diff --git a/.mailmap b/.mailmap
index 43031441b2d9..a58762564479 100644
--- a/.mailmap
+++ b/.mailmap
@@ -117,6 +117,7 @@ Changbin Du <changbin.du@intel.com> <changbin.du@gmail.com>
 Changbin Du <changbin.du@intel.com> <changbin.du@intel.com>
 Chao Yu <chao@kernel.org> <chao2.yu@samsung.com>
 Chao Yu <chao@kernel.org> <yuchao0@huawei.com>
+Chester Lin <chester62515@gmail.com> <clin@suse.com>
 Chris Chiu <chris.chiu@canonical.com> <chiu@endlessm.com>
 Chris Chiu <chris.chiu@canonical.com> <chiu@endlessos.org>
 Chris Lew <quic_clew@quicinc.com> <clew@codeaurora.org>
@@ -383,9 +384,10 @@ Matthias Fuchs <socketcan@esd.eu> <matthias.fuchs@esd.eu>
 Matthieu Baerts <matttbe@kernel.org> <matthieu.baerts@tessares.net>
 Matthieu CASTET <castet.matthieu@free.fr>
 Matti Vaittinen <mazziesaccount@gmail.com> <matti.vaittinen@fi.rohmeurope.com>
-Matt Ranostay <matt.ranostay@konsulko.com> <matt@ranostay.consulting>
-Matt Ranostay <mranostay@gmail.com> Matthew Ranostay <mranostay@embeddedalley.com>
-Matt Ranostay <mranostay@gmail.com> <matt.ranostay@intel.com>
+Matt Ranostay <matt@ranostay.sg> <matt.ranostay@konsulko.com>
+Matt Ranostay <matt@ranostay.sg> <matt@ranostay.consulting>
+Matt Ranostay <matt@ranostay.sg> Matthew Ranostay <mranostay@embeddedalley.com>
+Matt Ranostay <matt@ranostay.sg> <matt.ranostay@intel.com>
 Matt Redfearn <matt.redfearn@mips.com> <matt.redfearn@imgtec.com>
 Maulik Shah <quic_mkshah@quicinc.com> <mkshah@codeaurora.org>
 Mauro Carvalho Chehab <mchehab@kernel.org> <maurochehab@gmail.com>
@@ -469,6 +471,8 @@ Paul E. McKenney <paulmck@kernel.org> <paulmck@linux.vnet.ibm.com>
 Paul E. McKenney <paulmck@kernel.org> <paulmck@us.ibm.com>
 Paul Mackerras <paulus@ozlabs.org> <paulus@samba.org>
 Paul Mackerras <paulus@ozlabs.org> <paulus@au1.ibm.com>
+Paul Moore <paul@paul-moore.com> <paul.moore@hp.com>
+Paul Moore <paul@paul-moore.com> <pmoore@redhat.com>
 Pavankumar Kondeti <quic_pkondeti@quicinc.com> <pkondeti@codeaurora.org>
 Peter A Jonsson <pj@ludd.ltu.se>
 Peter Oruba <peter.oruba@amd.com>
@@ -533,6 +537,8 @@ Sebastian Reichel <sre@kernel.org> <sebastian.reichel@collabora.co.uk>
 Sebastian Reichel <sre@kernel.org> <sre@debian.org>
 Sedat Dilek <sedat.dilek@gmail.com> <sedat.dilek@credativ.de>
 Senthilkumar N L <quic_snlakshm@quicinc.com> <snlakshm@codeaurora.org>
+Serge Hallyn <sergeh@kernel.org> <serge.hallyn@canonical.com>
+Serge Hallyn <sergeh@kernel.org> <serue@us.ibm.com>
 Seth Forshee <sforshee@kernel.org> <seth.forshee@canonical.com>
 Shannon Nelson <shannon.nelson@amd.com> <snelson@pensando.io>
 Shannon Nelson <shannon.nelson@amd.com> <shannon.nelson@intel.com>
diff --git a/Documentation/ABI/testing/sysfs-bus-coresight-devices-tmc b/Documentation/ABI/testing/sysfs-bus-coresight-devices-tmc
index 6aa527296c71..96aafa66b4a5 100644
--- a/Documentation/ABI/testing/sysfs-bus-coresight-devices-tmc
+++ b/Documentation/ABI/testing/sysfs-bus-coresight-devices-tmc
@@ -91,3 +91,19 @@ Contact:	Mathieu Poirier <mathieu.poirier@linaro.org>
 Description:	(RW) Size of the trace buffer for TMC-ETR when used in SYSFS
 		mode. Writable only for TMC-ETR configurations. The value
 		should be aligned to the kernel pagesize.
+
+What:		/sys/bus/coresight/devices/<memory_map>.tmc/buf_modes_available
+Date:		August 2023
+KernelVersion:	6.7
+Contact:	Anshuman Khandual <anshuman.khandual@arm.com>
+Description:	(Read) Shows all supported Coresight TMC-ETR buffer modes available
+		for the users to configure explicitly. This file is avaialble only
+		for TMC ETR devices.
+
+What:		/sys/bus/coresight/devices/<memory_map>.tmc/buf_mode_preferred
+Date:		August 2023
+KernelVersion:	6.7
+Contact:	Anshuman Khandual <anshuman.khandual@arm.com>
+Description:	(RW) Current Coresight TMC-ETR buffer mode selected. But user could
+		only provide a mode which is supported for a given ETR device. This
+		file is available only for TMC ETR devices.
diff --git a/Documentation/ABI/testing/sysfs-bus-coresight-devices-tpdm b/Documentation/ABI/testing/sysfs-bus-coresight-devices-tpdm
index 4a58e649550d..4dd49b159543 100644
--- a/Documentation/ABI/testing/sysfs-bus-coresight-devices-tpdm
+++ b/Documentation/ABI/testing/sysfs-bus-coresight-devices-tpdm
@@ -11,3 +11,162 @@ Description:
 		Accepts only one of the 2 values -  1 or 2.
 		1 : Generate 64 bits data
 		2 : Generate 32 bits data
+
+What:		/sys/bus/coresight/devices/<tpdm-name>/reset_dataset
+Date:		March 2023
+KernelVersion	6.7
+Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Description:
+		(Write) Reset the dataset of the tpdm.
+
+		Accepts only one value -  1.
+		1 : Reset the dataset of the tpdm
+
+What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_trig_type
+Date:		March 2023
+KernelVersion	6.7
+Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Description:
+		(RW) Set/Get the trigger type of the DSB for tpdm.
+
+		Accepts only one of the 2 values -  0 or 1.
+		0 : Set the DSB trigger type to false
+		1 : Set the DSB trigger type to true
+
+What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_trig_ts
+Date:		March 2023
+KernelVersion	6.7
+Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Description:
+		(RW) Set/Get the trigger timestamp of the DSB for tpdm.
+
+		Accepts only one of the 2 values -  0 or 1.
+		0 : Set the DSB trigger type to false
+		1 : Set the DSB trigger type to true
+
+What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_mode
+Date:		March 2023
+KernelVersion	6.7
+Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Description:
+		(RW) Set/Get the programming mode of the DSB for tpdm.
+
+		Accepts the value needs to be greater than 0. What data
+		bits do is listed below.
+		Bit[0:1] : Test mode control bit for choosing the inputs.
+		Bit[3] : Set to 0 for low performance mode. Set to 1 for high
+		performance mode.
+		Bit[4:8] : Select byte lane for high performance mode.
+
+What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_edge/ctrl_idx
+Date:		March 2023
+KernelVersion	6.7
+Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Description:
+		(RW) Set/Get the index number of the edge detection for the DSB
+		subunit TPDM. Since there are at most 256 edge detections, this
+		value ranges from 0 to 255.
+
+What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_edge/ctrl_val
+Date:		March 2023
+KernelVersion	6.7
+Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Description:
+		Write a data to control the edge detection corresponding to
+		the index number. Before writing data to this sysfs file,
+		"ctrl_idx" should be written first to configure the index
+		number of the edge detection which needs to be controlled.
+
+		Accepts only one of the following values.
+		0 - Rising edge detection
+		1 - Falling edge detection
+		2 - Rising and falling edge detection (toggle detection)
+
+
+What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_edge/ctrl_mask
+Date:		March 2023
+KernelVersion	6.7
+Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Description:
+		Write a data to mask the edge detection corresponding to the index
+		number. Before writing data to this sysfs file, "ctrl_idx" should
+		be written first to configure the index number of the edge detection
+		which needs to be masked.
+
+		Accepts only one of the 2 values -  0 or 1.
+
+What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_edge/edcr[0:15]
+Date:		March 2023
+KernelVersion	6.7
+Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Description:
+		Read a set of the edge control value of the DSB in TPDM.
+
+What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_edge/edcmr[0:7]
+Date:		March 2023
+KernelVersion	6.7
+Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Description:
+		Read a set of the edge control mask of the DSB in TPDM.
+
+What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_trig_patt/xpr[0:7]
+Date:		March 2023
+KernelVersion	6.7
+Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Description:
+		(RW) Set/Get the value of the trigger pattern for the DSB
+		subunit TPDM.
+
+What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_trig_patt/xpmr[0:7]
+Date:		March 2023
+KernelVersion	6.7
+Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Description:
+		(RW) Set/Get the mask of the trigger pattern for the DSB
+		subunit TPDM.
+
+What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_patt/tpr[0:7]
+Date:		March 2023
+KernelVersion	6.7
+Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Description:
+		(RW) Set/Get the value of the pattern for the DSB subunit TPDM.
+
+What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_patt/tpmr[0:7]
+Date:		March 2023
+KernelVersion	6.7
+Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Description:
+		(RW) Set/Get the mask of the pattern for the DSB subunit TPDM.
+
+What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_patt/enable_ts
+Date:		March 2023
+KernelVersion	6.7
+Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Description:
+		(Write) Set the pattern timestamp of DSB tpdm. Read
+		the pattern timestamp of DSB tpdm.
+
+		Accepts only one of the 2 values -  0 or 1.
+		0 : Disable DSB pattern timestamp.
+		1 : Enable DSB pattern timestamp.
+
+What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_patt/set_type
+Date:		March 2023
+KernelVersion	6.7
+Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Description:
+		(Write) Set the pattern type of DSB tpdm. Read
+		the pattern type of DSB tpdm.
+
+		Accepts only one of the 2 values -  0 or 1.
+		0 : Set the DSB pattern type to value.
+		1 : Set the DSB pattern type to toggle.
+
+What:		/sys/bus/coresight/devices/<tpdm-name>/dsb_msr/msr[0:31]
+Date:		March 2023
+KernelVersion	6.7
+Contact:	Jinlong Mao (QUIC) <quic_jinlmao@quicinc.com>, Tao Zhang (QUIC) <quic_taozha@quicinc.com>
+Description:
+		(RW) Set/Get the MSR(mux select register) for the DSB subunit
+		TPDM.
diff --git a/Documentation/ABI/testing/sysfs-bus-event_source-devices-caps b/Documentation/ABI/testing/sysfs-bus-event_source-devices-caps
index 8757dcf41c08..a5f506f7d481 100644
--- a/Documentation/ABI/testing/sysfs-bus-event_source-devices-caps
+++ b/Documentation/ABI/testing/sysfs-bus-event_source-devices-caps
@@ -16,3 +16,9 @@ Description:
 		Example output in powerpc:
 		grep . /sys/bus/event_source/devices/cpu/caps/*
 		/sys/bus/event_source/devices/cpu/caps/pmu_name:POWER9
+
+		The "branch_counter_nr" in the supported platform exposes the
+		maximum number of counters which can be shown in the u64 counters
+		of PERF_SAMPLE_BRANCH_COUNTERS, while the "branch_counter_width"
+		exposes the width of each counter. Both of them can be used by
+		the perf tool to parse the logged counters in each branch.
diff --git a/Documentation/ABI/testing/sysfs-bus-iio b/Documentation/ABI/testing/sysfs-bus-iio
index 19cde14f3869..0eadc08c3a13 100644
--- a/Documentation/ABI/testing/sysfs-bus-iio
+++ b/Documentation/ABI/testing/sysfs-bus-iio
@@ -2254,3 +2254,21 @@ Description:
 		If a label is defined for this event add that to the event
 		specific attributes. This is useful for userspace to be able to
 		better identify an individual event.
+
+What:		/sys/.../events/in_accel_gesture_tap_wait_timeout
+KernelVersion:	6.7
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Enable tap gesture confirmation with timeout.
+
+What:		/sys/.../events/in_accel_gesture_tap_wait_dur
+KernelVersion:	6.7
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Timeout value in seconds for tap gesture confirmation.
+
+What:		/sys/.../events/in_accel_gesture_tap_wait_dur_available
+KernelVersion:	6.7
+Contact:	linux-iio@vger.kernel.org
+Description:
+		List of available timeout value for tap gesture confirmation.
diff --git a/Documentation/ABI/testing/sysfs-bus-optee-devices b/Documentation/ABI/testing/sysfs-bus-optee-devices
index 0f58701367b6..af31e5a22d89 100644
--- a/Documentation/ABI/testing/sysfs-bus-optee-devices
+++ b/Documentation/ABI/testing/sysfs-bus-optee-devices
@@ -6,3 +6,12 @@ Description:
 		OP-TEE bus provides reference to registered drivers under this directory. The <uuid>
 		matches Trusted Application (TA) driver and corresponding TA in secure OS. Drivers
 		are free to create needed API under optee-ta-<uuid> directory.
+
+What:		/sys/bus/tee/devices/optee-ta-<uuid>/need_supplicant
+Date:		November 2023
+KernelVersion:	6.7
+Contact:	op-tee@lists.trustedfirmware.org
+Description:
+		Allows to distinguish whether an OP-TEE based TA/device requires user-space
+		tee-supplicant to function properly or not. This attribute will be present for
+		devices which depend on tee-supplicant to be running.
diff --git a/Documentation/ABI/testing/sysfs-class-hwmon b/Documentation/ABI/testing/sysfs-class-hwmon
index 638f4c6d4ec7..3dac923c9b0e 100644
--- a/Documentation/ABI/testing/sysfs-class-hwmon
+++ b/Documentation/ABI/testing/sysfs-class-hwmon
@@ -381,6 +381,15 @@ Description:
 
 		RW
 
+What:		/sys/class/hwmon/hwmonX/tempY_max_alarm
+Description:
+		Maximum temperature alarm flag.
+
+		- 0: OK
+		- 1: temperature has reached tempY_max
+
+		RO
+
 What:		/sys/class/hwmon/hwmonX/tempY_min
 Description:
 		Temperature min value.
@@ -389,6 +398,15 @@ Description:
 
 		RW
 
+What:		/sys/class/hwmon/hwmonX/tempY_min_alarm
+Description:
+		Minimum temperature alarm flag.
+
+		- 0: OK
+		- 1: temperature has reached tempY_min
+
+		RO
+
 What:		/sys/class/hwmon/hwmonX/tempY_max_hyst
 Description:
 		Temperature hysteresis value for max limit.
@@ -434,12 +452,7 @@ Description:
 		- 0: OK
 		- 1: temperature has reached tempY_crit
 
-		RW
-
-		Contrary to regular alarm flags which clear themselves
-		automatically when read, this one sticks until cleared by
-		the user. This is done by writing 0 to the file. Writing
-		other values is unsupported.
+		RO
 
 What:		/sys/class/hwmon/hwmonX/tempY_crit_hyst
 Description:
@@ -462,6 +475,15 @@ Description:
 
 		RW
 
+What:		/sys/class/hwmon/hwmonX/tempY_emergency_alarm
+Description:
+		Emergency high temperature alarm flag.
+
+		- 0: OK
+		- 1: temperature has reached tempY_emergency
+
+		RO
+
 What:		/sys/class/hwmon/hwmonX/tempY_emergency_hyst
 Description:
 		Temperature hysteresis value for emergency limit.
@@ -887,15 +909,15 @@ Description:
 
 		RW
 
-What:		/sys/class/hwmon/hwmonX/humidityY_input
+What:		/sys/class/hwmon/hwmonX/humidityY_alarm
 Description:
-		Humidity
+		Humidity limit detection
 
-		Unit: milli-percent (per cent mille, pcm)
+		- 0: OK
+		- 1: Humidity limit has been reached
 
 		RO
 
-
 What:		/sys/class/hwmon/hwmonX/humidityY_enable
 Description:
 		Enable or disable the sensors
@@ -908,6 +930,74 @@ Description:
 
 		RW
 
+What:		/sys/class/hwmon/hwmonX/humidityY_fault
+Description:
+		Reports a humidity sensor failure.
+
+		- 1: Failed
+		- 0: Ok
+
+		RO
+
+What:		/sys/class/hwmon/hwmonX/humidityY_input
+Description:
+		Humidity
+
+		Unit: milli-percent (per cent mille, pcm)
+
+		RO
+
+What:		/sys/class/hwmon/hwmonX/humidityY_label
+Description:
+		Suggested humidity channel label.
+
+		Text string
+
+		Should only be created if the driver has hints about what
+		this humidity channel is being used for, and user-space
+		doesn't. In all other cases, the label is provided by
+		user-space.
+
+		RO
+
+What:		/sys/class/hwmon/hwmonX/humidityY_max
+Description:
+		Humidity max value.
+
+		Unit: milli-percent (per cent mille, pcm)
+
+		RW
+
+What:		/sys/class/hwmon/hwmonX/humidityY_max_hyst
+Description:
+		Humidity hysteresis value for max limit.
+
+		Unit: milli-percent (per cent mille, pcm)
+
+		Must be reported as an absolute humidity, NOT a delta
+		from the max value.
+
+		RW
+
+What:		/sys/class/hwmon/hwmonX/humidityY_min
+Description:
+		Humidity min value.
+
+		Unit: milli-percent (per cent mille, pcm)
+
+		RW
+
+What:		/sys/class/hwmon/hwmonX/humidityY_min_hyst
+Description:
+		Humidity hysteresis value for min limit.
+
+		Unit: milli-percent (per cent mille, pcm)
+
+		Must be reported as an absolute humidity, NOT a delta
+		from the min value.
+
+		RW
+
 What:		/sys/class/hwmon/hwmonX/humidityY_rated_min
 Description:
 		Minimum rated humidity.
diff --git a/Documentation/ABI/testing/sysfs-driver-habanalabs b/Documentation/ABI/testing/sysfs-driver-habanalabs
index c63ca1ad500d..89fe3b09d4ad 100644
--- a/Documentation/ABI/testing/sysfs-driver-habanalabs
+++ b/Documentation/ABI/testing/sysfs-driver-habanalabs
@@ -149,6 +149,12 @@ Contact:        ogabbay@kernel.org
 Description:    Displays the current clock frequency, in Hz, of the MME compute
                 engine. This property is valid only for the Goya ASIC family
 
+What:           /sys/class/accel/accel<n>/device/module_id
+Date:           Nov 2023
+KernelVersion:  not yet upstreamed
+Contact:        ogabbay@kernel.org
+Description:    Displays the device's module id
+
 What:           /sys/class/accel/accel<n>/device/pci_addr
 Date:           Jan 2019
 KernelVersion:  5.1
diff --git a/Documentation/ABI/testing/sysfs-driver-ufs b/Documentation/ABI/testing/sysfs-driver-ufs
index 0c7efaf62de0..b73067bb5ea2 100644
--- a/Documentation/ABI/testing/sysfs-driver-ufs
+++ b/Documentation/ABI/testing/sysfs-driver-ufs
@@ -1223,6 +1223,55 @@ Description:	This file shows the total latency (in micro seconds) of write
 
 		The file is read only.
 
+What:		/sys/bus/platform/drivers/ufshcd/*/power_info/lane
+What:		/sys/bus/platform/devices/*.ufs/power_info/lane
+Date:		September 2023
+Contact:	Can Guo <quic_cang@quicinc.com>
+Description:	This file shows how many lanes are enabled on the UFS link,
+		i.e., an output 2 means UFS link is operating with 2 lanes.
+
+		The file is read only.
+
+What:		/sys/bus/platform/drivers/ufshcd/*/power_info/mode
+What:		/sys/bus/platform/devices/*.ufs/power_info/mode
+Date:		September 2023
+Contact:	Can Guo <quic_cang@quicinc.com>
+Description:	This file shows the PA power mode of UFS.
+
+		The file is read only.
+
+What:		/sys/bus/platform/drivers/ufshcd/*/power_info/rate
+What:		/sys/bus/platform/devices/*.ufs/power_info/rate
+Date:		September 2023
+Contact:	Can Guo <quic_cang@quicinc.com>
+Description:	This file shows the speed rate of UFS link.
+
+		The file is read only.
+
+What:		/sys/bus/platform/drivers/ufshcd/*/power_info/gear
+What:		/sys/bus/platform/devices/*.ufs/power_info/gear
+Date:		September 2023
+Contact:	Can Guo <quic_cang@quicinc.com>
+Description:	This file shows the gear of UFS link.
+
+		The file is read only.
+
+What:		/sys/bus/platform/drivers/ufshcd/*/power_info/dev_pm
+What:		/sys/bus/platform/devices/*.ufs/power_info/dev_pm
+Date:		September 2023
+Contact:	Can Guo <quic_cang@quicinc.com>
+Description:	This file shows the UFS device power mode.
+
+		The file is read only.
+
+What:		/sys/bus/platform/drivers/ufshcd/*/power_info/link_state
+What:		/sys/bus/platform/devices/*.ufs/power_info/link_state
+Date:		September 2023
+Contact:	Can Guo <quic_cang@quicinc.com>
+Description:	This file shows the state of UFS link.
+
+		The file is read only.
+
 What:		/sys/bus/platform/drivers/ufshcd/*/device_descriptor/wb_presv_us_en
 What:		/sys/bus/platform/devices/*.ufs/device_descriptor/wb_presv_us_en
 Date:		June 2020
diff --git a/Documentation/accel/qaic/aic100.rst b/Documentation/accel/qaic/aic100.rst
index c80d0f1307db..590dae77ea12 100644
--- a/Documentation/accel/qaic/aic100.rst
+++ b/Documentation/accel/qaic/aic100.rst
@@ -36,8 +36,9 @@ AIC100 DID (0xa100).
 
 AIC100 does not implement FLR (function level reset).
 
-AIC100 implements MSI but does not implement MSI-X. AIC100 requires 17 MSIs to
-operate (1 for MHI, 16 for the DMA Bridge).
+AIC100 implements MSI but does not implement MSI-X. AIC100 prefers 17 MSIs to
+operate (1 for MHI, 16 for the DMA Bridge). Falling back to 1 MSI is possible in
+scenarios where reserving 32 MSIs isn't feasible.
 
 As a PCIe device, AIC100 utilizes BARs to provide host interfaces to the device
 hardware. AIC100 provides 3, 64-bit BARs.
@@ -220,10 +221,14 @@ of the defined channels, and their uses.
 +----------------+---------+----------+----------------------------------------+
 | QAIC_DEBUG     | 18 & 19 | AMSS     | Not used.                              |
 +----------------+---------+----------+----------------------------------------+
-| QAIC_TIMESYNC  | 20 & 21 | SBL/AMSS | Used to synchronize timestamps in the  |
+| QAIC_TIMESYNC  | 20 & 21 | SBL      | Used to synchronize timestamps in the  |
 |                |         |          | device side logs with the host time    |
 |                |         |          | source.                                |
 +----------------+---------+----------+----------------------------------------+
+| QAIC_TIMESYNC  | 22 & 23 | AMSS     | Used to periodically synchronize       |
+| _PERIODIC      |         |          | timestamps in the device side logs with|
+|                |         |          | the host time source.                  |
++----------------+---------+----------+----------------------------------------+
 
 DMA Bridge
 ==========
diff --git a/Documentation/accel/qaic/qaic.rst b/Documentation/accel/qaic/qaic.rst
index c88502383136..f81020736ebf 100644
--- a/Documentation/accel/qaic/qaic.rst
+++ b/Documentation/accel/qaic/qaic.rst
@@ -10,6 +10,9 @@ accelerator products.
 Interrupts
 ==========
 
+IRQ Storm Mitigation
+--------------------
+
 While the AIC100 DMA Bridge hardware implements an IRQ storm mitigation
 mechanism, it is still possible for an IRQ storm to occur. A storm can happen
 if the workload is particularly quick, and the host is responsive. If the host
@@ -35,6 +38,26 @@ generates 100k IRQs per second (per /proc/interrupts) is reduced to roughly 64
 IRQs over 5 minutes while keeping the host system stable, and having the same
 workload throughput performance (within run to run noise variation).
 
+Single MSI Mode
+---------------
+
+MultiMSI is not well supported on all systems; virtualized ones even less so
+(circa 2023). Between hypervisors masking the PCIe MSI capability structure to
+large memory requirements for vIOMMUs (required for supporting MultiMSI), it is
+useful to be able to fall back to a single MSI when needed.
+
+To support this fallback, we allow the case where only one MSI is able to be
+allocated, and share that one MSI between MHI and the DBCs. The device detects
+when only one MSI has been configured and directs the interrupts for the DBCs
+to the interrupt normally used for MHI. Unfortunately this means that the
+interrupt handlers for every DBC and MHI wake up for every interrupt that
+arrives; however, the DBC threaded irq handlers only are started when work to be
+done is detected (MHI will always start its threaded handler).
+
+If the DBC is configured to force MSI interrupts, this can circumvent the
+software IRQ storm mitigation mentioned above. Since the MSI is shared it is
+never disabled, allowing each new entry to the FIFO to trigger a new interrupt.
+
 
 Neural Network Control (NNC) Protocol
 =====================================
@@ -178,3 +201,8 @@ overrides this for that call. Default is 5000 (5 seconds).
 
 Sets the polling interval in microseconds (us) when datapath polling is active.
 Takes effect at the next polling interval. Default is 100 (100 us).
+
+**timesync_delay_ms (unsigned int)**
+
+Sets the time interval in milliseconds (ms) between two consecutive timesync
+operations. Default is 1000 (1000 ms).
diff --git a/Documentation/admin-guide/blockdev/zram.rst b/Documentation/admin-guide/blockdev/zram.rst
index e4551579cb12..ee2b0030d416 100644
--- a/Documentation/admin-guide/blockdev/zram.rst
+++ b/Documentation/admin-guide/blockdev/zram.rst
@@ -328,7 +328,7 @@ as idle::
 From now on, any pages on zram are idle pages. The idle mark
 will be removed until someone requests access of the block.
 IOW, unless there is access request, those pages are still idle pages.
-Additionally, when CONFIG_ZRAM_MEMORY_TRACKING is enabled pages can be
+Additionally, when CONFIG_ZRAM_TRACK_ENTRY_ACTIME is enabled pages can be
 marked as idle based on how long (in seconds) it's been since they were
 last accessed::
 
diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst
index 3f85254f3cef..cf5651a11df8 100644
--- a/Documentation/admin-guide/cgroup-v2.rst
+++ b/Documentation/admin-guide/cgroup-v2.rst
@@ -2358,11 +2358,11 @@ Cpuset Interface Files
 	partition or scheduling domain.  The set of exclusive CPUs is
 	determined by the value of its "cpuset.cpus.exclusive.effective".
 
-	When set to "isolated", the CPUs in that partition will
-	be in an isolated state without any load balancing from the
-	scheduler.  Tasks placed in such a partition with multiple
-	CPUs should be carefully distributed and bound to each of the
-	individual CPUs for optimal performance.
+	When set to "isolated", the CPUs in that partition will be in
+	an isolated state without any load balancing from the scheduler
+	and excluded from the unbound workqueues.  Tasks placed in such
+	a partition with multiple CPUs should be carefully distributed
+	and bound to each of the individual CPUs for optimal performance.
 
 	A partition root ("root" or "isolated") can be in one of the
 	two possible states - valid or invalid.  An invalid partition
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 65731b060e3f..2f3b732a0914 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -5302,6 +5302,12 @@
 			Dump ftrace buffer after reporting RCU CPU
 			stall warning.
 
+	rcupdate.rcu_cpu_stall_notifiers= [KNL]
+			Provide RCU CPU stall notifiers, but see the
+			warnings in the RCU_CPU_STALL_NOTIFIER Kconfig
+			option's help text.  TL;DR:  You almost certainly
+			do not want rcupdate.rcu_cpu_stall_notifiers.
+
 	rcupdate.rcu_cpu_stall_suppress= [KNL]
 			Suppress RCU CPU stall warning messages.
 
@@ -5544,6 +5550,13 @@
 			print every Nth verbose statement, where N is the value
 			specified.
 
+	regulator_ignore_unused
+			[REGULATOR]
+			Prevents regulator framework from disabling regulators
+			that are unused, due no driver claiming them. This may
+			be useful for debug and development, but should not be
+			needed on a platform with proper driver support.
+
 	relax_domain_level=
 			[KNL, SMP] Set scheduler's default relax_domain_level.
 			See Documentation/admin-guide/cgroup-v1/cpusets.rst.
diff --git a/Documentation/admin-guide/mm/pagemap.rst b/Documentation/admin-guide/mm/pagemap.rst
index fe17cf210426..f5f065c67615 100644
--- a/Documentation/admin-guide/mm/pagemap.rst
+++ b/Documentation/admin-guide/mm/pagemap.rst
@@ -253,6 +253,7 @@ Following flags about pages are currently supported:
 - ``PAGE_IS_SWAPPED`` - Page is in swapped
 - ``PAGE_IS_PFNZERO`` - Page has zero PFN
 - ``PAGE_IS_HUGE`` - Page is THP or Hugetlb backed
+- ``PAGE_IS_SOFT_DIRTY`` - Page is soft-dirty
 
 The ``struct pm_scan_arg`` is used as the argument of the IOCTL.
 
diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst
index 6584a1f9bfe3..bc578663619d 100644
--- a/Documentation/admin-guide/sysctl/kernel.rst
+++ b/Documentation/admin-guide/sysctl/kernel.rst
@@ -594,6 +594,9 @@ default (``MSGMNB``).
 ``msgmni`` is the maximum number of IPC queues. 32000 by default
 (``MSGMNI``).
 
+All of these parameters are set per ipc namespace. The maximum number of bytes
+in POSIX message queues is limited by ``RLIMIT_MSGQUEUE``. This limit is
+respected hierarchically in the each user namespace.
 
 msg_next_id, sem_next_id, and shm_next_id (System V IPC)
 ========================================================
@@ -1274,15 +1277,20 @@ are doing anyway :)
 shmall
 ======
 
-This parameter sets the total amount of shared memory pages that
-can be used system wide. Hence, ``shmall`` should always be at least
-``ceil(shmmax/PAGE_SIZE)``.
+This parameter sets the total amount of shared memory pages that can be used
+inside ipc namespace. The shared memory pages counting occurs for each ipc
+namespace separately and is not inherited. Hence, ``shmall`` should always be at
+least ``ceil(shmmax/PAGE_SIZE)``.
 
 If you are not sure what the default ``PAGE_SIZE`` is on your Linux
 system, you can run the following command::
 
 	# getconf PAGE_SIZE
 
+To reduce or disable the ability to allocate shared memory, you must create a
+new ipc namespace, set this parameter to the required value and prohibit the
+creation of a new ipc namespace in the current user namespace or cgroups can
+be used.
 
 shmmax
 ======
diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst
index f47f63bcf67c..bfdf236e2af3 100644
--- a/Documentation/arch/arm64/silicon-errata.rst
+++ b/Documentation/arch/arm64/silicon-errata.rst
@@ -117,6 +117,10 @@ stable kernels.
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Cortex-A76      | #1463225        | ARM64_ERRATUM_1463225       |
 +----------------+-----------------+-----------------+-----------------------------+
+| ARM            | Cortex-A76      | #1490853        | N/A                         |
++----------------+-----------------+-----------------+-----------------------------+
+| ARM            | Cortex-A77      | #1491015        | N/A                         |
++----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Cortex-A77      | #1508412        | ARM64_ERRATUM_1508412       |
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Cortex-A710     | #2119858        | ARM64_ERRATUM_2119858       |
@@ -127,6 +131,8 @@ stable kernels.
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Cortex-A715     | #2645198        | ARM64_ERRATUM_2645198       |
 +----------------+-----------------+-----------------+-----------------------------+
+| ARM            | Cortex-X1       | #1502854        | N/A                         |
++----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Cortex-X2       | #2119858        | ARM64_ERRATUM_2119858       |
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Cortex-X2       | #2224489        | ARM64_ERRATUM_2224489       |
@@ -135,6 +141,8 @@ stable kernels.
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Neoverse-N1     | #1349291        | N/A                         |
 +----------------+-----------------+-----------------+-----------------------------+
+| ARM            | Neoverse-N1     | #1490853        | N/A                         |
++----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Neoverse-N1     | #1542419        | ARM64_ERRATUM_1542419       |
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Neoverse-N2     | #2139208        | ARM64_ERRATUM_2139208       |
@@ -143,6 +151,8 @@ stable kernels.
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Neoverse-N2     | #2253138        | ARM64_ERRATUM_2253138       |
 +----------------+-----------------+-----------------+-----------------------------+
+| ARM            | Neoverse-V1     | #1619801        | N/A                         |
++----------------+-----------------+-----------------+-----------------------------+
 | ARM            | MMU-500         | #841119,826419  | N/A                         |
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | MMU-600         | #1076982,1209401| N/A                         |
diff --git a/Documentation/arch/loongarch/introduction.rst b/Documentation/arch/loongarch/introduction.rst
index 8c568cfc2107..5e6db78abeaf 100644
--- a/Documentation/arch/loongarch/introduction.rst
+++ b/Documentation/arch/loongarch/introduction.rst
@@ -375,9 +375,9 @@ Developer web site of Loongson and LoongArch (Software and Documentation):
 
 Documentation of LoongArch ISA:
 
-  https://github.com/loongson/LoongArch-Documentation/releases/latest/download/LoongArch-Vol1-v1.02-CN.pdf (in Chinese)
+  https://github.com/loongson/LoongArch-Documentation/releases/latest/download/LoongArch-Vol1-v1.10-CN.pdf (in Chinese)
 
-  https://github.com/loongson/LoongArch-Documentation/releases/latest/download/LoongArch-Vol1-v1.02-EN.pdf (in English)
+  https://github.com/loongson/LoongArch-Documentation/releases/latest/download/LoongArch-Vol1-v1.10-EN.pdf (in English)
 
 Documentation of LoongArch ELF psABI:
 
diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst
index 723408e399ab..7985c6615f3c 100644
--- a/Documentation/bpf/kfuncs.rst
+++ b/Documentation/bpf/kfuncs.rst
@@ -135,6 +135,30 @@ Either way, the returned buffer is either NULL, or of size buffer_szk. Without t
 annotation, the verifier will reject the program if a null pointer is passed in with
 a nonzero size.
 
+2.2.5 __str Annotation
+----------------------------
+This annotation is used to indicate that the argument is a constant string.
+
+An example is given below::
+
+        __bpf_kfunc bpf_get_file_xattr(..., const char *name__str, ...)
+        {
+        ...
+        }
+
+In this case, ``bpf_get_file_xattr()`` can be called as::
+
+        bpf_get_file_xattr(..., "xattr_name", ...);
+
+Or::
+
+        const char name[] = "xattr_name";  /* This need to be global */
+        int BPF_PROG(...)
+        {
+                ...
+                bpf_get_file_xattr(..., name, ...);
+                ...
+        }
 
 .. _BPF_kfunc_nodef:
 
diff --git a/Documentation/core-api/dma-api-howto.rst b/Documentation/core-api/dma-api-howto.rst
index 72f6cdb6be1c..e8a55f9d61db 100644
--- a/Documentation/core-api/dma-api-howto.rst
+++ b/Documentation/core-api/dma-api-howto.rst
@@ -8,7 +8,7 @@ Dynamic DMA mapping Guide
 
 This is a guide to device driver writers on how to use the DMA API
 with example pseudo-code.  For a concise description of the API, see
-DMA-API.txt.
+Documentation/core-api/dma-api.rst.
 
 CPU and DMA addresses
 =====================
diff --git a/Documentation/core-api/dma-api.rst b/Documentation/core-api/dma-api.rst
index 829f20a193ca..8e3cce3d0a23 100644
--- a/Documentation/core-api/dma-api.rst
+++ b/Documentation/core-api/dma-api.rst
@@ -448,7 +448,7 @@ DMA address entries returned.
 
 Synchronise a single contiguous or scatter/gather mapping for the CPU
 and device. With the sync_sg API, all the parameters must be the same
-as those passed into the single mapping API. With the sync_single API,
+as those passed into the sg mapping API. With the sync_single API,
 you can use dma_handle and size parameters that aren't identical to
 those passed into the single mapping API to do a partial sync.
 
diff --git a/Documentation/core-api/maple_tree.rst b/Documentation/core-api/maple_tree.rst
index 96f3d5f076b5..ccdd1615cf97 100644
--- a/Documentation/core-api/maple_tree.rst
+++ b/Documentation/core-api/maple_tree.rst
@@ -81,6 +81,9 @@ section.
 Sometimes it is necessary to ensure the next call to store to a maple tree does
 not allocate memory, please see :ref:`maple-tree-advanced-api` for this use case.
 
+You can use mtree_dup() to duplicate an entire maple tree. It is a more
+efficient way than inserting all elements one by one into a new tree.
+
 Finally, you can remove all entries from a maple tree by calling
 mtree_destroy().  If the maple tree entries are pointers, you may wish to free
 the entries first.
@@ -112,6 +115,7 @@ Takes ma_lock internally:
  * mtree_insert()
  * mtree_insert_range()
  * mtree_erase()
+ * mtree_dup()
  * mtree_destroy()
  * mt_set_in_rcu()
  * mt_clear_in_rcu()
diff --git a/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml b/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml
index 749ee54a3ff8..6f7543463d89 100644
--- a/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml
+++ b/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml
@@ -35,7 +35,9 @@ properties:
               - ampere,mtjade-bmc
               - aspeed,ast2500-evb
               - asrock,e3c246d4i-bmc
+              - asrock,e3c256d4i-bmc
               - asrock,romed8hm3-bmc
+              - asrock,spc621d8hm3-bmc
               - bytedance,g220a-bmc
               - facebook,cmm-bmc
               - facebook,minipack-bmc
diff --git a/Documentation/devicetree/bindings/arm/qcom,coresight-tpdm.yaml b/Documentation/devicetree/bindings/arm/qcom,coresight-tpdm.yaml
index 3bad47b7b02b..61ddc3b5b247 100644
--- a/Documentation/devicetree/bindings/arm/qcom,coresight-tpdm.yaml
+++ b/Documentation/devicetree/bindings/arm/qcom,coresight-tpdm.yaml
@@ -44,6 +44,23 @@ properties:
     minItems: 1
     maxItems: 2
 
+  qcom,dsb-element-size:
+    description:
+      Specifies the DSB(Discrete Single Bit) element size supported by
+      the monitor. The associated aggregator will read this size before it
+      is enabled. DSB element size currently only supports 32-bit and 64-bit.
+    $ref: /schemas/types.yaml#/definitions/uint8
+    enum: [32, 64]
+
+  qcom,dsb-msrs-num:
+    description:
+      Specifies the number of DSB(Discrete Single Bit) MSR(mux select register)
+      registers supported by the monitor. If this property is not configured
+      or set to 0, it means this DSB TPDM doesn't support MSR.
+    $ref: /schemas/types.yaml#/definitions/uint32
+    minimum: 0
+    maximum: 32
+
   clocks:
     maxItems: 1
 
@@ -77,6 +94,9 @@ examples:
       compatible = "qcom,coresight-tpdm", "arm,primecell";
       reg = <0x0684c000 0x1000>;
 
+      qcom,dsb-element-size = /bits/ 8 <32>;
+      qcom,dsb-msrs-num = <16>;
+
       clocks = <&aoss_qmp>;
       clock-names = "apb_pclk";
 
diff --git a/Documentation/devicetree/bindings/arm/qcom.yaml b/Documentation/devicetree/bindings/arm/qcom.yaml
index 7f80f48a0954..88b84035e7b1 100644
--- a/Documentation/devicetree/bindings/arm/qcom.yaml
+++ b/Documentation/devicetree/bindings/arm/qcom.yaml
@@ -186,9 +186,19 @@ properties:
 
       - items:
           - enum:
+              - microsoft,dempsey
+              - microsoft,makepeace
+              - microsoft,moneypenny
               - samsung,s3ve3g
           - const: qcom,msm8226
 
+      - items:
+          - enum:
+              - microsoft,superman-lte
+              - microsoft,tesla
+          - const: qcom,msm8926
+          - const: qcom,msm8226
+
       - items:
           - enum:
               - longcheer,l9100
diff --git a/Documentation/devicetree/bindings/arm/rockchip.yaml b/Documentation/devicetree/bindings/arm/rockchip.yaml
index 5f7c6c4aad8f..9f6155e4deed 100644
--- a/Documentation/devicetree/bindings/arm/rockchip.yaml
+++ b/Documentation/devicetree/bindings/arm/rockchip.yaml
@@ -237,6 +237,11 @@ properties:
           - const: geekbuying,geekbox
           - const: rockchip,rk3368
 
+      - description: Geniatech XPI-3128
+        items:
+          - const: geniatech,xpi-3128
+          - const: rockchip,rk3128
+
       - description: Google Bob (Asus Chromebook Flip C101PA)
         items:
           - const: google,bob-rev13
@@ -674,9 +679,11 @@ properties:
           - const: pine64,soquartz
           - const: rockchip,rk3566
 
-      - description: Powkiddy RGB30
+      - description: Powkiddy RK3566 Handheld Gaming Console
         items:
-          - const: powkiddy,rgb30
+          - enum:
+              - powkiddy,rgb30
+              - powkiddy,rk2023
           - const: rockchip,rk3566
 
       - description: Radxa Compute Module 3(CM3)
diff --git a/Documentation/devicetree/bindings/arm/samsung/samsung-boards.yaml b/Documentation/devicetree/bindings/arm/samsung/samsung-boards.yaml
index e3ffd8159ab6..01dcbd8aa703 100644
--- a/Documentation/devicetree/bindings/arm/samsung/samsung-boards.yaml
+++ b/Documentation/devicetree/bindings/arm/samsung/samsung-boards.yaml
@@ -230,6 +230,12 @@ properties:
               - samsung,exynosautov9-sadk   # Samsung Exynos Auto v9 SADK
           - const: samsung,exynosautov9
 
+      - description: Exynos Auto v920 based boards
+        items:
+          - enum:
+              - samsung,exynosautov920-sadk   # Samsung Exynos Auto v920 SADK
+          - const: samsung,exynosautov920
+
 required:
   - compatible
 
diff --git a/Documentation/devicetree/bindings/arm/sunxi.yaml b/Documentation/devicetree/bindings/arm/sunxi.yaml
index 11c5ce941dd7..6f481e8bb4bf 100644
--- a/Documentation/devicetree/bindings/arm/sunxi.yaml
+++ b/Documentation/devicetree/bindings/arm/sunxi.yaml
@@ -1013,6 +1013,11 @@ properties:
           - const: xunlong,orangepi-zero2
           - const: allwinner,sun50i-h616
 
+      - description: Xunlong OrangePi Zero 2W
+        items:
+          - const: xunlong,orangepi-zero2w
+          - const: allwinner,sun50i-h618
+
       - description: Xunlong OrangePi Zero 3
         items:
           - const: xunlong,orangepi-zero3
diff --git a/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml b/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml
index ca4f7d1cefaa..09e43157cc71 100644
--- a/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml
+++ b/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml
@@ -16,6 +16,7 @@ properties:
           - qcom,sa8775p-inline-crypto-engine
           - qcom,sm8450-inline-crypto-engine
           - qcom,sm8550-inline-crypto-engine
+          - qcom,sm8650-inline-crypto-engine
       - const: qcom,inline-crypto-engine
 
   reg:
diff --git a/Documentation/devicetree/bindings/crypto/qcom,prng.yaml b/Documentation/devicetree/bindings/crypto/qcom,prng.yaml
index 13070db0f70c..89c88004b41b 100644
--- a/Documentation/devicetree/bindings/crypto/qcom,prng.yaml
+++ b/Documentation/devicetree/bindings/crypto/qcom,prng.yaml
@@ -21,6 +21,7 @@ properties:
               - qcom,sc7280-trng
               - qcom,sm8450-trng
               - qcom,sm8550-trng
+              - qcom,sm8650-trng
           - const: qcom,trng
 
   reg:
diff --git a/Documentation/devicetree/bindings/crypto/qcom-qce.yaml b/Documentation/devicetree/bindings/crypto/qcom-qce.yaml
index 8e665d910e6e..eeb8a956d7cb 100644
--- a/Documentation/devicetree/bindings/crypto/qcom-qce.yaml
+++ b/Documentation/devicetree/bindings/crypto/qcom-qce.yaml
@@ -48,6 +48,7 @@ properties:
               - qcom,sm8350-qce
               - qcom,sm8450-qce
               - qcom,sm8550-qce
+              - qcom,sm8650-qce
           - const: qcom,sm8150-qce
           - const: qcom,qce
 
diff --git a/Documentation/devicetree/bindings/display/panel/newvision,nv3051d.yaml b/Documentation/devicetree/bindings/display/panel/newvision,nv3051d.yaml
index cce775a87f87..7a634fbc465e 100644
--- a/Documentation/devicetree/bindings/display/panel/newvision,nv3051d.yaml
+++ b/Documentation/devicetree/bindings/display/panel/newvision,nv3051d.yaml
@@ -21,7 +21,7 @@ properties:
       - enum:
           - anbernic,rg351v-panel
           - anbernic,rg353p-panel
-          - anbernic,rg353v-panel
+          - powkiddy,rk2023-panel
       - const: newvision,nv3051d
 
   reg: true
diff --git a/Documentation/devicetree/bindings/gpu/arm,mali-midgard.yaml b/Documentation/devicetree/bindings/gpu/arm,mali-midgard.yaml
index ca02baba5526..0801da33a385 100644
--- a/Documentation/devicetree/bindings/gpu/arm,mali-midgard.yaml
+++ b/Documentation/devicetree/bindings/gpu/arm,mali-midgard.yaml
@@ -40,6 +40,11 @@ properties:
               - rockchip,rk3288-mali
               - samsung,exynos5433-mali
           - const: arm,mali-t760
+      - items:
+          - enum:
+              - samsung,exynos7-mali
+          - const: samsung,exynos5433-mali
+          - const: arm,mali-t760
       - items:
           - enum:
               - rockchip,rk3399-mali
diff --git a/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.yaml b/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.yaml
index dae55b8a267b..dc078ceeca9a 100644
--- a/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.yaml
+++ b/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.yaml
@@ -17,6 +17,7 @@ properties:
   compatible:
     enum:
       - brcm,2711-v3d
+      - brcm,2712-v3d
       - brcm,7268-v3d
       - brcm,7278-v3d
 
diff --git a/Documentation/devicetree/bindings/hwinfo/samsung,exynos-chipid.yaml b/Documentation/devicetree/bindings/hwinfo/samsung,exynos-chipid.yaml
index 95cbdcb56efe..780ccb5ee9b4 100644
--- a/Documentation/devicetree/bindings/hwinfo/samsung,exynos-chipid.yaml
+++ b/Documentation/devicetree/bindings/hwinfo/samsung,exynos-chipid.yaml
@@ -11,9 +11,21 @@ maintainers:
 
 properties:
   compatible:
-    enum:
-      - samsung,exynos4210-chipid
-      - samsung,exynos850-chipid
+    oneOf:
+      - enum:
+          - samsung,exynos4210-chipid
+          - samsung,exynos850-chipid
+      - items:
+          - enum:
+              - samsung,exynos5433-chipid
+              - samsung,exynos7-chipid
+          - const: samsung,exynos4210-chipid
+      - items:
+          - enum:
+              - samsung,exynos7885-chipid
+              - samsung,exynosautov9-chipid
+              - samsung,exynosautov920-chipid
+          - const: samsung,exynos850-chipid
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/i2c/i2c-exynos5.yaml b/Documentation/devicetree/bindings/i2c/i2c-exynos5.yaml
index 3e52a0db6c41..c1f5d2cb7709 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-exynos5.yaml
+++ b/Documentation/devicetree/bindings/i2c/i2c-exynos5.yaml
@@ -25,7 +25,15 @@ properties:
           - samsung,exynos5250-hsi2c    # Exynos5250 and Exynos5420
           - samsung,exynos5260-hsi2c    # Exynos5260
           - samsung,exynos7-hsi2c       # Exynos7
-          - samsung,exynosautov9-hsi2c  # ExynosAutoV9 and Exynos850
+          - samsung,exynosautov9-hsi2c
+      - items:
+          - enum:
+              - samsung,exynos5433-hsi2c
+          - const: samsung,exynos7-hsi2c
+      - items:
+          - enum:
+              - samsung,exynos850-hsi2c
+          - const: samsung,exynosautov9-hsi2c
       - const: samsung,exynos5-hsi2c    # Exynos5250 and Exynos5420
         deprecated: true
 
diff --git a/Documentation/devicetree/bindings/i2c/samsung,s3c2410-i2c.yaml b/Documentation/devicetree/bindings/i2c/samsung,s3c2410-i2c.yaml
index b204e35e4f8d..1303502cf265 100644
--- a/Documentation/devicetree/bindings/i2c/samsung,s3c2410-i2c.yaml
+++ b/Documentation/devicetree/bindings/i2c/samsung,s3c2410-i2c.yaml
@@ -11,14 +11,20 @@ maintainers:
 
 properties:
   compatible:
-    enum:
-      - samsung,s3c2410-i2c
-      - samsung,s3c2440-i2c
-        # For s3c2440-like I2C used inside HDMIPHY block found on several SoCs:
-      - samsung,s3c2440-hdmiphy-i2c
-        # For s3c2440-like I2C used as a host to SATA PHY controller on an
-        # internal bus:
-      - samsung,exynos5-sata-phy-i2c
+    oneOf:
+      - enum:
+          - samsung,s3c2410-i2c
+          - samsung,s3c2440-i2c
+            # For s3c2440-like I2C used inside HDMIPHY block found on several SoCs:
+          - samsung,s3c2440-hdmiphy-i2c
+            # For s3c2440-like I2C used as a host to SATA PHY controller on an
+            # internal bus:
+          - samsung,exynos5-sata-phy-i2c
+      - items:
+          - enum:
+              - samsung,exynos7885-i2c
+              - samsung,exynos850-i2c
+          - const: samsung,s3c2440-i2c
 
   '#address-cells':
     const: 1
diff --git a/Documentation/devicetree/bindings/iio/adc/maxim,max34408.yaml b/Documentation/devicetree/bindings/iio/adc/maxim,max34408.yaml
new file mode 100644
index 000000000000..4cba856e8d47
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/adc/maxim,max34408.yaml
@@ -0,0 +1,139 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iio/adc/maxim,max34408.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Maxim MAX34408/MAX34409 current monitors with overcurrent control
+
+maintainers:
+  - Ivan Mikhaylov <fr0st61te@gmail.com>
+
+description: |
+  The MAX34408/MAX34409 are two- and four-channel current monitors that are
+  configured and monitored with a standard I2C/SMBus serial interface. Each
+  unidirectional current sensor offers precision high-side operation with a
+  low full-scale sense voltage. The devices automatically sequence through
+  two or four channels and collect the current-sense samples and average them
+  to reduce the effect of impulse noise. The raw ADC samples are compared to
+  user-programmable digital thresholds to indicate overcurrent conditions.
+  Overcurrent conditions trigger a hardware output to provide an immediate
+  indication to shut down any necessary external circuitry.
+
+  Specifications about the devices can be found at:
+  https://www.analog.com/media/en/technical-documentation/data-sheets/MAX34408-MAX34409.pdf
+
+properties:
+  compatible:
+    enum:
+      - maxim,max34408
+      - maxim,max34409
+
+  "#address-cells":
+    const: 1
+
+  "#size-cells":
+    const: 0
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  powerdown-gpios:
+    description:
+      Shutdown Output. Open-drain output. This output transitions to high impedance
+      when any of the digital comparator thresholds are exceeded as long as the ENA
+      pin is high.
+    maxItems: 1
+
+  powerdown-status-gpios:
+    description:
+      SHTDN Enable Input. CMOS digital input. Connect to GND to clear the latch and
+      unconditionally deassert (force low) the SHTDN output and reset the shutdown
+      delay. Connect to VDD to enable normal latch operation of the SHTDN output.
+    maxItems: 1
+
+  vdd-supply: true
+
+patternProperties:
+  "^channel@[0-3]$":
+    $ref: adc.yaml
+    type: object
+    description:
+      Represents the internal channels of the ADC.
+
+    properties:
+      reg:
+        items:
+          - minimum: 0
+            maximum: 3
+
+      maxim,rsense-val-micro-ohms:
+        description:
+          Adjust the Rsense value to monitor higher or lower current levels for
+          input.
+        enum: [250, 500, 1000, 5000, 10000, 50000, 100000, 200000, 500000]
+        default: 1000
+
+    required:
+      - reg
+      - maxim,rsense-val-micro-ohms
+
+    unevaluatedProperties: false
+
+required:
+  - compatible
+  - reg
+
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: maxim,max34408
+    then:
+      patternProperties:
+        "^channel@[2-3]$": false
+        "^channel@[0-1]$":
+          properties:
+            reg:
+              maximum: 1
+    else:
+      patternProperties:
+        "^channel@[0-3]$":
+          properties:
+            reg:
+              maximum: 3
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        adc@1e {
+              compatible = "maxim,max34409";
+              reg = <0x1e>;
+              powerdown-gpios = <&gpio0 1 GPIO_ACTIVE_LOW>;
+              powerdown-status-gpios = <&gpio0 2 GPIO_ACTIVE_HIGH>;
+
+              #address-cells = <1>;
+              #size-cells = <0>;
+
+              channel@0 {
+                  reg = <0x0>;
+                  maxim,rsense-val-micro-ohms = <5000>;
+              };
+
+              channel@1 {
+                  reg = <0x1>;
+                  maxim,rsense-val-micro-ohms = <10000>;
+             };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/iio/adc/samsung,exynos-adc.yaml b/Documentation/devicetree/bindings/iio/adc/samsung,exynos-adc.yaml
index 582d0a03b814..4e40f6bed5db 100644
--- a/Documentation/devicetree/bindings/iio/adc/samsung,exynos-adc.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/samsung,exynos-adc.yaml
@@ -11,18 +11,23 @@ maintainers:
 
 properties:
   compatible:
-    enum:
-      - samsung,exynos-adc-v1                 # Exynos5250
-      - samsung,exynos-adc-v2
-      - samsung,exynos3250-adc
-      - samsung,exynos4212-adc                # Exynos4212 and Exynos4412
-      - samsung,exynos7-adc
-      - samsung,s3c2410-adc
-      - samsung,s3c2416-adc
-      - samsung,s3c2440-adc
-      - samsung,s3c2443-adc
-      - samsung,s3c6410-adc
-      - samsung,s5pv210-adc
+    oneOf:
+      - enum:
+          - samsung,exynos-adc-v1                 # Exynos5250
+          - samsung,exynos-adc-v2
+          - samsung,exynos3250-adc
+          - samsung,exynos4212-adc                # Exynos4212 and Exynos4412
+          - samsung,exynos7-adc
+          - samsung,s3c2410-adc
+          - samsung,s3c2416-adc
+          - samsung,s3c2440-adc
+          - samsung,s3c2443-adc
+          - samsung,s3c6410-adc
+          - samsung,s5pv210-adc
+      - items:
+          - enum:
+              - samsung,exynos5433-adc
+          - const: samsung,exynos7-adc
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/iio/imu/adi,adis16460.yaml b/Documentation/devicetree/bindings/iio/imu/adi,adis16460.yaml
index 4e43c80e5119..4cacc9948726 100644
--- a/Documentation/devicetree/bindings/iio/imu/adi,adis16460.yaml
+++ b/Documentation/devicetree/bindings/iio/imu/adi,adis16460.yaml
@@ -25,6 +25,10 @@ properties:
 
   spi-cpol: true
 
+  spi-cs-inactive-delay-ns:
+    minimum: 16000
+    default: 16000
+
   interrupts:
     maxItems: 1
 
diff --git a/Documentation/devicetree/bindings/iio/imu/adi,adis16475.yaml b/Documentation/devicetree/bindings/iio/imu/adi,adis16475.yaml
index c73533c54588..9b7ad609f7db 100644
--- a/Documentation/devicetree/bindings/iio/imu/adi,adis16475.yaml
+++ b/Documentation/devicetree/bindings/iio/imu/adi,adis16475.yaml
@@ -47,6 +47,10 @@ properties:
   spi-max-frequency:
     maximum: 2000000
 
+  spi-cs-inactive-delay-ns:
+    minimum: 16000
+    default: 16000
+
   interrupts:
     maxItems: 1
 
diff --git a/Documentation/devicetree/bindings/iio/imu/bosch,bmi323.yaml b/Documentation/devicetree/bindings/iio/imu/bosch,bmi323.yaml
new file mode 100644
index 000000000000..64ef26e19669
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/imu/bosch,bmi323.yaml
@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iio/imu/bosch,bmi323.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Bosch BMI323 6-Axis IMU
+
+maintainers:
+  - Jagath Jog J <jagathjog1996@gmail.com>
+
+description:
+  BMI323 is a 6-axis inertial measurement unit that supports acceleration and
+  gyroscopic measurements with hardware fifo buffering. Sensor also provides
+  events information such as motion, steps, orientation, single and double
+  tap detection.
+
+properties:
+  compatible:
+    const: bosch,bmi323
+
+  reg:
+    maxItems: 1
+
+  vdd-supply: true
+  vddio-supply: true
+
+  interrupts:
+    minItems: 1
+    maxItems: 2
+
+  interrupt-names:
+    minItems: 1
+    maxItems: 2
+    items:
+      enum:
+        - INT1
+        - INT2
+
+  drive-open-drain:
+    description:
+      set if the specified interrupt pin should be configured as
+      open drain. If not set, defaults to push-pull.
+
+  mount-matrix:
+    description:
+      an optional 3x3 mounting rotation matrix.
+
+required:
+  - compatible
+  - reg
+  - vdd-supply
+  - vddio-supply
+
+allOf:
+  - $ref: /schemas/spi/spi-peripheral-props.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    // Example for I2C
+    #include <dt-bindings/interrupt-controller/irq.h>
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        imu@68 {
+            compatible = "bosch,bmi323";
+            reg = <0x68>;
+            vddio-supply = <&vddio>;
+            vdd-supply = <&vdd>;
+            interrupt-parent = <&gpio1>;
+            interrupts = <29 IRQ_TYPE_EDGE_RISING>;
+            interrupt-names = "INT1";
+        };
+    };
diff --git a/Documentation/devicetree/bindings/iio/temperature/microchip,mcp9600.yaml b/Documentation/devicetree/bindings/iio/temperature/microchip,mcp9600.yaml
new file mode 100644
index 000000000000..d2cafa38a544
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/temperature/microchip,mcp9600.yaml
@@ -0,0 +1,70 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iio/temperature/microchip,mcp9600.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Microchip MCP9600 thermocouple EMF converter
+
+maintainers:
+  - Andrew Hepp <andrew.hepp@ahepp.dev>
+
+description:
+  https://ww1.microchip.com/downloads/en/DeviceDoc/MCP960X-Data-Sheet-20005426.pdf
+
+properties:
+  compatible:
+    const: microchip,mcp9600
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    minItems: 1
+    maxItems: 6
+
+  interrupt-names:
+    minItems: 1
+    maxItems: 6
+    items:
+      enum:
+        - open-circuit
+        - short-circuit
+        - alert1
+        - alert2
+        - alert3
+        - alert4
+
+  thermocouple-type:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description:
+      Type of thermocouple (THERMOCOUPLE_TYPE_K if omitted).
+      Use defines in dt-bindings/iio/temperature/thermocouple.h.
+      Supported types are B, E, J, K, N, R, S, T.
+
+  vdd-supply: true
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/iio/temperature/thermocouple.h>
+    #include <dt-bindings/interrupt-controller/irq.h>
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        temperature-sensor@60 {
+            compatible = "microchip,mcp9600";
+            reg = <0x60>;
+            interrupt-parent = <&gpio>;
+            interrupts = <25 IRQ_TYPE_EDGE_RISING>;
+            interrupt-names = "open-circuit";
+            thermocouple-type = <THERMOCOUPLE_TYPE_K>;
+            vdd-supply = <&vdd>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/input/touchscreen/samsung,s6sy761.txt b/Documentation/devicetree/bindings/input/touchscreen/samsung,s6sy761.txt
deleted file mode 100644
index 6805d10d226d..000000000000
--- a/Documentation/devicetree/bindings/input/touchscreen/samsung,s6sy761.txt
+++ /dev/null
@@ -1,32 +0,0 @@
-* Samsung S6SY761 touchscreen controller
-
-Required properties:
-- compatible		: must be "samsung,s6sy761"
-- reg			: I2C slave address, (e.g. 0x48)
-- interrupts		: interrupt specification
-- avdd-supply		: analogic power supply
-- vdd-supply		: power supply
-
-Optional properties:
-- touchscreen-size-x	: see touchscreen.txt. This property is embedded in the
-			  device. If defined it forces a different x resolution.
-- touchscreen-size-y	: see touchscreen.txt. This property is embedded in the
-			  device. If defined it forces a different y resolution.
-
-Example:
-
-i2c@00000000 {
-
-	/* ... */
-
-	touchscreen@48 {
-		compatible = "samsung,s6sy761";
-		reg = <0x48>;
-		interrupt-parent = <&gpa1>;
-		interrupts = <1 IRQ_TYPE_NONE>;
-		avdd-supply = <&ldo30_reg>;
-		vdd-supply = <&ldo31_reg>;
-		touchscreen-size-x = <4096>;
-		touchscreen-size-y = <4096>;
-	};
-};
diff --git a/Documentation/devicetree/bindings/input/touchscreen/samsung,s6sy761.yaml b/Documentation/devicetree/bindings/input/touchscreen/samsung,s6sy761.yaml
new file mode 100644
index 000000000000..1ffd17af3c53
--- /dev/null
+++ b/Documentation/devicetree/bindings/input/touchscreen/samsung,s6sy761.yaml
@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/input/touchscreen/samsung,s6sy761.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Samsung S6SY761 touchscreen controller
+
+maintainers:
+  - Andi Shyti <andi.shyti@kernel.org>
+
+allOf:
+  - $ref: touchscreen.yaml#
+
+properties:
+  compatible:
+    const: samsung,s6sy761
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  avdd-supply: true
+  vdd-supply: true
+
+unevaluatedProperties: false
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - avdd-supply
+  - vdd-supply
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        touchscreen@48 {
+            compatible = "samsung,s6sy761";
+            reg = <0x48>;
+            interrupt-parent = <&gpa1>;
+            interrupts = <1 IRQ_TYPE_LEVEL_HIGH>;
+            avdd-supply = <&ldo30_reg>;
+            vdd-supply = <&ldo31_reg>;
+            touchscreen-size-x = <4096>;
+            touchscreen-size-y = <4096>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/mfd/samsung,exynos5433-lpass.yaml b/Documentation/devicetree/bindings/mfd/samsung,exynos5433-lpass.yaml
index b97b06848729..f154103f32cc 100644
--- a/Documentation/devicetree/bindings/mfd/samsung,exynos5433-lpass.yaml
+++ b/Documentation/devicetree/bindings/mfd/samsung,exynos5433-lpass.yaml
@@ -85,7 +85,7 @@ examples:
         };
 
         i2s@11440000 {
-            compatible = "samsung,exynos7-i2s";
+            compatible = "samsung,exynos5433-i2s", "samsung,exynos7-i2s";
             reg = <0x11440000 0x100>;
             dmas = <&adma 0>, <&adma 2>;
             dma-names = "tx", "rx";
diff --git a/Documentation/devicetree/bindings/mmc/arasan,sdhci.yaml b/Documentation/devicetree/bindings/mmc/arasan,sdhci.yaml
index 3e99801f77d2..9075add020bf 100644
--- a/Documentation/devicetree/bindings/mmc/arasan,sdhci.yaml
+++ b/Documentation/devicetree/bindings/mmc/arasan,sdhci.yaml
@@ -226,8 +226,8 @@ examples:
           interrupt-parent = <&gic>;
           interrupts = <0 48 4>;
           reg = <0xff160000 0x1000>;
-          clocks = <&clk200>, <&clk200>;
-          clock-names = "clk_xin", "clk_ahb";
+          clocks = <&clk200>, <&clk200>, <&clk1200>;
+          clock-names = "clk_xin", "clk_ahb", "gate";
           clock-output-names = "clk_out_sd0", "clk_in_sd0";
           #clock-cells = <1>;
           clk-phase-sd-hs = <63>, <72>;
@@ -239,8 +239,8 @@ examples:
           interrupt-parent = <&gic>;
           interrupts = <0 126 4>;
           reg = <0xf1040000 0x10000>;
-          clocks = <&clk200>, <&clk200>;
-          clock-names = "clk_xin", "clk_ahb";
+          clocks = <&clk200>, <&clk200>, <&clk1200>;
+          clock-names = "clk_xin", "clk_ahb", "gate";
           clock-output-names = "clk_out_sd0", "clk_in_sd0";
           #clock-cells = <1>;
           clk-phase-sd-hs = <132>, <60>;
diff --git a/Documentation/devicetree/bindings/mmc/samsung,exynos-dw-mshc.yaml b/Documentation/devicetree/bindings/mmc/samsung,exynos-dw-mshc.yaml
index 6ee78a38bd74..5fe65795f796 100644
--- a/Documentation/devicetree/bindings/mmc/samsung,exynos-dw-mshc.yaml
+++ b/Documentation/devicetree/bindings/mmc/samsung,exynos-dw-mshc.yaml
@@ -14,15 +14,22 @@ maintainers:
 
 properties:
   compatible:
-    enum:
-      - samsung,exynos4210-dw-mshc
-      - samsung,exynos4412-dw-mshc
-      - samsung,exynos5250-dw-mshc
-      - samsung,exynos5420-dw-mshc
-      - samsung,exynos5420-dw-mshc-smu
-      - samsung,exynos7-dw-mshc
-      - samsung,exynos7-dw-mshc-smu
-      - axis,artpec8-dw-mshc
+    oneOf:
+      - enum:
+          - axis,artpec8-dw-mshc
+          - samsung,exynos4210-dw-mshc
+          - samsung,exynos4412-dw-mshc
+          - samsung,exynos5250-dw-mshc
+          - samsung,exynos5420-dw-mshc
+          - samsung,exynos5420-dw-mshc-smu
+          - samsung,exynos7-dw-mshc
+          - samsung,exynos7-dw-mshc-smu
+      - items:
+          - enum:
+              - samsung,exynos5433-dw-mshc-smu
+              - samsung,exynos7885-dw-mshc-smu
+              - samsung,exynos850-dw-mshc-smu
+          - const: samsung,exynos7-dw-mshc-smu
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml b/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml
index a43eb837f8da..42804d955293 100644
--- a/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml
+++ b/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml
@@ -19,6 +19,7 @@ properties:
       - rockchip,rk3568-dwcmshc
       - rockchip,rk3588-dwcmshc
       - snps,dwcmshc-sdhci
+      - thead,th1520-dwcmshc
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/net/marvell,aquantia.yaml b/Documentation/devicetree/bindings/net/marvell,aquantia.yaml
new file mode 100644
index 000000000000..9854fab4c4db
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/marvell,aquantia.yaml
@@ -0,0 +1,116 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/net/marvell,aquantia.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Marvell Aquantia Ethernet PHY
+
+maintainers:
+  - Christian Marangi <ansuelsmth@gmail.com>
+
+description: |
+  Marvell Aquantia Ethernet PHY require a firmware to be loaded to actually
+  work.
+
+  This can be done and is implemented by OEM in 3 different way:
+    - Attached SPI flash directly to the PHY with the firmware. The PHY
+      will self load the firmware in the presence of this configuration.
+    - Read from a dedicated partition on system NAND declared in an
+      NVMEM cell, and loaded to the PHY using its mailbox interface.
+    - Manually provided firmware loaded from a file in the filesystem.
+
+allOf:
+  - $ref: ethernet-phy.yaml#
+
+select:
+  properties:
+    compatible:
+      contains:
+        enum:
+          - ethernet-phy-id03a1.b445
+          - ethernet-phy-id03a1.b460
+          - ethernet-phy-id03a1.b4a2
+          - ethernet-phy-id03a1.b4d0
+          - ethernet-phy-id03a1.b4e0
+          - ethernet-phy-id03a1.b5c2
+          - ethernet-phy-id03a1.b4b0
+          - ethernet-phy-id03a1.b662
+          - ethernet-phy-id03a1.b712
+          - ethernet-phy-id31c3.1c12
+  required:
+    - compatible
+
+properties:
+  reg:
+    maxItems: 1
+
+  firmware-name:
+    description: specify the name of PHY firmware to load
+
+  nvmem-cells:
+    description: phandle to the firmware nvmem cell
+    maxItems: 1
+
+  nvmem-cell-names:
+    const: firmware
+
+required:
+  - compatible
+  - reg
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    mdio {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        ethernet-phy@0 {
+            compatible = "ethernet-phy-id31c3.1c12",
+                         "ethernet-phy-ieee802.3-c45";
+
+            reg = <0>;
+            firmware-name = "AQR-G4_v5.4.C-AQR_CIG_WF-1945_0x8_ID44776_VER1630.cld";
+        };
+
+        ethernet-phy@1 {
+            compatible = "ethernet-phy-id31c3.1c12",
+                         "ethernet-phy-ieee802.3-c45";
+
+            reg = <1>;
+            nvmem-cells = <&aqr_fw>;
+            nvmem-cell-names = "firmware";
+        };
+    };
+
+    flash {
+        compatible = "jedec,spi-nor";
+        #address-cells = <1>;
+        #size-cells = <1>;
+
+        partitions {
+            compatible = "fixed-partitions";
+            #address-cells = <1>;
+            #size-cells = <1>;
+
+            /* ... */
+
+            partition@650000 {
+                compatible = "nvmem-cells";
+                label = "0:ethphyfw";
+                reg = <0x650000 0x80000>;
+                read-only;
+                #address-cells = <1>;
+                #size-cells = <1>;
+
+                aqr_fw: aqr_fw@0 {
+                    reg = <0x0 0x5f42a>;
+                };
+            };
+
+            /* ... */
+
+        };
+    };
diff --git a/Documentation/devicetree/bindings/net/renesas,etheravb.yaml b/Documentation/devicetree/bindings/net/renesas,etheravb.yaml
index 5d074f27d462..d3306b186000 100644
--- a/Documentation/devicetree/bindings/net/renesas,etheravb.yaml
+++ b/Documentation/devicetree/bindings/net/renesas,etheravb.yaml
@@ -55,7 +55,7 @@ properties:
 
       - items:
           - enum:
-              - renesas,r9a07g043-gbeth # RZ/G2UL
+              - renesas,r9a07g043-gbeth # RZ/G2UL and RZ/Five
               - renesas,r9a07g044-gbeth # RZ/G2{L,LC}
               - renesas,r9a07g054-gbeth # RZ/V2L
           - const: renesas,rzg2l-gbeth  # RZ/{G2L,G2UL,V2L} family
diff --git a/Documentation/devicetree/bindings/net/xlnx,axi-ethernet.yaml b/Documentation/devicetree/bindings/net/xlnx,axi-ethernet.yaml
index 1d33d80af11c..bbe89ea9590c 100644
--- a/Documentation/devicetree/bindings/net/xlnx,axi-ethernet.yaml
+++ b/Documentation/devicetree/bindings/net/xlnx,axi-ethernet.yaml
@@ -122,6 +122,20 @@ properties:
       and "phy-handle" should point to an external PHY if exists.
     maxItems: 1
 
+  dmas:
+    minItems: 2
+    maxItems: 32
+    description: TX and RX DMA channel phandle
+
+  dma-names:
+    items:
+      pattern: "^[tr]x_chan([0-9]|1[0-5])$"
+    description:
+      Should be "tx_chan0", "tx_chan1" ... "tx_chan15" for DMA Tx channel
+      Should be "rx_chan0", "rx_chan1" ... "rx_chan15" for DMA Rx channel
+    minItems: 2
+    maxItems: 32
+
 required:
   - compatible
   - interrupts
@@ -143,6 +157,8 @@ examples:
         clocks = <&axi_clk>, <&axi_clk>, <&pl_enet_ref_clk>, <&mgt_clk>;
         phy-mode = "mii";
         reg = <0x40c00000 0x40000>,<0x50c00000 0x40000>;
+        dmas = <&xilinx_dma 0>, <&xilinx_dma 1>;
+        dma-names = "tx_chan0", "rx_chan0";
         xlnx,rxcsum = <0x2>;
         xlnx,rxmem = <0x800>;
         xlnx,txcsum = <0x2>;
diff --git a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml
index 2c3d6553a7ba..6c03f2d5fca3 100644
--- a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml
+++ b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml
@@ -36,6 +36,8 @@ properties:
       - qcom,sm8450-qmp-gen4x2-pcie-phy
       - qcom,sm8550-qmp-gen3x2-pcie-phy
       - qcom,sm8550-qmp-gen4x2-pcie-phy
+      - qcom,sm8650-qmp-gen3x2-pcie-phy
+      - qcom,sm8650-qmp-gen4x2-pcie-phy
 
   reg:
     minItems: 1
@@ -147,6 +149,8 @@ allOf:
               - qcom,sm8450-qmp-gen3x2-pcie-phy
               - qcom,sm8550-qmp-gen3x2-pcie-phy
               - qcom,sm8550-qmp-gen4x2-pcie-phy
+              - qcom,sm8650-qmp-gen3x2-pcie-phy
+              - qcom,sm8650-qmp-gen4x2-pcie-phy
     then:
       properties:
         clocks:
@@ -189,6 +193,7 @@ allOf:
           contains:
             enum:
               - qcom,sm8550-qmp-gen4x2-pcie-phy
+              - qcom,sm8650-qmp-gen4x2-pcie-phy
     then:
       properties:
         resets:
diff --git a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml
index f3a3296c811c..8474eef8d0ff 100644
--- a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml
+++ b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml
@@ -32,6 +32,7 @@ properties:
       - qcom,sm8350-qmp-ufs-phy
       - qcom,sm8450-qmp-ufs-phy
       - qcom,sm8550-qmp-ufs-phy
+      - qcom,sm8650-qmp-ufs-phy
 
   reg:
     maxItems: 1
@@ -112,6 +113,7 @@ allOf:
               - qcom,sm8250-qmp-ufs-phy
               - qcom,sm8350-qmp-ufs-phy
               - qcom,sm8550-qmp-ufs-phy
+              - qcom,sm8650-qmp-ufs-phy
     then:
       properties:
         clocks:
diff --git a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-usb43dp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-usb43dp-phy.yaml
index 9af203dc8793..ae83cb8cb21f 100644
--- a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-usb43dp-phy.yaml
+++ b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-usb43dp-phy.yaml
@@ -27,6 +27,7 @@ properties:
       - qcom,sm8350-qmp-usb3-dp-phy
       - qcom,sm8450-qmp-usb3-dp-phy
       - qcom,sm8550-qmp-usb3-dp-phy
+      - qcom,sm8650-qmp-usb3-dp-phy
 
   reg:
     maxItems: 1
@@ -128,6 +129,7 @@ allOf:
             - qcom,sc8280xp-qmp-usb43dp-phy
             - qcom,sm6350-qmp-usb3-dp-phy
             - qcom,sm8550-qmp-usb3-dp-phy
+            - qcom,sm8650-qmp-usb3-dp-phy
     then:
       required:
         - power-domains
diff --git a/Documentation/devicetree/bindings/phy/qcom,snps-eusb2-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,snps-eusb2-phy.yaml
index c95828607ab6..8f5d7362046c 100644
--- a/Documentation/devicetree/bindings/phy/qcom,snps-eusb2-phy.yaml
+++ b/Documentation/devicetree/bindings/phy/qcom,snps-eusb2-phy.yaml
@@ -18,6 +18,7 @@ properties:
       - items:
           - enum:
               - qcom,sdx75-snps-eusb2-phy
+              - qcom,sm8650-snps-eusb2-phy
           - const: qcom,sm8550-snps-eusb2-phy
       - const: qcom,sm8550-snps-eusb2-phy
 
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sm8650-lpass-lpi-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sm8650-lpass-lpi-pinctrl.yaml
new file mode 100644
index 000000000000..f5736ed140ee
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,sm8650-lpass-lpi-pinctrl.yaml
@@ -0,0 +1,148 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pinctrl/qcom,sm8650-lpass-lpi-pinctrl.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm SM8650 SoC LPASS LPI TLMM
+
+maintainers:
+  - Krzysztof Kozlowski <krzysztof.kozlowski@linaro.org>
+  - Srinivas Kandagatla <srinivas.kandagatla@linaro.org>
+
+description:
+  Top Level Mode Multiplexer pin controller in the Low Power Audio SubSystem
+  (LPASS) Low Power Island (LPI) of Qualcomm SM8650 SoC.
+
+properties:
+  compatible:
+    const: qcom,sm8650-lpass-lpi-pinctrl
+
+  reg:
+    items:
+      - description: LPASS LPI TLMM Control and Status registers
+
+  clocks:
+    items:
+      - description: LPASS Core voting clock
+      - description: LPASS Audio voting clock
+
+  clock-names:
+    items:
+      - const: core
+      - const: audio
+
+  gpio-controller: true
+
+  "#gpio-cells":
+    description: Specifying the pin number and flags, as defined in
+      include/dt-bindings/gpio/gpio.h
+    const: 2
+
+  gpio-ranges:
+    maxItems: 1
+
+patternProperties:
+  "-state$":
+    oneOf:
+      - $ref: "#/$defs/qcom-sm8650-lpass-state"
+      - patternProperties:
+          "-pins$":
+            $ref: "#/$defs/qcom-sm8650-lpass-state"
+        additionalProperties: false
+
+$defs:
+  qcom-sm8650-lpass-state:
+    type: object
+    description:
+      Pinctrl node's client devices use subnodes for desired pin configuration.
+      Client device subnodes use below standard properties.
+    $ref: /schemas/pinctrl/pincfg-node.yaml
+
+    properties:
+      pins:
+        description:
+          List of gpio pins affected by the properties specified in this
+          subnode.
+        items:
+          pattern: "^gpio([0-9]|1[0-9]|2[0-2])$"
+
+      function:
+        enum: [ dmic1_clk, dmic1_data, dmic2_clk, dmic2_data, dmic3_clk,
+                dmic3_data, dmic4_clk, dmic4_data, ext_mclk1_a, ext_mclk1_b,
+                ext_mclk1_c, ext_mclk1_d, ext_mclk1_e, gpio, i2s0_clk,
+                i2s0_data, i2s0_ws, i2s1_clk, i2s1_data, i2s1_ws, i2s2_clk,
+                i2s2_data, i2s2_ws, i2s3_clk, i2s3_data, i2s3_ws, i2s4_clk,
+                i2s4_data, i2s4_ws, qca_swr_clk, qca_swr_data, slimbus_clk,
+                slimbus_data, swr_rx_clk, swr_rx_data, swr_tx_clk, swr_tx_data,
+                wsa_swr_clk, wsa_swr_data, wsa2_swr_clk, wsa2_swr_data ]
+        description:
+          Specify the alternative function to be configured for the specified
+          pins.
+
+      drive-strength:
+        enum: [2, 4, 6, 8, 10, 12, 14, 16]
+        default: 2
+        description:
+          Selects the drive strength for the specified pins, in mA.
+
+      slew-rate:
+        enum: [0, 1, 2, 3]
+        default: 0
+        description: |
+          0: No adjustments
+          1: Higher Slew rate (faster edges)
+          2: Lower Slew rate (slower edges)
+          3: Reserved (No adjustments)
+
+      bias-bus-hold: true
+      bias-pull-down: true
+      bias-pull-up: true
+      bias-disable: true
+      input-enable: true
+      output-high: true
+      output-low: true
+
+    required:
+      - pins
+      - function
+
+    additionalProperties: false
+
+allOf:
+  - $ref: pinctrl.yaml#
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - clock-names
+  - gpio-controller
+  - "#gpio-cells"
+  - gpio-ranges
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/sound/qcom,q6dsp-lpass-ports.h>
+
+    lpass_tlmm: pinctrl@6e80000 {
+        compatible = "qcom,sm8650-lpass-lpi-pinctrl";
+        reg = <0x06e80000 0x20000>;
+
+        clocks = <&q6prmcc LPASS_HW_MACRO_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>,
+                 <&q6prmcc LPASS_HW_DCODEC_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>;
+        clock-names = "core", "audio";
+
+        gpio-controller;
+        #gpio-cells = <2>;
+        gpio-ranges = <&lpass_tlmm 0 0 23>;
+
+        tx-swr-sleep-clk-state {
+            pins = "gpio0";
+            function = "swr_tx_clk";
+            drive-strength = <2>;
+            bias-pull-down;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sm8650-tlmm.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sm8650-tlmm.yaml
new file mode 100644
index 000000000000..30432be8fb07
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,sm8650-tlmm.yaml
@@ -0,0 +1,147 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pinctrl/qcom,sm8650-tlmm.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Technologies, Inc. SM8650 TLMM block
+
+maintainers:
+  - Bjorn Andersson <andersson@kernel.org>
+
+description:
+  Top Level Mode Multiplexer pin controller in Qualcomm SM8650 SoC.
+
+allOf:
+  - $ref: /schemas/pinctrl/qcom,tlmm-common.yaml#
+
+properties:
+  compatible:
+    const: qcom,sm8650-tlmm
+
+  reg:
+    maxItems: 1
+
+  interrupts: true
+  interrupt-controller: true
+  "#interrupt-cells": true
+  gpio-controller: true
+
+  gpio-reserved-ranges:
+    minItems: 1
+    maxItems: 105
+
+  gpio-line-names:
+    maxItems: 210
+
+  "#gpio-cells": true
+  gpio-ranges: true
+  wakeup-parent: true
+
+patternProperties:
+  "-state$":
+    oneOf:
+      - $ref: "#/$defs/qcom-sm8650-tlmm-state"
+      - patternProperties:
+          "-pins$":
+            $ref: "#/$defs/qcom-sm8650-tlmm-state"
+        additionalProperties: false
+
+$defs:
+  qcom-sm8650-tlmm-state:
+    type: object
+    description:
+      Pinctrl node's client devices use subnodes for desired pin configuration.
+      Client device subnodes use below standard properties.
+    $ref: qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state
+    unevaluatedProperties: false
+
+    properties:
+      pins:
+        description:
+          List of gpio pins affected by the properties specified in this
+          subnode.
+        items:
+          oneOf:
+            - pattern: "^gpio([0-9]|[1-9][0-9]|1[0-9][0-9]|20[0-9])$"
+            - enum: [ ufs_reset, sdc2_clk, sdc2_cmd, sdc2_data ]
+        minItems: 1
+        maxItems: 36
+
+      function:
+        description:
+          Specify the alternative function to be configured for the specified
+          pins.
+        enum: [ gpio, aoss_cti, atest_char, atest_usb, audio_ext_mclk0,
+                audio_ext_mclk1, audio_ref_clk, cam_aon_mclk2, cam_aon_mclk4,
+                cam_mclk, cci_async_in, cci_i2c_scl, cci_i2c_sda, cci_timer,
+                cmu_rng, coex_uart1_rx, coex_uart1_tx, coex_uart2_rx,
+                coex_uart2_tx, cri_trng, dbg_out_clk, ddr_bist_complete,
+                ddr_bist_fail, ddr_bist_start, ddr_bist_stop, ddr_pxi0,
+                ddr_pxi1, ddr_pxi2, ddr_pxi3, do_not, dp_hot, gcc_gp1,
+                gcc_gp2, gcc_gp3, gnss_adc0, gnss_adc1, i2chub0_se0,
+                i2chub0_se1, i2chub0_se2, i2chub0_se3, i2chub0_se4,
+                i2chub0_se5, i2chub0_se6, i2chub0_se7, i2chub0_se8,
+                i2chub0_se9, i2s0_data0, i2s0_data1, i2s0_sck, i2s0_ws,
+                i2s1_data0, i2s1_data1, i2s1_sck, i2s1_ws, ibi_i3c,
+                jitter_bist, mdp_vsync, mdp_vsync0_out, mdp_vsync1_out,
+                mdp_vsync2_out, mdp_vsync3_out, mdp_vsync_e, nav_gpio0,
+                nav_gpio1, nav_gpio2, nav_gpio3, pcie0_clk_req_n,
+                pcie1_clk_req_n, phase_flag, pll_bist_sync, pll_clk_aux,
+                prng_rosc0, prng_rosc1, prng_rosc2, prng_rosc3, qdss_cti,
+                qdss_gpio, qlink_big_enable, qlink_big_request,
+                qlink_little_enable, qlink_little_request, qlink_wmss,
+                qspi0, qspi1, qspi2, qspi3, qspi_clk, qspi_cs, qup1_se0,
+                qup1_se1, qup1_se2, qup1_se3, qup1_se4, qup1_se5, qup1_se6,
+                qup1_se7, qup2_se0, qup2_se1, qup2_se2, qup2_se3, qup2_se4,
+                qup2_se5, qup2_se6, qup2_se7, sd_write_protect, sdc40, sdc41,
+                sdc42, sdc43, sdc4_clk, sdc4_cmd, tb_trig_sdc2, tb_trig_sdc4,
+                tgu_ch0_trigout, tgu_ch1_trigout, tgu_ch2_trigout,
+                tgu_ch3_trigout, tmess_prng0, tmess_prng1, tmess_prng2,
+                tmess_prng3, tsense_pwm1, tsense_pwm2, tsense_pwm3, uim0_clk,
+                uim0_data, uim0_present, uim0_reset, uim1_clk, uim1_data,
+                uim1_present, uim1_reset, usb1_hs, usb_phy, vfr_0, vfr_1,
+                vsense_trigger_mirnat ]
+
+    required:
+      - pins
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    tlmm: pinctrl@f100000 {
+        compatible = "qcom,sm8650-tlmm";
+        reg = <0x0f100000 0x300000>;
+        gpio-controller;
+        #gpio-cells = <2>;
+        gpio-ranges = <&tlmm 0 0 211>;
+        interrupt-controller;
+        #interrupt-cells = <2>;
+        interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
+
+        gpio-wo-state {
+            pins = "gpio1";
+            function = "gpio";
+        };
+
+        uart-w-state {
+            rx-pins {
+                pins = "gpio60";
+                function = "qup1_se7";
+                bias-pull-up;
+            };
+
+            tx-pins {
+                pins = "gpio61";
+                function = "qup1_se7";
+                bias-disable;
+            };
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl-wakeup-interrupt.yaml b/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl-wakeup-interrupt.yaml
index 1de91a51234d..1b75abebb953 100644
--- a/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl-wakeup-interrupt.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl-wakeup-interrupt.yaml
@@ -28,15 +28,22 @@ description: |
 
 properties:
   compatible:
-    enum:
-      - samsung,s3c2410-wakeup-eint
-      - samsung,s3c2412-wakeup-eint
-      - samsung,s3c64xx-wakeup-eint
-      - samsung,s5pv210-wakeup-eint
-      - samsung,exynos4210-wakeup-eint
-      - samsung,exynos7-wakeup-eint
-      - samsung,exynos850-wakeup-eint
-      - samsung,exynosautov9-wakeup-eint
+    oneOf:
+      - enum:
+          - samsung,s3c2410-wakeup-eint
+          - samsung,s3c2412-wakeup-eint
+          - samsung,s3c64xx-wakeup-eint
+          - samsung,s5pv210-wakeup-eint
+          - samsung,exynos4210-wakeup-eint
+          - samsung,exynos7-wakeup-eint
+          - samsung,exynos850-wakeup-eint
+          - samsung,exynosautov9-wakeup-eint
+          - samsung,exynosautov920-wakeup-eint
+      - items:
+          - enum:
+              - samsung,exynos5433-wakeup-eint
+              - samsung,exynos7885-wakeup-eint
+          - const: samsung,exynos7-wakeup-eint
 
   interrupts:
     description:
@@ -99,6 +106,7 @@ allOf:
             enum:
               - samsung,exynos850-wakeup-eint
               - samsung,exynosautov9-wakeup-eint
+              - samsung,exynosautov920-wakeup-eint
     then:
       properties:
         interrupts: false
diff --git a/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml
index 26614621774a..9f04a0c76403 100644
--- a/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml
@@ -53,6 +53,7 @@ properties:
       - samsung,exynos7885-pinctrl
       - samsung,exynos850-pinctrl
       - samsung,exynosautov9-pinctrl
+      - samsung,exynosautov920-pinctrl
       - tesla,fsd-pinctrl
 
   interrupts:
@@ -313,7 +314,8 @@ examples:
         pinctrl-0 = <&initial_alive>;
 
         wakeup-interrupt-controller {
-            compatible = "samsung,exynos7-wakeup-eint";
+            compatible = "samsung,exynos5433-wakeup-eint",
+                         "samsung,exynos7-wakeup-eint";
             interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
         };
 
diff --git a/Documentation/devicetree/bindings/power/reset/nvmem-reboot-mode.yaml b/Documentation/devicetree/bindings/power/reset/nvmem-reboot-mode.yaml
index 14a262bcbf7c..627f8a6078c2 100644
--- a/Documentation/devicetree/bindings/power/reset/nvmem-reboot-mode.yaml
+++ b/Documentation/devicetree/bindings/power/reset/nvmem-reboot-mode.yaml
@@ -28,17 +28,15 @@ properties:
     items:
       - const: reboot-mode
 
-patternProperties:
-  "^mode-.+":
-    $ref: /schemas/types.yaml#/definitions/uint32
-    description: Vendor-specific mode value written to the mode register
+allOf:
+  - $ref: reboot-mode.yaml#
 
 required:
   - compatible
   - nvmem-cells
   - nvmem-cell-names
 
-additionalProperties: false
+unevaluatedProperties: false
 
 examples:
   - |
diff --git a/Documentation/devicetree/bindings/power/reset/syscon-reboot-mode.yaml b/Documentation/devicetree/bindings/power/reset/syscon-reboot-mode.yaml
index 9b1ffceefe3d..b6acff199cde 100644
--- a/Documentation/devicetree/bindings/power/reset/syscon-reboot-mode.yaml
+++ b/Documentation/devicetree/bindings/power/reset/syscon-reboot-mode.yaml
@@ -29,12 +29,10 @@ properties:
     $ref: /schemas/types.yaml#/definitions/uint32
     description: Offset in the register map for the mode register (in bytes)
 
-patternProperties:
-  "^mode-.+":
-    $ref: /schemas/types.yaml#/definitions/uint32
-    description: Vendor-specific mode value written to the mode register
+allOf:
+  - $ref: reboot-mode.yaml#
 
-additionalProperties: false
+unevaluatedProperties: false
 
 required:
   - compatible
diff --git a/Documentation/devicetree/bindings/pwm/pwm-samsung.yaml b/Documentation/devicetree/bindings/pwm/pwm-samsung.yaml
index 2162f661ed5a..16de6434693b 100644
--- a/Documentation/devicetree/bindings/pwm/pwm-samsung.yaml
+++ b/Documentation/devicetree/bindings/pwm/pwm-samsung.yaml
@@ -29,7 +29,10 @@ properties:
           - samsung,exynos4210-pwm          # 32-bit, Exynos
       - items:
           - enum:
+              - samsung,exynos5433-pwm
+              - samsung,exynos7-pwm
               - samsung,exynosautov9-pwm
+              - samsung,exynosautov920-pwm
           - const: samsung,exynos4210-pwm
 
   reg:
diff --git a/Documentation/devicetree/bindings/regulator/fixed-regulator.yaml b/Documentation/devicetree/bindings/regulator/fixed-regulator.yaml
index ce7751b9129c..9ff9abf2691a 100644
--- a/Documentation/devicetree/bindings/regulator/fixed-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/fixed-regulator.yaml
@@ -105,6 +105,8 @@ properties:
     description:
       Interrupt signaling a critical under-voltage event.
 
+  system-critical-regulator: true
+
 required:
   - compatible
   - regulator-name
diff --git a/Documentation/devicetree/bindings/regulator/qcom,smd-rpm-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom,smd-rpm-regulator.yaml
index 9ea8ac0786ac..f2fd2df68a9e 100644
--- a/Documentation/devicetree/bindings/regulator/qcom,smd-rpm-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/qcom,smd-rpm-regulator.yaml
@@ -47,6 +47,9 @@ description:
   For pm8916, s1, s2, s3, s4, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11,
   l12, l13, l14, l15, l16, l17, l18
 
+  For pm8937, s1, s2, s3, s4, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10,
+  l11, l12, l13, l14, l15, l16, l17, l18, l19, l20, l21, l22, l23
+
   For pm8941, s1, s2, s3, s4, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11,
   l12, l13, l14, l15, l16, l17, l18, l19, l20, l21, l22, l23, l24, lvs1, lvs2,
   lvs3, 5vs1, 5vs2
@@ -92,6 +95,7 @@ properties:
       - qcom,rpm-pm8841-regulators
       - qcom,rpm-pm8909-regulators
       - qcom,rpm-pm8916-regulators
+      - qcom,rpm-pm8937-regulators
       - qcom,rpm-pm8941-regulators
       - qcom,rpm-pm8950-regulators
       - qcom,rpm-pm8953-regulators
diff --git a/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.yaml
index 7a1b7d2abbd4..aea849e8eadf 100644
--- a/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.yaml
@@ -22,6 +22,7 @@ properties:
       - qcom,pm8841-regulators
       - qcom,pm8909-regulators
       - qcom,pm8916-regulators
+      - qcom,pm8937-regulators
       - qcom,pm8941-regulators
       - qcom,pm8950-regulators
       - qcom,pm8994-regulators
@@ -291,6 +292,24 @@ allOf:
       patternProperties:
         "^vdd_s[1-3]-supply$": true
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,pm8937-regulators
+    then:
+      properties:
+        vdd_l1_l19-supply: true
+        vdd_l20_l21-supply: true
+        vdd_l2_l23-supply: true
+        vdd_l3-supply: true
+        vdd_l4_l5_l6_l7_l16-supply: true
+        vdd_l8_l11_l12_l17_l22-supply: true
+        vdd_l9_l10_l13_l14_l15_l18-supply: true
+      patternProperties:
+        "^vdd_s[1-6]-supply$": true
+
   - if:
       properties:
         compatible:
diff --git a/Documentation/devicetree/bindings/regulator/regulator.yaml b/Documentation/devicetree/bindings/regulator/regulator.yaml
index 9daf0fc2465f..1ef380d1515e 100644
--- a/Documentation/devicetree/bindings/regulator/regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/regulator.yaml
@@ -114,6 +114,11 @@ properties:
     description: Enable pull down resistor when the regulator is disabled.
     type: boolean
 
+  system-critical-regulator:
+    description: Set if the regulator is critical to system stability or
+      functionality.
+    type: boolean
+
   regulator-over-current-protection:
     description: Enable over current protection.
     type: boolean
@@ -181,6 +186,14 @@ properties:
       be enabled but limit setting can be omitted. Limit is given as microvolt
       offset from voltage set to regulator.
 
+  regulator-uv-less-critical-window-ms:
+    description: Specifies the time window (in milliseconds) following a
+      critical under-voltage event during which the system can continue to
+      operate safely while performing less critical operations. This property
+      provides a defined duration before a more severe reaction to the
+      under-voltage event is needed, allowing for certain non-urgent actions to
+      be carried out in preparation for potential power loss.
+
   regulator-temp-protection-kelvin:
     description: Set over temperature protection limit. This is a limit where
       hardware performs emergency shutdown. Zero can be passed to disable
diff --git a/Documentation/devicetree/bindings/rtc/s3c-rtc.yaml b/Documentation/devicetree/bindings/rtc/s3c-rtc.yaml
index d51b236939bf..bf4e11d6dffb 100644
--- a/Documentation/devicetree/bindings/rtc/s3c-rtc.yaml
+++ b/Documentation/devicetree/bindings/rtc/s3c-rtc.yaml
@@ -17,6 +17,11 @@ properties:
           - samsung,s3c2416-rtc
           - samsung,s3c2443-rtc
           - samsung,s3c6410-rtc
+      - items:
+          - enum:
+              - samsung,exynos7-rtc
+              - samsung,exynos850-rtc
+          - const: samsung,s3c6410-rtc
       - const: samsung,exynos3250-rtc
         deprecated: true
 
diff --git a/Documentation/devicetree/bindings/serial/samsung_uart.yaml b/Documentation/devicetree/bindings/serial/samsung_uart.yaml
index ac60ab1e35e3..e32c1b462836 100644
--- a/Documentation/devicetree/bindings/serial/samsung_uart.yaml
+++ b/Documentation/devicetree/bindings/serial/samsung_uart.yaml
@@ -18,9 +18,6 @@ description: |+
 properties:
   compatible:
     oneOf:
-      - items:
-          - const: samsung,exynosautov9-uart
-          - const: samsung,exynos850-uart
       - enum:
           - apple,s5l-uart
           - axis,artpec8-uart
@@ -29,6 +26,19 @@ properties:
           - samsung,exynos4210-uart
           - samsung,exynos5433-uart
           - samsung,exynos850-uart
+      - items:
+          - enum:
+              - samsung,exynos7-uart
+          - const: samsung,exynos4210-uart
+      - items:
+          - enum:
+              - samsung,exynos7885-uart
+          - const: samsung,exynos5433-uart
+      - items:
+          - enum:
+              - samsung,exynosautov9-uart
+              - samsung,exynosautov920-uart
+          - const: samsung,exynos850-uart
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/soc/rockchip/grf.yaml b/Documentation/devicetree/bindings/soc/rockchip/grf.yaml
index e4fa6a07b4fa..1309bf5ae0cd 100644
--- a/Documentation/devicetree/bindings/soc/rockchip/grf.yaml
+++ b/Documentation/devicetree/bindings/soc/rockchip/grf.yaml
@@ -233,6 +233,7 @@ allOf:
               - rockchip,rk3399-grf
               - rockchip,rk3399-pmugrf
               - rockchip,rk3568-pmugrf
+              - rockchip,rk3588-pmugrf
               - rockchip,rv1108-grf
               - rockchip,rv1108-pmugrf
 
diff --git a/Documentation/devicetree/bindings/soc/samsung/exynos-pmu.yaml b/Documentation/devicetree/bindings/soc/samsung/exynos-pmu.yaml
index e1d716df5dfa..28e2cb50d85e 100644
--- a/Documentation/devicetree/bindings/soc/samsung/exynos-pmu.yaml
+++ b/Documentation/devicetree/bindings/soc/samsung/exynos-pmu.yaml
@@ -48,6 +48,13 @@ properties:
               - samsung,exynos850-pmu
               - samsung-s5pv210-pmu
           - const: syscon
+      - items:
+          - enum:
+              - samsung,exynos7885-pmu
+              - samsung,exynosautov9-pmu
+              - samsung,exynosautov920-pmu
+          - const: samsung,exynos7-pmu
+          - const: syscon
       - items:
           - enum:
               - samsung,exynos3250-pmu
diff --git a/Documentation/devicetree/bindings/soc/samsung/exynos-usi.yaml b/Documentation/devicetree/bindings/soc/samsung/exynos-usi.yaml
index a6836904a4f8..61be1f2ddbe7 100644
--- a/Documentation/devicetree/bindings/soc/samsung/exynos-usi.yaml
+++ b/Documentation/devicetree/bindings/soc/samsung/exynos-usi.yaml
@@ -24,7 +24,9 @@ properties:
   compatible:
     oneOf:
       - items:
-          - const: samsung,exynosautov9-usi
+          - enum:
+              - samsung,exynosautov9-usi
+              - samsung,exynosautov920-usi
           - const: samsung,exynos850-usi
       - enum:
           - samsung,exynos850-usi
@@ -155,7 +157,7 @@ examples:
         };
 
         hsi2c_0: i2c@13820000 {
-            compatible = "samsung,exynosautov9-hsi2c";
+            compatible = "samsung,exynos850-hsi2c", "samsung,exynosautov9-hsi2c";
             reg = <0x13820000 0xc0>;
             interrupts = <GIC_SPI 227 IRQ_TYPE_LEVEL_HIGH>;
             #address-cells = <1>;
diff --git a/Documentation/devicetree/bindings/soc/samsung/samsung,exynos-sysreg.yaml b/Documentation/devicetree/bindings/soc/samsung/samsung,exynos-sysreg.yaml
index 163e912e9cad..2de4301a467d 100644
--- a/Documentation/devicetree/bindings/soc/samsung/samsung,exynos-sysreg.yaml
+++ b/Documentation/devicetree/bindings/soc/samsung/samsung,exynos-sysreg.yaml
@@ -50,6 +50,11 @@ properties:
               - samsung,exynosautov9-peric1-sysreg
           - const: samsung,exynosautov9-sysreg
           - const: syscon
+      - items:
+          - enum:
+              - samsung,exynosautov920-peric0-sysreg
+              - samsung,exynosautov920-peric1-sysreg
+          - const: syscon
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/sound/adi,max98363.yaml b/Documentation/devicetree/bindings/sound/adi,max98363.yaml
index a844b63f3930..c388cda56011 100644
--- a/Documentation/devicetree/bindings/sound/adi,max98363.yaml
+++ b/Documentation/devicetree/bindings/sound/adi,max98363.yaml
@@ -39,7 +39,7 @@ unevaluatedProperties: false
 
 examples:
   - |
-    soundwire-controller@3250000 {
+    soundwire@3250000 {
         #address-cells = <2>;
         #size-cells = <0>;
         reg = <0x3250000 0x2000>;
diff --git a/Documentation/devicetree/bindings/sound/es8328.txt b/Documentation/devicetree/bindings/sound/es8328.txt
deleted file mode 100644
index 33fbf058c997..000000000000
--- a/Documentation/devicetree/bindings/sound/es8328.txt
+++ /dev/null
@@ -1,38 +0,0 @@
-Everest ES8328 audio CODEC
-
-This device supports both I2C and SPI.
-
-Required properties:
-
-  - compatible  : Should be "everest,es8328" or "everest,es8388"
-  - DVDD-supply : Regulator providing digital core supply voltage 1.8 - 3.6V
-  - AVDD-supply : Regulator providing analog supply voltage 3.3V
-  - PVDD-supply : Regulator providing digital IO supply voltage 1.8 - 3.6V
-  - IPVDD-supply : Regulator providing analog output voltage 3.3V
-  - clocks : A 22.5792 or 11.2896 MHz clock
-  - reg : the I2C address of the device for I2C, the chip select number for SPI
-
-Pins on the device (for linking into audio routes):
-
-  * LOUT1
-  * LOUT2
-  * ROUT1
-  * ROUT2
-  * LINPUT1
-  * RINPUT1
-  * LINPUT2
-  * RINPUT2
-  * Mic Bias
-
-
-Example:
-
-codec: es8328@11 {
-	compatible = "everest,es8328";
-	DVDD-supply = <&reg_3p3v>;
-	AVDD-supply = <&reg_3p3v>;
-	PVDD-supply = <&reg_3p3v>;
-	HPVDD-supply = <&reg_3p3v>;
-	clocks = <&clks 169>;
-	reg = <0x11>;
-};
diff --git a/Documentation/devicetree/bindings/sound/everest,es8328.yaml b/Documentation/devicetree/bindings/sound/everest,es8328.yaml
new file mode 100644
index 000000000000..a0f4670fa38c
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/everest,es8328.yaml
@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/everest,es8328.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Everest ES8328 audio CODEC
+
+description:
+  Everest Audio Codec, which can be connected via I2C or SPI.
+  Pins on the device (for linking into audio routes) are
+  * LOUT1
+  * LOUT2
+  * ROUT1
+  * ROUT2
+  * LINPUT1
+  * RINPUT1
+  * LINPUT2
+  * RINPUT2
+  * Mic Bias
+
+maintainers:
+  - David Yang <yangxiaohua@everest-semi.com>
+
+properties:
+  compatible:
+    enum:
+      - everest,es8328
+      - everest,es8388
+
+  reg:
+    maxItems: 1
+
+  "#sound-dai-cells":
+    const: 0
+
+  clocks:
+    items:
+      - description: A 22.5792 or 11.2896 MHz clock
+
+  DVDD-supply:
+    description: Regulator providing digital core supply voltage 1.8 - 3.6V
+
+  AVDD-supply:
+    description: Regulator providing analog supply voltage 3.3V
+
+  PVDD-supply:
+    description: Regulator providing digital IO supply voltage 1.8 - 3.6V
+
+  HPVDD-supply:
+    description: Regulator providing analog output voltage 3.3V
+
+required:
+  - compatible
+  - clocks
+  - DVDD-supply
+  - AVDD-supply
+  - PVDD-supply
+  - HPVDD-supply
+
+additionalProperties: false
+
+examples:
+  - |
+    i2c {
+      #address-cells = <1>;
+      #size-cells = <0>;
+      es8328: codec@11 {
+        compatible = "everest,es8328";
+        reg = <0x11>;
+        AVDD-supply = <&reg_3p3v>;
+        DVDD-supply = <&reg_3p3v>;
+        HPVDD-supply = <&reg_3p3v>;
+        PVDD-supply = <&reg_3p3v>;
+        clocks = <&clks 169>;
+      };
+    };
diff --git a/Documentation/devicetree/bindings/sound/nuvoton,nau8821.yaml b/Documentation/devicetree/bindings/sound/nuvoton,nau8821.yaml
index 3e54abd4ca74..3380b6aa9542 100644
--- a/Documentation/devicetree/bindings/sound/nuvoton,nau8821.yaml
+++ b/Documentation/devicetree/bindings/sound/nuvoton,nau8821.yaml
@@ -89,6 +89,14 @@ properties:
     $ref: /schemas/types.yaml#/definitions/uint32
     default: 3072000
 
+  nuvoton,dmic-slew-rate:
+    description: The range 0 to 7 represents the speed of DMIC slew rate.
+        The lowest value 0 means the slowest rate and the highest value
+        7 means the fastest rate.
+    $ref: /schemas/types.yaml#/definitions/uint32
+    maximum: 7
+    default: 0
+
   nuvoton,left-input-single-end:
     description: Enable left input with single-ended settings if set.
         For the headset mic application, the single-ended control is
@@ -127,6 +135,7 @@ examples:
             nuvoton,jack-insert-debounce = <7>;
             nuvoton,jack-eject-debounce = <0>;
             nuvoton,dmic-clk-threshold = <3072000>;
+            nuvoton,dmic-slew-rate= <0>;
             #sound-dai-cells = <0>;
         };
     };
diff --git a/Documentation/devicetree/bindings/sound/qcom,sm8250.yaml b/Documentation/devicetree/bindings/sound/qcom,sm8250.yaml
index e082a4fe095d..ec641fa2cd4b 100644
--- a/Documentation/devicetree/bindings/sound/qcom,sm8250.yaml
+++ b/Documentation/devicetree/bindings/sound/qcom,sm8250.yaml
@@ -21,6 +21,10 @@ properties:
               - lenovo,yoga-c630-sndcard
               - qcom,db845c-sndcard
           - const: qcom,sdm845-sndcard
+      - items:
+          - enum:
+              - qcom,sm8550-sndcard
+          - const: qcom,sm8450-sndcard
       - enum:
           - qcom,apq8016-sbc-sndcard
           - qcom,msm8916-qdsp6-sndcard
diff --git a/Documentation/devicetree/bindings/sound/qcom,wsa883x.yaml b/Documentation/devicetree/bindings/sound/qcom,wsa883x.yaml
index ba572a7f4f3c..8e462cdf0018 100644
--- a/Documentation/devicetree/bindings/sound/qcom,wsa883x.yaml
+++ b/Documentation/devicetree/bindings/sound/qcom,wsa883x.yaml
@@ -52,7 +52,7 @@ examples:
   - |
     #include <dt-bindings/gpio/gpio.h>
 
-    soundwire-controller@3250000 {
+    soundwire@3250000 {
         #address-cells = <2>;
         #size-cells = <0>;
         reg = <0x3250000 0x2000>;
diff --git a/Documentation/devicetree/bindings/sound/qcom,wsa8840.yaml b/Documentation/devicetree/bindings/sound/qcom,wsa8840.yaml
index e6723c9e312a..d717017b0fdb 100644
--- a/Documentation/devicetree/bindings/sound/qcom,wsa8840.yaml
+++ b/Documentation/devicetree/bindings/sound/qcom,wsa8840.yaml
@@ -48,7 +48,7 @@ examples:
   - |
     #include <dt-bindings/gpio/gpio.h>
 
-    soundwire-controller {
+    soundwire {
         #address-cells = <2>;
         #size-cells = <0>;
 
diff --git a/Documentation/devicetree/bindings/sound/renesas,rsnd.yaml b/Documentation/devicetree/bindings/sound/renesas,rsnd.yaml
index 13a5a0a10fe6..1174205286d4 100644
--- a/Documentation/devicetree/bindings/sound/renesas,rsnd.yaml
+++ b/Documentation/devicetree/bindings/sound/renesas,rsnd.yaml
@@ -9,20 +9,6 @@ title: Renesas R-Car Sound Driver
 maintainers:
   - Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>
 
-definitions:
-  port-def:
-    $ref: audio-graph-port.yaml#/definitions/port-base
-    unevaluatedProperties: false
-    patternProperties:
-      "^endpoint(@[0-9a-f]+)?":
-        $ref: audio-graph-port.yaml#/definitions/endpoint-base
-        properties:
-          playback:
-            $ref: /schemas/types.yaml#/definitions/phandle-array
-          capture:
-            $ref: /schemas/types.yaml#/definitions/phandle-array
-        unevaluatedProperties: false
-
 properties:
 
   compatible:
@@ -125,7 +111,17 @@ properties:
 
   # ports is below
   port:
-    $ref: "#/definitions/port-def"
+    $ref: audio-graph-port.yaml#/definitions/port-base
+    unevaluatedProperties: false
+    patternProperties:
+      "^endpoint(@[0-9a-f]+)?":
+        $ref: audio-graph-port.yaml#/definitions/endpoint-base
+        properties:
+          playback:
+            $ref: /schemas/types.yaml#/definitions/phandle-array
+          capture:
+            $ref: /schemas/types.yaml#/definitions/phandle-array
+        unevaluatedProperties: false
 
   rcar_sound,dvc:
     description: DVC subnode.
@@ -269,7 +265,7 @@ patternProperties:
     unevaluatedProperties: false
     patternProperties:
       '^port(@[0-9a-f]+)?$':
-        $ref: "#/definitions/port-def"
+        $ref: "#/properties/port"
 
 required:
   - compatible
diff --git a/Documentation/devicetree/bindings/sound/renesas,rz-ssi.yaml b/Documentation/devicetree/bindings/sound/renesas,rz-ssi.yaml
index 3b5ae45eee4a..8b9695f5decc 100644
--- a/Documentation/devicetree/bindings/sound/renesas,rz-ssi.yaml
+++ b/Documentation/devicetree/bindings/sound/renesas,rz-ssi.yaml
@@ -16,7 +16,7 @@ properties:
   compatible:
     items:
       - enum:
-          - renesas,r9a07g043-ssi  # RZ/G2UL
+          - renesas,r9a07g043-ssi  # RZ/G2UL and RZ/Five
           - renesas,r9a07g044-ssi  # RZ/G2{L,LC}
           - renesas,r9a07g054-ssi  # RZ/V2L
       - const: renesas,rz-ssi
diff --git a/Documentation/devicetree/bindings/sound/samsung-i2s.yaml b/Documentation/devicetree/bindings/sound/samsung-i2s.yaml
index 30b3b6e9824b..f45f73b5056d 100644
--- a/Documentation/devicetree/bindings/sound/samsung-i2s.yaml
+++ b/Documentation/devicetree/bindings/sound/samsung-i2s.yaml
@@ -44,13 +44,18 @@ properties:
       frequencies supported by Exynos7 I2S and 7.1 channel TDM support
       for playback and capture TDM (Time division multiplexing) to allow
       transfer of multiple channel audio data on single data line.
-    enum:
-      - samsung,s3c6410-i2s
-      - samsung,s5pv210-i2s
-      - samsung,exynos5420-i2s
-      - samsung,exynos7-i2s
-      - samsung,exynos7-i2s1
-      - tesla,fsd-i2s
+    oneOf:
+      - enum:
+          - samsung,s3c6410-i2s
+          - samsung,s5pv210-i2s
+          - samsung,exynos5420-i2s
+          - samsung,exynos7-i2s
+          - samsung,exynos7-i2s1
+          - tesla,fsd-i2s
+      - items:
+          - enum:
+              - samsung,exynos5433-i2s
+          - const: samsung,exynos7-i2s
 
   '#address-cells':
     const: 1
diff --git a/Documentation/devicetree/bindings/sound/sound-card-common.yaml b/Documentation/devicetree/bindings/sound/sound-card-common.yaml
index 3a941177f684..721950f65748 100644
--- a/Documentation/devicetree/bindings/sound/sound-card-common.yaml
+++ b/Documentation/devicetree/bindings/sound/sound-card-common.yaml
@@ -17,6 +17,13 @@ properties:
       pair of strings, the first being the connection's sink, the second
       being the connection's source.
 
+  ignore-suspend-widgets:
+    $ref: /schemas/types.yaml#/definitions/non-unique-string-array
+    description: |
+      A list of audio sound widgets which are marked ignoring system suspend.
+      Paths between these endpoints are still active over suspend of the main
+      application processor that the current operating system is running.
+
   model:
     $ref: /schemas/types.yaml#/definitions/string
     description: User specified audio sound card name
diff --git a/Documentation/devicetree/bindings/spi/adi,axi-spi-engine.txt b/Documentation/devicetree/bindings/spi/adi,axi-spi-engine.txt
deleted file mode 100644
index 8a18d71e6879..000000000000
--- a/Documentation/devicetree/bindings/spi/adi,axi-spi-engine.txt
+++ /dev/null
@@ -1,31 +0,0 @@
-Analog Devices AXI SPI Engine controller Device Tree Bindings
-
-Required properties:
-- compatible		: Must be "adi,axi-spi-engine-1.00.a""
-- reg			: Physical base address and size of the register map.
-- interrupts		: Property with a value describing the interrupt
-			  number.
-- clock-names		: List of input clock names - "s_axi_aclk", "spi_clk"
-- clocks		: Clock phandles and specifiers (See clock bindings for
-			  details on clock-names and clocks).
-- #address-cells	: Must be <1>
-- #size-cells		: Must be <0>
-
-Optional subnodes:
-	Subnodes are use to represent the SPI slave devices connected to the SPI
-	master. They follow the generic SPI bindings as outlined in spi-bus.txt.
-
-Example:
-
-    spi@@44a00000 {
-		compatible = "adi,axi-spi-engine-1.00.a";
-		reg = <0x44a00000 0x1000>;
-		interrupts = <0 56 4>;
-		clocks = <&clkc 15 &clkc 15>;
-		clock-names = "s_axi_aclk", "spi_clk";
-
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		/* SPI devices */
-    };
diff --git a/Documentation/devicetree/bindings/spi/adi,axi-spi-engine.yaml b/Documentation/devicetree/bindings/spi/adi,axi-spi-engine.yaml
new file mode 100644
index 000000000000..d48faa42d025
--- /dev/null
+++ b/Documentation/devicetree/bindings/spi/adi,axi-spi-engine.yaml
@@ -0,0 +1,66 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/spi/adi,axi-spi-engine.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Analog Devices AXI SPI Engine Controller
+
+description: |
+  The AXI SPI Engine controller is part of the SPI Engine framework[1] and
+  allows memory mapped access to the SPI Engine control bus. This allows it
+  to be used as a general purpose software driven SPI controller as well as
+  some optional advanced acceleration and offloading capabilities.
+
+  [1] https://wiki.analog.com/resources/fpga/peripherals/spi_engine
+
+maintainers:
+  - Michael Hennerich <Michael.Hennerich@analog.com>
+  - Nuno Sá <nuno.sa@analog.com>
+
+allOf:
+  - $ref: /schemas/spi/spi-controller.yaml#
+
+properties:
+  compatible:
+    const: adi,axi-spi-engine-1.00.a
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  clocks:
+    items:
+      - description: The AXI interconnect clock.
+      - description: The SPI controller clock.
+
+  clock-names:
+    items:
+      - const: s_axi_aclk
+      - const: spi_clk
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - clocks
+  - clock-names
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    spi@44a00000 {
+        compatible = "adi,axi-spi-engine-1.00.a";
+        reg = <0x44a00000 0x1000>;
+        interrupts = <0 56 4>;
+        clocks = <&clkc 15>, <&clkc 15>;
+        clock-names = "s_axi_aclk", "spi_clk";
+
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        /* SPI devices */
+    };
diff --git a/Documentation/devicetree/bindings/spi/renesas,rspi.yaml b/Documentation/devicetree/bindings/spi/renesas,rspi.yaml
index 4d8ec69214c9..0ef3f8421986 100644
--- a/Documentation/devicetree/bindings/spi/renesas,rspi.yaml
+++ b/Documentation/devicetree/bindings/spi/renesas,rspi.yaml
@@ -21,7 +21,7 @@ properties:
           - enum:
               - renesas,rspi-r7s72100  # RZ/A1H
               - renesas,rspi-r7s9210   # RZ/A2
-              - renesas,r9a07g043-rspi # RZ/G2UL
+              - renesas,r9a07g043-rspi # RZ/G2UL and RZ/Five
               - renesas,r9a07g044-rspi # RZ/G2{L,LC}
               - renesas,r9a07g054-rspi # RZ/V2L
           - const: renesas,rspi-rz
diff --git a/Documentation/devicetree/bindings/spi/st,stm32-spi.yaml b/Documentation/devicetree/bindings/spi/st,stm32-spi.yaml
index ae0f082bd377..5754d603f34f 100644
--- a/Documentation/devicetree/bindings/spi/st,stm32-spi.yaml
+++ b/Documentation/devicetree/bindings/spi/st,stm32-spi.yaml
@@ -23,6 +23,7 @@ properties:
   compatible:
     enum:
       - st,stm32f4-spi
+      - st,stm32f7-spi
       - st,stm32h7-spi
 
   reg:
diff --git a/Documentation/devicetree/bindings/trivial-devices.yaml b/Documentation/devicetree/bindings/trivial-devices.yaml
index c3190f2a168a..441b55723675 100644
--- a/Documentation/devicetree/bindings/trivial-devices.yaml
+++ b/Documentation/devicetree/bindings/trivial-devices.yaml
@@ -125,6 +125,8 @@ properties:
           - mps,mp2973
             # Monolithic Power Systems Inc. multi-phase controller mp2975
           - mps,mp2975
+            # Monolithic Power Systems Inc. multi-phase hot-swap controller mp5990
+          - mps,mp5990
             # Honeywell Humidicon HIH-6130 humidity/temperature sensor
           - honeywell,hi6130
             # IBM Common Form Factor Power Supply Versions (all versions)
diff --git a/Documentation/devicetree/bindings/usb/usb-hcd.yaml b/Documentation/devicetree/bindings/usb/usb-hcd.yaml
index 692dd60e3f73..45a19d4928af 100644
--- a/Documentation/devicetree/bindings/usb/usb-hcd.yaml
+++ b/Documentation/devicetree/bindings/usb/usb-hcd.yaml
@@ -41,7 +41,7 @@ examples:
   - |
     usb {
         phys = <&usb2_phy1>, <&usb3_phy1>;
-        phy-names = "usb";
+        phy-names = "usb2", "usb3";
         #address-cells = <1>;
         #size-cells = <0>;
 
diff --git a/Documentation/devicetree/bindings/w1/amd,axi-1wire-host.yaml b/Documentation/devicetree/bindings/w1/amd,axi-1wire-host.yaml
new file mode 100644
index 000000000000..ef70fa2c0c5d
--- /dev/null
+++ b/Documentation/devicetree/bindings/w1/amd,axi-1wire-host.yaml
@@ -0,0 +1,44 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/w1/amd,axi-1wire-host.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: AMD AXI 1-wire bus host for programmable logic
+
+maintainers:
+  - Kris Chaplin <kris.chaplin@amd.com>
+
+properties:
+  compatible:
+    const: amd,axi-1wire-host
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - interrupts
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    onewire@a0000000 {
+        compatible = "amd,axi-1wire-host";
+        reg = <0xa0000000 0x10000>;
+        clocks = <&zynqmp_clk 0x47>;
+        interrupts = <GIC_SPI 0x59 IRQ_TYPE_LEVEL_HIGH>;
+    };
+
+...
diff --git a/Documentation/filesystems/erofs.rst b/Documentation/filesystems/erofs.rst
index 57c6ae23b3fc..cc4626d6ee4f 100644
--- a/Documentation/filesystems/erofs.rst
+++ b/Documentation/filesystems/erofs.rst
@@ -91,6 +91,10 @@ compatibility checking tool (fsck.erofs), and a debugging tool (dump.erofs):
 
 - git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git
 
+For more information, please also refer to the documentation site:
+
+- https://erofs.docs.kernel.org
+
 Bugs and patches are welcome, please kindly help us and send to the following
 linux-erofs mailing list:
 
diff --git a/Documentation/filesystems/fuse-io.rst b/Documentation/filesystems/fuse-io.rst
index 255a368fe534..6464de4266ad 100644
--- a/Documentation/filesystems/fuse-io.rst
+++ b/Documentation/filesystems/fuse-io.rst
@@ -15,7 +15,8 @@ The direct-io mode can be selected with the FOPEN_DIRECT_IO flag in the
 FUSE_OPEN reply.
 
 In direct-io mode the page cache is completely bypassed for reads and writes.
-No read-ahead takes place. Shared mmap is disabled.
+No read-ahead takes place. Shared mmap is disabled by default. To allow shared
+mmap, the FUSE_DIRECT_IO_ALLOW_MMAP flag may be enabled in the FUSE_INIT reply.
 
 In cached mode reads may be satisfied from the page cache, and data may be
 read-ahead by the kernel to fill the cache.  The cache is always kept consistent
diff --git a/Documentation/filesystems/locking.rst b/Documentation/filesystems/locking.rst
index 7be2900806c8..559857831e92 100644
--- a/Documentation/filesystems/locking.rst
+++ b/Documentation/filesystems/locking.rst
@@ -17,7 +17,8 @@ dentry_operations
 
 prototypes::
 
-	int (*d_revalidate)(struct dentry *, unsigned int);
+	int (*d_revalidate)(struct dentry *, const struct qstr *,
+			    unsigned int);
 	int (*d_weak_revalidate)(struct dentry *, unsigned int);
 	int (*d_hash)(const struct dentry *, struct qstr *);
 	int (*d_compare)(const struct dentry *,
@@ -261,7 +262,7 @@ prototypes::
 			struct folio *src, enum migrate_mode);
 	int (*launder_folio)(struct folio *);
 	bool (*is_partially_uptodate)(struct folio *, size_t from, size_t count);
-	int (*error_remove_page)(struct address_space *, struct page *);
+	int (*error_remove_folio)(struct address_space *, struct folio *);
 	int (*swap_activate)(struct swap_info_struct *sis, struct file *f, sector_t *span)
 	int (*swap_deactivate)(struct file *);
 	int (*swap_rw)(struct kiocb *iocb, struct iov_iter *iter);
@@ -287,7 +288,7 @@ direct_IO:
 migrate_folio:		yes (both)
 launder_folio:		yes
 is_partially_uptodate:	yes
-error_remove_page:	yes
+error_remove_folio:	yes
 swap_activate:		no
 swap_deactivate:	no
 swap_rw:		yes, unlocks
diff --git a/Documentation/filesystems/porting.rst b/Documentation/filesystems/porting.rst
index 878e72b2f8b7..ced3a6761329 100644
--- a/Documentation/filesystems/porting.rst
+++ b/Documentation/filesystems/porting.rst
@@ -1061,3 +1061,15 @@ export_operations ->encode_fh() no longer has a default implementation to
 encode FILEID_INO32_GEN* file handles.
 Filesystems that used the default implementation may use the generic helper
 generic_encode_ino32_fh() explicitly.
+
+---
+
+**recommended**
+
+Block device freezing and thawing have been moved to holder operations.
+
+Before this change, get_active_super() would only be able to find the
+superblock of the main block device, i.e., the one stored in sb->s_bdev. Block
+device freezing now works for any block device owned by a given superblock, not
+just the main block device. The get_active_super() helper and bd_fsfreeze_sb
+pointer are gone.
diff --git a/Documentation/filesystems/squashfs.rst b/Documentation/filesystems/squashfs.rst
index df42106bae71..4af8d6207509 100644
--- a/Documentation/filesystems/squashfs.rst
+++ b/Documentation/filesystems/squashfs.rst
@@ -64,6 +64,66 @@ obtained from this site also.
 The squashfs-tools development tree is now located on kernel.org
 	git://git.kernel.org/pub/scm/fs/squashfs/squashfs-tools.git
 
+2.1 Mount options
+-----------------
+===================    =========================================================
+errors=%s              Specify whether squashfs errors trigger a kernel panic
+                       or not
+
+		       ==========  =============================================
+                         continue  errors don't trigger a panic (default)
+                            panic  trigger a panic when errors are encountered,
+                                   similar to several other filesystems (e.g.
+                                   btrfs, ext4, f2fs, GFS2, jfs, ntfs, ubifs)
+
+                                   This allows a kernel dump to be saved,
+                                   useful for analyzing and debugging the
+                                   corruption.
+                       ==========  =============================================
+threads=%s             Select the decompression mode or the number of threads
+
+                       If SQUASHFS_CHOICE_DECOMP_BY_MOUNT is set:
+
+		       ==========  =============================================
+                           single  use single-threaded decompression (default)
+
+                                   Only one block (data or metadata) can be
+                                   decompressed at any one time. This limits
+                                   CPU and memory usage to a minimum, but it
+                                   also gives poor performance on parallel I/O
+                                   workloads when using multiple CPU machines
+                                   due to waiting on decompressor availability.
+                            multi  use up to two parallel decompressors per core
+
+                                   If you have a parallel I/O workload and your
+                                   system has enough memory, using this option
+                                   may improve overall I/O performance. It
+                                   dynamically allocates decompressors on a
+                                   demand basis.
+                           percpu  use a maximum of one decompressor per core
+
+                                   It uses percpu variables to ensure
+                                   decompression is load-balanced across the
+                                   cores.
+                        1|2|3|...  configure the number of threads used for
+                                   decompression
+
+                                   The upper limit is num_online_cpus() * 2.
+                       ==========  =============================================
+
+                       If SQUASHFS_CHOICE_DECOMP_BY_MOUNT is **not** set and
+                       SQUASHFS_DECOMP_MULTI, SQUASHFS_MOUNT_DECOMP_THREADS are
+                       both set:
+
+		       ==========  =============================================
+                          2|3|...  configure the number of threads used for
+                                   decompression
+
+                                   The upper limit is num_online_cpus() * 2.
+                       ==========  =============================================
+
+===================    =========================================================
+
 3. Squashfs Filesystem Design
 -----------------------------
 
diff --git a/Documentation/filesystems/vfs.rst b/Documentation/filesystems/vfs.rst
index 99acc2e98673..5c100d42586b 100644
--- a/Documentation/filesystems/vfs.rst
+++ b/Documentation/filesystems/vfs.rst
@@ -437,7 +437,7 @@ field.  This is a pointer to a "struct inode_operations" which describes
 the methods that can be performed on individual inodes.
 
 
-struct xattr_handlers
+struct xattr_handler
 ---------------------
 
 On filesystems that support extended attributes (xattrs), the s_xattr
@@ -823,7 +823,7 @@ cache in your filesystem.  The following members are defined:
 		bool (*is_partially_uptodate) (struct folio *, size_t from,
 					       size_t count);
 		void (*is_dirty_writeback)(struct folio *, bool *, bool *);
-		int (*error_remove_page) (struct mapping *mapping, struct page *page);
+		int (*error_remove_folio)(struct mapping *mapping, struct folio *);
 		int (*swap_activate)(struct swap_info_struct *sis, struct file *f, sector_t *span)
 		int (*swap_deactivate)(struct file *);
 		int (*swap_rw)(struct kiocb *iocb, struct iov_iter *iter);
@@ -1034,8 +1034,8 @@ cache in your filesystem.  The following members are defined:
 	VM if a folio should be treated as dirty or writeback for the
 	purposes of stalling.
 
-``error_remove_page``
-	normally set to generic_error_remove_page if truncation is ok
+``error_remove_folio``
+	normally set to generic_error_remove_folio if truncation is ok
 	for this address space.  Used for memory failure handling.
 	Setting this implies you deal with pages going away under you,
 	unless you have them locked or reference counts increased.
@@ -1252,7 +1252,8 @@ defined:
 .. code-block:: c
 
 	struct dentry_operations {
-		int (*d_revalidate)(struct dentry *, unsigned int);
+		int (*d_revalidate)(struct dentry *, const struct qstr *,
+				    unsigned int);
 		int (*d_weak_revalidate)(struct dentry *, unsigned int);
 		int (*d_hash)(const struct dentry *, struct qstr *);
 		int (*d_compare)(const struct dentry *,
@@ -1285,6 +1286,14 @@ defined:
 	they can change and, in d_inode case, even become NULL under
 	us).
 
+	d_revalidate also provides the name-under-lookup for cases where
+	there are particular filename encoding semantics to be handled
+	during revalidation.  Note that, if comparing with
+	dentry->d_name, the later can change from under d_revalidate, so
+	it must be protected with ->d_lock before accessing.  The
+	exception is when revalidating negative dentries for creation,
+	in which case the parent inode prevents it from changing.
+
 	If a situation is encountered that rcu-walk cannot handle,
 	return
 	-ECHILD and it will be called again in ref-walk mode.
diff --git a/Documentation/gpu/amdgpu/display/dc-debug.rst b/Documentation/gpu/amdgpu/display/dc-debug.rst
index 40c55a618918..817631b1dbf3 100644
--- a/Documentation/gpu/amdgpu/display/dc-debug.rst
+++ b/Documentation/gpu/amdgpu/display/dc-debug.rst
@@ -75,3 +75,44 @@ change in real-time by using something like::
 
 When reporting a bug related to DC, consider attaching this log before and
 after you reproduce the bug.
+
+DMUB Firmware Debug
+===================
+
+Sometimes, dmesg logs aren't enough. This is especially true if a feature is
+implemented primarily in DMUB firmware. In such cases, all we see in dmesg when
+an issue arises is some generic timeout error. So, to get more relevant
+information, we can trace DMUB commands by enabling the relevant bits in
+`amdgpu_dm_dmub_trace_mask`.
+
+Currently, we support the tracing of the following groups:
+
+Trace Groups
+------------
+
+.. csv-table::
+   :header-rows: 1
+   :widths: 1, 1
+   :file: ./trace-groups-table.csv
+
+**Note: Not all ASICs support all of the listed trace groups**
+
+So, to enable just PSR tracing you can use the following command::
+
+  # echo 0x8020 > /sys/kernel/debug/dri/0/amdgpu_dm_dmub_trace_mask
+
+Then, you need to enable logging trace events to the buffer, which you can do
+using the following::
+
+  # echo 1 > /sys/kernel/debug/dri/0/amdgpu_dm_dmcub_trace_event_en
+
+Lastly, after you are able to reproduce the issue you are trying to debug,
+you can disable tracing and read the trace log by using the following::
+
+  # echo 0 > /sys/kernel/debug/dri/0/amdgpu_dm_dmcub_trace_event_en
+  # cat /sys/kernel/debug/dri/0/amdgpu_dm_dmub_tracebuffer
+
+So, when reporting bugs related to features such as PSR and ABM, consider
+enabling the relevant bits in the mask before reproducing the issue and
+attach the log that you obtain from the trace buffer in any bug reports that you
+create.
diff --git a/Documentation/gpu/amdgpu/display/trace-groups-table.csv b/Documentation/gpu/amdgpu/display/trace-groups-table.csv
new file mode 100644
index 000000000000..3f6a50d1d883
--- /dev/null
+++ b/Documentation/gpu/amdgpu/display/trace-groups-table.csv
@@ -0,0 +1,29 @@
+Name, Mask Value
+INFO, 0x1
+IRQ SVC, 0x2
+VBIOS, 0x4
+REGISTER, 0x8
+PHY DBG, 0x10
+PSR, 0x20
+AUX, 0x40
+SMU, 0x80
+MALL, 0x100
+ABM, 0x200
+ALPM, 0x400
+TIMER, 0x800
+HW LOCK MGR, 0x1000
+INBOX1, 0x2000
+PHY SEQ, 0x4000
+PSR STATE, 0x8000
+ZSTATE, 0x10000
+TRANSMITTER CTL, 0x20000
+PANEL CNTL, 0x40000
+FAMS, 0x80000
+DPIA, 0x100000
+SUBVP, 0x200000
+INBOX0, 0x400000
+SDP, 0x4000000
+REPLAY, 0x8000000
+REPLAY RESIDENCY, 0x20000000
+CURSOR INFO, 0x80000000
+IPS, 0x100000000
diff --git a/Documentation/gpu/drm-kms-helpers.rst b/Documentation/gpu/drm-kms-helpers.rst
index b748b8ae70b2..59cfe8a7a8ba 100644
--- a/Documentation/gpu/drm-kms-helpers.rst
+++ b/Documentation/gpu/drm-kms-helpers.rst
@@ -363,6 +363,12 @@ EDID Helper Functions Reference
 .. kernel-doc:: drivers/gpu/drm/drm_edid.c
    :export:
 
+.. kernel-doc:: include/drm/drm_eld.h
+   :internal:
+
+.. kernel-doc:: drivers/gpu/drm/drm_eld.c
+   :export:
+
 SCDC Helper Functions Reference
 ===============================
 
diff --git a/Documentation/gpu/drm-mm.rst b/Documentation/gpu/drm-mm.rst
index 602010cb6894..acc5901ac840 100644
--- a/Documentation/gpu/drm-mm.rst
+++ b/Documentation/gpu/drm-mm.rst
@@ -552,6 +552,12 @@ Overview
 .. kernel-doc:: drivers/gpu/drm/scheduler/sched_main.c
    :doc: Overview
 
+Flow Control
+------------
+
+.. kernel-doc:: drivers/gpu/drm/scheduler/sched_main.c
+   :doc: Flow Control
+
 Scheduler Function References
 -----------------------------
 
diff --git a/Documentation/gpu/todo.rst b/Documentation/gpu/todo.rst
index 03fe5d1247be..b62c7fa0c2bc 100644
--- a/Documentation/gpu/todo.rst
+++ b/Documentation/gpu/todo.rst
@@ -621,6 +621,23 @@ Contact: Javier Martinez Canillas <javierm@redhat.com>
 
 Level: Intermediate
 
+Clean up and document former selftests suites
+---------------------------------------------
+
+Some KUnit test suites (drm_buddy, drm_cmdline_parser, drm_damage_helper,
+drm_format, drm_framebuffer, drm_dp_mst_helper, drm_mm, drm_plane_helper and
+drm_rect) are former selftests suites that have been converted over when KUnit
+was first introduced.
+
+These suites were fairly undocumented, and with different goals than what unit
+tests can be. Trying to identify what each test in these suites actually test
+for, whether that makes sense for a unit test, and either remove it if it
+doesn't or document it if it does would be of great help.
+
+Contact: Maxime Ripard <mripard@kernel.org>
+
+Level: Intermediate
+
 Enable trinity for DRM
 ----------------------
 
diff --git a/Documentation/hwmon/index.rst b/Documentation/hwmon/index.rst
index 72f4e6065bae..095c36f5e8a1 100644
--- a/Documentation/hwmon/index.rst
+++ b/Documentation/hwmon/index.rst
@@ -159,6 +159,7 @@ Hardware Monitoring Kernel Drivers
    mp2888
    mp2975
    mp5023
+   mp5990
    nct6683
    nct6775
    nct7802
diff --git a/Documentation/hwmon/mp5990.rst b/Documentation/hwmon/mp5990.rst
new file mode 100644
index 000000000000..6f2f0c099d44
--- /dev/null
+++ b/Documentation/hwmon/mp5990.rst
@@ -0,0 +1,84 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Kernel driver mp5990
+====================
+
+Supported chips:
+
+  * MPS MP5990
+
+    Prefix: 'mp5990'
+
+  * Datasheet
+
+    Publicly available at the MPS website : https://www.monolithicpower.com/en/mp5990.html
+
+Author:
+
+	Peter Yin <peteryin.openbmc@gmail.com>
+
+Description
+-----------
+
+This driver implements support for Monolithic Power Systems, Inc. (MPS)
+MP5990 Hot-Swap Controller.
+
+Device compliant with:
+
+- PMBus rev 1.3 interface.
+
+Device supports direct and linear format for reading input voltage,
+output voltage, output current, input power and temperature.
+
+The driver exports the following attributes via the 'sysfs' files
+for input voltage:
+
+**in1_input**
+
+**in1_label**
+
+**in1_max**
+
+**in1_max_alarm**
+
+**in1_min**
+
+**in1_min_alarm**
+
+The driver provides the following attributes for output voltage:
+
+**in2_input**
+
+**in2_label**
+
+**in2_alarm**
+
+The driver provides the following attributes for output current:
+
+**curr1_input**
+
+**curr1_label**
+
+**curr1_alarm**
+
+**curr1_max**
+
+The driver provides the following attributes for input power:
+
+**power1_input**
+
+**power1_label**
+
+**power1_alarm**
+
+The driver provides the following attributes for temperature:
+
+**temp1_input**
+
+**temp1_max**
+
+**temp1_max_alarm**
+
+**temp1_crit**
+
+**temp1_crit_alarm**
diff --git a/Documentation/mm/arch_pgtable_helpers.rst b/Documentation/mm/arch_pgtable_helpers.rst
index c82e3ee20e51..2466d3363af7 100644
--- a/Documentation/mm/arch_pgtable_helpers.rst
+++ b/Documentation/mm/arch_pgtable_helpers.rst
@@ -18,8 +18,6 @@ PTE Page Table Helpers
 +---------------------------+--------------------------------------------------+
 | pte_same                  | Tests whether both PTE entries are the same      |
 +---------------------------+--------------------------------------------------+
-| pte_bad                   | Tests a non-table mapped PTE                     |
-+---------------------------+--------------------------------------------------+
 | pte_present               | Tests a valid mapped PTE                         |
 +---------------------------+--------------------------------------------------+
 | pte_young                 | Tests a young PTE                                |
diff --git a/Documentation/netlink/specs/devlink.yaml b/Documentation/netlink/specs/devlink.yaml
index 572d83a414d0..43067e1f63aa 100644
--- a/Documentation/netlink/specs/devlink.yaml
+++ b/Documentation/netlink/specs/devlink.yaml
@@ -1484,8 +1484,8 @@ operations:
       dont-validate: [ strict ]
       flags: [ admin-perm ]
       do:
-        pre: devlink-nl-pre-doit
-        post: devlink-nl-post-doit
+        pre: devlink-nl-pre-doit-dev-lock
+        post: devlink-nl-post-doit-dev-lock
         request:
           attributes:
             - bus-name
diff --git a/Documentation/networking/device_drivers/ethernet/marvell/octeon_ep.rst b/Documentation/networking/device_drivers/ethernet/marvell/octeon_ep.rst
index cad96c8d1f97..613a818d5db6 100644
--- a/Documentation/networking/device_drivers/ethernet/marvell/octeon_ep.rst
+++ b/Documentation/networking/device_drivers/ethernet/marvell/octeon_ep.rst
@@ -24,6 +24,10 @@ Supported Devices
 Currently, this driver support following devices:
  * Network controller: Cavium, Inc. Device b200
  * Network controller: Cavium, Inc. Device b400
+ * Network controller: Cavium, Inc. Device b900
+ * Network controller: Cavium, Inc. Device ba00
+ * Network controller: Cavium, Inc. Device bc00
+ * Network controller: Cavium, Inc. Device bd00
 
 Interface Control
 =================
diff --git a/Documentation/networking/device_drivers/wifi/index.rst b/Documentation/networking/device_drivers/wifi/index.rst
index bf91a87c7acf..fb394f5de4a9 100644
--- a/Documentation/networking/device_drivers/wifi/index.rst
+++ b/Documentation/networking/device_drivers/wifi/index.rst
@@ -10,7 +10,6 @@ Contents:
 
    intel/ipw2100
    intel/ipw2200
-   ray_cs
 
 .. only::  subproject and html
 
diff --git a/Documentation/networking/device_drivers/wifi/ray_cs.rst b/Documentation/networking/device_drivers/wifi/ray_cs.rst
deleted file mode 100644
index 9a46d1ae8f20..000000000000
--- a/Documentation/networking/device_drivers/wifi/ray_cs.rst
+++ /dev/null
@@ -1,165 +0,0 @@
-.. SPDX-License-Identifier: GPL-2.0
-
-.. include:: <isonum.txt>
-
-=========================
-Raylink wireless LAN card
-=========================
-
-September 21, 1999
-
-Copyright |copy| 1998  Corey Thomas (corey@world.std.com)
-
-This file is the documentation for the Raylink Wireless LAN card driver for
-Linux.  The Raylink wireless LAN card is a PCMCIA card which provides IEEE
-802.11 compatible wireless network connectivity at 1 and 2 megabits/second.
-See http://www.raytheon.com/micro/raylink/ for more information on the Raylink
-card.  This driver is in early development and does have bugs.  See the known
-bugs and limitations at the end of this document for more information.
-This driver also works with WebGear's Aviator 2.4 and Aviator Pro
-wireless LAN cards.
-
-As of kernel 2.3.18, the ray_cs driver is part of the Linux kernel
-source.  My web page for the development of ray_cs is at
-http://web.ralinktech.com/ralink/Home/Support/Linux.html
-and I can be emailed at corey@world.std.com
-
-The kernel driver is based on ray_cs-1.62.tgz
-
-The driver at my web page is intended to be used as an add on to
-David Hinds pcmcia package.  All the command line parameters are
-available when compiled as a module.  When built into the kernel, only
-the essid= string parameter is available via the kernel command line.
-This will change after the method of sorting out parameters for all
-the PCMCIA drivers is agreed upon.  If you must have a built in driver
-with nondefault parameters, they can be edited in
-/usr/src/linux/drivers/net/pcmcia/ray_cs.c.  Searching for module_param
-will find them all.
-
-Information on card services is available at:
-
-	http://pcmcia-cs.sourceforge.net/
-
-
-Card services user programs are still required for PCMCIA devices.
-pcmcia-cs-3.1.1 or greater is required for the kernel version of
-the driver.
-
-Currently, ray_cs is not part of David Hinds card services package,
-so the following magic is required.
-
-At the end of the /etc/pcmcia/config.opts file, add the line:
-source ./ray_cs.opts
-This will make card services read the ray_cs.opts file
-when starting.  Create the file /etc/pcmcia/ray_cs.opts containing the
-following::
-
-  #### start of /etc/pcmcia/ray_cs.opts ###################
-  # Configuration options for Raylink Wireless LAN PCMCIA card
-  device "ray_cs"
-    class "network" module "misc/ray_cs"
-
-  card "RayLink PC Card WLAN Adapter"
-    manfid 0x01a6, 0x0000
-    bind "ray_cs"
-
-  module "misc/ray_cs" opts ""
-  #### end of /etc/pcmcia/ray_cs.opts #####################
-
-
-To join an existing network with
-different parameters, contact the network administrator for the
-configuration information, and edit /etc/pcmcia/ray_cs.opts.
-Add the parameters below between the empty quotes.
-
-Parameters for ray_cs driver which may be specified in ray_cs.opts:
-
-=============== =============== =============================================
-bc              integer         0 = normal mode (802.11 timing),
-				1 = slow down inter frame timing to allow
-				operation with older breezecom access
-				points.
-
-beacon_period	integer         beacon period in Kilo-microseconds,
-
-				legal values = must be integer multiple
-				of hop dwell
-
-				default = 256
-
-country         integer         1 = USA (default),
-				2 = Europe,
-				3 = Japan,
-				4 = Korea,
-				5 = Spain,
-				6 = France,
-				7 = Israel,
-				8 = Australia
-
-essid		string		ESS ID - network name to join
-
-				string with maximum length of 32 chars
-				default value = "ADHOC_ESSID"
-
-hop_dwell	integer         hop dwell time in Kilo-microseconds
-
-				legal values = 16,32,64,128(default),256
-
-irq_mask	integer         linux standard 16 bit value 1bit/IRQ
-
-				lsb is IRQ 0, bit 1 is IRQ 1 etc.
-				Used to restrict choice of IRQ's to use.
-				Recommended method for controlling
-				interrupts is in /etc/pcmcia/config.opts
-
-net_type	integer		0 (default) = adhoc network,
-				1 = infrastructure
-
-phy_addr	string          string containing new MAC address in
-				hex, must start with x eg
-				x00008f123456
-
-psm		integer         0 = continuously active,
-				1 = power save mode (not useful yet)
-
-pc_debug	integer		(0-5) larger values for more verbose
-				logging.  Replaces ray_debug.
-
-ray_debug	integer		Replaced with pc_debug
-
-ray_mem_speed   integer         defaults to 500
-
-sniffer         integer         0 = not sniffer (default),
-				1 = sniffer which can be used to record all
-				network traffic using tcpdump or similar,
-				but no normal network use is allowed.
-
-translate	integer		0 = no translation (encapsulate frames),
-				1 = translation    (RFC1042/802.1)
-=============== =============== =============================================
-
-More on sniffer mode:
-
-tcpdump does not understand 802.11 headers, so it can't
-interpret the contents, but it can record to a file.  This is only
-useful for debugging 802.11 lowlevel protocols that are not visible to
-linux.  If you want to watch ftp xfers, or do similar things, you
-don't need to use sniffer mode.  Also, some packet types are never
-sent up by the card, so you will never see them (ack, rts, cts, probe
-etc.)  There is a simple program (showcap) included in the ray_cs
-package which parses the 802.11 headers.
-
-Known Problems and missing features
-
-	Does not work with non x86
-
-	Does not work with SMP
-
-	Support for defragmenting frames is not yet debugged, and in
-	fact is known to not work.  I have never encountered a net set
-	up to fragment, but still, it should be fixed.
-
-	The ioctl support is incomplete.  The hardware address cannot be set
-	using ifconfig yet.  If a different hardware address is needed, it may
-	be set using the phy_addr parameter in ray_cs.opts.  This requires
-	a card insertion to take effect.
diff --git a/Documentation/sound/soc/dapm.rst b/Documentation/sound/soc/dapm.rst
index 8e44107933ab..c3154ce6e1b2 100644
--- a/Documentation/sound/soc/dapm.rst
+++ b/Documentation/sound/soc/dapm.rst
@@ -234,7 +234,7 @@ corresponding soft power control. In this case it is necessary to create
 a virtual widget - a widget with no control bits e.g.
 ::
 
-  SND_SOC_DAPM_MIXER("AC97 Mixer", SND_SOC_DAPM_NOPM, 0, 0, NULL, 0),
+  SND_SOC_DAPM_MIXER("AC97 Mixer", SND_SOC_NOPM, 0, 0, NULL, 0),
 
 This can be used to merge to signal paths together in software.
 
diff --git a/Documentation/trace/coresight/coresight.rst b/Documentation/trace/coresight/coresight.rst
index 4a71ea6cb390..d4f93d6a2d63 100644
--- a/Documentation/trace/coresight/coresight.rst
+++ b/Documentation/trace/coresight/coresight.rst
@@ -130,7 +130,7 @@ Misc:
 Device Tree Bindings
 --------------------
 
-See Documentation/devicetree/bindings/arm/arm,coresight-\*.yaml for details.
+See ``Documentation/devicetree/bindings/arm/arm,coresight-*.yaml`` for details.
 
 As of this writing drivers for ITM, STMs and CTIs are not provided but are
 expected to be added as the solution matures.
@@ -624,6 +624,10 @@ They are also listed in the folder /sys/bus/event_source/devices/cs_etm/format/
    * - timestamp
      - Session local version of the system wide setting: :ref:`ETMv4_MODE_TIMESTAMP
        <coresight-timestamp>`
+   * - cc_threshold
+     - Cycle count threshold value. If nothing is provided here or the provided value is 0, then the
+       default value i.e 0x100 will be used. If provided value is less than minimum cycles threshold
+       value, as indicated via TRCIDR3.CCITMIN, then the minimum value will be used instead.
 
 How to use the STM module
 -------------------------
diff --git a/Documentation/trace/ftrace.rst b/Documentation/trace/ftrace.rst
index 23572f6697c0..9ef362dae9df 100644
--- a/Documentation/trace/ftrace.rst
+++ b/Documentation/trace/ftrace.rst
@@ -2574,7 +2574,7 @@ want, depending on your needs.
 
 - The cpu number on which the function executed is default
   enabled.  It is sometimes better to only trace one cpu (see
-  tracing_cpu_mask file) or you might sometimes see unordered
+  tracing_cpumask file) or you might sometimes see unordered
   function calls while cpu tracing switch.
 
 	- hide: echo nofuncgraph-cpu > trace_options
diff --git a/Documentation/translations/sp_SP/process/handling-regressions.rst b/Documentation/translations/sp_SP/process/handling-regressions.rst
new file mode 100644
index 000000000000..aa0988985c55
--- /dev/null
+++ b/Documentation/translations/sp_SP/process/handling-regressions.rst
@@ -0,0 +1,797 @@
+.. include:: ../disclaimer-sp.rst
+
+:Translator: Sergio González Collado <sergio.collado@gmail.com>
+
+.. _sp_handling_regressions:
+
+Gestión de regresiones
+++++++++++++++++++++++
+
+*No causamos regresiones* -- este documento describe la que es la "primera
+regla del desarrollo del kernel de Linux" y que implica en la práctica para
+los desarrolladores. Y complementa la documentación:
+Documentation/admin-guide/reporting-regressions.rst, que cubre el tema
+desde el punto de vista de un usuario; si nunca ha leído ese texto, realice
+al menos una lectura rápida del mismo antes de continuar.
+
+Las partes importantes (el "TL;DR")
+===================================
+
+#.  Asegúrese de que los suscriptores a la lista `regression mailing list
+    <https://lore.kernel.org/regressions/>`_ (regressions@lists.linux.dev)
+    son conocedores con rapidez de cualquier nuevo informe de regresión:
+
+    * Cuando se reciba un correo que no incluyó a la lista, inclúyalo en la
+      conversación de los correos, mandando un breve "Reply-all" con la
+      lista en CCed.
+
+    * Mande o redirija cualquier informe originado en los gestores de bugs
+      a la lista.
+
+#. Haga que el bot del kernel de Linux "regzbot" realice el seguimiento del
+   incidente (esto es opcional, pero recomendado).
+
+    * Para reportes enviados por correo, verificar si contiene alguna línea
+      como ``#regzbot introduced v5.13..v5.14-rc1``. Si no, mandar una
+      respuesta (con la lista de regresiones en CC) que contenga un párrafo
+      como el siguiente, lo que le indica a regzbot cuando empezó a suceder
+      el incidente::
+
+       #regzbot ^introduced 1f2e3d4c5b6a
+
+    * Cuando se mandan informes desde un gestor de incidentes a la lista de
+      regresiones(ver más arriba), incluir un párrafo como el siguiente::
+
+       #regzbot introduced: v5.13..v5.14-rc1
+       #regzbot from: Some N. Ice Human <some.human@example.com>
+       #regzbot monitor: http://some.bugtracker.example.com/ticket?id=123456789
+
+#. Cuando se manden correcciones para las regresiones, añadir etiquetas
+   "Link:" a la descripción, apuntado a todos los sitios donde se informó
+   del incidente, como se indica en el documento:
+   Documentation/process/submitting-patches.rst y
+   :ref:`Documentation/process/5.Posting.rst <development_posting>`.
+
+#. Intente arreglar las regresiones rápidamente una vez la causa haya sido
+   identificada; las correcciones para la mayor parte de las regresiones
+   deberían ser integradas en menos de dos semanas, pero algunas pueden
+   resolverse en dos o tres días.
+
+Detalles importantes para desarrolladores en la regresiones de kernel de Linux
+==============================================================================
+
+Puntos básicos importantes más en detalle
+-----------------------------------------
+
+Qué hacer cuando se recibe un aviso de regresión.
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Asegúrese de que el programa de gestión de regresiones del kernel de Linux
+y los subscritos a la lista de correo `regression mailing list
+<https://lore.kernel.org/regressions/>`_ (regressions@lists.linux.dev) son
+conocedores de cualquier nuevo informe de regresión:
+
+ * Cuando se recibe un informe por email que no tiene en CC la lista,
+   inmediatamente meterla en el la cadena de emails mandado al menos un
+   breve "Reply-all" con la lista en CC; Intentar asegurar que la lista es
+   añadida en CC de nuevo en caso de que alguna respuesta la omita de la
+   lista.
+
+ * Si un informe enviado a un gestor de defectos, llega a su correo,
+   reenvíelo o redirijalo a la lista. Considere verificar los archivos de
+   la lista de antemano, si la persona que lo ha informado, lo ha enviado
+   anteriormente, como se indica en:
+   Documentation/admin-guide/reporting-issues.rst.
+
+Cuando se realice cualquiera de las acciones anteriores, considere
+inmediatamente iniciar el seguimiento de la regresión con "regzbot" el
+gestor de regresiones del kernel de Linux.
+
+ * Para los informes enviados por email, verificar si se ha incluido un
+   comando a "regzbot", como ``#regzbot introduced 1f2e3d4c5b6a``. Si no es
+   así, envíe una respuesta (con la lista de regresiones en CC) con un
+   párrafo como el siguiente::
+
+       #regzbot ^introduced: v5.13..v5.14-rc1
+
+   Esto indica a regzbot el rango de versiones en el cual es defecto
+   comenzó a suceder; Puede especificar un rango usando los identificadores
+   de los commits así como un único commit, en caso en el que el informante
+   haya identificado el commit causante con 'bisect'.
+
+   Tenga en cuenta que el acento circunflejo (^) antes de "introduced":
+   Esto indica a regzbot, que debe tratar el email padre (el que ha sido
+   respondido) como el informeinicial para la regresión que quiere ser
+   seguida. Esto es importante, ya que regzbot buscará más tarde parches
+   con etiquetas "Link:" que apunten al al informe de losarchivos de
+   lore.kernel.org.
+
+ * Cuando mande informes de regresiones a un gestor de defectos, incluya un
+   párrafo con los siguientes comandos a regzbot::
+
+       #regzbot introduced: 1f2e3d4c5b6a
+       #regzbot from: Some N. Ice Human <some.human@example.com>
+       #regzbot monitor: http://some.bugtracker.example.com/ticket?id=123456789
+
+   Regzbot asociará automáticamente parches con el informe que contengan
+   las etiquetas "Link:" apuntando a su email o el ticket indicado.
+
+Qué es importante cuando se corrigen regresiones
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+No se necesita hacer nada especial cuando se mandan las correcciones para
+las regresiones únicamente recordar lo que se explica en los documentos:
+Documentation/process/submitting-patches.rst,
+:ref:`Documentation/process/5.Posting.rst <development_posting>`, y
+Documentation/process/stable-kernel-rules.rst
+
+ * Apunte a todos los lugares donde el incidente se reportó usando la
+   etiqueta "Link:" ::
+
+       Link: https://lore.kernel.org/r/30th.anniversary.repost@klaava.Helsinki.FI/
+       Link: https://bugzilla.kernel.org/show_bug.cgi?id=1234567890
+
+ * Añada la etiqueta "Fixes:" para indicar el commit causante de la
+   regresión.
+
+ * Si el culpable ha sido "mergeado" en un ciclo de desarrollo anterior,
+   marque explícitamente el fix para retro-importarlo usando la etiqueta
+   ``Cc: stable@vger.kernel.org`` tag.
+
+Todo esto se espera y es importante en una regresión, ya que estas
+etiquetas son de gran valor para todos (incluido usted) que pueda estar
+mirando en ese incidente semanas, meses o años después. Estas etiquetas son
+también cruciales para las herramientas y scripts usados por otros
+desarrolladores del kernel o distribuciones de Linux; una de esas
+herramientas es regzbot, el cual depende mucho de las etiquetas "Link:"
+para asociar los informes por regresiones con los cambios que las
+resuelven.
+
+
+Priorización del trabajo en arreglar regresiones
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Al final, los desarrolladores deberían hacer lo posible para evitar a los
+usuarios situaciones donde una regresión les deje solo tres opciones:
+
+ * Ejecutar el kernel con una regresión que afecta seriamente al uso.
+
+ * Cambiar a un kernel nuevo o mas antiguo -- rebajarse a una versión
+   soportada del kernel que no tenga las funcionalidades requeridas.
+
+ * Continuar ejecutando una versión desfasada y potencialmente insegura del
+   kernel por más de dos semanas después de que el causante de una regresión
+   fuese identificado.
+
+Cómo se ejecuta esto depende mucho de la situación. A continuación se
+presentan unas reglas generales, en orden de importancia:
+
+ * Priorice el trabajo en la gestión de los informes de la regresión y
+   arreglar la regresión por encima de cualquier otro trabajo en el kernel
+   de Linux, a menos que lo último afecte profundamente a efectos de
+   seguridad, o cause errores en los que haya pérdida o daño de datos.
+
+ * Considere siempre revertir los commits responsables y re-aplicarlos
+   después, junto con las correcciones necesarias, ya que esto puede la
+   forma menos peligrosa y más rápida de arreglar la regresión.
+
+ * Los desarrolladores deberían gestionar la regresión en todos los kernels
+   soportados de la serie, pero son libres de delegar el trabajo al equipo
+   permanente el incidente no hubiese ocurrido en la línea principal.
+
+ * Intente resolver cualquier regresión que apareciera en el ciclo de
+   desarrollo antes de que este acabe. Si se teme que una corrección
+   pudiera ser demasiado arriesgada para aplicarla días antes de una
+   liberación de la línea principal de desarrollo, dejar decidir a Linus:
+   mande la corrección a él de forma separada, tan pronto como sea posible
+   con una explicación de la situación. El podrá decidir, y posponer la
+   liberación si fuese necesario, por ejemplo si aparecieran múltiples
+   cambios como ese.
+
+ * Gestione las regresiones en la rama estable, de largo término, o la
+   propia rama principal de las versiones, con más urgencia que la
+   regresiones en las preliberaciones. Esto cambia después de la liberación
+   de la quinta pre-liberación, aka "-rc5": la rama principal entonces se
+   vuelve más importante, asegurar que todas las mejoras y correcciones son
+   idealmente testeados juntos por al menos una semana antes de que Linux
+   libere la nueva versión en la rama principal.
+
+ * Intente arreglar regresiones en un intervalo de una semana después de
+   que se ha identificado el responsable, si el incidente fue introducido
+   en alguno de los siguientes casos:
+
+    * una versión estable/largo-plazo reciente
+
+    * en el último ciclo de desarrollo de la rama principal
+
+   En el último caso (por ejemplo v5.14), intentar gestionar las
+   regresiones incluso más rápido, si la versión estable precedente (v5.13)
+   ha de ser abandonada pronto o ya se ha etiquetado como de final de vida
+   (EOL de las siglas en inglés End-of-Life) -- esto sucede usualmente
+   sobre tres o cuatro semanas después de una liberación de una versión en
+   la rama principal.
+
+ * Intente arreglar cualquier otra regresión en un periodo de dos semanas
+   después de que el culpable haya sido identificado. Dos o tres semanas
+   adicionales son aceptables para regresiones de rendimiento y otros
+   incidentes que son molestos, pero no bloquean a nadie la ejecución de
+   Linux (a menos que se un incidente en el ciclo de desarrollo actual, en
+   ese caso se debería gestionar antes de la liberación de la versión).
+   Unas semanas son aceptables si la regresión únicamente puede ser
+   arreglada con un cambio arriesgado y al mismo tiempo únicamente afecta a
+   unos pocos usuarios; también está bien si se usa tanto tiempo como fuera
+   necesario si la regresión está presente en la segunda versión más nueva
+   de largo plazo del kernel.
+
+Nota: Los intervalos de tiempo mencionados anteriormente para la resolución
+de las regresiones, incluyen la verificación de esta, revisión e inclusión
+en la rama principal, idealmente con la corrección incluida en la rama
+"linux-next" al menos brevemente. Esto conllevará retrasos que también se
+tienen tener en cuenta.
+
+Se espera que los maintainers de los subsistemas, ayuden en conseguir esos
+tiempos, haciendo revisiones con prontitud y gestionando con rapidez los
+parches aceptados. Esto puede resultar en tener que mandar peticiones de
+git-pull antes o de forma más frecuente que lo normal; dependiendo del
+arreglo, podría incluso ser aceptable saltarse la verificación en
+linux-next. Especialmente para las correcciones en las ramas de los kernels
+estable y de largo plazo necesitan ser gestionadas rápidamente, y las
+correcciones necesitan ser incluidas en la rama principal antes de que
+puedan ser incluidas posteriormente a las series precedentes.
+
+
+Más aspectos sobre regresiones que los desarrolladores deben saber
+------------------------------------------------------------------
+
+Cómo tratar con cambios donde se sabe que hay riesgo de regresión
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Evalué cómo de grande es el riesgo de una regresión, por ejemplo realizando
+una búsqueda en las distribuciones de linux y en Git forges. Considere
+también preguntar a otros desarrolladores o proyectos que pudieran ser
+afectados para evaluar o incluso testear el cambio propuesto; si
+apareciesen problemas, quizás se pudiera encontrar una solución aceptable
+para todos.
+
+Si al final, el riesgo de la regresión parece ser relativamente pequeño,
+entonces adelante con el cambio, pero siempre informe a todas las partes
+involucradas del posible riesgo. Por tanto, asegúrese de que la descripción
+del parche, se hace explícito este hecho. Una vez el cambio ha sido
+integrado, informe al gestor de regresiones de Linux y a las listas de
+correo de regresiones sobre el riesgo, de manera que cualquiera que tenga
+el cambio en el radar, en el caso de que aparezcan reportes. Dependiendo
+del riesgo, quizás se quiera preguntar al mantenedor del subsistema, que
+mencione el hecho en su línea principal de desarrollo.
+
+¿Qué más hay que saber sobre regresiones?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Repase la documentación: Documentation/admin-guide/reporting-regressions.rst,
+esta cubre otros aspectos a tener a en cuenta y conocer:
+
+ * la finalidad de la "regla de no regresión"
+
+ * qué incidencias no se califican como regresión
+
+ * quién es el responsable de identificar la causa raíz de una regresión
+
+ * cómo gestionar situaciones difíciles, como por ejemplo cuando una
+   regresión es causada por una corrección de seguridad o cuando una
+   regresión causa otra regresión
+
+A quién preguntar por consejo cuando se trata de regresiones
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Mande un email a la lista de correo de regresiones
+(regressions@lists.linux.dev) y CC al seguidor de regresiones del kernel de
+Linux (regressions@leemhuis.info); Si el incidente pudiera ser mejor
+gestionarlo en privado, puede omitirse la lista.
+
+
+Más sobre la gestión de regresiones con regzbot
+-----------------------------------------------
+
+¿Por qué el kernel de Linux tiene un gestor de regresiones, y por qué se usa regzbot?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Reglas como "no regresiones" necesitan asegurar que se cumplen, de otro
+modo se romperían accidentalmente o a propósito. La historia ha mostrado
+que esto es verdad también para el kernel de Linux. Esto es por lo que
+Thorsten Leemhuis se ofreció como voluntario para dar una solución a esto,
+con el gestor de regresiones del kernel de Linux. A nadie se le paga por
+hacer esto, y esa es la razón por la gestión de regresiones es un servicio
+con el "mejor esfuerzo".
+
+Intentos iniciales de gestionar manualmente las regresiones han demostrado
+que es una tarea extenuante y frustrante, y por esa razón se dejaron de
+hacer después de un tiempo. Para evitar que volviese a suceder esto,
+Thorsten desarrollo regbot para facilitar el trabajo, con el objetivo a
+largo plazo de automatizar la gestión de regresiones tanto como fuese
+posible para cualquiera que estuviese involucrado.
+
+¿Cómo funciona el seguimiento de regresiones con regzbot?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+El bot monitoriza las respuestas de los informes de las regresiones
+identificadas. Adicionalmente mira si se han publicado o enviado parches
+que hagan referencia a esos informes con la etiqueta: "Link:"; respuestas a
+esos parches también se siguen. Combinando esta información, también
+proporciona una buena imagen del estado actual del proceso de corrección.
+
+Regzbot intenta hacer todo este trabajo con tan poco retraso como sea
+posible tanto para la gente que lo reporta, como para los desarrolladores.
+De hecho, solo los informantes son requeridos para una tarea adicional:
+necesitan informar a regzbot con el comando ``#regzbot introduced``
+indicado anteriormente; si no hacen esto, alguien más puede hacerlo usando
+``#regzbot ^introduced``.
+
+Para los desarrolladores normalmente no hay un trabajo adicional que
+realizar, únicamente necesitan asegurarse una cosa, que ya se hacía mucho
+antes de que regzbot apareciera: añadir las etiquetas "Link:" a la
+descripción del parche apuntando a todos los informes sobre el error
+corregido.
+
+¿Tengo que usar regzbot?
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Hacerlo es por el bien de todo el mundo, tanto los mantenedores del kernel,
+como Linus Torvalds dependen parcialmente en regzbot para seguir su trabajo
+-- por ejemplo cuando deciden liberar una nueva versión o ampliar la fase de
+desarrollo. Para esto necesitan conocer todas las regresiones que están sin
+corregir; para esto, es conocido que Linux mira los informes semanales que
+manda regzbot.
+
+¿He de informar a regzbot cada regresión que encuentre?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Idealmente, sí: todos somos humanos y olvidamos fácilmente los problemas
+cuando algo más importante aparece inesperadamente -- por ejemplo un
+problema mayor en el kernel de Linux o algo en la vida real que nos mantenga
+alejados de los teclados por un tiempo. Por eso es mejor informar a regzbot
+sobre cada regresión, excepto cuando inmediatamente escribimos un parche y
+los mandamos al árbol de desarrollo en el que se integran habitualmente a
+la serie del kernel.
+
+¿Cómo ver qué regresiones esta siguiendo regbot actualmente?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Verifique el `interfaz web de regzbot <https://linux-regtracking.leemhuis.info/regzbot/>`_
+para ver la última información; o `busque el último informe de regresiones
+<https://lore.kernel.org/lkml/?q=%22Linux+regressions+report%22+f%3Aregzbot>`_,
+el cual suele ser enviado por regzbot una vez a la semana el domingo por la
+noche (UTC), lo cual es unas horas antes de que Linus normalmente anuncie
+las "(pre-)releases".
+
+¿Qué sitios supervisa regzbot?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Regzbot supervisa las listas de correo más importantes de Linux, como
+también las de los repositorios linux-next, mainline y stable/longterm.
+
+
+¿Qué tipos de incidentes han de ser monitorizados por regzbot?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+El bot debe hacer seguimiento de las regresiones, y por tanto por favor,
+no involucre a regzbot para incidencias normales. Pero es correcto para
+el gestor de incidencias de kernel de Linux, monitorizar incidentes
+graves, como informes sobre cuelgues, corrupción de datos o errores
+internos (Panic, Oops, BUG(), warning, ...).
+
+
+¿Puedo añadir una regresión detectada por un sistema de CI al seguimiento de regzbot?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Siéntase libre de hacerlo, si la regresión en concreto puede tener un
+impacto en casos de uso prácticos y por tanto ser detectado por los usuarios;
+Así, por favor no involucre a regzbot en regresiones teóricas que
+difícilmente pudieran manifestarse en un uso real.
+
+¿Cómo interactuar con regzbot?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Usando el comando 'regzbot' en una respuesta directa o indirecta al correo
+con el informe de regresión. Ese comando necesita estar en su propio
+párrafo (debe estar separado del resto del texto usando líneas en blanco):
+
+Por ejemplo ``#regzbot introduced <version or commit>``, que hace que regzbot
+considere el correo como un informe de regressión que se ha de añadir al
+seguimiento, como se ha descrito anteriormente; ``#regzbot ^introduced <version or commit>``
+es otro ejemplo del comando, el cual indica a regzbot que considere el email
+anterior como el informe de una regresión que se ha de comenzar a monitorizar.
+
+Una vez uno de esos dos comandos se ha utilizado, se pueden usar otros
+comandos regzbot en respuestas directas o indirectas al informe. Puede
+escribirlos debajo de uno de los comandos anteriormente usados o en las
+respuestas al correo en el que se uso como respuesta a ese correo:
+
+ * Definir o actualizar el título::
+
+       #regzbot title: foo
+
+ * Monitorizar una discusión o un tiquet de bugzilla.kernel.org donde
+   aspectos adicionales del incidente o de la corrección se están
+   comentando -- por ejemplo presentar un parche que corrige la regresión::
+
+       #regzbot monitor: https://lore.kernel.org/all/30th.anniversary.repost@klaava.Helsinki.FI/
+
+  Monitorizar solamente funciona para lore.kernel.org y bugzilla.kernel.org;
+  regzbot considerará todos los mensajes en ese hilo o el tiquet como
+  relacionados al proceso de corrección.
+
+ * Indicar a un lugar donde más detalles de interés, como un mensaje en una
+   lista de correo o un tiquet en un gestor de incidencias que pueden estar
+   levemente relacionados, pero con un tema diferente::
+
+       #regzbot link: https://bugzilla.kernel.org/show_bug.cgi?id=123456789
+
+ * Identificar una regresión como corregida por un commit que se ha mandado
+   aguas arriba o se ha publicado::
+
+        #regzbot fixed-by: 1f2e3d4c5d
+
+
+ * Identificar una regresión como un duplicado de otra que ya es seguida
+   por regzbot::
+
+        #regzbot dup-of: https://lore.kernel.org/all/30th.anniversary.repost@klaava.Helsinki.FI/
+
+ * Identificar una regresión como inválida::
+
+       #regzbot invalid: wasn't a regression, problem has always existed
+
+
+¿Algo más que decir sobre regzbot y sus comandos?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Hay información más detallada y actualizada sobre el bot de seguimiento de
+regresiones del kernel de Linux en: `project page <https://gitlab.com/knurd42/regzbot>`_,
+y entre otros contiene una  `guia de inicio <https://gitlab.com/knurd42/regzbot/-/blob/main/docs/getting_started.md>`_
+y `documentación de referencia <https://gitlab.com/knurd42/regzbot/-/blob/main/docs/reference.md>`_
+Ambos contienen más detalles que las secciones anteriores.
+
+
+Citas de Linus sobre regresiones
+--------------------------------
+
+A continuación se encuentran unos ejemplos reales (traducidos) de como
+Linus Torvalds espera que se gestionen las regresiones:
+
+
+ * De 2017-10-26 (1/2)
+   <https://lore.kernel.org/lkml/CA+55aFwiiQYJ+YoLKCXjN_beDVfu38mg=Ggg5LFOcqHE8Qi7Zw@mail.gmail.com/>`_::
+
+     Si rompes la configuración de los espacios de usuario ESO ES UNA REGRESIÓN.
+
+     No está bien decir "pero nosotros arreglaremos la configuración del espacio
+     de usuario".
+
+     Realmente. NO ESTÁ BIEN.
+
+     [...]
+
+     La primera regla es:
+
+     - no causamos regresiones
+
+     y el corolario es que cuando una regresión pasa, lo admitimos y lo
+     arreglamos, en vez de echar la culpa al espacio de usuario.
+
+     El hecho de que aparentemente se haya negado la regresión durante
+     tres semanas, significa que lo revertiré y dejaré de integrar peticiones
+     de apparmor hasta que la gente involucrada entienda como se hace
+     el desarrollo del kernel.
+
+
+ * De `2017-10-26 (2/2)
+   <https://lore.kernel.org/lkml/CA+55aFxW7NMAMvYhkvz1UPbUTUJewRt6Yb51QAx5RtrWOwjebg@mail.gmail.com/>`_::
+
+       La gente debería sentirse libre de actualizar su kernel y simplemente
+       no preocuparse por ello.
+
+       Me niego a imponer una limitación del tipo "solo puede actualizar
+       el kernel si actualiza otro programa". Si el kernel trabaja para tí,
+       la regla es que continúe trabajando para tí.
+
+       Ha habido algunas excepciones, pero son pocas y separadas entre sí, y
+       generalmente tienen una razón fundamental para haber sucedido, que era
+       básicamente inevitable, y la gente intentó evitarlas por todos los
+       medios. Quizás no podamos mantener el hardware más, después de que han
+       pasado décadas y nadie los usacon kernel modernos. Quizás haya un
+       problema de seguridad serio con cómo hicimos las cosas, y la gente
+       depende de un modelo fundamentalmente roto. Quizás haya algún otro roto
+       fundamental, que tenga que tener una _flag_ y por razones internas y
+       fundamentales.
+
+       Y nótese que esto trata sobre *romper* los entornos de la gente.
+
+       Cambios de comportamiento pasan, y quizás no se mantengan algunas
+       funcionalidades más. Hay un número de campos en /proc/<pid>/stat que
+       se imprimen como ceros, simplemente porque ni siquiera existen ya en
+       kernel, o porque mostrarlos era un error (típica una fuga de
+       información). Pero los números se sustituyeron por ceros, así que
+       el código que se usaba para parsear esos campos todavía existe. El
+       usuario puede no ver todo lo que podía ver antes, y por eso el
+       omportamiento es claramente diferente, pero las cosas todavía
+       _funcionan_, incluso si no se puede mostrar información sensible
+       (o que no es ya importante).
+
+       Pero si algo realmente se rompe, entonces el cambio debe de arreglarse
+       o revertirse. Y se arregla en el *kernel*. No diciendo "bueno, arreglaremos
+       tu espacio de usuario". Ha sido un cambio en el kernel el que creo
+       el problema, entonces ha de ser el kernel el que lo corrija, porque
+       tenemos un modelo de "actualización". Pero no tenemos una "actualización
+       con el nuevo espacio de usuario".
+
+       Y yo seriamente me negaré a coger código de gente que no entiende y
+       honre esta sencilla regla.
+
+       Y esta regla no va a cambiar.
+
+       Y sí, me doy cuenta que el kernel es "especial" en este respecto. Y
+       estoy orgulloso de ello.
+
+       Y he visto, y puedo señalar, muchos proyectos que dicen "Tenemos que
+       romper ese caso de uso para poder hacer progresos" o "estabas basandote
+       en comportamientos no documentados, debe ser duro ser tú" o "hay una
+       forma mejor de hacer lo que quieres hacer, y tienes que cambiar a esa
+       nueva forma", y yo simplemente no pienso que eso sea aceptable fuera
+       de una fase alfa muy temprana que tenga usuarios experimentales que
+       saben a lo que se han apuntado. El kernel no ha estado en esta
+       situación en las dos últimas décadas.
+
+       Nosotros rompemos la API _dentro_ del kernel todo el tiempo. Y
+       arreglaremos los problemas internos diciendo "tú ahora necesitas
+       hacer XYZ", pero entonces es sobre la API interna del kernel y la
+       gente que hace esto entonces tendrá obviamente que arreglar todos
+       los usos de esa API del kernel. Nadie puede decir "ahora, yo he roto
+       la API que usas, y ahora tú necesitas arreglarlo". Quién rompa algo,
+       lo arregla también.
+
+       Y nosotros, simplemente, no rompemos el espacio de usuario.
+
+ * De `2020-05-21
+   <https://lore.kernel.org/all/CAHk-=wiVi7mSrsMP=fLXQrXK_UimybW=ziLOwSzFTtoXUacWVQ@mail.gmail.com/>`_::
+
+       Las reglas sobre regresiones nunca han sido sobre ningún tipo de
+       comportamiento documentado, o dónde está situado el código.
+
+       Las reglas sobre regresiones son siempre sobre "roturas en el
+       flujo de trabajo del usuario".
+
+       Los usuarios son literalmente la _única_ cosa que importa.
+
+       Argumentaciones como "no debería haber usado esto" o "ese
+       comportamiento es indefinido, es su culpa que su aplicación no
+       funcione" o "eso solía funcionar únicamente por un bug del kernel" son
+       irrelevantes.
+
+       Ahora, la realidad nunca es blanca o negra. Así hemos tenido situaciones
+       como "un serio incidente de seguridad" etc que solamente nos fuerza
+       a hacer cambios que pueden romper el espacio de usuario. Pero incluso
+       entonces la regla es que realmente no hay otras opciones para que
+       las cosas sigan funcionando.
+
+       Y obviamente, si los usuarios tardan años en darse cuenta que algo
+       se ha roto, o si hay formas adecuadas para sortear la rotura que
+       no causen muchos problemas para los usuarios (por ejemplo: "hay un
+       puñado de usuarios, y estos pueden usar la línea de comandos del
+       kernel para evitarlos"; ese tipo de casos), en esos casos se ha sido
+       un poco menos estricto.
+
+       Pero no, "eso que está documentado que está roto" (si es dado a que
+       el código estaba en preparación o porque el manual dice otra cosa) eso
+       es irrelevante. Si preparar el código es tan útil que la gente,
+       acaba usando, esto implica que básicamente es código del kernel con
+       una señal diciendo "por favor limpiar esto".
+
+       El otro lado de la moneda es que la gente que habla sobre "estabilidad
+       de las APIs" están totalmente equivocados. Las APIs tampoco importan.
+       Se puede hacer cualquier cambio que se quiera a una API ... siempre y
+       cuando nadie se de cuenta.
+
+       De nuevo, la regla de las regresiones no trata sobre la documentación,
+       tampoco sobre las APIs y tampoco sobre las fases de la Luna.
+
+       Únicamente trata sobre "hemos causado problemas al espacio de usuario que
+       antes funcionaba".
+
+ * De `2017-11-05
+   <https://lore.kernel.org/all/CA+55aFzUvbGjD8nQ-+3oiMBx14c_6zOj2n7KLN3UsJ-qsd4Dcw@mail.gmail.com/>`_::
+
+       Y nuestra regla sobre las regresiones nunca ha sido "el comportamiento
+       no cambia". Eso podría significar que nunca podríamos hacer ningún
+       cambio.
+
+       Por ejemplo, hacemos cosas como añadir una nueva gestión de
+       errores etc todo el tiempo, con lo cual a veces incluso añadimos
+       tests en el directorio de kselftest.
+
+       Así que claramente cambia el comportamiento todo el tiempo y
+       nosotros no consideramos eso una regresión per se.
+
+       La regla para regresiones para el kernel es para cuando se
+       rompe algo en el espacio de usuario. No en algún test. No en
+       "mira, antes podía hacer X, y ahora no puedo".
+
+ * De `2018-08-03
+   <https://lore.kernel.org/all/CA+55aFwWZX=CXmWDTkDGb36kf12XmTehmQjbiMPCqCRG2hi9kw@mail.gmail.com/>`_::
+
+       ESTÁS OLVIDANDO LA REGLA #1 DEL KERNEL.
+
+       No hacemos regresiones, y no hacemos regresiones porque estás 100%
+       equivocado.
+
+       Y la razón que apuntas en tú opinión es exactamente *PORQUÉ* estás
+       equivocado.
+
+       Tus "buenas razones" son honradas y pura basura.
+
+       El punto de "no hacemos regresiones" es para que la gente pueda
+       actualizar el kernel y nunca tengan que preocuparse por ello.
+
+       > El kernel tiene un bug que ha de ser arreglado
+
+       Eso es *TOTALMENTE* insustancial.
+
+       Chicos, si algo estaba roto o no, NO IMPORTA.
+
+       ¿Porqué?
+
+       Los errores pasan. Eso es un hecho de la vida. Discutir que
+       "tenemos que romper algo porque estábamos arreglando un error" es
+       una locura. Arreglamos decenas de errores cada dia, pensando que
+       "arreglando un bug" significa que podemos romper otra cosa es algo
+       que simplemente NO ES VERDAD.
+
+       Así que los bugs no son realmente relevantes para la discusión. Estos
+       suceden y se detectan, se arreglan, y no tienen nada que ver con
+       "rompemos a los usuarios".
+
+       Porque la única cosa que importa ES EL USUARIO.
+
+       ¿Cómo de complicado es eso de comprender?
+
+       Cualquier persona que use "pero no funcionaba correctamente" es
+       un argumento no tiene la razón. Con respecto al USUARIO, no era
+       erróneo - funcionaba para él/ella.
+
+       Quizás funcionaba *porque* el usuario había tenido el bug en cuenta,
+       y quizás funcionaba porque el usuario no lo había notado - de nuevo
+       no importa. Funcionaba para el usuario.
+
+       Romper el flujo del trabajo de un usuario, debido a un "bug" es la
+       PEOR razón que se pueda usar.
+
+       Es básicamente decir "He cogido algo que funcionaba, y lo he roto,
+       pero ahora es mejor". ¿No ves que un argumento como este es j*didamente
+       absurdo?
+
+       y sin usuarios, tu programa no es un programa, es una pieza de
+       código sin finalidad que puedes perfectamente tirar a la basura.
+
+       Seriamente. Esto es *porque* la regla #1 para el desarrollo del
+       kernel es "no rompemos el espacio de usuario". Porque "He arreglado
+       un error" PARA NADA ES UN ARGUMENTO si esa corrección del código
+       rompe el espacio de usuario.
+
+       si actualizamos el kernel TODO EL TIEMPO, sin actualizar ningún otro
+       programa en absoluto. Y esto es absolutamente necesario, porque
+       las dependencias son terribles.
+
+       Y esto es necesario simplemente porque yo como desarrollador del
+       kernel no actualizo al azar otras herramientas que ni siquiera me
+       importan como desarrollador del kernel, y yo quiero que mis usuarios
+       se sientan a salvo haciendo lo mismo.
+
+       Así que no. Tu regla está COMPLETAMENTE equivocada. Si no puedes
+       actualizar el kernel sin actualizar otro binario al azar, entonces
+       tenemos un problema.
+
+ * De `2021-06-05
+   <https://lore.kernel.org/all/CAHk-=wiUVqHN76YUwhkjZzwTdjMMJf_zN4+u7vEJjmEGh3recw@mail.gmail.com/>`_::
+
+       NO HAY ARGUMENTOS VÁLIDOS PARA UNA REGRESIÓN.
+
+       Honestamente, la gente de seguridad necesita entender que "no funciona"
+       no es un caso de éxito sobre seguridad. Es un caso de fallo.
+
+       Sí, "no funciona" puede ser seguro. Pero en este caso es totalmente
+       inutil.
+
+ * De `2011-05-06 (1/3)
+   <https://lore.kernel.org/all/BANLkTim9YvResB+PwRp7QTK-a5VNg2PvmQ@mail.gmail.com/>`_::
+
+       La compatibilidad de los binarios es más importante.
+
+       Y si los binarios no usan el interfaz para parsear el formato
+       (o justamente lo parsea incorrectamente - como el reciente ejemplo
+       de añadir uuid al /proc/self/mountinfo), entonces es una regresión.
+
+       Y las regresiones se revierten, a menos que haya problemas de
+       seguridad o similares que nos hagan decir "Dios mío, realmente
+       tenemos que romper las cosas".
+
+       No entiendo porqué esta simple lógica es tan difícil para algunos
+       desarrolladores del kernel. La realidad importa. Sus deseos personales
+       NO IMPORTAN NADA.
+
+       Si se crea un interface que puede usarse sin parsear la
+       descripción del interface, entonces estaḿos atascados en el interface.
+       La teoría simplemente no importa.
+
+       Podrias alludar a arreglar las herramientas, e intentar evitar los
+       errores de compatibilidad de ese modo. No hay tampoco tantos de esos.
+
+   De `2011-05-06 (2/3)
+   <https://lore.kernel.org/all/BANLkTi=KVXjKR82sqsz4gwjr+E0vtqCmvA@mail.gmail.com/>`_::
+
+       Esto claramente NO es un tracepoint interno. Por definición. Y está
+       siendo usado por powertop.
+
+   De `2011-05-06 (3/3)
+   <https://lore.kernel.org/all/BANLkTinazaXRdGovYL7rRVp+j6HbJ7pzhg@mail.gmail.com/>`_::
+
+       Tenemos programas que usan esa ABI y si eso se rompe eso es una
+       regresión.
+
+ * De `2012-07-06 <https://lore.kernel.org/all/CA+55aFwnLJ+0sjx92EGREGTWOx84wwKaraSzpTNJwPVV8edw8g@mail.gmail.com/>`_::
+
+       > Ahora esto me ha dejado preguntandome si Debian _inestable_
+       realmente califica
+       > como espacio de usuario estándar.
+
+       Oh, si el kernel rompe algún espacio de usuario estándar, eso cuenta.
+       Muchísima gente usa Debian inestable.
+
+ * De `2019-09-15
+   <https://lore.kernel.org/lkml/CAHk-=wiP4K8DRJWsCo=20hn_6054xBamGKF2kPgUzpB5aMaofA@mail.gmail.com/>`_::
+
+       Una reversión _en particular_ en el último minuto en el último commit
+       (no teniendo en cuenta el propio cambio de versión) justo antes
+       de la liberación, y aunque es bastante incómodo, quizás también es
+       instructivo.
+
+       Lo que es instructivo sobre esto es que he revertido un commit que no
+       tenía ningún error. De hecho, hacía exactamente lo que pretendía, y lo
+       hacía muy bien. De hecho lo hacía _tan_ bien que los muy mejorados
+       patrones de IO que causaba han acabado revelando una regresión observable
+       desde el espacio de usuario, debido a un error real en un componente
+       no relacionado en absoluto.
+
+       De todas maneras, los detalles actuales de esta regresión no son la
+       razón por la que señalo esto como instructivo. Es más que es un ejemplo
+       ilustrativo sobre lo que cuenta como una regresión, y lo que conlleva
+       la regla del kernel de "no regresiones". El commit que ha sido revertido
+       no cambiaba ninguna API, y no introducía ningún error nuevo en el código.
+       Pero acabó exponiendo otro problema, y como eso causaba que la
+       actualización del kernel fallara para el usuario. Así que ha sido
+       revertido.
+
+       El foco aquí, es que hemos hecho la reversión basándonos en el
+       comportamiento reportado en el espacio de usuario, no basado en
+       conceptos como "cambios de ABI" o "provocaba un error". Los mejores
+       patrones de IO que se han presentado debido al cambio únicamente han
+       expuesto un viejo error, y la gente ya dependía del benigno
+       comportamiento de ese viejo error.
+
+       Y que no haya miedo, reintroduciremos el arreglo que mejoraba los
+       patrones de IO una vez hayamos decidido cómo gestionar el hecho de
+       que hay una interacción incorrecta con un interfaz en el que la
+       gente dependía de ese comportamiento previo. Es únicamente que
+       tenemos que ver cómo gestionamos y cómo lo hacemos (no hay menos de
+       tres parches diferentes de tres desarrolladores distintos que estamos
+       evaluando, ... puede haber más por llegar). Mientras tanto, he
+       revertido lo que exponía el problema a los usuarios de esta release,
+       incluso cuando espero que el fix será reintroducido (quizás insertado
+       a posteriormente como un parche estable) una vez lleguemos a un
+       acuerdo sobre cómo se ha de exponer el error.
+
+       Lo que hay que recordar de todo el asunto no es sobre si el cambio
+       de kernel-espacio-de-usuario ABI, o la corrección de un error, o si
+       el código antiguo "en primer lugar nunca debería haber estado ahí".
+       Es sobre si algo rompe el actual flujo de trabajo del usuario.
+
+       De todas formas, esto era mi pequeña aclaración en todo este
+       tema de la regresión. Ya que es la "primera regla de la programación
+       del kernel", me ha parecido que quizás es bueno mencionarlo de
+       vez en cuando.
diff --git a/Documentation/translations/sp_SP/process/index.rst b/Documentation/translations/sp_SP/process/index.rst
index d6f3ccfb160e..a082e162168a 100644
--- a/Documentation/translations/sp_SP/process/index.rst
+++ b/Documentation/translations/sp_SP/process/index.rst
@@ -24,3 +24,4 @@
    contribution-maturity-model
    security-bugs
    embargoed-hardware-issues
+   handling-regressions
diff --git a/Documentation/translations/zh_CN/arch/loongarch/introduction.rst b/Documentation/translations/zh_CN/arch/loongarch/introduction.rst
index 59d6bf33050c..bf463c5a4c51 100644
--- a/Documentation/translations/zh_CN/arch/loongarch/introduction.rst
+++ b/Documentation/translations/zh_CN/arch/loongarch/introduction.rst
@@ -338,9 +338,9 @@ Loongson与LoongArch的开发者网站（软件与文档资源）：
 
 LoongArch指令集架构的文档：
 
-  https://github.com/loongson/LoongArch-Documentation/releases/latest/download/LoongArch-Vol1-v1.02-CN.pdf （中文版）
+  https://github.com/loongson/LoongArch-Documentation/releases/latest/download/LoongArch-Vol1-v1.10-CN.pdf （中文版）
 
-  https://github.com/loongson/LoongArch-Documentation/releases/latest/download/LoongArch-Vol1-v1.02-EN.pdf （英文版）
+  https://github.com/loongson/LoongArch-Documentation/releases/latest/download/LoongArch-Vol1-v1.10-EN.pdf （英文版）
 
 LoongArch的ELF psABI文档：
 
diff --git a/Documentation/translations/zh_CN/dev-tools/testing-overview.rst b/Documentation/translations/zh_CN/dev-tools/testing-overview.rst
index 69e7e4cb2002..c91f9b60f9f1 100644
--- a/Documentation/translations/zh_CN/dev-tools/testing-overview.rst
+++ b/Documentation/translations/zh_CN/dev-tools/testing-overview.rst
@@ -3,7 +3,7 @@
 .. include:: ../disclaimer-zh_CN.rst
 
 :Original: Documentation/dev-tools/testing-overview.rst
-:Translator: 胡皓文 Hu Haowen <src.res.211@gmail.com>
+:Translator: 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 ============
 内核测试指南
diff --git a/Documentation/translations/zh_TW/IRQ.txt b/Documentation/translations/zh_TW/IRQ.txt
index fd78ca720298..8115a7618307 100644
--- a/Documentation/translations/zh_TW/IRQ.txt
+++ b/Documentation/translations/zh_TW/IRQ.txt
@@ -7,7 +7,7 @@ help.  Contact the Chinese maintainer if this translation is outdated
 or if there is a problem with the translation.
 
 Maintainer: Eric W. Biederman <ebiederman@xmission.com>
-Traditional Chinese maintainer: Hu Haowen <src.res.211@gmail.com>
+Traditional Chinese maintainer: Hu Haowen <2023002089@link.tyut.edu.cn>
 ---------------------------------------------------------------------
 Documentation/core-api/irq/index.rst 的繁體中文翻譯
 
@@ -16,9 +16,9 @@ Documentation/core-api/irq/index.rst 的繁體中文翻譯
 者翻譯存在問題，請聯繫繁體中文版維護者。
 
 英文版維護者： Eric W. Biederman <ebiederman@xmission.com>
-繁體中文版維護者： 胡皓文 Hu Haowen <src.res.211@gmail.com>
-繁體中文版翻譯者： 胡皓文 Hu Haowen <src.res.211@gmail.com>
-繁體中文版校譯者： 胡皓文 Hu Haowen <src.res.211@gmail.com>
+繁體中文版維護者： 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
+繁體中文版翻譯者： 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
+繁體中文版校譯者： 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 
 以下爲正文
diff --git a/Documentation/translations/zh_TW/admin-guide/README.rst b/Documentation/translations/zh_TW/admin-guide/README.rst
index 4cb581f5994a..a6e34c200ea3 100644
--- a/Documentation/translations/zh_TW/admin-guide/README.rst
+++ b/Documentation/translations/zh_TW/admin-guide/README.rst
@@ -7,7 +7,7 @@
 :譯者:
 
  吳想成 Wu XiangCheng <bobwxc@email.cn>
- 胡皓文 Hu Haowen <src.res.211@gmail.com>
+ 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 Linux內核6.x版本 <http://kernel.org/>
 =========================================
diff --git a/Documentation/translations/zh_TW/admin-guide/bug-bisect.rst b/Documentation/translations/zh_TW/admin-guide/bug-bisect.rst
index 3f10a9f8f223..1efe913b8da0 100644
--- a/Documentation/translations/zh_TW/admin-guide/bug-bisect.rst
+++ b/Documentation/translations/zh_TW/admin-guide/bug-bisect.rst
@@ -7,7 +7,7 @@
 :譯者:
 
  吳想成 Wu XiangCheng <bobwxc@email.cn>
- 胡皓文 Hu Haowen <src.res.211@gmail.com>
+ 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 二分（bisect）缺陷
 +++++++++++++++++++
diff --git a/Documentation/translations/zh_TW/admin-guide/bug-hunting.rst b/Documentation/translations/zh_TW/admin-guide/bug-hunting.rst
index 631fd2650929..c139ec99cab1 100644
--- a/Documentation/translations/zh_TW/admin-guide/bug-hunting.rst
+++ b/Documentation/translations/zh_TW/admin-guide/bug-hunting.rst
@@ -7,7 +7,7 @@
 :譯者:
 
  吳想成 Wu XiangCheng <bobwxc@email.cn>
- 胡皓文 Hu Haowen <src.res.211@gmail.com>
+ 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 追蹤缺陷
 =========
diff --git a/Documentation/translations/zh_TW/admin-guide/clearing-warn-once.rst b/Documentation/translations/zh_TW/admin-guide/clearing-warn-once.rst
index 6961006b4a2d..a3e82ff9daac 100644
--- a/Documentation/translations/zh_TW/admin-guide/clearing-warn-once.rst
+++ b/Documentation/translations/zh_TW/admin-guide/clearing-warn-once.rst
@@ -2,7 +2,7 @@
 
 .. include:: ../disclaimer-zh_TW.rst
 
-:Translator: 胡皓文 Hu Haowen <src.res.211@gmail.com>
+:Translator: 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 清除 WARN_ONCE
 --------------
diff --git a/Documentation/translations/zh_TW/admin-guide/cpu-load.rst b/Documentation/translations/zh_TW/admin-guide/cpu-load.rst
index cc046f3b7ffa..4c25a2105b39 100644
--- a/Documentation/translations/zh_TW/admin-guide/cpu-load.rst
+++ b/Documentation/translations/zh_TW/admin-guide/cpu-load.rst
@@ -2,7 +2,7 @@
 
 .. include:: ../disclaimer-zh_TW.rst
 
-:Translator: 胡皓文 Hu Haowen <src.res.211@gmail.com>
+:Translator: 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 ========
 CPU 負載
diff --git a/Documentation/translations/zh_TW/admin-guide/index.rst b/Documentation/translations/zh_TW/admin-guide/index.rst
index aba8939351e0..9335c0e9105d 100644
--- a/Documentation/translations/zh_TW/admin-guide/index.rst
+++ b/Documentation/translations/zh_TW/admin-guide/index.rst
@@ -4,7 +4,7 @@
 
 :Original: :doc:`../../../admin-guide/index`
 :Translator: Alex Shi <alex.shi@linux.alibaba.com>
-             胡皓文 Hu Haowen <src.res.211@gmail.com>
+             胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 Linux 內核用戶和管理員指南
 ==========================
diff --git a/Documentation/translations/zh_TW/admin-guide/init.rst b/Documentation/translations/zh_TW/admin-guide/init.rst
index be6e34f5f7fa..4cef1994c650 100644
--- a/Documentation/translations/zh_TW/admin-guide/init.rst
+++ b/Documentation/translations/zh_TW/admin-guide/init.rst
@@ -7,7 +7,7 @@
 :譯者:
 
  吳想成 Wu XiangCheng <bobwxc@email.cn>
- 胡皓文 Hu Haowen <src.res.211@gmail.com>
+ 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 解釋“No working init found.”啓動掛起消息
 =========================================
diff --git a/Documentation/translations/zh_TW/admin-guide/reporting-issues.rst b/Documentation/translations/zh_TW/admin-guide/reporting-issues.rst
index fe5a5a07d51a..bc132b25f2ae 100644
--- a/Documentation/translations/zh_TW/admin-guide/reporting-issues.rst
+++ b/Documentation/translations/zh_TW/admin-guide/reporting-issues.rst
@@ -9,7 +9,7 @@
 :譯者:
 
  吳想成 Wu XiangCheng <bobwxc@email.cn>
- 胡皓文 Hu Haowen <src.res.211@gmail.com>
+ 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 
 報告問題
diff --git a/Documentation/translations/zh_TW/admin-guide/security-bugs.rst b/Documentation/translations/zh_TW/admin-guide/security-bugs.rst
index c0e9fc247695..cfe1e58e116b 100644
--- a/Documentation/translations/zh_TW/admin-guide/security-bugs.rst
+++ b/Documentation/translations/zh_TW/admin-guide/security-bugs.rst
@@ -7,7 +7,7 @@
 :譯者:
 
  吳想成 Wu XiangCheng <bobwxc@email.cn>
- 胡皓文 Hu Haowen <src.res.211@gmail.com>
+ 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 安全缺陷
 =========
diff --git a/Documentation/translations/zh_TW/admin-guide/tainted-kernels.rst b/Documentation/translations/zh_TW/admin-guide/tainted-kernels.rst
index 47629f6b05de..0d8046576d04 100644
--- a/Documentation/translations/zh_TW/admin-guide/tainted-kernels.rst
+++ b/Documentation/translations/zh_TW/admin-guide/tainted-kernels.rst
@@ -7,7 +7,7 @@
 :譯者:
 
  吳想成 Wu XiangCheng <bobwxc@email.cn>
- 胡皓文 Hu Haowen <src.res.211@gmail.com>
+ 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 受污染的內核
 -------------
diff --git a/Documentation/translations/zh_TW/admin-guide/unicode.rst b/Documentation/translations/zh_TW/admin-guide/unicode.rst
index a2b48b5d0a64..f43edb2b5ed0 100644
--- a/Documentation/translations/zh_TW/admin-guide/unicode.rst
+++ b/Documentation/translations/zh_TW/admin-guide/unicode.rst
@@ -7,7 +7,7 @@
 :譯者:
 
  吳想成 Wu XiangCheng <bobwxc@email.cn>
- 胡皓文 Hu Haowen <src.res.211@gmail.com>
+ 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 Unicode（統一碼）支持
 ======================
diff --git a/Documentation/translations/zh_TW/arch/arm64/amu.rst b/Documentation/translations/zh_TW/arch/arm64/amu.rst
index 1b451eae2bee..3726c1671ab6 100644
--- a/Documentation/translations/zh_TW/arch/arm64/amu.rst
+++ b/Documentation/translations/zh_TW/arch/arm64/amu.rst
@@ -5,7 +5,7 @@
 :Original: :ref:`Documentation/arch/arm64/amu.rst <amu_index>`
 
 Translator: Bailu Lin <bailu.lin@vivo.com>
-            Hu Haowen <src.res.211@gmail.com>
+            Hu Haowen <2023002089@link.tyut.edu.cn>
 
 ==================================
 AArch64 Linux 中擴展的活動監控單元
diff --git a/Documentation/translations/zh_TW/arch/arm64/booting.txt b/Documentation/translations/zh_TW/arch/arm64/booting.txt
index be0de91ecebd..f1ac96370ace 100644
--- a/Documentation/translations/zh_TW/arch/arm64/booting.txt
+++ b/Documentation/translations/zh_TW/arch/arm64/booting.txt
@@ -10,7 +10,7 @@ or if there is a problem with the translation.
 
 M:	Will Deacon <will.deacon@arm.com>
 zh_CN:	Fu Wei <wefu@redhat.com>
-zh_TW:	Hu Haowen <src.res.211@gmail.com>
+zh_TW:	Hu Haowen <2023002089@link.tyut.edu.cn>
 C:	55f058e7574c3615dea4615573a19bdb258696c6
 ---------------------------------------------------------------------
 Documentation/arch/arm64/booting.rst 的中文翻譯
@@ -23,7 +23,7 @@ Documentation/arch/arm64/booting.rst 的中文翻譯
 中文版維護者： 傅煒  Fu Wei <wefu@redhat.com>
 中文版翻譯者： 傅煒  Fu Wei <wefu@redhat.com>
 中文版校譯者： 傅煒  Fu Wei <wefu@redhat.com>
-繁體中文版校譯者： 胡皓文  Hu Haowen <src.res.211@gmail.com>
+繁體中文版校譯者： 胡皓文  Hu Haowen <2023002089@link.tyut.edu.cn>
 本文翻譯提交時的 Git 檢出點爲： 55f058e7574c3615dea4615573a19bdb258696c6
 
 以下爲正文
diff --git a/Documentation/translations/zh_TW/arch/arm64/elf_hwcaps.rst b/Documentation/translations/zh_TW/arch/arm64/elf_hwcaps.rst
index d2c1c2f23812..cada25303e8d 100644
--- a/Documentation/translations/zh_TW/arch/arm64/elf_hwcaps.rst
+++ b/Documentation/translations/zh_TW/arch/arm64/elf_hwcaps.rst
@@ -5,7 +5,7 @@
 :Original: :ref:`Documentation/arch/arm64/elf_hwcaps.rst <elf_hwcaps_index>`
 
 Translator: Bailu Lin <bailu.lin@vivo.com>
-            Hu Haowen <src.res.211@gmail.com>
+            Hu Haowen <2023002089@link.tyut.edu.cn>
 
 ================
 ARM64 ELF hwcaps
diff --git a/Documentation/translations/zh_TW/arch/arm64/hugetlbpage.rst b/Documentation/translations/zh_TW/arch/arm64/hugetlbpage.rst
index a17858c978d6..b6849935e028 100644
--- a/Documentation/translations/zh_TW/arch/arm64/hugetlbpage.rst
+++ b/Documentation/translations/zh_TW/arch/arm64/hugetlbpage.rst
@@ -5,7 +5,7 @@
 :Original: :ref:`Documentation/arch/arm64/hugetlbpage.rst <hugetlbpage_index>`
 
 Translator: Bailu Lin <bailu.lin@vivo.com>
-            Hu Haowen <src.res.211@gmail.com>
+            Hu Haowen <2023002089@link.tyut.edu.cn>
 
 =====================
 ARM64中的 HugeTLBpage
diff --git a/Documentation/translations/zh_TW/arch/arm64/index.rst b/Documentation/translations/zh_TW/arch/arm64/index.rst
index a62b5f06b66c..86014346792e 100644
--- a/Documentation/translations/zh_TW/arch/arm64/index.rst
+++ b/Documentation/translations/zh_TW/arch/arm64/index.rst
@@ -4,7 +4,7 @@
 
 :Original: :ref:`Documentation/arch/arm64/index.rst <arm64_index>`
 :Translator: Bailu Lin <bailu.lin@vivo.com>
-             Hu Haowen <src.res.211@gmail.com>
+             Hu Haowen <2023002089@link.tyut.edu.cn>
 
 .. _tw_arm64_index:
 
diff --git a/Documentation/translations/zh_TW/arch/arm64/legacy_instructions.txt b/Documentation/translations/zh_TW/arch/arm64/legacy_instructions.txt
index 7d1f0593d7ca..5c664555a71a 100644
--- a/Documentation/translations/zh_TW/arch/arm64/legacy_instructions.txt
+++ b/Documentation/translations/zh_TW/arch/arm64/legacy_instructions.txt
@@ -11,7 +11,7 @@ or if there is a problem with the translation.
 Maintainer: Punit Agrawal <punit.agrawal@arm.com>
             Suzuki K. Poulose <suzuki.poulose@arm.com>
 Chinese maintainer: Fu Wei <wefu@redhat.com>
-Traditional Chinese maintainer: Hu Haowen <src.res.211@gmail.com>
+Traditional Chinese maintainer: Hu Haowen <2023002089@link.tyut.edu.cn>
 ---------------------------------------------------------------------
 Documentation/arch/arm64/legacy_instructions.rst 的中文翻譯
 
@@ -26,7 +26,7 @@ Documentation/arch/arm64/legacy_instructions.rst 的中文翻譯
 中文版維護者： 傅煒  Fu Wei <wefu@redhat.com>
 中文版翻譯者： 傅煒  Fu Wei <wefu@redhat.com>
 中文版校譯者： 傅煒  Fu Wei <wefu@redhat.com>
-繁體中文版校譯者：胡皓文  Hu Haowen <src.res.211@gmail.com>
+繁體中文版校譯者：胡皓文  Hu Haowen <2023002089@link.tyut.edu.cn>
 
 以下爲正文
 ---------------------------------------------------------------------
diff --git a/Documentation/translations/zh_TW/arch/arm64/memory.txt b/Documentation/translations/zh_TW/arch/arm64/memory.txt
index e41c518e71c6..6ee2239c293f 100644
--- a/Documentation/translations/zh_TW/arch/arm64/memory.txt
+++ b/Documentation/translations/zh_TW/arch/arm64/memory.txt
@@ -10,7 +10,7 @@ or if there is a problem with the translation.
 
 Maintainer: Catalin Marinas <catalin.marinas@arm.com>
 Chinese maintainer: Fu Wei <wefu@redhat.com>
-Traditional Chinese maintainer: Hu Haowen <src.res.211@gmail.com>
+Traditional Chinese maintainer: Hu Haowen <2023002089@link.tyut.edu.cn>
 ---------------------------------------------------------------------
 Documentation/arch/arm64/memory.rst 的中文翻譯
 
@@ -24,7 +24,7 @@ Documentation/arch/arm64/memory.rst 的中文翻譯
 中文版維護者： 傅煒  Fu Wei <wefu@redhat.com>
 中文版翻譯者： 傅煒  Fu Wei <wefu@redhat.com>
 中文版校譯者： 傅煒  Fu Wei <wefu@redhat.com>
-繁體中文版校譯者： 胡皓文  Hu Haowen <src.res.211@gmail.com>
+繁體中文版校譯者： 胡皓文  Hu Haowen <2023002089@link.tyut.edu.cn>
 
 以下爲正文
 ---------------------------------------------------------------------
diff --git a/Documentation/translations/zh_TW/arch/arm64/perf.rst b/Documentation/translations/zh_TW/arch/arm64/perf.rst
index 405d5f66964f..ce083ba63872 100644
--- a/Documentation/translations/zh_TW/arch/arm64/perf.rst
+++ b/Documentation/translations/zh_TW/arch/arm64/perf.rst
@@ -5,7 +5,7 @@
 :Original: :ref:`Documentation/arch/arm64/perf.rst <perf_index>`
 
 Translator: Bailu Lin <bailu.lin@vivo.com>
-            Hu Haowen <src.res.211@gmail.com>
+            Hu Haowen <2023002089@link.tyut.edu.cn>
 
 =============
 Perf 事件屬性
diff --git a/Documentation/translations/zh_TW/arch/arm64/silicon-errata.txt b/Documentation/translations/zh_TW/arch/arm64/silicon-errata.txt
index 70371807ca83..16d73b6c309f 100644
--- a/Documentation/translations/zh_TW/arch/arm64/silicon-errata.txt
+++ b/Documentation/translations/zh_TW/arch/arm64/silicon-errata.txt
@@ -10,7 +10,7 @@ or if there is a problem with the translation.
 
 M:	Will Deacon <will.deacon@arm.com>
 zh_CN:	Fu Wei <wefu@redhat.com>
-zh_TW:	Hu Haowen <src.res.211@gmail.com>
+zh_TW:	Hu Haowen <2023002089@link.tyut.edu.cn>
 C:	1926e54f115725a9248d0c4c65c22acaf94de4c4
 ---------------------------------------------------------------------
 Documentation/arch/arm64/silicon-errata.rst 的中文翻譯
@@ -23,7 +23,7 @@ Documentation/arch/arm64/silicon-errata.rst 的中文翻譯
 中文版維護者： 傅煒  Fu Wei <wefu@redhat.com>
 中文版翻譯者： 傅煒  Fu Wei <wefu@redhat.com>
 中文版校譯者： 傅煒  Fu Wei <wefu@redhat.com>
-繁體中文版校譯者： 胡皓文  Hu Haowen <src.res.211@gmail.com>
+繁體中文版校譯者： 胡皓文  Hu Haowen <2023002089@link.tyut.edu.cn>
 本文翻譯提交時的 Git 檢出點爲： 1926e54f115725a9248d0c4c65c22acaf94de4c4
 
 以下爲正文
diff --git a/Documentation/translations/zh_TW/arch/arm64/tagged-pointers.txt b/Documentation/translations/zh_TW/arch/arm64/tagged-pointers.txt
index 9812d99549ba..e86ffa893ef6 100644
--- a/Documentation/translations/zh_TW/arch/arm64/tagged-pointers.txt
+++ b/Documentation/translations/zh_TW/arch/arm64/tagged-pointers.txt
@@ -10,7 +10,7 @@ or if there is a problem with the translation.
 
 Maintainer: Will Deacon <will.deacon@arm.com>
 Chinese maintainer: Fu Wei <wefu@redhat.com>
-Traditional Chinese maintainer: Hu Haowen <src.res.211@gmail.com>
+Traditional Chinese maintainer: Hu Haowen <2023002089@link.tyut.edu.cn>
 ---------------------------------------------------------------------
 Documentation/arch/arm64/tagged-pointers.rst 的中文翻譯
 
@@ -22,7 +22,7 @@ Documentation/arch/arm64/tagged-pointers.rst 的中文翻譯
 中文版維護者： 傅煒  Fu Wei <wefu@redhat.com>
 中文版翻譯者： 傅煒  Fu Wei <wefu@redhat.com>
 中文版校譯者： 傅煒  Fu Wei <wefu@redhat.com>
-繁體中文版校譯者： 胡皓文  Hu Haowen <src.res.211@gmail.com>
+繁體中文版校譯者： 胡皓文  Hu Haowen <2023002089@link.tyut.edu.cn>
 
 以下爲正文
 ---------------------------------------------------------------------
diff --git a/Documentation/translations/zh_TW/dev-tools/sparse.rst b/Documentation/translations/zh_TW/dev-tools/sparse.rst
index 11d64709d6a4..55f0ad2c0beb 100644
--- a/Documentation/translations/zh_TW/dev-tools/sparse.rst
+++ b/Documentation/translations/zh_TW/dev-tools/sparse.rst
@@ -6,19 +6,19 @@ communicating in English you can also ask the Chinese maintainer for
 help.  Contact the Chinese maintainer if this translation is outdated
 or if there is a problem with the translation.
 
-Traditional Chinese maintainer: Hu Haowen <src.res.211@gmail.com>
----------------------------------------------------------------------
+Traditional Chinese maintainer: Hu Haowen <2023002089@link.tyut.edu.cn>
+-------------------------------------------------------------------------
 Documentation/dev-tools/sparse.rst 的繁體中文翻譯
 
 如果想評論或更新本文的內容，請直接聯繫原文檔的維護者。如果你使用英文
 交流有困難的話，也可以向繁體中文版維護者求助。如果本翻譯更新不及時或
 者翻譯存在問題，請聯繫繁體中文版維護者。
 
-繁體中文版維護者： 胡皓文 Hu Haowen <src.res.211@gmail.com>
-繁體中文版翻譯者： 胡皓文 Hu Haowen <src.res.211@gmail.com>
+繁體中文版維護者： 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
+繁體中文版翻譯者： 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 以下爲正文
----------------------------------------------------------------------
+-------------------------------------------------------------------------
 
 Copyright 2004 Linus Torvalds
 Copyright 2004 Pavel Machek <pavel@ucw.cz>
diff --git a/Documentation/translations/zh_TW/dev-tools/testing-overview.rst b/Documentation/translations/zh_TW/dev-tools/testing-overview.rst
index fb3f691f46c3..3b08aad1da00 100644
--- a/Documentation/translations/zh_TW/dev-tools/testing-overview.rst
+++ b/Documentation/translations/zh_TW/dev-tools/testing-overview.rst
@@ -3,7 +3,7 @@
 .. include:: ../disclaimer-zh_TW.rst
 
 :Original: Documentation/dev-tools/testing-overview.rst
-:Translator: 胡皓文 Hu Haowen <src.res.211@gmail.com>
+:Translator: 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 ============
 內核測試指南
diff --git a/Documentation/translations/zh_TW/disclaimer-zh_TW.rst b/Documentation/translations/zh_TW/disclaimer-zh_TW.rst
index 0d0ffb1ca4e8..28b734c223b6 100644
--- a/Documentation/translations/zh_TW/disclaimer-zh_TW.rst
+++ b/Documentation/translations/zh_TW/disclaimer-zh_TW.rst
@@ -7,5 +7,5 @@
 
 .. note::
      如果您發現本文檔與原始文件有任何不同或者有翻譯問題，請聯繫該文件的譯者，
-     或者發送電子郵件給胡皓文以獲取幫助：<src.res.211@gmail.com>。
+     或者發送電子郵件給胡皓文以獲取幫助：<2023002089@link.tyut.edu.cn>。
 
diff --git a/Documentation/translations/zh_TW/filesystems/debugfs.rst b/Documentation/translations/zh_TW/filesystems/debugfs.rst
index 78e2e08af95e..cda7d0e18b9b 100644
--- a/Documentation/translations/zh_TW/filesystems/debugfs.rst
+++ b/Documentation/translations/zh_TW/filesystems/debugfs.rst
@@ -14,7 +14,7 @@ Debugfs
 	中文版維護者： 羅楚成 Chucheng Luo <luochucheng@vivo.com>
 	中文版翻譯者： 羅楚成 Chucheng Luo <luochucheng@vivo.com>
 	中文版校譯者:  羅楚成 Chucheng Luo <luochucheng@vivo.com>
-	繁體中文版校譯者: 胡皓文 Hu Haowen <src.res.211@gmail.com>
+	繁體中文版校譯者: 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 
 
diff --git a/Documentation/translations/zh_TW/filesystems/index.rst b/Documentation/translations/zh_TW/filesystems/index.rst
index d7f9d61f654c..88f0e632bfe2 100644
--- a/Documentation/translations/zh_TW/filesystems/index.rst
+++ b/Documentation/translations/zh_TW/filesystems/index.rst
@@ -4,7 +4,7 @@
 
 :Original: :ref:`Documentation/filesystems/index.rst <filesystems_index>`
 :Translator: Wang Wenhu <wenhu.wang@vivo.com>
-             Hu Haowen <src.res.211@gmail.com>
+             Hu Haowen <2023002089@link.tyut.edu.cn>
 
 .. _tw_filesystems_index:
 
diff --git a/Documentation/translations/zh_TW/filesystems/sysfs.txt b/Documentation/translations/zh_TW/filesystems/sysfs.txt
index ebe90651fc3b..978462d5fe14 100644
--- a/Documentation/translations/zh_TW/filesystems/sysfs.txt
+++ b/Documentation/translations/zh_TW/filesystems/sysfs.txt
@@ -22,7 +22,7 @@ Documentation/filesystems/sysfs.rst 的中文翻譯
 中文版維護者： 傅煒 Fu Wei <tekkamanninja@gmail.com>
 中文版翻譯者： 傅煒 Fu Wei <tekkamanninja@gmail.com>
 中文版校譯者： 傅煒 Fu Wei <tekkamanninja@gmail.com>
-繁體中文版校譯者：胡皓文 Hu Haowen <src.res.211@gmail.com>
+繁體中文版校譯者：胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 
 以下爲正文
diff --git a/Documentation/translations/zh_TW/filesystems/virtiofs.rst b/Documentation/translations/zh_TW/filesystems/virtiofs.rst
index 6150ad964e78..704a0ee44fd2 100644
--- a/Documentation/translations/zh_TW/filesystems/virtiofs.rst
+++ b/Documentation/translations/zh_TW/filesystems/virtiofs.rst
@@ -10,7 +10,7 @@
 	中文版維護者： 王文虎 Wang Wenhu <wenhu.wang@vivo.com>
 	中文版翻譯者： 王文虎 Wang Wenhu <wenhu.wang@vivo.com>
 	中文版校譯者： 王文虎 Wang Wenhu <wenhu.wang@vivo.com>
-	繁體中文版校譯者：胡皓文 Hu Haowen <src.res.211@gmail.com>
+	繁體中文版校譯者：胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 ===========================================
 virtiofs: virtio-fs 主機<->客機共享文件系統
diff --git a/Documentation/translations/zh_TW/gpio.txt b/Documentation/translations/zh_TW/gpio.txt
index 555e4b11a5c7..b9b48012c62e 100644
--- a/Documentation/translations/zh_TW/gpio.txt
+++ b/Documentation/translations/zh_TW/gpio.txt
@@ -8,7 +8,7 @@ or if there is a problem with the translation.
 
 Maintainer: Grant Likely <grant.likely@secretlab.ca>
 		Linus Walleij <linus.walleij@linaro.org>
-Traditional Chinese maintainer: Hu Haowen <src.res.211@gmail.com>
+Traditional Chinese maintainer: Hu Haowen <2023002089@link.tyut.edu.cn>
 ---------------------------------------------------------------------
 Documentation/admin-guide/gpio 的繁體中文翻譯
 
@@ -18,9 +18,9 @@ Documentation/admin-guide/gpio 的繁體中文翻譯
 
 英文版維護者： Grant Likely <grant.likely@secretlab.ca>
 		Linus Walleij <linus.walleij@linaro.org>
-繁體中文版維護者： 胡皓文 Hu Haowen <src.res.211@gmail.com>
-繁體中文版翻譯者： 胡皓文 Hu Haowen <src.res.211@gmail.com>
-繁體中文版校譯者： 胡皓文 Hu Haowen <src.res.211@gmail.com>
+繁體中文版維護者： 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
+繁體中文版翻譯者： 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
+繁體中文版校譯者： 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 以下爲正文
 ---------------------------------------------------------------------
diff --git a/Documentation/translations/zh_TW/index.rst b/Documentation/translations/zh_TW/index.rst
index 563ac9bfc66b..660a74d2023c 100644
--- a/Documentation/translations/zh_TW/index.rst
+++ b/Documentation/translations/zh_TW/index.rst
@@ -15,7 +15,7 @@
 
 .. note::
    內核文檔繁體中文版的翻譯工作正在進行中。如果您願意並且有時間參與這項工
-   作，歡迎提交補丁給胡皓文 <src.res.211@gmail.com>。
+   作，歡迎提交補丁給胡皓文 <2023002089@link.tyut.edu.cn>。
 
 與Linux 內核社區一起工作
 ------------------------
diff --git a/Documentation/translations/zh_TW/io_ordering.txt b/Documentation/translations/zh_TW/io_ordering.txt
index 03f86840c139..00b374092d7e 100644
--- a/Documentation/translations/zh_TW/io_ordering.txt
+++ b/Documentation/translations/zh_TW/io_ordering.txt
@@ -6,7 +6,7 @@ communicating in English you can also ask the Chinese maintainer for
 help.  Contact the Chinese maintainer if this translation is outdated
 or if there is a problem with the translation.
 
-Traditional Chinese maintainer: Hu Haowen <src.res.211@gmail.com>
+Traditional Chinese maintainer: Hu Haowen <2023002089@link.tyut.edu.cn>
 ---------------------------------------------------------------------
 Documentation/driver-api/io_ordering.rst 的繁體中文翻譯
 
@@ -14,9 +14,9 @@ Documentation/driver-api/io_ordering.rst 的繁體中文翻譯
 交流有困難的話，也可以向繁體中文版維護者求助。如果本翻譯更新不及時或
 者翻譯存在問題，請聯繫繁體中文版維護者。
 
-繁體中文版維護者： 胡皓文 Hu Haowen <src.res.211@gmail.com>
-繁體中文版翻譯者： 胡皓文 Hu Haowen <src.res.211@gmail.com>
-繁體中文版校譯者： 胡皓文 Hu Haowen <src.res.211@gmail.com>
+繁體中文版維護者： 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
+繁體中文版翻譯者： 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
+繁體中文版校譯者： 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 
 以下爲正文
diff --git a/Documentation/translations/zh_TW/process/1.Intro.rst b/Documentation/translations/zh_TW/process/1.Intro.rst
index 6e754ac48964..345c4cbe9b55 100644
--- a/Documentation/translations/zh_TW/process/1.Intro.rst
+++ b/Documentation/translations/zh_TW/process/1.Intro.rst
@@ -11,7 +11,7 @@
 :校譯:
 
  吳想成 Wu XiangCheng <bobwxc@email.cn>
- 胡皓文 Hu Haowen <src.res.211@gmail.com>
+ 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 .. _tw_development_process_intro:
 
diff --git a/Documentation/translations/zh_TW/process/2.Process.rst b/Documentation/translations/zh_TW/process/2.Process.rst
index 49385d65c216..f45ddba6238f 100644
--- a/Documentation/translations/zh_TW/process/2.Process.rst
+++ b/Documentation/translations/zh_TW/process/2.Process.rst
@@ -11,7 +11,7 @@
 :校譯:
 
  吳想成 Wu XiangCheng <bobwxc@email.cn>
- 胡皓文 Hu Haowen <src.res.211@gmail.com>
+ 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 .. _tw_development_process:
 
diff --git a/Documentation/translations/zh_TW/process/3.Early-stage.rst b/Documentation/translations/zh_TW/process/3.Early-stage.rst
index a6959e6350f4..a58fc9e0ea99 100644
--- a/Documentation/translations/zh_TW/process/3.Early-stage.rst
+++ b/Documentation/translations/zh_TW/process/3.Early-stage.rst
@@ -11,7 +11,7 @@
 :校譯:
 
  吳想成 Wu XiangCheng <bobwxc@email.cn>
- 胡皓文 Hu Haowen <src.res.211@gmail.com>
+ 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 .. _tw_development_early_stage:
 
diff --git a/Documentation/translations/zh_TW/process/4.Coding.rst b/Documentation/translations/zh_TW/process/4.Coding.rst
index 7a4e01eabd81..bdd2abe4daf4 100644
--- a/Documentation/translations/zh_TW/process/4.Coding.rst
+++ b/Documentation/translations/zh_TW/process/4.Coding.rst
@@ -11,7 +11,7 @@
 :校譯:
 
  吳想成 Wu XiangCheng <bobwxc@email.cn>
- 胡皓文 Hu Haowen <src.res.211@gmail.com>
+ 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 .. _tw_development_coding:
 
diff --git a/Documentation/translations/zh_TW/process/5.Posting.rst b/Documentation/translations/zh_TW/process/5.Posting.rst
index d398dda427aa..7d66a1c638be 100644
--- a/Documentation/translations/zh_TW/process/5.Posting.rst
+++ b/Documentation/translations/zh_TW/process/5.Posting.rst
@@ -11,7 +11,7 @@
 :校譯:
 
  吳想成 Wu XiangCheng <bobwxc@email.cn>
- 胡皓文 Hu Haowen <src.res.211@gmail.com>
+ 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 .. _tw_development_posting:
 
diff --git a/Documentation/translations/zh_TW/process/6.Followthrough.rst b/Documentation/translations/zh_TW/process/6.Followthrough.rst
index bcc885ae1b8e..f3b195966632 100644
--- a/Documentation/translations/zh_TW/process/6.Followthrough.rst
+++ b/Documentation/translations/zh_TW/process/6.Followthrough.rst
@@ -11,7 +11,7 @@
 :校譯:
 
  吳想成 Wu XiangCheng <bobwxc@email.cn>
- 胡皓文 Hu Haowen <src.res.211@gmail.com>
+ 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 .. _tw_development_followthrough:
 
diff --git a/Documentation/translations/zh_TW/process/7.AdvancedTopics.rst b/Documentation/translations/zh_TW/process/7.AdvancedTopics.rst
index db74d8ca3f3b..b449d67e3ad9 100644
--- a/Documentation/translations/zh_TW/process/7.AdvancedTopics.rst
+++ b/Documentation/translations/zh_TW/process/7.AdvancedTopics.rst
@@ -11,7 +11,7 @@
 :校譯:
 
  吳想成 Wu XiangCheng <bobwxc@email.cn>
- 胡皓文 Hu Haowen <src.res.211@gmail.com>
+ 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 .. _tw_development_advancedtopics:
 
diff --git a/Documentation/translations/zh_TW/process/8.Conclusion.rst b/Documentation/translations/zh_TW/process/8.Conclusion.rst
index a0c00741f912..d1634421b62c 100644
--- a/Documentation/translations/zh_TW/process/8.Conclusion.rst
+++ b/Documentation/translations/zh_TW/process/8.Conclusion.rst
@@ -10,7 +10,7 @@
 :校譯:
 
  吳想成 Wu XiangCheng <bobwxc@email.cn>
- 胡皓文 Hu Haowen <src.res.211@gmail.com>
+ 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 .. _tw_development_conclusion:
 
diff --git a/Documentation/translations/zh_TW/process/code-of-conduct-interpretation.rst b/Documentation/translations/zh_TW/process/code-of-conduct-interpretation.rst
index 48df918000e9..fbe66b001322 100644
--- a/Documentation/translations/zh_TW/process/code-of-conduct-interpretation.rst
+++ b/Documentation/translations/zh_TW/process/code-of-conduct-interpretation.rst
@@ -4,7 +4,7 @@
 
 :Original: :ref:`Documentation/process/code-of-conduct-interpretation.rst <code_of_conduct_interpretation>`
 :Translator: Alex Shi <alex.shi@linux.alibaba.com>
-             Hu Haowen <src.res.211@gmail.com>
+             Hu Haowen <2023002089@link.tyut.edu.cn>
 
 .. _tw_code_of_conduct_interpretation:
 
diff --git a/Documentation/translations/zh_TW/process/code-of-conduct.rst b/Documentation/translations/zh_TW/process/code-of-conduct.rst
index a7a31de03526..d24f1695bd02 100644
--- a/Documentation/translations/zh_TW/process/code-of-conduct.rst
+++ b/Documentation/translations/zh_TW/process/code-of-conduct.rst
@@ -4,7 +4,7 @@
 
 :Original: :ref:`Documentation/process/code-of-conduct.rst <code_of_conduct>`
 :Translator: Alex Shi <alex.shi@linux.alibaba.com>
-             Hu Haowen <src.res.211@gmail.com>
+             Hu Haowen <2023002089@link.tyut.edu.cn>
 
 .. _tw_code_of_conduct:
 
diff --git a/Documentation/translations/zh_TW/process/coding-style.rst b/Documentation/translations/zh_TW/process/coding-style.rst
index 5749363de421..f11dbb65ca21 100644
--- a/Documentation/translations/zh_TW/process/coding-style.rst
+++ b/Documentation/translations/zh_TW/process/coding-style.rst
@@ -17,7 +17,7 @@
  - 管旭東 Xudong Guan <xudong.guan@gmail.com>
  - Li Zefan <lizf@cn.fujitsu.com>
  - Wang Chen <wangchen@cn.fujitsu.com>
- - Hu Haowen <src.res.211@gmail.com>
+ - Hu Haowen <2023002089@link.tyut.edu.cn>
 
 Linux 內核代碼風格
 ==================
diff --git a/Documentation/translations/zh_TW/process/development-process.rst b/Documentation/translations/zh_TW/process/development-process.rst
index 7d803d3db89e..6eb49d317046 100644
--- a/Documentation/translations/zh_TW/process/development-process.rst
+++ b/Documentation/translations/zh_TW/process/development-process.rst
@@ -4,7 +4,7 @@
 
 :Original: :ref:`Documentation/process/development-process.rst <development_process_main>`
 :Translator: Alex Shi <alex.shi@linux.alibaba.com>
-             Hu Haowen <src.res.211@gmail.com>
+             Hu Haowen <2023002089@link.tyut.edu.cn>
 
 .. _tw_development_process_main:
 
diff --git a/Documentation/translations/zh_TW/process/email-clients.rst b/Documentation/translations/zh_TW/process/email-clients.rst
index 55e10d3fc28a..a5ac9400a9f5 100644
--- a/Documentation/translations/zh_TW/process/email-clients.rst
+++ b/Documentation/translations/zh_TW/process/email-clients.rst
@@ -15,7 +15,7 @@
  - Yinglin Luan <synmyth@gmail.com>
  - Xiaochen Wang <wangxiaochen0@gmail.com>
  - yaxinsn <yaxinsn@163.com>
- - Hu Haowen <src.res.211@gmail.com>
+ - Hu Haowen <2023002089@link.tyut.edu.cn>
 
 Linux郵件客戶端配置信息
 =======================
diff --git a/Documentation/translations/zh_TW/process/embargoed-hardware-issues.rst b/Documentation/translations/zh_TW/process/embargoed-hardware-issues.rst
index b9f6ab7b6666..3cce7db2ab7e 100644
--- a/Documentation/translations/zh_TW/process/embargoed-hardware-issues.rst
+++ b/Documentation/translations/zh_TW/process/embargoed-hardware-issues.rst
@@ -4,7 +4,7 @@
 
 :Original: :ref:`Documentation/process/embargoed-hardware-issues.rst <embargoed_hardware_issues>`
 :Translator: Alex Shi <alex.shi@linux.alibaba.com>
-             Hu Haowen <src.res.211@gmail.com>
+             Hu Haowen <2023002089@link.tyut.edu.cn>
 
 被限制的硬件問題
 ================
diff --git a/Documentation/translations/zh_TW/process/howto.rst b/Documentation/translations/zh_TW/process/howto.rst
index 306f5b77b4b8..80c416483e73 100644
--- a/Documentation/translations/zh_TW/process/howto.rst
+++ b/Documentation/translations/zh_TW/process/howto.rst
@@ -16,7 +16,7 @@
                    鍾宇  TripleX Chung <xxx.phy@gmail.com>
                    陳琦  Maggie Chen <chenqi@beyondsoft.com>
                    王聰  Wang Cong <xiyou.wangcong@gmail.com>
-                   胡皓文 Hu Haowen <src.res.211@gmail.com>
+                   胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 如何參與Linux內核開發
 =====================
diff --git a/Documentation/translations/zh_TW/process/index.rst b/Documentation/translations/zh_TW/process/index.rst
index 6a0d98b2f9ee..65922d9faa20 100644
--- a/Documentation/translations/zh_TW/process/index.rst
+++ b/Documentation/translations/zh_TW/process/index.rst
@@ -9,7 +9,7 @@
 
 :Original: :ref:`Documentation/process/index.rst <process_index>`
 :Translator: Alex Shi <alex.shi@linux.alibaba.com>
-             Hu Haowen <src.res.211@gmail.com>
+             Hu Haowen <2023002089@link.tyut.edu.cn>
 
 .. _tw_process_index:
 
diff --git a/Documentation/translations/zh_TW/process/kernel-driver-statement.rst b/Documentation/translations/zh_TW/process/kernel-driver-statement.rst
index e967089d2e1f..23d5cae9685b 100644
--- a/Documentation/translations/zh_TW/process/kernel-driver-statement.rst
+++ b/Documentation/translations/zh_TW/process/kernel-driver-statement.rst
@@ -6,7 +6,7 @@
 
 :Original: :ref:`Documentation/process/kernel-driver-statement.rst <process_statement_driver>`
 :Translator: Alex Shi <alex.shi@linux.alibaba.com>
-             Hu Haowen <src.res.211@gmail.com>
+             Hu Haowen <2023002089@link.tyut.edu.cn>
 
 內核驅動聲明
 ------------
diff --git a/Documentation/translations/zh_TW/process/kernel-enforcement-statement.rst b/Documentation/translations/zh_TW/process/kernel-enforcement-statement.rst
index 2861f4a15721..524eb4ac26cc 100644
--- a/Documentation/translations/zh_TW/process/kernel-enforcement-statement.rst
+++ b/Documentation/translations/zh_TW/process/kernel-enforcement-statement.rst
@@ -6,7 +6,7 @@
 
 :Original: :ref:`Documentation/process/kernel-enforcement-statement.rst <process_statement_kernel>`
 :Translator: Alex Shi <alex.shi@linux.alibaba.com>
-             Hu Haowen <src.res.211@gmail.com>
+             Hu Haowen <2023002089@link.tyut.edu.cn>
 
 Linux 內核執行聲明
 ------------------
diff --git a/Documentation/translations/zh_TW/process/license-rules.rst b/Documentation/translations/zh_TW/process/license-rules.rst
index 2c43bcf2ac79..594255856b68 100644
--- a/Documentation/translations/zh_TW/process/license-rules.rst
+++ b/Documentation/translations/zh_TW/process/license-rules.rst
@@ -4,7 +4,7 @@
 
 :Original: :ref:`Documentation/process/license-rules.rst <kernel_licensing>`
 :Translator: Alex Shi <alex.shi@linux.alibaba.com>
-             Hu Haowen <src.res.211@gmail.com>
+             Hu Haowen <2023002089@link.tyut.edu.cn>
 
 .. _tw_kernel_licensing:
 
diff --git a/Documentation/translations/zh_TW/process/magic-number.rst b/Documentation/translations/zh_TW/process/magic-number.rst
index 5657d5cd18d4..199cd5d63973 100644
--- a/Documentation/translations/zh_TW/process/magic-number.rst
+++ b/Documentation/translations/zh_TW/process/magic-number.rst
@@ -12,7 +12,7 @@
         中文版維護者： 賈威威 Jia Wei Wei <harryxiyou@gmail.com>
         中文版翻譯者： 賈威威 Jia Wei Wei <harryxiyou@gmail.com>
         中文版校譯者： 賈威威 Jia Wei Wei <harryxiyou@gmail.com>
-                      胡皓文 Hu Haowen <src.res.211@gmail.com>
+                      胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 Linux 魔術數
 ============
diff --git a/Documentation/translations/zh_TW/process/management-style.rst b/Documentation/translations/zh_TW/process/management-style.rst
index f3913e3c159d..7cb912e89032 100644
--- a/Documentation/translations/zh_TW/process/management-style.rst
+++ b/Documentation/translations/zh_TW/process/management-style.rst
@@ -4,7 +4,7 @@
 
 :Original: :ref:`Documentation/process/management-style.rst <managementstyle>`
 :Translator: Alex Shi <alex.shi@linux.alibaba.com>
-             Hu Haowen <src.res.211@gmail.com>
+             Hu Haowen <2023002089@link.tyut.edu.cn>
 
 .. _tw_managementstyle:
 
diff --git a/Documentation/translations/zh_TW/process/programming-language.rst b/Documentation/translations/zh_TW/process/programming-language.rst
index e33389676eed..d2c64a5599e8 100644
--- a/Documentation/translations/zh_TW/process/programming-language.rst
+++ b/Documentation/translations/zh_TW/process/programming-language.rst
@@ -4,7 +4,7 @@
 
 :Original: :ref:`Documentation/process/programming-language.rst <programming_language>`
 :Translator: Alex Shi <alex.shi@linux.alibaba.com>
-             Hu Haowen <src.res.211@gmail.com>
+             Hu Haowen <2023002089@link.tyut.edu.cn>
 
 .. _tw_programming_language:
 
diff --git a/Documentation/translations/zh_TW/process/stable-api-nonsense.rst b/Documentation/translations/zh_TW/process/stable-api-nonsense.rst
index 6839d25bb22a..4b8597fed5ae 100644
--- a/Documentation/translations/zh_TW/process/stable-api-nonsense.rst
+++ b/Documentation/translations/zh_TW/process/stable-api-nonsense.rst
@@ -12,7 +12,7 @@
         中文版維護者： 鍾宇  TripleX Chung <xxx.phy@gmail.com>
         中文版翻譯者： 鍾宇  TripleX Chung <xxx.phy@gmail.com>
         中文版校譯者： 李陽  Li Yang <leoyang.li@nxp.com>
-                      胡皓文 Hu Haowen <src.res.211@gmail.com>
+                      胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 Linux 內核驅動接口
 ==================
diff --git a/Documentation/translations/zh_TW/process/stable-kernel-rules.rst b/Documentation/translations/zh_TW/process/stable-kernel-rules.rst
index bd82a8ff3969..2f8f064f8629 100644
--- a/Documentation/translations/zh_TW/process/stable-kernel-rules.rst
+++ b/Documentation/translations/zh_TW/process/stable-kernel-rules.rst
@@ -15,7 +15,7 @@
         中文版校譯者：
             - 李陽  Li Yang <leoyang.li@nxp.com>
             - Kangkai Yin <e12051@motorola.com>
-            - 胡皓文 Hu Haowen <src.res.211@gmail.com>
+            - 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 所有你想知道的事情 - 關於linux穩定版發佈
 ========================================
diff --git a/Documentation/translations/zh_TW/process/submit-checklist.rst b/Documentation/translations/zh_TW/process/submit-checklist.rst
index 942962d1e2f4..e57ef0f99e0c 100644
--- a/Documentation/translations/zh_TW/process/submit-checklist.rst
+++ b/Documentation/translations/zh_TW/process/submit-checklist.rst
@@ -6,7 +6,7 @@
 :Translator:
  - Alex Shi <alexs@kernel.org>
  - Wu XiangCheng <bobwxc@email.cn>
- - Hu Haowen <src.res.211@gmail.com>
+ - Hu Haowen <2023002089@link.tyut.edu.cn>
 
 .. _tw_submitchecklist:
 
diff --git a/Documentation/translations/zh_TW/process/submitting-patches.rst b/Documentation/translations/zh_TW/process/submitting-patches.rst
index 8272b3218b54..99fa0f2fe6f4 100644
--- a/Documentation/translations/zh_TW/process/submitting-patches.rst
+++ b/Documentation/translations/zh_TW/process/submitting-patches.rst
@@ -14,7 +14,7 @@
 :校譯:
  - 李陽 Li Yang <leoyang.li@nxp.com>
  - 王聰 Wang Cong <xiyou.wangcong@gmail.com>
- - 胡皓文 Hu Haowen <src.res.211@gmail.com>
+ - 胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 
 提交補丁：如何讓你的改動進入內核
diff --git a/Documentation/translations/zh_TW/process/volatile-considered-harmful.rst b/Documentation/translations/zh_TW/process/volatile-considered-harmful.rst
index a609620affb0..e2723f3cbbb0 100644
--- a/Documentation/translations/zh_TW/process/volatile-considered-harmful.rst
+++ b/Documentation/translations/zh_TW/process/volatile-considered-harmful.rst
@@ -17,7 +17,7 @@
         中文版校譯者： 張漢輝  Eugene Teo <eugeneteo@kernel.sg>
                        楊瑞  Dave Young <hidave.darkstar@gmail.com>
                        時奎亮 Alex Shi <alex.shi@linux.alibaba.com>
-                       胡皓文 Hu Haowen <src.res.211@gmail.com>
+                       胡皓文 Hu Haowen <2023002089@link.tyut.edu.cn>
 
 爲什麼不應該使用“volatile”類型
 ==============================
diff --git a/Documentation/userspace-api/index.rst b/Documentation/userspace-api/index.rst
index 031df47a7c19..8be8b1979194 100644
--- a/Documentation/userspace-api/index.rst
+++ b/Documentation/userspace-api/index.rst
@@ -33,6 +33,7 @@ place where this information is gathered.
    sysfs-platform_profile
    vduse
    futex2
+   lsm
 
 .. only::  subproject and html
 
diff --git a/Documentation/userspace-api/landlock.rst b/Documentation/userspace-api/landlock.rst
index 2e3822677061..68498ca64dc9 100644
--- a/Documentation/userspace-api/landlock.rst
+++ b/Documentation/userspace-api/landlock.rst
@@ -75,7 +75,8 @@ to be explicit about the denied-by-default access rights.
             LANDLOCK_ACCESS_FS_MAKE_BLOCK |
             LANDLOCK_ACCESS_FS_MAKE_SYM |
             LANDLOCK_ACCESS_FS_REFER |
-            LANDLOCK_ACCESS_FS_TRUNCATE,
+            LANDLOCK_ACCESS_FS_TRUNCATE |
+            LANDLOCK_ACCESS_FS_IOCTL,
         .handled_access_net =
             LANDLOCK_ACCESS_NET_BIND_TCP |
             LANDLOCK_ACCESS_NET_CONNECT_TCP,
@@ -84,10 +85,10 @@ to be explicit about the denied-by-default access rights.
 Because we may not know on which kernel version an application will be
 executed, it is safer to follow a best-effort security approach.  Indeed, we
 should try to protect users as much as possible whatever the kernel they are
-using.  To avoid binary enforcement (i.e. either all security features or
-none), we can leverage a dedicated Landlock command to get the current version
-of the Landlock ABI and adapt the handled accesses.  Let's check if we should
-remove access rights which are only supported in higher versions of the ABI.
+using.
+
+To be compatible with older Linux versions, we detect the available Landlock ABI
+version, and only use the available subset of access rights:
 
 .. code-block:: c
 
@@ -113,6 +114,10 @@ remove access rights which are only supported in higher versions of the ABI.
         ruleset_attr.handled_access_net &=
             ~(LANDLOCK_ACCESS_NET_BIND_TCP |
               LANDLOCK_ACCESS_NET_CONNECT_TCP);
+        __attribute__((fallthrough));
+    case 4:
+        /* Removes LANDLOCK_ACCESS_FS_IOCTL for ABI < 5 */
+        ruleset_attr.handled_access_fs &= ~LANDLOCK_ACCESS_FS_IOCTL;
     }
 
 This enables to create an inclusive ruleset that will contain our rules.
@@ -224,6 +229,7 @@ access rights per directory enables to change the location of such directory
 without relying on the destination directory access rights (except those that
 are required for this operation, see ``LANDLOCK_ACCESS_FS_REFER``
 documentation).
+
 Having self-sufficient hierarchies also helps to tighten the required access
 rights to the minimal set of data.  This also helps avoid sinkhole directories,
 i.e.  directories where data can be linked to but not linked from.  However,
@@ -317,18 +323,24 @@ It should also be noted that truncating files does not require the
 system call, this can also be done through :manpage:`open(2)` with the flags
 ``O_RDONLY | O_TRUNC``.
 
-When opening a file, the availability of the ``LANDLOCK_ACCESS_FS_TRUNCATE``
-right is associated with the newly created file descriptor and will be used for
-subsequent truncation attempts using :manpage:`ftruncate(2)`.  The behavior is
-similar to opening a file for reading or writing, where permissions are checked
-during :manpage:`open(2)`, but not during the subsequent :manpage:`read(2)` and
+The truncate right is associated with the opened file (see below).
+
+Rights associated with file descriptors
+---------------------------------------
+
+When opening a file, the availability of the ``LANDLOCK_ACCESS_FS_TRUNCATE`` and
+``LANDLOCK_ACCESS_FS_IOCTL`` rights is associated with the newly created file
+descriptor and will be used for subsequent truncation and ioctl attempts using
+:manpage:`ftruncate(2)` and :manpage:`ioctl(2)`.  The behavior is similar to
+opening a file for reading or writing, where permissions are checked during
+:manpage:`open(2)`, but not during the subsequent :manpage:`read(2)` and
 :manpage:`write(2)` calls.
 
-As a consequence, it is possible to have multiple open file descriptors for the
-same file, where one grants the right to truncate the file and the other does
-not.  It is also possible to pass such file descriptors between processes,
-keeping their Landlock properties, even when these processes do not have an
-enforced Landlock ruleset.
+As a consequence, it is possible to have multiple open file descriptors
+referring to the same file, where one grants the truncate or ioctl right and the
+other does not.  It is also possible to pass such file descriptors between
+processes, keeping their Landlock properties, even when these processes do not
+have an enforced Landlock ruleset.
 
 Compatibility
 =============
@@ -457,6 +469,28 @@ Memory usage
 Kernel memory allocated to create rulesets is accounted and can be restricted
 by the Documentation/admin-guide/cgroup-v1/memory.rst.
 
+IOCTL support
+-------------
+
+The ``LANDLOCK_ACCESS_FS_IOCTL`` access right restricts the use of
+:manpage:`ioctl(2)`, but it only applies to newly opened files.  This means
+specifically that pre-existing file descriptors like stdin, stdout and stderr
+are unaffected.
+
+Users should be aware that TTY devices have traditionally permitted to control
+other processes on the same TTY through the ``TIOCSTI`` and ``TIOCLINUX`` IOCTL
+commands.  It is therefore recommended to close inherited TTY file descriptors,
+or to reopen them from ``/proc/self/fd/*`` without the
+``LANDLOCK_ACCESS_FS_IOCTL`` right, if possible.  The :manpage:`isatty(3)`
+function checks whether a given file descriptor is a TTY.
+
+Landlock's IOCTL support is coarse-grained at the moment, but may become more
+fine-grained in the future.  Until then, users are advised to establish the
+guarantees that they need through the file hierarchy, by only permitting the
+``LANDLOCK_ACCESS_FS_IOCTL`` right on files where it is really harmless.  In
+cases where you can control the mounts, the ``nodev`` mount option can help to
+rule out that device files can be accessed.
+
 Previous limitations
 ====================
 
@@ -494,6 +528,16 @@ bind and connect actions to only a set of allowed ports thanks to the new
 ``LANDLOCK_ACCESS_NET_BIND_TCP`` and ``LANDLOCK_ACCESS_NET_CONNECT_TCP``
 access rights.
 
+IOCTL (ABI < 5)
+---------------
+
+IOCTL operations could not be denied before the fifth Landlock ABI, so
+:manpage:`ioctl(2)` is always allowed when using a kernel that only supports an
+earlier ABI.
+
+Starting with the Landlock ABI version 5, it is possible to restrict the use of
+:manpage:`ioctl(2)` using the new ``LANDLOCK_ACCESS_FS_IOCTL`` access right.
+
 .. _kernel_support:
 
 Kernel support
diff --git a/Documentation/userspace-api/lsm.rst b/Documentation/userspace-api/lsm.rst
new file mode 100644
index 000000000000..a76da373841b
--- /dev/null
+++ b/Documentation/userspace-api/lsm.rst
@@ -0,0 +1,73 @@
+.. SPDX-License-Identifier: GPL-2.0
+.. Copyright (C) 2022 Casey Schaufler <casey@schaufler-ca.com>
+.. Copyright (C) 2022 Intel Corporation
+
+=====================================
+Linux Security Modules
+=====================================
+
+:Author: Casey Schaufler
+:Date: July 2023
+
+Linux security modules (LSM) provide a mechanism to implement
+additional access controls to the Linux security policies.
+
+The various security modules may support any of these attributes:
+
+``LSM_ATTR_CURRENT`` is the current, active security context of the
+process.
+The proc filesystem provides this value in ``/proc/self/attr/current``.
+This is supported by the SELinux, Smack and AppArmor security modules.
+Smack also provides this value in ``/proc/self/attr/smack/current``.
+AppArmor also provides this value in ``/proc/self/attr/apparmor/current``.
+
+``LSM_ATTR_EXEC`` is the security context of the process at the time the
+current image was executed.
+The proc filesystem provides this value in ``/proc/self/attr/exec``.
+This is supported by the SELinux and AppArmor security modules.
+AppArmor also provides this value in ``/proc/self/attr/apparmor/exec``.
+
+``LSM_ATTR_FSCREATE`` is the security context of the process used when
+creating file system objects.
+The proc filesystem provides this value in ``/proc/self/attr/fscreate``.
+This is supported by the SELinux security module.
+
+``LSM_ATTR_KEYCREATE`` is the security context of the process used when
+creating key objects.
+The proc filesystem provides this value in ``/proc/self/attr/keycreate``.
+This is supported by the SELinux security module.
+
+``LSM_ATTR_PREV`` is the security context of the process at the time the
+current security context was set.
+The proc filesystem provides this value in ``/proc/self/attr/prev``.
+This is supported by the SELinux and AppArmor security modules.
+AppArmor also provides this value in ``/proc/self/attr/apparmor/prev``.
+
+``LSM_ATTR_SOCKCREATE`` is the security context of the process used when
+creating socket objects.
+The proc filesystem provides this value in ``/proc/self/attr/sockcreate``.
+This is supported by the SELinux security module.
+
+Kernel interface
+================
+
+Set a security attribute of the current process
+-----------------------------------------------
+
+.. kernel-doc:: security/lsm_syscalls.c
+    :identifiers: sys_lsm_set_self_attr
+
+Get the specified security attributes of the current process
+------------------------------------------------------------
+
+.. kernel-doc:: security/lsm_syscalls.c
+    :identifiers: sys_lsm_get_self_attr
+
+.. kernel-doc:: security/lsm_syscalls.c
+    :identifiers: sys_lsm_list_modules
+
+Additional documentation
+========================
+
+* Documentation/security/lsm.rst
+* Documentation/security/lsm-development.rst
diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst
index 7025b3751027..926241e23aeb 100644
--- a/Documentation/virt/kvm/api.rst
+++ b/Documentation/virt/kvm/api.rst
@@ -147,10 +147,29 @@ described as 'basic' will be available.
 The new VM has no virtual cpus and no memory.
 You probably want to use 0 as machine type.
 
+X86:
+^^^^
+
+Supported X86 VM types can be queried via KVM_CAP_VM_TYPES.
+
+S390:
+^^^^^
+
 In order to create user controlled virtual machines on S390, check
 KVM_CAP_S390_UCONTROL and use the flag KVM_VM_S390_UCONTROL as
 privileged user (CAP_SYS_ADMIN).
 
+MIPS:
+^^^^^
+
+To use hardware assisted virtualization on MIPS (VZ ASE) rather than
+the default trap & emulate implementation (which changes the virtual
+memory layout to fit in user mode), check KVM_CAP_MIPS_VZ and use the
+flag KVM_VM_MIPS_VZ.
+
+ARM64:
+^^^^^^
+
 On arm64, the physical address size for a VM (IPA Size limit) is limited
 to 40bits by default. The limit can be configured if the host supports the
 extension KVM_CAP_ARM_VM_IPA_SIZE. When supported, use
@@ -6192,6 +6211,130 @@ to know what fields can be changed for the system register described by
 ``op0, op1, crn, crm, op2``. KVM rejects ID register values that describe a
 superset of the features supported by the system.
 
+4.140 KVM_SET_USER_MEMORY_REGION2
+---------------------------------
+
+:Capability: KVM_CAP_USER_MEMORY2
+:Architectures: all
+:Type: vm ioctl
+:Parameters: struct kvm_userspace_memory_region2 (in)
+:Returns: 0 on success, -1 on error
+
+KVM_SET_USER_MEMORY_REGION2 is an extension to KVM_SET_USER_MEMORY_REGION that
+allows mapping guest_memfd memory into a guest.  All fields shared with
+KVM_SET_USER_MEMORY_REGION identically.  Userspace can set KVM_MEM_GUEST_MEMFD
+in flags to have KVM bind the memory region to a given guest_memfd range of
+[guest_memfd_offset, guest_memfd_offset + memory_size].  The target guest_memfd
+must point at a file created via KVM_CREATE_GUEST_MEMFD on the current VM, and
+the target range must not be bound to any other memory region.  All standard
+bounds checks apply (use common sense).
+
+::
+
+  struct kvm_userspace_memory_region2 {
+	__u32 slot;
+	__u32 flags;
+	__u64 guest_phys_addr;
+	__u64 memory_size; /* bytes */
+	__u64 userspace_addr; /* start of the userspace allocated memory */
+	__u64 guest_memfd_offset;
+	__u32 guest_memfd;
+	__u32 pad1;
+	__u64 pad2[14];
+  };
+
+A KVM_MEM_GUEST_MEMFD region _must_ have a valid guest_memfd (private memory) and
+userspace_addr (shared memory).  However, "valid" for userspace_addr simply
+means that the address itself must be a legal userspace address.  The backing
+mapping for userspace_addr is not required to be valid/populated at the time of
+KVM_SET_USER_MEMORY_REGION2, e.g. shared memory can be lazily mapped/allocated
+on-demand.
+
+When mapping a gfn into the guest, KVM selects shared vs. private, i.e consumes
+userspace_addr vs. guest_memfd, based on the gfn's KVM_MEMORY_ATTRIBUTE_PRIVATE
+state.  At VM creation time, all memory is shared, i.e. the PRIVATE attribute
+is '0' for all gfns.  Userspace can control whether memory is shared/private by
+toggling KVM_MEMORY_ATTRIBUTE_PRIVATE via KVM_SET_MEMORY_ATTRIBUTES as needed.
+
+4.141 KVM_SET_MEMORY_ATTRIBUTES
+-------------------------------
+
+:Capability: KVM_CAP_MEMORY_ATTRIBUTES
+:Architectures: x86
+:Type: vm ioctl
+:Parameters: struct kvm_memory_attributes (in)
+:Returns: 0 on success, <0 on error
+
+KVM_SET_MEMORY_ATTRIBUTES allows userspace to set memory attributes for a range
+of guest physical memory.
+
+::
+
+  struct kvm_memory_attributes {
+	__u64 address;
+	__u64 size;
+	__u64 attributes;
+	__u64 flags;
+  };
+
+  #define KVM_MEMORY_ATTRIBUTE_PRIVATE           (1ULL << 3)
+
+The address and size must be page aligned.  The supported attributes can be
+retrieved via ioctl(KVM_CHECK_EXTENSION) on KVM_CAP_MEMORY_ATTRIBUTES.  If
+executed on a VM, KVM_CAP_MEMORY_ATTRIBUTES precisely returns the attributes
+supported by that VM.  If executed at system scope, KVM_CAP_MEMORY_ATTRIBUTES
+returns all attributes supported by KVM.  The only attribute defined at this
+time is KVM_MEMORY_ATTRIBUTE_PRIVATE, which marks the associated gfn as being
+guest private memory.
+
+Note, there is no "get" API.  Userspace is responsible for explicitly tracking
+the state of a gfn/page as needed.
+
+The "flags" field is reserved for future extensions and must be '0'.
+
+4.142 KVM_CREATE_GUEST_MEMFD
+----------------------------
+
+:Capability: KVM_CAP_GUEST_MEMFD
+:Architectures: none
+:Type: vm ioctl
+:Parameters: struct kvm_create_guest_memfd(in)
+:Returns: 0 on success, <0 on error
+
+KVM_CREATE_GUEST_MEMFD creates an anonymous file and returns a file descriptor
+that refers to it.  guest_memfd files are roughly analogous to files created
+via memfd_create(), e.g. guest_memfd files live in RAM, have volatile storage,
+and are automatically released when the last reference is dropped.  Unlike
+"regular" memfd_create() files, guest_memfd files are bound to their owning
+virtual machine (see below), cannot be mapped, read, or written by userspace,
+and cannot be resized  (guest_memfd files do however support PUNCH_HOLE).
+
+::
+
+  struct kvm_create_guest_memfd {
+	__u64 size;
+	__u64 flags;
+	__u64 reserved[6];
+  };
+
+Conceptually, the inode backing a guest_memfd file represents physical memory,
+i.e. is coupled to the virtual machine as a thing, not to a "struct kvm".  The
+file itself, which is bound to a "struct kvm", is that instance's view of the
+underlying memory, e.g. effectively provides the translation of guest addresses
+to host memory.  This allows for use cases where multiple KVM structures are
+used to manage a single virtual machine, e.g. when performing intrahost
+migration of a virtual machine.
+
+KVM currently only supports mapping guest_memfd via KVM_SET_USER_MEMORY_REGION2,
+and more specifically via the guest_memfd and guest_memfd_offset fields in
+"struct kvm_userspace_memory_region2", where guest_memfd_offset is the offset
+into the guest_memfd instance.  For a given guest_memfd file, there can be at
+most one mapping per page, i.e. binding multiple memory regions to a single
+guest_memfd range is not allowed (any number of memory regions can be bound to
+a single guest_memfd file, but the bound ranges must not overlap).
+
+See KVM_SET_USER_MEMORY_REGION2 for additional details.
+
 5. The kvm_run structure
 ========================
 
@@ -6824,6 +6967,30 @@ array field represents return values. The userspace should update the return
 values of SBI call before resuming the VCPU. For more details on RISC-V SBI
 spec refer, https://github.com/riscv/riscv-sbi-doc.
 
+::
+
+		/* KVM_EXIT_MEMORY_FAULT */
+		struct {
+  #define KVM_MEMORY_EXIT_FLAG_PRIVATE	(1ULL << 3)
+			__u64 flags;
+			__u64 gpa;
+			__u64 size;
+		} memory_fault;
+
+KVM_EXIT_MEMORY_FAULT indicates the vCPU has encountered a memory fault that
+could not be resolved by KVM.  The 'gpa' and 'size' (in bytes) describe the
+guest physical address range [gpa, gpa + size) of the fault.  The 'flags' field
+describes properties of the faulting access that are likely pertinent:
+
+ - KVM_MEMORY_EXIT_FLAG_PRIVATE - When set, indicates the memory fault occurred
+   on a private memory access.  When clear, indicates the fault occurred on a
+   shared access.
+
+Note!  KVM_EXIT_MEMORY_FAULT is unique among all KVM exit reasons in that it
+accompanies a return code of '-1', not '0'!  errno will always be set to EFAULT
+or EHWPOISON when KVM exits with KVM_EXIT_MEMORY_FAULT, userspace should assume
+kvm_run.exit_reason is stale/undefined for all other error numbers.
+
 ::
 
     /* KVM_EXIT_NOTIFY */
@@ -7858,6 +8025,27 @@ This capability is aimed to mitigate the threat that malicious VMs can
 cause CPU stuck (due to event windows don't open up) and make the CPU
 unavailable to host or other VMs.
 
+7.34 KVM_CAP_MEMORY_FAULT_INFO
+------------------------------
+
+:Architectures: x86
+:Returns: Informational only, -EINVAL on direct KVM_ENABLE_CAP.
+
+The presence of this capability indicates that KVM_RUN will fill
+kvm_run.memory_fault if KVM cannot resolve a guest page fault VM-Exit, e.g. if
+there is a valid memslot but no backing VMA for the corresponding host virtual
+address.
+
+The information in kvm_run.memory_fault is valid if and only if KVM_RUN returns
+an error with errno=EFAULT or errno=EHWPOISON *and* kvm_run.exit_reason is set
+to KVM_EXIT_MEMORY_FAULT.
+
+Note: Userspaces which attempt to resolve memory faults so that they can retry
+KVM_RUN are encouraged to guard against repeatedly receiving the same
+error/annotated fault.
+
+See KVM_EXIT_MEMORY_FAULT for more information.
+
 8. Other capabilities.
 ======================
 
@@ -8596,6 +8784,19 @@ block sizes is exposed in KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES as a
 64-bit bitmap (each bit describing a block size). The default value is
 0, to disable the eager page splitting.
 
+8.41 KVM_CAP_VM_TYPES
+---------------------
+
+:Capability: KVM_CAP_MEMORY_ATTRIBUTES
+:Architectures: x86
+:Type: system ioctl
+
+This capability returns a bitmap of support VM types.  The 1-setting of bit @n
+means the VM type with value @n is supported.  Possible values of @n are::
+
+  #define KVM_X86_DEFAULT_VM	0
+  #define KVM_X86_SW_PROTECTED_VM	1
+
 9. Known KVM API problems
 =========================
 
diff --git a/MAINTAINERS b/MAINTAINERS
index ea790149af79..5d1bef4d72bc 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -893,6 +893,14 @@ Q:	https://patchwork.kernel.org/project/linux-rdma/list/
 F:	drivers/infiniband/hw/efa/
 F:	include/uapi/rdma/efa-abi.h
 
+AMD AXI W1 DRIVER
+M:	Kris Chaplin <kris.chaplin@amd.com>
+R:	Thomas Delev <thomas.delev@amd.com>
+R:	Michal Simek <michal.simek@amd.com>
+S:	Maintained
+F:	Documentation/devicetree/bindings/w1/amd,axi-1wire-host.yaml
+F:	drivers/w1/masters/amd_axi_w1.c
+
 AMD CDX BUS DRIVER
 M:	Nipun Gupta <nipun.gupta@amd.com>
 M:	Nikhil Agarwal <nikhil.agarwal@amd.com>
@@ -2063,7 +2071,6 @@ ARM/CORESIGHT FRAMEWORK AND DRIVERS
 M:	Suzuki K Poulose <suzuki.poulose@arm.com>
 R:	Mike Leach <mike.leach@linaro.org>
 R:	James Clark <james.clark@arm.com>
-R:	Leo Yan <leo.yan@linaro.org>
 L:	coresight@lists.linaro.org (moderated for non-subscribers)
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
@@ -2405,7 +2412,6 @@ F:	drivers/memory/atmel*
 F:	drivers/watchdog/sama5d4_wdt.c
 F:	include/soc/at91/
 X:	drivers/input/touchscreen/atmel_mxt_ts.c
-X:	drivers/net/wireless/atmel/
 N:	at91
 N:	atmel
 
@@ -3309,13 +3315,6 @@ T:	git git://github.com/ndyer/linux.git
 F:	Documentation/devicetree/bindings/input/atmel,maxtouch.yaml
 F:	drivers/input/touchscreen/atmel_mxt_ts.c
 
-ATMEL WIRELESS DRIVER
-L:	linux-wireless@vger.kernel.org
-S:	Orphan
-W:	http://www.thekelleys.org.uk/atmel
-W:	http://atmelwlandriver.sourceforge.net/
-F:	drivers/net/wireless/atmel/atmel*
-
 ATOMIC INFRASTRUCTURE
 M:	Will Deacon <will@kernel.org>
 M:	Peter Zijlstra <peterz@infradead.org>
@@ -3350,13 +3349,17 @@ M:	Eric Paris <eparis@redhat.com>
 L:	audit@vger.kernel.org
 S:	Supported
 W:	https://github.com/linux-audit
-T:	git git://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/audit.git
+Q:	https://patchwork.kernel.org/project/audit/list
+B:	mailto:audit@vger.kernel.org
+P:	https://github.com/linux-audit/audit-kernel/blob/main/README.md
+T:	git https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/audit.git
 F:	include/asm-generic/audit_*.h
 F:	include/linux/audit.h
 F:	include/linux/audit_arch.h
 F:	include/uapi/linux/audit.h
 F:	kernel/audit*
 F:	lib/*audit.c
+K:	\baudit_[a-z_0-9]\+\b
 
 AUXILIARY BUS DRIVER
 M:	Greg Kroah-Hartman <gregkh@linuxfoundation.org>
@@ -3415,6 +3418,16 @@ W:	https://ez.analog.com/linux-software-drivers
 F:	Documentation/devicetree/bindings/hwmon/adi,axi-fan-control.yaml
 F:	drivers/hwmon/axi-fan-control.c
 
+AXI SPI ENGINE
+M:	Michael Hennerich <michael.hennerich@analog.com>
+M:	Nuno Sá <nuno.sa@analog.com>
+R:	David Lechner <dlechner@baylibre.com>
+L:	linux-spi@vger.kernel.org
+S:	Supported
+W:	https://ez.analog.com/linux-software-drivers
+F:	Documentation/devicetree/bindings/spi/adi,axi-spi-engine.yaml
+F:	drivers/spi/spi-axi-spi-engine.c
+
 AXXIA I2C CONTROLLER
 M:	Krzysztof Adamski <krzysztof.adamski@nokia.com>
 L:	linux-i2c@vger.kernel.org
@@ -3647,6 +3660,13 @@ S:	Maintained
 F:	Documentation/devicetree/bindings/iio/accel/bosch,bma400.yaml
 F:	drivers/iio/accel/bma400*
 
+BOSCH SENSORTEC BMI323 IMU IIO DRIVER
+M:	Jagath Jog J <jagathjog1996@gmail.com>
+L:	linux-iio@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/iio/imu/bosch,bmi323.yaml
+F:	drivers/iio/imu/bmi323/
+
 BPF JIT for ARM
 M:	Russell King <linux@armlinux.org.uk>
 M:	Puranjay Mohan <puranjay12@gmail.com>
@@ -6503,8 +6523,7 @@ T:	git git://anongit.freedesktop.org/drm/drm-misc
 F:	drivers/gpu/drm/sun4i/sun8i*
 
 DRM DRIVER FOR ARM PL111 CLCD
-M:	Emma Anholt <emma@anholt.net>
-S:	Supported
+S:	Orphan
 T:	git git://anongit.freedesktop.org/drm/drm-misc
 F:	drivers/gpu/drm/pl111/
 
@@ -6619,8 +6638,7 @@ F:	Documentation/devicetree/bindings/display/panel/himax,hx8394.yaml
 F:	drivers/gpu/drm/panel/panel-himax-hx8394.c
 
 DRM DRIVER FOR HX8357D PANELS
-M:	Emma Anholt <emma@anholt.net>
-S:	Maintained
+S:	Orphan
 T:	git git://anongit.freedesktop.org/drm/drm-misc
 F:	Documentation/devicetree/bindings/display/himax,hx8357d.txt
 F:	drivers/gpu/drm/tiny/hx8357d.c
@@ -7213,8 +7231,8 @@ F:	Documentation/devicetree/bindings/display/ti/
 F:	drivers/gpu/drm/omapdrm/
 
 DRM DRIVERS FOR V3D
-M:	Emma Anholt <emma@anholt.net>
 M:	Melissa Wen <mwen@igalia.com>
+M:	Maíra Canal <mcanal@igalia.com>
 S:	Supported
 T:	git git://anongit.freedesktop.org/drm/drm-misc
 F:	Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.yaml
@@ -7222,7 +7240,6 @@ F:	drivers/gpu/drm/v3d/
 F:	include/uapi/drm/v3d_drm.h
 
 DRM DRIVERS FOR VC4
-M:	Emma Anholt <emma@anholt.net>
 M:	Maxime Ripard <mripard@kernel.org>
 S:	Supported
 T:	git git://github.com/anholt/linux
@@ -7855,6 +7872,7 @@ R:	Yue Hu <huyue2@coolpad.com>
 R:	Jeffle Xu <jefflexu@linux.alibaba.com>
 L:	linux-erofs@lists.ozlabs.org
 S:	Maintained
+W:	https://erofs.docs.kernel.org
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git
 F:	Documentation/ABI/testing/sysfs-fs-erofs
 F:	Documentation/filesystems/erofs.rst
@@ -9627,6 +9645,7 @@ F:	drivers/crypto/hisilicon/sgl.c
 F:	include/linux/hisi_acc_qm.h
 
 HISILICON ROCE DRIVER
+M:	Chengchang Tang <tangchengchang@huawei.com>
 M:	Junxian Huang <huangjunxian6@hisilicon.com>
 L:	linux-rdma@vger.kernel.org
 S:	Maintained
@@ -9708,11 +9727,6 @@ S:	Maintained
 F:	Documentation/devicetree/bindings/iio/pressure/honeywell,mprls0025pa.yaml
 F:	drivers/iio/pressure/mprls0025pa.c
 
-HOST AP DRIVER
-L:	linux-wireless@vger.kernel.org
-S:	Obsolete
-F:	drivers/net/wireless/intersil/hostap/
-
 HP BIOSCFG DRIVER
 M:	Jorge Lopez <jorge.lopez2@hp.com>
 L:	platform-driver-x86@vger.kernel.org
@@ -10332,8 +10346,8 @@ IIO LIGHT SENSOR GAIN-TIME-SCALE HELPERS
 M:	Matti Vaittinen <mazziesaccount@gmail.com>
 L:	linux-iio@vger.kernel.org
 S:	Maintained
-F:	drivers/iio/light/gain-time-scale-helper.c
-F:	drivers/iio/light/gain-time-scale-helper.h
+F:	drivers/iio/industrialio-gts-helper.c
+F:	include/linux/iio/iio-gts-helper.h
 
 IIO MULTIPLEXER
 M:	Peter Rosin <peda@axentia.se>
@@ -10638,9 +10652,9 @@ M:	Rodrigo Vivi <rodrigo.vivi@intel.com>
 M:	Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
 L:	intel-gfx@lists.freedesktop.org
 S:	Supported
-W:	https://01.org/linuxgraphics/
+W:	https://drm.pages.freedesktop.org/intel-docs/
 Q:	http://patchwork.freedesktop.org/project/intel-gfx/
-B:	https://gitlab.freedesktop.org/drm/intel/-/wikis/How-to-file-i915-bugs
+B:	https://drm.pages.freedesktop.org/intel-docs/how-to-file-i915-bugs.html
 C:	irc://irc.oftc.net/intel-gfx
 T:	git git://anongit.freedesktop.org/drm-intel
 F:	Documentation/ABI/testing/sysfs-driver-intel-i915-hwmon
@@ -11024,7 +11038,6 @@ F:	drivers/net/wireless/intel/iwlwifi/
 
 INTEL WMI SLIM BOOTLOADER (SBL) FIRMWARE UPDATE DRIVER
 M:	Jithu Joseph <jithu.joseph@intel.com>
-R:	Maurice Ma <maurice.ma@intel.com>
 S:	Maintained
 W:	https://slimbootloader.github.io/security/firmware-update.html
 F:	drivers/platform/x86/intel/wmi/sbl-fw-update.c
@@ -12208,6 +12221,13 @@ F:	include/linux/nd.h
 F:	include/uapi/linux/ndctl.h
 F:	tools/testing/nvdimm/
 
+LIBRARY CODE
+M:	Andrew Morton <akpm@linux-foundation.org>
+L:	linux-kernel@vger.kernel.org
+S:	Supported
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-nonmm-unstable
+F:	lib/*
+
 LICENSES and SPDX stuff
 M:	Thomas Gleixner <tglx@linutronix.de>
 M:	Greg Kroah-Hartman <gregkh@linuxfoundation.org>
@@ -12446,6 +12466,12 @@ T:	git git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git for-next/har
 F:	Documentation/admin-guide/LSM/LoadPin.rst
 F:	security/loadpin/
 
+LOCKDOWN SECURITY MODULE
+L:	linux-security-module@vger.kernel.org
+S:	Odd Fixes
+T:	git https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/lsm.git
+F:	security/lockdown/
+
 LOCKING PRIMITIVES
 M:	Peter Zijlstra <peterz@infradead.org>
 M:	Ingo Molnar <mingo@redhat.com>
@@ -13778,7 +13804,6 @@ F:	drivers/net/ethernet/mellanox/mlxfw/
 MELLANOX HARDWARE PLATFORM SUPPORT
 M:	Hans de Goede <hdegoede@redhat.com>
 M:	Ilpo Järvinen <ilpo.jarvinen@linux.intel.com>
-M:	Mark Gross <markgross@kernel.org>
 M:	Vadim Pasternak <vadimp@nvidia.com>
 L:	platform-driver-x86@vger.kernel.org
 S:	Supported
@@ -14387,7 +14412,6 @@ F:	drivers/platform/surface/surface_gpe.c
 MICROSOFT SURFACE HARDWARE PLATFORM SUPPORT
 M:	Hans de Goede <hdegoede@redhat.com>
 M:	Ilpo Järvinen <ilpo.jarvinen@linux.intel.com>
-M:	Mark Gross <markgross@kernel.org>
 M:	Maximilian Luz <luzmaximilian@gmail.com>
 L:	platform-driver-x86@vger.kernel.org
 S:	Maintained
@@ -14994,6 +15018,7 @@ M:	Jakub Kicinski <kuba@kernel.org>
 M:	Paolo Abeni <pabeni@redhat.com>
 L:	netdev@vger.kernel.org
 S:	Maintained
+P:	Documentation/process/maintainer-netdev.rst
 Q:	https://patchwork.kernel.org/project/netdevbpf/list/
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net.git
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git
@@ -15045,6 +15070,7 @@ M:	Jakub Kicinski <kuba@kernel.org>
 M:	Paolo Abeni <pabeni@redhat.com>
 L:	netdev@vger.kernel.org
 S:	Maintained
+P:	Documentation/process/maintainer-netdev.rst
 Q:	https://patchwork.kernel.org/project/netdevbpf/list/
 B:	mailto:netdev@vger.kernel.org
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net.git
@@ -15055,6 +15081,7 @@ F:	Documentation/networking/
 F:	Documentation/process/maintainer-netdev.rst
 F:	Documentation/userspace-api/netlink/
 F:	include/linux/in.h
+F:	include/linux/indirect_call_wrapper.h
 F:	include/linux/net.h
 F:	include/linux/netdevice.h
 F:	include/net/
@@ -16268,13 +16295,6 @@ T:	git git://git.kernel.org/pub/scm/linux/kernel/git/hubcap/linux.git
 F:	Documentation/filesystems/orangefs.rst
 F:	fs/orangefs/
 
-ORINOCO DRIVER
-L:	linux-wireless@vger.kernel.org
-S:	Orphan
-W:	https://wireless.wiki.kernel.org/en/users/Drivers/orinoco
-W:	http://www.nongnu.org/orinoco/
-F:	drivers/net/wireless/intersil/orinoco/
-
 OV2659 OMNIVISION SENSOR DRIVER
 M:	"Lad, Prabhakar" <prabhakar.csengg@gmail.com>
 L:	linux-media@vger.kernel.org
@@ -18168,11 +18188,6 @@ F:	drivers/ras/
 F:	include/linux/ras.h
 F:	include/ras/ras_event.h
 
-RAYLINK/WEBGEAR 802.11 WIRELESS LAN DRIVER
-L:	linux-wireless@vger.kernel.org
-S:	Orphan
-F:	drivers/net/wireless/legacy/ray*
-
 RC-CORE / LIRC FRAMEWORK
 M:	Sean Young <sean@mess.org>
 L:	linux-media@vger.kernel.org
@@ -19218,7 +19233,6 @@ R:	Alim Akhtar <alim.akhtar@samsung.com>
 L:	linux-samsung-soc@vger.kernel.org
 S:	Maintained
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git
-T:	git git://git.kernel.org/pub/scm/linux/kernel/git/snawrocki/clk.git
 F:	Documentation/devicetree/bindings/clock/samsung,*.yaml
 F:	Documentation/devicetree/bindings/clock/samsung,s3c*
 F:	drivers/clk/samsung/
@@ -19506,12 +19520,17 @@ SECURITY SUBSYSTEM
 M:	Paul Moore <paul@paul-moore.com>
 M:	James Morris <jmorris@namei.org>
 M:	"Serge E. Hallyn" <serge@hallyn.com>
-L:	linux-security-module@vger.kernel.org (suggested Cc:)
+L:	linux-security-module@vger.kernel.org
 S:	Supported
-W:	http://kernsec.org/
-T:	git git://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/lsm.git
+Q:	https://patchwork.kernel.org/project/linux-security-module/list
+B:	mailto:linux-security-module@vger.kernel.org
+P:	https://github.com/LinuxSecurityModule/kernel/blob/main/README.md
+T:	git https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/lsm.git
+F:	include/uapi/linux/lsm.h
 F:	security/
+F:	tools/testing/selftests/lsm/
 X:	security/selinux/
+K:	\bsecurity_[a-z_0-9]\+\b
 
 SELINUX SECURITY MODULE
 M:	Paul Moore <paul@paul-moore.com>
@@ -19519,9 +19538,11 @@ M:	Stephen Smalley <stephen.smalley.work@gmail.com>
 M:	Eric Paris <eparis@parisplace.org>
 L:	selinux@vger.kernel.org
 S:	Supported
-W:	https://selinuxproject.org
 W:	https://github.com/SELinuxProject
-T:	git git://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/selinux.git
+Q:	https://patchwork.kernel.org/project/selinux/list
+B:	mailto:selinux@vger.kernel.org
+P:	https://github.com/SELinuxProject/selinux-kernel/blob/main/README.md
+T:	git https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/selinux.git
 F:	Documentation/ABI/removed/sysfs-selinux-checkreqprot
 F:	Documentation/ABI/removed/sysfs-selinux-disable
 F:	Documentation/admin-guide/LSM/SELinux.rst
@@ -20425,7 +20446,7 @@ F:	drivers/pinctrl/spear/
 SPI NOR SUBSYSTEM
 M:	Tudor Ambarus <tudor.ambarus@linaro.org>
 M:	Pratyush Yadav <pratyush@kernel.org>
-R:	Michael Walle <michael@walle.cc>
+M:	Michael Walle <michael@walle.cc>
 L:	linux-mtd@lists.infradead.org
 S:	Maintained
 W:	http://www.linux-mtd.infradead.org/
@@ -22120,7 +22141,7 @@ F:	kernel/trace/trace_osnoise.c
 F:	kernel/trace/trace_sched_wakeup.c
 
 TRADITIONAL CHINESE DOCUMENTATION
-M:	Hu Haowen <src.res.211@gmail.com>
+M:	Hu Haowen <2023002089@link.tyut.edu.cn>
 S:	Maintained
 W:	https://github.com/srcres258/linux-doc
 T:	git git://github.com/srcres258/linux-doc.git doc-zh-tw
@@ -22679,11 +22700,6 @@ F:	drivers/usb/gadget/function/*uvc*
 F:	drivers/usb/gadget/legacy/webcam.c
 F:	include/uapi/linux/usb/g_uvc.h
 
-USB WIRELESS RNDIS DRIVER (rndis_wlan)
-L:	linux-wireless@vger.kernel.org
-S:	Orphan
-F:	drivers/net/wireless/legacy/rndis_wlan.c
-
 USB XHCI DRIVER
 M:	Mathias Nyman <mathias.nyman@intel.com>
 L:	linux-usb@vger.kernel.org
@@ -22691,12 +22707,6 @@ S:	Supported
 F:	drivers/usb/host/pci-quirks*
 F:	drivers/usb/host/xhci*
 
-USB ZD1201 DRIVER
-L:	linux-wireless@vger.kernel.org
-S:	Orphan
-W:	http://linux-lc100020.sourceforge.net
-F:	drivers/net/wireless/zydas/zd1201.*
-
 USER DATAGRAM PROTOCOL (UDP)
 M:	Willem de Bruijn <willemdebruijn.kernel@gmail.com>
 S:	Maintained
@@ -23498,11 +23508,6 @@ M:	Miloslav Trmac <mitr@volny.cz>
 S:	Maintained
 F:	drivers/input/misc/wistron_btns.c
 
-WL3501 WIRELESS PCMCIA CARD DRIVER
-L:	linux-wireless@vger.kernel.org
-S:	Orphan
-F:	drivers/net/wireless/legacy/wl3501*
-
 WMI BINARY MOF DRIVER
 M:	Armin Wolf <W_Armin@gmx.de>
 R:	Thomas Weißschuh <linux@weissschuh.net>
@@ -23664,7 +23669,6 @@ F:	drivers/platform/x86/x86-android-tablets/
 X86 PLATFORM DRIVERS
 M:	Hans de Goede <hdegoede@redhat.com>
 M:	Ilpo Järvinen <ilpo.jarvinen@linux.intel.com>
-M:	Mark Gross <markgross@kernel.org>
 L:	platform-driver-x86@vger.kernel.org
 S:	Maintained
 Q:	https://patchwork.kernel.org/project/platform-driver-x86/list/
diff --git a/Makefile b/Makefile
index 724c79bebe72..2cfd71ae3a86 100644
--- a/Makefile
+++ b/Makefile
@@ -982,6 +982,8 @@ NOSTDINC_FLAGS += -nostdinc
 # perform bounds checking.
 KBUILD_CFLAGS += $(call cc-option, -fstrict-flex-arrays=3)
 
+KBUILD_CFLAGS += $(call cc-option, -Wstringop-overflow)
+
 # disable invalid "can't wrap" optimizations for signed / pointers
 KBUILD_CFLAGS	+= -fno-strict-overflow
 
@@ -1576,7 +1578,8 @@ help:
 	 echo  '                    (default: $(INSTALL_HDR_PATH))'; \
 	 echo  ''
 	@echo  'Static analysers:'
-	@echo  '  checkstack      - Generate a list of stack hogs'
+	@echo  '  checkstack      - Generate a list of stack hogs and consider all functions'
+	@echo  '                    with a stack size larger than MINSTACKSIZE (default: 100)'
 	@echo  '  versioncheck    - Sanity check on version.h usage'
 	@echo  '  includecheck    - Check for duplicate included header files'
 	@echo  '  export_report   - List the usages of all exported symbols'
@@ -2016,9 +2019,10 @@ CHECKSTACK_ARCH := $(SUBARCH)
 else
 CHECKSTACK_ARCH := $(ARCH)
 endif
+MINSTACKSIZE	?= 100
 checkstack:
 	$(OBJDUMP) -d vmlinux $$(find . -name '*.ko') | \
-	$(PERL) $(srctree)/scripts/checkstack.pl $(CHECKSTACK_ARCH)
+	$(PERL) $(srctree)/scripts/checkstack.pl $(CHECKSTACK_ARCH) $(MINSTACKSIZE)
 
 kernelrelease:
 	@$(filechk_kernel.release)
diff --git a/Next/SHA1s b/Next/SHA1s
new file mode 100644
index 000000000000..0f61fc285564
--- /dev/null
+++ b/Next/SHA1s
@@ -0,0 +1,371 @@
+Name		SHA1
+----		----
+origin		c2d5304e6c648ebcf653bace7e51e0e6742e46c8
+fixes		2dde18cd1d8fac735875f2e4987f11817cc0bc2c
+mm-hotfixes	25996b6ebbb739057c96c2b798190d40a6f9e868
+kbuild-current	98b1cc82c4affc16f5598d4fa14b1858671b2263
+arc-current	0bb80ecc33a8fb5a682236443c1e740d5c917d1d
+arm-current	2dde18cd1d8fac735875f2e4987f11817cc0bc2c
+arm64-fixes	4785aa8028536c2be656d22c74ec1995b97056f3
+arm-soc-fixes	736a4aad8a9fdcf577ffa33b33df240c67557af8
+davinci-current	06c2afb862f9da8dc5efa4b6076a0e48c3fbaaa5
+drivers-memory-fixes	b85ea95d086471afb4ad062012a4d73cd328fa86
+tee-fixes	ceaa837f96adb69c0df0397937cd74991d5d821a
+m68k-current	03191fb3db3d2585725bcffe0356d12fcfd27c4c
+powerpc-fixes	644b6025bcaff59737270d812c70302f5a8d4a8f
+s390-fixes	ffc253263a1375a65fa6c9f62a893e9767fbebfa
+sparc		2d2b17d08bfc3d98cf93622e8c6543eaaf02dabe
+fscrypt-current	4bcf6f827a79c59806c695dc280e763c5b6a6813
+fsverity-current	a075bacde257f755bea0e53400c9f1cdd1b8e8e6
+net		9c6dc13106f2dd2d6819d66618b25a6f41f0ee6a
+bpf		acb12c859ac7c36d6d7632280fd1e263188cb07f
+ipsec		76df934c6d5f5c93ba7a0112b1818620ddc10b19
+netfilter	54d4434da824460a190d547404530eff12a7907d
+ipvs		a63b6622120cd03a304796dbccb80655b3a21798
+wireless	7a934b5cc3f452df6f9a4903450fc103dee98ee8
+wpan		2d1c882d4434a27a026b98b602f232c7919849c5
+rdma-fixes	b6f09b16558f31d93cdcda3cab90a2d309a7c823
+sound-current	d04ce4113cb4e5c2deddcb161db42a25917f285f
+sound-asoc-fixes	14e8442e0789598514f3c9de014950de9feda7a4
+regmap-fixes	b85ea95d086471afb4ad062012a4d73cd328fa86
+regulator-fixes	b85ea95d086471afb4ad062012a4d73cd328fa86
+spi-fixes	b85ea95d086471afb4ad062012a4d73cd328fa86
+pci-current	b85ea95d086471afb4ad062012a4d73cd328fa86
+driver-core.current	98b1cc82c4affc16f5598d4fa14b1858671b2263
+tty.current	98b1cc82c4affc16f5598d4fa14b1858671b2263
+usb.current	16b7e0cccb243033de4406ffb4d892365041a1e7
+usb-serial-fixes	a1092619dd28ac0fcf23016160a2fdccd98ef935
+phy		0f40d5099cd6d828fd7de6227d3eabe86016724c
+staging.current	98b1cc82c4affc16f5598d4fa14b1858671b2263
+iio-fixes	54cf39ec16335dadbe1ba008d8e5e98dae3e26f8
+counter-current	58720809f52779dc0f08e53e54b014209d13eebb
+char-misc.current	98b1cc82c4affc16f5598d4fa14b1858671b2263
+soundwire-fixes	b85ea95d086471afb4ad062012a4d73cd328fa86
+thunderbolt-fixes	480713b1ba8eac4617936f8404da34bda991c30e
+input-current	c1f342f35f820b33390571293498c3e2e9bc77ec
+crypto-current	9aedd10fe38418319bd8ed55dc68a40ec04aaa05
+vfio-fixes	c777b11d34e0f47dbbc4b018ef65ad030f2b283a
+kselftest-fixes	b85ea95d086471afb4ad062012a4d73cd328fa86
+modules-fixes	f412eef03938d3a40d4f6f5a79d0f98ed89b596d
+dmaengine-fixes	b85ea95d086471afb4ad062012a4d73cd328fa86
+backlight-fixes	88603b6dc419445847923fcb7fe5080067a30f98
+mtd-fixes	7c1b1906229db88c487e21e1ecb622db64a1830d
+mfd-fixes	88603b6dc419445847923fcb7fe5080067a30f98
+v4l-dvb-fixes	32138be394e5d32c095a413949e6ab4875b2aec0
+reset-fixes	3a2390c6777e3f6662980c6cfc25cafe9e4fef98
+mips-fixes	b85ea95d086471afb4ad062012a4d73cd328fa86
+at91-fixes	b85ea95d086471afb4ad062012a4d73cd328fa86
+omap-fixes	0b9a4a67c60d3e15b39a69d480a50ce7eeff9bc1
+kvm-fixes	ffc253263a1375a65fa6c9f62a893e9767fbebfa
+kvms390-fixes	27072b8e18a73ffeffb1c140939023915a35134b
+hwmon-fixes	58ebe7fb6eb2deed0fd05cf57911fea3f36124eb
+nvdimm-fixes	33908660e814203e996f6e775d033c5c32fcf9a7
+cxl-fixes	8f61d48c83f6e3525a770e44692604595693f787
+btrfs-fixes	9b71b2975adf06744176efd6fee4153b3c39c184
+vfs-fixes	dc32464a5fe4946fe1a4d8f8e29961dc411933c5
+dma-mapping-fixes	d5090484b021794271280ab64d20253883b7f6fd
+drivers-x86-fixes	a6584711e64d9d12ab79a450ec3628fd35e4f476
+samsung-krzk-fixes	b85ea95d086471afb4ad062012a4d73cd328fa86
+pinctrl-samsung-fixes	b85ea95d086471afb4ad062012a4d73cd328fa86
+devicetree-fixes	8f51593cdcab82fb23ef2e1a0010b2e6f99aae02
+dt-krzk-fixes	b85ea95d086471afb4ad062012a4d73cd328fa86
+scsi-fixes	2a0508d9d08f0c3e354044d4f48466ee0d225041
+drm-fixes	98b1cc82c4affc16f5598d4fa14b1858671b2263
+drm-intel-fixes	0561794b6b642b84b879bf97061c4b4fa692839e
+mmc-fixes	1de1b77982e1a1df9707cb11f9b1789e6b8919d4
+rtc-fixes	08279468a294d8c996a657ecc9e51bd5c084c75d
+gnss-fixes	98b1cc82c4affc16f5598d4fa14b1858671b2263
+hyperv-fixes	7e8037b099c0bbe8f2109dc452dbcab8d400fc53
+soc-fsl-fixes	06c2afb862f9da8dc5efa4b6076a0e48c3fbaaa5
+risc-v-fixes	3ca112b71f35dd5d99fc4571a56b5fc6f0c15814
+riscv-dt-fixes	e80ed63affc9a9b4aacb44180ecd7ed601839599
+riscv-soc-fixes	b85ea95d086471afb4ad062012a4d73cd328fa86
+fpga-fixes	b85ea95d086471afb4ad062012a4d73cd328fa86
+spdx		d2f51b3516dade79269ff45eae2a7668ae711b25
+gpio-brgl-fixes	b85ea95d086471afb4ad062012a4d73cd328fa86
+gpio-intel-fixes	b85ea95d086471afb4ad062012a4d73cd328fa86
+pinctrl-intel-fixes	b85ea95d086471afb4ad062012a4d73cd328fa86
+erofs-fixes	62b241efff99fc4d88a86f1c67c7516e31f432a3
+kunit-fixes	1bddcf77ce6668692fc15e968fd0870d5524d112
+ubifs-fixes	2241ab53cbb5cdb08a6b2d4688feb13971058f65
+memblock-fixes	55122e0130e51eb71f5ec62d10525db0468f28e8
+nfsd-fixes	bf51c52a1f3c238d72c64e14d5e7702d3a245b82
+irqchip-fixes	b673fe1a6229a49be5394f4e539055d9ce685615
+renesas-fixes	9eab43facdadb7d00456c2657001ae2e5353c814
+broadcom-fixes	9abf2313adc1ca1b6180c508c25f22f9395cc780
+perf-current	98b1cc82c4affc16f5598d4fa14b1858671b2263
+efi-fixes	c03d21f05e76b25f907684bdf874308dcefab385
+zstd-fixes	77618db346455129424fadbbaec596a09feaf3bb
+battery-fixes	f37669119423ca852ca855b24732f25c0737aa57
+uml-fixes	73a23d7710331a530e972903318528b75e5a5f58
+asahi-soc-fixes	568035b01cfb107af8d2e4bd2fb9aea22cf5b868
+iommufd-fixes	98594181944daa201481ad63242806beb7c89ff4
+rust-fixes	cfd96726e61136e68a168813cedc4084f626208b
+v9fs-fixes	2dde18cd1d8fac735875f2e4987f11817cc0bc2c
+w1-fixes	b85ea95d086471afb4ad062012a4d73cd328fa86
+pmdomain-fixes	b131329b9bfbd1b4c0c5e088cb0c6ec03a12930f
+overlayfs-fixes	37f32f52643869131ec01bb69bdf9f404f6109fb
+drm-misc-fixes	ab93edb2f94c3c0d5965be3815782472adbe3f52
+kspp		1ee60356c2dca938362528404af95b8ef3e49b6a
+kspp-gustavo	1b56c90018f0e2bc76064ed6c90d146d5a7f998f
+mm-stable	b85ea95d086471afb4ad062012a4d73cd328fa86
+mm-nonmm-stable	b85ea95d086471afb4ad062012a4d73cd328fa86
+mm		2e2d7c098f1b1a118482cf2de73bbb46ee0ba18d
+kbuild		39d5b46d86e7226eafa32556ddd7abb0382c01b1
+clang-format	5d0c230f1de8c7515b6567d9afba1f196fb4e2f4
+perf		acbf6de674ef7b1b5870b25e7b3c695bf84273d0
+compiler-attributes	5d0c230f1de8c7515b6567d9afba1f196fb4e2f4
+dma-mapping	53c87e846e335e3c18044c397cc35178163d7827
+asm-generic	550087a0ba91eb001c139f563a193b1e9ef5a8dd
+arc		0bb80ecc33a8fb5a682236443c1e740d5c917d1d
+arm		c7368ddba2ffcc5d200122c5bb122c3825ecb976
+arm64		f86128050d2d854035bfa461aadf36e6951b2bac
+arm-perf	b805cafc604bfdb671fae7347a57f51154afa735
+arm-soc		1b52f65d88adb1fac2193ed5d38b51ad13c7be76
+amlogic		996fc07dce7991dd12ac0b7c378ce65080036492
+asahi-soc	ffc253263a1375a65fa6c9f62a893e9767fbebfa
+aspeed		e9be9b34b841d2b1691c285ef0e85232734e28b2
+at91		b766b70117ac57d13dbbd6a6f4f0a75d57f9290e
+broadcom	62a3c97f8167c88a5623d45c8984927c82c286be
+davinci		06c2afb862f9da8dc5efa4b6076a0e48c3fbaaa5
+drivers-memory	0d6c918011ce4764ed277de4726a468b7ffe5fed
+imx-mxs		fa81543ef85463e53bd2e3193a182ec1ff800ee9
+mediatek	9802b60bd6d895a8be9c44cc5f74bb11131aa8bc
+mvebu		93e6b023e552e7af30ba89439a1486160779010f
+omap		cb1114df7bb08a0be22ae1652b0c5ff9602ed391
+qcom		2b05c2dc230b58fa09358009e83eb5556ab72dc1
+renesas		19c4da034485378938f6251c62d5dfee47e0f8e5
+reset		417a3a5ae44a14b4eeb3df514b2c006b19771f82
+rockchip	2a883579c894fcedd543233e5089479bfa0c9cde
+samsung-krzk	505d8d66267f31835c86cdf6cf80b075b6d9bd95
+scmi		b419bd935f13421f87738dd83a18dbdabb4651fb
+stm32		9a7308232a118b3837800337381bb1cbae30ae1d
+sunxi		715a9e70bdd4d5f8a6875a0b27ebd36d7eb58d1b
+tee		352b5f33b950a2130413b8278c859677d81e7fee
+tegra		650220c2b474332f78dbb20651f0e98d77af6fde
+ti		b85ea95d086471afb4ad062012a4d73cd328fa86
+xilinx		1d4723dec415d594a21190183ed62a9e3aadff91
+clk		b85ea95d086471afb4ad062012a4d73cd328fa86
+clk-imx		2838820800dcaa60f32bef715c7e2e87e049aebc
+clk-renesas	5ab16198b431ca4885dbcc3433527fdf5b66be3c
+csky		2c40c1c6adab90ee4660caf03722b3a3ec67767b
+loongarch	c517fd2738f472eb0d1db60a70d91629349a9bf8
+m68k		03191fb3db3d2585725bcffe0356d12fcfd27c4c
+m68knommu	98b1cc82c4affc16f5598d4fa14b1858671b2263
+microblaze	ffb0399437ef582b035089d1617bbb0ea174cd0c
+mips		b85ea95d086471afb4ad062012a4d73cd328fa86
+openrisc	c289330331eb93bc6a3c68b9119ccd7d4285a4a2
+parisc-hd	2546732f135427a14e8cf0d0b1a72862368d86e8
+powerpc		707df298cbde200b939c70be2577b20775fe3345
+soc-fsl		fb9c384625dd604e8a5be1f42b35e83104b90670
+risc-v		3ca112b71f35dd5d99fc4571a56b5fc6f0c15814
+riscv-dt	b85ea95d086471afb4ad062012a4d73cd328fa86
+riscv-soc	b85ea95d086471afb4ad062012a4d73cd328fa86
+s390		02e790ee3077c0571794d0ab8f71413edbe129cc
+sh		b85ea95d086471afb4ad062012a4d73cd328fa86
+uml		974b808d85abbc03c3914af63d60d5816aabf2ca
+xtensa		a83a72730c330a57c1a07cd98603140f5fbe0bf9
+bcachefs	e3759384e88e515bce2749a7946f1f0dd0251786
+pidfd		a901a3568fd26ca9c4a82d8bc5ed5b3ed844d451
+fscrypt		15baf55481de700f8c4494cddb80ec4f4575548b
+afs		0a278bc196e784b4849c1581266421adbca1989f
+btrfs		3fdca4a7353dcdfeb95da8e13a2b6b135709f2ec
+ceph		d1a66ad3bd278641cbae1537e2297cb78618b745
+cifs		98b1cc82c4affc16f5598d4fa14b1858671b2263
+configfs	4425c1d9b44ded655d2668e1ce95a62bccf7b21b
+ecryptfs	a3d78fe3e1ae8c6a1901635c54a1a799656f72c8
+erofs		1a0ac8bd7a4fa5b2f4ef14c3b1e9d6e5a5faae06
+exfat		1373ca10ec04afba9199de1fab01fde91338a78b
+ext3		7f680e5f256f346a5d3cd83a17c84bb6bc950008
+ext4		91562895f8030cb9a0470b1db49de79346a69f91
+f2fs		6bc40e44f1ddef16a787f3501b97f1fff909177c
+fsverity	919dc320956ea353a7fb2d84265195ad5ef525ac
+fuse		18437366a3a10577852f1626bc0dc28ae91573a7
+gfs2		8d859dc44163102336550ca74d82d7362bcacf53
+jfs		cca974daeb6c43ea971f8ceff5a7080d7d49ee30
+ksmbd		98b1cc82c4affc16f5598d4fa14b1858671b2263
+nfs		f003a717ae9086b1e8a4663124a96862df7282e7
+nfs-anna	379e4adfddd6a2f95a4f2029b8ddcbacf92b21f9
+nfsd		21bc7f605adb9625e9db69fdca07c2bedb6bf6a3
+ntfs3		ffc253263a1375a65fa6c9f62a893e9767fbebfa
+orangefs	31720a2b109b3080eb77e97b8f6f50a27b4ae599
+overlayfs	484d4fbfdafea581fdc2fd04df6781c3a59122c9
+ubifs		75690493591fe283e4c92a3ba7c4420e9858abdb
+v9fs		ce07087964208eee2ca2f9ee4a98f8b5d9027fe6
+v9fs-ericvh	2dde18cd1d8fac735875f2e4987f11817cc0bc2c
+xfs		7930d9e103700cde15833638855b750715c12091
+zonefs		8812387d056957355ef1d026cd38bed3830649db
+iomap		3ac974796e5d94509b85a403449132ea660127c2
+djw-vfs		ce85a1e04645b1ed386b074297df27ab5b8801c0
+file-locks	e0152e7481c6c63764d6ea8ee41af5cf9dfac5e9
+iversion	e0152e7481c6c63764d6ea8ee41af5cf9dfac5e9
+vfs-brauner	e0bd765766c9110c21ff09922159f8137e2877d9
+vfs		bbe6a7c899e7f265c5a6d01a178336a405e98ed6
+printk		b4908d68609b57ad1ba4b80bd72c4d2260387e31
+pci		b85ea95d086471afb4ad062012a4d73cd328fa86
+pstore		1d49dee6b691d47acd5c299c89d9e32acea1d20d
+hid		dbef8373cf6b6b27715dd34597312e6401c152b5
+i2c		382561d16854a747e6df71034da08d20d6013dfe
+i3c		b4da37db3e2cfc1c60875b0c10cfc556d5342a3a
+dmi		13a0ac816d22aa47d6c393f14a99f39e49b960df
+hwmon-staging	c03cd01890c5e41138cc5709e37859bc917aab5d
+jc_docs		7c49ca6b021380f7be67ec9896392ff316a8ac71
+v4l-dvb		3e238417254bfdcc23fe207780b59cbb08656762
+v4l-dvb-next	3e238417254bfdcc23fe207780b59cbb08656762
+pm		e8afe36a728732d1d0110c1c63b74fa856e549bd
+cpufreq-arm	5b5b5806f22390808b8e8fa180fe35b003a4a74d
+cpupower	b85ea95d086471afb4ad062012a4d73cd328fa86
+devfreq		b85ea95d086471afb4ad062012a4d73cd328fa86
+pmdomain	9e0cceadb7a5099c637e787191a9adbf9ec424cd
+opp		41907aa4ae37a3f9ce092927e20afb649fc6fb19
+thermal		9618efe343ead954ca5c23856ae23d0a29e7d4b9
+dlm		0c08699744d20ce0bac22b9f291a646a0302e51f
+rdma		057a30168175048be9e9b30f0cafd26f5043eb07
+net-next	3a17ea77da31ef0b5ab04111583bfcd38f56fcba
+bpf-next	3cbbf9192abdc9183eb215b5e8b06c778e5c2214
+ipsec-next	18de1e517ed37ebaf33e771e46faf052e966e163
+mlx5-next	82f9378c443c206d3f9e45844306e5270e7e4109
+netfilter-next	ac40916a3f7243efbe6e129ebf495b5c33a3adfe
+ipvs-next	9cdee063476988102bbc5e0e9551e10c5ed00d3e
+bluetooth	d5742377e3ac807f1879bb6d8571c8787622a37d
+wireless-next	0a78bb64a49995c7377b1d716bbff7a9d92212cb
+wpan-next	18b849f12dcc34ec4cb9c8fadeb503b069499ba4
+wpan-staging	18b849f12dcc34ec4cb9c8fadeb503b069499ba4
+mtd		b85ea95d086471afb4ad062012a4d73cd328fa86
+nand		acb1fd579efbcac26ce8f9c4fc8bd82f7eaa56e9
+spi-nor		4137f324cb29a689e8519d8f7f52d3443bac934b
+crypto		7c18e3c6b309aebc0fde38bda5ff30ea22c8a56c
+drm		c79b972eb88b077d2765e7790d0902b3dc94d55c
+drm-ci		ad6bfe1b66a5c146ec236847eca7af4c8806d666
+drm-misc	a4dea9a06f72c7885f8d4dccedec7e477878d798
+amdgpu		699d392903c3cebb7d2a2a3505ec9047c419dcd7
+drm-intel	297c76d94c8911b5d7b58afc51cfde715dd155fe
+drm-tegra	2429b3c529da29d4277d519bd66d034842dcd70c
+drm-msm		b85ea95d086471afb4ad062012a4d73cd328fa86
+drm-msm-lumag	d3b4075b173f033387b614297bb4d998cf22c8bd
+etnaviv		925b10728f20c24516031060f3d01e4410c4a52a
+fbdev		8b01ee6ce9bf8f4be2c961a574a451eb16dc5ee8
+regmap		d958d97848a6604d024221920d300d07869715a2
+sound		5a77457232fa0453da4d3d5a7d530129944aeeb5
+ieee1394	c12d7aa7ffa4c61443241fbc1ee405acf4aa17de
+sound-asoc	a2d01aefded184c5ccf09450179999c8187c6481
+modules		4652b8e4f3ffa48c706ec334f048c217a7d9750d
+input		ccb9e9dd2a996f5814d7072b968b3d2f4b094188
+block		50e8a66babc480f4dad816285d48ca59ee50509d
+device-mapper	13648e04a9b831b3dfa5cf3887dfa6cf8fe5fe69
+libata		804901fdd637c7af8b767c329fd84aa0d3d617ec
+pcmcia		4f733de8b78a209501041a4b0a44c83ece0e8933
+mmc		3f00051234f02d0d9d1f63b9a334d0fd4c65b6ca
+mfd		2b481822446e30943fb7c02744a7b49ebec0e696
+backlight	d5272d39995f4150062a67e6f2cef556edece740
+battery		b55d073e6501dc6077edaa945a6dad8ac5c8bbab
+regulator	753e4d5c433da57da75dd4c3e1aececc8e874a62
+security	afe576a62062cf944372fff2b6510b621ec454f2
+apparmor	a7e405a2de69fe5e6657046e978a81683b140051
+integrity	b836c4d29f2744200b2af41e14bf50758dddc818
+safesetid	64b634830c919979de4b18163e15d30df66e64a8
+selinux		ae254858ce0745aba25d107159b580ab5fdada5b
+smack		3ad49d37cf5759c3b8b68d02e3563f633d9c1aee
+tomoyo		0bb80ecc33a8fb5a682236443c1e740d5c917d1d
+tpmdd		c2d5304e6c648ebcf653bace7e51e0e6742e46c8
+watchdog	b85ea95d086471afb4ad062012a4d73cd328fa86
+iommu		e8cca466a84a75f8ff2a7a31173c99ee6d1c59d2
+audit		24fade412acf3e349e337b9472a5a8f7eb94cfb8
+devicetree	fe612629746cf5cc7040529f780d46929605d0a6
+dt-krzk		b85ea95d086471afb4ad062012a4d73cd328fa86
+mailbox		96cb7a4e296da9aaae0ad61394fdb2828e0a21b5
+spi		4c3ff31a85e39d7e216e86934b694b6846433435
+tip		92fddfbe99ffc16cb2990eab74ea0da6238ba7cd
+clockevents	c28ca80ba3b531a79402d61046aef83272f86b08
+edac		0c7c7ba0c7215de44410e9c4acce28d762dd907a
+irqchip		19b5a44bee16518104e8a159ab9a60788292fbd4
+ftrace		e1742fa172d553ecc8e9077a713315404be36c13
+rcu		68f721e904b253ae9f782db0362f608048225403
+kvm		6c370dc65374db5afbc5c6c64c662f922a2555ad
+kvm-arm		123f42f0ad6815014f54d0cc6eb9039c46ee2907
+kvms390		70fea30195168fd84e7076720c984f0ac1af5b09
+kvm-ppc		707df298cbde200b939c70be2577b20775fe3345
+kvm-riscv	d9c00f44e5de542340cce1d09e2c990e16c0ed3a
+kvm-x86		45b890f7689eb0aba454fc5831d2d79763781677
+xen-tip		295b202227e98edb2fb5cc29b6ec4b96b2792d9c
+percpu		3fcf62f24c80e2472f472778f317e062841c2d37
+workqueues	d5ce8f4ed90bf34b9b2c5459665cee6ee79fe101
+drivers-x86	b85ea95d086471afb4ad062012a4d73cd328fa86
+chrome-platform	d131f1f3b459980d38a59adc3598c96cc3a6ad5e
+chrome-platform-firmware	ecea08916418a94f99f89c543303877cb6e08a11
+hsi		fa72d143471d04ce3055d8dad9743b08c19e4060
+leds		1b929c02afd37871d5afb9d498426f83432e71c2
+leds-lj		b9604be241587fb29c0f40450e53d0a37dc611b5
+ipmi		e89a0391d9208db19c75751559dd5087484a2a25
+driver-core	98b1cc82c4affc16f5598d4fa14b1858671b2263
+usb		98b1cc82c4affc16f5598d4fa14b1858671b2263
+thunderbolt	655b8af57d31e07340b023a807e43dadf81a0818
+usb-serial	98b1cc82c4affc16f5598d4fa14b1858671b2263
+tty		98b1cc82c4affc16f5598d4fa14b1858671b2263
+char-misc	98b1cc82c4affc16f5598d4fa14b1858671b2263
+accel		ce06bc9d7209f3a585c5154841eeb0b788ff75c7
+coresight	60e5f23dc5d68ec01e6dae8f4311230c7d2ccb8a
+fastrpc		b85ea95d086471afb4ad062012a4d73cd328fa86
+fpga		5496fb8eedd637e1e9d87655f86dc816afd5ad68
+icc		b85ea95d086471afb4ad062012a4d73cd328fa86
+iio		6543f376ec8aa90a6c1ed44b765f4f0d6c3eb1db
+phy-next	772dd70a5ed6845d87738f82c788c9ff4e37fd7f
+soundwire	b85ea95d086471afb4ad062012a4d73cd328fa86
+extcon		b85ea95d086471afb4ad062012a4d73cd328fa86
+gnss		98b1cc82c4affc16f5598d4fa14b1858671b2263
+vfio		2b88119e35b00d8cb418d86abbace3b90a993bd7
+w1		271c81935801d6449bb7bab5ccfc6cd38238c62b
+staging		98b1cc82c4affc16f5598d4fa14b1858671b2263
+counter-next	7904cdf1397c9391178ce53a7ebfa099c6bc4a59
+mux		44c026a73be8038f03dbdeef028b642880cf1511
+dmaengine	b85ea95d086471afb4ad062012a4d73cd328fa86
+cgroup		787521fc03808faa10a0bf77874d0342654c3dc8
+scsi		2a0508d9d08f0c3e354044d4f48466ee0d225041
+scsi-mkp	6a965ee1892a7a44f6d8f4a0b9fb5f775a8b4ccb
+vhost		e07754e0a1ea2d63fb29574253d1fd7405607343
+rpmsg		cae0e61beb7acb1c54a17e5418978f096d66def6
+gpio		0bb80ecc33a8fb5a682236443c1e740d5c917d1d
+gpio-brgl	d652049e118ffe9227155f7b9b85faf15fc91f8f
+gpio-intel	92fc925f838660eec25862a7fa7e6ef79d22f3ea
+pinctrl		91acee35452f14f25de8e862fa243592baadcd43
+pinctrl-intel	c5860e4a2737a8b29dc426c800d01c5be6aad811
+pinctrl-renesas	583d8073205566468abf3a34000fccdd1d19c9cc
+pinctrl-samsung	e4e2fbe7d7d70ce52adeca7ef933488ebfe78f03
+pwm		d27abbfd4888d79dd24baf50e774631046ac4732
+userns		05bd6e0242b4c059ec6797e3d721ecca6d15ff8d
+ktest		7dc8e24f0e09834341f84d37433840b353d64bc8
+kselftest	b85ea95d086471afb4ad062012a4d73cd328fa86
+kunit		b85ea95d086471afb4ad062012a4d73cd328fa86
+kunit-next	b85ea95d086471afb4ad062012a4d73cd328fa86
+livepatching	602bf18307981f3bfd9ebf19921791a4256d3fd1
+rtc		3d762e21d56370a43478b55e604b4a83dd85aafc
+nvdimm		9ea459e477dc09370cdd8ee13b61aefe8cd1f20a
+at24		b85ea95d086471afb4ad062012a4d73cd328fa86
+ntb		9341b37ec17a8793e8439e9b18354ba69556b786
+seccomp		31c65705a8cfa5f80d3824c686ab74b0409ee76d
+fsi		f04d61a379d65794d5d85168b84dcdf01d426f7c
+slimbus		b85ea95d086471afb4ad062012a4d73cd328fa86
+nvmem		b85ea95d086471afb4ad062012a4d73cd328fa86
+xarray		2a15de80dd0f7e04a823291aa9eb49c5294f56af
+hyperv		ce9ecca0238b140b88f43859b211c9fdfd8e5b70
+auxdisplay	35b464e32c8bccef435e415db955787ead4ab44c
+kgdb		23816724fdbd47c28bc998866fd7bc5ad9f0e535
+hmm		0bb80ecc33a8fb5a682236443c1e740d5c917d1d
+cfi		06c2afb862f9da8dc5efa4b6076a0e48c3fbaaa5
+mhi		83c5dcf78ab992181c7d8bd5e94a0fdbde4c0efe
+memblock	e96c6b8f212a510c9b22362de519f6e1d7920de5
+cxl		5d09c63f11f083707b60c8ea0bb420651c47740f
+zstd		3f832dfb8a8eafee3cecd479d99651a64a61485a
+efi		5329aa5101f73c451bcd48deaf3f296685849d9c
+unicode		c1869dbd29578abdbf383ffa70c36f3f555ababb
+slab		dd374e220ba492f95344a638b1efe5b2744fdd73
+random		512dee0c00ad9e9c7ae9f11fc6743702ea40caff
+landlock	323f65ded48d9f4619b9b3d80f952471cf469b11
+rust		3857af38e57a80b15b994e19d1f4301cac796481
+sysctl		8b793bcda61f6c3ed4f5b2ded7530ef6749580cb
+execve		21ca59b365c091d583f36ac753eaa8baf947be6f
+bitmap		bdcb37a5d8de3253da48b120e3f10863696fb654
+hte		b85ea95d086471afb4ad062012a4d73cd328fa86
+nolibc		b85ea95d086471afb4ad062012a4d73cd328fa86
+tsm		f4738f56d1dc62aaba69b33702a5ab098f1b8c63
+iommufd		b2b67c997bf74453f3469d8b54e4859f190943bd
diff --git a/Next/Trees b/Next/Trees
new file mode 100644
index 000000000000..13b9c07ee35b
--- /dev/null
+++ b/Next/Trees
@@ -0,0 +1,373 @@
+Trees included into this release:
+
+Name		Type	Url
+----		----	---
+origin		git	git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git#master
+fixes		git	git://git.kernel.org/pub/scm/linux/kernel/git/sfr/next-fixes.git#fixes
+mm-hotfixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-hotfixes-unstable
+kbuild-current	git	git://git.kernel.org/pub/scm/linux/kernel/git/masahiroy/linux-kbuild.git#fixes
+arc-current	git	git://git.kernel.org/pub/scm/linux/kernel/git/vgupta/arc.git#for-curr
+arm-current	git	git://git.armlinux.org.uk/~rmk/linux-arm.git#fixes
+arm64-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux#for-next/fixes
+arm-soc-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git#arm/fixes
+davinci-current	git	git://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#davinci/for-current
+drivers-memory-fixes	git	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git#fixes
+tee-fixes	git	https://git.linaro.org/people/jens.wiklander/linux-tee.git#fixes
+m68k-current	git	git://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git#for-linus
+powerpc-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git#fixes
+s390-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git#fixes
+sparc		git	git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc.git#master
+fscrypt-current	git	git://git.kernel.org/pub/scm/fs/fscrypt/linux.git#for-current
+fsverity-current	git	git://git.kernel.org/pub/scm/fs/fsverity/linux.git#for-current
+net		git	git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net.git#main
+bpf		git	git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git#master
+ipsec		git	git://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec.git#master
+netfilter	git	git://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf.git#main
+ipvs		git	git://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs.git#main
+wireless	git	git://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless.git#for-next
+wpan		git	git://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan.git#master
+rdma-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git#for-rc
+sound-current	git	git://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git#for-linus
+sound-asoc-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git#for-linus
+regmap-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git#for-linus
+regulator-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git#for-linus
+spi-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git#for-linus
+pci-current	git	git://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git#for-linus
+driver-core.current	git	git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git#driver-core-linus
+tty.current	git	git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git#tty-linus
+usb.current	git	git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git#usb-linus
+usb-serial-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git#usb-linus
+phy		git	git://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git#fixes
+staging.current	git	git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git#staging-linus
+iio-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git#fixes-togreg
+counter-current	git	git://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git#counter-current
+char-misc.current	git	git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git#char-misc-linus
+soundwire-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git#fixes
+thunderbolt-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git#fixes
+input-current	git	git://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git#for-linus
+crypto-current	git	git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6.git#master
+vfio-fixes	git	git://github.com/awilliam/linux-vfio.git#for-linus
+kselftest-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#fixes
+modules-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/mcgrof/linux.git#modules-linus
+dmaengine-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git#fixes
+backlight-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git#for-backlight-fixes
+mtd-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#mtd/fixes
+mfd-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git#for-mfd-fixes
+v4l-dvb-fixes	git	https://git.linuxtv.org/media_stage.git#fixes
+reset-fixes	git	https://git.pengutronix.de/git/pza/linux#reset/fixes
+mips-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git#mips-fixes
+at91-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git#at91-fixes
+omap-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/tmlind/linux-omap.git#fixes
+kvm-fixes	git	git://git.kernel.org/pub/scm/virt/kvm/kvm.git#master
+kvms390-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git#master
+hwmon-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git#hwmon
+nvdimm-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git#libnvdimm-fixes
+cxl-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git#fixes
+btrfs-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git#next-fixes
+vfs-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git#fixes
+dma-mapping-fixes	git	git://git.infradead.org/users/hch/dma-mapping.git#for-linus
+drivers-x86-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git#fixes
+samsung-krzk-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git#fixes
+pinctrl-samsung-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git#fixes
+devicetree-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git#dt/linus
+dt-krzk-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git#fixes
+scsi-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git#fixes
+drm-fixes	git	git://git.freedesktop.org/git/drm/drm.git#drm-fixes
+drm-intel-fixes	git	git://anongit.freedesktop.org/drm-intel#for-linux-next-fixes
+mmc-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git#fixes
+rtc-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git#rtc-fixes
+gnss-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git#gnss-linus
+hyperv-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git#hyperv-fixes
+soc-fsl-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/leo/linux.git#fix
+risc-v-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git#fixes
+riscv-dt-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-dt-fixes
+riscv-soc-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-soc-fixes
+fpga-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git#fixes
+spdx		git	git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/spdx.git#spdx-linus
+gpio-brgl-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#gpio/for-current
+gpio-intel-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git#fixes
+pinctrl-intel-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git#fixes
+erofs-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git#fixes
+kunit-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#kunit-fixes
+ubifs-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/rw/ubifs.git#fixes
+memblock-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git#fixes
+nfsd-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/cel/linux#nfsd-fixes
+irqchip-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/maz/arm-platforms.git#irq/irqchip-fixes
+renesas-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git#fixes
+broadcom-fixes	git	https://github.com/Broadcom/stblinux.git#fixes
+perf-current	git	git://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools#perf-tools
+efi-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git#urgent
+zstd-fixes	git	https://github.com/terrelln/linux.git#zstd-linus
+battery-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git#fixes
+uml-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git#fixes
+asahi-soc-fixes	git	https://github.com/AsahiLinux/linux.git#asahi-soc/fixes
+iommufd-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git#for-rc
+rust-fixes	git	https://github.com/Rust-for-Linux/linux.git#rust-fixes
+v9fs-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git#fixes/next
+w1-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git#fixes
+pmdomain-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git#fixes
+overlayfs-fixes	git	git://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git#ovl-fixes
+drm-misc-fixes	git	git://anongit.freedesktop.org/drm/drm-misc#for-linux-next-fixes
+kspp		git	git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/kspp
+kspp-gustavo	git	git://git.kernel.org/pub/scm/linux/kernel/git/gustavoars/linux.git#for-next/kspp
+mm-stable	git	git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-stable
+mm-nonmm-stable	git	git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-nonmm-stable
+mm		git	git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-everything
+kbuild		git	git://git.kernel.org/pub/scm/linux/kernel/git/masahiroy/linux-kbuild.git#for-next
+clang-format	git	https://github.com/ojeda/linux.git#clang-format
+perf		git	git://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git#perf-tools-next
+compiler-attributes	git	https://github.com/ojeda/linux.git#compiler-attributes
+dma-mapping	git	git://git.infradead.org/users/hch/dma-mapping.git#for-next
+asm-generic	git	git://git.kernel.org/pub/scm/linux/kernel/git/arnd/asm-generic.git#master
+arc		git	git://git.kernel.org/pub/scm/linux/kernel/git/vgupta/arc.git#for-next
+arm		git	git://git.armlinux.org.uk/~rmk/linux-arm.git#for-next
+arm64		git	git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux#for-next/core
+arm-perf	git	git://git.kernel.org/pub/scm/linux/kernel/git/will/linux.git#for-next/perf
+arm-soc		git	git://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git#for-next
+amlogic		git	git://git.kernel.org/pub/scm/linux/kernel/git/amlogic/linux.git#for-next
+asahi-soc	git	https://github.com/AsahiLinux/linux.git#asahi-soc/for-next
+aspeed		git	git://git.kernel.org/pub/scm/linux/kernel/git/joel/bmc.git#for-next
+at91		git	git://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git#at91-next
+broadcom	git	https://github.com/Broadcom/stblinux.git#next
+davinci		git	git://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#davinci/for-next
+drivers-memory	git	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git#for-next
+imx-mxs		git	git://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux.git#for-next
+mediatek	git	git://git.kernel.org/pub/scm/linux/kernel/git/matthias.bgg/linux.git#for-next
+mvebu		git	git://git.kernel.org/pub/scm/linux/kernel/git/gclement/mvebu.git#for-next
+omap		git	git://git.kernel.org/pub/scm/linux/kernel/git/tmlind/linux-omap.git#for-next
+qcom		git	git://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux.git#for-next
+renesas		git	git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git#next
+reset		git	https://git.pengutronix.de/git/pza/linux#reset/next
+rockchip	git	git://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip.git#for-next
+samsung-krzk	git	git://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git#for-next
+scmi		git	git://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux.git#for-linux-next
+stm32		git	git://git.kernel.org/pub/scm/linux/kernel/git/atorgue/stm32.git#stm32-next
+sunxi		git	git://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux.git#sunxi/for-next
+tee		git	https://git.linaro.org/people/jens.wiklander/linux-tee.git#next
+tegra		git	git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux.git#for-next
+ti		git	git://git.kernel.org/pub/scm/linux/kernel/git/ti/linux.git#ti-next
+xilinx		git	git://github.com/Xilinx/linux-xlnx.git#for-next
+clk		git	git://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git#clk-next
+clk-imx		git	git://git.kernel.org/pub/scm/linux/kernel/git/abelvesa/linux.git#for-next
+clk-renesas	git	git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git#renesas-clk
+csky		git	git://github.com/c-sky/csky-linux.git#linux-next
+loongarch	git	git://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson.git#loongarch-next
+m68k		git	git://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git#for-next
+m68knommu	git	git://git.kernel.org/pub/scm/linux/kernel/git/gerg/m68knommu.git#for-next
+microblaze	git	git://git.monstr.eu/linux-2.6-microblaze.git#next
+mips		git	git://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git#mips-next
+openrisc	git	git://github.com/openrisc/linux.git#for-next
+parisc-hd	git	git://git.kernel.org/pub/scm/linux/kernel/git/deller/parisc-linux.git#for-next
+powerpc		git	git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git#next
+soc-fsl		git	git://git.kernel.org/pub/scm/linux/kernel/git/leo/linux.git#next
+risc-v		git	git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git#for-next
+riscv-dt	git	git://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-dt-for-next
+riscv-soc	git	git://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-soc-for-next
+s390		git	git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git#for-next
+sh		git	git:git.kernel.org/pub/scm/linux/kernel/git/glaubitz/sh-linux.git#for-next
+uml		git	git://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git#next
+xtensa		git	git://github.com/jcmvbkbc/linux-xtensa.git#xtensa-for-next
+bcachefs	git	https://evilpiepirate.org/git/bcachefs.git#for-next
+pidfd		git	git://git.kernel.org/pub/scm/linux/kernel/git/brauner/linux.git#for-next
+fscrypt		git	git://git.kernel.org/pub/scm/fs/fscrypt/linux.git#for-next
+afs		git	git://git.kernel.org/pub/scm/linux/kernel/git/dhowells/linux-fs.git#afs-next
+btrfs		git	git://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git#for-next
+ceph		git	git://github.com/ceph/ceph-client.git#master
+cifs		git	git://git.samba.org/sfrench/cifs-2.6.git#for-next
+configfs	git	git://git.infradead.org/users/hch/configfs.git#for-next
+ecryptfs	git	git://git.kernel.org/pub/scm/linux/kernel/git/tyhicks/ecryptfs.git#next
+erofs		git	git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git#dev
+exfat		git	git://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/exfat.git#dev
+ext3		git	git://git.kernel.org/pub/scm/linux/kernel/git/jack/linux-fs.git#for_next
+ext4		git	git://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git#dev
+f2fs		git	git://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git#dev
+fsverity	git	git://git.kernel.org/pub/scm/fs/fsverity/linux.git#for-next
+fuse		git	git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git#for-next
+gfs2		git	git://git.kernel.org/pub/scm/linux/kernel/git/gfs2/linux-gfs2.git#for-next
+jfs		git	git://github.com/kleikamp/linux-shaggy.git#jfs-next
+ksmbd		git	https://github.com/smfrench/smb3-kernel.git#ksmbd-for-next
+nfs		git	git://git.linux-nfs.org/projects/trondmy/nfs-2.6.git#linux-next
+nfs-anna	git	git://git.linux-nfs.org/projects/anna/linux-nfs.git#linux-next
+nfsd		git	git://git.kernel.org/pub/scm/linux/kernel/git/cel/linux#nfsd-next
+ntfs3		git	https://github.com/Paragon-Software-Group/linux-ntfs3.git#master
+orangefs	git	git://git.kernel.org/pub/scm/linux/kernel/git/hubcap/linux#for-next
+overlayfs	git	git://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git#overlayfs-next
+ubifs		git	git://git.kernel.org/pub/scm/linux/kernel/git/rw/ubifs.git#next
+v9fs		git	git://github.com/martinetd/linux#9p-next
+v9fs-ericvh	git	git://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git#ericvh/for-next
+xfs		git	git://git.kernel.org/pub/scm/fs/xfs/xfs-linux.git#for-next
+zonefs		git	git://git.kernel.org/pub/scm/linux/kernel/git/dlemoal/zonefs.git#for-next
+iomap		git	git://git.kernel.org/pub/scm/fs/xfs/xfs-linux.git#iomap-for-next
+djw-vfs		git	git://git.kernel.org/pub/scm/fs/xfs/xfs-linux.git#vfs-for-next
+file-locks	git	git://git.kernel.org/pub/scm/linux/kernel/git/jlayton/linux.git#locks-next
+iversion	git	git://git.kernel.org/pub/scm/linux/kernel/git/jlayton/linux.git#iversion-next
+vfs-brauner	git	git://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git#vfs.all
+vfs		git	git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git#for-next
+printk		git	git://git.kernel.org/pub/scm/linux/kernel/git/printk/linux.git#for-next
+pci		git	git://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git#next
+pstore		git	git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/pstore
+hid		git	git://git.kernel.org/pub/scm/linux/kernel/git/hid/hid.git#for-next
+i2c		git	git://git.kernel.org/pub/scm/linux/kernel/git/wsa/linux.git#i2c/for-next
+i3c		git	git://git.kernel.org/pub/scm/linux/kernel/git/i3c/linux.git#i3c/next
+dmi		git	git://git.kernel.org/pub/scm/linux/kernel/git/jdelvare/staging.git#dmi-for-next
+hwmon-staging	git	git://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git#hwmon-next
+jc_docs		git	git://git.lwn.net/linux.git#docs-next
+v4l-dvb		git	git://linuxtv.org/media_tree.git#master
+v4l-dvb-next	git	git://linuxtv.org/mchehab/media-next.git#master
+pm		git	git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git#linux-next
+cpufreq-arm	git	git://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git#cpufreq/arm/linux-next
+cpupower	git	git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux.git#cpupower
+devfreq		git	git://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/linux.git#devfreq-next
+pmdomain	git	git://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git#next
+opp		git	git://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git#opp/linux-next
+thermal		git	git://git.kernel.org/pub/scm/linux/kernel/git/thermal/linux.git#thermal/linux-next
+dlm		git	git://git.kernel.org/pub/scm/linux/kernel/git/teigland/linux-dlm.git#next
+rdma		git	git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git#for-next
+net-next	git	git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git#main
+bpf-next	git	git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git#for-next
+ipsec-next	git	git://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec-next.git#master
+mlx5-next	git	git://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux.git#mlx5-next
+netfilter-next	git	git://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf-next.git#main
+ipvs-next	git	git://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs-next.git#main
+bluetooth	git	git://git.kernel.org/pub/scm/linux/kernel/git/bluetooth/bluetooth-next.git#master
+wireless-next	git	git://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless-next.git#for-next
+wpan-next	git	git://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git#master
+wpan-staging	git	git://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git#staging
+mtd		git	git://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#mtd/next
+nand		git	git://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#nand/next
+spi-nor		git	git://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#spi-nor/next
+crypto		git	git://git.kernel.org/pub/scm/linux/kernel/git/herbert/cryptodev-2.6.git#master
+drm		git	git://git.freedesktop.org/git/drm/drm.git#drm-next
+drm-ci		git	git://git.freedesktop.org/git/drm/drm.git#topic/drm-ci
+drm-misc	git	git://anongit.freedesktop.org/drm/drm-misc#for-linux-next
+amdgpu		git	https://gitlab.freedesktop.org/agd5f/linux#drm-next
+drm-intel	git	git://anongit.freedesktop.org/drm-intel#for-linux-next
+drm-tegra	git	https://gitlab.freedesktop.org/drm/tegra.git#for-next
+drm-msm		git	https://gitlab.freedesktop.org/drm/msm.git#msm-next
+drm-msm-lumag	git	https://gitlab.freedesktop.org/lumag/msm.git#msm-next-lumag
+etnaviv		git	https://git.pengutronix.de/git/lst/linux#etnaviv/next
+fbdev		git	git://git.kernel.org/pub/scm/linux/kernel/git/deller/linux-fbdev.git#for-next
+regmap		git	git://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git#for-next
+sound		git	git://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git#for-next
+ieee1394	git	https://git.kernel.org/pub/scm/linux/kernel/git/ieee1394/linux1394.git#for-next
+sound-asoc	git	git://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git#for-next
+modules		git	git://git.kernel.org/pub/scm/linux/kernel/git/mcgrof/linux.git#modules-next
+input		git	git://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git#next
+block		git	git://git.kernel.dk/linux-block.git#for-next
+device-mapper	git	git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git#for-next
+libata		git	git://git.kernel.org/pub/scm/linux/kernel/git/dlemoal/libata.git#for-next
+pcmcia		git	git://git.kernel.org/pub/scm/linux/kernel/git/brodo/linux.git#pcmcia-next
+mmc		git	git://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git#next
+mfd		git	git://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git#for-mfd-next
+backlight	git	git://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git#for-backlight-next
+battery		git	git://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git#for-next
+regulator	git	git://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git#for-next
+security	git	git://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/lsm.git#next
+apparmor	git	git://git.kernel.org/pub/scm/linux/kernel/git/jj/linux-apparmor#apparmor-next
+integrity	git	git://git.kernel.org/pub/scm/linux/kernel/git/zohar/linux-integrity#next-integrity
+safesetid	git	https://github.com/micah-morton/linux.git#safesetid-next
+selinux		git	git://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/selinux.git#next
+smack		git	git://github.com/cschaufler/smack-next#next
+tomoyo		git	https://scm.osdn.net/gitroot/tomoyo/tomoyo-test1.git#master
+tpmdd		git	git://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git#next
+watchdog	git	git://www.linux-watchdog.org/linux-watchdog-next.git#master
+iommu		git	git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu.git#next
+audit		git	git://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/audit.git#next
+devicetree	git	git://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git#for-next
+dt-krzk		git	git://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git#for-next
+mailbox		git	git://git.kernel.org/pub/scm/linux/kernel/git/jassibrar/mailbox.git#for-next
+spi		git	git://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git#for-next
+tip		git	git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git#master
+clockevents	git	git://git.linaro.org/people/daniel.lezcano/linux.git#timers/drivers/next
+edac		git	git://git.kernel.org/pub/scm/linux/kernel/git/ras/ras.git#edac-for-next
+irqchip		git	git://git.kernel.org/pub/scm/linux/kernel/git/maz/arm-platforms.git#irq/irqchip-next
+ftrace		git	git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#for-next
+rcu		git	git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git#rcu/next
+kvm		git	git://git.kernel.org/pub/scm/virt/kvm/kvm.git#next
+kvm-arm		git	git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git#next
+kvms390		git	git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git#next
+kvm-ppc		git	git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git#topic/ppc-kvm
+kvm-riscv	git	https://github.com/kvm-riscv/linux.git#riscv_kvm_next
+kvm-x86		git	https://github.com/kvm-x86/linux.git#next
+xen-tip		git	git://git.kernel.org/pub/scm/linux/kernel/git/xen/tip.git#linux-next
+percpu		git	git://git.kernel.org/pub/scm/linux/kernel/git/dennis/percpu.git#for-next
+workqueues	git	git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq.git#for-next
+drivers-x86	git	git://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git#for-next
+chrome-platform	git	git://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git#for-next
+chrome-platform-firmware	git	git://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git#for-firmware-next
+hsi		git	git://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-hsi.git#for-next
+leds		git	git://git.kernel.org/pub/scm/linux/kernel/git/pavel/linux-leds.git#for-next
+leds-lj		git	git://git.kernel.org/pub/scm/linux/kernel/git/lee/leds.git#for-leds-next
+ipmi		git	git://github.com/cminyard/linux-ipmi.git#for-next
+driver-core	git	git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git#driver-core-next
+usb		git	git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git#usb-next
+thunderbolt	git	git://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git#next
+usb-serial	git	git://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git#usb-next
+tty		git	git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git#tty-next
+char-misc	git	git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git#char-misc-next
+accel		git	git://git.kernel.org/pub/scm/linux/kernel/git/ogabbay/linux.git#habanalabs-next
+coresight	git	git://git.kernel.org/pub/scm/linux/kernel/git/coresight/linux.git#next
+fastrpc		git	git://git.kernel.org/pub/scm/linux/kernel/git/srini/fastrpc.git#for-next
+fpga		git	git://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git#for-next
+icc		git	git://git.kernel.org/pub/scm/linux/kernel/git/djakov/icc.git#icc-next
+iio		git	git://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git#togreg
+phy-next	git	git://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git#next
+soundwire	git	git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git#next
+extcon		git	git://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/extcon.git#extcon-next
+gnss		git	git://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git#gnss-next
+vfio		git	git://github.com/awilliam/linux-vfio.git#next
+w1		git	git://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git#for-next
+staging		git	git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git#staging-next
+counter-next	git	git://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git#counter-next
+mux		git	https://gitlab.com/peda-linux/mux.git#for-next
+dmaengine	git	git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git#next
+cgroup		git	git://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git#for-next
+scsi		git	git://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi.git#for-next
+scsi-mkp	git	git://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git#for-next
+vhost		git	git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git#linux-next
+rpmsg		git	git://git.kernel.org/pub/scm/linux/kernel/git/remoteproc/linux.git#for-next
+gpio		git	git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-gpio.git#for-next
+gpio-brgl	git	git://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#gpio/for-next
+gpio-intel	git	git://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git#for-next
+pinctrl		git	git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-pinctrl.git#for-next
+pinctrl-intel	git	git://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git#for-next
+pinctrl-renesas	git	git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git#renesas-pinctrl
+pinctrl-samsung	git	git://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git#for-next
+pwm		git	git://git.kernel.org/pub/scm/linux/kernel/git/thierry.reding/linux-pwm.git#for-next
+userns		git	git://git.kernel.org/pub/scm/linux/kernel/git/ebiederm/user-namespace.git#for-next
+ktest		git	git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-ktest.git#for-next
+kselftest	git	git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#next
+kunit		git	git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#test
+kunit-next	git	git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#kunit
+livepatching	git	git://git.kernel.org/pub/scm/linux/kernel/git/livepatching/livepatching#for-next
+rtc		git	git://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git#rtc-next
+nvdimm		git	git://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git#libnvdimm-for-next
+at24		git	git://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#at24/for-next
+ntb		git	https://github.com/jonmason/ntb.git#ntb-next
+seccomp		git	git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/seccomp
+fsi		git	git://git.kernel.org/pub/scm/linux/kernel/git/joel/fsi.git#next
+slimbus		git	git://git.kernel.org/pub/scm/linux/kernel/git/srini/slimbus.git#for-next
+nvmem		git	git://git.kernel.org/pub/scm/linux/kernel/git/srini/nvmem.git#for-next
+xarray		git	git://git.infradead.org/users/willy/xarray.git#main
+hyperv		git	git://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git#hyperv-next
+auxdisplay	git	https://github.com/ojeda/linux.git#auxdisplay
+kgdb		git	git://git.kernel.org/pub/scm/linux/kernel/git/danielt/linux.git#kgdb/for-next
+hmm		git	git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git#hmm
+cfi		git	git://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#cfi/next
+mhi		git	git://git.kernel.org/pub/scm/linux/kernel/git/mani/mhi.git#mhi-next
+memblock	git	git://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git#for-next
+cxl		git	git://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git#next
+zstd		git	https://github.com/terrelln/linux.git#zstd-next
+efi		git	git://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git#next
+unicode		git	git://git.kernel.org/pub/scm/linux/kernel/git/krisman/unicode.git#for-next
+slab		git	git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab.git#slab/for-next
+random		git	git://git.kernel.org/pub/scm/linux/kernel/git/crng/random.git#master
+landlock	git	git://git.kernel.org/pub/scm/linux/kernel/git/mic/linux.git#next
+rust		git	https://github.com/Rust-for-Linux/linux.git#rust-next
+sysctl		git	git://git.kernel.org/pub/scm/linux/kernel/git/mcgrof/linux.git#sysctl-next
+execve		git	git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/execve
+bitmap		git	https://github.com/norov/linux.git#bitmap-for-next
+hte		git	git://git.kernel.org/pub/scm/linux/kernel/git/pateldipen1984/linux.git#for-next
+nolibc		git	git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#nolibc
+tsm		git	git://git.kernel.org/pub/scm/linux/kernel/git/djbw/linux#tsm-next
+iommufd		git	git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git#for-next
diff --git a/Next/merge.log b/Next/merge.log
new file mode 100644
index 000000000000..29b7b8bd8659
--- /dev/null
+++ b/Next/merge.log
@@ -0,0 +1,4137 @@
+$ date -R
+Wed, 22 Nov 2023 09:04:15 +1100
+$ git checkout master
+Already on 'master'
+$ git reset --hard stable
+HEAD is now at 98b1cc82c4af Linux 6.7-rc2
+Merging origin/master (c2d5304e6c64 Merge tag 'platform-drivers-x86-v6.7-2' of git://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86)
+$ git merge -m Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git origin/master
+Updating 98b1cc82c4af..c2d5304e6c64
+Fast-forward (no commit created; -m option ignored)
+ Documentation/filesystems/erofs.rst          |  4 ++
+ MAINTAINERS                                  |  5 +-
+ drivers/platform/x86/amd/pmc/pmc.c           | 31 +--------
+ drivers/platform/x86/hp/hp-bioscfg/bioscfg.c | 26 ++++----
+ drivers/platform/x86/ideapad-laptop.c        | 11 ++--
+ drivers/platform/x86/intel/telemetry/core.c  |  4 +-
+ fs/erofs/Kconfig                             |  2 +-
+ fs/erofs/data.c                              |  5 +-
+ fs/erofs/inode.c                             | 98 ++++++++++------------------
+ 9 files changed, 64 insertions(+), 122 deletions(-)
+Merging fixes/fixes (2dde18cd1d8f Linux 6.5)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/sfr/next-fixes.git fixes/fixes
+Already up to date.
+Merging mm-hotfixes/mm-hotfixes-unstable (25996b6ebbb7 kexec-fix-kexec_file-dependencies-fix)
+$ git merge -m Merge branch 'mm-hotfixes-unstable' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-hotfixes/mm-hotfixes-unstable
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .mailmap                                   |  1 +
+ MAINTAINERS                                |  7 ++++++
+ arch/powerpc/Kconfig                       |  4 +--
+ arch/riscv/Kconfig                         |  4 +--
+ arch/s390/Kconfig                          |  4 +--
+ arch/x86/Kconfig                           |  4 +--
+ drivers/base/memory.c                      | 18 +++++++++++---
+ fs/proc/task_mmu.c                         | 26 +++++++++++++++----
+ fs/squashfs/block.c                        |  2 +-
+ include/linux/hugetlb.h                    |  5 +---
+ include/linux/mmzone.h                     | 14 ++++++++---
+ kernel/Kconfig.kexec                       |  2 ++
+ lib/group_cpus.c                           | 22 +++++++++++-----
+ mm/damon/core.c                            |  1 +
+ mm/filemap.c                               |  2 +-
+ mm/hugetlb.c                               |  7 ++++++
+ mm/kmemleak.c                              | 40 ++++++++++++++++--------------
+ mm/madvise.c                               | 11 ++++++++
+ mm/memcontrol.c                            |  2 +-
+ mm/memory.c                                |  1 +
+ mm/memory_hotplug.c                        | 15 ++++++-----
+ mm/shmem.c                                 | 19 +++++++++++++-
+ mm/sparse.c                                | 17 +++++++------
+ scripts/checkstack.pl                      |  8 ++----
+ tools/testing/selftests/mm/pagemap_ioctl.c |  9 ++++---
+ 25 files changed, 168 insertions(+), 77 deletions(-)
+Merging kbuild-current/fixes (98b1cc82c4af Linux 6.7-rc2)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/masahiroy/linux-kbuild.git kbuild-current/fixes
+Already up to date.
+Merging arc-current/for-curr (0bb80ecc33a8 Linux 6.6-rc1)
+$ git merge -m Merge branch 'for-curr' of git://git.kernel.org/pub/scm/linux/kernel/git/vgupta/arc.git arc-current/for-curr
+Already up to date.
+Merging arm-current/fixes (2dde18cd1d8f Linux 6.5)
+$ git merge -m Merge branch 'fixes' of git://git.armlinux.org.uk/~rmk/linux-arm.git arm-current/fixes
+Already up to date.
+Merging arm64-fixes/for-next/fixes (4785aa802853 cpuidle, ACPI: Evaluate LPI arch_flags for broadcast timer)
+$ git merge -m Merge branch 'for-next/fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux arm64-fixes/for-next/fixes
+Already up to date.
+Merging arm-soc-fixes/arm/fixes (736a4aad8a9f Merge tag 'renesas-fixes-for-v6.6-tag3' of git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel into arm/fixes)
+$ git merge -m Merge branch 'arm/fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git arm-soc-fixes/arm/fixes
+Already up to date.
+Merging davinci-current/davinci/for-current (06c2afb862f9 Linux 6.5-rc1)
+$ git merge -m Merge branch 'davinci/for-current' of git://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git davinci-current/davinci/for-current
+Already up to date.
+Merging drivers-memory-fixes/fixes (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git drivers-memory-fixes/fixes
+Already up to date.
+Merging tee-fixes/fixes (ceaa837f96ad Linux 6.2-rc8)
+$ git merge -m Merge branch 'fixes' of https://git.linaro.org/people/jens.wiklander/linux-tee.git tee-fixes/fixes
+Already up to date.
+Merging m68k-current/for-linus (03191fb3db3d m68k: lib: Include <linux/libgcc.h> for __muldi3())
+$ git merge -m Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git m68k-current/for-linus
+Already up to date.
+Merging powerpc-fixes/fixes (644b6025bcaf powerpc/rtas: Fix ppc_rtas_rmo_buf_show() kernel-doc)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git powerpc-fixes/fixes
+Already up to date.
+Merging s390-fixes/fixes (ffc253263a13 Linux 6.6)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git s390-fixes/fixes
+Already up to date.
+Merging sparc/master (2d2b17d08bfc sparc: Unbreak the build)
+$ git merge -m Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc.git sparc/master
+Auto-merging arch/sparc/mm/srmmu.c
+Merge made by the 'ort' strategy.
+Merging fscrypt-current/for-current (4bcf6f827a79 fscrypt: check for NULL keyring in fscrypt_put_master_key_activeref())
+$ git merge -m Merge branch 'for-current' of git://git.kernel.org/pub/scm/fs/fscrypt/linux.git fscrypt-current/for-current
+Already up to date.
+Merging fsverity-current/for-current (a075bacde257 fsverity: don't drop pagecache at end of FS_IOC_ENABLE_VERITY)
+$ git merge -m Merge branch 'for-current' of git://git.kernel.org/pub/scm/fs/fsverity/linux.git fsverity-current/for-current
+Already up to date.
+Merging net/main (9c6dc13106f2 MAINTAINERS: Add indirect_call_wrapper.h to NETWORKING [GENERAL])
+$ git merge -m Merge branch 'main' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net.git net/main
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                                        |  3 +
+ .../net/ethernet/marvell/octeontx2/nic/otx2_pf.c   |  2 +
+ drivers/net/ethernet/stmicro/stmmac/Kconfig        |  2 +-
+ drivers/net/ethernet/wangxun/libwx/wx_hw.c         |  8 +-
+ drivers/net/ethernet/wangxun/ngbe/ngbe_main.c      |  4 +-
+ drivers/net/ethernet/wangxun/txgbe/txgbe_main.c    |  4 +-
+ drivers/net/hyperv/netvsc_drv.c                    | 68 +++++++++++-----
+ drivers/net/usb/aqc111.c                           |  8 +-
+ drivers/net/wireguard/device.c                     |  4 +-
+ drivers/net/wireguard/receive.c                    | 12 +--
+ drivers/net/wireguard/send.c                       |  3 +-
+ drivers/s390/net/Kconfig                           |  3 +-
+ drivers/s390/net/ism_drv.c                         | 93 +++++++++++-----------
+ include/trace/events/rxrpc.h                       |  2 +-
+ net/ipv4/inet_diag.c                               |  1 +
+ net/ipv4/raw_diag.c                                |  1 +
+ net/ipv4/route.c                                   |  2 +-
+ net/ipv4/tcp_diag.c                                |  1 +
+ net/ipv4/udp_diag.c                                |  1 +
+ net/mptcp/mptcp_diag.c                             |  1 +
+ net/packet/diag.c                                  |  1 +
+ net/rxrpc/conn_client.c                            |  7 +-
+ net/rxrpc/input.c                                  | 61 +++++++-------
+ net/sctp/diag.c                                    |  1 +
+ net/smc/smc_diag.c                                 |  1 +
+ net/tipc/diag.c                                    |  1 +
+ net/unix/diag.c                                    |  1 +
+ net/vmw_vsock/diag.c                               |  1 +
+ net/xdp/xsk_diag.c                                 |  1 +
+ tools/testing/selftests/net/rtnetlink.sh           |  2 +-
+ 30 files changed, 169 insertions(+), 131 deletions(-)
+Merging bpf/master (acb12c859ac7 Merge branch 'verify-callbacks-as-if-they-are-called-unknown-number-of-times')
+$ git merge -m Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git bpf/master
+Merge made by the 'ort' strategy.
+ drivers/net/netkit.c                               |  22 +-
+ drivers/net/veth.c                                 |  44 +--
+ drivers/net/vrf.c                                  |  38 +-
+ include/linux/bpf_verifier.h                       |  16 +
+ include/linux/netdevice.h                          |  30 +-
+ include/net/netkit.h                               |   6 +
+ kernel/bpf/verifier.c                              | 438 +++++++++++++--------
+ net/core/dev.c                                     |  57 ++-
+ net/core/filter.c                                  |  19 +-
+ .../testing/selftests/bpf/prog_tests/tc_redirect.c | 315 +++++++++------
+ tools/testing/selftests/bpf/prog_tests/verifier.c  |   2 +
+ tools/testing/selftests/bpf/progs/bpf_loop_bench.c |  13 +-
+ tools/testing/selftests/bpf/progs/cb_refs.c        |   1 +
+ .../testing/selftests/bpf/progs/exceptions_fail.c  |   2 +
+ tools/testing/selftests/bpf/progs/strobemeta.h     |  78 ++--
+ .../bpf/progs/verifier_iterating_callbacks.c       | 242 ++++++++++++
+ .../bpf/progs/verifier_subprog_precision.c         |  86 +++-
+ .../selftests/bpf/progs/xdp_synproxy_kern.c        |  84 ++--
+ 18 files changed, 1060 insertions(+), 433 deletions(-)
+ create mode 100644 tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c
+Merging ipsec/master (76df934c6d5f MAINTAINERS: Add netdev subsystem profile link)
+$ git merge -m Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec.git ipsec/master
+Already up to date.
+Merging netfilter/main (54d4434da824 Merge branch 'hv_netvsc-fix-race-of-netvsc-vf-register-and-slave-bit')
+$ git merge -m Merge branch 'main' of git://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf.git netfilter/main
+Already up to date.
+Merging ipvs/main (a63b6622120c net/sched: act_ct: additional checks for outdated flows)
+$ git merge -m Merge branch 'main' of git://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs.git ipvs/main
+Already up to date.
+Merging wireless/for-next (7a934b5cc3f4 wifi: mt76: mt7921: fix 6GHz disabled by the missing default CLC config)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless.git wireless/for-next
+Merge made by the 'ort' strategy.
+ drivers/net/wireless/mediatek/mt76/mt7921/mcu.c | 1 +
+ 1 file changed, 1 insertion(+)
+Merging wpan/master (2d1c882d4434 Merge tag 'mlx5-fixes-2023-10-12' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux)
+$ git merge -m Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan.git wpan/master
+Already up to date.
+Merging rdma-fixes/for-rc (b6f09b16558f MAINTAINERS: Add Chengchang Tang as Hisilicon RoCE maintainer)
+$ git merge -m Merge branch 'for-rc' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git rdma-fixes/for-rc
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                                |  1 +
+ drivers/infiniband/core/verbs.c            |  2 +-
+ drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 13 +++++++++----
+ drivers/infiniband/hw/irdma/hw.c           |  6 +++++-
+ drivers/infiniband/hw/irdma/main.c         |  2 +-
+ drivers/infiniband/hw/irdma/main.h         |  2 +-
+ drivers/infiniband/hw/irdma/verbs.c        | 28 +++++++++++++++++++++-------
+ drivers/infiniband/hw/irdma/verbs.h        |  1 +
+ 8 files changed, 40 insertions(+), 15 deletions(-)
+Merging sound-current/for-linus (d04ce4113cb4 ALSA: cs35l41: Fix for old systems which do not support command)
+$ git merge -m Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git sound-current/for-linus
+Merge made by the 'ort' strategy.
+ include/sound/cs35l41.h        |  2 +-
+ sound/pci/hda/cs35l41_hda.c    | 28 ++++++++++++----------------
+ sound/pci/hda/patch_realtek.c  | 10 ++++++++++
+ sound/soc/codecs/cs35l41-lib.c |  6 ++++--
+ sound/soc/codecs/cs35l41.c     |  4 ++--
+ 5 files changed, 29 insertions(+), 21 deletions(-)
+Merging sound-asoc-fixes/for-linus (14e8442e0789 ASoC: fsl_sai: Fix no frame sync clock issue on i.MX8MP)
+$ git merge -m Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git sound-asoc-fixes/for-linus
+Merge made by the 'ort' strategy.
+ sound/soc/amd/yc/acp6x-mach.c         |  7 +++++++
+ sound/soc/codecs/cs43130.c            |  6 +++---
+ sound/soc/codecs/nau8822.c            |  9 ++++++---
+ sound/soc/codecs/wm8974.c             |  6 +++---
+ sound/soc/fsl/fsl_sai.c               | 21 +++++++++++++++++++++
+ sound/soc/intel/skylake/skl-pcm.c     |  9 +++++++--
+ sound/soc/intel/skylake/skl-sst-ipc.c |  4 +++-
+ sound/soc/sof/topology.c              |  4 +++-
+ 8 files changed, 53 insertions(+), 13 deletions(-)
+Merging regmap-fixes/for-linus (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git regmap-fixes/for-linus
+Already up to date.
+Merging regulator-fixes/for-linus (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git regulator-fixes/for-linus
+Already up to date.
+Merging spi-fixes/for-linus (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git spi-fixes/for-linus
+Already up to date.
+Merging pci-current/for-linus (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git pci-current/for-linus
+Already up to date.
+Merging driver-core.current/driver-core-linus (98b1cc82c4af Linux 6.7-rc2)
+$ git merge -m Merge branch 'driver-core-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git driver-core.current/driver-core-linus
+Already up to date.
+Merging tty.current/tty-linus (98b1cc82c4af Linux 6.7-rc2)
+$ git merge -m Merge branch 'tty-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git tty.current/tty-linus
+Already up to date.
+Merging usb.current/usb-linus (16b7e0cccb24 USB: xhci-plat: fix legacy PHY double init)
+$ git merge -m Merge branch 'usb-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git usb.current/usb-linus
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/usb/usb-hcd.yaml |    2 +-
+ drivers/phy/Kconfig                                |    1 -
+ drivers/phy/Makefile                               |    1 -
+ drivers/phy/realtek/Kconfig                        |   32 -
+ drivers/phy/realtek/Makefile                       |    3 -
+ drivers/phy/realtek/phy-rtk-usb2.c                 | 1325 --------------------
+ drivers/phy/realtek/phy-rtk-usb3.c                 |  761 -----------
+ drivers/thunderbolt/switch.c                       |    6 +-
+ drivers/thunderbolt/tb.c                           |   12 +-
+ drivers/usb/cdns3/cdnsp-ring.c                     |    3 +
+ drivers/usb/core/config.c                          |    3 +-
+ drivers/usb/core/hub.c                             |   23 -
+ drivers/usb/dwc2/hcd_intr.c                        |   15 +-
+ drivers/usb/dwc3/core.c                            |    2 +
+ drivers/usb/dwc3/drd.c                             |    2 +-
+ drivers/usb/dwc3/dwc3-rtk.c                        |    8 +-
+ drivers/usb/host/xhci-mtk-sch.c                    |   13 +-
+ drivers/usb/host/xhci-mtk.h                        |    2 +
+ drivers/usb/host/xhci-plat.c                       |   50 +-
+ drivers/usb/misc/usb-ljca.c                        |   14 +-
+ drivers/usb/typec/tcpm/tcpm.c                      |   12 +-
+ drivers/usb/typec/tipd/core.c                      |   14 +-
+ include/linux/usb/phy.h                            |   13 -
+ 23 files changed, 101 insertions(+), 2216 deletions(-)
+ delete mode 100644 drivers/phy/realtek/Kconfig
+ delete mode 100644 drivers/phy/realtek/Makefile
+ delete mode 100644 drivers/phy/realtek/phy-rtk-usb2.c
+ delete mode 100644 drivers/phy/realtek/phy-rtk-usb3.c
+Merging usb-serial-fixes/usb-linus (a1092619dd28 USB: serial: option: fix FM101R-GL defines)
+$ git merge -m Merge branch 'usb-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git usb-serial-fixes/usb-linus
+Merge made by the 'ort' strategy.
+ drivers/usb/serial/option.c | 7 ++++---
+ 1 file changed, 4 insertions(+), 3 deletions(-)
+Merging phy/fixes (0f40d5099cd6 phy: ti: gmii-sel: Fix register offset when parent is not a syscon node)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git phy/fixes
+Merge made by the 'ort' strategy.
+ drivers/phy/ti/phy-gmii-sel.c | 5 ++++-
+ 1 file changed, 4 insertions(+), 1 deletion(-)
+Merging staging.current/staging-linus (98b1cc82c4af Linux 6.7-rc2)
+$ git merge -m Merge branch 'staging-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git staging.current/staging-linus
+Already up to date.
+Merging iio-fixes/fixes-togreg (54cf39ec1633 iio: common: ms_sensors: ms_sensors_i2c: fix humidity conversion time table)
+$ git merge -m Merge branch 'fixes-togreg' of git://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git iio-fixes/fixes-togreg
+Merge made by the 'ort' strategy.
+ drivers/iio/accel/kionix-kx022a.c              | 37 ++++++++++++++++++--------
+ drivers/iio/common/ms_sensors/ms_sensors_i2c.c |  4 +--
+ 2 files changed, 28 insertions(+), 13 deletions(-)
+Merging counter-current/counter-current (58720809f527 Linux 6.6-rc6)
+$ git merge -m Merge branch 'counter-current' of git://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git counter-current/counter-current
+Already up to date.
+Merging char-misc.current/char-misc-linus (98b1cc82c4af Linux 6.7-rc2)
+$ git merge -m Merge branch 'char-misc-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git char-misc.current/char-misc-linus
+Already up to date.
+Merging soundwire-fixes/fixes (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git soundwire-fixes/fixes
+Already up to date.
+Merging thunderbolt-fixes/fixes (480713b1ba8e thunderbolt: Only add device router DP IN to the head of the DP resource list)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git thunderbolt-fixes/fixes
+Already up to date.
+Merging input-current/for-linus (c1f342f35f82 Input: psmouse - enable Synaptics InterTouch for ThinkPad L14 G1)
+$ git merge -m Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git input-current/for-linus
+Merge made by the 'ort' strategy.
+ drivers/input/mouse/synaptics.c | 1 +
+ 1 file changed, 1 insertion(+)
+Merging crypto-current/master (9aedd10fe384 crypto: ahash - Set using_shash for cloned ahash wrapper over shash)
+$ git merge -m Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6.git crypto-current/master
+Already up to date.
+Merging vfio-fixes/for-linus (c777b11d34e0 vfio/mdev: Fix a null-ptr-deref bug for mdev_unregister_parent())
+$ git merge -m Merge branch 'for-linus' of git://github.com/awilliam/linux-vfio.git vfio-fixes/for-linus
+Already up to date.
+Merging kselftest-fixes/fixes (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kselftest-fixes/fixes
+Already up to date.
+Merging modules-fixes/modules-linus (f412eef03938 Documentation: livepatch: module-elf-format: Remove local klp_modinfo definition)
+$ git merge -m Merge branch 'modules-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mcgrof/linux.git modules-fixes/modules-linus
+Already up to date.
+Merging dmaengine-fixes/fixes (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git dmaengine-fixes/fixes
+Already up to date.
+Merging backlight-fixes/for-backlight-fixes (88603b6dc419 Linux 6.2-rc2)
+$ git merge -m Merge branch 'for-backlight-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git backlight-fixes/for-backlight-fixes
+Already up to date.
+Merging mtd-fixes/mtd/fixes (7c1b1906229d mtd: spinand: gigadevice: Fix the get ecc status issue)
+$ git merge -m Merge branch 'mtd/fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git mtd-fixes/mtd/fixes
+Merge made by the 'ort' strategy.
+ drivers/mtd/nand/spi/gigadevice.c | 6 ++++--
+ 1 file changed, 4 insertions(+), 2 deletions(-)
+Merging mfd-fixes/for-mfd-fixes (88603b6dc419 Linux 6.2-rc2)
+$ git merge -m Merge branch 'for-mfd-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git mfd-fixes/for-mfd-fixes
+Already up to date.
+Merging v4l-dvb-fixes/fixes (32138be394e5 Merge tag 'media-renesas-fixes-20231113' of git://git.kernel.org/pub/scm/linux/kernel/git/pinchartl/linux.git)
+$ git merge -m Merge branch 'fixes' of https://git.linuxtv.org/media_stage.git v4l-dvb-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/media/pci/mgb4/Kconfig                  |  1 +
+ drivers/media/pci/mgb4/mgb4_core.c              | 20 ++++++++++++-----
+ drivers/media/platform/renesas/vsp1/vsp1_pipe.c |  2 +-
+ drivers/media/platform/renesas/vsp1/vsp1_rpf.c  | 10 +--------
+ drivers/media/platform/renesas/vsp1/vsp1_rwpf.c |  8 +++++--
+ drivers/media/platform/renesas/vsp1/vsp1_rwpf.h |  4 +++-
+ drivers/media/platform/renesas/vsp1/vsp1_wpf.c  | 29 +++----------------------
+ include/uapi/linux/v4l2-subdev.h                |  2 +-
+ 8 files changed, 31 insertions(+), 45 deletions(-)
+Merging reset-fixes/reset/fixes (3a2390c6777e reset: uniphier-glue: Fix possible null-ptr-deref)
+$ git merge -m Merge branch 'reset/fixes' of https://git.pengutronix.de/git/pza/linux reset-fixes/reset/fixes
+Already up to date.
+Merging mips-fixes/mips-fixes (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'mips-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git mips-fixes/mips-fixes
+Already up to date.
+Merging at91-fixes/at91-fixes (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'at91-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git at91-fixes/at91-fixes
+Already up to date.
+Merging omap-fixes/fixes (0b9a4a67c60d clk: ti: Fix missing omap5 mcbsp functional clock and aliases)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tmlind/linux-omap.git omap-fixes/fixes
+Already up to date.
+Merging kvm-fixes/master (ffc253263a13 Linux 6.6)
+$ git merge -m Merge branch 'master' of git://git.kernel.org/pub/scm/virt/kvm/kvm.git kvm-fixes/master
+Already up to date.
+Merging kvms390-fixes/master (27072b8e18a7 KVM: s390/mm: Properly reset no-dat)
+$ git merge -m Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git kvms390-fixes/master
+Merge made by the 'ort' strategy.
+ arch/s390/kvm/vsie.c   | 4 ----
+ arch/s390/mm/pgtable.c | 2 +-
+ 2 files changed, 1 insertion(+), 5 deletions(-)
+Merging hwmon-fixes/hwmon (58ebe7fb6eb2 hwmon: max31827: include regulator header)
+$ git merge -m Merge branch 'hwmon' of git://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git hwmon-fixes/hwmon
+Merge made by the 'ort' strategy.
+ drivers/hwmon/ltc2991.c  | 2 +-
+ drivers/hwmon/max31827.c | 1 +
+ 2 files changed, 2 insertions(+), 1 deletion(-)
+Merging nvdimm-fixes/libnvdimm-fixes (33908660e814 ACPI: NFIT: Fix incorrect calculation of idt size)
+$ git merge -m Merge branch 'libnvdimm-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git nvdimm-fixes/libnvdimm-fixes
+Already up to date.
+Merging cxl-fixes/fixes (8f61d48c83f6 tools/testing/cxl: Slow down the mock firmware transfer)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git cxl-fixes/fixes
+Already up to date.
+Merging btrfs-fixes/next-fixes (9b71b2975adf Merge branch 'misc-6.7' into next-fixes)
+$ git merge -m Merge branch 'next-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git btrfs-fixes/next-fixes
+Merge made by the 'ort' strategy.
+ fs/btrfs/transaction.c  |  2 +-
+ fs/btrfs/tree-checker.c | 39 +++++++++++++++++++++++++++++++++++++++
+ 2 files changed, 40 insertions(+), 1 deletion(-)
+Merging vfs-fixes/fixes (dc32464a5fe4 ceph_wait_on_conflict_unlink(): grab reference before dropping ->d_lock)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git vfs-fixes/fixes
+Already up to date.
+Merging dma-mapping-fixes/for-linus (d5090484b021 swiotlb: do not try to allocate a TLB bigger than MAX_ORDER pages)
+$ git merge -m Merge branch 'for-linus' of git://git.infradead.org/users/hch/dma-mapping.git dma-mapping-fixes/for-linus
+Already up to date.
+Merging drivers-x86-fixes/fixes (a6584711e64d platform/x86: intel_telemetry: Fix kernel doc descriptions)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git drivers-x86-fixes/fixes
+Already up to date.
+Merging samsung-krzk-fixes/fixes (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git samsung-krzk-fixes/fixes
+Already up to date.
+Merging pinctrl-samsung-fixes/fixes (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git pinctrl-samsung-fixes/fixes
+Already up to date.
+Merging devicetree-fixes/dt/linus (8f51593cdcab dt: dt-extract-compatibles: Don't follow symlinks when walking tree)
+$ git merge -m Merge branch 'dt/linus' of git://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git devicetree-fixes/dt/linus
+Merge made by the 'ort' strategy.
+ scripts/dtc/dt-extract-compatibles | 14 ++++++++++++--
+ 1 file changed, 12 insertions(+), 2 deletions(-)
+Merging dt-krzk-fixes/fixes (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git dt-krzk-fixes/fixes
+Already up to date.
+Merging scsi-fixes/fixes (2a0508d9d08f Merge branch '6.7/scsi-staging' into 6.7/scsi-fixes)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git scsi-fixes/fixes
+Already up to date.
+Merging drm-fixes/drm-fixes (98b1cc82c4af Linux 6.7-rc2)
+$ git merge -m Merge branch 'drm-fixes' of git://git.freedesktop.org/git/drm/drm.git drm-fixes/drm-fixes
+Already up to date.
+Merging drm-intel-fixes/for-linux-next-fixes (0561794b6b64 drm/i915: do not clean GT table on error path)
+$ git merge -m Merge branch 'for-linux-next-fixes' of git://anongit.freedesktop.org/drm-intel drm-intel-fixes/for-linux-next-fixes
+Merge made by the 'ort' strategy.
+ drivers/gpu/drm/i915/display/intel_dp_mst.c | 16 ++++++++--------
+ drivers/gpu/drm/i915/gt/intel_gt.c          | 11 -----------
+ drivers/gpu/drm/i915/i915_driver.c          |  4 +---
+ 3 files changed, 9 insertions(+), 22 deletions(-)
+Merging mmc-fixes/fixes (1de1b77982e1 mmc: cqhci: Fix task clearing in CQE error recovery)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git mmc-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/mmc/core/block.c      |  2 ++
+ drivers/mmc/core/core.c       |  9 +++++++--
+ drivers/mmc/host/cqhci-core.c | 44 +++++++++++++++++++++----------------------
+ 3 files changed, 31 insertions(+), 24 deletions(-)
+Merging rtc-fixes/rtc-fixes (08279468a294 rtc: sunplus: fix format string for printing resource)
+$ git merge -m Merge branch 'rtc-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git rtc-fixes/rtc-fixes
+Already up to date.
+Merging gnss-fixes/gnss-linus (98b1cc82c4af Linux 6.7-rc2)
+$ git merge -m Merge branch 'gnss-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git gnss-fixes/gnss-linus
+Already up to date.
+Merging hyperv-fixes/hyperv-fixes (7e8037b099c0 x86/hyperv: Fix the detection of E820_TYPE_PRAM in a Gen2 VM)
+$ git merge -m Merge branch 'hyperv-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git hyperv-fixes/hyperv-fixes
+Merge made by the 'ort' strategy.
+ arch/x86/hyperv/hv_init.c   | 25 +++++++++++++++++++++----
+ tools/hv/hv_kvp_daemon.c    | 20 ++++++++++++--------
+ tools/hv/hv_set_ifconfig.sh |  4 ++--
+ 3 files changed, 35 insertions(+), 14 deletions(-)
+Merging soc-fsl-fixes/fix (06c2afb862f9 Linux 6.5-rc1)
+$ git merge -m Merge branch 'fix' of git://git.kernel.org/pub/scm/linux/kernel/git/leo/linux.git soc-fsl-fixes/fix
+Already up to date.
+Merging risc-v-fixes/fixes (3ca112b71f35 Merge tag 'probes-fixes-v6.7-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git risc-v-fixes/fixes
+Already up to date.
+Merging riscv-dt-fixes/riscv-dt-fixes (e80ed63affc9 riscv: dts: sophgo: remove address-cells from intc node)
+$ git merge -m Merge branch 'riscv-dt-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-dt-fixes/riscv-dt-fixes
+Merge made by the 'ort' strategy.
+ arch/riscv/boot/dts/sophgo/cv1800b.dtsi | 1 -
+ 1 file changed, 1 deletion(-)
+Merging riscv-soc-fixes/riscv-soc-fixes (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'riscv-soc-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-soc-fixes/riscv-soc-fixes
+Already up to date.
+Merging fpga-fixes/fixes (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git fpga-fixes/fixes
+Already up to date.
+Merging spdx/spdx-linus (d2f51b3516da Merge tag 'rtc-6.7' of git://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux)
+$ git merge -m Merge branch 'spdx-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/spdx.git spdx/spdx-linus
+Already up to date.
+Merging gpio-brgl-fixes/gpio/for-current (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'gpio/for-current' of git://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git gpio-brgl-fixes/gpio/for-current
+Already up to date.
+Merging gpio-intel-fixes/fixes (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git gpio-intel-fixes/fixes
+Already up to date.
+Merging pinctrl-intel-fixes/fixes (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git pinctrl-intel-fixes/fixes
+Already up to date.
+Merging erofs-fixes/fixes (62b241efff99 MAINTAINERS: erofs: add EROFS webpage)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git erofs-fixes/fixes
+Already up to date.
+Merging kunit-fixes/kunit-fixes (1bddcf77ce66 kunit: test: Avoid cast warning when adding kfree() as an action)
+$ git merge -m Merge branch 'kunit-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit-fixes/kunit-fixes
+Merge made by the 'ort' strategy.
+ lib/kunit/kunit-test.c |  2 +-
+ lib/kunit/test.c       | 42 ++++++++++++++++++++++++++++++++++++++++--
+ 2 files changed, 41 insertions(+), 3 deletions(-)
+Merging ubifs-fixes/fixes (2241ab53cbb5 Linux 6.2-rc5)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/rw/ubifs.git ubifs-fixes/fixes
+Already up to date.
+Merging memblock-fixes/fixes (55122e0130e5 memblock tests: fix warning ‘struct seq_file’ declared inside parameter list)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git memblock-fixes/fixes
+Already up to date.
+Merging nfsd-fixes/nfsd-fixes (bf51c52a1f3c NFSD: Fix checksum mismatches in the duplicate reply cache)
+$ git merge -m Merge branch 'nfsd-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/cel/linux nfsd-fixes/nfsd-fixes
+Already up to date.
+Merging irqchip-fixes/irq/irqchip-fixes (b673fe1a6229 MAINTAINERS: Remove myself from the general IRQ subsystem maintenance)
+$ git merge -m Merge branch 'irq/irqchip-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/maz/arm-platforms.git irqchip-fixes/irq/irqchip-fixes
+Already up to date.
+Merging renesas-fixes/fixes (9eab43facdad soc: renesas: ARCH_R9A07G043 depends on !RISCV_ISA_ZICBOM)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git renesas-fixes/fixes
+Already up to date.
+Merging broadcom-fixes/fixes (9abf2313adc1 Linux 6.1-rc1)
+$ git merge -m Merge branch 'fixes' of https://github.com/Broadcom/stblinux.git broadcom-fixes/fixes
+Already up to date.
+Merging perf-current/perf-tools (98b1cc82c4af Linux 6.7-rc2)
+$ git merge -m Merge branch 'perf-tools' of git://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools perf-current/perf-tools
+Already up to date.
+Merging efi-fixes/urgent (c03d21f05e76 Merge 3rd batch of EFI fixes into efi/urgent)
+$ git merge -m Merge branch 'urgent' of git://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git efi-fixes/urgent
+Already up to date.
+Merging zstd-fixes/zstd-linus (77618db34645 zstd: Fix array-index-out-of-bounds UBSAN warning)
+$ git merge -m Merge branch 'zstd-linus' of https://github.com/terrelln/linux.git zstd-fixes/zstd-linus
+Already up to date.
+Merging battery-fixes/fixes (f37669119423 power: supply: cw2015: correct time_to_empty units in sysfs)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git battery-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/power/supply/cw2015_battery.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging uml-fixes/fixes (73a23d771033 um: harddog: fix modular build)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git uml-fixes/fixes
+Already up to date.
+Merging asahi-soc-fixes/asahi-soc/fixes (568035b01cfb Linux 6.0-rc1)
+$ git merge -m Merge branch 'asahi-soc/fixes' of https://github.com/AsahiLinux/linux.git asahi-soc-fixes/asahi-soc/fixes
+Already up to date.
+Merging iommufd-fixes/for-rc (98594181944d iommufd/selftest: Fix _test_mock_dirty_bitmaps())
+$ git merge -m Merge branch 'for-rc' of git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git iommufd-fixes/for-rc
+Merge made by the 'ort' strategy.
+ tools/testing/selftests/iommu/iommufd_utils.h | 13 ++++++-------
+ 1 file changed, 6 insertions(+), 7 deletions(-)
+Merging rust-fixes/rust-fixes (cfd96726e611 rust: docs: fix logo replacement)
+$ git merge -m Merge branch 'rust-fixes' of https://github.com/Rust-for-Linux/linux.git rust-fixes/rust-fixes
+Already up to date.
+Merging v9fs-fixes/fixes/next (2dde18cd1d8f Linux 6.5)
+$ git merge -m Merge branch 'fixes/next' of git://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git v9fs-fixes/fixes/next
+Already up to date.
+Merging w1-fixes/fixes (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git w1-fixes/fixes
+Already up to date.
+Merging pmdomain-fixes/fixes (b131329b9bfb pmdomain: amlogic: Fix mask for the second NNA mem PD domain)
+$ git merge -m Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git pmdomain-fixes/fixes
+Already up to date.
+Merging overlayfs-fixes/ovl-fixes (37f32f526438 ovl: fix memory leak in ovl_parse_param())
+$ git merge -m Merge branch 'ovl-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git overlayfs-fixes/ovl-fixes
+Already up to date.
+Merging drm-misc-fixes/for-linux-next-fixes (ab93edb2f94c nouveau/gsp: allocate enough space for all channel ids.)
+$ git merge -m Merge branch 'for-linux-next-fixes' of git://anongit.freedesktop.org/drm/drm-misc drm-misc-fixes/for-linux-next-fixes
+Merge made by the 'ort' strategy.
+ drivers/accel/ivpu/ivpu_hw_37xx.c               | 46 +++++++++---------
+ drivers/gpu/drm/ast/ast_drv.h                   | 13 +++++-
+ drivers/gpu/drm/ast/ast_mode.c                  | 62 ++++++++++++++++++++++---
+ drivers/gpu/drm/nouveau/nvkm/engine/fifo/r535.c |  2 +-
+ drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c  |  9 ++--
+ drivers/gpu/drm/panel/panel-simple.c            | 13 +++---
+ drivers/gpu/drm/rockchip/rockchip_drm_vop.c     | 14 ++++--
+ 7 files changed, 113 insertions(+), 46 deletions(-)
+Merging kspp/for-next/kspp (1ee60356c2dc gcc-plugins: randstruct: Only warn about true flexible arrays)
+$ git merge -m Merge branch 'for-next/kspp' of git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git kspp/for-next/kspp
+Already up to date.
+Merging kspp-gustavo/for-next/kspp (1b56c90018f0 Makefile: Enable -Wstringop-overflow globally)
+$ git merge -m Merge branch 'for-next/kspp' of git://git.kernel.org/pub/scm/linux/kernel/git/gustavoars/linux.git kspp-gustavo/for-next/kspp
+Auto-merging Makefile
+Merge made by the 'ort' strategy.
+ Makefile                               | 2 ++
+ arch/powerpc/crypto/aes-gcm-p10-glue.c | 2 +-
+ arch/powerpc/lib/sstep.c               | 4 ++--
+ scripts/Makefile.extrawarn             | 2 --
+ 4 files changed, 5 insertions(+), 5 deletions(-)
+Merging mm-stable/mm-stable (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'mm-stable' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-stable/mm-stable
+Already up to date.
+Merging mm-nonmm-stable/mm-nonmm-stable (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'mm-nonmm-stable' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-nonmm-stable/mm-nonmm-stable
+Already up to date.
+Merging mm/mm-everything (2e2d7c098f1b Merge branch 'mm-nonmm-unstable' into mm-everything)
+$ git merge -m Merge branch 'mm-everything' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm/mm-everything
+Auto-merging Makefile
+Auto-merging fs/bcachefs/fs.c
+Auto-merging fs/btrfs/inode.c
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/blockdev/zram.rst |    2 +-
+ Documentation/admin-guide/mm/pagemap.rst    |    1 +
+ Documentation/core-api/maple_tree.rst       |    4 +
+ Documentation/filesystems/locking.rst       |    4 +-
+ Documentation/filesystems/squashfs.rst      |   60 ++
+ Documentation/filesystems/vfs.rst           |    6 +-
+ Documentation/mm/arch_pgtable_helpers.rst   |    2 -
+ Makefile                                    |    6 +-
+ arch/Kconfig                                |   13 -
+ arch/loongarch/kernel/numa.c                |   28 +-
+ arch/powerpc/kexec/core_64.c                |    3 +-
+ arch/x86/mm/numa.c                          |   34 +-
+ block/fops.c                                |    2 +-
+ drivers/block/zram/Kconfig                  |   15 +-
+ drivers/block/zram/zram_drv.c               |   27 +-
+ drivers/block/zram/zram_drv.h               |    2 +-
+ drivers/dax/bus.c                           |    3 +
+ drivers/dax/bus.h                           |    1 +
+ drivers/dax/cxl.c                           |    1 +
+ drivers/dax/dax-private.h                   |    1 +
+ drivers/dax/hmem/hmem.c                     |    1 +
+ drivers/dax/kmem.c                          |    8 +-
+ drivers/dax/pmem.c                          |    1 +
+ fs/afs/write.c                              |    8 +-
+ fs/bcachefs/fs.c                            |    2 +-
+ fs/btrfs/inode.c                            |    2 +-
+ fs/buffer.c                                 |  130 ++--
+ fs/ceph/addr.c                              |    4 +-
+ fs/exec.c                                   |    3 +-
+ fs/ext2/inode.c                             |    2 +-
+ fs/ext4/inline.c                            |    3 +-
+ fs/ext4/inode.c                             |    6 +-
+ fs/f2fs/compress.c                          |    2 +-
+ fs/f2fs/inode.c                             |    2 +-
+ fs/gfs2/aops.c                              |   41 +-
+ fs/hugetlbfs/inode.c                        |    6 +-
+ fs/iomap/buffered-io.c                      |   14 +-
+ fs/nfs/file.c                               |    2 +-
+ fs/nilfs2/btnode.c                          |   62 +-
+ fs/nilfs2/file.c                            |   28 +-
+ fs/nilfs2/gcinode.c                         |    4 +-
+ fs/nilfs2/inode.c                           |   11 +-
+ fs/nilfs2/ioctl.c                           |   10 +-
+ fs/nilfs2/mdt.c                             |   23 +-
+ fs/nilfs2/page.c                            |   68 +-
+ fs/nilfs2/page.h                            |    6 +-
+ fs/nilfs2/segment.c                         |  157 ++--
+ fs/ntfs/aops.c                              |    6 +-
+ fs/ocfs2/aops.c                             |    2 +-
+ fs/proc/task_mmu.c                          |   17 +-
+ fs/smb/client/file.c                        |    6 +-
+ fs/squashfs/file.c                          |    3 +-
+ fs/squashfs/file_direct.c                   |    6 +-
+ fs/xfs/xfs_aops.c                           |    2 +-
+ fs/zonefs/file.c                            |    2 +-
+ include/linux/fs.h                          |    2 +-
+ include/linux/gfp_types.h                   |    5 +-
+ include/linux/highmem.h                     |   76 ++
+ include/linux/init_task.h                   |    7 -
+ include/linux/maple_tree.h                  |  411 ++++++-----
+ include/linux/memblock.h                    |    1 +
+ include/linux/memremap.h                    |   12 -
+ include/linux/mm.h                          |   40 +-
+ include/linux/mm_types.h                    |    3 +-
+ include/linux/mmzone.h                      |    4 +
+ include/linux/page-flags.h                  |    9 +-
+ include/linux/sched.h                       |    2 -
+ include/linux/sched/signal.h                |    4 +-
+ include/linux/stackdepot.h                  |   59 +-
+ include/linux/swap.h                        |    6 +
+ include/linux/vm_event_item.h               |    3 -
+ include/uapi/linux/fs.h                     |    1 +
+ init/init_task.c                            |   10 +-
+ kernel/fork.c                               |   66 +-
+ kernel/reboot.c                             |   17 +-
+ kernel/signal.c                             |   28 +-
+ lib/Kconfig                                 |   10 +
+ lib/Kconfig.kasan                           |    2 +-
+ lib/maple_tree.c                            | 1066 +++++++++++++++++----------
+ lib/stackdepot.c                            |  456 ++++++++----
+ lib/test_maple_tree.c                       |  332 +++++----
+ mm/damon/core-test.h                        |   15 +-
+ mm/filemap.c                                |    2 +-
+ mm/folio-compat.c                           |    4 +-
+ mm/huge_memory.c                            |    4 +-
+ mm/internal.h                               |   23 +-
+ mm/kasan/common.c                           |    8 +-
+ mm/kasan/generic.c                          |   57 +-
+ mm/kasan/kasan.h                            |    2 +-
+ mm/kasan/quarantine.c                       |   26 +-
+ mm/kasan/report_tags.c                      |   27 +-
+ mm/kasan/tags.c                             |   24 +-
+ mm/kmemleak.c                               |    8 +-
+ mm/kmsan/core.c                             |    7 +-
+ mm/ksm.c                                    |    4 +-
+ mm/memblock.c                               |   34 +
+ mm/memcontrol.c                             |    8 +
+ mm/memory-failure.c                         |  115 ++-
+ mm/memory.c                                 |   33 +-
+ mm/memory_hotplug.c                         |  210 ++++--
+ mm/mempool.c                                |    8 +-
+ mm/memremap.c                               |   14 +-
+ mm/mmap.c                                   |   46 +-
+ mm/page-writeback.c                         |   54 +-
+ mm/page_alloc.c                             |   56 +-
+ mm/page_owner.c                             |    9 +-
+ mm/page_poison.c                            |    8 +-
+ mm/process_vm_access.c                      |   15 +-
+ mm/shmem.c                                  |    6 +-
+ mm/truncate.c                               |   51 +-
+ mm/util.c                                   |    8 +-
+ mm/vmscan.c                                 |   36 +-
+ mm/vmstat.c                                 |    6 +-
+ mm/zswap.c                                  |    5 +
+ scripts/checkpatch.pl                       |    2 +-
+ scripts/checkstack.pl                       |   19 +-
+ tools/include/linux/rwsem.h                 |    4 +
+ tools/include/linux/spinlock.h              |    1 +
+ tools/include/uapi/linux/fs.h               |    1 +
+ tools/testing/radix-tree/linux.c            |   45 +-
+ tools/testing/radix-tree/linux/maple_tree.h |    2 +-
+ tools/testing/radix-tree/maple.c            |  390 +++++++++-
+ tools/testing/selftests/mm/vm_util.c        |   80 +-
+ 123 files changed, 3107 insertions(+), 1867 deletions(-)
+Merging kbuild/for-next (39d5b46d86e7 kconfig: do not clear SYMBOL_DEF_USER when the value is out of range)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/masahiroy/linux-kbuild.git kbuild/for-next
+Merge made by the 'ort' strategy.
+ scripts/genksyms/genksyms.c | 22 +---------------------
+ scripts/kconfig/confdata.c  |  2 +-
+ 2 files changed, 2 insertions(+), 22 deletions(-)
+Merging clang-format/clang-format (5d0c230f1de8 Linux 6.5-rc4)
+$ git merge -m Merge branch 'clang-format' of https://github.com/ojeda/linux.git clang-format/clang-format
+Already up to date.
+Merging perf/perf-tools-next (acbf6de674ef perf vendor events riscv: Add StarFive Dubhe-80 JSON file)
+$ git merge -m Merge branch 'perf-tools-next' of git://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git perf/perf-tools-next
+Auto-merging tools/perf/util/Build
+Merge made by the 'ort' strategy.
+ tools/build/Makefile.feature                       |   1 +
+ tools/build/feature/Makefile                       |   4 +
+ tools/build/feature/test-dwarf_getcfi.c            |   9 +
+ tools/include/asm-generic/unaligned.h              |   1 +
+ tools/include/uapi/linux/perf_event.h              |  13 ++
+ tools/perf/.gitignore                              |   1 +
+ tools/perf/Documentation/perf-config.txt           |   4 +
+ tools/perf/Documentation/perf-list.txt             |  12 +-
+ tools/perf/Documentation/perf-record.txt           |   4 +
+ tools/perf/Makefile.config                         |  13 +-
+ tools/perf/arch/loongarch/annotate/instructions.c  |   6 +-
+ tools/perf/builtin-annotate.c                      |   7 +-
+ tools/perf/builtin-inject.c                        |   6 +
+ tools/perf/builtin-record.c                        |   2 +
+ tools/perf/builtin-report.c                        |   7 +
+ tools/perf/pmu-events/arch/riscv/mapfile.csv       |   1 +
+ .../arch/riscv/starfive/dubhe-80/common.json       | 172 +++++++++++++++++
+ .../arch/riscv/starfive/dubhe-80/firmware.json     |  68 +++++++
+ tools/perf/tests/attr.c                            |   2 +-
+ tools/perf/tests/builtin-test.c                    |  15 ++
+ tools/perf/tests/code-reading.c                    |  78 +++-----
+ tools/perf/tests/make                              |   2 +-
+ tools/perf/tests/parse-events.c                    |   4 +-
+ tools/perf/tests/shell/record_offcpu.sh            |   4 +-
+ tools/perf/tests/shell/stat_all_pmu.sh             |   2 +-
+ tools/perf/tests/tests.h                           |   1 +
+ tools/perf/trace/beauty/prctl_option.sh            |   4 +-
+ tools/perf/trace/beauty/socket.sh                  |   4 +-
+ tools/perf/ui/browsers/annotate.c                  |  18 +-
+ tools/perf/util/Build                              |   1 +
+ tools/perf/util/annotate.c                         | 211 +++++++++++++--------
+ tools/perf/util/annotate.h                         |  56 +++---
+ tools/perf/util/block-info.c                       |   4 +-
+ tools/perf/util/block-range.c                      |   7 +-
+ tools/perf/util/bpf-event.c                        |   3 +
+ tools/perf/util/debuginfo.c                        | 205 ++++++++++++++++++++
+ tools/perf/util/debuginfo.h                        |  64 +++++++
+ tools/perf/util/dwarf-aux.c                        | 132 ++++++++++++-
+ tools/perf/util/dwarf-aux.h                        |  34 +++-
+ tools/perf/util/env.c                              |  18 ++
+ tools/perf/util/env.h                              |   7 +
+ tools/perf/util/event.c                            |   4 +-
+ tools/perf/util/evsel.c                            |  35 +++-
+ tools/perf/util/evsel.h                            |   1 +
+ tools/perf/util/header.c                           |  26 ++-
+ tools/perf/util/machine.c                          |  10 +-
+ tools/perf/util/parse-branch-options.c             |   1 +
+ tools/perf/util/perf_event_attr_fprintf.c          |   1 +
+ tools/perf/util/probe-finder.c                     | 201 +-------------------
+ tools/perf/util/probe-finder.h                     |  19 +-
+ tools/perf/util/s390-cpumcf-kernel.h               |   2 +
+ tools/perf/util/s390-sample-raw.c                  | 113 ++++++++++-
+ tools/perf/util/sample.h                           |   1 +
+ tools/perf/util/session.c                          |  20 +-
+ tools/perf/util/sort.c                             |  14 +-
+ tools/perf/util/symbol_conf.h                      |   4 +-
+ tools/perf/util/thread.h                           |  14 ++
+ 57 files changed, 1226 insertions(+), 447 deletions(-)
+ create mode 100644 tools/build/feature/test-dwarf_getcfi.c
+ create mode 100644 tools/perf/pmu-events/arch/riscv/starfive/dubhe-80/common.json
+ create mode 100644 tools/perf/pmu-events/arch/riscv/starfive/dubhe-80/firmware.json
+ create mode 100644 tools/perf/util/debuginfo.c
+ create mode 100644 tools/perf/util/debuginfo.h
+Merging compiler-attributes/compiler-attributes (5d0c230f1de8 Linux 6.5-rc4)
+$ git merge -m Merge branch 'compiler-attributes' of https://github.com/ojeda/linux.git compiler-attributes/compiler-attributes
+Already up to date.
+Merging dma-mapping/for-next (53c87e846e33 swiotlb: fix out-of-bounds TLB allocations with CONFIG_SWIOTLB_DYNAMIC)
+$ git merge -m Merge branch 'for-next' of git://git.infradead.org/users/hch/dma-mapping.git dma-mapping/for-next
+Already up to date.
+Merging asm-generic/master (550087a0ba91 hexagon: Remove unusable symbols from the ptrace.h uapi)
+$ git merge -m Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/arnd/asm-generic.git asm-generic/master
+Already up to date.
+Merging arc/for-next (0bb80ecc33a8 Linux 6.6-rc1)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/vgupta/arc.git arc/for-next
+Already up to date.
+Merging arm/for-next (c7368ddba2ff ARM: 9326/1: make <linux/uaccess.h> self-contained for ARM)
+$ git merge -m Merge branch 'for-next' of git://git.armlinux.org.uk/~rmk/linux-arm.git arm/for-next
+Already up to date.
+Merging arm64/for-next/core (f86128050d2d arm64/syscall: Remove duplicate declaration)
+$ git merge -m Merge branch 'for-next/core' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux arm64/for-next/core
+Already up to date.
+Merging arm-perf/for-next/perf (b805cafc604b perf: hisi: Fix use-after-free when register pmu fails)
+$ git merge -m Merge branch 'for-next/perf' of git://git.kernel.org/pub/scm/linux/kernel/git/will/linux.git arm-perf/for-next/perf
+Already up to date.
+Merging arm-soc/for-next (1b52f65d88ad Merge branch 'soc/defconfig' into for-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git arm-soc/for-next
+Auto-merging arch/arm/boot/dts/qcom/qcom-msm8226.dtsi
+Auto-merging drivers/soc/renesas/Kconfig
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/arm/qcom.yaml    |   10 +
+ arch/arm/arm-soc-for-next-contents.txt             |  269 +++++
+ .../intel/ixp/intel-ixp42x-usrobotics-usr8200.dts  |   22 +
+ arch/arm/boot/dts/qcom/Makefile                    |    5 +
+ .../dts/qcom/qcom-msm8226-microsoft-common.dtsi    |  327 ++++++
+ .../dts/qcom/qcom-msm8226-microsoft-dempsey.dts    |   17 +
+ .../dts/qcom/qcom-msm8226-microsoft-makepeace.dts  |   17 +
+ .../dts/qcom/qcom-msm8226-microsoft-moneypenny.dts |   23 +
+ arch/arm/boot/dts/qcom/qcom-msm8226.dtsi           |    4 +-
+ .../qcom/qcom-msm8926-microsoft-superman-lte.dts   |   53 +
+ .../boot/dts/qcom/qcom-msm8926-microsoft-tesla.dts |   67 ++
+ arch/arm/boot/dts/qcom/qcom-msm8974.dtsi           |   80 +-
+ arch/arm64/boot/dts/qcom/ipq5018-rdp432-c2.dts     |   12 +
+ arch/arm64/boot/dts/qcom/ipq5018.dtsi              |   78 ++
+ arch/arm64/boot/dts/qcom/ipq5332.dtsi              |   23 +-
+ arch/arm64/boot/dts/qcom/ipq6018.dtsi              |    4 +-
+ arch/arm64/boot/dts/qcom/ipq8074.dtsi              |    4 +-
+ arch/arm64/boot/dts/qcom/ipq9574-rdp-common.dtsi   |  147 +++
+ arch/arm64/boot/dts/qcom/ipq9574-rdp418.dts        |   63 +-
+ arch/arm64/boot/dts/qcom/ipq9574-rdp433.dts        |   91 +-
+ arch/arm64/boot/dts/qcom/ipq9574-rdp449.dts        |   65 +-
+ arch/arm64/boot/dts/qcom/ipq9574-rdp453.dts        |   65 +-
+ arch/arm64/boot/dts/qcom/ipq9574-rdp454.dts        |   66 +-
+ arch/arm64/boot/dts/qcom/ipq9574.dtsi              |   25 +-
+ .../boot/dts/qcom/msm8916-alcatel-idol347.dts      |  173 ++-
+ arch/arm64/boot/dts/qcom/msm8916-asus-z00l.dts     |   21 +
+ .../boot/dts/qcom/msm8916-longcheer-l8150.dts      |   31 +-
+ .../boot/dts/qcom/msm8916-longcheer-l8910.dts      |  103 ++
+ arch/arm64/boot/dts/qcom/msm8916-modem-qdsp6.dtsi  |  148 +++
+ .../dts/qcom/msm8916-samsung-a2015-common.dtsi     |   71 ++
+ .../boot/dts/qcom/msm8916-samsung-a3u-eur.dts      |   10 +-
+ .../dts/qcom/msm8916-samsung-e2015-common.dtsi     |    4 +
+ .../boot/dts/qcom/msm8916-samsung-grandmax.dts     |    4 +
+ .../boot/dts/qcom/msm8916-samsung-gt5-common.dtsi  |   92 +-
+ arch/arm64/boot/dts/qcom/msm8916-samsung-gt510.dts |   26 +
+ arch/arm64/boot/dts/qcom/msm8916-samsung-gt58.dts  |   46 +
+ .../boot/dts/qcom/msm8916-samsung-j5-common.dtsi   |   14 +
+ arch/arm64/boot/dts/qcom/msm8916-samsung-j5.dts    |    4 +
+ .../boot/dts/qcom/msm8916-samsung-serranove.dts    |   13 +
+ arch/arm64/boot/dts/qcom/msm8916-thwc-uf896.dts    |    8 +-
+ arch/arm64/boot/dts/qcom/msm8916-thwc-ufi001c.dts  |    8 +-
+ .../boot/dts/qcom/msm8916-wingtech-wt88047.dts     |   74 ++
+ .../arm64/boot/dts/qcom/msm8916-yiming-uz801v3.dts |    8 +-
+ arch/arm64/boot/dts/qcom/msm8916.dtsi              |   49 +
+ .../boot/dts/qcom/msm8939-longcheer-l9100.dts      |   55 +
+ arch/arm64/boot/dts/qcom/msm8939-samsung-a7.dts    |   70 ++
+ arch/arm64/boot/dts/qcom/msm8939.dtsi              |   79 ++
+ arch/arm64/boot/dts/qcom/pm7250b.dtsi              |    2 +-
+ arch/arm64/boot/dts/qcom/qrb4210-rb2.dts           |    1 -
+ arch/arm64/boot/dts/qcom/sa8775p.dtsi              | 1106 ++++++++++++++++++++
+ arch/arm64/boot/dts/qcom/sc7280.dtsi               |  167 +++
+ .../dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts     |   11 +
+ arch/arm64/boot/dts/qcom/sdm670.dtsi               |    2 +-
+ .../arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi |   23 +
+ .../dts/qcom/sdm845-xiaomi-beryllium-common.dtsi   |   22 +
+ .../dts/qcom/sm6375-sony-xperia-murray-pdx225.dts  |   29 +-
+ arch/arm64/boot/dts/qcom/sm6375.dtsi               |   43 +
+ arch/arm64/boot/dts/qcom/sm8450.dtsi               |    5 +
+ arch/arm64/boot/dts/qcom/sm8550.dtsi               |    5 +
+ 59 files changed, 3616 insertions(+), 448 deletions(-)
+ create mode 100644 arch/arm/arm-soc-for-next-contents.txt
+ create mode 100644 arch/arm/boot/dts/qcom/qcom-msm8226-microsoft-common.dtsi
+ create mode 100644 arch/arm/boot/dts/qcom/qcom-msm8226-microsoft-dempsey.dts
+ create mode 100644 arch/arm/boot/dts/qcom/qcom-msm8226-microsoft-makepeace.dts
+ create mode 100644 arch/arm/boot/dts/qcom/qcom-msm8226-microsoft-moneypenny.dts
+ create mode 100644 arch/arm/boot/dts/qcom/qcom-msm8926-microsoft-superman-lte.dts
+ create mode 100644 arch/arm/boot/dts/qcom/qcom-msm8926-microsoft-tesla.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/ipq9574-rdp-common.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/msm8916-modem-qdsp6.dtsi
+Merging amlogic/for-next (996fc07dce79 Merge branch 'v6.7/defconfig' into for-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/amlogic/linux.git amlogic/for-next
+Merge made by the 'ort' strategy.
+Merging asahi-soc/asahi-soc/for-next (ffc253263a13 Linux 6.6)
+$ git merge -m Merge branch 'asahi-soc/for-next' of https://github.com/AsahiLinux/linux.git asahi-soc/asahi-soc/for-next
+Already up to date.
+Merging aspeed/for-next (e9be9b34b841 ARM: dts: aspeed: Add ASRock E3C256D4I BMC)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/joel/bmc.git aspeed/for-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/arm/aspeed/aspeed.yaml     |   2 +
+ arch/arm/boot/dts/aspeed/Makefile                  |   2 +
+ .../dts/aspeed/aspeed-bmc-asrock-e3c256d4i.dts     | 316 ++++++++++++++++++++
+ .../dts/aspeed/aspeed-bmc-asrock-spc621d8hm3.dts   | 318 +++++++++++++++++++++
+ 4 files changed, 638 insertions(+)
+ create mode 100644 arch/arm/boot/dts/aspeed/aspeed-bmc-asrock-e3c256d4i.dts
+ create mode 100644 arch/arm/boot/dts/aspeed/aspeed-bmc-asrock-spc621d8hm3.dts
+Merging at91/at91-next (b766b70117ac Merge branch 'at91-dt' into at91-next)
+$ git merge -m Merge branch 'at91-next' of git://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git at91/at91-next
+Merge made by the 'ort' strategy.
+ arch/arm/boot/dts/microchip/at91-sam9x60_curiosity.dts | 3 +++
+ arch/arm/boot/dts/microchip/at91-sam9x60ek.dts         | 3 +++
+ arch/arm/mach-at91/pm.c                                | 3 +++
+ 3 files changed, 9 insertions(+)
+Merging broadcom/next (62a3c97f8167 Merge branch 'devicetree/next' into next)
+$ git merge -m Merge branch 'next' of https://github.com/Broadcom/stblinux.git broadcom/next
+Merge made by the 'ort' strategy.
+Merging davinci/davinci/for-next (06c2afb862f9 Linux 6.5-rc1)
+$ git merge -m Merge branch 'davinci/for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git davinci/davinci/for-next
+Already up to date.
+Merging drivers-memory/for-next (0d6c918011ce memory: tegra: Skip SID programming if SID registers aren't set)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git drivers-memory/for-next
+Merge made by the 'ort' strategy.
+ drivers/memory/tegra/tegra186.c | 17 +++++++++++++++++
+ include/soc/tegra/mc.h          |  1 +
+ 2 files changed, 18 insertions(+)
+Merging imx-mxs/for-next (fa81543ef854 Merge branch 'imx/defconfig' into for-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux.git imx-mxs/for-next
+Merge made by the 'ort' strategy.
+Merging mediatek/for-next (9802b60bd6d8 Merge branch 'v6.6-next/soc' into for-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/matthias.bgg/linux.git mediatek/for-next
+Merge made by the 'ort' strategy.
+Merging mvebu/for-next (93e6b023e552 Merge branch 'mvebu/dt64' into mvebu/for-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/gclement/mvebu.git mvebu/for-next
+Merge made by the 'ort' strategy.
+Merging omap/for-next (cb1114df7bb0 Merge branch 'fixes' into for-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/tmlind/linux-omap.git omap/for-next
+Merge made by the 'ort' strategy.
+Merging qcom/for-next (2b05c2dc230b Merge branches 'arm64-defconfig-for-6.7', 'arm64-fixes-for-6.6', 'arm64-for-6.7', 'clk-for-6.7', 'drivers-for-6.7' and 'dts-for-6.7' into for-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux.git qcom/for-next
+Merge made by the 'ort' strategy.
+Merging renesas/next (19c4da034485 Merge branch 'renesas-dts-for-v6.8' into renesas-next)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git renesas/next
+Merge made by the 'ort' strategy.
+ arch/arm/boot/dts/renesas/iwg20d-q7-dbcm-ca.dtsi   | 15 +++++
+ .../dts/renesas/r8a7745-iwg22d-sodimm-dbhd-ca.dts  | 15 +++++
+ .../arm/boot/dts/renesas/r8a7745-iwg22d-sodimm.dts |  4 +-
+ arch/arm/boot/dts/renesas/r8a7779-marzen.dts       | 48 +++++++++++++---
+ arch/arm/boot/dts/renesas/r8a7790-lager.dts        | 23 ++++++--
+ arch/arm/boot/dts/renesas/r8a7790-stout.dts        | 15 +++++
+ arch/arm/boot/dts/renesas/r8a7791-koelsch.dts      | 30 +++++++++-
+ arch/arm/boot/dts/renesas/r8a7791-porter.dts       | 26 ++++++++-
+ arch/arm/boot/dts/renesas/r8a7792-blanche.dts      | 49 ++++++++++++++++
+ arch/arm/boot/dts/renesas/r8a7792-wheat.dts        | 21 +++++++
+ arch/arm/boot/dts/renesas/r8a7793-gose.dts         | 28 +++++++++-
+ arch/arm/boot/dts/renesas/r8a7794-alt.dts          |  4 +-
+ arch/arm/boot/dts/renesas/r8a7794-silk.dts         | 17 +++++-
+ arch/arm64/boot/dts/renesas/draak.dtsi             |  6 ++
+ arch/arm64/boot/dts/renesas/ebisu.dtsi             |  6 ++
+ arch/arm64/boot/dts/renesas/r8a77970-eagle.dts     | 15 +++++
+ arch/arm64/boot/dts/renesas/rzg2l-smarc-som.dtsi   |  1 +
+ arch/arm64/boot/dts/renesas/rzg2lc-smarc-som.dtsi  |  1 +
+ arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi   | 49 ++++++++++++++++
+ arch/arm64/boot/dts/renesas/rzg3s-smarc.dtsi       | 65 ++++++++++++++++++++++
+ arch/riscv/boot/dts/renesas/r9a07g043f.dtsi        |  4 ++
+ 21 files changed, 420 insertions(+), 22 deletions(-)
+Merging reset/reset/next (417a3a5ae44a reset: ti: syscon: remove unneeded call to platform_set_drvdata())
+$ git merge -m Merge branch 'reset/next' of https://git.pengutronix.de/git/pza/linux reset/reset/next
+Already up to date.
+Merging rockchip/for-next (2a883579c894 Merge branch 'v6.8-armsoc/dts64' into for-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip.git rockchip/for-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/arm/rockchip.yaml          |  11 +-
+ .../devicetree/bindings/soc/rockchip/grf.yaml      |   1 +
+ arch/arm/boot/dts/rockchip/Makefile                |   1 +
+ arch/arm/boot/dts/rockchip/rk3128-xpi-3128.dts     | 397 ++++++++++
+ arch/arm/boot/dts/rockchip/rk3128.dtsi             |  11 +
+ arch/arm64/boot/dts/rockchip/Makefile              |   1 +
+ arch/arm64/boot/dts/rockchip/rk3328.dtsi           |   6 +-
+ arch/arm64/boot/dts/rockchip/rk3399.dtsi           |   6 +-
+ .../boot/dts/rockchip/rk3566-powkiddy-rgb30.dts    | 152 +---
+ .../boot/dts/rockchip/rk3566-powkiddy-rk2023.dts   |  38 +
+ .../boot/dts/rockchip/rk3566-powkiddy-rk2023.dtsi  | 875 +++++++++++++++++++++
+ arch/arm64/boot/dts/rockchip/rk356x.dtsi           |   2 +-
+ arch/arm64/boot/dts/rockchip/rk3588-evb1-v10.dts   |  97 +++
+ drivers/clk/rockchip/clk-rk3128.c                  |   2 +-
+ drivers/clk/rockchip/clk-rk3568.c                  |   2 +
+ 15 files changed, 1458 insertions(+), 144 deletions(-)
+ create mode 100644 arch/arm/boot/dts/rockchip/rk3128-xpi-3128.dts
+ create mode 100644 arch/arm64/boot/dts/rockchip/rk3566-powkiddy-rk2023.dts
+ create mode 100644 arch/arm64/boot/dts/rockchip/rk3566-powkiddy-rk2023.dtsi
+Merging samsung-krzk/for-next (505d8d66267f Merge branch 'next/clk' into for-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git samsung-krzk/for-next
+Merge made by the 'ort' strategy.
+ .../bindings/arm/samsung/samsung-boards.yaml       |   6 +
+ .../devicetree/bindings/gpu/arm,mali-midgard.yaml  |   5 +
+ .../bindings/hwinfo/samsung,exynos-chipid.yaml     |  18 ++-
+ .../devicetree/bindings/i2c/i2c-exynos5.yaml       |  10 +-
+ .../bindings/i2c/samsung,s3c2410-i2c.yaml          |  22 +--
+ .../bindings/iio/adc/samsung,exynos-adc.yaml       |  29 ++--
+ .../bindings/mfd/samsung,exynos5433-lpass.yaml     |   2 +-
+ .../bindings/mmc/samsung,exynos-dw-mshc.yaml       |  25 ++--
+ .../devicetree/bindings/pwm/pwm-samsung.yaml       |   3 +
+ Documentation/devicetree/bindings/rtc/s3c-rtc.yaml |   5 +
+ .../devicetree/bindings/serial/samsung_uart.yaml   |  16 ++-
+ .../bindings/soc/samsung/exynos-pmu.yaml           |   7 +
+ .../bindings/soc/samsung/exynos-usi.yaml           |   6 +-
+ .../soc/samsung/samsung,exynos-sysreg.yaml         |   5 +
+ .../devicetree/bindings/sound/samsung-i2s.yaml     |  19 ++-
+ arch/arm/boot/dts/samsung/exynos4.dtsi             |  26 ++--
+ arch/arm/boot/dts/samsung/exynos4x12.dtsi          |  22 ++-
+ arch/arm/boot/dts/samsung/s5pv210.dtsi             |  18 +--
+ arch/arm64/boot/dts/exynos/exynos5433.dtsi         |  60 +++++---
+ arch/arm64/boot/dts/exynos/exynos7.dtsi            |  18 +--
+ arch/arm64/boot/dts/exynos/exynos7885.dtsi         |  45 ++++--
+ arch/arm64/boot/dts/exynos/exynos850.dtsi          |  34 +++--
+ arch/arm64/boot/dts/exynos/exynosautov9-sadk.dts   |  51 +++++++
+ arch/arm64/boot/dts/exynos/exynosautov9.dtsi       |   6 +-
+ drivers/clk/samsung/clk-cpu.h                      |  30 ++--
+ drivers/clk/samsung/clk.h                          | 157 +++++++++++----------
+ drivers/soc/samsung/exynos-chipid.c                |   1 +
+ 27 files changed, 420 insertions(+), 226 deletions(-)
+Merging scmi/for-linux-next (b419bd935f13 Merge branch 'for-next/scmi/updates' of git://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux into for-linux-next)
+$ git merge -m Merge branch 'for-linux-next' of git://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux.git scmi/for-linux-next
+Merge made by the 'ort' strategy.
+ drivers/firmware/arm_ffa/driver.c     | 70 +++++++++++++++++++++++------------
+ drivers/firmware/arm_scmi/clock.c     |  2 +-
+ drivers/firmware/arm_scmi/driver.c    | 12 ++++--
+ drivers/firmware/arm_scmi/perf.c      | 35 ++++++++++++------
+ drivers/firmware/arm_scmi/power.c     |  2 +-
+ drivers/firmware/arm_scmi/powercap.c  |  2 +-
+ drivers/firmware/arm_scmi/protocols.h |  3 +-
+ drivers/firmware/arm_scmi/reset.c     |  3 +-
+ drivers/firmware/arm_scmi/sensors.c   |  2 +-
+ drivers/firmware/arm_scmi/voltage.c   |  2 +-
+ include/linux/arm_ffa.h               |  2 +
+ 11 files changed, 89 insertions(+), 46 deletions(-)
+Merging stm32/stm32-next (9a7308232a11 ARM: dts: stm32: add SPI support on STM32F746)
+$ git merge -m Merge branch 'stm32-next' of git://git.kernel.org/pub/scm/linux/kernel/git/atorgue/stm32.git stm32/stm32-next
+Merge made by the 'ort' strategy.
+ arch/arm/boot/dts/st/stm32f469-disco.dts | 15 +++-----
+ arch/arm/boot/dts/st/stm32f746.dtsi      | 61 ++++++++++++++++++++++++++++++++
+ 2 files changed, 65 insertions(+), 11 deletions(-)
+Merging sunxi/sunxi/for-next (715a9e70bdd4 Merge branch 'sunxi/clk-for-6.8' into sunxi/for-next)
+$ git merge -m Merge branch 'sunxi/for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux.git sunxi/sunxi/for-next
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/arm/sunxi.yaml   |   5 +
+ .../dts/allwinner/sun50i-h616-orangepi-zero.dtsi   |   3 -
+ .../dts/allwinner/sun50i-h616-orangepi-zero2.dts   |   3 +
+ .../dts/allwinner/sun50i-h618-orangepi-zero2w.dts  | 176 +++++++++++++++++++++
+ .../dts/allwinner/sun50i-h618-orangepi-zero3.dts   |   2 +
+ drivers/clk/sunxi-ng/ccu_nkm.c                     |   5 +-
+ 6 files changed, 188 insertions(+), 6 deletions(-)
+ create mode 100644 arch/arm64/boot/dts/allwinner/sun50i-h618-orangepi-zero2w.dts
+Merging tee/next (352b5f33b950 Merge branch 'ffa_notif_for_v6.8' into next)
+$ git merge -m Merge branch 'next' of https://git.linaro.org/people/jens.wiklander/linux-tee.git tee/next
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-bus-optee-devices |   9 ++
+ drivers/firmware/arm_scmi/optee.c                 |   4 +
+ drivers/tee/optee/call.c                          | 163 ++++++++++++++++++++--
+ drivers/tee/optee/core.c                          |  18 +--
+ drivers/tee/optee/device.c                        |  17 ++-
+ drivers/tee/optee/ffa_abi.c                       | 107 ++++++++++++--
+ drivers/tee/optee/optee_ffa.h                     |  28 +++-
+ drivers/tee/optee/optee_private.h                 |  40 +++++-
+ drivers/tee/optee/smc_abi.c                       |  67 ++++-----
+ drivers/tee/tee_core.c                            |   8 ++
+ include/linux/tee_drv.h                           |  16 +++
+ 11 files changed, 393 insertions(+), 84 deletions(-)
+Merging tegra/for-next (650220c2b474 Merge branch for-6.7/arm64/dt into for-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux.git tegra/for-next
+Merge made by the 'ort' strategy.
+Merging ti/ti-next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'ti-next' of git://git.kernel.org/pub/scm/linux/kernel/git/ti/linux.git ti/ti-next
+Already up to date.
+Merging xilinx/for-next (1d4723dec415 Merge branch 'zynqmp/soc' into for-next)
+$ git merge -m Merge branch 'for-next' of git://github.com/Xilinx/linux-xlnx.git xilinx/for-next
+Merge made by the 'ort' strategy.
+ arch/arm64/boot/dts/xilinx/Makefile                |  9 ++++----
+ arch/arm64/boot/dts/xilinx/zynqmp-sm-k26-revA.dts  |  8 +++----
+ .../boot/dts/xilinx/zynqmp-zc1751-xm015-dc1.dts    |  8 +++++--
+ .../boot/dts/xilinx/zynqmp-zc1751-xm016-dc2.dts    | 16 ++++++++-----
+ .../boot/dts/xilinx/zynqmp-zc1751-xm017-dc3.dts    |  8 +++++--
+ .../boot/dts/xilinx/zynqmp-zc1751-xm018-dc4.dts    | 26 +++++++++++++---------
+ .../boot/dts/xilinx/zynqmp-zc1751-xm019-dc5.dts    |  8 +++++--
+ arch/arm64/boot/dts/xilinx/zynqmp-zcu100-revC.dts  |  4 ++--
+ arch/arm64/boot/dts/xilinx/zynqmp.dtsi             | 14 +++---------
+ drivers/soc/xilinx/xlnx_event_manager.c            |  7 ++++--
+ drivers/soc/xilinx/zynqmp_power.c                  | 14 +++++-------
+ 11 files changed, 66 insertions(+), 56 deletions(-)
+Merging clk/clk-next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'clk-next' of git://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git clk/clk-next
+Already up to date.
+Merging clk-imx/for-next (2838820800dc clk: imx: imx8qm/qxp: add more resources to whitelist)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/abelvesa/linux.git clk-imx/for-next
+Already up to date.
+Merging clk-renesas/renesas-clk (5ab16198b431 clk: renesas: r8a779g0: Add PCIe clocks)
+$ git merge -m Merge branch 'renesas-clk' of git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git clk-renesas/renesas-clk
+Merge made by the 'ort' strategy.
+ drivers/clk/renesas/r8a779g0-cpg-mssr.c | 3 +++
+ 1 file changed, 3 insertions(+)
+Merging csky/linux-next (2c40c1c6adab Merge tag 'usb-6.7-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb)
+$ git merge -m Merge branch 'linux-next' of git://github.com/c-sky/csky-linux.git csky/linux-next
+Already up to date.
+Merging loongarch/loongarch-next (c517fd2738f4 Docs/zh_CN/LoongArch: Update links in LoongArch introduction.rst)
+$ git merge -m Merge branch 'loongarch-next' of git://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson.git loongarch/loongarch-next
+Merge made by the 'ort' strategy.
+ Documentation/arch/loongarch/introduction.rst      |  4 ++--
+ .../zh_CN/arch/loongarch/introduction.rst          |  4 ++--
+ arch/loongarch/Makefile                            |  3 +++
+ arch/loongarch/include/asm/asmmacro.h              |  3 +--
+ arch/loongarch/include/asm/percpu.h                | 11 ++++-----
+ arch/loongarch/include/asm/setup.h                 |  2 +-
+ arch/loongarch/kernel/relocate.c                   | 10 +++++++-
+ arch/loongarch/kernel/time.c                       | 27 +++++++++-------------
+ arch/loongarch/mm/pgtable.c                        |  4 ++--
+ 9 files changed, 36 insertions(+), 32 deletions(-)
+Merging m68k/for-next (03191fb3db3d m68k: lib: Include <linux/libgcc.h> for __muldi3())
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git m68k/for-next
+Already up to date.
+Merging m68knommu/for-next (98b1cc82c4af Linux 6.7-rc2)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/gerg/m68knommu.git m68knommu/for-next
+Already up to date.
+Merging microblaze/next (ffb0399437ef microblaze: defconfig: Enable the Marvell phy driver)
+$ git merge -m Merge branch 'next' of git://git.monstr.eu/linux-2.6-microblaze.git microblaze/next
+Merge made by the 'ort' strategy.
+ arch/microblaze/configs/mmu_defconfig | 13 +++++++------
+ 1 file changed, 7 insertions(+), 6 deletions(-)
+Merging mips/mips-next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'mips-next' of git://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git mips/mips-next
+Already up to date.
+Merging openrisc/for-next (c289330331eb openrisc: Remove kernel-doc marker from ioremap comment)
+$ git merge -m Merge branch 'for-next' of git://github.com/openrisc/linux.git openrisc/for-next
+Already up to date.
+Merging parisc-hd/for-next (2546732f1354 parisc: Use correct alignment in __bug_table)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/deller/parisc-linux.git parisc-hd/for-next
+Merge made by the 'ort' strategy.
+ arch/parisc/include/asm/alternative.h |  9 ++++++---
+ arch/parisc/include/asm/assembly.h    |  1 +
+ arch/parisc/include/asm/bug.h         | 12 +++++++++---
+ arch/parisc/include/asm/jump_label.h  |  2 ++
+ arch/parisc/include/asm/uaccess.h     |  1 +
+ arch/parisc/kernel/vmlinux.lds.S      |  4 ++--
+ 6 files changed, 21 insertions(+), 8 deletions(-)
+Merging powerpc/next (707df298cbde Merge tag 'powerpc-6.7-1' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git powerpc/next
+Already up to date.
+Merging soc-fsl/next (fb9c384625dd bus: fsl-mc: fsl-mc-allocator: Drop a write-only variable)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/leo/linux.git soc-fsl/next
+Already up to date.
+Merging risc-v/for-next (3ca112b71f35 Merge tag 'probes-fixes-v6.7-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git risc-v/for-next
+Already up to date.
+Merging riscv-dt/riscv-dt-for-next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'riscv-dt-for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-dt/riscv-dt-for-next
+Already up to date.
+Merging riscv-soc/riscv-soc-for-next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'riscv-soc-for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-soc/riscv-soc-for-next
+Already up to date.
+Merging s390/for-next (02e790ee3077 s390/mm: make pte_free_tlb() similar to pXd_free_tlb())
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git s390/for-next
+Already up to date.
+Merging sh/for-next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'for-next' of git:git.kernel.org/pub/scm/linux/kernel/git/glaubitz/sh-linux.git sh/for-next
+Already up to date.
+Merging uml/next (974b808d85ab um: virt-pci: fix missing declaration warning)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git uml/next
+Already up to date.
+Merging xtensa/xtensa-for-next (a83a72730c33 xtensa: import ESP32S3 core variant)
+$ git merge -m Merge branch 'xtensa-for-next' of git://github.com/jcmvbkbc/linux-xtensa.git xtensa/xtensa-for-next
+Merge made by the 'ort' strategy.
+ arch/xtensa/variants/esp32/include/variant/core.h  | 655 ++++++++++++++++++++
+ .../variants/esp32/include/variant/tie-asm.h       | 323 ++++++++++
+ arch/xtensa/variants/esp32/include/variant/tie.h   | 182 ++++++
+ .../xtensa/variants/esp32s3/include/variant/core.h | 681 +++++++++++++++++++++
+ .../variants/esp32s3/include/variant/tie-asm.h     | 429 +++++++++++++
+ arch/xtensa/variants/esp32s3/include/variant/tie.h | 207 +++++++
+ 6 files changed, 2477 insertions(+)
+ create mode 100644 arch/xtensa/variants/esp32/include/variant/core.h
+ create mode 100644 arch/xtensa/variants/esp32/include/variant/tie-asm.h
+ create mode 100644 arch/xtensa/variants/esp32/include/variant/tie.h
+ create mode 100644 arch/xtensa/variants/esp32s3/include/variant/core.h
+ create mode 100644 arch/xtensa/variants/esp32s3/include/variant/tie-asm.h
+ create mode 100644 arch/xtensa/variants/esp32s3/include/variant/tie.h
+Merging bcachefs/for-next (e3759384e88e bcachefs: six locks: Simplify optimistic spinning)
+$ git merge -m Merge branch 'for-next' of https://evilpiepirate.org/git/bcachefs.git bcachefs/for-next
+Auto-merging fs/bcachefs/fs.c
+Merge made by the 'ort' strategy.
+ arch/powerpc/kernel/firmware.c         |   2 +
+ drivers/md/bcache/btree.c              |  14 +-
+ drivers/md/bcache/journal.c            |  20 +-
+ drivers/md/bcache/movinggc.c           |  16 +-
+ drivers/md/bcache/request.c            |  74 ++--
+ drivers/md/bcache/request.h            |   2 +-
+ drivers/md/bcache/super.c              |  40 +-
+ drivers/md/bcache/writeback.c          |  16 +-
+ fs/bcachefs/Kconfig                    |  10 +
+ fs/bcachefs/Makefile                   |   1 +
+ fs/bcachefs/alloc_background.c         | 105 ++++--
+ fs/bcachefs/alloc_foreground.c         |  27 ++
+ fs/bcachefs/backpointers.c             | 116 +++---
+ fs/bcachefs/bcachefs.h                 |  20 +-
+ fs/bcachefs/bcachefs_format.h          |  16 +-
+ fs/bcachefs/bcachefs_ioctl.h           |  18 +-
+ fs/bcachefs/bkey_methods.h             |   2 -
+ fs/bcachefs/btree_gc.c                 |  12 +-
+ fs/bcachefs/btree_io.c                 |  13 +-
+ fs/bcachefs/btree_iter.c               | 151 +-------
+ fs/bcachefs/btree_iter.h               |  28 +-
+ fs/bcachefs/btree_journal_iter.c       |  37 +-
+ fs/bcachefs/btree_journal_iter.h       |  10 +-
+ fs/bcachefs/btree_key_cache.c          |  23 +-
+ fs/bcachefs/btree_locking.c            |   4 +-
+ fs/bcachefs/btree_trans_commit.c       | 118 ++----
+ fs/bcachefs/btree_types.h              |  36 +-
+ fs/bcachefs/btree_update.c             |  27 +-
+ fs/bcachefs/btree_update.h             |  47 +--
+ fs/bcachefs/btree_update_interior.c    |  53 +--
+ fs/bcachefs/btree_write_buffer.c       | 666 +++++++++++++++++++++++----------
+ fs/bcachefs/btree_write_buffer.h       |  53 ++-
+ fs/bcachefs/btree_write_buffer_types.h |  63 ++--
+ fs/bcachefs/buckets.c                  |  48 +--
+ fs/bcachefs/buckets.h                  |   3 +
+ fs/bcachefs/chardev.c                  |   2 +-
+ fs/bcachefs/clock.c                    |   3 +-
+ fs/bcachefs/darray.c                   |  23 ++
+ fs/bcachefs/darray.h                   |  34 +-
+ fs/bcachefs/data_update.c              |   4 +-
+ fs/bcachefs/dirent.c                   |   5 +-
+ fs/bcachefs/dirent.h                   |   3 +-
+ fs/bcachefs/ec.c                       |  12 +-
+ fs/bcachefs/ec_types.h                 |   2 +-
+ fs/bcachefs/errcode.h                  |   2 -
+ fs/bcachefs/fs-io-direct.c             |   8 +-
+ fs/bcachefs/fs-io.c                    |   6 +-
+ fs/bcachefs/fs.c                       |   9 +-
+ fs/bcachefs/fs.h                       |   9 +-
+ fs/bcachefs/fsck.c                     | 113 +++---
+ fs/bcachefs/inode.c                    |  20 +-
+ fs/bcachefs/io_misc.c                  |  12 +-
+ fs/bcachefs/io_read.c                  |   2 +-
+ fs/bcachefs/io_write.c                 |  20 +-
+ fs/bcachefs/io_write.h                 |   3 +-
+ fs/bcachefs/journal.c                  | 113 ++++--
+ fs/bcachefs/journal.h                  |   4 +-
+ fs/bcachefs/journal_io.c               |  85 ++++-
+ fs/bcachefs/journal_io.h               |   2 +-
+ fs/bcachefs/journal_reclaim.c          |  94 +++--
+ fs/bcachefs/journal_reclaim.h          |  16 +-
+ fs/bcachefs/journal_types.h            |  15 +-
+ fs/bcachefs/logged_ops.c               |   4 +-
+ fs/bcachefs/lru.c                      |   2 +-
+ fs/bcachefs/migrate.c                  |   2 +-
+ fs/bcachefs/move.c                     | 154 +++++---
+ fs/bcachefs/move.h                     |   4 +-
+ fs/bcachefs/movinggc.c                 |   7 +-
+ fs/bcachefs/opts.h                     |   5 -
+ fs/bcachefs/rebalance.c                |   6 +-
+ fs/bcachefs/recovery.c                 | 133 ++++---
+ fs/bcachefs/reflink.c                  |   2 +-
+ fs/bcachefs/replicas.c                 |  58 +--
+ fs/bcachefs/replicas.h                 |  20 +-
+ fs/bcachefs/replicas_types.h           |   4 +-
+ fs/bcachefs/sb-clean.c                 |   2 +-
+ fs/bcachefs/sb-errors.h                |   3 +-
+ fs/bcachefs/six.c                      | 119 ++----
+ fs/bcachefs/six.h                      |   7 -
+ fs/bcachefs/snapshot.c                 |  12 +-
+ fs/bcachefs/str_hash.h                 |  25 +-
+ fs/bcachefs/subvolume.c                |   6 +-
+ fs/bcachefs/super-io.c                 |  32 +-
+ fs/bcachefs/super-io.h                 |   1 +
+ fs/bcachefs/super.c                    |  12 +-
+ fs/bcachefs/super_types.h              |   1 +
+ fs/bcachefs/trace.h                    |  75 +++-
+ fs/bcachefs/util.c                     | 145 +++----
+ fs/bcachefs/util.h                     |  33 +-
+ include/linux/closure.h                |   9 +-
+ kernel/kthread.c                       |   3 +-
+ lib/closure.c                          |   5 +-
+ 92 files changed, 2027 insertions(+), 1376 deletions(-)
+ create mode 100644 fs/bcachefs/darray.c
+Merging pidfd/for-next (a901a3568fd2 Merge tag 'iomap-6.5-merge-1' of git://git.kernel.org/pub/scm/fs/xfs/xfs-linux)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/brauner/linux.git pidfd/for-next
+Already up to date.
+Merging fscrypt/for-next (15baf55481de fscrypt: track master key presence separately from secret)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/fs/fscrypt/linux.git fscrypt/for-next
+Already up to date.
+Merging afs/afs-next (0a278bc196e7 afs: Automatically generate trace tag enums)
+$ git merge -m Merge branch 'afs-next' of git://git.kernel.org/pub/scm/linux/kernel/git/dhowells/linux-fs.git afs/afs-next
+Merge made by the 'ort' strategy.
+ include/trace/events/afs.h | 475 ++++++++++++++-------------------------------
+ 1 file changed, 148 insertions(+), 327 deletions(-)
+Merging btrfs/for-next (3fdca4a7353d Merge branch 'for-next-next-v6.7-20231120' into for-next-20231120)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git btrfs/for-next
+Merge made by the 'ort' strategy.
+ fs/btrfs/ctree.c         | 48 ++++++++++++++++++++++++++++--------------------
+ fs/btrfs/ctree.h         |  9 +++------
+ fs/btrfs/delayed-inode.c | 45 ++++++++++++++++++++++++---------------------
+ fs/btrfs/disk-io.c       |  5 +----
+ fs/btrfs/disk-io.h       |  2 --
+ fs/btrfs/super.c         |  5 ++++-
+ 6 files changed, 60 insertions(+), 54 deletions(-)
+Merging ceph/master (d1a66ad3bd27 ceph: fix deadlock or deadcode of misusing dget())
+$ git merge -m Merge branch 'master' of git://github.com/ceph/ceph-client.git ceph/master
+Merge made by the 'ort' strategy.
+ fs/ceph/caps.c                  |  9 +++------
+ include/linux/ceph/osd_client.h |  3 ++-
+ net/ceph/osd_client.c           | 32 ++++++++++++++++++--------------
+ 3 files changed, 23 insertions(+), 21 deletions(-)
+Merging cifs/for-next (98b1cc82c4af Linux 6.7-rc2)
+$ git merge -m Merge branch 'for-next' of git://git.samba.org/sfrench/cifs-2.6.git cifs/for-next
+Already up to date.
+Merging configfs/for-next (4425c1d9b44d configfs: improve item creation performance)
+$ git merge -m Merge branch 'for-next' of git://git.infradead.org/users/hch/configfs.git configfs/for-next
+Auto-merging fs/configfs/inode.c
+Merge made by the 'ort' strategy.
+ fs/configfs/configfs_internal.h |  4 ++--
+ fs/configfs/dir.c               | 42 +++++++++++++++++++++++++++++++----------
+ fs/configfs/inode.c             | 24 -----------------------
+ 3 files changed, 34 insertions(+), 36 deletions(-)
+Merging ecryptfs/next (a3d78fe3e1ae fs: ecryptfs: comment typo fix)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/tyhicks/ecryptfs.git ecryptfs/next
+Auto-merging fs/ecryptfs/crypto.c
+Auto-merging fs/ecryptfs/read_write.c
+Merge made by the 'ort' strategy.
+ fs/ecryptfs/crypto.c   | 2 +-
+ fs/ecryptfs/keystore.c | 2 +-
+ 2 files changed, 2 insertions(+), 2 deletions(-)
+Merging erofs/dev (1a0ac8bd7a4f erofs: fix erofs_insert_workgroup() lockref usage)
+$ git merge -m Merge branch 'dev' of git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git erofs/dev
+Already up to date.
+Merging exfat/dev (1373ca10ec04 exfat: fix ctime is not updated)
+$ git merge -m Merge branch 'dev' of git://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/exfat.git exfat/dev
+Already up to date.
+Merging ext3/for_next (7f680e5f256f Pull ext2 conversion of directory code to folios.)
+$ git merge -m Merge branch 'for_next' of git://git.kernel.org/pub/scm/linux/kernel/git/jack/linux-fs.git ext3/for_next
+Merge made by the 'ort' strategy.
+Merging ext4/dev (91562895f803 ext4: properly sync file size update after O_SYNC direct IO)
+$ git merge -m Merge branch 'dev' of git://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git ext4/dev
+Already up to date.
+Merging f2fs/dev (6bc40e44f1dd Merge tag 'ovl-fixes-6.7-rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs)
+$ git merge -m Merge branch 'dev' of git://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git f2fs/dev
+Already up to date.
+Merging fsverity/for-next (919dc320956e fsverity: skip PKCS#7 parser when keyring is empty)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/fs/fsverity/linux.git fsverity/for-next
+Already up to date.
+Merging fuse/for-next (18437366a3a1 fuse: share lookup state between submount and its parent)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git fuse/for-next
+Auto-merging fs/fuse/fuse_i.h
+Auto-merging fs/fuse/inode.c
+Merge made by the 'ort' strategy.
+ Documentation/filesystems/fuse-io.rst |  3 +-
+ fs/fuse/file.c                        |  6 +--
+ fs/fuse/fuse_i.h                      | 19 +++++++-
+ fs/fuse/inode.c                       | 81 ++++++++++++++++++++++++++++++++---
+ include/uapi/linux/fuse.h             | 10 +++--
+ 5 files changed, 103 insertions(+), 16 deletions(-)
+Merging gfs2/for-next (8d859dc44163 gfs2: Fix inode_go_instantiate description)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/gfs2/linux-gfs2.git gfs2/for-next
+Merge made by the 'ort' strategy.
+ fs/gfs2/dentry.c |  6 ++----
+ fs/gfs2/glock.c  | 39 ++++++++++++++++++++++++++++++++++++++-
+ fs/gfs2/glock.h  |  1 +
+ fs/gfs2/glops.c  |  2 +-
+ fs/gfs2/inode.c  |  8 ++++----
+ 5 files changed, 46 insertions(+), 10 deletions(-)
+Merging jfs/jfs-next (cca974daeb6c jfs: fix shift-out-of-bounds in dbJoin)
+$ git merge -m Merge branch 'jfs-next' of git://github.com/kleikamp/linux-shaggy.git jfs/jfs-next
+Merge made by the 'ort' strategy.
+ fs/jfs/jfs_dmap.c  | 65 +++++++++++++++++++++++++++++++-----------------------
+ fs/jfs/jfs_dtree.c |  7 +++++-
+ fs/jfs/jfs_mount.c |  6 ++---
+ 3 files changed, 47 insertions(+), 31 deletions(-)
+Merging ksmbd/ksmbd-for-next (98b1cc82c4af Linux 6.7-rc2)
+$ git merge -m Merge branch 'ksmbd-for-next' of https://github.com/smfrench/smb3-kernel.git ksmbd/ksmbd-for-next
+Already up to date.
+Merging nfs/linux-next (f003a717ae90 nfs: Convert nfs_symlink() to use a folio)
+$ git merge -m Merge branch 'linux-next' of git://git.linux-nfs.org/projects/trondmy/nfs-2.6.git nfs/linux-next
+Already up to date.
+Merging nfs-anna/linux-next (379e4adfddd6 NFSv4.1: fixup use EXCHGID4_FLAG_USE_PNFS_DS for DS server)
+$ git merge -m Merge branch 'linux-next' of git://git.linux-nfs.org/projects/anna/linux-nfs.git nfs-anna/linux-next
+Already up to date.
+Merging nfsd/nfsd-next (21bc7f605adb NFSD: Document lack of f_pos_lock in nfsd_readdir())
+$ git merge -m Merge branch 'nfsd-next' of git://git.kernel.org/pub/scm/linux/kernel/git/cel/linux nfsd/nfsd-next
+Merge made by the 'ort' strategy.
+ fs/nfsd/Kconfig       | 16 +++++++++
+ fs/nfsd/filecache.c   |  2 +-
+ fs/nfsd/nfs4recover.c | 97 +++++++++++++++++++++++++++++++++------------------
+ fs/nfsd/nfscache.c    |  6 +---
+ fs/nfsd/nfsctl.c      |  6 ++++
+ fs/nfsd/nfssvc.c      |  7 ++--
+ fs/nfsd/trace.h       | 22 ------------
+ fs/nfsd/vfs.c         | 20 +++++++++--
+ 8 files changed, 107 insertions(+), 69 deletions(-)
+Merging ntfs3/master (ffc253263a13 Linux 6.6)
+$ git merge -m Merge branch 'master' of https://github.com/Paragon-Software-Group/linux-ntfs3.git ntfs3/master
+Already up to date.
+Merging orangefs/for-next (31720a2b109b orangefs: Fix kmemleak in orangefs_{kernel,client}_debug_init())
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/hubcap/linux orangefs/for-next
+Already up to date.
+Merging overlayfs/overlayfs-next (484d4fbfdafe ovl: stop using d_alloc_anon()/d_instantiate_anon())
+$ git merge -m Merge branch 'overlayfs-next' of git://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git overlayfs/overlayfs-next
+Merge made by the 'ort' strategy.
+ fs/overlayfs/export.c | 23 +----------------------
+ 1 file changed, 1 insertion(+), 22 deletions(-)
+Merging ubifs/next (75690493591f ubifs: ubifs_link: Fix wrong name len calculating when UBIFS is encrypted)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/rw/ubifs.git ubifs/next
+Already up to date.
+Merging v9fs/9p-next (ce0708796420 9p/net: fix possible memory leak in p9_check_errors())
+$ git merge -m Merge branch '9p-next' of git://github.com/martinetd/linux v9fs/9p-next
+Already up to date.
+Merging v9fs-ericvh/ericvh/for-next (2dde18cd1d8f Linux 6.5)
+$ git merge -m Merge branch 'ericvh/for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git v9fs-ericvh/ericvh/for-next
+Already up to date.
+Merging xfs/for-next (7930d9e10370 xfs: recovery should not clear di_flushiter unconditionally)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/fs/xfs/xfs-linux.git xfs/for-next
+Already up to date.
+Merging zonefs/for-next (8812387d0569 zonefs: set FMODE_CAN_ODIRECT instead of a dummy direct_IO method)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/dlemoal/zonefs.git zonefs/for-next
+Already up to date.
+Merging iomap/iomap-for-next (3ac974796e5d iomap: fix short copy in iomap_write_iter())
+$ git merge -m Merge branch 'iomap-for-next' of git://git.kernel.org/pub/scm/fs/xfs/xfs-linux.git iomap/iomap-for-next
+Already up to date.
+Merging djw-vfs/vfs-for-next (ce85a1e04645 xfs: stabilize fs summary counters for online fsck)
+$ git merge -m Merge branch 'vfs-for-next' of git://git.kernel.org/pub/scm/fs/xfs/xfs-linux.git djw-vfs/vfs-for-next
+Already up to date.
+Merging file-locks/locks-next (e0152e7481c6 Merge tag 'riscv-for-linus-6.6-mw1' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux)
+$ git merge -m Merge branch 'locks-next' of git://git.kernel.org/pub/scm/linux/kernel/git/jlayton/linux.git file-locks/locks-next
+Already up to date.
+Merging iversion/iversion-next (e0152e7481c6 Merge tag 'riscv-for-linus-6.6-mw1' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux)
+$ git merge -m Merge branch 'iversion-next' of git://git.kernel.org/pub/scm/linux/kernel/git/jlayton/linux.git iversion/iversion-next
+Already up to date.
+Merging vfs-brauner/vfs.all (e0bd765766c9 Merge branch 'vfs.mount' into vfs.all)
+$ git merge -m Merge branch 'vfs.all' of git://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git vfs-brauner/vfs.all
+Auto-merging fs/bcachefs/super-io.c
+Auto-merging fs/bcachefs/super_types.h
+Auto-merging fs/btrfs/super.c
+Auto-merging fs/buffer.c
+CONFLICT (content): Merge conflict in fs/buffer.c
+Auto-merging fs/ext4/inode.c
+Auto-merging fs/gfs2/glock.c
+Auto-merging fs/hugetlbfs/inode.c
+Auto-merging fs/nilfs2/inode.c
+Auto-merging fs/ntfs/aops.c
+Auto-merging include/linux/fs.h
+Auto-merging mm/hugetlb.c
+Auto-merging mm/page-writeback.c
+Recorded preimage for 'fs/buffer.c'
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+Recorded resolution for 'fs/buffer.c'.
+[master 9ac8bdfbfc5d] Merge branch 'vfs.all' of git://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/filesystems/porting.rst       |  12 +
+ arch/alpha/kernel/syscalls/syscall.tbl      |   2 +
+ arch/arm/tools/syscall.tbl                  |   2 +
+ arch/arm64/include/asm/unistd32.h           |   4 +
+ arch/m68k/kernel/syscalls/syscall.tbl       |   2 +
+ arch/microblaze/kernel/syscalls/syscall.tbl |   2 +
+ arch/mips/kernel/syscalls/syscall_n32.tbl   |   2 +
+ arch/mips/kernel/syscalls/syscall_n64.tbl   |   2 +
+ arch/mips/kernel/syscalls/syscall_o32.tbl   |   2 +
+ arch/parisc/kernel/syscalls/syscall.tbl     |   2 +
+ arch/powerpc/kernel/syscalls/syscall.tbl    |   2 +
+ arch/s390/kernel/syscalls/syscall.tbl       |   2 +
+ arch/sh/kernel/syscalls/syscall.tbl         |   2 +
+ arch/sparc/kernel/syscalls/syscall.tbl      |   2 +
+ arch/x86/entry/syscalls/syscall_32.tbl      |   2 +
+ arch/x86/entry/syscalls/syscall_64.tbl      |   2 +
+ arch/xtensa/kernel/syscalls/syscall.tbl     |   2 +
+ block/Kconfig                               |  20 +
+ block/bdev.c                                | 258 +++++++-----
+ drivers/md/dm.c                             |   4 +-
+ fs/aio.c                                    |  16 +-
+ fs/autofs/inode.c                           |  56 +--
+ fs/bcachefs/fs-ioctl.c                      |   4 +-
+ fs/bcachefs/super-io.c                      |  19 +-
+ fs/bcachefs/super_types.h                   |   1 +
+ fs/btrfs/extent_io.c                        |  52 +--
+ fs/btrfs/subpage.c                          |   4 +-
+ fs/btrfs/super.c                            |   2 +
+ fs/buffer.c                                 | 108 ++---
+ fs/dax.c                                    |   2 +-
+ fs/direct-io.c                              |   2 +-
+ fs/ecryptfs/inode.c                         |  12 +-
+ fs/ext4/inode.c                             |   4 +-
+ fs/ext4/ioctl.c                             |   4 +-
+ fs/ext4/super.c                             |   8 +-
+ fs/f2fs/file.c                              |   4 +-
+ fs/gfs2/glock.c                             |   2 +-
+ fs/gfs2/ops_fstype.c                        |   2 +-
+ fs/hugetlbfs/inode.c                        |   4 +-
+ fs/inode.c                                  |  10 +-
+ fs/internal.h                               |   2 +
+ fs/libfs.c                                  |  14 +-
+ fs/mount.h                                  |  27 +-
+ fs/namei.c                                  |   4 +-
+ fs/namespace.c                              | 631 +++++++++++++++++++++++-----
+ fs/nfs/write.c                              |  12 +-
+ fs/nilfs2/inode.c                           |   4 +-
+ fs/nilfs2/super.c                           |   8 -
+ fs/ntfs/aops.c                              |  10 +-
+ fs/open.c                                   |   3 +-
+ fs/overlayfs/inode.c                        |  10 +-
+ fs/overlayfs/overlayfs.h                    |   8 +
+ fs/pnode.c                                  |   2 +-
+ fs/proc_namespace.c                         |  13 +-
+ fs/stat.c                                   |  15 +-
+ fs/super.c                                  | 450 ++++++++++++--------
+ fs/xfs/xfs_fsops.c                          |   4 +-
+ fs/xfs/xfs_inode.h                          |   8 +
+ fs/xfs/xfs_ioctl.c                          |  34 +-
+ fs/xfs/xfs_iops.c                           |   7 +
+ fs/xfs/xfs_super.c                          |  24 +-
+ include/linux/blk_types.h                   |   8 +-
+ include/linux/blkdev.h                      |  29 +-
+ include/linux/fs.h                          |  31 +-
+ include/linux/mount.h                       |   5 +-
+ include/linux/pagemap.h                     |  17 +
+ include/linux/syscalls.h                    |   8 +
+ include/linux/writeback.h                   |   1 -
+ include/uapi/asm-generic/unistd.h           |   8 +-
+ include/uapi/linux/fcntl.h                  |   3 +
+ include/uapi/linux/mount.h                  |  62 +++
+ include/uapi/linux/stat.h                   |   1 +
+ kernel/user_namespace.c                     |  16 +-
+ lib/iov_iter.c                              |   2 +-
+ mm/hugetlb.c                                |   2 +-
+ mm/migrate.c                                |   6 +-
+ mm/page-writeback.c                         |   2 +-
+ 77 files changed, 1451 insertions(+), 684 deletions(-)
+Merging vfs/for-next (bbe6a7c899e7 bch2_ioctl_subvolume_destroy(): fix locking)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git vfs/for-next
+Auto-merging fs/bcachefs/fs-ioctl.c
+Auto-merging fs/namei.c
+Merge made by the 'ort' strategy.
+ fs/bcachefs/fs-ioctl.c | 31 +++++++++++++++++--------------
+ fs/namei.c             | 16 +++++++++++++---
+ include/linux/namei.h  |  1 +
+ 3 files changed, 31 insertions(+), 17 deletions(-)
+Merging printk/for-next (b4908d68609b Merge branch 'for-6.7' into for-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/printk/linux.git printk/for-next
+Merge made by the 'ort' strategy.
+Merging pci/next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git pci/next
+Already up to date.
+Merging pstore/for-next/pstore (1d49dee6b691 pstore/ram: Fix crash when setting number of cpus to an odd number)
+$ git merge -m Merge branch 'for-next/pstore' of git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git pstore/for-next/pstore
+Merge made by the 'ort' strategy.
+ fs/pstore/ram.c | 1 +
+ 1 file changed, 1 insertion(+)
+Merging hid/for-next (dbef8373cf6b Merge branch 'for-6.7/upstream-fixes' into for-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/hid/hid.git hid/for-next
+Merge made by the 'ort' strategy.
+ drivers/hid/Kconfig           |   9 +
+ drivers/hid/Makefile          |   1 +
+ drivers/hid/hid-apple.c       |   2 +
+ drivers/hid/hid-core.c        |  12 +-
+ drivers/hid/hid-debug.c       |   3 +
+ drivers/hid/hid-glorious.c    |  16 +-
+ drivers/hid/hid-ids.h         |  13 +-
+ drivers/hid/hid-logitech-dj.c |  11 +-
+ drivers/hid/hid-mcp2200.c     | 392 ++++++++++++++++++++++++++++++++++++++++++
+ drivers/hid/hid-mcp2221.c     |  76 +++++---
+ drivers/hid/hid-quirks.c      |   1 +
+ include/linux/hid.h           |   3 +
+ 12 files changed, 499 insertions(+), 40 deletions(-)
+ create mode 100644 drivers/hid/hid-mcp2200.c
+Merging i2c/i2c/for-next (382561d16854 i2c: ocores: Move system PM hooks to the NOIRQ phase)
+$ git merge -m Merge branch 'i2c/for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/wsa/linux.git i2c/i2c/for-next
+Already up to date.
+Merging i3c/i3c/next (b4da37db3e2c i3c: master: Fix build error)
+$ git merge -m Merge branch 'i3c/next' of git://git.kernel.org/pub/scm/linux/kernel/git/i3c/linux.git i3c/i3c/next
+Merge made by the 'ort' strategy.
+ drivers/i3c/master.c                     | 12 ++++++--
+ drivers/i3c/master/mipi-i3c-hci/cmd_v1.c |  7 +++--
+ drivers/i3c/master/mipi-i3c-hci/core.c   | 49 +++++++++++++++++++++++++++++++-
+ drivers/i3c/master/mipi-i3c-hci/dma.c    |  4 ++-
+ drivers/i3c/master/mipi-i3c-hci/hci.h    |  1 +
+ 5 files changed, 66 insertions(+), 7 deletions(-)
+Merging dmi/dmi-for-next (13a0ac816d22 firmware: dmi: Fortify entry point length checks)
+$ git merge -m Merge branch 'dmi-for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/jdelvare/staging.git dmi/dmi-for-next
+Already up to date.
+Merging hwmon-staging/hwmon-next (c03cd01890c5 ABI: sysfs-class-hwmon: document emergency/max/min temperature alarms)
+$ git merge -m Merge branch 'hwmon-next' of git://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git hwmon-staging/hwmon-next
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-class-hwmon        | 110 +++++++++++--
+ .../devicetree/bindings/trivial-devices.yaml       |   2 +
+ Documentation/hwmon/index.rst                      |   1 +
+ Documentation/hwmon/mp5990.rst                     |  84 ++++++++++
+ drivers/hwmon/aspeed-pwm-tacho.c                   |  26 ++-
+ drivers/hwmon/emc1403.c                            |   6 +-
+ drivers/hwmon/ltc2991.c                            |  20 ++-
+ drivers/hwmon/max6650.c                            |   8 +-
+ drivers/hwmon/nct6775-core.c                       |  26 +--
+ drivers/hwmon/nct6775-i2c.c                        |  14 +-
+ drivers/hwmon/nct6775-platform.c                   |  26 +--
+ drivers/hwmon/nct6775.h                            |   2 +-
+ drivers/hwmon/npcm750-pwm-fan.c                    |  30 +++-
+ drivers/hwmon/pmbus/Kconfig                        |   9 ++
+ drivers/hwmon/pmbus/Makefile                       |   1 +
+ drivers/hwmon/pmbus/lm25066.c                      |  14 +-
+ drivers/hwmon/pmbus/mp5990.c                       | 179 +++++++++++++++++++++
+ drivers/hwmon/sht4x.c                              |   3 +-
+ 18 files changed, 472 insertions(+), 89 deletions(-)
+ create mode 100644 Documentation/hwmon/mp5990.rst
+ create mode 100644 drivers/hwmon/pmbus/mp5990.c
+Merging jc_docs/docs-next (7c49ca6b0213 Documentation: Fix filename typo in ftrace doc)
+$ git merge -m Merge branch 'docs-next' of git://git.lwn.net/linux.git jc_docs/docs-next
+Auto-merging Documentation/filesystems/vfs.rst
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ Documentation/core-api/dma-api-howto.rst           |   2 +-
+ Documentation/core-api/dma-api.rst                 |   2 +-
+ Documentation/filesystems/vfs.rst                  |   2 +-
+ Documentation/trace/ftrace.rst                     |   2 +-
+ .../sp_SP/process/handling-regressions.rst         | 797 +++++++++++++++++++++
+ Documentation/translations/sp_SP/process/index.rst |   1 +
+ .../zh_CN/dev-tools/testing-overview.rst           |   2 +-
+ Documentation/translations/zh_TW/IRQ.txt           |   8 +-
+ .../translations/zh_TW/admin-guide/README.rst      |   2 +-
+ .../translations/zh_TW/admin-guide/bug-bisect.rst  |   2 +-
+ .../translations/zh_TW/admin-guide/bug-hunting.rst |   2 +-
+ .../zh_TW/admin-guide/clearing-warn-once.rst       |   2 +-
+ .../translations/zh_TW/admin-guide/cpu-load.rst    |   2 +-
+ .../translations/zh_TW/admin-guide/index.rst       |   2 +-
+ .../translations/zh_TW/admin-guide/init.rst        |   2 +-
+ .../zh_TW/admin-guide/reporting-issues.rst         |   2 +-
+ .../zh_TW/admin-guide/security-bugs.rst            |   2 +-
+ .../zh_TW/admin-guide/tainted-kernels.rst          |   2 +-
+ .../translations/zh_TW/admin-guide/unicode.rst     |   2 +-
+ .../translations/zh_TW/arch/arm64/amu.rst          |   2 +-
+ .../translations/zh_TW/arch/arm64/booting.txt      |   4 +-
+ .../translations/zh_TW/arch/arm64/elf_hwcaps.rst   |   2 +-
+ .../translations/zh_TW/arch/arm64/hugetlbpage.rst  |   2 +-
+ .../translations/zh_TW/arch/arm64/index.rst        |   2 +-
+ .../zh_TW/arch/arm64/legacy_instructions.txt       |   4 +-
+ .../translations/zh_TW/arch/arm64/memory.txt       |   4 +-
+ .../translations/zh_TW/arch/arm64/perf.rst         |   2 +-
+ .../zh_TW/arch/arm64/silicon-errata.txt            |   4 +-
+ .../zh_TW/arch/arm64/tagged-pointers.txt           |   4 +-
+ .../translations/zh_TW/dev-tools/sparse.rst        |  10 +-
+ .../zh_TW/dev-tools/testing-overview.rst           |   2 +-
+ .../translations/zh_TW/disclaimer-zh_TW.rst        |   2 +-
+ .../translations/zh_TW/filesystems/debugfs.rst     |   2 +-
+ .../translations/zh_TW/filesystems/index.rst       |   2 +-
+ .../translations/zh_TW/filesystems/sysfs.txt       |   2 +-
+ .../translations/zh_TW/filesystems/virtiofs.rst    |   2 +-
+ Documentation/translations/zh_TW/gpio.txt          |   8 +-
+ Documentation/translations/zh_TW/index.rst         |   2 +-
+ Documentation/translations/zh_TW/io_ordering.txt   |   8 +-
+ .../translations/zh_TW/process/1.Intro.rst         |   2 +-
+ .../translations/zh_TW/process/2.Process.rst       |   2 +-
+ .../translations/zh_TW/process/3.Early-stage.rst   |   2 +-
+ .../translations/zh_TW/process/4.Coding.rst        |   2 +-
+ .../translations/zh_TW/process/5.Posting.rst       |   2 +-
+ .../translations/zh_TW/process/6.Followthrough.rst |   2 +-
+ .../zh_TW/process/7.AdvancedTopics.rst             |   2 +-
+ .../translations/zh_TW/process/8.Conclusion.rst    |   2 +-
+ .../process/code-of-conduct-interpretation.rst     |   2 +-
+ .../translations/zh_TW/process/code-of-conduct.rst |   2 +-
+ .../translations/zh_TW/process/coding-style.rst    |   2 +-
+ .../zh_TW/process/development-process.rst          |   2 +-
+ .../translations/zh_TW/process/email-clients.rst   |   2 +-
+ .../zh_TW/process/embargoed-hardware-issues.rst    |   2 +-
+ Documentation/translations/zh_TW/process/howto.rst |   2 +-
+ Documentation/translations/zh_TW/process/index.rst |   2 +-
+ .../zh_TW/process/kernel-driver-statement.rst      |   2 +-
+ .../zh_TW/process/kernel-enforcement-statement.rst |   2 +-
+ .../translations/zh_TW/process/license-rules.rst   |   2 +-
+ .../translations/zh_TW/process/magic-number.rst    |   2 +-
+ .../zh_TW/process/management-style.rst             |   2 +-
+ .../zh_TW/process/programming-language.rst         |   2 +-
+ .../zh_TW/process/stable-api-nonsense.rst          |   2 +-
+ .../zh_TW/process/stable-kernel-rules.rst          |   2 +-
+ .../zh_TW/process/submit-checklist.rst             |   2 +-
+ .../zh_TW/process/submitting-patches.rst           |   2 +-
+ .../zh_TW/process/volatile-considered-harmful.rst  |   2 +-
+ MAINTAINERS                                        |   2 +-
+ 67 files changed, 881 insertions(+), 83 deletions(-)
+ create mode 100644 Documentation/translations/sp_SP/process/handling-regressions.rst
+Merging v4l-dvb/master (3e238417254b media: nuvoton: VIDEO_NPCM_VCD_ECE should depend on ARCH_NPCM)
+$ git merge -m Merge branch 'master' of git://linuxtv.org/media_tree.git v4l-dvb/master
+Already up to date.
+Merging v4l-dvb-next/master (3e238417254b media: nuvoton: VIDEO_NPCM_VCD_ECE should depend on ARCH_NPCM)
+$ git merge -m Merge branch 'master' of git://linuxtv.org/mchehab/media-next.git v4l-dvb-next/master
+Already up to date.
+Merging pm/linux-next (e8afe36a7287 Merge branch 'pm-tools' into linux-next)
+$ git merge -m Merge branch 'linux-next' of git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git pm/linux-next
+Merge made by the 'ort' strategy.
+ drivers/acpi/Makefile              |   2 +-
+ drivers/acpi/acpi_video.c          |  58 +--
+ drivers/acpi/bus.c                 |  32 +-
+ drivers/acpi/device_pm.c           |  13 +
+ drivers/acpi/internal.h            |   9 +
+ drivers/acpi/mipi-disco-img.c      | 725 +++++++++++++++++++++++++++++++++++++
+ drivers/acpi/processor_idle.c      |   2 +-
+ drivers/acpi/property.c            | 102 +++++-
+ drivers/acpi/resource.c            |   7 +
+ drivers/acpi/scan.c                |  56 ++-
+ drivers/thermal/thermal_core.c     |  43 ++-
+ include/acpi/acpi_bus.h            |  94 +++++
+ include/acpi/video.h               |   9 +
+ include/linux/property.h           |   7 +
+ include/linux/thermal.h            |   2 +
+ tools/power/pm-graph/sleepgraph.py |   2 +-
+ 16 files changed, 1065 insertions(+), 98 deletions(-)
+ create mode 100644 drivers/acpi/mipi-disco-img.c
+Merging cpufreq-arm/cpufreq/arm/linux-next (5b5b5806f223 cpufreq: qcom-nvmem: Introduce cpufreq for ipq95xx)
+$ git merge -m Merge branch 'cpufreq/arm/linux-next' of git://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git cpufreq-arm/cpufreq/arm/linux-next
+Already up to date.
+Merging cpupower/cpupower (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'cpupower' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux.git cpupower/cpupower
+Already up to date.
+Merging devfreq/devfreq-next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'devfreq-next' of git://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/linux.git devfreq/devfreq-next
+Already up to date.
+Merging pmdomain/next (9e0cceadb7a5 pmdomain: Merge branch fixes into next)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git pmdomain/next
+Already up to date.
+Merging opp/opp/linux-next (41907aa4ae37 OPP: Level zero is valid)
+$ git merge -m Merge branch 'opp/linux-next' of git://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git opp/opp/linux-next
+Merge made by the 'ort' strategy.
+ drivers/opp/core.c          | 24 ++++++++++++++++++++----
+ drivers/opp/of.c            |  8 +++++++-
+ drivers/opp/ti-opp-supply.c | 13 +++----------
+ include/linux/pm_opp.h      |  5 ++++-
+ 4 files changed, 34 insertions(+), 16 deletions(-)
+Merging thermal/thermal/linux-next (9618efe343ea thermal/qcom/tsens: Drop ops_v0_1)
+$ git merge -m Merge branch 'thermal/linux-next' of git://git.kernel.org/pub/scm/linux/kernel/git/thermal/linux.git thermal/thermal/linux-next
+Already up to date.
+Merging dlm/next (0c08699744d2 dlm: implement EXPORT_OP_ASYNC_LOCK)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/teigland/linux-dlm.git dlm/next
+Merge made by the 'ort' strategy.
+ fs/dlm/lowcomms.c | 14 +++++++-------
+ fs/dlm/plock.c    | 20 +++++---------------
+ fs/gfs2/export.c  |  1 +
+ fs/ocfs2/export.c |  1 +
+ 4 files changed, 14 insertions(+), 22 deletions(-)
+Merging rdma/for-next (057a30168175 RDMA/irdma: Use crypto_shash_digest() in irdma_ieq_check_mpacrc())
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git rdma/for-next
+Merge made by the 'ort' strategy.
+ drivers/infiniband/hw/bnxt_re/ib_verbs.c   |  59 +++++++-----
+ drivers/infiniband/hw/bnxt_re/ib_verbs.h   |   7 ++
+ drivers/infiniband/hw/bnxt_re/qplib_fp.c   | 138 ++++++++++++++---------------
+ drivers/infiniband/hw/bnxt_re/qplib_fp.h   |  20 ++++-
+ drivers/infiniband/hw/bnxt_re/qplib_rcfw.c |  17 ++--
+ drivers/infiniband/hw/bnxt_re/qplib_rcfw.h |   4 +-
+ drivers/infiniband/hw/bnxt_re/qplib_res.c  |   2 +-
+ drivers/infiniband/hw/bnxt_re/qplib_res.h  |  37 +++++---
+ drivers/infiniband/hw/hfi1/user_exp_rcv.c  |   4 +-
+ drivers/infiniband/hw/hfi1/user_sdma.c     |   4 +-
+ drivers/infiniband/hw/irdma/utils.c        |  11 +--
+ drivers/infiniband/sw/siw/siw.h            |   3 +-
+ drivers/infiniband/sw/siw/siw_mem.c        | 109 +++++++++--------------
+ drivers/infiniband/sw/siw/siw_mem.h        |   5 +-
+ drivers/infiniband/sw/siw/siw_qp_tx.c      |  12 +--
+ drivers/infiniband/sw/siw/siw_verbs.c      |  19 +---
+ include/uapi/rdma/bnxt_re-abi.h            |   9 ++
+ 17 files changed, 236 insertions(+), 224 deletions(-)
+Merging net-next/main (3a17ea77da31 Merge branch 'mlxsw-preparations-for-support-of-cff-flood-mode')
+$ git merge -m Merge branch 'main' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git net-next/main
+Auto-merging include/linux/netdevice.h
+Auto-merging net/core/dev.c
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/net/marvell,aquantia.yaml  |  116 ++
+ .../devicetree/bindings/net/renesas,etheravb.yaml  |    2 +-
+ .../devicetree/bindings/net/xlnx,axi-ethernet.yaml |   16 +
+ Documentation/netlink/specs/devlink.yaml           |    4 +-
+ .../device_drivers/ethernet/marvell/octeon_ep.rst  |    4 +
+ drivers/net/bonding/bond_alb.c                     |    3 +-
+ drivers/net/bonding/bond_main.c                    |   34 +-
+ drivers/net/ethernet/broadcom/bnxt/bnxt.c          |  515 +++++----
+ drivers/net/ethernet/broadcom/bnxt/bnxt.h          |   59 +-
+ drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c  |   11 +-
+ drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c      |   23 +-
+ drivers/net/ethernet/cadence/macb.h                |   15 +-
+ drivers/net/ethernet/cadence/macb_main.c           |   42 +-
+ drivers/net/ethernet/cadence/macb_ptp.c            |   28 +-
+ drivers/net/ethernet/google/gve/gve_dqo.h          |    3 +
+ drivers/net/ethernet/google/gve/gve_main.c         |   13 +
+ drivers/net/ethernet/google/gve/gve_tx_dqo.c       |   37 +-
+ drivers/net/ethernet/intel/i40e/i40e.h             |  148 ++-
+ drivers/net/ethernet/intel/i40e/i40e_adminq.c      |  161 ++-
+ drivers/net/ethernet/intel/i40e/i40e_common.c      |   68 +-
+ drivers/net/ethernet/intel/i40e/i40e_dcb.c         |    9 +-
+ drivers/net/ethernet/intel/i40e/i40e_dcb_nl.c      |   29 +-
+ drivers/net/ethernet/intel/i40e/i40e_debug.h       |    1 +
+ drivers/net/ethernet/intel/i40e/i40e_debugfs.c     |    7 +-
+ drivers/net/ethernet/intel/i40e/i40e_ethtool.c     |  246 ++--
+ drivers/net/ethernet/intel/i40e/i40e_main.c        |  620 +++++------
+ drivers/net/ethernet/intel/i40e/i40e_nvm.c         |   10 +-
+ drivers/net/ethernet/intel/i40e/i40e_prototype.h   |   70 ++
+ drivers/net/ethernet/intel/i40e/i40e_ptp.c         |   32 +-
+ drivers/net/ethernet/intel/i40e/i40e_register.h    |    1 +
+ drivers/net/ethernet/intel/i40e/i40e_txrx.c        |   20 +-
+ drivers/net/ethernet/intel/i40e/i40e_txrx.h        |    8 +-
+ drivers/net/ethernet/intel/i40e/i40e_type.h        |   51 +-
+ drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c |   20 +-
+ drivers/net/ethernet/intel/ice/ice.h               |   13 +-
+ drivers/net/ethernet/intel/ice/ice_devlink.c       |   29 +
+ drivers/net/ethernet/intel/ice/ice_devlink.h       |    1 +
+ drivers/net/ethernet/intel/ice/ice_eswitch.c       |  576 +++++-----
+ drivers/net/ethernet/intel/ice/ice_eswitch.h       |   22 +-
+ drivers/net/ethernet/intel/ice/ice_eswitch_br.c    |   22 +-
+ drivers/net/ethernet/intel/ice/ice_lib.c           |   81 +-
+ drivers/net/ethernet/intel/ice/ice_main.c          |    6 +-
+ drivers/net/ethernet/intel/ice/ice_repr.c          |  195 ++--
+ drivers/net/ethernet/intel/ice/ice_repr.h          |    9 +-
+ drivers/net/ethernet/intel/ice/ice_sriov.c         |   20 +-
+ drivers/net/ethernet/intel/ice/ice_tc_lib.c        |    4 +-
+ drivers/net/ethernet/intel/ice/ice_vf_lib.c        |    9 +-
+ drivers/net/ethernet/intel/ice/ice_vf_lib.h        |    2 +-
+ drivers/net/ethernet/intel/igc/igc.h               |   21 +-
+ drivers/net/ethernet/intel/igc/igc_base.h          |    4 +
+ drivers/net/ethernet/intel/igc/igc_defines.h       |    2 +
+ drivers/net/ethernet/intel/igc/igc_main.c          |   65 +-
+ drivers/net/ethernet/intel/igc/igc_ptp.c           |   50 +-
+ drivers/net/ethernet/intel/igc/igc_regs.h          |    5 +
+ drivers/net/ethernet/marvell/octeon_ep/Makefile    |    3 +-
+ .../net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c |  886 +++++++++++++++
+ .../net/ethernet/marvell/octeon_ep/octep_config.h  |    3 +-
+ .../net/ethernet/marvell/octeon_ep/octep_main.c    |   75 +-
+ .../net/ethernet/marvell/octeon_ep/octep_main.h    |   15 +
+ .../marvell/octeon_ep/octep_regs_cnxk_pf.h         |  400 +++++++
+ drivers/net/ethernet/marvell/octeon_ep/octep_tx.c  |    5 +-
+ drivers/net/ethernet/marvell/octeon_ep/octep_tx.h  |    3 -
+ drivers/net/ethernet/mediatek/mtk_eth_soc.c        |    5 +-
+ drivers/net/ethernet/mediatek/mtk_wed.c            |   10 +-
+ drivers/net/ethernet/mediatek/mtk_wed_wo.c         |    3 +-
+ .../net/ethernet/mellanox/mlx5/core/diag/crdump.c  |    5 +-
+ .../net/ethernet/mellanox/mlx5/core/en/rep/tc.c    |   14 +
+ .../ethernet/mellanox/mlx5/core/en/tc/act/pedit.c  |    3 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_main.c  |    1 -
+ drivers/net/ethernet/mellanox/mlx5/core/en_rep.c   |   26 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_stats.h |    2 +
+ drivers/net/ethernet/mellanox/mlx5/core/en_tc.c    |   24 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_tc.h    |    2 -
+ drivers/net/ethernet/mellanox/mlx5/core/fs_core.h  |    2 +-
+ .../net/ethernet/mellanox/mlx5/core/fs_counters.c  |    2 +-
+ drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c |   32 +-
+ .../net/ethernet/mellanox/mlx5/core/lib/clock.c    |   78 +-
+ drivers/net/ethernet/mellanox/mlx5/core/main.c     |   20 +-
+ drivers/net/ethernet/mellanox/mlxsw/cmd.h          |   11 +
+ drivers/net/ethernet/mellanox/mlxsw/core.c         |    7 +
+ drivers/net/ethernet/mellanox/mlxsw/core.h         |    9 +
+ drivers/net/ethernet/mellanox/mlxsw/pci.c          |  117 +-
+ drivers/net/ethernet/mellanox/mlxsw/reg.h          |   94 +-
+ drivers/net/ethernet/mellanox/mlxsw/resources.h    |    2 +
+ drivers/net/ethernet/mellanox/mlxsw/spectrum.h     |    2 +
+ drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c |   44 +-
+ .../net/ethernet/mellanox/mlxsw/spectrum_router.c  |   20 +-
+ drivers/net/ethernet/microchip/lan743x_main.h      |    2 +-
+ drivers/net/ethernet/netronome/nfp/nfp_net.h       |   36 +
+ .../net/ethernet/netronome/nfp/nfp_net_common.c    |  183 +++
+ drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h  |   15 +
+ .../net/ethernet/netronome/nfp/nfp_net_ethtool.c   |  369 ++++++
+ drivers/net/ethernet/realtek/r8169_main.c          |  139 +--
+ drivers/net/ethernet/stmicro/stmmac/stmmac_main.c  |   29 +-
+ drivers/net/ethernet/ti/am65-cpsw-ethtool.c        |   26 +
+ drivers/net/ethernet/ti/am65-cpsw-nuss.c           |  197 ++--
+ drivers/net/ethernet/xilinx/Kconfig                |    1 +
+ drivers/net/ethernet/xilinx/xilinx_axienet.h       |   35 +
+ drivers/net/ethernet/xilinx/xilinx_axienet_main.c  |  683 +++++++++---
+ drivers/net/macvlan.c                              |   14 +-
+ drivers/net/phy/Kconfig                            |    5 +-
+ drivers/net/phy/Makefile                           |    6 +-
+ drivers/net/phy/aquantia.h                         |   16 -
+ drivers/net/phy/aquantia/Kconfig                   |    6 +
+ drivers/net/phy/aquantia/Makefile                  |    6 +
+ drivers/net/phy/aquantia/aquantia.h                |  117 ++
+ drivers/net/phy/aquantia/aquantia_firmware.c       |  370 ++++++
+ drivers/net/phy/{ => aquantia}/aquantia_hwmon.c    |   14 -
+ drivers/net/phy/{ => aquantia}/aquantia_main.c     |   61 +-
+ drivers/net/phy/bcm-phy-ptp.c                      |   15 +-
+ drivers/net/phy/broadcom.c                         |    2 +
+ drivers/net/phy/dp83640.c                          |   24 +-
+ drivers/net/phy/micrel.c                           |   44 +-
+ drivers/net/phy/mscc/mscc_ptp.c                    |   18 +-
+ drivers/net/phy/nxp-c45-tja11xx.c                  |   17 +-
+ drivers/net/phy/phy.c                              |   28 +-
+ drivers/net/phy/phylink.c                          |   22 +-
+ drivers/net/phy/sfp-bus.c                          |    2 +-
+ drivers/net/usb/ax88179_178a.c                     |    2 -
+ drivers/net/vxlan/vxlan_core.c                     |   23 +-
+ drivers/pci/pci.c                                  |    3 +
+ drivers/pci/quirks.c                               |   13 +
+ drivers/ptp/ptp_ines.c                             |   16 +-
+ include/linux/ethtool.h                            |    8 +
+ include/linux/indirect_call_wrapper.h              |    2 +-
+ include/linux/linkmode.h                           |    5 +
+ include/linux/mii_timestamper.h                    |    4 +-
+ include/linux/mlx5/mlx5_ifc.h                      |    5 +-
+ include/linux/netdevice.h                          |    3 +
+ include/linux/netlink.h                            |    1 +
+ include/linux/phy.h                                |    6 +-
+ include/net/ip_tunnels.h                           |   11 +
+ include/net/netlink.h                              |   14 +
+ include/net/vxlan.h                                |   33 +-
+ include/uapi/linux/batadv_packet.h                 |   45 +-
+ include/uapi/linux/if_link.h                       |    8 +
+ net/8021q/vlan_dev.c                               |   15 +-
+ net/batman-adv/Makefile                            |    1 +
+ net/batman-adv/bridge_loop_avoidance.c             |    2 +-
+ net/batman-adv/fragmentation.c                     |    8 +-
+ net/batman-adv/gateway_client.c                    |    2 +-
+ net/batman-adv/main.c                              |    5 +-
+ net/batman-adv/main.h                              |    2 +-
+ net/batman-adv/multicast.c                         |  129 ++-
+ net/batman-adv/multicast.h                         |   30 +-
+ net/batman-adv/multicast_forw.c                    | 1178 ++++++++++++++++++++
+ net/batman-adv/netlink.c                           |    2 +-
+ net/batman-adv/originator.c                        |   28 +
+ net/batman-adv/originator.h                        |    3 +
+ net/batman-adv/routing.c                           |   70 ++
+ net/batman-adv/routing.h                           |   11 +
+ net/batman-adv/soft-interface.c                    |   18 +-
+ net/batman-adv/types.h                             |   70 ++
+ net/core/dev.c                                     |   10 +-
+ net/core/dev_ioctl.c                               |    7 +-
+ net/core/rtnetlink.c                               |    2 +-
+ net/devlink/core.c                                 |    4 +-
+ net/devlink/dev.c                                  |    8 +
+ net/devlink/devl_internal.h                        |   21 +-
+ net/devlink/health.c                               |    3 +-
+ net/devlink/netlink.c                              |   45 +-
+ net/devlink/netlink_gen.c                          |    4 +-
+ net/devlink/netlink_gen.h                          |    5 +
+ net/devlink/region.c                               |    3 +-
+ net/dsa/tag_rtl4_a.c                               |    5 +-
+ net/ethtool/common.c                               |    6 +
+ net/ipv4/tcp_ipv4.c                                |    6 +
+ net/ipv4/tcp_timer.c                               |    4 +-
+ net/ipv6/tcp_ipv6.c                                |    9 +-
+ net/ncsi/internal.h                                |    7 +-
+ net/ncsi/ncsi-cmd.c                                |    3 +-
+ net/ncsi/ncsi-manage.c                             |   29 +-
+ net/ncsi/ncsi-netlink.c                            |    4 +-
+ net/ncsi/ncsi-pkt.h                                |   17 +-
+ net/ncsi/ncsi-rsp.c                                |   67 +-
+ net/netlink/af_netlink.c                           |    3 +-
+ net/sched/cls_u32.c                                |   36 +-
+ net/sched/sch_generic.c                            |    5 +-
+ net/tipc/netlink_compat.c                          |    2 +-
+ .../selftests/drivers/net/mlxsw/pci_reset.sh       |   58 +
+ tools/testing/selftests/net/Makefile               |    1 +
+ tools/testing/selftests/net/cmsg_sender.c          |   50 +-
+ tools/testing/selftests/net/fq_band_pktlimit.sh    |   57 +
+ tools/testing/selftests/net/net_helper.sh          |   22 +
+ tools/testing/selftests/net/udpgro.sh              |   13 +-
+ tools/testing/selftests/net/udpgro_bench.sh        |    5 +-
+ tools/testing/selftests/net/udpgro_frglist.sh      |    5 +-
+ .../selftests/tc-testing/plugin-lib/nsPlugin.py    |  192 ++--
+ .../selftests/tc-testing/tc-tests/filters/u32.json |   57 +
+ tools/testing/selftests/tc-testing/tdc.py          |    3 +-
+ tools/testing/selftests/tc-testing/tdc.sh          |   69 +-
+ 191 files changed, 8115 insertions(+), 2497 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/net/marvell,aquantia.yaml
+ create mode 100644 drivers/net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c
+ create mode 100644 drivers/net/ethernet/marvell/octeon_ep/octep_regs_cnxk_pf.h
+ delete mode 100644 drivers/net/phy/aquantia.h
+ create mode 100644 drivers/net/phy/aquantia/Kconfig
+ create mode 100644 drivers/net/phy/aquantia/Makefile
+ create mode 100644 drivers/net/phy/aquantia/aquantia.h
+ create mode 100644 drivers/net/phy/aquantia/aquantia_firmware.c
+ rename drivers/net/phy/{ => aquantia}/aquantia_hwmon.c (90%)
+ rename drivers/net/phy/{ => aquantia}/aquantia_main.c (91%)
+ create mode 100644 net/batman-adv/multicast_forw.c
+ create mode 100755 tools/testing/selftests/drivers/net/mlxsw/pci_reset.sh
+ create mode 100755 tools/testing/selftests/net/fq_band_pktlimit.sh
+ create mode 100755 tools/testing/selftests/net/net_helper.sh
+Merging bpf-next/for-next (3cbbf9192abd Merge branch 'selftests-bpf-update-multiple-prog_tests-to-use-assert_-macros')
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git bpf-next/for-next
+Auto-merging include/linux/bpf.h
+Auto-merging include/linux/bpf_verifier.h
+Auto-merging kernel/bpf/verifier.c
+Auto-merging kernel/cgroup/cgroup.c
+Merge made by the 'ort' strategy.
+ Documentation/bpf/kfuncs.rst                       |   24 +
+ include/linux/bpf.h                                |    6 +-
+ include/linux/bpf_verifier.h                       |   77 +
+ include/linux/cgroup-defs.h                        |    1 +
+ include/linux/cgroup.h                             |    4 +-
+ include/linux/compiler-gcc.h                       |    2 +-
+ include/linux/tnum.h                               |    4 +
+ include/uapi/linux/bpf.h                           |   29 +-
+ kernel/bpf/btf.c                                   |   11 +-
+ kernel/bpf/helpers.c                               |   46 +-
+ kernel/bpf/log.c                                   |  480 +++++
+ kernel/bpf/lpm_trie.c                              |    3 +
+ kernel/bpf/stackmap.c                              |   11 +-
+ kernel/bpf/syscall.c                               |    3 +-
+ kernel/bpf/task_iter.c                             |   29 +-
+ kernel/bpf/tnum.c                                  |    7 +-
+ kernel/bpf/verifier.c                              | 1782 +++++++---------
+ kernel/cgroup/cgroup-internal.h                    |    4 +-
+ kernel/cgroup/cgroup-v1.c                          |   34 +
+ kernel/cgroup/cgroup.c                             |   45 +-
+ kernel/trace/bpf_trace.c                           |   12 +-
+ lib/test_bpf.c                                     |   16 -
+ tools/bpf/bpftool/Documentation/bpftool.rst        |    2 +-
+ tools/include/uapi/linux/bpf.h                     |   29 +-
+ tools/lib/bpf/libbpf_common.h                      |   13 +-
+ tools/testing/selftests/bpf/cgroup_helpers.c       |  116 +-
+ tools/testing/selftests/bpf/cgroup_helpers.h       |    4 +-
+ tools/testing/selftests/bpf/config.aarch64         |   17 +-
+ tools/testing/selftests/bpf/config.s390x           |    9 -
+ tools/testing/selftests/bpf/config.vm              |   12 +
+ tools/testing/selftests/bpf/config.x86_64          |   12 -
+ .../selftests/bpf/map_tests/map_percpu_stats.c     |   39 +-
+ tools/testing/selftests/bpf/prog_tests/align.c     |   42 +-
+ tools/testing/selftests/bpf/prog_tests/bind_perm.c |    6 +-
+ tools/testing/selftests/bpf/prog_tests/bpf_iter.c  |   87 +-
+ .../testing/selftests/bpf/prog_tests/bpf_obj_id.c  |  210 +-
+ .../testing/selftests/bpf/prog_tests/bpf_tcp_ca.c  |   48 +-
+ .../selftests/bpf/prog_tests/bpf_verif_scale.c     |    2 +-
+ tools/testing/selftests/bpf/prog_tests/btf.c       |    1 +
+ .../selftests/bpf/prog_tests/cgroup1_hierarchy.c   |  158 ++
+ .../testing/selftests/bpf/prog_tests/cgroup_v1v2.c |    2 +-
+ .../selftests/bpf/prog_tests/local_kptr_stash.c    |   33 +
+ tools/testing/selftests/bpf/prog_tests/log_buf.c   |    4 +-
+ .../testing/selftests/bpf/prog_tests/reg_bounds.c  | 2124 ++++++++++++++++++++
+ tools/testing/selftests/bpf/prog_tests/spin_lock.c |   14 +-
+ tools/testing/selftests/bpf/prog_tests/tc_opts.c   |    6 +-
+ tools/testing/selftests/bpf/prog_tests/vmlinux.c   |   16 +-
+ .../selftests/bpf/progs/bpf_iter_task_stack.c      |    5 +
+ .../selftests/bpf/progs/exceptions_assert.c        |   40 +-
+ tools/testing/selftests/bpf/progs/iters.c          |   22 +
+ .../testing/selftests/bpf/progs/local_kptr_stash.c |   71 +
+ tools/testing/selftests/bpf/progs/pyperf180.c      |   22 +
+ .../selftests/bpf/progs/refcounted_kptr_fail.c     |   19 +
+ .../selftests/bpf/progs/test_cgroup1_hierarchy.c   |   71 +
+ .../testing/selftests/bpf/progs/verifier_bounds.c  |    2 +
+ tools/testing/selftests/bpf/test_loader.c          |   35 +-
+ tools/testing/selftests/bpf/test_maps.c            |   17 +-
+ tools/testing/selftests/bpf/test_maps.h            |    5 +
+ tools/testing/selftests/bpf/test_sock_addr.c       |    2 +-
+ tools/testing/selftests/bpf/test_verifier.c        |    2 +-
+ tools/testing/selftests/bpf/testing_helpers.c      |    4 +-
+ tools/testing/selftests/bpf/veristat.c             |   89 +-
+ tools/testing/selftests/bpf/vmtest.sh              |    4 +-
+ 63 files changed, 4513 insertions(+), 1533 deletions(-)
+ create mode 100644 tools/testing/selftests/bpf/config.vm
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/cgroup1_hierarchy.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/reg_bounds.c
+ create mode 100644 tools/testing/selftests/bpf/progs/test_cgroup1_hierarchy.c
+Merging ipsec-next/master (18de1e517ed3 gve: add gve_features_check())
+$ git merge -m Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec-next.git ipsec-next/master
+Already up to date.
+Merging mlx5-next/mlx5-next (82f9378c443c net/mlx5: Handle IPsec steering upon master unbind/bind)
+$ git merge -m Merge branch 'mlx5-next' of git://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux.git mlx5-next/mlx5-next
+Already up to date.
+Merging netfilter-next/main (ac40916a3f72 rtnetlink: introduce nlmsg_new_large and use it in rtnl_getlink)
+$ git merge -m Merge branch 'main' of git://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf-next.git netfilter-next/main
+Already up to date.
+Merging ipvs-next/main (9cdee0634769 netfilter: nf_tables: Carry reset boolean in nft_set_dump_ctx)
+$ git merge -m Merge branch 'main' of git://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs-next.git ipvs-next/main
+Already up to date.
+Merging bluetooth/master (d5742377e3ac Bluetooth: qca: Set both WIDEBAND_SPEECH and LE_STATES quirks for QCA2066)
+$ git merge -m Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/bluetooth/bluetooth-next.git bluetooth/master
+Merge made by the 'ort' strategy.
+ drivers/bluetooth/hci_qca.c      |   1 +
+ drivers/bluetooth/hci_vhci.c     |  10 +--
+ include/net/bluetooth/hci_core.h |  24 +++++++
+ net/bluetooth/hci_conn.c         |  32 ++++++++-
+ net/bluetooth/hci_event.c        |   8 +--
+ net/bluetooth/iso.c              | 146 +++++++++++++++++++++++++++++++++++++--
+ 6 files changed, 201 insertions(+), 20 deletions(-)
+Merging wireless-next/for-next (0a78bb64a499 wifi: rtw89: pci: update interrupt mitigation register for 8922AE)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless-next.git wireless-next/for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../networking/device_drivers/wifi/index.rst       |    1 -
+ .../networking/device_drivers/wifi/ray_cs.rst      |  165 -
+ MAINTAINERS                                        |   41 -
+ drivers/net/wireless/Kconfig                       |    3 -
+ drivers/net/wireless/Makefile                      |    2 -
+ drivers/net/wireless/atmel/Kconfig                 |   35 -
+ drivers/net/wireless/atmel/Makefile                |    4 -
+ drivers/net/wireless/atmel/atmel.c                 | 4452 -----------
+ drivers/net/wireless/atmel/atmel.h                 |   31 -
+ drivers/net/wireless/atmel/atmel_cs.c              |  292 -
+ drivers/net/wireless/atmel/atmel_pci.c             |   65 -
+ .../broadcom/brcm80211/brcmfmac/cfg80211.c         |    2 +-
+ .../net/wireless/broadcom/brcm80211/brcmfmac/p2p.c |    2 +-
+ .../wireless/broadcom/brcm80211/brcmsmac/channel.c |    6 +-
+ .../net/wireless/broadcom/brcm80211/brcmsmac/dma.c |    3 +-
+ .../wireless/broadcom/brcm80211/brcmsmac/main.c    |    4 +-
+ drivers/net/wireless/cisco/Kconfig                 |   59 -
+ drivers/net/wireless/cisco/Makefile                |    3 -
+ drivers/net/wireless/cisco/airo.c                  | 8288 --------------------
+ drivers/net/wireless/cisco/airo.h                  |   10 -
+ drivers/net/wireless/cisco/airo_cs.c               |  218 -
+ drivers/net/wireless/intel/iwlegacy/4965-mac.c     |    6 -
+ drivers/net/wireless/intersil/Kconfig              |    2 -
+ drivers/net/wireless/intersil/Makefile             |    2 -
+ drivers/net/wireless/intersil/hostap/Kconfig       |   95 -
+ drivers/net/wireless/intersil/hostap/Makefile      |    8 -
+ drivers/net/wireless/intersil/hostap/hostap.h      |   98 -
+ .../net/wireless/intersil/hostap/hostap_80211.h    |   97 -
+ .../net/wireless/intersil/hostap/hostap_80211_rx.c | 1116 ---
+ .../net/wireless/intersil/hostap/hostap_80211_tx.c |  554 --
+ drivers/net/wireless/intersil/hostap/hostap_ap.c   | 3277 --------
+ drivers/net/wireless/intersil/hostap/hostap_ap.h   |  264 -
+ .../net/wireless/intersil/hostap/hostap_common.h   |  420 -
+ .../net/wireless/intersil/hostap/hostap_config.h   |   49 -
+ drivers/net/wireless/intersil/hostap/hostap_cs.c   |  710 --
+ .../net/wireless/intersil/hostap/hostap_download.c |  810 --
+ drivers/net/wireless/intersil/hostap/hostap_hw.c   | 3387 --------
+ drivers/net/wireless/intersil/hostap/hostap_info.c |  509 --
+ .../net/wireless/intersil/hostap/hostap_ioctl.c    | 3847 ---------
+ drivers/net/wireless/intersil/hostap/hostap_main.c | 1123 ---
+ drivers/net/wireless/intersil/hostap/hostap_pci.c  |  445 --
+ drivers/net/wireless/intersil/hostap/hostap_plx.c  |  617 --
+ drivers/net/wireless/intersil/hostap/hostap_proc.c |  411 -
+ drivers/net/wireless/intersil/hostap/hostap_wlan.h | 1051 ---
+ drivers/net/wireless/intersil/orinoco/Kconfig      |  143 -
+ drivers/net/wireless/intersil/orinoco/Makefile     |   15 -
+ drivers/net/wireless/intersil/orinoco/airport.c    |  268 -
+ drivers/net/wireless/intersil/orinoco/cfg.c        |  291 -
+ drivers/net/wireless/intersil/orinoco/cfg.h        |   15 -
+ drivers/net/wireless/intersil/orinoco/fw.c         |  387 -
+ drivers/net/wireless/intersil/orinoco/fw.h         |   21 -
+ drivers/net/wireless/intersil/orinoco/hermes.c     |  778 --
+ drivers/net/wireless/intersil/orinoco/hermes.h     |  534 --
+ drivers/net/wireless/intersil/orinoco/hermes_dld.c |  477 --
+ drivers/net/wireless/intersil/orinoco/hermes_dld.h |   52 -
+ drivers/net/wireless/intersil/orinoco/hermes_rid.h |  165 -
+ drivers/net/wireless/intersil/orinoco/hw.c         | 1362 ----
+ drivers/net/wireless/intersil/orinoco/hw.h         |   60 -
+ drivers/net/wireless/intersil/orinoco/main.c       | 2414 ------
+ drivers/net/wireless/intersil/orinoco/main.h       |   50 -
+ drivers/net/wireless/intersil/orinoco/mic.c        |   89 -
+ drivers/net/wireless/intersil/orinoco/mic.h        |   23 -
+ drivers/net/wireless/intersil/orinoco/orinoco.h    |  251 -
+ drivers/net/wireless/intersil/orinoco/orinoco_cs.c |  350 -
+ .../net/wireless/intersil/orinoco/orinoco_nortel.c |  314 -
+ .../net/wireless/intersil/orinoco/orinoco_pci.c    |  257 -
+ .../net/wireless/intersil/orinoco/orinoco_pci.h    |   54 -
+ .../net/wireless/intersil/orinoco/orinoco_plx.c    |  362 -
+ .../net/wireless/intersil/orinoco/orinoco_tmd.c    |  237 -
+ .../net/wireless/intersil/orinoco/orinoco_usb.c    | 1787 -----
+ drivers/net/wireless/intersil/orinoco/scan.c       |  259 -
+ drivers/net/wireless/intersil/orinoco/scan.h       |   21 -
+ .../net/wireless/intersil/orinoco/spectrum_cs.c    |  328 -
+ drivers/net/wireless/intersil/orinoco/wext.c       | 1428 ----
+ drivers/net/wireless/intersil/orinoco/wext.h       |   13 -
+ drivers/net/wireless/legacy/Kconfig                |   55 -
+ drivers/net/wireless/legacy/Makefile               |    6 -
+ drivers/net/wireless/legacy/ray_cs.c               | 2824 -------
+ drivers/net/wireless/legacy/ray_cs.h               |   74 -
+ drivers/net/wireless/legacy/rayctl.h               |  734 --
+ drivers/net/wireless/legacy/rndis_wlan.c           | 3760 ---------
+ drivers/net/wireless/legacy/wl3501.h               |  615 --
+ drivers/net/wireless/legacy/wl3501_cs.c            | 2036 -----
+ drivers/net/wireless/marvell/libertas/Kconfig      |    9 +-
+ drivers/net/wireless/marvell/libertas/Makefile     |    1 -
+ drivers/net/wireless/marvell/libertas/if_cs.c      |  957 ---
+ drivers/net/wireless/microchip/wilc1000/cfg80211.c |   24 +-
+ drivers/net/wireless/microchip/wilc1000/hif.c      |   46 +-
+ drivers/net/wireless/microchip/wilc1000/hif.h      |   42 +-
+ drivers/net/wireless/microchip/wilc1000/sdio.c     |    9 +-
+ drivers/net/wireless/purelifi/plfxlc/usb.c         |    5 +-
+ drivers/net/wireless/ralink/rt2x00/rt2800.h        |    4 +
+ drivers/net/wireless/ralink/rt2x00/rt2800lib.c     |   83 +-
+ drivers/net/wireless/ralink/rt2x00/rt2x00.h        |    3 +
+ drivers/net/wireless/ralink/rt2x00/rt2x00dev.c     |    3 +
+ drivers/net/wireless/ralink/rt2x00/rt2x00mac.c     |   11 +
+ .../wireless/realtek/rtlwifi/rtl8192c/phy_common.c |    4 -
+ .../net/wireless/realtek/rtlwifi/rtl8192ee/dm.c    |   11 +-
+ .../net/wireless/realtek/rtlwifi/rtl8192ee/hw.c    |    1 -
+ drivers/net/wireless/realtek/rtlwifi/wifi.h        |   15 -
+ drivers/net/wireless/realtek/rtw88/mac80211.c      |    4 +-
+ drivers/net/wireless/realtek/rtw88/tx.c            |    3 +-
+ drivers/net/wireless/realtek/rtw89/cam.c           |   16 +-
+ drivers/net/wireless/realtek/rtw89/coex.c          |    4 +-
+ drivers/net/wireless/realtek/rtw89/core.c          |   90 +-
+ drivers/net/wireless/realtek/rtw89/core.h          |    9 +-
+ drivers/net/wireless/realtek/rtw89/mac.h           |    1 +
+ drivers/net/wireless/realtek/rtw89/pci.c           |  323 +-
+ drivers/net/wireless/realtek/rtw89/pci.h           |  515 ++
+ drivers/net/wireless/realtek/rtw89/pci_be.c        |  509 ++
+ drivers/net/wireless/realtek/rtw89/reg.h           |  356 +
+ drivers/net/wireless/realtek/rtw89/rtw8851b.c      |    1 +
+ drivers/net/wireless/realtek/rtw89/rtw8851be.c     |    3 +
+ drivers/net/wireless/realtek/rtw89/rtw8852a.c      |    1 +
+ drivers/net/wireless/realtek/rtw89/rtw8852ae.c     |    4 +
+ drivers/net/wireless/realtek/rtw89/rtw8852b.c      |    1 +
+ drivers/net/wireless/realtek/rtw89/rtw8852be.c     |    4 +
+ drivers/net/wireless/realtek/rtw89/rtw8852c.c      |    1 +
+ drivers/net/wireless/realtek/rtw89/rtw8852ce.c     |    4 +
+ drivers/net/wireless/realtek/rtw89/rtw8922ae.c     |   86 +
+ drivers/net/wireless/realtek/rtw89/ser.c           |    6 +
+ drivers/net/wireless/realtek/rtw89/txrx.h          |    4 +
+ drivers/net/wireless/zydas/Kconfig                 |   19 -
+ drivers/net/wireless/zydas/Makefile                |    2 -
+ drivers/net/wireless/zydas/zd1201.c                | 1909 -----
+ drivers/net/wireless/zydas/zd1201.h                |  144 -
+ 126 files changed, 2004 insertions(+), 58736 deletions(-)
+ delete mode 100644 Documentation/networking/device_drivers/wifi/ray_cs.rst
+ delete mode 100644 drivers/net/wireless/atmel/atmel.c
+ delete mode 100644 drivers/net/wireless/atmel/atmel.h
+ delete mode 100644 drivers/net/wireless/atmel/atmel_cs.c
+ delete mode 100644 drivers/net/wireless/atmel/atmel_pci.c
+ delete mode 100644 drivers/net/wireless/cisco/Kconfig
+ delete mode 100644 drivers/net/wireless/cisco/Makefile
+ delete mode 100644 drivers/net/wireless/cisco/airo.c
+ delete mode 100644 drivers/net/wireless/cisco/airo.h
+ delete mode 100644 drivers/net/wireless/cisco/airo_cs.c
+ delete mode 100644 drivers/net/wireless/intersil/hostap/Kconfig
+ delete mode 100644 drivers/net/wireless/intersil/hostap/Makefile
+ delete mode 100644 drivers/net/wireless/intersil/hostap/hostap.h
+ delete mode 100644 drivers/net/wireless/intersil/hostap/hostap_80211.h
+ delete mode 100644 drivers/net/wireless/intersil/hostap/hostap_80211_rx.c
+ delete mode 100644 drivers/net/wireless/intersil/hostap/hostap_80211_tx.c
+ delete mode 100644 drivers/net/wireless/intersil/hostap/hostap_ap.c
+ delete mode 100644 drivers/net/wireless/intersil/hostap/hostap_ap.h
+ delete mode 100644 drivers/net/wireless/intersil/hostap/hostap_common.h
+ delete mode 100644 drivers/net/wireless/intersil/hostap/hostap_config.h
+ delete mode 100644 drivers/net/wireless/intersil/hostap/hostap_cs.c
+ delete mode 100644 drivers/net/wireless/intersil/hostap/hostap_download.c
+ delete mode 100644 drivers/net/wireless/intersil/hostap/hostap_hw.c
+ delete mode 100644 drivers/net/wireless/intersil/hostap/hostap_info.c
+ delete mode 100644 drivers/net/wireless/intersil/hostap/hostap_ioctl.c
+ delete mode 100644 drivers/net/wireless/intersil/hostap/hostap_main.c
+ delete mode 100644 drivers/net/wireless/intersil/hostap/hostap_pci.c
+ delete mode 100644 drivers/net/wireless/intersil/hostap/hostap_plx.c
+ delete mode 100644 drivers/net/wireless/intersil/hostap/hostap_proc.c
+ delete mode 100644 drivers/net/wireless/intersil/hostap/hostap_wlan.h
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/Kconfig
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/Makefile
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/airport.c
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/cfg.c
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/cfg.h
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/fw.c
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/fw.h
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/hermes.c
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/hermes.h
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/hermes_dld.c
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/hermes_dld.h
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/hermes_rid.h
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/hw.c
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/hw.h
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/main.c
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/main.h
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/mic.c
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/mic.h
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/orinoco.h
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/orinoco_cs.c
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/orinoco_nortel.c
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/orinoco_pci.c
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/orinoco_pci.h
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/orinoco_plx.c
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/orinoco_tmd.c
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/orinoco_usb.c
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/scan.c
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/scan.h
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/spectrum_cs.c
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/wext.c
+ delete mode 100644 drivers/net/wireless/intersil/orinoco/wext.h
+ delete mode 100644 drivers/net/wireless/legacy/Kconfig
+ delete mode 100644 drivers/net/wireless/legacy/Makefile
+ delete mode 100644 drivers/net/wireless/legacy/ray_cs.c
+ delete mode 100644 drivers/net/wireless/legacy/ray_cs.h
+ delete mode 100644 drivers/net/wireless/legacy/rayctl.h
+ delete mode 100644 drivers/net/wireless/legacy/rndis_wlan.c
+ delete mode 100644 drivers/net/wireless/legacy/wl3501.h
+ delete mode 100644 drivers/net/wireless/legacy/wl3501_cs.c
+ delete mode 100644 drivers/net/wireless/marvell/libertas/if_cs.c
+ create mode 100644 drivers/net/wireless/realtek/rtw89/pci_be.c
+ create mode 100644 drivers/net/wireless/realtek/rtw89/rtw8922ae.c
+ delete mode 100644 drivers/net/wireless/zydas/zd1201.c
+ delete mode 100644 drivers/net/wireless/zydas/zd1201.h
+Merging wpan-next/master (18b849f12dcc ieee802154: ca8210: Remove stray gpiod_unexport() call)
+$ git merge -m Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git wpan-next/master
+Already up to date.
+Merging wpan-staging/staging (18b849f12dcc ieee802154: ca8210: Remove stray gpiod_unexport() call)
+$ git merge -m Merge branch 'staging' of git://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git wpan-staging/staging
+Already up to date.
+Merging mtd/mtd/next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'mtd/next' of git://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git mtd/mtd/next
+Already up to date.
+Merging nand/nand/next (acb1fd579efb mtd: rawnand: meson: initialize clock register)
+$ git merge -m Merge branch 'nand/next' of git://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git nand/nand/next
+Merge made by the 'ort' strategy.
+ drivers/mtd/nand/raw/brcmnand/bcm63138_nand.c |  2 +-
+ drivers/mtd/nand/raw/brcmnand/bcm6368_nand.c  |  2 +-
+ drivers/mtd/nand/raw/brcmnand/bcma_nand.c     |  2 +-
+ drivers/mtd/nand/raw/brcmnand/brcmnand.c      |  4 +---
+ drivers/mtd/nand/raw/brcmnand/brcmnand.h      |  2 +-
+ drivers/mtd/nand/raw/brcmnand/brcmstb_nand.c  |  2 +-
+ drivers/mtd/nand/raw/brcmnand/iproc_nand.c    |  2 +-
+ drivers/mtd/nand/raw/fsl_ifc_nand.c           |  2 +-
+ drivers/mtd/nand/raw/meson_nand.c             |  4 +++-
+ drivers/mtd/nand/raw/txx9ndfmc.c              | 13 +++++--------
+ 10 files changed, 16 insertions(+), 19 deletions(-)
+Merging spi-nor/spi-nor/next (4137f324cb29 MAINTAINERS: spi-nor: add myself as maintainer)
+$ git merge -m Merge branch 'spi-nor/next' of git://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git spi-nor/spi-nor/next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                 |  2 +-
+ drivers/mtd/spi-nor/atmel.c | 16 +++++++---------
+ drivers/mtd/spi-nor/core.c  |  5 ++---
+ drivers/mtd/spi-nor/core.h  |  6 +++---
+ drivers/mtd/spi-nor/sst.c   |  6 +++---
+ drivers/mtd/spi-nor/swp.c   | 25 ++++++++++++-------------
+ 6 files changed, 28 insertions(+), 32 deletions(-)
+Merging crypto/master (7c18e3c6b309 hwrng: virtio - remove #ifdef guards for PM functions)
+$ git merge -m Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/cryptodev-2.6.git crypto/master
+Merge made by the 'ort' strategy.
+ .../bindings/crypto/qcom,inline-crypto-engine.yaml |  1 +
+ .../devicetree/bindings/crypto/qcom,prng.yaml      |  1 +
+ .../devicetree/bindings/crypto/qcom-qce.yaml       |  1 +
+ arch/x86/crypto/sha1_ssse3_glue.c                  |  7 ++-
+ arch/x86/crypto/sha256_ssse3_glue.c                |  7 ++-
+ crypto/drbg.c                                      | 40 ++++++----------
+ crypto/rsa.c                                       |  2 +
+ crypto/shash.c                                     |  6 +--
+ crypto/testmgr.c                                   | 25 ++--------
+ drivers/char/hw_random/ingenic-rng.c               |  2 +-
+ drivers/char/hw_random/virtio-rng.c                |  8 +---
+ drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c  | 25 ++--------
+ drivers/crypto/ccree/cc_aead.c                     | 10 ++--
+ drivers/crypto/ccree/cc_cipher.c                   | 10 ++--
+ drivers/crypto/hisilicon/debugfs.c                 | 50 ++++++++++----------
+ drivers/crypto/hisilicon/qm.c                      | 22 ++++++---
+ drivers/crypto/hisilicon/zip/zip_main.c            | 54 +++++++++++-----------
+ drivers/crypto/intel/qat/qat_common/adf_sysfs.c    |  3 +-
+ drivers/crypto/marvell/cesa/cesa.c                 |  6 +--
+ drivers/crypto/n2_core.c                           |  9 +++-
+ drivers/crypto/starfive/jh7110-cryp.c              | 10 ++--
+ lib/crypto/aesgcm.c                                | 13 ++++++
+ 22 files changed, 150 insertions(+), 162 deletions(-)
+Merging drm/drm-next (c79b972eb88b Merge tag 'drm-misc-next-2023-11-17' of git://anongit.freedesktop.org/drm/drm-misc into drm-next)
+$ git merge -m Merge branch 'drm-next' of git://git.freedesktop.org/git/drm/drm.git drm/drm-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/accel/ivpu/ivpu_hw_37xx.c
+CONFLICT (content): Merge conflict in drivers/accel/ivpu/ivpu_hw_37xx.c
+Recorded preimage for 'drivers/accel/ivpu/ivpu_hw_37xx.c'
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+Recorded resolution for 'drivers/accel/ivpu/ivpu_hw_37xx.c'.
+[master 4d57d41bd8b9] Merge branch 'drm-next' of git://git.freedesktop.org/git/drm/drm.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/accel/qaic/aic100.rst                |   11 +-
+ Documentation/accel/qaic/qaic.rst                  |   28 +
+ .../devicetree/bindings/gpu/brcm,bcm-v3d.yaml      |    1 +
+ Documentation/gpu/drm-kms-helpers.rst              |    6 +
+ Documentation/gpu/drm-mm.rst                       |    6 +
+ Documentation/gpu/todo.rst                         |   17 +
+ MAINTAINERS                                        |    9 +-
+ drivers/accel/ivpu/Kconfig                         |   11 +-
+ drivers/accel/ivpu/ivpu_debugfs.c                  |   57 +
+ drivers/accel/ivpu/ivpu_drv.c                      |   49 +-
+ drivers/accel/ivpu/ivpu_drv.h                      |   18 +-
+ drivers/accel/ivpu/ivpu_fw.c                       |   79 +-
+ drivers/accel/ivpu/ivpu_fw.h                       |    1 +
+ drivers/accel/ivpu/ivpu_gem.c                      |  692 +++----
+ drivers/accel/ivpu/ivpu_gem.h                      |   75 +-
+ drivers/accel/ivpu/ivpu_hw.h                       |   20 +
+ drivers/accel/ivpu/ivpu_hw_37xx.c                  |   67 +-
+ drivers/accel/ivpu/ivpu_hw_37xx_reg.h              |    2 +
+ drivers/accel/ivpu/ivpu_hw_40xx.c                  |   69 +-
+ drivers/accel/ivpu/ivpu_ipc.c                      |  251 ++-
+ drivers/accel/ivpu/ivpu_ipc.h                      |   33 +-
+ drivers/accel/ivpu/ivpu_job.c                      |  103 +-
+ drivers/accel/ivpu/ivpu_job.h                      |    4 +-
+ drivers/accel/ivpu/ivpu_jsm_msg.c                  |   38 +
+ drivers/accel/ivpu/ivpu_jsm_msg.h                  |    1 +
+ drivers/accel/ivpu/ivpu_mmu.c                      |   44 +-
+ drivers/accel/ivpu/ivpu_mmu_context.c              |  153 +-
+ drivers/accel/ivpu/ivpu_mmu_context.h              |   11 +-
+ drivers/accel/ivpu/ivpu_pm.c                       |   72 +-
+ drivers/accel/ivpu/ivpu_pm.h                       |    3 +
+ drivers/accel/ivpu/vpu_boot_api.h                  |   90 +-
+ drivers/accel/ivpu/vpu_jsm_api.h                   |  309 +++-
+ drivers/accel/qaic/Makefile                        |    3 +-
+ drivers/accel/qaic/mhi_controller.c                |   42 +-
+ drivers/accel/qaic/mhi_controller.h                |    2 +-
+ drivers/accel/qaic/qaic.h                          |    6 +
+ drivers/accel/qaic/qaic_control.c                  |    2 +-
+ drivers/accel/qaic/qaic_data.c                     |  129 +-
+ drivers/accel/qaic/qaic_drv.c                      |   48 +-
+ drivers/accel/qaic/qaic_timesync.c                 |  395 ++++
+ drivers/accel/qaic/qaic_timesync.h                 |   11 +
+ drivers/gpu/drm/Kconfig                            |   12 +-
+ drivers/gpu/drm/Makefile                           |    1 +
+ .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c    |    2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c        |   15 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_device.c         |   14 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_job.c            |    2 +-
+ drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c  |    1 +
+ drivers/gpu/drm/drm_atomic_helper.c                |    4 +-
+ drivers/gpu/drm/drm_client.c                       |   12 +-
+ drivers/gpu/drm/drm_connector.c                    |    6 +
+ drivers/gpu/drm/drm_crtc_internal.h                |    2 +
+ drivers/gpu/drm/drm_edid.c                         |   43 +-
+ drivers/gpu/drm/drm_eld.c                          |   55 +
+ drivers/gpu/drm/drm_file.c                         |    2 +-
+ drivers/gpu/drm/drm_flip_work.c                    |   27 +-
+ drivers/gpu/drm/drm_format_helper.c                |  215 ++-
+ drivers/gpu/drm/drm_framebuffer.c                  |   75 +-
+ drivers/gpu/drm/drm_gem_atomic_helper.c            |    9 +
+ drivers/gpu/drm/drm_gpuvm.c                        | 1131 +++++++++++-
+ drivers/gpu/drm/drm_internal.h                     |    6 +
+ drivers/gpu/drm/drm_ioctl.c                        |    1 +
+ drivers/gpu/drm/drm_mipi_dbi.c                     |   19 +-
+ drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c       |    2 +-
+ drivers/gpu/drm/etnaviv/etnaviv_gpu.c              |    2 +-
+ drivers/gpu/drm/etnaviv/etnaviv_sched.c            |    2 +-
+ drivers/gpu/drm/gud/gud_pipe.c                     |   30 +-
+ drivers/gpu/drm/i915/display/intel_audio.c         |    1 +
+ .../gpu/drm/i915/display/intel_crtc_state_dump.c   |    1 +
+ drivers/gpu/drm/i915/display/intel_sdvo.c          |    1 +
+ drivers/gpu/drm/lima/lima_device.c                 |    2 +-
+ drivers/gpu/drm/lima/lima_sched.c                  |    4 +-
+ drivers/gpu/drm/msm/adreno/adreno_device.c         |    6 +-
+ drivers/gpu/drm/msm/msm_gem_submit.c               |    2 +-
+ drivers/gpu/drm/msm/msm_ringbuffer.c               |    2 +-
+ drivers/gpu/drm/nouveau/dispnv50/disp.c            |    1 +
+ drivers/gpu/drm/nouveau/nouveau_bo.c               |   11 +-
+ drivers/gpu/drm/nouveau/nouveau_bo.h               |    5 +
+ drivers/gpu/drm/nouveau/nouveau_drm.c              |    5 +-
+ drivers/gpu/drm/nouveau/nouveau_drv.h              |   10 +-
+ drivers/gpu/drm/nouveau/nouveau_gem.c              |   10 +-
+ drivers/gpu/drm/nouveau/nouveau_sched.c            |    4 +-
+ drivers/gpu/drm/nouveau/nouveau_uvmm.c             |  174 +-
+ drivers/gpu/drm/nouveau/nouveau_uvmm.h             |    8 -
+ drivers/gpu/drm/omapdrm/omap_drv.c                 |    9 +-
+ drivers/gpu/drm/panel/panel-edp.c                  |   58 +-
+ drivers/gpu/drm/panel/panel-novatek-nt35510.c      |    2 +-
+ drivers/gpu/drm/panfrost/panfrost_device.c         |   78 +-
+ drivers/gpu/drm/panfrost/panfrost_device.h         |   13 +
+ drivers/gpu/drm/panfrost/panfrost_drv.c            |    5 +-
+ drivers/gpu/drm/panfrost/panfrost_dump.c           |   12 +-
+ drivers/gpu/drm/panfrost/panfrost_gpu.c            |   85 +-
+ drivers/gpu/drm/panfrost/panfrost_job.c            |    4 +-
+ drivers/gpu/drm/panfrost/panfrost_regs.h           |    1 +
+ drivers/gpu/drm/radeon/radeon_audio.c              |    1 +
+ drivers/gpu/drm/scheduler/gpu_scheduler_trace.h    |    2 +-
+ drivers/gpu/drm/scheduler/sched_entity.c           |    4 +-
+ drivers/gpu/drm/scheduler/sched_main.c             |  486 +++--
+ drivers/gpu/drm/solomon/ssd130x.c                  |   38 +-
+ drivers/gpu/drm/tegra/hdmi.c                       |    1 +
+ drivers/gpu/drm/tegra/sor.c                        |    1 +
+ drivers/gpu/drm/tests/Makefile                     |    4 +-
+ drivers/gpu/drm/tests/drm_buddy_test.c             |  465 -----
+ drivers/gpu/drm/tests/drm_format_helper_test.c     |   72 +-
+ drivers/gpu/drm/tests/drm_mm_test.c                | 1904 --------------------
+ drivers/gpu/drm/tidss/tidss_kms.c                  |    4 -
+ drivers/gpu/drm/tiny/cirrus.c                      |    3 +-
+ drivers/gpu/drm/tiny/ili9225.c                     |   10 +-
+ drivers/gpu/drm/tiny/ofdrm.c                       |   16 +-
+ drivers/gpu/drm/tiny/repaper.c                     |   10 +-
+ drivers/gpu/drm/tiny/simpledrm.c                   |   43 +-
+ drivers/gpu/drm/tiny/st7586.c                      |   19 +-
+ drivers/gpu/drm/v3d/Makefile                       |    3 +-
+ drivers/gpu/drm/v3d/v3d_debugfs.c                  |  170 +-
+ drivers/gpu/drm/v3d/v3d_drv.c                      |   46 +-
+ drivers/gpu/drm/v3d/v3d_drv.h                      |   31 +
+ drivers/gpu/drm/v3d/v3d_gem.c                      |   15 +-
+ drivers/gpu/drm/v3d/v3d_irq.c                      |   93 +-
+ drivers/gpu/drm/v3d/v3d_regs.h                     |   90 +-
+ drivers/gpu/drm/v3d/v3d_sched.c                    |   81 +-
+ drivers/gpu/drm/v3d/v3d_sysfs.c                    |   69 +
+ drivers/gpu/drm/virtio/virtgpu_drv.h               |    5 +
+ drivers/gpu/drm/virtio/virtgpu_ioctl.c             |   41 +-
+ drivers/video/fbdev/simplefb.c                     |  128 +-
+ include/drm/drm_edid.h                             |  148 --
+ include/drm/drm_eld.h                              |  164 ++
+ include/drm/drm_flip_work.h                        |   20 +-
+ include/drm/drm_format_helper.h                    |   81 +-
+ include/drm/drm_gem.h                              |   32 +-
+ include/drm/drm_gem_atomic_helper.h                |   10 +
+ include/drm/drm_gpuvm.h                            |  521 +++++-
+ include/drm/drm_mipi_dbi.h                         |    4 +-
+ include/drm/gpu_scheduler.h                        |   50 +-
+ include/linux/iosys-map.h                          |   44 +-
+ include/uapi/drm/drm.h                             |   20 +
+ include/uapi/drm/drm_mode.h                        |   10 +
+ include/uapi/drm/ivpu_accel.h                      |    2 +-
+ include/uapi/drm/qaic_accel.h                      |    5 +-
+ include/uapi/drm/v3d_drm.h                         |    5 +
+ include/uapi/drm/virtgpu_drm.h                     |    2 +
+ sound/core/pcm_drm_eld.c                           |    1 +
+ sound/soc/codecs/hdac_hdmi.c                       |    1 +
+ sound/soc/codecs/hdmi-codec.c                      |    1 +
+ sound/x86/intel_hdmi_audio.c                       |    1 +
+ 144 files changed, 5773 insertions(+), 4376 deletions(-)
+ create mode 100644 drivers/accel/qaic/qaic_timesync.c
+ create mode 100644 drivers/accel/qaic/qaic_timesync.h
+ create mode 100644 drivers/gpu/drm/drm_eld.c
+ create mode 100644 drivers/gpu/drm/v3d/v3d_sysfs.c
+ create mode 100644 include/drm/drm_eld.h
+Merging drm-ci/topic/drm-ci (ad6bfe1b66a5 drm: ci: docs: fix build warning - add missing escape)
+$ git merge -m Merge branch 'topic/drm-ci' of git://git.freedesktop.org/git/drm/drm.git drm-ci/topic/drm-ci
+Already up to date.
+Merging drm-misc/for-linux-next (a4dea9a06f72 drm/edid/firmware: drop drm_kms_helper.edid_firmware backward compat)
+$ git merge -m Merge branch 'for-linux-next' of git://anongit.freedesktop.org/drm/drm-misc drm-misc/for-linux-next
+Merge made by the 'ort' strategy.
+ .../bindings/display/panel/newvision,nv3051d.yaml  |  2 +-
+ drivers/accel/qaic/qaic_data.c                     |  2 +-
+ drivers/dma-buf/dma-buf.c                          |  4 +-
+ drivers/gpu/drm/armada/armada_crtc.c               |  5 +-
+ drivers/gpu/drm/armada/armada_drv.c                |  5 +-
+ .../gpu/drm/bridge/cadence/cdns-mhdp8546-core.c    | 22 ++++-----
+ drivers/gpu/drm/bridge/ti-tpd12s015.c              |  6 +--
+ drivers/gpu/drm/drm_edid_load.c                    | 16 ------
+ drivers/gpu/drm/drm_kms_helper_common.c            | 32 ------------
+ drivers/gpu/drm/etnaviv/etnaviv_drv.c              |  6 +--
+ drivers/gpu/drm/etnaviv/etnaviv_gpu.c              |  5 +-
+ drivers/gpu/drm/imx/dcss/dcss-drv.c                |  6 +--
+ drivers/gpu/drm/imx/lcdc/imx-lcdc.c                |  6 +--
+ drivers/gpu/drm/kmb/kmb_drv.c                      |  5 +-
+ drivers/gpu/drm/mediatek/mtk_disp_ovl_adaptor.c    |  5 +-
+ drivers/gpu/drm/mediatek/mtk_ethdr.c               |  5 +-
+ drivers/gpu/drm/meson/meson_dw_mipi_dsi.c          |  6 +--
+ drivers/gpu/drm/nouveau/nouveau_platform.c         |  5 +-
+ drivers/gpu/drm/nouveau/nv04_fence.c               |  2 +-
+ drivers/gpu/drm/panel/panel-elida-kd35t133.c       | 37 ++------------
+ drivers/gpu/drm/panel/panel-newvision-nv3051d.c    | 57 +++++++++++++++++-----
+ drivers/gpu/drm/rockchip/rk3066_hdmi.c             | 46 +++++++----------
+ drivers/gpu/drm/rockchip/rockchip_drm_vop2.c       |  5 ++
+ drivers/gpu/drm/rockchip/rockchip_vop2_reg.c       |  2 +
+ drivers/gpu/drm/sprd/sprd_dpu.c                    |  6 +--
+ drivers/gpu/drm/sprd/sprd_drm.c                    |  5 +-
+ drivers/gpu/drm/sprd/sprd_dsi.c                    |  6 +--
+ drivers/gpu/drm/tilcdc/tilcdc_drv.c                |  2 +-
+ drivers/gpu/drm/tiny/arcpgu.c                      |  6 +--
+ include/drm/drm_edid.h                             |  5 --
+ include/drm/drm_print.h                            |  2 +-
+ 31 files changed, 123 insertions(+), 201 deletions(-)
+Merging amdgpu/drm-next (699d392903c3 drm/amdgpu: Add function parameter 'xcc_mask' not described in 'amdgpu_vm_flush_compute_tlb')
+$ git merge -m Merge branch 'drm-next' of https://gitlab.freedesktop.org/agd5f/linux amdgpu/drm-next
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+Merge made by the 'ort' strategy.
+ Documentation/gpu/amdgpu/display/dc-debug.rst      |  41 ++
+ .../gpu/amdgpu/display/trace-groups-table.csv      |  29 +
+ drivers/gpu/drm/amd/amdgpu/Makefile                |   2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu.h                |   5 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c         |  29 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h         |   5 -
+ .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c    |   2 +-
+ .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c    |   2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c  |   4 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c  |   4 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c  |   4 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c   |  28 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c     |  69 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_device.c         |  52 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c        |   4 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c          |   2 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c            |   2 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c          | 247 ++++++++
+ drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.h          |  49 ++
+ drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h          |  15 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c             |  45 ++
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h             |   5 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c           |  44 ++
+ drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h     | 692 ++++++++++-----------
+ .../gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm |   6 +
+ drivers/gpu/drm/amd/amdkfd/kfd_priv.h              |  10 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_process.c           |  31 -
+ drivers/gpu/drm/amd/display/Makefile               |   1 +
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c  | 107 +++-
+ .../amd/display/amdgpu_dm/amdgpu_dm_mst_types.c    |  46 +-
+ drivers/gpu/drm/amd/display/dc/Makefile            |   5 +-
+ .../drm/amd/display/dc/clk_mgr/dcn35/dcn35_smu.c   |   3 +
+ drivers/gpu/drm/amd/display/dc/core/dc_resource.c  |  47 +-
+ drivers/gpu/drm/amd/display/dc/dc.h                |   3 +-
+ drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c       |  29 +-
+ drivers/gpu/drm/amd/display/dc/dce100/Makefile     |  46 --
+ drivers/gpu/drm/amd/display/dc/dce110/Makefile     |   4 +-
+ drivers/gpu/drm/amd/display/dc/dce112/Makefile     |   3 +-
+ drivers/gpu/drm/amd/display/dc/dce120/Makefile     |   2 +-
+ drivers/gpu/drm/amd/display/dc/dce80/Makefile      |   3 +-
+ drivers/gpu/drm/amd/display/dc/dcn10/Makefile      |   2 +-
+ drivers/gpu/drm/amd/display/dc/dcn20/Makefile      |   2 +-
+ drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.h  |   6 +-
+ drivers/gpu/drm/amd/display/dc/dcn201/Makefile     |   2 +-
+ drivers/gpu/drm/amd/display/dc/dcn21/Makefile      |   2 +-
+ drivers/gpu/drm/amd/display/dc/dcn30/Makefile      |   1 -
+ drivers/gpu/drm/amd/display/dc/dcn301/Makefile     |   2 +-
+ drivers/gpu/drm/amd/display/dc/dcn302/Makefile     |   2 +-
+ drivers/gpu/drm/amd/display/dc/dcn303/Makefile     |   2 +-
+ drivers/gpu/drm/amd/display/dc/dcn31/Makefile      |   2 +-
+ drivers/gpu/drm/amd/display/dc/dcn314/Makefile     |   2 +-
+ drivers/gpu/drm/amd/display/dc/dcn315/Makefile     |  30 -
+ drivers/gpu/drm/amd/display/dc/dcn316/Makefile     |  30 -
+ drivers/gpu/drm/amd/display/dc/dcn32/Makefile      |   2 +-
+ drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c   |  13 +-
+ .../amd/display/dc/dcn32/dcn32_resource_helpers.c  |   2 +-
+ drivers/gpu/drm/amd/display/dc/dcn321/Makefile     |   2 +-
+ drivers/gpu/drm/amd/display/dc/dcn35/Makefile      |   2 +-
+ drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dccg.c  |  30 +
+ drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dccg.h  |   7 +-
+ .../gpu/drm/amd/display/dc/dcn35/dcn35_pg_cntl.c   |  10 +-
+ .../gpu/drm/amd/display/dc/dcn35/dcn35_pg_cntl.h   |   1 -
+ .../gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c   |   2 +-
+ .../gpu/drm/amd/display/dc/hwss/dce/dce_hwseq.h    |  15 +-
+ .../drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c    |  40 +-
+ drivers/gpu/drm/amd/display/dc/inc/hw/pg_cntl.h    |   2 -
+ drivers/gpu/drm/amd/display/dc/inc/resource.h      |  12 +
+ drivers/gpu/drm/amd/display/dc/resource/Makefile   | 199 ++++++
+ .../dc/{ => resource}/dce100/dce100_resource.c     |   0
+ .../dc/{ => resource}/dce100/dce100_resource.h     |   0
+ .../dc/{ => resource}/dce110/dce110_resource.c     |   0
+ .../dc/{ => resource}/dce110/dce110_resource.h     |   0
+ .../dc/{ => resource}/dce112/dce112_resource.c     |   0
+ .../dc/{ => resource}/dce112/dce112_resource.h     |   0
+ .../dc/{ => resource}/dce120/dce120_resource.c     |   2 +-
+ .../dc/{ => resource}/dce120/dce120_resource.h     |   0
+ .../amd/display/dc/resource/dce80/CMakeLists.txt   |   4 +
+ .../dc/{ => resource}/dce80/dce80_resource.c       |   0
+ .../dc/{ => resource}/dce80/dce80_resource.h       |   0
+ .../dc/{ => resource}/dcn10/dcn10_resource.c       |  25 +-
+ .../dc/{ => resource}/dcn10/dcn10_resource.h       |   0
+ .../dc/{ => resource}/dcn20/dcn20_resource.c       |  24 +-
+ .../dc/{ => resource}/dcn20/dcn20_resource.h       |   0
+ .../dc/{ => resource}/dcn201/dcn201_resource.c     |  14 +-
+ .../dc/{ => resource}/dcn201/dcn201_resource.h     |   0
+ .../dc/{ => resource}/dcn21/dcn21_resource.c       |   6 +-
+ .../dc/{ => resource}/dcn21/dcn21_resource.h       |   0
+ .../dc/{ => resource}/dcn30/dcn30_resource.c       |   2 +-
+ .../dc/{ => resource}/dcn30/dcn30_resource.h       |   0
+ .../dc/{ => resource}/dcn301/dcn301_resource.c     |   4 +-
+ .../dc/{ => resource}/dcn301/dcn301_resource.h     |   0
+ .../dc/{ => resource}/dcn302/dcn302_resource.c     |   4 +-
+ .../dc/{ => resource}/dcn302/dcn302_resource.h     |   0
+ .../dc/{ => resource}/dcn303/dcn303_resource.c     |   4 +-
+ .../dc/{ => resource}/dcn303/dcn303_resource.h     |   0
+ .../dc/{ => resource}/dcn31/dcn31_resource.c       |   2 +-
+ .../dc/{ => resource}/dcn31/dcn31_resource.h       |   0
+ .../dc/{ => resource}/dcn314/dcn314_resource.c     |   0
+ .../dc/{ => resource}/dcn314/dcn314_resource.h     |   0
+ .../dc/{ => resource}/dcn315/dcn315_resource.c     |   0
+ .../dc/{ => resource}/dcn315/dcn315_resource.h     |   0
+ .../dc/{ => resource}/dcn316/dcn316_resource.c     |   0
+ .../dc/{ => resource}/dcn316/dcn316_resource.h     |   0
+ .../dc/{ => resource}/dcn32/dcn32_resource.c       |   4 +-
+ .../dc/{ => resource}/dcn32/dcn32_resource.h       |   0
+ .../dc/{ => resource}/dcn321/dcn321_resource.c     |   2 +-
+ .../dc/{ => resource}/dcn321/dcn321_resource.h     |   0
+ .../dc/{ => resource}/dcn35/dcn35_resource.c       |  20 +-
+ .../dc/{ => resource}/dcn35/dcn35_resource.h       |   1 +
+ drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h    |  81 ++-
+ .../gpu/drm/amd/display/include/hdcp_msg_types.h   |   5 +
+ .../drm/amd/display/modules/freesync/freesync.c    |   6 +-
+ .../amd/include/asic_reg/dcn/dcn_3_5_0_sh_mask.h   |   8 +
+ drivers/gpu/drm/amd/include/kgd_pp_interface.h     |  17 +-
+ drivers/gpu/drm/amd/pm/amdgpu_dpm.c                |   8 +-
+ drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c   |  11 +-
+ drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.c |   7 +-
+ drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.h |   2 +-
+ drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.c |   9 +-
+ drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.h |   2 +-
+ .../gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.c   |   9 +-
+ .../gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.h   |   2 +-
+ drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h       |   2 +-
+ drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c          |  12 +-
+ drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h       |   3 -
+ drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c     |  40 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c   |  36 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c   |  18 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c   |  36 +-
+ drivers/gpu/drm/radeon/clearstate_evergreen.h      |   8 +-
+ 130 files changed, 1755 insertions(+), 836 deletions(-)
+ create mode 100644 Documentation/gpu/amdgpu/display/trace-groups-table.csv
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.h
+ delete mode 100644 drivers/gpu/drm/amd/display/dc/dce100/Makefile
+ delete mode 100644 drivers/gpu/drm/amd/display/dc/dcn315/Makefile
+ delete mode 100644 drivers/gpu/drm/amd/display/dc/dcn316/Makefile
+ create mode 100644 drivers/gpu/drm/amd/display/dc/resource/Makefile
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dce100/dce100_resource.c (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dce100/dce100_resource.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dce110/dce110_resource.c (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dce110/dce110_resource.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dce112/dce112_resource.c (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dce112/dce112_resource.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dce120/dce120_resource.c (99%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dce120/dce120_resource.h (100%)
+ create mode 100644 drivers/gpu/drm/amd/display/dc/resource/dce80/CMakeLists.txt
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dce80/dce80_resource.c (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dce80/dce80_resource.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn10/dcn10_resource.c (99%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn10/dcn10_resource.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn20/dcn20_resource.c (99%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn20/dcn20_resource.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn201/dcn201_resource.c (99%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn201/dcn201_resource.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn21/dcn21_resource.c (99%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn21/dcn21_resource.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn30/dcn30_resource.c (99%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn30/dcn30_resource.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn301/dcn301_resource.c (99%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn301/dcn301_resource.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn302/dcn302_resource.c (99%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn302/dcn302_resource.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn303/dcn303_resource.c (99%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn303/dcn303_resource.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn31/dcn31_resource.c (99%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn31/dcn31_resource.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn314/dcn314_resource.c (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn314/dcn314_resource.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn315/dcn315_resource.c (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn315/dcn315_resource.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn316/dcn316_resource.c (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn316/dcn316_resource.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn32/dcn32_resource.c (99%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn32/dcn32_resource.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn321/dcn321_resource.c (99%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn321/dcn321_resource.h (100%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn35/dcn35_resource.c (98%)
+ rename drivers/gpu/drm/amd/display/dc/{ => resource}/dcn35/dcn35_resource.h (99%)
+Merging drm-intel/for-linux-next (297c76d94c89 drm/i915/dp: Reuse intel_dp_{max,effective}_data_rate in intel_link_compute_m_n())
+$ git merge -m Merge branch 'for-linux-next' of git://anongit.freedesktop.org/drm-intel drm-intel/for-linux-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+Auto-merging drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
+CONFLICT (content): Merge conflict in drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
+Auto-merging drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
+CONFLICT (content): Merge conflict in drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
+Auto-merging drivers/gpu/drm/display/drm_dp_helper.c
+Auto-merging drivers/gpu/drm/display/drm_dp_mst_topology.c
+Auto-merging drivers/gpu/drm/i915/display/intel_audio.c
+Auto-merging drivers/gpu/drm/i915/display/intel_cdclk.c
+Auto-merging drivers/gpu/drm/i915/display/intel_display_core.h
+Auto-merging drivers/gpu/drm/i915/display/intel_display_debugfs.c
+Auto-merging drivers/gpu/drm/i915/display/intel_dp.c
+Auto-merging drivers/gpu/drm/i915/display/intel_dp_mst.c
+CONFLICT (content): Merge conflict in drivers/gpu/drm/i915/display/intel_dp_mst.c
+Auto-merging drivers/gpu/drm/i915/display/intel_fb.c
+Auto-merging drivers/gpu/drm/i915/gt/intel_gt.h
+Auto-merging drivers/gpu/drm/i915/i915_driver.c
+Auto-merging drivers/gpu/drm/i915/i915_drv.h
+Auto-merging drivers/gpu/drm/i915/i915_gem.c
+Auto-merging drivers/gpu/drm/i915/i915_gpu_error.c
+Auto-merging drivers/gpu/drm/nouveau/dispnv50/disp.c
+Auto-merging include/drm/display/drm_dp_helper.h
+Recorded preimage for 'drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c'
+Resolved 'drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c' using previous resolution.
+Resolved 'drivers/gpu/drm/i915/display/intel_dp_mst.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+Recorded resolution for 'drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c'.
+[master 94aa4a27a805] Merge branch 'for-linux-next' of git://anongit.freedesktop.org/drm-intel
+$ git diff -M --stat --summary HEAD^..
+ MAINTAINERS                                        |   4 +-
+ drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c  |   7 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c  |   3 +-
+ .../amd/display/amdgpu_dm/amdgpu_dm_mst_types.c    |   7 +-
+ drivers/gpu/drm/display/drm_dp_helper.c            | 161 +++++
+ drivers/gpu/drm/display/drm_dp_mst_topology.c      | 234 ++++++--
+ drivers/gpu/drm/i915/Kconfig                       |   2 +-
+ drivers/gpu/drm/i915/Makefile                      | 179 +++---
+ drivers/gpu/drm/i915/display/hsw_ips.c             |   4 +-
+ drivers/gpu/drm/i915/display/i9xx_wm.c             |   2 +-
+ drivers/gpu/drm/i915/display/icl_dsi.c             |  10 +-
+ drivers/gpu/drm/i915/display/intel_atomic.c        |   3 -
+ drivers/gpu/drm/i915/display/intel_atomic_plane.c  |  83 +--
+ drivers/gpu/drm/i915/display/intel_audio.c         |  16 +-
+ drivers/gpu/drm/i915/display/intel_backlight.c     |   9 +-
+ drivers/gpu/drm/i915/display/intel_bios.c          |  32 +-
+ drivers/gpu/drm/i915/display/intel_cdclk.c         |   5 +-
+ drivers/gpu/drm/i915/display/intel_crt.c           |   4 +-
+ drivers/gpu/drm/i915/display/intel_cx0_phy.c       | 160 +++--
+ drivers/gpu/drm/i915/display/intel_cx0_phy.h       |  16 +-
+ drivers/gpu/drm/i915/display/intel_ddi.c           | 166 ++++--
+ drivers/gpu/drm/i915/display/intel_ddi.h           |   5 +
+ drivers/gpu/drm/i915/display/intel_display.c       | 431 ++++----------
+ drivers/gpu/drm/i915/display/intel_display.h       |   9 +-
+ drivers/gpu/drm/i915/display/intel_display_core.h  |  20 +-
+ .../gpu/drm/i915/display/intel_display_debugfs.c   |  98 +++
+ .../i915/display/intel_display_debugfs_params.c    | 176 ++++++
+ .../i915/display/intel_display_debugfs_params.h    |  13 +
+ .../gpu/drm/i915/display/intel_display_device.c    |  13 +-
+ .../gpu/drm/i915/display/intel_display_device.h    |   1 +
+ .../gpu/drm/i915/display/intel_display_driver.c    |   7 +
+ .../gpu/drm/i915/display/intel_display_params.c    | 217 +++++++
+ .../gpu/drm/i915/display/intel_display_params.h    |  61 ++
+ drivers/gpu/drm/i915/display/intel_display_power.c |  14 +-
+ .../drm/i915/display/intel_display_power_well.c    |  23 +-
+ drivers/gpu/drm/i915/display/intel_display_reset.c |   2 +-
+ drivers/gpu/drm/i915/display/intel_display_types.h |  25 +-
+ drivers/gpu/drm/i915/display/intel_dp.c            | 502 ++++++++++++----
+ drivers/gpu/drm/i915/display/intel_dp.h            |  26 +-
+ drivers/gpu/drm/i915/display/intel_dp_aux.c        |  99 ++--
+ .../gpu/drm/i915/display/intel_dp_aux_backlight.c  |   4 +-
+ drivers/gpu/drm/i915/display/intel_dp_aux_regs.h   |  14 +-
+ drivers/gpu/drm/i915/display/intel_dp_mst.c        | 656 +++++++++++++++++----
+ drivers/gpu/drm/i915/display/intel_dp_mst.h        |   5 +
+ drivers/gpu/drm/i915/display/intel_dpio_phy.c      | 171 +++---
+ drivers/gpu/drm/i915/display/intel_dpio_phy.h      |   5 +
+ drivers/gpu/drm/i915/display/intel_dpll.c          | 270 +++++++--
+ drivers/gpu/drm/i915/display/intel_dpll.h          |   9 +-
+ drivers/gpu/drm/i915/display/intel_dpll_mgr.c      | 187 +++---
+ drivers/gpu/drm/i915/display/intel_dpll_mgr.h      |   6 +
+ drivers/gpu/drm/i915/display/intel_dpt.c           |   6 +-
+ drivers/gpu/drm/i915/display/intel_dsb.c           |  98 ++-
+ drivers/gpu/drm/i915/display/intel_dsb_buffer.c    |  82 +++
+ drivers/gpu/drm/i915/display/intel_dsb_buffer.h    |  29 +
+ drivers/gpu/drm/i915/display/intel_fb.c            |   6 +-
+ drivers/gpu/drm/i915/display/intel_fbc.c           |  57 +-
+ drivers/gpu/drm/i915/display/intel_fdi.c           |   8 +-
+ drivers/gpu/drm/i915/display/intel_frontbuffer.c   |   2 -
+ drivers/gpu/drm/i915/display/intel_hdcp.c          |  37 +-
+ drivers/gpu/drm/i915/display/intel_hdcp.h          |   8 +-
+ drivers/gpu/drm/i915/display/intel_hdmi.c          |  10 -
+ drivers/gpu/drm/i915/display/intel_hotplug_irq.c   |  16 +
+ drivers/gpu/drm/i915/display/intel_link_bw.c       |  30 +-
+ drivers/gpu/drm/i915/display/intel_link_bw.h       |   1 +
+ drivers/gpu/drm/i915/display/intel_lvds.c          |   4 +-
+ drivers/gpu/drm/i915/display/intel_modeset_setup.c |   6 +
+ .../gpu/drm/i915/display/intel_modeset_verify.c    |   2 +-
+ drivers/gpu/drm/i915/display/intel_opregion.c      |   2 +-
+ drivers/gpu/drm/i915/display/intel_panel.c         |   4 +-
+ drivers/gpu/drm/i915/display/intel_pch_display.c   |   1 +
+ drivers/gpu/drm/i915/display/intel_pps.c           |   2 +-
+ drivers/gpu/drm/i915/display/intel_psr.c           | 372 +++++++++---
+ drivers/gpu/drm/i915/display/intel_psr.h           |   7 +
+ drivers/gpu/drm/i915/display/intel_psr_regs.h      |   2 +
+ drivers/gpu/drm/i915/display/intel_qp_tables.c     |   3 -
+ drivers/gpu/drm/i915/display/intel_sprite.c        |   7 +-
+ drivers/gpu/drm/i915/display/intel_vdsc.c          |  29 +-
+ drivers/gpu/drm/i915/display/skl_universal_plane.c |  25 +-
+ drivers/gpu/drm/i915/display/skl_watermark.c       |   5 +-
+ drivers/gpu/drm/i915/display/vlv_dsi.c             |  21 +-
+ .../gpu/drm/i915/gem/i915_gem_object_frontbuffer.h |   1 +
+ drivers/gpu/drm/i915/gt/intel_engine_pm.h          |   1 +
+ drivers/gpu/drm/i915/gt/intel_gt.h                 |  14 +
+ drivers/gpu/drm/i915/gt/intel_gt_engines_debugfs.c |   2 +-
+ drivers/gpu/drm/i915/gvt/cmd_parser.c              |   2 +-
+ drivers/gpu/drm/i915/gvt/fb_decoder.c              |   6 +-
+ drivers/gpu/drm/i915/gvt/handlers.c                |   3 +-
+ drivers/gpu/drm/i915/i915_debugfs.c                |   4 +-
+ drivers/gpu/drm/i915/i915_driver.c                 |   8 +-
+ drivers/gpu/drm/i915/i915_drv.h                    |  20 +-
+ drivers/gpu/drm/i915/i915_gem.c                    |   2 -
+ drivers/gpu/drm/i915/i915_gpu_error.c              |   5 +-
+ drivers/gpu/drm/i915/i915_gpu_error.h              |   2 +
+ drivers/gpu/drm/i915/i915_params.c                 |  89 ---
+ drivers/gpu/drm/i915/i915_params.h                 |  22 -
+ drivers/gpu/drm/i915/i915_reg.h                    |   2 -
+ drivers/gpu/drm/i915/i915_utils.h                  |   2 +-
+ drivers/gpu/drm/i915/intel_runtime_pm.h            |   2 -
+ drivers/gpu/drm/i915/selftests/intel_uncore.c      |   2 +
+ drivers/gpu/drm/i915/soc/intel_gmch.c              |  27 +-
+ drivers/gpu/drm/i915/vlv_sideband.c                |  12 +-
+ drivers/gpu/drm/i915/vlv_sideband.h                |   6 +-
+ drivers/gpu/drm/nouveau/dispnv50/disp.c            |   9 +-
+ drivers/gpu/drm/tests/drm_dp_mst_helper_test.c     | 166 +++++-
+ include/drm/display/drm_dp.h                       |  25 +
+ include/drm/display/drm_dp_helper.h                |  32 +
+ include/drm/display/drm_dp_mst_helper.h            |  16 +-
+ 107 files changed, 3971 insertions(+), 1569 deletions(-)
+ create mode 100644 drivers/gpu/drm/i915/display/intel_display_debugfs_params.c
+ create mode 100644 drivers/gpu/drm/i915/display/intel_display_debugfs_params.h
+ create mode 100644 drivers/gpu/drm/i915/display/intel_display_params.c
+ create mode 100644 drivers/gpu/drm/i915/display/intel_display_params.h
+ create mode 100644 drivers/gpu/drm/i915/display/intel_dsb_buffer.c
+ create mode 100644 drivers/gpu/drm/i915/display/intel_dsb_buffer.h
+Merging drm-tegra/for-next (2429b3c529da drm/tegra: Avoid potential 32-bit integer overflow)
+$ git merge -m Merge branch 'for-next' of https://gitlab.freedesktop.org/drm/tegra.git drm-tegra/for-next
+Already up to date.
+Merging drm-msm/msm-next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'msm-next' of https://gitlab.freedesktop.org/drm/msm.git drm-msm/msm-next
+Already up to date.
+Merging drm-msm-lumag/msm-next-lumag (d3b4075b173f drm/msm/dp: use correct lifetime device for devm_drm_bridge_add)
+$ git merge -m Merge branch 'msm-next-lumag' of https://gitlab.freedesktop.org/lumag/msm.git drm-msm-lumag/msm-next-lumag
+Already up to date.
+Merging etnaviv/etnaviv/next (925b10728f20 drm/etnaviv: disable MLCG and pulse eater on GPU reset)
+$ git merge -m Merge branch 'etnaviv/next' of https://git.pengutronix.de/git/lst/linux etnaviv/etnaviv/next
+Auto-merging drivers/gpu/drm/etnaviv/etnaviv_drv.c
+Auto-merging drivers/gpu/drm/etnaviv/etnaviv_gpu.c
+Merge made by the 'ort' strategy.
+ drivers/gpu/drm/etnaviv/etnaviv_drv.c     | 95 ++++++++++++++++++++-----------
+ drivers/gpu/drm/etnaviv/etnaviv_gem.c     | 12 ++--
+ drivers/gpu/drm/etnaviv/etnaviv_gpu.c     | 13 ++++-
+ drivers/gpu/drm/etnaviv/etnaviv_mmu.c     |  4 +-
+ drivers/gpu/drm/etnaviv/etnaviv_perfmon.c |  4 +-
+ 5 files changed, 84 insertions(+), 44 deletions(-)
+Merging fbdev/for-next (8b01ee6ce9bf fbdev: imxfb: add '*/' on a separate line in block comment)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/deller/linux-fbdev.git fbdev/for-next
+Merge made by the 'ort' strategy.
+ drivers/video/fbdev/imxfb.c | 179 +++++++++++++++++++++++++-------------------
+ 1 file changed, 102 insertions(+), 77 deletions(-)
+Merging regmap/for-next (d958d97848a6 regmap: kunit: add noinc write test)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git regmap/for-next
+Merge made by the 'ort' strategy.
+ drivers/base/regmap/internal.h       |  1 +
+ drivers/base/regmap/regmap-kunit.c   | 60 ++++++++++++++++++++++++++++++++++++
+ drivers/base/regmap/regmap-raw-ram.c | 23 ++++++++++----
+ 3 files changed, 78 insertions(+), 6 deletions(-)
+Merging sound/for-next (5a77457232fa ALSA: wavefront: copy userspace array safely)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git sound/for-next
+Merge made by the 'ort' strategy.
+ sound/isa/wavefront/wavefront_fx.c | 6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+Merging ieee1394/for-next (c12d7aa7ffa4 firewire: Annotate struct fw_node with __counted_by)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ieee1394/linux1394.git ieee1394/for-next
+Already up to date.
+Merging sound-asoc/for-next (a2d01aefded1 Merge remote-tracking branch 'asoc/for-6.8' into asoc-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git sound-asoc/for-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/sound/adi,max98363.yaml    |   2 +-
+ Documentation/devicetree/bindings/sound/es8328.txt |  38 ----
+ .../devicetree/bindings/sound/everest,es8328.yaml  |  77 +++++++
+ .../devicetree/bindings/sound/nuvoton,nau8821.yaml |   9 +
+ .../devicetree/bindings/sound/qcom,sm8250.yaml     |   4 +
+ .../devicetree/bindings/sound/qcom,wsa883x.yaml    |   2 +-
+ .../devicetree/bindings/sound/qcom,wsa8840.yaml    |   2 +-
+ .../devicetree/bindings/sound/renesas,rsnd.yaml    |  28 ++-
+ .../devicetree/bindings/sound/renesas,rz-ssi.yaml  |   2 +-
+ .../bindings/sound/sound-card-common.yaml          |   7 +
+ Documentation/sound/soc/dapm.rst                   |   2 +-
+ include/sound/rt5682s.h                            |   8 +
+ include/sound/sof/dai-imx.h                        |   7 +
+ include/sound/sof/dai.h                            |   2 +
+ include/uapi/sound/sof/tokens.h                    |   4 +
+ sound/soc/amd/acp/Kconfig                          |  13 ++
+ sound/soc/amd/acp/Makefile                         |   2 +
+ sound/soc/amd/acp/acp63.c                          |  22 +-
+ sound/soc/amd/acp/acp70.c                          |  28 +--
+ sound/soc/codecs/aw88395/aw88395_device.h          |   1 +
+ sound/soc/codecs/aw88395/aw88395_lib.c             | 124 ++++-------
+ sound/soc/codecs/aw88395/aw88395_reg.h             |   3 -
+ sound/soc/codecs/aw88399.c                         |   1 -
+ sound/soc/codecs/aw88399.h                         |   1 +
+ sound/soc/codecs/cs42l43.c                         |   4 +-
+ sound/soc/codecs/cs43130.c                         | 209 ++++++++++---------
+ sound/soc/codecs/cs43130.h                         |   1 +
+ sound/soc/codecs/es8326.c                          | 229 +++++++++++++--------
+ sound/soc/codecs/nau8810.c                         |   9 +-
+ sound/soc/codecs/nau8821.c                         |   7 +
+ sound/soc/codecs/nau8821.h                         |   3 +
+ sound/soc/codecs/rt5682s.c                         |  23 +++
+ sound/soc/codecs/rt5682s.h                         |   7 +
+ sound/soc/fsl/fsl_mqs.c                            |   1 -
+ sound/soc/fsl/imx-rpmsg.c                          |  61 +++++-
+ sound/soc/intel/avs/boards/da7219.c                |  10 +-
+ sound/soc/intel/avs/boards/dmic.c                  |  10 +-
+ sound/soc/intel/avs/boards/es8336.c                |  10 +-
+ sound/soc/intel/avs/boards/hdaudio.c               |  10 +-
+ sound/soc/intel/avs/boards/i2s_test.c              |  10 +-
+ sound/soc/intel/avs/boards/max98357a.c             |  10 +-
+ sound/soc/intel/avs/boards/max98373.c              |  10 +-
+ sound/soc/intel/avs/boards/max98927.c              |  10 +-
+ sound/soc/intel/avs/boards/nau8825.c               |  10 +-
+ sound/soc/intel/avs/boards/probe.c                 |  10 +-
+ sound/soc/intel/avs/boards/rt274.c                 |  10 +-
+ sound/soc/intel/avs/boards/rt286.c                 |  10 +-
+ sound/soc/intel/avs/boards/rt298.c                 |  10 +-
+ sound/soc/intel/avs/boards/rt5514.c                |  10 +-
+ sound/soc/intel/avs/boards/rt5663.c                |  10 +-
+ sound/soc/intel/avs/boards/rt5682.c                |  10 +-
+ sound/soc/intel/avs/boards/ssm4567.c               |  10 +-
+ sound/soc/intel/boards/sof_rt5682.c                |   9 +
+ sound/soc/intel/common/soc-acpi-intel-mtl-match.c  |  12 ++
+ sound/soc/soc-dapm.c                               |  30 ++-
+ sound/soc/sof/imx/imx8m.c                          |   7 +
+ sound/soc/sof/ipc3-pcm.c                           |  11 +
+ sound/soc/sof/ipc3-topology.c                      |  46 +++++
+ sound/soc/sof/mediatek/adsp_helper.h               |   4 -
+ sound/soc/sof/mediatek/mt8186/mt8186.c             |  49 -----
+ sound/soc/sof/mediatek/mt8195/mt8195.c             |  49 -----
+ sound/soc/sof/sof-audio.h                          |   1 +
+ sound/soc/sof/topology.c                           |   5 +
+ sound/soc/sti/sti_uniperif.c                       |   7 +-
+ sound/soc/tegra/tegra_pcm.c                        |   4 +-
+ 65 files changed, 836 insertions(+), 511 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/sound/es8328.txt
+ create mode 100644 Documentation/devicetree/bindings/sound/everest,es8328.yaml
+ mode change 100644 => 100755 sound/soc/codecs/es8326.c
+Merging modules/modules-next (4652b8e4f3ff Merge tag '6.7-rc-ksmbd-server-fixes' of git://git.samba.org/ksmbd)
+$ git merge -m Merge branch 'modules-next' of git://git.kernel.org/pub/scm/linux/kernel/git/mcgrof/linux.git modules/modules-next
+Already up to date.
+Merging input/next (ccb9e9dd2a99 dt-bindings: input: samsung,s6sy761: convert to DT schema)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git input/next
+Merge made by the 'ort' strategy.
+ .../bindings/input/touchscreen/samsung,s6sy761.txt | 32 -------------
+ .../input/touchscreen/samsung,s6sy761.yaml         | 54 ++++++++++++++++++++++
+ 2 files changed, 54 insertions(+), 32 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/input/touchscreen/samsung,s6sy761.txt
+ create mode 100644 Documentation/devicetree/bindings/input/touchscreen/samsung,s6sy761.yaml
+Merging block/for-next (50e8a66babc4 Merge branch 'block-6.7' into for-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.dk/linux-block.git block/for-next
+Auto-merging drivers/md/bcache/btree.c
+Auto-merging drivers/md/bcache/super.c
+Auto-merging drivers/md/bcache/writeback.c
+Merge made by the 'ort' strategy.
+ block/blk-cgroup.c            |  13 +++++
+ block/blk-cgroup.h            |   2 -
+ block/blk-pm.c                |  33 ++----------
+ block/blk-throttle.c          |   2 +
+ drivers/block/nbd.c           | 117 +++++++++++++++++++++++++++---------------
+ drivers/block/null_blk/main.c |  25 ++++-----
+ drivers/md/bcache/bcache.h    |   1 +
+ drivers/md/bcache/btree.c     |  11 +++-
+ drivers/md/bcache/super.c     |   4 +-
+ drivers/md/bcache/sysfs.c     |   2 +-
+ drivers/md/bcache/writeback.c |  24 ++++++---
+ drivers/md/md.c               |   3 +-
+ drivers/s390/block/dasd.c     |  24 +++++----
+ drivers/s390/block/dasd_int.h |   2 +-
+ include/linux/blk-pm.h        |   1 -
+ io_uring/fs.c                 |   2 +-
+ io_uring/rsrc.c               |   2 +-
+ 17 files changed, 159 insertions(+), 109 deletions(-)
+Merging device-mapper/for-next (13648e04a9b8 dm-crypt: start allocating with MAX_ORDER)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git device-mapper/for-next
+Already up to date.
+Merging libata/for-next (804901fdd637 ata: pata_pxa: convert not to use dma_request_slave_channel())
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/dlemoal/libata.git libata/for-next
+Merge made by the 'ort' strategy.
+ drivers/ata/pata_pxa.c | 7 +++----
+ 1 file changed, 3 insertions(+), 4 deletions(-)
+Merging pcmcia/pcmcia-next (4f733de8b78a pcmcia: tcic: remove unneeded "&" in call to setup_timer())
+$ git merge -m Merge branch 'pcmcia-next' of git://git.kernel.org/pub/scm/linux/kernel/git/brodo/linux.git pcmcia/pcmcia-next
+Already up to date.
+Merging mmc/next (3f00051234f0 mmc: Merge branch fixes into next)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git mmc/next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/mmc/arasan,sdhci.yaml      |   8 +-
+ .../bindings/mmc/snps,dwcmshc-sdhci.yaml           |   1 +
+ drivers/mmc/core/mmc.c                             |   5 -
+ drivers/mmc/host/meson-mx-sdhc-mmc.c               |  26 +-
+ drivers/mmc/host/mmci.c                            |  69 +++-
+ drivers/mmc/host/mmci.h                            |   2 +
+ drivers/mmc/host/sdhci-of-dwcmshc.c                | 349 +++++++++++++++++++++
+ drivers/mmc/host/sdhci.c                           |   3 +-
+ drivers/mmc/host/sdhci.h                           |   1 +
+ include/linux/mmc/card.h                           |   3 -
+ include/linux/mmc/core.h                           |   1 -
+ include/linux/mmc/mmc.h                            |  10 -
+ 12 files changed, 431 insertions(+), 47 deletions(-)
+Merging mfd/for-mfd-next (2b481822446e mfd: lpc_ich: Mark *_gpio_offsets data with const)
+$ git merge -m Merge branch 'for-mfd-next' of git://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git mfd/for-mfd-next
+Already up to date.
+Merging backlight/for-backlight-next (d5272d39995f dt-bindings: backlight: Add brightness-levels related common properties)
+$ git merge -m Merge branch 'for-backlight-next' of git://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git backlight/for-backlight-next
+Already up to date.
+Merging battery/for-next (b55d073e6501 power: supply: bq256xx: fix some problem in bq256xx_hw_init)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git battery/for-next
+Merge made by the 'ort' strategy.
+ .../bindings/power/reset/nvmem-reboot-mode.yaml    |  8 +++-----
+ .../bindings/power/reset/syscon-reboot-mode.yaml   |  8 +++-----
+ drivers/power/reset/as3722-poweroff.c              |  6 ++----
+ drivers/power/reset/at91-poweroff.c                | 11 +++++------
+ drivers/power/reset/at91-reset.c                   | 11 +++++------
+ drivers/power/reset/at91-sama5d2_shdwc.c           | 11 +++++------
+ drivers/power/reset/atc260x-poweroff.c             |  6 ++----
+ drivers/power/reset/ltc2952-poweroff.c             |  5 ++---
+ drivers/power/reset/mt6323-poweroff.c              |  6 ++----
+ drivers/power/reset/pwr-mlxbf.c                    | 20 +++++++++++++++-----
+ drivers/power/reset/qnap-poweroff.c                |  5 ++---
+ drivers/power/reset/regulator-poweroff.c           |  6 ++----
+ drivers/power/reset/restart-poweroff.c             |  6 ++----
+ drivers/power/reset/rmobile-reset.c                |  5 ++---
+ drivers/power/reset/syscon-poweroff.c              |  6 ++----
+ drivers/power/reset/tps65086-restart.c             | 12 +++++++-----
+ drivers/power/supply/bq256xx_charger.c             |  5 ++++-
+ drivers/power/supply/bq27xxx_battery.c             | 22 ++++++++++++++++++++++
+ drivers/power/supply/bq27xxx_battery_i2c.c         |  1 +
+ include/linux/power/bq27xxx_battery.h              |  1 +
+ 20 files changed, 89 insertions(+), 72 deletions(-)
+Merging regulator/for-next (753e4d5c433d regulator: add under-voltage support (part 2))
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git regulator/for-next
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/kernel-parameters.txt    |   7 ++
+ .../bindings/regulator/fixed-regulator.yaml        |   2 +
+ .../bindings/regulator/qcom,smd-rpm-regulator.yaml |   4 +
+ .../bindings/regulator/qcom,spmi-regulator.yaml    |  19 ++++
+ .../devicetree/bindings/regulator/regulator.yaml   |  13 +++
+ drivers/regulator/core.c                           | 107 ++++++++++++++++-----
+ drivers/regulator/of_regulator.c                   |   9 ++
+ drivers/regulator/palmas-regulator.c               |   2 +-
+ drivers/regulator/qcom_smd-regulator.c             |  34 +++++++
+ drivers/regulator/qcom_spmi-regulator.c            |  34 +++++++
+ drivers/regulator/stpmic1_regulator.c              |   2 +-
+ include/linux/regulator/machine.h                  |  18 ++++
+ 12 files changed, 225 insertions(+), 26 deletions(-)
+Merging security/next (afe576a62062 MAINTAINERS: add an entry for the lockdown LSM)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/lsm.git security/next
+Auto-merging .mailmap
+Auto-merging MAINTAINERS
+Auto-merging arch/alpha/kernel/syscalls/syscall.tbl
+CONFLICT (content): Merge conflict in arch/alpha/kernel/syscalls/syscall.tbl
+Auto-merging arch/arm/tools/syscall.tbl
+CONFLICT (content): Merge conflict in arch/arm/tools/syscall.tbl
+Auto-merging arch/arm64/include/asm/unistd32.h
+CONFLICT (content): Merge conflict in arch/arm64/include/asm/unistd32.h
+Auto-merging arch/m68k/kernel/syscalls/syscall.tbl
+CONFLICT (content): Merge conflict in arch/m68k/kernel/syscalls/syscall.tbl
+Auto-merging arch/microblaze/kernel/syscalls/syscall.tbl
+CONFLICT (content): Merge conflict in arch/microblaze/kernel/syscalls/syscall.tbl
+Auto-merging arch/mips/kernel/syscalls/syscall_n32.tbl
+CONFLICT (content): Merge conflict in arch/mips/kernel/syscalls/syscall_n32.tbl
+Auto-merging arch/mips/kernel/syscalls/syscall_n64.tbl
+CONFLICT (content): Merge conflict in arch/mips/kernel/syscalls/syscall_n64.tbl
+Auto-merging arch/mips/kernel/syscalls/syscall_o32.tbl
+CONFLICT (content): Merge conflict in arch/mips/kernel/syscalls/syscall_o32.tbl
+Auto-merging arch/parisc/kernel/syscalls/syscall.tbl
+CONFLICT (content): Merge conflict in arch/parisc/kernel/syscalls/syscall.tbl
+Auto-merging arch/powerpc/kernel/syscalls/syscall.tbl
+CONFLICT (content): Merge conflict in arch/powerpc/kernel/syscalls/syscall.tbl
+Auto-merging arch/s390/kernel/syscalls/syscall.tbl
+CONFLICT (content): Merge conflict in arch/s390/kernel/syscalls/syscall.tbl
+Auto-merging arch/sh/kernel/syscalls/syscall.tbl
+CONFLICT (content): Merge conflict in arch/sh/kernel/syscalls/syscall.tbl
+Auto-merging arch/sparc/kernel/syscalls/syscall.tbl
+CONFLICT (content): Merge conflict in arch/sparc/kernel/syscalls/syscall.tbl
+Auto-merging arch/x86/entry/syscalls/syscall_32.tbl
+CONFLICT (content): Merge conflict in arch/x86/entry/syscalls/syscall_32.tbl
+Auto-merging arch/x86/entry/syscalls/syscall_64.tbl
+CONFLICT (content): Merge conflict in arch/x86/entry/syscalls/syscall_64.tbl
+Auto-merging arch/xtensa/kernel/syscalls/syscall.tbl
+CONFLICT (content): Merge conflict in arch/xtensa/kernel/syscalls/syscall.tbl
+Auto-merging include/linux/syscalls.h
+Auto-merging include/uapi/asm-generic/unistd.h
+CONFLICT (content): Merge conflict in include/uapi/asm-generic/unistd.h
+Resolved 'arch/alpha/kernel/syscalls/syscall.tbl' using previous resolution.
+Resolved 'arch/arm/tools/syscall.tbl' using previous resolution.
+Resolved 'arch/arm64/include/asm/unistd32.h' using previous resolution.
+Resolved 'arch/m68k/kernel/syscalls/syscall.tbl' using previous resolution.
+Resolved 'arch/microblaze/kernel/syscalls/syscall.tbl' using previous resolution.
+Resolved 'arch/mips/kernel/syscalls/syscall_n32.tbl' using previous resolution.
+Resolved 'arch/mips/kernel/syscalls/syscall_n64.tbl' using previous resolution.
+Resolved 'arch/mips/kernel/syscalls/syscall_o32.tbl' using previous resolution.
+Resolved 'arch/parisc/kernel/syscalls/syscall.tbl' using previous resolution.
+Resolved 'arch/powerpc/kernel/syscalls/syscall.tbl' using previous resolution.
+Resolved 'arch/s390/kernel/syscalls/syscall.tbl' using previous resolution.
+Resolved 'arch/sh/kernel/syscalls/syscall.tbl' using previous resolution.
+Resolved 'arch/sparc/kernel/syscalls/syscall.tbl' using previous resolution.
+Resolved 'arch/x86/entry/syscalls/syscall_32.tbl' using previous resolution.
+Resolved 'arch/x86/entry/syscalls/syscall_64.tbl' using previous resolution.
+Resolved 'arch/xtensa/kernel/syscalls/syscall.tbl' using previous resolution.
+Resolved 'include/uapi/asm-generic/unistd.h' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 217cb642310c] Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/lsm.git
+$ git diff -M --stat --summary HEAD^..
+ .mailmap                                           |   4 +
+ Documentation/userspace-api/index.rst              |   1 +
+ Documentation/userspace-api/lsm.rst                |  73 ++++++
+ MAINTAINERS                                        |  17 +-
+ arch/alpha/kernel/syscalls/syscall.tbl             |   3 +
+ arch/arm/tools/syscall.tbl                         |   3 +
+ arch/arm64/include/asm/unistd.h                    |   2 +-
+ arch/arm64/include/asm/unistd32.h                  |   6 +
+ arch/m68k/kernel/syscalls/syscall.tbl              |   3 +
+ arch/microblaze/kernel/syscalls/syscall.tbl        |   3 +
+ arch/mips/kernel/syscalls/syscall_n32.tbl          |   3 +
+ arch/mips/kernel/syscalls/syscall_n64.tbl          |   3 +
+ arch/mips/kernel/syscalls/syscall_o32.tbl          |   3 +
+ arch/parisc/kernel/syscalls/syscall.tbl            |   3 +
+ arch/powerpc/kernel/syscalls/syscall.tbl           |   3 +
+ arch/s390/kernel/syscalls/syscall.tbl              |   3 +
+ arch/sh/kernel/syscalls/syscall.tbl                |   3 +
+ arch/sparc/kernel/syscalls/syscall.tbl             |   3 +
+ arch/x86/entry/syscalls/syscall_32.tbl             |   3 +
+ arch/x86/entry/syscalls/syscall_64.tbl             |   3 +
+ arch/xtensa/kernel/syscalls/syscall.tbl            |   3 +
+ fs/proc/base.c                                     |  29 +--
+ fs/proc/internal.h                                 |   2 +-
+ include/linux/lsm_hook_defs.h                      |   4 +
+ include/linux/lsm_hooks.h                          |  17 +-
+ include/linux/security.h                           |  47 +++-
+ include/linux/syscalls.h                           |   6 +
+ include/uapi/asm-generic/unistd.h                  |   9 +-
+ include/uapi/linux/lsm.h                           |  89 +++++++
+ kernel/sys_ni.c                                    |   3 +
+ security/Makefile                                  |   1 +
+ security/apparmor/include/procattr.h               |   2 +-
+ security/apparmor/lsm.c                            |  90 ++++++-
+ security/apparmor/procattr.c                       |  10 +-
+ security/bpf/hooks.c                               |   9 +-
+ security/commoncap.c                               |   8 +-
+ security/landlock/cred.c                           |   2 +-
+ security/landlock/fs.c                             |   2 +-
+ security/landlock/net.c                            |   2 +-
+ security/landlock/ptrace.c                         |   2 +-
+ security/landlock/setup.c                          |   6 +
+ security/landlock/setup.h                          |   1 +
+ security/loadpin/loadpin.c                         |   9 +-
+ security/lockdown/lockdown.c                       |   8 +-
+ security/lsm_syscalls.c                            | 120 +++++++++
+ security/safesetid/lsm.c                           |   9 +-
+ security/security.c                                | 255 ++++++++++++++++++-
+ security/selinux/hooks.c                           | 145 ++++++++---
+ security/smack/smack_lsm.c                         |  92 ++++++-
+ security/tomoyo/tomoyo.c                           |   9 +-
+ security/yama/yama_lsm.c                           |   8 +-
+ .../perf/arch/mips/entry/syscalls/syscall_n64.tbl  |   3 +
+ tools/perf/arch/powerpc/entry/syscalls/syscall.tbl |   3 +
+ tools/perf/arch/s390/entry/syscalls/syscall.tbl    |   3 +
+ tools/perf/arch/x86/entry/syscalls/syscall_64.tbl  |   3 +
+ tools/testing/selftests/Makefile                   |   1 +
+ tools/testing/selftests/lsm/.gitignore             |   1 +
+ tools/testing/selftests/lsm/Makefile               |  17 ++
+ tools/testing/selftests/lsm/common.c               |  89 +++++++
+ tools/testing/selftests/lsm/common.h               |  33 +++
+ tools/testing/selftests/lsm/config                 |   3 +
+ .../testing/selftests/lsm/lsm_get_self_attr_test.c | 275 +++++++++++++++++++++
+ .../testing/selftests/lsm/lsm_list_modules_test.c  | 140 +++++++++++
+ .../testing/selftests/lsm/lsm_set_self_attr_test.c |  74 ++++++
+ 64 files changed, 1692 insertions(+), 97 deletions(-)
+ create mode 100644 Documentation/userspace-api/lsm.rst
+ create mode 100644 include/uapi/linux/lsm.h
+ create mode 100644 security/lsm_syscalls.c
+ create mode 100644 tools/testing/selftests/lsm/.gitignore
+ create mode 100644 tools/testing/selftests/lsm/Makefile
+ create mode 100644 tools/testing/selftests/lsm/common.c
+ create mode 100644 tools/testing/selftests/lsm/common.h
+ create mode 100644 tools/testing/selftests/lsm/config
+ create mode 100644 tools/testing/selftests/lsm/lsm_get_self_attr_test.c
+ create mode 100644 tools/testing/selftests/lsm/lsm_list_modules_test.c
+ create mode 100644 tools/testing/selftests/lsm/lsm_set_self_attr_test.c
+$ git am -3 ../patches/0001-fix-up-for-LSM-wireup-Linux-Security-Module-syscalls.patch
+Applying: fix up for "LSM: wireup Linux Security Module syscalls"
+$ git reset HEAD^
+Unstaged changes after reset:
+M	arch/arm64/include/asm/unistd.h
+$ git add -A .
+$ git commit -v -a --amend
+[master 58272d2c48ff] Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/lsm.git
+ Date: Wed Nov 22 12:11:56 2023 +1100
+Merging apparmor/apparmor-next (a7e405a2de69 apparmor: add missing params to aa_may_ptrace kernel-doc comments)
+$ git merge -m Merge branch 'apparmor-next' of git://git.kernel.org/pub/scm/linux/kernel/git/jj/linux-apparmor apparmor/apparmor-next
+Auto-merging security/apparmor/lsm.c
+Merge made by the 'ort' strategy.
+ security/apparmor/Kconfig      | 12 ++++++------
+ security/apparmor/apparmorfs.c | 16 ++++++++--------
+ security/apparmor/crypto.c     |  6 +++---
+ security/apparmor/domain.c     |  2 +-
+ security/apparmor/lsm.c        |  2 +-
+ security/apparmor/task.c       |  2 ++
+ 6 files changed, 21 insertions(+), 19 deletions(-)
+Merging integrity/next-integrity (b836c4d29f27 ima: detect changes to the backing overlay file)
+$ git merge -m Merge branch 'next-integrity' of git://git.kernel.org/pub/scm/linux/kernel/git/zohar/linux-integrity integrity/next-integrity
+Already up to date.
+Merging safesetid/safesetid-next (64b634830c91 LSM: SafeSetID: add setgroups() testing to selftest)
+$ git merge -m Merge branch 'safesetid-next' of https://github.com/micah-morton/linux.git safesetid/safesetid-next
+Already up to date.
+Merging selinux/next (ae254858ce07 selinux: introduce an initial SID for early boot processes)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/selinux.git selinux/next
+Auto-merging MAINTAINERS
+Auto-merging security/selinux/hooks.c
+Merge made by the 'ort' strategy.
+ MAINTAINERS                                      |   6 +-
+ security/selinux/hooks.c                         |  28 +++++
+ security/selinux/include/initial_sid_to_string.h |   2 +-
+ security/selinux/include/policycap.h             |   1 +
+ security/selinux/include/policycap_names.h       |   1 +
+ security/selinux/include/security.h              |   6 +
+ security/selinux/selinuxfs.c                     | 148 +++++++++++------------
+ security/selinux/ss/avtab.c                      | 101 +++++++---------
+ security/selinux/ss/policydb.c                   |  38 ++++--
+ security/selinux/ss/services.c                   |  13 +-
+ 10 files changed, 191 insertions(+), 153 deletions(-)
+Merging smack/next (3ad49d37cf57 smackfs: Prevent underflow in smk_set_cipso())
+$ git merge -m Merge branch 'next' of git://github.com/cschaufler/smack-next smack/next
+Already up to date.
+Merging tomoyo/master (0bb80ecc33a8 Linux 6.6-rc1)
+$ git merge -m Merge branch 'master' of https://scm.osdn.net/gitroot/tomoyo/tomoyo-test1.git tomoyo/master
+Already up to date.
+Merging tpmdd/next (c2d5304e6c64 Merge tag 'platform-drivers-x86-v6.7-2' of git://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git tpmdd/next
+Already up to date.
+Merging watchdog/master (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'master' of git://www.linux-watchdog.org/linux-watchdog-next.git watchdog/master
+Already up to date.
+Merging iommu/next (e8cca466a84a Merge branches 'iommu/fixes', 'arm/tegra', 'arm/smmu', 'virtio', 'x86/vt-d', 'x86/amd', 'core' and 's390' into next)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu.git iommu/next
+Already up to date.
+Merging audit/next (24fade412acf Automated merge of 'dev' into 'next')
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/audit.git audit/next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS    |  6 +++++-
+ kernel/audit.c | 31 ++++++++++++++++++++++++-------
+ 2 files changed, 29 insertions(+), 8 deletions(-)
+Merging devicetree/for-next (fe612629746c dt-bindings: soc: fsl: cpm_qe: cpm1-scc-qmc: Add support for QMC HDLC)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git devicetree/for-next
+Already up to date.
+Merging dt-krzk/for-next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git dt-krzk/for-next
+Already up to date.
+Merging mailbox/for-next (96cb7a4e296d dt-bindings: mailbox: qcom-ipcc: document the SM8650 Inter-Processor Communication Controller)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/jassibrar/mailbox.git mailbox/for-next
+Already up to date.
+Merging spi/for-next (4c3ff31a85e3 spi: axi-spi-engine improvements)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git spi/for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/spi/adi,axi-spi-engine.txt |  31 --
+ .../bindings/spi/adi,axi-spi-engine.yaml           |  66 +++
+ .../devicetree/bindings/spi/renesas,rspi.yaml      |   2 +-
+ .../devicetree/bindings/spi/st,stm32-spi.yaml      |   1 +
+ MAINTAINERS                                        |  10 +
+ drivers/input/rmi4/rmi_spi.c                       |   2 +-
+ drivers/mmc/host/mmc_spi.c                         |   2 +-
+ drivers/net/ethernet/micrel/ks8851_spi.c           |   4 +-
+ drivers/spi/Kconfig                                |   3 +-
+ drivers/spi/spi-axi-spi-engine.c                   | 403 ++++++++++++-------
+ drivers/spi/spi-cadence-xspi.c                     |   1 -
+ drivers/spi/spi-ingenic.c                          |  15 +-
+ drivers/spi/spi-intel.c                            |  10 +-
+ drivers/spi/spi-stm32.c                            | 447 ++++++++++++++-------
+ drivers/spi/spi-ti-qspi.c                          |  13 +-
+ drivers/usb/gadget/udc/max3420_udc.c               |   2 +-
+ include/linux/spi/spi.h                            |   2 -
+ 17 files changed, 663 insertions(+), 351 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/spi/adi,axi-spi-engine.txt
+ create mode 100644 Documentation/devicetree/bindings/spi/adi,axi-spi-engine.yaml
+Merging tip/master (92fddfbe99ff Merge x86/apic into tip/master)
+$ git merge -m Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git tip/master
+Auto-merging arch/x86/Kconfig
+Auto-merging include/linux/mm_types.h
+Auto-merging include/linux/sched.h
+Merge made by the 'ort' strategy.
+ .../testing/sysfs-bus-event_source-devices-caps    |   6 +
+ arch/powerpc/perf/core-book3s.c                    |   2 +-
+ arch/x86/Kconfig                                   |  12 +
+ arch/x86/entry/calling.h                           |  14 +-
+ arch/x86/entry/entry_32.S                          |   2 +-
+ arch/x86/entry/entry_64.S                          |  35 +-
+ arch/x86/events/amd/core.c                         |   2 +-
+ arch/x86/events/core.c                             |   4 +-
+ arch/x86/events/intel/core.c                       | 147 +++++--
+ arch/x86/events/intel/cstate.c                     | 162 +++++--
+ arch/x86/events/intel/ds.c                         |   4 +-
+ arch/x86/events/intel/lbr.c                        |  85 +++-
+ arch/x86/events/perf_event.h                       |  12 +
+ arch/x86/events/perf_event_flags.h                 |   2 +
+ arch/x86/include/asm/apic.h                        |   2 -
+ arch/x86/include/asm/apicdef.h                     | 276 +-----------
+ arch/x86/include/asm/barrier.h                     |  18 -
+ arch/x86/include/asm/cpufeatures.h                 |   2 +-
+ arch/x86/include/asm/current.h                     |   7 +
+ arch/x86/include/asm/fpu/sched.h                   |  10 +-
+ arch/x86/include/asm/msr-index.h                   |   5 +
+ arch/x86/include/asm/paravirt.h                    |   8 +-
+ arch/x86/include/asm/paravirt_types.h              |   8 +-
+ arch/x86/include/asm/percpu.h                      | 174 ++++++--
+ arch/x86/include/asm/perf_event.h                  |   4 +
+ arch/x86/include/asm/preempt.h                     |   2 +-
+ arch/x86/include/asm/processor.h                   |  21 +
+ arch/x86/include/asm/setup.h                       |   2 -
+ arch/x86/kernel/alternative.c                      |   8 +-
+ arch/x86/kernel/apic/apic_flat_64.c                |   2 -
+ arch/x86/kernel/apic/apic_noop.c                   |   1 -
+ arch/x86/kernel/apic/apic_numachip.c               |   2 -
+ arch/x86/kernel/apic/bigsmp_32.c                   |   1 -
+ arch/x86/kernel/apic/probe_32.c                    |   1 -
+ arch/x86/kernel/apic/x2apic_cluster.c              |   1 -
+ arch/x86/kernel/apic/x2apic_phys.c                 |   1 -
+ arch/x86/kernel/apic/x2apic_uv_x.c                 |   1 -
+ arch/x86/kernel/callthunks.c                       |   2 +-
+ arch/x86/kernel/cpu/amd.c                          |   3 +
+ arch/x86/kernel/cpu/common.c                       |   8 +
+ arch/x86/kernel/cpu/hygon.c                        |   3 +
+ arch/x86/kernel/cpu/mce/core.c                     |  19 +-
+ arch/x86/kernel/cpu/microcode/amd.c                |  39 +-
+ arch/x86/kernel/cpu/microcode/core.c               |  15 +-
+ arch/x86/kernel/cpu/microcode/intel.c              |  17 +-
+ arch/x86/kernel/cpu/microcode/internal.h           |  14 +-
+ arch/x86/kernel/cpu/mtrr/generic.c                 |  14 +-
+ arch/x86/kernel/head_64.S                          |   2 +-
+ arch/x86/kernel/process_32.c                       |   7 +-
+ arch/x86/kernel/process_64.c                       |   7 +-
+ arch/x86/kernel/setup.c                            |   4 +-
+ arch/x86/kernel/smpboot.c                          |   1 +
+ arch/x86/kernel/vmlinux.lds.S                      |   1 +
+ arch/x86/lib/cmpxchg16b_emu.S                      |  12 +-
+ arch/x86/lib/cmpxchg8b_emu.S                       |  30 +-
+ arch/x86/lib/misc.c                                |   2 +-
+ arch/x86/platform/uv/uv_irq.c                      |   2 +-
+ arch/x86/xen/xen-asm.S                             |  10 +-
+ drivers/iommu/amd/iommu.c                          |   4 +-
+ drivers/iommu/intel/irq_remapping.c                |   2 +-
+ drivers/pci/controller/pci-hyperv.c                |   7 -
+ include/linux/cleanup.h                            |  52 ++-
+ include/linux/compiler.h                           |   2 +-
+ include/linux/mm_types.h                           |   3 +
+ include/linux/mutex.h                              |   3 +-
+ include/linux/perf_event.h                         |  22 +-
+ include/linux/rwsem.h                              |   8 +-
+ include/linux/sched.h                              |  26 +-
+ include/linux/spinlock.h                           |  15 +
+ include/trace/events/sched.h                       |  15 +-
+ include/uapi/linux/perf_event.h                    |  13 +
+ kernel/events/core.c                               |  48 ++-
+ kernel/sched/core.c                                |  50 ++-
+ kernel/sched/deadline.c                            | 477 +++++++++++++--------
+ kernel/sched/debug.c                               |  11 +-
+ kernel/sched/fair.c                                | 254 +++++------
+ kernel/sched/idle.c                                |  30 ++
+ kernel/sched/rt.c                                  |  15 +-
+ kernel/sched/sched.h                               |  51 ++-
+ kernel/sched/stop_task.c                           |  13 +-
+ tools/objtool/check.c                              |   2 +-
+ 81 files changed, 1456 insertions(+), 925 deletions(-)
+Merging clockevents/timers/drivers/next (c28ca80ba3b5 clocksource: ep93xx: Add driver for Cirrus Logic EP93xx)
+$ git merge -m Merge branch 'timers/drivers/next' of git://git.linaro.org/people/daniel.lezcano/linux.git clockevents/timers/drivers/next
+Already up to date.
+Merging edac/edac-for-next (0c7c7ba0c721 EDAC/fsl_ddr: Convert to platform remove callback returning void)
+$ git merge -m Merge branch 'edac-for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/ras/ras.git edac/edac-for-next
+Merge made by the 'ort' strategy.
+ drivers/edac/altera_edac.c      | 21 ++++++---------------
+ drivers/edac/armada_xp_edac.c   | 12 ++++--------
+ drivers/edac/aspeed_edac.c      |  6 ++----
+ drivers/edac/bluefield_edac.c   |  6 ++----
+ drivers/edac/cell_edac.c        |  5 ++---
+ drivers/edac/cpc925_edac.c      |  6 ++----
+ drivers/edac/dmc520_edac.c      |  6 ++----
+ drivers/edac/fsl_ddr_edac.c     |  3 +--
+ drivers/edac/fsl_ddr_edac.h     |  2 +-
+ drivers/edac/highbank_l2_edac.c |  5 ++---
+ drivers/edac/highbank_mc_edac.c |  5 ++---
+ drivers/edac/layerscape_edac.c  |  2 +-
+ drivers/edac/mpc85xx_edac.c     | 13 +++++--------
+ drivers/edac/npcm_edac.c        |  6 ++----
+ drivers/edac/octeon_edac-l2c.c  |  6 ++----
+ drivers/edac/octeon_edac-lmc.c  |  5 ++---
+ drivers/edac/octeon_edac-pc.c   |  5 ++---
+ drivers/edac/octeon_edac-pci.c  |  6 ++----
+ drivers/edac/ppc4xx_edac.c      |  7 ++-----
+ drivers/edac/qcom_edac.c        |  6 ++----
+ drivers/edac/synopsys_edac.c    |  6 ++----
+ drivers/edac/ti_edac.c          |  6 ++----
+ drivers/edac/xgene_edac.c       |  6 ++----
+ drivers/edac/zynqmp_edac.c      |  6 ++----
+ 24 files changed, 54 insertions(+), 103 deletions(-)
+Merging irqchip/irq/irqchip-next (19b5a44bee16 irqchip: Add support for Amlogic-C3 SoCs)
+$ git merge -m Merge branch 'irq/irqchip-next' of git://git.kernel.org/pub/scm/linux/kernel/git/maz/arm-platforms.git irqchip/irq/irqchip-next
+Already up to date.
+Merging ftrace/for-next (e1742fa172d5 Merge probes/for-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git ftrace/for-next
+Merge made by the 'ort' strategy.
+Merging rcu/rcu/next (68f721e904b2 rcu: Restrict access to RCU CPU stall notifiers)
+$ git merge -m Merge branch 'rcu/next' of git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git rcu/rcu/next
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/kernel-parameters.txt |  6 ++++++
+ include/linux/rcu_notifier.h                    |  6 +++---
+ kernel/rcu/Kconfig.debug                        | 25 +++++++++++++++++++++++++
+ kernel/rcu/rcu.h                                |  8 +++++---
+ kernel/rcu/rcutorture.c                         | 12 +++++++-----
+ kernel/rcu/tree_stall.h                         | 11 ++++++++++-
+ kernel/rcu/update.c                             |  6 ++++++
+ 7 files changed, 62 insertions(+), 12 deletions(-)
+Merging kvm/next (6c370dc65374 Merge branch 'kvm-guestmemfd' into HEAD)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/virt/kvm/kvm.git kvm/next
+Auto-merging include/linux/pagemap.h
+CONFLICT (content): Merge conflict in include/linux/pagemap.h
+Auto-merging mm/migrate.c
+Recorded preimage for 'include/linux/pagemap.h'
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+Recorded resolution for 'include/linux/pagemap.h'.
+[master 9bb0f4aeaaa9] Merge branch 'next' of git://git.kernel.org/pub/scm/virt/kvm/kvm.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/virt/kvm/api.rst                     | 201 ++++++++
+ arch/arm64/include/asm/kvm_host.h                  |   2 -
+ arch/arm64/kvm/Kconfig                             |   2 +-
+ arch/loongarch/include/asm/kvm_host.h              |   1 -
+ arch/loongarch/kvm/Kconfig                         |   2 +-
+ arch/mips/include/asm/kvm_host.h                   |   2 -
+ arch/mips/kvm/Kconfig                              |   2 +-
+ arch/powerpc/include/asm/kvm_host.h                |   2 -
+ arch/powerpc/kvm/Kconfig                           |   8 +-
+ arch/powerpc/kvm/book3s_hv.c                       |   2 +-
+ arch/powerpc/kvm/powerpc.c                         |   7 +-
+ arch/riscv/include/asm/kvm_host.h                  |   2 -
+ arch/riscv/kvm/Kconfig                             |   2 +-
+ arch/x86/include/asm/kvm_host.h                    |  17 +-
+ arch/x86/include/uapi/asm/kvm.h                    |   3 +
+ arch/x86/kvm/Kconfig                               |  14 +-
+ arch/x86/kvm/debugfs.c                             |   2 +-
+ arch/x86/kvm/mmu/mmu.c                             | 271 ++++++++++-
+ arch/x86/kvm/mmu/mmu_internal.h                    |   2 +
+ arch/x86/kvm/vmx/vmx.c                             |  11 +-
+ arch/x86/kvm/x86.c                                 |  26 +-
+ fs/anon_inodes.c                                   |  51 +-
+ fs/userfaultfd.c                                   |   5 +-
+ include/linux/anon_inodes.h                        |   4 +-
+ include/linux/kvm_host.h                           | 142 +++++-
+ include/linux/kvm_types.h                          |   1 +
+ include/linux/pagemap.h                            |  19 +-
+ include/uapi/linux/kvm.h                           |  49 ++
+ io_uring/io_uring.c                                |   3 +-
+ mm/compaction.c                                    |  43 +-
+ mm/migrate.c                                       |   2 +
+ tools/testing/selftests/kvm/Makefile               |   3 +
+ .../selftests/kvm/aarch64/page_fault_test.c        |   2 +-
+ tools/testing/selftests/kvm/dirty_log_test.c       |   2 +-
+ tools/testing/selftests/kvm/guest_memfd_test.c     | 200 ++++++++
+ .../testing/selftests/kvm/include/kvm_util_base.h  | 150 +++++-
+ tools/testing/selftests/kvm/include/test_util.h    |   5 +
+ tools/testing/selftests/kvm/include/ucall_common.h |  11 +
+ .../selftests/kvm/include/x86_64/processor.h       |  15 +
+ tools/testing/selftests/kvm/kvm_page_table_test.c  |   2 +-
+ tools/testing/selftests/kvm/lib/kvm_util.c         | 224 +++++----
+ tools/testing/selftests/kvm/lib/memstress.c        |   3 +-
+ tools/testing/selftests/kvm/s390x/cmma_test.c      |  11 +-
+ .../testing/selftests/kvm/set_memory_region_test.c | 153 +++++-
+ .../kvm/x86_64/private_mem_conversions_test.c      | 482 +++++++++++++++++++
+ .../kvm/x86_64/private_mem_kvm_exits_test.c        | 120 +++++
+ .../selftests/kvm/x86_64/ucna_injection_test.c     |   2 +-
+ virt/kvm/Kconfig                                   |  17 +
+ virt/kvm/Makefile.kvm                              |   1 +
+ virt/kvm/dirty_ring.c                              |   2 +-
+ virt/kvm/guest_memfd.c                             | 534 +++++++++++++++++++++
+ virt/kvm/kvm_main.c                                | 507 ++++++++++++++++---
+ virt/kvm/kvm_mm.h                                  |  26 +
+ 53 files changed, 3065 insertions(+), 307 deletions(-)
+ create mode 100644 tools/testing/selftests/kvm/guest_memfd_test.c
+ create mode 100644 tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.c
+ create mode 100644 tools/testing/selftests/kvm/x86_64/private_mem_kvm_exits_test.c
+ create mode 100644 virt/kvm/guest_memfd.c
+$ git am -3 ../patches/0001-fs-Convert-error_remove_page-to-error_remove_folio.patch
+Applying: fs: Convert error_remove_page to error_remove_folio
+$ git reset HEAD^
+Unstaged changes after reset:
+M	virt/kvm/guest_memfd.c
+$ git add -A .
+$ git commit -v -a --amend
+[master b7d266ba9a5b] Merge branch 'next' of git://git.kernel.org/pub/scm/virt/kvm/kvm.git
+ Date: Wed Nov 22 12:53:54 2023 +1100
+$ git am -3 ../patches/0001-fix-up-for-KVM-Add-KVM_CREATE_GUEST_MEMFD-ioctl-for-.patch
+Applying: fix up for "KVM: Add KVM_CREATE_GUEST_MEMFD ioctl() for guest-specific backing memory"
+$ git reset HEAD^
+Unstaged changes after reset:
+M	virt/kvm/guest_memfd.c
+$ git add -A .
+$ git commit -v -a --amend
+[master ed4eb02ba77f] Merge branch 'next' of git://git.kernel.org/pub/scm/virt/kvm/kvm.git
+ Date: Wed Nov 22 12:53:54 2023 +1100
+Merging kvm-arm/next (123f42f0ad68 Merge branch kvm-arm64/pmu_pmcr_n into kvmarm/next)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git kvm-arm/next
+Already up to date.
+Merging kvms390/next (70fea3019516 KVM: s390: add tracepoint in gmap notifier)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git kvms390/next
+Already up to date.
+Merging kvm-ppc/topic/ppc-kvm (707df298cbde Merge tag 'powerpc-6.7-1' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux)
+$ git merge -m Merge branch 'topic/ppc-kvm' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git kvm-ppc/topic/ppc-kvm
+Already up to date.
+Merging kvm-riscv/riscv_kvm_next (d9c00f44e5de KVM: riscv: selftests: Add SBI DBCN extension to get-reg-list test)
+$ git merge -m Merge branch 'riscv_kvm_next' of https://github.com/kvm-riscv/linux.git kvm-riscv/riscv_kvm_next
+Already up to date.
+Merging kvm-x86/next (45b890f7689e Merge tag 'kvmarm-6.7' of git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm into HEAD)
+$ git merge -m Merge branch 'next' of https://github.com/kvm-x86/linux.git kvm-x86/next
+Already up to date.
+Merging xen-tip/linux-next (295b202227e9 xen: privcmd: Replace zero-length array with flex-array member and use __counted_by)
+$ git merge -m Merge branch 'linux-next' of git://git.kernel.org/pub/scm/linux/kernel/git/xen/tip.git xen-tip/linux-next
+Merge made by the 'ort' strategy.
+ drivers/xen/privcmd.c     | 2 +-
+ drivers/xen/swiotlb-xen.c | 1 +
+ 2 files changed, 2 insertions(+), 1 deletion(-)
+Merging percpu/for-next (3fcf62f24c80 Merge branch 'for-6.6' into for-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/dennis/percpu.git percpu/for-next
+Merge made by the 'ort' strategy.
+Merging workqueues/for-next (d5ce8f4ed90b Merge branch 'for-6.7' into for-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq.git workqueues/for-next
+Merge made by the 'ort' strategy.
+Merging drivers-x86/for-next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git drivers-x86/for-next
+Already up to date.
+Merging chrome-platform/for-next (d131f1f3b459 platform/chrome: sensorhub: Implement quickselect for median calculation)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git chrome-platform/for-next
+Merge made by the 'ort' strategy.
+ drivers/platform/chrome/cros_ec_sensorhub_ring.c | 74 ++++++++++++++++--------
+ 1 file changed, 51 insertions(+), 23 deletions(-)
+Merging chrome-platform-firmware/for-firmware-next (ecea08916418 firmware: coreboot: framebuffer: Avoid invalid zero physical address)
+$ git merge -m Merge branch 'for-firmware-next' of git://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git chrome-platform-firmware/for-firmware-next
+Merge made by the 'ort' strategy.
+ drivers/firmware/google/framebuffer-coreboot.c | 3 +++
+ 1 file changed, 3 insertions(+)
+Merging hsi/for-next (fa72d143471d HSI: omap_ssi: Remove usage of the deprecated ida_simple_xx() API)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-hsi.git hsi/for-next
+Merge made by the 'ort' strategy.
+ drivers/hsi/controllers/omap_ssi_core.c | 6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+Merging leds/for-next (1b929c02afd3 Linux 6.2-rc1)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/pavel/linux-leds.git leds/for-next
+Already up to date.
+Merging leds-lj/for-leds-next (b9604be24158 leds: lp5521: Add an error check in lp5521_post_init_device)
+$ git merge -m Merge branch 'for-leds-next' of git://git.kernel.org/pub/scm/linux/kernel/git/lee/leds.git leds-lj/for-leds-next
+Already up to date.
+Merging ipmi/for-next (e89a0391d920 ipmi: si: Use device_get_match_data())
+$ git merge -m Merge branch 'for-next' of git://github.com/cminyard/linux-ipmi.git ipmi/for-next
+Merge made by the 'ort' strategy.
+ drivers/char/ipmi/ipmi_si_platform.c | 12 ++++--------
+ 1 file changed, 4 insertions(+), 8 deletions(-)
+Merging driver-core/driver-core-next (98b1cc82c4af Linux 6.7-rc2)
+$ git merge -m Merge branch 'driver-core-next' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/driver-core.git driver-core/driver-core-next
+Already up to date.
+Merging usb/usb-next (98b1cc82c4af Linux 6.7-rc2)
+$ git merge -m Merge branch 'usb-next' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git usb/usb-next
+Already up to date.
+Merging thunderbolt/next (655b8af57d31 thunderbolt: Remove duplicated re-assignment of pointer 'out')
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git thunderbolt/next
+Merge made by the 'ort' strategy.
+ drivers/thunderbolt/tunnel.c | 2 --
+ 1 file changed, 2 deletions(-)
+Merging usb-serial/usb-next (98b1cc82c4af Linux 6.7-rc2)
+$ git merge -m Merge branch 'usb-next' of git://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git usb-serial/usb-next
+Already up to date.
+Merging tty/tty-next (98b1cc82c4af Linux 6.7-rc2)
+$ git merge -m Merge branch 'tty-next' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git tty/tty-next
+Already up to date.
+Merging char-misc/char-misc-next (98b1cc82c4af Linux 6.7-rc2)
+$ git merge -m Merge branch 'char-misc-next' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git char-misc/char-misc-next
+Already up to date.
+Merging accel/habanalabs-next (ce06bc9d7209 accel/habanalabs: expose module id through sysfs)
+$ git merge -m Merge branch 'habanalabs-next' of git://git.kernel.org/pub/scm/linux/kernel/git/ogabbay/linux.git accel/habanalabs-next
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-driver-habanalabs  |   6 ++
+ drivers/accel/habanalabs/common/device.c           |  25 +++--
+ drivers/accel/habanalabs/common/firmware_if.c      | 115 ++++++---------------
+ drivers/accel/habanalabs/common/habanalabs.h       |  10 ++
+ drivers/accel/habanalabs/common/habanalabs_drv.c   |  37 +++++++
+ drivers/accel/habanalabs/common/hwmon.c            |   4 -
+ drivers/accel/habanalabs/common/memory.c           |   7 +-
+ drivers/accel/habanalabs/common/mmu/mmu.c          |   1 +
+ drivers/accel/habanalabs/common/sysfs.c            |  13 +++
+ drivers/accel/habanalabs/gaudi2/gaudi2.c           |  56 +++++-----
+ .../include/gaudi2/asic_reg/gaudi2_regs.h          |   1 +
+ .../habanalabs/include/hw_ip/pci/pci_general.h     |   1 +
+ 12 files changed, 148 insertions(+), 128 deletions(-)
+Merging coresight/next (60e5f23dc5d6 coresight: ultrasoc-smb: Use guards to cleanup)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/coresight/linux.git coresight/next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../ABI/testing/sysfs-bus-coresight-devices-tmc    |  16 +
+ .../ABI/testing/sysfs-bus-coresight-devices-tpdm   | 159 +++++
+ Documentation/arch/arm64/silicon-errata.rst        |  10 +
+ .../bindings/arm/qcom,coresight-tpdm.yaml          |  20 +
+ Documentation/trace/coresight/coresight.rst        |   6 +-
+ MAINTAINERS                                        |   1 -
+ drivers/hwtracing/coresight/coresight-core.c       |   3 +
+ drivers/hwtracing/coresight/coresight-dummy.c      |   5 +-
+ drivers/hwtracing/coresight/coresight-etm-perf.c   |   6 +-
+ drivers/hwtracing/coresight/coresight-etm4x-core.c |  56 +-
+ drivers/hwtracing/coresight/coresight-etm4x.h      |   2 +-
+ drivers/hwtracing/coresight/coresight-funnel.c     |   5 +-
+ drivers/hwtracing/coresight/coresight-replicator.c |   5 +-
+ drivers/hwtracing/coresight/coresight-tmc-core.c   |  15 +-
+ drivers/hwtracing/coresight/coresight-tmc-etr.c    | 111 +++-
+ drivers/hwtracing/coresight/coresight-tmc.h        |   3 +
+ drivers/hwtracing/coresight/coresight-tpda.c       | 126 +++-
+ drivers/hwtracing/coresight/coresight-tpda.h       |   2 +
+ drivers/hwtracing/coresight/coresight-tpdm.c       | 718 ++++++++++++++++++++-
+ drivers/hwtracing/coresight/coresight-tpdm.h       | 161 +++++
+ drivers/hwtracing/coresight/coresight-trbe.c       |  28 +-
+ drivers/hwtracing/coresight/coresight-trbe.h       |   2 +
+ drivers/hwtracing/coresight/ultrasoc-smb.c         | 112 ++--
+ drivers/hwtracing/coresight/ultrasoc-smb.h         |   6 +-
+ drivers/hwtracing/ptt/hisi_ptt.c                   |  33 +-
+ drivers/hwtracing/ptt/hisi_ptt.h                   |   1 +
+ include/linux/coresight.h                          |   1 +
+ 27 files changed, 1460 insertions(+), 153 deletions(-)
+Merging fastrpc/for-next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/srini/fastrpc.git fastrpc/for-next
+Already up to date.
+Merging fpga/for-next (5496fb8eedd6 drivers/fpga: use standard array-copy function)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git fpga/for-next
+Merge made by the 'ort' strategy.
+ drivers/fpga/dfl.c | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+Merging icc/icc-next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'icc-next' of git://git.kernel.org/pub/scm/linux/kernel/git/djakov/icc.git icc/icc-next
+Already up to date.
+Merging iio/togreg (6543f376ec8a iio: buffer: document known issue)
+$ git merge -m Merge branch 'togreg' of git://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git iio/togreg
+Auto-merging .mailmap
+Auto-merging MAINTAINERS
+Auto-merging include/linux/property.h
+Merge made by the 'ort' strategy.
+ .mailmap                                           |    7 +-
+ Documentation/ABI/testing/sysfs-bus-iio            |   18 +
+ .../bindings/iio/adc/maxim,max34408.yaml           |  139 ++
+ .../devicetree/bindings/iio/imu/adi,adis16460.yaml |    4 +
+ .../devicetree/bindings/iio/imu/adi,adis16475.yaml |    4 +
+ .../devicetree/bindings/iio/imu/bosch,bmi323.yaml  |   77 +
+ .../iio/temperature/microchip,mcp9600.yaml         |   70 +
+ MAINTAINERS                                        |   11 +-
+ drivers/base/property.c                            |   37 +-
+ drivers/iio/adc/Kconfig                            |   11 +
+ drivers/iio/adc/Makefile                           |    1 +
+ drivers/iio/adc/max34408.c                         |  276 +++
+ drivers/iio/frequency/adf4377.c                    |   16 +-
+ drivers/iio/frequency/admv1014.c                   |   31 +-
+ drivers/iio/imu/Kconfig                            |    1 +
+ drivers/iio/imu/Makefile                           |    1 +
+ drivers/iio/imu/adis.c                             |   18 +-
+ drivers/iio/imu/bmi323/Kconfig                     |   33 +
+ drivers/iio/imu/bmi323/Makefile                    |    7 +
+ drivers/iio/imu/bmi323/bmi323.h                    |  209 ++
+ drivers/iio/imu/bmi323/bmi323_core.c               | 2139 ++++++++++++++++++++
+ drivers/iio/imu/bmi323/bmi323_i2c.c                |  121 ++
+ drivers/iio/imu/bmi323/bmi323_spi.c                |   92 +
+ drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c  |    5 +-
+ drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c |    5 +-
+ drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c   |    5 +-
+ drivers/iio/industrialio-buffer.c                  |   16 +
+ drivers/iio/industrialio-core.c                    |   63 +
+ drivers/iio/light/rohm-bu27008.c                   |  201 +-
+ drivers/iio/magnetometer/tmag5273.c                |   10 +-
+ drivers/iio/pressure/bmp280-core.c                 |   42 +-
+ drivers/iio/pressure/bmp280-i2c.c                  |    8 +-
+ drivers/iio/pressure/bmp280-spi.c                  |   10 +-
+ drivers/iio/pressure/bmp280.h                      |    6 +-
+ drivers/iio/proximity/sx9324.c                     |   24 +-
+ drivers/iio/resolver/ad2s1210.c                    |  160 +-
+ drivers/iio/temperature/Kconfig                    |   10 +
+ drivers/iio/temperature/Makefile                   |    1 +
+ drivers/iio/temperature/mcp9600.c                  |  139 ++
+ include/linux/property.h                           |   12 +
+ 40 files changed, 3899 insertions(+), 141 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/iio/adc/maxim,max34408.yaml
+ create mode 100644 Documentation/devicetree/bindings/iio/imu/bosch,bmi323.yaml
+ create mode 100644 Documentation/devicetree/bindings/iio/temperature/microchip,mcp9600.yaml
+ create mode 100644 drivers/iio/adc/max34408.c
+ create mode 100644 drivers/iio/imu/bmi323/Kconfig
+ create mode 100644 drivers/iio/imu/bmi323/Makefile
+ create mode 100644 drivers/iio/imu/bmi323/bmi323.h
+ create mode 100644 drivers/iio/imu/bmi323/bmi323_core.c
+ create mode 100644 drivers/iio/imu/bmi323/bmi323_i2c.c
+ create mode 100644 drivers/iio/imu/bmi323/bmi323_spi.c
+ create mode 100644 drivers/iio/temperature/mcp9600.c
+Merging phy-next/next (772dd70a5ed6 phy: core: Remove usage of the deprecated ida_simple_xx() API)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git phy-next/next
+Merge made by the 'ort' strategy.
+ .../bindings/phy/qcom,sc8280xp-qmp-pcie-phy.yaml   |  5 ++
+ .../bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml    |  2 +
+ .../phy/qcom,sc8280xp-qmp-usb43dp-phy.yaml         |  2 +
+ .../bindings/phy/qcom,snps-eusb2-phy.yaml          |  1 +
+ drivers/phy/phy-core.c                             |  4 +-
+ drivers/phy/qualcomm/phy-qcom-qmp-combo.c          |  4 +
+ drivers/phy/qualcomm/phy-qcom-qmp-pcie.c           | 65 ++++++++++++++++
+ drivers/phy/qualcomm/phy-qcom-qmp-pcs-ufs-v6.h     |  1 +
+ .../qualcomm/phy-qcom-qmp-qserdes-txrx-ufs-v6.h    |  7 ++
+ drivers/phy/qualcomm/phy-qcom-qmp-ufs.c            | 86 ++++++++++++++++++++++
+ 10 files changed, 175 insertions(+), 2 deletions(-)
+Merging soundwire/next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git soundwire/next
+Already up to date.
+Merging extcon/extcon-next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'extcon-next' of git://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/extcon.git extcon/extcon-next
+Already up to date.
+Merging gnss/gnss-next (98b1cc82c4af Linux 6.7-rc2)
+$ git merge -m Merge branch 'gnss-next' of git://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git gnss/gnss-next
+Already up to date.
+Merging vfio/next (2b88119e35b0 vfio/mtty: Enable migration support)
+$ git merge -m Merge branch 'next' of git://github.com/awilliam/linux-vfio.git vfio/next
+Already up to date.
+Merging w1/for-next (271c81935801 w1: Add AXI 1-wire host driver for AMD programmable logic IP core)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git w1/for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/w1/amd,axi-1wire-host.yaml |  44 +++
+ MAINTAINERS                                        |   8 +
+ drivers/w1/masters/Kconfig                         |  11 +
+ drivers/w1/masters/Makefile                        |   1 +
+ drivers/w1/masters/amd_axi_w1.c                    | 395 +++++++++++++++++++++
+ 5 files changed, 459 insertions(+)
+ create mode 100644 Documentation/devicetree/bindings/w1/amd,axi-1wire-host.yaml
+ create mode 100644 drivers/w1/masters/amd_axi_w1.c
+Merging staging/staging-next (98b1cc82c4af Linux 6.7-rc2)
+$ git merge -m Merge branch 'staging-next' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git staging/staging-next
+Already up to date.
+Merging counter-next/counter-next (7904cdf1397c counter: chrdev: remove a typo in header file comment)
+$ git merge -m Merge branch 'counter-next' of git://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git counter-next/counter-next
+Already up to date.
+Merging mux/for-next (44c026a73be8 Linux 6.4-rc3)
+$ git merge -m Merge branch 'for-next' of https://gitlab.com/peda-linux/mux.git mux/for-next
+Already up to date.
+Merging dmaengine/next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git dmaengine/next
+Already up to date.
+Merging cgroup/for-next (787521fc0380 Merge branch 'for-6.8' into for-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git cgroup/for-next
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/cgroup-v2.rst           |  10 +-
+ include/linux/cgroup-defs.h                       |   7 +
+ include/linux/workqueue.h                         |   2 +-
+ kernel/cgroup/cpuset.c                            | 286 ++++++++++++++++------
+ kernel/cgroup/legacy_freezer.c                    |   8 +-
+ kernel/cgroup/rstat.c                             |  43 ++--
+ kernel/workqueue.c                                | 165 +++++++++----
+ tools/testing/selftests/cgroup/test_cpuset_prs.sh | 216 ++++++++++------
+ tools/testing/selftests/cgroup/test_freezer.c     |   2 +-
+ 9 files changed, 518 insertions(+), 221 deletions(-)
+Merging scsi/for-next (2a0508d9d08f Merge branch '6.7/scsi-staging' into 6.7/scsi-fixes)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi.git scsi/for-next
+Already up to date.
+Merging scsi-mkp/for-next (6a965ee1892a scsi: mpt3sas: Suppress a warning in debug kernel)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git scsi-mkp/for-next
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-driver-ufs   |  49 ++++++
+ drivers/scsi/3w-sas.c                        |   3 +-
+ drivers/scsi/bnx2fc/bnx2fc_fcoe.c            |  14 +-
+ drivers/scsi/ch.c                            |  12 +-
+ drivers/scsi/csiostor/csio_init.c            |   3 +-
+ drivers/scsi/elx/libefc/efc_node.h           |  12 +-
+ drivers/scsi/lpfc/lpfc.h                     |   1 +
+ drivers/scsi/lpfc/lpfc_attr.c                |   4 +-
+ drivers/scsi/lpfc/lpfc_els.c                 |  53 +++---
+ drivers/scsi/lpfc/lpfc_hbadisc.c             |   2 +-
+ drivers/scsi/lpfc/lpfc_init.c                |   4 +-
+ drivers/scsi/lpfc/lpfc_mbox.c                |   6 +-
+ drivers/scsi/lpfc/lpfc_mem.c                 |  47 +++---
+ drivers/scsi/lpfc/lpfc_sli.c                 |  20 ++-
+ drivers/scsi/lpfc/lpfc_sli.h                 |  10 +-
+ drivers/scsi/lpfc/lpfc_version.h             |   2 +-
+ drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h         | 231 +++++++++------------------
+ drivers/scsi/mpt3sas/mpi/mpi2_image.h        |  32 ++--
+ drivers/scsi/mpt3sas/mpi/mpi2_ioc.h          |  27 ++--
+ drivers/scsi/mpt3sas/mpt3sas_base.c          |  35 ++--
+ drivers/scsi/mpt3sas/mpt3sas_base.h          |   3 +-
+ drivers/scsi/mpt3sas/mpt3sas_config.c        |   6 +-
+ drivers/scsi/mpt3sas/mpt3sas_ctl.c           |  42 ++---
+ drivers/scsi/mpt3sas/mpt3sas_scsih.c         |  56 +++----
+ drivers/scsi/mpt3sas/mpt3sas_transport.c     |   9 +-
+ drivers/scsi/mpt3sas/mpt3sas_trigger_pages.h |  44 ++---
+ drivers/scsi/mpt3sas/mpt3sas_warpdrive.c     |   3 +-
+ drivers/ufs/core/ufs-sysfs.c                 | 120 ++++++++++++++
+ include/ufs/unipro.h                         |   4 +-
+ 29 files changed, 466 insertions(+), 388 deletions(-)
+Merging vhost/linux-next (e07754e0a1ea vhost-vdpa: fix use after free in vhost_vdpa_probe())
+$ git merge -m Merge branch 'linux-next' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git vhost/linux-next
+Already up to date.
+Merging rpmsg/for-next (cae0e61beb7a arm64: dts: imx8mp: Add reserve-memory nodes for DSP)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/remoteproc/linux.git rpmsg/for-next
+Merge made by the 'ort' strategy.
+ arch/arm64/boot/dts/freescale/imx8mp-evk.dts | 22 ++++++++++++++++++++++
+ drivers/remoteproc/imx_dsp_rproc.c           |  1 +
+ 2 files changed, 23 insertions(+)
+Merging gpio/for-next (0bb80ecc33a8 Linux 6.6-rc1)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-gpio.git gpio/for-next
+Already up to date.
+Merging gpio-brgl/gpio/for-next (d652049e118f gpio: mockup: initialize a managed pointer in place)
+$ git merge -m Merge branch 'gpio/for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git gpio-brgl/gpio/for-next
+Merge made by the 'ort' strategy.
+ drivers/gpio/gpio-mockup.c | 3 +--
+ drivers/gpio/gpio-sifive.c | 1 -
+ 2 files changed, 1 insertion(+), 3 deletions(-)
+Merging gpio-intel/for-next (92fc925f8386 gpio: tangier: simplify locking using cleanup helpers)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git gpio-intel/for-next
+Merge made by the 'ort' strategy.
+ drivers/gpio/gpio-elkhartlake.c | 14 +--------
+ drivers/gpio/gpio-tangier.c     | 63 +++++++++++++----------------------------
+ drivers/gpio/gpio-tangier.h     |  4 +--
+ 3 files changed, 22 insertions(+), 59 deletions(-)
+Merging pinctrl/for-next (91acee35452f Merge branch 'devel' into for-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-pinctrl.git pinctrl/for-next
+Merge made by the 'ort' strategy.
+ .../pinctrl/qcom,sm8650-lpass-lpi-pinctrl.yaml     |  148 ++
+ .../bindings/pinctrl/qcom,sm8650-tlmm.yaml         |  147 ++
+ drivers/pinctrl/Kconfig                            |   16 +
+ drivers/pinctrl/Makefile                           |    1 +
+ drivers/pinctrl/cirrus/Kconfig                     |    3 +-
+ drivers/pinctrl/core.c                             |    6 +-
+ drivers/pinctrl/nxp/pinctrl-s32cc.c                |    4 +-
+ drivers/pinctrl/pinctrl-cy8c95x0.c                 |    1 +
+ drivers/pinctrl/pinctrl-tps6594.c                  |  373 +++++
+ drivers/pinctrl/qcom/Kconfig                       |   10 +
+ drivers/pinctrl/qcom/Kconfig.msm                   |    8 +
+ drivers/pinctrl/qcom/Makefile                      |    2 +
+ drivers/pinctrl/qcom/pinctrl-lpass-lpi.c           |   69 +-
+ drivers/pinctrl/qcom/pinctrl-lpass-lpi.h           |    7 +
+ drivers/pinctrl/qcom/pinctrl-msm.c                 |   42 +
+ drivers/pinctrl/qcom/pinctrl-msm.h                 |    5 +
+ drivers/pinctrl/qcom/pinctrl-sm8650-lpass-lpi.c    |  255 +++
+ drivers/pinctrl/qcom/pinctrl-sm8650.c              | 1762 ++++++++++++++++++++
+ drivers/pinctrl/stm32/pinctrl-stm32.c              |   13 +-
+ drivers/pinctrl/tegra/pinctrl-tegra.c              |    8 +
+ 20 files changed, 2847 insertions(+), 33 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/pinctrl/qcom,sm8650-lpass-lpi-pinctrl.yaml
+ create mode 100644 Documentation/devicetree/bindings/pinctrl/qcom,sm8650-tlmm.yaml
+ create mode 100644 drivers/pinctrl/pinctrl-tps6594.c
+ create mode 100644 drivers/pinctrl/qcom/pinctrl-sm8650-lpass-lpi.c
+ create mode 100644 drivers/pinctrl/qcom/pinctrl-sm8650.c
+Merging pinctrl-intel/for-next (c5860e4a2737 pinctrl: intel: Add a generic Intel pin control platform driver)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git pinctrl-intel/for-next
+Merge made by the 'ort' strategy.
+ drivers/pinctrl/intel/Kconfig                  |  12 +-
+ drivers/pinctrl/intel/Makefile                 |   1 +
+ drivers/pinctrl/intel/pinctrl-alderlake.c      |   5 +-
+ drivers/pinctrl/intel/pinctrl-broxton.c        |   5 +-
+ drivers/pinctrl/intel/pinctrl-cannonlake.c     |   5 +-
+ drivers/pinctrl/intel/pinctrl-cedarfork.c      |   5 +-
+ drivers/pinctrl/intel/pinctrl-denverton.c      |   5 +-
+ drivers/pinctrl/intel/pinctrl-elkhartlake.c    |   5 +-
+ drivers/pinctrl/intel/pinctrl-emmitsburg.c     |   5 +-
+ drivers/pinctrl/intel/pinctrl-geminilake.c     |   5 +-
+ drivers/pinctrl/intel/pinctrl-icelake.c        |   5 +-
+ drivers/pinctrl/intel/pinctrl-intel-platform.c | 225 +++++++++++++++++++++++++
+ drivers/pinctrl/intel/pinctrl-intel.c          |  32 ++--
+ drivers/pinctrl/intel/pinctrl-intel.h          |  14 +-
+ drivers/pinctrl/intel/pinctrl-jasperlake.c     |   5 +-
+ drivers/pinctrl/intel/pinctrl-lakefield.c      |   5 +-
+ drivers/pinctrl/intel/pinctrl-lewisburg.c      |   5 +-
+ drivers/pinctrl/intel/pinctrl-meteorlake.c     |   5 +-
+ drivers/pinctrl/intel/pinctrl-sunrisepoint.c   |   5 +-
+ drivers/pinctrl/intel/pinctrl-tangier.c        |  16 +-
+ drivers/pinctrl/intel/pinctrl-tigerlake.c      |   5 +-
+ 21 files changed, 294 insertions(+), 81 deletions(-)
+ create mode 100644 drivers/pinctrl/intel/pinctrl-intel-platform.c
+Merging pinctrl-renesas/renesas-pinctrl (583d80732055 pinctrl: renesas: rzn1: Convert to platform remove callback returning void)
+$ git merge -m Merge branch 'renesas-pinctrl' of git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git pinctrl-renesas/renesas-pinctrl
+Already up to date.
+Merging pinctrl-samsung/for-next (e4e2fbe7d7d7 MAINTAINERS: Remove snawrocki's git tree)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git pinctrl-samsung/for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../pinctrl/samsung,pinctrl-wakeup-interrupt.yaml  | 26 ++++++++++++++--------
+ .../bindings/pinctrl/samsung,pinctrl.yaml          |  4 +++-
+ MAINTAINERS                                        |  1 -
+ 3 files changed, 20 insertions(+), 11 deletions(-)
+Merging pwm/for-next (d27abbfd4888 pwm: Fix double shift bug)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/thierry.reding/linux-pwm.git pwm/for-next
+Already up to date.
+Merging userns/for-next (05bd6e0242b4 Merge of unpriv-ipc-sysctls-for-v6.2, and fix-atomic_lock_inc_below-for-v6.2 for testing in linux-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiederm/user-namespace.git userns/for-next
+Auto-merging Documentation/admin-guide/sysctl/kernel.rst
+Auto-merging ipc/ipc_sysctl.c
+Auto-merging ipc/mq_sysctl.c
+Auto-merging kernel/ucount.c
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/sysctl/kernel.rst | 14 ++++++++---
+ ipc/ipc_sysctl.c                            | 37 +++++++++++++++++++++++++++--
+ ipc/mq_sysctl.c                             | 36 ++++++++++++++++++++++++++++
+ kernel/ucount.c                             | 16 ++++++-------
+ 4 files changed, 89 insertions(+), 14 deletions(-)
+Merging ktest/for-next (7dc8e24f0e09 ktest: Restore stty setting at first in dodie)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-ktest.git ktest/for-next
+Already up to date.
+Merging kselftest/next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kselftest/next
+Already up to date.
+Merging kunit/test (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'test' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit/test
+Already up to date.
+Merging kunit-next/kunit (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'kunit' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit-next/kunit
+Already up to date.
+Merging livepatching/for-next (602bf1830798 Merge branch 'for-6.7' into for-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/livepatching/livepatching livepatching/for-next
+Merge made by the 'ort' strategy.
+Merging rtc/rtc-next (3d762e21d563 rtc: cmos: Use ACPI alarm for non-Intel x86 systems too)
+$ git merge -m Merge branch 'rtc-next' of git://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git rtc/rtc-next
+Merge made by the 'ort' strategy.
+ drivers/rtc/rtc-cmos.c | 18 ++++++++++++------
+ 1 file changed, 12 insertions(+), 6 deletions(-)
+Merging nvdimm/libnvdimm-for-next (9ea459e477dc libnvdimm: remove kernel-doc warnings:)
+$ git merge -m Merge branch 'libnvdimm-for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git nvdimm/libnvdimm-for-next
+Already up to date.
+Merging at24/at24/for-next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'at24/for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git at24/at24/for-next
+Already up to date.
+Merging ntb/ntb-next (9341b37ec17a ntb_perf: Fix printk format)
+$ git merge -m Merge branch 'ntb-next' of https://github.com/jonmason/ntb.git ntb/ntb-next
+Merge made by the 'ort' strategy.
+ drivers/ntb/hw/intel/ntb_hw_gen1.c | 2 +-
+ drivers/ntb/test/ntb_perf.c        | 2 +-
+ 2 files changed, 2 insertions(+), 2 deletions(-)
+Merging seccomp/for-next/seccomp (31c65705a8cf perf/benchmark: fix seccomp_unotify benchmark for 32-bit)
+$ git merge -m Merge branch 'for-next/seccomp' of git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git seccomp/for-next/seccomp
+Already up to date.
+Merging fsi/next (f04d61a379d6 fsi: fix some spelling mistakes in comment)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/joel/fsi.git fsi/next
+Already up to date.
+Merging slimbus/for-next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/srini/slimbus.git slimbus/for-next
+Already up to date.
+Merging nvmem/for-next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/srini/nvmem.git nvmem/for-next
+Already up to date.
+Merging xarray/main (2a15de80dd0f idr: fix param name in idr_alloc_cyclic() doc)
+$ git merge -m Merge branch 'main' of git://git.infradead.org/users/willy/xarray.git xarray/main
+Already up to date.
+Merging hyperv/hyperv-next (ce9ecca0238b Linux 6.6-rc2)
+$ git merge -m Merge branch 'hyperv-next' of git://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git hyperv/hyperv-next
+Already up to date.
+Merging auxdisplay/auxdisplay (35b464e32c8b auxdisplay: hd44780: move cursor home after clear display command)
+$ git merge -m Merge branch 'auxdisplay' of https://github.com/ojeda/linux.git auxdisplay/auxdisplay
+Already up to date.
+Merging kgdb/kgdb/for-next (23816724fdbd kdb: Corrects comment for kdballocenv)
+$ git merge -m Merge branch 'kgdb/for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/danielt/linux.git kgdb/kgdb/for-next
+Already up to date.
+Merging hmm/hmm (0bb80ecc33a8 Linux 6.6-rc1)
+$ git merge -m Merge branch 'hmm' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git hmm/hmm
+Already up to date.
+Merging cfi/cfi/next (06c2afb862f9 Linux 6.5-rc1)
+$ git merge -m Merge branch 'cfi/next' of git://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git cfi/cfi/next
+Already up to date.
+Merging mhi/mhi-next (83c5dcf78ab9 bus: mhi: host: Add alignment check for event ring read pointer)
+$ git merge -m Merge branch 'mhi-next' of git://git.kernel.org/pub/scm/linux/kernel/git/mani/mhi.git mhi/mhi-next
+Merge made by the 'ort' strategy.
+ drivers/bus/mhi/ep/main.c          | 68 ++++++++++++++++++++++++++++----------
+ drivers/bus/mhi/host/init.c        |  1 +
+ drivers/bus/mhi/host/internal.h    |  2 +-
+ drivers/bus/mhi/host/main.c        |  8 +++--
+ drivers/bus/mhi/host/pci_generic.c | 22 ++++++++++++
+ drivers/bus/mhi/host/pm.c          | 24 ++++++++++----
+ include/linux/mhi.h                |  4 +++
+ 7 files changed, 100 insertions(+), 29 deletions(-)
+Merging memblock/for-next (e96c6b8f212a memblock: report failures when memblock_can_resize is not set)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git memblock/for-next
+Already up to date.
+Merging cxl/next (5d09c63f11f0 cxl/hdm: Remove broken error path)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git cxl/next
+Already up to date.
+Merging zstd/zstd-next (3f832dfb8a8e zstd: fix g_debuglevel export warning)
+$ git merge -m Merge branch 'zstd-next' of https://github.com/terrelln/linux.git zstd/zstd-next
+Merge made by the 'ort' strategy.
+ include/linux/zstd.h                           |    2 +-
+ include/linux/zstd_errors.h                    |   23 +-
+ include/linux/zstd_lib.h                       |  697 ++++++++--
+ lib/zstd/Makefile                              |    2 +-
+ lib/zstd/common/allocations.h                  |   56 +
+ lib/zstd/common/bits.h                         |  149 ++
+ lib/zstd/common/bitstream.h                    |   53 +-
+ lib/zstd/common/compiler.h                     |   14 +-
+ lib/zstd/common/cpu.h                          |    3 +-
+ lib/zstd/common/debug.c                        |    5 +-
+ lib/zstd/common/debug.h                        |    3 +-
+ lib/zstd/common/entropy_common.c               |   42 +-
+ lib/zstd/common/error_private.c                |   12 +-
+ lib/zstd/common/error_private.h                |    3 +-
+ lib/zstd/common/fse.h                          |   89 +-
+ lib/zstd/common/fse_decompress.c               |   94 +-
+ lib/zstd/common/huf.h                          |  234 +---
+ lib/zstd/common/mem.h                          |    2 +-
+ lib/zstd/common/portability_macros.h           |   26 +-
+ lib/zstd/common/zstd_common.c                  |   38 +-
+ lib/zstd/common/zstd_deps.h                    |   16 +-
+ lib/zstd/common/zstd_internal.h                |   99 +-
+ lib/zstd/compress/clevels.h                    |    3 +-
+ lib/zstd/compress/fse_compress.c               |   59 +-
+ lib/zstd/compress/hist.c                       |    3 +-
+ lib/zstd/compress/hist.h                       |    3 +-
+ lib/zstd/compress/huf_compress.c               |  372 +++--
+ lib/zstd/compress/zstd_compress.c              | 1758 +++++++++++++++++-------
+ lib/zstd/compress/zstd_compress_internal.h     |  333 +++--
+ lib/zstd/compress/zstd_compress_literals.c     |  155 ++-
+ lib/zstd/compress/zstd_compress_literals.h     |   25 +-
+ lib/zstd/compress/zstd_compress_sequences.c    |    7 +-
+ lib/zstd/compress/zstd_compress_sequences.h    |    3 +-
+ lib/zstd/compress/zstd_compress_superblock.c   |   47 +-
+ lib/zstd/compress/zstd_compress_superblock.h   |    3 +-
+ lib/zstd/compress/zstd_cwksp.h                 |  149 +-
+ lib/zstd/compress/zstd_double_fast.c           |  129 +-
+ lib/zstd/compress/zstd_double_fast.h           |    6 +-
+ lib/zstd/compress/zstd_fast.c                  |  578 ++++++--
+ lib/zstd/compress/zstd_fast.h                  |    6 +-
+ lib/zstd/compress/zstd_lazy.c                  |  518 +++----
+ lib/zstd/compress/zstd_lazy.h                  |    7 +-
+ lib/zstd/compress/zstd_ldm.c                   |   11 +-
+ lib/zstd/compress/zstd_ldm.h                   |    3 +-
+ lib/zstd/compress/zstd_ldm_geartab.h           |    3 +-
+ lib/zstd/compress/zstd_opt.c                   |  187 +--
+ lib/zstd/compress/zstd_opt.h                   |    3 +-
+ lib/zstd/decompress/huf_decompress.c           |  772 +++++++----
+ lib/zstd/decompress/zstd_ddict.c               |    9 +-
+ lib/zstd/decompress/zstd_ddict.h               |    3 +-
+ lib/zstd/decompress/zstd_decompress.c          |  259 +++-
+ lib/zstd/decompress/zstd_decompress_block.c    |  283 ++--
+ lib/zstd/decompress/zstd_decompress_block.h    |    8 +-
+ lib/zstd/decompress/zstd_decompress_internal.h |    7 +-
+ lib/zstd/decompress_sources.h                  |    2 +-
+ lib/zstd/zstd_common_module.c                  |    5 +-
+ lib/zstd/zstd_compress_module.c                |    2 +-
+ lib/zstd/zstd_decompress_module.c              |    4 +-
+ 58 files changed, 4791 insertions(+), 2596 deletions(-)
+ create mode 100644 lib/zstd/common/allocations.h
+ create mode 100644 lib/zstd/common/bits.h
+Merging efi/next (5329aa5101f7 efivarfs: Add uid/gid mount options)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git efi/next
+Already up to date.
+Merging unicode/for-next (c1869dbd2957 Merge branch 'unicode/negative-dentries' into unicode/for-next)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/krisman/unicode.git unicode/for-next
+Auto-merging Documentation/filesystems/locking.rst
+Auto-merging Documentation/filesystems/vfs.rst
+Auto-merging fs/ecryptfs/inode.c
+Auto-merging fs/gfs2/dentry.c
+Auto-merging fs/libfs.c
+Auto-merging fs/namei.c
+Auto-merging fs/proc/base.c
+Merge made by the 'ort' strategy.
+ Documentation/filesystems/locking.rst |   3 +-
+ Documentation/filesystems/vfs.rst     |  11 +++-
+ fs/9p/vfs_dentry.c                    |  11 +++-
+ fs/afs/dir.c                          |   6 +-
+ fs/afs/dynroot.c                      |   4 +-
+ fs/ceph/dir.c                         |   5 +-
+ fs/coda/dir.c                         |   3 +-
+ fs/crypto/fname.c                     |   3 +-
+ fs/dcache.c                           |   8 +++
+ fs/ecryptfs/dentry.c                  |   6 +-
+ fs/ecryptfs/inode.c                   |   8 +++
+ fs/exfat/namei.c                      |   3 +-
+ fs/ext4/namei.c                       |  35 ++---------
+ fs/f2fs/namei.c                       |  25 ++------
+ fs/fat/namei_vfat.c                   |   6 +-
+ fs/fuse/dir.c                         |   3 +-
+ fs/gfs2/dentry.c                      |   4 +-
+ fs/hfs/sysdep.c                       |   3 +-
+ fs/jfs/namei.c                        |   3 +-
+ fs/kernfs/dir.c                       |   3 +-
+ fs/libfs.c                            | 107 ++++++++++++++++++++++++----------
+ fs/namei.c                            |  18 +++---
+ fs/nfs/dir.c                          |   9 ++-
+ fs/ocfs2/dcache.c                     |   4 +-
+ fs/orangefs/dcache.c                  |   3 +-
+ fs/overlayfs/super.c                  |  20 ++++---
+ fs/proc/base.c                        |   6 +-
+ fs/proc/fd.c                          |   3 +-
+ fs/proc/generic.c                     |   6 +-
+ fs/proc/proc_sysctl.c                 |   3 +-
+ fs/reiserfs/xattr.c                   |   3 +-
+ fs/smb/client/dir.c                   |   3 +-
+ fs/vboxsf/dir.c                       |   4 +-
+ include/linux/dcache.h                |  10 +++-
+ include/linux/fscrypt.h               |   4 +-
+ 35 files changed, 219 insertions(+), 137 deletions(-)
+Merging slab/slab/for-next (dd374e220ba4 slub: Update frozen slabs documentations in the source)
+$ git merge -m Merge branch 'slab/for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab.git slab/slab/for-next
+Merge made by the 'ort' strategy.
+ mm/slub.c | 386 +++++++++++++++++++++++++++++---------------------------------
+ 1 file changed, 181 insertions(+), 205 deletions(-)
+Merging random/master (512dee0c00ad Merge tag 'x86-urgent-2023-01-04' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip)
+$ git merge -m Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/crng/random.git random/master
+Already up to date.
+Merging landlock/next (323f65ded48d landlock: Document IOCTL support)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/mic/linux.git landlock/next
+Auto-merging security/landlock/fs.c
+Merge made by the 'ort' strategy.
+ Documentation/userspace-api/landlock.rst     |  74 +++-
+ include/uapi/linux/landlock.h                |  58 +++-
+ samples/landlock/sandboxer.c                 |  13 +-
+ security/landlock/fs.c                       | 150 +++++++-
+ security/landlock/fs.h                       |  11 +
+ security/landlock/limits.h                   |  15 +-
+ security/landlock/ruleset.c                  |   5 +-
+ security/landlock/ruleset.h                  |   2 +-
+ security/landlock/syscalls.c                 |  10 +-
+ tools/testing/selftests/landlock/base_test.c |   2 +-
+ tools/testing/selftests/landlock/fs_test.c   | 498 ++++++++++++++++++++++++++-
+ 11 files changed, 789 insertions(+), 49 deletions(-)
+Merging rust/rust-next (3857af38e57a docs: rust: add "The Rust experiment" section)
+$ git merge -m Merge branch 'rust-next' of https://github.com/Rust-for-Linux/linux.git rust/rust-next
+Already up to date.
+Merging sysctl/sysctl-next (8b793bcda61f watchdog: move softlockup_panic back to early_param)
+$ git merge -m Merge branch 'sysctl-next' of git://git.kernel.org/pub/scm/linux/kernel/git/mcgrof/linux.git sysctl/sysctl-next
+Already up to date.
+Merging execve/for-next/execve (21ca59b365c0 binfmt_misc: enable sandboxed mounts)
+$ git merge -m Merge branch 'for-next/execve' of git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git execve/for-next/execve
+Already up to date.
+Merging bitmap/bitmap-for-next (bdcb37a5d8de buildid: reduce header file dependencies for module)
+$ git merge -m Merge branch 'bitmap-for-next' of https://github.com/norov/linux.git bitmap/bitmap-for-next
+Already up to date.
+Merging hte/for-next (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/pateldipen1984/linux.git hte/for-next
+Already up to date.
+Merging nolibc/nolibc (b85ea95d0864 Linux 6.7-rc1)
+$ git merge -m Merge branch 'nolibc' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git nolibc/nolibc
+Already up to date.
+Merging tsm/tsm-next (f4738f56d1dc virt: tdx-guest: Add Quote generation support using TSM_REPORTS)
+$ git merge -m Merge branch 'tsm-next' of git://git.kernel.org/pub/scm/linux/kernel/git/djbw/linux tsm/tsm-next
+Already up to date.
+Merging iommufd/for-next (b2b67c997bf7 iommufd: Organize the mock domain alloc functions closer to Joerg's tree)
+$ git merge -m Merge branch 'for-next' of git://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git iommufd/for-next
+Already up to date.
diff --git a/arch/Kconfig b/arch/Kconfig
index f4b210ab0612..bfcc7c2dc039 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -301,17 +301,8 @@ config ARCH_HAS_DMA_CLEAR_UNCACHED
 config ARCH_HAS_CPU_FINALIZE_INIT
 	bool
 
-# Select if arch init_task must go in the __init_task_data section
-config ARCH_TASK_STRUCT_ON_STACK
-	bool
-
-# Select if arch has its private alloc_task_struct() function
-config ARCH_TASK_STRUCT_ALLOCATOR
-	bool
-
 config HAVE_ARCH_THREAD_STRUCT_WHITELIST
 	bool
-	depends on !ARCH_TASK_STRUCT_ALLOCATOR
 	help
 	  An architecture should select this to provide hardened usercopy
 	  knowledge about what region of the thread_struct should be
@@ -320,10 +311,6 @@ config HAVE_ARCH_THREAD_STRUCT_WHITELIST
 	  should be implemented. Without this, the entire thread_struct
 	  field in task_struct will be left whitelisted.
 
-# Select if arch has its private alloc_thread_stack() function
-config ARCH_THREAD_STACK_ALLOCATOR
-	bool
-
 # Select if arch wants to size task_struct dynamically via arch_task_struct_size:
 config ARCH_WANTS_DYNAMIC_TASK_STRUCT
 	bool
diff --git a/arch/alpha/kernel/syscalls/syscall.tbl b/arch/alpha/kernel/syscalls/syscall.tbl
index 18c842ca6c32..8ff110826ce2 100644
--- a/arch/alpha/kernel/syscalls/syscall.tbl
+++ b/arch/alpha/kernel/syscalls/syscall.tbl
@@ -496,3 +496,8 @@
 564	common	futex_wake			sys_futex_wake
 565	common	futex_wait			sys_futex_wait
 566	common	futex_requeue			sys_futex_requeue
+567	common	statmount			sys_statmount
+568	common	listmount			sys_listmount
+569	common	lsm_get_self_attr		sys_lsm_get_self_attr
+570	common	lsm_set_self_attr		sys_lsm_set_self_attr
+571	common	lsm_list_modules		sys_lsm_list_modules
diff --git a/arch/arm/arm-soc-for-next-contents.txt b/arch/arm/arm-soc-for-next-contents.txt
new file mode 100644
index 000000000000..a1069cdcf85d
--- /dev/null
+++ b/arch/arm/arm-soc-for-next-contents.txt
@@ -0,0 +1,269 @@
+soc/arm
+	patch
+		MAINTAINERS: Add entry for AMD PENSANDO
+		arm64: Add config for AMD Pensando SoC platforms
+	renesas/arm
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-arm-soc-for-v6.7-tag1
+	patch
+		arm: debug: reuse the config DEBUG_OMAP2UART{1,2} for OMAP{3,4,5}
+
+soc/dt
+	patch
+		dt-bindings: arm: add AMD Pensando boards
+		arm64: dts: Add AMD Pensando Elba SoC support
+	ux500/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-nomadik tags/ux500-dts-for-armsoc
+	renesas/dt-bindings
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-dt-bindings-for-v6.7-tag1
+	renesas/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-dts-for-v6.7-tag1
+	dts/cleanup-arm64
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt tags/dt64-cleanup-6.7
+	dts/cleanup-arm32
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt tags/dt-cleanup-6.7
+	samsung/dt64
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux tags/samsung-dt64-6.7
+	samsung/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux tags/samsung-dt-6.7
+	ixp4xx/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-nomadik tags/ixp4xx-for-soc-v6.7
+	at91/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux tags/at91-dt-6.7
+	omap/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/tmlind/linux-omap tags/omap-for-v6.7/dt-signed
+	sti/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/pchotard/sti tags/sti-dt-for-v6.7-round1
+	renesas/dt-bindings-2
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-dt-bindings-for-v6.7-tag2
+	renesas/dt-2
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-dts-for-v6.7-tag2
+	tegra/dt-bindings
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-6.7-dt-bindings
+	tegra/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-6.7-arm-dt
+	tegra/dt64
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-6.7-arm64-dt
+	sunxi/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux tags/sunxi-dt-for-6.7-1
+	riscv/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux tags/riscv-dt-for-v6.6-final
+	imx/dt-bindings
+		git://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux tags/imx-bindings-6.7
+	imx/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux tags/imx-dt-6.7
+	imx/dt64
+		git://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux tags/imx-dt64-6.7
+	qcom/dt64
+		https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux tags/qcom-arm64-for-6.7
+	qcom/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux tags/qcom-dts-for-6.7
+	aspeed/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/joel/bmc tags/aspeed-6.7-devicetree
+	amlogic/dt64
+		https://git.kernel.org/pub/scm/linux/kernel/git/amlogic/linux tags/amlogic-arm64-dt-for-v6.7
+	stm32/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/atorgue/stm32 tags/stm32-dt-for-v6.7-1
+	patch
+		arm/arm64: dts: Removed undocumented and unused "pl022,hierarchy" property
+	riscv/dt-2
+		https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux tags/riscv-dt-for-v6.7
+	rockchip/dt64
+		git://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip tags/v6.7-rockchip-dts64-1
+	rockchip/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip tags/v6.7-rockchip-dts32-1
+	mediatek/dts64-2
+		https://git.kernel.org/pub/scm/linux/kernel/git/matthias.bgg/linux tags/v6.6-next-dts64.3
+	<no branch> (56bc311585206a8955de793301d4f84fb4ad2ee6)
+		https://git.kernel.org/pub/scm/linux/kernel/git/ti/linux tags/ti-k3-dt-for-v6.7
+	marvell/dts
+		git://git.kernel.org/pub/scm/linux/kernel/git/gclement/mvebu tags/mvebu-dt64-6.7-1
+	patch
+		arm64: dts: Update cache properties for socionext
+		riscv: dts: thead: convert isa detection to new properties
+		arm64: dts: socionext: add missing cache properties
+	samsung/dt-2
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux tags/samsung-dt64-6.7-2
+	samsung/dt-3
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux tags/samsung-dt-6.7-2
+	broadcom/dt
+		https://github.com/Broadcom/stblinux tags/arm-soc/for-6.7/devicetree
+	ti/dt-2
+		https://git.kernel.org/pub/scm/linux/kernel/git/ti/linux tags/ti-k3-dt-for-v6.7-part2
+	rockchip/dt-2
+		git://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip tags/v6.7-rockchip-dts64-2
+
+soc/drivers
+	patch
+		soc: samsung: exynos-chipid: Convert to platform remove callback returning void
+	drivers/soc-cleanup
+		https://git.pengutronix.de/git/ukl/linux tags/platform-remove-void-soc-for-6.7-rc
+	renesas/drivers
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-drivers-for-v6.7-tag1
+	mediatek/drivers
+		https://git.kernel.org/pub/scm/linux/kernel/git/matthias.bgg/linux tags/v6.6-next-soc
+	vexpress/drivers
+		git://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux tags/vexpress-update-6.7
+	firmware/scmi
+		git://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux tags/scmi-updates-6.7
+	firmware/ffa
+		git://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux tags/ffa-updates-6.7
+	tegra/soc
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-6.7-soc
+	tegra/firmware
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-6.7-firmware
+	tegra/memory
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-6.7-memory
+	aspeed/soc-driver
+		git://git.kernel.org/pub/scm/linux/kernel/git/joel/bmc tags/aspeed-6.7-soc
+	drivers/memory
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl tags/memory-controller-drv-6.7
+	meson/firmware
+		https://git.kernel.org/pub/scm/linux/kernel/git/amlogic/linux tags/amlogic-drivers-for-v6.7
+	riscv/drivers
+		https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux tags/riscv-soc-for-v6.7
+	qcom/drivers
+		https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux tags/qcom-drivers-for-6.7
+	rockchips/drivers
+		git://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip tags/v6.7-rockchip-drivers1
+	ti/drivers
+		https://git.kernel.org/pub/scm/linux/kernel/git/ti/linux tags/ti-driver-soc-for-v6.7
+	mvebu/arm
+		git://git.kernel.org/pub/scm/linux/kernel/git/gclement/mvebu tags/mvebu-arm-6.7-1
+	bcm/drivers
+		https://github.com/Broadcom/stblinux tags/arm-soc/for-6.7/drivers
+	qcom/drivers-2
+		https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux tags/qcom-drivers-for-6.7-2
+
+soc/defconfig
+	patch
+		ARM: defconfig: cleanup orphaned CONFIGs
+	renesas/defconfig
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-arm-defconfig-for-v6.7-tag1
+	renesas/defconfig-2
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-arm-defconfig-for-v6.7-tag2
+	imx/defconfig
+		git://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux tags/imx-defconfig-6.7
+	qcom/defconfig
+		https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux tags/qcom-arm64-defconfig-for-6.7
+	aspeed/defconfig
+		git://git.kernel.org/pub/scm/linux/kernel/git/joel/bmc tags/aspeed-6.7-defconfig
+	samsung/defconfig
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux tags/samsung-defconfig-6.7
+	amlogic/defconfig
+		https://git.kernel.org/pub/scm/linux/kernel/git/amlogic/linux tags/amlogic-defconfig-for-v6.7
+	qcom/defconfig-2
+		https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux tags/qcom-arm64-defconfig-for-6.7-2
+
+soc/late
+
+arm/fixes
+	<no branch> (1531309aa2092a96c092fa662863ffa53da3ba93)
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-fixes-for-v6.6-tag2
+	<no branch> (1558b1a8dd388f5fcc3abc1e24de854a295044c3)
+		git://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux tags/imx-fixes-6.6-2
+	qcom/dt-fixes
+		https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux tags/qcom-arm64-fixes-for-6.6
+	riscv/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux tags/riscv-dt-for-v6.6-final
+	patch
+		riscv: dts: thead: set dma-noncoherent to soc bus
+	<no branch> (8cd79b729e746cb167f1563d015a93fc0a079899)
+		git://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip tags/v6.6-rockchip-dtsfixes1
+	omap/fixes
+		git://git.kernel.org/pub/scm/linux/kernel/git/tmlind/linux-omap tags/omap-fixes-audio-clock-and-modem-signed
+	<no branch> (1122a9c2ebe02435014a0f40dcc50f4333286f0f)
+		git://git.kernel.org/pub/scm/linux/kernel/git/gclement/mvebu tags/mvebu-fixes-6.6-1
+	<no branch> (9eab43facdadb7d00456c2657001ae2e5353c814)
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-fixes-for-v6.6-tag3
+
+soc/dt-late
+	patch
+		dt-bindings: arm: add AMD Pensando boards
+		arm64: dts: Add AMD Pensando Elba SoC support
+	ux500/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-nomadik tags/ux500-dts-for-armsoc
+	renesas/dt-bindings
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-dt-bindings-for-v6.7-tag1
+	renesas/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-dts-for-v6.7-tag1
+	dts/cleanup-arm64
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt tags/dt64-cleanup-6.7
+	dts/cleanup-arm32
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt tags/dt-cleanup-6.7
+	samsung/dt64
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux tags/samsung-dt64-6.7
+	samsung/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux tags/samsung-dt-6.7
+	ixp4xx/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-nomadik tags/ixp4xx-for-soc-v6.7
+	at91/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux tags/at91-dt-6.7
+	omap/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/tmlind/linux-omap tags/omap-for-v6.7/dt-signed
+	sti/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/pchotard/sti tags/sti-dt-for-v6.7-round1
+	renesas/dt-bindings-2
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-dt-bindings-for-v6.7-tag2
+	renesas/dt-2
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-dts-for-v6.7-tag2
+	tegra/dt-bindings
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-6.7-dt-bindings
+	tegra/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-6.7-arm-dt
+	tegra/dt64
+		git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux tags/tegra-for-6.7-arm64-dt
+	sunxi/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux tags/sunxi-dt-for-6.7-1
+	riscv/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux tags/riscv-dt-for-v6.6-final
+	imx/dt-bindings
+		git://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux tags/imx-bindings-6.7
+	imx/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux tags/imx-dt-6.7
+	imx/dt64
+		git://git.kernel.org/pub/scm/linux/kernel/git/shawnguo/linux tags/imx-dt64-6.7
+	qcom/dt64
+		https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux tags/qcom-arm64-for-6.7
+	qcom/dt
+		https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux tags/qcom-dts-for-6.7
+	aspeed/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/joel/bmc tags/aspeed-6.7-devicetree
+	amlogic/dt64
+		https://git.kernel.org/pub/scm/linux/kernel/git/amlogic/linux tags/amlogic-arm64-dt-for-v6.7
+	stm32/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/atorgue/stm32 tags/stm32-dt-for-v6.7-1
+	patch
+		arm/arm64: dts: Removed undocumented and unused "pl022,hierarchy" property
+	riscv/dt-2
+		https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux tags/riscv-dt-for-v6.7
+	rockchip/dt64
+		git://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip tags/v6.7-rockchip-dts64-1
+	rockchip/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip tags/v6.7-rockchip-dts32-1
+	mediatek/dts64-2
+		https://git.kernel.org/pub/scm/linux/kernel/git/matthias.bgg/linux tags/v6.6-next-dts64.3
+	<no branch> (56bc311585206a8955de793301d4f84fb4ad2ee6)
+		https://git.kernel.org/pub/scm/linux/kernel/git/ti/linux tags/ti-k3-dt-for-v6.7
+	marvell/dts
+		git://git.kernel.org/pub/scm/linux/kernel/git/gclement/mvebu tags/mvebu-dt64-6.7-1
+	patch
+		arm64: dts: Update cache properties for socionext
+		riscv: dts: thead: convert isa detection to new properties
+		arm64: dts: socionext: add missing cache properties
+	samsung/dt-2
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux tags/samsung-dt64-6.7-2
+	samsung/dt-3
+		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux tags/samsung-dt-6.7-2
+	broadcom/dt
+		https://github.com/Broadcom/stblinux tags/arm-soc/for-6.7/devicetree
+	ti/dt-2
+		https://git.kernel.org/pub/scm/linux/kernel/git/ti/linux tags/ti-k3-dt-for-v6.7-part2
+	rockchip/dt-2
+		git://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip tags/v6.7-rockchip-dts64-2
+	qcom/dt-2
+		https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux tags/qcom-arm64-for-6.7-2
+	qcom/dt32-2
+		https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux tags/qcom-dts-for-6.7-2
+	patch
+		ARM: dts: usr8200: Fix phy registers
+
diff --git a/arch/arm/boot/dts/aspeed/Makefile b/arch/arm/boot/dts/aspeed/Makefile
index d3ac20e316d0..3e3e6b96cb79 100644
--- a/arch/arm/boot/dts/aspeed/Makefile
+++ b/arch/arm/boot/dts/aspeed/Makefile
@@ -9,7 +9,9 @@ dtb-$(CONFIG_ARCH_ASPEED) += \
 	aspeed-bmc-ampere-mtmitchell.dtb \
 	aspeed-bmc-arm-stardragon4800-rep2.dtb \
 	aspeed-bmc-asrock-e3c246d4i.dtb \
+	aspeed-bmc-asrock-e3c256d4i.dtb \
 	aspeed-bmc-asrock-romed8hm3.dtb \
+	aspeed-bmc-asrock-spc621d8hm3.dtb \
 	aspeed-bmc-bytedance-g220a.dtb \
 	aspeed-bmc-delta-ahe50dc.dtb \
 	aspeed-bmc-facebook-bletchley.dtb \
diff --git a/arch/arm/boot/dts/aspeed/aspeed-bmc-asrock-e3c256d4i.dts b/arch/arm/boot/dts/aspeed/aspeed-bmc-asrock-e3c256d4i.dts
new file mode 100644
index 000000000000..64796fb82032
--- /dev/null
+++ b/arch/arm/boot/dts/aspeed/aspeed-bmc-asrock-e3c256d4i.dts
@@ -0,0 +1,316 @@
+// SPDX-License-Identifier: GPL-2.0+
+/dts-v1/;
+
+#include "aspeed-g5.dtsi"
+#include <dt-bindings/gpio/aspeed-gpio.h>
+#include <dt-bindings/i2c/i2c.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/watchdog/aspeed-wdt.h>
+
+/{
+	model = "ASRock E3C256D4I BMC";
+	compatible = "asrock,e3c256d4i-bmc", "aspeed,ast2500";
+
+	aliases {
+		serial4 = &uart5;
+
+		i2c20 = &i2c2mux0ch0;
+		i2c21 = &i2c2mux0ch1;
+		i2c22 = &i2c2mux0ch2;
+		i2c23 = &i2c2mux0ch3;
+	};
+
+	chosen {
+		stdout-path = &uart5;
+		bootargs = "console=tty0 console=ttyS4,115200 earlycon";
+	};
+
+	memory@80000000 {
+		reg = <0x80000000 0x20000000>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		heartbeat {
+			gpios = <&gpio ASPEED_GPIO(H, 6) GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "timer";
+		};
+
+		system-fault {
+			gpios = <&gpio ASPEED_GPIO(Z, 2) GPIO_ACTIVE_LOW>;
+			panic-indicator;
+		};
+	};
+
+	iio-hwmon {
+		compatible = "iio-hwmon";
+		io-channels = <&adc 0>, <&adc 1>, <&adc 2>, <&adc 3>,
+			<&adc 4>, <&adc 5>, <&adc 6>, <&adc 7>,
+			<&adc 8>, <&adc 9>, <&adc 10>, <&adc 11>,
+			<&adc 12>, <&adc 13>, <&adc 14>, <&adc 15>;
+	};
+};
+
+&fmc {
+	status = "okay";
+	flash@0 {
+		status = "okay";
+		m25p,fast-read;
+		label = "bmc";
+		spi-max-frequency = <100000000>; /* 100 MHz */
+#include "openbmc-flash-layout-64.dtsi"
+	};
+};
+
+&uart1 {
+	status = "okay";
+};
+
+&uart2 {
+	status = "okay";
+};
+
+&uart3 {
+	status = "okay";
+};
+
+&uart4 {
+	status = "okay";
+};
+
+&uart5 {
+	status = "okay";
+};
+
+&uart_routing {
+	status = "okay";
+};
+
+&mac0 {
+	status = "okay";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_rgmii1_default &pinctrl_mdio1_default>;
+
+	nvmem-cells = <&eth0_macaddress>;
+	nvmem-cell-names = "mac-address";
+};
+
+&i2c0 {
+	status = "okay";
+};
+
+&i2c1 {
+	status = "okay";
+};
+
+&i2c2 {
+	status = "okay";
+
+	i2c-mux@70 {
+		compatible = "nxp,pca9545";
+		reg = <0x70>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		i2c2mux0ch0: i2c@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+		};
+
+		i2c2mux0ch1: i2c@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+		};
+
+		i2c2mux0ch2: i2c@2 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <2>;
+		};
+
+		i2c2mux0ch3: i2c@3 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <3>;
+		};
+	};
+};
+
+&i2c3 {
+	status = "okay";
+};
+
+&i2c4 {
+	status = "okay";
+};
+
+&i2c5 {
+	status = "okay";
+};
+
+&i2c6 {
+	status = "okay";
+};
+
+&i2c7 {
+	status = "okay";
+};
+
+&i2c9 {
+	status = "okay";
+};
+
+&i2c10 {
+	status = "okay";
+};
+
+&i2c11 {
+	status = "okay";
+
+	vrm@60 {
+		compatible = "renesas,isl69269", "isl69269";
+		reg = <0x60>;
+	};
+};
+
+&i2c12 {
+	status = "okay";
+
+	/* FRU eeprom */
+	eeprom@57 {
+		compatible = "st,24c128", "atmel,24c128";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		reg = <0x57>;
+		pagesize = <16>;
+
+		eth0_macaddress: macaddress@3f80 {
+			reg = <0x3f80 6>;
+		};
+	};
+};
+
+&video {
+	status = "okay";
+};
+
+&vhub {
+	status = "okay";
+};
+
+&lpc_ctrl {
+	status = "okay";
+};
+
+&lpc_snoop {
+	status = "okay";
+	snoop-ports = <0x80>;
+};
+
+&kcs3 {
+	status = "okay";
+	aspeed,lpc-io-reg = <0xca2>;
+};
+
+&peci0 {
+	status = "okay";
+};
+
+&wdt1 {
+	aspeed,reset-mask = <(AST2500_WDT_RESET_DEFAULT & ~AST2500_WDT_RESET_LPC)>;
+};
+
+&wdt2 {
+	aspeed,reset-mask = <(AST2500_WDT_RESET_DEFAULT & ~AST2500_WDT_RESET_LPC)>;
+};
+
+&pwm_tacho {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm0_default /* CPU */
+		&pinctrl_pwm2_default      /* rear */
+		&pinctrl_pwm4_default>;    /* front */
+
+	/* CPU */
+	fan@0 {
+		reg = <0x00>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x00>;
+	};
+
+	/* rear */
+	fan@2 {
+		reg = <0x02>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x02>;
+	};
+
+	/* front */
+	fan@4 {
+		reg = <0x04>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x04>;
+	};
+};
+
+&gpio {
+	status = "okay";
+	gpio-line-names =
+		/*  A */ "", "", "NMI_BTN_N", "BMC_NMI", "", "", "", "",
+		/*  B */ "", "", "", "", "", "", "", "",
+		/*  C */ "", "", "", "", "", "", "", "",
+		/*  D */ "BMC_PSIN", "BMC_PSOUT", "BMC_RESETCON", "RESETCON",
+			"", "", "", "",
+		/*  E */ "", "", "", "", "", "", "", "",
+		/*  F */ "LOCATORLED_STATUS_N", "LOCATORBTN", "", "",
+			"", "", "BMC_PCH_SCI_LPC", "BMC_NCSI_MUX_CTL",
+		/*  G */ "HWM_BAT_EN", "CHASSIS_ID0", "CHASSIS_ID1", "CHASSIS_ID2",
+			"", "", "", "",
+		/*  H */ "FM_ME_RCVR_N", "O_PWROK", "", "D4_DIMM_EVENT_3V_N",
+			"MFG_MODE_N", "BMC_RTCRST", "BMC_HB_LED_N", "BMC_CASEOPEN",
+		/*  I */ "", "", "", "", "", "", "", "",
+		/*  J */ "BMC_READY", "BMC_PCH_BIOS_CS_N", "BMC_SMI", "", "", "", "", "",
+		/*  K */ "", "", "", "", "", "", "", "",
+		/*  L */ "", "", "", "", "", "", "", "",
+		/*  M */ "", "", "", "", "", "", "", "",
+		/*  N */ "", "", "", "", "", "", "", "",
+		/*  O */ "", "", "", "", "", "", "", "",
+		/*  P */ "", "", "", "", "", "", "", "",
+		/*  Q */ "", "", "", "", "", "", "", "",
+		/*  R */ "", "", "", "", "", "", "", "",
+		/*  S */ "PCHHOT_BMC_N", "", "RSMRST", "", "", "", "", "",
+		/*  T */ "", "", "", "", "", "", "", "",
+		/*  U */ "", "", "", "", "", "", "", "",
+		/*  V */ "", "", "", "", "", "", "", "",
+		/*  W */ "", "", "", "", "", "", "", "",
+		/*  X */ "", "", "", "", "", "", "", "",
+		/*  Y */ "SLP_S3", "SLP_S5", "", "", "", "", "", "",
+		/*  Z */ "CPU_CATERR_BMC_N", "", "SYSTEM_FAULT_LED_N", "BMC_THROTTLE_N",
+			"", "", "", "",
+		/* AA */ "CPU1_THERMTRIP_LATCH_N", "", "CPU1_PROCHOT_N", "",
+			"", "", "IRQ_SMI_ACTIVE_N", "FM_BIOS_POST_CMPLT_N",
+		/* AB */ "", "", "ME_OVERRIDE", "BMC_DMI_MODIFY", "", "", "", "",
+		/* AC */ "", "", "", "", "", "", "", "";
+};
+
+&adc {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_adc0_default /* 3VSB */
+		&pinctrl_adc1_default	   /* 5VSB */
+		&pinctrl_adc2_default	   /* CPU1 */
+		&pinctrl_adc3_default	   /* VCCSA */
+		&pinctrl_adc4_default	   /* VCCM */
+		&pinctrl_adc5_default	   /* V10M */
+		&pinctrl_adc6_default	   /* VCCIO */
+		&pinctrl_adc7_default	   /* VCCGT */
+		&pinctrl_adc8_default	   /* VPPM */
+		&pinctrl_adc9_default	   /* BAT */
+		&pinctrl_adc10_default	   /* 3V */
+		&pinctrl_adc11_default	   /* 5V */
+		&pinctrl_adc12_default	   /* 12V */
+		&pinctrl_adc13_default	   /* GND */
+		&pinctrl_adc14_default	   /* GND */
+		&pinctrl_adc15_default>;   /* GND */
+};
diff --git a/arch/arm/boot/dts/aspeed/aspeed-bmc-asrock-spc621d8hm3.dts b/arch/arm/boot/dts/aspeed/aspeed-bmc-asrock-spc621d8hm3.dts
new file mode 100644
index 000000000000..e77435fdb3b1
--- /dev/null
+++ b/arch/arm/boot/dts/aspeed/aspeed-bmc-asrock-spc621d8hm3.dts
@@ -0,0 +1,318 @@
+// SPDX-License-Identifier: GPL-2.0+
+/dts-v1/;
+
+#include "aspeed-g5.dtsi"
+#include <dt-bindings/gpio/aspeed-gpio.h>
+#include <dt-bindings/i2c/i2c.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/{
+	model = "ASRock SPC621D8HM3 BMC";
+	compatible = "asrock,spc621d8hm3-bmc", "aspeed,ast2500";
+
+	aliases {
+		serial4 = &uart5;
+
+		i2c20 = &i2c1mux0ch0;
+		i2c21 = &i2c1mux0ch1;
+	};
+
+	chosen {
+		stdout-path = &uart5;
+		bootargs = "console=tty0 console=ttyS4,115200 earlycon";
+	};
+
+	memory@80000000 {
+		reg = <0x80000000 0x20000000>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		heartbeat {
+			gpios = <&gpio ASPEED_GPIO(H, 6) GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "timer";
+		};
+
+		system-fault {
+			gpios = <&gpio ASPEED_GPIO(Z, 2) GPIO_ACTIVE_LOW>;
+			panic-indicator;
+		};
+	};
+
+	iio-hwmon {
+		compatible = "iio-hwmon";
+		io-channels = <&adc 0>, <&adc 1>, <&adc 2>, <&adc 3>,
+			<&adc 4>, <&adc 5>, <&adc 6>, <&adc 7>,
+			<&adc 8>, <&adc 9>, <&adc 10>, <&adc 11>,
+			<&adc 12>, <&adc 13>, <&adc 14>, <&adc 15>;
+	};
+};
+
+&fmc {
+	status = "okay";
+	flash@0 {
+		status = "okay";
+		m25p,fast-read;
+		label = "bmc";
+		spi-max-frequency = <50000000>; /* 50 MHz */
+#include "openbmc-flash-layout-64.dtsi"
+	};
+};
+
+&uart5 {
+	status = "okay";
+};
+
+&vuart {
+	status = "okay";
+	aspeed,lpc-io-reg = <0x2f8>;
+	aspeed,lpc-interrupts = <3 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&mac0 {
+	status = "okay";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_rgmii1_default &pinctrl_mdio1_default>;
+
+	nvmem-cells = <&eth0_macaddress>;
+	nvmem-cell-names = "mac-address";
+};
+
+&i2c0 {
+	status = "okay";
+};
+
+&i2c1 {
+	status = "okay";
+
+	/* hardware monitor/thermal sensor */
+	temperature-sensor@29 {
+		compatible = "nuvoton,nct7802";
+		reg = <0x29>;
+	};
+
+	/* motherboard temp sensor (TMP1, near BMC) */
+	temperature-sensor@4c {
+		compatible = "nuvoton,w83773g";
+		reg = <0x4c>;
+	};
+
+	/* motherboard FRU eeprom */
+	eeprom@50 {
+		compatible = "st,24c128", "atmel,24c128";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		reg = <0x50>;
+		pagesize = <16>;
+
+		eth0_macaddress: macaddress@3f80 {
+			reg = <0x3f80 6>;
+		};
+	};
+
+	/* M.2 slot smbus mux */
+	i2c-mux@71 {
+		compatible = "nxp,pca9545";
+		reg = <0x71>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		i2c1mux0ch0: i2c@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+		};
+
+		i2c1mux0ch1: i2c@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+		};
+	};
+};
+
+&i2c2 {
+	status = "okay";
+};
+
+&i2c3 {
+	status = "okay";
+};
+
+&i2c4 {
+	status = "okay";
+};
+
+&i2c5 {
+	status = "okay";
+};
+
+&i2c6 {
+	status = "okay";
+};
+
+&i2c7 {
+	status = "okay";
+};
+
+&i2c8 {
+	status = "okay";
+};
+
+&i2c9 {
+	status = "okay";
+};
+
+&i2c10 {
+	status = "okay";
+};
+
+&i2c11 {
+	status = "okay";
+};
+
+&i2c12 {
+	status = "okay";
+};
+
+&i2c13 {
+	status = "okay";
+};
+
+&video {
+	status = "okay";
+};
+
+&vhub {
+	status = "okay";
+};
+
+&lpc_ctrl {
+	status = "okay";
+};
+
+&lpc_snoop {
+	status = "okay";
+	snoop-ports = <0x80>;
+};
+
+&kcs3 {
+	status = "okay";
+	aspeed,lpc-io-reg = <0xca2>;
+};
+
+&peci0 {
+	status = "okay";
+};
+
+&pwm_tacho {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm0_default
+		&pinctrl_pwm2_default
+		&pinctrl_pwm3_default
+		&pinctrl_pwm4_default>;
+
+	fan@0 {
+		reg = <0x00>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x00>;
+	};
+
+	fan@2 {
+		reg = <0x02>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x02>;
+	};
+
+	fan@3 {
+		reg = <0x03>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x03>;
+	};
+
+	fan@4 {
+		reg = <0x04>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x04>;
+	};
+};
+
+&gpio {
+	status = "okay";
+	gpio-line-names =
+		/*  A */ "LOCATORLED_STATUS_N", "LOCATORBTN_N",
+			"BMC_READY_N", "FM_SPD_DDRCPU_LVLSHFT_EN",
+			"", "", "", "",
+		/*  B */ "NODE_ID_1", "NODE_ID_2", "PSU_FAN_FAIL_N", "",
+			"", "", "", "GPIO_RST",
+		/*  C */ "", "", "", "", "", "", "", "",
+		/*  D */ "FP_PWR_BTN_MUX_N", "FM_BMC_PWRBTN_OUT_N",
+			"FP_RST_BTN_N", "RST_BMC_RSTBTN_OUT_N",
+			"NMI_BTN_N", "BMC_NMI",
+			"", "",
+		/*  E */ "", "", "", "FM_ME_RCVR_N", "", "", "", "",
+		/*  F */ "BMC_SMB_SEL_N", "FM_CPU2_DISABLE_COD_N",
+			"FM_REMOTE_DEBUG_BMC_EN", "FM_CPU_ERR0_LVT3_EN",
+			"FM_CPU_ERR1_LVT3_EN", "FM_CPU_ERR2_LVT3_EN",
+			"FM_MEM_THERM_EVENT_CPU1_LVT3_N", "FM_MEM_THERM_EVENT_CPU2_LVT3_N",
+		/*  G */ "HWM_BAT_EN", "", "BMC_PHYRST_N", "FM_BIOS_SPI_BMC_CTRL",
+			"BMC_ALERT1_N", "BMC_ALERT2_N", "BMC_ALERT3_N", "IRQ_SML0_ALERT_N",
+		/*  H */ "BMC_SMB_PRESENT_1_N", "FM_PCH_CORE_VID_0", "FM_PCH_CORE_VID_1", "",
+			"FM_MFG_MODE", "BMC_RTCRST", "BMC_HB_LED_N", "BMC_CASEOPEN",
+		/*  I */ "IRQ_PVDDQ_ABCD_CPU1_VRHOT_LVC3_N", "IRQ_PVDDQ_ABCD_CPU2_VRHOT_LVC3_N",
+			"IRQ_PVDDQ_EFGH_CPU1_VRHOT_LVC3_N", "IRQ_PVDDQ_EFGH_CPU2_VRHOT_LVC3_N",
+			"", "", "", "",
+		/*  J */ "", "", "", "", "", "", "", "",
+		/*  K */ "", "", "", "", "", "", "", "",
+		/*  L */ "", "", "", "", "", "", "", "",
+		/*  M */ "FM_PVCCIN_CPU1_PWR_IN_ALERT_N", "FM_PVCCIN_CPU2_PWR_IN_ALERT_N",
+			"IRQ_PVCCIN_CPU1_VRHOT_LVC3_N", "IRQ_PVCCIN_CPU2_VRHOT_LVC3_N",
+			"FM_CPU1_PROCHOT_BMC_LVC3_N", "",
+			"FM_CPU1_MEMHOT_OUT_N", "FM_CPU2_MEMHOT_OUT_N",
+		/*  N */ "", "", "", "", "", "", "", "",
+		/*  O */ "", "", "", "", "", "", "", "",
+		/*  P */ "", "", "", "", "", "", "", "",
+		/*  Q */ "", "", "", "", "", "", "RST_GLB_RST_WARN_N", "PCIE_WAKE_N",
+		/*  R */ "", "", "FM_BMC_SUSACK_N", "FM_BMC_EUP_LOT6_N",
+			"", "FM_BMC_PCH_SCI_LPC_N", "", "",
+		/*  S */ "FM_DBP_PRESENT_N", "FM_CPU2_SKTOCC_LCT3_N",
+			"FM_CPU1_FIVR_FAULT_LVT3", "FM_CPU2_FIVR_FAULT_LVT3",
+			 "", "", "", "",
+		/*  T */ "", "", "", "", "", "", "", "",
+		/*  U */ "", "", "", "", "", "", "", "",
+		/*  V */ "", "", "", "", "", "", "", "",
+		/*  W */ "", "", "", "", "", "", "", "",
+		/*  X */ "", "", "", "", "", "", "", "",
+		/*  Y */ "FM_SLPS3_N", "FM_SLPS4_N", "", "FM_BMC_ONCTL_N_PLD",
+			"", "", "", "",
+		/*  Z */ "FM_CPU_MSMI_CATERR_LVT3_N", "", "SYSTEM_FAULT_LED_N", "BMC_THROTTLE_N",
+			"", "", "", "",
+		/* AA */ "FM_CPU1_THERMTRIP_LATCH_LVT3_N", "FM_CPU2_THERMTRIP_LATCH_LVT3_N",
+			"FM_BIOS_POST_COMPLT_N", "DBP_BMC_SYSPWROK",
+			"", "IRQ_SML0_ALERT_MUX_N",
+			"IRQ_SMI_ACTIVE_N", "IRQ_NMI_EVENT_N",
+		/* AB */ "FM_PCH_BMC_THERMTRIP_N", "PWRGD_SYS_PWROK",
+			"ME_OVERRIDE", "IRQ_BMC_PCH_SMI_LPC_N",
+			"", "", "", "",
+		/* AC */ "", "", "", "", "", "", "", "";
+};
+
+&adc {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_adc0_default /* 3VSB */
+		&pinctrl_adc1_default	   /* 5VSB */
+		&pinctrl_adc2_default	   /* CPU1 */
+		&pinctrl_adc3_default	   /* NC */
+		&pinctrl_adc4_default	   /* VCCMABCD */
+		&pinctrl_adc5_default	   /* VCCMEFGH */
+		&pinctrl_adc6_default	   /* NC */
+		&pinctrl_adc7_default	   /* NC */
+		&pinctrl_adc8_default	   /* PVNN_PCH */
+		&pinctrl_adc9_default	   /* 1P05PCH */
+		&pinctrl_adc10_default	   /* 1P8PCH */
+		&pinctrl_adc11_default	   /* BAT */
+		&pinctrl_adc12_default	   /* 3V */
+		&pinctrl_adc13_default	   /* 5V */
+		&pinctrl_adc14_default	   /* 12V */
+		&pinctrl_adc15_default>;   /* GND */
+};
diff --git a/arch/arm/boot/dts/intel/ixp/intel-ixp42x-usrobotics-usr8200.dts b/arch/arm/boot/dts/intel/ixp/intel-ixp42x-usrobotics-usr8200.dts
index 90fd51b36e7d..2c89db34c8d8 100644
--- a/arch/arm/boot/dts/intel/ixp/intel-ixp42x-usrobotics-usr8200.dts
+++ b/arch/arm/boot/dts/intel/ixp/intel-ixp42x-usrobotics-usr8200.dts
@@ -165,6 +165,24 @@ mdio {
 				#address-cells = <1>;
 				#size-cells = <0>;
 
+				/*
+				 * PHY 0..4 are internal to the MV88E6060 switch but appear
+				 * as independent devices.
+				 */
+				phy0: ethernet-phy@0 {
+					reg = <0>;
+				};
+				phy1: ethernet-phy@1 {
+					reg = <1>;
+				};
+				phy2: ethernet-phy@2 {
+					reg = <2>;
+				};
+				phy3: ethernet-phy@3 {
+					reg = <3>;
+				};
+
+				/* Altima AMI101L used by the WAN port */
 				phy9: ethernet-phy@9 {
 					reg = <9>;
 				};
@@ -181,21 +199,25 @@ ports {
 						port@0 {
 							reg = <0>;
 							label = "lan1";
+							phy-handle = <&phy0>;
 						};
 
 						port@1 {
 							reg = <1>;
 							label = "lan2";
+							phy-handle = <&phy1>;
 						};
 
 						port@2 {
 							reg = <2>;
 							label = "lan3";
+							phy-handle = <&phy2>;
 						};
 
 						port@3 {
 							reg = <3>;
 							label = "lan4";
+							phy-handle = <&phy3>;
 						};
 
 						port@5 {
diff --git a/arch/arm/boot/dts/microchip/at91-sam9x60_curiosity.dts b/arch/arm/boot/dts/microchip/at91-sam9x60_curiosity.dts
index 83372c1f291b..c6fbdd29019f 100644
--- a/arch/arm/boot/dts/microchip/at91-sam9x60_curiosity.dts
+++ b/arch/arm/boot/dts/microchip/at91-sam9x60_curiosity.dts
@@ -249,6 +249,8 @@ &macb0 {
 
 	ethernet-phy@0 {
 		reg = <0x0>;
+		interrupt-parent = <&pioB>;
+		interrupts = <8 IRQ_TYPE_LEVEL_LOW>;
 	};
 };
 
@@ -353,6 +355,7 @@ AT91_PIOB 4 AT91_PERIPH_A AT91_PINCTRL_NONE	/* PB4 periph A */
 				 AT91_PIOB 5 AT91_PERIPH_A AT91_PINCTRL_NONE	/* PB5 periph A */
 				 AT91_PIOB 6 AT91_PERIPH_A AT91_PINCTRL_NONE	/* PB6 periph A */
 				 AT91_PIOB 7 AT91_PERIPH_A AT91_PINCTRL_NONE	/* PB7 periph A */
+				 AT91_PIOB 8 AT91_PERIPH_GPIO AT91_PINCTRL_NONE	/* PB8 IRQ GPIO */
 				 AT91_PIOB 9 AT91_PERIPH_A AT91_PINCTRL_NONE	/* PB9 periph A */
 				 AT91_PIOB 10 AT91_PERIPH_A AT91_PINCTRL_NONE>;	/* PB10 periph A */
 		};
diff --git a/arch/arm/boot/dts/microchip/at91-sam9x60ek.dts b/arch/arm/boot/dts/microchip/at91-sam9x60ek.dts
index 5cd593028aff..f3cbb675cea4 100644
--- a/arch/arm/boot/dts/microchip/at91-sam9x60ek.dts
+++ b/arch/arm/boot/dts/microchip/at91-sam9x60ek.dts
@@ -292,6 +292,8 @@ &macb0 {
 
 	ethernet-phy@0 {
 		reg = <0x0>;
+		interrupt-parent = <&pioB>;
+		interrupts = <8 IRQ_TYPE_LEVEL_LOW>;
 	};
 };
 
@@ -465,6 +467,7 @@ AT91_PIOB 4 AT91_PERIPH_A AT91_PINCTRL_NONE	/* PB4 periph A */
 				 AT91_PIOB 5 AT91_PERIPH_A AT91_PINCTRL_NONE	/* PB5 periph A */
 				 AT91_PIOB 6 AT91_PERIPH_A AT91_PINCTRL_NONE	/* PB6 periph A */
 				 AT91_PIOB 7 AT91_PERIPH_A AT91_PINCTRL_NONE	/* PB7 periph A */
+				 AT91_PIOB 8 AT91_PERIPH_GPIO AT91_PINCTRL_NONE	/* PB8 IRQ GPIO */
 				 AT91_PIOB 9 AT91_PERIPH_A AT91_PINCTRL_NONE	/* PB9 periph A */
 				 AT91_PIOB 10 AT91_PERIPH_A AT91_PINCTRL_NONE>;	/* PB10 periph A */
 		};
diff --git a/arch/arm/boot/dts/qcom/Makefile b/arch/arm/boot/dts/qcom/Makefile
index 3dfb1c8cefb8..a3d293e40820 100644
--- a/arch/arm/boot/dts/qcom/Makefile
+++ b/arch/arm/boot/dts/qcom/Makefile
@@ -23,12 +23,17 @@ dtb-$(CONFIG_ARCH_QCOM) += \
 	qcom-ipq4019-ap.dk07.1-c2.dtb \
 	qcom-ipq8064-ap148.dtb \
 	qcom-ipq8064-rb3011.dtb \
+	qcom-msm8226-microsoft-dempsey.dtb \
+	qcom-msm8226-microsoft-makepeace.dtb \
+	qcom-msm8226-microsoft-moneypenny.dtb \
 	qcom-msm8226-samsung-s3ve3g.dtb \
 	qcom-msm8660-surf.dtb \
 	qcom-msm8916-samsung-e5.dtb \
 	qcom-msm8916-samsung-e7.dtb \
 	qcom-msm8916-samsung-grandmax.dtb \
 	qcom-msm8916-samsung-serranove.dtb \
+	qcom-msm8926-microsoft-superman-lte.dtb \
+	qcom-msm8926-microsoft-tesla.dtb \
 	qcom-msm8960-cdp.dtb \
 	qcom-msm8960-samsung-expressatt.dtb \
 	qcom-msm8974-lge-nexus5-hammerhead.dtb \
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8226-microsoft-common.dtsi b/arch/arm/boot/dts/qcom/qcom-msm8226-microsoft-common.dtsi
new file mode 100644
index 000000000000..a6fd3acf0bae
--- /dev/null
+++ b/arch/arm/boot/dts/qcom/qcom-msm8226-microsoft-common.dtsi
@@ -0,0 +1,327 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Common Board Device Tree for Microsoft MSM8x26-based Lumias
+ *
+ * Copyright (c) 2023, Jack Matthews <jm5112356@gmail.com>
+ * Copyright (c) 2023, Ivaylo Ivanov <ivo.ivanov.ivanov1@gmail.com>
+ * Copyright (c) 2023, Dominik Kobinski <dominikkobinski314@gmail.com>
+ * Copyright (c) 2023, Rayyan Ansari <rayyan@ansari.sh>
+ */
+
+#include "qcom-msm8226.dtsi"
+#include "qcom-pm8226.dtsi"
+#include <dt-bindings/input/input.h>
+
+/*
+ * Delete all generic (msm8226.dtsi) reserved
+ * memory mappings which are different on these devices.
+ */
+/delete-node/ &smem_region;
+
+/ {
+	aliases {
+		mmc0 = &sdhc_1; /* eMMC */
+		mmc1 = &sdhc_2; /* microSD */
+		display0 = &framebuffer;
+	};
+
+	chosen {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		stdout-path = "display0";
+
+		framebuffer: framebuffer@3200000 {
+			compatible = "simple-framebuffer";
+			reg = <0x3200000 0x800000>;
+			format = "a8r8g8b8";
+			width = <720>;
+			height = <1280>;
+			stride = <(720 * 4)>;
+
+			clocks = <&mmcc MDSS_AHB_CLK>,
+				 <&mmcc MDSS_AXI_CLK>,
+				 <&mmcc MDSS_BYTE0_CLK>,
+				 <&mmcc MDSS_MDP_CLK>,
+				 <&mmcc MDSS_PCLK0_CLK>,
+				 <&mmcc MDSS_VSYNC_CLK>;
+			power-domains = <&mmcc MDSS_GDSC>;
+		};
+	};
+
+	gpio_keys: gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&gpio_keys_default>;
+		pinctrl-names = "default";
+
+		label = "GPIO Buttons";
+
+		key-volume-up {
+			label = "Volume Up";
+			gpios = <&tlmm 106 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEUP>;
+		};
+	};
+
+	/*
+	 * This device being a WP platform has a different
+	 * memory layout than other Android based devices.
+	 * This smem memory region is directly copied from
+	 * the original UEFI firmware.
+	 */
+	reserved-memory {
+		display_reserved: framebuffer@3200000 {
+			reg = <0x03200000 0x800000>;
+			no-map;
+		};
+
+		smem_region: smem@fa00000 {
+			reg = <0x0fa00000 0x100000>;
+			no-map;
+		};
+	};
+};
+
+&blsp1_i2c5 {
+	status = "okay";
+
+	touchscreen: touchscreen@4b {
+		compatible = "syna,rmi4-i2c";
+		reg = <0x4b>;
+
+		interrupts-extended = <&tlmm 17 IRQ_TYPE_EDGE_FALLING>;
+		vdd-supply = <&pm8226_l15>;
+		vio-supply = <&pm8226_l6>;
+
+		pinctrl-0 = <&touchscreen_default>;
+		pinctrl-names = "default";
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		rmi4-f01@1 {
+			reg = <0x01>;
+			syna,nosleep-mode = <1>;
+		};
+
+		rmi4-f11@11 {
+			reg = <0x11>;
+			syna,sensor-type = <1>;
+		};
+	};
+};
+
+&blsp1_uart3 {
+	status = "okay";
+};
+
+&pm8226_resin {
+	linux,code = <KEY_VOLUMEDOWN>;
+	status = "okay";
+};
+
+&pm8226_vib {
+	status = "okay";
+};
+
+&rpm_requests {
+	regulators {
+		compatible = "qcom,rpm-pm8226-regulators";
+
+		/* These values were taken from the original firmware DSDT */
+		pm8226_s1: s1 {
+			regulator-min-microvolt = <1050000>;
+			regulator-max-microvolt = <1050000>;
+		};
+
+		pm8226_s3: s3 {
+			regulator-min-microvolt = <1300000>;
+			regulator-max-microvolt = <1300000>;
+		};
+
+		pm8226_s4: s4 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2100000>;
+		};
+
+		pm8226_s5: s5 {
+			regulator-min-microvolt = <1150000>;
+			regulator-max-microvolt = <1150000>;
+		};
+
+		pm8226_l3: l3 {
+			regulator-min-microvolt = <750000>;
+			regulator-max-microvolt = <1150000>;
+		};
+
+		pm8226_l4: l4 {
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+		};
+
+		pm8226_l5: l5 {
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+		};
+
+		pm8226_l6: l6 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+
+		pm8226_l7: l7 {
+			regulator-min-microvolt = <1850000>;
+			regulator-max-microvolt = <1850000>;
+		};
+
+		pm8226_l8: l8 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-always-on;
+		};
+
+		pm8226_l9: l9 {
+			regulator-min-microvolt = <2050000>;
+			regulator-max-microvolt = <2050000>;
+		};
+
+		pm8226_l10: l10 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+
+		pm8226_l12: l12 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+
+		pm8226_l14: l14 {
+			regulator-min-microvolt = <2750000>;
+			regulator-max-microvolt = <2750000>;
+		};
+
+		pm8226_l15: l15 {
+			regulator-min-microvolt = <2850000>;
+			regulator-max-microvolt = <2850000>;
+		};
+
+		pm8226_l16: l16 {
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3000000>;
+		};
+
+		pm8226_l17: l17 {
+			regulator-min-microvolt = <2950000>;
+			regulator-max-microvolt = <2950000>;
+			regulator-system-load = <200000>;
+			regulator-allow-set-load;
+		};
+
+		pm8226_l18: l18 {
+			regulator-min-microvolt = <2950000>;
+			regulator-max-microvolt = <2950000>;
+			regulator-system-load = <200000>;
+			regulator-allow-set-load;
+		};
+
+		pm8226_l19: l19 {
+			regulator-min-microvolt = <2850000>;
+			regulator-max-microvolt = <2850000>;
+		};
+
+		pm8226_l20: l20 {
+			regulator-min-microvolt = <3075000>;
+			regulator-max-microvolt = <3075000>;
+		};
+
+		pm8226_l21: l21 {
+			regulator-min-microvolt = <1850000>;
+			regulator-max-microvolt = <2950000>;
+		};
+
+		pm8226_l22: l22 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+
+		pm8226_l23: l23 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+
+		pm8226_l24: l24 {
+			regulator-min-microvolt = <1300000>;
+			regulator-max-microvolt = <1300000>;
+		};
+
+		pm8226_l25: l25 {
+			regulator-min-microvolt = <1775000>;
+			regulator-max-microvolt = <2075000>;
+		};
+
+		pm8226_l27: l27 {
+			regulator-min-microvolt = <2050000>;
+			regulator-max-microvolt = <2050000>;
+		};
+
+		pm8226_l28: l28 {
+			regulator-min-microvolt = <2000000>;
+			regulator-max-microvolt = <3000000>;
+		};
+	};
+};
+
+&sdhc_1 {
+	vmmc-supply = <&pm8226_l17>;
+	vqmmc-supply = <&pm8226_l6>;
+
+	status = "okay";
+};
+
+&sdhc_2 {
+	vmmc-supply = <&pm8226_l18>;
+	vqmmc-supply = <&pm8226_l21>;
+
+	status = "okay";
+};
+
+&usb {
+	extcon = <&smbb>;
+	dr_mode = "peripheral";
+
+	status = "okay";
+};
+
+&usb_hs_phy {
+	extcon = <&smbb>;
+	v1p8-supply = <&pm8226_l10>;
+	v3p3-supply = <&pm8226_l20>;
+};
+
+&tlmm {
+	gpio_keys_default: gpio-keys-default-state {
+		pins = "gpio106", "gpio107", "gpio108";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	touchscreen_default: touchscreen-default-state {
+		irq-pins {
+			pins = "gpio17";
+			function = "gpio";
+			drive-strength = <8>;
+			bias-pull-up;
+		};
+
+		reset-pins {
+			pins = "gpio16";
+			function = "gpio";
+			drive-strength = <8>;
+			bias-disable;
+			output-high;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8226-microsoft-dempsey.dts b/arch/arm/boot/dts/qcom/qcom-msm8226-microsoft-dempsey.dts
new file mode 100644
index 000000000000..2c664b5934ec
--- /dev/null
+++ b/arch/arm/boot/dts/qcom/qcom-msm8226-microsoft-dempsey.dts
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2023, Jack Matthews <jm5112356@gmail.com>
+ * Copyright (c) 2023, Ivaylo Ivanov <ivo.ivanov.ivanov1@gmail.com>
+ * Copyright (c) 2023, Dominik Kobinski <dominikkobinski314@gmail.com>
+ * Copyright (c) 2023, Rayyan Ansari <rayyan@ansari.sh>
+ */
+
+/dts-v1/;
+
+#include "qcom-msm8226-microsoft-common.dtsi"
+
+/ {
+	model = "Microsoft Lumia 640";
+	compatible = "microsoft,dempsey", "qcom,msm8226";
+	chassis-type = "handset";
+};
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8226-microsoft-makepeace.dts b/arch/arm/boot/dts/qcom/qcom-msm8226-microsoft-makepeace.dts
new file mode 100644
index 000000000000..731c5c375678
--- /dev/null
+++ b/arch/arm/boot/dts/qcom/qcom-msm8226-microsoft-makepeace.dts
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2023, Jack Matthews <jm5112356@gmail.com>
+ * Copyright (c) 2023, Ivaylo Ivanov <ivo.ivanov.ivanov1@gmail.com>
+ * Copyright (c) 2023, Dominik Kobinski <dominikkobinski314@gmail.com>
+ * Copyright (c) 2023, Rayyan Ansari <rayyan@ansari.sh>
+ */
+
+/dts-v1/;
+
+#include "qcom-msm8226-microsoft-common.dtsi"
+
+/ {
+	model = "Microsoft Lumia 640 XL";
+	compatible = "microsoft,makepeace", "qcom,msm8226";
+	chassis-type = "handset";
+};
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8226-microsoft-moneypenny.dts b/arch/arm/boot/dts/qcom/qcom-msm8226-microsoft-moneypenny.dts
new file mode 100644
index 000000000000..992b7115b5f8
--- /dev/null
+++ b/arch/arm/boot/dts/qcom/qcom-msm8226-microsoft-moneypenny.dts
@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2023, Jack Matthews <jm5112356@gmail.com>
+ * Copyright (c) 2023, Ivaylo Ivanov <ivo.ivanov.ivanov1@gmail.com>
+ * Copyright (c) 2023, Dominik Kobinski <dominikkobinski314@gmail.com>
+ * Copyright (c) 2023, Rayyan Ansari <rayyan@ansari.sh>
+ */
+
+/dts-v1/;
+
+#include "qcom-msm8226-microsoft-common.dtsi"
+
+/ {
+	model = "Nokia Lumia 630";
+	compatible = "microsoft,moneypenny", "qcom,msm8226";
+	chassis-type = "handset";
+};
+
+&framebuffer {
+	width = <480>;
+	height = <854>;
+	stride = <(480 * 4)>;
+};
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8226.dtsi b/arch/arm/boot/dts/qcom/qcom-msm8226.dtsi
index 97a377b5a0ec..5cd03ea7b084 100644
--- a/arch/arm/boot/dts/qcom/qcom-msm8226.dtsi
+++ b/arch/arm/boot/dts/qcom/qcom-msm8226.dtsi
@@ -442,8 +442,8 @@ mmcc: clock-controller@fd8c0000 {
 				 <&gcc GPLL0_VOTE>,
 				 <&gcc GPLL1_VOTE>,
 				 <&rpmcc RPM_SMD_GFX3D_CLK_SRC>,
-				 <0>,
-				 <0>;
+				 <&mdss_dsi0_phy 1>,
+				 <&mdss_dsi0_phy 0>;
 			clock-names = "xo",
 				      "mmss_gpll0_vote",
 				      "gpll0_vote",
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8926-microsoft-superman-lte.dts b/arch/arm/boot/dts/qcom/qcom-msm8926-microsoft-superman-lte.dts
new file mode 100644
index 000000000000..9b48661d69c5
--- /dev/null
+++ b/arch/arm/boot/dts/qcom/qcom-msm8926-microsoft-superman-lte.dts
@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2023, Jack Matthews <jm5112356@gmail.com>
+ * Copyright (c) 2023, Ivaylo Ivanov <ivo.ivanov.ivanov1@gmail.com>
+ * Copyright (c) 2023, Dominik Kobinski <dominikkobinski314@gmail.com>
+ * Copyright (c) 2023, Rayyan Ansari <rayyan@ansari.sh>
+ */
+
+/dts-v1/;
+
+#include "qcom-msm8226-microsoft-common.dtsi"
+
+/* This device has touchscreen on i2c3 instead */
+/delete-node/ &touchscreen;
+
+/ {
+	model = "Nokia Lumia 735";
+	compatible = "microsoft,superman-lte", "qcom,msm8926", "qcom,msm8226";
+	chassis-type = "handset";
+};
+
+&blsp1_i2c3 {
+	status = "okay";
+
+	touchscreen: touchscreen@4b {
+		compatible = "syna,rmi4-i2c";
+		reg = <0x4b>;
+
+		interrupts-extended = <&tlmm 17 IRQ_TYPE_EDGE_FALLING>;
+		vdd-supply = <&pm8226_l15>;
+		vio-supply = <&pm8226_l6>;
+
+		pinctrl-0 = <&touchscreen_default>;
+		pinctrl-names = "default";
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		rmi4-f01@1 {
+			reg = <0x01>;
+			syna,nosleep-mode = <1>;
+		};
+
+		rmi4-f11@11 {
+			reg = <0x11>;
+			syna,sensor-type = <1>;
+		};
+	};
+};
+
+&blsp1_i2c5 {
+	status = "disabled";
+};
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8926-microsoft-tesla.dts b/arch/arm/boot/dts/qcom/qcom-msm8926-microsoft-tesla.dts
new file mode 100644
index 000000000000..53a6d4e85959
--- /dev/null
+++ b/arch/arm/boot/dts/qcom/qcom-msm8926-microsoft-tesla.dts
@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2023, Jack Matthews <jm5112356@gmail.com>
+ * Copyright (c) 2023, Ivaylo Ivanov <ivo.ivanov.ivanov1@gmail.com>
+ * Copyright (c) 2023, Dominik Kobinski <dominikkobinski314@gmail.com>
+ * Copyright (c) 2023, Rayyan Ansari <rayyan@ansari.sh>
+ */
+
+/dts-v1/;
+
+#include "qcom-msm8226-microsoft-common.dtsi"
+
+/* This device has touchscreen on i2c1 instead */
+/delete-node/ &touchscreen;
+
+/ {
+	model = "Nokia Lumia 830";
+	compatible = "microsoft,tesla", "qcom,msm8926", "qcom,msm8226";
+	chassis-type = "handset";
+};
+
+&blsp1_i2c1 {
+	status = "okay";
+
+	touchscreen: touchscreen@4b {
+		compatible = "syna,rmi4-i2c";
+		reg = <0x4b>;
+
+		interrupts-extended = <&tlmm 17 IRQ_TYPE_EDGE_FALLING>;
+		vdd-supply = <&pm8226_l15>;
+		vio-supply = <&pm8226_l6>;
+
+		pinctrl-0 = <&touchscreen_default>;
+		pinctrl-names = "default";
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		rmi4-f01@1 {
+			reg = <0x01>;
+			syna,nosleep-mode = <1>;
+		};
+
+		rmi4-f12@12 {
+			reg = <0x12>;
+			syna,sensor-type = <1>;
+		};
+	};
+};
+
+&blsp1_i2c5 {
+	status = "disabled";
+};
+
+&gpio_keys {
+	key-camera-snapshot {
+		label = "Camera Snapshot";
+		gpios = <&tlmm 107 GPIO_ACTIVE_LOW>;
+		linux,code = <KEY_CAMERA>;
+	};
+
+	key-camera-focus {
+		label = "Camera Focus";
+		gpios = <&tlmm 108 GPIO_ACTIVE_LOW>;
+		linux,code = <KEY_CAMERA_FOCUS>;
+	};
+};
diff --git a/arch/arm/boot/dts/qcom/qcom-msm8974.dtsi b/arch/arm/boot/dts/qcom/qcom-msm8974.dtsi
index 0bc2e66d15b1..76006c3c4af2 100644
--- a/arch/arm/boot/dts/qcom/qcom-msm8974.dtsi
+++ b/arch/arm/boot/dts/qcom/qcom-msm8974.dtsi
@@ -346,6 +346,12 @@ apcs: syscon@f9011000 {
 			reg = <0xf9011000 0x1000>;
 		};
 
+		saw_l2: power-controller@f9012000 {
+			compatible = "qcom,saw2";
+			reg = <0xf9012000 0x1000>;
+			regulator;
+		};
+
 		timer@f9020000 {
 			#address-cells = <1>;
 			#size-cells = <1>;
@@ -405,52 +411,46 @@ frame@f9028000 {
 			};
 		};
 
-		saw0: power-controller@f9089000 {
-			compatible = "qcom,msm8974-saw2-v2.1-cpu", "qcom,saw2";
-			reg = <0xf9089000 0x1000>, <0xf9009000 0x1000>;
-		};
-
-		saw1: power-controller@f9099000 {
-			compatible = "qcom,msm8974-saw2-v2.1-cpu", "qcom,saw2";
-			reg = <0xf9099000 0x1000>, <0xf9009000 0x1000>;
-		};
-
-		saw2: power-controller@f90a9000 {
-			compatible = "qcom,msm8974-saw2-v2.1-cpu", "qcom,saw2";
-			reg = <0xf90a9000 0x1000>, <0xf9009000 0x1000>;
-		};
-
-		saw3: power-controller@f90b9000 {
-			compatible = "qcom,msm8974-saw2-v2.1-cpu", "qcom,saw2";
-			reg = <0xf90b9000 0x1000>, <0xf9009000 0x1000>;
-		};
-
-		saw_l2: power-controller@f9012000 {
-			compatible = "qcom,saw2";
-			reg = <0xf9012000 0x1000>;
-			regulator;
-		};
-
 		acc0: power-manager@f9088000 {
 			compatible = "qcom,kpss-acc-v2";
 			reg = <0xf9088000 0x1000>, <0xf9008000 0x1000>;
 		};
 
+		saw0: power-controller@f9089000 {
+			compatible = "qcom,msm8974-saw2-v2.1-cpu", "qcom,saw2";
+			reg = <0xf9089000 0x1000>, <0xf9009000 0x1000>;
+		};
+
 		acc1: power-manager@f9098000 {
 			compatible = "qcom,kpss-acc-v2";
 			reg = <0xf9098000 0x1000>, <0xf9008000 0x1000>;
 		};
 
+		saw1: power-controller@f9099000 {
+			compatible = "qcom,msm8974-saw2-v2.1-cpu", "qcom,saw2";
+			reg = <0xf9099000 0x1000>, <0xf9009000 0x1000>;
+		};
+
 		acc2: power-manager@f90a8000 {
 			compatible = "qcom,kpss-acc-v2";
 			reg = <0xf90a8000 0x1000>, <0xf9008000 0x1000>;
 		};
 
+		saw2: power-controller@f90a9000 {
+			compatible = "qcom,msm8974-saw2-v2.1-cpu", "qcom,saw2";
+			reg = <0xf90a9000 0x1000>, <0xf9009000 0x1000>;
+		};
+
 		acc3: power-manager@f90b8000 {
 			compatible = "qcom,kpss-acc-v2";
 			reg = <0xf90b8000 0x1000>, <0xf9008000 0x1000>;
 		};
 
+		saw3: power-controller@f90b9000 {
+			compatible = "qcom,msm8974-saw2-v2.1-cpu", "qcom,saw2";
+			reg = <0xf90b9000 0x1000>, <0xf9009000 0x1000>;
+		};
+
 		sdhc_1: mmc@f9824900 {
 			compatible = "qcom,msm8974-sdhci", "qcom,sdhci-msm-v4";
 			reg = <0xf9824900 0x11c>, <0xf9824000 0x800>;
@@ -1063,6 +1063,15 @@ kpss_out: endpoint {
 			};
 		};
 
+		bimc: interconnect@fc380000 {
+			reg = <0xfc380000 0x6a000>;
+			compatible = "qcom,msm8974-bimc";
+			#interconnect-cells = <1>;
+			clock-names = "bus", "bus_a";
+			clocks = <&rpmcc RPM_SMD_BIMC_CLK>,
+				 <&rpmcc RPM_SMD_BIMC_A_CLK>;
+		};
+
 		gcc: clock-controller@fc400000 {
 			compatible = "qcom,gcc-msm8974";
 			#clock-cells = <1>;
@@ -1101,22 +1110,13 @@ pronto_master_stats: sram@1f50 {
 			};
 		};
 
-		bimc: interconnect@fc380000 {
-			reg = <0xfc380000 0x6a000>;
-			compatible = "qcom,msm8974-bimc";
-			#interconnect-cells = <1>;
-			clock-names = "bus", "bus_a";
-			clocks = <&rpmcc RPM_SMD_BIMC_CLK>,
-			         <&rpmcc RPM_SMD_BIMC_A_CLK>;
-		};
-
 		snoc: interconnect@fc460000 {
 			reg = <0xfc460000 0x4000>;
 			compatible = "qcom,msm8974-snoc";
 			#interconnect-cells = <1>;
 			clock-names = "bus", "bus_a";
 			clocks = <&rpmcc RPM_SMD_SNOC_CLK>,
-			         <&rpmcc RPM_SMD_SNOC_A_CLK>;
+				 <&rpmcc RPM_SMD_SNOC_A_CLK>;
 		};
 
 		pnoc: interconnect@fc468000 {
@@ -1125,7 +1125,7 @@ pnoc: interconnect@fc468000 {
 			#interconnect-cells = <1>;
 			clock-names = "bus", "bus_a";
 			clocks = <&rpmcc RPM_SMD_PNOC_CLK>,
-			         <&rpmcc RPM_SMD_PNOC_A_CLK>;
+				 <&rpmcc RPM_SMD_PNOC_A_CLK>;
 		};
 
 		ocmemnoc: interconnect@fc470000 {
@@ -1134,7 +1134,7 @@ ocmemnoc: interconnect@fc470000 {
 			#interconnect-cells = <1>;
 			clock-names = "bus", "bus_a";
 			clocks = <&rpmcc RPM_SMD_OCMEMGX_CLK>,
-			         <&rpmcc RPM_SMD_OCMEMGX_A_CLK>;
+				 <&rpmcc RPM_SMD_OCMEMGX_A_CLK>;
 		};
 
 		mmssnoc: interconnect@fc478000 {
@@ -1143,7 +1143,7 @@ mmssnoc: interconnect@fc478000 {
 			#interconnect-cells = <1>;
 			clock-names = "bus", "bus_a";
 			clocks = <&mmcc MMSS_S0_AXI_CLK>,
-			         <&mmcc MMSS_S0_AXI_CLK>;
+				 <&mmcc MMSS_S0_AXI_CLK>;
 		};
 
 		cnoc: interconnect@fc480000 {
@@ -1152,7 +1152,7 @@ cnoc: interconnect@fc480000 {
 			#interconnect-cells = <1>;
 			clock-names = "bus", "bus_a";
 			clocks = <&rpmcc RPM_SMD_CNOC_CLK>,
-			         <&rpmcc RPM_SMD_CNOC_A_CLK>;
+				 <&rpmcc RPM_SMD_CNOC_A_CLK>;
 		};
 
 		tsens: thermal-sensor@fc4a9000 {
diff --git a/arch/arm/boot/dts/renesas/iwg20d-q7-dbcm-ca.dtsi b/arch/arm/boot/dts/renesas/iwg20d-q7-dbcm-ca.dtsi
index e10f99278c77..de52218ceaa4 100644
--- a/arch/arm/boot/dts/renesas/iwg20d-q7-dbcm-ca.dtsi
+++ b/arch/arm/boot/dts/renesas/iwg20d-q7-dbcm-ca.dtsi
@@ -27,6 +27,15 @@ hdmi_con_out: endpoint {
 			};
 		};
 	};
+
+	reg_1p8v: regulator-1p8v {
+		compatible = "regulator-fixed";
+		regulator-name = "1P8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
 };
 
 &can1 {
@@ -69,6 +78,12 @@ hdmi@39 {
 		clocks = <&cec_clock>;
 		clock-names = "cec";
 
+		avdd-supply = <&reg_1p8v>;
+		dvdd-supply = <&reg_1p8v>;
+		pvdd-supply = <&reg_1p8v>;
+		dvdd-3v-supply = <&reg_3p3v>;
+		bgvdd-supply = <&reg_1p8v>;
+
 		adi,input-depth = <8>;
 		adi,input-colorspace = "rgb";
 		adi,input-clock = "1x";
diff --git a/arch/arm/boot/dts/renesas/r8a7745-iwg22d-sodimm-dbhd-ca.dts b/arch/arm/boot/dts/renesas/r8a7745-iwg22d-sodimm-dbhd-ca.dts
index b1f679da36b2..a0b574398055 100644
--- a/arch/arm/boot/dts/renesas/r8a7745-iwg22d-sodimm-dbhd-ca.dts
+++ b/arch/arm/boot/dts/renesas/r8a7745-iwg22d-sodimm-dbhd-ca.dts
@@ -34,6 +34,15 @@ hdmi_con: endpoint {
 			};
 		};
 	};
+
+	reg_1p8v: regulator-1p8v {
+		compatible = "regulator-fixed";
+		regulator-name = "1P8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
 };
 
 &du {
@@ -81,6 +90,12 @@ hdmi@39 {
 		clock-names = "cec";
 		pd-gpios = <&gpio2 24 GPIO_ACTIVE_HIGH>;
 
+		avdd-supply = <&reg_1p8v>;
+		dvdd-supply = <&reg_1p8v>;
+		pvdd-supply = <&reg_1p8v>;
+		dvdd-3v-supply = <&reg_3p3v>;
+		bgvdd-supply = <&reg_1p8v>;
+
 		adi,input-depth = <8>;
 		adi,input-colorspace = "rgb";
 		adi,input-clock = "1x";
diff --git a/arch/arm/boot/dts/renesas/r8a7745-iwg22d-sodimm.dts b/arch/arm/boot/dts/renesas/r8a7745-iwg22d-sodimm.dts
index c105932f642e..24411044ef6c 100644
--- a/arch/arm/boot/dts/renesas/r8a7745-iwg22d-sodimm.dts
+++ b/arch/arm/boot/dts/renesas/r8a7745-iwg22d-sodimm.dts
@@ -182,7 +182,7 @@ sgtl5000: codec@a {
 		VDDIO-supply = <&reg_3p3v>;
 	};
 
-	stmpe811@44 {
+	port-expander@44 {
 		compatible = "st,stmpe811";
 		reg = <0x44>;
 		interrupt-parent = <&gpio4>;
@@ -197,7 +197,7 @@ stmpe811@44 {
 		/* internal ADC reference */
 		st,ref-sel = <0>;
 
-		stmpe_touchscreen {
+		touchscreen {
 			compatible = "st,stmpe-ts";
 			/* 8 sample average control */
 			st,ave-ctrl = <3>;
diff --git a/arch/arm/boot/dts/renesas/r8a7779-marzen.dts b/arch/arm/boot/dts/renesas/r8a7779-marzen.dts
index 08ea149b1ee6..9b13e8d1538b 100644
--- a/arch/arm/boot/dts/renesas/r8a7779-marzen.dts
+++ b/arch/arm/boot/dts/renesas/r8a7779-marzen.dts
@@ -52,10 +52,10 @@ vccq_sdhi0: regulator-vccq-sdhi0 {
 		states = <3300000 1>, <1800000 0>;
 	};
 
-	keyboard-irq {
+	keypad-0 {
 		compatible = "gpio-keys";
 
-		pinctrl-0 = <&keyboard_irq_pins>;
+		pinctrl-0 = <&keypad0_pins>;
 		pinctrl-names = "default";
 
 		interrupt-parent = <&gpio0>;
@@ -76,11 +76,11 @@ key-2 {
 		};
 	};
 
-	keyboard-gpio {
+	keypad-1 {
 		compatible = "gpio-keys-polled";
 		poll-interval = <50>;
 
-		pinctrl-0 = <&keyboard_gpio_pins>;
+		pinctrl-0 = <&keypad1_pins>;
 		pinctrl-names = "default";
 
 		key-3 {
@@ -193,7 +193,7 @@ endpoint {
 };
 
 &gpio0 {
-	keyboard-irq-hog {
+	keypad0-hog {
 		gpio-hog;
 		gpios = <17 GPIO_ACTIVE_LOW>, <18 GPIO_ACTIVE_LOW>;
 		input;
@@ -215,6 +215,35 @@ &extal_clk {
 };
 
 &lbsc {
+	flash@0 {
+		compatible = "cfi-flash";
+		reg = <0x0 0x04000000>;
+		pinctrl-0 = <&flash_pins>;
+		pinctrl-names = "default";
+		bank-width = <2>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "uboot";
+				reg = <0x00000000 0x00040000>;
+				read-only;
+			};
+			partition@40000 {
+				label = "uboot-env";
+				reg = <0x00040000 0x00040000>;
+				read-only;
+			};
+			partition@80000 {
+				label = "flash";
+				reg = <0x00080000 0x03f80000>;
+			};
+		};
+	};
+
 	ethernet@18000000 {
 		compatible = "smsc,lan89218", "smsc,lan9115";
 		reg = <0x18000000 0x100>;
@@ -266,6 +295,11 @@ lbsc {
 		};
 	};
 
+	flash_pins: flash {
+		groups = "lbsc_cs0";
+		function = "lbsc";
+	};
+
 	scif2_pins: scif2 {
 		groups = "scif2_data_c";
 		function = "scif2";
@@ -286,11 +320,11 @@ hspi0_pins: hspi0 {
 		function = "hspi0";
 	};
 
-	keyboard_irq_pins: keyboard-irq {
+	keypad0_pins: keypad-0 {
 		pins = "GP_0_17", "GP_0_18";
 		bias-pull-up;
 	};
-	keyboard_gpio_pins: keyboard-gpio {
+	keypad1_pins: keypad-1 {
 		pins = "GP_0_19", "GP_0_20";
 		bias-pull-up;
 	};
diff --git a/arch/arm/boot/dts/renesas/r8a7790-lager.dts b/arch/arm/boot/dts/renesas/r8a7790-lager.dts
index 4d666ad8b114..2fba4d084001 100644
--- a/arch/arm/boot/dts/renesas/r8a7790-lager.dts
+++ b/arch/arm/boot/dts/renesas/r8a7790-lager.dts
@@ -122,6 +122,15 @@ led8 {
 		};
 	};
 
+	fixedregulator1v8: regulator-1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-1.8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
 	fixedregulator3v3: regulator-3v3 {
 		compatible = "regulator-fixed";
 		regulator-name = "fixed-3.3V";
@@ -303,7 +312,7 @@ gpioi2c2: i2c-9 {
 	 *
 	 * IIC0/I2C0 does not appear to support fallback to GPIO.
 	 */
-	i2cexio0: i2c-10 {
+	i2cexio0: i2c-mux1 {
 		compatible = "i2c-demux-pinctrl";
 		i2c-parent = <&iic0>, <&i2c0>;
 		i2c-bus-name = "i2c-exio0";
@@ -316,7 +325,7 @@ i2cexio0: i2c-10 {
 	 * This is similar to the arangement described for i2cexio0 (above)
 	 * with a fallback to GPIO also provided.
 	 */
-	i2cexio1: i2c-11 {
+	i2cexio1: i2c-mux2 {
 		compatible = "i2c-demux-pinctrl";
 		i2c-parent = <&iic1>, <&i2c1>, <&gpioi2c1>;
 		i2c-bus-name = "i2c-exio1";
@@ -328,7 +337,7 @@ i2cexio1: i2c-11 {
 	 * IIC2 and I2C2 may be switched using pinmux.
 	 * A fallback to GPIO is also provided.
 	 */
-	i2chdmi: i2c-12 {
+	i2chdmi: i2c-mux3 {
 		compatible = "i2c-demux-pinctrl";
 		i2c-parent = <&iic2>, <&i2c2>, <&gpioi2c2>;
 		i2c-bus-name = "i2c-hdmi";
@@ -361,6 +370,12 @@ hdmi@39 {
 			clocks = <&cec_clock>;
 			clock-names = "cec";
 
+			avdd-supply = <&fixedregulator1v8>;
+			dvdd-supply = <&fixedregulator1v8>;
+			pvdd-supply = <&fixedregulator1v8>;
+			dvdd-3v-supply = <&fixedregulator3v3>;
+			bgvdd-supply = <&fixedregulator1v8>;
+
 			adi,input-depth = <8>;
 			adi,input-colorspace = "rgb";
 			adi,input-clock = "1x";
@@ -417,7 +432,7 @@ adv7612_out: endpoint {
 	 * IIC3 and I2C3 may be switched using pinmux.
 	 * IIC3/I2C3 does not appear to support fallback to GPIO.
 	 */
-	i2cpwr: i2c-13 {
+	i2cpwr: i2c-mux4 {
 		compatible = "i2c-demux-pinctrl";
 		pinctrl-names = "default";
 		pinctrl-0 = <&pmic_irq_pins>;
diff --git a/arch/arm/boot/dts/renesas/r8a7790-stout.dts b/arch/arm/boot/dts/renesas/r8a7790-stout.dts
index fe14727eefe1..f9bc5b4f019d 100644
--- a/arch/arm/boot/dts/renesas/r8a7790-stout.dts
+++ b/arch/arm/boot/dts/renesas/r8a7790-stout.dts
@@ -44,6 +44,15 @@ led5 {
 		};
 	};
 
+	fixedregulator1v8: regulator-1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-1.8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
 	fixedregulator3v3: regulator-3v3 {
 		compatible = "regulator-fixed";
 		regulator-name = "fixed-3.3V";
@@ -296,6 +305,12 @@ hdmi@39 {
 		clocks = <&osc4_clk>;
 		clock-names = "cec";
 
+		avdd-supply = <&fixedregulator1v8>;
+		dvdd-supply = <&fixedregulator1v8>;
+		pvdd-supply = <&fixedregulator1v8>;
+		dvdd-3v-supply = <&fixedregulator3v3>;
+		bgvdd-supply = <&fixedregulator1v8>;
+
 		adi,input-depth = <8>;
 		adi,input-colorspace = "rgb";
 		adi,input-clock = "1x";
diff --git a/arch/arm/boot/dts/renesas/r8a7791-koelsch.dts b/arch/arm/boot/dts/renesas/r8a7791-koelsch.dts
index 545515b41ea3..e9c13bb03772 100644
--- a/arch/arm/boot/dts/renesas/r8a7791-koelsch.dts
+++ b/arch/arm/boot/dts/renesas/r8a7791-koelsch.dts
@@ -174,6 +174,24 @@ led8 {
 		};
 	};
 
+	reg_1p8v: regulator-1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-1.8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	reg_3p3v: regulator-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-3.3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
 	vcc_sdhi0: regulator-vcc-sdhi0 {
 		compatible = "regulator-fixed";
 
@@ -340,7 +358,7 @@ gpioi2c4: i2c-11 {
 	 * I2C1 is routed to EXIO connector B, pins 64 (SCL) + 66 (SDA).
 	 * A fallback to GPIO is provided.
 	 */
-	i2cexio1: i2c-12 {
+	i2cexio1: i2c-mux1 {
 		compatible = "i2c-demux-pinctrl";
 		i2c-parent = <&i2c1>, <&gpioi2c1>;
 		i2c-bus-name = "i2c-exio1";
@@ -351,7 +369,7 @@ i2cexio1: i2c-12 {
 	/*
 	 * A fallback to GPIO is provided for I2C2.
 	 */
-	i2chdmi: i2c-13 {
+	i2chdmi: i2c-mux2 {
 		compatible = "i2c-demux-pinctrl";
 		i2c-parent = <&i2c2>, <&gpioi2c2>;
 		i2c-bus-name = "i2c-hdmi";
@@ -384,6 +402,12 @@ hdmi@39 {
 			clocks = <&cec_clock>;
 			clock-names = "cec";
 
+			avdd-supply = <&reg_1p8v>;
+			dvdd-supply = <&reg_1p8v>;
+			pvdd-supply = <&reg_1p8v>;
+			dvdd-3v-supply = <&reg_3p3v>;
+			bgvdd-supply = <&reg_1p8v>;
+
 			adi,input-depth = <8>;
 			adi,input-colorspace = "rgb";
 			adi,input-clock = "1x";
@@ -446,7 +470,7 @@ eeprom@50 {
 	 * I2C4 is routed to EXIO connector E, pins 37 (SCL) + 39 (SDA).
 	 * A fallback to GPIO is provided.
 	 */
-	i2cexio4: i2c-14 {
+	i2cexio4: i2c-mux3 {
 		compatible = "i2c-demux-pinctrl";
 		i2c-parent = <&i2c4>, <&gpioi2c4>;
 		i2c-bus-name = "i2c-exio4";
diff --git a/arch/arm/boot/dts/renesas/r8a7791-porter.dts b/arch/arm/boot/dts/renesas/r8a7791-porter.dts
index ec0a20d5130d..7e8bc06715f6 100644
--- a/arch/arm/boot/dts/renesas/r8a7791-porter.dts
+++ b/arch/arm/boot/dts/renesas/r8a7791-porter.dts
@@ -47,6 +47,24 @@ memory@200000000 {
 		reg = <2 0x00000000 0 0x40000000>;
 	};
 
+	reg_1p8v: regulator-1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-1.8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	reg_3p3v: regulator-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-3.3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
 	vcc_sdhi0: regulator-vcc-sdhi0 {
 		compatible = "regulator-fixed";
 
@@ -148,7 +166,7 @@ gpioi2c2: i2c-9 {
 	/*
 	 * A fallback to GPIO is provided for I2C2.
 	 */
-	i2chdmi: i2c-10 {
+	i2chdmi: i2c-mux1 {
 		compatible = "i2c-demux-pinctrl";
 		i2c-parent = <&i2c2>, <&gpioi2c2>;
 		i2c-bus-name = "i2c-hdmi";
@@ -179,6 +197,12 @@ hdmi@39 {
 			interrupt-parent = <&gpio3>;
 			interrupts = <29 IRQ_TYPE_LEVEL_LOW>;
 
+			avdd-supply = <&reg_1p8v>;
+			dvdd-supply = <&reg_1p8v>;
+			pvdd-supply = <&reg_1p8v>;
+			dvdd-3v-supply = <&reg_3p3v>;
+			bgvdd-supply = <&reg_1p8v>;
+
 			adi,input-depth = <8>;
 			adi,input-colorspace = "rgb";
 			adi,input-clock = "1x";
diff --git a/arch/arm/boot/dts/renesas/r8a7792-blanche.dts b/arch/arm/boot/dts/renesas/r8a7792-blanche.dts
index e793134f32a3..4f9838cf97ee 100644
--- a/arch/arm/boot/dts/renesas/r8a7792-blanche.dts
+++ b/arch/arm/boot/dts/renesas/r8a7792-blanche.dts
@@ -30,6 +30,15 @@ memory@40000000 {
 		reg = <0 0x40000000 0 0x40000000>;
 	};
 
+	d1_8v: regulator-1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "D1.8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
 	d3_3v: regulator-3v3 {
 		compatible = "regulator-fixed";
 		regulator-name = "D3.3V";
@@ -182,6 +191,35 @@ &can_clk {
 };
 
 &lbsc {
+	flash@0 {
+		compatible = "cfi-flash";
+		reg = <0x00000000 0x04000000>;
+		pinctrl-0 = <&flash_pins>;
+		pinctrl-names = "default";
+		bank-width = <2>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "uboot";
+				reg = <0x00000000 0x00040000>;
+				read-only;
+			};
+			partition@40000 {
+				label = "uboot-env";
+				reg = <0x00040000 0x00040000>;
+				read-only;
+			};
+			partition@80000 {
+				label = "flash";
+				reg = <0x00080000 0x03f80000>;
+			};
+		};
+	};
+
 	ethernet@18000000 {
 		compatible = "smsc,lan89218", "smsc,lan9115";
 		reg = <0x18000000 0x100>;
@@ -240,6 +278,11 @@ du1_pins: du1 {
 		function = "du1";
 	};
 
+	flash_pins: flash {
+		groups = "lbsc_cs0";
+		function = "lbsc";
+	};
+
 	keyboard_pins: keyboard {
 		pins = "GP_3_10", "GP_3_11", "GP_3_12", "GP_3_15", "GP_11_2";
 		bias-pull-up;
@@ -296,6 +339,12 @@ hdmi@39 {
 		interrupt-parent = <&irqc>;
 		interrupts = <3 IRQ_TYPE_EDGE_FALLING>;
 
+		avdd-supply = <&d1_8v>;
+		dvdd-supply = <&d1_8v>;
+		pvdd-supply = <&d1_8v>;
+		dvdd-3v-supply = <&d3_3v>;
+		bgvdd-supply = <&d1_8v>;
+
 		adi,input-depth = <8>;
 		adi,input-colorspace = "rgb";
 		adi,input-clock = "1x";
diff --git a/arch/arm/boot/dts/renesas/r8a7792-wheat.dts b/arch/arm/boot/dts/renesas/r8a7792-wheat.dts
index f87e78fe3f6e..000f21a2a863 100644
--- a/arch/arm/boot/dts/renesas/r8a7792-wheat.dts
+++ b/arch/arm/boot/dts/renesas/r8a7792-wheat.dts
@@ -29,6 +29,15 @@ memory@40000000 {
 		reg = <0 0x40000000 0 0x40000000>;
 	};
 
+	d1_8v: regulator-1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "D1.8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
 	d3_3v: regulator-3v3 {
 		compatible = "regulator-fixed";
 		regulator-name = "D3.3V";
@@ -254,6 +263,12 @@ hdmi@3d {
 		reg = <0x3d>, <0x4d>, <0x2d>, <0x5d>;
 		reg-names = "main", "edid", "cec", "packet";
 
+		avdd-supply = <&d1_8v>;
+		dvdd-supply = <&d1_8v>;
+		pvdd-supply = <&d1_8v>;
+		dvdd-3v-supply = <&d3_3v>;
+		bgvdd-supply = <&d1_8v>;
+
 		adi,input-depth = <8>;
 		adi,input-colorspace = "rgb";
 		adi,input-clock = "1x";
@@ -283,6 +298,12 @@ hdmi@39 {
 		reg = <0x39>, <0x49>, <0x29>, <0x59>;
 		reg-names = "main", "edid", "cec", "packet";
 
+		avdd-supply = <&d1_8v>;
+		dvdd-supply = <&d1_8v>;
+		pvdd-supply = <&d1_8v>;
+		dvdd-3v-supply = <&d3_3v>;
+		bgvdd-supply = <&d1_8v>;
+
 		adi,input-depth = <8>;
 		adi,input-colorspace = "rgb";
 		adi,input-clock = "1x";
diff --git a/arch/arm/boot/dts/renesas/r8a7793-gose.dts b/arch/arm/boot/dts/renesas/r8a7793-gose.dts
index 79b537b24642..1744fdbf9e0c 100644
--- a/arch/arm/boot/dts/renesas/r8a7793-gose.dts
+++ b/arch/arm/boot/dts/renesas/r8a7793-gose.dts
@@ -165,6 +165,24 @@ led8 {
 		};
 	};
 
+	reg_1p8v: regulator-1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-1.8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	reg_3p3v: regulator-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-3.3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
 	vcc_sdhi0: regulator-vcc-sdhi0 {
 		compatible = "regulator-fixed";
 
@@ -324,7 +342,7 @@ gpioi2c4: i2c-10 {
 	/*
 	 * A fallback to GPIO is provided for I2C2.
 	 */
-	i2chdmi: i2c-11 {
+	i2chdmi: i2c-mux1 {
 		compatible = "i2c-demux-pinctrl";
 		i2c-parent = <&i2c2>, <&gpioi2c2>;
 		i2c-bus-name = "i2c-hdmi";
@@ -368,6 +386,12 @@ hdmi@39 {
 			interrupt-parent = <&gpio3>;
 			interrupts = <29 IRQ_TYPE_LEVEL_LOW>;
 
+			avdd-supply = <&reg_1p8v>;
+			dvdd-supply = <&reg_1p8v>;
+			pvdd-supply = <&reg_1p8v>;
+			dvdd-3v-supply = <&reg_3p3v>;
+			bgvdd-supply = <&reg_1p8v>;
+
 			adi,input-depth = <8>;
 			adi,input-colorspace = "rgb";
 			adi,input-clock = "1x";
@@ -430,7 +454,7 @@ eeprom@50 {
 	 * I2C4 is routed to EXIO connector E, pins 37 (SCL) + 39 (SDA).
 	 * A fallback to GPIO is provided.
 	 */
-	i2cexio4: i2c-12 {
+	i2cexio4: i2c-mux2 {
 		compatible = "i2c-demux-pinctrl";
 		i2c-parent = <&i2c4>, <&gpioi2c4>;
 		i2c-bus-name = "i2c-exio4";
diff --git a/arch/arm/boot/dts/renesas/r8a7794-alt.dts b/arch/arm/boot/dts/renesas/r8a7794-alt.dts
index 08df031bc27c..c0d067df22a0 100644
--- a/arch/arm/boot/dts/renesas/r8a7794-alt.dts
+++ b/arch/arm/boot/dts/renesas/r8a7794-alt.dts
@@ -192,7 +192,7 @@ gpioi2c4: i2c-10 {
 	/*
 	 * A fallback to GPIO is provided for I2C1.
 	 */
-	i2chdmi: i2c-11 {
+	i2chdmi: i2c-mux1 {
 		compatible = "i2c-demux-pinctrl";
 		i2c-parent = <&i2c1>, <&gpioi2c1>;
 		i2c-bus-name = "i2c-hdmi";
@@ -222,7 +222,7 @@ eeprom@50 {
 	 * I2C4 is routed to EXIO connector B, pins 73 (SCL) + 74 (SDA).
 	 * A fallback to GPIO is provided.
 	 */
-	i2cexio4: i2c-14 {
+	i2cexio4: i2c-mux2 {
 		compatible = "i2c-demux-pinctrl";
 		i2c-parent = <&i2c4>, <&gpioi2c4>;
 		i2c-bus-name = "i2c-exio4";
diff --git a/arch/arm/boot/dts/renesas/r8a7794-silk.dts b/arch/arm/boot/dts/renesas/r8a7794-silk.dts
index b7af1befa126..43d480a7f3ea 100644
--- a/arch/arm/boot/dts/renesas/r8a7794-silk.dts
+++ b/arch/arm/boot/dts/renesas/r8a7794-silk.dts
@@ -102,6 +102,15 @@ key-d {
 		};
 	};
 
+	d1_8v: regulator-d1-8v {
+		compatible = "regulator-fixed";
+		regulator-name = "D1.8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
 	d3_3v: regulator-d3-3v {
 		compatible = "regulator-fixed";
 		regulator-name = "D3.3V";
@@ -225,7 +234,7 @@ gpioi2c1: i2c-9 {
 	/*
 	 * A fallback to GPIO is provided for I2C1.
 	 */
-	i2chdmi: i2c-10 {
+	i2chdmi: i2c-mux1 {
 		compatible = "i2c-demux-pinctrl";
 		i2c-parent = <&i2c1>, <&gpioi2c1>;
 		i2c-bus-name = "i2c-hdmi";
@@ -256,6 +265,12 @@ hdmi@39 {
 			interrupt-parent = <&gpio5>;
 			interrupts = <23 IRQ_TYPE_LEVEL_LOW>;
 
+			avdd-supply = <&d1_8v>;
+			dvdd-supply = <&d1_8v>;
+			pvdd-supply = <&d1_8v>;
+			dvdd-3v-supply = <&d3_3v>;
+			bgvdd-supply = <&d1_8v>;
+
 			adi,input-depth = <8>;
 			adi,input-colorspace = "rgb";
 			adi,input-clock = "1x";
diff --git a/arch/arm/boot/dts/rockchip/Makefile b/arch/arm/boot/dts/rockchip/Makefile
index 0f46e18fe275..58868cf0510b 100644
--- a/arch/arm/boot/dts/rockchip/Makefile
+++ b/arch/arm/boot/dts/rockchip/Makefile
@@ -10,6 +10,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += \
 	rk3066a-mk808.dtb \
 	rk3066a-rayeager.dtb \
 	rk3128-evb.dtb \
+	rk3128-xpi-3128.dtb \
 	rk3188-bqedison2qc.dtb \
 	rk3188-px3-evb.dtb \
 	rk3188-radxarock.dtb \
diff --git a/arch/arm/boot/dts/rockchip/rk3128-xpi-3128.dts b/arch/arm/boot/dts/rockchip/rk3128-xpi-3128.dts
new file mode 100644
index 000000000000..61b9f069c8a2
--- /dev/null
+++ b/arch/arm/boot/dts/rockchip/rk3128-xpi-3128.dts
@@ -0,0 +1,397 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+
+/dts-v1/;
+
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
+#include "rk3128.dtsi"
+
+/ {
+	model = "Geniatech XPI-3128";
+	compatible = "geniatech,xpi-3128", "rockchip,rk3128";
+
+	aliases {
+		gpio0 = &gpio0;
+		gpio1 = &gpio1;
+		gpio2 = &gpio2;
+		gpio3 = &gpio3;
+		mmc0 = &emmc;
+		mmc1 = &sdmmc;
+		serial0 = &uart1;
+	};
+
+	memory@60000000 {
+		device_type = "memory";
+		reg = <0x60000000 0x40000000>;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	adc-keys {
+		compatible = "adc-keys";
+		io-channels = <&saradc 1>;
+		io-channel-names = "buttons";
+		keyup-threshold-microvolt = <3300000>;
+
+		button-recovery {
+			label = "Recovery";
+			linux,code = <KEY_VENDOR>;
+			press-threshold-microvolt = <0>;
+		};
+	};
+
+	dc_5v: dc-5v-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "DC_5V";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	/*
+	 * This is a vbus-supply, which also supplies the GL852G usb hub,
+	 * thus has to be always-on
+	 */
+	host_pwr_5v: host-pwr-5v-regulator {
+		compatible = "regulator-fixed";
+		gpio = <&gpio3 RK_PC4 GPIO_ACTIVE_HIGH>;
+		startup-delay-us = <1500>;
+		regulator-name = "HOST_PWR_5V";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&dc_5v>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&host_drv>;
+		enable-active-high;
+		regulator-always-on;
+	};
+
+	ir-receiver {
+		compatible = "gpio-ir-receiver";
+		gpios = <&gpio3 RK_PD2 GPIO_ACTIVE_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&ir_int>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led-power {
+			gpios = <&gpio0 RK_PD2 GPIO_ACTIVE_HIGH>;
+			function = LED_FUNCTION_POWER;
+			color = <LED_COLOR_ID_BLUE>;
+			default-state = "on";
+			pinctrl-names = "default";
+			pinctrl-0 = <&power_led>;
+		};
+
+		led-spd {
+			gpios = <&gpio3 RK_PB3 GPIO_ACTIVE_LOW>;
+			function = LED_FUNCTION_LAN;
+			color = <LED_COLOR_ID_GREEN>;
+			/*
+			 * currently not allowed to be set as per
+			 * https://www.kernel.org/doc/Documentation/devicetree/bindings/leds/common.yaml
+			 * and needs to set in userspace:
+			 *
+			 * linux,default-trigger = "netdev";
+			 */
+			pinctrl-names = "default";
+			pinctrl-0 = <&spd_led>;
+		};
+	};
+
+	mcu3v3: mcu3v3-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "MCU3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc_io>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vcc_ddr: vcc-ddr-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_DDR";
+		regulator-min-microvolt = <1500000>;
+		regulator-max-microvolt = <1500000>;
+		vin-supply = <&vcc_sys>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vcc_io: vcc-io-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_IO";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc_sys>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vcc_lan: vcc-lan-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_LAN";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc_io>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vcc_sd: vcc-sd-regulator {
+		compatible = "regulator-fixed";
+		gpio = <&gpio1 RK_PB6 GPIO_ACTIVE_LOW>;
+		startup-delay-us = <500>;
+		regulator-name = "VCC_SD";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc_io>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&sdmmc_pwren>;
+	};
+
+	vcc_sys: vcc-sys-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_SYS";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&dc_5v>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vcc33_hdmi: vcc33-hdmi-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC33_HDMI";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcca_33>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vcca_33: vcca-33-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "VCCA_33";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc_sys>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vdd_11: vdd-11-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "VDD_11";
+		regulator-min-microvolt = <1100000>;
+		regulator-max-microvolt = <1100000>;
+		vin-supply = <&vcc_sys>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vdd11_hdmi: vdd11-hdmi-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "VDD11_HDMI";
+		regulator-min-microvolt = <1100000>;
+		regulator-max-microvolt = <1100000>;
+		vin-supply = <&vdd_11>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vdd_arm: vdd-arm-regulator {
+		compatible = "pwm-regulator";
+		regulator-name = "VDD_ARM";
+		pwms = <&pwm1 0 25000 1>;
+		pwm-supply = <&vcc_sys>;
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <1400000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	/*
+	 * As per schematics vdd_log is minimum 900 mV, maximum 1400 mV.
+	 * Since there are HW blocks in PD_LOGIC (which are all driven by
+	 * this supply), that either do not have a driver at all or the
+	 * driver does not implement regulator support we have to make
+	 * sure here that the voltage never drops below 1050 mV.
+	 */
+	vdd_log: vdd-log-regulator {
+		compatible = "pwm-regulator";
+		regulator-name = "VDD_LOG";
+		pwms = <&pwm2 0 25000 1>;
+		pwm-dutycycle-range = <30 100>;
+		pwm-supply = <&vcc_sys>;
+		regulator-min-microvolt = <1050000>;
+		regulator-max-microvolt = <1400000>;
+		regulator-ramp-delay = <4000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+};
+
+&cpu0 {
+	cpu-supply = <&vdd_arm>;
+};
+
+&emmc {
+	bus-width = <8>;
+	vmmc-supply = <&vcc_io>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>;
+	cap-mmc-highspeed;
+	mmc-ddr-3_3v;
+	no-sd;
+	no-sdio;
+	status = "okay";
+};
+
+&gpio0 {
+	gpio-line-names = /* GPIO0 A0-A7 */
+			  "", "", "HEADER_5", "HEADER_3",
+			  "", "", "", "",
+			  /* GPIO0 B0-B7 */
+			  "HEADER_22", "HEADER_23", "", "HEADER_19",
+			  "HEADER_26", "HEADER_21", "HEADER_24", "",
+			  /* GPIO0 C0-C7 */
+			  "", "HEADER_18", "", "",
+			  "", "", "", "",
+			  /* GPIO0 D0-D7 */
+			  "HEADER_36", "", "", "",
+			  "", "", "HEADER_13", "";
+};
+
+&gpio1 {
+	gpio-line-names = /* GPIO1 A0-A7 */
+			  "HEADER_7", "HEADER_35", "HEADER_33", "HEADER_37",
+			  "HEADER_40", "HEADER_38", "", "",
+			  /* GPIO1 B0-B7 */
+			  "HEADER_11", "", "", "HEADER_29",
+			  "HEADER_31", "", "", "",
+			  /* GPIO1 C0-C7 */
+			  "", "", "", "",
+			  "", "", "", "",
+			  /* GPIO1 D0-D7 */
+			  "", "", "", "",
+			  "", "", "", "";
+};
+
+&gpio2 {
+	gpio-line-names = /* GPIO2 A0-A7 */
+			  "", "", "", "",
+			  "", "", "", "",
+			  /* GPIO2 B0-B7 */
+			  "", "", "", "",
+			  "", "", "", "",
+			  /* GPIO2 C0-C7 */
+			  "", "", "", "",
+			  "HEADER_27", "", "", "",
+			  /* GPIO2 D0-D7 */
+			  "", "", "HEADER_8", "HEADER_10",
+			  "", "", "", "";
+};
+
+&gpio3 {
+	gpio-line-names = /* GPIO3 A0-A7 */
+			  "", "", "", "",
+			  "", "", "", "",
+			  /* GPIO3 B0-B7 */
+			  "", "", "", "",
+			  "", "", "", "",
+			  /* GPIO3 C0-C7 */
+			  "", "HEADER_32", "", "",
+			  "", "", "", "HEADER_12",
+			  /* GPIO3 D0-D7 */
+			  "", "", "", "HEADER_15",
+			  "", "", "", "";
+};
+
+&pinctrl {
+	dp83848c {
+		dp83848c_rst: dp83848c-rst {
+			rockchip,pins = <2 RK_PD0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	ir-receiver {
+		ir_int: ir-int {
+			rockchip,pins = <3 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	leds {
+		power_led: power-led {
+			rockchip,pins = <0 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		spd_led: spd-led {
+			rockchip,pins = <3 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	usb2 {
+		host_drv: host-drv {
+			rockchip,pins = <3 RK_PC4 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+};
+
+&pwm1 {
+	status = "okay";
+};
+
+&pwm2 {
+	status = "okay";
+};
+
+&saradc {
+	vref-supply = <&vcc_io>;
+	status = "okay";
+};
+
+&sdmmc {
+	bus-width = <4>;
+	vmmc-supply = <&vcc_sd>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&sdmmc_bus4 &sdmmc_clk &sdmmc_cmd &sdmmc_det>;
+	disable-wp;
+	cap-sd-highspeed;
+	no-mmc;
+	no-sdio;
+	status = "okay";
+};
+
+&uart1 {
+	status = "okay";
+};
+
+&usb_host_ehci {
+	status = "okay";
+};
+
+&usb_otg {
+	vusb_a-supply = <&vcc_io>;
+	vusb_d-supply = <&vdd_11>;
+	status = "okay";
+};
+
+&usb2phy {
+	status = "okay";
+};
+
+&usb2phy_host {
+	status = "okay";
+};
+
+&usb2phy_otg {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/rockchip/rk3128.dtsi b/arch/arm/boot/dts/rockchip/rk3128.dtsi
index 7bf557c99561..4e8b38604ecd 100644
--- a/arch/arm/boot/dts/rockchip/rk3128.dtsi
+++ b/arch/arm/boot/dts/rockchip/rk3128.dtsi
@@ -154,6 +154,9 @@ usb_otg: usb@10180000 {
 		clocks = <&cru HCLK_OTG>;
 		clock-names = "otg";
 		dr_mode = "otg";
+		g-np-tx-fifo-size = <16>;
+		g-rx-fifo-size = <280>;
+		g-tx-fifo-size = <256 128 128 64 32 16>;
 		phys = <&usb2phy_otg>;
 		phy-names = "usb2-phy";
 		status = "disabled";
@@ -163,6 +166,7 @@ usb_host_ehci: usb@101c0000 {
 		compatible = "generic-ehci";
 		reg = <0x101c0000 0x20000>;
 		interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&cru HCLK_HOST2>;
 		phys = <&usb2phy_host>;
 		phy-names = "usb";
 		status = "disabled";
@@ -172,6 +176,7 @@ usb_host_ohci: usb@101e0000 {
 		compatible = "generic-ohci";
 		reg = <0x101e0000 0x20000>;
 		interrupts = <GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&cru HCLK_HOST2>;
 		phys = <&usb2phy_host>;
 		phy-names = "usb";
 		status = "disabled";
@@ -261,6 +266,8 @@ usb2phy: usb2phy@17c {
 			clocks = <&cru SCLK_OTGPHY0>;
 			clock-names = "phyclk";
 			clock-output-names = "usb480m_phy";
+			assigned-clocks = <&cru SCLK_USB480M>;
+			assigned-clock-parents = <&usb2phy>;
 			#clock-cells = <0>;
 			status = "disabled";
 
@@ -843,6 +850,10 @@ sdmmc_cmd: sdmmc-cmd {
 				rockchip,pins = <1 RK_PB7 1 &pcfg_pull_default>;
 			};
 
+			sdmmc_det: sdmmc-det {
+				rockchip,pins = <1 RK_PC1 1 &pcfg_pull_default>;
+			};
+
 			sdmmc_wp: sdmmc-wp {
 				rockchip,pins = <1 RK_PA7 1 &pcfg_pull_default>;
 			};
diff --git a/arch/arm/boot/dts/samsung/exynos4.dtsi b/arch/arm/boot/dts/samsung/exynos4.dtsi
index f775b9377a38..7f981b5c0d64 100644
--- a/arch/arm/boot/dts/samsung/exynos4.dtsi
+++ b/arch/arm/boot/dts/samsung/exynos4.dtsi
@@ -203,16 +203,16 @@ dsi_0: dsi@11c80000 {
 
 		camera: camera@11800000 {
 			compatible = "samsung,fimc";
+			ranges = <0x0 0x11800000 0xa0000>;
 			status = "disabled";
 			#address-cells = <1>;
 			#size-cells = <1>;
 			#clock-cells = <1>;
 			clock-output-names = "cam_a_clkout", "cam_b_clkout";
-			ranges;
 
-			fimc_0: fimc@11800000 {
+			fimc_0: fimc@0 {
 				compatible = "samsung,exynos4210-fimc";
-				reg = <0x11800000 0x1000>;
+				reg = <0x0 0x1000>;
 				interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clock CLK_FIMC0>,
 					 <&clock CLK_SCLK_FIMC0>;
@@ -223,9 +223,9 @@ fimc_0: fimc@11800000 {
 				status = "disabled";
 			};
 
-			fimc_1: fimc@11810000 {
+			fimc_1: fimc@10000 {
 				compatible = "samsung,exynos4210-fimc";
-				reg = <0x11810000 0x1000>;
+				reg = <0x00010000 0x1000>;
 				interrupts = <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clock CLK_FIMC1>,
 					 <&clock CLK_SCLK_FIMC1>;
@@ -236,9 +236,9 @@ fimc_1: fimc@11810000 {
 				status = "disabled";
 			};
 
-			fimc_2: fimc@11820000 {
+			fimc_2: fimc@20000 {
 				compatible = "samsung,exynos4210-fimc";
-				reg = <0x11820000 0x1000>;
+				reg = <0x00020000 0x1000>;
 				interrupts = <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clock CLK_FIMC2>,
 					 <&clock CLK_SCLK_FIMC2>;
@@ -249,9 +249,9 @@ fimc_2: fimc@11820000 {
 				status = "disabled";
 			};
 
-			fimc_3: fimc@11830000 {
+			fimc_3: fimc@30000 {
 				compatible = "samsung,exynos4210-fimc";
-				reg = <0x11830000 0x1000>;
+				reg = <0x00030000 0x1000>;
 				interrupts = <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clock CLK_FIMC3>,
 					 <&clock CLK_SCLK_FIMC3>;
@@ -262,9 +262,9 @@ fimc_3: fimc@11830000 {
 				status = "disabled";
 			};
 
-			csis_0: csis@11880000 {
+			csis_0: csis@80000 {
 				compatible = "samsung,exynos4210-csis";
-				reg = <0x11880000 0x4000>;
+				reg = <0x00080000 0x4000>;
 				interrupts = <GIC_SPI 78 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clock CLK_CSIS0>,
 					 <&clock CLK_SCLK_CSIS0>;
@@ -278,9 +278,9 @@ csis_0: csis@11880000 {
 				#size-cells = <0>;
 			};
 
-			csis_1: csis@11890000 {
+			csis_1: csis@90000 {
 				compatible = "samsung,exynos4210-csis";
-				reg = <0x11890000 0x4000>;
+				reg = <0x00090000 0x4000>;
 				interrupts = <GIC_SPI 80 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clock CLK_CSIS1>,
 					 <&clock CLK_SCLK_CSIS1>;
diff --git a/arch/arm/boot/dts/samsung/exynos4x12.dtsi b/arch/arm/boot/dts/samsung/exynos4x12.dtsi
index 84c1db221c98..b4b5e769145b 100644
--- a/arch/arm/boot/dts/samsung/exynos4x12.dtsi
+++ b/arch/arm/boot/dts/samsung/exynos4x12.dtsi
@@ -451,14 +451,15 @@ &combiner {
 };
 
 &camera {
+	ranges = <0x0 0x11800000 0xba1000>;
 	clocks = <&clock CLK_SCLK_CAM0>, <&clock CLK_SCLK_CAM1>,
 		 <&clock CLK_PIXELASYNCM0>, <&clock CLK_PIXELASYNCM1>;
 	clock-names = "sclk_cam0", "sclk_cam1", "pxl_async0", "pxl_async1";
 
 	/* fimc_[0-3] are configured outside, under phandles */
-	fimc_lite_0: fimc-lite@12390000 {
+	fimc_lite_0: fimc-lite@b90000 {
 		compatible = "samsung,exynos4212-fimc-lite";
-		reg = <0x12390000 0x1000>;
+		reg = <0x00b90000 0x1000>;
 		interrupts = <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>;
 		power-domains = <&pd_isp>;
 		clocks = <&isp_clock CLK_ISP_FIMC_LITE0>;
@@ -467,9 +468,9 @@ fimc_lite_0: fimc-lite@12390000 {
 		status = "disabled";
 	};
 
-	fimc_lite_1: fimc-lite@123a0000 {
+	fimc_lite_1: fimc-lite@ba0000 {
 		compatible = "samsung,exynos4212-fimc-lite";
-		reg = <0x123a0000 0x1000>;
+		reg = <0x00ba0000 0x1000>;
 		interrupts = <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>;
 		power-domains = <&pd_isp>;
 		clocks = <&isp_clock CLK_ISP_FIMC_LITE1>;
@@ -478,9 +479,9 @@ fimc_lite_1: fimc-lite@123a0000 {
 		status = "disabled";
 	};
 
-	fimc_is: fimc-is@12000000 {
+	fimc_is: fimc-is@800000 {
 		compatible = "samsung,exynos4212-fimc-is";
-		reg = <0x12000000 0x260000>;
+		reg = <0x00800000 0x260000>;
 		interrupts = <GIC_SPI 90 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>;
 		power-domains = <&pd_isp>;
@@ -516,18 +517,15 @@ fimc_is: fimc-is@12000000 {
 		iommus = <&sysmmu_fimc_isp>, <&sysmmu_fimc_drc>,
 			 <&sysmmu_fimc_fd>, <&sysmmu_fimc_mcuctl>;
 		iommu-names = "isp", "drc", "fd", "mcuctl";
+		samsung,pmu-syscon = <&pmu_system_controller>;
 		#address-cells = <1>;
 		#size-cells = <1>;
 		ranges;
 		status = "disabled";
 
-		pmu@10020000 {
-			reg = <0x10020000 0x3000>;
-		};
-
-		i2c1_isp: i2c-isp@12140000 {
+		i2c1_isp: i2c-isp@940000 {
 			compatible = "samsung,exynos4212-i2c-isp";
-			reg = <0x12140000 0x100>;
+			reg = <0x00940000 0x100>;
 			clocks = <&isp_clock CLK_ISP_I2C1_ISP>;
 			clock-names = "i2c_isp";
 			#address-cells = <1>;
diff --git a/arch/arm/boot/dts/samsung/s5pv210.dtsi b/arch/arm/boot/dts/samsung/s5pv210.dtsi
index f7de5b5f2f38..ed560c9a3aa1 100644
--- a/arch/arm/boot/dts/samsung/s5pv210.dtsi
+++ b/arch/arm/boot/dts/samsung/s5pv210.dtsi
@@ -549,17 +549,17 @@ i2c1: i2c@fab00000 {
 
 		camera: camera@fa600000 {
 			compatible = "samsung,fimc";
+			ranges = <0x0 0xfa600000 0xe01000>;
 			clocks = <&clocks SCLK_CAM0>, <&clocks SCLK_CAM1>;
 			clock-names = "sclk_cam0", "sclk_cam1";
 			#address-cells = <1>;
 			#size-cells = <1>;
 			#clock-cells = <1>;
 			clock-output-names = "cam_a_clkout", "cam_b_clkout";
-			ranges;
 
-			csis0: csis@fa600000 {
+			csis0: csis@0 {
 				compatible = "samsung,s5pv210-csis";
-				reg = <0xfa600000 0x4000>;
+				reg = <0x00000000 0x4000>;
 				interrupt-parent = <&vic2>;
 				interrupts = <29>;
 				clocks = <&clocks CLK_CSIS>,
@@ -572,9 +572,9 @@ csis0: csis@fa600000 {
 				#size-cells = <0>;
 			};
 
-			fimc0: fimc@fb200000 {
+			fimc0: fimc@c00000 {
 				compatible = "samsung,s5pv210-fimc";
-				reg = <0xfb200000 0x1000>;
+				reg = <0x00c00000 0x1000>;
 				interrupts = <5>;
 				interrupt-parent = <&vic2>;
 				clocks = <&clocks CLK_FIMC0>,
@@ -586,9 +586,9 @@ fimc0: fimc@fb200000 {
 				samsung,cam-if;
 			};
 
-			fimc1: fimc@fb300000 {
+			fimc1: fimc@d00000 {
 				compatible = "samsung,s5pv210-fimc";
-				reg = <0xfb300000 0x1000>;
+				reg = <0x00d00000 0x1000>;
 				interrupt-parent = <&vic2>;
 				interrupts = <6>;
 				clocks = <&clocks CLK_FIMC1>,
@@ -602,9 +602,9 @@ fimc1: fimc@fb300000 {
 				samsung,lcd-wb;
 			};
 
-			fimc2: fimc@fb400000 {
+			fimc2: fimc@e00000 {
 				compatible = "samsung,s5pv210-fimc";
-				reg = <0xfb400000 0x1000>;
+				reg = <0x00e00000 0x1000>;
 				interrupt-parent = <&vic2>;
 				interrupts = <7>;
 				clocks = <&clocks CLK_FIMC2>,
diff --git a/arch/arm/boot/dts/st/stm32f469-disco.dts b/arch/arm/boot/dts/st/stm32f469-disco.dts
index cbbd521bf010..8a4f8ddd083d 100644
--- a/arch/arm/boot/dts/st/stm32f469-disco.dts
+++ b/arch/arm/boot/dts/st/stm32f469-disco.dts
@@ -69,16 +69,9 @@ aliases {
 		serial0 = &usart3;
 	};
 
-	mmc_vcard: mmc_vcard {
+	vcc_3v3: vcc-3v3 {
 		compatible = "regulator-fixed";
-		regulator-name = "mmc_vcard";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-	};
-
-	vdd_dsi: vdd-dsi {
-		compatible = "regulator-fixed";
-		regulator-name = "vdd_dsi";
+		regulator-name = "vcc_3v3";
 		regulator-min-microvolt = <3300000>;
 		regulator-max-microvolt = <3300000>;
 	};
@@ -164,7 +157,7 @@ panel@0 {
 		compatible = "orisetech,otm8009a";
 		reg = <0>; /* dsi virtual channel (0..3) */
 		reset-gpios = <&gpioh 7 GPIO_ACTIVE_LOW>;
-		power-supply = <&vdd_dsi>;
+		power-supply = <&vcc_3v3>;
 		status = "okay";
 
 		port {
@@ -219,7 +212,7 @@ timer@2 {
 
 &sdio {
 	status = "okay";
-	vmmc-supply = <&mmc_vcard>;
+	vmmc-supply = <&vcc_3v3>;
 	cd-gpios = <&gpiog 2 GPIO_ACTIVE_LOW>;
 	broken-cd;
 	pinctrl-names = "default", "opendrain";
diff --git a/arch/arm/boot/dts/st/stm32f746.dtsi b/arch/arm/boot/dts/st/stm32f746.dtsi
index 53a8e2dec9a4..65c72b6fcc83 100644
--- a/arch/arm/boot/dts/st/stm32f746.dtsi
+++ b/arch/arm/boot/dts/st/stm32f746.dtsi
@@ -274,6 +274,26 @@ gcan3: gcan@40003600 {
 			clocks = <&rcc 0 STM32F7_APB1_CLOCK(CAN3)>;
 		};
 
+		spi2: spi@40003800 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32f7-spi";
+			reg = <0x40003800 0x400>;
+			interrupts = <36>;
+			clocks = <&rcc 0 STM32F7_APB1_CLOCK(SPI2)>;
+			status = "disabled";
+		};
+
+		spi3: spi@40003c00 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32f7-spi";
+			reg = <0x40003c00 0x400>;
+			interrupts = <51>;
+			clocks = <&rcc 0 STM32F7_APB1_CLOCK(SPI3)>;
+			status = "disabled";
+		};
+
 		usart2: serial@40004400 {
 			compatible = "st,stm32f7-uart";
 			reg = <0x40004400 0x400>;
@@ -491,9 +511,30 @@ sdio1: mmc@40012c00 {
 			status = "disabled";
 		};
 
+		spi1: spi@40013000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32f7-spi";
+			reg = <0x40013000 0x400>;
+			interrupts = <35>;
+			clocks = <&rcc 0 STM32F7_APB2_CLOCK(SPI1)>;
+			status = "disabled";
+		};
+
+		spi4: spi@40013400 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32f7-spi";
+			reg = <0x40013400 0x400>;
+			interrupts = <84>;
+			clocks = <&rcc 0 STM32F7_APB2_CLOCK(SPI4)>;
+			status = "disabled";
+		};
+
 		syscfg: syscon@40013800 {
 			compatible = "st,stm32-syscfg", "syscon";
 			reg = <0x40013800 0x400>;
+			clocks = <&rcc 0 STM32F7_APB2_CLOCK(SYSCFG)>;
 		};
 
 		exti: interrupt-controller@40013c00 {
@@ -554,6 +595,26 @@ pwm {
 			};
 		};
 
+		spi5: spi@40015000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32f7-spi";
+			reg = <0x40015000 0x400>;
+			interrupts = <85>;
+			clocks = <&rcc 0 STM32F7_APB2_CLOCK(SPI5)>;
+			status = "disabled";
+		};
+
+		spi6: spi@40015400 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "st,stm32f7-spi";
+			reg = <0x40015400 0x400>;
+			interrupts = <86>;
+			clocks = <&rcc 0 STM32F7_APB2_CLOCK(SPI6)>;
+			status = "disabled";
+		};
+
 		ltdc: display-controller@40016800 {
 			compatible = "st,stm32-ltdc";
 			reg = <0x40016800 0x200>;
diff --git a/arch/arm/mach-at91/pm.c b/arch/arm/mach-at91/pm.c
index 1a26af0fabc7..345b91dc6627 100644
--- a/arch/arm/mach-at91/pm.c
+++ b/arch/arm/mach-at91/pm.c
@@ -1103,6 +1103,7 @@ static void __init at91_pm_secure_init(void)
 	if (res.a0 == 0) {
 		pr_info("AT91: Secure PM: suspend mode set to %s\n",
 			pm_modes[suspend_mode].pattern);
+		soc_pm.data.mode = suspend_mode;
 		return;
 	}
 
@@ -1112,6 +1113,7 @@ static void __init at91_pm_secure_init(void)
 	res = sam_smccc_call(SAMA5_SMC_SIP_GET_SUSPEND_MODE, 0, 0);
 	if (res.a0 == 0) {
 		pr_warn("AT91: Secure PM: failed to get default mode\n");
+		soc_pm.data.mode = -1;
 		return;
 	}
 
@@ -1119,6 +1121,7 @@ static void __init at91_pm_secure_init(void)
 		pm_modes[suspend_mode].pattern);
 
 	soc_pm.data.suspend_mode = res.a1;
+	soc_pm.data.mode = soc_pm.data.suspend_mode;
 }
 static const struct of_device_id atmel_shdwc_ids[] = {
 	{ .compatible = "atmel,sama5d2-shdwc" },
diff --git a/arch/arm/tools/syscall.tbl b/arch/arm/tools/syscall.tbl
index 584f9528c996..b6c9e01e14f5 100644
--- a/arch/arm/tools/syscall.tbl
+++ b/arch/arm/tools/syscall.tbl
@@ -470,3 +470,8 @@
 454	common	futex_wake			sys_futex_wake
 455	common	futex_wait			sys_futex_wait
 456	common	futex_requeue			sys_futex_requeue
+457	common	statmount			sys_statmount
+458	common	listmount			sys_listmount
+459	common	lsm_get_self_attr		sys_lsm_get_self_attr
+460	common	lsm_set_self_attr		sys_lsm_set_self_attr
+461	common	lsm_list_modules		sys_lsm_list_modules
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h616-orangepi-zero.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-h616-orangepi-zero.dtsi
index 15290e6892fc..fc7315b94406 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-h616-orangepi-zero.dtsi
+++ b/arch/arm64/boot/dts/allwinner/sun50i-h616-orangepi-zero.dtsi
@@ -68,10 +68,7 @@ &ehci1 {
 &emac0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&ext_rgmii_pins>;
-	phy-mode = "rgmii";
 	phy-handle = <&ext_rgmii_phy>;
-	allwinner,rx-delay-ps = <3100>;
-	allwinner,tx-delay-ps = <700>;
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h616-orangepi-zero2.dts b/arch/arm64/boot/dts/allwinner/sun50i-h616-orangepi-zero2.dts
index d83852e72f06..b5d713926a34 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-h616-orangepi-zero2.dts
+++ b/arch/arm64/boot/dts/allwinner/sun50i-h616-orangepi-zero2.dts
@@ -13,6 +13,9 @@ / {
 };
 
 &emac0 {
+	allwinner,rx-delay-ps = <3100>;
+	allwinner,tx-delay-ps = <700>;
+	phy-mode = "rgmii";
 	phy-supply = <&reg_dcdce>;
 };
 
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h618-orangepi-zero2w.dts b/arch/arm64/boot/dts/allwinner/sun50i-h618-orangepi-zero2w.dts
new file mode 100644
index 000000000000..21ca1977055d
--- /dev/null
+++ b/arch/arm64/boot/dts/allwinner/sun50i-h618-orangepi-zero2w.dts
@@ -0,0 +1,176 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (C) 2023 Arm Ltd.
+ */
+
+/dts-v1/;
+
+#include "sun50i-h616.dtsi"
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/leds/common.h>
+
+/ {
+	model = "OrangePi Zero 2W";
+	compatible = "xunlong,orangepi-zero2w", "allwinner,sun50i-h618";
+
+	aliases {
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led-0 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_GREEN>;
+			gpios = <&pio 2 13 GPIO_ACTIVE_HIGH>; /* PC13 */
+		};
+	};
+
+	reg_vcc5v: vcc5v {
+		/* board wide 5V supply directly from the USB-C socket */
+		compatible = "regulator-fixed";
+		regulator-name = "vcc-5v";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+	};
+
+	reg_vcc3v3: vcc3v3 {
+		/* SY8089 DC/DC converter */
+		compatible = "regulator-fixed";
+		regulator-name = "vcc-3v3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&reg_vcc5v>;
+		regulator-always-on;
+	};
+};
+
+&ehci1 {
+	status = "okay";
+};
+
+/* USB 2 & 3 are on the FPC connector (or the exansion board) */
+
+&mmc0 {
+	cd-gpios = <&pio 5 6 GPIO_ACTIVE_LOW>;	/* PF6 */
+	bus-width = <4>;
+	vmmc-supply = <&reg_vcc3v3>;
+	status = "okay";
+};
+
+&ohci1 {
+	status = "okay";
+};
+
+&pio {
+	vcc-pc-supply = <&reg_dldo1>;
+	vcc-pf-supply = <&reg_dldo1>;	/* internally via VCC-IO */
+	vcc-pg-supply = <&reg_aldo1>;
+	vcc-ph-supply = <&reg_dldo1>;	/* internally via VCC-IO */
+	vcc-pi-supply = <&reg_dldo1>;
+};
+
+&r_i2c {
+	status = "okay";
+
+	axp313: pmic@36 {
+		compatible = "x-powers,axp313a";
+		reg = <0x36>;
+		#interrupt-cells = <1>;
+		interrupt-controller;
+		interrupt-parent = <&pio>;
+		interrupts = <2 9 IRQ_TYPE_LEVEL_LOW>;	/* PC9 */
+
+		vin1-supply = <&reg_vcc5v>;
+		vin2-supply = <&reg_vcc5v>;
+		vin3-supply = <&reg_vcc5v>;
+
+		regulators {
+			/* Supplies VCC-PLL and DRAM */
+			reg_aldo1: aldo1 {
+				regulator-always-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcc1v8";
+			};
+
+			/* Supplies VCC-IO, so needs to be always on. */
+			reg_dldo1: dldo1 {
+				regulator-always-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcc3v3";
+			};
+
+			reg_dcdc1: dcdc1 {
+				regulator-always-on;
+				regulator-min-microvolt = <810000>;
+				regulator-max-microvolt = <990000>;
+				regulator-name = "vdd-gpu-sys";
+			};
+
+			reg_dcdc2: dcdc2 {
+				regulator-always-on;
+				regulator-min-microvolt = <810000>;
+				regulator-max-microvolt = <1100000>;
+				regulator-name = "vdd-cpu";
+			};
+
+			reg_dcdc3: dcdc3 {
+				regulator-always-on;
+				regulator-min-microvolt = <1100000>;
+				regulator-max-microvolt = <1100000>;
+				regulator-name = "vdd-dram";
+			};
+		};
+	};
+};
+
+&spi0  {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi0_pins>, <&spi0_cs0_pin>;
+
+	flash@0 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <40000000>;
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_ph_pins>;
+	status = "okay";
+};
+
+&usbotg {
+	/*
+	 * PHY0 pins are connected to a USB-C socket, but a role switch
+	 * is not implemented: both CC pins are pulled to GND.
+	 * The VBUS pins power the device, so a fixed peripheral mode
+	 * is the best choice.
+	 * The board can be powered via GPIOs, in this case port0 *can*
+	 * act as a host (with a cable/adapter ignoring CC), as VBUS is
+	 * then provided by the GPIOs. Any user of this setup would
+	 * need to adjust the DT accordingly: dr_mode set to "host",
+	 * enabling OHCI0 and EHCI0.
+	 */
+	dr_mode = "peripheral";
+	status = "okay";
+};
+
+&usbphy {
+	usb1_vbus-supply = <&reg_vcc5v>;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h618-orangepi-zero3.dts b/arch/arm64/boot/dts/allwinner/sun50i-h618-orangepi-zero3.dts
index 00fe28caac93..b3b1b8692125 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-h618-orangepi-zero3.dts
+++ b/arch/arm64/boot/dts/allwinner/sun50i-h618-orangepi-zero3.dts
@@ -13,6 +13,8 @@ / {
 };
 
 &emac0 {
+	allwinner,tx-delay-ps = <700>;
+	phy-mode = "rgmii-rxid";
 	phy-supply = <&reg_dldo1>;
 };
 
diff --git a/arch/arm64/boot/dts/exynos/exynos5433.dtsi b/arch/arm64/boot/dts/exynos/exynos5433.dtsi
index 91ae0462a706..7fbbec04bff0 100644
--- a/arch/arm64/boot/dts/exynos/exynos5433.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos5433.dtsi
@@ -361,7 +361,8 @@ soc: soc@0 {
 		ranges = <0x0 0x0 0x0 0x18000000>;
 
 		chipid@10000000 {
-			compatible = "samsung,exynos4210-chipid";
+			compatible = "samsung,exynos5433-chipid",
+				     "samsung,exynos4210-chipid";
 			reg = <0x10000000 0x100>;
 		};
 
@@ -850,7 +851,8 @@ pinctrl_alive: pinctrl@10580000 {
 			reg = <0x10580000 0x1a20>, <0x11090000 0x100>;
 
 			wakeup-interrupt-controller {
-				compatible = "samsung,exynos7-wakeup-eint";
+				compatible = "samsung,exynos5433-wakeup-eint",
+					     "samsung,exynos7-wakeup-eint";
 				interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
 			};
 		};
@@ -1546,7 +1548,7 @@ spi_4: spi@14d00000 {
 		};
 
 		adc: adc@14d10000 {
-			compatible = "samsung,exynos7-adc";
+			compatible = "samsung,exynos5433-adc", "samsung,exynos7-adc";
 			reg = <0x14d10000 0x100>;
 			interrupts = <GIC_SPI 438 IRQ_TYPE_LEVEL_HIGH>;
 			clock-names = "adc";
@@ -1556,7 +1558,7 @@ adc: adc@14d10000 {
 		};
 
 		i2s1: i2s@14d60000 {
-			compatible = "samsung,exynos7-i2s";
+			compatible = "samsung,exynos5433-i2s", "samsung,exynos7-i2s";
 			reg = <0x14d60000 0x100>;
 			dmas = <&pdma0 31>, <&pdma0 30>;
 			dma-names = "tx", "rx";
@@ -1571,7 +1573,7 @@ i2s1: i2s@14d60000 {
 		};
 
 		pwm: pwm@14dd0000 {
-			compatible = "samsung,exynos4210-pwm";
+			compatible = "samsung,exynos5433-pwm", "samsung,exynos4210-pwm";
 			reg = <0x14dd0000 0x100>;
 			interrupts = <GIC_SPI 416 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 417 IRQ_TYPE_LEVEL_HIGH>,
@@ -1586,7 +1588,8 @@ pwm: pwm@14dd0000 {
 		};
 
 		hsi2c_0: i2c@14e40000 {
-			compatible = "samsung,exynos7-hsi2c";
+			compatible = "samsung,exynos5433-hsi2c",
+				     "samsung,exynos7-hsi2c";
 			reg = <0x14e40000 0x1000>;
 			interrupts = <GIC_SPI 428 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -1599,7 +1602,8 @@ hsi2c_0: i2c@14e40000 {
 		};
 
 		hsi2c_1: i2c@14e50000 {
-			compatible = "samsung,exynos7-hsi2c";
+			compatible = "samsung,exynos5433-hsi2c",
+				     "samsung,exynos7-hsi2c";
 			reg = <0x14e50000 0x1000>;
 			interrupts = <GIC_SPI 429 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -1612,7 +1616,8 @@ hsi2c_1: i2c@14e50000 {
 		};
 
 		hsi2c_2: i2c@14e60000 {
-			compatible = "samsung,exynos7-hsi2c";
+			compatible = "samsung,exynos5433-hsi2c",
+				     "samsung,exynos7-hsi2c";
 			reg = <0x14e60000 0x1000>;
 			interrupts = <GIC_SPI 430 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -1625,7 +1630,8 @@ hsi2c_2: i2c@14e60000 {
 		};
 
 		hsi2c_3: i2c@14e70000 {
-			compatible = "samsung,exynos7-hsi2c";
+			compatible = "samsung,exynos5433-hsi2c",
+				     "samsung,exynos7-hsi2c";
 			reg = <0x14e70000 0x1000>;
 			interrupts = <GIC_SPI 431 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -1638,7 +1644,8 @@ hsi2c_3: i2c@14e70000 {
 		};
 
 		hsi2c_4: i2c@14ec0000 {
-			compatible = "samsung,exynos7-hsi2c";
+			compatible = "samsung,exynos5433-hsi2c",
+				     "samsung,exynos7-hsi2c";
 			reg = <0x14ec0000 0x1000>;
 			interrupts = <GIC_SPI 424 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -1651,7 +1658,8 @@ hsi2c_4: i2c@14ec0000 {
 		};
 
 		hsi2c_5: i2c@14ed0000 {
-			compatible = "samsung,exynos7-hsi2c";
+			compatible = "samsung,exynos5433-hsi2c",
+				     "samsung,exynos7-hsi2c";
 			reg = <0x14ed0000 0x1000>;
 			interrupts = <GIC_SPI 425 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -1664,7 +1672,8 @@ hsi2c_5: i2c@14ed0000 {
 		};
 
 		hsi2c_6: i2c@14ee0000 {
-			compatible = "samsung,exynos7-hsi2c";
+			compatible = "samsung,exynos5433-hsi2c",
+				     "samsung,exynos7-hsi2c";
 			reg = <0x14ee0000 0x1000>;
 			interrupts = <GIC_SPI 426 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -1677,7 +1686,8 @@ hsi2c_6: i2c@14ee0000 {
 		};
 
 		hsi2c_7: i2c@14ef0000 {
-			compatible = "samsung,exynos7-hsi2c";
+			compatible = "samsung,exynos5433-hsi2c",
+				     "samsung,exynos7-hsi2c";
 			reg = <0x14ef0000 0x1000>;
 			interrupts = <GIC_SPI 427 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -1690,7 +1700,8 @@ hsi2c_7: i2c@14ef0000 {
 		};
 
 		hsi2c_8: i2c@14d90000 {
-			compatible = "samsung,exynos7-hsi2c";
+			compatible = "samsung,exynos5433-hsi2c",
+				     "samsung,exynos7-hsi2c";
 			reg = <0x14d90000 0x1000>;
 			interrupts = <GIC_SPI 443 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -1703,7 +1714,8 @@ hsi2c_8: i2c@14d90000 {
 		};
 
 		hsi2c_9: i2c@14da0000 {
-			compatible = "samsung,exynos7-hsi2c";
+			compatible = "samsung,exynos5433-hsi2c",
+				     "samsung,exynos7-hsi2c";
 			reg = <0x14da0000 0x1000>;
 			interrupts = <GIC_SPI 444 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -1716,7 +1728,8 @@ hsi2c_9: i2c@14da0000 {
 		};
 
 		hsi2c_10: i2c@14de0000 {
-			compatible = "samsung,exynos7-hsi2c";
+			compatible = "samsung,exynos5433-hsi2c",
+				     "samsung,exynos7-hsi2c";
 			reg = <0x14de0000 0x1000>;
 			interrupts = <GIC_SPI 445 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -1729,7 +1742,8 @@ hsi2c_10: i2c@14de0000 {
 		};
 
 		hsi2c_11: i2c@14df0000 {
-			compatible = "samsung,exynos7-hsi2c";
+			compatible = "samsung,exynos5433-hsi2c",
+				     "samsung,exynos7-hsi2c";
 			reg = <0x14df0000 0x1000>;
 			interrupts = <GIC_SPI 446 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -1820,7 +1834,8 @@ usbhost_dwc3: usb@0 {
 		};
 
 		mshc_0: mmc@15540000 {
-			compatible = "samsung,exynos7-dw-mshc-smu";
+			compatible = "samsung,exynos5433-dw-mshc-smu",
+				     "samsung,exynos7-dw-mshc-smu";
 			interrupts = <GIC_SPI 225 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -1833,7 +1848,8 @@ mshc_0: mmc@15540000 {
 		};
 
 		mshc_1: mmc@15550000 {
-			compatible = "samsung,exynos7-dw-mshc-smu";
+			compatible = "samsung,exynos5433-dw-mshc-smu",
+				     "samsung,exynos7-dw-mshc-smu";
 			interrupts = <GIC_SPI 226 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -1846,7 +1862,8 @@ mshc_1: mmc@15550000 {
 		};
 
 		mshc_2: mmc@15560000 {
-			compatible = "samsung,exynos7-dw-mshc-smu";
+			compatible = "samsung,exynos5433-dw-mshc-smu",
+				     "samsung,exynos7-dw-mshc-smu";
 			interrupts = <GIC_SPI 227 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -1897,7 +1914,8 @@ adma: dma-controller@11420000 {
 			};
 
 			i2s0: i2s@11440000 {
-				compatible = "samsung,exynos7-i2s";
+				compatible = "samsung,exynos5433-i2s",
+					     "samsung,exynos7-i2s";
 				reg = <0x11440000 0x100>;
 				dmas = <&adma 0>, <&adma 2>;
 				dma-names = "tx", "rx";
diff --git a/arch/arm64/boot/dts/exynos/exynos7.dtsi b/arch/arm64/boot/dts/exynos/exynos7.dtsi
index 6ed80ddf3369..9cb6bd61262e 100644
--- a/arch/arm64/boot/dts/exynos/exynos7.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos7.dtsi
@@ -128,7 +128,8 @@ soc: soc@0 {
 		ranges = <0 0 0 0x18000000>;
 
 		chipid@10000000 {
-			compatible = "samsung,exynos4210-chipid";
+			compatible = "samsung,exynos7-chipid",
+				     "samsung,exynos4210-chipid";
 			reg = <0x10000000 0x100>;
 		};
 
@@ -279,7 +280,7 @@ clock_fsys1: clock-controller@156e0000 {
 		};
 
 		serial_0: serial@13630000 {
-			compatible = "samsung,exynos4210-uart";
+			compatible = "samsung,exynos7-uart", "samsung,exynos4210-uart";
 			reg = <0x13630000 0x100>;
 			interrupts = <GIC_SPI 440 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock_peric0 PCLK_UART0>,
@@ -289,7 +290,7 @@ serial_0: serial@13630000 {
 		};
 
 		serial_1: serial@14c20000 {
-			compatible = "samsung,exynos4210-uart";
+			compatible = "samsung,exynos7-uart", "samsung,exynos4210-uart";
 			reg = <0x14c20000 0x100>;
 			interrupts = <GIC_SPI 456 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock_peric1 PCLK_UART1>,
@@ -299,7 +300,7 @@ serial_1: serial@14c20000 {
 		};
 
 		serial_2: serial@14c30000 {
-			compatible = "samsung,exynos4210-uart";
+			compatible = "samsung,exynos7-uart", "samsung,exynos4210-uart";
 			reg = <0x14c30000 0x100>;
 			interrupts = <GIC_SPI 457 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock_peric1 PCLK_UART2>,
@@ -309,7 +310,7 @@ serial_2: serial@14c30000 {
 		};
 
 		serial_3: serial@14c40000 {
-			compatible = "samsung,exynos4210-uart";
+			compatible = "samsung,exynos7-uart", "samsung,exynos4210-uart";
 			reg = <0x14c40000 0x100>;
 			interrupts = <GIC_SPI 458 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clock_peric1 PCLK_UART3>,
@@ -539,7 +540,7 @@ pmu_system_controller: system-controller@105c0000 {
 		};
 
 		rtc: rtc@10590000 {
-			compatible = "samsung,s3c6410-rtc";
+			compatible = "samsung,exynos7-rtc", "samsung,s3c6410-rtc";
 			reg = <0x10590000 0x100>;
 			interrupts = <GIC_SPI 355 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 356 IRQ_TYPE_LEVEL_HIGH>;
@@ -559,7 +560,8 @@ watchdog: watchdog@101d0000 {
 		};
 
 		gpu: gpu@14ac0000 {
-			compatible = "samsung,exynos5433-mali", "arm,mali-t760";
+			compatible = "samsung,exynos7-mali",
+				     "samsung,exynos5433-mali", "arm,mali-t760";
 			reg = <0x14ac0000 0x5000>;
 			interrupts = <GIC_SPI 241 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 242 IRQ_TYPE_LEVEL_HIGH>,
@@ -619,7 +621,7 @@ adc: adc@13620000 {
 		};
 
 		pwm: pwm@136c0000 {
-			compatible = "samsung,exynos4210-pwm";
+			compatible = "samsung,exynos7-pwm", "samsung,exynos4210-pwm";
 			reg = <0x136c0000 0x100>;
 			interrupts = <GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 450 IRQ_TYPE_LEVEL_HIGH>,
diff --git a/arch/arm64/boot/dts/exynos/exynos7885.dtsi b/arch/arm64/boot/dts/exynos/exynos7885.dtsi
index d69fc2392bd0..008228fb319a 100644
--- a/arch/arm64/boot/dts/exynos/exynos7885.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos7885.dtsi
@@ -172,7 +172,8 @@ soc: soc@0 {
 		ranges = <0x0 0x0 0x0 0x20000000>;
 
 		chipid@10000000 {
-			compatible = "samsung,exynos850-chipid";
+			compatible = "samsung,exynos7885-chipid",
+				     "samsung,exynos850-chipid";
 			reg = <0x10000000 0x24>;
 		};
 
@@ -264,7 +265,8 @@ pinctrl_alive: pinctrl@11cb0000 {
 			reg = <0x11cb0000 0x1000>;
 
 			wakeup-interrupt-controller {
-				compatible = "samsung,exynos7-wakeup-eint";
+				compatible = "samsung,exynos7885-wakeup-eint",
+					     "samsung,exynos7-wakeup-eint";
 				interrupt-parent = <&gic>;
 				interrupts = <GIC_SPI 130 IRQ_TYPE_LEVEL_HIGH>;
 			};
@@ -289,12 +291,14 @@ pinctrl_dispaud: pinctrl@148f0000 {
 		};
 
 		pmu_system_controller: system-controller@11c80000 {
-			compatible = "samsung,exynos7-pmu", "syscon";
+			compatible = "samsung,exynos7885-pmu",
+				     "samsung,exynos7-pmu", "syscon";
 			reg = <0x11c80000 0x10000>;
 		};
 
 		mmc_0: mmc@13500000 {
-			compatible = "samsung,exynos7-dw-mshc-smu";
+			compatible = "samsung,exynos7885-dw-mshc-smu",
+				     "samsung,exynos7-dw-mshc-smu";
 			reg = <0x13500000 0x2000>;
 			interrupts = <GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -307,7 +311,8 @@ mmc_0: mmc@13500000 {
 		};
 
 		serial_0: serial@13800000 {
-			compatible = "samsung,exynos5433-uart";
+			compatible = "samsung,exynos7885-uart",
+				     "samsung,exynos5433-uart";
 			reg = <0x13800000 0x100>;
 			interrupts = <GIC_SPI 246 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
@@ -320,7 +325,8 @@ serial_0: serial@13800000 {
 		};
 
 		serial_1: serial@13810000 {
-			compatible = "samsung,exynos5433-uart";
+			compatible = "samsung,exynos7885-uart",
+				     "samsung,exynos5433-uart";
 			reg = <0x13810000 0x100>;
 			interrupts = <GIC_SPI 247 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
@@ -333,7 +339,8 @@ serial_1: serial@13810000 {
 		};
 
 		serial_2: serial@13820000 {
-			compatible = "samsung,exynos5433-uart";
+			compatible = "samsung,exynos7885-uart",
+				     "samsung,exynos5433-uart";
 			reg = <0x13820000 0x100>;
 			interrupts = <GIC_SPI 279 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
@@ -346,7 +353,8 @@ serial_2: serial@13820000 {
 		};
 
 		i2c_0: i2c@13830000 {
-			compatible = "samsung,s3c2440-i2c";
+			compatible = "samsung,exynos7885-i2c",
+				     "samsung,s3c2440-i2c";
 			reg = <0x13830000 0x100>;
 			interrupts = <GIC_SPI 248 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -359,7 +367,8 @@ i2c_0: i2c@13830000 {
 		};
 
 		i2c_1: i2c@13840000 {
-			compatible = "samsung,s3c2440-i2c";
+			compatible = "samsung,exynos7885-i2c",
+				     "samsung,s3c2440-i2c";
 			reg = <0x13840000 0x100>;
 			interrupts = <GIC_SPI 249 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -372,7 +381,8 @@ i2c_1: i2c@13840000 {
 		};
 
 		i2c_2: i2c@13850000 {
-			compatible = "samsung,s3c2440-i2c";
+			compatible = "samsung,exynos7885-i2c",
+				     "samsung,s3c2440-i2c";
 			reg = <0x13850000 0x100>;
 			interrupts = <GIC_SPI 250 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -385,7 +395,8 @@ i2c_2: i2c@13850000 {
 		};
 
 		i2c_3: i2c@13860000 {
-			compatible = "samsung,s3c2440-i2c";
+			compatible = "samsung,exynos7885-i2c",
+				     "samsung,s3c2440-i2c";
 			reg = <0x13860000 0x100>;
 			interrupts = <GIC_SPI 251 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -398,7 +409,8 @@ i2c_3: i2c@13860000 {
 		};
 
 		i2c_4: i2c@13870000 {
-			compatible = "samsung,s3c2440-i2c";
+			compatible = "samsung,exynos7885-i2c",
+				     "samsung,s3c2440-i2c";
 			reg = <0x13870000 0x100>;
 			interrupts = <GIC_SPI 252 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -411,7 +423,8 @@ i2c_4: i2c@13870000 {
 		};
 
 		i2c_5: i2c@13880000 {
-			compatible = "samsung,s3c2440-i2c";
+			compatible = "samsung,exynos7885-i2c",
+				     "samsung,s3c2440-i2c";
 			reg = <0x13880000 0x100>;
 			interrupts = <GIC_SPI 253 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -424,7 +437,8 @@ i2c_5: i2c@13880000 {
 		};
 
 		i2c_6: i2c@13890000 {
-			compatible = "samsung,s3c2440-i2c";
+			compatible = "samsung,exynos7885-i2c",
+				     "samsung,s3c2440-i2c";
 			reg = <0x13890000 0x100>;
 			interrupts = <GIC_SPI 254 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -437,7 +451,8 @@ i2c_6: i2c@13890000 {
 		};
 
 		i2c_7: i2c@11cd0000 {
-			compatible = "samsung,s3c2440-i2c";
+			compatible = "samsung,exynos7885-i2c",
+				     "samsung,s3c2440-i2c";
 			reg = <0x11cd0000 0x100>;
 			interrupts = <GIC_SPI 255 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/exynos/exynos850.dtsi b/arch/arm64/boot/dts/exynos/exynos850.dtsi
index 53104e65b9c6..df5ea43ebcad 100644
--- a/arch/arm64/boot/dts/exynos/exynos850.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos850.dtsi
@@ -396,7 +396,7 @@ pinctrl_aud: pinctrl@14a60000 {
 		};
 
 		rtc: rtc@11a30000 {
-			compatible = "samsung,s3c6410-rtc";
+			compatible = "samsung,exynos850-rtc", "samsung,s3c6410-rtc";
 			reg = <0x11a30000 0x100>;
 			interrupts = <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>;
@@ -406,7 +406,8 @@ rtc: rtc@11a30000 {
 		};
 
 		mmc_0: mmc@12100000 {
-			compatible = "samsung,exynos7-dw-mshc-smu";
+			compatible = "samsung,exynos850-dw-mshc-smu",
+				     "samsung,exynos7-dw-mshc-smu";
 			reg = <0x12100000 0x2000>;
 			interrupts = <GIC_SPI 452 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -419,7 +420,7 @@ mmc_0: mmc@12100000 {
 		};
 
 		i2c_0: i2c@13830000 {
-			compatible = "samsung,s3c2440-i2c";
+			compatible = "samsung,exynos850-i2c", "samsung,s3c2440-i2c";
 			reg = <0x13830000 0x100>;
 			interrupts = <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -432,7 +433,7 @@ i2c_0: i2c@13830000 {
 		};
 
 		i2c_1: i2c@13840000 {
-			compatible = "samsung,s3c2440-i2c";
+			compatible = "samsung,exynos850-i2c", "samsung,s3c2440-i2c";
 			reg = <0x13840000 0x100>;
 			interrupts = <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -445,7 +446,7 @@ i2c_1: i2c@13840000 {
 		};
 
 		i2c_2: i2c@13850000 {
-			compatible = "samsung,s3c2440-i2c";
+			compatible = "samsung,exynos850-i2c", "samsung,s3c2440-i2c";
 			reg = <0x13850000 0x100>;
 			interrupts = <GIC_SPI 198 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -458,7 +459,7 @@ i2c_2: i2c@13850000 {
 		};
 
 		i2c_3: i2c@13860000 {
-			compatible = "samsung,s3c2440-i2c";
+			compatible = "samsung,exynos850-i2c", "samsung,s3c2440-i2c";
 			reg = <0x13860000 0x100>;
 			interrupts = <GIC_SPI 199 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -471,7 +472,7 @@ i2c_3: i2c@13860000 {
 		};
 
 		i2c_4: i2c@13870000 {
-			compatible = "samsung,s3c2440-i2c";
+			compatible = "samsung,exynos850-i2c", "samsung,s3c2440-i2c";
 			reg = <0x13870000 0x100>;
 			interrupts = <GIC_SPI 200 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -485,7 +486,7 @@ i2c_4: i2c@13870000 {
 
 		/* I2C_5 (also called CAM_PMIC_I2C in TRM) */
 		i2c_5: i2c@13880000 {
-			compatible = "samsung,s3c2440-i2c";
+			compatible = "samsung,exynos850-i2c", "samsung,s3c2440-i2c";
 			reg = <0x13880000 0x100>;
 			interrupts = <GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -499,7 +500,7 @@ i2c_5: i2c@13880000 {
 
 		/* I2C_6 (also called MOTOR_I2C in TRM) */
 		i2c_6: i2c@13890000 {
-			compatible = "samsung,s3c2440-i2c";
+			compatible = "samsung,exynos850-i2c", "samsung,s3c2440-i2c";
 			reg = <0x13890000 0x100>;
 			interrupts = <GIC_SPI 202 IRQ_TYPE_LEVEL_HIGH>;
 			#address-cells = <1>;
@@ -640,7 +641,8 @@ usi_hsi2c_0: usi@138a00c0 {
 			status = "disabled";
 
 			hsi2c_0: i2c@138a0000 {
-				compatible = "samsung,exynosautov9-hsi2c";
+				compatible = "samsung,exynos850-hsi2c",
+					     "samsung,exynosautov9-hsi2c";
 				reg = <0x138a0000 0xc0>;
 				interrupts = <GIC_SPI 193 IRQ_TYPE_LEVEL_HIGH>;
 				#address-cells = <1>;
@@ -668,7 +670,8 @@ usi_hsi2c_1: usi@138b00c0 {
 			status = "disabled";
 
 			hsi2c_1: i2c@138b0000 {
-				compatible = "samsung,exynosautov9-hsi2c";
+				compatible = "samsung,exynos850-hsi2c",
+					     "samsung,exynosautov9-hsi2c";
 				reg = <0x138b0000 0xc0>;
 				interrupts = <GIC_SPI 194 IRQ_TYPE_LEVEL_HIGH>;
 				#address-cells = <1>;
@@ -696,7 +699,8 @@ usi_hsi2c_2: usi@138c00c0 {
 			status = "disabled";
 
 			hsi2c_2: i2c@138c0000 {
-				compatible = "samsung,exynosautov9-hsi2c";
+				compatible = "samsung,exynos850-hsi2c",
+					     "samsung,exynosautov9-hsi2c";
 				reg = <0x138c0000 0xc0>;
 				interrupts = <GIC_SPI 195 IRQ_TYPE_LEVEL_HIGH>;
 				#address-cells = <1>;
@@ -738,7 +742,8 @@ usi_cmgp0: usi@11d000c0 {
 			status = "disabled";
 
 			hsi2c_3: i2c@11d00000 {
-				compatible = "samsung,exynosautov9-hsi2c";
+				compatible = "samsung,exynos850-hsi2c",
+					     "samsung,exynosautov9-hsi2c";
 				reg = <0x11d00000 0xc0>;
 				interrupts = <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>;
 				#address-cells = <1>;
@@ -778,7 +783,8 @@ usi_cmgp1: usi@11d200c0 {
 			status = "disabled";
 
 			hsi2c_4: i2c@11d20000 {
-				compatible = "samsung,exynosautov9-hsi2c";
+				compatible = "samsung,exynos850-hsi2c",
+					     "samsung,exynosautov9-hsi2c";
 				reg = <0x11d20000 0xc0>;
 				interrupts = <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>;
 				#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/exynos/exynosautov9-sadk.dts b/arch/arm64/boot/dts/exynos/exynosautov9-sadk.dts
index bc1815f6ada2..de2c1de51a76 100644
--- a/arch/arm64/boot/dts/exynos/exynosautov9-sadk.dts
+++ b/arch/arm64/boot/dts/exynos/exynosautov9-sadk.dts
@@ -9,6 +9,7 @@
 /dts-v1/;
 #include "exynosautov9.dtsi"
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
 
 / {
 	model = "Samsung ExynosAuto v9 SADK board";
@@ -32,6 +33,31 @@ memory@80000000 {
 		      <0xa 0x00000000 0x2 0x00000000>;
 	};
 
+	gpio-keys {
+		compatible = "gpio-keys";
+		pinctrl-names = "default";
+		pinctrl-0 = <&key_wakeup &key_volup &key_voldown>;
+
+		key-wakeup {
+			label = "Wakeup";
+			linux,code = <KEY_WAKEUP>;
+			gpios = <&gpa0 0 GPIO_ACTIVE_LOW>;
+			wakeup-source;
+		};
+
+		key-volume-down {
+			label = "Volume Down";
+			linux,code = <KEY_VOLUMEDOWN>;
+			gpios = <&gpg2 0 GPIO_ACTIVE_LOW>;
+		};
+
+		key-volume-up {
+			label = "Volume Up";
+			linux,code = <KEY_VOLUMEUP>;
+			gpios = <&gpg1 5 GPIO_ACTIVE_LOW>;
+		};
+	};
+
 	ufs_0_fixed_vcc_reg: regulator-0 {
 		compatible = "regulator-fixed";
 		regulator-name = "ufs-vcc";
@@ -49,6 +75,31 @@ ufs_1_fixed_vcc_reg: regulator-1 {
 	};
 };
 
+&pinctrl_alive {
+	key_wakeup: key-wakeup-pins {
+		samsung,pins = "gpa0-0";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_EINT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS5420_PIN_DRV_LV1>;
+	};
+};
+
+&pinctrl_peric1 {
+	key_voldown: key-voldown-pins {
+		samsung,pins = "gpg2-0";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_EINT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS5420_PIN_DRV_LV1>;
+	};
+
+	key_volup: key-volup-pins {
+		samsung,pins = "gpg1-5";
+		samsung,pin-function = <EXYNOS_PIN_FUNC_EINT>;
+		samsung,pin-pud = <EXYNOS_PIN_PULL_NONE>;
+		samsung,pin-drv = <EXYNOS5420_PIN_DRV_LV1>;
+	};
+};
+
 &pwm {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pwm_tout3>;
diff --git a/arch/arm64/boot/dts/exynos/exynosautov9.dtsi b/arch/arm64/boot/dts/exynos/exynosautov9.dtsi
index b228cd7e351e..417aa56a81f6 100644
--- a/arch/arm64/boot/dts/exynos/exynosautov9.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynosautov9.dtsi
@@ -166,7 +166,8 @@ soc: soc@0 {
 		ranges = <0x0 0x0 0x0 0x20000000>;
 
 		chipid@10000000 {
-			compatible = "samsung,exynos850-chipid";
+			compatible = "samsung,exynosautov9-chipid",
+				     "samsung,exynos850-chipid";
 			reg = <0x10000000 0x24>;
 		};
 
@@ -349,7 +350,8 @@ pinctrl_peric1: pinctrl@10830000 {
 		};
 
 		pmu_system_controller: system-controller@10460000 {
-			compatible = "samsung,exynos7-pmu", "syscon";
+			compatible = "samsung,exynosautov9-pmu",
+				     "samsung,exynos7-pmu", "syscon";
 			reg = <0x10460000 0x10000>;
 
 			reboot: syscon-reboot {
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-evk.dts b/arch/arm64/boot/dts/freescale/imx8mp-evk.dts
index cc9d468b43ab..f87fa5a948cc 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-evk.dts
@@ -137,6 +137,28 @@ simple-audio-card,codec {
 		};
 
 	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		dsp_vdev0vring0: vdev0vring0@942f0000 {
+			reg = <0 0x942f0000 0 0x8000>;
+			no-map;
+		};
+
+		dsp_vdev0vring1: vdev0vring1@942f8000 {
+			reg = <0 0x942f8000 0 0x8000>;
+			no-map;
+		};
+
+		dsp_vdev0buffer: vdev0buffer@94300000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0x94300000 0 0x100000>;
+			no-map;
+		};
+	};
 };
 
 &flexspi {
diff --git a/arch/arm64/boot/dts/qcom/ipq5018-rdp432-c2.dts b/arch/arm64/boot/dts/qcom/ipq5018-rdp432-c2.dts
index e636a1cb9b77..8460b538eb6a 100644
--- a/arch/arm64/boot/dts/qcom/ipq5018-rdp432-c2.dts
+++ b/arch/arm64/boot/dts/qcom/ipq5018-rdp432-c2.dts
@@ -67,6 +67,18 @@ data-pins {
 	};
 };
 
+&usb {
+	status = "okay";
+};
+
+&usb_dwc {
+	dr_mode = "host";
+};
+
+&usbphy0 {
+	status = "okay";
+};
+
 &xo_board_clk {
 	clock-frequency = <24000000>;
 };
diff --git a/arch/arm64/boot/dts/qcom/ipq5018.dtsi b/arch/arm64/boot/dts/qcom/ipq5018.dtsi
index 38ffdc3cbdcd..0b739077ed70 100644
--- a/arch/arm64/boot/dts/qcom/ipq5018.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq5018.dtsi
@@ -94,6 +94,19 @@ soc: soc@0 {
 		#size-cells = <1>;
 		ranges = <0 0 0 0xffffffff>;
 
+		usbphy0: phy@5b000 {
+			compatible = "qcom,ipq5018-usb-hsphy";
+			reg = <0x0005b000 0x120>;
+
+			clocks = <&gcc GCC_USB0_PHY_CFG_AHB_CLK>;
+
+			resets = <&gcc GCC_QUSB2_0_PHY_BCR>;
+
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
 		tlmm: pinctrl@1000000 {
 			compatible = "qcom,ipq5018-tlmm";
 			reg = <0x01000000 0x300000>;
@@ -146,6 +159,16 @@ sdhc_1: mmc@7804000 {
 			status = "disabled";
 		};
 
+		blsp_dma: dma-controller@7884000 {
+			compatible = "qcom,bam-v1.7.0";
+			reg = <0x07884000 0x1d000>;
+			interrupts = <GIC_SPI 238 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gcc GCC_BLSP1_AHB_CLK>;
+			clock-names = "bam_clk";
+			#dma-cells = <1>;
+			qcom,ee = <0>;
+		};
+
 		blsp1_uart1: serial@78af000 {
 			compatible = "qcom,msm-uartdm-v1.4", "qcom,msm-uartdm";
 			reg = <0x078af000 0x200>;
@@ -156,6 +179,61 @@ blsp1_uart1: serial@78af000 {
 			status = "disabled";
 		};
 
+		blsp1_spi1: spi@78b5000 {
+			compatible = "qcom,spi-qup-v2.2.1";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x078b5000 0x600>;
+			interrupts = <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gcc GCC_BLSP1_QUP1_SPI_APPS_CLK>,
+				 <&gcc GCC_BLSP1_AHB_CLK>;
+			clock-names = "core", "iface";
+			dmas = <&blsp_dma 4>, <&blsp_dma 5>;
+			dma-names = "tx", "rx";
+			status = "disabled";
+		};
+
+		usb: usb@8af8800 {
+			compatible = "qcom,ipq5018-dwc3", "qcom,dwc3";
+			reg = <0x08af8800 0x400>;
+
+			interrupts = <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "hs_phy_irq";
+
+			clocks = <&gcc GCC_USB0_MASTER_CLK>,
+				 <&gcc GCC_SYS_NOC_USB0_AXI_CLK>,
+				 <&gcc GCC_USB0_SLEEP_CLK>,
+				 <&gcc GCC_USB0_MOCK_UTMI_CLK>;
+			clock-names = "core",
+				      "iface",
+				      "sleep",
+				      "mock_utmi";
+
+			resets = <&gcc GCC_USB0_BCR>;
+
+			qcom,select-utmi-as-pipe-clk;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+
+			status = "disabled";
+
+			usb_dwc: usb@8a00000 {
+				compatible = "snps,dwc3";
+				reg = <0x08a00000 0xe000>;
+				clocks = <&gcc GCC_USB0_MOCK_UTMI_CLK>;
+				clock-names = "ref";
+				interrupts = <GIC_SPI 140 IRQ_TYPE_LEVEL_HIGH>;
+				phy-names = "usb2-phy";
+				phys = <&usbphy0>;
+				tx-fifo-resize;
+				snps,is-utmi-l1-suspend;
+				snps,hird-threshold = /bits/ 8 <0x0>;
+				snps,dis_u2_susphy_quirk;
+				snps,dis_u3_susphy_quirk;
+			};
+		};
+
 		intc: interrupt-controller@b000000 {
 			compatible = "qcom,msm-qgic2";
 			reg = <0x0b000000 0x1000>,  /* GICD */
diff --git a/arch/arm64/boot/dts/qcom/ipq5332.dtsi b/arch/arm64/boot/dts/qcom/ipq5332.dtsi
index d3fef2f80a81..42e2e48b2bc3 100644
--- a/arch/arm64/boot/dts/qcom/ipq5332.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq5332.dtsi
@@ -91,11 +91,19 @@ memory@40000000 {
 	};
 
 	cpu_opp_table: opp-table-cpu {
-		compatible = "operating-points-v2";
+		compatible = "operating-points-v2-kryo-cpu";
 		opp-shared;
+		nvmem-cells = <&cpu_speed_bin>;
 
-		opp-1488000000 {
-			opp-hz = /bits/ 64 <1488000000>;
+		opp-1100000000 {
+			opp-hz = /bits/ 64 <1100000000>;
+			opp-supported-hw = <0x7>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp-1500000000 {
+			opp-hz = /bits/ 64 <1500000000>;
+			opp-supported-hw = <0x3>;
 			clock-latency-ns = <200000>;
 		};
 	};
@@ -163,6 +171,11 @@ qfprom: efuse@a4000 {
 			reg = <0x000a4000 0x721>;
 			#address-cells = <1>;
 			#size-cells = <1>;
+
+			cpu_speed_bin: cpu-speed-bin@1d {
+				reg = <0x1d 0x2>;
+				bits = <7 2>;
+			};
 		};
 
 		rng: rng@e3000 {
@@ -390,8 +403,8 @@ apcs_glb: mailbox@b111000 {
 				     "qcom,ipq6018-apcs-apps-global";
 			reg = <0x0b111000 0x1000>;
 			#clock-cells = <1>;
-			clocks = <&a53pll>, <&xo_board>;
-			clock-names = "pll", "xo";
+			clocks = <&a53pll>, <&xo_board>, <&gcc GPLL0>;
+			clock-names = "pll", "xo", "gpll0";
 			#mbox-cells = <1>;
 		};
 
diff --git a/arch/arm64/boot/dts/qcom/ipq6018.dtsi b/arch/arm64/boot/dts/qcom/ipq6018.dtsi
index e59b9df96c7e..9aec89d5e095 100644
--- a/arch/arm64/boot/dts/qcom/ipq6018.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq6018.dtsi
@@ -611,8 +611,8 @@ apcs_glb: mailbox@b111000 {
 			compatible = "qcom,ipq6018-apcs-apps-global";
 			reg = <0x0 0x0b111000 0x0 0x1000>;
 			#clock-cells = <1>;
-			clocks = <&a53pll>, <&xo>;
-			clock-names = "pll", "xo";
+			clocks = <&a53pll>, <&xo>, <&gcc GPLL0>;
+			clock-names = "pll", "xo", "gpll0";
 			#mbox-cells = <1>;
 		};
 
diff --git a/arch/arm64/boot/dts/qcom/ipq8074.dtsi b/arch/arm64/boot/dts/qcom/ipq8074.dtsi
index 2f275c84e566..5d05819f356d 100644
--- a/arch/arm64/boot/dts/qcom/ipq8074.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq8074.dtsi
@@ -708,8 +708,8 @@ apcs_glb: mailbox@b111000 {
 			compatible = "qcom,ipq8074-apcs-apps-global",
 				     "qcom,ipq6018-apcs-apps-global";
 			reg = <0x0b111000 0x1000>;
-			clocks = <&a53pll>, <&xo>;
-			clock-names = "pll", "xo";
+			clocks = <&a53pll>, <&xo>, <&gcc GPLL0>;
+			clock-names = "pll", "xo", "gpll0";
 
 			#clock-cells = <1>;
 			#mbox-cells = <1>;
diff --git a/arch/arm64/boot/dts/qcom/ipq9574-rdp-common.dtsi b/arch/arm64/boot/dts/qcom/ipq9574-rdp-common.dtsi
new file mode 100644
index 000000000000..49c9b6478357
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/ipq9574-rdp-common.dtsi
@@ -0,0 +1,147 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * IPQ9574 RDP board common device tree source
+ *
+ * Copyright (c) 2020-2021 The Linux Foundation. All rights reserved.
+ * Copyright (c) 2023, Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include "ipq9574.dtsi"
+
+/ {
+	aliases {
+		serial0 = &blsp1_uart2;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	regulator_fixed_3p3: s3300 {
+		compatible = "regulator-fixed";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		regulator-always-on;
+		regulator-name = "fixed_3p3";
+	};
+
+	regulator_fixed_0p925: s0925 {
+		compatible = "regulator-fixed";
+		regulator-min-microvolt = <925000>;
+		regulator-max-microvolt = <925000>;
+		regulator-boot-on;
+		regulator-always-on;
+		regulator-name = "fixed_0p925";
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		pinctrl-0 = <&gpio_keys_default>;
+		pinctrl-names = "default";
+
+		button-wps {
+			label = "wps";
+			linux,code = <KEY_WPS_BUTTON>;
+			gpios = <&tlmm 37 GPIO_ACTIVE_LOW>;
+			debounce-interval = <60>;
+		};
+	};
+};
+
+&blsp1_spi0 {
+	pinctrl-0 = <&spi_0_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	flash@0 {
+		compatible = "micron,n25q128a11", "jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <50000000>;
+	};
+};
+
+&blsp1_uart2 {
+	pinctrl-0 = <&uart2_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&rpm_requests {
+	regulators {
+		compatible = "qcom,rpm-mp5496-regulators";
+
+		ipq9574_s1: s1 {
+		/*
+		 * During kernel bootup, the SoC runs at 800MHz with 875mV set by the bootloaders.
+		 * During regulator registration, kernel not knowing the initial voltage,
+		 * considers it as zero and brings up the regulators with minimum supported voltage.
+		 * Update the regulator-min-microvolt with SVS voltage of 725mV so that
+		 * the regulators are brought up with 725mV which is sufficient for all the
+		 * corner parts to operate at 800MHz
+		 */
+			regulator-min-microvolt = <725000>;
+			regulator-max-microvolt = <1075000>;
+		};
+
+		mp5496_l2: l2 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-always-on;
+			regulator-boot-on;
+		};
+	};
+};
+
+&sleep_clk {
+	clock-frequency = <32000>;
+};
+
+&tlmm {
+	spi_0_pins: spi-0-state {
+		pins = "gpio11", "gpio12", "gpio13", "gpio14";
+		function = "blsp0_spi";
+		drive-strength = <8>;
+		bias-disable;
+	};
+
+	gpio_keys_default: gpio-keys-default-state {
+		pins = "gpio37";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-pull-up;
+	};
+};
+
+&usb_0_dwc3 {
+	dr_mode = "host";
+};
+
+&usb_0_qmpphy {
+	vdda-pll-supply = <&mp5496_l2>;
+	vdda-phy-supply = <&regulator_fixed_0p925>;
+
+	status = "okay";
+};
+
+&usb_0_qusbphy {
+	vdd-supply = <&regulator_fixed_0p925>;
+	vdda-pll-supply = <&mp5496_l2>;
+	vdda-phy-dpdm-supply = <&regulator_fixed_3p3>;
+
+	status = "okay";
+};
+
+&usb3 {
+	status = "okay";
+};
+
+&xo_board_clk {
+	clock-frequency = <24000000>;
+};
diff --git a/arch/arm64/boot/dts/qcom/ipq9574-rdp418.dts b/arch/arm64/boot/dts/qcom/ipq9574-rdp418.dts
index 2b093e02637b..f4f9199d4ab1 100644
--- a/arch/arm64/boot/dts/qcom/ipq9574-rdp418.dts
+++ b/arch/arm64/boot/dts/qcom/ipq9574-rdp418.dts
@@ -8,58 +8,12 @@
 
 /dts-v1/;
 
-#include "ipq9574.dtsi"
+#include "ipq9574-rdp-common.dtsi"
 
 / {
 	model = "Qualcomm Technologies, Inc. IPQ9574/AP-AL02-C2";
 	compatible = "qcom,ipq9574-ap-al02-c2", "qcom,ipq9574";
 
-	aliases {
-		serial0 = &blsp1_uart2;
-	};
-
-	chosen {
-		stdout-path = "serial0:115200n8";
-	};
-};
-
-&blsp1_spi0 {
-	pinctrl-0 = <&spi_0_pins>;
-	pinctrl-names = "default";
-	status = "okay";
-
-	flash@0 {
-		compatible = "micron,n25q128a11", "jedec,spi-nor";
-		reg = <0>;
-		#address-cells = <1>;
-		#size-cells = <1>;
-		spi-max-frequency = <50000000>;
-	};
-};
-
-&blsp1_uart2 {
-	pinctrl-0 = <&uart2_pins>;
-	pinctrl-names = "default";
-	status = "okay";
-};
-
-&rpm_requests {
-	regulators {
-		compatible = "qcom,rpm-mp5496-regulators";
-
-		ipq9574_s1: s1 {
-		/*
-		 * During kernel bootup, the SoC runs at 800MHz with 875mV set by the bootloaders.
-		 * During regulator registration, kernel not knowing the initial voltage,
-		 * considers it as zero and brings up the regulators with minimum supported voltage.
-		 * Update the regulator-min-microvolt with SVS voltage of 725mV so that
-		 * the regulators are brought up with 725mV which is sufficient for all the
-		 * corner parts to operate at 800MHz
-		 */
-			regulator-min-microvolt = <725000>;
-			regulator-max-microvolt = <1075000>;
-		};
-	};
 };
 
 &sdhc_1 {
@@ -74,10 +28,6 @@ &sdhc_1 {
 	status = "okay";
 };
 
-&sleep_clk {
-	clock-frequency = <32000>;
-};
-
 &tlmm {
 	sdc_default_state: sdc-default-state {
 		clk-pins {
@@ -110,15 +60,4 @@ rclk-pins {
 			bias-pull-down;
 		};
 	};
-
-	spi_0_pins: spi-0-state {
-		pins = "gpio11", "gpio12", "gpio13", "gpio14";
-		function = "blsp0_spi";
-		drive-strength = <8>;
-		bias-disable;
-	};
-};
-
-&xo_board_clk {
-	clock-frequency = <24000000>;
 };
diff --git a/arch/arm64/boot/dts/qcom/ipq9574-rdp433.dts b/arch/arm64/boot/dts/qcom/ipq9574-rdp433.dts
index 877026ccc6e2..1bb8d96c9a82 100644
--- a/arch/arm64/boot/dts/qcom/ipq9574-rdp433.dts
+++ b/arch/arm64/boot/dts/qcom/ipq9574-rdp433.dts
@@ -8,69 +8,11 @@
 
 /dts-v1/;
 
-#include "ipq9574.dtsi"
+#include "ipq9574-rdp-common.dtsi"
 
 / {
 	model = "Qualcomm Technologies, Inc. IPQ9574/AP-AL02-C7";
 	compatible = "qcom,ipq9574-ap-al02-c7", "qcom,ipq9574";
-
-	aliases {
-		serial0 = &blsp1_uart2;
-	};
-
-	chosen {
-		stdout-path = "serial0:115200n8";
-	};
-
-	regulator_fixed_3p3: s3300 {
-		compatible = "regulator-fixed";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		regulator-boot-on;
-		regulator-always-on;
-		regulator-name = "fixed_3p3";
-	};
-
-	regulator_fixed_0p925: s0925 {
-		compatible = "regulator-fixed";
-		regulator-min-microvolt = <925000>;
-		regulator-max-microvolt = <925000>;
-		regulator-boot-on;
-		regulator-always-on;
-		regulator-name = "fixed_0p925";
-	};
-};
-
-&blsp1_uart2 {
-	pinctrl-0 = <&uart2_pins>;
-	pinctrl-names = "default";
-	status = "okay";
-};
-
-&rpm_requests {
-	regulators {
-		compatible = "qcom,rpm-mp5496-regulators";
-
-		ipq9574_s1: s1 {
-		/*
-		 * During kernel bootup, the SoC runs at 800MHz with 875mV set by the bootloaders.
-		 * During regulator registration, kernel not knowing the initial voltage,
-		 * considers it as zero and brings up the regulators with minimum supported voltage.
-		 * Update the regulator-min-microvolt with SVS voltage of 725mV so that
-		 * the regulators are brought up with 725mV which is sufficient for all the
-		 * corner parts to operate at 800MHz
-		 */
-			regulator-min-microvolt = <725000>;
-			regulator-max-microvolt = <1075000>;
-		};
-
-		mp5496_l2: l2 {
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1800000>;
-			regulator-always-on;
-			regulator-boot-on;
-		};
-	};
 };
 
 &sdhc_1 {
@@ -85,10 +27,6 @@ &sdhc_1 {
 	status = "okay";
 };
 
-&sleep_clk {
-	clock-frequency = <32000>;
-};
-
 &tlmm {
 	sdc_default_state: sdc-default-state {
 		clk-pins {
@@ -122,30 +60,3 @@ rclk-pins {
 		};
 	};
 };
-
-&usb_0_dwc3 {
-	dr_mode = "host";
-};
-
-&usb_0_qmpphy {
-	vdda-pll-supply = <&mp5496_l2>;
-	vdda-phy-supply = <&regulator_fixed_0p925>;
-
-	status = "okay";
-};
-
-&usb_0_qusbphy {
-	vdd-supply = <&regulator_fixed_0p925>;
-	vdda-pll-supply = <&mp5496_l2>;
-	vdda-phy-dpdm-supply = <&regulator_fixed_3p3>;
-
-	status = "okay";
-};
-
-&usb3 {
-	status = "okay";
-};
-
-&xo_board_clk {
-	clock-frequency = <24000000>;
-};
diff --git a/arch/arm64/boot/dts/qcom/ipq9574-rdp449.dts b/arch/arm64/boot/dts/qcom/ipq9574-rdp449.dts
index c8fa54e1a62c..d36d1078763e 100644
--- a/arch/arm64/boot/dts/qcom/ipq9574-rdp449.dts
+++ b/arch/arm64/boot/dts/qcom/ipq9574-rdp449.dts
@@ -8,73 +8,10 @@
 
 /dts-v1/;
 
-#include "ipq9574.dtsi"
+#include "ipq9574-rdp-common.dtsi"
 
 / {
 	model = "Qualcomm Technologies, Inc. IPQ9574/AP-AL02-C6";
 	compatible = "qcom,ipq9574-ap-al02-c6", "qcom,ipq9574";
 
-	aliases {
-		serial0 = &blsp1_uart2;
-	};
-
-	chosen {
-		stdout-path = "serial0:115200n8";
-	};
-};
-
-&blsp1_spi0 {
-	pinctrl-0 = <&spi_0_pins>;
-	pinctrl-names = "default";
-	status = "okay";
-
-	flash@0 {
-		compatible = "micron,n25q128a11", "jedec,spi-nor";
-		reg = <0>;
-		#address-cells = <1>;
-		#size-cells = <1>;
-		spi-max-frequency = <50000000>;
-	};
-};
-
-&blsp1_uart2 {
-	pinctrl-0 = <&uart2_pins>;
-	pinctrl-names = "default";
-	status = "okay";
-};
-
-&rpm_requests {
-	regulators {
-		compatible = "qcom,rpm-mp5496-regulators";
-
-		ipq9574_s1: s1 {
-		/*
-		 * During kernel bootup, the SoC runs at 800MHz with 875mV set by the bootloaders.
-		 * During regulator registration, kernel not knowing the initial voltage,
-		 * considers it as zero and brings up the regulators with minimum supported voltage.
-		 * Update the regulator-min-microvolt with SVS voltage of 725mV so that
-		 * the regulators are brought up with 725mV which is sufficient for all the
-		 * corner parts to operate at 800MHz
-		 */
-			regulator-min-microvolt = <725000>;
-			regulator-max-microvolt = <1075000>;
-		};
-	};
-};
-
-&sleep_clk {
-	clock-frequency = <32000>;
-};
-
-&tlmm {
-	spi_0_pins: spi-0-state {
-		pins = "gpio11", "gpio12", "gpio13", "gpio14";
-		function = "blsp0_spi";
-		drive-strength = <8>;
-		bias-disable;
-	};
-};
-
-&xo_board_clk {
-	clock-frequency = <24000000>;
 };
diff --git a/arch/arm64/boot/dts/qcom/ipq9574-rdp453.dts b/arch/arm64/boot/dts/qcom/ipq9574-rdp453.dts
index f01de6628c3b..c30c9fbedf26 100644
--- a/arch/arm64/boot/dts/qcom/ipq9574-rdp453.dts
+++ b/arch/arm64/boot/dts/qcom/ipq9574-rdp453.dts
@@ -8,73 +8,10 @@
 
 /dts-v1/;
 
-#include "ipq9574.dtsi"
+#include "ipq9574-rdp-common.dtsi"
 
 / {
 	model = "Qualcomm Technologies, Inc. IPQ9574/AP-AL02-C8";
 	compatible = "qcom,ipq9574-ap-al02-c8", "qcom,ipq9574";
 
-	aliases {
-		serial0 = &blsp1_uart2;
-	};
-
-	chosen {
-		stdout-path = "serial0:115200n8";
-	};
-};
-
-&blsp1_spi0 {
-	pinctrl-0 = <&spi_0_pins>;
-	pinctrl-names = "default";
-	status = "okay";
-
-	flash@0 {
-		compatible = "micron,n25q128a11", "jedec,spi-nor";
-		reg = <0>;
-		#address-cells = <1>;
-		#size-cells = <1>;
-		spi-max-frequency = <50000000>;
-	};
-};
-
-&blsp1_uart2 {
-	pinctrl-0 = <&uart2_pins>;
-	pinctrl-names = "default";
-	status = "okay";
-};
-
-&rpm_requests {
-	regulators {
-		compatible = "qcom,rpm-mp5496-regulators";
-
-		ipq9574_s1: s1 {
-		/*
-		 * During kernel bootup, the SoC runs at 800MHz with 875mV set by the bootloaders.
-		 * During regulator registration, kernel not knowing the initial voltage,
-		 * considers it as zero and brings up the regulators with minimum supported voltage.
-		 * Update the regulator-min-microvolt with SVS voltage of 725mV so that
-		 * the regulators are brought up with 725mV which is sufficient for all the
-		 * corner parts to operate at 800MHz
-		 */
-			regulator-min-microvolt = <725000>;
-			regulator-max-microvolt = <1075000>;
-		};
-	};
-};
-
-&sleep_clk {
-	clock-frequency = <32000>;
-};
-
-&tlmm {
-	spi_0_pins: spi-0-state {
-		pins = "gpio11", "gpio12", "gpio13", "gpio14";
-		function = "blsp0_spi";
-		drive-strength = <8>;
-		bias-disable;
-	};
-};
-
-&xo_board_clk {
-	clock-frequency = <24000000>;
 };
diff --git a/arch/arm64/boot/dts/qcom/ipq9574-rdp454.dts b/arch/arm64/boot/dts/qcom/ipq9574-rdp454.dts
index 6efae3426cb8..0dc382f5d5ec 100644
--- a/arch/arm64/boot/dts/qcom/ipq9574-rdp454.dts
+++ b/arch/arm64/boot/dts/qcom/ipq9574-rdp454.dts
@@ -8,73 +8,9 @@
 
 /dts-v1/;
 
-#include "ipq9574.dtsi"
+#include "ipq9574-rdp-common.dtsi"
 
 / {
 	model = "Qualcomm Technologies, Inc. IPQ9574/AP-AL02-C9";
 	compatible = "qcom,ipq9574-ap-al02-c9", "qcom,ipq9574";
-
-	aliases {
-		serial0 = &blsp1_uart2;
-	};
-
-	chosen {
-		stdout-path = "serial0:115200n8";
-	};
-};
-
-&blsp1_spi0 {
-	pinctrl-0 = <&spi_0_pins>;
-	pinctrl-names = "default";
-	status = "okay";
-
-	flash@0 {
-		compatible = "micron,n25q128a11", "jedec,spi-nor";
-		reg = <0>;
-		#address-cells = <1>;
-		#size-cells = <1>;
-		spi-max-frequency = <50000000>;
-	};
-};
-
-&blsp1_uart2 {
-	pinctrl-0 = <&uart2_pins>;
-	pinctrl-names = "default";
-	status = "okay";
-};
-
-&rpm_requests {
-	regulators {
-		compatible = "qcom,rpm-mp5496-regulators";
-
-		ipq9574_s1: s1 {
-		/*
-		 * During kernel bootup, the SoC runs at 800MHz with 875mV set by the bootloaders.
-		 * During regulator registration, kernel not knowing the initial voltage,
-		 * considers it as zero and brings up the regulators with minimum supported voltage.
-		 * Update the regulator-min-microvolt with SVS voltage of 725mV so that
-		 * the regulators are brought up with 725mV which is sufficient for all the
-		 * corner parts to operate at 800MHz
-		 */
-			regulator-min-microvolt = <725000>;
-			regulator-max-microvolt = <1075000>;
-		};
-	};
-};
-
-&sleep_clk {
-	clock-frequency = <32000>;
-};
-
-&tlmm {
-	spi_0_pins: spi-0-state {
-		pins = "gpio11", "gpio12", "gpio13", "gpio14";
-		function = "blsp0_spi";
-		drive-strength = <8>;
-		bias-disable;
-	};
-};
-
-&xo_board_clk {
-	clock-frequency = <24000000>;
 };
diff --git a/arch/arm64/boot/dts/qcom/ipq9574.dtsi b/arch/arm64/boot/dts/qcom/ipq9574.dtsi
index 8a72ad4afd03..5f83ee42a719 100644
--- a/arch/arm64/boot/dts/qcom/ipq9574.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq9574.dtsi
@@ -106,42 +106,56 @@ memory@40000000 {
 	};
 
 	cpu_opp_table: opp-table-cpu {
-		compatible = "operating-points-v2";
+		compatible = "operating-points-v2-kryo-cpu";
 		opp-shared;
+		nvmem-cells = <&cpu_speed_bin>;
 
 		opp-936000000 {
 			opp-hz = /bits/ 64 <936000000>;
 			opp-microvolt = <725000>;
+			opp-supported-hw = <0xf>;
 			clock-latency-ns = <200000>;
 		};
 
 		opp-1104000000 {
 			opp-hz = /bits/ 64 <1104000000>;
 			opp-microvolt = <787500>;
+			opp-supported-hw = <0xf>;
+			clock-latency-ns = <200000>;
+		};
+
+		opp-1200000000 {
+			opp-hz = /bits/ 64 <1200000000>;
+			opp-microvolt = <862500>;
+			opp-supported-hw = <0xf>;
 			clock-latency-ns = <200000>;
 		};
 
 		opp-1416000000 {
 			opp-hz = /bits/ 64 <1416000000>;
 			opp-microvolt = <862500>;
+			opp-supported-hw = <0x7>;
 			clock-latency-ns = <200000>;
 		};
 
 		opp-1488000000 {
 			opp-hz = /bits/ 64 <1488000000>;
 			opp-microvolt = <925000>;
+			opp-supported-hw = <0x7>;
 			clock-latency-ns = <200000>;
 		};
 
 		opp-1800000000 {
 			opp-hz = /bits/ 64 <1800000000>;
 			opp-microvolt = <987500>;
+			opp-supported-hw = <0x5>;
 			clock-latency-ns = <200000>;
 		};
 
 		opp-2208000000 {
 			opp-hz = /bits/ 64 <2208000000>;
 			opp-microvolt = <1062500>;
+			opp-supported-hw = <0x1>;
 			clock-latency-ns = <200000>;
 		};
 	};
@@ -223,6 +237,11 @@ qfprom: efuse@a4000 {
 			reg = <0x000a4000 0x5a1>;
 			#address-cells = <1>;
 			#size-cells = <1>;
+
+			cpu_speed_bin: cpu-speed-bin@15 {
+				reg = <0x15 0x2>;
+				bits = <7 2>;
+			};
 		};
 
 		cryptobam: dma-controller@704000 {
@@ -652,8 +671,8 @@ apcs_glb: mailbox@b111000 {
 				     "qcom,ipq6018-apcs-apps-global";
 			reg = <0x0b111000 0x1000>;
 			#clock-cells = <1>;
-			clocks = <&a73pll>, <&xo_board_clk>;
-			clock-names = "pll", "xo";
+			clocks = <&a73pll>, <&xo_board_clk>, <&gcc GPLL0>;
+			clock-names = "pll", "xo", "gpll0";
 			#mbox-cells = <1>;
 		};
 
diff --git a/arch/arm64/boot/dts/qcom/msm8916-alcatel-idol347.dts b/arch/arm64/boot/dts/qcom/msm8916-alcatel-idol347.dts
index aa4c1ab1e673..3459145516a1 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-alcatel-idol347.dts
+++ b/arch/arm64/boot/dts/qcom/msm8916-alcatel-idol347.dts
@@ -3,6 +3,8 @@
 /dts-v1/;
 
 #include "msm8916-pm8916.dtsi"
+#include "msm8916-modem-qdsp6.dtsi"
+
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/leds/common.h>
@@ -22,6 +24,19 @@ chosen {
 		stdout-path = "serial0";
 	};
 
+	reserved-memory {
+		/delete-node/ reserved@86680000;
+		/delete-node/ rmtfs@86700000;
+
+		rmtfs: rmtfs@86680000 {
+			compatible = "qcom,rmtfs-mem";
+			reg = <0x0 0x86680000 0x0 0x160000>;
+			no-map;
+
+			qcom,client-id = <1>;
+		};
+	};
+
 	gpio-keys {
 		compatible = "gpio-keys";
 
@@ -50,6 +65,17 @@ led-0 {
 		};
 	};
 
+	reg_headphones_avdd: regulator-headphones-avdd {
+		compatible = "regulator-fixed";
+		regulator-name = "headphones_avdd";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		gpio = <&tlmm 121 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		pinctrl-0 = <&headphones_avdd_default>;
+		pinctrl-names = "default";
+	};
+
 	usb_id: usb-id {
 		compatible = "linux,extcon-usb-gpio";
 		id-gpios = <&tlmm 69 GPIO_ACTIVE_HIGH>;
@@ -58,8 +84,41 @@ usb_id: usb-id {
 	};
 };
 
-&blsp_uart2 {
+&blsp_i2c3 {
 	status = "okay";
+
+	headphones: audio-codec@10 {
+		compatible = "asahi-kasei,ak4375";
+		reg = <0x10>;
+		avdd-supply = <&reg_headphones_avdd>;
+		tvdd-supply = <&pm8916_l6>;
+		pdn-gpios = <&tlmm 114 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&headphones_pdn_default>;
+		pinctrl-names = "default";
+		#sound-dai-cells = <0>;
+	};
+
+	speaker_codec_top: audio-codec@34 {
+		compatible = "nxp,tfa9897";
+		reg = <0x34>;
+		vddd-supply = <&pm8916_l6>;
+		rcv-gpios = <&tlmm 50 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&speaker_top_default>;
+		pinctrl-names = "default";
+		sound-name-prefix = "Speaker Top";
+		#sound-dai-cells = <0>;
+	};
+
+	speaker_codec_bottom: audio-codec@36 {
+		compatible = "nxp,tfa9897";
+		reg = <0x36>;
+		vddd-supply = <&pm8916_l6>;
+		rcv-gpios = <&tlmm 111 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&speaker_bottom_default>;
+		pinctrl-names = "default";
+		sound-name-prefix = "Speaker Bottom";
+		#sound-dai-cells = <0>;
+	};
 };
 
 &blsp_i2c4 {
@@ -153,6 +212,22 @@ led@1 {
 	};
 };
 
+&blsp_uart2 {
+	status = "okay";
+};
+
+&mpss_mem {
+	reg = <0x0 0x86800000 0x0 0x5000000>;
+};
+
+&pm8916_codec {
+	qcom,micbias1-ext-cap;
+	qcom,micbias-lvl = <2800>;
+	qcom,mbhc-vthreshold-low = <75 100 120 180 500>;
+	qcom,mbhc-vthreshold-high = <75 100 120 180 500>;
+	qcom,hphl-jack-type-normally-open;
+};
+
 &pm8916_resin {
 	status = "okay";
 	linux,code = <KEY_VOLUMEDOWN>;
@@ -169,6 +244,17 @@ &pm8916_vib {
 	status = "okay";
 };
 
+&q6afedai {
+	dai@18 {
+		reg = <SECONDARY_MI2S_RX>;
+		qcom,sd-lines = <0>;
+	};
+	dai@22 {
+		reg = <QUATERNARY_MI2S_RX>;
+		qcom,sd-lines = <0>;
+	};
+};
+
 &sdhc_1 {
 	status = "okay";
 };
@@ -183,6 +269,54 @@ &sdhc_2 {
 	cd-gpios = <&tlmm 38 GPIO_ACTIVE_LOW>;
 };
 
+&sound {
+	/* Add pin switches for speakers to allow disabling them individually */
+	model = "alcatel-idol3";
+	widgets =
+		"Speaker", "Speaker Top",
+		"Speaker", "Speaker Bottom";
+	pin-switches = "Speaker Top", "Speaker Bottom";
+	audio-routing =
+		"Speaker Top", "Speaker Top OUT",
+		"Speaker Bottom", "Speaker Bottom OUT",
+		"AMIC1", "MIC BIAS External1",
+		"AMIC2", "MIC BIAS Internal2",
+		"AMIC3", "MIC BIAS External1";
+
+	pinctrl-0 = <&cdc_pdm_default &pri_mi2s_default &pri_mi2s_ws_default &sec_mi2s_default>;
+	pinctrl-1 = <&cdc_pdm_sleep &pri_mi2s_sleep &pri_mi2s_ws_sleep &sec_mi2s_sleep>;
+	pinctrl-names = "default", "sleep";
+
+	sound_link_backend2: backend2-dai-link {
+		link-name = "Quaternary MI2S";
+
+		cpu {
+			sound-dai = <&q6afedai QUATERNARY_MI2S_RX>;
+		};
+		platform {
+			sound-dai = <&q6routing>;
+		};
+		codec {
+			sound-dai = <&speaker_codec_top>, <&speaker_codec_bottom>;
+		};
+	};
+};
+
+&sound_link_backend0 {
+	/* Primary MI2S is not used, replace with Secondary MI2S for headphones */
+	link-name = "Secondary MI2S";
+
+	cpu {
+		sound-dai = <&q6afedai SECONDARY_MI2S_RX>;
+	};
+	platform {
+		sound-dai = <&q6routing>;
+	};
+	codec {
+		sound-dai = <&headphones>;
+	};
+};
+
 &usb {
 	status = "okay";
 	extcon = <&usb_id>, <&usb_id>;
@@ -212,6 +346,15 @@ &wcnss_mem {
 	status = "okay";
 };
 
+/* Only some of the pins are used */
+&pri_mi2s_default {
+	pins = "gpio113", "gpio115";
+};
+
+&pri_mi2s_sleep {
+	pins = "gpio113", "gpio115";
+};
+
 &tlmm {
 	accel_int_default: accel-int-default-state {
 		pins = "gpio31";
@@ -245,6 +388,20 @@ gyro_int_default: gyro-int-default-state {
 		bias-disable;
 	};
 
+	headphones_avdd_default: headphones-avdd-default-state {
+		pins = "gpio121";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	headphones_pdn_default: headphones-pdn-default-state {
+		pins = "gpio114";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
 	/*
 	 * The OEM wired an additional GPIO to be asserted so that
 	 * the si-en,sn3190 LED IC works. Since this GPIO is not
@@ -291,6 +448,20 @@ sdc2_cd_default: sdc2-cd-default-state {
 		bias-disable;
 	};
 
+	speaker_bottom_default: speaker-bottom-default-state {
+		pins = "gpio111";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	speaker_top_default: speaker-top-default-state {
+		pins = "gpio50";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
 	ts_int_reset_default: ts-int-reset-default-state {
 		pins = "gpio13", "gpio100";
 		function = "gpio";
diff --git a/arch/arm64/boot/dts/qcom/msm8916-asus-z00l.dts b/arch/arm64/boot/dts/qcom/msm8916-asus-z00l.dts
index a8be6ff66893..77618c7374df 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-asus-z00l.dts
+++ b/arch/arm64/boot/dts/qcom/msm8916-asus-z00l.dts
@@ -3,6 +3,8 @@
 /dts-v1/;
 
 #include "msm8916-pm8916.dtsi"
+#include "msm8916-modem-qdsp6.dtsi"
+
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/interrupt-controller/irq.h>
@@ -130,6 +132,18 @@ &blsp_uart2 {
 	status = "okay";
 };
 
+&mpss_mem {
+	reg = <0x0 0x86800000 0x0 0x5500000>;
+};
+
+&pm8916_codec {
+	qcom,micbias-lvl = <2800>;
+	qcom,mbhc-vthreshold-low = <75 150 237 450 500>;
+	qcom,mbhc-vthreshold-high = <75 150 237 450 500>;
+	qcom,micbias1-ext-cap;
+	qcom,hphl-jack-type-normally-open;
+};
+
 &pm8916_rpm_regulators {
 	pm8916_l17: l17 {
 		regulator-min-microvolt = <2850000>;
@@ -151,6 +165,13 @@ &sdhc_2 {
 	cd-gpios = <&tlmm 38 GPIO_ACTIVE_LOW>;
 };
 
+&sound {
+	audio-routing =
+		"AMIC1", "MIC BIAS External1",
+		"AMIC2", "MIC BIAS Internal2",
+		"AMIC3", "MIC BIAS External1";
+};
+
 &usb {
 	status = "okay";
 	extcon = <&usb_id>, <&usb_id>;
diff --git a/arch/arm64/boot/dts/qcom/msm8916-longcheer-l8150.dts b/arch/arm64/boot/dts/qcom/msm8916-longcheer-l8150.dts
index 47d1c5cb13f4..37fa55166918 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-longcheer-l8150.dts
+++ b/arch/arm64/boot/dts/qcom/msm8916-longcheer-l8150.dts
@@ -3,6 +3,8 @@
 /dts-v1/;
 
 #include "msm8916-pm8916.dtsi"
+#include "msm8916-modem-qdsp6.dtsi"
+
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/interrupt-controller/irq.h>
@@ -25,17 +27,26 @@ chosen {
 
 	/*
 	 * For some reason, the signed wcnss firmware is not relocatable.
-	 * It must be loaded at 0x8b600000. All other firmware is relocatable,
-	 * so place wcnss at the fixed address and then all other firmware
-	 * regions will be automatically allocated at a fitting place.
+	 * It must be loaded at 0x8b600000. Unfortunately, this also means that
+	 * mpss_mem does not fit when loaded to the typical address at 0x86800000.
+	 *
+	 * Load wcnss_mem to the fixed address and relocate mpss_mem to the next
+	 * working higher address. For some reason the modem firmware does not
+	 * boot when placed at 0x8a800000 to 0x8e800000.
 	 */
 	reserved-memory {
+		/delete-node/ mpss@86800000;
 		/delete-node/ wcnss;
 
 		wcnss_mem: wcnss@8b600000 {
 			reg = <0x0 0x8b600000 0x0 0x600000>;
 			no-map;
 		};
+
+		mpss_mem: mpss@8e800000 {
+			reg = <0x0 0x8e800000 0x0 0x5000000>;
+			no-map;
+		};
 	};
 
 	gpio-keys {
@@ -225,6 +236,13 @@ &blsp_uart2 {
 	status = "okay";
 };
 
+&pm8916_codec {
+	qcom,micbias-lvl = <2800>;
+	qcom,mbhc-vthreshold-low = <75 150 237 450 500>;
+	qcom,mbhc-vthreshold-high = <75 150 237 450 500>;
+	qcom,hphl-jack-type-normally-open;
+};
+
 &pm8916_resin {
 	status = "okay";
 	linux,code = <KEY_VOLUMEDOWN>;
@@ -254,6 +272,13 @@ &sdhc_2 {
 	non-removable;
 };
 
+&sound {
+	audio-routing =
+		"AMIC1", "MIC BIAS Internal1",
+		"AMIC2", "MIC BIAS Internal2",
+		"AMIC3", "MIC BIAS Internal3";
+};
+
 &usb {
 	status = "okay";
 	dr_mode = "peripheral";
diff --git a/arch/arm64/boot/dts/qcom/msm8916-longcheer-l8910.dts b/arch/arm64/boot/dts/qcom/msm8916-longcheer-l8910.dts
index 41cadb906b98..3b7fdb6797a9 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-longcheer-l8910.dts
+++ b/arch/arm64/boot/dts/qcom/msm8916-longcheer-l8910.dts
@@ -3,9 +3,12 @@
 /dts-v1/;
 
 #include "msm8916-pm8916.dtsi"
+#include "msm8916-modem-qdsp6.dtsi"
+
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
 
 / {
 	model = "BQ Aquaris X5 (Longcheer L8910)";
@@ -22,6 +25,16 @@ chosen {
 		stdout-path = "serial0";
 	};
 
+	speaker_amp: audio-amplifier {
+		compatible = "awinic,aw8738";
+		mode-gpios = <&tlmm 114 GPIO_ACTIVE_HIGH>;
+		awinic,mode = <5>;
+		sound-name-prefix = "Speaker Amp";
+
+		pinctrl-0 = <&spk_ext_pa_default>;
+		pinctrl-names = "default";
+	};
+
 	flash-led-controller {
 		compatible = "ocs,ocp8110";
 		enable-gpios = <&tlmm 49 GPIO_ACTIVE_HIGH>;
@@ -74,6 +87,46 @@ usb_id: usb-id {
 	};
 };
 
+&blsp_i2c2 {
+	status = "okay";
+
+	led-controller@30 {
+		compatible = "kinetic,ktd2026";
+		reg = <0x30>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		vin-supply = <&pm8916_l17>;
+		vio-supply = <&pm8916_l6>;
+
+		pinctrl-0 = <&status_led_default>;
+		pinctrl-names = "default";
+
+		multi-led {
+			color = <LED_COLOR_ID_RGB>;
+			function = LED_FUNCTION_STATUS;
+
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			led@0 {
+				reg = <0>;
+				color = <LED_COLOR_ID_RED>;
+			};
+
+			led@1 {
+				reg = <1>;
+				color = <LED_COLOR_ID_GREEN>;
+			};
+
+			led@2 {
+				reg = <2>;
+				color = <LED_COLOR_ID_BLUE>;
+			};
+		};
+	};
+};
+
 &blsp_i2c3 {
 	status = "okay";
 
@@ -107,6 +160,27 @@ &blsp_uart2 {
 	status = "okay";
 };
 
+&mpss_mem {
+	reg = <0x0 0x86800000 0x0 0x5000000>;
+};
+
+&pm8916_codec {
+	qcom,micbias-lvl = <2800>;
+	qcom,mbhc-vthreshold-low = <75 100 120 180 500>;
+	qcom,mbhc-vthreshold-high = <75 100 120 180 500>;
+	qcom,hphl-jack-type-normally-open;
+};
+
+&pm8916_gpios {
+	status_led_default: status-led-default-state {
+		pins = "gpio3";
+		function = PMIC_GPIO_FUNC_NORMAL;
+		power-source = <PM8916_GPIO_VPH>;
+		bias-disable;
+		output-high;
+	};
+};
+
 &pm8916_resin {
 	status = "okay";
 	linux,code = <KEY_VOLUMEDOWN>;
@@ -137,6 +211,28 @@ &sdhc_2 {
 	cd-gpios = <&tlmm 38 GPIO_ACTIVE_LOW>;
 };
 
+&sound {
+	/*
+	 * Provide widgets/pin-switches to allow enabling speaker separately.
+	 * The hardware does not provide a way to disable the output via the
+	 * headphone jack when the speaker is enabled.
+	 */
+	model = "bq-paella";
+	widgets =
+		"Speaker", "Speaker",
+		"Headphone", "Headphones";
+	pin-switches = "Speaker";
+	audio-routing =
+		"Speaker", "Speaker Amp OUT",
+		"Speaker Amp IN", "HPH_R",
+		"Headphones", "HPH_L",
+		"Headphones", "HPH_R",
+		"AMIC1", "MIC BIAS External1",
+		"AMIC2", "MIC BIAS Internal2",
+		"AMIC3", "MIC BIAS External1";
+	aux-devs = <&speaker_amp>;
+};
+
 &usb {
 	status = "okay";
 	extcon = <&usb_id>, <&usb_id>;
@@ -205,6 +301,13 @@ sdc2_cd_default: sdc2-cd-default-state {
 		bias-disable;
 	};
 
+	spk_ext_pa_default: spk-ext-pa-default-state {
+		pins = "gpio114";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
 	usb_id_default: usb-id-default-state {
 		pins = "gpio110";
 		function = "gpio";
diff --git a/arch/arm64/boot/dts/qcom/msm8916-modem-qdsp6.dtsi b/arch/arm64/boot/dts/qcom/msm8916-modem-qdsp6.dtsi
new file mode 100644
index 000000000000..039961622633
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/msm8916-modem-qdsp6.dtsi
@@ -0,0 +1,148 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+/*
+ * msm8916-modem-qdsp6.dtsi describes the typical modem setup on MSM8916 devices
+ * (or similar SoCs) with audio routed via the QDSP6 services provided by the
+ * modem firmware. The digital/analog codec in the SoC/PMIC is used by default,
+ * but boards can define additional codecs by adding additional backend DAI links.
+ */
+
+#include <dt-bindings/sound/qcom,q6afe.h>
+#include <dt-bindings/sound/qcom,q6asm.h>
+
+&apr {
+	status = "okay";
+};
+
+&bam_dmux {
+	status = "okay";
+};
+
+&bam_dmux_dma {
+	status = "okay";
+};
+
+&lpass {
+	status = "reserved"; /* Controlled by QDSP6 */
+};
+
+&lpass_codec {
+	status = "okay";
+};
+
+&mba_mem {
+	status = "okay";
+};
+
+&mpss {
+	status = "okay";
+};
+
+&mpss_mem {
+	status = "okay";
+};
+
+&pm8916_codec {
+	status = "okay";
+};
+
+&q6afedai {
+	dai@16 {
+		reg = <PRIMARY_MI2S_RX>;
+		qcom,sd-lines = <0 1>;
+	};
+	dai@20 {
+		reg = <TERTIARY_MI2S_TX>;
+		qcom,sd-lines = <0 1>;
+	};
+};
+
+&q6asmdai {
+	dai@0 {
+		reg = <0>;
+		direction = <Q6ASM_DAI_RX>;
+	};
+	dai@1 {
+		reg = <1>;
+		direction = <Q6ASM_DAI_TX>;
+	};
+	dai@2 {
+		reg = <2>;
+		direction = <Q6ASM_DAI_RX>;
+	};
+	dai@3 {
+		reg = <3>;
+		direction = <Q6ASM_DAI_RX>;
+		is-compress-dai;
+	};
+};
+
+&sound {
+	compatible = "qcom,msm8916-qdsp6-sndcard";
+	model = "msm8916";
+
+	pinctrl-0 = <&cdc_pdm_default>;
+	pinctrl-1 = <&cdc_pdm_sleep>;
+	pinctrl-names = "default", "sleep";
+
+	status = "okay";
+
+	frontend0-dai-link {
+		link-name = "MultiMedia1";
+
+		cpu {
+			sound-dai = <&q6asmdai MSM_FRONTEND_DAI_MULTIMEDIA1>;
+		};
+	};
+
+	frontend1-dai-link {
+		link-name = "MultiMedia2";
+
+		cpu {
+			sound-dai = <&q6asmdai MSM_FRONTEND_DAI_MULTIMEDIA2>;
+		};
+	};
+
+	frontend2-dai-link {
+		link-name = "MultiMedia3";
+
+		cpu {
+			sound-dai = <&q6asmdai MSM_FRONTEND_DAI_MULTIMEDIA3>;
+		};
+	};
+
+	frontend3-dai-link {
+		link-name = "MultiMedia4";
+
+		cpu {
+			sound-dai = <&q6asmdai MSM_FRONTEND_DAI_MULTIMEDIA4>;
+		};
+	};
+
+	sound_link_backend0: backend0-dai-link {
+		link-name = "Primary MI2S";
+
+		cpu {
+			sound-dai = <&q6afedai PRIMARY_MI2S_RX>;
+		};
+		platform {
+			sound-dai = <&q6routing>;
+		};
+		codec {
+			sound-dai = <&lpass_codec 0>, <&pm8916_codec 0>;
+		};
+	};
+
+	sound_link_backend1: backend1-dai-link {
+		link-name = "Tertiary MI2S";
+
+		cpu {
+			sound-dai = <&q6afedai TERTIARY_MI2S_TX>;
+		};
+		platform {
+			sound-dai = <&q6routing>;
+		};
+		codec {
+			sound-dai = <&lpass_codec 1>, <&pm8916_codec 1>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-a2015-common.dtsi b/arch/arm64/boot/dts/qcom/msm8916-samsung-a2015-common.dtsi
index 0b29132b74e1..2937495940ea 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-samsung-a2015-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-a2015-common.dtsi
@@ -1,10 +1,13 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
 #include "msm8916-pm8916.dtsi"
+#include "msm8916-modem-qdsp6.dtsi"
+
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+#include <dt-bindings/sound/apq8016-lpass.h>
 
 / {
 	aliases {
@@ -196,6 +199,18 @@ vibrator: vibrator {
 	};
 };
 
+&blsp_i2c1 {
+	status = "okay";
+
+	speaker_codec: audio-codec@34 {
+		compatible = "nxp,tfa9895";
+		reg = <0x34>;
+		vddd-supply = <&pm8916_l5>;
+		sound-name-prefix = "Speaker";
+		#sound-dai-cells = <0>;
+	};
+};
+
 &blsp_i2c2 {
 	status = "okay";
 
@@ -243,6 +258,25 @@ &gpu {
 	status = "okay";
 };
 
+/*
+ * For some reason the speaker amplifier is connected to the second SD line
+ * (MI2S_2_D1) instead of the first (MI2S_2_D0). This must be configured in the
+ * device tree, otherwise audio will seemingly play fine on the wrong SD line
+ * but the speaker stays silent.
+ *
+ * When routing audio via QDSP6 (the default) the &lpass node is reserved and
+ * the definitions from &q6afedai are used. When the modem is disabled audio can
+ * be alternatively routed directly to the LPASS hardware with reduced latency.
+ * The definitions for &lpass are here for completeness to simplify changing the
+ * setup with minor changes to the DT (either manually or with DT overlays).
+ */
+&lpass {
+	dai-link@3 {
+		reg = <MI2S_QUATERNARY>;
+		qcom,playback-sd-lines = <1>;
+	};
+};
+
 &mdss {
 	status = "okay";
 };
@@ -253,6 +287,10 @@ &mdss_dsi0 {
 	pinctrl-1 = <&mdss_sleep>;
 };
 
+&mpss_mem {
+	reg = <0x0 0x86800000 0x0 0x5400000>;
+};
+
 &pm8916_resin {
 	status = "okay";
 	linux,code = <KEY_VOLUMEDOWN>;
@@ -265,6 +303,13 @@ pm8916_l17: l17 {
 	};
 };
 
+&q6afedai {
+	dai@22 {
+		reg = <QUATERNARY_MI2S_RX>;
+		qcom,sd-lines = <1>;
+	};
+};
+
 &sdhc_1 {
 	status = "okay";
 };
@@ -279,6 +324,32 @@ &sdhc_2 {
 	cd-gpios = <&tlmm 38 GPIO_ACTIVE_LOW>;
 };
 
+&sound {
+	model = "samsung-a2015";
+	audio-routing =
+		"AMIC1", "MIC BIAS External1",
+		"AMIC2", "MIC BIAS Internal2",
+		"AMIC3", "MIC BIAS External1";
+
+	pinctrl-0 = <&cdc_pdm_default &sec_mi2s_default>;
+	pinctrl-1 = <&cdc_pdm_sleep &sec_mi2s_sleep>;
+	pinctrl-names = "default", "sleep";
+
+	sound_link_backend2: backend2-dai-link {
+		link-name = "Quaternary MI2S";
+
+		cpu {
+			sound-dai = <&q6afedai QUATERNARY_MI2S_RX>;
+		};
+		platform {
+			sound-dai = <&q6routing>;
+		};
+		codec {
+			sound-dai = <&speaker_codec>;
+		};
+	};
+};
+
 &usb {
 	status = "okay";
 	extcon = <&muic>, <&muic>;
diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-a3u-eur.dts b/arch/arm64/boot/dts/qcom/msm8916-samsung-a3u-eur.dts
index f5a808369518..3b934f5eba47 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-samsung-a3u-eur.dts
+++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-a3u-eur.dts
@@ -49,11 +49,6 @@ reg_key_led: regulator-key-led {
 	};
 };
 
-&touchkey {
-	vcc-supply = <&reg_touch_key>;
-	vdd-supply = <&reg_key_led>;
-};
-
 &accelerometer {
 	mount-matrix = "0", "1", "0",
 		       "1", "0", "0",
@@ -108,6 +103,11 @@ &mdss_dsi0_out {
 	remote-endpoint = <&panel_in>;
 };
 
+&touchkey {
+	vcc-supply = <&reg_touch_key>;
+	vdd-supply = <&reg_key_led>;
+};
+
 &vibrator {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-e2015-common.dtsi b/arch/arm64/boot/dts/qcom/msm8916-samsung-e2015-common.dtsi
index 0824ab041d80..3c49dac92d2d 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-samsung-e2015-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-e2015-common.dtsi
@@ -65,6 +65,10 @@ accelerometer@1d {
 	};
 };
 
+&mpss_mem {
+	reg = <0x0 0x86800000 0x0 0x5a00000>;
+};
+
 &reg_motor_vdd {
 	regulator-min-microvolt = <3300000>;
 	regulator-max-microvolt = <3300000>;
diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-grandmax.dts b/arch/arm64/boot/dts/qcom/msm8916-samsung-grandmax.dts
index 3f145dde4059..5882b3a593b8 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-samsung-grandmax.dts
+++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-grandmax.dts
@@ -49,6 +49,10 @@ &reg_touch_key {
 	status = "disabled";
 };
 
+&sound {
+	model = "samsung-gmax"; /* No secondary microphone */
+};
+
 &tlmm {
 	gpio_leds_default: gpio-led-default-state {
 		pins = "gpio60";
diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-gt5-common.dtsi b/arch/arm64/boot/dts/qcom/msm8916-samsung-gt5-common.dtsi
index c19cf20d7427..fbd2caf405d5 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-samsung-gt5-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-gt5-common.dtsi
@@ -3,9 +3,12 @@
 /dts-v1/;
 
 #include "msm8916-pm8916.dtsi"
+#include "msm8916-modem-qdsp6.dtsi"
+
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/sound/apq8016-lpass.h>
 
 / {
 	aliases {
@@ -65,25 +68,6 @@ hall-sensor-switch {
 	};
 };
 
-&blsp_i2c4 {
-	status = "okay";
-
-	fuelgauge@36 {
-		compatible = "maxim,max77849-battery";
-		reg = <0x36>;
-
-		maxim,rsns-microohm = <10000>;
-		maxim,over-heat-temp = <600>;
-		maxim,over-volt = <4400>;
-
-		interrupt-parent = <&tlmm>;
-		interrupts = <121 IRQ_TYPE_EDGE_FALLING>;
-
-		pinctrl-0 = <&fuelgauge_int_default>;
-		pinctrl-names = "default";
-	};
-};
-
 &blsp_i2c2 {
 	status = "okay";
 
@@ -112,10 +96,52 @@ accelerometer@1d {
 	};
 };
 
+&blsp_i2c4 {
+	status = "okay";
+
+	fuelgauge@36 {
+		compatible = "maxim,max77849-battery";
+		reg = <0x36>;
+
+		maxim,rsns-microohm = <10000>;
+		maxim,over-heat-temp = <600>;
+		maxim,over-volt = <4400>;
+
+		interrupt-parent = <&tlmm>;
+		interrupts = <121 IRQ_TYPE_EDGE_FALLING>;
+
+		pinctrl-0 = <&fuelgauge_int_default>;
+		pinctrl-names = "default";
+	};
+};
+
 &blsp_uart2 {
 	status = "okay";
 };
 
+/*
+ * For some reason the speaker amplifier is connected to the second SD line
+ * (MI2S_2_D1) instead of the first (MI2S_2_D0). This must be configured in the
+ * device tree, otherwise audio will seemingly play fine on the wrong SD line
+ * but the speaker stays silent.
+ *
+ * When routing audio via QDSP6 (the default) the &lpass node is reserved and
+ * the definitions from &q6afedai are used. When the modem is disabled audio can
+ * be alternatively routed directly to the LPASS hardware with reduced latency.
+ * The definitions for &lpass are here for completeness to simplify changing the
+ * setup with minor changes to the DT (either manually or with DT overlays).
+ */
+&lpass {
+	dai-link@3 {
+		reg = <MI2S_QUATERNARY>;
+		qcom,playback-sd-lines = <1>;
+	};
+};
+
+&mpss_mem {
+	reg = <0x0 0x86800000 0x0 0x5400000>;
+};
+
 &pm8916_resin {
 	linux,code = <KEY_VOLUMEDOWN>;
 	status = "okay";
@@ -133,6 +159,13 @@ &pm8916_usbin {
 	status = "okay";
 };
 
+&q6afedai {
+	dai@22 {
+		reg = <QUATERNARY_MI2S_RX>;
+		qcom,sd-lines = <1>;
+	};
+};
+
 &sdhc_1 {
 	status = "okay";
 };
@@ -147,6 +180,27 @@ &sdhc_2 {
 	status = "okay";
 };
 
+&sound {
+	audio-routing =
+		"AMIC1", "MIC BIAS External1",
+		"AMIC2", "MIC BIAS Internal2",
+		"AMIC3", "MIC BIAS External1";
+
+	sound_link_backend2: backend2-dai-link {
+		link-name = "Quaternary MI2S";
+
+		cpu {
+			sound-dai = <&q6afedai QUATERNARY_MI2S_RX>;
+		};
+		platform {
+			sound-dai = <&q6routing>;
+		};
+		codec {
+			sound-dai = <&speaker_codec>;
+		};
+	};
+};
+
 &usb {
 	dr_mode = "peripheral";
 	extcon = <&pm8916_usbin>;
diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-gt510.dts b/arch/arm64/boot/dts/qcom/msm8916-samsung-gt510.dts
index 75c4854ecd64..5b34529b816c 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-samsung-gt510.dts
+++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-gt510.dts
@@ -9,6 +9,14 @@ / {
 	compatible = "samsung,gt510", "qcom,msm8916";
 	chassis-type = "tablet";
 
+	speaker_codec: audio-codec {
+		compatible = "maxim,max98357a";
+		sdmode-gpios = <&tlmm 55 GPIO_ACTIVE_HIGH>;
+		#sound-dai-cells = <0>;
+		pinctrl-0 = <&audio_sdmode_default>;
+		pinctrl-names = "default";
+	};
+
 	clk_pwm: pwm {
 		compatible = "clk-pwm";
 		#pwm-cells = <2>;
@@ -112,6 +120,10 @@ touchscreen@4a {
 	};
 };
 
+&gpu {
+	status = "okay";
+};
+
 &mdss {
 	status = "okay";
 };
@@ -142,7 +154,21 @@ &mdss_dsi0_out {
 	remote-endpoint = <&panel_in>;
 };
 
+&sound {
+	model = "samsung-gt510";
+	pinctrl-0 = <&cdc_pdm_default &sec_mi2s_default>;
+	pinctrl-1 = <&cdc_pdm_sleep &sec_mi2s_sleep>;
+	pinctrl-names = "default", "sleep";
+};
+
 &tlmm {
+	audio_sdmode_default: audio-sdmode-default-state {
+		pins = "gpio55";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
 	buckbooster_en_default: buckbooster-en-default-state {
 		pins = "gpio51";
 		function = "gpio";
diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-gt58.dts b/arch/arm64/boot/dts/qcom/msm8916-samsung-gt58.dts
index 11359bcc27b3..579312ed53ce 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-samsung-gt58.dts
+++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-gt58.dts
@@ -35,6 +35,26 @@ reg_vdd_tsp: regulator-vdd-tsp {
 		pinctrl-names = "default";
 	};
 
+	i2c-amplifier {
+		compatible = "i2c-gpio";
+		sda-gpios = <&tlmm 55 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+		scl-gpios = <&tlmm 56 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+
+		pinctrl-0 = <&amp_i2c_default>;
+		pinctrl-names = "default";
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		speaker_codec: audio-codec@34 {
+			compatible = "nxp,tfa9895";
+			reg = <0x34>;
+			vddd-supply = <&pm8916_l5>;
+			sound-name-prefix = "Speaker";
+			#sound-dai-cells = <0>;
+		};
+	};
+
 	vibrator {
 		compatible = "gpio-vibrator";
 		enable-gpios = <&tlmm 76 GPIO_ACTIVE_HIGH>;
@@ -64,6 +84,10 @@ touchscreen@20 {
 	};
 };
 
+&gpu {
+	status = "okay";
+};
+
 &mdss {
 	status = "okay";
 };
@@ -94,7 +118,21 @@ &mdss_dsi0_out {
 	remote-endpoint = <&panel_in>;
 };
 
+&sound {
+	model = "samsung-a2015";
+	pinctrl-0 = <&cdc_pdm_default &sec_mi2s_default &secondary_mic_default>;
+	pinctrl-1 = <&cdc_pdm_sleep &sec_mi2s_sleep &secondary_mic_default>;
+	pinctrl-names = "default", "sleep";
+};
+
 &tlmm {
+	amp_i2c_default: amp-i2c-default-state {
+		pins = "gpio55", "gpio56";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
 	buckbooster_en_default: buckbooster-en-default-state {
 		pins = "gpio8";
 		function = "gpio";
@@ -123,6 +161,14 @@ reg_tsp_en_default: reg-tsp-en-default-state {
 		bias-disable;
 	};
 
+	secondary_mic_default: secondary-mic-default-state {
+		pins = "gpio98";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+		output-high;
+	};
+
 	tsp_int_default: tsp-int-default-state {
 		pins = "gpio13";
 		function = "gpio";
diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-j5-common.dtsi b/arch/arm64/boot/dts/qcom/msm8916-samsung-j5-common.dtsi
index fe59be3505fe..5ca2ada266f4 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-samsung-j5-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-j5-common.dtsi
@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
 #include "msm8916-pm8916.dtsi"
+#include "msm8916-modem-qdsp6.dtsi"
+
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/interrupt-controller/irq.h>
@@ -135,6 +137,10 @@ &blsp_uart2 {
 	status = "okay";
 };
 
+&mpss_mem {
+	reg = <0x0 0x86800000 0x0 0x5800000>;
+};
+
 &pm8916_resin {
 	status = "okay";
 	linux,code = <KEY_VOLUMEDOWN>;
@@ -154,6 +160,14 @@ &sdhc_2 {
 	cd-gpios = <&tlmm 38 GPIO_ACTIVE_LOW>;
 };
 
+&sound {
+	model = "msm8916-1mic";
+	audio-routing =
+		"AMIC1", "MIC BIAS External1",
+		"AMIC2", "MIC BIAS Internal2",
+		"AMIC3", "MIC BIAS External1";
+};
+
 &usb {
 	extcon = <&muic>, <&muic>;
 	status = "okay";
diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-j5.dts b/arch/arm64/boot/dts/qcom/msm8916-samsung-j5.dts
index 58c2f5a70e78..ba8650971d6a 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-samsung-j5.dts
+++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-j5.dts
@@ -19,6 +19,10 @@ &blsp_i2c5 {
 	status = "disabled";
 };
 
+&pm8916_codec {
+	qcom,micbias1-ext-cap;
+};
+
 &touchscreen {
 	/* FIXME: Missing sm5703-mfd driver to power up vdd-supply */
 };
diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-serranove.dts b/arch/arm64/boot/dts/qcom/msm8916-samsung-serranove.dts
index 68da2a2d3077..5ce8f1350abc 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-samsung-serranove.dts
+++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-serranove.dts
@@ -6,6 +6,8 @@
 /dts-v1/;
 
 #include "msm8916-pm8916.dtsi"
+#include "msm8916-modem-qdsp6.dtsi"
+
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/interrupt-controller/irq.h>
@@ -319,6 +321,10 @@ &blsp_uart2 {
 	status = "okay";
 };
 
+&mpss_mem {
+	reg = <0x0 0x86800000 0x0 0x5a00000>;
+};
+
 &pm8916_resin {
 	status = "okay";
 	linux,code = <KEY_VOLUMEDOWN>;
@@ -350,6 +356,13 @@ &sdhc_2 {
 	no-1-8-v;
 };
 
+&sound {
+	audio-routing =
+		"AMIC1", "MIC BIAS External1",
+		"AMIC2", "MIC BIAS Internal2",
+		"AMIC3", "MIC BIAS External1";
+};
+
 &usb {
 	status = "okay";
 	extcon = <&muic>, <&muic>;
diff --git a/arch/arm64/boot/dts/qcom/msm8916-thwc-uf896.dts b/arch/arm64/boot/dts/qcom/msm8916-thwc-uf896.dts
index 6fe1850ba20e..f34997500891 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-thwc-uf896.dts
+++ b/arch/arm64/boot/dts/qcom/msm8916-thwc-uf896.dts
@@ -13,16 +13,16 @@ &button_restart {
 	gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
 };
 
-&led_r {
-	gpios = <&tlmm 82 GPIO_ACTIVE_HIGH>;
+&led_b {
+	gpios = <&tlmm 81 GPIO_ACTIVE_HIGH>;
 };
 
 &led_g {
 	gpios = <&tlmm 83 GPIO_ACTIVE_HIGH>;
 };
 
-&led_b {
-	gpios = <&tlmm 81 GPIO_ACTIVE_HIGH>;
+&led_r {
+	gpios = <&tlmm 82 GPIO_ACTIVE_HIGH>;
 };
 
 &button_default {
diff --git a/arch/arm64/boot/dts/qcom/msm8916-thwc-ufi001c.dts b/arch/arm64/boot/dts/qcom/msm8916-thwc-ufi001c.dts
index 16d4a91022be..6cb3911ba1c9 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-thwc-ufi001c.dts
+++ b/arch/arm64/boot/dts/qcom/msm8916-thwc-ufi001c.dts
@@ -13,16 +13,16 @@ &button_restart {
 	gpios = <&tlmm 37 GPIO_ACTIVE_HIGH>;
 };
 
-&led_r {
-	gpios = <&tlmm 22 GPIO_ACTIVE_HIGH>;
+&led_b {
+	gpios = <&tlmm 20 GPIO_ACTIVE_HIGH>;
 };
 
 &led_g {
 	gpios = <&tlmm 21 GPIO_ACTIVE_HIGH>;
 };
 
-&led_b {
-	gpios = <&tlmm 20 GPIO_ACTIVE_HIGH>;
+&led_r {
+	gpios = <&tlmm 22 GPIO_ACTIVE_HIGH>;
 };
 
 &mpss {
diff --git a/arch/arm64/boot/dts/qcom/msm8916-wingtech-wt88047.dts b/arch/arm64/boot/dts/qcom/msm8916-wingtech-wt88047.dts
index 419f35c1fc92..d4b88c787e59 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-wingtech-wt88047.dts
+++ b/arch/arm64/boot/dts/qcom/msm8916-wingtech-wt88047.dts
@@ -6,6 +6,8 @@
 /dts-v1/;
 
 #include "msm8916-pm8916.dtsi"
+#include "msm8916-modem-qdsp6.dtsi"
+
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/leds/common.h>
@@ -25,6 +27,28 @@ chosen {
 		stdout-path = "serial0";
 	};
 
+	speaker_amp: audio-amplifier {
+		compatible = "simple-audio-amplifier";
+		enable-gpios = <&tlmm 117 GPIO_ACTIVE_HIGH>;
+		sound-name-prefix = "Speaker Amp";
+		pinctrl-0 = <&speaker_amp_default>;
+		pinctrl-names = "default";
+	};
+
+	/*
+	 * This seems to be actually an analog switch that either routes audio
+	 * to the headphone jack or nowhere. Given that we need to enable a GPIO
+	 * to get sound on headphones, modelling it as simple-audio-amplifier
+	 * works just fine.
+	 */
+	headphones_switch: audio-switch {
+		compatible = "simple-audio-amplifier";
+		enable-gpios = <&tlmm 8 GPIO_ACTIVE_HIGH>;
+		sound-name-prefix = "Headphones Switch";
+		pinctrl-0 = <&headphones_switch_default>;
+		pinctrl-names = "default";
+	};
+
 	flash-led-controller {
 		compatible = "ocs,ocp8110";
 		enable-gpios = <&tlmm 31 GPIO_ACTIVE_HIGH>;
@@ -146,6 +170,18 @@ &blsp_uart2 {
 	status = "okay";
 };
 
+&mpss_mem {
+	reg = <0x0 0x86800000 0x0 0x5100000>;
+};
+
+&pm8916_codec {
+	qcom,micbias1-ext-cap;
+	qcom,micbias-lvl = <2800>;
+	qcom,mbhc-vthreshold-low = <75 100 120 180 500>;
+	qcom,mbhc-vthreshold-high = <75 100 120 180 500>;
+	qcom,hphl-jack-type-normally-open;
+};
+
 &pm8916_resin {
 	status = "okay";
 	linux,code = <KEY_VOLUMEDOWN>;
@@ -180,6 +216,30 @@ &sdhc_2 {
 	non-removable;
 };
 
+&sound {
+	/*
+	 * Provide widgets/pin-switches to allow enabling speaker and headphones
+	 * separately. Both are routed via the HPH_L/HPH_R pins of the codec.
+	 */
+	model = "wt88047";
+	widgets =
+		"Speaker", "Speaker",
+		"Headphone", "Headphones";
+	pin-switches = "Speaker", "Headphones";
+	audio-routing =
+		"Speaker", "Speaker Amp OUTL",
+		"Speaker", "Speaker Amp OUTR",
+		"Speaker Amp INL", "HPH_R",
+		"Speaker Amp INR", "HPH_R",
+		"Headphones", "Headphones Switch OUTL",
+		"Headphones", "Headphones Switch OUTR",
+		"Headphones Switch INL", "HPH_L",
+		"Headphones Switch INR", "HPH_R",
+		"AMIC1", "MIC BIAS External1",
+		"AMIC2", "MIC BIAS Internal2";
+	aux-devs = <&speaker_amp>, <&headphones_switch>;
+};
+
 &usb {
 	status = "okay";
 	extcon = <&usb_id>, <&usb_id>;
@@ -226,6 +286,13 @@ gpio_keys_default: gpio-keys-default-state {
 		bias-pull-up;
 	};
 
+	headphones_switch_default: headphones-switch-default-state {
+		pins = "gpio8";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
 	imu_default: imu-default-state {
 		pins = "gpio115";
 		function = "gpio";
@@ -234,6 +301,13 @@ imu_default: imu-default-state {
 		bias-disable;
 	};
 
+	speaker_amp_default: speaker-amp-default-state {
+		pins = "gpio117";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
 	touchscreen_default: touchscreen-default-state {
 		touchscreen-pins {
 			pins = "gpio13";
diff --git a/arch/arm64/boot/dts/qcom/msm8916-yiming-uz801v3.dts b/arch/arm64/boot/dts/qcom/msm8916-yiming-uz801v3.dts
index 5e6ba8c58bb5..a98efcfe78b7 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-yiming-uz801v3.dts
+++ b/arch/arm64/boot/dts/qcom/msm8916-yiming-uz801v3.dts
@@ -13,16 +13,16 @@ &button_restart {
 	gpios = <&tlmm 23 GPIO_ACTIVE_LOW>;
 };
 
-&led_r {
-	gpios = <&tlmm 7 GPIO_ACTIVE_HIGH>;
+&led_b {
+	gpios = <&tlmm 6 GPIO_ACTIVE_HIGH>;
 };
 
 &led_g {
 	gpios = <&tlmm 8 GPIO_ACTIVE_HIGH>;
 };
 
-&led_b {
-	gpios = <&tlmm 6 GPIO_ACTIVE_HIGH>;
+&led_r {
+	gpios = <&tlmm 7 GPIO_ACTIVE_HIGH>;
 };
 
 &button_default {
diff --git a/arch/arm64/boot/dts/qcom/msm8916.dtsi b/arch/arm64/boot/dts/qcom/msm8916.dtsi
index 4f799b536a92..e8a14dd7e7c2 100644
--- a/arch/arm64/boot/dts/qcom/msm8916.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8916.dtsi
@@ -10,6 +10,7 @@
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/reset/qcom,gcc-msm8916.h>
+#include <dt-bindings/soc/qcom,apr.h>
 #include <dt-bindings/thermal/thermal.h>
 
 / {
@@ -1989,6 +1990,54 @@ smd-edge {
 
 				label = "hexagon";
 
+				apr: apr {
+					compatible = "qcom,apr-v2";
+					qcom,smd-channels = "apr_audio_svc";
+					qcom,domain = <APR_DOMAIN_ADSP>;
+					#address-cells = <1>;
+					#size-cells = <0>;
+					status = "disabled";
+
+					q6core: service@3 {
+						compatible = "qcom,q6core";
+						reg = <APR_SVC_ADSP_CORE>;
+					};
+
+					q6afe: service@4 {
+						compatible = "qcom,q6afe";
+						reg = <APR_SVC_AFE>;
+
+						q6afedai: dais {
+							compatible = "qcom,q6afe-dais";
+							#address-cells = <1>;
+							#size-cells = <0>;
+							#sound-dai-cells = <1>;
+						};
+					};
+
+					q6asm: service@7 {
+						compatible = "qcom,q6asm";
+						reg = <APR_SVC_ASM>;
+
+						q6asmdai: dais {
+							compatible = "qcom,q6asm-dais";
+							#address-cells = <1>;
+							#size-cells = <0>;
+							#sound-dai-cells = <1>;
+						};
+					};
+
+					q6adm: service@8 {
+						compatible = "qcom,q6adm";
+						reg = <APR_SVC_ADM>;
+
+						q6routing: routing {
+							compatible = "qcom,q6adm-routing";
+							#sound-dai-cells = <0>;
+						};
+					};
+				};
+
 				fastrpc {
 					compatible = "qcom,fastrpc";
 					qcom,smd-channels = "fastrpcsmd-apps-dsp";
diff --git a/arch/arm64/boot/dts/qcom/msm8939-longcheer-l9100.dts b/arch/arm64/boot/dts/qcom/msm8939-longcheer-l9100.dts
index 6802714fda3f..ff532fdc1826 100644
--- a/arch/arm64/boot/dts/qcom/msm8939-longcheer-l9100.dts
+++ b/arch/arm64/boot/dts/qcom/msm8939-longcheer-l9100.dts
@@ -8,6 +8,7 @@
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
 #include <dt-bindings/pinctrl/qcom,pmic-mpp.h>
 
 / {
@@ -120,6 +121,46 @@ usb_id: usb-id {
 
 };
 
+&blsp_i2c2 {
+	status = "okay";
+
+	led-controller@30 {
+		compatible = "kinetic,ktd2026";
+		reg = <0x30>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		vin-supply = <&pm8916_l17>;
+		vio-supply = <&pm8916_l6>;
+
+		pinctrl-0 = <&status_led_default>;
+		pinctrl-names = "default";
+
+		multi-led {
+			color = <LED_COLOR_ID_RGB>;
+			function = LED_FUNCTION_STATUS;
+
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			led@0 {
+				reg = <0>;
+				color = <LED_COLOR_ID_RED>;
+			};
+
+			led@1 {
+				reg = <1>;
+				color = <LED_COLOR_ID_GREEN>;
+			};
+
+			led@2 {
+				reg = <2>;
+				color = <LED_COLOR_ID_BLUE>;
+			};
+		};
+	};
+};
+
 &blsp_i2c3 {
 	status = "okay";
 
@@ -184,6 +225,16 @@ &blsp_uart2 {
 	status = "okay";
 };
 
+&pm8916_gpios {
+	status_led_default: status-led-default-state {
+		pins = "gpio3";
+		function = PMIC_GPIO_FUNC_NORMAL;
+		power-source = <PM8916_GPIO_VPH>;
+		bias-disable;
+		output-high;
+	};
+};
+
 &pm8916_mpps {
 	pwm_out: mpp4-state {
 		pins = "mpp4";
@@ -247,6 +298,10 @@ &wcnss_iris {
 	compatible = "qcom,wcn3620";
 };
 
+&wcnss_mem {
+	status = "okay";
+};
+
 &tlmm {
 	button_backlight_default: button-backlight-default-state {
 		pins = "gpio17";
diff --git a/arch/arm64/boot/dts/qcom/msm8939-samsung-a7.dts b/arch/arm64/boot/dts/qcom/msm8939-samsung-a7.dts
index fccd8fec8b8f..aa6c39482a2f 100644
--- a/arch/arm64/boot/dts/qcom/msm8939-samsung-a7.dts
+++ b/arch/arm64/boot/dts/qcom/msm8939-samsung-a7.dts
@@ -3,10 +3,12 @@
 /dts-v1/;
 
 #include "msm8939-pm8916.dtsi"
+#include "msm8916-modem-qdsp6.dtsi"
 
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/sound/apq8016-lpass.h>
 
 / {
 	model = "Samsung Galaxy A7 (2015)";
@@ -287,6 +289,18 @@ muic: extcon@25 {
 	};
 };
 
+&blsp_i2c2 {
+	status = "okay";
+
+	speaker_codec: audio-codec@34 {
+		compatible = "nxp,tfa9895";
+		reg = <0x34>;
+		vddd-supply = <&pm8916_l5>;
+		sound-name-prefix = "Speaker";
+		#sound-dai-cells = <0>;
+	};
+};
+
 &blsp_i2c5 {
 	status = "okay";
 
@@ -309,6 +323,29 @@ &blsp_uart2 {
 	status = "okay";
 };
 
+/*
+ * For some reason the speaker amplifier is connected to the second SD line
+ * (MI2S_2_D1) instead of the first (MI2S_2_D0). This must be configured in the
+ * device tree, otherwise audio will seemingly play fine on the wrong SD line
+ * but the speaker stays silent.
+ *
+ * When routing audio via QDSP6 (the default) the &lpass node is reserved and
+ * the definitions from &q6afedai are used. When the modem is disabled audio can
+ * be alternatively routed directly to the LPASS hardware with reduced latency.
+ * The definitions for &lpass are here for completeness to simplify changing the
+ * setup with minor changes to the DT (either manually or with DT overlays).
+ */
+&lpass {
+	dai-link@3 {
+		reg = <MI2S_QUATERNARY>;
+		qcom,playback-sd-lines = <1>;
+	};
+};
+
+&mpss_mem {
+	reg = <0x0 0x86800000 0x0 0x5800000>;
+};
+
 &pm8916_resin {
 	linux,code = <KEY_VOLUMEDOWN>;
 	status = "okay";
@@ -321,6 +358,13 @@ pm8916_l17: l17 {
 	};
 };
 
+&q6afedai {
+	dai@22 {
+		reg = <QUATERNARY_MI2S_RX>;
+		qcom,sd-lines = <1>;
+	};
+};
+
 &sdhc_1 {
 	status = "okay";
 };
@@ -335,6 +379,32 @@ &sdhc_2 {
 	status = "okay";
 };
 
+&sound {
+	model = "samsung-a2015";
+	audio-routing =
+		"AMIC1", "MIC BIAS External1",
+		"AMIC2", "MIC BIAS Internal2",
+		"AMIC3", "MIC BIAS External1";
+
+	pinctrl-0 = <&cdc_pdm_default &sec_mi2s_default>;
+	pinctrl-1 = <&cdc_pdm_sleep &sec_mi2s_sleep>;
+	pinctrl-names = "default", "sleep";
+
+	sound_link_backend2: backend2-dai-link {
+		link-name = "Quaternary MI2S";
+
+		cpu {
+			sound-dai = <&q6afedai QUATERNARY_MI2S_RX>;
+		};
+		platform {
+			sound-dai = <&q6routing>;
+		};
+		codec {
+			sound-dai = <&speaker_codec>;
+		};
+	};
+};
+
 &usb {
 	extcon = <&muic>, <&muic>;
 	status = "okay";
diff --git a/arch/arm64/boot/dts/qcom/msm8939.dtsi b/arch/arm64/boot/dts/qcom/msm8939.dtsi
index 324b5d26db40..95610a32750a 100644
--- a/arch/arm64/boot/dts/qcom/msm8939.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8939.dtsi
@@ -10,6 +10,7 @@
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/reset/qcom,gcc-msm8939.h>
+#include <dt-bindings/soc/qcom,apr.h>
 #include <dt-bindings/thermal/thermal.h>
 
 / {
@@ -1537,6 +1538,20 @@ spmi_bus: spmi@200f000 {
 			#interrupt-cells = <4>;
 		};
 
+		bam_dmux_dma: dma-controller@4044000 {
+			compatible = "qcom,bam-v1.7.0";
+			reg = <0x04044000 0x19000>;
+			interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
+			#dma-cells = <1>;
+			qcom,ee = <0>;
+
+			num-channels = <6>;
+			qcom,num-ees = <1>;
+			qcom,powered-remotely;
+
+			status = "disabled";
+		};
+
 		mpss: remoteproc@4080000 {
 			compatible = "qcom,msm8916-mss-pil";
 			reg = <0x04080000 0x100>, <0x04020000 0x040>;
@@ -1569,6 +1584,22 @@ mpss: remoteproc@4080000 {
 			qcom,halt-regs = <&tcsr 0x18000 0x19000 0x1a000>;
 			status = "disabled";
 
+			bam_dmux: bam-dmux {
+				compatible = "qcom,bam-dmux";
+
+				interrupt-parent = <&hexagon_smsm>;
+				interrupts = <1 IRQ_TYPE_EDGE_BOTH>, <11 IRQ_TYPE_EDGE_BOTH>;
+				interrupt-names = "pc", "pc-ack";
+
+				qcom,smem-states = <&apps_smsm 1>, <&apps_smsm 11>;
+				qcom,smem-state-names = "pc", "pc-ack";
+
+				dmas = <&bam_dmux_dma 4>, <&bam_dmux_dma 5>;
+				dma-names = "tx", "rx";
+
+				status = "disabled";
+			};
+
 			mba {
 				memory-region = <&mba_mem>;
 			};
@@ -1585,6 +1616,54 @@ smd-edge {
 				qcom,remote-pid = <1>;
 
 				label = "hexagon";
+
+				apr: apr {
+					compatible = "qcom,apr-v2";
+					qcom,smd-channels = "apr_audio_svc";
+					qcom,domain = <APR_DOMAIN_ADSP>;
+					#address-cells = <1>;
+					#size-cells = <0>;
+					status = "disabled";
+
+					q6core: service@3 {
+						compatible = "qcom,q6core";
+						reg = <APR_SVC_ADSP_CORE>;
+					};
+
+					q6afe: service@4 {
+						compatible = "qcom,q6afe";
+						reg = <APR_SVC_AFE>;
+
+						q6afedai: dais {
+							compatible = "qcom,q6afe-dais";
+							#address-cells = <1>;
+							#size-cells = <0>;
+							#sound-dai-cells = <1>;
+						};
+					};
+
+					q6asm: service@7 {
+						compatible = "qcom,q6asm";
+						reg = <APR_SVC_ASM>;
+
+						q6asmdai: dais {
+							compatible = "qcom,q6asm-dais";
+							#address-cells = <1>;
+							#size-cells = <0>;
+							#sound-dai-cells = <1>;
+						};
+					};
+
+					q6adm: service@8 {
+						compatible = "qcom,q6adm";
+						reg = <APR_SVC_ADM>;
+
+						q6routing: routing {
+							compatible = "qcom,q6adm-routing";
+							#sound-dai-cells = <0>;
+						};
+					};
+				};
 			};
 		};
 
diff --git a/arch/arm64/boot/dts/qcom/pm7250b.dtsi b/arch/arm64/boot/dts/qcom/pm7250b.dtsi
index df0afe82f250..3bf7cf5d1700 100644
--- a/arch/arm64/boot/dts/qcom/pm7250b.dtsi
+++ b/arch/arm64/boot/dts/qcom/pm7250b.dtsi
@@ -148,7 +148,7 @@ pm7250b_adc_tm: adc-tm@3500 {
 			status = "disabled";
 		};
 
-		pm7250b_gpios: pinctrl@c000 {
+		pm7250b_gpios: gpio@c000 {
 			compatible = "qcom,pm7250b-gpio", "qcom,spmi-gpio";
 			reg = <0xc000>;
 			gpio-controller;
diff --git a/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts b/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts
index a7278a9472ed..9738c0dacd58 100644
--- a/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts
+++ b/arch/arm64/boot/dts/qcom/qrb4210-rb2.dts
@@ -518,7 +518,6 @@ &usb {
 
 &usb_dwc3 {
 	maximum-speed = "super-speed";
-	dr_mode = "peripheral";
 };
 
 &usb_hsphy {
diff --git a/arch/arm64/boot/dts/qcom/sa8775p.dtsi b/arch/arm64/boot/dts/qcom/sa8775p.dtsi
index b6a93b11cbbd..13dd44dd9ed1 100644
--- a/arch/arm64/boot/dts/qcom/sa8775p.dtsi
+++ b/arch/arm64/boot/dts/qcom/sa8775p.dtsi
@@ -1487,6 +1487,11 @@ &config_noc SLAVE_QUP_3 QCOM_ICC_TAG_ALWAYS>,
 			};
 		};
 
+		rng: rng@10d2000 {
+			compatible = "qcom,sa8775p-trng", "qcom,trng";
+			reg = <0 0x010d2000 0 0x1000>;
+		};
+
 		ufs_mem_hc: ufs@1d84000 {
 			compatible = "qcom,sa8775p-ufshc", "qcom,ufshc", "jedec,ufs-2.0";
 			reg = <0x0 0x01d84000 0x0 0x3000>;
@@ -1910,6 +1915,50 @@ pdc: interrupt-controller@b220000 {
 			interrupt-controller;
 		};
 
+		tsens2: thermal-sensor@c251000 {
+			compatible = "qcom,sa8775p-tsens", "qcom,tsens-v2";
+			reg = <0x0 0x0c251000 0x0 0x1ff>,
+			      <0x0 0x0c224000 0x0 0x8>;
+			interrupts = <GIC_SPI 572 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 609 IRQ_TYPE_LEVEL_HIGH>;
+			#qcom,sensors = <13>;
+			interrupt-names = "uplow", "critical";
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens3: thermal-sensor@c252000 {
+			compatible = "qcom,sa8775p-tsens", "qcom,tsens-v2";
+			reg = <0x0 0x0c252000 0x0 0x1ff>,
+			      <0x0 0x0c225000 0x0 0x8>;
+			interrupts = <GIC_SPI 573 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 610 IRQ_TYPE_LEVEL_HIGH>;
+			#qcom,sensors = <13>;
+			interrupt-names = "uplow", "critical";
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens0: thermal-sensor@c263000 {
+			compatible = "qcom,sa8775p-tsens", "qcom,tsens-v2";
+			reg = <0x0 0x0c263000 0x0 0x1ff>,
+			      <0x0 0x0c222000 0x0 0x8>;
+			interrupts = <GIC_SPI 506 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 508 IRQ_TYPE_LEVEL_HIGH>;
+			#qcom,sensors = <12>;
+			interrupt-names = "uplow", "critical";
+			#thermal-sensor-cells = <1>;
+		};
+
+		tsens1: thermal-sensor@c265000 {
+			compatible = "qcom,sa8775p-tsens", "qcom,tsens-v2";
+			reg = <0x0 0x0c265000 0x0 0x1ff>,
+			      <0x0 0x0c223000 0x0 0x8>;
+			interrupts = <GIC_SPI 507 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 509 IRQ_TYPE_LEVEL_HIGH>;
+			#qcom,sensors = <12>;
+			interrupt-names = "uplow", "critical";
+			#thermal-sensor-cells = <1>;
+		};
+
 		aoss_qmp: power-management@c300000 {
 			compatible = "qcom,sa8775p-aoss-qmp", "qcom,aoss-qmp";
 			reg = <0x0 0x0c300000 0x0 0x400>;
@@ -1920,6 +1969,11 @@ IPCC_MPROC_SIGNAL_GLINK_QMP
 			#clock-cells = <0>;
 		};
 
+		sram@c3f0000 {
+			compatible = "qcom,rpmh-stats";
+			reg = <0x0 0x0c3f0000 0x0 0x400>;
+		};
+
 		spmi_bus: spmi@c440000 {
 			compatible = "qcom,spmi-pmic-arb";
 			reg = <0x0 0x0c440000 0x0 0x1100>,
@@ -2401,6 +2455,1058 @@ ethernet0: ethernet@23040000 {
 		};
 	};
 
+	thermal-zones {
+		aoss-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens0 0>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-0-0-0-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens0 1>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-0-1-0-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens0 2>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-0-2-0-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens0 3>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-0-3-0-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens0 4>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpuss-0-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens0 5>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpuss-1-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens0 6>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpuss-2-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens0 7>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		audio-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens0 8>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		camss-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens0 9>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		pcie-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens0 10>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpuss-0-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens0 11>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		aoss-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens1 0>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-0-0-1-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens1 1>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-0-1-1-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens1 2>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-0-2-1-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens1 3>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-0-3-1-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens1 4>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpuss-3-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens1 5>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpuss-4-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens1 6>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		gpuss-5-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens1 7>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		video-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens1 8>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		camss-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens1 9>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		pcie-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens1 10>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpuss-0-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens1 11>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		aoss-2-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens2 0>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-1-0-0-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens2 1>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-1-1-0-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens2 2>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-1-2-0-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens2 3>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-1-3-0-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens2 4>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-0-0-0-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens2 5>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-0-1-0-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens2 6>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-0-2-0-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens2 7>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-1-0-0-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens2 8>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-1-1-0-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens2 9>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-1-2-0-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens2 10>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		ddrss-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens2 11>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpuss-1-0-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens2 12>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		aoss-3-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens3 0>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-1-0-1-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens3 1>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-1-1-1-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens3 2>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-1-2-1-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens3 3>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpu-1-3-1-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens3 4>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-0-0-1-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens3 5>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-0-1-1-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens3 6>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-0-2-1-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens3 7>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-1-0-1-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens3 8>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-1-1-1-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens3 9>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		nsp-1-2-1-thermal {
+			polling-delay-passive = <10>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens3 10>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		ddrss-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens3 11>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+
+		cpuss-1-1-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&tsens3 12>;
+
+			trips {
+				trip-point0 {
+					temperature = <105000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+
+				trip-point1 {
+					temperature = <115000>;
+					hysteresis = <5000>;
+					type = "passive";
+				};
+			};
+		};
+	};
+
 	arch_timer: timer {
 		compatible = "arm,armv8-timer";
 		interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_LOW)>,
diff --git a/arch/arm64/boot/dts/qcom/sc7280.dtsi b/arch/arm64/boot/dts/qcom/sc7280.dtsi
index 66f1eb83cca7..04bf85b0399a 100644
--- a/arch/arm64/boot/dts/qcom/sc7280.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc7280.dtsi
@@ -2034,6 +2034,11 @@ uart15: serial@a9c000 {
 			};
 		};
 
+		rng: rng@10d3000 {
+			compatible = "qcom,sc7280-trng", "qcom,trng";
+			reg = <0 0x010d3000 0 0x1000>;
+		};
+
 		cnoc2: interconnect@1500000 {
 			reg = <0 0x01500000 0 0x1000>;
 			compatible = "qcom,sc7280-cnoc2";
@@ -3399,6 +3404,32 @@ usb_1_qmpphy: phy@88e8000 {
 
 			#clock-cells = <1>;
 			#phy-cells = <1>;
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					usb_dp_qmpphy_out: endpoint {
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					usb_dp_qmpphy_usb_ss_in: endpoint {
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					usb_dp_qmpphy_dp_in: endpoint {
+					};
+				};
+			};
 		};
 
 		usb_2: usb@8cf8800 {
@@ -3793,6 +3824,86 @@ videocc: clock-controller@aaf0000 {
 			#power-domain-cells = <1>;
 		};
 
+		cci0: cci@ac4a000 {
+			compatible = "qcom,sc7280-cci", "qcom,msm8996-cci";
+			reg = <0 0x0ac4a000 0 0x1000>;
+			interrupts = <GIC_SPI 460 IRQ_TYPE_EDGE_RISING>;
+			power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>;
+
+			clocks = <&camcc CAM_CC_CAMNOC_AXI_CLK>,
+				 <&camcc CAM_CC_SLOW_AHB_CLK_SRC>,
+				 <&camcc CAM_CC_CPAS_AHB_CLK>,
+				 <&camcc CAM_CC_CCI_0_CLK>,
+				 <&camcc CAM_CC_CCI_0_CLK_SRC>;
+			clock-names = "camnoc_axi",
+				      "slow_ahb_src",
+				      "cpas_ahb",
+				      "cci",
+				      "cci_src";
+			pinctrl-0 = <&cci0_default &cci1_default>;
+			pinctrl-1 = <&cci0_sleep &cci1_sleep>;
+			pinctrl-names = "default", "sleep";
+
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			status = "disabled";
+
+			cci0_i2c0: i2c-bus@0 {
+				reg = <0>;
+				clock-frequency = <1000000>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+
+			cci0_i2c1: i2c-bus@1 {
+				reg = <1>;
+				clock-frequency = <1000000>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+		};
+
+		cci1: cci@ac4b000 {
+			compatible = "qcom,sc7280-cci", "qcom,msm8996-cci";
+			reg = <0 0x0ac4b000 0 0x1000>;
+			interrupts = <GIC_SPI 271 IRQ_TYPE_EDGE_RISING>;
+			power-domains = <&camcc CAM_CC_TITAN_TOP_GDSC>;
+
+			clocks = <&camcc CAM_CC_CAMNOC_AXI_CLK>,
+				 <&camcc CAM_CC_SLOW_AHB_CLK_SRC>,
+				 <&camcc CAM_CC_CPAS_AHB_CLK>,
+				 <&camcc CAM_CC_CCI_1_CLK>,
+				 <&camcc CAM_CC_CCI_1_CLK_SRC>;
+			clock-names = "camnoc_axi",
+				      "slow_ahb_src",
+				      "cpas_ahb",
+				      "cci",
+				      "cci_src";
+			pinctrl-0 = <&cci2_default &cci3_default>;
+			pinctrl-1 = <&cci2_sleep &cci3_sleep>;
+			pinctrl-names = "default", "sleep";
+
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			status = "disabled";
+
+			cci1_i2c0: i2c-bus@0 {
+				reg = <0>;
+				clock-frequency = <1000000>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+
+			cci1_i2c1: i2c-bus@1 {
+				reg = <1>;
+				clock-frequency = <1000000>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+		};
+
 		camcc: clock-controller@ad00000 {
 			compatible = "qcom,sc7280-camcc";
 			reg = <0 0x0ad00000 0 0x10000>;
@@ -4298,6 +4409,62 @@ tlmm: pinctrl@f100000 {
 			gpio-ranges = <&tlmm 0 0 175>;
 			wakeup-parent = <&pdc>;
 
+			cci0_default: cci0-default-state {
+				pins = "gpio69", "gpio70";
+				function = "cci_i2c";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			cci0_sleep: cci0-sleep-state {
+				pins = "gpio69", "gpio70";
+				function = "cci_i2c";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			cci1_default: cci1-default-state {
+				pins = "gpio71", "gpio72";
+				function = "cci_i2c";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			cci1_sleep: cci1-sleep-state {
+				pins = "gpio71", "gpio72";
+				function = "cci_i2c";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			cci2_default: cci2-default-state {
+				pins = "gpio73", "gpio74";
+				function = "cci_i2c";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			cci2_sleep: cci2-sleep-state {
+				pins = "gpio73", "gpio74";
+				function = "cci_i2c";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
+			cci3_default: cci3-default-state {
+				pins = "gpio75", "gpio76";
+				function = "cci_i2c";
+				drive-strength = <2>;
+				bias-pull-up;
+			};
+
+			cci3_sleep: cci3-sleep-state {
+				pins = "gpio75", "gpio76";
+				function = "cci_i2c";
+				drive-strength = <2>;
+				bias-pull-down;
+			};
+
 			dp_hot_plug_det: dp-hot-plug-det-state {
 				pins = "gpio47";
 				function = "dp_hot";
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts b/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts
index 38edaf51aa34..f2055899ae7a 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts
+++ b/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts
@@ -82,6 +82,9 @@ switch-lid {
 	leds {
 		compatible = "gpio-leds";
 
+		pinctrl-names = "default";
+		pinctrl-0 = <&cam_indicator_en>;
+
 		led-camera-indicator {
 			label = "white:camera-indicator";
 			function = LED_FUNCTION_INDICATOR;
@@ -601,6 +604,7 @@ mdss0_dp3_out: endpoint {
 };
 
 &mdss0_dp3_phy {
+	compatible = "qcom,sc8280xp-edp-phy";
 	vdda-phy-supply = <&vreg_l6b>;
 	vdda-pll-supply = <&vreg_l3b>;
 
@@ -1277,6 +1281,13 @@ hstp-sw-ctrl-pins {
 		};
 	};
 
+	cam_indicator_en: cam-indicator-en-state {
+		pins = "gpio28";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+	};
+
 	edp_reg_en: edp-reg-en-state {
 		pins = "gpio25";
 		function = "gpio";
diff --git a/arch/arm64/boot/dts/qcom/sdm670.dtsi b/arch/arm64/boot/dts/qcom/sdm670.dtsi
index ba2043d67370..6d9843d05cb3 100644
--- a/arch/arm64/boot/dts/qcom/sdm670.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm670.dtsi
@@ -1532,7 +1532,7 @@ osm_l3: interconnect@17d41000 {
 		};
 
 		cpufreq_hw: cpufreq@17d43000 {
-			compatible = "qcom,cpufreq-hw";
+			compatible = "qcom,sdm670-cpufreq-hw", "qcom,cpufreq-hw";
 			reg = <0 0x17d43000 0 0x1400>, <0 0x17d45800 0 0x1400>;
 			reg-names = "freq-domain0", "freq-domain1";
 
diff --git a/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi
index b523b5fff702..e821103d49c0 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi
@@ -8,6 +8,7 @@
 /dts-v1/;
 
 #include <dt-bindings/input/linux-event-codes.h>
+#include <dt-bindings/leds/common.h>
 #include <dt-bindings/regulator/qcom,rpmh-regulator.h>
 #include <dt-bindings/sound/qcom,q6afe.h>
 #include <dt-bindings/sound/qcom,q6asm.h>
@@ -484,6 +485,28 @@ &pmi8998_charger {
 	status = "okay";
 };
 
+&pmi8998_flash {
+	status = "okay";
+
+	led-0 {
+		function = LED_FUNCTION_FLASH;
+		color = <LED_COLOR_ID_WHITE>;
+		led-sources = <1>;
+		led-max-microamp = <500000>;
+		flash-max-microamp = <1500000>;
+		flash-max-timeout-us = <1280000>;
+	};
+
+	led-1 {
+		function = LED_FUNCTION_FLASH;
+		color = <LED_COLOR_ID_YELLOW>;
+		led-sources = <2>;
+		led-max-microamp = <500000>;
+		flash-max-microamp = <1500000>;
+		flash-max-timeout-us = <1280000>;
+	};
+};
+
 &q6afedai {
 	qi2s@22 {
 		reg = <22>;
diff --git a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi
index 93b1582e807d..617b17b2d7d9 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi
@@ -355,6 +355,28 @@ &pmi8998_charger {
 	status = "okay";
 };
 
+&pmi8998_flash {
+	status = "okay";
+
+	led-0 {
+		function = LED_FUNCTION_FLASH;
+		color = <LED_COLOR_ID_WHITE>;
+		led-sources = <1>;
+		led-max-microamp = <500000>;
+		flash-max-microamp = <1500000>;
+		flash-max-timeout-us = <1280000>;
+	};
+
+	led-1 {
+		function = LED_FUNCTION_FLASH;
+		color = <LED_COLOR_ID_YELLOW>;
+		led-sources = <2>;
+		led-max-microamp = <500000>;
+		flash-max-microamp = <1500000>;
+		flash-max-timeout-us = <1280000>;
+	};
+};
+
 &pm8998_resin {
 	linux,code = <KEY_VOLUMEDOWN>;
 	status = "okay";
diff --git a/arch/arm64/boot/dts/qcom/sm6375-sony-xperia-murray-pdx225.dts b/arch/arm64/boot/dts/qcom/sm6375-sony-xperia-murray-pdx225.dts
index b2f1bb1d58e9..0ce4fa8de8b0 100644
--- a/arch/arm64/boot/dts/qcom/sm6375-sony-xperia-murray-pdx225.dts
+++ b/arch/arm64/boot/dts/qcom/sm6375-sony-xperia-murray-pdx225.dts
@@ -187,6 +187,11 @@ &remoteproc_cdsp {
 	status = "okay";
 };
 
+&remoteproc_mss {
+	firmware-name = "qcom/sm6375/Sony/murray/modem.mbn";
+	status = "okay";
+};
+
 &rpm_requests {
 	regulators-0 {
 		compatible = "qcom,rpm-pm6125-regulators";
@@ -238,8 +243,8 @@ pm6125_l6: l6 {
 		};
 
 		pm6125_l7: l7 {
-			regulator-min-microvolt = <720000>;
-			regulator-max-microvolt = <1050000>;
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
 		};
 
 		pm6125_l8: l8 {
@@ -306,7 +311,7 @@ pm6125_l20: l20 {
 
 		pm6125_l21: l21 {
 			regulator-min-microvolt = <3000000>;
-			regulator-max-microvolt = <3400000>;
+			regulator-max-microvolt = <3312000>;
 		};
 
 		pm6125_l22: l22 {
@@ -317,7 +322,7 @@ pm6125_l22: l22 {
 
 		pm6125_l23: l23 {
 			regulator-min-microvolt = <3000000>;
-			regulator-max-microvolt = <3400000>;
+			regulator-max-microvolt = <3312000>;
 		};
 
 		pm6125_l24: l24 {
@@ -340,8 +345,8 @@ pmr735a_l1: l1 {
 		};
 
 		pmr735a_l2: l2 {
-			regulator-min-microvolt = <352000>;
-			regulator-max-microvolt = <796000>;
+			regulator-min-microvolt = <640000>;
+			regulator-max-microvolt = <640000>;
 		};
 
 		pmr735a_l3: l3 {
@@ -425,6 +430,18 @@ &usb_1_dwc3 {
 };
 
 &usb_1_hsphy {
+	vdda-pll-supply = <&pm6125_l7>;
+	vdda18-supply = <&pm6125_l10>;
+	vdda33-supply = <&pmr735a_l7>;
+	status = "okay";
+};
+
+&wifi {
+	vdd-0.8-cx-mx-supply = <&pmr735a_l2>;
+	vdd-1.8-xo-supply = <&pm6125_l16>;
+	vdd-1.3-rfa-supply = <&pm6125_l2>;
+	vdd-3.3-ch0-supply = <&pm6125_l23>;
+	vdd-3.3-ch1-supply = <&pm6125_l21>;
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/qcom/sm6375.dtsi b/arch/arm64/boot/dts/qcom/sm6375.dtsi
index e7ff55443da7..2fba0e7ea4e6 100644
--- a/arch/arm64/boot/dts/qcom/sm6375.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm6375.dtsi
@@ -896,6 +896,36 @@ qup_spi0_default: qup-spi0-default-state {
 				drive-strength = <6>;
 				bias-disable;
 			};
+
+			qup_uart1_default: qup-uart1-default-state {
+				cts-pins {
+					pins = "gpio61";
+					function = "qup01";
+					drive-strength = <2>;
+					bias-pull-down;
+				};
+
+				rts-pins {
+					pins = "gpio62";
+					function = "qup01";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				tx-pins {
+					pins = "gpio63";
+					function = "qup01";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				rx-pins {
+					pins = "gpio64";
+					function = "qup01";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+			};
 		};
 
 		gcc: clock-controller@1400000 {
@@ -1111,6 +1141,19 @@ spi1: spi@4a84000 {
 				status = "disabled";
 			};
 
+			uart1: serial@4a84000 {
+				compatible = "qcom,geni-uart";
+				reg = <0x0 0x04a84000 0x0 0x4000>;
+				interrupts = <GIC_SPI 328 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GCC_QUPV3_WRAP0_S1_CLK>;
+				clock-names = "se";
+				power-domains = <&rpmpd SM6375_VDDCX>;
+				operating-points-v2 = <&qup_opp_table>;
+				pinctrl-0 = <&qup_uart1_default>;
+				pinctrl-names = "default";
+				status = "disabled";
+			};
+
 			i2c2: i2c@4a88000 {
 				compatible = "qcom,geni-i2c";
 				reg = <0x0 0x04a88000 0x0 0x4000>;
diff --git a/arch/arm64/boot/dts/qcom/sm8450.dtsi b/arch/arm64/boot/dts/qcom/sm8450.dtsi
index 1783fa78bdbc..bde9c1093384 100644
--- a/arch/arm64/boot/dts/qcom/sm8450.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8450.dtsi
@@ -1739,6 +1739,11 @@ spi14: spi@a98000 {
 			};
 		};
 
+		rng: rng@10c3000 {
+			compatible = "qcom,sm8450-trng", "qcom,trng";
+			reg = <0 0x010c3000 0 0x1000>;
+		};
+
 		pcie0: pci@1c00000 {
 			compatible = "qcom,pcie-sm8450-pcie0";
 			reg = <0 0x01c00000 0 0x3000>,
diff --git a/arch/arm64/boot/dts/qcom/sm8550.dtsi b/arch/arm64/boot/dts/qcom/sm8550.dtsi
index 7b9ddde0b2c9..7bafb3d88d69 100644
--- a/arch/arm64/boot/dts/qcom/sm8550.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8550.dtsi
@@ -1677,6 +1677,11 @@ mmss_noc: interconnect@1780000 {
 			qcom,bcm-voters = <&apps_bcm_voter>;
 		};
 
+		rng: rng@10c3000 {
+			compatible = "qcom,sm8550-trng", "qcom,trng";
+			reg = <0 0x010c3000 0 0x1000>;
+		};
+
 		pcie0: pci@1c00000 {
 			device_type = "pci";
 			compatible = "qcom,pcie-sm8550";
diff --git a/arch/arm64/boot/dts/renesas/draak.dtsi b/arch/arm64/boot/dts/renesas/draak.dtsi
index ef3bb835d5c0..ce57c9ad9672 100644
--- a/arch/arm64/boot/dts/renesas/draak.dtsi
+++ b/arch/arm64/boot/dts/renesas/draak.dtsi
@@ -374,6 +374,12 @@ hdmi-encoder@39 {
 		interrupt-parent = <&gpio1>;
 		interrupts = <28 IRQ_TYPE_LEVEL_LOW>;
 
+		avdd-supply = <&reg_1p8v>;
+		dvdd-supply = <&reg_1p8v>;
+		pvdd-supply = <&reg_1p8v>;
+		dvdd-3v-supply = <&reg_3p3v>;
+		bgvdd-supply = <&reg_1p8v>;
+
 		adi,input-depth = <8>;
 		adi,input-colorspace = "rgb";
 		adi,input-clock = "1x";
diff --git a/arch/arm64/boot/dts/renesas/ebisu.dtsi b/arch/arm64/boot/dts/renesas/ebisu.dtsi
index f1a5778ef115..cba2fde9dd36 100644
--- a/arch/arm64/boot/dts/renesas/ebisu.dtsi
+++ b/arch/arm64/boot/dts/renesas/ebisu.dtsi
@@ -403,6 +403,12 @@ hdmi-encoder@39 {
 		interrupt-parent = <&gpio1>;
 		interrupts = <1 IRQ_TYPE_LEVEL_LOW>;
 
+		avdd-supply = <&reg_1p8v>;
+		dvdd-supply = <&reg_1p8v>;
+		pvdd-supply = <&reg_1p8v>;
+		dvdd-3v-supply = <&reg_3p3v>;
+		bgvdd-supply = <&reg_1p8v>;
+
 		adi,input-depth = <8>;
 		adi,input-colorspace = "rgb";
 		adi,input-clock = "1x";
diff --git a/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts b/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts
index 405404c0843d..0608dce92e40 100644
--- a/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts
@@ -29,6 +29,15 @@ chosen {
 		stdout-path = "serial0:115200n8";
 	};
 
+	d1p8: regulator-fixed {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-1.8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
 	d3p3: regulator-fixed {
 		compatible = "regulator-fixed";
 		regulator-name = "fixed-3.3V";
@@ -166,6 +175,12 @@ hdmi@39 {
 		interrupt-parent = <&gpio1>;
 		interrupts = <20 IRQ_TYPE_LEVEL_LOW>;
 
+		avdd-supply = <&d1p8>;
+		dvdd-supply = <&d1p8>;
+		pvdd-supply = <&d1p8>;
+		dvdd-3v-supply = <&d3p3>;
+		bgvdd-supply = <&d1p8>;
+
 		adi,input-depth = <8>;
 		adi,input-colorspace = "rgb";
 		adi,input-clock = "1x";
diff --git a/arch/arm64/boot/dts/renesas/rzg2l-smarc-som.dtsi b/arch/arm64/boot/dts/renesas/rzg2l-smarc-som.dtsi
index 547859c388ce..4409c47239b9 100644
--- a/arch/arm64/boot/dts/renesas/rzg2l-smarc-som.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg2l-smarc-som.dtsi
@@ -312,6 +312,7 @@ flash@0 {
 		m25p,fast-read;
 		spi-max-frequency = <50000000>;
 		spi-rx-bus-width = <4>;
+		spi-tx-bus-width = <4>;
 
 		partitions {
 			compatible = "fixed-partitions";
diff --git a/arch/arm64/boot/dts/renesas/rzg2lc-smarc-som.dtsi b/arch/arm64/boot/dts/renesas/rzg2lc-smarc-som.dtsi
index 56ff92453976..5e4209d6fb42 100644
--- a/arch/arm64/boot/dts/renesas/rzg2lc-smarc-som.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg2lc-smarc-som.dtsi
@@ -241,6 +241,7 @@ flash@0 {
 		m25p,fast-read;
 		spi-max-frequency = <50000000>;
 		spi-rx-bus-width = <4>;
+		spi-tx-bus-width = <4>;
 
 		partitions {
 			compatible = "fixed-partitions";
diff --git a/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi b/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi
index a199de8f8b02..01a4a9da7afc 100644
--- a/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg3s-smarc-som.dtsi
@@ -13,14 +13,21 @@
  * @SW_SD0_DEV_SEL:
  *	0 - SD0 is connected to eMMC
  *	1 - SD0 is connected to uSD0 card
+ * @SW_SD2_EN:
+ *	0 - SCIF1, SSI0, IRQ0, IRQ1 connected to SoC
+ *	1 - SD2 is connected to SoC
  */
 #define SW_SD0_DEV_SEL	1
+#define SW_SD2_EN	1
 
 / {
 	compatible = "renesas,rzg3s-smarcm", "renesas,r9a08g045s33", "renesas,r9a08g045";
 
 	aliases {
 		mmc0 = &sdhi0;
+#if SW_SD2_EN
+		mmc2 = &sdhi2;
+#endif
 	};
 
 	chosen {
@@ -63,6 +70,15 @@ reg_1p8v: regulator1 {
 		regulator-always-on;
 	};
 #endif
+
+	vcc_sdhi2: regulator2 {
+		compatible = "regulator-fixed";
+		regulator-name = "SDHI2 Vcc";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpios = <&pinctrl RZG2L_GPIO(8, 1) GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
 };
 
 &extal_clk {
@@ -100,6 +116,17 @@ &sdhi0 {
 };
 #endif
 
+#if SW_SD2_EN
+&sdhi2 {
+	pinctrl-0 = <&sdhi2_pins>;
+	pinctrl-names = "default";
+	vmmc-supply = <&vcc_sdhi2>;
+	bus-width = <4>;
+	max-frequency = <50000000>;
+	status = "okay";
+};
+#endif
+
 &pinctrl {
 	sdhi0_pins: sd0 {
 		data {
@@ -139,4 +166,26 @@ sdhi0_emmc_pins: sd0-emmc {
 		       "SD0_CLK", "SD0_CMD", "SD0_RST#";
 		power-source = <1800>;
 	};
+
+	sdhi2_pins: sd2 {
+		data {
+			pins = "P11_2", "P11_3", "P12_0", "P12_1";
+			input-enable;
+		};
+
+		ctrl {
+			pins = "P11_1";
+			input-enable;
+		};
+
+		mux {
+			pinmux = <RZG2L_PORT_PINMUX(11, 0, 8)>, /* SD2_CLK */
+				 <RZG2L_PORT_PINMUX(11, 1, 8)>, /* SD2_CMD */
+				 <RZG2L_PORT_PINMUX(11, 2, 8)>, /* SD2_DATA0 */
+				 <RZG2L_PORT_PINMUX(11, 3, 8)>, /* SD2_DATA1 */
+				 <RZG2L_PORT_PINMUX(12, 0, 8)>, /* SD2_DATA2 */
+				 <RZG2L_PORT_PINMUX(12, 1, 8)>, /* SD2_DATA3 */
+				 <RZG2L_PORT_PINMUX(14, 1, 7)>; /* SD2_CD# */
+		};
+	};
 };
diff --git a/arch/arm64/boot/dts/renesas/rzg3s-smarc.dtsi b/arch/arm64/boot/dts/renesas/rzg3s-smarc.dtsi
index e7073a09ed2e..214520137230 100644
--- a/arch/arm64/boot/dts/renesas/rzg3s-smarc.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg3s-smarc.dtsi
@@ -11,6 +11,26 @@
 / {
 	aliases {
 		serial0 = &scif0;
+		mmc1 = &sdhi1;
+	};
+
+	vcc_sdhi1: regulator-vcc-sdhi1 {
+		compatible = "regulator-fixed";
+		regulator-name = "SDHI1 Vcc";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpios = <&pinctrl RZG2L_GPIO(2, 3) GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	vccq_sdhi1: regulator-vccq-sdhi1 {
+		compatible = "regulator-gpio";
+		regulator-name = "SDHI1 VccQ";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		gpios = <&pinctrl RZG2L_GPIO(4, 2) GPIO_ACTIVE_HIGH>;
+		gpios-states = <1>;
+		states = <3300000 1>, <1800000 0>;
 	};
 };
 
@@ -19,6 +39,38 @@ scif0_pins: scif0 {
 		pinmux = <RZG2L_PORT_PINMUX(6, 3, 1)>, /* RXD */
 			 <RZG2L_PORT_PINMUX(6, 4, 1)>; /* TXD */
 	};
+
+	sdhi1_pins: sd1 {
+		data {
+			pins = "SD1_DATA0", "SD1_DATA1", "SD1_DATA2", "SD1_DATA3";
+			power-source = <3300>;
+		};
+
+		ctrl {
+			pins = "SD1_CLK", "SD1_CMD";
+			power-source = <3300>;
+		};
+
+		cd {
+			pinmux = <RZG2L_PORT_PINMUX(0, 2, 1)>; /* SD1_CD */
+		};
+	};
+
+	sdhi1_pins_uhs: sd1-uhs {
+		data {
+			pins = "SD1_DATA0", "SD1_DATA1", "SD1_DATA2", "SD1_DATA3";
+			power-source = <1800>;
+		};
+
+		ctrl {
+			pins = "SD1_CLK", "SD1_CMD";
+			power-source = <1800>;
+		};
+
+		cd {
+			pinmux = <RZG2L_PORT_PINMUX(0, 2, 1)>; /* SD1_CD */
+		};
+	};
 };
 
 &scif0 {
@@ -26,3 +78,16 @@ &scif0 {
 	pinctrl-0 = <&scif0_pins>;
 	status = "okay";
 };
+
+&sdhi1 {
+	pinctrl-0 = <&sdhi1_pins>;
+	pinctrl-1 = <&sdhi1_pins_uhs>;
+	pinctrl-names = "default", "state_uhs";
+	vmmc-supply = <&vcc_sdhi1>;
+	vqmmc-supply = <&vccq_sdhi1>;
+	bus-width = <4>;
+	sd-uhs-sdr50;
+	sd-uhs-sdr104;
+	max-frequency = <125000000>;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/rockchip/Makefile b/arch/arm64/boot/dts/rockchip/Makefile
index a18f33bf0c0e..f969618da352 100644
--- a/arch/arm64/boot/dts/rockchip/Makefile
+++ b/arch/arm64/boot/dts/rockchip/Makefile
@@ -78,6 +78,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-anbernic-rg503.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-pinenote-v1.1.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-pinenote-v1.2.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-powkiddy-rgb30.dtb
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-powkiddy-rk2023.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-quartz64-a.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-quartz64-b.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-radxa-cm3-io.dtb
diff --git a/arch/arm64/boot/dts/rockchip/rk3328.dtsi b/arch/arm64/boot/dts/rockchip/rk3328.dtsi
index e729e7a22b23..e18f7c1c0724 100644
--- a/arch/arm64/boot/dts/rockchip/rk3328.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3328.dtsi
@@ -668,7 +668,7 @@ vpu_mmu: iommu@ff350800 {
 
 	vdec: video-codec@ff360000 {
 		compatible = "rockchip,rk3328-vdec", "rockchip,rk3399-vdec";
-		reg = <0x0 0xff360000 0x0 0x400>;
+		reg = <0x0 0xff360000 0x0 0x480>;
 		interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cru ACLK_RKVDEC>, <&cru HCLK_RKVDEC>,
 			 <&cru SCLK_VDEC_CABAC>, <&cru SCLK_VDEC_CORE>;
@@ -916,6 +916,8 @@ gmac2io: ethernet@ff540000 {
 		resets = <&cru SRST_GMAC2IO_A>;
 		reset-names = "stmmaceth";
 		rockchip,grf = <&grf>;
+		tx-fifo-depth = <2048>;
+		rx-fifo-depth = <4096>;
 		snps,txpbl = <0x4>;
 		status = "disabled";
 	};
@@ -938,6 +940,8 @@ gmac2phy: ethernet@ff550000 {
 		reset-names = "stmmaceth";
 		phy-mode = "rmii";
 		phy-handle = <&phy>;
+		tx-fifo-depth = <2048>;
+		rx-fifo-depth = <4096>;
 		snps,txpbl = <0x4>;
 		clock_in_out = "output";
 		status = "disabled";
diff --git a/arch/arm64/boot/dts/rockchip/rk3399.dtsi b/arch/arm64/boot/dts/rockchip/rk3399.dtsi
index faf02e59d6c7..da0dfb237f85 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3399.dtsi
@@ -1109,7 +1109,9 @@ power-domain@RK3399_PD_VCODEC {
 			power-domain@RK3399_PD_VDU {
 				reg = <RK3399_PD_VDU>;
 				clocks = <&cru ACLK_VDU>,
-					 <&cru HCLK_VDU>;
+					 <&cru HCLK_VDU>,
+					 <&cru SCLK_VDU_CA>,
+					 <&cru SCLK_VDU_CORE>;
 				pm_qos = <&qos_video_m1_r>,
 					 <&qos_video_m1_w>;
 				#power-domain-cells = <0>;
@@ -1384,7 +1386,7 @@ vpu_mmu: iommu@ff650800 {
 
 	vdec: video-codec@ff660000 {
 		compatible = "rockchip,rk3399-vdec";
-		reg = <0x0 0xff660000 0x0 0x400>;
+		reg = <0x0 0xff660000 0x0 0x480>;
 		interrupts = <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH 0>;
 		clocks = <&cru ACLK_VDU>, <&cru HCLK_VDU>,
 			 <&cru SCLK_VDU_CA>, <&cru SCLK_VDU_CORE>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3566-powkiddy-rgb30.dts b/arch/arm64/boot/dts/rockchip/rk3566-powkiddy-rgb30.dts
index 1ead3c5c24b3..0ac64f043b80 100644
--- a/arch/arm64/boot/dts/rockchip/rk3566-powkiddy-rgb30.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3566-powkiddy-rgb30.dts
@@ -5,67 +5,11 @@
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/linux-event-codes.h>
 #include <dt-bindings/pinctrl/rockchip.h>
-#include "rk3566-anbernic-rg353x.dtsi"
+#include "rk3566-powkiddy-rk2023.dtsi"
 
 / {
 	model = "RGB30";
 	compatible = "powkiddy,rgb30", "rockchip,rk3566";
-
-	aliases {
-		mmc1 = &sdmmc0;
-		mmc2 = &sdmmc1;
-		mmc3 = &sdmmc2;
-	};
-
-	battery: battery {
-		compatible = "simple-battery";
-		charge-full-design-microamp-hours = <3151000>;
-		charge-term-current-microamp = <300000>;
-		constant-charge-current-max-microamp = <2000000>;
-		constant-charge-voltage-max-microvolt = <4250000>;
-		factory-internal-resistance-micro-ohms = <117000>;
-		voltage-max-design-microvolt = <4172000>;
-		voltage-min-design-microvolt = <3400000>;
-
-		ocv-capacity-celsius = <20>;
-		ocv-capacity-table-0 =  <4172000 100>, <4092000 95>, <4035000 90>, <3990000 85>,
-					<3939000 80>, <3895000 75>, <3852000 70>, <3807000 65>,
-					<3762000 60>, <3713000 55>, <3672000 50>, <3647000 45>,
-					<3629000 40>, <3613000 35>, <3598000 30>, <3578000 25>,
-					<3550000 20>, <3519000 15>, <3479000 10>, <3438000 5>,
-					<3400000 0>;
-	};
-
-	/*
-	 * Channels reversed for speakers. Headphones automatically switch via hardware when
-	 * detected with no ability to control output in software. Headphones appear to be mono
-	 * (each output channel receives all audio). No microphone support on 3.5mm jack.
-	 */
-	sound {
-		compatible = "simple-audio-card";
-		simple-audio-card,name = "rk817_ext";
-		simple-audio-card,format = "i2s";
-		simple-audio-card,mclk-fs = <256>;
-		simple-audio-card,widgets =
-			"Headphone", "Headphones";
-		simple-audio-card,routing =
-			"Headphones", "HPOL",
-			"Headphones", "HPOR";
-
-		simple-audio-card,codec {
-			sound-dai = <&rk817>;
-		};
-
-		simple-audio-card,cpu {
-			sound-dai = <&i2s1_8ch>;
-		};
-	};
-};
-
-/delete-node/ &adc_keys;
-
-&chosen {
-	/delete-property/ stdout-path;
 };
 
 &cru {
@@ -75,87 +19,21 @@ &cru {
 			       <200000000>, <292500000>;
 };
 
-&gpio_keys_control {
-	button-r1 {
-		gpios = <&gpio3 RK_PB3 GPIO_ACTIVE_LOW>;
-		label = "TR";
-		linux,code = <BTN_TR>;
-	};
-
-	button-r2 {
-		gpios = <&gpio3 RK_PB4 GPIO_ACTIVE_LOW>;
-		label = "TR2";
-		linux,code = <BTN_TR2>;
-	};
-};
-
-/delete-node/ &{/i2c@fdd40000/regulator@40};
-
-&i2c0 {
-	vdd_cpu: regulator@1c {
-		compatible = "tcs,tcs4525";
-		reg = <0x1c>;
-		fcs,suspend-voltage-selector = <1>;
-		regulator-always-on;
-		regulator-boot-on;
-		regulator-min-microvolt = <712500>;
-		regulator-max-microvolt = <1390000>;
-		regulator-name = "vdd_cpu";
-		regulator-ramp-delay = <2300>;
-		vin-supply = <&vcc_sys>;
-		regulator-state-mem {
-			regulator-off-in-suspend;
+&dsi0 {
+	panel: panel@0 {
+		compatible = "powkiddy,rgb30-panel";
+		reg = <0>;
+		backlight = <&backlight>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&lcd_rst>;
+		reset-gpios = <&gpio4 RK_PA0 GPIO_ACTIVE_LOW>;
+		vcc-supply = <&vcc3v3_lcd0_n>;
+		iovcc-supply = <&vcc3v3_lcd0_n>;
+
+		port {
+			mipi_in_panel: endpoint {
+				remote-endpoint = <&mipi_out_panel>;
+			};
 		};
 	};
 };
-
-/*
- * Device has 2 red LEDs instead of an amber and a red. Relabel LEDs as
- * red_led0 and red_led1.
- */
-/delete-node/ &{/pwm-leds/led-1};
-/delete-node/ &{/pwm-leds/led-2};
-
-&leds {
-	red_led0: led-1 {
-		color = <LED_COLOR_ID_RED>;
-		function = LED_FUNCTION_CHARGING;
-		max-brightness = <255>;
-		pwms = <&pwm7 0 25000 0>;
-	};
-
-	red_led1: led-2 {
-		color = <LED_COLOR_ID_RED>;
-		default-state = "off";
-		function = LED_FUNCTION_STATUS;
-		max-brightness = <255>;
-		pwms = <&pwm0 0 25000 0>;
-	};
-};
-
-&panel {
-	compatible = "powkiddy,rgb30-panel";
-	vcc-supply = <&vcc3v3_lcd0_n>;
-	iovcc-supply = <&vcc3v3_lcd0_n>;
-	/delete-property/ vdd-supply;
-};
-
-&pwm5 {
-	status = "disabled";
-};
-
-&rk817 {
-	rk817_charger: charger {
-		monitored-battery = <&battery>;
-		rockchip,resistor-sense-micro-ohms = <10000>;
-		rockchip,sleep-enter-current-microamp = <300000>;
-		rockchip,sleep-filter-current-microamp = <100000>;
-	};
-};
-
-/* There is no UART header visible on the board for this device. */
-&uart2 {
-	status = "disabled";
-};
-
-/delete-node/ &vibrator;
diff --git a/arch/arm64/boot/dts/rockchip/rk3566-powkiddy-rk2023.dts b/arch/arm64/boot/dts/rockchip/rk3566-powkiddy-rk2023.dts
new file mode 100644
index 000000000000..ba32d0793dca
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3566-powkiddy-rk2023.dts
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/linux-event-codes.h>
+#include <dt-bindings/pinctrl/rockchip.h>
+#include "rk3566-powkiddy-rk2023.dtsi"
+
+/ {
+	model = "RK2023";
+	compatible = "powkiddy,rk2023", "rockchip,rk3566";
+};
+
+&cru {
+	assigned-clocks = <&pmucru CLK_RTC_32K>, <&cru PLL_GPLL>,
+			  <&pmucru PLL_PPLL>, <&cru PLL_VPLL>;
+	assigned-clock-rates = <32768>, <1200000000>,
+			  <200000000>, <115200000>;
+};
+
+&dsi0 {
+	panel: panel@0 {
+		compatible = "powkiddy,rk2023-panel", "newvision,nv3051d";
+		reg = <0>;
+		backlight = <&backlight>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&lcd_rst>;
+		reset-gpios = <&gpio4 RK_PA0 GPIO_ACTIVE_LOW>;
+		vdd-supply = <&vcc3v3_lcd0_n>;
+
+		port {
+			mipi_in_panel: endpoint {
+				remote-endpoint = <&mipi_out_panel>;
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3566-powkiddy-rk2023.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-powkiddy-rk2023.dtsi
new file mode 100644
index 000000000000..0fa8f06f94cd
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3566-powkiddy-rk2023.dtsi
@@ -0,0 +1,875 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/linux-event-codes.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/rockchip.h>
+#include <dt-bindings/soc/rockchip,vop2.h>
+#include "rk3566.dtsi"
+
+/ {
+	aliases {
+		mmc1 = &sdmmc0;
+		mmc2 = &sdmmc1;
+		mmc3 = &sdmmc2;
+	};
+
+	adc-joystick {
+		compatible = "adc-joystick";
+		io-channels = <&adc_mux 0>,
+			      <&adc_mux 1>,
+			      <&adc_mux 2>,
+			      <&adc_mux 3>;
+		pinctrl-0 = <&joy_mux_en>;
+		pinctrl-names = "default";
+		poll-interval = <60>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		axis@0 {
+			reg = <0>;
+			abs-flat = <32>;
+			abs-fuzz = <32>;
+			abs-range = <1023 15>;
+			linux,code = <ABS_X>;
+		};
+
+		axis@1 {
+			reg = <1>;
+			abs-flat = <32>;
+			abs-fuzz = <32>;
+			abs-range = <15 1023>;
+			linux,code = <ABS_RX>;
+		};
+
+		axis@2 {
+			reg = <2>;
+			abs-flat = <32>;
+			abs-fuzz = <32>;
+			abs-range = <15 1023>;
+			linux,code = <ABS_Y>;
+		};
+
+		axis@3 {
+			reg = <3>;
+			abs-flat = <32>;
+			abs-fuzz = <32>;
+			abs-range = <1023 15>;
+			linux,code = <ABS_RY>;
+		};
+	};
+
+	adc_mux: adc-mux {
+		compatible = "io-channel-mux";
+		channels = "left_x", "right_x", "left_y", "right_y";
+		#io-channel-cells = <1>;
+		io-channels = <&saradc 3>;
+		io-channel-names = "parent";
+		mux-controls = <&gpio_mux>;
+		settle-time-us = <100>;
+	};
+
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		power-supply = <&vcc_sys>;
+		pwms = <&pwm4 0 25000 0>;
+	};
+
+	battery: battery {
+		compatible = "simple-battery";
+		charge-full-design-microamp-hours = <3151000>;
+		charge-term-current-microamp = <300000>;
+		constant-charge-current-max-microamp = <2000000>;
+		constant-charge-voltage-max-microvolt = <4250000>;
+		factory-internal-resistance-micro-ohms = <117000>;
+		voltage-max-design-microvolt = <4172000>;
+		voltage-min-design-microvolt = <3400000>;
+
+		ocv-capacity-celsius = <20>;
+		ocv-capacity-table-0 =  <4172000 100>, <4092000 95>, <4035000 90>, <3990000 85>,
+					<3939000 80>, <3895000 75>, <3852000 70>, <3807000 65>,
+					<3762000 60>, <3713000 55>, <3672000 50>, <3647000 45>,
+					<3629000 40>, <3613000 35>, <3598000 30>, <3578000 25>,
+					<3550000 20>, <3519000 15>, <3479000 10>, <3438000 5>,
+					<3400000 0>;
+	};
+
+	gpio_keys_control: gpio-keys-control {
+		compatible = "gpio-keys";
+		pinctrl-0 = <&btn_pins_ctrl>;
+		pinctrl-names = "default";
+
+		button-a {
+			gpios = <&gpio3 RK_PC2 GPIO_ACTIVE_LOW>;
+			label = "EAST";
+			linux,code = <BTN_EAST>;
+		};
+
+		button-b {
+			gpios = <&gpio3 RK_PC3 GPIO_ACTIVE_LOW>;
+			label = "SOUTH";
+			linux,code = <BTN_SOUTH>;
+		};
+
+		button-down {
+			gpios = <&gpio3 RK_PA4 GPIO_ACTIVE_LOW>;
+			label = "DPAD-DOWN";
+			linux,code = <BTN_DPAD_DOWN>;
+		};
+
+		button-l1 {
+			gpios = <&gpio3 RK_PB1 GPIO_ACTIVE_LOW>;
+			label = "TL";
+			linux,code = <BTN_TL>;
+		};
+
+		button-l2 {
+			gpios = <&gpio3 RK_PB2 GPIO_ACTIVE_LOW>;
+			label = "TL2";
+			linux,code = <BTN_TL2>;
+		};
+
+		button-left {
+			gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_LOW>;
+			label = "DPAD-LEFT";
+			linux,code = <BTN_DPAD_LEFT>;
+		};
+
+		button-r1 {
+			gpios = <&gpio3 RK_PB3 GPIO_ACTIVE_LOW>;
+			label = "TR";
+			linux,code = <BTN_TR>;
+		};
+
+		button-r2 {
+			gpios = <&gpio3 RK_PB4 GPIO_ACTIVE_LOW>;
+			label = "TR2";
+			linux,code = <BTN_TR2>;
+		};
+
+		button-right {
+			gpios = <&gpio3 RK_PA5 GPIO_ACTIVE_LOW>;
+			label = "DPAD-RIGHT";
+			linux,code = <BTN_DPAD_RIGHT>;
+		};
+
+		button-select {
+			gpios = <&gpio3 RK_PB6 GPIO_ACTIVE_LOW>;
+			label = "SELECT";
+			linux,code = <BTN_SELECT>;
+		};
+
+		button-start {
+			gpios = <&gpio3 RK_PB5 GPIO_ACTIVE_LOW>;
+			label = "START";
+			linux,code = <BTN_START>;
+		};
+
+		button-thumbl {
+			gpios = <&gpio3 RK_PA1 GPIO_ACTIVE_LOW>;
+			label = "THUMBL";
+			linux,code = <BTN_THUMBL>;
+		};
+
+		button-thumbr {
+			gpios = <&gpio3 RK_PA2 GPIO_ACTIVE_LOW>;
+			label = "THUMBR";
+			linux,code = <BTN_THUMBR>;
+		};
+
+		button-up {
+			gpios = <&gpio3 RK_PA3 GPIO_ACTIVE_LOW>;
+			label = "DPAD-UP";
+			linux,code = <BTN_DPAD_UP>;
+		};
+
+		button-x {
+			gpios = <&gpio3 RK_PC0 GPIO_ACTIVE_LOW>;
+			label = "NORTH";
+			linux,code = <BTN_NORTH>;
+		};
+
+		button-y {
+			gpios = <&gpio3 RK_PC1 GPIO_ACTIVE_LOW>;
+			label = "WEST";
+			linux,code = <BTN_WEST>;
+		};
+	};
+
+	gpio_keys_vol: gpio-keys-vol {
+		compatible = "gpio-keys";
+		autorepeat;
+		pinctrl-0 = <&btn_pins_vol>;
+		pinctrl-names = "default";
+
+		button-vol-down {
+			gpios = <&gpio3 RK_PB0 GPIO_ACTIVE_LOW>;
+			label = "VOLUMEDOWN";
+			linux,code = <KEY_VOLUMEDOWN>;
+		};
+
+		button-vol-up {
+			gpios = <&gpio3 RK_PA7 GPIO_ACTIVE_LOW>;
+			label = "VOLUMEUP";
+			linux,code = <KEY_VOLUMEUP>;
+		};
+	};
+
+	gpio_mux: mux-controller {
+		compatible = "gpio-mux";
+		mux-gpios = <&gpio0 RK_PB6 GPIO_ACTIVE_LOW>,
+			    <&gpio0 RK_PB7 GPIO_ACTIVE_LOW>;
+		#mux-control-cells = <0>;
+	};
+
+	hdmi-con {
+		compatible = "hdmi-connector";
+		ddc-i2c-bus = <&i2c5>;
+		type = "c";
+
+		port {
+			hdmi_con_in: endpoint {
+				remote-endpoint = <&hdmi_out_con>;
+			};
+		};
+	};
+
+	/*
+	 * Device also includes an always on LED that is wired to the 5V input
+	 * voltage and is on when the device is plugged in.
+	 */
+	leds: pwm-leds {
+		compatible = "pwm-leds";
+
+		green_led: led-0 {
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_STATUS;
+			max-brightness = <255>;
+			pwms = <&pwm6 0 25000 0>;
+		};
+
+		red_led: led-1 {
+			color = <LED_COLOR_ID_RED>;
+			function = LED_FUNCTION_CHARGING;
+			max-brightness = <255>;
+			pwms = <&pwm7 0 25000 0>;
+		};
+	};
+
+	sdio_pwrseq: sdio-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		clocks = <&rk817 1>;
+		clock-names = "ext_clock";
+		pinctrl-0 = <&wifi_enable_h>;
+		pinctrl-names = "default";
+		post-power-on-delay-ms = <200>;
+		reset-gpios = <&gpio4 RK_PA2 GPIO_ACTIVE_LOW>;
+	};
+
+	/*
+	 * Channels reversed for speakers. Headphones automatically switch via hardware when
+	 * detected with no ability to control output in software. Headphones appear to be mono
+	 * (each output channel receives all audio). No microphone support on 3.5mm jack.
+	 */
+	sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "rk817_ext";
+		simple-audio-card,format = "i2s";
+		simple-audio-card,mclk-fs = <256>;
+		simple-audio-card,widgets =
+			"Headphone", "Headphones";
+		simple-audio-card,routing =
+			"Headphones", "HPOL",
+			"Headphones", "HPOR";
+
+		simple-audio-card,codec {
+			sound-dai = <&rk817>;
+		};
+
+		simple-audio-card,cpu {
+			sound-dai = <&i2s1_8ch>;
+		};
+	};
+
+	vcc3v3_lcd0_n: regulator-vcc3v3-lcd0 {
+		compatible = "regulator-fixed";
+		gpio = <&gpio0 RK_PC2 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		pinctrl-0 = <&vcc_lcd_h>;
+		pinctrl-names = "default";
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-name = "vcc3v3_lcd0_n";
+		vin-supply = <&vcc_3v3>;
+		regulator-state-mem {
+			regulator-off-in-suspend;
+		};
+	};
+
+	vcc_sys: regulator-vcc-sys {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3800000>;
+		regulator-max-microvolt = <3800000>;
+		regulator-name = "vcc_sys";
+	};
+
+	vcc_wifi: regulator-vcc-wifi {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&vcc_wifi_h>;
+		pinctrl-names = "default";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-name = "vcc_wifi";
+	};
+};
+
+&combphy1 {
+	status = "okay";
+};
+
+&cpu0 {
+	cpu-supply = <&vdd_cpu>;
+};
+
+&cpu1 {
+	cpu-supply = <&vdd_cpu>;
+};
+
+&cpu2 {
+	cpu-supply = <&vdd_cpu>;
+};
+
+&cpu3 {
+	cpu-supply = <&vdd_cpu>;
+};
+
+&dsi0 {
+	status = "okay";
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	ports {
+		dsi0_in: port@0 {
+			reg = <0>;
+			dsi0_in_vp1: endpoint {
+				remote-endpoint = <&vp1_out_dsi0>;
+			};
+		};
+
+		dsi0_out: port@1 {
+			reg = <1>;
+			mipi_out_panel: endpoint {
+				remote-endpoint = <&mipi_in_panel>;
+			};
+		};
+	};
+};
+
+&dsi_dphy0 {
+	status = "okay";
+};
+
+&gpu {
+	mali-supply = <&vdd_gpu>;
+	status = "okay";
+};
+
+&hdmi {
+	ddc-i2c-bus = <&i2c5>;
+	pinctrl-0 = <&hdmitxm0_cec>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&hdmi_in {
+	hdmi_in_vp0: endpoint {
+		remote-endpoint = <&vp0_out_hdmi>;
+	};
+};
+
+&hdmi_out {
+	hdmi_out_con: endpoint {
+		remote-endpoint = <&hdmi_con_in>;
+	};
+};
+
+&hdmi_sound {
+	status = "okay";
+};
+
+&i2c0 {
+	status = "okay";
+
+	rk817: pmic@20 {
+		compatible = "rockchip,rk817";
+		reg = <0x20>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PA3 IRQ_TYPE_LEVEL_LOW>;
+		clock-output-names = "rk808-clkout1", "rk808-clkout2";
+		clock-names = "mclk";
+		clocks = <&cru I2S1_MCLKOUT_TX>;
+		assigned-clocks = <&cru I2S1_MCLKOUT_TX>;
+		assigned-clock-parents = <&cru CLK_I2S1_8CH_TX>;
+		#clock-cells = <1>;
+		#sound-dai-cells = <0>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&i2s1m0_mclk>, <&pmic_int_l>;
+		wakeup-source;
+
+		vcc1-supply = <&vcc_sys>;
+		vcc2-supply = <&vcc_sys>;
+		vcc3-supply = <&vcc_sys>;
+		vcc4-supply = <&vcc_sys>;
+		vcc5-supply = <&vcc_sys>;
+		vcc6-supply = <&vcc_sys>;
+		vcc7-supply = <&vcc_sys>;
+		vcc8-supply = <&vcc_sys>;
+		vcc9-supply = <&dcdc_boost>;
+
+		regulators {
+			vdd_logic: DCDC_REG1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-ramp-delay = <6001>;
+				regulator-initial-mode = <0x2>;
+				regulator-name = "vdd_logic";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+					regulator-suspend-microvolt = <900000>;
+				};
+			};
+
+			vdd_gpu: DCDC_REG2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-ramp-delay = <6001>;
+				regulator-initial-mode = <0x2>;
+				regulator-name = "vdd_gpu";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_ddr: DCDC_REG3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-initial-mode = <0x2>;
+				regulator-name = "vcc_ddr";
+				regulator-state-mem {
+					regulator-on-in-suspend;
+				};
+			};
+
+			vcc_3v3: DCDC_REG4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-initial-mode = <0x2>;
+				regulator-name = "vcc_3v3";
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <3300000>;
+				};
+			};
+
+			vcca1v8_pmu: LDO_REG1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcca1v8_pmu";
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+
+			vdda_0v9: LDO_REG2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-name = "vdda_0v9";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdda0v9_pmu: LDO_REG3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-name = "vdda0v9_pmu";
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <900000>;
+				};
+			};
+
+			vccio_acodec: LDO_REG4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vccio_acodec";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vccio_sd: LDO_REG5 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vccio_sd";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc3v3_pmu: LDO_REG6 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcc3v3_pmu";
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <3300000>;
+				};
+			};
+
+			vcc_1v8: LDO_REG7 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcc_1v8";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc1v8_dvp: LDO_REG8 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcc1v8_dvp";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc2v8_dvp: LDO_REG9 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2800000>;
+				regulator-name = "vcc2v8_dvp";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			dcdc_boost: BOOST {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <4700000>;
+				regulator-max-microvolt = <5400000>;
+				regulator-name = "boost";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			otg_switch: OTG_SWITCH {
+				regulator-name = "otg_switch";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+		};
+
+		rk817_charger: charger {
+			monitored-battery = <&battery>;
+			rockchip,resistor-sense-micro-ohms = <10000>;
+			rockchip,sleep-enter-current-microamp = <300000>;
+			rockchip,sleep-filter-current-microamp = <100000>;
+		};
+	};
+
+	vdd_cpu: regulator@1c {
+		compatible = "tcs,tcs4525";
+		reg = <0x1c>;
+		fcs,suspend-voltage-selector = <1>;
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <712500>;
+		regulator-max-microvolt = <1390000>;
+		regulator-name = "vdd_cpu";
+		regulator-ramp-delay = <2300>;
+		vin-supply = <&vcc_sys>;
+		regulator-state-mem {
+			regulator-off-in-suspend;
+		};
+	};
+};
+
+&i2c5 {
+	pinctrl-0 = <&i2c5m1_xfer>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&i2s0_8ch {
+	status = "okay";
+};
+
+&i2s1_8ch {
+	pinctrl-0 = <&i2s1m0_sclktx
+		     &i2s1m0_lrcktx
+		     &i2s1m0_sdi0
+		     &i2s1m0_sdo0>;
+	pinctrl-names = "default";
+	rockchip,trcm-sync-tx-only;
+	status = "okay";
+};
+
+&pinctrl {
+	gpio-btns {
+		btn_pins_ctrl: btn-pins-ctrl {
+			rockchip,pins =
+				<3 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PB3 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PB4 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PB6 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PC0 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PC3 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+
+		btn_pins_vol: btn-pins-vol {
+			rockchip,pins =
+				<3 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
+	joy-mux {
+		joy_mux_en: joy-mux-en {
+			rockchip,pins =
+				<0 RK_PB5 RK_FUNC_GPIO &pcfg_output_low>;
+		};
+	};
+
+	gpio-lcd {
+		lcd_rst: lcd-rst {
+			rockchip,pins =
+				<4 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	pmic {
+		pmic_int_l: pmic-int-l {
+			rockchip,pins =
+				<0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
+	sdio-pwrseq {
+		wifi_enable_h: wifi-enable-h {
+			rockchip,pins =
+				<4 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	vcc3v3-lcd {
+		vcc_lcd_h: vcc-lcd-h {
+			rockchip,pins =
+				<0 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	vcc-wifi {
+		vcc_wifi_h: vcc-wifi-h {
+			rockchip,pins =
+				<0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+};
+
+&pmu_io_domains {
+	status = "okay";
+	pmuio1-supply = <&vcc3v3_pmu>;
+	pmuio2-supply = <&vcc3v3_pmu>;
+	vccio1-supply = <&vccio_acodec>;
+	vccio3-supply = <&vccio_sd>;
+	vccio4-supply = <&vcc_1v8>;
+	vccio5-supply = <&vcc_3v3>;
+	vccio6-supply = <&vcc1v8_dvp>;
+	vccio7-supply = <&vcc_3v3>;
+};
+
+&pwm4 {
+	status = "okay";
+};
+
+&pwm6 {
+	status = "okay";
+};
+
+&pwm7 {
+	status = "okay";
+};
+
+&saradc {
+	vref-supply = <&vcc_1v8>;
+	status = "okay";
+};
+
+&sdmmc0 {
+	bus-width = <4>;
+	cap-sd-highspeed;
+	cd-gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_LOW>;
+	disable-wp;
+	pinctrl-0 = <&sdmmc0_bus4 &sdmmc0_clk &sdmmc0_cmd &sdmmc0_det>;
+	pinctrl-names = "default";
+	sd-uhs-sdr104;
+	vmmc-supply = <&vcc_3v3>;
+	vqmmc-supply = <&vccio_sd>;
+	status = "okay";
+};
+
+&sdmmc1 {
+	bus-width = <4>;
+	cap-sd-highspeed;
+	cd-gpios = <&gpio2 RK_PB2 GPIO_ACTIVE_LOW>;
+	disable-wp;
+	pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk &sdmmc1_det>;
+	pinctrl-names = "default";
+	sd-uhs-sdr104;
+	vmmc-supply = <&vcc_3v3>;
+	vqmmc-supply = <&vcc1v8_dvp>;
+	status = "okay";
+};
+
+&sdmmc2 {
+	bus-width = <4>;
+	cap-sd-highspeed;
+	cap-sdio-irq;
+	keep-power-in-suspend;
+	mmc-pwrseq = <&sdio_pwrseq>;
+	non-removable;
+	pinctrl-0 = <&sdmmc2m0_bus4 &sdmmc2m0_cmd &sdmmc2m0_clk>;
+	pinctrl-names = "default";
+	vmmc-supply = <&vcc_wifi>;
+	vqmmc-supply = <&vcca1v8_pmu>;
+	status = "okay";
+};
+
+&tsadc {
+	rockchip,hw-tshut-mode = <1>;
+	rockchip,hw-tshut-polarity = <0>;
+	status = "okay";
+};
+
+&uart1 {
+	pinctrl-0 = <&uart1m1_xfer &uart1m1_ctsn &uart1m1_rtsn>;
+	pinctrl-names = "default";
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		compatible = "realtek,rtl8821cs-bt", "realtek,rtl8723bs-bt";
+		device-wake-gpios = <&gpio4 4 GPIO_ACTIVE_HIGH>;
+		enable-gpios = <&gpio4 3 GPIO_ACTIVE_HIGH>;
+		host-wake-gpios = <&gpio4 5 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&usb_host0_xhci {
+	dr_mode = "peripheral";
+	phys = <&usb2phy0_otg>;
+	phy-names = "usb2-phy";
+	status = "okay";
+};
+
+&usb_host1_ehci {
+	status = "okay";
+};
+
+&usb_host1_ohci {
+	status = "okay";
+};
+
+&usb_host1_xhci {
+	phy-names = "usb2-phy", "usb3-phy";
+	phys = <&usb2phy1_host>, <&combphy1 PHY_TYPE_USB3>;
+	status = "okay";
+};
+
+&usb2phy0 {
+	status = "okay";
+};
+
+&usb2phy0_otg {
+	status = "okay";
+};
+
+&usb2phy1 {
+	status = "okay";
+};
+
+&usb2phy1_host {
+	status = "okay";
+};
+
+&vop {
+	assigned-clocks = <&cru DCLK_VOP0>, <&cru DCLK_VOP1>;
+	assigned-clock-parents = <&pmucru PLL_HPLL>, <&cru PLL_VPLL>;
+	status = "okay";
+};
+
+&vop_mmu {
+	status = "okay";
+};
+
+&vp0 {
+	vp0_out_hdmi: endpoint@ROCKCHIP_VOP2_EP_HDMI0 {
+		reg = <ROCKCHIP_VOP2_EP_HDMI0>;
+		remote-endpoint = <&hdmi_in_vp0>;
+	};
+};
+
+&vp1 {
+	vp1_out_dsi0: endpoint@ROCKCHIP_VOP2_EP_MIPI0 {
+		reg = <ROCKCHIP_VOP2_EP_MIPI0>;
+		remote-endpoint = <&dsi0_in_vp1>;
+	};
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk356x.dtsi b/arch/arm64/boot/dts/rockchip/rk356x.dtsi
index 0964761e3ce9..c19c0f1b3778 100644
--- a/arch/arm64/boot/dts/rockchip/rk356x.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk356x.dtsi
@@ -977,7 +977,7 @@ pcie2x1: pcie@fe260000 {
 			     <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>;
-		interrupt-names = "sys", "pmc", "msi", "legacy", "err";
+		interrupt-names = "sys", "pmc", "msg", "legacy", "err";
 		bus-range = <0x0 0xf>;
 		clocks = <&cru ACLK_PCIE20_MST>, <&cru ACLK_PCIE20_SLV>,
 			 <&cru ACLK_PCIE20_DBI>, <&cru PCLK_PCIE20>,
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-evb1-v10.dts b/arch/arm64/boot/dts/rockchip/rk3588-evb1-v10.dts
index b9d789d57862..3369065f897d 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-evb1-v10.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-evb1-v10.dts
@@ -56,6 +56,63 @@ button-escape {
 		};
 	};
 
+	analog-sound {
+		compatible = "simple-audio-card";
+		pinctrl-names = "default";
+		pinctrl-0 = <&hp_detect>;
+		simple-audio-card,name = "RK3588 EVB1 Audio";
+		simple-audio-card,aux-devs = <&amp_headphone>, <&amp_speaker>;
+		simple-audio-card,bitclock-master = <&masterdai>;
+		simple-audio-card,format = "i2s";
+		simple-audio-card,frame-master = <&masterdai>;
+		simple-audio-card,hp-det-gpio = <&gpio1 RK_PD5 GPIO_ACTIVE_LOW>;
+		simple-audio-card,mclk-fs = <256>;
+		simple-audio-card,pin-switches = "Headphones", "Speaker";
+		simple-audio-card,routing =
+			"Speaker Amplifier INL", "LOUT2",
+			"Speaker Amplifier INR", "ROUT2",
+			"Speaker", "Speaker Amplifier OUTL",
+			"Speaker", "Speaker Amplifier OUTR",
+			"Headphones Amplifier INL", "LOUT1",
+			"Headphones Amplifier INR", "ROUT1",
+			"Headphones", "Headphones Amplifier OUTL",
+			"Headphones", "Headphones Amplifier OUTR",
+			"LINPUT1", "Onboard Microphone",
+			"RINPUT1", "Onboard Microphone",
+			"LINPUT2", "Microphone Jack",
+			"RINPUT2", "Microphone Jack";
+		simple-audio-card,widgets =
+			"Microphone", "Microphone Jack",
+			"Microphone", "Onboard Microphone",
+			"Headphone", "Headphones",
+			"Speaker", "Speaker";
+
+		simple-audio-card,cpu {
+			sound-dai = <&i2s0_8ch>;
+		};
+
+		masterdai: simple-audio-card,codec {
+			sound-dai = <&es8388>;
+			system-clock-frequency = <12288000>;
+		};
+	};
+
+	amp_headphone: headphone-amplifier {
+		compatible = "simple-audio-amplifier";
+		enable-gpios = <&gpio1 RK_PD2 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&headphone_amplifier_en>;
+		sound-name-prefix = "Headphones Amplifier";
+	};
+
+	amp_speaker: speaker-amplifier {
+		compatible = "simple-audio-amplifier";
+		enable-gpios = <&gpio1 RK_PD3 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&speaker_amplifier_en>;
+		sound-name-prefix = "Speaker Amplifier";
+	};
+
 	backlight: backlight {
 		compatible = "pwm-backlight";
 		power-supply = <&vcc12v_dcin>;
@@ -240,6 +297,32 @@ hym8563: rtc@51 {
 	};
 };
 
+&i2c7 {
+	status = "okay";
+
+	es8388: audio-codec@11 {
+		compatible = "everest,es8388";
+		reg = <0x11>;
+		clocks = <&cru I2S0_8CH_MCLKOUT>;
+		assigned-clocks = <&cru I2S0_8CH_MCLKOUT>;
+		assigned-clock-rates = <12288000>;
+		AVDD-supply = <&avcc_1v8_codec_s0>;
+		DVDD-supply = <&avcc_1v8_codec_s0>;
+		HPVDD-supply = <&vcc_3v3_s0>;
+		PVDD-supply = <&vcc_3v3_s0>;
+		#sound-dai-cells = <0>;
+	};
+};
+
+&i2s0_8ch {
+	pinctrl-0 = <&i2s0_lrck
+		     &i2s0_mclk
+		     &i2s0_sclk
+		     &i2s0_sdi0
+		     &i2s0_sdo0>;
+	status = "okay";
+};
+
 &mdio0 {
 	rgmii_phy: ethernet-phy@1 {
 		/* RTL8211F */
@@ -273,6 +356,20 @@ &pcie3x4 {
 };
 
 &pinctrl {
+	audio {
+		hp_detect: headphone-detect {
+			rockchip,pins = <1 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		headphone_amplifier_en: headphone-amplifier-en {
+			rockchip,pins = <1 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		speaker_amplifier_en: speaker-amplifier-en {
+			rockchip,pins = <1 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	rtl8111 {
 		rtl8111_isolate: rtl8111-isolate {
 			rockchip,pins = <1 RK_PA4 RK_FUNC_GPIO &pcfg_pull_up>;
diff --git a/arch/arm64/boot/dts/xilinx/Makefile b/arch/arm64/boot/dts/xilinx/Makefile
index 5e40c0b4fa0a..1068b0fa8e98 100644
--- a/arch/arm64/boot/dts/xilinx/Makefile
+++ b/arch/arm64/boot/dts/xilinx/Makefile
@@ -22,11 +22,10 @@ dtb-$(CONFIG_ARCH_ZYNQMP) += zynqmp-sm-k26-revA.dtb
 dtb-$(CONFIG_ARCH_ZYNQMP) += zynqmp-smk-k26-revA.dtb
 
 zynqmp-sm-k26-revA-sck-kv-g-revA-dtbs := zynqmp-sm-k26-revA.dtb zynqmp-sck-kv-g-revA.dtbo
+dtb-$(CONFIG_ARCH_ZYNQMP) += zynqmp-sm-k26-revA-sck-kv-g-revA.dtb
 zynqmp-sm-k26-revA-sck-kv-g-revB-dtbs := zynqmp-sm-k26-revA.dtb zynqmp-sck-kv-g-revB.dtbo
+dtb-$(CONFIG_ARCH_ZYNQMP) += zynqmp-sm-k26-revA-sck-kv-g-revB.dtb
 zynqmp-smk-k26-revA-sck-kv-g-revA-dtbs := zynqmp-smk-k26-revA.dtb zynqmp-sck-kv-g-revA.dtbo
+dtb-$(CONFIG_ARCH_ZYNQMP) += zynqmp-smk-k26-revA-sck-kv-g-revA.dtb
 zynqmp-smk-k26-revA-sck-kv-g-revB-dtbs := zynqmp-smk-k26-revA.dtb zynqmp-sck-kv-g-revB.dtbo
-
-zynqmp-sm-k26-revA-sck-kr-g-revA-dtbs := zynqmp-sm-k26-revA.dtb zynqmp-sck-kr-g-revA.dtbo
-zynqmp-sm-k26-revA-sck-kr-g-revB-dtbs := zynqmp-sm-k26-revA.dtb zynqmp-sck-kr-g-revB.dtbo
-zynqmp-smk-k26-revA-sck-kr-g-revA-dtbs := zynqmp-smk-k26-revA.dtb zynqmp-sck-kr-g-revA.dtbo
-zynqmp-smk-k26-revA-sck-kr-g-revB-dtbs := zynqmp-smk-k26-revA.dtb zynqmp-sck-kr-g-revB.dtbo
+dtb-$(CONFIG_ARCH_ZYNQMP) += zynqmp-smk-k26-revA-sck-kv-g-revB.dtb
diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-sm-k26-revA.dts b/arch/arm64/boot/dts/xilinx/zynqmp-sm-k26-revA.dts
index c4774a42d5fc..51622896b1b1 100644
--- a/arch/arm64/boot/dts/xilinx/zynqmp-sm-k26-revA.dts
+++ b/arch/arm64/boot/dts/xilinx/zynqmp-sm-k26-revA.dts
@@ -131,9 +131,7 @@ mux {
 &qspi { /* MIO 0-5 - U143 */
 	status = "okay";
 	spi_flash: flash@0 { /* MT25QU512A */
-		compatible = "mt25qu512a", "jedec,spi-nor"; /* 64MB */
-		#address-cells = <1>;
-		#size-cells = <1>;
+		compatible = "jedec,spi-nor"; /* 64MB */
 		reg = <0>;
 		spi-tx-bus-width = <4>;
 		spi-rx-bus-width = <4>;
@@ -222,9 +220,9 @@ partition@2280000 {
 				label = "Secure OS Storage";
 				reg = <0x2280000 0x20000>; /* 128KB */
 			};
-			partition@22A0000 {
+			partition@22a0000 {
 				label = "User";
-				reg = <0x22A0000 0x1d60000>; /* 29.375 MB */
+				reg = <0x22a0000 0x1d60000>; /* 29.375 MB */
 			};
 		};
 	};
diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm015-dc1.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm015-dc1.dts
index e821d55d8d5a..73491626e01e 100644
--- a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm015-dc1.dts
+++ b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm015-dc1.dts
@@ -98,8 +98,12 @@ &gem3 {
 	phy-mode = "rgmii-id";
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_gem3_default>;
-	phy0: ethernet-phy@0 {
-		reg = <0>;
+	mdio: mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		phy0: ethernet-phy@0 {
+			reg = <0>;
+		};
 	};
 };
 
diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm016-dc2.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm016-dc2.dts
index b59e11316b4b..f767708fb50d 100644
--- a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm016-dc2.dts
+++ b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm016-dc2.dts
@@ -91,12 +91,16 @@ &gem2 {
 	phy-mode = "rgmii-id";
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_gem2_default>;
-	phy0: ethernet-phy@5 {
-		reg = <5>;
-		ti,rx-internal-delay = <0x8>;
-		ti,tx-internal-delay = <0xa>;
-		ti,fifo-depth = <0x1>;
-		ti,dp83867-rxctrl-strap-quirk;
+	mdio: mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		phy0: ethernet-phy@5 {
+			reg = <5>;
+			ti,rx-internal-delay = <0x8>;
+			ti,tx-internal-delay = <0xa>;
+			ti,fifo-depth = <0x1>;
+			ti,dp83867-rxctrl-strap-quirk;
+		};
 	};
 };
 
diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm017-dc3.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm017-dc3.dts
index 38b0a312171b..f553b317e6b2 100644
--- a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm017-dc3.dts
+++ b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm017-dc3.dts
@@ -88,8 +88,12 @@ &gem0 {
 	status = "okay";
 	phy-handle = <&phy0>;
 	phy-mode = "rgmii-id";
-	phy0: ethernet-phy@0 { /* VSC8211 */
-		reg = <0>;
+	mdio: mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		phy0: ethernet-phy@0 { /* VSC8211 */
+			reg = <0>;
+		};
 	};
 };
 
diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm018-dc4.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm018-dc4.dts
index 6636e76545a5..6ec1d9813973 100644
--- a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm018-dc4.dts
+++ b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm018-dc4.dts
@@ -116,17 +116,21 @@ &gem0 {
 	status = "okay";
 	phy-mode = "rgmii-id";
 	phy-handle = <&ethernet_phy0>;
-	ethernet_phy0: ethernet-phy@0 { /* Marvell 88e1512 */
-		reg = <0>;
-	};
-	ethernet_phy7: ethernet-phy@7 { /* Vitesse VSC8211 */
-		reg = <7>;
-	};
-	ethernet_phy3: ethernet-phy@3 { /* Realtek RTL8211DN */
-		reg = <3>;
-	};
-	ethernet_phy8: ethernet-phy@8 { /* Vitesse VSC8211 */
-		reg = <8>;
+	mdio: mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		ethernet_phy0: ethernet-phy@0 { /* Marvell 88e1512 */
+			reg = <0>;
+		};
+		ethernet_phy7: ethernet-phy@7 { /* Vitesse VSC8211 */
+			reg = <7>;
+		};
+		ethernet_phy3: ethernet-phy@3 { /* Realtek RTL8211DN */
+			reg = <3>;
+		};
+		ethernet_phy8: ethernet-phy@8 { /* Vitesse VSC8211 */
+			reg = <8>;
+		};
 	};
 };
 
diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm019-dc5.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm019-dc5.dts
index 0d2ea9c09a0a..b1857e17ab7e 100644
--- a/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm019-dc5.dts
+++ b/arch/arm64/boot/dts/xilinx/zynqmp-zc1751-xm019-dc5.dts
@@ -77,8 +77,12 @@ &gem1 {
 	phy-mode = "rgmii-id";
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_gem1_default>;
-	phy0: ethernet-phy@0 {
-		reg = <0>;
+	mdio: mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		phy0: ethernet-phy@0 {
+			reg = <0>;
+		};
 	};
 };
 
diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-zcu100-revC.dts b/arch/arm64/boot/dts/xilinx/zynqmp-zcu100-revC.dts
index d0091d3cb764..52f998c22538 100644
--- a/arch/arm64/boot/dts/xilinx/zynqmp-zcu100-revC.dts
+++ b/arch/arm64/boot/dts/xilinx/zynqmp-zcu100-revC.dts
@@ -123,13 +123,13 @@ ina226 {
 		io-channels = <&u35 0>, <&u35 1>, <&u35 2>, <&u35 3>;
 	};
 
-	si5335_0: si5335_0 { /* clk0_usb - u23 */
+	si5335_0: si5335-0 { /* clk0_usb - u23 */
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <26000000>;
 	};
 
-	si5335_1: si5335_1 { /* clk1_dp - u23 */
+	si5335_1: si5335-1 { /* clk1_dp - u23 */
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <27000000>;
diff --git a/arch/arm64/boot/dts/xilinx/zynqmp.dtsi b/arch/arm64/boot/dts/xilinx/zynqmp.dtsi
index b61fc99cd911..f5e1eb8cb3b7 100644
--- a/arch/arm64/boot/dts/xilinx/zynqmp.dtsi
+++ b/arch/arm64/boot/dts/xilinx/zynqmp.dtsi
@@ -129,7 +129,7 @@ rproc_1_fw_image: memory@3ef00000 {
 		};
 	};
 
-	zynqmp_ipi: zynqmp_ipi {
+	zynqmp_ipi: zynqmp-ipi {
 		bootph-all;
 		compatible = "xlnx,zynqmp-ipi-mailbox";
 		interrupt-parent = <&gic>;
@@ -194,12 +194,12 @@ zynqmp_power: zynqmp-power {
 				mbox-names = "tx", "rx";
 			};
 
-			nvmem_firmware {
+			nvmem-firmware {
 				compatible = "xlnx,zynqmp-nvmem-fw";
 				#address-cells = <1>;
 				#size-cells = <1>;
 
-				soc_revision: soc_revision@0 {
+				soc_revision: soc-revision@0 {
 					reg = <0x0 0x4>;
 				};
 			};
@@ -584,8 +584,6 @@ gem0: ethernet@ff0b0000 {
 				     <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>;
 			reg = <0x0 0xff0b0000 0x0 0x1000>;
 			clock-names = "pclk", "hclk", "tx_clk", "rx_clk", "tsu_clk";
-			#address-cells = <1>;
-			#size-cells = <0>;
 			iommus = <&smmu 0x874>;
 			power-domains = <&zynqmp_firmware PD_ETH_0>;
 			resets = <&zynqmp_reset ZYNQMP_RESET_GEM0>;
@@ -600,8 +598,6 @@ gem1: ethernet@ff0c0000 {
 				     <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>;
 			reg = <0x0 0xff0c0000 0x0 0x1000>;
 			clock-names = "pclk", "hclk", "tx_clk", "rx_clk", "tsu_clk";
-			#address-cells = <1>;
-			#size-cells = <0>;
 			iommus = <&smmu 0x875>;
 			power-domains = <&zynqmp_firmware PD_ETH_1>;
 			resets = <&zynqmp_reset ZYNQMP_RESET_GEM1>;
@@ -616,8 +612,6 @@ gem2: ethernet@ff0d0000 {
 				     <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>;
 			reg = <0x0 0xff0d0000 0x0 0x1000>;
 			clock-names = "pclk", "hclk", "tx_clk", "rx_clk", "tsu_clk";
-			#address-cells = <1>;
-			#size-cells = <0>;
 			iommus = <&smmu 0x876>;
 			power-domains = <&zynqmp_firmware PD_ETH_2>;
 			resets = <&zynqmp_reset ZYNQMP_RESET_GEM2>;
@@ -632,8 +626,6 @@ gem3: ethernet@ff0e0000 {
 				     <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>;
 			reg = <0x0 0xff0e0000 0x0 0x1000>;
 			clock-names = "pclk", "hclk", "tx_clk", "rx_clk", "tsu_clk";
-			#address-cells = <1>;
-			#size-cells = <0>;
 			iommus = <&smmu 0x877>;
 			power-domains = <&zynqmp_firmware PD_ETH_3>;
 			resets = <&zynqmp_reset ZYNQMP_RESET_GEM3>;
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 824f29f04916..567bbc417e4b 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -954,8 +954,6 @@ int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
 int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
 			      struct kvm_vcpu_events *events);
 
-#define KVM_ARCH_WANT_MMU_NOTIFIER
-
 void kvm_arm_halt_guest(struct kvm *kvm);
 void kvm_arm_resume_guest(struct kvm *kvm);
 
diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h
index 531effca5f1f..491b2b9bd553 100644
--- a/arch/arm64/include/asm/unistd.h
+++ b/arch/arm64/include/asm/unistd.h
@@ -39,7 +39,7 @@
 #define __ARM_NR_compat_set_tls		(__ARM_NR_COMPAT_BASE + 5)
 #define __ARM_NR_COMPAT_END		(__ARM_NR_COMPAT_BASE + 0x800)
 
-#define __NR_compat_syscalls		457
+#define __NR_compat_syscalls		462
 #endif
 
 #define __ARCH_WANT_SYS_CLONE
diff --git a/arch/arm64/include/asm/unistd32.h b/arch/arm64/include/asm/unistd32.h
index 9f7c1bf99526..7118282d1c79 100644
--- a/arch/arm64/include/asm/unistd32.h
+++ b/arch/arm64/include/asm/unistd32.h
@@ -919,6 +919,16 @@ __SYSCALL(__NR_futex_wake, sys_futex_wake)
 __SYSCALL(__NR_futex_wait, sys_futex_wait)
 #define __NR_futex_requeue 456
 __SYSCALL(__NR_futex_requeue, sys_futex_requeue)
+#define __NR_statmount 457
+__SYSCALL(__NR_statmount, sys_statmount)
+#define __NR_listmount 458
+__SYSCALL(__NR_listmount, sys_listmount)
+#define __NR_lsm_get_self_attr 459
+__SYSCALL(__NR_lsm_get_self_attr, sys_lsm_get_self_attr)
+#define __NR_lsm_set_self_attr 460
+__SYSCALL(__NR_lsm_set_self_attr, sys_lsm_set_self_attr)
+#define __NR_lsm_list_modules 461
+__SYSCALL(__NR_lsm_list_modules, sys_lsm_list_modules)
 
 /*
  * Please add new compat syscalls above this comment and update
diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig
index 83c1e09be42e..1a777715199f 100644
--- a/arch/arm64/kvm/Kconfig
+++ b/arch/arm64/kvm/Kconfig
@@ -22,7 +22,7 @@ menuconfig KVM
 	bool "Kernel-based Virtual Machine (KVM) support"
 	depends on HAVE_KVM
 	select KVM_GENERIC_HARDWARE_ENABLING
-	select MMU_NOTIFIER
+	select KVM_GENERIC_MMU_NOTIFIER
 	select PREEMPT_NOTIFIERS
 	select HAVE_KVM_CPU_RELAX_INTERCEPT
 	select KVM_MMIO
diff --git a/arch/loongarch/Makefile b/arch/loongarch/Makefile
index 9eeb0c05f3f4..204b94b2e6aa 100644
--- a/arch/loongarch/Makefile
+++ b/arch/loongarch/Makefile
@@ -68,6 +68,7 @@ LDFLAGS_vmlinux			+= -static -n -nostdlib
 ifdef CONFIG_AS_HAS_EXPLICIT_RELOCS
 cflags-y			+= $(call cc-option,-mexplicit-relocs)
 KBUILD_CFLAGS_KERNEL		+= $(call cc-option,-mdirect-extern-access)
+KBUILD_CFLAGS_KERNEL		+= $(call cc-option,-fdirect-access-external-data)
 KBUILD_AFLAGS_MODULE		+= $(call cc-option,-fno-direct-access-external-data)
 KBUILD_CFLAGS_MODULE		+= $(call cc-option,-fno-direct-access-external-data)
 KBUILD_AFLAGS_MODULE		+= $(call cc-option,-mno-relax) $(call cc-option,-Wa$(comma)-mno-relax)
@@ -142,6 +143,8 @@ vdso-install-y += arch/loongarch/vdso/vdso.so.dbg
 
 all:	$(notdir $(KBUILD_IMAGE))
 
+vmlinuz.efi: vmlinux.efi
+
 vmlinux.elf vmlinux.efi vmlinuz.efi: vmlinux
 	$(Q)$(MAKE) $(build)=$(boot) $(bootvars-y) $(boot)/$@
 
diff --git a/arch/loongarch/include/asm/asmmacro.h b/arch/loongarch/include/asm/asmmacro.h
index c9544f358c33..655db7d7a427 100644
--- a/arch/loongarch/include/asm/asmmacro.h
+++ b/arch/loongarch/include/asm/asmmacro.h
@@ -609,8 +609,7 @@
 	lu32i.d	\reg, 0
 	lu52i.d	\reg, \reg, 0
 	.pushsection ".la_abs", "aw", %progbits
-	768:
-	.dword	768b-766b
+	.dword	766b
 	.dword	\sym
 	.popsection
 #endif
diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h
index 11328700d4fa..b108301c2e5a 100644
--- a/arch/loongarch/include/asm/kvm_host.h
+++ b/arch/loongarch/include/asm/kvm_host.h
@@ -183,7 +183,6 @@ void kvm_flush_tlb_all(void);
 void kvm_flush_tlb_gpa(struct kvm_vcpu *vcpu, unsigned long gpa);
 int kvm_handle_mm_fault(struct kvm_vcpu *vcpu, unsigned long badv, bool write);
 
-#define KVM_ARCH_WANT_MMU_NOTIFIER
 void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
 int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end, bool blockable);
 int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
diff --git a/arch/loongarch/include/asm/percpu.h b/arch/loongarch/include/asm/percpu.h
index ed5da02b1cf6..9b36ac003f89 100644
--- a/arch/loongarch/include/asm/percpu.h
+++ b/arch/loongarch/include/asm/percpu.h
@@ -40,13 +40,13 @@ static __always_inline unsigned long __percpu_##op(void *ptr,		\
 	switch (size) {							\
 	case 4:								\
 		__asm__ __volatile__(					\
-		"am"#asm_op".w"	" %[ret], %[val], %[ptr]	\n"		\
+		"am"#asm_op".w"	" %[ret], %[val], %[ptr]	\n"	\
 		: [ret] "=&r" (ret), [ptr] "+ZB"(*(u32 *)ptr)		\
 		: [val] "r" (val));					\
 		break;							\
 	case 8:								\
 		__asm__ __volatile__(					\
-		"am"#asm_op".d" " %[ret], %[val], %[ptr]	\n"		\
+		"am"#asm_op".d" " %[ret], %[val], %[ptr]	\n"	\
 		: [ret] "=&r" (ret), [ptr] "+ZB"(*(u64 *)ptr)		\
 		: [val] "r" (val));					\
 		break;							\
@@ -63,7 +63,7 @@ PERCPU_OP(and, and, &)
 PERCPU_OP(or, or, |)
 #undef PERCPU_OP
 
-static __always_inline unsigned long __percpu_read(void *ptr, int size)
+static __always_inline unsigned long __percpu_read(void __percpu *ptr, int size)
 {
 	unsigned long ret;
 
@@ -100,7 +100,7 @@ static __always_inline unsigned long __percpu_read(void *ptr, int size)
 	return ret;
 }
 
-static __always_inline void __percpu_write(void *ptr, unsigned long val, int size)
+static __always_inline void __percpu_write(void __percpu *ptr, unsigned long val, int size)
 {
 	switch (size) {
 	case 1:
@@ -132,8 +132,7 @@ static __always_inline void __percpu_write(void *ptr, unsigned long val, int siz
 	}
 }
 
-static __always_inline unsigned long __percpu_xchg(void *ptr, unsigned long val,
-						   int size)
+static __always_inline unsigned long __percpu_xchg(void *ptr, unsigned long val, int size)
 {
 	switch (size) {
 	case 1:
diff --git a/arch/loongarch/include/asm/setup.h b/arch/loongarch/include/asm/setup.h
index a0bc159ce8bd..ee52fb1e9963 100644
--- a/arch/loongarch/include/asm/setup.h
+++ b/arch/loongarch/include/asm/setup.h
@@ -25,7 +25,7 @@ extern void set_merr_handler(unsigned long offset, void *addr, unsigned long len
 #ifdef CONFIG_RELOCATABLE
 
 struct rela_la_abs {
-	long offset;
+	long pc;
 	long symvalue;
 };
 
diff --git a/arch/loongarch/kernel/numa.c b/arch/loongarch/kernel/numa.c
index 6e65ff12d5c7..8fe21f868f72 100644
--- a/arch/loongarch/kernel/numa.c
+++ b/arch/loongarch/kernel/numa.c
@@ -226,32 +226,6 @@ static void __init node_mem_init(unsigned int node)
 
 #ifdef CONFIG_ACPI_NUMA
 
-/*
- * Sanity check to catch more bad NUMA configurations (they are amazingly
- * common).  Make sure the nodes cover all memory.
- */
-static bool __init numa_meminfo_cover_memory(const struct numa_meminfo *mi)
-{
-	int i;
-	u64 numaram, biosram;
-
-	numaram = 0;
-	for (i = 0; i < mi->nr_blks; i++) {
-		u64 s = mi->blk[i].start >> PAGE_SHIFT;
-		u64 e = mi->blk[i].end >> PAGE_SHIFT;
-
-		numaram += e - s;
-		numaram -= __absent_pages_in_range(mi->blk[i].nid, s, e);
-		if ((s64)numaram < 0)
-			numaram = 0;
-	}
-	max_pfn = max_low_pfn;
-	biosram = max_pfn - absent_pages_in_range(0, max_pfn);
-
-	BUG_ON((s64)(biosram - numaram) >= (1 << (20 - PAGE_SHIFT)));
-	return true;
-}
-
 static void __init add_node_intersection(u32 node, u64 start, u64 size, u32 type)
 {
 	static unsigned long num_physpages;
@@ -396,7 +370,7 @@ int __init init_numa_memory(void)
 		return -EINVAL;
 
 	init_node_memblock();
-	if (numa_meminfo_cover_memory(&numa_meminfo) == false)
+	if (!memblock_validate_numa_coverage(SZ_1M))
 		return -EINVAL;
 
 	for_each_node_mask(node, node_possible_map) {
diff --git a/arch/loongarch/kernel/relocate.c b/arch/loongarch/kernel/relocate.c
index 6c3eff9af9fb..1acfa704c8d0 100644
--- a/arch/loongarch/kernel/relocate.c
+++ b/arch/loongarch/kernel/relocate.c
@@ -52,7 +52,7 @@ static inline void __init relocate_absolute(long random_offset)
 	for (p = begin; (void *)p < end; p++) {
 		long v = p->symvalue;
 		uint32_t lu12iw, ori, lu32id, lu52id;
-		union loongarch_instruction *insn = (void *)p - p->offset;
+		union loongarch_instruction *insn = (void *)p->pc;
 
 		lu12iw = (v >> 12) & 0xfffff;
 		ori    = v & 0xfff;
@@ -102,6 +102,14 @@ static inline __init unsigned long get_random_boot(void)
 	return hash;
 }
 
+static int __init nokaslr(char *p)
+{
+	pr_info("KASLR is disabled.\n");
+
+	return 0; /* Print a notice and silence the boot warning */
+}
+early_param("nokaslr", nokaslr);
+
 static inline __init bool kaslr_disabled(void)
 {
 	char *str;
diff --git a/arch/loongarch/kernel/time.c b/arch/loongarch/kernel/time.c
index 3064af94db9c..e7015f7b70e3 100644
--- a/arch/loongarch/kernel/time.c
+++ b/arch/loongarch/kernel/time.c
@@ -58,14 +58,16 @@ static int constant_set_state_oneshot(struct clock_event_device *evt)
 	return 0;
 }
 
-static int constant_set_state_oneshot_stopped(struct clock_event_device *evt)
+static int constant_set_state_periodic(struct clock_event_device *evt)
 {
+	unsigned long period;
 	unsigned long timer_config;
 
 	raw_spin_lock(&state_lock);
 
-	timer_config = csr_read64(LOONGARCH_CSR_TCFG);
-	timer_config &= ~CSR_TCFG_EN;
+	period = const_clock_freq / HZ;
+	timer_config = period & CSR_TCFG_VAL;
+	timer_config |= (CSR_TCFG_PERIOD | CSR_TCFG_EN);
 	csr_write64(timer_config, LOONGARCH_CSR_TCFG);
 
 	raw_spin_unlock(&state_lock);
@@ -73,16 +75,14 @@ static int constant_set_state_oneshot_stopped(struct clock_event_device *evt)
 	return 0;
 }
 
-static int constant_set_state_periodic(struct clock_event_device *evt)
+static int constant_set_state_shutdown(struct clock_event_device *evt)
 {
-	unsigned long period;
 	unsigned long timer_config;
 
 	raw_spin_lock(&state_lock);
 
-	period = const_clock_freq / HZ;
-	timer_config = period & CSR_TCFG_VAL;
-	timer_config |= (CSR_TCFG_PERIOD | CSR_TCFG_EN);
+	timer_config = csr_read64(LOONGARCH_CSR_TCFG);
+	timer_config &= ~CSR_TCFG_EN;
 	csr_write64(timer_config, LOONGARCH_CSR_TCFG);
 
 	raw_spin_unlock(&state_lock);
@@ -90,11 +90,6 @@ static int constant_set_state_periodic(struct clock_event_device *evt)
 	return 0;
 }
 
-static int constant_set_state_shutdown(struct clock_event_device *evt)
-{
-	return 0;
-}
-
 static int constant_timer_next_event(unsigned long delta, struct clock_event_device *evt)
 {
 	unsigned long timer_config;
@@ -161,7 +156,7 @@ int constant_clockevent_init(void)
 	cd->rating = 320;
 	cd->cpumask = cpumask_of(cpu);
 	cd->set_state_oneshot = constant_set_state_oneshot;
-	cd->set_state_oneshot_stopped = constant_set_state_oneshot_stopped;
+	cd->set_state_oneshot_stopped = constant_set_state_shutdown;
 	cd->set_state_periodic = constant_set_state_periodic;
 	cd->set_state_shutdown = constant_set_state_shutdown;
 	cd->set_next_event = constant_timer_next_event;
diff --git a/arch/loongarch/kvm/Kconfig b/arch/loongarch/kvm/Kconfig
index fda425babfb2..f22bae89b07d 100644
--- a/arch/loongarch/kvm/Kconfig
+++ b/arch/loongarch/kvm/Kconfig
@@ -26,9 +26,9 @@ config KVM
 	select HAVE_KVM_VCPU_ASYNC_IOCTL
 	select KVM_GENERIC_DIRTYLOG_READ_PROTECT
 	select KVM_GENERIC_HARDWARE_ENABLING
+	select KVM_GENERIC_MMU_NOTIFIER
 	select KVM_MMIO
 	select KVM_XFER_TO_GUEST_WORK
-	select MMU_NOTIFIER
 	select PREEMPT_NOTIFIERS
 	help
 	  Support hosting virtualized guest machines using
diff --git a/arch/loongarch/mm/pgtable.c b/arch/loongarch/mm/pgtable.c
index 71d0539e2d0b..2aae72e63871 100644
--- a/arch/loongarch/mm/pgtable.c
+++ b/arch/loongarch/mm/pgtable.c
@@ -13,13 +13,13 @@ struct page *dmw_virt_to_page(unsigned long kaddr)
 {
 	return pfn_to_page(virt_to_pfn(kaddr));
 }
-EXPORT_SYMBOL_GPL(dmw_virt_to_page);
+EXPORT_SYMBOL(dmw_virt_to_page);
 
 struct page *tlb_virt_to_page(unsigned long kaddr)
 {
 	return pfn_to_page(pte_pfn(*virt_to_kpte(kaddr)));
 }
-EXPORT_SYMBOL_GPL(tlb_virt_to_page);
+EXPORT_SYMBOL(tlb_virt_to_page);
 
 pgd_t *pgd_alloc(struct mm_struct *mm)
 {
diff --git a/arch/m68k/kernel/syscalls/syscall.tbl b/arch/m68k/kernel/syscalls/syscall.tbl
index 7a4b780e82cb..7fd43fd4c9f2 100644
--- a/arch/m68k/kernel/syscalls/syscall.tbl
+++ b/arch/m68k/kernel/syscalls/syscall.tbl
@@ -456,3 +456,8 @@
 454	common	futex_wake			sys_futex_wake
 455	common	futex_wait			sys_futex_wait
 456	common	futex_requeue			sys_futex_requeue
+457	common	statmount			sys_statmount
+458	common	listmount			sys_listmount
+459	common	lsm_get_self_attr		sys_lsm_get_self_attr
+460	common	lsm_set_self_attr		sys_lsm_set_self_attr
+461	common	lsm_list_modules		sys_lsm_list_modules
diff --git a/arch/microblaze/configs/mmu_defconfig b/arch/microblaze/configs/mmu_defconfig
index 7b2d7f6f23c0..4da7bc4ac4a3 100644
--- a/arch/microblaze/configs/mmu_defconfig
+++ b/arch/microblaze/configs/mmu_defconfig
@@ -3,11 +3,9 @@ CONFIG_POSIX_MQUEUE=y
 CONFIG_AUDIT=y
 CONFIG_IKCONFIG=y
 CONFIG_IKCONFIG_PROC=y
-CONFIG_SYSFS_DEPRECATED=y
-CONFIG_SYSFS_DEPRECATED_V2=y
+CONFIG_EXPERT=y
 # CONFIG_BASE_FULL is not set
 CONFIG_KALLSYMS_ALL=y
-CONFIG_EXPERT=y
 CONFIG_XILINX_MICROBLAZE0_USE_MSR_INSTR=1
 CONFIG_XILINX_MICROBLAZE0_USE_PCMP_INSTR=1
 CONFIG_XILINX_MICROBLAZE0_USE_BARREL=1
@@ -20,7 +18,6 @@ CONFIG_CMDLINE_FORCE=y
 CONFIG_HIGHMEM=y
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
-# CONFIG_BLK_DEV_BSG is not set
 CONFIG_PARTITION_ADVANCED=y
 # CONFIG_EFI_PARTITION is not set
 CONFIG_CMA=y
@@ -28,6 +25,10 @@ CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
 CONFIG_INET=y
+CONFIG_IP_PNP=y
+CONFIG_IP_PNP_DHCP=y
+CONFIG_IP_PNP_BOOTP=y
+CONFIG_IP_PNP_RARP=y
 # CONFIG_IPV6 is not set
 CONFIG_BRIDGE=m
 CONFIG_PCI=y
@@ -43,6 +44,7 @@ CONFIG_NETDEVICES=y
 CONFIG_XILINX_EMACLITE=y
 CONFIG_XILINX_AXI_EMAC=y
 CONFIG_XILINX_LL_TEMAC=y
+CONFIG_MARVELL_PHY=y
 # CONFIG_INPUT is not set
 # CONFIG_SERIO is not set
 # CONFIG_VT is not set
@@ -77,14 +79,13 @@ CONFIG_TMPFS=y
 CONFIG_CRAMFS=y
 CONFIG_ROMFS_FS=y
 CONFIG_NFS_FS=y
+CONFIG_ROOT_NFS=y
 CONFIG_CIFS=y
-CONFIG_CIFS_STATS2=y
 CONFIG_ENCRYPTED_KEYS=y
 CONFIG_DMA_CMA=y
 CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_KGDB=y
 CONFIG_KGDB_TESTS=y
 CONFIG_KGDB_KDB=y
-CONFIG_DEBUG_SLAB=y
 CONFIG_DETECT_HUNG_TASK=y
 CONFIG_DEBUG_SPINLOCK=y
diff --git a/arch/microblaze/kernel/syscalls/syscall.tbl b/arch/microblaze/kernel/syscalls/syscall.tbl
index 5b6a0b02b7de..b00ab2cabab9 100644
--- a/arch/microblaze/kernel/syscalls/syscall.tbl
+++ b/arch/microblaze/kernel/syscalls/syscall.tbl
@@ -462,3 +462,8 @@
 454	common	futex_wake			sys_futex_wake
 455	common	futex_wait			sys_futex_wait
 456	common	futex_requeue			sys_futex_requeue
+457	common	statmount			sys_statmount
+458	common	listmount			sys_listmount
+459	common	lsm_get_self_attr		sys_lsm_get_self_attr
+460	common	lsm_set_self_attr		sys_lsm_set_self_attr
+461	common	lsm_list_modules		sys_lsm_list_modules
diff --git a/arch/mips/include/asm/kvm_host.h b/arch/mips/include/asm/kvm_host.h
index 54a85f1d4f2c..179f320cc231 100644
--- a/arch/mips/include/asm/kvm_host.h
+++ b/arch/mips/include/asm/kvm_host.h
@@ -810,8 +810,6 @@ int kvm_mips_mkclean_gpa_pt(struct kvm *kvm, gfn_t start_gfn, gfn_t end_gfn);
 pgd_t *kvm_pgd_alloc(void);
 void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu);
 
-#define KVM_ARCH_WANT_MMU_NOTIFIER
-
 /* Emulation */
 enum emulation_result update_pc(struct kvm_vcpu *vcpu, u32 cause);
 int kvm_get_badinstr(u32 *opc, struct kvm_vcpu *vcpu, u32 *out);
diff --git a/arch/mips/kernel/syscalls/syscall_n32.tbl b/arch/mips/kernel/syscalls/syscall_n32.tbl
index a842b41c8e06..83cfc9eb6b88 100644
--- a/arch/mips/kernel/syscalls/syscall_n32.tbl
+++ b/arch/mips/kernel/syscalls/syscall_n32.tbl
@@ -395,3 +395,8 @@
 454	n32	futex_wake			sys_futex_wake
 455	n32	futex_wait			sys_futex_wait
 456	n32	futex_requeue			sys_futex_requeue
+457	n32	statmount			sys_statmount
+458	n32	listmount			sys_listmount
+459	n32	lsm_get_self_attr		sys_lsm_get_self_attr
+460	n32	lsm_set_self_attr		sys_lsm_set_self_attr
+461	n32	lsm_list_modules		sys_lsm_list_modules
diff --git a/arch/mips/kernel/syscalls/syscall_n64.tbl b/arch/mips/kernel/syscalls/syscall_n64.tbl
index 116ff501bf92..532b855df589 100644
--- a/arch/mips/kernel/syscalls/syscall_n64.tbl
+++ b/arch/mips/kernel/syscalls/syscall_n64.tbl
@@ -371,3 +371,8 @@
 454	n64	futex_wake			sys_futex_wake
 455	n64	futex_wait			sys_futex_wait
 456	n64	futex_requeue			sys_futex_requeue
+457	n64	statmount			sys_statmount
+458	n64	listmount			sys_listmount
+459	n64	lsm_get_self_attr		sys_lsm_get_self_attr
+460	n64	lsm_set_self_attr		sys_lsm_set_self_attr
+461	n64	lsm_list_modules		sys_lsm_list_modules
diff --git a/arch/mips/kernel/syscalls/syscall_o32.tbl b/arch/mips/kernel/syscalls/syscall_o32.tbl
index 525cc54bc63b..d5fda9f29086 100644
--- a/arch/mips/kernel/syscalls/syscall_o32.tbl
+++ b/arch/mips/kernel/syscalls/syscall_o32.tbl
@@ -444,3 +444,8 @@
 454	o32	futex_wake			sys_futex_wake
 455	o32	futex_wait			sys_futex_wait
 456	o32	futex_requeue			sys_futex_requeue
+457	o32	statmount			sys_statmount
+458	o32	listmount			sys_listmount
+459	o32	lsm_get_self_attr		sys_lsm_get_self_attr
+460	032	lsm_set_self_attr		sys_lsm_set_self_attr
+461	o32	lsm_list_modules		sys_lsm_list_modules
diff --git a/arch/mips/kvm/Kconfig b/arch/mips/kvm/Kconfig
index a8cdba75f98d..c04987d2ed2e 100644
--- a/arch/mips/kvm/Kconfig
+++ b/arch/mips/kvm/Kconfig
@@ -25,7 +25,7 @@ config KVM
 	select HAVE_KVM_EVENTFD
 	select HAVE_KVM_VCPU_ASYNC_IOCTL
 	select KVM_MMIO
-	select MMU_NOTIFIER
+	select KVM_GENERIC_MMU_NOTIFIER
 	select INTERVAL_TREE
 	select KVM_GENERIC_HARDWARE_ENABLING
 	help
diff --git a/arch/parisc/include/asm/alternative.h b/arch/parisc/include/asm/alternative.h
index 1ed45fd085d3..1eb488f25b83 100644
--- a/arch/parisc/include/asm/alternative.h
+++ b/arch/parisc/include/asm/alternative.h
@@ -34,7 +34,8 @@ void apply_alternatives(struct alt_instr *start, struct alt_instr *end,
 
 /* Alternative SMP implementation. */
 #define ALTERNATIVE(cond, replacement)		"!0:"	\
-	".section .altinstructions, \"aw\"	!"	\
+	".section .altinstructions, \"a\"	!"	\
+	".align 4				!"	\
 	".word (0b-4-.)				!"	\
 	".hword 1, " __stringify(cond) "	!"	\
 	".word " __stringify(replacement) "	!"	\
@@ -44,7 +45,8 @@ void apply_alternatives(struct alt_instr *start, struct alt_instr *end,
 
 /* to replace one single instructions by a new instruction */
 #define ALTERNATIVE(from, to, cond, replacement)\
-	.section .altinstructions, "aw"	!	\
+	.section .altinstructions, "a"	!	\
+	.align 4			!	\
 	.word (from - .)		!	\
 	.hword (to - from)/4, cond	!	\
 	.word replacement		!	\
@@ -52,7 +54,8 @@ void apply_alternatives(struct alt_instr *start, struct alt_instr *end,
 
 /* to replace multiple instructions by new code */
 #define ALTERNATIVE_CODE(from, num_instructions, cond, new_instr_ptr)\
-	.section .altinstructions, "aw"	!	\
+	.section .altinstructions, "a"	!	\
+	.align 4			!	\
 	.word (from - .)		!	\
 	.hword -num_instructions, cond	!	\
 	.word (new_instr_ptr - .)	!	\
diff --git a/arch/parisc/include/asm/assembly.h b/arch/parisc/include/asm/assembly.h
index 75677b526b2b..74d17d7e759d 100644
--- a/arch/parisc/include/asm/assembly.h
+++ b/arch/parisc/include/asm/assembly.h
@@ -574,6 +574,7 @@
 	 */
 #define ASM_EXCEPTIONTABLE_ENTRY(fault_addr, except_addr)	\
 	.section __ex_table,"aw"			!	\
+	.align 4					!	\
 	.word (fault_addr - .), (except_addr - .)	!	\
 	.previous
 
diff --git a/arch/parisc/include/asm/bug.h b/arch/parisc/include/asm/bug.h
index 4b6d60b94124..b6800de9c645 100644
--- a/arch/parisc/include/asm/bug.h
+++ b/arch/parisc/include/asm/bug.h
@@ -29,12 +29,14 @@
 		asm volatile("\n"					\
 			     "1:\t" PARISC_BUG_BREAK_ASM "\n"		\
 			     "\t.pushsection __bug_table,\"aw\"\n"	\
+			     "\t.align %4\n"				\
 			     "2:\t" ASM_WORD_INSN "1b, %c0\n"		\
 			     "\t.short %c1, %c2\n"			\
 			     "\t.org 2b+%c3\n"				\
 			     "\t.popsection"				\
 			     : : "i" (__FILE__), "i" (__LINE__),	\
-			     "i" (0), "i" (sizeof(struct bug_entry)) ); \
+			     "i" (0), "i" (sizeof(struct bug_entry)),	\
+			     "i" (sizeof(long)) );			\
 		unreachable();						\
 	} while(0)
 
@@ -52,13 +54,15 @@
 		asm volatile("\n"					\
 			     "1:\t" PARISC_BUG_BREAK_ASM "\n"		\
 			     "\t.pushsection __bug_table,\"aw\"\n"	\
+			     "\t.align %4\n"				\
 			     "2:\t" ASM_WORD_INSN "1b, %c0\n"		\
 			     "\t.short %c1, %c2\n"			\
 			     "\t.org 2b+%c3\n"				\
 			     "\t.popsection"				\
 			     : : "i" (__FILE__), "i" (__LINE__),	\
 			     "i" (BUGFLAG_WARNING|(flags)),		\
-			     "i" (sizeof(struct bug_entry)) );		\
+			     "i" (sizeof(struct bug_entry)),		\
+			     "i" (sizeof(long)) );			\
 	} while(0)
 #else
 #define __WARN_FLAGS(flags)						\
@@ -66,12 +70,14 @@
 		asm volatile("\n"					\
 			     "1:\t" PARISC_BUG_BREAK_ASM "\n"		\
 			     "\t.pushsection __bug_table,\"aw\"\n"	\
+			     "\t.align %4\n"				\
 			     "2:\t" ASM_WORD_INSN "1b\n"		\
 			     "\t.short %c0\n"				\
 			     "\t.org 2b+%c1\n"				\
 			     "\t.popsection"				\
 			     : : "i" (BUGFLAG_WARNING|(flags)),		\
-			     "i" (sizeof(struct bug_entry)) );		\
+			     "i" (sizeof(struct bug_entry)),		\
+			     "i" (sizeof(long)) );			\
 	} while(0)
 #endif
 
diff --git a/arch/parisc/include/asm/jump_label.h b/arch/parisc/include/asm/jump_label.h
index af2a598bc0f8..984ddc003101 100644
--- a/arch/parisc/include/asm/jump_label.h
+++ b/arch/parisc/include/asm/jump_label.h
@@ -15,6 +15,7 @@ static __always_inline bool arch_static_branch(struct static_key *key, bool bran
 	asm_volatile_goto("1:\n\t"
 		 "nop\n\t"
 		 ".pushsection __jump_table,  \"aw\"\n\t"
+		 ".align 4\n\t"
 		 ".word 1b - ., %l[l_yes] - .\n\t"
 		 __stringify(ASM_ULONG_INSN) " %c0 - .\n\t"
 		 ".popsection\n\t"
@@ -30,6 +31,7 @@ static __always_inline bool arch_static_branch_jump(struct static_key *key, bool
 	asm_volatile_goto("1:\n\t"
 		 "b,n %l[l_yes]\n\t"
 		 ".pushsection __jump_table,  \"aw\"\n\t"
+		 ".align 4\n\t"
 		 ".word 1b - ., %l[l_yes] - .\n\t"
 		 __stringify(ASM_ULONG_INSN) " %c0 - .\n\t"
 		 ".popsection\n\t"
diff --git a/arch/parisc/include/asm/uaccess.h b/arch/parisc/include/asm/uaccess.h
index 2bf660eabe42..4165079898d9 100644
--- a/arch/parisc/include/asm/uaccess.h
+++ b/arch/parisc/include/asm/uaccess.h
@@ -41,6 +41,7 @@ struct exception_table_entry {
 
 #define ASM_EXCEPTIONTABLE_ENTRY( fault_addr, except_addr )\
 	".section __ex_table,\"aw\"\n"			   \
+	".align 4\n"					   \
 	".word (" #fault_addr " - .), (" #except_addr " - .)\n\t" \
 	".previous\n"
 
diff --git a/arch/parisc/kernel/syscalls/syscall.tbl b/arch/parisc/kernel/syscalls/syscall.tbl
index a47798fed54e..b236a84c4e12 100644
--- a/arch/parisc/kernel/syscalls/syscall.tbl
+++ b/arch/parisc/kernel/syscalls/syscall.tbl
@@ -455,3 +455,8 @@
 454	common	futex_wake			sys_futex_wake
 455	common	futex_wait			sys_futex_wait
 456	common	futex_requeue			sys_futex_requeue
+457	common	statmount			sys_statmount
+458	common	listmount			sys_listmount
+459	common	lsm_get_self_attr		sys_lsm_get_self_attr
+460	common	lsm_set_self_attr		sys_lsm_set_self_attr
+461	common	lsm_list_modules		sys_lsm_list_modules
diff --git a/arch/parisc/kernel/vmlinux.lds.S b/arch/parisc/kernel/vmlinux.lds.S
index 58694d1989c2..190e623e3ade 100644
--- a/arch/parisc/kernel/vmlinux.lds.S
+++ b/arch/parisc/kernel/vmlinux.lds.S
@@ -130,7 +130,7 @@ SECTIONS
 	RO_DATA(8)
 
 	/* unwind info */
-	.PARISC.unwind : {
+	.PARISC.unwind ALIGN(4) : {
 		__start___unwind = .;
 		*(.PARISC.unwind)
 		__stop___unwind = .;
@@ -149,7 +149,7 @@ SECTIONS
 
 	/* PA-RISC locks requires 16-byte alignment */
 	. = ALIGN(16);
-	.data..lock_aligned : {
+	.data..lock_aligned ALIGN(16) : {
 		*(.data..lock_aligned)
 	}
 
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 6f105ee4f3cf..1f11a62809f2 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -608,10 +608,10 @@ config ARCH_SUPPORTS_KEXEC
 	def_bool PPC_BOOK3S || PPC_E500 || (44x && !SMP)
 
 config ARCH_SUPPORTS_KEXEC_FILE
-	def_bool PPC64 && CRYPTO=y && CRYPTO_SHA256=y
+	def_bool PPC64
 
 config ARCH_SUPPORTS_KEXEC_PURGATORY
-	def_bool KEXEC_FILE
+	def_bool y
 
 config ARCH_SELECTS_KEXEC_FILE
 	def_bool y
diff --git a/arch/powerpc/crypto/aes-gcm-p10-glue.c b/arch/powerpc/crypto/aes-gcm-p10-glue.c
index 4b6e899895e7..f62ee54076c0 100644
--- a/arch/powerpc/crypto/aes-gcm-p10-glue.c
+++ b/arch/powerpc/crypto/aes-gcm-p10-glue.c
@@ -37,7 +37,7 @@ asmlinkage void aes_p10_gcm_encrypt(u8 *in, u8 *out, size_t len,
 				    void *rkey, u8 *iv, void *Xi);
 asmlinkage void aes_p10_gcm_decrypt(u8 *in, u8 *out, size_t len,
 				    void *rkey, u8 *iv, void *Xi);
-asmlinkage void gcm_init_htable(unsigned char htable[256], unsigned char Xi[16]);
+asmlinkage void gcm_init_htable(unsigned char htable[], unsigned char Xi[]);
 asmlinkage void gcm_ghash_p10(unsigned char *Xi, unsigned char *Htable,
 		unsigned char *aad, unsigned int alen);
 
diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h
index 8799b37be295..8abac532146e 100644
--- a/arch/powerpc/include/asm/kvm_host.h
+++ b/arch/powerpc/include/asm/kvm_host.h
@@ -63,8 +63,6 @@
 
 #include <linux/mmu_notifier.h>
 
-#define KVM_ARCH_WANT_MMU_NOTIFIER
-
 #define HPTEG_CACHE_NUM			(1 << 15)
 #define HPTEG_HASH_BITS_PTE		13
 #define HPTEG_HASH_BITS_PTE_LONG	12
diff --git a/arch/powerpc/kernel/firmware.c b/arch/powerpc/kernel/firmware.c
index 20328f72f9f2..8987eee33dc8 100644
--- a/arch/powerpc/kernel/firmware.c
+++ b/arch/powerpc/kernel/firmware.c
@@ -23,6 +23,8 @@ EXPORT_SYMBOL_GPL(powerpc_firmware_features);
 
 #if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_KVM_GUEST)
 DEFINE_STATIC_KEY_FALSE(kvm_guest);
+EXPORT_SYMBOL_GPL(kvm_guest);
+
 int __init check_kvm_guest(void)
 {
 	struct device_node *hyper_node;
diff --git a/arch/powerpc/kernel/syscalls/syscall.tbl b/arch/powerpc/kernel/syscalls/syscall.tbl
index 7fab411378f2..17173b82ca21 100644
--- a/arch/powerpc/kernel/syscalls/syscall.tbl
+++ b/arch/powerpc/kernel/syscalls/syscall.tbl
@@ -543,3 +543,8 @@
 454	common	futex_wake			sys_futex_wake
 455	common	futex_wait			sys_futex_wait
 456	common	futex_requeue			sys_futex_requeue
+457	common	statmount			sys_statmount
+458	common	listmount			sys_listmount
+459	common	lsm_get_self_attr		sys_lsm_get_self_attr
+460	common	lsm_set_self_attr		sys_lsm_set_self_attr
+461	common	lsm_list_modules		sys_lsm_list_modules
diff --git a/arch/powerpc/kexec/core_64.c b/arch/powerpc/kexec/core_64.c
index 0bee7ca9a77c..762e4d09aacf 100644
--- a/arch/powerpc/kexec/core_64.c
+++ b/arch/powerpc/kexec/core_64.c
@@ -283,8 +283,7 @@ static void kexec_prepare_cpus(void)
  * We could use a smaller stack if we don't care about anything using
  * current, but that audit has not been performed.
  */
-static union thread_union kexec_stack __init_task_data =
-	{ };
+static union thread_union kexec_stack = { };
 
 /*
  * For similar reasons to the stack above, the kexecing CPU needs to be on a
diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig
index 902611954200..b33358ee6424 100644
--- a/arch/powerpc/kvm/Kconfig
+++ b/arch/powerpc/kvm/Kconfig
@@ -42,7 +42,7 @@ config KVM_BOOK3S_64_HANDLER
 config KVM_BOOK3S_PR_POSSIBLE
 	bool
 	select KVM_MMIO
-	select MMU_NOTIFIER
+	select KVM_GENERIC_MMU_NOTIFIER
 
 config KVM_BOOK3S_HV_POSSIBLE
 	bool
@@ -85,7 +85,7 @@ config KVM_BOOK3S_64_HV
 	tristate "KVM for POWER7 and later using hypervisor mode in host"
 	depends on KVM_BOOK3S_64 && PPC_POWERNV
 	select KVM_BOOK3S_HV_POSSIBLE
-	select MMU_NOTIFIER
+	select KVM_GENERIC_MMU_NOTIFIER
 	select CMA
 	help
 	  Support running unmodified book3s_64 guest kernels in
@@ -194,7 +194,7 @@ config KVM_E500V2
 	depends on !CONTEXT_TRACKING_USER
 	select KVM
 	select KVM_MMIO
-	select MMU_NOTIFIER
+	select KVM_GENERIC_MMU_NOTIFIER
 	help
 	  Support running unmodified E500 guest kernels in virtual machines on
 	  E500v2 host processors.
@@ -211,7 +211,7 @@ config KVM_E500MC
 	select KVM
 	select KVM_MMIO
 	select KVM_BOOKE_HV
-	select MMU_NOTIFIER
+	select KVM_GENERIC_MMU_NOTIFIER
 	help
 	  Support running unmodified E500MC/E5500/E6500 guest kernels in
 	  virtual machines on E500MC/E5500/E6500 host processors.
diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
index 1ed6ec140701..b5b04dd5d0ee 100644
--- a/arch/powerpc/kvm/book3s_hv.c
+++ b/arch/powerpc/kvm/book3s_hv.c
@@ -6210,7 +6210,7 @@ static int kvmhv_svm_off(struct kvm *kvm)
 	}
 
 	srcu_idx = srcu_read_lock(&kvm->srcu);
-	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
+	for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
 		struct kvm_memory_slot *memslot;
 		struct kvm_memslots *slots = __kvm_memslots(kvm, i);
 		int bkt;
diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
index f6af752698d0..c39945a7fce3 100644
--- a/arch/powerpc/kvm/powerpc.c
+++ b/arch/powerpc/kvm/powerpc.c
@@ -632,13 +632,8 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 		break;
 #endif
 	case KVM_CAP_SYNC_MMU:
-#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
-		r = hv_enabled;
-#elif defined(KVM_ARCH_WANT_MMU_NOTIFIER)
+		BUILD_BUG_ON(!IS_ENABLED(CONFIG_KVM_GENERIC_MMU_NOTIFIER));
 		r = 1;
-#else
-		r = 0;
-#endif
 		break;
 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
 	case KVM_CAP_PPC_HTAB_FD:
diff --git a/arch/powerpc/lib/sstep.c b/arch/powerpc/lib/sstep.c
index a4ab8625061a..a13f05cfc7db 100644
--- a/arch/powerpc/lib/sstep.c
+++ b/arch/powerpc/lib/sstep.c
@@ -688,7 +688,7 @@ static nokprobe_inline int do_vec_load(int rn, unsigned long ea,
 	if (err)
 		return err;
 	if (unlikely(cross_endian))
-		do_byte_reverse(&u.b[ea & 0xf], size);
+		do_byte_reverse(&u.b[ea & 0xf], min_t(size_t, size, sizeof(u)));
 	preempt_disable();
 	if (regs->msr & MSR_VEC)
 		put_vr(rn, &u.v);
@@ -719,7 +719,7 @@ static nokprobe_inline int do_vec_store(int rn, unsigned long ea,
 		u.v = current->thread.vr_state.vr[rn];
 	preempt_enable();
 	if (unlikely(cross_endian))
-		do_byte_reverse(&u.b[ea & 0xf], size);
+		do_byte_reverse(&u.b[ea & 0xf], min_t(size_t, size, sizeof(u)));
 	return copy_mem_out(&u.b[ea & 0xf], ea, size, regs);
 }
 #endif /* CONFIG_ALTIVEC */
diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c
index 10b946e9c6e7..b7ff680cde96 100644
--- a/arch/powerpc/perf/core-book3s.c
+++ b/arch/powerpc/perf/core-book3s.c
@@ -2312,7 +2312,7 @@ static void record_and_restart(struct perf_event *event, unsigned long val,
 			struct cpu_hw_events *cpuhw;
 			cpuhw = this_cpu_ptr(&cpu_hw_events);
 			power_pmu_bhrb_read(event, cpuhw);
-			perf_sample_save_brstack(&data, event, &cpuhw->bhrb_stack);
+			perf_sample_save_brstack(&data, event, &cpuhw->bhrb_stack, NULL);
 		}
 
 		if (event->attr.sample_type & PERF_SAMPLE_DATA_SRC &&
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 95a2a06acc6a..afdf05ad78ad 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -702,9 +702,7 @@ config ARCH_SELECTS_KEXEC_FILE
 	select KEXEC_ELF
 
 config ARCH_SUPPORTS_KEXEC_PURGATORY
-	def_bool KEXEC_FILE
-	depends on CRYPTO=y
-	depends on CRYPTO_SHA256=y
+	def_bool ARCH_SUPPORTS_KEXEC_FILE
 
 config ARCH_SUPPORTS_CRASH_DUMP
 	def_bool y
diff --git a/arch/riscv/boot/dts/renesas/r9a07g043f.dtsi b/arch/riscv/boot/dts/renesas/r9a07g043f.dtsi
index b0796015e36b..a92cfcfc021b 100644
--- a/arch/riscv/boot/dts/renesas/r9a07g043f.dtsi
+++ b/arch/riscv/boot/dts/renesas/r9a07g043f.dtsi
@@ -24,6 +24,10 @@ cpu0: cpu@0 {
 			reg = <0x0>;
 			status = "okay";
 			riscv,isa = "rv64imafdc";
+			riscv,isa-base = "rv64i";
+			riscv,isa-extensions = "i", "m", "a", "f", "d", "c",
+					       "zicntr", "zicsr", "zifencei",
+					       "zihpm";
 			mmu-type = "riscv,sv39";
 			i-cache-size = <0x8000>;
 			i-cache-line-size = <0x40>;
diff --git a/arch/riscv/boot/dts/sophgo/cv1800b.dtsi b/arch/riscv/boot/dts/sophgo/cv1800b.dtsi
index df40e87ee063..aec6401a467b 100644
--- a/arch/riscv/boot/dts/sophgo/cv1800b.dtsi
+++ b/arch/riscv/boot/dts/sophgo/cv1800b.dtsi
@@ -34,7 +34,6 @@ cpu0: cpu@0 {
 			cpu0_intc: interrupt-controller {
 				compatible = "riscv,cpu-intc";
 				interrupt-controller;
-				#address-cells = <0>;
 				#interrupt-cells = <1>;
 			};
 		};
diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h
index 0eefd9c991ae..6964dd235e97 100644
--- a/arch/riscv/include/asm/kvm_host.h
+++ b/arch/riscv/include/asm/kvm_host.h
@@ -267,8 +267,6 @@ struct kvm_vcpu_arch {
 static inline void kvm_arch_sync_events(struct kvm *kvm) {}
 static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
 
-#define KVM_ARCH_WANT_MMU_NOTIFIER
-
 #define KVM_RISCV_GSTAGE_TLB_MIN_ORDER		12
 
 void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid,
diff --git a/arch/riscv/kvm/Kconfig b/arch/riscv/kvm/Kconfig
index dfc237d7875b..ae2e05f050ec 100644
--- a/arch/riscv/kvm/Kconfig
+++ b/arch/riscv/kvm/Kconfig
@@ -30,7 +30,7 @@ config KVM
 	select KVM_GENERIC_HARDWARE_ENABLING
 	select KVM_MMIO
 	select KVM_XFER_TO_GUEST_WORK
-	select MMU_NOTIFIER
+	select KVM_GENERIC_MMU_NOTIFIER
 	select PREEMPT_NOTIFIERS
 	help
 	  Support hosting virtualized guest machines.
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 3bec98d20283..d5d8f99d1f25 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -254,13 +254,13 @@ config ARCH_SUPPORTS_KEXEC
 	def_bool y
 
 config ARCH_SUPPORTS_KEXEC_FILE
-	def_bool CRYPTO && CRYPTO_SHA256 && CRYPTO_SHA256_S390
+	def_bool y
 
 config ARCH_SUPPORTS_KEXEC_SIG
 	def_bool MODULE_SIG_FORMAT
 
 config ARCH_SUPPORTS_KEXEC_PURGATORY
-	def_bool KEXEC_FILE
+	def_bool y
 
 config ARCH_SUPPORTS_CRASH_DUMP
 	def_bool y
diff --git a/arch/s390/kernel/syscalls/syscall.tbl b/arch/s390/kernel/syscalls/syscall.tbl
index 86fec9b080f6..095bb86339a7 100644
--- a/arch/s390/kernel/syscalls/syscall.tbl
+++ b/arch/s390/kernel/syscalls/syscall.tbl
@@ -459,3 +459,8 @@
 454  common	futex_wake		sys_futex_wake			sys_futex_wake
 455  common	futex_wait		sys_futex_wait			sys_futex_wait
 456  common	futex_requeue		sys_futex_requeue		sys_futex_requeue
+457  common	statmount		sys_statmount			sys_statmount
+458  common	listmount		sys_listmount			sys_listmount
+459  common	lsm_get_self_attr	sys_lsm_get_self_attr		sys_lsm_get_self_attr
+460  common	lsm_set_self_attr	sys_lsm_set_self_attr		sys_lsm_set_self_attr
+461  common	lsm_list_modules	sys_lsm_list_modules		sys_lsm_list_modules
diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c
index 02dcbe82a8e5..8207a892bbe2 100644
--- a/arch/s390/kvm/vsie.c
+++ b/arch/s390/kvm/vsie.c
@@ -587,10 +587,6 @@ void kvm_s390_vsie_gmap_notifier(struct gmap *gmap, unsigned long start,
 
 	if (!gmap_is_shadow(gmap))
 		return;
-	if (start >= 1UL << 31)
-		/* We are only interested in prefix pages */
-		return;
-
 	/*
 	 * Only new shadow blocks are added to the list during runtime,
 	 * therefore we can safely reference them all the time.
diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index 3bd2ab2a9a34..5cb92941540b 100644
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -756,7 +756,7 @@ void ptep_zap_unused(struct mm_struct *mm, unsigned long addr,
 		pte_clear(mm, addr, ptep);
 	}
 	if (reset)
-		pgste_val(pgste) &= ~_PGSTE_GPS_USAGE_MASK;
+		pgste_val(pgste) &= ~(_PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT);
 	pgste_set_unlock(ptep, pgste);
 	preempt_enable();
 }
diff --git a/arch/sh/kernel/syscalls/syscall.tbl b/arch/sh/kernel/syscalls/syscall.tbl
index 363fae0fe9bf..86fe269f0220 100644
--- a/arch/sh/kernel/syscalls/syscall.tbl
+++ b/arch/sh/kernel/syscalls/syscall.tbl
@@ -459,3 +459,8 @@
 454	common	futex_wake			sys_futex_wake
 455	common	futex_wait			sys_futex_wait
 456	common	futex_requeue			sys_futex_requeue
+457	common	statmount			sys_statmount
+458	common	listmount			sys_listmount
+459	common	lsm_get_self_attr		sys_lsm_get_self_attr
+460	common	lsm_set_self_attr		sys_lsm_set_self_attr
+461	common	lsm_list_modules		sys_lsm_list_modules
diff --git a/arch/sparc/kernel/syscalls/syscall.tbl b/arch/sparc/kernel/syscalls/syscall.tbl
index 7bcaa3d5ea44..b23d59313589 100644
--- a/arch/sparc/kernel/syscalls/syscall.tbl
+++ b/arch/sparc/kernel/syscalls/syscall.tbl
@@ -502,3 +502,8 @@
 454	common	futex_wake			sys_futex_wake
 455	common	futex_wait			sys_futex_wait
 456	common	futex_requeue			sys_futex_requeue
+457	common	statmount			sys_statmount
+458	common	listmount			sys_listmount
+459	common	lsm_get_self_attr		sys_lsm_get_self_attr
+460	common	lsm_set_self_attr		sys_lsm_set_self_attr
+461	common	lsm_list_modules		sys_lsm_list_modules
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 3762f41bb092..21890efbfdd8 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -2072,7 +2072,7 @@ config ARCH_SUPPORTS_KEXEC
 	def_bool y
 
 config ARCH_SUPPORTS_KEXEC_FILE
-	def_bool X86_64 && CRYPTO && CRYPTO_SHA256
+	def_bool X86_64
 
 config ARCH_SELECTS_KEXEC_FILE
 	def_bool y
@@ -2080,7 +2080,7 @@ config ARCH_SELECTS_KEXEC_FILE
 	select HAVE_IMA_KEXEC if IMA
 
 config ARCH_SUPPORTS_KEXEC_PURGATORY
-	def_bool KEXEC_FILE
+	def_bool y
 
 config ARCH_SUPPORTS_KEXEC_SIG
 	def_bool y
@@ -2429,6 +2429,18 @@ source "kernel/livepatch/Kconfig"
 
 endmenu
 
+config CC_HAS_NAMED_AS
+	def_bool CC_IS_GCC && GCC_VERSION >= 120100
+
+config USE_X86_SEG_SUPPORT
+	def_bool y
+	depends on CC_HAS_NAMED_AS
+	#
+	# -fsanitize=kernel-address (KASAN) is at the moment incompatible
+	# with named address spaces - see GCC PR sanitizer/111736.
+	#
+	depends on !KASAN
+
 config CC_HAS_SLS
 	def_bool $(cc-option,-mharden-sls=all)
 
diff --git a/arch/x86/crypto/sha1_ssse3_glue.c b/arch/x86/crypto/sha1_ssse3_glue.c
index 959afa705e95..ab8bc54f254d 100644
--- a/arch/x86/crypto/sha1_ssse3_glue.c
+++ b/arch/x86/crypto/sha1_ssse3_glue.c
@@ -2,8 +2,8 @@
 /*
  * Cryptographic API.
  *
- * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using
- * Supplemental SSE3 instructions.
+ * Glue code for the SHA1 Secure Hash Algorithm assembler implementations
+ * using SSSE3, AVX, AVX2, and SHA-NI instructions.
  *
  * This file is based on sha1_generic.c
  *
@@ -28,6 +28,9 @@
 #include <asm/simd.h>
 
 static const struct x86_cpu_id module_cpu_ids[] = {
+#ifdef CONFIG_AS_SHA1_NI
+	X86_MATCH_FEATURE(X86_FEATURE_SHA_NI, NULL),
+#endif
 	X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL),
 	X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL),
 	X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL),
diff --git a/arch/x86/crypto/sha256_ssse3_glue.c b/arch/x86/crypto/sha256_ssse3_glue.c
index 4c0383a90e11..e04a43d9f7d5 100644
--- a/arch/x86/crypto/sha256_ssse3_glue.c
+++ b/arch/x86/crypto/sha256_ssse3_glue.c
@@ -1,8 +1,8 @@
 /*
  * Cryptographic API.
  *
- * Glue code for the SHA256 Secure Hash Algorithm assembler
- * implementation using supplemental SSE3 / AVX / AVX2 instructions.
+ * Glue code for the SHA256 Secure Hash Algorithm assembler implementations
+ * using SSSE3, AVX, AVX2, and SHA-NI instructions.
  *
  * This file is based on sha256_generic.c
  *
@@ -45,6 +45,9 @@ asmlinkage void sha256_transform_ssse3(struct sha256_state *state,
 				       const u8 *data, int blocks);
 
 static const struct x86_cpu_id module_cpu_ids[] = {
+#ifdef CONFIG_AS_SHA256_NI
+	X86_MATCH_FEATURE(X86_FEATURE_SHA_NI, NULL),
+#endif
 	X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL),
 	X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL),
 	X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL),
diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h
index f6907627172b..e59d3073e7cf 100644
--- a/arch/x86/entry/calling.h
+++ b/arch/x86/entry/calling.h
@@ -173,10 +173,9 @@ For 32-bit we have the following conventions - kernel is built with
 .endm
 
 #define THIS_CPU_user_pcid_flush_mask   \
-	PER_CPU_VAR(cpu_tlbstate) + TLB_STATE_user_pcid_flush_mask
+	PER_CPU_VAR(cpu_tlbstate + TLB_STATE_user_pcid_flush_mask)
 
-.macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req
-	ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
+.macro SWITCH_TO_USER_CR3 scratch_reg:req scratch_reg2:req
 	mov	%cr3, \scratch_reg
 
 	ALTERNATIVE "jmp .Lwrcr3_\@", "", X86_FEATURE_PCID
@@ -206,13 +205,20 @@ For 32-bit we have the following conventions - kernel is built with
 	/* Flip the PGD to the user version */
 	orq     $(PTI_USER_PGTABLE_MASK), \scratch_reg
 	mov	\scratch_reg, %cr3
+.endm
+
+.macro SWITCH_TO_USER_CR3_NOSTACK scratch_reg:req scratch_reg2:req
+	ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
+	SWITCH_TO_USER_CR3 \scratch_reg \scratch_reg2
 .Lend_\@:
 .endm
 
 .macro SWITCH_TO_USER_CR3_STACK	scratch_reg:req
+	ALTERNATIVE "jmp .Lend_\@", "", X86_FEATURE_PTI
 	pushq	%rax
-	SWITCH_TO_USER_CR3_NOSTACK scratch_reg=\scratch_reg scratch_reg2=%rax
+	SWITCH_TO_USER_CR3 scratch_reg=\scratch_reg scratch_reg2=%rax
 	popq	%rax
+.Lend_\@:
 .endm
 
 .macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req
diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index c73047bf9f4b..4e295798638b 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -305,7 +305,7 @@
 .macro CHECK_AND_APPLY_ESPFIX
 #ifdef CONFIG_X86_ESPFIX32
 #define GDT_ESPFIX_OFFSET (GDT_ENTRY_ESPFIX_SS * 8)
-#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + GDT_ESPFIX_OFFSET
+#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page + GDT_ESPFIX_OFFSET)
 
 	ALTERNATIVE	"jmp .Lend_\@", "", X86_BUG_ESPFIX
 
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index de6469dffe3a..567d973eed03 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -190,7 +190,7 @@ SYM_FUNC_START(__switch_to_asm)
 
 #ifdef CONFIG_STACKPROTECTOR
 	movq	TASK_stack_canary(%rsi), %rbx
-	movq	%rbx, PER_CPU_VAR(fixed_percpu_data) + FIXED_stack_canary
+	movq	%rbx, PER_CPU_VAR(fixed_percpu_data + FIXED_stack_canary)
 #endif
 
 	/*
@@ -559,17 +559,27 @@ __irqentry_text_end:
 SYM_CODE_START_LOCAL(common_interrupt_return)
 SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
 	IBRS_EXIT
-#ifdef CONFIG_DEBUG_ENTRY
-	/* Assert that pt_regs indicates user mode. */
-	testb	$3, CS(%rsp)
-	jnz	1f
-	ud2
-1:
-#endif
 #ifdef CONFIG_XEN_PV
 	ALTERNATIVE "", "jmp xenpv_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV
 #endif
+#ifdef CONFIG_PAGE_TABLE_ISOLATION
+	ALTERNATIVE "", "jmp .Lpti_restore_regs_and_return_to_usermode", X86_FEATURE_PTI
+#endif
+
+	STACKLEAK_ERASE
+	POP_REGS
+	add	$8, %rsp	/* orig_ax */
+	UNWIND_HINT_IRET_REGS
+
+.Lswapgs_and_iret:
+	swapgs
+	/* Assert that the IRET frame indicates user mode. */
+	testb	$3, 8(%rsp)
+	jnz	.Lnative_iret
+	ud2
 
+#ifdef CONFIG_PAGE_TABLE_ISOLATION
+.Lpti_restore_regs_and_return_to_usermode:
 	POP_REGS pop_rdi=0
 
 	/*
@@ -596,13 +606,14 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
 	 */
 	STACKLEAK_ERASE_NOCLOBBER
 
-	SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
+	push	%rax
+	SWITCH_TO_USER_CR3 scratch_reg=%rdi scratch_reg2=%rax
+	pop	%rax
 
 	/* Restore RDI. */
 	popq	%rdi
-	swapgs
-	jmp	.Lnative_iret
-
+	jmp	.Lswapgs_and_iret
+#endif
 
 SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
 #ifdef CONFIG_DEBUG_ENTRY
diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl
index c8fac5205803..5f8591ce7f25 100644
--- a/arch/x86/entry/syscalls/syscall_32.tbl
+++ b/arch/x86/entry/syscalls/syscall_32.tbl
@@ -461,3 +461,8 @@
 454	i386	futex_wake		sys_futex_wake
 455	i386	futex_wait		sys_futex_wait
 456	i386	futex_requeue		sys_futex_requeue
+457	i386	statmount		sys_statmount
+458	i386	listmount		sys_listmount
+459	i386	lsm_get_self_attr	sys_lsm_get_self_attr
+460	i386	lsm_set_self_attr	sys_lsm_set_self_attr
+461	i386	lsm_list_modules	sys_lsm_list_modules
diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl
index 8cb8bf68721c..7e8d46f4147f 100644
--- a/arch/x86/entry/syscalls/syscall_64.tbl
+++ b/arch/x86/entry/syscalls/syscall_64.tbl
@@ -378,6 +378,11 @@
 454	common	futex_wake		sys_futex_wake
 455	common	futex_wait		sys_futex_wait
 456	common	futex_requeue		sys_futex_requeue
+457	common	statmount		sys_statmount
+458	common	listmount		sys_listmount
+459	common	lsm_get_self_attr	sys_lsm_get_self_attr
+460	common	lsm_set_self_attr	sys_lsm_set_self_attr
+461	common	lsm_list_modules	sys_lsm_list_modules
 
 #
 # Due to a historical design error, certain syscalls are numbered differently
diff --git a/arch/x86/events/amd/core.c b/arch/x86/events/amd/core.c
index e24976593a29..4ee6390b45c9 100644
--- a/arch/x86/events/amd/core.c
+++ b/arch/x86/events/amd/core.c
@@ -940,7 +940,7 @@ static int amd_pmu_v2_handle_irq(struct pt_regs *regs)
 			continue;
 
 		if (has_branch_stack(event))
-			perf_sample_save_brstack(&data, event, &cpuc->lbr_stack);
+			perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL);
 
 		if (perf_event_overflow(event, &data, regs))
 			x86_pmu_stop(event, 0);
diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c
index 40ad1425ffa2..09050641ce5d 100644
--- a/arch/x86/events/core.c
+++ b/arch/x86/events/core.c
@@ -601,7 +601,7 @@ int x86_pmu_hw_config(struct perf_event *event)
 		}
 	}
 
-	if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK)
+	if (branch_sample_call_stack(event))
 		event->attach_state |= PERF_ATTACH_TASK_DATA;
 
 	/*
@@ -1702,7 +1702,7 @@ int x86_pmu_handle_irq(struct pt_regs *regs)
 		perf_sample_data_init(&data, 0, event->hw.last_period);
 
 		if (has_branch_stack(event))
-			perf_sample_save_brstack(&data, event, &cpuc->lbr_stack);
+			perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL);
 
 		if (perf_event_overflow(event, &data, regs))
 			x86_pmu_stop(event, 0);
diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c
index a08f794a0e79..69b8bd4e99ad 100644
--- a/arch/x86/events/intel/core.c
+++ b/arch/x86/events/intel/core.c
@@ -2527,9 +2527,14 @@ static void intel_pmu_assign_event(struct perf_event *event, int idx)
 		perf_report_aux_output_id(event, idx);
 }
 
+static __always_inline bool intel_pmu_needs_branch_stack(struct perf_event *event)
+{
+	return event->hw.flags & PERF_X86_EVENT_NEEDS_BRANCH_STACK;
+}
+
 static void intel_pmu_del_event(struct perf_event *event)
 {
-	if (needs_branch_stack(event))
+	if (intel_pmu_needs_branch_stack(event))
 		intel_pmu_lbr_del(event);
 	if (event->attr.precise_ip)
 		intel_pmu_pebs_del(event);
@@ -2787,6 +2792,7 @@ static void intel_pmu_enable_fixed(struct perf_event *event)
 
 static void intel_pmu_enable_event(struct perf_event *event)
 {
+	u64 enable_mask = ARCH_PERFMON_EVENTSEL_ENABLE;
 	struct hw_perf_event *hwc = &event->hw;
 	int idx = hwc->idx;
 
@@ -2795,8 +2801,10 @@ static void intel_pmu_enable_event(struct perf_event *event)
 
 	switch (idx) {
 	case 0 ... INTEL_PMC_IDX_FIXED - 1:
+		if (branch_sample_counters(event))
+			enable_mask |= ARCH_PERFMON_EVENTSEL_BR_CNTR;
 		intel_set_masks(event, idx);
-		__x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
+		__x86_pmu_enable_event(hwc, enable_mask);
 		break;
 	case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS - 1:
 	case INTEL_PMC_IDX_METRIC_BASE ... INTEL_PMC_IDX_METRIC_END:
@@ -2820,7 +2828,7 @@ static void intel_pmu_add_event(struct perf_event *event)
 {
 	if (event->attr.precise_ip)
 		intel_pmu_pebs_add(event);
-	if (needs_branch_stack(event))
+	if (intel_pmu_needs_branch_stack(event))
 		intel_pmu_lbr_add(event);
 }
 
@@ -3047,7 +3055,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
 		perf_sample_data_init(&data, 0, event->hw.last_period);
 
 		if (has_branch_stack(event))
-			perf_sample_save_brstack(&data, event, &cpuc->lbr_stack);
+			intel_pmu_lbr_save_brstack(&data, cpuc, event);
 
 		if (perf_event_overflow(event, &data, regs))
 			x86_pmu_stop(event, 0);
@@ -3612,6 +3620,13 @@ intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
 	if (cpuc->excl_cntrs)
 		return intel_get_excl_constraints(cpuc, event, idx, c2);
 
+	/* Not all counters support the branch counter feature. */
+	if (branch_sample_counters(event)) {
+		c2 = dyn_constraint(cpuc, c2, idx);
+		c2->idxmsk64 &= x86_pmu.lbr_counters;
+		c2->weight = hweight64(c2->idxmsk64);
+	}
+
 	return c2;
 }
 
@@ -3897,7 +3912,62 @@ static int intel_pmu_hw_config(struct perf_event *event)
 			x86_pmu.pebs_aliases(event);
 	}
 
-	if (needs_branch_stack(event)) {
+	if (needs_branch_stack(event) && is_sampling_event(event))
+		event->hw.flags  |= PERF_X86_EVENT_NEEDS_BRANCH_STACK;
+
+	if (branch_sample_counters(event)) {
+		struct perf_event *leader, *sibling;
+		int num = 0;
+
+		if (!(x86_pmu.flags & PMU_FL_BR_CNTR) ||
+		    (event->attr.config & ~INTEL_ARCH_EVENT_MASK))
+			return -EINVAL;
+
+		/*
+		 * The branch counter logging is not supported in the call stack
+		 * mode yet, since we cannot simply flush the LBR during e.g.,
+		 * multiplexing. Also, there is no obvious usage with the call
+		 * stack mode. Simply forbids it for now.
+		 *
+		 * If any events in the group enable the branch counter logging
+		 * feature, the group is treated as a branch counter logging
+		 * group, which requires the extra space to store the counters.
+		 */
+		leader = event->group_leader;
+		if (branch_sample_call_stack(leader))
+			return -EINVAL;
+		if (branch_sample_counters(leader))
+			num++;
+		leader->hw.flags |= PERF_X86_EVENT_BRANCH_COUNTERS;
+
+		for_each_sibling_event(sibling, leader) {
+			if (branch_sample_call_stack(sibling))
+				return -EINVAL;
+			if (branch_sample_counters(sibling))
+				num++;
+		}
+
+		if (num > fls(x86_pmu.lbr_counters))
+			return -EINVAL;
+		/*
+		 * Only applying the PERF_SAMPLE_BRANCH_COUNTERS doesn't
+		 * require any branch stack setup.
+		 * Clear the bit to avoid unnecessary branch stack setup.
+		 */
+		if (0 == (event->attr.branch_sample_type &
+			  ~(PERF_SAMPLE_BRANCH_PLM_ALL |
+			    PERF_SAMPLE_BRANCH_COUNTERS)))
+			event->hw.flags  &= ~PERF_X86_EVENT_NEEDS_BRANCH_STACK;
+
+		/*
+		 * Force the leader to be a LBR event. So LBRs can be reset
+		 * with the leader event. See intel_pmu_lbr_del() for details.
+		 */
+		if (!intel_pmu_needs_branch_stack(leader))
+			return -EINVAL;
+	}
+
+	if (intel_pmu_needs_branch_stack(event)) {
 		ret = intel_pmu_setup_lbr_filter(event);
 		if (ret)
 			return ret;
@@ -4375,8 +4445,13 @@ cmt_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
 	 */
 	if (event->attr.precise_ip == 3) {
 		/* Force instruction:ppp on PMC0, 1 and Fixed counter 0 */
-		if (constraint_match(&fixed0_constraint, event->hw.config))
-			return &fixed0_counter0_1_constraint;
+		if (constraint_match(&fixed0_constraint, event->hw.config)) {
+			/* The fixed counter 0 doesn't support LBR event logging. */
+			if (branch_sample_counters(event))
+				return &counter0_1_constraint;
+			else
+				return &fixed0_counter0_1_constraint;
+		}
 
 		switch (c->idxmsk64 & 0x3ull) {
 		case 0x1:
@@ -4555,7 +4630,7 @@ int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu)
 			goto err;
 	}
 
-	if (x86_pmu.flags & (PMU_FL_EXCL_CNTRS | PMU_FL_TFA)) {
+	if (x86_pmu.flags & (PMU_FL_EXCL_CNTRS | PMU_FL_TFA | PMU_FL_BR_CNTR)) {
 		size_t sz = X86_PMC_IDX_MAX * sizeof(struct event_constraint);
 
 		cpuc->constraint_list = kzalloc_node(sz, GFP_KERNEL, cpu_to_node(cpu));
@@ -4660,7 +4735,7 @@ static void intel_pmu_check_hybrid_pmus(struct x86_hybrid_pmu *pmu)
 	if (pmu->intel_cap.pebs_output_pt_available)
 		pmu->pmu.capabilities |= PERF_PMU_CAP_AUX_OUTPUT;
 	else
-		pmu->pmu.capabilities |= ~PERF_PMU_CAP_AUX_OUTPUT;
+		pmu->pmu.capabilities &= ~PERF_PMU_CAP_AUX_OUTPUT;
 
 	intel_pmu_check_event_constraints(pmu->event_constraints,
 					  pmu->num_counters,
@@ -5527,11 +5602,41 @@ static ssize_t branches_show(struct device *cdev,
 
 static DEVICE_ATTR_RO(branches);
 
+static ssize_t branch_counter_nr_show(struct device *cdev,
+				      struct device_attribute *attr,
+				      char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "%d\n", fls(x86_pmu.lbr_counters));
+}
+
+static DEVICE_ATTR_RO(branch_counter_nr);
+
+static ssize_t branch_counter_width_show(struct device *cdev,
+					 struct device_attribute *attr,
+					 char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "%d\n", LBR_INFO_BR_CNTR_BITS);
+}
+
+static DEVICE_ATTR_RO(branch_counter_width);
+
 static struct attribute *lbr_attrs[] = {
 	&dev_attr_branches.attr,
+	&dev_attr_branch_counter_nr.attr,
+	&dev_attr_branch_counter_width.attr,
 	NULL
 };
 
+static umode_t
+lbr_is_visible(struct kobject *kobj, struct attribute *attr, int i)
+{
+	/* branches */
+	if (i == 0)
+		return x86_pmu.lbr_nr ? attr->mode : 0;
+
+	return (x86_pmu.flags & PMU_FL_BR_CNTR) ? attr->mode : 0;
+}
+
 static char pmu_name_str[30];
 
 static ssize_t pmu_name_show(struct device *cdev,
@@ -5558,6 +5663,15 @@ static struct attribute *intel_pmu_attrs[] = {
 	NULL,
 };
 
+static umode_t
+default_is_visible(struct kobject *kobj, struct attribute *attr, int i)
+{
+	if (attr == &dev_attr_allow_tsx_force_abort.attr)
+		return x86_pmu.flags & PMU_FL_TFA ? attr->mode : 0;
+
+	return attr->mode;
+}
+
 static umode_t
 tsx_is_visible(struct kobject *kobj, struct attribute *attr, int i)
 {
@@ -5579,27 +5693,12 @@ mem_is_visible(struct kobject *kobj, struct attribute *attr, int i)
 	return pebs_is_visible(kobj, attr, i);
 }
 
-static umode_t
-lbr_is_visible(struct kobject *kobj, struct attribute *attr, int i)
-{
-	return x86_pmu.lbr_nr ? attr->mode : 0;
-}
-
 static umode_t
 exra_is_visible(struct kobject *kobj, struct attribute *attr, int i)
 {
 	return x86_pmu.version >= 2 ? attr->mode : 0;
 }
 
-static umode_t
-default_is_visible(struct kobject *kobj, struct attribute *attr, int i)
-{
-	if (attr == &dev_attr_allow_tsx_force_abort.attr)
-		return x86_pmu.flags & PMU_FL_TFA ? attr->mode : 0;
-
-	return attr->mode;
-}
-
 static struct attribute_group group_events_td  = {
 	.name = "events",
 };
diff --git a/arch/x86/events/intel/cstate.c b/arch/x86/events/intel/cstate.c
index cbeb6d2bf5b4..4b50a3a9818a 100644
--- a/arch/x86/events/intel/cstate.c
+++ b/arch/x86/events/intel/cstate.c
@@ -41,7 +41,7 @@
  *	MSR_CORE_C1_RES: CORE C1 Residency Counter
  *			 perf code: 0x00
  *			 Available model: SLM,AMT,GLM,CNL,ICX,TNT,ADL,RPL
- *					  MTL
+ *					  MTL,SRF,GRR
  *			 Scope: Core (each processor core has a MSR)
  *	MSR_CORE_C3_RESIDENCY: CORE C3 Residency Counter
  *			       perf code: 0x01
@@ -52,7 +52,8 @@
  *			       perf code: 0x02
  *			       Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,
  *						SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX,
- *						TGL,TNT,RKL,ADL,RPL,SPR,MTL
+ *						TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF,
+ *						GRR
  *			       Scope: Core
  *	MSR_CORE_C7_RESIDENCY: CORE C7 Residency Counter
  *			       perf code: 0x03
@@ -75,7 +76,7 @@
  *			       perf code: 0x02
  *			       Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,
  *						SKL,KNL,GLM,CNL,KBL,CML,ICL,ICX,
- *						TGL,TNT,RKL,ADL,RPL,SPR,MTL
+ *						TGL,TNT,RKL,ADL,RPL,SPR,MTL,SRF
  *			       Scope: Package (physical package)
  *	MSR_PKG_C7_RESIDENCY:  Package C7 Residency Counter.
  *			       perf code: 0x03
@@ -97,6 +98,10 @@
  *			       Available model: HSW ULT,KBL,GLM,CNL,CML,ICL,TGL,
  *						TNT,RKL,ADL,RPL,MTL
  *			       Scope: Package (physical package)
+ *	MSR_MODULE_C6_RES_MS:  Module C6 Residency Counter.
+ *			       perf code: 0x00
+ *			       Available model: SRF,GRR
+ *			       Scope: A cluster of cores shared L2 cache
  *
  */
 
@@ -130,6 +135,7 @@ static ssize_t cstate_get_attr_cpumask(struct device *dev,
 struct cstate_model {
 	unsigned long		core_events;
 	unsigned long		pkg_events;
+	unsigned long		module_events;
 	unsigned long		quirks;
 };
 
@@ -189,20 +195,20 @@ static struct attribute *attrs_empty[] = {
  * "events" group (with empty attrs) before updating
  * it with detected events.
  */
-static struct attribute_group core_events_attr_group = {
+static struct attribute_group cstate_events_attr_group = {
 	.name = "events",
 	.attrs = attrs_empty,
 };
 
-DEFINE_CSTATE_FORMAT_ATTR(core_event, event, "config:0-63");
-static struct attribute *core_format_attrs[] = {
-	&format_attr_core_event.attr,
+DEFINE_CSTATE_FORMAT_ATTR(cstate_event, event, "config:0-63");
+static struct attribute *cstate_format_attrs[] = {
+	&format_attr_cstate_event.attr,
 	NULL,
 };
 
-static struct attribute_group core_format_attr_group = {
+static struct attribute_group cstate_format_attr_group = {
 	.name = "format",
-	.attrs = core_format_attrs,
+	.attrs = cstate_format_attrs,
 };
 
 static cpumask_t cstate_core_cpu_mask;
@@ -217,9 +223,9 @@ static struct attribute_group cpumask_attr_group = {
 	.attrs = cstate_cpumask_attrs,
 };
 
-static const struct attribute_group *core_attr_groups[] = {
-	&core_events_attr_group,
-	&core_format_attr_group,
+static const struct attribute_group *cstate_attr_groups[] = {
+	&cstate_events_attr_group,
+	&cstate_format_attr_group,
 	&cpumask_attr_group,
 	NULL,
 };
@@ -268,30 +274,30 @@ static struct perf_msr pkg_msr[] = {
 	[PERF_CSTATE_PKG_C10_RES] = { MSR_PKG_C10_RESIDENCY,	&group_cstate_pkg_c10,	test_msr },
 };
 
-static struct attribute_group pkg_events_attr_group = {
-	.name = "events",
-	.attrs = attrs_empty,
-};
+static cpumask_t cstate_pkg_cpu_mask;
 
-DEFINE_CSTATE_FORMAT_ATTR(pkg_event, event, "config:0-63");
-static struct attribute *pkg_format_attrs[] = {
-	&format_attr_pkg_event.attr,
-	NULL,
-};
-static struct attribute_group pkg_format_attr_group = {
-	.name = "format",
-	.attrs = pkg_format_attrs,
+/* cstate_module PMU */
+static struct pmu cstate_module_pmu;
+static bool has_cstate_module;
+
+enum perf_cstate_module_events {
+	PERF_CSTATE_MODULE_C6_RES = 0,
+
+	PERF_CSTATE_MODULE_EVENT_MAX,
 };
 
-static cpumask_t cstate_pkg_cpu_mask;
+PMU_EVENT_ATTR_STRING(c6-residency, attr_cstate_module_c6, "event=0x00");
 
-static const struct attribute_group *pkg_attr_groups[] = {
-	&pkg_events_attr_group,
-	&pkg_format_attr_group,
-	&cpumask_attr_group,
-	NULL,
+static unsigned long module_msr_mask;
+
+PMU_EVENT_GROUP(events, cstate_module_c6);
+
+static struct perf_msr module_msr[] = {
+	[PERF_CSTATE_MODULE_C6_RES]  = { MSR_MODULE_C6_RES_MS,	&group_cstate_module_c6,	test_msr },
 };
 
+static cpumask_t cstate_module_cpu_mask;
+
 static ssize_t cstate_get_attr_cpumask(struct device *dev,
 				       struct device_attribute *attr,
 				       char *buf)
@@ -302,6 +308,8 @@ static ssize_t cstate_get_attr_cpumask(struct device *dev,
 		return cpumap_print_to_pagebuf(true, buf, &cstate_core_cpu_mask);
 	else if (pmu == &cstate_pkg_pmu)
 		return cpumap_print_to_pagebuf(true, buf, &cstate_pkg_cpu_mask);
+	else if (pmu == &cstate_module_pmu)
+		return cpumap_print_to_pagebuf(true, buf, &cstate_module_cpu_mask);
 	else
 		return 0;
 }
@@ -342,6 +350,15 @@ static int cstate_pmu_event_init(struct perf_event *event)
 		event->hw.event_base = pkg_msr[cfg].msr;
 		cpu = cpumask_any_and(&cstate_pkg_cpu_mask,
 				      topology_die_cpumask(event->cpu));
+	} else if (event->pmu == &cstate_module_pmu) {
+		if (cfg >= PERF_CSTATE_MODULE_EVENT_MAX)
+			return -EINVAL;
+		cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_MODULE_EVENT_MAX);
+		if (!(module_msr_mask & (1 << cfg)))
+			return -EINVAL;
+		event->hw.event_base = module_msr[cfg].msr;
+		cpu = cpumask_any_and(&cstate_module_cpu_mask,
+				      topology_cluster_cpumask(event->cpu));
 	} else {
 		return -ENOENT;
 	}
@@ -429,6 +446,17 @@ static int cstate_cpu_exit(unsigned int cpu)
 			perf_pmu_migrate_context(&cstate_pkg_pmu, cpu, target);
 		}
 	}
+
+	if (has_cstate_module &&
+	    cpumask_test_and_clear_cpu(cpu, &cstate_module_cpu_mask)) {
+
+		target = cpumask_any_but(topology_cluster_cpumask(cpu), cpu);
+		/* Migrate events if there is a valid target */
+		if (target < nr_cpu_ids) {
+			cpumask_set_cpu(target, &cstate_module_cpu_mask);
+			perf_pmu_migrate_context(&cstate_module_pmu, cpu, target);
+		}
+	}
 	return 0;
 }
 
@@ -455,6 +483,15 @@ static int cstate_cpu_init(unsigned int cpu)
 	if (has_cstate_pkg && target >= nr_cpu_ids)
 		cpumask_set_cpu(cpu, &cstate_pkg_cpu_mask);
 
+	/*
+	 * If this is the first online thread of that cluster, set it
+	 * in the cluster cpu mask as the designated reader.
+	 */
+	target = cpumask_any_and(&cstate_module_cpu_mask,
+				 topology_cluster_cpumask(cpu));
+	if (has_cstate_module && target >= nr_cpu_ids)
+		cpumask_set_cpu(cpu, &cstate_module_cpu_mask);
+
 	return 0;
 }
 
@@ -477,8 +514,13 @@ static const struct attribute_group *pkg_attr_update[] = {
 	NULL,
 };
 
+static const struct attribute_group *module_attr_update[] = {
+	&group_cstate_module_c6,
+	NULL
+};
+
 static struct pmu cstate_core_pmu = {
-	.attr_groups	= core_attr_groups,
+	.attr_groups	= cstate_attr_groups,
 	.attr_update	= core_attr_update,
 	.name		= "cstate_core",
 	.task_ctx_nr	= perf_invalid_context,
@@ -493,7 +535,7 @@ static struct pmu cstate_core_pmu = {
 };
 
 static struct pmu cstate_pkg_pmu = {
-	.attr_groups	= pkg_attr_groups,
+	.attr_groups	= cstate_attr_groups,
 	.attr_update	= pkg_attr_update,
 	.name		= "cstate_pkg",
 	.task_ctx_nr	= perf_invalid_context,
@@ -507,6 +549,21 @@ static struct pmu cstate_pkg_pmu = {
 	.module		= THIS_MODULE,
 };
 
+static struct pmu cstate_module_pmu = {
+	.attr_groups	= cstate_attr_groups,
+	.attr_update	= module_attr_update,
+	.name		= "cstate_module",
+	.task_ctx_nr	= perf_invalid_context,
+	.event_init	= cstate_pmu_event_init,
+	.add		= cstate_pmu_event_add,
+	.del		= cstate_pmu_event_del,
+	.start		= cstate_pmu_event_start,
+	.stop		= cstate_pmu_event_stop,
+	.read		= cstate_pmu_event_update,
+	.capabilities	= PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE,
+	.module		= THIS_MODULE,
+};
+
 static const struct cstate_model nhm_cstates __initconst = {
 	.core_events		= BIT(PERF_CSTATE_CORE_C3_RES) |
 				  BIT(PERF_CSTATE_CORE_C6_RES),
@@ -621,6 +678,22 @@ static const struct cstate_model glm_cstates __initconst = {
 				  BIT(PERF_CSTATE_PKG_C10_RES),
 };
 
+static const struct cstate_model grr_cstates __initconst = {
+	.core_events		= BIT(PERF_CSTATE_CORE_C1_RES) |
+				  BIT(PERF_CSTATE_CORE_C6_RES),
+
+	.module_events		= BIT(PERF_CSTATE_MODULE_C6_RES),
+};
+
+static const struct cstate_model srf_cstates __initconst = {
+	.core_events		= BIT(PERF_CSTATE_CORE_C1_RES) |
+				  BIT(PERF_CSTATE_CORE_C6_RES),
+
+	.pkg_events		= BIT(PERF_CSTATE_PKG_C6_RES),
+
+	.module_events		= BIT(PERF_CSTATE_MODULE_C6_RES),
+};
+
 
 static const struct x86_cpu_id intel_cstates_match[] __initconst = {
 	X86_MATCH_INTEL_FAM6_MODEL(NEHALEM,		&nhm_cstates),
@@ -673,6 +746,8 @@ static const struct x86_cpu_id intel_cstates_match[] __initconst = {
 	X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT,	&glm_cstates),
 	X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_L,	&glm_cstates),
 	X86_MATCH_INTEL_FAM6_MODEL(ATOM_GRACEMONT,	&adl_cstates),
+	X86_MATCH_INTEL_FAM6_MODEL(ATOM_CRESTMONT_X,	&srf_cstates),
+	X86_MATCH_INTEL_FAM6_MODEL(ATOM_CRESTMONT,	&grr_cstates),
 
 	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_L,		&icl_cstates),
 	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE,		&icl_cstates),
@@ -714,10 +789,14 @@ static int __init cstate_probe(const struct cstate_model *cm)
 	pkg_msr_mask = perf_msr_probe(pkg_msr, PERF_CSTATE_PKG_EVENT_MAX,
 				      true, (void *) &cm->pkg_events);
 
+	module_msr_mask = perf_msr_probe(module_msr, PERF_CSTATE_MODULE_EVENT_MAX,
+				      true, (void *) &cm->module_events);
+
 	has_cstate_core = !!core_msr_mask;
 	has_cstate_pkg  = !!pkg_msr_mask;
+	has_cstate_module  = !!module_msr_mask;
 
-	return (has_cstate_core || has_cstate_pkg) ? 0 : -ENODEV;
+	return (has_cstate_core || has_cstate_pkg || has_cstate_module) ? 0 : -ENODEV;
 }
 
 static inline void cstate_cleanup(void)
@@ -730,6 +809,9 @@ static inline void cstate_cleanup(void)
 
 	if (has_cstate_pkg)
 		perf_pmu_unregister(&cstate_pkg_pmu);
+
+	if (has_cstate_module)
+		perf_pmu_unregister(&cstate_module_pmu);
 }
 
 static int __init cstate_init(void)
@@ -766,6 +848,16 @@ static int __init cstate_init(void)
 			return err;
 		}
 	}
+
+	if (has_cstate_module) {
+		err = perf_pmu_register(&cstate_module_pmu, cstate_module_pmu.name, -1);
+		if (err) {
+			has_cstate_module = false;
+			pr_info("Failed to register cstate cluster pmu\n");
+			cstate_cleanup();
+			return err;
+		}
+	}
 	return 0;
 }
 
diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c
index bf97ab904d40..d49d661ec0a7 100644
--- a/arch/x86/events/intel/ds.c
+++ b/arch/x86/events/intel/ds.c
@@ -1755,7 +1755,7 @@ static void setup_pebs_fixed_sample_data(struct perf_event *event,
 		setup_pebs_time(event, data, pebs->tsc);
 
 	if (has_branch_stack(event))
-		perf_sample_save_brstack(data, event, &cpuc->lbr_stack);
+		perf_sample_save_brstack(data, event, &cpuc->lbr_stack, NULL);
 }
 
 static void adaptive_pebs_save_regs(struct pt_regs *regs,
@@ -1912,7 +1912,7 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event,
 
 		if (has_branch_stack(event)) {
 			intel_pmu_store_pebs_lbrs(lbr);
-			perf_sample_save_brstack(data, event, &cpuc->lbr_stack);
+			intel_pmu_lbr_save_brstack(data, cpuc, event);
 		}
 	}
 
diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c
index c3b0d15a9841..78cd5084104e 100644
--- a/arch/x86/events/intel/lbr.c
+++ b/arch/x86/events/intel/lbr.c
@@ -676,6 +676,25 @@ void intel_pmu_lbr_del(struct perf_event *event)
 	WARN_ON_ONCE(cpuc->lbr_users < 0);
 	WARN_ON_ONCE(cpuc->lbr_pebs_users < 0);
 	perf_sched_cb_dec(event->pmu);
+
+	/*
+	 * The logged occurrences information is only valid for the
+	 * current LBR group. If another LBR group is scheduled in
+	 * later, the information from the stale LBRs will be wrongly
+	 * interpreted. Reset the LBRs here.
+	 *
+	 * Only clear once for a branch counter group with the leader
+	 * event. Because
+	 * - Cannot simply reset the LBRs with the !cpuc->lbr_users.
+	 *   Because it's possible that the last LBR user is not in a
+	 *   branch counter group, e.g., a branch_counters group +
+	 *   several normal LBR events.
+	 * - The LBR reset can be done with any one of the events in a
+	 *   branch counter group, since they are always scheduled together.
+	 *   It's easy to force the leader event an LBR event.
+	 */
+	if (is_branch_counters_group(event) && event == event->group_leader)
+		intel_pmu_lbr_reset();
 }
 
 static inline bool vlbr_exclude_host(void)
@@ -866,6 +885,8 @@ static __always_inline u16 get_lbr_cycles(u64 info)
 	return cycles;
 }
 
+static_assert((64 - PERF_BRANCH_ENTRY_INFO_BITS_MAX) > LBR_INFO_BR_CNTR_NUM * LBR_INFO_BR_CNTR_BITS);
+
 static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc,
 				struct lbr_entry *entries)
 {
@@ -898,11 +919,67 @@ static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc,
 		e->abort	= !!(info & LBR_INFO_ABORT);
 		e->cycles	= get_lbr_cycles(info);
 		e->type		= get_lbr_br_type(info);
+
+		/*
+		 * Leverage the reserved field of cpuc->lbr_entries[i] to
+		 * temporarily store the branch counters information.
+		 * The later code will decide what content can be disclosed
+		 * to the perf tool. Pleae see intel_pmu_lbr_counters_reorder().
+		 */
+		e->reserved	= (info >> LBR_INFO_BR_CNTR_OFFSET) & LBR_INFO_BR_CNTR_FULL_MASK;
 	}
 
 	cpuc->lbr_stack.nr = i;
 }
 
+/*
+ * The enabled order may be different from the counter order.
+ * Update the lbr_counters with the enabled order.
+ */
+static void intel_pmu_lbr_counters_reorder(struct cpu_hw_events *cpuc,
+					   struct perf_event *event)
+{
+	int i, j, pos = 0, order[X86_PMC_IDX_MAX];
+	struct perf_event *leader, *sibling;
+	u64 src, dst, cnt;
+
+	leader = event->group_leader;
+	if (branch_sample_counters(leader))
+		order[pos++] = leader->hw.idx;
+
+	for_each_sibling_event(sibling, leader) {
+		if (!branch_sample_counters(sibling))
+			continue;
+		order[pos++] = sibling->hw.idx;
+	}
+
+	WARN_ON_ONCE(!pos);
+
+	for (i = 0; i < cpuc->lbr_stack.nr; i++) {
+		src = cpuc->lbr_entries[i].reserved;
+		dst = 0;
+		for (j = 0; j < pos; j++) {
+			cnt = (src >> (order[j] * LBR_INFO_BR_CNTR_BITS)) & LBR_INFO_BR_CNTR_MASK;
+			dst |= cnt << j * LBR_INFO_BR_CNTR_BITS;
+		}
+		cpuc->lbr_counters[i] = dst;
+		cpuc->lbr_entries[i].reserved = 0;
+	}
+}
+
+void intel_pmu_lbr_save_brstack(struct perf_sample_data *data,
+				struct cpu_hw_events *cpuc,
+				struct perf_event *event)
+{
+	if (is_branch_counters_group(event)) {
+		intel_pmu_lbr_counters_reorder(cpuc, event);
+		perf_sample_save_brstack(data, event, &cpuc->lbr_stack, cpuc->lbr_counters);
+		return;
+	}
+
+	perf_sample_save_brstack(data, event, &cpuc->lbr_stack, NULL);
+}
+
 static void intel_pmu_arch_lbr_read(struct cpu_hw_events *cpuc)
 {
 	intel_pmu_store_lbr(cpuc, NULL);
@@ -1173,8 +1250,10 @@ intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)
 	for (i = 0; i < cpuc->lbr_stack.nr; ) {
 		if (!cpuc->lbr_entries[i].from) {
 			j = i;
-			while (++j < cpuc->lbr_stack.nr)
+			while (++j < cpuc->lbr_stack.nr) {
 				cpuc->lbr_entries[j-1] = cpuc->lbr_entries[j];
+				cpuc->lbr_counters[j-1] = cpuc->lbr_counters[j];
+			}
 			cpuc->lbr_stack.nr--;
 			if (!cpuc->lbr_entries[i].from)
 				continue;
@@ -1525,8 +1604,12 @@ void __init intel_pmu_arch_lbr_init(void)
 	x86_pmu.lbr_mispred = ecx.split.lbr_mispred;
 	x86_pmu.lbr_timed_lbr = ecx.split.lbr_timed_lbr;
 	x86_pmu.lbr_br_type = ecx.split.lbr_br_type;
+	x86_pmu.lbr_counters = ecx.split.lbr_counters;
 	x86_pmu.lbr_nr = lbr_nr;
 
+	if (!!x86_pmu.lbr_counters)
+		x86_pmu.flags |= PMU_FL_BR_CNTR;
+
 	if (x86_pmu.lbr_mispred)
 		static_branch_enable(&x86_lbr_mispred);
 	if (x86_pmu.lbr_timed_lbr)
diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h
index 53dd5d495ba6..fb56518356ec 100644
--- a/arch/x86/events/perf_event.h
+++ b/arch/x86/events/perf_event.h
@@ -110,6 +110,11 @@ static inline bool is_topdown_event(struct perf_event *event)
 	return is_metric_event(event) || is_slots_event(event);
 }
 
+static inline bool is_branch_counters_group(struct perf_event *event)
+{
+	return event->group_leader->hw.flags & PERF_X86_EVENT_BRANCH_COUNTERS;
+}
+
 struct amd_nb {
 	int nb_id;  /* NorthBridge id */
 	int refcnt; /* reference count */
@@ -283,6 +288,7 @@ struct cpu_hw_events {
 	int				lbr_pebs_users;
 	struct perf_branch_stack	lbr_stack;
 	struct perf_branch_entry	lbr_entries[MAX_LBR_ENTRIES];
+	u64				lbr_counters[MAX_LBR_ENTRIES]; /* branch stack extra */
 	union {
 		struct er_account		*lbr_sel;
 		struct er_account		*lbr_ctl;
@@ -888,6 +894,7 @@ struct x86_pmu {
 	unsigned int	lbr_mispred:1;
 	unsigned int	lbr_timed_lbr:1;
 	unsigned int	lbr_br_type:1;
+	unsigned int	lbr_counters:4;
 
 	void		(*lbr_reset)(void);
 	void		(*lbr_read)(struct cpu_hw_events *cpuc);
@@ -1012,6 +1019,7 @@ do {									\
 #define PMU_FL_INSTR_LATENCY	0x80 /* Support Instruction Latency in PEBS Memory Info Record */
 #define PMU_FL_MEM_LOADS_AUX	0x100 /* Require an auxiliary event for the complete memory info */
 #define PMU_FL_RETIRE_LATENCY	0x200 /* Support Retire Latency in PEBS */
+#define PMU_FL_BR_CNTR		0x400 /* Support branch counter logging */
 
 #define EVENT_VAR(_id)  event_attr_##_id
 #define EVENT_PTR(_id) &event_attr_##_id.attr.attr
@@ -1552,6 +1560,10 @@ void intel_pmu_store_pebs_lbrs(struct lbr_entry *lbr);
 
 void intel_ds_init(void);
 
+void intel_pmu_lbr_save_brstack(struct perf_sample_data *data,
+				struct cpu_hw_events *cpuc,
+				struct perf_event *event);
+
 void intel_pmu_lbr_swap_task_ctx(struct perf_event_pmu_context *prev_epc,
 				 struct perf_event_pmu_context *next_epc);
 
diff --git a/arch/x86/events/perf_event_flags.h b/arch/x86/events/perf_event_flags.h
index 1dc19b9b4426..6c977c19f2cd 100644
--- a/arch/x86/events/perf_event_flags.h
+++ b/arch/x86/events/perf_event_flags.h
@@ -20,3 +20,5 @@ PERF_ARCH(TOPDOWN,		0x04000) /* Count Topdown slots/metrics events */
 PERF_ARCH(PEBS_STLAT,		0x08000) /* st+stlat data address sampling */
 PERF_ARCH(AMD_BRS,		0x10000) /* AMD Branch Sampling */
 PERF_ARCH(PEBS_LAT_HYBRID,	0x20000) /* ld and st lat for hybrid */
+PERF_ARCH(NEEDS_BRANCH_STACK,	0x40000) /* require branch stack setup */
+PERF_ARCH(BRANCH_COUNTERS,	0x80000) /* logs the counters in the extra space of each branch */
diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c
index 21556ad87f4b..8f3a4d16bb79 100644
--- a/arch/x86/hyperv/hv_init.c
+++ b/arch/x86/hyperv/hv_init.c
@@ -15,6 +15,7 @@
 #include <linux/io.h>
 #include <asm/apic.h>
 #include <asm/desc.h>
+#include <asm/e820/api.h>
 #include <asm/sev.h>
 #include <asm/ibt.h>
 #include <asm/hypervisor.h>
@@ -286,15 +287,31 @@ static int hv_cpu_die(unsigned int cpu)
 
 static int __init hv_pci_init(void)
 {
-	int gen2vm = efi_enabled(EFI_BOOT);
+	bool gen2vm = efi_enabled(EFI_BOOT);
 
 	/*
-	 * For Generation-2 VM, we exit from pci_arch_init() by returning 0.
-	 * The purpose is to suppress the harmless warning:
+	 * A Generation-2 VM doesn't support legacy PCI/PCIe, so both
+	 * raw_pci_ops and raw_pci_ext_ops are NULL, and pci_subsys_init() ->
+	 * pcibios_init() doesn't call pcibios_resource_survey() ->
+	 * e820__reserve_resources_late(); as a result, any emulated persistent
+	 * memory of E820_TYPE_PRAM (12) via the kernel parameter
+	 * memmap=nn[KMG]!ss is not added into iomem_resource and hence can't be
+	 * detected by register_e820_pmem(). Fix this by directly calling
+	 * e820__reserve_resources_late() here: e820__reserve_resources_late()
+	 * depends on e820__reserve_resources(), which has been called earlier
+	 * from setup_arch(). Note: e820__reserve_resources_late() also adds
+	 * any memory of E820_TYPE_PMEM (7) into iomem_resource, and
+	 * acpi_nfit_register_region() -> acpi_nfit_insert_resource() ->
+	 * region_intersects() returns REGION_INTERSECTS, so the memory of
+	 * E820_TYPE_PMEM won't get added twice.
+	 *
+	 * We return 0 here so that pci_arch_init() won't print the warning:
 	 * "PCI: Fatal: No config space access function found"
 	 */
-	if (gen2vm)
+	if (gen2vm) {
+		e820__reserve_resources_late();
 		return 0;
+	}
 
 	/* For Generation-1 VM, we'll proceed in pci_arch_init().  */
 	return 1;
diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h
index d21f48f1c242..9d159b771dc8 100644
--- a/arch/x86/include/asm/apic.h
+++ b/arch/x86/include/asm/apic.h
@@ -272,8 +272,6 @@ struct apic {
 	void	(*send_IPI_all)(int vector);
 	void	(*send_IPI_self)(int vector);
 
-	enum apic_delivery_modes delivery_mode;
-
 	u32	disable_esr		: 1,
 		dest_mode_logical	: 1,
 		x2apic_set_max_apicid	: 1,
diff --git a/arch/x86/include/asm/apicdef.h b/arch/x86/include/asm/apicdef.h
index 4b125e5b3187..094106b6a538 100644
--- a/arch/x86/include/asm/apicdef.h
+++ b/arch/x86/include/asm/apicdef.h
@@ -20,6 +20,13 @@
  */
 #define IO_APIC_SLOT_SIZE		1024
 
+#define APIC_DELIVERY_MODE_FIXED	0
+#define APIC_DELIVERY_MODE_LOWESTPRIO	1
+#define APIC_DELIVERY_MODE_SMI		2
+#define APIC_DELIVERY_MODE_NMI		4
+#define APIC_DELIVERY_MODE_INIT		5
+#define APIC_DELIVERY_MODE_EXTINT	7
+
 #define	APIC_ID		0x20
 
 #define	APIC_LVR	0x30
@@ -165,279 +172,10 @@
 #define APIC_CPUID(apicid)	((apicid) & XAPIC_DEST_CPUS_MASK)
 #define NUM_APIC_CLUSTERS	((BAD_APICID + 1) >> XAPIC_DEST_CPUS_SHIFT)
 
-#ifndef __ASSEMBLY__
-/*
- * the local APIC register structure, memory mapped. Not terribly well
- * tested, but we might eventually use this one in the future - the
- * problem why we cannot use it right now is the P5 APIC, it has an
- * errata which cannot take 8-bit reads and writes, only 32-bit ones ...
- */
-#define u32 unsigned int
-
-struct local_apic {
-
-/*000*/	struct { u32 __reserved[4]; } __reserved_01;
-
-/*010*/	struct { u32 __reserved[4]; } __reserved_02;
-
-/*020*/	struct { /* APIC ID Register */
-		u32   __reserved_1	: 24,
-			phys_apic_id	:  4,
-			__reserved_2	:  4;
-		u32 __reserved[3];
-	} id;
-
-/*030*/	const
-	struct { /* APIC Version Register */
-		u32   version		:  8,
-			__reserved_1	:  8,
-			max_lvt		:  8,
-			__reserved_2	:  8;
-		u32 __reserved[3];
-	} version;
-
-/*040*/	struct { u32 __reserved[4]; } __reserved_03;
-
-/*050*/	struct { u32 __reserved[4]; } __reserved_04;
-
-/*060*/	struct { u32 __reserved[4]; } __reserved_05;
-
-/*070*/	struct { u32 __reserved[4]; } __reserved_06;
-
-/*080*/	struct { /* Task Priority Register */
-		u32   priority	:  8,
-			__reserved_1	: 24;
-		u32 __reserved_2[3];
-	} tpr;
-
-/*090*/	const
-	struct { /* Arbitration Priority Register */
-		u32   priority	:  8,
-			__reserved_1	: 24;
-		u32 __reserved_2[3];
-	} apr;
-
-/*0A0*/	const
-	struct { /* Processor Priority Register */
-		u32   priority	:  8,
-			__reserved_1	: 24;
-		u32 __reserved_2[3];
-	} ppr;
-
-/*0B0*/	struct { /* End Of Interrupt Register */
-		u32   eoi;
-		u32 __reserved[3];
-	} eoi;
-
-/*0C0*/	struct { u32 __reserved[4]; } __reserved_07;
-
-/*0D0*/	struct { /* Logical Destination Register */
-		u32   __reserved_1	: 24,
-			logical_dest	:  8;
-		u32 __reserved_2[3];
-	} ldr;
-
-/*0E0*/	struct { /* Destination Format Register */
-		u32   __reserved_1	: 28,
-			model		:  4;
-		u32 __reserved_2[3];
-	} dfr;
-
-/*0F0*/	struct { /* Spurious Interrupt Vector Register */
-		u32	spurious_vector	:  8,
-			apic_enabled	:  1,
-			focus_cpu	:  1,
-			__reserved_2	: 22;
-		u32 __reserved_3[3];
-	} svr;
-
-/*100*/	struct { /* In Service Register */
-/*170*/		u32 bitfield;
-		u32 __reserved[3];
-	} isr [8];
-
-/*180*/	struct { /* Trigger Mode Register */
-/*1F0*/		u32 bitfield;
-		u32 __reserved[3];
-	} tmr [8];
-
-/*200*/	struct { /* Interrupt Request Register */
-/*270*/		u32 bitfield;
-		u32 __reserved[3];
-	} irr [8];
-
-/*280*/	union { /* Error Status Register */
-		struct {
-			u32   send_cs_error			:  1,
-				receive_cs_error		:  1,
-				send_accept_error		:  1,
-				receive_accept_error		:  1,
-				__reserved_1			:  1,
-				send_illegal_vector		:  1,
-				receive_illegal_vector		:  1,
-				illegal_register_address	:  1,
-				__reserved_2			: 24;
-			u32 __reserved_3[3];
-		} error_bits;
-		struct {
-			u32 errors;
-			u32 __reserved_3[3];
-		} all_errors;
-	} esr;
-
-/*290*/	struct { u32 __reserved[4]; } __reserved_08;
-
-/*2A0*/	struct { u32 __reserved[4]; } __reserved_09;
-
-/*2B0*/	struct { u32 __reserved[4]; } __reserved_10;
-
-/*2C0*/	struct { u32 __reserved[4]; } __reserved_11;
-
-/*2D0*/	struct { u32 __reserved[4]; } __reserved_12;
-
-/*2E0*/	struct { u32 __reserved[4]; } __reserved_13;
-
-/*2F0*/	struct { u32 __reserved[4]; } __reserved_14;
-
-/*300*/	struct { /* Interrupt Command Register 1 */
-		u32   vector			:  8,
-			delivery_mode		:  3,
-			destination_mode	:  1,
-			delivery_status		:  1,
-			__reserved_1		:  1,
-			level			:  1,
-			trigger			:  1,
-			__reserved_2		:  2,
-			shorthand		:  2,
-			__reserved_3		:  12;
-		u32 __reserved_4[3];
-	} icr1;
-
-/*310*/	struct { /* Interrupt Command Register 2 */
-		union {
-			u32   __reserved_1	: 24,
-				phys_dest	:  4,
-				__reserved_2	:  4;
-			u32   __reserved_3	: 24,
-				logical_dest	:  8;
-		} dest;
-		u32 __reserved_4[3];
-	} icr2;
-
-/*320*/	struct { /* LVT - Timer */
-		u32   vector		:  8,
-			__reserved_1	:  4,
-			delivery_status	:  1,
-			__reserved_2	:  3,
-			mask		:  1,
-			timer_mode	:  1,
-			__reserved_3	: 14;
-		u32 __reserved_4[3];
-	} lvt_timer;
-
-/*330*/	struct { /* LVT - Thermal Sensor */
-		u32  vector		:  8,
-			delivery_mode	:  3,
-			__reserved_1	:  1,
-			delivery_status	:  1,
-			__reserved_2	:  3,
-			mask		:  1,
-			__reserved_3	: 15;
-		u32 __reserved_4[3];
-	} lvt_thermal;
-
-/*340*/	struct { /* LVT - Performance Counter */
-		u32   vector		:  8,
-			delivery_mode	:  3,
-			__reserved_1	:  1,
-			delivery_status	:  1,
-			__reserved_2	:  3,
-			mask		:  1,
-			__reserved_3	: 15;
-		u32 __reserved_4[3];
-	} lvt_pc;
-
-/*350*/	struct { /* LVT - LINT0 */
-		u32   vector		:  8,
-			delivery_mode	:  3,
-			__reserved_1	:  1,
-			delivery_status	:  1,
-			polarity	:  1,
-			remote_irr	:  1,
-			trigger		:  1,
-			mask		:  1,
-			__reserved_2	: 15;
-		u32 __reserved_3[3];
-	} lvt_lint0;
-
-/*360*/	struct { /* LVT - LINT1 */
-		u32   vector		:  8,
-			delivery_mode	:  3,
-			__reserved_1	:  1,
-			delivery_status	:  1,
-			polarity	:  1,
-			remote_irr	:  1,
-			trigger		:  1,
-			mask		:  1,
-			__reserved_2	: 15;
-		u32 __reserved_3[3];
-	} lvt_lint1;
-
-/*370*/	struct { /* LVT - Error */
-		u32   vector		:  8,
-			__reserved_1	:  4,
-			delivery_status	:  1,
-			__reserved_2	:  3,
-			mask		:  1,
-			__reserved_3	: 15;
-		u32 __reserved_4[3];
-	} lvt_error;
-
-/*380*/	struct { /* Timer Initial Count Register */
-		u32   initial_count;
-		u32 __reserved_2[3];
-	} timer_icr;
-
-/*390*/	const
-	struct { /* Timer Current Count Register */
-		u32   curr_count;
-		u32 __reserved_2[3];
-	} timer_ccr;
-
-/*3A0*/	struct { u32 __reserved[4]; } __reserved_16;
-
-/*3B0*/	struct { u32 __reserved[4]; } __reserved_17;
-
-/*3C0*/	struct { u32 __reserved[4]; } __reserved_18;
-
-/*3D0*/	struct { u32 __reserved[4]; } __reserved_19;
-
-/*3E0*/	struct { /* Timer Divide Configuration Register */
-		u32   divisor		:  4,
-			__reserved_1	: 28;
-		u32 __reserved_2[3];
-	} timer_dcr;
-
-/*3F0*/	struct { u32 __reserved[4]; } __reserved_20;
-
-} __attribute__ ((packed));
-
-#undef u32
-
 #ifdef CONFIG_X86_32
  #define BAD_APICID 0xFFu
 #else
  #define BAD_APICID 0xFFFFu
 #endif
 
-enum apic_delivery_modes {
-	APIC_DELIVERY_MODE_FIXED	= 0,
-	APIC_DELIVERY_MODE_LOWESTPRIO   = 1,
-	APIC_DELIVERY_MODE_SMI		= 2,
-	APIC_DELIVERY_MODE_NMI		= 4,
-	APIC_DELIVERY_MODE_INIT		= 5,
-	APIC_DELIVERY_MODE_EXTINT	= 7,
-};
-
-#endif /* !__ASSEMBLY__ */
 #endif /* _ASM_X86_APICDEF_H */
diff --git a/arch/x86/include/asm/barrier.h b/arch/x86/include/asm/barrier.h
index 35389b2af88e..0216f63a366b 100644
--- a/arch/x86/include/asm/barrier.h
+++ b/arch/x86/include/asm/barrier.h
@@ -81,22 +81,4 @@ do {									\
 
 #include <asm-generic/barrier.h>
 
-/*
- * Make previous memory operations globally visible before
- * a WRMSR.
- *
- * MFENCE makes writes visible, but only affects load/store
- * instructions.  WRMSR is unfortunately not a load/store
- * instruction and is unaffected by MFENCE.  The LFENCE ensures
- * that the WRMSR is not reordered.
- *
- * Most WRMSRs are full serializing instructions themselves and
- * do not require this barrier.  This is only required for the
- * IA32_TSC_DEADLINE and X2APIC MSRs.
- */
-static inline void weak_wrmsr_fence(void)
-{
-	asm volatile("mfence; lfence" : : : "memory");
-}
-
 #endif /* _ASM_X86_BARRIER_H */
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 4af140cf5719..3e973ff20e23 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -308,10 +308,10 @@
 #define X86_FEATURE_SMBA		(11*32+21) /* "" Slow Memory Bandwidth Allocation */
 #define X86_FEATURE_BMEC		(11*32+22) /* "" Bandwidth Monitoring Event Configuration */
 #define X86_FEATURE_USER_SHSTK		(11*32+23) /* Shadow stack support for user mode applications */
-
 #define X86_FEATURE_SRSO		(11*32+24) /* "" AMD BTB untrain RETs */
 #define X86_FEATURE_SRSO_ALIAS		(11*32+25) /* "" AMD BTB untrain RETs through aliasing */
 #define X86_FEATURE_IBPB_ON_VMEXIT	(11*32+26) /* "" Issue an IBPB only on VMEXIT */
+#define X86_FEATURE_APIC_MSRS_FENCE	(11*32+27) /* "" IA32_TSC_DEADLINE and X2APIC MSRs need fencing */
 
 /* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */
 #define X86_FEATURE_AVX_VNNI		(12*32+ 4) /* AVX VNNI instructions */
diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
index a1168e7b69e5..0538d2436673 100644
--- a/arch/x86/include/asm/current.h
+++ b/arch/x86/include/asm/current.h
@@ -36,8 +36,15 @@ static_assert(sizeof(struct pcpu_hot) == 64);
 
 DECLARE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot);
 
+/* const-qualified alias to pcpu_hot, aliased by linker. */
+DECLARE_PER_CPU_ALIGNED(const struct pcpu_hot __percpu_seg_override,
+			const_pcpu_hot);
+
 static __always_inline struct task_struct *get_current(void)
 {
+	if (IS_ENABLED(CONFIG_USE_X86_SEG_SUPPORT))
+		return const_pcpu_hot.current_task;
+
 	return this_cpu_read_stable(pcpu_hot.current_task);
 }
 
diff --git a/arch/x86/include/asm/fpu/sched.h b/arch/x86/include/asm/fpu/sched.h
index ca6e5e5f16b2..c485f1944c5f 100644
--- a/arch/x86/include/asm/fpu/sched.h
+++ b/arch/x86/include/asm/fpu/sched.h
@@ -37,10 +37,12 @@ extern void fpu_flush_thread(void);
  * The FPU context is only stored/restored for a user task and
  * PF_KTHREAD is used to distinguish between kernel and user threads.
  */
-static inline void switch_fpu_prepare(struct fpu *old_fpu, int cpu)
+static inline void switch_fpu_prepare(struct task_struct *old, int cpu)
 {
 	if (cpu_feature_enabled(X86_FEATURE_FPU) &&
-	    !(current->flags & (PF_KTHREAD | PF_USER_WORKER))) {
+	    !(old->flags & (PF_KTHREAD | PF_USER_WORKER))) {
+		struct fpu *old_fpu = &old->thread.fpu;
+
 		save_fpregs_to_fpstate(old_fpu);
 		/*
 		 * The save operation preserved register state, so the
@@ -60,10 +62,10 @@ static inline void switch_fpu_prepare(struct fpu *old_fpu, int cpu)
  * Delay loading of the complete FPU state until the return to userland.
  * PKRU is handled separately.
  */
-static inline void switch_fpu_finish(void)
+static inline void switch_fpu_finish(struct task_struct *new)
 {
 	if (cpu_feature_enabled(X86_FEATURE_FPU))
-		set_thread_flag(TIF_NEED_FPU_LOAD);
+		set_tsk_thread_flag(new, TIF_NEED_FPU_LOAD);
 }
 
 #endif /* _ASM_X86_FPU_SCHED_H */
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index d7036982332e..a565a2e70f30 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1255,6 +1255,7 @@ enum kvm_apicv_inhibit {
 };
 
 struct kvm_arch {
+	unsigned long vm_type;
 	unsigned long n_used_mmu_pages;
 	unsigned long n_requested_mmu_pages;
 	unsigned long n_max_mmu_pages;
@@ -1848,6 +1849,9 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu);
 void kvm_mmu_init_vm(struct kvm *kvm);
 void kvm_mmu_uninit_vm(struct kvm *kvm);
 
+void kvm_mmu_init_memslot_memory_attributes(struct kvm *kvm,
+					    struct kvm_memory_slot *slot);
+
 void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu);
 void kvm_mmu_reset_context(struct kvm_vcpu *vcpu);
 void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
@@ -2086,6 +2090,12 @@ void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd);
 void kvm_configure_mmu(bool enable_tdp, int tdp_forced_root_level,
 		       int tdp_max_root_level, int tdp_huge_page_level);
 
+#ifdef CONFIG_KVM_PRIVATE_MEM
+#define kvm_arch_has_private_mem(kvm) ((kvm)->arch.vm_type != KVM_X86_DEFAULT_VM)
+#else
+#define kvm_arch_has_private_mem(kvm) false
+#endif
+
 static inline u16 kvm_read_ldt(void)
 {
 	u16 ldt;
@@ -2133,16 +2143,15 @@ enum {
 #define HF_SMM_MASK		(1 << 1)
 #define HF_SMM_INSIDE_NMI_MASK	(1 << 2)
 
-# define __KVM_VCPU_MULTIPLE_ADDRESS_SPACE
-# define KVM_ADDRESS_SPACE_NUM 2
+# define KVM_MAX_NR_ADDRESS_SPACES	2
+/* SMM is currently unsupported for guests with private memory. */
+# define kvm_arch_nr_memslot_as_ids(kvm) (kvm_arch_has_private_mem(kvm) ? 1 : 2)
 # define kvm_arch_vcpu_memslots_id(vcpu) ((vcpu)->arch.hflags & HF_SMM_MASK ? 1 : 0)
 # define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, (role).smm)
 #else
 # define kvm_memslots_for_spte_role(kvm, role) __kvm_memslots(kvm, 0)
 #endif
 
-#define KVM_ARCH_WANT_MMU_NOTIFIER
-
 int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v);
 int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu);
 int kvm_cpu_has_extint(struct kvm_vcpu *v);
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index 1d51e1850ed0..737a52b89e64 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -237,6 +237,11 @@
 #define LBR_INFO_CYCLES			0xffff
 #define LBR_INFO_BR_TYPE_OFFSET		56
 #define LBR_INFO_BR_TYPE		(0xfull << LBR_INFO_BR_TYPE_OFFSET)
+#define LBR_INFO_BR_CNTR_OFFSET		32
+#define LBR_INFO_BR_CNTR_NUM		4
+#define LBR_INFO_BR_CNTR_BITS		2
+#define LBR_INFO_BR_CNTR_MASK		GENMASK_ULL(LBR_INFO_BR_CNTR_BITS - 1, 0)
+#define LBR_INFO_BR_CNTR_FULL_MASK	GENMASK_ULL(LBR_INFO_BR_CNTR_NUM * LBR_INFO_BR_CNTR_BITS - 1, 0)
 
 #define MSR_ARCH_LBR_CTL		0x000014ce
 #define ARCH_LBR_CTL_LBREN		BIT(0)
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index 6c8ff12140ae..693c61dbdd9c 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -743,16 +743,14 @@ void native_pv_lock_init(void) __init;
 
 #else  /* __ASSEMBLY__ */
 
-#define _PVSITE(ptype, ops, word, algn)		\
+#define _PVSITE(ptype, ops)			\
 771:;						\
 	ops;					\
 772:;						\
 	.pushsection .parainstructions,"a";	\
-	 .align	algn;				\
-	 word 771b;				\
+	 .long 771b-.;				\
 	 .byte ptype;				\
 	 .byte 772b-771b;			\
-	 _ASM_ALIGN;				\
 	.popsection
 
 
@@ -760,7 +758,7 @@ void native_pv_lock_init(void) __init;
 #ifdef CONFIG_PARAVIRT_XXL
 
 #define PARA_PATCH(off)		((off) / 8)
-#define PARA_SITE(ptype, ops)	_PVSITE(ptype, ops, .quad, 8)
+#define PARA_SITE(ptype, ops)	_PVSITE(ptype, ops)
 #define PARA_INDIRECT(addr)	*addr(%rip)
 
 #ifdef CONFIG_DEBUG_ENTRY
diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 772d03487520..f4fb2e3ec7b8 100644
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -5,10 +5,10 @@
 #ifndef __ASSEMBLY__
 /* These all sit in the .parainstructions section to tell us what to patch. */
 struct paravirt_patch_site {
-	u8 *instr;		/* original instructions */
+	s32 instr_offset;	/* original instructions */
 	u8 type;		/* type of this instruction */
 	u8 len;			/* length of original instruction */
-};
+} __packed;
 #endif
 
 #ifdef CONFIG_PARAVIRT
@@ -263,11 +263,9 @@ extern struct paravirt_patch_template pv_ops;
 #define _paravirt_alt(insn_string, type)		\
 	"771:\n\t" insn_string "\n" "772:\n"		\
 	".pushsection .parainstructions,\"a\"\n"	\
-	_ASM_ALIGN "\n"					\
-	_ASM_PTR " 771b\n"				\
+	"  .long 771b-.\n"				\
 	"  .byte " type "\n"				\
 	"  .byte 772b-771b\n"				\
-	_ASM_ALIGN "\n"					\
 	".popsection\n"
 
 /* Generate patchable code, with the default asm parameters. */
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 20624b80f890..b86b27d15e52 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -4,17 +4,21 @@
 
 #ifdef CONFIG_X86_64
 #define __percpu_seg		gs
+#define __percpu_rel		(%rip)
 #else
 #define __percpu_seg		fs
+#define __percpu_rel
 #endif
 
 #ifdef __ASSEMBLY__
 
 #ifdef CONFIG_SMP
-#define PER_CPU_VAR(var)	%__percpu_seg:var
-#else /* ! SMP */
-#define PER_CPU_VAR(var)	var
-#endif	/* SMP */
+#define __percpu		%__percpu_seg:
+#else
+#define __percpu
+#endif
+
+#define PER_CPU_VAR(var)	__percpu(var)__percpu_rel
 
 #ifdef CONFIG_X86_64_SMP
 #define INIT_PER_CPU_VAR(var)  init_per_cpu__##var
@@ -28,26 +32,69 @@
 #include <linux/stringify.h>
 
 #ifdef CONFIG_SMP
+
+#ifdef CONFIG_CC_HAS_NAMED_AS
+
+#ifdef CONFIG_X86_64
+#define __percpu_seg_override	__seg_gs
+#else
+#define __percpu_seg_override	__seg_fs
+#endif
+
+#define __percpu_prefix		""
+
+#else /* CONFIG_CC_HAS_NAMED_AS */
+
+#define __percpu_seg_override
 #define __percpu_prefix		"%%"__stringify(__percpu_seg)":"
+
+#endif /* CONFIG_CC_HAS_NAMED_AS */
+
+#define __force_percpu_prefix	"%%"__stringify(__percpu_seg)":"
 #define __my_cpu_offset		this_cpu_read(this_cpu_off)
 
+#ifdef CONFIG_USE_X86_SEG_SUPPORT
+/*
+ * Efficient implementation for cases in which the compiler supports
+ * named address spaces.  Allows the compiler to perform additional
+ * optimizations that can save more instructions.
+ */
+#define arch_raw_cpu_ptr(ptr)					\
+({								\
+	unsigned long tcp_ptr__;				\
+	tcp_ptr__ = __raw_cpu_read(, this_cpu_off);		\
+								\
+	tcp_ptr__ += (unsigned long)(ptr);			\
+	(typeof(*(ptr)) __kernel __force *)tcp_ptr__;		\
+})
+#else /* CONFIG_USE_X86_SEG_SUPPORT */
 /*
  * Compared to the generic __my_cpu_offset version, the following
  * saves one instruction and avoids clobbering a temp register.
  */
-#define arch_raw_cpu_ptr(ptr)				\
-({							\
-	unsigned long tcp_ptr__;			\
-	asm ("add " __percpu_arg(1) ", %0"		\
-	     : "=r" (tcp_ptr__)				\
-	     : "m" (this_cpu_off), "0" (ptr));		\
-	(typeof(*(ptr)) __kernel __force *)tcp_ptr__;	\
+#define arch_raw_cpu_ptr(ptr)					\
+({								\
+	unsigned long tcp_ptr__;				\
+	asm ("mov " __percpu_arg(1) ", %0"			\
+	     : "=r" (tcp_ptr__)					\
+	     : "m" (__my_cpu_var(this_cpu_off)));		\
+								\
+	tcp_ptr__ += (unsigned long)(ptr);			\
+	(typeof(*(ptr)) __kernel __force *)tcp_ptr__;		\
 })
-#else
+#endif /* CONFIG_USE_X86_SEG_SUPPORT */
+
+#else /* CONFIG_SMP */
+#define __percpu_seg_override
 #define __percpu_prefix		""
-#endif
+#define __force_percpu_prefix	""
+#endif /* CONFIG_SMP */
 
+#define __my_cpu_type(var)	typeof(var) __percpu_seg_override
+#define __my_cpu_ptr(ptr)	(__my_cpu_type(*ptr) *)(uintptr_t)(ptr)
+#define __my_cpu_var(var)	(*__my_cpu_ptr(&var))
 #define __percpu_arg(x)		__percpu_prefix "%" #x
+#define __force_percpu_arg(x)	__force_percpu_prefix "%" #x
 
 /*
  * Initialized pointers to per-cpu variables needed for the boot
@@ -107,14 +154,14 @@ do {									\
 		(void)pto_tmp__;					\
 	}								\
 	asm qual(__pcpu_op2_##size(op, "%[val]", __percpu_arg([var]))	\
-	    : [var] "+m" (_var)						\
+	    : [var] "+m" (__my_cpu_var(_var))				\
 	    : [val] __pcpu_reg_imm_##size(pto_val__));			\
 } while (0)
 
 #define percpu_unary_op(size, qual, op, _var)				\
 ({									\
 	asm qual (__pcpu_op1_##size(op, __percpu_arg([var]))		\
-	    : [var] "+m" (_var));					\
+	    : [var] "+m" (__my_cpu_var(_var)));				\
 })
 
 /*
@@ -144,16 +191,16 @@ do {									\
 	__pcpu_type_##size pfo_val__;					\
 	asm qual (__pcpu_op2_##size(op, __percpu_arg([var]), "%[val]")	\
 	    : [val] __pcpu_reg_##size("=", pfo_val__)			\
-	    : [var] "m" (_var));					\
+	    : [var] "m" (__my_cpu_var(_var)));				\
 	(typeof(_var))(unsigned long) pfo_val__;			\
 })
 
 #define percpu_stable_op(size, op, _var)				\
 ({									\
 	__pcpu_type_##size pfo_val__;					\
-	asm(__pcpu_op2_##size(op, __percpu_arg(P[var]), "%[val]")	\
+	asm(__pcpu_op2_##size(op, __force_percpu_arg(a[var]), "%[val]")	\
 	    : [val] __pcpu_reg_##size("=", pfo_val__)			\
-	    : [var] "p" (&(_var)));					\
+	    : [var] "i" (&(_var)));					\
 	(typeof(_var))(unsigned long) pfo_val__;			\
 })
 
@@ -166,7 +213,7 @@ do {									\
 	asm qual (__pcpu_op2_##size("xadd", "%[tmp]",			\
 				     __percpu_arg([var]))		\
 		  : [tmp] __pcpu_reg_##size("+", paro_tmp__),		\
-		    [var] "+m" (_var)					\
+		    [var] "+m" (__my_cpu_var(_var))			\
 		  : : "memory");					\
 	(typeof(_var))(unsigned long) (paro_tmp__ + _val);		\
 })
@@ -187,7 +234,7 @@ do {									\
 				    __percpu_arg([var]))		\
 		  "\n\tjnz 1b"						\
 		  : [oval] "=&a" (pxo_old__),				\
-		    [var] "+m" (_var)					\
+		    [var] "+m" (__my_cpu_var(_var))			\
 		  : [nval] __pcpu_reg_##size(, pxo_new__)		\
 		  : "memory");						\
 	(typeof(_var))(unsigned long) pxo_old__;			\
@@ -204,7 +251,7 @@ do {									\
 	asm qual (__pcpu_op2_##size("cmpxchg", "%[nval]",		\
 				    __percpu_arg([var]))		\
 		  : [oval] "+a" (pco_old__),				\
-		    [var] "+m" (_var)					\
+		    [var] "+m" (__my_cpu_var(_var))			\
 		  : [nval] __pcpu_reg_##size(, pco_new__)		\
 		  : "memory");						\
 	(typeof(_var))(unsigned long) pco_old__;			\
@@ -221,7 +268,7 @@ do {									\
 		  CC_SET(z)						\
 		  : CC_OUT(z) (success),				\
 		    [oval] "+a" (pco_old__),				\
-		    [var] "+m" (_var)					\
+		    [var] "+m" (__my_cpu_var(_var))			\
 		  : [nval] __pcpu_reg_##size(, pco_new__)		\
 		  : "memory");						\
 	if (unlikely(!success))						\
@@ -244,7 +291,7 @@ do {									\
 									\
 	asm qual (ALTERNATIVE("call this_cpu_cmpxchg8b_emu",		\
 			      "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \
-		  : [var] "+m" (_var),					\
+		  : [var] "+m" (__my_cpu_var(_var)),			\
 		    "+a" (old__.low),					\
 		    "+d" (old__.high)					\
 		  : "b" (new__.low),					\
@@ -276,7 +323,7 @@ do {									\
 			      "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \
 		  CC_SET(z)						\
 		  : CC_OUT(z) (success),				\
-		    [var] "+m" (_var),					\
+		    [var] "+m" (__my_cpu_var(_var)),			\
 		    "+a" (old__.low),					\
 		    "+d" (old__.high)					\
 		  : "b" (new__.low),					\
@@ -313,7 +360,7 @@ do {									\
 									\
 	asm qual (ALTERNATIVE("call this_cpu_cmpxchg16b_emu",		\
 			      "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \
-		  : [var] "+m" (_var),					\
+		  : [var] "+m" (__my_cpu_var(_var)),			\
 		    "+a" (old__.low),					\
 		    "+d" (old__.high)					\
 		  : "b" (new__.low),					\
@@ -345,7 +392,7 @@ do {									\
 			      "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \
 		  CC_SET(z)						\
 		  : CC_OUT(z) (success),				\
-		    [var] "+m" (_var),					\
+		    [var] "+m" (__my_cpu_var(_var)),			\
 		    "+a" (old__.low),					\
 		    "+d" (old__.high)					\
 		  : "b" (new__.low),					\
@@ -366,9 +413,9 @@ do {									\
  * accessed while this_cpu_read_stable() allows the value to be cached.
  * this_cpu_read_stable() is more efficient and can be used if its value
  * is guaranteed to be valid across cpus.  The current users include
- * get_current() and get_thread_info() both of which are actually
- * per-thread variables implemented as per-cpu variables and thus
- * stable for the duration of the respective task.
+ * pcpu_hot.current_task and pcpu_hot.top_of_stack, both of which are
+ * actually per-thread variables implemented as per-CPU variables and
+ * thus stable for the duration of the respective task.
  */
 #define this_cpu_read_stable_1(pcp)	percpu_stable_op(1, "mov", pcp)
 #define this_cpu_read_stable_2(pcp)	percpu_stable_op(2, "mov", pcp)
@@ -376,13 +423,66 @@ do {									\
 #define this_cpu_read_stable_8(pcp)	percpu_stable_op(8, "mov", pcp)
 #define this_cpu_read_stable(pcp)	__pcpu_size_call_return(this_cpu_read_stable_, pcp)
 
+#ifdef CONFIG_USE_X86_SEG_SUPPORT
+
+#define __raw_cpu_read(qual, pcp)					\
+({									\
+	*(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp));		\
+})
+
+#define __raw_cpu_write(qual, pcp, val)					\
+do {									\
+	*(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val);	\
+} while (0)
+
+#define raw_cpu_read_1(pcp)		__raw_cpu_read(, pcp)
+#define raw_cpu_read_2(pcp)		__raw_cpu_read(, pcp)
+#define raw_cpu_read_4(pcp)		__raw_cpu_read(, pcp)
+#define raw_cpu_write_1(pcp, val)	__raw_cpu_write(, pcp, val)
+#define raw_cpu_write_2(pcp, val)	__raw_cpu_write(, pcp, val)
+#define raw_cpu_write_4(pcp, val)	__raw_cpu_write(, pcp, val)
+
+#define this_cpu_read_1(pcp)		__raw_cpu_read(volatile, pcp)
+#define this_cpu_read_2(pcp)		__raw_cpu_read(volatile, pcp)
+#define this_cpu_read_4(pcp)		__raw_cpu_read(volatile, pcp)
+#define this_cpu_write_1(pcp, val)	__raw_cpu_write(volatile, pcp, val)
+#define this_cpu_write_2(pcp, val)	__raw_cpu_write(volatile, pcp, val)
+#define this_cpu_write_4(pcp, val)	__raw_cpu_write(volatile, pcp, val)
+
+#ifdef CONFIG_X86_64
+#define raw_cpu_read_8(pcp)		__raw_cpu_read(, pcp)
+#define raw_cpu_write_8(pcp, val)	__raw_cpu_write(, pcp, val)
+
+#define this_cpu_read_8(pcp)		__raw_cpu_read(volatile, pcp)
+#define this_cpu_write_8(pcp, val)	__raw_cpu_write(volatile, pcp, val)
+#endif
+
+#else /* CONFIG_USE_X86_SEG_SUPPORT */
+
 #define raw_cpu_read_1(pcp)		percpu_from_op(1, , "mov", pcp)
 #define raw_cpu_read_2(pcp)		percpu_from_op(2, , "mov", pcp)
 #define raw_cpu_read_4(pcp)		percpu_from_op(4, , "mov", pcp)
-
 #define raw_cpu_write_1(pcp, val)	percpu_to_op(1, , "mov", (pcp), val)
 #define raw_cpu_write_2(pcp, val)	percpu_to_op(2, , "mov", (pcp), val)
 #define raw_cpu_write_4(pcp, val)	percpu_to_op(4, , "mov", (pcp), val)
+
+#define this_cpu_read_1(pcp)		percpu_from_op(1, volatile, "mov", pcp)
+#define this_cpu_read_2(pcp)		percpu_from_op(2, volatile, "mov", pcp)
+#define this_cpu_read_4(pcp)		percpu_from_op(4, volatile, "mov", pcp)
+#define this_cpu_write_1(pcp, val)	percpu_to_op(1, volatile, "mov", (pcp), val)
+#define this_cpu_write_2(pcp, val)	percpu_to_op(2, volatile, "mov", (pcp), val)
+#define this_cpu_write_4(pcp, val)	percpu_to_op(4, volatile, "mov", (pcp), val)
+
+#ifdef CONFIG_X86_64
+#define raw_cpu_read_8(pcp)		percpu_from_op(8, , "mov", pcp)
+#define raw_cpu_write_8(pcp, val)	percpu_to_op(8, , "mov", (pcp), val)
+
+#define this_cpu_read_8(pcp)		percpu_from_op(8, volatile, "mov", pcp)
+#define this_cpu_write_8(pcp, val)	percpu_to_op(8, volatile, "mov", (pcp), val)
+#endif
+
+#endif /* CONFIG_USE_X86_SEG_SUPPORT */
+
 #define raw_cpu_add_1(pcp, val)		percpu_add_op(1, , (pcp), val)
 #define raw_cpu_add_2(pcp, val)		percpu_add_op(2, , (pcp), val)
 #define raw_cpu_add_4(pcp, val)		percpu_add_op(4, , (pcp), val)
@@ -408,12 +508,6 @@ do {									\
 #define raw_cpu_xchg_2(pcp, val)	raw_percpu_xchg_op(pcp, val)
 #define raw_cpu_xchg_4(pcp, val)	raw_percpu_xchg_op(pcp, val)
 
-#define this_cpu_read_1(pcp)		percpu_from_op(1, volatile, "mov", pcp)
-#define this_cpu_read_2(pcp)		percpu_from_op(2, volatile, "mov", pcp)
-#define this_cpu_read_4(pcp)		percpu_from_op(4, volatile, "mov", pcp)
-#define this_cpu_write_1(pcp, val)	percpu_to_op(1, volatile, "mov", (pcp), val)
-#define this_cpu_write_2(pcp, val)	percpu_to_op(2, volatile, "mov", (pcp), val)
-#define this_cpu_write_4(pcp, val)	percpu_to_op(4, volatile, "mov", (pcp), val)
 #define this_cpu_add_1(pcp, val)	percpu_add_op(1, volatile, (pcp), val)
 #define this_cpu_add_2(pcp, val)	percpu_add_op(2, volatile, (pcp), val)
 #define this_cpu_add_4(pcp, val)	percpu_add_op(4, volatile, (pcp), val)
@@ -452,8 +546,6 @@ do {									\
  * 32 bit must fall back to generic operations.
  */
 #ifdef CONFIG_X86_64
-#define raw_cpu_read_8(pcp)			percpu_from_op(8, , "mov", pcp)
-#define raw_cpu_write_8(pcp, val)		percpu_to_op(8, , "mov", (pcp), val)
 #define raw_cpu_add_8(pcp, val)			percpu_add_op(8, , (pcp), val)
 #define raw_cpu_and_8(pcp, val)			percpu_to_op(8, , "and", (pcp), val)
 #define raw_cpu_or_8(pcp, val)			percpu_to_op(8, , "or", (pcp), val)
@@ -462,8 +554,6 @@ do {									\
 #define raw_cpu_cmpxchg_8(pcp, oval, nval)	percpu_cmpxchg_op(8, , pcp, oval, nval)
 #define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval)	percpu_try_cmpxchg_op(8, , pcp, ovalp, nval)
 
-#define this_cpu_read_8(pcp)			percpu_from_op(8, volatile, "mov", pcp)
-#define this_cpu_write_8(pcp, val)		percpu_to_op(8, volatile, "mov", (pcp), val)
 #define this_cpu_add_8(pcp, val)		percpu_add_op(8, volatile, (pcp), val)
 #define this_cpu_and_8(pcp, val)		percpu_to_op(8, volatile, "and", (pcp), val)
 #define this_cpu_or_8(pcp, val)			percpu_to_op(8, volatile, "or", (pcp), val)
@@ -494,7 +584,7 @@ static inline bool x86_this_cpu_variable_test_bit(int nr,
 	asm volatile("btl "__percpu_arg(2)",%1"
 			CC_SET(c)
 			: CC_OUT(c) (oldbit)
-			: "m" (*(unsigned long __percpu *)addr), "Ir" (nr));
+			: "m" (*__my_cpu_ptr((unsigned long __percpu *)(addr))), "Ir" (nr));
 
 	return oldbit;
 }
diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h
index 2618ec7c3d1d..3736b8a46c04 100644
--- a/arch/x86/include/asm/perf_event.h
+++ b/arch/x86/include/asm/perf_event.h
@@ -31,6 +31,7 @@
 #define ARCH_PERFMON_EVENTSEL_ENABLE			(1ULL << 22)
 #define ARCH_PERFMON_EVENTSEL_INV			(1ULL << 23)
 #define ARCH_PERFMON_EVENTSEL_CMASK			0xFF000000ULL
+#define ARCH_PERFMON_EVENTSEL_BR_CNTR			(1ULL << 35)
 
 #define INTEL_FIXED_BITS_MASK				0xFULL
 #define INTEL_FIXED_BITS_STRIDE			4
@@ -223,6 +224,9 @@ union cpuid28_ecx {
 		unsigned int    lbr_timed_lbr:1;
 		/* Branch Type Field Supported */
 		unsigned int    lbr_br_type:1;
+		unsigned int	reserved:13;
+		/* Branch counters (Event Logging) Supported */
+		unsigned int	lbr_counters:4;
 	} split;
 	unsigned int            full;
 };
diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
index 4527e1430c6d..4b2a35d8d56a 100644
--- a/arch/x86/include/asm/preempt.h
+++ b/arch/x86/include/asm/preempt.h
@@ -92,7 +92,7 @@ static __always_inline void __preempt_count_sub(int val)
  */
 static __always_inline bool __preempt_count_dec_and_test(void)
 {
-	return GEN_UNARY_RMWcc("decl", pcpu_hot.preempt_count, e,
+	return GEN_UNARY_RMWcc("decl", __my_cpu_var(pcpu_hot.preempt_count), e,
 			       __percpu_arg([var]));
 }
 
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index ae81a7191c1c..061aa86b4662 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -533,6 +533,9 @@ static __always_inline unsigned long current_top_of_stack(void)
 	 *  and around vm86 mode and sp0 on x86_64 is special because of the
 	 *  entry trampoline.
 	 */
+	if (IS_ENABLED(CONFIG_USE_X86_SEG_SUPPORT))
+		return const_pcpu_hot.top_of_stack;
+
 	return this_cpu_read_stable(pcpu_hot.top_of_stack);
 }
 
@@ -749,4 +752,22 @@ enum mds_mitigations {
 
 extern bool gds_ucode_mitigated(void);
 
+/*
+ * Make previous memory operations globally visible before
+ * a WRMSR.
+ *
+ * MFENCE makes writes visible, but only affects load/store
+ * instructions.  WRMSR is unfortunately not a load/store
+ * instruction and is unaffected by MFENCE.  The LFENCE ensures
+ * that the WRMSR is not reordered.
+ *
+ * Most WRMSRs are full serializing instructions themselves and
+ * do not require this barrier.  This is only required for the
+ * IA32_TSC_DEADLINE and X2APIC MSRs.
+ */
+static inline void weak_wrmsr_fence(void)
+{
+	alternative("mfence; lfence", "", ALT_NOT(X86_FEATURE_APIC_MSRS_FENCE));
+}
+
 #endif /* _ASM_X86_PROCESSOR_H */
diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h
index bf483fcb4e57..5c83729c8e71 100644
--- a/arch/x86/include/asm/setup.h
+++ b/arch/x86/include/asm/setup.h
@@ -31,8 +31,6 @@
 #include <asm/bootparam.h>
 #include <asm/x86_init.h>
 
-extern u64 relocated_ramdisk;
-
 /* Interrupt control for vSMPowered x86_64 systems */
 #ifdef CONFIG_X86_64
 void vsmp_init(void);
diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h
index 1a6a1f987949..a448d0964fc0 100644
--- a/arch/x86/include/uapi/asm/kvm.h
+++ b/arch/x86/include/uapi/asm/kvm.h
@@ -562,4 +562,7 @@ struct kvm_pmu_event_filter {
 /* x86-specific KVM_EXIT_HYPERCALL flags. */
 #define KVM_EXIT_HYPERCALL_LONG_MODE	BIT(0)
 
+#define KVM_X86_DEFAULT_VM	0
+#define KVM_X86_SW_PROTECTED_VM	1
+
 #endif /* _ASM_X86_KVM_H */
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index 73be3931e4f0..be35c8ccf826 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -1431,20 +1431,22 @@ void __init_or_module apply_paravirt(struct paravirt_patch_site *start,
 {
 	struct paravirt_patch_site *p;
 	char insn_buff[MAX_PATCH_LEN];
+	u8 *instr;
 
 	for (p = start; p < end; p++) {
 		unsigned int used;
 
+		instr = (u8 *)&p->instr_offset + p->instr_offset;
 		BUG_ON(p->len > MAX_PATCH_LEN);
 		/* prep the buffer with the original instructions */
-		memcpy(insn_buff, p->instr, p->len);
-		used = paravirt_patch(p->type, insn_buff, (unsigned long)p->instr, p->len);
+		memcpy(insn_buff, instr, p->len);
+		used = paravirt_patch(p->type, insn_buff, (unsigned long)instr, p->len);
 
 		BUG_ON(used > p->len);
 
 		/* Pad the rest with nops */
 		add_nops(insn_buff + used, p->len - used);
-		text_poke_early(p->instr, insn_buff, p->len);
+		text_poke_early(instr, insn_buff, p->len);
 	}
 }
 extern struct paravirt_patch_site __start_parainstructions[],
diff --git a/arch/x86/kernel/apic/apic_flat_64.c b/arch/x86/kernel/apic/apic_flat_64.c
index 7139867d69cd..b295a056a4fc 100644
--- a/arch/x86/kernel/apic/apic_flat_64.c
+++ b/arch/x86/kernel/apic/apic_flat_64.c
@@ -82,7 +82,6 @@ static struct apic apic_flat __ro_after_init = {
 	.acpi_madt_oem_check		= flat_acpi_madt_oem_check,
 	.apic_id_registered		= default_apic_id_registered,
 
-	.delivery_mode			= APIC_DELIVERY_MODE_FIXED,
 	.dest_mode_logical		= true,
 
 	.disable_esr			= 0,
@@ -154,7 +153,6 @@ static struct apic apic_physflat __ro_after_init = {
 	.acpi_madt_oem_check		= physflat_acpi_madt_oem_check,
 	.apic_id_registered		= default_apic_id_registered,
 
-	.delivery_mode			= APIC_DELIVERY_MODE_FIXED,
 	.dest_mode_logical		= false,
 
 	.disable_esr			= 0,
diff --git a/arch/x86/kernel/apic/apic_noop.c b/arch/x86/kernel/apic/apic_noop.c
index b00d52ae84fa..9f1d553eb48f 100644
--- a/arch/x86/kernel/apic/apic_noop.c
+++ b/arch/x86/kernel/apic/apic_noop.c
@@ -47,7 +47,6 @@ static void noop_apic_write(u32 reg, u32 val)
 struct apic apic_noop __ro_after_init = {
 	.name				= "noop",
 
-	.delivery_mode			= APIC_DELIVERY_MODE_FIXED,
 	.dest_mode_logical		= true,
 
 	.disable_esr			= 0,
diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c
index 456a14c44f67..7d0c51b9d3bc 100644
--- a/arch/x86/kernel/apic/apic_numachip.c
+++ b/arch/x86/kernel/apic/apic_numachip.c
@@ -222,7 +222,6 @@ static const struct apic apic_numachip1 __refconst = {
 	.probe				= numachip1_probe,
 	.acpi_madt_oem_check		= numachip1_acpi_madt_oem_check,
 
-	.delivery_mode			= APIC_DELIVERY_MODE_FIXED,
 	.dest_mode_logical		= false,
 
 	.disable_esr			= 0,
@@ -259,7 +258,6 @@ static const struct apic apic_numachip2 __refconst = {
 	.probe				= numachip2_probe,
 	.acpi_madt_oem_check		= numachip2_acpi_madt_oem_check,
 
-	.delivery_mode			= APIC_DELIVERY_MODE_FIXED,
 	.dest_mode_logical		= false,
 
 	.disable_esr			= 0,
diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c
index 7ee3c486cb33..5a0d60b38e6b 100644
--- a/arch/x86/kernel/apic/bigsmp_32.c
+++ b/arch/x86/kernel/apic/bigsmp_32.c
@@ -80,7 +80,6 @@ static struct apic apic_bigsmp __ro_after_init = {
 	.name				= "bigsmp",
 	.probe				= probe_bigsmp,
 
-	.delivery_mode			= APIC_DELIVERY_MODE_FIXED,
 	.dest_mode_logical		= false,
 
 	.disable_esr			= 1,
diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c
index 5eb3fbe472da..c0f78059f06a 100644
--- a/arch/x86/kernel/apic/probe_32.c
+++ b/arch/x86/kernel/apic/probe_32.c
@@ -45,7 +45,6 @@ static struct apic apic_default __ro_after_init = {
 	.probe				= probe_default,
 	.apic_id_registered		= default_apic_id_registered,
 
-	.delivery_mode			= APIC_DELIVERY_MODE_FIXED,
 	.dest_mode_logical		= true,
 
 	.disable_esr			= 0,
diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c
index a8306089c91b..28a7d3f2312d 100644
--- a/arch/x86/kernel/apic/x2apic_cluster.c
+++ b/arch/x86/kernel/apic/x2apic_cluster.c
@@ -227,7 +227,6 @@ static struct apic apic_x2apic_cluster __ro_after_init = {
 	.probe				= x2apic_cluster_probe,
 	.acpi_madt_oem_check		= x2apic_acpi_madt_oem_check,
 
-	.delivery_mode			= APIC_DELIVERY_MODE_FIXED,
 	.dest_mode_logical		= true,
 
 	.disable_esr			= 0,
diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c
index 558a4a8824f4..409815a40668 100644
--- a/arch/x86/kernel/apic/x2apic_phys.c
+++ b/arch/x86/kernel/apic/x2apic_phys.c
@@ -145,7 +145,6 @@ static struct apic apic_x2apic_phys __ro_after_init = {
 	.probe				= x2apic_phys_probe,
 	.acpi_madt_oem_check		= x2apic_acpi_madt_oem_check,
 
-	.delivery_mode			= APIC_DELIVERY_MODE_FIXED,
 	.dest_mode_logical		= false,
 
 	.disable_esr			= 0,
diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c
index 1b0d7336a28f..f1766b18dcd0 100644
--- a/arch/x86/kernel/apic/x2apic_uv_x.c
+++ b/arch/x86/kernel/apic/x2apic_uv_x.c
@@ -805,7 +805,6 @@ static struct apic apic_x2apic_uv_x __ro_after_init = {
 	.probe				= uv_probe,
 	.acpi_madt_oem_check		= uv_acpi_madt_oem_check,
 
-	.delivery_mode			= APIC_DELIVERY_MODE_FIXED,
 	.dest_mode_logical		= false,
 
 	.disable_esr			= 0,
diff --git a/arch/x86/kernel/callthunks.c b/arch/x86/kernel/callthunks.c
index e9ad518a5003..57e5c2e75c2a 100644
--- a/arch/x86/kernel/callthunks.c
+++ b/arch/x86/kernel/callthunks.c
@@ -240,7 +240,7 @@ patch_paravirt_call_sites(struct paravirt_patch_site *start,
 	struct paravirt_patch_site *p;
 
 	for (p = start; p < end; p++)
-		patch_call(p->instr, ct);
+		patch_call((void *)&p->instr_offset + p->instr_offset, ct);
 }
 
 static __init_or_module void
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index a7eab05e5f29..841e21213668 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -1162,6 +1162,9 @@ static void init_amd(struct cpuinfo_x86 *c)
 	if (!cpu_has(c, X86_FEATURE_HYPERVISOR) &&
 	     cpu_has_amd_erratum(c, amd_erratum_1485))
 		msr_set_bit(MSR_ZEN4_BP_CFG, MSR_ZEN4_BP_CFG_SHARED_BTB_FIX_BIT);
+
+	/* AMD CPUs don't need fencing after x2APIC/TSC_DEADLINE MSR writes. */
+	clear_cpu_cap(c, X86_FEATURE_APIC_MSRS_FENCE);
 }
 
 #ifdef CONFIG_X86_32
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index b14fc8c1c953..4d4b87c6885d 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1856,6 +1856,13 @@ static void identify_cpu(struct cpuinfo_x86 *c)
 	c->topo.apicid = apic->phys_pkg_id(c->topo.initial_apicid, 0);
 #endif
 
+
+	/*
+	 * Set default APIC and TSC_DEADLINE MSR fencing flag. AMD and
+	 * Hygon will clear it in ->c_init() below.
+	 */
+	set_cpu_cap(c, X86_FEATURE_APIC_MSRS_FENCE);
+
 	/*
 	 * Vendor-specific initialization.  In this section we
 	 * canonicalize the feature flags, meaning if there are
@@ -2049,6 +2056,7 @@ DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = {
 	.top_of_stack	= TOP_OF_INIT_STACK,
 };
 EXPORT_PER_CPU_SYMBOL(pcpu_hot);
+EXPORT_PER_CPU_SYMBOL(const_pcpu_hot);
 
 #ifdef CONFIG_X86_64
 DEFINE_PER_CPU_FIRST(struct fixed_percpu_data,
diff --git a/arch/x86/kernel/cpu/hygon.c b/arch/x86/kernel/cpu/hygon.c
index 6f247d66758d..f0cd95502faa 100644
--- a/arch/x86/kernel/cpu/hygon.c
+++ b/arch/x86/kernel/cpu/hygon.c
@@ -354,6 +354,9 @@ static void init_hygon(struct cpuinfo_x86 *c)
 		set_cpu_bug(c, X86_BUG_SYSRET_SS_ATTRS);
 
 	check_null_seg_clears_base(c);
+
+	/* Hygon CPUs don't need fencing after x2APIC/TSC_DEADLINE MSR writes. */
+	clear_cpu_cap(c, X86_FEATURE_APIC_MSRS_FENCE);
 }
 
 static void cpu_detect_tlb_hygon(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c
index 7b397370b4d6..1642018dd6c9 100644
--- a/arch/x86/kernel/cpu/mce/core.c
+++ b/arch/x86/kernel/cpu/mce/core.c
@@ -44,6 +44,7 @@
 #include <linux/sync_core.h>
 #include <linux/task_work.h>
 #include <linux/hardirq.h>
+#include <linux/kexec.h>
 
 #include <asm/intel-family.h>
 #include <asm/processor.h>
@@ -233,6 +234,7 @@ static noinstr void mce_panic(const char *msg, struct mce *final, char *exp)
 	struct llist_node *pending;
 	struct mce_evt_llist *l;
 	int apei_err = 0;
+	struct page *p;
 
 	/*
 	 * Allow instrumentation around external facilities usage. Not that it
@@ -286,6 +288,20 @@ static noinstr void mce_panic(const char *msg, struct mce *final, char *exp)
 	if (!fake_panic) {
 		if (panic_timeout == 0)
 			panic_timeout = mca_cfg.panic_timeout;
+
+		/*
+		 * Kdump skips the poisoned page in order to avoid
+		 * touching the error bits again. Poison the page even
+		 * if the error is fatal and the machine is about to
+		 * panic.
+		 */
+		if (kexec_crash_loaded()) {
+			if (final && (final->status & MCI_STATUS_ADDRV)) {
+				p = pfn_to_online_page(final->addr >> PAGE_SHIFT);
+				if (p)
+					SetPageHWPoison(p);
+			}
+		}
 		panic(msg);
 	} else
 		pr_emerg(HW_ERR "Fake kernel panic: %s\n", msg);
@@ -2568,9 +2584,6 @@ static int mce_device_create(unsigned int cpu)
 	int err;
 	int i, j;
 
-	if (!mce_available(&boot_cpu_data))
-		return -EIO;
-
 	dev = per_cpu(mce_device, cpu);
 	if (dev)
 		return 0;
diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c
index 9373ec01c5ae..13b45b9c806d 100644
--- a/arch/x86/kernel/cpu/microcode/amd.c
+++ b/arch/x86/kernel/cpu/microcode/amd.c
@@ -104,8 +104,6 @@ struct cont_desc {
 	size_t		     size;
 };
 
-static u32 ucode_new_rev;
-
 /*
  * Microcode patch container file is prepended to the initrd in cpio
  * format. See Documentation/arch/x86/microcode.rst
@@ -442,12 +440,11 @@ static int __apply_microcode_amd(struct microcode_amd *mc)
  *
  * Returns true if container found (sets @desc), false otherwise.
  */
-static bool early_apply_microcode(u32 cpuid_1_eax, void *ucode, size_t size)
+static bool early_apply_microcode(u32 cpuid_1_eax, u32 old_rev, void *ucode, size_t size)
 {
 	struct cont_desc desc = { 0 };
 	struct microcode_amd *mc;
 	bool ret = false;
-	u32 rev, dummy;
 
 	desc.cpuid_1_eax = cpuid_1_eax;
 
@@ -457,22 +454,15 @@ static bool early_apply_microcode(u32 cpuid_1_eax, void *ucode, size_t size)
 	if (!mc)
 		return ret;
 
-	native_rdmsr(MSR_AMD64_PATCH_LEVEL, rev, dummy);
-
 	/*
 	 * Allow application of the same revision to pick up SMT-specific
 	 * changes even if the revision of the other SMT thread is already
 	 * up-to-date.
 	 */
-	if (rev > mc->hdr.patch_id)
+	if (old_rev > mc->hdr.patch_id)
 		return ret;
 
-	if (!__apply_microcode_amd(mc)) {
-		ucode_new_rev = mc->hdr.patch_id;
-		ret = true;
-	}
-
-	return ret;
+	return !__apply_microcode_amd(mc);
 }
 
 static bool get_builtin_microcode(struct cpio_data *cp, unsigned int family)
@@ -506,9 +496,12 @@ static void __init find_blobs_in_containers(unsigned int cpuid_1_eax, struct cpi
 	*ret = cp;
 }
 
-void __init load_ucode_amd_bsp(unsigned int cpuid_1_eax)
+void __init load_ucode_amd_bsp(struct early_load_data *ed, unsigned int cpuid_1_eax)
 {
 	struct cpio_data cp = { };
+	u32 dummy;
+
+	native_rdmsr(MSR_AMD64_PATCH_LEVEL, ed->old_rev, dummy);
 
 	/* Needed in load_microcode_amd() */
 	ucode_cpu_info[0].cpu_sig.sig = cpuid_1_eax;
@@ -517,7 +510,8 @@ void __init load_ucode_amd_bsp(unsigned int cpuid_1_eax)
 	if (!(cp.data && cp.size))
 		return;
 
-	early_apply_microcode(cpuid_1_eax, cp.data, cp.size);
+	if (early_apply_microcode(cpuid_1_eax, ed->old_rev, cp.data, cp.size))
+		native_rdmsr(MSR_AMD64_PATCH_LEVEL, ed->new_rev, dummy);
 }
 
 static enum ucode_state load_microcode_amd(u8 family, const u8 *data, size_t size);
@@ -625,10 +619,8 @@ void reload_ucode_amd(unsigned int cpu)
 	rdmsr(MSR_AMD64_PATCH_LEVEL, rev, dummy);
 
 	if (rev < mc->hdr.patch_id) {
-		if (!__apply_microcode_amd(mc)) {
-			ucode_new_rev = mc->hdr.patch_id;
-			pr_info("reload patch_level=0x%08x\n", ucode_new_rev);
-		}
+		if (!__apply_microcode_amd(mc))
+			pr_info_once("reload revision: 0x%08x\n", mc->hdr.patch_id);
 	}
 }
 
@@ -649,8 +641,6 @@ static int collect_cpu_info_amd(int cpu, struct cpu_signature *csig)
 	if (p && (p->patch_id == csig->rev))
 		uci->mc = p->data;
 
-	pr_info("CPU%d: patch_level=0x%08x\n", cpu, csig->rev);
-
 	return 0;
 }
 
@@ -691,8 +681,6 @@ static enum ucode_state apply_microcode_amd(int cpu)
 	rev = mc_amd->hdr.patch_id;
 	ret = UCODE_UPDATED;
 
-	pr_info("CPU%d: new patch_level=0x%08x\n", cpu, rev);
-
 out:
 	uci->cpu_sig.rev = rev;
 	c->microcode	 = rev;
@@ -935,11 +923,6 @@ struct microcode_ops * __init init_amd_microcode(void)
 		pr_warn("AMD CPU family 0x%x not supported\n", c->x86);
 		return NULL;
 	}
-
-	if (ucode_new_rev)
-		pr_info_once("microcode updated early to new patch_level=0x%08x\n",
-			     ucode_new_rev);
-
 	return &microcode_amd_ops;
 }
 
diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c
index 666d25bbc5ad..232026a239a6 100644
--- a/arch/x86/kernel/cpu/microcode/core.c
+++ b/arch/x86/kernel/cpu/microcode/core.c
@@ -41,8 +41,6 @@
 
 #include "internal.h"
 
-#define DRIVER_VERSION	"2.2"
-
 static struct microcode_ops	*microcode_ops;
 bool dis_ucode_ldr = true;
 
@@ -77,6 +75,8 @@ static u32 final_levels[] = {
 	0, /* T-101 terminator */
 };
 
+struct early_load_data early_data;
+
 /*
  * Check the current patch level on this CPU.
  *
@@ -155,9 +155,9 @@ void __init load_ucode_bsp(void)
 		return;
 
 	if (intel)
-		load_ucode_intel_bsp();
+		load_ucode_intel_bsp(&early_data);
 	else
-		load_ucode_amd_bsp(cpuid_1_eax);
+		load_ucode_amd_bsp(&early_data, cpuid_1_eax);
 }
 
 void load_ucode_ap(void)
@@ -828,6 +828,11 @@ static int __init microcode_init(void)
 	if (!microcode_ops)
 		return -ENODEV;
 
+	pr_info_once("Current revision: 0x%08x\n", (early_data.new_rev ?: early_data.old_rev));
+
+	if (early_data.new_rev)
+		pr_info_once("Updated early from: 0x%08x\n", early_data.old_rev);
+
 	microcode_pdev = platform_device_register_simple("microcode", -1, NULL, 0);
 	if (IS_ERR(microcode_pdev))
 		return PTR_ERR(microcode_pdev);
@@ -846,8 +851,6 @@ static int __init microcode_init(void)
 	cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/microcode:online",
 			  mc_cpu_online, mc_cpu_down_prep);
 
-	pr_info("Microcode Update Driver: v%s.", DRIVER_VERSION);
-
 	return 0;
 
  out_pdev:
diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c
index 6024feb98d29..070426b9895f 100644
--- a/arch/x86/kernel/cpu/microcode/intel.c
+++ b/arch/x86/kernel/cpu/microcode/intel.c
@@ -339,16 +339,9 @@ static enum ucode_state __apply_microcode(struct ucode_cpu_info *uci,
 static enum ucode_state apply_microcode_early(struct ucode_cpu_info *uci)
 {
 	struct microcode_intel *mc = uci->mc;
-	enum ucode_state ret;
-	u32 cur_rev, date;
+	u32 cur_rev;
 
-	ret = __apply_microcode(uci, mc, &cur_rev);
-	if (ret == UCODE_UPDATED) {
-		date = mc->hdr.date;
-		pr_info_once("updated early: 0x%x -> 0x%x, date = %04x-%02x-%02x\n",
-			     cur_rev, mc->hdr.rev, date & 0xffff, date >> 24, (date >> 16) & 0xff);
-	}
-	return ret;
+	return __apply_microcode(uci, mc, &cur_rev);
 }
 
 static __init bool load_builtin_intel_microcode(struct cpio_data *cp)
@@ -413,13 +406,17 @@ static int __init save_builtin_microcode(void)
 early_initcall(save_builtin_microcode);
 
 /* Load microcode on BSP from initrd or builtin blobs */
-void __init load_ucode_intel_bsp(void)
+void __init load_ucode_intel_bsp(struct early_load_data *ed)
 {
 	struct ucode_cpu_info uci;
 
+	ed->old_rev = intel_get_microcode_revision();
+
 	uci.mc = get_microcode_blob(&uci, false);
 	if (uci.mc && apply_microcode_early(&uci) == UCODE_UPDATED)
 		ucode_patch_va = UCODE_BSP_LOADED;
+
+	ed->new_rev = uci.cpu_sig.rev;
 }
 
 void load_ucode_intel_ap(void)
diff --git a/arch/x86/kernel/cpu/microcode/internal.h b/arch/x86/kernel/cpu/microcode/internal.h
index f8047b12329a..21776c529fa9 100644
--- a/arch/x86/kernel/cpu/microcode/internal.h
+++ b/arch/x86/kernel/cpu/microcode/internal.h
@@ -37,6 +37,12 @@ struct microcode_ops {
 				use_nmi		: 1;
 };
 
+struct early_load_data {
+	u32 old_rev;
+	u32 new_rev;
+};
+
+extern struct early_load_data early_data;
 extern struct ucode_cpu_info ucode_cpu_info[];
 struct cpio_data find_microcode_in_initrd(const char *path);
 
@@ -92,14 +98,14 @@ extern bool dis_ucode_ldr;
 extern bool force_minrev;
 
 #ifdef CONFIG_CPU_SUP_AMD
-void load_ucode_amd_bsp(unsigned int family);
+void load_ucode_amd_bsp(struct early_load_data *ed, unsigned int family);
 void load_ucode_amd_ap(unsigned int family);
 int save_microcode_in_initrd_amd(unsigned int family);
 void reload_ucode_amd(unsigned int cpu);
 struct microcode_ops *init_amd_microcode(void);
 void exit_amd_microcode(void);
 #else /* CONFIG_CPU_SUP_AMD */
-static inline void load_ucode_amd_bsp(unsigned int family) { }
+static inline void load_ucode_amd_bsp(struct early_load_data *ed, unsigned int family) { }
 static inline void load_ucode_amd_ap(unsigned int family) { }
 static inline int save_microcode_in_initrd_amd(unsigned int family) { return -EINVAL; }
 static inline void reload_ucode_amd(unsigned int cpu) { }
@@ -108,12 +114,12 @@ static inline void exit_amd_microcode(void) { }
 #endif /* !CONFIG_CPU_SUP_AMD */
 
 #ifdef CONFIG_CPU_SUP_INTEL
-void load_ucode_intel_bsp(void);
+void load_ucode_intel_bsp(struct early_load_data *ed);
 void load_ucode_intel_ap(void);
 void reload_ucode_intel(void);
 struct microcode_ops *init_intel_microcode(void);
 #else /* CONFIG_CPU_SUP_INTEL */
-static inline void load_ucode_intel_bsp(void) { }
+static inline void load_ucode_intel_bsp(struct early_load_data *ed) { }
 static inline void load_ucode_intel_ap(void) { }
 static inline void reload_ucode_intel(void) { }
 static inline struct microcode_ops *init_intel_microcode(void) { return NULL; }
diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c
index 2d6aa5d2e3d7..d3524778a545 100644
--- a/arch/x86/kernel/cpu/mtrr/generic.c
+++ b/arch/x86/kernel/cpu/mtrr/generic.c
@@ -428,6 +428,10 @@ void __init mtrr_copy_map(void)
  * from the x86_init.hyper.init_platform() hook.  It can be called only once.
  * The MTRR state can't be changed afterwards.  To ensure that, X86_FEATURE_MTRR
  * is cleared.
+ *
+ * @var: MTRR variable range array to use
+ * @num_var: length of the @var array
+ * @def_type: default caching type
  */
 void mtrr_overwrite_state(struct mtrr_var_range *var, unsigned int num_var,
 			  mtrr_type def_type)
@@ -492,13 +496,15 @@ static u8 type_merge(u8 type, u8 new_type, u8 *uniform)
 /**
  * mtrr_type_lookup - look up memory type in MTRR
  *
+ * @start: Begin of the physical address range
+ * @end: End of the physical address range
+ * @uniform: output argument:
+ *  - 1: the returned MTRR type is valid for the whole region
+ *  - 0: otherwise
+ *
  * Return Values:
  * MTRR_TYPE_(type)  - The effective MTRR type for the region
  * MTRR_TYPE_INVALID - MTRR is disabled
- *
- * Output Argument:
- * uniform - Set to 1 when the returned MTRR type is valid for the whole
- *	     region, set to 0 else.
  */
 u8 mtrr_type_lookup(u64 start, u64 end, u8 *uniform)
 {
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 086a2c3aaaa0..3dcabbc49149 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -448,7 +448,7 @@ SYM_CODE_START(soft_restart_cpu)
 	UNWIND_HINT_END_OF_STACK
 
 	/* Find the idle task stack */
-	movq	PER_CPU_VAR(pcpu_hot) + X86_current_task, %rcx
+	movq	PER_CPU_VAR(pcpu_hot + X86_current_task), %rcx
 	movq	TASK_threadsp(%rcx), %rsp
 
 	jmp	.Ljump_to_C_code
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 708c87b88cc1..0917c7f25720 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -156,13 +156,12 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 {
 	struct thread_struct *prev = &prev_p->thread,
 			     *next = &next_p->thread;
-	struct fpu *prev_fpu = &prev->fpu;
 	int cpu = smp_processor_id();
 
 	/* never put a printk in __switch_to... printk() calls wake_up*() indirectly */
 
-	if (!test_thread_flag(TIF_NEED_FPU_LOAD))
-		switch_fpu_prepare(prev_fpu, cpu);
+	if (!test_tsk_thread_flag(prev_p, TIF_NEED_FPU_LOAD))
+		switch_fpu_prepare(prev_p, cpu);
 
 	/*
 	 * Save away %gs. No need to save %fs, as it was saved on the
@@ -209,7 +208,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 
 	raw_cpu_write(pcpu_hot.current_task, next_p);
 
-	switch_fpu_finish();
+	switch_fpu_finish(next_p);
 
 	/* Load the Intel cache allocation PQR MSR. */
 	resctrl_sched_in(next_p);
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 33b268747bb7..1553e19904e0 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -562,14 +562,13 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 {
 	struct thread_struct *prev = &prev_p->thread;
 	struct thread_struct *next = &next_p->thread;
-	struct fpu *prev_fpu = &prev->fpu;
 	int cpu = smp_processor_id();
 
 	WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) &&
 		     this_cpu_read(pcpu_hot.hardirq_stack_inuse));
 
-	if (!test_thread_flag(TIF_NEED_FPU_LOAD))
-		switch_fpu_prepare(prev_fpu, cpu);
+	if (!test_tsk_thread_flag(prev_p, TIF_NEED_FPU_LOAD))
+		switch_fpu_prepare(prev_p, cpu);
 
 	/* We must save %fs and %gs before load_TLS() because
 	 * %fs and %gs may be cleared by load_TLS().
@@ -623,7 +622,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 	raw_cpu_write(pcpu_hot.current_task, next_p);
 	raw_cpu_write(pcpu_hot.top_of_stack, task_top_of_stack(next_p));
 
-	switch_fpu_finish();
+	switch_fpu_finish(next_p);
 
 	/* Reload sp0. */
 	update_task_stack(next_p);
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 1526747bedf2..ec2c21a1844e 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -226,8 +226,6 @@ static void __init reserve_brk(void)
 	_brk_start = 0;
 }
 
-u64 relocated_ramdisk;
-
 #ifdef CONFIG_BLK_DEV_INITRD
 
 static u64 __init get_ramdisk_image(void)
@@ -261,7 +259,7 @@ static void __init relocate_initrd(void)
 	u64 area_size     = PAGE_ALIGN(ramdisk_size);
 
 	/* We need to move the initrd down into directly mapped mem */
-	relocated_ramdisk = memblock_phys_alloc_range(area_size, PAGE_SIZE, 0,
+	u64 relocated_ramdisk = memblock_phys_alloc_range(area_size, PAGE_SIZE, 0,
 						      PFN_PHYS(max_pfn_mapped));
 	if (!relocated_ramdisk)
 		panic("Cannot find place for new RAMDISK of size %lld\n",
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 2cc2aa120b4b..3f57ce68a3f1 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -757,6 +757,7 @@ const struct cpumask *cpu_clustergroup_mask(int cpu)
 {
 	return cpu_l2c_shared_mask(cpu);
 }
+EXPORT_SYMBOL_GPL(cpu_clustergroup_mask);
 
 static void impress_friends(void)
 {
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 54a5596adaa6..1239be7cc8d8 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -46,6 +46,7 @@ ENTRY(phys_startup_64)
 #endif
 
 jiffies = jiffies_64;
+const_pcpu_hot = pcpu_hot;
 
 #if defined(CONFIG_X86_64)
 /*
diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig
index 950c12868d30..c1716e83d176 100644
--- a/arch/x86/kvm/Kconfig
+++ b/arch/x86/kvm/Kconfig
@@ -24,7 +24,7 @@ config KVM
 	depends on HIGH_RES_TIMERS
 	depends on X86_LOCAL_APIC
 	select PREEMPT_NOTIFIERS
-	select MMU_NOTIFIER
+	select KVM_GENERIC_MMU_NOTIFIER
 	select HAVE_KVM_IRQCHIP
 	select HAVE_KVM_PFNCACHE
 	select HAVE_KVM_IRQFD
@@ -77,6 +77,18 @@ config KVM_WERROR
 
 	  If in doubt, say "N".
 
+config KVM_SW_PROTECTED_VM
+	bool "Enable support for KVM software-protected VMs"
+	depends on EXPERT
+	depends on X86_64
+	select KVM_GENERIC_PRIVATE_MEM
+	help
+	  Enable support for KVM software-protected VMs.  Currently "protected"
+	  means the VM can be backed with memory provided by
+	  KVM_CREATE_GUEST_MEMFD.
+
+	  If unsure, say "N".
+
 config KVM_INTEL
 	tristate "KVM for Intel (and compatible) processors support"
 	depends on KVM && IA32_FEAT_CTL
diff --git a/arch/x86/kvm/debugfs.c b/arch/x86/kvm/debugfs.c
index ee8c4c3496ed..42026b3f3ff3 100644
--- a/arch/x86/kvm/debugfs.c
+++ b/arch/x86/kvm/debugfs.c
@@ -111,7 +111,7 @@ static int kvm_mmu_rmaps_stat_show(struct seq_file *m, void *v)
 	mutex_lock(&kvm->slots_lock);
 	write_lock(&kvm->mmu_lock);
 
-	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
+	for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
 		int bkt;
 
 		slots = __kvm_memslots(kvm, i);
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index c57e181bba21..59b026b6ad2a 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -795,16 +795,26 @@ static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
 	return &slot->arch.lpage_info[level - 2][idx];
 }
 
+/*
+ * The most significant bit in disallow_lpage tracks whether or not memory
+ * attributes are mixed, i.e. not identical for all gfns at the current level.
+ * The lower order bits are used to refcount other cases where a hugepage is
+ * disallowed, e.g. if KVM has shadow a page table at the gfn.
+ */
+#define KVM_LPAGE_MIXED_FLAG	BIT(31)
+
 static void update_gfn_disallow_lpage_count(const struct kvm_memory_slot *slot,
 					    gfn_t gfn, int count)
 {
 	struct kvm_lpage_info *linfo;
-	int i;
+	int old, i;
 
 	for (i = PG_LEVEL_2M; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) {
 		linfo = lpage_info_slot(gfn, slot, i);
+
+		old = linfo->disallow_lpage;
 		linfo->disallow_lpage += count;
-		WARN_ON_ONCE(linfo->disallow_lpage < 0);
+		WARN_ON_ONCE((old ^ linfo->disallow_lpage) & KVM_LPAGE_MIXED_FLAG);
 	}
 }
 
@@ -3056,7 +3066,7 @@ static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
  *
  * There are several ways to safely use this helper:
  *
- * - Check mmu_invalidate_retry_hva() after grabbing the mapping level, before
+ * - Check mmu_invalidate_retry_gfn() after grabbing the mapping level, before
  *   consuming it.  In this case, mmu_lock doesn't need to be held during the
  *   lookup, but it does need to be held while checking the MMU notifier.
  *
@@ -3137,9 +3147,9 @@ static int host_pfn_mapping_level(struct kvm *kvm, gfn_t gfn,
 	return level;
 }
 
-int kvm_mmu_max_mapping_level(struct kvm *kvm,
-			      const struct kvm_memory_slot *slot, gfn_t gfn,
-			      int max_level)
+static int __kvm_mmu_max_mapping_level(struct kvm *kvm,
+				       const struct kvm_memory_slot *slot,
+				       gfn_t gfn, int max_level, bool is_private)
 {
 	struct kvm_lpage_info *linfo;
 	int host_level;
@@ -3151,6 +3161,9 @@ int kvm_mmu_max_mapping_level(struct kvm *kvm,
 			break;
 	}
 
+	if (is_private)
+		return max_level;
+
 	if (max_level == PG_LEVEL_4K)
 		return PG_LEVEL_4K;
 
@@ -3158,6 +3171,16 @@ int kvm_mmu_max_mapping_level(struct kvm *kvm,
 	return min(host_level, max_level);
 }
 
+int kvm_mmu_max_mapping_level(struct kvm *kvm,
+			      const struct kvm_memory_slot *slot, gfn_t gfn,
+			      int max_level)
+{
+	bool is_private = kvm_slot_can_be_private(slot) &&
+			  kvm_mem_is_private(kvm, gfn);
+
+	return __kvm_mmu_max_mapping_level(kvm, slot, gfn, max_level, is_private);
+}
+
 void kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
 {
 	struct kvm_memory_slot *slot = fault->slot;
@@ -3178,8 +3201,9 @@ void kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
 	 * Enforce the iTLB multihit workaround after capturing the requested
 	 * level, which will be used to do precise, accurate accounting.
 	 */
-	fault->req_level = kvm_mmu_max_mapping_level(vcpu->kvm, slot,
-						     fault->gfn, fault->max_level);
+	fault->req_level = __kvm_mmu_max_mapping_level(vcpu->kvm, slot,
+						       fault->gfn, fault->max_level,
+						       fault->is_private);
 	if (fault->req_level == PG_LEVEL_4K || fault->huge_page_disallowed)
 		return;
 
@@ -3739,7 +3763,7 @@ static int mmu_first_shadow_root_alloc(struct kvm *kvm)
 	    kvm_page_track_write_tracking_enabled(kvm))
 		goto out_success;
 
-	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
+	for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
 		slots = __kvm_memslots(kvm, i);
 		kvm_for_each_memslot(slot, bkt, slots) {
 			/*
@@ -4259,6 +4283,55 @@ void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work)
 	kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, 0, true, NULL);
 }
 
+static inline u8 kvm_max_level_for_order(int order)
+{
+	BUILD_BUG_ON(KVM_MAX_HUGEPAGE_LEVEL > PG_LEVEL_1G);
+
+	KVM_MMU_WARN_ON(order != KVM_HPAGE_GFN_SHIFT(PG_LEVEL_1G) &&
+			order != KVM_HPAGE_GFN_SHIFT(PG_LEVEL_2M) &&
+			order != KVM_HPAGE_GFN_SHIFT(PG_LEVEL_4K));
+
+	if (order >= KVM_HPAGE_GFN_SHIFT(PG_LEVEL_1G))
+		return PG_LEVEL_1G;
+
+	if (order >= KVM_HPAGE_GFN_SHIFT(PG_LEVEL_2M))
+		return PG_LEVEL_2M;
+
+	return PG_LEVEL_4K;
+}
+
+static void kvm_mmu_prepare_memory_fault_exit(struct kvm_vcpu *vcpu,
+					      struct kvm_page_fault *fault)
+{
+	kvm_prepare_memory_fault_exit(vcpu, fault->gfn << PAGE_SHIFT,
+				      PAGE_SIZE, fault->write, fault->exec,
+				      fault->is_private);
+}
+
+static int kvm_faultin_pfn_private(struct kvm_vcpu *vcpu,
+				   struct kvm_page_fault *fault)
+{
+	int max_order, r;
+
+	if (!kvm_slot_can_be_private(fault->slot)) {
+		kvm_mmu_prepare_memory_fault_exit(vcpu, fault);
+		return -EFAULT;
+	}
+
+	r = kvm_gmem_get_pfn(vcpu->kvm, fault->slot, fault->gfn, &fault->pfn,
+			     &max_order);
+	if (r) {
+		kvm_mmu_prepare_memory_fault_exit(vcpu, fault);
+		return r;
+	}
+
+	fault->max_level = min(kvm_max_level_for_order(max_order),
+			       fault->max_level);
+	fault->map_writable = !(fault->slot->flags & KVM_MEM_READONLY);
+
+	return RET_PF_CONTINUE;
+}
+
 static int __kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
 {
 	struct kvm_memory_slot *slot = fault->slot;
@@ -4291,6 +4364,14 @@ static int __kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
 			return RET_PF_EMULATE;
 	}
 
+	if (fault->is_private != kvm_mem_is_private(vcpu->kvm, fault->gfn)) {
+		kvm_mmu_prepare_memory_fault_exit(vcpu, fault);
+		return -EFAULT;
+	}
+
+	if (fault->is_private)
+		return kvm_faultin_pfn_private(vcpu, fault);
+
 	async = false;
 	fault->pfn = __gfn_to_pfn_memslot(slot, fault->gfn, false, false, &async,
 					  fault->write, &fault->map_writable,
@@ -4366,7 +4447,7 @@ static bool is_page_fault_stale(struct kvm_vcpu *vcpu,
 		return true;
 
 	return fault->slot &&
-	       mmu_invalidate_retry_hva(vcpu->kvm, fault->mmu_seq, fault->hva);
+	       mmu_invalidate_retry_gfn(vcpu->kvm, fault->mmu_seq, fault->gfn);
 }
 
 static int direct_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
@@ -6228,7 +6309,7 @@ static bool kvm_rmap_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_e
 	if (!kvm_memslots_have_rmaps(kvm))
 		return flush;
 
-	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
+	for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
 		slots = __kvm_memslots(kvm, i);
 
 		kvm_for_each_memslot_in_gfn_range(&iter, slots, gfn_start, gfn_end) {
@@ -6260,7 +6341,9 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
 
 	write_lock(&kvm->mmu_lock);
 
-	kvm_mmu_invalidate_begin(kvm, 0, -1ul);
+	kvm_mmu_invalidate_begin(kvm);
+
+	kvm_mmu_invalidate_range_add(kvm, gfn_start, gfn_end);
 
 	flush = kvm_rmap_zap_gfn_range(kvm, gfn_start, gfn_end);
 
@@ -6270,7 +6353,7 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
 	if (flush)
 		kvm_flush_remote_tlbs_range(kvm, gfn_start, gfn_end - gfn_start);
 
-	kvm_mmu_invalidate_end(kvm, 0, -1ul);
+	kvm_mmu_invalidate_end(kvm);
 
 	write_unlock(&kvm->mmu_lock);
 }
@@ -6723,7 +6806,7 @@ void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen)
 	 * modifier prior to checking for a wrap of the MMIO generation so
 	 * that a wrap in any address space is detected.
 	 */
-	gen &= ~((u64)KVM_ADDRESS_SPACE_NUM - 1);
+	gen &= ~((u64)kvm_arch_nr_memslot_as_ids(kvm) - 1);
 
 	/*
 	 * The very rare case: if the MMIO generation number has wrapped,
@@ -7176,3 +7259,163 @@ void kvm_mmu_pre_destroy_vm(struct kvm *kvm)
 	if (kvm->arch.nx_huge_page_recovery_thread)
 		kthread_stop(kvm->arch.nx_huge_page_recovery_thread);
 }
+
+#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
+bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm,
+					struct kvm_gfn_range *range)
+{
+	/*
+	 * Zap SPTEs even if the slot can't be mapped PRIVATE.  KVM x86 only
+	 * supports KVM_MEMORY_ATTRIBUTE_PRIVATE, and so it *seems* like KVM
+	 * can simply ignore such slots.  But if userspace is making memory
+	 * PRIVATE, then KVM must prevent the guest from accessing the memory
+	 * as shared.  And if userspace is making memory SHARED and this point
+	 * is reached, then at least one page within the range was previously
+	 * PRIVATE, i.e. the slot's possible hugepage ranges are changing.
+	 * Zapping SPTEs in this case ensures KVM will reassess whether or not
+	 * a hugepage can be used for affected ranges.
+	 */
+	if (WARN_ON_ONCE(!kvm_arch_has_private_mem(kvm)))
+		return false;
+
+	return kvm_unmap_gfn_range(kvm, range);
+}
+
+static bool hugepage_test_mixed(struct kvm_memory_slot *slot, gfn_t gfn,
+				int level)
+{
+	return lpage_info_slot(gfn, slot, level)->disallow_lpage & KVM_LPAGE_MIXED_FLAG;
+}
+
+static void hugepage_clear_mixed(struct kvm_memory_slot *slot, gfn_t gfn,
+				 int level)
+{
+	lpage_info_slot(gfn, slot, level)->disallow_lpage &= ~KVM_LPAGE_MIXED_FLAG;
+}
+
+static void hugepage_set_mixed(struct kvm_memory_slot *slot, gfn_t gfn,
+			       int level)
+{
+	lpage_info_slot(gfn, slot, level)->disallow_lpage |= KVM_LPAGE_MIXED_FLAG;
+}
+
+static bool hugepage_has_attrs(struct kvm *kvm, struct kvm_memory_slot *slot,
+			       gfn_t gfn, int level, unsigned long attrs)
+{
+	const unsigned long start = gfn;
+	const unsigned long end = start + KVM_PAGES_PER_HPAGE(level);
+
+	if (level == PG_LEVEL_2M)
+		return kvm_range_has_memory_attributes(kvm, start, end, attrs);
+
+	for (gfn = start; gfn < end; gfn += KVM_PAGES_PER_HPAGE(level - 1)) {
+		if (hugepage_test_mixed(slot, gfn, level - 1) ||
+		    attrs != kvm_get_memory_attributes(kvm, gfn))
+			return false;
+	}
+	return true;
+}
+
+bool kvm_arch_post_set_memory_attributes(struct kvm *kvm,
+					 struct kvm_gfn_range *range)
+{
+	unsigned long attrs = range->arg.attributes;
+	struct kvm_memory_slot *slot = range->slot;
+	int level;
+
+	lockdep_assert_held_write(&kvm->mmu_lock);
+	lockdep_assert_held(&kvm->slots_lock);
+
+	/*
+	 * Calculate which ranges can be mapped with hugepages even if the slot
+	 * can't map memory PRIVATE.  KVM mustn't create a SHARED hugepage over
+	 * a range that has PRIVATE GFNs, and conversely converting a range to
+	 * SHARED may now allow hugepages.
+	 */
+	if (WARN_ON_ONCE(!kvm_arch_has_private_mem(kvm)))
+		return false;
+
+	/*
+	 * The sequence matters here: upper levels consume the result of lower
+	 * level's scanning.
+	 */
+	for (level = PG_LEVEL_2M; level <= KVM_MAX_HUGEPAGE_LEVEL; level++) {
+		gfn_t nr_pages = KVM_PAGES_PER_HPAGE(level);
+		gfn_t gfn = gfn_round_for_level(range->start, level);
+
+		/* Process the head page if it straddles the range. */
+		if (gfn != range->start || gfn + nr_pages > range->end) {
+			/*
+			 * Skip mixed tracking if the aligned gfn isn't covered
+			 * by the memslot, KVM can't use a hugepage due to the
+			 * misaligned address regardless of memory attributes.
+			 */
+			if (gfn >= slot->base_gfn) {
+				if (hugepage_has_attrs(kvm, slot, gfn, level, attrs))
+					hugepage_clear_mixed(slot, gfn, level);
+				else
+					hugepage_set_mixed(slot, gfn, level);
+			}
+			gfn += nr_pages;
+		}
+
+		/*
+		 * Pages entirely covered by the range are guaranteed to have
+		 * only the attributes which were just set.
+		 */
+		for ( ; gfn + nr_pages <= range->end; gfn += nr_pages)
+			hugepage_clear_mixed(slot, gfn, level);
+
+		/*
+		 * Process the last tail page if it straddles the range and is
+		 * contained by the memslot.  Like the head page, KVM can't
+		 * create a hugepage if the slot size is misaligned.
+		 */
+		if (gfn < range->end &&
+		    (gfn + nr_pages) <= (slot->base_gfn + slot->npages)) {
+			if (hugepage_has_attrs(kvm, slot, gfn, level, attrs))
+				hugepage_clear_mixed(slot, gfn, level);
+			else
+				hugepage_set_mixed(slot, gfn, level);
+		}
+	}
+	return false;
+}
+
+void kvm_mmu_init_memslot_memory_attributes(struct kvm *kvm,
+					    struct kvm_memory_slot *slot)
+{
+	int level;
+
+	if (!kvm_arch_has_private_mem(kvm))
+		return;
+
+	for (level = PG_LEVEL_2M; level <= KVM_MAX_HUGEPAGE_LEVEL; level++) {
+		/*
+		 * Don't bother tracking mixed attributes for pages that can't
+		 * be huge due to alignment, i.e. process only pages that are
+		 * entirely contained by the memslot.
+		 */
+		gfn_t end = gfn_round_for_level(slot->base_gfn + slot->npages, level);
+		gfn_t start = gfn_round_for_level(slot->base_gfn, level);
+		gfn_t nr_pages = KVM_PAGES_PER_HPAGE(level);
+		gfn_t gfn;
+
+		if (start < slot->base_gfn)
+			start += nr_pages;
+
+		/*
+		 * Unlike setting attributes, every potential hugepage needs to
+		 * be manually checked as the attributes may already be mixed.
+		 */
+		for (gfn = start; gfn < end; gfn += nr_pages) {
+			unsigned long attrs = kvm_get_memory_attributes(kvm, gfn);
+
+			if (hugepage_has_attrs(kvm, slot, gfn, level, attrs))
+				hugepage_clear_mixed(slot, gfn, level);
+			else
+				hugepage_set_mixed(slot, gfn, level);
+		}
+	}
+}
+#endif
diff --git a/arch/x86/kvm/mmu/mmu_internal.h b/arch/x86/kvm/mmu/mmu_internal.h
index decc1f153669..b66a7d47e0e4 100644
--- a/arch/x86/kvm/mmu/mmu_internal.h
+++ b/arch/x86/kvm/mmu/mmu_internal.h
@@ -201,6 +201,7 @@ struct kvm_page_fault {
 
 	/* Derived from mmu and global state.  */
 	const bool is_tdp;
+	const bool is_private;
 	const bool nx_huge_page_workaround_enabled;
 
 	/*
@@ -296,6 +297,7 @@ static inline int kvm_mmu_do_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
 		.max_level = KVM_MAX_HUGEPAGE_LEVEL,
 		.req_level = PG_LEVEL_4K,
 		.goal_level = PG_LEVEL_4K,
+		.is_private = kvm_mem_is_private(vcpu->kvm, cr2_or_gpa >> PAGE_SHIFT),
 	};
 	int r;
 
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index be20a60047b1..40e3780d73ae 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -6757,10 +6757,10 @@ static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu)
 		return;
 
 	/*
-	 * Grab the memslot so that the hva lookup for the mmu_notifier retry
-	 * is guaranteed to use the same memslot as the pfn lookup, i.e. rely
-	 * on the pfn lookup's validation of the memslot to ensure a valid hva
-	 * is used for the retry check.
+	 * Explicitly grab the memslot using KVM's internal slot ID to ensure
+	 * KVM doesn't unintentionally grab a userspace memslot.  It _should_
+	 * be impossible for userspace to create a memslot for the APIC when
+	 * APICv is enabled, but paranoia won't hurt in this case.
 	 */
 	slot = id_to_memslot(slots, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT);
 	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
@@ -6785,8 +6785,7 @@ static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu)
 		return;
 
 	read_lock(&vcpu->kvm->mmu_lock);
-	if (mmu_invalidate_retry_hva(kvm, mmu_seq,
-				     gfn_to_hva_memslot(slot, gfn))) {
+	if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) {
 		kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
 		read_unlock(&vcpu->kvm->mmu_lock);
 		goto out;
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 2c924075f6f1..6d0772b47041 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -4548,6 +4548,13 @@ static int kvm_ioctl_get_supported_hv_cpuid(struct kvm_vcpu *vcpu,
 	return 0;
 }
 
+static bool kvm_is_vm_type_supported(unsigned long type)
+{
+	return type == KVM_X86_DEFAULT_VM ||
+	       (type == KVM_X86_SW_PROTECTED_VM &&
+		IS_ENABLED(CONFIG_KVM_SW_PROTECTED_VM) && tdp_enabled);
+}
+
 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 {
 	int r = 0;
@@ -4625,6 +4632,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 	case KVM_CAP_ENABLE_CAP:
 	case KVM_CAP_VM_DISABLE_NX_HUGE_PAGES:
 	case KVM_CAP_IRQFD_RESAMPLE:
+	case KVM_CAP_MEMORY_FAULT_INFO:
 		r = 1;
 		break;
 	case KVM_CAP_EXIT_HYPERCALL:
@@ -4738,6 +4746,11 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 	case KVM_CAP_X86_NOTIFY_VMEXIT:
 		r = kvm_caps.has_notify_vmexit;
 		break;
+	case KVM_CAP_VM_TYPES:
+		r = BIT(KVM_X86_DEFAULT_VM);
+		if (kvm_is_vm_type_supported(KVM_X86_SW_PROTECTED_VM))
+			r |= BIT(KVM_X86_SW_PROTECTED_VM);
+		break;
 	default:
 		break;
 	}
@@ -11081,6 +11094,7 @@ static int vcpu_run(struct kvm_vcpu *vcpu)
 {
 	int r;
 
+	vcpu->run->exit_reason = KVM_EXIT_UNKNOWN;
 	vcpu->arch.l1tf_flush_l1d = true;
 
 	for (;;) {
@@ -12434,9 +12448,11 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 	int ret;
 	unsigned long flags;
 
-	if (type)
+	if (!kvm_is_vm_type_supported(type))
 		return -EINVAL;
 
+	kvm->arch.vm_type = type;
+
 	ret = kvm_page_track_init(kvm);
 	if (ret)
 		goto out;
@@ -12575,8 +12591,8 @@ void __user * __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa,
 		hva = slot->userspace_addr;
 	}
 
-	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
-		struct kvm_userspace_memory_region m;
+	for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
+		struct kvm_userspace_memory_region2 m;
 
 		m.slot = id | (i << 16);
 		m.flags = 0;
@@ -12726,6 +12742,10 @@ static int kvm_alloc_memslot_metadata(struct kvm *kvm,
 		}
 	}
 
+#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
+	kvm_mmu_init_memslot_memory_attributes(kvm, slot);
+#endif
+
 	if (kvm_page_track_create_memslot(kvm, slot, npages))
 		goto out_free;
 
diff --git a/arch/x86/lib/cmpxchg16b_emu.S b/arch/x86/lib/cmpxchg16b_emu.S
index 6962df315793..4fb44894ad87 100644
--- a/arch/x86/lib/cmpxchg16b_emu.S
+++ b/arch/x86/lib/cmpxchg16b_emu.S
@@ -23,14 +23,14 @@ SYM_FUNC_START(this_cpu_cmpxchg16b_emu)
 	cli
 
 	/* if (*ptr == old) */
-	cmpq	PER_CPU_VAR(0(%rsi)), %rax
+	cmpq	__percpu (%rsi), %rax
 	jne	.Lnot_same
-	cmpq	PER_CPU_VAR(8(%rsi)), %rdx
+	cmpq	__percpu 8(%rsi), %rdx
 	jne	.Lnot_same
 
 	/* *ptr = new */
-	movq	%rbx, PER_CPU_VAR(0(%rsi))
-	movq	%rcx, PER_CPU_VAR(8(%rsi))
+	movq	%rbx, __percpu (%rsi)
+	movq	%rcx, __percpu 8(%rsi)
 
 	/* set ZF in EFLAGS to indicate success */
 	orl	$X86_EFLAGS_ZF, (%rsp)
@@ -42,8 +42,8 @@ SYM_FUNC_START(this_cpu_cmpxchg16b_emu)
 	/* *ptr != old */
 
 	/* old = *ptr */
-	movq	PER_CPU_VAR(0(%rsi)), %rax
-	movq	PER_CPU_VAR(8(%rsi)), %rdx
+	movq	__percpu (%rsi), %rax
+	movq	__percpu 8(%rsi), %rdx
 
 	/* clear ZF in EFLAGS to indicate failure */
 	andl	$(~X86_EFLAGS_ZF), (%rsp)
diff --git a/arch/x86/lib/cmpxchg8b_emu.S b/arch/x86/lib/cmpxchg8b_emu.S
index 873e4ef23e49..1c96be769adc 100644
--- a/arch/x86/lib/cmpxchg8b_emu.S
+++ b/arch/x86/lib/cmpxchg8b_emu.S
@@ -24,12 +24,12 @@ SYM_FUNC_START(cmpxchg8b_emu)
 	pushfl
 	cli
 
-	cmpl	0(%esi), %eax
+	cmpl	(%esi), %eax
 	jne	.Lnot_same
 	cmpl	4(%esi), %edx
 	jne	.Lnot_same
 
-	movl	%ebx, 0(%esi)
+	movl	%ebx, (%esi)
 	movl	%ecx, 4(%esi)
 
 	orl	$X86_EFLAGS_ZF, (%esp)
@@ -38,7 +38,7 @@ SYM_FUNC_START(cmpxchg8b_emu)
 	RET
 
 .Lnot_same:
-	movl	0(%esi), %eax
+	movl	(%esi), %eax
 	movl	4(%esi), %edx
 
 	andl	$(~X86_EFLAGS_ZF), (%esp)
@@ -53,18 +53,30 @@ EXPORT_SYMBOL(cmpxchg8b_emu)
 
 #ifndef CONFIG_UML
 
+/*
+ * Emulate 'cmpxchg8b %fs:(%rsi)'
+ *
+ * Inputs:
+ * %esi : memory location to compare
+ * %eax : low 32 bits of old value
+ * %edx : high 32 bits of old value
+ * %ebx : low 32 bits of new value
+ * %ecx : high 32 bits of new value
+ *
+ * Notably this is not LOCK prefixed and is not safe against NMIs
+ */
 SYM_FUNC_START(this_cpu_cmpxchg8b_emu)
 
 	pushfl
 	cli
 
-	cmpl	PER_CPU_VAR(0(%esi)), %eax
+	cmpl	__percpu (%esi), %eax
 	jne	.Lnot_same2
-	cmpl	PER_CPU_VAR(4(%esi)), %edx
+	cmpl	__percpu 4(%esi), %edx
 	jne	.Lnot_same2
 
-	movl	%ebx, PER_CPU_VAR(0(%esi))
-	movl	%ecx, PER_CPU_VAR(4(%esi))
+	movl	%ebx, __percpu (%esi)
+	movl	%ecx, __percpu 4(%esi)
 
 	orl	$X86_EFLAGS_ZF, (%esp)
 
@@ -72,8 +84,8 @@ SYM_FUNC_START(this_cpu_cmpxchg8b_emu)
 	RET
 
 .Lnot_same2:
-	movl	PER_CPU_VAR(0(%esi)), %eax
-	movl	PER_CPU_VAR(4(%esi)), %edx
+	movl	__percpu (%esi), %eax
+	movl	__percpu 4(%esi), %edx
 
 	andl	$(~X86_EFLAGS_ZF), (%esp)
 
diff --git a/arch/x86/lib/misc.c b/arch/x86/lib/misc.c
index 92cd8ecc3a2c..40b81c338ae5 100644
--- a/arch/x86/lib/misc.c
+++ b/arch/x86/lib/misc.c
@@ -8,7 +8,7 @@
  */
 int num_digits(int val)
 {
-	int m = 10;
+	long long m = 10;
 	int d = 1;
 
 	if (val < 0) {
diff --git a/arch/x86/mm/numa.c b/arch/x86/mm/numa.c
index b29ceb19e46e..adc497b93f03 100644
--- a/arch/x86/mm/numa.c
+++ b/arch/x86/mm/numa.c
@@ -449,37 +449,6 @@ int __node_distance(int from, int to)
 }
 EXPORT_SYMBOL(__node_distance);
 
-/*
- * Sanity check to catch more bad NUMA configurations (they are amazingly
- * common).  Make sure the nodes cover all memory.
- */
-static bool __init numa_meminfo_cover_memory(const struct numa_meminfo *mi)
-{
-	u64 numaram, e820ram;
-	int i;
-
-	numaram = 0;
-	for (i = 0; i < mi->nr_blks; i++) {
-		u64 s = mi->blk[i].start >> PAGE_SHIFT;
-		u64 e = mi->blk[i].end >> PAGE_SHIFT;
-		numaram += e - s;
-		numaram -= __absent_pages_in_range(mi->blk[i].nid, s, e);
-		if ((s64)numaram < 0)
-			numaram = 0;
-	}
-
-	e820ram = max_pfn - absent_pages_in_range(0, max_pfn);
-
-	/* We seem to lose 3 pages somewhere. Allow 1M of slack. */
-	if ((s64)(e820ram - numaram) >= (1 << (20 - PAGE_SHIFT))) {
-		printk(KERN_ERR "NUMA: nodes only cover %LuMB of your %LuMB e820 RAM. Not used.\n",
-		       (numaram << PAGE_SHIFT) >> 20,
-		       (e820ram << PAGE_SHIFT) >> 20);
-		return false;
-	}
-	return true;
-}
-
 /*
  * Mark all currently memblock-reserved physical memory (which covers the
  * kernel's own memory ranges) as hot-unswappable.
@@ -585,7 +554,8 @@ static int __init numa_register_memblks(struct numa_meminfo *mi)
 			return -EINVAL;
 		}
 	}
-	if (!numa_meminfo_cover_memory(mi))
+
+	if (!memblock_validate_numa_coverage(SZ_1M))
 		return -EINVAL;
 
 	/* Finally register nodes. */
diff --git a/arch/x86/platform/uv/uv_irq.c b/arch/x86/platform/uv/uv_irq.c
index 4221259a5870..a379501b7a69 100644
--- a/arch/x86/platform/uv/uv_irq.c
+++ b/arch/x86/platform/uv/uv_irq.c
@@ -35,7 +35,7 @@ static void uv_program_mmr(struct irq_cfg *cfg, struct uv_irq_2_mmr_pnode *info)
 	mmr_value = 0;
 	entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
 	entry->vector		= cfg->vector;
-	entry->delivery_mode	= apic->delivery_mode;
+	entry->delivery_mode	= APIC_DELIVERY_MODE_FIXED;
 	entry->dest_mode	= apic->dest_mode_logical;
 	entry->polarity		= 0;
 	entry->trigger		= 0;
diff --git a/arch/x86/xen/xen-asm.S b/arch/x86/xen/xen-asm.S
index 9e5e68008785..448958ddbaf8 100644
--- a/arch/x86/xen/xen-asm.S
+++ b/arch/x86/xen/xen-asm.S
@@ -28,7 +28,7 @@
  * non-zero.
  */
 SYM_FUNC_START(xen_irq_disable_direct)
-	movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
+	movb $1, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_mask)
 	RET
 SYM_FUNC_END(xen_irq_disable_direct)
 
@@ -69,7 +69,7 @@ SYM_FUNC_END(check_events)
 SYM_FUNC_START(xen_irq_enable_direct)
 	FRAME_BEGIN
 	/* Unmask events */
-	movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
+	movb $0, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_mask)
 
 	/*
 	 * Preempt here doesn't matter because that will deal with any
@@ -78,7 +78,7 @@ SYM_FUNC_START(xen_irq_enable_direct)
 	 */
 
 	/* Test for pending */
-	testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
+	testb $0xff, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_pending)
 	jz 1f
 
 	call check_events
@@ -97,7 +97,7 @@ SYM_FUNC_END(xen_irq_enable_direct)
  * x86 use opposite senses (mask vs enable).
  */
 SYM_FUNC_START(xen_save_fl_direct)
-	testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
+	testb $0xff, PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_mask)
 	setz %ah
 	addb %ah, %ah
 	RET
@@ -113,7 +113,7 @@ SYM_FUNC_END(xen_read_cr2);
 
 SYM_FUNC_START(xen_read_cr2_direct)
 	FRAME_BEGIN
-	_ASM_MOV PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_arch_cr2, %_ASM_AX
+	_ASM_MOV PER_CPU_VAR(xen_vcpu_info + XEN_vcpu_info_arch_cr2), %_ASM_AX
 	FRAME_END
 	RET
 SYM_FUNC_END(xen_read_cr2_direct);
diff --git a/arch/xtensa/kernel/syscalls/syscall.tbl b/arch/xtensa/kernel/syscalls/syscall.tbl
index 06eefa9c1458..dd116598fb25 100644
--- a/arch/xtensa/kernel/syscalls/syscall.tbl
+++ b/arch/xtensa/kernel/syscalls/syscall.tbl
@@ -427,3 +427,8 @@
 454	common	futex_wake			sys_futex_wake
 455	common	futex_wait			sys_futex_wait
 456	common	futex_requeue			sys_futex_requeue
+457	common	statmount			sys_statmount
+458	common	listmount			sys_listmount
+459	common	lsm_get_self_attr		sys_lsm_get_self_attr
+460	common	lsm_set_self_attr		sys_lsm_set_self_attr
+461	common	lsm_list_modules		sys_lsm_list_modules
diff --git a/arch/xtensa/variants/esp32/include/variant/core.h b/arch/xtensa/variants/esp32/include/variant/core.h
new file mode 100644
index 000000000000..f3f4e45f001f
--- /dev/null
+++ b/arch/xtensa/variants/esp32/include/variant/core.h
@@ -0,0 +1,655 @@
+/* 
+ * xtensa/config/core-isa.h -- HAL definitions that are dependent on Xtensa
+ *				processor CORE configuration
+ *
+ *  See <xtensa/config/core.h>, which includes this file, for more details.
+ */
+
+/* Xtensa processor core configuration information.
+
+   Copyright (c) 1999-2016 Tensilica Inc.
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be included
+   in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+   IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+   CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+   TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+   SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.  */
+
+#ifndef _XTENSA_CORE_CONFIGURATION_H
+#define _XTENSA_CORE_CONFIGURATION_H
+
+
+/****************************************************************************
+	    Parameters Useful for Any Code, USER or PRIVILEGED
+ ****************************************************************************/
+
+/*
+ *  Note:  Macros of the form XCHAL_HAVE_*** have a value of 1 if the option is
+ *  configured, and a value of 0 otherwise.  These macros are always defined.
+ */
+
+
+/*----------------------------------------------------------------------
+				ISA
+  ----------------------------------------------------------------------*/
+
+#define XCHAL_HAVE_BE			0	/* big-endian byte ordering */
+#define XCHAL_HAVE_WINDOWED		1	/* windowed registers option */
+#define XCHAL_NUM_AREGS			64	/* num of physical addr regs */
+#define XCHAL_NUM_AREGS_LOG2		6	/* log2(XCHAL_NUM_AREGS) */
+#define XCHAL_MAX_INSTRUCTION_SIZE	3	/* max instr bytes (3..8) */
+#define XCHAL_HAVE_DEBUG		1	/* debug option */
+#define XCHAL_HAVE_DENSITY		1	/* 16-bit instructions */
+#define XCHAL_HAVE_LOOPS		1	/* zero-overhead loops */
+#define XCHAL_LOOP_BUFFER_SIZE		256	/* zero-ov. loop instr buffer size */
+#define XCHAL_HAVE_NSA			1	/* NSA/NSAU instructions */
+#define XCHAL_HAVE_MINMAX		1	/* MIN/MAX instructions */
+#define XCHAL_HAVE_SEXT			1	/* SEXT instruction */
+#define XCHAL_HAVE_DEPBITS		0	/* DEPBITS instruction */
+#define XCHAL_HAVE_CLAMPS		1	/* CLAMPS instruction */
+#define XCHAL_HAVE_MUL16		1	/* MUL16S/MUL16U instructions */
+#define XCHAL_HAVE_MUL32		1	/* MULL instruction */
+#define XCHAL_HAVE_MUL32_HIGH		1	/* MULUH/MULSH instructions */
+#define XCHAL_HAVE_DIV32		1	/* QUOS/QUOU/REMS/REMU instructions */
+#define XCHAL_HAVE_L32R			1	/* L32R instruction */
+#define XCHAL_HAVE_ABSOLUTE_LITERALS	0	/* non-PC-rel (extended) L32R */
+#define XCHAL_HAVE_CONST16		0	/* CONST16 instruction */
+#define XCHAL_HAVE_ADDX			1	/* ADDX#/SUBX# instructions */
+#define XCHAL_HAVE_WIDE_BRANCHES	0	/* B*.W18 or B*.W15 instr's */
+#define XCHAL_HAVE_PREDICTED_BRANCHES	0	/* B[EQ/EQZ/NE/NEZ]T instr's */
+#define XCHAL_HAVE_CALL4AND12		1	/* (obsolete option) */
+#define XCHAL_HAVE_ABS			1	/* ABS instruction */
+/*#define XCHAL_HAVE_POPC		0*/	/* POPC instruction */
+/*#define XCHAL_HAVE_CRC		0*/	/* CRC instruction */
+#define XCHAL_HAVE_RELEASE_SYNC		1	/* L32AI/S32RI instructions */
+#define XCHAL_HAVE_S32C1I		1	/* S32C1I instruction */
+#define XCHAL_HAVE_SPECULATION		0	/* speculation */
+#define XCHAL_HAVE_FULL_RESET		1	/* all regs/state reset */
+#define XCHAL_NUM_CONTEXTS		1	/* */
+#define XCHAL_NUM_MISC_REGS		4	/* num of scratch regs (0..4) */
+#define XCHAL_HAVE_TAP_MASTER		0	/* JTAG TAP control instr's */
+#define XCHAL_HAVE_PRID			1	/* processor ID register */
+#define XCHAL_HAVE_EXTERN_REGS		1	/* WER/RER instructions */
+#define XCHAL_HAVE_MX			0	/* MX core (Tensilica internal) */
+#define XCHAL_HAVE_MP_INTERRUPTS	0	/* interrupt distributor port */
+#define XCHAL_HAVE_MP_RUNSTALL		0	/* core RunStall control port */
+#define XCHAL_HAVE_PSO			0	/* Power Shut-Off */
+#define XCHAL_HAVE_PSO_CDM		0	/* core/debug/mem pwr domains */
+#define XCHAL_HAVE_PSO_FULL_RETENTION	0	/* all regs preserved on PSO */
+#define XCHAL_HAVE_THREADPTR		1	/* THREADPTR register */
+#define XCHAL_HAVE_BOOLEANS		1	/* boolean registers */
+#define XCHAL_HAVE_CP			1	/* CPENABLE reg (coprocessor) */
+#define XCHAL_CP_MAXCFG			8	/* max allowed cp id plus one */
+#define XCHAL_HAVE_MAC16		1	/* MAC16 package */
+
+#define XCHAL_HAVE_FUSION		 0	/* Fusion*/
+#define XCHAL_HAVE_FUSION_FP	 0	        /* Fusion FP option */
+#define XCHAL_HAVE_FUSION_LOW_POWER 0	/* Fusion Low Power option */
+#define XCHAL_HAVE_FUSION_AES	 0	        /* Fusion BLE/Wifi AES-128 CCM option */
+#define XCHAL_HAVE_FUSION_CONVENC	 0       /* Fusion Conv Encode option */
+#define XCHAL_HAVE_FUSION_LFSR_CRC	 0	/* Fusion LFSR-CRC option */
+#define XCHAL_HAVE_FUSION_BITOPS	 0	/* Fusion Bit Operations Support option */
+#define XCHAL_HAVE_FUSION_AVS	 0	/* Fusion AVS option */
+#define XCHAL_HAVE_FUSION_16BIT_BASEBAND	 0	/* Fusion 16-bit Baseband option */
+#define XCHAL_HAVE_FUSION_VITERBI        0     /* Fusion Viterbi option */
+#define XCHAL_HAVE_FUSION_SOFTDEMAP      0   /* Fusion Soft Bit Demap option */
+#define XCHAL_HAVE_HIFIPRO		0	/* HiFiPro Audio Engine pkg */
+#define XCHAL_HAVE_HIFI4		0	/* HiFi4 Audio Engine pkg */
+#define XCHAL_HAVE_HIFI4_VFPU		0	/* HiFi4 Audio Engine VFPU option */
+#define XCHAL_HAVE_HIFI3		0	/* HiFi3 Audio Engine pkg */
+#define XCHAL_HAVE_HIFI3_VFPU		0	/* HiFi3 Audio Engine VFPU option */
+#define XCHAL_HAVE_HIFI2		0	/* HiFi2 Audio Engine pkg */
+#define XCHAL_HAVE_HIFI2EP		0	/* HiFi2EP */
+#define XCHAL_HAVE_HIFI_MINI		0	
+
+
+#define XCHAL_HAVE_VECTORFPU2005	0	/* vector or user floating-point pkg */
+#define XCHAL_HAVE_USER_DPFPU         0       /* user DP floating-point pkg */
+#define XCHAL_HAVE_USER_SPFPU         0       /* user DP floating-point pkg */
+#define XCHAL_HAVE_FP                 1      /* single prec floating point */
+#define XCHAL_HAVE_FP_DIV             1  /* FP with DIV instructions */
+#define XCHAL_HAVE_FP_RECIP           1        /* FP with RECIP instructions */
+#define XCHAL_HAVE_FP_SQRT            1 /* FP with SQRT instructions */
+#define XCHAL_HAVE_FP_RSQRT           1        /* FP with RSQRT instructions */
+#define XCHAL_HAVE_DFP                        0     /* double precision FP pkg */
+#define XCHAL_HAVE_DFP_DIV            0 /* DFP with DIV instructions */
+#define XCHAL_HAVE_DFP_RECIP          0       /* DFP with RECIP instructions*/
+#define XCHAL_HAVE_DFP_SQRT           0        /* DFP with SQRT instructions */
+#define XCHAL_HAVE_DFP_RSQRT          0       /* DFP with RSQRT instructions*/
+#define XCHAL_HAVE_DFP_ACCEL		1	/* double precision FP acceleration pkg */
+#define XCHAL_HAVE_DFP_accel		XCHAL_HAVE_DFP_ACCEL				/* for backward compatibility */
+
+#define XCHAL_HAVE_DFPU_SINGLE_ONLY    1                 	/* DFPU Coprocessor, single precision only */
+#define XCHAL_HAVE_DFPU_SINGLE_DOUBLE  0               	/* DFPU Coprocessor, single and double precision */
+#define XCHAL_HAVE_VECTRA1		0	/* Vectra I  pkg */
+#define XCHAL_HAVE_VECTRALX		0	/* Vectra LX pkg */
+#define XCHAL_HAVE_PDX4		        0	/* PDX4 */
+#define XCHAL_HAVE_CONNXD2		0	/* ConnX D2 pkg */
+#define XCHAL_HAVE_CONNXD2_DUALLSFLIX   0	/* ConnX D2 & Dual LoadStore Flix */
+#define XCHAL_HAVE_BBE16		0	/* ConnX BBE16 pkg */
+#define XCHAL_HAVE_BBE16_RSQRT		0	/* BBE16 & vector recip sqrt */
+#define XCHAL_HAVE_BBE16_VECDIV		0	/* BBE16 & vector divide */
+#define XCHAL_HAVE_BBE16_DESPREAD	0	/* BBE16 & despread */
+#define XCHAL_HAVE_BBENEP		0	/* ConnX BBENEP pkgs */
+#define XCHAL_HAVE_BSP3			0	/* ConnX BSP3 pkg */
+#define XCHAL_HAVE_BSP3_TRANSPOSE	0	/* BSP3 & transpose32x32 */
+#define XCHAL_HAVE_SSP16		0	/* ConnX SSP16 pkg */
+#define XCHAL_HAVE_SSP16_VITERBI	0	/* SSP16 & viterbi */
+#define XCHAL_HAVE_TURBO16		0	/* ConnX Turbo16 pkg */
+#define XCHAL_HAVE_BBP16		0	/* ConnX BBP16 pkg */
+#define XCHAL_HAVE_FLIX3		0	/* basic 3-way FLIX option */
+#define XCHAL_HAVE_GRIVPEP              0   /*  GRIVPEP is General Release of IVPEP */
+#define XCHAL_HAVE_GRIVPEP_HISTOGRAM    0   /* Histogram option on GRIVPEP */
+
+
+/*----------------------------------------------------------------------
+				MISC
+  ----------------------------------------------------------------------*/
+
+#define XCHAL_NUM_LOADSTORE_UNITS	1	/* load/store units */
+#define XCHAL_NUM_WRITEBUFFER_ENTRIES	4	/* size of write buffer */
+#define XCHAL_INST_FETCH_WIDTH		4	/* instr-fetch width in bytes */
+#define XCHAL_DATA_WIDTH		4	/* data width in bytes */
+#define XCHAL_DATA_PIPE_DELAY		2	/* d-side pipeline delay
+						   (1 = 5-stage, 2 = 7-stage) */
+#define XCHAL_CLOCK_GATING_GLOBAL	1	/* global clock gating */
+#define XCHAL_CLOCK_GATING_FUNCUNIT	1	/* funct. unit clock gating */
+/*  In T1050, applies to selected core load and store instructions (see ISA): */
+#define XCHAL_UNALIGNED_LOAD_EXCEPTION	0	/* unaligned loads cause exc. */
+#define XCHAL_UNALIGNED_STORE_EXCEPTION	0	/* unaligned stores cause exc.*/
+#define XCHAL_UNALIGNED_LOAD_HW		1	/* unaligned loads work in hw */
+#define XCHAL_UNALIGNED_STORE_HW	1	/* unaligned stores work in hw*/
+
+#define XCHAL_SW_VERSION		1100003	/* sw version of this header */
+
+#define XCHAL_CORE_ID			"esp32_v3_49_prod"	/* alphanum core name
+						   (CoreID) set in the Xtensa
+						   Processor Generator */
+
+#define XCHAL_BUILD_UNIQUE_ID		0x0005FE96	/* 22-bit sw build ID */
+
+/*
+ *  These definitions describe the hardware targeted by this software.
+ */
+#define XCHAL_HW_CONFIGID0		0xC2BCFFFE	/* ConfigID hi 32 bits*/
+#define XCHAL_HW_CONFIGID1		0x1CC5FE96	/* ConfigID lo 32 bits*/
+#define XCHAL_HW_VERSION_NAME		"LX6.0.3"	/* full version name */
+#define XCHAL_HW_VERSION_MAJOR		2600	/* major ver# of targeted hw */
+#define XCHAL_HW_VERSION_MINOR		3	/* minor ver# of targeted hw */
+#define XCHAL_HW_VERSION		260003	/* major*100+minor */
+#define XCHAL_HW_REL_LX6		1
+#define XCHAL_HW_REL_LX6_0		1
+#define XCHAL_HW_REL_LX6_0_3		1
+#define XCHAL_HW_CONFIGID_RELIABLE	1
+/*  If software targets a *range* of hardware versions, these are the bounds: */
+#define XCHAL_HW_MIN_VERSION_MAJOR	2600	/* major v of earliest tgt hw */
+#define XCHAL_HW_MIN_VERSION_MINOR	3	/* minor v of earliest tgt hw */
+#define XCHAL_HW_MIN_VERSION		260003	/* earliest targeted hw */
+#define XCHAL_HW_MAX_VERSION_MAJOR	2600	/* major v of latest tgt hw */
+#define XCHAL_HW_MAX_VERSION_MINOR	3	/* minor v of latest tgt hw */
+#define XCHAL_HW_MAX_VERSION		260003	/* latest targeted hw */
+
+
+/*----------------------------------------------------------------------
+				CACHE
+  ----------------------------------------------------------------------*/
+
+#define XCHAL_ICACHE_LINESIZE		4	/* I-cache line size in bytes */
+#define XCHAL_DCACHE_LINESIZE		4	/* D-cache line size in bytes */
+#define XCHAL_ICACHE_LINEWIDTH		2	/* log2(I line size in bytes) */
+#define XCHAL_DCACHE_LINEWIDTH		2	/* log2(D line size in bytes) */
+
+#define XCHAL_ICACHE_SIZE		0	/* I-cache size in bytes or 0 */
+#define XCHAL_DCACHE_SIZE		0	/* D-cache size in bytes or 0 */
+
+#define XCHAL_DCACHE_IS_WRITEBACK	0	/* writeback feature */
+#define XCHAL_DCACHE_IS_COHERENT	0	/* MP coherence feature */
+
+#define XCHAL_HAVE_PREFETCH		0	/* PREFCTL register */
+#define XCHAL_HAVE_PREFETCH_L1		0	/* prefetch to L1 dcache */
+#define XCHAL_PREFETCH_CASTOUT_LINES	0	/* dcache pref. castout bufsz */
+#define XCHAL_PREFETCH_ENTRIES		0	/* cache prefetch entries */
+#define XCHAL_PREFETCH_BLOCK_ENTRIES	0	/* prefetch block streams */
+#define XCHAL_HAVE_CACHE_BLOCKOPS	0	/* block prefetch for caches */
+#define XCHAL_HAVE_ICACHE_TEST		0	/* Icache test instructions */
+#define XCHAL_HAVE_DCACHE_TEST		0	/* Dcache test instructions */
+#define XCHAL_HAVE_ICACHE_DYN_WAYS	0	/* Icache dynamic way support */
+#define XCHAL_HAVE_DCACHE_DYN_WAYS	0	/* Dcache dynamic way support */
+
+
+
+
+/****************************************************************************
+    Parameters Useful for PRIVILEGED (Supervisory or Non-Virtualized) Code
+ ****************************************************************************/
+
+
+#ifndef XTENSA_HAL_NON_PRIVILEGED_ONLY
+
+/*----------------------------------------------------------------------
+				CACHE
+  ----------------------------------------------------------------------*/
+
+#define XCHAL_HAVE_PIF			1	/* any outbound PIF present */
+#define XCHAL_HAVE_AXI			0	/* AXI bus */
+
+#define XCHAL_HAVE_PIF_WR_RESP			0	/* pif write response */
+#define XCHAL_HAVE_PIF_REQ_ATTR			0	/* pif attribute */
+
+/*  If present, cache size in bytes == (ways * 2^(linewidth + setwidth)).  */
+
+/*  Number of cache sets in log2(lines per way):  */
+#define XCHAL_ICACHE_SETWIDTH		0
+#define XCHAL_DCACHE_SETWIDTH		0
+
+/*  Cache set associativity (number of ways):  */
+#define XCHAL_ICACHE_WAYS		1
+#define XCHAL_DCACHE_WAYS		1
+
+/*  Cache features:  */
+#define XCHAL_ICACHE_LINE_LOCKABLE	0
+#define XCHAL_DCACHE_LINE_LOCKABLE	0
+#define XCHAL_ICACHE_ECC_PARITY		0
+#define XCHAL_DCACHE_ECC_PARITY		0
+
+/*  Cache access size in bytes (affects operation of SICW instruction):  */
+#define XCHAL_ICACHE_ACCESS_SIZE	1
+#define XCHAL_DCACHE_ACCESS_SIZE	1
+
+#define XCHAL_DCACHE_BANKS		0	/* number of banks */
+
+/*  Number of encoded cache attr bits (see <xtensa/hal.h> for decoded bits):  */
+#define XCHAL_CA_BITS			4
+
+
+/*----------------------------------------------------------------------
+			INTERNAL I/D RAM/ROMs and XLMI
+  ----------------------------------------------------------------------*/
+
+#define XCHAL_NUM_INSTROM		1	/* number of core instr. ROMs */
+#define XCHAL_NUM_INSTRAM		2	/* number of core instr. RAMs */
+#define XCHAL_NUM_DATAROM		1	/* number of core data ROMs */
+#define XCHAL_NUM_DATARAM		2	/* number of core data RAMs */
+#define XCHAL_NUM_URAM			0	/* number of core unified RAMs*/
+#define XCHAL_NUM_XLMI			1	/* number of core XLMI ports */
+
+/*  Instruction ROM 0:  */
+#define XCHAL_INSTROM0_VADDR		0x40800000	/* virtual address */
+#define XCHAL_INSTROM0_PADDR		0x40800000	/* physical address */
+#define XCHAL_INSTROM0_SIZE		4194304	/* size in bytes */
+#define XCHAL_INSTROM0_ECC_PARITY	0	/* ECC/parity type, 0=none */
+
+/*  Instruction RAM 0:  */
+#define XCHAL_INSTRAM0_VADDR		0x40000000	/* virtual address */
+#define XCHAL_INSTRAM0_PADDR		0x40000000	/* physical address */
+#define XCHAL_INSTRAM0_SIZE		4194304	/* size in bytes */
+#define XCHAL_INSTRAM0_ECC_PARITY	0	/* ECC/parity type, 0=none */
+
+/*  Instruction RAM 1:  */
+#define XCHAL_INSTRAM1_VADDR		0x40400000	/* virtual address */
+#define XCHAL_INSTRAM1_PADDR		0x40400000	/* physical address */
+#define XCHAL_INSTRAM1_SIZE		4194304	/* size in bytes */
+#define XCHAL_INSTRAM1_ECC_PARITY	0	/* ECC/parity type, 0=none */
+
+/*  Data ROM 0:  */
+#define XCHAL_DATAROM0_VADDR		0x3F400000	/* virtual address */
+#define XCHAL_DATAROM0_PADDR		0x3F400000	/* physical address */
+#define XCHAL_DATAROM0_SIZE		4194304	/* size in bytes */
+#define XCHAL_DATAROM0_ECC_PARITY	0	/* ECC/parity type, 0=none */
+#define XCHAL_DATAROM0_BANKS		1	/* number of banks */
+
+/*  Data RAM 0:  */
+#define XCHAL_DATARAM0_VADDR		0x3FF80000	/* virtual address */
+#define XCHAL_DATARAM0_PADDR		0x3FF80000	/* physical address */
+#define XCHAL_DATARAM0_SIZE		524288	/* size in bytes */
+#define XCHAL_DATARAM0_ECC_PARITY	0	/* ECC/parity type, 0=none */
+#define XCHAL_DATARAM0_BANKS		1	/* number of banks */
+
+/*  Data RAM 1:  */
+#define XCHAL_DATARAM1_VADDR		0x3F800000	/* virtual address */
+#define XCHAL_DATARAM1_PADDR		0x3F800000	/* physical address */
+#define XCHAL_DATARAM1_SIZE		4194304	/* size in bytes */
+#define XCHAL_DATARAM1_ECC_PARITY	0	/* ECC/parity type, 0=none */
+#define XCHAL_DATARAM1_BANKS		1	/* number of banks */
+
+/*  XLMI Port 0:  */
+#define XCHAL_XLMI0_VADDR		0x3FF00000	/* virtual address */
+#define XCHAL_XLMI0_PADDR		0x3FF00000	/* physical address */
+#define XCHAL_XLMI0_SIZE		524288	/* size in bytes */
+#define XCHAL_XLMI0_ECC_PARITY	0	/* ECC/parity type, 0=none */
+
+#define XCHAL_HAVE_IMEM_LOADSTORE	1	/* can load/store to IROM/IRAM*/
+
+
+/*----------------------------------------------------------------------
+			INTERRUPTS and TIMERS
+  ----------------------------------------------------------------------*/
+
+#define XCHAL_HAVE_INTERRUPTS		1	/* interrupt option */
+#define XCHAL_HAVE_HIGHPRI_INTERRUPTS	1	/* med/high-pri. interrupts */
+#define XCHAL_HAVE_NMI			1	/* non-maskable interrupt */
+#define XCHAL_HAVE_CCOUNT		1	/* CCOUNT reg. (timer option) */
+#define XCHAL_NUM_TIMERS		3	/* number of CCOMPAREn regs */
+#define XCHAL_NUM_INTERRUPTS		32	/* number of interrupts */
+#define XCHAL_NUM_INTERRUPTS_LOG2	5	/* ceil(log2(NUM_INTERRUPTS)) */
+#define XCHAL_NUM_EXTINTERRUPTS		26	/* num of external interrupts */
+#define XCHAL_NUM_INTLEVELS		6	/* number of interrupt levels
+						   (not including level zero) */
+#define XCHAL_EXCM_LEVEL		3	/* level masked by PS.EXCM */
+	/* (always 1 in XEA1; levels 2 .. EXCM_LEVEL are "medium priority") */
+
+/*  Masks of interrupts at each interrupt level:  */
+#define XCHAL_INTLEVEL1_MASK		0x000637FF
+#define XCHAL_INTLEVEL2_MASK		0x00380000
+#define XCHAL_INTLEVEL3_MASK		0x28C08800
+#define XCHAL_INTLEVEL4_MASK		0x53000000
+#define XCHAL_INTLEVEL5_MASK		0x84010000
+#define XCHAL_INTLEVEL6_MASK		0x00000000
+#define XCHAL_INTLEVEL7_MASK		0x00004000
+
+/*  Masks of interrupts at each range 1..n of interrupt levels:  */
+#define XCHAL_INTLEVEL1_ANDBELOW_MASK	0x000637FF
+#define XCHAL_INTLEVEL2_ANDBELOW_MASK	0x003E37FF
+#define XCHAL_INTLEVEL3_ANDBELOW_MASK	0x28FEBFFF
+#define XCHAL_INTLEVEL4_ANDBELOW_MASK	0x7BFEBFFF
+#define XCHAL_INTLEVEL5_ANDBELOW_MASK	0xFFFFBFFF
+#define XCHAL_INTLEVEL6_ANDBELOW_MASK	0xFFFFBFFF
+#define XCHAL_INTLEVEL7_ANDBELOW_MASK	0xFFFFFFFF
+
+/*  Level of each interrupt:  */
+#define XCHAL_INT0_LEVEL		1
+#define XCHAL_INT1_LEVEL		1
+#define XCHAL_INT2_LEVEL		1
+#define XCHAL_INT3_LEVEL		1
+#define XCHAL_INT4_LEVEL		1
+#define XCHAL_INT5_LEVEL		1
+#define XCHAL_INT6_LEVEL		1
+#define XCHAL_INT7_LEVEL		1
+#define XCHAL_INT8_LEVEL		1
+#define XCHAL_INT9_LEVEL		1
+#define XCHAL_INT10_LEVEL		1
+#define XCHAL_INT11_LEVEL		3
+#define XCHAL_INT12_LEVEL		1
+#define XCHAL_INT13_LEVEL		1
+#define XCHAL_INT14_LEVEL		7
+#define XCHAL_INT15_LEVEL		3
+#define XCHAL_INT16_LEVEL		5
+#define XCHAL_INT17_LEVEL		1
+#define XCHAL_INT18_LEVEL		1
+#define XCHAL_INT19_LEVEL		2
+#define XCHAL_INT20_LEVEL		2
+#define XCHAL_INT21_LEVEL		2
+#define XCHAL_INT22_LEVEL		3
+#define XCHAL_INT23_LEVEL		3
+#define XCHAL_INT24_LEVEL		4
+#define XCHAL_INT25_LEVEL		4
+#define XCHAL_INT26_LEVEL		5
+#define XCHAL_INT27_LEVEL		3
+#define XCHAL_INT28_LEVEL		4
+#define XCHAL_INT29_LEVEL		3
+#define XCHAL_INT30_LEVEL		4
+#define XCHAL_INT31_LEVEL		5
+#define XCHAL_DEBUGLEVEL		6	/* debug interrupt level */
+#define XCHAL_HAVE_DEBUG_EXTERN_INT	1	/* OCD external db interrupt */
+#define XCHAL_NMILEVEL			7	/* NMI "level" (for use with
+						   EXCSAVE/EPS/EPC_n, RFI n) */
+
+/*  Type of each interrupt:  */
+#define XCHAL_INT0_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT1_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT2_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT3_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT4_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT5_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT6_TYPE 	XTHAL_INTTYPE_TIMER
+#define XCHAL_INT7_TYPE 	XTHAL_INTTYPE_SOFTWARE
+#define XCHAL_INT8_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT9_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT10_TYPE 	XTHAL_INTTYPE_EXTERN_EDGE
+#define XCHAL_INT11_TYPE 	XTHAL_INTTYPE_PROFILING
+#define XCHAL_INT12_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT13_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT14_TYPE 	XTHAL_INTTYPE_NMI
+#define XCHAL_INT15_TYPE 	XTHAL_INTTYPE_TIMER
+#define XCHAL_INT16_TYPE 	XTHAL_INTTYPE_TIMER
+#define XCHAL_INT17_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT18_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT19_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT20_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT21_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT22_TYPE 	XTHAL_INTTYPE_EXTERN_EDGE
+#define XCHAL_INT23_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT24_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT25_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT26_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT27_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT28_TYPE 	XTHAL_INTTYPE_EXTERN_EDGE
+#define XCHAL_INT29_TYPE 	XTHAL_INTTYPE_SOFTWARE
+#define XCHAL_INT30_TYPE 	XTHAL_INTTYPE_EXTERN_EDGE
+#define XCHAL_INT31_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+
+/*  Masks of interrupts for each type of interrupt:  */
+#define XCHAL_INTTYPE_MASK_UNCONFIGURED	0x00000000
+#define XCHAL_INTTYPE_MASK_SOFTWARE	0x20000080
+#define XCHAL_INTTYPE_MASK_EXTERN_EDGE	0x50400400
+#define XCHAL_INTTYPE_MASK_EXTERN_LEVEL	0x8FBE333F
+#define XCHAL_INTTYPE_MASK_TIMER	0x00018040
+#define XCHAL_INTTYPE_MASK_NMI		0x00004000
+#define XCHAL_INTTYPE_MASK_WRITE_ERROR	0x00000000
+#define XCHAL_INTTYPE_MASK_PROFILING	0x00000800
+
+/*  Interrupt numbers assigned to specific interrupt sources:  */
+#define XCHAL_TIMER0_INTERRUPT		6	/* CCOMPARE0 */
+#define XCHAL_TIMER1_INTERRUPT		15	/* CCOMPARE1 */
+#define XCHAL_TIMER2_INTERRUPT		16	/* CCOMPARE2 */
+#define XCHAL_TIMER3_INTERRUPT		XTHAL_TIMER_UNCONFIGURED
+#define XCHAL_NMI_INTERRUPT		14	/* non-maskable interrupt */
+#define XCHAL_PROFILING_INTERRUPT	11	/* profiling interrupt */
+
+/*  Interrupt numbers for levels at which only one interrupt is configured:  */
+#define XCHAL_INTLEVEL7_NUM		14
+/*  (There are many interrupts each at level(s) 1, 2, 3, 4, 5.)  */
+
+
+/*
+ *  External interrupt mapping.
+ *  These macros describe how Xtensa processor interrupt numbers
+ *  (as numbered internally, eg. in INTERRUPT and INTENABLE registers)
+ *  map to external BInterrupt<n> pins, for those interrupts
+ *  configured as external (level-triggered, edge-triggered, or NMI).
+ *  See the Xtensa processor databook for more details.
+ */
+
+/*  Core interrupt numbers mapped to each EXTERNAL BInterrupt pin number:  */
+#define XCHAL_EXTINT0_NUM		0	/* (intlevel 1) */
+#define XCHAL_EXTINT1_NUM		1	/* (intlevel 1) */
+#define XCHAL_EXTINT2_NUM		2	/* (intlevel 1) */
+#define XCHAL_EXTINT3_NUM		3	/* (intlevel 1) */
+#define XCHAL_EXTINT4_NUM		4	/* (intlevel 1) */
+#define XCHAL_EXTINT5_NUM		5	/* (intlevel 1) */
+#define XCHAL_EXTINT6_NUM		8	/* (intlevel 1) */
+#define XCHAL_EXTINT7_NUM		9	/* (intlevel 1) */
+#define XCHAL_EXTINT8_NUM		10	/* (intlevel 1) */
+#define XCHAL_EXTINT9_NUM		12	/* (intlevel 1) */
+#define XCHAL_EXTINT10_NUM		13	/* (intlevel 1) */
+#define XCHAL_EXTINT11_NUM		14	/* (intlevel 7) */
+#define XCHAL_EXTINT12_NUM		17	/* (intlevel 1) */
+#define XCHAL_EXTINT13_NUM		18	/* (intlevel 1) */
+#define XCHAL_EXTINT14_NUM		19	/* (intlevel 2) */
+#define XCHAL_EXTINT15_NUM		20	/* (intlevel 2) */
+#define XCHAL_EXTINT16_NUM		21	/* (intlevel 2) */
+#define XCHAL_EXTINT17_NUM		22	/* (intlevel 3) */
+#define XCHAL_EXTINT18_NUM		23	/* (intlevel 3) */
+#define XCHAL_EXTINT19_NUM		24	/* (intlevel 4) */
+#define XCHAL_EXTINT20_NUM		25	/* (intlevel 4) */
+#define XCHAL_EXTINT21_NUM		26	/* (intlevel 5) */
+#define XCHAL_EXTINT22_NUM		27	/* (intlevel 3) */
+#define XCHAL_EXTINT23_NUM		28	/* (intlevel 4) */
+#define XCHAL_EXTINT24_NUM		30	/* (intlevel 4) */
+#define XCHAL_EXTINT25_NUM		31	/* (intlevel 5) */
+/*  EXTERNAL BInterrupt pin numbers mapped to each core interrupt number:  */
+#define XCHAL_INT0_EXTNUM		0	/* (intlevel 1) */
+#define XCHAL_INT1_EXTNUM		1	/* (intlevel 1) */
+#define XCHAL_INT2_EXTNUM		2	/* (intlevel 1) */
+#define XCHAL_INT3_EXTNUM		3	/* (intlevel 1) */
+#define XCHAL_INT4_EXTNUM		4	/* (intlevel 1) */
+#define XCHAL_INT5_EXTNUM		5	/* (intlevel 1) */
+#define XCHAL_INT8_EXTNUM		6	/* (intlevel 1) */
+#define XCHAL_INT9_EXTNUM		7	/* (intlevel 1) */
+#define XCHAL_INT10_EXTNUM		8	/* (intlevel 1) */
+#define XCHAL_INT12_EXTNUM		9	/* (intlevel 1) */
+#define XCHAL_INT13_EXTNUM		10	/* (intlevel 1) */
+#define XCHAL_INT14_EXTNUM		11	/* (intlevel 7) */
+#define XCHAL_INT17_EXTNUM		12	/* (intlevel 1) */
+#define XCHAL_INT18_EXTNUM		13	/* (intlevel 1) */
+#define XCHAL_INT19_EXTNUM		14	/* (intlevel 2) */
+#define XCHAL_INT20_EXTNUM		15	/* (intlevel 2) */
+#define XCHAL_INT21_EXTNUM		16	/* (intlevel 2) */
+#define XCHAL_INT22_EXTNUM		17	/* (intlevel 3) */
+#define XCHAL_INT23_EXTNUM		18	/* (intlevel 3) */
+#define XCHAL_INT24_EXTNUM		19	/* (intlevel 4) */
+#define XCHAL_INT25_EXTNUM		20	/* (intlevel 4) */
+#define XCHAL_INT26_EXTNUM		21	/* (intlevel 5) */
+#define XCHAL_INT27_EXTNUM		22	/* (intlevel 3) */
+#define XCHAL_INT28_EXTNUM		23	/* (intlevel 4) */
+#define XCHAL_INT30_EXTNUM		24	/* (intlevel 4) */
+#define XCHAL_INT31_EXTNUM		25	/* (intlevel 5) */
+
+
+/*----------------------------------------------------------------------
+			EXCEPTIONS and VECTORS
+  ----------------------------------------------------------------------*/
+
+#define XCHAL_XEA_VERSION		2	/* Xtensa Exception Architecture
+						   number: 1 == XEA1 (old)
+							   2 == XEA2 (new)
+							   0 == XEAX (extern) or TX */
+#define XCHAL_HAVE_XEA1			0	/* Exception Architecture 1 */
+#define XCHAL_HAVE_XEA2			1	/* Exception Architecture 2 */
+#define XCHAL_HAVE_XEAX			0	/* External Exception Arch. */
+#define XCHAL_HAVE_EXCEPTIONS		1	/* exception option */
+#define XCHAL_HAVE_HALT			0	/* halt architecture option */
+#define XCHAL_HAVE_BOOTLOADER		0	/* boot loader (for TX) */
+#define XCHAL_HAVE_MEM_ECC_PARITY	0	/* local memory ECC/parity */
+#define XCHAL_HAVE_VECTOR_SELECT	1	/* relocatable vectors */
+#define XCHAL_HAVE_VECBASE		1	/* relocatable vectors */
+#define XCHAL_VECBASE_RESET_VADDR	0x40000000  /* VECBASE reset value */
+#define XCHAL_VECBASE_RESET_PADDR	0x40000000
+#define XCHAL_RESET_VECBASE_OVERLAP	0
+
+#define XCHAL_RESET_VECTOR0_VADDR	0x50000000
+#define XCHAL_RESET_VECTOR0_PADDR	0x50000000
+#define XCHAL_RESET_VECTOR1_VADDR	0x40000400
+#define XCHAL_RESET_VECTOR1_PADDR	0x40000400
+#define XCHAL_RESET_VECTOR_VADDR	0x40000400
+#define XCHAL_RESET_VECTOR_PADDR	0x40000400
+#define XCHAL_USER_VECOFS		0x00000340
+#define XCHAL_USER_VECTOR_VADDR		0x40000340
+#define XCHAL_USER_VECTOR_PADDR		0x40000340
+#define XCHAL_KERNEL_VECOFS		0x00000300
+#define XCHAL_KERNEL_VECTOR_VADDR	0x40000300
+#define XCHAL_KERNEL_VECTOR_PADDR	0x40000300
+#define XCHAL_DOUBLEEXC_VECOFS		0x000003C0
+#define XCHAL_DOUBLEEXC_VECTOR_VADDR	0x400003C0
+#define XCHAL_DOUBLEEXC_VECTOR_PADDR	0x400003C0
+#define XCHAL_WINDOW_OF4_VECOFS		0x00000000
+#define XCHAL_WINDOW_UF4_VECOFS		0x00000040
+#define XCHAL_WINDOW_OF8_VECOFS		0x00000080
+#define XCHAL_WINDOW_UF8_VECOFS		0x000000C0
+#define XCHAL_WINDOW_OF12_VECOFS	0x00000100
+#define XCHAL_WINDOW_UF12_VECOFS	0x00000140
+#define XCHAL_WINDOW_VECTORS_VADDR	0x40000000
+#define XCHAL_WINDOW_VECTORS_PADDR	0x40000000
+#define XCHAL_INTLEVEL2_VECOFS		0x00000180
+#define XCHAL_INTLEVEL2_VECTOR_VADDR	0x40000180
+#define XCHAL_INTLEVEL2_VECTOR_PADDR	0x40000180
+#define XCHAL_INTLEVEL3_VECOFS		0x000001C0
+#define XCHAL_INTLEVEL3_VECTOR_VADDR	0x400001C0
+#define XCHAL_INTLEVEL3_VECTOR_PADDR	0x400001C0
+#define XCHAL_INTLEVEL4_VECOFS		0x00000200
+#define XCHAL_INTLEVEL4_VECTOR_VADDR	0x40000200
+#define XCHAL_INTLEVEL4_VECTOR_PADDR	0x40000200
+#define XCHAL_INTLEVEL5_VECOFS		0x00000240
+#define XCHAL_INTLEVEL5_VECTOR_VADDR	0x40000240
+#define XCHAL_INTLEVEL5_VECTOR_PADDR	0x40000240
+#define XCHAL_INTLEVEL6_VECOFS		0x00000280
+#define XCHAL_INTLEVEL6_VECTOR_VADDR	0x40000280
+#define XCHAL_INTLEVEL6_VECTOR_PADDR	0x40000280
+#define XCHAL_DEBUG_VECOFS		XCHAL_INTLEVEL6_VECOFS
+#define XCHAL_DEBUG_VECTOR_VADDR	XCHAL_INTLEVEL6_VECTOR_VADDR
+#define XCHAL_DEBUG_VECTOR_PADDR	XCHAL_INTLEVEL6_VECTOR_PADDR
+#define XCHAL_NMI_VECOFS		0x000002C0
+#define XCHAL_NMI_VECTOR_VADDR		0x400002C0
+#define XCHAL_NMI_VECTOR_PADDR		0x400002C0
+#define XCHAL_INTLEVEL7_VECOFS		XCHAL_NMI_VECOFS
+#define XCHAL_INTLEVEL7_VECTOR_VADDR	XCHAL_NMI_VECTOR_VADDR
+#define XCHAL_INTLEVEL7_VECTOR_PADDR	XCHAL_NMI_VECTOR_PADDR
+
+
+/*----------------------------------------------------------------------
+				DEBUG MODULE
+  ----------------------------------------------------------------------*/
+
+/*  Misc  */
+#define XCHAL_HAVE_DEBUG_ERI		1	/* ERI to debug module */
+#define XCHAL_HAVE_DEBUG_APB		1	/* APB to debug module */
+#define XCHAL_HAVE_DEBUG_JTAG		1	/* JTAG to debug module */
+
+/*  On-Chip Debug (OCD)  */
+#define XCHAL_HAVE_OCD			1	/* OnChipDebug option */
+#define XCHAL_NUM_IBREAK		2	/* number of IBREAKn regs */
+#define XCHAL_NUM_DBREAK		2	/* number of DBREAKn regs */
+#define XCHAL_HAVE_OCD_DIR_ARRAY	0	/* faster OCD option (to LX4) */
+#define XCHAL_HAVE_OCD_LS32DDR		1	/* L32DDR/S32DDR (faster OCD) */
+
+/*  TRAX (in core)  */
+#define XCHAL_HAVE_TRAX			1	/* TRAX in debug module */
+#define XCHAL_TRAX_MEM_SIZE		16384	/* TRAX memory size in bytes */
+#define XCHAL_TRAX_MEM_SHAREABLE	1	/* start/end regs; ready sig. */
+#define XCHAL_TRAX_ATB_WIDTH		32	/* ATB width (bits), 0=no ATB */
+#define XCHAL_TRAX_TIME_WIDTH		0	/* timestamp bitwidth, 0=none */
+
+/*  Perf counters  */
+#define XCHAL_NUM_PERF_COUNTERS		2	/* performance counters */
+
+
+/*----------------------------------------------------------------------
+				MMU
+  ----------------------------------------------------------------------*/
+
+/*  See core-matmap.h header file for more details.  */
+
+#define XCHAL_HAVE_TLBS			1	/* inverse of HAVE_CACHEATTR */
+#define XCHAL_HAVE_SPANNING_WAY		1	/* one way maps I+D 4GB vaddr */
+#define XCHAL_SPANNING_WAY		0	/* TLB spanning way number */
+#define XCHAL_HAVE_IDENTITY_MAP		1	/* vaddr == paddr always */
+#define XCHAL_HAVE_CACHEATTR		0	/* CACHEATTR register present */
+#define XCHAL_HAVE_MIMIC_CACHEATTR	1	/* region protection */
+#define XCHAL_HAVE_XLT_CACHEATTR	0	/* region prot. w/translation */
+#define XCHAL_HAVE_PTP_MMU		0	/* full MMU (with page table
+						   [autorefill] and protection)
+						   usable for an MMU-based OS */
+/*  If none of the above last 4 are set, it's a custom TLB configuration.  */
+
+#define XCHAL_MMU_ASID_BITS		0	/* number of bits in ASIDs */
+#define XCHAL_MMU_RINGS			1	/* number of rings (1..4) */
+#define XCHAL_MMU_RING_BITS		0	/* num of bits in RING field */
+
+#endif /* !XTENSA_HAL_NON_PRIVILEGED_ONLY */
+
+
+#endif /* _XTENSA_CORE_CONFIGURATION_H */
+
diff --git a/arch/xtensa/variants/esp32/include/variant/tie-asm.h b/arch/xtensa/variants/esp32/include/variant/tie-asm.h
new file mode 100644
index 000000000000..831d8676ae2c
--- /dev/null
+++ b/arch/xtensa/variants/esp32/include/variant/tie-asm.h
@@ -0,0 +1,323 @@
+/* 
+ * tie-asm.h -- compile-time HAL assembler definitions dependent on CORE & TIE
+ *
+ *  NOTE:  This header file is not meant to be included directly.
+ */
+
+/* This header file contains assembly-language definitions (assembly
+   macros, etc.) for this specific Xtensa processor's TIE extensions
+   and options.  It is customized to this Xtensa processor configuration.
+
+   Customer ID=11657; Build=0x5fe96; Copyright (c) 1999-2016 Cadence Design Systems Inc.
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be included
+   in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+   IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+   CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+   TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+   SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.  */
+
+#ifndef _XTENSA_CORE_TIE_ASM_H
+#define _XTENSA_CORE_TIE_ASM_H
+
+/*  Selection parameter values for save-area save/restore macros:  */
+/*  Option vs. TIE:  */
+#define XTHAL_SAS_TIE	0x0001	/* custom extension or coprocessor */
+#define XTHAL_SAS_OPT	0x0002	/* optional (and not a coprocessor) */
+#define XTHAL_SAS_ANYOT	0x0003	/* both of the above */
+/*  Whether used automatically by compiler:  */
+#define XTHAL_SAS_NOCC	0x0004	/* not used by compiler w/o special opts/code */
+#define XTHAL_SAS_CC	0x0008	/* used by compiler without special opts/code */
+#define XTHAL_SAS_ANYCC	0x000C	/* both of the above */
+/*  ABI handling across function calls:  */
+#define XTHAL_SAS_CALR	0x0010	/* caller-saved */
+#define XTHAL_SAS_CALE	0x0020	/* callee-saved */
+#define XTHAL_SAS_GLOB	0x0040	/* global across function calls (in thread) */
+#define XTHAL_SAS_ANYABI	0x0070	/* all of the above three */
+/*  Misc  */
+#define XTHAL_SAS_ALL	0xFFFF	/* include all default NCP contents */
+#define XTHAL_SAS3(optie,ccuse,abi)	( ((optie) & XTHAL_SAS_ANYOT)  \
+					| ((ccuse) & XTHAL_SAS_ANYCC)  \
+					| ((abi)   & XTHAL_SAS_ANYABI) )
+
+
+    /*
+      *  Macro to store all non-coprocessor (extra) custom TIE and optional state
+      *  (not including zero-overhead loop registers).
+      *  Required parameters:
+      *      ptr         Save area pointer address register (clobbered)
+      *                  (register must contain a 4 byte aligned address).
+      *      at1..at4    Four temporary address registers (first XCHAL_NCP_NUM_ATMPS
+      *                  registers are clobbered, the remaining are unused).
+      *  Optional parameters:
+      *      continue    If macro invoked as part of a larger store sequence, set to 1
+      *                  if this is not the first in the sequence.  Defaults to 0.
+      *      ofs         Offset from start of larger sequence (from value of first ptr
+      *                  in sequence) at which to store.  Defaults to next available space
+      *                  (or 0 if <continue> is 0).
+      *      select      Select what category(ies) of registers to store, as a bitmask
+      *                  (see XTHAL_SAS_xxx constants).  Defaults to all registers.
+      *      alloc       Select what category(ies) of registers to allocate; if any
+      *                  category is selected here that is not in <select>, space for
+      *                  the corresponding registers is skipped without doing any store.
+      */
+    .macro xchal_ncp_store  ptr at1 at2 at3 at4  continue=0 ofs=-1 select=XTHAL_SAS_ALL alloc=0
+	xchal_sa_start	\continue, \ofs
+	// Optional global registers used by default by the compiler:
+	.ifeq (XTHAL_SAS_OPT | XTHAL_SAS_CC | XTHAL_SAS_GLOB) & ~(\select)
+	xchal_sa_align	\ptr, 0, 1016, 4, 4
+	rur.THREADPTR	\at1		// threadptr option
+	s32i	\at1, \ptr, .Lxchal_ofs_+0
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 4
+	.elseif ((XTHAL_SAS_OPT | XTHAL_SAS_CC | XTHAL_SAS_GLOB) & ~(\alloc)) == 0
+	xchal_sa_align	\ptr, 0, 1016, 4, 4
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 4
+	.endif
+	// Optional caller-saved registers used by default by the compiler:
+	.ifeq (XTHAL_SAS_OPT | XTHAL_SAS_CC | XTHAL_SAS_CALR) & ~(\select)
+	xchal_sa_align	\ptr, 0, 1012, 4, 4
+	rsr.ACCLO	\at1		// MAC16 option
+	s32i	\at1, \ptr, .Lxchal_ofs_+0
+	rsr.ACCHI	\at1		// MAC16 option
+	s32i	\at1, \ptr, .Lxchal_ofs_+4
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 8
+	.elseif ((XTHAL_SAS_OPT | XTHAL_SAS_CC | XTHAL_SAS_CALR) & ~(\alloc)) == 0
+	xchal_sa_align	\ptr, 0, 1012, 4, 4
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 8
+	.endif
+	// Optional caller-saved registers not used by default by the compiler:
+	.ifeq (XTHAL_SAS_OPT | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\select)
+	xchal_sa_align	\ptr, 0, 996, 4, 4
+	rsr.BR	\at1		// boolean option
+	s32i	\at1, \ptr, .Lxchal_ofs_+0
+	rsr.SCOMPARE1	\at1		// conditional store option
+	s32i	\at1, \ptr, .Lxchal_ofs_+4
+	rsr.M0	\at1		// MAC16 option
+	s32i	\at1, \ptr, .Lxchal_ofs_+8
+	rsr.M1	\at1		// MAC16 option
+	s32i	\at1, \ptr, .Lxchal_ofs_+12
+	rsr.M2	\at1		// MAC16 option
+	s32i	\at1, \ptr, .Lxchal_ofs_+16
+	rsr.M3	\at1		// MAC16 option
+	s32i	\at1, \ptr, .Lxchal_ofs_+20
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 24
+	.elseif ((XTHAL_SAS_OPT | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\alloc)) == 0
+	xchal_sa_align	\ptr, 0, 996, 4, 4
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 24
+	.endif
+	// Custom caller-saved registers not used by default by the compiler:
+	.ifeq (XTHAL_SAS_TIE | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\select)
+	xchal_sa_align	\ptr, 0, 1008, 4, 4
+	rur.F64R_LO	\at1		// ureg 234
+	s32i	\at1, \ptr, .Lxchal_ofs_+0
+	rur.F64R_HI	\at1		// ureg 235
+	s32i	\at1, \ptr, .Lxchal_ofs_+4
+	rur.F64S	\at1		// ureg 236
+	s32i	\at1, \ptr, .Lxchal_ofs_+8
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 12
+	.elseif ((XTHAL_SAS_TIE | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\alloc)) == 0
+	xchal_sa_align	\ptr, 0, 1008, 4, 4
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 12
+	.endif
+    .endm	// xchal_ncp_store
+
+    /*
+      *  Macro to load all non-coprocessor (extra) custom TIE and optional state
+      *  (not including zero-overhead loop registers).
+      *  Required parameters:
+      *      ptr         Save area pointer address register (clobbered)
+      *                  (register must contain a 4 byte aligned address).
+      *      at1..at4    Four temporary address registers (first XCHAL_NCP_NUM_ATMPS
+      *                  registers are clobbered, the remaining are unused).
+      *  Optional parameters:
+      *      continue    If macro invoked as part of a larger load sequence, set to 1
+      *                  if this is not the first in the sequence.  Defaults to 0.
+      *      ofs         Offset from start of larger sequence (from value of first ptr
+      *                  in sequence) at which to load.  Defaults to next available space
+      *                  (or 0 if <continue> is 0).
+      *      select      Select what category(ies) of registers to load, as a bitmask
+      *                  (see XTHAL_SAS_xxx constants).  Defaults to all registers.
+      *      alloc       Select what category(ies) of registers to allocate; if any
+      *                  category is selected here that is not in <select>, space for
+      *                  the corresponding registers is skipped without doing any load.
+      */
+    .macro xchal_ncp_load  ptr at1 at2 at3 at4  continue=0 ofs=-1 select=XTHAL_SAS_ALL alloc=0
+	xchal_sa_start	\continue, \ofs
+	// Optional global registers used by default by the compiler:
+	.ifeq (XTHAL_SAS_OPT | XTHAL_SAS_CC | XTHAL_SAS_GLOB) & ~(\select)
+	xchal_sa_align	\ptr, 0, 1016, 4, 4
+	l32i	\at1, \ptr, .Lxchal_ofs_+0
+	wur.THREADPTR	\at1		// threadptr option
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 4
+	.elseif ((XTHAL_SAS_OPT | XTHAL_SAS_CC | XTHAL_SAS_GLOB) & ~(\alloc)) == 0
+	xchal_sa_align	\ptr, 0, 1016, 4, 4
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 4
+	.endif
+	// Optional caller-saved registers used by default by the compiler:
+	.ifeq (XTHAL_SAS_OPT | XTHAL_SAS_CC | XTHAL_SAS_CALR) & ~(\select)
+	xchal_sa_align	\ptr, 0, 1012, 4, 4
+	l32i	\at1, \ptr, .Lxchal_ofs_+0
+	wsr.ACCLO	\at1		// MAC16 option
+	l32i	\at1, \ptr, .Lxchal_ofs_+4
+	wsr.ACCHI	\at1		// MAC16 option
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 8
+	.elseif ((XTHAL_SAS_OPT | XTHAL_SAS_CC | XTHAL_SAS_CALR) & ~(\alloc)) == 0
+	xchal_sa_align	\ptr, 0, 1012, 4, 4
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 8
+	.endif
+	// Optional caller-saved registers not used by default by the compiler:
+	.ifeq (XTHAL_SAS_OPT | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\select)
+	xchal_sa_align	\ptr, 0, 996, 4, 4
+	l32i	\at1, \ptr, .Lxchal_ofs_+0
+	wsr.BR	\at1		// boolean option
+	l32i	\at1, \ptr, .Lxchal_ofs_+4
+	wsr.SCOMPARE1	\at1		// conditional store option
+	l32i	\at1, \ptr, .Lxchal_ofs_+8
+	wsr.M0	\at1		// MAC16 option
+	l32i	\at1, \ptr, .Lxchal_ofs_+12
+	wsr.M1	\at1		// MAC16 option
+	l32i	\at1, \ptr, .Lxchal_ofs_+16
+	wsr.M2	\at1		// MAC16 option
+	l32i	\at1, \ptr, .Lxchal_ofs_+20
+	wsr.M3	\at1		// MAC16 option
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 24
+	.elseif ((XTHAL_SAS_OPT | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\alloc)) == 0
+	xchal_sa_align	\ptr, 0, 996, 4, 4
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 24
+	.endif
+	// Custom caller-saved registers not used by default by the compiler:
+	.ifeq (XTHAL_SAS_TIE | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\select)
+	xchal_sa_align	\ptr, 0, 1008, 4, 4
+	l32i	\at1, \ptr, .Lxchal_ofs_+0
+	wur.F64R_LO	\at1		// ureg 234
+	l32i	\at1, \ptr, .Lxchal_ofs_+4
+	wur.F64R_HI	\at1		// ureg 235
+	l32i	\at1, \ptr, .Lxchal_ofs_+8
+	wur.F64S	\at1		// ureg 236
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 12
+	.elseif ((XTHAL_SAS_TIE | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\alloc)) == 0
+	xchal_sa_align	\ptr, 0, 1008, 4, 4
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 12
+	.endif
+    .endm	// xchal_ncp_load
+
+
+#define XCHAL_NCP_NUM_ATMPS	1
+
+    /* 
+     *  Macro to store the state of TIE coprocessor FPU.
+     *  Required parameters:
+     *      ptr         Save area pointer address register (clobbered)
+     *                  (register must contain a 4 byte aligned address).
+     *      at1..at4    Four temporary address registers (first XCHAL_CP0_NUM_ATMPS
+     *                  registers are clobbered, the remaining are unused).
+     *  Optional parameters are the same as for xchal_ncp_store.
+     */
+#define xchal_cp_FPU_store	xchal_cp0_store
+    .macro	xchal_cp0_store  ptr at1 at2 at3 at4  continue=0 ofs=-1 select=XTHAL_SAS_ALL alloc=0
+	xchal_sa_start \continue, \ofs
+	// Custom caller-saved registers not used by default by the compiler:
+	.ifeq (XTHAL_SAS_TIE | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\select)
+	xchal_sa_align	\ptr, 0, 948, 4, 4
+	rur.FCR	\at1		// ureg 232
+	s32i	\at1, \ptr, .Lxchal_ofs_+0
+	rur.FSR	\at1		// ureg 233
+	s32i	\at1, \ptr, .Lxchal_ofs_+4
+	ssi	f0, \ptr, .Lxchal_ofs_+8
+	ssi	f1, \ptr, .Lxchal_ofs_+12
+	ssi	f2, \ptr, .Lxchal_ofs_+16
+	ssi	f3, \ptr, .Lxchal_ofs_+20
+	ssi	f4, \ptr, .Lxchal_ofs_+24
+	ssi	f5, \ptr, .Lxchal_ofs_+28
+	ssi	f6, \ptr, .Lxchal_ofs_+32
+	ssi	f7, \ptr, .Lxchal_ofs_+36
+	ssi	f8, \ptr, .Lxchal_ofs_+40
+	ssi	f9, \ptr, .Lxchal_ofs_+44
+	ssi	f10, \ptr, .Lxchal_ofs_+48
+	ssi	f11, \ptr, .Lxchal_ofs_+52
+	ssi	f12, \ptr, .Lxchal_ofs_+56
+	ssi	f13, \ptr, .Lxchal_ofs_+60
+	ssi	f14, \ptr, .Lxchal_ofs_+64
+	ssi	f15, \ptr, .Lxchal_ofs_+68
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 72
+	.elseif ((XTHAL_SAS_TIE | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\alloc)) == 0
+	xchal_sa_align	\ptr, 0, 948, 4, 4
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 72
+	.endif
+    .endm	// xchal_cp0_store
+
+    /* 
+     *  Macro to load the state of TIE coprocessor FPU.
+     *  Required parameters:
+     *      ptr         Save area pointer address register (clobbered)
+     *                  (register must contain a 4 byte aligned address).
+     *      at1..at4    Four temporary address registers (first XCHAL_CP0_NUM_ATMPS
+     *                  registers are clobbered, the remaining are unused).
+     *  Optional parameters are the same as for xchal_ncp_load.
+     */
+#define xchal_cp_FPU_load	xchal_cp0_load
+    .macro	xchal_cp0_load  ptr at1 at2 at3 at4  continue=0 ofs=-1 select=XTHAL_SAS_ALL alloc=0
+	xchal_sa_start \continue, \ofs
+	// Custom caller-saved registers not used by default by the compiler:
+	.ifeq (XTHAL_SAS_TIE | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\select)
+	xchal_sa_align	\ptr, 0, 948, 4, 4
+	l32i	\at1, \ptr, .Lxchal_ofs_+0
+	wur.FCR	\at1		// ureg 232
+	l32i	\at1, \ptr, .Lxchal_ofs_+4
+	wur.FSR	\at1		// ureg 233
+	lsi	f0, \ptr, .Lxchal_ofs_+8
+	lsi	f1, \ptr, .Lxchal_ofs_+12
+	lsi	f2, \ptr, .Lxchal_ofs_+16
+	lsi	f3, \ptr, .Lxchal_ofs_+20
+	lsi	f4, \ptr, .Lxchal_ofs_+24
+	lsi	f5, \ptr, .Lxchal_ofs_+28
+	lsi	f6, \ptr, .Lxchal_ofs_+32
+	lsi	f7, \ptr, .Lxchal_ofs_+36
+	lsi	f8, \ptr, .Lxchal_ofs_+40
+	lsi	f9, \ptr, .Lxchal_ofs_+44
+	lsi	f10, \ptr, .Lxchal_ofs_+48
+	lsi	f11, \ptr, .Lxchal_ofs_+52
+	lsi	f12, \ptr, .Lxchal_ofs_+56
+	lsi	f13, \ptr, .Lxchal_ofs_+60
+	lsi	f14, \ptr, .Lxchal_ofs_+64
+	lsi	f15, \ptr, .Lxchal_ofs_+68
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 72
+	.elseif ((XTHAL_SAS_TIE | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\alloc)) == 0
+	xchal_sa_align	\ptr, 0, 948, 4, 4
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 72
+	.endif
+    .endm	// xchal_cp0_load
+
+#define XCHAL_CP0_NUM_ATMPS	1
+#define XCHAL_SA_NUM_ATMPS	1
+
+	/*  Empty macros for unconfigured coprocessors:  */
+	.macro xchal_cp1_store	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp1_load	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp2_store	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp2_load	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp3_store	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp3_load	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp4_store	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp4_load	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp5_store	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp5_load	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp6_store	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp6_load	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp7_store	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp7_load	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+
+#endif /*_XTENSA_CORE_TIE_ASM_H*/
+
diff --git a/arch/xtensa/variants/esp32/include/variant/tie.h b/arch/xtensa/variants/esp32/include/variant/tie.h
new file mode 100644
index 000000000000..e178799708e2
--- /dev/null
+++ b/arch/xtensa/variants/esp32/include/variant/tie.h
@@ -0,0 +1,182 @@
+/* 
+ * tie.h -- compile-time HAL definitions dependent on CORE & TIE configuration
+ *
+ *  NOTE:  This header file is not meant to be included directly.
+ */
+
+/* This header file describes this specific Xtensa processor's TIE extensions
+   that extend basic Xtensa core functionality.  It is customized to this
+   Xtensa processor configuration.
+
+   Customer ID=11657; Build=0x5fe96; Copyright (c) 1999-2016 Cadence Design Systems Inc.
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be included
+   in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+   IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+   CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+   TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+   SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.  */
+
+#ifndef _XTENSA_CORE_TIE_H
+#define _XTENSA_CORE_TIE_H
+
+#define XCHAL_CP_NUM			1	/* number of coprocessors */
+#define XCHAL_CP_MAX			1	/* max CP ID + 1 (0 if none) */
+#define XCHAL_CP_MASK			0x01	/* bitmask of all CPs by ID */
+#define XCHAL_CP_PORT_MASK		0x00	/* bitmask of only port CPs */
+
+/*  Basic parameters of each coprocessor:  */
+#define XCHAL_CP0_NAME			"FPU"
+#define XCHAL_CP0_IDENT			FPU
+#define XCHAL_CP0_SA_SIZE		72	/* size of state save area */
+#define XCHAL_CP0_SA_ALIGN		4	/* min alignment of save area */
+#define XCHAL_CP_ID_FPU             	0	/* coprocessor ID (0..7) */
+
+/*  Filler info for unassigned coprocessors, to simplify arrays etc:  */
+#define XCHAL_CP1_SA_SIZE		0
+#define XCHAL_CP1_SA_ALIGN		1
+#define XCHAL_CP2_SA_SIZE		0
+#define XCHAL_CP2_SA_ALIGN		1
+#define XCHAL_CP3_SA_SIZE		0
+#define XCHAL_CP3_SA_ALIGN		1
+#define XCHAL_CP4_SA_SIZE		0
+#define XCHAL_CP4_SA_ALIGN		1
+#define XCHAL_CP5_SA_SIZE		0
+#define XCHAL_CP5_SA_ALIGN		1
+#define XCHAL_CP6_SA_SIZE		0
+#define XCHAL_CP6_SA_ALIGN		1
+#define XCHAL_CP7_SA_SIZE		0
+#define XCHAL_CP7_SA_ALIGN		1
+
+/*  Save area for non-coprocessor optional and custom (TIE) state:  */
+#define XCHAL_NCP_SA_SIZE		48
+#define XCHAL_NCP_SA_ALIGN		4
+
+/*  Total save area for optional and custom state (NCP + CPn):  */
+#define XCHAL_TOTAL_SA_SIZE		128	/* with 16-byte align padding */
+#define XCHAL_TOTAL_SA_ALIGN		4	/* actual minimum alignment */
+
+/*
+ * Detailed contents of save areas.
+ * NOTE:  caller must define the XCHAL_SA_REG macro (not defined here)
+ * before expanding the XCHAL_xxx_SA_LIST() macros.
+ *
+ * XCHAL_SA_REG(s,ccused,abikind,kind,opt,name,galign,align,asize,
+ *		dbnum,base,regnum,bitsz,gapsz,reset,x...)
+ *
+ *	s = passed from XCHAL_*_LIST(s), eg. to select how to expand
+ *	ccused = set if used by compiler without special options or code
+ *	abikind = 0 (caller-saved), 1 (callee-saved), or 2 (thread-global)
+ *	kind = 0 (special reg), 1 (TIE user reg), or 2 (TIE regfile reg)
+ *	opt = 0 (custom TIE extension or coprocessor), or 1 (optional reg)
+ *	name = lowercase reg name (no quotes)
+ *	galign = group byte alignment (power of 2) (galign >= align)
+ *	align = register byte alignment (power of 2)
+ *	asize = allocated size in bytes (asize*8 == bitsz + gapsz + padsz)
+ *	  (not including any pad bytes required to galign this or next reg)
+ *	dbnum = unique target number f/debug (see <xtensa-libdb-macros.h>)
+ *	base = reg shortname w/o index (or sr=special, ur=TIE user reg)
+ *	regnum = reg index in regfile, or special/TIE-user reg number
+ *	bitsz = number of significant bits (regfile width, or ur/sr mask bits)
+ *	gapsz = intervening bits, if bitsz bits not stored contiguously
+ *	(padsz = pad bits at end [TIE regfile] or at msbits [ur,sr] of asize)
+ *	reset = register reset value (or 0 if undefined at reset)
+ *	x = reserved for future use (0 until then)
+ *
+ *  To filter out certain registers, e.g. to expand only the non-global
+ *  registers used by the compiler, you can do something like this:
+ *
+ *  #define XCHAL_SA_REG(s,ccused,p...)	SELCC##ccused(p)
+ *  #define SELCC0(p...)
+ *  #define SELCC1(abikind,p...)	SELAK##abikind(p)
+ *  #define SELAK0(p...)		REG(p)
+ *  #define SELAK1(p...)		REG(p)
+ *  #define SELAK2(p...)
+ *  #define REG(kind,tie,name,galn,aln,asz,csz,dbnum,base,rnum,bsz,rst,x...) \
+ *		...what you want to expand...
+ */
+
+#define XCHAL_NCP_SA_NUM	12
+#define XCHAL_NCP_SA_LIST(s)	\
+ XCHAL_SA_REG(s,1,2,1,1,      threadptr, 4, 4, 4,0x03E7,  ur,231, 32,0,0,0) \
+ XCHAL_SA_REG(s,1,0,0,1,          acclo, 4, 4, 4,0x0210,  sr,16 , 32,0,0,0) \
+ XCHAL_SA_REG(s,1,0,0,1,          acchi, 4, 4, 4,0x0211,  sr,17 ,  8,0,0,0) \
+ XCHAL_SA_REG(s,0,0,0,1,             br, 4, 4, 4,0x0204,  sr,4  , 16,0,0,0) \
+ XCHAL_SA_REG(s,0,0,0,1,      scompare1, 4, 4, 4,0x020C,  sr,12 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,0,1,             m0, 4, 4, 4,0x0220,  sr,32 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,0,1,             m1, 4, 4, 4,0x0221,  sr,33 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,0,1,             m2, 4, 4, 4,0x0222,  sr,34 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,0,1,             m3, 4, 4, 4,0x0223,  sr,35 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,        f64r_lo, 4, 4, 4,0x03EA,  ur,234, 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,        f64r_hi, 4, 4, 4,0x03EB,  ur,235, 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,           f64s, 4, 4, 4,0x03EC,  ur,236, 32,0,0,0)
+
+#define XCHAL_CP0_SA_NUM	18
+#define XCHAL_CP0_SA_LIST(s)	\
+ XCHAL_SA_REG(s,0,0,1,0,            fcr, 4, 4, 4,0x03E8,  ur,232, 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,            fsr, 4, 4, 4,0x03E9,  ur,233, 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             f0, 4, 4, 4,0x0030,   f,0  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             f1, 4, 4, 4,0x0031,   f,1  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             f2, 4, 4, 4,0x0032,   f,2  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             f3, 4, 4, 4,0x0033,   f,3  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             f4, 4, 4, 4,0x0034,   f,4  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             f5, 4, 4, 4,0x0035,   f,5  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             f6, 4, 4, 4,0x0036,   f,6  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             f7, 4, 4, 4,0x0037,   f,7  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             f8, 4, 4, 4,0x0038,   f,8  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             f9, 4, 4, 4,0x0039,   f,9  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,            f10, 4, 4, 4,0x003A,   f,10 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,            f11, 4, 4, 4,0x003B,   f,11 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,            f12, 4, 4, 4,0x003C,   f,12 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,            f13, 4, 4, 4,0x003D,   f,13 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,            f14, 4, 4, 4,0x003E,   f,14 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,            f15, 4, 4, 4,0x003F,   f,15 , 32,0,0,0)
+
+#define XCHAL_CP1_SA_NUM	0
+#define XCHAL_CP1_SA_LIST(s)	/* empty */
+
+#define XCHAL_CP2_SA_NUM	0
+#define XCHAL_CP2_SA_LIST(s)	/* empty */
+
+#define XCHAL_CP3_SA_NUM	0
+#define XCHAL_CP3_SA_LIST(s)	/* empty */
+
+#define XCHAL_CP4_SA_NUM	0
+#define XCHAL_CP4_SA_LIST(s)	/* empty */
+
+#define XCHAL_CP5_SA_NUM	0
+#define XCHAL_CP5_SA_LIST(s)	/* empty */
+
+#define XCHAL_CP6_SA_NUM	0
+#define XCHAL_CP6_SA_LIST(s)	/* empty */
+
+#define XCHAL_CP7_SA_NUM	0
+#define XCHAL_CP7_SA_LIST(s)	/* empty */
+
+/* Byte length of instruction from its first nibble (op0 field), per FLIX.  */
+#define XCHAL_OP0_FORMAT_LENGTHS	3,3,3,3,3,3,3,3,2,2,2,2,2,2,3,3
+/* Byte length of instruction from its first byte, per FLIX.  */
+#define XCHAL_BYTE0_FORMAT_LENGTHS	\
+	3,3,3,3,3,3,3,3,2,2,2,2,2,2,3,3, 3,3,3,3,3,3,3,3,2,2,2,2,2,2,3,3,\
+	3,3,3,3,3,3,3,3,2,2,2,2,2,2,3,3, 3,3,3,3,3,3,3,3,2,2,2,2,2,2,3,3,\
+	3,3,3,3,3,3,3,3,2,2,2,2,2,2,3,3, 3,3,3,3,3,3,3,3,2,2,2,2,2,2,3,3,\
+	3,3,3,3,3,3,3,3,2,2,2,2,2,2,3,3, 3,3,3,3,3,3,3,3,2,2,2,2,2,2,3,3,\
+	3,3,3,3,3,3,3,3,2,2,2,2,2,2,3,3, 3,3,3,3,3,3,3,3,2,2,2,2,2,2,3,3,\
+	3,3,3,3,3,3,3,3,2,2,2,2,2,2,3,3, 3,3,3,3,3,3,3,3,2,2,2,2,2,2,3,3,\
+	3,3,3,3,3,3,3,3,2,2,2,2,2,2,3,3, 3,3,3,3,3,3,3,3,2,2,2,2,2,2,3,3,\
+	3,3,3,3,3,3,3,3,2,2,2,2,2,2,3,3, 3,3,3,3,3,3,3,3,2,2,2,2,2,2,3,3
+
+#endif /*_XTENSA_CORE_TIE_H*/
+
diff --git a/arch/xtensa/variants/esp32s3/include/variant/core.h b/arch/xtensa/variants/esp32s3/include/variant/core.h
new file mode 100644
index 000000000000..6c8fc963c0aa
--- /dev/null
+++ b/arch/xtensa/variants/esp32s3/include/variant/core.h
@@ -0,0 +1,681 @@
+/*
+ * xtensa/config/core-isa.h -- HAL definitions that are dependent on Xtensa
+ *				processor CORE configuration
+ *
+ *  See <xtensa/config/core.h>, which includes this file, for more details.
+ */
+
+/* Xtensa processor core configuration information.
+
+   Customer ID=15128; Build=0x90f1f; Copyright (c) 1999-2021 Tensilica Inc.
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be included
+   in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+   IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+   CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+   TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+   SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.  */
+
+#ifndef _XTENSA_CORE_CONFIGURATION_H
+#define _XTENSA_CORE_CONFIGURATION_H
+
+
+/****************************************************************************
+	    Parameters Useful for Any Code, USER or PRIVILEGED
+ ****************************************************************************/
+
+/*
+ *  Note:  Macros of the form XCHAL_HAVE_*** have a value of 1 if the option is
+ *  configured, and a value of 0 otherwise.  These macros are always defined.
+ */
+
+
+/*----------------------------------------------------------------------
+				ISA
+  ----------------------------------------------------------------------*/
+
+#define XCHAL_HAVE_BE			0	/* big-endian byte ordering */
+#define XCHAL_HAVE_WINDOWED		1	/* windowed registers option */
+#define XCHAL_NUM_AREGS			64	/* num of physical addr regs */
+#define XCHAL_NUM_AREGS_LOG2		6	/* log2(XCHAL_NUM_AREGS) */
+#define XCHAL_MAX_INSTRUCTION_SIZE	4	/* max instr bytes (3..8) */
+#define XCHAL_HAVE_DEBUG		1	/* debug option */
+#define XCHAL_HAVE_DENSITY		1	/* 16-bit instructions */
+#define XCHAL_HAVE_LOOPS		1	/* zero-overhead loops */
+#define XCHAL_LOOP_BUFFER_SIZE		256	/* zero-ov. loop instr buffer size */
+#define XCHAL_HAVE_NSA			1	/* NSA/NSAU instructions */
+#define XCHAL_HAVE_MINMAX		1	/* MIN/MAX instructions */
+#define XCHAL_HAVE_SEXT			1	/* SEXT instruction */
+#define XCHAL_HAVE_DEPBITS		0	/* DEPBITS instruction */
+#define XCHAL_HAVE_CLAMPS		1	/* CLAMPS instruction */
+#define XCHAL_HAVE_MUL16		1	/* MUL16S/MUL16U instructions */
+#define XCHAL_HAVE_MUL32		1	/* MULL instruction */
+#define XCHAL_HAVE_MUL32_HIGH		1	/* MULUH/MULSH instructions */
+#define XCHAL_HAVE_DIV32		1	/* QUOS/QUOU/REMS/REMU instructions */
+#define XCHAL_HAVE_L32R			1	/* L32R instruction */
+#define XCHAL_HAVE_ABSOLUTE_LITERALS	0	/* non-PC-rel (extended) L32R */
+#define XCHAL_HAVE_CONST16		0	/* CONST16 instruction */
+#define XCHAL_HAVE_ADDX			1	/* ADDX#/SUBX# instructions */
+#define XCHAL_HAVE_EXCLUSIVE            0	/* L32EX/S32EX instructions */
+#define XCHAL_HAVE_WIDE_BRANCHES	0	/* B*.W18 or B*.W15 instr's */
+#define XCHAL_HAVE_PREDICTED_BRANCHES	0	/* B[EQ/EQZ/NE/NEZ]T instr's */
+#define XCHAL_HAVE_CALL4AND12		1	/* (obsolete option) */
+#define XCHAL_HAVE_ABS			1	/* ABS instruction */
+/*#define XCHAL_HAVE_POPC		0*/	/* POPC instruction */
+/*#define XCHAL_HAVE_CRC		0*/	/* CRC instruction */
+#define XCHAL_HAVE_RELEASE_SYNC		1	/* L32AI/S32RI instructions */
+#define XCHAL_HAVE_S32C1I		1	/* S32C1I instruction */
+#define XCHAL_HAVE_SPECULATION		0	/* speculation */
+#define XCHAL_HAVE_FULL_RESET		1	/* all regs/state reset */
+#define XCHAL_NUM_CONTEXTS		1	/* */
+#define XCHAL_NUM_MISC_REGS		4	/* num of scratch regs (0..4) */
+#define XCHAL_HAVE_TAP_MASTER		0	/* JTAG TAP control instr's */
+#define XCHAL_HAVE_PRID			1	/* processor ID register */
+#define XCHAL_HAVE_EXTERN_REGS		1	/* WER/RER instructions */
+#define XCHAL_HAVE_MX			0	/* MX core (Tensilica internal) */
+#define XCHAL_HAVE_MP_INTERRUPTS	0	/* interrupt distributor port */
+#define XCHAL_HAVE_MP_RUNSTALL		0	/* core RunStall control port */
+#define XCHAL_HAVE_PSO			0	/* Power Shut-Off */
+#define XCHAL_HAVE_PSO_CDM		0	/* core/debug/mem pwr domains */
+#define XCHAL_HAVE_PSO_FULL_RETENTION	0	/* all regs preserved on PSO */
+#define XCHAL_HAVE_THREADPTR		1	/* THREADPTR register */
+#define XCHAL_HAVE_BOOLEANS		1	/* boolean registers */
+#define XCHAL_HAVE_CP			1	/* CPENABLE reg (coprocessor) */
+#define XCHAL_CP_MAXCFG			8	/* max allowed cp id plus one */
+#define XCHAL_HAVE_MAC16		1	/* MAC16 package */
+
+#define XCHAL_HAVE_FUSION		0	/* Fusion*/
+#define XCHAL_HAVE_FUSION_FP		0	/* Fusion FP option */
+#define XCHAL_HAVE_FUSION_LOW_POWER	0	/* Fusion Low Power option */
+#define XCHAL_HAVE_FUSION_AES		0	/* Fusion BLE/Wifi AES-128 CCM option */
+#define XCHAL_HAVE_FUSION_CONVENC	0	/* Fusion Conv Encode option */
+#define XCHAL_HAVE_FUSION_LFSR_CRC	0	/* Fusion LFSR-CRC option */
+#define XCHAL_HAVE_FUSION_BITOPS	0	/* Fusion Bit Operations Support option */
+#define XCHAL_HAVE_FUSION_AVS		0	/* Fusion AVS option */
+#define XCHAL_HAVE_FUSION_16BIT_BASEBAND 0	/* Fusion 16-bit Baseband option */
+#define XCHAL_HAVE_FUSION_VITERBI	0	/* Fusion Viterbi option */
+#define XCHAL_HAVE_FUSION_SOFTDEMAP	0	/* Fusion Soft Bit Demap option */
+#define XCHAL_HAVE_HIFIPRO		0	/* HiFiPro Audio Engine pkg */
+#define XCHAL_HAVE_HIFI5		0	/* HiFi5 Audio Engine pkg */
+#define XCHAL_HAVE_HIFI5_NN_MAC		0	/* HiFi5 Audio Engine NN-MAC option */
+#define XCHAL_HAVE_HIFI5_VFPU		0	/* HiFi5 Audio Engine Single-Precision VFPU option */
+#define XCHAL_HAVE_HIFI5_HP_VFPU	0	/* HiFi5 Audio Engine Half-Precision VFPU option */
+#define XCHAL_HAVE_HIFI4		0	/* HiFi4 Audio Engine pkg */
+#define XCHAL_HAVE_HIFI4_VFPU		0	/* HiFi4 Audio Engine VFPU option */
+#define XCHAL_HAVE_HIFI3		0	/* HiFi3 Audio Engine pkg */
+#define XCHAL_HAVE_HIFI3_VFPU		0	/* HiFi3 Audio Engine VFPU option */
+#define XCHAL_HAVE_HIFI3Z		0	/* HiFi3Z Audio Engine pkg */
+#define XCHAL_HAVE_HIFI3Z_VFPU	0	/* HiFi3Z Audio Engine VFPU option */
+#define XCHAL_HAVE_HIFI2		0	/* HiFi2 Audio Engine pkg */
+#define XCHAL_HAVE_HIFI2EP		0	/* HiFi2EP */
+#define XCHAL_HAVE_HIFI_MINI		0
+
+
+
+#define XCHAL_HAVE_VECTORFPU2005	0	/* vector floating-point pkg */
+#define XCHAL_HAVE_USER_DPFPU		0       /* user DP floating-point pkg */
+#define XCHAL_HAVE_USER_SPFPU		0       /* user SP floating-point pkg */
+#define XCHAL_HAVE_FP			1	/* single prec floating point */
+#define XCHAL_HAVE_FP_DIV		1	/* FP with DIV instructions */
+#define XCHAL_HAVE_FP_RECIP		1	/* FP with RECIP instructions */
+#define XCHAL_HAVE_FP_SQRT		1	/* FP with SQRT instructions */
+#define XCHAL_HAVE_FP_RSQRT		1	/* FP with RSQRT instructions */
+#define XCHAL_HAVE_DFP			0	/* double precision FP pkg */
+#define XCHAL_HAVE_DFP_DIV		0	/* DFP with DIV instructions */
+#define XCHAL_HAVE_DFP_RECIP		0	/* DFP with RECIP instructions*/
+#define XCHAL_HAVE_DFP_SQRT		0	/* DFP with SQRT instructions */
+#define XCHAL_HAVE_DFP_RSQRT		0	/* DFP with RSQRT instructions*/
+#define XCHAL_HAVE_DFP_ACCEL		0	/* double precision FP acceleration pkg */
+#define XCHAL_HAVE_DFP_accel		XCHAL_HAVE_DFP_ACCEL	/* for backward compatibility */
+
+#define XCHAL_HAVE_DFPU_SINGLE_ONLY	1	/* DFPU Coprocessor, single precision only */
+#define XCHAL_HAVE_DFPU_SINGLE_DOUBLE	0	/* DFPU Coprocessor, single and double precision */
+#define XCHAL_HAVE_VECTRA1		0	/* Vectra I  pkg */
+#define XCHAL_HAVE_VECTRALX		0	/* Vectra LX pkg */
+
+#define XCHAL_HAVE_FUSIONG		0    /* FusionG */
+#define XCHAL_HAVE_FUSIONG3		0    /* FusionG3 */
+#define XCHAL_HAVE_FUSIONG6		0    /* FusionG6 */
+#define XCHAL_HAVE_FUSIONG_SP_VFPU	0    /* sp_vfpu option on FusionG */
+#define XCHAL_HAVE_FUSIONG_DP_VFPU	0    /* dp_vfpu option on FusionG */
+#define XCHAL_FUSIONG_SIMD32		0    /* simd32 for FusionG */
+
+#define XCHAL_HAVE_PDX			0    /* PDX */
+#define XCHAL_PDX_SIMD32		0    /* simd32 for PDX */
+#define XCHAL_HAVE_PDX4			0    /* PDX4 */
+#define XCHAL_HAVE_PDX8			0    /* PDX8 */
+#define XCHAL_HAVE_PDX16		0    /* PDX16 */
+
+#define XCHAL_HAVE_CONNXD2		0	/* ConnX D2 pkg */
+#define XCHAL_HAVE_CONNXD2_DUALLSFLIX   0	/* ConnX D2 & Dual LoadStore Flix */
+#define XCHAL_HAVE_BBE16		0	/* ConnX BBE16 pkg */
+#define XCHAL_HAVE_BBE16_RSQRT		0	/* BBE16 & vector recip sqrt */
+#define XCHAL_HAVE_BBE16_VECDIV		0	/* BBE16 & vector divide */
+#define XCHAL_HAVE_BBE16_DESPREAD	0	/* BBE16 & despread */
+#define XCHAL_HAVE_BBENEP		0	/* ConnX BBENEP pkgs */
+#define XCHAL_HAVE_BBENEP_SP_VFPU	0      /* sp_vfpu option on BBE-EP */
+#define XCHAL_HAVE_BSP3			0	/* ConnX BSP3 pkg */
+#define XCHAL_HAVE_BSP3_TRANSPOSE	0	/* BSP3 & transpose32x32 */
+#define XCHAL_HAVE_SSP16		0	/* ConnX SSP16 pkg */
+#define XCHAL_HAVE_SSP16_VITERBI	0	/* SSP16 & viterbi */
+#define XCHAL_HAVE_TURBO16		0	/* ConnX Turbo16 pkg */
+#define XCHAL_HAVE_BBP16		0	/* ConnX BBP16 pkg */
+#define XCHAL_HAVE_FLIX3		0	/* basic 3-way FLIX option */
+#define XCHAL_HAVE_GRIVPEP		0	/* General Release of IVPEP */
+#define XCHAL_HAVE_GRIVPEP_HISTOGRAM	0       /* Histogram option on GRIVPEP */
+
+#define XCHAL_HAVE_VISION	        0     /* Vision P5/P6 */
+#define XCHAL_VISION_SIMD16             0     /* simd16 for Vision P5/P6 */
+#define XCHAL_VISION_TYPE               0     /* Vision P5, P6, or P3 */
+#define XCHAL_VISION_QUAD_MAC_TYPE      0     /* quad_mac option on Vision P6 */
+#define XCHAL_HAVE_VISION_HISTOGRAM     0     /* histogram option on Vision P5/P6 */
+#define XCHAL_HAVE_VISION_SP_VFPU       0     /* sp_vfpu option on Vision P5/P6 */
+#define XCHAL_HAVE_VISION_HP_VFPU       0     /* hp_vfpu option on Vision P6 */
+
+#define XCHAL_HAVE_VISIONC	        0     /* Vision C */
+
+/*----------------------------------------------------------------------
+				MISC
+  ----------------------------------------------------------------------*/
+
+#define XCHAL_NUM_LOADSTORE_UNITS	1	/* load/store units */
+#define XCHAL_NUM_WRITEBUFFER_ENTRIES	4	/* size of write buffer */
+#define XCHAL_INST_FETCH_WIDTH		4	/* instr-fetch width in bytes */
+#define XCHAL_DATA_WIDTH		16	/* data width in bytes */
+#define XCHAL_DATA_PIPE_DELAY		1	/* d-side pipeline delay
+						   (1 = 5-stage, 2 = 7-stage) */
+#define XCHAL_CLOCK_GATING_GLOBAL	1	/* global clock gating */
+#define XCHAL_CLOCK_GATING_FUNCUNIT	1	/* funct. unit clock gating */
+/*  In T1050, applies to selected core load and store instructions (see ISA): */
+#define XCHAL_UNALIGNED_LOAD_EXCEPTION	0	/* unaligned loads cause exc. */
+#define XCHAL_UNALIGNED_STORE_EXCEPTION	0	/* unaligned stores cause exc.*/
+#define XCHAL_UNALIGNED_LOAD_HW		1	/* unaligned loads work in hw */
+#define XCHAL_UNALIGNED_STORE_HW	1	/* unaligned stores work in hw*/
+
+#define XCHAL_SW_VERSION		1200012	/* sw version of this header */
+
+#define XCHAL_CORE_ID			"LX7_ESP32_S3_MP"	/* alphanum core name
+						   (CoreID) set in the Xtensa
+						   Processor Generator */
+
+#define XCHAL_BUILD_UNIQUE_ID		0x00090F1F	/* 22-bit sw build ID */
+
+/*
+ *  These definitions describe the hardware targeted by this software.
+ */
+#define XCHAL_HW_CONFIGID0		0xC2F0FFFE	/* ConfigID hi 32 bits*/
+#define XCHAL_HW_CONFIGID1		0x23090F1F	/* ConfigID lo 32 bits*/
+#define XCHAL_HW_VERSION_NAME		"LX7.0.12"	/* full version name */
+#define XCHAL_HW_VERSION_MAJOR		2700	/* major ver# of targeted hw */
+#define XCHAL_HW_VERSION_MINOR		12	/* minor ver# of targeted hw */
+#define XCHAL_HW_VERSION		270012	/* major*100+minor */
+#define XCHAL_HW_REL_LX7		1
+#define XCHAL_HW_REL_LX7_0		1
+#define XCHAL_HW_REL_LX7_0_12		1
+#define XCHAL_HW_CONFIGID_RELIABLE	1
+/*  If software targets a *range* of hardware versions, these are the bounds: */
+#define XCHAL_HW_MIN_VERSION_MAJOR	2700	/* major v of earliest tgt hw */
+#define XCHAL_HW_MIN_VERSION_MINOR	12	/* minor v of earliest tgt hw */
+#define XCHAL_HW_MIN_VERSION		270012	/* earliest targeted hw */
+#define XCHAL_HW_MAX_VERSION_MAJOR	2700	/* major v of latest tgt hw */
+#define XCHAL_HW_MAX_VERSION_MINOR	12	/* minor v of latest tgt hw */
+#define XCHAL_HW_MAX_VERSION		270012	/* latest targeted hw */
+
+
+/*----------------------------------------------------------------------
+				CACHE
+  ----------------------------------------------------------------------*/
+
+#define XCHAL_ICACHE_LINESIZE		4	/* I-cache line size in bytes */
+#define XCHAL_DCACHE_LINESIZE		16	/* D-cache line size in bytes */
+#define XCHAL_ICACHE_LINEWIDTH		2	/* log2(I line size in bytes) */
+#define XCHAL_DCACHE_LINEWIDTH		4	/* log2(D line size in bytes) */
+
+#define XCHAL_ICACHE_SIZE		0	/* I-cache size in bytes or 0 */
+#define XCHAL_DCACHE_SIZE		0	/* D-cache size in bytes or 0 */
+
+#define XCHAL_DCACHE_IS_WRITEBACK	0	/* writeback feature */
+#define XCHAL_DCACHE_IS_COHERENT	0	/* MP coherence feature */
+
+#define XCHAL_HAVE_PREFETCH		0	/* PREFCTL register */
+#define XCHAL_HAVE_PREFETCH_L1		0	/* prefetch to L1 dcache */
+#define XCHAL_PREFETCH_CASTOUT_LINES	0	/* dcache pref. castout bufsz */
+#define XCHAL_PREFETCH_ENTRIES		0	/* cache prefetch entries */
+#define XCHAL_PREFETCH_BLOCK_ENTRIES	0	/* prefetch block streams */
+#define XCHAL_HAVE_CACHE_BLOCKOPS	0	/* block prefetch for caches */
+#define XCHAL_HAVE_ICACHE_TEST		0	/* Icache test instructions */
+#define XCHAL_HAVE_DCACHE_TEST		0	/* Dcache test instructions */
+#define XCHAL_HAVE_ICACHE_DYN_WAYS	0	/* Icache dynamic way support */
+#define XCHAL_HAVE_DCACHE_DYN_WAYS	0	/* Dcache dynamic way support */
+
+
+
+
+/****************************************************************************
+    Parameters Useful for PRIVILEGED (Supervisory or Non-Virtualized) Code
+ ****************************************************************************/
+
+
+#ifndef XTENSA_HAL_NON_PRIVILEGED_ONLY
+
+/*----------------------------------------------------------------------
+				CACHE
+  ----------------------------------------------------------------------*/
+
+#define XCHAL_HAVE_PIF			1	/* any outbound bus present */
+
+#define XCHAL_HAVE_AXI			0	/* AXI bus */
+#define XCHAL_HAVE_AXI_ECC		0	/* ECC on AXI bus */
+#define XCHAL_HAVE_ACELITE		0	/* ACELite bus */
+
+#define XCHAL_HAVE_PIF_WR_RESP			0	/* pif write response */
+#define XCHAL_HAVE_PIF_REQ_ATTR			1	/* pif attribute */
+
+/*  If present, cache size in bytes == (ways * 2^(linewidth + setwidth)).  */
+
+/*  Number of cache sets in log2(lines per way):  */
+#define XCHAL_ICACHE_SETWIDTH		0
+#define XCHAL_DCACHE_SETWIDTH		0
+
+/*  Cache set associativity (number of ways):  */
+#define XCHAL_ICACHE_WAYS		1
+#define XCHAL_DCACHE_WAYS		1
+
+/*  Cache features:  */
+#define XCHAL_ICACHE_LINE_LOCKABLE	0
+#define XCHAL_DCACHE_LINE_LOCKABLE	0
+#define XCHAL_ICACHE_ECC_PARITY		0
+#define XCHAL_DCACHE_ECC_PARITY		0
+#define XCHAL_ICACHE_ECC_WIDTH		1
+#define XCHAL_DCACHE_ECC_WIDTH		1
+
+/*  Cache access size in bytes (affects operation of SICW instruction):  */
+#define XCHAL_ICACHE_ACCESS_SIZE	1
+#define XCHAL_DCACHE_ACCESS_SIZE	1
+
+#define XCHAL_DCACHE_BANKS		0	/* number of banks */
+
+/*  Number of encoded cache attr bits (see <xtensa/hal.h> for decoded bits):  */
+#define XCHAL_CA_BITS			4
+
+
+/*----------------------------------------------------------------------
+			INTERNAL I/D RAM/ROMs and XLMI
+  ----------------------------------------------------------------------*/
+#define XCHAL_NUM_INSTROM		0	/* number of core instr. ROMs */
+#define XCHAL_NUM_INSTRAM		1	/* number of core instr. RAMs */
+#define XCHAL_NUM_DATAROM		0	/* number of core data ROMs */
+#define XCHAL_NUM_DATARAM		1	/* number of core data RAMs */
+#define XCHAL_NUM_URAM			0	/* number of core unified RAMs*/
+#define XCHAL_NUM_XLMI			0	/* number of core XLMI ports */
+
+/*  Instruction RAM 0:  */
+#define XCHAL_INSTRAM0_VADDR		0x40000000	/* virtual address */
+#define XCHAL_INSTRAM0_PADDR		0x40000000	/* physical address */
+#define XCHAL_INSTRAM0_SIZE		67108864	/* size in bytes */
+#define XCHAL_INSTRAM0_ECC_PARITY	0	/* ECC/parity type, 0=none */
+#define XCHAL_HAVE_INSTRAM0		1
+#define XCHAL_INSTRAM0_HAVE_IDMA	0	/* idma supported by this local memory */
+
+/*  Data RAM 0:  */
+#define XCHAL_DATARAM0_VADDR		0x3C000000	/* virtual address */
+#define XCHAL_DATARAM0_PADDR		0x3C000000	/* physical address */
+#define XCHAL_DATARAM0_SIZE		67108864	/* size in bytes */
+#define XCHAL_DATARAM0_ECC_PARITY	0	/* ECC/parity type, 0=none */
+#define XCHAL_DATARAM0_BANKS		1	/* number of banks */
+#define XCHAL_HAVE_DATARAM0		1
+#define XCHAL_DATARAM0_HAVE_IDMA	0	/* idma supported by this local memory */
+
+#define XCHAL_HAVE_IDMA			0
+#define XCHAL_HAVE_IDMA_TRANSPOSE	0
+
+#define XCHAL_HAVE_IMEM_LOADSTORE	1	/* can load/store to IROM/IRAM*/
+
+
+/*----------------------------------------------------------------------
+			INTERRUPTS and TIMERS
+  ----------------------------------------------------------------------*/
+
+#define XCHAL_HAVE_INTERRUPTS		1	/* interrupt option */
+#define XCHAL_HAVE_HIGHPRI_INTERRUPTS	1	/* med/high-pri. interrupts */
+#define XCHAL_HAVE_NMI			1	/* non-maskable interrupt */
+#define XCHAL_HAVE_CCOUNT		1	/* CCOUNT reg. (timer option) */
+#define XCHAL_NUM_TIMERS		3	/* number of CCOMPAREn regs */
+#define XCHAL_NUM_INTERRUPTS		32	/* number of interrupts */
+#define XCHAL_NUM_INTERRUPTS_LOG2	5	/* ceil(log2(NUM_INTERRUPTS)) */
+#define XCHAL_NUM_EXTINTERRUPTS		26	/* num of external interrupts */
+#define XCHAL_NUM_INTLEVELS		6	/* number of interrupt levels
+						   (not including level zero) */
+#define XCHAL_EXCM_LEVEL		3	/* level masked by PS.EXCM */
+	/* (always 1 in XEA1; levels 2 .. EXCM_LEVEL are "medium priority") */
+
+/*  Masks of interrupts at each interrupt level:  */
+#define XCHAL_INTLEVEL1_MASK		0x000637FF
+#define XCHAL_INTLEVEL2_MASK		0x00380000
+#define XCHAL_INTLEVEL3_MASK		0x28C08800
+#define XCHAL_INTLEVEL4_MASK		0x53000000
+#define XCHAL_INTLEVEL5_MASK		0x84010000
+#define XCHAL_INTLEVEL6_MASK		0x00000000
+#define XCHAL_INTLEVEL7_MASK		0x00004000
+
+/*  Masks of interrupts at each range 1..n of interrupt levels:  */
+#define XCHAL_INTLEVEL1_ANDBELOW_MASK	0x000637FF
+#define XCHAL_INTLEVEL2_ANDBELOW_MASK	0x003E37FF
+#define XCHAL_INTLEVEL3_ANDBELOW_MASK	0x28FEBFFF
+#define XCHAL_INTLEVEL4_ANDBELOW_MASK	0x7BFEBFFF
+#define XCHAL_INTLEVEL5_ANDBELOW_MASK	0xFFFFBFFF
+#define XCHAL_INTLEVEL6_ANDBELOW_MASK	0xFFFFBFFF
+#define XCHAL_INTLEVEL7_ANDBELOW_MASK	0xFFFFFFFF
+
+/*  Level of each interrupt:  */
+#define XCHAL_INT0_LEVEL		1
+#define XCHAL_INT1_LEVEL		1
+#define XCHAL_INT2_LEVEL		1
+#define XCHAL_INT3_LEVEL		1
+#define XCHAL_INT4_LEVEL		1
+#define XCHAL_INT5_LEVEL		1
+#define XCHAL_INT6_LEVEL		1
+#define XCHAL_INT7_LEVEL		1
+#define XCHAL_INT8_LEVEL		1
+#define XCHAL_INT9_LEVEL		1
+#define XCHAL_INT10_LEVEL		1
+#define XCHAL_INT11_LEVEL		3
+#define XCHAL_INT12_LEVEL		1
+#define XCHAL_INT13_LEVEL		1
+#define XCHAL_INT14_LEVEL		7
+#define XCHAL_INT15_LEVEL		3
+#define XCHAL_INT16_LEVEL		5
+#define XCHAL_INT17_LEVEL		1
+#define XCHAL_INT18_LEVEL		1
+#define XCHAL_INT19_LEVEL		2
+#define XCHAL_INT20_LEVEL		2
+#define XCHAL_INT21_LEVEL		2
+#define XCHAL_INT22_LEVEL		3
+#define XCHAL_INT23_LEVEL		3
+#define XCHAL_INT24_LEVEL		4
+#define XCHAL_INT25_LEVEL		4
+#define XCHAL_INT26_LEVEL		5
+#define XCHAL_INT27_LEVEL		3
+#define XCHAL_INT28_LEVEL		4
+#define XCHAL_INT29_LEVEL		3
+#define XCHAL_INT30_LEVEL		4
+#define XCHAL_INT31_LEVEL		5
+#define XCHAL_DEBUGLEVEL		6	/* debug interrupt level */
+#define XCHAL_HAVE_DEBUG_EXTERN_INT	1	/* OCD external db interrupt */
+#define XCHAL_NMILEVEL			7	/* NMI "level" (for use with
+						   EXCSAVE/EPS/EPC_n, RFI n) */
+
+/*  Type of each interrupt:  */
+#define XCHAL_INT0_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT1_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT2_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT3_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT4_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT5_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT6_TYPE 	XTHAL_INTTYPE_TIMER
+#define XCHAL_INT7_TYPE 	XTHAL_INTTYPE_SOFTWARE
+#define XCHAL_INT8_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT9_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT10_TYPE 	XTHAL_INTTYPE_EXTERN_EDGE
+#define XCHAL_INT11_TYPE 	XTHAL_INTTYPE_PROFILING
+#define XCHAL_INT12_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT13_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT14_TYPE 	XTHAL_INTTYPE_NMI
+#define XCHAL_INT15_TYPE 	XTHAL_INTTYPE_TIMER
+#define XCHAL_INT16_TYPE 	XTHAL_INTTYPE_TIMER
+#define XCHAL_INT17_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT18_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT19_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT20_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT21_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT22_TYPE 	XTHAL_INTTYPE_EXTERN_EDGE
+#define XCHAL_INT23_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT24_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT25_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT26_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT27_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+#define XCHAL_INT28_TYPE 	XTHAL_INTTYPE_EXTERN_EDGE
+#define XCHAL_INT29_TYPE 	XTHAL_INTTYPE_SOFTWARE
+#define XCHAL_INT30_TYPE 	XTHAL_INTTYPE_EXTERN_EDGE
+#define XCHAL_INT31_TYPE 	XTHAL_INTTYPE_EXTERN_LEVEL
+
+/*  Masks of interrupts for each type of interrupt:  */
+#define XCHAL_INTTYPE_MASK_UNCONFIGURED	0x00000000
+#define XCHAL_INTTYPE_MASK_SOFTWARE	0x20000080
+#define XCHAL_INTTYPE_MASK_EXTERN_EDGE	0x50400400
+#define XCHAL_INTTYPE_MASK_EXTERN_LEVEL	0x8FBE333F
+#define XCHAL_INTTYPE_MASK_TIMER	0x00018040
+#define XCHAL_INTTYPE_MASK_NMI		0x00004000
+#define XCHAL_INTTYPE_MASK_WRITE_ERROR	0x00000000
+#define XCHAL_INTTYPE_MASK_PROFILING	0x00000800
+#define XCHAL_INTTYPE_MASK_IDMA_DONE	0x00000000
+#define XCHAL_INTTYPE_MASK_IDMA_ERR	0x00000000
+#define XCHAL_INTTYPE_MASK_GS_ERR	0x00000000
+
+/*  Interrupt numbers assigned to specific interrupt sources:  */
+#define XCHAL_TIMER0_INTERRUPT		6	/* CCOMPARE0 */
+#define XCHAL_TIMER1_INTERRUPT		15	/* CCOMPARE1 */
+#define XCHAL_TIMER2_INTERRUPT		16	/* CCOMPARE2 */
+#define XCHAL_TIMER3_INTERRUPT		XTHAL_TIMER_UNCONFIGURED
+#define XCHAL_NMI_INTERRUPT		14	/* non-maskable interrupt */
+#define XCHAL_PROFILING_INTERRUPT	11
+
+/*  Interrupt numbers for levels at which only one interrupt is configured:  */
+#define XCHAL_INTLEVEL7_NUM		14
+/*  (There are many interrupts each at level(s) 1, 2, 3, 4, 5.)  */
+
+
+/*
+ *  External interrupt mapping.
+ *  These macros describe how Xtensa processor interrupt numbers
+ *  (as numbered internally, eg. in INTERRUPT and INTENABLE registers)
+ *  map to external BInterrupt<n> pins, for those interrupts
+ *  configured as external (level-triggered, edge-triggered, or NMI).
+ *  See the Xtensa processor databook for more details.
+ */
+
+/*  Core interrupt numbers mapped to each EXTERNAL BInterrupt pin number:  */
+#define XCHAL_EXTINT0_NUM		0	/* (intlevel 1) */
+#define XCHAL_EXTINT1_NUM		1	/* (intlevel 1) */
+#define XCHAL_EXTINT2_NUM		2	/* (intlevel 1) */
+#define XCHAL_EXTINT3_NUM		3	/* (intlevel 1) */
+#define XCHAL_EXTINT4_NUM		4	/* (intlevel 1) */
+#define XCHAL_EXTINT5_NUM		5	/* (intlevel 1) */
+#define XCHAL_EXTINT6_NUM		8	/* (intlevel 1) */
+#define XCHAL_EXTINT7_NUM		9	/* (intlevel 1) */
+#define XCHAL_EXTINT8_NUM		10	/* (intlevel 1) */
+#define XCHAL_EXTINT9_NUM		12	/* (intlevel 1) */
+#define XCHAL_EXTINT10_NUM		13	/* (intlevel 1) */
+#define XCHAL_EXTINT11_NUM		14	/* (intlevel 7) */
+#define XCHAL_EXTINT12_NUM		17	/* (intlevel 1) */
+#define XCHAL_EXTINT13_NUM		18	/* (intlevel 1) */
+#define XCHAL_EXTINT14_NUM		19	/* (intlevel 2) */
+#define XCHAL_EXTINT15_NUM		20	/* (intlevel 2) */
+#define XCHAL_EXTINT16_NUM		21	/* (intlevel 2) */
+#define XCHAL_EXTINT17_NUM		22	/* (intlevel 3) */
+#define XCHAL_EXTINT18_NUM		23	/* (intlevel 3) */
+#define XCHAL_EXTINT19_NUM		24	/* (intlevel 4) */
+#define XCHAL_EXTINT20_NUM		25	/* (intlevel 4) */
+#define XCHAL_EXTINT21_NUM		26	/* (intlevel 5) */
+#define XCHAL_EXTINT22_NUM		27	/* (intlevel 3) */
+#define XCHAL_EXTINT23_NUM		28	/* (intlevel 4) */
+#define XCHAL_EXTINT24_NUM		30	/* (intlevel 4) */
+#define XCHAL_EXTINT25_NUM		31	/* (intlevel 5) */
+/*  EXTERNAL BInterrupt pin numbers mapped to each core interrupt number:  */
+#define XCHAL_INT0_EXTNUM		0	/* (intlevel 1) */
+#define XCHAL_INT1_EXTNUM		1	/* (intlevel 1) */
+#define XCHAL_INT2_EXTNUM		2	/* (intlevel 1) */
+#define XCHAL_INT3_EXTNUM		3	/* (intlevel 1) */
+#define XCHAL_INT4_EXTNUM		4	/* (intlevel 1) */
+#define XCHAL_INT5_EXTNUM		5	/* (intlevel 1) */
+#define XCHAL_INT8_EXTNUM		6	/* (intlevel 1) */
+#define XCHAL_INT9_EXTNUM		7	/* (intlevel 1) */
+#define XCHAL_INT10_EXTNUM		8	/* (intlevel 1) */
+#define XCHAL_INT12_EXTNUM		9	/* (intlevel 1) */
+#define XCHAL_INT13_EXTNUM		10	/* (intlevel 1) */
+#define XCHAL_INT14_EXTNUM		11	/* (intlevel 7) */
+#define XCHAL_INT17_EXTNUM		12	/* (intlevel 1) */
+#define XCHAL_INT18_EXTNUM		13	/* (intlevel 1) */
+#define XCHAL_INT19_EXTNUM		14	/* (intlevel 2) */
+#define XCHAL_INT20_EXTNUM		15	/* (intlevel 2) */
+#define XCHAL_INT21_EXTNUM		16	/* (intlevel 2) */
+#define XCHAL_INT22_EXTNUM		17	/* (intlevel 3) */
+#define XCHAL_INT23_EXTNUM		18	/* (intlevel 3) */
+#define XCHAL_INT24_EXTNUM		19	/* (intlevel 4) */
+#define XCHAL_INT25_EXTNUM		20	/* (intlevel 4) */
+#define XCHAL_INT26_EXTNUM		21	/* (intlevel 5) */
+#define XCHAL_INT27_EXTNUM		22	/* (intlevel 3) */
+#define XCHAL_INT28_EXTNUM		23	/* (intlevel 4) */
+#define XCHAL_INT30_EXTNUM		24	/* (intlevel 4) */
+#define XCHAL_INT31_EXTNUM		25	/* (intlevel 5) */
+
+
+/*----------------------------------------------------------------------
+			EXCEPTIONS and VECTORS
+  ----------------------------------------------------------------------*/
+
+#define XCHAL_XEA_VERSION		2	/* Xtensa Exception Architecture
+						   number: 1 == XEA1 (old)
+							   2 == XEA2 (new)
+							   0 == XEAX (extern) or TX */
+#define XCHAL_HAVE_XEA1			0	/* Exception Architecture 1 */
+#define XCHAL_HAVE_XEA2			1	/* Exception Architecture 2 */
+#define XCHAL_HAVE_XEAX			0	/* External Exception Arch. */
+#define XCHAL_HAVE_EXCEPTIONS		1	/* exception option */
+#define XCHAL_HAVE_HALT			0	/* halt architecture option */
+#define XCHAL_HAVE_BOOTLOADER		0	/* boot loader (for TX) */
+#define XCHAL_HAVE_MEM_ECC_PARITY	0	/* local memory ECC/parity */
+#define XCHAL_HAVE_VECTOR_SELECT	1	/* relocatable vectors */
+#define XCHAL_HAVE_VECBASE		1	/* relocatable vectors */
+#define XCHAL_VECBASE_RESET_VADDR	0x40000000  /* VECBASE reset value */
+#define XCHAL_VECBASE_RESET_PADDR	0x40000000
+#define XCHAL_RESET_VECBASE_OVERLAP	0
+
+#define XCHAL_RESET_VECTOR0_VADDR	0x50000000
+#define XCHAL_RESET_VECTOR0_PADDR	0x50000000
+#define XCHAL_RESET_VECTOR1_VADDR	0x40000400
+#define XCHAL_RESET_VECTOR1_PADDR	0x40000400
+#define XCHAL_RESET_VECTOR_VADDR	0x40000400
+#define XCHAL_RESET_VECTOR_PADDR	0x40000400
+#define XCHAL_USER_VECOFS		0x00000340
+#define XCHAL_USER_VECTOR_VADDR		0x40000340
+#define XCHAL_USER_VECTOR_PADDR		0x40000340
+#define XCHAL_KERNEL_VECOFS		0x00000300
+#define XCHAL_KERNEL_VECTOR_VADDR	0x40000300
+#define XCHAL_KERNEL_VECTOR_PADDR	0x40000300
+#define XCHAL_DOUBLEEXC_VECOFS		0x000003C0
+#define XCHAL_DOUBLEEXC_VECTOR_VADDR	0x400003C0
+#define XCHAL_DOUBLEEXC_VECTOR_PADDR	0x400003C0
+#define XCHAL_WINDOW_OF4_VECOFS		0x00000000
+#define XCHAL_WINDOW_UF4_VECOFS		0x00000040
+#define XCHAL_WINDOW_OF8_VECOFS		0x00000080
+#define XCHAL_WINDOW_UF8_VECOFS		0x000000C0
+#define XCHAL_WINDOW_OF12_VECOFS	0x00000100
+#define XCHAL_WINDOW_UF12_VECOFS	0x00000140
+#define XCHAL_WINDOW_VECTORS_VADDR	0x40000000
+#define XCHAL_WINDOW_VECTORS_PADDR	0x40000000
+#define XCHAL_INTLEVEL2_VECOFS		0x00000180
+#define XCHAL_INTLEVEL2_VECTOR_VADDR	0x40000180
+#define XCHAL_INTLEVEL2_VECTOR_PADDR	0x40000180
+#define XCHAL_INTLEVEL3_VECOFS		0x000001C0
+#define XCHAL_INTLEVEL3_VECTOR_VADDR	0x400001C0
+#define XCHAL_INTLEVEL3_VECTOR_PADDR	0x400001C0
+#define XCHAL_INTLEVEL4_VECOFS		0x00000200
+#define XCHAL_INTLEVEL4_VECTOR_VADDR	0x40000200
+#define XCHAL_INTLEVEL4_VECTOR_PADDR	0x40000200
+#define XCHAL_INTLEVEL5_VECOFS		0x00000240
+#define XCHAL_INTLEVEL5_VECTOR_VADDR	0x40000240
+#define XCHAL_INTLEVEL5_VECTOR_PADDR	0x40000240
+#define XCHAL_INTLEVEL6_VECOFS		0x00000280
+#define XCHAL_INTLEVEL6_VECTOR_VADDR	0x40000280
+#define XCHAL_INTLEVEL6_VECTOR_PADDR	0x40000280
+#define XCHAL_DEBUG_VECOFS		XCHAL_INTLEVEL6_VECOFS
+#define XCHAL_DEBUG_VECTOR_VADDR	XCHAL_INTLEVEL6_VECTOR_VADDR
+#define XCHAL_DEBUG_VECTOR_PADDR	XCHAL_INTLEVEL6_VECTOR_PADDR
+#define XCHAL_NMI_VECOFS		0x000002C0
+#define XCHAL_NMI_VECTOR_VADDR		0x400002C0
+#define XCHAL_NMI_VECTOR_PADDR		0x400002C0
+#define XCHAL_INTLEVEL7_VECOFS		XCHAL_NMI_VECOFS
+#define XCHAL_INTLEVEL7_VECTOR_VADDR	XCHAL_NMI_VECTOR_VADDR
+#define XCHAL_INTLEVEL7_VECTOR_PADDR	XCHAL_NMI_VECTOR_PADDR
+
+
+/*----------------------------------------------------------------------
+				DEBUG MODULE
+  ----------------------------------------------------------------------*/
+
+/*  Misc  */
+#define XCHAL_HAVE_DEBUG_ERI		1	/* ERI to debug module */
+#define XCHAL_HAVE_DEBUG_APB		0	/* APB to debug module */
+#define XCHAL_HAVE_DEBUG_JTAG		1	/* JTAG to debug module */
+
+/*  On-Chip Debug (OCD)  */
+#define XCHAL_HAVE_OCD			1	/* OnChipDebug option */
+#define XCHAL_NUM_IBREAK		2	/* number of IBREAKn regs */
+#define XCHAL_NUM_DBREAK		2	/* number of DBREAKn regs */
+#define XCHAL_HAVE_OCD_DIR_ARRAY	0	/* faster OCD option (to LX4) */
+#define XCHAL_HAVE_OCD_LS32DDR		1	/* L32DDR/S32DDR (faster OCD) */
+
+/*  TRAX (in core)  */
+#define XCHAL_HAVE_TRAX			1	/* TRAX in debug module */
+#define XCHAL_TRAX_MEM_SIZE		16384	/* TRAX memory size in bytes */
+#define XCHAL_TRAX_MEM_SHAREABLE	1	/* start/end regs; ready sig. */
+#define XCHAL_TRAX_ATB_WIDTH		0	/* ATB width (bits), 0=no ATB */
+#define XCHAL_TRAX_TIME_WIDTH		0	/* timestamp bitwidth, 0=none */
+
+/*  Perf counters  */
+#define XCHAL_NUM_PERF_COUNTERS		2	/* performance counters */
+
+
+/*----------------------------------------------------------------------
+				MMU
+  ----------------------------------------------------------------------*/
+
+/*  See core-matmap.h header file for more details.  */
+
+#define XCHAL_HAVE_TLBS			1	/* inverse of HAVE_CACHEATTR */
+#define XCHAL_HAVE_SPANNING_WAY		1	/* one way maps I+D 4GB vaddr */
+#define XCHAL_SPANNING_WAY		0	/* TLB spanning way number */
+#define XCHAL_HAVE_IDENTITY_MAP		1	/* vaddr == paddr always */
+#define XCHAL_HAVE_CACHEATTR		0	/* CACHEATTR register present */
+#define XCHAL_HAVE_MIMIC_CACHEATTR	1	/* region protection */
+#define XCHAL_HAVE_XLT_CACHEATTR	0	/* region prot. w/translation */
+#define XCHAL_HAVE_PTP_MMU		0	/* full MMU (with page table
+						   [autorefill] and protection)
+						   usable for an MMU-based OS */
+
+/*  If none of the above last 5 are set, it's a custom TLB configuration.  */
+
+#define XCHAL_MMU_ASID_BITS		0	/* number of bits in ASIDs */
+#define XCHAL_MMU_RINGS			1	/* number of rings (1..4) */
+#define XCHAL_MMU_RING_BITS		0	/* num of bits in RING field */
+
+/*----------------------------------------------------------------------
+				MPU
+  ----------------------------------------------------------------------*/
+#define XCHAL_HAVE_MPU			0
+#define XCHAL_MPU_ENTRIES		0
+
+#define XCHAL_MPU_ALIGN_REQ		1	/* MPU requires alignment of entries to background map */
+#define XCHAL_MPU_BACKGROUND_ENTRIES	0	/* number of entries in bg map*/
+#define XCHAL_MPU_BG_CACHEADRDIS	0	/* default CACHEADRDIS for bg */
+
+#define XCHAL_MPU_ALIGN_BITS		0
+#define XCHAL_MPU_ALIGN			0
+
+#endif /* !XTENSA_HAL_NON_PRIVILEGED_ONLY */
+
+
+#endif /* _XTENSA_CORE_CONFIGURATION_H */
diff --git a/arch/xtensa/variants/esp32s3/include/variant/tie-asm.h b/arch/xtensa/variants/esp32s3/include/variant/tie-asm.h
new file mode 100644
index 000000000000..104dc10fd414
--- /dev/null
+++ b/arch/xtensa/variants/esp32s3/include/variant/tie-asm.h
@@ -0,0 +1,429 @@
+/*
+ * tie-asm.h -- compile-time HAL assembler definitions dependent on CORE & TIE
+ *
+ *  NOTE:  This header file is not meant to be included directly.
+ */
+
+/* This header file contains assembly-language definitions (assembly
+   macros, etc.) for this specific Xtensa processor's TIE extensions
+   and options.  It is customized to this Xtensa processor configuration.
+
+   Customer ID=15128; Build=0x90f1f; Copyright (c) 1999-2021 Cadence Design Systems Inc.
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be included
+   in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+   IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+   CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+   TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+   SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.  */
+
+#ifndef _XTENSA_CORE_TIE_ASM_H
+#define _XTENSA_CORE_TIE_ASM_H
+
+/*  Selection parameter values for save-area save/restore macros:  */
+/*  Option vs. TIE:  */
+#define XTHAL_SAS_TIE	0x0001	/* custom extension or coprocessor */
+#define XTHAL_SAS_OPT	0x0002	/* optional (and not a coprocessor) */
+#define XTHAL_SAS_ANYOT	0x0003	/* both of the above */
+/*  Whether used automatically by compiler:  */
+#define XTHAL_SAS_NOCC	0x0004	/* not used by compiler w/o special opts/code */
+#define XTHAL_SAS_CC	0x0008	/* used by compiler without special opts/code */
+#define XTHAL_SAS_ANYCC	0x000C	/* both of the above */
+/*  ABI handling across function calls:  */
+#define XTHAL_SAS_CALR	0x0010	/* caller-saved */
+#define XTHAL_SAS_CALE	0x0020	/* callee-saved */
+#define XTHAL_SAS_GLOB	0x0040	/* global across function calls (in thread) */
+#define XTHAL_SAS_ANYABI	0x0070	/* all of the above three */
+/*  Misc  */
+#define XTHAL_SAS_ALL	0xFFFF	/* include all default NCP contents */
+#define XTHAL_SAS3(optie,ccuse,abi)	( ((optie) & XTHAL_SAS_ANYOT)  \
+					| ((ccuse) & XTHAL_SAS_ANYCC)  \
+					| ((abi)   & XTHAL_SAS_ANYABI) )
+
+
+    /*
+      *  Macro to store all non-coprocessor (extra) custom TIE and optional state
+      *  (not including zero-overhead loop registers).
+      *  Required parameters:
+      *      ptr         Save area pointer address register (clobbered)
+      *                  (register must contain a 4 byte aligned address).
+      *      at1..at4    Four temporary address registers (first XCHAL_NCP_NUM_ATMPS
+      *                  registers are clobbered, the remaining are unused).
+      *  Optional parameters:
+      *      continue    If macro invoked as part of a larger store sequence, set to 1
+      *                  if this is not the first in the sequence.  Defaults to 0.
+      *      ofs         Offset from start of larger sequence (from value of first ptr
+      *                  in sequence) at which to store.  Defaults to next available space
+      *                  (or 0 if <continue> is 0).
+      *      select      Select what category(ies) of registers to store, as a bitmask
+      *                  (see XTHAL_SAS_xxx constants).  Defaults to all registers.
+      *      alloc       Select what category(ies) of registers to allocate; if any
+      *                  category is selected here that is not in <select>, space for
+      *                  the corresponding registers is skipped without doing any store.
+      */
+    .macro xchal_ncp_store  ptr at1 at2 at3 at4  continue=0 ofs=-1 select=XTHAL_SAS_ALL alloc=0
+	xchal_sa_start	\continue, \ofs
+	// Optional global registers used by default by the compiler:
+	.ifeq (XTHAL_SAS_OPT | XTHAL_SAS_CC | XTHAL_SAS_GLOB) & ~(\select)
+	xchal_sa_align	\ptr, 0, 1016, 4, 4
+	rur.THREADPTR	\at1		// threadptr option
+	s32i	\at1, \ptr, .Lxchal_ofs_+0
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 4
+	.elseif ((XTHAL_SAS_OPT | XTHAL_SAS_CC | XTHAL_SAS_GLOB) & ~(\alloc)) == 0
+	xchal_sa_align	\ptr, 0, 1016, 4, 4
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 4
+	.endif
+	// Optional caller-saved registers used by default by the compiler:
+	.ifeq (XTHAL_SAS_OPT | XTHAL_SAS_CC | XTHAL_SAS_CALR) & ~(\select)
+	xchal_sa_align	\ptr, 0, 1012, 4, 4
+	rsr.ACCLO	\at1		// MAC16 option
+	s32i	\at1, \ptr, .Lxchal_ofs_+0
+	rsr.ACCHI	\at1		// MAC16 option
+	s32i	\at1, \ptr, .Lxchal_ofs_+4
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 8
+	.elseif ((XTHAL_SAS_OPT | XTHAL_SAS_CC | XTHAL_SAS_CALR) & ~(\alloc)) == 0
+	xchal_sa_align	\ptr, 0, 1012, 4, 4
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 8
+	.endif
+	// Optional caller-saved registers not used by default by the compiler:
+	.ifeq (XTHAL_SAS_OPT | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\select)
+	xchal_sa_align	\ptr, 0, 996, 4, 4
+	rsr.BR	\at1		// boolean option
+	s32i	\at1, \ptr, .Lxchal_ofs_+0
+	rsr.SCOMPARE1	\at1		// conditional store option
+	s32i	\at1, \ptr, .Lxchal_ofs_+4
+	rsr.M0	\at1		// MAC16 option
+	s32i	\at1, \ptr, .Lxchal_ofs_+8
+	rsr.M1	\at1		// MAC16 option
+	s32i	\at1, \ptr, .Lxchal_ofs_+12
+	rsr.M2	\at1		// MAC16 option
+	s32i	\at1, \ptr, .Lxchal_ofs_+16
+	rsr.M3	\at1		// MAC16 option
+	s32i	\at1, \ptr, .Lxchal_ofs_+20
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 24
+	.elseif ((XTHAL_SAS_OPT | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\alloc)) == 0
+	xchal_sa_align	\ptr, 0, 996, 4, 4
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 24
+	.endif
+    .endm	// xchal_ncp_store
+
+    /*
+      *  Macro to load all non-coprocessor (extra) custom TIE and optional state
+      *  (not including zero-overhead loop registers).
+      *  Required parameters:
+      *      ptr         Save area pointer address register (clobbered)
+      *                  (register must contain a 4 byte aligned address).
+      *      at1..at4    Four temporary address registers (first XCHAL_NCP_NUM_ATMPS
+      *                  registers are clobbered, the remaining are unused).
+      *  Optional parameters:
+      *      continue    If macro invoked as part of a larger load sequence, set to 1
+      *                  if this is not the first in the sequence.  Defaults to 0.
+      *      ofs         Offset from start of larger sequence (from value of first ptr
+      *                  in sequence) at which to load.  Defaults to next available space
+      *                  (or 0 if <continue> is 0).
+      *      select      Select what category(ies) of registers to load, as a bitmask
+      *                  (see XTHAL_SAS_xxx constants).  Defaults to all registers.
+      *      alloc       Select what category(ies) of registers to allocate; if any
+      *                  category is selected here that is not in <select>, space for
+      *                  the corresponding registers is skipped without doing any load.
+      */
+    .macro xchal_ncp_load  ptr at1 at2 at3 at4  continue=0 ofs=-1 select=XTHAL_SAS_ALL alloc=0
+	xchal_sa_start	\continue, \ofs
+	// Optional global registers used by default by the compiler:
+	.ifeq (XTHAL_SAS_OPT | XTHAL_SAS_CC | XTHAL_SAS_GLOB) & ~(\select)
+	xchal_sa_align	\ptr, 0, 1016, 4, 4
+	l32i	\at1, \ptr, .Lxchal_ofs_+0
+	wur.THREADPTR	\at1		// threadptr option
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 4
+	.elseif ((XTHAL_SAS_OPT | XTHAL_SAS_CC | XTHAL_SAS_GLOB) & ~(\alloc)) == 0
+	xchal_sa_align	\ptr, 0, 1016, 4, 4
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 4
+	.endif
+	// Optional caller-saved registers used by default by the compiler:
+	.ifeq (XTHAL_SAS_OPT | XTHAL_SAS_CC | XTHAL_SAS_CALR) & ~(\select)
+	xchal_sa_align	\ptr, 0, 1012, 4, 4
+	l32i	\at1, \ptr, .Lxchal_ofs_+0
+	wsr.ACCLO	\at1		// MAC16 option
+	l32i	\at1, \ptr, .Lxchal_ofs_+4
+	wsr.ACCHI	\at1		// MAC16 option
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 8
+	.elseif ((XTHAL_SAS_OPT | XTHAL_SAS_CC | XTHAL_SAS_CALR) & ~(\alloc)) == 0
+	xchal_sa_align	\ptr, 0, 1012, 4, 4
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 8
+	.endif
+	// Optional caller-saved registers not used by default by the compiler:
+	.ifeq (XTHAL_SAS_OPT | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\select)
+	xchal_sa_align	\ptr, 0, 996, 4, 4
+	l32i	\at1, \ptr, .Lxchal_ofs_+0
+	wsr.BR	\at1		// boolean option
+	l32i	\at1, \ptr, .Lxchal_ofs_+4
+	wsr.SCOMPARE1	\at1		// conditional store option
+	l32i	\at1, \ptr, .Lxchal_ofs_+8
+	wsr.M0	\at1		// MAC16 option
+	l32i	\at1, \ptr, .Lxchal_ofs_+12
+	wsr.M1	\at1		// MAC16 option
+	l32i	\at1, \ptr, .Lxchal_ofs_+16
+	wsr.M2	\at1		// MAC16 option
+	l32i	\at1, \ptr, .Lxchal_ofs_+20
+	wsr.M3	\at1		// MAC16 option
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 24
+	.elseif ((XTHAL_SAS_OPT | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\alloc)) == 0
+	xchal_sa_align	\ptr, 0, 996, 4, 4
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 24
+	.endif
+    .endm	// xchal_ncp_load
+
+
+#define XCHAL_NCP_NUM_ATMPS	1
+
+    /*
+     *  Macro to store the state of TIE coprocessor FPU.
+     *  Required parameters:
+     *      ptr         Save area pointer address register (clobbered)
+     *                  (register must contain a 4 byte aligned address).
+     *      at1..at4    Four temporary address registers (first XCHAL_CP0_NUM_ATMPS
+     *                  registers are clobbered, the remaining are unused).
+     *  Optional parameters are the same as for xchal_ncp_store.
+     */
+#define xchal_cp_FPU_store	xchal_cp0_store
+    .macro	xchal_cp0_store  ptr at1 at2 at3 at4  continue=0 ofs=-1 select=XTHAL_SAS_ALL alloc=0
+	xchal_sa_start \continue, \ofs
+	// Custom caller-saved registers not used by default by the compiler:
+	.ifeq (XTHAL_SAS_TIE | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\select)
+	xchal_sa_align	\ptr, 0, 948, 4, 4
+	rur.FCR	\at1		// ureg 232
+	s32i	\at1, \ptr, .Lxchal_ofs_+0
+	rur.FSR	\at1		// ureg 233
+	s32i	\at1, \ptr, .Lxchal_ofs_+4
+	ssi	f0, \ptr, .Lxchal_ofs_+8
+	ssi	f1, \ptr, .Lxchal_ofs_+12
+	ssi	f2, \ptr, .Lxchal_ofs_+16
+	ssi	f3, \ptr, .Lxchal_ofs_+20
+	ssi	f4, \ptr, .Lxchal_ofs_+24
+	ssi	f5, \ptr, .Lxchal_ofs_+28
+	ssi	f6, \ptr, .Lxchal_ofs_+32
+	ssi	f7, \ptr, .Lxchal_ofs_+36
+	ssi	f8, \ptr, .Lxchal_ofs_+40
+	ssi	f9, \ptr, .Lxchal_ofs_+44
+	ssi	f10, \ptr, .Lxchal_ofs_+48
+	ssi	f11, \ptr, .Lxchal_ofs_+52
+	ssi	f12, \ptr, .Lxchal_ofs_+56
+	ssi	f13, \ptr, .Lxchal_ofs_+60
+	ssi	f14, \ptr, .Lxchal_ofs_+64
+	ssi	f15, \ptr, .Lxchal_ofs_+68
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 72
+	.elseif ((XTHAL_SAS_TIE | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\alloc)) == 0
+	xchal_sa_align	\ptr, 0, 948, 4, 4
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 72
+	.endif
+    .endm	// xchal_cp0_store
+
+    /*
+     *  Macro to load the state of TIE coprocessor FPU.
+     *  Required parameters:
+     *      ptr         Save area pointer address register (clobbered)
+     *                  (register must contain a 4 byte aligned address).
+     *      at1..at4    Four temporary address registers (first XCHAL_CP0_NUM_ATMPS
+     *                  registers are clobbered, the remaining are unused).
+     *  Optional parameters are the same as for xchal_ncp_load.
+     */
+#define xchal_cp_FPU_load	xchal_cp0_load
+    .macro	xchal_cp0_load  ptr at1 at2 at3 at4  continue=0 ofs=-1 select=XTHAL_SAS_ALL alloc=0
+	xchal_sa_start \continue, \ofs
+	// Custom caller-saved registers not used by default by the compiler:
+	.ifeq (XTHAL_SAS_TIE | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\select)
+	xchal_sa_align	\ptr, 0, 948, 4, 4
+	l32i	\at1, \ptr, .Lxchal_ofs_+0
+	wur.FCR	\at1		// ureg 232
+	l32i	\at1, \ptr, .Lxchal_ofs_+4
+	wur.FSR	\at1		// ureg 233
+	lsi	f0, \ptr, .Lxchal_ofs_+8
+	lsi	f1, \ptr, .Lxchal_ofs_+12
+	lsi	f2, \ptr, .Lxchal_ofs_+16
+	lsi	f3, \ptr, .Lxchal_ofs_+20
+	lsi	f4, \ptr, .Lxchal_ofs_+24
+	lsi	f5, \ptr, .Lxchal_ofs_+28
+	lsi	f6, \ptr, .Lxchal_ofs_+32
+	lsi	f7, \ptr, .Lxchal_ofs_+36
+	lsi	f8, \ptr, .Lxchal_ofs_+40
+	lsi	f9, \ptr, .Lxchal_ofs_+44
+	lsi	f10, \ptr, .Lxchal_ofs_+48
+	lsi	f11, \ptr, .Lxchal_ofs_+52
+	lsi	f12, \ptr, .Lxchal_ofs_+56
+	lsi	f13, \ptr, .Lxchal_ofs_+60
+	lsi	f14, \ptr, .Lxchal_ofs_+64
+	lsi	f15, \ptr, .Lxchal_ofs_+68
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 72
+	.elseif ((XTHAL_SAS_TIE | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\alloc)) == 0
+	xchal_sa_align	\ptr, 0, 948, 4, 4
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 72
+	.endif
+    .endm	// xchal_cp0_load
+
+#define XCHAL_CP0_NUM_ATMPS	1
+    /*
+     *  Macro to store the state of TIE coprocessor cop_ai.
+     *  Required parameters:
+     *      ptr         Save area pointer address register (clobbered)
+     *                  (register must contain a 16 byte aligned address).
+     *      at1..at4    Four temporary address registers (first XCHAL_CP3_NUM_ATMPS
+     *                  registers are clobbered, the remaining are unused).
+     *  Optional parameters are the same as for xchal_ncp_store.
+     */
+#define xchal_cp_cop_ai_store	xchal_cp3_store
+    .macro	xchal_cp3_store  ptr at1 at2 at3 at4  continue=0 ofs=-1 select=XTHAL_SAS_ALL alloc=0
+	xchal_sa_start \continue, \ofs
+	// Custom caller-saved registers not used by default by the compiler:
+	.ifeq (XTHAL_SAS_TIE | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\select)
+	xchal_sa_align	\ptr, 0, 0, 16, 16
+	rur.ACCX_0	\at1		// ureg 0
+	s32i	\at1, \ptr, .Lxchal_ofs_+0
+	rur.ACCX_1	\at1		// ureg 1
+	s32i	\at1, \ptr, .Lxchal_ofs_+4
+	rur.QACC_H_0	\at1		// ureg 2
+	s32i	\at1, \ptr, .Lxchal_ofs_+8
+	rur.QACC_H_1	\at1		// ureg 3
+	s32i	\at1, \ptr, .Lxchal_ofs_+12
+	rur.QACC_H_2	\at1		// ureg 4
+	s32i	\at1, \ptr, .Lxchal_ofs_+16
+	rur.QACC_H_3	\at1		// ureg 5
+	s32i	\at1, \ptr, .Lxchal_ofs_+20
+	rur.QACC_H_4	\at1		// ureg 6
+	s32i	\at1, \ptr, .Lxchal_ofs_+24
+	rur.QACC_L_0	\at1		// ureg 7
+	s32i	\at1, \ptr, .Lxchal_ofs_+28
+	rur.QACC_L_1	\at1		// ureg 8
+	s32i	\at1, \ptr, .Lxchal_ofs_+32
+	rur.QACC_L_2	\at1		// ureg 9
+	s32i	\at1, \ptr, .Lxchal_ofs_+36
+	rur.QACC_L_3	\at1		// ureg 10
+	s32i	\at1, \ptr, .Lxchal_ofs_+40
+	rur.QACC_L_4	\at1		// ureg 11
+	s32i	\at1, \ptr, .Lxchal_ofs_+44
+	rur.SAR_BYTE	\at1		// ureg 13
+	s32i	\at1, \ptr, .Lxchal_ofs_+48
+	rur.FFT_BIT_WIDTH	\at1		// ureg 14
+	s32i	\at1, \ptr, .Lxchal_ofs_+52
+	rur.UA_STATE_0	\at1		// ureg 15
+	s32i	\at1, \ptr, .Lxchal_ofs_+56
+	rur.UA_STATE_1	\at1		// ureg 16
+	s32i	\at1, \ptr, .Lxchal_ofs_+60
+	rur.UA_STATE_2	\at1		// ureg 17
+	s32i	\at1, \ptr, .Lxchal_ofs_+64
+	rur.UA_STATE_3	\at1		// ureg 18
+	s32i	\at1, \ptr, .Lxchal_ofs_+68
+	st.qr	q0, \ptr, .Lxchal_ofs_+80
+	st.qr	q1, \ptr, .Lxchal_ofs_+96
+	st.qr	q2, \ptr, .Lxchal_ofs_+112
+	addi	\ptr, \ptr, 128
+	st.qr	q3, \ptr, .Lxchal_ofs_+0
+	st.qr	q4, \ptr, .Lxchal_ofs_+16
+	st.qr	q5, \ptr, .Lxchal_ofs_+32
+	st.qr	q6, \ptr, .Lxchal_ofs_+48
+	st.qr	q7, \ptr, .Lxchal_ofs_+64
+	.set	.Lxchal_pofs_, .Lxchal_pofs_ + 128
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 80
+	.elseif ((XTHAL_SAS_TIE | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\alloc)) == 0
+	xchal_sa_align	\ptr, 0, 0, 16, 16
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 208
+	.endif
+    .endm	// xchal_cp3_store
+
+    /*
+     *  Macro to load the state of TIE coprocessor cop_ai.
+     *  Required parameters:
+     *      ptr         Save area pointer address register (clobbered)
+     *                  (register must contain a 16 byte aligned address).
+     *      at1..at4    Four temporary address registers (first XCHAL_CP3_NUM_ATMPS
+     *                  registers are clobbered, the remaining are unused).
+     *  Optional parameters are the same as for xchal_ncp_load.
+     */
+#define xchal_cp_cop_ai_load	xchal_cp3_load
+    .macro	xchal_cp3_load  ptr at1 at2 at3 at4  continue=0 ofs=-1 select=XTHAL_SAS_ALL alloc=0
+	xchal_sa_start \continue, \ofs
+	// Custom caller-saved registers not used by default by the compiler:
+	.ifeq (XTHAL_SAS_TIE | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\select)
+	xchal_sa_align	\ptr, 0, 0, 16, 16
+	l32i	\at1, \ptr, .Lxchal_ofs_+0
+	wur.ACCX_0	\at1		// ureg 0
+	l32i	\at1, \ptr, .Lxchal_ofs_+4
+	wur.ACCX_1	\at1		// ureg 1
+	l32i	\at1, \ptr, .Lxchal_ofs_+8
+	wur.QACC_H_0	\at1		// ureg 2
+	l32i	\at1, \ptr, .Lxchal_ofs_+12
+	wur.QACC_H_1	\at1		// ureg 3
+	l32i	\at1, \ptr, .Lxchal_ofs_+16
+	wur.QACC_H_2	\at1		// ureg 4
+	l32i	\at1, \ptr, .Lxchal_ofs_+20
+	wur.QACC_H_3	\at1		// ureg 5
+	l32i	\at1, \ptr, .Lxchal_ofs_+24
+	wur.QACC_H_4	\at1		// ureg 6
+	l32i	\at1, \ptr, .Lxchal_ofs_+28
+	wur.QACC_L_0	\at1		// ureg 7
+	l32i	\at1, \ptr, .Lxchal_ofs_+32
+	wur.QACC_L_1	\at1		// ureg 8
+	l32i	\at1, \ptr, .Lxchal_ofs_+36
+	wur.QACC_L_2	\at1		// ureg 9
+	l32i	\at1, \ptr, .Lxchal_ofs_+40
+	wur.QACC_L_3	\at1		// ureg 10
+	l32i	\at1, \ptr, .Lxchal_ofs_+44
+	wur.QACC_L_4	\at1		// ureg 11
+	l32i	\at1, \ptr, .Lxchal_ofs_+48
+	wur.SAR_BYTE	\at1		// ureg 13
+	l32i	\at1, \ptr, .Lxchal_ofs_+52
+	wur.FFT_BIT_WIDTH	\at1		// ureg 14
+	l32i	\at1, \ptr, .Lxchal_ofs_+56
+	wur.UA_STATE_0	\at1		// ureg 15
+	l32i	\at1, \ptr, .Lxchal_ofs_+60
+	wur.UA_STATE_1	\at1		// ureg 16
+	l32i	\at1, \ptr, .Lxchal_ofs_+64
+	wur.UA_STATE_2	\at1		// ureg 17
+	l32i	\at1, \ptr, .Lxchal_ofs_+68
+	wur.UA_STATE_3	\at1		// ureg 18
+	ld.qr	q0, \ptr, .Lxchal_ofs_+80
+	ld.qr	q1, \ptr, .Lxchal_ofs_+96
+	ld.qr	q2, \ptr, .Lxchal_ofs_+112
+	addi	\ptr, \ptr, 128
+	ld.qr	q3, \ptr, .Lxchal_ofs_+0
+	ld.qr	q4, \ptr, .Lxchal_ofs_+16
+	ld.qr	q5, \ptr, .Lxchal_ofs_+32
+	ld.qr	q6, \ptr, .Lxchal_ofs_+48
+	ld.qr	q7, \ptr, .Lxchal_ofs_+64
+	.set	.Lxchal_pofs_, .Lxchal_pofs_ + 128
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 80
+	.elseif ((XTHAL_SAS_TIE | XTHAL_SAS_NOCC | XTHAL_SAS_CALR) & ~(\alloc)) == 0
+	xchal_sa_align	\ptr, 0, 0, 16, 16
+	.set	.Lxchal_ofs_, .Lxchal_ofs_ + 208
+	.endif
+    .endm	// xchal_cp3_load
+
+#define XCHAL_CP3_NUM_ATMPS	1
+#define XCHAL_SA_NUM_ATMPS	1
+
+	/*  Empty macros for unconfigured coprocessors:  */
+	.macro xchal_cp1_store	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp1_load	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp2_store	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp2_load	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp4_store	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp4_load	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp5_store	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp5_load	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp6_store	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp6_load	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp7_store	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+	.macro xchal_cp7_load	p a b c d continue=0 ofs=-1 select=-1 ; .endm
+
+#endif /*_XTENSA_CORE_TIE_ASM_H*/
diff --git a/arch/xtensa/variants/esp32s3/include/variant/tie.h b/arch/xtensa/variants/esp32s3/include/variant/tie.h
new file mode 100644
index 000000000000..e7671b1979e4
--- /dev/null
+++ b/arch/xtensa/variants/esp32s3/include/variant/tie.h
@@ -0,0 +1,207 @@
+/*
+ * tie.h -- compile-time HAL definitions dependent on CORE & TIE configuration
+ *
+ *  NOTE:  This header file is not meant to be included directly.
+ */
+
+/* This header file describes this specific Xtensa processor's TIE extensions
+   that extend basic Xtensa core functionality.  It is customized to this
+   Xtensa processor configuration.
+
+   Customer ID=15128; Build=0x90f1f; Copyright (c) 1999-2021 Cadence Design Systems Inc.
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be included
+   in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+   IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+   CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+   TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+   SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.  */
+
+#ifndef _XTENSA_CORE_TIE_H
+#define _XTENSA_CORE_TIE_H
+
+#define XCHAL_CP_NUM			2	/* number of coprocessors */
+#define XCHAL_CP_MAX			4	/* max CP ID + 1 (0 if none) */
+#define XCHAL_CP_MASK			0x09	/* bitmask of all CPs by ID */
+#define XCHAL_CP_PORT_MASK		0x00	/* bitmask of only port CPs */
+
+/*  Basic parameters of each coprocessor:  */
+#define XCHAL_CP0_NAME			"FPU"
+#define XCHAL_CP0_IDENT			FPU
+#define XCHAL_CP0_SA_SIZE		72	/* size of state save area */
+#define XCHAL_CP0_SA_ALIGN		4	/* min alignment of save area */
+#define XCHAL_CP_ID_FPU             	0	/* coprocessor ID (0..7) */
+#define XCHAL_CP3_NAME			"cop_ai"
+#define XCHAL_CP3_IDENT			cop_ai
+#define XCHAL_CP3_SA_SIZE		208	/* size of state save area */
+#define XCHAL_CP3_SA_ALIGN		16	/* min alignment of save area */
+#define XCHAL_CP_ID_COP_AI          	3	/* coprocessor ID (0..7) */
+
+/*  Filler info for unassigned coprocessors, to simplify arrays etc:  */
+#define XCHAL_CP1_SA_SIZE		0
+#define XCHAL_CP1_SA_ALIGN		1
+#define XCHAL_CP2_SA_SIZE		0
+#define XCHAL_CP2_SA_ALIGN		1
+#define XCHAL_CP4_SA_SIZE		0
+#define XCHAL_CP4_SA_ALIGN		1
+#define XCHAL_CP5_SA_SIZE		0
+#define XCHAL_CP5_SA_ALIGN		1
+#define XCHAL_CP6_SA_SIZE		0
+#define XCHAL_CP6_SA_ALIGN		1
+#define XCHAL_CP7_SA_SIZE		0
+#define XCHAL_CP7_SA_ALIGN		1
+
+/*  Save area for non-coprocessor optional and custom (TIE) state:  */
+#define XCHAL_NCP_SA_SIZE		36
+#define XCHAL_NCP_SA_ALIGN		4
+
+/*  Total save area for optional and custom state (NCP + CPn):  */
+#define XCHAL_TOTAL_SA_SIZE		336	/* with 16-byte align padding */
+#define XCHAL_TOTAL_SA_ALIGN		16	/* actual minimum alignment */
+
+/*
+ * Detailed contents of save areas.
+ * NOTE:  caller must define the XCHAL_SA_REG macro (not defined here)
+ * before expanding the XCHAL_xxx_SA_LIST() macros.
+ *
+ * XCHAL_SA_REG(s,ccused,abikind,kind,opt,name,galign,align,asize,
+ *		dbnum,base,regnum,bitsz,gapsz,reset,x...)
+ *
+ *	s = passed from XCHAL_*_LIST(s), eg. to select how to expand
+ *	ccused = set if used by compiler without special options or code
+ *	abikind = 0 (caller-saved), 1 (callee-saved), or 2 (thread-global)
+ *	kind = 0 (special reg), 1 (TIE user reg), or 2 (TIE regfile reg)
+ *	opt = 0 (custom TIE extension or coprocessor), or 1 (optional reg)
+ *	name = lowercase reg name (no quotes)
+ *	galign = group byte alignment (power of 2) (galign >= align)
+ *	align = register byte alignment (power of 2)
+ *	asize = allocated size in bytes (asize*8 == bitsz + gapsz + padsz)
+ *	  (not including any pad bytes required to galign this or next reg)
+ *	dbnum = unique target number f/debug (see <xtensa-libdb-macros.h>)
+ *	base = reg shortname w/o index (or sr=special, ur=TIE user reg)
+ *	regnum = reg index in regfile, or special/TIE-user reg number
+ *	bitsz = number of significant bits (regfile width, or ur/sr mask bits)
+ *	gapsz = intervening bits, if bitsz bits not stored contiguously
+ *	(padsz = pad bits at end [TIE regfile] or at msbits [ur,sr] of asize)
+ *	reset = register reset value (or 0 if undefined at reset)
+ *	x = reserved for future use (0 until then)
+ *
+ *  To filter out certain registers, e.g. to expand only the non-global
+ *  registers used by the compiler, you can do something like this:
+ *
+ *  #define XCHAL_SA_REG(s,ccused,p...)	SELCC##ccused(p)
+ *  #define SELCC0(p...)
+ *  #define SELCC1(abikind,p...)	SELAK##abikind(p)
+ *  #define SELAK0(p...)		REG(p)
+ *  #define SELAK1(p...)		REG(p)
+ *  #define SELAK2(p...)
+ *  #define REG(kind,tie,name,galn,aln,asz,csz,dbnum,base,rnum,bsz,rst,x...) \
+ *		...what you want to expand...
+ */
+
+#define XCHAL_NCP_SA_NUM	9
+#define XCHAL_NCP_SA_LIST(s)	\
+ XCHAL_SA_REG(s,1,2,1,1,      threadptr, 4, 4, 4,0x03E7,  ur,231, 32,0,0,0) \
+ XCHAL_SA_REG(s,1,0,0,1,          acclo, 4, 4, 4,0x0210,  sr,16 , 32,0,0,0) \
+ XCHAL_SA_REG(s,1,0,0,1,          acchi, 4, 4, 4,0x0211,  sr,17 ,  8,0,0,0) \
+ XCHAL_SA_REG(s,0,0,0,1,             br, 4, 4, 4,0x0204,  sr,4  , 16,0,0,0) \
+ XCHAL_SA_REG(s,0,0,0,1,      scompare1, 4, 4, 4,0x020C,  sr,12 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,0,1,             m0, 4, 4, 4,0x0220,  sr,32 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,0,1,             m1, 4, 4, 4,0x0221,  sr,33 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,0,1,             m2, 4, 4, 4,0x0222,  sr,34 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,0,1,             m3, 4, 4, 4,0x0223,  sr,35 , 32,0,0,0)
+
+#define XCHAL_CP0_SA_NUM	18
+#define XCHAL_CP0_SA_LIST(s)	\
+ XCHAL_SA_REG(s,0,0,1,0,            fcr, 4, 4, 4,0x03E8,  ur,232, 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,            fsr, 4, 4, 4,0x03E9,  ur,233, 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             f0, 4, 4, 4,0x0030,   f,0  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             f1, 4, 4, 4,0x0031,   f,1  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             f2, 4, 4, 4,0x0032,   f,2  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             f3, 4, 4, 4,0x0033,   f,3  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             f4, 4, 4, 4,0x0034,   f,4  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             f5, 4, 4, 4,0x0035,   f,5  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             f6, 4, 4, 4,0x0036,   f,6  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             f7, 4, 4, 4,0x0037,   f,7  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             f8, 4, 4, 4,0x0038,   f,8  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             f9, 4, 4, 4,0x0039,   f,9  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,            f10, 4, 4, 4,0x003A,   f,10 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,            f11, 4, 4, 4,0x003B,   f,11 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,            f12, 4, 4, 4,0x003C,   f,12 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,            f13, 4, 4, 4,0x003D,   f,13 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,            f14, 4, 4, 4,0x003E,   f,14 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,            f15, 4, 4, 4,0x003F,   f,15 , 32,0,0,0)
+
+#define XCHAL_CP1_SA_NUM	0
+#define XCHAL_CP1_SA_LIST(s)	/* empty */
+
+#define XCHAL_CP2_SA_NUM	0
+#define XCHAL_CP2_SA_LIST(s)	/* empty */
+
+#define XCHAL_CP3_SA_NUM	26
+#define XCHAL_CP3_SA_LIST(s)	\
+ XCHAL_SA_REG(s,0,0,1,0,         accx_0,16, 4, 4,0x0300,  ur,0  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,         accx_1, 4, 4, 4,0x0301,  ur,1  ,  8,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,       qacc_h_0, 4, 4, 4,0x0302,  ur,2  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,       qacc_h_1, 4, 4, 4,0x0303,  ur,3  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,       qacc_h_2, 4, 4, 4,0x0304,  ur,4  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,       qacc_h_3, 4, 4, 4,0x0305,  ur,5  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,       qacc_h_4, 4, 4, 4,0x0306,  ur,6  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,       qacc_l_0, 4, 4, 4,0x0307,  ur,7  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,       qacc_l_1, 4, 4, 4,0x0308,  ur,8  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,       qacc_l_2, 4, 4, 4,0x0309,  ur,9  , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,       qacc_l_3, 4, 4, 4,0x030A,  ur,10 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,       qacc_l_4, 4, 4, 4,0x030B,  ur,11 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,       sar_byte, 4, 4, 4,0x030D,  ur,13 ,  4,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,  fft_bit_width, 4, 4, 4,0x030E,  ur,14 ,  4,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,     ua_state_0, 4, 4, 4,0x030F,  ur,15 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,     ua_state_1, 4, 4, 4,0x0310,  ur,16 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,     ua_state_2, 4, 4, 4,0x0311,  ur,17 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,1,0,     ua_state_3, 4, 4, 4,0x0312,  ur,18 , 32,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             q0,16,16,16,0x1008,   q,0  ,128,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             q1,16,16,16,0x1009,   q,1  ,128,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             q2,16,16,16,0x100A,   q,2  ,128,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             q3,16,16,16,0x100B,   q,3  ,128,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             q4,16,16,16,0x100C,   q,4  ,128,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             q5,16,16,16,0x100D,   q,5  ,128,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             q6,16,16,16,0x100E,   q,6  ,128,0,0,0) \
+ XCHAL_SA_REG(s,0,0,2,0,             q7,16,16,16,0x100F,   q,7  ,128,0,0,0)
+
+#define XCHAL_CP4_SA_NUM	0
+#define XCHAL_CP4_SA_LIST(s)	/* empty */
+
+#define XCHAL_CP5_SA_NUM	0
+#define XCHAL_CP5_SA_LIST(s)	/* empty */
+
+#define XCHAL_CP6_SA_NUM	0
+#define XCHAL_CP6_SA_LIST(s)	/* empty */
+
+#define XCHAL_CP7_SA_NUM	0
+#define XCHAL_CP7_SA_LIST(s)	/* empty */
+
+/* Byte length of instruction from its first nibble (op0 field), per FLIX.  */
+#define XCHAL_OP0_FORMAT_LENGTHS	3,3,3,3,3,3,3,3,2,2,2,2,2,2,4,4
+/* Byte length of instruction from its first byte, per FLIX.  */
+#define XCHAL_BYTE0_FORMAT_LENGTHS	\
+	3,3,3,3,3,3,3,3,2,2,2,2,2,2,4,4, 3,3,3,3,3,3,3,3,2,2,2,2,2,2,4,4,\
+	3,3,3,3,3,3,3,3,2,2,2,2,2,2,4,4, 3,3,3,3,3,3,3,3,2,2,2,2,2,2,4,4,\
+	3,3,3,3,3,3,3,3,2,2,2,2,2,2,4,4, 3,3,3,3,3,3,3,3,2,2,2,2,2,2,4,4,\
+	3,3,3,3,3,3,3,3,2,2,2,2,2,2,4,4, 3,3,3,3,3,3,3,3,2,2,2,2,2,2,4,4,\
+	3,3,3,3,3,3,3,3,2,2,2,2,2,2,4,4, 3,3,3,3,3,3,3,3,2,2,2,2,2,2,4,4,\
+	3,3,3,3,3,3,3,3,2,2,2,2,2,2,4,4, 3,3,3,3,3,3,3,3,2,2,2,2,2,2,4,4,\
+	3,3,3,3,3,3,3,3,2,2,2,2,2,2,4,4, 3,3,3,3,3,3,3,3,2,2,2,2,2,2,4,4,\
+	3,3,3,3,3,3,3,3,2,2,2,2,2,2,4,4, 3,3,3,3,3,3,3,3,2,2,2,2,2,2,4,4
+
+#endif /*_XTENSA_CORE_TIE_H*/
diff --git a/block/Kconfig b/block/Kconfig
index 55ae2286a4de..1de4682d48cc 100644
--- a/block/Kconfig
+++ b/block/Kconfig
@@ -78,6 +78,26 @@ config BLK_DEV_INTEGRITY_T10
 	select CRC_T10DIF
 	select CRC64_ROCKSOFT
 
+config BLK_DEV_WRITE_MOUNTED
+	bool "Allow writing to mounted block devices"
+	default y
+	help
+	When a block device is mounted, writing to its buffer cache is very
+	likely going to cause filesystem corruption. It is also rather easy to
+	crash the kernel in this way since the filesystem has no practical way
+	of detecting these writes to buffer cache and verifying its metadata
+	integrity. However there are some setups that need this capability
+	like running fsck on read-only mounted root device, modifying some
+	features on mounted ext4 filesystem, and similar. If you say N, the
+	kernel will prevent processes from writing to block devices that are
+	mounted by filesystems which provides some more protection from runaway
+	privileged processes and generally makes it much harder to crash
+	filesystem drivers. Note however that this does not prevent
+	underlying device(s) from being modified by other means, e.g. by
+	directly submitting SCSI commands or through access to lower layers of
+	storage stack. If in doubt, say Y. The configuration can be overridden
+	with the bdev_allow_write_mounted boot option.
+
 config BLK_DEV_ZONED
 	bool "Zoned block device support"
 	select MQ_IOSCHED_DEADLINE
diff --git a/block/bdev.c b/block/bdev.c
index e4cfb7adb645..6f73b02d549c 100644
--- a/block/bdev.c
+++ b/block/bdev.c
@@ -30,6 +30,9 @@
 #include "../fs/internal.h"
 #include "blk.h"
 
+/* Should we allow writing to mounted block devices? */
+static bool bdev_allow_write_mounted = IS_ENABLED(CONFIG_BLK_DEV_WRITE_MOUNTED);
+
 struct bdev_inode {
 	struct block_device bdev;
 	struct inode vfs_inode;
@@ -207,85 +210,88 @@ int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend)
 EXPORT_SYMBOL(sync_blockdev_range);
 
 /**
- * freeze_bdev - lock a filesystem and force it into a consistent state
+ * bdev_freeze - lock a filesystem and force it into a consistent state
  * @bdev:	blockdevice to lock
  *
  * If a superblock is found on this device, we take the s_umount semaphore
  * on it to make sure nobody unmounts until the snapshot creation is done.
  * The reference counter (bd_fsfreeze_count) guarantees that only the last
  * unfreeze process can unfreeze the frozen filesystem actually when multiple
- * freeze requests arrive simultaneously. It counts up in freeze_bdev() and
- * count down in thaw_bdev(). When it becomes 0, thaw_bdev() will unfreeze
+ * freeze requests arrive simultaneously. It counts up in bdev_freeze() and
+ * count down in bdev_thaw(). When it becomes 0, thaw_bdev() will unfreeze
  * actually.
+ *
+ * Return: On success zero is returned, negative error code on failure.
  */
-int freeze_bdev(struct block_device *bdev)
+int bdev_freeze(struct block_device *bdev)
 {
-	struct super_block *sb;
 	int error = 0;
 
 	mutex_lock(&bdev->bd_fsfreeze_mutex);
-	if (++bdev->bd_fsfreeze_count > 1)
-		goto done;
-
-	sb = get_active_super(bdev);
-	if (!sb)
-		goto sync;
-	if (sb->s_op->freeze_super)
-		error = sb->s_op->freeze_super(sb, FREEZE_HOLDER_USERSPACE);
-	else
-		error = freeze_super(sb, FREEZE_HOLDER_USERSPACE);
-	deactivate_super(sb);
 
-	if (error) {
-		bdev->bd_fsfreeze_count--;
-		goto done;
+	if (atomic_inc_return(&bdev->bd_fsfreeze_count) > 1) {
+		mutex_unlock(&bdev->bd_fsfreeze_mutex);
+		return 0;
 	}
-	bdev->bd_fsfreeze_sb = sb;
 
-sync:
-	sync_blockdev(bdev);
-done:
+	mutex_lock(&bdev->bd_holder_lock);
+	if (bdev->bd_holder_ops && bdev->bd_holder_ops->freeze) {
+		error = bdev->bd_holder_ops->freeze(bdev);
+		lockdep_assert_not_held(&bdev->bd_holder_lock);
+	} else {
+		mutex_unlock(&bdev->bd_holder_lock);
+		error = sync_blockdev(bdev);
+	}
+
+	if (error)
+		atomic_dec(&bdev->bd_fsfreeze_count);
+
 	mutex_unlock(&bdev->bd_fsfreeze_mutex);
 	return error;
 }
-EXPORT_SYMBOL(freeze_bdev);
+EXPORT_SYMBOL(bdev_freeze);
 
 /**
- * thaw_bdev - unlock filesystem
+ * bdev_thaw - unlock filesystem
  * @bdev:	blockdevice to unlock
  *
- * Unlocks the filesystem and marks it writeable again after freeze_bdev().
+ * Unlocks the filesystem and marks it writeable again after bdev_freeze().
+ *
+ * Return: On success zero is returned, negative error code on failure.
  */
-int thaw_bdev(struct block_device *bdev)
+int bdev_thaw(struct block_device *bdev)
 {
-	struct super_block *sb;
-	int error = -EINVAL;
+	int error = -EINVAL, nr_freeze;
 
 	mutex_lock(&bdev->bd_fsfreeze_mutex);
-	if (!bdev->bd_fsfreeze_count)
+
+	/*
+	 * If this returns < 0 it means that @bd_fsfreeze_count was
+	 * already 0 and no decrement was performed.
+	 */
+	nr_freeze = atomic_dec_if_positive(&bdev->bd_fsfreeze_count);
+	if (nr_freeze < 0)
 		goto out;
 
 	error = 0;
-	if (--bdev->bd_fsfreeze_count > 0)
+	if (nr_freeze > 0)
 		goto out;
 
-	sb = bdev->bd_fsfreeze_sb;
-	if (!sb)
-		goto out;
+	mutex_lock(&bdev->bd_holder_lock);
+	if (bdev->bd_holder_ops && bdev->bd_holder_ops->thaw) {
+		error = bdev->bd_holder_ops->thaw(bdev);
+		lockdep_assert_not_held(&bdev->bd_holder_lock);
+	} else {
+		mutex_unlock(&bdev->bd_holder_lock);
+	}
 
-	if (sb->s_op->thaw_super)
-		error = sb->s_op->thaw_super(sb, FREEZE_HOLDER_USERSPACE);
-	else
-		error = thaw_super(sb, FREEZE_HOLDER_USERSPACE);
 	if (error)
-		bdev->bd_fsfreeze_count++;
-	else
-		bdev->bd_fsfreeze_sb = NULL;
+		atomic_inc(&bdev->bd_fsfreeze_count);
 out:
 	mutex_unlock(&bdev->bd_fsfreeze_mutex);
 	return error;
 }
-EXPORT_SYMBOL(thaw_bdev);
+EXPORT_SYMBOL(bdev_thaw);
 
 /*
  * pseudo-fs
@@ -425,6 +431,8 @@ void bdev_set_nr_sectors(struct block_device *bdev, sector_t sectors)
 
 void bdev_add(struct block_device *bdev, dev_t dev)
 {
+	if (bdev_stable_writes(bdev))
+		mapping_set_stable_writes(bdev->bd_inode->i_mapping);
 	bdev->bd_dev = dev;
 	bdev->bd_inode->i_rdev = dev;
 	bdev->bd_inode->i_ino = dev;
@@ -727,9 +735,60 @@ void blkdev_put_no_open(struct block_device *bdev)
 {
 	put_device(&bdev->bd_device);
 }
-	
+
+static bool bdev_writes_blocked(struct block_device *bdev)
+{
+	return bdev->bd_writers == -1;
+}
+
+static void bdev_block_writes(struct block_device *bdev)
+{
+	bdev->bd_writers = -1;
+}
+
+static void bdev_unblock_writes(struct block_device *bdev)
+{
+	bdev->bd_writers = 0;
+}
+
+static bool bdev_may_open(struct block_device *bdev, blk_mode_t mode)
+{
+	if (bdev_allow_write_mounted)
+		return true;
+	/* Writes blocked? */
+	if (mode & BLK_OPEN_WRITE && bdev_writes_blocked(bdev))
+		return false;
+	if (mode & BLK_OPEN_RESTRICT_WRITES && bdev->bd_writers > 0)
+		return false;
+	return true;
+}
+
+static void bdev_claim_write_access(struct block_device *bdev, blk_mode_t mode)
+{
+	if (bdev_allow_write_mounted)
+		return;
+
+	/* Claim exclusive or shared write access. */
+	if (mode & BLK_OPEN_RESTRICT_WRITES)
+		bdev_block_writes(bdev);
+	else if (mode & BLK_OPEN_WRITE)
+		bdev->bd_writers++;
+}
+
+static void bdev_yield_write_access(struct block_device *bdev, blk_mode_t mode)
+{
+	if (bdev_allow_write_mounted)
+		return;
+
+	/* Yield exclusive or shared write access. */
+	if (mode & BLK_OPEN_RESTRICT_WRITES)
+		bdev_unblock_writes(bdev);
+	else if (mode & BLK_OPEN_WRITE)
+		bdev->bd_writers--;
+}
+
 /**
- * blkdev_get_by_dev - open a block device by device number
+ * bdev_open_by_dev - open a block device by device number
  * @dev: device number of block device to open
  * @mode: open mode (BLK_OPEN_*)
  * @holder: exclusive holder identifier
@@ -741,32 +800,44 @@ void blkdev_put_no_open(struct block_device *bdev)
  *
  * Use this interface ONLY if you really do not have anything better - i.e. when
  * you are behind a truly sucky interface and all you are given is a device
- * number.  Everything else should use blkdev_get_by_path().
+ * number.  Everything else should use bdev_open_by_path().
  *
  * CONTEXT:
  * Might sleep.
  *
  * RETURNS:
- * Reference to the block_device on success, ERR_PTR(-errno) on failure.
+ * Handle with a reference to the block_device on success, ERR_PTR(-errno) on
+ * failure.
  */
-struct block_device *blkdev_get_by_dev(dev_t dev, blk_mode_t mode, void *holder,
-		const struct blk_holder_ops *hops)
+struct bdev_handle *bdev_open_by_dev(dev_t dev, blk_mode_t mode, void *holder,
+				     const struct blk_holder_ops *hops)
 {
-	bool unblock_events = true;
+	struct bdev_handle *handle = kmalloc(sizeof(struct bdev_handle),
+					     GFP_KERNEL);
 	struct block_device *bdev;
+	bool unblock_events = true;
 	struct gendisk *disk;
 	int ret;
 
+	if (!handle)
+		return ERR_PTR(-ENOMEM);
+
 	ret = devcgroup_check_permission(DEVCG_DEV_BLOCK,
 			MAJOR(dev), MINOR(dev),
 			((mode & BLK_OPEN_READ) ? DEVCG_ACC_READ : 0) |
 			((mode & BLK_OPEN_WRITE) ? DEVCG_ACC_WRITE : 0));
 	if (ret)
-		return ERR_PTR(ret);
+		goto free_handle;
+
+	/* Blocking writes requires exclusive opener */
+	if (mode & BLK_OPEN_RESTRICT_WRITES && !holder)
+		return ERR_PTR(-EINVAL);
 
 	bdev = blkdev_get_no_open(dev);
-	if (!bdev)
-		return ERR_PTR(-ENXIO);
+	if (!bdev) {
+		ret = -ENXIO;
+		goto free_handle;
+	}
 	disk = bdev->bd_disk;
 
 	if (holder) {
@@ -789,12 +860,16 @@ struct block_device *blkdev_get_by_dev(dev_t dev, blk_mode_t mode, void *holder,
 		goto abort_claiming;
 	if (!try_module_get(disk->fops->owner))
 		goto abort_claiming;
+	ret = -EBUSY;
+	if (!bdev_may_open(bdev, mode))
+		goto abort_claiming;
 	if (bdev_is_partition(bdev))
 		ret = blkdev_get_part(bdev, mode);
 	else
 		ret = blkdev_get_whole(bdev, mode);
 	if (ret)
 		goto put_module;
+	bdev_claim_write_access(bdev, mode);
 	if (holder) {
 		bd_finish_claiming(bdev, holder, hops);
 
@@ -815,7 +890,10 @@ struct block_device *blkdev_get_by_dev(dev_t dev, blk_mode_t mode, void *holder,
 
 	if (unblock_events)
 		disk_unblock_events(disk);
-	return bdev;
+	handle->bdev = bdev;
+	handle->holder = holder;
+	handle->mode = mode;
+	return handle;
 put_module:
 	module_put(disk->fops->owner);
 abort_claiming:
@@ -825,34 +903,14 @@ struct block_device *blkdev_get_by_dev(dev_t dev, blk_mode_t mode, void *holder,
 	disk_unblock_events(disk);
 put_blkdev:
 	blkdev_put_no_open(bdev);
+free_handle:
+	kfree(handle);
 	return ERR_PTR(ret);
 }
-EXPORT_SYMBOL(blkdev_get_by_dev);
-
-struct bdev_handle *bdev_open_by_dev(dev_t dev, blk_mode_t mode, void *holder,
-				     const struct blk_holder_ops *hops)
-{
-	struct bdev_handle *handle = kmalloc(sizeof(*handle), GFP_KERNEL);
-	struct block_device *bdev;
-
-	if (!handle)
-		return ERR_PTR(-ENOMEM);
-	bdev = blkdev_get_by_dev(dev, mode, holder, hops);
-	if (IS_ERR(bdev)) {
-		kfree(handle);
-		return ERR_CAST(bdev);
-	}
-	handle->bdev = bdev;
-	handle->holder = holder;
-	if (holder)
-		mode |= BLK_OPEN_EXCL;
-	handle->mode = mode;
-	return handle;
-}
 EXPORT_SYMBOL(bdev_open_by_dev);
 
 /**
- * blkdev_get_by_path - open a block device by name
+ * bdev_open_by_path - open a block device by name
  * @path: path to the block device to open
  * @mode: open mode (BLK_OPEN_*)
  * @holder: exclusive holder identifier
@@ -866,29 +924,9 @@ EXPORT_SYMBOL(bdev_open_by_dev);
  * Might sleep.
  *
  * RETURNS:
- * Reference to the block_device on success, ERR_PTR(-errno) on failure.
+ * Handle with a reference to the block_device on success, ERR_PTR(-errno) on
+ * failure.
  */
-struct block_device *blkdev_get_by_path(const char *path, blk_mode_t mode,
-		void *holder, const struct blk_holder_ops *hops)
-{
-	struct block_device *bdev;
-	dev_t dev;
-	int error;
-
-	error = lookup_bdev(path, &dev);
-	if (error)
-		return ERR_PTR(error);
-
-	bdev = blkdev_get_by_dev(dev, mode, holder, hops);
-	if (!IS_ERR(bdev) && (mode & BLK_OPEN_WRITE) && bdev_read_only(bdev)) {
-		blkdev_put(bdev, holder);
-		return ERR_PTR(-EACCES);
-	}
-
-	return bdev;
-}
-EXPORT_SYMBOL(blkdev_get_by_path);
-
 struct bdev_handle *bdev_open_by_path(const char *path, blk_mode_t mode,
 		void *holder, const struct blk_holder_ops *hops)
 {
@@ -911,8 +949,9 @@ struct bdev_handle *bdev_open_by_path(const char *path, blk_mode_t mode,
 }
 EXPORT_SYMBOL(bdev_open_by_path);
 
-void blkdev_put(struct block_device *bdev, void *holder)
+void bdev_release(struct bdev_handle *handle)
 {
+	struct block_device *bdev = handle->bdev;
 	struct gendisk *disk = bdev->bd_disk;
 
 	/*
@@ -926,8 +965,10 @@ void blkdev_put(struct block_device *bdev, void *holder)
 		sync_blockdev(bdev);
 
 	mutex_lock(&disk->open_mutex);
-	if (holder)
-		bd_end_claim(bdev, holder);
+	bdev_yield_write_access(bdev, handle->mode);
+
+	if (handle->holder)
+		bd_end_claim(bdev, handle->holder);
 
 	/*
 	 * Trigger event checking and tell drivers to flush MEDIA_CHANGE
@@ -944,12 +985,6 @@ void blkdev_put(struct block_device *bdev, void *holder)
 
 	module_put(disk->fops->owner);
 	blkdev_put_no_open(bdev);
-}
-EXPORT_SYMBOL(blkdev_put);
-
-void bdev_release(struct bdev_handle *handle)
-{
-	blkdev_put(handle->bdev, handle->holder);
 	kfree(handle);
 }
 EXPORT_SYMBOL(bdev_release);
@@ -1100,3 +1135,12 @@ void bdev_statx_dioalign(struct inode *inode, struct kstat *stat)
 
 	blkdev_put_no_open(bdev);
 }
+
+static int __init setup_bdev_allow_write_mounted(char *str)
+{
+	if (kstrtobool(str, &bdev_allow_write_mounted))
+		pr_warn("Invalid option string for bdev_allow_write_mounted:"
+			" '%s'\n", str);
+	return 1;
+}
+__setup("bdev_allow_write_mounted=", setup_bdev_allow_write_mounted);
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index 4a42ea2972ad..4b48c2c44098 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -577,6 +577,7 @@ static void blkg_destroy_all(struct gendisk *disk)
 	struct request_queue *q = disk->queue;
 	struct blkcg_gq *blkg, *n;
 	int count = BLKG_DESTROY_BATCH_SIZE;
+	int i;
 
 restart:
 	spin_lock_irq(&q->queue_lock);
@@ -602,6 +603,18 @@ static void blkg_destroy_all(struct gendisk *disk)
 		}
 	}
 
+	/*
+	 * Mark policy deactivated since policy offline has been done, and
+	 * the free is scheduled, so future blkcg_deactivate_policy() can
+	 * be bypassed
+	 */
+	for (i = 0; i < BLKCG_MAX_POLS; i++) {
+		struct blkcg_policy *pol = blkcg_policy[i];
+
+		if (pol)
+			__clear_bit(pol->plid, q->blkcg_pols);
+	}
+
 	q->root_blkg = NULL;
 	spin_unlock_irq(&q->queue_lock);
 }
diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h
index 624c03c8fe64..fd482439afbc 100644
--- a/block/blk-cgroup.h
+++ b/block/blk-cgroup.h
@@ -249,8 +249,6 @@ static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg,
 {
 	struct blkcg_gq *blkg;
 
-	WARN_ON_ONCE(!rcu_read_lock_held());
-
 	if (blkcg == &blkcg_root)
 		return q->root_blkg;
 
diff --git a/block/blk-pm.c b/block/blk-pm.c
index 6b72b2e03fc8..42e842074715 100644
--- a/block/blk-pm.c
+++ b/block/blk-pm.c
@@ -163,38 +163,15 @@ EXPORT_SYMBOL(blk_pre_runtime_resume);
  * @q: the queue of the device
  *
  * Description:
- *    For historical reasons, this routine merely calls blk_set_runtime_active()
- *    to do the real work of restarting the queue.  It does this regardless of
- *    whether the device's runtime-resume succeeded; even if it failed the
+ *    Restart the queue of a runtime suspended device. It does this regardless
+ *    of whether the device's runtime-resume succeeded; even if it failed the
  *    driver or error handler will need to communicate with the device.
  *
  *    This function should be called near the end of the device's
- *    runtime_resume callback.
+ *    runtime_resume callback to correct queue runtime PM status and re-enable
+ *    peeking requests from the queue.
  */
 void blk_post_runtime_resume(struct request_queue *q)
-{
-	blk_set_runtime_active(q);
-}
-EXPORT_SYMBOL(blk_post_runtime_resume);
-
-/**
- * blk_set_runtime_active - Force runtime status of the queue to be active
- * @q: the queue of the device
- *
- * If the device is left runtime suspended during system suspend the resume
- * hook typically resumes the device and corrects runtime status
- * accordingly. However, that does not affect the queue runtime PM status
- * which is still "suspended". This prevents processing requests from the
- * queue.
- *
- * This function can be used in driver's resume hook to correct queue
- * runtime PM status and re-enable peeking requests from the queue. It
- * should be called before first request is added to the queue.
- *
- * This function is also called by blk_post_runtime_resume() for
- * runtime resumes.  It does everything necessary to restart the queue.
- */
-void blk_set_runtime_active(struct request_queue *q)
 {
 	int old_status;
 
@@ -211,4 +188,4 @@ void blk_set_runtime_active(struct request_queue *q)
 	if (old_status != RPM_ACTIVE)
 		blk_clear_pm_only(q);
 }
-EXPORT_SYMBOL(blk_set_runtime_active);
+EXPORT_SYMBOL(blk_post_runtime_resume);
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index 13e4377a8b28..16f5766620a4 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -1320,6 +1320,7 @@ static void tg_conf_updated(struct throtl_grp *tg, bool global)
 		   tg_bps_limit(tg, READ), tg_bps_limit(tg, WRITE),
 		   tg_iops_limit(tg, READ), tg_iops_limit(tg, WRITE));
 
+	rcu_read_lock();
 	/*
 	 * Update has_rules[] flags for the updated tg's subtree.  A tg is
 	 * considered to have rules if either the tg itself or any of its
@@ -1347,6 +1348,7 @@ static void tg_conf_updated(struct throtl_grp *tg, bool global)
 		this_tg->latency_target = max(this_tg->latency_target,
 				parent_tg->latency_target);
 	}
+	rcu_read_unlock();
 
 	/*
 	 * We're already holding queue_lock and know @tg is valid.  Let's
diff --git a/block/fops.c b/block/fops.c
index 0abaac705daf..0bdad1e8d514 100644
--- a/block/fops.c
+++ b/block/fops.c
@@ -500,7 +500,7 @@ const struct address_space_operations def_blk_aops = {
 	.readahead		= blkdev_readahead,
 	.writepages		= blkdev_writepages,
 	.is_partially_uptodate  = iomap_is_partially_uptodate,
-	.error_remove_page	= generic_error_remove_page,
+	.error_remove_folio	= generic_error_remove_folio,
 	.migrate_folio		= filemap_migrate_folio,
 };
 #endif /* CONFIG_BUFFER_HEAD */
diff --git a/crypto/drbg.c b/crypto/drbg.c
index e01f8c7769d0..3addce90930c 100644
--- a/crypto/drbg.c
+++ b/crypto/drbg.c
@@ -111,9 +111,9 @@
  * as stdrng. Each DRBG receives an increasing cra_priority values the later
  * they are defined in this array (see drbg_fill_array).
  *
- * HMAC DRBGs are favored over Hash DRBGs over CTR DRBGs, and
- * the SHA256 / AES 256 over other ciphers. Thus, the favored
- * DRBGs are the latest entries in this array.
+ * HMAC DRBGs are favored over Hash DRBGs over CTR DRBGs, and the
+ * HMAC-SHA512 / SHA256 / AES 256 over other ciphers. Thus, the
+ * favored DRBGs are the latest entries in this array.
  */
 static const struct drbg_core drbg_cores[] = {
 #ifdef CONFIG_CRYPTO_DRBG_CTR
@@ -139,12 +139,6 @@ static const struct drbg_core drbg_cores[] = {
 #endif /* CONFIG_CRYPTO_DRBG_CTR */
 #ifdef CONFIG_CRYPTO_DRBG_HASH
 	{
-		.flags = DRBG_HASH | DRBG_STRENGTH128,
-		.statelen = 55, /* 440 bits */
-		.blocklen_bytes = 20,
-		.cra_name = "sha1",
-		.backend_cra_name = "sha1",
-	}, {
 		.flags = DRBG_HASH | DRBG_STRENGTH256,
 		.statelen = 111, /* 888 bits */
 		.blocklen_bytes = 48,
@@ -166,12 +160,6 @@ static const struct drbg_core drbg_cores[] = {
 #endif /* CONFIG_CRYPTO_DRBG_HASH */
 #ifdef CONFIG_CRYPTO_DRBG_HMAC
 	{
-		.flags = DRBG_HMAC | DRBG_STRENGTH128,
-		.statelen = 20, /* block length of cipher */
-		.blocklen_bytes = 20,
-		.cra_name = "hmac_sha1",
-		.backend_cra_name = "hmac(sha1)",
-	}, {
 		.flags = DRBG_HMAC | DRBG_STRENGTH256,
 		.statelen = 48, /* block length of cipher */
 		.blocklen_bytes = 48,
@@ -648,8 +636,6 @@ MODULE_ALIAS_CRYPTO("drbg_pr_hmac_sha384");
 MODULE_ALIAS_CRYPTO("drbg_nopr_hmac_sha384");
 MODULE_ALIAS_CRYPTO("drbg_pr_hmac_sha256");
 MODULE_ALIAS_CRYPTO("drbg_nopr_hmac_sha256");
-MODULE_ALIAS_CRYPTO("drbg_pr_hmac_sha1");
-MODULE_ALIAS_CRYPTO("drbg_nopr_hmac_sha1");
 
 /* update function of HMAC DRBG as defined in 10.1.2.2 */
 static int drbg_hmac_update(struct drbg_state *drbg, struct list_head *seed,
@@ -768,8 +754,6 @@ MODULE_ALIAS_CRYPTO("drbg_pr_sha384");
 MODULE_ALIAS_CRYPTO("drbg_nopr_sha384");
 MODULE_ALIAS_CRYPTO("drbg_pr_sha256");
 MODULE_ALIAS_CRYPTO("drbg_nopr_sha256");
-MODULE_ALIAS_CRYPTO("drbg_pr_sha1");
-MODULE_ALIAS_CRYPTO("drbg_nopr_sha1");
 
 /*
  * Increment buffer
@@ -1475,11 +1459,11 @@ static int drbg_generate(struct drbg_state *drbg,
 		int err = 0;
 		pr_devel("DRBG: start to perform self test\n");
 		if (drbg->core->flags & DRBG_HMAC)
-			err = alg_test("drbg_pr_hmac_sha256",
-				       "drbg_pr_hmac_sha256", 0, 0);
+			err = alg_test("drbg_pr_hmac_sha512",
+				       "drbg_pr_hmac_sha512", 0, 0);
 		else if (drbg->core->flags & DRBG_CTR)
-			err = alg_test("drbg_pr_ctr_aes128",
-				       "drbg_pr_ctr_aes128", 0, 0);
+			err = alg_test("drbg_pr_ctr_aes256",
+				       "drbg_pr_ctr_aes256", 0, 0);
 		else
 			err = alg_test("drbg_pr_sha256",
 				       "drbg_pr_sha256", 0, 0);
@@ -2017,11 +2001,13 @@ static inline int __init drbg_healthcheck_sanity(void)
 		return 0;
 
 #ifdef CONFIG_CRYPTO_DRBG_CTR
-	drbg_convert_tfm_core("drbg_nopr_ctr_aes128", &coreref, &pr);
-#elif defined CONFIG_CRYPTO_DRBG_HASH
+	drbg_convert_tfm_core("drbg_nopr_ctr_aes256", &coreref, &pr);
+#endif
+#ifdef CONFIG_CRYPTO_DRBG_HASH
 	drbg_convert_tfm_core("drbg_nopr_sha256", &coreref, &pr);
-#else
-	drbg_convert_tfm_core("drbg_nopr_hmac_sha256", &coreref, &pr);
+#endif
+#ifdef CONFIG_CRYPTO_DRBG_HMAC
+	drbg_convert_tfm_core("drbg_nopr_hmac_sha512", &coreref, &pr);
 #endif
 
 	drbg = kzalloc(sizeof(struct drbg_state), GFP_KERNEL);
diff --git a/crypto/rsa.c b/crypto/rsa.c
index c79613cdce6e..b9cd11fb7d36 100644
--- a/crypto/rsa.c
+++ b/crypto/rsa.c
@@ -220,6 +220,8 @@ static int rsa_check_exponent_fips(MPI e)
 	}
 
 	e_max = mpi_alloc(0);
+	if (!e_max)
+		return -ENOMEM;
 	mpi_set_bit(e_max, 256);
 
 	if (mpi_cmp(e, e_max) >= 0) {
diff --git a/crypto/shash.c b/crypto/shash.c
index d5194221c88c..c3f7f6a25280 100644
--- a/crypto/shash.c
+++ b/crypto/shash.c
@@ -23,12 +23,8 @@ static inline struct crypto_istat_hash *shash_get_stat(struct shash_alg *alg)
 
 static inline int crypto_shash_errstat(struct shash_alg *alg, int err)
 {
-	if (!IS_ENABLED(CONFIG_CRYPTO_STATS))
-		return err;
-
-	if (err && err != -EINPROGRESS && err != -EBUSY)
+	if (IS_ENABLED(CONFIG_CRYPTO_STATS) && err)
 		atomic64_inc(&shash_get_stat(alg)->err_cnt);
-
 	return err;
 }
 
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index 15c7a3011269..59f0540d442e 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -4845,14 +4845,6 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.suite = {
 			.drbg = __VECS(drbg_nopr_ctr_aes256_tv_template)
 		}
-	}, {
-		/*
-		 * There is no need to specifically test the DRBG with every
-		 * backend cipher -- covered by drbg_nopr_hmac_sha256 test
-		 */
-		.alg = "drbg_nopr_hmac_sha1",
-		.fips_allowed = 1,
-		.test = alg_test_null,
 	}, {
 		.alg = "drbg_nopr_hmac_sha256",
 		.test = alg_test_drbg,
@@ -4861,7 +4853,10 @@ static const struct alg_test_desc alg_test_descs[] = {
 			.drbg = __VECS(drbg_nopr_hmac_sha256_tv_template)
 		}
 	}, {
-		/* covered by drbg_nopr_hmac_sha256 test */
+		/*
+		 * There is no need to specifically test the DRBG with every
+		 * backend cipher -- covered by drbg_nopr_hmac_sha512 test
+		 */
 		.alg = "drbg_nopr_hmac_sha384",
 		.test = alg_test_null,
 	}, {
@@ -4871,10 +4866,6 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.suite = {
 			.drbg = __VECS(drbg_nopr_hmac_sha512_tv_template)
 		}
-	}, {
-		.alg = "drbg_nopr_sha1",
-		.fips_allowed = 1,
-		.test = alg_test_null,
 	}, {
 		.alg = "drbg_nopr_sha256",
 		.test = alg_test_drbg,
@@ -4906,10 +4897,6 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.alg = "drbg_pr_ctr_aes256",
 		.fips_allowed = 1,
 		.test = alg_test_null,
-	}, {
-		.alg = "drbg_pr_hmac_sha1",
-		.fips_allowed = 1,
-		.test = alg_test_null,
 	}, {
 		.alg = "drbg_pr_hmac_sha256",
 		.test = alg_test_drbg,
@@ -4925,10 +4912,6 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.alg = "drbg_pr_hmac_sha512",
 		.test = alg_test_null,
 		.fips_allowed = 1,
-	}, {
-		.alg = "drbg_pr_sha1",
-		.fips_allowed = 1,
-		.test = alg_test_null,
 	}, {
 		.alg = "drbg_pr_sha256",
 		.test = alg_test_drbg,
diff --git a/drivers/accel/habanalabs/common/device.c b/drivers/accel/habanalabs/common/device.c
index 9711e8fc979d..a365791a9f5c 100644
--- a/drivers/accel/habanalabs/common/device.c
+++ b/drivers/accel/habanalabs/common/device.c
@@ -853,6 +853,9 @@ static int device_early_init(struct hl_device *hdev)
 		gaudi2_set_asic_funcs(hdev);
 		strscpy(hdev->asic_name, "GAUDI2B", sizeof(hdev->asic_name));
 		break;
+	case ASIC_GAUDI2C:
+		gaudi2_set_asic_funcs(hdev);
+		strscpy(hdev->asic_name, "GAUDI2C", sizeof(hdev->asic_name));
 		break;
 	default:
 		dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
@@ -1041,18 +1044,21 @@ static bool is_pci_link_healthy(struct hl_device *hdev)
 	return (vendor_id == PCI_VENDOR_ID_HABANALABS);
 }
 
-static void hl_device_eq_heartbeat(struct hl_device *hdev)
+static int hl_device_eq_heartbeat_check(struct hl_device *hdev)
 {
-	u64 event_mask = HL_NOTIFIER_EVENT_DEVICE_RESET | HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE;
 	struct asic_fixed_properties *prop = &hdev->asic_prop;
 
 	if (!prop->cpucp_info.eq_health_check_supported)
-		return;
+		return 0;
 
-	if (hdev->eq_heartbeat_received)
+	if (hdev->eq_heartbeat_received) {
 		hdev->eq_heartbeat_received = false;
-	else
-		hl_device_cond_reset(hdev, HL_DRV_RESET_HARD, event_mask);
+	} else {
+		dev_err(hdev->dev, "EQ heartbeat event was not received!\n");
+		return -EIO;
+	}
+
+	return 0;
 }
 
 static void hl_device_heartbeat(struct work_struct *work)
@@ -1069,10 +1075,9 @@ static void hl_device_heartbeat(struct work_struct *work)
 	/*
 	 * For EQ health check need to check if driver received the heartbeat eq event
 	 * in order to validate the eq is working.
+	 * Only if both the EQ is healthy and we managed to send the next heartbeat reschedule.
 	 */
-	hl_device_eq_heartbeat(hdev);
-
-	if (!hdev->asic_funcs->send_heartbeat(hdev))
+	if ((!hl_device_eq_heartbeat_check(hdev)) && (!hdev->asic_funcs->send_heartbeat(hdev)))
 		goto reschedule;
 
 	if (hl_device_operational(hdev, NULL))
@@ -2035,7 +2040,7 @@ int hl_device_cond_reset(struct hl_device *hdev, u32 flags, u64 event_mask)
 	if (ctx)
 		hl_ctx_put(ctx);
 
-	return hl_device_reset(hdev, flags);
+	return hl_device_reset(hdev, flags | HL_DRV_RESET_HARD);
 }
 
 static void hl_notifier_event_send(struct hl_notifier_event *notifier_event, u64 event_mask)
diff --git a/drivers/accel/habanalabs/common/firmware_if.c b/drivers/accel/habanalabs/common/firmware_if.c
index 47e8384134aa..9e9dfe013659 100644
--- a/drivers/accel/habanalabs/common/firmware_if.c
+++ b/drivers/accel/habanalabs/common/firmware_if.c
@@ -646,39 +646,27 @@ int hl_fw_send_heartbeat(struct hl_device *hdev)
 	return rc;
 }
 
-static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val,
-								u32 sts_val)
+static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val, u32 sts_val)
 {
 	bool err_exists = false;
 
 	if (!(err_val & CPU_BOOT_ERR0_ENABLED))
 		return false;
 
-	if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL) {
-		dev_err(hdev->dev,
-			"Device boot error - DRAM initialization failed\n");
-		err_exists = true;
-	}
+	if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL)
+		dev_err(hdev->dev, "Device boot error - DRAM initialization failed\n");
 
-	if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED) {
+	if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED)
 		dev_err(hdev->dev, "Device boot error - FIT image corrupted\n");
-		err_exists = true;
-	}
 
-	if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL) {
-		dev_err(hdev->dev,
-			"Device boot error - Thermal Sensor initialization failed\n");
-		err_exists = true;
-	}
+	if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL)
+		dev_err(hdev->dev, "Device boot error - Thermal Sensor initialization failed\n");
 
 	if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED) {
 		if (hdev->bmc_enable) {
-			dev_err(hdev->dev,
-				"Device boot error - Skipped waiting for BMC\n");
-			err_exists = true;
+			dev_err(hdev->dev, "Device boot error - Skipped waiting for BMC\n");
 		} else {
-			dev_info(hdev->dev,
-				"Device boot message - Skipped waiting for BMC\n");
+			dev_info(hdev->dev, "Device boot message - Skipped waiting for BMC\n");
 			/* This is an info so we don't want it to disable the
 			 * device
 			 */
@@ -686,48 +674,29 @@ static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val,
 		}
 	}
 
-	if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY) {
-		dev_err(hdev->dev,
-			"Device boot error - Serdes data from BMC not available\n");
-		err_exists = true;
-	}
+	if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY)
+		dev_err(hdev->dev, "Device boot error - Serdes data from BMC not available\n");
 
-	if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL) {
-		dev_err(hdev->dev,
-			"Device boot error - NIC F/W initialization failed\n");
-		err_exists = true;
-	}
+	if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL)
+		dev_err(hdev->dev, "Device boot error - NIC F/W initialization failed\n");
 
-	if (err_val & CPU_BOOT_ERR0_SECURITY_NOT_RDY) {
-		dev_err(hdev->dev,
-			"Device boot warning - security not ready\n");
-		err_exists = true;
-	}
+	if (err_val & CPU_BOOT_ERR0_SECURITY_NOT_RDY)
+		dev_err(hdev->dev, "Device boot warning - security not ready\n");
 
-	if (err_val & CPU_BOOT_ERR0_SECURITY_FAIL) {
+	if (err_val & CPU_BOOT_ERR0_SECURITY_FAIL)
 		dev_err(hdev->dev, "Device boot error - security failure\n");
-		err_exists = true;
-	}
 
-	if (err_val & CPU_BOOT_ERR0_EFUSE_FAIL) {
+	if (err_val & CPU_BOOT_ERR0_EFUSE_FAIL)
 		dev_err(hdev->dev, "Device boot error - eFuse failure\n");
-		err_exists = true;
-	}
 
-	if (err_val & CPU_BOOT_ERR0_SEC_IMG_VER_FAIL) {
+	if (err_val & CPU_BOOT_ERR0_SEC_IMG_VER_FAIL)
 		dev_err(hdev->dev, "Device boot error - Failed to load preboot secondary image\n");
-		err_exists = true;
-	}
 
-	if (err_val & CPU_BOOT_ERR0_PLL_FAIL) {
+	if (err_val & CPU_BOOT_ERR0_PLL_FAIL)
 		dev_err(hdev->dev, "Device boot error - PLL failure\n");
-		err_exists = true;
-	}
 
-	if (err_val & CPU_BOOT_ERR0_TMP_THRESH_INIT_FAIL) {
+	if (err_val & CPU_BOOT_ERR0_TMP_THRESH_INIT_FAIL)
 		dev_err(hdev->dev, "Device boot error - Failed to set threshold for temperature sensor\n");
-		err_exists = true;
-	}
 
 	if (err_val & CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL) {
 		/* Ignore this bit, don't prevent driver loading */
@@ -735,52 +704,32 @@ static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val,
 		err_val &= ~CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL;
 	}
 
-	if (err_val & CPU_BOOT_ERR0_BINNING_FAIL) {
+	if (err_val & CPU_BOOT_ERR0_BINNING_FAIL)
 		dev_err(hdev->dev, "Device boot error - binning failure\n");
-		err_exists = true;
-	}
 
 	if (sts_val & CPU_BOOT_DEV_STS0_ENABLED)
 		dev_dbg(hdev->dev, "Device status0 %#x\n", sts_val);
 
+	if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED)
+		dev_err(hdev->dev, "Device boot warning - Skipped DRAM initialization\n");
+
+	if (err_val & CPU_BOOT_ERR_ENG_ARC_MEM_SCRUB_FAIL)
+		dev_err(hdev->dev, "Device boot error - ARC memory scrub failed\n");
+
+	/* All warnings should go here in order not to reach the unknown error validation */
 	if (err_val & CPU_BOOT_ERR0_EEPROM_FAIL) {
 		dev_err(hdev->dev, "Device boot error - EEPROM failure detected\n");
 		err_exists = true;
 	}
 
-	/* All warnings should go here in order not to reach the unknown error validation */
-	if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED) {
-		dev_warn(hdev->dev,
-			"Device boot warning - Skipped DRAM initialization\n");
-		/* This is a warning so we don't want it to disable the
-		 * device
-		 */
-		err_val &= ~CPU_BOOT_ERR0_DRAM_SKIPPED;
-	}
+	if (err_val & CPU_BOOT_ERR0_PRI_IMG_VER_FAIL)
+		dev_warn(hdev->dev, "Device boot warning - Failed to load preboot primary image\n");
 
-	if (err_val & CPU_BOOT_ERR0_PRI_IMG_VER_FAIL) {
-		dev_warn(hdev->dev,
-			"Device boot warning - Failed to load preboot primary image\n");
-		/* This is a warning so we don't want it to disable the
-		 * device as we have a secondary preboot image
-		 */
-		err_val &= ~CPU_BOOT_ERR0_PRI_IMG_VER_FAIL;
-	}
+	if (err_val & CPU_BOOT_ERR0_TPM_FAIL)
+		dev_warn(hdev->dev, "Device boot warning - TPM failure\n");
 
-	if (err_val & CPU_BOOT_ERR0_TPM_FAIL) {
-		dev_warn(hdev->dev,
-			"Device boot warning - TPM failure\n");
-		/* This is a warning so we don't want it to disable the
-		 * device
-		 */
-		err_val &= ~CPU_BOOT_ERR0_TPM_FAIL;
-	}
-
-	if (!err_exists && (err_val & ~CPU_BOOT_ERR0_ENABLED)) {
-		dev_err(hdev->dev,
-			"Device boot error - unknown ERR0 error 0x%08x\n", err_val);
+	if (err_val & CPU_BOOT_ERR_FATAL_MASK)
 		err_exists = true;
-	}
 
 	/* return error only if it's in the predefined mask */
 	if (err_exists && ((err_val & ~CPU_BOOT_ERR0_ENABLED) &
diff --git a/drivers/accel/habanalabs/common/habanalabs.h b/drivers/accel/habanalabs/common/habanalabs.h
index 1655c101c705..7b0209e5bad6 100644
--- a/drivers/accel/habanalabs/common/habanalabs.h
+++ b/drivers/accel/habanalabs/common/habanalabs.h
@@ -1262,6 +1262,7 @@ struct hl_dec {
  * @ASIC_GAUDI_SEC: Gaudi secured device (HL-2000).
  * @ASIC_GAUDI2: Gaudi2 device.
  * @ASIC_GAUDI2B: Gaudi2B device.
+ * @ASIC_GAUDI2C: Gaudi2C device.
  */
 enum hl_asic_type {
 	ASIC_INVALID,
@@ -1270,6 +1271,7 @@ enum hl_asic_type {
 	ASIC_GAUDI_SEC,
 	ASIC_GAUDI2,
 	ASIC_GAUDI2B,
+	ASIC_GAUDI2C,
 };
 
 struct hl_cs_parser;
@@ -3594,6 +3596,14 @@ static inline bool hl_is_fw_sw_ver_below(struct hl_device *hdev, u32 fw_sw_major
 	return false;
 }
 
+static inline bool hl_is_fw_sw_ver_equal_or_greater(struct hl_device *hdev, u32 fw_sw_major,
+							u32 fw_sw_minor)
+{
+	return (hdev->fw_sw_major_ver > fw_sw_major ||
+			(hdev->fw_sw_major_ver == fw_sw_major &&
+					hdev->fw_sw_minor_ver >= fw_sw_minor));
+}
+
 /*
  * Kernel module functions that can be accessed by entire module
  */
diff --git a/drivers/accel/habanalabs/common/habanalabs_drv.c b/drivers/accel/habanalabs/common/habanalabs_drv.c
index 306a5bc9bf89..e542fd40e16c 100644
--- a/drivers/accel/habanalabs/common/habanalabs_drv.c
+++ b/drivers/accel/habanalabs/common/habanalabs_drv.c
@@ -141,6 +141,9 @@ static enum hl_asic_type get_asic_type(struct hl_device *hdev)
 		case REV_ID_B:
 			asic_type = ASIC_GAUDI2B;
 			break;
+		case REV_ID_C:
+			asic_type = ASIC_GAUDI2C;
+			break;
 		default:
 			break;
 		}
@@ -670,6 +673,38 @@ static pci_ers_result_t hl_pci_err_slot_reset(struct pci_dev *pdev)
 	return PCI_ERS_RESULT_RECOVERED;
 }
 
+static void hl_pci_reset_prepare(struct pci_dev *pdev)
+{
+	struct hl_device *hdev;
+
+	hdev = pci_get_drvdata(pdev);
+	if (!hdev)
+		return;
+
+	hdev->disabled = true;
+}
+
+static void hl_pci_reset_done(struct pci_dev *pdev)
+{
+	struct hl_device *hdev;
+	u32 flags;
+
+	hdev = pci_get_drvdata(pdev);
+	if (!hdev)
+		return;
+
+	/*
+	 * Schedule a thread to trigger hard reset.
+	 * The reason for this handler, is for rare cases where the driver is up
+	 * and FLR occurs. This is valid only when working with no VM, so FW handles FLR
+	 * and resets the device. FW will go back preboot stage, so driver needs to perform
+	 * hard reset in order to load FW fit again.
+	 */
+	flags = HL_DRV_RESET_HARD | HL_DRV_RESET_BYPASS_REQ_TO_FW;
+
+	hl_device_reset(hdev, flags);
+}
+
 static const struct dev_pm_ops hl_pm_ops = {
 	.suspend = hl_pmops_suspend,
 	.resume = hl_pmops_resume,
@@ -679,6 +714,8 @@ static const struct pci_error_handlers hl_pci_err_handler = {
 	.error_detected = hl_pci_err_detected,
 	.slot_reset = hl_pci_err_slot_reset,
 	.resume = hl_pci_err_resume,
+	.reset_prepare = hl_pci_reset_prepare,
+	.reset_done = hl_pci_reset_done,
 };
 
 static struct pci_driver hl_pci_driver = {
diff --git a/drivers/accel/habanalabs/common/hwmon.c b/drivers/accel/habanalabs/common/hwmon.c
index 8598056216e7..1ee2ee07e9ed 100644
--- a/drivers/accel/habanalabs/common/hwmon.c
+++ b/drivers/accel/habanalabs/common/hwmon.c
@@ -578,10 +578,6 @@ int hl_get_temperature(struct hl_device *hdev,
 				CPUCP_PKT_CTL_OPCODE_SHIFT);
 	pkt.sensor_index = __cpu_to_le16(sensor_index);
 	pkt.type = __cpu_to_le16(attr);
-
-	dev_dbg(hdev->dev, "get temp, ctl 0x%x, sensor %d, type %d\n",
-		pkt.ctl, pkt.sensor_index, pkt.type);
-
 	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),
 						0, &result);
 
diff --git a/drivers/accel/habanalabs/common/memory.c b/drivers/accel/habanalabs/common/memory.c
index 0b8689fe0b64..3348ad12c237 100644
--- a/drivers/accel/habanalabs/common/memory.c
+++ b/drivers/accel/habanalabs/common/memory.c
@@ -955,8 +955,8 @@ static int map_phys_pg_pack(struct hl_ctx *ctx, u64 vaddr,
 				(i + 1) == phys_pg_pack->npages);
 		if (rc) {
 			dev_err(hdev->dev,
-				"map failed for handle %u, npages: %llu, mapped: %llu",
-				phys_pg_pack->handle, phys_pg_pack->npages,
+				"map failed (%d) for handle %u, npages: %llu, mapped: %llu\n",
+				rc, phys_pg_pack->handle, phys_pg_pack->npages,
 				mapped_pg_cnt);
 			goto err;
 		}
@@ -1186,7 +1186,8 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, u64 *device
 
 	rc = map_phys_pg_pack(ctx, ret_vaddr, phys_pg_pack);
 	if (rc) {
-		dev_err(hdev->dev, "mapping page pack failed for handle %u\n", handle);
+		dev_err(hdev->dev, "mapping page pack failed (%d) for handle %u\n",
+			rc, handle);
 		mutex_unlock(&hdev->mmu_lock);
 		goto map_err;
 	}
diff --git a/drivers/accel/habanalabs/common/mmu/mmu.c b/drivers/accel/habanalabs/common/mmu/mmu.c
index b2145716c605..b654302a68fc 100644
--- a/drivers/accel/habanalabs/common/mmu/mmu.c
+++ b/drivers/accel/habanalabs/common/mmu/mmu.c
@@ -596,6 +596,7 @@ int hl_mmu_if_set_funcs(struct hl_device *hdev)
 		break;
 	case ASIC_GAUDI2:
 	case ASIC_GAUDI2B:
+	case ASIC_GAUDI2C:
 		/* MMUs in Gaudi2 are always host resident */
 		hl_mmu_v2_hr_set_funcs(hdev, &hdev->mmu_func[MMU_HR_PGT]);
 		break;
diff --git a/drivers/accel/habanalabs/common/sysfs.c b/drivers/accel/habanalabs/common/sysfs.c
index 01f89f029355..8d2164691d81 100644
--- a/drivers/accel/habanalabs/common/sysfs.c
+++ b/drivers/accel/habanalabs/common/sysfs.c
@@ -251,6 +251,9 @@ static ssize_t device_type_show(struct device *dev,
 	case ASIC_GAUDI2B:
 		str = "GAUDI2B";
 		break;
+	case ASIC_GAUDI2C:
+		str = "GAUDI2C";
+		break;
 	default:
 		dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
 				hdev->asic_type);
@@ -383,6 +386,14 @@ static ssize_t security_enabled_show(struct device *dev,
 	return sprintf(buf, "%d\n", hdev->asic_prop.fw_security_enabled);
 }
 
+static ssize_t module_id_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct hl_device *hdev = dev_get_drvdata(dev);
+
+	return sprintf(buf, "%u\n", le32_to_cpu(hdev->asic_prop.cpucp_info.card_location));
+}
+
 static DEVICE_ATTR_RO(armcp_kernel_ver);
 static DEVICE_ATTR_RO(armcp_ver);
 static DEVICE_ATTR_RO(cpld_ver);
@@ -402,6 +413,7 @@ static DEVICE_ATTR_RO(thermal_ver);
 static DEVICE_ATTR_RO(uboot_ver);
 static DEVICE_ATTR_RO(fw_os_ver);
 static DEVICE_ATTR_RO(security_enabled);
+static DEVICE_ATTR_RO(module_id);
 
 static struct bin_attribute bin_attr_eeprom = {
 	.attr = {.name = "eeprom", .mode = (0444)},
@@ -427,6 +439,7 @@ static struct attribute *hl_dev_attrs[] = {
 	&dev_attr_uboot_ver.attr,
 	&dev_attr_fw_os_ver.attr,
 	&dev_attr_security_enabled.attr,
+	&dev_attr_module_id.attr,
 	NULL,
 };
 
diff --git a/drivers/accel/habanalabs/gaudi2/gaudi2.c b/drivers/accel/habanalabs/gaudi2/gaudi2.c
index 819660c684cf..77c480725a84 100644
--- a/drivers/accel/habanalabs/gaudi2/gaudi2.c
+++ b/drivers/accel/habanalabs/gaudi2/gaudi2.c
@@ -7860,36 +7860,36 @@ static bool gaudi2_handle_ecc_event(struct hl_device *hdev, u16 event_type,
 
 static void handle_lower_qman_data_on_err(struct hl_device *hdev, u64 qman_base, u64 event_mask)
 {
-	u32 lo, hi, cq_ptr_size, arc_cq_ptr_size;
-	u64 cq_ptr, arc_cq_ptr, cp_current_inst;
-
-	lo = RREG32(qman_base + QM_CQ_PTR_LO_4_OFFSET);
-	hi = RREG32(qman_base + QM_CQ_PTR_HI_4_OFFSET);
-	cq_ptr = ((u64) hi) << 32 | lo;
-	cq_ptr_size = RREG32(qman_base + QM_CQ_TSIZE_4_OFFSET);
-
-	lo = RREG32(qman_base + QM_ARC_CQ_PTR_LO_OFFSET);
-	hi = RREG32(qman_base + QM_ARC_CQ_PTR_HI_OFFSET);
-	arc_cq_ptr = ((u64) hi) << 32 | lo;
-	arc_cq_ptr_size = RREG32(qman_base + QM_ARC_CQ_TSIZE_OFFSET);
+	u32 lo, hi, cq_ptr_size, cp_sts;
+	u64 cq_ptr, cp_current_inst;
+	bool is_arc_cq;
+
+	cp_sts = RREG32(qman_base + QM_CP_STS_4_OFFSET);
+	is_arc_cq = FIELD_GET(PDMA0_QM_CP_STS_CUR_CQ_MASK, cp_sts); /* 0 - legacy CQ, 1 - ARC_CQ */
+
+	if (is_arc_cq) {
+		lo = RREG32(qman_base + QM_ARC_CQ_PTR_LO_OFFSET);
+		hi = RREG32(qman_base + QM_ARC_CQ_PTR_HI_OFFSET);
+		cq_ptr = ((u64) hi) << 32 | lo;
+		cq_ptr_size = RREG32(qman_base + QM_ARC_CQ_TSIZE_OFFSET);
+	} else {
+		lo = RREG32(qman_base + QM_CQ_PTR_LO_4_OFFSET);
+		hi = RREG32(qman_base + QM_CQ_PTR_HI_4_OFFSET);
+		cq_ptr = ((u64) hi) << 32 | lo;
+		cq_ptr_size = RREG32(qman_base + QM_CQ_TSIZE_4_OFFSET);
+	}
 
 	lo = RREG32(qman_base + QM_CP_CURRENT_INST_LO_4_OFFSET);
 	hi = RREG32(qman_base + QM_CP_CURRENT_INST_HI_4_OFFSET);
 	cp_current_inst = ((u64) hi) << 32 | lo;
 
 	dev_info(hdev->dev,
-		"LowerQM. CQ: {ptr %#llx, size %u}, ARC_CQ: {ptr %#llx, size %u}, CP: {instruction %#llx}\n",
-		cq_ptr, cq_ptr_size, arc_cq_ptr, arc_cq_ptr_size, cp_current_inst);
+		"LowerQM. %sCQ: {ptr %#llx, size %u}, CP: {instruction %#llx}\n",
+		is_arc_cq ? "ARC_" : "", cq_ptr, cq_ptr_size, cp_current_inst);
 
 	if (event_mask & HL_NOTIFIER_EVENT_UNDEFINED_OPCODE) {
-		if (arc_cq_ptr) {
-			hdev->captured_err_info.undef_opcode.cq_addr = arc_cq_ptr;
-			hdev->captured_err_info.undef_opcode.cq_size = arc_cq_ptr_size;
-		} else {
-			hdev->captured_err_info.undef_opcode.cq_addr = cq_ptr;
-			hdev->captured_err_info.undef_opcode.cq_size = cq_ptr_size;
-		}
-
+		hdev->captured_err_info.undef_opcode.cq_addr = cq_ptr;
+		hdev->captured_err_info.undef_opcode.cq_size = cq_ptr_size;
 		hdev->captured_err_info.undef_opcode.stream_id = QMAN_STREAMS;
 	}
 }
@@ -7929,21 +7929,19 @@ static int gaudi2_handle_qman_err_generic(struct hl_device *hdev, u16 event_type
 				error_count++;
 			}
 
-		if (i == QMAN_STREAMS && error_count) {
-			/* check for undefined opcode */
-			if (glbl_sts_val & PDMA0_QM_GLBL_ERR_STS_CP_UNDEF_CMD_ERR_MASK &&
-					hdev->captured_err_info.undef_opcode.write_enable) {
+		/* check for undefined opcode */
+		if (glbl_sts_val & PDMA0_QM_GLBL_ERR_STS_CP_UNDEF_CMD_ERR_MASK) {
+			*event_mask |= HL_NOTIFIER_EVENT_UNDEFINED_OPCODE;
+			if (hdev->captured_err_info.undef_opcode.write_enable) {
 				memset(&hdev->captured_err_info.undef_opcode, 0,
 						sizeof(hdev->captured_err_info.undef_opcode));
-
-				hdev->captured_err_info.undef_opcode.write_enable = false;
 				hdev->captured_err_info.undef_opcode.timestamp = ktime_get();
 				hdev->captured_err_info.undef_opcode.engine_id =
 							gaudi2_queue_id_to_engine_id[qid_base];
-				*event_mask |= HL_NOTIFIER_EVENT_UNDEFINED_OPCODE;
 			}
 
-			handle_lower_qman_data_on_err(hdev, qman_base, *event_mask);
+			if (i == QMAN_STREAMS)
+				handle_lower_qman_data_on_err(hdev, qman_base, *event_mask);
 		}
 	}
 
@@ -10007,6 +10005,8 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent
 		error_count = gaudi2_handle_pcie_drain(hdev, &eq_entry->pcie_drain_ind_data);
 		reset_flags |= HL_DRV_RESET_FW_FATAL_ERR;
 		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;
+		if (hl_is_fw_sw_ver_equal_or_greater(hdev, 1, 13))
+			is_critical = true;
 		break;
 
 	case GAUDI2_EVENT_PSOC59_RPM_ERROR_OR_DRAIN:
diff --git a/drivers/accel/habanalabs/include/gaudi2/asic_reg/gaudi2_regs.h b/drivers/accel/habanalabs/include/gaudi2/asic_reg/gaudi2_regs.h
index a08378d0802b..8018214a7b59 100644
--- a/drivers/accel/habanalabs/include/gaudi2/asic_reg/gaudi2_regs.h
+++ b/drivers/accel/habanalabs/include/gaudi2/asic_reg/gaudi2_regs.h
@@ -250,6 +250,7 @@
 #define QM_ARC_CQ_PTR_HI_OFFSET		(mmPDMA0_QM_ARC_CQ_PTR_HI - mmPDMA0_QM_BASE)
 #define QM_ARC_CQ_TSIZE_OFFSET		(mmPDMA0_QM_ARC_CQ_TSIZE - mmPDMA0_QM_BASE)
 
+#define QM_CP_STS_4_OFFSET		(mmPDMA0_QM_CP_STS_4 - mmPDMA0_QM_BASE)
 #define QM_CP_CURRENT_INST_LO_4_OFFSET	(mmPDMA0_QM_CP_CURRENT_INST_LO_4 - mmPDMA0_QM_BASE)
 #define QM_CP_CURRENT_INST_HI_4_OFFSET	(mmPDMA0_QM_CP_CURRENT_INST_HI_4 - mmPDMA0_QM_BASE)
 
diff --git a/drivers/accel/habanalabs/include/hw_ip/pci/pci_general.h b/drivers/accel/habanalabs/include/hw_ip/pci/pci_general.h
index f5d497dc9bdc..4f951cada077 100644
--- a/drivers/accel/habanalabs/include/hw_ip/pci/pci_general.h
+++ b/drivers/accel/habanalabs/include/hw_ip/pci/pci_general.h
@@ -25,6 +25,7 @@ enum hl_revision_id {
 	REV_ID_INVALID				= 0x00,
 	REV_ID_A				= 0x01,
 	REV_ID_B				= 0x02,
+	REV_ID_C				= 0x03
 };
 
 #endif /* INCLUDE_PCI_GENERAL_H_ */
diff --git a/drivers/accel/ivpu/Kconfig b/drivers/accel/ivpu/Kconfig
index 1a4c4ed9d113..682c53245286 100644
--- a/drivers/accel/ivpu/Kconfig
+++ b/drivers/accel/ivpu/Kconfig
@@ -1,16 +1,17 @@
 # SPDX-License-Identifier: GPL-2.0-only
 
 config DRM_ACCEL_IVPU
-	tristate "Intel VPU for Meteor Lake and newer"
+	tristate "Intel NPU (Neural Processing Unit)"
 	depends on DRM_ACCEL
 	depends on X86_64 && !UML
 	depends on PCI && PCI_MSI
 	select FW_LOADER
-	select SHMEM
+	select DRM_GEM_SHMEM_HELPER
 	select GENERIC_ALLOCATOR
 	help
-	  Choose this option if you have a system that has an 14th generation Intel CPU
-	  or newer. VPU stands for Versatile Processing Unit and it's a CPU-integrated
-	  inference accelerator for Computer Vision and Deep Learning applications.
+	  Choose this option if you have a system with an 14th generation
+	  Intel CPU (Meteor Lake) or newer. Intel NPU (formerly called Intel VPU)
+	  is a CPU-integrated inference accelerator for Computer Vision
+	  and Deep Learning applications.
 
 	  If "M" is selected, the module will be called intel_vpu.
diff --git a/drivers/accel/ivpu/ivpu_debugfs.c b/drivers/accel/ivpu/ivpu_debugfs.c
index ea453b985b49..19035230563d 100644
--- a/drivers/accel/ivpu/ivpu_debugfs.c
+++ b/drivers/accel/ivpu/ivpu_debugfs.c
@@ -14,6 +14,7 @@
 #include "ivpu_fw.h"
 #include "ivpu_fw_log.h"
 #include "ivpu_gem.h"
+#include "ivpu_hw.h"
 #include "ivpu_jsm_msg.h"
 #include "ivpu_pm.h"
 
@@ -115,6 +116,31 @@ static const struct drm_debugfs_info vdev_debugfs_list[] = {
 	{"reset_pending", reset_pending_show, 0},
 };
 
+static ssize_t
+dvfs_mode_fops_write(struct file *file, const char __user *user_buf, size_t size, loff_t *pos)
+{
+	struct ivpu_device *vdev = file->private_data;
+	struct ivpu_fw_info *fw = vdev->fw;
+	u32 dvfs_mode;
+	int ret;
+
+	ret = kstrtou32_from_user(user_buf, size, 0, &dvfs_mode);
+	if (ret < 0)
+		return ret;
+
+	fw->dvfs_mode = dvfs_mode;
+
+	ivpu_pm_schedule_recovery(vdev);
+
+	return size;
+}
+
+static const struct file_operations dvfs_mode_fops = {
+	.owner = THIS_MODULE,
+	.open = simple_open,
+	.write = dvfs_mode_fops_write,
+};
+
 static int fw_log_show(struct seq_file *s, void *v)
 {
 	struct ivpu_device *vdev = s->private;
@@ -151,6 +177,30 @@ static const struct file_operations fw_log_fops = {
 	.release = single_release,
 };
 
+static ssize_t
+fw_profiling_freq_fops_write(struct file *file, const char __user *user_buf,
+			     size_t size, loff_t *pos)
+{
+	struct ivpu_device *vdev = file->private_data;
+	bool enable;
+	int ret;
+
+	ret = kstrtobool_from_user(user_buf, size, &enable);
+	if (ret < 0)
+		return ret;
+
+	ivpu_hw_profiling_freq_drive(vdev, enable);
+	ivpu_pm_schedule_recovery(vdev);
+
+	return size;
+}
+
+static const struct file_operations fw_profiling_freq_fops = {
+	.owner = THIS_MODULE,
+	.open = simple_open,
+	.write = fw_profiling_freq_fops_write,
+};
+
 static ssize_t
 fw_trace_destination_mask_fops_write(struct file *file, const char __user *user_buf,
 				     size_t size, loff_t *pos)
@@ -280,6 +330,9 @@ void ivpu_debugfs_init(struct ivpu_device *vdev)
 	debugfs_create_file("force_recovery", 0200, debugfs_root, vdev,
 			    &ivpu_force_recovery_fops);
 
+	debugfs_create_file("dvfs_mode", 0200, debugfs_root, vdev,
+			    &dvfs_mode_fops);
+
 	debugfs_create_file("fw_log", 0644, debugfs_root, vdev,
 			    &fw_log_fops);
 	debugfs_create_file("fw_trace_destination_mask", 0200, debugfs_root, vdev,
@@ -291,4 +344,8 @@ void ivpu_debugfs_init(struct ivpu_device *vdev)
 
 	debugfs_create_file("reset_engine", 0200, debugfs_root, vdev,
 			    &ivpu_reset_engine_fops);
+
+	if (ivpu_hw_gen(vdev) >= IVPU_HW_40XX)
+		debugfs_create_file("fw_profiling_freq_drive", 0200,
+				    debugfs_root, vdev, &fw_profiling_freq_fops);
 }
diff --git a/drivers/accel/ivpu/ivpu_drv.c b/drivers/accel/ivpu/ivpu_drv.c
index 790603017653..64927682161b 100644
--- a/drivers/accel/ivpu/ivpu_drv.c
+++ b/drivers/accel/ivpu/ivpu_drv.c
@@ -31,8 +31,6 @@
 			   __stringify(DRM_IVPU_DRIVER_MINOR) "."
 #endif
 
-static const struct drm_driver driver;
-
 static struct lock_class_key submitted_jobs_xa_lock_class_key;
 
 int ivpu_dbg_mask;
@@ -41,7 +39,7 @@ MODULE_PARM_DESC(dbg_mask, "Driver debug mask. See IVPU_DBG_* macros.");
 
 int ivpu_test_mode;
 module_param_named_unsafe(test_mode, ivpu_test_mode, int, 0644);
-MODULE_PARM_DESC(test_mode, "Test mode: 0 - normal operation, 1 - fw unit test, 2 - null hw");
+MODULE_PARM_DESC(test_mode, "Test mode mask. See IVPU_TEST_MODE_* macros.");
 
 u8 ivpu_pll_min_ratio;
 module_param_named(pll_min_ratio, ivpu_pll_min_ratio, byte, 0644);
@@ -93,8 +91,8 @@ static void file_priv_release(struct kref *ref)
 	ivpu_dbg(vdev, FILE, "file_priv release: ctx %u\n", file_priv->ctx.id);
 
 	ivpu_cmdq_release_all(file_priv);
-	ivpu_bo_remove_all_bos_from_context(&file_priv->ctx);
 	ivpu_jsm_context_release(vdev, file_priv->ctx.id);
+	ivpu_bo_remove_all_bos_from_context(vdev, &file_priv->ctx);
 	ivpu_mmu_user_context_fini(vdev, &file_priv->ctx);
 	drm_WARN_ON(&vdev->drm, xa_erase_irq(&vdev->context_xa, file_priv->ctx.id) != file_priv);
 	mutex_destroy(&file_priv->lock);
@@ -317,16 +315,14 @@ static int ivpu_wait_for_ready(struct ivpu_device *vdev)
 	unsigned long timeout;
 	int ret;
 
-	if (ivpu_test_mode == IVPU_TEST_MODE_FW_TEST)
+	if (ivpu_test_mode & IVPU_TEST_MODE_FW_TEST)
 		return 0;
 
-	ivpu_ipc_consumer_add(vdev, &cons, IVPU_IPC_CHAN_BOOT_MSG);
+	ivpu_ipc_consumer_add(vdev, &cons, IVPU_IPC_CHAN_BOOT_MSG, NULL);
 
 	timeout = jiffies + msecs_to_jiffies(vdev->timeout.boot);
 	while (1) {
-		ret = ivpu_ipc_irq_handler(vdev);
-		if (ret)
-			break;
+		ivpu_ipc_irq_handler(vdev, NULL);
 		ret = ivpu_ipc_receive(vdev, &cons, &ipc_hdr, NULL, 0);
 		if (ret != -ETIMEDOUT || time_after_eq(jiffies, timeout))
 			break;
@@ -362,7 +358,7 @@ int ivpu_boot(struct ivpu_device *vdev)
 	int ret;
 
 	/* Update boot params located at first 4KB of FW memory */
-	ivpu_fw_boot_params_setup(vdev, vdev->fw->mem->kvaddr);
+	ivpu_fw_boot_params_setup(vdev, ivpu_bo_vaddr(vdev->fw->mem));
 
 	ret = ivpu_hw_boot_fw(vdev);
 	if (ret) {
@@ -414,7 +410,9 @@ static const struct drm_driver driver = {
 
 	.open = ivpu_open,
 	.postclose = ivpu_postclose,
-	.gem_prime_import = ivpu_gem_prime_import,
+
+	.gem_create_object = ivpu_gem_create_object,
+	.gem_prime_import_sg_table = drm_gem_shmem_prime_import_sg_table,
 
 	.ioctls = ivpu_drm_ioctls,
 	.num_ioctls = ARRAY_SIZE(ivpu_drm_ioctls),
@@ -427,6 +425,13 @@ static const struct drm_driver driver = {
 	.minor = DRM_IVPU_DRIVER_MINOR,
 };
 
+static irqreturn_t ivpu_irq_thread_handler(int irq, void *arg)
+{
+	struct ivpu_device *vdev = arg;
+
+	return ivpu_ipc_irq_thread_handler(vdev);
+}
+
 static int ivpu_irq_init(struct ivpu_device *vdev)
 {
 	struct pci_dev *pdev = to_pci_dev(vdev->drm.dev);
@@ -440,8 +445,8 @@ static int ivpu_irq_init(struct ivpu_device *vdev)
 
 	vdev->irq = pci_irq_vector(pdev, 0);
 
-	ret = devm_request_irq(vdev->drm.dev, vdev->irq, vdev->hw->ops->irq_handler,
-			       IRQF_NO_AUTOEN, DRIVER_NAME, vdev);
+	ret = devm_request_threaded_irq(vdev->drm.dev, vdev->irq, vdev->hw->ops->irq_handler,
+					ivpu_irq_thread_handler, IRQF_NO_AUTOEN, DRIVER_NAME, vdev);
 	if (ret)
 		ivpu_err(vdev, "Failed to request an IRQ %d\n", ret);
 
@@ -533,6 +538,11 @@ static int ivpu_dev_init(struct ivpu_device *vdev)
 	xa_init_flags(&vdev->context_xa, XA_FLAGS_ALLOC);
 	xa_init_flags(&vdev->submitted_jobs_xa, XA_FLAGS_ALLOC1);
 	lockdep_set_class(&vdev->submitted_jobs_xa.xa_lock, &submitted_jobs_xa_lock_class_key);
+	INIT_LIST_HEAD(&vdev->bo_list);
+
+	ret = drmm_mutex_init(&vdev->drm, &vdev->bo_list_lock);
+	if (ret)
+		goto err_xa_destroy;
 
 	ret = ivpu_pci_init(vdev);
 	if (ret)
@@ -550,7 +560,7 @@ static int ivpu_dev_init(struct ivpu_device *vdev)
 	/* Power up early so the rest of init code can access VPU registers */
 	ret = ivpu_hw_power_up(vdev);
 	if (ret)
-		goto err_xa_destroy;
+		goto err_power_down;
 
 	ret = ivpu_mmu_global_context_init(vdev);
 	if (ret)
@@ -574,20 +584,15 @@ static int ivpu_dev_init(struct ivpu_device *vdev)
 
 	ivpu_pm_init(vdev);
 
-	ret = ivpu_job_done_thread_init(vdev);
-	if (ret)
-		goto err_ipc_fini;
-
 	ret = ivpu_boot(vdev);
 	if (ret)
-		goto err_job_done_thread_fini;
+		goto err_ipc_fini;
 
+	ivpu_job_done_consumer_init(vdev);
 	ivpu_pm_enable(vdev);
 
 	return 0;
 
-err_job_done_thread_fini:
-	ivpu_job_done_thread_fini(vdev);
 err_ipc_fini:
 	ivpu_ipc_fini(vdev);
 err_fw_fini:
@@ -612,7 +617,7 @@ static void ivpu_dev_fini(struct ivpu_device *vdev)
 	ivpu_shutdown(vdev);
 	if (IVPU_WA(d3hot_after_power_off))
 		pci_set_power_state(to_pci_dev(vdev->drm.dev), PCI_D3hot);
-	ivpu_job_done_thread_fini(vdev);
+	ivpu_job_done_consumer_fini(vdev);
 	ivpu_pm_cancel_recovery(vdev);
 
 	ivpu_ipc_fini(vdev);
diff --git a/drivers/accel/ivpu/ivpu_drv.h b/drivers/accel/ivpu/ivpu_drv.h
index 417ddeca8517..ebc4b84f27b2 100644
--- a/drivers/accel/ivpu/ivpu_drv.h
+++ b/drivers/accel/ivpu/ivpu_drv.h
@@ -17,9 +17,10 @@
 #include <uapi/drm/ivpu_accel.h>
 
 #include "ivpu_mmu_context.h"
+#include "ivpu_ipc.h"
 
 #define DRIVER_NAME "intel_vpu"
-#define DRIVER_DESC "Driver for Intel Versatile Processing Unit (VPU)"
+#define DRIVER_DESC "Driver for Intel NPU (Neural Processing Unit)"
 #define DRIVER_DATE "20230117"
 
 #define PCI_DEVICE_ID_MTL   0x7d1d
@@ -88,6 +89,7 @@ struct ivpu_wa_table {
 	bool d3hot_after_power_off;
 	bool interrupt_clear_with_0;
 	bool disable_clock_relinquish;
+	bool disable_d0i3_msg;
 };
 
 struct ivpu_hw_info;
@@ -115,8 +117,11 @@ struct ivpu_device {
 	struct xarray context_xa;
 	struct xa_limit context_xa_limit;
 
+	struct mutex bo_list_lock; /* Protects bo_list */
+	struct list_head bo_list;
+
 	struct xarray submitted_jobs_xa;
-	struct task_struct *job_done_thread;
+	struct ivpu_ipc_consumer job_done_consumer;
 
 	atomic64_t unique_id_counter;
 
@@ -126,6 +131,7 @@ struct ivpu_device {
 		int tdr;
 		int reschedule_suspend;
 		int autosuspend;
+		int d0i3_entry_msg;
 	} timeout;
 };
 
@@ -148,9 +154,11 @@ extern u8 ivpu_pll_min_ratio;
 extern u8 ivpu_pll_max_ratio;
 extern bool ivpu_disable_mmu_cont_pages;
 
-#define IVPU_TEST_MODE_DISABLED  0
-#define IVPU_TEST_MODE_FW_TEST   1
-#define IVPU_TEST_MODE_NULL_HW   2
+#define IVPU_TEST_MODE_FW_TEST            BIT(0)
+#define IVPU_TEST_MODE_NULL_HW            BIT(1)
+#define IVPU_TEST_MODE_NULL_SUBMISSION    BIT(2)
+#define IVPU_TEST_MODE_D0I3_MSG_DISABLE   BIT(4)
+#define IVPU_TEST_MODE_D0I3_MSG_ENABLE    BIT(5)
 extern int ivpu_test_mode;
 
 struct ivpu_file_priv *ivpu_file_priv_get(struct ivpu_file_priv *file_priv);
diff --git a/drivers/accel/ivpu/ivpu_fw.c b/drivers/accel/ivpu/ivpu_fw.c
index 691da521dde5..6576232f3e67 100644
--- a/drivers/accel/ivpu/ivpu_fw.c
+++ b/drivers/accel/ivpu/ivpu_fw.c
@@ -33,12 +33,17 @@
 
 #define ADDR_TO_L2_CACHE_CFG(addr) ((addr) >> 31)
 
-#define IVPU_FW_CHECK_API(vdev, fw_hdr, name, min_major) \
+/* Check if FW API is compatible with the driver */
+#define IVPU_FW_CHECK_API_COMPAT(vdev, fw_hdr, name, min_major) \
 	ivpu_fw_check_api(vdev, fw_hdr, #name, \
 			  VPU_##name##_API_VER_INDEX, \
 			  VPU_##name##_API_VER_MAJOR, \
 			  VPU_##name##_API_VER_MINOR, min_major)
 
+/* Check if API version is lower that the given version */
+#define IVPU_FW_CHECK_API_VER_LT(vdev, fw_hdr, name, major, minor) \
+	ivpu_fw_check_api_ver_lt(vdev, fw_hdr, #name, VPU_##name##_API_VER_INDEX, major, minor)
+
 static char *ivpu_firmware;
 module_param_named_unsafe(firmware, ivpu_firmware, charp, 0644);
 MODULE_PARM_DESC(firmware, "VPU firmware binary in /lib/firmware/..");
@@ -105,6 +110,19 @@ ivpu_fw_check_api(struct ivpu_device *vdev, const struct vpu_firmware_header *fw
 	return 0;
 }
 
+static bool
+ivpu_fw_check_api_ver_lt(struct ivpu_device *vdev, const struct vpu_firmware_header *fw_hdr,
+			 const char *str, int index, u16 major, u16 minor)
+{
+	u16 fw_major = (u16)(fw_hdr->api_version[index] >> 16);
+	u16 fw_minor = (u16)(fw_hdr->api_version[index]);
+
+	if (fw_major < major || (fw_major == major && fw_minor < minor))
+		return true;
+
+	return false;
+}
+
 static int ivpu_fw_parse(struct ivpu_device *vdev)
 {
 	struct ivpu_fw_info *fw = vdev->fw;
@@ -164,9 +182,9 @@ static int ivpu_fw_parse(struct ivpu_device *vdev)
 	ivpu_info(vdev, "Firmware: %s, version: %s", fw->name,
 		  (const char *)fw_hdr + VPU_FW_HEADER_SIZE);
 
-	if (IVPU_FW_CHECK_API(vdev, fw_hdr, BOOT, 3))
+	if (IVPU_FW_CHECK_API_COMPAT(vdev, fw_hdr, BOOT, 3))
 		return -EINVAL;
-	if (IVPU_FW_CHECK_API(vdev, fw_hdr, JSM, 3))
+	if (IVPU_FW_CHECK_API_COMPAT(vdev, fw_hdr, JSM, 3))
 		return -EINVAL;
 
 	fw->runtime_addr = runtime_addr;
@@ -182,6 +200,8 @@ static int ivpu_fw_parse(struct ivpu_device *vdev)
 	fw->trace_destination_mask = VPU_TRACE_DESTINATION_VERBOSE_TRACING;
 	fw->trace_hw_component_mask = -1;
 
+	fw->dvfs_mode = 0;
+
 	ivpu_dbg(vdev, FW_BOOT, "Size: file %lu image %u runtime %u shavenn %u\n",
 		 fw->file->size, fw->image_size, fw->runtime_size, fw->shave_nn_size);
 	ivpu_dbg(vdev, FW_BOOT, "Address: runtime 0x%llx, load 0x%llx, entry point 0x%llx\n",
@@ -195,6 +215,24 @@ static void ivpu_fw_release(struct ivpu_device *vdev)
 	release_firmware(vdev->fw->file);
 }
 
+/* Initialize workarounds that depend on FW version */
+static void
+ivpu_fw_init_wa(struct ivpu_device *vdev)
+{
+	const struct vpu_firmware_header *fw_hdr = (const void *)vdev->fw->file->data;
+
+	if (IVPU_FW_CHECK_API_VER_LT(vdev, fw_hdr, BOOT, 3, 17) ||
+	    (ivpu_hw_gen(vdev) > IVPU_HW_37XX) ||
+	    (ivpu_test_mode & IVPU_TEST_MODE_D0I3_MSG_DISABLE))
+		vdev->wa.disable_d0i3_msg = true;
+
+	/* Force enable the feature for testing purposes */
+	if (ivpu_test_mode & IVPU_TEST_MODE_D0I3_MSG_ENABLE)
+		vdev->wa.disable_d0i3_msg = false;
+
+	IVPU_PRINT_WA(disable_d0i3_msg);
+}
+
 static int ivpu_fw_update_global_range(struct ivpu_device *vdev)
 {
 	struct ivpu_fw_info *fw = vdev->fw;
@@ -248,7 +286,7 @@ static int ivpu_fw_mem_init(struct ivpu_device *vdev)
 
 	if (fw->shave_nn_size) {
 		fw->mem_shave_nn = ivpu_bo_alloc_internal(vdev, vdev->hw->ranges.shave.start,
-							  fw->shave_nn_size, DRM_IVPU_BO_UNCACHED);
+							  fw->shave_nn_size, DRM_IVPU_BO_WC);
 		if (!fw->mem_shave_nn) {
 			ivpu_err(vdev, "Failed to allocate shavenn buffer\n");
 			ret = -ENOMEM;
@@ -297,6 +335,8 @@ int ivpu_fw_init(struct ivpu_device *vdev)
 	if (ret)
 		goto err_fw_release;
 
+	ivpu_fw_init_wa(vdev);
+
 	ret = ivpu_fw_mem_init(vdev);
 	if (ret)
 		goto err_fw_release;
@@ -422,14 +462,31 @@ static void ivpu_fw_boot_params_print(struct ivpu_device *vdev, struct vpu_boot_
 		 boot_params->punit_telemetry_sram_size);
 	ivpu_dbg(vdev, FW_BOOT, "boot_params.vpu_telemetry_enable = 0x%x\n",
 		 boot_params->vpu_telemetry_enable);
+	ivpu_dbg(vdev, FW_BOOT, "boot_params.dvfs_mode = %u\n",
+		 boot_params->dvfs_mode);
+	ivpu_dbg(vdev, FW_BOOT, "boot_params.d0i3_delayed_entry = %d\n",
+		 boot_params->d0i3_delayed_entry);
+	ivpu_dbg(vdev, FW_BOOT, "boot_params.d0i3_residency_time_us = %lld\n",
+		 boot_params->d0i3_residency_time_us);
+	ivpu_dbg(vdev, FW_BOOT, "boot_params.d0i3_entry_vpu_ts = %llu\n",
+		 boot_params->d0i3_entry_vpu_ts);
 }
 
 void ivpu_fw_boot_params_setup(struct ivpu_device *vdev, struct vpu_boot_params *boot_params)
 {
 	struct ivpu_bo *ipc_mem_rx = vdev->ipc->mem_rx;
 
-	/* In case of warm boot we only have to reset the entrypoint addr */
+	/* In case of warm boot only update variable params */
 	if (!ivpu_fw_is_cold_boot(vdev)) {
+		boot_params->d0i3_residency_time_us =
+			ktime_us_delta(ktime_get_boottime(), vdev->hw->d0i3_entry_host_ts);
+		boot_params->d0i3_entry_vpu_ts = vdev->hw->d0i3_entry_vpu_ts;
+
+		ivpu_dbg(vdev, FW_BOOT, "boot_params.d0i3_residency_time_us = %lld\n",
+			 boot_params->d0i3_residency_time_us);
+		ivpu_dbg(vdev, FW_BOOT, "boot_params.d0i3_entry_vpu_ts = %llu\n",
+			 boot_params->d0i3_entry_vpu_ts);
+
 		boot_params->save_restore_ret_address = 0;
 		vdev->pm->is_warmboot = true;
 		wmb(); /* Flush WC buffers after writing save_restore_ret_address */
@@ -442,6 +499,13 @@ void ivpu_fw_boot_params_setup(struct ivpu_device *vdev, struct vpu_boot_params
 	boot_params->vpu_id = to_pci_dev(vdev->drm.dev)->bus->number;
 	boot_params->frequency = ivpu_hw_reg_pll_freq_get(vdev);
 
+	/*
+	 * This param is a debug firmware feature.  It switches default clock
+	 * to higher resolution one for fine-grained and more accurate firmware
+	 * task profiling.
+	 */
+	boot_params->perf_clk_frequency = ivpu_hw_profiling_freq_get(vdev);
+
 	/*
 	 * Uncached region of VPU address space, covers IPC buffers, job queues
 	 * and log buffers, programmable to L2$ Uncached by VPU MTRR
@@ -493,6 +557,11 @@ void ivpu_fw_boot_params_setup(struct ivpu_device *vdev, struct vpu_boot_params
 	boot_params->punit_telemetry_sram_base = ivpu_hw_reg_telemetry_offset_get(vdev);
 	boot_params->punit_telemetry_sram_size = ivpu_hw_reg_telemetry_size_get(vdev);
 	boot_params->vpu_telemetry_enable = ivpu_hw_reg_telemetry_enable_get(vdev);
+	boot_params->dvfs_mode = vdev->fw->dvfs_mode;
+	if (!IVPU_WA(disable_d0i3_msg))
+		boot_params->d0i3_delayed_entry = 1;
+	boot_params->d0i3_residency_time_us = 0;
+	boot_params->d0i3_entry_vpu_ts = 0;
 
 	wmb(); /* Flush WC buffers after writing bootparams */
 
diff --git a/drivers/accel/ivpu/ivpu_fw.h b/drivers/accel/ivpu/ivpu_fw.h
index 10ae2847f0ef..66b60fa161b5 100644
--- a/drivers/accel/ivpu/ivpu_fw.h
+++ b/drivers/accel/ivpu/ivpu_fw.h
@@ -27,6 +27,7 @@ struct ivpu_fw_info {
 	u32 trace_level;
 	u32 trace_destination_mask;
 	u64 trace_hw_component_mask;
+	u32 dvfs_mode;
 };
 
 int ivpu_fw_init(struct ivpu_device *vdev);
diff --git a/drivers/accel/ivpu/ivpu_gem.c b/drivers/accel/ivpu/ivpu_gem.c
index c91852f2edc8..1dda4f38ea25 100644
--- a/drivers/accel/ivpu/ivpu_gem.c
+++ b/drivers/accel/ivpu/ivpu_gem.c
@@ -20,215 +20,18 @@
 #include "ivpu_mmu.h"
 #include "ivpu_mmu_context.h"
 
-MODULE_IMPORT_NS(DMA_BUF);
-
 static const struct drm_gem_object_funcs ivpu_gem_funcs;
 
-static struct lock_class_key prime_bo_lock_class_key;
-
-static int __must_check prime_alloc_pages_locked(struct ivpu_bo *bo)
-{
-	/* Pages are managed by the underlying dma-buf */
-	return 0;
-}
-
-static void prime_free_pages_locked(struct ivpu_bo *bo)
-{
-	/* Pages are managed by the underlying dma-buf */
-}
-
-static int prime_map_pages_locked(struct ivpu_bo *bo)
-{
-	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
-	struct sg_table *sgt;
-
-	sgt = dma_buf_map_attachment_unlocked(bo->base.import_attach, DMA_BIDIRECTIONAL);
-	if (IS_ERR(sgt)) {
-		ivpu_err(vdev, "Failed to map attachment: %ld\n", PTR_ERR(sgt));
-		return PTR_ERR(sgt);
-	}
-
-	bo->sgt = sgt;
-	return 0;
-}
-
-static void prime_unmap_pages_locked(struct ivpu_bo *bo)
-{
-	dma_buf_unmap_attachment_unlocked(bo->base.import_attach, bo->sgt, DMA_BIDIRECTIONAL);
-	bo->sgt = NULL;
-}
-
-static const struct ivpu_bo_ops prime_ops = {
-	.type = IVPU_BO_TYPE_PRIME,
-	.name = "prime",
-	.alloc_pages = prime_alloc_pages_locked,
-	.free_pages = prime_free_pages_locked,
-	.map_pages = prime_map_pages_locked,
-	.unmap_pages = prime_unmap_pages_locked,
-};
-
-static int __must_check shmem_alloc_pages_locked(struct ivpu_bo *bo)
-{
-	int npages = ivpu_bo_size(bo) >> PAGE_SHIFT;
-	struct page **pages;
-
-	pages = drm_gem_get_pages(&bo->base);
-	if (IS_ERR(pages))
-		return PTR_ERR(pages);
-
-	if (bo->flags & DRM_IVPU_BO_WC)
-		set_pages_array_wc(pages, npages);
-	else if (bo->flags & DRM_IVPU_BO_UNCACHED)
-		set_pages_array_uc(pages, npages);
-
-	bo->pages = pages;
-	return 0;
-}
-
-static void shmem_free_pages_locked(struct ivpu_bo *bo)
-{
-	if (ivpu_bo_cache_mode(bo) != DRM_IVPU_BO_CACHED)
-		set_pages_array_wb(bo->pages, ivpu_bo_size(bo) >> PAGE_SHIFT);
-
-	drm_gem_put_pages(&bo->base, bo->pages, true, false);
-	bo->pages = NULL;
-}
-
-static int ivpu_bo_map_pages_locked(struct ivpu_bo *bo)
-{
-	int npages = ivpu_bo_size(bo) >> PAGE_SHIFT;
-	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
-	struct sg_table *sgt;
-	int ret;
-
-	sgt = drm_prime_pages_to_sg(&vdev->drm, bo->pages, npages);
-	if (IS_ERR(sgt)) {
-		ivpu_err(vdev, "Failed to allocate sgtable\n");
-		return PTR_ERR(sgt);
-	}
-
-	ret = dma_map_sgtable(vdev->drm.dev, sgt, DMA_BIDIRECTIONAL, 0);
-	if (ret) {
-		ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret);
-		goto err_free_sgt;
-	}
-
-	bo->sgt = sgt;
-	return 0;
-
-err_free_sgt:
-	kfree(sgt);
-	return ret;
-}
-
-static void ivpu_bo_unmap_pages_locked(struct ivpu_bo *bo)
-{
-	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
-
-	dma_unmap_sgtable(vdev->drm.dev, bo->sgt, DMA_BIDIRECTIONAL, 0);
-	sg_free_table(bo->sgt);
-	kfree(bo->sgt);
-	bo->sgt = NULL;
-}
-
-static const struct ivpu_bo_ops shmem_ops = {
-	.type = IVPU_BO_TYPE_SHMEM,
-	.name = "shmem",
-	.alloc_pages = shmem_alloc_pages_locked,
-	.free_pages = shmem_free_pages_locked,
-	.map_pages = ivpu_bo_map_pages_locked,
-	.unmap_pages = ivpu_bo_unmap_pages_locked,
-};
-
-static int __must_check internal_alloc_pages_locked(struct ivpu_bo *bo)
-{
-	unsigned int i, npages = ivpu_bo_size(bo) >> PAGE_SHIFT;
-	struct page **pages;
-	int ret;
-
-	pages = kvmalloc_array(npages, sizeof(*bo->pages), GFP_KERNEL);
-	if (!pages)
-		return -ENOMEM;
-
-	for (i = 0; i < npages; i++) {
-		pages[i] = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
-		if (!pages[i]) {
-			ret = -ENOMEM;
-			goto err_free_pages;
-		}
-		cond_resched();
-	}
-
-	bo->pages = pages;
-	return 0;
-
-err_free_pages:
-	while (i--)
-		put_page(pages[i]);
-	kvfree(pages);
-	return ret;
-}
-
-static void internal_free_pages_locked(struct ivpu_bo *bo)
-{
-	unsigned int i, npages = ivpu_bo_size(bo) >> PAGE_SHIFT;
-
-	if (ivpu_bo_cache_mode(bo) != DRM_IVPU_BO_CACHED)
-		set_pages_array_wb(bo->pages, ivpu_bo_size(bo) >> PAGE_SHIFT);
-
-	for (i = 0; i < npages; i++)
-		put_page(bo->pages[i]);
-
-	kvfree(bo->pages);
-	bo->pages = NULL;
-}
-
-static const struct ivpu_bo_ops internal_ops = {
-	.type = IVPU_BO_TYPE_INTERNAL,
-	.name = "internal",
-	.alloc_pages = internal_alloc_pages_locked,
-	.free_pages = internal_free_pages_locked,
-	.map_pages = ivpu_bo_map_pages_locked,
-	.unmap_pages = ivpu_bo_unmap_pages_locked,
-};
-
-static int __must_check ivpu_bo_alloc_and_map_pages_locked(struct ivpu_bo *bo)
-{
-	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
-	int ret;
-
-	lockdep_assert_held(&bo->lock);
-	drm_WARN_ON(&vdev->drm, bo->sgt);
-
-	ret = bo->ops->alloc_pages(bo);
-	if (ret) {
-		ivpu_err(vdev, "Failed to allocate pages for BO: %d", ret);
-		return ret;
-	}
-
-	ret = bo->ops->map_pages(bo);
-	if (ret) {
-		ivpu_err(vdev, "Failed to map pages for BO: %d", ret);
-		goto err_free_pages;
-	}
-	return ret;
-
-err_free_pages:
-	bo->ops->free_pages(bo);
-	return ret;
-}
-
-static void ivpu_bo_unmap_and_free_pages(struct ivpu_bo *bo)
+static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action)
 {
-	mutex_lock(&bo->lock);
-
-	WARN_ON(!bo->sgt);
-	bo->ops->unmap_pages(bo);
-	WARN_ON(bo->sgt);
-	bo->ops->free_pages(bo);
-	WARN_ON(bo->pages);
-
-	mutex_unlock(&bo->lock);
+	if (bo->ctx)
+		ivpu_dbg(vdev, BO, "%6s: size %zu has_pages %d dma_mapped %d handle %u ctx %d vpu_addr 0x%llx mmu_mapped %d\n",
+			 action, ivpu_bo_size(bo), (bool)bo->base.pages, (bool)bo->base.sgt,
+			 bo->handle, bo->ctx->id, bo->vpu_addr, bo->mmu_mapped);
+	else
+		ivpu_dbg(vdev, BO, "%6s: size %zu has_pages %d dma_mapped %d handle %u (not added to context)\n",
+			 action, ivpu_bo_size(bo), (bool)bo->base.pages, (bool)bo->base.sgt,
+			 bo->handle);
 }
 
 /*
@@ -245,21 +48,24 @@ int __must_check ivpu_bo_pin(struct ivpu_bo *bo)
 
 	mutex_lock(&bo->lock);
 
-	if (!bo->vpu_addr) {
-		ivpu_err(vdev, "vpu_addr not set for BO ctx_id: %d handle: %d\n",
-			 bo->ctx->id, bo->handle);
+	ivpu_dbg_bo(vdev, bo, "pin");
+
+	if (!bo->ctx) {
+		ivpu_err(vdev, "vpu_addr not allocated for BO %d\n", bo->handle);
 		ret = -EINVAL;
 		goto unlock;
 	}
 
-	if (!bo->sgt) {
-		ret = ivpu_bo_alloc_and_map_pages_locked(bo);
-		if (ret)
+	if (!bo->mmu_mapped) {
+		struct sg_table *sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
+
+		if (IS_ERR(sgt)) {
+			ret = PTR_ERR(sgt);
+			ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret);
 			goto unlock;
-	}
+		}
 
-	if (!bo->mmu_mapped) {
-		ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, bo->sgt,
+		ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt,
 					       ivpu_bo_is_snooped(bo));
 		if (ret) {
 			ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret);
@@ -281,248 +87,213 @@ ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx,
 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
 	int ret;
 
-	if (!range) {
-		if (bo->flags & DRM_IVPU_BO_SHAVE_MEM)
-			range = &vdev->hw->ranges.shave;
-		else if (bo->flags & DRM_IVPU_BO_DMA_MEM)
-			range = &vdev->hw->ranges.dma;
-		else
-			range = &vdev->hw->ranges.user;
-	}
+	mutex_lock(&bo->lock);
 
-	mutex_lock(&ctx->lock);
-	ret = ivpu_mmu_context_insert_node_locked(ctx, range, ivpu_bo_size(bo), &bo->mm_node);
+	ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node);
 	if (!ret) {
 		bo->ctx = ctx;
 		bo->vpu_addr = bo->mm_node.start;
-		list_add_tail(&bo->ctx_node, &ctx->bo_list);
+	} else {
+		ivpu_err(vdev, "Failed to add BO to context %u: %d\n", ctx->id, ret);
 	}
-	mutex_unlock(&ctx->lock);
+
+	ivpu_dbg_bo(vdev, bo, "alloc");
+
+	mutex_unlock(&bo->lock);
 
 	return ret;
 }
 
-static void ivpu_bo_free_vpu_addr(struct ivpu_bo *bo)
+static void ivpu_bo_unbind_locked(struct ivpu_bo *bo)
 {
 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
-	struct ivpu_mmu_context *ctx = bo->ctx;
 
-	ivpu_dbg(vdev, BO, "remove from ctx: ctx %d vpu_addr 0x%llx allocated %d mmu_mapped %d\n",
-		 ctx->id, bo->vpu_addr, (bool)bo->sgt, bo->mmu_mapped);
+	lockdep_assert_held(&bo->lock);
 
-	mutex_lock(&bo->lock);
+	ivpu_dbg_bo(vdev, bo, "unbind");
+
+	/* TODO: dma_unmap */
 
 	if (bo->mmu_mapped) {
-		drm_WARN_ON(&vdev->drm, !bo->sgt);
-		ivpu_mmu_context_unmap_sgt(vdev, ctx, bo->vpu_addr, bo->sgt);
+		drm_WARN_ON(&vdev->drm, !bo->ctx);
+		drm_WARN_ON(&vdev->drm, !bo->vpu_addr);
+		drm_WARN_ON(&vdev->drm, !bo->base.sgt);
+		ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt);
 		bo->mmu_mapped = false;
 	}
 
-	mutex_lock(&ctx->lock);
-	list_del(&bo->ctx_node);
-	bo->vpu_addr = 0;
-	bo->ctx = NULL;
-	ivpu_mmu_context_remove_node_locked(ctx, &bo->mm_node);
-	mutex_unlock(&ctx->lock);
+	if (bo->ctx) {
+		ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node);
+		bo->vpu_addr = 0;
+		bo->ctx = NULL;
+	}
+}
 
+static void ivpu_bo_unbind(struct ivpu_bo *bo)
+{
+	mutex_lock(&bo->lock);
+	ivpu_bo_unbind_locked(bo);
 	mutex_unlock(&bo->lock);
 }
 
-void ivpu_bo_remove_all_bos_from_context(struct ivpu_mmu_context *ctx)
+void ivpu_bo_remove_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
 {
-	struct ivpu_bo *bo, *tmp;
+	struct ivpu_bo *bo;
+
+	if (drm_WARN_ON(&vdev->drm, !ctx))
+		return;
 
-	list_for_each_entry_safe(bo, tmp, &ctx->bo_list, ctx_node)
-		ivpu_bo_free_vpu_addr(bo);
+	mutex_lock(&vdev->bo_list_lock);
+	list_for_each_entry(bo, &vdev->bo_list, bo_list_node) {
+		mutex_lock(&bo->lock);
+		if (bo->ctx == ctx)
+			ivpu_bo_unbind_locked(bo);
+		mutex_unlock(&bo->lock);
+	}
+	mutex_unlock(&vdev->bo_list_lock);
 }
 
-static struct ivpu_bo *
-ivpu_bo_alloc(struct ivpu_device *vdev, struct ivpu_mmu_context *mmu_context,
-	      u64 size, u32 flags, const struct ivpu_bo_ops *ops,
-	      const struct ivpu_addr_range *range, u64 user_ptr)
+struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size)
 {
 	struct ivpu_bo *bo;
-	int ret = 0;
-
-	if (drm_WARN_ON(&vdev->drm, size == 0 || !PAGE_ALIGNED(size)))
-		return ERR_PTR(-EINVAL);
 
-	switch (flags & DRM_IVPU_BO_CACHE_MASK) {
-	case DRM_IVPU_BO_CACHED:
-	case DRM_IVPU_BO_UNCACHED:
-	case DRM_IVPU_BO_WC:
-		break;
-	default:
+	if (size == 0 || !PAGE_ALIGNED(size))
 		return ERR_PTR(-EINVAL);
-	}
 
 	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
 	if (!bo)
 		return ERR_PTR(-ENOMEM);
 
-	mutex_init(&bo->lock);
-	bo->base.funcs = &ivpu_gem_funcs;
-	bo->flags = flags;
-	bo->ops = ops;
-	bo->user_ptr = user_ptr;
-
-	if (ops->type == IVPU_BO_TYPE_SHMEM)
-		ret = drm_gem_object_init(&vdev->drm, &bo->base, size);
-	else
-		drm_gem_private_object_init(&vdev->drm, &bo->base, size);
-
-	if (ret) {
-		ivpu_err(vdev, "Failed to initialize drm object\n");
-		goto err_free;
-	}
-
-	if (flags & DRM_IVPU_BO_MAPPABLE) {
-		ret = drm_gem_create_mmap_offset(&bo->base);
-		if (ret) {
-			ivpu_err(vdev, "Failed to allocate mmap offset\n");
-			goto err_release;
-		}
-	}
-
-	if (mmu_context) {
-		ret = ivpu_bo_alloc_vpu_addr(bo, mmu_context, range);
-		if (ret) {
-			ivpu_err(vdev, "Failed to add BO to context: %d\n", ret);
-			goto err_release;
-		}
-	}
+	bo->base.base.funcs = &ivpu_gem_funcs;
+	bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */
 
-	return bo;
+	INIT_LIST_HEAD(&bo->bo_list_node);
+	mutex_init(&bo->lock);
 
-err_release:
-	drm_gem_object_release(&bo->base);
-err_free:
-	kfree(bo);
-	return ERR_PTR(ret);
+	return &bo->base.base;
 }
 
-static void ivpu_bo_free(struct drm_gem_object *obj)
+static struct ivpu_bo *
+ivpu_bo_create(struct ivpu_device *vdev, u64 size, u32 flags)
 {
-	struct ivpu_bo *bo = to_ivpu_bo(obj);
-	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
-
-	if (bo->ctx)
-		ivpu_dbg(vdev, BO, "free: ctx %d vpu_addr 0x%llx allocated %d mmu_mapped %d\n",
-			 bo->ctx->id, bo->vpu_addr, (bool)bo->sgt, bo->mmu_mapped);
-	else
-		ivpu_dbg(vdev, BO, "free: ctx (released) allocated %d mmu_mapped %d\n",
-			 (bool)bo->sgt, bo->mmu_mapped);
-
-	drm_WARN_ON(&vdev->drm, !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ));
+	struct drm_gem_shmem_object *shmem;
+	struct ivpu_bo *bo;
 
-	vunmap(bo->kvaddr);
+	switch (flags & DRM_IVPU_BO_CACHE_MASK) {
+	case DRM_IVPU_BO_CACHED:
+	case DRM_IVPU_BO_WC:
+		break;
+	default:
+		return ERR_PTR(-EINVAL);
+	}
 
-	if (bo->ctx)
-		ivpu_bo_free_vpu_addr(bo);
+	shmem = drm_gem_shmem_create(&vdev->drm, size);
+	if (IS_ERR(shmem))
+		return ERR_CAST(shmem);
 
-	if (bo->sgt)
-		ivpu_bo_unmap_and_free_pages(bo);
+	bo = to_ivpu_bo(&shmem->base);
+	bo->base.map_wc = flags & DRM_IVPU_BO_WC;
+	bo->flags = flags;
 
-	if (bo->base.import_attach)
-		drm_prime_gem_destroy(&bo->base, bo->sgt);
+	mutex_lock(&vdev->bo_list_lock);
+	list_add_tail(&bo->bo_list_node, &vdev->bo_list);
+	mutex_unlock(&vdev->bo_list_lock);
 
-	drm_gem_object_release(&bo->base);
+	ivpu_dbg(vdev, BO, "create: vpu_addr 0x%llx size %zu flags 0x%x\n",
+		 bo->vpu_addr, bo->base.base.size, flags);
 
-	mutex_destroy(&bo->lock);
-	kfree(bo);
+	return bo;
 }
 
-static int ivpu_bo_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
+static int ivpu_bo_open(struct drm_gem_object *obj, struct drm_file *file)
 {
+	struct ivpu_file_priv *file_priv = file->driver_priv;
+	struct ivpu_device *vdev = file_priv->vdev;
 	struct ivpu_bo *bo = to_ivpu_bo(obj);
-	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
-
-	ivpu_dbg(vdev, BO, "mmap: ctx %u handle %u vpu_addr 0x%llx size %zu type %s",
-		 bo->ctx->id, bo->handle, bo->vpu_addr, ivpu_bo_size(bo), bo->ops->name);
+	struct ivpu_addr_range *range;
 
-	if (obj->import_attach) {
-		/* Drop the reference drm_gem_mmap_obj() acquired.*/
-		drm_gem_object_put(obj);
-		vma->vm_private_data = NULL;
-		return dma_buf_mmap(obj->dma_buf, vma, 0);
-	}
-
-	vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND);
-	vma->vm_page_prot = ivpu_bo_pgprot(bo, vm_get_page_prot(vma->vm_flags));
+	if (bo->flags & DRM_IVPU_BO_SHAVE_MEM)
+		range = &vdev->hw->ranges.shave;
+	else if (bo->flags & DRM_IVPU_BO_DMA_MEM)
+		range = &vdev->hw->ranges.dma;
+	else
+		range = &vdev->hw->ranges.user;
 
-	return 0;
+	return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range);
 }
 
-static struct sg_table *ivpu_bo_get_sg_table(struct drm_gem_object *obj)
+static void ivpu_bo_free(struct drm_gem_object *obj)
 {
+	struct ivpu_device *vdev = to_ivpu_device(obj->dev);
 	struct ivpu_bo *bo = to_ivpu_bo(obj);
-	loff_t npages = obj->size >> PAGE_SHIFT;
-	int ret = 0;
 
-	mutex_lock(&bo->lock);
+	mutex_lock(&vdev->bo_list_lock);
+	list_del(&bo->bo_list_node);
+	mutex_unlock(&vdev->bo_list_lock);
 
-	if (!bo->sgt)
-		ret = ivpu_bo_alloc_and_map_pages_locked(bo);
+	drm_WARN_ON(&vdev->drm, !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ));
 
-	mutex_unlock(&bo->lock);
+	ivpu_dbg_bo(vdev, bo, "free");
 
-	if (ret)
-		return ERR_PTR(ret);
+	ivpu_bo_unbind(bo);
+	mutex_destroy(&bo->lock);
 
-	return drm_prime_pages_to_sg(obj->dev, bo->pages, npages);
+	drm_WARN_ON(obj->dev, bo->base.pages_use_count > 1);
+	drm_gem_shmem_free(&bo->base);
 }
 
-static vm_fault_t ivpu_vm_fault(struct vm_fault *vmf)
-{
-	struct vm_area_struct *vma = vmf->vma;
-	struct drm_gem_object *obj = vma->vm_private_data;
-	struct ivpu_bo *bo = to_ivpu_bo(obj);
-	loff_t npages = obj->size >> PAGE_SHIFT;
-	pgoff_t page_offset;
-	struct page *page;
-	vm_fault_t ret;
-	int err;
-
-	mutex_lock(&bo->lock);
-
-	if (!bo->sgt) {
-		err = ivpu_bo_alloc_and_map_pages_locked(bo);
-		if (err) {
-			ret = vmf_error(err);
-			goto unlock;
-		}
-	}
-
-	/* We don't use vmf->pgoff since that has the fake offset */
-	page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
-	if (page_offset >= npages) {
-		ret = VM_FAULT_SIGBUS;
-	} else {
-		page = bo->pages[page_offset];
-		ret = vmf_insert_pfn(vma, vmf->address, page_to_pfn(page));
-	}
-
-unlock:
-	mutex_unlock(&bo->lock);
+static const struct dma_buf_ops ivpu_bo_dmabuf_ops =  {
+	.cache_sgt_mapping = true,
+	.attach = drm_gem_map_attach,
+	.detach = drm_gem_map_detach,
+	.map_dma_buf = drm_gem_map_dma_buf,
+	.unmap_dma_buf = drm_gem_unmap_dma_buf,
+	.release = drm_gem_dmabuf_release,
+	.mmap = drm_gem_dmabuf_mmap,
+	.vmap = drm_gem_dmabuf_vmap,
+	.vunmap = drm_gem_dmabuf_vunmap,
+};
 
-	return ret;
+static struct dma_buf *ivpu_bo_export(struct drm_gem_object *obj, int flags)
+{
+	struct drm_device *dev = obj->dev;
+	struct dma_buf_export_info exp_info = {
+		.exp_name = KBUILD_MODNAME,
+		.owner = dev->driver->fops->owner,
+		.ops = &ivpu_bo_dmabuf_ops,
+		.size = obj->size,
+		.flags = flags,
+		.priv = obj,
+		.resv = obj->resv,
+	};
+	void *sgt;
+
+	/*
+	 * Make sure that pages are allocated and dma-mapped before exporting the bo.
+	 * DMA-mapping is required if the bo will be imported to the same device.
+	 */
+	sgt = drm_gem_shmem_get_pages_sgt(to_drm_gem_shmem_obj(obj));
+	if (IS_ERR(sgt))
+		return sgt;
+
+	return drm_gem_dmabuf_export(dev, &exp_info);
 }
 
-static const struct vm_operations_struct ivpu_vm_ops = {
-	.fault = ivpu_vm_fault,
-	.open = drm_gem_vm_open,
-	.close = drm_gem_vm_close,
-};
-
 static const struct drm_gem_object_funcs ivpu_gem_funcs = {
 	.free = ivpu_bo_free,
-	.mmap = ivpu_bo_mmap,
-	.vm_ops = &ivpu_vm_ops,
-	.get_sg_table = ivpu_bo_get_sg_table,
+	.open = ivpu_bo_open,
+	.export = ivpu_bo_export,
+	.print_info = drm_gem_shmem_object_print_info,
+	.pin = drm_gem_shmem_object_pin,
+	.unpin = drm_gem_shmem_object_unpin,
+	.get_sg_table = drm_gem_shmem_object_get_sg_table,
+	.vmap = drm_gem_shmem_object_vmap,
+	.vunmap = drm_gem_shmem_object_vunmap,
+	.mmap = drm_gem_shmem_object_mmap,
+	.vm_ops = &drm_gem_shmem_vm_ops,
 };
 
-int
-ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
+int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 {
 	struct ivpu_file_priv *file_priv = file->driver_priv;
 	struct ivpu_device *vdev = file_priv->vdev;
@@ -537,23 +308,20 @@ ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 	if (size == 0)
 		return -EINVAL;
 
-	bo = ivpu_bo_alloc(vdev, &file_priv->ctx, size, args->flags, &shmem_ops, NULL, 0);
+	bo = ivpu_bo_create(vdev, size, args->flags);
 	if (IS_ERR(bo)) {
 		ivpu_err(vdev, "Failed to create BO: %pe (ctx %u size %llu flags 0x%x)",
 			 bo, file_priv->ctx.id, args->size, args->flags);
 		return PTR_ERR(bo);
 	}
 
-	ret = drm_gem_handle_create(file, &bo->base, &bo->handle);
+	ret = drm_gem_handle_create(file, &bo->base.base, &bo->handle);
 	if (!ret) {
 		args->vpu_addr = bo->vpu_addr;
 		args->handle = bo->handle;
 	}
 
-	drm_gem_object_put(&bo->base);
-
-	ivpu_dbg(vdev, BO, "alloc shmem: ctx %u vpu_addr 0x%llx size %zu flags 0x%x\n",
-		 file_priv->ctx.id, bo->vpu_addr, ivpu_bo_size(bo), bo->flags);
+	drm_gem_object_put(&bo->base.base);
 
 	return ret;
 }
@@ -563,8 +331,8 @@ ivpu_bo_alloc_internal(struct ivpu_device *vdev, u64 vpu_addr, u64 size, u32 fla
 {
 	const struct ivpu_addr_range *range;
 	struct ivpu_addr_range fixed_range;
+	struct iosys_map map;
 	struct ivpu_bo *bo;
-	pgprot_t prot;
 	int ret;
 
 	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(vpu_addr));
@@ -578,81 +346,42 @@ ivpu_bo_alloc_internal(struct ivpu_device *vdev, u64 vpu_addr, u64 size, u32 fla
 		range = &vdev->hw->ranges.global;
 	}
 
-	bo = ivpu_bo_alloc(vdev, &vdev->gctx, size, flags, &internal_ops, range, 0);
+	bo = ivpu_bo_create(vdev, size, flags);
 	if (IS_ERR(bo)) {
 		ivpu_err(vdev, "Failed to create BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)",
 			 bo, vpu_addr, size, flags);
 		return NULL;
 	}
 
-	ret = ivpu_bo_pin(bo);
+	ret = ivpu_bo_alloc_vpu_addr(bo, &vdev->gctx, range);
 	if (ret)
 		goto err_put;
 
-	if (ivpu_bo_cache_mode(bo) != DRM_IVPU_BO_CACHED)
-		drm_clflush_pages(bo->pages, ivpu_bo_size(bo) >> PAGE_SHIFT);
-
-	if (bo->flags & DRM_IVPU_BO_WC)
-		set_pages_array_wc(bo->pages, ivpu_bo_size(bo) >> PAGE_SHIFT);
-	else if (bo->flags & DRM_IVPU_BO_UNCACHED)
-		set_pages_array_uc(bo->pages, ivpu_bo_size(bo) >> PAGE_SHIFT);
-
-	prot = ivpu_bo_pgprot(bo, PAGE_KERNEL);
-	bo->kvaddr = vmap(bo->pages, ivpu_bo_size(bo) >> PAGE_SHIFT, VM_MAP, prot);
-	if (!bo->kvaddr) {
-		ivpu_err(vdev, "Failed to map BO into kernel virtual memory\n");
+	ret = ivpu_bo_pin(bo);
+	if (ret)
 		goto err_put;
-	}
 
-	ivpu_dbg(vdev, BO, "alloc internal: ctx 0 vpu_addr 0x%llx size %zu flags 0x%x\n",
-		 bo->vpu_addr, ivpu_bo_size(bo), flags);
+	ret = drm_gem_shmem_vmap(&bo->base, &map);
+	if (ret)
+		goto err_put;
 
 	return bo;
 
 err_put:
-	drm_gem_object_put(&bo->base);
+	drm_gem_object_put(&bo->base.base);
 	return NULL;
 }
 
 void ivpu_bo_free_internal(struct ivpu_bo *bo)
 {
-	drm_gem_object_put(&bo->base);
-}
-
-struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev, struct dma_buf *buf)
-{
-	struct ivpu_device *vdev = to_ivpu_device(dev);
-	struct dma_buf_attachment *attach;
-	struct ivpu_bo *bo;
-
-	attach = dma_buf_attach(buf, dev->dev);
-	if (IS_ERR(attach))
-		return ERR_CAST(attach);
-
-	get_dma_buf(buf);
+	struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr);
 
-	bo = ivpu_bo_alloc(vdev, NULL, buf->size, DRM_IVPU_BO_MAPPABLE, &prime_ops, NULL, 0);
-	if (IS_ERR(bo)) {
-		ivpu_err(vdev, "Failed to import BO: %pe (size %lu)", bo, buf->size);
-		goto err_detach;
-	}
-
-	lockdep_set_class(&bo->lock, &prime_bo_lock_class_key);
-
-	bo->base.import_attach = attach;
-
-	return &bo->base;
-
-err_detach:
-	dma_buf_detach(buf, attach);
-	dma_buf_put(buf);
-	return ERR_CAST(bo);
+	drm_gem_shmem_vunmap(&bo->base, &map);
+	drm_gem_object_put(&bo->base.base);
 }
 
 int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 {
-	struct ivpu_file_priv *file_priv = file->driver_priv;
-	struct ivpu_device *vdev = to_ivpu_device(dev);
 	struct drm_ivpu_bo_info *args = data;
 	struct drm_gem_object *obj;
 	struct ivpu_bo *bo;
@@ -665,21 +394,12 @@ int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file
 	bo = to_ivpu_bo(obj);
 
 	mutex_lock(&bo->lock);
-
-	if (!bo->ctx) {
-		ret = ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, NULL);
-		if (ret) {
-			ivpu_err(vdev, "Failed to allocate vpu_addr: %d\n", ret);
-			goto unlock;
-		}
-	}
-
 	args->flags = bo->flags;
 	args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node);
 	args->vpu_addr = bo->vpu_addr;
 	args->size = obj->size;
-unlock:
 	mutex_unlock(&bo->lock);
+
 	drm_gem_object_put(obj);
 	return ret;
 }
@@ -714,41 +434,41 @@ static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p)
 {
 	unsigned long dma_refcount = 0;
 
-	if (bo->base.dma_buf && bo->base.dma_buf->file)
-		dma_refcount = atomic_long_read(&bo->base.dma_buf->file->f_count);
+	mutex_lock(&bo->lock);
+
+	if (bo->base.base.dma_buf && bo->base.base.dma_buf->file)
+		dma_refcount = atomic_long_read(&bo->base.base.dma_buf->file->f_count);
+
+	drm_printf(p, "%-3u %-6d 0x%-12llx %-10lu 0x%-8x %-4u %-8lu",
+		   bo->ctx->id, bo->handle, bo->vpu_addr, bo->base.base.size,
+		   bo->flags, kref_read(&bo->base.base.refcount), dma_refcount);
+
+	if (bo->base.base.import_attach)
+		drm_printf(p, " imported");
+
+	if (bo->base.pages)
+		drm_printf(p, " has_pages");
 
-	drm_printf(p, "%5u %6d %16llx %10lu %10u %12lu %14s\n",
-		   bo->ctx->id, bo->handle, bo->vpu_addr, ivpu_bo_size(bo),
-		   kref_read(&bo->base.refcount), dma_refcount, bo->ops->name);
+	if (bo->mmu_mapped)
+		drm_printf(p, " mmu_mapped");
+
+	drm_printf(p, "\n");
+
+	mutex_unlock(&bo->lock);
 }
 
 void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p)
 {
 	struct ivpu_device *vdev = to_ivpu_device(dev);
-	struct ivpu_file_priv *file_priv;
-	unsigned long ctx_id;
 	struct ivpu_bo *bo;
 
-	drm_printf(p, "%5s %6s %16s %10s %10s %12s %14s\n",
-		   "ctx", "handle", "vpu_addr", "size", "refcount", "dma_refcount", "type");
+	drm_printf(p, "%-3s %-6s %-14s %-10s %-10s %-4s %-8s %s\n",
+		   "ctx", "handle", "vpu_addr", "size", "flags", "refs", "dma_refs", "attribs");
 
-	mutex_lock(&vdev->gctx.lock);
-	list_for_each_entry(bo, &vdev->gctx.bo_list, ctx_node)
+	mutex_lock(&vdev->bo_list_lock);
+	list_for_each_entry(bo, &vdev->bo_list, bo_list_node)
 		ivpu_bo_print_info(bo, p);
-	mutex_unlock(&vdev->gctx.lock);
-
-	xa_for_each(&vdev->context_xa, ctx_id, file_priv) {
-		file_priv = ivpu_file_priv_get_by_ctx_id(vdev, ctx_id);
-		if (!file_priv)
-			continue;
-
-		mutex_lock(&file_priv->ctx.lock);
-		list_for_each_entry(bo, &file_priv->ctx.bo_list, ctx_node)
-			ivpu_bo_print_info(bo, p);
-		mutex_unlock(&file_priv->ctx.lock);
-
-		ivpu_file_priv_put(&file_priv);
-	}
+	mutex_unlock(&vdev->bo_list_lock);
 }
 
 void ivpu_bo_list_print(struct drm_device *dev)
diff --git a/drivers/accel/ivpu/ivpu_gem.h b/drivers/accel/ivpu/ivpu_gem.h
index a0b4d4a32b3b..d75cad0d3c74 100644
--- a/drivers/accel/ivpu/ivpu_gem.h
+++ b/drivers/accel/ivpu/ivpu_gem.h
@@ -6,84 +6,52 @@
 #define __IVPU_GEM_H__
 
 #include <drm/drm_gem.h>
+#include <drm/drm_gem_shmem_helper.h>
 #include <drm/drm_mm.h>
 
-struct dma_buf;
-struct ivpu_bo_ops;
 struct ivpu_file_priv;
 
 struct ivpu_bo {
-	struct drm_gem_object base;
-	const struct ivpu_bo_ops *ops;
-
+	struct drm_gem_shmem_object base;
 	struct ivpu_mmu_context *ctx;
-	struct list_head ctx_node;
+	struct list_head bo_list_node;
 	struct drm_mm_node mm_node;
 
-	struct mutex lock; /* Protects: pages, sgt, mmu_mapped */
-	struct sg_table *sgt;
-	struct page **pages;
-	bool mmu_mapped;
-
-	void *kvaddr;
+	struct mutex lock; /* Protects: ctx, mmu_mapped, vpu_addr */
 	u64 vpu_addr;
 	u32 handle;
 	u32 flags;
-	uintptr_t user_ptr;
-	u32 job_status;
-};
-
-enum ivpu_bo_type {
-	IVPU_BO_TYPE_SHMEM = 1,
-	IVPU_BO_TYPE_INTERNAL,
-	IVPU_BO_TYPE_PRIME,
-};
-
-struct ivpu_bo_ops {
-	enum ivpu_bo_type type;
-	const char *name;
-	int (*alloc_pages)(struct ivpu_bo *bo);
-	void (*free_pages)(struct ivpu_bo *bo);
-	int (*map_pages)(struct ivpu_bo *bo);
-	void (*unmap_pages)(struct ivpu_bo *bo);
+	u32 job_status; /* Valid only for command buffer */
+	bool mmu_mapped;
 };
 
 int ivpu_bo_pin(struct ivpu_bo *bo);
-void ivpu_bo_remove_all_bos_from_context(struct ivpu_mmu_context *ctx);
-void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p);
-void ivpu_bo_list_print(struct drm_device *dev);
+void ivpu_bo_remove_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx);
 
-struct ivpu_bo *
-ivpu_bo_alloc_internal(struct ivpu_device *vdev, u64 vpu_addr, u64 size, u32 flags);
+struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size);
+struct ivpu_bo *ivpu_bo_alloc_internal(struct ivpu_device *vdev, u64 vpu_addr, u64 size, u32 flags);
 void ivpu_bo_free_internal(struct ivpu_bo *bo);
-struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf);
-void ivpu_bo_unmap_sgt_and_remove_from_context(struct ivpu_bo *bo);
 
 int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file);
 int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file);
 int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file);
 
+void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p);
+void ivpu_bo_list_print(struct drm_device *dev);
+
 static inline struct ivpu_bo *to_ivpu_bo(struct drm_gem_object *obj)
 {
-	return container_of(obj, struct ivpu_bo, base);
+	return container_of(obj, struct ivpu_bo, base.base);
 }
 
 static inline void *ivpu_bo_vaddr(struct ivpu_bo *bo)
 {
-	return bo->kvaddr;
+	return bo->base.vaddr;
 }
 
 static inline size_t ivpu_bo_size(struct ivpu_bo *bo)
 {
-	return bo->base.size;
-}
-
-static inline struct page *ivpu_bo_get_page(struct ivpu_bo *bo, u64 offset)
-{
-	if (offset > ivpu_bo_size(bo) || !bo->pages)
-		return NULL;
-
-	return bo->pages[offset / PAGE_SIZE];
+	return bo->base.base.size;
 }
 
 static inline u32 ivpu_bo_cache_mode(struct ivpu_bo *bo)
@@ -96,20 +64,9 @@ static inline bool ivpu_bo_is_snooped(struct ivpu_bo *bo)
 	return ivpu_bo_cache_mode(bo) == DRM_IVPU_BO_CACHED;
 }
 
-static inline pgprot_t ivpu_bo_pgprot(struct ivpu_bo *bo, pgprot_t prot)
-{
-	if (bo->flags & DRM_IVPU_BO_WC)
-		return pgprot_writecombine(prot);
-
-	if (bo->flags & DRM_IVPU_BO_UNCACHED)
-		return pgprot_noncached(prot);
-
-	return prot;
-}
-
 static inline struct ivpu_device *ivpu_bo_to_vdev(struct ivpu_bo *bo)
 {
-	return to_ivpu_device(bo->base.dev);
+	return to_ivpu_device(bo->base.base.dev);
 }
 
 static inline void *ivpu_to_cpu_addr(struct ivpu_bo *bo, u32 vpu_addr)
diff --git a/drivers/accel/ivpu/ivpu_hw.h b/drivers/accel/ivpu/ivpu_hw.h
index 1079e06255ba..b2909168a0a6 100644
--- a/drivers/accel/ivpu/ivpu_hw.h
+++ b/drivers/accel/ivpu/ivpu_hw.h
@@ -15,8 +15,11 @@ struct ivpu_hw_ops {
 	int (*power_down)(struct ivpu_device *vdev);
 	int (*reset)(struct ivpu_device *vdev);
 	bool (*is_idle)(struct ivpu_device *vdev);
+	int (*wait_for_idle)(struct ivpu_device *vdev);
 	void (*wdt_disable)(struct ivpu_device *vdev);
 	void (*diagnose_failure)(struct ivpu_device *vdev);
+	u32 (*profiling_freq_get)(struct ivpu_device *vdev);
+	void (*profiling_freq_drive)(struct ivpu_device *vdev, bool enable);
 	u32 (*reg_pll_freq_get)(struct ivpu_device *vdev);
 	u32 (*reg_telemetry_offset_get)(struct ivpu_device *vdev);
 	u32 (*reg_telemetry_size_get)(struct ivpu_device *vdev);
@@ -58,6 +61,8 @@ struct ivpu_hw_info {
 	u32 sku;
 	u16 config;
 	int dma_bits;
+	ktime_t d0i3_entry_host_ts;
+	u64 d0i3_entry_vpu_ts;
 };
 
 extern const struct ivpu_hw_ops ivpu_hw_37xx_ops;
@@ -85,6 +90,11 @@ static inline bool ivpu_hw_is_idle(struct ivpu_device *vdev)
 	return vdev->hw->ops->is_idle(vdev);
 };
 
+static inline int ivpu_hw_wait_for_idle(struct ivpu_device *vdev)
+{
+	return vdev->hw->ops->wait_for_idle(vdev);
+};
+
 static inline int ivpu_hw_power_down(struct ivpu_device *vdev)
 {
 	ivpu_dbg(vdev, PM, "HW power down\n");
@@ -104,6 +114,16 @@ static inline void ivpu_hw_wdt_disable(struct ivpu_device *vdev)
 	vdev->hw->ops->wdt_disable(vdev);
 };
 
+static inline u32 ivpu_hw_profiling_freq_get(struct ivpu_device *vdev)
+{
+	return vdev->hw->ops->profiling_freq_get(vdev);
+};
+
+static inline void ivpu_hw_profiling_freq_drive(struct ivpu_device *vdev, bool enable)
+{
+	return vdev->hw->ops->profiling_freq_drive(vdev, enable);
+};
+
 /* Register indirect accesses */
 static inline u32 ivpu_hw_reg_pll_freq_get(struct ivpu_device *vdev)
 {
diff --git a/drivers/accel/ivpu/ivpu_hw_37xx.c b/drivers/accel/ivpu/ivpu_hw_37xx.c
index 5c0246b9e522..050f068a0566 100644
--- a/drivers/accel/ivpu/ivpu_hw_37xx.c
+++ b/drivers/accel/ivpu/ivpu_hw_37xx.c
@@ -29,6 +29,7 @@
 
 #define PLL_REF_CLK_FREQ	     (50 * 1000000)
 #define PLL_SIMULATION_FREQ	     (10 * 1000000)
+#define PLL_PROF_CLK_FREQ	     (38400 * 1000)
 #define PLL_DEFAULT_EPP_VALUE	     0x80
 
 #define TIM_SAFE_ENABLE		     0xf1d0dead
@@ -37,7 +38,7 @@
 #define TIMEOUT_US		     (150 * USEC_PER_MSEC)
 #define PWR_ISLAND_STATUS_TIMEOUT_US (5 * USEC_PER_MSEC)
 #define PLL_TIMEOUT_US		     (1500 * USEC_PER_MSEC)
-#define IDLE_TIMEOUT_US		     (500 * USEC_PER_MSEC)
+#define IDLE_TIMEOUT_US		     (5 * USEC_PER_MSEC)
 
 #define ICB_0_IRQ_MASK ((REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT)) | \
 			(REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT)) | \
@@ -90,6 +91,7 @@ static void ivpu_hw_timeouts_init(struct ivpu_device *vdev)
 	vdev->timeout.tdr = 2000;
 	vdev->timeout.reschedule_suspend = 10;
 	vdev->timeout.autosuspend = 10;
+	vdev->timeout.d0i3_entry_msg = 5;
 }
 
 static int ivpu_pll_wait_for_cmd_send(struct ivpu_device *vdev)
@@ -502,6 +504,16 @@ static int ivpu_boot_pwr_domain_enable(struct ivpu_device *vdev)
 	return ret;
 }
 
+static int ivpu_boot_pwr_domain_disable(struct ivpu_device *vdev)
+{
+	ivpu_boot_dpu_active_drive(vdev, false);
+	ivpu_boot_pwr_island_isolation_drive(vdev, true);
+	ivpu_boot_pwr_island_trickle_drive(vdev, false);
+	ivpu_boot_pwr_island_drive(vdev, false);
+
+	return ivpu_boot_wait_for_pwr_island_status(vdev, 0x0);
+}
+
 static void ivpu_boot_no_snoop_enable(struct ivpu_device *vdev)
 {
 	u32 val = REGV_RD32(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES);
@@ -600,25 +612,17 @@ static int ivpu_hw_37xx_info_init(struct ivpu_device *vdev)
 
 static int ivpu_hw_37xx_reset(struct ivpu_device *vdev)
 {
-	int ret;
-	u32 val;
-
-	if (IVPU_WA(punit_disabled))
-		return 0;
+	int ret = 0;
 
-	ret = REGB_POLL_FLD(VPU_37XX_BUTTRESS_VPU_IP_RESET, TRIGGER, 0, TIMEOUT_US);
-	if (ret) {
-		ivpu_err(vdev, "Timed out waiting for TRIGGER bit\n");
-		return ret;
+	if (ivpu_boot_pwr_domain_disable(vdev)) {
+		ivpu_err(vdev, "Failed to disable power domain\n");
+		ret = -EIO;
 	}
 
-	val = REGB_RD32(VPU_37XX_BUTTRESS_VPU_IP_RESET);
-	val = REG_SET_FLD(VPU_37XX_BUTTRESS_VPU_IP_RESET, TRIGGER, val);
-	REGB_WR32(VPU_37XX_BUTTRESS_VPU_IP_RESET, val);
-
-	ret = REGB_POLL_FLD(VPU_37XX_BUTTRESS_VPU_IP_RESET, TRIGGER, 0, TIMEOUT_US);
-	if (ret)
-		ivpu_err(vdev, "Timed out waiting for RESET completion\n");
+	if (ivpu_pll_disable(vdev)) {
+		ivpu_err(vdev, "Failed to disable PLL\n");
+		ret = -EIO;
+	}
 
 	return ret;
 }
@@ -651,10 +655,6 @@ static int ivpu_hw_37xx_power_up(struct ivpu_device *vdev)
 {
 	int ret;
 
-	ret = ivpu_hw_37xx_reset(vdev);
-	if (ret)
-		ivpu_warn(vdev, "Failed to reset HW: %d\n", ret);
-
 	ret = ivpu_hw_37xx_d0i3_disable(vdev);
 	if (ret)
 		ivpu_warn(vdev, "Failed to disable D0I3: %d\n", ret);
@@ -718,16 +718,27 @@ static bool ivpu_hw_37xx_is_idle(struct ivpu_device *vdev)
 	       REG_TEST_FLD(VPU_37XX_BUTTRESS_VPU_STATUS, IDLE, val);
 }
 
+static int ivpu_hw_37xx_wait_for_idle(struct ivpu_device *vdev)
+{
+	return REGB_POLL_FLD(VPU_37XX_BUTTRESS_VPU_STATUS, IDLE, 0x1, IDLE_TIMEOUT_US);
+}
+
+static void ivpu_hw_37xx_save_d0i3_entry_timestamp(struct ivpu_device *vdev)
+{
+	vdev->hw->d0i3_entry_host_ts = ktime_get_boottime();
+	vdev->hw->d0i3_entry_vpu_ts = REGV_RD64(VPU_37XX_CPU_SS_TIM_PERF_FREE_CNT);
+}
+
 static int ivpu_hw_37xx_power_down(struct ivpu_device *vdev)
 {
 	int ret = 0;
 
-	if (!ivpu_hw_37xx_is_idle(vdev) && ivpu_hw_37xx_reset(vdev))
-		ivpu_err(vdev, "Failed to reset the VPU\n");
+	ivpu_hw_37xx_save_d0i3_entry_timestamp(vdev);
 
-	if (ivpu_pll_disable(vdev)) {
-		ivpu_err(vdev, "Failed to disable PLL\n");
-		ret = -EIO;
+	if (!ivpu_hw_37xx_is_idle(vdev)) {
+		ivpu_warn(vdev, "VPU not idle during power down\n");
+		if (ivpu_hw_37xx_reset(vdev))
+			ivpu_warn(vdev, "Failed to reset the VPU\n");
 	}
 
 	if (ivpu_hw_37xx_d0i3_enable(vdev)) {
@@ -756,6 +767,16 @@ static void ivpu_hw_37xx_wdt_disable(struct ivpu_device *vdev)
 	REGV_WR32(VPU_37XX_CPU_SS_TIM_GEN_CONFIG, val);
 }
 
+static u32 ivpu_hw_37xx_profiling_freq_get(struct ivpu_device *vdev)
+{
+	return PLL_PROF_CLK_FREQ;
+}
+
+static void ivpu_hw_37xx_profiling_freq_drive(struct ivpu_device *vdev, bool enable)
+{
+	/* Profiling freq - is a debug feature. Unavailable on VPU 37XX. */
+}
+
 static u32 ivpu_hw_37xx_pll_to_freq(u32 ratio, u32 config)
 {
 	u32 pll_clock = PLL_REF_CLK_FREQ * ratio;
@@ -867,17 +888,20 @@ static void ivpu_hw_37xx_irq_noc_firewall_handler(struct ivpu_device *vdev)
 }
 
 /* Handler for IRQs from VPU core (irqV) */
-static u32 ivpu_hw_37xx_irqv_handler(struct ivpu_device *vdev, int irq)
+static bool ivpu_hw_37xx_irqv_handler(struct ivpu_device *vdev, int irq, bool *wake_thread)
 {
 	u32 status = REGV_RD32(VPU_37XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK;
 
+	if (!status)
+		return false;
+
 	REGV_WR32(VPU_37XX_HOST_SS_ICB_CLEAR_0, status);
 
 	if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT, status))
 		ivpu_mmu_irq_evtq_handler(vdev);
 
 	if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT, status))
-		ivpu_ipc_irq_handler(vdev);
+		ivpu_ipc_irq_handler(vdev, wake_thread);
 
 	if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT, status))
 		ivpu_dbg(vdev, IRQ, "MMU sync complete\n");
@@ -894,17 +918,17 @@ static u32 ivpu_hw_37xx_irqv_handler(struct ivpu_device *vdev, int irq)
 	if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, status))
 		ivpu_hw_37xx_irq_noc_firewall_handler(vdev);
 
-	return status;
+	return true;
 }
 
 /* Handler for IRQs from Buttress core (irqB) */
-static u32 ivpu_hw_37xx_irqb_handler(struct ivpu_device *vdev, int irq)
+static bool ivpu_hw_37xx_irqb_handler(struct ivpu_device *vdev, int irq)
 {
 	u32 status = REGB_RD32(VPU_37XX_BUTTRESS_INTERRUPT_STAT) & BUTTRESS_IRQ_MASK;
 	bool schedule_recovery = false;
 
-	if (status == 0)
-		return 0;
+	if (!status)
+		return false;
 
 	if (REG_TEST_FLD(VPU_37XX_BUTTRESS_INTERRUPT_STAT, FREQ_CHANGE, status))
 		ivpu_dbg(vdev, IRQ, "FREQ_CHANGE irq: %08x",
@@ -940,23 +964,27 @@ static u32 ivpu_hw_37xx_irqb_handler(struct ivpu_device *vdev, int irq)
 	if (schedule_recovery)
 		ivpu_pm_schedule_recovery(vdev);
 
-	return status;
+	return true;
 }
 
 static irqreturn_t ivpu_hw_37xx_irq_handler(int irq, void *ptr)
 {
 	struct ivpu_device *vdev = ptr;
-	u32 ret_irqv, ret_irqb;
+	bool irqv_handled, irqb_handled, wake_thread = false;
 
 	REGB_WR32(VPU_37XX_BUTTRESS_GLOBAL_INT_MASK, 0x1);
 
-	ret_irqv = ivpu_hw_37xx_irqv_handler(vdev, irq);
-	ret_irqb = ivpu_hw_37xx_irqb_handler(vdev, irq);
+	irqv_handled = ivpu_hw_37xx_irqv_handler(vdev, irq, &wake_thread);
+	irqb_handled = ivpu_hw_37xx_irqb_handler(vdev, irq);
 
 	/* Re-enable global interrupts to re-trigger MSI for pending interrupts */
 	REGB_WR32(VPU_37XX_BUTTRESS_GLOBAL_INT_MASK, 0x0);
 
-	return IRQ_RETVAL(ret_irqb | ret_irqv);
+	if (wake_thread)
+		return IRQ_WAKE_THREAD;
+	if (irqv_handled || irqb_handled)
+		return IRQ_HANDLED;
+	return IRQ_NONE;
 }
 
 static void ivpu_hw_37xx_diagnose_failure(struct ivpu_device *vdev)
@@ -993,11 +1021,14 @@ const struct ivpu_hw_ops ivpu_hw_37xx_ops = {
 	.info_init = ivpu_hw_37xx_info_init,
 	.power_up = ivpu_hw_37xx_power_up,
 	.is_idle = ivpu_hw_37xx_is_idle,
+	.wait_for_idle = ivpu_hw_37xx_wait_for_idle,
 	.power_down = ivpu_hw_37xx_power_down,
 	.reset = ivpu_hw_37xx_reset,
 	.boot_fw = ivpu_hw_37xx_boot_fw,
 	.wdt_disable = ivpu_hw_37xx_wdt_disable,
 	.diagnose_failure = ivpu_hw_37xx_diagnose_failure,
+	.profiling_freq_get = ivpu_hw_37xx_profiling_freq_get,
+	.profiling_freq_drive = ivpu_hw_37xx_profiling_freq_drive,
 	.reg_pll_freq_get = ivpu_hw_37xx_reg_pll_freq_get,
 	.reg_telemetry_offset_get = ivpu_hw_37xx_reg_telemetry_offset_get,
 	.reg_telemetry_size_get = ivpu_hw_37xx_reg_telemetry_size_get,
diff --git a/drivers/accel/ivpu/ivpu_hw_37xx_reg.h b/drivers/accel/ivpu/ivpu_hw_37xx_reg.h
index 4083beb5e9db..f6fec1919202 100644
--- a/drivers/accel/ivpu/ivpu_hw_37xx_reg.h
+++ b/drivers/accel/ivpu/ivpu_hw_37xx_reg.h
@@ -240,6 +240,8 @@
 #define VPU_37XX_CPU_SS_TIM_GEN_CONFIG					0x06021008u
 #define VPU_37XX_CPU_SS_TIM_GEN_CONFIG_WDOG_TO_INT_CLR_MASK		BIT_MASK(9)
 
+#define VPU_37XX_CPU_SS_TIM_PERF_FREE_CNT				0x06029000u
+
 #define VPU_37XX_CPU_SS_DOORBELL_0					0x06300000u
 #define VPU_37XX_CPU_SS_DOORBELL_0_SET_MASK				BIT_MASK(0)
 
diff --git a/drivers/accel/ivpu/ivpu_hw_40xx.c b/drivers/accel/ivpu/ivpu_hw_40xx.c
index e691c49c9841..eba2fdef2ace 100644
--- a/drivers/accel/ivpu/ivpu_hw_40xx.c
+++ b/drivers/accel/ivpu/ivpu_hw_40xx.c
@@ -39,6 +39,7 @@
 #define TIMEOUT_US		     (150 * USEC_PER_MSEC)
 #define PWR_ISLAND_STATUS_TIMEOUT_US (5 * USEC_PER_MSEC)
 #define PLL_TIMEOUT_US		     (1500 * USEC_PER_MSEC)
+#define IDLE_TIMEOUT_US		     (5 * USEC_PER_MSEC)
 
 #define WEIGHTS_DEFAULT              0xf711f711u
 #define WEIGHTS_ATS_DEFAULT          0x0000f711u
@@ -139,18 +140,21 @@ static void ivpu_hw_timeouts_init(struct ivpu_device *vdev)
 		vdev->timeout.tdr = 2000000;
 		vdev->timeout.reschedule_suspend = 1000;
 		vdev->timeout.autosuspend = -1;
+		vdev->timeout.d0i3_entry_msg = 500;
 	} else if (ivpu_is_simics(vdev)) {
 		vdev->timeout.boot = 50;
 		vdev->timeout.jsm = 500;
 		vdev->timeout.tdr = 10000;
 		vdev->timeout.reschedule_suspend = 10;
 		vdev->timeout.autosuspend = -1;
+		vdev->timeout.d0i3_entry_msg = 100;
 	} else {
 		vdev->timeout.boot = 1000;
 		vdev->timeout.jsm = 500;
 		vdev->timeout.tdr = 2000;
 		vdev->timeout.reschedule_suspend = 10;
 		vdev->timeout.autosuspend = 10;
+		vdev->timeout.d0i3_entry_msg = 5;
 	}
 }
 
@@ -824,12 +828,6 @@ static int ivpu_hw_40xx_power_up(struct ivpu_device *vdev)
 {
 	int ret;
 
-	ret = ivpu_hw_40xx_reset(vdev);
-	if (ret) {
-		ivpu_err(vdev, "Failed to reset HW: %d\n", ret);
-		return ret;
-	}
-
 	ret = ivpu_hw_40xx_d0i3_disable(vdev);
 	if (ret)
 		ivpu_warn(vdev, "Failed to disable D0I3: %d\n", ret);
@@ -898,10 +896,23 @@ static bool ivpu_hw_40xx_is_idle(struct ivpu_device *vdev)
 	       REG_TEST_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, IDLE, val);
 }
 
+static int ivpu_hw_40xx_wait_for_idle(struct ivpu_device *vdev)
+{
+	return REGB_POLL_FLD(VPU_40XX_BUTTRESS_VPU_STATUS, IDLE, 0x1, IDLE_TIMEOUT_US);
+}
+
+static void ivpu_hw_40xx_save_d0i3_entry_timestamp(struct ivpu_device *vdev)
+{
+	vdev->hw->d0i3_entry_host_ts = ktime_get_boottime();
+	vdev->hw->d0i3_entry_vpu_ts = REGV_RD64(VPU_40XX_CPU_SS_TIM_PERF_EXT_FREE_CNT);
+}
+
 static int ivpu_hw_40xx_power_down(struct ivpu_device *vdev)
 {
 	int ret = 0;
 
+	ivpu_hw_40xx_save_d0i3_entry_timestamp(vdev);
+
 	if (!ivpu_hw_40xx_is_idle(vdev) && ivpu_hw_40xx_reset(vdev))
 		ivpu_warn(vdev, "Failed to reset the VPU\n");
 
@@ -933,6 +944,19 @@ static void ivpu_hw_40xx_wdt_disable(struct ivpu_device *vdev)
 	REGV_WR32(VPU_40XX_CPU_SS_TIM_GEN_CONFIG, val);
 }
 
+static u32 ivpu_hw_40xx_profiling_freq_get(struct ivpu_device *vdev)
+{
+	return vdev->hw->pll.profiling_freq;
+}
+
+static void ivpu_hw_40xx_profiling_freq_drive(struct ivpu_device *vdev, bool enable)
+{
+	if (enable)
+		vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_HIGH;
+	else
+		vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT;
+}
+
 /* Register indirect accesses */
 static u32 ivpu_hw_40xx_reg_pll_freq_get(struct ivpu_device *vdev)
 {
@@ -1023,13 +1047,12 @@ static void ivpu_hw_40xx_irq_noc_firewall_handler(struct ivpu_device *vdev)
 }
 
 /* Handler for IRQs from VPU core (irqV) */
-static irqreturn_t ivpu_hw_40xx_irqv_handler(struct ivpu_device *vdev, int irq)
+static bool ivpu_hw_40xx_irqv_handler(struct ivpu_device *vdev, int irq, bool *wake_thread)
 {
 	u32 status = REGV_RD32(VPU_40XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK;
-	irqreturn_t ret = IRQ_NONE;
 
 	if (!status)
-		return IRQ_NONE;
+		return false;
 
 	REGV_WR32(VPU_40XX_HOST_SS_ICB_CLEAR_0, status);
 
@@ -1037,7 +1060,7 @@ static irqreturn_t ivpu_hw_40xx_irqv_handler(struct ivpu_device *vdev, int irq)
 		ivpu_mmu_irq_evtq_handler(vdev);
 
 	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT, status))
-		ret |= ivpu_ipc_irq_handler(vdev);
+		ivpu_ipc_irq_handler(vdev, wake_thread);
 
 	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT, status))
 		ivpu_dbg(vdev, IRQ, "MMU sync complete\n");
@@ -1054,17 +1077,17 @@ static irqreturn_t ivpu_hw_40xx_irqv_handler(struct ivpu_device *vdev, int irq)
 	if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, status))
 		ivpu_hw_40xx_irq_noc_firewall_handler(vdev);
 
-	return ret;
+	return true;
 }
 
 /* Handler for IRQs from Buttress core (irqB) */
-static irqreturn_t ivpu_hw_40xx_irqb_handler(struct ivpu_device *vdev, int irq)
+static bool ivpu_hw_40xx_irqb_handler(struct ivpu_device *vdev, int irq)
 {
 	bool schedule_recovery = false;
 	u32 status = REGB_RD32(VPU_40XX_BUTTRESS_INTERRUPT_STAT) & BUTTRESS_IRQ_MASK;
 
-	if (status == 0)
-		return IRQ_NONE;
+	if (!status)
+		return false;
 
 	if (REG_TEST_FLD(VPU_40XX_BUTTRESS_INTERRUPT_STAT, FREQ_CHANGE, status))
 		ivpu_dbg(vdev, IRQ, "FREQ_CHANGE");
@@ -1116,26 +1139,27 @@ static irqreturn_t ivpu_hw_40xx_irqb_handler(struct ivpu_device *vdev, int irq)
 	if (schedule_recovery)
 		ivpu_pm_schedule_recovery(vdev);
 
-	return IRQ_HANDLED;
+	return true;
 }
 
 static irqreturn_t ivpu_hw_40xx_irq_handler(int irq, void *ptr)
 {
+	bool irqv_handled, irqb_handled, wake_thread = false;
 	struct ivpu_device *vdev = ptr;
-	irqreturn_t ret = IRQ_NONE;
 
 	REGB_WR32(VPU_40XX_BUTTRESS_GLOBAL_INT_MASK, 0x1);
 
-	ret |= ivpu_hw_40xx_irqv_handler(vdev, irq);
-	ret |= ivpu_hw_40xx_irqb_handler(vdev, irq);
+	irqv_handled = ivpu_hw_40xx_irqv_handler(vdev, irq, &wake_thread);
+	irqb_handled = ivpu_hw_40xx_irqb_handler(vdev, irq);
 
 	/* Re-enable global interrupts to re-trigger MSI for pending interrupts */
 	REGB_WR32(VPU_40XX_BUTTRESS_GLOBAL_INT_MASK, 0x0);
 
-	if (ret & IRQ_WAKE_THREAD)
+	if (wake_thread)
 		return IRQ_WAKE_THREAD;
-
-	return ret;
+	if (irqv_handled || irqb_handled)
+		return IRQ_HANDLED;
+	return IRQ_NONE;
 }
 
 static void ivpu_hw_40xx_diagnose_failure(struct ivpu_device *vdev)
@@ -1185,11 +1209,14 @@ const struct ivpu_hw_ops ivpu_hw_40xx_ops = {
 	.info_init = ivpu_hw_40xx_info_init,
 	.power_up = ivpu_hw_40xx_power_up,
 	.is_idle = ivpu_hw_40xx_is_idle,
+	.wait_for_idle = ivpu_hw_40xx_wait_for_idle,
 	.power_down = ivpu_hw_40xx_power_down,
 	.reset = ivpu_hw_40xx_reset,
 	.boot_fw = ivpu_hw_40xx_boot_fw,
 	.wdt_disable = ivpu_hw_40xx_wdt_disable,
 	.diagnose_failure = ivpu_hw_40xx_diagnose_failure,
+	.profiling_freq_get = ivpu_hw_40xx_profiling_freq_get,
+	.profiling_freq_drive = ivpu_hw_40xx_profiling_freq_drive,
 	.reg_pll_freq_get = ivpu_hw_40xx_reg_pll_freq_get,
 	.reg_telemetry_offset_get = ivpu_hw_40xx_reg_telemetry_offset_get,
 	.reg_telemetry_size_get = ivpu_hw_40xx_reg_telemetry_size_get,
diff --git a/drivers/accel/ivpu/ivpu_ipc.c b/drivers/accel/ivpu/ivpu_ipc.c
index a4ca40b184d4..e86621f16f85 100644
--- a/drivers/accel/ivpu/ivpu_ipc.c
+++ b/drivers/accel/ivpu/ivpu_ipc.c
@@ -5,7 +5,7 @@
 
 #include <linux/genalloc.h>
 #include <linux/highmem.h>
-#include <linux/kthread.h>
+#include <linux/pm_runtime.h>
 #include <linux/wait.h>
 
 #include "ivpu_drv.h"
@@ -17,19 +17,12 @@
 #include "ivpu_pm.h"
 
 #define IPC_MAX_RX_MSG	128
-#define IS_KTHREAD()	(get_current()->flags & PF_KTHREAD)
 
 struct ivpu_ipc_tx_buf {
 	struct ivpu_ipc_hdr ipc;
 	struct vpu_jsm_msg jsm;
 };
 
-struct ivpu_ipc_rx_msg {
-	struct list_head link;
-	struct ivpu_ipc_hdr *ipc_hdr;
-	struct vpu_jsm_msg *jsm_msg;
-};
-
 static void ivpu_ipc_msg_dump(struct ivpu_device *vdev, char *c,
 			      struct ivpu_ipc_hdr *ipc_hdr, u32 vpu_addr)
 {
@@ -139,8 +132,49 @@ static void ivpu_ipc_tx(struct ivpu_device *vdev, u32 vpu_addr)
 	ivpu_hw_reg_ipc_tx_set(vdev, vpu_addr);
 }
 
-void
-ivpu_ipc_consumer_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, u32 channel)
+static void
+ivpu_ipc_rx_msg_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
+		    struct ivpu_ipc_hdr *ipc_hdr, struct vpu_jsm_msg *jsm_msg)
+{
+	struct ivpu_ipc_info *ipc = vdev->ipc;
+	struct ivpu_ipc_rx_msg *rx_msg;
+
+	lockdep_assert_held(&ipc->cons_lock);
+	lockdep_assert_irqs_disabled();
+
+	rx_msg = kzalloc(sizeof(*rx_msg), GFP_ATOMIC);
+	if (!rx_msg) {
+		ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg);
+		return;
+	}
+
+	atomic_inc(&ipc->rx_msg_count);
+
+	rx_msg->ipc_hdr = ipc_hdr;
+	rx_msg->jsm_msg = jsm_msg;
+	rx_msg->callback = cons->rx_callback;
+
+	if (rx_msg->callback) {
+		list_add_tail(&rx_msg->link, &ipc->cb_msg_list);
+	} else {
+		spin_lock(&cons->rx_lock);
+		list_add_tail(&rx_msg->link, &cons->rx_msg_list);
+		spin_unlock(&cons->rx_lock);
+		wake_up(&cons->rx_msg_wq);
+	}
+}
+
+static void
+ivpu_ipc_rx_msg_del(struct ivpu_device *vdev, struct ivpu_ipc_rx_msg *rx_msg)
+{
+	list_del(&rx_msg->link);
+	ivpu_ipc_rx_mark_free(vdev, rx_msg->ipc_hdr, rx_msg->jsm_msg);
+	atomic_dec(&vdev->ipc->rx_msg_count);
+	kfree(rx_msg);
+}
+
+void ivpu_ipc_consumer_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
+			   u32 channel, ivpu_ipc_rx_callback_t rx_callback)
 {
 	struct ivpu_ipc_info *ipc = vdev->ipc;
 
@@ -148,13 +182,15 @@ ivpu_ipc_consumer_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
 	cons->channel = channel;
 	cons->tx_vpu_addr = 0;
 	cons->request_id = 0;
-	spin_lock_init(&cons->rx_msg_lock);
+	cons->aborted = false;
+	cons->rx_callback = rx_callback;
+	spin_lock_init(&cons->rx_lock);
 	INIT_LIST_HEAD(&cons->rx_msg_list);
 	init_waitqueue_head(&cons->rx_msg_wq);
 
-	spin_lock_irq(&ipc->cons_list_lock);
+	spin_lock_irq(&ipc->cons_lock);
 	list_add_tail(&cons->link, &ipc->cons_list);
-	spin_unlock_irq(&ipc->cons_list_lock);
+	spin_unlock_irq(&ipc->cons_lock);
 }
 
 void ivpu_ipc_consumer_del(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons)
@@ -162,18 +198,14 @@ void ivpu_ipc_consumer_del(struct ivpu_device *vdev, struct ivpu_ipc_consumer *c
 	struct ivpu_ipc_info *ipc = vdev->ipc;
 	struct ivpu_ipc_rx_msg *rx_msg, *r;
 
-	spin_lock_irq(&ipc->cons_list_lock);
+	spin_lock_irq(&ipc->cons_lock);
 	list_del(&cons->link);
-	spin_unlock_irq(&ipc->cons_list_lock);
-
-	spin_lock_irq(&cons->rx_msg_lock);
-	list_for_each_entry_safe(rx_msg, r, &cons->rx_msg_list, link) {
-		list_del(&rx_msg->link);
-		ivpu_ipc_rx_mark_free(vdev, rx_msg->ipc_hdr, rx_msg->jsm_msg);
-		atomic_dec(&ipc->rx_msg_count);
-		kfree(rx_msg);
-	}
-	spin_unlock_irq(&cons->rx_msg_lock);
+	spin_unlock_irq(&ipc->cons_lock);
+
+	spin_lock_irq(&cons->rx_lock);
+	list_for_each_entry_safe(rx_msg, r, &cons->rx_msg_list, link)
+		ivpu_ipc_rx_msg_del(vdev, rx_msg);
+	spin_unlock_irq(&cons->rx_lock);
 
 	ivpu_ipc_tx_release(vdev, cons->tx_vpu_addr);
 }
@@ -202,52 +234,61 @@ ivpu_ipc_send(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, struct v
 	return ret;
 }
 
+static bool ivpu_ipc_rx_need_wakeup(struct ivpu_ipc_consumer *cons)
+{
+	bool ret;
+
+	spin_lock_irq(&cons->rx_lock);
+	ret = !list_empty(&cons->rx_msg_list) || cons->aborted;
+	spin_unlock_irq(&cons->rx_lock);
+
+	return ret;
+}
+
 int ivpu_ipc_receive(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
 		     struct ivpu_ipc_hdr *ipc_buf,
-		     struct vpu_jsm_msg *ipc_payload, unsigned long timeout_ms)
+		     struct vpu_jsm_msg *jsm_msg, unsigned long timeout_ms)
 {
-	struct ivpu_ipc_info *ipc = vdev->ipc;
 	struct ivpu_ipc_rx_msg *rx_msg;
 	int wait_ret, ret = 0;
 
+	if (drm_WARN_ONCE(&vdev->drm, cons->rx_callback, "Consumer works only in async mode\n"))
+		return -EINVAL;
+
 	wait_ret = wait_event_timeout(cons->rx_msg_wq,
-				      (IS_KTHREAD() && kthread_should_stop()) ||
-				      !list_empty(&cons->rx_msg_list),
+				      ivpu_ipc_rx_need_wakeup(cons),
 				      msecs_to_jiffies(timeout_ms));
 
-	if (IS_KTHREAD() && kthread_should_stop())
-		return -EINTR;
-
 	if (wait_ret == 0)
 		return -ETIMEDOUT;
 
-	spin_lock_irq(&cons->rx_msg_lock);
+	spin_lock_irq(&cons->rx_lock);
+	if (cons->aborted) {
+		spin_unlock_irq(&cons->rx_lock);
+		return -ECANCELED;
+	}
 	rx_msg = list_first_entry_or_null(&cons->rx_msg_list, struct ivpu_ipc_rx_msg, link);
 	if (!rx_msg) {
-		spin_unlock_irq(&cons->rx_msg_lock);
+		spin_unlock_irq(&cons->rx_lock);
 		return -EAGAIN;
 	}
-	list_del(&rx_msg->link);
-	spin_unlock_irq(&cons->rx_msg_lock);
 
 	if (ipc_buf)
 		memcpy(ipc_buf, rx_msg->ipc_hdr, sizeof(*ipc_buf));
 	if (rx_msg->jsm_msg) {
-		u32 size = min_t(int, rx_msg->ipc_hdr->data_size, sizeof(*ipc_payload));
+		u32 size = min_t(int, rx_msg->ipc_hdr->data_size, sizeof(*jsm_msg));
 
 		if (rx_msg->jsm_msg->result != VPU_JSM_STATUS_SUCCESS) {
 			ivpu_dbg(vdev, IPC, "IPC resp result error: %d\n", rx_msg->jsm_msg->result);
 			ret = -EBADMSG;
 		}
 
-		if (ipc_payload)
-			memcpy(ipc_payload, rx_msg->jsm_msg, size);
+		if (jsm_msg)
+			memcpy(jsm_msg, rx_msg->jsm_msg, size);
 	}
 
-	ivpu_ipc_rx_mark_free(vdev, rx_msg->ipc_hdr, rx_msg->jsm_msg);
-	atomic_dec(&ipc->rx_msg_count);
-	kfree(rx_msg);
-
+	ivpu_ipc_rx_msg_del(vdev, rx_msg);
+	spin_unlock_irq(&cons->rx_lock);
 	return ret;
 }
 
@@ -260,7 +301,7 @@ ivpu_ipc_send_receive_internal(struct ivpu_device *vdev, struct vpu_jsm_msg *req
 	struct ivpu_ipc_consumer cons;
 	int ret;
 
-	ivpu_ipc_consumer_add(vdev, &cons, channel);
+	ivpu_ipc_consumer_add(vdev, &cons, channel, NULL);
 
 	ret = ivpu_ipc_send(vdev, &cons, req);
 	if (ret) {
@@ -285,23 +326,19 @@ ivpu_ipc_send_receive_internal(struct ivpu_device *vdev, struct vpu_jsm_msg *req
 	return ret;
 }
 
-int ivpu_ipc_send_receive(struct ivpu_device *vdev, struct vpu_jsm_msg *req,
-			  enum vpu_ipc_msg_type expected_resp_type,
-			  struct vpu_jsm_msg *resp, u32 channel,
-			  unsigned long timeout_ms)
+int ivpu_ipc_send_receive_active(struct ivpu_device *vdev, struct vpu_jsm_msg *req,
+				 enum vpu_ipc_msg_type expected_resp, struct vpu_jsm_msg *resp,
+				 u32 channel, unsigned long timeout_ms)
 {
 	struct vpu_jsm_msg hb_req = { .type = VPU_JSM_MSG_QUERY_ENGINE_HB };
 	struct vpu_jsm_msg hb_resp;
 	int ret, hb_ret;
 
-	ret = ivpu_rpm_get(vdev);
-	if (ret < 0)
-		return ret;
+	drm_WARN_ON(&vdev->drm, pm_runtime_status_suspended(vdev->drm.dev));
 
-	ret = ivpu_ipc_send_receive_internal(vdev, req, expected_resp_type, resp,
-					     channel, timeout_ms);
+	ret = ivpu_ipc_send_receive_internal(vdev, req, expected_resp, resp, channel, timeout_ms);
 	if (ret != -ETIMEDOUT)
-		goto rpm_put;
+		return ret;
 
 	hb_ret = ivpu_ipc_send_receive_internal(vdev, &hb_req, VPU_JSM_MSG_QUERY_ENGINE_HB_DONE,
 						&hb_resp, VPU_IPC_CHAN_ASYNC_CMD,
@@ -311,7 +348,21 @@ int ivpu_ipc_send_receive(struct ivpu_device *vdev, struct vpu_jsm_msg *req,
 		ivpu_pm_schedule_recovery(vdev);
 	}
 
-rpm_put:
+	return ret;
+}
+
+int ivpu_ipc_send_receive(struct ivpu_device *vdev, struct vpu_jsm_msg *req,
+			  enum vpu_ipc_msg_type expected_resp, struct vpu_jsm_msg *resp,
+			  u32 channel, unsigned long timeout_ms)
+{
+	int ret;
+
+	ret = ivpu_rpm_get(vdev);
+	if (ret < 0)
+		return ret;
+
+	ret = ivpu_ipc_send_receive_active(vdev, req, expected_resp, resp, channel, timeout_ms);
+
 	ivpu_rpm_put(vdev);
 	return ret;
 }
@@ -329,35 +380,7 @@ ivpu_ipc_match_consumer(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons
 	return false;
 }
 
-static void
-ivpu_ipc_dispatch(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
-		  struct ivpu_ipc_hdr *ipc_hdr, struct vpu_jsm_msg *jsm_msg)
-{
-	struct ivpu_ipc_info *ipc = vdev->ipc;
-	struct ivpu_ipc_rx_msg *rx_msg;
-	unsigned long flags;
-
-	lockdep_assert_held(&ipc->cons_list_lock);
-
-	rx_msg = kzalloc(sizeof(*rx_msg), GFP_ATOMIC);
-	if (!rx_msg) {
-		ivpu_ipc_rx_mark_free(vdev, ipc_hdr, jsm_msg);
-		return;
-	}
-
-	atomic_inc(&ipc->rx_msg_count);
-
-	rx_msg->ipc_hdr = ipc_hdr;
-	rx_msg->jsm_msg = jsm_msg;
-
-	spin_lock_irqsave(&cons->rx_msg_lock, flags);
-	list_add_tail(&rx_msg->link, &cons->rx_msg_list);
-	spin_unlock_irqrestore(&cons->rx_msg_lock, flags);
-
-	wake_up(&cons->rx_msg_wq);
-}
-
-int ivpu_ipc_irq_handler(struct ivpu_device *vdev)
+void ivpu_ipc_irq_handler(struct ivpu_device *vdev, bool *wake_thread)
 {
 	struct ivpu_ipc_info *ipc = vdev->ipc;
 	struct ivpu_ipc_consumer *cons;
@@ -375,7 +398,7 @@ int ivpu_ipc_irq_handler(struct ivpu_device *vdev)
 		vpu_addr = ivpu_hw_reg_ipc_rx_addr_get(vdev);
 		if (vpu_addr == REG_IO_ERROR) {
 			ivpu_err_ratelimited(vdev, "Failed to read IPC rx addr register\n");
-			return -EIO;
+			return;
 		}
 
 		ipc_hdr = ivpu_to_cpu_addr(ipc->mem_rx, vpu_addr);
@@ -405,15 +428,15 @@ int ivpu_ipc_irq_handler(struct ivpu_device *vdev)
 		}
 
 		dispatched = false;
-		spin_lock_irqsave(&ipc->cons_list_lock, flags);
+		spin_lock_irqsave(&ipc->cons_lock, flags);
 		list_for_each_entry(cons, &ipc->cons_list, link) {
 			if (ivpu_ipc_match_consumer(vdev, cons, ipc_hdr, jsm_msg)) {
-				ivpu_ipc_dispatch(vdev, cons, ipc_hdr, jsm_msg);
+				ivpu_ipc_rx_msg_add(vdev, cons, ipc_hdr, jsm_msg);
 				dispatched = true;
 				break;
 			}
 		}
-		spin_unlock_irqrestore(&ipc->cons_list_lock, flags);
+		spin_unlock_irqrestore(&ipc->cons_lock, flags);
 
 		if (!dispatched) {
 			ivpu_dbg(vdev, IPC, "IPC RX msg 0x%x dropped (no consumer)\n", vpu_addr);
@@ -421,7 +444,28 @@ int ivpu_ipc_irq_handler(struct ivpu_device *vdev)
 		}
 	}
 
-	return 0;
+	if (wake_thread)
+		*wake_thread = !list_empty(&ipc->cb_msg_list);
+}
+
+irqreturn_t ivpu_ipc_irq_thread_handler(struct ivpu_device *vdev)
+{
+	struct ivpu_ipc_info *ipc = vdev->ipc;
+	struct ivpu_ipc_rx_msg *rx_msg, *r;
+	struct list_head cb_msg_list;
+
+	INIT_LIST_HEAD(&cb_msg_list);
+
+	spin_lock_irq(&ipc->cons_lock);
+	list_splice_tail_init(&ipc->cb_msg_list, &cb_msg_list);
+	spin_unlock_irq(&ipc->cons_lock);
+
+	list_for_each_entry_safe(rx_msg, r, &cb_msg_list, link) {
+		rx_msg->callback(vdev, rx_msg->ipc_hdr, rx_msg->jsm_msg);
+		ivpu_ipc_rx_msg_del(vdev, rx_msg);
+	}
+
+	return IRQ_HANDLED;
 }
 
 int ivpu_ipc_init(struct ivpu_device *vdev)
@@ -456,10 +500,10 @@ int ivpu_ipc_init(struct ivpu_device *vdev)
 		goto err_free_rx;
 	}
 
+	spin_lock_init(&ipc->cons_lock);
 	INIT_LIST_HEAD(&ipc->cons_list);
-	spin_lock_init(&ipc->cons_list_lock);
+	INIT_LIST_HEAD(&ipc->cb_msg_list);
 	drmm_mutex_init(&vdev->drm, &ipc->lock);
-
 	ivpu_ipc_reset(vdev);
 	return 0;
 
@@ -472,6 +516,13 @@ int ivpu_ipc_init(struct ivpu_device *vdev)
 
 void ivpu_ipc_fini(struct ivpu_device *vdev)
 {
+	struct ivpu_ipc_info *ipc = vdev->ipc;
+
+	drm_WARN_ON(&vdev->drm, ipc->on);
+	drm_WARN_ON(&vdev->drm, !list_empty(&ipc->cons_list));
+	drm_WARN_ON(&vdev->drm, !list_empty(&ipc->cb_msg_list));
+	drm_WARN_ON(&vdev->drm, atomic_read(&ipc->rx_msg_count) > 0);
+
 	ivpu_ipc_mem_fini(vdev);
 }
 
@@ -488,16 +539,27 @@ void ivpu_ipc_disable(struct ivpu_device *vdev)
 {
 	struct ivpu_ipc_info *ipc = vdev->ipc;
 	struct ivpu_ipc_consumer *cons, *c;
-	unsigned long flags;
+	struct ivpu_ipc_rx_msg *rx_msg, *r;
+
+	drm_WARN_ON(&vdev->drm, !list_empty(&ipc->cb_msg_list));
 
 	mutex_lock(&ipc->lock);
 	ipc->on = false;
 	mutex_unlock(&ipc->lock);
 
-	spin_lock_irqsave(&ipc->cons_list_lock, flags);
-	list_for_each_entry_safe(cons, c, &ipc->cons_list, link)
+	spin_lock_irq(&ipc->cons_lock);
+	list_for_each_entry_safe(cons, c, &ipc->cons_list, link) {
+		spin_lock(&cons->rx_lock);
+		if (!cons->rx_callback)
+			cons->aborted = true;
+		list_for_each_entry_safe(rx_msg, r, &cons->rx_msg_list, link)
+			ivpu_ipc_rx_msg_del(vdev, rx_msg);
+		spin_unlock(&cons->rx_lock);
 		wake_up(&cons->rx_msg_wq);
-	spin_unlock_irqrestore(&ipc->cons_list_lock, flags);
+	}
+	spin_unlock_irq(&ipc->cons_lock);
+
+	drm_WARN_ON(&vdev->drm, atomic_read(&ipc->rx_msg_count) > 0);
 }
 
 void ivpu_ipc_reset(struct ivpu_device *vdev)
@@ -505,6 +567,7 @@ void ivpu_ipc_reset(struct ivpu_device *vdev)
 	struct ivpu_ipc_info *ipc = vdev->ipc;
 
 	mutex_lock(&ipc->lock);
+	drm_WARN_ON(&vdev->drm, ipc->on);
 
 	memset(ivpu_bo_vaddr(ipc->mem_tx), 0, ivpu_bo_size(ipc->mem_tx));
 	memset(ivpu_bo_vaddr(ipc->mem_rx), 0, ivpu_bo_size(ipc->mem_rx));
diff --git a/drivers/accel/ivpu/ivpu_ipc.h b/drivers/accel/ivpu/ivpu_ipc.h
index 68f5b6668e00..40ca3cc4e61f 100644
--- a/drivers/accel/ivpu/ivpu_ipc.h
+++ b/drivers/accel/ivpu/ivpu_ipc.h
@@ -42,13 +42,26 @@ struct ivpu_ipc_hdr {
 	u8 status;
 } __packed __aligned(IVPU_IPC_ALIGNMENT);
 
+typedef void (*ivpu_ipc_rx_callback_t)(struct ivpu_device *vdev,
+				       struct ivpu_ipc_hdr *ipc_hdr,
+				       struct vpu_jsm_msg *jsm_msg);
+
+struct ivpu_ipc_rx_msg {
+	struct list_head link;
+	struct ivpu_ipc_hdr *ipc_hdr;
+	struct vpu_jsm_msg *jsm_msg;
+	ivpu_ipc_rx_callback_t callback;
+};
+
 struct ivpu_ipc_consumer {
 	struct list_head link;
 	u32 channel;
 	u32 tx_vpu_addr;
 	u32 request_id;
+	bool aborted;
+	ivpu_ipc_rx_callback_t rx_callback;
 
-	spinlock_t rx_msg_lock; /* Protects rx_msg_list */
+	spinlock_t rx_lock; /* Protects rx_msg_list and aborted */
 	struct list_head rx_msg_list;
 	wait_queue_head_t rx_msg_wq;
 };
@@ -60,8 +73,9 @@ struct ivpu_ipc_info {
 
 	atomic_t rx_msg_count;
 
-	spinlock_t cons_list_lock; /* Protects cons_list */
+	spinlock_t cons_lock; /* Protects cons_list and cb_msg_list */
 	struct list_head cons_list;
+	struct list_head cb_msg_list;
 
 	atomic_t request_id;
 	struct mutex lock; /* Lock on status */
@@ -75,19 +89,22 @@ void ivpu_ipc_enable(struct ivpu_device *vdev);
 void ivpu_ipc_disable(struct ivpu_device *vdev);
 void ivpu_ipc_reset(struct ivpu_device *vdev);
 
-int ivpu_ipc_irq_handler(struct ivpu_device *vdev);
+void ivpu_ipc_irq_handler(struct ivpu_device *vdev, bool *wake_thread);
+irqreturn_t ivpu_ipc_irq_thread_handler(struct ivpu_device *vdev);
 
 void ivpu_ipc_consumer_add(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
-			   u32 channel);
+			   u32 channel, ivpu_ipc_rx_callback_t callback);
 void ivpu_ipc_consumer_del(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons);
 
 int ivpu_ipc_receive(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons,
-		     struct ivpu_ipc_hdr *ipc_buf, struct vpu_jsm_msg *ipc_payload,
+		     struct ivpu_ipc_hdr *ipc_buf, struct vpu_jsm_msg *jsm_msg,
 		     unsigned long timeout_ms);
 
+int ivpu_ipc_send_receive_active(struct ivpu_device *vdev, struct vpu_jsm_msg *req,
+				 enum vpu_ipc_msg_type expected_resp, struct vpu_jsm_msg *resp,
+				 u32 channel, unsigned long timeout_ms);
 int ivpu_ipc_send_receive(struct ivpu_device *vdev, struct vpu_jsm_msg *req,
-			  enum vpu_ipc_msg_type expected_resp_type,
-			  struct vpu_jsm_msg *resp, u32 channel,
-			  unsigned long timeout_ms);
+			  enum vpu_ipc_msg_type expected_resp, struct vpu_jsm_msg *resp,
+			  u32 channel, unsigned long timeout_ms);
 
 #endif /* __IVPU_IPC_H__ */
diff --git a/drivers/accel/ivpu/ivpu_job.c b/drivers/accel/ivpu/ivpu_job.c
index 8983e3a4fdf9..7206cf9cdb4a 100644
--- a/drivers/accel/ivpu/ivpu_job.c
+++ b/drivers/accel/ivpu/ivpu_job.c
@@ -7,7 +7,6 @@
 
 #include <linux/bitfield.h>
 #include <linux/highmem.h>
-#include <linux/kthread.h>
 #include <linux/pci.h>
 #include <linux/module.h>
 #include <uapi/drm/ivpu_accel.h>
@@ -24,10 +23,6 @@
 #define JOB_ID_CONTEXT_MASK  GENMASK(31, 8)
 #define JOB_MAX_BUFFER_COUNT 65535
 
-static unsigned int ivpu_tdr_timeout_ms;
-module_param_named(tdr_timeout_ms, ivpu_tdr_timeout_ms, uint, 0644);
-MODULE_PARM_DESC(tdr_timeout_ms, "Timeout for device hang detection, in milliseconds, 0 - default");
-
 static void ivpu_cmdq_ring_db(struct ivpu_device *vdev, struct ivpu_cmdq *cmdq)
 {
 	ivpu_hw_reg_db_set(vdev, cmdq->db_id);
@@ -196,6 +191,8 @@ static int ivpu_cmdq_push_job(struct ivpu_cmdq *cmdq, struct ivpu_job *job)
 	entry->batch_buf_addr = job->cmd_buf_vpu_addr;
 	entry->job_id = job->job_id;
 	entry->flags = 0;
+	if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_SUBMISSION))
+		entry->flags = VPU_JOB_FLAGS_NULL_SUBMISSION_MASK;
 	wmb(); /* Ensure that tail is updated after filling entry */
 	header->tail = next_entry;
 	wmb(); /* Flush WC buffer for jobq header */
@@ -264,7 +261,7 @@ static void job_release(struct kref *ref)
 
 	for (i = 0; i < job->bo_count; i++)
 		if (job->bos[i])
-			drm_gem_object_put(&job->bos[i]->base);
+			drm_gem_object_put(&job->bos[i]->base.base);
 
 	dma_fence_put(job->done_fence);
 	ivpu_file_priv_put(&job->file_priv);
@@ -340,23 +337,12 @@ static int ivpu_job_done(struct ivpu_device *vdev, u32 job_id, u32 job_status)
 	ivpu_dbg(vdev, JOB, "Job complete:  id %3u ctx %2d engine %d status 0x%x\n",
 		 job->job_id, job->file_priv->ctx.id, job->engine_idx, job_status);
 
+	ivpu_stop_job_timeout_detection(vdev);
+
 	job_put(job);
 	return 0;
 }
 
-static void ivpu_job_done_message(struct ivpu_device *vdev, void *msg)
-{
-	struct vpu_ipc_msg_payload_job_done *payload;
-	struct vpu_jsm_msg *job_ret_msg = msg;
-	int ret;
-
-	payload = (struct vpu_ipc_msg_payload_job_done *)&job_ret_msg->payload;
-
-	ret = ivpu_job_done(vdev, payload->job_id, payload->job_status);
-	if (ret)
-		ivpu_err(vdev, "Failed to finish job %d: %d\n", payload->job_id, ret);
-}
-
 void ivpu_jobs_abort_all(struct ivpu_device *vdev)
 {
 	struct ivpu_job *job;
@@ -398,11 +384,13 @@ static int ivpu_direct_job_submission(struct ivpu_job *job)
 	if (ret)
 		goto err_xa_erase;
 
+	ivpu_start_job_timeout_detection(vdev);
+
 	ivpu_dbg(vdev, JOB, "Job submitted: id %3u addr 0x%llx ctx %2d engine %d next %d\n",
 		 job->job_id, job->cmd_buf_vpu_addr, file_priv->ctx.id,
 		 job->engine_idx, cmdq->jobq->header.tail);
 
-	if (ivpu_test_mode == IVPU_TEST_MODE_NULL_HW) {
+	if (ivpu_test_mode & IVPU_TEST_MODE_NULL_HW) {
 		ivpu_job_done(vdev, job->job_id, VPU_JSM_STATUS_SUCCESS);
 		cmdq->jobq->header.head = cmdq->jobq->header.tail;
 		wmb(); /* Flush WC buffer for jobq header */
@@ -448,7 +436,7 @@ ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32
 	}
 
 	bo = job->bos[CMD_BUF_IDX];
-	if (!dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_READ)) {
+	if (!dma_resv_test_signaled(bo->base.base.resv, DMA_RESV_USAGE_READ)) {
 		ivpu_warn(vdev, "Buffer is already in use\n");
 		return -EBUSY;
 	}
@@ -468,7 +456,7 @@ ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32
 	}
 
 	for (i = 0; i < buf_count; i++) {
-		ret = dma_resv_reserve_fences(job->bos[i]->base.resv, 1);
+		ret = dma_resv_reserve_fences(job->bos[i]->base.base.resv, 1);
 		if (ret) {
 			ivpu_warn(vdev, "Failed to reserve fences: %d\n", ret);
 			goto unlock_reservations;
@@ -477,7 +465,7 @@ ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32
 
 	for (i = 0; i < buf_count; i++) {
 		usage = (i == CMD_BUF_IDX) ? DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_BOOKKEEP;
-		dma_resv_add_fence(job->bos[i]->base.resv, job->done_fence, usage);
+		dma_resv_add_fence(job->bos[i]->base.base.resv, job->done_fence, usage);
 	}
 
 unlock_reservations:
@@ -562,61 +550,36 @@ int ivpu_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 	return ret;
 }
 
-static int ivpu_job_done_thread(void *arg)
+static void
+ivpu_job_done_callback(struct ivpu_device *vdev, struct ivpu_ipc_hdr *ipc_hdr,
+		       struct vpu_jsm_msg *jsm_msg)
 {
-	struct ivpu_device *vdev = (struct ivpu_device *)arg;
-	struct ivpu_ipc_consumer cons;
-	struct vpu_jsm_msg jsm_msg;
-	bool jobs_submitted;
-	unsigned int timeout;
+	struct vpu_ipc_msg_payload_job_done *payload;
 	int ret;
 
-	ivpu_dbg(vdev, JOB, "Started %s\n", __func__);
-
-	ivpu_ipc_consumer_add(vdev, &cons, VPU_IPC_CHAN_JOB_RET);
-
-	while (!kthread_should_stop()) {
-		timeout = ivpu_tdr_timeout_ms ? ivpu_tdr_timeout_ms : vdev->timeout.tdr;
-		jobs_submitted = !xa_empty(&vdev->submitted_jobs_xa);
-		ret = ivpu_ipc_receive(vdev, &cons, NULL, &jsm_msg, timeout);
-		if (!ret) {
-			ivpu_job_done_message(vdev, &jsm_msg);
-		} else if (ret == -ETIMEDOUT) {
-			if (jobs_submitted && !xa_empty(&vdev->submitted_jobs_xa)) {
-				ivpu_err(vdev, "TDR detected, timeout %d ms", timeout);
-				ivpu_hw_diagnose_failure(vdev);
-				ivpu_pm_schedule_recovery(vdev);
-			}
-		}
+	if (!jsm_msg) {
+		ivpu_err(vdev, "IPC message has no JSM payload\n");
+		return;
 	}
 
-	ivpu_ipc_consumer_del(vdev, &cons);
-
-	ivpu_jobs_abort_all(vdev);
+	if (jsm_msg->result != VPU_JSM_STATUS_SUCCESS) {
+		ivpu_err(vdev, "Invalid JSM message result: %d\n", jsm_msg->result);
+		return;
+	}
 
-	ivpu_dbg(vdev, JOB, "Stopped %s\n", __func__);
-	return 0;
+	payload = (struct vpu_ipc_msg_payload_job_done *)&jsm_msg->payload;
+	ret = ivpu_job_done(vdev, payload->job_id, payload->job_status);
+	if (!ret && !xa_empty(&vdev->submitted_jobs_xa))
+		ivpu_start_job_timeout_detection(vdev);
 }
 
-int ivpu_job_done_thread_init(struct ivpu_device *vdev)
+void ivpu_job_done_consumer_init(struct ivpu_device *vdev)
 {
-	struct task_struct *thread;
-
-	thread = kthread_run(&ivpu_job_done_thread, (void *)vdev, "ivpu_job_done_thread");
-	if (IS_ERR(thread)) {
-		ivpu_err(vdev, "Failed to start job completion thread\n");
-		return -EIO;
-	}
-
-	get_task_struct(thread);
-	wake_up_process(thread);
-
-	vdev->job_done_thread = thread;
-
-	return 0;
+	ivpu_ipc_consumer_add(vdev, &vdev->job_done_consumer,
+			      VPU_IPC_CHAN_JOB_RET, ivpu_job_done_callback);
 }
 
-void ivpu_job_done_thread_fini(struct ivpu_device *vdev)
+void ivpu_job_done_consumer_fini(struct ivpu_device *vdev)
 {
-	kthread_stop_put(vdev->job_done_thread);
+	ivpu_ipc_consumer_del(vdev, &vdev->job_done_consumer);
 }
diff --git a/drivers/accel/ivpu/ivpu_job.h b/drivers/accel/ivpu/ivpu_job.h
index 5514c2d8a609..45a2f2ec82e5 100644
--- a/drivers/accel/ivpu/ivpu_job.h
+++ b/drivers/accel/ivpu/ivpu_job.h
@@ -59,8 +59,8 @@ int ivpu_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 void ivpu_cmdq_release_all(struct ivpu_file_priv *file_priv);
 void ivpu_cmdq_reset_all_contexts(struct ivpu_device *vdev);
 
-int ivpu_job_done_thread_init(struct ivpu_device *vdev);
-void ivpu_job_done_thread_fini(struct ivpu_device *vdev);
+void ivpu_job_done_consumer_init(struct ivpu_device *vdev);
+void ivpu_job_done_consumer_fini(struct ivpu_device *vdev);
 
 void ivpu_jobs_abort_all(struct ivpu_device *vdev);
 
diff --git a/drivers/accel/ivpu/ivpu_jsm_msg.c b/drivers/accel/ivpu/ivpu_jsm_msg.c
index 0c2fe7142024..8cea0dd731b9 100644
--- a/drivers/accel/ivpu/ivpu_jsm_msg.c
+++ b/drivers/accel/ivpu/ivpu_jsm_msg.c
@@ -4,6 +4,7 @@
  */
 
 #include "ivpu_drv.h"
+#include "ivpu_hw.h"
 #include "ivpu_ipc.h"
 #include "ivpu_jsm_msg.h"
 
@@ -36,6 +37,17 @@ const char *ivpu_jsm_msg_type_to_str(enum vpu_ipc_msg_type type)
 	IVPU_CASE_TO_STR(VPU_JSM_MSG_DESTROY_CMD_QUEUE);
 	IVPU_CASE_TO_STR(VPU_JSM_MSG_SET_CONTEXT_SCHED_PROPERTIES);
 	IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_REGISTER_DB);
+	IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_RESUME_CMDQ);
+	IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_SUSPEND_CMDQ);
+	IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_RESUME_CMDQ_RSP);
+	IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_SUSPEND_CMDQ_DONE);
+	IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG);
+	IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG_RSP);
+	IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_SCHEDULING_LOG_NOTIFICATION);
+	IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_ENGINE_RESUME);
+	IVPU_CASE_TO_STR(VPU_JSM_MSG_HWS_RESUME_ENGINE_DONE);
+	IVPU_CASE_TO_STR(VPU_JSM_MSG_STATE_DUMP);
+	IVPU_CASE_TO_STR(VPU_JSM_MSG_STATE_DUMP_RSP);
 	IVPU_CASE_TO_STR(VPU_JSM_MSG_BLOB_DEINIT);
 	IVPU_CASE_TO_STR(VPU_JSM_MSG_DYNDBG_CONTROL);
 	IVPU_CASE_TO_STR(VPU_JSM_MSG_JOB_DONE);
@@ -65,6 +77,12 @@ const char *ivpu_jsm_msg_type_to_str(enum vpu_ipc_msg_type type)
 	IVPU_CASE_TO_STR(VPU_JSM_MSG_SET_CONTEXT_SCHED_PROPERTIES_RSP);
 	IVPU_CASE_TO_STR(VPU_JSM_MSG_BLOB_DEINIT_DONE);
 	IVPU_CASE_TO_STR(VPU_JSM_MSG_DYNDBG_CONTROL_RSP);
+	IVPU_CASE_TO_STR(VPU_JSM_MSG_PWR_D0I3_ENTER);
+	IVPU_CASE_TO_STR(VPU_JSM_MSG_PWR_D0I3_ENTER_DONE);
+	IVPU_CASE_TO_STR(VPU_JSM_MSG_DCT_ENABLE);
+	IVPU_CASE_TO_STR(VPU_JSM_MSG_DCT_ENABLE_DONE);
+	IVPU_CASE_TO_STR(VPU_JSM_MSG_DCT_DISABLE);
+	IVPU_CASE_TO_STR(VPU_JSM_MSG_DCT_DISABLE_DONE);
 	}
 	#undef IVPU_CASE_TO_STR
 
@@ -243,3 +261,23 @@ int ivpu_jsm_context_release(struct ivpu_device *vdev, u32 host_ssid)
 	return ivpu_ipc_send_receive(vdev, &req, VPU_JSM_MSG_SSID_RELEASE_DONE, &resp,
 				     VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm);
 }
+
+int ivpu_jsm_pwr_d0i3_enter(struct ivpu_device *vdev)
+{
+	struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_PWR_D0I3_ENTER };
+	struct vpu_jsm_msg resp;
+	int ret;
+
+	if (IVPU_WA(disable_d0i3_msg))
+		return 0;
+
+	req.payload.pwr_d0i3_enter.send_response = 1;
+
+	ret = ivpu_ipc_send_receive_active(vdev, &req, VPU_JSM_MSG_PWR_D0I3_ENTER_DONE,
+					   &resp, VPU_IPC_CHAN_GEN_CMD,
+					   vdev->timeout.d0i3_entry_msg);
+	if (ret)
+		return ret;
+
+	return ivpu_hw_wait_for_idle(vdev);
+}
diff --git a/drivers/accel/ivpu/ivpu_jsm_msg.h b/drivers/accel/ivpu/ivpu_jsm_msg.h
index 66979a948c7c..ae75e5dbcc41 100644
--- a/drivers/accel/ivpu/ivpu_jsm_msg.h
+++ b/drivers/accel/ivpu/ivpu_jsm_msg.h
@@ -22,4 +22,5 @@ int ivpu_jsm_trace_get_capability(struct ivpu_device *vdev, u32 *trace_destinati
 int ivpu_jsm_trace_set_config(struct ivpu_device *vdev, u32 trace_level, u32 trace_destination_mask,
 			      u64 trace_hw_component_mask);
 int ivpu_jsm_context_release(struct ivpu_device *vdev, u32 host_ssid);
+int ivpu_jsm_pwr_d0i3_enter(struct ivpu_device *vdev);
 #endif
diff --git a/drivers/accel/ivpu/ivpu_mmu.c b/drivers/accel/ivpu/ivpu_mmu.c
index 2538c78fbebe..2228c44b115f 100644
--- a/drivers/accel/ivpu/ivpu_mmu.c
+++ b/drivers/accel/ivpu/ivpu_mmu.c
@@ -230,7 +230,12 @@
 				  (REG_FLD(IVPU_MMU_REG_GERROR, MSI_PRIQ_ABT)) | \
 				  (REG_FLD(IVPU_MMU_REG_GERROR, MSI_ABT)))
 
-static char *ivpu_mmu_event_to_str(u32 cmd)
+#define IVPU_MMU_CERROR_NONE         0x0
+#define IVPU_MMU_CERROR_ILL          0x1
+#define IVPU_MMU_CERROR_ABT          0x2
+#define IVPU_MMU_CERROR_ATC_INV_SYNC 0x3
+
+static const char *ivpu_mmu_event_to_str(u32 cmd)
 {
 	switch (cmd) {
 	case IVPU_MMU_EVT_F_UUT:
@@ -276,6 +281,22 @@ static char *ivpu_mmu_event_to_str(u32 cmd)
 	}
 }
 
+static const char *ivpu_mmu_cmdq_err_to_str(u32 err)
+{
+	switch (err) {
+	case IVPU_MMU_CERROR_NONE:
+		return "No CMDQ Error";
+	case IVPU_MMU_CERROR_ILL:
+		return "Illegal command";
+	case IVPU_MMU_CERROR_ABT:
+		return "External abort on CMDQ read";
+	case IVPU_MMU_CERROR_ATC_INV_SYNC:
+		return "Sync failed to complete ATS invalidation";
+	default:
+		return "Unknown CMDQ Error";
+	}
+}
+
 static void ivpu_mmu_config_check(struct ivpu_device *vdev)
 {
 	u32 val_ref;
@@ -479,10 +500,7 @@ static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev)
 	u64 val;
 	int ret;
 
-	val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_SYNC) |
-	      FIELD_PREP(IVPU_MMU_CMD_SYNC_0_CS, 0x2) |
-	      FIELD_PREP(IVPU_MMU_CMD_SYNC_0_MSH, 0x3) |
-	      FIELD_PREP(IVPU_MMU_CMD_SYNC_0_MSI_ATTR, 0xf);
+	val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_SYNC);
 
 	ret = ivpu_mmu_cmdq_cmd_write(vdev, "SYNC", val, 0);
 	if (ret)
@@ -492,8 +510,15 @@ static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev)
 	REGV_WR32(IVPU_MMU_REG_CMDQ_PROD, q->prod);
 
 	ret = ivpu_mmu_cmdq_wait_for_cons(vdev);
-	if (ret)
-		ivpu_err(vdev, "Timed out waiting for consumer: %d\n", ret);
+	if (ret) {
+		u32 err;
+
+		val = REGV_RD32(IVPU_MMU_REG_CMDQ_CONS);
+		err = REG_GET_FLD(IVPU_MMU_REG_CMDQ_CONS, ERR, val);
+
+		ivpu_err(vdev, "Timed out waiting for MMU consumer: %d, error: %s\n", ret,
+			 ivpu_mmu_cmdq_err_to_str(err));
+	}
 
 	return ret;
 }
@@ -750,9 +775,12 @@ int ivpu_mmu_init(struct ivpu_device *vdev)
 
 	ivpu_dbg(vdev, MMU, "Init..\n");
 
-	drmm_mutex_init(&vdev->drm, &mmu->lock);
 	ivpu_mmu_config_check(vdev);
 
+	ret = drmm_mutex_init(&vdev->drm, &mmu->lock);
+	if (ret)
+		return ret;
+
 	ret = ivpu_mmu_structs_alloc(vdev);
 	if (ret)
 		return ret;
diff --git a/drivers/accel/ivpu/ivpu_mmu_context.c b/drivers/accel/ivpu/ivpu_mmu_context.c
index c1050a2df954..12a8c09d4547 100644
--- a/drivers/accel/ivpu/ivpu_mmu_context.c
+++ b/drivers/accel/ivpu/ivpu_mmu_context.c
@@ -5,6 +5,9 @@
 
 #include <linux/bitfield.h>
 #include <linux/highmem.h>
+#include <linux/set_memory.h>
+
+#include <drm/drm_cache.h>
 
 #include "ivpu_drv.h"
 #include "ivpu_hw.h"
@@ -39,12 +42,57 @@
 #define IVPU_MMU_ENTRY_MAPPED  (IVPU_MMU_ENTRY_FLAG_AF | IVPU_MMU_ENTRY_FLAG_USER | \
 				IVPU_MMU_ENTRY_FLAG_NG | IVPU_MMU_ENTRY_VALID)
 
+static void *ivpu_pgtable_alloc_page(struct ivpu_device *vdev, dma_addr_t *dma)
+{
+	dma_addr_t dma_addr;
+	struct page *page;
+	void *cpu;
+
+	page = alloc_page(GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
+	if (!page)
+		return NULL;
+
+	set_pages_array_wc(&page, 1);
+
+	dma_addr = dma_map_page(vdev->drm.dev, page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
+	if (dma_mapping_error(vdev->drm.dev, dma_addr))
+		goto err_free_page;
+
+	cpu = vmap(&page, 1, VM_MAP, pgprot_writecombine(PAGE_KERNEL));
+	if (!cpu)
+		goto err_dma_unmap_page;
+
+
+	*dma = dma_addr;
+	return cpu;
+
+err_dma_unmap_page:
+	dma_unmap_page(vdev->drm.dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
+
+err_free_page:
+	put_page(page);
+	return NULL;
+}
+
+static void ivpu_pgtable_free_page(struct ivpu_device *vdev, u64 *cpu_addr, dma_addr_t dma_addr)
+{
+	struct page *page;
+
+	if (cpu_addr) {
+		page = vmalloc_to_page(cpu_addr);
+		vunmap(cpu_addr);
+		dma_unmap_page(vdev->drm.dev, dma_addr & ~IVPU_MMU_ENTRY_FLAGS_MASK, PAGE_SIZE,
+			       DMA_BIDIRECTIONAL);
+		set_pages_array_wb(&page, 1);
+		put_page(page);
+	}
+}
+
 static int ivpu_mmu_pgtable_init(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)
 {
 	dma_addr_t pgd_dma;
 
-	pgtable->pgd_dma_ptr = dma_alloc_coherent(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pgd_dma,
-						  GFP_KERNEL);
+	pgtable->pgd_dma_ptr = ivpu_pgtable_alloc_page(vdev, &pgd_dma);
 	if (!pgtable->pgd_dma_ptr)
 		return -ENOMEM;
 
@@ -53,13 +101,6 @@ static int ivpu_mmu_pgtable_init(struct ivpu_device *vdev, struct ivpu_mmu_pgtab
 	return 0;
 }
 
-static void ivpu_mmu_pgtable_free(struct ivpu_device *vdev, u64 *cpu_addr, dma_addr_t dma_addr)
-{
-	if (cpu_addr)
-		dma_free_coherent(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, cpu_addr,
-				  dma_addr & ~IVPU_MMU_ENTRY_FLAGS_MASK);
-}
-
 static void ivpu_mmu_pgtables_free(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)
 {
 	int pgd_idx, pud_idx, pmd_idx;
@@ -84,19 +125,19 @@ static void ivpu_mmu_pgtables_free(struct ivpu_device *vdev, struct ivpu_mmu_pgt
 				pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];
 				pte_dma = pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx];
 
-				ivpu_mmu_pgtable_free(vdev, pte_dma_ptr, pte_dma);
+				ivpu_pgtable_free_page(vdev, pte_dma_ptr, pte_dma);
 			}
 
 			kfree(pgtable->pte_ptrs[pgd_idx][pud_idx]);
-			ivpu_mmu_pgtable_free(vdev, pmd_dma_ptr, pmd_dma);
+			ivpu_pgtable_free_page(vdev, pmd_dma_ptr, pmd_dma);
 		}
 
 		kfree(pgtable->pmd_ptrs[pgd_idx]);
 		kfree(pgtable->pte_ptrs[pgd_idx]);
-		ivpu_mmu_pgtable_free(vdev, pud_dma_ptr, pud_dma);
+		ivpu_pgtable_free_page(vdev, pud_dma_ptr, pud_dma);
 	}
 
-	ivpu_mmu_pgtable_free(vdev, pgtable->pgd_dma_ptr, pgtable->pgd_dma);
+	ivpu_pgtable_free_page(vdev, pgtable->pgd_dma_ptr, pgtable->pgd_dma);
 }
 
 static u64*
@@ -108,7 +149,7 @@ ivpu_mmu_ensure_pud(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable,
 	if (pud_dma_ptr)
 		return pud_dma_ptr;
 
-	pud_dma_ptr = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pud_dma, GFP_KERNEL);
+	pud_dma_ptr = ivpu_pgtable_alloc_page(vdev, &pud_dma);
 	if (!pud_dma_ptr)
 		return NULL;
 
@@ -131,7 +172,7 @@ ivpu_mmu_ensure_pud(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable,
 	kfree(pgtable->pmd_ptrs[pgd_idx]);
 
 err_free_pud_dma_ptr:
-	ivpu_mmu_pgtable_free(vdev, pud_dma_ptr, pud_dma);
+	ivpu_pgtable_free_page(vdev, pud_dma_ptr, pud_dma);
 	return NULL;
 }
 
@@ -145,7 +186,7 @@ ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable,
 	if (pmd_dma_ptr)
 		return pmd_dma_ptr;
 
-	pmd_dma_ptr = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pmd_dma, GFP_KERNEL);
+	pmd_dma_ptr = ivpu_pgtable_alloc_page(vdev, &pmd_dma);
 	if (!pmd_dma_ptr)
 		return NULL;
 
@@ -160,7 +201,7 @@ ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable,
 	return pmd_dma_ptr;
 
 err_free_pmd_dma_ptr:
-	ivpu_mmu_pgtable_free(vdev, pmd_dma_ptr, pmd_dma);
+	ivpu_pgtable_free_page(vdev, pmd_dma_ptr, pmd_dma);
 	return NULL;
 }
 
@@ -174,7 +215,7 @@ ivpu_mmu_ensure_pte(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable,
 	if (pte_dma_ptr)
 		return pte_dma_ptr;
 
-	pte_dma_ptr = dma_alloc_wc(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pte_dma, GFP_KERNEL);
+	pte_dma_ptr = ivpu_pgtable_alloc_page(vdev, &pte_dma);
 	if (!pte_dma_ptr)
 		return NULL;
 
@@ -249,38 +290,6 @@ static void ivpu_mmu_context_unmap_page(struct ivpu_mmu_context *ctx, u64 vpu_ad
 	ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] = IVPU_MMU_ENTRY_INVALID;
 }
 
-static void
-ivpu_mmu_context_flush_page_tables(struct ivpu_mmu_context *ctx, u64 vpu_addr, size_t size)
-{
-	struct ivpu_mmu_pgtable *pgtable = &ctx->pgtable;
-	u64 end_addr = vpu_addr + size;
-
-	/* Align to PMD entry (2 MB) */
-	vpu_addr &= ~(IVPU_MMU_PTE_MAP_SIZE - 1);
-
-	while (vpu_addr < end_addr) {
-		int pgd_idx = FIELD_GET(IVPU_MMU_PGD_INDEX_MASK, vpu_addr);
-		u64 pud_end = (pgd_idx + 1) * (u64)IVPU_MMU_PUD_MAP_SIZE;
-
-		while (vpu_addr < end_addr && vpu_addr < pud_end) {
-			int pud_idx = FIELD_GET(IVPU_MMU_PUD_INDEX_MASK, vpu_addr);
-			u64 pmd_end = (pud_idx + 1) * (u64)IVPU_MMU_PMD_MAP_SIZE;
-
-			while (vpu_addr < end_addr && vpu_addr < pmd_end) {
-				int pmd_idx = FIELD_GET(IVPU_MMU_PMD_INDEX_MASK, vpu_addr);
-
-				clflush_cache_range(pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx],
-						    IVPU_MMU_PGTABLE_SIZE);
-				vpu_addr += IVPU_MMU_PTE_MAP_SIZE;
-			}
-			clflush_cache_range(pgtable->pmd_ptrs[pgd_idx][pud_idx],
-					    IVPU_MMU_PGTABLE_SIZE);
-		}
-		clflush_cache_range(pgtable->pud_ptrs[pgd_idx], IVPU_MMU_PGTABLE_SIZE);
-	}
-	clflush_cache_range(pgtable->pgd_dma_ptr, IVPU_MMU_PGTABLE_SIZE);
-}
-
 static int
 ivpu_mmu_context_map_pages(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
 			   u64 vpu_addr, dma_addr_t dma_addr, size_t size, u64 prot)
@@ -327,6 +336,9 @@ ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
 	u64 prot;
 	u64 i;
 
+	if (drm_WARN_ON(&vdev->drm, !ctx))
+		return -EINVAL;
+
 	if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE))
 		return -EINVAL;
 
@@ -349,10 +361,11 @@ ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
 			mutex_unlock(&ctx->lock);
 			return ret;
 		}
-		ivpu_mmu_context_flush_page_tables(ctx, vpu_addr, size);
 		vpu_addr += size;
 	}
 
+	/* Ensure page table modifications are flushed from wc buffers to memory */
+	wmb();
 	mutex_unlock(&ctx->lock);
 
 	ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id);
@@ -369,8 +382,8 @@ ivpu_mmu_context_unmap_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ct
 	int ret;
 	u64 i;
 
-	if (!IS_ALIGNED(vpu_addr, IVPU_MMU_PAGE_SIZE))
-		ivpu_warn(vdev, "Unaligned vpu_addr: 0x%llx\n", vpu_addr);
+	if (drm_WARN_ON(&vdev->drm, !ctx))
+		return;
 
 	mutex_lock(&ctx->lock);
 
@@ -378,10 +391,11 @@ ivpu_mmu_context_unmap_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ct
 		size_t size = sg_dma_len(sg) + sg->offset;
 
 		ivpu_mmu_context_unmap_pages(ctx, vpu_addr, size);
-		ivpu_mmu_context_flush_page_tables(ctx, vpu_addr, size);
 		vpu_addr += size;
 	}
 
+	/* Ensure page table modifications are flushed from wc buffers to memory */
+	wmb();
 	mutex_unlock(&ctx->lock);
 
 	ret = ivpu_mmu_invalidate_tlb(vdev, ctx->id);
@@ -390,28 +404,34 @@ ivpu_mmu_context_unmap_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ct
 }
 
 int
-ivpu_mmu_context_insert_node_locked(struct ivpu_mmu_context *ctx,
-				    const struct ivpu_addr_range *range,
-				    u64 size, struct drm_mm_node *node)
+ivpu_mmu_context_insert_node(struct ivpu_mmu_context *ctx, const struct ivpu_addr_range *range,
+			     u64 size, struct drm_mm_node *node)
 {
-	lockdep_assert_held(&ctx->lock);
+	int ret;
+
+	WARN_ON(!range);
 
+	mutex_lock(&ctx->lock);
 	if (!ivpu_disable_mmu_cont_pages && size >= IVPU_MMU_CONT_PAGES_SIZE) {
-		if (!drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_CONT_PAGES_SIZE, 0,
-						 range->start, range->end, DRM_MM_INSERT_BEST))
-			return 0;
+		ret = drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_CONT_PAGES_SIZE, 0,
+						  range->start, range->end, DRM_MM_INSERT_BEST);
+		if (!ret)
+			goto unlock;
 	}
 
-	return drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_PAGE_SIZE, 0,
-					   range->start, range->end, DRM_MM_INSERT_BEST);
+	ret = drm_mm_insert_node_in_range(&ctx->mm, node, size, IVPU_MMU_PAGE_SIZE, 0,
+					  range->start, range->end, DRM_MM_INSERT_BEST);
+unlock:
+	mutex_unlock(&ctx->lock);
+	return ret;
 }
 
 void
-ivpu_mmu_context_remove_node_locked(struct ivpu_mmu_context *ctx, struct drm_mm_node *node)
+ivpu_mmu_context_remove_node(struct ivpu_mmu_context *ctx, struct drm_mm_node *node)
 {
-	lockdep_assert_held(&ctx->lock);
-
+	mutex_lock(&ctx->lock);
 	drm_mm_remove_node(node);
+	mutex_unlock(&ctx->lock);
 }
 
 static int
@@ -421,7 +441,6 @@ ivpu_mmu_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, u3
 	int ret;
 
 	mutex_init(&ctx->lock);
-	INIT_LIST_HEAD(&ctx->bo_list);
 
 	ret = ivpu_mmu_pgtable_init(vdev, &ctx->pgtable);
 	if (ret) {
diff --git a/drivers/accel/ivpu/ivpu_mmu_context.h b/drivers/accel/ivpu/ivpu_mmu_context.h
index f15d8c630d8a..535db3a1fc74 100644
--- a/drivers/accel/ivpu/ivpu_mmu_context.h
+++ b/drivers/accel/ivpu/ivpu_mmu_context.h
@@ -23,10 +23,9 @@ struct ivpu_mmu_pgtable {
 };
 
 struct ivpu_mmu_context {
-	struct mutex lock; /* protects: mm, pgtable, bo_list */
+	struct mutex lock; /* Protects: mm, pgtable */
 	struct drm_mm mm;
 	struct ivpu_mmu_pgtable pgtable;
-	struct list_head bo_list;
 	u32 id;
 };
 
@@ -39,11 +38,9 @@ int ivpu_mmu_user_context_init(struct ivpu_device *vdev, struct ivpu_mmu_context
 void ivpu_mmu_user_context_fini(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx);
 void ivpu_mmu_user_context_mark_invalid(struct ivpu_device *vdev, u32 ssid);
 
-int ivpu_mmu_context_insert_node_locked(struct ivpu_mmu_context *ctx,
-					const struct ivpu_addr_range *range,
-					u64 size, struct drm_mm_node *node);
-void ivpu_mmu_context_remove_node_locked(struct ivpu_mmu_context *ctx,
-					 struct drm_mm_node *node);
+int ivpu_mmu_context_insert_node(struct ivpu_mmu_context *ctx, const struct ivpu_addr_range *range,
+				 u64 size, struct drm_mm_node *node);
+void ivpu_mmu_context_remove_node(struct ivpu_mmu_context *ctx, struct drm_mm_node *node);
 
 int ivpu_mmu_context_map_sgt(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
 			     u64 vpu_addr, struct sg_table *sgt, bool llc_coherent);
diff --git a/drivers/accel/ivpu/ivpu_pm.c b/drivers/accel/ivpu/ivpu_pm.c
index e9b16cbc26f4..0af8864cb3b5 100644
--- a/drivers/accel/ivpu/ivpu_pm.c
+++ b/drivers/accel/ivpu/ivpu_pm.c
@@ -15,6 +15,7 @@
 #include "ivpu_fw.h"
 #include "ivpu_ipc.h"
 #include "ivpu_job.h"
+#include "ivpu_jsm_msg.h"
 #include "ivpu_mmu.h"
 #include "ivpu_pm.h"
 
@@ -22,6 +23,10 @@ static bool ivpu_disable_recovery;
 module_param_named_unsafe(disable_recovery, ivpu_disable_recovery, bool, 0644);
 MODULE_PARM_DESC(disable_recovery, "Disables recovery when VPU hang is detected");
 
+static unsigned long ivpu_tdr_timeout_ms;
+module_param_named(tdr_timeout_ms, ivpu_tdr_timeout_ms, ulong, 0644);
+MODULE_PARM_DESC(tdr_timeout_ms, "Timeout for device hang detection, in milliseconds, 0 - default");
+
 #define PM_RESCHEDULE_LIMIT     5
 
 static void ivpu_pm_prepare_cold_boot(struct ivpu_device *vdev)
@@ -69,27 +74,31 @@ static int ivpu_resume(struct ivpu_device *vdev)
 	ret = ivpu_hw_power_up(vdev);
 	if (ret) {
 		ivpu_err(vdev, "Failed to power up HW: %d\n", ret);
-		return ret;
+		goto err_power_down;
 	}
 
 	ret = ivpu_mmu_enable(vdev);
 	if (ret) {
 		ivpu_err(vdev, "Failed to resume MMU: %d\n", ret);
-		ivpu_hw_power_down(vdev);
-		return ret;
+		goto err_power_down;
 	}
 
 	ret = ivpu_boot(vdev);
-	if (ret) {
-		ivpu_mmu_disable(vdev);
-		ivpu_hw_power_down(vdev);
-		if (!ivpu_fw_is_cold_boot(vdev)) {
-			ivpu_warn(vdev, "Failed to resume the FW: %d. Retrying cold boot..\n", ret);
-			ivpu_pm_prepare_cold_boot(vdev);
-			goto retry;
-		} else {
-			ivpu_err(vdev, "Failed to resume the FW: %d\n", ret);
-		}
+	if (ret)
+		goto err_mmu_disable;
+
+	return 0;
+
+err_mmu_disable:
+	ivpu_mmu_disable(vdev);
+err_power_down:
+	ivpu_hw_power_down(vdev);
+
+	if (!ivpu_fw_is_cold_boot(vdev)) {
+		ivpu_pm_prepare_cold_boot(vdev);
+		goto retry;
+	} else {
+		ivpu_err(vdev, "Failed to resume the FW: %d\n", ret);
 	}
 
 	return ret;
@@ -136,6 +145,31 @@ void ivpu_pm_schedule_recovery(struct ivpu_device *vdev)
 	}
 }
 
+static void ivpu_job_timeout_work(struct work_struct *work)
+{
+	struct ivpu_pm_info *pm = container_of(work, struct ivpu_pm_info, job_timeout_work.work);
+	struct ivpu_device *vdev = pm->vdev;
+	unsigned long timeout_ms = ivpu_tdr_timeout_ms ? ivpu_tdr_timeout_ms : vdev->timeout.tdr;
+
+	ivpu_err(vdev, "TDR detected, timeout %lu ms", timeout_ms);
+	ivpu_hw_diagnose_failure(vdev);
+
+	ivpu_pm_schedule_recovery(vdev);
+}
+
+void ivpu_start_job_timeout_detection(struct ivpu_device *vdev)
+{
+	unsigned long timeout_ms = ivpu_tdr_timeout_ms ? ivpu_tdr_timeout_ms : vdev->timeout.tdr;
+
+	/* No-op if already queued */
+	queue_delayed_work(system_wq, &vdev->pm->job_timeout_work, msecs_to_jiffies(timeout_ms));
+}
+
+void ivpu_stop_job_timeout_detection(struct ivpu_device *vdev)
+{
+	cancel_delayed_work_sync(&vdev->pm->job_timeout_work);
+}
+
 int ivpu_pm_suspend_cb(struct device *dev)
 {
 	struct drm_device *drm = dev_get_drvdata(dev);
@@ -153,6 +187,8 @@ int ivpu_pm_suspend_cb(struct device *dev)
 		}
 	}
 
+	ivpu_jsm_pwr_d0i3_enter(vdev);
+
 	ivpu_suspend(vdev);
 	ivpu_pm_prepare_warm_boot(vdev);
 
@@ -188,6 +224,7 @@ int ivpu_pm_runtime_suspend_cb(struct device *dev)
 {
 	struct drm_device *drm = dev_get_drvdata(dev);
 	struct ivpu_device *vdev = to_ivpu_device(drm);
+	bool hw_is_idle = true;
 	int ret;
 
 	ivpu_dbg(vdev, PM, "Runtime suspend..\n");
@@ -200,11 +237,16 @@ int ivpu_pm_runtime_suspend_cb(struct device *dev)
 		return -EAGAIN;
 	}
 
+	if (!vdev->pm->suspend_reschedule_counter)
+		hw_is_idle = false;
+	else if (ivpu_jsm_pwr_d0i3_enter(vdev))
+		hw_is_idle = false;
+
 	ret = ivpu_suspend(vdev);
 	if (ret)
 		ivpu_err(vdev, "Failed to set suspend VPU: %d\n", ret);
 
-	if (!vdev->pm->suspend_reschedule_counter) {
+	if (!hw_is_idle) {
 		ivpu_warn(vdev, "VPU failed to enter idle, force suspended.\n");
 		ivpu_pm_prepare_cold_boot(vdev);
 	} else {
@@ -304,6 +346,7 @@ void ivpu_pm_init(struct ivpu_device *vdev)
 
 	atomic_set(&pm->in_reset, 0);
 	INIT_WORK(&pm->recovery_work, ivpu_pm_recovery_work);
+	INIT_DELAYED_WORK(&pm->job_timeout_work, ivpu_job_timeout_work);
 
 	if (ivpu_disable_recovery)
 		delay = -1;
@@ -318,6 +361,7 @@ void ivpu_pm_init(struct ivpu_device *vdev)
 
 void ivpu_pm_cancel_recovery(struct ivpu_device *vdev)
 {
+	drm_WARN_ON(&vdev->drm, delayed_work_pending(&vdev->pm->job_timeout_work));
 	cancel_work_sync(&vdev->pm->recovery_work);
 }
 
diff --git a/drivers/accel/ivpu/ivpu_pm.h b/drivers/accel/ivpu/ivpu_pm.h
index 044db150be07..97c6e0b0aa42 100644
--- a/drivers/accel/ivpu/ivpu_pm.h
+++ b/drivers/accel/ivpu/ivpu_pm.h
@@ -12,6 +12,7 @@ struct ivpu_device;
 
 struct ivpu_pm_info {
 	struct ivpu_device *vdev;
+	struct delayed_work job_timeout_work;
 	struct work_struct recovery_work;
 	atomic_t in_reset;
 	atomic_t reset_counter;
@@ -37,5 +38,7 @@ int __must_check ivpu_rpm_get_if_active(struct ivpu_device *vdev);
 void ivpu_rpm_put(struct ivpu_device *vdev);
 
 void ivpu_pm_schedule_recovery(struct ivpu_device *vdev);
+void ivpu_start_job_timeout_detection(struct ivpu_device *vdev);
+void ivpu_stop_job_timeout_detection(struct ivpu_device *vdev);
 
 #endif /* __IVPU_PM_H__ */
diff --git a/drivers/accel/ivpu/vpu_boot_api.h b/drivers/accel/ivpu/vpu_boot_api.h
index 6b71be92ba65..04c954258563 100644
--- a/drivers/accel/ivpu/vpu_boot_api.h
+++ b/drivers/accel/ivpu/vpu_boot_api.h
@@ -11,7 +11,10 @@
  *  The bellow values will be used to construct the version info this way:
  *  fw_bin_header->api_version[VPU_BOOT_API_VER_ID] = (VPU_BOOT_API_VER_MAJOR << 16) |
  *  VPU_BOOT_API_VER_MINOR;
- *  VPU_BOOT_API_VER_PATCH will be ignored. KMD and compatibility is not affected if this changes.
+ *  VPU_BOOT_API_VER_PATCH will be ignored. KMD and compatibility is not affected if this changes
+ *  This information is collected by using vpuip_2/application/vpuFirmware/make_std_fw_image.py
+ *  If a header is missing this info we ignore the header, if a header is missing or contains
+ *  partial info a build error will be generated.
  */
 
 /*
@@ -24,12 +27,12 @@
  * Minor version changes when API backward compatibility is preserved.
  * Resets to 0 if Major version is incremented.
  */
-#define VPU_BOOT_API_VER_MINOR 12
+#define VPU_BOOT_API_VER_MINOR 20
 
 /*
  * API header changed (field names, documentation, formatting) but API itself has not been changed
  */
-#define VPU_BOOT_API_VER_PATCH 2
+#define VPU_BOOT_API_VER_PATCH 4
 
 /*
  * Index in the API version table
@@ -63,6 +66,12 @@ struct vpu_firmware_header {
 	/* Size of memory require for firmware execution */
 	u32 runtime_size;
 	u32 shave_nn_fw_size;
+	/* Size of primary preemption buffer. */
+	u32 preemption_buffer_1_size;
+	/* Size of secondary preemption buffer. */
+	u32 preemption_buffer_2_size;
+	/* Space reserved for future preemption-related fields. */
+	u32 preemption_reserved[6];
 };
 
 /*
@@ -89,6 +98,14 @@ enum VPU_BOOT_L2_CACHE_CFG_TYPE {
 	VPU_BOOT_L2_CACHE_CFG_NUM = 2
 };
 
+/** VPU MCA ECC signalling mode. By default, no signalling is used */
+enum VPU_BOOT_MCA_ECC_SIGNAL_TYPE {
+	VPU_BOOT_MCA_ECC_NONE = 0,
+	VPU_BOOT_MCA_ECC_CORR = 1,
+	VPU_BOOT_MCA_ECC_FATAL = 2,
+	VPU_BOOT_MCA_ECC_BOTH = 3
+};
+
 /**
  * Logging destinations.
  *
@@ -131,9 +148,11 @@ enum vpu_trace_destination {
 #define VPU_TRACE_PROC_BIT_ACT_SHV_3 22
 #define VPU_TRACE_PROC_NO_OF_HW_DEVS 23
 
-/* KMB HW component IDs are sequential, so define first and last IDs. */
-#define VPU_TRACE_PROC_BIT_KMB_FIRST VPU_TRACE_PROC_BIT_LRT
-#define VPU_TRACE_PROC_BIT_KMB_LAST  VPU_TRACE_PROC_BIT_SHV_15
+/* VPU 30xx HW component IDs are sequential, so define first and last IDs. */
+#define VPU_TRACE_PROC_BIT_30XX_FIRST VPU_TRACE_PROC_BIT_LRT
+#define VPU_TRACE_PROC_BIT_30XX_LAST  VPU_TRACE_PROC_BIT_SHV_15
+#define VPU_TRACE_PROC_BIT_KMB_FIRST  VPU_TRACE_PROC_BIT_30XX_FIRST
+#define VPU_TRACE_PROC_BIT_KMB_LAST   VPU_TRACE_PROC_BIT_30XX_LAST
 
 struct vpu_boot_l2_cache_config {
 	u8 use;
@@ -148,6 +167,25 @@ struct vpu_warm_boot_section {
 	u32 is_clear_op;
 };
 
+/*
+ * When HW scheduling mode is enabled, a present period is defined.
+ * It will be used by VPU to swap between normal and focus priorities
+ * to prevent starving of normal priority band (when implemented).
+ * Host must provide a valid value at boot time in
+ * `vpu_focus_present_timer_ms`. If the value provided by the host is not within the
+ * defined range a default value will be used. Here we define the min. and max.
+ * allowed values and the and default value of the present period. Units are milliseconds.
+ */
+#define VPU_PRESENT_CALL_PERIOD_MS_DEFAULT 50
+#define VPU_PRESENT_CALL_PERIOD_MS_MIN	   16
+#define VPU_PRESENT_CALL_PERIOD_MS_MAX	   10000
+
+/**
+ * Macros to enable various operation modes within the VPU.
+ * To be defined as part of 32 bit mask.
+ */
+#define VPU_OP_MODE_SURVIVABILITY 0x1
+
 struct vpu_boot_params {
 	u32 magic;
 	u32 vpu_id;
@@ -218,6 +256,7 @@ struct vpu_boot_params {
 	 * the threshold will not be logged); applies to every enabled logging
 	 * destination and loggable HW component. See 'mvLog_t' enum for acceptable
 	 * values.
+	 * TODO: EISW-33556: Move log level definition (mvLog_t) to this file.
 	 */
 	u32 default_trace_level;
 	u32 boot_type;
@@ -249,7 +288,36 @@ struct vpu_boot_params {
 	u32 temp_sensor_period_ms;
 	/** PLL ratio for efficient clock frequency */
 	u32 pn_freq_pll_ratio;
-	u32 pad4[28];
+	/** DVFS Mode: Default: 0, Max Performance: 1, On Demand: 2, Power Save: 3 */
+	u32 dvfs_mode;
+	/**
+	 * Depending on DVFS Mode:
+	 * On-demand: Default if 0.
+	 *    Bit 0-7   - uint8_t: Highest residency percent
+	 *    Bit 8-15  - uint8_t: High residency percent
+	 *    Bit 16-23 - uint8_t: Low residency percent
+	 *    Bit 24-31 - uint8_t: Lowest residency percent
+	 *    Bit 32-35 - unsigned 4b: PLL Ratio increase amount on highest residency
+	 *    Bit 36-39 - unsigned 4b: PLL Ratio increase amount on high residency
+	 *    Bit 40-43 - unsigned 4b: PLL Ratio decrease amount on low residency
+	 *    Bit 44-47 - unsigned 4b: PLL Ratio decrease amount on lowest frequency
+	 *    Bit 48-55 - uint8_t: Period (ms) for residency decisions
+	 *    Bit 56-63 - uint8_t: Averaging windows (as multiples of period. Max: 30 decimal)
+	 * Power Save/Max Performance: Unused
+	 */
+	u64 dvfs_param;
+	/**
+	 * D0i3 delayed entry
+	 * Bit0: Disable CPU state save on D0i2 entry flow.
+	 *       0: Every D0i2 entry saves state. Save state IPC message ignored.
+	 *       1: IPC message required to save state on D0i3 entry flow.
+	 */
+	u32 d0i3_delayed_entry;
+	/* Time spent by VPU in D0i3 state */
+	u64 d0i3_residency_time_us;
+	/* Value of VPU perf counter at the time of entering D0i3 state . */
+	u64 d0i3_entry_vpu_ts;
+	u32 pad4[20];
 	/* Warm boot information: 0x400 - 0x43F */
 	u32 warm_boot_sections_count;
 	u32 warm_boot_start_address_reference;
@@ -274,8 +342,12 @@ struct vpu_boot_params {
 	u32 vpu_scheduling_mode;
 	/* Present call period in milliseconds. */
 	u32 vpu_focus_present_timer_ms;
-	/* Unused/reserved: 0x478 - 0xFFF */
-	u32 pad6[738];
+	/* VPU ECC Signaling */
+	u32 vpu_uses_ecc_mca_signal;
+	/* Values defined by VPU_OP_MODE* macros */
+	u32 vpu_operation_mode;
+	/* Unused/reserved: 0x480 - 0xFFF */
+	u32 pad6[736];
 };
 
 /*
diff --git a/drivers/accel/ivpu/vpu_jsm_api.h b/drivers/accel/ivpu/vpu_jsm_api.h
index 2949ec8365bd..7da7622742be 100644
--- a/drivers/accel/ivpu/vpu_jsm_api.h
+++ b/drivers/accel/ivpu/vpu_jsm_api.h
@@ -22,12 +22,12 @@
 /*
  * Minor version changes when API backward compatibility is preserved.
  */
-#define VPU_JSM_API_VER_MINOR 0
+#define VPU_JSM_API_VER_MINOR 15
 
 /*
  * API header changed (field names, documentation, formatting) but API itself has not been changed
  */
-#define VPU_JSM_API_VER_PATCH 1
+#define VPU_JSM_API_VER_PATCH 0
 
 /*
  * Index in the API version table
@@ -84,11 +84,13 @@
  * Job flags bit masks.
  */
 #define VPU_JOB_FLAGS_NULL_SUBMISSION_MASK 0x00000001
+#define VPU_JOB_FLAGS_PRIVATE_DATA_MASK	   0xFF000000
 
 /*
  * Sizes of the reserved areas in jobs, in bytes.
  */
-#define VPU_JOB_RESERVED_BYTES	     16
+#define VPU_JOB_RESERVED_BYTES 8
+
 /*
  * Sizes of the reserved areas in job queues, in bytes.
  */
@@ -108,6 +110,20 @@
  */
 #define VPU_DYNDBG_CMD_MAX_LEN 96
 
+/*
+ * For HWS command queue scheduling, we can prioritise command queues inside the
+ * same process with a relative in-process priority. Valid values for relative
+ * priority are given below - max and min.
+ */
+#define VPU_HWS_COMMAND_QUEUE_MAX_IN_PROCESS_PRIORITY 7
+#define VPU_HWS_COMMAND_QUEUE_MIN_IN_PROCESS_PRIORITY -7
+
+/*
+ * For HWS priority scheduling, we can have multiple realtime priority bands.
+ * They are numbered 0 to a MAX.
+ */
+#define VPU_HWS_MAX_REALTIME_PRIORITY_LEVEL 31U
+
 /*
  * Job format.
  */
@@ -117,8 +133,14 @@ struct vpu_job_queue_entry {
 	u32 flags; /**< Flags bit field, see VPU_JOB_FLAGS_* above */
 	u64 root_page_table_addr; /**< Address of root page table to use for this job */
 	u64 root_page_table_update_counter; /**< Page tables update events counter */
-	u64 preemption_buffer_address; /**< Address of the preemption buffer to use for this job */
-	u64 preemption_buffer_size; /**< Size of the preemption buffer to use for this job */
+	u64 primary_preempt_buf_addr;
+	/**< Address of the primary preemption buffer to use for this job */
+	u32 primary_preempt_buf_size;
+	/**< Size of the primary preemption buffer to use for this job */
+	u32 secondary_preempt_buf_size;
+	/**< Size of secondary preemption buffer to use for this job */
+	u64 secondary_preempt_buf_addr;
+	/**< Address of secondary preemption buffer to use for this job */
 	u8 reserved_0[VPU_JOB_RESERVED_BYTES];
 };
 
@@ -152,6 +174,46 @@ enum vpu_trace_entity_type {
 	VPU_TRACE_ENTITY_TYPE_HW_COMPONENT = 2,
 };
 
+/*
+ * HWS specific log buffer header details.
+ * Total size is 32 bytes.
+ */
+struct vpu_hws_log_buffer_header {
+	/* Written by VPU after adding a log entry. Initialised by host to 0. */
+	u32 first_free_entry_index;
+	/* Incremented by VPU every time the VPU overwrites the 0th entry;
+	 * initialised by host to 0.
+	 */
+	u32 wraparound_count;
+	/*
+	 * This is the number of buffers that can be stored in the log buffer provided by the host.
+	 * It is written by host before passing buffer to VPU. VPU should consider it read-only.
+	 */
+	u64 num_of_entries;
+	u64 reserved[2];
+};
+
+/*
+ * HWS specific log buffer entry details.
+ * Total size is 32 bytes.
+ */
+struct vpu_hws_log_buffer_entry {
+	/* VPU timestamp must be an invariant timer tick (not impacted by DVFS) */
+	u64 vpu_timestamp;
+	/*
+	 * Operation type:
+	 *     0 - context state change
+	 *     1 - queue new work
+	 *     2 - queue unwait sync object
+	 *     3 - queue no more work
+	 *     4 - queue wait sync object
+	 */
+	u32 operation_type;
+	u32 reserved;
+	/* Operation data depends on operation type */
+	u64 operation_data[2];
+};
+
 /*
  * Host <-> VPU IPC messages types.
  */
@@ -228,6 +290,23 @@ enum vpu_ipc_msg_type {
 	 * deallocated or reassigned to another context.
 	 */
 	VPU_JSM_MSG_HWS_REGISTER_DB = 0x1117,
+	/** Control command: Log buffer setting */
+	VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG = 0x1118,
+	/* Control command: Suspend command queue. */
+	VPU_JSM_MSG_HWS_SUSPEND_CMDQ = 0x1119,
+	/* Control command: Resume command queue */
+	VPU_JSM_MSG_HWS_RESUME_CMDQ = 0x111a,
+	/* Control command: Resume engine after reset */
+	VPU_JSM_MSG_HWS_ENGINE_RESUME = 0x111b,
+	/* Control command: Enable survivability/DCT mode */
+	VPU_JSM_MSG_DCT_ENABLE = 0x111c,
+	/* Control command: Disable survivability/DCT mode */
+	VPU_JSM_MSG_DCT_DISABLE = 0x111d,
+	/**
+	 * Dump VPU state. To be used for debug purposes only.
+	 * NOTE: Please introduce new ASYNC commands before this one. *
+	 */
+	VPU_JSM_MSG_STATE_DUMP = 0x11FF,
 	/* IPC Host -> Device, General commands */
 	VPU_JSM_MSG_GENERAL_CMD = 0x1200,
 	VPU_JSM_MSG_BLOB_DEINIT = VPU_JSM_MSG_GENERAL_CMD,
@@ -236,6 +315,10 @@ enum vpu_ipc_msg_type {
 	 * Linux command: `echo '<dyndbg_cmd>' > <debugfs>/dynamic_debug/control`.
 	 */
 	VPU_JSM_MSG_DYNDBG_CONTROL = 0x1201,
+	/**
+	 * Perform the save procedure for the D0i3 entry
+	 */
+	VPU_JSM_MSG_PWR_D0I3_ENTER = 0x1202,
 	/* IPC Device -> Host, Job completion */
 	VPU_JSM_MSG_JOB_DONE = 0x2100,
 	/* IPC Device -> Host, Async command completion */
@@ -304,11 +387,35 @@ enum vpu_ipc_msg_type {
 	VPU_JSM_MSG_DESTROY_CMD_QUEUE_RSP = 0x2216,
 	/** Response to control command: Set context scheduling properties */
 	VPU_JSM_MSG_SET_CONTEXT_SCHED_PROPERTIES_RSP = 0x2217,
+	/** Response to control command: Log buffer setting */
+	VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG_RSP = 0x2218,
+	/* IPC Device -> Host, HWS notify index entry of log buffer written */
+	VPU_JSM_MSG_HWS_SCHEDULING_LOG_NOTIFICATION = 0x2219,
+	/* IPC Device -> Host, HWS completion of a context suspend request */
+	VPU_JSM_MSG_HWS_SUSPEND_CMDQ_DONE = 0x221a,
+	/* Response to control command: Resume command queue */
+	VPU_JSM_MSG_HWS_RESUME_CMDQ_RSP = 0x221b,
+	/* Response to control command: Resume engine command response */
+	VPU_JSM_MSG_HWS_RESUME_ENGINE_DONE = 0x221c,
+	/* Response to control command: Enable survivability/DCT mode */
+	VPU_JSM_MSG_DCT_ENABLE_DONE = 0x221d,
+	/* Response to control command: Disable survivability/DCT mode */
+	VPU_JSM_MSG_DCT_DISABLE_DONE = 0x221e,
+	/**
+	 * Response to state dump control command.
+	 * NOTE: Please introduce new ASYNC responses before this one. *
+	 */
+	VPU_JSM_MSG_STATE_DUMP_RSP = 0x22FF,
 	/* IPC Device -> Host, General command completion */
 	VPU_JSM_MSG_GENERAL_CMD_DONE = 0x2300,
 	VPU_JSM_MSG_BLOB_DEINIT_DONE = VPU_JSM_MSG_GENERAL_CMD_DONE,
 	/** Response to VPU_JSM_MSG_DYNDBG_CONTROL. */
 	VPU_JSM_MSG_DYNDBG_CONTROL_RSP = 0x2301,
+	/**
+	 * Acknowledgment of completion of the save procedure initiated by
+	 * VPU_JSM_MSG_PWR_D0I3_ENTER
+	 */
+	VPU_JSM_MSG_PWR_D0I3_ENTER_DONE = 0x2302,
 };
 
 enum vpu_ipc_msg_status { VPU_JSM_MSG_FREE, VPU_JSM_MSG_ALLOCATED };
@@ -593,12 +700,12 @@ struct vpu_ipc_msg_payload_hws_priority_band_setup {
 	 * Default quantum in 100ns units for scheduling across processes
 	 * within a priority band
 	 */
-	u64 process_quantum[VPU_HWS_NUM_PRIORITY_BANDS];
+	u32 process_quantum[VPU_HWS_NUM_PRIORITY_BANDS];
 	/*
 	 * Default grace period in 100ns units for processes that preempt each
 	 * other within a priority band
 	 */
-	u64 process_grace_period[VPU_HWS_NUM_PRIORITY_BANDS];
+	u32 process_grace_period[VPU_HWS_NUM_PRIORITY_BANDS];
 	/*
 	 * For normal priority band, specifies the target VPU percentage
 	 * in situations when it's starved by the focus band.
@@ -608,32 +715,51 @@ struct vpu_ipc_msg_payload_hws_priority_band_setup {
 	u32 reserved_0;
 };
 
-/* HWS create command queue request */
+/*
+ * @brief HWS create command queue request.
+ * Host will create a command queue via this command.
+ * Note: Cmdq group is a handle of an object which
+ * may contain one or more command queues.
+ * @see VPU_JSM_MSG_CREATE_CMD_QUEUE
+ * @see VPU_JSM_MSG_CREATE_CMD_QUEUE_RSP
+ */
 struct vpu_ipc_msg_payload_hws_create_cmdq {
 	/* Process id */
 	u64 process_id;
 	/* Host SSID */
 	u32 host_ssid;
-	/* Zero Padding */
-	u32 reserved;
+	/* Engine for which queue is being created */
+	u32 engine_idx;
+	/*
+	 * Cmdq group may be set to 0 or equal to
+	 * cmdq_id while each priority band contains
+	 * only single engine instances.
+	 */
+	u64 cmdq_group;
 	/* Command queue id */
 	u64 cmdq_id;
 	/* Command queue base */
 	u64 cmdq_base;
 	/* Command queue size */
 	u32 cmdq_size;
-	/* Reserved */
+	/* Zero padding */
 	u32 reserved_0;
 };
 
-/* HWS create command queue response */
+/*
+ * @brief HWS create command queue response.
+ * @see VPU_JSM_MSG_CREATE_CMD_QUEUE
+ * @see VPU_JSM_MSG_CREATE_CMD_QUEUE_RSP
+ */
 struct vpu_ipc_msg_payload_hws_create_cmdq_rsp {
 	/* Process id */
 	u64 process_id;
 	/* Host SSID */
 	u32 host_ssid;
-	/* Zero Padding */
-	u32 reserved;
+	/* Engine for which queue is being created */
+	u32 engine_idx;
+	/* Command queue group */
+	u64 cmdq_group;
 	/* Command queue id */
 	u64 cmdq_id;
 };
@@ -661,7 +787,7 @@ struct vpu_ipc_msg_payload_hws_set_context_sched_properties {
 	/* Inside realtime band assigns a further priority */
 	u32 realtime_priority_level;
 	/* Priority relative to other contexts in the same process */
-	u32 in_process_priority;
+	s32 in_process_priority;
 	/* Zero padding / Reserved */
 	u32 reserved_1;
 	/* Context quantum relative to other contexts of same priority in the same process */
@@ -694,6 +820,123 @@ struct vpu_jsm_hws_register_db {
 	u64 cmdq_size;
 };
 
+/*
+ * @brief Structure to set another buffer to be used for scheduling-related logging.
+ * The size of the logging buffer and the number of entries is defined as part of the
+ * buffer itself as described next.
+ * The log buffer received from the host is made up of;
+ *   - header:     32 bytes in size, as shown in 'struct vpu_hws_log_buffer_header'.
+ *                 The header contains the number of log entries in the buffer.
+ *   - log entry:  0 to n-1, each log entry is 32 bytes in size, as shown in
+ *                 'struct vpu_hws_log_buffer_entry'.
+ *                 The entry contains the VPU timestamp, operation type and data.
+ * The host should provide the notify index value of log buffer to VPU. This is a
+ * value defined within the log buffer and when written to will generate the
+ * scheduling log notification.
+ * The host should set engine_idx and vpu_log_buffer_va to 0 to disable logging
+ * for a particular engine.
+ * VPU will handle one log buffer for each of supported engines.
+ * VPU should allow the logging to consume one host_ssid.
+ * @see VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG
+ * @see VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG_RSP
+ * @see VPU_JSM_MSG_HWS_SCHEDULING_LOG_NOTIFICATION
+ */
+struct vpu_ipc_msg_payload_hws_set_scheduling_log {
+	/* Engine ordinal */
+	u32 engine_idx;
+	/* Host SSID */
+	u32 host_ssid;
+	/*
+	 * VPU log buffer virtual address.
+	 * Set to 0 to disable logging for this engine.
+	 */
+	u64 vpu_log_buffer_va;
+	/*
+	 * Notify index of log buffer. VPU_JSM_MSG_HWS_SCHEDULING_LOG_NOTIFICATION
+	 * is generated when an event log is written to this index.
+	 */
+	u64 notify_index;
+};
+
+/*
+ * @brief The scheduling log notification is generated by VPU when it writes
+ * an event into the log buffer at the notify_index. VPU notifies host with
+ * VPU_JSM_MSG_HWS_SCHEDULING_LOG_NOTIFICATION. This is an asynchronous
+ * message from VPU to host.
+ * @see VPU_JSM_MSG_HWS_SCHEDULING_LOG_NOTIFICATION
+ * @see VPU_JSM_MSG_HWS_SET_SCHEDULING_LOG
+ */
+struct vpu_ipc_msg_payload_hws_scheduling_log_notification {
+	/* Engine ordinal */
+	u32 engine_idx;
+	/* Zero Padding */
+	u32 reserved_0;
+};
+
+/*
+ * @brief HWS suspend command queue request and done structure.
+ * Host will request the suspend of contexts and VPU will;
+ *   - Suspend all work on this context
+ *   - Preempt any running work
+ *   - Asynchronously perform the above and return success immediately once
+ *     all items above are started successfully
+ *   - Notify the host of completion of these operations via
+ *     VPU_JSM_MSG_HWS_SUSPEND_CMDQ_DONE
+ *   - Reject any other context operations on a context with an in-flight
+ *     suspend request running
+ * Same structure used when VPU notifies host of completion of a context suspend
+ * request. The ids and suspend fence value reported in this command will match
+ * the one in the request from the host to suspend the context. Once suspend is
+ * complete, VPU will not access any data relating to this command queue until
+ * it is resumed.
+ * @see VPU_JSM_MSG_HWS_SUSPEND_CMDQ
+ * @see VPU_JSM_MSG_HWS_SUSPEND_CMDQ_DONE
+ */
+struct vpu_ipc_msg_payload_hws_suspend_cmdq {
+	/* Host SSID */
+	u32 host_ssid;
+	/* Zero Padding */
+	u32 reserved_0;
+	/* Command queue id */
+	u64 cmdq_id;
+	/*
+	 * Suspend fence value - reported by the VPU suspend context
+	 * completed once suspend is complete.
+	 */
+	u64 suspend_fence_value;
+};
+
+/*
+ * @brief HWS Resume command queue request / response structure.
+ * Host will request the resume of a context;
+ *  - VPU will resume all work on this context
+ *  - Scheduler will allow this context to be scheduled
+ * @see VPU_JSM_MSG_HWS_RESUME_CMDQ
+ * @see VPU_JSM_MSG_HWS_RESUME_CMDQ_RSP
+ */
+struct vpu_ipc_msg_payload_hws_resume_cmdq {
+	/* Host SSID */
+	u32 host_ssid;
+	/* Zero Padding */
+	u32 reserved_0;
+	/* Command queue id */
+	u64 cmdq_id;
+};
+
+/*
+ * @brief HWS Resume engine request / response structure.
+ * After a HWS engine reset, all scheduling is stopped on VPU until a engine resume.
+ * Host shall send this command to resume scheduling of any valid queue.
+ * @see VPU_JSM_MSG_HWS_RESUME_ENGINE
+ * @see VPU_JSM_MSG_HWS_RESUME_ENGINE_DONE
+ */
+struct vpu_ipc_msg_payload_hws_resume_engine {
+	/* Engine to be resumed */
+	u32 engine_idx;
+	/* Reserved */
+	u32 reserved_0;
+};
+
 /**
  * Payload for VPU_JSM_MSG_TRACE_SET_CONFIG[_RSP] and
  * VPU_JSM_MSG_TRACE_GET_CONFIG_RSP messages.
@@ -938,6 +1181,35 @@ struct vpu_ipc_msg_payload_dyndbg_control {
 	char dyndbg_cmd[VPU_DYNDBG_CMD_MAX_LEN];
 };
 
+/**
+ * Payload for VPU_JSM_MSG_PWR_D0I3_ENTER
+ *
+ * This is a bi-directional payload.
+ */
+struct vpu_ipc_msg_payload_pwr_d0i3_enter {
+	/**
+	 * 0: VPU_JSM_MSG_PWR_D0I3_ENTER_DONE is not sent to the host driver
+	 *    The driver will poll for D0i2 Idle state transitions.
+	 * 1: VPU_JSM_MSG_PWR_D0I3_ENTER_DONE is sent after VPU state save is complete
+	 */
+	u32 send_response;
+	u32 reserved_0;
+};
+
+/**
+ * Payload for VPU_JSM_MSG_DCT_ENABLE message.
+ *
+ * Default values for DCT active/inactive times are 5.3ms and 30ms respectively,
+ * corresponding to a 85% duty cycle. This payload allows the host to tune these
+ * values according to application requirements.
+ */
+struct vpu_ipc_msg_payload_pwr_dct_control {
+	/** Duty cycle active time in microseconds */
+	u32 dct_active_us;
+	/** Duty cycle inactive time in microseconds */
+	u32 dct_inactive_us;
+};
+
 /*
  * Payloads union, used to define complete message format.
  */
@@ -974,6 +1246,13 @@ union vpu_ipc_msg_payload {
 	struct vpu_ipc_msg_payload_hws_destroy_cmdq hws_destroy_cmdq;
 	struct vpu_ipc_msg_payload_hws_set_context_sched_properties
 		hws_set_context_sched_properties;
+	struct vpu_ipc_msg_payload_hws_set_scheduling_log hws_set_scheduling_log;
+	struct vpu_ipc_msg_payload_hws_scheduling_log_notification hws_scheduling_log_notification;
+	struct vpu_ipc_msg_payload_hws_suspend_cmdq hws_suspend_cmdq;
+	struct vpu_ipc_msg_payload_hws_resume_cmdq hws_resume_cmdq;
+	struct vpu_ipc_msg_payload_hws_resume_engine hws_resume_engine;
+	struct vpu_ipc_msg_payload_pwr_d0i3_enter pwr_d0i3_enter;
+	struct vpu_ipc_msg_payload_pwr_dct_control pwr_dct_control;
 };
 
 /*
diff --git a/drivers/accel/qaic/Makefile b/drivers/accel/qaic/Makefile
index 2418418f7a50..3f7f6dfde7f2 100644
--- a/drivers/accel/qaic/Makefile
+++ b/drivers/accel/qaic/Makefile
@@ -9,4 +9,5 @@ qaic-y := \
 	mhi_controller.o \
 	qaic_control.o \
 	qaic_data.o \
-	qaic_drv.o
+	qaic_drv.o \
+	qaic_timesync.o
diff --git a/drivers/accel/qaic/mhi_controller.c b/drivers/accel/qaic/mhi_controller.c
index 5036e58e7235..5d3cc30009cc 100644
--- a/drivers/accel/qaic/mhi_controller.c
+++ b/drivers/accel/qaic/mhi_controller.c
@@ -348,7 +348,7 @@ static struct mhi_channel_config aic100_channels[] = {
 		.local_elements = 0,
 		.event_ring = 0,
 		.dir = DMA_TO_DEVICE,
-		.ee_mask = MHI_CH_EE_SBL | MHI_CH_EE_AMSS,
+		.ee_mask = MHI_CH_EE_SBL,
 		.pollcfg = 0,
 		.doorbell = MHI_DB_BRST_DISABLE,
 		.lpm_notify = false,
@@ -364,7 +364,39 @@ static struct mhi_channel_config aic100_channels[] = {
 		.local_elements = 0,
 		.event_ring = 0,
 		.dir = DMA_FROM_DEVICE,
-		.ee_mask = MHI_CH_EE_SBL | MHI_CH_EE_AMSS,
+		.ee_mask = MHI_CH_EE_SBL,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.name = "QAIC_TIMESYNC_PERIODIC",
+		.num = 22,
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_TO_DEVICE,
+		.ee_mask = MHI_CH_EE_AMSS,
+		.pollcfg = 0,
+		.doorbell = MHI_DB_BRST_DISABLE,
+		.lpm_notify = false,
+		.offload_channel = false,
+		.doorbell_mode_switch = false,
+		.auto_queue = false,
+		.wake_capable = false,
+	},
+	{
+		.num = 23,
+		.name = "QAIC_TIMESYNC_PERIODIC",
+		.num_elements = 32,
+		.local_elements = 0,
+		.event_ring = 0,
+		.dir = DMA_FROM_DEVICE,
+		.ee_mask = MHI_CH_EE_AMSS,
 		.pollcfg = 0,
 		.doorbell = MHI_DB_BRST_DISABLE,
 		.lpm_notify = false,
@@ -468,7 +500,7 @@ static int mhi_reset_and_async_power_up(struct mhi_controller *mhi_cntrl)
 }
 
 struct mhi_controller *qaic_mhi_register_controller(struct pci_dev *pci_dev, void __iomem *mhi_bar,
-						    int mhi_irq)
+						    int mhi_irq, bool shared_msi)
 {
 	struct mhi_controller *mhi_cntrl;
 	int ret;
@@ -500,6 +532,10 @@ struct mhi_controller *qaic_mhi_register_controller(struct pci_dev *pci_dev, voi
 		return ERR_PTR(-ENOMEM);
 
 	mhi_cntrl->irq[0] = mhi_irq;
+
+	if (shared_msi) /* MSI shared with data path, no IRQF_NO_SUSPEND */
+		mhi_cntrl->irq_flags = IRQF_SHARED;
+
 	mhi_cntrl->fw_image = "qcom/aic100/sbl.bin";
 
 	/* use latest configured timeout */
diff --git a/drivers/accel/qaic/mhi_controller.h b/drivers/accel/qaic/mhi_controller.h
index 2ae45d768e24..500e7f4af2af 100644
--- a/drivers/accel/qaic/mhi_controller.h
+++ b/drivers/accel/qaic/mhi_controller.h
@@ -8,7 +8,7 @@
 #define MHICONTROLLERQAIC_H_
 
 struct mhi_controller *qaic_mhi_register_controller(struct pci_dev *pci_dev, void __iomem *mhi_bar,
-						    int mhi_irq);
+						    int mhi_irq, bool shared_msi);
 void qaic_mhi_free_controller(struct mhi_controller *mhi_cntrl, bool link_up);
 void qaic_mhi_start_reset(struct mhi_controller *mhi_cntrl);
 void qaic_mhi_reset_done(struct mhi_controller *mhi_cntrl);
diff --git a/drivers/accel/qaic/qaic.h b/drivers/accel/qaic/qaic.h
index e3f4c30f3ffd..bc40d52dc010 100644
--- a/drivers/accel/qaic/qaic.h
+++ b/drivers/accel/qaic/qaic.h
@@ -123,6 +123,8 @@ struct qaic_device {
 	struct srcu_struct	dev_lock;
 	/* true: Device under reset; false: Device not under reset */
 	bool			in_reset;
+	/* true: single MSI is used to operate device */
+	bool			single_msi;
 	/*
 	 * true: A tx MHI transaction has failed and a rx buffer is still queued
 	 * in control device. Such a buffer is considered lost rx buffer
@@ -137,6 +139,10 @@ struct qaic_device {
 	u32 (*gen_crc)(void *msg);
 	/* Validate the CRC of a control message */
 	bool (*valid_crc)(void *msg);
+	/* MHI "QAIC_TIMESYNC" channel device */
+	struct mhi_device	*qts_ch;
+	/* Work queue for tasks related to MHI "QAIC_TIMESYNC" channel */
+	struct workqueue_struct	*qts_wq;
 };
 
 struct qaic_drm_device {
diff --git a/drivers/accel/qaic/qaic_control.c b/drivers/accel/qaic/qaic_control.c
index 388abd40024b..84915824be54 100644
--- a/drivers/accel/qaic/qaic_control.c
+++ b/drivers/accel/qaic/qaic_control.c
@@ -1138,7 +1138,7 @@ static int abort_dma_cont(struct qaic_device *qdev, struct wrapper_list *wrapper
 		if (!list_is_first(&wrapper->list, &wrappers->list))
 			kref_put(&wrapper->ref_count, free_wrapper);
 
-	wrapper = add_wrapper(wrappers, offsetof(struct wrapper_msg, trans) + sizeof(*out_trans));
+	wrapper = add_wrapper(wrappers, sizeof(*wrapper));
 
 	if (!wrapper)
 		return -ENOMEM;
diff --git a/drivers/accel/qaic/qaic_data.c b/drivers/accel/qaic/qaic_data.c
index 4a8e43a7a6a4..8998c28e566e 100644
--- a/drivers/accel/qaic/qaic_data.c
+++ b/drivers/accel/qaic/qaic_data.c
@@ -51,6 +51,7 @@
 		})
 #define NUM_EVENTS	128
 #define NUM_DELAYS	10
+#define fifo_at(base, offset) ((base) + (offset) * get_dbc_req_elem_size())
 
 static unsigned int wait_exec_default_timeout_ms = 5000; /* 5 sec default */
 module_param(wait_exec_default_timeout_ms, uint, 0600);
@@ -451,7 +452,7 @@ static int create_sgt(struct qaic_device *qdev, struct sg_table **sgt_out, u64 s
 		 * later
 		 */
 		buf_extra = (PAGE_SIZE - size % PAGE_SIZE) % PAGE_SIZE;
-		max_order = min(MAX_ORDER - 1, get_order(size));
+		max_order = min(MAX_ORDER, get_order(size));
 	} else {
 		/* allocate a single page for book keeping */
 		nr_pages = 1;
@@ -1058,6 +1059,16 @@ int qaic_attach_slice_bo_ioctl(struct drm_device *dev, void *data, struct drm_fi
 	return ret;
 }
 
+static inline u32 fifo_space_avail(u32 head, u32 tail, u32 q_size)
+{
+	u32 avail = head - tail - 1;
+
+	if (head <= tail)
+		avail += q_size;
+
+	return avail;
+}
+
 static inline int copy_exec_reqs(struct qaic_device *qdev, struct bo_slice *slice, u32 dbc_id,
 				 u32 head, u32 *ptail)
 {
@@ -1066,27 +1077,20 @@ static inline int copy_exec_reqs(struct qaic_device *qdev, struct bo_slice *slic
 	u32 tail = *ptail;
 	u32 avail;
 
-	avail = head - tail;
-	if (head <= tail)
-		avail += dbc->nelem;
-
-	--avail;
-
+	avail = fifo_space_avail(head, tail, dbc->nelem);
 	if (avail < slice->nents)
 		return -EAGAIN;
 
 	if (tail + slice->nents > dbc->nelem) {
 		avail = dbc->nelem - tail;
 		avail = min_t(u32, avail, slice->nents);
-		memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs,
-		       sizeof(*reqs) * avail);
+		memcpy(fifo_at(dbc->req_q_base, tail), reqs, sizeof(*reqs) * avail);
 		reqs += avail;
 		avail = slice->nents - avail;
 		if (avail)
 			memcpy(dbc->req_q_base, reqs, sizeof(*reqs) * avail);
 	} else {
-		memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs,
-		       sizeof(*reqs) * slice->nents);
+		memcpy(fifo_at(dbc->req_q_base, tail), reqs, sizeof(*reqs) * slice->nents);
 	}
 
 	*ptail = (tail + slice->nents) % dbc->nelem;
@@ -1094,46 +1098,31 @@ static inline int copy_exec_reqs(struct qaic_device *qdev, struct bo_slice *slic
 	return 0;
 }
 
-/*
- * Based on the value of resize we may only need to transmit first_n
- * entries and the last entry, with last_bytes to send from the last entry.
- * Note that first_n could be 0.
- */
 static inline int copy_partial_exec_reqs(struct qaic_device *qdev, struct bo_slice *slice,
-					 u64 resize, u32 dbc_id, u32 head, u32 *ptail)
+					 u64 resize, struct dma_bridge_chan *dbc, u32 head,
+					 u32 *ptail)
 {
-	struct dma_bridge_chan *dbc = &qdev->dbc[dbc_id];
 	struct dbc_req *reqs = slice->reqs;
 	struct dbc_req *last_req;
 	u32 tail = *ptail;
-	u64 total_bytes;
 	u64 last_bytes;
 	u32 first_n;
 	u32 avail;
-	int ret;
-	int i;
-
-	avail = head - tail;
-	if (head <= tail)
-		avail += dbc->nelem;
 
-	--avail;
+	avail = fifo_space_avail(head, tail, dbc->nelem);
 
-	total_bytes = 0;
-	for (i = 0; i < slice->nents; i++) {
-		total_bytes += le32_to_cpu(reqs[i].len);
-		if (total_bytes >= resize)
+	/*
+	 * After this for loop is complete, first_n represents the index
+	 * of the last DMA request of this slice that needs to be
+	 * transferred after resizing and last_bytes represents DMA size
+	 * of that request.
+	 */
+	last_bytes = resize;
+	for (first_n = 0; first_n < slice->nents; first_n++)
+		if (last_bytes > le32_to_cpu(reqs[first_n].len))
+			last_bytes -= le32_to_cpu(reqs[first_n].len);
+		else
 			break;
-	}
-
-	if (total_bytes < resize) {
-		/* User space should have used the full buffer path. */
-		ret = -EINVAL;
-		return ret;
-	}
-
-	first_n = i;
-	last_bytes = i ? resize + le32_to_cpu(reqs[i].len) - total_bytes : resize;
 
 	if (avail < (first_n + 1))
 		return -EAGAIN;
@@ -1142,22 +1131,21 @@ static inline int copy_partial_exec_reqs(struct qaic_device *qdev, struct bo_sli
 		if (tail + first_n > dbc->nelem) {
 			avail = dbc->nelem - tail;
 			avail = min_t(u32, avail, first_n);
-			memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs,
-			       sizeof(*reqs) * avail);
+			memcpy(fifo_at(dbc->req_q_base, tail), reqs, sizeof(*reqs) * avail);
 			last_req = reqs + avail;
 			avail = first_n - avail;
 			if (avail)
 				memcpy(dbc->req_q_base, last_req, sizeof(*reqs) * avail);
 		} else {
-			memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs,
-			       sizeof(*reqs) * first_n);
+			memcpy(fifo_at(dbc->req_q_base, tail), reqs, sizeof(*reqs) * first_n);
 		}
 	}
 
-	/* Copy over the last entry. Here we need to adjust len to the left over
+	/*
+	 * Copy over the last entry. Here we need to adjust len to the left over
 	 * size, and set src and dst to the entry it is copied to.
 	 */
-	last_req = dbc->req_q_base + (tail + first_n) % dbc->nelem * get_dbc_req_elem_size();
+	last_req = fifo_at(dbc->req_q_base, (tail + first_n) % dbc->nelem);
 	memcpy(last_req, reqs + slice->nents - 1, sizeof(*reqs));
 
 	/*
@@ -1168,6 +1156,9 @@ static inline int copy_partial_exec_reqs(struct qaic_device *qdev, struct bo_sli
 	last_req->len = cpu_to_le32((u32)last_bytes);
 	last_req->src_addr = reqs[first_n].src_addr;
 	last_req->dest_addr = reqs[first_n].dest_addr;
+	if (!last_bytes)
+		/* Disable DMA transfer */
+		last_req->cmd = GENMASK(7, 2) & reqs[first_n].cmd;
 
 	*ptail = (tail + first_n + 1) % dbc->nelem;
 
@@ -1227,26 +1218,17 @@ static int send_bo_list_to_device(struct qaic_device *qdev, struct drm_file *fil
 		bo->req_id = dbc->next_req_id++;
 
 		list_for_each_entry(slice, &bo->slices, slice) {
-			/*
-			 * If this slice does not fall under the given
-			 * resize then skip this slice and continue the loop
-			 */
-			if (is_partial && pexec[i].resize && pexec[i].resize <= slice->offset)
-				continue;
-
 			for (j = 0; j < slice->nents; j++)
 				slice->reqs[j].req_id = cpu_to_le16(bo->req_id);
 
-			/*
-			 * If it is a partial execute ioctl call then check if
-			 * resize has cut this slice short then do a partial copy
-			 * else do complete copy
-			 */
-			if (is_partial && pexec[i].resize &&
-			    pexec[i].resize < slice->offset + slice->size)
+			if (is_partial && (!pexec[i].resize || pexec[i].resize <= slice->offset))
+				/* Configure the slice for no DMA transfer */
+				ret = copy_partial_exec_reqs(qdev, slice, 0, dbc, head, tail);
+			else if (is_partial && pexec[i].resize < slice->offset + slice->size)
+				/* Configure the slice to be partially DMA transferred */
 				ret = copy_partial_exec_reqs(qdev, slice,
-							     pexec[i].resize - slice->offset,
-							     dbc->id, head, tail);
+							     pexec[i].resize - slice->offset, dbc,
+							     head, tail);
 			else
 				ret = copy_exec_reqs(qdev, slice, dbc->id, head, tail);
 			if (ret) {
@@ -1466,6 +1448,16 @@ irqreturn_t dbc_irq_handler(int irq, void *data)
 
 	rcu_id = srcu_read_lock(&dbc->ch_lock);
 
+	if (datapath_polling) {
+		srcu_read_unlock(&dbc->ch_lock, rcu_id);
+		/*
+		 * Normally datapath_polling will not have irqs enabled, but
+		 * when running with only one MSI the interrupt is shared with
+		 * MHI so it cannot be disabled. Return ASAP instead.
+		 */
+		return IRQ_HANDLED;
+	}
+
 	if (!dbc->usr) {
 		srcu_read_unlock(&dbc->ch_lock, rcu_id);
 		return IRQ_HANDLED;
@@ -1488,7 +1480,8 @@ irqreturn_t dbc_irq_handler(int irq, void *data)
 		return IRQ_NONE;
 	}
 
-	disable_irq_nosync(irq);
+	if (!dbc->qdev->single_msi)
+		disable_irq_nosync(irq);
 	srcu_read_unlock(&dbc->ch_lock, rcu_id);
 	return IRQ_WAKE_THREAD;
 }
@@ -1559,12 +1552,12 @@ irqreturn_t dbc_irq_threaded_fn(int irq, void *data)
 	u32 tail;
 
 	rcu_id = srcu_read_lock(&dbc->ch_lock);
+	qdev = dbc->qdev;
 
 	head = readl(dbc->dbc_base + RSPHP_OFF);
 	if (head == U32_MAX) /* PCI link error */
 		goto error_out;
 
-	qdev = dbc->qdev;
 read_fifo:
 
 	if (!event_count) {
@@ -1645,14 +1638,14 @@ irqreturn_t dbc_irq_threaded_fn(int irq, void *data)
 	goto read_fifo;
 
 normal_out:
-	if (likely(!datapath_polling))
+	if (!qdev->single_msi && likely(!datapath_polling))
 		enable_irq(irq);
-	else
+	else if (unlikely(datapath_polling))
 		schedule_work(&dbc->poll_work);
 	/* checking the fifo and enabling irqs is a race, missed event check */
 	tail = readl(dbc->dbc_base + RSPTP_OFF);
 	if (tail != U32_MAX && head != tail) {
-		if (likely(!datapath_polling))
+		if (!qdev->single_msi && likely(!datapath_polling))
 			disable_irq_nosync(irq);
 		goto read_fifo;
 	}
@@ -1661,9 +1654,9 @@ irqreturn_t dbc_irq_threaded_fn(int irq, void *data)
 
 error_out:
 	srcu_read_unlock(&dbc->ch_lock, rcu_id);
-	if (likely(!datapath_polling))
+	if (!qdev->single_msi && likely(!datapath_polling))
 		enable_irq(irq);
-	else
+	else if (unlikely(datapath_polling))
 		schedule_work(&dbc->poll_work);
 
 	return IRQ_HANDLED;
diff --git a/drivers/accel/qaic/qaic_drv.c b/drivers/accel/qaic/qaic_drv.c
index 6f58095767df..b12226385003 100644
--- a/drivers/accel/qaic/qaic_drv.c
+++ b/drivers/accel/qaic/qaic_drv.c
@@ -27,6 +27,7 @@
 
 #include "mhi_controller.h"
 #include "qaic.h"
+#include "qaic_timesync.h"
 
 MODULE_IMPORT_NS(DMA_BUF);
 
@@ -324,6 +325,7 @@ static void cleanup_qdev(struct qaic_device *qdev)
 	cleanup_srcu_struct(&qdev->dev_lock);
 	pci_set_drvdata(qdev->pdev, NULL);
 	destroy_workqueue(qdev->cntl_wq);
+	destroy_workqueue(qdev->qts_wq);
 }
 
 static struct qaic_device *create_qdev(struct pci_dev *pdev, const struct pci_device_id *id)
@@ -347,6 +349,12 @@ static struct qaic_device *create_qdev(struct pci_dev *pdev, const struct pci_de
 	if (!qdev->cntl_wq)
 		return NULL;
 
+	qdev->qts_wq = alloc_workqueue("qaic_ts", WQ_UNBOUND, 0);
+	if (!qdev->qts_wq) {
+		destroy_workqueue(qdev->cntl_wq);
+		return NULL;
+	}
+
 	pci_set_drvdata(pdev, qdev);
 	qdev->pdev = pdev;
 
@@ -424,14 +432,24 @@ static int init_msi(struct qaic_device *qdev, struct pci_dev *pdev)
 	int i;
 
 	/* Managed release since we use pcim_enable_device */
-	ret = pci_alloc_irq_vectors(pdev, 1, 32, PCI_IRQ_MSI);
-	if (ret < 0)
-		return ret;
+	ret = pci_alloc_irq_vectors(pdev, 32, 32, PCI_IRQ_MSI);
+	if (ret == -ENOSPC) {
+		ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
+		if (ret < 0)
+			return ret;
 
-	if (ret < 32) {
-		pci_err(pdev, "%s: Requested 32 MSIs. Obtained %d MSIs which is less than the 32 required.\n",
-			__func__, ret);
-		return -ENODEV;
+		/*
+		 * Operate in one MSI mode. All interrupts will be directed to
+		 * MSI0; every interrupt will wake up all the interrupt handlers
+		 * (MHI and DBC[0-15]). Since the interrupt is now shared, it is
+		 * not disabled during DBC threaded handler, but only one thread
+		 * will be allowed to run per DBC, so while it can be
+		 * interrupted, it shouldn't race with itself.
+		 */
+		qdev->single_msi = true;
+		pci_info(pdev, "Allocating 32 MSIs failed, operating in 1 MSI mode. Performance may be impacted.\n");
+	} else if (ret < 0) {
+		return ret;
 	}
 
 	mhi_irq = pci_irq_vector(pdev, 0);
@@ -439,15 +457,17 @@ static int init_msi(struct qaic_device *qdev, struct pci_dev *pdev)
 		return mhi_irq;
 
 	for (i = 0; i < qdev->num_dbc; ++i) {
-		ret = devm_request_threaded_irq(&pdev->dev, pci_irq_vector(pdev, i + 1),
+		ret = devm_request_threaded_irq(&pdev->dev,
+						pci_irq_vector(pdev, qdev->single_msi ? 0 : i + 1),
 						dbc_irq_handler, dbc_irq_threaded_fn, IRQF_SHARED,
 						"qaic_dbc", &qdev->dbc[i]);
 		if (ret)
 			return ret;
 
 		if (datapath_polling) {
-			qdev->dbc[i].irq = pci_irq_vector(pdev, i + 1);
-			disable_irq_nosync(qdev->dbc[i].irq);
+			qdev->dbc[i].irq = pci_irq_vector(pdev, qdev->single_msi ? 0 : i + 1);
+			if (!qdev->single_msi)
+				disable_irq_nosync(qdev->dbc[i].irq);
 			INIT_WORK(&qdev->dbc[i].poll_work, irq_polling_work);
 		}
 	}
@@ -479,7 +499,8 @@ static int qaic_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		goto cleanup_qdev;
 	}
 
-	qdev->mhi_cntrl = qaic_mhi_register_controller(pdev, qdev->bar_0, mhi_irq);
+	qdev->mhi_cntrl = qaic_mhi_register_controller(pdev, qdev->bar_0, mhi_irq,
+						       qdev->single_msi);
 	if (IS_ERR(qdev->mhi_cntrl)) {
 		ret = PTR_ERR(qdev->mhi_cntrl);
 		goto cleanup_qdev;
@@ -586,6 +607,10 @@ static int __init qaic_init(void)
 		goto free_pci;
 	}
 
+	ret = qaic_timesync_init();
+	if (ret)
+		pr_debug("qaic: qaic_timesync_init failed %d\n", ret);
+
 	return 0;
 
 free_pci:
@@ -611,6 +636,7 @@ static void __exit qaic_exit(void)
 	 * reinitializing the link_up state after the cleanup is done.
 	 */
 	link_up = true;
+	qaic_timesync_deinit();
 	mhi_driver_unregister(&qaic_mhi_driver);
 	pci_unregister_driver(&qaic_pci_driver);
 }
diff --git a/drivers/accel/qaic/qaic_timesync.c b/drivers/accel/qaic/qaic_timesync.c
new file mode 100644
index 000000000000..301f4462d51b
--- /dev/null
+++ b/drivers/accel/qaic/qaic_timesync.c
@@ -0,0 +1,395 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved. */
+
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/math64.h>
+#include <linux/mhi.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/time64.h>
+#include <linux/timer.h>
+
+#include "qaic.h"
+#include "qaic_timesync.h"
+
+#define QTIMER_REG_OFFSET			0xa28
+#define QAIC_TIMESYNC_SIGNATURE			0x55aa
+#define QAIC_CONV_QTIMER_TO_US(qtimer)		(mul_u64_u32_div(qtimer, 10, 192))
+
+static unsigned int timesync_delay_ms = 1000; /* 1 sec default */
+module_param(timesync_delay_ms, uint, 0600);
+MODULE_PARM_DESC(timesync_delay_ms, "Delay in ms between two consecutive timesync operations");
+
+enum qts_msg_type {
+	QAIC_TS_CMD_TO_HOST,
+	QAIC_TS_SYNC_REQ,
+	QAIC_TS_ACK_TO_HOST,
+	QAIC_TS_MSG_TYPE_MAX
+};
+
+/**
+ * struct qts_hdr - Timesync message header structure.
+ * @signature: Unique signature to identify the timesync message.
+ * @reserved_1: Reserved for future use.
+ * @reserved_2: Reserved for future use.
+ * @msg_type: sub-type of the timesync message.
+ * @reserved_3: Reserved for future use.
+ */
+struct qts_hdr {
+	__le16	signature;
+	__le16	reserved_1;
+	u8	reserved_2;
+	u8	msg_type;
+	__le16	reserved_3;
+} __packed;
+
+/**
+ * struct qts_timeval - Structure to carry time information.
+ * @tv_sec: Seconds part of the time.
+ * @tv_usec: uS (microseconds) part of the time.
+ */
+struct qts_timeval {
+	__le64	tv_sec;
+	__le64	tv_usec;
+} __packed;
+
+/**
+ * struct qts_host_time_sync_msg_data - Structure to denote the timesync message.
+ * @header: Header of the timesync message.
+ * @data: Time information.
+ */
+struct qts_host_time_sync_msg_data {
+	struct	qts_hdr header;
+	struct	qts_timeval data;
+} __packed;
+
+/**
+ * struct mqts_dev - MHI QAIC Timesync Control device.
+ * @qdev: Pointer to the root device struct driven by QAIC driver.
+ * @mhi_dev: Pointer to associated MHI device.
+ * @timer: Timer handle used for timesync.
+ * @qtimer_addr: Device QTimer register pointer.
+ * @buff_in_use: atomic variable to track if the sync_msg buffer is in use.
+ * @dev: Device pointer to qdev->pdev->dev stored for easy access.
+ * @sync_msg: Buffer used to send timesync message over MHI.
+ */
+struct mqts_dev {
+	struct qaic_device *qdev;
+	struct mhi_device *mhi_dev;
+	struct timer_list timer;
+	void __iomem *qtimer_addr;
+	atomic_t buff_in_use;
+	struct device *dev;
+	struct qts_host_time_sync_msg_data *sync_msg;
+};
+
+struct qts_resp_msg {
+	struct qts_hdr	hdr;
+} __packed;
+
+struct qts_resp {
+	struct qts_resp_msg	data;
+	struct work_struct	work;
+	struct qaic_device	*qdev;
+};
+
+#ifdef readq
+static u64 read_qtimer(const volatile void __iomem *addr)
+{
+	return readq(addr);
+}
+#else
+static u64 read_qtimer(const volatile void __iomem *addr)
+{
+	u64 low, high;
+
+	low = readl(addr);
+	high = readl(addr + sizeof(u32));
+	return low | (high << 32);
+}
+#endif
+
+static void qaic_timesync_ul_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)
+{
+	struct mqts_dev *mqtsdev = dev_get_drvdata(&mhi_dev->dev);
+
+	dev_dbg(mqtsdev->dev, "%s status: %d xfer_len: %zu\n", __func__,
+		mhi_result->transaction_status, mhi_result->bytes_xferd);
+
+	atomic_set(&mqtsdev->buff_in_use, 0);
+}
+
+static void qaic_timesync_dl_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)
+{
+	struct mqts_dev *mqtsdev = dev_get_drvdata(&mhi_dev->dev);
+
+	dev_err(mqtsdev->dev, "%s no data expected on dl channel\n", __func__);
+}
+
+static void qaic_timesync_timer(struct timer_list *t)
+{
+	struct mqts_dev *mqtsdev = from_timer(mqtsdev, t, timer);
+	struct qts_host_time_sync_msg_data *sync_msg;
+	u64 device_qtimer_us;
+	u64 device_qtimer;
+	u64 host_time_us;
+	u64 offset_us;
+	u64 host_sec;
+	int ret;
+
+	if (atomic_read(&mqtsdev->buff_in_use)) {
+		dev_dbg(mqtsdev->dev, "%s buffer not free, schedule next cycle\n", __func__);
+		goto mod_timer;
+	}
+	atomic_set(&mqtsdev->buff_in_use, 1);
+
+	sync_msg = mqtsdev->sync_msg;
+	sync_msg->header.signature = cpu_to_le16(QAIC_TIMESYNC_SIGNATURE);
+	sync_msg->header.msg_type = QAIC_TS_SYNC_REQ;
+	/* Read host UTC time and convert to uS*/
+	host_time_us = div_u64(ktime_get_real_ns(), NSEC_PER_USEC);
+	device_qtimer = read_qtimer(mqtsdev->qtimer_addr);
+	device_qtimer_us = QAIC_CONV_QTIMER_TO_US(device_qtimer);
+	/* Offset between host UTC and device time */
+	offset_us = host_time_us - device_qtimer_us;
+
+	host_sec = div_u64(offset_us, USEC_PER_SEC);
+	sync_msg->data.tv_usec = cpu_to_le64(offset_us - host_sec * USEC_PER_SEC);
+	sync_msg->data.tv_sec = cpu_to_le64(host_sec);
+	ret = mhi_queue_buf(mqtsdev->mhi_dev, DMA_TO_DEVICE, sync_msg, sizeof(*sync_msg), MHI_EOT);
+	if (ret && (ret != -EAGAIN)) {
+		dev_err(mqtsdev->dev, "%s unable to queue to mhi:%d\n", __func__, ret);
+		return;
+	} else if (ret == -EAGAIN) {
+		atomic_set(&mqtsdev->buff_in_use, 0);
+	}
+
+mod_timer:
+	ret = mod_timer(t, jiffies + msecs_to_jiffies(timesync_delay_ms));
+	if (ret)
+		dev_err(mqtsdev->dev, "%s mod_timer error:%d\n", __func__, ret);
+}
+
+static int qaic_timesync_probe(struct mhi_device *mhi_dev, const struct mhi_device_id *id)
+{
+	struct qaic_device *qdev = pci_get_drvdata(to_pci_dev(mhi_dev->mhi_cntrl->cntrl_dev));
+	struct mqts_dev *mqtsdev;
+	struct timer_list *timer;
+	int ret;
+
+	mqtsdev = kzalloc(sizeof(*mqtsdev), GFP_KERNEL);
+	if (!mqtsdev) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	timer = &mqtsdev->timer;
+	mqtsdev->mhi_dev = mhi_dev;
+	mqtsdev->qdev = qdev;
+	mqtsdev->dev = &qdev->pdev->dev;
+
+	mqtsdev->sync_msg = kzalloc(sizeof(*mqtsdev->sync_msg), GFP_KERNEL);
+	if (!mqtsdev->sync_msg) {
+		ret = -ENOMEM;
+		goto free_mqts_dev;
+	}
+	atomic_set(&mqtsdev->buff_in_use, 0);
+
+	ret = mhi_prepare_for_transfer(mhi_dev);
+	if (ret)
+		goto free_sync_msg;
+
+	/* Qtimer register pointer */
+	mqtsdev->qtimer_addr = qdev->bar_0 + QTIMER_REG_OFFSET;
+	timer_setup(timer, qaic_timesync_timer, 0);
+	timer->expires = jiffies + msecs_to_jiffies(timesync_delay_ms);
+	add_timer(timer);
+	dev_set_drvdata(&mhi_dev->dev, mqtsdev);
+
+	return 0;
+
+free_sync_msg:
+	kfree(mqtsdev->sync_msg);
+free_mqts_dev:
+	kfree(mqtsdev);
+out:
+	return ret;
+};
+
+static void qaic_timesync_remove(struct mhi_device *mhi_dev)
+{
+	struct mqts_dev *mqtsdev = dev_get_drvdata(&mhi_dev->dev);
+
+	del_timer_sync(&mqtsdev->timer);
+	mhi_unprepare_from_transfer(mqtsdev->mhi_dev);
+	kfree(mqtsdev->sync_msg);
+	kfree(mqtsdev);
+}
+
+static const struct mhi_device_id qaic_timesync_match_table[] = {
+	{ .chan = "QAIC_TIMESYNC_PERIODIC"},
+	{},
+};
+
+MODULE_DEVICE_TABLE(mhi, qaic_timesync_match_table);
+
+static struct mhi_driver qaic_timesync_driver = {
+	.id_table = qaic_timesync_match_table,
+	.remove = qaic_timesync_remove,
+	.probe = qaic_timesync_probe,
+	.ul_xfer_cb = qaic_timesync_ul_xfer_cb,
+	.dl_xfer_cb = qaic_timesync_dl_xfer_cb,
+	.driver = {
+		.name = "qaic_timesync_periodic",
+	},
+};
+
+static void qaic_boot_timesync_worker(struct work_struct *work)
+{
+	struct qts_resp *resp = container_of(work, struct qts_resp, work);
+	struct qts_host_time_sync_msg_data *req;
+	struct qts_resp_msg data = resp->data;
+	struct qaic_device *qdev = resp->qdev;
+	struct mhi_device *mhi_dev;
+	struct timespec64 ts;
+	int ret;
+
+	mhi_dev = qdev->qts_ch;
+	/* Queue the response message beforehand to avoid race conditions */
+	ret = mhi_queue_buf(mhi_dev, DMA_FROM_DEVICE, &resp->data, sizeof(resp->data), MHI_EOT);
+	if (ret) {
+		kfree(resp);
+		dev_warn(&mhi_dev->dev, "Failed to re-queue response buffer %d\n", ret);
+		return;
+	}
+
+	switch (data.hdr.msg_type) {
+	case QAIC_TS_CMD_TO_HOST:
+		req = kzalloc(sizeof(*req), GFP_KERNEL);
+		if (!req)
+			break;
+
+		req->header = data.hdr;
+		req->header.msg_type = QAIC_TS_SYNC_REQ;
+		ktime_get_real_ts64(&ts);
+		req->data.tv_sec = cpu_to_le64(ts.tv_sec);
+		req->data.tv_usec = cpu_to_le64(div_u64(ts.tv_nsec, NSEC_PER_USEC));
+
+		ret = mhi_queue_buf(mhi_dev, DMA_TO_DEVICE, req, sizeof(*req), MHI_EOT);
+		if (ret) {
+			kfree(req);
+			dev_dbg(&mhi_dev->dev, "Failed to send request message. Error %d\n", ret);
+		}
+		break;
+	case QAIC_TS_ACK_TO_HOST:
+		dev_dbg(&mhi_dev->dev, "ACK received from device\n");
+		break;
+	default:
+		dev_err(&mhi_dev->dev, "Invalid message type %u.\n", data.hdr.msg_type);
+	}
+}
+
+static int qaic_boot_timesync_queue_resp(struct mhi_device *mhi_dev, struct qaic_device *qdev)
+{
+	struct qts_resp *resp;
+	int ret;
+
+	resp = kzalloc(sizeof(*resp), GFP_KERNEL);
+	if (!resp)
+		return -ENOMEM;
+
+	resp->qdev = qdev;
+	INIT_WORK(&resp->work, qaic_boot_timesync_worker);
+
+	ret = mhi_queue_buf(mhi_dev, DMA_FROM_DEVICE, &resp->data, sizeof(resp->data), MHI_EOT);
+	if (ret) {
+		kfree(resp);
+		dev_warn(&mhi_dev->dev, "Failed to queue response buffer %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void qaic_boot_timesync_remove(struct mhi_device *mhi_dev)
+{
+	struct qaic_device *qdev;
+
+	qdev = dev_get_drvdata(&mhi_dev->dev);
+	mhi_unprepare_from_transfer(qdev->qts_ch);
+	qdev->qts_ch = NULL;
+}
+
+static int qaic_boot_timesync_probe(struct mhi_device *mhi_dev, const struct mhi_device_id *id)
+{
+	struct qaic_device *qdev = pci_get_drvdata(to_pci_dev(mhi_dev->mhi_cntrl->cntrl_dev));
+	int ret;
+
+	ret = mhi_prepare_for_transfer(mhi_dev);
+	if (ret)
+		return ret;
+
+	qdev->qts_ch = mhi_dev;
+	dev_set_drvdata(&mhi_dev->dev, qdev);
+
+	ret = qaic_boot_timesync_queue_resp(mhi_dev, qdev);
+	if (ret) {
+		dev_set_drvdata(&mhi_dev->dev, NULL);
+		qdev->qts_ch = NULL;
+		mhi_unprepare_from_transfer(mhi_dev);
+	}
+
+	return ret;
+}
+
+static void qaic_boot_timesync_ul_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)
+{
+	kfree(mhi_result->buf_addr);
+}
+
+static void qaic_boot_timesync_dl_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)
+{
+	struct qts_resp *resp = container_of(mhi_result->buf_addr, struct qts_resp, data);
+
+	if (mhi_result->transaction_status || mhi_result->bytes_xferd != sizeof(resp->data)) {
+		kfree(resp);
+		return;
+	}
+
+	queue_work(resp->qdev->qts_wq, &resp->work);
+}
+
+static const struct mhi_device_id qaic_boot_timesync_match_table[] = {
+	{ .chan = "QAIC_TIMESYNC"},
+	{},
+};
+
+static struct mhi_driver qaic_boot_timesync_driver = {
+	.id_table = qaic_boot_timesync_match_table,
+	.remove = qaic_boot_timesync_remove,
+	.probe = qaic_boot_timesync_probe,
+	.ul_xfer_cb = qaic_boot_timesync_ul_xfer_cb,
+	.dl_xfer_cb = qaic_boot_timesync_dl_xfer_cb,
+	.driver = {
+		.name = "qaic_timesync",
+	},
+};
+
+int qaic_timesync_init(void)
+{
+	int ret;
+
+	ret = mhi_driver_register(&qaic_timesync_driver);
+	if (ret)
+		return ret;
+	ret = mhi_driver_register(&qaic_boot_timesync_driver);
+
+	return ret;
+}
+
+void qaic_timesync_deinit(void)
+{
+	mhi_driver_unregister(&qaic_boot_timesync_driver);
+	mhi_driver_unregister(&qaic_timesync_driver);
+}
diff --git a/drivers/accel/qaic/qaic_timesync.h b/drivers/accel/qaic/qaic_timesync.h
new file mode 100644
index 000000000000..851b7acd43bb
--- /dev/null
+++ b/drivers/accel/qaic/qaic_timesync.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: GPL-2.0-only
+ *
+ * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+
+#ifndef __QAIC_TIMESYNC_H__
+#define __QAIC_TIMESYNC_H__
+
+int qaic_timesync_init(void);
+void qaic_timesync_deinit(void);
+#endif /* __QAIC_TIMESYNC_H__ */
diff --git a/drivers/acpi/Makefile b/drivers/acpi/Makefile
index eaa09bf52f17..d367e649714f 100644
--- a/drivers/acpi/Makefile
+++ b/drivers/acpi/Makefile
@@ -37,7 +37,7 @@ acpi-$(CONFIG_ACPI_SLEEP)	+= proc.o
 # ACPI Bus and Device Drivers
 #
 acpi-y				+= bus.o glue.o
-acpi-y				+= scan.o
+acpi-y				+= scan.o mipi-disco-img.o
 acpi-y				+= resource.o
 acpi-y				+= acpi_processor.o
 acpi-y				+= processor_core.o
diff --git a/drivers/acpi/acpi_video.c b/drivers/acpi/acpi_video.c
index 0b7a01f38b65..5eded14f8853 100644
--- a/drivers/acpi/acpi_video.c
+++ b/drivers/acpi/acpi_video.c
@@ -67,7 +67,7 @@ MODULE_PARM_DESC(hw_changes_brightness,
 static bool device_id_scheme = false;
 module_param(device_id_scheme, bool, 0444);
 
-static int only_lcd = -1;
+static int only_lcd;
 module_param(only_lcd, int, 0444);
 
 static bool may_report_brightness_keys;
@@ -2031,7 +2031,7 @@ static int acpi_video_bus_add(struct acpi_device *device)
 	 * HP ZBook Fury 16 G10 requires ACPI video's child devices have _PS0
 	 * evaluated to have functional panel brightness control.
 	 */
-	acpi_device_fix_up_power_extended(device);
+	acpi_device_fix_up_power_children(device);
 
 	pr_info("%s [%s] (multi-head: %s  rom: %s  post: %s)\n",
 	       ACPI_VIDEO_DEVICE_NAME, acpi_device_bid(device),
@@ -2141,57 +2141,6 @@ static int __init intel_opregion_present(void)
 	return opregion;
 }
 
-/* Check if the chassis-type indicates there is no builtin LCD panel */
-static bool dmi_is_desktop(void)
-{
-	const char *chassis_type;
-	unsigned long type;
-
-	chassis_type = dmi_get_system_info(DMI_CHASSIS_TYPE);
-	if (!chassis_type)
-		return false;
-
-	if (kstrtoul(chassis_type, 10, &type) != 0)
-		return false;
-
-	switch (type) {
-	case 0x03: /* Desktop */
-	case 0x04: /* Low Profile Desktop */
-	case 0x05: /* Pizza Box */
-	case 0x06: /* Mini Tower */
-	case 0x07: /* Tower */
-	case 0x10: /* Lunch Box */
-	case 0x11: /* Main Server Chassis */
-		return true;
-	}
-
-	return false;
-}
-
-/*
- * We're seeing a lot of bogus backlight interfaces on newer machines
- * without a LCD such as desktops, servers and HDMI sticks. Checking the
- * lcd flag fixes this, enable this by default on any machines which are:
- * 1.  Win8 ready (where we also prefer the native backlight driver, so
- *     normally the acpi_video code should not register there anyways); *and*
- * 2.1 Report a desktop/server DMI chassis-type, or
- * 2.2 Are an ACPI-reduced-hardware platform (and thus won't use the EC for
-       backlight control)
- */
-static bool should_check_lcd_flag(void)
-{
-	if (!acpi_osi_is_win8())
-		return false;
-
-	if (dmi_is_desktop())
-		return true;
-
-	if (acpi_reduced_hardware())
-		return true;
-
-	return false;
-}
-
 int acpi_video_register(void)
 {
 	int ret = 0;
@@ -2205,9 +2154,6 @@ int acpi_video_register(void)
 		goto leave;
 	}
 
-	if (only_lcd == -1)
-		only_lcd = should_check_lcd_flag();
-
 	dmi_check_system(video_dmi_table);
 
 	ret = acpi_bus_register_driver(&acpi_video_bus);
diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c
index 72e64c0718c9..569bd15f211b 100644
--- a/drivers/acpi/bus.c
+++ b/drivers/acpi/bus.c
@@ -408,7 +408,7 @@ static void acpi_bus_decode_usb_osc(const char *msg, u32 bits)
 static u8 sb_usb_uuid_str[] = "23A0D13A-26AB-486C-9C5F-0FFA525A575A";
 static void acpi_bus_osc_negotiate_usb_control(void)
 {
-	u32 capbuf[3];
+	u32 capbuf[3], *capbuf_ret;
 	struct acpi_osc_context context = {
 		.uuid_str = sb_usb_uuid_str,
 		.rev = 1,
@@ -428,7 +428,12 @@ static void acpi_bus_osc_negotiate_usb_control(void)
 	control = OSC_USB_USB3_TUNNELING | OSC_USB_DP_TUNNELING |
 		  OSC_USB_PCIE_TUNNELING | OSC_USB_XDOMAIN;
 
-	capbuf[OSC_QUERY_DWORD] = 0;
+	/*
+	 * Run _OSC first with query bit set, trying to get control over
+	 * all tunneling. The platform can then clear out bits in the
+	 * control dword that it does not want to grant to the OS.
+	 */
+	capbuf[OSC_QUERY_DWORD] = OSC_QUERY_ENABLE;
 	capbuf[OSC_SUPPORT_DWORD] = 0;
 	capbuf[OSC_CONTROL_DWORD] = control;
 
@@ -441,8 +446,29 @@ static void acpi_bus_osc_negotiate_usb_control(void)
 		goto out_free;
 	}
 
+	/*
+	 * Run _OSC again now with query bit clear and the control dword
+	 * matching what the platform granted (which may not have all
+	 * the control bits set).
+	 */
+	capbuf_ret = context.ret.pointer;
+
+	capbuf[OSC_QUERY_DWORD] = 0;
+	capbuf[OSC_CONTROL_DWORD] = capbuf_ret[OSC_CONTROL_DWORD];
+
+	kfree(context.ret.pointer);
+
+	status = acpi_run_osc(handle, &context);
+	if (ACPI_FAILURE(status))
+		return;
+
+	if (context.ret.length != sizeof(capbuf)) {
+		pr_info("USB4 _OSC: returned invalid length buffer\n");
+		goto out_free;
+	}
+
 	osc_sb_native_usb4_control =
-		control &  acpi_osc_ctx_get_pci_control(&context);
+		control & acpi_osc_ctx_get_pci_control(&context);
 
 	acpi_bus_decode_usb_osc("USB4 _OSC: OS supports", control);
 	acpi_bus_decode_usb_osc("USB4 _OSC: OS controls",
diff --git a/drivers/acpi/device_pm.c b/drivers/acpi/device_pm.c
index f007116a8427..3b4d048c4941 100644
--- a/drivers/acpi/device_pm.c
+++ b/drivers/acpi/device_pm.c
@@ -397,6 +397,19 @@ void acpi_device_fix_up_power_extended(struct acpi_device *adev)
 }
 EXPORT_SYMBOL_GPL(acpi_device_fix_up_power_extended);
 
+/**
+ * acpi_device_fix_up_power_children - Force a device's children into D0.
+ * @adev: Parent device object whose children's power state is to be fixed up.
+ *
+ * Call acpi_device_fix_up_power() for @adev's children so long as they
+ * are reported as present and enabled.
+ */
+void acpi_device_fix_up_power_children(struct acpi_device *adev)
+{
+	acpi_dev_for_each_child(adev, fix_up_power_if_applicable, NULL);
+}
+EXPORT_SYMBOL_GPL(acpi_device_fix_up_power_children);
+
 int acpi_device_update_power(struct acpi_device *device, int *state_p)
 {
 	int state;
diff --git a/drivers/acpi/internal.h b/drivers/acpi/internal.h
index 866c7c4ed233..86b670637f2f 100644
--- a/drivers/acpi/internal.h
+++ b/drivers/acpi/internal.h
@@ -276,4 +276,13 @@ void acpi_init_lpit(void);
 static inline void acpi_init_lpit(void) { }
 #endif
 
+/*--------------------------------------------------------------------------
+		ACPI _CRS CSI-2 and MIPI DisCo for Imaging
+  -------------------------------------------------------------------------- */
+
+void acpi_mipi_check_crs_csi2(acpi_handle handle);
+void acpi_mipi_scan_crs_csi2(void);
+void acpi_mipi_init_crs_csi2_swnodes(void);
+void acpi_mipi_crs_csi2_cleanup(void);
+
 #endif /* _ACPI_INTERNAL_H_ */
diff --git a/drivers/acpi/mipi-disco-img.c b/drivers/acpi/mipi-disco-img.c
new file mode 100644
index 000000000000..7286cf4579bc
--- /dev/null
+++ b/drivers/acpi/mipi-disco-img.c
@@ -0,0 +1,725 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * MIPI DisCo for Imaging support.
+ *
+ * Copyright (C) 2023 Intel Corporation
+ *
+ * Support MIPI DisCo for Imaging by parsing ACPI _CRS CSI-2 records defined in
+ * Section 6.4.3.8.2.4 "Camera Serial Interface (CSI-2) Connection Resource
+ * Descriptor" of ACPI 6.5 and using device properties defined by the MIPI DisCo
+ * for Imaging specification.
+ *
+ * The implementation looks for the information in the ACPI namespace (CSI-2
+ * resource descriptors in _CRS) and constructs software nodes compatible with
+ * Documentation/firmware-guide/acpi/dsd/graph.rst to represent the CSI-2
+ * connection graph.  The software nodes are then populated with the data
+ * extracted from the _CRS CSI-2 resource descriptors and the MIPI DisCo
+ * for Imaging device properties present in _DSD for the ACPI device objects
+ * with CSI-2 connections.
+ */
+
+#include <linux/acpi.h>
+#include <linux/limits.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/overflow.h>
+#include <linux/types.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+#include <media/v4l2-fwnode.h>
+
+#include "internal.h"
+
+static LIST_HEAD(acpi_mipi_crs_csi2_list);
+
+static void acpi_mipi_data_tag(acpi_handle handle, void *context)
+{
+}
+
+/* Connection data extracted from one _CRS CSI-2 resource descriptor. */
+struct crs_csi2_connection {
+	struct list_head entry;
+	struct acpi_resource_csi2_serialbus csi2_data;
+	acpi_handle remote_handle;
+	char remote_name[];
+};
+
+/* Data extracted from _CRS CSI-2 resource descriptors for one device. */
+struct crs_csi2 {
+	struct list_head entry;
+	acpi_handle handle;
+	struct acpi_device_software_nodes *swnodes;
+	struct list_head connections;
+	u32 port_count;
+};
+
+struct csi2_resources_walk_data {
+	acpi_handle handle;
+	struct list_head connections;
+};
+
+static acpi_status parse_csi2_resource(struct acpi_resource *res, void *context)
+{
+	struct csi2_resources_walk_data *crwd = context;
+	struct acpi_resource_csi2_serialbus *csi2_res;
+	struct acpi_resource_source *csi2_res_src;
+	u16 csi2_res_src_length;
+	struct crs_csi2_connection *conn;
+	acpi_handle remote_handle;
+
+	if (res->type != ACPI_RESOURCE_TYPE_SERIAL_BUS)
+		return AE_OK;
+
+	csi2_res = &res->data.csi2_serial_bus;
+
+	if (csi2_res->type != ACPI_RESOURCE_SERIAL_TYPE_CSI2)
+		return AE_OK;
+
+	csi2_res_src = &csi2_res->resource_source;
+	if (ACPI_FAILURE(acpi_get_handle(NULL, csi2_res_src->string_ptr,
+					 &remote_handle))) {
+		acpi_handle_debug(crwd->handle,
+				  "unable to find resource source\n");
+		return AE_OK;
+	}
+	csi2_res_src_length = csi2_res_src->string_length;
+	if (!csi2_res_src_length) {
+		acpi_handle_debug(crwd->handle,
+				  "invalid resource source string length\n");
+		return AE_OK;
+	}
+
+	conn = kmalloc(struct_size(conn, remote_name, csi2_res_src_length + 1),
+		       GFP_KERNEL);
+	if (!conn)
+		return AE_OK;
+
+	conn->csi2_data = *csi2_res;
+	strscpy(conn->remote_name, csi2_res_src->string_ptr, csi2_res_src_length);
+	conn->csi2_data.resource_source.string_ptr = conn->remote_name;
+	conn->remote_handle = remote_handle;
+
+	list_add(&conn->entry, &crwd->connections);
+
+	return AE_OK;
+}
+
+static struct crs_csi2 *acpi_mipi_add_crs_csi2(acpi_handle handle,
+					       struct list_head *list)
+{
+	struct crs_csi2 *csi2;
+
+	csi2 = kzalloc(sizeof(*csi2), GFP_KERNEL);
+	if (!csi2)
+		return NULL;
+
+	csi2->handle = handle;
+	INIT_LIST_HEAD(&csi2->connections);
+	csi2->port_count = 1;
+
+	if (ACPI_FAILURE(acpi_attach_data(handle, acpi_mipi_data_tag, csi2))) {
+		kfree(csi2);
+		return NULL;
+	}
+
+	list_add(&csi2->entry, list);
+
+	return csi2;
+}
+
+static struct crs_csi2 *acpi_mipi_get_crs_csi2(acpi_handle handle)
+{
+	struct crs_csi2 *csi2;
+
+	if (ACPI_FAILURE(acpi_get_data_full(handle, acpi_mipi_data_tag,
+					    (void **)&csi2, NULL)))
+		return NULL;
+
+	return csi2;
+}
+
+static void csi_csr2_release_connections(struct list_head *list)
+{
+	struct crs_csi2_connection *conn, *conn_tmp;
+
+	list_for_each_entry_safe(conn, conn_tmp, list, entry) {
+		list_del(&conn->entry);
+		kfree(conn);
+	}
+}
+
+static void acpi_mipi_del_crs_csi2(struct crs_csi2 *csi2)
+{
+	list_del(&csi2->entry);
+	acpi_detach_data(csi2->handle, acpi_mipi_data_tag);
+	kfree(csi2->swnodes);
+	csi_csr2_release_connections(&csi2->connections);
+	kfree(csi2);
+}
+
+/**
+ * acpi_mipi_check_crs_csi2 - Look for CSI-2 resources in _CRS
+ * @handle: Device object handle to evaluate _CRS for.
+ *
+ * Find all CSI-2 resource descriptors in the given device's _CRS
+ * and collect them into a list.
+ */
+void acpi_mipi_check_crs_csi2(acpi_handle handle)
+{
+	struct csi2_resources_walk_data crwd = {
+		.handle = handle,
+		.connections = LIST_HEAD_INIT(crwd.connections),
+	};
+	struct crs_csi2 *csi2;
+
+	/*
+	 * Avoid allocating _CRS CSI-2 objects for devices without any CSI-2
+	 * resource descriptions in _CRS to reduce overhead.
+	 */
+	acpi_walk_resources(handle, METHOD_NAME__CRS, parse_csi2_resource, &crwd);
+	if (list_empty(&crwd.connections))
+		return;
+
+	/*
+	 * Create a _CRS CSI-2 entry to store the extracted connection
+	 * information and add it to the global list.
+	 */
+	csi2 = acpi_mipi_add_crs_csi2(handle, &acpi_mipi_crs_csi2_list);
+	if (!csi2) {
+		csi_csr2_release_connections(&crwd.connections);
+		return; /* Nothing really can be done about this. */
+	}
+
+	list_replace(&crwd.connections, &csi2->connections);
+}
+
+#define NO_CSI2_PORT (UINT_MAX - 1)
+
+static void alloc_crs_csi2_swnodes(struct crs_csi2 *csi2)
+{
+	size_t port_count = csi2->port_count;
+	struct acpi_device_software_nodes *swnodes;
+	size_t alloc_size;
+	unsigned int i;
+
+	/*
+	 * Allocate memory for ports, node pointers (number of nodes +
+	 * 1 (guardian), nodes (root + number of ports * 2 (because for
+	 * every port there is an endpoint)).
+	 */
+	if (check_mul_overflow(sizeof(*swnodes->ports) +
+			       sizeof(*swnodes->nodes) * 2 +
+			       sizeof(*swnodes->nodeptrs) * 2,
+			       port_count, &alloc_size) ||
+	    check_add_overflow(sizeof(*swnodes) +
+			       sizeof(*swnodes->nodes) +
+			       sizeof(*swnodes->nodeptrs) * 2,
+			       alloc_size, &alloc_size)) {
+		acpi_handle_info(csi2->handle,
+				 "too many _CRS CSI-2 resource handles (%zu)",
+				 port_count);
+		return;
+	}
+
+	swnodes = kmalloc(alloc_size, GFP_KERNEL);
+	if (!swnodes)
+		return;
+
+	swnodes->ports = (struct acpi_device_software_node_port *)(swnodes + 1);
+	swnodes->nodes = (struct software_node *)(swnodes->ports + port_count);
+	swnodes->nodeptrs = (const struct software_node **)(swnodes->nodes + 1 +
+				2 * port_count);
+	swnodes->num_ports = port_count;
+
+	for (i = 0; i < 2 * port_count + 1; i++)
+		swnodes->nodeptrs[i] = &swnodes->nodes[i];
+
+	swnodes->nodeptrs[i] = NULL;
+
+	for (i = 0; i < port_count; i++)
+		swnodes->ports[i].port_nr = NO_CSI2_PORT;
+
+	csi2->swnodes = swnodes;
+}
+
+#define ACPI_CRS_CSI2_PHY_TYPE_C	0
+#define ACPI_CRS_CSI2_PHY_TYPE_D	1
+
+static unsigned int next_csi2_port_index(struct acpi_device_software_nodes *swnodes,
+					 unsigned int port_nr)
+{
+	unsigned int i;
+
+	for (i = 0; i < swnodes->num_ports; i++) {
+		struct acpi_device_software_node_port *port = &swnodes->ports[i];
+
+		if (port->port_nr == port_nr)
+			return i;
+
+		if (port->port_nr == NO_CSI2_PORT) {
+			port->port_nr = port_nr;
+			return i;
+		}
+	}
+
+	return NO_CSI2_PORT;
+}
+
+/* Print graph port name into a buffer, return non-zero on failure. */
+#define GRAPH_PORT_NAME(var, num)					    \
+	(snprintf((var), sizeof(var), SWNODE_GRAPH_PORT_NAME_FMT, (num)) >= \
+	 sizeof(var))
+
+static void extract_crs_csi2_conn_info(acpi_handle local_handle,
+				       struct acpi_device_software_nodes *local_swnodes,
+				       struct crs_csi2_connection *conn)
+{
+	struct crs_csi2 *remote_csi2 = acpi_mipi_get_crs_csi2(conn->remote_handle);
+	struct acpi_device_software_nodes *remote_swnodes;
+	struct acpi_device_software_node_port *local_port, *remote_port;
+	struct software_node *local_node, *remote_node;
+	unsigned int local_index, remote_index;
+	unsigned int bus_type;
+
+	/*
+	 * If the previous steps have failed to make room for a _CRS CSI-2
+	 * representation for the remote end of the given connection, skip it.
+	 */
+	if (!remote_csi2)
+		return;
+
+	remote_swnodes = remote_csi2->swnodes;
+	if (!remote_swnodes)
+		return;
+
+	switch (conn->csi2_data.phy_type) {
+	case ACPI_CRS_CSI2_PHY_TYPE_C:
+		bus_type = V4L2_FWNODE_BUS_TYPE_CSI2_CPHY;
+		break;
+
+	case ACPI_CRS_CSI2_PHY_TYPE_D:
+		bus_type = V4L2_FWNODE_BUS_TYPE_CSI2_DPHY;
+		break;
+
+	default:
+		acpi_handle_info(local_handle, "unknown CSI-2 PHY type %u\n",
+				 conn->csi2_data.phy_type);
+		return;
+	}
+
+	local_index = next_csi2_port_index(local_swnodes,
+					   conn->csi2_data.local_port_instance);
+	if (WARN_ON_ONCE(local_index >= local_swnodes->num_ports))
+		return;
+
+	remote_index = next_csi2_port_index(remote_swnodes,
+					    conn->csi2_data.resource_source.index);
+	if (WARN_ON_ONCE(remote_index >= remote_swnodes->num_ports))
+		return;
+
+	local_port = &local_swnodes->ports[local_index];
+	local_node = &local_swnodes->nodes[ACPI_DEVICE_SWNODE_EP(local_index)];
+	local_port->crs_csi2_local = true;
+
+	remote_port = &remote_swnodes->ports[remote_index];
+	remote_node = &remote_swnodes->nodes[ACPI_DEVICE_SWNODE_EP(remote_index)];
+
+	local_port->remote_ep[0] = SOFTWARE_NODE_REFERENCE(remote_node);
+	remote_port->remote_ep[0] = SOFTWARE_NODE_REFERENCE(local_node);
+
+	local_port->ep_props[ACPI_DEVICE_SWNODE_EP_REMOTE_EP] =
+			PROPERTY_ENTRY_REF_ARRAY("remote-endpoint",
+						 local_port->remote_ep);
+
+	local_port->ep_props[ACPI_DEVICE_SWNODE_EP_BUS_TYPE] =
+			PROPERTY_ENTRY_U32("bus-type", bus_type);
+
+	local_port->ep_props[ACPI_DEVICE_SWNODE_EP_REG] =
+			PROPERTY_ENTRY_U32("reg", 0);
+
+	local_port->port_props[ACPI_DEVICE_SWNODE_PORT_REG] =
+			PROPERTY_ENTRY_U32("reg", conn->csi2_data.local_port_instance);
+
+	if (GRAPH_PORT_NAME(local_port->port_name,
+			    conn->csi2_data.local_port_instance))
+		acpi_handle_info(local_handle, "local port %u name too long",
+				 conn->csi2_data.local_port_instance);
+
+	remote_port->ep_props[ACPI_DEVICE_SWNODE_EP_REMOTE_EP] =
+			PROPERTY_ENTRY_REF_ARRAY("remote-endpoint",
+						 remote_port->remote_ep);
+
+	remote_port->ep_props[ACPI_DEVICE_SWNODE_EP_BUS_TYPE] =
+			PROPERTY_ENTRY_U32("bus-type", bus_type);
+
+	remote_port->ep_props[ACPI_DEVICE_SWNODE_EP_REG] =
+			PROPERTY_ENTRY_U32("reg", 0);
+
+	remote_port->port_props[ACPI_DEVICE_SWNODE_PORT_REG] =
+			PROPERTY_ENTRY_U32("reg", conn->csi2_data.resource_source.index);
+
+	if (GRAPH_PORT_NAME(remote_port->port_name,
+			    conn->csi2_data.resource_source.index))
+		acpi_handle_info(local_handle, "remote port %u name too long",
+				 conn->csi2_data.resource_source.index);
+}
+
+static void prepare_crs_csi2_swnodes(struct crs_csi2 *csi2)
+{
+	struct acpi_device_software_nodes *local_swnodes = csi2->swnodes;
+	acpi_handle local_handle = csi2->handle;
+	struct crs_csi2_connection *conn;
+
+	/* Bail out if the allocation of swnodes has failed. */
+	if (!local_swnodes)
+		return;
+
+	list_for_each_entry(conn, &csi2->connections, entry)
+		extract_crs_csi2_conn_info(local_handle, local_swnodes, conn);
+}
+
+/**
+ * acpi_mipi_scan_crs_csi2 - Create ACPI _CRS CSI-2 software nodes
+ *
+ * Note that this function must be called before any struct acpi_device objects
+ * are bound to any ACPI drivers or scan handlers, so it cannot assume the
+ * existence of struct acpi_device objects for every device present in the ACPI
+ * namespace.
+ *
+ * acpi_scan_lock in scan.c must be held when calling this function.
+ */
+void acpi_mipi_scan_crs_csi2(void)
+{
+	struct crs_csi2 *csi2;
+	LIST_HEAD(aux_list);
+
+	/* Count references to each ACPI handle in the CSI-2 connection graph. */
+	list_for_each_entry(csi2, &acpi_mipi_crs_csi2_list, entry) {
+		struct crs_csi2_connection *conn;
+
+		list_for_each_entry(conn, &csi2->connections, entry) {
+			struct crs_csi2 *remote_csi2;
+
+			csi2->port_count++;
+
+			remote_csi2 = acpi_mipi_get_crs_csi2(conn->remote_handle);
+			if (remote_csi2) {
+				remote_csi2->port_count++;
+				continue;
+			}
+			/*
+			 * The remote endpoint has no _CRS CSI-2 list entry yet,
+			 * so create one for it and add it to the list.
+			 */
+			acpi_mipi_add_crs_csi2(conn->remote_handle, &aux_list);
+		}
+	}
+	list_splice(&aux_list, &acpi_mipi_crs_csi2_list);
+
+	/*
+	 * Allocate software nodes for representing the CSI-2 information.
+	 *
+	 * This needs to be done for all of the list entries in one go, because
+	 * they may point to each other without restrictions and the next step
+	 * relies on the availability of swnodes memory for each list entry.
+	 */
+	list_for_each_entry(csi2, &acpi_mipi_crs_csi2_list, entry)
+		alloc_crs_csi2_swnodes(csi2);
+
+	/*
+	 * Set up software node properties using data from _CRS CSI-2 resource
+	 * descriptors.
+	 */
+	list_for_each_entry(csi2, &acpi_mipi_crs_csi2_list, entry)
+		prepare_crs_csi2_swnodes(csi2);
+}
+
+/*
+ * Get the index of the next property in the property array, with a given
+ * maximum value.
+ */
+#define NEXT_PROPERTY(index, max)			\
+	(WARN_ON((index) > ACPI_DEVICE_SWNODE_##max) ?	\
+	 ACPI_DEVICE_SWNODE_##max : (index)++)
+
+static void init_csi2_port_local(struct acpi_device *adev,
+				 struct acpi_device_software_node_port *port,
+				 struct fwnode_handle *port_fwnode,
+				 unsigned int index)
+{
+	acpi_handle handle = acpi_device_handle(adev);
+	unsigned int num_link_freqs;
+	int ret;
+
+	ret = fwnode_property_count_u64(port_fwnode, "mipi-img-link-frequencies");
+	if (ret <= 0)
+		return;
+
+	num_link_freqs = ret;
+	if (num_link_freqs > ACPI_DEVICE_CSI2_DATA_LANES) {
+		acpi_handle_info(handle, "Too many link frequencies: %u\n",
+				 num_link_freqs);
+		num_link_freqs = ACPI_DEVICE_CSI2_DATA_LANES;
+	}
+
+	ret = fwnode_property_read_u64_array(port_fwnode,
+					     "mipi-img-link-frequencies",
+					     port->link_frequencies,
+					     num_link_freqs);
+	if (ret) {
+		acpi_handle_info(handle, "Unable to get link frequencies (%d)\n",
+				 ret);
+		return;
+	}
+
+	port->ep_props[NEXT_PROPERTY(index, EP_LINK_FREQUENCIES)] =
+				PROPERTY_ENTRY_U64_ARRAY_LEN("link-frequencies",
+							     port->link_frequencies,
+							     num_link_freqs);
+}
+
+static void init_csi2_port(struct acpi_device *adev,
+			   struct acpi_device_software_nodes *swnodes,
+			   struct acpi_device_software_node_port *port,
+			   struct fwnode_handle *port_fwnode,
+			   unsigned int port_index)
+{
+	unsigned int ep_prop_index = ACPI_DEVICE_SWNODE_EP_CLOCK_LANES;
+	acpi_handle handle = acpi_device_handle(adev);
+	u8 val[ACPI_DEVICE_CSI2_DATA_LANES];
+	int num_lanes = 0;
+	int ret;
+
+	if (GRAPH_PORT_NAME(port->port_name, port->port_nr))
+		return;
+
+	swnodes->nodes[ACPI_DEVICE_SWNODE_PORT(port_index)] =
+			SOFTWARE_NODE(port->port_name, port->port_props,
+				      &swnodes->nodes[ACPI_DEVICE_SWNODE_ROOT]);
+
+	ret = fwnode_property_read_u8(port_fwnode, "mipi-img-clock-lane", val);
+	if (!ret)
+		port->ep_props[NEXT_PROPERTY(ep_prop_index, EP_CLOCK_LANES)] =
+			PROPERTY_ENTRY_U32("clock-lanes", val[0]);
+
+	ret = fwnode_property_count_u8(port_fwnode, "mipi-img-data-lanes");
+	if (ret > 0) {
+		num_lanes = ret;
+
+		if (num_lanes > ACPI_DEVICE_CSI2_DATA_LANES) {
+			acpi_handle_info(handle, "Too many data lanes: %u\n",
+					 num_lanes);
+			num_lanes = ACPI_DEVICE_CSI2_DATA_LANES;
+		}
+
+		ret = fwnode_property_read_u8_array(port_fwnode,
+						    "mipi-img-data-lanes",
+						    val, num_lanes);
+		if (!ret) {
+			unsigned int i;
+
+			for (i = 0; i < num_lanes; i++)
+				port->data_lanes[i] = val[i];
+
+			port->ep_props[NEXT_PROPERTY(ep_prop_index, EP_DATA_LANES)] =
+				PROPERTY_ENTRY_U32_ARRAY_LEN("data-lanes",
+							     port->data_lanes,
+							     num_lanes);
+		}
+	}
+
+	ret = fwnode_property_count_u8(port_fwnode, "mipi-img-lane-polarities");
+	if (ret < 0) {
+		acpi_handle_debug(handle, "Lane polarity bytes missing\n");
+	} else if (ret * BITS_PER_TYPE(u8) < num_lanes + 1) {
+		acpi_handle_info(handle, "Too few lane polarity bits (%zu vs. %d)\n",
+				 ret * BITS_PER_TYPE(u8), num_lanes + 1);
+	} else {
+		unsigned long mask = 0;
+		int byte_count = ret;
+		unsigned int i;
+
+		/*
+		 * The total number of lanes is ACPI_DEVICE_CSI2_DATA_LANES + 1
+		 * (data lanes + clock lane).  It is not expected to ever be
+		 * greater than the number of bits in an unsigned long
+		 * variable, but ensure that this is the case.
+		 */
+		BUILD_BUG_ON(BITS_PER_TYPE(unsigned long) <= ACPI_DEVICE_CSI2_DATA_LANES);
+
+		if (byte_count > sizeof(mask)) {
+			acpi_handle_info(handle, "Too many lane polarities: %d\n",
+					 byte_count);
+			byte_count = sizeof(mask);
+		}
+		fwnode_property_read_u8_array(port_fwnode, "mipi-img-lane-polarities",
+					      val, byte_count);
+
+		for (i = 0; i < byte_count; i++)
+			mask |= (unsigned long)val[i] << BITS_PER_TYPE(u8) * i;
+
+		for (i = 0; i <= num_lanes; i++)
+			port->lane_polarities[i] = test_bit(i, &mask);
+
+		port->ep_props[NEXT_PROPERTY(ep_prop_index, EP_LANE_POLARITIES)] =
+				PROPERTY_ENTRY_U32_ARRAY_LEN("lane-polarities",
+							     port->lane_polarities,
+							     num_lanes + 1);
+	}
+
+	swnodes->nodes[ACPI_DEVICE_SWNODE_EP(port_index)] =
+		SOFTWARE_NODE("endpoint@0", swnodes->ports[port_index].ep_props,
+			      &swnodes->nodes[ACPI_DEVICE_SWNODE_PORT(port_index)]);
+
+	if (port->crs_csi2_local)
+		init_csi2_port_local(adev, port, port_fwnode, ep_prop_index);
+}
+
+#define MIPI_IMG_PORT_PREFIX "mipi-img-port-"
+
+static struct fwnode_handle *get_mipi_port_handle(struct fwnode_handle *adev_fwnode,
+						  unsigned int port_nr)
+{
+	char port_name[sizeof(MIPI_IMG_PORT_PREFIX) + 2];
+
+	if (snprintf(port_name, sizeof(port_name), "%s%u",
+		     MIPI_IMG_PORT_PREFIX, port_nr) >= sizeof(port_name))
+		return NULL;
+
+	return fwnode_get_named_child_node(adev_fwnode, port_name);
+}
+
+static void init_crs_csi2_swnodes(struct crs_csi2 *csi2)
+{
+	struct acpi_buffer buffer = { .length = ACPI_ALLOCATE_BUFFER };
+	struct acpi_device_software_nodes *swnodes = csi2->swnodes;
+	acpi_handle handle = csi2->handle;
+	unsigned int prop_index = 0;
+	struct fwnode_handle *adev_fwnode;
+	struct acpi_device *adev;
+	acpi_status status;
+	unsigned int i;
+	u32 val;
+	int ret;
+
+	/*
+	 * Bail out if the swnodes are not available (either they have not been
+	 * allocated or they have been assigned to the device already).
+	 */
+	if (!swnodes)
+		return;
+
+	adev = acpi_fetch_acpi_dev(handle);
+	if (!adev)
+		return;
+
+	adev_fwnode = acpi_fwnode_handle(adev);
+
+	/*
+	 * If the "rotation" property is not present, but _PLD is there,
+	 * evaluate it to get the "rotation" value.
+	 */
+	if (!fwnode_property_present(adev_fwnode, "rotation")) {
+		struct acpi_pld_info *pld;
+
+		status = acpi_get_physical_device_location(handle, &pld);
+		if (ACPI_SUCCESS(status)) {
+			swnodes->dev_props[NEXT_PROPERTY(prop_index, DEV_ROTATION)] =
+					PROPERTY_ENTRY_U32("rotation",
+							   pld->rotation * 45U);
+			kfree(pld);
+		}
+	}
+
+	if (!fwnode_property_read_u32(adev_fwnode, "mipi-img-clock-frequency", &val))
+		swnodes->dev_props[NEXT_PROPERTY(prop_index, DEV_CLOCK_FREQUENCY)] =
+			PROPERTY_ENTRY_U32("clock-frequency", val);
+
+	if (!fwnode_property_read_u32(adev_fwnode, "mipi-img-led-max-current", &val))
+		swnodes->dev_props[NEXT_PROPERTY(prop_index, DEV_LED_MAX_MICROAMP)] =
+			PROPERTY_ENTRY_U32("led-max-microamp", val);
+
+	if (!fwnode_property_read_u32(adev_fwnode, "mipi-img-flash-max-current", &val))
+		swnodes->dev_props[NEXT_PROPERTY(prop_index, DEV_FLASH_MAX_MICROAMP)] =
+			PROPERTY_ENTRY_U32("flash-max-microamp", val);
+
+	if (!fwnode_property_read_u32(adev_fwnode, "mipi-img-flash-max-timeout-us", &val))
+		swnodes->dev_props[NEXT_PROPERTY(prop_index, DEV_FLASH_MAX_TIMEOUT_US)] =
+			PROPERTY_ENTRY_U32("flash-max-timeout-us", val);
+
+	status = acpi_get_name(handle, ACPI_FULL_PATHNAME, &buffer);
+	if (ACPI_FAILURE(status)) {
+		acpi_handle_info(handle, "Unable to get the path name\n");
+		return;
+	}
+
+	swnodes->nodes[ACPI_DEVICE_SWNODE_ROOT] =
+			SOFTWARE_NODE(buffer.pointer, swnodes->dev_props, NULL);
+
+	for (i = 0; i < swnodes->num_ports; i++) {
+		struct acpi_device_software_node_port *port = &swnodes->ports[i];
+		struct fwnode_handle *port_fwnode;
+
+		/*
+		 * The MIPI DisCo for Imaging specification defines _DSD device
+		 * properties for providing CSI-2 port parameters that can be
+		 * accessed through the generic device properties framework.  To
+		 * access them, it is first necessary to find the data node
+		 * representing the port under the given ACPI device object.
+		 */
+		port_fwnode = get_mipi_port_handle(adev_fwnode, port->port_nr);
+		if (!port_fwnode) {
+			acpi_handle_info(handle,
+					 "MIPI port name too long for port %u\n",
+					 port->port_nr);
+			continue;
+		}
+
+		init_csi2_port(adev, swnodes, port, port_fwnode, i);
+
+		fwnode_handle_put(port_fwnode);
+	}
+
+	ret = software_node_register_node_group(swnodes->nodeptrs);
+	if (ret < 0) {
+		acpi_handle_info(handle,
+				 "Unable to register software nodes (%d)\n", ret);
+		return;
+	}
+
+	adev->swnodes = swnodes;
+	adev_fwnode->secondary = software_node_fwnode(swnodes->nodes);
+
+	/*
+	 * Prevents the swnodes from this csi2 entry from being assigned again
+	 * or freed prematurely.
+	 */
+	csi2->swnodes = NULL;
+}
+
+/**
+ * acpi_mipi_init_crs_csi2_swnodes - Initialize _CRS CSI-2 software nodes
+ *
+ * Use MIPI DisCo for Imaging device properties to finalize the initialization
+ * of CSI-2 software nodes for all ACPI device objects that have been already
+ * enumerated.
+ */
+void acpi_mipi_init_crs_csi2_swnodes(void)
+{
+	struct crs_csi2 *csi2, *csi2_tmp;
+
+	list_for_each_entry_safe(csi2, csi2_tmp, &acpi_mipi_crs_csi2_list, entry)
+		init_crs_csi2_swnodes(csi2);
+}
+
+/**
+ * acpi_mipi_crs_csi2_cleanup - Free _CRS CSI-2 temporary data
+ */
+void acpi_mipi_crs_csi2_cleanup(void)
+{
+	struct crs_csi2 *csi2, *csi2_tmp;
+
+	list_for_each_entry_safe(csi2, csi2_tmp, &acpi_mipi_crs_csi2_list, entry)
+		acpi_mipi_del_crs_csi2(csi2);
+}
diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c
index 3a34a8c425fe..55437f5e0c3a 100644
--- a/drivers/acpi/processor_idle.c
+++ b/drivers/acpi/processor_idle.c
@@ -592,7 +592,7 @@ static int acpi_idle_play_dead(struct cpuidle_device *dev, int index)
 	while (1) {
 
 		if (cx->entry_method == ACPI_CSTATE_HALT)
-			safe_halt();
+			raw_safe_halt();
 		else if (cx->entry_method == ACPI_CSTATE_SYSTEMIO) {
 			io_idle(cx->address);
 		} else
diff --git a/drivers/acpi/property.c b/drivers/acpi/property.c
index 6979a3f9f90a..07d76fb740b6 100644
--- a/drivers/acpi/property.c
+++ b/drivers/acpi/property.c
@@ -2,14 +2,17 @@
 /*
  * ACPI device specific properties support.
  *
- * Copyright (C) 2014, Intel Corporation
+ * Copyright (C) 2014 - 2023, Intel Corporation
  * All rights reserved.
  *
  * Authors: Mika Westerberg <mika.westerberg@linux.intel.com>
  *          Darren Hart <dvhart@linux.intel.com>
  *          Rafael J. Wysocki <rafael.j.wysocki@intel.com>
+ *          Sakari Ailus <sakari.ailus@linux.intel.com>
  */
 
+#define pr_fmt(fmt) "ACPI: " fmt
+
 #include <linux/acpi.h>
 #include <linux/device.h>
 #include <linux/export.h>
@@ -800,28 +803,16 @@ static int acpi_get_ref_args(struct fwnode_reference_args *args,
 {
 	u32 nargs = 0, i;
 
-	/*
-	 * Find the referred data extension node under the
-	 * referred device node.
-	 */
-	for (; *element < end && (*element)->type == ACPI_TYPE_STRING;
-	     (*element)++) {
-		const char *child_name = (*element)->string.pointer;
-
-		ref_fwnode = acpi_fwnode_get_named_child_node(ref_fwnode, child_name);
-		if (!ref_fwnode)
-			return -EINVAL;
-	}
-
 	/*
 	 * Assume the following integer elements are all args. Stop counting on
-	 * the first reference or end of the package arguments. In case of
-	 * neither reference, nor integer, return an error, we can't parse it.
+	 * the first reference (possibly represented as a string) or end of the
+	 * package arguments. In case of neither reference, nor integer, return
+	 * an error, we can't parse it.
 	 */
 	for (i = 0; (*element) + i < end && i < num_args; i++) {
 		acpi_object_type type = (*element)[i].type;
 
-		if (type == ACPI_TYPE_LOCAL_REFERENCE)
+		if (type == ACPI_TYPE_LOCAL_REFERENCE || type == ACPI_TYPE_STRING)
 			break;
 
 		if (type == ACPI_TYPE_INTEGER)
@@ -845,6 +836,44 @@ static int acpi_get_ref_args(struct fwnode_reference_args *args,
 	return 0;
 }
 
+static struct fwnode_handle *acpi_parse_string_ref(const struct fwnode_handle *fwnode,
+						   const char *refstring)
+{
+	acpi_handle scope, handle;
+	struct acpi_data_node *dn;
+	struct acpi_device *device;
+	acpi_status status;
+
+	if (is_acpi_device_node(fwnode)) {
+		scope = to_acpi_device_node(fwnode)->handle;
+	} else if (is_acpi_data_node(fwnode)) {
+		scope = to_acpi_data_node(fwnode)->handle;
+	} else {
+		pr_debug("Bad node type for node %pfw\n", fwnode);
+		return NULL;
+	}
+
+	status = acpi_get_handle(scope, refstring, &handle);
+	if (ACPI_FAILURE(status)) {
+		acpi_handle_debug(scope, "Unable to get an ACPI handle for %s\n",
+				  refstring);
+		return NULL;
+	}
+
+	device = acpi_fetch_acpi_dev(handle);
+	if (device)
+		return acpi_fwnode_handle(device);
+
+	status = acpi_get_data_full(handle, acpi_nondev_subnode_tag,
+				    (void **)&dn, NULL);
+	if (ACPI_FAILURE(status) || !dn) {
+		acpi_handle_debug(handle, "Subnode not found\n");
+		return NULL;
+	}
+
+	return &dn->fwnode;
+}
+
 /**
  * __acpi_node_get_property_reference - returns handle to the referenced object
  * @fwnode: Firmware node to get the property from
@@ -887,6 +916,7 @@ int __acpi_node_get_property_reference(const struct fwnode_handle *fwnode,
 	const union acpi_object *element, *end;
 	const union acpi_object *obj;
 	const struct acpi_device_data *data;
+	struct fwnode_handle *ref_fwnode;
 	struct acpi_device *device;
 	int ret, idx = 0;
 
@@ -910,16 +940,30 @@ int __acpi_node_get_property_reference(const struct fwnode_handle *fwnode,
 
 		args->fwnode = acpi_fwnode_handle(device);
 		args->nargs = 0;
+
+		return 0;
+	case ACPI_TYPE_STRING:
+		if (index)
+			return -ENOENT;
+
+		ref_fwnode = acpi_parse_string_ref(fwnode, obj->string.pointer);
+		if (!ref_fwnode)
+			return -EINVAL;
+
+		args->fwnode = ref_fwnode;
+		args->nargs = 0;
+
 		return 0;
 	case ACPI_TYPE_PACKAGE:
 		/*
 		 * If it is not a single reference, then it is a package of
-		 * references followed by number of ints as follows:
+		 * references, followed by number of ints as follows:
 		 *
 		 *  Package () { REF, INT, REF, INT, INT }
 		 *
-		 * The index argument is then used to determine which reference
-		 * the caller wants (along with the arguments).
+		 * Here, REF may be either a local reference or a string. The
+		 * index argument is then used to determine which reference the
+		 * caller wants (along with the arguments).
 		 */
 		break;
 	default:
@@ -950,6 +994,24 @@ int __acpi_node_get_property_reference(const struct fwnode_handle *fwnode,
 			if (idx == index)
 				return 0;
 
+			break;
+		case ACPI_TYPE_STRING:
+			ref_fwnode = acpi_parse_string_ref(fwnode,
+							   element->string.pointer);
+			if (!ref_fwnode)
+				return -EINVAL;
+
+			element++;
+
+			ret = acpi_get_ref_args(idx == index ? args : NULL,
+						ref_fwnode, &element, end,
+						num_args);
+			if (ret < 0)
+				return ret;
+
+			if (idx == index)
+				return 0;
+
 			break;
 		case ACPI_TYPE_INTEGER:
 			if (idx == index)
diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c
index 15a3bdbd0755..9bd9f79cd409 100644
--- a/drivers/acpi/resource.c
+++ b/drivers/acpi/resource.c
@@ -447,6 +447,13 @@ static const struct dmi_system_id irq1_level_low_skip_override[] = {
 			DMI_MATCH(DMI_BOARD_NAME, "B1402CBA"),
 		},
 	},
+	{
+		/* Asus ExpertBook B1402CVA */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_MATCH(DMI_BOARD_NAME, "B1402CVA"),
+		},
+	},
 	{
 		/* Asus ExpertBook B1502CBA */
 		.matches = {
diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c
index fa5dd71a80fa..b8df04779904 100644
--- a/drivers/acpi/scan.c
+++ b/drivers/acpi/scan.c
@@ -1976,7 +1976,7 @@ static void acpi_scan_init_hotplug(struct acpi_device *adev)
 	}
 }
 
-static u32 acpi_scan_check_dep(acpi_handle handle, bool check_dep)
+static u32 acpi_scan_check_dep(acpi_handle handle)
 {
 	struct acpi_handle_list dep_devices;
 	acpi_status status;
@@ -1989,8 +1989,7 @@ static u32 acpi_scan_check_dep(acpi_handle handle, bool check_dep)
 	 * 2. ACPI nodes describing USB ports.
 	 * Still, checking for _HID catches more then just these cases ...
 	 */
-	if (!check_dep || !acpi_has_method(handle, "_DEP") ||
-	    !acpi_has_method(handle, "_HID"))
+	if (!acpi_has_method(handle, "_DEP") || !acpi_has_method(handle, "_HID"))
 		return 0;
 
 	status = acpi_evaluate_reference(handle, "_DEP", NULL, &dep_devices);
@@ -2036,7 +2035,13 @@ static u32 acpi_scan_check_dep(acpi_handle handle, bool check_dep)
 	return count;
 }
 
-static acpi_status acpi_bus_check_add(acpi_handle handle, bool check_dep,
+static acpi_status acpi_scan_check_crs_csi2_cb(acpi_handle handle, u32 a, void *b, void **c)
+{
+	acpi_mipi_check_crs_csi2(handle);
+	return AE_OK;
+}
+
+static acpi_status acpi_bus_check_add(acpi_handle handle, bool first_pass,
 				      struct acpi_device **adev_p)
 {
 	struct acpi_device *device = acpi_fetch_acpi_dev(handle);
@@ -2054,9 +2059,25 @@ static acpi_status acpi_bus_check_add(acpi_handle handle, bool check_dep,
 		if (acpi_device_should_be_hidden(handle))
 			return AE_OK;
 
-		/* Bail out if there are dependencies. */
-		if (acpi_scan_check_dep(handle, check_dep) > 0)
-			return AE_CTRL_DEPTH;
+		if (first_pass) {
+			acpi_mipi_check_crs_csi2(handle);
+
+			/* Bail out if there are dependencies. */
+			if (acpi_scan_check_dep(handle) > 0) {
+				/*
+				 * The entire CSI-2 connection graph needs to be
+				 * extracted before any drivers or scan handlers
+				 * are bound to struct device objects, so scan
+				 * _CRS CSI-2 resource descriptors for all
+				 * devices below the current handle.
+				 */
+				acpi_walk_namespace(ACPI_TYPE_DEVICE, handle,
+						    ACPI_UINT32_MAX,
+						    acpi_scan_check_crs_csi2_cb,
+						    NULL, NULL, NULL);
+				return AE_CTRL_DEPTH;
+			}
+		}
 
 		fallthrough;
 	case ACPI_TYPE_ANY:	/* for ACPI_ROOT_OBJECT */
@@ -2079,10 +2100,10 @@ static acpi_status acpi_bus_check_add(acpi_handle handle, bool check_dep,
 	}
 
 	/*
-	 * If check_dep is true at this point, the device has no dependencies,
+	 * If first_pass is true at this point, the device has no dependencies,
 	 * or the creation of the device object would have been postponed above.
 	 */
-	acpi_add_single_object(&device, handle, type, !check_dep);
+	acpi_add_single_object(&device, handle, type, !first_pass);
 	if (!device)
 		return AE_CTRL_DEPTH;
 
@@ -2426,6 +2447,13 @@ static void acpi_scan_postponed_branch(acpi_handle handle)
 
 	acpi_walk_namespace(ACPI_TYPE_ANY, handle, ACPI_UINT32_MAX,
 			    acpi_bus_check_add_2, NULL, NULL, (void **)&adev);
+
+	/*
+	 * Populate the ACPI _CRS CSI-2 software nodes for the ACPI devices that
+	 * have been added above.
+	 */
+	acpi_mipi_init_crs_csi2_swnodes();
+
 	acpi_bus_attach(adev, NULL);
 }
 
@@ -2494,12 +2522,22 @@ int acpi_bus_scan(acpi_handle handle)
 	if (!device)
 		return -ENODEV;
 
+	/*
+	 * Set up ACPI _CRS CSI-2 software nodes using information extracted
+	 * from the _CRS CSI-2 resource descriptors during the ACPI namespace
+	 * walk above and MIPI DisCo for Imaging device properties.
+	 */
+	acpi_mipi_scan_crs_csi2();
+	acpi_mipi_init_crs_csi2_swnodes();
+
 	acpi_bus_attach(device, (void *)true);
 
 	/* Pass 2: Enumerate all of the remaining devices. */
 
 	acpi_scan_postponed();
 
+	acpi_mipi_crs_csi2_cleanup();
+
 	return 0;
 }
 EXPORT_SYMBOL(acpi_bus_scan);
diff --git a/drivers/ata/pata_pxa.c b/drivers/ata/pata_pxa.c
index 5275c6464f57..538bd3423d85 100644
--- a/drivers/ata/pata_pxa.c
+++ b/drivers/ata/pata_pxa.c
@@ -274,10 +274,9 @@ static int pxa_ata_probe(struct platform_device *pdev)
 	/*
 	 * Request the DMA channel
 	 */
-	data->dma_chan =
-		dma_request_slave_channel(&pdev->dev, "data");
-	if (!data->dma_chan)
-		return -EBUSY;
+	data->dma_chan = dma_request_chan(&pdev->dev, "data");
+	if (IS_ERR(data->dma_chan))
+		return PTR_ERR(data->dma_chan);
 	ret = dmaengine_slave_config(data->dma_chan, &config);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "dma configuration failed: %d\n", ret);
diff --git a/drivers/base/memory.c b/drivers/base/memory.c
index f3b9a4d0fa3b..8a13babd826c 100644
--- a/drivers/base/memory.c
+++ b/drivers/base/memory.c
@@ -180,6 +180,9 @@ static inline unsigned long memblk_nr_poison(struct memory_block *mem)
 }
 #endif
 
+/*
+ * Must acquire mem_hotplug_lock in write mode.
+ */
 static int memory_block_online(struct memory_block *mem)
 {
 	unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr);
@@ -204,10 +207,11 @@ static int memory_block_online(struct memory_block *mem)
 	if (mem->altmap)
 		nr_vmemmap_pages = mem->altmap->free;
 
+	mem_hotplug_begin();
 	if (nr_vmemmap_pages) {
 		ret = mhp_init_memmap_on_memory(start_pfn, nr_vmemmap_pages, zone);
 		if (ret)
-			return ret;
+			goto out;
 	}
 
 	ret = online_pages(start_pfn + nr_vmemmap_pages,
@@ -215,7 +219,7 @@ static int memory_block_online(struct memory_block *mem)
 	if (ret) {
 		if (nr_vmemmap_pages)
 			mhp_deinit_memmap_on_memory(start_pfn, nr_vmemmap_pages);
-		return ret;
+		goto out;
 	}
 
 	/*
@@ -227,9 +231,14 @@ static int memory_block_online(struct memory_block *mem)
 					  nr_vmemmap_pages);
 
 	mem->zone = zone;
+out:
+	mem_hotplug_done();
 	return ret;
 }
 
+/*
+ * Must acquire mem_hotplug_lock in write mode.
+ */
 static int memory_block_offline(struct memory_block *mem)
 {
 	unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr);
@@ -247,6 +256,7 @@ static int memory_block_offline(struct memory_block *mem)
 	if (mem->altmap)
 		nr_vmemmap_pages = mem->altmap->free;
 
+	mem_hotplug_begin();
 	if (nr_vmemmap_pages)
 		adjust_present_page_count(pfn_to_page(start_pfn), mem->group,
 					  -nr_vmemmap_pages);
@@ -258,13 +268,15 @@ static int memory_block_offline(struct memory_block *mem)
 		if (nr_vmemmap_pages)
 			adjust_present_page_count(pfn_to_page(start_pfn),
 						  mem->group, nr_vmemmap_pages);
-		return ret;
+		goto out;
 	}
 
 	if (nr_vmemmap_pages)
 		mhp_deinit_memmap_on_memory(start_pfn, nr_vmemmap_pages);
 
 	mem->zone = NULL;
+out:
+	mem_hotplug_done();
 	return ret;
 }
 
diff --git a/drivers/base/property.c b/drivers/base/property.c
index 8c40abed7852..8f8e2a6816bc 100644
--- a/drivers/base/property.c
+++ b/drivers/base/property.c
@@ -473,7 +473,7 @@ int fwnode_property_match_string(const struct fwnode_handle *fwnode,
 	const char **values;
 	int nval, ret;
 
-	nval = fwnode_property_read_string_array(fwnode, propname, NULL, 0);
+	nval = fwnode_property_string_array_count(fwnode, propname);
 	if (nval < 0)
 		return nval;
 
@@ -498,6 +498,41 @@ int fwnode_property_match_string(const struct fwnode_handle *fwnode,
 }
 EXPORT_SYMBOL_GPL(fwnode_property_match_string);
 
+/**
+ * fwnode_property_match_property_string - find a property string value in an array and return index
+ * @fwnode: Firmware node to get the property of
+ * @propname: Name of the property holding the string value
+ * @array: String array to search in
+ * @n: Size of the @array
+ *
+ * Find a property string value in a given @array and if it is found return
+ * the index back.
+ *
+ * Return: index, starting from %0, if the string value was found in the @array (success),
+ *	   %-ENOENT when the string value was not found in the @array,
+ *	   %-EINVAL if given arguments are not valid,
+ *	   %-ENODATA if the property does not have a value,
+ *	   %-EPROTO or %-EILSEQ if the property is not a string,
+ *	   %-ENXIO if no suitable firmware interface is present.
+ */
+int fwnode_property_match_property_string(const struct fwnode_handle *fwnode,
+	const char *propname, const char * const *array, size_t n)
+{
+	const char *string;
+	int ret;
+
+	ret = fwnode_property_read_string(fwnode, propname, &string);
+	if (ret)
+		return ret;
+
+	ret = match_string(array, n, string);
+	if (ret < 0)
+		ret = -ENOENT;
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(fwnode_property_match_property_string);
+
 /**
  * fwnode_property_get_reference_args() - Find a reference with arguments
  * @fwnode:	Firmware node where to look for the reference
diff --git a/drivers/base/regmap/internal.h b/drivers/base/regmap/internal.h
index 9a9ea514c2d8..583dd5d7d46b 100644
--- a/drivers/base/regmap/internal.h
+++ b/drivers/base/regmap/internal.h
@@ -318,6 +318,7 @@ struct regmap_ram_data {
 	bool *read;
 	bool *written;
 	enum regmap_endian reg_endian;
+	bool (*noinc_reg)(struct regmap_ram_data *data, unsigned int reg);
 };
 
 /*
diff --git a/drivers/base/regmap/regmap-kunit.c b/drivers/base/regmap/regmap-kunit.c
index e14cc03a17f6..026bdcb45127 100644
--- a/drivers/base/regmap/regmap-kunit.c
+++ b/drivers/base/regmap/regmap-kunit.c
@@ -1186,6 +1186,65 @@ static void raw_write(struct kunit *test)
 	regmap_exit(map);
 }
 
+static bool reg_zero(struct device *dev, unsigned int reg)
+{
+	return reg == 0;
+}
+
+static bool ram_reg_zero(struct regmap_ram_data *data, unsigned int reg)
+{
+	return reg == 0;
+}
+
+static void raw_noinc_write(struct kunit *test)
+{
+	struct raw_test_types *t = (struct raw_test_types *)test->param_value;
+	struct regmap *map;
+	struct regmap_config config;
+	struct regmap_ram_data *data;
+	unsigned int val, val_test, val_last;
+	u16 val_array[BLOCK_TEST_SIZE];
+
+	config = raw_regmap_config;
+	config.volatile_reg = reg_zero;
+	config.writeable_noinc_reg = reg_zero;
+	config.readable_noinc_reg = reg_zero;
+
+	map = gen_raw_regmap(&config, t, &data);
+	KUNIT_ASSERT_FALSE(test, IS_ERR(map));
+	if (IS_ERR(map))
+		return;
+
+	data->noinc_reg = ram_reg_zero;
+
+	get_random_bytes(&val_array, sizeof(val_array));
+
+	if (config.val_format_endian == REGMAP_ENDIAN_BIG) {
+		val_test = be16_to_cpu(val_array[1]) + 100;
+		val_last = be16_to_cpu(val_array[BLOCK_TEST_SIZE - 1]);
+	} else {
+		val_test = le16_to_cpu(val_array[1]) + 100;
+		val_last = le16_to_cpu(val_array[BLOCK_TEST_SIZE - 1]);
+	}
+
+	/* Put some data into the register following the noinc register */
+	KUNIT_EXPECT_EQ(test, 0, regmap_write(map, 1, val_test));
+
+	/* Write some data to the noinc register */
+	KUNIT_EXPECT_EQ(test, 0, regmap_noinc_write(map, 0, val_array,
+						    sizeof(val_array)));
+
+	/* We should read back the last value written */
+	KUNIT_EXPECT_EQ(test, 0, regmap_read(map, 0, &val));
+	KUNIT_ASSERT_EQ(test, val_last, val);
+
+	/* Make sure we didn't touch the register after the noinc register */
+	KUNIT_EXPECT_EQ(test, 0, regmap_read(map, 1, &val));
+	KUNIT_ASSERT_EQ(test, val_test, val);
+
+	regmap_exit(map);
+}
+
 static void raw_sync(struct kunit *test)
 {
 	struct raw_test_types *t = (struct raw_test_types *)test->param_value;
@@ -1284,6 +1343,7 @@ static struct kunit_case regmap_test_cases[] = {
 	KUNIT_CASE_PARAM(raw_read_defaults, raw_test_types_gen_params),
 	KUNIT_CASE_PARAM(raw_write_read_single, raw_test_types_gen_params),
 	KUNIT_CASE_PARAM(raw_write, raw_test_types_gen_params),
+	KUNIT_CASE_PARAM(raw_noinc_write, raw_test_types_gen_params),
 	KUNIT_CASE_PARAM(raw_sync, raw_test_cache_types_gen_params),
 	{}
 };
diff --git a/drivers/base/regmap/regmap-raw-ram.c b/drivers/base/regmap/regmap-raw-ram.c
index c9b800885f3b..463adafa9532 100644
--- a/drivers/base/regmap/regmap-raw-ram.c
+++ b/drivers/base/regmap/regmap-raw-ram.c
@@ -41,10 +41,15 @@ static int regmap_raw_ram_gather_write(void *context,
 		return -EINVAL;
 
 	r = decode_reg(data->reg_endian, reg);
-	memcpy(&our_buf[r], val, val_len);
-
-	for (i = 0; i < val_len / 2; i++)
-		data->written[r + i] = true;
+	if (data->noinc_reg && data->noinc_reg(data, r)) {
+		memcpy(&our_buf[r], val + val_len - 2, 2);
+		data->written[r] = true;
+	} else {
+		memcpy(&our_buf[r], val, val_len);
+
+		for (i = 0; i < val_len / 2; i++)
+			data->written[r + i] = true;
+	}
 	
 	return 0;
 }
@@ -70,10 +75,16 @@ static int regmap_raw_ram_read(void *context,
 		return -EINVAL;
 
 	r = decode_reg(data->reg_endian, reg);
-	memcpy(val, &our_buf[r], val_len);
-
-	for (i = 0; i < val_len / 2; i++)
-		data->read[r + i] = true;
+	if (data->noinc_reg && data->noinc_reg(data, r)) {
+		for (i = 0; i < val_len; i += 2)
+			memcpy(val + i, &our_buf[r], 2);
+		data->read[r] = true;
+	} else {
+		memcpy(val, &our_buf[r], val_len);
+
+		for (i = 0; i < val_len / 2; i++)
+			data->read[r + i] = true;
+	}
 
 	return 0;
 }
diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c
index 855fdf5c3b4e..b6414e1e645b 100644
--- a/drivers/block/nbd.c
+++ b/drivers/block/nbd.c
@@ -67,6 +67,7 @@ struct nbd_sock {
 struct recv_thread_args {
 	struct work_struct work;
 	struct nbd_device *nbd;
+	struct nbd_sock *nsock;
 	int index;
 };
 
@@ -395,6 +396,22 @@ static u32 req_to_nbd_cmd_type(struct request *req)
 	}
 }
 
+static struct nbd_config *nbd_get_config_unlocked(struct nbd_device *nbd)
+{
+	if (refcount_inc_not_zero(&nbd->config_refs)) {
+		/*
+		 * Add smp_mb__after_atomic to ensure that reading nbd->config_refs
+		 * and reading nbd->config is ordered. The pair is the barrier in
+		 * nbd_alloc_and_init_config(), avoid nbd->config_refs is set
+		 * before nbd->config.
+		 */
+		smp_mb__after_atomic();
+		return nbd->config;
+	}
+
+	return NULL;
+}
+
 static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req)
 {
 	struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req);
@@ -409,13 +426,13 @@ static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req)
 		return BLK_EH_DONE;
 	}
 
-	if (!refcount_inc_not_zero(&nbd->config_refs)) {
+	config = nbd_get_config_unlocked(nbd);
+	if (!config) {
 		cmd->status = BLK_STS_TIMEOUT;
 		__clear_bit(NBD_CMD_INFLIGHT, &cmd->flags);
 		mutex_unlock(&cmd->lock);
 		goto done;
 	}
-	config = nbd->config;
 
 	if (config->num_connections > 1 ||
 	    (config->num_connections == 1 && nbd->tag_set.timeout)) {
@@ -489,15 +506,9 @@ static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req)
 	return BLK_EH_DONE;
 }
 
-/*
- *  Send or receive packet. Return a positive value on success and
- *  negtive value on failue, and never return 0.
- */
-static int sock_xmit(struct nbd_device *nbd, int index, int send,
-		     struct iov_iter *iter, int msg_flags, int *sent)
+static int __sock_xmit(struct nbd_device *nbd, struct socket *sock, int send,
+		       struct iov_iter *iter, int msg_flags, int *sent)
 {
-	struct nbd_config *config = nbd->config;
-	struct socket *sock = config->socks[index]->sock;
 	int result;
 	struct msghdr msg;
 	unsigned int noreclaim_flag;
@@ -540,6 +551,19 @@ static int sock_xmit(struct nbd_device *nbd, int index, int send,
 	return result;
 }
 
+/*
+ *  Send or receive packet. Return a positive value on success and
+ *  negtive value on failure, and never return 0.
+ */
+static int sock_xmit(struct nbd_device *nbd, int index, int send,
+		     struct iov_iter *iter, int msg_flags, int *sent)
+{
+	struct nbd_config *config = nbd->config;
+	struct socket *sock = config->socks[index]->sock;
+
+	return __sock_xmit(nbd, sock, send, iter, msg_flags, sent);
+}
+
 /*
  * Different settings for sk->sk_sndtimeo can result in different return values
  * if there is a signal pending when we enter sendmsg, because reasons?
@@ -696,7 +720,7 @@ static int nbd_send_cmd(struct nbd_device *nbd, struct nbd_cmd *cmd, int index)
 	return 0;
 }
 
-static int nbd_read_reply(struct nbd_device *nbd, int index,
+static int nbd_read_reply(struct nbd_device *nbd, struct socket *sock,
 			  struct nbd_reply *reply)
 {
 	struct kvec iov = {.iov_base = reply, .iov_len = sizeof(*reply)};
@@ -705,7 +729,7 @@ static int nbd_read_reply(struct nbd_device *nbd, int index,
 
 	reply->magic = 0;
 	iov_iter_kvec(&to, ITER_DEST, &iov, 1, sizeof(*reply));
-	result = sock_xmit(nbd, index, 0, &to, MSG_WAITALL, NULL);
+	result = __sock_xmit(nbd, sock, 0, &to, MSG_WAITALL, NULL);
 	if (result < 0) {
 		if (!nbd_disconnected(nbd->config))
 			dev_err(disk_to_dev(nbd->disk),
@@ -829,14 +853,14 @@ static void recv_work(struct work_struct *work)
 	struct nbd_device *nbd = args->nbd;
 	struct nbd_config *config = nbd->config;
 	struct request_queue *q = nbd->disk->queue;
-	struct nbd_sock *nsock;
+	struct nbd_sock *nsock = args->nsock;
 	struct nbd_cmd *cmd;
 	struct request *rq;
 
 	while (1) {
 		struct nbd_reply reply;
 
-		if (nbd_read_reply(nbd, args->index, &reply))
+		if (nbd_read_reply(nbd, nsock->sock, &reply))
 			break;
 
 		/*
@@ -871,7 +895,6 @@ static void recv_work(struct work_struct *work)
 		percpu_ref_put(&q->q_usage_counter);
 	}
 
-	nsock = config->socks[args->index];
 	mutex_lock(&nsock->tx_lock);
 	nbd_mark_nsock_dead(nbd, nsock, 1);
 	mutex_unlock(&nsock->tx_lock);
@@ -977,12 +1000,12 @@ static int nbd_handle_cmd(struct nbd_cmd *cmd, int index)
 	struct nbd_sock *nsock;
 	int ret;
 
-	if (!refcount_inc_not_zero(&nbd->config_refs)) {
+	config = nbd_get_config_unlocked(nbd);
+	if (!config) {
 		dev_err_ratelimited(disk_to_dev(nbd->disk),
 				    "Socks array is empty\n");
 		return -EINVAL;
 	}
-	config = nbd->config;
 
 	if (index >= config->num_connections) {
 		dev_err_ratelimited(disk_to_dev(nbd->disk),
@@ -1215,6 +1238,7 @@ static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg)
 		INIT_WORK(&args->work, recv_work);
 		args->index = i;
 		args->nbd = nbd;
+		args->nsock = nsock;
 		nsock->cookie++;
 		mutex_unlock(&nsock->tx_lock);
 		sockfd_put(old);
@@ -1397,6 +1421,7 @@ static int nbd_start_device(struct nbd_device *nbd)
 		refcount_inc(&nbd->config_refs);
 		INIT_WORK(&args->work, recv_work);
 		args->nbd = nbd;
+		args->nsock = config->socks[i];
 		args->index = i;
 		queue_work(nbd->recv_workq, &args->work);
 	}
@@ -1530,17 +1555,20 @@ static int nbd_ioctl(struct block_device *bdev, blk_mode_t mode,
 	return error;
 }
 
-static struct nbd_config *nbd_alloc_config(void)
+static int nbd_alloc_and_init_config(struct nbd_device *nbd)
 {
 	struct nbd_config *config;
 
+	if (WARN_ON(nbd->config))
+		return -EINVAL;
+
 	if (!try_module_get(THIS_MODULE))
-		return ERR_PTR(-ENODEV);
+		return -ENODEV;
 
 	config = kzalloc(sizeof(struct nbd_config), GFP_NOFS);
 	if (!config) {
 		module_put(THIS_MODULE);
-		return ERR_PTR(-ENOMEM);
+		return -ENOMEM;
 	}
 
 	atomic_set(&config->recv_threads, 0);
@@ -1548,12 +1576,24 @@ static struct nbd_config *nbd_alloc_config(void)
 	init_waitqueue_head(&config->conn_wait);
 	config->blksize_bits = NBD_DEF_BLKSIZE_BITS;
 	atomic_set(&config->live_connections, 0);
-	return config;
+
+	nbd->config = config;
+	/*
+	 * Order refcount_set(&nbd->config_refs, 1) and nbd->config assignment,
+	 * its pair is the barrier in nbd_get_config_unlocked().
+	 * So nbd_get_config_unlocked() won't see nbd->config as null after
+	 * refcount_inc_not_zero() succeed.
+	 */
+	smp_mb__before_atomic();
+	refcount_set(&nbd->config_refs, 1);
+
+	return 0;
 }
 
 static int nbd_open(struct gendisk *disk, blk_mode_t mode)
 {
 	struct nbd_device *nbd;
+	struct nbd_config *config;
 	int ret = 0;
 
 	mutex_lock(&nbd_index_mutex);
@@ -1566,27 +1606,25 @@ static int nbd_open(struct gendisk *disk, blk_mode_t mode)
 		ret = -ENXIO;
 		goto out;
 	}
-	if (!refcount_inc_not_zero(&nbd->config_refs)) {
-		struct nbd_config *config;
 
+	config = nbd_get_config_unlocked(nbd);
+	if (!config) {
 		mutex_lock(&nbd->config_lock);
 		if (refcount_inc_not_zero(&nbd->config_refs)) {
 			mutex_unlock(&nbd->config_lock);
 			goto out;
 		}
-		config = nbd_alloc_config();
-		if (IS_ERR(config)) {
-			ret = PTR_ERR(config);
+		ret = nbd_alloc_and_init_config(nbd);
+		if (ret) {
 			mutex_unlock(&nbd->config_lock);
 			goto out;
 		}
-		nbd->config = config;
-		refcount_set(&nbd->config_refs, 1);
+
 		refcount_inc(&nbd->refs);
 		mutex_unlock(&nbd->config_lock);
 		if (max_part)
 			set_bit(GD_NEED_PART_SCAN, &disk->state);
-	} else if (nbd_disconnected(nbd->config)) {
+	} else if (nbd_disconnected(config)) {
 		if (max_part)
 			set_bit(GD_NEED_PART_SCAN, &disk->state);
 	}
@@ -1990,22 +2028,17 @@ static int nbd_genl_connect(struct sk_buff *skb, struct genl_info *info)
 		pr_err("nbd%d already in use\n", index);
 		return -EBUSY;
 	}
-	if (WARN_ON(nbd->config)) {
-		mutex_unlock(&nbd->config_lock);
-		nbd_put(nbd);
-		return -EINVAL;
-	}
-	config = nbd_alloc_config();
-	if (IS_ERR(config)) {
+
+	ret = nbd_alloc_and_init_config(nbd);
+	if (ret) {
 		mutex_unlock(&nbd->config_lock);
 		nbd_put(nbd);
 		pr_err("couldn't allocate config\n");
-		return PTR_ERR(config);
+		return ret;
 	}
-	nbd->config = config;
-	refcount_set(&nbd->config_refs, 1);
-	set_bit(NBD_RT_BOUND, &config->runtime_flags);
 
+	config = nbd->config;
+	set_bit(NBD_RT_BOUND, &config->runtime_flags);
 	ret = nbd_genl_size_set(info, nbd);
 	if (ret)
 		goto out;
@@ -2208,7 +2241,8 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info)
 	}
 	mutex_unlock(&nbd_index_mutex);
 
-	if (!refcount_inc_not_zero(&nbd->config_refs)) {
+	config = nbd_get_config_unlocked(nbd);
+	if (!config) {
 		dev_err(nbd_to_dev(nbd),
 			"not configured, cannot reconfigure\n");
 		nbd_put(nbd);
@@ -2216,7 +2250,6 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info)
 	}
 
 	mutex_lock(&nbd->config_lock);
-	config = nbd->config;
 	if (!test_bit(NBD_RT_BOUND, &config->runtime_flags) ||
 	    !nbd->pid) {
 		dev_err(nbd_to_dev(nbd),
diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c
index 22a3cf7f32e2..3021d58ca51c 100644
--- a/drivers/block/null_blk/main.c
+++ b/drivers/block/null_blk/main.c
@@ -1464,19 +1464,13 @@ blk_status_t null_process_cmd(struct nullb_cmd *cmd, enum req_op op,
 	return BLK_STS_OK;
 }
 
-static blk_status_t null_handle_cmd(struct nullb_cmd *cmd, sector_t sector,
-				    sector_t nr_sectors, enum req_op op)
+static void null_handle_cmd(struct nullb_cmd *cmd, sector_t sector,
+			    sector_t nr_sectors, enum req_op op)
 {
 	struct nullb_device *dev = cmd->nq->dev;
 	struct nullb *nullb = dev->nullb;
 	blk_status_t sts;
 
-	if (test_bit(NULLB_DEV_FL_THROTTLED, &dev->flags)) {
-		sts = null_handle_throttled(cmd);
-		if (sts != BLK_STS_OK)
-			return sts;
-	}
-
 	if (op == REQ_OP_FLUSH) {
 		cmd->error = errno_to_blk_status(null_handle_flush(nullb));
 		goto out;
@@ -1493,7 +1487,6 @@ static blk_status_t null_handle_cmd(struct nullb_cmd *cmd, sector_t sector,
 
 out:
 	nullb_complete_cmd(cmd);
-	return BLK_STS_OK;
 }
 
 static enum hrtimer_restart nullb_bwtimer_fn(struct hrtimer *timer)
@@ -1724,8 +1717,6 @@ static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
 	cmd->fake_timeout = should_timeout_request(rq) ||
 		blk_should_fake_timeout(rq->q);
 
-	blk_mq_start_request(rq);
-
 	if (should_requeue_request(rq)) {
 		/*
 		 * Alternate between hitting the core BUSY path, and the
@@ -1738,6 +1729,15 @@ static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
 		return BLK_STS_OK;
 	}
 
+	if (test_bit(NULLB_DEV_FL_THROTTLED, &nq->dev->flags)) {
+		blk_status_t sts = null_handle_throttled(cmd);
+
+		if (sts != BLK_STS_OK)
+			return sts;
+	}
+
+	blk_mq_start_request(rq);
+
 	if (is_poll) {
 		spin_lock(&nq->poll_lock);
 		list_add_tail(&rq->queuelist, &nq->poll_list);
@@ -1747,7 +1747,8 @@ static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
 	if (cmd->fake_timeout)
 		return BLK_STS_OK;
 
-	return null_handle_cmd(cmd, sector, nr_sectors, req_op(rq));
+	null_handle_cmd(cmd, sector, nr_sectors, req_op(rq));
+	return BLK_STS_OK;
 }
 
 static void null_queue_rqs(struct request **rqlist)
diff --git a/drivers/block/zram/Kconfig b/drivers/block/zram/Kconfig
index 0386b7da02aa..7b29cce60ab2 100644
--- a/drivers/block/zram/Kconfig
+++ b/drivers/block/zram/Kconfig
@@ -59,8 +59,8 @@ config ZRAM_WRITEBACK
        bool "Write back incompressible or idle page to backing device"
        depends on ZRAM
        help
-	 With incompressible page, there is no memory saving to keep it
-	 in memory. Instead, write it out to backing device.
+	 This lets zram entries (incompressible or idle pages) be written
+	 back to a backing device, helping save memory.
 	 For this feature, admin should set up backing device via
 	 /sys/block/zramX/backing_dev.
 
@@ -69,9 +69,18 @@ config ZRAM_WRITEBACK
 
 	 See Documentation/admin-guide/blockdev/zram.rst for more information.
 
+config ZRAM_TRACK_ENTRY_ACTIME
+	bool "Track access time of zram entries"
+	depends on ZRAM
+	help
+	  With this feature zram tracks access time of every stored
+	  entry (page), which can be used for a more fine grained IDLE
+	  pages writeback.
+
 config ZRAM_MEMORY_TRACKING
 	bool "Track zRam block status"
 	depends on ZRAM && DEBUG_FS
+	select ZRAM_TRACK_ENTRY_ACTIME
 	help
 	  With this feature, admin can track the state of allocated blocks
 	  of zRAM. Admin could see the information via
@@ -86,4 +95,4 @@ config ZRAM_MULTI_COMP
 	  This will enable multi-compression streams, so that ZRAM can
 	  re-compress pages using a potentially slower but more effective
 	  compression algorithm. Note, that IDLE page recompression
-	  requires ZRAM_MEMORY_TRACKING.
+	  requires ZRAM_TRACK_ENTRY_ACTIME.
diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c
index d77d3664ca08..f6b286e7f310 100644
--- a/drivers/block/zram/zram_drv.c
+++ b/drivers/block/zram/zram_drv.c
@@ -174,6 +174,14 @@ static inline u32 zram_get_priority(struct zram *zram, u32 index)
 	return prio & ZRAM_COMP_PRIORITY_MASK;
 }
 
+static void zram_accessed(struct zram *zram, u32 index)
+{
+	zram_clear_flag(zram, index, ZRAM_IDLE);
+#ifdef CONFIG_ZRAM_TRACK_ENTRY_ACTIME
+	zram->table[index].ac_time = ktime_get_boottime();
+#endif
+}
+
 static inline void update_used_max(struct zram *zram,
 					const unsigned long pages)
 {
@@ -293,8 +301,9 @@ static void mark_idle(struct zram *zram, ktime_t cutoff)
 		zram_slot_lock(zram, index);
 		if (zram_allocated(zram, index) &&
 				!zram_test_flag(zram, index, ZRAM_UNDER_WB)) {
-#ifdef CONFIG_ZRAM_MEMORY_TRACKING
-			is_idle = !cutoff || ktime_after(cutoff, zram->table[index].ac_time);
+#ifdef CONFIG_ZRAM_TRACK_ENTRY_ACTIME
+			is_idle = !cutoff || ktime_after(cutoff,
+							 zram->table[index].ac_time);
 #endif
 			if (is_idle)
 				zram_set_flag(zram, index, ZRAM_IDLE);
@@ -317,7 +326,7 @@ static ssize_t idle_store(struct device *dev,
 		 */
 		u64 age_sec;
 
-		if (IS_ENABLED(CONFIG_ZRAM_MEMORY_TRACKING) && !kstrtoull(buf, 0, &age_sec))
+		if (IS_ENABLED(CONFIG_ZRAM_TRACK_ENTRY_ACTIME) && !kstrtoull(buf, 0, &age_sec))
 			cutoff_time = ktime_sub(ktime_get_boottime(),
 					ns_to_ktime(age_sec * NSEC_PER_SEC));
 		else
@@ -841,12 +850,6 @@ static void zram_debugfs_destroy(void)
 	debugfs_remove_recursive(zram_debugfs_root);
 }
 
-static void zram_accessed(struct zram *zram, u32 index)
-{
-	zram_clear_flag(zram, index, ZRAM_IDLE);
-	zram->table[index].ac_time = ktime_get_boottime();
-}
-
 static ssize_t read_block_state(struct file *file, char __user *buf,
 				size_t count, loff_t *ppos)
 {
@@ -930,10 +933,6 @@ static void zram_debugfs_unregister(struct zram *zram)
 #else
 static void zram_debugfs_create(void) {};
 static void zram_debugfs_destroy(void) {};
-static void zram_accessed(struct zram *zram, u32 index)
-{
-	zram_clear_flag(zram, index, ZRAM_IDLE);
-};
 static void zram_debugfs_register(struct zram *zram) {};
 static void zram_debugfs_unregister(struct zram *zram) {};
 #endif
@@ -1254,7 +1253,7 @@ static void zram_free_page(struct zram *zram, size_t index)
 {
 	unsigned long handle;
 
-#ifdef CONFIG_ZRAM_MEMORY_TRACKING
+#ifdef CONFIG_ZRAM_TRACK_ENTRY_ACTIME
 	zram->table[index].ac_time = 0;
 #endif
 	if (zram_test_flag(zram, index, ZRAM_IDLE))
diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h
index d090753f97be..3b94d12f41b4 100644
--- a/drivers/block/zram/zram_drv.h
+++ b/drivers/block/zram/zram_drv.h
@@ -69,7 +69,7 @@ struct zram_table_entry {
 		unsigned long element;
 	};
 	unsigned long flags;
-#ifdef CONFIG_ZRAM_MEMORY_TRACKING
+#ifdef CONFIG_ZRAM_TRACK_ENTRY_ACTIME
 	ktime_t ac_time;
 #endif
 };
diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c
index 067e248e3599..35f74f209d1f 100644
--- a/drivers/bluetooth/hci_qca.c
+++ b/drivers/bluetooth/hci_qca.c
@@ -2039,6 +2039,7 @@ static const struct qca_device_data qca_soc_data_wcn3998 __maybe_unused = {
 static const struct qca_device_data qca_soc_data_qca2066 __maybe_unused = {
 	.soc_type = QCA_QCA2066,
 	.num_vregs = 0,
+	.capabilities = QCA_CAP_WIDEBAND_SPEECH | QCA_CAP_VALID_LE_STATES,
 };
 
 static const struct qca_device_data qca_soc_data_qca6390 __maybe_unused = {
diff --git a/drivers/bluetooth/hci_vhci.c b/drivers/bluetooth/hci_vhci.c
index f3892e9ce800..572d68d52965 100644
--- a/drivers/bluetooth/hci_vhci.c
+++ b/drivers/bluetooth/hci_vhci.c
@@ -11,6 +11,7 @@
 #include <linux/module.h>
 #include <asm/unaligned.h>
 
+#include <linux/atomic.h>
 #include <linux/kernel.h>
 #include <linux/init.h>
 #include <linux/slab.h>
@@ -44,6 +45,7 @@ struct vhci_data {
 	bool wakeup;
 	__u16 msft_opcode;
 	bool aosp_capable;
+	atomic_t initialized;
 };
 
 static int vhci_open_dev(struct hci_dev *hdev)
@@ -75,11 +77,10 @@ static int vhci_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
 
 	memcpy(skb_push(skb, 1), &hci_skb_pkt_type(skb), 1);
 
-	mutex_lock(&data->open_mutex);
 	skb_queue_tail(&data->readq, skb);
-	mutex_unlock(&data->open_mutex);
 
-	wake_up_interruptible(&data->read_wait);
+	if (atomic_read(&data->initialized))
+		wake_up_interruptible(&data->read_wait);
 	return 0;
 }
 
@@ -464,7 +465,8 @@ static int __vhci_create_device(struct vhci_data *data, __u8 opcode)
 	skb_put_u8(skb, 0xff);
 	skb_put_u8(skb, opcode);
 	put_unaligned_le16(hdev->id, skb_put(skb, 2));
-	skb_queue_tail(&data->readq, skb);
+	skb_queue_head(&data->readq, skb);
+	atomic_inc(&data->initialized);
 
 	wake_up_interruptible(&data->read_wait);
 	return 0;
diff --git a/drivers/bus/mhi/ep/main.c b/drivers/bus/mhi/ep/main.c
index 600881808982..e2513f5f47a6 100644
--- a/drivers/bus/mhi/ep/main.c
+++ b/drivers/bus/mhi/ep/main.c
@@ -71,45 +71,77 @@ static int mhi_ep_send_event(struct mhi_ep_cntrl *mhi_cntrl, u32 ring_idx,
 static int mhi_ep_send_completion_event(struct mhi_ep_cntrl *mhi_cntrl, struct mhi_ep_ring *ring,
 					struct mhi_ring_element *tre, u32 len, enum mhi_ev_ccs code)
 {
-	struct mhi_ring_element event = {};
+	struct mhi_ring_element *event;
+	int ret;
+
+	event = kzalloc(sizeof(struct mhi_ring_element), GFP_KERNEL);
+	if (!event)
+		return -ENOMEM;
 
-	event.ptr = cpu_to_le64(ring->rbase + ring->rd_offset * sizeof(*tre));
-	event.dword[0] = MHI_TRE_EV_DWORD0(code, len);
-	event.dword[1] = MHI_TRE_EV_DWORD1(ring->ch_id, MHI_PKT_TYPE_TX_EVENT);
+	event->ptr = cpu_to_le64(ring->rbase + ring->rd_offset * sizeof(*tre));
+	event->dword[0] = MHI_TRE_EV_DWORD0(code, len);
+	event->dword[1] = MHI_TRE_EV_DWORD1(ring->ch_id, MHI_PKT_TYPE_TX_EVENT);
 
-	return mhi_ep_send_event(mhi_cntrl, ring->er_index, &event, MHI_TRE_DATA_GET_BEI(tre));
+	ret = mhi_ep_send_event(mhi_cntrl, ring->er_index, event, MHI_TRE_DATA_GET_BEI(tre));
+	kfree(event);
+
+	return ret;
 }
 
 int mhi_ep_send_state_change_event(struct mhi_ep_cntrl *mhi_cntrl, enum mhi_state state)
 {
-	struct mhi_ring_element event = {};
+	struct mhi_ring_element *event;
+	int ret;
+
+	event = kzalloc(sizeof(struct mhi_ring_element), GFP_KERNEL);
+	if (!event)
+		return -ENOMEM;
 
-	event.dword[0] = MHI_SC_EV_DWORD0(state);
-	event.dword[1] = MHI_SC_EV_DWORD1(MHI_PKT_TYPE_STATE_CHANGE_EVENT);
+	event->dword[0] = MHI_SC_EV_DWORD0(state);
+	event->dword[1] = MHI_SC_EV_DWORD1(MHI_PKT_TYPE_STATE_CHANGE_EVENT);
 
-	return mhi_ep_send_event(mhi_cntrl, 0, &event, 0);
+	ret = mhi_ep_send_event(mhi_cntrl, 0, event, 0);
+	kfree(event);
+
+	return ret;
 }
 
 int mhi_ep_send_ee_event(struct mhi_ep_cntrl *mhi_cntrl, enum mhi_ee_type exec_env)
 {
-	struct mhi_ring_element event = {};
+	struct mhi_ring_element *event;
+	int ret;
+
+	event = kzalloc(sizeof(struct mhi_ring_element), GFP_KERNEL);
+	if (!event)
+		return -ENOMEM;
 
-	event.dword[0] = MHI_EE_EV_DWORD0(exec_env);
-	event.dword[1] = MHI_SC_EV_DWORD1(MHI_PKT_TYPE_EE_EVENT);
+	event->dword[0] = MHI_EE_EV_DWORD0(exec_env);
+	event->dword[1] = MHI_SC_EV_DWORD1(MHI_PKT_TYPE_EE_EVENT);
 
-	return mhi_ep_send_event(mhi_cntrl, 0, &event, 0);
+	ret = mhi_ep_send_event(mhi_cntrl, 0, event, 0);
+	kfree(event);
+
+	return ret;
 }
 
 static int mhi_ep_send_cmd_comp_event(struct mhi_ep_cntrl *mhi_cntrl, enum mhi_ev_ccs code)
 {
 	struct mhi_ep_ring *ring = &mhi_cntrl->mhi_cmd->ring;
-	struct mhi_ring_element event = {};
+	struct mhi_ring_element *event;
+	int ret;
+
+	event = kzalloc(sizeof(struct mhi_ring_element), GFP_KERNEL);
+	if (!event)
+		return -ENOMEM;
 
-	event.ptr = cpu_to_le64(ring->rbase + ring->rd_offset * sizeof(struct mhi_ring_element));
-	event.dword[0] = MHI_CC_EV_DWORD0(code);
-	event.dword[1] = MHI_CC_EV_DWORD1(MHI_PKT_TYPE_CMD_COMPLETION_EVENT);
+	event->ptr = cpu_to_le64(ring->rbase + ring->rd_offset * sizeof(struct mhi_ring_element));
+	event->dword[0] = MHI_CC_EV_DWORD0(code);
+	event->dword[1] = MHI_CC_EV_DWORD1(MHI_PKT_TYPE_CMD_COMPLETION_EVENT);
 
-	return mhi_ep_send_event(mhi_cntrl, 0, &event, 0);
+	ret = mhi_ep_send_event(mhi_cntrl, 0, event, 0);
+	kfree(event);
+
+	return ret;
 }
 
 static int mhi_ep_process_cmd_ring(struct mhi_ep_ring *ring, struct mhi_ring_element *el)
diff --git a/drivers/bus/mhi/host/init.c b/drivers/bus/mhi/host/init.c
index f78aefd2d7a3..65ceac1837f9 100644
--- a/drivers/bus/mhi/host/init.c
+++ b/drivers/bus/mhi/host/init.c
@@ -881,6 +881,7 @@ static int parse_config(struct mhi_controller *mhi_cntrl,
 	if (!mhi_cntrl->timeout_ms)
 		mhi_cntrl->timeout_ms = MHI_TIMEOUT_MS;
 
+	mhi_cntrl->ready_timeout_ms = config->ready_timeout_ms;
 	mhi_cntrl->bounce_buf = config->use_bounce_buf;
 	mhi_cntrl->buffer_len = config->buf_len;
 	if (!mhi_cntrl->buffer_len)
diff --git a/drivers/bus/mhi/host/internal.h b/drivers/bus/mhi/host/internal.h
index 2e139e76de4c..30ac415a3000 100644
--- a/drivers/bus/mhi/host/internal.h
+++ b/drivers/bus/mhi/host/internal.h
@@ -321,7 +321,7 @@ int __must_check mhi_read_reg_field(struct mhi_controller *mhi_cntrl,
 				    u32 *out);
 int __must_check mhi_poll_reg_field(struct mhi_controller *mhi_cntrl,
 				    void __iomem *base, u32 offset, u32 mask,
-				    u32 val, u32 delayus);
+				    u32 val, u32 delayus, u32 timeout_ms);
 void mhi_write_reg(struct mhi_controller *mhi_cntrl, void __iomem *base,
 		   u32 offset, u32 val);
 int __must_check mhi_write_reg_field(struct mhi_controller *mhi_cntrl,
diff --git a/drivers/bus/mhi/host/main.c b/drivers/bus/mhi/host/main.c
index dcf627b36e82..d80975f4bba8 100644
--- a/drivers/bus/mhi/host/main.c
+++ b/drivers/bus/mhi/host/main.c
@@ -40,10 +40,11 @@ int __must_check mhi_read_reg_field(struct mhi_controller *mhi_cntrl,
 
 int __must_check mhi_poll_reg_field(struct mhi_controller *mhi_cntrl,
 				    void __iomem *base, u32 offset,
-				    u32 mask, u32 val, u32 delayus)
+				    u32 mask, u32 val, u32 delayus,
+				    u32 timeout_ms)
 {
 	int ret;
-	u32 out, retry = (mhi_cntrl->timeout_ms * 1000) / delayus;
+	u32 out, retry = (timeout_ms * 1000) / delayus;
 
 	while (retry--) {
 		ret = mhi_read_reg_field(mhi_cntrl, base, offset, mask, &out);
@@ -268,7 +269,8 @@ static void mhi_del_ring_element(struct mhi_controller *mhi_cntrl,
 
 static bool is_valid_ring_ptr(struct mhi_ring *ring, dma_addr_t addr)
 {
-	return addr >= ring->iommu_base && addr < ring->iommu_base + ring->len;
+	return addr >= ring->iommu_base && addr < ring->iommu_base + ring->len &&
+			!(addr & (sizeof(struct mhi_ring_element) - 1));
 }
 
 int mhi_destroy_device(struct device *dev, void *data)
diff --git a/drivers/bus/mhi/host/pci_generic.c b/drivers/bus/mhi/host/pci_generic.c
index 08f3f039dbdd..cd6cd14b3d29 100644
--- a/drivers/bus/mhi/host/pci_generic.c
+++ b/drivers/bus/mhi/host/pci_generic.c
@@ -269,6 +269,16 @@ static struct mhi_event_config modem_qcom_v1_mhi_events[] = {
 	MHI_EVENT_CONFIG_HW_DATA(5, 2048, 101)
 };
 
+static const struct mhi_controller_config modem_qcom_v2_mhiv_config = {
+	.max_channels = 128,
+	.timeout_ms = 8000,
+	.ready_timeout_ms = 50000,
+	.num_channels = ARRAY_SIZE(modem_qcom_v1_mhi_channels),
+	.ch_cfg = modem_qcom_v1_mhi_channels,
+	.num_events = ARRAY_SIZE(modem_qcom_v1_mhi_events),
+	.event_cfg = modem_qcom_v1_mhi_events,
+};
+
 static const struct mhi_controller_config modem_qcom_v1_mhiv_config = {
 	.max_channels = 128,
 	.timeout_ms = 8000,
@@ -278,6 +288,16 @@ static const struct mhi_controller_config modem_qcom_v1_mhiv_config = {
 	.event_cfg = modem_qcom_v1_mhi_events,
 };
 
+static const struct mhi_pci_dev_info mhi_qcom_sdx75_info = {
+	.name = "qcom-sdx75m",
+	.fw = "qcom/sdx75m/xbl.elf",
+	.edl = "qcom/sdx75m/edl.mbn",
+	.config = &modem_qcom_v2_mhiv_config,
+	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
+	.dma_data_width = 32,
+	.sideband_wake = false,
+};
+
 static const struct mhi_pci_dev_info mhi_qcom_sdx65_info = {
 	.name = "qcom-sdx65m",
 	.fw = "qcom/sdx65m/xbl.elf",
@@ -600,6 +620,8 @@ static const struct pci_device_id mhi_pci_id_table[] = {
 		.driver_data = (kernel_ulong_t) &mhi_telit_fn990_info },
 	{ PCI_DEVICE(PCI_VENDOR_ID_QCOM, 0x0308),
 		.driver_data = (kernel_ulong_t) &mhi_qcom_sdx65_info },
+	{ PCI_DEVICE(PCI_VENDOR_ID_QCOM, 0x0309),
+		.driver_data = (kernel_ulong_t) &mhi_qcom_sdx75_info },
 	{ PCI_DEVICE(PCI_VENDOR_ID_QUECTEL, 0x1001), /* EM120R-GL (sdx24) */
 		.driver_data = (kernel_ulong_t) &mhi_quectel_em1xx_info },
 	{ PCI_DEVICE(PCI_VENDOR_ID_QUECTEL, 0x1002), /* EM160R-GL (sdx24) */
diff --git a/drivers/bus/mhi/host/pm.c b/drivers/bus/mhi/host/pm.c
index 8a4362d75fc4..a2f2feef1476 100644
--- a/drivers/bus/mhi/host/pm.c
+++ b/drivers/bus/mhi/host/pm.c
@@ -163,6 +163,7 @@ int mhi_ready_state_transition(struct mhi_controller *mhi_cntrl)
 	enum mhi_pm_state cur_state;
 	struct device *dev = &mhi_cntrl->mhi_dev->dev;
 	u32 interval_us = 25000; /* poll register field every 25 milliseconds */
+	u32 timeout_ms;
 	int ret, i;
 
 	/* Check if device entered error state */
@@ -173,14 +174,18 @@ int mhi_ready_state_transition(struct mhi_controller *mhi_cntrl)
 
 	/* Wait for RESET to be cleared and READY bit to be set by the device */
 	ret = mhi_poll_reg_field(mhi_cntrl, mhi_cntrl->regs, MHICTRL,
-				 MHICTRL_RESET_MASK, 0, interval_us);
+				 MHICTRL_RESET_MASK, 0, interval_us,
+				 mhi_cntrl->timeout_ms);
 	if (ret) {
 		dev_err(dev, "Device failed to clear MHI Reset\n");
 		return ret;
 	}
 
+	timeout_ms = mhi_cntrl->ready_timeout_ms ?
+		mhi_cntrl->ready_timeout_ms : mhi_cntrl->timeout_ms;
 	ret = mhi_poll_reg_field(mhi_cntrl, mhi_cntrl->regs, MHISTATUS,
-				 MHISTATUS_READY_MASK, 1, interval_us);
+				 MHISTATUS_READY_MASK, 1, interval_us,
+				 timeout_ms);
 	if (ret) {
 		dev_err(dev, "Device failed to enter MHI Ready\n");
 		return ret;
@@ -479,7 +484,7 @@ static void mhi_pm_disable_transition(struct mhi_controller *mhi_cntrl)
 
 		/* Wait for the reset bit to be cleared by the device */
 		ret = mhi_poll_reg_field(mhi_cntrl, mhi_cntrl->regs, MHICTRL,
-				 MHICTRL_RESET_MASK, 0, 25000);
+				 MHICTRL_RESET_MASK, 0, 25000, mhi_cntrl->timeout_ms);
 		if (ret)
 			dev_err(dev, "Device failed to clear MHI Reset\n");
 
@@ -492,8 +497,8 @@ static void mhi_pm_disable_transition(struct mhi_controller *mhi_cntrl)
 		if (!MHI_IN_PBL(mhi_get_exec_env(mhi_cntrl))) {
 			/* wait for ready to be set */
 			ret = mhi_poll_reg_field(mhi_cntrl, mhi_cntrl->regs,
-						 MHISTATUS,
-						 MHISTATUS_READY_MASK, 1, 25000);
+						 MHISTATUS, MHISTATUS_READY_MASK,
+						 1, 25000, mhi_cntrl->timeout_ms);
 			if (ret)
 				dev_err(dev, "Device failed to enter READY state\n");
 		}
@@ -1111,7 +1116,8 @@ int mhi_async_power_up(struct mhi_controller *mhi_cntrl)
 	if (state == MHI_STATE_SYS_ERR) {
 		mhi_set_mhi_state(mhi_cntrl, MHI_STATE_RESET);
 		ret = mhi_poll_reg_field(mhi_cntrl, mhi_cntrl->regs, MHICTRL,
-				 MHICTRL_RESET_MASK, 0, interval_us);
+				 MHICTRL_RESET_MASK, 0, interval_us,
+				 mhi_cntrl->timeout_ms);
 		if (ret) {
 			dev_info(dev, "Failed to reset MHI due to syserr state\n");
 			goto error_exit;
@@ -1202,14 +1208,18 @@ EXPORT_SYMBOL_GPL(mhi_power_down);
 int mhi_sync_power_up(struct mhi_controller *mhi_cntrl)
 {
 	int ret = mhi_async_power_up(mhi_cntrl);
+	u32 timeout_ms;
 
 	if (ret)
 		return ret;
 
+	/* Some devices need more time to set ready during power up */
+	timeout_ms = mhi_cntrl->ready_timeout_ms ?
+		mhi_cntrl->ready_timeout_ms : mhi_cntrl->timeout_ms;
 	wait_event_timeout(mhi_cntrl->state_event,
 			   MHI_IN_MISSION_MODE(mhi_cntrl->ee) ||
 			   MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state),
-			   msecs_to_jiffies(mhi_cntrl->timeout_ms));
+			   msecs_to_jiffies(timeout_ms));
 
 	ret = (MHI_IN_MISSION_MODE(mhi_cntrl->ee)) ? 0 : -ETIMEDOUT;
 	if (ret)
diff --git a/drivers/char/hw_random/ingenic-rng.c b/drivers/char/hw_random/ingenic-rng.c
index 4f18c3fa5427..c74ded64fbe3 100644
--- a/drivers/char/hw_random/ingenic-rng.c
+++ b/drivers/char/hw_random/ingenic-rng.c
@@ -11,7 +11,7 @@
 #include <linux/io.h>
 #include <linux/iopoll.h>
 #include <linux/module.h>
-#include <linux/of_device.h>
+#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 
diff --git a/drivers/char/hw_random/virtio-rng.c b/drivers/char/hw_random/virtio-rng.c
index e41a84e6b4b5..58d92d62ddfe 100644
--- a/drivers/char/hw_random/virtio-rng.c
+++ b/drivers/char/hw_random/virtio-rng.c
@@ -208,7 +208,6 @@ static void virtrng_scan(struct virtio_device *vdev)
 		vi->hwrng_register_done = true;
 }
 
-#ifdef CONFIG_PM_SLEEP
 static int virtrng_freeze(struct virtio_device *vdev)
 {
 	remove_common(vdev);
@@ -238,7 +237,6 @@ static int virtrng_restore(struct virtio_device *vdev)
 
 	return err;
 }
-#endif
 
 static const struct virtio_device_id id_table[] = {
 	{ VIRTIO_ID_RNG, VIRTIO_DEV_ANY_ID },
@@ -252,10 +250,8 @@ static struct virtio_driver virtio_rng_driver = {
 	.probe =	virtrng_probe,
 	.remove =	virtrng_remove,
 	.scan =		virtrng_scan,
-#ifdef CONFIG_PM_SLEEP
-	.freeze =	virtrng_freeze,
-	.restore =	virtrng_restore,
-#endif
+	.freeze =	pm_sleep_ptr(virtrng_freeze),
+	.restore =	pm_sleep_ptr(virtrng_restore),
 };
 
 module_virtio_driver(virtio_rng_driver);
diff --git a/drivers/char/ipmi/ipmi_si_platform.c b/drivers/char/ipmi/ipmi_si_platform.c
index c3d8ac7873ba..cd2edd8f8a03 100644
--- a/drivers/char/ipmi/ipmi_si_platform.c
+++ b/drivers/char/ipmi/ipmi_si_platform.c
@@ -11,10 +11,11 @@
 
 #include <linux/types.h>
 #include <linux/module.h>
-#include <linux/of_device.h>
-#include <linux/of_platform.h>
+#include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
 #include <linux/acpi.h>
 #include "ipmi_si.h"
 #include "ipmi_dmi.h"
@@ -224,7 +225,6 @@ MODULE_DEVICE_TABLE(of, of_ipmi_match);
 
 static int of_ipmi_probe(struct platform_device *pdev)
 {
-	const struct of_device_id *match;
 	struct si_sm_io io;
 	struct resource resource;
 	const __be32 *regsize, *regspacing, *regshift;
@@ -237,10 +237,6 @@ static int of_ipmi_probe(struct platform_device *pdev)
 
 	dev_info(&pdev->dev, "probing via device tree\n");
 
-	match = of_match_device(of_ipmi_match, &pdev->dev);
-	if (!match)
-		return -ENODEV;
-
 	if (!of_device_is_available(np))
 		return -EINVAL;
 
@@ -269,7 +265,7 @@ static int of_ipmi_probe(struct platform_device *pdev)
 	}
 
 	memset(&io, 0, sizeof(io));
-	io.si_type	= (unsigned long) match->data;
+	io.si_type	= (enum si_type)device_get_match_data(&pdev->dev);
 	io.addr_source	= SI_DEVICETREE;
 	io.irq_setup	= ipmi_std_irq_setup;
 
diff --git a/drivers/clk/renesas/r8a779g0-cpg-mssr.c b/drivers/clk/renesas/r8a779g0-cpg-mssr.c
index 7cc580d67362..5974adcef3ed 100644
--- a/drivers/clk/renesas/r8a779g0-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a779g0-cpg-mssr.c
@@ -192,6 +192,8 @@ static const struct mssr_mod_clk r8a779g0_mod_clks[] __initconst = {
 	DEF_MOD("msi3",		621,	R8A779G0_CLK_MSO),
 	DEF_MOD("msi4",		622,	R8A779G0_CLK_MSO),
 	DEF_MOD("msi5",		623,	R8A779G0_CLK_MSO),
+	DEF_MOD("pciec0",	624,	R8A779G0_CLK_S0D2_HSC),
+	DEF_MOD("pscie1",	625,	R8A779G0_CLK_S0D2_HSC),
 	DEF_MOD("pwm",		628,	R8A779G0_CLK_SASYNCPERD4),
 	DEF_MOD("rpc-if",	629,	R8A779G0_CLK_RPCD2),
 	DEF_MOD("scif0",	702,	R8A779G0_CLK_SASYNCPERD4),
@@ -235,6 +237,7 @@ static const struct mssr_mod_clk r8a779g0_mod_clks[] __initconst = {
 	DEF_MOD("pfc2",		917,	R8A779G0_CLK_CL16M),
 	DEF_MOD("pfc3",		918,	R8A779G0_CLK_CL16M),
 	DEF_MOD("tsc",		919,	R8A779G0_CLK_CL16M),
+	DEF_MOD("tsn",		2723,	R8A779G0_CLK_S0D4_HSC),
 	DEF_MOD("ssiu",		2926,	R8A779G0_CLK_S0D6_PER),
 	DEF_MOD("ssi",		2927,	R8A779G0_CLK_S0D6_PER),
 };
diff --git a/drivers/clk/rockchip/clk-rk3128.c b/drivers/clk/rockchip/clk-rk3128.c
index aa53797dbfc1..7782785a86e6 100644
--- a/drivers/clk/rockchip/clk-rk3128.c
+++ b/drivers/clk/rockchip/clk-rk3128.c
@@ -490,7 +490,7 @@ static struct rockchip_clk_branch common_clk_branches[] __initdata = {
 	GATE(HCLK_I2S_2CH, "hclk_i2s_2ch", "hclk_peri", 0, RK2928_CLKGATE_CON(7), 2, GFLAGS),
 	GATE(0, "hclk_usb_peri", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(9), 13, GFLAGS),
 	GATE(HCLK_HOST2, "hclk_host2", "hclk_peri", 0, RK2928_CLKGATE_CON(7), 3, GFLAGS),
-	GATE(HCLK_OTG, "hclk_otg", "hclk_peri", 0, RK2928_CLKGATE_CON(3), 13, GFLAGS),
+	GATE(HCLK_OTG, "hclk_otg", "hclk_peri", 0, RK2928_CLKGATE_CON(5), 13, GFLAGS),
 	GATE(0, "hclk_peri_ahb", "hclk_peri", CLK_IGNORE_UNUSED, RK2928_CLKGATE_CON(9), 14, GFLAGS),
 	GATE(HCLK_SPDIF, "hclk_spdif", "hclk_peri", 0, RK2928_CLKGATE_CON(10), 9, GFLAGS),
 	GATE(HCLK_TSP, "hclk_tsp", "hclk_peri", 0, RK2928_CLKGATE_CON(10), 12, GFLAGS),
diff --git a/drivers/clk/rockchip/clk-rk3568.c b/drivers/clk/rockchip/clk-rk3568.c
index 16dabe2b9c47..bfbcbb744327 100644
--- a/drivers/clk/rockchip/clk-rk3568.c
+++ b/drivers/clk/rockchip/clk-rk3568.c
@@ -72,12 +72,14 @@ static struct rockchip_pll_rate_table rk3568_pll_rates[] = {
 	RK3036_PLL_RATE(408000000, 1, 68, 2, 2, 1, 0),
 	RK3036_PLL_RATE(312000000, 1, 78, 6, 1, 1, 0),
 	RK3036_PLL_RATE(297000000, 2, 99, 4, 1, 1, 0),
+	RK3036_PLL_RATE(292500000, 1, 195, 4, 4, 1, 0),
 	RK3036_PLL_RATE(241500000, 2, 161, 4, 2, 1, 0),
 	RK3036_PLL_RATE(216000000, 1, 72, 4, 2, 1, 0),
 	RK3036_PLL_RATE(200000000, 1, 100, 3, 4, 1, 0),
 	RK3036_PLL_RATE(148500000, 1, 99, 4, 4, 1, 0),
 	RK3036_PLL_RATE(135000000, 2, 45, 4, 1, 1, 0),
 	RK3036_PLL_RATE(119000000, 3, 119, 4, 2, 1, 0),
+	RK3036_PLL_RATE(115200000, 1, 24, 5, 1, 1, 0),
 	RK3036_PLL_RATE(108000000, 2, 45, 5, 1, 1, 0),
 	RK3036_PLL_RATE(101000000, 1, 101, 6, 4, 1, 0),
 	RK3036_PLL_RATE(100000000, 1, 150, 6, 6, 1, 0),
diff --git a/drivers/clk/samsung/clk-cpu.h b/drivers/clk/samsung/clk-cpu.h
index fc9f67a3b22e..0164bd9ad021 100644
--- a/drivers/clk/samsung/clk-cpu.h
+++ b/drivers/clk/samsung/clk-cpu.h
@@ -11,10 +11,10 @@
 #include "clk.h"
 
 /**
- * struct exynos_cpuclk_data: config data to setup cpu clocks.
- * @prate: frequency of the primary parent clock (in KHz).
- * @div0: value to be programmed in the div_cpu0 register.
- * @div1: value to be programmed in the div_cpu1 register.
+ * struct exynos_cpuclk_cfg_data - config data to setup cpu clocks
+ * @prate: frequency of the primary parent clock (in KHz)
+ * @div0: value to be programmed in the div_cpu0 register
+ * @div1: value to be programmed in the div_cpu1 register
  *
  * This structure holds the divider configuration data for dividers in the CPU
  * clock domain. The parent frequency at which these divider values are valid is
@@ -29,17 +29,17 @@ struct exynos_cpuclk_cfg_data {
 };
 
 /**
- * struct exynos_cpuclk: information about clock supplied to a CPU core.
- * @hw:	handle between CCF and CPU clock.
- * @alt_parent: alternate parent clock to use when switching the speed
- *	of the primary parent clock.
- * @ctrl_base:	base address of the clock controller.
- * @lock: cpu clock domain register access lock.
- * @cfg: cpu clock rate configuration data.
- * @num_cfgs: number of array elements in @cfg array.
- * @clk_nb: clock notifier registered for changes in clock speed of the
- *	primary parent clock.
- * @flags: configuration flags for the CPU clock.
+ * struct exynos_cpuclk - information about clock supplied to a CPU core
+ * @hw:		handle between CCF and CPU clock
+ * @alt_parent:	alternate parent clock to use when switching the speed
+ *		of the primary parent clock
+ * @ctrl_base:	base address of the clock controller
+ * @lock:	cpu clock domain register access lock
+ * @cfg:	cpu clock rate configuration data
+ * @num_cfgs:	number of array elements in @cfg array
+ * @clk_nb:	clock notifier registered for changes in clock speed of the
+ *		primary parent clock
+ * @flags:	configuration flags for the CPU clock
  *
  * This structure holds information required for programming the CPU clock for
  * various clock speeds.
diff --git a/drivers/clk/samsung/clk.h b/drivers/clk/samsung/clk.h
index ab9c3d7a25b3..516b716407e5 100644
--- a/drivers/clk/samsung/clk.h
+++ b/drivers/clk/samsung/clk.h
@@ -14,11 +14,11 @@
 #include "clk-pll.h"
 
 /**
- * struct samsung_clk_provider: information about clock provider
- * @reg_base: virtual address for the register base.
- * @dev: clock provider device needed for runtime PM.
- * @lock: maintains exclusion between callbacks for a given clock-provider.
- * @clk_data: holds clock related data like clk_hw* and number of clocks.
+ * struct samsung_clk_provider - information about clock provider
+ * @reg_base: virtual address for the register base
+ * @dev: clock provider device needed for runtime PM
+ * @lock: maintains exclusion between callbacks for a given clock-provider
+ * @clk_data: holds clock related data like clk_hw* and number of clocks
  */
 struct samsung_clk_provider {
 	void __iomem *reg_base;
@@ -29,10 +29,10 @@ struct samsung_clk_provider {
 };
 
 /**
- * struct samsung_clock_alias: information about mux clock
- * @id: platform specific id of the clock.
- * @dev_name: name of the device to which this clock belongs.
- * @alias: optional clock alias name to be assigned to this clock.
+ * struct samsung_clock_alias - information about mux clock
+ * @id: platform specific id of the clock
+ * @dev_name: name of the device to which this clock belongs
+ * @alias: optional clock alias name to be assigned to this clock
  */
 struct samsung_clock_alias {
 	unsigned int		id;
@@ -50,12 +50,12 @@ struct samsung_clock_alias {
 #define MHZ (1000 * 1000)
 
 /**
- * struct samsung_fixed_rate_clock: information about fixed-rate clock
- * @id: platform specific id of the clock.
- * @name: name of this fixed-rate clock.
- * @parent_name: optional parent clock name.
- * @flags: optional fixed-rate clock flags.
- * @fixed-rate: fixed clock rate of this clock.
+ * struct samsung_fixed_rate_clock - information about fixed-rate clock
+ * @id: platform specific id of the clock
+ * @name: name of this fixed-rate clock
+ * @parent_name: optional parent clock name
+ * @flags: optional fixed-rate clock flags
+ * @fixed_rate: fixed clock rate of this clock
  */
 struct samsung_fixed_rate_clock {
 	unsigned int		id;
@@ -74,14 +74,14 @@ struct samsung_fixed_rate_clock {
 		.fixed_rate	= frate,		\
 	}
 
-/*
- * struct samsung_fixed_factor_clock: information about fixed-factor clock
- * @id: platform specific id of the clock.
- * @name: name of this fixed-factor clock.
- * @parent_name: parent clock name.
- * @mult: fixed multiplication factor.
- * @div: fixed division factor.
- * @flags: optional fixed-factor clock flags.
+/**
+ * struct samsung_fixed_factor_clock - information about fixed-factor clock
+ * @id: platform specific id of the clock
+ * @name: name of this fixed-factor clock
+ * @parent_name: parent clock name
+ * @mult: fixed multiplication factor
+ * @div: fixed division factor
+ * @flags: optional fixed-factor clock flags
  */
 struct samsung_fixed_factor_clock {
 	unsigned int		id;
@@ -103,16 +103,16 @@ struct samsung_fixed_factor_clock {
 	}
 
 /**
- * struct samsung_mux_clock: information about mux clock
- * @id: platform specific id of the clock.
- * @name: name of this mux clock.
- * @parent_names: array of pointer to parent clock names.
- * @num_parents: number of parents listed in @parent_names.
- * @flags: optional flags for basic clock.
- * @offset: offset of the register for configuring the mux.
- * @shift: starting bit location of the mux control bit-field in @reg.
- * @width: width of the mux control bit-field in @reg.
- * @mux_flags: flags for mux-type clock.
+ * struct samsung_mux_clock - information about mux clock
+ * @id: platform specific id of the clock
+ * @name: name of this mux clock
+ * @parent_names: array of pointer to parent clock names
+ * @num_parents: number of parents listed in @parent_names
+ * @flags: optional flags for basic clock
+ * @offset: offset of the register for configuring the mux
+ * @shift: starting bit location of the mux control bit-field in @reg
+ * @width: width of the mux control bit-field in @reg
+ * @mux_flags: flags for mux-type clock
  */
 struct samsung_mux_clock {
 	unsigned int		id;
@@ -146,14 +146,16 @@ struct samsung_mux_clock {
 	__MUX(_id, cname, pnames, o, s, w, f, mf)
 
 /**
- * @id: platform specific id of the clock.
- * struct samsung_div_clock: information about div clock
- * @name: name of this div clock.
- * @parent_name: name of the parent clock.
- * @flags: optional flags for basic clock.
- * @offset: offset of the register for configuring the div.
- * @shift: starting bit location of the div control bit-field in @reg.
- * @div_flags: flags for div-type clock.
+ * struct samsung_div_clock - information about div clock
+ * @id: platform specific id of the clock
+ * @name: name of this div clock
+ * @parent_name: name of the parent clock
+ * @flags: optional flags for basic clock
+ * @offset: offset of the register for configuring the div
+ * @shift: starting bit location of the div control bit-field in @reg
+ * @width: width of the bitfield
+ * @div_flags: flags for div-type clock
+ * @table: array of divider/value pairs ending with a div set to 0
  */
 struct samsung_div_clock {
 	unsigned int		id;
@@ -190,14 +192,14 @@ struct samsung_div_clock {
 	__DIV(_id, cname, pname, o, s, w, 0, 0, t)
 
 /**
- * struct samsung_gate_clock: information about gate clock
- * @id: platform specific id of the clock.
- * @name: name of this gate clock.
- * @parent_name: name of the parent clock.
- * @flags: optional flags for basic clock.
- * @offset: offset of the register for configuring the gate.
- * @bit_idx: bit index of the gate control bit-field in @reg.
- * @gate_flags: flags for gate-type clock.
+ * struct samsung_gate_clock - information about gate clock
+ * @id: platform specific id of the clock
+ * @name: name of this gate clock
+ * @parent_name: name of the parent clock
+ * @flags: optional flags for basic clock
+ * @offset: offset of the register for configuring the gate
+ * @bit_idx: bit index of the gate control bit-field in @reg
+ * @gate_flags: flags for gate-type clock
  */
 struct samsung_gate_clock {
 	unsigned int		id;
@@ -226,9 +228,9 @@ struct samsung_gate_clock {
 #define PNAME(x) static const char * const x[] __initconst
 
 /**
- * struct samsung_clk_reg_dump: register dump of clock controller registers.
- * @offset: clock register offset from the controller base address.
- * @value: the value to be register at offset.
+ * struct samsung_clk_reg_dump - register dump of clock controller registers
+ * @offset: clock register offset from the controller base address
+ * @value: the value to be register at offset
  */
 struct samsung_clk_reg_dump {
 	u32	offset;
@@ -236,14 +238,15 @@ struct samsung_clk_reg_dump {
 };
 
 /**
- * struct samsung_pll_clock: information about pll clock
- * @id: platform specific id of the clock.
- * @name: name of this pll clock.
- * @parent_name: name of the parent clock.
- * @flags: optional flags for basic clock.
- * @con_offset: offset of the register for configuring the PLL.
- * @lock_offset: offset of the register for locking the PLL.
- * @type: Type of PLL to be registered.
+ * struct samsung_pll_clock - information about pll clock
+ * @id: platform specific id of the clock
+ * @name: name of this pll clock
+ * @parent_name: name of the parent clock
+ * @flags: optional flags for basic clock
+ * @con_offset: offset of the register for configuring the PLL
+ * @lock_offset: offset of the register for locking the PLL
+ * @type: type of PLL to be registered
+ * @rate_table: array of PLL settings for possible PLL rates
  */
 struct samsung_pll_clock {
 	unsigned int		id;
@@ -302,39 +305,51 @@ struct samsung_clock_reg_cache {
 	unsigned int rsuspend_num;
 };
 
+/**
+ * struct samsung_cmu_info - all clocks information needed for CMU registration
+ * @pll_clks: list of PLL clocks
+ * @nr_pll_clks: count of clocks in @pll_clks
+ * @mux_clks: list of mux clocks
+ * @nr_mux_clks: count of clocks in @mux_clks
+ * @div_clks: list of div clocks
+ * @nr_div_clks: count of clocks in @div_clks
+ * @gate_clks: list of gate clocks
+ * @nr_gate_clks: count of clocks in @gate_clks
+ * @fixed_clks: list of fixed clocks
+ * @nr_fixed_clks: count clocks in @fixed_clks
+ * @fixed_factor_clks: list of fixed factor clocks
+ * @nr_fixed_factor_clks: count of clocks in @fixed_factor_clks
+ * @nr_clk_ids: total number of clocks with IDs assigned
+ * @cpu_clks: list of CPU clocks
+ * @nr_cpu_clks: count of clocks in @cpu_clks
+ * @clk_regs: list of clock registers
+ * @nr_clk_regs: count of clock registers in @clk_regs
+ * @suspend_regs: list of clock registers to set before suspend
+ * @nr_suspend_regs: count of clock registers in @suspend_regs
+ * @clk_name: name of the parent clock needed for CMU register access
+ */
 struct samsung_cmu_info {
-	/* list of pll clocks and respective count */
 	const struct samsung_pll_clock *pll_clks;
 	unsigned int nr_pll_clks;
-	/* list of mux clocks and respective count */
 	const struct samsung_mux_clock *mux_clks;
 	unsigned int nr_mux_clks;
-	/* list of div clocks and respective count */
 	const struct samsung_div_clock *div_clks;
 	unsigned int nr_div_clks;
-	/* list of gate clocks and respective count */
 	const struct samsung_gate_clock *gate_clks;
 	unsigned int nr_gate_clks;
-	/* list of fixed clocks and respective count */
 	const struct samsung_fixed_rate_clock *fixed_clks;
 	unsigned int nr_fixed_clks;
-	/* list of fixed factor clocks and respective count */
 	const struct samsung_fixed_factor_clock *fixed_factor_clks;
 	unsigned int nr_fixed_factor_clks;
-	/* total number of clocks with IDs assigned*/
 	unsigned int nr_clk_ids;
-	/* list of cpu clocks and respective count */
 	const struct samsung_cpu_clock *cpu_clks;
 	unsigned int nr_cpu_clks;
 
-	/* list and number of clocks registers */
 	const unsigned long *clk_regs;
 	unsigned int nr_clk_regs;
 
-	/* list and number of clocks registers to set before suspend */
 	const struct samsung_clk_reg_dump *suspend_regs;
 	unsigned int nr_suspend_regs;
-	/* name of the parent clock needed for CMU register access */
 	const char *clk_name;
 };
 
diff --git a/drivers/clk/sunxi-ng/ccu_nkm.c b/drivers/clk/sunxi-ng/ccu_nkm.c
index eed64547ad42..853f84398e2b 100644
--- a/drivers/clk/sunxi-ng/ccu_nkm.c
+++ b/drivers/clk/sunxi-ng/ccu_nkm.c
@@ -21,17 +21,16 @@ static unsigned long ccu_nkm_find_best_with_parent_adj(struct ccu_common *common
 						       unsigned long *parent, unsigned long rate,
 						       struct _ccu_nkm *nkm)
 {
-	unsigned long best_rate = 0, best_parent_rate = *parent, tmp_parent = *parent;
+	unsigned long best_rate = 0, best_parent_rate = *parent;
 	unsigned long best_n = 0, best_k = 0, best_m = 0;
 	unsigned long _n, _k, _m;
 
 	for (_k = nkm->min_k; _k <= nkm->max_k; _k++) {
 		for (_n = nkm->min_n; _n <= nkm->max_n; _n++) {
 			for (_m = nkm->min_m; _m <= nkm->max_m; _m++) {
-				unsigned long tmp_rate;
+				unsigned long tmp_rate, tmp_parent;
 
 				tmp_parent = clk_hw_round_rate(parent_hw, rate * _m / (_n * _k));
-
 				tmp_rate = tmp_parent * _n * _k / _m;
 
 				if (ccu_is_better_rate(common, rate, tmp_rate, best_rate) ||
diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c
index d70b105dcfa1..753f67a36dc5 100644
--- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c
+++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c
@@ -30,33 +30,16 @@ static int sun8i_ss_hashkey(struct sun8i_ss_hash_tfm_ctx *tfmctx, const u8 *key,
 			    unsigned int keylen)
 {
 	struct crypto_shash *xtfm;
-	struct shash_desc *sdesc;
-	size_t len;
-	int ret = 0;
+	int ret;
 
 	xtfm = crypto_alloc_shash("sha1", 0, CRYPTO_ALG_NEED_FALLBACK);
 	if (IS_ERR(xtfm))
 		return PTR_ERR(xtfm);
 
-	len = sizeof(*sdesc) + crypto_shash_descsize(xtfm);
-	sdesc = kmalloc(len, GFP_KERNEL);
-	if (!sdesc) {
-		ret = -ENOMEM;
-		goto err_hashkey_sdesc;
-	}
-	sdesc->tfm = xtfm;
-
-	ret = crypto_shash_init(sdesc);
-	if (ret) {
-		dev_err(tfmctx->ss->dev, "shash init error ret=%d\n", ret);
-		goto err_hashkey;
-	}
-	ret = crypto_shash_finup(sdesc, key, keylen, tfmctx->key);
+	ret = crypto_shash_tfm_digest(xtfm, key, keylen, tfmctx->key);
 	if (ret)
-		dev_err(tfmctx->ss->dev, "shash finup error\n");
-err_hashkey:
-	kfree(sdesc);
-err_hashkey_sdesc:
+		dev_err(tfmctx->ss->dev, "shash digest error ret=%d\n", ret);
+
 	crypto_free_shash(xtfm);
 	return ret;
 }
diff --git a/drivers/crypto/ccree/cc_aead.c b/drivers/crypto/ccree/cc_aead.c
index 109ffb375fc6..5ef39d682389 100644
--- a/drivers/crypto/ccree/cc_aead.c
+++ b/drivers/crypto/ccree/cc_aead.c
@@ -2569,9 +2569,13 @@ static struct cc_crypto_alg *cc_create_aead_alg(struct cc_alg_template *tmpl,
 
 	alg = &tmpl->template_aead;
 
-	snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
-	snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
-		 tmpl->driver_name);
+	if (snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s",
+		     tmpl->name) >= CRYPTO_MAX_ALG_NAME)
+		return ERR_PTR(-EINVAL);
+	if (snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
+		     tmpl->driver_name) >= CRYPTO_MAX_ALG_NAME)
+		return ERR_PTR(-EINVAL);
+
 	alg->base.cra_module = THIS_MODULE;
 	alg->base.cra_priority = CC_CRA_PRIO;
 
diff --git a/drivers/crypto/ccree/cc_cipher.c b/drivers/crypto/ccree/cc_cipher.c
index 2cd44d7457a4..d229e54fa935 100644
--- a/drivers/crypto/ccree/cc_cipher.c
+++ b/drivers/crypto/ccree/cc_cipher.c
@@ -1427,9 +1427,13 @@ static struct cc_crypto_alg *cc_create_alg(const struct cc_alg_template *tmpl,
 
 	memcpy(alg, &tmpl->template_skcipher, sizeof(*alg));
 
-	snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
-	snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
-		 tmpl->driver_name);
+	if (snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s",
+		     tmpl->name) >= CRYPTO_MAX_ALG_NAME)
+		return ERR_PTR(-EINVAL);
+	if (snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
+		     tmpl->driver_name) >= CRYPTO_MAX_ALG_NAME)
+		return ERR_PTR(-EINVAL);
+
 	alg->base.cra_module = THIS_MODULE;
 	alg->base.cra_priority = CC_CRA_PRIO;
 	alg->base.cra_blocksize = tmpl->blocksize;
diff --git a/drivers/crypto/hisilicon/debugfs.c b/drivers/crypto/hisilicon/debugfs.c
index 7e8186fe0512..415139b4abc1 100644
--- a/drivers/crypto/hisilicon/debugfs.c
+++ b/drivers/crypto/hisilicon/debugfs.c
@@ -53,34 +53,34 @@ static struct qm_dfx_item qm_dfx_files[] = {
 #define CNT_CYC_REGS_NUM		10
 static const struct debugfs_reg32 qm_dfx_regs[] = {
 	/* XXX_CNT are reading clear register */
-	{"QM_ECC_1BIT_CNT               ",  0x104000ull},
-	{"QM_ECC_MBIT_CNT               ",  0x104008ull},
-	{"QM_DFX_MB_CNT                 ",  0x104018ull},
-	{"QM_DFX_DB_CNT                 ",  0x104028ull},
-	{"QM_DFX_SQE_CNT                ",  0x104038ull},
-	{"QM_DFX_CQE_CNT                ",  0x104048ull},
-	{"QM_DFX_SEND_SQE_TO_ACC_CNT    ",  0x104050ull},
-	{"QM_DFX_WB_SQE_FROM_ACC_CNT    ",  0x104058ull},
-	{"QM_DFX_ACC_FINISH_CNT         ",  0x104060ull},
-	{"QM_DFX_CQE_ERR_CNT            ",  0x1040b4ull},
-	{"QM_DFX_FUNS_ACTIVE_ST         ",  0x200ull},
-	{"QM_ECC_1BIT_INF               ",  0x104004ull},
-	{"QM_ECC_MBIT_INF               ",  0x10400cull},
-	{"QM_DFX_ACC_RDY_VLD0           ",  0x1040a0ull},
-	{"QM_DFX_ACC_RDY_VLD1           ",  0x1040a4ull},
-	{"QM_DFX_AXI_RDY_VLD            ",  0x1040a8ull},
-	{"QM_DFX_FF_ST0                 ",  0x1040c8ull},
-	{"QM_DFX_FF_ST1                 ",  0x1040ccull},
-	{"QM_DFX_FF_ST2                 ",  0x1040d0ull},
-	{"QM_DFX_FF_ST3                 ",  0x1040d4ull},
-	{"QM_DFX_FF_ST4                 ",  0x1040d8ull},
-	{"QM_DFX_FF_ST5                 ",  0x1040dcull},
-	{"QM_DFX_FF_ST6                 ",  0x1040e0ull},
-	{"QM_IN_IDLE_ST                 ",  0x1040e4ull},
+	{"QM_ECC_1BIT_CNT               ",  0x104000},
+	{"QM_ECC_MBIT_CNT               ",  0x104008},
+	{"QM_DFX_MB_CNT                 ",  0x104018},
+	{"QM_DFX_DB_CNT                 ",  0x104028},
+	{"QM_DFX_SQE_CNT                ",  0x104038},
+	{"QM_DFX_CQE_CNT                ",  0x104048},
+	{"QM_DFX_SEND_SQE_TO_ACC_CNT    ",  0x104050},
+	{"QM_DFX_WB_SQE_FROM_ACC_CNT    ",  0x104058},
+	{"QM_DFX_ACC_FINISH_CNT         ",  0x104060},
+	{"QM_DFX_CQE_ERR_CNT            ",  0x1040b4},
+	{"QM_DFX_FUNS_ACTIVE_ST         ",  0x200},
+	{"QM_ECC_1BIT_INF               ",  0x104004},
+	{"QM_ECC_MBIT_INF               ",  0x10400c},
+	{"QM_DFX_ACC_RDY_VLD0           ",  0x1040a0},
+	{"QM_DFX_ACC_RDY_VLD1           ",  0x1040a4},
+	{"QM_DFX_AXI_RDY_VLD            ",  0x1040a8},
+	{"QM_DFX_FF_ST0                 ",  0x1040c8},
+	{"QM_DFX_FF_ST1                 ",  0x1040cc},
+	{"QM_DFX_FF_ST2                 ",  0x1040d0},
+	{"QM_DFX_FF_ST3                 ",  0x1040d4},
+	{"QM_DFX_FF_ST4                 ",  0x1040d8},
+	{"QM_DFX_FF_ST5                 ",  0x1040dc},
+	{"QM_DFX_FF_ST6                 ",  0x1040e0},
+	{"QM_IN_IDLE_ST                 ",  0x1040e4},
 };
 
 static const struct debugfs_reg32 qm_vf_dfx_regs[] = {
-	{"QM_DFX_FUNS_ACTIVE_ST         ",  0x200ull},
+	{"QM_DFX_FUNS_ACTIVE_ST         ",  0x200},
 };
 
 /* define the QM's dfx regs region and region length */
diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c
index 18599f3634c3..4d91a249be74 100644
--- a/drivers/crypto/hisilicon/qm.c
+++ b/drivers/crypto/hisilicon/qm.c
@@ -129,16 +129,21 @@
 #define QM_FIFO_OVERFLOW_TYPE		0xc0
 #define QM_FIFO_OVERFLOW_TYPE_SHIFT	6
 #define QM_FIFO_OVERFLOW_VF		0x3f
+#define QM_FIFO_OVERFLOW_QP_SHIFT	16
 #define QM_ABNORMAL_INF01		0x100014
 #define QM_DB_TIMEOUT_TYPE		0xc0
 #define QM_DB_TIMEOUT_TYPE_SHIFT	6
 #define QM_DB_TIMEOUT_VF		0x3f
+#define QM_DB_TIMEOUT_QP_SHIFT		16
+#define QM_ABNORMAL_INF02		0x100018
+#define QM_AXI_POISON_ERR		BIT(22)
 #define QM_RAS_CE_ENABLE		0x1000ec
 #define QM_RAS_FE_ENABLE		0x1000f0
 #define QM_RAS_NFE_ENABLE		0x1000f4
 #define QM_RAS_CE_THRESHOLD		0x1000f8
 #define QM_RAS_CE_TIMES_PER_IRQ		1
 #define QM_OOO_SHUTDOWN_SEL		0x1040f8
+#define QM_AXI_RRESP_ERR		BIT(0)
 #define QM_ECC_MBIT			BIT(2)
 #define QM_DB_TIMEOUT			BIT(10)
 #define QM_OF_FIFO_OF			BIT(11)
@@ -1406,7 +1411,7 @@ static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status)
 {
 	const struct hisi_qm_hw_error *err;
 	struct device *dev = &qm->pdev->dev;
-	u32 reg_val, type, vf_num;
+	u32 reg_val, type, vf_num, qp_id;
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(qm_hw_error); i++) {
@@ -1422,19 +1427,24 @@ static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status)
 			type = (reg_val & QM_DB_TIMEOUT_TYPE) >>
 			       QM_DB_TIMEOUT_TYPE_SHIFT;
 			vf_num = reg_val & QM_DB_TIMEOUT_VF;
-			dev_err(dev, "qm %s doorbell timeout in function %u\n",
-				qm_db_timeout[type], vf_num);
+			qp_id = reg_val >> QM_DB_TIMEOUT_QP_SHIFT;
+			dev_err(dev, "qm %s doorbell timeout in function %u qp %u\n",
+				qm_db_timeout[type], vf_num, qp_id);
 		} else if (err->int_msk & QM_OF_FIFO_OF) {
 			reg_val = readl(qm->io_base + QM_ABNORMAL_INF00);
 			type = (reg_val & QM_FIFO_OVERFLOW_TYPE) >>
 			       QM_FIFO_OVERFLOW_TYPE_SHIFT;
 			vf_num = reg_val & QM_FIFO_OVERFLOW_VF;
-
+			qp_id = reg_val >> QM_FIFO_OVERFLOW_QP_SHIFT;
 			if (type < ARRAY_SIZE(qm_fifo_overflow))
-				dev_err(dev, "qm %s fifo overflow in function %u\n",
-					qm_fifo_overflow[type], vf_num);
+				dev_err(dev, "qm %s fifo overflow in function %u qp %u\n",
+					qm_fifo_overflow[type], vf_num, qp_id);
 			else
 				dev_err(dev, "unknown error type\n");
+		} else if (err->int_msk & QM_AXI_RRESP_ERR) {
+			reg_val = readl(qm->io_base + QM_ABNORMAL_INF02);
+			if (reg_val & QM_AXI_POISON_ERR)
+				dev_err(dev, "qm axi poison error happened\n");
 		}
 	}
 }
diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c
index db4c964cd649..d6672b777efc 100644
--- a/drivers/crypto/hisilicon/zip/zip_main.c
+++ b/drivers/crypto/hisilicon/zip/zip_main.c
@@ -270,28 +270,28 @@ static const u64 core_offsets[] = {
 };
 
 static const struct debugfs_reg32 hzip_dfx_regs[] = {
-	{"HZIP_GET_BD_NUM                ",  0x00ull},
-	{"HZIP_GET_RIGHT_BD              ",  0x04ull},
-	{"HZIP_GET_ERROR_BD              ",  0x08ull},
-	{"HZIP_DONE_BD_NUM               ",  0x0cull},
-	{"HZIP_WORK_CYCLE                ",  0x10ull},
-	{"HZIP_IDLE_CYCLE                ",  0x18ull},
-	{"HZIP_MAX_DELAY                 ",  0x20ull},
-	{"HZIP_MIN_DELAY                 ",  0x24ull},
-	{"HZIP_AVG_DELAY                 ",  0x28ull},
-	{"HZIP_MEM_VISIBLE_DATA          ",  0x30ull},
-	{"HZIP_MEM_VISIBLE_ADDR          ",  0x34ull},
-	{"HZIP_CONSUMED_BYTE             ",  0x38ull},
-	{"HZIP_PRODUCED_BYTE             ",  0x40ull},
-	{"HZIP_COMP_INF                  ",  0x70ull},
-	{"HZIP_PRE_OUT                   ",  0x78ull},
-	{"HZIP_BD_RD                     ",  0x7cull},
-	{"HZIP_BD_WR                     ",  0x80ull},
-	{"HZIP_GET_BD_AXI_ERR_NUM        ",  0x84ull},
-	{"HZIP_GET_BD_PARSE_ERR_NUM      ",  0x88ull},
-	{"HZIP_ADD_BD_AXI_ERR_NUM        ",  0x8cull},
-	{"HZIP_DECOMP_STF_RELOAD_CURR_ST ",  0x94ull},
-	{"HZIP_DECOMP_LZ77_CURR_ST       ",  0x9cull},
+	{"HZIP_GET_BD_NUM                ",  0x00},
+	{"HZIP_GET_RIGHT_BD              ",  0x04},
+	{"HZIP_GET_ERROR_BD              ",  0x08},
+	{"HZIP_DONE_BD_NUM               ",  0x0c},
+	{"HZIP_WORK_CYCLE                ",  0x10},
+	{"HZIP_IDLE_CYCLE                ",  0x18},
+	{"HZIP_MAX_DELAY                 ",  0x20},
+	{"HZIP_MIN_DELAY                 ",  0x24},
+	{"HZIP_AVG_DELAY                 ",  0x28},
+	{"HZIP_MEM_VISIBLE_DATA          ",  0x30},
+	{"HZIP_MEM_VISIBLE_ADDR          ",  0x34},
+	{"HZIP_CONSUMED_BYTE             ",  0x38},
+	{"HZIP_PRODUCED_BYTE             ",  0x40},
+	{"HZIP_COMP_INF                  ",  0x70},
+	{"HZIP_PRE_OUT                   ",  0x78},
+	{"HZIP_BD_RD                     ",  0x7c},
+	{"HZIP_BD_WR                     ",  0x80},
+	{"HZIP_GET_BD_AXI_ERR_NUM        ",  0x84},
+	{"HZIP_GET_BD_PARSE_ERR_NUM      ",  0x88},
+	{"HZIP_ADD_BD_AXI_ERR_NUM        ",  0x8c},
+	{"HZIP_DECOMP_STF_RELOAD_CURR_ST ",  0x94},
+	{"HZIP_DECOMP_LZ77_CURR_ST       ",  0x9c},
 };
 
 static const struct debugfs_reg32 hzip_com_dfx_regs[] = {
@@ -303,11 +303,11 @@ static const struct debugfs_reg32 hzip_com_dfx_regs[] = {
 };
 
 static const struct debugfs_reg32 hzip_dump_dfx_regs[] = {
-	{"HZIP_GET_BD_NUM                ",  0x00ull},
-	{"HZIP_GET_RIGHT_BD              ",  0x04ull},
-	{"HZIP_GET_ERROR_BD              ",  0x08ull},
-	{"HZIP_DONE_BD_NUM               ",  0x0cull},
-	{"HZIP_MAX_DELAY                 ",  0x20ull},
+	{"HZIP_GET_BD_NUM                ",  0x00},
+	{"HZIP_GET_RIGHT_BD              ",  0x04},
+	{"HZIP_GET_ERROR_BD              ",  0x08},
+	{"HZIP_DONE_BD_NUM               ",  0x0c},
+	{"HZIP_MAX_DELAY                 ",  0x20},
 };
 
 /* define the ZIP's dfx regs region and region length */
diff --git a/drivers/crypto/intel/qat/qat_common/adf_sysfs.c b/drivers/crypto/intel/qat/qat_common/adf_sysfs.c
index ddffc98119c6..6f0b3629da13 100644
--- a/drivers/crypto/intel/qat/qat_common/adf_sysfs.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs.c
@@ -242,7 +242,8 @@ static ssize_t rp2srv_store(struct device *dev, struct device_attribute *attr,
 			    const char *buf, size_t count)
 {
 	struct adf_accel_dev *accel_dev;
-	int ring, num_rings, ret;
+	int num_rings, ret;
+	unsigned int ring;
 
 	accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev));
 	if (!accel_dev)
diff --git a/drivers/crypto/marvell/cesa/cesa.c b/drivers/crypto/marvell/cesa/cesa.c
index 5744df30c838..5fd31ba715c2 100644
--- a/drivers/crypto/marvell/cesa/cesa.c
+++ b/drivers/crypto/marvell/cesa/cesa.c
@@ -488,7 +488,7 @@ static int mv_cesa_probe(struct platform_device *pdev)
 
 	for (i = 0; i < caps->nengines; i++) {
 		struct mv_cesa_engine *engine = &cesa->engines[i];
-		char res_name[7];
+		char res_name[16];
 
 		engine->id = i;
 		spin_lock_init(&engine->lock);
@@ -509,7 +509,7 @@ static int mv_cesa_probe(struct platform_device *pdev)
 		 * Not all platforms can gate the CESA clocks: do not complain
 		 * if the clock does not exist.
 		 */
-		snprintf(res_name, sizeof(res_name), "cesa%d", i);
+		snprintf(res_name, sizeof(res_name), "cesa%u", i);
 		engine->clk = devm_clk_get(dev, res_name);
 		if (IS_ERR(engine->clk)) {
 			engine->clk = devm_clk_get(dev, NULL);
@@ -517,7 +517,7 @@ static int mv_cesa_probe(struct platform_device *pdev)
 				engine->clk = NULL;
 		}
 
-		snprintf(res_name, sizeof(res_name), "cesaz%d", i);
+		snprintf(res_name, sizeof(res_name), "cesaz%u", i);
 		engine->zclk = devm_clk_get(dev, res_name);
 		if (IS_ERR(engine->zclk))
 			engine->zclk = NULL;
diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c
index caea98622c33..37d958cfa9bb 100644
--- a/drivers/crypto/n2_core.c
+++ b/drivers/crypto/n2_core.c
@@ -1382,8 +1382,12 @@ static int __n2_register_one_hmac(struct n2_ahash_alg *n2ahash)
 	ahash->setkey = n2_hmac_async_setkey;
 
 	base = &ahash->halg.base;
-	snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", p->child_alg);
-	snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s-n2", p->child_alg);
+	if (snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)",
+		     p->child_alg) >= CRYPTO_MAX_ALG_NAME)
+		goto out_free_p;
+	if (snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s-n2",
+		     p->child_alg) >= CRYPTO_MAX_ALG_NAME)
+		goto out_free_p;
 
 	base->cra_ctxsize = sizeof(struct n2_hmac_ctx);
 	base->cra_init = n2_hmac_cra_init;
@@ -1394,6 +1398,7 @@ static int __n2_register_one_hmac(struct n2_ahash_alg *n2ahash)
 	if (err) {
 		pr_err("%s alg registration failed\n", base->cra_name);
 		list_del(&p->derived.entry);
+out_free_p:
 		kfree(p);
 	} else {
 		pr_info("%s alg registered\n", base->cra_name);
diff --git a/drivers/crypto/starfive/jh7110-cryp.c b/drivers/crypto/starfive/jh7110-cryp.c
index 08e974e0dd12..3a67ddc4d936 100644
--- a/drivers/crypto/starfive/jh7110-cryp.c
+++ b/drivers/crypto/starfive/jh7110-cryp.c
@@ -180,12 +180,8 @@ static int starfive_cryp_probe(struct platform_device *pdev)
 	spin_unlock(&dev_list.lock);
 
 	ret = starfive_dma_init(cryp);
-	if (ret) {
-		if (ret == -EPROBE_DEFER)
-			goto err_probe_defer;
-		else
-			goto err_dma_init;
-	}
+	if (ret)
+		goto err_dma_init;
 
 	/* Initialize crypto engine */
 	cryp->engine = crypto_engine_alloc_init(&pdev->dev, 1);
@@ -233,7 +229,7 @@ static int starfive_cryp_probe(struct platform_device *pdev)
 
 	tasklet_kill(&cryp->aes_done);
 	tasklet_kill(&cryp->hash_done);
-err_probe_defer:
+
 	return ret;
 }
 
diff --git a/drivers/dax/bus.c b/drivers/dax/bus.c
index 1659b787b65f..1ff1ab5fa105 100644
--- a/drivers/dax/bus.c
+++ b/drivers/dax/bus.c
@@ -367,6 +367,7 @@ static ssize_t create_store(struct device *dev, struct device_attribute *attr,
 			.dax_region = dax_region,
 			.size = 0,
 			.id = -1,
+			.memmap_on_memory = false,
 		};
 		struct dev_dax *dev_dax = devm_create_dev_dax(&data);
 
@@ -1400,6 +1401,8 @@ struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data)
 	dev_dax->align = dax_region->align;
 	ida_init(&dev_dax->ida);
 
+	dev_dax->memmap_on_memory = data->memmap_on_memory;
+
 	inode = dax_inode(dax_dev);
 	dev->devt = inode->i_rdev;
 	dev->bus = &dax_bus_type;
diff --git a/drivers/dax/bus.h b/drivers/dax/bus.h
index 1ccd23360124..cbbf64443098 100644
--- a/drivers/dax/bus.h
+++ b/drivers/dax/bus.h
@@ -23,6 +23,7 @@ struct dev_dax_data {
 	struct dev_pagemap *pgmap;
 	resource_size_t size;
 	int id;
+	bool memmap_on_memory;
 };
 
 struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data);
diff --git a/drivers/dax/cxl.c b/drivers/dax/cxl.c
index 8bc9d04034d6..c696837ab23c 100644
--- a/drivers/dax/cxl.c
+++ b/drivers/dax/cxl.c
@@ -26,6 +26,7 @@ static int cxl_dax_region_probe(struct device *dev)
 		.dax_region = dax_region,
 		.id = -1,
 		.size = range_len(&cxlr_dax->hpa_range),
+		.memmap_on_memory = true,
 	};
 
 	return PTR_ERR_OR_ZERO(devm_create_dev_dax(&data));
diff --git a/drivers/dax/dax-private.h b/drivers/dax/dax-private.h
index 27cf2daaaa79..446617b73aea 100644
--- a/drivers/dax/dax-private.h
+++ b/drivers/dax/dax-private.h
@@ -70,6 +70,7 @@ struct dev_dax {
 	struct ida ida;
 	struct device dev;
 	struct dev_pagemap *pgmap;
+	bool memmap_on_memory;
 	int nr_range;
 	struct dev_dax_range {
 		unsigned long pgoff;
diff --git a/drivers/dax/hmem/hmem.c b/drivers/dax/hmem/hmem.c
index 5d2ddef0f8f5..b9da69f92697 100644
--- a/drivers/dax/hmem/hmem.c
+++ b/drivers/dax/hmem/hmem.c
@@ -36,6 +36,7 @@ static int dax_hmem_probe(struct platform_device *pdev)
 		.dax_region = dax_region,
 		.id = -1,
 		.size = region_idle ? 0 : range_len(&mri->range),
+		.memmap_on_memory = false,
 	};
 
 	return PTR_ERR_OR_ZERO(devm_create_dev_dax(&data));
diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c
index 369c698b7706..42ee360cf4e3 100644
--- a/drivers/dax/kmem.c
+++ b/drivers/dax/kmem.c
@@ -12,6 +12,7 @@
 #include <linux/mm.h>
 #include <linux/mman.h>
 #include <linux/memory-tiers.h>
+#include <linux/memory_hotplug.h>
 #include "dax-private.h"
 #include "bus.h"
 
@@ -93,6 +94,7 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax)
 	struct dax_kmem_data *data;
 	struct memory_dev_type *mtype;
 	int i, rc, mapped = 0;
+	mhp_t mhp_flags;
 	int numa_node;
 	int adist = MEMTIER_DEFAULT_DAX_ADISTANCE;
 
@@ -179,12 +181,16 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax)
 		 */
 		res->flags = IORESOURCE_SYSTEM_RAM;
 
+		mhp_flags = MHP_NID_IS_MGID;
+		if (dev_dax->memmap_on_memory)
+			mhp_flags |= MHP_MEMMAP_ON_MEMORY;
+
 		/*
 		 * Ensure that future kexec'd kernels will not treat
 		 * this as RAM automatically.
 		 */
 		rc = add_memory_driver_managed(data->mgid, range.start,
-				range_len(&range), kmem_name, MHP_NID_IS_MGID);
+				range_len(&range), kmem_name, mhp_flags);
 
 		if (rc) {
 			dev_warn(dev, "mapping%d: %#llx-%#llx memory add failed\n",
diff --git a/drivers/dax/pmem.c b/drivers/dax/pmem.c
index ae0cb113a5d3..f3c6c67b8412 100644
--- a/drivers/dax/pmem.c
+++ b/drivers/dax/pmem.c
@@ -63,6 +63,7 @@ static struct dev_dax *__dax_pmem_probe(struct device *dev)
 		.id = id,
 		.pgmap = &pgmap,
 		.size = range_len(&range),
+		.memmap_on_memory = false,
 	};
 
 	return devm_create_dev_dax(&data);
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index 21916bba77d5..8fe5aa67b167 100644
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -46,12 +46,12 @@ static char *dmabuffs_dname(struct dentry *dentry, char *buffer, int buflen)
 {
 	struct dma_buf *dmabuf;
 	char name[DMA_BUF_NAME_LEN];
-	size_t ret = 0;
+	ssize_t ret = 0;
 
 	dmabuf = dentry->d_fsdata;
 	spin_lock(&dmabuf->name_lock);
 	if (dmabuf->name)
-		ret = strlcpy(name, dmabuf->name, DMA_BUF_NAME_LEN);
+		ret = strscpy(name, dmabuf->name, sizeof(name));
 	spin_unlock(&dmabuf->name_lock);
 
 	return dynamic_dname(buffer, buflen, "/%s:%s",
diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c
index 8b31cd54bdb6..ae17ce4d9722 100644
--- a/drivers/edac/altera_edac.c
+++ b/drivers/edac/altera_edac.c
@@ -22,6 +22,7 @@
 #include <linux/of_platform.h>
 #include <linux/panic_notifier.h>
 #include <linux/platform_device.h>
+#include <linux/property.h>
 #include <linux/regmap.h>
 #include <linux/types.h>
 #include <linux/uaccess.h>
@@ -279,7 +280,6 @@ static int a10_unmask_irq(struct platform_device *pdev, u32 mask)
 
 static int altr_sdram_probe(struct platform_device *pdev)
 {
-	const struct of_device_id *id;
 	struct edac_mc_layer layers[2];
 	struct mem_ctl_info *mci;
 	struct altr_sdram_mc_data *drvdata;
@@ -290,10 +290,6 @@ static int altr_sdram_probe(struct platform_device *pdev)
 	int irq, irq2, res = 0;
 	unsigned long mem_size, irqflags = 0;
 
-	id = of_match_device(altr_sdram_ctrl_of_match, &pdev->dev);
-	if (!id)
-		return -ENODEV;
-
 	/* Grab the register range from the sdr controller in device tree */
 	mc_vbase = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
 						   "altr,sdr-syscon");
@@ -304,8 +300,7 @@ static int altr_sdram_probe(struct platform_device *pdev)
 	}
 
 	/* Check specific dependencies for the module */
-	priv = of_match_node(altr_sdram_ctrl_of_match,
-			     pdev->dev.of_node)->data;
+	priv = device_get_match_data(&pdev->dev);
 
 	/* Validate the SDRAM controller has ECC enabled */
 	if (regmap_read(mc_vbase, priv->ecc_ctrl_offset, &read_reg) ||
@@ -459,15 +454,13 @@ static int altr_sdram_probe(struct platform_device *pdev)
 	return res;
 }
 
-static int altr_sdram_remove(struct platform_device *pdev)
+static void altr_sdram_remove(struct platform_device *pdev)
 {
 	struct mem_ctl_info *mci = platform_get_drvdata(pdev);
 
 	edac_mc_del_mc(&pdev->dev);
 	edac_mc_free(mci);
 	platform_set_drvdata(pdev, NULL);
-
-	return 0;
 }
 
 /*
@@ -489,7 +482,7 @@ static const struct dev_pm_ops altr_sdram_pm_ops = {
 
 static struct platform_driver altr_sdram_edac_driver = {
 	.probe = altr_sdram_probe,
-	.remove = altr_sdram_remove,
+	.remove_new = altr_sdram_remove,
 	.driver = {
 		.name = "altr_sdram_edac",
 #ifdef CONFIG_PM
@@ -812,7 +805,7 @@ static int altr_edac_device_probe(struct platform_device *pdev)
 	return res;
 }
 
-static int altr_edac_device_remove(struct platform_device *pdev)
+static void altr_edac_device_remove(struct platform_device *pdev)
 {
 	struct edac_device_ctl_info *dci = platform_get_drvdata(pdev);
 	struct altr_edac_device_dev *drvdata = dci->pvt_info;
@@ -820,13 +813,11 @@ static int altr_edac_device_remove(struct platform_device *pdev)
 	debugfs_remove_recursive(drvdata->debugfs_dir);
 	edac_device_del_device(&pdev->dev);
 	edac_device_free_ctl_info(dci);
-
-	return 0;
 }
 
 static struct platform_driver altr_edac_device_driver = {
 	.probe =  altr_edac_device_probe,
-	.remove = altr_edac_device_remove,
+	.remove_new = altr_edac_device_remove,
 	.driver = {
 		.name = "altr_edac_device",
 		.of_match_table = altr_edac_device_of_match,
diff --git a/drivers/edac/armada_xp_edac.c b/drivers/edac/armada_xp_edac.c
index c4bd2fb9c46b..5bcd34f23baf 100644
--- a/drivers/edac/armada_xp_edac.c
+++ b/drivers/edac/armada_xp_edac.c
@@ -351,20 +351,18 @@ static int axp_mc_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int axp_mc_remove(struct platform_device *pdev)
+static void axp_mc_remove(struct platform_device *pdev)
 {
 	struct mem_ctl_info *mci = platform_get_drvdata(pdev);
 
 	edac_mc_del_mc(&pdev->dev);
 	edac_mc_free(mci);
 	platform_set_drvdata(pdev, NULL);
-
-	return 0;
 }
 
 static struct platform_driver axp_mc_driver = {
 	.probe = axp_mc_probe,
-	.remove = axp_mc_remove,
+	.remove_new = axp_mc_remove,
 	.driver = {
 		.name = "armada_xp_mc_edac",
 		.of_match_table = of_match_ptr(axp_mc_of_match),
@@ -564,7 +562,7 @@ static int aurora_l2_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int aurora_l2_remove(struct platform_device *pdev)
+static void aurora_l2_remove(struct platform_device *pdev)
 {
 	struct edac_device_ctl_info *dci = platform_get_drvdata(pdev);
 #ifdef CONFIG_EDAC_DEBUG
@@ -575,13 +573,11 @@ static int aurora_l2_remove(struct platform_device *pdev)
 	edac_device_del_device(&pdev->dev);
 	edac_device_free_ctl_info(dci);
 	platform_set_drvdata(pdev, NULL);
-
-	return 0;
 }
 
 static struct platform_driver aurora_l2_driver = {
 	.probe = aurora_l2_probe,
-	.remove = aurora_l2_remove,
+	.remove_new = aurora_l2_remove,
 	.driver = {
 		.name = "aurora_l2_edac",
 		.of_match_table = of_match_ptr(aurora_l2_of_match),
diff --git a/drivers/edac/aspeed_edac.c b/drivers/edac/aspeed_edac.c
index 6bd5f8815919..157a480eb761 100644
--- a/drivers/edac/aspeed_edac.c
+++ b/drivers/edac/aspeed_edac.c
@@ -357,7 +357,7 @@ static int aspeed_probe(struct platform_device *pdev)
 }
 
 
-static int aspeed_remove(struct platform_device *pdev)
+static void aspeed_remove(struct platform_device *pdev)
 {
 	struct mem_ctl_info *mci;
 
@@ -369,8 +369,6 @@ static int aspeed_remove(struct platform_device *pdev)
 	mci = edac_mc_del_mc(&pdev->dev);
 	if (mci)
 		edac_mc_free(mci);
-
-	return 0;
 }
 
 
@@ -389,7 +387,7 @@ static struct platform_driver aspeed_driver = {
 		.of_match_table = aspeed_of_match
 	},
 	.probe		= aspeed_probe,
-	.remove		= aspeed_remove
+	.remove_new	= aspeed_remove
 };
 module_platform_driver(aspeed_driver);
 
diff --git a/drivers/edac/bluefield_edac.c b/drivers/edac/bluefield_edac.c
index e4736eb37bfb..5b3164560648 100644
--- a/drivers/edac/bluefield_edac.c
+++ b/drivers/edac/bluefield_edac.c
@@ -323,14 +323,12 @@ static int bluefield_edac_mc_probe(struct platform_device *pdev)
 
 }
 
-static int bluefield_edac_mc_remove(struct platform_device *pdev)
+static void bluefield_edac_mc_remove(struct platform_device *pdev)
 {
 	struct mem_ctl_info *mci = platform_get_drvdata(pdev);
 
 	edac_mc_del_mc(&pdev->dev);
 	edac_mc_free(mci);
-
-	return 0;
 }
 
 static const struct acpi_device_id bluefield_mc_acpi_ids[] = {
@@ -346,7 +344,7 @@ static struct platform_driver bluefield_edac_mc_driver = {
 		.acpi_match_table = bluefield_mc_acpi_ids,
 	},
 	.probe = bluefield_edac_mc_probe,
-	.remove = bluefield_edac_mc_remove,
+	.remove_new = bluefield_edac_mc_remove,
 };
 
 module_platform_driver(bluefield_edac_mc_driver);
diff --git a/drivers/edac/cell_edac.c b/drivers/edac/cell_edac.c
index bc1f3416400e..2000f66fbf5c 100644
--- a/drivers/edac/cell_edac.c
+++ b/drivers/edac/cell_edac.c
@@ -234,12 +234,11 @@ static int cell_edac_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int cell_edac_remove(struct platform_device *pdev)
+static void cell_edac_remove(struct platform_device *pdev)
 {
 	struct mem_ctl_info *mci = edac_mc_del_mc(&pdev->dev);
 	if (mci)
 		edac_mc_free(mci);
-	return 0;
 }
 
 static struct platform_driver cell_edac_driver = {
@@ -247,7 +246,7 @@ static struct platform_driver cell_edac_driver = {
 		.name	= "cbe-mic",
 	},
 	.probe		= cell_edac_probe,
-	.remove		= cell_edac_remove,
+	.remove_new	= cell_edac_remove,
 };
 
 static int __init cell_edac_init(void)
diff --git a/drivers/edac/cpc925_edac.c b/drivers/edac/cpc925_edac.c
index 9797e6d60dde..5075dc7526e3 100644
--- a/drivers/edac/cpc925_edac.c
+++ b/drivers/edac/cpc925_edac.c
@@ -1010,7 +1010,7 @@ static int cpc925_probe(struct platform_device *pdev)
 	return res;
 }
 
-static int cpc925_remove(struct platform_device *pdev)
+static void cpc925_remove(struct platform_device *pdev)
 {
 	struct mem_ctl_info *mci = platform_get_drvdata(pdev);
 
@@ -1023,13 +1023,11 @@ static int cpc925_remove(struct platform_device *pdev)
 
 	edac_mc_del_mc(&pdev->dev);
 	edac_mc_free(mci);
-
-	return 0;
 }
 
 static struct platform_driver cpc925_edac_driver = {
 	.probe = cpc925_probe,
-	.remove = cpc925_remove,
+	.remove_new = cpc925_remove,
 	.driver = {
 		   .name = "cpc925_edac",
 	}
diff --git a/drivers/edac/dmc520_edac.c b/drivers/edac/dmc520_edac.c
index 1fa5ca57e9ec..4e30b989a1a4 100644
--- a/drivers/edac/dmc520_edac.c
+++ b/drivers/edac/dmc520_edac.c
@@ -602,7 +602,7 @@ static int dmc520_edac_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int dmc520_edac_remove(struct platform_device *pdev)
+static void dmc520_edac_remove(struct platform_device *pdev)
 {
 	u32 reg_val, idx, irq_mask_all = 0;
 	struct mem_ctl_info *mci;
@@ -626,8 +626,6 @@ static int dmc520_edac_remove(struct platform_device *pdev)
 
 	edac_mc_del_mc(&pdev->dev);
 	edac_mc_free(mci);
-
-	return 0;
 }
 
 static const struct of_device_id dmc520_edac_driver_id[] = {
@@ -644,7 +642,7 @@ static struct platform_driver dmc520_edac_driver = {
 	},
 
 	.probe = dmc520_edac_probe,
-	.remove = dmc520_edac_remove
+	.remove_new = dmc520_edac_remove
 };
 
 module_platform_driver(dmc520_edac_driver);
diff --git a/drivers/edac/fsl_ddr_edac.c b/drivers/edac/fsl_ddr_edac.c
index b81757555a8a..d148d262d0d4 100644
--- a/drivers/edac/fsl_ddr_edac.c
+++ b/drivers/edac/fsl_ddr_edac.c
@@ -612,7 +612,7 @@ int fsl_mc_err_probe(struct platform_device *op)
 	return res;
 }
 
-int fsl_mc_err_remove(struct platform_device *op)
+void fsl_mc_err_remove(struct platform_device *op)
 {
 	struct mem_ctl_info *mci = dev_get_drvdata(&op->dev);
 	struct fsl_mc_pdata *pdata = mci->pvt_info;
@@ -629,5 +629,4 @@ int fsl_mc_err_remove(struct platform_device *op)
 
 	edac_mc_del_mc(&op->dev);
 	edac_mc_free(mci);
-	return 0;
 }
diff --git a/drivers/edac/fsl_ddr_edac.h b/drivers/edac/fsl_ddr_edac.h
index 332439d7b2d9..c0994a2a003c 100644
--- a/drivers/edac/fsl_ddr_edac.h
+++ b/drivers/edac/fsl_ddr_edac.h
@@ -72,5 +72,5 @@ struct fsl_mc_pdata {
 	int irq;
 };
 int fsl_mc_err_probe(struct platform_device *op);
-int fsl_mc_err_remove(struct platform_device *op);
+void fsl_mc_err_remove(struct platform_device *op);
 #endif
diff --git a/drivers/edac/highbank_l2_edac.c b/drivers/edac/highbank_l2_edac.c
index 140d4431bd0d..5646c049a934 100644
--- a/drivers/edac/highbank_l2_edac.c
+++ b/drivers/edac/highbank_l2_edac.c
@@ -118,18 +118,17 @@ static int highbank_l2_err_probe(struct platform_device *pdev)
 	return res;
 }
 
-static int highbank_l2_err_remove(struct platform_device *pdev)
+static void highbank_l2_err_remove(struct platform_device *pdev)
 {
 	struct edac_device_ctl_info *dci = platform_get_drvdata(pdev);
 
 	edac_device_del_device(&pdev->dev);
 	edac_device_free_ctl_info(dci);
-	return 0;
 }
 
 static struct platform_driver highbank_l2_edac_driver = {
 	.probe = highbank_l2_err_probe,
-	.remove = highbank_l2_err_remove,
+	.remove_new = highbank_l2_err_remove,
 	.driver = {
 		.name = "hb_l2_edac",
 		.of_match_table = hb_l2_err_of_match,
diff --git a/drivers/edac/highbank_mc_edac.c b/drivers/edac/highbank_mc_edac.c
index a0c04a7f95e9..1c5b888ab11d 100644
--- a/drivers/edac/highbank_mc_edac.c
+++ b/drivers/edac/highbank_mc_edac.c
@@ -251,18 +251,17 @@ static int highbank_mc_probe(struct platform_device *pdev)
 	return res;
 }
 
-static int highbank_mc_remove(struct platform_device *pdev)
+static void highbank_mc_remove(struct platform_device *pdev)
 {
 	struct mem_ctl_info *mci = platform_get_drvdata(pdev);
 
 	edac_mc_del_mc(&pdev->dev);
 	edac_mc_free(mci);
-	return 0;
 }
 
 static struct platform_driver highbank_mc_edac_driver = {
 	.probe = highbank_mc_probe,
-	.remove = highbank_mc_remove,
+	.remove_new = highbank_mc_remove,
 	.driver = {
 		.name = "hb_mc_edac",
 		.of_match_table = hb_ddr_ctrl_of_match,
diff --git a/drivers/edac/layerscape_edac.c b/drivers/edac/layerscape_edac.c
index 7c5e2b3c0daa..d2f895033280 100644
--- a/drivers/edac/layerscape_edac.c
+++ b/drivers/edac/layerscape_edac.c
@@ -27,7 +27,7 @@ MODULE_DEVICE_TABLE(of, fsl_ddr_mc_err_of_match);
 
 static struct platform_driver fsl_ddr_mc_err_driver = {
 	.probe = fsl_mc_err_probe,
-	.remove = fsl_mc_err_remove,
+	.remove_new = fsl_mc_err_remove,
 	.driver = {
 		.name = "fsl_ddr_mc_err",
 		.of_match_table = fsl_ddr_mc_err_of_match,
diff --git a/drivers/edac/mpc85xx_edac.c b/drivers/edac/mpc85xx_edac.c
index 2b5703e5066e..c1bc53f4e184 100644
--- a/drivers/edac/mpc85xx_edac.c
+++ b/drivers/edac/mpc85xx_edac.c
@@ -300,7 +300,7 @@ static int mpc85xx_pci_err_probe(struct platform_device *op)
 	return res;
 }
 
-static int mpc85xx_pci_err_remove(struct platform_device *op)
+static void mpc85xx_pci_err_remove(struct platform_device *op)
 {
 	struct edac_pci_ctl_info *pci = dev_get_drvdata(&op->dev);
 	struct mpc85xx_pci_pdata *pdata = pci->pvt_info;
@@ -312,8 +312,6 @@ static int mpc85xx_pci_err_remove(struct platform_device *op)
 
 	edac_pci_del_device(&op->dev);
 	edac_pci_free_ctl_info(pci);
-
-	return 0;
 }
 
 static const struct platform_device_id mpc85xx_pci_err_match[] = {
@@ -325,7 +323,7 @@ static const struct platform_device_id mpc85xx_pci_err_match[] = {
 
 static struct platform_driver mpc85xx_pci_err_driver = {
 	.probe = mpc85xx_pci_err_probe,
-	.remove = mpc85xx_pci_err_remove,
+	.remove_new = mpc85xx_pci_err_remove,
 	.id_table = mpc85xx_pci_err_match,
 	.driver = {
 		.name = "mpc85xx_pci_err",
@@ -591,7 +589,7 @@ static int mpc85xx_l2_err_probe(struct platform_device *op)
 	return res;
 }
 
-static int mpc85xx_l2_err_remove(struct platform_device *op)
+static void mpc85xx_l2_err_remove(struct platform_device *op)
 {
 	struct edac_device_ctl_info *edac_dev = dev_get_drvdata(&op->dev);
 	struct mpc85xx_l2_pdata *pdata = edac_dev->pvt_info;
@@ -606,7 +604,6 @@ static int mpc85xx_l2_err_remove(struct platform_device *op)
 	out_be32(pdata->l2_vbase + MPC85XX_L2_ERRDIS, orig_l2_err_disable);
 	edac_device_del_device(&op->dev);
 	edac_device_free_ctl_info(edac_dev);
-	return 0;
 }
 
 static const struct of_device_id mpc85xx_l2_err_of_match[] = {
@@ -630,7 +627,7 @@ MODULE_DEVICE_TABLE(of, mpc85xx_l2_err_of_match);
 
 static struct platform_driver mpc85xx_l2_err_driver = {
 	.probe = mpc85xx_l2_err_probe,
-	.remove = mpc85xx_l2_err_remove,
+	.remove_new = mpc85xx_l2_err_remove,
 	.driver = {
 		.name = "mpc85xx_l2_err",
 		.of_match_table = mpc85xx_l2_err_of_match,
@@ -659,7 +656,7 @@ MODULE_DEVICE_TABLE(of, mpc85xx_mc_err_of_match);
 
 static struct platform_driver mpc85xx_mc_err_driver = {
 	.probe = fsl_mc_err_probe,
-	.remove = fsl_mc_err_remove,
+	.remove_new = fsl_mc_err_remove,
 	.driver = {
 		.name = "mpc85xx_mc_err",
 		.of_match_table = mpc85xx_mc_err_of_match,
diff --git a/drivers/edac/npcm_edac.c b/drivers/edac/npcm_edac.c
index 6d15c1550263..2e2133b784e9 100644
--- a/drivers/edac/npcm_edac.c
+++ b/drivers/edac/npcm_edac.c
@@ -410,7 +410,7 @@ static int edac_probe(struct platform_device *pdev)
 	return rc;
 }
 
-static int edac_remove(struct platform_device *pdev)
+static void edac_remove(struct platform_device *pdev)
 {
 	struct mem_ctl_info *mci = platform_get_drvdata(pdev);
 	struct priv_data *priv = mci->pvt_info;
@@ -426,8 +426,6 @@ static int edac_remove(struct platform_device *pdev)
 	regmap_write(npcm_regmap, pdata->ctl_int_mask_master,
 		     pdata->int_mask_master_global_mask);
 	regmap_update_bits(npcm_regmap, pdata->ctl_ecc_en, pdata->ecc_en_mask, 0);
-
-	return 0;
 }
 
 static const struct npcm_platform_data npcm750_edac = {
@@ -533,7 +531,7 @@ static struct platform_driver npcm_edac_driver = {
 		.of_match_table = npcm_edac_of_match,
 	},
 	.probe = edac_probe,
-	.remove = edac_remove,
+	.remove_new = edac_remove,
 };
 
 module_platform_driver(npcm_edac_driver);
diff --git a/drivers/edac/octeon_edac-l2c.c b/drivers/edac/octeon_edac-l2c.c
index c33059e9b0be..4015eb9af6fe 100644
--- a/drivers/edac/octeon_edac-l2c.c
+++ b/drivers/edac/octeon_edac-l2c.c
@@ -184,19 +184,17 @@ static int octeon_l2c_probe(struct platform_device *pdev)
 	return -ENXIO;
 }
 
-static int octeon_l2c_remove(struct platform_device *pdev)
+static void octeon_l2c_remove(struct platform_device *pdev)
 {
 	struct edac_device_ctl_info *l2c = platform_get_drvdata(pdev);
 
 	edac_device_del_device(&pdev->dev);
 	edac_device_free_ctl_info(l2c);
-
-	return 0;
 }
 
 static struct platform_driver octeon_l2c_driver = {
 	.probe = octeon_l2c_probe,
-	.remove = octeon_l2c_remove,
+	.remove_new = octeon_l2c_remove,
 	.driver = {
 		   .name = "octeon_l2c_edac",
 	}
diff --git a/drivers/edac/octeon_edac-lmc.c b/drivers/edac/octeon_edac-lmc.c
index aeb222ca3ed1..18615cbcd9ea 100644
--- a/drivers/edac/octeon_edac-lmc.c
+++ b/drivers/edac/octeon_edac-lmc.c
@@ -302,18 +302,17 @@ static int octeon_lmc_edac_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int octeon_lmc_edac_remove(struct platform_device *pdev)
+static void octeon_lmc_edac_remove(struct platform_device *pdev)
 {
 	struct mem_ctl_info *mci = platform_get_drvdata(pdev);
 
 	edac_mc_del_mc(&pdev->dev);
 	edac_mc_free(mci);
-	return 0;
 }
 
 static struct platform_driver octeon_lmc_edac_driver = {
 	.probe = octeon_lmc_edac_probe,
-	.remove = octeon_lmc_edac_remove,
+	.remove_new = octeon_lmc_edac_remove,
 	.driver = {
 		   .name = "octeon_lmc_edac",
 	}
diff --git a/drivers/edac/octeon_edac-pc.c b/drivers/edac/octeon_edac-pc.c
index 754eced59c32..ea8a8e337b1e 100644
--- a/drivers/edac/octeon_edac-pc.c
+++ b/drivers/edac/octeon_edac-pc.c
@@ -119,19 +119,18 @@ static int co_cache_error_probe(struct platform_device *pdev)
 	return -ENXIO;
 }
 
-static int co_cache_error_remove(struct platform_device *pdev)
+static void co_cache_error_remove(struct platform_device *pdev)
 {
 	struct co_cache_error *p = platform_get_drvdata(pdev);
 
 	unregister_co_cache_error_notifier(&p->notifier);
 	edac_device_del_device(&pdev->dev);
 	edac_device_free_ctl_info(p->ed);
-	return 0;
 }
 
 static struct platform_driver co_cache_error_driver = {
 	.probe = co_cache_error_probe,
-	.remove = co_cache_error_remove,
+	.remove_new = co_cache_error_remove,
 	.driver = {
 		   .name = "octeon_pc_edac",
 	}
diff --git a/drivers/edac/octeon_edac-pci.c b/drivers/edac/octeon_edac-pci.c
index 28b238eecefc..108ad9493cfb 100644
--- a/drivers/edac/octeon_edac-pci.c
+++ b/drivers/edac/octeon_edac-pci.c
@@ -87,19 +87,17 @@ static int octeon_pci_probe(struct platform_device *pdev)
 	return res;
 }
 
-static int octeon_pci_remove(struct platform_device *pdev)
+static void octeon_pci_remove(struct platform_device *pdev)
 {
 	struct edac_pci_ctl_info *pci = platform_get_drvdata(pdev);
 
 	edac_pci_del_device(&pdev->dev);
 	edac_pci_free_ctl_info(pci);
-
-	return 0;
 }
 
 static struct platform_driver octeon_pci_driver = {
 	.probe = octeon_pci_probe,
-	.remove = octeon_pci_remove,
+	.remove_new = octeon_pci_remove,
 	.driver = {
 		   .name = "octeon_pci_edac",
 	}
diff --git a/drivers/edac/ppc4xx_edac.c b/drivers/edac/ppc4xx_edac.c
index 046969b4e82e..1eea3341a916 100644
--- a/drivers/edac/ppc4xx_edac.c
+++ b/drivers/edac/ppc4xx_edac.c
@@ -1329,8 +1329,7 @@ static int ppc4xx_edac_probe(struct platform_device *op)
  *
  * Unconditionally returns 0.
  */
-static int
-ppc4xx_edac_remove(struct platform_device *op)
+static void ppc4xx_edac_remove(struct platform_device *op)
 {
 	struct mem_ctl_info *mci = dev_get_drvdata(&op->dev);
 	struct ppc4xx_edac_pdata *pdata = mci->pvt_info;
@@ -1344,8 +1343,6 @@ ppc4xx_edac_remove(struct platform_device *op)
 
 	edac_mc_del_mc(mci->pdev);
 	edac_mc_free(mci);
-
-	return 0;
 }
 
 /**
@@ -1379,7 +1376,7 @@ ppc4xx_edac_opstate_init(void)
 
 static struct platform_driver ppc4xx_edac_driver = {
 	.probe			= ppc4xx_edac_probe,
-	.remove			= ppc4xx_edac_remove,
+	.remove_new		= ppc4xx_edac_remove,
 	.driver = {
 		.name = PPC4XX_EDAC_MODULE_NAME,
 		.of_match_table = ppc4xx_edac_match,
diff --git a/drivers/edac/qcom_edac.c b/drivers/edac/qcom_edac.c
index b2db545c6810..5539917c01dd 100644
--- a/drivers/edac/qcom_edac.c
+++ b/drivers/edac/qcom_edac.c
@@ -390,14 +390,12 @@ static int qcom_llcc_edac_probe(struct platform_device *pdev)
 	return rc;
 }
 
-static int qcom_llcc_edac_remove(struct platform_device *pdev)
+static void qcom_llcc_edac_remove(struct platform_device *pdev)
 {
 	struct edac_device_ctl_info *edev_ctl = dev_get_drvdata(&pdev->dev);
 
 	edac_device_del_device(edev_ctl->dev);
 	edac_device_free_ctl_info(edev_ctl);
-
-	return 0;
 }
 
 static const struct platform_device_id qcom_llcc_edac_id_table[] = {
@@ -408,7 +406,7 @@ MODULE_DEVICE_TABLE(platform, qcom_llcc_edac_id_table);
 
 static struct platform_driver qcom_llcc_edac_driver = {
 	.probe = qcom_llcc_edac_probe,
-	.remove = qcom_llcc_edac_remove,
+	.remove_new = qcom_llcc_edac_remove,
 	.driver = {
 		.name = "qcom_llcc_edac",
 	},
diff --git a/drivers/edac/synopsys_edac.c b/drivers/edac/synopsys_edac.c
index c4fc64cbecd0..709babce43ba 100644
--- a/drivers/edac/synopsys_edac.c
+++ b/drivers/edac/synopsys_edac.c
@@ -1410,7 +1410,7 @@ static int mc_probe(struct platform_device *pdev)
  *
  * Return: Unconditionally 0
  */
-static int mc_remove(struct platform_device *pdev)
+static void mc_remove(struct platform_device *pdev)
 {
 	struct mem_ctl_info *mci = platform_get_drvdata(pdev);
 	struct synps_edac_priv *priv = mci->pvt_info;
@@ -1425,8 +1425,6 @@ static int mc_remove(struct platform_device *pdev)
 
 	edac_mc_del_mc(&pdev->dev);
 	edac_mc_free(mci);
-
-	return 0;
 }
 
 static struct platform_driver synps_edac_mc_driver = {
@@ -1435,7 +1433,7 @@ static struct platform_driver synps_edac_mc_driver = {
 		   .of_match_table = synps_edac_match,
 		   },
 	.probe = mc_probe,
-	.remove = mc_remove,
+	.remove_new = mc_remove,
 };
 
 module_platform_driver(synps_edac_mc_driver);
diff --git a/drivers/edac/ti_edac.c b/drivers/edac/ti_edac.c
index 6971ded598de..29723c9592f7 100644
--- a/drivers/edac/ti_edac.c
+++ b/drivers/edac/ti_edac.c
@@ -312,19 +312,17 @@ static int ti_edac_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int ti_edac_remove(struct platform_device *pdev)
+static void ti_edac_remove(struct platform_device *pdev)
 {
 	struct mem_ctl_info *mci = platform_get_drvdata(pdev);
 
 	edac_mc_del_mc(&pdev->dev);
 	edac_mc_free(mci);
-
-	return 0;
 }
 
 static struct platform_driver ti_edac_driver = {
 	.probe = ti_edac_probe,
-	.remove = ti_edac_remove,
+	.remove_new = ti_edac_remove,
 	.driver = {
 		   .name = EDAC_MOD_NAME,
 		   .of_match_table = ti_edac_of_match,
diff --git a/drivers/edac/xgene_edac.c b/drivers/edac/xgene_edac.c
index c52b9dd9154c..1b50f8160013 100644
--- a/drivers/edac/xgene_edac.c
+++ b/drivers/edac/xgene_edac.c
@@ -1960,7 +1960,7 @@ static int xgene_edac_probe(struct platform_device *pdev)
 	return rc;
 }
 
-static int xgene_edac_remove(struct platform_device *pdev)
+static void xgene_edac_remove(struct platform_device *pdev)
 {
 	struct xgene_edac *edac = dev_get_drvdata(&pdev->dev);
 	struct xgene_edac_mc_ctx *mcu;
@@ -1981,8 +1981,6 @@ static int xgene_edac_remove(struct platform_device *pdev)
 
 	list_for_each_entry_safe(node, temp_node, &edac->socs, next)
 		xgene_edac_soc_remove(node);
-
-	return 0;
 }
 
 static const struct of_device_id xgene_edac_of_match[] = {
@@ -1993,7 +1991,7 @@ MODULE_DEVICE_TABLE(of, xgene_edac_of_match);
 
 static struct platform_driver xgene_edac_driver = {
 	.probe = xgene_edac_probe,
-	.remove = xgene_edac_remove,
+	.remove_new = xgene_edac_remove,
 	.driver = {
 		.name = "xgene-edac",
 		.of_match_table = xgene_edac_of_match,
diff --git a/drivers/edac/zynqmp_edac.c b/drivers/edac/zynqmp_edac.c
index ac7d1e0b324c..2d9a5cfd8931 100644
--- a/drivers/edac/zynqmp_edac.c
+++ b/drivers/edac/zynqmp_edac.c
@@ -426,7 +426,7 @@ static int edac_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int edac_remove(struct platform_device *pdev)
+static void edac_remove(struct platform_device *pdev)
 {
 	struct edac_device_ctl_info *dci = platform_get_drvdata(pdev);
 	struct edac_priv *priv = dci->pvt_info;
@@ -440,8 +440,6 @@ static int edac_remove(struct platform_device *pdev)
 
 	edac_device_del_device(&pdev->dev);
 	edac_device_free_ctl_info(dci);
-
-	return 0;
 }
 
 static const struct of_device_id zynqmp_ocm_edac_match[] = {
@@ -457,7 +455,7 @@ static struct platform_driver zynqmp_ocm_edac_driver = {
 		   .of_match_table = zynqmp_ocm_edac_match,
 		   },
 	.probe = edac_probe,
-	.remove = edac_remove,
+	.remove_new = edac_remove,
 };
 
 module_platform_driver(zynqmp_ocm_edac_driver);
diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c
index 07b72c679247..6146b2927d5c 100644
--- a/drivers/firmware/arm_ffa/driver.c
+++ b/drivers/firmware/arm_ffa/driver.c
@@ -99,6 +99,7 @@ struct ffa_drv_info {
 	void *tx_buffer;
 	bool mem_ops_native;
 	bool bitmap_created;
+	bool notif_enabled;
 	unsigned int sched_recv_irq;
 	unsigned int cpuhp_state;
 	struct ffa_pcpu_irq __percpu *irq_pcpu;
@@ -782,7 +783,7 @@ static void ffa_notification_info_get(void)
 			if (ids_processed >= max_ids - 1)
 				break;
 
-			part_id = packed_id_list[++ids_processed];
+			part_id = packed_id_list[ids_processed++];
 
 			if (!ids_count[list]) { /* Global Notification */
 				__do_sched_recv_cb(part_id, 0, false);
@@ -794,7 +795,7 @@ static void ffa_notification_info_get(void)
 				if (ids_processed >= max_ids - 1)
 					break;
 
-				vcpu_id = packed_id_list[++ids_processed];
+				vcpu_id = packed_id_list[ids_processed++];
 
 				__do_sched_recv_cb(part_id, vcpu_id, true);
 			}
@@ -889,6 +890,8 @@ static int ffa_memory_lend(struct ffa_mem_ops_args *args)
 
 #define FFA_SECURE_PARTITION_ID_FLAG	BIT(15)
 
+#define ffa_notifications_disabled()	(!drv_info->notif_enabled)
+
 enum notify_type {
 	NON_SECURE_VM,
 	SECURE_PARTITION,
@@ -908,6 +911,9 @@ static int ffa_sched_recv_cb_update(u16 part_id, ffa_sched_recv_cb callback,
 	struct ffa_dev_part_info *partition;
 	bool cb_valid;
 
+	if (ffa_notifications_disabled())
+		return -EOPNOTSUPP;
+
 	partition = xa_load(&drv_info->partition_info, part_id);
 	write_lock(&partition->rw_lock);
 
@@ -1001,6 +1007,9 @@ static int ffa_notify_relinquish(struct ffa_device *dev, int notify_id)
 	int rc;
 	enum notify_type type = ffa_notify_type_get(dev->vm_id);
 
+	if (ffa_notifications_disabled())
+		return -EOPNOTSUPP;
+
 	if (notify_id >= FFA_MAX_NOTIFICATIONS)
 		return -EINVAL;
 
@@ -1027,6 +1036,9 @@ static int ffa_notify_request(struct ffa_device *dev, bool is_per_vcpu,
 	u32 flags = 0;
 	enum notify_type type = ffa_notify_type_get(dev->vm_id);
 
+	if (ffa_notifications_disabled())
+		return -EOPNOTSUPP;
+
 	if (notify_id >= FFA_MAX_NOTIFICATIONS)
 		return -EINVAL;
 
@@ -1057,6 +1069,9 @@ static int ffa_notify_send(struct ffa_device *dev, int notify_id,
 {
 	u32 flags = 0;
 
+	if (ffa_notifications_disabled())
+		return -EOPNOTSUPP;
+
 	if (is_per_vcpu)
 		flags |= (PER_VCPU_NOTIFICATION_FLAG | vcpu << 16);
 
@@ -1233,7 +1248,7 @@ static void ffa_partitions_cleanup(void)
 	if (!count)
 		return;
 
-	info = kcalloc(count, sizeof(**info), GFP_KERNEL);
+	info = kcalloc(count, sizeof(*info), GFP_KERNEL);
 	if (!info)
 		return;
 
@@ -1311,8 +1326,10 @@ static int ffa_sched_recv_irq_map(void)
 
 static void ffa_sched_recv_irq_unmap(void)
 {
-	if (drv_info->sched_recv_irq)
+	if (drv_info->sched_recv_irq) {
 		irq_dispose_mapping(drv_info->sched_recv_irq);
+		drv_info->sched_recv_irq = 0;
+	}
 }
 
 static int ffa_cpuhp_pcpu_irq_enable(unsigned int cpu)
@@ -1329,17 +1346,23 @@ static int ffa_cpuhp_pcpu_irq_disable(unsigned int cpu)
 
 static void ffa_uninit_pcpu_irq(void)
 {
-	if (drv_info->cpuhp_state)
+	if (drv_info->cpuhp_state) {
 		cpuhp_remove_state(drv_info->cpuhp_state);
+		drv_info->cpuhp_state = 0;
+	}
 
-	if (drv_info->notif_pcpu_wq)
+	if (drv_info->notif_pcpu_wq) {
 		destroy_workqueue(drv_info->notif_pcpu_wq);
+		drv_info->notif_pcpu_wq = NULL;
+	}
 
 	if (drv_info->sched_recv_irq)
 		free_percpu_irq(drv_info->sched_recv_irq, drv_info->irq_pcpu);
 
-	if (drv_info->irq_pcpu)
+	if (drv_info->irq_pcpu) {
 		free_percpu(drv_info->irq_pcpu);
+		drv_info->irq_pcpu = NULL;
+	}
 }
 
 static int ffa_init_pcpu_irq(unsigned int irq)
@@ -1388,22 +1411,23 @@ static void ffa_notifications_cleanup(void)
 		ffa_notification_bitmap_destroy();
 		drv_info->bitmap_created = false;
 	}
+	drv_info->notif_enabled = false;
 }
 
-static int ffa_notifications_setup(void)
+static void ffa_notifications_setup(void)
 {
 	int ret, irq;
 
 	ret = ffa_features(FFA_NOTIFICATION_BITMAP_CREATE, 0, NULL, NULL);
 	if (ret) {
-		pr_err("Notifications not supported, continuing with it ..\n");
-		return 0;
+		pr_info("Notifications not supported, continuing with it ..\n");
+		return;
 	}
 
 	ret = ffa_notification_bitmap_create();
 	if (ret) {
-		pr_err("notification_bitmap_create error %d\n", ret);
-		return ret;
+		pr_info("Notification bitmap create error %d\n", ret);
+		return;
 	}
 	drv_info->bitmap_created = true;
 
@@ -1422,14 +1446,11 @@ static int ffa_notifications_setup(void)
 	hash_init(drv_info->notifier_hash);
 	mutex_init(&drv_info->notify_lock);
 
-	/* Register internal scheduling callback */
-	ret = ffa_sched_recv_cb_update(drv_info->vm_id, ffa_self_notif_handle,
-				       drv_info, true);
-	if (!ret)
-		return ret;
+	drv_info->notif_enabled = true;
+	return;
 cleanup:
+	pr_info("Notification setup failed %d, not enabled\n", ret);
 	ffa_notifications_cleanup();
-	return ret;
 }
 
 static int __init ffa_init(void)
@@ -1483,17 +1504,18 @@ static int __init ffa_init(void)
 	mutex_init(&drv_info->rx_lock);
 	mutex_init(&drv_info->tx_lock);
 
-	ffa_setup_partitions();
-
 	ffa_set_up_mem_ops_native_flag();
 
-	ret = ffa_notifications_setup();
+	ffa_notifications_setup();
+
+	ffa_setup_partitions();
+
+	ret = ffa_sched_recv_cb_update(drv_info->vm_id, ffa_self_notif_handle,
+				       drv_info, true);
 	if (ret)
-		goto partitions_cleanup;
+		pr_info("Failed to register driver sched callback %d\n", ret);
 
 	return 0;
-partitions_cleanup:
-	ffa_partitions_cleanup();
 free_pages:
 	if (drv_info->tx_buffer)
 		free_pages_exact(drv_info->tx_buffer, RXTX_BUFFER_SIZE);
diff --git a/drivers/firmware/arm_scmi/clock.c b/drivers/firmware/arm_scmi/clock.c
index 42b81c181d68..98511a3aa367 100644
--- a/drivers/firmware/arm_scmi/clock.c
+++ b/drivers/firmware/arm_scmi/clock.c
@@ -318,7 +318,7 @@ static int scmi_clock_attributes_get(const struct scmi_protocol_handle *ph,
 	if (!ret && PROTOCOL_REV_MAJOR(version) >= 0x2) {
 		if (SUPPORTS_EXTENDED_NAMES(attributes))
 			ph->hops->extended_name_get(ph, CLOCK_NAME_GET, clk_id,
-						    clk->name,
+						    NULL, clk->name,
 						    SCMI_MAX_STR_SIZE);
 
 		if (SUPPORTS_RATE_CHANGED_NOTIF(attributes))
diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c
index 09371f40d61f..3eb19ed6f148 100644
--- a/drivers/firmware/arm_scmi/driver.c
+++ b/drivers/firmware/arm_scmi/driver.c
@@ -1438,6 +1438,7 @@ struct scmi_msg_resp_domain_name_get {
  * @ph: A protocol handle reference.
  * @cmd_id: The specific command ID to use.
  * @res_id: The specific resource ID to use.
+ * @flags: A pointer to specific flags to use, if any.
  * @name: A pointer to the preallocated area where the retrieved name will be
  *	  stored as a NULL terminated string.
  * @len: The len in bytes of the @name char array.
@@ -1445,19 +1446,22 @@ struct scmi_msg_resp_domain_name_get {
  * Return: 0 on Succcess
  */
 static int scmi_common_extended_name_get(const struct scmi_protocol_handle *ph,
-					 u8 cmd_id, u32 res_id, char *name,
-					 size_t len)
+					 u8 cmd_id, u32 res_id, u32 *flags,
+					 char *name, size_t len)
 {
 	int ret;
+	size_t txlen;
 	struct scmi_xfer *t;
 	struct scmi_msg_resp_domain_name_get *resp;
 
-	ret = ph->xops->xfer_get_init(ph, cmd_id, sizeof(res_id),
-				      sizeof(*resp), &t);
+	txlen = !flags ? sizeof(res_id) : sizeof(res_id) + sizeof(*flags);
+	ret = ph->xops->xfer_get_init(ph, cmd_id, txlen, sizeof(*resp), &t);
 	if (ret)
 		goto out;
 
 	put_unaligned_le32(res_id, t->tx.buf);
+	if (flags)
+		put_unaligned_le32(*flags, t->tx.buf + sizeof(res_id));
 	resp = t->rx.buf;
 
 	ret = ph->xops->do_xfer(ph, t);
diff --git a/drivers/firmware/arm_scmi/optee.c b/drivers/firmware/arm_scmi/optee.c
index e123de6e8c67..25bfb465484d 100644
--- a/drivers/firmware/arm_scmi/optee.c
+++ b/drivers/firmware/arm_scmi/optee.c
@@ -440,6 +440,10 @@ static int scmi_optee_chan_setup(struct scmi_chan_info *cinfo, struct device *de
 	if (ret)
 		goto err_free_shm;
 
+	ret = tee_client_system_session(scmi_optee_private->tee_ctx, channel->tee_session);
+	if (ret)
+		dev_warn(dev, "Could not switch to system session, do best effort\n");
+
 	ret = get_channel(channel);
 	if (ret)
 		goto err_close_sess;
diff --git a/drivers/firmware/arm_scmi/perf.c b/drivers/firmware/arm_scmi/perf.c
index c2435be0ae1b..d1323c5d9c27 100644
--- a/drivers/firmware/arm_scmi/perf.c
+++ b/drivers/firmware/arm_scmi/perf.c
@@ -288,7 +288,7 @@ scmi_perf_domain_attributes_get(const struct scmi_protocol_handle *ph,
 	if (!ret && PROTOCOL_REV_MAJOR(version) >= 0x3 &&
 	    SUPPORTS_EXTENDED_NAMES(flags))
 		ph->hops->extended_name_get(ph, PERF_DOMAIN_NAME_GET,
-					    dom_info->id, dom_info->info.name,
+					    dom_info->id, NULL, dom_info->info.name,
 					    SCMI_MAX_STR_SIZE);
 
 	if (dom_info->level_indexing_mode) {
@@ -504,6 +504,9 @@ static int scmi_perf_limits_set(const struct scmi_protocol_handle *ph,
 	if (IS_ERR(dom))
 		return PTR_ERR(dom);
 
+	if (!dom->set_limits)
+		return -EOPNOTSUPP;
+
 	if (PROTOCOL_REV_MAJOR(pi->version) >= 0x3 && !max_perf && !min_perf)
 		return -EINVAL;
 
@@ -654,6 +657,9 @@ static int scmi_perf_level_set(const struct scmi_protocol_handle *ph,
 	if (IS_ERR(dom))
 		return PTR_ERR(dom);
 
+	if (!dom->info.set_perf)
+		return -EOPNOTSUPP;
+
 	if (dom->level_indexing_mode) {
 		struct scmi_opp *opp;
 
@@ -756,29 +762,36 @@ static void scmi_perf_domain_init_fc(const struct scmi_protocol_handle *ph,
 				     u32 domain, struct scmi_fc_info **p_fc)
 {
 	struct scmi_fc_info *fc;
+	struct perf_dom_info *dom;
+
+	dom = scmi_perf_domain_lookup(ph, domain);
+	if (IS_ERR(dom))
+		return;
 
 	fc = devm_kcalloc(ph->dev, PERF_FC_MAX, sizeof(*fc), GFP_KERNEL);
 	if (!fc)
 		return;
 
-	ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,
-				   PERF_LEVEL_SET, 4, domain,
-				   &fc[PERF_FC_LEVEL].set_addr,
-				   &fc[PERF_FC_LEVEL].set_db);
-
 	ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,
 				   PERF_LEVEL_GET, 4, domain,
 				   &fc[PERF_FC_LEVEL].get_addr, NULL);
 
-	ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,
-				   PERF_LIMITS_SET, 8, domain,
-				   &fc[PERF_FC_LIMIT].set_addr,
-				   &fc[PERF_FC_LIMIT].set_db);
-
 	ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,
 				   PERF_LIMITS_GET, 8, domain,
 				   &fc[PERF_FC_LIMIT].get_addr, NULL);
 
+	if (dom->info.set_perf)
+		ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,
+					   PERF_LEVEL_SET, 4, domain,
+					   &fc[PERF_FC_LEVEL].set_addr,
+					   &fc[PERF_FC_LEVEL].set_db);
+
+	if (dom->set_limits)
+		ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,
+					   PERF_LIMITS_SET, 8, domain,
+					   &fc[PERF_FC_LIMIT].set_addr,
+					   &fc[PERF_FC_LIMIT].set_db);
+
 	*p_fc = fc;
 }
 
diff --git a/drivers/firmware/arm_scmi/power.c b/drivers/firmware/arm_scmi/power.c
index 356e83631664..077767d6e902 100644
--- a/drivers/firmware/arm_scmi/power.c
+++ b/drivers/firmware/arm_scmi/power.c
@@ -133,7 +133,7 @@ scmi_power_domain_attributes_get(const struct scmi_protocol_handle *ph,
 	if (!ret && PROTOCOL_REV_MAJOR(version) >= 0x3 &&
 	    SUPPORTS_EXTENDED_NAMES(flags)) {
 		ph->hops->extended_name_get(ph, POWER_DOMAIN_NAME_GET,
-					    domain, dom_info->name,
+					    domain, NULL, dom_info->name,
 					    SCMI_MAX_STR_SIZE);
 	}
 
diff --git a/drivers/firmware/arm_scmi/powercap.c b/drivers/firmware/arm_scmi/powercap.c
index cb5617443a14..62a7780fedbb 100644
--- a/drivers/firmware/arm_scmi/powercap.c
+++ b/drivers/firmware/arm_scmi/powercap.c
@@ -270,7 +270,7 @@ scmi_powercap_domain_attributes_get(const struct scmi_protocol_handle *ph,
 	 */
 	if (!ret && SUPPORTS_EXTENDED_NAMES(flags))
 		ph->hops->extended_name_get(ph, POWERCAP_DOMAIN_NAME_GET,
-					    domain, dom_info->name,
+					    domain, NULL, dom_info->name,
 					    SCMI_MAX_STR_SIZE);
 
 	return ret;
diff --git a/drivers/firmware/arm_scmi/protocols.h b/drivers/firmware/arm_scmi/protocols.h
index 78e1a01eb656..b3c6314bb4b8 100644
--- a/drivers/firmware/arm_scmi/protocols.h
+++ b/drivers/firmware/arm_scmi/protocols.h
@@ -256,7 +256,8 @@ struct scmi_fc_info {
  */
 struct scmi_proto_helpers_ops {
 	int (*extended_name_get)(const struct scmi_protocol_handle *ph,
-				 u8 cmd_id, u32 res_id, char *name, size_t len);
+				 u8 cmd_id, u32 res_id, u32 *flags, char *name,
+				 size_t len);
 	void *(*iter_response_init)(const struct scmi_protocol_handle *ph,
 				    struct scmi_iterator_ops *ops,
 				    unsigned int max_resources, u8 msg_id,
diff --git a/drivers/firmware/arm_scmi/reset.c b/drivers/firmware/arm_scmi/reset.c
index e9afa8cab730..7217fd7c6afa 100644
--- a/drivers/firmware/arm_scmi/reset.c
+++ b/drivers/firmware/arm_scmi/reset.c
@@ -128,7 +128,8 @@ scmi_reset_domain_attributes_get(const struct scmi_protocol_handle *ph,
 	if (!ret && PROTOCOL_REV_MAJOR(version) >= 0x3 &&
 	    SUPPORTS_EXTENDED_NAMES(attributes))
 		ph->hops->extended_name_get(ph, RESET_DOMAIN_NAME_GET, domain,
-					    dom_info->name, SCMI_MAX_STR_SIZE);
+					    NULL, dom_info->name,
+					    SCMI_MAX_STR_SIZE);
 
 	return ret;
 }
diff --git a/drivers/firmware/arm_scmi/sensors.c b/drivers/firmware/arm_scmi/sensors.c
index 0b5853fa9d87..9952a7bc6682 100644
--- a/drivers/firmware/arm_scmi/sensors.c
+++ b/drivers/firmware/arm_scmi/sensors.c
@@ -644,7 +644,7 @@ iter_sens_descr_process_response(const struct scmi_protocol_handle *ph,
 	if (PROTOCOL_REV_MAJOR(si->version) >= 0x3 &&
 	    SUPPORTS_EXTENDED_NAMES(attrl))
 		ph->hops->extended_name_get(ph, SENSOR_NAME_GET, s->id,
-					    s->name, SCMI_MAX_STR_SIZE);
+					    NULL, s->name, SCMI_MAX_STR_SIZE);
 
 	if (s->extended_scalar_attrs) {
 		s->sensor_power = le32_to_cpu(sdesc->power);
diff --git a/drivers/firmware/arm_scmi/voltage.c b/drivers/firmware/arm_scmi/voltage.c
index eaa8d944926a..36e2df77738c 100644
--- a/drivers/firmware/arm_scmi/voltage.c
+++ b/drivers/firmware/arm_scmi/voltage.c
@@ -242,7 +242,7 @@ static int scmi_voltage_descriptors_get(const struct scmi_protocol_handle *ph,
 			if (SUPPORTS_EXTENDED_NAMES(attributes))
 				ph->hops->extended_name_get(ph,
 							VOLTAGE_DOMAIN_NAME_GET,
-							v->id, v->name,
+							v->id, NULL, v->name,
 							SCMI_MAX_STR_SIZE);
 			if (SUPPORTS_ASYNC_LEVEL_SET(attributes))
 				v->async_level_set = true;
diff --git a/drivers/firmware/google/framebuffer-coreboot.c b/drivers/firmware/google/framebuffer-coreboot.c
index c323a818805c..5c84bbebfef8 100644
--- a/drivers/firmware/google/framebuffer-coreboot.c
+++ b/drivers/firmware/google/framebuffer-coreboot.c
@@ -36,6 +36,9 @@ static int framebuffer_probe(struct coreboot_device *dev)
 		.format = NULL,
 	};
 
+	if (!fb->physical_address)
+		return -ENODEV;
+
 	for (i = 0; i < ARRAY_SIZE(formats); ++i) {
 		if (fb->bits_per_pixel     == formats[i].bits_per_pixel &&
 		    fb->red_mask_pos       == formats[i].red.offset &&
diff --git a/drivers/fpga/dfl.c b/drivers/fpga/dfl.c
index dd7a783d53b5..e69b9f1f2a50 100644
--- a/drivers/fpga/dfl.c
+++ b/drivers/fpga/dfl.c
@@ -2008,8 +2008,8 @@ long dfl_feature_ioctl_set_irq(struct platform_device *pdev,
 	    (hdr.start + hdr.count < hdr.start))
 		return -EINVAL;
 
-	fds = memdup_user((void __user *)(arg + sizeof(hdr)),
-			  array_size(hdr.count, sizeof(s32)));
+	fds = memdup_array_user((void __user *)(arg + sizeof(hdr)),
+				hdr.count, sizeof(s32));
 	if (IS_ERR(fds))
 		return PTR_ERR(fds);
 
diff --git a/drivers/gpio/gpio-elkhartlake.c b/drivers/gpio/gpio-elkhartlake.c
index a9c8b16215be..887c0fe99d39 100644
--- a/drivers/gpio/gpio-elkhartlake.c
+++ b/drivers/gpio/gpio-elkhartlake.c
@@ -55,18 +55,6 @@ static int ehl_gpio_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int ehl_gpio_suspend(struct device *dev)
-{
-	return tng_gpio_suspend(dev);
-}
-
-static int ehl_gpio_resume(struct device *dev)
-{
-	return tng_gpio_resume(dev);
-}
-
-static DEFINE_SIMPLE_DEV_PM_OPS(ehl_gpio_pm_ops, ehl_gpio_suspend, ehl_gpio_resume);
-
 static const struct platform_device_id ehl_gpio_ids[] = {
 	{ "gpio-elkhartlake" },
 	{ }
@@ -76,7 +64,7 @@ MODULE_DEVICE_TABLE(platform, ehl_gpio_ids);
 static struct platform_driver ehl_gpio_driver = {
 	.driver	= {
 		.name	= "gpio-elkhartlake",
-		.pm	= pm_sleep_ptr(&ehl_gpio_pm_ops),
+		.pm	= pm_sleep_ptr(&tng_gpio_pm_ops),
 	},
 	.probe		= ehl_gpio_probe,
 	.id_table	= ehl_gpio_ids,
diff --git a/drivers/gpio/gpio-mockup.c b/drivers/gpio/gpio-mockup.c
index 4870e267a402..455eecf6380e 100644
--- a/drivers/gpio/gpio-mockup.c
+++ b/drivers/gpio/gpio-mockup.c
@@ -354,7 +354,6 @@ static const struct file_operations gpio_mockup_debugfs_ops = {
 static void gpio_mockup_debugfs_setup(struct device *dev,
 				      struct gpio_mockup_chip *chip)
 {
-	struct device *child __free(put_device) = NULL;
 	struct gpio_mockup_dbgfs_private *priv;
 	struct gpio_chip *gc;
 	const char *devname;
@@ -367,7 +366,7 @@ static void gpio_mockup_debugfs_setup(struct device *dev,
 	 * There can only be a single GPIO device per platform device in
 	 * gpio-mockup so using device_find_any_child() is OK.
 	 */
-	child = device_find_any_child(dev);
+	struct device *child __free(put_device) = device_find_any_child(dev);
 	if (!child)
 		return;
 
diff --git a/drivers/gpio/gpio-sifive.c b/drivers/gpio/gpio-sifive.c
index 8decd9b5d229..067c8edb62e2 100644
--- a/drivers/gpio/gpio-sifive.c
+++ b/drivers/gpio/gpio-sifive.c
@@ -250,7 +250,6 @@ static int sifive_gpio_probe(struct platform_device *pdev)
 	girq->handler = handle_bad_irq;
 	girq->default_type = IRQ_TYPE_NONE;
 
-	platform_set_drvdata(pdev, chip);
 	return gpiochip_add_data(&chip->gc, chip);
 }
 
diff --git a/drivers/gpio/gpio-tangier.c b/drivers/gpio/gpio-tangier.c
index 7ce3eddaed25..b75e0b12087a 100644
--- a/drivers/gpio/gpio-tangier.c
+++ b/drivers/gpio/gpio-tangier.c
@@ -10,6 +10,7 @@
  */
 
 #include <linux/bitops.h>
+#include <linux/cleanup.h>
 #include <linux/device.h>
 #include <linux/errno.h>
 #include <linux/export.h>
@@ -19,6 +20,7 @@
 #include <linux/math.h>
 #include <linux/module.h>
 #include <linux/pinctrl/pinconf-generic.h>
+#include <linux/pm.h>
 #include <linux/spinlock.h>
 #include <linux/string_helpers.h>
 #include <linux/types.h>
@@ -91,37 +93,31 @@ static int tng_gpio_get(struct gpio_chip *chip, unsigned int offset)
 static void tng_gpio_set(struct gpio_chip *chip, unsigned int offset, int value)
 {
 	struct tng_gpio *priv = gpiochip_get_data(chip);
-	unsigned long flags;
 	void __iomem *reg;
 	u8 shift;
 
 	reg = gpio_reg_and_bit(chip, offset, value ? GPSR : GPCR, &shift);
 
-	raw_spin_lock_irqsave(&priv->lock, flags);
+	guard(raw_spinlock_irqsave)(&priv->lock);
 
 	writel(BIT(shift), reg);
-
-	raw_spin_unlock_irqrestore(&priv->lock, flags);
 }
 
 static int tng_gpio_direction_input(struct gpio_chip *chip, unsigned int offset)
 {
 	struct tng_gpio *priv = gpiochip_get_data(chip);
-	unsigned long flags;
 	void __iomem *gpdr;
 	u32 value;
 	u8 shift;
 
 	gpdr = gpio_reg_and_bit(chip, offset, GPDR, &shift);
 
-	raw_spin_lock_irqsave(&priv->lock, flags);
+	guard(raw_spinlock_irqsave)(&priv->lock);
 
 	value = readl(gpdr);
 	value &= ~BIT(shift);
 	writel(value, gpdr);
 
-	raw_spin_unlock_irqrestore(&priv->lock, flags);
-
 	return 0;
 }
 
@@ -129,21 +125,18 @@ static int tng_gpio_direction_output(struct gpio_chip *chip, unsigned int offset
 				     int value)
 {
 	struct tng_gpio *priv = gpiochip_get_data(chip);
-	unsigned long flags;
 	void __iomem *gpdr;
 	u8 shift;
 
 	gpdr = gpio_reg_and_bit(chip, offset, GPDR, &shift);
 	tng_gpio_set(chip, offset, value);
 
-	raw_spin_lock_irqsave(&priv->lock, flags);
+	guard(raw_spinlock_irqsave)(&priv->lock);
 
 	value = readl(gpdr);
 	value |= BIT(shift);
 	writel(value, gpdr);
 
-	raw_spin_unlock_irqrestore(&priv->lock, flags);
-
 	return 0;
 }
 
@@ -164,14 +157,13 @@ static int tng_gpio_set_debounce(struct gpio_chip *chip, unsigned int offset,
 				 unsigned int debounce)
 {
 	struct tng_gpio *priv = gpiochip_get_data(chip);
-	unsigned long flags;
 	void __iomem *gfbr;
 	u32 value;
 	u8 shift;
 
 	gfbr = gpio_reg_and_bit(chip, offset, GFBR, &shift);
 
-	raw_spin_lock_irqsave(&priv->lock, flags);
+	guard(raw_spinlock_irqsave)(&priv->lock);
 
 	value = readl(gfbr);
 	if (debounce)
@@ -180,8 +172,6 @@ static int tng_gpio_set_debounce(struct gpio_chip *chip, unsigned int offset,
 		value |= BIT(shift);
 	writel(value, gfbr);
 
-	raw_spin_unlock_irqrestore(&priv->lock, flags);
-
 	return 0;
 }
 
@@ -207,27 +197,25 @@ static void tng_irq_ack(struct irq_data *d)
 {
 	struct tng_gpio *priv = irq_data_get_irq_chip_data(d);
 	irq_hw_number_t gpio = irqd_to_hwirq(d);
-	unsigned long flags;
 	void __iomem *gisr;
 	u8 shift;
 
 	gisr = gpio_reg_and_bit(&priv->chip, gpio, GISR, &shift);
 
-	raw_spin_lock_irqsave(&priv->lock, flags);
+	guard(raw_spinlock_irqsave)(&priv->lock);
+
 	writel(BIT(shift), gisr);
-	raw_spin_unlock_irqrestore(&priv->lock, flags);
 }
 
 static void tng_irq_unmask_mask(struct tng_gpio *priv, u32 gpio, bool unmask)
 {
-	unsigned long flags;
 	void __iomem *gimr;
 	u32 value;
 	u8 shift;
 
 	gimr = gpio_reg_and_bit(&priv->chip, gpio, GIMR, &shift);
 
-	raw_spin_lock_irqsave(&priv->lock, flags);
+	guard(raw_spinlock_irqsave)(&priv->lock);
 
 	value = readl(gimr);
 	if (unmask)
@@ -235,8 +223,6 @@ static void tng_irq_unmask_mask(struct tng_gpio *priv, u32 gpio, bool unmask)
 	else
 		value &= ~BIT(shift);
 	writel(value, gimr);
-
-	raw_spin_unlock_irqrestore(&priv->lock, flags);
 }
 
 static void tng_irq_mask(struct irq_data *d)
@@ -267,10 +253,9 @@ static int tng_irq_set_type(struct irq_data *d, unsigned int type)
 	void __iomem *gitr = gpio_reg(&priv->chip, gpio, GITR);
 	void __iomem *glpr = gpio_reg(&priv->chip, gpio, GLPR);
 	u8 shift = gpio % 32;
-	unsigned long flags;
 	u32 value;
 
-	raw_spin_lock_irqsave(&priv->lock, flags);
+	guard(raw_spinlock_irqsave)(&priv->lock);
 
 	value = readl(grer);
 	if (type & IRQ_TYPE_EDGE_RISING)
@@ -311,8 +296,6 @@ static int tng_irq_set_type(struct irq_data *d, unsigned int type)
 		irq_set_handler_locked(d, handle_edge_irq);
 	}
 
-	raw_spin_unlock_irqrestore(&priv->lock, flags);
-
 	return 0;
 }
 
@@ -324,10 +307,11 @@ static int tng_irq_set_wake(struct irq_data *d, unsigned int on)
 	void __iomem *gwmr = gpio_reg(&priv->chip, gpio, priv->wake_regs.gwmr);
 	void __iomem *gwsr = gpio_reg(&priv->chip, gpio, priv->wake_regs.gwsr);
 	u8 shift = gpio % 32;
-	unsigned long flags;
 	u32 value;
 
-	raw_spin_lock_irqsave(&priv->lock, flags);
+	dev_dbg(priv->dev, "%s wake for gpio %lu\n", str_enable_disable(on), gpio);
+
+	guard(raw_spinlock_irqsave)(&priv->lock);
 
 	/* Clear the existing wake status */
 	writel(BIT(shift), gwsr);
@@ -339,9 +323,6 @@ static int tng_irq_set_wake(struct irq_data *d, unsigned int on)
 		value &= ~BIT(shift);
 	writel(value, gwmr);
 
-	raw_spin_unlock_irqrestore(&priv->lock, flags);
-
-	dev_dbg(priv->dev, "%s wake for gpio %lu\n", str_enable_disable(on), gpio);
 	return 0;
 }
 
@@ -477,14 +458,13 @@ int devm_tng_gpio_probe(struct device *dev, struct tng_gpio *gpio)
 }
 EXPORT_SYMBOL_NS_GPL(devm_tng_gpio_probe, GPIO_TANGIER);
 
-int tng_gpio_suspend(struct device *dev)
+static int tng_gpio_suspend(struct device *dev)
 {
 	struct tng_gpio *priv = dev_get_drvdata(dev);
 	struct tng_gpio_context *ctx = priv->ctx;
-	unsigned long flags;
 	unsigned int base;
 
-	raw_spin_lock_irqsave(&priv->lock, flags);
+	guard(raw_spinlock_irqsave)(&priv->lock);
 
 	for (base = 0; base < priv->chip.ngpio; base += 32, ctx++) {
 		/* GPLR is RO, values read will be restored using GPSR */
@@ -498,20 +478,16 @@ int tng_gpio_suspend(struct device *dev)
 		ctx->gwmr = readl(gpio_reg(&priv->chip, base, priv->wake_regs.gwmr));
 	}
 
-	raw_spin_unlock_irqrestore(&priv->lock, flags);
-
 	return 0;
 }
-EXPORT_SYMBOL_NS_GPL(tng_gpio_suspend, GPIO_TANGIER);
 
-int tng_gpio_resume(struct device *dev)
+static int tng_gpio_resume(struct device *dev)
 {
 	struct tng_gpio *priv = dev_get_drvdata(dev);
 	struct tng_gpio_context *ctx = priv->ctx;
-	unsigned long flags;
 	unsigned int base;
 
-	raw_spin_lock_irqsave(&priv->lock, flags);
+	guard(raw_spinlock_irqsave)(&priv->lock);
 
 	for (base = 0; base < priv->chip.ngpio; base += 32, ctx++) {
 		/* GPLR is RO, values read will be restored using GPSR */
@@ -525,11 +501,10 @@ int tng_gpio_resume(struct device *dev)
 		writel(ctx->gwmr, gpio_reg(&priv->chip, base, priv->wake_regs.gwmr));
 	}
 
-	raw_spin_unlock_irqrestore(&priv->lock, flags);
-
 	return 0;
 }
-EXPORT_SYMBOL_NS_GPL(tng_gpio_resume, GPIO_TANGIER);
+
+EXPORT_NS_GPL_SIMPLE_DEV_PM_OPS(tng_gpio_pm_ops, tng_gpio_suspend, tng_gpio_resume, GPIO_TANGIER);
 
 MODULE_AUTHOR("Andy Shevchenko <andriy.shevchenko@linux.intel.com>");
 MODULE_AUTHOR("Pandith N <pandith.n@intel.com>");
diff --git a/drivers/gpio/gpio-tangier.h b/drivers/gpio/gpio-tangier.h
index 16c4f22908fb..ca7ab6cf6fa5 100644
--- a/drivers/gpio/gpio-tangier.h
+++ b/drivers/gpio/gpio-tangier.h
@@ -13,6 +13,7 @@
 #define _GPIO_TANGIER_H_
 
 #include <linux/gpio/driver.h>
+#include <linux/pm.h>
 #include <linux/spinlock_types.h>
 #include <linux/types.h>
 
@@ -111,7 +112,6 @@ struct tng_gpio {
 
 int devm_tng_gpio_probe(struct device *dev, struct tng_gpio *gpio);
 
-int tng_gpio_suspend(struct device *dev);
-int tng_gpio_resume(struct device *dev);
+extern const struct dev_pm_ops tng_gpio_pm_ops;
 
 #endif /* _GPIO_TANGIER_H_ */
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index 3eee8636f847..cdbc56e07649 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -75,15 +75,15 @@ config DRM_KUNIT_TEST_HELPERS
 config DRM_KUNIT_TEST
 	tristate "KUnit tests for DRM" if !KUNIT_ALL_TESTS
 	depends on DRM && KUNIT
-	select PRIME_NUMBERS
+	select DRM_BUDDY
 	select DRM_DISPLAY_DP_HELPER
 	select DRM_DISPLAY_HELPER
-	select DRM_LIB_RANDOM
-	select DRM_KMS_HELPER
-	select DRM_BUDDY
+	select DRM_EXEC
 	select DRM_EXPORT_FOR_TESTS if m
+	select DRM_KMS_HELPER
 	select DRM_KUNIT_TEST_HELPERS
-	select DRM_EXEC
+	select DRM_LIB_RANDOM
+	select PRIME_NUMBERS
 	default KUNIT_ALL_TESTS
 	help
 	  This builds unit tests for DRM. This option is not useful for
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index 8e1bde059170..cdbe91ac0bfc 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -22,6 +22,7 @@ drm-y := \
 	drm_drv.o \
 	drm_dumb_buffers.o \
 	drm_edid.o \
+	drm_eld.o \
 	drm_encoder.o \
 	drm_file.o \
 	drm_fourcc.o \
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 2afecc55090f..260e32ef7bae 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -80,7 +80,7 @@ amdgpu-y += amdgpu_device.o amdgpu_doorbell_mgr.o amdgpu_kms.o \
 	amdgpu_umc.o smu_v11_0_i2c.o amdgpu_fru_eeprom.o amdgpu_rap.o \
 	amdgpu_fw_attestation.o amdgpu_securedisplay.o \
 	amdgpu_eeprom.o amdgpu_mca.o amdgpu_psp_ta.o amdgpu_lsdma.o \
-	amdgpu_ring_mux.o amdgpu_xcp.o
+	amdgpu_ring_mux.o amdgpu_xcp.o amdgpu_seq64.o
 
 amdgpu-$(CONFIG_PROC_FS) += amdgpu_fdinfo.o
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 9d92ca157677..4d100f8d75bf 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -109,6 +109,7 @@
 #include "amdgpu_mca.h"
 #include "amdgpu_ras.h"
 #include "amdgpu_xcp.h"
+#include "amdgpu_seq64.h"
 
 #define MAX_GPU_INSTANCE		64
 
@@ -468,6 +469,7 @@ struct amdgpu_fpriv {
 	struct amdgpu_vm	vm;
 	struct amdgpu_bo_va	*prt_va;
 	struct amdgpu_bo_va	*csa_va;
+	struct amdgpu_bo_va	*seq64_va;
 	struct mutex		bo_list_lock;
 	struct idr		bo_list_handles;
 	struct amdgpu_ctx_mgr	ctx_mgr;
@@ -986,6 +988,9 @@ struct amdgpu_device {
 	/* GDS */
 	struct amdgpu_gds		gds;
 
+	/* for userq and VM fences */
+	struct amdgpu_seq64		seq64;
+
 	/* KFD */
 	struct amdgpu_kfd_dev		kfd;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index b8412202a1b0..6ab17330a6ed 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -710,35 +710,6 @@ bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid)
 	return false;
 }
 
-int amdgpu_amdkfd_flush_gpu_tlb_vmid(struct amdgpu_device *adev,
-				     uint16_t vmid)
-{
-	if (adev->family == AMDGPU_FAMILY_AI) {
-		int i;
-
-		for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS)
-			amdgpu_gmc_flush_gpu_tlb(adev, vmid, i, 0);
-	} else {
-		amdgpu_gmc_flush_gpu_tlb(adev, vmid, AMDGPU_GFXHUB(0), 0);
-	}
-
-	return 0;
-}
-
-int amdgpu_amdkfd_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
-				      uint16_t pasid,
-				      enum TLB_FLUSH_TYPE flush_type,
-				      uint32_t inst)
-{
-	bool all_hub = false;
-
-	if (adev->family == AMDGPU_FAMILY_AI ||
-	    adev->family == AMDGPU_FAMILY_RV)
-		all_hub = true;
-
-	return amdgpu_gmc_flush_gpu_tlb_pasid(adev, pasid, flush_type, all_hub, inst);
-}
-
 bool amdgpu_amdkfd_have_atomics_support(struct amdgpu_device *adev)
 {
 	return adev->have_atomics_support;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index dac983da961d..16794c2eea35 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -162,11 +162,6 @@ int amdgpu_amdkfd_submit_ib(struct amdgpu_device *adev,
 				uint32_t *ib_cmd, uint32_t ib_len);
 void amdgpu_amdkfd_set_compute_idle(struct amdgpu_device *adev, bool idle);
 bool amdgpu_amdkfd_have_atomics_support(struct amdgpu_device *adev);
-int amdgpu_amdkfd_flush_gpu_tlb_vmid(struct amdgpu_device *adev,
-				uint16_t vmid);
-int amdgpu_amdkfd_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
-				uint16_t pasid, enum TLB_FLUSH_TYPE flush_type,
-				uint32_t inst);
 
 bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c
index 625db444df1c..899e31e3a5e8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c
@@ -200,7 +200,7 @@ int kgd_arcturus_hqd_sdma_dump(struct amdgpu_device *adev,
 #undef HQD_N_REGS
 #define HQD_N_REGS (19+6+7+10)
 
-	*dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL);
+	*dump = kmalloc_array(HQD_N_REGS, sizeof(**dump), GFP_KERNEL);
 	if (*dump == NULL)
 		return -ENOMEM;
 
@@ -290,7 +290,7 @@ static int suspend_resume_compute_scheduler(struct amdgpu_device *adev, bool sus
 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 		struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
 
-		if (!(ring && ring->sched.thread))
+		if (!(ring && drm_sched_wqueue_ready(&ring->sched)))
 			continue;
 
 		/* stop secheduler and drain ring. */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c
index f6598b9e4faa..a5c7259cf2a3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c
@@ -141,7 +141,7 @@ static int kgd_gfx_v9_4_3_hqd_sdma_dump(struct amdgpu_device *adev,
 		(*dump)[i++][1] = RREG32(addr);         \
 	} while (0)
 
-	*dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL);
+	*dump = kmalloc_array(HQD_N_REGS, sizeof(**dump), GFP_KERNEL);
 	if (*dump == NULL)
 		return -ENOMEM;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
index 6bf448ab3dff..ca4a6b82817f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
@@ -214,7 +214,7 @@ static int kgd_hqd_dump(struct amdgpu_device *adev,
 		(*dump)[i++][1] = RREG32(addr);		\
 	} while (0)
 
-	*dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL);
+	*dump = kmalloc_array(HQD_N_REGS, sizeof(**dump), GFP_KERNEL);
 	if (*dump == NULL)
 		return -ENOMEM;
 
@@ -301,7 +301,7 @@ static int kgd_hqd_sdma_dump(struct amdgpu_device *adev,
 #undef HQD_N_REGS
 #define HQD_N_REGS (19+4)
 
-	*dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL);
+	*dump = kmalloc_array(HQD_N_REGS, sizeof(**dump), GFP_KERNEL);
 	if (*dump == NULL)
 		return -ENOMEM;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
index cd06e4a6d1da..0f3e2944edd7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
@@ -238,7 +238,7 @@ static int kgd_hqd_dump(struct amdgpu_device *adev,
 		(*dump)[i++][1] = RREG32(addr);		\
 	} while (0)
 
-	*dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL);
+	*dump = kmalloc_array(HQD_N_REGS, sizeof(**dump), GFP_KERNEL);
 	if (*dump == NULL)
 		return -ENOMEM;
 
@@ -324,7 +324,7 @@ static int kgd_hqd_sdma_dump(struct amdgpu_device *adev,
 #undef HQD_N_REGS
 #define HQD_N_REGS (19+4+2+3+7)
 
-	*dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL);
+	*dump = kmalloc_array(HQD_N_REGS, sizeof(**dump), GFP_KERNEL);
 	if (*dump == NULL)
 		return -ENOMEM;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
index 00fbc0f44c92..5a35a8ca8922 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
@@ -363,7 +363,7 @@ int kgd_gfx_v9_hqd_dump(struct amdgpu_device *adev,
 		(*dump)[i++][1] = RREG32(addr);		\
 	} while (0)
 
-	*dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL);
+	*dump = kmalloc_array(HQD_N_REGS, sizeof(**dump), GFP_KERNEL);
 	if (*dump == NULL)
 		return -ENOMEM;
 
@@ -460,7 +460,7 @@ static int kgd_hqd_sdma_dump(struct amdgpu_device *adev,
 #undef HQD_N_REGS
 #define HQD_N_REGS (19+6+7+10)
 
-	*dump = kmalloc_array(HQD_N_REGS * 2, sizeof(uint32_t), GFP_KERNEL);
+	*dump = kmalloc_array(HQD_N_REGS, sizeof(**dump), GFP_KERNEL);
 	if (*dump == NULL)
 		return -ENOMEM;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 41fbc4fd0fac..b98c0969d3be 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -2825,12 +2825,6 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef)
 	if (ret)
 		goto validate_map_fail;
 
-	ret = process_sync_pds_resv(process_info, &sync_obj);
-	if (ret) {
-		pr_debug("Memory eviction: Failed to sync to PD BO moving fence. Try again\n");
-		goto validate_map_fail;
-	}
-
 	/* Validate BOs and map them to GPUVM (update VM page tables). */
 	list_for_each_entry(mem, &process_info->kfd_bo_list,
 			    validate_list) {
@@ -2881,6 +2875,19 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef)
 	if (failed_size)
 		pr_debug("0x%lx/0x%lx in system\n", failed_size, total_size);
 
+	/* Update mappings not managed by KFD */
+	list_for_each_entry(peer_vm, &process_info->vm_list_head,
+			vm_list_node) {
+		struct amdgpu_device *adev = amdgpu_ttm_adev(
+			peer_vm->root.bo->tbo.bdev);
+
+		ret = amdgpu_vm_handle_moved(adev, peer_vm, &exec.ticket);
+		if (ret) {
+			pr_debug("Memory eviction: handle moved failed. Try again\n");
+			goto validate_map_fail;
+		}
+	}
+
 	/* Update page directories */
 	ret = process_update_pds(process_info, &sync_obj);
 	if (ret) {
@@ -2888,6 +2895,15 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef)
 		goto validate_map_fail;
 	}
 
+	/* Sync with fences on all the page tables. They implicitly depend on any
+	 * move fences from amdgpu_vm_handle_moved above.
+	 */
+	ret = process_sync_pds_resv(process_info, &sync_obj);
+	if (ret) {
+		pr_debug("Memory eviction: Failed to sync to PD BO moving fence. Try again\n");
+		goto validate_map_fail;
+	}
+
 	/* Wait for validate and PT updates to finish */
 	amdgpu_sync_wait(&sync_obj, false);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
index 7473a42f7d45..96f63fd39b9e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
@@ -103,7 +103,7 @@ int amdgpu_connector_get_monitor_bpc(struct drm_connector *connector)
 	struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
 	struct amdgpu_connector_atom_dig *dig_connector;
 	int bpc = 8;
-	unsigned mode_clock, max_tmds_clock;
+	unsigned int mode_clock, max_tmds_clock;
 
 	switch (connector->connector_type) {
 	case DRM_MODE_CONNECTOR_DVII:
@@ -255,6 +255,7 @@ struct edid *amdgpu_connector_edid(struct drm_connector *connector)
 		return amdgpu_connector->edid;
 	} else if (edid_blob) {
 		struct edid *edid = kmemdup(edid_blob->data, edid_blob->length, GFP_KERNEL);
+
 		if (edid)
 			amdgpu_connector->edid = edid;
 	}
@@ -581,6 +582,7 @@ static int amdgpu_connector_set_property(struct drm_connector *connector,
 			amdgpu_encoder = to_amdgpu_encoder(connector->encoder);
 		} else {
 			const struct drm_connector_helper_funcs *connector_funcs = connector->helper_private;
+
 			amdgpu_encoder = to_amdgpu_encoder(connector_funcs->best_encoder(connector));
 		}
 
@@ -797,6 +799,7 @@ static int amdgpu_connector_set_lcd_property(struct drm_connector *connector,
 		amdgpu_encoder = to_amdgpu_encoder(connector->encoder);
 	else {
 		const struct drm_connector_helper_funcs *connector_funcs = connector->helper_private;
+
 		amdgpu_encoder = to_amdgpu_encoder(connector_funcs->best_encoder(connector));
 	}
 
@@ -979,6 +982,41 @@ amdgpu_connector_check_hpd_status_unchanged(struct drm_connector *connector)
 	return false;
 }
 
+static void amdgpu_connector_shared_ddc(enum drm_connector_status *status,
+					struct drm_connector *connector,
+					struct amdgpu_connector *amdgpu_connector)
+{
+	struct drm_connector *list_connector;
+	struct drm_connector_list_iter iter;
+	struct amdgpu_connector *list_amdgpu_connector;
+	struct drm_device *dev = connector->dev;
+	struct amdgpu_device *adev = drm_to_adev(dev);
+
+	if (amdgpu_connector->shared_ddc && *status == connector_status_connected) {
+		drm_connector_list_iter_begin(dev, &iter);
+		drm_for_each_connector_iter(list_connector,
+					    &iter) {
+			if (connector == list_connector)
+				continue;
+			list_amdgpu_connector = to_amdgpu_connector(list_connector);
+			if (list_amdgpu_connector->shared_ddc &&
+			    list_amdgpu_connector->ddc_bus->rec.i2c_id ==
+			     amdgpu_connector->ddc_bus->rec.i2c_id) {
+				/* cases where both connectors are digital */
+				if (list_connector->connector_type != DRM_MODE_CONNECTOR_VGA) {
+					/* hpd is our only option in this case */
+					if (!amdgpu_display_hpd_sense(adev,
+								      amdgpu_connector->hpd.hpd)) {
+						amdgpu_connector_free_edid(connector);
+						*status = connector_status_disconnected;
+					}
+				}
+			}
+		}
+		drm_connector_list_iter_end(&iter);
+	}
+}
+
 /*
  * DVI is complicated
  * Do a DDC probe, if DDC probe passes, get the full EDID so
@@ -1065,32 +1103,7 @@ amdgpu_connector_dvi_detect(struct drm_connector *connector, bool force)
 			 * DDC line.  The latter is more complex because with DVI<->HDMI adapters
 			 * you don't really know what's connected to which port as both are digital.
 			 */
-			if (amdgpu_connector->shared_ddc && (ret == connector_status_connected)) {
-				struct drm_connector *list_connector;
-				struct drm_connector_list_iter iter;
-				struct amdgpu_connector *list_amdgpu_connector;
-
-				drm_connector_list_iter_begin(dev, &iter);
-				drm_for_each_connector_iter(list_connector,
-							    &iter) {
-					if (connector == list_connector)
-						continue;
-					list_amdgpu_connector = to_amdgpu_connector(list_connector);
-					if (list_amdgpu_connector->shared_ddc &&
-					    (list_amdgpu_connector->ddc_bus->rec.i2c_id ==
-					     amdgpu_connector->ddc_bus->rec.i2c_id)) {
-						/* cases where both connectors are digital */
-						if (list_connector->connector_type != DRM_MODE_CONNECTOR_VGA) {
-							/* hpd is our only option in this case */
-							if (!amdgpu_display_hpd_sense(adev, amdgpu_connector->hpd.hpd)) {
-								amdgpu_connector_free_edid(connector);
-								ret = connector_status_disconnected;
-							}
-						}
-					}
-				}
-				drm_connector_list_iter_end(&iter);
-			}
+			 amdgpu_connector_shared_ddc(&ret, connector, amdgpu_connector);
 		}
 	}
 
@@ -1192,6 +1205,7 @@ amdgpu_connector_dvi_encoder(struct drm_connector *connector)
 static void amdgpu_connector_dvi_force(struct drm_connector *connector)
 {
 	struct amdgpu_connector *amdgpu_connector = to_amdgpu_connector(connector);
+
 	if (connector->force == DRM_FORCE_ON)
 		amdgpu_connector->use_digital = false;
 	if (connector->force == DRM_FORCE_ON_DIGITAL)
@@ -1426,6 +1440,7 @@ amdgpu_connector_dp_detect(struct drm_connector *connector, bool force)
 				ret = connector_status_connected;
 			else if (amdgpu_connector->dac_load_detect) { /* try load detection */
 				const struct drm_encoder_helper_funcs *encoder_funcs = encoder->helper_private;
+
 				ret = encoder_funcs->detect(encoder, connector);
 			}
 		}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
index a53f436fa9f1..c1efa13bccbb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
@@ -1665,9 +1665,9 @@ static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
 	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
 		struct amdgpu_ring *ring = adev->rings[i];
 
-		if (!ring || !ring->sched.thread)
+		if (!ring || !drm_sched_wqueue_ready(&ring->sched))
 			continue;
-		kthread_park(ring->sched.thread);
+		drm_sched_wqueue_stop(&ring->sched);
 	}
 
 	seq_puts(m, "run ib test:\n");
@@ -1681,9 +1681,9 @@ static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
 	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
 		struct amdgpu_ring *ring = adev->rings[i];
 
-		if (!ring || !ring->sched.thread)
+		if (!ring || !drm_sched_wqueue_ready(&ring->sched))
 			continue;
-		kthread_unpark(ring->sched.thread);
+		drm_sched_wqueue_start(&ring->sched);
 	}
 
 	up_write(&adev->reset_domain->sem);
@@ -1903,7 +1903,8 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
 
 	ring = adev->rings[val];
 
-	if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
+	if (!ring || !ring->funcs->preempt_ib ||
+	    !drm_sched_wqueue_ready(&ring->sched))
 		return -EINVAL;
 
 	/* the last preemption failed */
@@ -1921,7 +1922,7 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
 		goto pro_end;
 
 	/* stop the scheduler */
-	kthread_park(ring->sched.thread);
+	drm_sched_wqueue_stop(&ring->sched);
 
 	/* preempt the IB */
 	r = amdgpu_ring_preempt_ib(ring);
@@ -1955,7 +1956,7 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
 
 failure:
 	/* restart the scheduler */
-	kthread_unpark(ring->sched.thread);
+	drm_sched_wqueue_start(&ring->sched);
 
 	up_read(&adev->reset_domain->sem);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 7eeaf0aa7f81..c0b3898f850b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -1551,11 +1551,15 @@ bool amdgpu_device_seamless_boot_supported(struct amdgpu_device *adev)
  * https://edc.intel.com/content/www/us/en/design/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-2/005/pci-express-support/
  * https://gitlab.freedesktop.org/drm/amd/-/issues/2663
  */
-static bool amdgpu_device_pcie_dynamic_switching_supported(void)
+static bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev)
 {
 #if IS_ENABLED(CONFIG_X86)
 	struct cpuinfo_x86 *c = &cpu_data(0);
 
+	/* eGPU change speeds based on USB4 fabric conditions */
+	if (dev_is_removable(adev->dev))
+		return true;
+
 	if (c->x86_vendor == X86_VENDOR_INTEL)
 		return false;
 #endif
@@ -2395,7 +2399,7 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
 		adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
 	if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
 		adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
-	if (!amdgpu_device_pcie_dynamic_switching_supported())
+	if (!amdgpu_device_pcie_dynamic_switching_supported(adev))
 		adev->pm.pp_feature &= ~PP_PCIE_DPM_MASK;
 
 	total = true;
@@ -2573,7 +2577,7 @@ static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
 			break;
 		}
 
-		r = drm_sched_init(&ring->sched, &amdgpu_sched_ops,
+		r = drm_sched_init(&ring->sched, &amdgpu_sched_ops, NULL,
 				   DRM_SCHED_PRIORITY_COUNT,
 				   ring->num_hw_submission, 0,
 				   timeout, adev->reset_domain->wq,
@@ -2676,6 +2680,12 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev)
 					goto init_failed;
 				}
 			}
+
+			r = amdgpu_seq64_init(adev);
+			if (r) {
+				DRM_ERROR("allocate seq64 failed %d\n", r);
+				goto init_failed;
+			}
 		}
 	}
 
@@ -3138,6 +3148,7 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
 			amdgpu_device_wb_fini(adev);
 			amdgpu_device_mem_scratch_fini(adev);
 			amdgpu_ib_pool_fini(adev);
+			amdgpu_seq64_fini(adev);
 		}
 
 		r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
@@ -4964,7 +4975,7 @@ bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
 		struct amdgpu_ring *ring = adev->rings[i];
 
-		if (!ring || !ring->sched.thread)
+		if (!ring || !drm_sched_wqueue_ready(&ring->sched))
 			continue;
 
 		spin_lock(&ring->sched.job_list_lock);
@@ -5103,7 +5114,7 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
 		struct amdgpu_ring *ring = adev->rings[i];
 
-		if (!ring || !ring->sched.thread)
+		if (!ring || !drm_sched_wqueue_ready(&ring->sched))
 			continue;
 
 		/* Clear job fence from fence drv to avoid force_completion
@@ -5592,7 +5603,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
 			struct amdgpu_ring *ring = tmp_adev->rings[i];
 
-			if (!ring || !ring->sched.thread)
+			if (!ring || !drm_sched_wqueue_ready(&ring->sched))
 				continue;
 
 			drm_sched_stop(&ring->sched, job ? &job->base : NULL);
@@ -5668,7 +5679,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
 			struct amdgpu_ring *ring = tmp_adev->rings[i];
 
-			if (!ring || !ring->sched.thread)
+			if (!ring || !drm_sched_wqueue_ready(&ring->sched))
 				continue;
 
 			drm_sched_start(&ring->sched, true);
@@ -5730,6 +5741,39 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 	return r;
 }
 
+/**
+ * amdgpu_device_partner_bandwidth - find the bandwidth of appropriate partner
+ *
+ * @adev: amdgpu_device pointer
+ * @speed: pointer to the speed of the link
+ * @width: pointer to the width of the link
+ *
+ * Evaluate the hierarchy to find the speed and bandwidth capabilities of the
+ * first physical partner to an AMD dGPU.
+ * This will exclude any virtual switches and links.
+ */
+static void amdgpu_device_partner_bandwidth(struct amdgpu_device *adev,
+					    enum pci_bus_speed *speed,
+					    enum pcie_link_width *width)
+{
+	struct pci_dev *parent = adev->pdev;
+
+	if (!speed || !width)
+		return;
+
+	*speed = PCI_SPEED_UNKNOWN;
+	*width = PCIE_LNK_WIDTH_UNKNOWN;
+
+	while ((parent = pci_upstream_bridge(parent))) {
+		/* skip upstream/downstream switches internal to dGPU*/
+		if (parent->vendor == PCI_VENDOR_ID_ATI)
+			continue;
+		*speed = pcie_get_speed_cap(parent);
+		*width = pcie_get_width_cap(parent);
+		break;
+	}
+}
+
 /**
  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
  *
@@ -5763,8 +5807,8 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
 	if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
 		return;
 
-	pcie_bandwidth_available(adev->pdev, NULL,
-				 &platform_speed_cap, &platform_link_width);
+	amdgpu_device_partner_bandwidth(adev, &platform_speed_cap,
+					&platform_link_width);
 
 	if (adev->pm.pcie_gen_mask == 0) {
 		/* asic caps */
@@ -5991,7 +6035,7 @@ pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_sta
 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
 			struct amdgpu_ring *ring = adev->rings[i];
 
-			if (!ring || !ring->sched.thread)
+			if (!ring || !drm_sched_wqueue_ready(&ring->sched))
 				continue;
 
 			drm_sched_stop(&ring->sched, NULL);
@@ -6119,7 +6163,7 @@ void amdgpu_pci_resume(struct pci_dev *pdev)
 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
 		struct amdgpu_ring *ring = adev->rings[i];
 
-		if (!ring || !ring->sched.thread)
+		if (!ring || !drm_sched_wqueue_ready(&ring->sched))
 			continue;
 
 		drm_sched_start(&ring->sched, true);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index e7e87a3b2601..decbbe3d4f06 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -42,6 +42,7 @@
 #include <linux/dma-fence-array.h>
 #include <linux/pci-p2pdma.h>
 #include <linux/pm_runtime.h>
+#include "amdgpu_trace.h"
 
 /**
  * amdgpu_dma_buf_attach - &dma_buf_ops.attach implementation
@@ -63,6 +64,7 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf,
 		attach->peer2peer = false;
 
 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
+	trace_amdgpu_runpm_reference_dumps(1, __func__);
 	if (r < 0)
 		goto out;
 
@@ -70,6 +72,7 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf,
 
 out:
 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+	trace_amdgpu_runpm_reference_dumps(0, __func__);
 	return r;
 }
 
@@ -90,6 +93,7 @@ static void amdgpu_dma_buf_detach(struct dma_buf *dmabuf,
 
 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+	trace_amdgpu_runpm_reference_dumps(0, __func__);
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
index dc230212746a..70bff8cecfda 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
@@ -183,6 +183,7 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amd
 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
 			       seq, flags | AMDGPU_FENCE_FLAG_INT);
 	pm_runtime_get_noresume(adev_to_drm(adev)->dev);
+	trace_amdgpu_runpm_reference_dumps(1, __func__);
 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];
 	if (unlikely(rcu_dereference_protected(*ptr, 1))) {
 		struct dma_fence *old;
@@ -310,6 +311,7 @@ bool amdgpu_fence_process(struct amdgpu_ring *ring)
 		dma_fence_put(fence);
 		pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+		trace_amdgpu_runpm_reference_dumps(0, __func__);
 	} while (last_seq != seq);
 
 	return true;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 1f357198533f..62bb7fc7448a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -115,7 +115,7 @@ int amdgpu_job_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	if (!entity)
 		return 0;
 
-	return drm_sched_job_init(&(*job)->base, entity, owner);
+	return drm_sched_job_init(&(*job)->base, entity, 1, owner);
 }
 
 int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 583cf03950cd..b5ebafd4a3ad 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -1428,6 +1428,8 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
 		fpriv->csa_va = NULL;
 	}
 
+	amdgpu_seq64_unmap(adev, fpriv);
+
 	pasid = fpriv->vm.pasid;
 	pd = amdgpu_bo_ref(fpriv->vm.root.bo);
 	if (!WARN_ON(amdgpu_bo_reserve(pd, true))) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c
new file mode 100644
index 000000000000..f3de02193138
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c
@@ -0,0 +1,247 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2023 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "amdgpu.h"
+#include "amdgpu_seq64.h"
+
+#include <drm/drm_exec.h>
+
+/**
+ * DOC: amdgpu_seq64
+ *
+ * amdgpu_seq64 allocates a 64bit memory on each request in sequence order.
+ * seq64 driver is required for user queue fence memory allocation, TLB
+ * counters and VM updates. It has maximum count of 32768 64 bit slots.
+ */
+
+/**
+ * amdgpu_seq64_map - Map the seq64 memory to VM
+ *
+ * @adev: amdgpu_device pointer
+ * @vm: vm pointer
+ * @bo_va: bo_va pointer
+ * @seq64_addr: seq64 vaddr start address
+ * @size: seq64 pool size
+ *
+ * Map the seq64 memory to the given VM.
+ *
+ * Returns:
+ * 0 on success or a negative error code on failure
+ */
+int amdgpu_seq64_map(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+		     struct amdgpu_bo_va **bo_va, u64 seq64_addr,
+		     uint32_t size)
+{
+	struct amdgpu_bo *bo;
+	struct drm_exec exec;
+	int r;
+
+	bo = adev->seq64.sbo;
+	if (!bo)
+		return -EINVAL;
+
+	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
+	drm_exec_until_all_locked(&exec) {
+		r = amdgpu_vm_lock_pd(vm, &exec, 0);
+		if (likely(!r))
+			r = drm_exec_lock_obj(&exec, &bo->tbo.base);
+		drm_exec_retry_on_contention(&exec);
+		if (unlikely(r))
+			goto error;
+	}
+
+	*bo_va = amdgpu_vm_bo_add(adev, vm, bo);
+	if (!*bo_va) {
+		r = -ENOMEM;
+		goto error;
+	}
+
+	r = amdgpu_vm_bo_map(adev, *bo_va, seq64_addr, 0, size,
+			     AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE |
+			     AMDGPU_PTE_EXECUTABLE);
+	if (r) {
+		DRM_ERROR("failed to do bo_map on userq sem, err=%d\n", r);
+		amdgpu_vm_bo_del(adev, *bo_va);
+		goto error;
+	}
+
+	r = amdgpu_vm_bo_update(adev, *bo_va, false);
+	if (r) {
+		DRM_ERROR("failed to do vm_bo_update on userq sem\n");
+		amdgpu_vm_bo_del(adev, *bo_va);
+		goto error;
+	}
+
+error:
+	drm_exec_fini(&exec);
+	return r;
+}
+
+/**
+ * amdgpu_seq64_unmap - Unmap the seq64 memory
+ *
+ * @adev: amdgpu_device pointer
+ * @fpriv: DRM file private
+ *
+ * Unmap the seq64 memory from the given VM.
+ */
+void amdgpu_seq64_unmap(struct amdgpu_device *adev, struct amdgpu_fpriv *fpriv)
+{
+	struct amdgpu_vm *vm;
+	struct amdgpu_bo *bo;
+	struct drm_exec exec;
+	int r;
+
+	if (!fpriv->seq64_va)
+		return;
+
+	bo = adev->seq64.sbo;
+	if (!bo)
+		return;
+
+	vm = &fpriv->vm;
+
+	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
+	drm_exec_until_all_locked(&exec) {
+		r = amdgpu_vm_lock_pd(vm, &exec, 0);
+		if (likely(!r))
+			r = drm_exec_lock_obj(&exec, &bo->tbo.base);
+		drm_exec_retry_on_contention(&exec);
+		if (unlikely(r))
+			goto error;
+	}
+
+	amdgpu_vm_bo_del(adev, fpriv->seq64_va);
+
+	fpriv->seq64_va = NULL;
+
+error:
+	drm_exec_fini(&exec);
+}
+
+/**
+ * amdgpu_seq64_alloc - Allocate a 64 bit memory
+ *
+ * @adev: amdgpu_device pointer
+ * @gpu_addr: allocated gpu VA start address
+ * @cpu_addr: allocated cpu VA start address
+ *
+ * Alloc a 64 bit memory from seq64 pool.
+ *
+ * Returns:
+ * 0 on success or a negative error code on failure
+ */
+int amdgpu_seq64_alloc(struct amdgpu_device *adev, u64 *gpu_addr,
+		       u64 **cpu_addr)
+{
+	unsigned long bit_pos;
+	u32 offset;
+
+	bit_pos = find_first_zero_bit(adev->seq64.used, adev->seq64.num_sem);
+
+	if (bit_pos < adev->seq64.num_sem) {
+		__set_bit(bit_pos, adev->seq64.used);
+		offset = bit_pos << 6; /* convert to qw offset */
+	} else {
+		return -EINVAL;
+	}
+
+	*gpu_addr = offset + AMDGPU_SEQ64_VADDR_START;
+	*cpu_addr = offset + adev->seq64.cpu_base_addr;
+
+	return 0;
+}
+
+/**
+ * amdgpu_seq64_free - Free the given 64 bit memory
+ *
+ * @adev: amdgpu_device pointer
+ * @gpu_addr: gpu start address to be freed
+ *
+ * Free the given 64 bit memory from seq64 pool.
+ *
+ */
+void amdgpu_seq64_free(struct amdgpu_device *adev, u64 gpu_addr)
+{
+	u32 offset;
+
+	offset = gpu_addr - AMDGPU_SEQ64_VADDR_START;
+
+	offset >>= 6;
+	if (offset < adev->seq64.num_sem)
+		__clear_bit(offset, adev->seq64.used);
+}
+
+/**
+ * amdgpu_seq64_fini - Cleanup seq64 driver
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Free the memory space allocated for seq64.
+ *
+ */
+void amdgpu_seq64_fini(struct amdgpu_device *adev)
+{
+	amdgpu_bo_free_kernel(&adev->seq64.sbo,
+			      NULL,
+			      (void **)&adev->seq64.cpu_base_addr);
+}
+
+/**
+ * amdgpu_seq64_init - Initialize seq64 driver
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Allocate the required memory space for seq64.
+ *
+ * Returns:
+ * 0 on success or a negative error code on failure
+ */
+int amdgpu_seq64_init(struct amdgpu_device *adev)
+{
+	int r;
+
+	if (adev->seq64.sbo)
+		return 0;
+
+	/*
+	 * AMDGPU_MAX_SEQ64_SLOTS * sizeof(u64) * 8 = AMDGPU_MAX_SEQ64_SLOTS
+	 * 64bit slots
+	 */
+	r = amdgpu_bo_create_kernel(adev, AMDGPU_SEQ64_SIZE,
+				    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
+				    &adev->seq64.sbo, NULL,
+				    (void **)&adev->seq64.cpu_base_addr);
+	if (r) {
+		dev_warn(adev->dev, "(%d) create seq64 failed\n", r);
+		return r;
+	}
+
+	memset(adev->seq64.cpu_base_addr, 0, AMDGPU_SEQ64_SIZE);
+
+	adev->seq64.num_sem = AMDGPU_MAX_SEQ64_SLOTS;
+	memset(&adev->seq64.used, 0, sizeof(adev->seq64.used));
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.h
new file mode 100644
index 000000000000..2196e72be508
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.h
@@ -0,0 +1,49 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2023 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __AMDGPU_SEQ64_H__
+#define __AMDGPU_SEQ64_H__
+
+#define AMDGPU_SEQ64_SIZE		(2ULL << 20)
+#define AMDGPU_MAX_SEQ64_SLOTS		(AMDGPU_SEQ64_SIZE / (sizeof(u64) * 8))
+#define AMDGPU_SEQ64_VADDR_OFFSET	0x50000
+#define AMDGPU_SEQ64_VADDR_START	(AMDGPU_VA_RESERVED_SIZE + AMDGPU_SEQ64_VADDR_OFFSET)
+
+struct amdgpu_seq64 {
+	struct amdgpu_bo *sbo;
+	u32 num_sem;
+	u64 *cpu_base_addr;
+	DECLARE_BITMAP(used, AMDGPU_MAX_SEQ64_SLOTS);
+};
+
+void amdgpu_seq64_fini(struct amdgpu_device *adev);
+int amdgpu_seq64_init(struct amdgpu_device *adev);
+int amdgpu_seq64_alloc(struct amdgpu_device *adev, u64 *gpu_addr, u64 **cpu_addr);
+void amdgpu_seq64_free(struct amdgpu_device *adev, u64 gpu_addr);
+int amdgpu_seq64_map(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+		     struct amdgpu_bo_va **bo_va, u64 seq64_addr, uint32_t size);
+void amdgpu_seq64_unmap(struct amdgpu_device *adev, struct amdgpu_fpriv *fpriv);
+
+#endif
+
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index 2fd1bfb35916..f539b1d00234 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -554,6 +554,21 @@ TRACE_EVENT(amdgpu_reset_reg_dumps,
 		      __entry->value)
 );
 
+TRACE_EVENT(amdgpu_runpm_reference_dumps,
+	    TP_PROTO(uint32_t index, const char *func),
+	    TP_ARGS(index, func),
+	    TP_STRUCT__entry(
+			     __field(uint32_t, index)
+			     __string(func, func)
+			     ),
+	    TP_fast_assign(
+			   __entry->index = index;
+			   __assign_str(func, func);
+			   ),
+	    TP_printk("amdgpu runpm reference dump 0x%x: 0x%s\n",
+		      __entry->index,
+		      __get_str(func))
+);
 #undef AMDGPU_JOB_GET_TIMELINE_NAME
 #endif
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index d1b8afd105c9..7da71b6a9dc6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -1437,6 +1437,51 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
 	return 0;
 }
 
+/**
+ * amdgpu_vm_flush_compute_tlb - Flush TLB on compute VM
+ *
+ * @adev: amdgpu_device pointer
+ * @vm: requested vm
+ * @flush_type: flush type
+ * @xcc_mask: mask of XCCs that belong to the compute partition in need of a TLB flush.
+ *
+ * Flush TLB if needed for a compute VM.
+ *
+ * Returns:
+ * 0 for success.
+ */
+int amdgpu_vm_flush_compute_tlb(struct amdgpu_device *adev,
+				struct amdgpu_vm *vm,
+				uint32_t flush_type,
+				uint32_t xcc_mask)
+{
+	uint64_t tlb_seq = amdgpu_vm_tlb_seq(vm);
+	bool all_hub = false;
+	int xcc = 0, r = 0;
+
+	WARN_ON_ONCE(!vm->is_compute_context);
+
+	/*
+	 * It can be that we race and lose here, but that is extremely unlikely
+	 * and the worst thing which could happen is that we flush the changes
+	 * into the TLB once more which is harmless.
+	 */
+	if (atomic64_xchg(&vm->kfd_last_flushed_seq, tlb_seq) == tlb_seq)
+		return 0;
+
+	if (adev->family == AMDGPU_FAMILY_AI ||
+	    adev->family == AMDGPU_FAMILY_RV)
+		all_hub = true;
+
+	for_each_inst(xcc, xcc_mask) {
+		r = amdgpu_gmc_flush_gpu_tlb_pasid(adev, vm->pasid, flush_type,
+						   all_hub, xcc);
+		if (r)
+			break;
+	}
+	return r;
+}
+
 /**
  * amdgpu_vm_bo_add - add a bo to a specific vm
  *
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 2cd86d2bf73f..b6cd565562ad 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -324,6 +324,7 @@ struct amdgpu_vm {
 	/* Last finished delayed update */
 	atomic64_t		tlb_seq;
 	struct dma_fence	*last_tlb_flush;
+	atomic64_t		kfd_last_flushed_seq;
 
 	/* How many times we had to re-generate the page tables */
 	uint64_t		generation;
@@ -445,6 +446,10 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
 int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
 			   struct amdgpu_vm *vm,
 			   struct ww_acquire_ctx *ticket);
+int amdgpu_vm_flush_compute_tlb(struct amdgpu_device *adev,
+				struct amdgpu_vm *vm,
+				uint32_t flush_type,
+				uint32_t xcc_mask);
 void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
 			    struct amdgpu_vm *vm, struct amdgpu_bo *bo);
 int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
index bd20cb3b9819..44d8c1a11e1b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
@@ -413,6 +413,38 @@ static ssize_t amdgpu_xgmi_show_num_links(struct device *dev,
 	return sysfs_emit(buf, "%s\n", buf);
 }
 
+static ssize_t amdgpu_xgmi_show_connected_port_num(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = drm_to_adev(ddev);
+	struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
+	int i, j, size = 0;
+	int current_node;
+	/*
+	 * get the node id in the sysfs for the current socket and show
+	 * it in the port num info output in the sysfs for easy reading.
+	 * it is NOT the one retrieved from xgmi ta.
+	 */
+	for (i = 0; i < top->num_nodes; i++) {
+		if (top->nodes[i].node_id == adev->gmc.xgmi.node_id) {
+			current_node = i;
+			break;
+		}
+	}
+
+	for (i = 0; i < top->num_nodes; i++) {
+		for (j = 0; j < top->nodes[i].num_links; j++)
+			/* node id in sysfs starts from 1 rather than 0 so +1 here */
+			size += sysfs_emit_at(buf, size, "%02x:%02x ->  %02x:%02x\n", current_node + 1,
+					      top->nodes[i].port_num[j].src_xgmi_port_num, i + 1,
+					      top->nodes[i].port_num[j].dst_xgmi_port_num);
+	}
+
+	return size;
+}
+
 #define AMDGPU_XGMI_SET_FICAA(o)	((o) | 0x456801)
 static ssize_t amdgpu_xgmi_show_error(struct device *dev,
 				      struct device_attribute *attr,
@@ -452,6 +484,7 @@ static DEVICE_ATTR(xgmi_physical_id, 0444, amdgpu_xgmi_show_physical_id, NULL);
 static DEVICE_ATTR(xgmi_error, S_IRUGO, amdgpu_xgmi_show_error, NULL);
 static DEVICE_ATTR(xgmi_num_hops, S_IRUGO, amdgpu_xgmi_show_num_hops, NULL);
 static DEVICE_ATTR(xgmi_num_links, S_IRUGO, amdgpu_xgmi_show_num_links, NULL);
+static DEVICE_ATTR(xgmi_port_num, S_IRUGO, amdgpu_xgmi_show_connected_port_num, NULL);
 
 static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev,
 					 struct amdgpu_hive_info *hive)
@@ -487,6 +520,13 @@ static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev,
 	if (ret)
 		pr_err("failed to create xgmi_num_links\n");
 
+	/* Create xgmi port num file if supported */
+	if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG) {
+		ret = device_create_file(adev->dev, &dev_attr_xgmi_port_num);
+		if (ret)
+			dev_err(adev->dev, "failed to create xgmi_port_num\n");
+	}
+
 	/* Create sysfs link to hive info folder on the first device */
 	if (hive->kobj.parent != (&adev->dev->kobj)) {
 		ret = sysfs_create_link(&adev->dev->kobj, &hive->kobj,
@@ -517,6 +557,8 @@ static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev,
 	device_remove_file(adev->dev, &dev_attr_xgmi_error);
 	device_remove_file(adev->dev, &dev_attr_xgmi_num_hops);
 	device_remove_file(adev->dev, &dev_attr_xgmi_num_links);
+	if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG)
+		device_remove_file(adev->dev, &dev_attr_xgmi_port_num);
 
 success:
 	return ret;
@@ -533,6 +575,8 @@ static void amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device *adev,
 	device_remove_file(adev->dev, &dev_attr_xgmi_error);
 	device_remove_file(adev->dev, &dev_attr_xgmi_num_hops);
 	device_remove_file(adev->dev, &dev_attr_xgmi_num_links);
+	if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG)
+		device_remove_file(adev->dev, &dev_attr_xgmi_port_num);
 
 	if (hive->kobj.parent != (&adev->dev->kobj))
 		sysfs_remove_link(&adev->dev->kobj,"xgmi_hive_info");
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
index d7cd5fa313ff..df75863393fc 100644
--- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
@@ -2069,7 +2069,7 @@ static const uint32_t cwsr_trap_aldebaran_hex[] = {
 };
 
 static const uint32_t cwsr_trap_gfx10_hex[] = {
-	0xbf820001, 0xbf820220,
+	0xbf820001, 0xbf820221,
 	0xb0804004, 0xb978f802,
 	0x8a78ff78, 0x00020006,
 	0xb97bf803, 0x876eff78,
@@ -2118,391 +2118,391 @@ static const uint32_t cwsr_trap_gfx10_hex[] = {
 	0xbf900004, 0xbf8cc07f,
 	0x877aff7f, 0x04000000,
 	0x8f7a857a, 0x886d7a6d,
-	0xbefa037e, 0x877bff7f,
-	0x0000ffff, 0xbefe03c1,
-	0xbeff03c1, 0xdc5f8000,
-	0x007a0000, 0x7e000280,
-	0xbefe037a, 0xbeff037b,
-	0xb97b02dc, 0x8f7b997b,
-	0xb97a3a05, 0x807a817a,
-	0xbf0d997b, 0xbf850002,
-	0x8f7a897a, 0xbf820001,
-	0x8f7a8a7a, 0xb97b1e06,
-	0x8f7b8a7b, 0x807a7b7a,
+	0x7e008200, 0xbefa037e,
 	0x877bff7f, 0x0000ffff,
-	0x807aff7a, 0x00000200,
-	0x807a7e7a, 0x827b807b,
-	0xd7610000, 0x00010870,
-	0xd7610000, 0x00010a71,
-	0xd7610000, 0x00010c72,
-	0xd7610000, 0x00010e73,
-	0xd7610000, 0x00011074,
-	0xd7610000, 0x00011275,
-	0xd7610000, 0x00011476,
-	0xd7610000, 0x00011677,
-	0xd7610000, 0x00011a79,
-	0xd7610000, 0x00011c7e,
-	0xd7610000, 0x00011e7f,
-	0xbefe03ff, 0x00003fff,
-	0xbeff0380, 0xdc5f8040,
-	0x007a0000, 0xd760007a,
-	0x00011d00, 0xd760007b,
-	0x00011f00, 0xbefe037a,
-	0xbeff037b, 0xbef4037e,
-	0x8775ff7f, 0x0000ffff,
-	0x8875ff75, 0x00040000,
-	0xbef60380, 0xbef703ff,
-	0x10807fac, 0xbef1037c,
-	0xbef00380, 0xb97302dc,
-	0x8f739973, 0xbefe03c1,
-	0x907c9973, 0x877c817c,
-	0xbf06817c, 0xbf850002,
-	0xbeff0380, 0xbf820002,
-	0xbeff03c1, 0xbf820009,
+	0xbefe03c1, 0xbeff03c1,
+	0xdc5f8000, 0x007a0000,
+	0x7e000280, 0xbefe037a,
+	0xbeff037b, 0xb97b02dc,
+	0x8f7b997b, 0xb97a3a05,
+	0x807a817a, 0xbf0d997b,
+	0xbf850002, 0x8f7a897a,
+	0xbf820001, 0x8f7a8a7a,
+	0xb97b1e06, 0x8f7b8a7b,
+	0x807a7b7a, 0x877bff7f,
+	0x0000ffff, 0x807aff7a,
+	0x00000200, 0x807a7e7a,
+	0x827b807b, 0xd7610000,
+	0x00010870, 0xd7610000,
+	0x00010a71, 0xd7610000,
+	0x00010c72, 0xd7610000,
+	0x00010e73, 0xd7610000,
+	0x00011074, 0xd7610000,
+	0x00011275, 0xd7610000,
+	0x00011476, 0xd7610000,
+	0x00011677, 0xd7610000,
+	0x00011a79, 0xd7610000,
+	0x00011c7e, 0xd7610000,
+	0x00011e7f, 0xbefe03ff,
+	0x00003fff, 0xbeff0380,
+	0xdc5f8040, 0x007a0000,
+	0xd760007a, 0x00011d00,
+	0xd760007b, 0x00011f00,
+	0xbefe037a, 0xbeff037b,
+	0xbef4037e, 0x8775ff7f,
+	0x0000ffff, 0x8875ff75,
+	0x00040000, 0xbef60380,
+	0xbef703ff, 0x10807fac,
+	0xbef1037c, 0xbef00380,
+	0xb97302dc, 0x8f739973,
+	0xbefe03c1, 0x907c9973,
+	0x877c817c, 0xbf06817c,
+	0xbf850002, 0xbeff0380,
+	0xbf820002, 0xbeff03c1,
+	0xbf820009, 0xbef603ff,
+	0x01000000, 0xe0704080,
+	0x705d0100, 0xe0704100,
+	0x705d0200, 0xe0704180,
+	0x705d0300, 0xbf820008,
 	0xbef603ff, 0x01000000,
-	0xe0704080, 0x705d0100,
-	0xe0704100, 0x705d0200,
-	0xe0704180, 0x705d0300,
-	0xbf820008, 0xbef603ff,
-	0x01000000, 0xe0704100,
-	0x705d0100, 0xe0704200,
-	0x705d0200, 0xe0704300,
-	0x705d0300, 0xb9703a05,
-	0x80708170, 0xbf0d9973,
-	0xbf850002, 0x8f708970,
-	0xbf820001, 0x8f708a70,
-	0xb97a1e06, 0x8f7a8a7a,
-	0x80707a70, 0x8070ff70,
-	0x00000200, 0xbef603ff,
-	0x01000000, 0x7e000280,
-	0x7e020280, 0x7e040280,
-	0xbefc0380, 0xd7610002,
-	0x0000f871, 0x807c817c,
-	0xd7610002, 0x0000f86c,
-	0x807c817c, 0x8a7aff6d,
-	0x80000000, 0xd7610002,
-	0x0000f87a, 0x807c817c,
-	0xd7610002, 0x0000f86e,
-	0x807c817c, 0xd7610002,
-	0x0000f86f, 0x807c817c,
-	0xd7610002, 0x0000f878,
-	0x807c817c, 0xb97af803,
-	0xd7610002, 0x0000f87a,
-	0x807c817c, 0xd7610002,
-	0x0000f87b, 0x807c817c,
-	0xb971f801, 0xd7610002,
-	0x0000f871, 0x807c817c,
-	0xb971f814, 0xd7610002,
-	0x0000f871, 0x807c817c,
-	0xb971f815, 0xd7610002,
-	0x0000f871, 0x807c817c,
-	0xbefe03ff, 0x0000ffff,
-	0xbeff0380, 0xe0704000,
-	0x705d0200, 0xbefe03c1,
+	0xe0704100, 0x705d0100,
+	0xe0704200, 0x705d0200,
+	0xe0704300, 0x705d0300,
 	0xb9703a05, 0x80708170,
 	0xbf0d9973, 0xbf850002,
 	0x8f708970, 0xbf820001,
 	0x8f708a70, 0xb97a1e06,
 	0x8f7a8a7a, 0x80707a70,
+	0x8070ff70, 0x00000200,
 	0xbef603ff, 0x01000000,
-	0xbef90380, 0xbefc0380,
-	0xbf800000, 0xbe802f00,
-	0xbe822f02, 0xbe842f04,
-	0xbe862f06, 0xbe882f08,
-	0xbe8a2f0a, 0xbe8c2f0c,
-	0xbe8e2f0e, 0xd7610002,
-	0x0000f200, 0x80798179,
-	0xd7610002, 0x0000f201,
+	0x7e000280, 0x7e020280,
+	0x7e040280, 0xbefc0380,
+	0xd7610002, 0x0000f871,
+	0x807c817c, 0xd7610002,
+	0x0000f86c, 0x807c817c,
+	0x8a7aff6d, 0x80000000,
+	0xd7610002, 0x0000f87a,
+	0x807c817c, 0xd7610002,
+	0x0000f86e, 0x807c817c,
+	0xd7610002, 0x0000f86f,
+	0x807c817c, 0xd7610002,
+	0x0000f878, 0x807c817c,
+	0xb97af803, 0xd7610002,
+	0x0000f87a, 0x807c817c,
+	0xd7610002, 0x0000f87b,
+	0x807c817c, 0xb971f801,
+	0xd7610002, 0x0000f871,
+	0x807c817c, 0xb971f814,
+	0xd7610002, 0x0000f871,
+	0x807c817c, 0xb971f815,
+	0xd7610002, 0x0000f871,
+	0x807c817c, 0xbefe03ff,
+	0x0000ffff, 0xbeff0380,
+	0xe0704000, 0x705d0200,
+	0xbefe03c1, 0xb9703a05,
+	0x80708170, 0xbf0d9973,
+	0xbf850002, 0x8f708970,
+	0xbf820001, 0x8f708a70,
+	0xb97a1e06, 0x8f7a8a7a,
+	0x80707a70, 0xbef603ff,
+	0x01000000, 0xbef90380,
+	0xbefc0380, 0xbf800000,
+	0xbe802f00, 0xbe822f02,
+	0xbe842f04, 0xbe862f06,
+	0xbe882f08, 0xbe8a2f0a,
+	0xbe8c2f0c, 0xbe8e2f0e,
+	0xd7610002, 0x0000f200,
 	0x80798179, 0xd7610002,
-	0x0000f202, 0x80798179,
-	0xd7610002, 0x0000f203,
+	0x0000f201, 0x80798179,
+	0xd7610002, 0x0000f202,
 	0x80798179, 0xd7610002,
-	0x0000f204, 0x80798179,
-	0xd7610002, 0x0000f205,
+	0x0000f203, 0x80798179,
+	0xd7610002, 0x0000f204,
 	0x80798179, 0xd7610002,
-	0x0000f206, 0x80798179,
-	0xd7610002, 0x0000f207,
+	0x0000f205, 0x80798179,
+	0xd7610002, 0x0000f206,
 	0x80798179, 0xd7610002,
-	0x0000f208, 0x80798179,
-	0xd7610002, 0x0000f209,
+	0x0000f207, 0x80798179,
+	0xd7610002, 0x0000f208,
 	0x80798179, 0xd7610002,
-	0x0000f20a, 0x80798179,
-	0xd7610002, 0x0000f20b,
+	0x0000f209, 0x80798179,
+	0xd7610002, 0x0000f20a,
 	0x80798179, 0xd7610002,
-	0x0000f20c, 0x80798179,
-	0xd7610002, 0x0000f20d,
+	0x0000f20b, 0x80798179,
+	0xd7610002, 0x0000f20c,
 	0x80798179, 0xd7610002,
-	0x0000f20e, 0x80798179,
-	0xd7610002, 0x0000f20f,
-	0x80798179, 0xbf06a079,
-	0xbf840006, 0xe0704000,
-	0x705d0200, 0x8070ff70,
-	0x00000080, 0xbef90380,
-	0x7e040280, 0x807c907c,
-	0xbf0aff7c, 0x00000060,
-	0xbf85ffbc, 0xbe802f00,
-	0xbe822f02, 0xbe842f04,
-	0xbe862f06, 0xbe882f08,
-	0xbe8a2f0a, 0xd7610002,
-	0x0000f200, 0x80798179,
-	0xd7610002, 0x0000f201,
+	0x0000f20d, 0x80798179,
+	0xd7610002, 0x0000f20e,
 	0x80798179, 0xd7610002,
-	0x0000f202, 0x80798179,
-	0xd7610002, 0x0000f203,
+	0x0000f20f, 0x80798179,
+	0xbf06a079, 0xbf840006,
+	0xe0704000, 0x705d0200,
+	0x8070ff70, 0x00000080,
+	0xbef90380, 0x7e040280,
+	0x807c907c, 0xbf0aff7c,
+	0x00000060, 0xbf85ffbc,
+	0xbe802f00, 0xbe822f02,
+	0xbe842f04, 0xbe862f06,
+	0xbe882f08, 0xbe8a2f0a,
+	0xd7610002, 0x0000f200,
 	0x80798179, 0xd7610002,
-	0x0000f204, 0x80798179,
-	0xd7610002, 0x0000f205,
+	0x0000f201, 0x80798179,
+	0xd7610002, 0x0000f202,
 	0x80798179, 0xd7610002,
-	0x0000f206, 0x80798179,
-	0xd7610002, 0x0000f207,
+	0x0000f203, 0x80798179,
+	0xd7610002, 0x0000f204,
 	0x80798179, 0xd7610002,
-	0x0000f208, 0x80798179,
-	0xd7610002, 0x0000f209,
+	0x0000f205, 0x80798179,
+	0xd7610002, 0x0000f206,
 	0x80798179, 0xd7610002,
-	0x0000f20a, 0x80798179,
-	0xd7610002, 0x0000f20b,
-	0x80798179, 0xe0704000,
-	0x705d0200, 0xbefe03c1,
-	0x907c9973, 0x877c817c,
-	0xbf06817c, 0xbf850002,
-	0xbeff0380, 0xbf820001,
-	0xbeff03c1, 0xb97b4306,
-	0x877bc17b, 0xbf840044,
-	0xbf8a0000, 0x877aff6d,
-	0x80000000, 0xbf840040,
-	0x8f7b867b, 0x8f7b827b,
-	0xbef6037b, 0xb9703a05,
-	0x80708170, 0xbf0d9973,
-	0xbf850002, 0x8f708970,
-	0xbf820001, 0x8f708a70,
-	0xb97a1e06, 0x8f7a8a7a,
-	0x80707a70, 0x8070ff70,
-	0x00000200, 0x8070ff70,
-	0x00000080, 0xbef603ff,
-	0x01000000, 0xd7650000,
-	0x000100c1, 0xd7660000,
-	0x000200c1, 0x16000084,
-	0x907c9973, 0x877c817c,
-	0xbf06817c, 0xbefc0380,
-	0xbf850012, 0xbe8303ff,
-	0x00000080, 0xbf800000,
-	0xbf800000, 0xbf800000,
-	0xd8d80000, 0x01000000,
-	0xbf8c0000, 0xe0704000,
-	0x705d0100, 0x807c037c,
-	0x80700370, 0xd5250000,
-	0x0001ff00, 0x00000080,
-	0xbf0a7b7c, 0xbf85fff4,
-	0xbf820011, 0xbe8303ff,
-	0x00000100, 0xbf800000,
-	0xbf800000, 0xbf800000,
-	0xd8d80000, 0x01000000,
-	0xbf8c0000, 0xe0704000,
-	0x705d0100, 0x807c037c,
-	0x80700370, 0xd5250000,
-	0x0001ff00, 0x00000100,
-	0xbf0a7b7c, 0xbf85fff4,
+	0x0000f207, 0x80798179,
+	0xd7610002, 0x0000f208,
+	0x80798179, 0xd7610002,
+	0x0000f209, 0x80798179,
+	0xd7610002, 0x0000f20a,
+	0x80798179, 0xd7610002,
+	0x0000f20b, 0x80798179,
+	0xe0704000, 0x705d0200,
 	0xbefe03c1, 0x907c9973,
 	0x877c817c, 0xbf06817c,
-	0xbf850004, 0xbef003ff,
-	0x00000200, 0xbeff0380,
-	0xbf820003, 0xbef003ff,
-	0x00000400, 0xbeff03c1,
-	0xb97b3a05, 0x807b817b,
-	0x8f7b827b, 0x907c9973,
+	0xbf850002, 0xbeff0380,
+	0xbf820001, 0xbeff03c1,
+	0xb97b4306, 0x877bc17b,
+	0xbf840044, 0xbf8a0000,
+	0x877aff6d, 0x80000000,
+	0xbf840040, 0x8f7b867b,
+	0x8f7b827b, 0xbef6037b,
+	0xb9703a05, 0x80708170,
+	0xbf0d9973, 0xbf850002,
+	0x8f708970, 0xbf820001,
+	0x8f708a70, 0xb97a1e06,
+	0x8f7a8a7a, 0x80707a70,
+	0x8070ff70, 0x00000200,
+	0x8070ff70, 0x00000080,
+	0xbef603ff, 0x01000000,
+	0xd7650000, 0x000100c1,
+	0xd7660000, 0x000200c1,
+	0x16000084, 0x907c9973,
 	0x877c817c, 0xbf06817c,
-	0xbf850017, 0xbef603ff,
-	0x01000000, 0xbefc0384,
-	0xbf0a7b7c, 0xbf840037,
-	0x7e008700, 0x7e028701,
-	0x7e048702, 0x7e068703,
-	0xe0704000, 0x705d0000,
-	0xe0704080, 0x705d0100,
-	0xe0704100, 0x705d0200,
-	0xe0704180, 0x705d0300,
-	0x807c847c, 0x8070ff70,
-	0x00000200, 0xbf0a7b7c,
-	0xbf85ffef, 0xbf820025,
+	0xbefc0380, 0xbf850012,
+	0xbe8303ff, 0x00000080,
+	0xbf800000, 0xbf800000,
+	0xbf800000, 0xd8d80000,
+	0x01000000, 0xbf8c0000,
+	0xe0704000, 0x705d0100,
+	0x807c037c, 0x80700370,
+	0xd5250000, 0x0001ff00,
+	0x00000080, 0xbf0a7b7c,
+	0xbf85fff4, 0xbf820011,
+	0xbe8303ff, 0x00000100,
+	0xbf800000, 0xbf800000,
+	0xbf800000, 0xd8d80000,
+	0x01000000, 0xbf8c0000,
+	0xe0704000, 0x705d0100,
+	0x807c037c, 0x80700370,
+	0xd5250000, 0x0001ff00,
+	0x00000100, 0xbf0a7b7c,
+	0xbf85fff4, 0xbefe03c1,
+	0x907c9973, 0x877c817c,
+	0xbf06817c, 0xbf850004,
+	0xbef003ff, 0x00000200,
+	0xbeff0380, 0xbf820003,
+	0xbef003ff, 0x00000400,
+	0xbeff03c1, 0xb97b3a05,
+	0x807b817b, 0x8f7b827b,
+	0x907c9973, 0x877c817c,
+	0xbf06817c, 0xbf850017,
 	0xbef603ff, 0x01000000,
 	0xbefc0384, 0xbf0a7b7c,
-	0xbf840011, 0x7e008700,
+	0xbf840037, 0x7e008700,
 	0x7e028701, 0x7e048702,
 	0x7e068703, 0xe0704000,
-	0x705d0000, 0xe0704100,
-	0x705d0100, 0xe0704200,
-	0x705d0200, 0xe0704300,
+	0x705d0000, 0xe0704080,
+	0x705d0100, 0xe0704100,
+	0x705d0200, 0xe0704180,
 	0x705d0300, 0x807c847c,
-	0x8070ff70, 0x00000400,
+	0x8070ff70, 0x00000200,
 	0xbf0a7b7c, 0xbf85ffef,
-	0xb97b1e06, 0x877bc17b,
-	0xbf84000c, 0x8f7b837b,
-	0x807b7c7b, 0xbefe03c1,
-	0xbeff0380, 0x7e008700,
+	0xbf820025, 0xbef603ff,
+	0x01000000, 0xbefc0384,
+	0xbf0a7b7c, 0xbf840011,
+	0x7e008700, 0x7e028701,
+	0x7e048702, 0x7e068703,
 	0xe0704000, 0x705d0000,
-	0x807c817c, 0x8070ff70,
-	0x00000080, 0xbf0a7b7c,
-	0xbf85fff8, 0xbf82013b,
-	0xbef4037e, 0x8775ff7f,
-	0x0000ffff, 0x8875ff75,
-	0x00040000, 0xbef60380,
-	0xbef703ff, 0x10807fac,
-	0xb97202dc, 0x8f729972,
-	0x876eff7f, 0x04000000,
-	0xbf840034, 0xbefe03c1,
-	0x907c9972, 0x877c817c,
-	0xbf06817c, 0xbf850002,
-	0xbeff0380, 0xbf820001,
-	0xbeff03c1, 0xb96f4306,
-	0x876fc16f, 0xbf840029,
-	0x8f6f866f, 0x8f6f826f,
-	0xbef6036f, 0xb9783a05,
-	0x80788178, 0xbf0d9972,
-	0xbf850002, 0x8f788978,
-	0xbf820001, 0x8f788a78,
-	0xb96e1e06, 0x8f6e8a6e,
-	0x80786e78, 0x8078ff78,
-	0x00000200, 0x8078ff78,
-	0x00000080, 0xbef603ff,
-	0x01000000, 0x907c9972,
-	0x877c817c, 0xbf06817c,
-	0xbefc0380, 0xbf850009,
-	0xe0310000, 0x781d0000,
-	0x807cff7c, 0x00000080,
-	0x8078ff78, 0x00000080,
-	0xbf0a6f7c, 0xbf85fff8,
-	0xbf820008, 0xe0310000,
-	0x781d0000, 0x807cff7c,
-	0x00000100, 0x8078ff78,
-	0x00000100, 0xbf0a6f7c,
-	0xbf85fff8, 0xbef80380,
+	0xe0704100, 0x705d0100,
+	0xe0704200, 0x705d0200,
+	0xe0704300, 0x705d0300,
+	0x807c847c, 0x8070ff70,
+	0x00000400, 0xbf0a7b7c,
+	0xbf85ffef, 0xb97b1e06,
+	0x877bc17b, 0xbf84000c,
+	0x8f7b837b, 0x807b7c7b,
+	0xbefe03c1, 0xbeff0380,
+	0x7e008700, 0xe0704000,
+	0x705d0000, 0x807c817c,
+	0x8070ff70, 0x00000080,
+	0xbf0a7b7c, 0xbf85fff8,
+	0xbf82013b, 0xbef4037e,
+	0x8775ff7f, 0x0000ffff,
+	0x8875ff75, 0x00040000,
+	0xbef60380, 0xbef703ff,
+	0x10807fac, 0xb97202dc,
+	0x8f729972, 0x876eff7f,
+	0x04000000, 0xbf840034,
 	0xbefe03c1, 0x907c9972,
 	0x877c817c, 0xbf06817c,
 	0xbf850002, 0xbeff0380,
 	0xbf820001, 0xbeff03c1,
-	0xb96f3a05, 0x806f816f,
-	0x8f6f826f, 0x907c9972,
-	0x877c817c, 0xbf06817c,
-	0xbf850024, 0xbef603ff,
-	0x01000000, 0xbeee0378,
+	0xb96f4306, 0x876fc16f,
+	0xbf840029, 0x8f6f866f,
+	0x8f6f826f, 0xbef6036f,
+	0xb9783a05, 0x80788178,
+	0xbf0d9972, 0xbf850002,
+	0x8f788978, 0xbf820001,
+	0x8f788a78, 0xb96e1e06,
+	0x8f6e8a6e, 0x80786e78,
 	0x8078ff78, 0x00000200,
-	0xbefc0384, 0xbf0a6f7c,
-	0xbf840050, 0xe0304000,
-	0x785d0000, 0xe0304080,
-	0x785d0100, 0xe0304100,
-	0x785d0200, 0xe0304180,
-	0x785d0300, 0xbf8c3f70,
-	0x7e008500, 0x7e028501,
-	0x7e048502, 0x7e068503,
-	0x807c847c, 0x8078ff78,
-	0x00000200, 0xbf0a6f7c,
-	0xbf85ffee, 0xe0304000,
-	0x6e5d0000, 0xe0304080,
-	0x6e5d0100, 0xe0304100,
-	0x6e5d0200, 0xe0304180,
-	0x6e5d0300, 0xbf8c3f70,
-	0xbf820034, 0xbef603ff,
-	0x01000000, 0xbeee0378,
-	0x8078ff78, 0x00000400,
-	0xbefc0384, 0xbf0a6f7c,
-	0xbf840012, 0xe0304000,
-	0x785d0000, 0xe0304100,
-	0x785d0100, 0xe0304200,
-	0x785d0200, 0xe0304300,
-	0x785d0300, 0xbf8c3f70,
-	0x7e008500, 0x7e028501,
-	0x7e048502, 0x7e068503,
-	0x807c847c, 0x8078ff78,
-	0x00000400, 0xbf0a6f7c,
-	0xbf85ffee, 0xb96f1e06,
-	0x876fc16f, 0xbf84000e,
-	0x8f6f836f, 0x806f7c6f,
-	0xbefe03c1, 0xbeff0380,
+	0x8078ff78, 0x00000080,
+	0xbef603ff, 0x01000000,
+	0x907c9972, 0x877c817c,
+	0xbf06817c, 0xbefc0380,
+	0xbf850009, 0xe0310000,
+	0x781d0000, 0x807cff7c,
+	0x00000080, 0x8078ff78,
+	0x00000080, 0xbf0a6f7c,
+	0xbf85fff8, 0xbf820008,
+	0xe0310000, 0x781d0000,
+	0x807cff7c, 0x00000100,
+	0x8078ff78, 0x00000100,
+	0xbf0a6f7c, 0xbf85fff8,
+	0xbef80380, 0xbefe03c1,
+	0x907c9972, 0x877c817c,
+	0xbf06817c, 0xbf850002,
+	0xbeff0380, 0xbf820001,
+	0xbeff03c1, 0xb96f3a05,
+	0x806f816f, 0x8f6f826f,
+	0x907c9972, 0x877c817c,
+	0xbf06817c, 0xbf850024,
+	0xbef603ff, 0x01000000,
+	0xbeee0378, 0x8078ff78,
+	0x00000200, 0xbefc0384,
+	0xbf0a6f7c, 0xbf840050,
 	0xe0304000, 0x785d0000,
+	0xe0304080, 0x785d0100,
+	0xe0304100, 0x785d0200,
+	0xe0304180, 0x785d0300,
 	0xbf8c3f70, 0x7e008500,
-	0x807c817c, 0x8078ff78,
-	0x00000080, 0xbf0a6f7c,
-	0xbf85fff7, 0xbeff03c1,
+	0x7e028501, 0x7e048502,
+	0x7e068503, 0x807c847c,
+	0x8078ff78, 0x00000200,
+	0xbf0a6f7c, 0xbf85ffee,
 	0xe0304000, 0x6e5d0000,
-	0xe0304100, 0x6e5d0100,
-	0xe0304200, 0x6e5d0200,
-	0xe0304300, 0x6e5d0300,
-	0xbf8c3f70, 0xb9783a05,
-	0x80788178, 0xbf0d9972,
-	0xbf850002, 0x8f788978,
-	0xbf820001, 0x8f788a78,
-	0xb96e1e06, 0x8f6e8a6e,
-	0x80786e78, 0x8078ff78,
-	0x00000200, 0x80f8ff78,
-	0x00000050, 0xbef603ff,
-	0x01000000, 0xbefc03ff,
-	0x0000006c, 0x80f89078,
-	0xf429003a, 0xf0000000,
-	0xbf8cc07f, 0x80fc847c,
-	0xbf800000, 0xbe803100,
-	0xbe823102, 0x80f8a078,
-	0xf42d003a, 0xf0000000,
-	0xbf8cc07f, 0x80fc887c,
-	0xbf800000, 0xbe803100,
-	0xbe823102, 0xbe843104,
-	0xbe863106, 0x80f8c078,
-	0xf431003a, 0xf0000000,
-	0xbf8cc07f, 0x80fc907c,
-	0xbf800000, 0xbe803100,
-	0xbe823102, 0xbe843104,
-	0xbe863106, 0xbe883108,
-	0xbe8a310a, 0xbe8c310c,
-	0xbe8e310e, 0xbf06807c,
-	0xbf84fff0, 0xba80f801,
-	0x00000000, 0xbf8a0000,
+	0xe0304080, 0x6e5d0100,
+	0xe0304100, 0x6e5d0200,
+	0xe0304180, 0x6e5d0300,
+	0xbf8c3f70, 0xbf820034,
+	0xbef603ff, 0x01000000,
+	0xbeee0378, 0x8078ff78,
+	0x00000400, 0xbefc0384,
+	0xbf0a6f7c, 0xbf840012,
+	0xe0304000, 0x785d0000,
+	0xe0304100, 0x785d0100,
+	0xe0304200, 0x785d0200,
+	0xe0304300, 0x785d0300,
+	0xbf8c3f70, 0x7e008500,
+	0x7e028501, 0x7e048502,
+	0x7e068503, 0x807c847c,
+	0x8078ff78, 0x00000400,
+	0xbf0a6f7c, 0xbf85ffee,
+	0xb96f1e06, 0x876fc16f,
+	0xbf84000e, 0x8f6f836f,
+	0x806f7c6f, 0xbefe03c1,
+	0xbeff0380, 0xe0304000,
+	0x785d0000, 0xbf8c3f70,
+	0x7e008500, 0x807c817c,
+	0x8078ff78, 0x00000080,
+	0xbf0a6f7c, 0xbf85fff7,
+	0xbeff03c1, 0xe0304000,
+	0x6e5d0000, 0xe0304100,
+	0x6e5d0100, 0xe0304200,
+	0x6e5d0200, 0xe0304300,
+	0x6e5d0300, 0xbf8c3f70,
 	0xb9783a05, 0x80788178,
 	0xbf0d9972, 0xbf850002,
 	0x8f788978, 0xbf820001,
 	0x8f788a78, 0xb96e1e06,
 	0x8f6e8a6e, 0x80786e78,
 	0x8078ff78, 0x00000200,
+	0x80f8ff78, 0x00000050,
 	0xbef603ff, 0x01000000,
-	0xf4211bfa, 0xf0000000,
-	0x80788478, 0xf4211b3a,
+	0xbefc03ff, 0x0000006c,
+	0x80f89078, 0xf429003a,
+	0xf0000000, 0xbf8cc07f,
+	0x80fc847c, 0xbf800000,
+	0xbe803100, 0xbe823102,
+	0x80f8a078, 0xf42d003a,
+	0xf0000000, 0xbf8cc07f,
+	0x80fc887c, 0xbf800000,
+	0xbe803100, 0xbe823102,
+	0xbe843104, 0xbe863106,
+	0x80f8c078, 0xf431003a,
+	0xf0000000, 0xbf8cc07f,
+	0x80fc907c, 0xbf800000,
+	0xbe803100, 0xbe823102,
+	0xbe843104, 0xbe863106,
+	0xbe883108, 0xbe8a310a,
+	0xbe8c310c, 0xbe8e310e,
+	0xbf06807c, 0xbf84fff0,
+	0xba80f801, 0x00000000,
+	0xbf8a0000, 0xb9783a05,
+	0x80788178, 0xbf0d9972,
+	0xbf850002, 0x8f788978,
+	0xbf820001, 0x8f788a78,
+	0xb96e1e06, 0x8f6e8a6e,
+	0x80786e78, 0x8078ff78,
+	0x00000200, 0xbef603ff,
+	0x01000000, 0xf4211bfa,
 	0xf0000000, 0x80788478,
-	0xf4211b7a, 0xf0000000,
-	0x80788478, 0xf4211c3a,
+	0xf4211b3a, 0xf0000000,
+	0x80788478, 0xf4211b7a,
 	0xf0000000, 0x80788478,
-	0xf4211c7a, 0xf0000000,
-	0x80788478, 0xf4211eba,
+	0xf4211c3a, 0xf0000000,
+	0x80788478, 0xf4211c7a,
 	0xf0000000, 0x80788478,
-	0xf4211efa, 0xf0000000,
-	0x80788478, 0xf4211e7a,
+	0xf4211eba, 0xf0000000,
+	0x80788478, 0xf4211efa,
 	0xf0000000, 0x80788478,
-	0xf4211cfa, 0xf0000000,
-	0x80788478, 0xf4211bba,
+	0xf4211e7a, 0xf0000000,
+	0x80788478, 0xf4211cfa,
 	0xf0000000, 0x80788478,
-	0xbf8cc07f, 0xb9eef814,
 	0xf4211bba, 0xf0000000,
 	0x80788478, 0xbf8cc07f,
-	0xb9eef815, 0xbefc036f,
-	0xbefe0370, 0xbeff0371,
-	0x876f7bff, 0x000003ff,
-	0xb9ef4803, 0x876f7bff,
-	0xfffff800, 0x906f8b6f,
-	0xb9efa2c3, 0xb9f3f801,
-	0xb96e3a05, 0x806e816e,
-	0xbf0d9972, 0xbf850002,
-	0x8f6e896e, 0xbf820001,
-	0x8f6e8a6e, 0xb96f1e06,
-	0x8f6f8a6f, 0x806e6f6e,
-	0x806eff6e, 0x00000200,
-	0x806e746e, 0x826f8075,
-	0x876fff6f, 0x0000ffff,
-	0xf4091c37, 0xfa000050,
-	0xf4091d37, 0xfa000060,
-	0xf4011e77, 0xfa000074,
-	0xbf8cc07f, 0x876dff6d,
-	0x0000ffff, 0x87fe7e7e,
-	0x87ea6a6a, 0xb9faf802,
-	0xbe80226c, 0xbf810000,
+	0xb9eef814, 0xf4211bba,
+	0xf0000000, 0x80788478,
+	0xbf8cc07f, 0xb9eef815,
+	0xbefc036f, 0xbefe0370,
+	0xbeff0371, 0x876f7bff,
+	0x000003ff, 0xb9ef4803,
+	0x876f7bff, 0xfffff800,
+	0x906f8b6f, 0xb9efa2c3,
+	0xb9f3f801, 0xb96e3a05,
+	0x806e816e, 0xbf0d9972,
+	0xbf850002, 0x8f6e896e,
+	0xbf820001, 0x8f6e8a6e,
+	0xb96f1e06, 0x8f6f8a6f,
+	0x806e6f6e, 0x806eff6e,
+	0x00000200, 0x806e746e,
+	0x826f8075, 0x876fff6f,
+	0x0000ffff, 0xf4091c37,
+	0xfa000050, 0xf4091d37,
+	0xfa000060, 0xf4011e77,
+	0xfa000074, 0xbf8cc07f,
+	0x876dff6d, 0x0000ffff,
+	0x87fe7e7e, 0x87ea6a6a,
+	0xb9faf802, 0xbe80226c,
+	0xbf810000, 0xbf9f0000,
 	0xbf9f0000, 0xbf9f0000,
 	0xbf9f0000, 0xbf9f0000,
-	0xbf9f0000, 0x00000000,
 };
 
 static const uint32_t cwsr_trap_gfx11_hex[] = {
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm
index fdab64624422..e0140df0b0ec 100644
--- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm
@@ -369,6 +369,12 @@ L_SLEEP:
 	s_or_b32	s_save_pc_hi, s_save_pc_hi, s_save_tmp
 
 #if NO_SQC_STORE
+#if ASIC_FAMILY <= CHIP_SIENNA_CICHLID
+	// gfx10: If there was a VALU exception, the exception state must be
+	// cleared before executing the VALU instructions below.
+	v_clrexcp
+#endif
+
 	// Trap temporaries must be saved via VGPR but all VGPRs are in use.
 	// There is no ttmp space to hold the resource constant for VGPR save.
 	// Save v0 by itself since it requires only two SGPRs.
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 9cc32f577e38..a40f8cfc6aa5 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -748,7 +748,6 @@ struct kfd_process_device {
 	/* VM context for GPUVM allocations */
 	struct file *drm_file;
 	void *drm_priv;
-	atomic64_t tlb_seq;
 
 	/* GPUVM allocations storage */
 	struct idr alloc_idr;
@@ -1462,7 +1461,14 @@ void kfd_signal_reset_event(struct kfd_node *dev);
 
 void kfd_signal_poison_consumed_event(struct kfd_node *dev, u32 pasid);
 
-void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type);
+static inline void kfd_flush_tlb(struct kfd_process_device *pdd,
+				 enum TLB_FLUSH_TYPE type)
+{
+	struct amdgpu_device *adev = pdd->dev->adev;
+	struct amdgpu_vm *vm = drm_priv_to_vm(pdd->drm_priv);
+
+	amdgpu_vm_flush_compute_tlb(adev, vm, type, pdd->dev->xcc_mask);
+}
 
 static inline bool kfd_flush_tlb_after_unmap(struct kfd_dev *dev)
 {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 7a33e06f5c90..c10d050e1a61 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -1667,7 +1667,6 @@ int kfd_process_device_init_vm(struct kfd_process_device *pdd,
 		return ret;
 	}
 	pdd->drm_priv = drm_file->private_data;
-	atomic64_set(&pdd->tlb_seq, 0);
 
 	ret = kfd_process_device_reserve_ib_mem(pdd);
 	if (ret)
@@ -2059,36 +2058,6 @@ int kfd_reserved_mem_mmap(struct kfd_node *dev, struct kfd_process *process,
 			       KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot);
 }
 
-void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type)
-{
-	struct amdgpu_vm *vm = drm_priv_to_vm(pdd->drm_priv);
-	uint64_t tlb_seq = amdgpu_vm_tlb_seq(vm);
-	struct kfd_node *dev = pdd->dev;
-	uint32_t xcc_mask = dev->xcc_mask;
-	int xcc = 0;
-
-	/*
-	 * It can be that we race and lose here, but that is extremely unlikely
-	 * and the worst thing which could happen is that we flush the changes
-	 * into the TLB once more which is harmless.
-	 */
-	if (atomic64_xchg(&pdd->tlb_seq, tlb_seq) == tlb_seq)
-		return;
-
-	if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
-		/* Nothing to flush until a VMID is assigned, which
-		 * only happens when the first queue is created.
-		 */
-		if (pdd->qpd.vmid)
-			amdgpu_amdkfd_flush_gpu_tlb_vmid(dev->adev,
-							pdd->qpd.vmid);
-	} else {
-		for_each_inst(xcc, xcc_mask)
-			amdgpu_amdkfd_flush_gpu_tlb_pasid(
-				dev->adev, pdd->process->pasid, type, xcc);
-	}
-}
-
 /* assumes caller holds process lock. */
 int kfd_process_drain_interrupts(struct kfd_process_device *pdd)
 {
diff --git a/drivers/gpu/drm/amd/display/Makefile b/drivers/gpu/drm/amd/display/Makefile
index af17ab8027df..71192fc81a20 100644
--- a/drivers/gpu/drm/amd/display/Makefile
+++ b/drivers/gpu/drm/amd/display/Makefile
@@ -30,6 +30,7 @@ subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/inc/
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/inc/hw
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/clk_mgr
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/hwss
+subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/dc/resource
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/modules/inc
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/modules/freesync
 subdir-ccflags-y += -I$(FULL_AMD_DISPLAY_PATH)/modules/color
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
index ee97814ebd99..aa43f1761acd 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
@@ -85,8 +85,10 @@
 #include <drm/drm_atomic_uapi.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
+#include <drm/drm_fixed.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_eld.h>
 #include <drm/drm_vblank.h>
 #include <drm/drm_audio_component.h>
 #include <drm/drm_gem_atomic_helper.h>
@@ -6909,8 +6911,8 @@ static int dm_encoder_helper_atomic_check(struct drm_encoder *encoder,
 	if (IS_ERR(mst_state))
 		return PTR_ERR(mst_state);
 
-	if (!mst_state->pbn_div)
-		mst_state->pbn_div = dm_mst_get_pbn_divider(aconnector->mst_root->dc_link);
+	if (!mst_state->pbn_div.full)
+		mst_state->pbn_div.full = dfixed_const(dm_mst_get_pbn_divider(aconnector->mst_root->dc_link));
 
 	if (!state->duplicated) {
 		int max_bpc = conn_state->max_requested_bpc;
@@ -6922,7 +6924,7 @@ static int dm_encoder_helper_atomic_check(struct drm_encoder *encoder,
 								    max_bpc);
 		bpp = convert_dc_color_depth_into_bpc(color_depth) * 3;
 		clock = adjusted_mode->clock;
-		dm_new_connector_state->pbn = drm_dp_calc_pbn_mode(clock, bpp, false);
+		dm_new_connector_state->pbn = drm_dp_calc_pbn_mode(clock, bpp << 4);
 	}
 
 	dm_new_connector_state->vcpi_slots =
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
index 13a177d34376..98b41ec7288e 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
@@ -2971,6 +2971,104 @@ static int allow_edp_hotplug_detection_set(void *data, u64 val)
 	return 0;
 }
 
+static int dmub_trace_mask_set(void *data, u64 val)
+{
+	struct amdgpu_device *adev = data;
+	struct dmub_srv *srv = adev->dm.dc->ctx->dmub_srv->dmub;
+	enum dmub_gpint_command cmd;
+	enum dmub_status status;
+	u64 mask = 0xffff;
+	u8 shift = 0;
+	u32 res;
+	int i;
+
+	if (!srv->fw_version)
+		return -EINVAL;
+
+	for (i = 0;  i < 4; i++) {
+		res = (val & mask) >> shift;
+
+		switch (i) {
+		case 0:
+			cmd = DMUB_GPINT__SET_TRACE_BUFFER_MASK_WORD0;
+			break;
+		case 1:
+			cmd = DMUB_GPINT__SET_TRACE_BUFFER_MASK_WORD1;
+			break;
+		case 2:
+			cmd = DMUB_GPINT__SET_TRACE_BUFFER_MASK_WORD2;
+			break;
+		case 3:
+			cmd = DMUB_GPINT__SET_TRACE_BUFFER_MASK_WORD3;
+			break;
+		}
+
+		status = dmub_srv_send_gpint_command(srv, cmd, res, 30);
+
+		if (status == DMUB_STATUS_TIMEOUT)
+			return -ETIMEDOUT;
+		else if (status == DMUB_STATUS_INVALID)
+			return -EINVAL;
+		else if (status != DMUB_STATUS_OK)
+			return -EIO;
+
+		usleep_range(100, 1000);
+
+		mask <<= 16;
+		shift += 16;
+	}
+
+	return 0;
+}
+
+static int dmub_trace_mask_show(void *data, u64 *val)
+{
+	enum dmub_gpint_command cmd = DMUB_GPINT__GET_TRACE_BUFFER_MASK_WORD0;
+	struct amdgpu_device *adev = data;
+	struct dmub_srv *srv = adev->dm.dc->ctx->dmub_srv->dmub;
+	enum dmub_status status;
+	u8 shift = 0;
+	u64 raw = 0;
+	u64 res = 0;
+	int i = 0;
+
+	if (!srv->fw_version)
+		return -EINVAL;
+
+	while (i < 4) {
+		status = dmub_srv_send_gpint_command(srv, cmd, 0, 30);
+
+		if (status == DMUB_STATUS_OK) {
+			status = dmub_srv_get_gpint_response(srv, (u32 *) &raw);
+
+			if (status == DMUB_STATUS_INVALID)
+				return -EINVAL;
+			else if (status != DMUB_STATUS_OK)
+				return -EIO;
+		} else if (status == DMUB_STATUS_TIMEOUT) {
+			return -ETIMEDOUT;
+		} else if (status == DMUB_STATUS_INVALID) {
+			return -EINVAL;
+		} else {
+			return -EIO;
+		}
+
+		usleep_range(100, 1000);
+
+		cmd++;
+		res |= (raw << shift);
+		shift += 16;
+		i++;
+	}
+
+	*val = res;
+
+	return 0;
+}
+
+DEFINE_DEBUGFS_ATTRIBUTE(dmub_trace_mask_fops, dmub_trace_mask_show,
+			 dmub_trace_mask_set, "0x%llx\n");
+
 /*
  * Set dmcub trace event IRQ enable or disable.
  * Usage to enable dmcub trace event IRQ: echo 1 > /sys/kernel/debug/dri/0/amdgpu_dm_dmcub_trace_event_en
@@ -3647,12 +3745,16 @@ static int capabilities_show(struct seq_file *m, void *unused)
 	bool mall_supported = dc->caps.mall_size_total;
 	bool subvp_supported = dc->caps.subvp_fw_processing_delay_us;
 	unsigned int mall_in_use = false;
-	unsigned int subvp_in_use = dc->cap_funcs.get_subvp_en(dc, dc->current_state);
+	unsigned int subvp_in_use = false;
+
 	struct hubbub *hubbub = dc->res_pool->hubbub;
 
 	if (hubbub->funcs->get_mall_en)
 		hubbub->funcs->get_mall_en(hubbub, &mall_in_use);
 
+	if (dc->cap_funcs.get_subvp_en)
+		subvp_in_use = dc->cap_funcs.get_subvp_en(dc, dc->current_state);
+
 	seq_printf(m, "mall supported: %s, enabled: %s\n",
 			   mall_supported ? "yes" : "no", mall_in_use ? "yes" : "no");
 	seq_printf(m, "sub-viewport supported: %s, enabled: %s\n",
@@ -3880,6 +3982,9 @@ void dtn_debugfs_init(struct amdgpu_device *adev)
 	debugfs_create_file_unsafe("amdgpu_dm_force_timing_sync", 0644, root,
 				   adev, &force_timing_sync_ops);
 
+	debugfs_create_file_unsafe("amdgpu_dm_dmub_trace_mask", 0644, root,
+				   adev, &dmub_trace_mask_fops);
+
 	debugfs_create_file_unsafe("amdgpu_dm_dmcub_trace_event_en", 0644, root,
 				   adev, &dmcub_trace_event_state_fops);
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
index c7a29bb737e2..eb6121ad92fd 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
@@ -31,6 +31,7 @@
 #include <drm/drm_probe_helper.h>
 #include <drm/amdgpu_drm.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_fixed.h>
 
 #include "dm_services.h"
 #include "amdgpu.h"
@@ -210,7 +211,7 @@ static void dm_helpers_construct_old_payload(
 			struct drm_dp_mst_atomic_payload *old_payload)
 {
 	struct drm_dp_mst_atomic_payload *pos;
-	int pbn_per_slot = mst_state->pbn_div;
+	int pbn_per_slot = dfixed_trunc(mst_state->pbn_div);
 	u8 next_payload_vc_start = mgr->next_start_slot;
 	u8 payload_vc_start = new_payload->vc_start_slot;
 	u8 allocated_time_slots;
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
index 11da0eebee6c..21b6ac6c3944 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
@@ -27,6 +27,7 @@
 #include <drm/display/drm_dp_mst_helper.h>
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
+#include <drm/drm_fixed.h>
 #include "dm_services.h"
 #include "amdgpu.h"
 #include "amdgpu_dm.h"
@@ -44,7 +45,7 @@
 #include "amdgpu_dm_debugfs.h"
 #endif
 
-#include "dc/dcn20/dcn20_resource.h"
+#include "dc/resource/dcn20/dcn20_resource.h"
 
 #define PEAK_FACTOR_X1000 1006
 
@@ -941,10 +942,10 @@ static int increase_dsc_bpp(struct drm_atomic_state *state,
 		link_timeslots_used = 0;
 
 		for (i = 0; i < count; i++)
-			link_timeslots_used += DIV_ROUND_UP(vars[i + k].pbn, mst_state->pbn_div);
+			link_timeslots_used += DIV_ROUND_UP(vars[i + k].pbn, dfixed_trunc(mst_state->pbn_div));
 
 		fair_pbn_alloc =
-			(63 - link_timeslots_used) / remaining_to_increase * mst_state->pbn_div;
+			(63 - link_timeslots_used) / remaining_to_increase * dfixed_trunc(mst_state->pbn_div);
 
 		if (initial_slack[next_index] > fair_pbn_alloc) {
 			vars[next_index].pbn += fair_pbn_alloc;
@@ -1602,9 +1603,8 @@ enum dc_status dm_dp_mst_is_port_support_mode(
 	struct dc_link_settings cur_link_settings;
 	unsigned int end_to_end_bw_in_kbps = 0;
 	unsigned int upper_link_bw_in_kbps = 0, down_link_bw_in_kbps = 0;
-	unsigned int max_compressed_bw_in_kbps = 0;
 	struct dc_dsc_bw_range bw_range = {0};
-	uint16_t full_pbn = aconnector->mst_output_port->full_pbn;
+	struct dc_dsc_config_options dsc_options = {0};
 
 	/*
 	 * Consider the case with the depth of the mst topology tree is equal or less than 2
@@ -1620,30 +1620,39 @@ enum dc_status dm_dp_mst_is_port_support_mode(
 	   (aconnector->mst_output_port->passthrough_aux ||
 	    aconnector->dsc_aux == &aconnector->mst_output_port->aux)) {
 		cur_link_settings = stream->link->verified_link_cap;
+		upper_link_bw_in_kbps = dc_link_bandwidth_kbps(aconnector->dc_link, &cur_link_settings);
+		down_link_bw_in_kbps = kbps_from_pbn(aconnector->mst_output_port->full_pbn);
 
-		upper_link_bw_in_kbps = dc_link_bandwidth_kbps(aconnector->dc_link,
-							       &cur_link_settings);
-		down_link_bw_in_kbps = kbps_from_pbn(full_pbn);
-
-		/* pick the bottleneck */
-		end_to_end_bw_in_kbps = min(upper_link_bw_in_kbps,
-					    down_link_bw_in_kbps);
-
-		/*
-		 * use the maximum dsc compression bandwidth as the required
-		 * bandwidth for the mode
-		 */
-		max_compressed_bw_in_kbps = bw_range.min_kbps;
+		/* pick the end to end bw bottleneck */
+		end_to_end_bw_in_kbps = min(upper_link_bw_in_kbps, down_link_bw_in_kbps);
 
-		if (end_to_end_bw_in_kbps < max_compressed_bw_in_kbps) {
-			DRM_DEBUG_DRIVER("Mode does not fit into DSC pass-through bandwidth validation\n");
+		if (end_to_end_bw_in_kbps < bw_range.min_kbps) {
+			DRM_DEBUG_DRIVER("maximum dsc compression cannot fit into end-to-end bandwidth\n");
 			return DC_FAIL_BANDWIDTH_VALIDATE;
 		}
+
+		if (end_to_end_bw_in_kbps < bw_range.stream_kbps) {
+			dc_dsc_get_default_config_option(stream->link->dc, &dsc_options);
+			dsc_options.max_target_bpp_limit_override_x16 = aconnector->base.display_info.max_dsc_bpp * 16;
+			if (dc_dsc_compute_config(stream->sink->ctx->dc->res_pool->dscs[0],
+					&stream->sink->dsc_caps.dsc_dec_caps,
+					&dsc_options,
+					end_to_end_bw_in_kbps,
+					&stream->timing,
+					dc_link_get_highest_encoding_format(stream->link),
+					&stream->timing.dsc_cfg)) {
+				stream->timing.flags.DSC = 1;
+				DRM_DEBUG_DRIVER("end-to-end bandwidth require dsc and dsc config found\n");
+			} else {
+				DRM_DEBUG_DRIVER("end-to-end bandwidth require dsc but dsc config not found\n");
+				return DC_FAIL_BANDWIDTH_VALIDATE;
+			}
+		}
 	} else {
 		/* check if mode could be supported within full_pbn */
 		bpp = convert_dc_color_depth_into_bpc(stream->timing.display_color_depth) * 3;
-		pbn = drm_dp_calc_pbn_mode(stream->timing.pix_clk_100hz / 10, bpp, false);
-		if (pbn > full_pbn)
+		pbn = drm_dp_calc_pbn_mode(stream->timing.pix_clk_100hz / 10, bpp << 4);
+		if (pbn > aconnector->mst_output_port->full_pbn)
 			return DC_FAIL_BANDWIDTH_VALIDATE;
 	}
 
diff --git a/drivers/gpu/drm/amd/display/dc/Makefile b/drivers/gpu/drm/amd/display/dc/Makefile
index 3a169b78e7e4..ab51a065cf0e 100644
--- a/drivers/gpu/drm/amd/display/dc/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/Makefile
@@ -22,7 +22,7 @@
 #
 # Makefile for Display Core (dc) component.
 
-DC_LIBS = basics bios dml clk_mgr dce gpio hwss irq link virtual dsc
+DC_LIBS = basics bios dml clk_mgr dce gpio hwss irq link virtual dsc resource
 
 ifdef CONFIG_DRM_AMD_DC_FP
 
@@ -38,8 +38,6 @@ DC_LIBS += dcn302
 DC_LIBS += dcn303
 DC_LIBS += dcn31
 DC_LIBS += dcn314
-DC_LIBS += dcn315
-DC_LIBS += dcn316
 DC_LIBS += dcn32
 DC_LIBS += dcn321
 DC_LIBS += dcn35
@@ -51,7 +49,6 @@ DC_LIBS += dce120
 
 DC_LIBS += dce112
 DC_LIBS += dce110
-DC_LIBS += dce100
 DC_LIBS += dce80
 
 ifdef CONFIG_DRM_AMD_DC_SI
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_smu.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_smu.c
index b6b8c3ca1572..af0a0f292595 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_smu.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_smu.c
@@ -116,6 +116,9 @@ static uint32_t dcn35_smu_wait_for_response(struct clk_mgr_internal *clk_mgr, un
 			msleep(delay_us/1000);
 		else if (delay_us > 0)
 			udelay(delay_us);
+
+		if (clk_mgr->base.ctx->dc->debug.disable_timeout)
+			max_retries++;
 	} while (max_retries--);
 
 	return res_val;
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
index a1f1d1003992..42a927710743 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
@@ -69,8 +69,8 @@
 #include "dcn314/dcn314_resource.h"
 #include "dcn315/dcn315_resource.h"
 #include "dcn316/dcn316_resource.h"
-#include "../dcn32/dcn32_resource.h"
-#include "../dcn321/dcn321_resource.h"
+#include "dcn32/dcn32_resource.h"
+#include "dcn321/dcn321_resource.h"
 #include "dcn35/dcn35_resource.h"
 
 #define VISUAL_CONFIRM_BASE_DEFAULT 3
@@ -1764,6 +1764,29 @@ int recource_find_free_pipe_not_used_in_cur_res_ctx(
 	return free_pipe_idx;
 }
 
+int recource_find_free_pipe_used_as_otg_master_in_cur_res_ctx(
+		const struct resource_context *cur_res_ctx,
+		struct resource_context *new_res_ctx,
+		const struct resource_pool *pool)
+{
+	int free_pipe_idx = FREE_PIPE_INDEX_NOT_FOUND;
+	const struct pipe_ctx *new_pipe, *cur_pipe;
+	int i;
+
+	for (i = 0; i < pool->pipe_count; i++) {
+		cur_pipe = &cur_res_ctx->pipe_ctx[i];
+		new_pipe = &new_res_ctx->pipe_ctx[i];
+
+		if (resource_is_pipe_type(cur_pipe, OTG_MASTER) &&
+				resource_is_pipe_type(new_pipe, FREE_PIPE)) {
+			free_pipe_idx = i;
+			break;
+		}
+	}
+
+	return free_pipe_idx;
+}
+
 int resource_find_free_pipe_used_as_cur_sec_dpp_in_mpcc_combine(
 		const struct resource_context *cur_res_ctx,
 		struct resource_context *new_res_ctx,
@@ -3586,6 +3609,7 @@ static void mark_seamless_boot_stream(
  *       |________|_______________|___________|_____________|
  */
 static bool acquire_otg_master_pipe_for_stream(
+		const struct dc_state *cur_ctx,
 		struct dc_state *new_ctx,
 		const struct resource_pool *pool,
 		struct dc_stream_state *stream)
@@ -3599,7 +3623,22 @@ static bool acquire_otg_master_pipe_for_stream(
 	int pipe_idx;
 	struct pipe_ctx *pipe_ctx = NULL;
 
-	pipe_idx = resource_find_any_free_pipe(&new_ctx->res_ctx, pool);
+	/*
+	 * Upper level code is responsible to optimize unnecessary addition and
+	 * removal for unchanged streams. So unchanged stream will keep the same
+	 * OTG master instance allocated. When current stream is removed and a
+	 * new stream is added, we want to reuse the OTG instance made available
+	 * by the removed stream first. If not found, we try to avoid of using
+	 * any free pipes already used in current context as this could tear
+	 * down exiting ODM/MPC/MPO configuration unnecessarily.
+	 */
+	pipe_idx = recource_find_free_pipe_used_as_otg_master_in_cur_res_ctx(
+			&cur_ctx->res_ctx, &new_ctx->res_ctx, pool);
+	if (pipe_idx == FREE_PIPE_INDEX_NOT_FOUND)
+		pipe_idx = recource_find_free_pipe_not_used_in_cur_res_ctx(
+				&cur_ctx->res_ctx, &new_ctx->res_ctx, pool);
+	if (pipe_idx == FREE_PIPE_INDEX_NOT_FOUND)
+		pipe_idx = resource_find_any_free_pipe(&new_ctx->res_ctx, pool);
 	if (pipe_idx != FREE_PIPE_INDEX_NOT_FOUND) {
 		pipe_ctx = &new_ctx->res_ctx.pipe_ctx[pipe_idx];
 		memset(pipe_ctx, 0, sizeof(*pipe_ctx));
@@ -3659,7 +3698,7 @@ enum dc_status resource_map_pool_resources(
 
 	if (!acquired)
 		/* acquire new resources */
-		acquired = acquire_otg_master_pipe_for_stream(
+		acquired = acquire_otg_master_pipe_for_stream(dc->current_state,
 				context, pool, stream);
 
 	pipe_ctx = resource_get_otg_master_for_stream(&context->res_ctx, stream);
diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h
index 9316b737a8ba..541e781267b9 100644
--- a/drivers/gpu/drm/amd/display/dc/dc.h
+++ b/drivers/gpu/drm/amd/display/dc/dc.h
@@ -49,7 +49,7 @@ struct aux_payload;
 struct set_config_cmd_payload;
 struct dmub_notification;
 
-#define DC_VER "3.2.259"
+#define DC_VER "3.2.260"
 
 #define MAX_SURFACES 3
 #define MAX_PLANES 6
@@ -978,6 +978,7 @@ struct dc_debug_options {
 	bool psp_disabled_wa;
 	unsigned int ips2_eval_delay_us;
 	unsigned int ips2_entry_delay_us;
+	bool disable_timeout;
 };
 
 struct gpu_info_soc_bounding_box_v1_0;
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
index 0e07699c1e83..53400cc05b5b 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
@@ -241,7 +241,12 @@ bool dc_dmub_srv_cmd_run_list(struct dc_dmub_srv *dc_dmub_srv, unsigned int coun
 
 	// Wait for DMUB to process command
 	if (wait_type != DM_DMUB_WAIT_TYPE_NO_WAIT) {
-		status = dmub_srv_wait_for_idle(dmub, 100000);
+		if (dc_dmub_srv->ctx->dc->debug.disable_timeout) {
+			do {
+				status = dmub_srv_wait_for_idle(dmub, 100000);
+			} while (status != DMUB_STATUS_OK);
+		} else
+			status = dmub_srv_wait_for_idle(dmub, 100000);
 
 		if (status != DMUB_STATUS_OK) {
 			DC_LOG_DEBUG("No reply for DMUB command: status=%d\n", status);
@@ -1147,10 +1152,16 @@ bool dc_dmub_srv_is_hw_pwr_up(struct dc_dmub_srv *dc_dmub_srv, bool wait)
 		return true;
 
 	if (wait) {
-		status = dmub_srv_wait_for_hw_pwr_up(dc_dmub_srv->dmub, 500000);
-		if (status != DMUB_STATUS_OK) {
-			DC_ERROR("Error querying DMUB hw power up status: error=%d\n", status);
-			return false;
+		if (dc_dmub_srv->ctx->dc->debug.disable_timeout) {
+			do {
+				status = dmub_srv_wait_for_hw_pwr_up(dc_dmub_srv->dmub, 500000);
+			} while (status != DMUB_STATUS_OK);
+		} else {
+			status = dmub_srv_wait_for_hw_pwr_up(dc_dmub_srv->dmub, 500000);
+			if (status != DMUB_STATUS_OK) {
+				DC_ERROR("Error querying DMUB hw power up status: error=%d\n", status);
+				return false;
+			}
 		}
 	} else
 		return dmub_srv_is_hw_pwr_up(dc_dmub_srv->dmub);
@@ -1187,7 +1198,7 @@ void dc_dmub_srv_exit_low_power_state(const struct dc *dc)
 	const uint32_t max_num_polls = 10000;
 	uint32_t allow_state = 0;
 	uint32_t commit_state = 0;
-	uint32_t i;
+	int i;
 
 	if (dc->debug.dmcub_emulation)
 		return;
@@ -1220,6 +1231,9 @@ void dc_dmub_srv_exit_low_power_state(const struct dc *dc)
 						break;
 
 					udelay(1);
+
+					if (dc->debug.disable_timeout)
+						i--;
 				}
 				ASSERT(i < max_num_polls);
 
@@ -1242,6 +1256,9 @@ void dc_dmub_srv_exit_low_power_state(const struct dc *dc)
 					break;
 
 				udelay(1);
+
+				if (dc->debug.disable_timeout)
+					i--;
 			}
 			ASSERT(i < max_num_polls);
 		}
diff --git a/drivers/gpu/drm/amd/display/dc/dce100/Makefile b/drivers/gpu/drm/amd/display/dc/dce100/Makefile
deleted file mode 100644
index 0d2f6bbf7558..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dce100/Makefile
+++ /dev/null
@@ -1,46 +0,0 @@
-#
-# Copyright 2017 Advanced Micro Devices, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the "Software"),
-# to deal in the Software without restriction, including without limitation
-# the rights to use, copy, modify, merge, publish, distribute, sublicense,
-# and/or sell copies of the Software, and to permit persons to whom the
-# Software is furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
-# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
-# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
-# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-# OTHER DEALINGS IN THE SOFTWARE.
-#
-#
-# Makefile for the 'controller' sub-component of DAL.
-# It provides the control and status of HW CRTC block.
-
-CFLAGS_$(AMDDALPATH)/dc/dce100/dce100_resource.o = $(call cc-disable-warning, override-init)
-
-DCE100 = dce100_resource.o
-
-AMD_DAL_DCE100 = $(addprefix $(AMDDALPATH)/dc/dce100/,$(DCE100))
-
-AMD_DISPLAY_FILES += $(AMD_DAL_DCE100)
-
-
-###############################################################################
-# DCE 10x
-###############################################################################
-ifdef 0#CONFIG_DRM_AMD_DC_DCE11_0
-TG_DCE100 = dce100_resource.o
-
-AMD_DAL_TG_DCE100 = $(addprefix \
-	$(AMDDALPATH)/dc/dce100/,$(TG_DCE100))
-
-AMD_DISPLAY_FILES += $(AMD_DAL_TG_DCE100)
-endif
-
diff --git a/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c b/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c
deleted file mode 100644
index 53a5f4cb648c..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c
+++ /dev/null
@@ -1,1179 +0,0 @@
-/*
- * Copyright 2012-15 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#include "dm_services.h"
-
-#include "link_encoder.h"
-#include "stream_encoder.h"
-
-#include "resource.h"
-#include "include/irq_service_interface.h"
-#include "virtual/virtual_stream_encoder.h"
-#include "dce110/dce110_resource.h"
-#include "dce110/dce110_timing_generator.h"
-#include "irq/dce110/irq_service_dce110.h"
-#include "dce/dce_link_encoder.h"
-#include "dce/dce_stream_encoder.h"
-#include "dce/dce_mem_input.h"
-#include "dce/dce_ipp.h"
-#include "dce/dce_transform.h"
-#include "dce/dce_opp.h"
-#include "dce/dce_clock_source.h"
-#include "dce/dce_audio.h"
-#include "dce/dce_hwseq.h"
-#include "dce100/dce100_hwseq.h"
-#include "dce/dce_panel_cntl.h"
-
-#include "reg_helper.h"
-
-#include "dce/dce_10_0_d.h"
-#include "dce/dce_10_0_sh_mask.h"
-
-#include "dce/dce_dmcu.h"
-#include "dce/dce_aux.h"
-#include "dce/dce_abm.h"
-#include "dce/dce_i2c.h"
-
-#include "dce100_resource.h"
-
-#ifndef mmMC_HUB_RDREQ_DMIF_LIMIT
-#include "gmc/gmc_8_2_d.h"
-#include "gmc/gmc_8_2_sh_mask.h"
-#endif
-
-#ifndef mmDP_DPHY_INTERNAL_CTRL
-	#define mmDP_DPHY_INTERNAL_CTRL 0x4aa7
-	#define mmDP0_DP_DPHY_INTERNAL_CTRL 0x4aa7
-	#define mmDP1_DP_DPHY_INTERNAL_CTRL 0x4ba7
-	#define mmDP2_DP_DPHY_INTERNAL_CTRL 0x4ca7
-	#define mmDP3_DP_DPHY_INTERNAL_CTRL 0x4da7
-	#define mmDP4_DP_DPHY_INTERNAL_CTRL 0x4ea7
-	#define mmDP5_DP_DPHY_INTERNAL_CTRL 0x4fa7
-	#define mmDP6_DP_DPHY_INTERNAL_CTRL 0x54a7
-	#define mmDP7_DP_DPHY_INTERNAL_CTRL 0x56a7
-	#define mmDP8_DP_DPHY_INTERNAL_CTRL 0x57a7
-#endif
-
-#ifndef mmBIOS_SCRATCH_2
-	#define mmBIOS_SCRATCH_2 0x05CB
-	#define mmBIOS_SCRATCH_3 0x05CC
-	#define mmBIOS_SCRATCH_6 0x05CF
-#endif
-
-#ifndef mmDP_DPHY_BS_SR_SWAP_CNTL
-	#define mmDP_DPHY_BS_SR_SWAP_CNTL                       0x4ADC
-	#define mmDP0_DP_DPHY_BS_SR_SWAP_CNTL                   0x4ADC
-	#define mmDP1_DP_DPHY_BS_SR_SWAP_CNTL                   0x4BDC
-	#define mmDP2_DP_DPHY_BS_SR_SWAP_CNTL                   0x4CDC
-	#define mmDP3_DP_DPHY_BS_SR_SWAP_CNTL                   0x4DDC
-	#define mmDP4_DP_DPHY_BS_SR_SWAP_CNTL                   0x4EDC
-	#define mmDP5_DP_DPHY_BS_SR_SWAP_CNTL                   0x4FDC
-	#define mmDP6_DP_DPHY_BS_SR_SWAP_CNTL                   0x54DC
-#endif
-
-#ifndef mmDP_DPHY_FAST_TRAINING
-	#define mmDP_DPHY_FAST_TRAINING                         0x4ABC
-	#define mmDP0_DP_DPHY_FAST_TRAINING                     0x4ABC
-	#define mmDP1_DP_DPHY_FAST_TRAINING                     0x4BBC
-	#define mmDP2_DP_DPHY_FAST_TRAINING                     0x4CBC
-	#define mmDP3_DP_DPHY_FAST_TRAINING                     0x4DBC
-	#define mmDP4_DP_DPHY_FAST_TRAINING                     0x4EBC
-	#define mmDP5_DP_DPHY_FAST_TRAINING                     0x4FBC
-	#define mmDP6_DP_DPHY_FAST_TRAINING                     0x54BC
-#endif
-
-static const struct dce110_timing_generator_offsets dce100_tg_offsets[] = {
-	{
-		.crtc = (mmCRTC0_CRTC_CONTROL - mmCRTC_CONTROL),
-		.dcp =  (mmDCP0_GRPH_CONTROL - mmGRPH_CONTROL),
-	},
-	{
-		.crtc = (mmCRTC1_CRTC_CONTROL - mmCRTC_CONTROL),
-		.dcp = (mmDCP1_GRPH_CONTROL - mmGRPH_CONTROL),
-	},
-	{
-		.crtc = (mmCRTC2_CRTC_CONTROL - mmCRTC_CONTROL),
-		.dcp = (mmDCP2_GRPH_CONTROL - mmGRPH_CONTROL),
-	},
-	{
-		.crtc = (mmCRTC3_CRTC_CONTROL - mmCRTC_CONTROL),
-		.dcp =  (mmDCP3_GRPH_CONTROL - mmGRPH_CONTROL),
-	},
-	{
-		.crtc = (mmCRTC4_CRTC_CONTROL - mmCRTC_CONTROL),
-		.dcp = (mmDCP4_GRPH_CONTROL - mmGRPH_CONTROL),
-	},
-	{
-		.crtc = (mmCRTC5_CRTC_CONTROL - mmCRTC_CONTROL),
-		.dcp = (mmDCP5_GRPH_CONTROL - mmGRPH_CONTROL),
-	}
-};
-
-/* set register offset */
-#define SR(reg_name)\
-	.reg_name = mm ## reg_name
-
-/* set register offset with instance */
-#define SRI(reg_name, block, id)\
-	.reg_name = mm ## block ## id ## _ ## reg_name
-
-#define ipp_regs(id)\
-[id] = {\
-		IPP_DCE100_REG_LIST_DCE_BASE(id)\
-}
-
-static const struct dce_ipp_registers ipp_regs[] = {
-		ipp_regs(0),
-		ipp_regs(1),
-		ipp_regs(2),
-		ipp_regs(3),
-		ipp_regs(4),
-		ipp_regs(5)
-};
-
-static const struct dce_ipp_shift ipp_shift = {
-		IPP_DCE100_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
-};
-
-static const struct dce_ipp_mask ipp_mask = {
-		IPP_DCE100_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
-};
-
-#define transform_regs(id)\
-[id] = {\
-		XFM_COMMON_REG_LIST_DCE100(id)\
-}
-
-static const struct dce_transform_registers xfm_regs[] = {
-		transform_regs(0),
-		transform_regs(1),
-		transform_regs(2),
-		transform_regs(3),
-		transform_regs(4),
-		transform_regs(5)
-};
-
-static const struct dce_transform_shift xfm_shift = {
-		XFM_COMMON_MASK_SH_LIST_DCE110(__SHIFT)
-};
-
-static const struct dce_transform_mask xfm_mask = {
-		XFM_COMMON_MASK_SH_LIST_DCE110(_MASK)
-};
-
-#define aux_regs(id)\
-[id] = {\
-	AUX_REG_LIST(id)\
-}
-
-static const struct dce110_link_enc_aux_registers link_enc_aux_regs[] = {
-		aux_regs(0),
-		aux_regs(1),
-		aux_regs(2),
-		aux_regs(3),
-		aux_regs(4),
-		aux_regs(5)
-};
-
-#define hpd_regs(id)\
-[id] = {\
-	HPD_REG_LIST(id)\
-}
-
-static const struct dce110_link_enc_hpd_registers link_enc_hpd_regs[] = {
-		hpd_regs(0),
-		hpd_regs(1),
-		hpd_regs(2),
-		hpd_regs(3),
-		hpd_regs(4),
-		hpd_regs(5)
-};
-
-#define link_regs(id)\
-[id] = {\
-	LE_DCE100_REG_LIST(id)\
-}
-
-static const struct dce110_link_enc_registers link_enc_regs[] = {
-	link_regs(0),
-	link_regs(1),
-	link_regs(2),
-	link_regs(3),
-	link_regs(4),
-	link_regs(5),
-	link_regs(6),
-};
-
-#define stream_enc_regs(id)\
-[id] = {\
-	SE_COMMON_REG_LIST_DCE_BASE(id),\
-	.AFMT_CNTL = 0,\
-}
-
-static const struct dce110_stream_enc_registers stream_enc_regs[] = {
-	stream_enc_regs(0),
-	stream_enc_regs(1),
-	stream_enc_regs(2),
-	stream_enc_regs(3),
-	stream_enc_regs(4),
-	stream_enc_regs(5),
-	stream_enc_regs(6)
-};
-
-static const struct dce_stream_encoder_shift se_shift = {
-		SE_COMMON_MASK_SH_LIST_DCE80_100(__SHIFT)
-};
-
-static const struct dce_stream_encoder_mask se_mask = {
-		SE_COMMON_MASK_SH_LIST_DCE80_100(_MASK)
-};
-
-static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
-	{ DCE_PANEL_CNTL_REG_LIST() }
-};
-
-static const struct dce_panel_cntl_shift panel_cntl_shift = {
-	DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_panel_cntl_mask panel_cntl_mask = {
-	DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
-};
-
-#define opp_regs(id)\
-[id] = {\
-	OPP_DCE_100_REG_LIST(id),\
-}
-
-static const struct dce_opp_registers opp_regs[] = {
-	opp_regs(0),
-	opp_regs(1),
-	opp_regs(2),
-	opp_regs(3),
-	opp_regs(4),
-	opp_regs(5)
-};
-
-static const struct dce_opp_shift opp_shift = {
-	OPP_COMMON_MASK_SH_LIST_DCE_100(__SHIFT)
-};
-
-static const struct dce_opp_mask opp_mask = {
-	OPP_COMMON_MASK_SH_LIST_DCE_100(_MASK)
-};
-#define aux_engine_regs(id)\
-[id] = {\
-	AUX_COMMON_REG_LIST(id), \
-	.AUX_RESET_MASK = 0 \
-}
-
-static const struct dce110_aux_registers aux_engine_regs[] = {
-		aux_engine_regs(0),
-		aux_engine_regs(1),
-		aux_engine_regs(2),
-		aux_engine_regs(3),
-		aux_engine_regs(4),
-		aux_engine_regs(5)
-};
-
-#define audio_regs(id)\
-[id] = {\
-	AUD_COMMON_REG_LIST(id)\
-}
-
-static const struct dce_audio_registers audio_regs[] = {
-	audio_regs(0),
-	audio_regs(1),
-	audio_regs(2),
-	audio_regs(3),
-	audio_regs(4),
-	audio_regs(5),
-	audio_regs(6),
-};
-
-static const struct dce_audio_shift audio_shift = {
-		AUD_COMMON_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_audio_mask audio_mask = {
-		AUD_COMMON_MASK_SH_LIST(_MASK)
-};
-
-#define clk_src_regs(id)\
-[id] = {\
-	CS_COMMON_REG_LIST_DCE_100_110(id),\
-}
-
-static const struct dce110_clk_src_regs clk_src_regs[] = {
-	clk_src_regs(0),
-	clk_src_regs(1),
-	clk_src_regs(2)
-};
-
-static const struct dce110_clk_src_shift cs_shift = {
-		CS_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
-};
-
-static const struct dce110_clk_src_mask cs_mask = {
-		CS_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
-};
-
-static const struct dce_dmcu_registers dmcu_regs = {
-		DMCU_DCE110_COMMON_REG_LIST()
-};
-
-static const struct dce_dmcu_shift dmcu_shift = {
-		DMCU_MASK_SH_LIST_DCE110(__SHIFT)
-};
-
-static const struct dce_dmcu_mask dmcu_mask = {
-		DMCU_MASK_SH_LIST_DCE110(_MASK)
-};
-
-static const struct dce_abm_registers abm_regs = {
-		ABM_DCE110_COMMON_REG_LIST()
-};
-
-static const struct dce_abm_shift abm_shift = {
-		ABM_MASK_SH_LIST_DCE110(__SHIFT)
-};
-
-static const struct dce_abm_mask abm_mask = {
-		ABM_MASK_SH_LIST_DCE110(_MASK)
-};
-
-#define DCFE_MEM_PWR_CTRL_REG_BASE 0x1b03
-
-static const struct bios_registers bios_regs = {
-	.BIOS_SCRATCH_3 = mmBIOS_SCRATCH_3,
-	.BIOS_SCRATCH_6 = mmBIOS_SCRATCH_6
-};
-
-static const struct resource_caps res_cap = {
-	.num_timing_generator = 6,
-	.num_audio = 6,
-	.num_stream_encoder = 6,
-	.num_pll = 3,
-	.num_ddc = 6,
-};
-
-static const struct dc_plane_cap plane_cap = {
-	.type = DC_PLANE_TYPE_DCE_RGB,
-
-	.pixel_format_support = {
-			.argb8888 = true,
-			.nv12 = false,
-			.fp16 = true
-	},
-
-	.max_upscale_factor = {
-			.argb8888 = 16000,
-			.nv12 = 1,
-			.fp16 = 1
-	},
-
-	.max_downscale_factor = {
-			.argb8888 = 250,
-			.nv12 = 1,
-			.fp16 = 1
-	}
-};
-
-static const struct dc_debug_options debug_defaults = {
-		.enable_legacy_fast_update = true,
-};
-
-#define CTX  ctx
-#define REG(reg) mm ## reg
-
-#ifndef mmCC_DC_HDMI_STRAPS
-#define mmCC_DC_HDMI_STRAPS 0x1918
-#define CC_DC_HDMI_STRAPS__HDMI_DISABLE_MASK 0x40
-#define CC_DC_HDMI_STRAPS__HDMI_DISABLE__SHIFT 0x6
-#define CC_DC_HDMI_STRAPS__AUDIO_STREAM_NUMBER_MASK 0x700
-#define CC_DC_HDMI_STRAPS__AUDIO_STREAM_NUMBER__SHIFT 0x8
-#endif
-
-static int map_transmitter_id_to_phy_instance(
-	enum transmitter transmitter)
-{
-	switch (transmitter) {
-	case TRANSMITTER_UNIPHY_A:
-		return 0;
-	case TRANSMITTER_UNIPHY_B:
-		return 1;
-	case TRANSMITTER_UNIPHY_C:
-		return 2;
-	case TRANSMITTER_UNIPHY_D:
-		return 3;
-	case TRANSMITTER_UNIPHY_E:
-		return 4;
-	case TRANSMITTER_UNIPHY_F:
-		return 5;
-	case TRANSMITTER_UNIPHY_G:
-		return 6;
-	default:
-		ASSERT(0);
-		return 0;
-	}
-}
-
-static void read_dce_straps(
-	struct dc_context *ctx,
-	struct resource_straps *straps)
-{
-	REG_GET_2(CC_DC_HDMI_STRAPS,
-			HDMI_DISABLE, &straps->hdmi_disable,
-			AUDIO_STREAM_NUMBER, &straps->audio_stream_number);
-
-	REG_GET(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO, &straps->dc_pinstraps_audio);
-}
-
-static struct audio *create_audio(
-		struct dc_context *ctx, unsigned int inst)
-{
-	return dce_audio_create(ctx, inst,
-			&audio_regs[inst], &audio_shift, &audio_mask);
-}
-
-static struct timing_generator *dce100_timing_generator_create(
-		struct dc_context *ctx,
-		uint32_t instance,
-		const struct dce110_timing_generator_offsets *offsets)
-{
-	struct dce110_timing_generator *tg110 =
-		kzalloc(sizeof(struct dce110_timing_generator), GFP_KERNEL);
-
-	if (!tg110)
-		return NULL;
-
-	dce110_timing_generator_construct(tg110, ctx, instance, offsets);
-	return &tg110->base;
-}
-
-static struct stream_encoder *dce100_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dce110_stream_encoder *enc110 =
-		kzalloc(sizeof(struct dce110_stream_encoder), GFP_KERNEL);
-
-	if (!enc110)
-		return NULL;
-
-	dce110_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id,
-					&stream_enc_regs[eng_id], &se_shift, &se_mask);
-	return &enc110->base;
-}
-
-#define SRII(reg_name, block, id)\
-	.reg_name[id] = mm ## block ## id ## _ ## reg_name
-
-static const struct dce_hwseq_registers hwseq_reg = {
-		HWSEQ_DCE10_REG_LIST()
-};
-
-static const struct dce_hwseq_shift hwseq_shift = {
-		HWSEQ_DCE10_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_hwseq_mask hwseq_mask = {
-		HWSEQ_DCE10_MASK_SH_LIST(_MASK)
-};
-
-static struct dce_hwseq *dce100_hwseq_create(
-	struct dc_context *ctx)
-{
-	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
-
-	if (hws) {
-		hws->ctx = ctx;
-		hws->regs = &hwseq_reg;
-		hws->shifts = &hwseq_shift;
-		hws->masks = &hwseq_mask;
-	}
-	return hws;
-}
-
-static const struct resource_create_funcs res_create_funcs = {
-	.read_dce_straps = read_dce_straps,
-	.create_audio = create_audio,
-	.create_stream_encoder = dce100_stream_encoder_create,
-	.create_hwseq = dce100_hwseq_create,
-};
-
-#define mi_inst_regs(id) { \
-	MI_DCE8_REG_LIST(id), \
-	.MC_HUB_RDREQ_DMIF_LIMIT = mmMC_HUB_RDREQ_DMIF_LIMIT \
-}
-static const struct dce_mem_input_registers mi_regs[] = {
-		mi_inst_regs(0),
-		mi_inst_regs(1),
-		mi_inst_regs(2),
-		mi_inst_regs(3),
-		mi_inst_regs(4),
-		mi_inst_regs(5),
-};
-
-static const struct dce_mem_input_shift mi_shifts = {
-		MI_DCE8_MASK_SH_LIST(__SHIFT),
-		.ENABLE = MC_HUB_RDREQ_DMIF_LIMIT__ENABLE__SHIFT
-};
-
-static const struct dce_mem_input_mask mi_masks = {
-		MI_DCE8_MASK_SH_LIST(_MASK),
-		.ENABLE = MC_HUB_RDREQ_DMIF_LIMIT__ENABLE_MASK
-};
-
-static const struct dce110_aux_registers_shift aux_shift = {
-	DCE10_AUX_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce110_aux_registers_mask aux_mask = {
-	DCE10_AUX_MASK_SH_LIST(_MASK)
-};
-
-static struct mem_input *dce100_mem_input_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_mem_input *dce_mi = kzalloc(sizeof(struct dce_mem_input),
-					       GFP_KERNEL);
-
-	if (!dce_mi) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dce_mem_input_construct(dce_mi, ctx, inst, &mi_regs[inst], &mi_shifts, &mi_masks);
-	dce_mi->wa.single_head_rdreq_dmif_limit = 2;
-	return &dce_mi->base;
-}
-
-static void dce100_transform_destroy(struct transform **xfm)
-{
-	kfree(TO_DCE_TRANSFORM(*xfm));
-	*xfm = NULL;
-}
-
-static struct transform *dce100_transform_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_transform *transform =
-		kzalloc(sizeof(struct dce_transform), GFP_KERNEL);
-
-	if (!transform)
-		return NULL;
-
-	dce_transform_construct(transform, ctx, inst,
-				&xfm_regs[inst], &xfm_shift, &xfm_mask);
-	return &transform->base;
-}
-
-static struct input_pixel_processor *dce100_ipp_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dce_ipp *ipp = kzalloc(sizeof(struct dce_ipp), GFP_KERNEL);
-
-	if (!ipp) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dce_ipp_construct(ipp, ctx, inst,
-			&ipp_regs[inst], &ipp_shift, &ipp_mask);
-	return &ipp->base;
-}
-
-static const struct encoder_feature_support link_enc_feature = {
-		.max_hdmi_deep_color = COLOR_DEPTH_121212,
-		.max_hdmi_pixel_clock = 300000,
-		.flags.bits.IS_HBR2_CAPABLE = true,
-		.flags.bits.IS_TPS3_CAPABLE = true
-};
-
-static struct link_encoder *dce100_link_encoder_create(
-	struct dc_context *ctx,
-	const struct encoder_init_data *enc_init_data)
-{
-	struct dce110_link_encoder *enc110 =
-		kzalloc(sizeof(struct dce110_link_encoder), GFP_KERNEL);
-	int link_regs_id;
-
-	if (!enc110)
-		return NULL;
-
-	link_regs_id =
-		map_transmitter_id_to_phy_instance(enc_init_data->transmitter);
-
-	dce110_link_encoder_construct(enc110,
-				      enc_init_data,
-				      &link_enc_feature,
-				      &link_enc_regs[link_regs_id],
-				      &link_enc_aux_regs[enc_init_data->channel - 1],
-				      &link_enc_hpd_regs[enc_init_data->hpd_source]);
-	return &enc110->base;
-}
-
-static struct panel_cntl *dce100_panel_cntl_create(const struct panel_cntl_init_data *init_data)
-{
-	struct dce_panel_cntl *panel_cntl =
-		kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
-
-	if (!panel_cntl)
-		return NULL;
-
-	dce_panel_cntl_construct(panel_cntl,
-			init_data,
-			&panel_cntl_regs[init_data->inst],
-			&panel_cntl_shift,
-			&panel_cntl_mask);
-
-	return &panel_cntl->base;
-}
-
-static struct output_pixel_processor *dce100_opp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce110_opp *opp =
-		kzalloc(sizeof(struct dce110_opp), GFP_KERNEL);
-
-	if (!opp)
-		return NULL;
-
-	dce110_opp_construct(opp,
-			     ctx, inst, &opp_regs[inst], &opp_shift, &opp_mask);
-	return &opp->base;
-}
-
-static struct dce_aux *dce100_aux_engine_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct aux_engine_dce110 *aux_engine =
-		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
-
-	if (!aux_engine)
-		return NULL;
-
-	dce110_aux_engine_construct(aux_engine, ctx, inst,
-				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
-				    &aux_engine_regs[inst],
-					&aux_mask,
-					&aux_shift,
-					ctx->dc->caps.extended_aux_timeout_support);
-
-	return &aux_engine->base;
-}
-#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
-
-static const struct dce_i2c_registers i2c_hw_regs[] = {
-		i2c_inst_regs(1),
-		i2c_inst_regs(2),
-		i2c_inst_regs(3),
-		i2c_inst_regs(4),
-		i2c_inst_regs(5),
-		i2c_inst_regs(6),
-};
-
-static const struct dce_i2c_shift i2c_shifts = {
-		I2C_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
-};
-
-static const struct dce_i2c_mask i2c_masks = {
-		I2C_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
-};
-
-static struct dce_i2c_hw *dce100_i2c_hw_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_i2c_hw *dce_i2c_hw =
-		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
-
-	if (!dce_i2c_hw)
-		return NULL;
-
-	dce100_i2c_hw_construct(dce_i2c_hw, ctx, inst,
-				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
-
-	return dce_i2c_hw;
-}
-static struct clock_source *dce100_clock_source_create(
-	struct dc_context *ctx,
-	struct dc_bios *bios,
-	enum clock_source_id id,
-	const struct dce110_clk_src_regs *regs,
-	bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src =
-		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dce110_clk_src_construct(clk_src, ctx, bios, id,
-			regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-
-	kfree(clk_src);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
-static void dce100_clock_source_destroy(struct clock_source **clk_src)
-{
-	kfree(TO_DCE110_CLK_SRC(*clk_src));
-	*clk_src = NULL;
-}
-
-static void dce100_resource_destruct(struct dce110_resource_pool *pool)
-{
-	unsigned int i;
-
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		if (pool->base.opps[i] != NULL)
-			dce110_opp_destroy(&pool->base.opps[i]);
-
-		if (pool->base.transforms[i] != NULL)
-			dce100_transform_destroy(&pool->base.transforms[i]);
-
-		if (pool->base.ipps[i] != NULL)
-			dce_ipp_destroy(&pool->base.ipps[i]);
-
-		if (pool->base.mis[i] != NULL) {
-			kfree(TO_DCE_MEM_INPUT(pool->base.mis[i]));
-			pool->base.mis[i] = NULL;
-		}
-
-		if (pool->base.timing_generators[i] != NULL)	{
-			kfree(DCE110TG_FROM_TG(pool->base.timing_generators[i]));
-			pool->base.timing_generators[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		if (pool->base.engines[i] != NULL)
-			dce110_engine_destroy(&pool->base.engines[i]);
-		if (pool->base.hw_i2cs[i] != NULL) {
-			kfree(pool->base.hw_i2cs[i]);
-			pool->base.hw_i2cs[i] = NULL;
-		}
-		if (pool->base.sw_i2cs[i] != NULL) {
-			kfree(pool->base.sw_i2cs[i]);
-			pool->base.sw_i2cs[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.stream_enc_count; i++) {
-		if (pool->base.stream_enc[i] != NULL)
-			kfree(DCE110STRENC_FROM_STRENC(pool->base.stream_enc[i]));
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] != NULL)
-			dce100_clock_source_destroy(&pool->base.clock_sources[i]);
-	}
-
-	if (pool->base.dp_clock_source != NULL)
-		dce100_clock_source_destroy(&pool->base.dp_clock_source);
-
-	for (i = 0; i < pool->base.audio_count; i++)	{
-		if (pool->base.audios[i] != NULL)
-			dce_aud_destroy(&pool->base.audios[i]);
-	}
-
-	if (pool->base.abm != NULL)
-				dce_abm_destroy(&pool->base.abm);
-
-	if (pool->base.dmcu != NULL)
-			dce_dmcu_destroy(&pool->base.dmcu);
-
-	if (pool->base.irqs != NULL)
-		dal_irq_service_destroy(&pool->base.irqs);
-}
-
-static enum dc_status build_mapped_resource(
-		const struct dc  *dc,
-		struct dc_state *context,
-		struct dc_stream_state *stream)
-{
-	struct pipe_ctx *pipe_ctx = resource_get_otg_master_for_stream(&context->res_ctx, stream);
-
-	if (!pipe_ctx)
-		return DC_ERROR_UNEXPECTED;
-
-	dce110_resource_build_pipe_hw_param(pipe_ctx);
-
-	resource_build_info_frame(pipe_ctx);
-
-	return DC_OK;
-}
-
-static bool dce100_validate_bandwidth(
-	struct dc  *dc,
-	struct dc_state *context,
-	bool fast_validate)
-{
-	int i;
-	bool at_least_one_pipe = false;
-
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		if (context->res_ctx.pipe_ctx[i].stream)
-			at_least_one_pipe = true;
-	}
-
-	if (at_least_one_pipe) {
-		/* TODO implement when needed but for now hardcode max value*/
-		context->bw_ctx.bw.dce.dispclk_khz = 681000;
-		context->bw_ctx.bw.dce.yclk_khz = 250000 * MEMORY_TYPE_MULTIPLIER_CZ;
-	} else {
-		context->bw_ctx.bw.dce.dispclk_khz = 0;
-		context->bw_ctx.bw.dce.yclk_khz = 0;
-	}
-
-	return true;
-}
-
-static bool dce100_validate_surface_sets(
-		struct dc_state *context)
-{
-	int i;
-
-	for (i = 0; i < context->stream_count; i++) {
-		if (context->stream_status[i].plane_count == 0)
-			continue;
-
-		if (context->stream_status[i].plane_count > 1)
-			return false;
-
-		if (context->stream_status[i].plane_states[0]->format
-				>= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN)
-			return false;
-	}
-
-	return true;
-}
-
-static enum dc_status dce100_validate_global(
-		struct dc  *dc,
-		struct dc_state *context)
-{
-	if (!dce100_validate_surface_sets(context))
-		return DC_FAIL_SURFACE_VALIDATE;
-
-	return DC_OK;
-}
-
-enum dc_status dce100_add_stream_to_ctx(
-		struct dc *dc,
-		struct dc_state *new_ctx,
-		struct dc_stream_state *dc_stream)
-{
-	enum dc_status result = DC_ERROR_UNEXPECTED;
-
-	result = resource_map_pool_resources(dc, new_ctx, dc_stream);
-
-	if (result == DC_OK)
-		result = resource_map_clock_resources(dc, new_ctx, dc_stream);
-
-	if (result == DC_OK)
-		result = build_mapped_resource(dc, new_ctx, dc_stream);
-
-	return result;
-}
-
-static void dce100_destroy_resource_pool(struct resource_pool **pool)
-{
-	struct dce110_resource_pool *dce110_pool = TO_DCE110_RES_POOL(*pool);
-
-	dce100_resource_destruct(dce110_pool);
-	kfree(dce110_pool);
-	*pool = NULL;
-}
-
-enum dc_status dce100_validate_plane(const struct dc_plane_state *plane_state, struct dc_caps *caps)
-{
-
-	if (plane_state->format < SURFACE_PIXEL_FORMAT_VIDEO_BEGIN)
-		return DC_OK;
-
-	return DC_FAIL_SURFACE_VALIDATE;
-}
-
-struct stream_encoder *dce100_find_first_free_match_stream_enc_for_link(
-		struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		struct dc_stream_state *stream)
-{
-	int i;
-	int j = -1;
-	struct dc_link *link = stream->link;
-
-	for (i = 0; i < pool->stream_enc_count; i++) {
-		if (!res_ctx->is_stream_enc_acquired[i] &&
-				pool->stream_enc[i]) {
-			/* Store first available for MST second display
-			 * in daisy chain use case
-			 */
-			j = i;
-			if (pool->stream_enc[i]->id ==
-					link->link_enc->preferred_engine)
-				return pool->stream_enc[i];
-		}
-	}
-
-	/*
-	 * below can happen in cases when stream encoder is acquired:
-	 * 1) for second MST display in chain, so preferred engine already
-	 * acquired;
-	 * 2) for another link, which preferred engine already acquired by any
-	 * MST configuration.
-	 *
-	 * If signal is of DP type and preferred engine not found, return last available
-	 *
-	 * TODO - This is just a patch up and a generic solution is
-	 * required for non DP connectors.
-	 */
-
-	if (j >= 0 && link->connector_signal == SIGNAL_TYPE_DISPLAY_PORT)
-		return pool->stream_enc[j];
-
-	return NULL;
-}
-
-static const struct resource_funcs dce100_res_pool_funcs = {
-	.destroy = dce100_destroy_resource_pool,
-	.link_enc_create = dce100_link_encoder_create,
-	.panel_cntl_create = dce100_panel_cntl_create,
-	.validate_bandwidth = dce100_validate_bandwidth,
-	.validate_plane = dce100_validate_plane,
-	.add_stream_to_ctx = dce100_add_stream_to_ctx,
-	.validate_global = dce100_validate_global,
-	.find_first_free_match_stream_enc_for_link = dce100_find_first_free_match_stream_enc_for_link
-};
-
-static bool dce100_resource_construct(
-	uint8_t num_virtual_links,
-	struct dc  *dc,
-	struct dce110_resource_pool *pool)
-{
-	unsigned int i;
-	struct dc_context *ctx = dc->ctx;
-	struct dc_bios *bp;
-
-	ctx->dc_bios->regs = &bios_regs;
-
-	pool->base.res_cap = &res_cap;
-	pool->base.funcs = &dce100_res_pool_funcs;
-	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
-
-	bp = ctx->dc_bios;
-
-	if (bp->fw_info_valid && bp->fw_info.external_clock_source_frequency_for_dp != 0) {
-		pool->base.dp_clock_source =
-				dce100_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_EXTERNAL, NULL, true);
-
-		pool->base.clock_sources[0] =
-				dce100_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0, &clk_src_regs[0], false);
-		pool->base.clock_sources[1] =
-				dce100_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false);
-		pool->base.clock_sources[2] =
-				dce100_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[2], false);
-		pool->base.clk_src_count = 3;
-
-	} else {
-		pool->base.dp_clock_source =
-				dce100_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0, &clk_src_regs[0], true);
-
-		pool->base.clock_sources[0] =
-				dce100_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false);
-		pool->base.clock_sources[1] =
-				dce100_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[2], false);
-		pool->base.clk_src_count = 2;
-	}
-
-	if (pool->base.dp_clock_source == NULL) {
-		dm_error("DC: failed to create dp clock source!\n");
-		BREAK_TO_DEBUGGER();
-		goto res_create_fail;
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			BREAK_TO_DEBUGGER();
-			goto res_create_fail;
-		}
-	}
-
-	pool->base.dmcu = dce_dmcu_create(ctx,
-			&dmcu_regs,
-			&dmcu_shift,
-			&dmcu_mask);
-	if (pool->base.dmcu == NULL) {
-		dm_error("DC: failed to create dmcu!\n");
-		BREAK_TO_DEBUGGER();
-		goto res_create_fail;
-	}
-
-	pool->base.abm = dce_abm_create(ctx,
-				&abm_regs,
-				&abm_shift,
-				&abm_mask);
-	if (pool->base.abm == NULL) {
-		dm_error("DC: failed to create abm!\n");
-		BREAK_TO_DEBUGGER();
-		goto res_create_fail;
-	}
-
-	{
-		struct irq_service_init_data init_data;
-		init_data.ctx = dc->ctx;
-		pool->base.irqs = dal_irq_service_dce110_create(&init_data);
-		if (!pool->base.irqs)
-			goto res_create_fail;
-	}
-
-	/*************************************************
-	*  Resource + asic cap harcoding                *
-	*************************************************/
-	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
-	pool->base.pipe_count = res_cap.num_timing_generator;
-	pool->base.timing_generator_count = pool->base.res_cap->num_timing_generator;
-	dc->caps.max_downscale_ratio = 200;
-	dc->caps.i2c_speed_in_khz = 40;
-	dc->caps.i2c_speed_in_khz = 40;
-	dc->caps.max_cursor_size = 128;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.dual_link_dvi = true;
-	dc->caps.disable_dp_clk_share = true;
-	dc->caps.extended_aux_timeout_support = false;
-	dc->debug = debug_defaults;
-
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		pool->base.timing_generators[i] =
-			dce100_timing_generator_create(
-				ctx,
-				i,
-				&dce100_tg_offsets[i]);
-		if (pool->base.timing_generators[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create tg!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.mis[i] = dce100_mem_input_create(ctx, i);
-		if (pool->base.mis[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create memory input!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.ipps[i] = dce100_ipp_create(ctx, i);
-		if (pool->base.ipps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create input pixel processor!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.transforms[i] = dce100_transform_create(ctx, i);
-		if (pool->base.transforms[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create transform!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.opps[i] = dce100_opp_create(ctx, i);
-		if (pool->base.opps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create output pixel processor!\n");
-			goto res_create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		pool->base.engines[i] = dce100_aux_engine_create(ctx, i);
-		if (pool->base.engines[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create aux engine!!\n");
-			goto res_create_fail;
-		}
-		pool->base.hw_i2cs[i] = dce100_i2c_hw_create(ctx, i);
-		if (pool->base.hw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create i2c engine!!\n");
-			goto res_create_fail;
-		}
-		pool->base.sw_i2cs[i] = NULL;
-	}
-
-	dc->caps.max_planes =  pool->base.pipe_count;
-
-	for (i = 0; i < dc->caps.max_planes; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	if (!resource_construct(num_virtual_links, dc, &pool->base,
-			&res_create_funcs))
-		goto res_create_fail;
-
-	/* Create hardware sequencer */
-	dce100_hw_sequencer_construct(dc);
-	return true;
-
-res_create_fail:
-	dce100_resource_destruct(pool);
-
-	return false;
-}
-
-struct resource_pool *dce100_create_resource_pool(
-	uint8_t num_virtual_links,
-	struct dc  *dc)
-{
-	struct dce110_resource_pool *pool =
-		kzalloc(sizeof(struct dce110_resource_pool), GFP_KERNEL);
-
-	if (!pool)
-		return NULL;
-
-	if (dce100_resource_construct(num_virtual_links, dc, pool))
-		return &pool->base;
-
-	kfree(pool);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
diff --git a/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.h b/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.h
deleted file mode 100644
index fecab7c560f5..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright 2017 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- *
- */
-/*
- * dce100_resource.h
- *
- *  Created on: 2016-01-20
- *      Author: qyang
- */
-
-#ifndef DCE100_RESOURCE_H_
-#define DCE100_RESOURCE_H_
-
-struct dc;
-struct resource_pool;
-struct dc_validation_set;
-
-struct resource_pool *dce100_create_resource_pool(
-	uint8_t num_virtual_links,
-	struct dc *dc);
-
-enum dc_status dce100_validate_plane(const struct dc_plane_state *plane_state, struct dc_caps *caps);
-
-enum dc_status dce100_add_stream_to_ctx(
-		struct dc *dc,
-		struct dc_state *new_ctx,
-		struct dc_stream_state *dc_stream);
-
-struct stream_encoder *dce100_find_first_free_match_stream_enc_for_link(
-		struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		struct dc_stream_state *stream);
-
-#endif /* DCE100_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/Makefile b/drivers/gpu/drm/amd/display/dc/dce110/Makefile
index 695a50ed5ad2..f0777d61c2cb 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dce110/Makefile
@@ -26,8 +26,8 @@
 CFLAGS_$(AMDDALPATH)/dc/dce110/dce110_resource.o = $(call cc-disable-warning, override-init)
 
 DCE110 = dce110_timing_generator.o \
-dce110_compressor.o dce110_resource.o \
-dce110_opp_regamma_v.o dce110_opp_csc_v.o dce110_timing_generator_v.o \
+dce110_compressor.o dce110_opp_regamma_v.o \
+dce110_opp_csc_v.o dce110_timing_generator_v.o \
 dce110_mem_input_v.o dce110_opp_v.o dce110_transform_v.o
 
 AMD_DAL_DCE110 = $(addprefix $(AMDDALPATH)/dc/dce110/,$(DCE110))
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c
deleted file mode 100644
index fe518fd27b08..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c
+++ /dev/null
@@ -1,1551 +0,0 @@
-/*
- * Copyright 2012-15 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#include "dm_services.h"
-
-#include "link_encoder.h"
-#include "stream_encoder.h"
-
-#include "resource.h"
-#include "dce110/dce110_resource.h"
-#include "include/irq_service_interface.h"
-#include "dce/dce_audio.h"
-#include "dce110/dce110_timing_generator.h"
-#include "irq/dce110/irq_service_dce110.h"
-#include "dce110/dce110_timing_generator_v.h"
-#include "dce/dce_link_encoder.h"
-#include "dce/dce_stream_encoder.h"
-#include "dce/dce_mem_input.h"
-#include "dce110/dce110_mem_input_v.h"
-#include "dce/dce_ipp.h"
-#include "dce/dce_transform.h"
-#include "dce110/dce110_transform_v.h"
-#include "dce/dce_opp.h"
-#include "dce110/dce110_opp_v.h"
-#include "dce/dce_clock_source.h"
-#include "dce/dce_hwseq.h"
-#include "dce110/dce110_hwseq.h"
-#include "dce/dce_aux.h"
-#include "dce/dce_abm.h"
-#include "dce/dce_dmcu.h"
-#include "dce/dce_i2c.h"
-#include "dce/dce_panel_cntl.h"
-
-#define DC_LOGGER \
-		dc->ctx->logger
-
-#include "dce110/dce110_compressor.h"
-
-#include "reg_helper.h"
-
-#include "dce/dce_11_0_d.h"
-#include "dce/dce_11_0_sh_mask.h"
-
-#ifndef mmMC_HUB_RDREQ_DMIF_LIMIT
-#include "gmc/gmc_8_2_d.h"
-#include "gmc/gmc_8_2_sh_mask.h"
-#endif
-
-#ifndef mmDP_DPHY_INTERNAL_CTRL
-	#define mmDP_DPHY_INTERNAL_CTRL 0x4aa7
-	#define mmDP0_DP_DPHY_INTERNAL_CTRL 0x4aa7
-	#define mmDP1_DP_DPHY_INTERNAL_CTRL 0x4ba7
-	#define mmDP2_DP_DPHY_INTERNAL_CTRL 0x4ca7
-	#define mmDP3_DP_DPHY_INTERNAL_CTRL 0x4da7
-	#define mmDP4_DP_DPHY_INTERNAL_CTRL 0x4ea7
-	#define mmDP5_DP_DPHY_INTERNAL_CTRL 0x4fa7
-	#define mmDP6_DP_DPHY_INTERNAL_CTRL 0x54a7
-	#define mmDP7_DP_DPHY_INTERNAL_CTRL 0x56a7
-	#define mmDP8_DP_DPHY_INTERNAL_CTRL 0x57a7
-#endif
-
-#ifndef mmBIOS_SCRATCH_2
-	#define mmBIOS_SCRATCH_2 0x05CB
-	#define mmBIOS_SCRATCH_3 0x05CC
-	#define mmBIOS_SCRATCH_6 0x05CF
-#endif
-
-#ifndef mmDP_DPHY_BS_SR_SWAP_CNTL
-	#define mmDP_DPHY_BS_SR_SWAP_CNTL                       0x4ADC
-	#define mmDP0_DP_DPHY_BS_SR_SWAP_CNTL                   0x4ADC
-	#define mmDP1_DP_DPHY_BS_SR_SWAP_CNTL                   0x4BDC
-	#define mmDP2_DP_DPHY_BS_SR_SWAP_CNTL                   0x4CDC
-	#define mmDP3_DP_DPHY_BS_SR_SWAP_CNTL                   0x4DDC
-	#define mmDP4_DP_DPHY_BS_SR_SWAP_CNTL                   0x4EDC
-	#define mmDP5_DP_DPHY_BS_SR_SWAP_CNTL                   0x4FDC
-	#define mmDP6_DP_DPHY_BS_SR_SWAP_CNTL                   0x54DC
-#endif
-
-#ifndef mmDP_DPHY_FAST_TRAINING
-	#define mmDP_DPHY_FAST_TRAINING                         0x4ABC
-	#define mmDP0_DP_DPHY_FAST_TRAINING                     0x4ABC
-	#define mmDP1_DP_DPHY_FAST_TRAINING                     0x4BBC
-	#define mmDP2_DP_DPHY_FAST_TRAINING                     0x4CBC
-	#define mmDP3_DP_DPHY_FAST_TRAINING                     0x4DBC
-	#define mmDP4_DP_DPHY_FAST_TRAINING                     0x4EBC
-	#define mmDP5_DP_DPHY_FAST_TRAINING                     0x4FBC
-	#define mmDP6_DP_DPHY_FAST_TRAINING                     0x54BC
-#endif
-
-#ifndef DPHY_RX_FAST_TRAINING_CAPABLE
-	#define DPHY_RX_FAST_TRAINING_CAPABLE 0x1
-#endif
-
-static const struct dce110_timing_generator_offsets dce110_tg_offsets[] = {
-	{
-		.crtc = (mmCRTC0_CRTC_CONTROL - mmCRTC_CONTROL),
-		.dcp =  (mmDCP0_GRPH_CONTROL - mmGRPH_CONTROL),
-	},
-	{
-		.crtc = (mmCRTC1_CRTC_CONTROL - mmCRTC_CONTROL),
-		.dcp = (mmDCP1_GRPH_CONTROL - mmGRPH_CONTROL),
-	},
-	{
-		.crtc = (mmCRTC2_CRTC_CONTROL - mmCRTC_CONTROL),
-		.dcp = (mmDCP2_GRPH_CONTROL - mmGRPH_CONTROL),
-	},
-	{
-		.crtc = (mmCRTC3_CRTC_CONTROL - mmCRTC_CONTROL),
-		.dcp =  (mmDCP3_GRPH_CONTROL - mmGRPH_CONTROL),
-	},
-	{
-		.crtc = (mmCRTC4_CRTC_CONTROL - mmCRTC_CONTROL),
-		.dcp = (mmDCP4_GRPH_CONTROL - mmGRPH_CONTROL),
-	},
-	{
-		.crtc = (mmCRTC5_CRTC_CONTROL - mmCRTC_CONTROL),
-		.dcp = (mmDCP5_GRPH_CONTROL - mmGRPH_CONTROL),
-	}
-};
-
-/* set register offset */
-#define SR(reg_name)\
-	.reg_name = mm ## reg_name
-
-/* set register offset with instance */
-#define SRI(reg_name, block, id)\
-	.reg_name = mm ## block ## id ## _ ## reg_name
-
-static const struct dce_dmcu_registers dmcu_regs = {
-		DMCU_DCE110_COMMON_REG_LIST()
-};
-
-static const struct dce_dmcu_shift dmcu_shift = {
-		DMCU_MASK_SH_LIST_DCE110(__SHIFT)
-};
-
-static const struct dce_dmcu_mask dmcu_mask = {
-		DMCU_MASK_SH_LIST_DCE110(_MASK)
-};
-
-static const struct dce_abm_registers abm_regs = {
-		ABM_DCE110_COMMON_REG_LIST()
-};
-
-static const struct dce_abm_shift abm_shift = {
-		ABM_MASK_SH_LIST_DCE110(__SHIFT)
-};
-
-static const struct dce_abm_mask abm_mask = {
-		ABM_MASK_SH_LIST_DCE110(_MASK)
-};
-
-#define ipp_regs(id)\
-[id] = {\
-		IPP_DCE110_REG_LIST_DCE_BASE(id)\
-}
-
-static const struct dce_ipp_registers ipp_regs[] = {
-		ipp_regs(0),
-		ipp_regs(1),
-		ipp_regs(2)
-};
-
-static const struct dce_ipp_shift ipp_shift = {
-		IPP_DCE100_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
-};
-
-static const struct dce_ipp_mask ipp_mask = {
-		IPP_DCE100_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
-};
-
-#define transform_regs(id)\
-[id] = {\
-		XFM_COMMON_REG_LIST_DCE110(id)\
-}
-
-static const struct dce_transform_registers xfm_regs[] = {
-		transform_regs(0),
-		transform_regs(1),
-		transform_regs(2)
-};
-
-static const struct dce_transform_shift xfm_shift = {
-		XFM_COMMON_MASK_SH_LIST_DCE110(__SHIFT)
-};
-
-static const struct dce_transform_mask xfm_mask = {
-		XFM_COMMON_MASK_SH_LIST_DCE110(_MASK)
-};
-
-#define aux_regs(id)\
-[id] = {\
-	AUX_REG_LIST(id)\
-}
-
-static const struct dce110_link_enc_aux_registers link_enc_aux_regs[] = {
-		aux_regs(0),
-		aux_regs(1),
-		aux_regs(2),
-		aux_regs(3),
-		aux_regs(4),
-		aux_regs(5)
-};
-
-#define hpd_regs(id)\
-[id] = {\
-	HPD_REG_LIST(id)\
-}
-
-static const struct dce110_link_enc_hpd_registers link_enc_hpd_regs[] = {
-		hpd_regs(0),
-		hpd_regs(1),
-		hpd_regs(2),
-		hpd_regs(3),
-		hpd_regs(4),
-		hpd_regs(5)
-};
-
-
-#define link_regs(id)\
-[id] = {\
-	LE_DCE110_REG_LIST(id)\
-}
-
-static const struct dce110_link_enc_registers link_enc_regs[] = {
-	link_regs(0),
-	link_regs(1),
-	link_regs(2),
-	link_regs(3),
-	link_regs(4),
-	link_regs(5),
-	link_regs(6),
-};
-
-#define stream_enc_regs(id)\
-[id] = {\
-	SE_COMMON_REG_LIST(id),\
-	.TMDS_CNTL = 0,\
-}
-
-static const struct dce110_stream_enc_registers stream_enc_regs[] = {
-	stream_enc_regs(0),
-	stream_enc_regs(1),
-	stream_enc_regs(2)
-};
-
-static const struct dce_stream_encoder_shift se_shift = {
-		SE_COMMON_MASK_SH_LIST_DCE110(__SHIFT)
-};
-
-static const struct dce_stream_encoder_mask se_mask = {
-		SE_COMMON_MASK_SH_LIST_DCE110(_MASK)
-};
-
-static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
-	{ DCE_PANEL_CNTL_REG_LIST() }
-};
-
-static const struct dce_panel_cntl_shift panel_cntl_shift = {
-	DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_panel_cntl_mask panel_cntl_mask = {
-	DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
-};
-
-static const struct dce110_aux_registers_shift aux_shift = {
-	DCE_AUX_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce110_aux_registers_mask aux_mask = {
-	DCE_AUX_MASK_SH_LIST(_MASK)
-};
-
-#define opp_regs(id)\
-[id] = {\
-	OPP_DCE_110_REG_LIST(id),\
-}
-
-static const struct dce_opp_registers opp_regs[] = {
-	opp_regs(0),
-	opp_regs(1),
-	opp_regs(2),
-	opp_regs(3),
-	opp_regs(4),
-	opp_regs(5)
-};
-
-static const struct dce_opp_shift opp_shift = {
-	OPP_COMMON_MASK_SH_LIST_DCE_110(__SHIFT)
-};
-
-static const struct dce_opp_mask opp_mask = {
-	OPP_COMMON_MASK_SH_LIST_DCE_110(_MASK)
-};
-
-#define aux_engine_regs(id)\
-[id] = {\
-	AUX_COMMON_REG_LIST(id), \
-	.AUX_RESET_MASK = 0 \
-}
-
-static const struct dce110_aux_registers aux_engine_regs[] = {
-		aux_engine_regs(0),
-		aux_engine_regs(1),
-		aux_engine_regs(2),
-		aux_engine_regs(3),
-		aux_engine_regs(4),
-		aux_engine_regs(5)
-};
-
-#define audio_regs(id)\
-[id] = {\
-	AUD_COMMON_REG_LIST(id)\
-}
-
-static const struct dce_audio_registers audio_regs[] = {
-	audio_regs(0),
-	audio_regs(1),
-	audio_regs(2),
-	audio_regs(3),
-	audio_regs(4),
-	audio_regs(5),
-	audio_regs(6),
-};
-
-static const struct dce_audio_shift audio_shift = {
-		AUD_COMMON_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_audio_mask audio_mask = {
-		AUD_COMMON_MASK_SH_LIST(_MASK)
-};
-
-/* AG TBD Needs to be reduced back to 3 pipes once dce10 hw sequencer implemented. */
-
-
-#define clk_src_regs(id)\
-[id] = {\
-	CS_COMMON_REG_LIST_DCE_100_110(id),\
-}
-
-static const struct dce110_clk_src_regs clk_src_regs[] = {
-	clk_src_regs(0),
-	clk_src_regs(1),
-	clk_src_regs(2)
-};
-
-static const struct dce110_clk_src_shift cs_shift = {
-		CS_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
-};
-
-static const struct dce110_clk_src_mask cs_mask = {
-		CS_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
-};
-
-static const struct bios_registers bios_regs = {
-	.BIOS_SCRATCH_3 = mmBIOS_SCRATCH_3,
-	.BIOS_SCRATCH_6 = mmBIOS_SCRATCH_6
-};
-
-static const struct resource_caps carrizo_resource_cap = {
-		.num_timing_generator = 3,
-		.num_video_plane = 1,
-		.num_audio = 3,
-		.num_stream_encoder = 3,
-		.num_pll = 2,
-		.num_ddc = 3,
-};
-
-static const struct resource_caps stoney_resource_cap = {
-		.num_timing_generator = 2,
-		.num_video_plane = 1,
-		.num_audio = 3,
-		.num_stream_encoder = 3,
-		.num_pll = 2,
-		.num_ddc = 3,
-};
-
-static const struct dc_plane_cap plane_cap = {
-		.type = DC_PLANE_TYPE_DCE_RGB,
-		.per_pixel_alpha = 1,
-
-		.pixel_format_support = {
-				.argb8888 = true,
-				.nv12 = false,
-				.fp16 = true
-		},
-
-		.max_upscale_factor = {
-				.argb8888 = 16000,
-				.nv12 = 1,
-				.fp16 = 1
-		},
-
-		.max_downscale_factor = {
-				.argb8888 = 250,
-				.nv12 = 1,
-				.fp16 = 1
-		},
-		64,
-		64
-};
-
-static const struct dc_debug_options debug_defaults = {
-		.enable_legacy_fast_update = true,
-};
-
-static const struct dc_plane_cap underlay_plane_cap = {
-		.type = DC_PLANE_TYPE_DCE_UNDERLAY,
-		.per_pixel_alpha = 1,
-
-		.pixel_format_support = {
-				.argb8888 = false,
-				.nv12 = true,
-				.fp16 = false
-		},
-
-		.max_upscale_factor = {
-				.argb8888 = 1,
-				.nv12 = 16000,
-				.fp16 = 1
-		},
-
-		.max_downscale_factor = {
-				.argb8888 = 1,
-				.nv12 = 250,
-				.fp16 = 1
-		},
-		64,
-		64
-};
-
-#define CTX  ctx
-#define REG(reg) mm ## reg
-
-#ifndef mmCC_DC_HDMI_STRAPS
-#define mmCC_DC_HDMI_STRAPS 0x4819
-#define CC_DC_HDMI_STRAPS__HDMI_DISABLE_MASK 0x40
-#define CC_DC_HDMI_STRAPS__HDMI_DISABLE__SHIFT 0x6
-#define CC_DC_HDMI_STRAPS__AUDIO_STREAM_NUMBER_MASK 0x700
-#define CC_DC_HDMI_STRAPS__AUDIO_STREAM_NUMBER__SHIFT 0x8
-#endif
-
-static int map_transmitter_id_to_phy_instance(
-	enum transmitter transmitter)
-{
-	switch (transmitter) {
-	case TRANSMITTER_UNIPHY_A:
-		return 0;
-	case TRANSMITTER_UNIPHY_B:
-		return 1;
-	case TRANSMITTER_UNIPHY_C:
-		return 2;
-	case TRANSMITTER_UNIPHY_D:
-		return 3;
-	case TRANSMITTER_UNIPHY_E:
-		return 4;
-	case TRANSMITTER_UNIPHY_F:
-		return 5;
-	case TRANSMITTER_UNIPHY_G:
-		return 6;
-	default:
-		ASSERT(0);
-		return 0;
-	}
-}
-
-static void read_dce_straps(
-	struct dc_context *ctx,
-	struct resource_straps *straps)
-{
-	REG_GET_2(CC_DC_HDMI_STRAPS,
-			HDMI_DISABLE, &straps->hdmi_disable,
-			AUDIO_STREAM_NUMBER, &straps->audio_stream_number);
-
-	REG_GET(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO, &straps->dc_pinstraps_audio);
-}
-
-static struct audio *create_audio(
-		struct dc_context *ctx, unsigned int inst)
-{
-	return dce_audio_create(ctx, inst,
-			&audio_regs[inst], &audio_shift, &audio_mask);
-}
-
-static struct timing_generator *dce110_timing_generator_create(
-		struct dc_context *ctx,
-		uint32_t instance,
-		const struct dce110_timing_generator_offsets *offsets)
-{
-	struct dce110_timing_generator *tg110 =
-		kzalloc(sizeof(struct dce110_timing_generator), GFP_KERNEL);
-
-	if (!tg110)
-		return NULL;
-
-	dce110_timing_generator_construct(tg110, ctx, instance, offsets);
-	return &tg110->base;
-}
-
-static struct stream_encoder *dce110_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dce110_stream_encoder *enc110 =
-		kzalloc(sizeof(struct dce110_stream_encoder), GFP_KERNEL);
-
-	if (!enc110)
-		return NULL;
-
-	dce110_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id,
-					&stream_enc_regs[eng_id],
-					&se_shift, &se_mask);
-	return &enc110->base;
-}
-
-#define SRII(reg_name, block, id)\
-	.reg_name[id] = mm ## block ## id ## _ ## reg_name
-
-static const struct dce_hwseq_registers hwseq_stoney_reg = {
-		HWSEQ_ST_REG_LIST()
-};
-
-static const struct dce_hwseq_registers hwseq_cz_reg = {
-		HWSEQ_CZ_REG_LIST()
-};
-
-static const struct dce_hwseq_shift hwseq_shift = {
-		HWSEQ_DCE11_MASK_SH_LIST(__SHIFT),
-};
-
-static const struct dce_hwseq_mask hwseq_mask = {
-		HWSEQ_DCE11_MASK_SH_LIST(_MASK),
-};
-
-static struct dce_hwseq *dce110_hwseq_create(
-	struct dc_context *ctx)
-{
-	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
-
-	if (hws) {
-		hws->ctx = ctx;
-		hws->regs = ASIC_REV_IS_STONEY(ctx->asic_id.hw_internal_rev) ?
-				&hwseq_stoney_reg : &hwseq_cz_reg;
-		hws->shifts = &hwseq_shift;
-		hws->masks = &hwseq_mask;
-		hws->wa.blnd_crtc_trigger = true;
-	}
-	return hws;
-}
-
-static const struct resource_create_funcs res_create_funcs = {
-	.read_dce_straps = read_dce_straps,
-	.create_audio = create_audio,
-	.create_stream_encoder = dce110_stream_encoder_create,
-	.create_hwseq = dce110_hwseq_create,
-};
-
-#define mi_inst_regs(id) { \
-	MI_DCE11_REG_LIST(id), \
-	.MC_HUB_RDREQ_DMIF_LIMIT = mmMC_HUB_RDREQ_DMIF_LIMIT \
-}
-static const struct dce_mem_input_registers mi_regs[] = {
-		mi_inst_regs(0),
-		mi_inst_regs(1),
-		mi_inst_regs(2),
-};
-
-static const struct dce_mem_input_shift mi_shifts = {
-		MI_DCE11_MASK_SH_LIST(__SHIFT),
-		.ENABLE = MC_HUB_RDREQ_DMIF_LIMIT__ENABLE__SHIFT
-};
-
-static const struct dce_mem_input_mask mi_masks = {
-		MI_DCE11_MASK_SH_LIST(_MASK),
-		.ENABLE = MC_HUB_RDREQ_DMIF_LIMIT__ENABLE_MASK
-};
-
-
-static struct mem_input *dce110_mem_input_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_mem_input *dce_mi = kzalloc(sizeof(struct dce_mem_input),
-					       GFP_KERNEL);
-
-	if (!dce_mi) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dce_mem_input_construct(dce_mi, ctx, inst, &mi_regs[inst], &mi_shifts, &mi_masks);
-	dce_mi->wa.single_head_rdreq_dmif_limit = 3;
-	return &dce_mi->base;
-}
-
-static void dce110_transform_destroy(struct transform **xfm)
-{
-	kfree(TO_DCE_TRANSFORM(*xfm));
-	*xfm = NULL;
-}
-
-static struct transform *dce110_transform_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_transform *transform =
-		kzalloc(sizeof(struct dce_transform), GFP_KERNEL);
-
-	if (!transform)
-		return NULL;
-
-	dce_transform_construct(transform, ctx, inst,
-				&xfm_regs[inst], &xfm_shift, &xfm_mask);
-	return &transform->base;
-}
-
-static struct input_pixel_processor *dce110_ipp_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dce_ipp *ipp = kzalloc(sizeof(struct dce_ipp), GFP_KERNEL);
-
-	if (!ipp) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dce_ipp_construct(ipp, ctx, inst,
-			&ipp_regs[inst], &ipp_shift, &ipp_mask);
-	return &ipp->base;
-}
-
-static const struct encoder_feature_support link_enc_feature = {
-		.max_hdmi_deep_color = COLOR_DEPTH_121212,
-		.max_hdmi_pixel_clock = 300000,
-		.flags.bits.IS_HBR2_CAPABLE = true,
-		.flags.bits.IS_TPS3_CAPABLE = true
-};
-
-static struct link_encoder *dce110_link_encoder_create(
-	struct dc_context *ctx,
-	const struct encoder_init_data *enc_init_data)
-{
-	struct dce110_link_encoder *enc110 =
-		kzalloc(sizeof(struct dce110_link_encoder), GFP_KERNEL);
-	int link_regs_id;
-
-	if (!enc110)
-		return NULL;
-
-	link_regs_id =
-		map_transmitter_id_to_phy_instance(enc_init_data->transmitter);
-
-	dce110_link_encoder_construct(enc110,
-				      enc_init_data,
-				      &link_enc_feature,
-				      &link_enc_regs[link_regs_id],
-				      &link_enc_aux_regs[enc_init_data->channel - 1],
-				      &link_enc_hpd_regs[enc_init_data->hpd_source]);
-	return &enc110->base;
-}
-
-static struct panel_cntl *dce110_panel_cntl_create(const struct panel_cntl_init_data *init_data)
-{
-	struct dce_panel_cntl *panel_cntl =
-		kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
-
-	if (!panel_cntl)
-		return NULL;
-
-	dce_panel_cntl_construct(panel_cntl,
-			init_data,
-			&panel_cntl_regs[init_data->inst],
-			&panel_cntl_shift,
-			&panel_cntl_mask);
-
-	return &panel_cntl->base;
-}
-
-static struct output_pixel_processor *dce110_opp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce110_opp *opp =
-		kzalloc(sizeof(struct dce110_opp), GFP_KERNEL);
-
-	if (!opp)
-		return NULL;
-
-	dce110_opp_construct(opp,
-			     ctx, inst, &opp_regs[inst], &opp_shift, &opp_mask);
-	return &opp->base;
-}
-
-static struct dce_aux *dce110_aux_engine_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct aux_engine_dce110 *aux_engine =
-		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
-
-	if (!aux_engine)
-		return NULL;
-
-	dce110_aux_engine_construct(aux_engine, ctx, inst,
-				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
-				    &aux_engine_regs[inst],
-					&aux_mask,
-					&aux_shift,
-					ctx->dc->caps.extended_aux_timeout_support);
-
-	return &aux_engine->base;
-}
-#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
-
-static const struct dce_i2c_registers i2c_hw_regs[] = {
-		i2c_inst_regs(1),
-		i2c_inst_regs(2),
-		i2c_inst_regs(3),
-		i2c_inst_regs(4),
-		i2c_inst_regs(5),
-		i2c_inst_regs(6),
-};
-
-static const struct dce_i2c_shift i2c_shifts = {
-		I2C_COMMON_MASK_SH_LIST_DCE110(__SHIFT)
-};
-
-static const struct dce_i2c_mask i2c_masks = {
-		I2C_COMMON_MASK_SH_LIST_DCE110(_MASK)
-};
-
-static struct dce_i2c_hw *dce110_i2c_hw_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_i2c_hw *dce_i2c_hw =
-		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
-
-	if (!dce_i2c_hw)
-		return NULL;
-
-	dce100_i2c_hw_construct(dce_i2c_hw, ctx, inst,
-				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
-
-	return dce_i2c_hw;
-}
-static struct clock_source *dce110_clock_source_create(
-	struct dc_context *ctx,
-	struct dc_bios *bios,
-	enum clock_source_id id,
-	const struct dce110_clk_src_regs *regs,
-	bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src =
-		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dce110_clk_src_construct(clk_src, ctx, bios, id,
-			regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-
-	kfree(clk_src);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
-static void dce110_clock_source_destroy(struct clock_source **clk_src)
-{
-	struct dce110_clk_src *dce110_clk_src;
-
-	if (!clk_src)
-		return;
-
-	dce110_clk_src = TO_DCE110_CLK_SRC(*clk_src);
-
-	kfree(dce110_clk_src->dp_ss_params);
-	kfree(dce110_clk_src->hdmi_ss_params);
-	kfree(dce110_clk_src->dvi_ss_params);
-
-	kfree(dce110_clk_src);
-	*clk_src = NULL;
-}
-
-static void dce110_resource_destruct(struct dce110_resource_pool *pool)
-{
-	unsigned int i;
-
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		if (pool->base.opps[i] != NULL)
-			dce110_opp_destroy(&pool->base.opps[i]);
-
-		if (pool->base.transforms[i] != NULL)
-			dce110_transform_destroy(&pool->base.transforms[i]);
-
-		if (pool->base.ipps[i] != NULL)
-			dce_ipp_destroy(&pool->base.ipps[i]);
-
-		if (pool->base.mis[i] != NULL) {
-			kfree(TO_DCE_MEM_INPUT(pool->base.mis[i]));
-			pool->base.mis[i] = NULL;
-		}
-
-		if (pool->base.timing_generators[i] != NULL)	{
-			kfree(DCE110TG_FROM_TG(pool->base.timing_generators[i]));
-			pool->base.timing_generators[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		if (pool->base.engines[i] != NULL)
-			dce110_engine_destroy(&pool->base.engines[i]);
-		if (pool->base.hw_i2cs[i] != NULL) {
-			kfree(pool->base.hw_i2cs[i]);
-			pool->base.hw_i2cs[i] = NULL;
-		}
-		if (pool->base.sw_i2cs[i] != NULL) {
-			kfree(pool->base.sw_i2cs[i]);
-			pool->base.sw_i2cs[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.stream_enc_count; i++) {
-		if (pool->base.stream_enc[i] != NULL)
-			kfree(DCE110STRENC_FROM_STRENC(pool->base.stream_enc[i]));
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] != NULL) {
-			dce110_clock_source_destroy(&pool->base.clock_sources[i]);
-		}
-	}
-
-	if (pool->base.dp_clock_source != NULL)
-		dce110_clock_source_destroy(&pool->base.dp_clock_source);
-
-	for (i = 0; i < pool->base.audio_count; i++)	{
-		if (pool->base.audios[i] != NULL) {
-			dce_aud_destroy(&pool->base.audios[i]);
-		}
-	}
-
-	if (pool->base.abm != NULL)
-		dce_abm_destroy(&pool->base.abm);
-
-	if (pool->base.dmcu != NULL)
-		dce_dmcu_destroy(&pool->base.dmcu);
-
-	if (pool->base.irqs != NULL) {
-		dal_irq_service_destroy(&pool->base.irqs);
-	}
-}
-
-
-static void get_pixel_clock_parameters(
-	const struct pipe_ctx *pipe_ctx,
-	struct pixel_clk_params *pixel_clk_params)
-{
-	const struct dc_stream_state *stream = pipe_ctx->stream;
-
-	/*TODO: is this halved for YCbCr 420? in that case we might want to move
-	 * the pixel clock normalization for hdmi up to here instead of doing it
-	 * in pll_adjust_pix_clk
-	 */
-	pixel_clk_params->requested_pix_clk_100hz = stream->timing.pix_clk_100hz;
-	pixel_clk_params->encoder_object_id = stream->link->link_enc->id;
-	pixel_clk_params->signal_type = pipe_ctx->stream->signal;
-	pixel_clk_params->controller_id = pipe_ctx->stream_res.tg->inst + 1;
-	/* TODO: un-hardcode*/
-	pixel_clk_params->requested_sym_clk = LINK_RATE_LOW *
-						LINK_RATE_REF_FREQ_IN_KHZ;
-	pixel_clk_params->flags.ENABLE_SS = 0;
-	pixel_clk_params->color_depth =
-		stream->timing.display_color_depth;
-	pixel_clk_params->flags.DISPLAY_BLANKED = 1;
-	pixel_clk_params->flags.SUPPORT_YCBCR420 = (stream->timing.pixel_encoding ==
-			PIXEL_ENCODING_YCBCR420);
-	pixel_clk_params->pixel_encoding = stream->timing.pixel_encoding;
-	if (stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR422) {
-		pixel_clk_params->color_depth = COLOR_DEPTH_888;
-	}
-	if (stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR420) {
-		pixel_clk_params->requested_pix_clk_100hz  = pixel_clk_params->requested_pix_clk_100hz / 2;
-	}
-	if (stream->timing.timing_3d_format == TIMING_3D_FORMAT_HW_FRAME_PACKING)
-		pixel_clk_params->requested_pix_clk_100hz *= 2;
-
-}
-
-void dce110_resource_build_pipe_hw_param(struct pipe_ctx *pipe_ctx)
-{
-	get_pixel_clock_parameters(pipe_ctx, &pipe_ctx->stream_res.pix_clk_params);
-	pipe_ctx->clock_source->funcs->get_pix_clk_dividers(
-		pipe_ctx->clock_source,
-		&pipe_ctx->stream_res.pix_clk_params,
-		&pipe_ctx->pll_settings);
-	resource_build_bit_depth_reduction_params(pipe_ctx->stream,
-			&pipe_ctx->stream->bit_depth_params);
-	pipe_ctx->stream->clamping.pixel_encoding = pipe_ctx->stream->timing.pixel_encoding;
-}
-
-static bool is_surface_pixel_format_supported(struct pipe_ctx *pipe_ctx, unsigned int underlay_idx)
-{
-	if (pipe_ctx->pipe_idx != underlay_idx)
-		return true;
-	if (!pipe_ctx->plane_state)
-		return false;
-	if (pipe_ctx->plane_state->format < SURFACE_PIXEL_FORMAT_VIDEO_BEGIN)
-		return false;
-	return true;
-}
-
-static enum dc_status build_mapped_resource(
-		const struct dc *dc,
-		struct dc_state *context,
-		struct dc_stream_state *stream)
-{
-	struct pipe_ctx *pipe_ctx = resource_get_otg_master_for_stream(&context->res_ctx, stream);
-
-	if (!pipe_ctx)
-		return DC_ERROR_UNEXPECTED;
-
-	if (!is_surface_pixel_format_supported(pipe_ctx,
-			dc->res_pool->underlay_pipe_index))
-		return DC_SURFACE_PIXEL_FORMAT_UNSUPPORTED;
-
-	dce110_resource_build_pipe_hw_param(pipe_ctx);
-
-	/* TODO: validate audio ASIC caps, encoder */
-
-	resource_build_info_frame(pipe_ctx);
-
-	return DC_OK;
-}
-
-static bool dce110_validate_bandwidth(
-	struct dc *dc,
-	struct dc_state *context,
-	bool fast_validate)
-{
-	bool result = false;
-
-	DC_LOG_BANDWIDTH_CALCS(
-		"%s: start",
-		__func__);
-
-	if (bw_calcs(
-			dc->ctx,
-			dc->bw_dceip,
-			dc->bw_vbios,
-			context->res_ctx.pipe_ctx,
-			dc->res_pool->pipe_count,
-			&context->bw_ctx.bw.dce))
-		result =  true;
-
-	if (!result)
-		DC_LOG_BANDWIDTH_VALIDATION("%s: %dx%d@%d Bandwidth validation failed!\n",
-			__func__,
-			context->streams[0]->timing.h_addressable,
-			context->streams[0]->timing.v_addressable,
-			context->streams[0]->timing.pix_clk_100hz / 10);
-
-	if (memcmp(&dc->current_state->bw_ctx.bw.dce,
-			&context->bw_ctx.bw.dce, sizeof(context->bw_ctx.bw.dce))) {
-
-		DC_LOG_BANDWIDTH_CALCS(
-			"%s: finish,\n"
-			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
-			"stutMark_b: %d stutMark_a: %d\n"
-			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
-			"stutMark_b: %d stutMark_a: %d\n"
-			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
-			"stutMark_b: %d stutMark_a: %d stutter_mode_enable: %d\n"
-			"cstate: %d pstate: %d nbpstate: %d sync: %d dispclk: %d\n"
-			"sclk: %d sclk_sleep: %d yclk: %d blackout_recovery_time_us: %d\n"
-			,
-			__func__,
-			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[0].b_mark,
-			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[0].a_mark,
-			context->bw_ctx.bw.dce.urgent_wm_ns[0].b_mark,
-			context->bw_ctx.bw.dce.urgent_wm_ns[0].a_mark,
-			context->bw_ctx.bw.dce.stutter_exit_wm_ns[0].b_mark,
-			context->bw_ctx.bw.dce.stutter_exit_wm_ns[0].a_mark,
-			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[1].b_mark,
-			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[1].a_mark,
-			context->bw_ctx.bw.dce.urgent_wm_ns[1].b_mark,
-			context->bw_ctx.bw.dce.urgent_wm_ns[1].a_mark,
-			context->bw_ctx.bw.dce.stutter_exit_wm_ns[1].b_mark,
-			context->bw_ctx.bw.dce.stutter_exit_wm_ns[1].a_mark,
-			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[2].b_mark,
-			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[2].a_mark,
-			context->bw_ctx.bw.dce.urgent_wm_ns[2].b_mark,
-			context->bw_ctx.bw.dce.urgent_wm_ns[2].a_mark,
-			context->bw_ctx.bw.dce.stutter_exit_wm_ns[2].b_mark,
-			context->bw_ctx.bw.dce.stutter_exit_wm_ns[2].a_mark,
-			context->bw_ctx.bw.dce.stutter_mode_enable,
-			context->bw_ctx.bw.dce.cpuc_state_change_enable,
-			context->bw_ctx.bw.dce.cpup_state_change_enable,
-			context->bw_ctx.bw.dce.nbp_state_change_enable,
-			context->bw_ctx.bw.dce.all_displays_in_sync,
-			context->bw_ctx.bw.dce.dispclk_khz,
-			context->bw_ctx.bw.dce.sclk_khz,
-			context->bw_ctx.bw.dce.sclk_deep_sleep_khz,
-			context->bw_ctx.bw.dce.yclk_khz,
-			context->bw_ctx.bw.dce.blackout_recovery_time_us);
-	}
-	return result;
-}
-
-static enum dc_status dce110_validate_plane(const struct dc_plane_state *plane_state,
-					    struct dc_caps *caps)
-{
-	if (((plane_state->dst_rect.width * 2) < plane_state->src_rect.width) ||
-	    ((plane_state->dst_rect.height * 2) < plane_state->src_rect.height))
-		return DC_FAIL_SURFACE_VALIDATE;
-
-	return DC_OK;
-}
-
-static bool dce110_validate_surface_sets(
-		struct dc_state *context)
-{
-	int i, j;
-
-	for (i = 0; i < context->stream_count; i++) {
-		if (context->stream_status[i].plane_count == 0)
-			continue;
-
-		if (context->stream_status[i].plane_count > 2)
-			return false;
-
-		for (j = 0; j < context->stream_status[i].plane_count; j++) {
-			struct dc_plane_state *plane =
-				context->stream_status[i].plane_states[j];
-
-			/* underlay validation */
-			if (plane->format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN) {
-
-				if ((plane->src_rect.width > 1920 ||
-					plane->src_rect.height > 1080))
-					return false;
-
-				/* we don't have the logic to support underlay
-				 * only yet so block the use case where we get
-				 * NV12 plane as top layer
-				 */
-				if (j == 0)
-					return false;
-
-				/* irrespective of plane format,
-				 * stream should be RGB encoded
-				 */
-				if (context->streams[i]->timing.pixel_encoding
-						!= PIXEL_ENCODING_RGB)
-					return false;
-
-			}
-
-		}
-	}
-
-	return true;
-}
-
-static enum dc_status dce110_validate_global(
-		struct dc *dc,
-		struct dc_state *context)
-{
-	if (!dce110_validate_surface_sets(context))
-		return DC_FAIL_SURFACE_VALIDATE;
-
-	return DC_OK;
-}
-
-static enum dc_status dce110_add_stream_to_ctx(
-		struct dc *dc,
-		struct dc_state *new_ctx,
-		struct dc_stream_state *dc_stream)
-{
-	enum dc_status result = DC_ERROR_UNEXPECTED;
-
-	result = resource_map_pool_resources(dc, new_ctx, dc_stream);
-
-	if (result == DC_OK)
-		result = resource_map_clock_resources(dc, new_ctx, dc_stream);
-
-
-	if (result == DC_OK)
-		result = build_mapped_resource(dc, new_ctx, dc_stream);
-
-	return result;
-}
-
-static struct pipe_ctx *dce110_acquire_underlay(
-		const struct dc_state *cur_ctx,
-		struct dc_state *new_ctx,
-		const struct resource_pool *pool,
-		const struct pipe_ctx *opp_head_pipe)
-{
-	struct dc_stream_state *stream = opp_head_pipe->stream;
-	struct dc *dc = stream->ctx->dc;
-	struct dce_hwseq *hws = dc->hwseq;
-	struct resource_context *res_ctx = &new_ctx->res_ctx;
-	unsigned int underlay_idx = pool->underlay_pipe_index;
-	struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[underlay_idx];
-
-	if (res_ctx->pipe_ctx[underlay_idx].stream)
-		return NULL;
-
-	pipe_ctx->stream_res.tg = pool->timing_generators[underlay_idx];
-	pipe_ctx->plane_res.mi = pool->mis[underlay_idx];
-	/*pipe_ctx->plane_res.ipp = res_ctx->pool->ipps[underlay_idx];*/
-	pipe_ctx->plane_res.xfm = pool->transforms[underlay_idx];
-	pipe_ctx->stream_res.opp = pool->opps[underlay_idx];
-	pipe_ctx->pipe_idx = underlay_idx;
-
-	pipe_ctx->stream = stream;
-
-	if (!dc->current_state->res_ctx.pipe_ctx[underlay_idx].stream) {
-		struct tg_color black_color = {0};
-		struct dc_bios *dcb = dc->ctx->dc_bios;
-
-		hws->funcs.enable_display_power_gating(
-				dc,
-				pipe_ctx->stream_res.tg->inst,
-				dcb, PIPE_GATING_CONTROL_DISABLE);
-
-		/*
-		 * This is for powering on underlay, so crtc does not
-		 * need to be enabled
-		 */
-
-		pipe_ctx->stream_res.tg->funcs->program_timing(pipe_ctx->stream_res.tg,
-				&stream->timing,
-				0,
-				0,
-				0,
-				0,
-				pipe_ctx->stream->signal,
-				false);
-
-		pipe_ctx->stream_res.tg->funcs->enable_advanced_request(
-				pipe_ctx->stream_res.tg,
-				true,
-				&stream->timing);
-
-		pipe_ctx->plane_res.mi->funcs->allocate_mem_input(pipe_ctx->plane_res.mi,
-				stream->timing.h_total,
-				stream->timing.v_total,
-				stream->timing.pix_clk_100hz / 10,
-				new_ctx->stream_count);
-
-		color_space_to_black_color(dc,
-				COLOR_SPACE_YCBCR601, &black_color);
-		pipe_ctx->stream_res.tg->funcs->set_blank_color(
-				pipe_ctx->stream_res.tg,
-				&black_color);
-	}
-
-	return pipe_ctx;
-}
-
-static void dce110_destroy_resource_pool(struct resource_pool **pool)
-{
-	struct dce110_resource_pool *dce110_pool = TO_DCE110_RES_POOL(*pool);
-
-	dce110_resource_destruct(dce110_pool);
-	kfree(dce110_pool);
-	*pool = NULL;
-}
-
-struct stream_encoder *dce110_find_first_free_match_stream_enc_for_link(
-		struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		struct dc_stream_state *stream)
-{
-	int i;
-	int j = -1;
-	struct dc_link *link = stream->link;
-
-	for (i = 0; i < pool->stream_enc_count; i++) {
-		if (!res_ctx->is_stream_enc_acquired[i] &&
-				pool->stream_enc[i]) {
-			/* Store first available for MST second display
-			 * in daisy chain use case
-			 */
-			j = i;
-			if (pool->stream_enc[i]->id ==
-					link->link_enc->preferred_engine)
-				return pool->stream_enc[i];
-		}
-	}
-
-	/*
-	 * For CZ and later, we can allow DIG FE and BE to differ for all display types
-	 */
-
-	if (j >= 0)
-		return pool->stream_enc[j];
-
-	return NULL;
-}
-
-
-static const struct resource_funcs dce110_res_pool_funcs = {
-	.destroy = dce110_destroy_resource_pool,
-	.link_enc_create = dce110_link_encoder_create,
-	.panel_cntl_create = dce110_panel_cntl_create,
-	.validate_bandwidth = dce110_validate_bandwidth,
-	.validate_plane = dce110_validate_plane,
-	.acquire_free_pipe_as_secondary_dpp_pipe = dce110_acquire_underlay,
-	.add_stream_to_ctx = dce110_add_stream_to_ctx,
-	.validate_global = dce110_validate_global,
-	.find_first_free_match_stream_enc_for_link = dce110_find_first_free_match_stream_enc_for_link
-};
-
-static bool underlay_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	struct dce110_timing_generator *dce110_tgv = kzalloc(sizeof(*dce110_tgv),
-							     GFP_KERNEL);
-	struct dce_transform *dce110_xfmv = kzalloc(sizeof(*dce110_xfmv),
-						    GFP_KERNEL);
-	struct dce_mem_input *dce110_miv = kzalloc(sizeof(*dce110_miv),
-						   GFP_KERNEL);
-	struct dce110_opp *dce110_oppv = kzalloc(sizeof(*dce110_oppv),
-						 GFP_KERNEL);
-
-	if (!dce110_tgv || !dce110_xfmv || !dce110_miv || !dce110_oppv) {
-		kfree(dce110_tgv);
-		kfree(dce110_xfmv);
-		kfree(dce110_miv);
-		kfree(dce110_oppv);
-		return false;
-	}
-
-	dce110_opp_v_construct(dce110_oppv, ctx);
-
-	dce110_timing_generator_v_construct(dce110_tgv, ctx);
-	dce110_mem_input_v_construct(dce110_miv, ctx);
-	dce110_transform_v_construct(dce110_xfmv, ctx);
-
-	pool->opps[pool->pipe_count] = &dce110_oppv->base;
-	pool->timing_generators[pool->pipe_count] = &dce110_tgv->base;
-	pool->mis[pool->pipe_count] = &dce110_miv->base;
-	pool->transforms[pool->pipe_count] = &dce110_xfmv->base;
-	pool->pipe_count++;
-
-	/* update the public caps to indicate an underlay is available */
-	ctx->dc->caps.max_slave_planes = 1;
-	ctx->dc->caps.max_slave_yuv_planes = 1;
-	ctx->dc->caps.max_slave_rgb_planes = 0;
-
-	return true;
-}
-
-static void bw_calcs_data_update_from_pplib(struct dc *dc)
-{
-	struct dm_pp_clock_levels clks = {0};
-
-	/*do system clock*/
-	dm_pp_get_clock_levels_by_type(
-			dc->ctx,
-			DM_PP_CLOCK_TYPE_ENGINE_CLK,
-			&clks);
-	/* convert all the clock fro kHz to fix point mHz */
-	dc->bw_vbios->high_sclk = bw_frc_to_fixed(
-			clks.clocks_in_khz[clks.num_levels-1], 1000);
-	dc->bw_vbios->mid1_sclk  = bw_frc_to_fixed(
-			clks.clocks_in_khz[clks.num_levels/8], 1000);
-	dc->bw_vbios->mid2_sclk  = bw_frc_to_fixed(
-			clks.clocks_in_khz[clks.num_levels*2/8], 1000);
-	dc->bw_vbios->mid3_sclk  = bw_frc_to_fixed(
-			clks.clocks_in_khz[clks.num_levels*3/8], 1000);
-	dc->bw_vbios->mid4_sclk  = bw_frc_to_fixed(
-			clks.clocks_in_khz[clks.num_levels*4/8], 1000);
-	dc->bw_vbios->mid5_sclk  = bw_frc_to_fixed(
-			clks.clocks_in_khz[clks.num_levels*5/8], 1000);
-	dc->bw_vbios->mid6_sclk  = bw_frc_to_fixed(
-			clks.clocks_in_khz[clks.num_levels*6/8], 1000);
-	dc->bw_vbios->low_sclk  = bw_frc_to_fixed(
-			clks.clocks_in_khz[0], 1000);
-	dc->sclk_lvls = clks;
-
-	/*do display clock*/
-	dm_pp_get_clock_levels_by_type(
-			dc->ctx,
-			DM_PP_CLOCK_TYPE_DISPLAY_CLK,
-			&clks);
-	dc->bw_vbios->high_voltage_max_dispclk = bw_frc_to_fixed(
-			clks.clocks_in_khz[clks.num_levels-1], 1000);
-	dc->bw_vbios->mid_voltage_max_dispclk  = bw_frc_to_fixed(
-			clks.clocks_in_khz[clks.num_levels>>1], 1000);
-	dc->bw_vbios->low_voltage_max_dispclk  = bw_frc_to_fixed(
-			clks.clocks_in_khz[0], 1000);
-
-	/*do memory clock*/
-	dm_pp_get_clock_levels_by_type(
-			dc->ctx,
-			DM_PP_CLOCK_TYPE_MEMORY_CLK,
-			&clks);
-
-	dc->bw_vbios->low_yclk = bw_frc_to_fixed(
-		clks.clocks_in_khz[0] * MEMORY_TYPE_MULTIPLIER_CZ, 1000);
-	dc->bw_vbios->mid_yclk = bw_frc_to_fixed(
-		clks.clocks_in_khz[clks.num_levels>>1] * MEMORY_TYPE_MULTIPLIER_CZ,
-		1000);
-	dc->bw_vbios->high_yclk = bw_frc_to_fixed(
-		clks.clocks_in_khz[clks.num_levels-1] * MEMORY_TYPE_MULTIPLIER_CZ,
-		1000);
-}
-
-static const struct resource_caps *dce110_resource_cap(
-	struct hw_asic_id *asic_id)
-{
-	if (ASIC_REV_IS_STONEY(asic_id->hw_internal_rev))
-		return &stoney_resource_cap;
-	else
-		return &carrizo_resource_cap;
-}
-
-static bool dce110_resource_construct(
-	uint8_t num_virtual_links,
-	struct dc *dc,
-	struct dce110_resource_pool *pool,
-	struct hw_asic_id asic_id)
-{
-	unsigned int i;
-	struct dc_context *ctx = dc->ctx;
-	struct dc_bios *bp;
-
-	ctx->dc_bios->regs = &bios_regs;
-
-	pool->base.res_cap = dce110_resource_cap(&ctx->asic_id);
-	pool->base.funcs = &dce110_res_pool_funcs;
-
-	/*************************************************
-	 *  Resource + asic cap harcoding                *
-	 *************************************************/
-
-	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
-	pool->base.underlay_pipe_index = pool->base.pipe_count;
-	pool->base.timing_generator_count = pool->base.res_cap->num_timing_generator;
-	dc->caps.max_downscale_ratio = 150;
-	dc->caps.i2c_speed_in_khz = 40;
-	dc->caps.i2c_speed_in_khz_hdcp = 40;
-	dc->caps.max_cursor_size = 128;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.is_apu = true;
-	dc->caps.extended_aux_timeout_support = false;
-	dc->debug = debug_defaults;
-
-	/*************************************************
-	 *  Create resources                             *
-	 *************************************************/
-
-	bp = ctx->dc_bios;
-
-	if (bp->fw_info_valid && bp->fw_info.external_clock_source_frequency_for_dp != 0) {
-		pool->base.dp_clock_source =
-				dce110_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_EXTERNAL, NULL, true);
-
-		pool->base.clock_sources[0] =
-				dce110_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0,
-						&clk_src_regs[0], false);
-		pool->base.clock_sources[1] =
-				dce110_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1,
-						&clk_src_regs[1], false);
-
-		pool->base.clk_src_count = 2;
-
-		/* TODO: find out if CZ support 3 PLLs */
-	}
-
-	if (pool->base.dp_clock_source == NULL) {
-		dm_error("DC: failed to create dp clock source!\n");
-		BREAK_TO_DEBUGGER();
-		goto res_create_fail;
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			BREAK_TO_DEBUGGER();
-			goto res_create_fail;
-		}
-	}
-
-	pool->base.dmcu = dce_dmcu_create(ctx,
-			&dmcu_regs,
-			&dmcu_shift,
-			&dmcu_mask);
-	if (pool->base.dmcu == NULL) {
-		dm_error("DC: failed to create dmcu!\n");
-		BREAK_TO_DEBUGGER();
-		goto res_create_fail;
-	}
-
-	pool->base.abm = dce_abm_create(ctx,
-			&abm_regs,
-			&abm_shift,
-			&abm_mask);
-	if (pool->base.abm == NULL) {
-		dm_error("DC: failed to create abm!\n");
-		BREAK_TO_DEBUGGER();
-		goto res_create_fail;
-	}
-
-	{
-		struct irq_service_init_data init_data;
-		init_data.ctx = dc->ctx;
-		pool->base.irqs = dal_irq_service_dce110_create(&init_data);
-		if (!pool->base.irqs)
-			goto res_create_fail;
-	}
-
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		pool->base.timing_generators[i] = dce110_timing_generator_create(
-				ctx, i, &dce110_tg_offsets[i]);
-		if (pool->base.timing_generators[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create tg!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.mis[i] = dce110_mem_input_create(ctx, i);
-		if (pool->base.mis[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create memory input!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.ipps[i] = dce110_ipp_create(ctx, i);
-		if (pool->base.ipps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create input pixel processor!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.transforms[i] = dce110_transform_create(ctx, i);
-		if (pool->base.transforms[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create transform!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.opps[i] = dce110_opp_create(ctx, i);
-		if (pool->base.opps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create output pixel processor!\n");
-			goto res_create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		pool->base.engines[i] = dce110_aux_engine_create(ctx, i);
-		if (pool->base.engines[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create aux engine!!\n");
-			goto res_create_fail;
-		}
-		pool->base.hw_i2cs[i] = dce110_i2c_hw_create(ctx, i);
-		if (pool->base.hw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create i2c engine!!\n");
-			goto res_create_fail;
-		}
-		pool->base.sw_i2cs[i] = NULL;
-	}
-
-	if (dc->config.fbc_support)
-		dc->fbc_compressor = dce110_compressor_create(ctx);
-
-	if (!underlay_create(ctx, &pool->base))
-		goto res_create_fail;
-
-	if (!resource_construct(num_virtual_links, dc, &pool->base,
-			&res_create_funcs))
-		goto res_create_fail;
-
-	/* Create hardware sequencer */
-	dce110_hw_sequencer_construct(dc);
-
-	dc->caps.max_planes =  pool->base.pipe_count;
-
-	for (i = 0; i < pool->base.underlay_pipe_index; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	dc->caps.planes[pool->base.underlay_pipe_index] = underlay_plane_cap;
-
-	bw_calcs_init(dc->bw_dceip, dc->bw_vbios, dc->ctx->asic_id);
-
-	bw_calcs_data_update_from_pplib(dc);
-
-	return true;
-
-res_create_fail:
-	dce110_resource_destruct(pool);
-	return false;
-}
-
-struct resource_pool *dce110_create_resource_pool(
-	uint8_t num_virtual_links,
-	struct dc *dc,
-	struct hw_asic_id asic_id)
-{
-	struct dce110_resource_pool *pool =
-		kzalloc(sizeof(struct dce110_resource_pool), GFP_KERNEL);
-
-	if (!pool)
-		return NULL;
-
-	if (dce110_resource_construct(num_virtual_links, dc, pool, asic_id))
-		return &pool->base;
-
-	kfree(pool);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.h b/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.h
deleted file mode 100644
index aa4531e0800e..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
-* Copyright 2012-15 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#ifndef __DC_RESOURCE_DCE110_H__
-#define __DC_RESOURCE_DCE110_H__
-
-#include "core_types.h"
-
-struct dc;
-struct resource_pool;
-
-#define TO_DCE110_RES_POOL(pool)\
-	container_of(pool, struct dce110_resource_pool, base)
-
-struct dce110_resource_pool {
-	struct resource_pool base;
-};
-
-void dce110_resource_build_pipe_hw_param(struct pipe_ctx *pipe_ctx);
-
-struct resource_pool *dce110_create_resource_pool(
-	uint8_t num_virtual_links,
-	struct dc *dc,
-	struct hw_asic_id asic_id);
-
-struct stream_encoder *dce110_find_first_free_match_stream_enc_for_link(
-		struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		struct dc_stream_state *stream);
-
-#endif /* __DC_RESOURCE_DCE110_H__ */
-
diff --git a/drivers/gpu/drm/amd/display/dc/dce112/Makefile b/drivers/gpu/drm/amd/display/dc/dce112/Makefile
index e846ef58cab3..7e92effec894 100644
--- a/drivers/gpu/drm/amd/display/dc/dce112/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dce112/Makefile
@@ -25,8 +25,7 @@
 
 CFLAGS_$(AMDDALPATH)/dc/dce112/dce112_resource.o = $(call cc-disable-warning, override-init)
 
-DCE112 = dce112_compressor.o \
-dce112_resource.o
+DCE112 = dce112_compressor.o
 
 AMD_DAL_DCE112 = $(addprefix $(AMDDALPATH)/dc/dce112/,$(DCE112))
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c
deleted file mode 100644
index d1edac46c9a0..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c
+++ /dev/null
@@ -1,1431 +0,0 @@
-/*
-* Copyright 2012-15 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#include "dm_services.h"
-
-#include "link_encoder.h"
-#include "stream_encoder.h"
-
-#include "resource.h"
-#include "include/irq_service_interface.h"
-#include "dce110/dce110_resource.h"
-#include "dce110/dce110_timing_generator.h"
-
-#include "irq/dce110/irq_service_dce110.h"
-#include "dce/dce_mem_input.h"
-#include "dce/dce_transform.h"
-#include "dce/dce_link_encoder.h"
-#include "dce/dce_stream_encoder.h"
-#include "dce/dce_audio.h"
-#include "dce/dce_opp.h"
-#include "dce/dce_ipp.h"
-#include "dce/dce_clock_source.h"
-
-#include "dce/dce_hwseq.h"
-#include "dce112/dce112_hwseq.h"
-#include "dce/dce_abm.h"
-#include "dce/dce_dmcu.h"
-#include "dce/dce_aux.h"
-#include "dce/dce_i2c.h"
-#include "dce/dce_panel_cntl.h"
-
-#include "reg_helper.h"
-
-#include "dce/dce_11_2_d.h"
-#include "dce/dce_11_2_sh_mask.h"
-
-#include "dce100/dce100_resource.h"
-#include "dce112_resource.h"
-
-#define DC_LOGGER				\
-		dc->ctx->logger
-
-#ifndef mmDP_DPHY_INTERNAL_CTRL
-	#define mmDP_DPHY_INTERNAL_CTRL 0x4aa7
-	#define mmDP0_DP_DPHY_INTERNAL_CTRL 0x4aa7
-	#define mmDP1_DP_DPHY_INTERNAL_CTRL 0x4ba7
-	#define mmDP2_DP_DPHY_INTERNAL_CTRL 0x4ca7
-	#define mmDP3_DP_DPHY_INTERNAL_CTRL 0x4da7
-	#define mmDP4_DP_DPHY_INTERNAL_CTRL 0x4ea7
-	#define mmDP5_DP_DPHY_INTERNAL_CTRL 0x4fa7
-	#define mmDP6_DP_DPHY_INTERNAL_CTRL 0x54a7
-	#define mmDP7_DP_DPHY_INTERNAL_CTRL 0x56a7
-	#define mmDP8_DP_DPHY_INTERNAL_CTRL 0x57a7
-#endif
-
-#ifndef mmBIOS_SCRATCH_2
-	#define mmBIOS_SCRATCH_2 0x05CB
-	#define mmBIOS_SCRATCH_3 0x05CC
-	#define mmBIOS_SCRATCH_6 0x05CF
-#endif
-
-#ifndef mmDP_DPHY_BS_SR_SWAP_CNTL
-	#define mmDP_DPHY_BS_SR_SWAP_CNTL                       0x4ADC
-	#define mmDP0_DP_DPHY_BS_SR_SWAP_CNTL                   0x4ADC
-	#define mmDP1_DP_DPHY_BS_SR_SWAP_CNTL                   0x4BDC
-	#define mmDP2_DP_DPHY_BS_SR_SWAP_CNTL                   0x4CDC
-	#define mmDP3_DP_DPHY_BS_SR_SWAP_CNTL                   0x4DDC
-	#define mmDP4_DP_DPHY_BS_SR_SWAP_CNTL                   0x4EDC
-	#define mmDP5_DP_DPHY_BS_SR_SWAP_CNTL                   0x4FDC
-	#define mmDP6_DP_DPHY_BS_SR_SWAP_CNTL                   0x54DC
-#endif
-
-#ifndef mmDP_DPHY_FAST_TRAINING
-	#define mmDP_DPHY_FAST_TRAINING                         0x4ABC
-	#define mmDP0_DP_DPHY_FAST_TRAINING                     0x4ABC
-	#define mmDP1_DP_DPHY_FAST_TRAINING                     0x4BBC
-	#define mmDP2_DP_DPHY_FAST_TRAINING                     0x4CBC
-	#define mmDP3_DP_DPHY_FAST_TRAINING                     0x4DBC
-	#define mmDP4_DP_DPHY_FAST_TRAINING                     0x4EBC
-	#define mmDP5_DP_DPHY_FAST_TRAINING                     0x4FBC
-	#define mmDP6_DP_DPHY_FAST_TRAINING                     0x54BC
-#endif
-
-enum dce112_clk_src_array_id {
-	DCE112_CLK_SRC_PLL0,
-	DCE112_CLK_SRC_PLL1,
-	DCE112_CLK_SRC_PLL2,
-	DCE112_CLK_SRC_PLL3,
-	DCE112_CLK_SRC_PLL4,
-	DCE112_CLK_SRC_PLL5,
-
-	DCE112_CLK_SRC_TOTAL
-};
-
-static const struct dce110_timing_generator_offsets dce112_tg_offsets[] = {
-	{
-		.crtc = (mmCRTC0_CRTC_CONTROL - mmCRTC_CONTROL),
-		.dcp =  (mmDCP0_GRPH_CONTROL - mmGRPH_CONTROL),
-	},
-	{
-		.crtc = (mmCRTC1_CRTC_CONTROL - mmCRTC_CONTROL),
-		.dcp = (mmDCP1_GRPH_CONTROL - mmGRPH_CONTROL),
-	},
-	{
-		.crtc = (mmCRTC2_CRTC_CONTROL - mmCRTC_CONTROL),
-		.dcp = (mmDCP2_GRPH_CONTROL - mmGRPH_CONTROL),
-	},
-	{
-		.crtc = (mmCRTC3_CRTC_CONTROL - mmCRTC_CONTROL),
-		.dcp = (mmDCP3_GRPH_CONTROL - mmGRPH_CONTROL),
-	},
-	{
-		.crtc = (mmCRTC4_CRTC_CONTROL - mmCRTC_CONTROL),
-		.dcp = (mmDCP4_GRPH_CONTROL - mmGRPH_CONTROL),
-	},
-	{
-		.crtc = (mmCRTC5_CRTC_CONTROL - mmCRTC_CONTROL),
-		.dcp = (mmDCP5_GRPH_CONTROL - mmGRPH_CONTROL),
-	}
-};
-
-/* set register offset */
-#define SR(reg_name)\
-	.reg_name = mm ## reg_name
-
-/* set register offset with instance */
-#define SRI(reg_name, block, id)\
-	.reg_name = mm ## block ## id ## _ ## reg_name
-
-static const struct dce_dmcu_registers dmcu_regs = {
-		DMCU_DCE110_COMMON_REG_LIST()
-};
-
-static const struct dce_dmcu_shift dmcu_shift = {
-		DMCU_MASK_SH_LIST_DCE110(__SHIFT)
-};
-
-static const struct dce_dmcu_mask dmcu_mask = {
-		DMCU_MASK_SH_LIST_DCE110(_MASK)
-};
-
-static const struct dce_abm_registers abm_regs = {
-		ABM_DCE110_COMMON_REG_LIST()
-};
-
-static const struct dce_abm_shift abm_shift = {
-		ABM_MASK_SH_LIST_DCE110(__SHIFT)
-};
-
-static const struct dce_abm_mask abm_mask = {
-		ABM_MASK_SH_LIST_DCE110(_MASK)
-};
-
-static const struct dce110_aux_registers_shift aux_shift = {
-	DCE_AUX_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce110_aux_registers_mask aux_mask = {
-	DCE_AUX_MASK_SH_LIST(_MASK)
-};
-
-#define ipp_regs(id)\
-[id] = {\
-		IPP_DCE110_REG_LIST_DCE_BASE(id)\
-}
-
-static const struct dce_ipp_registers ipp_regs[] = {
-		ipp_regs(0),
-		ipp_regs(1),
-		ipp_regs(2),
-		ipp_regs(3),
-		ipp_regs(4),
-		ipp_regs(5)
-};
-
-static const struct dce_ipp_shift ipp_shift = {
-		IPP_DCE100_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
-};
-
-static const struct dce_ipp_mask ipp_mask = {
-		IPP_DCE100_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
-};
-
-#define transform_regs(id)\
-[id] = {\
-		XFM_COMMON_REG_LIST_DCE110(id)\
-}
-
-static const struct dce_transform_registers xfm_regs[] = {
-		transform_regs(0),
-		transform_regs(1),
-		transform_regs(2),
-		transform_regs(3),
-		transform_regs(4),
-		transform_regs(5)
-};
-
-static const struct dce_transform_shift xfm_shift = {
-		XFM_COMMON_MASK_SH_LIST_DCE110(__SHIFT)
-};
-
-static const struct dce_transform_mask xfm_mask = {
-		XFM_COMMON_MASK_SH_LIST_DCE110(_MASK)
-};
-
-#define aux_regs(id)\
-[id] = {\
-	AUX_REG_LIST(id)\
-}
-
-static const struct dce110_link_enc_aux_registers link_enc_aux_regs[] = {
-		aux_regs(0),
-		aux_regs(1),
-		aux_regs(2),
-		aux_regs(3),
-		aux_regs(4),
-		aux_regs(5)
-};
-
-static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
-	{ DCE_PANEL_CNTL_REG_LIST() }
-};
-
-static const struct dce_panel_cntl_shift panel_cntl_shift = {
-	DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_panel_cntl_mask panel_cntl_mask = {
-	DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
-};
-
-#define hpd_regs(id)\
-[id] = {\
-	HPD_REG_LIST(id)\
-}
-
-static const struct dce110_link_enc_hpd_registers link_enc_hpd_regs[] = {
-		hpd_regs(0),
-		hpd_regs(1),
-		hpd_regs(2),
-		hpd_regs(3),
-		hpd_regs(4),
-		hpd_regs(5)
-};
-
-#define link_regs(id)\
-[id] = {\
-	LE_DCE110_REG_LIST(id)\
-}
-
-static const struct dce110_link_enc_registers link_enc_regs[] = {
-	link_regs(0),
-	link_regs(1),
-	link_regs(2),
-	link_regs(3),
-	link_regs(4),
-	link_regs(5),
-	link_regs(6),
-};
-
-#define stream_enc_regs(id)\
-[id] = {\
-	SE_COMMON_REG_LIST(id),\
-	.TMDS_CNTL = 0,\
-}
-
-static const struct dce110_stream_enc_registers stream_enc_regs[] = {
-	stream_enc_regs(0),
-	stream_enc_regs(1),
-	stream_enc_regs(2),
-	stream_enc_regs(3),
-	stream_enc_regs(4),
-	stream_enc_regs(5)
-};
-
-static const struct dce_stream_encoder_shift se_shift = {
-		SE_COMMON_MASK_SH_LIST_DCE112(__SHIFT)
-};
-
-static const struct dce_stream_encoder_mask se_mask = {
-		SE_COMMON_MASK_SH_LIST_DCE112(_MASK)
-};
-
-#define opp_regs(id)\
-[id] = {\
-	OPP_DCE_112_REG_LIST(id),\
-}
-
-static const struct dce_opp_registers opp_regs[] = {
-	opp_regs(0),
-	opp_regs(1),
-	opp_regs(2),
-	opp_regs(3),
-	opp_regs(4),
-	opp_regs(5)
-};
-
-static const struct dce_opp_shift opp_shift = {
-	OPP_COMMON_MASK_SH_LIST_DCE_112(__SHIFT)
-};
-
-static const struct dce_opp_mask opp_mask = {
-	OPP_COMMON_MASK_SH_LIST_DCE_112(_MASK)
-};
-
-#define aux_engine_regs(id)\
-[id] = {\
-	AUX_COMMON_REG_LIST(id), \
-	.AUX_RESET_MASK = 0 \
-}
-
-static const struct dce110_aux_registers aux_engine_regs[] = {
-		aux_engine_regs(0),
-		aux_engine_regs(1),
-		aux_engine_regs(2),
-		aux_engine_regs(3),
-		aux_engine_regs(4),
-		aux_engine_regs(5)
-};
-
-#define audio_regs(id)\
-[id] = {\
-	AUD_COMMON_REG_LIST(id)\
-}
-
-static const struct dce_audio_registers audio_regs[] = {
-	audio_regs(0),
-	audio_regs(1),
-	audio_regs(2),
-	audio_regs(3),
-	audio_regs(4),
-	audio_regs(5)
-};
-
-static const struct dce_audio_shift audio_shift = {
-		AUD_COMMON_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_audio_mask audio_mask = {
-		AUD_COMMON_MASK_SH_LIST(_MASK)
-};
-
-#define clk_src_regs(index, id)\
-[index] = {\
-	CS_COMMON_REG_LIST_DCE_112(id),\
-}
-
-static const struct dce110_clk_src_regs clk_src_regs[] = {
-	clk_src_regs(0, A),
-	clk_src_regs(1, B),
-	clk_src_regs(2, C),
-	clk_src_regs(3, D),
-	clk_src_regs(4, E),
-	clk_src_regs(5, F)
-};
-
-static const struct dce110_clk_src_shift cs_shift = {
-		CS_COMMON_MASK_SH_LIST_DCE_112(__SHIFT)
-};
-
-static const struct dce110_clk_src_mask cs_mask = {
-		CS_COMMON_MASK_SH_LIST_DCE_112(_MASK)
-};
-
-static const struct bios_registers bios_regs = {
-	.BIOS_SCRATCH_3 = mmBIOS_SCRATCH_3,
-	.BIOS_SCRATCH_6 = mmBIOS_SCRATCH_6
-};
-
-static const struct resource_caps polaris_10_resource_cap = {
-		.num_timing_generator = 6,
-		.num_audio = 6,
-		.num_stream_encoder = 6,
-		.num_pll = 8, /* why 8? 6 combo PHY PLL + 2 regular PLLs? */
-		.num_ddc = 6,
-};
-
-static const struct resource_caps polaris_11_resource_cap = {
-		.num_timing_generator = 5,
-		.num_audio = 5,
-		.num_stream_encoder = 5,
-		.num_pll = 8, /* why 8? 6 combo PHY PLL + 2 regular PLLs? */
-		.num_ddc = 5,
-};
-
-static const struct dc_plane_cap plane_cap = {
-	.type = DC_PLANE_TYPE_DCE_RGB,
-
-	.pixel_format_support = {
-			.argb8888 = true,
-			.nv12 = false,
-			.fp16 = true
-	},
-
-	.max_upscale_factor = {
-			.argb8888 = 16000,
-			.nv12 = 1,
-			.fp16 = 1
-	},
-
-	.max_downscale_factor = {
-			.argb8888 = 250,
-			.nv12 = 1,
-			.fp16 = 1
-	},
-	64,
-	64
-};
-
-static const struct dc_debug_options debug_defaults = {
-		.enable_legacy_fast_update = true,
-};
-
-#define CTX  ctx
-#define REG(reg) mm ## reg
-
-#ifndef mmCC_DC_HDMI_STRAPS
-#define mmCC_DC_HDMI_STRAPS 0x4819
-#define CC_DC_HDMI_STRAPS__HDMI_DISABLE_MASK 0x40
-#define CC_DC_HDMI_STRAPS__HDMI_DISABLE__SHIFT 0x6
-#define CC_DC_HDMI_STRAPS__AUDIO_STREAM_NUMBER_MASK 0x700
-#define CC_DC_HDMI_STRAPS__AUDIO_STREAM_NUMBER__SHIFT 0x8
-#endif
-
-static int map_transmitter_id_to_phy_instance(
-	enum transmitter transmitter)
-{
-	switch (transmitter) {
-	case TRANSMITTER_UNIPHY_A:
-		return 0;
-	case TRANSMITTER_UNIPHY_B:
-		return 1;
-	case TRANSMITTER_UNIPHY_C:
-		return 2;
-	case TRANSMITTER_UNIPHY_D:
-		return 3;
-	case TRANSMITTER_UNIPHY_E:
-		return 4;
-	case TRANSMITTER_UNIPHY_F:
-		return 5;
-	case TRANSMITTER_UNIPHY_G:
-		return 6;
-	default:
-		ASSERT(0);
-		return 0;
-	}
-}
-
-static void read_dce_straps(
-	struct dc_context *ctx,
-	struct resource_straps *straps)
-{
-	REG_GET_2(CC_DC_HDMI_STRAPS,
-			HDMI_DISABLE, &straps->hdmi_disable,
-			AUDIO_STREAM_NUMBER, &straps->audio_stream_number);
-
-	REG_GET(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO, &straps->dc_pinstraps_audio);
-}
-
-static struct audio *create_audio(
-		struct dc_context *ctx, unsigned int inst)
-{
-	return dce_audio_create(ctx, inst,
-			&audio_regs[inst], &audio_shift, &audio_mask);
-}
-
-
-static struct timing_generator *dce112_timing_generator_create(
-		struct dc_context *ctx,
-		uint32_t instance,
-		const struct dce110_timing_generator_offsets *offsets)
-{
-	struct dce110_timing_generator *tg110 =
-		kzalloc(sizeof(struct dce110_timing_generator), GFP_KERNEL);
-
-	if (!tg110)
-		return NULL;
-
-	dce110_timing_generator_construct(tg110, ctx, instance, offsets);
-	return &tg110->base;
-}
-
-static struct stream_encoder *dce112_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dce110_stream_encoder *enc110 =
-		kzalloc(sizeof(struct dce110_stream_encoder), GFP_KERNEL);
-
-	if (!enc110)
-		return NULL;
-
-	dce110_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id,
-					&stream_enc_regs[eng_id],
-					&se_shift, &se_mask);
-	return &enc110->base;
-}
-
-#define SRII(reg_name, block, id)\
-	.reg_name[id] = mm ## block ## id ## _ ## reg_name
-
-static const struct dce_hwseq_registers hwseq_reg = {
-		HWSEQ_DCE112_REG_LIST()
-};
-
-static const struct dce_hwseq_shift hwseq_shift = {
-		HWSEQ_DCE112_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_hwseq_mask hwseq_mask = {
-		HWSEQ_DCE112_MASK_SH_LIST(_MASK)
-};
-
-static struct dce_hwseq *dce112_hwseq_create(
-	struct dc_context *ctx)
-{
-	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
-
-	if (hws) {
-		hws->ctx = ctx;
-		hws->regs = &hwseq_reg;
-		hws->shifts = &hwseq_shift;
-		hws->masks = &hwseq_mask;
-	}
-	return hws;
-}
-
-static const struct resource_create_funcs res_create_funcs = {
-	.read_dce_straps = read_dce_straps,
-	.create_audio = create_audio,
-	.create_stream_encoder = dce112_stream_encoder_create,
-	.create_hwseq = dce112_hwseq_create,
-};
-
-#define mi_inst_regs(id) { MI_DCE11_2_REG_LIST(id) }
-static const struct dce_mem_input_registers mi_regs[] = {
-		mi_inst_regs(0),
-		mi_inst_regs(1),
-		mi_inst_regs(2),
-		mi_inst_regs(3),
-		mi_inst_regs(4),
-		mi_inst_regs(5),
-};
-
-static const struct dce_mem_input_shift mi_shifts = {
-		MI_DCE11_2_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_mem_input_mask mi_masks = {
-		MI_DCE11_2_MASK_SH_LIST(_MASK)
-};
-
-static struct mem_input *dce112_mem_input_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_mem_input *dce_mi = kzalloc(sizeof(struct dce_mem_input),
-					       GFP_KERNEL);
-
-	if (!dce_mi) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dce112_mem_input_construct(dce_mi, ctx, inst, &mi_regs[inst], &mi_shifts, &mi_masks);
-	return &dce_mi->base;
-}
-
-static void dce112_transform_destroy(struct transform **xfm)
-{
-	kfree(TO_DCE_TRANSFORM(*xfm));
-	*xfm = NULL;
-}
-
-static struct transform *dce112_transform_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_transform *transform =
-		kzalloc(sizeof(struct dce_transform), GFP_KERNEL);
-
-	if (!transform)
-		return NULL;
-
-	dce_transform_construct(transform, ctx, inst,
-				&xfm_regs[inst], &xfm_shift, &xfm_mask);
-	transform->lb_memory_size = 0x1404; /*5124*/
-	return &transform->base;
-}
-
-static const struct encoder_feature_support link_enc_feature = {
-		.max_hdmi_deep_color = COLOR_DEPTH_121212,
-		.max_hdmi_pixel_clock = 600000,
-		.hdmi_ycbcr420_supported = true,
-		.dp_ycbcr420_supported = false,
-		.flags.bits.IS_HBR2_CAPABLE = true,
-		.flags.bits.IS_HBR3_CAPABLE = true,
-		.flags.bits.IS_TPS3_CAPABLE = true,
-		.flags.bits.IS_TPS4_CAPABLE = true
-};
-
-static struct link_encoder *dce112_link_encoder_create(
-	struct dc_context *ctx,
-	const struct encoder_init_data *enc_init_data)
-{
-	struct dce110_link_encoder *enc110 =
-		kzalloc(sizeof(struct dce110_link_encoder), GFP_KERNEL);
-	int link_regs_id;
-
-	if (!enc110)
-		return NULL;
-
-	link_regs_id =
-		map_transmitter_id_to_phy_instance(enc_init_data->transmitter);
-
-	dce110_link_encoder_construct(enc110,
-				      enc_init_data,
-				      &link_enc_feature,
-				      &link_enc_regs[link_regs_id],
-				      &link_enc_aux_regs[enc_init_data->channel - 1],
-				      &link_enc_hpd_regs[enc_init_data->hpd_source]);
-	return &enc110->base;
-}
-
-static struct panel_cntl *dce112_panel_cntl_create(const struct panel_cntl_init_data *init_data)
-{
-	struct dce_panel_cntl *panel_cntl =
-		kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
-
-	if (!panel_cntl)
-		return NULL;
-
-	dce_panel_cntl_construct(panel_cntl,
-			init_data,
-			&panel_cntl_regs[init_data->inst],
-			&panel_cntl_shift,
-			&panel_cntl_mask);
-
-	return &panel_cntl->base;
-}
-
-static struct input_pixel_processor *dce112_ipp_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dce_ipp *ipp = kzalloc(sizeof(struct dce_ipp), GFP_KERNEL);
-
-	if (!ipp) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dce_ipp_construct(ipp, ctx, inst,
-			&ipp_regs[inst], &ipp_shift, &ipp_mask);
-	return &ipp->base;
-}
-
-static struct output_pixel_processor *dce112_opp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce110_opp *opp =
-		kzalloc(sizeof(struct dce110_opp), GFP_KERNEL);
-
-	if (!opp)
-		return NULL;
-
-	dce110_opp_construct(opp,
-			     ctx, inst, &opp_regs[inst], &opp_shift, &opp_mask);
-	return &opp->base;
-}
-
-static struct dce_aux *dce112_aux_engine_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct aux_engine_dce110 *aux_engine =
-		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
-
-	if (!aux_engine)
-		return NULL;
-
-	dce110_aux_engine_construct(aux_engine, ctx, inst,
-				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
-				    &aux_engine_regs[inst],
-					&aux_mask,
-					&aux_shift,
-					ctx->dc->caps.extended_aux_timeout_support);
-
-	return &aux_engine->base;
-}
-#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
-
-static const struct dce_i2c_registers i2c_hw_regs[] = {
-		i2c_inst_regs(1),
-		i2c_inst_regs(2),
-		i2c_inst_regs(3),
-		i2c_inst_regs(4),
-		i2c_inst_regs(5),
-		i2c_inst_regs(6),
-};
-
-static const struct dce_i2c_shift i2c_shifts = {
-		I2C_COMMON_MASK_SH_LIST_DCE110(__SHIFT)
-};
-
-static const struct dce_i2c_mask i2c_masks = {
-		I2C_COMMON_MASK_SH_LIST_DCE110(_MASK)
-};
-
-static struct dce_i2c_hw *dce112_i2c_hw_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_i2c_hw *dce_i2c_hw =
-		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
-
-	if (!dce_i2c_hw)
-		return NULL;
-
-	dce112_i2c_hw_construct(dce_i2c_hw, ctx, inst,
-				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
-
-	return dce_i2c_hw;
-}
-static struct clock_source *dce112_clock_source_create(
-	struct dc_context *ctx,
-	struct dc_bios *bios,
-	enum clock_source_id id,
-	const struct dce110_clk_src_regs *regs,
-	bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src =
-		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dce112_clk_src_construct(clk_src, ctx, bios, id,
-			regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-
-	kfree(clk_src);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
-static void dce112_clock_source_destroy(struct clock_source **clk_src)
-{
-	kfree(TO_DCE110_CLK_SRC(*clk_src));
-	*clk_src = NULL;
-}
-
-static void dce112_resource_destruct(struct dce110_resource_pool *pool)
-{
-	unsigned int i;
-
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		if (pool->base.opps[i] != NULL)
-			dce110_opp_destroy(&pool->base.opps[i]);
-
-		if (pool->base.transforms[i] != NULL)
-			dce112_transform_destroy(&pool->base.transforms[i]);
-
-		if (pool->base.ipps[i] != NULL)
-			dce_ipp_destroy(&pool->base.ipps[i]);
-
-		if (pool->base.mis[i] != NULL) {
-			kfree(TO_DCE_MEM_INPUT(pool->base.mis[i]));
-			pool->base.mis[i] = NULL;
-		}
-
-		if (pool->base.timing_generators[i] != NULL) {
-			kfree(DCE110TG_FROM_TG(pool->base.timing_generators[i]));
-			pool->base.timing_generators[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		if (pool->base.engines[i] != NULL)
-			dce110_engine_destroy(&pool->base.engines[i]);
-		if (pool->base.hw_i2cs[i] != NULL) {
-			kfree(pool->base.hw_i2cs[i]);
-			pool->base.hw_i2cs[i] = NULL;
-		}
-		if (pool->base.sw_i2cs[i] != NULL) {
-			kfree(pool->base.sw_i2cs[i]);
-			pool->base.sw_i2cs[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.stream_enc_count; i++) {
-		if (pool->base.stream_enc[i] != NULL)
-			kfree(DCE110STRENC_FROM_STRENC(pool->base.stream_enc[i]));
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] != NULL) {
-			dce112_clock_source_destroy(&pool->base.clock_sources[i]);
-		}
-	}
-
-	if (pool->base.dp_clock_source != NULL)
-		dce112_clock_source_destroy(&pool->base.dp_clock_source);
-
-	for (i = 0; i < pool->base.audio_count; i++)	{
-		if (pool->base.audios[i] != NULL) {
-			dce_aud_destroy(&pool->base.audios[i]);
-		}
-	}
-
-	if (pool->base.abm != NULL)
-		dce_abm_destroy(&pool->base.abm);
-
-	if (pool->base.dmcu != NULL)
-		dce_dmcu_destroy(&pool->base.dmcu);
-
-	if (pool->base.irqs != NULL) {
-		dal_irq_service_destroy(&pool->base.irqs);
-	}
-}
-
-static struct clock_source *find_matching_pll(
-		struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		const struct dc_stream_state *const stream)
-{
-	switch (stream->link->link_enc->transmitter) {
-	case TRANSMITTER_UNIPHY_A:
-		return pool->clock_sources[DCE112_CLK_SRC_PLL0];
-	case TRANSMITTER_UNIPHY_B:
-		return pool->clock_sources[DCE112_CLK_SRC_PLL1];
-	case TRANSMITTER_UNIPHY_C:
-		return pool->clock_sources[DCE112_CLK_SRC_PLL2];
-	case TRANSMITTER_UNIPHY_D:
-		return pool->clock_sources[DCE112_CLK_SRC_PLL3];
-	case TRANSMITTER_UNIPHY_E:
-		return pool->clock_sources[DCE112_CLK_SRC_PLL4];
-	case TRANSMITTER_UNIPHY_F:
-		return pool->clock_sources[DCE112_CLK_SRC_PLL5];
-	default:
-		return NULL;
-	}
-
-	return NULL;
-}
-
-static enum dc_status build_mapped_resource(
-		const struct dc *dc,
-		struct dc_state *context,
-		struct dc_stream_state *stream)
-{
-	struct pipe_ctx *pipe_ctx = resource_get_otg_master_for_stream(&context->res_ctx, stream);
-
-	if (!pipe_ctx)
-		return DC_ERROR_UNEXPECTED;
-
-	dce110_resource_build_pipe_hw_param(pipe_ctx);
-
-	resource_build_info_frame(pipe_ctx);
-
-	return DC_OK;
-}
-
-bool dce112_validate_bandwidth(
-	struct dc *dc,
-	struct dc_state *context,
-	bool fast_validate)
-{
-	bool result = false;
-
-	DC_LOG_BANDWIDTH_CALCS(
-		"%s: start",
-		__func__);
-
-	if (bw_calcs(
-			dc->ctx,
-			dc->bw_dceip,
-			dc->bw_vbios,
-			context->res_ctx.pipe_ctx,
-			dc->res_pool->pipe_count,
-			&context->bw_ctx.bw.dce))
-		result = true;
-
-	if (!result)
-		DC_LOG_BANDWIDTH_VALIDATION(
-			"%s: Bandwidth validation failed!",
-			__func__);
-
-	if (memcmp(&dc->current_state->bw_ctx.bw.dce,
-			&context->bw_ctx.bw.dce, sizeof(context->bw_ctx.bw.dce))) {
-
-		DC_LOG_BANDWIDTH_CALCS(
-			"%s: finish,\n"
-			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
-			"stutMark_b: %d stutMark_a: %d\n"
-			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
-			"stutMark_b: %d stutMark_a: %d\n"
-			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
-			"stutMark_b: %d stutMark_a: %d stutter_mode_enable: %d\n"
-			"cstate: %d pstate: %d nbpstate: %d sync: %d dispclk: %d\n"
-			"sclk: %d sclk_sleep: %d yclk: %d blackout_recovery_time_us: %d\n"
-			,
-			__func__,
-			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[0].b_mark,
-			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[0].a_mark,
-			context->bw_ctx.bw.dce.urgent_wm_ns[0].b_mark,
-			context->bw_ctx.bw.dce.urgent_wm_ns[0].a_mark,
-			context->bw_ctx.bw.dce.stutter_exit_wm_ns[0].b_mark,
-			context->bw_ctx.bw.dce.stutter_exit_wm_ns[0].a_mark,
-			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[1].b_mark,
-			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[1].a_mark,
-			context->bw_ctx.bw.dce.urgent_wm_ns[1].b_mark,
-			context->bw_ctx.bw.dce.urgent_wm_ns[1].a_mark,
-			context->bw_ctx.bw.dce.stutter_exit_wm_ns[1].b_mark,
-			context->bw_ctx.bw.dce.stutter_exit_wm_ns[1].a_mark,
-			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[2].b_mark,
-			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[2].a_mark,
-			context->bw_ctx.bw.dce.urgent_wm_ns[2].b_mark,
-			context->bw_ctx.bw.dce.urgent_wm_ns[2].a_mark,
-			context->bw_ctx.bw.dce.stutter_exit_wm_ns[2].b_mark,
-			context->bw_ctx.bw.dce.stutter_exit_wm_ns[2].a_mark,
-			context->bw_ctx.bw.dce.stutter_mode_enable,
-			context->bw_ctx.bw.dce.cpuc_state_change_enable,
-			context->bw_ctx.bw.dce.cpup_state_change_enable,
-			context->bw_ctx.bw.dce.nbp_state_change_enable,
-			context->bw_ctx.bw.dce.all_displays_in_sync,
-			context->bw_ctx.bw.dce.dispclk_khz,
-			context->bw_ctx.bw.dce.sclk_khz,
-			context->bw_ctx.bw.dce.sclk_deep_sleep_khz,
-			context->bw_ctx.bw.dce.yclk_khz,
-			context->bw_ctx.bw.dce.blackout_recovery_time_us);
-	}
-	return result;
-}
-
-enum dc_status resource_map_phy_clock_resources(
-		const struct dc *dc,
-		struct dc_state *context,
-		struct dc_stream_state *stream)
-{
-
-	/* acquire new resources */
-	struct pipe_ctx *pipe_ctx = resource_get_otg_master_for_stream(
-			&context->res_ctx, stream);
-
-	if (!pipe_ctx)
-		return DC_ERROR_UNEXPECTED;
-
-	if (dc_is_dp_signal(pipe_ctx->stream->signal)
-		|| dc_is_virtual_signal(pipe_ctx->stream->signal))
-		pipe_ctx->clock_source =
-				dc->res_pool->dp_clock_source;
-	else {
-		if (stream && stream->link && stream->link->link_enc)
-			pipe_ctx->clock_source = find_matching_pll(
-				&context->res_ctx, dc->res_pool,
-				stream);
-	}
-
-	if (pipe_ctx->clock_source == NULL)
-		return DC_NO_CLOCK_SOURCE_RESOURCE;
-
-	resource_reference_clock_source(
-		&context->res_ctx,
-		dc->res_pool,
-		pipe_ctx->clock_source);
-
-	return DC_OK;
-}
-
-static bool dce112_validate_surface_sets(
-		struct dc_state *context)
-{
-	int i;
-
-	for (i = 0; i < context->stream_count; i++) {
-		if (context->stream_status[i].plane_count == 0)
-			continue;
-
-		if (context->stream_status[i].plane_count > 1)
-			return false;
-
-		if (context->stream_status[i].plane_states[0]->format
-				>= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN)
-			return false;
-	}
-
-	return true;
-}
-
-enum dc_status dce112_add_stream_to_ctx(
-		struct dc *dc,
-		struct dc_state *new_ctx,
-		struct dc_stream_state *dc_stream)
-{
-	enum dc_status result;
-
-	result = resource_map_pool_resources(dc, new_ctx, dc_stream);
-
-	if (result == DC_OK)
-		result = resource_map_phy_clock_resources(dc, new_ctx, dc_stream);
-
-
-	if (result == DC_OK)
-		result = build_mapped_resource(dc, new_ctx, dc_stream);
-
-	return result;
-}
-
-static enum dc_status dce112_validate_global(
-		struct dc *dc,
-		struct dc_state *context)
-{
-	if (!dce112_validate_surface_sets(context))
-		return DC_FAIL_SURFACE_VALIDATE;
-
-	return DC_OK;
-}
-
-static void dce112_destroy_resource_pool(struct resource_pool **pool)
-{
-	struct dce110_resource_pool *dce110_pool = TO_DCE110_RES_POOL(*pool);
-
-	dce112_resource_destruct(dce110_pool);
-	kfree(dce110_pool);
-	*pool = NULL;
-}
-
-static const struct resource_funcs dce112_res_pool_funcs = {
-	.destroy = dce112_destroy_resource_pool,
-	.link_enc_create = dce112_link_encoder_create,
-	.panel_cntl_create = dce112_panel_cntl_create,
-	.validate_bandwidth = dce112_validate_bandwidth,
-	.validate_plane = dce100_validate_plane,
-	.add_stream_to_ctx = dce112_add_stream_to_ctx,
-	.validate_global = dce112_validate_global,
-	.find_first_free_match_stream_enc_for_link = dce110_find_first_free_match_stream_enc_for_link
-};
-
-static void bw_calcs_data_update_from_pplib(struct dc *dc)
-{
-	struct dm_pp_clock_levels_with_latency eng_clks = {0};
-	struct dm_pp_clock_levels_with_latency mem_clks = {0};
-	struct dm_pp_wm_sets_with_clock_ranges clk_ranges = {0};
-	struct dm_pp_clock_levels clks = {0};
-	int memory_type_multiplier = MEMORY_TYPE_MULTIPLIER_CZ;
-
-	if (dc->bw_vbios && dc->bw_vbios->memory_type == bw_def_hbm)
-		memory_type_multiplier = MEMORY_TYPE_HBM;
-
-	/*do system clock  TODO PPLIB: after PPLIB implement,
-	 * then remove old way
-	 */
-	if (!dm_pp_get_clock_levels_by_type_with_latency(
-			dc->ctx,
-			DM_PP_CLOCK_TYPE_ENGINE_CLK,
-			&eng_clks)) {
-
-		/* This is only for temporary */
-		dm_pp_get_clock_levels_by_type(
-				dc->ctx,
-				DM_PP_CLOCK_TYPE_ENGINE_CLK,
-				&clks);
-		/* convert all the clock fro kHz to fix point mHz */
-		dc->bw_vbios->high_sclk = bw_frc_to_fixed(
-				clks.clocks_in_khz[clks.num_levels-1], 1000);
-		dc->bw_vbios->mid1_sclk  = bw_frc_to_fixed(
-				clks.clocks_in_khz[clks.num_levels/8], 1000);
-		dc->bw_vbios->mid2_sclk  = bw_frc_to_fixed(
-				clks.clocks_in_khz[clks.num_levels*2/8], 1000);
-		dc->bw_vbios->mid3_sclk  = bw_frc_to_fixed(
-				clks.clocks_in_khz[clks.num_levels*3/8], 1000);
-		dc->bw_vbios->mid4_sclk  = bw_frc_to_fixed(
-				clks.clocks_in_khz[clks.num_levels*4/8], 1000);
-		dc->bw_vbios->mid5_sclk  = bw_frc_to_fixed(
-				clks.clocks_in_khz[clks.num_levels*5/8], 1000);
-		dc->bw_vbios->mid6_sclk  = bw_frc_to_fixed(
-				clks.clocks_in_khz[clks.num_levels*6/8], 1000);
-		dc->bw_vbios->low_sclk  = bw_frc_to_fixed(
-				clks.clocks_in_khz[0], 1000);
-
-		/*do memory clock*/
-		dm_pp_get_clock_levels_by_type(
-				dc->ctx,
-				DM_PP_CLOCK_TYPE_MEMORY_CLK,
-				&clks);
-
-		dc->bw_vbios->low_yclk = bw_frc_to_fixed(
-			clks.clocks_in_khz[0] * memory_type_multiplier, 1000);
-		dc->bw_vbios->mid_yclk = bw_frc_to_fixed(
-			clks.clocks_in_khz[clks.num_levels>>1] * memory_type_multiplier,
-			1000);
-		dc->bw_vbios->high_yclk = bw_frc_to_fixed(
-			clks.clocks_in_khz[clks.num_levels-1] * memory_type_multiplier,
-			1000);
-
-		return;
-	}
-
-	/* convert all the clock fro kHz to fix point mHz  TODO: wloop data */
-	dc->bw_vbios->high_sclk = bw_frc_to_fixed(
-		eng_clks.data[eng_clks.num_levels-1].clocks_in_khz, 1000);
-	dc->bw_vbios->mid1_sclk  = bw_frc_to_fixed(
-		eng_clks.data[eng_clks.num_levels/8].clocks_in_khz, 1000);
-	dc->bw_vbios->mid2_sclk  = bw_frc_to_fixed(
-		eng_clks.data[eng_clks.num_levels*2/8].clocks_in_khz, 1000);
-	dc->bw_vbios->mid3_sclk  = bw_frc_to_fixed(
-		eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz, 1000);
-	dc->bw_vbios->mid4_sclk  = bw_frc_to_fixed(
-		eng_clks.data[eng_clks.num_levels*4/8].clocks_in_khz, 1000);
-	dc->bw_vbios->mid5_sclk  = bw_frc_to_fixed(
-		eng_clks.data[eng_clks.num_levels*5/8].clocks_in_khz, 1000);
-	dc->bw_vbios->mid6_sclk  = bw_frc_to_fixed(
-		eng_clks.data[eng_clks.num_levels*6/8].clocks_in_khz, 1000);
-	dc->bw_vbios->low_sclk  = bw_frc_to_fixed(
-			eng_clks.data[0].clocks_in_khz, 1000);
-
-	/*do memory clock*/
-	dm_pp_get_clock_levels_by_type_with_latency(
-			dc->ctx,
-			DM_PP_CLOCK_TYPE_MEMORY_CLK,
-			&mem_clks);
-
-	/* we don't need to call PPLIB for validation clock since they
-	 * also give us the highest sclk and highest mclk (UMA clock).
-	 * ALSO always convert UMA clock (from PPLIB)  to YCLK (HW formula):
-	 * YCLK = UMACLK*m_memoryTypeMultiplier
-	 */
-	dc->bw_vbios->low_yclk = bw_frc_to_fixed(
-		mem_clks.data[0].clocks_in_khz * memory_type_multiplier, 1000);
-	dc->bw_vbios->mid_yclk = bw_frc_to_fixed(
-		mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz * memory_type_multiplier,
-		1000);
-	dc->bw_vbios->high_yclk = bw_frc_to_fixed(
-		mem_clks.data[mem_clks.num_levels-1].clocks_in_khz * memory_type_multiplier,
-		1000);
-
-	/* Now notify PPLib/SMU about which Watermarks sets they should select
-	 * depending on DPM state they are in. And update BW MGR GFX Engine and
-	 * Memory clock member variables for Watermarks calculations for each
-	 * Watermark Set
-	 */
-	clk_ranges.num_wm_sets = 4;
-	clk_ranges.wm_clk_ranges[0].wm_set_id = WM_SET_A;
-	clk_ranges.wm_clk_ranges[0].wm_min_eng_clk_in_khz =
-			eng_clks.data[0].clocks_in_khz;
-	clk_ranges.wm_clk_ranges[0].wm_max_eng_clk_in_khz =
-			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz - 1;
-	clk_ranges.wm_clk_ranges[0].wm_min_mem_clk_in_khz =
-			mem_clks.data[0].clocks_in_khz;
-	clk_ranges.wm_clk_ranges[0].wm_max_mem_clk_in_khz =
-			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz - 1;
-
-	clk_ranges.wm_clk_ranges[1].wm_set_id = WM_SET_B;
-	clk_ranges.wm_clk_ranges[1].wm_min_eng_clk_in_khz =
-			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz;
-	/* 5 GHz instead of data[7].clockInKHz to cover Overdrive */
-	clk_ranges.wm_clk_ranges[1].wm_max_eng_clk_in_khz = 5000000;
-	clk_ranges.wm_clk_ranges[1].wm_min_mem_clk_in_khz =
-			mem_clks.data[0].clocks_in_khz;
-	clk_ranges.wm_clk_ranges[1].wm_max_mem_clk_in_khz =
-			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz - 1;
-
-	clk_ranges.wm_clk_ranges[2].wm_set_id = WM_SET_C;
-	clk_ranges.wm_clk_ranges[2].wm_min_eng_clk_in_khz =
-			eng_clks.data[0].clocks_in_khz;
-	clk_ranges.wm_clk_ranges[2].wm_max_eng_clk_in_khz =
-			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz - 1;
-	clk_ranges.wm_clk_ranges[2].wm_min_mem_clk_in_khz =
-			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz;
-	/* 5 GHz instead of data[2].clockInKHz to cover Overdrive */
-	clk_ranges.wm_clk_ranges[2].wm_max_mem_clk_in_khz = 5000000;
-
-	clk_ranges.wm_clk_ranges[3].wm_set_id = WM_SET_D;
-	clk_ranges.wm_clk_ranges[3].wm_min_eng_clk_in_khz =
-			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz;
-	/* 5 GHz instead of data[7].clockInKHz to cover Overdrive */
-	clk_ranges.wm_clk_ranges[3].wm_max_eng_clk_in_khz = 5000000;
-	clk_ranges.wm_clk_ranges[3].wm_min_mem_clk_in_khz =
-			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz;
-	/* 5 GHz instead of data[2].clockInKHz to cover Overdrive */
-	clk_ranges.wm_clk_ranges[3].wm_max_mem_clk_in_khz = 5000000;
-
-	/* Notify PP Lib/SMU which Watermarks to use for which clock ranges */
-	dm_pp_notify_wm_clock_changes(dc->ctx, &clk_ranges);
-}
-
-static const struct resource_caps *dce112_resource_cap(
-	struct hw_asic_id *asic_id)
-{
-	if (ASIC_REV_IS_POLARIS11_M(asic_id->hw_internal_rev) ||
-	    ASIC_REV_IS_POLARIS12_V(asic_id->hw_internal_rev))
-		return &polaris_11_resource_cap;
-	else
-		return &polaris_10_resource_cap;
-}
-
-static bool dce112_resource_construct(
-	uint8_t num_virtual_links,
-	struct dc *dc,
-	struct dce110_resource_pool *pool)
-{
-	unsigned int i;
-	struct dc_context *ctx = dc->ctx;
-
-	ctx->dc_bios->regs = &bios_regs;
-
-	pool->base.res_cap = dce112_resource_cap(&ctx->asic_id);
-	pool->base.funcs = &dce112_res_pool_funcs;
-
-	/*************************************************
-	 *  Resource + asic cap harcoding                *
-	 *************************************************/
-	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
-	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
-	pool->base.timing_generator_count = pool->base.res_cap->num_timing_generator;
-	dc->caps.max_downscale_ratio = 200;
-	dc->caps.i2c_speed_in_khz = 100;
-	dc->caps.i2c_speed_in_khz_hdcp = 100; /*1.4 w/a not applied by default*/
-	dc->caps.max_cursor_size = 128;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.dual_link_dvi = true;
-	dc->caps.extended_aux_timeout_support = false;
-	dc->debug = debug_defaults;
-
-	/*************************************************
-	 *  Create resources                             *
-	 *************************************************/
-
-	pool->base.clock_sources[DCE112_CLK_SRC_PLL0] =
-			dce112_clock_source_create(
-				ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL0,
-				&clk_src_regs[0], false);
-	pool->base.clock_sources[DCE112_CLK_SRC_PLL1] =
-			dce112_clock_source_create(
-				ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL1,
-				&clk_src_regs[1], false);
-	pool->base.clock_sources[DCE112_CLK_SRC_PLL2] =
-			dce112_clock_source_create(
-				ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL2,
-				&clk_src_regs[2], false);
-	pool->base.clock_sources[DCE112_CLK_SRC_PLL3] =
-			dce112_clock_source_create(
-				ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL3,
-				&clk_src_regs[3], false);
-	pool->base.clock_sources[DCE112_CLK_SRC_PLL4] =
-			dce112_clock_source_create(
-				ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL4,
-				&clk_src_regs[4], false);
-	pool->base.clock_sources[DCE112_CLK_SRC_PLL5] =
-			dce112_clock_source_create(
-				ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL5,
-				&clk_src_regs[5], false);
-	pool->base.clk_src_count = DCE112_CLK_SRC_TOTAL;
-
-	pool->base.dp_clock_source =  dce112_clock_source_create(
-		ctx, ctx->dc_bios,
-		CLOCK_SOURCE_ID_DP_DTO, &clk_src_regs[0], true);
-
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			BREAK_TO_DEBUGGER();
-			goto res_create_fail;
-		}
-	}
-
-	pool->base.dmcu = dce_dmcu_create(ctx,
-			&dmcu_regs,
-			&dmcu_shift,
-			&dmcu_mask);
-	if (pool->base.dmcu == NULL) {
-		dm_error("DC: failed to create dmcu!\n");
-		BREAK_TO_DEBUGGER();
-		goto res_create_fail;
-	}
-
-	pool->base.abm = dce_abm_create(ctx,
-			&abm_regs,
-			&abm_shift,
-			&abm_mask);
-	if (pool->base.abm == NULL) {
-		dm_error("DC: failed to create abm!\n");
-		BREAK_TO_DEBUGGER();
-		goto res_create_fail;
-	}
-
-	{
-		struct irq_service_init_data init_data;
-		init_data.ctx = dc->ctx;
-		pool->base.irqs = dal_irq_service_dce110_create(&init_data);
-		if (!pool->base.irqs)
-			goto res_create_fail;
-	}
-
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		pool->base.timing_generators[i] =
-				dce112_timing_generator_create(
-					ctx,
-					i,
-					&dce112_tg_offsets[i]);
-		if (pool->base.timing_generators[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create tg!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.mis[i] = dce112_mem_input_create(ctx, i);
-		if (pool->base.mis[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create memory input!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.ipps[i] = dce112_ipp_create(ctx, i);
-		if (pool->base.ipps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create input pixel processor!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.transforms[i] = dce112_transform_create(ctx, i);
-		if (pool->base.transforms[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create transform!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.opps[i] = dce112_opp_create(
-			ctx,
-			i);
-		if (pool->base.opps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create output pixel processor!\n");
-			goto res_create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		pool->base.engines[i] = dce112_aux_engine_create(ctx, i);
-		if (pool->base.engines[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create aux engine!!\n");
-			goto res_create_fail;
-		}
-		pool->base.hw_i2cs[i] = dce112_i2c_hw_create(ctx, i);
-		if (pool->base.hw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create i2c engine!!\n");
-			goto res_create_fail;
-		}
-		pool->base.sw_i2cs[i] = NULL;
-	}
-
-	if (!resource_construct(num_virtual_links, dc, &pool->base,
-			  &res_create_funcs))
-		goto res_create_fail;
-
-	dc->caps.max_planes =  pool->base.pipe_count;
-
-	for (i = 0; i < dc->caps.max_planes; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	/* Create hardware sequencer */
-	dce112_hw_sequencer_construct(dc);
-
-	bw_calcs_init(dc->bw_dceip, dc->bw_vbios, dc->ctx->asic_id);
-
-	bw_calcs_data_update_from_pplib(dc);
-
-	return true;
-
-res_create_fail:
-	dce112_resource_destruct(pool);
-	return false;
-}
-
-struct resource_pool *dce112_create_resource_pool(
-	uint8_t num_virtual_links,
-	struct dc *dc)
-{
-	struct dce110_resource_pool *pool =
-		kzalloc(sizeof(struct dce110_resource_pool), GFP_KERNEL);
-
-	if (!pool)
-		return NULL;
-
-	if (dce112_resource_construct(num_virtual_links, dc, pool))
-		return &pool->base;
-
-	kfree(pool);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.h b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.h
deleted file mode 100644
index 1f57ebc6f9b4..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.h
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
-* Copyright 2012-15 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#ifndef __DC_RESOURCE_DCE112_H__
-#define __DC_RESOURCE_DCE112_H__
-
-#include "core_types.h"
-
-struct dc;
-struct resource_pool;
-
-struct resource_pool *dce112_create_resource_pool(
-	uint8_t num_virtual_links,
-	struct dc *dc);
-
-enum dc_status dce112_validate_with_context(
-		struct dc *dc,
-		const struct dc_validation_set set[],
-		int set_count,
-		struct dc_state *context,
-		struct dc_state *old_context);
-
-bool dce112_validate_bandwidth(
-	struct dc *dc,
-	struct dc_state *context,
-	bool fast_validate);
-
-enum dc_status dce112_add_stream_to_ctx(
-		struct dc *dc,
-		struct dc_state *new_ctx,
-		struct dc_stream_state *dc_stream);
-
-
-#endif /* __DC_RESOURCE_DCE112_H__ */
-
diff --git a/drivers/gpu/drm/amd/display/dc/dce120/Makefile b/drivers/gpu/drm/amd/display/dc/dce120/Makefile
index 097cf407a15d..1e3ef68a452a 100644
--- a/drivers/gpu/drm/amd/display/dc/dce120/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dce120/Makefile
@@ -26,7 +26,7 @@
 
 CFLAGS_$(AMDDALPATH)/dc/dce120/dce120_resource.o = $(call cc-disable-warning, override-init)
 
-DCE120 = dce120_resource.o dce120_timing_generator.o \
+DCE120 = dce120_timing_generator.o
 
 AMD_DAL_DCE120 = $(addprefix $(AMDDALPATH)/dc/dce120/,$(DCE120))
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c b/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c
deleted file mode 100644
index 962de79be169..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c
+++ /dev/null
@@ -1,1288 +0,0 @@
-/*
-* Copyright 2012-15 Advanced Micro Devices, Inc.cls
-*
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#include "dm_services.h"
-
-
-#include "stream_encoder.h"
-#include "resource.h"
-#include "include/irq_service_interface.h"
-#include "dce120_resource.h"
-
-#include "dce112/dce112_resource.h"
-
-#include "dce110/dce110_resource.h"
-#include "virtual/virtual_stream_encoder.h"
-#include "dce120_timing_generator.h"
-#include "irq/dce120/irq_service_dce120.h"
-#include "dce/dce_opp.h"
-#include "dce/dce_clock_source.h"
-#include "dce/dce_ipp.h"
-#include "dce/dce_mem_input.h"
-#include "dce/dce_panel_cntl.h"
-
-#include "dce110/dce110_hwseq.h"
-#include "dce120/dce120_hwseq.h"
-#include "dce/dce_transform.h"
-#include "clk_mgr.h"
-#include "dce/dce_audio.h"
-#include "dce/dce_link_encoder.h"
-#include "dce/dce_stream_encoder.h"
-#include "dce/dce_hwseq.h"
-#include "dce/dce_abm.h"
-#include "dce/dce_dmcu.h"
-#include "dce/dce_aux.h"
-#include "dce/dce_i2c.h"
-
-#include "dce/dce_12_0_offset.h"
-#include "dce/dce_12_0_sh_mask.h"
-#include "soc15_hw_ip.h"
-#include "vega10_ip_offset.h"
-#include "nbio/nbio_6_1_offset.h"
-#include "mmhub/mmhub_1_0_offset.h"
-#include "mmhub/mmhub_1_0_sh_mask.h"
-#include "reg_helper.h"
-
-#include "dce100/dce100_resource.h"
-
-#ifndef mmDP0_DP_DPHY_INTERNAL_CTRL
-	#define mmDP0_DP_DPHY_INTERNAL_CTRL		0x210f
-	#define mmDP0_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
-	#define mmDP1_DP_DPHY_INTERNAL_CTRL		0x220f
-	#define mmDP1_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
-	#define mmDP2_DP_DPHY_INTERNAL_CTRL		0x230f
-	#define mmDP2_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
-	#define mmDP3_DP_DPHY_INTERNAL_CTRL		0x240f
-	#define mmDP3_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
-	#define mmDP4_DP_DPHY_INTERNAL_CTRL		0x250f
-	#define mmDP4_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
-	#define mmDP5_DP_DPHY_INTERNAL_CTRL		0x260f
-	#define mmDP5_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
-	#define mmDP6_DP_DPHY_INTERNAL_CTRL		0x270f
-	#define mmDP6_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
-#endif
-
-enum dce120_clk_src_array_id {
-	DCE120_CLK_SRC_PLL0,
-	DCE120_CLK_SRC_PLL1,
-	DCE120_CLK_SRC_PLL2,
-	DCE120_CLK_SRC_PLL3,
-	DCE120_CLK_SRC_PLL4,
-	DCE120_CLK_SRC_PLL5,
-
-	DCE120_CLK_SRC_TOTAL
-};
-
-static const struct dce110_timing_generator_offsets dce120_tg_offsets[] = {
-	{
-		.crtc = (mmCRTC0_CRTC_CONTROL - mmCRTC0_CRTC_CONTROL),
-	},
-	{
-		.crtc = (mmCRTC1_CRTC_CONTROL - mmCRTC0_CRTC_CONTROL),
-	},
-	{
-		.crtc = (mmCRTC2_CRTC_CONTROL - mmCRTC0_CRTC_CONTROL),
-	},
-	{
-		.crtc = (mmCRTC3_CRTC_CONTROL - mmCRTC0_CRTC_CONTROL),
-	},
-	{
-		.crtc = (mmCRTC4_CRTC_CONTROL - mmCRTC0_CRTC_CONTROL),
-	},
-	{
-		.crtc = (mmCRTC5_CRTC_CONTROL - mmCRTC0_CRTC_CONTROL),
-	}
-};
-
-/* begin *********************
- * macros to expend register list macro defined in HW object header file */
-
-#define BASE_INNER(seg) \
-	DCE_BASE__INST0_SEG ## seg
-
-#define NBIO_BASE_INNER(seg) \
-	NBIF_BASE__INST0_SEG ## seg
-
-#define NBIO_BASE(seg) \
-	NBIO_BASE_INNER(seg)
-
-/* compile time expand base address. */
-#define BASE(seg) \
-	BASE_INNER(seg)
-
-#define SR(reg_name)\
-		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) +  \
-					mm ## reg_name
-
-#define SRI(reg_name, block, id)\
-	.reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-/* MMHUB */
-#define MMHUB_BASE_INNER(seg) \
-	MMHUB_BASE__INST0_SEG ## seg
-
-#define MMHUB_BASE(seg) \
-	MMHUB_BASE_INNER(seg)
-
-#define MMHUB_SR(reg_name)\
-		.reg_name = MMHUB_BASE(mm ## reg_name ## _BASE_IDX) +  \
-					mm ## reg_name
-
-/* macros to expend register list macro defined in HW object header file
- * end *********************/
-
-
-static const struct dce_dmcu_registers dmcu_regs = {
-		DMCU_DCE110_COMMON_REG_LIST()
-};
-
-static const struct dce_dmcu_shift dmcu_shift = {
-		DMCU_MASK_SH_LIST_DCE110(__SHIFT)
-};
-
-static const struct dce_dmcu_mask dmcu_mask = {
-		DMCU_MASK_SH_LIST_DCE110(_MASK)
-};
-
-static const struct dce_abm_registers abm_regs = {
-		ABM_DCE110_COMMON_REG_LIST()
-};
-
-static const struct dce_abm_shift abm_shift = {
-		ABM_MASK_SH_LIST_DCE110(__SHIFT)
-};
-
-static const struct dce_abm_mask abm_mask = {
-		ABM_MASK_SH_LIST_DCE110(_MASK)
-};
-
-#define ipp_regs(id)\
-[id] = {\
-		IPP_DCE110_REG_LIST_DCE_BASE(id)\
-}
-
-static const struct dce_ipp_registers ipp_regs[] = {
-		ipp_regs(0),
-		ipp_regs(1),
-		ipp_regs(2),
-		ipp_regs(3),
-		ipp_regs(4),
-		ipp_regs(5)
-};
-
-static const struct dce_ipp_shift ipp_shift = {
-		IPP_DCE120_MASK_SH_LIST_SOC_BASE(__SHIFT)
-};
-
-static const struct dce_ipp_mask ipp_mask = {
-		IPP_DCE120_MASK_SH_LIST_SOC_BASE(_MASK)
-};
-
-#define transform_regs(id)\
-[id] = {\
-		XFM_COMMON_REG_LIST_DCE110(id)\
-}
-
-static const struct dce_transform_registers xfm_regs[] = {
-		transform_regs(0),
-		transform_regs(1),
-		transform_regs(2),
-		transform_regs(3),
-		transform_regs(4),
-		transform_regs(5)
-};
-
-static const struct dce_transform_shift xfm_shift = {
-		XFM_COMMON_MASK_SH_LIST_SOC_BASE(__SHIFT)
-};
-
-static const struct dce_transform_mask xfm_mask = {
-		XFM_COMMON_MASK_SH_LIST_SOC_BASE(_MASK)
-};
-
-#define aux_regs(id)\
-[id] = {\
-	AUX_REG_LIST(id)\
-}
-
-static const struct dce110_link_enc_aux_registers link_enc_aux_regs[] = {
-		aux_regs(0),
-		aux_regs(1),
-		aux_regs(2),
-		aux_regs(3),
-		aux_regs(4),
-		aux_regs(5)
-};
-
-#define hpd_regs(id)\
-[id] = {\
-	HPD_REG_LIST(id)\
-}
-
-static const struct dce110_link_enc_hpd_registers link_enc_hpd_regs[] = {
-		hpd_regs(0),
-		hpd_regs(1),
-		hpd_regs(2),
-		hpd_regs(3),
-		hpd_regs(4),
-		hpd_regs(5)
-};
-
-#define link_regs(id)\
-[id] = {\
-	LE_DCE120_REG_LIST(id), \
-	SRI(DP_DPHY_INTERNAL_CTRL, DP, id) \
-}
-
-static const struct dce110_link_enc_registers link_enc_regs[] = {
-	link_regs(0),
-	link_regs(1),
-	link_regs(2),
-	link_regs(3),
-	link_regs(4),
-	link_regs(5),
-	link_regs(6),
-};
-
-
-#define stream_enc_regs(id)\
-[id] = {\
-	SE_COMMON_REG_LIST(id),\
-	.TMDS_CNTL = 0,\
-}
-
-static const struct dce110_stream_enc_registers stream_enc_regs[] = {
-	stream_enc_regs(0),
-	stream_enc_regs(1),
-	stream_enc_regs(2),
-	stream_enc_regs(3),
-	stream_enc_regs(4),
-	stream_enc_regs(5)
-};
-
-static const struct dce_stream_encoder_shift se_shift = {
-		SE_COMMON_MASK_SH_LIST_DCE120(__SHIFT)
-};
-
-static const struct dce_stream_encoder_mask se_mask = {
-		SE_COMMON_MASK_SH_LIST_DCE120(_MASK)
-};
-
-static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
-	{ DCE_PANEL_CNTL_REG_LIST() }
-};
-
-static const struct dce_panel_cntl_shift panel_cntl_shift = {
-	DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_panel_cntl_mask panel_cntl_mask = {
-	DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
-};
-
-static const struct dce110_aux_registers_shift aux_shift = {
-	DCE12_AUX_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce110_aux_registers_mask aux_mask = {
-	DCE12_AUX_MASK_SH_LIST(_MASK)
-};
-
-#define opp_regs(id)\
-[id] = {\
-	OPP_DCE_120_REG_LIST(id),\
-}
-
-static const struct dce_opp_registers opp_regs[] = {
-	opp_regs(0),
-	opp_regs(1),
-	opp_regs(2),
-	opp_regs(3),
-	opp_regs(4),
-	opp_regs(5)
-};
-
-static const struct dce_opp_shift opp_shift = {
-	OPP_COMMON_MASK_SH_LIST_DCE_120(__SHIFT)
-};
-
-static const struct dce_opp_mask opp_mask = {
-	OPP_COMMON_MASK_SH_LIST_DCE_120(_MASK)
-};
- #define aux_engine_regs(id)\
-[id] = {\
-	AUX_COMMON_REG_LIST(id), \
-	.AUX_RESET_MASK = 0 \
-}
-
-static const struct dce110_aux_registers aux_engine_regs[] = {
-		aux_engine_regs(0),
-		aux_engine_regs(1),
-		aux_engine_regs(2),
-		aux_engine_regs(3),
-		aux_engine_regs(4),
-		aux_engine_regs(5)
-};
-
-#define audio_regs(id)\
-[id] = {\
-	AUD_COMMON_REG_LIST(id)\
-}
-
-static const struct dce_audio_registers audio_regs[] = {
-	audio_regs(0),
-	audio_regs(1),
-	audio_regs(2),
-	audio_regs(3),
-	audio_regs(4),
-	audio_regs(5),
-	audio_regs(6),
-};
-
-#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
-		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
-
-static const struct dce_audio_shift audio_shift = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_audio_mask audio_mask = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
-};
-
-static int map_transmitter_id_to_phy_instance(
-	enum transmitter transmitter)
-{
-	switch (transmitter) {
-	case TRANSMITTER_UNIPHY_A:
-		return 0;
-	case TRANSMITTER_UNIPHY_B:
-		return 1;
-	case TRANSMITTER_UNIPHY_C:
-		return 2;
-	case TRANSMITTER_UNIPHY_D:
-		return 3;
-	case TRANSMITTER_UNIPHY_E:
-		return 4;
-	case TRANSMITTER_UNIPHY_F:
-		return 5;
-	case TRANSMITTER_UNIPHY_G:
-		return 6;
-	default:
-		ASSERT(0);
-		return 0;
-	}
-}
-
-#define clk_src_regs(index, id)\
-[index] = {\
-	CS_COMMON_REG_LIST_DCE_112(id),\
-}
-
-static const struct dce110_clk_src_regs clk_src_regs[] = {
-	clk_src_regs(0, A),
-	clk_src_regs(1, B),
-	clk_src_regs(2, C),
-	clk_src_regs(3, D),
-	clk_src_regs(4, E),
-	clk_src_regs(5, F)
-};
-
-static const struct dce110_clk_src_shift cs_shift = {
-		CS_COMMON_MASK_SH_LIST_DCE_112(__SHIFT)
-};
-
-static const struct dce110_clk_src_mask cs_mask = {
-		CS_COMMON_MASK_SH_LIST_DCE_112(_MASK)
-};
-
-static struct output_pixel_processor *dce120_opp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce110_opp *opp =
-		kzalloc(sizeof(struct dce110_opp), GFP_KERNEL);
-
-	if (!opp)
-		return NULL;
-
-	dce110_opp_construct(opp,
-			     ctx, inst, &opp_regs[inst], &opp_shift, &opp_mask);
-	return &opp->base;
-}
-static struct dce_aux *dce120_aux_engine_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct aux_engine_dce110 *aux_engine =
-		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
-
-	if (!aux_engine)
-		return NULL;
-
-	dce110_aux_engine_construct(aux_engine, ctx, inst,
-				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
-				    &aux_engine_regs[inst],
-					&aux_mask,
-					&aux_shift,
-					ctx->dc->caps.extended_aux_timeout_support);
-
-	return &aux_engine->base;
-}
-#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
-
-static const struct dce_i2c_registers i2c_hw_regs[] = {
-		i2c_inst_regs(1),
-		i2c_inst_regs(2),
-		i2c_inst_regs(3),
-		i2c_inst_regs(4),
-		i2c_inst_regs(5),
-		i2c_inst_regs(6),
-};
-
-static const struct dce_i2c_shift i2c_shifts = {
-		I2C_COMMON_MASK_SH_LIST_DCE110(__SHIFT)
-};
-
-static const struct dce_i2c_mask i2c_masks = {
-		I2C_COMMON_MASK_SH_LIST_DCE110(_MASK)
-};
-
-static struct dce_i2c_hw *dce120_i2c_hw_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_i2c_hw *dce_i2c_hw =
-		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
-
-	if (!dce_i2c_hw)
-		return NULL;
-
-	dce112_i2c_hw_construct(dce_i2c_hw, ctx, inst,
-				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
-
-	return dce_i2c_hw;
-}
-static const struct bios_registers bios_regs = {
-	.BIOS_SCRATCH_3 = mmBIOS_SCRATCH_3 + NBIO_BASE(mmBIOS_SCRATCH_3_BASE_IDX),
-	.BIOS_SCRATCH_6 = mmBIOS_SCRATCH_6 + NBIO_BASE(mmBIOS_SCRATCH_6_BASE_IDX)
-};
-
-static const struct resource_caps res_cap = {
-		.num_timing_generator = 6,
-		.num_audio = 7,
-		.num_stream_encoder = 6,
-		.num_pll = 6,
-		.num_ddc = 6,
-};
-
-static const struct dc_plane_cap plane_cap = {
-	.type = DC_PLANE_TYPE_DCE_RGB,
-
-	.pixel_format_support = {
-			.argb8888 = true,
-			.nv12 = false,
-			.fp16 = true
-	},
-
-	.max_upscale_factor = {
-			.argb8888 = 16000,
-			.nv12 = 1,
-			.fp16 = 1
-	},
-
-	.max_downscale_factor = {
-			.argb8888 = 250,
-			.nv12 = 1,
-			.fp16 = 1
-	}
-};
-
-static const struct dc_debug_options debug_defaults = {
-		.disable_clock_gate = true,
-		.enable_legacy_fast_update = true,
-};
-
-static struct clock_source *dce120_clock_source_create(
-	struct dc_context *ctx,
-	struct dc_bios *bios,
-	enum clock_source_id id,
-	const struct dce110_clk_src_regs *regs,
-	bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src =
-		kzalloc(sizeof(*clk_src), GFP_KERNEL);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dce112_clk_src_construct(clk_src, ctx, bios, id,
-				     regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-
-	kfree(clk_src);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
-static void dce120_clock_source_destroy(struct clock_source **clk_src)
-{
-	kfree(TO_DCE110_CLK_SRC(*clk_src));
-	*clk_src = NULL;
-}
-
-
-static bool dce120_hw_sequencer_create(struct dc *dc)
-{
-	/* All registers used by dce11.2 match those in dce11 in offset and
-	 * structure
-	 */
-	dce120_hw_sequencer_construct(dc);
-
-	/*TODO	Move to separate file and Override what is needed */
-
-	return true;
-}
-
-static struct timing_generator *dce120_timing_generator_create(
-		struct dc_context *ctx,
-		uint32_t instance,
-		const struct dce110_timing_generator_offsets *offsets)
-{
-	struct dce110_timing_generator *tg110 =
-		kzalloc(sizeof(struct dce110_timing_generator), GFP_KERNEL);
-
-	if (!tg110)
-		return NULL;
-
-	dce120_timing_generator_construct(tg110, ctx, instance, offsets);
-	return &tg110->base;
-}
-
-static void dce120_transform_destroy(struct transform **xfm)
-{
-	kfree(TO_DCE_TRANSFORM(*xfm));
-	*xfm = NULL;
-}
-
-static void dce120_resource_destruct(struct dce110_resource_pool *pool)
-{
-	unsigned int i;
-
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		if (pool->base.opps[i] != NULL)
-			dce110_opp_destroy(&pool->base.opps[i]);
-
-		if (pool->base.transforms[i] != NULL)
-			dce120_transform_destroy(&pool->base.transforms[i]);
-
-		if (pool->base.ipps[i] != NULL)
-			dce_ipp_destroy(&pool->base.ipps[i]);
-
-		if (pool->base.mis[i] != NULL) {
-			kfree(TO_DCE_MEM_INPUT(pool->base.mis[i]));
-			pool->base.mis[i] = NULL;
-		}
-
-		if (pool->base.irqs != NULL) {
-			dal_irq_service_destroy(&pool->base.irqs);
-		}
-
-		if (pool->base.timing_generators[i] != NULL) {
-			kfree(DCE110TG_FROM_TG(pool->base.timing_generators[i]));
-			pool->base.timing_generators[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		if (pool->base.engines[i] != NULL)
-			dce110_engine_destroy(&pool->base.engines[i]);
-		if (pool->base.hw_i2cs[i] != NULL) {
-			kfree(pool->base.hw_i2cs[i]);
-			pool->base.hw_i2cs[i] = NULL;
-		}
-		if (pool->base.sw_i2cs[i] != NULL) {
-			kfree(pool->base.sw_i2cs[i]);
-			pool->base.sw_i2cs[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.audio_count; i++) {
-		if (pool->base.audios[i])
-			dce_aud_destroy(&pool->base.audios[i]);
-	}
-
-	for (i = 0; i < pool->base.stream_enc_count; i++) {
-		if (pool->base.stream_enc[i] != NULL)
-			kfree(DCE110STRENC_FROM_STRENC(pool->base.stream_enc[i]));
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] != NULL)
-			dce120_clock_source_destroy(
-				&pool->base.clock_sources[i]);
-	}
-
-	if (pool->base.dp_clock_source != NULL)
-		dce120_clock_source_destroy(&pool->base.dp_clock_source);
-
-	if (pool->base.abm != NULL)
-		dce_abm_destroy(&pool->base.abm);
-
-	if (pool->base.dmcu != NULL)
-		dce_dmcu_destroy(&pool->base.dmcu);
-}
-
-static void read_dce_straps(
-	struct dc_context *ctx,
-	struct resource_straps *straps)
-{
-	uint32_t reg_val = dm_read_reg_soc15(ctx, mmCC_DC_MISC_STRAPS, 0);
-
-	straps->audio_stream_number = get_reg_field_value(reg_val,
-							  CC_DC_MISC_STRAPS,
-							  AUDIO_STREAM_NUMBER);
-	straps->hdmi_disable = get_reg_field_value(reg_val,
-						   CC_DC_MISC_STRAPS,
-						   HDMI_DISABLE);
-
-	reg_val = dm_read_reg_soc15(ctx, mmDC_PINSTRAPS, 0);
-	straps->dc_pinstraps_audio = get_reg_field_value(reg_val,
-							 DC_PINSTRAPS,
-							 DC_PINSTRAPS_AUDIO);
-}
-
-static struct audio *create_audio(
-		struct dc_context *ctx, unsigned int inst)
-{
-	return dce_audio_create(ctx, inst,
-			&audio_regs[inst], &audio_shift, &audio_mask);
-}
-
-static const struct encoder_feature_support link_enc_feature = {
-		.max_hdmi_deep_color = COLOR_DEPTH_121212,
-		.max_hdmi_pixel_clock = 600000,
-		.hdmi_ycbcr420_supported = true,
-		.dp_ycbcr420_supported = false,
-		.flags.bits.IS_HBR2_CAPABLE = true,
-		.flags.bits.IS_HBR3_CAPABLE = true,
-		.flags.bits.IS_TPS3_CAPABLE = true,
-		.flags.bits.IS_TPS4_CAPABLE = true,
-};
-
-static struct link_encoder *dce120_link_encoder_create(
-	struct dc_context *ctx,
-	const struct encoder_init_data *enc_init_data)
-{
-	struct dce110_link_encoder *enc110 =
-		kzalloc(sizeof(struct dce110_link_encoder), GFP_KERNEL);
-	int link_regs_id;
-
-	if (!enc110)
-		return NULL;
-
-	link_regs_id =
-		map_transmitter_id_to_phy_instance(enc_init_data->transmitter);
-
-	dce110_link_encoder_construct(enc110,
-				      enc_init_data,
-				      &link_enc_feature,
-				      &link_enc_regs[link_regs_id],
-				      &link_enc_aux_regs[enc_init_data->channel - 1],
-				      &link_enc_hpd_regs[enc_init_data->hpd_source]);
-
-	return &enc110->base;
-}
-
-static struct panel_cntl *dce120_panel_cntl_create(const struct panel_cntl_init_data *init_data)
-{
-	struct dce_panel_cntl *panel_cntl =
-		kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
-
-	if (!panel_cntl)
-		return NULL;
-
-	dce_panel_cntl_construct(panel_cntl,
-			init_data,
-			&panel_cntl_regs[init_data->inst],
-			&panel_cntl_shift,
-			&panel_cntl_mask);
-
-	return &panel_cntl->base;
-}
-
-static struct input_pixel_processor *dce120_ipp_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dce_ipp *ipp = kzalloc(sizeof(struct dce_ipp), GFP_KERNEL);
-
-	if (!ipp) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dce_ipp_construct(ipp, ctx, inst,
-			&ipp_regs[inst], &ipp_shift, &ipp_mask);
-	return &ipp->base;
-}
-
-static struct stream_encoder *dce120_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dce110_stream_encoder *enc110 =
-		kzalloc(sizeof(struct dce110_stream_encoder), GFP_KERNEL);
-
-	if (!enc110)
-		return NULL;
-
-	dce110_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id,
-					&stream_enc_regs[eng_id],
-					&se_shift, &se_mask);
-	return &enc110->base;
-}
-
-#define SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-static const struct dce_hwseq_registers hwseq_reg = {
-		HWSEQ_DCE120_REG_LIST()
-};
-
-static const struct dce_hwseq_shift hwseq_shift = {
-		HWSEQ_DCE12_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_hwseq_mask hwseq_mask = {
-		HWSEQ_DCE12_MASK_SH_LIST(_MASK)
-};
-
-/* HWSEQ regs for VG20 */
-static const struct dce_hwseq_registers dce121_hwseq_reg = {
-		HWSEQ_VG20_REG_LIST()
-};
-
-static const struct dce_hwseq_shift dce121_hwseq_shift = {
-		HWSEQ_VG20_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_hwseq_mask dce121_hwseq_mask = {
-		HWSEQ_VG20_MASK_SH_LIST(_MASK)
-};
-
-static struct dce_hwseq *dce120_hwseq_create(
-	struct dc_context *ctx)
-{
-	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
-
-	if (hws) {
-		hws->ctx = ctx;
-		hws->regs = &hwseq_reg;
-		hws->shifts = &hwseq_shift;
-		hws->masks = &hwseq_mask;
-	}
-	return hws;
-}
-
-static struct dce_hwseq *dce121_hwseq_create(
-	struct dc_context *ctx)
-{
-	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
-
-	if (hws) {
-		hws->ctx = ctx;
-		hws->regs = &dce121_hwseq_reg;
-		hws->shifts = &dce121_hwseq_shift;
-		hws->masks = &dce121_hwseq_mask;
-	}
-	return hws;
-}
-
-static const struct resource_create_funcs res_create_funcs = {
-	.read_dce_straps = read_dce_straps,
-	.create_audio = create_audio,
-	.create_stream_encoder = dce120_stream_encoder_create,
-	.create_hwseq = dce120_hwseq_create,
-};
-
-static const struct resource_create_funcs dce121_res_create_funcs = {
-	.read_dce_straps = read_dce_straps,
-	.create_audio = create_audio,
-	.create_stream_encoder = dce120_stream_encoder_create,
-	.create_hwseq = dce121_hwseq_create,
-};
-
-
-#define mi_inst_regs(id) { MI_DCE12_REG_LIST(id) }
-static const struct dce_mem_input_registers mi_regs[] = {
-		mi_inst_regs(0),
-		mi_inst_regs(1),
-		mi_inst_regs(2),
-		mi_inst_regs(3),
-		mi_inst_regs(4),
-		mi_inst_regs(5),
-};
-
-static const struct dce_mem_input_shift mi_shifts = {
-		MI_DCE12_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_mem_input_mask mi_masks = {
-		MI_DCE12_MASK_SH_LIST(_MASK)
-};
-
-static struct mem_input *dce120_mem_input_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_mem_input *dce_mi = kzalloc(sizeof(struct dce_mem_input),
-					       GFP_KERNEL);
-
-	if (!dce_mi) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dce120_mem_input_construct(dce_mi, ctx, inst, &mi_regs[inst], &mi_shifts, &mi_masks);
-	return &dce_mi->base;
-}
-
-static struct transform *dce120_transform_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_transform *transform =
-		kzalloc(sizeof(struct dce_transform), GFP_KERNEL);
-
-	if (!transform)
-		return NULL;
-
-	dce_transform_construct(transform, ctx, inst,
-				&xfm_regs[inst], &xfm_shift, &xfm_mask);
-	transform->lb_memory_size = 0x1404; /*5124*/
-	return &transform->base;
-}
-
-static void dce120_destroy_resource_pool(struct resource_pool **pool)
-{
-	struct dce110_resource_pool *dce110_pool = TO_DCE110_RES_POOL(*pool);
-
-	dce120_resource_destruct(dce110_pool);
-	kfree(dce110_pool);
-	*pool = NULL;
-}
-
-static const struct resource_funcs dce120_res_pool_funcs = {
-	.destroy = dce120_destroy_resource_pool,
-	.link_enc_create = dce120_link_encoder_create,
-	.panel_cntl_create = dce120_panel_cntl_create,
-	.validate_bandwidth = dce112_validate_bandwidth,
-	.validate_plane = dce100_validate_plane,
-	.add_stream_to_ctx = dce112_add_stream_to_ctx,
-	.find_first_free_match_stream_enc_for_link = dce110_find_first_free_match_stream_enc_for_link
-};
-
-static void bw_calcs_data_update_from_pplib(struct dc *dc)
-{
-	struct dm_pp_clock_levels_with_latency eng_clks = {0};
-	struct dm_pp_clock_levels_with_latency mem_clks = {0};
-	struct dm_pp_wm_sets_with_clock_ranges clk_ranges = {0};
-	int i;
-	unsigned int clk;
-	unsigned int latency;
-	/*original logic in dal3*/
-	int memory_type_multiplier = MEMORY_TYPE_MULTIPLIER_CZ;
-
-	/*do system clock*/
-	if (!dm_pp_get_clock_levels_by_type_with_latency(
-				dc->ctx,
-				DM_PP_CLOCK_TYPE_ENGINE_CLK,
-				&eng_clks) || eng_clks.num_levels == 0) {
-
-		eng_clks.num_levels = 8;
-		clk = 300000;
-
-		for (i = 0; i < eng_clks.num_levels; i++) {
-			eng_clks.data[i].clocks_in_khz = clk;
-			clk += 100000;
-		}
-	}
-
-	/* convert all the clock fro kHz to fix point mHz  TODO: wloop data */
-	dc->bw_vbios->high_sclk = bw_frc_to_fixed(
-		eng_clks.data[eng_clks.num_levels-1].clocks_in_khz, 1000);
-	dc->bw_vbios->mid1_sclk  = bw_frc_to_fixed(
-		eng_clks.data[eng_clks.num_levels/8].clocks_in_khz, 1000);
-	dc->bw_vbios->mid2_sclk  = bw_frc_to_fixed(
-		eng_clks.data[eng_clks.num_levels*2/8].clocks_in_khz, 1000);
-	dc->bw_vbios->mid3_sclk  = bw_frc_to_fixed(
-		eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz, 1000);
-	dc->bw_vbios->mid4_sclk  = bw_frc_to_fixed(
-		eng_clks.data[eng_clks.num_levels*4/8].clocks_in_khz, 1000);
-	dc->bw_vbios->mid5_sclk  = bw_frc_to_fixed(
-		eng_clks.data[eng_clks.num_levels*5/8].clocks_in_khz, 1000);
-	dc->bw_vbios->mid6_sclk  = bw_frc_to_fixed(
-		eng_clks.data[eng_clks.num_levels*6/8].clocks_in_khz, 1000);
-	dc->bw_vbios->low_sclk  = bw_frc_to_fixed(
-			eng_clks.data[0].clocks_in_khz, 1000);
-
-	/*do memory clock*/
-	if (!dm_pp_get_clock_levels_by_type_with_latency(
-			dc->ctx,
-			DM_PP_CLOCK_TYPE_MEMORY_CLK,
-			&mem_clks) || mem_clks.num_levels == 0) {
-
-		mem_clks.num_levels = 3;
-		clk = 250000;
-		latency = 45;
-
-		for (i = 0; i < eng_clks.num_levels; i++) {
-			mem_clks.data[i].clocks_in_khz = clk;
-			mem_clks.data[i].latency_in_us = latency;
-			clk += 500000;
-			latency -= 5;
-		}
-
-	}
-
-	/* we don't need to call PPLIB for validation clock since they
-	 * also give us the highest sclk and highest mclk (UMA clock).
-	 * ALSO always convert UMA clock (from PPLIB)  to YCLK (HW formula):
-	 * YCLK = UMACLK*m_memoryTypeMultiplier
-	 */
-	if (dc->bw_vbios->memory_type == bw_def_hbm)
-		memory_type_multiplier = MEMORY_TYPE_HBM;
-
-	dc->bw_vbios->low_yclk = bw_frc_to_fixed(
-		mem_clks.data[0].clocks_in_khz * memory_type_multiplier, 1000);
-	dc->bw_vbios->mid_yclk = bw_frc_to_fixed(
-		mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz * memory_type_multiplier,
-		1000);
-	dc->bw_vbios->high_yclk = bw_frc_to_fixed(
-		mem_clks.data[mem_clks.num_levels-1].clocks_in_khz * memory_type_multiplier,
-		1000);
-
-	/* Now notify PPLib/SMU about which Watermarks sets they should select
-	 * depending on DPM state they are in. And update BW MGR GFX Engine and
-	 * Memory clock member variables for Watermarks calculations for each
-	 * Watermark Set
-	 */
-	clk_ranges.num_wm_sets = 4;
-	clk_ranges.wm_clk_ranges[0].wm_set_id = WM_SET_A;
-	clk_ranges.wm_clk_ranges[0].wm_min_eng_clk_in_khz =
-			eng_clks.data[0].clocks_in_khz;
-	clk_ranges.wm_clk_ranges[0].wm_max_eng_clk_in_khz =
-			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz - 1;
-	clk_ranges.wm_clk_ranges[0].wm_min_mem_clk_in_khz =
-			mem_clks.data[0].clocks_in_khz;
-	clk_ranges.wm_clk_ranges[0].wm_max_mem_clk_in_khz =
-			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz - 1;
-
-	clk_ranges.wm_clk_ranges[1].wm_set_id = WM_SET_B;
-	clk_ranges.wm_clk_ranges[1].wm_min_eng_clk_in_khz =
-			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz;
-	/* 5 GHz instead of data[7].clockInKHz to cover Overdrive */
-	clk_ranges.wm_clk_ranges[1].wm_max_eng_clk_in_khz = 5000000;
-	clk_ranges.wm_clk_ranges[1].wm_min_mem_clk_in_khz =
-			mem_clks.data[0].clocks_in_khz;
-	clk_ranges.wm_clk_ranges[1].wm_max_mem_clk_in_khz =
-			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz - 1;
-
-	clk_ranges.wm_clk_ranges[2].wm_set_id = WM_SET_C;
-	clk_ranges.wm_clk_ranges[2].wm_min_eng_clk_in_khz =
-			eng_clks.data[0].clocks_in_khz;
-	clk_ranges.wm_clk_ranges[2].wm_max_eng_clk_in_khz =
-			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz - 1;
-	clk_ranges.wm_clk_ranges[2].wm_min_mem_clk_in_khz =
-			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz;
-	/* 5 GHz instead of data[2].clockInKHz to cover Overdrive */
-	clk_ranges.wm_clk_ranges[2].wm_max_mem_clk_in_khz = 5000000;
-
-	clk_ranges.wm_clk_ranges[3].wm_set_id = WM_SET_D;
-	clk_ranges.wm_clk_ranges[3].wm_min_eng_clk_in_khz =
-			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz;
-	/* 5 GHz instead of data[7].clockInKHz to cover Overdrive */
-	clk_ranges.wm_clk_ranges[3].wm_max_eng_clk_in_khz = 5000000;
-	clk_ranges.wm_clk_ranges[3].wm_min_mem_clk_in_khz =
-			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz;
-	/* 5 GHz instead of data[2].clockInKHz to cover Overdrive */
-	clk_ranges.wm_clk_ranges[3].wm_max_mem_clk_in_khz = 5000000;
-
-	/* Notify PP Lib/SMU which Watermarks to use for which clock ranges */
-	dm_pp_notify_wm_clock_changes(dc->ctx, &clk_ranges);
-}
-
-static uint32_t read_pipe_fuses(struct dc_context *ctx)
-{
-	uint32_t value = dm_read_reg_soc15(ctx, mmCC_DC_PIPE_DIS, 0);
-	/* VG20 support max 6 pipes */
-	value = value & 0x3f;
-	return value;
-}
-
-static bool dce120_resource_construct(
-	uint8_t num_virtual_links,
-	struct dc *dc,
-	struct dce110_resource_pool *pool)
-{
-	unsigned int i;
-	int j;
-	struct dc_context *ctx = dc->ctx;
-	struct irq_service_init_data irq_init_data;
-	static const struct resource_create_funcs *res_funcs;
-	bool is_vg20 = ASICREV_IS_VEGA20_P(ctx->asic_id.hw_internal_rev);
-	uint32_t pipe_fuses;
-
-	ctx->dc_bios->regs = &bios_regs;
-
-	pool->base.res_cap = &res_cap;
-	pool->base.funcs = &dce120_res_pool_funcs;
-
-	/* TODO: Fill more data from GreenlandAsicCapability.cpp */
-	pool->base.pipe_count = res_cap.num_timing_generator;
-	pool->base.timing_generator_count = pool->base.res_cap->num_timing_generator;
-	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
-
-	dc->caps.max_downscale_ratio = 200;
-	dc->caps.i2c_speed_in_khz = 100;
-	dc->caps.i2c_speed_in_khz_hdcp = 100; /*1.4 w/a not applied by default*/
-	dc->caps.max_cursor_size = 128;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.dual_link_dvi = true;
-	dc->caps.psp_setup_panel_mode = true;
-	dc->caps.extended_aux_timeout_support = false;
-	dc->debug = debug_defaults;
-
-	/*************************************************
-	 *  Create resources                             *
-	 *************************************************/
-
-	pool->base.clock_sources[DCE120_CLK_SRC_PLL0] =
-			dce120_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL0,
-				&clk_src_regs[0], false);
-	pool->base.clock_sources[DCE120_CLK_SRC_PLL1] =
-			dce120_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL1,
-				&clk_src_regs[1], false);
-	pool->base.clock_sources[DCE120_CLK_SRC_PLL2] =
-			dce120_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL2,
-				&clk_src_regs[2], false);
-	pool->base.clock_sources[DCE120_CLK_SRC_PLL3] =
-			dce120_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL3,
-				&clk_src_regs[3], false);
-	pool->base.clock_sources[DCE120_CLK_SRC_PLL4] =
-			dce120_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL4,
-				&clk_src_regs[4], false);
-	pool->base.clock_sources[DCE120_CLK_SRC_PLL5] =
-			dce120_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL5,
-				&clk_src_regs[5], false);
-	pool->base.clk_src_count = DCE120_CLK_SRC_TOTAL;
-
-	pool->base.dp_clock_source =
-			dce120_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_ID_DP_DTO,
-				&clk_src_regs[0], true);
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			BREAK_TO_DEBUGGER();
-			goto clk_src_create_fail;
-		}
-	}
-
-	pool->base.dmcu = dce_dmcu_create(ctx,
-			&dmcu_regs,
-			&dmcu_shift,
-			&dmcu_mask);
-	if (pool->base.dmcu == NULL) {
-		dm_error("DC: failed to create dmcu!\n");
-		BREAK_TO_DEBUGGER();
-		goto res_create_fail;
-	}
-
-	pool->base.abm = dce_abm_create(ctx,
-			&abm_regs,
-			&abm_shift,
-			&abm_mask);
-	if (pool->base.abm == NULL) {
-		dm_error("DC: failed to create abm!\n");
-		BREAK_TO_DEBUGGER();
-		goto res_create_fail;
-	}
-
-
-	irq_init_data.ctx = dc->ctx;
-	pool->base.irqs = dal_irq_service_dce120_create(&irq_init_data);
-	if (!pool->base.irqs)
-		goto irqs_create_fail;
-
-	/* VG20: Pipe harvesting enabled, retrieve valid pipe fuses */
-	if (is_vg20)
-		pipe_fuses = read_pipe_fuses(ctx);
-
-	/* index to valid pipe resource */
-	j = 0;
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		if (is_vg20) {
-			if ((pipe_fuses & (1 << i)) != 0) {
-				dm_error("DC: skip invalid pipe %d!\n", i);
-				continue;
-			}
-		}
-
-		pool->base.timing_generators[j] =
-				dce120_timing_generator_create(
-					ctx,
-					i,
-					&dce120_tg_offsets[i]);
-		if (pool->base.timing_generators[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create tg!\n");
-			goto controller_create_fail;
-		}
-
-		pool->base.mis[j] = dce120_mem_input_create(ctx, i);
-
-		if (pool->base.mis[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create memory input!\n");
-			goto controller_create_fail;
-		}
-
-		pool->base.ipps[j] = dce120_ipp_create(ctx, i);
-		if (pool->base.ipps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create input pixel processor!\n");
-			goto controller_create_fail;
-		}
-
-		pool->base.transforms[j] = dce120_transform_create(ctx, i);
-		if (pool->base.transforms[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create transform!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.opps[j] = dce120_opp_create(
-			ctx,
-			i);
-		if (pool->base.opps[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create output pixel processor!\n");
-		}
-
-		/* check next valid pipe */
-		j++;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		pool->base.engines[i] = dce120_aux_engine_create(ctx, i);
-		if (pool->base.engines[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create aux engine!!\n");
-			goto res_create_fail;
-		}
-		pool->base.hw_i2cs[i] = dce120_i2c_hw_create(ctx, i);
-		if (pool->base.hw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create i2c engine!!\n");
-			goto res_create_fail;
-		}
-		pool->base.sw_i2cs[i] = NULL;
-	}
-
-	/* valid pipe num */
-	pool->base.pipe_count = j;
-	pool->base.timing_generator_count = j;
-
-	if (is_vg20)
-		res_funcs = &dce121_res_create_funcs;
-	else
-		res_funcs = &res_create_funcs;
-
-	if (!resource_construct(num_virtual_links, dc, &pool->base, res_funcs))
-		goto res_create_fail;
-
-	/* Create hardware sequencer */
-	if (!dce120_hw_sequencer_create(dc))
-		goto controller_create_fail;
-
-	dc->caps.max_planes =  pool->base.pipe_count;
-
-	for (i = 0; i < dc->caps.max_planes; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	bw_calcs_init(dc->bw_dceip, dc->bw_vbios, dc->ctx->asic_id);
-
-	bw_calcs_data_update_from_pplib(dc);
-
-	return true;
-
-irqs_create_fail:
-controller_create_fail:
-clk_src_create_fail:
-res_create_fail:
-
-	dce120_resource_destruct(pool);
-
-	return false;
-}
-
-struct resource_pool *dce120_create_resource_pool(
-	uint8_t num_virtual_links,
-	struct dc *dc)
-{
-	struct dce110_resource_pool *pool =
-		kzalloc(sizeof(struct dce110_resource_pool), GFP_KERNEL);
-
-	if (!pool)
-		return NULL;
-
-	if (dce120_resource_construct(num_virtual_links, dc, pool))
-		return &pool->base;
-
-	kfree(pool);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.h b/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.h
deleted file mode 100644
index 3d1f3cf012f4..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
-* Copyright 2012-15 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#ifndef __DC_RESOURCE_DCE120_H__
-#define __DC_RESOURCE_DCE120_H__
-
-#include "core_types.h"
-
-struct dc;
-struct resource_pool;
-
-struct resource_pool *dce120_create_resource_pool(
-	uint8_t num_virtual_links,
-	struct dc *dc);
-
-#endif /* __DC_RESOURCE_DCE120_H__ */
-
diff --git a/drivers/gpu/drm/amd/display/dc/dce80/Makefile b/drivers/gpu/drm/amd/display/dc/dce80/Makefile
index 93dd68c31275..7eefffbdc925 100644
--- a/drivers/gpu/drm/amd/display/dc/dce80/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dce80/Makefile
@@ -25,8 +25,7 @@
 
 CFLAGS_$(AMDDALPATH)/dc/dce80/dce80_resource.o = $(call cc-disable-warning, override-init)
 
-DCE80 = dce80_timing_generator.o \
-	dce80_resource.o
+DCE80 = dce80_timing_generator.o
 
 AMD_DAL_DCE80 = $(addprefix $(AMDDALPATH)/dc/dce80/,$(DCE80))
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c b/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c
deleted file mode 100644
index 35a2cce0c2b8..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c
+++ /dev/null
@@ -1,1544 +0,0 @@
-/*
- * Copyright 2012-15 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#include "dce/dce_8_0_d.h"
-#include "dce/dce_8_0_sh_mask.h"
-
-#include "dm_services.h"
-
-#include "link_encoder.h"
-#include "stream_encoder.h"
-
-#include "resource.h"
-#include "include/irq_service_interface.h"
-#include "irq/dce80/irq_service_dce80.h"
-#include "dce110/dce110_timing_generator.h"
-#include "dce110/dce110_resource.h"
-#include "dce80/dce80_timing_generator.h"
-#include "dce/dce_mem_input.h"
-#include "dce/dce_link_encoder.h"
-#include "dce/dce_stream_encoder.h"
-#include "dce/dce_ipp.h"
-#include "dce/dce_transform.h"
-#include "dce/dce_opp.h"
-#include "dce/dce_clock_source.h"
-#include "dce/dce_audio.h"
-#include "dce/dce_hwseq.h"
-#include "dce80/dce80_hwseq.h"
-#include "dce100/dce100_resource.h"
-#include "dce/dce_panel_cntl.h"
-
-#include "reg_helper.h"
-
-#include "dce/dce_dmcu.h"
-#include "dce/dce_aux.h"
-#include "dce/dce_abm.h"
-#include "dce/dce_i2c.h"
-/* TODO remove this include */
-
-#ifndef mmMC_HUB_RDREQ_DMIF_LIMIT
-#include "gmc/gmc_7_1_d.h"
-#include "gmc/gmc_7_1_sh_mask.h"
-#endif
-
-#include "dce80/dce80_resource.h"
-
-#ifndef mmDP_DPHY_INTERNAL_CTRL
-#define mmDP_DPHY_INTERNAL_CTRL                         0x1CDE
-#define mmDP0_DP_DPHY_INTERNAL_CTRL                     0x1CDE
-#define mmDP1_DP_DPHY_INTERNAL_CTRL                     0x1FDE
-#define mmDP2_DP_DPHY_INTERNAL_CTRL                     0x42DE
-#define mmDP3_DP_DPHY_INTERNAL_CTRL                     0x45DE
-#define mmDP4_DP_DPHY_INTERNAL_CTRL                     0x48DE
-#define mmDP5_DP_DPHY_INTERNAL_CTRL                     0x4BDE
-#define mmDP6_DP_DPHY_INTERNAL_CTRL                     0x4EDE
-#endif
-
-
-#ifndef mmBIOS_SCRATCH_2
-	#define mmBIOS_SCRATCH_2 0x05CB
-	#define mmBIOS_SCRATCH_3 0x05CC
-	#define mmBIOS_SCRATCH_6 0x05CF
-#endif
-
-#ifndef mmDP_DPHY_FAST_TRAINING
-	#define mmDP_DPHY_FAST_TRAINING                         0x1CCE
-	#define mmDP0_DP_DPHY_FAST_TRAINING                     0x1CCE
-	#define mmDP1_DP_DPHY_FAST_TRAINING                     0x1FCE
-	#define mmDP2_DP_DPHY_FAST_TRAINING                     0x42CE
-	#define mmDP3_DP_DPHY_FAST_TRAINING                     0x45CE
-	#define mmDP4_DP_DPHY_FAST_TRAINING                     0x48CE
-	#define mmDP5_DP_DPHY_FAST_TRAINING                     0x4BCE
-	#define mmDP6_DP_DPHY_FAST_TRAINING                     0x4ECE
-#endif
-
-
-#ifndef mmHPD_DC_HPD_CONTROL
-	#define mmHPD_DC_HPD_CONTROL                            0x189A
-	#define mmHPD0_DC_HPD_CONTROL                           0x189A
-	#define mmHPD1_DC_HPD_CONTROL                           0x18A2
-	#define mmHPD2_DC_HPD_CONTROL                           0x18AA
-	#define mmHPD3_DC_HPD_CONTROL                           0x18B2
-	#define mmHPD4_DC_HPD_CONTROL                           0x18BA
-	#define mmHPD5_DC_HPD_CONTROL                           0x18C2
-#endif
-
-#define DCE11_DIG_FE_CNTL 0x4a00
-#define DCE11_DIG_BE_CNTL 0x4a47
-#define DCE11_DP_SEC 0x4ac3
-
-static const struct dce110_timing_generator_offsets dce80_tg_offsets[] = {
-		{
-			.crtc = (mmCRTC0_CRTC_CONTROL - mmCRTC_CONTROL),
-			.dcp =  (mmGRPH_CONTROL - mmGRPH_CONTROL),
-			.dmif = (mmDMIF_PG0_DPG_WATERMARK_MASK_CONTROL
-					- mmDPG_WATERMARK_MASK_CONTROL),
-		},
-		{
-			.crtc = (mmCRTC1_CRTC_CONTROL - mmCRTC_CONTROL),
-			.dcp = (mmDCP1_GRPH_CONTROL - mmGRPH_CONTROL),
-			.dmif = (mmDMIF_PG1_DPG_WATERMARK_MASK_CONTROL
-					- mmDPG_WATERMARK_MASK_CONTROL),
-		},
-		{
-			.crtc = (mmCRTC2_CRTC_CONTROL - mmCRTC_CONTROL),
-			.dcp = (mmDCP2_GRPH_CONTROL - mmGRPH_CONTROL),
-			.dmif = (mmDMIF_PG2_DPG_WATERMARK_MASK_CONTROL
-					- mmDPG_WATERMARK_MASK_CONTROL),
-		},
-		{
-			.crtc = (mmCRTC3_CRTC_CONTROL - mmCRTC_CONTROL),
-			.dcp = (mmDCP3_GRPH_CONTROL - mmGRPH_CONTROL),
-			.dmif = (mmDMIF_PG3_DPG_WATERMARK_MASK_CONTROL
-					- mmDPG_WATERMARK_MASK_CONTROL),
-		},
-		{
-			.crtc = (mmCRTC4_CRTC_CONTROL - mmCRTC_CONTROL),
-			.dcp = (mmDCP4_GRPH_CONTROL - mmGRPH_CONTROL),
-			.dmif = (mmDMIF_PG4_DPG_WATERMARK_MASK_CONTROL
-					- mmDPG_WATERMARK_MASK_CONTROL),
-		},
-		{
-			.crtc = (mmCRTC5_CRTC_CONTROL - mmCRTC_CONTROL),
-			.dcp = (mmDCP5_GRPH_CONTROL - mmGRPH_CONTROL),
-			.dmif = (mmDMIF_PG5_DPG_WATERMARK_MASK_CONTROL
-					- mmDPG_WATERMARK_MASK_CONTROL),
-		}
-};
-
-/* set register offset */
-#define SR(reg_name)\
-	.reg_name = mm ## reg_name
-
-/* set register offset with instance */
-#define SRI(reg_name, block, id)\
-	.reg_name = mm ## block ## id ## _ ## reg_name
-
-#define ipp_regs(id)\
-[id] = {\
-		IPP_COMMON_REG_LIST_DCE_BASE(id)\
-}
-
-static const struct dce_ipp_registers ipp_regs[] = {
-		ipp_regs(0),
-		ipp_regs(1),
-		ipp_regs(2),
-		ipp_regs(3),
-		ipp_regs(4),
-		ipp_regs(5)
-};
-
-static const struct dce_ipp_shift ipp_shift = {
-		IPP_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
-};
-
-static const struct dce_ipp_mask ipp_mask = {
-		IPP_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
-};
-
-#define transform_regs(id)\
-[id] = {\
-		XFM_COMMON_REG_LIST_DCE80(id)\
-}
-
-static const struct dce_transform_registers xfm_regs[] = {
-		transform_regs(0),
-		transform_regs(1),
-		transform_regs(2),
-		transform_regs(3),
-		transform_regs(4),
-		transform_regs(5)
-};
-
-static const struct dce_transform_shift xfm_shift = {
-		XFM_COMMON_MASK_SH_LIST_DCE80(__SHIFT)
-};
-
-static const struct dce_transform_mask xfm_mask = {
-		XFM_COMMON_MASK_SH_LIST_DCE80(_MASK)
-};
-
-#define aux_regs(id)\
-[id] = {\
-	AUX_REG_LIST(id)\
-}
-
-static const struct dce110_link_enc_aux_registers link_enc_aux_regs[] = {
-	aux_regs(0),
-	aux_regs(1),
-	aux_regs(2),
-	aux_regs(3),
-	aux_regs(4),
-	aux_regs(5)
-};
-
-#define hpd_regs(id)\
-[id] = {\
-	HPD_REG_LIST(id)\
-}
-
-static const struct dce110_link_enc_hpd_registers link_enc_hpd_regs[] = {
-		hpd_regs(0),
-		hpd_regs(1),
-		hpd_regs(2),
-		hpd_regs(3),
-		hpd_regs(4),
-		hpd_regs(5)
-};
-
-#define link_regs(id)\
-[id] = {\
-	LE_DCE80_REG_LIST(id)\
-}
-
-static const struct dce110_link_enc_registers link_enc_regs[] = {
-	link_regs(0),
-	link_regs(1),
-	link_regs(2),
-	link_regs(3),
-	link_regs(4),
-	link_regs(5),
-	link_regs(6),
-};
-
-#define stream_enc_regs(id)\
-[id] = {\
-	SE_COMMON_REG_LIST_DCE_BASE(id),\
-	.AFMT_CNTL = 0,\
-}
-
-static const struct dce110_stream_enc_registers stream_enc_regs[] = {
-	stream_enc_regs(0),
-	stream_enc_regs(1),
-	stream_enc_regs(2),
-	stream_enc_regs(3),
-	stream_enc_regs(4),
-	stream_enc_regs(5),
-	stream_enc_regs(6)
-};
-
-static const struct dce_stream_encoder_shift se_shift = {
-		SE_COMMON_MASK_SH_LIST_DCE80_100(__SHIFT)
-};
-
-static const struct dce_stream_encoder_mask se_mask = {
-		SE_COMMON_MASK_SH_LIST_DCE80_100(_MASK)
-};
-
-static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
-	{ DCE_PANEL_CNTL_REG_LIST() }
-};
-
-static const struct dce_panel_cntl_shift panel_cntl_shift = {
-	DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_panel_cntl_mask panel_cntl_mask = {
-	DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
-};
-
-#define opp_regs(id)\
-[id] = {\
-	OPP_DCE_80_REG_LIST(id),\
-}
-
-static const struct dce_opp_registers opp_regs[] = {
-	opp_regs(0),
-	opp_regs(1),
-	opp_regs(2),
-	opp_regs(3),
-	opp_regs(4),
-	opp_regs(5)
-};
-
-static const struct dce_opp_shift opp_shift = {
-	OPP_COMMON_MASK_SH_LIST_DCE_80(__SHIFT)
-};
-
-static const struct dce_opp_mask opp_mask = {
-	OPP_COMMON_MASK_SH_LIST_DCE_80(_MASK)
-};
-
-static const struct dce110_aux_registers_shift aux_shift = {
-	DCE10_AUX_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce110_aux_registers_mask aux_mask = {
-	DCE10_AUX_MASK_SH_LIST(_MASK)
-};
-
-#define aux_engine_regs(id)\
-[id] = {\
-	AUX_COMMON_REG_LIST(id), \
-	.AUX_RESET_MASK = 0 \
-}
-
-static const struct dce110_aux_registers aux_engine_regs[] = {
-		aux_engine_regs(0),
-		aux_engine_regs(1),
-		aux_engine_regs(2),
-		aux_engine_regs(3),
-		aux_engine_regs(4),
-		aux_engine_regs(5)
-};
-
-#define audio_regs(id)\
-[id] = {\
-	AUD_COMMON_REG_LIST(id)\
-}
-
-static const struct dce_audio_registers audio_regs[] = {
-	audio_regs(0),
-	audio_regs(1),
-	audio_regs(2),
-	audio_regs(3),
-	audio_regs(4),
-	audio_regs(5),
-	audio_regs(6),
-};
-
-static const struct dce_audio_shift audio_shift = {
-		AUD_COMMON_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_audio_mask audio_mask = {
-		AUD_COMMON_MASK_SH_LIST(_MASK)
-};
-
-#define clk_src_regs(id)\
-[id] = {\
-	CS_COMMON_REG_LIST_DCE_80(id),\
-}
-
-
-static const struct dce110_clk_src_regs clk_src_regs[] = {
-	clk_src_regs(0),
-	clk_src_regs(1),
-	clk_src_regs(2)
-};
-
-static const struct dce110_clk_src_shift cs_shift = {
-		CS_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
-};
-
-static const struct dce110_clk_src_mask cs_mask = {
-		CS_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
-};
-
-static const struct bios_registers bios_regs = {
-	.BIOS_SCRATCH_3 = mmBIOS_SCRATCH_3,
-	.BIOS_SCRATCH_6 = mmBIOS_SCRATCH_6
-};
-
-static const struct resource_caps res_cap = {
-		.num_timing_generator = 6,
-		.num_audio = 6,
-		.num_stream_encoder = 6,
-		.num_pll = 3,
-		.num_ddc = 6,
-};
-
-static const struct resource_caps res_cap_81 = {
-		.num_timing_generator = 4,
-		.num_audio = 7,
-		.num_stream_encoder = 7,
-		.num_pll = 3,
-		.num_ddc = 6,
-};
-
-static const struct resource_caps res_cap_83 = {
-		.num_timing_generator = 2,
-		.num_audio = 6,
-		.num_stream_encoder = 6,
-		.num_pll = 2,
-		.num_ddc = 2,
-};
-
-static const struct dc_plane_cap plane_cap = {
-	.type = DC_PLANE_TYPE_DCE_RGB,
-
-	.pixel_format_support = {
-			.argb8888 = true,
-			.nv12 = false,
-			.fp16 = true
-	},
-
-	.max_upscale_factor = {
-			.argb8888 = 16000,
-			.nv12 = 1,
-			.fp16 = 1
-	},
-
-	.max_downscale_factor = {
-			.argb8888 = 250,
-			.nv12 = 1,
-			.fp16 = 1
-	}
-};
-
-static const struct dc_debug_options debug_defaults = {
-		.enable_legacy_fast_update = true,
-};
-
-static const struct dce_dmcu_registers dmcu_regs = {
-		DMCU_DCE80_REG_LIST()
-};
-
-static const struct dce_dmcu_shift dmcu_shift = {
-		DMCU_MASK_SH_LIST_DCE80(__SHIFT)
-};
-
-static const struct dce_dmcu_mask dmcu_mask = {
-		DMCU_MASK_SH_LIST_DCE80(_MASK)
-};
-static const struct dce_abm_registers abm_regs = {
-		ABM_DCE110_COMMON_REG_LIST()
-};
-
-static const struct dce_abm_shift abm_shift = {
-		ABM_MASK_SH_LIST_DCE110(__SHIFT)
-};
-
-static const struct dce_abm_mask abm_mask = {
-		ABM_MASK_SH_LIST_DCE110(_MASK)
-};
-
-#define CTX  ctx
-#define REG(reg) mm ## reg
-
-#ifndef mmCC_DC_HDMI_STRAPS
-#define mmCC_DC_HDMI_STRAPS 0x1918
-#define CC_DC_HDMI_STRAPS__HDMI_DISABLE_MASK 0x40
-#define CC_DC_HDMI_STRAPS__HDMI_DISABLE__SHIFT 0x6
-#define CC_DC_HDMI_STRAPS__AUDIO_STREAM_NUMBER_MASK 0x700
-#define CC_DC_HDMI_STRAPS__AUDIO_STREAM_NUMBER__SHIFT 0x8
-#endif
-
-static int map_transmitter_id_to_phy_instance(
-	enum transmitter transmitter)
-{
-	switch (transmitter) {
-	case TRANSMITTER_UNIPHY_A:
-		return 0;
-	case TRANSMITTER_UNIPHY_B:
-		return 1;
-	case TRANSMITTER_UNIPHY_C:
-		return 2;
-	case TRANSMITTER_UNIPHY_D:
-		return 3;
-	case TRANSMITTER_UNIPHY_E:
-		return 4;
-	case TRANSMITTER_UNIPHY_F:
-		return 5;
-	case TRANSMITTER_UNIPHY_G:
-		return 6;
-	default:
-		ASSERT(0);
-		return 0;
-	}
-}
-
-static void read_dce_straps(
-	struct dc_context *ctx,
-	struct resource_straps *straps)
-{
-	REG_GET_2(CC_DC_HDMI_STRAPS,
-			HDMI_DISABLE, &straps->hdmi_disable,
-			AUDIO_STREAM_NUMBER, &straps->audio_stream_number);
-
-	REG_GET(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO, &straps->dc_pinstraps_audio);
-}
-
-static struct audio *create_audio(
-		struct dc_context *ctx, unsigned int inst)
-{
-	return dce_audio_create(ctx, inst,
-			&audio_regs[inst], &audio_shift, &audio_mask);
-}
-
-static struct timing_generator *dce80_timing_generator_create(
-		struct dc_context *ctx,
-		uint32_t instance,
-		const struct dce110_timing_generator_offsets *offsets)
-{
-	struct dce110_timing_generator *tg110 =
-		kzalloc(sizeof(struct dce110_timing_generator), GFP_KERNEL);
-
-	if (!tg110)
-		return NULL;
-
-	dce80_timing_generator_construct(tg110, ctx, instance, offsets);
-	return &tg110->base;
-}
-
-static struct output_pixel_processor *dce80_opp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce110_opp *opp =
-		kzalloc(sizeof(struct dce110_opp), GFP_KERNEL);
-
-	if (!opp)
-		return NULL;
-
-	dce110_opp_construct(opp,
-			     ctx, inst, &opp_regs[inst], &opp_shift, &opp_mask);
-	return &opp->base;
-}
-
-static struct dce_aux *dce80_aux_engine_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct aux_engine_dce110 *aux_engine =
-		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
-
-	if (!aux_engine)
-		return NULL;
-
-	dce110_aux_engine_construct(aux_engine, ctx, inst,
-				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
-				    &aux_engine_regs[inst],
-					&aux_mask,
-					&aux_shift,
-					ctx->dc->caps.extended_aux_timeout_support);
-
-	return &aux_engine->base;
-}
-#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
-
-static const struct dce_i2c_registers i2c_hw_regs[] = {
-		i2c_inst_regs(1),
-		i2c_inst_regs(2),
-		i2c_inst_regs(3),
-		i2c_inst_regs(4),
-		i2c_inst_regs(5),
-		i2c_inst_regs(6),
-};
-
-static const struct dce_i2c_shift i2c_shifts = {
-		I2C_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
-};
-
-static const struct dce_i2c_mask i2c_masks = {
-		I2C_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
-};
-
-static struct dce_i2c_hw *dce80_i2c_hw_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_i2c_hw *dce_i2c_hw =
-		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
-
-	if (!dce_i2c_hw)
-		return NULL;
-
-	dce_i2c_hw_construct(dce_i2c_hw, ctx, inst,
-				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
-
-	return dce_i2c_hw;
-}
-
-static struct dce_i2c_sw *dce80_i2c_sw_create(
-	struct dc_context *ctx)
-{
-	struct dce_i2c_sw *dce_i2c_sw =
-		kzalloc(sizeof(struct dce_i2c_sw), GFP_KERNEL);
-
-	if (!dce_i2c_sw)
-		return NULL;
-
-	dce_i2c_sw_construct(dce_i2c_sw, ctx);
-
-	return dce_i2c_sw;
-}
-static struct stream_encoder *dce80_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dce110_stream_encoder *enc110 =
-		kzalloc(sizeof(struct dce110_stream_encoder), GFP_KERNEL);
-
-	if (!enc110)
-		return NULL;
-
-	dce110_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id,
-					&stream_enc_regs[eng_id],
-					&se_shift, &se_mask);
-	return &enc110->base;
-}
-
-#define SRII(reg_name, block, id)\
-	.reg_name[id] = mm ## block ## id ## _ ## reg_name
-
-static const struct dce_hwseq_registers hwseq_reg = {
-		HWSEQ_DCE8_REG_LIST()
-};
-
-static const struct dce_hwseq_shift hwseq_shift = {
-		HWSEQ_DCE8_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_hwseq_mask hwseq_mask = {
-		HWSEQ_DCE8_MASK_SH_LIST(_MASK)
-};
-
-static struct dce_hwseq *dce80_hwseq_create(
-	struct dc_context *ctx)
-{
-	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
-
-	if (hws) {
-		hws->ctx = ctx;
-		hws->regs = &hwseq_reg;
-		hws->shifts = &hwseq_shift;
-		hws->masks = &hwseq_mask;
-	}
-	return hws;
-}
-
-static const struct resource_create_funcs res_create_funcs = {
-	.read_dce_straps = read_dce_straps,
-	.create_audio = create_audio,
-	.create_stream_encoder = dce80_stream_encoder_create,
-	.create_hwseq = dce80_hwseq_create,
-};
-
-#define mi_inst_regs(id) { \
-	MI_DCE8_REG_LIST(id), \
-	.MC_HUB_RDREQ_DMIF_LIMIT = mmMC_HUB_RDREQ_DMIF_LIMIT \
-}
-static const struct dce_mem_input_registers mi_regs[] = {
-		mi_inst_regs(0),
-		mi_inst_regs(1),
-		mi_inst_regs(2),
-		mi_inst_regs(3),
-		mi_inst_regs(4),
-		mi_inst_regs(5),
-};
-
-static const struct dce_mem_input_shift mi_shifts = {
-		MI_DCE8_MASK_SH_LIST(__SHIFT),
-		.ENABLE = MC_HUB_RDREQ_DMIF_LIMIT__ENABLE__SHIFT
-};
-
-static const struct dce_mem_input_mask mi_masks = {
-		MI_DCE8_MASK_SH_LIST(_MASK),
-		.ENABLE = MC_HUB_RDREQ_DMIF_LIMIT__ENABLE_MASK
-};
-
-static struct mem_input *dce80_mem_input_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_mem_input *dce_mi = kzalloc(sizeof(struct dce_mem_input),
-					       GFP_KERNEL);
-
-	if (!dce_mi) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dce_mem_input_construct(dce_mi, ctx, inst, &mi_regs[inst], &mi_shifts, &mi_masks);
-	dce_mi->wa.single_head_rdreq_dmif_limit = 2;
-	return &dce_mi->base;
-}
-
-static void dce80_transform_destroy(struct transform **xfm)
-{
-	kfree(TO_DCE_TRANSFORM(*xfm));
-	*xfm = NULL;
-}
-
-static struct transform *dce80_transform_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_transform *transform =
-		kzalloc(sizeof(struct dce_transform), GFP_KERNEL);
-
-	if (!transform)
-		return NULL;
-
-	dce_transform_construct(transform, ctx, inst,
-				&xfm_regs[inst], &xfm_shift, &xfm_mask);
-	transform->prescaler_on = false;
-	return &transform->base;
-}
-
-static const struct encoder_feature_support link_enc_feature = {
-		.max_hdmi_deep_color = COLOR_DEPTH_121212,
-		.max_hdmi_pixel_clock = 297000,
-		.flags.bits.IS_HBR2_CAPABLE = true,
-		.flags.bits.IS_TPS3_CAPABLE = true
-};
-
-static struct link_encoder *dce80_link_encoder_create(
-	struct dc_context *ctx,
-	const struct encoder_init_data *enc_init_data)
-{
-	struct dce110_link_encoder *enc110 =
-		kzalloc(sizeof(struct dce110_link_encoder), GFP_KERNEL);
-	int link_regs_id;
-
-	if (!enc110)
-		return NULL;
-
-	link_regs_id =
-		map_transmitter_id_to_phy_instance(enc_init_data->transmitter);
-
-	dce110_link_encoder_construct(enc110,
-				      enc_init_data,
-				      &link_enc_feature,
-				      &link_enc_regs[link_regs_id],
-				      &link_enc_aux_regs[enc_init_data->channel - 1],
-				      &link_enc_hpd_regs[enc_init_data->hpd_source]);
-	return &enc110->base;
-}
-
-static struct panel_cntl *dce80_panel_cntl_create(const struct panel_cntl_init_data *init_data)
-{
-	struct dce_panel_cntl *panel_cntl =
-		kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
-
-	if (!panel_cntl)
-		return NULL;
-
-	dce_panel_cntl_construct(panel_cntl,
-			init_data,
-			&panel_cntl_regs[init_data->inst],
-			&panel_cntl_shift,
-			&panel_cntl_mask);
-
-	return &panel_cntl->base;
-}
-
-static struct clock_source *dce80_clock_source_create(
-	struct dc_context *ctx,
-	struct dc_bios *bios,
-	enum clock_source_id id,
-	const struct dce110_clk_src_regs *regs,
-	bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src =
-		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dce110_clk_src_construct(clk_src, ctx, bios, id,
-			regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-
-	kfree(clk_src);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
-static void dce80_clock_source_destroy(struct clock_source **clk_src)
-{
-	kfree(TO_DCE110_CLK_SRC(*clk_src));
-	*clk_src = NULL;
-}
-
-static struct input_pixel_processor *dce80_ipp_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dce_ipp *ipp = kzalloc(sizeof(struct dce_ipp), GFP_KERNEL);
-
-	if (!ipp) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dce_ipp_construct(ipp, ctx, inst,
-			&ipp_regs[inst], &ipp_shift, &ipp_mask);
-	return &ipp->base;
-}
-
-static void dce80_resource_destruct(struct dce110_resource_pool *pool)
-{
-	unsigned int i;
-
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		if (pool->base.opps[i] != NULL)
-			dce110_opp_destroy(&pool->base.opps[i]);
-
-		if (pool->base.transforms[i] != NULL)
-			dce80_transform_destroy(&pool->base.transforms[i]);
-
-		if (pool->base.ipps[i] != NULL)
-			dce_ipp_destroy(&pool->base.ipps[i]);
-
-		if (pool->base.mis[i] != NULL) {
-			kfree(TO_DCE_MEM_INPUT(pool->base.mis[i]));
-			pool->base.mis[i] = NULL;
-		}
-
-		if (pool->base.timing_generators[i] != NULL)	{
-			kfree(DCE110TG_FROM_TG(pool->base.timing_generators[i]));
-			pool->base.timing_generators[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		if (pool->base.engines[i] != NULL)
-			dce110_engine_destroy(&pool->base.engines[i]);
-		if (pool->base.hw_i2cs[i] != NULL) {
-			kfree(pool->base.hw_i2cs[i]);
-			pool->base.hw_i2cs[i] = NULL;
-		}
-		if (pool->base.sw_i2cs[i] != NULL) {
-			kfree(pool->base.sw_i2cs[i]);
-			pool->base.sw_i2cs[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.stream_enc_count; i++) {
-		if (pool->base.stream_enc[i] != NULL)
-			kfree(DCE110STRENC_FROM_STRENC(pool->base.stream_enc[i]));
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] != NULL) {
-			dce80_clock_source_destroy(&pool->base.clock_sources[i]);
-		}
-	}
-
-	if (pool->base.abm != NULL)
-			dce_abm_destroy(&pool->base.abm);
-
-	if (pool->base.dmcu != NULL)
-			dce_dmcu_destroy(&pool->base.dmcu);
-
-	if (pool->base.dp_clock_source != NULL)
-		dce80_clock_source_destroy(&pool->base.dp_clock_source);
-
-	for (i = 0; i < pool->base.audio_count; i++)	{
-		if (pool->base.audios[i] != NULL) {
-			dce_aud_destroy(&pool->base.audios[i]);
-		}
-	}
-
-	if (pool->base.irqs != NULL) {
-		dal_irq_service_destroy(&pool->base.irqs);
-	}
-}
-
-static bool dce80_validate_bandwidth(
-	struct dc *dc,
-	struct dc_state *context,
-	bool fast_validate)
-{
-	int i;
-	bool at_least_one_pipe = false;
-
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		if (context->res_ctx.pipe_ctx[i].stream)
-			at_least_one_pipe = true;
-	}
-
-	if (at_least_one_pipe) {
-		/* TODO implement when needed but for now hardcode max value*/
-		context->bw_ctx.bw.dce.dispclk_khz = 681000;
-		context->bw_ctx.bw.dce.yclk_khz = 250000 * MEMORY_TYPE_MULTIPLIER_CZ;
-	} else {
-		context->bw_ctx.bw.dce.dispclk_khz = 0;
-		context->bw_ctx.bw.dce.yclk_khz = 0;
-	}
-
-	return true;
-}
-
-static bool dce80_validate_surface_sets(
-		struct dc_state *context)
-{
-	int i;
-
-	for (i = 0; i < context->stream_count; i++) {
-		if (context->stream_status[i].plane_count == 0)
-			continue;
-
-		if (context->stream_status[i].plane_count > 1)
-			return false;
-
-		if (context->stream_status[i].plane_states[0]->format
-				>= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN)
-			return false;
-	}
-
-	return true;
-}
-
-static enum dc_status dce80_validate_global(
-		struct dc *dc,
-		struct dc_state *context)
-{
-	if (!dce80_validate_surface_sets(context))
-		return DC_FAIL_SURFACE_VALIDATE;
-
-	return DC_OK;
-}
-
-static void dce80_destroy_resource_pool(struct resource_pool **pool)
-{
-	struct dce110_resource_pool *dce110_pool = TO_DCE110_RES_POOL(*pool);
-
-	dce80_resource_destruct(dce110_pool);
-	kfree(dce110_pool);
-	*pool = NULL;
-}
-
-static const struct resource_funcs dce80_res_pool_funcs = {
-	.destroy = dce80_destroy_resource_pool,
-	.link_enc_create = dce80_link_encoder_create,
-	.panel_cntl_create = dce80_panel_cntl_create,
-	.validate_bandwidth = dce80_validate_bandwidth,
-	.validate_plane = dce100_validate_plane,
-	.add_stream_to_ctx = dce100_add_stream_to_ctx,
-	.validate_global = dce80_validate_global,
-	.find_first_free_match_stream_enc_for_link = dce100_find_first_free_match_stream_enc_for_link
-};
-
-static bool dce80_construct(
-	uint8_t num_virtual_links,
-	struct dc *dc,
-	struct dce110_resource_pool *pool)
-{
-	unsigned int i;
-	struct dc_context *ctx = dc->ctx;
-	struct dc_bios *bp;
-
-	ctx->dc_bios->regs = &bios_regs;
-
-	pool->base.res_cap = &res_cap;
-	pool->base.funcs = &dce80_res_pool_funcs;
-
-
-	/*************************************************
-	 *  Resource + asic cap harcoding                *
-	 *************************************************/
-	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
-	pool->base.pipe_count = res_cap.num_timing_generator;
-	pool->base.timing_generator_count = res_cap.num_timing_generator;
-	dc->caps.max_downscale_ratio = 200;
-	dc->caps.i2c_speed_in_khz = 40;
-	dc->caps.i2c_speed_in_khz_hdcp = 40;
-	dc->caps.max_cursor_size = 128;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.dual_link_dvi = true;
-	dc->caps.extended_aux_timeout_support = false;
-	dc->debug = debug_defaults;
-
-	/*************************************************
-	 *  Create resources                             *
-	 *************************************************/
-
-	bp = ctx->dc_bios;
-
-	if (bp->fw_info_valid && bp->fw_info.external_clock_source_frequency_for_dp != 0) {
-		pool->base.dp_clock_source =
-				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_EXTERNAL, NULL, true);
-
-		pool->base.clock_sources[0] =
-				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0, &clk_src_regs[0], false);
-		pool->base.clock_sources[1] =
-				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false);
-		pool->base.clock_sources[2] =
-				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[2], false);
-		pool->base.clk_src_count = 3;
-
-	} else {
-		pool->base.dp_clock_source =
-				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0, &clk_src_regs[0], true);
-
-		pool->base.clock_sources[0] =
-				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false);
-		pool->base.clock_sources[1] =
-				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[2], false);
-		pool->base.clk_src_count = 2;
-	}
-
-	if (pool->base.dp_clock_source == NULL) {
-		dm_error("DC: failed to create dp clock source!\n");
-		BREAK_TO_DEBUGGER();
-		goto res_create_fail;
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			BREAK_TO_DEBUGGER();
-			goto res_create_fail;
-		}
-	}
-
-	pool->base.dmcu = dce_dmcu_create(ctx,
-			&dmcu_regs,
-			&dmcu_shift,
-			&dmcu_mask);
-	if (pool->base.dmcu == NULL) {
-		dm_error("DC: failed to create dmcu!\n");
-		BREAK_TO_DEBUGGER();
-		goto res_create_fail;
-	}
-
-	pool->base.abm = dce_abm_create(ctx,
-			&abm_regs,
-			&abm_shift,
-			&abm_mask);
-	if (pool->base.abm == NULL) {
-		dm_error("DC: failed to create abm!\n");
-		BREAK_TO_DEBUGGER();
-		goto res_create_fail;
-	}
-
-	{
-		struct irq_service_init_data init_data;
-		init_data.ctx = dc->ctx;
-		pool->base.irqs = dal_irq_service_dce80_create(&init_data);
-		if (!pool->base.irqs)
-			goto res_create_fail;
-	}
-
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		pool->base.timing_generators[i] = dce80_timing_generator_create(
-				ctx, i, &dce80_tg_offsets[i]);
-		if (pool->base.timing_generators[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create tg!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.mis[i] = dce80_mem_input_create(ctx, i);
-		if (pool->base.mis[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create memory input!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.ipps[i] = dce80_ipp_create(ctx, i);
-		if (pool->base.ipps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create input pixel processor!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.transforms[i] = dce80_transform_create(ctx, i);
-		if (pool->base.transforms[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create transform!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.opps[i] = dce80_opp_create(ctx, i);
-		if (pool->base.opps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create output pixel processor!\n");
-			goto res_create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		pool->base.engines[i] = dce80_aux_engine_create(ctx, i);
-		if (pool->base.engines[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create aux engine!!\n");
-			goto res_create_fail;
-		}
-		pool->base.hw_i2cs[i] = dce80_i2c_hw_create(ctx, i);
-		if (pool->base.hw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create i2c engine!!\n");
-			goto res_create_fail;
-		}
-		pool->base.sw_i2cs[i] = dce80_i2c_sw_create(ctx);
-		if (pool->base.sw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create sw i2c!!\n");
-			goto res_create_fail;
-		}
-	}
-
-	dc->caps.max_planes =  pool->base.pipe_count;
-
-	for (i = 0; i < dc->caps.max_planes; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	dc->caps.disable_dp_clk_share = true;
-
-	if (!resource_construct(num_virtual_links, dc, &pool->base,
-			&res_create_funcs))
-		goto res_create_fail;
-
-	/* Create hardware sequencer */
-	dce80_hw_sequencer_construct(dc);
-
-	return true;
-
-res_create_fail:
-	dce80_resource_destruct(pool);
-	return false;
-}
-
-struct resource_pool *dce80_create_resource_pool(
-	uint8_t num_virtual_links,
-	struct dc *dc)
-{
-	struct dce110_resource_pool *pool =
-		kzalloc(sizeof(struct dce110_resource_pool), GFP_KERNEL);
-
-	if (!pool)
-		return NULL;
-
-	if (dce80_construct(num_virtual_links, dc, pool))
-		return &pool->base;
-
-	kfree(pool);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
-static bool dce81_construct(
-	uint8_t num_virtual_links,
-	struct dc *dc,
-	struct dce110_resource_pool *pool)
-{
-	unsigned int i;
-	struct dc_context *ctx = dc->ctx;
-	struct dc_bios *bp;
-
-	ctx->dc_bios->regs = &bios_regs;
-
-	pool->base.res_cap = &res_cap_81;
-	pool->base.funcs = &dce80_res_pool_funcs;
-
-
-	/*************************************************
-	 *  Resource + asic cap harcoding                *
-	 *************************************************/
-	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
-	pool->base.pipe_count = res_cap_81.num_timing_generator;
-	pool->base.timing_generator_count = res_cap_81.num_timing_generator;
-	dc->caps.max_downscale_ratio = 200;
-	dc->caps.i2c_speed_in_khz = 40;
-	dc->caps.i2c_speed_in_khz_hdcp = 40;
-	dc->caps.max_cursor_size = 128;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.is_apu = true;
-
-	/*************************************************
-	 *  Create resources                             *
-	 *************************************************/
-
-	bp = ctx->dc_bios;
-
-	if (bp->fw_info_valid && bp->fw_info.external_clock_source_frequency_for_dp != 0) {
-		pool->base.dp_clock_source =
-				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_EXTERNAL, NULL, true);
-
-		pool->base.clock_sources[0] =
-				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0, &clk_src_regs[0], false);
-		pool->base.clock_sources[1] =
-				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false);
-		pool->base.clock_sources[2] =
-				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[2], false);
-		pool->base.clk_src_count = 3;
-
-	} else {
-		pool->base.dp_clock_source =
-				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0, &clk_src_regs[0], true);
-
-		pool->base.clock_sources[0] =
-				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false);
-		pool->base.clock_sources[1] =
-				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[2], false);
-		pool->base.clk_src_count = 2;
-	}
-
-	if (pool->base.dp_clock_source == NULL) {
-		dm_error("DC: failed to create dp clock source!\n");
-		BREAK_TO_DEBUGGER();
-		goto res_create_fail;
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			BREAK_TO_DEBUGGER();
-			goto res_create_fail;
-		}
-	}
-
-	pool->base.dmcu = dce_dmcu_create(ctx,
-			&dmcu_regs,
-			&dmcu_shift,
-			&dmcu_mask);
-	if (pool->base.dmcu == NULL) {
-		dm_error("DC: failed to create dmcu!\n");
-		BREAK_TO_DEBUGGER();
-		goto res_create_fail;
-	}
-
-	pool->base.abm = dce_abm_create(ctx,
-			&abm_regs,
-			&abm_shift,
-			&abm_mask);
-	if (pool->base.abm == NULL) {
-		dm_error("DC: failed to create abm!\n");
-		BREAK_TO_DEBUGGER();
-		goto res_create_fail;
-	}
-
-	{
-		struct irq_service_init_data init_data;
-		init_data.ctx = dc->ctx;
-		pool->base.irqs = dal_irq_service_dce80_create(&init_data);
-		if (!pool->base.irqs)
-			goto res_create_fail;
-	}
-
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		pool->base.timing_generators[i] = dce80_timing_generator_create(
-				ctx, i, &dce80_tg_offsets[i]);
-		if (pool->base.timing_generators[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create tg!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.mis[i] = dce80_mem_input_create(ctx, i);
-		if (pool->base.mis[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create memory input!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.ipps[i] = dce80_ipp_create(ctx, i);
-		if (pool->base.ipps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create input pixel processor!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.transforms[i] = dce80_transform_create(ctx, i);
-		if (pool->base.transforms[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create transform!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.opps[i] = dce80_opp_create(ctx, i);
-		if (pool->base.opps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create output pixel processor!\n");
-			goto res_create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		pool->base.engines[i] = dce80_aux_engine_create(ctx, i);
-		if (pool->base.engines[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create aux engine!!\n");
-			goto res_create_fail;
-		}
-		pool->base.hw_i2cs[i] = dce80_i2c_hw_create(ctx, i);
-		if (pool->base.hw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create i2c engine!!\n");
-			goto res_create_fail;
-		}
-		pool->base.sw_i2cs[i] = dce80_i2c_sw_create(ctx);
-		if (pool->base.sw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create sw i2c!!\n");
-			goto res_create_fail;
-		}
-	}
-
-	dc->caps.max_planes =  pool->base.pipe_count;
-
-	for (i = 0; i < dc->caps.max_planes; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	dc->caps.disable_dp_clk_share = true;
-
-	if (!resource_construct(num_virtual_links, dc, &pool->base,
-			&res_create_funcs))
-		goto res_create_fail;
-
-	/* Create hardware sequencer */
-	dce80_hw_sequencer_construct(dc);
-
-	return true;
-
-res_create_fail:
-	dce80_resource_destruct(pool);
-	return false;
-}
-
-struct resource_pool *dce81_create_resource_pool(
-	uint8_t num_virtual_links,
-	struct dc *dc)
-{
-	struct dce110_resource_pool *pool =
-		kzalloc(sizeof(struct dce110_resource_pool), GFP_KERNEL);
-
-	if (!pool)
-		return NULL;
-
-	if (dce81_construct(num_virtual_links, dc, pool))
-		return &pool->base;
-
-	kfree(pool);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
-static bool dce83_construct(
-	uint8_t num_virtual_links,
-	struct dc *dc,
-	struct dce110_resource_pool *pool)
-{
-	unsigned int i;
-	struct dc_context *ctx = dc->ctx;
-	struct dc_bios *bp;
-
-	ctx->dc_bios->regs = &bios_regs;
-
-	pool->base.res_cap = &res_cap_83;
-	pool->base.funcs = &dce80_res_pool_funcs;
-
-
-	/*************************************************
-	 *  Resource + asic cap harcoding                *
-	 *************************************************/
-	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
-	pool->base.pipe_count = res_cap_83.num_timing_generator;
-	pool->base.timing_generator_count = res_cap_83.num_timing_generator;
-	dc->caps.max_downscale_ratio = 200;
-	dc->caps.i2c_speed_in_khz = 40;
-	dc->caps.i2c_speed_in_khz_hdcp = 40;
-	dc->caps.max_cursor_size = 128;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.is_apu = true;
-	dc->debug = debug_defaults;
-
-	/*************************************************
-	 *  Create resources                             *
-	 *************************************************/
-
-	bp = ctx->dc_bios;
-
-	if (bp->fw_info_valid && bp->fw_info.external_clock_source_frequency_for_dp != 0) {
-		pool->base.dp_clock_source =
-				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_EXTERNAL, NULL, true);
-
-		pool->base.clock_sources[0] =
-				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[0], false);
-		pool->base.clock_sources[1] =
-				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[1], false);
-		pool->base.clk_src_count = 2;
-
-	} else {
-		pool->base.dp_clock_source =
-				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[0], true);
-
-		pool->base.clock_sources[0] =
-				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[1], false);
-		pool->base.clk_src_count = 1;
-	}
-
-	if (pool->base.dp_clock_source == NULL) {
-		dm_error("DC: failed to create dp clock source!\n");
-		BREAK_TO_DEBUGGER();
-		goto res_create_fail;
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			BREAK_TO_DEBUGGER();
-			goto res_create_fail;
-		}
-	}
-
-	pool->base.dmcu = dce_dmcu_create(ctx,
-			&dmcu_regs,
-			&dmcu_shift,
-			&dmcu_mask);
-	if (pool->base.dmcu == NULL) {
-		dm_error("DC: failed to create dmcu!\n");
-		BREAK_TO_DEBUGGER();
-		goto res_create_fail;
-	}
-
-	pool->base.abm = dce_abm_create(ctx,
-			&abm_regs,
-			&abm_shift,
-			&abm_mask);
-	if (pool->base.abm == NULL) {
-		dm_error("DC: failed to create abm!\n");
-		BREAK_TO_DEBUGGER();
-		goto res_create_fail;
-	}
-
-	{
-		struct irq_service_init_data init_data;
-		init_data.ctx = dc->ctx;
-		pool->base.irqs = dal_irq_service_dce80_create(&init_data);
-		if (!pool->base.irqs)
-			goto res_create_fail;
-	}
-
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		pool->base.timing_generators[i] = dce80_timing_generator_create(
-				ctx, i, &dce80_tg_offsets[i]);
-		if (pool->base.timing_generators[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create tg!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.mis[i] = dce80_mem_input_create(ctx, i);
-		if (pool->base.mis[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create memory input!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.ipps[i] = dce80_ipp_create(ctx, i);
-		if (pool->base.ipps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create input pixel processor!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.transforms[i] = dce80_transform_create(ctx, i);
-		if (pool->base.transforms[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create transform!\n");
-			goto res_create_fail;
-		}
-
-		pool->base.opps[i] = dce80_opp_create(ctx, i);
-		if (pool->base.opps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create output pixel processor!\n");
-			goto res_create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		pool->base.engines[i] = dce80_aux_engine_create(ctx, i);
-		if (pool->base.engines[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create aux engine!!\n");
-			goto res_create_fail;
-		}
-		pool->base.hw_i2cs[i] = dce80_i2c_hw_create(ctx, i);
-		if (pool->base.hw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create i2c engine!!\n");
-			goto res_create_fail;
-		}
-		pool->base.sw_i2cs[i] = dce80_i2c_sw_create(ctx);
-		if (pool->base.sw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create sw i2c!!\n");
-			goto res_create_fail;
-		}
-	}
-
-	dc->caps.max_planes =  pool->base.pipe_count;
-
-	for (i = 0; i < dc->caps.max_planes; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	dc->caps.disable_dp_clk_share = true;
-
-	if (!resource_construct(num_virtual_links, dc, &pool->base,
-			&res_create_funcs))
-		goto res_create_fail;
-
-	/* Create hardware sequencer */
-	dce80_hw_sequencer_construct(dc);
-
-	return true;
-
-res_create_fail:
-	dce80_resource_destruct(pool);
-	return false;
-}
-
-struct resource_pool *dce83_create_resource_pool(
-	uint8_t num_virtual_links,
-	struct dc *dc)
-{
-	struct dce110_resource_pool *pool =
-		kzalloc(sizeof(struct dce110_resource_pool), GFP_KERNEL);
-
-	if (!pool)
-		return NULL;
-
-	if (dce83_construct(num_virtual_links, dc, pool))
-		return &pool->base;
-
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.h b/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.h
deleted file mode 100644
index eff31ab83a39..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
-* Copyright 2012-15 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#ifndef __DC_RESOURCE_DCE80_H__
-#define __DC_RESOURCE_DCE80_H__
-
-#include "core_types.h"
-
-struct dc;
-struct resource_pool;
-
-struct resource_pool *dce80_create_resource_pool(
-	uint8_t num_virtual_links,
-	struct dc *dc);
-
-struct resource_pool *dce81_create_resource_pool(
-	uint8_t num_virtual_links,
-	struct dc *dc);
-
-struct resource_pool *dce83_create_resource_pool(
-	uint8_t num_virtual_links,
-	struct dc *dc);
-
-#endif /* __DC_RESOURCE_DCE80_H__ */
-
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/Makefile b/drivers/gpu/drm/amd/display/dc/dcn10/Makefile
index 2d2007c3e2b6..1eb7418ced3a 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/Makefile
@@ -22,7 +22,7 @@
 #
 # Makefile for DCN.
 
-DCN10 = dcn10_init.o dcn10_resource.o dcn10_ipp.o \
+DCN10 = dcn10_init.o dcn10_ipp.o \
 		dcn10_hw_sequencer_debug.o \
 		dcn10_dpp.o dcn10_opp.o dcn10_optc.o \
 		dcn10_hubp.o dcn10_mpc.o \
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
deleted file mode 100644
index b94c5c97eee7..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
+++ /dev/null
@@ -1,1686 +0,0 @@
-/*
-* Copyright 2016 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#include "dm_services.h"
-#include "dc.h"
-
-#include "dcn10_init.h"
-
-#include "resource.h"
-#include "include/irq_service_interface.h"
-#include "dcn10_resource.h"
-#include "dcn10_ipp.h"
-#include "dcn10_mpc.h"
-#include "irq/dcn10/irq_service_dcn10.h"
-#include "dcn10_dpp.h"
-#include "dcn10_optc.h"
-#include "dcn10/dcn10_hwseq.h"
-#include "dce110/dce110_hwseq.h"
-#include "dcn10_opp.h"
-#include "dcn10_link_encoder.h"
-#include "dcn10_stream_encoder.h"
-#include "dce/dce_clock_source.h"
-#include "dce/dce_audio.h"
-#include "dce/dce_hwseq.h"
-#include "virtual/virtual_stream_encoder.h"
-#include "dce110/dce110_resource.h"
-#include "dce112/dce112_resource.h"
-#include "dcn10_hubp.h"
-#include "dcn10_hubbub.h"
-#include "dce/dce_panel_cntl.h"
-
-#include "soc15_hw_ip.h"
-#include "vega10_ip_offset.h"
-
-#include "dcn/dcn_1_0_offset.h"
-#include "dcn/dcn_1_0_sh_mask.h"
-
-#include "nbio/nbio_7_0_offset.h"
-
-#include "mmhub/mmhub_9_1_offset.h"
-#include "mmhub/mmhub_9_1_sh_mask.h"
-
-#include "reg_helper.h"
-#include "dce/dce_abm.h"
-#include "dce/dce_dmcu.h"
-#include "dce/dce_aux.h"
-#include "dce/dce_i2c.h"
-
-#ifndef mmDP0_DP_DPHY_INTERNAL_CTRL
-	#define mmDP0_DP_DPHY_INTERNAL_CTRL		0x210f
-	#define mmDP0_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
-	#define mmDP1_DP_DPHY_INTERNAL_CTRL		0x220f
-	#define mmDP1_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
-	#define mmDP2_DP_DPHY_INTERNAL_CTRL		0x230f
-	#define mmDP2_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
-	#define mmDP3_DP_DPHY_INTERNAL_CTRL		0x240f
-	#define mmDP3_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
-	#define mmDP4_DP_DPHY_INTERNAL_CTRL		0x250f
-	#define mmDP4_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
-	#define mmDP5_DP_DPHY_INTERNAL_CTRL		0x260f
-	#define mmDP5_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
-	#define mmDP6_DP_DPHY_INTERNAL_CTRL		0x270f
-	#define mmDP6_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
-#endif
-
-
-enum dcn10_clk_src_array_id {
-	DCN10_CLK_SRC_PLL0,
-	DCN10_CLK_SRC_PLL1,
-	DCN10_CLK_SRC_PLL2,
-	DCN10_CLK_SRC_PLL3,
-	DCN10_CLK_SRC_TOTAL,
-	DCN101_CLK_SRC_TOTAL = DCN10_CLK_SRC_PLL3
-};
-
-/* begin *********************
- * macros to expend register list macro defined in HW object header file */
-
-/* DCN */
-#define BASE_INNER(seg) \
-	DCE_BASE__INST0_SEG ## seg
-
-#define BASE(seg) \
-	BASE_INNER(seg)
-
-#define SR(reg_name)\
-		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) +  \
-					mm ## reg_name
-
-#define SRI(reg_name, block, id)\
-	.reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-
-#define SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define VUPDATE_SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(mm ## reg_name ## 0 ## _ ## block ## id ## _BASE_IDX) + \
-					mm ## reg_name ## 0 ## _ ## block ## id
-
-/* set field/register/bitfield name */
-#define SFRB(field_name, reg_name, bitfield, post_fix)\
-	.field_name = reg_name ## __ ## bitfield ## post_fix
-
-/* NBIO */
-#define NBIO_BASE_INNER(seg) \
-	NBIF_BASE__INST0_SEG ## seg
-
-#define NBIO_BASE(seg) \
-	NBIO_BASE_INNER(seg)
-
-#define NBIO_SR(reg_name)\
-		.reg_name = NBIO_BASE(mm ## reg_name ## _BASE_IDX) +  \
-					mm ## reg_name
-
-/* MMHUB */
-#define MMHUB_BASE_INNER(seg) \
-	MMHUB_BASE__INST0_SEG ## seg
-
-#define MMHUB_BASE(seg) \
-	MMHUB_BASE_INNER(seg)
-
-#define MMHUB_SR(reg_name)\
-		.reg_name = MMHUB_BASE(mm ## reg_name ## _BASE_IDX) +  \
-					mm ## reg_name
-
-/* macros to expend register list macro defined in HW object header file
- * end *********************/
-
-
-static const struct dce_dmcu_registers dmcu_regs = {
-		DMCU_DCN10_REG_LIST()
-};
-
-static const struct dce_dmcu_shift dmcu_shift = {
-		DMCU_MASK_SH_LIST_DCN10(__SHIFT)
-};
-
-static const struct dce_dmcu_mask dmcu_mask = {
-		DMCU_MASK_SH_LIST_DCN10(_MASK)
-};
-
-static const struct dce_abm_registers abm_regs = {
-		ABM_DCN10_REG_LIST(0)
-};
-
-static const struct dce_abm_shift abm_shift = {
-		ABM_MASK_SH_LIST_DCN10(__SHIFT)
-};
-
-static const struct dce_abm_mask abm_mask = {
-		ABM_MASK_SH_LIST_DCN10(_MASK)
-};
-
-#define stream_enc_regs(id)\
-[id] = {\
-	SE_DCN_REG_LIST(id)\
-}
-
-static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
-	stream_enc_regs(0),
-	stream_enc_regs(1),
-	stream_enc_regs(2),
-	stream_enc_regs(3),
-};
-
-static const struct dcn10_stream_encoder_shift se_shift = {
-		SE_COMMON_MASK_SH_LIST_DCN10(__SHIFT)
-};
-
-static const struct dcn10_stream_encoder_mask se_mask = {
-		SE_COMMON_MASK_SH_LIST_DCN10(_MASK)
-};
-
-#define audio_regs(id)\
-[id] = {\
-		AUD_COMMON_REG_LIST(id)\
-}
-
-static const struct dce_audio_registers audio_regs[] = {
-	audio_regs(0),
-	audio_regs(1),
-	audio_regs(2),
-	audio_regs(3),
-};
-
-#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
-		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
-
-static const struct dce_audio_shift audio_shift = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_audio_mask audio_mask = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
-};
-
-#define aux_regs(id)\
-[id] = {\
-	AUX_REG_LIST(id)\
-}
-
-static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
-		aux_regs(0),
-		aux_regs(1),
-		aux_regs(2),
-		aux_regs(3)
-};
-
-#define hpd_regs(id)\
-[id] = {\
-	HPD_REG_LIST(id)\
-}
-
-static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
-		hpd_regs(0),
-		hpd_regs(1),
-		hpd_regs(2),
-		hpd_regs(3)
-};
-
-#define link_regs(id)\
-[id] = {\
-	LE_DCN10_REG_LIST(id), \
-	SRI(DP_DPHY_INTERNAL_CTRL, DP, id) \
-}
-
-static const struct dcn10_link_enc_registers link_enc_regs[] = {
-	link_regs(0),
-	link_regs(1),
-	link_regs(2),
-	link_regs(3)
-};
-
-static const struct dcn10_link_enc_shift le_shift = {
-		LINK_ENCODER_MASK_SH_LIST_DCN10(__SHIFT)
-};
-
-static const struct dcn10_link_enc_mask le_mask = {
-		LINK_ENCODER_MASK_SH_LIST_DCN10(_MASK)
-};
-
-static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
-	{ DCN_PANEL_CNTL_REG_LIST() }
-};
-
-static const struct dce_panel_cntl_shift panel_cntl_shift = {
-	DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_panel_cntl_mask panel_cntl_mask = {
-	DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
-};
-
-static const struct dce110_aux_registers_shift aux_shift = {
-	DCN10_AUX_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce110_aux_registers_mask aux_mask = {
-	DCN10_AUX_MASK_SH_LIST(_MASK)
-};
-
-#define ipp_regs(id)\
-[id] = {\
-	IPP_REG_LIST_DCN10(id),\
-}
-
-static const struct dcn10_ipp_registers ipp_regs[] = {
-	ipp_regs(0),
-	ipp_regs(1),
-	ipp_regs(2),
-	ipp_regs(3),
-};
-
-static const struct dcn10_ipp_shift ipp_shift = {
-		IPP_MASK_SH_LIST_DCN10(__SHIFT)
-};
-
-static const struct dcn10_ipp_mask ipp_mask = {
-		IPP_MASK_SH_LIST_DCN10(_MASK),
-};
-
-#define opp_regs(id)\
-[id] = {\
-	OPP_REG_LIST_DCN10(id),\
-}
-
-static const struct dcn10_opp_registers opp_regs[] = {
-	opp_regs(0),
-	opp_regs(1),
-	opp_regs(2),
-	opp_regs(3),
-};
-
-static const struct dcn10_opp_shift opp_shift = {
-		OPP_MASK_SH_LIST_DCN10(__SHIFT)
-};
-
-static const struct dcn10_opp_mask opp_mask = {
-		OPP_MASK_SH_LIST_DCN10(_MASK),
-};
-
-#define aux_engine_regs(id)\
-[id] = {\
-	AUX_COMMON_REG_LIST(id), \
-	.AUX_RESET_MASK = 0 \
-}
-
-static const struct dce110_aux_registers aux_engine_regs[] = {
-		aux_engine_regs(0),
-		aux_engine_regs(1),
-		aux_engine_regs(2),
-		aux_engine_regs(3),
-		aux_engine_regs(4),
-		aux_engine_regs(5)
-};
-
-#define tf_regs(id)\
-[id] = {\
-	TF_REG_LIST_DCN10(id),\
-}
-
-static const struct dcn_dpp_registers tf_regs[] = {
-	tf_regs(0),
-	tf_regs(1),
-	tf_regs(2),
-	tf_regs(3),
-};
-
-static const struct dcn_dpp_shift tf_shift = {
-	TF_REG_LIST_SH_MASK_DCN10(__SHIFT),
-	TF_DEBUG_REG_LIST_SH_DCN10
-
-};
-
-static const struct dcn_dpp_mask tf_mask = {
-	TF_REG_LIST_SH_MASK_DCN10(_MASK),
-	TF_DEBUG_REG_LIST_MASK_DCN10
-};
-
-static const struct dcn_mpc_registers mpc_regs = {
-		MPC_COMMON_REG_LIST_DCN1_0(0),
-		MPC_COMMON_REG_LIST_DCN1_0(1),
-		MPC_COMMON_REG_LIST_DCN1_0(2),
-		MPC_COMMON_REG_LIST_DCN1_0(3),
-		MPC_OUT_MUX_COMMON_REG_LIST_DCN1_0(0),
-		MPC_OUT_MUX_COMMON_REG_LIST_DCN1_0(1),
-		MPC_OUT_MUX_COMMON_REG_LIST_DCN1_0(2),
-		MPC_OUT_MUX_COMMON_REG_LIST_DCN1_0(3)
-};
-
-static const struct dcn_mpc_shift mpc_shift = {
-	MPC_COMMON_MASK_SH_LIST_DCN1_0(__SHIFT),\
-	SFRB(CUR_VUPDATE_LOCK_SET, CUR0_VUPDATE_LOCK_SET0, CUR0_VUPDATE_LOCK_SET, __SHIFT)
-};
-
-static const struct dcn_mpc_mask mpc_mask = {
-	MPC_COMMON_MASK_SH_LIST_DCN1_0(_MASK),\
-	SFRB(CUR_VUPDATE_LOCK_SET, CUR0_VUPDATE_LOCK_SET0, CUR0_VUPDATE_LOCK_SET, _MASK)
-};
-
-#define tg_regs(id)\
-[id] = {TG_COMMON_REG_LIST_DCN1_0(id)}
-
-static const struct dcn_optc_registers tg_regs[] = {
-	tg_regs(0),
-	tg_regs(1),
-	tg_regs(2),
-	tg_regs(3),
-};
-
-static const struct dcn_optc_shift tg_shift = {
-	TG_COMMON_MASK_SH_LIST_DCN1_0(__SHIFT)
-};
-
-static const struct dcn_optc_mask tg_mask = {
-	TG_COMMON_MASK_SH_LIST_DCN1_0(_MASK)
-};
-
-static const struct bios_registers bios_regs = {
-		NBIO_SR(BIOS_SCRATCH_3),
-		NBIO_SR(BIOS_SCRATCH_6)
-};
-
-#define hubp_regs(id)\
-[id] = {\
-	HUBP_REG_LIST_DCN10(id)\
-}
-
-static const struct dcn_mi_registers hubp_regs[] = {
-	hubp_regs(0),
-	hubp_regs(1),
-	hubp_regs(2),
-	hubp_regs(3),
-};
-
-static const struct dcn_mi_shift hubp_shift = {
-		HUBP_MASK_SH_LIST_DCN10(__SHIFT)
-};
-
-static const struct dcn_mi_mask hubp_mask = {
-		HUBP_MASK_SH_LIST_DCN10(_MASK)
-};
-
-static const struct dcn_hubbub_registers hubbub_reg = {
-		HUBBUB_REG_LIST_DCN10(0)
-};
-
-static const struct dcn_hubbub_shift hubbub_shift = {
-		HUBBUB_MASK_SH_LIST_DCN10(__SHIFT)
-};
-
-static const struct dcn_hubbub_mask hubbub_mask = {
-		HUBBUB_MASK_SH_LIST_DCN10(_MASK)
-};
-
-static int map_transmitter_id_to_phy_instance(
-	enum transmitter transmitter)
-{
-	switch (transmitter) {
-	case TRANSMITTER_UNIPHY_A:
-		return 0;
-	break;
-	case TRANSMITTER_UNIPHY_B:
-		return 1;
-	break;
-	case TRANSMITTER_UNIPHY_C:
-		return 2;
-	break;
-	case TRANSMITTER_UNIPHY_D:
-		return 3;
-	break;
-	default:
-		ASSERT(0);
-		return 0;
-	}
-}
-
-#define clk_src_regs(index, pllid)\
-[index] = {\
-	CS_COMMON_REG_LIST_DCN1_0(index, pllid),\
-}
-
-static const struct dce110_clk_src_regs clk_src_regs[] = {
-	clk_src_regs(0, A),
-	clk_src_regs(1, B),
-	clk_src_regs(2, C),
-	clk_src_regs(3, D)
-};
-
-static const struct dce110_clk_src_shift cs_shift = {
-		CS_COMMON_MASK_SH_LIST_DCN1_0(__SHIFT)
-};
-
-static const struct dce110_clk_src_mask cs_mask = {
-		CS_COMMON_MASK_SH_LIST_DCN1_0(_MASK)
-};
-
-static const struct resource_caps res_cap = {
-		.num_timing_generator = 4,
-		.num_opp = 4,
-		.num_video_plane = 4,
-		.num_audio = 4,
-		.num_stream_encoder = 4,
-		.num_pll = 4,
-		.num_ddc = 4,
-};
-
-static const struct resource_caps rv2_res_cap = {
-		.num_timing_generator = 3,
-		.num_opp = 3,
-		.num_video_plane = 3,
-		.num_audio = 3,
-		.num_stream_encoder = 3,
-		.num_pll = 3,
-		.num_ddc = 4,
-};
-
-static const struct dc_plane_cap plane_cap = {
-	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.per_pixel_alpha = true,
-
-	.pixel_format_support = {
-			.argb8888 = true,
-			.nv12 = true,
-			.fp16 = true,
-			.p010 = true
-	},
-
-	.max_upscale_factor = {
-			.argb8888 = 16000,
-			.nv12 = 16000,
-			.fp16 = 1
-	},
-
-	.max_downscale_factor = {
-			.argb8888 = 250,
-			.nv12 = 250,
-			.fp16 = 1
-	}
-};
-
-static const struct dc_debug_options debug_defaults_drv = {
-		.sanity_checks = true,
-		.disable_dmcu = false,
-		.force_abm_enable = false,
-		.timing_trace = false,
-		.clock_trace = true,
-
-		/* raven smu dones't allow 0 disp clk,
-		 * smu min disp clk limit is 50Mhz
-		 * keep min disp clk 100Mhz avoid smu hang
-		 */
-		.min_disp_clk_khz = 100000,
-
-		.disable_pplib_clock_request = false,
-		.disable_pplib_wm_range = false,
-		.pplib_wm_report_mode = WM_REPORT_DEFAULT,
-		.pipe_split_policy = MPC_SPLIT_DYNAMIC,
-		.force_single_disp_pipe_split = true,
-		.disable_dcc = DCC_ENABLE,
-		.voltage_align_fclk = true,
-		.disable_stereo_support = true,
-		.vsr_support = true,
-		.performance_trace = false,
-		.az_endpoint_mute_only = true,
-		.recovery_enabled = false, /*enable this by default after testing.*/
-		.max_downscale_src_width = 3840,
-		.underflow_assert_delay_us = 0xFFFFFFFF,
-		.enable_legacy_fast_update = true,
-		.using_dml2 = false,
-};
-
-static const struct dc_debug_options debug_defaults_diags = {
-		.disable_dmcu = false,
-		.force_abm_enable = false,
-		.timing_trace = true,
-		.clock_trace = true,
-		.disable_stutter = true,
-		.disable_pplib_clock_request = true,
-		.disable_pplib_wm_range = true,
-		.underflow_assert_delay_us = 0xFFFFFFFF,
-};
-
-static void dcn10_dpp_destroy(struct dpp **dpp)
-{
-	kfree(TO_DCN10_DPP(*dpp));
-	*dpp = NULL;
-}
-
-static struct dpp *dcn10_dpp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn10_dpp *dpp =
-		kzalloc(sizeof(struct dcn10_dpp), GFP_KERNEL);
-
-	if (!dpp)
-		return NULL;
-
-	dpp1_construct(dpp, ctx, inst,
-		       &tf_regs[inst], &tf_shift, &tf_mask);
-	return &dpp->base;
-}
-
-static struct input_pixel_processor *dcn10_ipp_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn10_ipp *ipp =
-		kzalloc(sizeof(struct dcn10_ipp), GFP_KERNEL);
-
-	if (!ipp) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dcn10_ipp_construct(ipp, ctx, inst,
-			&ipp_regs[inst], &ipp_shift, &ipp_mask);
-	return &ipp->base;
-}
-
-
-static struct output_pixel_processor *dcn10_opp_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn10_opp *opp =
-		kzalloc(sizeof(struct dcn10_opp), GFP_KERNEL);
-
-	if (!opp) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dcn10_opp_construct(opp, ctx, inst,
-			&opp_regs[inst], &opp_shift, &opp_mask);
-	return &opp->base;
-}
-
-static struct dce_aux *dcn10_aux_engine_create(struct dc_context *ctx,
-					       uint32_t inst)
-{
-	struct aux_engine_dce110 *aux_engine =
-		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
-
-	if (!aux_engine)
-		return NULL;
-
-	dce110_aux_engine_construct(aux_engine, ctx, inst,
-				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
-				    &aux_engine_regs[inst],
-					&aux_mask,
-					&aux_shift,
-					ctx->dc->caps.extended_aux_timeout_support);
-
-	return &aux_engine->base;
-}
-#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
-
-static const struct dce_i2c_registers i2c_hw_regs[] = {
-		i2c_inst_regs(1),
-		i2c_inst_regs(2),
-		i2c_inst_regs(3),
-		i2c_inst_regs(4),
-		i2c_inst_regs(5),
-		i2c_inst_regs(6),
-};
-
-static const struct dce_i2c_shift i2c_shifts = {
-		I2C_COMMON_MASK_SH_LIST_DCE110(__SHIFT)
-};
-
-static const struct dce_i2c_mask i2c_masks = {
-		I2C_COMMON_MASK_SH_LIST_DCE110(_MASK)
-};
-
-static struct dce_i2c_hw *dcn10_i2c_hw_create(struct dc_context *ctx,
-					      uint32_t inst)
-{
-	struct dce_i2c_hw *dce_i2c_hw =
-		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
-
-	if (!dce_i2c_hw)
-		return NULL;
-
-	dcn1_i2c_hw_construct(dce_i2c_hw, ctx, inst,
-				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
-
-	return dce_i2c_hw;
-}
-static struct mpc *dcn10_mpc_create(struct dc_context *ctx)
-{
-	struct dcn10_mpc *mpc10 = kzalloc(sizeof(struct dcn10_mpc),
-					  GFP_KERNEL);
-
-	if (!mpc10)
-		return NULL;
-
-	dcn10_mpc_construct(mpc10, ctx,
-			&mpc_regs,
-			&mpc_shift,
-			&mpc_mask,
-			4);
-
-	return &mpc10->base;
-}
-
-static struct hubbub *dcn10_hubbub_create(struct dc_context *ctx)
-{
-	struct dcn10_hubbub *dcn10_hubbub = kzalloc(sizeof(struct dcn10_hubbub),
-					  GFP_KERNEL);
-
-	if (!dcn10_hubbub)
-		return NULL;
-
-	hubbub1_construct(&dcn10_hubbub->base, ctx,
-			&hubbub_reg,
-			&hubbub_shift,
-			&hubbub_mask);
-
-	return &dcn10_hubbub->base;
-}
-
-static struct timing_generator *dcn10_timing_generator_create(
-		struct dc_context *ctx,
-		uint32_t instance)
-{
-	struct optc *tgn10 =
-		kzalloc(sizeof(struct optc), GFP_KERNEL);
-
-	if (!tgn10)
-		return NULL;
-
-	tgn10->base.inst = instance;
-	tgn10->base.ctx = ctx;
-
-	tgn10->tg_regs = &tg_regs[instance];
-	tgn10->tg_shift = &tg_shift;
-	tgn10->tg_mask = &tg_mask;
-
-	dcn10_timing_generator_init(tgn10);
-
-	return &tgn10->base;
-}
-
-static const struct encoder_feature_support link_enc_feature = {
-		.max_hdmi_deep_color = COLOR_DEPTH_121212,
-		.max_hdmi_pixel_clock = 600000,
-		.hdmi_ycbcr420_supported = true,
-		.dp_ycbcr420_supported = true,
-		.flags.bits.IS_HBR2_CAPABLE = true,
-		.flags.bits.IS_HBR3_CAPABLE = true,
-		.flags.bits.IS_TPS3_CAPABLE = true,
-		.flags.bits.IS_TPS4_CAPABLE = true
-};
-
-static struct link_encoder *dcn10_link_encoder_create(
-	struct dc_context *ctx,
-	const struct encoder_init_data *enc_init_data)
-{
-	struct dcn10_link_encoder *enc10 =
-		kzalloc(sizeof(struct dcn10_link_encoder), GFP_KERNEL);
-	int link_regs_id;
-
-	if (!enc10)
-		return NULL;
-
-	link_regs_id =
-		map_transmitter_id_to_phy_instance(enc_init_data->transmitter);
-
-	dcn10_link_encoder_construct(enc10,
-				      enc_init_data,
-				      &link_enc_feature,
-				      &link_enc_regs[link_regs_id],
-				      &link_enc_aux_regs[enc_init_data->channel - 1],
-				      &link_enc_hpd_regs[enc_init_data->hpd_source],
-				      &le_shift,
-				      &le_mask);
-
-	return &enc10->base;
-}
-
-static struct panel_cntl *dcn10_panel_cntl_create(const struct panel_cntl_init_data *init_data)
-{
-	struct dce_panel_cntl *panel_cntl =
-		kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
-
-	if (!panel_cntl)
-		return NULL;
-
-	dce_panel_cntl_construct(panel_cntl,
-			init_data,
-			&panel_cntl_regs[init_data->inst],
-			&panel_cntl_shift,
-			&panel_cntl_mask);
-
-	return &panel_cntl->base;
-}
-
-static struct clock_source *dcn10_clock_source_create(
-	struct dc_context *ctx,
-	struct dc_bios *bios,
-	enum clock_source_id id,
-	const struct dce110_clk_src_regs *regs,
-	bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src =
-		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dce112_clk_src_construct(clk_src, ctx, bios, id,
-			regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-
-	kfree(clk_src);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
-static void read_dce_straps(
-	struct dc_context *ctx,
-	struct resource_straps *straps)
-{
-	generic_reg_get(ctx, mmDC_PINSTRAPS + BASE(mmDC_PINSTRAPS_BASE_IDX),
-		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
-}
-
-static struct audio *create_audio(
-		struct dc_context *ctx, unsigned int inst)
-{
-	return dce_audio_create(ctx, inst,
-			&audio_regs[inst], &audio_shift, &audio_mask);
-}
-
-static struct stream_encoder *dcn10_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dcn10_stream_encoder *enc1 =
-		kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
-
-	if (!enc1)
-		return NULL;
-
-	dcn10_stream_encoder_construct(enc1, ctx, ctx->dc_bios, eng_id,
-					&stream_enc_regs[eng_id],
-					&se_shift, &se_mask);
-	return &enc1->base;
-}
-
-static const struct dce_hwseq_registers hwseq_reg = {
-		HWSEQ_DCN1_REG_LIST()
-};
-
-static const struct dce_hwseq_shift hwseq_shift = {
-		HWSEQ_DCN1_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_hwseq_mask hwseq_mask = {
-		HWSEQ_DCN1_MASK_SH_LIST(_MASK)
-};
-
-static struct dce_hwseq *dcn10_hwseq_create(
-	struct dc_context *ctx)
-{
-	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
-
-	if (hws) {
-		hws->ctx = ctx;
-		hws->regs = &hwseq_reg;
-		hws->shifts = &hwseq_shift;
-		hws->masks = &hwseq_mask;
-		hws->wa.DEGVIDCN10_253 = true;
-		hws->wa.false_optc_underflow = true;
-		hws->wa.DEGVIDCN10_254 = true;
-
-		if ((ctx->asic_id.chip_family == FAMILY_RV) &&
-			ASICREV_IS_RAVEN2(ctx->asic_id.hw_internal_rev))
-			switch (ctx->asic_id.pci_revision_id) {
-			case PRID_POLLOCK_94:
-			case PRID_POLLOCK_95:
-			case PRID_POLLOCK_E9:
-			case PRID_POLLOCK_EA:
-			case PRID_POLLOCK_EB:
-				hws->wa.wait_hubpret_read_start_during_mpo_transition = true;
-				break;
-			default:
-				hws->wa.wait_hubpret_read_start_during_mpo_transition = false;
-				break;
-			}
-	}
-	return hws;
-}
-
-static const struct resource_create_funcs res_create_funcs = {
-	.read_dce_straps = read_dce_straps,
-	.create_audio = create_audio,
-	.create_stream_encoder = dcn10_stream_encoder_create,
-	.create_hwseq = dcn10_hwseq_create,
-};
-
-static void dcn10_clock_source_destroy(struct clock_source **clk_src)
-{
-	kfree(TO_DCE110_CLK_SRC(*clk_src));
-	*clk_src = NULL;
-}
-
-static struct pp_smu_funcs *dcn10_pp_smu_create(struct dc_context *ctx)
-{
-	struct pp_smu_funcs *pp_smu = kzalloc(sizeof(*pp_smu), GFP_KERNEL);
-
-	if (!pp_smu)
-		return pp_smu;
-
-	dm_pp_get_funcs(ctx, pp_smu);
-	return pp_smu;
-}
-
-static void dcn10_resource_destruct(struct dcn10_resource_pool *pool)
-{
-	unsigned int i;
-
-	for (i = 0; i < pool->base.stream_enc_count; i++) {
-		if (pool->base.stream_enc[i] != NULL) {
-			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
-			pool->base.stream_enc[i] = NULL;
-		}
-	}
-
-	if (pool->base.mpc != NULL) {
-		kfree(TO_DCN10_MPC(pool->base.mpc));
-		pool->base.mpc = NULL;
-	}
-
-	kfree(pool->base.hubbub);
-	pool->base.hubbub = NULL;
-
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		if (pool->base.opps[i] != NULL)
-			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
-
-		if (pool->base.dpps[i] != NULL)
-			dcn10_dpp_destroy(&pool->base.dpps[i]);
-
-		if (pool->base.ipps[i] != NULL)
-			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
-
-		if (pool->base.hubps[i] != NULL) {
-			kfree(TO_DCN10_HUBP(pool->base.hubps[i]));
-			pool->base.hubps[i] = NULL;
-		}
-
-		if (pool->base.irqs != NULL) {
-			dal_irq_service_destroy(&pool->base.irqs);
-		}
-
-		if (pool->base.timing_generators[i] != NULL)	{
-			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
-			pool->base.timing_generators[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		if (pool->base.engines[i] != NULL)
-			dce110_engine_destroy(&pool->base.engines[i]);
-		kfree(pool->base.hw_i2cs[i]);
-		pool->base.hw_i2cs[i] = NULL;
-		kfree(pool->base.sw_i2cs[i]);
-		pool->base.sw_i2cs[i] = NULL;
-	}
-
-	for (i = 0; i < pool->base.audio_count; i++) {
-		if (pool->base.audios[i])
-			dce_aud_destroy(&pool->base.audios[i]);
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] != NULL) {
-			dcn10_clock_source_destroy(&pool->base.clock_sources[i]);
-			pool->base.clock_sources[i] = NULL;
-		}
-	}
-
-	if (pool->base.dp_clock_source != NULL) {
-		dcn10_clock_source_destroy(&pool->base.dp_clock_source);
-		pool->base.dp_clock_source = NULL;
-	}
-
-	if (pool->base.abm != NULL)
-		dce_abm_destroy(&pool->base.abm);
-
-	if (pool->base.dmcu != NULL)
-		dce_dmcu_destroy(&pool->base.dmcu);
-
-	kfree(pool->base.pp_smu);
-}
-
-static struct hubp *dcn10_hubp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn10_hubp *hubp1 =
-		kzalloc(sizeof(struct dcn10_hubp), GFP_KERNEL);
-
-	if (!hubp1)
-		return NULL;
-
-	dcn10_hubp_construct(hubp1, ctx, inst,
-			     &hubp_regs[inst], &hubp_shift, &hubp_mask);
-	return &hubp1->base;
-}
-
-static void get_pixel_clock_parameters(
-	const struct pipe_ctx *pipe_ctx,
-	struct pixel_clk_params *pixel_clk_params)
-{
-	const struct dc_stream_state *stream = pipe_ctx->stream;
-	pixel_clk_params->requested_pix_clk_100hz = stream->timing.pix_clk_100hz;
-	pixel_clk_params->encoder_object_id = stream->link->link_enc->id;
-	pixel_clk_params->signal_type = pipe_ctx->stream->signal;
-	pixel_clk_params->controller_id = pipe_ctx->stream_res.tg->inst + 1;
-	/* TODO: un-hardcode*/
-	pixel_clk_params->requested_sym_clk = LINK_RATE_LOW *
-		LINK_RATE_REF_FREQ_IN_KHZ;
-	pixel_clk_params->flags.ENABLE_SS = 0;
-	pixel_clk_params->color_depth =
-		stream->timing.display_color_depth;
-	pixel_clk_params->flags.DISPLAY_BLANKED = 1;
-	pixel_clk_params->pixel_encoding = stream->timing.pixel_encoding;
-
-	if (stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR422)
-		pixel_clk_params->color_depth = COLOR_DEPTH_888;
-
-	if (stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR420)
-		pixel_clk_params->requested_pix_clk_100hz  /= 2;
-	if (stream->timing.timing_3d_format == TIMING_3D_FORMAT_HW_FRAME_PACKING)
-		pixel_clk_params->requested_pix_clk_100hz *= 2;
-
-}
-
-static void build_clamping_params(struct dc_stream_state *stream)
-{
-	stream->clamping.clamping_level = CLAMPING_FULL_RANGE;
-	stream->clamping.c_depth = stream->timing.display_color_depth;
-	stream->clamping.pixel_encoding = stream->timing.pixel_encoding;
-}
-
-static void build_pipe_hw_param(struct pipe_ctx *pipe_ctx)
-{
-
-	get_pixel_clock_parameters(pipe_ctx, &pipe_ctx->stream_res.pix_clk_params);
-
-	pipe_ctx->clock_source->funcs->get_pix_clk_dividers(
-		pipe_ctx->clock_source,
-		&pipe_ctx->stream_res.pix_clk_params,
-		&pipe_ctx->pll_settings);
-
-	pipe_ctx->stream->clamping.pixel_encoding = pipe_ctx->stream->timing.pixel_encoding;
-
-	resource_build_bit_depth_reduction_params(pipe_ctx->stream,
-					&pipe_ctx->stream->bit_depth_params);
-	build_clamping_params(pipe_ctx->stream);
-}
-
-static enum dc_status build_mapped_resource(
-		const struct dc *dc,
-		struct dc_state *context,
-		struct dc_stream_state *stream)
-{
-	struct pipe_ctx *pipe_ctx = resource_get_otg_master_for_stream(&context->res_ctx, stream);
-
-	if (!pipe_ctx)
-		return DC_ERROR_UNEXPECTED;
-
-	build_pipe_hw_param(pipe_ctx);
-	return DC_OK;
-}
-
-static enum dc_status dcn10_add_stream_to_ctx(
-		struct dc *dc,
-		struct dc_state *new_ctx,
-		struct dc_stream_state *dc_stream)
-{
-	enum dc_status result = DC_ERROR_UNEXPECTED;
-
-	result = resource_map_pool_resources(dc, new_ctx, dc_stream);
-
-	if (result == DC_OK)
-		result = resource_map_phy_clock_resources(dc, new_ctx, dc_stream);
-
-
-	if (result == DC_OK)
-		result = build_mapped_resource(dc, new_ctx, dc_stream);
-
-	return result;
-}
-
-static struct pipe_ctx *dcn10_acquire_free_pipe_for_layer(
-		const struct dc_state *cur_ctx,
-		struct dc_state *new_ctx,
-		const struct resource_pool *pool,
-		const struct pipe_ctx *opp_head_pipe)
-{
-	struct resource_context *res_ctx = &new_ctx->res_ctx;
-	struct pipe_ctx *head_pipe = resource_get_otg_master_for_stream(res_ctx, opp_head_pipe->stream);
-	struct pipe_ctx *idle_pipe = resource_find_free_secondary_pipe_legacy(res_ctx, pool, head_pipe);
-
-	if (!head_pipe) {
-		ASSERT(0);
-		return NULL;
-	}
-
-	if (!idle_pipe)
-		return NULL;
-
-	idle_pipe->stream = head_pipe->stream;
-	idle_pipe->stream_res.tg = head_pipe->stream_res.tg;
-	idle_pipe->stream_res.abm = head_pipe->stream_res.abm;
-	idle_pipe->stream_res.opp = head_pipe->stream_res.opp;
-
-	idle_pipe->plane_res.hubp = pool->hubps[idle_pipe->pipe_idx];
-	idle_pipe->plane_res.ipp = pool->ipps[idle_pipe->pipe_idx];
-	idle_pipe->plane_res.dpp = pool->dpps[idle_pipe->pipe_idx];
-	idle_pipe->plane_res.mpcc_inst = pool->dpps[idle_pipe->pipe_idx]->inst;
-
-	return idle_pipe;
-}
-
-static bool dcn10_get_dcc_compression_cap(const struct dc *dc,
-		const struct dc_dcc_surface_param *input,
-		struct dc_surface_dcc_cap *output)
-{
-	return dc->res_pool->hubbub->funcs->get_dcc_compression_cap(
-			dc->res_pool->hubbub,
-			input,
-			output);
-}
-
-static void dcn10_destroy_resource_pool(struct resource_pool **pool)
-{
-	struct dcn10_resource_pool *dcn10_pool = TO_DCN10_RES_POOL(*pool);
-
-	dcn10_resource_destruct(dcn10_pool);
-	kfree(dcn10_pool);
-	*pool = NULL;
-}
-
-static bool dcn10_validate_bandwidth(
-		struct dc *dc,
-		struct dc_state *context,
-		bool fast_validate)
-{
-	bool voltage_supported;
-
-	DC_FP_START();
-	voltage_supported = dcn_validate_bandwidth(dc, context, fast_validate);
-	DC_FP_END();
-
-	return voltage_supported;
-}
-
-static enum dc_status dcn10_validate_plane(const struct dc_plane_state *plane_state, struct dc_caps *caps)
-{
-	if (plane_state->format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN
-			&& caps->max_video_width != 0
-			&& plane_state->src_rect.width > caps->max_video_width)
-		return DC_FAIL_SURFACE_VALIDATE;
-
-	return DC_OK;
-}
-
-static enum dc_status dcn10_validate_global(struct dc *dc, struct dc_state *context)
-{
-	int i, j;
-	bool video_down_scaled = false;
-	bool video_large = false;
-	bool desktop_large = false;
-	bool dcc_disabled = false;
-	bool mpo_enabled = false;
-
-	for (i = 0; i < context->stream_count; i++) {
-		if (context->stream_status[i].plane_count == 0)
-			continue;
-
-		if (context->stream_status[i].plane_count > 2)
-			return DC_FAIL_UNSUPPORTED_1;
-
-		if (context->stream_status[i].plane_count > 1)
-			mpo_enabled = true;
-
-		for (j = 0; j < context->stream_status[i].plane_count; j++) {
-			struct dc_plane_state *plane =
-				context->stream_status[i].plane_states[j];
-
-
-			if (plane->format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN) {
-
-				if (plane->src_rect.width > plane->dst_rect.width ||
-						plane->src_rect.height > plane->dst_rect.height)
-					video_down_scaled = true;
-
-				if (plane->src_rect.width >= 3840)
-					video_large = true;
-
-			} else {
-				if (plane->src_rect.width >= 3840)
-					desktop_large = true;
-				if (!plane->dcc.enable)
-					dcc_disabled = true;
-			}
-		}
-	}
-
-	/* Disable MPO in multi-display configurations. */
-	if (context->stream_count > 1 && mpo_enabled)
-		return DC_FAIL_UNSUPPORTED_1;
-
-	/*
-	 * Workaround: On DCN10 there is UMC issue that causes underflow when
-	 * playing 4k video on 4k desktop with video downscaled and single channel
-	 * memory
-	 */
-	if (video_large && desktop_large && video_down_scaled && dcc_disabled &&
-			dc->dcn_soc->number_of_channels == 1)
-		return DC_FAIL_SURFACE_VALIDATE;
-
-	return DC_OK;
-}
-
-static enum dc_status dcn10_patch_unknown_plane_state(struct dc_plane_state *plane_state)
-{
-	enum surface_pixel_format surf_pix_format = plane_state->format;
-	unsigned int bpp = resource_pixel_format_to_bpp(surf_pix_format);
-
-	enum swizzle_mode_values swizzle = DC_SW_LINEAR;
-
-	if (bpp == 64)
-		swizzle = DC_SW_64KB_D;
-	else
-		swizzle = DC_SW_64KB_S;
-
-	plane_state->tiling_info.gfx9.swizzle = swizzle;
-	return DC_OK;
-}
-
-struct stream_encoder *dcn10_find_first_free_match_stream_enc_for_link(
-		struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		struct dc_stream_state *stream)
-{
-	int i;
-	int j = -1;
-	struct dc_link *link = stream->link;
-
-	for (i = 0; i < pool->stream_enc_count; i++) {
-		if (!res_ctx->is_stream_enc_acquired[i] &&
-				pool->stream_enc[i]) {
-			/* Store first available for MST second display
-			 * in daisy chain use case
-			 */
-			j = i;
-			if (link->ep_type == DISPLAY_ENDPOINT_PHY && pool->stream_enc[i]->id ==
-					link->link_enc->preferred_engine)
-				return pool->stream_enc[i];
-		}
-	}
-
-	/*
-	 * For CZ and later, we can allow DIG FE and BE to differ for all display types
-	 */
-
-	if (j >= 0)
-		return pool->stream_enc[j];
-
-	return NULL;
-}
-
-static const struct dc_cap_funcs cap_funcs = {
-	.get_dcc_compression_cap = dcn10_get_dcc_compression_cap
-};
-
-static const struct resource_funcs dcn10_res_pool_funcs = {
-	.destroy = dcn10_destroy_resource_pool,
-	.link_enc_create = dcn10_link_encoder_create,
-	.panel_cntl_create = dcn10_panel_cntl_create,
-	.validate_bandwidth = dcn10_validate_bandwidth,
-	.acquire_free_pipe_as_secondary_dpp_pipe = dcn10_acquire_free_pipe_for_layer,
-	.validate_plane = dcn10_validate_plane,
-	.validate_global = dcn10_validate_global,
-	.add_stream_to_ctx = dcn10_add_stream_to_ctx,
-	.patch_unknown_plane_state = dcn10_patch_unknown_plane_state,
-	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link
-};
-
-static uint32_t read_pipe_fuses(struct dc_context *ctx)
-{
-	uint32_t value = dm_read_reg_soc15(ctx, mmCC_DC_PIPE_DIS, 0);
-	/* RV1 support max 4 pipes */
-	value = value & 0xf;
-	return value;
-}
-
-static bool verify_clock_values(struct dm_pp_clock_levels_with_voltage *clks)
-{
-	int i;
-
-	if (clks->num_levels == 0)
-		return false;
-
-	for (i = 0; i < clks->num_levels; i++)
-		/* Ensure that the result is sane */
-		if (clks->data[i].clocks_in_khz == 0)
-			return false;
-
-	return true;
-}
-
-static bool dcn10_resource_construct(
-	uint8_t num_virtual_links,
-	struct dc *dc,
-	struct dcn10_resource_pool *pool)
-{
-	int i;
-	int j;
-	struct dc_context *ctx = dc->ctx;
-	uint32_t pipe_fuses = read_pipe_fuses(ctx);
-	struct dm_pp_clock_levels_with_voltage fclks = {0}, dcfclks = {0};
-	int min_fclk_khz, min_dcfclk_khz, socclk_khz;
-	bool res;
-
-	ctx->dc_bios->regs = &bios_regs;
-
-	if (ctx->dce_version == DCN_VERSION_1_01)
-		pool->base.res_cap = &rv2_res_cap;
-	else
-		pool->base.res_cap = &res_cap;
-	pool->base.funcs = &dcn10_res_pool_funcs;
-
-	/*
-	 * TODO fill in from actual raven resource when we create
-	 * more than virtual encoder
-	 */
-
-	/*************************************************
-	 *  Resource + asic cap harcoding                *
-	 *************************************************/
-	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
-
-	/* max pipe num for ASIC before check pipe fuses */
-	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
-
-	if (dc->ctx->dce_version == DCN_VERSION_1_01)
-		pool->base.pipe_count = 3;
-	dc->caps.max_video_width = 3840;
-	dc->caps.max_downscale_ratio = 200;
-	dc->caps.i2c_speed_in_khz = 100;
-	dc->caps.i2c_speed_in_khz_hdcp = 100; /*1.4 w/a not applied by default*/
-	dc->caps.max_cursor_size = 256;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.max_slave_planes = 1;
-	dc->caps.max_slave_yuv_planes = 1;
-	dc->caps.max_slave_rgb_planes = 0;
-	dc->caps.is_apu = true;
-	dc->caps.post_blend_color_processing = false;
-	dc->caps.extended_aux_timeout_support = false;
-
-	/* Raven DP PHY HBR2 eye diagram pattern is not stable. Use TP4 */
-	dc->caps.force_dp_tps4_for_cp2520 = true;
-
-	/* Color pipeline capabilities */
-	dc->caps.color.dpp.dcn_arch = 1;
-	dc->caps.color.dpp.input_lut_shared = 1;
-	dc->caps.color.dpp.icsc = 1;
-	dc->caps.color.dpp.dgam_ram = 1;
-	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
-	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.dpp.dgam_rom_caps.pq = 0;
-	dc->caps.color.dpp.dgam_rom_caps.hlg = 0;
-	dc->caps.color.dpp.post_csc = 0;
-	dc->caps.color.dpp.gamma_corr = 0;
-	dc->caps.color.dpp.dgam_rom_for_yuv = 1;
-
-	dc->caps.color.dpp.hw_3d_lut = 0;
-	dc->caps.color.dpp.ogam_ram = 1; // RGAM on DCN1
-	dc->caps.color.dpp.ogam_rom_caps.srgb = 1;
-	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 1;
-	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
-	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
-	dc->caps.color.dpp.ocsc = 1;
-
-	/* no post-blend color operations */
-	dc->caps.color.mpc.gamut_remap = 0;
-	dc->caps.color.mpc.num_3dluts = 0;
-	dc->caps.color.mpc.shared_3d_lut = 0;
-	dc->caps.color.mpc.ogam_ram = 0;
-	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
-	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
-	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
-	dc->caps.color.mpc.ocsc = 0;
-
-	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
-		dc->debug = debug_defaults_drv;
-	else
-		dc->debug = debug_defaults_diags;
-
-	/*************************************************
-	 *  Create resources                             *
-	 *************************************************/
-
-	pool->base.clock_sources[DCN10_CLK_SRC_PLL0] =
-			dcn10_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL0,
-				&clk_src_regs[0], false);
-	pool->base.clock_sources[DCN10_CLK_SRC_PLL1] =
-			dcn10_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL1,
-				&clk_src_regs[1], false);
-	pool->base.clock_sources[DCN10_CLK_SRC_PLL2] =
-			dcn10_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL2,
-				&clk_src_regs[2], false);
-
-	if (dc->ctx->dce_version == DCN_VERSION_1_0) {
-		pool->base.clock_sources[DCN10_CLK_SRC_PLL3] =
-				dcn10_clock_source_create(ctx, ctx->dc_bios,
-					CLOCK_SOURCE_COMBO_PHY_PLL3,
-					&clk_src_regs[3], false);
-	}
-
-	pool->base.clk_src_count = DCN10_CLK_SRC_TOTAL;
-
-	if (dc->ctx->dce_version == DCN_VERSION_1_01)
-		pool->base.clk_src_count = DCN101_CLK_SRC_TOTAL;
-
-	pool->base.dp_clock_source =
-			dcn10_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_ID_DP_DTO,
-				/* todo: not reuse phy_pll registers */
-				&clk_src_regs[0], true);
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			BREAK_TO_DEBUGGER();
-			goto fail;
-		}
-	}
-
-	pool->base.dmcu = dcn10_dmcu_create(ctx,
-			&dmcu_regs,
-			&dmcu_shift,
-			&dmcu_mask);
-	if (pool->base.dmcu == NULL) {
-		dm_error("DC: failed to create dmcu!\n");
-		BREAK_TO_DEBUGGER();
-		goto fail;
-	}
-
-	pool->base.abm = dce_abm_create(ctx,
-			&abm_regs,
-			&abm_shift,
-			&abm_mask);
-	if (pool->base.abm == NULL) {
-		dm_error("DC: failed to create abm!\n");
-		BREAK_TO_DEBUGGER();
-		goto fail;
-	}
-
-	dml_init_instance(&dc->dml, &dcn1_0_soc, &dcn1_0_ip, DML_PROJECT_RAVEN1);
-	memcpy(dc->dcn_ip, &dcn10_ip_defaults, sizeof(dcn10_ip_defaults));
-	memcpy(dc->dcn_soc, &dcn10_soc_defaults, sizeof(dcn10_soc_defaults));
-
-	DC_FP_START();
-	dcn10_resource_construct_fp(dc);
-	DC_FP_END();
-
-	if (!dc->config.is_vmin_only_asic)
-		if (ASICREV_IS_RAVEN2(dc->ctx->asic_id.hw_internal_rev))
-			switch (dc->ctx->asic_id.pci_revision_id) {
-			case PRID_DALI_DE:
-			case PRID_DALI_DF:
-			case PRID_DALI_E3:
-			case PRID_DALI_E4:
-			case PRID_POLLOCK_94:
-			case PRID_POLLOCK_95:
-			case PRID_POLLOCK_E9:
-			case PRID_POLLOCK_EA:
-			case PRID_POLLOCK_EB:
-				dc->config.is_vmin_only_asic = true;
-				break;
-			default:
-				break;
-			}
-
-	pool->base.pp_smu = dcn10_pp_smu_create(ctx);
-
-	/*
-	 * Right now SMU/PPLIB and DAL all have the AZ D3 force PME notification *
-	 * implemented. So AZ D3 should work.For issue 197007.                   *
-	 */
-	if (pool->base.pp_smu != NULL
-			&& pool->base.pp_smu->rv_funcs.set_pme_wa_enable != NULL)
-		dc->debug.az_endpoint_mute_only = false;
-
-
-	if (!dc->debug.disable_pplib_clock_request) {
-		/*
-		 * TODO: This is not the proper way to obtain
-		 * fabric_and_dram_bandwidth, should be min(fclk, memclk).
-		 */
-		res = dm_pp_get_clock_levels_by_type_with_voltage(
-				ctx, DM_PP_CLOCK_TYPE_FCLK, &fclks);
-
-		DC_FP_START();
-
-		if (res)
-			res = verify_clock_values(&fclks);
-
-		if (res)
-			dcn_bw_update_from_pplib_fclks(dc, &fclks);
-		else
-			BREAK_TO_DEBUGGER();
-
-		DC_FP_END();
-
-		res = dm_pp_get_clock_levels_by_type_with_voltage(
-			ctx, DM_PP_CLOCK_TYPE_DCFCLK, &dcfclks);
-
-		DC_FP_START();
-
-		if (res)
-			res = verify_clock_values(&dcfclks);
-
-		if (res)
-			dcn_bw_update_from_pplib_dcfclks(dc, &dcfclks);
-		else
-			BREAK_TO_DEBUGGER();
-
-		DC_FP_END();
-	}
-
-	dcn_bw_sync_calcs_and_dml(dc);
-	if (!dc->debug.disable_pplib_wm_range) {
-		dc->res_pool = &pool->base;
-		DC_FP_START();
-		dcn_get_soc_clks(
-			dc, &min_fclk_khz, &min_dcfclk_khz, &socclk_khz);
-		DC_FP_END();
-		dcn_bw_notify_pplib_of_wm_ranges(
-			dc, min_fclk_khz, min_dcfclk_khz, socclk_khz);
-	}
-
-	{
-		struct irq_service_init_data init_data;
-		init_data.ctx = dc->ctx;
-		pool->base.irqs = dal_irq_service_dcn10_create(&init_data);
-		if (!pool->base.irqs)
-			goto fail;
-	}
-
-	/* index to valid pipe resource  */
-	j = 0;
-	/* mem input -> ipp -> dpp -> opp -> TG */
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		/* if pipe is disabled, skip instance of HW pipe,
-		 * i.e, skip ASIC register instance
-		 */
-		if ((pipe_fuses & (1 << i)) != 0)
-			continue;
-
-		pool->base.hubps[j] = dcn10_hubp_create(ctx, i);
-		if (pool->base.hubps[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create memory input!\n");
-			goto fail;
-		}
-
-		pool->base.ipps[j] = dcn10_ipp_create(ctx, i);
-		if (pool->base.ipps[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create input pixel processor!\n");
-			goto fail;
-		}
-
-		pool->base.dpps[j] = dcn10_dpp_create(ctx, i);
-		if (pool->base.dpps[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create dpp!\n");
-			goto fail;
-		}
-
-		pool->base.opps[j] = dcn10_opp_create(ctx, i);
-		if (pool->base.opps[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create output pixel processor!\n");
-			goto fail;
-		}
-
-		pool->base.timing_generators[j] = dcn10_timing_generator_create(
-				ctx, i);
-		if (pool->base.timing_generators[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create tg!\n");
-			goto fail;
-		}
-		/* check next valid pipe */
-		j++;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		pool->base.engines[i] = dcn10_aux_engine_create(ctx, i);
-		if (pool->base.engines[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create aux engine!!\n");
-			goto fail;
-		}
-		pool->base.hw_i2cs[i] = dcn10_i2c_hw_create(ctx, i);
-		if (pool->base.hw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create hw i2c!!\n");
-			goto fail;
-		}
-		pool->base.sw_i2cs[i] = NULL;
-	}
-
-	/* valid pipe num */
-	pool->base.pipe_count = j;
-	pool->base.timing_generator_count = j;
-
-	/* within dml lib, it is hard code to 4. If ASIC pipe is fused,
-	 * the value may be changed
-	 */
-	dc->dml.ip.max_num_dpp = pool->base.pipe_count;
-	dc->dcn_ip->max_num_dpp = pool->base.pipe_count;
-
-	pool->base.mpc = dcn10_mpc_create(ctx);
-	if (pool->base.mpc == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mpc!\n");
-		goto fail;
-	}
-
-	pool->base.hubbub = dcn10_hubbub_create(ctx);
-	if (pool->base.hubbub == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create hubbub!\n");
-		goto fail;
-	}
-
-	if (!resource_construct(num_virtual_links, dc, &pool->base,
-			&res_create_funcs))
-		goto fail;
-
-	dcn10_hw_sequencer_construct(dc);
-	dc->caps.max_planes =  pool->base.pipe_count;
-
-	for (i = 0; i < dc->caps.max_planes; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	dc->cap_funcs = cap_funcs;
-
-	return true;
-
-fail:
-
-	dcn10_resource_destruct(pool);
-
-	return false;
-}
-
-struct resource_pool *dcn10_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc)
-{
-	struct dcn10_resource_pool *pool =
-		kzalloc(sizeof(struct dcn10_resource_pool), GFP_KERNEL);
-
-	if (!pool)
-		return NULL;
-
-	if (dcn10_resource_construct(init_data->num_virtual_links, dc, pool))
-		return &pool->base;
-
-	kfree(pool);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.h
deleted file mode 100644
index bf8e33cd8147..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.h
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
-* Copyright 2016 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#ifndef __DC_RESOURCE_DCN10_H__
-#define __DC_RESOURCE_DCN10_H__
-
-#include "core_types.h"
-#include "dml/dcn10/dcn10_fpu.h"
-
-#define TO_DCN10_RES_POOL(pool)\
-	container_of(pool, struct dcn10_resource_pool, base)
-
-struct dc;
-struct resource_pool;
-struct _vcs_dpi_display_pipe_params_st;
-
-extern struct _vcs_dpi_ip_params_st dcn1_0_ip;
-extern struct _vcs_dpi_soc_bounding_box_st dcn1_0_soc;
-
-struct dcn10_resource_pool {
-	struct resource_pool base;
-};
-struct resource_pool *dcn10_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc);
-
-struct stream_encoder *dcn10_find_first_free_match_stream_enc_for_link(
-		struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		struct dc_stream_state *stream);
-
-
-#endif /* __DC_RESOURCE_DCN10_H__ */
-
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/Makefile b/drivers/gpu/drm/amd/display/dc/dcn20/Makefile
index d7dc9696a8c8..1cac1eca8111 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/Makefile
@@ -2,7 +2,7 @@
 #
 # Makefile for DCN.
 
-DCN20 = dcn20_resource.o dcn20_init.o dcn20_dpp.o dcn20_dpp_cm.o dcn20_hubp.o \
+DCN20 = dcn20_init.o dcn20_dpp.o dcn20_dpp_cm.o dcn20_hubp.o \
 		dcn20_mpc.o dcn20_opp.o dcn20_hubbub.o dcn20_optc.o dcn20_mmhubbub.o \
 		dcn20_stream_encoder.o dcn20_link_encoder.o dcn20_dccg.o \
 		dcn20_vmid.o dcn20_dwb.o dcn20_dwb_scl.o
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.h
index ab6d09c6fe34..76da59d8caaf 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.h
@@ -291,7 +291,11 @@
 	type SYMCLKB_FE_SRC_SEL;\
 	type SYMCLKC_FE_SRC_SEL;\
 	type SYMCLKD_FE_SRC_SEL;\
-	type SYMCLKE_FE_SRC_SEL;
+	type SYMCLKE_FE_SRC_SEL;\
+	type DTBCLK_P0_GATE_DISABLE;\
+	type DTBCLK_P1_GATE_DISABLE;\
+	type DTBCLK_P2_GATE_DISABLE;\
+	type DTBCLK_P3_GATE_DISABLE;\
 
 struct dccg_shift {
 	DCCG_REG_FIELD_LIST(uint8_t)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c
deleted file mode 100644
index 0a422fbb14bc..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c
+++ /dev/null
@@ -1,2789 +0,0 @@
-/*
-* Copyright 2016 Advanced Micro Devices, Inc.
- * Copyright 2019 Raptor Engineering, LLC
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#include <linux/slab.h>
-
-#include "dm_services.h"
-#include "dc.h"
-
-#include "dcn20_init.h"
-
-#include "resource.h"
-#include "include/irq_service_interface.h"
-#include "dcn20/dcn20_resource.h"
-
-#include "dml/dcn20/dcn20_fpu.h"
-
-#include "dcn10/dcn10_hubp.h"
-#include "dcn10/dcn10_ipp.h"
-#include "dcn20_hubbub.h"
-#include "dcn20_mpc.h"
-#include "dcn20_hubp.h"
-#include "irq/dcn20/irq_service_dcn20.h"
-#include "dcn20_dpp.h"
-#include "dcn20_optc.h"
-#include "dcn20/dcn20_hwseq.h"
-#include "dce110/dce110_hwseq.h"
-#include "dcn10/dcn10_resource.h"
-#include "dcn20_opp.h"
-
-#include "dcn20_dsc.h"
-
-#include "dcn20_link_encoder.h"
-#include "dcn20_stream_encoder.h"
-#include "dce/dce_clock_source.h"
-#include "dce/dce_audio.h"
-#include "dce/dce_hwseq.h"
-#include "virtual/virtual_stream_encoder.h"
-#include "dce110/dce110_resource.h"
-#include "dml/display_mode_vba.h"
-#include "dcn20_dccg.h"
-#include "dcn20_vmid.h"
-#include "dce/dce_panel_cntl.h"
-
-#include "navi10_ip_offset.h"
-
-#include "dcn/dcn_2_0_0_offset.h"
-#include "dcn/dcn_2_0_0_sh_mask.h"
-#include "dpcs/dpcs_2_0_0_offset.h"
-#include "dpcs/dpcs_2_0_0_sh_mask.h"
-
-#include "nbio/nbio_2_3_offset.h"
-
-#include "dcn20/dcn20_dwb.h"
-#include "dcn20/dcn20_mmhubbub.h"
-
-#include "mmhub/mmhub_2_0_0_offset.h"
-#include "mmhub/mmhub_2_0_0_sh_mask.h"
-
-#include "reg_helper.h"
-#include "dce/dce_abm.h"
-#include "dce/dce_dmcu.h"
-#include "dce/dce_aux.h"
-#include "dce/dce_i2c.h"
-#include "vm_helper.h"
-#include "link_enc_cfg.h"
-
-#include "amdgpu_socbb.h"
-
-#include "link.h"
-#define DC_LOGGER_INIT(logger)
-
-#ifndef mmDP0_DP_DPHY_INTERNAL_CTRL
-	#define mmDP0_DP_DPHY_INTERNAL_CTRL		0x210f
-	#define mmDP0_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
-	#define mmDP1_DP_DPHY_INTERNAL_CTRL		0x220f
-	#define mmDP1_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
-	#define mmDP2_DP_DPHY_INTERNAL_CTRL		0x230f
-	#define mmDP2_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
-	#define mmDP3_DP_DPHY_INTERNAL_CTRL		0x240f
-	#define mmDP3_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
-	#define mmDP4_DP_DPHY_INTERNAL_CTRL		0x250f
-	#define mmDP4_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
-	#define mmDP5_DP_DPHY_INTERNAL_CTRL		0x260f
-	#define mmDP5_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
-	#define mmDP6_DP_DPHY_INTERNAL_CTRL		0x270f
-	#define mmDP6_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
-#endif
-
-
-enum dcn20_clk_src_array_id {
-	DCN20_CLK_SRC_PLL0,
-	DCN20_CLK_SRC_PLL1,
-	DCN20_CLK_SRC_PLL2,
-	DCN20_CLK_SRC_PLL3,
-	DCN20_CLK_SRC_PLL4,
-	DCN20_CLK_SRC_PLL5,
-	DCN20_CLK_SRC_TOTAL
-};
-
-/* begin *********************
- * macros to expend register list macro defined in HW object header file */
-
-/* DCN */
-#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
-
-#define BASE(seg) BASE_INNER(seg)
-
-#define SR(reg_name)\
-		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) +  \
-					mm ## reg_name
-
-#define SRI(reg_name, block, id)\
-	.reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define SRI2_DWB(reg_name, block, id)\
-	.reg_name = BASE(mm ## reg_name ## _BASE_IDX) + \
-					mm ## reg_name
-#define SF_DWB(reg_name, field_name, post_fix)\
-	.field_name = reg_name ## __ ## field_name ## post_fix
-
-#define SF_DWB2(reg_name, block, id, field_name, post_fix)	\
-	.field_name = reg_name ## __ ## field_name ## post_fix
-
-#define SRIR(var_name, reg_name, block, id)\
-	.var_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define DCCG_SRII(reg_name, block, id)\
-	.block ## _ ## reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define VUPDATE_SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(mm ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
-					mm ## reg_name ## _ ## block ## id
-
-/* NBIO */
-#define NBIO_BASE_INNER(seg) \
-	NBIO_BASE__INST0_SEG ## seg
-
-#define NBIO_BASE(seg) \
-	NBIO_BASE_INNER(seg)
-
-#define NBIO_SR(reg_name)\
-		.reg_name = NBIO_BASE(mm ## reg_name ## _BASE_IDX) + \
-					mm ## reg_name
-
-/* MMHUB */
-#define MMHUB_BASE_INNER(seg) \
-	MMHUB_BASE__INST0_SEG ## seg
-
-#define MMHUB_BASE(seg) \
-	MMHUB_BASE_INNER(seg)
-
-#define MMHUB_SR(reg_name)\
-		.reg_name = MMHUB_BASE(mmMM ## reg_name ## _BASE_IDX) + \
-					mmMM ## reg_name
-
-static const struct bios_registers bios_regs = {
-		NBIO_SR(BIOS_SCRATCH_3),
-		NBIO_SR(BIOS_SCRATCH_6)
-};
-
-#define clk_src_regs(index, pllid)\
-[index] = {\
-	CS_COMMON_REG_LIST_DCN2_0(index, pllid),\
-}
-
-static const struct dce110_clk_src_regs clk_src_regs[] = {
-	clk_src_regs(0, A),
-	clk_src_regs(1, B),
-	clk_src_regs(2, C),
-	clk_src_regs(3, D),
-	clk_src_regs(4, E),
-	clk_src_regs(5, F)
-};
-
-static const struct dce110_clk_src_shift cs_shift = {
-		CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
-};
-
-static const struct dce110_clk_src_mask cs_mask = {
-		CS_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
-};
-
-static const struct dce_dmcu_registers dmcu_regs = {
-		DMCU_DCN10_REG_LIST()
-};
-
-static const struct dce_dmcu_shift dmcu_shift = {
-		DMCU_MASK_SH_LIST_DCN10(__SHIFT)
-};
-
-static const struct dce_dmcu_mask dmcu_mask = {
-		DMCU_MASK_SH_LIST_DCN10(_MASK)
-};
-
-static const struct dce_abm_registers abm_regs = {
-		ABM_DCN20_REG_LIST()
-};
-
-static const struct dce_abm_shift abm_shift = {
-		ABM_MASK_SH_LIST_DCN20(__SHIFT)
-};
-
-static const struct dce_abm_mask abm_mask = {
-		ABM_MASK_SH_LIST_DCN20(_MASK)
-};
-
-#define audio_regs(id)\
-[id] = {\
-		AUD_COMMON_REG_LIST(id)\
-}
-
-static const struct dce_audio_registers audio_regs[] = {
-	audio_regs(0),
-	audio_regs(1),
-	audio_regs(2),
-	audio_regs(3),
-	audio_regs(4),
-	audio_regs(5),
-	audio_regs(6),
-};
-
-#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
-		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
-
-static const struct dce_audio_shift audio_shift = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_audio_mask audio_mask = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
-};
-
-#define stream_enc_regs(id)\
-[id] = {\
-	SE_DCN2_REG_LIST(id)\
-}
-
-static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
-	stream_enc_regs(0),
-	stream_enc_regs(1),
-	stream_enc_regs(2),
-	stream_enc_regs(3),
-	stream_enc_regs(4),
-	stream_enc_regs(5),
-};
-
-static const struct dcn10_stream_encoder_shift se_shift = {
-		SE_COMMON_MASK_SH_LIST_DCN20(__SHIFT)
-};
-
-static const struct dcn10_stream_encoder_mask se_mask = {
-		SE_COMMON_MASK_SH_LIST_DCN20(_MASK)
-};
-
-
-#define aux_regs(id)\
-[id] = {\
-	DCN2_AUX_REG_LIST(id)\
-}
-
-static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
-		aux_regs(0),
-		aux_regs(1),
-		aux_regs(2),
-		aux_regs(3),
-		aux_regs(4),
-		aux_regs(5)
-};
-
-#define hpd_regs(id)\
-[id] = {\
-	HPD_REG_LIST(id)\
-}
-
-static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
-		hpd_regs(0),
-		hpd_regs(1),
-		hpd_regs(2),
-		hpd_regs(3),
-		hpd_regs(4),
-		hpd_regs(5)
-};
-
-#define link_regs(id, phyid)\
-[id] = {\
-	LE_DCN10_REG_LIST(id), \
-	UNIPHY_DCN2_REG_LIST(phyid), \
-	DPCS_DCN2_REG_LIST(id), \
-	SRI(DP_DPHY_INTERNAL_CTRL, DP, id) \
-}
-
-static const struct dcn10_link_enc_registers link_enc_regs[] = {
-	link_regs(0, A),
-	link_regs(1, B),
-	link_regs(2, C),
-	link_regs(3, D),
-	link_regs(4, E),
-	link_regs(5, F)
-};
-
-static const struct dcn10_link_enc_shift le_shift = {
-	LINK_ENCODER_MASK_SH_LIST_DCN20(__SHIFT),\
-	DPCS_DCN2_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn10_link_enc_mask le_mask = {
-	LINK_ENCODER_MASK_SH_LIST_DCN20(_MASK),\
-	DPCS_DCN2_MASK_SH_LIST(_MASK)
-};
-
-static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
-	{ DCN_PANEL_CNTL_REG_LIST() }
-};
-
-static const struct dce_panel_cntl_shift panel_cntl_shift = {
-	DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_panel_cntl_mask panel_cntl_mask = {
-	DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
-};
-
-#define ipp_regs(id)\
-[id] = {\
-	IPP_REG_LIST_DCN20(id),\
-}
-
-static const struct dcn10_ipp_registers ipp_regs[] = {
-	ipp_regs(0),
-	ipp_regs(1),
-	ipp_regs(2),
-	ipp_regs(3),
-	ipp_regs(4),
-	ipp_regs(5),
-};
-
-static const struct dcn10_ipp_shift ipp_shift = {
-		IPP_MASK_SH_LIST_DCN20(__SHIFT)
-};
-
-static const struct dcn10_ipp_mask ipp_mask = {
-		IPP_MASK_SH_LIST_DCN20(_MASK),
-};
-
-#define opp_regs(id)\
-[id] = {\
-	OPP_REG_LIST_DCN20(id),\
-}
-
-static const struct dcn20_opp_registers opp_regs[] = {
-	opp_regs(0),
-	opp_regs(1),
-	opp_regs(2),
-	opp_regs(3),
-	opp_regs(4),
-	opp_regs(5),
-};
-
-static const struct dcn20_opp_shift opp_shift = {
-		OPP_MASK_SH_LIST_DCN20(__SHIFT)
-};
-
-static const struct dcn20_opp_mask opp_mask = {
-		OPP_MASK_SH_LIST_DCN20(_MASK)
-};
-
-#define aux_engine_regs(id)\
-[id] = {\
-	AUX_COMMON_REG_LIST0(id), \
-	.AUXN_IMPCAL = 0, \
-	.AUXP_IMPCAL = 0, \
-	.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
-}
-
-static const struct dce110_aux_registers aux_engine_regs[] = {
-		aux_engine_regs(0),
-		aux_engine_regs(1),
-		aux_engine_regs(2),
-		aux_engine_regs(3),
-		aux_engine_regs(4),
-		aux_engine_regs(5)
-};
-
-#define tf_regs(id)\
-[id] = {\
-	TF_REG_LIST_DCN20(id),\
-	TF_REG_LIST_DCN20_COMMON_APPEND(id),\
-}
-
-static const struct dcn2_dpp_registers tf_regs[] = {
-	tf_regs(0),
-	tf_regs(1),
-	tf_regs(2),
-	tf_regs(3),
-	tf_regs(4),
-	tf_regs(5),
-};
-
-static const struct dcn2_dpp_shift tf_shift = {
-		TF_REG_LIST_SH_MASK_DCN20(__SHIFT),
-		TF_DEBUG_REG_LIST_SH_DCN20
-};
-
-static const struct dcn2_dpp_mask tf_mask = {
-		TF_REG_LIST_SH_MASK_DCN20(_MASK),
-		TF_DEBUG_REG_LIST_MASK_DCN20
-};
-
-#define dwbc_regs_dcn2(id)\
-[id] = {\
-	DWBC_COMMON_REG_LIST_DCN2_0(id),\
-		}
-
-static const struct dcn20_dwbc_registers dwbc20_regs[] = {
-	dwbc_regs_dcn2(0),
-};
-
-static const struct dcn20_dwbc_shift dwbc20_shift = {
-	DWBC_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
-};
-
-static const struct dcn20_dwbc_mask dwbc20_mask = {
-	DWBC_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
-};
-
-#define mcif_wb_regs_dcn2(id)\
-[id] = {\
-	MCIF_WB_COMMON_REG_LIST_DCN2_0(id),\
-		}
-
-static const struct dcn20_mmhubbub_registers mcif_wb20_regs[] = {
-	mcif_wb_regs_dcn2(0),
-};
-
-static const struct dcn20_mmhubbub_shift mcif_wb20_shift = {
-	MCIF_WB_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
-};
-
-static const struct dcn20_mmhubbub_mask mcif_wb20_mask = {
-	MCIF_WB_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
-};
-
-static const struct dcn20_mpc_registers mpc_regs = {
-		MPC_REG_LIST_DCN2_0(0),
-		MPC_REG_LIST_DCN2_0(1),
-		MPC_REG_LIST_DCN2_0(2),
-		MPC_REG_LIST_DCN2_0(3),
-		MPC_REG_LIST_DCN2_0(4),
-		MPC_REG_LIST_DCN2_0(5),
-		MPC_OUT_MUX_REG_LIST_DCN2_0(0),
-		MPC_OUT_MUX_REG_LIST_DCN2_0(1),
-		MPC_OUT_MUX_REG_LIST_DCN2_0(2),
-		MPC_OUT_MUX_REG_LIST_DCN2_0(3),
-		MPC_OUT_MUX_REG_LIST_DCN2_0(4),
-		MPC_OUT_MUX_REG_LIST_DCN2_0(5),
-		MPC_DBG_REG_LIST_DCN2_0()
-};
-
-static const struct dcn20_mpc_shift mpc_shift = {
-	MPC_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT),
-	MPC_DEBUG_REG_LIST_SH_DCN20
-};
-
-static const struct dcn20_mpc_mask mpc_mask = {
-	MPC_COMMON_MASK_SH_LIST_DCN2_0(_MASK),
-	MPC_DEBUG_REG_LIST_MASK_DCN20
-};
-
-#define tg_regs(id)\
-[id] = {TG_COMMON_REG_LIST_DCN2_0(id)}
-
-
-static const struct dcn_optc_registers tg_regs[] = {
-	tg_regs(0),
-	tg_regs(1),
-	tg_regs(2),
-	tg_regs(3),
-	tg_regs(4),
-	tg_regs(5)
-};
-
-static const struct dcn_optc_shift tg_shift = {
-	TG_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
-};
-
-static const struct dcn_optc_mask tg_mask = {
-	TG_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
-};
-
-#define hubp_regs(id)\
-[id] = {\
-	HUBP_REG_LIST_DCN20(id)\
-}
-
-static const struct dcn_hubp2_registers hubp_regs[] = {
-		hubp_regs(0),
-		hubp_regs(1),
-		hubp_regs(2),
-		hubp_regs(3),
-		hubp_regs(4),
-		hubp_regs(5)
-};
-
-static const struct dcn_hubp2_shift hubp_shift = {
-		HUBP_MASK_SH_LIST_DCN20(__SHIFT)
-};
-
-static const struct dcn_hubp2_mask hubp_mask = {
-		HUBP_MASK_SH_LIST_DCN20(_MASK)
-};
-
-static const struct dcn_hubbub_registers hubbub_reg = {
-		HUBBUB_REG_LIST_DCN20(0)
-};
-
-static const struct dcn_hubbub_shift hubbub_shift = {
-		HUBBUB_MASK_SH_LIST_DCN20(__SHIFT)
-};
-
-static const struct dcn_hubbub_mask hubbub_mask = {
-		HUBBUB_MASK_SH_LIST_DCN20(_MASK)
-};
-
-#define vmid_regs(id)\
-[id] = {\
-		DCN20_VMID_REG_LIST(id)\
-}
-
-static const struct dcn_vmid_registers vmid_regs[] = {
-	vmid_regs(0),
-	vmid_regs(1),
-	vmid_regs(2),
-	vmid_regs(3),
-	vmid_regs(4),
-	vmid_regs(5),
-	vmid_regs(6),
-	vmid_regs(7),
-	vmid_regs(8),
-	vmid_regs(9),
-	vmid_regs(10),
-	vmid_regs(11),
-	vmid_regs(12),
-	vmid_regs(13),
-	vmid_regs(14),
-	vmid_regs(15)
-};
-
-static const struct dcn20_vmid_shift vmid_shifts = {
-		DCN20_VMID_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn20_vmid_mask vmid_masks = {
-		DCN20_VMID_MASK_SH_LIST(_MASK)
-};
-
-static const struct dce110_aux_registers_shift aux_shift = {
-		DCN_AUX_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce110_aux_registers_mask aux_mask = {
-		DCN_AUX_MASK_SH_LIST(_MASK)
-};
-
-static int map_transmitter_id_to_phy_instance(
-	enum transmitter transmitter)
-{
-	switch (transmitter) {
-	case TRANSMITTER_UNIPHY_A:
-		return 0;
-	break;
-	case TRANSMITTER_UNIPHY_B:
-		return 1;
-	break;
-	case TRANSMITTER_UNIPHY_C:
-		return 2;
-	break;
-	case TRANSMITTER_UNIPHY_D:
-		return 3;
-	break;
-	case TRANSMITTER_UNIPHY_E:
-		return 4;
-	break;
-	case TRANSMITTER_UNIPHY_F:
-		return 5;
-	break;
-	default:
-		ASSERT(0);
-		return 0;
-	}
-}
-
-#define dsc_regsDCN20(id)\
-[id] = {\
-	DSC_REG_LIST_DCN20(id)\
-}
-
-static const struct dcn20_dsc_registers dsc_regs[] = {
-	dsc_regsDCN20(0),
-	dsc_regsDCN20(1),
-	dsc_regsDCN20(2),
-	dsc_regsDCN20(3),
-	dsc_regsDCN20(4),
-	dsc_regsDCN20(5)
-};
-
-static const struct dcn20_dsc_shift dsc_shift = {
-	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
-};
-
-static const struct dcn20_dsc_mask dsc_mask = {
-	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
-};
-
-static const struct dccg_registers dccg_regs = {
-		DCCG_REG_LIST_DCN2()
-};
-
-static const struct dccg_shift dccg_shift = {
-		DCCG_MASK_SH_LIST_DCN2(__SHIFT)
-};
-
-static const struct dccg_mask dccg_mask = {
-		DCCG_MASK_SH_LIST_DCN2(_MASK)
-};
-
-static const struct resource_caps res_cap_nv10 = {
-		.num_timing_generator = 6,
-		.num_opp = 6,
-		.num_video_plane = 6,
-		.num_audio = 7,
-		.num_stream_encoder = 6,
-		.num_pll = 6,
-		.num_dwb = 1,
-		.num_ddc = 6,
-		.num_vmid = 16,
-		.num_dsc = 6,
-};
-
-static const struct dc_plane_cap plane_cap = {
-	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.per_pixel_alpha = true,
-
-	.pixel_format_support = {
-			.argb8888 = true,
-			.nv12 = true,
-			.fp16 = true,
-			.p010 = true
-	},
-
-	.max_upscale_factor = {
-			.argb8888 = 16000,
-			.nv12 = 16000,
-			.fp16 = 1
-	},
-
-	.max_downscale_factor = {
-			.argb8888 = 250,
-			.nv12 = 250,
-			.fp16 = 1
-	},
-	16,
-	16
-};
-static const struct resource_caps res_cap_nv14 = {
-		.num_timing_generator = 5,
-		.num_opp = 5,
-		.num_video_plane = 5,
-		.num_audio = 6,
-		.num_stream_encoder = 5,
-		.num_pll = 5,
-		.num_dwb = 1,
-		.num_ddc = 5,
-		.num_vmid = 16,
-		.num_dsc = 5,
-};
-
-static const struct dc_debug_options debug_defaults_drv = {
-		.disable_dmcu = false,
-		.force_abm_enable = false,
-		.timing_trace = false,
-		.clock_trace = true,
-		.disable_pplib_clock_request = true,
-		.pipe_split_policy = MPC_SPLIT_AVOID_MULT_DISP,
-		.force_single_disp_pipe_split = false,
-		.disable_dcc = DCC_ENABLE,
-		.vsr_support = true,
-		.performance_trace = false,
-		.max_downscale_src_width = 5120,/*upto 5K*/
-		.disable_pplib_wm_range = false,
-		.scl_reset_length10 = true,
-		.sanity_checks = false,
-		.underflow_assert_delay_us = 0xFFFFFFFF,
-		.enable_legacy_fast_update = true,
-		.using_dml2 = false,
-};
-
-void dcn20_dpp_destroy(struct dpp **dpp)
-{
-	kfree(TO_DCN20_DPP(*dpp));
-	*dpp = NULL;
-}
-
-struct dpp *dcn20_dpp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn20_dpp *dpp =
-		kzalloc(sizeof(struct dcn20_dpp), GFP_ATOMIC);
-
-	if (!dpp)
-		return NULL;
-
-	if (dpp2_construct(dpp, ctx, inst,
-			&tf_regs[inst], &tf_shift, &tf_mask))
-		return &dpp->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(dpp);
-	return NULL;
-}
-
-struct input_pixel_processor *dcn20_ipp_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn10_ipp *ipp =
-		kzalloc(sizeof(struct dcn10_ipp), GFP_ATOMIC);
-
-	if (!ipp) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dcn20_ipp_construct(ipp, ctx, inst,
-			&ipp_regs[inst], &ipp_shift, &ipp_mask);
-	return &ipp->base;
-}
-
-
-struct output_pixel_processor *dcn20_opp_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_opp *opp =
-		kzalloc(sizeof(struct dcn20_opp), GFP_ATOMIC);
-
-	if (!opp) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dcn20_opp_construct(opp, ctx, inst,
-			&opp_regs[inst], &opp_shift, &opp_mask);
-	return &opp->base;
-}
-
-struct dce_aux *dcn20_aux_engine_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct aux_engine_dce110 *aux_engine =
-		kzalloc(sizeof(struct aux_engine_dce110), GFP_ATOMIC);
-
-	if (!aux_engine)
-		return NULL;
-
-	dce110_aux_engine_construct(aux_engine, ctx, inst,
-				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
-				    &aux_engine_regs[inst],
-					&aux_mask,
-					&aux_shift,
-					ctx->dc->caps.extended_aux_timeout_support);
-
-	return &aux_engine->base;
-}
-#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
-
-static const struct dce_i2c_registers i2c_hw_regs[] = {
-		i2c_inst_regs(1),
-		i2c_inst_regs(2),
-		i2c_inst_regs(3),
-		i2c_inst_regs(4),
-		i2c_inst_regs(5),
-		i2c_inst_regs(6),
-};
-
-static const struct dce_i2c_shift i2c_shifts = {
-		I2C_COMMON_MASK_SH_LIST_DCN2(__SHIFT)
-};
-
-static const struct dce_i2c_mask i2c_masks = {
-		I2C_COMMON_MASK_SH_LIST_DCN2(_MASK)
-};
-
-struct dce_i2c_hw *dcn20_i2c_hw_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_i2c_hw *dce_i2c_hw =
-		kzalloc(sizeof(struct dce_i2c_hw), GFP_ATOMIC);
-
-	if (!dce_i2c_hw)
-		return NULL;
-
-	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
-				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
-
-	return dce_i2c_hw;
-}
-struct mpc *dcn20_mpc_create(struct dc_context *ctx)
-{
-	struct dcn20_mpc *mpc20 = kzalloc(sizeof(struct dcn20_mpc),
-					  GFP_ATOMIC);
-
-	if (!mpc20)
-		return NULL;
-
-	dcn20_mpc_construct(mpc20, ctx,
-			&mpc_regs,
-			&mpc_shift,
-			&mpc_mask,
-			6);
-
-	return &mpc20->base;
-}
-
-struct hubbub *dcn20_hubbub_create(struct dc_context *ctx)
-{
-	int i;
-	struct dcn20_hubbub *hubbub = kzalloc(sizeof(struct dcn20_hubbub),
-					  GFP_ATOMIC);
-
-	if (!hubbub)
-		return NULL;
-
-	hubbub2_construct(hubbub, ctx,
-			&hubbub_reg,
-			&hubbub_shift,
-			&hubbub_mask);
-
-	for (i = 0; i < res_cap_nv10.num_vmid; i++) {
-		struct dcn20_vmid *vmid = &hubbub->vmid[i];
-
-		vmid->ctx = ctx;
-
-		vmid->regs = &vmid_regs[i];
-		vmid->shifts = &vmid_shifts;
-		vmid->masks = &vmid_masks;
-	}
-
-	return &hubbub->base;
-}
-
-struct timing_generator *dcn20_timing_generator_create(
-		struct dc_context *ctx,
-		uint32_t instance)
-{
-	struct optc *tgn10 =
-		kzalloc(sizeof(struct optc), GFP_ATOMIC);
-
-	if (!tgn10)
-		return NULL;
-
-	tgn10->base.inst = instance;
-	tgn10->base.ctx = ctx;
-
-	tgn10->tg_regs = &tg_regs[instance];
-	tgn10->tg_shift = &tg_shift;
-	tgn10->tg_mask = &tg_mask;
-
-	dcn20_timing_generator_init(tgn10);
-
-	return &tgn10->base;
-}
-
-static const struct encoder_feature_support link_enc_feature = {
-		.max_hdmi_deep_color = COLOR_DEPTH_121212,
-		.max_hdmi_pixel_clock = 600000,
-		.hdmi_ycbcr420_supported = true,
-		.dp_ycbcr420_supported = true,
-		.fec_supported = true,
-		.flags.bits.IS_HBR2_CAPABLE = true,
-		.flags.bits.IS_HBR3_CAPABLE = true,
-		.flags.bits.IS_TPS3_CAPABLE = true,
-		.flags.bits.IS_TPS4_CAPABLE = true
-};
-
-struct link_encoder *dcn20_link_encoder_create(
-	struct dc_context *ctx,
-	const struct encoder_init_data *enc_init_data)
-{
-	struct dcn20_link_encoder *enc20 =
-		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
-	int link_regs_id;
-
-	if (!enc20)
-		return NULL;
-
-	link_regs_id =
-		map_transmitter_id_to_phy_instance(enc_init_data->transmitter);
-
-	dcn20_link_encoder_construct(enc20,
-				      enc_init_data,
-				      &link_enc_feature,
-				      &link_enc_regs[link_regs_id],
-				      &link_enc_aux_regs[enc_init_data->channel - 1],
-				      &link_enc_hpd_regs[enc_init_data->hpd_source],
-				      &le_shift,
-				      &le_mask);
-
-	return &enc20->enc10.base;
-}
-
-static struct panel_cntl *dcn20_panel_cntl_create(const struct panel_cntl_init_data *init_data)
-{
-	struct dce_panel_cntl *panel_cntl =
-		kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
-
-	if (!panel_cntl)
-		return NULL;
-
-	dce_panel_cntl_construct(panel_cntl,
-			init_data,
-			&panel_cntl_regs[init_data->inst],
-			&panel_cntl_shift,
-			&panel_cntl_mask);
-
-	return &panel_cntl->base;
-}
-
-static struct clock_source *dcn20_clock_source_create(
-	struct dc_context *ctx,
-	struct dc_bios *bios,
-	enum clock_source_id id,
-	const struct dce110_clk_src_regs *regs,
-	bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src =
-		kzalloc(sizeof(struct dce110_clk_src), GFP_ATOMIC);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dcn20_clk_src_construct(clk_src, ctx, bios, id,
-			regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-
-	kfree(clk_src);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
-static void read_dce_straps(
-	struct dc_context *ctx,
-	struct resource_straps *straps)
-{
-	generic_reg_get(ctx, mmDC_PINSTRAPS + BASE(mmDC_PINSTRAPS_BASE_IDX),
-		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
-}
-
-static struct audio *dcn20_create_audio(
-		struct dc_context *ctx, unsigned int inst)
-{
-	return dce_audio_create(ctx, inst,
-			&audio_regs[inst], &audio_shift, &audio_mask);
-}
-
-struct stream_encoder *dcn20_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dcn10_stream_encoder *enc1 =
-		kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
-
-	if (!enc1)
-		return NULL;
-
-	if (ASICREV_IS_NAVI14_M(ctx->asic_id.hw_internal_rev)) {
-		if (eng_id >= ENGINE_ID_DIGD)
-			eng_id++;
-	}
-
-	dcn20_stream_encoder_construct(enc1, ctx, ctx->dc_bios, eng_id,
-					&stream_enc_regs[eng_id],
-					&se_shift, &se_mask);
-
-	return &enc1->base;
-}
-
-static const struct dce_hwseq_registers hwseq_reg = {
-		HWSEQ_DCN2_REG_LIST()
-};
-
-static const struct dce_hwseq_shift hwseq_shift = {
-		HWSEQ_DCN2_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_hwseq_mask hwseq_mask = {
-		HWSEQ_DCN2_MASK_SH_LIST(_MASK)
-};
-
-struct dce_hwseq *dcn20_hwseq_create(
-	struct dc_context *ctx)
-{
-	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
-
-	if (hws) {
-		hws->ctx = ctx;
-		hws->regs = &hwseq_reg;
-		hws->shifts = &hwseq_shift;
-		hws->masks = &hwseq_mask;
-	}
-	return hws;
-}
-
-static const struct resource_create_funcs res_create_funcs = {
-	.read_dce_straps = read_dce_straps,
-	.create_audio = dcn20_create_audio,
-	.create_stream_encoder = dcn20_stream_encoder_create,
-	.create_hwseq = dcn20_hwseq_create,
-};
-
-static void dcn20_pp_smu_destroy(struct pp_smu_funcs **pp_smu);
-
-void dcn20_clock_source_destroy(struct clock_source **clk_src)
-{
-	kfree(TO_DCE110_CLK_SRC(*clk_src));
-	*clk_src = NULL;
-}
-
-
-struct display_stream_compressor *dcn20_dsc_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_dsc *dsc =
-		kzalloc(sizeof(struct dcn20_dsc), GFP_ATOMIC);
-
-	if (!dsc) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
-	return &dsc->base;
-}
-
-void dcn20_dsc_destroy(struct display_stream_compressor **dsc)
-{
-	kfree(container_of(*dsc, struct dcn20_dsc, base));
-	*dsc = NULL;
-}
-
-
-static void dcn20_resource_destruct(struct dcn20_resource_pool *pool)
-{
-	unsigned int i;
-
-	for (i = 0; i < pool->base.stream_enc_count; i++) {
-		if (pool->base.stream_enc[i] != NULL) {
-			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
-			pool->base.stream_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		if (pool->base.dscs[i] != NULL)
-			dcn20_dsc_destroy(&pool->base.dscs[i]);
-	}
-
-	if (pool->base.mpc != NULL) {
-		kfree(TO_DCN20_MPC(pool->base.mpc));
-		pool->base.mpc = NULL;
-	}
-	if (pool->base.hubbub != NULL) {
-		kfree(pool->base.hubbub);
-		pool->base.hubbub = NULL;
-	}
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		if (pool->base.dpps[i] != NULL)
-			dcn20_dpp_destroy(&pool->base.dpps[i]);
-
-		if (pool->base.ipps[i] != NULL)
-			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
-
-		if (pool->base.hubps[i] != NULL) {
-			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
-			pool->base.hubps[i] = NULL;
-		}
-
-		if (pool->base.irqs != NULL) {
-			dal_irq_service_destroy(&pool->base.irqs);
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		if (pool->base.engines[i] != NULL)
-			dce110_engine_destroy(&pool->base.engines[i]);
-		if (pool->base.hw_i2cs[i] != NULL) {
-			kfree(pool->base.hw_i2cs[i]);
-			pool->base.hw_i2cs[i] = NULL;
-		}
-		if (pool->base.sw_i2cs[i] != NULL) {
-			kfree(pool->base.sw_i2cs[i]);
-			pool->base.sw_i2cs[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
-		if (pool->base.opps[i] != NULL)
-			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		if (pool->base.timing_generators[i] != NULL)	{
-			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
-			pool->base.timing_generators[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
-		if (pool->base.dwbc[i] != NULL) {
-			kfree(TO_DCN20_DWBC(pool->base.dwbc[i]));
-			pool->base.dwbc[i] = NULL;
-		}
-		if (pool->base.mcif_wb[i] != NULL) {
-			kfree(TO_DCN20_MMHUBBUB(pool->base.mcif_wb[i]));
-			pool->base.mcif_wb[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.audio_count; i++) {
-		if (pool->base.audios[i])
-			dce_aud_destroy(&pool->base.audios[i]);
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] != NULL) {
-			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
-			pool->base.clock_sources[i] = NULL;
-		}
-	}
-
-	if (pool->base.dp_clock_source != NULL) {
-		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
-		pool->base.dp_clock_source = NULL;
-	}
-
-
-	if (pool->base.abm != NULL)
-		dce_abm_destroy(&pool->base.abm);
-
-	if (pool->base.dmcu != NULL)
-		dce_dmcu_destroy(&pool->base.dmcu);
-
-	if (pool->base.dccg != NULL)
-		dcn_dccg_destroy(&pool->base.dccg);
-
-	if (pool->base.pp_smu != NULL)
-		dcn20_pp_smu_destroy(&pool->base.pp_smu);
-
-	if (pool->base.oem_device != NULL) {
-		struct dc *dc = pool->base.oem_device->ctx->dc;
-
-		dc->link_srv->destroy_ddc_service(&pool->base.oem_device);
-	}
-}
-
-struct hubp *dcn20_hubp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn20_hubp *hubp2 =
-		kzalloc(sizeof(struct dcn20_hubp), GFP_ATOMIC);
-
-	if (!hubp2)
-		return NULL;
-
-	if (hubp2_construct(hubp2, ctx, inst,
-			&hubp_regs[inst], &hubp_shift, &hubp_mask))
-		return &hubp2->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(hubp2);
-	return NULL;
-}
-
-static void get_pixel_clock_parameters(
-	struct pipe_ctx *pipe_ctx,
-	struct pixel_clk_params *pixel_clk_params)
-{
-	const struct dc_stream_state *stream = pipe_ctx->stream;
-	struct pipe_ctx *odm_pipe;
-	int opp_cnt = 1;
-	struct dc_link *link = stream->link;
-	struct link_encoder *link_enc = NULL;
-	struct dc *dc = pipe_ctx->stream->ctx->dc;
-	struct dce_hwseq *hws = dc->hwseq;
-
-	for (odm_pipe = pipe_ctx->next_odm_pipe; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe)
-		opp_cnt++;
-
-	pixel_clk_params->requested_pix_clk_100hz = stream->timing.pix_clk_100hz;
-
-	link_enc = link_enc_cfg_get_link_enc(link);
-	if (link_enc)
-		pixel_clk_params->encoder_object_id = link_enc->id;
-
-	pixel_clk_params->signal_type = pipe_ctx->stream->signal;
-	pixel_clk_params->controller_id = pipe_ctx->stream_res.tg->inst + 1;
-	/* TODO: un-hardcode*/
-	/* TODO - DP2.0 HW: calculate requested_sym_clk for UHBR rates */
-	pixel_clk_params->requested_sym_clk = LINK_RATE_LOW *
-		LINK_RATE_REF_FREQ_IN_KHZ;
-	pixel_clk_params->flags.ENABLE_SS = 0;
-	pixel_clk_params->color_depth =
-		stream->timing.display_color_depth;
-	pixel_clk_params->flags.DISPLAY_BLANKED = 1;
-	pixel_clk_params->pixel_encoding = stream->timing.pixel_encoding;
-
-	if (stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR422)
-		pixel_clk_params->color_depth = COLOR_DEPTH_888;
-
-	if (opp_cnt == 4)
-		pixel_clk_params->requested_pix_clk_100hz /= 4;
-	else if (optc2_is_two_pixels_per_containter(&stream->timing) || opp_cnt == 2)
-		pixel_clk_params->requested_pix_clk_100hz /= 2;
-	else if (hws->funcs.is_dp_dig_pixel_rate_div_policy) {
-		if (hws->funcs.is_dp_dig_pixel_rate_div_policy(pipe_ctx))
-			pixel_clk_params->requested_pix_clk_100hz /= 2;
-	}
-
-	if (stream->timing.timing_3d_format == TIMING_3D_FORMAT_HW_FRAME_PACKING)
-		pixel_clk_params->requested_pix_clk_100hz *= 2;
-
-}
-
-static void build_clamping_params(struct dc_stream_state *stream)
-{
-	stream->clamping.clamping_level = CLAMPING_FULL_RANGE;
-	stream->clamping.c_depth = stream->timing.display_color_depth;
-	stream->clamping.pixel_encoding = stream->timing.pixel_encoding;
-}
-
-static enum dc_status build_pipe_hw_param(struct pipe_ctx *pipe_ctx)
-{
-
-	get_pixel_clock_parameters(pipe_ctx, &pipe_ctx->stream_res.pix_clk_params);
-
-	pipe_ctx->clock_source->funcs->get_pix_clk_dividers(
-		pipe_ctx->clock_source,
-		&pipe_ctx->stream_res.pix_clk_params,
-		&pipe_ctx->pll_settings);
-
-	pipe_ctx->stream->clamping.pixel_encoding = pipe_ctx->stream->timing.pixel_encoding;
-
-	resource_build_bit_depth_reduction_params(pipe_ctx->stream,
-					&pipe_ctx->stream->bit_depth_params);
-	build_clamping_params(pipe_ctx->stream);
-
-	return DC_OK;
-}
-
-enum dc_status dcn20_build_mapped_resource(const struct dc *dc, struct dc_state *context, struct dc_stream_state *stream)
-{
-	enum dc_status status = DC_OK;
-	struct pipe_ctx *pipe_ctx = resource_get_otg_master_for_stream(&context->res_ctx, stream);
-
-	if (!pipe_ctx)
-		return DC_ERROR_UNEXPECTED;
-
-
-	status = build_pipe_hw_param(pipe_ctx);
-
-	return status;
-}
-
-
-void dcn20_acquire_dsc(const struct dc *dc,
-			struct resource_context *res_ctx,
-			struct display_stream_compressor **dsc,
-			int pipe_idx)
-{
-	int i;
-	const struct resource_pool *pool = dc->res_pool;
-	struct display_stream_compressor *dsc_old = dc->current_state->res_ctx.pipe_ctx[pipe_idx].stream_res.dsc;
-
-	ASSERT(*dsc == NULL); /* If this ASSERT fails, dsc was not released properly */
-	*dsc = NULL;
-
-	/* Always do 1-to-1 mapping when number of DSCs is same as number of pipes */
-	if (pool->res_cap->num_dsc == pool->res_cap->num_opp) {
-		*dsc = pool->dscs[pipe_idx];
-		res_ctx->is_dsc_acquired[pipe_idx] = true;
-		return;
-	}
-
-	/* Return old DSC to avoid the need for re-programming */
-	if (dsc_old && !res_ctx->is_dsc_acquired[dsc_old->inst]) {
-		*dsc = dsc_old;
-		res_ctx->is_dsc_acquired[dsc_old->inst] = true;
-		return ;
-	}
-
-	/* Find first free DSC */
-	for (i = 0; i < pool->res_cap->num_dsc; i++)
-		if (!res_ctx->is_dsc_acquired[i]) {
-			*dsc = pool->dscs[i];
-			res_ctx->is_dsc_acquired[i] = true;
-			break;
-		}
-}
-
-void dcn20_release_dsc(struct resource_context *res_ctx,
-			const struct resource_pool *pool,
-			struct display_stream_compressor **dsc)
-{
-	int i;
-
-	for (i = 0; i < pool->res_cap->num_dsc; i++)
-		if (pool->dscs[i] == *dsc) {
-			res_ctx->is_dsc_acquired[i] = false;
-			*dsc = NULL;
-			break;
-		}
-}
-
-
-
-enum dc_status dcn20_add_dsc_to_stream_resource(struct dc *dc,
-		struct dc_state *dc_ctx,
-		struct dc_stream_state *dc_stream)
-{
-	enum dc_status result = DC_OK;
-	int i;
-
-	/* Get a DSC if required and available */
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe_ctx = &dc_ctx->res_ctx.pipe_ctx[i];
-
-		if (pipe_ctx->top_pipe)
-			continue;
-
-		if (pipe_ctx->stream != dc_stream)
-			continue;
-
-		if (pipe_ctx->stream_res.dsc)
-			continue;
-
-		dcn20_acquire_dsc(dc, &dc_ctx->res_ctx, &pipe_ctx->stream_res.dsc, i);
-
-		/* The number of DSCs can be less than the number of pipes */
-		if (!pipe_ctx->stream_res.dsc) {
-			result = DC_NO_DSC_RESOURCE;
-		}
-
-		break;
-	}
-
-	return result;
-}
-
-
-static enum dc_status remove_dsc_from_stream_resource(struct dc *dc,
-		struct dc_state *new_ctx,
-		struct dc_stream_state *dc_stream)
-{
-	struct pipe_ctx *pipe_ctx = NULL;
-	int i;
-
-	for (i = 0; i < MAX_PIPES; i++) {
-		if (new_ctx->res_ctx.pipe_ctx[i].stream == dc_stream && !new_ctx->res_ctx.pipe_ctx[i].top_pipe) {
-			pipe_ctx = &new_ctx->res_ctx.pipe_ctx[i];
-
-			if (pipe_ctx->stream_res.dsc)
-				dcn20_release_dsc(&new_ctx->res_ctx, dc->res_pool, &pipe_ctx->stream_res.dsc);
-		}
-	}
-
-	if (!pipe_ctx)
-		return DC_ERROR_UNEXPECTED;
-	else
-		return DC_OK;
-}
-
-
-enum dc_status dcn20_add_stream_to_ctx(struct dc *dc, struct dc_state *new_ctx, struct dc_stream_state *dc_stream)
-{
-	enum dc_status result = DC_ERROR_UNEXPECTED;
-
-	result = resource_map_pool_resources(dc, new_ctx, dc_stream);
-
-	if (result == DC_OK)
-		result = resource_map_phy_clock_resources(dc, new_ctx, dc_stream);
-
-	/* Get a DSC if required and available */
-	if (result == DC_OK && dc_stream->timing.flags.DSC)
-		result = dcn20_add_dsc_to_stream_resource(dc, new_ctx, dc_stream);
-
-	if (result == DC_OK)
-		result = dcn20_build_mapped_resource(dc, new_ctx, dc_stream);
-
-	return result;
-}
-
-
-enum dc_status dcn20_remove_stream_from_ctx(struct dc *dc, struct dc_state *new_ctx, struct dc_stream_state *dc_stream)
-{
-	enum dc_status result = DC_OK;
-
-	result = remove_dsc_from_stream_resource(dc, new_ctx, dc_stream);
-
-	return result;
-}
-
-/**
- * dcn20_split_stream_for_odm - Check if stream can be splited for ODM
- *
- * @dc: DC object with resource pool info required for pipe split
- * @res_ctx: Persistent state of resources
- * @prev_odm_pipe: Reference to the previous ODM pipe
- * @next_odm_pipe: Reference to the next ODM pipe
- *
- * This function takes a logically active pipe and a logically free pipe and
- * halves all the scaling parameters that need to be halved while populating
- * the free pipe with the required resources and configuring the next/previous
- * ODM pipe pointers.
- *
- * Return:
- * Return true if split stream for ODM is possible, otherwise, return false.
- */
-bool dcn20_split_stream_for_odm(
-		const struct dc *dc,
-		struct resource_context *res_ctx,
-		struct pipe_ctx *prev_odm_pipe,
-		struct pipe_ctx *next_odm_pipe)
-{
-	int pipe_idx = next_odm_pipe->pipe_idx;
-	const struct resource_pool *pool = dc->res_pool;
-
-	*next_odm_pipe = *prev_odm_pipe;
-
-	next_odm_pipe->pipe_idx = pipe_idx;
-	next_odm_pipe->plane_res.mi = pool->mis[next_odm_pipe->pipe_idx];
-	next_odm_pipe->plane_res.hubp = pool->hubps[next_odm_pipe->pipe_idx];
-	next_odm_pipe->plane_res.ipp = pool->ipps[next_odm_pipe->pipe_idx];
-	next_odm_pipe->plane_res.xfm = pool->transforms[next_odm_pipe->pipe_idx];
-	next_odm_pipe->plane_res.dpp = pool->dpps[next_odm_pipe->pipe_idx];
-	next_odm_pipe->plane_res.mpcc_inst = pool->dpps[next_odm_pipe->pipe_idx]->inst;
-	next_odm_pipe->stream_res.dsc = NULL;
-	if (prev_odm_pipe->next_odm_pipe && prev_odm_pipe->next_odm_pipe != next_odm_pipe) {
-		next_odm_pipe->next_odm_pipe = prev_odm_pipe->next_odm_pipe;
-		next_odm_pipe->next_odm_pipe->prev_odm_pipe = next_odm_pipe;
-	}
-	if (prev_odm_pipe->top_pipe && prev_odm_pipe->top_pipe->next_odm_pipe) {
-		prev_odm_pipe->top_pipe->next_odm_pipe->bottom_pipe = next_odm_pipe;
-		next_odm_pipe->top_pipe = prev_odm_pipe->top_pipe->next_odm_pipe;
-	}
-	if (prev_odm_pipe->bottom_pipe && prev_odm_pipe->bottom_pipe->next_odm_pipe) {
-		prev_odm_pipe->bottom_pipe->next_odm_pipe->top_pipe = next_odm_pipe;
-		next_odm_pipe->bottom_pipe = prev_odm_pipe->bottom_pipe->next_odm_pipe;
-	}
-	prev_odm_pipe->next_odm_pipe = next_odm_pipe;
-	next_odm_pipe->prev_odm_pipe = prev_odm_pipe;
-
-	if (prev_odm_pipe->plane_state) {
-		struct scaler_data *sd = &prev_odm_pipe->plane_res.scl_data;
-		int new_width;
-
-		/* HACTIVE halved for odm combine */
-		sd->h_active /= 2;
-		/* Calculate new vp and recout for left pipe */
-		/* Need at least 16 pixels width per side */
-		if (sd->recout.x + 16 >= sd->h_active)
-			return false;
-		new_width = sd->h_active - sd->recout.x;
-		sd->viewport.width -= dc_fixpt_floor(dc_fixpt_mul_int(
-				sd->ratios.horz, sd->recout.width - new_width));
-		sd->viewport_c.width -= dc_fixpt_floor(dc_fixpt_mul_int(
-				sd->ratios.horz_c, sd->recout.width - new_width));
-		sd->recout.width = new_width;
-
-		/* Calculate new vp and recout for right pipe */
-		sd = &next_odm_pipe->plane_res.scl_data;
-		/* HACTIVE halved for odm combine */
-		sd->h_active /= 2;
-		/* Need at least 16 pixels width per side */
-		if (new_width <= 16)
-			return false;
-		new_width = sd->recout.width + sd->recout.x - sd->h_active;
-		sd->viewport.width -= dc_fixpt_floor(dc_fixpt_mul_int(
-				sd->ratios.horz, sd->recout.width - new_width));
-		sd->viewport_c.width -= dc_fixpt_floor(dc_fixpt_mul_int(
-				sd->ratios.horz_c, sd->recout.width - new_width));
-		sd->recout.width = new_width;
-		sd->viewport.x += dc_fixpt_floor(dc_fixpt_mul_int(
-				sd->ratios.horz, sd->h_active - sd->recout.x));
-		sd->viewport_c.x += dc_fixpt_floor(dc_fixpt_mul_int(
-				sd->ratios.horz_c, sd->h_active - sd->recout.x));
-		sd->recout.x = 0;
-	}
-	if (!next_odm_pipe->top_pipe)
-		next_odm_pipe->stream_res.opp = pool->opps[next_odm_pipe->pipe_idx];
-	else
-		next_odm_pipe->stream_res.opp = next_odm_pipe->top_pipe->stream_res.opp;
-	if (next_odm_pipe->stream->timing.flags.DSC == 1 && !next_odm_pipe->top_pipe) {
-		dcn20_acquire_dsc(dc, res_ctx, &next_odm_pipe->stream_res.dsc, next_odm_pipe->pipe_idx);
-		ASSERT(next_odm_pipe->stream_res.dsc);
-		if (next_odm_pipe->stream_res.dsc == NULL)
-			return false;
-	}
-
-	return true;
-}
-
-void dcn20_split_stream_for_mpc(
-		struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		struct pipe_ctx *primary_pipe,
-		struct pipe_ctx *secondary_pipe)
-{
-	int pipe_idx = secondary_pipe->pipe_idx;
-	struct pipe_ctx *sec_bot_pipe = secondary_pipe->bottom_pipe;
-
-	*secondary_pipe = *primary_pipe;
-	secondary_pipe->bottom_pipe = sec_bot_pipe;
-
-	secondary_pipe->pipe_idx = pipe_idx;
-	secondary_pipe->plane_res.mi = pool->mis[secondary_pipe->pipe_idx];
-	secondary_pipe->plane_res.hubp = pool->hubps[secondary_pipe->pipe_idx];
-	secondary_pipe->plane_res.ipp = pool->ipps[secondary_pipe->pipe_idx];
-	secondary_pipe->plane_res.xfm = pool->transforms[secondary_pipe->pipe_idx];
-	secondary_pipe->plane_res.dpp = pool->dpps[secondary_pipe->pipe_idx];
-	secondary_pipe->plane_res.mpcc_inst = pool->dpps[secondary_pipe->pipe_idx]->inst;
-	secondary_pipe->stream_res.dsc = NULL;
-	if (primary_pipe->bottom_pipe && primary_pipe->bottom_pipe != secondary_pipe) {
-		ASSERT(!secondary_pipe->bottom_pipe);
-		secondary_pipe->bottom_pipe = primary_pipe->bottom_pipe;
-		secondary_pipe->bottom_pipe->top_pipe = secondary_pipe;
-	}
-	primary_pipe->bottom_pipe = secondary_pipe;
-	secondary_pipe->top_pipe = primary_pipe;
-
-	ASSERT(primary_pipe->plane_state);
-}
-
-unsigned int dcn20_calc_max_scaled_time(
-		unsigned int time_per_pixel,
-		enum mmhubbub_wbif_mode mode,
-		unsigned int urgent_watermark)
-{
-	unsigned int time_per_byte = 0;
-	unsigned int total_y_free_entry = 0x200; /* two memory piece for luma */
-	unsigned int total_c_free_entry = 0x140; /* two memory piece for chroma */
-	unsigned int small_free_entry, max_free_entry;
-	unsigned int buf_lh_capability;
-	unsigned int max_scaled_time;
-
-	if (mode == PACKED_444) /* packed mode */
-		time_per_byte = time_per_pixel/4;
-	else if (mode == PLANAR_420_8BPC)
-		time_per_byte  = time_per_pixel;
-	else if (mode == PLANAR_420_10BPC) /* p010 */
-		time_per_byte  = time_per_pixel * 819/1024;
-
-	if (time_per_byte == 0)
-		time_per_byte = 1;
-
-	small_free_entry  = (total_y_free_entry > total_c_free_entry) ? total_c_free_entry : total_y_free_entry;
-	max_free_entry    = (mode == PACKED_444) ? total_y_free_entry + total_c_free_entry : small_free_entry;
-	buf_lh_capability = max_free_entry*time_per_byte*32/16; /* there is 4bit fraction */
-	max_scaled_time   = buf_lh_capability - urgent_watermark;
-	return max_scaled_time;
-}
-
-void dcn20_set_mcif_arb_params(
-		struct dc *dc,
-		struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		int pipe_cnt)
-{
-	enum mmhubbub_wbif_mode wbif_mode;
-	struct mcif_arb_params *wb_arb_params;
-	int i, j, dwb_pipe;
-
-	/* Writeback MCIF_WB arbitration parameters */
-	dwb_pipe = 0;
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-
-		if (!context->res_ctx.pipe_ctx[i].stream)
-			continue;
-
-		for (j = 0; j < MAX_DWB_PIPES; j++) {
-			if (context->res_ctx.pipe_ctx[i].stream->writeback_info[j].wb_enabled == false)
-				continue;
-
-			//wb_arb_params = &context->res_ctx.pipe_ctx[i].stream->writeback_info[j].mcif_arb_params;
-			wb_arb_params = &context->bw_ctx.bw.dcn.bw_writeback.mcif_wb_arb[dwb_pipe];
-
-			if (context->res_ctx.pipe_ctx[i].stream->writeback_info[j].dwb_params.out_format == dwb_scaler_mode_yuv420) {
-				if (context->res_ctx.pipe_ctx[i].stream->writeback_info[j].dwb_params.output_depth == DWB_OUTPUT_PIXEL_DEPTH_8BPC)
-					wbif_mode = PLANAR_420_8BPC;
-				else
-					wbif_mode = PLANAR_420_10BPC;
-			} else
-				wbif_mode = PACKED_444;
-
-			DC_FP_START();
-			dcn20_fpu_set_wb_arb_params(wb_arb_params, context, pipes, pipe_cnt, i);
-			DC_FP_END();
-
-			wb_arb_params->slice_lines = 32;
-			wb_arb_params->arbitration_slice = 2;
-			wb_arb_params->max_scaled_time = dcn20_calc_max_scaled_time(wb_arb_params->time_per_pixel,
-				wbif_mode,
-				wb_arb_params->cli_watermark[0]); /* assume 4 watermark sets have the same value */
-
-			dwb_pipe++;
-
-			if (dwb_pipe >= MAX_DWB_PIPES)
-				return;
-		}
-		if (dwb_pipe >= MAX_DWB_PIPES)
-			return;
-	}
-}
-
-bool dcn20_validate_dsc(struct dc *dc, struct dc_state *new_ctx)
-{
-	int i;
-
-	/* Validate DSC config, dsc count validation is already done */
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe_ctx = &new_ctx->res_ctx.pipe_ctx[i];
-		struct dc_stream_state *stream = pipe_ctx->stream;
-		struct dsc_config dsc_cfg;
-		struct pipe_ctx *odm_pipe;
-		int opp_cnt = 1;
-
-		for (odm_pipe = pipe_ctx->next_odm_pipe; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe)
-			opp_cnt++;
-
-		/* Only need to validate top pipe */
-		if (pipe_ctx->top_pipe || pipe_ctx->prev_odm_pipe || !stream || !stream->timing.flags.DSC)
-			continue;
-
-		dsc_cfg.pic_width = (stream->timing.h_addressable + stream->timing.h_border_left
-				+ stream->timing.h_border_right) / opp_cnt;
-		dsc_cfg.pic_height = stream->timing.v_addressable + stream->timing.v_border_top
-				+ stream->timing.v_border_bottom;
-		dsc_cfg.pixel_encoding = stream->timing.pixel_encoding;
-		dsc_cfg.color_depth = stream->timing.display_color_depth;
-		dsc_cfg.is_odm = pipe_ctx->next_odm_pipe ? true : false;
-		dsc_cfg.dc_dsc_cfg = stream->timing.dsc_cfg;
-		dsc_cfg.dc_dsc_cfg.num_slices_h /= opp_cnt;
-
-		if (!pipe_ctx->stream_res.dsc->funcs->dsc_validate_stream(pipe_ctx->stream_res.dsc, &dsc_cfg))
-			return false;
-	}
-	return true;
-}
-
-struct pipe_ctx *dcn20_find_secondary_pipe(struct dc *dc,
-		struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		const struct pipe_ctx *primary_pipe)
-{
-	struct pipe_ctx *secondary_pipe = NULL;
-
-	if (dc && primary_pipe) {
-		int j;
-		int preferred_pipe_idx = 0;
-
-		/* first check the prev dc state:
-		 * if this primary pipe has a bottom pipe in prev. state
-		 * and if the bottom pipe is still available (which it should be),
-		 * pick that pipe as secondary
-		 * Same logic applies for ODM pipes
-		 */
-		if (dc->current_state->res_ctx.pipe_ctx[primary_pipe->pipe_idx].next_odm_pipe) {
-			preferred_pipe_idx = dc->current_state->res_ctx.pipe_ctx[primary_pipe->pipe_idx].next_odm_pipe->pipe_idx;
-			if (res_ctx->pipe_ctx[preferred_pipe_idx].stream == NULL) {
-				secondary_pipe = &res_ctx->pipe_ctx[preferred_pipe_idx];
-				secondary_pipe->pipe_idx = preferred_pipe_idx;
-			}
-		}
-		if (secondary_pipe == NULL &&
-				dc->current_state->res_ctx.pipe_ctx[primary_pipe->pipe_idx].bottom_pipe) {
-			preferred_pipe_idx = dc->current_state->res_ctx.pipe_ctx[primary_pipe->pipe_idx].bottom_pipe->pipe_idx;
-			if (res_ctx->pipe_ctx[preferred_pipe_idx].stream == NULL) {
-				secondary_pipe = &res_ctx->pipe_ctx[preferred_pipe_idx];
-				secondary_pipe->pipe_idx = preferred_pipe_idx;
-			}
-		}
-
-		/*
-		 * if this primary pipe does not have a bottom pipe in prev. state
-		 * start backward and find a pipe that did not used to be a bottom pipe in
-		 * prev. dc state. This way we make sure we keep the same assignment as
-		 * last state and will not have to reprogram every pipe
-		 */
-		if (secondary_pipe == NULL) {
-			for (j = dc->res_pool->pipe_count - 1; j >= 0; j--) {
-				if (dc->current_state->res_ctx.pipe_ctx[j].top_pipe == NULL
-						&& dc->current_state->res_ctx.pipe_ctx[j].prev_odm_pipe == NULL) {
-					preferred_pipe_idx = j;
-
-					if (res_ctx->pipe_ctx[preferred_pipe_idx].stream == NULL) {
-						secondary_pipe = &res_ctx->pipe_ctx[preferred_pipe_idx];
-						secondary_pipe->pipe_idx = preferred_pipe_idx;
-						break;
-					}
-				}
-			}
-		}
-		/*
-		 * We should never hit this assert unless assignments are shuffled around
-		 * if this happens we will prob. hit a vsync tdr
-		 */
-		ASSERT(secondary_pipe);
-		/*
-		 * search backwards for the second pipe to keep pipe
-		 * assignment more consistent
-		 */
-		if (secondary_pipe == NULL) {
-			for (j = dc->res_pool->pipe_count - 1; j >= 0; j--) {
-				preferred_pipe_idx = j;
-
-				if (res_ctx->pipe_ctx[preferred_pipe_idx].stream == NULL) {
-					secondary_pipe = &res_ctx->pipe_ctx[preferred_pipe_idx];
-					secondary_pipe->pipe_idx = preferred_pipe_idx;
-					break;
-				}
-			}
-		}
-	}
-
-	return secondary_pipe;
-}
-
-void dcn20_merge_pipes_for_validate(
-		struct dc *dc,
-		struct dc_state *context)
-{
-	int i;
-
-	/* merge previously split odm pipes since mode support needs to make the decision */
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-		struct pipe_ctx *odm_pipe = pipe->next_odm_pipe;
-
-		if (pipe->prev_odm_pipe)
-			continue;
-
-		pipe->next_odm_pipe = NULL;
-		while (odm_pipe) {
-			struct pipe_ctx *next_odm_pipe = odm_pipe->next_odm_pipe;
-
-			odm_pipe->plane_state = NULL;
-			odm_pipe->stream = NULL;
-			odm_pipe->top_pipe = NULL;
-			odm_pipe->bottom_pipe = NULL;
-			odm_pipe->prev_odm_pipe = NULL;
-			odm_pipe->next_odm_pipe = NULL;
-			if (odm_pipe->stream_res.dsc)
-				dcn20_release_dsc(&context->res_ctx, dc->res_pool, &odm_pipe->stream_res.dsc);
-			/* Clear plane_res and stream_res */
-			memset(&odm_pipe->plane_res, 0, sizeof(odm_pipe->plane_res));
-			memset(&odm_pipe->stream_res, 0, sizeof(odm_pipe->stream_res));
-			odm_pipe = next_odm_pipe;
-		}
-		if (pipe->plane_state)
-			resource_build_scaling_params(pipe);
-	}
-
-	/* merge previously mpc split pipes since mode support needs to make the decision */
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-		struct pipe_ctx *hsplit_pipe = pipe->bottom_pipe;
-
-		if (!hsplit_pipe || hsplit_pipe->plane_state != pipe->plane_state)
-			continue;
-
-		pipe->bottom_pipe = hsplit_pipe->bottom_pipe;
-		if (hsplit_pipe->bottom_pipe)
-			hsplit_pipe->bottom_pipe->top_pipe = pipe;
-		hsplit_pipe->plane_state = NULL;
-		hsplit_pipe->stream = NULL;
-		hsplit_pipe->top_pipe = NULL;
-		hsplit_pipe->bottom_pipe = NULL;
-
-		/* Clear plane_res and stream_res */
-		memset(&hsplit_pipe->plane_res, 0, sizeof(hsplit_pipe->plane_res));
-		memset(&hsplit_pipe->stream_res, 0, sizeof(hsplit_pipe->stream_res));
-		if (pipe->plane_state)
-			resource_build_scaling_params(pipe);
-	}
-}
-
-int dcn20_validate_apply_pipe_split_flags(
-		struct dc *dc,
-		struct dc_state *context,
-		int vlevel,
-		int *split,
-		bool *merge)
-{
-	int i, pipe_idx, vlevel_split;
-	int plane_count = 0;
-	bool force_split = false;
-	bool avoid_split = dc->debug.pipe_split_policy == MPC_SPLIT_AVOID;
-	struct vba_vars_st *v = &context->bw_ctx.dml.vba;
-	int max_mpc_comb = v->maxMpcComb;
-
-	if (context->stream_count > 1) {
-		if (dc->debug.pipe_split_policy == MPC_SPLIT_AVOID_MULT_DISP)
-			avoid_split = true;
-	} else if (dc->debug.force_single_disp_pipe_split)
-			force_split = true;
-
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-
-		/**
-		 * Workaround for avoiding pipe-split in cases where we'd split
-		 * planes that are too small, resulting in splits that aren't
-		 * valid for the scaler.
-		 */
-		if (pipe->plane_state &&
-		    (pipe->plane_state->dst_rect.width <= 16 ||
-		     pipe->plane_state->dst_rect.height <= 16 ||
-		     pipe->plane_state->src_rect.width <= 16 ||
-		     pipe->plane_state->src_rect.height <= 16))
-			avoid_split = true;
-
-		/* TODO: fix dc bugs and remove this split threshold thing */
-		if (pipe->stream && !pipe->prev_odm_pipe &&
-				(!pipe->top_pipe || pipe->top_pipe->plane_state != pipe->plane_state))
-			++plane_count;
-	}
-	if (plane_count > dc->res_pool->pipe_count / 2)
-		avoid_split = true;
-
-	/* W/A: Mode timing with borders may not work well with pipe split, avoid for this corner case */
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-		struct dc_crtc_timing timing;
-
-		if (!pipe->stream)
-			continue;
-		else {
-			timing = pipe->stream->timing;
-			if (timing.h_border_left + timing.h_border_right
-					+ timing.v_border_top + timing.v_border_bottom > 0) {
-				avoid_split = true;
-				break;
-			}
-		}
-	}
-
-	/* Avoid split loop looks for lowest voltage level that allows most unsplit pipes possible */
-	if (avoid_split) {
-		for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
-			if (!context->res_ctx.pipe_ctx[i].stream)
-				continue;
-
-			for (vlevel_split = vlevel; vlevel <= context->bw_ctx.dml.soc.num_states; vlevel++)
-				if (v->NoOfDPP[vlevel][0][pipe_idx] == 1 &&
-						v->ModeSupport[vlevel][0])
-					break;
-			/* Impossible to not split this pipe */
-			if (vlevel > context->bw_ctx.dml.soc.num_states)
-				vlevel = vlevel_split;
-			else
-				max_mpc_comb = 0;
-			pipe_idx++;
-		}
-		v->maxMpcComb = max_mpc_comb;
-	}
-
-	/* Split loop sets which pipe should be split based on dml outputs and dc flags */
-	for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-		int pipe_plane = v->pipe_plane[pipe_idx];
-		bool split4mpc = context->stream_count == 1 && plane_count == 1
-				&& dc->config.enable_4to1MPC && dc->res_pool->pipe_count >= 4;
-
-		if (!context->res_ctx.pipe_ctx[i].stream)
-			continue;
-
-		if (split4mpc || v->NoOfDPP[vlevel][max_mpc_comb][pipe_plane] == 4)
-			split[i] = 4;
-		else if (force_split || v->NoOfDPP[vlevel][max_mpc_comb][pipe_plane] == 2)
-				split[i] = 2;
-
-		if ((pipe->stream->view_format ==
-				VIEW_3D_FORMAT_SIDE_BY_SIDE ||
-				pipe->stream->view_format ==
-				VIEW_3D_FORMAT_TOP_AND_BOTTOM) &&
-				(pipe->stream->timing.timing_3d_format ==
-				TIMING_3D_FORMAT_TOP_AND_BOTTOM ||
-				 pipe->stream->timing.timing_3d_format ==
-				TIMING_3D_FORMAT_SIDE_BY_SIDE))
-			split[i] = 2;
-		if (dc->debug.force_odm_combine & (1 << pipe->stream_res.tg->inst)) {
-			split[i] = 2;
-			v->ODMCombineEnablePerState[vlevel][pipe_plane] = dm_odm_combine_mode_2to1;
-		}
-		if (dc->debug.force_odm_combine_4to1 & (1 << pipe->stream_res.tg->inst)) {
-			split[i] = 4;
-			v->ODMCombineEnablePerState[vlevel][pipe_plane] = dm_odm_combine_mode_4to1;
-		}
-		/*420 format workaround*/
-		if (pipe->stream->timing.h_addressable > 7680 &&
-				pipe->stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR420) {
-			split[i] = 4;
-		}
-		v->ODMCombineEnabled[pipe_plane] =
-			v->ODMCombineEnablePerState[vlevel][pipe_plane];
-
-		if (v->ODMCombineEnabled[pipe_plane] == dm_odm_combine_mode_disabled) {
-			if (resource_get_mpc_slice_count(pipe) == 2) {
-				/*If need split for mpc but 2 way split already*/
-				if (split[i] == 4)
-					split[i] = 2; /* 2 -> 4 MPC */
-				else if (split[i] == 2)
-					split[i] = 0; /* 2 -> 2 MPC */
-				else if (pipe->top_pipe && pipe->top_pipe->plane_state == pipe->plane_state)
-					merge[i] = true; /* 2 -> 1 MPC */
-			} else if (resource_get_mpc_slice_count(pipe) == 4) {
-				/*If need split for mpc but 4 way split already*/
-				if (split[i] == 2 && ((pipe->top_pipe && !pipe->top_pipe->top_pipe)
-						|| !pipe->bottom_pipe)) {
-					merge[i] = true; /* 4 -> 2 MPC */
-				} else if (split[i] == 0 && pipe->top_pipe &&
-						pipe->top_pipe->plane_state == pipe->plane_state)
-					merge[i] = true; /* 4 -> 1 MPC */
-				split[i] = 0;
-			} else if (resource_get_odm_slice_count(pipe) > 1) {
-				/* ODM -> MPC transition */
-				if (pipe->prev_odm_pipe) {
-					split[i] = 0;
-					merge[i] = true;
-				}
-			}
-		} else {
-			if (resource_get_odm_slice_count(pipe) == 2) {
-				/*If need split for odm but 2 way split already*/
-				if (split[i] == 4)
-					split[i] = 2; /* 2 -> 4 ODM */
-				else if (split[i] == 2)
-					split[i] = 0; /* 2 -> 2 ODM */
-				else if (pipe->prev_odm_pipe) {
-					ASSERT(0); /* NOT expected yet */
-					merge[i] = true; /* exit ODM */
-				}
-			} else if (resource_get_odm_slice_count(pipe) == 4) {
-				/*If need split for odm but 4 way split already*/
-				if (split[i] == 2 && ((pipe->prev_odm_pipe && !pipe->prev_odm_pipe->prev_odm_pipe)
-						|| !pipe->next_odm_pipe)) {
-					merge[i] = true; /* 4 -> 2 ODM */
-				} else if (split[i] == 0 && pipe->prev_odm_pipe) {
-					ASSERT(0); /* NOT expected yet */
-					merge[i] = true; /* exit ODM */
-				}
-				split[i] = 0;
-			} else if (resource_get_mpc_slice_count(pipe) > 1) {
-				/* MPC -> ODM transition */
-				ASSERT(0); /* NOT expected yet */
-				if (pipe->top_pipe && pipe->top_pipe->plane_state == pipe->plane_state) {
-					split[i] = 0;
-					merge[i] = true;
-				}
-			}
-		}
-
-		/* Adjust dppclk when split is forced, do not bother with dispclk */
-		if (split[i] != 0 && v->NoOfDPP[vlevel][max_mpc_comb][pipe_idx] == 1) {
-			DC_FP_START();
-			dcn20_fpu_adjust_dppclk(v, vlevel, max_mpc_comb, pipe_idx, false);
-			DC_FP_END();
-		}
-		pipe_idx++;
-	}
-
-	return vlevel;
-}
-
-bool dcn20_fast_validate_bw(
-		struct dc *dc,
-		struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		int *pipe_cnt_out,
-		int *pipe_split_from,
-		int *vlevel_out,
-		bool fast_validate)
-{
-	bool out = false;
-	int split[MAX_PIPES] = { 0 };
-	int pipe_cnt, i, pipe_idx, vlevel;
-
-	ASSERT(pipes);
-	if (!pipes)
-		return false;
-
-	dcn20_merge_pipes_for_validate(dc, context);
-
-	DC_FP_START();
-	pipe_cnt = dc->res_pool->funcs->populate_dml_pipes(dc, context, pipes, fast_validate);
-	DC_FP_END();
-
-	*pipe_cnt_out = pipe_cnt;
-
-	if (!pipe_cnt) {
-		out = true;
-		goto validate_out;
-	}
-
-	vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
-
-	if (vlevel > context->bw_ctx.dml.soc.num_states)
-		goto validate_fail;
-
-	vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, vlevel, split, NULL);
-
-	/*initialize pipe_just_split_from to invalid idx*/
-	for (i = 0; i < MAX_PIPES; i++)
-		pipe_split_from[i] = -1;
-
-	for (i = 0, pipe_idx = -1; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-		struct pipe_ctx *hsplit_pipe = pipe->bottom_pipe;
-
-		if (!pipe->stream || pipe_split_from[i] >= 0)
-			continue;
-
-		pipe_idx++;
-
-		if (!pipe->top_pipe && !pipe->plane_state && context->bw_ctx.dml.vba.ODMCombineEnabled[pipe_idx]) {
-			hsplit_pipe = dcn20_find_secondary_pipe(dc, &context->res_ctx, dc->res_pool, pipe);
-			ASSERT(hsplit_pipe);
-			if (!dcn20_split_stream_for_odm(
-					dc, &context->res_ctx,
-					pipe, hsplit_pipe))
-				goto validate_fail;
-			pipe_split_from[hsplit_pipe->pipe_idx] = pipe_idx;
-			dcn20_build_mapped_resource(dc, context, pipe->stream);
-		}
-
-		if (!pipe->plane_state)
-			continue;
-		/* Skip 2nd half of already split pipe */
-		if (pipe->top_pipe && pipe->plane_state == pipe->top_pipe->plane_state)
-			continue;
-
-		/* We do not support mpo + odm at the moment */
-		if (hsplit_pipe && hsplit_pipe->plane_state != pipe->plane_state
-				&& context->bw_ctx.dml.vba.ODMCombineEnabled[pipe_idx])
-			goto validate_fail;
-
-		if (split[i] == 2) {
-			if (!hsplit_pipe || hsplit_pipe->plane_state != pipe->plane_state) {
-				/* pipe not split previously needs split */
-				hsplit_pipe = dcn20_find_secondary_pipe(dc, &context->res_ctx, dc->res_pool, pipe);
-				ASSERT(hsplit_pipe);
-				if (!hsplit_pipe) {
-					DC_FP_START();
-					dcn20_fpu_adjust_dppclk(&context->bw_ctx.dml.vba, vlevel, context->bw_ctx.dml.vba.maxMpcComb, pipe_idx, true);
-					DC_FP_END();
-					continue;
-				}
-				if (context->bw_ctx.dml.vba.ODMCombineEnabled[pipe_idx]) {
-					if (!dcn20_split_stream_for_odm(
-							dc, &context->res_ctx,
-							pipe, hsplit_pipe))
-						goto validate_fail;
-					dcn20_build_mapped_resource(dc, context, pipe->stream);
-				} else {
-					dcn20_split_stream_for_mpc(
-							&context->res_ctx, dc->res_pool,
-							pipe, hsplit_pipe);
-					resource_build_scaling_params(pipe);
-					resource_build_scaling_params(hsplit_pipe);
-				}
-				pipe_split_from[hsplit_pipe->pipe_idx] = pipe_idx;
-			}
-		} else if (hsplit_pipe && hsplit_pipe->plane_state == pipe->plane_state) {
-			/* merge should already have been done */
-			ASSERT(0);
-		}
-	}
-	/* Actual dsc count per stream dsc validation*/
-	if (!dcn20_validate_dsc(dc, context)) {
-		context->bw_ctx.dml.vba.ValidationStatus[context->bw_ctx.dml.vba.soc.num_states] =
-				DML_FAIL_DSC_VALIDATION_FAILURE;
-		goto validate_fail;
-	}
-
-	*vlevel_out = vlevel;
-
-	out = true;
-	goto validate_out;
-
-validate_fail:
-	out = false;
-
-validate_out:
-	return out;
-}
-
-bool dcn20_validate_bandwidth(struct dc *dc, struct dc_state *context,
-		bool fast_validate)
-{
-	bool voltage_supported;
-	display_e2e_pipe_params_st *pipes;
-
-	pipes = kcalloc(dc->res_pool->pipe_count, sizeof(display_e2e_pipe_params_st), GFP_KERNEL);
-	if (!pipes)
-		return false;
-
-	DC_FP_START();
-	voltage_supported = dcn20_validate_bandwidth_fp(dc, context, fast_validate, pipes);
-	DC_FP_END();
-
-	kfree(pipes);
-	return voltage_supported;
-}
-
-struct pipe_ctx *dcn20_acquire_free_pipe_for_layer(
-		const struct dc_state *cur_ctx,
-		struct dc_state *new_ctx,
-		const struct resource_pool *pool,
-		const struct pipe_ctx *opp_head)
-{
-	struct resource_context *res_ctx = &new_ctx->res_ctx;
-	struct pipe_ctx *otg_master = resource_get_otg_master_for_stream(res_ctx, opp_head->stream);
-	struct pipe_ctx *sec_dpp_pipe = resource_find_free_secondary_pipe_legacy(res_ctx, pool, otg_master);
-
-	ASSERT(otg_master);
-
-	if (!sec_dpp_pipe)
-		return NULL;
-
-	sec_dpp_pipe->stream = opp_head->stream;
-	sec_dpp_pipe->stream_res.tg = opp_head->stream_res.tg;
-	sec_dpp_pipe->stream_res.opp = opp_head->stream_res.opp;
-
-	sec_dpp_pipe->plane_res.hubp = pool->hubps[sec_dpp_pipe->pipe_idx];
-	sec_dpp_pipe->plane_res.ipp = pool->ipps[sec_dpp_pipe->pipe_idx];
-	sec_dpp_pipe->plane_res.dpp = pool->dpps[sec_dpp_pipe->pipe_idx];
-	sec_dpp_pipe->plane_res.mpcc_inst = pool->dpps[sec_dpp_pipe->pipe_idx]->inst;
-
-	return sec_dpp_pipe;
-}
-
-bool dcn20_get_dcc_compression_cap(const struct dc *dc,
-		const struct dc_dcc_surface_param *input,
-		struct dc_surface_dcc_cap *output)
-{
-	return dc->res_pool->hubbub->funcs->get_dcc_compression_cap(
-			dc->res_pool->hubbub,
-			input,
-			output);
-}
-
-static void dcn20_destroy_resource_pool(struct resource_pool **pool)
-{
-	struct dcn20_resource_pool *dcn20_pool = TO_DCN20_RES_POOL(*pool);
-
-	dcn20_resource_destruct(dcn20_pool);
-	kfree(dcn20_pool);
-	*pool = NULL;
-}
-
-
-static struct dc_cap_funcs cap_funcs = {
-	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
-};
-
-
-enum dc_status dcn20_patch_unknown_plane_state(struct dc_plane_state *plane_state)
-{
-	enum surface_pixel_format surf_pix_format = plane_state->format;
-	unsigned int bpp = resource_pixel_format_to_bpp(surf_pix_format);
-
-	plane_state->tiling_info.gfx9.swizzle = DC_SW_64KB_S;
-	if (bpp == 64)
-		plane_state->tiling_info.gfx9.swizzle = DC_SW_64KB_D;
-
-	return DC_OK;
-}
-
-void dcn20_release_pipe(struct dc_state *context,
-			struct pipe_ctx *pipe,
-			const struct resource_pool *pool)
-{
-	if (resource_is_pipe_type(pipe, OPP_HEAD) && pipe->stream_res.dsc)
-		dcn20_release_dsc(&context->res_ctx, pool, &pipe->stream_res.dsc);
-	memset(pipe, 0, sizeof(*pipe));
-}
-
-static const struct resource_funcs dcn20_res_pool_funcs = {
-	.destroy = dcn20_destroy_resource_pool,
-	.link_enc_create = dcn20_link_encoder_create,
-	.panel_cntl_create = dcn20_panel_cntl_create,
-	.validate_bandwidth = dcn20_validate_bandwidth,
-	.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
-	.release_pipe = dcn20_release_pipe,
-	.add_stream_to_ctx = dcn20_add_stream_to_ctx,
-	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
-	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
-	.populate_dml_writeback_from_context = dcn20_populate_dml_writeback_from_context,
-	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
-	.set_mcif_arb_params = dcn20_set_mcif_arb_params,
-	.populate_dml_pipes = dcn20_populate_dml_pipes_from_context,
-	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link
-};
-
-bool dcn20_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t pipe_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < pipe_count; i++) {
-		struct dcn20_dwbc *dwbc20 = kzalloc(sizeof(struct dcn20_dwbc),
-						    GFP_KERNEL);
-
-		if (!dwbc20) {
-			dm_error("DC: failed to create dwbc20!\n");
-			return false;
-		}
-		dcn20_dwbc_construct(dwbc20, ctx,
-				&dwbc20_regs[i],
-				&dwbc20_shift,
-				&dwbc20_mask,
-				i);
-		pool->dwbc[i] = &dwbc20->base;
-	}
-	return true;
-}
-
-bool dcn20_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t pipe_count = pool->res_cap->num_dwb;
-
-	ASSERT(pipe_count > 0);
-
-	for (i = 0; i < pipe_count; i++) {
-		struct dcn20_mmhubbub *mcif_wb20 = kzalloc(sizeof(struct dcn20_mmhubbub),
-						    GFP_KERNEL);
-
-		if (!mcif_wb20) {
-			dm_error("DC: failed to create mcif_wb20!\n");
-			return false;
-		}
-
-		dcn20_mmhubbub_construct(mcif_wb20, ctx,
-				&mcif_wb20_regs[i],
-				&mcif_wb20_shift,
-				&mcif_wb20_mask,
-				i);
-
-		pool->mcif_wb[i] = &mcif_wb20->base;
-	}
-	return true;
-}
-
-static struct pp_smu_funcs *dcn20_pp_smu_create(struct dc_context *ctx)
-{
-	struct pp_smu_funcs *pp_smu = kzalloc(sizeof(*pp_smu), GFP_ATOMIC);
-
-	if (!pp_smu)
-		return pp_smu;
-
-	dm_pp_get_funcs(ctx, pp_smu);
-
-	if (pp_smu->ctx.ver != PP_SMU_VER_NV)
-		pp_smu = memset(pp_smu, 0, sizeof(struct pp_smu_funcs));
-
-	return pp_smu;
-}
-
-static void dcn20_pp_smu_destroy(struct pp_smu_funcs **pp_smu)
-{
-	if (pp_smu && *pp_smu) {
-		kfree(*pp_smu);
-		*pp_smu = NULL;
-	}
-}
-
-static struct _vcs_dpi_soc_bounding_box_st *get_asic_rev_soc_bb(
-	uint32_t hw_internal_rev)
-{
-	if (ASICREV_IS_NAVI14_M(hw_internal_rev))
-		return &dcn2_0_nv14_soc;
-
-	if (ASICREV_IS_NAVI12_P(hw_internal_rev))
-		return &dcn2_0_nv12_soc;
-
-	return &dcn2_0_soc;
-}
-
-static struct _vcs_dpi_ip_params_st *get_asic_rev_ip_params(
-	uint32_t hw_internal_rev)
-{
-	/* NV14 */
-	if (ASICREV_IS_NAVI14_M(hw_internal_rev))
-		return &dcn2_0_nv14_ip;
-
-	/* NV12 and NV10 */
-	return &dcn2_0_ip;
-}
-
-static enum dml_project get_dml_project_version(uint32_t hw_internal_rev)
-{
-	return DML_PROJECT_NAVI10v2;
-}
-
-static bool init_soc_bounding_box(struct dc *dc,
-				  struct dcn20_resource_pool *pool)
-{
-	struct _vcs_dpi_soc_bounding_box_st *loaded_bb =
-			get_asic_rev_soc_bb(dc->ctx->asic_id.hw_internal_rev);
-	struct _vcs_dpi_ip_params_st *loaded_ip =
-			get_asic_rev_ip_params(dc->ctx->asic_id.hw_internal_rev);
-
-	DC_LOGGER_INIT(dc->ctx->logger);
-
-	if (pool->base.pp_smu) {
-		struct pp_smu_nv_clock_table max_clocks = {0};
-		unsigned int uclk_states[8] = {0};
-		unsigned int num_states = 0;
-		enum pp_smu_status status;
-		bool clock_limits_available = false;
-		bool uclk_states_available = false;
-
-		if (pool->base.pp_smu->nv_funcs.get_uclk_dpm_states) {
-			status = (pool->base.pp_smu->nv_funcs.get_uclk_dpm_states)
-				(&pool->base.pp_smu->nv_funcs.pp_smu, uclk_states, &num_states);
-
-			uclk_states_available = (status == PP_SMU_RESULT_OK);
-		}
-
-		if (pool->base.pp_smu->nv_funcs.get_maximum_sustainable_clocks) {
-			status = (*pool->base.pp_smu->nv_funcs.get_maximum_sustainable_clocks)
-					(&pool->base.pp_smu->nv_funcs.pp_smu, &max_clocks);
-			/* SMU cannot set DCF clock to anything equal to or higher than SOC clock
-			 */
-			if (max_clocks.dcfClockInKhz >= max_clocks.socClockInKhz)
-				max_clocks.dcfClockInKhz = max_clocks.socClockInKhz - 1000;
-			clock_limits_available = (status == PP_SMU_RESULT_OK);
-		}
-
-		if (clock_limits_available && uclk_states_available && num_states) {
-			DC_FP_START();
-			dcn20_update_bounding_box(dc, loaded_bb, &max_clocks, uclk_states, num_states);
-			DC_FP_END();
-		} else if (clock_limits_available) {
-			DC_FP_START();
-			dcn20_cap_soc_clocks(loaded_bb, max_clocks);
-			DC_FP_END();
-		}
-	}
-
-	loaded_ip->max_num_otg = pool->base.res_cap->num_timing_generator;
-	loaded_ip->max_num_dpp = pool->base.pipe_count;
-	DC_FP_START();
-	dcn20_patch_bounding_box(dc, loaded_bb);
-	DC_FP_END();
-	return true;
-}
-
-static bool dcn20_resource_construct(
-	uint8_t num_virtual_links,
-	struct dc *dc,
-	struct dcn20_resource_pool *pool)
-{
-	int i;
-	struct dc_context *ctx = dc->ctx;
-	struct irq_service_init_data init_data;
-	struct ddc_service_init_data ddc_init_data = {0};
-	struct _vcs_dpi_soc_bounding_box_st *loaded_bb =
-			get_asic_rev_soc_bb(ctx->asic_id.hw_internal_rev);
-	struct _vcs_dpi_ip_params_st *loaded_ip =
-			get_asic_rev_ip_params(ctx->asic_id.hw_internal_rev);
-	enum dml_project dml_project_version =
-			get_dml_project_version(ctx->asic_id.hw_internal_rev);
-
-	ctx->dc_bios->regs = &bios_regs;
-	pool->base.funcs = &dcn20_res_pool_funcs;
-
-	if (ASICREV_IS_NAVI14_M(ctx->asic_id.hw_internal_rev)) {
-		pool->base.res_cap = &res_cap_nv14;
-		pool->base.pipe_count = 5;
-		pool->base.mpcc_count = 5;
-	} else {
-		pool->base.res_cap = &res_cap_nv10;
-		pool->base.pipe_count = 6;
-		pool->base.mpcc_count = 6;
-	}
-	/*************************************************
-	 *  Resource + asic cap harcoding                *
-	 *************************************************/
-	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
-
-	dc->caps.max_downscale_ratio = 200;
-	dc->caps.i2c_speed_in_khz = 100;
-	dc->caps.i2c_speed_in_khz_hdcp = 100; /*1.4 w/a not applied by default*/
-	dc->caps.max_cursor_size = 256;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.dmdata_alloc_size = 2048;
-
-	dc->caps.max_slave_planes = 1;
-	dc->caps.max_slave_yuv_planes = 1;
-	dc->caps.max_slave_rgb_planes = 1;
-	dc->caps.post_blend_color_processing = true;
-	dc->caps.force_dp_tps4_for_cp2520 = true;
-	dc->caps.extended_aux_timeout_support = true;
-
-	/* Color pipeline capabilities */
-	dc->caps.color.dpp.dcn_arch = 1;
-	dc->caps.color.dpp.input_lut_shared = 0;
-	dc->caps.color.dpp.icsc = 1;
-	dc->caps.color.dpp.dgam_ram = 1;
-	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
-	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.dpp.dgam_rom_caps.pq = 0;
-	dc->caps.color.dpp.dgam_rom_caps.hlg = 0;
-	dc->caps.color.dpp.post_csc = 0;
-	dc->caps.color.dpp.gamma_corr = 0;
-	dc->caps.color.dpp.dgam_rom_for_yuv = 1;
-
-	dc->caps.color.dpp.hw_3d_lut = 1;
-	dc->caps.color.dpp.ogam_ram = 1;
-	// no OGAM ROM on DCN2, only MPC ROM
-	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
-	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
-	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
-	dc->caps.color.dpp.ocsc = 0;
-
-	dc->caps.color.mpc.gamut_remap = 0;
-	dc->caps.color.mpc.num_3dluts = 0;
-	dc->caps.color.mpc.shared_3d_lut = 0;
-	dc->caps.color.mpc.ogam_ram = 1;
-	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
-	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
-	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
-	dc->caps.color.mpc.ocsc = 1;
-
-	dc->caps.dp_hdmi21_pcon_support = true;
-
-	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
-		dc->debug = debug_defaults_drv;
-
-	//dcn2.0x
-	dc->work_arounds.dedcn20_305_wa = true;
-
-	// Init the vm_helper
-	if (dc->vm_helper)
-		vm_helper_init(dc->vm_helper, 16);
-
-	/*************************************************
-	 *  Create resources                             *
-	 *************************************************/
-
-	pool->base.clock_sources[DCN20_CLK_SRC_PLL0] =
-			dcn20_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL0,
-				&clk_src_regs[0], false);
-	pool->base.clock_sources[DCN20_CLK_SRC_PLL1] =
-			dcn20_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL1,
-				&clk_src_regs[1], false);
-	pool->base.clock_sources[DCN20_CLK_SRC_PLL2] =
-			dcn20_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL2,
-				&clk_src_regs[2], false);
-	pool->base.clock_sources[DCN20_CLK_SRC_PLL3] =
-			dcn20_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL3,
-				&clk_src_regs[3], false);
-	pool->base.clock_sources[DCN20_CLK_SRC_PLL4] =
-			dcn20_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL4,
-				&clk_src_regs[4], false);
-	pool->base.clock_sources[DCN20_CLK_SRC_PLL5] =
-			dcn20_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL5,
-				&clk_src_regs[5], false);
-	pool->base.clk_src_count = DCN20_CLK_SRC_TOTAL;
-	/* todo: not reuse phy_pll registers */
-	pool->base.dp_clock_source =
-			dcn20_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_ID_DP_DTO,
-				&clk_src_regs[0], true);
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-
-	pool->base.dccg = dccg2_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
-	if (pool->base.dccg == NULL) {
-		dm_error("DC: failed to create dccg!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	pool->base.dmcu = dcn20_dmcu_create(ctx,
-			&dmcu_regs,
-			&dmcu_shift,
-			&dmcu_mask);
-	if (pool->base.dmcu == NULL) {
-		dm_error("DC: failed to create dmcu!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	pool->base.abm = dce_abm_create(ctx,
-			&abm_regs,
-			&abm_shift,
-			&abm_mask);
-	if (pool->base.abm == NULL) {
-		dm_error("DC: failed to create abm!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	pool->base.pp_smu = dcn20_pp_smu_create(ctx);
-
-
-	if (!init_soc_bounding_box(dc, pool)) {
-		dm_error("DC: failed to initialize soc bounding box!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	dml_init_instance(&dc->dml, loaded_bb, loaded_ip, dml_project_version);
-
-	if (!dc->debug.disable_pplib_wm_range) {
-		struct pp_smu_wm_range_sets ranges = {0};
-		int i = 0;
-
-		ranges.num_reader_wm_sets = 0;
-
-		if (loaded_bb->num_states == 1) {
-			ranges.reader_wm_sets[0].wm_inst = i;
-			ranges.reader_wm_sets[0].min_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
-			ranges.reader_wm_sets[0].max_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
-			ranges.reader_wm_sets[0].min_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
-			ranges.reader_wm_sets[0].max_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
-
-			ranges.num_reader_wm_sets = 1;
-		} else if (loaded_bb->num_states > 1) {
-			for (i = 0; i < 4 && i < loaded_bb->num_states; i++) {
-				ranges.reader_wm_sets[i].wm_inst = i;
-				ranges.reader_wm_sets[i].min_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
-				ranges.reader_wm_sets[i].max_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
-				DC_FP_START();
-				dcn20_fpu_set_wm_ranges(i, &ranges, loaded_bb);
-				DC_FP_END();
-
-				ranges.num_reader_wm_sets = i + 1;
-			}
-
-			ranges.reader_wm_sets[0].min_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
-			ranges.reader_wm_sets[ranges.num_reader_wm_sets - 1].max_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
-		}
-
-		ranges.num_writer_wm_sets = 1;
-
-		ranges.writer_wm_sets[0].wm_inst = 0;
-		ranges.writer_wm_sets[0].min_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
-		ranges.writer_wm_sets[0].max_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
-		ranges.writer_wm_sets[0].min_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
-		ranges.writer_wm_sets[0].max_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
-
-		/* Notify PP Lib/SMU which Watermarks to use for which clock ranges */
-		if (pool->base.pp_smu->nv_funcs.set_wm_ranges)
-			pool->base.pp_smu->nv_funcs.set_wm_ranges(&pool->base.pp_smu->nv_funcs.pp_smu, &ranges);
-	}
-
-	init_data.ctx = dc->ctx;
-	pool->base.irqs = dal_irq_service_dcn20_create(&init_data);
-	if (!pool->base.irqs)
-		goto create_fail;
-
-	/* mem input -> ipp -> dpp -> opp -> TG */
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		pool->base.hubps[i] = dcn20_hubp_create(ctx, i);
-		if (pool->base.hubps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create memory input!\n");
-			goto create_fail;
-		}
-
-		pool->base.ipps[i] = dcn20_ipp_create(ctx, i);
-		if (pool->base.ipps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create input pixel processor!\n");
-			goto create_fail;
-		}
-
-		pool->base.dpps[i] = dcn20_dpp_create(ctx, i);
-		if (pool->base.dpps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create dpps!\n");
-			goto create_fail;
-		}
-	}
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		pool->base.engines[i] = dcn20_aux_engine_create(ctx, i);
-		if (pool->base.engines[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create aux engine!!\n");
-			goto create_fail;
-		}
-		pool->base.hw_i2cs[i] = dcn20_i2c_hw_create(ctx, i);
-		if (pool->base.hw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create hw i2c!!\n");
-			goto create_fail;
-		}
-		pool->base.sw_i2cs[i] = NULL;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
-		pool->base.opps[i] = dcn20_opp_create(ctx, i);
-		if (pool->base.opps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create output pixel processor!\n");
-			goto create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		pool->base.timing_generators[i] = dcn20_timing_generator_create(
-				ctx, i);
-		if (pool->base.timing_generators[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create tg!\n");
-			goto create_fail;
-		}
-	}
-
-	pool->base.timing_generator_count = i;
-
-	pool->base.mpc = dcn20_mpc_create(ctx);
-	if (pool->base.mpc == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mpc!\n");
-		goto create_fail;
-	}
-
-	pool->base.hubbub = dcn20_hubbub_create(ctx);
-	if (pool->base.hubbub == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create hubbub!\n");
-		goto create_fail;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		pool->base.dscs[i] = dcn20_dsc_create(ctx, i);
-		if (pool->base.dscs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create display stream compressor %d!\n", i);
-			goto create_fail;
-		}
-	}
-
-	if (!dcn20_dwbc_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create dwbc!\n");
-		goto create_fail;
-	}
-	if (!dcn20_mmhubbub_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mcif_wb!\n");
-		goto create_fail;
-	}
-
-	if (!resource_construct(num_virtual_links, dc, &pool->base,
-			&res_create_funcs))
-		goto create_fail;
-
-	dcn20_hw_sequencer_construct(dc);
-
-	// IF NV12, set PG function pointer to NULL. It's not that
-	// PG isn't supported for NV12, it's that we don't want to
-	// program the registers because that will cause more power
-	// to be consumed. We could have created dcn20_init_hw to get
-	// the same effect by checking ASIC rev, but there was a
-	// request at some point to not check ASIC rev on hw sequencer.
-	if (ASICREV_IS_NAVI12_P(dc->ctx->asic_id.hw_internal_rev)) {
-		dc->hwseq->funcs.enable_power_gating_plane = NULL;
-		dc->debug.disable_dpp_power_gate = true;
-		dc->debug.disable_hubp_power_gate = true;
-	}
-
-
-	dc->caps.max_planes =  pool->base.pipe_count;
-
-	for (i = 0; i < dc->caps.max_planes; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	dc->cap_funcs = cap_funcs;
-
-	if (dc->ctx->dc_bios->fw_info.oem_i2c_present) {
-		ddc_init_data.ctx = dc->ctx;
-		ddc_init_data.link = NULL;
-		ddc_init_data.id.id = dc->ctx->dc_bios->fw_info.oem_i2c_obj_id;
-		ddc_init_data.id.enum_id = 0;
-		ddc_init_data.id.type = OBJECT_TYPE_GENERIC;
-		pool->base.oem_device = dc->link_srv->create_ddc_service(&ddc_init_data);
-	} else {
-		pool->base.oem_device = NULL;
-	}
-
-	return true;
-
-create_fail:
-
-	dcn20_resource_destruct(pool);
-
-	return false;
-}
-
-struct resource_pool *dcn20_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc)
-{
-	struct dcn20_resource_pool *pool =
-		kzalloc(sizeof(struct dcn20_resource_pool), GFP_ATOMIC);
-
-	if (!pool)
-		return NULL;
-
-	if (dcn20_resource_construct(init_data->num_virtual_links, dc, pool))
-		return &pool->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(pool);
-	return NULL;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.h
deleted file mode 100644
index 37ecaccc5d12..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.h
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
-* Copyright 2017 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#ifndef __DC_RESOURCE_DCN20_H__
-#define __DC_RESOURCE_DCN20_H__
-
-#include "core_types.h"
-#include "dml/dcn20/dcn20_fpu.h"
-
-#define TO_DCN20_RES_POOL(pool)\
-	container_of(pool, struct dcn20_resource_pool, base)
-
-struct dc;
-struct resource_pool;
-struct _vcs_dpi_display_pipe_params_st;
-
-extern struct _vcs_dpi_ip_params_st dcn2_0_ip;
-extern struct _vcs_dpi_ip_params_st dcn2_0_nv14_ip;
-extern struct _vcs_dpi_soc_bounding_box_st dcn2_0_soc;
-extern struct _vcs_dpi_soc_bounding_box_st dcn2_0_nv14_soc;
-extern struct _vcs_dpi_soc_bounding_box_st dcn2_0_nv12_soc;
-
-struct dcn20_resource_pool {
-	struct resource_pool base;
-};
-struct resource_pool *dcn20_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc);
-
-struct link_encoder *dcn20_link_encoder_create(
-	struct dc_context *ctx,
-	const struct encoder_init_data *enc_init_data);
-
-unsigned int dcn20_calc_max_scaled_time(
-		unsigned int time_per_pixel,
-		enum mmhubbub_wbif_mode mode,
-		unsigned int urgent_watermark);
-
-struct pipe_ctx *dcn20_acquire_free_pipe_for_layer(
-		const struct dc_state *cur_ctx,
-		struct dc_state *new_ctx,
-		const struct resource_pool *pool,
-		const struct pipe_ctx *opp_head_pipe);
-void dcn20_release_pipe(struct dc_state *context,
-			struct pipe_ctx *pipe,
-			const struct resource_pool *pool);
-struct stream_encoder *dcn20_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx);
-
-struct dce_hwseq *dcn20_hwseq_create(
-	struct dc_context *ctx);
-
-bool dcn20_get_dcc_compression_cap(const struct dc *dc,
-		const struct dc_dcc_surface_param *input,
-		struct dc_surface_dcc_cap *output);
-
-void dcn20_dpp_destroy(struct dpp **dpp);
-
-struct dpp *dcn20_dpp_create(
-	struct dc_context *ctx,
-	uint32_t inst);
-
-struct input_pixel_processor *dcn20_ipp_create(
-	struct dc_context *ctx, uint32_t inst);
-
-struct output_pixel_processor *dcn20_opp_create(
-	struct dc_context *ctx, uint32_t inst);
-
-struct dce_aux *dcn20_aux_engine_create(
-	struct dc_context *ctx, uint32_t inst);
-
-struct dce_i2c_hw *dcn20_i2c_hw_create(
-	struct dc_context *ctx,
-	uint32_t inst);
-
-void dcn20_clock_source_destroy(struct clock_source **clk_src);
-
-struct display_stream_compressor *dcn20_dsc_create(
-	struct dc_context *ctx, uint32_t inst);
-void dcn20_dsc_destroy(struct display_stream_compressor **dsc);
-
-struct hubp *dcn20_hubp_create(
-	struct dc_context *ctx,
-	uint32_t inst);
-struct timing_generator *dcn20_timing_generator_create(
-		struct dc_context *ctx,
-		uint32_t instance);
-struct mpc *dcn20_mpc_create(struct dc_context *ctx);
-struct hubbub *dcn20_hubbub_create(struct dc_context *ctx);
-
-bool dcn20_dwbc_create(struct dc_context *ctx, struct resource_pool *pool);
-bool dcn20_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool);
-
-void dcn20_set_mcif_arb_params(
-		struct dc *dc,
-		struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		int pipe_cnt);
-bool dcn20_validate_bandwidth(struct dc *dc, struct dc_state *context, bool fast_validate);
-void dcn20_merge_pipes_for_validate(
-		struct dc *dc,
-		struct dc_state *context);
-int dcn20_validate_apply_pipe_split_flags(
-		struct dc *dc,
-		struct dc_state *context,
-		int vlevel,
-		int *split,
-		bool *merge);
-void dcn20_release_dsc(struct resource_context *res_ctx,
-			const struct resource_pool *pool,
-			struct display_stream_compressor **dsc);
-bool dcn20_validate_dsc(struct dc *dc, struct dc_state *new_ctx);
-void dcn20_split_stream_for_mpc(
-		struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		struct pipe_ctx *primary_pipe,
-		struct pipe_ctx *secondary_pipe);
-bool dcn20_split_stream_for_odm(
-		const struct dc *dc,
-		struct resource_context *res_ctx,
-		struct pipe_ctx *prev_odm_pipe,
-		struct pipe_ctx *next_odm_pipe);
-void dcn20_acquire_dsc(const struct dc *dc,
-			struct resource_context *res_ctx,
-			struct display_stream_compressor **dsc,
-			int pipe_idx);
-struct pipe_ctx *dcn20_find_secondary_pipe(struct dc *dc,
-		struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		const struct pipe_ctx *primary_pipe);
-bool dcn20_fast_validate_bw(
-		struct dc *dc,
-		struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		int *pipe_cnt_out,
-		int *pipe_split_from,
-		int *vlevel_out,
-		bool fast_validate);
-
-enum dc_status dcn20_build_mapped_resource(const struct dc *dc, struct dc_state *context, struct dc_stream_state *stream);
-enum dc_status dcn20_add_stream_to_ctx(struct dc *dc, struct dc_state *new_ctx, struct dc_stream_state *dc_stream);
-enum dc_status dcn20_add_dsc_to_stream_resource(struct dc *dc, struct dc_state *dc_ctx, struct dc_stream_state *dc_stream);
-enum dc_status dcn20_remove_stream_from_ctx(struct dc *dc, struct dc_state *new_ctx, struct dc_stream_state *dc_stream);
-enum dc_status dcn20_patch_unknown_plane_state(struct dc_plane_state *plane_state);
-
-#endif /* __DC_RESOURCE_DCN20_H__ */
-
diff --git a/drivers/gpu/drm/amd/display/dc/dcn201/Makefile b/drivers/gpu/drm/amd/display/dc/dcn201/Makefile
index 3a41a97b0729..c069a894db92 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn201/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn201/Makefile
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: MIT
 #
 # Makefile for DCN.
-DCN201 = dcn201_init.o dcn201_resource.o \
+DCN201 = dcn201_init.o \
 	dcn201_hubbub.o\
 	dcn201_mpc.o dcn201_hubp.o dcn201_opp.o dcn201_optc.o dcn201_dpp.o \
 	dcn201_dccg.o dcn201_link_encoder.o
diff --git a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_resource.c b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_resource.c
deleted file mode 100644
index bca22d867696..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_resource.c
+++ /dev/null
@@ -1,1308 +0,0 @@
-/*
-* Copyright 2016 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#include "dm_services.h"
-#include "dc.h"
-
-#include "dcn201_init.h"
-#include "dml/dcn20/dcn20_fpu.h"
-#include "resource.h"
-#include "include/irq_service_interface.h"
-#include "dcn201_resource.h"
-
-#include "dcn20/dcn20_resource.h"
-
-#include "dcn10/dcn10_hubp.h"
-#include "dcn10/dcn10_ipp.h"
-#include "dcn201_mpc.h"
-#include "dcn201_hubp.h"
-#include "irq/dcn201/irq_service_dcn201.h"
-#include "dcn201/dcn201_dpp.h"
-#include "dcn201/dcn201_hubbub.h"
-#include "dcn201_dccg.h"
-#include "dcn201_optc.h"
-#include "dcn201/dcn201_hwseq.h"
-#include "dce110/dce110_hwseq.h"
-#include "dcn201_opp.h"
-#include "dcn201/dcn201_link_encoder.h"
-#include "dcn20/dcn20_stream_encoder.h"
-#include "dce/dce_clock_source.h"
-#include "dce/dce_audio.h"
-#include "dce/dce_hwseq.h"
-#include "virtual/virtual_stream_encoder.h"
-#include "dce110/dce110_resource.h"
-#include "dce/dce_aux.h"
-#include "dce/dce_i2c.h"
-#include "dcn201_hubbub.h"
-#include "dcn10/dcn10_resource.h"
-
-#include "cyan_skillfish_ip_offset.h"
-
-#include "dcn/dcn_2_0_3_offset.h"
-#include "dcn/dcn_2_0_3_sh_mask.h"
-#include "dpcs/dpcs_2_0_3_offset.h"
-#include "dpcs/dpcs_2_0_3_sh_mask.h"
-
-#include "mmhub/mmhub_2_0_0_offset.h"
-#include "mmhub/mmhub_2_0_0_sh_mask.h"
-#include "nbio/nbio_7_4_offset.h"
-
-#include "reg_helper.h"
-
-#define MIN_DISP_CLK_KHZ 100000
-#define MIN_DPP_CLK_KHZ 100000
-
-static struct _vcs_dpi_ip_params_st dcn201_ip = {
-	.gpuvm_enable = 0,
-	.hostvm_enable = 0,
-	.gpuvm_max_page_table_levels = 4,
-	.hostvm_max_page_table_levels = 4,
-	.hostvm_cached_page_table_levels = 0,
-	.pte_group_size_bytes = 2048,
-	.rob_buffer_size_kbytes = 168,
-	.det_buffer_size_kbytes = 164,
-	.dpte_buffer_size_in_pte_reqs_luma = 84,
-	.pde_proc_buffer_size_64k_reqs = 48,
-	.dpp_output_buffer_pixels = 2560,
-	.opp_output_buffer_lines = 1,
-	.pixel_chunk_size_kbytes = 8,
-	.pte_chunk_size_kbytes = 2,
-	.meta_chunk_size_kbytes = 2,
-	.writeback_chunk_size_kbytes = 2,
-	.line_buffer_size_bits = 789504,
-	.is_line_buffer_bpp_fixed = 0,
-	.line_buffer_fixed_bpp = 0,
-	.dcc_supported = true,
-	.max_line_buffer_lines = 12,
-	.writeback_luma_buffer_size_kbytes = 12,
-	.writeback_chroma_buffer_size_kbytes = 8,
-	.writeback_chroma_line_buffer_width_pixels = 4,
-	.writeback_max_hscl_ratio = 1,
-	.writeback_max_vscl_ratio = 1,
-	.writeback_min_hscl_ratio = 1,
-	.writeback_min_vscl_ratio = 1,
-	.writeback_max_hscl_taps = 12,
-	.writeback_max_vscl_taps = 12,
-	.writeback_line_buffer_luma_buffer_size = 0,
-	.writeback_line_buffer_chroma_buffer_size = 9600,
-	.cursor_buffer_size = 8,
-	.cursor_chunk_size = 2,
-	.max_num_otg = 2,
-	.max_num_dpp = 4,
-	.max_num_wb = 0,
-	.max_dchub_pscl_bw_pix_per_clk = 4,
-	.max_pscl_lb_bw_pix_per_clk = 2,
-	.max_lb_vscl_bw_pix_per_clk = 4,
-	.max_vscl_hscl_bw_pix_per_clk = 4,
-	.max_hscl_ratio = 8,
-	.max_vscl_ratio = 8,
-	.hscl_mults = 4,
-	.vscl_mults = 4,
-	.max_hscl_taps = 8,
-	.max_vscl_taps = 8,
-	.dispclk_ramp_margin_percent = 1,
-	.underscan_factor = 1.10,
-	.min_vblank_lines = 30,
-	.dppclk_delay_subtotal = 77,
-	.dppclk_delay_scl_lb_only = 16,
-	.dppclk_delay_scl = 50,
-	.dppclk_delay_cnvc_formatter = 8,
-	.dppclk_delay_cnvc_cursor = 6,
-	.dispclk_delay_subtotal = 87,
-	.dcfclk_cstate_latency = 10,
-	.max_inter_dcn_tile_repeaters = 8,
-	.number_of_cursors = 1,
-};
-
-static struct _vcs_dpi_soc_bounding_box_st dcn201_soc = {
-	.clock_limits = {
-			{
-				.state = 0,
-				.dscclk_mhz = 400.0,
-				.dcfclk_mhz = 1000.0,
-				.fabricclk_mhz = 200.0,
-				.dispclk_mhz = 300.0,
-				.dppclk_mhz = 300.0,
-				.phyclk_mhz = 810.0,
-				.socclk_mhz = 1254.0,
-				.dram_speed_mts = 2000.0,
-			},
-			{
-				.state = 1,
-				.dscclk_mhz = 400.0,
-				.dcfclk_mhz = 1000.0,
-				.fabricclk_mhz = 250.0,
-				.dispclk_mhz = 1200.0,
-				.dppclk_mhz = 1200.0,
-				.phyclk_mhz = 810.0,
-				.socclk_mhz = 1254.0,
-				.dram_speed_mts = 3600.0,
-			},
-			{
-				.state = 2,
-				.dscclk_mhz = 400.0,
-				.dcfclk_mhz = 1000.0,
-				.fabricclk_mhz = 750.0,
-				.dispclk_mhz = 1200.0,
-				.dppclk_mhz = 1200.0,
-				.phyclk_mhz = 810.0,
-				.socclk_mhz = 1254.0,
-				.dram_speed_mts = 6800.0,
-			},
-			{
-				.state = 3,
-				.dscclk_mhz = 400.0,
-				.dcfclk_mhz = 1000.0,
-				.fabricclk_mhz = 250.0,
-				.dispclk_mhz = 1200.0,
-				.dppclk_mhz = 1200.0,
-				.phyclk_mhz = 810.0,
-				.socclk_mhz = 1254.0,
-				.dram_speed_mts = 14000.0,
-			},
-			{
-				.state = 4,
-				.dscclk_mhz = 400.0,
-				.dcfclk_mhz = 1000.0,
-				.fabricclk_mhz = 750.0,
-				.dispclk_mhz = 1200.0,
-				.dppclk_mhz = 1200.0,
-				.phyclk_mhz = 810.0,
-				.socclk_mhz = 1254.0,
-				.dram_speed_mts = 14000.0,
-			}
-		},
-	.num_states = 4,
-	.sr_exit_time_us = 9.0,
-	.sr_enter_plus_exit_time_us = 11.0,
-	.urgent_latency_us = 4.0,
-	.urgent_latency_pixel_data_only_us = 4.0,
-	.urgent_latency_pixel_mixed_with_vm_data_us = 4.0,
-	.urgent_latency_vm_data_only_us = 4.0,
-	.urgent_out_of_order_return_per_channel_pixel_only_bytes = 256,
-	.urgent_out_of_order_return_per_channel_pixel_and_vm_bytes = 256,
-	.urgent_out_of_order_return_per_channel_vm_only_bytes = 256,
-	.pct_ideal_dram_sdp_bw_after_urgent_pixel_only = 80.0,
-	.pct_ideal_dram_sdp_bw_after_urgent_pixel_and_vm = 80.0,
-	.pct_ideal_dram_sdp_bw_after_urgent_vm_only = 80.0,
-	.max_avg_sdp_bw_use_normal_percent = 80.0,
-	.max_avg_dram_bw_use_normal_percent = 69.0,
-	.writeback_latency_us = 12.0,
-	.ideal_dram_bw_after_urgent_percent = 80.0,
-	.max_request_size_bytes = 256,
-	.dram_channel_width_bytes = 2,
-	.fabric_datapath_to_dcn_data_return_bytes = 64,
-	.dcn_downspread_percent = 0.3,
-	.downspread_percent = 0.3,
-	.dram_page_open_time_ns = 50.0,
-	.dram_rw_turnaround_time_ns = 17.5,
-	.dram_return_buffer_per_channel_bytes = 8192,
-	.round_trip_ping_latency_dcfclk_cycles = 128,
-	.urgent_out_of_order_return_per_channel_bytes = 256,
-	.channel_interleave_bytes = 256,
-	.num_banks = 8,
-	.num_chans = 16,
-	.vmm_page_size_bytes = 4096,
-	.dram_clock_change_latency_us = 250.0,
-	.writeback_dram_clock_change_latency_us = 23.0,
-	.return_bus_width_bytes = 64,
-	.dispclk_dppclk_vco_speed_mhz = 3000,
-	.use_urgent_burst_bw = 0,
-};
-
-enum dcn20_clk_src_array_id {
-	DCN20_CLK_SRC_PLL0,
-	DCN20_CLK_SRC_PLL1,
-	DCN20_CLK_SRC_TOTAL_DCN201
-};
-
-/* begin *********************
- * macros to expend register list macro defined in HW object header file */
-
-/* DCN */
-
-#undef BASE_INNER
-#define BASE_INNER(seg) DMU_BASE__INST0_SEG ## seg
-
-#define BASE(seg) BASE_INNER(seg)
-
-#define SR(reg_name)\
-		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) +  \
-					mm ## reg_name
-
-#define SRI(reg_name, block, id)\
-	.reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define SRIR(var_name, reg_name, block, id)\
-	.var_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define SRI_IX(reg_name, block, id)\
-	.reg_name = ix ## block ## id ## _ ## reg_name
-
-#define DCCG_SRII(reg_name, block, id)\
-	.block ## _ ## reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define VUPDATE_SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(mm ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
-					mm ## reg_name ## _ ## block ## id
-
-/* NBIO */
-#define NBIO_BASE_INNER(seg) \
-	NBIO_BASE__INST0_SEG ## seg
-
-#define NBIO_BASE(seg) \
-	NBIO_BASE_INNER(seg)
-
-#define NBIO_SR(reg_name)\
-		.reg_name = NBIO_BASE(mm ## reg_name ## _BASE_IDX) + \
-					mm ## reg_name
-
-/* MMHUB */
-#define MMHUB_BASE_INNER(seg) \
-	MMHUB_BASE__INST0_SEG ## seg
-
-#define MMHUB_BASE(seg) \
-	MMHUB_BASE_INNER(seg)
-
-#define MMHUB_SR(reg_name)\
-		.reg_name = MMHUB_BASE(mmMM ## reg_name ## _BASE_IDX) + \
-					mmMM ## reg_name
-
-static const struct bios_registers bios_regs = {
-		NBIO_SR(BIOS_SCRATCH_3),
-		NBIO_SR(BIOS_SCRATCH_6)
-};
-
-#define clk_src_regs(index, pllid)\
-[index] = {\
-	CS_COMMON_REG_LIST_DCN201(index, pllid),\
-}
-
-static const struct dce110_clk_src_regs clk_src_regs[] = {
-	clk_src_regs(0, A),
-	clk_src_regs(1, B)
-};
-
-static const struct dce110_clk_src_shift cs_shift = {
-		CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
-};
-
-static const struct dce110_clk_src_mask cs_mask = {
-		CS_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
-};
-
-#define audio_regs(id)\
-[id] = {\
-		AUD_COMMON_REG_LIST(id)\
-}
-
-static const struct dce_audio_registers audio_regs[] = {
-	audio_regs(0),
-	audio_regs(1),
-};
-
-#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
-		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
-
-static const struct dce_audio_shift audio_shift = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_audio_mask audio_mask = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
-};
-
-#define stream_enc_regs(id)\
-[id] = {\
-	SE_DCN2_REG_LIST(id)\
-}
-
-static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
-	stream_enc_regs(0),
-	stream_enc_regs(1)
-};
-
-static const struct dcn10_stream_encoder_shift se_shift = {
-		SE_COMMON_MASK_SH_LIST_DCN20(__SHIFT)
-};
-
-static const struct dcn10_stream_encoder_mask se_mask = {
-		SE_COMMON_MASK_SH_LIST_DCN20(_MASK)
-};
-
-static const struct dce110_aux_registers_shift aux_shift = {
-	DCN_AUX_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce110_aux_registers_mask aux_mask = {
-	DCN_AUX_MASK_SH_LIST(_MASK)
-};
-
-#define aux_regs(id)\
-[id] = {\
-	DCN2_AUX_REG_LIST(id)\
-}
-
-static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
-		aux_regs(0),
-		aux_regs(1),
-};
-
-#define hpd_regs(id)\
-[id] = {\
-	HPD_REG_LIST(id)\
-}
-
-static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
-		hpd_regs(0),
-		hpd_regs(1),
-};
-
-#define link_regs(id, phyid)\
-[id] = {\
-	LE_DCN_COMMON_REG_LIST(id), \
-	UNIPHY_DCN2_REG_LIST(phyid) \
-}
-
-static const struct dcn10_link_enc_registers link_enc_regs[] = {
-	link_regs(0, A),
-	link_regs(1, B),
-};
-
-#define LINK_ENCODER_MASK_SH_LIST_DCN201(mask_sh)\
-	LINK_ENCODER_MASK_SH_LIST_DCN20(mask_sh)
-
-static const struct dcn10_link_enc_shift le_shift = {
-		LINK_ENCODER_MASK_SH_LIST_DCN201(__SHIFT)
-};
-
-static const struct dcn10_link_enc_mask le_mask = {
-		LINK_ENCODER_MASK_SH_LIST_DCN201(_MASK)
-};
-
-#define ipp_regs(id)\
-[id] = {\
-		IPP_REG_LIST_DCN201(id),\
-}
-
-static const struct dcn10_ipp_registers ipp_regs[] = {
-	ipp_regs(0),
-	ipp_regs(1),
-	ipp_regs(2),
-	ipp_regs(3),
-};
-
-static const struct dcn10_ipp_shift ipp_shift = {
-		IPP_MASK_SH_LIST_DCN201(__SHIFT)
-};
-
-static const struct dcn10_ipp_mask ipp_mask = {
-		IPP_MASK_SH_LIST_DCN201(_MASK)
-};
-
-#define opp_regs(id)\
-[id] = {\
-	OPP_REG_LIST_DCN201(id),\
-}
-
-static const struct dcn201_opp_registers opp_regs[] = {
-	opp_regs(0),
-	opp_regs(1),
-};
-
-static const struct dcn201_opp_shift opp_shift = {
-		OPP_MASK_SH_LIST_DCN201(__SHIFT)
-};
-
-static const struct dcn201_opp_mask opp_mask = {
-		OPP_MASK_SH_LIST_DCN201(_MASK)
-};
-
-#define aux_engine_regs(id)\
-[id] = {\
-	AUX_COMMON_REG_LIST0(id), \
-	.AUX_RESET_MASK = 0 \
-}
-
-static const struct dce110_aux_registers aux_engine_regs[] = {
-		aux_engine_regs(0),
-		aux_engine_regs(1)
-};
-
-#define tf_regs(id)\
-[id] = {\
-	TF_REG_LIST_DCN201(id),\
-}
-
-static const struct dcn201_dpp_registers tf_regs[] = {
-	tf_regs(0),
-	tf_regs(1),
-	tf_regs(2),
-	tf_regs(3),
-};
-
-static const struct dcn201_dpp_shift tf_shift = {
-		TF_REG_LIST_SH_MASK_DCN201(__SHIFT)
-};
-
-static const struct dcn201_dpp_mask tf_mask = {
-		TF_REG_LIST_SH_MASK_DCN201(_MASK)
-};
-
-static const struct dcn201_mpc_registers mpc_regs = {
-		MPC_REG_LIST_DCN201(0),
-		MPC_REG_LIST_DCN201(1),
-		MPC_REG_LIST_DCN201(2),
-		MPC_REG_LIST_DCN201(3),
-		MPC_REG_LIST_DCN201(4),
-		MPC_OUT_MUX_REG_LIST_DCN201(0),
-		MPC_OUT_MUX_REG_LIST_DCN201(1),
-};
-
-static const struct dcn201_mpc_shift mpc_shift = {
-	MPC_COMMON_MASK_SH_LIST_DCN201(__SHIFT)
-};
-
-static const struct dcn201_mpc_mask mpc_mask = {
-	MPC_COMMON_MASK_SH_LIST_DCN201(_MASK)
-};
-
-#define tg_regs_dcn201(id)\
-[id] = {TG_COMMON_REG_LIST_DCN201(id)}
-
-static const struct dcn_optc_registers tg_regs[] = {
-	tg_regs_dcn201(0),
-	tg_regs_dcn201(1)
-};
-
-static const struct dcn_optc_shift tg_shift = {
-	TG_COMMON_MASK_SH_LIST_DCN201(__SHIFT)
-};
-
-static const struct dcn_optc_mask tg_mask = {
-	TG_COMMON_MASK_SH_LIST_DCN201(_MASK)
-};
-
-#define hubp_regsDCN201(id)\
-[id] = {\
-	HUBP_REG_LIST_DCN201(id)\
-}
-
-static const struct dcn201_hubp_registers hubp_regs[] = {
-		hubp_regsDCN201(0),
-		hubp_regsDCN201(1),
-		hubp_regsDCN201(2),
-		hubp_regsDCN201(3)
-};
-
-static const struct dcn201_hubp_shift hubp_shift = {
-		HUBP_MASK_SH_LIST_DCN201(__SHIFT)
-};
-
-static const struct dcn201_hubp_mask hubp_mask = {
-		HUBP_MASK_SH_LIST_DCN201(_MASK)
-};
-
-static const struct dcn_hubbub_registers hubbub_reg = {
-		HUBBUB_REG_LIST_DCN201(0)
-};
-
-static const struct dcn_hubbub_shift hubbub_shift = {
-		HUBBUB_MASK_SH_LIST_DCN201(__SHIFT)
-};
-
-static const struct dcn_hubbub_mask hubbub_mask = {
-		HUBBUB_MASK_SH_LIST_DCN201(_MASK)
-};
-
-
-static const struct dccg_registers dccg_regs = {
-		DCCG_COMMON_REG_LIST_DCN_BASE()
-};
-
-static const struct dccg_shift dccg_shift = {
-		DCCG_COMMON_MASK_SH_LIST_DCN_COMMON_BASE(__SHIFT)
-};
-
-static const struct dccg_mask dccg_mask = {
-		DCCG_COMMON_MASK_SH_LIST_DCN_COMMON_BASE(_MASK)
-};
-
-static const struct resource_caps res_cap_dnc201 = {
-		.num_timing_generator = 2,
-		.num_opp = 2,
-		.num_video_plane = 4,
-		.num_audio = 2,
-		.num_stream_encoder = 2,
-		.num_pll = 2,
-		.num_ddc = 2,
-};
-
-static const struct dc_plane_cap plane_cap = {
-	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.per_pixel_alpha = true,
-
-	.pixel_format_support = {
-			.argb8888 = true,
-			.nv12 = false,
-			.fp16 = true,
-			.p010 = false,
-	},
-
-	.max_upscale_factor = {
-			.argb8888 = 16000,
-			.nv12 = 16000,
-			.fp16 = 1
-	},
-
-	.max_downscale_factor = {
-			.argb8888 = 250,
-			.nv12 = 250,
-			.fp16 = 250
-	},
-	64,
-	64
-};
-
-static const struct dc_debug_options debug_defaults_drv = {
-		.disable_dmcu = true,
-		.force_abm_enable = false,
-		.timing_trace = false,
-		.clock_trace = true,
-		.disable_pplib_clock_request = true,
-		.pipe_split_policy = MPC_SPLIT_DYNAMIC,
-		.force_single_disp_pipe_split = false,
-		.disable_dcc = DCC_ENABLE,
-		.vsr_support = true,
-		.performance_trace = false,
-		.az_endpoint_mute_only = true,
-		.max_downscale_src_width = 3840,
-		.disable_pplib_wm_range = true,
-		.scl_reset_length10 = true,
-		.sanity_checks = false,
-		.underflow_assert_delay_us = 0xFFFFFFFF,
-		.enable_tri_buf = false,
-		.enable_legacy_fast_update = true,
-		.using_dml2 = false,
-};
-
-static void dcn201_dpp_destroy(struct dpp **dpp)
-{
-	kfree(TO_DCN201_DPP(*dpp));
-	*dpp = NULL;
-}
-
-static struct dpp *dcn201_dpp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn201_dpp *dpp =
-		kzalloc(sizeof(struct dcn201_dpp), GFP_ATOMIC);
-
-	if (!dpp)
-		return NULL;
-
-	if (dpp201_construct(dpp, ctx, inst,
-			&tf_regs[inst], &tf_shift, &tf_mask))
-		return &dpp->base;
-
-	kfree(dpp);
-	return NULL;
-}
-
-static struct input_pixel_processor *dcn201_ipp_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn10_ipp *ipp =
-		kzalloc(sizeof(struct dcn10_ipp), GFP_ATOMIC);
-
-	if (!ipp) {
-		return NULL;
-	}
-
-	dcn20_ipp_construct(ipp, ctx, inst,
-			&ipp_regs[inst], &ipp_shift, &ipp_mask);
-	return &ipp->base;
-}
-
-
-static struct output_pixel_processor *dcn201_opp_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn201_opp *opp =
-		kzalloc(sizeof(struct dcn201_opp), GFP_ATOMIC);
-
-	if (!opp) {
-		return NULL;
-	}
-
-	dcn201_opp_construct(opp, ctx, inst,
-			&opp_regs[inst], &opp_shift, &opp_mask);
-	return &opp->base;
-}
-
-static struct dce_aux *dcn201_aux_engine_create(struct dc_context *ctx,
-						uint32_t inst)
-{
-	struct aux_engine_dce110 *aux_engine =
-		kzalloc(sizeof(struct aux_engine_dce110), GFP_ATOMIC);
-
-	if (!aux_engine)
-		return NULL;
-
-	dce110_aux_engine_construct(aux_engine, ctx, inst,
-				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
-				    &aux_engine_regs[inst],
-					&aux_mask,
-					&aux_shift,
-					ctx->dc->caps.extended_aux_timeout_support);
-
-	return &aux_engine->base;
-}
-#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
-
-static const struct dce_i2c_registers i2c_hw_regs[] = {
-		i2c_inst_regs(1),
-		i2c_inst_regs(2),
-};
-
-static const struct dce_i2c_shift i2c_shifts = {
-		I2C_COMMON_MASK_SH_LIST_DCN2(__SHIFT)
-};
-
-static const struct dce_i2c_mask i2c_masks = {
-		I2C_COMMON_MASK_SH_LIST_DCN2(_MASK)
-};
-
-static struct dce_i2c_hw *dcn201_i2c_hw_create(struct dc_context *ctx,
-					       uint32_t inst)
-{
-	struct dce_i2c_hw *dce_i2c_hw =
-		kzalloc(sizeof(struct dce_i2c_hw), GFP_ATOMIC);
-
-	if (!dce_i2c_hw)
-		return NULL;
-
-	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
-				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
-
-	return dce_i2c_hw;
-}
-
-static struct mpc *dcn201_mpc_create(struct dc_context *ctx, uint32_t num_mpcc)
-{
-	struct dcn201_mpc *mpc201 = kzalloc(sizeof(struct dcn201_mpc),
-					    GFP_ATOMIC);
-
-	if (!mpc201)
-		return NULL;
-
-	dcn201_mpc_construct(mpc201, ctx,
-			&mpc_regs,
-			&mpc_shift,
-			&mpc_mask,
-			num_mpcc);
-
-	return &mpc201->base;
-}
-
-static struct hubbub *dcn201_hubbub_create(struct dc_context *ctx)
-{
-	struct dcn20_hubbub *hubbub = kzalloc(sizeof(struct dcn20_hubbub),
-					  GFP_ATOMIC);
-
-	if (!hubbub)
-		return NULL;
-
-	hubbub201_construct(hubbub, ctx,
-			&hubbub_reg,
-			&hubbub_shift,
-			&hubbub_mask);
-
-	return &hubbub->base;
-}
-
-static struct timing_generator *dcn201_timing_generator_create(
-		struct dc_context *ctx,
-		uint32_t instance)
-{
-	struct optc *tgn10 =
-		kzalloc(sizeof(struct optc), GFP_ATOMIC);
-
-	if (!tgn10)
-		return NULL;
-
-	tgn10->base.inst = instance;
-	tgn10->base.ctx = ctx;
-
-	tgn10->tg_regs = &tg_regs[instance];
-	tgn10->tg_shift = &tg_shift;
-	tgn10->tg_mask = &tg_mask;
-
-	dcn201_timing_generator_init(tgn10);
-
-	return &tgn10->base;
-}
-
-static const struct encoder_feature_support link_enc_feature = {
-		.max_hdmi_deep_color = COLOR_DEPTH_121212,
-		.max_hdmi_pixel_clock = 600000,
-		.hdmi_ycbcr420_supported = true,
-		.dp_ycbcr420_supported = true,
-		.fec_supported = true,
-		.flags.bits.IS_HBR2_CAPABLE = true,
-		.flags.bits.IS_HBR3_CAPABLE = true,
-		.flags.bits.IS_TPS3_CAPABLE = true,
-		.flags.bits.IS_TPS4_CAPABLE = true
-};
-
-static struct link_encoder *dcn201_link_encoder_create(
-	struct dc_context *ctx,
-	const struct encoder_init_data *enc_init_data)
-{
-	struct dcn20_link_encoder *enc20 =
-		kzalloc(sizeof(struct dcn20_link_encoder), GFP_ATOMIC);
-	struct dcn10_link_encoder *enc10 = &enc20->enc10;
-
-	if (!enc20)
-		return NULL;
-
-	dcn201_link_encoder_construct(enc20,
-			enc_init_data,
-			&link_enc_feature,
-			&link_enc_regs[enc_init_data->transmitter],
-			&link_enc_aux_regs[enc_init_data->channel - 1],
-			&link_enc_hpd_regs[enc_init_data->hpd_source],
-			&le_shift,
-			&le_mask);
-
-	return &enc10->base;
-}
-
-static struct clock_source *dcn201_clock_source_create(
-	struct dc_context *ctx,
-	struct dc_bios *bios,
-	enum clock_source_id id,
-	const struct dce110_clk_src_regs *regs,
-	bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src =
-		kzalloc(sizeof(struct dce110_clk_src), GFP_ATOMIC);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dce112_clk_src_construct(clk_src, ctx, bios, id,
-			regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-	kfree(clk_src);
-	return NULL;
-}
-
-static void read_dce_straps(
-	struct dc_context *ctx,
-	struct resource_straps *straps)
-{
-	generic_reg_get(ctx, mmDC_PINSTRAPS + BASE(mmDC_PINSTRAPS_BASE_IDX),
-
-		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
-}
-
-static struct audio *dcn201_create_audio(
-		struct dc_context *ctx, unsigned int inst)
-{
-	return dce_audio_create(ctx, inst,
-			&audio_regs[inst], &audio_shift, &audio_mask);
-}
-
-static struct stream_encoder *dcn201_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dcn10_stream_encoder *enc1 =
-		kzalloc(sizeof(struct dcn10_stream_encoder), GFP_ATOMIC);
-
-	if (!enc1)
-		return NULL;
-
-	dcn20_stream_encoder_construct(enc1, ctx, ctx->dc_bios, eng_id,
-					&stream_enc_regs[eng_id],
-					&se_shift, &se_mask);
-
-	return &enc1->base;
-}
-
-static const struct dce_hwseq_registers hwseq_reg = {
-		HWSEQ_DCN201_REG_LIST()
-};
-
-static const struct dce_hwseq_shift hwseq_shift = {
-		HWSEQ_DCN201_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_hwseq_mask hwseq_mask = {
-		HWSEQ_DCN201_MASK_SH_LIST(_MASK)
-};
-
-static struct dce_hwseq *dcn201_hwseq_create(
-	struct dc_context *ctx)
-{
-	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_ATOMIC);
-
-	if (hws) {
-		hws->ctx = ctx;
-		hws->regs = &hwseq_reg;
-		hws->shifts = &hwseq_shift;
-		hws->masks = &hwseq_mask;
-	}
-	return hws;
-}
-
-static const struct resource_create_funcs res_create_funcs = {
-	.read_dce_straps = read_dce_straps,
-	.create_audio = dcn201_create_audio,
-	.create_stream_encoder = dcn201_stream_encoder_create,
-	.create_hwseq = dcn201_hwseq_create,
-};
-
-static void dcn201_clock_source_destroy(struct clock_source **clk_src)
-{
-	kfree(TO_DCE110_CLK_SRC(*clk_src));
-	*clk_src = NULL;
-}
-
-static void dcn201_resource_destruct(struct dcn201_resource_pool *pool)
-{
-	unsigned int i;
-
-	for (i = 0; i < pool->base.stream_enc_count; i++) {
-		if (pool->base.stream_enc[i] != NULL) {
-			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
-			pool->base.stream_enc[i] = NULL;
-		}
-	}
-
-
-	if (pool->base.mpc != NULL) {
-		kfree(TO_DCN201_MPC(pool->base.mpc));
-		pool->base.mpc = NULL;
-	}
-
-	if (pool->base.hubbub != NULL) {
-		kfree(pool->base.hubbub);
-		pool->base.hubbub = NULL;
-	}
-
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		if (pool->base.dpps[i] != NULL)
-			dcn201_dpp_destroy(&pool->base.dpps[i]);
-
-		if (pool->base.ipps[i] != NULL)
-			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
-
-		if (pool->base.hubps[i] != NULL) {
-			kfree(TO_DCN10_HUBP(pool->base.hubps[i]));
-			pool->base.hubps[i] = NULL;
-		}
-
-		if (pool->base.irqs != NULL) {
-			dal_irq_service_destroy(&pool->base.irqs);
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
-		if (pool->base.opps[i] != NULL)
-			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		if (pool->base.timing_generators[i] != NULL)	{
-			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
-			pool->base.timing_generators[i] = NULL;
-		}
-	}
-	for (i = 0; i < pool->base.audio_count; i++) {
-		if (pool->base.audios[i])
-			dce_aud_destroy(&pool->base.audios[i]);
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] != NULL) {
-			dcn201_clock_source_destroy(&pool->base.clock_sources[i]);
-			pool->base.clock_sources[i] = NULL;
-		}
-	}
-
-	if (pool->base.dp_clock_source != NULL) {
-		dcn201_clock_source_destroy(&pool->base.dp_clock_source);
-		pool->base.dp_clock_source = NULL;
-	}
-
-	if (pool->base.dccg != NULL)
-		dcn_dccg_destroy(&pool->base.dccg);
-}
-
-static struct hubp *dcn201_hubp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn201_hubp *hubp201 =
-		kzalloc(sizeof(struct dcn201_hubp), GFP_ATOMIC);
-
-	if (!hubp201)
-		return NULL;
-
-	if (dcn201_hubp_construct(hubp201, ctx, inst,
-			&hubp_regs[inst], &hubp_shift, &hubp_mask))
-		return &hubp201->base;
-
-	kfree(hubp201);
-	return NULL;
-}
-
-static struct pipe_ctx *dcn201_acquire_free_pipe_for_layer(
-		const struct dc_state *cur_ctx,
-		struct dc_state *new_ctx,
-		const struct resource_pool *pool,
-		const struct pipe_ctx *opp_head_pipe)
-{
-	struct resource_context *res_ctx = &new_ctx->res_ctx;
-	struct pipe_ctx *head_pipe = resource_get_otg_master_for_stream(res_ctx, opp_head_pipe->stream);
-	struct pipe_ctx *idle_pipe = resource_find_free_secondary_pipe_legacy(res_ctx, pool, head_pipe);
-
-	if (!head_pipe)
-		ASSERT(0);
-
-	if (!idle_pipe)
-		return NULL;
-
-	idle_pipe->stream = head_pipe->stream;
-	idle_pipe->stream_res.tg = head_pipe->stream_res.tg;
-	idle_pipe->stream_res.opp = head_pipe->stream_res.opp;
-
-	idle_pipe->plane_res.hubp = pool->hubps[idle_pipe->pipe_idx];
-	idle_pipe->plane_res.ipp = pool->ipps[idle_pipe->pipe_idx];
-	idle_pipe->plane_res.dpp = pool->dpps[idle_pipe->pipe_idx];
-	idle_pipe->plane_res.mpcc_inst = pool->dpps[idle_pipe->pipe_idx]->inst;
-
-	return idle_pipe;
-}
-
-static bool dcn201_get_dcc_compression_cap(const struct dc *dc,
-		const struct dc_dcc_surface_param *input,
-		struct dc_surface_dcc_cap *output)
-{
-	return dc->res_pool->hubbub->funcs->get_dcc_compression_cap(
-			dc->res_pool->hubbub,
-			input,
-			output);
-}
-
-static void dcn201_populate_dml_writeback_from_context(struct dc *dc,
-						       struct resource_context *res_ctx,
-						       display_e2e_pipe_params_st *pipes)
-{
-	DC_FP_START();
-	dcn201_populate_dml_writeback_from_context_fpu(dc, res_ctx, pipes);
-	DC_FP_END();
-}
-
-static void dcn201_destroy_resource_pool(struct resource_pool **pool)
-{
-	struct dcn201_resource_pool *dcn201_pool = TO_DCN201_RES_POOL(*pool);
-
-	dcn201_resource_destruct(dcn201_pool);
-	kfree(dcn201_pool);
-	*pool = NULL;
-}
-
-static void dcn201_link_init(struct dc_link *link)
-{
-	if (link->ctx->dc_bios->integrated_info)
-		link->dp_ss_off = !link->ctx->dc_bios->integrated_info->dp_ss_control;
-}
-
-static struct dc_cap_funcs cap_funcs = {
-	.get_dcc_compression_cap = dcn201_get_dcc_compression_cap,
-};
-
-static struct resource_funcs dcn201_res_pool_funcs = {
-	.link_init = dcn201_link_init,
-	.destroy = dcn201_destroy_resource_pool,
-	.link_enc_create = dcn201_link_encoder_create,
-	.panel_cntl_create = NULL,
-	.validate_bandwidth = dcn20_validate_bandwidth,
-	.populate_dml_pipes = dcn20_populate_dml_pipes_from_context,
-	.add_stream_to_ctx = dcn20_add_stream_to_ctx,
-	.add_dsc_to_stream_resource = NULL,
-	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
-	.acquire_free_pipe_as_secondary_dpp_pipe = dcn201_acquire_free_pipe_for_layer,
-	.release_pipe = dcn20_release_pipe,
-	.populate_dml_writeback_from_context = dcn201_populate_dml_writeback_from_context,
-	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
-	.set_mcif_arb_params = dcn20_set_mcif_arb_params,
-	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link
-};
-
-static bool dcn201_resource_construct(
-	uint8_t num_virtual_links,
-	struct dc *dc,
-	struct dcn201_resource_pool *pool)
-{
-	int i;
-	struct dc_context *ctx = dc->ctx;
-
-	ctx->dc_bios->regs = &bios_regs;
-
-	pool->base.res_cap = &res_cap_dnc201;
-	pool->base.funcs = &dcn201_res_pool_funcs;
-
-	/*************************************************
-	 *  Resource + asic cap harcoding                *
-	 *************************************************/
-	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
-
-	pool->base.pipe_count = 4;
-	pool->base.mpcc_count = 5;
-	dc->caps.max_downscale_ratio = 200;
-	dc->caps.i2c_speed_in_khz = 100;
-	dc->caps.i2c_speed_in_khz_hdcp = 5; /*1.5 w/a applied by default*/
-	dc->caps.max_cursor_size = 256;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.dmdata_alloc_size = 2048;
-
-	dc->caps.max_slave_planes = 1;
-	dc->caps.max_slave_yuv_planes = 1;
-	dc->caps.max_slave_rgb_planes = 1;
-	dc->caps.post_blend_color_processing = true;
-	dc->caps.force_dp_tps4_for_cp2520 = true;
-	dc->caps.extended_aux_timeout_support = true;
-
-	/* Color pipeline capabilities */
-	dc->caps.color.dpp.dcn_arch = 1;
-	dc->caps.color.dpp.input_lut_shared = 0;
-	dc->caps.color.dpp.icsc = 1;
-	dc->caps.color.dpp.dgam_ram = 1;
-	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
-	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.dpp.dgam_rom_caps.pq = 0;
-	dc->caps.color.dpp.dgam_rom_caps.hlg = 0;
-	dc->caps.color.dpp.post_csc = 0;
-	dc->caps.color.dpp.gamma_corr = 0;
-	dc->caps.color.dpp.dgam_rom_for_yuv = 1;
-
-	dc->caps.color.dpp.hw_3d_lut = 1;
-	dc->caps.color.dpp.ogam_ram = 1;
-	// no OGAM ROM on DCN2
-	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
-	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
-	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
-	dc->caps.color.dpp.ocsc = 0;
-
-	dc->caps.color.mpc.gamut_remap = 0;
-	dc->caps.color.mpc.num_3dluts = 0;
-	dc->caps.color.mpc.shared_3d_lut = 0;
-	dc->caps.color.mpc.ogam_ram = 1;
-	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
-	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
-	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
-	dc->caps.color.mpc.ocsc = 1;
-
-	dc->debug = debug_defaults_drv;
-
-	/*a0 only, remove later*/
-	dc->work_arounds.no_connect_phy_config  = true;
-	dc->work_arounds.dedcn20_305_wa = true;
-	/*************************************************
-	 *  Create resources                             *
-	 *************************************************/
-
-	pool->base.clock_sources[DCN20_CLK_SRC_PLL0] =
-			dcn201_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL0,
-				&clk_src_regs[0], false);
-	pool->base.clock_sources[DCN20_CLK_SRC_PLL1] =
-			dcn201_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL1,
-				&clk_src_regs[1], false);
-
-	pool->base.clk_src_count = DCN20_CLK_SRC_TOTAL_DCN201;
-
-	/* todo: not reuse phy_pll registers */
-	pool->base.dp_clock_source =
-			dcn201_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_ID_DP_DTO,
-				&clk_src_regs[0], true);
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			goto create_fail;
-		}
-	}
-
-	pool->base.dccg = dccg201_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
-	if (pool->base.dccg == NULL) {
-		dm_error("DC: failed to create dccg!\n");
-		goto create_fail;
-	}
-
-	dcn201_ip.max_num_otg = pool->base.res_cap->num_timing_generator;
-	dcn201_ip.max_num_dpp = pool->base.pipe_count;
-	dml_init_instance(&dc->dml, &dcn201_soc, &dcn201_ip, DML_PROJECT_DCN201);
-	{
-		struct irq_service_init_data init_data;
-		init_data.ctx = dc->ctx;
-		pool->base.irqs = dal_irq_service_dcn201_create(&init_data);
-		if (!pool->base.irqs)
-			goto create_fail;
-	}
-
-	/* mem input -> ipp -> dpp -> opp -> TG */
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		pool->base.hubps[i] = dcn201_hubp_create(ctx, i);
-		if (pool->base.hubps[i] == NULL) {
-			dm_error(
-				"DC: failed to create memory input!\n");
-			goto create_fail;
-		}
-
-		pool->base.ipps[i] = dcn201_ipp_create(ctx, i);
-		if (pool->base.ipps[i] == NULL) {
-			dm_error(
-				"DC: failed to create input pixel processor!\n");
-			goto create_fail;
-		}
-
-		pool->base.dpps[i] = dcn201_dpp_create(ctx, i);
-		if (pool->base.dpps[i] == NULL) {
-			dm_error(
-				"DC: failed to create dpps!\n");
-			goto create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
-		pool->base.opps[i] = dcn201_opp_create(ctx, i);
-		if (pool->base.opps[i] == NULL) {
-			dm_error(
-				"DC: failed to create output pixel processor!\n");
-			goto create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		pool->base.engines[i] = dcn201_aux_engine_create(ctx, i);
-		if (pool->base.engines[i] == NULL) {
-			dm_error(
-				"DC:failed to create aux engine!!\n");
-			goto create_fail;
-		}
-		pool->base.hw_i2cs[i] = dcn201_i2c_hw_create(ctx, i);
-		if (pool->base.hw_i2cs[i] == NULL) {
-			dm_error(
-				"DC:failed to create hw i2c!!\n");
-			goto create_fail;
-		}
-		pool->base.sw_i2cs[i] = NULL;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		pool->base.timing_generators[i] = dcn201_timing_generator_create(
-				ctx, i);
-		if (pool->base.timing_generators[i] == NULL) {
-			dm_error("DC: failed to create tg!\n");
-			goto create_fail;
-		}
-	}
-
-	pool->base.timing_generator_count = i;
-
-	pool->base.mpc = dcn201_mpc_create(ctx, pool->base.mpcc_count);
-	if (pool->base.mpc == NULL) {
-		dm_error("DC: failed to create mpc!\n");
-		goto create_fail;
-	}
-
-	pool->base.hubbub = dcn201_hubbub_create(ctx);
-	if (pool->base.hubbub == NULL) {
-		dm_error("DC: failed to create hubbub!\n");
-		goto create_fail;
-	}
-
-	if (!resource_construct(num_virtual_links, dc, &pool->base,
-			&res_create_funcs))
-		goto create_fail;
-
-	dcn201_hw_sequencer_construct(dc);
-
-	dc->caps.max_planes =  pool->base.pipe_count;
-
-	for (i = 0; i < dc->caps.max_planes; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	dc->cap_funcs = cap_funcs;
-
-	return true;
-
-create_fail:
-
-	dcn201_resource_destruct(pool);
-
-	return false;
-}
-
-struct resource_pool *dcn201_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc)
-{
-	struct dcn201_resource_pool *pool =
-		kzalloc(sizeof(struct dcn201_resource_pool), GFP_ATOMIC);
-
-	if (!pool)
-		return NULL;
-
-	if (dcn201_resource_construct(init_data->num_virtual_links, dc, pool))
-		return &pool->base;
-
-	kfree(pool);
-	return NULL;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_resource.h b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_resource.h
deleted file mode 100644
index e0467d17d4ae..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_resource.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
-* Copyright 2017 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#ifndef __DC_RESOURCE_DCN201_H__
-#define __DC_RESOURCE_DCN201_H__
-
-#include "core_types.h"
-
-#define RRDPCS_PHY_DP_TX_PSTATE_POWER_UP    0x00000000
-#define RRDPCS_PHY_DP_TX_PSTATE_HOLD        0x00000001
-#define RRDPCS_PHY_DP_TX_PSTATE_HOLD_OFF    0x00000002
-#define RRDPCS_PHY_DP_TX_PSTATE_POWER_DOWN  0x00000003
-
-#define TO_DCN201_RES_POOL(pool)\
-	container_of(pool, struct dcn201_resource_pool, base)
-
-struct dc;
-struct resource_pool;
-struct _vcs_dpi_display_pipe_params_st;
-
-struct dcn201_resource_pool {
-	struct resource_pool base;
-};
-struct resource_pool *dcn201_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc);
-
-#endif /* __DC_RESOURCE_DCN201_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/Makefile b/drivers/gpu/drm/amd/display/dc/dcn21/Makefile
index ce1be0afae4a..dd1eea7212f4 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn21/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn21/Makefile
@@ -2,7 +2,7 @@
 #
 # Makefile for DCN21.
 
-DCN21 = dcn21_init.o dcn21_hubp.o dcn21_hubbub.o dcn21_resource.o \
+DCN21 = dcn21_init.o dcn21_hubp.o dcn21_hubbub.o \
 	 dcn21_link_encoder.o dcn21_dccg.o
 
 AMD_DAL_DCN21 = $(addprefix $(AMDDALPATH)/dc/dcn21/,$(DCN21))
diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c
deleted file mode 100644
index 42277b280586..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c
+++ /dev/null
@@ -1,1745 +0,0 @@
-/*
-* Copyright 2018 Advanced Micro Devices, Inc.
- * Copyright 2019 Raptor Engineering, LLC
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#include <linux/slab.h>
-
-#include "dm_services.h"
-#include "dc.h"
-
-#include "dcn21_init.h"
-
-#include "resource.h"
-#include "include/irq_service_interface.h"
-#include "dcn20/dcn20_resource.h"
-#include "dcn21/dcn21_resource.h"
-
-#include "dml/dcn20/dcn20_fpu.h"
-
-#include "clk_mgr.h"
-#include "dcn10/dcn10_hubp.h"
-#include "dcn10/dcn10_ipp.h"
-#include "dcn20/dcn20_hubbub.h"
-#include "dcn20/dcn20_mpc.h"
-#include "dcn20/dcn20_hubp.h"
-#include "dcn21_hubp.h"
-#include "irq/dcn21/irq_service_dcn21.h"
-#include "dcn20/dcn20_dpp.h"
-#include "dcn20/dcn20_optc.h"
-#include "dcn21/dcn21_hwseq.h"
-#include "dce110/dce110_hwseq.h"
-#include "dcn20/dcn20_opp.h"
-#include "dcn20/dcn20_dsc.h"
-#include "dcn21/dcn21_link_encoder.h"
-#include "dcn20/dcn20_stream_encoder.h"
-#include "dce/dce_clock_source.h"
-#include "dce/dce_audio.h"
-#include "dce/dce_hwseq.h"
-#include "virtual/virtual_stream_encoder.h"
-#include "dml/display_mode_vba.h"
-#include "dcn20/dcn20_dccg.h"
-#include "dcn21/dcn21_dccg.h"
-#include "dcn21_hubbub.h"
-#include "dcn10/dcn10_resource.h"
-#include "dce/dce_panel_cntl.h"
-
-#include "dcn20/dcn20_dwb.h"
-#include "dcn20/dcn20_mmhubbub.h"
-#include "dpcs/dpcs_2_1_0_offset.h"
-#include "dpcs/dpcs_2_1_0_sh_mask.h"
-
-#include "renoir_ip_offset.h"
-#include "dcn/dcn_2_1_0_offset.h"
-#include "dcn/dcn_2_1_0_sh_mask.h"
-
-#include "nbio/nbio_7_0_offset.h"
-
-#include "mmhub/mmhub_2_0_0_offset.h"
-#include "mmhub/mmhub_2_0_0_sh_mask.h"
-
-#include "reg_helper.h"
-#include "dce/dce_abm.h"
-#include "dce/dce_dmcu.h"
-#include "dce/dce_aux.h"
-#include "dce/dce_i2c.h"
-#include "dcn21_resource.h"
-#include "vm_helper.h"
-#include "dcn20/dcn20_vmid.h"
-#include "dce/dmub_psr.h"
-#include "dce/dmub_abm.h"
-
-/* begin *********************
- * macros to expend register list macro defined in HW object header file */
-
-/* DCN */
-#define BASE_INNER(seg) DMU_BASE__INST0_SEG ## seg
-
-#define BASE(seg) BASE_INNER(seg)
-
-#define SR(reg_name)\
-		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) +  \
-					mm ## reg_name
-
-#define SRI(reg_name, block, id)\
-	.reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define SRIR(var_name, reg_name, block, id)\
-	.var_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define DCCG_SRII(reg_name, block, id)\
-	.block ## _ ## reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define VUPDATE_SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(mm ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
-					mm ## reg_name ## _ ## block ## id
-
-/* NBIO */
-#define NBIO_BASE_INNER(seg) \
-	NBIF0_BASE__INST0_SEG ## seg
-
-#define NBIO_BASE(seg) \
-	NBIO_BASE_INNER(seg)
-
-#define NBIO_SR(reg_name)\
-		.reg_name = NBIO_BASE(mm ## reg_name ## _BASE_IDX) + \
-					mm ## reg_name
-
-/* MMHUB */
-#define MMHUB_BASE_INNER(seg) \
-	MMHUB_BASE__INST0_SEG ## seg
-
-#define MMHUB_BASE(seg) \
-	MMHUB_BASE_INNER(seg)
-
-#define MMHUB_SR(reg_name)\
-		.reg_name = MMHUB_BASE(mmMM ## reg_name ## _BASE_IDX) + \
-					mmMM ## reg_name
-
-#define clk_src_regs(index, pllid)\
-[index] = {\
-	CS_COMMON_REG_LIST_DCN2_1(index, pllid),\
-}
-
-static const struct dce110_clk_src_regs clk_src_regs[] = {
-	clk_src_regs(0, A),
-	clk_src_regs(1, B),
-	clk_src_regs(2, C),
-	clk_src_regs(3, D),
-	clk_src_regs(4, E),
-};
-
-static const struct dce110_clk_src_shift cs_shift = {
-		CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
-};
-
-static const struct dce110_clk_src_mask cs_mask = {
-		CS_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
-};
-
-static const struct bios_registers bios_regs = {
-		NBIO_SR(BIOS_SCRATCH_3),
-		NBIO_SR(BIOS_SCRATCH_6)
-};
-
-static const struct dce_dmcu_registers dmcu_regs = {
-		DMCU_DCN20_REG_LIST()
-};
-
-static const struct dce_dmcu_shift dmcu_shift = {
-		DMCU_MASK_SH_LIST_DCN10(__SHIFT)
-};
-
-static const struct dce_dmcu_mask dmcu_mask = {
-		DMCU_MASK_SH_LIST_DCN10(_MASK)
-};
-
-static const struct dce_abm_registers abm_regs = {
-		ABM_DCN20_REG_LIST()
-};
-
-static const struct dce_abm_shift abm_shift = {
-		ABM_MASK_SH_LIST_DCN20(__SHIFT)
-};
-
-static const struct dce_abm_mask abm_mask = {
-		ABM_MASK_SH_LIST_DCN20(_MASK)
-};
-
-#define audio_regs(id)\
-[id] = {\
-		AUD_COMMON_REG_LIST(id)\
-}
-
-static const struct dce_audio_registers audio_regs[] = {
-	audio_regs(0),
-	audio_regs(1),
-	audio_regs(2),
-	audio_regs(3),
-	audio_regs(4),
-	audio_regs(5),
-};
-
-#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
-		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
-
-static const struct dce_audio_shift audio_shift = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_audio_mask audio_mask = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
-};
-
-static const struct dccg_registers dccg_regs = {
-		DCCG_COMMON_REG_LIST_DCN_BASE()
-};
-
-static const struct dccg_shift dccg_shift = {
-		DCCG_MASK_SH_LIST_DCN2_1(__SHIFT)
-};
-
-static const struct dccg_mask dccg_mask = {
-		DCCG_MASK_SH_LIST_DCN2_1(_MASK)
-};
-
-#define opp_regs(id)\
-[id] = {\
-	OPP_REG_LIST_DCN20(id),\
-}
-
-static const struct dcn20_opp_registers opp_regs[] = {
-	opp_regs(0),
-	opp_regs(1),
-	opp_regs(2),
-	opp_regs(3),
-	opp_regs(4),
-	opp_regs(5),
-};
-
-static const struct dcn20_opp_shift opp_shift = {
-		OPP_MASK_SH_LIST_DCN20(__SHIFT)
-};
-
-static const struct dcn20_opp_mask opp_mask = {
-		OPP_MASK_SH_LIST_DCN20(_MASK)
-};
-
-#define tg_regs(id)\
-[id] = {TG_COMMON_REG_LIST_DCN2_0(id)}
-
-static const struct dcn_optc_registers tg_regs[] = {
-	tg_regs(0),
-	tg_regs(1),
-	tg_regs(2),
-	tg_regs(3)
-};
-
-static const struct dcn_optc_shift tg_shift = {
-	TG_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
-};
-
-static const struct dcn_optc_mask tg_mask = {
-	TG_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
-};
-
-static const struct dcn20_mpc_registers mpc_regs = {
-		MPC_REG_LIST_DCN2_0(0),
-		MPC_REG_LIST_DCN2_0(1),
-		MPC_REG_LIST_DCN2_0(2),
-		MPC_REG_LIST_DCN2_0(3),
-		MPC_REG_LIST_DCN2_0(4),
-		MPC_REG_LIST_DCN2_0(5),
-		MPC_OUT_MUX_REG_LIST_DCN2_0(0),
-		MPC_OUT_MUX_REG_LIST_DCN2_0(1),
-		MPC_OUT_MUX_REG_LIST_DCN2_0(2),
-		MPC_OUT_MUX_REG_LIST_DCN2_0(3),
-		MPC_DBG_REG_LIST_DCN2_0()
-};
-
-static const struct dcn20_mpc_shift mpc_shift = {
-	MPC_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT),
-	MPC_DEBUG_REG_LIST_SH_DCN20
-};
-
-static const struct dcn20_mpc_mask mpc_mask = {
-	MPC_COMMON_MASK_SH_LIST_DCN2_0(_MASK),
-	MPC_DEBUG_REG_LIST_MASK_DCN20
-};
-
-#define hubp_regs(id)\
-[id] = {\
-	HUBP_REG_LIST_DCN21(id)\
-}
-
-static const struct dcn_hubp2_registers hubp_regs[] = {
-		hubp_regs(0),
-		hubp_regs(1),
-		hubp_regs(2),
-		hubp_regs(3)
-};
-
-static const struct dcn_hubp2_shift hubp_shift = {
-		HUBP_MASK_SH_LIST_DCN21(__SHIFT)
-};
-
-static const struct dcn_hubp2_mask hubp_mask = {
-		HUBP_MASK_SH_LIST_DCN21(_MASK)
-};
-
-static const struct dcn_hubbub_registers hubbub_reg = {
-		HUBBUB_REG_LIST_DCN21()
-};
-
-static const struct dcn_hubbub_shift hubbub_shift = {
-		HUBBUB_MASK_SH_LIST_DCN21(__SHIFT)
-};
-
-static const struct dcn_hubbub_mask hubbub_mask = {
-		HUBBUB_MASK_SH_LIST_DCN21(_MASK)
-};
-
-
-#define vmid_regs(id)\
-[id] = {\
-		DCN20_VMID_REG_LIST(id)\
-}
-
-static const struct dcn_vmid_registers vmid_regs[] = {
-	vmid_regs(0),
-	vmid_regs(1),
-	vmid_regs(2),
-	vmid_regs(3),
-	vmid_regs(4),
-	vmid_regs(5),
-	vmid_regs(6),
-	vmid_regs(7),
-	vmid_regs(8),
-	vmid_regs(9),
-	vmid_regs(10),
-	vmid_regs(11),
-	vmid_regs(12),
-	vmid_regs(13),
-	vmid_regs(14),
-	vmid_regs(15)
-};
-
-static const struct dcn20_vmid_shift vmid_shifts = {
-		DCN20_VMID_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn20_vmid_mask vmid_masks = {
-		DCN20_VMID_MASK_SH_LIST(_MASK)
-};
-
-#define dsc_regsDCN20(id)\
-[id] = {\
-	DSC_REG_LIST_DCN20(id)\
-}
-
-static const struct dcn20_dsc_registers dsc_regs[] = {
-	dsc_regsDCN20(0),
-	dsc_regsDCN20(1),
-	dsc_regsDCN20(2),
-	dsc_regsDCN20(3),
-	dsc_regsDCN20(4),
-	dsc_regsDCN20(5)
-};
-
-static const struct dcn20_dsc_shift dsc_shift = {
-	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
-};
-
-static const struct dcn20_dsc_mask dsc_mask = {
-	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
-};
-
-#define ipp_regs(id)\
-[id] = {\
-	IPP_REG_LIST_DCN20(id),\
-}
-
-static const struct dcn10_ipp_registers ipp_regs[] = {
-	ipp_regs(0),
-	ipp_regs(1),
-	ipp_regs(2),
-	ipp_regs(3),
-};
-
-static const struct dcn10_ipp_shift ipp_shift = {
-		IPP_MASK_SH_LIST_DCN20(__SHIFT)
-};
-
-static const struct dcn10_ipp_mask ipp_mask = {
-		IPP_MASK_SH_LIST_DCN20(_MASK),
-};
-
-#define opp_regs(id)\
-[id] = {\
-	OPP_REG_LIST_DCN20(id),\
-}
-
-
-#define aux_engine_regs(id)\
-[id] = {\
-	AUX_COMMON_REG_LIST0(id), \
-	.AUXN_IMPCAL = 0, \
-	.AUXP_IMPCAL = 0, \
-	.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
-}
-
-static const struct dce110_aux_registers aux_engine_regs[] = {
-		aux_engine_regs(0),
-		aux_engine_regs(1),
-		aux_engine_regs(2),
-		aux_engine_regs(3),
-		aux_engine_regs(4),
-};
-
-#define tf_regs(id)\
-[id] = {\
-	TF_REG_LIST_DCN20(id),\
-	TF_REG_LIST_DCN20_COMMON_APPEND(id),\
-}
-
-static const struct dcn2_dpp_registers tf_regs[] = {
-	tf_regs(0),
-	tf_regs(1),
-	tf_regs(2),
-	tf_regs(3),
-};
-
-static const struct dcn2_dpp_shift tf_shift = {
-		TF_REG_LIST_SH_MASK_DCN20(__SHIFT),
-		TF_DEBUG_REG_LIST_SH_DCN20
-};
-
-static const struct dcn2_dpp_mask tf_mask = {
-		TF_REG_LIST_SH_MASK_DCN20(_MASK),
-		TF_DEBUG_REG_LIST_MASK_DCN20
-};
-
-#define stream_enc_regs(id)\
-[id] = {\
-	SE_DCN2_REG_LIST(id)\
-}
-
-static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
-	stream_enc_regs(0),
-	stream_enc_regs(1),
-	stream_enc_regs(2),
-	stream_enc_regs(3),
-	stream_enc_regs(4),
-};
-
-static const struct dce110_aux_registers_shift aux_shift = {
-	DCN_AUX_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce110_aux_registers_mask aux_mask = {
-	DCN_AUX_MASK_SH_LIST(_MASK)
-};
-
-static const struct dcn10_stream_encoder_shift se_shift = {
-		SE_COMMON_MASK_SH_LIST_DCN20(__SHIFT)
-};
-
-static const struct dcn10_stream_encoder_mask se_mask = {
-		SE_COMMON_MASK_SH_LIST_DCN20(_MASK)
-};
-
-static void dcn21_pp_smu_destroy(struct pp_smu_funcs **pp_smu);
-
-static struct input_pixel_processor *dcn21_ipp_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn10_ipp *ipp =
-		kzalloc(sizeof(struct dcn10_ipp), GFP_KERNEL);
-
-	if (!ipp) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dcn20_ipp_construct(ipp, ctx, inst,
-			&ipp_regs[inst], &ipp_shift, &ipp_mask);
-	return &ipp->base;
-}
-
-static struct dpp *dcn21_dpp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn20_dpp *dpp =
-		kzalloc(sizeof(struct dcn20_dpp), GFP_KERNEL);
-
-	if (!dpp)
-		return NULL;
-
-	if (dpp2_construct(dpp, ctx, inst,
-			&tf_regs[inst], &tf_shift, &tf_mask))
-		return &dpp->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(dpp);
-	return NULL;
-}
-
-static struct dce_aux *dcn21_aux_engine_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct aux_engine_dce110 *aux_engine =
-		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
-
-	if (!aux_engine)
-		return NULL;
-
-	dce110_aux_engine_construct(aux_engine, ctx, inst,
-				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
-				    &aux_engine_regs[inst],
-					&aux_mask,
-					&aux_shift,
-					ctx->dc->caps.extended_aux_timeout_support);
-
-	return &aux_engine->base;
-}
-
-#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
-
-static const struct dce_i2c_registers i2c_hw_regs[] = {
-		i2c_inst_regs(1),
-		i2c_inst_regs(2),
-		i2c_inst_regs(3),
-		i2c_inst_regs(4),
-		i2c_inst_regs(5),
-};
-
-static const struct dce_i2c_shift i2c_shifts = {
-		I2C_COMMON_MASK_SH_LIST_DCN2(__SHIFT)
-};
-
-static const struct dce_i2c_mask i2c_masks = {
-		I2C_COMMON_MASK_SH_LIST_DCN2(_MASK)
-};
-
-static struct dce_i2c_hw *dcn21_i2c_hw_create(struct dc_context *ctx,
-					      uint32_t inst)
-{
-	struct dce_i2c_hw *dce_i2c_hw =
-		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
-
-	if (!dce_i2c_hw)
-		return NULL;
-
-	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
-				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
-
-	return dce_i2c_hw;
-}
-
-static const struct resource_caps res_cap_rn = {
-		.num_timing_generator = 4,
-		.num_opp = 4,
-		.num_video_plane = 4,
-		.num_audio = 4, // 4 audio endpoints.  4 audio streams
-		.num_stream_encoder = 5,
-		.num_pll = 5,  // maybe 3 because the last two used for USB-c
-		.num_dwb = 1,
-		.num_ddc = 5,
-		.num_vmid = 16,
-		.num_dsc = 3,
-};
-
-#ifdef DIAGS_BUILD
-static const struct resource_caps res_cap_rn_FPGA_4pipe = {
-		.num_timing_generator = 4,
-		.num_opp = 4,
-		.num_video_plane = 4,
-		.num_audio = 7,
-		.num_stream_encoder = 4,
-		.num_pll = 4,
-		.num_dwb = 1,
-		.num_ddc = 4,
-		.num_dsc = 0,
-};
-
-static const struct resource_caps res_cap_rn_FPGA_2pipe_dsc = {
-		.num_timing_generator = 2,
-		.num_opp = 2,
-		.num_video_plane = 2,
-		.num_audio = 7,
-		.num_stream_encoder = 2,
-		.num_pll = 4,
-		.num_dwb = 1,
-		.num_ddc = 4,
-		.num_dsc = 2,
-};
-#endif
-
-static const struct dc_plane_cap plane_cap = {
-	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.per_pixel_alpha = true,
-
-	.pixel_format_support = {
-			.argb8888 = true,
-			.nv12 = true,
-			.fp16 = true,
-			.p010 = true
-	},
-
-	.max_upscale_factor = {
-			.argb8888 = 16000,
-			.nv12 = 16000,
-			.fp16 = 16000
-	},
-
-	.max_downscale_factor = {
-			.argb8888 = 250,
-			.nv12 = 250,
-			.fp16 = 250
-	},
-	64,
-	64
-};
-
-static const struct dc_debug_options debug_defaults_drv = {
-		.disable_dmcu = false,
-		.force_abm_enable = false,
-		.timing_trace = false,
-		.clock_trace = true,
-		.disable_pplib_clock_request = true,
-		.min_disp_clk_khz = 100000,
-		.pipe_split_policy = MPC_SPLIT_DYNAMIC,
-		.force_single_disp_pipe_split = false,
-		.disable_dcc = DCC_ENABLE,
-		.vsr_support = true,
-		.performance_trace = false,
-		.max_downscale_src_width = 4096,
-		.disable_pplib_wm_range = false,
-		.scl_reset_length10 = true,
-		.sanity_checks = true,
-		.disable_48mhz_pwrdwn = false,
-		.usbc_combo_phy_reset_wa = true,
-		.dmub_command_table = true,
-		.use_max_lb = true,
-		.enable_legacy_fast_update = true,
-		.using_dml2 = false,
-};
-
-static const struct dc_panel_config panel_config_defaults = {
-		.psr = {
-			.disable_psr = false,
-			.disallow_psrsu = false,
-			.disallow_replay = false,
-		},
-		.ilr = {
-			.optimize_edp_link_rate = true,
-		},
-};
-
-enum dcn20_clk_src_array_id {
-	DCN20_CLK_SRC_PLL0,
-	DCN20_CLK_SRC_PLL1,
-	DCN20_CLK_SRC_PLL2,
-	DCN20_CLK_SRC_PLL3,
-	DCN20_CLK_SRC_PLL4,
-	DCN20_CLK_SRC_TOTAL_DCN21
-};
-
-static void dcn21_resource_destruct(struct dcn21_resource_pool *pool)
-{
-	unsigned int i;
-
-	for (i = 0; i < pool->base.stream_enc_count; i++) {
-		if (pool->base.stream_enc[i] != NULL) {
-			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
-			pool->base.stream_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		if (pool->base.dscs[i] != NULL)
-			dcn20_dsc_destroy(&pool->base.dscs[i]);
-	}
-
-	if (pool->base.mpc != NULL) {
-		kfree(TO_DCN20_MPC(pool->base.mpc));
-		pool->base.mpc = NULL;
-	}
-	if (pool->base.hubbub != NULL) {
-		kfree(pool->base.hubbub);
-		pool->base.hubbub = NULL;
-	}
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		if (pool->base.dpps[i] != NULL)
-			dcn20_dpp_destroy(&pool->base.dpps[i]);
-
-		if (pool->base.ipps[i] != NULL)
-			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
-
-		if (pool->base.hubps[i] != NULL) {
-			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
-			pool->base.hubps[i] = NULL;
-		}
-
-		if (pool->base.irqs != NULL) {
-			dal_irq_service_destroy(&pool->base.irqs);
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		if (pool->base.engines[i] != NULL)
-			dce110_engine_destroy(&pool->base.engines[i]);
-		if (pool->base.hw_i2cs[i] != NULL) {
-			kfree(pool->base.hw_i2cs[i]);
-			pool->base.hw_i2cs[i] = NULL;
-		}
-		if (pool->base.sw_i2cs[i] != NULL) {
-			kfree(pool->base.sw_i2cs[i]);
-			pool->base.sw_i2cs[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
-		if (pool->base.opps[i] != NULL)
-			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		if (pool->base.timing_generators[i] != NULL)	{
-			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
-			pool->base.timing_generators[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
-		if (pool->base.dwbc[i] != NULL) {
-			kfree(TO_DCN20_DWBC(pool->base.dwbc[i]));
-			pool->base.dwbc[i] = NULL;
-		}
-		if (pool->base.mcif_wb[i] != NULL) {
-			kfree(TO_DCN20_MMHUBBUB(pool->base.mcif_wb[i]));
-			pool->base.mcif_wb[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.audio_count; i++) {
-		if (pool->base.audios[i])
-			dce_aud_destroy(&pool->base.audios[i]);
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] != NULL) {
-			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
-			pool->base.clock_sources[i] = NULL;
-		}
-	}
-
-	if (pool->base.dp_clock_source != NULL) {
-		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
-		pool->base.dp_clock_source = NULL;
-	}
-
-	if (pool->base.abm != NULL) {
-		if (pool->base.abm->ctx->dc->config.disable_dmcu)
-			dmub_abm_destroy(&pool->base.abm);
-		else
-			dce_abm_destroy(&pool->base.abm);
-	}
-
-	if (pool->base.dmcu != NULL)
-		dce_dmcu_destroy(&pool->base.dmcu);
-
-	if (pool->base.psr != NULL)
-		dmub_psr_destroy(&pool->base.psr);
-
-	if (pool->base.dccg != NULL)
-		dcn_dccg_destroy(&pool->base.dccg);
-
-	if (pool->base.pp_smu != NULL)
-		dcn21_pp_smu_destroy(&pool->base.pp_smu);
-}
-
-bool dcn21_fast_validate_bw(struct dc *dc,
-			    struct dc_state *context,
-			    display_e2e_pipe_params_st *pipes,
-			    int *pipe_cnt_out,
-			    int *pipe_split_from,
-			    int *vlevel_out,
-			    bool fast_validate)
-{
-	bool out = false;
-	int split[MAX_PIPES] = { 0 };
-	int pipe_cnt, i, pipe_idx, vlevel;
-
-	ASSERT(pipes);
-	if (!pipes)
-		return false;
-
-	dcn20_merge_pipes_for_validate(dc, context);
-
-	DC_FP_START();
-	pipe_cnt = dc->res_pool->funcs->populate_dml_pipes(dc, context, pipes, fast_validate);
-	DC_FP_END();
-
-	*pipe_cnt_out = pipe_cnt;
-
-	if (!pipe_cnt) {
-		out = true;
-		goto validate_out;
-	}
-	/*
-	 * DML favors voltage over p-state, but we're more interested in
-	 * supporting p-state over voltage. We can't support p-state in
-	 * prefetch mode > 0 so try capping the prefetch mode to start.
-	 */
-	context->bw_ctx.dml.soc.allow_dram_self_refresh_or_dram_clock_change_in_vblank =
-				dm_allow_self_refresh_and_mclk_switch;
-	vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
-
-	if (vlevel > context->bw_ctx.dml.soc.num_states) {
-		/*
-		 * If mode is unsupported or there's still no p-state support then
-		 * fall back to favoring voltage.
-		 *
-		 * We don't actually support prefetch mode 2, so require that we
-		 * at least support prefetch mode 1.
-		 */
-		context->bw_ctx.dml.soc.allow_dram_self_refresh_or_dram_clock_change_in_vblank =
-					dm_allow_self_refresh;
-		vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
-		if (vlevel > context->bw_ctx.dml.soc.num_states)
-			goto validate_fail;
-	}
-
-	vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, vlevel, split, NULL);
-
-	for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-		struct pipe_ctx *mpo_pipe = pipe->bottom_pipe;
-		struct vba_vars_st *vba = &context->bw_ctx.dml.vba;
-
-		if (!pipe->stream)
-			continue;
-
-		/* We only support full screen mpo with ODM */
-		if (vba->ODMCombineEnabled[vba->pipe_plane[pipe_idx]] != dm_odm_combine_mode_disabled
-				&& pipe->plane_state && mpo_pipe
-				&& memcmp(&mpo_pipe->plane_state->clip_rect,
-						&pipe->stream->src,
-						sizeof(struct rect)) != 0) {
-			ASSERT(mpo_pipe->plane_state != pipe->plane_state);
-			goto validate_fail;
-		}
-		pipe_idx++;
-	}
-
-	/*initialize pipe_just_split_from to invalid idx*/
-	for (i = 0; i < MAX_PIPES; i++)
-		pipe_split_from[i] = -1;
-
-	for (i = 0, pipe_idx = -1; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-		struct pipe_ctx *hsplit_pipe = pipe->bottom_pipe;
-
-		if (!pipe->stream || pipe_split_from[i] >= 0)
-			continue;
-
-		pipe_idx++;
-
-		if (!pipe->top_pipe && !pipe->plane_state && context->bw_ctx.dml.vba.ODMCombineEnabled[pipe_idx]) {
-			hsplit_pipe = dcn20_find_secondary_pipe(dc, &context->res_ctx, dc->res_pool, pipe);
-			ASSERT(hsplit_pipe);
-			if (!dcn20_split_stream_for_odm(
-					dc, &context->res_ctx,
-					pipe, hsplit_pipe))
-				goto validate_fail;
-			pipe_split_from[hsplit_pipe->pipe_idx] = pipe_idx;
-			dcn20_build_mapped_resource(dc, context, pipe->stream);
-		}
-
-		if (!pipe->plane_state)
-			continue;
-		/* Skip 2nd half of already split pipe */
-		if (pipe->top_pipe && pipe->plane_state == pipe->top_pipe->plane_state)
-			continue;
-
-		if (split[i] == 2) {
-			if (!hsplit_pipe || hsplit_pipe->plane_state != pipe->plane_state) {
-				/* pipe not split previously needs split */
-				hsplit_pipe = dcn20_find_secondary_pipe(dc, &context->res_ctx, dc->res_pool, pipe);
-				ASSERT(hsplit_pipe);
-				if (!hsplit_pipe) {
-					DC_FP_START();
-					dcn20_fpu_adjust_dppclk(&context->bw_ctx.dml.vba, vlevel, context->bw_ctx.dml.vba.maxMpcComb, pipe_idx, true);
-					DC_FP_END();
-					continue;
-				}
-				if (context->bw_ctx.dml.vba.ODMCombineEnabled[pipe_idx]) {
-					if (!dcn20_split_stream_for_odm(
-							dc, &context->res_ctx,
-							pipe, hsplit_pipe))
-						goto validate_fail;
-					dcn20_build_mapped_resource(dc, context, pipe->stream);
-				} else {
-					dcn20_split_stream_for_mpc(
-							&context->res_ctx, dc->res_pool,
-							pipe, hsplit_pipe);
-					resource_build_scaling_params(pipe);
-					resource_build_scaling_params(hsplit_pipe);
-				}
-				pipe_split_from[hsplit_pipe->pipe_idx] = pipe_idx;
-			}
-		} else if (hsplit_pipe && hsplit_pipe->plane_state == pipe->plane_state) {
-			/* merge should already have been done */
-			ASSERT(0);
-		}
-	}
-	/* Actual dsc count per stream dsc validation*/
-	if (!dcn20_validate_dsc(dc, context)) {
-		context->bw_ctx.dml.vba.ValidationStatus[context->bw_ctx.dml.vba.soc.num_states] =
-				DML_FAIL_DSC_VALIDATION_FAILURE;
-		goto validate_fail;
-	}
-
-	*vlevel_out = vlevel;
-
-	out = true;
-	goto validate_out;
-
-validate_fail:
-	out = false;
-
-validate_out:
-	return out;
-}
-
-/*
- * Some of the functions further below use the FPU, so we need to wrap this
- * with DC_FP_START()/DC_FP_END(). Use the same approach as for
- * dcn20_validate_bandwidth in dcn20_resource.c.
- */
-static bool dcn21_validate_bandwidth(struct dc *dc, struct dc_state *context,
-		bool fast_validate)
-{
-	bool voltage_supported;
-	display_e2e_pipe_params_st *pipes;
-
-	pipes = kcalloc(dc->res_pool->pipe_count, sizeof(display_e2e_pipe_params_st), GFP_KERNEL);
-	if (!pipes)
-		return false;
-
-	DC_FP_START();
-	voltage_supported = dcn21_validate_bandwidth_fp(dc, context, fast_validate, pipes);
-	DC_FP_END();
-
-	kfree(pipes);
-	return voltage_supported;
-}
-
-static void dcn21_destroy_resource_pool(struct resource_pool **pool)
-{
-	struct dcn21_resource_pool *dcn21_pool = TO_DCN21_RES_POOL(*pool);
-
-	dcn21_resource_destruct(dcn21_pool);
-	kfree(dcn21_pool);
-	*pool = NULL;
-}
-
-static struct clock_source *dcn21_clock_source_create(
-		struct dc_context *ctx,
-		struct dc_bios *bios,
-		enum clock_source_id id,
-		const struct dce110_clk_src_regs *regs,
-		bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src =
-		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dcn20_clk_src_construct(clk_src, ctx, bios, id,
-			regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-
-	kfree(clk_src);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
-static struct hubp *dcn21_hubp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn21_hubp *hubp21 =
-		kzalloc(sizeof(struct dcn21_hubp), GFP_KERNEL);
-
-	if (!hubp21)
-		return NULL;
-
-	if (hubp21_construct(hubp21, ctx, inst,
-			&hubp_regs[inst], &hubp_shift, &hubp_mask))
-		return &hubp21->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(hubp21);
-	return NULL;
-}
-
-static struct hubbub *dcn21_hubbub_create(struct dc_context *ctx)
-{
-	int i;
-
-	struct dcn20_hubbub *hubbub = kzalloc(sizeof(struct dcn20_hubbub),
-					  GFP_KERNEL);
-
-	if (!hubbub)
-		return NULL;
-
-	hubbub21_construct(hubbub, ctx,
-			&hubbub_reg,
-			&hubbub_shift,
-			&hubbub_mask);
-
-	for (i = 0; i < res_cap_rn.num_vmid; i++) {
-		struct dcn20_vmid *vmid = &hubbub->vmid[i];
-
-		vmid->ctx = ctx;
-
-		vmid->regs = &vmid_regs[i];
-		vmid->shifts = &vmid_shifts;
-		vmid->masks = &vmid_masks;
-	}
-	hubbub->num_vmid = res_cap_rn.num_vmid;
-
-	return &hubbub->base;
-}
-
-static struct output_pixel_processor *dcn21_opp_create(struct dc_context *ctx,
-						       uint32_t inst)
-{
-	struct dcn20_opp *opp =
-		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
-
-	if (!opp) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dcn20_opp_construct(opp, ctx, inst,
-			&opp_regs[inst], &opp_shift, &opp_mask);
-	return &opp->base;
-}
-
-static struct timing_generator *dcn21_timing_generator_create(struct dc_context *ctx,
-							      uint32_t instance)
-{
-	struct optc *tgn10 =
-		kzalloc(sizeof(struct optc), GFP_KERNEL);
-
-	if (!tgn10)
-		return NULL;
-
-	tgn10->base.inst = instance;
-	tgn10->base.ctx = ctx;
-
-	tgn10->tg_regs = &tg_regs[instance];
-	tgn10->tg_shift = &tg_shift;
-	tgn10->tg_mask = &tg_mask;
-
-	dcn20_timing_generator_init(tgn10);
-
-	return &tgn10->base;
-}
-
-static struct mpc *dcn21_mpc_create(struct dc_context *ctx)
-{
-	struct dcn20_mpc *mpc20 = kzalloc(sizeof(struct dcn20_mpc),
-					  GFP_KERNEL);
-
-	if (!mpc20)
-		return NULL;
-
-	dcn20_mpc_construct(mpc20, ctx,
-			&mpc_regs,
-			&mpc_shift,
-			&mpc_mask,
-			6);
-
-	return &mpc20->base;
-}
-
-static void read_dce_straps(
-	struct dc_context *ctx,
-	struct resource_straps *straps)
-{
-	generic_reg_get(ctx, mmDC_PINSTRAPS + BASE(mmDC_PINSTRAPS_BASE_IDX),
-		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
-
-}
-
-
-static struct display_stream_compressor *dcn21_dsc_create(struct dc_context *ctx,
-							  uint32_t inst)
-{
-	struct dcn20_dsc *dsc =
-		kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
-
-	if (!dsc) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
-	return &dsc->base;
-}
-
-static struct pp_smu_funcs *dcn21_pp_smu_create(struct dc_context *ctx)
-{
-	struct pp_smu_funcs *pp_smu = kzalloc(sizeof(*pp_smu), GFP_KERNEL);
-
-	if (!pp_smu)
-		return pp_smu;
-
-	dm_pp_get_funcs(ctx, pp_smu);
-
-	if (pp_smu->ctx.ver != PP_SMU_VER_RN)
-		pp_smu = memset(pp_smu, 0, sizeof(struct pp_smu_funcs));
-
-
-	return pp_smu;
-}
-
-static void dcn21_pp_smu_destroy(struct pp_smu_funcs **pp_smu)
-{
-	if (pp_smu && *pp_smu) {
-		kfree(*pp_smu);
-		*pp_smu = NULL;
-	}
-}
-
-static struct audio *dcn21_create_audio(
-		struct dc_context *ctx, unsigned int inst)
-{
-	return dce_audio_create(ctx, inst,
-			&audio_regs[inst], &audio_shift, &audio_mask);
-}
-
-static struct dc_cap_funcs cap_funcs = {
-	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
-};
-
-static struct stream_encoder *dcn21_stream_encoder_create(enum engine_id eng_id,
-							  struct dc_context *ctx)
-{
-	struct dcn10_stream_encoder *enc1 =
-		kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
-
-	if (!enc1)
-		return NULL;
-
-	dcn20_stream_encoder_construct(enc1, ctx, ctx->dc_bios, eng_id,
-					&stream_enc_regs[eng_id],
-					&se_shift, &se_mask);
-
-	return &enc1->base;
-}
-
-static const struct dce_hwseq_registers hwseq_reg = {
-		HWSEQ_DCN21_REG_LIST()
-};
-
-static const struct dce_hwseq_shift hwseq_shift = {
-		HWSEQ_DCN21_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_hwseq_mask hwseq_mask = {
-		HWSEQ_DCN21_MASK_SH_LIST(_MASK)
-};
-
-static struct dce_hwseq *dcn21_hwseq_create(
-	struct dc_context *ctx)
-{
-	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
-
-	if (hws) {
-		hws->ctx = ctx;
-		hws->regs = &hwseq_reg;
-		hws->shifts = &hwseq_shift;
-		hws->masks = &hwseq_mask;
-		hws->wa.DEGVIDCN21 = true;
-		hws->wa.disallow_self_refresh_during_multi_plane_transition = true;
-	}
-	return hws;
-}
-
-static const struct resource_create_funcs res_create_funcs = {
-	.read_dce_straps = read_dce_straps,
-	.create_audio = dcn21_create_audio,
-	.create_stream_encoder = dcn21_stream_encoder_create,
-	.create_hwseq = dcn21_hwseq_create,
-};
-
-static const struct encoder_feature_support link_enc_feature = {
-		.max_hdmi_deep_color = COLOR_DEPTH_121212,
-		.max_hdmi_pixel_clock = 600000,
-		.hdmi_ycbcr420_supported = true,
-		.dp_ycbcr420_supported = true,
-		.fec_supported = true,
-		.flags.bits.IS_HBR2_CAPABLE = true,
-		.flags.bits.IS_HBR3_CAPABLE = true,
-		.flags.bits.IS_TPS3_CAPABLE = true,
-		.flags.bits.IS_TPS4_CAPABLE = true
-};
-
-
-#define link_regs(id, phyid)\
-[id] = {\
-	LE_DCN2_REG_LIST(id), \
-	UNIPHY_DCN2_REG_LIST(phyid), \
-	DPCS_DCN21_REG_LIST(id), \
-	SRI(DP_DPHY_INTERNAL_CTRL, DP, id) \
-}
-
-static const struct dcn10_link_enc_registers link_enc_regs[] = {
-	link_regs(0, A),
-	link_regs(1, B),
-	link_regs(2, C),
-	link_regs(3, D),
-	link_regs(4, E),
-};
-
-static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
-	{ DCN_PANEL_CNTL_REG_LIST() }
-};
-
-static const struct dce_panel_cntl_shift panel_cntl_shift = {
-	DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_panel_cntl_mask panel_cntl_mask = {
-	DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
-};
-
-#define aux_regs(id)\
-[id] = {\
-	DCN2_AUX_REG_LIST(id)\
-}
-
-static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
-		aux_regs(0),
-		aux_regs(1),
-		aux_regs(2),
-		aux_regs(3),
-		aux_regs(4)
-};
-
-#define hpd_regs(id)\
-[id] = {\
-	HPD_REG_LIST(id)\
-}
-
-static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
-		hpd_regs(0),
-		hpd_regs(1),
-		hpd_regs(2),
-		hpd_regs(3),
-		hpd_regs(4)
-};
-
-static const struct dcn10_link_enc_shift le_shift = {
-	LINK_ENCODER_MASK_SH_LIST_DCN20(__SHIFT),\
-	DPCS_DCN21_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn10_link_enc_mask le_mask = {
-	LINK_ENCODER_MASK_SH_LIST_DCN20(_MASK),\
-	DPCS_DCN21_MASK_SH_LIST(_MASK)
-};
-
-static int map_transmitter_id_to_phy_instance(
-	enum transmitter transmitter)
-{
-	switch (transmitter) {
-	case TRANSMITTER_UNIPHY_A:
-		return 0;
-	break;
-	case TRANSMITTER_UNIPHY_B:
-		return 1;
-	break;
-	case TRANSMITTER_UNIPHY_C:
-		return 2;
-	break;
-	case TRANSMITTER_UNIPHY_D:
-		return 3;
-	break;
-	case TRANSMITTER_UNIPHY_E:
-		return 4;
-	break;
-	default:
-		ASSERT(0);
-		return 0;
-	}
-}
-
-static struct link_encoder *dcn21_link_encoder_create(
-	struct dc_context *ctx,
-	const struct encoder_init_data *enc_init_data)
-{
-	struct dcn21_link_encoder *enc21 =
-		kzalloc(sizeof(struct dcn21_link_encoder), GFP_KERNEL);
-	int link_regs_id;
-
-	if (!enc21)
-		return NULL;
-
-	link_regs_id =
-		map_transmitter_id_to_phy_instance(enc_init_data->transmitter);
-
-	dcn21_link_encoder_construct(enc21,
-				      enc_init_data,
-				      &link_enc_feature,
-				      &link_enc_regs[link_regs_id],
-				      &link_enc_aux_regs[enc_init_data->channel - 1],
-				      &link_enc_hpd_regs[enc_init_data->hpd_source],
-				      &le_shift,
-				      &le_mask);
-
-	return &enc21->enc10.base;
-}
-
-static struct panel_cntl *dcn21_panel_cntl_create(const struct panel_cntl_init_data *init_data)
-{
-	struct dce_panel_cntl *panel_cntl =
-		kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
-
-	if (!panel_cntl)
-		return NULL;
-
-	dce_panel_cntl_construct(panel_cntl,
-			init_data,
-			&panel_cntl_regs[init_data->inst],
-			&panel_cntl_shift,
-			&panel_cntl_mask);
-
-	return &panel_cntl->base;
-}
-
-static void dcn21_get_panel_config_defaults(struct dc_panel_config *panel_config)
-{
-	*panel_config = panel_config_defaults;
-}
-
-#define CTX ctx
-
-#define REG(reg_name) \
-	(DCN_BASE.instance[0].segment[mm ## reg_name ## _BASE_IDX] + mm ## reg_name)
-
-static uint32_t read_pipe_fuses(struct dc_context *ctx)
-{
-	uint32_t value = REG_READ(CC_DC_PIPE_DIS);
-	/* RV1 support max 4 pipes */
-	value = value & 0xf;
-	return value;
-}
-
-static enum dc_status dcn21_patch_unknown_plane_state(struct dc_plane_state *plane_state)
-{
-	if (plane_state->ctx->dc->debug.disable_dcc == DCC_ENABLE) {
-		plane_state->dcc.enable = 1;
-		/* align to our worst case block width */
-		plane_state->dcc.meta_pitch = ((plane_state->src_rect.width + 1023) / 1024) * 1024;
-	}
-
-	return dcn20_patch_unknown_plane_state(plane_state);
-}
-
-static const struct resource_funcs dcn21_res_pool_funcs = {
-	.destroy = dcn21_destroy_resource_pool,
-	.link_enc_create = dcn21_link_encoder_create,
-	.panel_cntl_create = dcn21_panel_cntl_create,
-	.validate_bandwidth = dcn21_validate_bandwidth,
-	.populate_dml_pipes = dcn21_populate_dml_pipes_from_context,
-	.add_stream_to_ctx = dcn20_add_stream_to_ctx,
-	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
-	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
-	.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
-	.release_pipe = dcn20_release_pipe,
-	.populate_dml_writeback_from_context = dcn20_populate_dml_writeback_from_context,
-	.patch_unknown_plane_state = dcn21_patch_unknown_plane_state,
-	.set_mcif_arb_params = dcn20_set_mcif_arb_params,
-	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
-	.update_bw_bounding_box = dcn21_update_bw_bounding_box,
-	.get_panel_config_defaults = dcn21_get_panel_config_defaults,
-};
-
-static bool dcn21_resource_construct(
-	uint8_t num_virtual_links,
-	struct dc *dc,
-	struct dcn21_resource_pool *pool)
-{
-	int i, j;
-	struct dc_context *ctx = dc->ctx;
-	struct irq_service_init_data init_data;
-	uint32_t pipe_fuses = read_pipe_fuses(ctx);
-	uint32_t num_pipes;
-
-	ctx->dc_bios->regs = &bios_regs;
-
-	pool->base.res_cap = &res_cap_rn;
-#ifdef DIAGS_BUILD
-	if (IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment))
-		//pool->base.res_cap = &res_cap_nv10_FPGA_2pipe_dsc;
-		pool->base.res_cap = &res_cap_rn_FPGA_4pipe;
-#endif
-
-	pool->base.funcs = &dcn21_res_pool_funcs;
-
-	/*************************************************
-	 *  Resource + asic cap harcoding                *
-	 *************************************************/
-	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
-
-	/* max pipe num for ASIC before check pipe fuses */
-	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
-
-	dc->caps.max_downscale_ratio = 200;
-	dc->caps.i2c_speed_in_khz = 100;
-	dc->caps.i2c_speed_in_khz_hdcp = 5; /*1.4 w/a applied by default*/
-	dc->caps.max_cursor_size = 256;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.dmdata_alloc_size = 2048;
-
-	dc->caps.max_slave_planes = 1;
-	dc->caps.max_slave_yuv_planes = 1;
-	dc->caps.max_slave_rgb_planes = 1;
-	dc->caps.post_blend_color_processing = true;
-	dc->caps.force_dp_tps4_for_cp2520 = true;
-	dc->caps.extended_aux_timeout_support = true;
-	dc->caps.dmcub_support = true;
-	dc->caps.is_apu = true;
-
-	/* Color pipeline capabilities */
-	dc->caps.color.dpp.dcn_arch = 1;
-	dc->caps.color.dpp.input_lut_shared = 0;
-	dc->caps.color.dpp.icsc = 1;
-	dc->caps.color.dpp.dgam_ram = 1;
-	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
-	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.dpp.dgam_rom_caps.pq = 0;
-	dc->caps.color.dpp.dgam_rom_caps.hlg = 0;
-	dc->caps.color.dpp.post_csc = 0;
-	dc->caps.color.dpp.gamma_corr = 0;
-	dc->caps.color.dpp.dgam_rom_for_yuv = 1;
-
-	dc->caps.color.dpp.hw_3d_lut = 1;
-	dc->caps.color.dpp.ogam_ram = 1;
-	// no OGAM ROM on DCN2
-	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
-	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
-	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
-	dc->caps.color.dpp.ocsc = 0;
-
-	dc->caps.color.mpc.gamut_remap = 0;
-	dc->caps.color.mpc.num_3dluts = 0;
-	dc->caps.color.mpc.shared_3d_lut = 0;
-	dc->caps.color.mpc.ogam_ram = 1;
-	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
-	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
-	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
-	dc->caps.color.mpc.ocsc = 1;
-
-	dc->caps.dp_hdmi21_pcon_support = true;
-
-	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
-		dc->debug = debug_defaults_drv;
-
-	// Init the vm_helper
-	if (dc->vm_helper)
-		vm_helper_init(dc->vm_helper, 16);
-
-	/*************************************************
-	 *  Create resources                             *
-	 *************************************************/
-
-	pool->base.clock_sources[DCN20_CLK_SRC_PLL0] =
-			dcn21_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL0,
-				&clk_src_regs[0], false);
-	pool->base.clock_sources[DCN20_CLK_SRC_PLL1] =
-			dcn21_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL1,
-				&clk_src_regs[1], false);
-	pool->base.clock_sources[DCN20_CLK_SRC_PLL2] =
-			dcn21_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL2,
-				&clk_src_regs[2], false);
-	pool->base.clock_sources[DCN20_CLK_SRC_PLL3] =
-			dcn21_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL3,
-				&clk_src_regs[3], false);
-	pool->base.clock_sources[DCN20_CLK_SRC_PLL4] =
-			dcn21_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL4,
-				&clk_src_regs[4], false);
-
-	pool->base.clk_src_count = DCN20_CLK_SRC_TOTAL_DCN21;
-
-	/* todo: not reuse phy_pll registers */
-	pool->base.dp_clock_source =
-			dcn21_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_ID_DP_DTO,
-				&clk_src_regs[0], true);
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-
-	pool->base.dccg = dccg21_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
-	if (pool->base.dccg == NULL) {
-		dm_error("DC: failed to create dccg!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	if (!dc->config.disable_dmcu) {
-		pool->base.dmcu = dcn21_dmcu_create(ctx,
-				&dmcu_regs,
-				&dmcu_shift,
-				&dmcu_mask);
-		if (pool->base.dmcu == NULL) {
-			dm_error("DC: failed to create dmcu!\n");
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-
-		dc->debug.dmub_command_table = false;
-	}
-
-	if (dc->config.disable_dmcu) {
-		pool->base.psr = dmub_psr_create(ctx);
-
-		if (pool->base.psr == NULL) {
-			dm_error("DC: failed to create psr obj!\n");
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-
-	if (dc->config.disable_dmcu)
-		pool->base.abm = dmub_abm_create(ctx,
-			&abm_regs,
-			&abm_shift,
-			&abm_mask);
-	else
-		pool->base.abm = dce_abm_create(ctx,
-			&abm_regs,
-			&abm_shift,
-			&abm_mask);
-
-	pool->base.pp_smu = dcn21_pp_smu_create(ctx);
-
-	num_pipes = dcn2_1_ip.max_num_dpp;
-
-	for (i = 0; i < dcn2_1_ip.max_num_dpp; i++)
-		if (pipe_fuses & 1 << i)
-			num_pipes--;
-	dcn2_1_ip.max_num_dpp = num_pipes;
-	dcn2_1_ip.max_num_otg = num_pipes;
-
-	dml_init_instance(&dc->dml, &dcn2_1_soc, &dcn2_1_ip, DML_PROJECT_DCN21);
-
-	init_data.ctx = dc->ctx;
-	pool->base.irqs = dal_irq_service_dcn21_create(&init_data);
-	if (!pool->base.irqs)
-		goto create_fail;
-
-	j = 0;
-	/* mem input -> ipp -> dpp -> opp -> TG */
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		/* if pipe is disabled, skip instance of HW pipe,
-		 * i.e, skip ASIC register instance
-		 */
-		if ((pipe_fuses & (1 << i)) != 0)
-			continue;
-
-		pool->base.hubps[j] = dcn21_hubp_create(ctx, i);
-		if (pool->base.hubps[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create memory input!\n");
-			goto create_fail;
-		}
-
-		pool->base.ipps[j] = dcn21_ipp_create(ctx, i);
-		if (pool->base.ipps[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create input pixel processor!\n");
-			goto create_fail;
-		}
-
-		pool->base.dpps[j] = dcn21_dpp_create(ctx, i);
-		if (pool->base.dpps[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create dpps!\n");
-			goto create_fail;
-		}
-
-		pool->base.opps[j] = dcn21_opp_create(ctx, i);
-		if (pool->base.opps[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create output pixel processor!\n");
-			goto create_fail;
-		}
-
-		pool->base.timing_generators[j] = dcn21_timing_generator_create(
-				ctx, i);
-		if (pool->base.timing_generators[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create tg!\n");
-			goto create_fail;
-		}
-		j++;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		pool->base.engines[i] = dcn21_aux_engine_create(ctx, i);
-		if (pool->base.engines[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create aux engine!!\n");
-			goto create_fail;
-		}
-		pool->base.hw_i2cs[i] = dcn21_i2c_hw_create(ctx, i);
-		if (pool->base.hw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create hw i2c!!\n");
-			goto create_fail;
-		}
-		pool->base.sw_i2cs[i] = NULL;
-	}
-
-	pool->base.timing_generator_count = j;
-	pool->base.pipe_count = j;
-	pool->base.mpcc_count = j;
-
-	pool->base.mpc = dcn21_mpc_create(ctx);
-	if (pool->base.mpc == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mpc!\n");
-		goto create_fail;
-	}
-
-	pool->base.hubbub = dcn21_hubbub_create(ctx);
-	if (pool->base.hubbub == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create hubbub!\n");
-		goto create_fail;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		pool->base.dscs[i] = dcn21_dsc_create(ctx, i);
-		if (pool->base.dscs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create display stream compressor %d!\n", i);
-			goto create_fail;
-		}
-	}
-
-	if (!dcn20_dwbc_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create dwbc!\n");
-		goto create_fail;
-	}
-	if (!dcn20_mmhubbub_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mcif_wb!\n");
-		goto create_fail;
-	}
-
-	if (!resource_construct(num_virtual_links, dc, &pool->base,
-			&res_create_funcs))
-		goto create_fail;
-
-	dcn21_hw_sequencer_construct(dc);
-
-	dc->caps.max_planes =  pool->base.pipe_count;
-
-	for (i = 0; i < dc->caps.max_planes; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	dc->cap_funcs = cap_funcs;
-
-	return true;
-
-create_fail:
-
-	dcn21_resource_destruct(pool);
-
-	return false;
-}
-
-struct resource_pool *dcn21_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc)
-{
-	struct dcn21_resource_pool *pool =
-		kzalloc(sizeof(struct dcn21_resource_pool), GFP_KERNEL);
-
-	if (!pool)
-		return NULL;
-
-	if (dcn21_resource_construct(init_data->num_virtual_links, dc, pool))
-		return &pool->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(pool);
-	return NULL;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.h b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.h
deleted file mode 100644
index f7ecc002c2f7..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.h
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright 2018 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#ifndef _DCN21_RESOURCE_H_
-#define _DCN21_RESOURCE_H_
-
-#include "core_types.h"
-
-#define TO_DCN21_RES_POOL(pool)\
-	container_of(pool, struct dcn21_resource_pool, base)
-
-struct dc;
-struct resource_pool;
-struct _vcs_dpi_display_pipe_params_st;
-
-extern struct _vcs_dpi_ip_params_st dcn2_1_ip;
-extern struct _vcs_dpi_soc_bounding_box_st dcn2_1_soc;
-
-struct dcn21_resource_pool {
-	struct resource_pool base;
-};
-struct resource_pool *dcn21_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc);
-bool dcn21_fast_validate_bw(
-		struct dc *dc,
-		struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		int *pipe_cnt_out,
-		int *pipe_split_from,
-		int *vlevel_out,
-		bool fast_validate);
-
-#endif /* _DCN21_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/Makefile b/drivers/gpu/drm/amd/display/dc/dcn30/Makefile
index af4d2065d2c1..9dcf06c0954d 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/Makefile
@@ -38,7 +38,6 @@ DCN30 := \
 	dcn30_dwb_cm.o \
 	dcn30_cm_common.o \
 	dcn30_mmhubbub.o \
-	dcn30_resource.o \
 	dcn30_dio_link_encoder.o
 
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c
deleted file mode 100644
index 7b259cb5f418..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c
+++ /dev/null
@@ -1,2611 +0,0 @@
-/*
- * Copyright 2020 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-
-#include "dm_services.h"
-#include "dc.h"
-
-#include "dcn30_init.h"
-
-#include "resource.h"
-#include "include/irq_service_interface.h"
-#include "dcn20/dcn20_resource.h"
-
-#include "dcn30_resource.h"
-
-#include "dcn10/dcn10_ipp.h"
-#include "dcn30/dcn30_hubbub.h"
-#include "dcn30/dcn30_mpc.h"
-#include "dcn30/dcn30_hubp.h"
-#include "irq/dcn30/irq_service_dcn30.h"
-#include "dcn30/dcn30_dpp.h"
-#include "dcn30/dcn30_optc.h"
-#include "dcn20/dcn20_hwseq.h"
-#include "dcn30/dcn30_hwseq.h"
-#include "dce110/dce110_hwseq.h"
-#include "dcn30/dcn30_opp.h"
-#include "dcn20/dcn20_dsc.h"
-#include "dcn30/dcn30_vpg.h"
-#include "dcn30/dcn30_afmt.h"
-#include "dcn30/dcn30_dio_stream_encoder.h"
-#include "dcn30/dcn30_dio_link_encoder.h"
-#include "dce/dce_clock_source.h"
-#include "dce/dce_audio.h"
-#include "dce/dce_hwseq.h"
-#include "clk_mgr.h"
-#include "virtual/virtual_stream_encoder.h"
-#include "dce110/dce110_resource.h"
-#include "dml/display_mode_vba.h"
-#include "dcn30/dcn30_dccg.h"
-#include "dcn10/dcn10_resource.h"
-#include "link.h"
-#include "dce/dce_panel_cntl.h"
-
-#include "dcn30/dcn30_dwb.h"
-#include "dcn30/dcn30_mmhubbub.h"
-
-#include "sienna_cichlid_ip_offset.h"
-#include "dcn/dcn_3_0_0_offset.h"
-#include "dcn/dcn_3_0_0_sh_mask.h"
-
-#include "nbio/nbio_7_4_offset.h"
-
-#include "dpcs/dpcs_3_0_0_offset.h"
-#include "dpcs/dpcs_3_0_0_sh_mask.h"
-
-#include "mmhub/mmhub_2_0_0_offset.h"
-#include "mmhub/mmhub_2_0_0_sh_mask.h"
-
-#include "reg_helper.h"
-#include "dce/dmub_abm.h"
-#include "dce/dmub_psr.h"
-#include "dce/dce_aux.h"
-#include "dce/dce_i2c.h"
-
-#include "dml/dcn30/dcn30_fpu.h"
-#include "dml/dcn30/display_mode_vba_30.h"
-#include "vm_helper.h"
-#include "dcn20/dcn20_vmid.h"
-#include "amdgpu_socbb.h"
-#include "dc_dmub_srv.h"
-
-#define DC_LOGGER \
-	dc->ctx->logger
-#define DC_LOGGER_INIT(logger)
-
-enum dcn30_clk_src_array_id {
-	DCN30_CLK_SRC_PLL0,
-	DCN30_CLK_SRC_PLL1,
-	DCN30_CLK_SRC_PLL2,
-	DCN30_CLK_SRC_PLL3,
-	DCN30_CLK_SRC_PLL4,
-	DCN30_CLK_SRC_PLL5,
-	DCN30_CLK_SRC_TOTAL
-};
-
-/* begin *********************
- * macros to expend register list macro defined in HW object header file
- */
-
-/* DCN */
-#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
-
-#define BASE(seg) BASE_INNER(seg)
-
-#define SR(reg_name)\
-		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) +  \
-					mm ## reg_name
-
-#define SRI(reg_name, block, id)\
-	.reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define SRI2(reg_name, block, id)\
-	.reg_name = BASE(mm ## reg_name ## _BASE_IDX) + \
-					mm ## reg_name
-
-#define SRIR(var_name, reg_name, block, id)\
-	.var_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define SRII_MPC_RMU(reg_name, block, id)\
-	.RMU##_##reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define SRII_DWB(reg_name, temp_name, block, id)\
-	.reg_name[id] = BASE(mm ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## temp_name
-
-#define SF_DWB2(reg_name, block, id, field_name, post_fix)	\
-	.field_name = reg_name ## __ ## field_name ## post_fix
-
-#define DCCG_SRII(reg_name, block, id)\
-	.block ## _ ## reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define VUPDATE_SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(mm ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
-					mm ## reg_name ## _ ## block ## id
-
-/* NBIO */
-#define NBIO_BASE_INNER(seg) \
-	NBIO_BASE__INST0_SEG ## seg
-
-#define NBIO_BASE(seg) \
-	NBIO_BASE_INNER(seg)
-
-#define NBIO_SR(reg_name)\
-		.reg_name = NBIO_BASE(mm ## reg_name ## _BASE_IDX) + \
-					mm ## reg_name
-
-/* MMHUB */
-#define MMHUB_BASE_INNER(seg) \
-	MMHUB_BASE__INST0_SEG ## seg
-
-#define MMHUB_BASE(seg) \
-	MMHUB_BASE_INNER(seg)
-
-#define MMHUB_SR(reg_name)\
-		.reg_name = MMHUB_BASE(mmMM ## reg_name ## _BASE_IDX) + \
-					mmMM ## reg_name
-
-/* CLOCK */
-#define CLK_BASE_INNER(seg) \
-	CLK_BASE__INST0_SEG ## seg
-
-#define CLK_BASE(seg) \
-	CLK_BASE_INNER(seg)
-
-#define CLK_SRI(reg_name, block, inst)\
-	.reg_name = CLK_BASE(mm ## block ## _ ## inst ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## _ ## inst ## _ ## reg_name
-
-
-static const struct bios_registers bios_regs = {
-		NBIO_SR(BIOS_SCRATCH_3),
-		NBIO_SR(BIOS_SCRATCH_6)
-};
-
-#define clk_src_regs(index, pllid)\
-[index] = {\
-	CS_COMMON_REG_LIST_DCN2_0(index, pllid),\
-}
-
-static const struct dce110_clk_src_regs clk_src_regs[] = {
-	clk_src_regs(0, A),
-	clk_src_regs(1, B),
-	clk_src_regs(2, C),
-	clk_src_regs(3, D),
-	clk_src_regs(4, E),
-	clk_src_regs(5, F)
-};
-
-static const struct dce110_clk_src_shift cs_shift = {
-		CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
-};
-
-static const struct dce110_clk_src_mask cs_mask = {
-		CS_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
-};
-
-#define abm_regs(id)\
-[id] = {\
-		ABM_DCN30_REG_LIST(id)\
-}
-
-static const struct dce_abm_registers abm_regs[] = {
-		abm_regs(0),
-		abm_regs(1),
-		abm_regs(2),
-		abm_regs(3),
-		abm_regs(4),
-		abm_regs(5),
-};
-
-static const struct dce_abm_shift abm_shift = {
-		ABM_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dce_abm_mask abm_mask = {
-		ABM_MASK_SH_LIST_DCN30(_MASK)
-};
-
-
-
-#define audio_regs(id)\
-[id] = {\
-		AUD_COMMON_REG_LIST(id)\
-}
-
-static const struct dce_audio_registers audio_regs[] = {
-	audio_regs(0),
-	audio_regs(1),
-	audio_regs(2),
-	audio_regs(3),
-	audio_regs(4),
-	audio_regs(5),
-	audio_regs(6)
-};
-
-#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
-		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
-
-static const struct dce_audio_shift audio_shift = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_audio_mask audio_mask = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
-};
-
-#define vpg_regs(id)\
-[id] = {\
-	VPG_DCN3_REG_LIST(id)\
-}
-
-static const struct dcn30_vpg_registers vpg_regs[] = {
-	vpg_regs(0),
-	vpg_regs(1),
-	vpg_regs(2),
-	vpg_regs(3),
-	vpg_regs(4),
-	vpg_regs(5),
-	vpg_regs(6),
-};
-
-static const struct dcn30_vpg_shift vpg_shift = {
-	DCN3_VPG_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn30_vpg_mask vpg_mask = {
-	DCN3_VPG_MASK_SH_LIST(_MASK)
-};
-
-#define afmt_regs(id)\
-[id] = {\
-	AFMT_DCN3_REG_LIST(id)\
-}
-
-static const struct dcn30_afmt_registers afmt_regs[] = {
-	afmt_regs(0),
-	afmt_regs(1),
-	afmt_regs(2),
-	afmt_regs(3),
-	afmt_regs(4),
-	afmt_regs(5),
-	afmt_regs(6),
-};
-
-static const struct dcn30_afmt_shift afmt_shift = {
-	DCN3_AFMT_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn30_afmt_mask afmt_mask = {
-	DCN3_AFMT_MASK_SH_LIST(_MASK)
-};
-
-#define stream_enc_regs(id)\
-[id] = {\
-	SE_DCN3_REG_LIST(id)\
-}
-
-static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
-	stream_enc_regs(0),
-	stream_enc_regs(1),
-	stream_enc_regs(2),
-	stream_enc_regs(3),
-	stream_enc_regs(4),
-	stream_enc_regs(5)
-};
-
-static const struct dcn10_stream_encoder_shift se_shift = {
-		SE_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn10_stream_encoder_mask se_mask = {
-		SE_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-
-#define aux_regs(id)\
-[id] = {\
-	DCN2_AUX_REG_LIST(id)\
-}
-
-static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
-		aux_regs(0),
-		aux_regs(1),
-		aux_regs(2),
-		aux_regs(3),
-		aux_regs(4),
-		aux_regs(5)
-};
-
-#define hpd_regs(id)\
-[id] = {\
-	HPD_REG_LIST(id)\
-}
-
-static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
-		hpd_regs(0),
-		hpd_regs(1),
-		hpd_regs(2),
-		hpd_regs(3),
-		hpd_regs(4),
-		hpd_regs(5)
-};
-
-#define link_regs(id, phyid)\
-[id] = {\
-	LE_DCN3_REG_LIST(id), \
-	UNIPHY_DCN2_REG_LIST(phyid), \
-	DPCS_DCN2_REG_LIST(id), \
-	SRI(DP_DPHY_INTERNAL_CTRL, DP, id) \
-}
-
-static const struct dce110_aux_registers_shift aux_shift = {
-	DCN_AUX_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce110_aux_registers_mask aux_mask = {
-	DCN_AUX_MASK_SH_LIST(_MASK)
-};
-
-static const struct dcn10_link_enc_registers link_enc_regs[] = {
-	link_regs(0, A),
-	link_regs(1, B),
-	link_regs(2, C),
-	link_regs(3, D),
-	link_regs(4, E),
-	link_regs(5, F)
-};
-
-static const struct dcn10_link_enc_shift le_shift = {
-	LINK_ENCODER_MASK_SH_LIST_DCN30(__SHIFT),\
-	DPCS_DCN2_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn10_link_enc_mask le_mask = {
-	LINK_ENCODER_MASK_SH_LIST_DCN30(_MASK),\
-	DPCS_DCN2_MASK_SH_LIST(_MASK)
-};
-
-
-static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
-	{ DCN_PANEL_CNTL_REG_LIST() }
-};
-
-static const struct dce_panel_cntl_shift panel_cntl_shift = {
-	DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_panel_cntl_mask panel_cntl_mask = {
-	DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
-};
-
-#define dpp_regs(id)\
-[id] = {\
-	DPP_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn3_dpp_registers dpp_regs[] = {
-	dpp_regs(0),
-	dpp_regs(1),
-	dpp_regs(2),
-	dpp_regs(3),
-	dpp_regs(4),
-	dpp_regs(5),
-};
-
-static const struct dcn3_dpp_shift tf_shift = {
-		DPP_REG_LIST_SH_MASK_DCN30(__SHIFT)
-};
-
-static const struct dcn3_dpp_mask tf_mask = {
-		DPP_REG_LIST_SH_MASK_DCN30(_MASK)
-};
-
-#define opp_regs(id)\
-[id] = {\
-	OPP_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn20_opp_registers opp_regs[] = {
-	opp_regs(0),
-	opp_regs(1),
-	opp_regs(2),
-	opp_regs(3),
-	opp_regs(4),
-	opp_regs(5)
-};
-
-static const struct dcn20_opp_shift opp_shift = {
-	OPP_MASK_SH_LIST_DCN20(__SHIFT)
-};
-
-static const struct dcn20_opp_mask opp_mask = {
-	OPP_MASK_SH_LIST_DCN20(_MASK)
-};
-
-#define aux_engine_regs(id)\
-[id] = {\
-	AUX_COMMON_REG_LIST0(id), \
-	.AUXN_IMPCAL = 0, \
-	.AUXP_IMPCAL = 0, \
-	.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
-}
-
-static const struct dce110_aux_registers aux_engine_regs[] = {
-		aux_engine_regs(0),
-		aux_engine_regs(1),
-		aux_engine_regs(2),
-		aux_engine_regs(3),
-		aux_engine_regs(4),
-		aux_engine_regs(5)
-};
-
-#define dwbc_regs_dcn3(id)\
-[id] = {\
-	DWBC_COMMON_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn30_dwbc_registers dwbc30_regs[] = {
-	dwbc_regs_dcn3(0),
-};
-
-static const struct dcn30_dwbc_shift dwbc30_shift = {
-	DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_dwbc_mask dwbc30_mask = {
-	DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define mcif_wb_regs_dcn3(id)\
-[id] = {\
-	MCIF_WB_COMMON_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn30_mmhubbub_registers mcif_wb30_regs[] = {
-	mcif_wb_regs_dcn3(0)
-};
-
-static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
-	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
-	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define dsc_regsDCN20(id)\
-[id] = {\
-	DSC_REG_LIST_DCN20(id)\
-}
-
-static const struct dcn20_dsc_registers dsc_regs[] = {
-	dsc_regsDCN20(0),
-	dsc_regsDCN20(1),
-	dsc_regsDCN20(2),
-	dsc_regsDCN20(3),
-	dsc_regsDCN20(4),
-	dsc_regsDCN20(5)
-};
-
-static const struct dcn20_dsc_shift dsc_shift = {
-	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
-};
-
-static const struct dcn20_dsc_mask dsc_mask = {
-	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
-};
-
-static const struct dcn30_mpc_registers mpc_regs = {
-		MPC_REG_LIST_DCN3_0(0),
-		MPC_REG_LIST_DCN3_0(1),
-		MPC_REG_LIST_DCN3_0(2),
-		MPC_REG_LIST_DCN3_0(3),
-		MPC_REG_LIST_DCN3_0(4),
-		MPC_REG_LIST_DCN3_0(5),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(0),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(1),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(2),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(3),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(4),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(5),
-		MPC_RMU_GLOBAL_REG_LIST_DCN3AG,
-		MPC_RMU_REG_LIST_DCN3AG(0),
-		MPC_RMU_REG_LIST_DCN3AG(1),
-		MPC_RMU_REG_LIST_DCN3AG(2),
-		MPC_DWB_MUX_REG_LIST_DCN3_0(0),
-};
-
-static const struct dcn30_mpc_shift mpc_shift = {
-	MPC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_mpc_mask mpc_mask = {
-	MPC_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define optc_regs(id)\
-[id] = {OPTC_COMMON_REG_LIST_DCN3_0(id)}
-
-
-static const struct dcn_optc_registers optc_regs[] = {
-	optc_regs(0),
-	optc_regs(1),
-	optc_regs(2),
-	optc_regs(3),
-	optc_regs(4),
-	optc_regs(5)
-};
-
-static const struct dcn_optc_shift optc_shift = {
-	OPTC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn_optc_mask optc_mask = {
-	OPTC_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define hubp_regs(id)\
-[id] = {\
-	HUBP_REG_LIST_DCN30(id)\
-}
-
-static const struct dcn_hubp2_registers hubp_regs[] = {
-		hubp_regs(0),
-		hubp_regs(1),
-		hubp_regs(2),
-		hubp_regs(3),
-		hubp_regs(4),
-		hubp_regs(5)
-};
-
-static const struct dcn_hubp2_shift hubp_shift = {
-		HUBP_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn_hubp2_mask hubp_mask = {
-		HUBP_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static const struct dcn_hubbub_registers hubbub_reg = {
-		HUBBUB_REG_LIST_DCN30(0)
-};
-
-static const struct dcn_hubbub_shift hubbub_shift = {
-		HUBBUB_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn_hubbub_mask hubbub_mask = {
-		HUBBUB_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static const struct dccg_registers dccg_regs = {
-		DCCG_REG_LIST_DCN30()
-};
-
-static const struct dccg_shift dccg_shift = {
-		DCCG_MASK_SH_LIST_DCN3(__SHIFT)
-};
-
-static const struct dccg_mask dccg_mask = {
-		DCCG_MASK_SH_LIST_DCN3(_MASK)
-};
-
-static const struct dce_hwseq_registers hwseq_reg = {
-		HWSEQ_DCN30_REG_LIST()
-};
-
-static const struct dce_hwseq_shift hwseq_shift = {
-		HWSEQ_DCN30_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_hwseq_mask hwseq_mask = {
-		HWSEQ_DCN30_MASK_SH_LIST(_MASK)
-};
-#define vmid_regs(id)\
-[id] = {\
-		DCN20_VMID_REG_LIST(id)\
-}
-
-static const struct dcn_vmid_registers vmid_regs[] = {
-	vmid_regs(0),
-	vmid_regs(1),
-	vmid_regs(2),
-	vmid_regs(3),
-	vmid_regs(4),
-	vmid_regs(5),
-	vmid_regs(6),
-	vmid_regs(7),
-	vmid_regs(8),
-	vmid_regs(9),
-	vmid_regs(10),
-	vmid_regs(11),
-	vmid_regs(12),
-	vmid_regs(13),
-	vmid_regs(14),
-	vmid_regs(15)
-};
-
-static const struct dcn20_vmid_shift vmid_shifts = {
-		DCN20_VMID_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn20_vmid_mask vmid_masks = {
-		DCN20_VMID_MASK_SH_LIST(_MASK)
-};
-
-static const struct resource_caps res_cap_dcn3 = {
-	.num_timing_generator = 6,
-	.num_opp = 6,
-	.num_video_plane = 6,
-	.num_audio = 6,
-	.num_stream_encoder = 6,
-	.num_pll = 6,
-	.num_dwb = 1,
-	.num_ddc = 6,
-	.num_vmid = 16,
-	.num_mpc_3dlut = 3,
-	.num_dsc = 6,
-};
-
-static const struct dc_plane_cap plane_cap = {
-	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.per_pixel_alpha = true,
-
-	.pixel_format_support = {
-			.argb8888 = true,
-			.nv12 = true,
-			.fp16 = true,
-			.p010 = true,
-			.ayuv = false,
-	},
-
-	.max_upscale_factor = {
-			.argb8888 = 16000,
-			.nv12 = 16000,
-			.fp16 = 16000
-	},
-
-	/* 6:1 downscaling ratio: 1000/6 = 166.666 */
-	.max_downscale_factor = {
-			.argb8888 = 167,
-			.nv12 = 167,
-			.fp16 = 167
-	},
-	16,
-	16
-};
-
-static const struct dc_debug_options debug_defaults_drv = {
-	.disable_dmcu = true, //No DMCU on DCN30
-	.force_abm_enable = false,
-	.timing_trace = false,
-	.clock_trace = true,
-	.disable_pplib_clock_request = true,
-	.pipe_split_policy = MPC_SPLIT_DYNAMIC,
-	.force_single_disp_pipe_split = false,
-	.disable_dcc = DCC_ENABLE,
-	.vsr_support = true,
-	.performance_trace = false,
-	.max_downscale_src_width = 7680,/*upto 8K*/
-	.disable_pplib_wm_range = false,
-	.scl_reset_length10 = true,
-	.sanity_checks = false,
-	.underflow_assert_delay_us = 0xFFFFFFFF,
-	.dwb_fi_phase = -1, // -1 = disable,
-	.dmub_command_table = true,
-	.use_max_lb = true,
-	.exit_idle_opt_for_cursor_updates = true,
-	.enable_legacy_fast_update = false,
-	.using_dml2 = false,
-};
-
-static const struct dc_panel_config panel_config_defaults = {
-	.psr = {
-		.disable_psr = false,
-		.disallow_psrsu = false,
-		.disallow_replay = false,
-	},
-};
-
-static void dcn30_dpp_destroy(struct dpp **dpp)
-{
-	kfree(TO_DCN20_DPP(*dpp));
-	*dpp = NULL;
-}
-
-static struct dpp *dcn30_dpp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn3_dpp *dpp =
-		kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
-
-	if (!dpp)
-		return NULL;
-
-	if (dpp3_construct(dpp, ctx, inst,
-			&dpp_regs[inst], &tf_shift, &tf_mask))
-		return &dpp->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(dpp);
-	return NULL;
-}
-
-static struct output_pixel_processor *dcn30_opp_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_opp *opp =
-		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
-
-	if (!opp) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dcn20_opp_construct(opp, ctx, inst,
-			&opp_regs[inst], &opp_shift, &opp_mask);
-	return &opp->base;
-}
-
-static struct dce_aux *dcn30_aux_engine_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct aux_engine_dce110 *aux_engine =
-		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
-
-	if (!aux_engine)
-		return NULL;
-
-	dce110_aux_engine_construct(aux_engine, ctx, inst,
-				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
-				    &aux_engine_regs[inst],
-					&aux_mask,
-					&aux_shift,
-					ctx->dc->caps.extended_aux_timeout_support);
-
-	return &aux_engine->base;
-}
-
-#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST_DCN30(id) }
-
-static const struct dce_i2c_registers i2c_hw_regs[] = {
-		i2c_inst_regs(1),
-		i2c_inst_regs(2),
-		i2c_inst_regs(3),
-		i2c_inst_regs(4),
-		i2c_inst_regs(5),
-		i2c_inst_regs(6),
-};
-
-static const struct dce_i2c_shift i2c_shifts = {
-		I2C_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dce_i2c_mask i2c_masks = {
-		I2C_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static struct dce_i2c_hw *dcn30_i2c_hw_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_i2c_hw *dce_i2c_hw =
-		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
-
-	if (!dce_i2c_hw)
-		return NULL;
-
-	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
-				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
-
-	return dce_i2c_hw;
-}
-
-static struct mpc *dcn30_mpc_create(
-		struct dc_context *ctx,
-		int num_mpcc,
-		int num_rmu)
-{
-	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc),
-					  GFP_KERNEL);
-
-	if (!mpc30)
-		return NULL;
-
-	dcn30_mpc_construct(mpc30, ctx,
-			&mpc_regs,
-			&mpc_shift,
-			&mpc_mask,
-			num_mpcc,
-			num_rmu);
-
-	return &mpc30->base;
-}
-
-static struct hubbub *dcn30_hubbub_create(struct dc_context *ctx)
-{
-	int i;
-
-	struct dcn20_hubbub *hubbub3 = kzalloc(sizeof(struct dcn20_hubbub),
-					  GFP_KERNEL);
-
-	if (!hubbub3)
-		return NULL;
-
-	hubbub3_construct(hubbub3, ctx,
-			&hubbub_reg,
-			&hubbub_shift,
-			&hubbub_mask);
-
-
-	for (i = 0; i < res_cap_dcn3.num_vmid; i++) {
-		struct dcn20_vmid *vmid = &hubbub3->vmid[i];
-
-		vmid->ctx = ctx;
-
-		vmid->regs = &vmid_regs[i];
-		vmid->shifts = &vmid_shifts;
-		vmid->masks = &vmid_masks;
-	}
-
-	return &hubbub3->base;
-}
-
-static struct timing_generator *dcn30_timing_generator_create(
-		struct dc_context *ctx,
-		uint32_t instance)
-{
-	struct optc *tgn10 =
-		kzalloc(sizeof(struct optc), GFP_KERNEL);
-
-	if (!tgn10)
-		return NULL;
-
-	tgn10->base.inst = instance;
-	tgn10->base.ctx = ctx;
-
-	tgn10->tg_regs = &optc_regs[instance];
-	tgn10->tg_shift = &optc_shift;
-	tgn10->tg_mask = &optc_mask;
-
-	dcn30_timing_generator_init(tgn10);
-
-	return &tgn10->base;
-}
-
-static const struct encoder_feature_support link_enc_feature = {
-		.max_hdmi_deep_color = COLOR_DEPTH_121212,
-		.max_hdmi_pixel_clock = 600000,
-		.hdmi_ycbcr420_supported = true,
-		.dp_ycbcr420_supported = true,
-		.fec_supported = true,
-		.flags.bits.IS_HBR2_CAPABLE = true,
-		.flags.bits.IS_HBR3_CAPABLE = true,
-		.flags.bits.IS_TPS3_CAPABLE = true,
-		.flags.bits.IS_TPS4_CAPABLE = true
-};
-
-static struct link_encoder *dcn30_link_encoder_create(
-	struct dc_context *ctx,
-	const struct encoder_init_data *enc_init_data)
-{
-	struct dcn20_link_encoder *enc20 =
-		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
-
-	if (!enc20)
-		return NULL;
-
-	dcn30_link_encoder_construct(enc20,
-			enc_init_data,
-			&link_enc_feature,
-			&link_enc_regs[enc_init_data->transmitter],
-			&link_enc_aux_regs[enc_init_data->channel - 1],
-			&link_enc_hpd_regs[enc_init_data->hpd_source],
-			&le_shift,
-			&le_mask);
-
-	return &enc20->enc10.base;
-}
-
-static struct panel_cntl *dcn30_panel_cntl_create(const struct panel_cntl_init_data *init_data)
-{
-	struct dce_panel_cntl *panel_cntl =
-		kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
-
-	if (!panel_cntl)
-		return NULL;
-
-	dce_panel_cntl_construct(panel_cntl,
-			init_data,
-			&panel_cntl_regs[init_data->inst],
-			&panel_cntl_shift,
-			&panel_cntl_mask);
-
-	return &panel_cntl->base;
-}
-
-static void read_dce_straps(
-	struct dc_context *ctx,
-	struct resource_straps *straps)
-{
-	generic_reg_get(ctx, mmDC_PINSTRAPS + BASE(mmDC_PINSTRAPS_BASE_IDX),
-		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
-
-}
-
-static struct audio *dcn30_create_audio(
-		struct dc_context *ctx, unsigned int inst)
-{
-	return dce_audio_create(ctx, inst,
-			&audio_regs[inst], &audio_shift, &audio_mask);
-}
-
-static struct vpg *dcn30_vpg_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn30_vpg *vpg3 = kzalloc(sizeof(struct dcn30_vpg), GFP_KERNEL);
-
-	if (!vpg3)
-		return NULL;
-
-	vpg3_construct(vpg3, ctx, inst,
-			&vpg_regs[inst],
-			&vpg_shift,
-			&vpg_mask);
-
-	return &vpg3->base;
-}
-
-static struct afmt *dcn30_afmt_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn30_afmt *afmt3 = kzalloc(sizeof(struct dcn30_afmt), GFP_KERNEL);
-
-	if (!afmt3)
-		return NULL;
-
-	afmt3_construct(afmt3, ctx, inst,
-			&afmt_regs[inst],
-			&afmt_shift,
-			&afmt_mask);
-
-	return &afmt3->base;
-}
-
-static struct stream_encoder *dcn30_stream_encoder_create(enum engine_id eng_id,
-							  struct dc_context *ctx)
-{
-	struct dcn10_stream_encoder *enc1;
-	struct vpg *vpg;
-	struct afmt *afmt;
-	int vpg_inst;
-	int afmt_inst;
-
-	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
-	if (eng_id <= ENGINE_ID_DIGF) {
-		vpg_inst = eng_id;
-		afmt_inst = eng_id;
-	} else
-		return NULL;
-
-	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
-	vpg = dcn30_vpg_create(ctx, vpg_inst);
-	afmt = dcn30_afmt_create(ctx, afmt_inst);
-
-	if (!enc1 || !vpg || !afmt) {
-		kfree(enc1);
-		kfree(vpg);
-		kfree(afmt);
-		return NULL;
-	}
-
-	dcn30_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
-					eng_id, vpg, afmt,
-					&stream_enc_regs[eng_id],
-					&se_shift, &se_mask);
-
-	return &enc1->base;
-}
-
-static struct dce_hwseq *dcn30_hwseq_create(struct dc_context *ctx)
-{
-	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
-
-	if (hws) {
-		hws->ctx = ctx;
-		hws->regs = &hwseq_reg;
-		hws->shifts = &hwseq_shift;
-		hws->masks = &hwseq_mask;
-	}
-	return hws;
-}
-static const struct resource_create_funcs res_create_funcs = {
-	.read_dce_straps = read_dce_straps,
-	.create_audio = dcn30_create_audio,
-	.create_stream_encoder = dcn30_stream_encoder_create,
-	.create_hwseq = dcn30_hwseq_create,
-};
-
-static void dcn30_resource_destruct(struct dcn30_resource_pool *pool)
-{
-	unsigned int i;
-
-	for (i = 0; i < pool->base.stream_enc_count; i++) {
-		if (pool->base.stream_enc[i] != NULL) {
-			if (pool->base.stream_enc[i]->vpg != NULL) {
-				kfree(DCN30_VPG_FROM_VPG(pool->base.stream_enc[i]->vpg));
-				pool->base.stream_enc[i]->vpg = NULL;
-			}
-			if (pool->base.stream_enc[i]->afmt != NULL) {
-				kfree(DCN30_AFMT_FROM_AFMT(pool->base.stream_enc[i]->afmt));
-				pool->base.stream_enc[i]->afmt = NULL;
-			}
-			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
-			pool->base.stream_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		if (pool->base.dscs[i] != NULL)
-			dcn20_dsc_destroy(&pool->base.dscs[i]);
-	}
-
-	if (pool->base.mpc != NULL) {
-		kfree(TO_DCN20_MPC(pool->base.mpc));
-		pool->base.mpc = NULL;
-	}
-	if (pool->base.hubbub != NULL) {
-		kfree(pool->base.hubbub);
-		pool->base.hubbub = NULL;
-	}
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		if (pool->base.dpps[i] != NULL)
-			dcn30_dpp_destroy(&pool->base.dpps[i]);
-
-		if (pool->base.ipps[i] != NULL)
-			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
-
-		if (pool->base.hubps[i] != NULL) {
-			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
-			pool->base.hubps[i] = NULL;
-		}
-
-		if (pool->base.irqs != NULL) {
-			dal_irq_service_destroy(&pool->base.irqs);
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		if (pool->base.engines[i] != NULL)
-			dce110_engine_destroy(&pool->base.engines[i]);
-		if (pool->base.hw_i2cs[i] != NULL) {
-			kfree(pool->base.hw_i2cs[i]);
-			pool->base.hw_i2cs[i] = NULL;
-		}
-		if (pool->base.sw_i2cs[i] != NULL) {
-			kfree(pool->base.sw_i2cs[i]);
-			pool->base.sw_i2cs[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
-		if (pool->base.opps[i] != NULL)
-			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		if (pool->base.timing_generators[i] != NULL)	{
-			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
-			pool->base.timing_generators[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
-		if (pool->base.dwbc[i] != NULL) {
-			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
-			pool->base.dwbc[i] = NULL;
-		}
-		if (pool->base.mcif_wb[i] != NULL) {
-			kfree(TO_DCN30_MMHUBBUB(pool->base.mcif_wb[i]));
-			pool->base.mcif_wb[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.audio_count; i++) {
-		if (pool->base.audios[i])
-			dce_aud_destroy(&pool->base.audios[i]);
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] != NULL) {
-			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
-			pool->base.clock_sources[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
-		if (pool->base.mpc_lut[i] != NULL) {
-			dc_3dlut_func_release(pool->base.mpc_lut[i]);
-			pool->base.mpc_lut[i] = NULL;
-		}
-		if (pool->base.mpc_shaper[i] != NULL) {
-			dc_transfer_func_release(pool->base.mpc_shaper[i]);
-			pool->base.mpc_shaper[i] = NULL;
-		}
-	}
-
-	if (pool->base.dp_clock_source != NULL) {
-		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
-		pool->base.dp_clock_source = NULL;
-	}
-
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		if (pool->base.multiple_abms[i] != NULL)
-			dce_abm_destroy(&pool->base.multiple_abms[i]);
-	}
-
-	if (pool->base.psr != NULL)
-		dmub_psr_destroy(&pool->base.psr);
-
-	if (pool->base.dccg != NULL)
-		dcn_dccg_destroy(&pool->base.dccg);
-
-	if (pool->base.oem_device != NULL) {
-		struct dc *dc = pool->base.oem_device->ctx->dc;
-
-		dc->link_srv->destroy_ddc_service(&pool->base.oem_device);
-	}
-}
-
-static struct hubp *dcn30_hubp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn20_hubp *hubp2 =
-		kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
-
-	if (!hubp2)
-		return NULL;
-
-	if (hubp3_construct(hubp2, ctx, inst,
-			&hubp_regs[inst], &hubp_shift, &hubp_mask))
-		return &hubp2->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(hubp2);
-	return NULL;
-}
-
-static bool dcn30_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t pipe_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < pipe_count; i++) {
-		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc),
-						    GFP_KERNEL);
-
-		if (!dwbc30) {
-			dm_error("DC: failed to create dwbc30!\n");
-			return false;
-		}
-
-		dcn30_dwbc_construct(dwbc30, ctx,
-				&dwbc30_regs[i],
-				&dwbc30_shift,
-				&dwbc30_mask,
-				i);
-
-		pool->dwbc[i] = &dwbc30->base;
-	}
-	return true;
-}
-
-static bool dcn30_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t pipe_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < pipe_count; i++) {
-		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub),
-						    GFP_KERNEL);
-
-		if (!mcif_wb30) {
-			dm_error("DC: failed to create mcif_wb30!\n");
-			return false;
-		}
-
-		dcn30_mmhubbub_construct(mcif_wb30, ctx,
-				&mcif_wb30_regs[i],
-				&mcif_wb30_shift,
-				&mcif_wb30_mask,
-				i);
-
-		pool->mcif_wb[i] = &mcif_wb30->base;
-	}
-	return true;
-}
-
-static struct display_stream_compressor *dcn30_dsc_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_dsc *dsc =
-		kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
-
-	if (!dsc) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
-	return &dsc->base;
-}
-
-enum dc_status dcn30_add_stream_to_ctx(struct dc *dc, struct dc_state *new_ctx, struct dc_stream_state *dc_stream)
-{
-
-	return dcn20_add_stream_to_ctx(dc, new_ctx, dc_stream);
-}
-
-static void dcn30_destroy_resource_pool(struct resource_pool **pool)
-{
-	struct dcn30_resource_pool *dcn30_pool = TO_DCN30_RES_POOL(*pool);
-
-	dcn30_resource_destruct(dcn30_pool);
-	kfree(dcn30_pool);
-	*pool = NULL;
-}
-
-static struct clock_source *dcn30_clock_source_create(
-		struct dc_context *ctx,
-		struct dc_bios *bios,
-		enum clock_source_id id,
-		const struct dce110_clk_src_regs *regs,
-		bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src =
-		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dcn3_clk_src_construct(clk_src, ctx, bios, id,
-			regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-
-	kfree(clk_src);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
-int dcn30_populate_dml_pipes_from_context(
-	struct dc *dc, struct dc_state *context,
-	display_e2e_pipe_params_st *pipes,
-	bool fast_validate)
-{
-	int i, pipe_cnt;
-	struct resource_context *res_ctx = &context->res_ctx;
-
-	DC_FP_START();
-	dcn20_populate_dml_pipes_from_context(dc, context, pipes, fast_validate);
-	DC_FP_END();
-
-	for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) {
-		if (!res_ctx->pipe_ctx[i].stream)
-			continue;
-
-		pipes[pipe_cnt++].pipe.scale_ratio_depth.lb_depth =
-			dm_lb_16;
-	}
-
-	return pipe_cnt;
-}
-
-void dcn30_populate_dml_writeback_from_context(
-	struct dc *dc, struct resource_context *res_ctx, display_e2e_pipe_params_st *pipes)
-{
-	DC_FP_START();
-	dcn30_fpu_populate_dml_writeback_from_context(dc, res_ctx, pipes);
-	DC_FP_END();
-}
-
-unsigned int dcn30_calc_max_scaled_time(
-		unsigned int time_per_pixel,
-		enum mmhubbub_wbif_mode mode,
-		unsigned int urgent_watermark)
-{
-	unsigned int time_per_byte = 0;
-	unsigned int total_free_entry = 0xb40;
-	unsigned int buf_lh_capability;
-	unsigned int max_scaled_time;
-
-	if (mode == PACKED_444) /* packed mode 32 bpp */
-		time_per_byte = time_per_pixel/4;
-	else if (mode == PACKED_444_FP16) /* packed mode 64 bpp */
-		time_per_byte = time_per_pixel/8;
-
-	if (time_per_byte == 0)
-		time_per_byte = 1;
-
-	buf_lh_capability = (total_free_entry*time_per_byte*32) >> 6; /* time_per_byte is in u6.6*/
-	max_scaled_time   = buf_lh_capability - urgent_watermark;
-	return max_scaled_time;
-}
-
-void dcn30_set_mcif_arb_params(
-		struct dc *dc,
-		struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		int pipe_cnt)
-{
-	enum mmhubbub_wbif_mode wbif_mode;
-	struct display_mode_lib *dml = &context->bw_ctx.dml;
-	struct mcif_arb_params *wb_arb_params;
-	int i, j, dwb_pipe;
-
-	/* Writeback MCIF_WB arbitration parameters */
-	dwb_pipe = 0;
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-
-		if (!context->res_ctx.pipe_ctx[i].stream)
-			continue;
-
-		for (j = 0; j < MAX_DWB_PIPES; j++) {
-			struct dc_writeback_info *writeback_info = &context->res_ctx.pipe_ctx[i].stream->writeback_info[j];
-
-			if (writeback_info->wb_enabled == false)
-				continue;
-
-			//wb_arb_params = &context->res_ctx.pipe_ctx[i].stream->writeback_info[j].mcif_arb_params;
-			wb_arb_params = &context->bw_ctx.bw.dcn.bw_writeback.mcif_wb_arb[dwb_pipe];
-
-			if (writeback_info->dwb_params.cnv_params.fc_out_format == DWB_OUT_FORMAT_64BPP_ARGB ||
-				writeback_info->dwb_params.cnv_params.fc_out_format == DWB_OUT_FORMAT_64BPP_RGBA)
-				wbif_mode = PACKED_444_FP16;
-			else
-				wbif_mode = PACKED_444;
-
-			DC_FP_START();
-			dcn30_fpu_set_mcif_arb_params(wb_arb_params, dml, pipes, pipe_cnt, j);
-			DC_FP_END();
-			wb_arb_params->time_per_pixel = (1000000 << 6) / context->res_ctx.pipe_ctx[i].stream->phy_pix_clk; /* time_per_pixel should be in u6.6 format */
-			wb_arb_params->slice_lines = 32;
-			wb_arb_params->arbitration_slice = 2; /* irrelevant since there is no YUV output */
-			wb_arb_params->max_scaled_time = dcn30_calc_max_scaled_time(wb_arb_params->time_per_pixel,
-					wbif_mode,
-					wb_arb_params->cli_watermark[0]); /* assume 4 watermark sets have the same value */
-
-			dwb_pipe++;
-
-			if (dwb_pipe >= MAX_DWB_PIPES)
-				return;
-		}
-		if (dwb_pipe >= MAX_DWB_PIPES)
-			return;
-	}
-
-}
-
-static struct dc_cap_funcs cap_funcs = {
-	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
-};
-
-bool dcn30_acquire_post_bldn_3dlut(
-		struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		int mpcc_id,
-		struct dc_3dlut **lut,
-		struct dc_transfer_func **shaper)
-{
-	int i;
-	bool ret = false;
-	union dc_3dlut_state *state;
-
-	ASSERT(*lut == NULL && *shaper == NULL);
-	*lut = NULL;
-	*shaper = NULL;
-
-	for (i = 0; i < pool->res_cap->num_mpc_3dlut; i++) {
-		if (!res_ctx->is_mpc_3dlut_acquired[i]) {
-			*lut = pool->mpc_lut[i];
-			*shaper = pool->mpc_shaper[i];
-			state = &pool->mpc_lut[i]->state;
-			res_ctx->is_mpc_3dlut_acquired[i] = true;
-			state->bits.rmu_idx_valid = 1;
-			state->bits.rmu_mux_num = i;
-			if (state->bits.rmu_mux_num == 0)
-				state->bits.mpc_rmu0_mux = mpcc_id;
-			else if (state->bits.rmu_mux_num == 1)
-				state->bits.mpc_rmu1_mux = mpcc_id;
-			else if (state->bits.rmu_mux_num == 2)
-				state->bits.mpc_rmu2_mux = mpcc_id;
-			ret = true;
-			break;
-		}
-	}
-	return ret;
-}
-
-bool dcn30_release_post_bldn_3dlut(
-		struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		struct dc_3dlut **lut,
-		struct dc_transfer_func **shaper)
-{
-	int i;
-	bool ret = false;
-
-	for (i = 0; i < pool->res_cap->num_mpc_3dlut; i++) {
-		if (pool->mpc_lut[i] == *lut && pool->mpc_shaper[i] == *shaper) {
-			res_ctx->is_mpc_3dlut_acquired[i] = false;
-			pool->mpc_lut[i]->state.raw = 0;
-			*lut = NULL;
-			*shaper = NULL;
-			ret = true;
-			break;
-		}
-	}
-	return ret;
-}
-
-static bool is_soc_bounding_box_valid(struct dc *dc)
-{
-	uint32_t hw_internal_rev = dc->ctx->asic_id.hw_internal_rev;
-
-	if (ASICREV_IS_SIENNA_CICHLID_P(hw_internal_rev))
-		return true;
-
-	return false;
-}
-
-static bool init_soc_bounding_box(struct dc *dc,
-				  struct dcn30_resource_pool *pool)
-{
-	struct _vcs_dpi_soc_bounding_box_st *loaded_bb = &dcn3_0_soc;
-	struct _vcs_dpi_ip_params_st *loaded_ip = &dcn3_0_ip;
-
-	DC_LOGGER_INIT(dc->ctx->logger);
-
-	if (!is_soc_bounding_box_valid(dc)) {
-		DC_LOG_ERROR("%s: not valid soc bounding box\n", __func__);
-		return false;
-	}
-
-	loaded_ip->max_num_otg = pool->base.res_cap->num_timing_generator;
-	loaded_ip->max_num_dpp = pool->base.pipe_count;
-	loaded_ip->clamp_min_dcfclk = dc->config.clamp_min_dcfclk;
-	dcn20_patch_bounding_box(dc, loaded_bb);
-	DC_FP_START();
-	patch_dcn30_soc_bounding_box(dc, &dcn3_0_soc);
-	DC_FP_END();
-
-	return true;
-}
-
-static bool dcn30_split_stream_for_mpc_or_odm(
-		const struct dc *dc,
-		struct resource_context *res_ctx,
-		struct pipe_ctx *pri_pipe,
-		struct pipe_ctx *sec_pipe,
-		bool odm)
-{
-	int pipe_idx = sec_pipe->pipe_idx;
-	const struct resource_pool *pool = dc->res_pool;
-
-	*sec_pipe = *pri_pipe;
-
-	sec_pipe->pipe_idx = pipe_idx;
-	sec_pipe->plane_res.mi = pool->mis[pipe_idx];
-	sec_pipe->plane_res.hubp = pool->hubps[pipe_idx];
-	sec_pipe->plane_res.ipp = pool->ipps[pipe_idx];
-	sec_pipe->plane_res.xfm = pool->transforms[pipe_idx];
-	sec_pipe->plane_res.dpp = pool->dpps[pipe_idx];
-	sec_pipe->plane_res.mpcc_inst = pool->dpps[pipe_idx]->inst;
-	sec_pipe->stream_res.dsc = NULL;
-	if (odm) {
-		if (pri_pipe->next_odm_pipe) {
-			ASSERT(pri_pipe->next_odm_pipe != sec_pipe);
-			sec_pipe->next_odm_pipe = pri_pipe->next_odm_pipe;
-			sec_pipe->next_odm_pipe->prev_odm_pipe = sec_pipe;
-		}
-		if (pri_pipe->top_pipe && pri_pipe->top_pipe->next_odm_pipe) {
-			pri_pipe->top_pipe->next_odm_pipe->bottom_pipe = sec_pipe;
-			sec_pipe->top_pipe = pri_pipe->top_pipe->next_odm_pipe;
-		}
-		if (pri_pipe->bottom_pipe && pri_pipe->bottom_pipe->next_odm_pipe) {
-			pri_pipe->bottom_pipe->next_odm_pipe->top_pipe = sec_pipe;
-			sec_pipe->bottom_pipe = pri_pipe->bottom_pipe->next_odm_pipe;
-		}
-		pri_pipe->next_odm_pipe = sec_pipe;
-		sec_pipe->prev_odm_pipe = pri_pipe;
-
-		if (!sec_pipe->top_pipe)
-			sec_pipe->stream_res.opp = pool->opps[pipe_idx];
-		else
-			sec_pipe->stream_res.opp = sec_pipe->top_pipe->stream_res.opp;
-		if (sec_pipe->stream->timing.flags.DSC == 1) {
-			dcn20_acquire_dsc(dc, res_ctx, &sec_pipe->stream_res.dsc, pipe_idx);
-			ASSERT(sec_pipe->stream_res.dsc);
-			if (sec_pipe->stream_res.dsc == NULL)
-				return false;
-		}
-	} else {
-		if (pri_pipe->bottom_pipe) {
-			ASSERT(pri_pipe->bottom_pipe != sec_pipe);
-			sec_pipe->bottom_pipe = pri_pipe->bottom_pipe;
-			sec_pipe->bottom_pipe->top_pipe = sec_pipe;
-		}
-		pri_pipe->bottom_pipe = sec_pipe;
-		sec_pipe->top_pipe = pri_pipe;
-
-		ASSERT(pri_pipe->plane_state);
-	}
-
-	return true;
-}
-
-static struct pipe_ctx *dcn30_find_split_pipe(
-		struct dc *dc,
-		struct dc_state *context,
-		int old_index)
-{
-	struct pipe_ctx *pipe = NULL;
-	int i;
-
-	if (old_index >= 0 && context->res_ctx.pipe_ctx[old_index].stream == NULL) {
-		pipe = &context->res_ctx.pipe_ctx[old_index];
-		pipe->pipe_idx = old_index;
-	}
-
-	if (!pipe)
-		for (i = dc->res_pool->pipe_count - 1; i >= 0; i--) {
-			if (dc->current_state->res_ctx.pipe_ctx[i].top_pipe == NULL
-					&& dc->current_state->res_ctx.pipe_ctx[i].prev_odm_pipe == NULL) {
-				if (context->res_ctx.pipe_ctx[i].stream == NULL) {
-					pipe = &context->res_ctx.pipe_ctx[i];
-					pipe->pipe_idx = i;
-					break;
-				}
-			}
-		}
-
-	/*
-	 * May need to fix pipes getting tossed from 1 opp to another on flip
-	 * Add for debugging transient underflow during topology updates:
-	 * ASSERT(pipe);
-	 */
-	if (!pipe)
-		for (i = dc->res_pool->pipe_count - 1; i >= 0; i--) {
-			if (context->res_ctx.pipe_ctx[i].stream == NULL) {
-				pipe = &context->res_ctx.pipe_ctx[i];
-				pipe->pipe_idx = i;
-				break;
-			}
-		}
-
-	return pipe;
-}
-
-noinline bool dcn30_internal_validate_bw(
-		struct dc *dc,
-		struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		int *pipe_cnt_out,
-		int *vlevel_out,
-		bool fast_validate,
-		bool allow_self_refresh_only)
-{
-	bool out = false;
-	bool repopulate_pipes = false;
-	int split[MAX_PIPES] = { 0 };
-	bool merge[MAX_PIPES] = { false };
-	bool newly_split[MAX_PIPES] = { false };
-	int pipe_cnt, i, pipe_idx, vlevel;
-	struct vba_vars_st *vba = &context->bw_ctx.dml.vba;
-
-	ASSERT(pipes);
-	if (!pipes)
-		return false;
-
-	context->bw_ctx.dml.vba.maxMpcComb = 0;
-	context->bw_ctx.dml.vba.VoltageLevel = 0;
-	context->bw_ctx.dml.vba.DRAMClockChangeSupport[0][0] = dm_dram_clock_change_vactive;
-	dc->res_pool->funcs->update_soc_for_wm_a(dc, context);
-	pipe_cnt = dc->res_pool->funcs->populate_dml_pipes(dc, context, pipes, fast_validate);
-
-	if (!pipe_cnt) {
-		out = true;
-		goto validate_out;
-	}
-
-	dml_log_pipe_params(&context->bw_ctx.dml, pipes, pipe_cnt);
-
-	if (!fast_validate || !allow_self_refresh_only) {
-		/*
-		 * DML favors voltage over p-state, but we're more interested in
-		 * supporting p-state over voltage. We can't support p-state in
-		 * prefetch mode > 0 so try capping the prefetch mode to start.
-		 */
-		context->bw_ctx.dml.soc.allow_dram_self_refresh_or_dram_clock_change_in_vblank =
-			dm_allow_self_refresh_and_mclk_switch;
-		vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
-		/* This may adjust vlevel and maxMpcComb */
-		if (vlevel < context->bw_ctx.dml.soc.num_states)
-			vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, vlevel, split, merge);
-	}
-	if (allow_self_refresh_only &&
-	    (fast_validate || vlevel == context->bw_ctx.dml.soc.num_states ||
-			vba->DRAMClockChangeSupport[vlevel][vba->maxMpcComb] == dm_dram_clock_change_unsupported)) {
-		/*
-		 * If mode is unsupported or there's still no p-state support
-		 * then fall back to favoring voltage.
-		 *
-		 * We don't actually support prefetch mode 2, so require that we
-		 * at least support prefetch mode 1.
-		 */
-		context->bw_ctx.dml.soc.allow_dram_self_refresh_or_dram_clock_change_in_vblank =
-			dm_allow_self_refresh;
-
-		vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
-		if (vlevel < context->bw_ctx.dml.soc.num_states) {
-			memset(split, 0, sizeof(split));
-			memset(merge, 0, sizeof(merge));
-			vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, vlevel, split, merge);
-		}
-	}
-
-	dml_log_mode_support_params(&context->bw_ctx.dml);
-
-	if (vlevel == context->bw_ctx.dml.soc.num_states)
-		goto validate_fail;
-
-	if (!dc->config.enable_windowed_mpo_odm) {
-		for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
-			struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-			struct pipe_ctx *mpo_pipe = pipe->bottom_pipe;
-
-			if (!pipe->stream)
-				continue;
-
-			/* We only support full screen mpo with ODM */
-			if (vba->ODMCombineEnabled[vba->pipe_plane[pipe_idx]] != dm_odm_combine_mode_disabled
-					&& pipe->plane_state && mpo_pipe
-					&& memcmp(&mpo_pipe->plane_state->clip_rect,
-							&pipe->stream->src,
-							sizeof(struct rect)) != 0) {
-				ASSERT(mpo_pipe->plane_state != pipe->plane_state);
-				goto validate_fail;
-			}
-			pipe_idx++;
-		}
-	}
-
-	/* merge pipes if necessary */
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-
-		/*skip pipes that don't need merging*/
-		if (!merge[i])
-			continue;
-
-		/* if ODM merge we ignore mpc tree, mpo pipes will have their own flags */
-		if (pipe->prev_odm_pipe) {
-			/*split off odm pipe*/
-			pipe->prev_odm_pipe->next_odm_pipe = pipe->next_odm_pipe;
-			if (pipe->next_odm_pipe)
-				pipe->next_odm_pipe->prev_odm_pipe = pipe->prev_odm_pipe;
-
-			pipe->bottom_pipe = NULL;
-			pipe->next_odm_pipe = NULL;
-			pipe->plane_state = NULL;
-			pipe->stream = NULL;
-			pipe->top_pipe = NULL;
-			pipe->prev_odm_pipe = NULL;
-			if (pipe->stream_res.dsc)
-				dcn20_release_dsc(&context->res_ctx, dc->res_pool, &pipe->stream_res.dsc);
-			memset(&pipe->plane_res, 0, sizeof(pipe->plane_res));
-			memset(&pipe->stream_res, 0, sizeof(pipe->stream_res));
-			repopulate_pipes = true;
-		} else if (pipe->top_pipe && pipe->top_pipe->plane_state == pipe->plane_state) {
-			struct pipe_ctx *top_pipe = pipe->top_pipe;
-			struct pipe_ctx *bottom_pipe = pipe->bottom_pipe;
-
-			top_pipe->bottom_pipe = bottom_pipe;
-			if (bottom_pipe)
-				bottom_pipe->top_pipe = top_pipe;
-
-			pipe->top_pipe = NULL;
-			pipe->bottom_pipe = NULL;
-			pipe->plane_state = NULL;
-			pipe->stream = NULL;
-			memset(&pipe->plane_res, 0, sizeof(pipe->plane_res));
-			memset(&pipe->stream_res, 0, sizeof(pipe->stream_res));
-			repopulate_pipes = true;
-		} else
-			ASSERT(0); /* Should never try to merge master pipe */
-
-	}
-
-	for (i = 0, pipe_idx = -1; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-		struct pipe_ctx *old_pipe = &dc->current_state->res_ctx.pipe_ctx[i];
-		struct pipe_ctx *hsplit_pipe = NULL;
-		bool odm;
-		int old_index = -1;
-
-		if (!pipe->stream || newly_split[i])
-			continue;
-
-		pipe_idx++;
-		odm = vba->ODMCombineEnabled[vba->pipe_plane[pipe_idx]] != dm_odm_combine_mode_disabled;
-
-		if (!pipe->plane_state && !odm)
-			continue;
-
-		if (split[i]) {
-			if (odm) {
-				if (split[i] == 4 && old_pipe->next_odm_pipe && old_pipe->next_odm_pipe->next_odm_pipe)
-					old_index = old_pipe->next_odm_pipe->next_odm_pipe->pipe_idx;
-				else if (old_pipe->next_odm_pipe)
-					old_index = old_pipe->next_odm_pipe->pipe_idx;
-			} else {
-				if (split[i] == 4 && old_pipe->bottom_pipe && old_pipe->bottom_pipe->bottom_pipe &&
-						old_pipe->bottom_pipe->bottom_pipe->plane_state == old_pipe->plane_state)
-					old_index = old_pipe->bottom_pipe->bottom_pipe->pipe_idx;
-				else if (old_pipe->bottom_pipe &&
-						old_pipe->bottom_pipe->plane_state == old_pipe->plane_state)
-					old_index = old_pipe->bottom_pipe->pipe_idx;
-			}
-			hsplit_pipe = dcn30_find_split_pipe(dc, context, old_index);
-			ASSERT(hsplit_pipe);
-			if (!hsplit_pipe)
-				goto validate_fail;
-
-			if (!dcn30_split_stream_for_mpc_or_odm(
-					dc, &context->res_ctx,
-					pipe, hsplit_pipe, odm))
-				goto validate_fail;
-
-			newly_split[hsplit_pipe->pipe_idx] = true;
-			repopulate_pipes = true;
-		}
-		if (split[i] == 4) {
-			struct pipe_ctx *pipe_4to1;
-
-			if (odm && old_pipe->next_odm_pipe)
-				old_index = old_pipe->next_odm_pipe->pipe_idx;
-			else if (!odm && old_pipe->bottom_pipe &&
-						old_pipe->bottom_pipe->plane_state == old_pipe->plane_state)
-				old_index = old_pipe->bottom_pipe->pipe_idx;
-			else
-				old_index = -1;
-			pipe_4to1 = dcn30_find_split_pipe(dc, context, old_index);
-			ASSERT(pipe_4to1);
-			if (!pipe_4to1)
-				goto validate_fail;
-			if (!dcn30_split_stream_for_mpc_or_odm(
-					dc, &context->res_ctx,
-					pipe, pipe_4to1, odm))
-				goto validate_fail;
-			newly_split[pipe_4to1->pipe_idx] = true;
-
-			if (odm && old_pipe->next_odm_pipe && old_pipe->next_odm_pipe->next_odm_pipe
-					&& old_pipe->next_odm_pipe->next_odm_pipe->next_odm_pipe)
-				old_index = old_pipe->next_odm_pipe->next_odm_pipe->next_odm_pipe->pipe_idx;
-			else if (!odm && old_pipe->bottom_pipe && old_pipe->bottom_pipe->bottom_pipe &&
-					old_pipe->bottom_pipe->bottom_pipe->bottom_pipe &&
-					old_pipe->bottom_pipe->bottom_pipe->bottom_pipe->plane_state == old_pipe->plane_state)
-				old_index = old_pipe->bottom_pipe->bottom_pipe->bottom_pipe->pipe_idx;
-			else
-				old_index = -1;
-			pipe_4to1 = dcn30_find_split_pipe(dc, context, old_index);
-			ASSERT(pipe_4to1);
-			if (!pipe_4to1)
-				goto validate_fail;
-			if (!dcn30_split_stream_for_mpc_or_odm(
-					dc, &context->res_ctx,
-					hsplit_pipe, pipe_4to1, odm))
-				goto validate_fail;
-			newly_split[pipe_4to1->pipe_idx] = true;
-		}
-		if (odm)
-			dcn20_build_mapped_resource(dc, context, pipe->stream);
-	}
-
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-
-		if (pipe->plane_state) {
-			if (!resource_build_scaling_params(pipe))
-				goto validate_fail;
-		}
-	}
-
-	/* Actual dsc count per stream dsc validation*/
-	if (!dcn20_validate_dsc(dc, context)) {
-		vba->ValidationStatus[vba->soc.num_states] = DML_FAIL_DSC_VALIDATION_FAILURE;
-		goto validate_fail;
-	}
-
-	if (repopulate_pipes)
-		pipe_cnt = dc->res_pool->funcs->populate_dml_pipes(dc, context, pipes, fast_validate);
-	context->bw_ctx.dml.vba.VoltageLevel = vlevel;
-	*vlevel_out = vlevel;
-	*pipe_cnt_out = pipe_cnt;
-
-	out = true;
-	goto validate_out;
-
-validate_fail:
-	out = false;
-
-validate_out:
-	return out;
-}
-
-static int get_refresh_rate(struct dc_state *context)
-{
-	int refresh_rate = 0;
-	int h_v_total = 0;
-	struct dc_crtc_timing *timing = NULL;
-
-	if (context == NULL || context->streams[0] == NULL)
-		return 0;
-
-	/* check if refresh rate at least 120hz */
-	timing = &context->streams[0]->timing;
-	if (timing == NULL)
-		return 0;
-
-	h_v_total = timing->h_total * timing->v_total;
-	if (h_v_total == 0)
-		return 0;
-
-	refresh_rate = ((timing->pix_clk_100hz * 100) / (h_v_total)) + 1;
-	return refresh_rate;
-}
-
-#define MAX_STRETCHED_V_BLANK 500 // in micro-seconds
-/*
- * Scaling factor for v_blank stretch calculations considering timing in
- * micro-seconds and pixel clock in 100hz.
- * Note: the parenthesis are necessary to ensure the correct order of
- * operation where V_SCALE is used.
- */
-#define V_SCALE (10000 / MAX_STRETCHED_V_BLANK)
-
-static int get_frame_rate_at_max_stretch_100hz(struct dc_state *context)
-{
-	struct dc_crtc_timing *timing = NULL;
-	uint32_t sec_per_100_lines;
-	uint32_t max_v_blank;
-	uint32_t curr_v_blank;
-	uint32_t v_stretch_max;
-	uint32_t stretched_frame_pix_cnt;
-	uint32_t scaled_stretched_frame_pix_cnt;
-	uint32_t scaled_refresh_rate;
-
-	if (context == NULL || context->streams[0] == NULL)
-		return 0;
-
-	/* check if refresh rate at least 120hz */
-	timing = &context->streams[0]->timing;
-	if (timing == NULL)
-		return 0;
-
-	sec_per_100_lines = timing->pix_clk_100hz / timing->h_total + 1;
-	max_v_blank = sec_per_100_lines / V_SCALE + 1;
-	curr_v_blank = timing->v_total - timing->v_addressable;
-	v_stretch_max = (max_v_blank > curr_v_blank) ? (max_v_blank - curr_v_blank) : (0);
-	stretched_frame_pix_cnt = (v_stretch_max + timing->v_total) * timing->h_total;
-	scaled_stretched_frame_pix_cnt = stretched_frame_pix_cnt / 10000;
-	scaled_refresh_rate = (timing->pix_clk_100hz) / scaled_stretched_frame_pix_cnt + 1;
-
-	return scaled_refresh_rate;
-}
-
-static bool is_refresh_rate_support_mclk_switch_using_fw_based_vblank_stretch(struct dc_state *context)
-{
-	int refresh_rate_max_stretch_100hz;
-	int min_refresh_100hz;
-
-	if (context == NULL || context->streams[0] == NULL)
-		return false;
-
-	refresh_rate_max_stretch_100hz = get_frame_rate_at_max_stretch_100hz(context);
-	min_refresh_100hz = context->streams[0]->timing.min_refresh_in_uhz / 10000;
-
-	if (refresh_rate_max_stretch_100hz < min_refresh_100hz)
-		return false;
-
-	return true;
-}
-
-bool dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch(struct dc *dc, struct dc_state *context)
-{
-	int refresh_rate = 0;
-	const int minimum_refreshrate_supported = 120;
-
-	if (context == NULL || context->streams[0] == NULL)
-		return false;
-
-	if (context->streams[0]->sink->edid_caps.panel_patch.disable_fams)
-		return false;
-
-	if (dc->debug.disable_fams)
-		return false;
-
-	if (!dc->caps.dmub_caps.mclk_sw)
-		return false;
-
-	if (context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching_shut_down)
-		return false;
-
-	/* more then 1 monitor connected */
-	if (context->stream_count != 1)
-		return false;
-
-	refresh_rate = get_refresh_rate(context);
-	if (refresh_rate < minimum_refreshrate_supported)
-		return false;
-
-	if (!is_refresh_rate_support_mclk_switch_using_fw_based_vblank_stretch(context))
-		return false;
-
-	if (!context->streams[0]->allow_freesync)
-		return false;
-
-	if (context->streams[0]->vrr_active_variable && dc->debug.disable_fams_gaming)
-		return false;
-
-	context->streams[0]->fpo_in_use = true;
-
-	return true;
-}
-
-/*
- * set up FPO watermarks, pstate, dram latency
- */
-void dcn30_setup_mclk_switch_using_fw_based_vblank_stretch(struct dc *dc, struct dc_state *context)
-{
-	ASSERT(dc != NULL && context != NULL);
-	if (dc == NULL || context == NULL)
-		return;
-
-	/* Set wm_a.pstate so high natural MCLK switches are impossible: 4 seconds */
-	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns = 4U * 1000U * 1000U * 1000U;
-}
-
-void dcn30_update_soc_for_wm_a(struct dc *dc, struct dc_state *context)
-{
-	DC_FP_START();
-	dcn30_fpu_update_soc_for_wm_a(dc, context);
-	DC_FP_END();
-}
-
-void dcn30_calculate_wm_and_dlg(
-		struct dc *dc, struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		int pipe_cnt,
-		int vlevel)
-{
-	DC_FP_START();
-	dcn30_fpu_calculate_wm_and_dlg(dc, context, pipes, pipe_cnt, vlevel);
-	DC_FP_END();
-}
-
-bool dcn30_validate_bandwidth(struct dc *dc,
-		struct dc_state *context,
-		bool fast_validate)
-{
-	bool out = false;
-
-	BW_VAL_TRACE_SETUP();
-
-	int vlevel = 0;
-	int pipe_cnt = 0;
-	display_e2e_pipe_params_st *pipes = kzalloc(dc->res_pool->pipe_count * sizeof(display_e2e_pipe_params_st), GFP_KERNEL);
-	DC_LOGGER_INIT(dc->ctx->logger);
-
-	BW_VAL_TRACE_COUNT();
-
-	DC_FP_START();
-	out = dcn30_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, fast_validate, true);
-	DC_FP_END();
-
-	if (pipe_cnt == 0)
-		goto validate_out;
-
-	if (!out)
-		goto validate_fail;
-
-	BW_VAL_TRACE_END_VOLTAGE_LEVEL();
-
-	if (fast_validate) {
-		BW_VAL_TRACE_SKIP(fast);
-		goto validate_out;
-	}
-
-	DC_FP_START();
-	if (dc->res_pool->funcs->calculate_wm_and_dlg)
-		dc->res_pool->funcs->calculate_wm_and_dlg(dc, context, pipes, pipe_cnt, vlevel);
-	DC_FP_END();
-
-	BW_VAL_TRACE_END_WATERMARKS();
-
-	goto validate_out;
-
-validate_fail:
-	DC_LOG_WARNING("Mode Validation Warning: %s failed validation.\n",
-		dml_get_status_message(context->bw_ctx.dml.vba.ValidationStatus[context->bw_ctx.dml.vba.soc.num_states]));
-
-	BW_VAL_TRACE_SKIP(fail);
-	out = false;
-
-validate_out:
-	kfree(pipes);
-
-	BW_VAL_TRACE_FINISH();
-
-	return out;
-}
-
-void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
-{
-	unsigned int i, j;
-	unsigned int num_states = 0;
-
-	unsigned int dcfclk_mhz[DC__VOLTAGE_STATES] = {0};
-	unsigned int dram_speed_mts[DC__VOLTAGE_STATES] = {0};
-	unsigned int optimal_uclk_for_dcfclk_sta_targets[DC__VOLTAGE_STATES] = {0};
-	unsigned int optimal_dcfclk_for_uclk[DC__VOLTAGE_STATES] = {0};
-
-	unsigned int dcfclk_sta_targets[DC__VOLTAGE_STATES] = {694, 875, 1000, 1200};
-	unsigned int num_dcfclk_sta_targets = 4;
-	unsigned int num_uclk_states;
-
-	struct dc_bounding_box_max_clk dcn30_bb_max_clk;
-
-	memset(&dcn30_bb_max_clk, 0, sizeof(dcn30_bb_max_clk));
-
-	if (dc->ctx->dc_bios->vram_info.num_chans)
-		dcn3_0_soc.num_chans = dc->ctx->dc_bios->vram_info.num_chans;
-
-	DC_FP_START();
-	dcn30_fpu_update_dram_channel_width_bytes(dc);
-	DC_FP_END();
-
-	if (bw_params->clk_table.entries[0].memclk_mhz) {
-
-		for (i = 0; i < MAX_NUM_DPM_LVL; i++) {
-			if (bw_params->clk_table.entries[i].dcfclk_mhz > dcn30_bb_max_clk.max_dcfclk_mhz)
-				dcn30_bb_max_clk.max_dcfclk_mhz = bw_params->clk_table.entries[i].dcfclk_mhz;
-			if (bw_params->clk_table.entries[i].dispclk_mhz > dcn30_bb_max_clk.max_dispclk_mhz)
-				dcn30_bb_max_clk.max_dispclk_mhz = bw_params->clk_table.entries[i].dispclk_mhz;
-			if (bw_params->clk_table.entries[i].dppclk_mhz > dcn30_bb_max_clk.max_dppclk_mhz)
-				dcn30_bb_max_clk.max_dppclk_mhz = bw_params->clk_table.entries[i].dppclk_mhz;
-			if (bw_params->clk_table.entries[i].phyclk_mhz > dcn30_bb_max_clk.max_phyclk_mhz)
-				dcn30_bb_max_clk.max_phyclk_mhz = bw_params->clk_table.entries[i].phyclk_mhz;
-		}
-
-		DC_FP_START();
-		dcn30_fpu_update_max_clk(&dcn30_bb_max_clk);
-		DC_FP_END();
-
-		if (dcn30_bb_max_clk.max_dcfclk_mhz > dcfclk_sta_targets[num_dcfclk_sta_targets-1]) {
-			// If max DCFCLK is greater than the max DCFCLK STA target, insert into the DCFCLK STA target array
-			dcfclk_sta_targets[num_dcfclk_sta_targets] = dcn30_bb_max_clk.max_dcfclk_mhz;
-			num_dcfclk_sta_targets++;
-		} else if (dcn30_bb_max_clk.max_dcfclk_mhz < dcfclk_sta_targets[num_dcfclk_sta_targets-1]) {
-			// If max DCFCLK is less than the max DCFCLK STA target, cap values and remove duplicates
-			for (i = 0; i < num_dcfclk_sta_targets; i++) {
-				if (dcfclk_sta_targets[i] > dcn30_bb_max_clk.max_dcfclk_mhz) {
-					dcfclk_sta_targets[i] = dcn30_bb_max_clk.max_dcfclk_mhz;
-					break;
-				}
-			}
-			// Update size of array since we "removed" duplicates
-			num_dcfclk_sta_targets = i + 1;
-		}
-
-		num_uclk_states = bw_params->clk_table.num_entries;
-
-		// Calculate optimal dcfclk for each uclk
-		for (i = 0; i < num_uclk_states; i++) {
-			DC_FP_START();
-			dcn30_fpu_get_optimal_dcfclk_fclk_for_uclk(bw_params->clk_table.entries[i].memclk_mhz * 16,
-					&optimal_dcfclk_for_uclk[i], NULL);
-			DC_FP_END();
-			if (optimal_dcfclk_for_uclk[i] < bw_params->clk_table.entries[0].dcfclk_mhz) {
-				optimal_dcfclk_for_uclk[i] = bw_params->clk_table.entries[0].dcfclk_mhz;
-			}
-		}
-
-		// Calculate optimal uclk for each dcfclk sta target
-		for (i = 0; i < num_dcfclk_sta_targets; i++) {
-			for (j = 0; j < num_uclk_states; j++) {
-				if (dcfclk_sta_targets[i] < optimal_dcfclk_for_uclk[j]) {
-					optimal_uclk_for_dcfclk_sta_targets[i] =
-							bw_params->clk_table.entries[j].memclk_mhz * 16;
-					break;
-				}
-			}
-		}
-
-		i = 0;
-		j = 0;
-		// create the final dcfclk and uclk table
-		while (i < num_dcfclk_sta_targets && j < num_uclk_states && num_states < DC__VOLTAGE_STATES) {
-			if (dcfclk_sta_targets[i] < optimal_dcfclk_for_uclk[j] && i < num_dcfclk_sta_targets) {
-				dcfclk_mhz[num_states] = dcfclk_sta_targets[i];
-				dram_speed_mts[num_states++] = optimal_uclk_for_dcfclk_sta_targets[i++];
-			} else {
-				if (j < num_uclk_states && optimal_dcfclk_for_uclk[j] <= dcn30_bb_max_clk.max_dcfclk_mhz) {
-					dcfclk_mhz[num_states] = optimal_dcfclk_for_uclk[j];
-					dram_speed_mts[num_states++] = bw_params->clk_table.entries[j++].memclk_mhz * 16;
-				} else {
-					j = num_uclk_states;
-				}
-			}
-		}
-
-		while (i < num_dcfclk_sta_targets && num_states < DC__VOLTAGE_STATES) {
-			dcfclk_mhz[num_states] = dcfclk_sta_targets[i];
-			dram_speed_mts[num_states++] = optimal_uclk_for_dcfclk_sta_targets[i++];
-		}
-
-		while (j < num_uclk_states && num_states < DC__VOLTAGE_STATES &&
-				optimal_dcfclk_for_uclk[j] <= dcn30_bb_max_clk.max_dcfclk_mhz) {
-			dcfclk_mhz[num_states] = optimal_dcfclk_for_uclk[j];
-			dram_speed_mts[num_states++] = bw_params->clk_table.entries[j++].memclk_mhz * 16;
-		}
-
-		dcn3_0_soc.num_states = num_states;
-		DC_FP_START();
-		dcn30_fpu_update_bw_bounding_box(dc, bw_params, &dcn30_bb_max_clk, dcfclk_mhz, dram_speed_mts);
-		DC_FP_END();
-	}
-}
-
-static void dcn30_get_panel_config_defaults(struct dc_panel_config *panel_config)
-{
-	*panel_config = panel_config_defaults;
-}
-
-static const struct resource_funcs dcn30_res_pool_funcs = {
-	.destroy = dcn30_destroy_resource_pool,
-	.link_enc_create = dcn30_link_encoder_create,
-	.panel_cntl_create = dcn30_panel_cntl_create,
-	.validate_bandwidth = dcn30_validate_bandwidth,
-	.calculate_wm_and_dlg = dcn30_calculate_wm_and_dlg,
-	.update_soc_for_wm_a = dcn30_update_soc_for_wm_a,
-	.populate_dml_pipes = dcn30_populate_dml_pipes_from_context,
-	.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
-	.release_pipe = dcn20_release_pipe,
-	.add_stream_to_ctx = dcn30_add_stream_to_ctx,
-	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
-	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
-	.populate_dml_writeback_from_context = dcn30_populate_dml_writeback_from_context,
-	.set_mcif_arb_params = dcn30_set_mcif_arb_params,
-	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
-	.acquire_post_bldn_3dlut = dcn30_acquire_post_bldn_3dlut,
-	.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
-	.update_bw_bounding_box = dcn30_update_bw_bounding_box,
-	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
-	.get_panel_config_defaults = dcn30_get_panel_config_defaults,
-};
-
-#define CTX ctx
-
-#define REG(reg_name) \
-	(DCN_BASE.instance[0].segment[mm ## reg_name ## _BASE_IDX] + mm ## reg_name)
-
-static uint32_t read_pipe_fuses(struct dc_context *ctx)
-{
-	uint32_t value = REG_READ(CC_DC_PIPE_DIS);
-	/* Support for max 6 pipes */
-	value = value & 0x3f;
-	return value;
-}
-
-static bool dcn30_resource_construct(
-	uint8_t num_virtual_links,
-	struct dc *dc,
-	struct dcn30_resource_pool *pool)
-{
-	int i;
-	struct dc_context *ctx = dc->ctx;
-	struct irq_service_init_data init_data;
-	struct ddc_service_init_data ddc_init_data = {0};
-	uint32_t pipe_fuses = read_pipe_fuses(ctx);
-	uint32_t num_pipes = 0;
-
-	if (!(pipe_fuses == 0 || pipe_fuses == 0x3e)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: Unexpected fuse recipe for navi2x !\n");
-		/* fault to single pipe */
-		pipe_fuses = 0x3e;
-	}
-
-	DC_FP_START();
-
-	ctx->dc_bios->regs = &bios_regs;
-
-	pool->base.res_cap = &res_cap_dcn3;
-
-	pool->base.funcs = &dcn30_res_pool_funcs;
-
-	/*************************************************
-	 *  Resource + asic cap harcoding                *
-	 *************************************************/
-	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
-	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
-	pool->base.mpcc_count = pool->base.res_cap->num_timing_generator;
-	dc->caps.max_downscale_ratio = 600;
-	dc->caps.i2c_speed_in_khz = 100;
-	dc->caps.i2c_speed_in_khz_hdcp = 100; /*1.4 w/a not applied by default*/
-	dc->caps.max_cursor_size = 256;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.dmdata_alloc_size = 2048;
-	dc->caps.mall_size_per_mem_channel = 8;
-	/* total size = mall per channel * num channels * 1024 * 1024 */
-	dc->caps.mall_size_total = dc->caps.mall_size_per_mem_channel * dc->ctx->dc_bios->vram_info.num_chans * 1048576;
-	dc->caps.cursor_cache_size = dc->caps.max_cursor_size * dc->caps.max_cursor_size * 8;
-
-	dc->caps.max_slave_planes = 2;
-	dc->caps.max_slave_yuv_planes = 2;
-	dc->caps.max_slave_rgb_planes = 2;
-	dc->caps.post_blend_color_processing = true;
-	dc->caps.force_dp_tps4_for_cp2520 = true;
-	dc->caps.extended_aux_timeout_support = true;
-	dc->caps.dmcub_support = true;
-
-	/* Color pipeline capabilities */
-	dc->caps.color.dpp.dcn_arch = 1;
-	dc->caps.color.dpp.input_lut_shared = 0;
-	dc->caps.color.dpp.icsc = 1;
-	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
-	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
-	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
-	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
-	dc->caps.color.dpp.post_csc = 1;
-	dc->caps.color.dpp.gamma_corr = 1;
-	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
-
-	dc->caps.color.dpp.hw_3d_lut = 1;
-	dc->caps.color.dpp.ogam_ram = 1;
-	// no OGAM ROM on DCN3
-	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
-	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
-	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
-	dc->caps.color.dpp.ocsc = 0;
-
-	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //3
-	dc->caps.color.mpc.ogam_ram = 1;
-	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
-	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
-	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
-	dc->caps.color.mpc.ocsc = 1;
-
-	dc->caps.dp_hdmi21_pcon_support = true;
-	dc->caps.max_v_total = (1 << 15) - 1;
-
-	/* read VBIOS LTTPR caps */
-	{
-		if (ctx->dc_bios->funcs->get_lttpr_caps) {
-			enum bp_result bp_query_result;
-			uint8_t is_vbios_lttpr_enable = 0;
-
-			bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
-			dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
-		}
-
-		if (ctx->dc_bios->funcs->get_lttpr_interop) {
-			enum bp_result bp_query_result;
-			uint8_t is_vbios_interop_enabled = 0;
-
-			bp_query_result = ctx->dc_bios->funcs->get_lttpr_interop(ctx->dc_bios,
-					&is_vbios_interop_enabled);
-			dc->caps.vbios_lttpr_aware = (bp_query_result == BP_RESULT_OK) && !!is_vbios_interop_enabled;
-		}
-	}
-
-	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
-		dc->debug = debug_defaults_drv;
-
-	// Init the vm_helper
-	if (dc->vm_helper)
-		vm_helper_init(dc->vm_helper, 16);
-
-	/*************************************************
-	 *  Create resources                             *
-	 *************************************************/
-
-	/* Clock Sources for Pixel Clock*/
-	pool->base.clock_sources[DCN30_CLK_SRC_PLL0] =
-			dcn30_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL0,
-				&clk_src_regs[0], false);
-	pool->base.clock_sources[DCN30_CLK_SRC_PLL1] =
-			dcn30_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL1,
-				&clk_src_regs[1], false);
-	pool->base.clock_sources[DCN30_CLK_SRC_PLL2] =
-			dcn30_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL2,
-				&clk_src_regs[2], false);
-	pool->base.clock_sources[DCN30_CLK_SRC_PLL3] =
-			dcn30_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL3,
-				&clk_src_regs[3], false);
-	pool->base.clock_sources[DCN30_CLK_SRC_PLL4] =
-			dcn30_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL4,
-				&clk_src_regs[4], false);
-	pool->base.clock_sources[DCN30_CLK_SRC_PLL5] =
-			dcn30_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL5,
-				&clk_src_regs[5], false);
-
-	pool->base.clk_src_count = DCN30_CLK_SRC_TOTAL;
-
-	/* todo: not reuse phy_pll registers */
-	pool->base.dp_clock_source =
-			dcn30_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_ID_DP_DTO,
-				&clk_src_regs[0], true);
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-
-	/* DCCG */
-	pool->base.dccg = dccg30_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
-	if (pool->base.dccg == NULL) {
-		dm_error("DC: failed to create dccg!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	/* PP Lib and SMU interfaces */
-	init_soc_bounding_box(dc, pool);
-
-	num_pipes = dcn3_0_ip.max_num_dpp;
-
-	for (i = 0; i < dcn3_0_ip.max_num_dpp; i++)
-		if (pipe_fuses & 1 << i)
-			num_pipes--;
-
-	dcn3_0_ip.max_num_dpp = num_pipes;
-	dcn3_0_ip.max_num_otg = num_pipes;
-
-	dml_init_instance(&dc->dml, &dcn3_0_soc, &dcn3_0_ip, DML_PROJECT_DCN30);
-
-	/* IRQ */
-	init_data.ctx = dc->ctx;
-	pool->base.irqs = dal_irq_service_dcn30_create(&init_data);
-	if (!pool->base.irqs)
-		goto create_fail;
-
-	/* HUBBUB */
-	pool->base.hubbub = dcn30_hubbub_create(ctx);
-	if (pool->base.hubbub == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create hubbub!\n");
-		goto create_fail;
-	}
-
-	/* HUBPs, DPPs, OPPs and TGs */
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		pool->base.hubps[i] = dcn30_hubp_create(ctx, i);
-		if (pool->base.hubps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create hubps!\n");
-			goto create_fail;
-		}
-
-		pool->base.dpps[i] = dcn30_dpp_create(ctx, i);
-		if (pool->base.dpps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create dpps!\n");
-			goto create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
-		pool->base.opps[i] = dcn30_opp_create(ctx, i);
-		if (pool->base.opps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create output pixel processor!\n");
-			goto create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		pool->base.timing_generators[i] = dcn30_timing_generator_create(
-				ctx, i);
-		if (pool->base.timing_generators[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create tg!\n");
-			goto create_fail;
-		}
-	}
-	pool->base.timing_generator_count = i;
-	/* PSR */
-	pool->base.psr = dmub_psr_create(ctx);
-
-	if (pool->base.psr == NULL) {
-		dm_error("DC: failed to create PSR obj!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	/* ABM */
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		pool->base.multiple_abms[i] = dmub_abm_create(ctx,
-				&abm_regs[i],
-				&abm_shift,
-				&abm_mask);
-		if (pool->base.multiple_abms[i] == NULL) {
-			dm_error("DC: failed to create abm for pipe %d!\n", i);
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-	/* MPC and DSC */
-	pool->base.mpc = dcn30_mpc_create(ctx, pool->base.mpcc_count, pool->base.res_cap->num_mpc_3dlut);
-	if (pool->base.mpc == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mpc!\n");
-		goto create_fail;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		pool->base.dscs[i] = dcn30_dsc_create(ctx, i);
-		if (pool->base.dscs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create display stream compressor %d!\n", i);
-			goto create_fail;
-		}
-	}
-
-	/* DWB and MMHUBBUB */
-	if (!dcn30_dwbc_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create dwbc!\n");
-		goto create_fail;
-	}
-
-	if (!dcn30_mmhubbub_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mcif_wb!\n");
-		goto create_fail;
-	}
-
-	/* AUX and I2C */
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		pool->base.engines[i] = dcn30_aux_engine_create(ctx, i);
-		if (pool->base.engines[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create aux engine!!\n");
-			goto create_fail;
-		}
-		pool->base.hw_i2cs[i] = dcn30_i2c_hw_create(ctx, i);
-		if (pool->base.hw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create hw i2c!!\n");
-			goto create_fail;
-		}
-		pool->base.sw_i2cs[i] = NULL;
-	}
-
-	/* Audio, Stream Encoders including DIG and virtual, MPC 3D LUTs */
-	if (!resource_construct(num_virtual_links, dc, &pool->base,
-			&res_create_funcs))
-		goto create_fail;
-
-	/* HW Sequencer and Plane caps */
-	dcn30_hw_sequencer_construct(dc);
-
-	dc->caps.max_planes =  pool->base.pipe_count;
-
-	for (i = 0; i < dc->caps.max_planes; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	dc->cap_funcs = cap_funcs;
-
-	if (dc->ctx->dc_bios->fw_info.oem_i2c_present) {
-		ddc_init_data.ctx = dc->ctx;
-		ddc_init_data.link = NULL;
-		ddc_init_data.id.id = dc->ctx->dc_bios->fw_info.oem_i2c_obj_id;
-		ddc_init_data.id.enum_id = 0;
-		ddc_init_data.id.type = OBJECT_TYPE_GENERIC;
-		pool->base.oem_device = dc->link_srv->create_ddc_service(&ddc_init_data);
-	} else {
-		pool->base.oem_device = NULL;
-	}
-
-	DC_FP_END();
-
-	return true;
-
-create_fail:
-
-	DC_FP_END();
-	dcn30_resource_destruct(pool);
-
-	return false;
-}
-
-struct resource_pool *dcn30_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc)
-{
-	struct dcn30_resource_pool *pool =
-		kzalloc(sizeof(struct dcn30_resource_pool), GFP_KERNEL);
-
-	if (!pool)
-		return NULL;
-
-	if (dcn30_resource_construct(init_data->num_virtual_links, dc, pool))
-		return &pool->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(pool);
-	return NULL;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.h b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.h
deleted file mode 100644
index 8e6b8b7368fd..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.h
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Copyright 2020 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#ifndef _DCN30_RESOURCE_H_
-#define _DCN30_RESOURCE_H_
-
-#include "core_types.h"
-
-#define TO_DCN30_RES_POOL(pool)\
-	container_of(pool, struct dcn30_resource_pool, base)
-
-struct dc;
-struct resource_pool;
-struct _vcs_dpi_display_pipe_params_st;
-
-extern struct _vcs_dpi_ip_params_st dcn3_0_ip;
-extern struct _vcs_dpi_soc_bounding_box_st dcn3_0_soc;
-
-struct dcn30_resource_pool {
-	struct resource_pool base;
-};
-struct resource_pool *dcn30_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc);
-
-void dcn30_set_mcif_arb_params(
-		struct dc *dc,
-		struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		int pipe_cnt);
-
-unsigned int dcn30_calc_max_scaled_time(
-		unsigned int time_per_pixel,
-		enum mmhubbub_wbif_mode mode,
-		unsigned int urgent_watermark);
-
-bool dcn30_validate_bandwidth(struct dc *dc, struct dc_state *context,
-		bool fast_validate);
-bool dcn30_internal_validate_bw(
-		struct dc *dc,
-		struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		int *pipe_cnt_out,
-		int *vlevel_out,
-		bool fast_validate,
-		bool allow_self_refresh_only);
-void dcn30_calculate_wm_and_dlg(
-		struct dc *dc, struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		int pipe_cnt,
-		int vlevel);
-void dcn30_update_soc_for_wm_a(struct dc *dc, struct dc_state *context);
-void dcn30_populate_dml_writeback_from_context(
-		struct dc *dc, struct resource_context *res_ctx, display_e2e_pipe_params_st *pipes);
-
-int dcn30_populate_dml_pipes_from_context(
-	struct dc *dc, struct dc_state *context,
-	display_e2e_pipe_params_st *pipes,
-	bool fast_validate);
-
-bool dcn30_acquire_post_bldn_3dlut(
-		struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		int mpcc_id,
-		struct dc_3dlut **lut,
-		struct dc_transfer_func **shaper);
-
-bool dcn30_release_post_bldn_3dlut(
-		struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		struct dc_3dlut **lut,
-		struct dc_transfer_func **shaper);
-
-enum dc_status dcn30_add_stream_to_ctx(
-		struct dc *dc,
-		struct dc_state *new_ctx,
-		struct dc_stream_state *dc_stream);
-
-void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params);
-
-bool dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch(struct dc *dc, struct dc_state *context);
-void dcn30_setup_mclk_switch_using_fw_based_vblank_stretch(struct dc *dc, struct dc_state *context);
-int dcn30_find_dummy_latency_index_for_fw_based_mclk_switch(struct dc *dc, struct dc_state *context,
-		display_e2e_pipe_params_st *pipes, int pipe_cnt, int vlevel);
-
-#endif /* _DCN30_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn301/Makefile b/drivers/gpu/drm/amd/display/dc/dcn301/Makefile
index 30fbc5e06dca..cbf59d7e78c4 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn301/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn301/Makefile
@@ -10,7 +10,7 @@
 #
 # Makefile for dcn30.
 
-DCN301 = dcn301_init.o dcn301_resource.o dcn301_dccg.o \
+DCN301 = dcn301_init.o dcn301_dccg.o \
 		dcn301_dio_link_encoder.o dcn301_panel_cntl.o dcn301_hubbub.o \
 		dcn301_optc.o
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c
deleted file mode 100644
index f3b75f283aa2..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c
+++ /dev/null
@@ -1,1728 +0,0 @@
-/*
- * Copyright 2019-2021 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-
-#include "dm_services.h"
-#include "dc.h"
-
-#include "dcn301_init.h"
-
-#include "resource.h"
-#include "include/irq_service_interface.h"
-#include "dcn30/dcn30_resource.h"
-#include "dcn301_resource.h"
-
-#include "dcn20/dcn20_resource.h"
-
-#include "dcn10/dcn10_ipp.h"
-#include "dcn301/dcn301_hubbub.h"
-#include "dcn30/dcn30_mpc.h"
-#include "dcn30/dcn30_hubp.h"
-#include "irq/dcn30/irq_service_dcn30.h"
-#include "dcn30/dcn30_dpp.h"
-#include "dcn301/dcn301_optc.h"
-#include "dcn20/dcn20_hwseq.h"
-#include "dcn30/dcn30_hwseq.h"
-#include "dce110/dce110_hwseq.h"
-#include "dcn30/dcn30_opp.h"
-#include "dcn20/dcn20_dsc.h"
-#include "dcn30/dcn30_vpg.h"
-#include "dcn30/dcn30_afmt.h"
-#include "dce/dce_clock_source.h"
-#include "dce/dce_audio.h"
-#include "dce/dce_hwseq.h"
-#include "clk_mgr.h"
-#include "virtual/virtual_stream_encoder.h"
-#include "dce110/dce110_resource.h"
-#include "dml/display_mode_vba.h"
-#include "dcn301/dcn301_dccg.h"
-#include "dcn10/dcn10_resource.h"
-#include "dcn30/dcn30_dio_stream_encoder.h"
-#include "dcn301/dcn301_dio_link_encoder.h"
-#include "dcn301_panel_cntl.h"
-
-#include "vangogh_ip_offset.h"
-
-#include "dcn30/dcn30_dwb.h"
-#include "dcn30/dcn30_mmhubbub.h"
-
-#include "dcn/dcn_3_0_1_offset.h"
-#include "dcn/dcn_3_0_1_sh_mask.h"
-
-#include "nbio/nbio_7_2_0_offset.h"
-
-#include "dpcs/dpcs_3_0_0_offset.h"
-#include "dpcs/dpcs_3_0_0_sh_mask.h"
-
-#include "reg_helper.h"
-#include "dce/dmub_abm.h"
-#include "dce/dce_aux.h"
-#include "dce/dce_i2c.h"
-
-#include "dml/dcn30/dcn30_fpu.h"
-
-#include "dml/dcn30/display_mode_vba_30.h"
-#include "dml/dcn301/dcn301_fpu.h"
-#include "vm_helper.h"
-#include "dcn20/dcn20_vmid.h"
-#include "amdgpu_socbb.h"
-
-#define TO_DCN301_RES_POOL(pool)\
-	container_of(pool, struct dcn301_resource_pool, base)
-
-#define DC_LOGGER \
-	dc->ctx->logger
-#define DC_LOGGER_INIT(logger)
-
-enum dcn301_clk_src_array_id {
-	DCN301_CLK_SRC_PLL0,
-	DCN301_CLK_SRC_PLL1,
-	DCN301_CLK_SRC_PLL2,
-	DCN301_CLK_SRC_PLL3,
-	DCN301_CLK_SRC_TOTAL
-};
-
-/* begin *********************
- * macros to expend register list macro defined in HW object header file
- */
-
-/* DCN */
-#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
-
-#define BASE(seg) BASE_INNER(seg)
-
-#define SR(reg_name)\
-		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) +  \
-					mm ## reg_name
-
-#define SRI(reg_name, block, id)\
-	.reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define SRI2(reg_name, block, id)\
-	.reg_name = BASE(mm ## reg_name ## _BASE_IDX) + \
-					mm ## reg_name
-
-#define SRIR(var_name, reg_name, block, id)\
-	.var_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define SRII2(reg_name_pre, reg_name_post, id)\
-	.reg_name_pre ## _ ##  reg_name_post[id] = BASE(mm ## reg_name_pre \
-			## id ## _ ## reg_name_post ## _BASE_IDX) + \
-			mm ## reg_name_pre ## id ## _ ## reg_name_post
-
-#define SRII_MPC_RMU(reg_name, block, id)\
-	.RMU##_##reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define SRII_DWB(reg_name, temp_name, block, id)\
-	.reg_name[id] = BASE(mm ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## temp_name
-
-#define SF_DWB2(reg_name, block, id, field_name, post_fix)	\
-	.field_name = reg_name ## __ ## field_name ## post_fix
-
-#define DCCG_SRII(reg_name, block, id)\
-	.block ## _ ## reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## id ## _ ## reg_name
-
-#define VUPDATE_SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(mm ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
-					mm ## reg_name ## _ ## block ## id
-
-/* NBIO */
-#define NBIO_BASE_INNER(seg) \
-	NBIO_BASE__INST0_SEG ## seg
-
-#define NBIO_BASE(seg) \
-	NBIO_BASE_INNER(seg)
-
-#define NBIO_SR(reg_name)\
-		.reg_name = NBIO_BASE(regBIF_BX0_ ## reg_name ## _BASE_IDX) + \
-					regBIF_BX0_ ## reg_name
-
-/* MMHUB */
-#define MMHUB_BASE_INNER(seg) \
-	MMHUB_BASE__INST0_SEG ## seg
-
-#define MMHUB_BASE(seg) \
-	MMHUB_BASE_INNER(seg)
-
-#define MMHUB_SR(reg_name)\
-		.reg_name = MMHUB_BASE(regMM ## reg_name ## _BASE_IDX) + \
-					regMM ## reg_name
-
-/* CLOCK */
-#define CLK_BASE_INNER(seg) \
-	CLK_BASE__INST0_SEG ## seg
-
-#define CLK_BASE(seg) \
-	CLK_BASE_INNER(seg)
-
-#define CLK_SRI(reg_name, block, inst)\
-	.reg_name = CLK_BASE(mm ## block ## _ ## inst ## _ ## reg_name ## _BASE_IDX) + \
-					mm ## block ## _ ## inst ## _ ## reg_name
-
-static const struct bios_registers bios_regs = {
-		NBIO_SR(BIOS_SCRATCH_3),
-		NBIO_SR(BIOS_SCRATCH_6)
-};
-
-#define clk_src_regs(index, pllid)\
-[index] = {\
-	CS_COMMON_REG_LIST_DCN3_01(index, pllid),\
-}
-
-static const struct dce110_clk_src_regs clk_src_regs[] = {
-	clk_src_regs(0, A),
-	clk_src_regs(1, B),
-	clk_src_regs(2, C),
-	clk_src_regs(3, D)
-};
-
-static const struct dce110_clk_src_shift cs_shift = {
-		CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
-};
-
-static const struct dce110_clk_src_mask cs_mask = {
-		CS_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
-};
-
-#define abm_regs(id)\
-[id] = {\
-		ABM_DCN301_REG_LIST(id)\
-}
-
-static const struct dce_abm_registers abm_regs[] = {
-		abm_regs(0),
-		abm_regs(1),
-		abm_regs(2),
-		abm_regs(3),
-};
-
-static const struct dce_abm_shift abm_shift = {
-		ABM_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dce_abm_mask abm_mask = {
-		ABM_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define audio_regs(id)\
-[id] = {\
-		AUD_COMMON_REG_LIST(id)\
-}
-
-static const struct dce_audio_registers audio_regs[] = {
-	audio_regs(0),
-	audio_regs(1),
-	audio_regs(2),
-	audio_regs(3),
-	audio_regs(4),
-	audio_regs(5),
-	audio_regs(6)
-};
-
-#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
-		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
-
-static const struct dce_audio_shift audio_shift = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_audio_mask audio_mask = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
-};
-
-#define vpg_regs(id)\
-[id] = {\
-	VPG_DCN3_REG_LIST(id)\
-}
-
-static const struct dcn30_vpg_registers vpg_regs[] = {
-	vpg_regs(0),
-	vpg_regs(1),
-	vpg_regs(2),
-	vpg_regs(3),
-};
-
-static const struct dcn30_vpg_shift vpg_shift = {
-	DCN3_VPG_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn30_vpg_mask vpg_mask = {
-	DCN3_VPG_MASK_SH_LIST(_MASK)
-};
-
-#define afmt_regs(id)\
-[id] = {\
-	AFMT_DCN3_REG_LIST(id)\
-}
-
-static const struct dcn30_afmt_registers afmt_regs[] = {
-	afmt_regs(0),
-	afmt_regs(1),
-	afmt_regs(2),
-	afmt_regs(3),
-};
-
-static const struct dcn30_afmt_shift afmt_shift = {
-	DCN3_AFMT_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn30_afmt_mask afmt_mask = {
-	DCN3_AFMT_MASK_SH_LIST(_MASK)
-};
-
-#define stream_enc_regs(id)\
-[id] = {\
-	SE_DCN3_REG_LIST(id)\
-}
-
-static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
-	stream_enc_regs(0),
-	stream_enc_regs(1),
-	stream_enc_regs(2),
-	stream_enc_regs(3),
-};
-
-static const struct dcn10_stream_encoder_shift se_shift = {
-		SE_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn10_stream_encoder_mask se_mask = {
-		SE_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-
-#define aux_regs(id)\
-[id] = {\
-	DCN2_AUX_REG_LIST(id)\
-}
-
-static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
-		aux_regs(0),
-		aux_regs(1),
-		aux_regs(2),
-		aux_regs(3),
-};
-
-#define hpd_regs(id)\
-[id] = {\
-	HPD_REG_LIST(id)\
-}
-
-static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
-		hpd_regs(0),
-		hpd_regs(1),
-		hpd_regs(2),
-		hpd_regs(3),
-};
-
-
-#define link_regs(id, phyid)\
-[id] = {\
-	LE_DCN301_REG_LIST(id), \
-	UNIPHY_DCN2_REG_LIST(phyid), \
-	DPCS_DCN2_REG_LIST(id), \
-	SRI(DP_DPHY_INTERNAL_CTRL, DP, id) \
-}
-
-static const struct dce110_aux_registers_shift aux_shift = {
-	DCN_AUX_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce110_aux_registers_mask aux_mask = {
-	DCN_AUX_MASK_SH_LIST(_MASK)
-};
-
-static const struct dcn10_link_enc_registers link_enc_regs[] = {
-	link_regs(0, A),
-	link_regs(1, B),
-	link_regs(2, C),
-	link_regs(3, D),
-};
-
-static const struct dcn10_link_enc_shift le_shift = {
-	LINK_ENCODER_MASK_SH_LIST_DCN301(__SHIFT),\
-	DPCS_DCN2_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn10_link_enc_mask le_mask = {
-	LINK_ENCODER_MASK_SH_LIST_DCN301(_MASK),\
-	DPCS_DCN2_MASK_SH_LIST(_MASK)
-};
-
-#define panel_cntl_regs(id)\
-[id] = {\
-	DCN301_PANEL_CNTL_REG_LIST(id),\
-}
-
-static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
-	panel_cntl_regs(0),
-	panel_cntl_regs(1),
-};
-
-static const struct dcn301_panel_cntl_shift panel_cntl_shift = {
-	DCN301_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn301_panel_cntl_mask panel_cntl_mask = {
-	DCN301_PANEL_CNTL_MASK_SH_LIST(_MASK)
-};
-
-#define dpp_regs(id)\
-[id] = {\
-	DPP_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn3_dpp_registers dpp_regs[] = {
-	dpp_regs(0),
-	dpp_regs(1),
-	dpp_regs(2),
-	dpp_regs(3),
-};
-
-static const struct dcn3_dpp_shift tf_shift = {
-		DPP_REG_LIST_SH_MASK_DCN30(__SHIFT)
-};
-
-static const struct dcn3_dpp_mask tf_mask = {
-		DPP_REG_LIST_SH_MASK_DCN30(_MASK)
-};
-
-#define opp_regs(id)\
-[id] = {\
-	OPP_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn20_opp_registers opp_regs[] = {
-	opp_regs(0),
-	opp_regs(1),
-	opp_regs(2),
-	opp_regs(3),
-};
-
-static const struct dcn20_opp_shift opp_shift = {
-	OPP_MASK_SH_LIST_DCN20(__SHIFT)
-};
-
-static const struct dcn20_opp_mask opp_mask = {
-	OPP_MASK_SH_LIST_DCN20(_MASK)
-};
-
-#define aux_engine_regs(id)\
-[id] = {\
-	AUX_COMMON_REG_LIST0(id), \
-	.AUXN_IMPCAL = 0, \
-	.AUXP_IMPCAL = 0, \
-	.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
-}
-
-static const struct dce110_aux_registers aux_engine_regs[] = {
-		aux_engine_regs(0),
-		aux_engine_regs(1),
-		aux_engine_regs(2),
-		aux_engine_regs(3),
-};
-
-#define dwbc_regs_dcn3(id)\
-[id] = {\
-	DWBC_COMMON_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn30_dwbc_registers dwbc30_regs[] = {
-	dwbc_regs_dcn3(0),
-};
-
-static const struct dcn30_dwbc_shift dwbc30_shift = {
-	DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_dwbc_mask dwbc30_mask = {
-	DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define mcif_wb_regs_dcn3(id)\
-[id] = {\
-	MCIF_WB_COMMON_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn30_mmhubbub_registers mcif_wb30_regs[] = {
-	mcif_wb_regs_dcn3(0)
-};
-
-static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
-	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
-	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define dsc_regsDCN20(id)\
-[id] = {\
-	DSC_REG_LIST_DCN20(id)\
-}
-
-static const struct dcn20_dsc_registers dsc_regs[] = {
-	dsc_regsDCN20(0),
-	dsc_regsDCN20(1),
-	dsc_regsDCN20(2),
-};
-
-static const struct dcn20_dsc_shift dsc_shift = {
-	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
-};
-
-static const struct dcn20_dsc_mask dsc_mask = {
-	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
-};
-
-static const struct dcn30_mpc_registers mpc_regs = {
-		MPC_REG_LIST_DCN3_0(0),
-		MPC_REG_LIST_DCN3_0(1),
-		MPC_REG_LIST_DCN3_0(2),
-		MPC_REG_LIST_DCN3_0(3),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(0),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(1),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(2),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(3),
-		MPC_RMU_GLOBAL_REG_LIST_DCN3AG,
-		MPC_RMU_REG_LIST_DCN3AG(0),
-		MPC_RMU_REG_LIST_DCN3AG(1),
-		MPC_DWB_MUX_REG_LIST_DCN3_0(0),
-};
-
-static const struct dcn30_mpc_shift mpc_shift = {
-	MPC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_mpc_mask mpc_mask = {
-	MPC_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define optc_regs(id)\
-[id] = {OPTC_COMMON_REG_LIST_DCN3_0(id)}
-
-
-static const struct dcn_optc_registers optc_regs[] = {
-	optc_regs(0),
-	optc_regs(1),
-	optc_regs(2),
-	optc_regs(3),
-};
-
-static const struct dcn_optc_shift optc_shift = {
-	OPTC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn_optc_mask optc_mask = {
-	OPTC_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define hubp_regs(id)\
-[id] = {\
-	HUBP_REG_LIST_DCN30(id)\
-}
-
-static const struct dcn_hubp2_registers hubp_regs[] = {
-		hubp_regs(0),
-		hubp_regs(1),
-		hubp_regs(2),
-		hubp_regs(3),
-};
-
-static const struct dcn_hubp2_shift hubp_shift = {
-		HUBP_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn_hubp2_mask hubp_mask = {
-		HUBP_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static const struct dcn_hubbub_registers hubbub_reg = {
-		HUBBUB_REG_LIST_DCN301(0)
-};
-
-static const struct dcn_hubbub_shift hubbub_shift = {
-		HUBBUB_MASK_SH_LIST_DCN301(__SHIFT)
-};
-
-static const struct dcn_hubbub_mask hubbub_mask = {
-		HUBBUB_MASK_SH_LIST_DCN301(_MASK)
-};
-
-static const struct dccg_registers dccg_regs = {
-		DCCG_REG_LIST_DCN301()
-};
-
-static const struct dccg_shift dccg_shift = {
-		DCCG_MASK_SH_LIST_DCN301(__SHIFT)
-};
-
-static const struct dccg_mask dccg_mask = {
-		DCCG_MASK_SH_LIST_DCN301(_MASK)
-};
-
-static const struct dce_hwseq_registers hwseq_reg = {
-		HWSEQ_DCN301_REG_LIST()
-};
-
-static const struct dce_hwseq_shift hwseq_shift = {
-		HWSEQ_DCN301_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_hwseq_mask hwseq_mask = {
-		HWSEQ_DCN301_MASK_SH_LIST(_MASK)
-};
-#define vmid_regs(id)\
-[id] = {\
-		DCN20_VMID_REG_LIST(id)\
-}
-
-static const struct dcn_vmid_registers vmid_regs[] = {
-	vmid_regs(0),
-	vmid_regs(1),
-	vmid_regs(2),
-	vmid_regs(3),
-	vmid_regs(4),
-	vmid_regs(5),
-	vmid_regs(6),
-	vmid_regs(7),
-	vmid_regs(8),
-	vmid_regs(9),
-	vmid_regs(10),
-	vmid_regs(11),
-	vmid_regs(12),
-	vmid_regs(13),
-	vmid_regs(14),
-	vmid_regs(15)
-};
-
-static const struct dcn20_vmid_shift vmid_shifts = {
-		DCN20_VMID_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn20_vmid_mask vmid_masks = {
-		DCN20_VMID_MASK_SH_LIST(_MASK)
-};
-
-static struct resource_caps res_cap_dcn301 = {
-	.num_timing_generator = 4,
-	.num_opp = 4,
-	.num_video_plane = 4,
-	.num_audio = 4,
-	.num_stream_encoder = 4,
-	.num_pll = 4,
-	.num_dwb = 1,
-	.num_ddc = 4,
-	.num_vmid = 16,
-	.num_mpc_3dlut = 2,
-	.num_dsc = 3,
-};
-
-static const struct dc_plane_cap plane_cap = {
-	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.per_pixel_alpha = true,
-
-	.pixel_format_support = {
-			.argb8888 = true,
-			.nv12 = true,
-			.fp16 = true,
-			.p010 = true,
-			.ayuv = false,
-	},
-
-	.max_upscale_factor = {
-			.argb8888 = 16000,
-			.nv12 = 16000,
-			.fp16 = 16000
-	},
-
-	/* 6:1 downscaling ratio: 1000/6 = 166.666 */
-	.max_downscale_factor = {
-			.argb8888 = 167,
-			.nv12 = 167,
-			.fp16 = 167 
-	},
-	64,
-	64
-};
-
-static const struct dc_debug_options debug_defaults_drv = {
-	.disable_dmcu = true,
-	.force_abm_enable = false,
-	.timing_trace = false,
-	.clock_trace = true,
-	.disable_dpp_power_gate = false,
-	.disable_hubp_power_gate = false,
-	.disable_clock_gate = true,
-	.disable_pplib_clock_request = true,
-	.disable_pplib_wm_range = true,
-	.pipe_split_policy = MPC_SPLIT_DYNAMIC,
-	.force_single_disp_pipe_split = false,
-	.disable_dcc = DCC_ENABLE,
-	.vsr_support = true,
-	.performance_trace = false,
-	.max_downscale_src_width = 7680,/*upto 8K*/
-	.scl_reset_length10 = true,
-	.sanity_checks = false,
-	.underflow_assert_delay_us = 0xFFFFFFFF,
-	.dwb_fi_phase = -1, // -1 = disable
-	.dmub_command_table = true,
-	.use_max_lb = false,
-	.exit_idle_opt_for_cursor_updates = true,
-	.using_dml2 = false,
-};
-
-static void dcn301_dpp_destroy(struct dpp **dpp)
-{
-	kfree(TO_DCN20_DPP(*dpp));
-	*dpp = NULL;
-}
-
-static struct dpp *dcn301_dpp_create(struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn3_dpp *dpp =
-		kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
-
-	if (!dpp)
-		return NULL;
-
-	if (dpp3_construct(dpp, ctx, inst,
-			&dpp_regs[inst], &tf_shift, &tf_mask))
-		return &dpp->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(dpp);
-	return NULL;
-}
-static struct output_pixel_processor *dcn301_opp_create(struct dc_context *ctx,
-							uint32_t inst)
-{
-	struct dcn20_opp *opp =
-		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
-
-	if (!opp) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dcn20_opp_construct(opp, ctx, inst,
-			&opp_regs[inst], &opp_shift, &opp_mask);
-	return &opp->base;
-}
-
-static struct dce_aux *dcn301_aux_engine_create(struct dc_context *ctx, uint32_t inst)
-{
-	struct aux_engine_dce110 *aux_engine =
-		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
-
-	if (!aux_engine)
-		return NULL;
-
-	dce110_aux_engine_construct(aux_engine, ctx, inst,
-				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
-				    &aux_engine_regs[inst],
-					&aux_mask,
-					&aux_shift,
-					ctx->dc->caps.extended_aux_timeout_support);
-
-	return &aux_engine->base;
-}
-#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
-
-static const struct dce_i2c_registers i2c_hw_regs[] = {
-		i2c_inst_regs(1),
-		i2c_inst_regs(2),
-		i2c_inst_regs(3),
-		i2c_inst_regs(4),
-};
-
-static const struct dce_i2c_shift i2c_shifts = {
-		I2C_COMMON_MASK_SH_LIST_DCN2(__SHIFT)
-};
-
-static const struct dce_i2c_mask i2c_masks = {
-		I2C_COMMON_MASK_SH_LIST_DCN2(_MASK)
-};
-
-static struct dce_i2c_hw *dcn301_i2c_hw_create(struct dc_context *ctx, uint32_t inst)
-{
-	struct dce_i2c_hw *dce_i2c_hw =
-		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
-
-	if (!dce_i2c_hw)
-		return NULL;
-
-	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
-				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
-
-	return dce_i2c_hw;
-}
-static struct mpc *dcn301_mpc_create(
-		struct dc_context *ctx,
-		int num_mpcc,
-		int num_rmu)
-{
-	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc),
-					  GFP_KERNEL);
-
-	if (!mpc30)
-		return NULL;
-
-	dcn30_mpc_construct(mpc30, ctx,
-			&mpc_regs,
-			&mpc_shift,
-			&mpc_mask,
-			num_mpcc,
-			num_rmu);
-
-	return &mpc30->base;
-}
-
-static struct hubbub *dcn301_hubbub_create(struct dc_context *ctx)
-{
-	int i;
-
-	struct dcn20_hubbub *hubbub3 = kzalloc(sizeof(struct dcn20_hubbub),
-					  GFP_KERNEL);
-
-	if (!hubbub3)
-		return NULL;
-
-	hubbub301_construct(hubbub3, ctx,
-			&hubbub_reg,
-			&hubbub_shift,
-			&hubbub_mask);
-
-
-	for (i = 0; i < res_cap_dcn301.num_vmid; i++) {
-		struct dcn20_vmid *vmid = &hubbub3->vmid[i];
-
-		vmid->ctx = ctx;
-
-		vmid->regs = &vmid_regs[i];
-		vmid->shifts = &vmid_shifts;
-		vmid->masks = &vmid_masks;
-	}
-
-	hubbub3->num_vmid = res_cap_dcn301.num_vmid;
-
-	return &hubbub3->base;
-}
-
-static struct timing_generator *dcn301_timing_generator_create(
-	struct dc_context *ctx, uint32_t instance)
-{
-	struct optc *tgn10 =
-		kzalloc(sizeof(struct optc), GFP_KERNEL);
-
-	if (!tgn10)
-		return NULL;
-
-	tgn10->base.inst = instance;
-	tgn10->base.ctx = ctx;
-
-	tgn10->tg_regs = &optc_regs[instance];
-	tgn10->tg_shift = &optc_shift;
-	tgn10->tg_mask = &optc_mask;
-
-	dcn301_timing_generator_init(tgn10);
-
-	return &tgn10->base;
-}
-
-static const struct encoder_feature_support link_enc_feature = {
-		.max_hdmi_deep_color = COLOR_DEPTH_121212,
-		.max_hdmi_pixel_clock = 600000,
-		.hdmi_ycbcr420_supported = true,
-		.dp_ycbcr420_supported = true,
-		.fec_supported = true,
-		.flags.bits.IS_HBR2_CAPABLE = true,
-		.flags.bits.IS_HBR3_CAPABLE = true,
-		.flags.bits.IS_TPS3_CAPABLE = true,
-		.flags.bits.IS_TPS4_CAPABLE = true
-};
-
-static struct link_encoder *dcn301_link_encoder_create(
-	struct dc_context *ctx,
-	const struct encoder_init_data *enc_init_data)
-{
-	struct dcn20_link_encoder *enc20 =
-		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
-
-	if (!enc20)
-		return NULL;
-
-	dcn301_link_encoder_construct(enc20,
-			enc_init_data,
-			&link_enc_feature,
-			&link_enc_regs[enc_init_data->transmitter],
-			&link_enc_aux_regs[enc_init_data->channel - 1],
-			&link_enc_hpd_regs[enc_init_data->hpd_source],
-			&le_shift,
-			&le_mask);
-
-	return &enc20->enc10.base;
-}
-
-static struct panel_cntl *dcn301_panel_cntl_create(const struct panel_cntl_init_data *init_data)
-{
-	struct dcn301_panel_cntl *panel_cntl =
-		kzalloc(sizeof(struct dcn301_panel_cntl), GFP_KERNEL);
-
-	if (!panel_cntl)
-		return NULL;
-
-	dcn301_panel_cntl_construct(panel_cntl,
-			init_data,
-			&panel_cntl_regs[init_data->inst],
-			&panel_cntl_shift,
-			&panel_cntl_mask);
-
-	return &panel_cntl->base;
-}
-
-
-#define CTX ctx
-
-#define REG(reg_name) \
-	(DCN_BASE.instance[0].segment[mm ## reg_name ## _BASE_IDX] + mm ## reg_name)
-
-static uint32_t read_pipe_fuses(struct dc_context *ctx)
-{
-	uint32_t value = REG_READ(CC_DC_PIPE_DIS);
-	/* RV1 support max 4 pipes */
-	value = value & 0xf;
-	return value;
-}
-
-
-static void read_dce_straps(
-	struct dc_context *ctx,
-	struct resource_straps *straps)
-{
-	generic_reg_get(ctx, mmDC_PINSTRAPS + BASE(mmDC_PINSTRAPS_BASE_IDX),
-		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
-
-}
-
-static struct audio *dcn301_create_audio(
-		struct dc_context *ctx, unsigned int inst)
-{
-	return dce_audio_create(ctx, inst,
-			&audio_regs[inst], &audio_shift, &audio_mask);
-}
-
-static struct vpg *dcn301_vpg_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn30_vpg *vpg3 = kzalloc(sizeof(struct dcn30_vpg), GFP_KERNEL);
-
-	if (!vpg3)
-		return NULL;
-
-	vpg3_construct(vpg3, ctx, inst,
-			&vpg_regs[inst],
-			&vpg_shift,
-			&vpg_mask);
-
-	return &vpg3->base;
-}
-
-static struct afmt *dcn301_afmt_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn30_afmt *afmt3 = kzalloc(sizeof(struct dcn30_afmt), GFP_KERNEL);
-
-	if (!afmt3)
-		return NULL;
-
-	afmt3_construct(afmt3, ctx, inst,
-			&afmt_regs[inst],
-			&afmt_shift,
-			&afmt_mask);
-
-	return &afmt3->base;
-}
-
-static struct stream_encoder *dcn301_stream_encoder_create(enum engine_id eng_id,
-							   struct dc_context *ctx)
-{
-	struct dcn10_stream_encoder *enc1;
-	struct vpg *vpg;
-	struct afmt *afmt;
-	int vpg_inst;
-	int afmt_inst;
-
-	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
-	if (eng_id <= ENGINE_ID_DIGF) {
-		vpg_inst = eng_id;
-		afmt_inst = eng_id;
-	} else
-		return NULL;
-
-	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
-	vpg = dcn301_vpg_create(ctx, vpg_inst);
-	afmt = dcn301_afmt_create(ctx, afmt_inst);
-
-	if (!enc1 || !vpg || !afmt) {
-		kfree(enc1);
-		kfree(vpg);
-		kfree(afmt);
-		return NULL;
-	}
-
-	dcn30_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
-					eng_id, vpg, afmt,
-					&stream_enc_regs[eng_id],
-					&se_shift, &se_mask);
-
-	return &enc1->base;
-}
-
-static struct dce_hwseq *dcn301_hwseq_create(struct dc_context *ctx)
-{
-	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
-
-	if (hws) {
-		hws->ctx = ctx;
-		hws->regs = &hwseq_reg;
-		hws->shifts = &hwseq_shift;
-		hws->masks = &hwseq_mask;
-	}
-	return hws;
-}
-static const struct resource_create_funcs res_create_funcs = {
-	.read_dce_straps = read_dce_straps,
-	.create_audio = dcn301_create_audio,
-	.create_stream_encoder = dcn301_stream_encoder_create,
-	.create_hwseq = dcn301_hwseq_create,
-};
-
-static void dcn301_destruct(struct dcn301_resource_pool *pool)
-{
-	unsigned int i;
-
-	for (i = 0; i < pool->base.stream_enc_count; i++) {
-		if (pool->base.stream_enc[i] != NULL) {
-			if (pool->base.stream_enc[i]->vpg != NULL) {
-				kfree(DCN30_VPG_FROM_VPG(pool->base.stream_enc[i]->vpg));
-				pool->base.stream_enc[i]->vpg = NULL;
-			}
-			if (pool->base.stream_enc[i]->afmt != NULL) {
-				kfree(DCN30_AFMT_FROM_AFMT(pool->base.stream_enc[i]->afmt));
-				pool->base.stream_enc[i]->afmt = NULL;
-			}
-			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
-			pool->base.stream_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		if (pool->base.dscs[i] != NULL)
-			dcn20_dsc_destroy(&pool->base.dscs[i]);
-	}
-
-	if (pool->base.mpc != NULL) {
-		kfree(TO_DCN20_MPC(pool->base.mpc));
-		pool->base.mpc = NULL;
-	}
-	if (pool->base.hubbub != NULL) {
-		kfree(pool->base.hubbub);
-		pool->base.hubbub = NULL;
-	}
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		if (pool->base.dpps[i] != NULL)
-			dcn301_dpp_destroy(&pool->base.dpps[i]);
-
-		if (pool->base.ipps[i] != NULL)
-			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
-
-		if (pool->base.hubps[i] != NULL) {
-			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
-			pool->base.hubps[i] = NULL;
-		}
-
-		if (pool->base.irqs != NULL) {
-			dal_irq_service_destroy(&pool->base.irqs);
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		if (pool->base.engines[i] != NULL)
-			dce110_engine_destroy(&pool->base.engines[i]);
-		if (pool->base.hw_i2cs[i] != NULL) {
-			kfree(pool->base.hw_i2cs[i]);
-			pool->base.hw_i2cs[i] = NULL;
-		}
-		if (pool->base.sw_i2cs[i] != NULL) {
-			kfree(pool->base.sw_i2cs[i]);
-			pool->base.sw_i2cs[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
-		if (pool->base.opps[i] != NULL)
-			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		if (pool->base.timing_generators[i] != NULL)	{
-			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
-			pool->base.timing_generators[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
-		if (pool->base.dwbc[i] != NULL) {
-			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
-			pool->base.dwbc[i] = NULL;
-		}
-		if (pool->base.mcif_wb[i] != NULL) {
-			kfree(TO_DCN30_MMHUBBUB(pool->base.mcif_wb[i]));
-			pool->base.mcif_wb[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.audio_count; i++) {
-		if (pool->base.audios[i])
-			dce_aud_destroy(&pool->base.audios[i]);
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] != NULL) {
-			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
-			pool->base.clock_sources[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
-		if (pool->base.mpc_lut[i] != NULL) {
-			dc_3dlut_func_release(pool->base.mpc_lut[i]);
-			pool->base.mpc_lut[i] = NULL;
-		}
-		if (pool->base.mpc_shaper[i] != NULL) {
-			dc_transfer_func_release(pool->base.mpc_shaper[i]);
-			pool->base.mpc_shaper[i] = NULL;
-		}
-	}
-
-	if (pool->base.dp_clock_source != NULL) {
-		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
-		pool->base.dp_clock_source = NULL;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		if (pool->base.multiple_abms[i] != NULL)
-			dce_abm_destroy(&pool->base.multiple_abms[i]);
-	}
-
-	if (pool->base.dccg != NULL)
-		dcn_dccg_destroy(&pool->base.dccg);
-}
-
-static struct hubp *dcn301_hubp_create(struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_hubp *hubp2 =
-		kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
-
-	if (!hubp2)
-		return NULL;
-
-	if (hubp3_construct(hubp2, ctx, inst,
-			&hubp_regs[inst], &hubp_shift, &hubp_mask))
-		return &hubp2->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(hubp2);
-	return NULL;
-}
-
-static bool dcn301_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t pipe_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < pipe_count; i++) {
-		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc),
-						    GFP_KERNEL);
-
-		if (!dwbc30) {
-			dm_error("DC: failed to create dwbc30!\n");
-			return false;
-		}
-
-		dcn30_dwbc_construct(dwbc30, ctx,
-				&dwbc30_regs[i],
-				&dwbc30_shift,
-				&dwbc30_mask,
-				i);
-
-		pool->dwbc[i] = &dwbc30->base;
-	}
-	return true;
-}
-
-static bool dcn301_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t pipe_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < pipe_count; i++) {
-		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub),
-						    GFP_KERNEL);
-
-		if (!mcif_wb30) {
-			dm_error("DC: failed to create mcif_wb30!\n");
-			return false;
-		}
-
-		dcn30_mmhubbub_construct(mcif_wb30, ctx,
-				&mcif_wb30_regs[i],
-				&mcif_wb30_shift,
-				&mcif_wb30_mask,
-				i);
-
-		pool->mcif_wb[i] = &mcif_wb30->base;
-	}
-	return true;
-}
-
-static struct display_stream_compressor *dcn301_dsc_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_dsc *dsc =
-		kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
-
-	if (!dsc) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
-	return &dsc->base;
-}
-
-
-static void dcn301_destroy_resource_pool(struct resource_pool **pool)
-{
-	struct dcn301_resource_pool *dcn301_pool = TO_DCN301_RES_POOL(*pool);
-
-	dcn301_destruct(dcn301_pool);
-	kfree(dcn301_pool);
-	*pool = NULL;
-}
-
-static struct clock_source *dcn301_clock_source_create(
-		struct dc_context *ctx,
-		struct dc_bios *bios,
-		enum clock_source_id id,
-		const struct dce110_clk_src_regs *regs,
-		bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src =
-		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dcn301_clk_src_construct(clk_src, ctx, bios, id,
-			regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-
-	kfree(clk_src);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
-static struct dc_cap_funcs cap_funcs = {
-	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
-};
-
-
-static bool is_soc_bounding_box_valid(struct dc *dc)
-{
-	uint32_t hw_internal_rev = dc->ctx->asic_id.hw_internal_rev;
-
-	if (ASICREV_IS_VANGOGH(hw_internal_rev))
-		return true;
-
-	return false;
-}
-
-static bool init_soc_bounding_box(struct dc *dc,
-				  struct dcn301_resource_pool *pool)
-{
-	struct _vcs_dpi_soc_bounding_box_st *loaded_bb = &dcn3_01_soc;
-	struct _vcs_dpi_ip_params_st *loaded_ip = &dcn3_01_ip;
-
-	DC_LOGGER_INIT(dc->ctx->logger);
-
-	if (!is_soc_bounding_box_valid(dc)) {
-		DC_LOG_ERROR("%s: not valid soc bounding box\n", __func__);
-		return false;
-	}
-
-	loaded_ip->max_num_otg = pool->base.res_cap->num_timing_generator;
-	loaded_ip->max_num_dpp = pool->base.pipe_count;
-	DC_FP_START();
-	dcn20_patch_bounding_box(dc, loaded_bb);
-	DC_FP_END();
-
-	if (dc->ctx->dc_bios->funcs->get_soc_bb_info) {
-		struct bp_soc_bb_info bb_info = {0};
-
-		if (dc->ctx->dc_bios->funcs->get_soc_bb_info(dc->ctx->dc_bios, &bb_info) == BP_RESULT_OK) {
-			DC_FP_START();
-			dcn301_fpu_init_soc_bounding_box(bb_info);
-			DC_FP_END();
-		}
-	}
-
-	return true;
-}
-
-
-static void set_wm_ranges(
-		struct pp_smu_funcs *pp_smu,
-		struct _vcs_dpi_soc_bounding_box_st *loaded_bb)
-{
-	struct pp_smu_wm_range_sets ranges = {0};
-	int i;
-
-	ranges.num_reader_wm_sets = 0;
-
-	if (loaded_bb->num_states == 1) {
-		ranges.reader_wm_sets[0].wm_inst = 0;
-		ranges.reader_wm_sets[0].min_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
-		ranges.reader_wm_sets[0].max_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
-		ranges.reader_wm_sets[0].min_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
-		ranges.reader_wm_sets[0].max_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
-
-		ranges.num_reader_wm_sets = 1;
-	} else if (loaded_bb->num_states > 1) {
-		for (i = 0; i < 4 && i < loaded_bb->num_states; i++) {
-			ranges.reader_wm_sets[i].wm_inst = i;
-			ranges.reader_wm_sets[i].min_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
-			ranges.reader_wm_sets[i].max_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
-			DC_FP_START();
-			dcn301_fpu_set_wm_ranges(i, &ranges, loaded_bb);
-			DC_FP_END();
-			ranges.num_reader_wm_sets = i + 1;
-		}
-
-		ranges.reader_wm_sets[0].min_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
-		ranges.reader_wm_sets[ranges.num_reader_wm_sets - 1].max_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
-	}
-
-	ranges.num_writer_wm_sets = 1;
-
-	ranges.writer_wm_sets[0].wm_inst = 0;
-	ranges.writer_wm_sets[0].min_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
-	ranges.writer_wm_sets[0].max_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
-	ranges.writer_wm_sets[0].min_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
-	ranges.writer_wm_sets[0].max_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
-
-	/* Notify PP Lib/SMU which Watermarks to use for which clock ranges */
-	pp_smu->nv_funcs.set_wm_ranges(&pp_smu->nv_funcs.pp_smu, &ranges);
-}
-
-static void dcn301_calculate_wm_and_dlg(
-		struct dc *dc, struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		int pipe_cnt,
-		int vlevel)
-{
-	DC_FP_START();
-	dcn301_calculate_wm_and_dlg_fp(dc, context, pipes, pipe_cnt, vlevel);
-	DC_FP_END();
-}
-
-static struct resource_funcs dcn301_res_pool_funcs = {
-	.destroy = dcn301_destroy_resource_pool,
-	.link_enc_create = dcn301_link_encoder_create,
-	.panel_cntl_create = dcn301_panel_cntl_create,
-	.validate_bandwidth = dcn30_validate_bandwidth,
-	.calculate_wm_and_dlg = dcn301_calculate_wm_and_dlg,
-	.update_soc_for_wm_a = dcn30_update_soc_for_wm_a,
-	.populate_dml_pipes = dcn30_populate_dml_pipes_from_context,
-	.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
-	.release_pipe = dcn20_release_pipe,
-	.add_stream_to_ctx = dcn30_add_stream_to_ctx,
-	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
-	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
-	.populate_dml_writeback_from_context = dcn30_populate_dml_writeback_from_context,
-	.set_mcif_arb_params = dcn30_set_mcif_arb_params,
-	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
-	.acquire_post_bldn_3dlut = dcn30_acquire_post_bldn_3dlut,
-	.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
-	.update_bw_bounding_box = dcn301_update_bw_bounding_box,
-	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state
-};
-
-static bool dcn301_resource_construct(
-	uint8_t num_virtual_links,
-	struct dc *dc,
-	struct dcn301_resource_pool *pool)
-{
-	int i, j;
-	struct dc_context *ctx = dc->ctx;
-	struct irq_service_init_data init_data;
-	uint32_t pipe_fuses = read_pipe_fuses(ctx);
-	uint32_t num_pipes = 0;
-
-	DC_LOGGER_INIT(dc->ctx->logger);
-
-	ctx->dc_bios->regs = &bios_regs;
-
-	if (dc->ctx->asic_id.chip_id == DEVICE_ID_VGH_1435)
-		res_cap_dcn301.num_pll = 2;
-	pool->base.res_cap = &res_cap_dcn301;
-
-	pool->base.funcs = &dcn301_res_pool_funcs;
-
-	/*************************************************
-	 *  Resource + asic cap harcoding                *
-	 *************************************************/
-	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
-	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
-	pool->base.mpcc_count = pool->base.res_cap->num_timing_generator;
-	dc->caps.max_downscale_ratio = 600;
-	dc->caps.i2c_speed_in_khz = 100;
-	dc->caps.i2c_speed_in_khz_hdcp = 5; /*1.4 w/a enabled by default*/
-	dc->caps.max_cursor_size = 256;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.dmdata_alloc_size = 2048;
-	dc->caps.max_slave_planes = 2;
-	dc->caps.max_slave_yuv_planes = 2;
-	dc->caps.max_slave_rgb_planes = 2;
-	dc->caps.is_apu = true;
-	dc->caps.post_blend_color_processing = true;
-	dc->caps.force_dp_tps4_for_cp2520 = true;
-	dc->caps.extended_aux_timeout_support = true;
-	dc->caps.dmcub_support = true;
-
-	/* Color pipeline capabilities */
-	dc->caps.color.dpp.dcn_arch = 1;
-	dc->caps.color.dpp.input_lut_shared = 0;
-	dc->caps.color.dpp.icsc = 1;
-	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
-	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
-	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
-	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
-	dc->caps.color.dpp.post_csc = 1;
-	dc->caps.color.dpp.gamma_corr = 1;
-	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
-
-	dc->caps.color.dpp.hw_3d_lut = 1;
-	dc->caps.color.dpp.ogam_ram = 1;
-	// no OGAM ROM on DCN301
-	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
-	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
-	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
-	dc->caps.color.dpp.ocsc = 0;
-
-	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //2
-	dc->caps.color.mpc.ogam_ram = 1;
-	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
-	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
-	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
-	dc->caps.color.mpc.ocsc = 1;
-
-	dc->caps.dp_hdmi21_pcon_support = true;
-
-	/* read VBIOS LTTPR caps */
-	if (ctx->dc_bios->funcs->get_lttpr_caps) {
-		enum bp_result bp_query_result;
-		uint8_t is_vbios_lttpr_enable = 0;
-
-		bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
-		dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
-	}
-
-	if (ctx->dc_bios->funcs->get_lttpr_interop) {
-		enum bp_result bp_query_result;
-		uint8_t is_vbios_interop_enabled = 0;
-
-		bp_query_result = ctx->dc_bios->funcs->get_lttpr_interop(ctx->dc_bios, &is_vbios_interop_enabled);
-		dc->caps.vbios_lttpr_aware = (bp_query_result == BP_RESULT_OK) && !!is_vbios_interop_enabled;
-	}
-
-	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
-		dc->debug = debug_defaults_drv;
-
-	// Init the vm_helper
-	if (dc->vm_helper)
-		vm_helper_init(dc->vm_helper, 16);
-
-	/*************************************************
-	 *  Create resources                             *
-	 *************************************************/
-
-	/* Clock Sources for Pixel Clock*/
-	pool->base.clock_sources[DCN301_CLK_SRC_PLL0] =
-			dcn301_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL0,
-				&clk_src_regs[0], false);
-	pool->base.clock_sources[DCN301_CLK_SRC_PLL1] =
-			dcn301_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL1,
-				&clk_src_regs[1], false);
-	pool->base.clock_sources[DCN301_CLK_SRC_PLL2] =
-			dcn301_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL2,
-				&clk_src_regs[2], false);
-	pool->base.clock_sources[DCN301_CLK_SRC_PLL3] =
-			dcn301_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL3,
-				&clk_src_regs[3], false);
-
-	pool->base.clk_src_count = DCN301_CLK_SRC_TOTAL;
-
-	/* todo: not reuse phy_pll registers */
-	pool->base.dp_clock_source =
-			dcn301_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_ID_DP_DTO,
-				&clk_src_regs[0], true);
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-
-	/* DCCG */
-	pool->base.dccg = dccg301_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
-	if (pool->base.dccg == NULL) {
-		dm_error("DC: failed to create dccg!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	init_soc_bounding_box(dc, pool);
-
-	if (!dc->debug.disable_pplib_wm_range && pool->base.pp_smu->nv_funcs.set_wm_ranges)
-		set_wm_ranges(pool->base.pp_smu, &dcn3_01_soc);
-
-	num_pipes = dcn3_01_ip.max_num_dpp;
-
-	for (i = 0; i < dcn3_01_ip.max_num_dpp; i++)
-		if (pipe_fuses & 1 << i)
-			num_pipes--;
-	dcn3_01_ip.max_num_dpp = num_pipes;
-	dcn3_01_ip.max_num_otg = num_pipes;
-
-
-	dml_init_instance(&dc->dml, &dcn3_01_soc, &dcn3_01_ip, DML_PROJECT_DCN30);
-
-	/* IRQ */
-	init_data.ctx = dc->ctx;
-	pool->base.irqs = dal_irq_service_dcn30_create(&init_data);
-	if (!pool->base.irqs)
-		goto create_fail;
-
-	/* HUBBUB */
-	pool->base.hubbub = dcn301_hubbub_create(ctx);
-	if (pool->base.hubbub == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create hubbub!\n");
-		goto create_fail;
-	}
-
-	j = 0;
-	/* HUBPs, DPPs, OPPs and TGs */
-	for (i = 0; i < pool->base.pipe_count; i++) {
-
-		/* if pipe is disabled, skip instance of HW pipe,
-		 * i.e, skip ASIC register instance
-		 */
-		if ((pipe_fuses & (1 << i)) != 0) {
-			DC_LOG_DEBUG("%s: fusing pipe %d\n", __func__, i);
-			continue;
-		}
-
-		pool->base.hubps[j] = dcn301_hubp_create(ctx, i);
-		if (pool->base.hubps[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create hubps!\n");
-			goto create_fail;
-		}
-
-		pool->base.dpps[j] = dcn301_dpp_create(ctx, i);
-		if (pool->base.dpps[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create dpps!\n");
-			goto create_fail;
-		}
-
-		pool->base.opps[j] = dcn301_opp_create(ctx, i);
-		if (pool->base.opps[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create output pixel processor!\n");
-			goto create_fail;
-		}
-
-		pool->base.timing_generators[j] = dcn301_timing_generator_create(ctx, i);
-		if (pool->base.timing_generators[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create tg!\n");
-			goto create_fail;
-		}
-		j++;
-	}
-	pool->base.timing_generator_count = j;
-	pool->base.pipe_count = j;
-	pool->base.mpcc_count = j;
-
-	/* ABM (or ABMs for NV2x) */
-	/* TODO: */
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		pool->base.multiple_abms[i] = dmub_abm_create(ctx,
-				&abm_regs[i],
-				&abm_shift,
-				&abm_mask);
-		if (pool->base.multiple_abms[i] == NULL) {
-			dm_error("DC: failed to create abm for pipe %d!\n", i);
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-
-	/* MPC and DSC */
-	pool->base.mpc = dcn301_mpc_create(ctx, pool->base.mpcc_count, pool->base.res_cap->num_mpc_3dlut);
-	if (pool->base.mpc == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mpc!\n");
-		goto create_fail;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		pool->base.dscs[i] = dcn301_dsc_create(ctx, i);
-		if (pool->base.dscs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create display stream compressor %d!\n", i);
-			goto create_fail;
-		}
-	}
-
-	/* DWB and MMHUBBUB */
-	if (!dcn301_dwbc_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create dwbc!\n");
-		goto create_fail;
-	}
-
-	if (!dcn301_mmhubbub_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mcif_wb!\n");
-		goto create_fail;
-	}
-
-	/* AUX and I2C */
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		pool->base.engines[i] = dcn301_aux_engine_create(ctx, i);
-		if (pool->base.engines[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create aux engine!!\n");
-			goto create_fail;
-		}
-		pool->base.hw_i2cs[i] = dcn301_i2c_hw_create(ctx, i);
-		if (pool->base.hw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create hw i2c!!\n");
-			goto create_fail;
-		}
-		pool->base.sw_i2cs[i] = NULL;
-	}
-
-	/* Audio, Stream Encoders including HPO and virtual, MPC 3D LUTs */
-	if (!resource_construct(num_virtual_links, dc, &pool->base,
-			&res_create_funcs))
-		goto create_fail;
-
-	/* HW Sequencer and Plane caps */
-	dcn301_hw_sequencer_construct(dc);
-
-	dc->caps.max_planes =  pool->base.pipe_count;
-
-	for (i = 0; i < dc->caps.max_planes; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	dc->cap_funcs = cap_funcs;
-
-	return true;
-
-create_fail:
-
-	dcn301_destruct(pool);
-
-	return false;
-}
-
-struct resource_pool *dcn301_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc)
-{
-	struct dcn301_resource_pool *pool =
-		kzalloc(sizeof(struct dcn301_resource_pool), GFP_KERNEL);
-
-	if (!pool)
-		return NULL;
-
-	if (dcn301_resource_construct(init_data->num_virtual_links, dc, pool))
-		return &pool->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(pool);
-	return NULL;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.h b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.h
deleted file mode 100644
index ae8672680cdd..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright 2020 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#ifndef _DCN301_RESOURCE_H_
-#define _DCN301_RESOURCE_H_
-
-#include "core_types.h"
-
-struct dc;
-struct resource_pool;
-struct _vcs_dpi_display_pipe_params_st;
-
-extern struct _vcs_dpi_ip_params_st dcn3_01_ip;
-extern struct _vcs_dpi_soc_bounding_box_st dcn3_01_soc;
-
-struct dcn301_resource_pool {
-	struct resource_pool base;
-};
-struct resource_pool *dcn301_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc);
-
-#endif /* _DCN301_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn302/Makefile b/drivers/gpu/drm/amd/display/dc/dcn302/Makefile
index 95b66baf39e9..0fcd03569d74 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn302/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn302/Makefile
@@ -5,7 +5,7 @@
 #
 # Makefile for dcn302.
 
-DCN3_02 = dcn302_init.o dcn302_resource.o
+DCN3_02 = dcn302_init.o
 
 AMD_DAL_DCN3_02 = $(addprefix $(AMDDALPATH)/dc/dcn302/,$(DCN3_02))
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c
deleted file mode 100644
index 63ac984a04f7..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c
+++ /dev/null
@@ -1,1518 +0,0 @@
-/*
- * Copyright 2020 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#include "dcn302_init.h"
-#include "dcn302_resource.h"
-#include "dcn302_dccg.h"
-#include "irq/dcn302/irq_service_dcn302.h"
-
-#include "dcn30/dcn30_dio_link_encoder.h"
-#include "dcn30/dcn30_dio_stream_encoder.h"
-#include "dcn30/dcn30_dwb.h"
-#include "dcn30/dcn30_dpp.h"
-#include "dcn30/dcn30_hubbub.h"
-#include "dcn30/dcn30_hubp.h"
-#include "dcn30/dcn30_mmhubbub.h"
-#include "dcn30/dcn30_mpc.h"
-#include "dcn30/dcn30_opp.h"
-#include "dcn30/dcn30_optc.h"
-#include "dcn30/dcn30_resource.h"
-
-#include "dcn20/dcn20_dsc.h"
-#include "dcn20/dcn20_resource.h"
-
-#include "dml/dcn30/dcn30_fpu.h"
-
-#include "dcn10/dcn10_resource.h"
-
-#include "link.h"
-#include "dce/dce_abm.h"
-#include "dce/dce_audio.h"
-#include "dce/dce_aux.h"
-#include "dce/dce_clock_source.h"
-#include "dce/dce_hwseq.h"
-#include "dce/dce_i2c_hw.h"
-#include "dce/dce_panel_cntl.h"
-#include "dce/dmub_abm.h"
-#include "dce/dmub_psr.h"
-#include "clk_mgr.h"
-
-#include "hw_sequencer_private.h"
-#include "reg_helper.h"
-#include "resource.h"
-#include "vm_helper.h"
-
-#include "dml/dcn302/dcn302_fpu.h"
-
-#include "dimgrey_cavefish_ip_offset.h"
-#include "dcn/dcn_3_0_2_offset.h"
-#include "dcn/dcn_3_0_2_sh_mask.h"
-#include "dpcs/dpcs_3_0_0_offset.h"
-#include "dpcs/dpcs_3_0_0_sh_mask.h"
-#include "nbio/nbio_7_4_offset.h"
-#include "amdgpu_socbb.h"
-
-#define DC_LOGGER \
-	dc->ctx->logger
-#define DC_LOGGER_INIT(logger)
-
-static const struct dc_debug_options debug_defaults_drv = {
-		.disable_dmcu = true,
-		.force_abm_enable = false,
-		.timing_trace = false,
-		.clock_trace = true,
-		.disable_pplib_clock_request = true,
-		.pipe_split_policy = MPC_SPLIT_DYNAMIC,
-		.force_single_disp_pipe_split = false,
-		.disable_dcc = DCC_ENABLE,
-		.vsr_support = true,
-		.performance_trace = false,
-		.max_downscale_src_width = 7680,/*upto 8K*/
-		.disable_pplib_wm_range = false,
-		.scl_reset_length10 = true,
-		.sanity_checks = false,
-		.underflow_assert_delay_us = 0xFFFFFFFF,
-		.dwb_fi_phase = -1, // -1 = disable,
-		.dmub_command_table = true,
-		.use_max_lb = true,
-		.exit_idle_opt_for_cursor_updates = true,
-		.enable_legacy_fast_update = false,
-		.using_dml2 = false,
-};
-
-static const struct dc_panel_config panel_config_defaults = {
-		.psr = {
-			.disable_psr = false,
-			.disallow_psrsu = false,
-			.disallow_replay = false,
-		},
-};
-
-enum dcn302_clk_src_array_id {
-	DCN302_CLK_SRC_PLL0,
-	DCN302_CLK_SRC_PLL1,
-	DCN302_CLK_SRC_PLL2,
-	DCN302_CLK_SRC_PLL3,
-	DCN302_CLK_SRC_PLL4,
-	DCN302_CLK_SRC_TOTAL
-};
-
-static const struct resource_caps res_cap_dcn302 = {
-		.num_timing_generator = 5,
-		.num_opp = 5,
-		.num_video_plane = 5,
-		.num_audio = 5,
-		.num_stream_encoder = 5,
-		.num_dwb = 1,
-		.num_ddc = 5,
-		.num_vmid = 16,
-		.num_mpc_3dlut = 2,
-		.num_dsc = 5,
-};
-
-static const struct dc_plane_cap plane_cap = {
-		.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-		.per_pixel_alpha = true,
-		.pixel_format_support = {
-				.argb8888 = true,
-				.nv12 = true,
-				.fp16 = true,
-				.p010 = true,
-				.ayuv = false,
-		},
-		.max_upscale_factor = {
-				.argb8888 = 16000,
-				.nv12 = 16000,
-				.fp16 = 16000
-		},
-		/* 6:1 downscaling ratio: 1000/6 = 166.666 */
-		.max_downscale_factor = {
-				.argb8888 = 167,
-				.nv12 = 167,
-				.fp16 = 167
-		},
-		16,
-		16
-};
-
-/* NBIO */
-#define NBIO_BASE_INNER(seg) \
-		NBIO_BASE__INST0_SEG ## seg
-
-#define NBIO_BASE(seg) \
-		NBIO_BASE_INNER(seg)
-
-#define NBIO_SR(reg_name)\
-		.reg_name = NBIO_BASE(mm ## reg_name ## _BASE_IDX) + \
-		mm ## reg_name
-
-/* DCN */
-#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
-
-#define BASE(seg) BASE_INNER(seg)
-
-#define SR(reg_name)\
-		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) + mm ## reg_name
-
-#define SF(reg_name, field_name, post_fix)\
-		.field_name = reg_name ## __ ## field_name ## post_fix
-
-#define SRI(reg_name, block, id)\
-		.reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + mm ## block ## id ## _ ## reg_name
-
-#define SRI2(reg_name, block, id)\
-		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) + mm ## reg_name
-
-#define SRII(reg_name, block, id)\
-		.reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		mm ## block ## id ## _ ## reg_name
-
-#define DCCG_SRII(reg_name, block, id)\
-		.block ## _ ## reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		mm ## block ## id ## _ ## reg_name
-
-#define VUPDATE_SRII(reg_name, block, id)\
-		.reg_name[id] = BASE(mm ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
-		mm ## reg_name ## _ ## block ## id
-
-#define SRII_DWB(reg_name, temp_name, block, id)\
-		.reg_name[id] = BASE(mm ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
-		mm ## block ## id ## _ ## temp_name
-
-#define SF_DWB2(reg_name, block, id, field_name, post_fix)	\
-	.field_name = reg_name ## __ ## field_name ## post_fix
-
-#define SRII_MPC_RMU(reg_name, block, id)\
-		.RMU##_##reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		mm ## block ## id ## _ ## reg_name
-
-static const struct dcn_hubbub_registers hubbub_reg = {
-		HUBBUB_REG_LIST_DCN30(0)
-};
-
-static const struct dcn_hubbub_shift hubbub_shift = {
-		HUBBUB_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn_hubbub_mask hubbub_mask = {
-		HUBBUB_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define vmid_regs(id)\
-		[id] = { DCN20_VMID_REG_LIST(id) }
-
-static const struct dcn_vmid_registers vmid_regs[] = {
-		vmid_regs(0),
-		vmid_regs(1),
-		vmid_regs(2),
-		vmid_regs(3),
-		vmid_regs(4),
-		vmid_regs(5),
-		vmid_regs(6),
-		vmid_regs(7),
-		vmid_regs(8),
-		vmid_regs(9),
-		vmid_regs(10),
-		vmid_regs(11),
-		vmid_regs(12),
-		vmid_regs(13),
-		vmid_regs(14),
-		vmid_regs(15)
-};
-
-static const struct dcn20_vmid_shift vmid_shifts = {
-		DCN20_VMID_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn20_vmid_mask vmid_masks = {
-		DCN20_VMID_MASK_SH_LIST(_MASK)
-};
-
-static struct hubbub *dcn302_hubbub_create(struct dc_context *ctx)
-{
-	int i;
-
-	struct dcn20_hubbub *hubbub3 = kzalloc(sizeof(struct dcn20_hubbub), GFP_KERNEL);
-
-	if (!hubbub3)
-		return NULL;
-
-	hubbub3_construct(hubbub3, ctx, &hubbub_reg, &hubbub_shift, &hubbub_mask);
-
-	for (i = 0; i < res_cap_dcn302.num_vmid; i++) {
-		struct dcn20_vmid *vmid = &hubbub3->vmid[i];
-
-		vmid->ctx = ctx;
-
-		vmid->regs = &vmid_regs[i];
-		vmid->shifts = &vmid_shifts;
-		vmid->masks = &vmid_masks;
-	}
-
-	return &hubbub3->base;
-}
-
-#define vpg_regs(id)\
-		[id] = { VPG_DCN3_REG_LIST(id) }
-
-static const struct dcn30_vpg_registers vpg_regs[] = {
-		vpg_regs(0),
-		vpg_regs(1),
-		vpg_regs(2),
-		vpg_regs(3),
-		vpg_regs(4),
-		vpg_regs(5)
-};
-
-static const struct dcn30_vpg_shift vpg_shift = {
-		DCN3_VPG_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn30_vpg_mask vpg_mask = {
-		DCN3_VPG_MASK_SH_LIST(_MASK)
-};
-
-static struct vpg *dcn302_vpg_create(struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn30_vpg *vpg3 = kzalloc(sizeof(struct dcn30_vpg), GFP_KERNEL);
-
-	if (!vpg3)
-		return NULL;
-
-	vpg3_construct(vpg3, ctx, inst, &vpg_regs[inst], &vpg_shift, &vpg_mask);
-
-	return &vpg3->base;
-}
-
-#define afmt_regs(id)\
-		[id] = { AFMT_DCN3_REG_LIST(id) }
-
-static const struct dcn30_afmt_registers afmt_regs[] = {
-		afmt_regs(0),
-		afmt_regs(1),
-		afmt_regs(2),
-		afmt_regs(3),
-		afmt_regs(4),
-		afmt_regs(5)
-};
-
-static const struct dcn30_afmt_shift afmt_shift = {
-		DCN3_AFMT_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn30_afmt_mask afmt_mask = {
-		DCN3_AFMT_MASK_SH_LIST(_MASK)
-};
-
-static struct afmt *dcn302_afmt_create(struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn30_afmt *afmt3 = kzalloc(sizeof(struct dcn30_afmt), GFP_KERNEL);
-
-	if (!afmt3)
-		return NULL;
-
-	afmt3_construct(afmt3, ctx, inst, &afmt_regs[inst], &afmt_shift, &afmt_mask);
-
-	return &afmt3->base;
-}
-
-#define audio_regs(id)\
-		[id] = { AUD_COMMON_REG_LIST(id) }
-
-static const struct dce_audio_registers audio_regs[] = {
-		audio_regs(0),
-		audio_regs(1),
-		audio_regs(2),
-		audio_regs(3),
-		audio_regs(4),
-		audio_regs(5),
-		audio_regs(6)
-};
-
-#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
-		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
-
-static const struct dce_audio_shift audio_shift = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_audio_mask audio_mask = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
-};
-
-static struct audio *dcn302_create_audio(struct dc_context *ctx, unsigned int inst)
-{
-	return dce_audio_create(ctx, inst, &audio_regs[inst], &audio_shift, &audio_mask);
-}
-
-#define stream_enc_regs(id)\
-		[id] = { SE_DCN3_REG_LIST(id) }
-
-static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
-		stream_enc_regs(0),
-		stream_enc_regs(1),
-		stream_enc_regs(2),
-		stream_enc_regs(3),
-		stream_enc_regs(4)
-};
-
-static const struct dcn10_stream_encoder_shift se_shift = {
-		SE_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn10_stream_encoder_mask se_mask = {
-		SE_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static struct stream_encoder *dcn302_stream_encoder_create(enum engine_id eng_id, struct dc_context *ctx)
-{
-	struct dcn10_stream_encoder *enc1;
-	struct vpg *vpg;
-	struct afmt *afmt;
-	int vpg_inst;
-	int afmt_inst;
-
-	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
-	if (eng_id <= ENGINE_ID_DIGE) {
-		vpg_inst = eng_id;
-		afmt_inst = eng_id;
-	} else
-		return NULL;
-
-	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
-	vpg = dcn302_vpg_create(ctx, vpg_inst);
-	afmt = dcn302_afmt_create(ctx, afmt_inst);
-
-	if (!enc1 || !vpg || !afmt) {
-		kfree(enc1);
-		kfree(vpg);
-		kfree(afmt);
-		return NULL;
-	}
-
-	dcn30_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios, eng_id, vpg, afmt, &stream_enc_regs[eng_id],
-			&se_shift, &se_mask);
-
-	return &enc1->base;
-}
-
-#define clk_src_regs(index, pllid)\
-		[index] = { CS_COMMON_REG_LIST_DCN3_02(index, pllid) }
-
-static const struct dce110_clk_src_regs clk_src_regs[] = {
-		clk_src_regs(0, A),
-		clk_src_regs(1, B),
-		clk_src_regs(2, C),
-		clk_src_regs(3, D),
-		clk_src_regs(4, E)
-};
-
-static const struct dce110_clk_src_shift cs_shift = {
-		CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
-};
-
-static const struct dce110_clk_src_mask cs_mask = {
-		CS_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
-};
-
-static struct clock_source *dcn302_clock_source_create(struct dc_context *ctx, struct dc_bios *bios,
-		enum clock_source_id id, const struct dce110_clk_src_regs *regs, bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src = kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dcn3_clk_src_construct(clk_src, ctx, bios, id, regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-
-	kfree(clk_src);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
-static const struct dce_hwseq_registers hwseq_reg = {
-		HWSEQ_DCN302_REG_LIST()
-};
-
-static const struct dce_hwseq_shift hwseq_shift = {
-		HWSEQ_DCN302_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_hwseq_mask hwseq_mask = {
-		HWSEQ_DCN302_MASK_SH_LIST(_MASK)
-};
-
-static struct dce_hwseq *dcn302_hwseq_create(struct dc_context *ctx)
-{
-	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
-
-	if (hws) {
-		hws->ctx = ctx;
-		hws->regs = &hwseq_reg;
-		hws->shifts = &hwseq_shift;
-		hws->masks = &hwseq_mask;
-	}
-	return hws;
-}
-
-#define hubp_regs(id)\
-		[id] = { HUBP_REG_LIST_DCN30(id) }
-
-static const struct dcn_hubp2_registers hubp_regs[] = {
-		hubp_regs(0),
-		hubp_regs(1),
-		hubp_regs(2),
-		hubp_regs(3),
-		hubp_regs(4)
-};
-
-static const struct dcn_hubp2_shift hubp_shift = {
-		HUBP_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn_hubp2_mask hubp_mask = {
-		HUBP_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static struct hubp *dcn302_hubp_create(struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_hubp *hubp2 = kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
-
-	if (!hubp2)
-		return NULL;
-
-	if (hubp3_construct(hubp2, ctx, inst, &hubp_regs[inst], &hubp_shift, &hubp_mask))
-		return &hubp2->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(hubp2);
-	return NULL;
-}
-
-#define dpp_regs(id)\
-		[id] = { DPP_REG_LIST_DCN30(id) }
-
-static const struct dcn3_dpp_registers dpp_regs[] = {
-		dpp_regs(0),
-		dpp_regs(1),
-		dpp_regs(2),
-		dpp_regs(3),
-		dpp_regs(4)
-};
-
-static const struct dcn3_dpp_shift tf_shift = {
-		DPP_REG_LIST_SH_MASK_DCN30(__SHIFT)
-};
-
-static const struct dcn3_dpp_mask tf_mask = {
-		DPP_REG_LIST_SH_MASK_DCN30(_MASK)
-};
-
-static struct dpp *dcn302_dpp_create(struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn3_dpp *dpp = kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
-
-	if (!dpp)
-		return NULL;
-
-	if (dpp3_construct(dpp, ctx, inst, &dpp_regs[inst], &tf_shift, &tf_mask))
-		return &dpp->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(dpp);
-	return NULL;
-}
-
-#define opp_regs(id)\
-		[id] = { OPP_REG_LIST_DCN30(id) }
-
-static const struct dcn20_opp_registers opp_regs[] = {
-		opp_regs(0),
-		opp_regs(1),
-		opp_regs(2),
-		opp_regs(3),
-		opp_regs(4)
-};
-
-static const struct dcn20_opp_shift opp_shift = {
-		OPP_MASK_SH_LIST_DCN20(__SHIFT)
-};
-
-static const struct dcn20_opp_mask opp_mask = {
-		OPP_MASK_SH_LIST_DCN20(_MASK)
-};
-
-static struct output_pixel_processor *dcn302_opp_create(struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_opp *opp = kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
-
-	if (!opp) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dcn20_opp_construct(opp, ctx, inst, &opp_regs[inst], &opp_shift, &opp_mask);
-	return &opp->base;
-}
-
-#define optc_regs(id)\
-		[id] = { OPTC_COMMON_REG_LIST_DCN3_0(id) }
-
-static const struct dcn_optc_registers optc_regs[] = {
-		optc_regs(0),
-		optc_regs(1),
-		optc_regs(2),
-		optc_regs(3),
-		optc_regs(4)
-};
-
-static const struct dcn_optc_shift optc_shift = {
-		OPTC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn_optc_mask optc_mask = {
-		OPTC_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static struct timing_generator *dcn302_timing_generator_create(struct dc_context *ctx, uint32_t instance)
-{
-	struct optc *tgn10 = kzalloc(sizeof(struct optc), GFP_KERNEL);
-
-	if (!tgn10)
-		return NULL;
-
-	tgn10->base.inst = instance;
-	tgn10->base.ctx = ctx;
-
-	tgn10->tg_regs = &optc_regs[instance];
-	tgn10->tg_shift = &optc_shift;
-	tgn10->tg_mask = &optc_mask;
-
-	dcn30_timing_generator_init(tgn10);
-
-	return &tgn10->base;
-}
-
-static const struct dcn30_mpc_registers mpc_regs = {
-		MPC_REG_LIST_DCN3_0(0),
-		MPC_REG_LIST_DCN3_0(1),
-		MPC_REG_LIST_DCN3_0(2),
-		MPC_REG_LIST_DCN3_0(3),
-		MPC_REG_LIST_DCN3_0(4),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(0),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(1),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(2),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(3),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(4),
-		MPC_RMU_GLOBAL_REG_LIST_DCN3AG,
-		MPC_RMU_REG_LIST_DCN3AG(0),
-		MPC_RMU_REG_LIST_DCN3AG(1),
-		MPC_RMU_REG_LIST_DCN3AG(2),
-		MPC_DWB_MUX_REG_LIST_DCN3_0(0),
-};
-
-static const struct dcn30_mpc_shift mpc_shift = {
-		MPC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_mpc_mask mpc_mask = {
-		MPC_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static struct mpc *dcn302_mpc_create(struct dc_context *ctx, int num_mpcc, int num_rmu)
-{
-	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc), GFP_KERNEL);
-
-	if (!mpc30)
-		return NULL;
-
-	dcn30_mpc_construct(mpc30, ctx, &mpc_regs, &mpc_shift, &mpc_mask, num_mpcc, num_rmu);
-
-	return &mpc30->base;
-}
-
-#define dsc_regsDCN20(id)\
-[id] = { DSC_REG_LIST_DCN20(id) }
-
-static const struct dcn20_dsc_registers dsc_regs[] = {
-		dsc_regsDCN20(0),
-		dsc_regsDCN20(1),
-		dsc_regsDCN20(2),
-		dsc_regsDCN20(3),
-		dsc_regsDCN20(4)
-};
-
-static const struct dcn20_dsc_shift dsc_shift = {
-		DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
-};
-
-static const struct dcn20_dsc_mask dsc_mask = {
-		DSC_REG_LIST_SH_MASK_DCN20(_MASK)
-};
-
-static struct display_stream_compressor *dcn302_dsc_create(struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_dsc *dsc = kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
-
-	if (!dsc) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
-	return &dsc->base;
-}
-
-#define dwbc_regs_dcn3(id)\
-[id] = { DWBC_COMMON_REG_LIST_DCN30(id) }
-
-static const struct dcn30_dwbc_registers dwbc30_regs[] = {
-		dwbc_regs_dcn3(0)
-};
-
-static const struct dcn30_dwbc_shift dwbc30_shift = {
-		DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_dwbc_mask dwbc30_mask = {
-		DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static bool dcn302_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t pipe_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < pipe_count; i++) {
-		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc), GFP_KERNEL);
-
-		if (!dwbc30) {
-			dm_error("DC: failed to create dwbc30!\n");
-			return false;
-		}
-
-		dcn30_dwbc_construct(dwbc30, ctx, &dwbc30_regs[i], &dwbc30_shift, &dwbc30_mask, i);
-
-		pool->dwbc[i] = &dwbc30->base;
-	}
-	return true;
-}
-
-#define mcif_wb_regs_dcn3(id)\
-[id] = { MCIF_WB_COMMON_REG_LIST_DCN30(id) }
-
-static const struct dcn30_mmhubbub_registers mcif_wb30_regs[] = {
-		mcif_wb_regs_dcn3(0)
-};
-
-static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
-		MCIF_WB_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
-		MCIF_WB_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static bool dcn302_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t pipe_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < pipe_count; i++) {
-		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub), GFP_KERNEL);
-
-		if (!mcif_wb30) {
-			dm_error("DC: failed to create mcif_wb30!\n");
-			return false;
-		}
-
-		dcn30_mmhubbub_construct(mcif_wb30, ctx, &mcif_wb30_regs[i], &mcif_wb30_shift, &mcif_wb30_mask, i);
-
-		pool->mcif_wb[i] = &mcif_wb30->base;
-	}
-	return true;
-}
-
-#define aux_engine_regs(id)\
-[id] = {\
-		AUX_COMMON_REG_LIST0(id), \
-		.AUXN_IMPCAL = 0, \
-		.AUXP_IMPCAL = 0, \
-		.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
-}
-
-static const struct dce110_aux_registers aux_engine_regs[] = {
-		aux_engine_regs(0),
-		aux_engine_regs(1),
-		aux_engine_regs(2),
-		aux_engine_regs(3),
-		aux_engine_regs(4)
-};
-
-static const struct dce110_aux_registers_shift aux_shift = {
-		DCN_AUX_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce110_aux_registers_mask aux_mask = {
-		DCN_AUX_MASK_SH_LIST(_MASK)
-};
-
-static struct dce_aux *dcn302_aux_engine_create(struct dc_context *ctx, uint32_t inst)
-{
-	struct aux_engine_dce110 *aux_engine = kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
-
-	if (!aux_engine)
-		return NULL;
-
-	dce110_aux_engine_construct(aux_engine, ctx, inst, SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
-			&aux_engine_regs[inst], &aux_mask, &aux_shift, ctx->dc->caps.extended_aux_timeout_support);
-
-	return &aux_engine->base;
-}
-
-#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
-
-static const struct dce_i2c_registers i2c_hw_regs[] = {
-		i2c_inst_regs(1),
-		i2c_inst_regs(2),
-		i2c_inst_regs(3),
-		i2c_inst_regs(4),
-		i2c_inst_regs(5)
-};
-
-static const struct dce_i2c_shift i2c_shifts = {
-		I2C_COMMON_MASK_SH_LIST_DCN2(__SHIFT)
-};
-
-static const struct dce_i2c_mask i2c_masks = {
-		I2C_COMMON_MASK_SH_LIST_DCN2(_MASK)
-};
-
-static struct dce_i2c_hw *dcn302_i2c_hw_create(struct dc_context *ctx, uint32_t inst)
-{
-	struct dce_i2c_hw *dce_i2c_hw = kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
-
-	if (!dce_i2c_hw)
-		return NULL;
-
-	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst, &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
-
-	return dce_i2c_hw;
-}
-
-static const struct encoder_feature_support link_enc_feature = {
-		.max_hdmi_deep_color = COLOR_DEPTH_121212,
-		.max_hdmi_pixel_clock = 600000,
-		.hdmi_ycbcr420_supported = true,
-		.dp_ycbcr420_supported = true,
-		.fec_supported = true,
-		.flags.bits.IS_HBR2_CAPABLE = true,
-		.flags.bits.IS_HBR3_CAPABLE = true,
-		.flags.bits.IS_TPS3_CAPABLE = true,
-		.flags.bits.IS_TPS4_CAPABLE = true
-};
-
-#define link_regs(id, phyid)\
-		[id] = {\
-				LE_DCN3_REG_LIST(id), \
-				UNIPHY_DCN2_REG_LIST(phyid), \
-				DPCS_DCN2_REG_LIST(id), \
-				SRI(DP_DPHY_INTERNAL_CTRL, DP, id) \
-		}
-
-static const struct dcn10_link_enc_registers link_enc_regs[] = {
-		link_regs(0, A),
-		link_regs(1, B),
-		link_regs(2, C),
-		link_regs(3, D),
-		link_regs(4, E)
-};
-
-static const struct dcn10_link_enc_shift le_shift = {
-		LINK_ENCODER_MASK_SH_LIST_DCN30(__SHIFT),
-		DPCS_DCN2_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn10_link_enc_mask le_mask = {
-		LINK_ENCODER_MASK_SH_LIST_DCN30(_MASK),
-		DPCS_DCN2_MASK_SH_LIST(_MASK)
-};
-
-#define aux_regs(id)\
-		[id] = { DCN2_AUX_REG_LIST(id) }
-
-static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
-		aux_regs(0),
-		aux_regs(1),
-		aux_regs(2),
-		aux_regs(3),
-		aux_regs(4)
-};
-
-#define hpd_regs(id)\
-		[id] = { HPD_REG_LIST(id) }
-
-static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
-		hpd_regs(0),
-		hpd_regs(1),
-		hpd_regs(2),
-		hpd_regs(3),
-		hpd_regs(4)
-};
-
-static struct link_encoder *dcn302_link_encoder_create(
-	struct dc_context *ctx,
-	const struct encoder_init_data *enc_init_data)
-{
-	struct dcn20_link_encoder *enc20 = kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
-
-	if (!enc20)
-		return NULL;
-
-	dcn30_link_encoder_construct(enc20, enc_init_data, &link_enc_feature,
-			&link_enc_regs[enc_init_data->transmitter], &link_enc_aux_regs[enc_init_data->channel - 1],
-			&link_enc_hpd_regs[enc_init_data->hpd_source], &le_shift, &le_mask);
-
-	return &enc20->enc10.base;
-}
-
-static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
-		{ DCN_PANEL_CNTL_REG_LIST() }
-};
-
-static const struct dce_panel_cntl_shift panel_cntl_shift = {
-		DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_panel_cntl_mask panel_cntl_mask = {
-		DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
-};
-
-static struct panel_cntl *dcn302_panel_cntl_create(const struct panel_cntl_init_data *init_data)
-{
-	struct dce_panel_cntl *panel_cntl = kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
-
-	if (!panel_cntl)
-		return NULL;
-
-	dce_panel_cntl_construct(panel_cntl, init_data, &panel_cntl_regs[init_data->inst],
-			&panel_cntl_shift, &panel_cntl_mask);
-
-	return &panel_cntl->base;
-}
-
-static void read_dce_straps(struct dc_context *ctx, struct resource_straps *straps)
-{
-	generic_reg_get(ctx, mmDC_PINSTRAPS + BASE(mmDC_PINSTRAPS_BASE_IDX),
-			FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
-}
-
-static const struct resource_create_funcs res_create_funcs = {
-		.read_dce_straps = read_dce_straps,
-		.create_audio = dcn302_create_audio,
-		.create_stream_encoder = dcn302_stream_encoder_create,
-		.create_hwseq = dcn302_hwseq_create,
-};
-
-static bool is_soc_bounding_box_valid(struct dc *dc)
-{
-	uint32_t hw_internal_rev = dc->ctx->asic_id.hw_internal_rev;
-
-	if (ASICREV_IS_DIMGREY_CAVEFISH_P(hw_internal_rev))
-		return true;
-
-	return false;
-}
-
-static bool init_soc_bounding_box(struct dc *dc,  struct resource_pool *pool)
-{
-	struct _vcs_dpi_soc_bounding_box_st *loaded_bb = &dcn3_02_soc;
-	struct _vcs_dpi_ip_params_st *loaded_ip = &dcn3_02_ip;
-
-	DC_LOGGER_INIT(dc->ctx->logger);
-
-	if (!is_soc_bounding_box_valid(dc)) {
-		DC_LOG_ERROR("%s: not valid soc bounding box\n", __func__);
-		return false;
-	}
-
-	loaded_ip->max_num_otg = pool->pipe_count;
-	loaded_ip->max_num_dpp = pool->pipe_count;
-	loaded_ip->clamp_min_dcfclk = dc->config.clamp_min_dcfclk;
-	DC_FP_START();
-	dcn20_patch_bounding_box(dc, loaded_bb);
-	DC_FP_END();
-
-	if (dc->ctx->dc_bios->funcs->get_soc_bb_info) {
-		struct bp_soc_bb_info bb_info = { 0 };
-
-		if (dc->ctx->dc_bios->funcs->get_soc_bb_info(
-			    dc->ctx->dc_bios, &bb_info) == BP_RESULT_OK) {
-
-				DC_FP_START();
-				dcn302_fpu_init_soc_bounding_box(bb_info);
-				DC_FP_END();
-		}
-	}
-
-	return true;
-}
-
-static void dcn302_resource_destruct(struct resource_pool *pool)
-{
-	unsigned int i;
-
-	for (i = 0; i < pool->stream_enc_count; i++) {
-		if (pool->stream_enc[i] != NULL) {
-			if (pool->stream_enc[i]->vpg != NULL) {
-				kfree(DCN30_VPG_FROM_VPG(pool->stream_enc[i]->vpg));
-				pool->stream_enc[i]->vpg = NULL;
-			}
-			if (pool->stream_enc[i]->afmt != NULL) {
-				kfree(DCN30_AFMT_FROM_AFMT(pool->stream_enc[i]->afmt));
-				pool->stream_enc[i]->afmt = NULL;
-			}
-			kfree(DCN10STRENC_FROM_STRENC(pool->stream_enc[i]));
-			pool->stream_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->res_cap->num_dsc; i++) {
-		if (pool->dscs[i] != NULL)
-			dcn20_dsc_destroy(&pool->dscs[i]);
-	}
-
-	if (pool->mpc != NULL) {
-		kfree(TO_DCN20_MPC(pool->mpc));
-		pool->mpc = NULL;
-	}
-
-	if (pool->hubbub != NULL) {
-		kfree(pool->hubbub);
-		pool->hubbub = NULL;
-	}
-
-	for (i = 0; i < pool->pipe_count; i++) {
-		if (pool->dpps[i] != NULL) {
-			kfree(TO_DCN20_DPP(pool->dpps[i]));
-			pool->dpps[i] = NULL;
-		}
-
-		if (pool->hubps[i] != NULL) {
-			kfree(TO_DCN20_HUBP(pool->hubps[i]));
-			pool->hubps[i] = NULL;
-		}
-
-		if (pool->irqs != NULL)
-			dal_irq_service_destroy(&pool->irqs);
-	}
-
-	for (i = 0; i < pool->res_cap->num_ddc; i++) {
-		if (pool->engines[i] != NULL)
-			dce110_engine_destroy(&pool->engines[i]);
-		if (pool->hw_i2cs[i] != NULL) {
-			kfree(pool->hw_i2cs[i]);
-			pool->hw_i2cs[i] = NULL;
-		}
-		if (pool->sw_i2cs[i] != NULL) {
-			kfree(pool->sw_i2cs[i]);
-			pool->sw_i2cs[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->res_cap->num_opp; i++) {
-		if (pool->opps[i] != NULL)
-			pool->opps[i]->funcs->opp_destroy(&pool->opps[i]);
-	}
-
-	for (i = 0; i < pool->res_cap->num_timing_generator; i++) {
-		if (pool->timing_generators[i] != NULL)	{
-			kfree(DCN10TG_FROM_TG(pool->timing_generators[i]));
-			pool->timing_generators[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->res_cap->num_dwb; i++) {
-		if (pool->dwbc[i] != NULL) {
-			kfree(TO_DCN30_DWBC(pool->dwbc[i]));
-			pool->dwbc[i] = NULL;
-		}
-		if (pool->mcif_wb[i] != NULL) {
-			kfree(TO_DCN30_MMHUBBUB(pool->mcif_wb[i]));
-			pool->mcif_wb[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->audio_count; i++) {
-		if (pool->audios[i])
-			dce_aud_destroy(&pool->audios[i]);
-	}
-
-	for (i = 0; i < pool->clk_src_count; i++) {
-		if (pool->clock_sources[i] != NULL)
-			dcn20_clock_source_destroy(&pool->clock_sources[i]);
-	}
-
-	if (pool->dp_clock_source != NULL)
-		dcn20_clock_source_destroy(&pool->dp_clock_source);
-
-	for (i = 0; i < pool->res_cap->num_mpc_3dlut; i++) {
-		if (pool->mpc_lut[i] != NULL) {
-			dc_3dlut_func_release(pool->mpc_lut[i]);
-			pool->mpc_lut[i] = NULL;
-		}
-		if (pool->mpc_shaper[i] != NULL) {
-			dc_transfer_func_release(pool->mpc_shaper[i]);
-			pool->mpc_shaper[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->pipe_count; i++) {
-		if (pool->multiple_abms[i] != NULL)
-			dce_abm_destroy(&pool->multiple_abms[i]);
-	}
-
-	if (pool->psr != NULL)
-		dmub_psr_destroy(&pool->psr);
-
-	if (pool->dccg != NULL)
-		dcn_dccg_destroy(&pool->dccg);
-
-	if (pool->oem_device != NULL) {
-		struct dc *dc = pool->oem_device->ctx->dc;
-
-		dc->link_srv->destroy_ddc_service(&pool->oem_device);
-	}
-}
-
-static void dcn302_destroy_resource_pool(struct resource_pool **pool)
-{
-	dcn302_resource_destruct(*pool);
-	kfree(*pool);
-	*pool = NULL;
-}
-
-void dcn302_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
-{
-	DC_FP_START();
-	dcn302_fpu_update_bw_bounding_box(dc, bw_params);
-	DC_FP_END();
-}
-
-static void dcn302_get_panel_config_defaults(struct dc_panel_config *panel_config)
-{
-	*panel_config = panel_config_defaults;
-}
-
-static struct resource_funcs dcn302_res_pool_funcs = {
-		.destroy = dcn302_destroy_resource_pool,
-		.link_enc_create = dcn302_link_encoder_create,
-		.panel_cntl_create = dcn302_panel_cntl_create,
-		.validate_bandwidth = dcn30_validate_bandwidth,
-		.calculate_wm_and_dlg = dcn30_calculate_wm_and_dlg,
-		.update_soc_for_wm_a = dcn30_update_soc_for_wm_a,
-		.populate_dml_pipes = dcn30_populate_dml_pipes_from_context,
-		.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
-		.release_pipe = dcn20_release_pipe,
-		.add_stream_to_ctx = dcn30_add_stream_to_ctx,
-		.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
-		.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
-		.populate_dml_writeback_from_context = dcn30_populate_dml_writeback_from_context,
-		.set_mcif_arb_params = dcn30_set_mcif_arb_params,
-		.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
-		.acquire_post_bldn_3dlut = dcn30_acquire_post_bldn_3dlut,
-		.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
-		.update_bw_bounding_box = dcn302_update_bw_bounding_box,
-		.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
-		.get_panel_config_defaults = dcn302_get_panel_config_defaults,
-};
-
-static struct dc_cap_funcs cap_funcs = {
-		.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
-};
-
-static const struct bios_registers bios_regs = {
-		NBIO_SR(BIOS_SCRATCH_3),
-		NBIO_SR(BIOS_SCRATCH_6)
-};
-
-static const struct dccg_registers dccg_regs = {
-		DCCG_REG_LIST_DCN3_02()
-};
-
-static const struct dccg_shift dccg_shift = {
-		DCCG_MASK_SH_LIST_DCN3_02(__SHIFT)
-};
-
-static const struct dccg_mask dccg_mask = {
-		DCCG_MASK_SH_LIST_DCN3_02(_MASK)
-};
-
-#define abm_regs(id)\
-		[id] = { ABM_DCN302_REG_LIST(id) }
-
-static const struct dce_abm_registers abm_regs[] = {
-		abm_regs(0),
-		abm_regs(1),
-		abm_regs(2),
-		abm_regs(3),
-		abm_regs(4)
-};
-
-static const struct dce_abm_shift abm_shift = {
-		ABM_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dce_abm_mask abm_mask = {
-		ABM_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static bool dcn302_resource_construct(
-		uint8_t num_virtual_links,
-		struct dc *dc,
-		struct resource_pool *pool)
-{
-	int i;
-	struct dc_context *ctx = dc->ctx;
-	struct irq_service_init_data init_data;
-	struct ddc_service_init_data ddc_init_data = {0};
-
-	ctx->dc_bios->regs = &bios_regs;
-
-	pool->res_cap = &res_cap_dcn302;
-
-	pool->funcs = &dcn302_res_pool_funcs;
-
-	/*************************************************
-	 *  Resource + asic cap harcoding                *
-	 *************************************************/
-	pool->underlay_pipe_index = NO_UNDERLAY_PIPE;
-	pool->pipe_count = pool->res_cap->num_timing_generator;
-	pool->mpcc_count = pool->res_cap->num_timing_generator;
-	dc->caps.max_downscale_ratio = 600;
-	dc->caps.i2c_speed_in_khz = 100;
-	dc->caps.i2c_speed_in_khz_hdcp = 5; /*1.4 w/a applied by derfault*/
-	dc->caps.max_cursor_size = 256;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.dmdata_alloc_size = 2048;
-	dc->caps.mall_size_per_mem_channel = 4;
-	/* total size = mall per channel * num channels * 1024 * 1024 */
-	dc->caps.mall_size_total = dc->caps.mall_size_per_mem_channel * dc->ctx->dc_bios->vram_info.num_chans * 1048576;
-	dc->caps.cursor_cache_size = dc->caps.max_cursor_size * dc->caps.max_cursor_size * 8;
-	dc->caps.max_slave_planes = 2;
-	dc->caps.max_slave_yuv_planes = 2;
-	dc->caps.max_slave_rgb_planes = 2;
-	dc->caps.post_blend_color_processing = true;
-	dc->caps.force_dp_tps4_for_cp2520 = true;
-	dc->caps.extended_aux_timeout_support = true;
-	dc->caps.dmcub_support = true;
-	dc->caps.max_v_total = (1 << 15) - 1;
-
-	/* Color pipeline capabilities */
-	dc->caps.color.dpp.dcn_arch = 1;
-	dc->caps.color.dpp.input_lut_shared = 0;
-	dc->caps.color.dpp.icsc = 1;
-	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
-	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
-	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
-	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
-	dc->caps.color.dpp.post_csc = 1;
-	dc->caps.color.dpp.gamma_corr = 1;
-	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
-
-	dc->caps.color.dpp.hw_3d_lut = 1;
-	dc->caps.color.dpp.ogam_ram = 1;
-	// no OGAM ROM on DCN3
-	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
-	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
-	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
-	dc->caps.color.dpp.ocsc = 0;
-
-	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->res_cap->num_mpc_3dlut; //3
-	dc->caps.color.mpc.ogam_ram = 1;
-	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
-	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
-	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
-	dc->caps.color.mpc.ocsc = 1;
-
-	dc->caps.dp_hdmi21_pcon_support = true;
-
-	/* read VBIOS LTTPR caps */
-	if (ctx->dc_bios->funcs->get_lttpr_caps) {
-		enum bp_result bp_query_result;
-		uint8_t is_vbios_lttpr_enable = 0;
-
-		bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
-		dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
-	}
-
-	if (ctx->dc_bios->funcs->get_lttpr_interop) {
-		enum bp_result bp_query_result;
-		uint8_t is_vbios_interop_enabled = 0;
-
-		bp_query_result = ctx->dc_bios->funcs->get_lttpr_interop(ctx->dc_bios,
-				&is_vbios_interop_enabled);
-		dc->caps.vbios_lttpr_aware = (bp_query_result == BP_RESULT_OK) && !!is_vbios_interop_enabled;
-	}
-
-	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
-		dc->debug = debug_defaults_drv;
-
-	// Init the vm_helper
-	if (dc->vm_helper)
-		vm_helper_init(dc->vm_helper, 16);
-
-	/*************************************************
-	 *  Create resources                             *
-	 *************************************************/
-
-	/* Clock Sources for Pixel Clock*/
-	pool->clock_sources[DCN302_CLK_SRC_PLL0] =
-			dcn302_clock_source_create(ctx, ctx->dc_bios,
-					CLOCK_SOURCE_COMBO_PHY_PLL0,
-					&clk_src_regs[0], false);
-	pool->clock_sources[DCN302_CLK_SRC_PLL1] =
-			dcn302_clock_source_create(ctx, ctx->dc_bios,
-					CLOCK_SOURCE_COMBO_PHY_PLL1,
-					&clk_src_regs[1], false);
-	pool->clock_sources[DCN302_CLK_SRC_PLL2] =
-			dcn302_clock_source_create(ctx, ctx->dc_bios,
-					CLOCK_SOURCE_COMBO_PHY_PLL2,
-					&clk_src_regs[2], false);
-	pool->clock_sources[DCN302_CLK_SRC_PLL3] =
-			dcn302_clock_source_create(ctx, ctx->dc_bios,
-					CLOCK_SOURCE_COMBO_PHY_PLL3,
-					&clk_src_regs[3], false);
-	pool->clock_sources[DCN302_CLK_SRC_PLL4] =
-			dcn302_clock_source_create(ctx, ctx->dc_bios,
-					CLOCK_SOURCE_COMBO_PHY_PLL4,
-					&clk_src_regs[4], false);
-
-	pool->clk_src_count = DCN302_CLK_SRC_TOTAL;
-
-	/* todo: not reuse phy_pll registers */
-	pool->dp_clock_source =
-			dcn302_clock_source_create(ctx, ctx->dc_bios,
-					CLOCK_SOURCE_ID_DP_DTO,
-					&clk_src_regs[0], true);
-
-	for (i = 0; i < pool->clk_src_count; i++) {
-		if (pool->clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-
-	/* DCCG */
-	pool->dccg = dccg30_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
-	if (pool->dccg == NULL) {
-		dm_error("DC: failed to create dccg!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	/* PP Lib and SMU interfaces */
-	init_soc_bounding_box(dc, pool);
-
-	/* DML */
-	dml_init_instance(&dc->dml, &dcn3_02_soc, &dcn3_02_ip, DML_PROJECT_DCN30);
-
-	/* IRQ */
-	init_data.ctx = dc->ctx;
-	pool->irqs = dal_irq_service_dcn302_create(&init_data);
-	if (!pool->irqs)
-		goto create_fail;
-
-	/* HUBBUB */
-	pool->hubbub = dcn302_hubbub_create(ctx);
-	if (pool->hubbub == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create hubbub!\n");
-		goto create_fail;
-	}
-
-	/* HUBPs, DPPs, OPPs and TGs */
-	for (i = 0; i < pool->pipe_count; i++) {
-		pool->hubps[i] = dcn302_hubp_create(ctx, i);
-		if (pool->hubps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create hubps!\n");
-			goto create_fail;
-		}
-
-		pool->dpps[i] = dcn302_dpp_create(ctx, i);
-		if (pool->dpps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create dpps!\n");
-			goto create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->res_cap->num_opp; i++) {
-		pool->opps[i] = dcn302_opp_create(ctx, i);
-		if (pool->opps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create output pixel processor!\n");
-			goto create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->res_cap->num_timing_generator; i++) {
-		pool->timing_generators[i] = dcn302_timing_generator_create(ctx, i);
-		if (pool->timing_generators[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create tg!\n");
-			goto create_fail;
-		}
-	}
-	pool->timing_generator_count = i;
-
-	/* PSR */
-	pool->psr = dmub_psr_create(ctx);
-	if (pool->psr == NULL) {
-		dm_error("DC: failed to create psr!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	/* ABMs */
-	for (i = 0; i < pool->res_cap->num_timing_generator; i++) {
-		pool->multiple_abms[i] = dmub_abm_create(ctx, &abm_regs[i], &abm_shift, &abm_mask);
-		if (pool->multiple_abms[i] == NULL) {
-			dm_error("DC: failed to create abm for pipe %d!\n", i);
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-
-	/* MPC and DSC */
-	pool->mpc = dcn302_mpc_create(ctx, pool->mpcc_count, pool->res_cap->num_mpc_3dlut);
-	if (pool->mpc == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mpc!\n");
-		goto create_fail;
-	}
-
-	for (i = 0; i < pool->res_cap->num_dsc; i++) {
-		pool->dscs[i] = dcn302_dsc_create(ctx, i);
-		if (pool->dscs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create display stream compressor %d!\n", i);
-			goto create_fail;
-		}
-	}
-
-	/* DWB and MMHUBBUB */
-	if (!dcn302_dwbc_create(ctx, pool)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create dwbc!\n");
-		goto create_fail;
-	}
-
-	if (!dcn302_mmhubbub_create(ctx, pool)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mcif_wb!\n");
-		goto create_fail;
-	}
-
-	/* AUX and I2C */
-	for (i = 0; i < pool->res_cap->num_ddc; i++) {
-		pool->engines[i] = dcn302_aux_engine_create(ctx, i);
-		if (pool->engines[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC:failed to create aux engine!!\n");
-			goto create_fail;
-		}
-		pool->hw_i2cs[i] = dcn302_i2c_hw_create(ctx, i);
-		if (pool->hw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC:failed to create hw i2c!!\n");
-			goto create_fail;
-		}
-		pool->sw_i2cs[i] = NULL;
-	}
-
-	/* Audio, Stream Encoders including HPO and virtual, MPC 3D LUTs */
-	if (!resource_construct(num_virtual_links, dc, pool,
-			&res_create_funcs))
-		goto create_fail;
-
-	/* HW Sequencer and Plane caps */
-	dcn302_hw_sequencer_construct(dc);
-
-	dc->caps.max_planes =  pool->pipe_count;
-
-	for (i = 0; i < dc->caps.max_planes; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	dc->cap_funcs = cap_funcs;
-
-	if (dc->ctx->dc_bios->fw_info.oem_i2c_present) {
-		ddc_init_data.ctx = dc->ctx;
-		ddc_init_data.link = NULL;
-		ddc_init_data.id.id = dc->ctx->dc_bios->fw_info.oem_i2c_obj_id;
-		ddc_init_data.id.enum_id = 0;
-		ddc_init_data.id.type = OBJECT_TYPE_GENERIC;
-		pool->oem_device = dc->link_srv->create_ddc_service(&ddc_init_data);
-	} else {
-		pool->oem_device = NULL;
-	}
-
-	return true;
-
-create_fail:
-
-	dcn302_resource_destruct(pool);
-
-	return false;
-}
-
-struct resource_pool *dcn302_create_resource_pool(const struct dc_init_data *init_data, struct dc *dc)
-{
-	struct resource_pool *pool = kzalloc(sizeof(struct resource_pool), GFP_KERNEL);
-
-	if (!pool)
-		return NULL;
-
-	if (dcn302_resource_construct(init_data->num_virtual_links, dc, pool))
-		return pool;
-
-	BREAK_TO_DEBUGGER();
-	kfree(pool);
-	return NULL;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.h b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.h
deleted file mode 100644
index 9f24e73b92b3..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2020 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#ifndef _DCN302_RESOURCE_H_
-#define _DCN302_RESOURCE_H_
-
-#include "core_types.h"
-
-extern struct _vcs_dpi_ip_params_st dcn3_02_ip;
-extern struct _vcs_dpi_soc_bounding_box_st dcn3_02_soc;
-
-struct resource_pool *dcn302_create_resource_pool(const struct dc_init_data *init_data, struct dc *dc);
-
-void dcn302_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params);
-
-#endif /* _DCN302_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn303/Makefile b/drivers/gpu/drm/amd/display/dc/dcn303/Makefile
index d7b3ad780e5d..a954e316aca2 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn303/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn303/Makefile
@@ -6,7 +6,7 @@
 #
 # Makefile for dcn303.
 
-DCN3_03 = dcn303_init.o dcn303_resource.o
+DCN3_03 = dcn303_init.o
 
 AMD_DAL_DCN3_03 = $(addprefix $(AMDDALPATH)/dc/dcn303/,$(DCN3_03))
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c b/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c
deleted file mode 100644
index 49cb7fde416a..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c
+++ /dev/null
@@ -1,1448 +0,0 @@
-// SPDX-License-Identifier: MIT
-/*
- * Copyright (C) 2021 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- */
-
-#include "dcn303_init.h"
-#include "dcn303_resource.h"
-#include "dcn303_dccg.h"
-#include "irq/dcn303/irq_service_dcn303.h"
-
-#include "dcn30/dcn30_dio_link_encoder.h"
-#include "dcn30/dcn30_dio_stream_encoder.h"
-#include "dcn30/dcn30_dpp.h"
-#include "dcn30/dcn30_dwb.h"
-#include "dcn30/dcn30_hubbub.h"
-#include "dcn30/dcn30_hubp.h"
-#include "dcn30/dcn30_mmhubbub.h"
-#include "dcn30/dcn30_mpc.h"
-#include "dcn30/dcn30_opp.h"
-#include "dcn30/dcn30_optc.h"
-#include "dcn30/dcn30_resource.h"
-
-#include "dcn20/dcn20_dsc.h"
-#include "dcn20/dcn20_resource.h"
-
-#include "dml/dcn30/dcn30_fpu.h"
-
-#include "dcn10/dcn10_resource.h"
-
-#include "link.h"
-
-#include "dce/dce_abm.h"
-#include "dce/dce_audio.h"
-#include "dce/dce_aux.h"
-#include "dce/dce_clock_source.h"
-#include "dce/dce_hwseq.h"
-#include "dce/dce_i2c_hw.h"
-#include "dce/dce_panel_cntl.h"
-#include "dce/dmub_abm.h"
-#include "dce/dmub_psr.h"
-#include "clk_mgr.h"
-
-#include "hw_sequencer_private.h"
-#include "reg_helper.h"
-#include "resource.h"
-#include "vm_helper.h"
-
-#include "sienna_cichlid_ip_offset.h"
-#include "dcn/dcn_3_0_3_offset.h"
-#include "dcn/dcn_3_0_3_sh_mask.h"
-#include "dpcs/dpcs_3_0_3_offset.h"
-#include "dpcs/dpcs_3_0_3_sh_mask.h"
-#include "nbio/nbio_2_3_offset.h"
-
-#include "dml/dcn303/dcn303_fpu.h"
-
-#define DC_LOGGER \
-	dc->ctx->logger
-#define DC_LOGGER_INIT(logger)
-
-
-static const struct dc_debug_options debug_defaults_drv = {
-		.disable_dmcu = true,
-		.force_abm_enable = false,
-		.timing_trace = false,
-		.clock_trace = true,
-		.disable_pplib_clock_request = true,
-		.pipe_split_policy = MPC_SPLIT_AVOID,
-		.force_single_disp_pipe_split = false,
-		.disable_dcc = DCC_ENABLE,
-		.vsr_support = true,
-		.performance_trace = false,
-		.max_downscale_src_width = 7680,/*upto 8K*/
-		.disable_pplib_wm_range = false,
-		.scl_reset_length10 = true,
-		.sanity_checks = false,
-		.underflow_assert_delay_us = 0xFFFFFFFF,
-		.dwb_fi_phase = -1, // -1 = disable,
-		.dmub_command_table = true,
-		.exit_idle_opt_for_cursor_updates = true,
-		.disable_idle_power_optimizations = false,
-		.using_dml2 = false,
-};
-
-static const struct dc_panel_config panel_config_defaults = {
-		.psr = {
-			.disable_psr = false,
-			.disallow_psrsu = false,
-			.disallow_replay = false,
-		},
-};
-
-enum dcn303_clk_src_array_id {
-	DCN303_CLK_SRC_PLL0,
-	DCN303_CLK_SRC_PLL1,
-	DCN303_CLK_SRC_TOTAL
-};
-
-static const struct resource_caps res_cap_dcn303 = {
-		.num_timing_generator = 2,
-		.num_opp = 2,
-		.num_video_plane = 2,
-		.num_audio = 2,
-		.num_stream_encoder = 2,
-		.num_dwb = 1,
-		.num_ddc = 2,
-		.num_vmid = 16,
-		.num_mpc_3dlut = 1,
-		.num_dsc = 2,
-};
-
-static const struct dc_plane_cap plane_cap = {
-		.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-		.per_pixel_alpha = true,
-		.pixel_format_support = {
-				.argb8888 = true,
-				.nv12 = true,
-				.fp16 = true,
-				.p010 = true,
-				.ayuv = false,
-		},
-		.max_upscale_factor = {
-				.argb8888 = 16000,
-				.nv12 = 16000,
-				.fp16 = 16000
-		},
-		.max_downscale_factor = {
-				.argb8888 = 600,
-				.nv12 = 600,
-				.fp16 = 600
-		},
-		16,
-		16
-};
-
-/* NBIO */
-#define NBIO_BASE_INNER(seg) \
-		NBIO_BASE__INST0_SEG ## seg
-
-#define NBIO_BASE(seg) \
-		NBIO_BASE_INNER(seg)
-
-#define NBIO_SR(reg_name)\
-		.reg_name = NBIO_BASE(mm ## reg_name ## _BASE_IDX) + \
-		mm ## reg_name
-
-/* DCN */
-#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
-
-#define BASE(seg) BASE_INNER(seg)
-
-#define SR(reg_name)\
-		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) + mm ## reg_name
-
-#define SF(reg_name, field_name, post_fix)\
-		.field_name = reg_name ## __ ## field_name ## post_fix
-
-#define SRI(reg_name, block, id)\
-		.reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + mm ## block ## id ## _ ## reg_name
-
-#define SRI2(reg_name, block, id)\
-		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) + mm ## reg_name
-
-#define SRII(reg_name, block, id)\
-		.reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		mm ## block ## id ## _ ## reg_name
-
-#define DCCG_SRII(reg_name, block, id)\
-		.block ## _ ## reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		mm ## block ## id ## _ ## reg_name
-
-#define VUPDATE_SRII(reg_name, block, id)\
-		.reg_name[id] = BASE(mm ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
-		mm ## reg_name ## _ ## block ## id
-
-#define SRII_DWB(reg_name, temp_name, block, id)\
-		.reg_name[id] = BASE(mm ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
-		mm ## block ## id ## _ ## temp_name
-
-#define SF_DWB2(reg_name, block, id, field_name, post_fix)	\
-	.field_name = reg_name ## __ ## field_name ## post_fix
-
-#define SRII_MPC_RMU(reg_name, block, id)\
-		.RMU##_##reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		mm ## block ## id ## _ ## reg_name
-
-static const struct dcn_hubbub_registers hubbub_reg = {
-		HUBBUB_REG_LIST_DCN30(0)
-};
-
-static const struct dcn_hubbub_shift hubbub_shift = {
-		HUBBUB_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn_hubbub_mask hubbub_mask = {
-		HUBBUB_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define vmid_regs(id)\
-		[id] = { DCN20_VMID_REG_LIST(id) }
-
-static const struct dcn_vmid_registers vmid_regs[] = {
-		vmid_regs(0),
-		vmid_regs(1),
-		vmid_regs(2),
-		vmid_regs(3),
-		vmid_regs(4),
-		vmid_regs(5),
-		vmid_regs(6),
-		vmid_regs(7),
-		vmid_regs(8),
-		vmid_regs(9),
-		vmid_regs(10),
-		vmid_regs(11),
-		vmid_regs(12),
-		vmid_regs(13),
-		vmid_regs(14),
-		vmid_regs(15)
-};
-
-static const struct dcn20_vmid_shift vmid_shifts = {
-		DCN20_VMID_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn20_vmid_mask vmid_masks = {
-		DCN20_VMID_MASK_SH_LIST(_MASK)
-};
-
-static struct hubbub *dcn303_hubbub_create(struct dc_context *ctx)
-{
-	int i;
-
-	struct dcn20_hubbub *hubbub3 = kzalloc(sizeof(struct dcn20_hubbub), GFP_KERNEL);
-
-	if (!hubbub3)
-		return NULL;
-
-	hubbub3_construct(hubbub3, ctx, &hubbub_reg, &hubbub_shift, &hubbub_mask);
-
-	for (i = 0; i < res_cap_dcn303.num_vmid; i++) {
-		struct dcn20_vmid *vmid = &hubbub3->vmid[i];
-
-		vmid->ctx = ctx;
-
-		vmid->regs = &vmid_regs[i];
-		vmid->shifts = &vmid_shifts;
-		vmid->masks = &vmid_masks;
-	}
-
-	return &hubbub3->base;
-}
-
-#define vpg_regs(id)\
-		[id] = { VPG_DCN3_REG_LIST(id) }
-
-static const struct dcn30_vpg_registers vpg_regs[] = {
-		vpg_regs(0),
-		vpg_regs(1),
-		vpg_regs(2)
-};
-
-static const struct dcn30_vpg_shift vpg_shift = {
-		DCN3_VPG_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn30_vpg_mask vpg_mask = {
-		DCN3_VPG_MASK_SH_LIST(_MASK)
-};
-
-static struct vpg *dcn303_vpg_create(struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn30_vpg *vpg3 = kzalloc(sizeof(struct dcn30_vpg), GFP_KERNEL);
-
-	if (!vpg3)
-		return NULL;
-
-	vpg3_construct(vpg3, ctx, inst, &vpg_regs[inst], &vpg_shift, &vpg_mask);
-
-	return &vpg3->base;
-}
-
-#define afmt_regs(id)\
-		[id] = { AFMT_DCN3_REG_LIST(id) }
-
-static const struct dcn30_afmt_registers afmt_regs[] = {
-		afmt_regs(0),
-		afmt_regs(1),
-		afmt_regs(2)
-};
-
-static const struct dcn30_afmt_shift afmt_shift = {
-		DCN3_AFMT_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn30_afmt_mask afmt_mask = {
-		DCN3_AFMT_MASK_SH_LIST(_MASK)
-};
-
-static struct afmt *dcn303_afmt_create(struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn30_afmt *afmt3 = kzalloc(sizeof(struct dcn30_afmt), GFP_KERNEL);
-
-	if (!afmt3)
-		return NULL;
-
-	afmt3_construct(afmt3, ctx, inst, &afmt_regs[inst], &afmt_shift, &afmt_mask);
-
-	return &afmt3->base;
-}
-
-#define audio_regs(id)\
-		[id] = { AUD_COMMON_REG_LIST(id) }
-
-static const struct dce_audio_registers audio_regs[] = {
-		audio_regs(0),
-		audio_regs(1),
-		audio_regs(2),
-		audio_regs(3),
-		audio_regs(4),
-		audio_regs(5),
-		audio_regs(6)
-};
-
-#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
-		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
-
-static const struct dce_audio_shift audio_shift = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_audio_mask audio_mask = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
-};
-
-static struct audio *dcn303_create_audio(struct dc_context *ctx, unsigned int inst)
-{
-	return dce_audio_create(ctx, inst, &audio_regs[inst], &audio_shift, &audio_mask);
-}
-
-#define stream_enc_regs(id)\
-		[id] = { SE_DCN3_REG_LIST(id) }
-
-static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
-		stream_enc_regs(0),
-		stream_enc_regs(1)
-};
-
-static const struct dcn10_stream_encoder_shift se_shift = {
-		SE_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn10_stream_encoder_mask se_mask = {
-		SE_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static struct stream_encoder *dcn303_stream_encoder_create(enum engine_id eng_id, struct dc_context *ctx)
-{
-	struct dcn10_stream_encoder *enc1;
-	struct vpg *vpg;
-	struct afmt *afmt;
-	int vpg_inst;
-	int afmt_inst;
-
-	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
-	if (eng_id <= ENGINE_ID_DIGB) {
-		vpg_inst = eng_id;
-		afmt_inst = eng_id;
-	} else
-		return NULL;
-
-	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
-	vpg = dcn303_vpg_create(ctx, vpg_inst);
-	afmt = dcn303_afmt_create(ctx, afmt_inst);
-
-	if (!enc1 || !vpg || !afmt) {
-		kfree(enc1);
-		kfree(vpg);
-		kfree(afmt);
-		return NULL;
-	}
-
-	dcn30_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios, eng_id, vpg, afmt, &stream_enc_regs[eng_id],
-			&se_shift, &se_mask);
-
-	return &enc1->base;
-}
-
-#define clk_src_regs(index, pllid)\
-		[index] = { CS_COMMON_REG_LIST_DCN3_03(index, pllid) }
-
-static const struct dce110_clk_src_regs clk_src_regs[] = {
-		clk_src_regs(0, A),
-		clk_src_regs(1, B)
-};
-
-static const struct dce110_clk_src_shift cs_shift = {
-		CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
-};
-
-static const struct dce110_clk_src_mask cs_mask = {
-		CS_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
-};
-
-static struct clock_source *dcn303_clock_source_create(struct dc_context *ctx, struct dc_bios *bios,
-		enum clock_source_id id, const struct dce110_clk_src_regs *regs, bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src = kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dcn3_clk_src_construct(clk_src, ctx, bios, id, regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-
-	kfree(clk_src);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
-static const struct dce_hwseq_registers hwseq_reg = {
-		HWSEQ_DCN303_REG_LIST()
-};
-
-static const struct dce_hwseq_shift hwseq_shift = {
-		HWSEQ_DCN303_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_hwseq_mask hwseq_mask = {
-		HWSEQ_DCN303_MASK_SH_LIST(_MASK)
-};
-
-static struct dce_hwseq *dcn303_hwseq_create(struct dc_context *ctx)
-{
-	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
-
-	if (hws) {
-		hws->ctx = ctx;
-		hws->regs = &hwseq_reg;
-		hws->shifts = &hwseq_shift;
-		hws->masks = &hwseq_mask;
-	}
-	return hws;
-}
-
-#define hubp_regs(id)\
-		[id] = { HUBP_REG_LIST_DCN30(id) }
-
-static const struct dcn_hubp2_registers hubp_regs[] = {
-		hubp_regs(0),
-		hubp_regs(1)
-};
-
-static const struct dcn_hubp2_shift hubp_shift = {
-		HUBP_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn_hubp2_mask hubp_mask = {
-		HUBP_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static struct hubp *dcn303_hubp_create(struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_hubp *hubp2 = kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
-
-	if (!hubp2)
-		return NULL;
-
-	if (hubp3_construct(hubp2, ctx, inst, &hubp_regs[inst], &hubp_shift, &hubp_mask))
-		return &hubp2->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(hubp2);
-	return NULL;
-}
-
-#define dpp_regs(id)\
-		[id] = { DPP_REG_LIST_DCN30(id) }
-
-static const struct dcn3_dpp_registers dpp_regs[] = {
-		dpp_regs(0),
-		dpp_regs(1)
-};
-
-static const struct dcn3_dpp_shift tf_shift = {
-		DPP_REG_LIST_SH_MASK_DCN30(__SHIFT)
-};
-
-static const struct dcn3_dpp_mask tf_mask = {
-		DPP_REG_LIST_SH_MASK_DCN30(_MASK)
-};
-
-static struct dpp *dcn303_dpp_create(struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn3_dpp *dpp = kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
-
-	if (!dpp)
-		return NULL;
-
-	if (dpp3_construct(dpp, ctx, inst, &dpp_regs[inst], &tf_shift, &tf_mask))
-		return &dpp->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(dpp);
-	return NULL;
-}
-
-#define opp_regs(id)\
-		[id] = { OPP_REG_LIST_DCN30(id) }
-
-static const struct dcn20_opp_registers opp_regs[] = {
-		opp_regs(0),
-		opp_regs(1)
-};
-
-static const struct dcn20_opp_shift opp_shift = {
-		OPP_MASK_SH_LIST_DCN20(__SHIFT)
-};
-
-static const struct dcn20_opp_mask opp_mask = {
-		OPP_MASK_SH_LIST_DCN20(_MASK)
-};
-
-static struct output_pixel_processor *dcn303_opp_create(struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_opp *opp = kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
-
-	if (!opp) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dcn20_opp_construct(opp, ctx, inst, &opp_regs[inst], &opp_shift, &opp_mask);
-	return &opp->base;
-}
-
-#define optc_regs(id)\
-		[id] = { OPTC_COMMON_REG_LIST_DCN3_0(id) }
-
-static const struct dcn_optc_registers optc_regs[] = {
-		optc_regs(0),
-		optc_regs(1)
-};
-
-static const struct dcn_optc_shift optc_shift = {
-		OPTC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn_optc_mask optc_mask = {
-		OPTC_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static struct timing_generator *dcn303_timing_generator_create(struct dc_context *ctx, uint32_t instance)
-{
-	struct optc *tgn10 = kzalloc(sizeof(struct optc), GFP_KERNEL);
-
-	if (!tgn10)
-		return NULL;
-
-	tgn10->base.inst = instance;
-	tgn10->base.ctx = ctx;
-
-	tgn10->tg_regs = &optc_regs[instance];
-	tgn10->tg_shift = &optc_shift;
-	tgn10->tg_mask = &optc_mask;
-
-	dcn30_timing_generator_init(tgn10);
-
-	return &tgn10->base;
-}
-
-static const struct dcn30_mpc_registers mpc_regs = {
-		MPC_REG_LIST_DCN3_0(0),
-		MPC_REG_LIST_DCN3_0(1),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(0),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(1),
-		MPC_RMU_GLOBAL_REG_LIST_DCN3AG,
-		MPC_RMU_REG_LIST_DCN3AG(0),
-		MPC_DWB_MUX_REG_LIST_DCN3_0(0),
-};
-
-static const struct dcn30_mpc_shift mpc_shift = {
-		MPC_COMMON_MASK_SH_LIST_DCN303(__SHIFT)
-};
-
-static const struct dcn30_mpc_mask mpc_mask = {
-		MPC_COMMON_MASK_SH_LIST_DCN303(_MASK)
-};
-
-static struct mpc *dcn303_mpc_create(struct dc_context *ctx, int num_mpcc, int num_rmu)
-{
-	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc), GFP_KERNEL);
-
-	if (!mpc30)
-		return NULL;
-
-	dcn30_mpc_construct(mpc30, ctx, &mpc_regs, &mpc_shift, &mpc_mask, num_mpcc, num_rmu);
-
-	return &mpc30->base;
-}
-
-#define dsc_regsDCN20(id)\
-[id] = { DSC_REG_LIST_DCN20(id) }
-
-static const struct dcn20_dsc_registers dsc_regs[] = {
-		dsc_regsDCN20(0),
-		dsc_regsDCN20(1)
-};
-
-static const struct dcn20_dsc_shift dsc_shift = {
-		DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
-};
-
-static const struct dcn20_dsc_mask dsc_mask = {
-		DSC_REG_LIST_SH_MASK_DCN20(_MASK)
-};
-
-static struct display_stream_compressor *dcn303_dsc_create(struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_dsc *dsc = kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
-
-	if (!dsc) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
-	return &dsc->base;
-}
-
-#define dwbc_regs_dcn3(id)\
-[id] = { DWBC_COMMON_REG_LIST_DCN30(id) }
-
-static const struct dcn30_dwbc_registers dwbc30_regs[] = {
-		dwbc_regs_dcn3(0)
-};
-
-static const struct dcn30_dwbc_shift dwbc30_shift = {
-		DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_dwbc_mask dwbc30_mask = {
-		DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static bool dcn303_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t pipe_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < pipe_count; i++) {
-		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc), GFP_KERNEL);
-
-		if (!dwbc30) {
-			dm_error("DC: failed to create dwbc30!\n");
-			return false;
-		}
-
-		dcn30_dwbc_construct(dwbc30, ctx, &dwbc30_regs[i], &dwbc30_shift, &dwbc30_mask, i);
-
-		pool->dwbc[i] = &dwbc30->base;
-	}
-	return true;
-}
-
-#define mcif_wb_regs_dcn3(id)\
-[id] = { MCIF_WB_COMMON_REG_LIST_DCN30(id) }
-
-static const struct dcn30_mmhubbub_registers mcif_wb30_regs[] = {
-		mcif_wb_regs_dcn3(0)
-};
-
-static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
-		MCIF_WB_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
-		MCIF_WB_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static bool dcn303_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t pipe_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < pipe_count; i++) {
-		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub), GFP_KERNEL);
-
-		if (!mcif_wb30) {
-			dm_error("DC: failed to create mcif_wb30!\n");
-			return false;
-		}
-
-		dcn30_mmhubbub_construct(mcif_wb30, ctx, &mcif_wb30_regs[i], &mcif_wb30_shift, &mcif_wb30_mask, i);
-
-		pool->mcif_wb[i] = &mcif_wb30->base;
-	}
-	return true;
-}
-
-#define aux_engine_regs(id)\
-[id] = {\
-		AUX_COMMON_REG_LIST0(id), \
-		.AUXN_IMPCAL = 0, \
-		.AUXP_IMPCAL = 0, \
-		.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
-}
-
-static const struct dce110_aux_registers aux_engine_regs[] = {
-		aux_engine_regs(0),
-		aux_engine_regs(1)
-};
-
-static const struct dce110_aux_registers_shift aux_shift = {
-		DCN_AUX_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce110_aux_registers_mask aux_mask = {
-		DCN_AUX_MASK_SH_LIST(_MASK)
-};
-
-static struct dce_aux *dcn303_aux_engine_create(struct dc_context *ctx, uint32_t inst)
-{
-	struct aux_engine_dce110 *aux_engine = kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
-
-	if (!aux_engine)
-		return NULL;
-
-	dce110_aux_engine_construct(aux_engine, ctx, inst, SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
-			&aux_engine_regs[inst], &aux_mask, &aux_shift, ctx->dc->caps.extended_aux_timeout_support);
-
-	return &aux_engine->base;
-}
-
-#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
-
-static const struct dce_i2c_registers i2c_hw_regs[] = {
-		i2c_inst_regs(1),
-		i2c_inst_regs(2)
-};
-
-static const struct dce_i2c_shift i2c_shifts = {
-		I2C_COMMON_MASK_SH_LIST_DCN2(__SHIFT)
-};
-
-static const struct dce_i2c_mask i2c_masks = {
-		I2C_COMMON_MASK_SH_LIST_DCN2(_MASK)
-};
-
-static struct dce_i2c_hw *dcn303_i2c_hw_create(struct dc_context *ctx, uint32_t inst)
-{
-	struct dce_i2c_hw *dce_i2c_hw = kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
-
-	if (!dce_i2c_hw)
-		return NULL;
-
-	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst, &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
-
-	return dce_i2c_hw;
-}
-
-static const struct encoder_feature_support link_enc_feature = {
-		.max_hdmi_deep_color = COLOR_DEPTH_121212,
-		.max_hdmi_pixel_clock = 600000,
-		.hdmi_ycbcr420_supported = true,
-		.dp_ycbcr420_supported = true,
-		.fec_supported = true,
-		.flags.bits.IS_HBR2_CAPABLE = true,
-		.flags.bits.IS_HBR3_CAPABLE = true,
-		.flags.bits.IS_TPS3_CAPABLE = true,
-		.flags.bits.IS_TPS4_CAPABLE = true
-};
-
-#define link_regs(id, phyid)\
-		[id] = {\
-				LE_DCN3_REG_LIST(id), \
-				UNIPHY_DCN2_REG_LIST(phyid), \
-				SRI(DP_DPHY_INTERNAL_CTRL, DP, id) \
-		}
-
-static const struct dcn10_link_enc_registers link_enc_regs[] = {
-		link_regs(0, A),
-		link_regs(1, B)
-};
-
-static const struct dcn10_link_enc_shift le_shift = {
-		LINK_ENCODER_MASK_SH_LIST_DCN30(__SHIFT),
-		DPCS_DCN2_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn10_link_enc_mask le_mask = {
-		LINK_ENCODER_MASK_SH_LIST_DCN30(_MASK),
-		DPCS_DCN2_MASK_SH_LIST(_MASK)
-};
-
-#define aux_regs(id)\
-		[id] = { DCN2_AUX_REG_LIST(id) }
-
-static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
-		aux_regs(0),
-		aux_regs(1)
-};
-
-#define hpd_regs(id)\
-		[id] = { HPD_REG_LIST(id) }
-
-static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
-		hpd_regs(0),
-		hpd_regs(1)
-};
-
-static struct link_encoder *dcn303_link_encoder_create(
-	struct dc_context *ctx,
-	const struct encoder_init_data *enc_init_data)
-{
-	struct dcn20_link_encoder *enc20 = kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
-
-	if (!enc20)
-		return NULL;
-
-	dcn30_link_encoder_construct(enc20, enc_init_data, &link_enc_feature,
-			&link_enc_regs[enc_init_data->transmitter], &link_enc_aux_regs[enc_init_data->channel - 1],
-			&link_enc_hpd_regs[enc_init_data->hpd_source], &le_shift, &le_mask);
-
-	return &enc20->enc10.base;
-}
-
-static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
-		{ DCN_PANEL_CNTL_REG_LIST() }
-};
-
-static const struct dce_panel_cntl_shift panel_cntl_shift = {
-		DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_panel_cntl_mask panel_cntl_mask = {
-		DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
-};
-
-static struct panel_cntl *dcn303_panel_cntl_create(const struct panel_cntl_init_data *init_data)
-{
-	struct dce_panel_cntl *panel_cntl = kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
-
-	if (!panel_cntl)
-		return NULL;
-
-	dce_panel_cntl_construct(panel_cntl, init_data, &panel_cntl_regs[init_data->inst],
-			&panel_cntl_shift, &panel_cntl_mask);
-
-	return &panel_cntl->base;
-}
-
-static void read_dce_straps(struct dc_context *ctx, struct resource_straps *straps)
-{
-	generic_reg_get(ctx, mmDC_PINSTRAPS + BASE(mmDC_PINSTRAPS_BASE_IDX),
-			FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
-}
-
-static const struct resource_create_funcs res_create_funcs = {
-		.read_dce_straps = read_dce_straps,
-		.create_audio = dcn303_create_audio,
-		.create_stream_encoder = dcn303_stream_encoder_create,
-		.create_hwseq = dcn303_hwseq_create,
-};
-
-static bool is_soc_bounding_box_valid(struct dc *dc)
-{
-	uint32_t hw_internal_rev = dc->ctx->asic_id.hw_internal_rev;
-
-	if (ASICREV_IS_BEIGE_GOBY_P(hw_internal_rev))
-		return true;
-
-	return false;
-}
-
-static bool init_soc_bounding_box(struct dc *dc,  struct resource_pool *pool)
-{
-	struct _vcs_dpi_soc_bounding_box_st *loaded_bb = &dcn3_03_soc;
-	struct _vcs_dpi_ip_params_st *loaded_ip = &dcn3_03_ip;
-
-	DC_LOGGER_INIT(dc->ctx->logger);
-
-	if (!is_soc_bounding_box_valid(dc)) {
-		DC_LOG_ERROR("%s: not valid soc bounding box/n", __func__);
-		return false;
-	}
-
-	loaded_ip->max_num_otg = pool->pipe_count;
-	loaded_ip->max_num_dpp = pool->pipe_count;
-	loaded_ip->clamp_min_dcfclk = dc->config.clamp_min_dcfclk;
-	DC_FP_START();
-	dcn20_patch_bounding_box(dc, loaded_bb);
-	DC_FP_END();
-
-	if (dc->ctx->dc_bios->funcs->get_soc_bb_info) {
-		struct bp_soc_bb_info bb_info = { 0 };
-
-		if (dc->ctx->dc_bios->funcs->get_soc_bb_info(
-			    dc->ctx->dc_bios, &bb_info) == BP_RESULT_OK) {
-					DC_FP_START();
-					dcn303_fpu_init_soc_bounding_box(bb_info);
-					DC_FP_END();
-		}
-	}
-
-	return true;
-}
-
-static void dcn303_resource_destruct(struct resource_pool *pool)
-{
-	unsigned int i;
-
-	for (i = 0; i < pool->stream_enc_count; i++) {
-		if (pool->stream_enc[i] != NULL) {
-			if (pool->stream_enc[i]->vpg != NULL) {
-				kfree(DCN30_VPG_FROM_VPG(pool->stream_enc[i]->vpg));
-				pool->stream_enc[i]->vpg = NULL;
-			}
-			if (pool->stream_enc[i]->afmt != NULL) {
-				kfree(DCN30_AFMT_FROM_AFMT(pool->stream_enc[i]->afmt));
-				pool->stream_enc[i]->afmt = NULL;
-			}
-			kfree(DCN10STRENC_FROM_STRENC(pool->stream_enc[i]));
-			pool->stream_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->res_cap->num_dsc; i++) {
-		if (pool->dscs[i] != NULL)
-			dcn20_dsc_destroy(&pool->dscs[i]);
-	}
-
-	if (pool->mpc != NULL) {
-		kfree(TO_DCN20_MPC(pool->mpc));
-		pool->mpc = NULL;
-	}
-
-	if (pool->hubbub != NULL) {
-		kfree(pool->hubbub);
-		pool->hubbub = NULL;
-	}
-
-	for (i = 0; i < pool->pipe_count; i++) {
-		if (pool->dpps[i] != NULL) {
-			kfree(TO_DCN20_DPP(pool->dpps[i]));
-			pool->dpps[i] = NULL;
-		}
-
-		if (pool->hubps[i] != NULL) {
-			kfree(TO_DCN20_HUBP(pool->hubps[i]));
-			pool->hubps[i] = NULL;
-		}
-
-		if (pool->irqs != NULL)
-			dal_irq_service_destroy(&pool->irqs);
-	}
-
-	for (i = 0; i < pool->res_cap->num_ddc; i++) {
-		if (pool->engines[i] != NULL)
-			dce110_engine_destroy(&pool->engines[i]);
-		if (pool->hw_i2cs[i] != NULL) {
-			kfree(pool->hw_i2cs[i]);
-			pool->hw_i2cs[i] = NULL;
-		}
-		if (pool->sw_i2cs[i] != NULL) {
-			kfree(pool->sw_i2cs[i]);
-			pool->sw_i2cs[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->res_cap->num_opp; i++) {
-		if (pool->opps[i] != NULL)
-			pool->opps[i]->funcs->opp_destroy(&pool->opps[i]);
-	}
-
-	for (i = 0; i < pool->res_cap->num_timing_generator; i++) {
-		if (pool->timing_generators[i] != NULL)	{
-			kfree(DCN10TG_FROM_TG(pool->timing_generators[i]));
-			pool->timing_generators[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->res_cap->num_dwb; i++) {
-		if (pool->dwbc[i] != NULL) {
-			kfree(TO_DCN30_DWBC(pool->dwbc[i]));
-			pool->dwbc[i] = NULL;
-		}
-		if (pool->mcif_wb[i] != NULL) {
-			kfree(TO_DCN30_MMHUBBUB(pool->mcif_wb[i]));
-			pool->mcif_wb[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->audio_count; i++) {
-		if (pool->audios[i])
-			dce_aud_destroy(&pool->audios[i]);
-	}
-
-	for (i = 0; i < pool->clk_src_count; i++) {
-		if (pool->clock_sources[i] != NULL)
-			dcn20_clock_source_destroy(&pool->clock_sources[i]);
-	}
-
-	if (pool->dp_clock_source != NULL)
-		dcn20_clock_source_destroy(&pool->dp_clock_source);
-
-	for (i = 0; i < pool->res_cap->num_mpc_3dlut; i++) {
-		if (pool->mpc_lut[i] != NULL) {
-			dc_3dlut_func_release(pool->mpc_lut[i]);
-			pool->mpc_lut[i] = NULL;
-		}
-		if (pool->mpc_shaper[i] != NULL) {
-			dc_transfer_func_release(pool->mpc_shaper[i]);
-			pool->mpc_shaper[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->pipe_count; i++) {
-		if (pool->multiple_abms[i] != NULL)
-			dce_abm_destroy(&pool->multiple_abms[i]);
-	}
-
-	if (pool->psr != NULL)
-		dmub_psr_destroy(&pool->psr);
-
-	if (pool->dccg != NULL)
-		dcn_dccg_destroy(&pool->dccg);
-
-	if (pool->oem_device != NULL) {
-		struct dc *dc = pool->oem_device->ctx->dc;
-
-		dc->link_srv->destroy_ddc_service(&pool->oem_device);
-	}
-}
-
-static void dcn303_destroy_resource_pool(struct resource_pool **pool)
-{
-	dcn303_resource_destruct(*pool);
-	kfree(*pool);
-	*pool = NULL;
-}
-
-static void dcn303_get_panel_config_defaults(struct dc_panel_config *panel_config)
-{
-	*panel_config = panel_config_defaults;
-}
-
-void dcn303_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
-{
-	DC_FP_START();
-	dcn303_fpu_update_bw_bounding_box(dc, bw_params);
-	DC_FP_END();
-}
-
-static struct resource_funcs dcn303_res_pool_funcs = {
-		.destroy = dcn303_destroy_resource_pool,
-		.link_enc_create = dcn303_link_encoder_create,
-		.panel_cntl_create = dcn303_panel_cntl_create,
-		.validate_bandwidth = dcn30_validate_bandwidth,
-		.calculate_wm_and_dlg = dcn30_calculate_wm_and_dlg,
-		.update_soc_for_wm_a = dcn30_update_soc_for_wm_a,
-		.populate_dml_pipes = dcn30_populate_dml_pipes_from_context,
-		.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
-		.release_pipe = dcn20_release_pipe,
-		.add_stream_to_ctx = dcn30_add_stream_to_ctx,
-		.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
-		.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
-		.populate_dml_writeback_from_context = dcn30_populate_dml_writeback_from_context,
-		.set_mcif_arb_params = dcn30_set_mcif_arb_params,
-		.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
-		.acquire_post_bldn_3dlut = dcn30_acquire_post_bldn_3dlut,
-		.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
-		.update_bw_bounding_box = dcn303_update_bw_bounding_box,
-		.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
-		.get_panel_config_defaults = dcn303_get_panel_config_defaults,
-};
-
-static struct dc_cap_funcs cap_funcs = {
-		.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
-};
-
-static const struct bios_registers bios_regs = {
-		NBIO_SR(BIOS_SCRATCH_3),
-		NBIO_SR(BIOS_SCRATCH_6)
-};
-
-static const struct dccg_registers dccg_regs = {
-		DCCG_REG_LIST_DCN3_03()
-};
-
-static const struct dccg_shift dccg_shift = {
-		DCCG_MASK_SH_LIST_DCN3_03(__SHIFT)
-};
-
-static const struct dccg_mask dccg_mask = {
-		DCCG_MASK_SH_LIST_DCN3_03(_MASK)
-};
-
-#define abm_regs(id)\
-		[id] = { ABM_DCN302_REG_LIST(id) }
-
-static const struct dce_abm_registers abm_regs[] = {
-		abm_regs(0),
-		abm_regs(1)
-};
-
-static const struct dce_abm_shift abm_shift = {
-		ABM_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dce_abm_mask abm_mask = {
-		ABM_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static bool dcn303_resource_construct(
-		uint8_t num_virtual_links,
-		struct dc *dc,
-		struct resource_pool *pool)
-{
-	int i;
-	struct dc_context *ctx = dc->ctx;
-	struct irq_service_init_data init_data;
-	struct ddc_service_init_data ddc_init_data;
-
-	ctx->dc_bios->regs = &bios_regs;
-
-	pool->res_cap = &res_cap_dcn303;
-
-	pool->funcs = &dcn303_res_pool_funcs;
-
-	/*************************************************
-	 *  Resource + asic cap harcoding                *
-	 *************************************************/
-	pool->underlay_pipe_index = NO_UNDERLAY_PIPE;
-	pool->pipe_count = pool->res_cap->num_timing_generator;
-	pool->mpcc_count = pool->res_cap->num_timing_generator;
-	dc->caps.max_downscale_ratio = 600;
-	dc->caps.i2c_speed_in_khz = 100;
-	dc->caps.i2c_speed_in_khz_hdcp = 5; /*1.4 w/a applied by derfault*/
-	dc->caps.max_cursor_size = 256;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.dmdata_alloc_size = 2048;
-	dc->caps.mall_size_per_mem_channel = 4;
-	/* total size = mall per channel * num channels * 1024 * 1024 */
-	dc->caps.mall_size_total = dc->caps.mall_size_per_mem_channel *
-				   dc->ctx->dc_bios->vram_info.num_chans *
-				   1024 * 1024;
-	dc->caps.cursor_cache_size =
-		dc->caps.max_cursor_size * dc->caps.max_cursor_size * 8;
-	dc->caps.max_slave_planes = 1;
-	dc->caps.post_blend_color_processing = true;
-	dc->caps.force_dp_tps4_for_cp2520 = true;
-	dc->caps.extended_aux_timeout_support = true;
-	dc->caps.dmcub_support = true;
-	dc->caps.max_v_total = (1 << 15) - 1;
-
-	/* Color pipeline capabilities */
-	dc->caps.color.dpp.dcn_arch = 1;
-	dc->caps.color.dpp.input_lut_shared = 0;
-	dc->caps.color.dpp.icsc = 1;
-	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
-	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
-	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
-	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
-	dc->caps.color.dpp.post_csc = 1;
-	dc->caps.color.dpp.gamma_corr = 1;
-	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
-
-	dc->caps.color.dpp.hw_3d_lut = 1;
-	dc->caps.color.dpp.ogam_ram = 1;
-	// no OGAM ROM on DCN3
-	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
-	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
-	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
-	dc->caps.color.dpp.ocsc = 0;
-
-	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->res_cap->num_mpc_3dlut; //3
-	dc->caps.color.mpc.ogam_ram = 1;
-	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
-	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
-	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
-	dc->caps.color.mpc.ocsc = 1;
-
-	dc->caps.dp_hdmi21_pcon_support = true;
-
-	dc->config.dc_mode_clk_limit_support = true;
-	/* read VBIOS LTTPR caps */
-	if (ctx->dc_bios->funcs->get_lttpr_caps) {
-		enum bp_result bp_query_result;
-		uint8_t is_vbios_lttpr_enable = 0;
-
-		bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
-		dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
-	}
-
-	if (ctx->dc_bios->funcs->get_lttpr_interop) {
-		enum bp_result bp_query_result;
-		uint8_t is_vbios_interop_enabled = 0;
-
-		bp_query_result = ctx->dc_bios->funcs->get_lttpr_interop(ctx->dc_bios, &is_vbios_interop_enabled);
-		dc->caps.vbios_lttpr_aware = (bp_query_result == BP_RESULT_OK) && !!is_vbios_interop_enabled;
-	}
-
-	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
-		dc->debug = debug_defaults_drv;
-
-	// Init the vm_helper
-	if (dc->vm_helper)
-		vm_helper_init(dc->vm_helper, 16);
-
-	/*************************************************
-	 *  Create resources                             *
-	 *************************************************/
-
-	/* Clock Sources for Pixel Clock*/
-	pool->clock_sources[DCN303_CLK_SRC_PLL0] =
-			dcn303_clock_source_create(ctx, ctx->dc_bios,
-					CLOCK_SOURCE_COMBO_PHY_PLL0,
-					&clk_src_regs[0], false);
-	pool->clock_sources[DCN303_CLK_SRC_PLL1] =
-			dcn303_clock_source_create(ctx, ctx->dc_bios,
-					CLOCK_SOURCE_COMBO_PHY_PLL1,
-					&clk_src_regs[1], false);
-
-	pool->clk_src_count = DCN303_CLK_SRC_TOTAL;
-
-	/* todo: not reuse phy_pll registers */
-	pool->dp_clock_source =
-			dcn303_clock_source_create(ctx, ctx->dc_bios,
-					CLOCK_SOURCE_ID_DP_DTO,
-					&clk_src_regs[0], true);
-
-	for (i = 0; i < pool->clk_src_count; i++) {
-		if (pool->clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-
-	/* DCCG */
-	pool->dccg = dccg30_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
-	if (pool->dccg == NULL) {
-		dm_error("DC: failed to create dccg!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	/* PP Lib and SMU interfaces */
-	init_soc_bounding_box(dc, pool);
-
-	/* DML */
-	dml_init_instance(&dc->dml, &dcn3_03_soc, &dcn3_03_ip, DML_PROJECT_DCN30);
-
-	/* IRQ */
-	init_data.ctx = dc->ctx;
-	pool->irqs = dal_irq_service_dcn303_create(&init_data);
-	if (!pool->irqs)
-		goto create_fail;
-
-	/* HUBBUB */
-	pool->hubbub = dcn303_hubbub_create(ctx);
-	if (pool->hubbub == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create hubbub!\n");
-		goto create_fail;
-	}
-
-	/* HUBPs, DPPs, OPPs and TGs */
-	for (i = 0; i < pool->pipe_count; i++) {
-		pool->hubps[i] = dcn303_hubp_create(ctx, i);
-		if (pool->hubps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create hubps!\n");
-			goto create_fail;
-		}
-
-		pool->dpps[i] = dcn303_dpp_create(ctx, i);
-		if (pool->dpps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create dpps!\n");
-			goto create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->res_cap->num_opp; i++) {
-		pool->opps[i] = dcn303_opp_create(ctx, i);
-		if (pool->opps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create output pixel processor!\n");
-			goto create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->res_cap->num_timing_generator; i++) {
-		pool->timing_generators[i] = dcn303_timing_generator_create(ctx, i);
-		if (pool->timing_generators[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create tg!\n");
-			goto create_fail;
-		}
-	}
-	pool->timing_generator_count = i;
-
-	/* PSR */
-	pool->psr = dmub_psr_create(ctx);
-	if (pool->psr == NULL) {
-		dm_error("DC: failed to create psr!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	/* ABM */
-	for (i = 0; i < pool->res_cap->num_timing_generator; i++) {
-		pool->multiple_abms[i] = dmub_abm_create(ctx, &abm_regs[i], &abm_shift, &abm_mask);
-		if (pool->multiple_abms[i] == NULL) {
-			dm_error("DC: failed to create abm for pipe %d!\n", i);
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-
-	/* MPC and DSC */
-	pool->mpc = dcn303_mpc_create(ctx, pool->mpcc_count, pool->res_cap->num_mpc_3dlut);
-	if (pool->mpc == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mpc!\n");
-		goto create_fail;
-	}
-
-	for (i = 0; i < pool->res_cap->num_dsc; i++) {
-		pool->dscs[i] = dcn303_dsc_create(ctx, i);
-		if (pool->dscs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create display stream compressor %d!\n", i);
-			goto create_fail;
-		}
-	}
-
-	/* DWB and MMHUBBUB */
-	if (!dcn303_dwbc_create(ctx, pool)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create dwbc!\n");
-		goto create_fail;
-	}
-
-	if (!dcn303_mmhubbub_create(ctx, pool)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mcif_wb!\n");
-		goto create_fail;
-	}
-
-	/* AUX and I2C */
-	for (i = 0; i < pool->res_cap->num_ddc; i++) {
-		pool->engines[i] = dcn303_aux_engine_create(ctx, i);
-		if (pool->engines[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC:failed to create aux engine!!\n");
-			goto create_fail;
-		}
-		pool->hw_i2cs[i] = dcn303_i2c_hw_create(ctx, i);
-		if (pool->hw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC:failed to create hw i2c!!\n");
-			goto create_fail;
-		}
-		pool->sw_i2cs[i] = NULL;
-	}
-
-	/* Audio, Stream Encoders including HPO and virtual, MPC 3D LUTs */
-	if (!resource_construct(num_virtual_links, dc, pool,
-			&res_create_funcs))
-		goto create_fail;
-
-	/* HW Sequencer and Plane caps */
-	dcn303_hw_sequencer_construct(dc);
-
-	dc->caps.max_planes =  pool->pipe_count;
-
-	for (i = 0; i < dc->caps.max_planes; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	dc->cap_funcs = cap_funcs;
-
-	if (dc->ctx->dc_bios->fw_info.oem_i2c_present) {
-		ddc_init_data.ctx = dc->ctx;
-		ddc_init_data.link = NULL;
-		ddc_init_data.id.id = dc->ctx->dc_bios->fw_info.oem_i2c_obj_id;
-		ddc_init_data.id.enum_id = 0;
-		ddc_init_data.id.type = OBJECT_TYPE_GENERIC;
-		pool->oem_device = dc->link_srv->create_ddc_service(&ddc_init_data);
-	} else {
-		pool->oem_device = NULL;
-	}
-
-	return true;
-
-create_fail:
-
-	dcn303_resource_destruct(pool);
-
-	return false;
-}
-
-struct resource_pool *dcn303_create_resource_pool(const struct dc_init_data *init_data, struct dc *dc)
-{
-	struct resource_pool *pool = kzalloc(sizeof(struct resource_pool), GFP_KERNEL);
-
-	if (!pool)
-		return NULL;
-
-	if (dcn303_resource_construct(init_data->num_virtual_links, dc, pool))
-		return pool;
-
-	BREAK_TO_DEBUGGER();
-	kfree(pool);
-	return NULL;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.h b/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.h
deleted file mode 100644
index 37cf1525820b..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.h
+++ /dev/null
@@ -1,38 +0,0 @@
-// SPDX-License-Identifier: MIT
-/*
- * Copyright (C) 2021 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- */
-
-#ifndef _DCN303_RESOURCE_H_
-#define _DCN303_RESOURCE_H_
-
-#include "core_types.h"
-
-extern struct _vcs_dpi_ip_params_st dcn3_03_ip;
-extern struct _vcs_dpi_soc_bounding_box_st dcn3_03_soc;
-
-struct resource_pool *dcn303_create_resource_pool(const struct dc_init_data *init_data, struct dc *dc);
-
-void dcn303_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params);
-
-#endif /* _DCN303_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/Makefile b/drivers/gpu/drm/amd/display/dc/dcn31/Makefile
index 96e45c9efb46..212287008c0a 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/Makefile
@@ -10,7 +10,7 @@
 #
 # Makefile for dcn31.
 
-DCN31 = dcn31_resource.o dcn31_hubbub.o dcn31_init.o dcn31_hubp.o \
+DCN31 = dcn31_hubbub.o dcn31_init.o dcn31_hubp.o \
 	dcn31_dccg.o dcn31_optc.o dcn31_dio_link_encoder.o dcn31_panel_cntl.o \
 	dcn31_apg.o dcn31_hpo_dp_stream_encoder.o dcn31_hpo_dp_link_encoder.o \
 	dcn31_afmt.o dcn31_vpg.o
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c
deleted file mode 100644
index 79416cfb22f0..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c
+++ /dev/null
@@ -1,2218 +0,0 @@
-/*
- * Copyright 2019 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-
-#include "dm_services.h"
-#include "dc.h"
-
-#include "dcn31/dcn31_init.h"
-
-#include "resource.h"
-#include "include/irq_service_interface.h"
-#include "dcn31_resource.h"
-
-#include "dcn20/dcn20_resource.h"
-#include "dcn30/dcn30_resource.h"
-
-#include "dml/dcn30/dcn30_fpu.h"
-
-#include "dcn10/dcn10_ipp.h"
-#include "dcn30/dcn30_hubbub.h"
-#include "dcn31/dcn31_hubbub.h"
-#include "dcn30/dcn30_mpc.h"
-#include "dcn31/dcn31_hubp.h"
-#include "irq/dcn31/irq_service_dcn31.h"
-#include "dcn30/dcn30_dpp.h"
-#include "dcn31/dcn31_optc.h"
-#include "dcn20/dcn20_hwseq.h"
-#include "dcn30/dcn30_hwseq.h"
-#include "dce110/dce110_hwseq.h"
-#include "dcn30/dcn30_opp.h"
-#include "dcn20/dcn20_dsc.h"
-#include "dcn30/dcn30_vpg.h"
-#include "dcn30/dcn30_afmt.h"
-#include "dcn30/dcn30_dio_stream_encoder.h"
-#include "dcn31/dcn31_hpo_dp_stream_encoder.h"
-#include "dcn31/dcn31_hpo_dp_link_encoder.h"
-#include "dcn31/dcn31_apg.h"
-#include "dcn31/dcn31_dio_link_encoder.h"
-#include "dcn31/dcn31_vpg.h"
-#include "dcn31/dcn31_afmt.h"
-#include "dce/dce_clock_source.h"
-#include "dce/dce_audio.h"
-#include "dce/dce_hwseq.h"
-#include "clk_mgr.h"
-#include "virtual/virtual_stream_encoder.h"
-#include "dce110/dce110_resource.h"
-#include "dml/display_mode_vba.h"
-#include "dml/dcn31/dcn31_fpu.h"
-#include "dcn31/dcn31_dccg.h"
-#include "dcn10/dcn10_resource.h"
-#include "dcn31_panel_cntl.h"
-
-#include "dcn30/dcn30_dwb.h"
-#include "dcn30/dcn30_mmhubbub.h"
-
-// TODO: change include headers /amd/include/asic_reg after upstream
-#include "yellow_carp_offset.h"
-#include "dcn/dcn_3_1_2_offset.h"
-#include "dcn/dcn_3_1_2_sh_mask.h"
-#include "nbio/nbio_7_2_0_offset.h"
-#include "dpcs/dpcs_4_2_0_offset.h"
-#include "dpcs/dpcs_4_2_0_sh_mask.h"
-#include "mmhub/mmhub_2_3_0_offset.h"
-#include "mmhub/mmhub_2_3_0_sh_mask.h"
-
-
-#define regDCHUBBUB_DEBUG_CTRL_0                                              0x04d6
-#define regDCHUBBUB_DEBUG_CTRL_0_BASE_IDX                                     2
-#define DCHUBBUB_DEBUG_CTRL_0__DET_DEPTH__SHIFT                               0x10
-#define DCHUBBUB_DEBUG_CTRL_0__DET_DEPTH_MASK                                 0x01FF0000L
-
-#include "reg_helper.h"
-#include "dce/dmub_abm.h"
-#include "dce/dmub_psr.h"
-#include "dce/dce_aux.h"
-#include "dce/dce_i2c.h"
-#include "dce/dmub_replay.h"
-
-#include "dml/dcn30/display_mode_vba_30.h"
-#include "vm_helper.h"
-#include "dcn20/dcn20_vmid.h"
-
-#include "link_enc_cfg.h"
-
-#define DC_LOGGER \
-	dc->ctx->logger
-#define DC_LOGGER_INIT(logger)
-
-enum dcn31_clk_src_array_id {
-	DCN31_CLK_SRC_PLL0,
-	DCN31_CLK_SRC_PLL1,
-	DCN31_CLK_SRC_PLL2,
-	DCN31_CLK_SRC_PLL3,
-	DCN31_CLK_SRC_PLL4,
-	DCN30_CLK_SRC_TOTAL
-};
-
-/* begin *********************
- * macros to expend register list macro defined in HW object header file
- */
-
-/* DCN */
-#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
-
-#define BASE(seg) BASE_INNER(seg)
-
-#define SR(reg_name)\
-		.reg_name = BASE(reg ## reg_name ## _BASE_IDX) +  \
-					reg ## reg_name
-
-#define SRI(reg_name, block, id)\
-	.reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define SRI2(reg_name, block, id)\
-	.reg_name = BASE(reg ## reg_name ## _BASE_IDX) + \
-					reg ## reg_name
-
-#define SRIR(var_name, reg_name, block, id)\
-	.var_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define SRII_MPC_RMU(reg_name, block, id)\
-	.RMU##_##reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define SRII_DWB(reg_name, temp_name, block, id)\
-	.reg_name[id] = BASE(reg ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## temp_name
-
-#define SF_DWB2(reg_name, block, id, field_name, post_fix)	\
-	.field_name = reg_name ## __ ## field_name ## post_fix
-
-#define DCCG_SRII(reg_name, block, id)\
-	.block ## _ ## reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define VUPDATE_SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(reg ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
-					reg ## reg_name ## _ ## block ## id
-
-/* NBIO */
-#define NBIO_BASE_INNER(seg) \
-	NBIO_BASE__INST0_SEG ## seg
-
-#define NBIO_BASE(seg) \
-	NBIO_BASE_INNER(seg)
-
-#define NBIO_SR(reg_name)\
-		.reg_name = NBIO_BASE(regBIF_BX1_ ## reg_name ## _BASE_IDX) + \
-					regBIF_BX1_ ## reg_name
-
-/* MMHUB */
-#define MMHUB_BASE_INNER(seg) \
-	MMHUB_BASE__INST0_SEG ## seg
-
-#define MMHUB_BASE(seg) \
-	MMHUB_BASE_INNER(seg)
-
-#define MMHUB_SR(reg_name)\
-		.reg_name = MMHUB_BASE(mm ## reg_name ## _BASE_IDX) + \
-					mm ## reg_name
-
-/* CLOCK */
-#define CLK_BASE_INNER(seg) \
-	CLK_BASE__INST0_SEG ## seg
-
-#define CLK_BASE(seg) \
-	CLK_BASE_INNER(seg)
-
-#define CLK_SRI(reg_name, block, inst)\
-	.reg_name = CLK_BASE(reg ## block ## _ ## inst ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## _ ## inst ## _ ## reg_name
-
-
-static const struct bios_registers bios_regs = {
-		NBIO_SR(BIOS_SCRATCH_3),
-		NBIO_SR(BIOS_SCRATCH_6)
-};
-
-#define clk_src_regs(index, pllid)\
-[index] = {\
-	CS_COMMON_REG_LIST_DCN3_0(index, pllid),\
-}
-
-static const struct dce110_clk_src_regs clk_src_regs[] = {
-	clk_src_regs(0, A),
-	clk_src_regs(1, B),
-	clk_src_regs(2, C),
-	clk_src_regs(3, D),
-	clk_src_regs(4, E)
-};
-/*pll_id being rempped in dmub, in driver it is logical instance*/
-static const struct dce110_clk_src_regs clk_src_regs_b0[] = {
-	clk_src_regs(0, A),
-	clk_src_regs(1, B),
-	clk_src_regs(2, F),
-	clk_src_regs(3, G),
-	clk_src_regs(4, E)
-};
-
-static const struct dce110_clk_src_shift cs_shift = {
-		CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
-};
-
-static const struct dce110_clk_src_mask cs_mask = {
-		CS_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
-};
-
-#define abm_regs(id)\
-[id] = {\
-		ABM_DCN302_REG_LIST(id)\
-}
-
-static const struct dce_abm_registers abm_regs[] = {
-		abm_regs(0),
-		abm_regs(1),
-		abm_regs(2),
-		abm_regs(3),
-};
-
-static const struct dce_abm_shift abm_shift = {
-		ABM_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dce_abm_mask abm_mask = {
-		ABM_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define audio_regs(id)\
-[id] = {\
-		AUD_COMMON_REG_LIST(id)\
-}
-
-static const struct dce_audio_registers audio_regs[] = {
-	audio_regs(0),
-	audio_regs(1),
-	audio_regs(2),
-	audio_regs(3),
-	audio_regs(4),
-	audio_regs(5),
-	audio_regs(6)
-};
-
-#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
-		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
-
-static const struct dce_audio_shift audio_shift = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_audio_mask audio_mask = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
-};
-
-#define vpg_regs(id)\
-[id] = {\
-	VPG_DCN31_REG_LIST(id)\
-}
-
-static const struct dcn31_vpg_registers vpg_regs[] = {
-	vpg_regs(0),
-	vpg_regs(1),
-	vpg_regs(2),
-	vpg_regs(3),
-	vpg_regs(4),
-	vpg_regs(5),
-	vpg_regs(6),
-	vpg_regs(7),
-	vpg_regs(8),
-	vpg_regs(9),
-};
-
-static const struct dcn31_vpg_shift vpg_shift = {
-	DCN31_VPG_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_vpg_mask vpg_mask = {
-	DCN31_VPG_MASK_SH_LIST(_MASK)
-};
-
-#define afmt_regs(id)\
-[id] = {\
-	AFMT_DCN31_REG_LIST(id)\
-}
-
-static const struct dcn31_afmt_registers afmt_regs[] = {
-	afmt_regs(0),
-	afmt_regs(1),
-	afmt_regs(2),
-	afmt_regs(3),
-	afmt_regs(4),
-	afmt_regs(5)
-};
-
-static const struct dcn31_afmt_shift afmt_shift = {
-	DCN31_AFMT_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_afmt_mask afmt_mask = {
-	DCN31_AFMT_MASK_SH_LIST(_MASK)
-};
-
-#define apg_regs(id)\
-[id] = {\
-	APG_DCN31_REG_LIST(id)\
-}
-
-static const struct dcn31_apg_registers apg_regs[] = {
-	apg_regs(0),
-	apg_regs(1),
-	apg_regs(2),
-	apg_regs(3)
-};
-
-static const struct dcn31_apg_shift apg_shift = {
-	DCN31_APG_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_apg_mask apg_mask = {
-		DCN31_APG_MASK_SH_LIST(_MASK)
-};
-
-#define stream_enc_regs(id)\
-[id] = {\
-	SE_DCN3_REG_LIST(id)\
-}
-
-/* Some encoders won't be initialized here - but they're logical, not physical. */
-static const struct dcn10_stream_enc_registers stream_enc_regs[ENGINE_ID_COUNT] = {
-	stream_enc_regs(0),
-	stream_enc_regs(1),
-	stream_enc_regs(2),
-	stream_enc_regs(3),
-	stream_enc_regs(4)
-};
-
-static const struct dcn10_stream_encoder_shift se_shift = {
-		SE_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn10_stream_encoder_mask se_mask = {
-		SE_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-
-#define aux_regs(id)\
-[id] = {\
-	DCN2_AUX_REG_LIST(id)\
-}
-
-static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
-		aux_regs(0),
-		aux_regs(1),
-		aux_regs(2),
-		aux_regs(3),
-		aux_regs(4)
-};
-
-#define hpd_regs(id)\
-[id] = {\
-	HPD_REG_LIST(id)\
-}
-
-static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
-		hpd_regs(0),
-		hpd_regs(1),
-		hpd_regs(2),
-		hpd_regs(3),
-		hpd_regs(4)
-};
-
-#define link_regs(id, phyid)\
-[id] = {\
-	LE_DCN31_REG_LIST(id), \
-	UNIPHY_DCN2_REG_LIST(phyid), \
-	DPCS_DCN31_REG_LIST(id), \
-}
-
-static const struct dce110_aux_registers_shift aux_shift = {
-	DCN_AUX_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce110_aux_registers_mask aux_mask = {
-	DCN_AUX_MASK_SH_LIST(_MASK)
-};
-
-static const struct dcn10_link_enc_registers link_enc_regs[] = {
-	link_regs(0, A),
-	link_regs(1, B),
-	link_regs(2, C),
-	link_regs(3, D),
-	link_regs(4, E)
-};
-
-static const struct dcn10_link_enc_shift le_shift = {
-	LINK_ENCODER_MASK_SH_LIST_DCN31(__SHIFT), \
-	DPCS_DCN31_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn10_link_enc_mask le_mask = {
-	LINK_ENCODER_MASK_SH_LIST_DCN31(_MASK), \
-	DPCS_DCN31_MASK_SH_LIST(_MASK)
-};
-
-#define hpo_dp_stream_encoder_reg_list(id)\
-[id] = {\
-	DCN3_1_HPO_DP_STREAM_ENC_REG_LIST(id)\
-}
-
-static const struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[] = {
-	hpo_dp_stream_encoder_reg_list(0),
-	hpo_dp_stream_encoder_reg_list(1),
-	hpo_dp_stream_encoder_reg_list(2),
-	hpo_dp_stream_encoder_reg_list(3),
-};
-
-static const struct dcn31_hpo_dp_stream_encoder_shift hpo_dp_se_shift = {
-	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_hpo_dp_stream_encoder_mask hpo_dp_se_mask = {
-	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(_MASK)
-};
-
-#define hpo_dp_link_encoder_reg_list(id)\
-[id] = {\
-	DCN3_1_HPO_DP_LINK_ENC_REG_LIST(id),\
-	DCN3_1_RDPCSTX_REG_LIST(0),\
-	DCN3_1_RDPCSTX_REG_LIST(1),\
-	DCN3_1_RDPCSTX_REG_LIST(2),\
-	DCN3_1_RDPCSTX_REG_LIST(3),\
-	DCN3_1_RDPCSTX_REG_LIST(4)\
-}
-
-static const struct dcn31_hpo_dp_link_encoder_registers hpo_dp_link_enc_regs[] = {
-	hpo_dp_link_encoder_reg_list(0),
-	hpo_dp_link_encoder_reg_list(1),
-};
-
-static const struct dcn31_hpo_dp_link_encoder_shift hpo_dp_le_shift = {
-	DCN3_1_HPO_DP_LINK_ENC_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_hpo_dp_link_encoder_mask hpo_dp_le_mask = {
-	DCN3_1_HPO_DP_LINK_ENC_MASK_SH_LIST(_MASK)
-};
-
-#define dpp_regs(id)\
-[id] = {\
-	DPP_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn3_dpp_registers dpp_regs[] = {
-	dpp_regs(0),
-	dpp_regs(1),
-	dpp_regs(2),
-	dpp_regs(3)
-};
-
-static const struct dcn3_dpp_shift tf_shift = {
-		DPP_REG_LIST_SH_MASK_DCN30(__SHIFT)
-};
-
-static const struct dcn3_dpp_mask tf_mask = {
-		DPP_REG_LIST_SH_MASK_DCN30(_MASK)
-};
-
-#define opp_regs(id)\
-[id] = {\
-	OPP_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn20_opp_registers opp_regs[] = {
-	opp_regs(0),
-	opp_regs(1),
-	opp_regs(2),
-	opp_regs(3)
-};
-
-static const struct dcn20_opp_shift opp_shift = {
-	OPP_MASK_SH_LIST_DCN20(__SHIFT)
-};
-
-static const struct dcn20_opp_mask opp_mask = {
-	OPP_MASK_SH_LIST_DCN20(_MASK)
-};
-
-#define aux_engine_regs(id)\
-[id] = {\
-	AUX_COMMON_REG_LIST0(id), \
-	.AUXN_IMPCAL = 0, \
-	.AUXP_IMPCAL = 0, \
-	.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
-}
-
-static const struct dce110_aux_registers aux_engine_regs[] = {
-		aux_engine_regs(0),
-		aux_engine_regs(1),
-		aux_engine_regs(2),
-		aux_engine_regs(3),
-		aux_engine_regs(4)
-};
-
-#define dwbc_regs_dcn3(id)\
-[id] = {\
-	DWBC_COMMON_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn30_dwbc_registers dwbc30_regs[] = {
-	dwbc_regs_dcn3(0),
-};
-
-static const struct dcn30_dwbc_shift dwbc30_shift = {
-	DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_dwbc_mask dwbc30_mask = {
-	DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define mcif_wb_regs_dcn3(id)\
-[id] = {\
-	MCIF_WB_COMMON_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn30_mmhubbub_registers mcif_wb30_regs[] = {
-	mcif_wb_regs_dcn3(0)
-};
-
-static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
-	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
-	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define dsc_regsDCN20(id)\
-[id] = {\
-	DSC_REG_LIST_DCN20(id)\
-}
-
-static const struct dcn20_dsc_registers dsc_regs[] = {
-	dsc_regsDCN20(0),
-	dsc_regsDCN20(1),
-	dsc_regsDCN20(2)
-};
-
-static const struct dcn20_dsc_shift dsc_shift = {
-	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
-};
-
-static const struct dcn20_dsc_mask dsc_mask = {
-	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
-};
-
-static const struct dcn30_mpc_registers mpc_regs = {
-		MPC_REG_LIST_DCN3_0(0),
-		MPC_REG_LIST_DCN3_0(1),
-		MPC_REG_LIST_DCN3_0(2),
-		MPC_REG_LIST_DCN3_0(3),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(0),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(1),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(2),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(3),
-		MPC_RMU_GLOBAL_REG_LIST_DCN3AG,
-		MPC_RMU_REG_LIST_DCN3AG(0),
-		MPC_RMU_REG_LIST_DCN3AG(1),
-		//MPC_RMU_REG_LIST_DCN3AG(2),
-		MPC_DWB_MUX_REG_LIST_DCN3_0(0),
-};
-
-static const struct dcn30_mpc_shift mpc_shift = {
-	MPC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_mpc_mask mpc_mask = {
-	MPC_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define optc_regs(id)\
-[id] = {OPTC_COMMON_REG_LIST_DCN3_1(id)}
-
-static const struct dcn_optc_registers optc_regs[] = {
-	optc_regs(0),
-	optc_regs(1),
-	optc_regs(2),
-	optc_regs(3)
-};
-
-static const struct dcn_optc_shift optc_shift = {
-	OPTC_COMMON_MASK_SH_LIST_DCN3_1(__SHIFT)
-};
-
-static const struct dcn_optc_mask optc_mask = {
-	OPTC_COMMON_MASK_SH_LIST_DCN3_1(_MASK)
-};
-
-#define hubp_regs(id)\
-[id] = {\
-	HUBP_REG_LIST_DCN30(id)\
-}
-
-static const struct dcn_hubp2_registers hubp_regs[] = {
-		hubp_regs(0),
-		hubp_regs(1),
-		hubp_regs(2),
-		hubp_regs(3)
-};
-
-
-static const struct dcn_hubp2_shift hubp_shift = {
-		HUBP_MASK_SH_LIST_DCN31(__SHIFT)
-};
-
-static const struct dcn_hubp2_mask hubp_mask = {
-		HUBP_MASK_SH_LIST_DCN31(_MASK)
-};
-static const struct dcn_hubbub_registers hubbub_reg = {
-		HUBBUB_REG_LIST_DCN31(0)
-};
-
-static const struct dcn_hubbub_shift hubbub_shift = {
-		HUBBUB_MASK_SH_LIST_DCN31(__SHIFT)
-};
-
-static const struct dcn_hubbub_mask hubbub_mask = {
-		HUBBUB_MASK_SH_LIST_DCN31(_MASK)
-};
-
-static const struct dccg_registers dccg_regs = {
-		DCCG_REG_LIST_DCN31()
-};
-
-static const struct dccg_shift dccg_shift = {
-		DCCG_MASK_SH_LIST_DCN31(__SHIFT)
-};
-
-static const struct dccg_mask dccg_mask = {
-		DCCG_MASK_SH_LIST_DCN31(_MASK)
-};
-
-
-#define SRII2(reg_name_pre, reg_name_post, id)\
-	.reg_name_pre ## _ ##  reg_name_post[id] = BASE(reg ## reg_name_pre \
-			## id ## _ ## reg_name_post ## _BASE_IDX) + \
-			reg ## reg_name_pre ## id ## _ ## reg_name_post
-
-
-#define HWSEQ_DCN31_REG_LIST()\
-	SR(DCHUBBUB_GLOBAL_TIMER_CNTL), \
-	SR(DCHUBBUB_ARB_HOSTVM_CNTL), \
-	SR(DIO_MEM_PWR_CTRL), \
-	SR(ODM_MEM_PWR_CTRL3), \
-	SR(DMU_MEM_PWR_CNTL), \
-	SR(MMHUBBUB_MEM_PWR_CNTL), \
-	SR(DCCG_GATE_DISABLE_CNTL), \
-	SR(DCCG_GATE_DISABLE_CNTL2), \
-	SR(DCFCLK_CNTL),\
-	SR(DC_MEM_GLOBAL_PWR_REQ_CNTL), \
-	SRII(PIXEL_RATE_CNTL, OTG, 0), \
-	SRII(PIXEL_RATE_CNTL, OTG, 1),\
-	SRII(PIXEL_RATE_CNTL, OTG, 2),\
-	SRII(PIXEL_RATE_CNTL, OTG, 3),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 0),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 1),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 2),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 3),\
-	SR(MICROSECOND_TIME_BASE_DIV), \
-	SR(MILLISECOND_TIME_BASE_DIV), \
-	SR(DISPCLK_FREQ_CHANGE_CNTL), \
-	SR(RBBMIF_TIMEOUT_DIS), \
-	SR(RBBMIF_TIMEOUT_DIS_2), \
-	SR(DCHUBBUB_CRC_CTRL), \
-	SR(DPP_TOP0_DPP_CRC_CTRL), \
-	SR(DPP_TOP0_DPP_CRC_VAL_B_A), \
-	SR(DPP_TOP0_DPP_CRC_VAL_R_G), \
-	SR(MPC_CRC_CTRL), \
-	SR(MPC_CRC_RESULT_GB), \
-	SR(MPC_CRC_RESULT_C), \
-	SR(MPC_CRC_RESULT_AR), \
-	SR(DOMAIN0_PG_CONFIG), \
-	SR(DOMAIN1_PG_CONFIG), \
-	SR(DOMAIN2_PG_CONFIG), \
-	SR(DOMAIN3_PG_CONFIG), \
-	SR(DOMAIN16_PG_CONFIG), \
-	SR(DOMAIN17_PG_CONFIG), \
-	SR(DOMAIN18_PG_CONFIG), \
-	SR(DOMAIN0_PG_STATUS), \
-	SR(DOMAIN1_PG_STATUS), \
-	SR(DOMAIN2_PG_STATUS), \
-	SR(DOMAIN3_PG_STATUS), \
-	SR(DOMAIN16_PG_STATUS), \
-	SR(DOMAIN17_PG_STATUS), \
-	SR(DOMAIN18_PG_STATUS), \
-	SR(D1VGA_CONTROL), \
-	SR(D2VGA_CONTROL), \
-	SR(D3VGA_CONTROL), \
-	SR(D4VGA_CONTROL), \
-	SR(D5VGA_CONTROL), \
-	SR(D6VGA_CONTROL), \
-	SR(DC_IP_REQUEST_CNTL), \
-	SR(AZALIA_AUDIO_DTO), \
-	SR(AZALIA_CONTROLLER_CLOCK_GATING), \
-	SR(HPO_TOP_HW_CONTROL)
-
-static const struct dce_hwseq_registers hwseq_reg = {
-		HWSEQ_DCN31_REG_LIST()
-};
-
-#define HWSEQ_DCN31_MASK_SH_LIST(mask_sh)\
-	HWSEQ_DCN_MASK_SH_LIST(mask_sh), \
-	HWS_SF(, DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, mask_sh), \
-	HWS_SF(, DCHUBBUB_ARB_HOSTVM_CNTL, DISABLE_HOSTVM_FORCE_ALLOW_PSTATE, mask_sh), \
-	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN0_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN1_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN2_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN3_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN16_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN17_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN18_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh), \
-	HWS_SF(, AZALIA_AUDIO_DTO, AZALIA_AUDIO_DTO_MODULE, mask_sh), \
-	HWS_SF(, HPO_TOP_CLOCK_CONTROL, HPO_HDMISTREAMCLK_G_GATE_DIS, mask_sh), \
-	HWS_SF(, DMU_MEM_PWR_CNTL, DMCU_ERAM_MEM_PWR_FORCE, mask_sh), \
-	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_UNASSIGNED_PWR_MODE, mask_sh), \
-	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_VBLANK_PWR_MODE, mask_sh), \
-	HWS_SF(, MMHUBBUB_MEM_PWR_CNTL, VGA_MEM_PWR_FORCE, mask_sh), \
-	HWS_SF(, DIO_MEM_PWR_CTRL, I2C_LIGHT_SLEEP_FORCE, mask_sh), \
-	HWS_SF(, HPO_TOP_HW_CONTROL, HPO_IO_EN, mask_sh)
-
-static const struct dce_hwseq_shift hwseq_shift = {
-		HWSEQ_DCN31_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_hwseq_mask hwseq_mask = {
-		HWSEQ_DCN31_MASK_SH_LIST(_MASK)
-};
-#define vmid_regs(id)\
-[id] = {\
-		DCN20_VMID_REG_LIST(id)\
-}
-
-static const struct dcn_vmid_registers vmid_regs[] = {
-	vmid_regs(0),
-	vmid_regs(1),
-	vmid_regs(2),
-	vmid_regs(3),
-	vmid_regs(4),
-	vmid_regs(5),
-	vmid_regs(6),
-	vmid_regs(7),
-	vmid_regs(8),
-	vmid_regs(9),
-	vmid_regs(10),
-	vmid_regs(11),
-	vmid_regs(12),
-	vmid_regs(13),
-	vmid_regs(14),
-	vmid_regs(15)
-};
-
-static const struct dcn20_vmid_shift vmid_shifts = {
-		DCN20_VMID_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn20_vmid_mask vmid_masks = {
-		DCN20_VMID_MASK_SH_LIST(_MASK)
-};
-
-static const struct resource_caps res_cap_dcn31 = {
-	.num_timing_generator = 4,
-	.num_opp = 4,
-	.num_video_plane = 4,
-	.num_audio = 5,
-	.num_stream_encoder = 5,
-	.num_dig_link_enc = 5,
-	.num_hpo_dp_stream_encoder = 4,
-	.num_hpo_dp_link_encoder = 2,
-	.num_pll = 5,
-	.num_dwb = 1,
-	.num_ddc = 5,
-	.num_vmid = 16,
-	.num_mpc_3dlut = 2,
-	.num_dsc = 3,
-};
-
-static const struct dc_plane_cap plane_cap = {
-	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.per_pixel_alpha = true,
-
-	.pixel_format_support = {
-			.argb8888 = true,
-			.nv12 = true,
-			.fp16 = true,
-			.p010 = true,
-			.ayuv = false,
-	},
-
-	.max_upscale_factor = {
-			.argb8888 = 16000,
-			.nv12 = 16000,
-			.fp16 = 16000
-	},
-
-	// 6:1 downscaling ratio: 1000/6 = 166.666
-	.max_downscale_factor = {
-			.argb8888 = 167,
-			.nv12 = 167,
-			.fp16 = 167
-	},
-	64,
-	64
-};
-
-static const struct dc_debug_options debug_defaults_drv = {
-	.disable_dmcu = true,
-	.force_abm_enable = false,
-	.timing_trace = false,
-	.clock_trace = true,
-	.disable_pplib_clock_request = false,
-	.pipe_split_policy = MPC_SPLIT_DYNAMIC,
-	.force_single_disp_pipe_split = false,
-	.disable_dcc = DCC_ENABLE,
-	.vsr_support = true,
-	.performance_trace = false,
-	.max_downscale_src_width = 4096,/*upto true 4K*/
-	.disable_pplib_wm_range = false,
-	.scl_reset_length10 = true,
-	.sanity_checks = true,
-	.underflow_assert_delay_us = 0xFFFFFFFF,
-	.dwb_fi_phase = -1, // -1 = disable,
-	.dmub_command_table = true,
-	.pstate_enabled = true,
-	.use_max_lb = true,
-	.enable_mem_low_power = {
-		.bits = {
-			.vga = true,
-			.i2c = true,
-			.dmcu = false, // This is previously known to cause hang on S3 cycles if enabled
-			.dscl = true,
-			.cm = true,
-			.mpc = true,
-			.optc = true,
-			.vpg = true,
-			.afmt = true,
-		}
-	},
-	.disable_z10 = true,
-	.enable_legacy_fast_update = true,
-	.enable_z9_disable_interface = true, /* Allow support for the PMFW interface for disable Z9*/
-	.dml_hostvm_override = DML_HOSTVM_OVERRIDE_FALSE,
-	.using_dml2 = false,
-};
-
-static const struct dc_panel_config panel_config_defaults = {
-	.psr = {
-		.disable_psr = false,
-		.disallow_psrsu = false,
-		.disallow_replay = false,
-	},
-	.ilr = {
-		.optimize_edp_link_rate = true,
-	},
-};
-
-static void dcn31_dpp_destroy(struct dpp **dpp)
-{
-	kfree(TO_DCN20_DPP(*dpp));
-	*dpp = NULL;
-}
-
-static struct dpp *dcn31_dpp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn3_dpp *dpp =
-		kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
-
-	if (!dpp)
-		return NULL;
-
-	if (dpp3_construct(dpp, ctx, inst,
-			&dpp_regs[inst], &tf_shift, &tf_mask))
-		return &dpp->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(dpp);
-	return NULL;
-}
-
-static struct output_pixel_processor *dcn31_opp_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_opp *opp =
-		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
-
-	if (!opp) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dcn20_opp_construct(opp, ctx, inst,
-			&opp_regs[inst], &opp_shift, &opp_mask);
-	return &opp->base;
-}
-
-static struct dce_aux *dcn31_aux_engine_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct aux_engine_dce110 *aux_engine =
-		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
-
-	if (!aux_engine)
-		return NULL;
-
-	dce110_aux_engine_construct(aux_engine, ctx, inst,
-				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
-				    &aux_engine_regs[inst],
-					&aux_mask,
-					&aux_shift,
-					ctx->dc->caps.extended_aux_timeout_support);
-
-	return &aux_engine->base;
-}
-#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST_DCN30(id) }
-
-static const struct dce_i2c_registers i2c_hw_regs[] = {
-		i2c_inst_regs(1),
-		i2c_inst_regs(2),
-		i2c_inst_regs(3),
-		i2c_inst_regs(4),
-		i2c_inst_regs(5),
-};
-
-static const struct dce_i2c_shift i2c_shifts = {
-		I2C_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dce_i2c_mask i2c_masks = {
-		I2C_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static struct dce_i2c_hw *dcn31_i2c_hw_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_i2c_hw *dce_i2c_hw =
-		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
-
-	if (!dce_i2c_hw)
-		return NULL;
-
-	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
-				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
-
-	return dce_i2c_hw;
-}
-static struct mpc *dcn31_mpc_create(
-		struct dc_context *ctx,
-		int num_mpcc,
-		int num_rmu)
-{
-	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc),
-					  GFP_KERNEL);
-
-	if (!mpc30)
-		return NULL;
-
-	dcn30_mpc_construct(mpc30, ctx,
-			&mpc_regs,
-			&mpc_shift,
-			&mpc_mask,
-			num_mpcc,
-			num_rmu);
-
-	return &mpc30->base;
-}
-
-static struct hubbub *dcn31_hubbub_create(struct dc_context *ctx)
-{
-	int i;
-
-	struct dcn20_hubbub *hubbub3 = kzalloc(sizeof(struct dcn20_hubbub),
-					  GFP_KERNEL);
-
-	if (!hubbub3)
-		return NULL;
-
-	hubbub31_construct(hubbub3, ctx,
-			&hubbub_reg,
-			&hubbub_shift,
-			&hubbub_mask,
-			dcn3_1_ip.det_buffer_size_kbytes,
-			dcn3_1_ip.pixel_chunk_size_kbytes,
-			dcn3_1_ip.config_return_buffer_size_in_kbytes);
-
-
-	for (i = 0; i < res_cap_dcn31.num_vmid; i++) {
-		struct dcn20_vmid *vmid = &hubbub3->vmid[i];
-
-		vmid->ctx = ctx;
-
-		vmid->regs = &vmid_regs[i];
-		vmid->shifts = &vmid_shifts;
-		vmid->masks = &vmid_masks;
-	}
-
-	return &hubbub3->base;
-}
-
-static struct timing_generator *dcn31_timing_generator_create(
-		struct dc_context *ctx,
-		uint32_t instance)
-{
-	struct optc *tgn10 =
-		kzalloc(sizeof(struct optc), GFP_KERNEL);
-
-	if (!tgn10)
-		return NULL;
-
-	tgn10->base.inst = instance;
-	tgn10->base.ctx = ctx;
-
-	tgn10->tg_regs = &optc_regs[instance];
-	tgn10->tg_shift = &optc_shift;
-	tgn10->tg_mask = &optc_mask;
-
-	dcn31_timing_generator_init(tgn10);
-
-	return &tgn10->base;
-}
-
-static const struct encoder_feature_support link_enc_feature = {
-		.max_hdmi_deep_color = COLOR_DEPTH_121212,
-		.max_hdmi_pixel_clock = 600000,
-		.hdmi_ycbcr420_supported = true,
-		.dp_ycbcr420_supported = true,
-		.fec_supported = true,
-		.flags.bits.IS_HBR2_CAPABLE = true,
-		.flags.bits.IS_HBR3_CAPABLE = true,
-		.flags.bits.IS_TPS3_CAPABLE = true,
-		.flags.bits.IS_TPS4_CAPABLE = true
-};
-
-static struct link_encoder *dcn31_link_encoder_create(
-	struct dc_context *ctx,
-	const struct encoder_init_data *enc_init_data)
-{
-	struct dcn20_link_encoder *enc20 =
-		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
-
-	if (!enc20)
-		return NULL;
-
-	dcn31_link_encoder_construct(enc20,
-			enc_init_data,
-			&link_enc_feature,
-			&link_enc_regs[enc_init_data->transmitter],
-			&link_enc_aux_regs[enc_init_data->channel - 1],
-			&link_enc_hpd_regs[enc_init_data->hpd_source],
-			&le_shift,
-			&le_mask);
-
-	return &enc20->enc10.base;
-}
-
-/* Create a minimal link encoder object not associated with a particular
- * physical connector.
- * resource_funcs.link_enc_create_minimal
- */
-static struct link_encoder *dcn31_link_enc_create_minimal(
-		struct dc_context *ctx, enum engine_id eng_id)
-{
-	struct dcn20_link_encoder *enc20;
-
-	if ((eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
-		return NULL;
-
-	enc20 = kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
-	if (!enc20)
-		return NULL;
-
-	dcn31_link_encoder_construct_minimal(
-			enc20,
-			ctx,
-			&link_enc_feature,
-			&link_enc_regs[eng_id - ENGINE_ID_DIGA],
-			eng_id);
-
-	return &enc20->enc10.base;
-}
-
-static struct panel_cntl *dcn31_panel_cntl_create(const struct panel_cntl_init_data *init_data)
-{
-	struct dcn31_panel_cntl *panel_cntl =
-		kzalloc(sizeof(struct dcn31_panel_cntl), GFP_KERNEL);
-
-	if (!panel_cntl)
-		return NULL;
-
-	dcn31_panel_cntl_construct(panel_cntl, init_data);
-
-	return &panel_cntl->base;
-}
-
-static void read_dce_straps(
-	struct dc_context *ctx,
-	struct resource_straps *straps)
-{
-	generic_reg_get(ctx, regDC_PINSTRAPS + BASE(regDC_PINSTRAPS_BASE_IDX),
-		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
-
-}
-
-static struct audio *dcn31_create_audio(
-		struct dc_context *ctx, unsigned int inst)
-{
-	return dce_audio_create(ctx, inst,
-			&audio_regs[inst], &audio_shift, &audio_mask);
-}
-
-static struct vpg *dcn31_vpg_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn31_vpg *vpg31 = kzalloc(sizeof(struct dcn31_vpg), GFP_KERNEL);
-
-	if (!vpg31)
-		return NULL;
-
-	vpg31_construct(vpg31, ctx, inst,
-			&vpg_regs[inst],
-			&vpg_shift,
-			&vpg_mask);
-
-	return &vpg31->base;
-}
-
-static struct afmt *dcn31_afmt_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn31_afmt *afmt31 = kzalloc(sizeof(struct dcn31_afmt), GFP_KERNEL);
-
-	if (!afmt31)
-		return NULL;
-
-	afmt31_construct(afmt31, ctx, inst,
-			&afmt_regs[inst],
-			&afmt_shift,
-			&afmt_mask);
-
-	// Light sleep by default, no need to power down here
-
-	return &afmt31->base;
-}
-
-static struct apg *dcn31_apg_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn31_apg *apg31 = kzalloc(sizeof(struct dcn31_apg), GFP_KERNEL);
-
-	if (!apg31)
-		return NULL;
-
-	apg31_construct(apg31, ctx, inst,
-			&apg_regs[inst],
-			&apg_shift,
-			&apg_mask);
-
-	return &apg31->base;
-}
-
-static struct stream_encoder *dcn31_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dcn10_stream_encoder *enc1;
-	struct vpg *vpg;
-	struct afmt *afmt;
-	int vpg_inst;
-	int afmt_inst;
-
-	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
-	if (eng_id <= ENGINE_ID_DIGF) {
-		vpg_inst = eng_id;
-		afmt_inst = eng_id;
-	} else
-		return NULL;
-
-	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
-	vpg = dcn31_vpg_create(ctx, vpg_inst);
-	afmt = dcn31_afmt_create(ctx, afmt_inst);
-
-	if (!enc1 || !vpg || !afmt) {
-		kfree(enc1);
-		kfree(vpg);
-		kfree(afmt);
-		return NULL;
-	}
-
-	dcn30_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
-					eng_id, vpg, afmt,
-					&stream_enc_regs[eng_id],
-					&se_shift, &se_mask);
-
-	return &enc1->base;
-}
-
-static struct hpo_dp_stream_encoder *dcn31_hpo_dp_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dcn31_hpo_dp_stream_encoder *hpo_dp_enc31;
-	struct vpg *vpg;
-	struct apg *apg;
-	uint32_t hpo_dp_inst;
-	uint32_t vpg_inst;
-	uint32_t apg_inst;
-
-	ASSERT((eng_id >= ENGINE_ID_HPO_DP_0) && (eng_id <= ENGINE_ID_HPO_DP_3));
-	hpo_dp_inst = eng_id - ENGINE_ID_HPO_DP_0;
-
-	/* Mapping of VPG register blocks to HPO DP block instance:
-	 * VPG[6] -> HPO_DP[0]
-	 * VPG[7] -> HPO_DP[1]
-	 * VPG[8] -> HPO_DP[2]
-	 * VPG[9] -> HPO_DP[3]
-	 */
-	vpg_inst = hpo_dp_inst + 6;
-
-	/* Mapping of APG register blocks to HPO DP block instance:
-	 * APG[0] -> HPO_DP[0]
-	 * APG[1] -> HPO_DP[1]
-	 * APG[2] -> HPO_DP[2]
-	 * APG[3] -> HPO_DP[3]
-	 */
-	apg_inst = hpo_dp_inst;
-
-	/* allocate HPO stream encoder and create VPG sub-block */
-	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_stream_encoder), GFP_KERNEL);
-	vpg = dcn31_vpg_create(ctx, vpg_inst);
-	apg = dcn31_apg_create(ctx, apg_inst);
-
-	if (!hpo_dp_enc31 || !vpg || !apg) {
-		kfree(hpo_dp_enc31);
-		kfree(vpg);
-		kfree(apg);
-		return NULL;
-	}
-
-	dcn31_hpo_dp_stream_encoder_construct(hpo_dp_enc31, ctx, ctx->dc_bios,
-					hpo_dp_inst, eng_id, vpg, apg,
-					&hpo_dp_stream_enc_regs[hpo_dp_inst],
-					&hpo_dp_se_shift, &hpo_dp_se_mask);
-
-	return &hpo_dp_enc31->base;
-}
-
-static struct hpo_dp_link_encoder *dcn31_hpo_dp_link_encoder_create(
-	uint8_t inst,
-	struct dc_context *ctx)
-{
-	struct dcn31_hpo_dp_link_encoder *hpo_dp_enc31;
-
-	/* allocate HPO link encoder */
-	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
-
-	hpo_dp_link_encoder31_construct(hpo_dp_enc31, ctx, inst,
-					&hpo_dp_link_enc_regs[inst],
-					&hpo_dp_le_shift, &hpo_dp_le_mask);
-
-	return &hpo_dp_enc31->base;
-}
-
-static struct dce_hwseq *dcn31_hwseq_create(
-	struct dc_context *ctx)
-{
-	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
-
-	if (hws) {
-		hws->ctx = ctx;
-		hws->regs = &hwseq_reg;
-		hws->shifts = &hwseq_shift;
-		hws->masks = &hwseq_mask;
-	}
-	return hws;
-}
-static const struct resource_create_funcs res_create_funcs = {
-	.read_dce_straps = read_dce_straps,
-	.create_audio = dcn31_create_audio,
-	.create_stream_encoder = dcn31_stream_encoder_create,
-	.create_hpo_dp_stream_encoder = dcn31_hpo_dp_stream_encoder_create,
-	.create_hpo_dp_link_encoder = dcn31_hpo_dp_link_encoder_create,
-	.create_hwseq = dcn31_hwseq_create,
-};
-
-static void dcn31_resource_destruct(struct dcn31_resource_pool *pool)
-{
-	unsigned int i;
-
-	for (i = 0; i < pool->base.stream_enc_count; i++) {
-		if (pool->base.stream_enc[i] != NULL) {
-			if (pool->base.stream_enc[i]->vpg != NULL) {
-				kfree(DCN30_VPG_FROM_VPG(pool->base.stream_enc[i]->vpg));
-				pool->base.stream_enc[i]->vpg = NULL;
-			}
-			if (pool->base.stream_enc[i]->afmt != NULL) {
-				kfree(DCN30_AFMT_FROM_AFMT(pool->base.stream_enc[i]->afmt));
-				pool->base.stream_enc[i]->afmt = NULL;
-			}
-			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
-			pool->base.stream_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.hpo_dp_stream_enc_count; i++) {
-		if (pool->base.hpo_dp_stream_enc[i] != NULL) {
-			if (pool->base.hpo_dp_stream_enc[i]->vpg != NULL) {
-				kfree(DCN30_VPG_FROM_VPG(pool->base.hpo_dp_stream_enc[i]->vpg));
-				pool->base.hpo_dp_stream_enc[i]->vpg = NULL;
-			}
-			if (pool->base.hpo_dp_stream_enc[i]->apg != NULL) {
-				kfree(DCN31_APG_FROM_APG(pool->base.hpo_dp_stream_enc[i]->apg));
-				pool->base.hpo_dp_stream_enc[i]->apg = NULL;
-			}
-			kfree(DCN3_1_HPO_DP_STREAM_ENC_FROM_HPO_STREAM_ENC(pool->base.hpo_dp_stream_enc[i]));
-			pool->base.hpo_dp_stream_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.hpo_dp_link_enc_count; i++) {
-		if (pool->base.hpo_dp_link_enc[i] != NULL) {
-			kfree(DCN3_1_HPO_DP_LINK_ENC_FROM_HPO_LINK_ENC(pool->base.hpo_dp_link_enc[i]));
-			pool->base.hpo_dp_link_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		if (pool->base.dscs[i] != NULL)
-			dcn20_dsc_destroy(&pool->base.dscs[i]);
-	}
-
-	if (pool->base.mpc != NULL) {
-		kfree(TO_DCN20_MPC(pool->base.mpc));
-		pool->base.mpc = NULL;
-	}
-	if (pool->base.hubbub != NULL) {
-		kfree(pool->base.hubbub);
-		pool->base.hubbub = NULL;
-	}
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		if (pool->base.dpps[i] != NULL)
-			dcn31_dpp_destroy(&pool->base.dpps[i]);
-
-		if (pool->base.ipps[i] != NULL)
-			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
-
-		if (pool->base.hubps[i] != NULL) {
-			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
-			pool->base.hubps[i] = NULL;
-		}
-
-		if (pool->base.irqs != NULL) {
-			dal_irq_service_destroy(&pool->base.irqs);
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		if (pool->base.engines[i] != NULL)
-			dce110_engine_destroy(&pool->base.engines[i]);
-		if (pool->base.hw_i2cs[i] != NULL) {
-			kfree(pool->base.hw_i2cs[i]);
-			pool->base.hw_i2cs[i] = NULL;
-		}
-		if (pool->base.sw_i2cs[i] != NULL) {
-			kfree(pool->base.sw_i2cs[i]);
-			pool->base.sw_i2cs[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
-		if (pool->base.opps[i] != NULL)
-			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		if (pool->base.timing_generators[i] != NULL)	{
-			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
-			pool->base.timing_generators[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
-		if (pool->base.dwbc[i] != NULL) {
-			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
-			pool->base.dwbc[i] = NULL;
-		}
-		if (pool->base.mcif_wb[i] != NULL) {
-			kfree(TO_DCN30_MMHUBBUB(pool->base.mcif_wb[i]));
-			pool->base.mcif_wb[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.audio_count; i++) {
-		if (pool->base.audios[i])
-			dce_aud_destroy(&pool->base.audios[i]);
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] != NULL) {
-			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
-			pool->base.clock_sources[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
-		if (pool->base.mpc_lut[i] != NULL) {
-			dc_3dlut_func_release(pool->base.mpc_lut[i]);
-			pool->base.mpc_lut[i] = NULL;
-		}
-		if (pool->base.mpc_shaper[i] != NULL) {
-			dc_transfer_func_release(pool->base.mpc_shaper[i]);
-			pool->base.mpc_shaper[i] = NULL;
-		}
-	}
-
-	if (pool->base.dp_clock_source != NULL) {
-		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
-		pool->base.dp_clock_source = NULL;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		if (pool->base.multiple_abms[i] != NULL)
-			dce_abm_destroy(&pool->base.multiple_abms[i]);
-	}
-
-	if (pool->base.psr != NULL)
-		dmub_psr_destroy(&pool->base.psr);
-
-	if (pool->base.replay != NULL)
-		dmub_replay_destroy(&pool->base.replay);
-
-	if (pool->base.dccg != NULL)
-		dcn_dccg_destroy(&pool->base.dccg);
-}
-
-static struct hubp *dcn31_hubp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn20_hubp *hubp2 =
-		kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
-
-	if (!hubp2)
-		return NULL;
-
-	if (hubp31_construct(hubp2, ctx, inst,
-			&hubp_regs[inst], &hubp_shift, &hubp_mask))
-		return &hubp2->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(hubp2);
-	return NULL;
-}
-
-static bool dcn31_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t pipe_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < pipe_count; i++) {
-		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc),
-						    GFP_KERNEL);
-
-		if (!dwbc30) {
-			dm_error("DC: failed to create dwbc30!\n");
-			return false;
-		}
-
-		dcn30_dwbc_construct(dwbc30, ctx,
-				&dwbc30_regs[i],
-				&dwbc30_shift,
-				&dwbc30_mask,
-				i);
-
-		pool->dwbc[i] = &dwbc30->base;
-	}
-	return true;
-}
-
-static bool dcn31_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t pipe_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < pipe_count; i++) {
-		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub),
-						    GFP_KERNEL);
-
-		if (!mcif_wb30) {
-			dm_error("DC: failed to create mcif_wb30!\n");
-			return false;
-		}
-
-		dcn30_mmhubbub_construct(mcif_wb30, ctx,
-				&mcif_wb30_regs[i],
-				&mcif_wb30_shift,
-				&mcif_wb30_mask,
-				i);
-
-		pool->mcif_wb[i] = &mcif_wb30->base;
-	}
-	return true;
-}
-
-static struct display_stream_compressor *dcn31_dsc_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_dsc *dsc =
-		kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
-
-	if (!dsc) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
-	return &dsc->base;
-}
-
-static void dcn31_destroy_resource_pool(struct resource_pool **pool)
-{
-	struct dcn31_resource_pool *dcn31_pool = TO_DCN31_RES_POOL(*pool);
-
-	dcn31_resource_destruct(dcn31_pool);
-	kfree(dcn31_pool);
-	*pool = NULL;
-}
-
-static struct clock_source *dcn31_clock_source_create(
-		struct dc_context *ctx,
-		struct dc_bios *bios,
-		enum clock_source_id id,
-		const struct dce110_clk_src_regs *regs,
-		bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src =
-		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dcn3_clk_src_construct(clk_src, ctx, bios, id,
-			regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-
-	kfree(clk_src);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
-static bool is_dual_plane(enum surface_pixel_format format)
-{
-	return format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN || format == SURFACE_PIXEL_FORMAT_GRPH_RGBE_ALPHA;
-}
-
-int dcn31x_populate_dml_pipes_from_context(struct dc *dc,
-					  struct dc_state *context,
-					  display_e2e_pipe_params_st *pipes,
-					  bool fast_validate)
-{
-	uint32_t pipe_cnt;
-	int i;
-
-	dc_assert_fp_enabled();
-
-	pipe_cnt = dcn20_populate_dml_pipes_from_context(dc, context, pipes, fast_validate);
-
-	for (i = 0; i < pipe_cnt; i++) {
-		pipes[i].pipe.src.gpuvm = 1;
-		if (dc->debug.dml_hostvm_override == DML_HOSTVM_NO_OVERRIDE) {
-			//pipes[pipe_cnt].pipe.src.hostvm = dc->res_pool->hubbub->riommu_active;
-			pipes[i].pipe.src.hostvm = dc->vm_pa_config.is_hvm_enabled;
-		} else if (dc->debug.dml_hostvm_override == DML_HOSTVM_OVERRIDE_FALSE)
-			pipes[i].pipe.src.hostvm = false;
-		else if (dc->debug.dml_hostvm_override == DML_HOSTVM_OVERRIDE_TRUE)
-			pipes[i].pipe.src.hostvm = true;
-	}
-	return pipe_cnt;
-}
-
-int dcn31_populate_dml_pipes_from_context(
-	struct dc *dc, struct dc_state *context,
-	display_e2e_pipe_params_st *pipes,
-	bool fast_validate)
-{
-	int i, pipe_cnt;
-	struct resource_context *res_ctx = &context->res_ctx;
-	struct pipe_ctx *pipe;
-	bool upscaled = false;
-
-	DC_FP_START();
-	dcn31x_populate_dml_pipes_from_context(dc, context, pipes, fast_validate);
-	DC_FP_END();
-
-	for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) {
-		struct dc_crtc_timing *timing;
-
-		if (!res_ctx->pipe_ctx[i].stream)
-			continue;
-		pipe = &res_ctx->pipe_ctx[i];
-		timing = &pipe->stream->timing;
-		if (pipe->plane_state &&
-				(pipe->plane_state->src_rect.height < pipe->plane_state->dst_rect.height ||
-				pipe->plane_state->src_rect.width < pipe->plane_state->dst_rect.width))
-			upscaled = true;
-
-		/*
-		 * Immediate flip can be set dynamically after enabling the plane.
-		 * We need to require support for immediate flip or underflow can be
-		 * intermittently experienced depending on peak b/w requirements.
-		 */
-		pipes[pipe_cnt].pipe.src.immediate_flip = true;
-		pipes[pipe_cnt].pipe.src.unbounded_req_mode = false;
-		pipes[pipe_cnt].pipe.src.gpuvm = true;
-		pipes[pipe_cnt].pipe.dest.vfront_porch = timing->v_front_porch;
-		pipes[pipe_cnt].pipe.src.dcc_rate = 3;
-		pipes[pipe_cnt].dout.dsc_input_bpc = 0;
-		DC_FP_START();
-		dcn31_zero_pipe_dcc_fraction(pipes, pipe_cnt);
-		DC_FP_END();
-
-
-		if (pipes[pipe_cnt].dout.dsc_enable) {
-			switch (timing->display_color_depth) {
-			case COLOR_DEPTH_888:
-				pipes[pipe_cnt].dout.dsc_input_bpc = 8;
-				break;
-			case COLOR_DEPTH_101010:
-				pipes[pipe_cnt].dout.dsc_input_bpc = 10;
-				break;
-			case COLOR_DEPTH_121212:
-				pipes[pipe_cnt].dout.dsc_input_bpc = 12;
-				break;
-			default:
-				ASSERT(0);
-				break;
-			}
-		}
-
-		pipe_cnt++;
-	}
-	context->bw_ctx.dml.ip.det_buffer_size_kbytes = DCN3_1_DEFAULT_DET_SIZE;
-	dc->config.enable_4to1MPC = false;
-	if (pipe_cnt == 1 && pipe->plane_state && !dc->debug.disable_z9_mpc) {
-		if (is_dual_plane(pipe->plane_state->format)
-				&& pipe->plane_state->src_rect.width <= 1920 && pipe->plane_state->src_rect.height <= 1080) {
-			dc->config.enable_4to1MPC = true;
-		} else if (!is_dual_plane(pipe->plane_state->format) && pipe->plane_state->src_rect.width <= 5120) {
-			/* Limit to 5k max to avoid forced pipe split when there is not enough detile for swath */
-			context->bw_ctx.dml.ip.det_buffer_size_kbytes = 192;
-			pipes[0].pipe.src.unbounded_req_mode = true;
-		}
-	} else if (context->stream_count >= dc->debug.crb_alloc_policy_min_disp_count
-			&& dc->debug.crb_alloc_policy > DET_SIZE_DEFAULT) {
-		context->bw_ctx.dml.ip.det_buffer_size_kbytes = dc->debug.crb_alloc_policy * 64;
-	} else if (context->stream_count >= 3 && upscaled) {
-		context->bw_ctx.dml.ip.det_buffer_size_kbytes = 192;
-	}
-
-	return pipe_cnt;
-}
-
-void dcn31_calculate_wm_and_dlg(
-		struct dc *dc, struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		int pipe_cnt,
-		int vlevel)
-{
-	DC_FP_START();
-	dcn31_calculate_wm_and_dlg_fp(dc, context, pipes, pipe_cnt, vlevel);
-	DC_FP_END();
-}
-
-void
-dcn31_populate_dml_writeback_from_context(struct dc *dc,
-					  struct resource_context *res_ctx,
-					  display_e2e_pipe_params_st *pipes)
-{
-	DC_FP_START();
-	dcn30_populate_dml_writeback_from_context(dc, res_ctx, pipes);
-	DC_FP_END();
-}
-
-void
-dcn31_set_mcif_arb_params(struct dc *dc,
-			  struct dc_state *context,
-			  display_e2e_pipe_params_st *pipes,
-			  int pipe_cnt)
-{
-	DC_FP_START();
-	dcn30_set_mcif_arb_params(dc, context, pipes, pipe_cnt);
-	DC_FP_END();
-}
-
-bool dcn31_validate_bandwidth(struct dc *dc,
-		struct dc_state *context,
-		bool fast_validate)
-{
-	bool out = false;
-
-	BW_VAL_TRACE_SETUP();
-
-	int vlevel = 0;
-	int pipe_cnt = 0;
-	display_e2e_pipe_params_st *pipes = kzalloc(dc->res_pool->pipe_count * sizeof(display_e2e_pipe_params_st), GFP_KERNEL);
-	DC_LOGGER_INIT(dc->ctx->logger);
-
-	BW_VAL_TRACE_COUNT();
-
-	DC_FP_START();
-	out = dcn30_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, fast_validate, true);
-	DC_FP_END();
-
-	// Disable fast_validate to set min dcfclk in alculate_wm_and_dlg
-	if (pipe_cnt == 0)
-		fast_validate = false;
-
-	if (!out)
-		goto validate_fail;
-
-	BW_VAL_TRACE_END_VOLTAGE_LEVEL();
-
-	if (fast_validate) {
-		BW_VAL_TRACE_SKIP(fast);
-		goto validate_out;
-	}
-	if (dc->res_pool->funcs->calculate_wm_and_dlg)
-		dc->res_pool->funcs->calculate_wm_and_dlg(dc, context, pipes, pipe_cnt, vlevel);
-
-	BW_VAL_TRACE_END_WATERMARKS();
-
-	goto validate_out;
-
-validate_fail:
-	DC_LOG_WARNING("Mode Validation Warning: %s failed validation.\n",
-		dml_get_status_message(context->bw_ctx.dml.vba.ValidationStatus[context->bw_ctx.dml.vba.soc.num_states]));
-
-	BW_VAL_TRACE_SKIP(fail);
-	out = false;
-
-validate_out:
-	kfree(pipes);
-
-	BW_VAL_TRACE_FINISH();
-
-	return out;
-}
-
-static void dcn31_get_panel_config_defaults(struct dc_panel_config *panel_config)
-{
-	*panel_config = panel_config_defaults;
-}
-
-static struct dc_cap_funcs cap_funcs = {
-	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
-};
-
-static struct resource_funcs dcn31_res_pool_funcs = {
-	.destroy = dcn31_destroy_resource_pool,
-	.link_enc_create = dcn31_link_encoder_create,
-	.link_enc_create_minimal = dcn31_link_enc_create_minimal,
-	.link_encs_assign = link_enc_cfg_link_encs_assign,
-	.link_enc_unassign = link_enc_cfg_link_enc_unassign,
-	.panel_cntl_create = dcn31_panel_cntl_create,
-	.validate_bandwidth = dcn31_validate_bandwidth,
-	.calculate_wm_and_dlg = dcn31_calculate_wm_and_dlg,
-	.update_soc_for_wm_a = dcn31_update_soc_for_wm_a,
-	.populate_dml_pipes = dcn31_populate_dml_pipes_from_context,
-	.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
-	.release_pipe = dcn20_release_pipe,
-	.add_stream_to_ctx = dcn30_add_stream_to_ctx,
-	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
-	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
-	.populate_dml_writeback_from_context = dcn31_populate_dml_writeback_from_context,
-	.set_mcif_arb_params = dcn31_set_mcif_arb_params,
-	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
-	.acquire_post_bldn_3dlut = dcn30_acquire_post_bldn_3dlut,
-	.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
-	.update_bw_bounding_box = dcn31_update_bw_bounding_box,
-	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
-	.get_panel_config_defaults = dcn31_get_panel_config_defaults,
-};
-
-static struct clock_source *dcn30_clock_source_create(
-		struct dc_context *ctx,
-		struct dc_bios *bios,
-		enum clock_source_id id,
-		const struct dce110_clk_src_regs *regs,
-		bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src =
-		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dcn31_clk_src_construct(clk_src, ctx, bios, id,
-			regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
-static bool dcn31_resource_construct(
-	uint8_t num_virtual_links,
-	struct dc *dc,
-	struct dcn31_resource_pool *pool)
-{
-	int i;
-	struct dc_context *ctx = dc->ctx;
-	struct irq_service_init_data init_data;
-
-	ctx->dc_bios->regs = &bios_regs;
-
-	pool->base.res_cap = &res_cap_dcn31;
-
-	pool->base.funcs = &dcn31_res_pool_funcs;
-
-	/*************************************************
-	 *  Resource + asic cap harcoding                *
-	 *************************************************/
-	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
-	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
-	pool->base.mpcc_count = pool->base.res_cap->num_timing_generator;
-	dc->caps.max_downscale_ratio = 600;
-	dc->caps.i2c_speed_in_khz = 100;
-	dc->caps.i2c_speed_in_khz_hdcp = 5; /*1.4 w/a applied by default*/
-	dc->caps.max_cursor_size = 256;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.dmdata_alloc_size = 2048;
-
-	dc->caps.max_slave_planes = 2;
-	dc->caps.max_slave_yuv_planes = 2;
-	dc->caps.max_slave_rgb_planes = 2;
-	dc->caps.post_blend_color_processing = true;
-	dc->caps.force_dp_tps4_for_cp2520 = true;
-	if (dc->config.forceHBR2CP2520)
-		dc->caps.force_dp_tps4_for_cp2520 = false;
-	dc->caps.dp_hpo = true;
-	dc->caps.dp_hdmi21_pcon_support = true;
-	dc->caps.edp_dsc_support = true;
-	dc->caps.extended_aux_timeout_support = true;
-	dc->caps.dmcub_support = true;
-	dc->caps.is_apu = true;
-	dc->caps.zstate_support = true;
-
-	/* Color pipeline capabilities */
-	dc->caps.color.dpp.dcn_arch = 1;
-	dc->caps.color.dpp.input_lut_shared = 0;
-	dc->caps.color.dpp.icsc = 1;
-	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
-	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
-	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
-	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
-	dc->caps.color.dpp.post_csc = 1;
-	dc->caps.color.dpp.gamma_corr = 1;
-	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
-
-	dc->caps.color.dpp.hw_3d_lut = 1;
-	dc->caps.color.dpp.ogam_ram = 1;
-	// no OGAM ROM on DCN301
-	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
-	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
-	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
-	dc->caps.color.dpp.ocsc = 0;
-
-	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //2
-	dc->caps.color.mpc.ogam_ram = 1;
-	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
-	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
-	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
-	dc->caps.color.mpc.ocsc = 1;
-
-	dc->config.use_old_fixed_vs_sequence = true;
-
-	/* Use pipe context based otg sync logic */
-	dc->config.use_pipe_ctx_sync_logic = true;
-
-	/* read VBIOS LTTPR caps */
-	{
-		if (ctx->dc_bios->funcs->get_lttpr_caps) {
-			enum bp_result bp_query_result;
-			uint8_t is_vbios_lttpr_enable = 0;
-
-			bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
-			dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
-		}
-
-		/* interop bit is implicit */
-		{
-			dc->caps.vbios_lttpr_aware = true;
-		}
-	}
-
-	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
-		dc->debug = debug_defaults_drv;
-
-	// Init the vm_helper
-	if (dc->vm_helper)
-		vm_helper_init(dc->vm_helper, 16);
-
-	/*************************************************
-	 *  Create resources                             *
-	 *************************************************/
-
-	/* Clock Sources for Pixel Clock*/
-	pool->base.clock_sources[DCN31_CLK_SRC_PLL0] =
-			dcn30_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL0,
-				&clk_src_regs[0], false);
-	pool->base.clock_sources[DCN31_CLK_SRC_PLL1] =
-			dcn30_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL1,
-				&clk_src_regs[1], false);
-	/*move phypllx_pixclk_resync to dmub next*/
-	if (dc->ctx->asic_id.hw_internal_rev == YELLOW_CARP_B0) {
-		pool->base.clock_sources[DCN31_CLK_SRC_PLL2] =
-			dcn30_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL2,
-				&clk_src_regs_b0[2], false);
-		pool->base.clock_sources[DCN31_CLK_SRC_PLL3] =
-			dcn30_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL3,
-				&clk_src_regs_b0[3], false);
-	} else {
-		pool->base.clock_sources[DCN31_CLK_SRC_PLL2] =
-			dcn30_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL2,
-				&clk_src_regs[2], false);
-		pool->base.clock_sources[DCN31_CLK_SRC_PLL3] =
-			dcn30_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL3,
-				&clk_src_regs[3], false);
-	}
-
-	pool->base.clock_sources[DCN31_CLK_SRC_PLL4] =
-			dcn30_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL4,
-				&clk_src_regs[4], false);
-
-	pool->base.clk_src_count = DCN30_CLK_SRC_TOTAL;
-
-	/* todo: not reuse phy_pll registers */
-	pool->base.dp_clock_source =
-			dcn31_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_ID_DP_DTO,
-				&clk_src_regs[0], true);
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-
-	/* TODO: DCCG */
-	pool->base.dccg = dccg31_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
-	if (pool->base.dccg == NULL) {
-		dm_error("DC: failed to create dccg!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	/* TODO: IRQ */
-	init_data.ctx = dc->ctx;
-	pool->base.irqs = dal_irq_service_dcn31_create(&init_data);
-	if (!pool->base.irqs)
-		goto create_fail;
-
-	/* HUBBUB */
-	pool->base.hubbub = dcn31_hubbub_create(ctx);
-	if (pool->base.hubbub == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create hubbub!\n");
-		goto create_fail;
-	}
-
-	/* HUBPs, DPPs, OPPs and TGs */
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		pool->base.hubps[i] = dcn31_hubp_create(ctx, i);
-		if (pool->base.hubps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create hubps!\n");
-			goto create_fail;
-		}
-
-		pool->base.dpps[i] = dcn31_dpp_create(ctx, i);
-		if (pool->base.dpps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create dpps!\n");
-			goto create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
-		pool->base.opps[i] = dcn31_opp_create(ctx, i);
-		if (pool->base.opps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create output pixel processor!\n");
-			goto create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		pool->base.timing_generators[i] = dcn31_timing_generator_create(
-				ctx, i);
-		if (pool->base.timing_generators[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create tg!\n");
-			goto create_fail;
-		}
-	}
-	pool->base.timing_generator_count = i;
-
-	/* PSR */
-	pool->base.psr = dmub_psr_create(ctx);
-	if (pool->base.psr == NULL) {
-		dm_error("DC: failed to create psr obj!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	/* Replay */
-	pool->base.replay = dmub_replay_create(ctx);
-	if (pool->base.replay == NULL) {
-		dm_error("DC: failed to create replay obj!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	/* ABM */
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		pool->base.multiple_abms[i] = dmub_abm_create(ctx,
-				&abm_regs[i],
-				&abm_shift,
-				&abm_mask);
-		if (pool->base.multiple_abms[i] == NULL) {
-			dm_error("DC: failed to create abm for pipe %d!\n", i);
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-
-	/* MPC and DSC */
-	pool->base.mpc = dcn31_mpc_create(ctx, pool->base.mpcc_count, pool->base.res_cap->num_mpc_3dlut);
-	if (pool->base.mpc == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mpc!\n");
-		goto create_fail;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		pool->base.dscs[i] = dcn31_dsc_create(ctx, i);
-		if (pool->base.dscs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create display stream compressor %d!\n", i);
-			goto create_fail;
-		}
-	}
-
-	/* DWB and MMHUBBUB */
-	if (!dcn31_dwbc_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create dwbc!\n");
-		goto create_fail;
-	}
-
-	if (!dcn31_mmhubbub_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mcif_wb!\n");
-		goto create_fail;
-	}
-
-	/* AUX and I2C */
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		pool->base.engines[i] = dcn31_aux_engine_create(ctx, i);
-		if (pool->base.engines[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create aux engine!!\n");
-			goto create_fail;
-		}
-		pool->base.hw_i2cs[i] = dcn31_i2c_hw_create(ctx, i);
-		if (pool->base.hw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create hw i2c!!\n");
-			goto create_fail;
-		}
-		pool->base.sw_i2cs[i] = NULL;
-	}
-
-	if (dc->ctx->asic_id.chip_family == FAMILY_YELLOW_CARP &&
-	    dc->ctx->asic_id.hw_internal_rev == YELLOW_CARP_B0 &&
-	    !dc->debug.dpia_debug.bits.disable_dpia) {
-		/* YELLOW CARP B0 has 4 DPIA's */
-		pool->base.usb4_dpia_count = 4;
-	}
-
-	if (dc->ctx->asic_id.chip_family == AMDGPU_FAMILY_GC_11_0_1)
-		pool->base.usb4_dpia_count = 4;
-
-	/* Audio, Stream Encoders including HPO and virtual, MPC 3D LUTs */
-	if (!resource_construct(num_virtual_links, dc, &pool->base,
-			&res_create_funcs))
-		goto create_fail;
-
-	/* HW Sequencer and Plane caps */
-	dcn31_hw_sequencer_construct(dc);
-
-	dc->caps.max_planes =  pool->base.pipe_count;
-
-	for (i = 0; i < dc->caps.max_planes; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	dc->cap_funcs = cap_funcs;
-
-	dc->dcn_ip->max_num_dpp = dcn3_1_ip.max_num_dpp;
-
-	return true;
-
-create_fail:
-	dcn31_resource_destruct(pool);
-
-	return false;
-}
-
-struct resource_pool *dcn31_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc)
-{
-	struct dcn31_resource_pool *pool =
-		kzalloc(sizeof(struct dcn31_resource_pool), GFP_KERNEL);
-
-	if (!pool)
-		return NULL;
-
-	if (dcn31_resource_construct(init_data->num_virtual_links, dc, pool))
-		return &pool->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(pool);
-	return NULL;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.h b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.h
deleted file mode 100644
index 901436591ed4..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.h
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Copyright 2020 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#ifndef _DCN31_RESOURCE_H_
-#define _DCN31_RESOURCE_H_
-
-#include "core_types.h"
-
-#define TO_DCN31_RES_POOL(pool)\
-	container_of(pool, struct dcn31_resource_pool, base)
-
-extern struct _vcs_dpi_ip_params_st dcn3_1_ip;
-
-struct dcn31_resource_pool {
-	struct resource_pool base;
-};
-
-bool dcn31_validate_bandwidth(struct dc *dc,
-		struct dc_state *context,
-		bool fast_validate);
-void dcn31_calculate_wm_and_dlg(
-		struct dc *dc, struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		int pipe_cnt,
-		int vlevel);
-int dcn31_populate_dml_pipes_from_context(
-	struct dc *dc, struct dc_state *context,
-	display_e2e_pipe_params_st *pipes,
-	bool fast_validate);
-void
-dcn31_populate_dml_writeback_from_context(struct dc *dc,
-					  struct resource_context *res_ctx,
-					  display_e2e_pipe_params_st *pipes);
-void
-dcn31_set_mcif_arb_params(struct dc *dc,
-			  struct dc_state *context,
-			  display_e2e_pipe_params_st *pipes,
-			  int pipe_cnt);
-
-struct resource_pool *dcn31_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc);
-
-/*temp: B0 specific before switch to dcn313 headers*/
-#ifndef regPHYPLLF_PIXCLK_RESYNC_CNTL
-#define regPHYPLLF_PIXCLK_RESYNC_CNTL 0x007e
-#define regPHYPLLF_PIXCLK_RESYNC_CNTL_BASE_IDX 1
-#define regPHYPLLG_PIXCLK_RESYNC_CNTL 0x005f
-#define regPHYPLLG_PIXCLK_RESYNC_CNTL_BASE_IDX 1
-
-//PHYPLLF_PIXCLK_RESYNC_CNTL
-#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_PIXCLK_RESYNC_ENABLE__SHIFT 0x0
-#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_DEEP_COLOR_DTO_ENABLE_STATUS__SHIFT 0x1
-#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_DCCG_DEEP_COLOR_CNTL__SHIFT 0x4
-#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_PIXCLK_ENABLE__SHIFT 0x8
-#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_PIXCLK_DOUBLE_RATE_ENABLE__SHIFT 0x9
-#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_PIXCLK_RESYNC_ENABLE_MASK 0x00000001L
-#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_DEEP_COLOR_DTO_ENABLE_STATUS_MASK 0x00000002L
-#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_DCCG_DEEP_COLOR_CNTL_MASK 0x00000030L
-#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_PIXCLK_ENABLE_MASK 0x00000100L
-#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_PIXCLK_DOUBLE_RATE_ENABLE_MASK 0x00000200L
-
-//PHYPLLG_PIXCLK_RESYNC_CNTL
-#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_PIXCLK_RESYNC_ENABLE__SHIFT 0x0
-#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_DEEP_COLOR_DTO_ENABLE_STATUS__SHIFT 0x1
-#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_DCCG_DEEP_COLOR_CNTL__SHIFT 0x4
-#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_PIXCLK_ENABLE__SHIFT 0x8
-#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_PIXCLK_DOUBLE_RATE_ENABLE__SHIFT 0x9
-#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_PIXCLK_RESYNC_ENABLE_MASK 0x00000001L
-#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_DEEP_COLOR_DTO_ENABLE_STATUS_MASK 0x00000002L
-#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_DCCG_DEEP_COLOR_CNTL_MASK 0x00000030L
-#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_PIXCLK_ENABLE_MASK 0x00000100L
-#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_PIXCLK_DOUBLE_RATE_ENABLE_MASK 0x00000200L
-#endif
-#endif /* _DCN31_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/Makefile b/drivers/gpu/drm/amd/display/dc/dcn314/Makefile
index 72456debb99f..6ea47e00d62d 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn314/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn314/Makefile
@@ -10,7 +10,7 @@
 #
 # Makefile for dcn314.
 
-DCN314 = dcn314_resource.o dcn314_init.o \
+DCN314 = dcn314_init.o \
 		dcn314_dio_stream_encoder.o dcn314_dccg.o dcn314_optc.o
 
 AMD_DAL_DCN314 = $(addprefix $(AMDDALPATH)/dc/dcn314/,$(DCN314))
diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c
deleted file mode 100644
index 677361d74a4e..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c
+++ /dev/null
@@ -1,2180 +0,0 @@
-// SPDX-License-Identifier: MIT
-/*
- * Copyright 2022 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-
-#include "dm_services.h"
-#include "dc.h"
-
-#include "dcn31/dcn31_init.h"
-#include "dcn314/dcn314_init.h"
-
-#include "resource.h"
-#include "include/irq_service_interface.h"
-#include "dcn314_resource.h"
-
-#include "dcn20/dcn20_resource.h"
-#include "dcn30/dcn30_resource.h"
-#include "dcn31/dcn31_resource.h"
-
-#include "dcn10/dcn10_ipp.h"
-#include "dcn30/dcn30_hubbub.h"
-#include "dcn31/dcn31_hubbub.h"
-#include "dcn30/dcn30_mpc.h"
-#include "dcn31/dcn31_hubp.h"
-#include "irq/dcn31/irq_service_dcn31.h"
-#include "irq/dcn314/irq_service_dcn314.h"
-#include "dcn30/dcn30_dpp.h"
-#include "dcn314/dcn314_optc.h"
-#include "dcn20/dcn20_hwseq.h"
-#include "dcn30/dcn30_hwseq.h"
-#include "dce110/dce110_hwseq.h"
-#include "dcn30/dcn30_opp.h"
-#include "dcn20/dcn20_dsc.h"
-#include "dcn30/dcn30_vpg.h"
-#include "dcn30/dcn30_afmt.h"
-#include "dcn31/dcn31_dio_link_encoder.h"
-#include "dcn314/dcn314_dio_stream_encoder.h"
-#include "dcn31/dcn31_hpo_dp_stream_encoder.h"
-#include "dcn31/dcn31_hpo_dp_link_encoder.h"
-#include "dcn31/dcn31_apg.h"
-#include "dcn31/dcn31_vpg.h"
-#include "dcn31/dcn31_afmt.h"
-#include "dce/dce_clock_source.h"
-#include "dce/dce_audio.h"
-#include "dce/dce_hwseq.h"
-#include "clk_mgr.h"
-#include "virtual/virtual_stream_encoder.h"
-#include "dce110/dce110_resource.h"
-#include "dml/display_mode_vba.h"
-#include "dml/dcn31/dcn31_fpu.h"
-#include "dml/dcn314/dcn314_fpu.h"
-#include "dcn314/dcn314_dccg.h"
-#include "dcn10/dcn10_resource.h"
-#include "dcn31/dcn31_panel_cntl.h"
-#include "dcn314/dcn314_hwseq.h"
-
-#include "dcn30/dcn30_dwb.h"
-#include "dcn30/dcn30_mmhubbub.h"
-
-#include "dcn/dcn_3_1_4_offset.h"
-#include "dcn/dcn_3_1_4_sh_mask.h"
-#include "dpcs/dpcs_3_1_4_offset.h"
-#include "dpcs/dpcs_3_1_4_sh_mask.h"
-
-#define DCHUBBUB_DEBUG_CTRL_0__DET_DEPTH__SHIFT		0x10
-#define DCHUBBUB_DEBUG_CTRL_0__DET_DEPTH_MASK		0x01FF0000L
-
-#define DSCC0_DSCC_CONFIG0__ICH_RESET_AT_END_OF_LINE__SHIFT                   0x0
-#define DSCC0_DSCC_CONFIG0__ICH_RESET_AT_END_OF_LINE_MASK                     0x0000000FL
-
-#include "reg_helper.h"
-#include "dce/dmub_abm.h"
-#include "dce/dmub_psr.h"
-#include "dce/dmub_replay.h"
-#include "dce/dce_aux.h"
-#include "dce/dce_i2c.h"
-#include "dml/dcn314/display_mode_vba_314.h"
-#include "vm_helper.h"
-#include "dcn20/dcn20_vmid.h"
-
-#include "link_enc_cfg.h"
-
-#define DCN_BASE__INST0_SEG1				0x000000C0
-#define DCN_BASE__INST0_SEG2				0x000034C0
-#define DCN_BASE__INST0_SEG3				0x00009000
-
-#define NBIO_BASE__INST0_SEG1				0x00000014
-
-#define MAX_INSTANCE					7
-#define MAX_SEGMENT					8
-
-#define regBIF_BX2_BIOS_SCRATCH_2			0x003a
-#define regBIF_BX2_BIOS_SCRATCH_2_BASE_IDX		1
-#define regBIF_BX2_BIOS_SCRATCH_3			0x003b
-#define regBIF_BX2_BIOS_SCRATCH_3_BASE_IDX		1
-#define regBIF_BX2_BIOS_SCRATCH_6			0x003e
-#define regBIF_BX2_BIOS_SCRATCH_6_BASE_IDX		1
-
-#define DC_LOGGER \
-	dc->ctx->logger
-#define DC_LOGGER_INIT(logger)
-
-enum dcn31_clk_src_array_id {
-	DCN31_CLK_SRC_PLL0,
-	DCN31_CLK_SRC_PLL1,
-	DCN31_CLK_SRC_PLL2,
-	DCN31_CLK_SRC_PLL3,
-	DCN31_CLK_SRC_PLL4,
-	DCN30_CLK_SRC_TOTAL
-};
-
-/* begin *********************
- * macros to expend register list macro defined in HW object header file
- */
-
-/* DCN */
-/* TODO awful hack. fixup dcn20_dwb.h */
-#undef BASE_INNER
-#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
-
-#define BASE(seg) BASE_INNER(seg)
-
-#define SR(reg_name)\
-		.reg_name = BASE(reg ## reg_name ## _BASE_IDX) +  \
-					reg ## reg_name
-
-#define SRI(reg_name, block, id)\
-	.reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define SRI2(reg_name, block, id)\
-	.reg_name = BASE(reg ## reg_name ## _BASE_IDX) + \
-					reg ## reg_name
-
-#define SRIR(var_name, reg_name, block, id)\
-	.var_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define SRII_MPC_RMU(reg_name, block, id)\
-	.RMU##_##reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define SRII_DWB(reg_name, temp_name, block, id)\
-	.reg_name[id] = BASE(reg ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## temp_name
-
-#define SF_DWB2(reg_name, block, id, field_name, post_fix)	\
-	.field_name = reg_name ## __ ## field_name ## post_fix
-
-#define DCCG_SRII(reg_name, block, id)\
-	.block ## _ ## reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define VUPDATE_SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(reg ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
-					reg ## reg_name ## _ ## block ## id
-
-/* NBIO */
-#define NBIO_BASE_INNER(seg) \
-	NBIO_BASE__INST0_SEG ## seg
-
-#define NBIO_BASE(seg) \
-	NBIO_BASE_INNER(seg)
-
-#define NBIO_SR(reg_name)\
-		.reg_name = NBIO_BASE(regBIF_BX2_ ## reg_name ## _BASE_IDX) + \
-					regBIF_BX2_ ## reg_name
-
-/* MMHUB */
-#define MMHUB_BASE_INNER(seg) \
-	MMHUB_BASE__INST0_SEG ## seg
-
-#define MMHUB_BASE(seg) \
-	MMHUB_BASE_INNER(seg)
-
-#define MMHUB_SR(reg_name)\
-		.reg_name = MMHUB_BASE(reg ## reg_name ## _BASE_IDX) + \
-					reg ## reg_name
-
-/* CLOCK */
-#define CLK_BASE_INNER(seg) \
-	CLK_BASE__INST0_SEG ## seg
-
-#define CLK_BASE(seg) \
-	CLK_BASE_INNER(seg)
-
-#define CLK_SRI(reg_name, block, inst)\
-	.reg_name = CLK_BASE(reg ## block ## _ ## inst ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## _ ## inst ## _ ## reg_name
-
-
-static const struct bios_registers bios_regs = {
-		NBIO_SR(BIOS_SCRATCH_3),
-		NBIO_SR(BIOS_SCRATCH_6)
-};
-
-#define clk_src_regs(index, pllid)\
-[index] = {\
-	CS_COMMON_REG_LIST_DCN3_0(index, pllid),\
-}
-
-static const struct dce110_clk_src_regs clk_src_regs[] = {
-	clk_src_regs(0, A),
-	clk_src_regs(1, B),
-	clk_src_regs(2, C),
-	clk_src_regs(3, D),
-	clk_src_regs(4, E)
-};
-
-static const struct dce110_clk_src_shift cs_shift = {
-		CS_COMMON_MASK_SH_LIST_DCN3_1_4(__SHIFT)
-};
-
-static const struct dce110_clk_src_mask cs_mask = {
-		CS_COMMON_MASK_SH_LIST_DCN3_1_4(_MASK)
-};
-
-#define abm_regs(id)\
-[id] = {\
-		ABM_DCN302_REG_LIST(id)\
-}
-
-static const struct dce_abm_registers abm_regs[] = {
-		abm_regs(0),
-		abm_regs(1),
-		abm_regs(2),
-		abm_regs(3),
-};
-
-static const struct dce_abm_shift abm_shift = {
-		ABM_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dce_abm_mask abm_mask = {
-		ABM_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define audio_regs(id)\
-[id] = {\
-		AUD_COMMON_REG_LIST(id)\
-}
-
-static const struct dce_audio_registers audio_regs[] = {
-	audio_regs(0),
-	audio_regs(1),
-	audio_regs(2),
-	audio_regs(3),
-	audio_regs(4),
-	audio_regs(5),
-	audio_regs(6)
-};
-
-#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
-		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
-
-static const struct dce_audio_shift audio_shift = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_audio_mask audio_mask = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
-};
-
-#define vpg_regs(id)\
-[id] = {\
-	VPG_DCN31_REG_LIST(id)\
-}
-
-static const struct dcn31_vpg_registers vpg_regs[] = {
-	vpg_regs(0),
-	vpg_regs(1),
-	vpg_regs(2),
-	vpg_regs(3),
-	vpg_regs(4),
-	vpg_regs(5),
-	vpg_regs(6),
-	vpg_regs(7),
-	vpg_regs(8),
-	vpg_regs(9),
-};
-
-static const struct dcn31_vpg_shift vpg_shift = {
-	DCN31_VPG_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_vpg_mask vpg_mask = {
-	DCN31_VPG_MASK_SH_LIST(_MASK)
-};
-
-#define afmt_regs(id)\
-[id] = {\
-	AFMT_DCN31_REG_LIST(id)\
-}
-
-static const struct dcn31_afmt_registers afmt_regs[] = {
-	afmt_regs(0),
-	afmt_regs(1),
-	afmt_regs(2),
-	afmt_regs(3),
-	afmt_regs(4),
-	afmt_regs(5)
-};
-
-static const struct dcn31_afmt_shift afmt_shift = {
-	DCN31_AFMT_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_afmt_mask afmt_mask = {
-	DCN31_AFMT_MASK_SH_LIST(_MASK)
-};
-
-#define apg_regs(id)\
-[id] = {\
-	APG_DCN31_REG_LIST(id)\
-}
-
-static const struct dcn31_apg_registers apg_regs[] = {
-	apg_regs(0),
-	apg_regs(1),
-	apg_regs(2),
-	apg_regs(3)
-};
-
-static const struct dcn31_apg_shift apg_shift = {
-	DCN31_APG_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_apg_mask apg_mask = {
-		DCN31_APG_MASK_SH_LIST(_MASK)
-};
-
-#define stream_enc_regs(id)\
-[id] = {\
-		SE_DCN314_REG_LIST(id)\
-}
-
-static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
-	stream_enc_regs(0),
-	stream_enc_regs(1),
-	stream_enc_regs(2),
-	stream_enc_regs(3),
-	stream_enc_regs(4)
-};
-
-static const struct dcn10_stream_encoder_shift se_shift = {
-		SE_COMMON_MASK_SH_LIST_DCN314(__SHIFT)
-};
-
-static const struct dcn10_stream_encoder_mask se_mask = {
-		SE_COMMON_MASK_SH_LIST_DCN314(_MASK)
-};
-
-
-#define aux_regs(id)\
-[id] = {\
-	DCN2_AUX_REG_LIST(id)\
-}
-
-static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
-		aux_regs(0),
-		aux_regs(1),
-		aux_regs(2),
-		aux_regs(3),
-		aux_regs(4)
-};
-
-#define hpd_regs(id)\
-[id] = {\
-	HPD_REG_LIST(id)\
-}
-
-static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
-		hpd_regs(0),
-		hpd_regs(1),
-		hpd_regs(2),
-		hpd_regs(3),
-		hpd_regs(4)
-};
-
-#define link_regs(id, phyid)\
-[id] = {\
-	LE_DCN31_REG_LIST(id), \
-	UNIPHY_DCN2_REG_LIST(phyid), \
-}
-
-static const struct dce110_aux_registers_shift aux_shift = {
-	DCN_AUX_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce110_aux_registers_mask aux_mask = {
-	DCN_AUX_MASK_SH_LIST(_MASK)
-};
-
-static const struct dcn10_link_enc_registers link_enc_regs[] = {
-	link_regs(0, A),
-	link_regs(1, B),
-	link_regs(2, C),
-	link_regs(3, D),
-	link_regs(4, E)
-};
-
-static const struct dcn10_link_enc_shift le_shift = {
-	LINK_ENCODER_MASK_SH_LIST_DCN31(__SHIFT),
-	DPCS_DCN31_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn10_link_enc_mask le_mask = {
-	LINK_ENCODER_MASK_SH_LIST_DCN31(_MASK),
-	DPCS_DCN31_MASK_SH_LIST(_MASK)
-};
-
-#define hpo_dp_stream_encoder_reg_list(id)\
-[id] = {\
-	DCN3_1_HPO_DP_STREAM_ENC_REG_LIST(id)\
-}
-
-static const struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[] = {
-	hpo_dp_stream_encoder_reg_list(0),
-	hpo_dp_stream_encoder_reg_list(1),
-	hpo_dp_stream_encoder_reg_list(2),
-	hpo_dp_stream_encoder_reg_list(3)
-};
-
-static const struct dcn31_hpo_dp_stream_encoder_shift hpo_dp_se_shift = {
-	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_hpo_dp_stream_encoder_mask hpo_dp_se_mask = {
-	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(_MASK)
-};
-
-
-#define hpo_dp_link_encoder_reg_list(id)\
-[id] = {\
-	DCN3_1_HPO_DP_LINK_ENC_REG_LIST(id),\
-	DCN3_1_RDPCSTX_REG_LIST(0),\
-	DCN3_1_RDPCSTX_REG_LIST(1),\
-	DCN3_1_RDPCSTX_REG_LIST(2),\
-}
-
-static const struct dcn31_hpo_dp_link_encoder_registers hpo_dp_link_enc_regs[] = {
-	hpo_dp_link_encoder_reg_list(0),
-	hpo_dp_link_encoder_reg_list(1),
-};
-
-static const struct dcn31_hpo_dp_link_encoder_shift hpo_dp_le_shift = {
-	DCN3_1_HPO_DP_LINK_ENC_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_hpo_dp_link_encoder_mask hpo_dp_le_mask = {
-	DCN3_1_HPO_DP_LINK_ENC_MASK_SH_LIST(_MASK)
-};
-
-#define dpp_regs(id)\
-[id] = {\
-	DPP_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn3_dpp_registers dpp_regs[] = {
-	dpp_regs(0),
-	dpp_regs(1),
-	dpp_regs(2),
-	dpp_regs(3)
-};
-
-static const struct dcn3_dpp_shift tf_shift = {
-		DPP_REG_LIST_SH_MASK_DCN30(__SHIFT)
-};
-
-static const struct dcn3_dpp_mask tf_mask = {
-		DPP_REG_LIST_SH_MASK_DCN30(_MASK)
-};
-
-#define opp_regs(id)\
-[id] = {\
-	OPP_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn20_opp_registers opp_regs[] = {
-	opp_regs(0),
-	opp_regs(1),
-	opp_regs(2),
-	opp_regs(3)
-};
-
-static const struct dcn20_opp_shift opp_shift = {
-	OPP_MASK_SH_LIST_DCN20(__SHIFT)
-};
-
-static const struct dcn20_opp_mask opp_mask = {
-	OPP_MASK_SH_LIST_DCN20(_MASK)
-};
-
-#define aux_engine_regs(id)\
-[id] = {\
-	AUX_COMMON_REG_LIST0(id), \
-	.AUXN_IMPCAL = 0, \
-	.AUXP_IMPCAL = 0, \
-	.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
-}
-
-static const struct dce110_aux_registers aux_engine_regs[] = {
-		aux_engine_regs(0),
-		aux_engine_regs(1),
-		aux_engine_regs(2),
-		aux_engine_regs(3),
-		aux_engine_regs(4)
-};
-
-#define dwbc_regs_dcn3(id)\
-[id] = {\
-	DWBC_COMMON_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn30_dwbc_registers dwbc30_regs[] = {
-	dwbc_regs_dcn3(0),
-};
-
-static const struct dcn30_dwbc_shift dwbc30_shift = {
-	DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_dwbc_mask dwbc30_mask = {
-	DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define mcif_wb_regs_dcn3(id)\
-[id] = {\
-	MCIF_WB_COMMON_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn30_mmhubbub_registers mcif_wb30_regs[] = {
-	mcif_wb_regs_dcn3(0)
-};
-
-static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
-	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
-	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define dsc_regsDCN314(id)\
-[id] = {\
-	DSC_REG_LIST_DCN20(id)\
-}
-
-static const struct dcn20_dsc_registers dsc_regs[] = {
-	dsc_regsDCN314(0),
-	dsc_regsDCN314(1),
-	dsc_regsDCN314(2),
-	dsc_regsDCN314(3)
-};
-
-static const struct dcn20_dsc_shift dsc_shift = {
-	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
-};
-
-static const struct dcn20_dsc_mask dsc_mask = {
-	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
-};
-
-static const struct dcn30_mpc_registers mpc_regs = {
-		MPC_REG_LIST_DCN3_0(0),
-		MPC_REG_LIST_DCN3_0(1),
-		MPC_REG_LIST_DCN3_0(2),
-		MPC_REG_LIST_DCN3_0(3),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(0),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(1),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(2),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(3),
-		MPC_RMU_GLOBAL_REG_LIST_DCN3AG,
-		MPC_RMU_REG_LIST_DCN3AG(0),
-		MPC_RMU_REG_LIST_DCN3AG(1),
-		//MPC_RMU_REG_LIST_DCN3AG(2),
-		MPC_DWB_MUX_REG_LIST_DCN3_0(0),
-};
-
-static const struct dcn30_mpc_shift mpc_shift = {
-	MPC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_mpc_mask mpc_mask = {
-	MPC_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define optc_regs(id)\
-[id] = {OPTC_COMMON_REG_LIST_DCN3_14(id)}
-
-static const struct dcn_optc_registers optc_regs[] = {
-	optc_regs(0),
-	optc_regs(1),
-	optc_regs(2),
-	optc_regs(3)
-};
-
-static const struct dcn_optc_shift optc_shift = {
-	OPTC_COMMON_MASK_SH_LIST_DCN3_14(__SHIFT)
-};
-
-static const struct dcn_optc_mask optc_mask = {
-	OPTC_COMMON_MASK_SH_LIST_DCN3_14(_MASK)
-};
-
-#define hubp_regs(id)\
-[id] = {\
-	HUBP_REG_LIST_DCN30(id)\
-}
-
-static const struct dcn_hubp2_registers hubp_regs[] = {
-		hubp_regs(0),
-		hubp_regs(1),
-		hubp_regs(2),
-		hubp_regs(3)
-};
-
-
-static const struct dcn_hubp2_shift hubp_shift = {
-		HUBP_MASK_SH_LIST_DCN31(__SHIFT)
-};
-
-static const struct dcn_hubp2_mask hubp_mask = {
-		HUBP_MASK_SH_LIST_DCN31(_MASK)
-};
-static const struct dcn_hubbub_registers hubbub_reg = {
-		HUBBUB_REG_LIST_DCN31(0)
-};
-
-static const struct dcn_hubbub_shift hubbub_shift = {
-		HUBBUB_MASK_SH_LIST_DCN31(__SHIFT)
-};
-
-static const struct dcn_hubbub_mask hubbub_mask = {
-		HUBBUB_MASK_SH_LIST_DCN31(_MASK)
-};
-
-static const struct dccg_registers dccg_regs = {
-		DCCG_REG_LIST_DCN314()
-};
-
-static const struct dccg_shift dccg_shift = {
-		DCCG_MASK_SH_LIST_DCN314(__SHIFT)
-};
-
-static const struct dccg_mask dccg_mask = {
-		DCCG_MASK_SH_LIST_DCN314(_MASK)
-};
-
-
-#define SRII2(reg_name_pre, reg_name_post, id)\
-	.reg_name_pre ## _ ##  reg_name_post[id] = BASE(reg ## reg_name_pre \
-			## id ## _ ## reg_name_post ## _BASE_IDX) + \
-			reg ## reg_name_pre ## id ## _ ## reg_name_post
-
-
-#define HWSEQ_DCN31_REG_LIST()\
-	SR(DCHUBBUB_GLOBAL_TIMER_CNTL), \
-	SR(DCHUBBUB_ARB_HOSTVM_CNTL), \
-	SR(DIO_MEM_PWR_CTRL), \
-	SR(ODM_MEM_PWR_CTRL3), \
-	SR(DMU_MEM_PWR_CNTL), \
-	SR(MMHUBBUB_MEM_PWR_CNTL), \
-	SR(DCCG_GATE_DISABLE_CNTL), \
-	SR(DCCG_GATE_DISABLE_CNTL2), \
-	SR(DCFCLK_CNTL),\
-	SR(DC_MEM_GLOBAL_PWR_REQ_CNTL), \
-	SRII(PIXEL_RATE_CNTL, OTG, 0), \
-	SRII(PIXEL_RATE_CNTL, OTG, 1),\
-	SRII(PIXEL_RATE_CNTL, OTG, 2),\
-	SRII(PIXEL_RATE_CNTL, OTG, 3),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 0),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 1),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 2),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 3),\
-	SR(MICROSECOND_TIME_BASE_DIV), \
-	SR(MILLISECOND_TIME_BASE_DIV), \
-	SR(DISPCLK_FREQ_CHANGE_CNTL), \
-	SR(RBBMIF_TIMEOUT_DIS), \
-	SR(RBBMIF_TIMEOUT_DIS_2), \
-	SR(DCHUBBUB_CRC_CTRL), \
-	SR(DPP_TOP0_DPP_CRC_CTRL), \
-	SR(DPP_TOP0_DPP_CRC_VAL_B_A), \
-	SR(DPP_TOP0_DPP_CRC_VAL_R_G), \
-	SR(MPC_CRC_CTRL), \
-	SR(MPC_CRC_RESULT_GB), \
-	SR(MPC_CRC_RESULT_C), \
-	SR(MPC_CRC_RESULT_AR), \
-	SR(DOMAIN0_PG_CONFIG), \
-	SR(DOMAIN1_PG_CONFIG), \
-	SR(DOMAIN2_PG_CONFIG), \
-	SR(DOMAIN3_PG_CONFIG), \
-	SR(DOMAIN16_PG_CONFIG), \
-	SR(DOMAIN17_PG_CONFIG), \
-	SR(DOMAIN18_PG_CONFIG), \
-	SR(DOMAIN19_PG_CONFIG), \
-	SR(DOMAIN0_PG_STATUS), \
-	SR(DOMAIN1_PG_STATUS), \
-	SR(DOMAIN2_PG_STATUS), \
-	SR(DOMAIN3_PG_STATUS), \
-	SR(DOMAIN16_PG_STATUS), \
-	SR(DOMAIN17_PG_STATUS), \
-	SR(DOMAIN18_PG_STATUS), \
-	SR(DOMAIN19_PG_STATUS), \
-	SR(D1VGA_CONTROL), \
-	SR(D2VGA_CONTROL), \
-	SR(D3VGA_CONTROL), \
-	SR(D4VGA_CONTROL), \
-	SR(D5VGA_CONTROL), \
-	SR(D6VGA_CONTROL), \
-	SR(DC_IP_REQUEST_CNTL), \
-	SR(AZALIA_AUDIO_DTO), \
-	SR(AZALIA_CONTROLLER_CLOCK_GATING), \
-	SR(HPO_TOP_HW_CONTROL)
-
-static const struct dce_hwseq_registers hwseq_reg = {
-		HWSEQ_DCN31_REG_LIST()
-};
-
-#define HWSEQ_DCN31_MASK_SH_LIST(mask_sh)\
-	HWSEQ_DCN_MASK_SH_LIST(mask_sh), \
-	HWS_SF(, DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, mask_sh), \
-	HWS_SF(, DCHUBBUB_ARB_HOSTVM_CNTL, DISABLE_HOSTVM_FORCE_ALLOW_PSTATE, mask_sh), \
-	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN0_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN1_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN2_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN3_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN16_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN17_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN18_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN19_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh), \
-	HWS_SF(, AZALIA_AUDIO_DTO, AZALIA_AUDIO_DTO_MODULE, mask_sh), \
-	HWS_SF(, HPO_TOP_CLOCK_CONTROL, HPO_HDMISTREAMCLK_G_GATE_DIS, mask_sh), \
-	HWS_SF(, DMU_MEM_PWR_CNTL, DMCU_ERAM_MEM_PWR_FORCE, mask_sh), \
-	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_UNASSIGNED_PWR_MODE, mask_sh), \
-	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_VBLANK_PWR_MODE, mask_sh), \
-	HWS_SF(, MMHUBBUB_MEM_PWR_CNTL, VGA_MEM_PWR_FORCE, mask_sh), \
-	HWS_SF(, DIO_MEM_PWR_CTRL, I2C_LIGHT_SLEEP_FORCE, mask_sh), \
-	HWS_SF(, HPO_TOP_HW_CONTROL, HPO_IO_EN, mask_sh)
-
-static const struct dce_hwseq_shift hwseq_shift = {
-		HWSEQ_DCN31_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_hwseq_mask hwseq_mask = {
-		HWSEQ_DCN31_MASK_SH_LIST(_MASK)
-};
-#define vmid_regs(id)\
-[id] = {\
-		DCN20_VMID_REG_LIST(id)\
-}
-
-static const struct dcn_vmid_registers vmid_regs[] = {
-	vmid_regs(0),
-	vmid_regs(1),
-	vmid_regs(2),
-	vmid_regs(3),
-	vmid_regs(4),
-	vmid_regs(5),
-	vmid_regs(6),
-	vmid_regs(7),
-	vmid_regs(8),
-	vmid_regs(9),
-	vmid_regs(10),
-	vmid_regs(11),
-	vmid_regs(12),
-	vmid_regs(13),
-	vmid_regs(14),
-	vmid_regs(15)
-};
-
-static const struct dcn20_vmid_shift vmid_shifts = {
-		DCN20_VMID_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn20_vmid_mask vmid_masks = {
-		DCN20_VMID_MASK_SH_LIST(_MASK)
-};
-
-static const struct resource_caps res_cap_dcn314 = {
-	.num_timing_generator = 4,
-	.num_opp = 4,
-	.num_video_plane = 4,
-	.num_audio = 5,
-	.num_stream_encoder = 5,
-	.num_dig_link_enc = 5,
-	.num_hpo_dp_stream_encoder = 4,
-	.num_hpo_dp_link_encoder = 2,
-	.num_pll = 5,
-	.num_dwb = 1,
-	.num_ddc = 5,
-	.num_vmid = 16,
-	.num_mpc_3dlut = 2,
-	.num_dsc = 4,
-};
-
-static const struct dc_plane_cap plane_cap = {
-	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.per_pixel_alpha = true,
-
-	.pixel_format_support = {
-			.argb8888 = true,
-			.nv12 = true,
-			.fp16 = true,
-			.p010 = true,
-			.ayuv = false,
-	},
-
-	.max_upscale_factor = {
-			.argb8888 = 16000,
-			.nv12 = 16000,
-			.fp16 = 16000
-	},
-
-	// 6:1 downscaling ratio: 1000/6 = 166.666
-	// 4:1 downscaling ratio for ARGB888 to prevent underflow during P010 playback: 1000/4 = 250
-	.max_downscale_factor = {
-			.argb8888 = 250,
-			.nv12 = 167,
-			.fp16 = 167
-	},
-	64,
-	64
-};
-
-static const struct dc_debug_options debug_defaults_drv = {
-	.disable_z10 = false,
-	.enable_z9_disable_interface = true,
-	.minimum_z8_residency_time = 2000,
-	.psr_skip_crtc_disable = true,
-	.replay_skip_crtc_disabled = true,
-	.disable_dmcu = true,
-	.force_abm_enable = false,
-	.timing_trace = false,
-	.clock_trace = true,
-	.disable_dpp_power_gate = false,
-	.disable_hubp_power_gate = false,
-	.disable_pplib_clock_request = false,
-	.pipe_split_policy = MPC_SPLIT_DYNAMIC,
-	.force_single_disp_pipe_split = false,
-	.disable_dcc = DCC_ENABLE,
-	.vsr_support = true,
-	.performance_trace = false,
-	.max_downscale_src_width = 4096,/*upto true 4k*/
-	.disable_pplib_wm_range = false,
-	.scl_reset_length10 = true,
-	.sanity_checks = true,
-	.underflow_assert_delay_us = 0xFFFFFFFF,
-	.dwb_fi_phase = -1, // -1 = disable,
-	.dmub_command_table = true,
-	.pstate_enabled = true,
-	.use_max_lb = true,
-	.enable_mem_low_power = {
-		.bits = {
-			.vga = true,
-			.i2c = true,
-			.dmcu = false, // This is previously known to cause hang on S3 cycles if enabled
-			.dscl = true,
-			.cm = true,
-			.mpc = true,
-			.optc = true,
-			.vpg = true,
-			.afmt = true,
-		}
-	},
-
-	.root_clock_optimization = {
-			.bits = {
-					.dpp = true,
-					.dsc = true,
-					.hdmistream = true,
-					.hdmichar = true,
-					.dpstream = true,
-					.symclk32_se = false,
-					.symclk32_le = true,
-					.symclk_fe = true,
-					.physymclk = true,
-					.dpiasymclk = true,
-			}
-	},
-
-	.seamless_boot_odm_combine = true,
-	.using_dml2 = false,
-};
-
-static const struct dc_debug_options debug_defaults_diags = {
-	.disable_dmcu = true,
-	.force_abm_enable = false,
-	.timing_trace = true,
-	.clock_trace = true,
-	.disable_dpp_power_gate = true,
-	.disable_hubp_power_gate = true,
-	.disable_clock_gate = true,
-	.disable_pplib_clock_request = true,
-	.disable_pplib_wm_range = true,
-	.disable_stutter = false,
-	.scl_reset_length10 = true,
-	.dwb_fi_phase = -1, // -1 = disable
-	.dmub_command_table = true,
-	.enable_tri_buf = true,
-	.use_max_lb = true
-};
-
-static const struct dc_panel_config panel_config_defaults = {
-	.psr = {
-		.disable_psr = false,
-		.disallow_psrsu = false,
-		.disallow_replay = false,
-	},
-	.ilr = {
-		.optimize_edp_link_rate = true,
-	},
-};
-
-static void dcn31_dpp_destroy(struct dpp **dpp)
-{
-	kfree(TO_DCN20_DPP(*dpp));
-	*dpp = NULL;
-}
-
-static struct dpp *dcn31_dpp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn3_dpp *dpp =
-		kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
-
-	if (!dpp)
-		return NULL;
-
-	if (dpp3_construct(dpp, ctx, inst,
-			&dpp_regs[inst], &tf_shift, &tf_mask))
-		return &dpp->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(dpp);
-	return NULL;
-}
-
-static struct output_pixel_processor *dcn31_opp_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_opp *opp =
-		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
-
-	if (!opp) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dcn20_opp_construct(opp, ctx, inst,
-			&opp_regs[inst], &opp_shift, &opp_mask);
-	return &opp->base;
-}
-
-static struct dce_aux *dcn31_aux_engine_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct aux_engine_dce110 *aux_engine =
-		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
-
-	if (!aux_engine)
-		return NULL;
-
-	dce110_aux_engine_construct(aux_engine, ctx, inst,
-				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
-				    &aux_engine_regs[inst],
-					&aux_mask,
-					&aux_shift,
-					ctx->dc->caps.extended_aux_timeout_support);
-
-	return &aux_engine->base;
-}
-#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST_DCN30(id) }
-
-static const struct dce_i2c_registers i2c_hw_regs[] = {
-		i2c_inst_regs(1),
-		i2c_inst_regs(2),
-		i2c_inst_regs(3),
-		i2c_inst_regs(4),
-		i2c_inst_regs(5),
-};
-
-static const struct dce_i2c_shift i2c_shifts = {
-		I2C_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dce_i2c_mask i2c_masks = {
-		I2C_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-/* ========================================================== */
-
-/*
- * DPIA index | Preferred Encoder     |    Host Router
- *   0        |      C                |       0
- *   1        |      First Available  |       0
- *   2        |      D                |       1
- *   3        |      First Available  |       1
- */
-/* ========================================================== */
-static const enum engine_id dpia_to_preferred_enc_id_table[] = {
-		ENGINE_ID_DIGC,
-		ENGINE_ID_DIGC,
-		ENGINE_ID_DIGD,
-		ENGINE_ID_DIGD
-};
-
-static enum engine_id dcn314_get_preferred_eng_id_dpia(unsigned int dpia_index)
-{
-	return dpia_to_preferred_enc_id_table[dpia_index];
-}
-
-static struct dce_i2c_hw *dcn31_i2c_hw_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_i2c_hw *dce_i2c_hw =
-		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
-
-	if (!dce_i2c_hw)
-		return NULL;
-
-	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
-				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
-
-	return dce_i2c_hw;
-}
-static struct mpc *dcn31_mpc_create(
-		struct dc_context *ctx,
-		int num_mpcc,
-		int num_rmu)
-{
-	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc),
-					  GFP_KERNEL);
-
-	if (!mpc30)
-		return NULL;
-
-	dcn30_mpc_construct(mpc30, ctx,
-			&mpc_regs,
-			&mpc_shift,
-			&mpc_mask,
-			num_mpcc,
-			num_rmu);
-
-	return &mpc30->base;
-}
-
-static struct hubbub *dcn31_hubbub_create(struct dc_context *ctx)
-{
-	int i;
-
-	struct dcn20_hubbub *hubbub3 = kzalloc(sizeof(struct dcn20_hubbub),
-					  GFP_KERNEL);
-
-	if (!hubbub3)
-		return NULL;
-
-	hubbub31_construct(hubbub3, ctx,
-			&hubbub_reg,
-			&hubbub_shift,
-			&hubbub_mask,
-			dcn3_14_ip.det_buffer_size_kbytes,
-			dcn3_14_ip.pixel_chunk_size_kbytes,
-			dcn3_14_ip.config_return_buffer_size_in_kbytes);
-
-
-	for (i = 0; i < res_cap_dcn314.num_vmid; i++) {
-		struct dcn20_vmid *vmid = &hubbub3->vmid[i];
-
-		vmid->ctx = ctx;
-
-		vmid->regs = &vmid_regs[i];
-		vmid->shifts = &vmid_shifts;
-		vmid->masks = &vmid_masks;
-	}
-
-	return &hubbub3->base;
-}
-
-static struct timing_generator *dcn31_timing_generator_create(
-		struct dc_context *ctx,
-		uint32_t instance)
-{
-	struct optc *tgn10 =
-		kzalloc(sizeof(struct optc), GFP_KERNEL);
-
-	if (!tgn10)
-		return NULL;
-
-	tgn10->base.inst = instance;
-	tgn10->base.ctx = ctx;
-
-	tgn10->tg_regs = &optc_regs[instance];
-	tgn10->tg_shift = &optc_shift;
-	tgn10->tg_mask = &optc_mask;
-
-	dcn314_timing_generator_init(tgn10);
-
-	return &tgn10->base;
-}
-
-static const struct encoder_feature_support link_enc_feature = {
-		.max_hdmi_deep_color = COLOR_DEPTH_121212,
-		.max_hdmi_pixel_clock = 600000,
-		.hdmi_ycbcr420_supported = true,
-		.dp_ycbcr420_supported = true,
-		.fec_supported = true,
-		.flags.bits.IS_HBR2_CAPABLE = true,
-		.flags.bits.IS_HBR3_CAPABLE = true,
-		.flags.bits.IS_TPS3_CAPABLE = true,
-		.flags.bits.IS_TPS4_CAPABLE = true
-};
-
-static struct link_encoder *dcn31_link_encoder_create(
-	struct dc_context *ctx,
-	const struct encoder_init_data *enc_init_data)
-{
-	struct dcn20_link_encoder *enc20 =
-		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
-
-	if (!enc20)
-		return NULL;
-
-	dcn31_link_encoder_construct(enc20,
-			enc_init_data,
-			&link_enc_feature,
-			&link_enc_regs[enc_init_data->transmitter],
-			&link_enc_aux_regs[enc_init_data->channel - 1],
-			&link_enc_hpd_regs[enc_init_data->hpd_source],
-			&le_shift,
-			&le_mask);
-
-	return &enc20->enc10.base;
-}
-
-/* Create a minimal link encoder object not associated with a particular
- * physical connector.
- * resource_funcs.link_enc_create_minimal
- */
-static struct link_encoder *dcn31_link_enc_create_minimal(
-		struct dc_context *ctx, enum engine_id eng_id)
-{
-	struct dcn20_link_encoder *enc20;
-
-	if ((eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
-		return NULL;
-
-	enc20 = kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
-	if (!enc20)
-		return NULL;
-
-	dcn31_link_encoder_construct_minimal(
-			enc20,
-			ctx,
-			&link_enc_feature,
-			&link_enc_regs[eng_id - ENGINE_ID_DIGA],
-			eng_id);
-
-	return &enc20->enc10.base;
-}
-
-static struct panel_cntl *dcn31_panel_cntl_create(const struct panel_cntl_init_data *init_data)
-{
-	struct dcn31_panel_cntl *panel_cntl =
-		kzalloc(sizeof(struct dcn31_panel_cntl), GFP_KERNEL);
-
-	if (!panel_cntl)
-		return NULL;
-
-	dcn31_panel_cntl_construct(panel_cntl, init_data);
-
-	return &panel_cntl->base;
-}
-
-static void read_dce_straps(
-	struct dc_context *ctx,
-	struct resource_straps *straps)
-{
-	generic_reg_get(ctx, regDC_PINSTRAPS + BASE(regDC_PINSTRAPS_BASE_IDX),
-		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
-
-}
-
-static struct audio *dcn31_create_audio(
-		struct dc_context *ctx, unsigned int inst)
-{
-	return dce_audio_create(ctx, inst,
-			&audio_regs[inst], &audio_shift, &audio_mask);
-}
-
-static struct vpg *dcn31_vpg_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn31_vpg *vpg31 = kzalloc(sizeof(struct dcn31_vpg), GFP_KERNEL);
-
-	if (!vpg31)
-		return NULL;
-
-	vpg31_construct(vpg31, ctx, inst,
-			&vpg_regs[inst],
-			&vpg_shift,
-			&vpg_mask);
-
-	return &vpg31->base;
-}
-
-static struct afmt *dcn31_afmt_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn31_afmt *afmt31 = kzalloc(sizeof(struct dcn31_afmt), GFP_KERNEL);
-
-	if (!afmt31)
-		return NULL;
-
-	afmt31_construct(afmt31, ctx, inst,
-			&afmt_regs[inst],
-			&afmt_shift,
-			&afmt_mask);
-
-	// Light sleep by default, no need to power down here
-
-	return &afmt31->base;
-}
-
-static struct apg *dcn31_apg_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn31_apg *apg31 = kzalloc(sizeof(struct dcn31_apg), GFP_KERNEL);
-
-	if (!apg31)
-		return NULL;
-
-	apg31_construct(apg31, ctx, inst,
-			&apg_regs[inst],
-			&apg_shift,
-			&apg_mask);
-
-	return &apg31->base;
-}
-
-static struct stream_encoder *dcn314_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dcn10_stream_encoder *enc1;
-	struct vpg *vpg;
-	struct afmt *afmt;
-	int vpg_inst;
-	int afmt_inst;
-
-	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
-	if (eng_id < ENGINE_ID_DIGF) {
-		vpg_inst = eng_id;
-		afmt_inst = eng_id;
-	} else
-		return NULL;
-
-	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
-	vpg = dcn31_vpg_create(ctx, vpg_inst);
-	afmt = dcn31_afmt_create(ctx, afmt_inst);
-
-	if (!enc1 || !vpg || !afmt) {
-		kfree(enc1);
-		kfree(vpg);
-		kfree(afmt);
-		return NULL;
-	}
-
-	dcn314_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
-					eng_id, vpg, afmt,
-					&stream_enc_regs[eng_id],
-					&se_shift, &se_mask);
-
-	return &enc1->base;
-}
-
-static struct hpo_dp_stream_encoder *dcn31_hpo_dp_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dcn31_hpo_dp_stream_encoder *hpo_dp_enc31;
-	struct vpg *vpg;
-	struct apg *apg;
-	uint32_t hpo_dp_inst;
-	uint32_t vpg_inst;
-	uint32_t apg_inst;
-
-	ASSERT((eng_id >= ENGINE_ID_HPO_DP_0) && (eng_id <= ENGINE_ID_HPO_DP_3));
-	hpo_dp_inst = eng_id - ENGINE_ID_HPO_DP_0;
-
-	/* Mapping of VPG register blocks to HPO DP block instance:
-	 * VPG[6] -> HPO_DP[0]
-	 * VPG[7] -> HPO_DP[1]
-	 * VPG[8] -> HPO_DP[2]
-	 * VPG[9] -> HPO_DP[3]
-	 */
-	//Uses offset index 5-8, but actually maps to vpg_inst 6-9
-	vpg_inst = hpo_dp_inst + 5;
-
-	/* Mapping of APG register blocks to HPO DP block instance:
-	 * APG[0] -> HPO_DP[0]
-	 * APG[1] -> HPO_DP[1]
-	 * APG[2] -> HPO_DP[2]
-	 * APG[3] -> HPO_DP[3]
-	 */
-	apg_inst = hpo_dp_inst;
-
-	/* allocate HPO stream encoder and create VPG sub-block */
-	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_stream_encoder), GFP_KERNEL);
-	vpg = dcn31_vpg_create(ctx, vpg_inst);
-	apg = dcn31_apg_create(ctx, apg_inst);
-
-	if (!hpo_dp_enc31 || !vpg || !apg) {
-		kfree(hpo_dp_enc31);
-		kfree(vpg);
-		kfree(apg);
-		return NULL;
-	}
-
-	dcn31_hpo_dp_stream_encoder_construct(hpo_dp_enc31, ctx, ctx->dc_bios,
-					hpo_dp_inst, eng_id, vpg, apg,
-					&hpo_dp_stream_enc_regs[hpo_dp_inst],
-					&hpo_dp_se_shift, &hpo_dp_se_mask);
-
-	return &hpo_dp_enc31->base;
-}
-
-static struct hpo_dp_link_encoder *dcn31_hpo_dp_link_encoder_create(
-	uint8_t inst,
-	struct dc_context *ctx)
-{
-	struct dcn31_hpo_dp_link_encoder *hpo_dp_enc31;
-
-	/* allocate HPO link encoder */
-	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
-
-	hpo_dp_link_encoder31_construct(hpo_dp_enc31, ctx, inst,
-					&hpo_dp_link_enc_regs[inst],
-					&hpo_dp_le_shift, &hpo_dp_le_mask);
-
-	return &hpo_dp_enc31->base;
-}
-
-static struct dce_hwseq *dcn314_hwseq_create(
-	struct dc_context *ctx)
-{
-	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
-
-	if (hws) {
-		hws->ctx = ctx;
-		hws->regs = &hwseq_reg;
-		hws->shifts = &hwseq_shift;
-		hws->masks = &hwseq_mask;
-	}
-	return hws;
-}
-static const struct resource_create_funcs res_create_funcs = {
-	.read_dce_straps = read_dce_straps,
-	.create_audio = dcn31_create_audio,
-	.create_stream_encoder = dcn314_stream_encoder_create,
-	.create_hpo_dp_stream_encoder = dcn31_hpo_dp_stream_encoder_create,
-	.create_hpo_dp_link_encoder = dcn31_hpo_dp_link_encoder_create,
-	.create_hwseq = dcn314_hwseq_create,
-};
-
-static void dcn314_resource_destruct(struct dcn314_resource_pool *pool)
-{
-	unsigned int i;
-
-	for (i = 0; i < pool->base.stream_enc_count; i++) {
-		if (pool->base.stream_enc[i] != NULL) {
-			if (pool->base.stream_enc[i]->vpg != NULL) {
-				kfree(DCN30_VPG_FROM_VPG(pool->base.stream_enc[i]->vpg));
-				pool->base.stream_enc[i]->vpg = NULL;
-			}
-			if (pool->base.stream_enc[i]->afmt != NULL) {
-				kfree(DCN30_AFMT_FROM_AFMT(pool->base.stream_enc[i]->afmt));
-				pool->base.stream_enc[i]->afmt = NULL;
-			}
-			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
-			pool->base.stream_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.hpo_dp_stream_enc_count; i++) {
-		if (pool->base.hpo_dp_stream_enc[i] != NULL) {
-			if (pool->base.hpo_dp_stream_enc[i]->vpg != NULL) {
-				kfree(DCN30_VPG_FROM_VPG(pool->base.hpo_dp_stream_enc[i]->vpg));
-				pool->base.hpo_dp_stream_enc[i]->vpg = NULL;
-			}
-			if (pool->base.hpo_dp_stream_enc[i]->apg != NULL) {
-				kfree(DCN31_APG_FROM_APG(pool->base.hpo_dp_stream_enc[i]->apg));
-				pool->base.hpo_dp_stream_enc[i]->apg = NULL;
-			}
-			kfree(DCN3_1_HPO_DP_STREAM_ENC_FROM_HPO_STREAM_ENC(pool->base.hpo_dp_stream_enc[i]));
-			pool->base.hpo_dp_stream_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.hpo_dp_link_enc_count; i++) {
-		if (pool->base.hpo_dp_link_enc[i] != NULL) {
-			kfree(DCN3_1_HPO_DP_LINK_ENC_FROM_HPO_LINK_ENC(pool->base.hpo_dp_link_enc[i]));
-			pool->base.hpo_dp_link_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		if (pool->base.dscs[i] != NULL)
-			dcn20_dsc_destroy(&pool->base.dscs[i]);
-	}
-
-	if (pool->base.mpc != NULL) {
-		kfree(TO_DCN20_MPC(pool->base.mpc));
-		pool->base.mpc = NULL;
-	}
-	if (pool->base.hubbub != NULL) {
-		kfree(pool->base.hubbub);
-		pool->base.hubbub = NULL;
-	}
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		if (pool->base.dpps[i] != NULL)
-			dcn31_dpp_destroy(&pool->base.dpps[i]);
-
-		if (pool->base.ipps[i] != NULL)
-			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
-
-		if (pool->base.hubps[i] != NULL) {
-			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
-			pool->base.hubps[i] = NULL;
-		}
-
-		if (pool->base.irqs != NULL)
-			dal_irq_service_destroy(&pool->base.irqs);
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		if (pool->base.engines[i] != NULL)
-			dce110_engine_destroy(&pool->base.engines[i]);
-		if (pool->base.hw_i2cs[i] != NULL) {
-			kfree(pool->base.hw_i2cs[i]);
-			pool->base.hw_i2cs[i] = NULL;
-		}
-		if (pool->base.sw_i2cs[i] != NULL) {
-			kfree(pool->base.sw_i2cs[i]);
-			pool->base.sw_i2cs[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
-		if (pool->base.opps[i] != NULL)
-			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		if (pool->base.timing_generators[i] != NULL)	{
-			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
-			pool->base.timing_generators[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
-		if (pool->base.dwbc[i] != NULL) {
-			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
-			pool->base.dwbc[i] = NULL;
-		}
-		if (pool->base.mcif_wb[i] != NULL) {
-			kfree(TO_DCN30_MMHUBBUB(pool->base.mcif_wb[i]));
-			pool->base.mcif_wb[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.audio_count; i++) {
-		if (pool->base.audios[i])
-			dce_aud_destroy(&pool->base.audios[i]);
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] != NULL) {
-			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
-			pool->base.clock_sources[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
-		if (pool->base.mpc_lut[i] != NULL) {
-			dc_3dlut_func_release(pool->base.mpc_lut[i]);
-			pool->base.mpc_lut[i] = NULL;
-		}
-		if (pool->base.mpc_shaper[i] != NULL) {
-			dc_transfer_func_release(pool->base.mpc_shaper[i]);
-			pool->base.mpc_shaper[i] = NULL;
-		}
-	}
-
-	if (pool->base.dp_clock_source != NULL) {
-		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
-		pool->base.dp_clock_source = NULL;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		if (pool->base.multiple_abms[i] != NULL)
-			dce_abm_destroy(&pool->base.multiple_abms[i]);
-	}
-
-	if (pool->base.psr != NULL)
-		dmub_psr_destroy(&pool->base.psr);
-
-	if (pool->base.replay != NULL)
-		dmub_replay_destroy(&pool->base.replay);
-
-	if (pool->base.dccg != NULL)
-		dcn_dccg_destroy(&pool->base.dccg);
-}
-
-static struct hubp *dcn31_hubp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn20_hubp *hubp2 =
-		kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
-
-	if (!hubp2)
-		return NULL;
-
-	if (hubp31_construct(hubp2, ctx, inst,
-			&hubp_regs[inst], &hubp_shift, &hubp_mask))
-		return &hubp2->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(hubp2);
-	return NULL;
-}
-
-static bool dcn31_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t pipe_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < pipe_count; i++) {
-		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc),
-						    GFP_KERNEL);
-
-		if (!dwbc30) {
-			dm_error("DC: failed to create dwbc30!\n");
-			return false;
-		}
-
-		dcn30_dwbc_construct(dwbc30, ctx,
-				&dwbc30_regs[i],
-				&dwbc30_shift,
-				&dwbc30_mask,
-				i);
-
-		pool->dwbc[i] = &dwbc30->base;
-	}
-	return true;
-}
-
-static bool dcn31_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t pipe_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < pipe_count; i++) {
-		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub),
-						    GFP_KERNEL);
-
-		if (!mcif_wb30) {
-			dm_error("DC: failed to create mcif_wb30!\n");
-			return false;
-		}
-
-		dcn30_mmhubbub_construct(mcif_wb30, ctx,
-				&mcif_wb30_regs[i],
-				&mcif_wb30_shift,
-				&mcif_wb30_mask,
-				i);
-
-		pool->mcif_wb[i] = &mcif_wb30->base;
-	}
-	return true;
-}
-
-static struct display_stream_compressor *dcn314_dsc_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_dsc *dsc =
-		kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
-
-	if (!dsc) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
-	return &dsc->base;
-}
-
-static void dcn314_destroy_resource_pool(struct resource_pool **pool)
-{
-	struct dcn314_resource_pool *dcn314_pool = TO_DCN314_RES_POOL(*pool);
-
-	dcn314_resource_destruct(dcn314_pool);
-	kfree(dcn314_pool);
-	*pool = NULL;
-}
-
-static struct clock_source *dcn31_clock_source_create(
-		struct dc_context *ctx,
-		struct dc_bios *bios,
-		enum clock_source_id id,
-		const struct dce110_clk_src_regs *regs,
-		bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src =
-		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dcn31_clk_src_construct(clk_src, ctx, bios, id,
-			regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-
-	BREAK_TO_DEBUGGER();
-	kfree(clk_src);
-	return NULL;
-}
-
-static int dcn314_populate_dml_pipes_from_context(
-	struct dc *dc, struct dc_state *context,
-	display_e2e_pipe_params_st *pipes,
-	bool fast_validate)
-{
-	int pipe_cnt;
-
-	DC_FP_START();
-	pipe_cnt = dcn314_populate_dml_pipes_from_context_fpu(dc, context, pipes, fast_validate);
-	DC_FP_END();
-
-	return pipe_cnt;
-}
-
-static struct dc_cap_funcs cap_funcs = {
-	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
-};
-
-static void dcn314_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
-{
-	DC_FP_START();
-	dcn314_update_bw_bounding_box_fpu(dc, bw_params);
-	DC_FP_END();
-}
-
-static void dcn314_get_panel_config_defaults(struct dc_panel_config *panel_config)
-{
-	*panel_config = panel_config_defaults;
-}
-
-static bool filter_modes_for_single_channel_workaround(struct dc *dc,
-		struct dc_state *context)
-{
-	// Filter 2K@240Hz+8K@24fps above combination timing if memory only has single dimm LPDDR
-	if (dc->clk_mgr->bw_params->vram_type == 34 &&
-	    dc->clk_mgr->bw_params->num_channels < 2 &&
-	    context->stream_count > 1) {
-		int total_phy_pix_clk = 0;
-
-		for (int i = 0; i < context->stream_count; i++)
-			if (context->res_ctx.pipe_ctx[i].stream)
-				total_phy_pix_clk += context->res_ctx.pipe_ctx[i].stream->phy_pix_clk;
-
-		if (total_phy_pix_clk >= (1148928+826260)) //2K@240Hz+8K@24fps
-			return true;
-	}
-	return false;
-}
-
-bool dcn314_validate_bandwidth(struct dc *dc,
-		struct dc_state *context,
-		bool fast_validate)
-{
-	bool out = false;
-
-	BW_VAL_TRACE_SETUP();
-
-	int vlevel = 0;
-	int pipe_cnt = 0;
-	display_e2e_pipe_params_st *pipes = kzalloc(dc->res_pool->pipe_count * sizeof(display_e2e_pipe_params_st), GFP_KERNEL);
-	DC_LOGGER_INIT(dc->ctx->logger);
-
-	BW_VAL_TRACE_COUNT();
-
-	if (filter_modes_for_single_channel_workaround(dc, context))
-		goto validate_fail;
-
-	DC_FP_START();
-	// do not support self refresh only
-	out = dcn30_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, fast_validate, false);
-	DC_FP_END();
-
-	// Disable fast_validate to set min dcfclk in calculate_wm_and_dlg
-	if (pipe_cnt == 0)
-		fast_validate = false;
-
-	if (!out)
-		goto validate_fail;
-
-	BW_VAL_TRACE_END_VOLTAGE_LEVEL();
-
-	if (fast_validate) {
-		BW_VAL_TRACE_SKIP(fast);
-		goto validate_out;
-	}
-	if (dc->res_pool->funcs->calculate_wm_and_dlg)
-		dc->res_pool->funcs->calculate_wm_and_dlg(dc, context, pipes, pipe_cnt, vlevel);
-
-	BW_VAL_TRACE_END_WATERMARKS();
-
-	goto validate_out;
-
-validate_fail:
-	DC_LOG_WARNING("Mode Validation Warning: %s failed validation.\n",
-		dml_get_status_message(context->bw_ctx.dml.vba.ValidationStatus[context->bw_ctx.dml.vba.soc.num_states]));
-
-	BW_VAL_TRACE_SKIP(fail);
-	out = false;
-
-validate_out:
-	kfree(pipes);
-
-	BW_VAL_TRACE_FINISH();
-
-	return out;
-}
-
-static struct resource_funcs dcn314_res_pool_funcs = {
-	.destroy = dcn314_destroy_resource_pool,
-	.link_enc_create = dcn31_link_encoder_create,
-	.link_enc_create_minimal = dcn31_link_enc_create_minimal,
-	.link_encs_assign = link_enc_cfg_link_encs_assign,
-	.link_enc_unassign = link_enc_cfg_link_enc_unassign,
-	.panel_cntl_create = dcn31_panel_cntl_create,
-	.validate_bandwidth = dcn314_validate_bandwidth,
-	.calculate_wm_and_dlg = dcn31_calculate_wm_and_dlg,
-	.update_soc_for_wm_a = dcn31_update_soc_for_wm_a,
-	.populate_dml_pipes = dcn314_populate_dml_pipes_from_context,
-	.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
-	.release_pipe = dcn20_release_pipe,
-	.add_stream_to_ctx = dcn30_add_stream_to_ctx,
-	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
-	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
-	.populate_dml_writeback_from_context = dcn30_populate_dml_writeback_from_context,
-	.set_mcif_arb_params = dcn30_set_mcif_arb_params,
-	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
-	.acquire_post_bldn_3dlut = dcn30_acquire_post_bldn_3dlut,
-	.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
-	.update_bw_bounding_box = dcn314_update_bw_bounding_box,
-	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
-	.get_panel_config_defaults = dcn314_get_panel_config_defaults,
-	.get_preferred_eng_id_dpia = dcn314_get_preferred_eng_id_dpia,
-};
-
-static struct clock_source *dcn30_clock_source_create(
-		struct dc_context *ctx,
-		struct dc_bios *bios,
-		enum clock_source_id id,
-		const struct dce110_clk_src_regs *regs,
-		bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src =
-		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dcn31_clk_src_construct(clk_src, ctx, bios, id,
-			regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-
-	BREAK_TO_DEBUGGER();
-	kfree(clk_src);
-	return NULL;
-}
-
-static bool dcn314_resource_construct(
-	uint8_t num_virtual_links,
-	struct dc *dc,
-	struct dcn314_resource_pool *pool)
-{
-	int i;
-	struct dc_context *ctx = dc->ctx;
-	struct irq_service_init_data init_data;
-
-	ctx->dc_bios->regs = &bios_regs;
-
-	pool->base.res_cap = &res_cap_dcn314;
-	pool->base.funcs = &dcn314_res_pool_funcs;
-
-	/*************************************************
-	 *  Resource + asic cap harcoding                *
-	 *************************************************/
-	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
-	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
-	pool->base.mpcc_count = pool->base.res_cap->num_timing_generator;
-	dc->caps.max_downscale_ratio = 400;
-	dc->caps.i2c_speed_in_khz = 100;
-	dc->caps.i2c_speed_in_khz_hdcp = 100;
-	dc->caps.max_cursor_size = 256;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.dmdata_alloc_size = 2048;
-	dc->caps.max_slave_planes = 2;
-	dc->caps.max_slave_yuv_planes = 2;
-	dc->caps.max_slave_rgb_planes = 2;
-	dc->caps.post_blend_color_processing = true;
-	dc->caps.force_dp_tps4_for_cp2520 = true;
-	if (dc->config.forceHBR2CP2520)
-		dc->caps.force_dp_tps4_for_cp2520 = false;
-	dc->caps.dp_hpo = true;
-	dc->caps.dp_hdmi21_pcon_support = true;
-	dc->caps.edp_dsc_support = true;
-	dc->caps.extended_aux_timeout_support = true;
-	dc->caps.dmcub_support = true;
-	dc->caps.is_apu = true;
-	dc->caps.seamless_odm = true;
-
-	dc->caps.zstate_support = true;
-
-	/* Color pipeline capabilities */
-	dc->caps.color.dpp.dcn_arch = 1;
-	dc->caps.color.dpp.input_lut_shared = 0;
-	dc->caps.color.dpp.icsc = 1;
-	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
-	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
-	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
-	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
-	dc->caps.color.dpp.post_csc = 1;
-	dc->caps.color.dpp.gamma_corr = 1;
-	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
-
-	dc->caps.color.dpp.hw_3d_lut = 1;
-	dc->caps.color.dpp.ogam_ram = 1;
-	// no OGAM ROM on DCN301
-	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
-	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
-	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
-	dc->caps.color.dpp.ocsc = 0;
-
-	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //2
-	dc->caps.color.mpc.ogam_ram = 1;
-	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
-	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
-	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
-	dc->caps.color.mpc.ocsc = 1;
-
-	dc->caps.max_disp_clock_khz_at_vmin = 650000;
-
-	/* Use pipe context based otg sync logic */
-	dc->config.use_pipe_ctx_sync_logic = true;
-
-	/* read VBIOS LTTPR caps */
-	{
-		if (ctx->dc_bios->funcs->get_lttpr_caps) {
-			enum bp_result bp_query_result;
-			uint8_t is_vbios_lttpr_enable = 0;
-
-			bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
-			dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
-		}
-
-		/* interop bit is implicit */
-		{
-			dc->caps.vbios_lttpr_aware = true;
-		}
-	}
-
-	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
-		dc->debug = debug_defaults_drv;
-	else
-		dc->debug = debug_defaults_diags;
-
-	/* Disable pipe power gating */
-	dc->debug.disable_dpp_power_gate = true;
-	dc->debug.disable_hubp_power_gate = true;
-
-	/* Disable root clock optimization */
-	dc->debug.root_clock_optimization.u32All = 0;
-
-	// Init the vm_helper
-	if (dc->vm_helper)
-		vm_helper_init(dc->vm_helper, 16);
-
-	/*************************************************
-	 *  Create resources                             *
-	 *************************************************/
-
-	/* Clock Sources for Pixel Clock*/
-	pool->base.clock_sources[DCN31_CLK_SRC_PLL0] =
-			dcn30_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL0,
-				&clk_src_regs[0], false);
-	pool->base.clock_sources[DCN31_CLK_SRC_PLL1] =
-			dcn30_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL1,
-				&clk_src_regs[1], false);
-	pool->base.clock_sources[DCN31_CLK_SRC_PLL2] =
-			dcn30_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL2,
-				&clk_src_regs[2], false);
-	pool->base.clock_sources[DCN31_CLK_SRC_PLL3] =
-			dcn30_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL3,
-				&clk_src_regs[3], false);
-	pool->base.clock_sources[DCN31_CLK_SRC_PLL4] =
-			dcn30_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL4,
-				&clk_src_regs[4], false);
-
-	pool->base.clk_src_count = DCN30_CLK_SRC_TOTAL;
-
-	/* todo: not reuse phy_pll registers */
-	pool->base.dp_clock_source =
-			dcn31_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_ID_DP_DTO,
-				&clk_src_regs[0], true);
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-
-	pool->base.dccg = dccg314_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
-	if (pool->base.dccg == NULL) {
-		dm_error("DC: failed to create dccg!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	init_data.ctx = dc->ctx;
-	pool->base.irqs = dal_irq_service_dcn314_create(&init_data);
-	if (!pool->base.irqs)
-		goto create_fail;
-
-	/* HUBBUB */
-	pool->base.hubbub = dcn31_hubbub_create(ctx);
-	if (pool->base.hubbub == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create hubbub!\n");
-		goto create_fail;
-	}
-
-	/* HUBPs, DPPs, OPPs and TGs */
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		pool->base.hubps[i] = dcn31_hubp_create(ctx, i);
-		if (pool->base.hubps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create hubps!\n");
-			goto create_fail;
-		}
-
-		pool->base.dpps[i] = dcn31_dpp_create(ctx, i);
-		if (pool->base.dpps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create dpps!\n");
-			goto create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
-		pool->base.opps[i] = dcn31_opp_create(ctx, i);
-		if (pool->base.opps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create output pixel processor!\n");
-			goto create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		pool->base.timing_generators[i] = dcn31_timing_generator_create(
-				ctx, i);
-		if (pool->base.timing_generators[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create tg!\n");
-			goto create_fail;
-		}
-	}
-	pool->base.timing_generator_count = i;
-
-	/* PSR */
-	pool->base.psr = dmub_psr_create(ctx);
-	if (pool->base.psr == NULL) {
-		dm_error("DC: failed to create psr obj!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	/* Replay */
-	pool->base.replay = dmub_replay_create(ctx);
-	if (pool->base.replay == NULL) {
-		dm_error("DC: failed to create replay obj!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	/* ABM */
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		pool->base.multiple_abms[i] = dmub_abm_create(ctx,
-				&abm_regs[i],
-				&abm_shift,
-				&abm_mask);
-		if (pool->base.multiple_abms[i] == NULL) {
-			dm_error("DC: failed to create abm for pipe %d!\n", i);
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-
-	/* MPC and DSC */
-	pool->base.mpc = dcn31_mpc_create(ctx, pool->base.mpcc_count, pool->base.res_cap->num_mpc_3dlut);
-	if (pool->base.mpc == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mpc!\n");
-		goto create_fail;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		pool->base.dscs[i] = dcn314_dsc_create(ctx, i);
-		if (pool->base.dscs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create display stream compressor %d!\n", i);
-			goto create_fail;
-		}
-	}
-
-	/* DWB and MMHUBBUB */
-	if (!dcn31_dwbc_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create dwbc!\n");
-		goto create_fail;
-	}
-
-	if (!dcn31_mmhubbub_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mcif_wb!\n");
-		goto create_fail;
-	}
-
-	/* AUX and I2C */
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		pool->base.engines[i] = dcn31_aux_engine_create(ctx, i);
-		if (pool->base.engines[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create aux engine!!\n");
-			goto create_fail;
-		}
-		pool->base.hw_i2cs[i] = dcn31_i2c_hw_create(ctx, i);
-		if (pool->base.hw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create hw i2c!!\n");
-			goto create_fail;
-		}
-		pool->base.sw_i2cs[i] = NULL;
-	}
-
-	/* DCN314 has 4 DPIA */
-	pool->base.usb4_dpia_count = 4;
-
-	/* Audio, Stream Encoders including HPO and virtual, MPC 3D LUTs */
-	if (!resource_construct(num_virtual_links, dc, &pool->base,
-			&res_create_funcs))
-		goto create_fail;
-
-	/* HW Sequencer and Plane caps */
-	dcn314_hw_sequencer_construct(dc);
-
-	dc->caps.max_planes =  pool->base.pipe_count;
-
-	for (i = 0; i < dc->caps.max_planes; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	dc->cap_funcs = cap_funcs;
-
-	dc->dcn_ip->max_num_dpp = dcn3_14_ip.max_num_dpp;
-
-	return true;
-
-create_fail:
-
-	dcn314_resource_destruct(pool);
-
-	return false;
-}
-
-struct resource_pool *dcn314_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc)
-{
-	struct dcn314_resource_pool *pool =
-		kzalloc(sizeof(struct dcn314_resource_pool), GFP_KERNEL);
-
-	if (!pool)
-		return NULL;
-
-	if (dcn314_resource_construct(init_data->num_virtual_links, dc, pool))
-		return &pool->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(pool);
-	return NULL;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.h b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.h
deleted file mode 100644
index 49ffe71018df..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright 2022 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#ifndef _DCN314_RESOURCE_H_
-#define _DCN314_RESOURCE_H_
-
-#include "core_types.h"
-
-extern struct _vcs_dpi_ip_params_st dcn3_14_ip;
-extern struct _vcs_dpi_soc_bounding_box_st dcn3_14_soc;
-
-#define TO_DCN314_RES_POOL(pool)\
-	container_of(pool, struct dcn314_resource_pool, base)
-
-struct dcn314_resource_pool {
-	struct resource_pool base;
-};
-
-bool dcn314_validate_bandwidth(struct dc *dc,
-		struct dc_state *context,
-		bool fast_validate);
-
-struct resource_pool *dcn314_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc);
-
-#endif /* _DCN314_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn315/Makefile b/drivers/gpu/drm/amd/display/dc/dcn315/Makefile
deleted file mode 100644
index 59381d24800b..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn315/Makefile
+++ /dev/null
@@ -1,30 +0,0 @@
-#
-# Copyright © 2021 Advanced Micro Devices, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the "Software"),
-# to deal in the Software without restriction, including without limitation
-# the rights to use, copy, modify, merge, publish, distribute, sublicense,
-# and/or sell copies of the Software, and to permit persons to whom the
-# Software is furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
-# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
-# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
-# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-# OTHER DEALINGS IN THE SOFTWARE.
-#
-# Authors: AMD
-#
-# Makefile for dcn315.
-
-DCN315 = dcn315_resource.o
-
-AMD_DAL_DCN315 = $(addprefix $(AMDDALPATH)/dc/dcn315/,$(DCN315))
-
-AMD_DISPLAY_FILES += $(AMD_DAL_DCN315)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c b/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c
deleted file mode 100644
index cb8024eee8e4..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c
+++ /dev/null
@@ -1,2151 +0,0 @@
-/*
- * Copyright 2021 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-
-#include "dm_services.h"
-#include "dc.h"
-
-#include "dcn31/dcn31_init.h"
-
-#include "resource.h"
-#include "include/irq_service_interface.h"
-#include "dcn315_resource.h"
-
-#include "dcn20/dcn20_resource.h"
-#include "dcn30/dcn30_resource.h"
-#include "dcn31/dcn31_resource.h"
-
-#include "dcn10/dcn10_ipp.h"
-#include "dcn30/dcn30_hubbub.h"
-#include "dcn31/dcn31_hubbub.h"
-#include "dcn30/dcn30_mpc.h"
-#include "dcn31/dcn31_hubp.h"
-#include "irq/dcn315/irq_service_dcn315.h"
-#include "dcn30/dcn30_dpp.h"
-#include "dcn31/dcn31_optc.h"
-#include "dcn20/dcn20_hwseq.h"
-#include "dcn30/dcn30_hwseq.h"
-#include "dce110/dce110_hwseq.h"
-#include "dcn30/dcn30_opp.h"
-#include "dcn20/dcn20_dsc.h"
-#include "dcn30/dcn30_vpg.h"
-#include "dcn30/dcn30_afmt.h"
-#include "dcn30/dcn30_dio_stream_encoder.h"
-#include "dcn31/dcn31_hpo_dp_stream_encoder.h"
-#include "dcn31/dcn31_hpo_dp_link_encoder.h"
-#include "dcn31/dcn31_apg.h"
-#include "dcn31/dcn31_dio_link_encoder.h"
-#include "dcn31/dcn31_vpg.h"
-#include "dcn31/dcn31_afmt.h"
-#include "dce/dce_clock_source.h"
-#include "dce/dce_audio.h"
-#include "dce/dce_hwseq.h"
-#include "clk_mgr.h"
-#include "virtual/virtual_stream_encoder.h"
-#include "dce110/dce110_resource.h"
-#include "dml/display_mode_vba.h"
-#include "dml/dcn31/dcn31_fpu.h"
-#include "dcn31/dcn31_dccg.h"
-#include "dcn10/dcn10_resource.h"
-#include "dcn31/dcn31_panel_cntl.h"
-
-#include "dcn30/dcn30_dwb.h"
-#include "dcn30/dcn30_mmhubbub.h"
-
-#include "dcn/dcn_3_1_5_offset.h"
-#include "dcn/dcn_3_1_5_sh_mask.h"
-#include "dpcs/dpcs_4_2_2_offset.h"
-#include "dpcs/dpcs_4_2_2_sh_mask.h"
-
-#define NBIO_BASE__INST0_SEG0                      0x00000000
-#define NBIO_BASE__INST0_SEG1                      0x00000014
-#define NBIO_BASE__INST0_SEG2                      0x00000D20
-#define NBIO_BASE__INST0_SEG3                      0x00010400
-#define NBIO_BASE__INST0_SEG4                      0x0241B000
-#define NBIO_BASE__INST0_SEG5                      0x04040000
-
-#define DPCS_BASE__INST0_SEG0                      0x00000012
-#define DPCS_BASE__INST0_SEG1                      0x000000C0
-#define DPCS_BASE__INST0_SEG2                      0x000034C0
-#define DPCS_BASE__INST0_SEG3                      0x00009000
-#define DPCS_BASE__INST0_SEG4                      0x02403C00
-#define DPCS_BASE__INST0_SEG5                      0
-
-#define DCN_BASE__INST0_SEG0                       0x00000012
-#define DCN_BASE__INST0_SEG1                       0x000000C0
-#define DCN_BASE__INST0_SEG2                       0x000034C0
-#define DCN_BASE__INST0_SEG3                       0x00009000
-#define DCN_BASE__INST0_SEG4                       0x02403C00
-#define DCN_BASE__INST0_SEG5                       0
-
-#define regBIF_BX_PF2_RSMU_INDEX                                                                        0x0000
-#define regBIF_BX_PF2_RSMU_INDEX_BASE_IDX                                                               1
-#define regBIF_BX_PF2_RSMU_DATA                                                                         0x0001
-#define regBIF_BX_PF2_RSMU_DATA_BASE_IDX                                                                1
-#define regBIF_BX2_BIOS_SCRATCH_6                                                                       0x003e
-#define regBIF_BX2_BIOS_SCRATCH_6_BASE_IDX                                                              1
-#define BIF_BX2_BIOS_SCRATCH_6__BIOS_SCRATCH_6__SHIFT                                                         0x0
-#define BIF_BX2_BIOS_SCRATCH_6__BIOS_SCRATCH_6_MASK                                                           0xFFFFFFFFL
-#define regBIF_BX2_BIOS_SCRATCH_2                                                                       0x003a
-#define regBIF_BX2_BIOS_SCRATCH_2_BASE_IDX                                                              1
-#define BIF_BX2_BIOS_SCRATCH_2__BIOS_SCRATCH_2__SHIFT                                                         0x0
-#define BIF_BX2_BIOS_SCRATCH_2__BIOS_SCRATCH_2_MASK                                                           0xFFFFFFFFL
-#define regBIF_BX2_BIOS_SCRATCH_3                                                                       0x003b
-#define regBIF_BX2_BIOS_SCRATCH_3_BASE_IDX                                                              1
-#define BIF_BX2_BIOS_SCRATCH_3__BIOS_SCRATCH_3__SHIFT                                                         0x0
-#define BIF_BX2_BIOS_SCRATCH_3__BIOS_SCRATCH_3_MASK                                                           0xFFFFFFFFL
-
-#define regDCHUBBUB_DEBUG_CTRL_0                                              0x04d6
-#define regDCHUBBUB_DEBUG_CTRL_0_BASE_IDX                                     2
-#define DCHUBBUB_DEBUG_CTRL_0__DET_DEPTH__SHIFT                               0x10
-#define DCHUBBUB_DEBUG_CTRL_0__DET_DEPTH_MASK                                 0x01FF0000L
-
-#include "reg_helper.h"
-#include "dce/dmub_abm.h"
-#include "dce/dmub_psr.h"
-#include "dce/dce_aux.h"
-#include "dce/dce_i2c.h"
-
-#include "dml/dcn30/display_mode_vba_30.h"
-#include "vm_helper.h"
-#include "dcn20/dcn20_vmid.h"
-
-#include "link_enc_cfg.h"
-
-#define DCN3_15_MAX_DET_SIZE 384
-#define DCN3_15_CRB_SEGMENT_SIZE_KB 64
-#define DCN3_15_MAX_DET_SEGS (DCN3_15_MAX_DET_SIZE / DCN3_15_CRB_SEGMENT_SIZE_KB)
-/* Minimum 3 extra segments need to be in compbuf and claimable to guarantee seamless mpo transitions */
-#define MIN_RESERVED_DET_SEGS 3
-
-enum dcn31_clk_src_array_id {
-	DCN31_CLK_SRC_PLL0,
-	DCN31_CLK_SRC_PLL1,
-	DCN31_CLK_SRC_PLL2,
-	DCN31_CLK_SRC_PLL3,
-	DCN31_CLK_SRC_PLL4,
-	DCN30_CLK_SRC_TOTAL
-};
-
-/* begin *********************
- * macros to expend register list macro defined in HW object header file
- */
-
-/* DCN */
-#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
-
-#define BASE(seg) BASE_INNER(seg)
-
-#define SR(reg_name)\
-		.reg_name = BASE(reg ## reg_name ## _BASE_IDX) +  \
-					reg ## reg_name
-
-#define SRI(reg_name, block, id)\
-	.reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define SRI2(reg_name, block, id)\
-	.reg_name = BASE(reg ## reg_name ## _BASE_IDX) + \
-					reg ## reg_name
-
-#define SRIR(var_name, reg_name, block, id)\
-	.var_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define SRII_MPC_RMU(reg_name, block, id)\
-	.RMU##_##reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define SRII_DWB(reg_name, temp_name, block, id)\
-	.reg_name[id] = BASE(reg ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## temp_name
-
-#define SF_DWB2(reg_name, block, id, field_name, post_fix)	\
-	.field_name = reg_name ## __ ## field_name ## post_fix
-
-#define DCCG_SRII(reg_name, block, id)\
-	.block ## _ ## reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define VUPDATE_SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(reg ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
-					reg ## reg_name ## _ ## block ## id
-
-/* NBIO */
-#define NBIO_BASE_INNER(seg) \
-	NBIO_BASE__INST0_SEG ## seg
-
-#define NBIO_BASE(seg) \
-	NBIO_BASE_INNER(seg)
-
-#define NBIO_SR(reg_name)\
-		.reg_name = NBIO_BASE(regBIF_BX2_ ## reg_name ## _BASE_IDX) + \
-					regBIF_BX2_ ## reg_name
-
-static const struct bios_registers bios_regs = {
-		NBIO_SR(BIOS_SCRATCH_3),
-		NBIO_SR(BIOS_SCRATCH_6)
-};
-
-#define clk_src_regs(index, pllid)\
-[index] = {\
-	CS_COMMON_REG_LIST_DCN3_0(index, pllid),\
-}
-
-static const struct dce110_clk_src_regs clk_src_regs[] = {
-	clk_src_regs(0, A),
-	clk_src_regs(1, B),
-	clk_src_regs(2, C),
-	clk_src_regs(3, D),
-	clk_src_regs(4, E)
-};
-
-static const struct dce110_clk_src_shift cs_shift = {
-		CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
-};
-
-static const struct dce110_clk_src_mask cs_mask = {
-		CS_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
-};
-
-#define abm_regs(id)\
-[id] = {\
-		ABM_DCN302_REG_LIST(id)\
-}
-
-static const struct dce_abm_registers abm_regs[] = {
-		abm_regs(0),
-		abm_regs(1),
-		abm_regs(2),
-		abm_regs(3),
-};
-
-static const struct dce_abm_shift abm_shift = {
-		ABM_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dce_abm_mask abm_mask = {
-		ABM_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define audio_regs(id)\
-[id] = {\
-		AUD_COMMON_REG_LIST(id)\
-}
-
-static const struct dce_audio_registers audio_regs[] = {
-	audio_regs(0),
-	audio_regs(1),
-	audio_regs(2),
-	audio_regs(3),
-	audio_regs(4),
-	audio_regs(5),
-	audio_regs(6)
-};
-
-#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
-		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
-
-static const struct dce_audio_shift audio_shift = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_audio_mask audio_mask = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
-};
-
-#define vpg_regs(id)\
-[id] = {\
-	VPG_DCN31_REG_LIST(id)\
-}
-
-static const struct dcn31_vpg_registers vpg_regs[] = {
-	vpg_regs(0),
-	vpg_regs(1),
-	vpg_regs(2),
-	vpg_regs(3),
-	vpg_regs(4),
-	vpg_regs(5),
-	vpg_regs(6),
-	vpg_regs(7),
-	vpg_regs(8),
-	vpg_regs(9),
-};
-
-static const struct dcn31_vpg_shift vpg_shift = {
-	DCN31_VPG_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_vpg_mask vpg_mask = {
-	DCN31_VPG_MASK_SH_LIST(_MASK)
-};
-
-#define afmt_regs(id)\
-[id] = {\
-	AFMT_DCN31_REG_LIST(id)\
-}
-
-static const struct dcn31_afmt_registers afmt_regs[] = {
-	afmt_regs(0),
-	afmt_regs(1),
-	afmt_regs(2),
-	afmt_regs(3),
-	afmt_regs(4),
-	afmt_regs(5)
-};
-
-static const struct dcn31_afmt_shift afmt_shift = {
-	DCN31_AFMT_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_afmt_mask afmt_mask = {
-	DCN31_AFMT_MASK_SH_LIST(_MASK)
-};
-
-#define apg_regs(id)\
-[id] = {\
-	APG_DCN31_REG_LIST(id)\
-}
-
-static const struct dcn31_apg_registers apg_regs[] = {
-	apg_regs(0),
-	apg_regs(1),
-	apg_regs(2),
-	apg_regs(3)
-};
-
-static const struct dcn31_apg_shift apg_shift = {
-	DCN31_APG_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_apg_mask apg_mask = {
-		DCN31_APG_MASK_SH_LIST(_MASK)
-};
-
-#define stream_enc_regs(id)\
-[id] = {\
-	SE_DCN3_REG_LIST(id)\
-}
-
-static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
-	stream_enc_regs(0),
-	stream_enc_regs(1),
-	stream_enc_regs(2),
-	stream_enc_regs(3),
-	stream_enc_regs(4)
-};
-
-static const struct dcn10_stream_encoder_shift se_shift = {
-		SE_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn10_stream_encoder_mask se_mask = {
-		SE_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-
-#define aux_regs(id)\
-[id] = {\
-	DCN2_AUX_REG_LIST(id)\
-}
-
-static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
-		aux_regs(0),
-		aux_regs(1),
-		aux_regs(2),
-		aux_regs(3),
-		aux_regs(4)
-};
-
-#define hpd_regs(id)\
-[id] = {\
-	HPD_REG_LIST(id)\
-}
-
-static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
-		hpd_regs(0),
-		hpd_regs(1),
-		hpd_regs(2),
-		hpd_regs(3),
-		hpd_regs(4)
-};
-
-#define link_regs(id, phyid)\
-[id] = {\
-	LE_DCN31_REG_LIST(id), \
-	UNIPHY_DCN2_REG_LIST(phyid), \
-	DPCS_DCN31_REG_LIST(id), \
-}
-
-static const struct dce110_aux_registers_shift aux_shift = {
-	DCN_AUX_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce110_aux_registers_mask aux_mask = {
-	DCN_AUX_MASK_SH_LIST(_MASK)
-};
-
-static const struct dcn10_link_enc_registers link_enc_regs[] = {
-	link_regs(0, A),
-	link_regs(1, B),
-	link_regs(2, C),
-	link_regs(3, D),
-	link_regs(4, E)
-};
-
-static const struct dcn10_link_enc_shift le_shift = {
-	LINK_ENCODER_MASK_SH_LIST_DCN31(__SHIFT), \
-	DPCS_DCN31_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn10_link_enc_mask le_mask = {
-	LINK_ENCODER_MASK_SH_LIST_DCN31(_MASK), \
-	DPCS_DCN31_MASK_SH_LIST(_MASK)
-};
-
-#define hpo_dp_stream_encoder_reg_list(id)\
-[id] = {\
-	DCN3_1_HPO_DP_STREAM_ENC_REG_LIST(id)\
-}
-
-static const struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[] = {
-	hpo_dp_stream_encoder_reg_list(0),
-	hpo_dp_stream_encoder_reg_list(1),
-	hpo_dp_stream_encoder_reg_list(2),
-	hpo_dp_stream_encoder_reg_list(3),
-};
-
-static const struct dcn31_hpo_dp_stream_encoder_shift hpo_dp_se_shift = {
-	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_hpo_dp_stream_encoder_mask hpo_dp_se_mask = {
-	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(_MASK)
-};
-
-
-#define hpo_dp_link_encoder_reg_list(id)\
-[id] = {\
-	DCN3_1_HPO_DP_LINK_ENC_REG_LIST(id),\
-	DCN3_1_RDPCSTX_REG_LIST(0),\
-	DCN3_1_RDPCSTX_REG_LIST(1),\
-	DCN3_1_RDPCSTX_REG_LIST(2),\
-	DCN3_1_RDPCSTX_REG_LIST(3),\
-	DCN3_1_RDPCSTX_REG_LIST(4)\
-}
-
-static const struct dcn31_hpo_dp_link_encoder_registers hpo_dp_link_enc_regs[] = {
-	hpo_dp_link_encoder_reg_list(0),
-	hpo_dp_link_encoder_reg_list(1),
-};
-
-static const struct dcn31_hpo_dp_link_encoder_shift hpo_dp_le_shift = {
-	DCN3_1_HPO_DP_LINK_ENC_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_hpo_dp_link_encoder_mask hpo_dp_le_mask = {
-	DCN3_1_HPO_DP_LINK_ENC_MASK_SH_LIST(_MASK)
-};
-
-#define dpp_regs(id)\
-[id] = {\
-	DPP_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn3_dpp_registers dpp_regs[] = {
-	dpp_regs(0),
-	dpp_regs(1),
-	dpp_regs(2),
-	dpp_regs(3)
-};
-
-static const struct dcn3_dpp_shift tf_shift = {
-		DPP_REG_LIST_SH_MASK_DCN30(__SHIFT)
-};
-
-static const struct dcn3_dpp_mask tf_mask = {
-		DPP_REG_LIST_SH_MASK_DCN30(_MASK)
-};
-
-#define opp_regs(id)\
-[id] = {\
-	OPP_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn20_opp_registers opp_regs[] = {
-	opp_regs(0),
-	opp_regs(1),
-	opp_regs(2),
-	opp_regs(3)
-};
-
-static const struct dcn20_opp_shift opp_shift = {
-	OPP_MASK_SH_LIST_DCN20(__SHIFT)
-};
-
-static const struct dcn20_opp_mask opp_mask = {
-	OPP_MASK_SH_LIST_DCN20(_MASK)
-};
-
-#define aux_engine_regs(id)\
-[id] = {\
-	AUX_COMMON_REG_LIST0(id), \
-	.AUXN_IMPCAL = 0, \
-	.AUXP_IMPCAL = 0, \
-	.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
-}
-
-static const struct dce110_aux_registers aux_engine_regs[] = {
-		aux_engine_regs(0),
-		aux_engine_regs(1),
-		aux_engine_regs(2),
-		aux_engine_regs(3),
-		aux_engine_regs(4)
-};
-
-#define dwbc_regs_dcn3(id)\
-[id] = {\
-	DWBC_COMMON_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn30_dwbc_registers dwbc30_regs[] = {
-	dwbc_regs_dcn3(0),
-};
-
-static const struct dcn30_dwbc_shift dwbc30_shift = {
-	DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_dwbc_mask dwbc30_mask = {
-	DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define mcif_wb_regs_dcn3(id)\
-[id] = {\
-	MCIF_WB_COMMON_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn30_mmhubbub_registers mcif_wb30_regs[] = {
-	mcif_wb_regs_dcn3(0)
-};
-
-static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
-	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
-	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define dsc_regsDCN20(id)\
-[id] = {\
-	DSC_REG_LIST_DCN20(id)\
-}
-
-static const struct dcn20_dsc_registers dsc_regs[] = {
-	dsc_regsDCN20(0),
-	dsc_regsDCN20(1),
-	dsc_regsDCN20(2)
-};
-
-static const struct dcn20_dsc_shift dsc_shift = {
-	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
-};
-
-static const struct dcn20_dsc_mask dsc_mask = {
-	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
-};
-
-static const struct dcn30_mpc_registers mpc_regs = {
-		MPC_REG_LIST_DCN3_0(0),
-		MPC_REG_LIST_DCN3_0(1),
-		MPC_REG_LIST_DCN3_0(2),
-		MPC_REG_LIST_DCN3_0(3),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(0),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(1),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(2),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(3),
-		MPC_DWB_MUX_REG_LIST_DCN3_0(0),
-};
-
-static const struct dcn30_mpc_shift mpc_shift = {
-	MPC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_mpc_mask mpc_mask = {
-	MPC_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define optc_regs(id)\
-[id] = {OPTC_COMMON_REG_LIST_DCN3_1(id)}
-
-static const struct dcn_optc_registers optc_regs[] = {
-	optc_regs(0),
-	optc_regs(1),
-	optc_regs(2),
-	optc_regs(3)
-};
-
-static const struct dcn_optc_shift optc_shift = {
-	OPTC_COMMON_MASK_SH_LIST_DCN3_1(__SHIFT)
-};
-
-static const struct dcn_optc_mask optc_mask = {
-	OPTC_COMMON_MASK_SH_LIST_DCN3_1(_MASK)
-};
-
-#define hubp_regs(id)\
-[id] = {\
-	HUBP_REG_LIST_DCN30(id)\
-}
-
-static const struct dcn_hubp2_registers hubp_regs[] = {
-		hubp_regs(0),
-		hubp_regs(1),
-		hubp_regs(2),
-		hubp_regs(3)
-};
-
-
-static const struct dcn_hubp2_shift hubp_shift = {
-		HUBP_MASK_SH_LIST_DCN31(__SHIFT)
-};
-
-static const struct dcn_hubp2_mask hubp_mask = {
-		HUBP_MASK_SH_LIST_DCN31(_MASK)
-};
-static const struct dcn_hubbub_registers hubbub_reg = {
-		HUBBUB_REG_LIST_DCN31(0)
-};
-
-static const struct dcn_hubbub_shift hubbub_shift = {
-		HUBBUB_MASK_SH_LIST_DCN31(__SHIFT)
-};
-
-static const struct dcn_hubbub_mask hubbub_mask = {
-		HUBBUB_MASK_SH_LIST_DCN31(_MASK)
-};
-
-static const struct dccg_registers dccg_regs = {
-		DCCG_REG_LIST_DCN31()
-};
-
-static const struct dccg_shift dccg_shift = {
-		DCCG_MASK_SH_LIST_DCN31(__SHIFT)
-};
-
-static const struct dccg_mask dccg_mask = {
-		DCCG_MASK_SH_LIST_DCN31(_MASK)
-};
-
-
-#define SRII2(reg_name_pre, reg_name_post, id)\
-	.reg_name_pre ## _ ##  reg_name_post[id] = BASE(reg ## reg_name_pre \
-			## id ## _ ## reg_name_post ## _BASE_IDX) + \
-			reg ## reg_name_pre ## id ## _ ## reg_name_post
-
-
-#define HWSEQ_DCN31_REG_LIST()\
-	SR(DCHUBBUB_GLOBAL_TIMER_CNTL), \
-	SR(DCHUBBUB_ARB_HOSTVM_CNTL), \
-	SR(DIO_MEM_PWR_CTRL), \
-	SR(ODM_MEM_PWR_CTRL3), \
-	SR(DMU_MEM_PWR_CNTL), \
-	SR(MMHUBBUB_MEM_PWR_CNTL), \
-	SR(DCCG_GATE_DISABLE_CNTL), \
-	SR(DCCG_GATE_DISABLE_CNTL2), \
-	SR(DCFCLK_CNTL),\
-	SR(DC_MEM_GLOBAL_PWR_REQ_CNTL), \
-	SRII(PIXEL_RATE_CNTL, OTG, 0), \
-	SRII(PIXEL_RATE_CNTL, OTG, 1),\
-	SRII(PIXEL_RATE_CNTL, OTG, 2),\
-	SRII(PIXEL_RATE_CNTL, OTG, 3),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 0),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 1),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 2),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 3),\
-	SR(MICROSECOND_TIME_BASE_DIV), \
-	SR(MILLISECOND_TIME_BASE_DIV), \
-	SR(DISPCLK_FREQ_CHANGE_CNTL), \
-	SR(RBBMIF_TIMEOUT_DIS), \
-	SR(RBBMIF_TIMEOUT_DIS_2), \
-	SR(DCHUBBUB_CRC_CTRL), \
-	SR(DPP_TOP0_DPP_CRC_CTRL), \
-	SR(DPP_TOP0_DPP_CRC_VAL_B_A), \
-	SR(DPP_TOP0_DPP_CRC_VAL_R_G), \
-	SR(MPC_CRC_CTRL), \
-	SR(MPC_CRC_RESULT_GB), \
-	SR(MPC_CRC_RESULT_C), \
-	SR(MPC_CRC_RESULT_AR), \
-	SR(DOMAIN0_PG_CONFIG), \
-	SR(DOMAIN1_PG_CONFIG), \
-	SR(DOMAIN2_PG_CONFIG), \
-	SR(DOMAIN3_PG_CONFIG), \
-	SR(DOMAIN16_PG_CONFIG), \
-	SR(DOMAIN17_PG_CONFIG), \
-	SR(DOMAIN18_PG_CONFIG), \
-	SR(DOMAIN0_PG_STATUS), \
-	SR(DOMAIN1_PG_STATUS), \
-	SR(DOMAIN2_PG_STATUS), \
-	SR(DOMAIN3_PG_STATUS), \
-	SR(DOMAIN16_PG_STATUS), \
-	SR(DOMAIN17_PG_STATUS), \
-	SR(DOMAIN18_PG_STATUS), \
-	SR(D1VGA_CONTROL), \
-	SR(D2VGA_CONTROL), \
-	SR(D3VGA_CONTROL), \
-	SR(D4VGA_CONTROL), \
-	SR(D5VGA_CONTROL), \
-	SR(D6VGA_CONTROL), \
-	SR(DC_IP_REQUEST_CNTL), \
-	SR(AZALIA_AUDIO_DTO), \
-	SR(AZALIA_CONTROLLER_CLOCK_GATING), \
-	SR(HPO_TOP_HW_CONTROL)
-
-static const struct dce_hwseq_registers hwseq_reg = {
-		HWSEQ_DCN31_REG_LIST()
-};
-
-#define HWSEQ_DCN31_MASK_SH_LIST(mask_sh)\
-	HWSEQ_DCN_MASK_SH_LIST(mask_sh), \
-	HWS_SF(, DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, mask_sh), \
-	HWS_SF(, DCHUBBUB_ARB_HOSTVM_CNTL, DISABLE_HOSTVM_FORCE_ALLOW_PSTATE, mask_sh), \
-	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN0_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN1_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN2_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN3_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN16_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN17_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN18_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh), \
-	HWS_SF(, AZALIA_AUDIO_DTO, AZALIA_AUDIO_DTO_MODULE, mask_sh), \
-	HWS_SF(, HPO_TOP_CLOCK_CONTROL, HPO_HDMISTREAMCLK_G_GATE_DIS, mask_sh), \
-	HWS_SF(, DMU_MEM_PWR_CNTL, DMCU_ERAM_MEM_PWR_FORCE, mask_sh), \
-	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_UNASSIGNED_PWR_MODE, mask_sh), \
-	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_VBLANK_PWR_MODE, mask_sh), \
-	HWS_SF(, MMHUBBUB_MEM_PWR_CNTL, VGA_MEM_PWR_FORCE, mask_sh), \
-	HWS_SF(, DIO_MEM_PWR_CTRL, I2C_LIGHT_SLEEP_FORCE, mask_sh), \
-	HWS_SF(, HPO_TOP_HW_CONTROL, HPO_IO_EN, mask_sh)
-
-static const struct dce_hwseq_shift hwseq_shift = {
-		HWSEQ_DCN31_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_hwseq_mask hwseq_mask = {
-		HWSEQ_DCN31_MASK_SH_LIST(_MASK)
-};
-#define vmid_regs(id)\
-[id] = {\
-		DCN20_VMID_REG_LIST(id)\
-}
-
-static const struct dcn_vmid_registers vmid_regs[] = {
-	vmid_regs(0),
-	vmid_regs(1),
-	vmid_regs(2),
-	vmid_regs(3),
-	vmid_regs(4),
-	vmid_regs(5),
-	vmid_regs(6),
-	vmid_regs(7),
-	vmid_regs(8),
-	vmid_regs(9),
-	vmid_regs(10),
-	vmid_regs(11),
-	vmid_regs(12),
-	vmid_regs(13),
-	vmid_regs(14),
-	vmid_regs(15)
-};
-
-static const struct dcn20_vmid_shift vmid_shifts = {
-		DCN20_VMID_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn20_vmid_mask vmid_masks = {
-		DCN20_VMID_MASK_SH_LIST(_MASK)
-};
-
-static const struct resource_caps res_cap_dcn31 = {
-	.num_timing_generator = 4,
-	.num_opp = 4,
-	.num_video_plane = 4,
-	.num_audio = 5,
-	.num_stream_encoder = 5,
-	.num_dig_link_enc = 5,
-	.num_hpo_dp_stream_encoder = 4,
-	.num_hpo_dp_link_encoder = 2,
-	.num_pll = 5,
-	.num_dwb = 1,
-	.num_ddc = 5,
-	.num_vmid = 16,
-	.num_mpc_3dlut = 2,
-	.num_dsc = 3,
-};
-
-static const struct dc_plane_cap plane_cap = {
-	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.per_pixel_alpha = true,
-
-	.pixel_format_support = {
-			.argb8888 = true,
-			.nv12 = true,
-			.fp16 = true,
-			.p010 = true,
-			.ayuv = false,
-	},
-
-	.max_upscale_factor = {
-			.argb8888 = 16000,
-			.nv12 = 16000,
-			.fp16 = 16000
-	},
-
-	// 6:1 downscaling ratio: 1000/6 = 166.666
-	.max_downscale_factor = {
-			.argb8888 = 167,
-			.nv12 = 167,
-			.fp16 = 167
-	},
-	64,
-	64
-};
-
-static const struct dc_debug_options debug_defaults_drv = {
-	.disable_z10 = true, /*hw not support it*/
-	.disable_dmcu = true,
-	.force_abm_enable = false,
-	.timing_trace = false,
-	.clock_trace = true,
-	.disable_pplib_clock_request = false,
-	.pipe_split_policy = MPC_SPLIT_DYNAMIC,
-	.force_single_disp_pipe_split = false,
-	.disable_dcc = DCC_ENABLE,
-	.vsr_support = true,
-	.performance_trace = false,
-	.max_downscale_src_width = 4096,/*upto true 4k*/
-	.disable_pplib_wm_range = false,
-	.scl_reset_length10 = true,
-	.sanity_checks = false,
-	.underflow_assert_delay_us = 0xFFFFFFFF,
-	.dwb_fi_phase = -1, // -1 = disable,
-	.dmub_command_table = true,
-	.pstate_enabled = true,
-	.use_max_lb = true,
-	.enable_mem_low_power = {
-		.bits = {
-			.vga = true,
-			.i2c = true,
-			.dmcu = false, // This is previously known to cause hang on S3 cycles if enabled
-			.dscl = true,
-			.cm = true,
-			.mpc = true,
-			.optc = true,
-			.vpg = true,
-			.afmt = true,
-		}
-	},
-	.enable_legacy_fast_update = true,
-	.psr_power_use_phy_fsm = 0,
-	.using_dml2 = false,
-};
-
-static const struct dc_panel_config panel_config_defaults = {
-	.psr = {
-		.disable_psr = false,
-		.disallow_psrsu = false,
-		.disallow_replay = false,
-	},
-	.ilr = {
-		.optimize_edp_link_rate = true,
-	},
-};
-
-static void dcn31_dpp_destroy(struct dpp **dpp)
-{
-	kfree(TO_DCN20_DPP(*dpp));
-	*dpp = NULL;
-}
-
-static struct dpp *dcn31_dpp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn3_dpp *dpp =
-		kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
-
-	if (!dpp)
-		return NULL;
-
-	if (dpp3_construct(dpp, ctx, inst,
-			&dpp_regs[inst], &tf_shift, &tf_mask))
-		return &dpp->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(dpp);
-	return NULL;
-}
-
-static struct output_pixel_processor *dcn31_opp_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_opp *opp =
-		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
-
-	if (!opp) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dcn20_opp_construct(opp, ctx, inst,
-			&opp_regs[inst], &opp_shift, &opp_mask);
-	return &opp->base;
-}
-
-static struct dce_aux *dcn31_aux_engine_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct aux_engine_dce110 *aux_engine =
-		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
-
-	if (!aux_engine)
-		return NULL;
-
-	dce110_aux_engine_construct(aux_engine, ctx, inst,
-				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
-				    &aux_engine_regs[inst],
-					&aux_mask,
-					&aux_shift,
-					ctx->dc->caps.extended_aux_timeout_support);
-
-	return &aux_engine->base;
-}
-#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST_DCN30(id) }
-
-static const struct dce_i2c_registers i2c_hw_regs[] = {
-		i2c_inst_regs(1),
-		i2c_inst_regs(2),
-		i2c_inst_regs(3),
-		i2c_inst_regs(4),
-		i2c_inst_regs(5),
-};
-
-static const struct dce_i2c_shift i2c_shifts = {
-		I2C_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dce_i2c_mask i2c_masks = {
-		I2C_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static struct dce_i2c_hw *dcn31_i2c_hw_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_i2c_hw *dce_i2c_hw =
-		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
-
-	if (!dce_i2c_hw)
-		return NULL;
-
-	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
-				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
-
-	return dce_i2c_hw;
-}
-static struct mpc *dcn31_mpc_create(
-		struct dc_context *ctx,
-		int num_mpcc,
-		int num_rmu)
-{
-	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc),
-					  GFP_KERNEL);
-
-	if (!mpc30)
-		return NULL;
-
-	dcn30_mpc_construct(mpc30, ctx,
-			&mpc_regs,
-			&mpc_shift,
-			&mpc_mask,
-			num_mpcc,
-			num_rmu);
-
-	return &mpc30->base;
-}
-
-static struct hubbub *dcn31_hubbub_create(struct dc_context *ctx)
-{
-	int i;
-
-	struct dcn20_hubbub *hubbub3 = kzalloc(sizeof(struct dcn20_hubbub),
-					  GFP_KERNEL);
-
-	if (!hubbub3)
-		return NULL;
-
-	hubbub31_construct(hubbub3, ctx,
-			&hubbub_reg,
-			&hubbub_shift,
-			&hubbub_mask,
-			dcn3_15_ip.det_buffer_size_kbytes,
-			dcn3_15_ip.pixel_chunk_size_kbytes,
-			dcn3_15_ip.config_return_buffer_size_in_kbytes);
-
-
-	for (i = 0; i < res_cap_dcn31.num_vmid; i++) {
-		struct dcn20_vmid *vmid = &hubbub3->vmid[i];
-
-		vmid->ctx = ctx;
-
-		vmid->regs = &vmid_regs[i];
-		vmid->shifts = &vmid_shifts;
-		vmid->masks = &vmid_masks;
-	}
-
-	return &hubbub3->base;
-}
-
-static struct timing_generator *dcn31_timing_generator_create(
-		struct dc_context *ctx,
-		uint32_t instance)
-{
-	struct optc *tgn10 =
-		kzalloc(sizeof(struct optc), GFP_KERNEL);
-
-	if (!tgn10)
-		return NULL;
-
-	tgn10->base.inst = instance;
-	tgn10->base.ctx = ctx;
-
-	tgn10->tg_regs = &optc_regs[instance];
-	tgn10->tg_shift = &optc_shift;
-	tgn10->tg_mask = &optc_mask;
-
-	dcn31_timing_generator_init(tgn10);
-
-	return &tgn10->base;
-}
-
-static const struct encoder_feature_support link_enc_feature = {
-		.max_hdmi_deep_color = COLOR_DEPTH_121212,
-		.max_hdmi_pixel_clock = 600000,
-		.hdmi_ycbcr420_supported = true,
-		.dp_ycbcr420_supported = true,
-		.fec_supported = true,
-		.flags.bits.IS_HBR2_CAPABLE = true,
-		.flags.bits.IS_HBR3_CAPABLE = true,
-		.flags.bits.IS_TPS3_CAPABLE = true,
-		.flags.bits.IS_TPS4_CAPABLE = true
-};
-
-static struct link_encoder *dcn31_link_encoder_create(
-	struct dc_context *ctx,
-	const struct encoder_init_data *enc_init_data)
-{
-	struct dcn20_link_encoder *enc20 =
-		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
-
-	if (!enc20)
-		return NULL;
-
-	dcn31_link_encoder_construct(enc20,
-			enc_init_data,
-			&link_enc_feature,
-			&link_enc_regs[enc_init_data->transmitter],
-			&link_enc_aux_regs[enc_init_data->channel - 1],
-			&link_enc_hpd_regs[enc_init_data->hpd_source],
-			&le_shift,
-			&le_mask);
-
-	return &enc20->enc10.base;
-}
-
-/* Create a minimal link encoder object not associated with a particular
- * physical connector.
- * resource_funcs.link_enc_create_minimal
- */
-static struct link_encoder *dcn31_link_enc_create_minimal(
-		struct dc_context *ctx, enum engine_id eng_id)
-{
-	struct dcn20_link_encoder *enc20;
-
-	if ((eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
-		return NULL;
-
-	enc20 = kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
-	if (!enc20)
-		return NULL;
-
-	dcn31_link_encoder_construct_minimal(
-			enc20,
-			ctx,
-			&link_enc_feature,
-			&link_enc_regs[eng_id - ENGINE_ID_DIGA],
-			eng_id);
-
-	return &enc20->enc10.base;
-}
-
-static struct panel_cntl *dcn31_panel_cntl_create(const struct panel_cntl_init_data *init_data)
-{
-	struct dcn31_panel_cntl *panel_cntl =
-		kzalloc(sizeof(struct dcn31_panel_cntl), GFP_KERNEL);
-
-	if (!panel_cntl)
-		return NULL;
-
-	dcn31_panel_cntl_construct(panel_cntl, init_data);
-
-	return &panel_cntl->base;
-}
-
-static void read_dce_straps(
-	struct dc_context *ctx,
-	struct resource_straps *straps)
-{
-	generic_reg_get(ctx, regDC_PINSTRAPS + BASE(regDC_PINSTRAPS_BASE_IDX),
-		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
-
-}
-
-static struct audio *dcn31_create_audio(
-		struct dc_context *ctx, unsigned int inst)
-{
-	return dce_audio_create(ctx, inst,
-			&audio_regs[inst], &audio_shift, &audio_mask);
-}
-
-static struct vpg *dcn31_vpg_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn31_vpg *vpg31 = kzalloc(sizeof(struct dcn31_vpg), GFP_KERNEL);
-
-	if (!vpg31)
-		return NULL;
-
-	vpg31_construct(vpg31, ctx, inst,
-			&vpg_regs[inst],
-			&vpg_shift,
-			&vpg_mask);
-
-	return &vpg31->base;
-}
-
-static struct afmt *dcn31_afmt_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn31_afmt *afmt31 = kzalloc(sizeof(struct dcn31_afmt), GFP_KERNEL);
-
-	if (!afmt31)
-		return NULL;
-
-	afmt31_construct(afmt31, ctx, inst,
-			&afmt_regs[inst],
-			&afmt_shift,
-			&afmt_mask);
-
-	// Light sleep by default, no need to power down here
-
-	return &afmt31->base;
-}
-
-static struct apg *dcn31_apg_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn31_apg *apg31 = kzalloc(sizeof(struct dcn31_apg), GFP_KERNEL);
-
-	if (!apg31)
-		return NULL;
-
-	apg31_construct(apg31, ctx, inst,
-			&apg_regs[inst],
-			&apg_shift,
-			&apg_mask);
-
-	return &apg31->base;
-}
-
-static struct stream_encoder *dcn315_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dcn10_stream_encoder *enc1;
-	struct vpg *vpg;
-	struct afmt *afmt;
-	int vpg_inst;
-	int afmt_inst;
-
-	/*PHYB is wired off in HW, allow front end to remapping, otherwise needs more changes*/
-
-	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
-	if (eng_id <= ENGINE_ID_DIGF) {
-		vpg_inst = eng_id;
-		afmt_inst = eng_id;
-	} else
-		return NULL;
-
-	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
-	vpg = dcn31_vpg_create(ctx, vpg_inst);
-	afmt = dcn31_afmt_create(ctx, afmt_inst);
-
-	if (!enc1 || !vpg || !afmt) {
-		kfree(enc1);
-		kfree(vpg);
-		kfree(afmt);
-		return NULL;
-	}
-
-	dcn30_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
-					eng_id, vpg, afmt,
-					&stream_enc_regs[eng_id],
-					&se_shift, &se_mask);
-
-	return &enc1->base;
-}
-
-static struct hpo_dp_stream_encoder *dcn31_hpo_dp_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dcn31_hpo_dp_stream_encoder *hpo_dp_enc31;
-	struct vpg *vpg;
-	struct apg *apg;
-	uint32_t hpo_dp_inst;
-	uint32_t vpg_inst;
-	uint32_t apg_inst;
-
-	ASSERT((eng_id >= ENGINE_ID_HPO_DP_0) && (eng_id <= ENGINE_ID_HPO_DP_3));
-	hpo_dp_inst = eng_id - ENGINE_ID_HPO_DP_0;
-
-	/* Mapping of VPG register blocks to HPO DP block instance:
-	 * VPG[6] -> HPO_DP[0]
-	 * VPG[7] -> HPO_DP[1]
-	 * VPG[8] -> HPO_DP[2]
-	 * VPG[9] -> HPO_DP[3]
-	 */
-	vpg_inst = hpo_dp_inst + 6;
-
-	/* Mapping of APG register blocks to HPO DP block instance:
-	 * APG[0] -> HPO_DP[0]
-	 * APG[1] -> HPO_DP[1]
-	 * APG[2] -> HPO_DP[2]
-	 * APG[3] -> HPO_DP[3]
-	 */
-	apg_inst = hpo_dp_inst;
-
-	/* allocate HPO stream encoder and create VPG sub-block */
-	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_stream_encoder), GFP_KERNEL);
-	vpg = dcn31_vpg_create(ctx, vpg_inst);
-	apg = dcn31_apg_create(ctx, apg_inst);
-
-	if (!hpo_dp_enc31 || !vpg || !apg) {
-		kfree(hpo_dp_enc31);
-		kfree(vpg);
-		kfree(apg);
-		return NULL;
-	}
-
-	dcn31_hpo_dp_stream_encoder_construct(hpo_dp_enc31, ctx, ctx->dc_bios,
-					hpo_dp_inst, eng_id, vpg, apg,
-					&hpo_dp_stream_enc_regs[hpo_dp_inst],
-					&hpo_dp_se_shift, &hpo_dp_se_mask);
-
-	return &hpo_dp_enc31->base;
-}
-
-static struct hpo_dp_link_encoder *dcn31_hpo_dp_link_encoder_create(
-	uint8_t inst,
-	struct dc_context *ctx)
-{
-	struct dcn31_hpo_dp_link_encoder *hpo_dp_enc31;
-
-	/* allocate HPO link encoder */
-	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
-
-	hpo_dp_link_encoder31_construct(hpo_dp_enc31, ctx, inst,
-					&hpo_dp_link_enc_regs[inst],
-					&hpo_dp_le_shift, &hpo_dp_le_mask);
-
-	return &hpo_dp_enc31->base;
-}
-
-static struct dce_hwseq *dcn31_hwseq_create(
-	struct dc_context *ctx)
-{
-	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
-
-	if (hws) {
-		hws->ctx = ctx;
-		hws->regs = &hwseq_reg;
-		hws->shifts = &hwseq_shift;
-		hws->masks = &hwseq_mask;
-	}
-	return hws;
-}
-static const struct resource_create_funcs res_create_funcs = {
-	.read_dce_straps = read_dce_straps,
-	.create_audio = dcn31_create_audio,
-	.create_stream_encoder = dcn315_stream_encoder_create,
-	.create_hpo_dp_stream_encoder = dcn31_hpo_dp_stream_encoder_create,
-	.create_hpo_dp_link_encoder = dcn31_hpo_dp_link_encoder_create,
-	.create_hwseq = dcn31_hwseq_create,
-};
-
-static void dcn315_resource_destruct(struct dcn315_resource_pool *pool)
-{
-	unsigned int i;
-
-	for (i = 0; i < pool->base.stream_enc_count; i++) {
-		if (pool->base.stream_enc[i] != NULL) {
-			if (pool->base.stream_enc[i]->vpg != NULL) {
-				kfree(DCN30_VPG_FROM_VPG(pool->base.stream_enc[i]->vpg));
-				pool->base.stream_enc[i]->vpg = NULL;
-			}
-			if (pool->base.stream_enc[i]->afmt != NULL) {
-				kfree(DCN30_AFMT_FROM_AFMT(pool->base.stream_enc[i]->afmt));
-				pool->base.stream_enc[i]->afmt = NULL;
-			}
-			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
-			pool->base.stream_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.hpo_dp_stream_enc_count; i++) {
-		if (pool->base.hpo_dp_stream_enc[i] != NULL) {
-			if (pool->base.hpo_dp_stream_enc[i]->vpg != NULL) {
-				kfree(DCN30_VPG_FROM_VPG(pool->base.hpo_dp_stream_enc[i]->vpg));
-				pool->base.hpo_dp_stream_enc[i]->vpg = NULL;
-			}
-			if (pool->base.hpo_dp_stream_enc[i]->apg != NULL) {
-				kfree(DCN31_APG_FROM_APG(pool->base.hpo_dp_stream_enc[i]->apg));
-				pool->base.hpo_dp_stream_enc[i]->apg = NULL;
-			}
-			kfree(DCN3_1_HPO_DP_STREAM_ENC_FROM_HPO_STREAM_ENC(pool->base.hpo_dp_stream_enc[i]));
-			pool->base.hpo_dp_stream_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.hpo_dp_link_enc_count; i++) {
-		if (pool->base.hpo_dp_link_enc[i] != NULL) {
-			kfree(DCN3_1_HPO_DP_LINK_ENC_FROM_HPO_LINK_ENC(pool->base.hpo_dp_link_enc[i]));
-			pool->base.hpo_dp_link_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		if (pool->base.dscs[i] != NULL)
-			dcn20_dsc_destroy(&pool->base.dscs[i]);
-	}
-
-	if (pool->base.mpc != NULL) {
-		kfree(TO_DCN20_MPC(pool->base.mpc));
-		pool->base.mpc = NULL;
-	}
-	if (pool->base.hubbub != NULL) {
-		kfree(pool->base.hubbub);
-		pool->base.hubbub = NULL;
-	}
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		if (pool->base.dpps[i] != NULL)
-			dcn31_dpp_destroy(&pool->base.dpps[i]);
-
-		if (pool->base.ipps[i] != NULL)
-			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
-
-		if (pool->base.hubps[i] != NULL) {
-			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
-			pool->base.hubps[i] = NULL;
-		}
-
-		if (pool->base.irqs != NULL) {
-			dal_irq_service_destroy(&pool->base.irqs);
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		if (pool->base.engines[i] != NULL)
-			dce110_engine_destroy(&pool->base.engines[i]);
-		if (pool->base.hw_i2cs[i] != NULL) {
-			kfree(pool->base.hw_i2cs[i]);
-			pool->base.hw_i2cs[i] = NULL;
-		}
-		if (pool->base.sw_i2cs[i] != NULL) {
-			kfree(pool->base.sw_i2cs[i]);
-			pool->base.sw_i2cs[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
-		if (pool->base.opps[i] != NULL)
-			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		if (pool->base.timing_generators[i] != NULL)	{
-			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
-			pool->base.timing_generators[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
-		if (pool->base.dwbc[i] != NULL) {
-			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
-			pool->base.dwbc[i] = NULL;
-		}
-		if (pool->base.mcif_wb[i] != NULL) {
-			kfree(TO_DCN30_MMHUBBUB(pool->base.mcif_wb[i]));
-			pool->base.mcif_wb[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.audio_count; i++) {
-		if (pool->base.audios[i])
-			dce_aud_destroy(&pool->base.audios[i]);
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] != NULL) {
-			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
-			pool->base.clock_sources[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
-		if (pool->base.mpc_lut[i] != NULL) {
-			dc_3dlut_func_release(pool->base.mpc_lut[i]);
-			pool->base.mpc_lut[i] = NULL;
-		}
-		if (pool->base.mpc_shaper[i] != NULL) {
-			dc_transfer_func_release(pool->base.mpc_shaper[i]);
-			pool->base.mpc_shaper[i] = NULL;
-		}
-	}
-
-	if (pool->base.dp_clock_source != NULL) {
-		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
-		pool->base.dp_clock_source = NULL;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		if (pool->base.multiple_abms[i] != NULL)
-			dce_abm_destroy(&pool->base.multiple_abms[i]);
-	}
-
-	if (pool->base.psr != NULL)
-		dmub_psr_destroy(&pool->base.psr);
-
-	if (pool->base.dccg != NULL)
-		dcn_dccg_destroy(&pool->base.dccg);
-}
-
-static struct hubp *dcn31_hubp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn20_hubp *hubp2 =
-		kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
-
-	if (!hubp2)
-		return NULL;
-
-	if (hubp31_construct(hubp2, ctx, inst,
-			&hubp_regs[inst], &hubp_shift, &hubp_mask))
-		return &hubp2->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(hubp2);
-	return NULL;
-}
-
-static bool dcn31_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t pipe_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < pipe_count; i++) {
-		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc),
-						    GFP_KERNEL);
-
-		if (!dwbc30) {
-			dm_error("DC: failed to create dwbc30!\n");
-			return false;
-		}
-
-		dcn30_dwbc_construct(dwbc30, ctx,
-				&dwbc30_regs[i],
-				&dwbc30_shift,
-				&dwbc30_mask,
-				i);
-
-		pool->dwbc[i] = &dwbc30->base;
-	}
-	return true;
-}
-
-static bool dcn31_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t pipe_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < pipe_count; i++) {
-		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub),
-						    GFP_KERNEL);
-
-		if (!mcif_wb30) {
-			dm_error("DC: failed to create mcif_wb30!\n");
-			return false;
-		}
-
-		dcn30_mmhubbub_construct(mcif_wb30, ctx,
-				&mcif_wb30_regs[i],
-				&mcif_wb30_shift,
-				&mcif_wb30_mask,
-				i);
-
-		pool->mcif_wb[i] = &mcif_wb30->base;
-	}
-	return true;
-}
-
-static struct display_stream_compressor *dcn31_dsc_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_dsc *dsc =
-		kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
-
-	if (!dsc) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
-	return &dsc->base;
-}
-
-static void dcn315_destroy_resource_pool(struct resource_pool **pool)
-{
-	struct dcn315_resource_pool *dcn31_pool = TO_DCN315_RES_POOL(*pool);
-
-	dcn315_resource_destruct(dcn31_pool);
-	kfree(dcn31_pool);
-	*pool = NULL;
-}
-
-static struct clock_source *dcn31_clock_source_create(
-		struct dc_context *ctx,
-		struct dc_bios *bios,
-		enum clock_source_id id,
-		const struct dce110_clk_src_regs *regs,
-		bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src =
-		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dcn31_clk_src_construct(clk_src, ctx, bios, id,
-			regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-
-	kfree(clk_src);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
-static bool is_dual_plane(enum surface_pixel_format format)
-{
-	return format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN || format == SURFACE_PIXEL_FORMAT_GRPH_RGBE_ALPHA;
-}
-
-static int source_format_to_bpp (enum source_format_class SourcePixelFormat)
-{
-	if (SourcePixelFormat == dm_444_64)
-		return 8;
-	else if (SourcePixelFormat == dm_444_16)
-		return 2;
-	else if (SourcePixelFormat == dm_444_8)
-		return 1;
-	else if (SourcePixelFormat == dm_rgbe_alpha)
-		return 5;
-	else if (SourcePixelFormat == dm_420_8)
-		return 3;
-	else if (SourcePixelFormat == dm_420_12)
-		return 6;
-	else
-		return 4;
-}
-
-static bool allow_pixel_rate_crb(struct dc *dc, struct dc_state *context)
-{
-	int i;
-	struct resource_context *res_ctx = &context->res_ctx;
-
-	/*Don't apply for single stream*/
-	if (context->stream_count < 2)
-		return false;
-
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		if (!res_ctx->pipe_ctx[i].stream)
-			continue;
-
-		/*Don't apply if scaling*/
-		if (res_ctx->pipe_ctx[i].stream->src.width != res_ctx->pipe_ctx[i].stream->dst.width ||
-				res_ctx->pipe_ctx[i].stream->src.height != res_ctx->pipe_ctx[i].stream->dst.height ||
-				(res_ctx->pipe_ctx[i].plane_state && (res_ctx->pipe_ctx[i].plane_state->src_rect.width
-														!= res_ctx->pipe_ctx[i].plane_state->dst_rect.width ||
-					res_ctx->pipe_ctx[i].plane_state->src_rect.height
-														!= res_ctx->pipe_ctx[i].plane_state->dst_rect.height)))
-			return false;
-		/*Don't apply if MPO to avoid transition issues*/
-		if (res_ctx->pipe_ctx[i].top_pipe && res_ctx->pipe_ctx[i].top_pipe->plane_state != res_ctx->pipe_ctx[i].plane_state)
-			return false;
-	}
-	return true;
-}
-
-static int dcn315_populate_dml_pipes_from_context(
-	struct dc *dc, struct dc_state *context,
-	display_e2e_pipe_params_st *pipes,
-	bool fast_validate)
-{
-	int i, pipe_cnt, crb_idx, crb_pipes;
-	struct resource_context *res_ctx = &context->res_ctx;
-	struct pipe_ctx *pipe = NULL;
-	const int max_usable_det = context->bw_ctx.dml.ip.config_return_buffer_size_in_kbytes - DCN3_15_MIN_COMPBUF_SIZE_KB;
-	int remaining_det_segs = max_usable_det / DCN3_15_CRB_SEGMENT_SIZE_KB;
-	bool pixel_rate_crb = allow_pixel_rate_crb(dc, context);
-
-	DC_FP_START();
-	dcn31x_populate_dml_pipes_from_context(dc, context, pipes, fast_validate);
-	DC_FP_END();
-
-	for (i = 0, pipe_cnt = 0, crb_pipes = 0; i < dc->res_pool->pipe_count; i++) {
-		struct dc_crtc_timing *timing;
-
-		if (!res_ctx->pipe_ctx[i].stream)
-			continue;
-		pipe = &res_ctx->pipe_ctx[i];
-		timing = &pipe->stream->timing;
-
-		/*
-		 * Immediate flip can be set dynamically after enabling the plane.
-		 * We need to require support for immediate flip or underflow can be
-		 * intermittently experienced depending on peak b/w requirements.
-		 */
-		pipes[pipe_cnt].pipe.src.immediate_flip = true;
-
-		pipes[pipe_cnt].pipe.src.unbounded_req_mode = false;
-		pipes[pipe_cnt].pipe.dest.vfront_porch = timing->v_front_porch;
-		pipes[pipe_cnt].pipe.src.dcc_rate = 3;
-		pipes[pipe_cnt].dout.dsc_input_bpc = 0;
-		DC_FP_START();
-		dcn31_zero_pipe_dcc_fraction(pipes, pipe_cnt);
-		if (pixel_rate_crb && !pipe->top_pipe && !pipe->prev_odm_pipe) {
-			int bpp = source_format_to_bpp(pipes[pipe_cnt].pipe.src.source_format);
-			/* Ceil to crb segment size */
-			int approx_det_segs_required_for_pstate = dcn_get_approx_det_segs_required_for_pstate(
-					&context->bw_ctx.dml.soc, timing->pix_clk_100hz, bpp, DCN3_15_CRB_SEGMENT_SIZE_KB);
-
-			if (approx_det_segs_required_for_pstate <= 2 * DCN3_15_MAX_DET_SEGS) {
-				bool split_required = approx_det_segs_required_for_pstate > DCN3_15_MAX_DET_SEGS;
-				split_required = split_required || timing->pix_clk_100hz >= dcn_get_max_non_odm_pix_rate_100hz(&dc->dml.soc);
-				split_required = split_required || (pipe->plane_state && pipe->plane_state->src_rect.width > 5120);
-
-				/* Minimum 2 segments to allow mpc/odm combine if its used later */
-				if (approx_det_segs_required_for_pstate < 2)
-					approx_det_segs_required_for_pstate = 2;
-				if (split_required)
-					approx_det_segs_required_for_pstate += approx_det_segs_required_for_pstate % 2;
-				pipes[pipe_cnt].pipe.src.det_size_override = approx_det_segs_required_for_pstate;
-				remaining_det_segs -= approx_det_segs_required_for_pstate;
-			} else
-				remaining_det_segs = -1;
-			crb_pipes++;
-		}
-		DC_FP_END();
-
-		if (pipes[pipe_cnt].dout.dsc_enable) {
-			switch (timing->display_color_depth) {
-			case COLOR_DEPTH_888:
-				pipes[pipe_cnt].dout.dsc_input_bpc = 8;
-				break;
-			case COLOR_DEPTH_101010:
-				pipes[pipe_cnt].dout.dsc_input_bpc = 10;
-				break;
-			case COLOR_DEPTH_121212:
-				pipes[pipe_cnt].dout.dsc_input_bpc = 12;
-				break;
-			default:
-				ASSERT(0);
-				break;
-			}
-		}
-		pipe_cnt++;
-	}
-
-	/* Spread remaining unreserved crb evenly among all pipes*/
-	if (pixel_rate_crb) {
-		for (i = 0, pipe_cnt = 0, crb_idx = 0; i < dc->res_pool->pipe_count; i++) {
-			pipe = &res_ctx->pipe_ctx[i];
-			if (!pipe->stream)
-				continue;
-
-			/* Do not use asymetric crb if not enough for pstate support */
-			if (remaining_det_segs < 0) {
-				pipes[pipe_cnt].pipe.src.det_size_override = 0;
-				pipe_cnt++;
-				continue;
-			}
-
-			if (!pipe->top_pipe && !pipe->prev_odm_pipe) {
-				bool split_required = pipe->stream->timing.pix_clk_100hz >= dcn_get_max_non_odm_pix_rate_100hz(&dc->dml.soc)
-						|| (pipe->plane_state && pipe->plane_state->src_rect.width > 5120);
-
-				if (remaining_det_segs > MIN_RESERVED_DET_SEGS)
-					pipes[pipe_cnt].pipe.src.det_size_override += (remaining_det_segs - MIN_RESERVED_DET_SEGS) / crb_pipes +
-							(crb_idx < (remaining_det_segs - MIN_RESERVED_DET_SEGS) % crb_pipes ? 1 : 0);
-				if (pipes[pipe_cnt].pipe.src.det_size_override > 2 * DCN3_15_MAX_DET_SEGS) {
-					/* Clamp to 2 pipe split max det segments */
-					remaining_det_segs += pipes[pipe_cnt].pipe.src.det_size_override - 2 * (DCN3_15_MAX_DET_SEGS);
-					pipes[pipe_cnt].pipe.src.det_size_override = 2 * DCN3_15_MAX_DET_SEGS;
-				}
-				if (pipes[pipe_cnt].pipe.src.det_size_override > DCN3_15_MAX_DET_SEGS || split_required) {
-					/* If we are splitting we must have an even number of segments */
-					remaining_det_segs += pipes[pipe_cnt].pipe.src.det_size_override % 2;
-					pipes[pipe_cnt].pipe.src.det_size_override -= pipes[pipe_cnt].pipe.src.det_size_override % 2;
-				}
-				/* Convert segments into size for DML use */
-				pipes[pipe_cnt].pipe.src.det_size_override *= DCN3_15_CRB_SEGMENT_SIZE_KB;
-
-				crb_idx++;
-			}
-			pipe_cnt++;
-		}
-	}
-
-	if (pipe_cnt)
-		context->bw_ctx.dml.ip.det_buffer_size_kbytes =
-				(max_usable_det / DCN3_15_CRB_SEGMENT_SIZE_KB / pipe_cnt) * DCN3_15_CRB_SEGMENT_SIZE_KB;
-	if (context->bw_ctx.dml.ip.det_buffer_size_kbytes > DCN3_15_MAX_DET_SIZE)
-		context->bw_ctx.dml.ip.det_buffer_size_kbytes = DCN3_15_MAX_DET_SIZE;
-
-	dc->config.enable_4to1MPC = false;
-	if (pipe_cnt == 1 && pipe->plane_state && !dc->debug.disable_z9_mpc) {
-		if (is_dual_plane(pipe->plane_state->format)
-				&& pipe->plane_state->src_rect.width <= 1920 && pipe->plane_state->src_rect.height <= 1080) {
-			dc->config.enable_4to1MPC = true;
-			context->bw_ctx.dml.ip.det_buffer_size_kbytes =
-					(max_usable_det / DCN3_15_CRB_SEGMENT_SIZE_KB / 4) * DCN3_15_CRB_SEGMENT_SIZE_KB;
-		} else if (!is_dual_plane(pipe->plane_state->format)
-				&& pipe->plane_state->src_rect.width <= 5120
-				&& pipe->stream->timing.pix_clk_100hz < dcn_get_max_non_odm_pix_rate_100hz(&dc->dml.soc)) {
-			/* Limit to 5k max to avoid forced pipe split when there is not enough detile for swath */
-			context->bw_ctx.dml.ip.det_buffer_size_kbytes = 192;
-			pipes[0].pipe.src.unbounded_req_mode = true;
-		}
-	}
-
-	return pipe_cnt;
-}
-
-static void dcn315_get_panel_config_defaults(struct dc_panel_config *panel_config)
-{
-	*panel_config = panel_config_defaults;
-}
-
-static struct dc_cap_funcs cap_funcs = {
-	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
-};
-
-static struct resource_funcs dcn315_res_pool_funcs = {
-	.destroy = dcn315_destroy_resource_pool,
-	.link_enc_create = dcn31_link_encoder_create,
-	.link_enc_create_minimal = dcn31_link_enc_create_minimal,
-	.link_encs_assign = link_enc_cfg_link_encs_assign,
-	.link_enc_unassign = link_enc_cfg_link_enc_unassign,
-	.panel_cntl_create = dcn31_panel_cntl_create,
-	.validate_bandwidth = dcn31_validate_bandwidth,
-	.calculate_wm_and_dlg = dcn31_calculate_wm_and_dlg,
-	.update_soc_for_wm_a = dcn315_update_soc_for_wm_a,
-	.populate_dml_pipes = dcn315_populate_dml_pipes_from_context,
-	.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
-	.release_pipe = dcn20_release_pipe,
-	.add_stream_to_ctx = dcn30_add_stream_to_ctx,
-	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
-	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
-	.populate_dml_writeback_from_context = dcn31_populate_dml_writeback_from_context,
-	.set_mcif_arb_params = dcn31_set_mcif_arb_params,
-	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
-	.acquire_post_bldn_3dlut = dcn30_acquire_post_bldn_3dlut,
-	.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
-	.update_bw_bounding_box = dcn315_update_bw_bounding_box,
-	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
-	.get_panel_config_defaults = dcn315_get_panel_config_defaults,
-};
-
-static bool dcn315_resource_construct(
-	uint8_t num_virtual_links,
-	struct dc *dc,
-	struct dcn315_resource_pool *pool)
-{
-	int i;
-	struct dc_context *ctx = dc->ctx;
-	struct irq_service_init_data init_data;
-
-	ctx->dc_bios->regs = &bios_regs;
-
-	pool->base.res_cap = &res_cap_dcn31;
-
-	pool->base.funcs = &dcn315_res_pool_funcs;
-
-	/*************************************************
-	 *  Resource + asic cap harcoding                *
-	 *************************************************/
-	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
-	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
-	pool->base.mpcc_count = pool->base.res_cap->num_timing_generator;
-	dc->caps.max_downscale_ratio = 600;
-	dc->caps.i2c_speed_in_khz = 100;
-	dc->caps.i2c_speed_in_khz_hdcp = 100;
-	dc->caps.max_cursor_size = 256;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.dmdata_alloc_size = 2048;
-	dc->caps.max_slave_planes = 2;
-	dc->caps.max_slave_yuv_planes = 2;
-	dc->caps.max_slave_rgb_planes = 2;
-	dc->caps.post_blend_color_processing = true;
-	dc->caps.force_dp_tps4_for_cp2520 = true;
-	if (dc->config.forceHBR2CP2520)
-		dc->caps.force_dp_tps4_for_cp2520 = false;
-	dc->caps.dp_hpo = true;
-	dc->caps.dp_hdmi21_pcon_support = true;
-	dc->caps.edp_dsc_support = true;
-	dc->caps.extended_aux_timeout_support = true;
-	dc->caps.dmcub_support = true;
-	dc->caps.is_apu = true;
-
-	/* Color pipeline capabilities */
-	dc->caps.color.dpp.dcn_arch = 1;
-	dc->caps.color.dpp.input_lut_shared = 0;
-	dc->caps.color.dpp.icsc = 1;
-	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
-	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
-	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
-	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
-	dc->caps.color.dpp.post_csc = 1;
-	dc->caps.color.dpp.gamma_corr = 1;
-	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
-
-	dc->caps.color.dpp.hw_3d_lut = 1;
-	dc->caps.color.dpp.ogam_ram = 1;
-	// no OGAM ROM on DCN301
-	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
-	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
-	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
-	dc->caps.color.dpp.ocsc = 0;
-
-	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //2
-	dc->caps.color.mpc.ogam_ram = 1;
-	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
-	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
-	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
-	dc->caps.color.mpc.ocsc = 1;
-
-	/* read VBIOS LTTPR caps */
-	{
-		if (ctx->dc_bios->funcs->get_lttpr_caps) {
-			enum bp_result bp_query_result;
-			uint8_t is_vbios_lttpr_enable = 0;
-
-			bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
-			dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
-		}
-
-		/* interop bit is implicit */
-		{
-			dc->caps.vbios_lttpr_aware = true;
-		}
-	}
-
-	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
-		dc->debug = debug_defaults_drv;
-
-	// Init the vm_helper
-	if (dc->vm_helper)
-		vm_helper_init(dc->vm_helper, 16);
-
-	/*************************************************
-	 *  Create resources                             *
-	 *************************************************/
-
-	/* Clock Sources for Pixel Clock*/
-	pool->base.clock_sources[DCN31_CLK_SRC_PLL0] =
-			dcn31_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL0,
-				&clk_src_regs[0], false);
-	pool->base.clock_sources[DCN31_CLK_SRC_PLL1] =
-			dcn31_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL1,
-				&clk_src_regs[1], false);
-	pool->base.clock_sources[DCN31_CLK_SRC_PLL2] =
-			dcn31_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL2,
-				&clk_src_regs[2], false);
-	pool->base.clock_sources[DCN31_CLK_SRC_PLL3] =
-			dcn31_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL3,
-				&clk_src_regs[3], false);
-	pool->base.clock_sources[DCN31_CLK_SRC_PLL4] =
-			dcn31_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL4,
-				&clk_src_regs[4], false);
-
-	pool->base.clk_src_count = DCN30_CLK_SRC_TOTAL;
-
-	/* todo: not reuse phy_pll registers */
-	pool->base.dp_clock_source =
-			dcn31_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_ID_DP_DTO,
-				&clk_src_regs[0], true);
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-
-	/* TODO: DCCG */
-	pool->base.dccg = dccg31_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
-	if (pool->base.dccg == NULL) {
-		dm_error("DC: failed to create dccg!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	/* TODO: IRQ */
-	init_data.ctx = dc->ctx;
-	pool->base.irqs = dal_irq_service_dcn315_create(&init_data);
-	if (!pool->base.irqs)
-		goto create_fail;
-
-	/* HUBBUB */
-	pool->base.hubbub = dcn31_hubbub_create(ctx);
-	if (pool->base.hubbub == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create hubbub!\n");
-		goto create_fail;
-	}
-
-	/* HUBPs, DPPs, OPPs and TGs */
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		pool->base.hubps[i] = dcn31_hubp_create(ctx, i);
-		if (pool->base.hubps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create hubps!\n");
-			goto create_fail;
-		}
-
-		pool->base.dpps[i] = dcn31_dpp_create(ctx, i);
-		if (pool->base.dpps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create dpps!\n");
-			goto create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
-		pool->base.opps[i] = dcn31_opp_create(ctx, i);
-		if (pool->base.opps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create output pixel processor!\n");
-			goto create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		pool->base.timing_generators[i] = dcn31_timing_generator_create(
-				ctx, i);
-		if (pool->base.timing_generators[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create tg!\n");
-			goto create_fail;
-		}
-	}
-	pool->base.timing_generator_count = i;
-
-	/* PSR */
-	pool->base.psr = dmub_psr_create(ctx);
-	if (pool->base.psr == NULL) {
-		dm_error("DC: failed to create psr obj!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	/* ABM */
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		pool->base.multiple_abms[i] = dmub_abm_create(ctx,
-				&abm_regs[i],
-				&abm_shift,
-				&abm_mask);
-		if (pool->base.multiple_abms[i] == NULL) {
-			dm_error("DC: failed to create abm for pipe %d!\n", i);
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-
-	/* MPC and DSC */
-	pool->base.mpc = dcn31_mpc_create(ctx, pool->base.mpcc_count, pool->base.res_cap->num_mpc_3dlut);
-	if (pool->base.mpc == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mpc!\n");
-		goto create_fail;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		pool->base.dscs[i] = dcn31_dsc_create(ctx, i);
-		if (pool->base.dscs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create display stream compressor %d!\n", i);
-			goto create_fail;
-		}
-	}
-
-	/* DWB and MMHUBBUB */
-	if (!dcn31_dwbc_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create dwbc!\n");
-		goto create_fail;
-	}
-
-	if (!dcn31_mmhubbub_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mcif_wb!\n");
-		goto create_fail;
-	}
-
-	/* AUX and I2C */
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		pool->base.engines[i] = dcn31_aux_engine_create(ctx, i);
-		if (pool->base.engines[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create aux engine!!\n");
-			goto create_fail;
-		}
-		pool->base.hw_i2cs[i] = dcn31_i2c_hw_create(ctx, i);
-		if (pool->base.hw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create hw i2c!!\n");
-			goto create_fail;
-		}
-		pool->base.sw_i2cs[i] = NULL;
-	}
-
-	/* Audio, Stream Encoders including HPO and virtual, MPC 3D LUTs */
-	if (!resource_construct(num_virtual_links, dc, &pool->base,
-			&res_create_funcs))
-		goto create_fail;
-
-	/* HW Sequencer and Plane caps */
-	dcn31_hw_sequencer_construct(dc);
-
-	dc->caps.max_planes =  pool->base.pipe_count;
-
-	for (i = 0; i < dc->caps.max_planes; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	dc->cap_funcs = cap_funcs;
-
-	dc->dcn_ip->max_num_dpp = dcn3_15_ip.max_num_dpp;
-
-	return true;
-
-create_fail:
-
-	dcn315_resource_destruct(pool);
-
-	return false;
-}
-
-struct resource_pool *dcn315_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc)
-{
-	struct dcn315_resource_pool *pool =
-		kzalloc(sizeof(struct dcn315_resource_pool), GFP_KERNEL);
-
-	if (!pool)
-		return NULL;
-
-	if (dcn315_resource_construct(init_data->num_virtual_links, dc, pool))
-		return &pool->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(pool);
-	return NULL;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.h b/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.h
deleted file mode 100644
index 22849eaa6f24..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright 2021 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#ifndef _DCN315_RESOURCE_H_
-#define _DCN315_RESOURCE_H_
-
-#include "core_types.h"
-
-#define TO_DCN315_RES_POOL(pool)\
-	container_of(pool, struct dcn315_resource_pool, base)
-
-extern struct _vcs_dpi_ip_params_st dcn3_15_ip;
-
-struct dcn315_resource_pool {
-	struct resource_pool base;
-};
-
-struct resource_pool *dcn315_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc);
-
-#endif /* _DCN315_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn316/Makefile b/drivers/gpu/drm/amd/display/dc/dcn316/Makefile
deleted file mode 100644
index 819d44a9439b..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn316/Makefile
+++ /dev/null
@@ -1,30 +0,0 @@
-#
-# Copyright 2021 Advanced Micro Devices, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the "Software"),
-# to deal in the Software without restriction, including without limitation
-# the rights to use, copy, modify, merge, publish, distribute, sublicense,
-# and/or sell copies of the Software, and to permit persons to whom the
-# Software is furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
-# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
-# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
-# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-# OTHER DEALINGS IN THE SOFTWARE.
-#
-# Authors: AMD
-#
-# Makefile for dcn316.
-
-DCN316 = dcn316_resource.o
-
-AMD_DAL_DCN316 = $(addprefix $(AMDDALPATH)/dc/dcn316/,$(DCN316))
-
-AMD_DISPLAY_FILES += $(AMD_DAL_DCN316)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c b/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c
deleted file mode 100644
index b9753d4606f8..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c
+++ /dev/null
@@ -1,2038 +0,0 @@
-/*
- * Copyright 2021 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-
-#include "dm_services.h"
-#include "dc.h"
-
-#include "dcn31/dcn31_init.h"
-
-#include "resource.h"
-#include "include/irq_service_interface.h"
-#include "dcn316_resource.h"
-
-#include "dcn20/dcn20_resource.h"
-#include "dcn30/dcn30_resource.h"
-#include "dcn31/dcn31_resource.h"
-
-#include "dcn10/dcn10_ipp.h"
-#include "dcn30/dcn30_hubbub.h"
-#include "dcn31/dcn31_hubbub.h"
-#include "dcn30/dcn30_mpc.h"
-#include "dcn31/dcn31_hubp.h"
-#include "irq/dcn31/irq_service_dcn31.h"
-#include "dcn30/dcn30_dpp.h"
-#include "dcn31/dcn31_optc.h"
-#include "dcn20/dcn20_hwseq.h"
-#include "dcn30/dcn30_hwseq.h"
-#include "dce110/dce110_hwseq.h"
-#include "dcn30/dcn30_opp.h"
-#include "dcn20/dcn20_dsc.h"
-#include "dcn30/dcn30_vpg.h"
-#include "dcn30/dcn30_afmt.h"
-#include "dcn30/dcn30_dio_stream_encoder.h"
-#include "dcn31/dcn31_hpo_dp_stream_encoder.h"
-#include "dcn31/dcn31_hpo_dp_link_encoder.h"
-#include "dcn31/dcn31_apg.h"
-#include "dcn31/dcn31_dio_link_encoder.h"
-#include "dcn31/dcn31_vpg.h"
-#include "dcn31/dcn31_afmt.h"
-#include "dce/dce_clock_source.h"
-#include "dce/dce_audio.h"
-#include "dce/dce_hwseq.h"
-#include "clk_mgr.h"
-#include "virtual/virtual_stream_encoder.h"
-#include "dce110/dce110_resource.h"
-#include "dml/display_mode_vba.h"
-#include "dml/dcn31/dcn31_fpu.h"
-#include "dcn31/dcn31_dccg.h"
-#include "dcn10/dcn10_resource.h"
-#include "dcn31/dcn31_panel_cntl.h"
-
-#include "dcn30/dcn30_dwb.h"
-#include "dcn30/dcn30_mmhubbub.h"
-
-#include "dcn/dcn_3_1_6_offset.h"
-#include "dcn/dcn_3_1_6_sh_mask.h"
-#include "dpcs/dpcs_4_2_3_offset.h"
-#include "dpcs/dpcs_4_2_3_sh_mask.h"
-
-#define regBIF_BX1_BIOS_SCRATCH_2                                                                       0x003a
-#define regBIF_BX1_BIOS_SCRATCH_2_BASE_IDX                                                              1
-#define regBIF_BX1_BIOS_SCRATCH_3                                                                       0x003b
-#define regBIF_BX1_BIOS_SCRATCH_3_BASE_IDX                                                              1
-#define regBIF_BX1_BIOS_SCRATCH_6                                                                       0x003e
-#define regBIF_BX1_BIOS_SCRATCH_6_BASE_IDX                                                              1
-
-#define regDCHUBBUB_DEBUG_CTRL_0                                              0x04d6
-#define regDCHUBBUB_DEBUG_CTRL_0_BASE_IDX                                     2
-#define DCHUBBUB_DEBUG_CTRL_0__DET_DEPTH__SHIFT                               0x10
-#define DCHUBBUB_DEBUG_CTRL_0__DET_DEPTH_MASK                                 0x01FF0000L
-
-#define DCN_BASE__INST0_SEG0                       0x00000012
-#define DCN_BASE__INST0_SEG1                       0x000000C0
-#define DCN_BASE__INST0_SEG2                       0x000034C0
-#define DCN_BASE__INST0_SEG3                       0x00009000
-#define DCN_BASE__INST0_SEG4                       0x02403C00
-#define DCN_BASE__INST0_SEG5                       0
-
-#define DPCS_BASE__INST0_SEG0                      0x00000012
-#define DPCS_BASE__INST0_SEG1                      0x000000C0
-#define DPCS_BASE__INST0_SEG2                      0x000034C0
-#define DPCS_BASE__INST0_SEG3                      0x00009000
-#define DPCS_BASE__INST0_SEG4                      0x02403C00
-#define DPCS_BASE__INST0_SEG5                      0
-
-#define NBIO_BASE__INST0_SEG0                      0x00000000
-#define NBIO_BASE__INST0_SEG1                      0x00000014
-#define NBIO_BASE__INST0_SEG2                      0x00000D20
-#define NBIO_BASE__INST0_SEG3                      0x00010400
-#define NBIO_BASE__INST0_SEG4                      0x0241B000
-#define NBIO_BASE__INST0_SEG5                      0x04040000
-
-#include "reg_helper.h"
-#include "dce/dmub_abm.h"
-#include "dce/dmub_psr.h"
-#include "dce/dce_aux.h"
-#include "dce/dce_i2c.h"
-
-#include "dml/dcn30/display_mode_vba_30.h"
-#include "vm_helper.h"
-#include "dcn20/dcn20_vmid.h"
-
-#include "link_enc_cfg.h"
-
-#define DCN3_16_MAX_DET_SIZE 384
-#define DCN3_16_MIN_COMPBUF_SIZE_KB 128
-#define DCN3_16_CRB_SEGMENT_SIZE_KB 64
-
-enum dcn31_clk_src_array_id {
-	DCN31_CLK_SRC_PLL0,
-	DCN31_CLK_SRC_PLL1,
-	DCN31_CLK_SRC_PLL2,
-	DCN31_CLK_SRC_PLL3,
-	DCN31_CLK_SRC_PLL4,
-	DCN30_CLK_SRC_TOTAL
-};
-
-/* begin *********************
- * macros to expend register list macro defined in HW object header file
- */
-
-/* DCN */
-#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
-
-#define BASE(seg) BASE_INNER(seg)
-
-#define SR(reg_name)\
-		.reg_name = BASE(reg ## reg_name ## _BASE_IDX) +  \
-					reg ## reg_name
-
-#define SRI(reg_name, block, id)\
-	.reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define SRI2(reg_name, block, id)\
-	.reg_name = BASE(reg ## reg_name ## _BASE_IDX) + \
-					reg ## reg_name
-
-#define SRIR(var_name, reg_name, block, id)\
-	.var_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define SRII_MPC_RMU(reg_name, block, id)\
-	.RMU##_##reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define SRII_DWB(reg_name, temp_name, block, id)\
-	.reg_name[id] = BASE(reg ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## temp_name
-
-#define SF_DWB2(reg_name, block, id, field_name, post_fix)	\
-	.field_name = reg_name ## __ ## field_name ## post_fix
-
-#define DCCG_SRII(reg_name, block, id)\
-	.block ## _ ## reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define VUPDATE_SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(reg ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
-					reg ## reg_name ## _ ## block ## id
-
-/* NBIO */
-#define NBIO_BASE_INNER(seg) \
-	NBIO_BASE__INST0_SEG ## seg
-
-#define NBIO_BASE(seg) \
-	NBIO_BASE_INNER(seg)
-
-#define NBIO_SR(reg_name)\
-		.reg_name = NBIO_BASE(regBIF_BX1_ ## reg_name ## _BASE_IDX) + \
-					regBIF_BX1_ ## reg_name
-
-static const struct bios_registers bios_regs = {
-		NBIO_SR(BIOS_SCRATCH_3),
-		NBIO_SR(BIOS_SCRATCH_6)
-};
-
-#define clk_src_regs(index, pllid)\
-[index] = {\
-	CS_COMMON_REG_LIST_DCN3_0(index, pllid),\
-}
-
-static const struct dce110_clk_src_regs clk_src_regs[] = {
-	clk_src_regs(0, A),
-	clk_src_regs(1, B),
-	clk_src_regs(2, C),
-	clk_src_regs(3, D),
-	clk_src_regs(4, E)
-};
-
-static const struct dce110_clk_src_shift cs_shift = {
-		CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
-};
-
-static const struct dce110_clk_src_mask cs_mask = {
-		CS_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
-};
-
-#define abm_regs(id)\
-[id] = {\
-		ABM_DCN302_REG_LIST(id)\
-}
-
-static const struct dce_abm_registers abm_regs[] = {
-		abm_regs(0),
-		abm_regs(1),
-		abm_regs(2),
-		abm_regs(3),
-};
-
-static const struct dce_abm_shift abm_shift = {
-		ABM_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dce_abm_mask abm_mask = {
-		ABM_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define audio_regs(id)\
-[id] = {\
-		AUD_COMMON_REG_LIST(id)\
-}
-
-static const struct dce_audio_registers audio_regs[] = {
-	audio_regs(0),
-	audio_regs(1),
-	audio_regs(2),
-	audio_regs(3),
-	audio_regs(4),
-	audio_regs(5),
-	audio_regs(6)
-};
-
-#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
-		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
-
-static const struct dce_audio_shift audio_shift = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_audio_mask audio_mask = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
-};
-
-#define vpg_regs(id)\
-[id] = {\
-	VPG_DCN31_REG_LIST(id)\
-}
-
-static const struct dcn31_vpg_registers vpg_regs[] = {
-	vpg_regs(0),
-	vpg_regs(1),
-	vpg_regs(2),
-	vpg_regs(3),
-	vpg_regs(4),
-	vpg_regs(5),
-	vpg_regs(6),
-	vpg_regs(7),
-	vpg_regs(8),
-	vpg_regs(9),
-};
-
-static const struct dcn31_vpg_shift vpg_shift = {
-	DCN31_VPG_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_vpg_mask vpg_mask = {
-	DCN31_VPG_MASK_SH_LIST(_MASK)
-};
-
-#define afmt_regs(id)\
-[id] = {\
-	AFMT_DCN31_REG_LIST(id)\
-}
-
-static const struct dcn31_afmt_registers afmt_regs[] = {
-	afmt_regs(0),
-	afmt_regs(1),
-	afmt_regs(2),
-	afmt_regs(3),
-	afmt_regs(4),
-	afmt_regs(5)
-};
-
-static const struct dcn31_afmt_shift afmt_shift = {
-	DCN31_AFMT_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_afmt_mask afmt_mask = {
-	DCN31_AFMT_MASK_SH_LIST(_MASK)
-};
-
-
-#define apg_regs(id)\
-[id] = {\
-	APG_DCN31_REG_LIST(id)\
-}
-
-static const struct dcn31_apg_registers apg_regs[] = {
-	apg_regs(0),
-	apg_regs(1),
-	apg_regs(2),
-	apg_regs(3)
-};
-
-static const struct dcn31_apg_shift apg_shift = {
-	DCN31_APG_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_apg_mask apg_mask = {
-		DCN31_APG_MASK_SH_LIST(_MASK)
-};
-
-
-#define stream_enc_regs(id)\
-[id] = {\
-	SE_DCN3_REG_LIST(id)\
-}
-
-static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
-	stream_enc_regs(0),
-	stream_enc_regs(1),
-	stream_enc_regs(2),
-	stream_enc_regs(3),
-	stream_enc_regs(4)
-};
-
-static const struct dcn10_stream_encoder_shift se_shift = {
-		SE_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn10_stream_encoder_mask se_mask = {
-		SE_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-
-#define aux_regs(id)\
-[id] = {\
-	DCN2_AUX_REG_LIST(id)\
-}
-
-static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
-		aux_regs(0),
-		aux_regs(1),
-		aux_regs(2),
-		aux_regs(3),
-		aux_regs(4)
-};
-
-#define hpd_regs(id)\
-[id] = {\
-	HPD_REG_LIST(id)\
-}
-
-static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
-		hpd_regs(0),
-		hpd_regs(1),
-		hpd_regs(2),
-		hpd_regs(3),
-		hpd_regs(4)
-};
-
-#define link_regs(id, phyid)\
-[id] = {\
-	LE_DCN31_REG_LIST(id), \
-	UNIPHY_DCN2_REG_LIST(phyid), \
-	DPCS_DCN31_REG_LIST(id), \
-}
-
-static const struct dce110_aux_registers_shift aux_shift = {
-	DCN_AUX_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce110_aux_registers_mask aux_mask = {
-	DCN_AUX_MASK_SH_LIST(_MASK)
-};
-
-static const struct dcn10_link_enc_registers link_enc_regs[] = {
-	link_regs(0, A),
-	link_regs(1, B),
-	link_regs(2, C),
-	link_regs(3, D),
-	link_regs(4, E)
-};
-
-static const struct dcn10_link_enc_shift le_shift = {
-	LINK_ENCODER_MASK_SH_LIST_DCN31(__SHIFT), \
-	DPCS_DCN31_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn10_link_enc_mask le_mask = {
-	LINK_ENCODER_MASK_SH_LIST_DCN31(_MASK), \
-	DPCS_DCN31_MASK_SH_LIST(_MASK)
-};
-
-
-
-#define hpo_dp_stream_encoder_reg_list(id)\
-[id] = {\
-	DCN3_1_HPO_DP_STREAM_ENC_REG_LIST(id)\
-}
-
-static const struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[] = {
-	hpo_dp_stream_encoder_reg_list(0),
-	hpo_dp_stream_encoder_reg_list(1),
-	hpo_dp_stream_encoder_reg_list(2),
-	hpo_dp_stream_encoder_reg_list(3),
-};
-
-static const struct dcn31_hpo_dp_stream_encoder_shift hpo_dp_se_shift = {
-	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_hpo_dp_stream_encoder_mask hpo_dp_se_mask = {
-	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(_MASK)
-};
-
-
-#define hpo_dp_link_encoder_reg_list(id)\
-[id] = {\
-	DCN3_1_HPO_DP_LINK_ENC_REG_LIST(id),\
-	DCN3_1_RDPCSTX_REG_LIST(0),\
-	DCN3_1_RDPCSTX_REG_LIST(1),\
-	DCN3_1_RDPCSTX_REG_LIST(2),\
-	DCN3_1_RDPCSTX_REG_LIST(3),\
-	DCN3_1_RDPCSTX_REG_LIST(4)\
-}
-
-static const struct dcn31_hpo_dp_link_encoder_registers hpo_dp_link_enc_regs[] = {
-	hpo_dp_link_encoder_reg_list(0),
-	hpo_dp_link_encoder_reg_list(1),
-};
-
-static const struct dcn31_hpo_dp_link_encoder_shift hpo_dp_le_shift = {
-	DCN3_1_HPO_DP_LINK_ENC_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_hpo_dp_link_encoder_mask hpo_dp_le_mask = {
-	DCN3_1_HPO_DP_LINK_ENC_MASK_SH_LIST(_MASK)
-};
-
-
-#define dpp_regs(id)\
-[id] = {\
-	DPP_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn3_dpp_registers dpp_regs[] = {
-	dpp_regs(0),
-	dpp_regs(1),
-	dpp_regs(2),
-	dpp_regs(3)
-};
-
-static const struct dcn3_dpp_shift tf_shift = {
-		DPP_REG_LIST_SH_MASK_DCN30(__SHIFT)
-};
-
-static const struct dcn3_dpp_mask tf_mask = {
-		DPP_REG_LIST_SH_MASK_DCN30(_MASK)
-};
-
-#define opp_regs(id)\
-[id] = {\
-	OPP_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn20_opp_registers opp_regs[] = {
-	opp_regs(0),
-	opp_regs(1),
-	opp_regs(2),
-	opp_regs(3)
-};
-
-static const struct dcn20_opp_shift opp_shift = {
-	OPP_MASK_SH_LIST_DCN20(__SHIFT)
-};
-
-static const struct dcn20_opp_mask opp_mask = {
-	OPP_MASK_SH_LIST_DCN20(_MASK)
-};
-
-#define aux_engine_regs(id)\
-[id] = {\
-	AUX_COMMON_REG_LIST0(id), \
-	.AUXN_IMPCAL = 0, \
-	.AUXP_IMPCAL = 0, \
-	.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
-}
-
-static const struct dce110_aux_registers aux_engine_regs[] = {
-		aux_engine_regs(0),
-		aux_engine_regs(1),
-		aux_engine_regs(2),
-		aux_engine_regs(3),
-		aux_engine_regs(4)
-};
-
-#define dwbc_regs_dcn3(id)\
-[id] = {\
-	DWBC_COMMON_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn30_dwbc_registers dwbc30_regs[] = {
-	dwbc_regs_dcn3(0),
-};
-
-static const struct dcn30_dwbc_shift dwbc30_shift = {
-	DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_dwbc_mask dwbc30_mask = {
-	DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define mcif_wb_regs_dcn3(id)\
-[id] = {\
-	MCIF_WB_COMMON_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn30_mmhubbub_registers mcif_wb30_regs[] = {
-	mcif_wb_regs_dcn3(0)
-};
-
-static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
-	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
-	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define dsc_regsDCN20(id)\
-[id] = {\
-	DSC_REG_LIST_DCN20(id)\
-}
-
-static const struct dcn20_dsc_registers dsc_regs[] = {
-	dsc_regsDCN20(0),
-	dsc_regsDCN20(1),
-	dsc_regsDCN20(2)
-};
-
-static const struct dcn20_dsc_shift dsc_shift = {
-	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
-};
-
-static const struct dcn20_dsc_mask dsc_mask = {
-	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
-};
-
-static const struct dcn30_mpc_registers mpc_regs = {
-		MPC_REG_LIST_DCN3_0(0),
-		MPC_REG_LIST_DCN3_0(1),
-		MPC_REG_LIST_DCN3_0(2),
-		MPC_REG_LIST_DCN3_0(3),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(0),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(1),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(2),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(3),
-		MPC_RMU_GLOBAL_REG_LIST_DCN3AG,
-		MPC_RMU_REG_LIST_DCN3AG(0),
-		MPC_RMU_REG_LIST_DCN3AG(1),
-		//MPC_RMU_REG_LIST_DCN3AG(2),
-		MPC_DWB_MUX_REG_LIST_DCN3_0(0),
-};
-
-static const struct dcn30_mpc_shift mpc_shift = {
-	MPC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_mpc_mask mpc_mask = {
-	MPC_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define optc_regs(id)\
-[id] = {OPTC_COMMON_REG_LIST_DCN3_1(id)}
-
-static const struct dcn_optc_registers optc_regs[] = {
-	optc_regs(0),
-	optc_regs(1),
-	optc_regs(2),
-	optc_regs(3)
-};
-
-static const struct dcn_optc_shift optc_shift = {
-	OPTC_COMMON_MASK_SH_LIST_DCN3_1(__SHIFT)
-};
-
-static const struct dcn_optc_mask optc_mask = {
-	OPTC_COMMON_MASK_SH_LIST_DCN3_1(_MASK)
-};
-
-#define hubp_regs(id)\
-[id] = {\
-	HUBP_REG_LIST_DCN30(id)\
-}
-
-static const struct dcn_hubp2_registers hubp_regs[] = {
-		hubp_regs(0),
-		hubp_regs(1),
-		hubp_regs(2),
-		hubp_regs(3)
-};
-
-
-static const struct dcn_hubp2_shift hubp_shift = {
-		HUBP_MASK_SH_LIST_DCN31(__SHIFT)
-};
-
-static const struct dcn_hubp2_mask hubp_mask = {
-		HUBP_MASK_SH_LIST_DCN31(_MASK)
-};
-static const struct dcn_hubbub_registers hubbub_reg = {
-		HUBBUB_REG_LIST_DCN31(0)
-};
-
-static const struct dcn_hubbub_shift hubbub_shift = {
-		HUBBUB_MASK_SH_LIST_DCN31(__SHIFT)
-};
-
-static const struct dcn_hubbub_mask hubbub_mask = {
-		HUBBUB_MASK_SH_LIST_DCN31(_MASK)
-};
-
-static const struct dccg_registers dccg_regs = {
-		DCCG_REG_LIST_DCN31()
-};
-
-static const struct dccg_shift dccg_shift = {
-		DCCG_MASK_SH_LIST_DCN31(__SHIFT)
-};
-
-static const struct dccg_mask dccg_mask = {
-		DCCG_MASK_SH_LIST_DCN31(_MASK)
-};
-
-
-#define SRII2(reg_name_pre, reg_name_post, id)\
-	.reg_name_pre ## _ ##  reg_name_post[id] = BASE(reg ## reg_name_pre \
-			## id ## _ ## reg_name_post ## _BASE_IDX) + \
-			reg ## reg_name_pre ## id ## _ ## reg_name_post
-
-
-#define HWSEQ_DCN31_REG_LIST()\
-	SR(DCHUBBUB_GLOBAL_TIMER_CNTL), \
-	SR(DCHUBBUB_ARB_HOSTVM_CNTL), \
-	SR(DIO_MEM_PWR_CTRL), \
-	SR(ODM_MEM_PWR_CTRL3), \
-	SR(DMU_MEM_PWR_CNTL), \
-	SR(MMHUBBUB_MEM_PWR_CNTL), \
-	SR(DCCG_GATE_DISABLE_CNTL), \
-	SR(DCCG_GATE_DISABLE_CNTL2), \
-	SR(DCFCLK_CNTL),\
-	SR(DC_MEM_GLOBAL_PWR_REQ_CNTL), \
-	SRII(PIXEL_RATE_CNTL, OTG, 0), \
-	SRII(PIXEL_RATE_CNTL, OTG, 1),\
-	SRII(PIXEL_RATE_CNTL, OTG, 2),\
-	SRII(PIXEL_RATE_CNTL, OTG, 3),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 0),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 1),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 2),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 3),\
-	SR(MICROSECOND_TIME_BASE_DIV), \
-	SR(MILLISECOND_TIME_BASE_DIV), \
-	SR(DISPCLK_FREQ_CHANGE_CNTL), \
-	SR(RBBMIF_TIMEOUT_DIS), \
-	SR(RBBMIF_TIMEOUT_DIS_2), \
-	SR(DCHUBBUB_CRC_CTRL), \
-	SR(DPP_TOP0_DPP_CRC_CTRL), \
-	SR(DPP_TOP0_DPP_CRC_VAL_B_A), \
-	SR(DPP_TOP0_DPP_CRC_VAL_R_G), \
-	SR(MPC_CRC_CTRL), \
-	SR(MPC_CRC_RESULT_GB), \
-	SR(MPC_CRC_RESULT_C), \
-	SR(MPC_CRC_RESULT_AR), \
-	SR(DOMAIN0_PG_CONFIG), \
-	SR(DOMAIN1_PG_CONFIG), \
-	SR(DOMAIN2_PG_CONFIG), \
-	SR(DOMAIN3_PG_CONFIG), \
-	SR(DOMAIN16_PG_CONFIG), \
-	SR(DOMAIN17_PG_CONFIG), \
-	SR(DOMAIN18_PG_CONFIG), \
-	SR(DOMAIN0_PG_STATUS), \
-	SR(DOMAIN1_PG_STATUS), \
-	SR(DOMAIN2_PG_STATUS), \
-	SR(DOMAIN3_PG_STATUS), \
-	SR(DOMAIN16_PG_STATUS), \
-	SR(DOMAIN17_PG_STATUS), \
-	SR(DOMAIN18_PG_STATUS), \
-	SR(D1VGA_CONTROL), \
-	SR(D2VGA_CONTROL), \
-	SR(D3VGA_CONTROL), \
-	SR(D4VGA_CONTROL), \
-	SR(D5VGA_CONTROL), \
-	SR(D6VGA_CONTROL), \
-	SR(DC_IP_REQUEST_CNTL), \
-	SR(AZALIA_AUDIO_DTO), \
-	SR(AZALIA_CONTROLLER_CLOCK_GATING), \
-	SR(HPO_TOP_HW_CONTROL)
-
-static const struct dce_hwseq_registers hwseq_reg = {
-		HWSEQ_DCN31_REG_LIST()
-};
-
-#define HWSEQ_DCN31_MASK_SH_LIST(mask_sh)\
-	HWSEQ_DCN_MASK_SH_LIST(mask_sh), \
-	HWS_SF(, DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, mask_sh), \
-	HWS_SF(, DCHUBBUB_ARB_HOSTVM_CNTL, DISABLE_HOSTVM_FORCE_ALLOW_PSTATE, mask_sh), \
-	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN0_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN1_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN2_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN3_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN16_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN17_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN18_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh), \
-	HWS_SF(, AZALIA_AUDIO_DTO, AZALIA_AUDIO_DTO_MODULE, mask_sh), \
-	HWS_SF(, HPO_TOP_CLOCK_CONTROL, HPO_HDMISTREAMCLK_G_GATE_DIS, mask_sh), \
-	HWS_SF(, DMU_MEM_PWR_CNTL, DMCU_ERAM_MEM_PWR_FORCE, mask_sh), \
-	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_UNASSIGNED_PWR_MODE, mask_sh), \
-	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_VBLANK_PWR_MODE, mask_sh), \
-	HWS_SF(, MMHUBBUB_MEM_PWR_CNTL, VGA_MEM_PWR_FORCE, mask_sh), \
-	HWS_SF(, DIO_MEM_PWR_CTRL, I2C_LIGHT_SLEEP_FORCE, mask_sh), \
-	HWS_SF(, HPO_TOP_HW_CONTROL, HPO_IO_EN, mask_sh)
-
-static const struct dce_hwseq_shift hwseq_shift = {
-		HWSEQ_DCN31_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_hwseq_mask hwseq_mask = {
-		HWSEQ_DCN31_MASK_SH_LIST(_MASK)
-};
-#define vmid_regs(id)\
-[id] = {\
-		DCN20_VMID_REG_LIST(id)\
-}
-
-static const struct dcn_vmid_registers vmid_regs[] = {
-	vmid_regs(0),
-	vmid_regs(1),
-	vmid_regs(2),
-	vmid_regs(3),
-	vmid_regs(4),
-	vmid_regs(5),
-	vmid_regs(6),
-	vmid_regs(7),
-	vmid_regs(8),
-	vmid_regs(9),
-	vmid_regs(10),
-	vmid_regs(11),
-	vmid_regs(12),
-	vmid_regs(13),
-	vmid_regs(14),
-	vmid_regs(15)
-};
-
-static const struct dcn20_vmid_shift vmid_shifts = {
-		DCN20_VMID_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn20_vmid_mask vmid_masks = {
-		DCN20_VMID_MASK_SH_LIST(_MASK)
-};
-
-static const struct resource_caps res_cap_dcn31 = {
-	.num_timing_generator = 4,
-	.num_opp = 4,
-	.num_video_plane = 4,
-	.num_audio = 5,
-	.num_stream_encoder = 5,
-	.num_dig_link_enc = 5,
-	.num_hpo_dp_stream_encoder = 4,
-	.num_hpo_dp_link_encoder = 2,
-	.num_pll = 5,
-	.num_dwb = 1,
-	.num_ddc = 5,
-	.num_vmid = 16,
-	.num_mpc_3dlut = 2,
-	.num_dsc = 3,
-};
-
-static const struct dc_plane_cap plane_cap = {
-	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.per_pixel_alpha = true,
-
-	.pixel_format_support = {
-			.argb8888 = true,
-			.nv12 = true,
-			.fp16 = true,
-			.p010 = true,
-			.ayuv = false,
-	},
-
-	.max_upscale_factor = {
-			.argb8888 = 16000,
-			.nv12 = 16000,
-			.fp16 = 16000
-	},
-
-	// 6:1 downscaling ratio: 1000/6 = 166.666
-	.max_downscale_factor = {
-			.argb8888 = 167,
-			.nv12 = 167,
-			.fp16 = 167
-	},
-	64,
-	64
-};
-
-static const struct dc_debug_options debug_defaults_drv = {
-	.disable_z10 = true, /*hw not support it*/
-	.disable_dmcu = true,
-	.force_abm_enable = false,
-	.timing_trace = false,
-	.clock_trace = true,
-	.disable_pplib_clock_request = false,
-	.pipe_split_policy = MPC_SPLIT_DYNAMIC,
-	.force_single_disp_pipe_split = false,
-	.disable_dcc = DCC_ENABLE,
-	.vsr_support = true,
-	.performance_trace = false,
-	.max_downscale_src_width = 4096,/*upto true 4k*/
-	.disable_pplib_wm_range = false,
-	.scl_reset_length10 = true,
-	.sanity_checks = false,
-	.underflow_assert_delay_us = 0xFFFFFFFF,
-	.dwb_fi_phase = -1, // -1 = disable,
-	.dmub_command_table = true,
-	.pstate_enabled = true,
-	.use_max_lb = true,
-	.enable_mem_low_power = {
-		.bits = {
-			.vga = true,
-			.i2c = true,
-			.dmcu = false, // This is previously known to cause hang on S3 cycles if enabled
-			.dscl = true,
-			.cm = true,
-			.mpc = true,
-			.optc = true,
-			.vpg = true,
-			.afmt = true,
-		}
-	},
-	.enable_legacy_fast_update = true,
-	.using_dml2 = false,
-};
-
-static const struct dc_panel_config panel_config_defaults = {
-	.psr = {
-		.disable_psr = false,
-		.disallow_psrsu = false,
-		.disallow_replay = false,
-	},
-	.ilr = {
-		.optimize_edp_link_rate = true,
-	},
-};
-
-static void dcn31_dpp_destroy(struct dpp **dpp)
-{
-	kfree(TO_DCN20_DPP(*dpp));
-	*dpp = NULL;
-}
-
-static struct dpp *dcn31_dpp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn3_dpp *dpp =
-		kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
-
-	if (!dpp)
-		return NULL;
-
-	if (dpp3_construct(dpp, ctx, inst,
-			&dpp_regs[inst], &tf_shift, &tf_mask))
-		return &dpp->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(dpp);
-	return NULL;
-}
-
-static struct output_pixel_processor *dcn31_opp_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_opp *opp =
-		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
-
-	if (!opp) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dcn20_opp_construct(opp, ctx, inst,
-			&opp_regs[inst], &opp_shift, &opp_mask);
-	return &opp->base;
-}
-
-static struct dce_aux *dcn31_aux_engine_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct aux_engine_dce110 *aux_engine =
-		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
-
-	if (!aux_engine)
-		return NULL;
-
-	dce110_aux_engine_construct(aux_engine, ctx, inst,
-				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
-				    &aux_engine_regs[inst],
-					&aux_mask,
-					&aux_shift,
-					ctx->dc->caps.extended_aux_timeout_support);
-
-	return &aux_engine->base;
-}
-#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST_DCN30(id) }
-
-static const struct dce_i2c_registers i2c_hw_regs[] = {
-		i2c_inst_regs(1),
-		i2c_inst_regs(2),
-		i2c_inst_regs(3),
-		i2c_inst_regs(4),
-		i2c_inst_regs(5),
-};
-
-static const struct dce_i2c_shift i2c_shifts = {
-		I2C_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dce_i2c_mask i2c_masks = {
-		I2C_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static struct dce_i2c_hw *dcn31_i2c_hw_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_i2c_hw *dce_i2c_hw =
-		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
-
-	if (!dce_i2c_hw)
-		return NULL;
-
-	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
-				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
-
-	return dce_i2c_hw;
-}
-static struct mpc *dcn31_mpc_create(
-		struct dc_context *ctx,
-		int num_mpcc,
-		int num_rmu)
-{
-	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc),
-					  GFP_KERNEL);
-
-	if (!mpc30)
-		return NULL;
-
-	dcn30_mpc_construct(mpc30, ctx,
-			&mpc_regs,
-			&mpc_shift,
-			&mpc_mask,
-			num_mpcc,
-			num_rmu);
-
-	return &mpc30->base;
-}
-
-static struct hubbub *dcn31_hubbub_create(struct dc_context *ctx)
-{
-	int i;
-
-	struct dcn20_hubbub *hubbub3 = kzalloc(sizeof(struct dcn20_hubbub),
-					  GFP_KERNEL);
-
-	if (!hubbub3)
-		return NULL;
-
-	hubbub31_construct(hubbub3, ctx,
-			&hubbub_reg,
-			&hubbub_shift,
-			&hubbub_mask,
-			dcn3_16_ip.det_buffer_size_kbytes,
-			dcn3_16_ip.pixel_chunk_size_kbytes,
-			dcn3_16_ip.config_return_buffer_size_in_kbytes);
-
-
-	for (i = 0; i < res_cap_dcn31.num_vmid; i++) {
-		struct dcn20_vmid *vmid = &hubbub3->vmid[i];
-
-		vmid->ctx = ctx;
-
-		vmid->regs = &vmid_regs[i];
-		vmid->shifts = &vmid_shifts;
-		vmid->masks = &vmid_masks;
-	}
-
-	return &hubbub3->base;
-}
-
-static struct timing_generator *dcn31_timing_generator_create(
-		struct dc_context *ctx,
-		uint32_t instance)
-{
-	struct optc *tgn10 =
-		kzalloc(sizeof(struct optc), GFP_KERNEL);
-
-	if (!tgn10)
-		return NULL;
-
-	tgn10->base.inst = instance;
-	tgn10->base.ctx = ctx;
-
-	tgn10->tg_regs = &optc_regs[instance];
-	tgn10->tg_shift = &optc_shift;
-	tgn10->tg_mask = &optc_mask;
-
-	dcn31_timing_generator_init(tgn10);
-
-	return &tgn10->base;
-}
-
-static const struct encoder_feature_support link_enc_feature = {
-		.max_hdmi_deep_color = COLOR_DEPTH_121212,
-		.max_hdmi_pixel_clock = 600000,
-		.hdmi_ycbcr420_supported = true,
-		.dp_ycbcr420_supported = true,
-		.fec_supported = true,
-		.flags.bits.IS_HBR2_CAPABLE = true,
-		.flags.bits.IS_HBR3_CAPABLE = true,
-		.flags.bits.IS_TPS3_CAPABLE = true,
-		.flags.bits.IS_TPS4_CAPABLE = true
-};
-
-static struct link_encoder *dcn31_link_encoder_create(
-	struct dc_context *ctx,
-	const struct encoder_init_data *enc_init_data)
-{
-	struct dcn20_link_encoder *enc20 =
-		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
-
-	if (!enc20)
-		return NULL;
-
-	dcn31_link_encoder_construct(enc20,
-			enc_init_data,
-			&link_enc_feature,
-			&link_enc_regs[enc_init_data->transmitter],
-			&link_enc_aux_regs[enc_init_data->channel - 1],
-			&link_enc_hpd_regs[enc_init_data->hpd_source],
-			&le_shift,
-			&le_mask);
-
-	return &enc20->enc10.base;
-}
-
-/* Create a minimal link encoder object not associated with a particular
- * physical connector.
- * resource_funcs.link_enc_create_minimal
- */
-static struct link_encoder *dcn31_link_enc_create_minimal(
-		struct dc_context *ctx, enum engine_id eng_id)
-{
-	struct dcn20_link_encoder *enc20;
-
-	if ((eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
-		return NULL;
-
-	enc20 = kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
-	if (!enc20)
-		return NULL;
-
-	dcn31_link_encoder_construct_minimal(
-			enc20,
-			ctx,
-			&link_enc_feature,
-			&link_enc_regs[eng_id - ENGINE_ID_DIGA],
-			eng_id);
-
-	return &enc20->enc10.base;
-}
-
-static struct panel_cntl *dcn31_panel_cntl_create(const struct panel_cntl_init_data *init_data)
-{
-	struct dcn31_panel_cntl *panel_cntl =
-		kzalloc(sizeof(struct dcn31_panel_cntl), GFP_KERNEL);
-
-	if (!panel_cntl)
-		return NULL;
-
-	dcn31_panel_cntl_construct(panel_cntl, init_data);
-
-	return &panel_cntl->base;
-}
-
-static void read_dce_straps(
-	struct dc_context *ctx,
-	struct resource_straps *straps)
-{
-	generic_reg_get(ctx, regDC_PINSTRAPS + BASE(regDC_PINSTRAPS_BASE_IDX),
-		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
-
-}
-
-static struct audio *dcn31_create_audio(
-		struct dc_context *ctx, unsigned int inst)
-{
-	return dce_audio_create(ctx, inst,
-			&audio_regs[inst], &audio_shift, &audio_mask);
-}
-
-static struct vpg *dcn31_vpg_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn31_vpg *vpg31 = kzalloc(sizeof(struct dcn31_vpg), GFP_KERNEL);
-
-	if (!vpg31)
-		return NULL;
-
-	vpg31_construct(vpg31, ctx, inst,
-			&vpg_regs[inst],
-			&vpg_shift,
-			&vpg_mask);
-
-	return &vpg31->base;
-}
-
-static struct afmt *dcn31_afmt_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn31_afmt *afmt31 = kzalloc(sizeof(struct dcn31_afmt), GFP_KERNEL);
-
-	if (!afmt31)
-		return NULL;
-
-	afmt31_construct(afmt31, ctx, inst,
-			&afmt_regs[inst],
-			&afmt_shift,
-			&afmt_mask);
-
-	// Light sleep by default, no need to power down here
-
-	return &afmt31->base;
-}
-
-
-static struct apg *dcn31_apg_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn31_apg *apg31 = kzalloc(sizeof(struct dcn31_apg), GFP_KERNEL);
-
-	if (!apg31)
-		return NULL;
-
-	apg31_construct(apg31, ctx, inst,
-			&apg_regs[inst],
-			&apg_shift,
-			&apg_mask);
-
-	return &apg31->base;
-}
-
-
-static struct stream_encoder *dcn316_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dcn10_stream_encoder *enc1;
-	struct vpg *vpg;
-	struct afmt *afmt;
-	int vpg_inst;
-	int afmt_inst;
-
-	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
-	if (eng_id <= ENGINE_ID_DIGF) {
-		vpg_inst = eng_id;
-		afmt_inst = eng_id;
-	} else
-		return NULL;
-
-	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
-	vpg = dcn31_vpg_create(ctx, vpg_inst);
-	afmt = dcn31_afmt_create(ctx, afmt_inst);
-
-	if (!enc1 || !vpg || !afmt) {
-		kfree(enc1);
-		kfree(vpg);
-		kfree(afmt);
-		return NULL;
-	}
-
-	dcn30_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
-					eng_id, vpg, afmt,
-					&stream_enc_regs[eng_id],
-					&se_shift, &se_mask);
-
-	return &enc1->base;
-}
-
-
-static struct hpo_dp_stream_encoder *dcn31_hpo_dp_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dcn31_hpo_dp_stream_encoder *hpo_dp_enc31;
-	struct vpg *vpg;
-	struct apg *apg;
-	uint32_t hpo_dp_inst;
-	uint32_t vpg_inst;
-	uint32_t apg_inst;
-
-	ASSERT((eng_id >= ENGINE_ID_HPO_DP_0) && (eng_id <= ENGINE_ID_HPO_DP_3));
-	hpo_dp_inst = eng_id - ENGINE_ID_HPO_DP_0;
-
-	/* Mapping of VPG register blocks to HPO DP block instance:
-	 * VPG[6] -> HPO_DP[0]
-	 * VPG[7] -> HPO_DP[1]
-	 * VPG[8] -> HPO_DP[2]
-	 * VPG[9] -> HPO_DP[3]
-	 */
-	vpg_inst = hpo_dp_inst + 6;
-
-	/* Mapping of APG register blocks to HPO DP block instance:
-	 * APG[0] -> HPO_DP[0]
-	 * APG[1] -> HPO_DP[1]
-	 * APG[2] -> HPO_DP[2]
-	 * APG[3] -> HPO_DP[3]
-	 */
-	apg_inst = hpo_dp_inst;
-
-	/* allocate HPO stream encoder and create VPG sub-block */
-	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_stream_encoder), GFP_KERNEL);
-	vpg = dcn31_vpg_create(ctx, vpg_inst);
-	apg = dcn31_apg_create(ctx, apg_inst);
-
-	if (!hpo_dp_enc31 || !vpg || !apg) {
-		kfree(hpo_dp_enc31);
-		kfree(vpg);
-		kfree(apg);
-		return NULL;
-	}
-
-	dcn31_hpo_dp_stream_encoder_construct(hpo_dp_enc31, ctx, ctx->dc_bios,
-					hpo_dp_inst, eng_id, vpg, apg,
-					&hpo_dp_stream_enc_regs[hpo_dp_inst],
-					&hpo_dp_se_shift, &hpo_dp_se_mask);
-
-	return &hpo_dp_enc31->base;
-}
-
-static struct hpo_dp_link_encoder *dcn31_hpo_dp_link_encoder_create(
-	uint8_t inst,
-	struct dc_context *ctx)
-{
-	struct dcn31_hpo_dp_link_encoder *hpo_dp_enc31;
-
-	/* allocate HPO link encoder */
-	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
-
-	hpo_dp_link_encoder31_construct(hpo_dp_enc31, ctx, inst,
-					&hpo_dp_link_enc_regs[inst],
-					&hpo_dp_le_shift, &hpo_dp_le_mask);
-
-	return &hpo_dp_enc31->base;
-}
-
-
-static struct dce_hwseq *dcn31_hwseq_create(
-	struct dc_context *ctx)
-{
-	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
-
-	if (hws) {
-		hws->ctx = ctx;
-		hws->regs = &hwseq_reg;
-		hws->shifts = &hwseq_shift;
-		hws->masks = &hwseq_mask;
-	}
-	return hws;
-}
-static const struct resource_create_funcs res_create_funcs = {
-	.read_dce_straps = read_dce_straps,
-	.create_audio = dcn31_create_audio,
-	.create_stream_encoder = dcn316_stream_encoder_create,
-	.create_hpo_dp_stream_encoder = dcn31_hpo_dp_stream_encoder_create,
-	.create_hpo_dp_link_encoder = dcn31_hpo_dp_link_encoder_create,
-	.create_hwseq = dcn31_hwseq_create,
-};
-
-static void dcn316_resource_destruct(struct dcn316_resource_pool *pool)
-{
-	unsigned int i;
-
-	for (i = 0; i < pool->base.stream_enc_count; i++) {
-		if (pool->base.stream_enc[i] != NULL) {
-			if (pool->base.stream_enc[i]->vpg != NULL) {
-				kfree(DCN30_VPG_FROM_VPG(pool->base.stream_enc[i]->vpg));
-				pool->base.stream_enc[i]->vpg = NULL;
-			}
-			if (pool->base.stream_enc[i]->afmt != NULL) {
-				kfree(DCN30_AFMT_FROM_AFMT(pool->base.stream_enc[i]->afmt));
-				pool->base.stream_enc[i]->afmt = NULL;
-			}
-			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
-			pool->base.stream_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.hpo_dp_stream_enc_count; i++) {
-		if (pool->base.hpo_dp_stream_enc[i] != NULL) {
-			if (pool->base.hpo_dp_stream_enc[i]->vpg != NULL) {
-				kfree(DCN30_VPG_FROM_VPG(pool->base.hpo_dp_stream_enc[i]->vpg));
-				pool->base.hpo_dp_stream_enc[i]->vpg = NULL;
-			}
-			if (pool->base.hpo_dp_stream_enc[i]->apg != NULL) {
-				kfree(DCN31_APG_FROM_APG(pool->base.hpo_dp_stream_enc[i]->apg));
-				pool->base.hpo_dp_stream_enc[i]->apg = NULL;
-			}
-			kfree(DCN3_1_HPO_DP_STREAM_ENC_FROM_HPO_STREAM_ENC(pool->base.hpo_dp_stream_enc[i]));
-			pool->base.hpo_dp_stream_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.hpo_dp_link_enc_count; i++) {
-		if (pool->base.hpo_dp_link_enc[i] != NULL) {
-			kfree(DCN3_1_HPO_DP_LINK_ENC_FROM_HPO_LINK_ENC(pool->base.hpo_dp_link_enc[i]));
-			pool->base.hpo_dp_link_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		if (pool->base.dscs[i] != NULL)
-			dcn20_dsc_destroy(&pool->base.dscs[i]);
-	}
-
-	if (pool->base.mpc != NULL) {
-		kfree(TO_DCN20_MPC(pool->base.mpc));
-		pool->base.mpc = NULL;
-	}
-	if (pool->base.hubbub != NULL) {
-		kfree(pool->base.hubbub);
-		pool->base.hubbub = NULL;
-	}
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		if (pool->base.dpps[i] != NULL)
-			dcn31_dpp_destroy(&pool->base.dpps[i]);
-
-		if (pool->base.ipps[i] != NULL)
-			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
-
-		if (pool->base.hubps[i] != NULL) {
-			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
-			pool->base.hubps[i] = NULL;
-		}
-
-		if (pool->base.irqs != NULL) {
-			dal_irq_service_destroy(&pool->base.irqs);
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		if (pool->base.engines[i] != NULL)
-			dce110_engine_destroy(&pool->base.engines[i]);
-		if (pool->base.hw_i2cs[i] != NULL) {
-			kfree(pool->base.hw_i2cs[i]);
-			pool->base.hw_i2cs[i] = NULL;
-		}
-		if (pool->base.sw_i2cs[i] != NULL) {
-			kfree(pool->base.sw_i2cs[i]);
-			pool->base.sw_i2cs[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
-		if (pool->base.opps[i] != NULL)
-			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		if (pool->base.timing_generators[i] != NULL)	{
-			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
-			pool->base.timing_generators[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
-		if (pool->base.dwbc[i] != NULL) {
-			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
-			pool->base.dwbc[i] = NULL;
-		}
-		if (pool->base.mcif_wb[i] != NULL) {
-			kfree(TO_DCN30_MMHUBBUB(pool->base.mcif_wb[i]));
-			pool->base.mcif_wb[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.audio_count; i++) {
-		if (pool->base.audios[i])
-			dce_aud_destroy(&pool->base.audios[i]);
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] != NULL) {
-			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
-			pool->base.clock_sources[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
-		if (pool->base.mpc_lut[i] != NULL) {
-			dc_3dlut_func_release(pool->base.mpc_lut[i]);
-			pool->base.mpc_lut[i] = NULL;
-		}
-		if (pool->base.mpc_shaper[i] != NULL) {
-			dc_transfer_func_release(pool->base.mpc_shaper[i]);
-			pool->base.mpc_shaper[i] = NULL;
-		}
-	}
-
-	if (pool->base.dp_clock_source != NULL) {
-		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
-		pool->base.dp_clock_source = NULL;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		if (pool->base.multiple_abms[i] != NULL)
-			dce_abm_destroy(&pool->base.multiple_abms[i]);
-	}
-
-	if (pool->base.psr != NULL)
-		dmub_psr_destroy(&pool->base.psr);
-
-	if (pool->base.dccg != NULL)
-		dcn_dccg_destroy(&pool->base.dccg);
-}
-
-static struct hubp *dcn31_hubp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn20_hubp *hubp2 =
-		kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
-
-	if (!hubp2)
-		return NULL;
-
-	if (hubp31_construct(hubp2, ctx, inst,
-			&hubp_regs[inst], &hubp_shift, &hubp_mask))
-		return &hubp2->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(hubp2);
-	return NULL;
-}
-
-static bool dcn31_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t pipe_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < pipe_count; i++) {
-		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc),
-						    GFP_KERNEL);
-
-		if (!dwbc30) {
-			dm_error("DC: failed to create dwbc30!\n");
-			return false;
-		}
-
-		dcn30_dwbc_construct(dwbc30, ctx,
-				&dwbc30_regs[i],
-				&dwbc30_shift,
-				&dwbc30_mask,
-				i);
-
-		pool->dwbc[i] = &dwbc30->base;
-	}
-	return true;
-}
-
-static bool dcn31_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t pipe_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < pipe_count; i++) {
-		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub),
-						    GFP_KERNEL);
-
-		if (!mcif_wb30) {
-			dm_error("DC: failed to create mcif_wb30!\n");
-			return false;
-		}
-
-		dcn30_mmhubbub_construct(mcif_wb30, ctx,
-				&mcif_wb30_regs[i],
-				&mcif_wb30_shift,
-				&mcif_wb30_mask,
-				i);
-
-		pool->mcif_wb[i] = &mcif_wb30->base;
-	}
-	return true;
-}
-
-static struct display_stream_compressor *dcn31_dsc_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_dsc *dsc =
-		kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
-
-	if (!dsc) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
-	return &dsc->base;
-}
-
-static void dcn316_destroy_resource_pool(struct resource_pool **pool)
-{
-	struct dcn316_resource_pool *dcn31_pool = TO_DCN316_RES_POOL(*pool);
-
-	dcn316_resource_destruct(dcn31_pool);
-	kfree(dcn31_pool);
-	*pool = NULL;
-}
-
-static struct clock_source *dcn31_clock_source_create(
-		struct dc_context *ctx,
-		struct dc_bios *bios,
-		enum clock_source_id id,
-		const struct dce110_clk_src_regs *regs,
-		bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src =
-		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dcn31_clk_src_construct(clk_src, ctx, bios, id,
-			regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-
-	kfree(clk_src);
-
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
-static bool is_dual_plane(enum surface_pixel_format format)
-{
-	return format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN || format == SURFACE_PIXEL_FORMAT_GRPH_RGBE_ALPHA;
-}
-
-static int dcn316_populate_dml_pipes_from_context(
-	struct dc *dc, struct dc_state *context,
-	display_e2e_pipe_params_st *pipes,
-	bool fast_validate)
-{
-	int i, pipe_cnt;
-	struct resource_context *res_ctx = &context->res_ctx;
-	struct pipe_ctx *pipe;
-	const int max_usable_det = context->bw_ctx.dml.ip.config_return_buffer_size_in_kbytes - DCN3_16_MIN_COMPBUF_SIZE_KB;
-
-	DC_FP_START();
-	dcn31x_populate_dml_pipes_from_context(dc, context, pipes, fast_validate);
-	DC_FP_END();
-
-	for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) {
-		struct dc_crtc_timing *timing;
-
-		if (!res_ctx->pipe_ctx[i].stream)
-			continue;
-		pipe = &res_ctx->pipe_ctx[i];
-		timing = &pipe->stream->timing;
-
-		/*
-		 * Immediate flip can be set dynamically after enabling the plane.
-		 * We need to require support for immediate flip or underflow can be
-		 * intermittently experienced depending on peak b/w requirements.
-		 */
-		pipes[pipe_cnt].pipe.src.immediate_flip = true;
-
-		pipes[pipe_cnt].pipe.src.unbounded_req_mode = false;
-		pipes[pipe_cnt].pipe.dest.vfront_porch = timing->v_front_porch;
-		pipes[pipe_cnt].pipe.src.dcc_rate = 3;
-		pipes[pipe_cnt].dout.dsc_input_bpc = 0;
-		DC_FP_START();
-		dcn31_zero_pipe_dcc_fraction(pipes, pipe_cnt);
-		DC_FP_END();
-
-		if (pipes[pipe_cnt].dout.dsc_enable) {
-			switch (timing->display_color_depth) {
-			case COLOR_DEPTH_888:
-				pipes[pipe_cnt].dout.dsc_input_bpc = 8;
-				break;
-			case COLOR_DEPTH_101010:
-				pipes[pipe_cnt].dout.dsc_input_bpc = 10;
-				break;
-			case COLOR_DEPTH_121212:
-				pipes[pipe_cnt].dout.dsc_input_bpc = 12;
-				break;
-			default:
-				ASSERT(0);
-				break;
-			}
-		}
-
-		pipe_cnt++;
-	}
-
-	if (pipe_cnt)
-		context->bw_ctx.dml.ip.det_buffer_size_kbytes =
-				(max_usable_det / DCN3_16_CRB_SEGMENT_SIZE_KB / pipe_cnt) * DCN3_16_CRB_SEGMENT_SIZE_KB;
-	if (context->bw_ctx.dml.ip.det_buffer_size_kbytes > DCN3_16_MAX_DET_SIZE)
-		context->bw_ctx.dml.ip.det_buffer_size_kbytes = DCN3_16_MAX_DET_SIZE;
-	ASSERT(context->bw_ctx.dml.ip.det_buffer_size_kbytes >= DCN3_16_DEFAULT_DET_SIZE);
-	dc->config.enable_4to1MPC = false;
-	if (pipe_cnt == 1 && pipe->plane_state && !dc->debug.disable_z9_mpc) {
-		if (is_dual_plane(pipe->plane_state->format)
-				&& pipe->plane_state->src_rect.width <= 1920 && pipe->plane_state->src_rect.height <= 1080) {
-			dc->config.enable_4to1MPC = true;
-			context->bw_ctx.dml.ip.det_buffer_size_kbytes =
-					(max_usable_det / DCN3_16_CRB_SEGMENT_SIZE_KB / 4) * DCN3_16_CRB_SEGMENT_SIZE_KB;
-		} else if (!is_dual_plane(pipe->plane_state->format)) {
-			context->bw_ctx.dml.ip.det_buffer_size_kbytes = 192;
-			pipes[0].pipe.src.unbounded_req_mode = true;
-		}
-	}
-
-	return pipe_cnt;
-}
-
-static void dcn316_get_panel_config_defaults(struct dc_panel_config *panel_config)
-{
-	*panel_config = panel_config_defaults;
-}
-
-static struct dc_cap_funcs cap_funcs = {
-	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
-};
-
-static struct resource_funcs dcn316_res_pool_funcs = {
-	.destroy = dcn316_destroy_resource_pool,
-	.link_enc_create = dcn31_link_encoder_create,
-	.link_enc_create_minimal = dcn31_link_enc_create_minimal,
-	.link_encs_assign = link_enc_cfg_link_encs_assign,
-	.link_enc_unassign = link_enc_cfg_link_enc_unassign,
-	.panel_cntl_create = dcn31_panel_cntl_create,
-	.validate_bandwidth = dcn31_validate_bandwidth,
-	.calculate_wm_and_dlg = dcn31_calculate_wm_and_dlg,
-	.update_soc_for_wm_a = dcn31_update_soc_for_wm_a,
-	.populate_dml_pipes = dcn316_populate_dml_pipes_from_context,
-	.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
-	.release_pipe = dcn20_release_pipe,
-	.add_stream_to_ctx = dcn30_add_stream_to_ctx,
-	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
-	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
-	.populate_dml_writeback_from_context = dcn31_populate_dml_writeback_from_context,
-	.set_mcif_arb_params = dcn31_set_mcif_arb_params,
-	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
-	.acquire_post_bldn_3dlut = dcn30_acquire_post_bldn_3dlut,
-	.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
-	.update_bw_bounding_box = dcn316_update_bw_bounding_box,
-	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
-	.get_panel_config_defaults = dcn316_get_panel_config_defaults,
-};
-
-static bool dcn316_resource_construct(
-	uint8_t num_virtual_links,
-	struct dc *dc,
-	struct dcn316_resource_pool *pool)
-{
-	int i;
-	struct dc_context *ctx = dc->ctx;
-	struct irq_service_init_data init_data;
-
-	ctx->dc_bios->regs = &bios_regs;
-
-	pool->base.res_cap = &res_cap_dcn31;
-
-	pool->base.funcs = &dcn316_res_pool_funcs;
-
-	/*************************************************
-	 *  Resource + asic cap harcoding                *
-	 *************************************************/
-	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
-	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
-	pool->base.mpcc_count = pool->base.res_cap->num_timing_generator;
-	dc->caps.max_downscale_ratio = 600;
-	dc->caps.i2c_speed_in_khz = 100;
-	dc->caps.i2c_speed_in_khz_hdcp = 5; /*1.5 w/a applied by default*/
-	dc->caps.max_cursor_size = 256;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.dmdata_alloc_size = 2048;
-	dc->caps.max_slave_planes = 2;
-	dc->caps.max_slave_yuv_planes = 2;
-	dc->caps.max_slave_rgb_planes = 2;
-	dc->caps.post_blend_color_processing = true;
-	dc->caps.force_dp_tps4_for_cp2520 = true;
-	if (dc->config.forceHBR2CP2520)
-		dc->caps.force_dp_tps4_for_cp2520 = false;
-	dc->caps.dp_hpo = true;
-	dc->caps.dp_hdmi21_pcon_support = true;
-	dc->caps.edp_dsc_support = true;
-	dc->caps.extended_aux_timeout_support = true;
-	dc->caps.dmcub_support = true;
-	dc->caps.is_apu = true;
-
-	/* Color pipeline capabilities */
-	dc->caps.color.dpp.dcn_arch = 1;
-	dc->caps.color.dpp.input_lut_shared = 0;
-	dc->caps.color.dpp.icsc = 1;
-	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
-	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
-	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
-	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
-	dc->caps.color.dpp.post_csc = 1;
-	dc->caps.color.dpp.gamma_corr = 1;
-	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
-
-	dc->caps.color.dpp.hw_3d_lut = 1;
-	dc->caps.color.dpp.ogam_ram = 1;
-	// no OGAM ROM on DCN301
-	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
-	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
-	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
-	dc->caps.color.dpp.ocsc = 0;
-
-	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //2
-	dc->caps.color.mpc.ogam_ram = 1;
-	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
-	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
-	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
-	dc->caps.color.mpc.ocsc = 1;
-
-	/* read VBIOS LTTPR caps */
-	{
-		if (ctx->dc_bios->funcs->get_lttpr_caps) {
-			enum bp_result bp_query_result;
-			uint8_t is_vbios_lttpr_enable = 0;
-
-			bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
-			dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
-		}
-
-		/* interop bit is implicit */
-		{
-			dc->caps.vbios_lttpr_aware = true;
-		}
-	}
-
-	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
-		dc->debug = debug_defaults_drv;
-
-	// Init the vm_helper
-	if (dc->vm_helper)
-		vm_helper_init(dc->vm_helper, 16);
-
-	/*************************************************
-	 *  Create resources                             *
-	 *************************************************/
-
-	/* Clock Sources for Pixel Clock*/
-	pool->base.clock_sources[DCN31_CLK_SRC_PLL0] =
-			dcn31_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL0,
-				&clk_src_regs[0], false);
-	pool->base.clock_sources[DCN31_CLK_SRC_PLL1] =
-			dcn31_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL1,
-				&clk_src_regs[1], false);
-	pool->base.clock_sources[DCN31_CLK_SRC_PLL2] =
-			dcn31_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL2,
-				&clk_src_regs[2], false);
-	pool->base.clock_sources[DCN31_CLK_SRC_PLL3] =
-			dcn31_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL3,
-				&clk_src_regs[3], false);
-	pool->base.clock_sources[DCN31_CLK_SRC_PLL4] =
-			dcn31_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL4,
-				&clk_src_regs[4], false);
-
-	pool->base.clk_src_count = DCN30_CLK_SRC_TOTAL;
-
-	/* todo: not reuse phy_pll registers */
-	pool->base.dp_clock_source =
-			dcn31_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_ID_DP_DTO,
-				&clk_src_regs[0], true);
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-
-	/* TODO: DCCG */
-	pool->base.dccg = dccg31_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
-	if (pool->base.dccg == NULL) {
-		dm_error("DC: failed to create dccg!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	/* TODO: IRQ */
-	init_data.ctx = dc->ctx;
-	pool->base.irqs = dal_irq_service_dcn31_create(&init_data);
-	if (!pool->base.irqs)
-		goto create_fail;
-
-	/* HUBBUB */
-	pool->base.hubbub = dcn31_hubbub_create(ctx);
-	if (pool->base.hubbub == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create hubbub!\n");
-		goto create_fail;
-	}
-
-	/* HUBPs, DPPs, OPPs and TGs */
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		pool->base.hubps[i] = dcn31_hubp_create(ctx, i);
-		if (pool->base.hubps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create hubps!\n");
-			goto create_fail;
-		}
-
-		pool->base.dpps[i] = dcn31_dpp_create(ctx, i);
-		if (pool->base.dpps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create dpps!\n");
-			goto create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
-		pool->base.opps[i] = dcn31_opp_create(ctx, i);
-		if (pool->base.opps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create output pixel processor!\n");
-			goto create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		pool->base.timing_generators[i] = dcn31_timing_generator_create(
-				ctx, i);
-		if (pool->base.timing_generators[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create tg!\n");
-			goto create_fail;
-		}
-	}
-	pool->base.timing_generator_count = i;
-
-	/* PSR */
-	pool->base.psr = dmub_psr_create(ctx);
-	if (pool->base.psr == NULL) {
-		dm_error("DC: failed to create psr obj!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	/* ABM */
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		pool->base.multiple_abms[i] = dmub_abm_create(ctx,
-				&abm_regs[i],
-				&abm_shift,
-				&abm_mask);
-		if (pool->base.multiple_abms[i] == NULL) {
-			dm_error("DC: failed to create abm for pipe %d!\n", i);
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-
-	/* MPC and DSC */
-	pool->base.mpc = dcn31_mpc_create(ctx, pool->base.mpcc_count, pool->base.res_cap->num_mpc_3dlut);
-	if (pool->base.mpc == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mpc!\n");
-		goto create_fail;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		pool->base.dscs[i] = dcn31_dsc_create(ctx, i);
-		if (pool->base.dscs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create display stream compressor %d!\n", i);
-			goto create_fail;
-		}
-	}
-
-	/* DWB and MMHUBBUB */
-	if (!dcn31_dwbc_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create dwbc!\n");
-		goto create_fail;
-	}
-
-	if (!dcn31_mmhubbub_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mcif_wb!\n");
-		goto create_fail;
-	}
-
-	/* AUX and I2C */
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		pool->base.engines[i] = dcn31_aux_engine_create(ctx, i);
-		if (pool->base.engines[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create aux engine!!\n");
-			goto create_fail;
-		}
-		pool->base.hw_i2cs[i] = dcn31_i2c_hw_create(ctx, i);
-		if (pool->base.hw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create hw i2c!!\n");
-			goto create_fail;
-		}
-		pool->base.sw_i2cs[i] = NULL;
-	}
-
-	/* Audio, Stream Encoders including HPO and virtual, MPC 3D LUTs */
-	if (!resource_construct(num_virtual_links, dc, &pool->base,
-			&res_create_funcs))
-		goto create_fail;
-
-	/* HW Sequencer and Plane caps */
-	dcn31_hw_sequencer_construct(dc);
-
-	dc->caps.max_planes =  pool->base.pipe_count;
-
-	for (i = 0; i < dc->caps.max_planes; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	dc->cap_funcs = cap_funcs;
-
-	dc->dcn_ip->max_num_dpp = dcn3_16_ip.max_num_dpp;
-
-	return true;
-
-create_fail:
-
-	dcn316_resource_destruct(pool);
-
-	return false;
-}
-
-struct resource_pool *dcn316_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc)
-{
-	struct dcn316_resource_pool *pool =
-		kzalloc(sizeof(struct dcn316_resource_pool), GFP_KERNEL);
-
-	if (!pool)
-		return NULL;
-
-	if (dcn316_resource_construct(init_data->num_virtual_links, dc, pool))
-		return &pool->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(pool);
-	return NULL;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.h b/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.h
deleted file mode 100644
index aba6d634131b..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright 2021 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#ifndef _DCN316_RESOURCE_H_
-#define _DCN316_RESOURCE_H_
-
-#include "core_types.h"
-
-#define TO_DCN316_RES_POOL(pool)\
-	container_of(pool, struct dcn316_resource_pool, base)
-
-extern struct _vcs_dpi_ip_params_st dcn3_16_ip;
-
-struct dcn316_resource_pool {
-	struct resource_pool base;
-};
-
-struct resource_pool *dcn316_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc);
-
-#endif /* _DCN316_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/Makefile b/drivers/gpu/drm/amd/display/dc/dcn32/Makefile
index 8bb251307247..3bb17dd01e4c 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/Makefile
@@ -10,7 +10,7 @@
 #
 # Makefile for dcn32.
 
-DCN32 = dcn32_resource.o dcn32_hubbub.o dcn32_init.o dcn32_dccg.o \
+DCN32 = dcn32_hubbub.o dcn32_init.o dcn32_dccg.o \
 		dcn32_dccg.o dcn32_optc.o dcn32_mmhubbub.o dcn32_hubp.o dcn32_dpp.o \
 		dcn32_dio_stream_encoder.o dcn32_dio_link_encoder.o dcn32_hpo_dp_link_encoder.o \
 		dcn32_resource_helpers.o dcn32_mpc.o
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c
index 994b21ed272f..1a2adb354718 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c
@@ -71,21 +71,24 @@ void mpc32_power_on_blnd_lut(
 {
 	struct dcn30_mpc *mpc30 = TO_DCN30_MPC(mpc);
 
+/*
 	if (mpc->ctx->dc->debug.enable_mem_low_power.bits.cm) {
 		if (power_on) {
 			REG_UPDATE(MPCC_MCM_MEM_PWR_CTRL[mpcc_id], MPCC_MCM_1DLUT_MEM_PWR_FORCE, 0);
 			REG_WAIT(MPCC_MCM_MEM_PWR_CTRL[mpcc_id], MPCC_MCM_1DLUT_MEM_PWR_STATE, 0, 1, 5);
 		} else if (!mpc->ctx->dc->debug.disable_mem_low_power) {
-			ASSERT(false);
-			/* TODO: change to mpc
-			 *  dpp_base->ctx->dc->optimized_required = true;
-			 *  dpp_base->deferred_reg_writes.bits.disable_blnd_lut = true;
-			 */
+			//TODO: change to mpc
+			dpp_base->ctx->dc->optimized_required = true;
+			dpp_base->deferred_reg_writes.bits.disable_blnd_lut = true;
 		}
 	} else {
 		REG_SET(MPCC_MCM_MEM_PWR_CTRL[mpcc_id], 0,
 				MPCC_MCM_1DLUT_MEM_PWR_FORCE, power_on == true ? 0 : 1);
 	}
+*/
+
+	REG_SET(MPCC_MCM_MEM_PWR_CTRL[mpcc_id], 0,
+			MPCC_MCM_1DLUT_MEM_PWR_FORCE, power_on == true ? 0 : 1);
 }
 
 static enum dc_lut_mode mpc32_get_post1dlut_current(struct mpc *mpc, uint32_t mpcc_id)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c
deleted file mode 100644
index 89b072447dba..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c
+++ /dev/null
@@ -1,2862 +0,0 @@
-// SPDX-License-Identifier: MIT
-/*
- * Copyright 2022 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#include "dm_services.h"
-#include "dc.h"
-
-#include "dcn32_init.h"
-
-#include "resource.h"
-#include "include/irq_service_interface.h"
-#include "dcn32_resource.h"
-
-#include "dcn20/dcn20_resource.h"
-#include "dcn30/dcn30_resource.h"
-
-#include "dcn10/dcn10_ipp.h"
-#include "dcn30/dcn30_hubbub.h"
-#include "dcn31/dcn31_hubbub.h"
-#include "dcn32/dcn32_hubbub.h"
-#include "dcn32/dcn32_mpc.h"
-#include "dcn32_hubp.h"
-#include "irq/dcn32/irq_service_dcn32.h"
-#include "dcn32/dcn32_dpp.h"
-#include "dcn32/dcn32_optc.h"
-#include "dcn20/dcn20_hwseq.h"
-#include "dcn30/dcn30_hwseq.h"
-#include "dce110/dce110_hwseq.h"
-#include "dcn30/dcn30_opp.h"
-#include "dcn20/dcn20_dsc.h"
-#include "dcn30/dcn30_vpg.h"
-#include "dcn30/dcn30_afmt.h"
-#include "dcn30/dcn30_dio_stream_encoder.h"
-#include "dcn32/dcn32_dio_stream_encoder.h"
-#include "dcn31/dcn31_hpo_dp_stream_encoder.h"
-#include "dcn31/dcn31_hpo_dp_link_encoder.h"
-#include "dcn32/dcn32_hpo_dp_link_encoder.h"
-#include "dcn31/dcn31_apg.h"
-#include "dcn31/dcn31_dio_link_encoder.h"
-#include "dcn32/dcn32_dio_link_encoder.h"
-#include "dce/dce_clock_source.h"
-#include "dce/dce_audio.h"
-#include "dce/dce_hwseq.h"
-#include "clk_mgr.h"
-#include "virtual/virtual_stream_encoder.h"
-#include "dml/display_mode_vba.h"
-#include "dcn32/dcn32_dccg.h"
-#include "dcn10/dcn10_resource.h"
-#include "link.h"
-#include "dcn31/dcn31_panel_cntl.h"
-
-#include "dcn30/dcn30_dwb.h"
-#include "dcn32/dcn32_mmhubbub.h"
-
-#include "dcn/dcn_3_2_0_offset.h"
-#include "dcn/dcn_3_2_0_sh_mask.h"
-#include "nbio/nbio_4_3_0_offset.h"
-
-#include "reg_helper.h"
-#include "dce/dmub_abm.h"
-#include "dce/dmub_psr.h"
-#include "dce/dce_aux.h"
-#include "dce/dce_i2c.h"
-
-#include "dml/dcn30/display_mode_vba_30.h"
-#include "vm_helper.h"
-#include "dcn20/dcn20_vmid.h"
-#include "dml/dcn32/dcn32_fpu.h"
-
-#include "dml2/dml2_wrapper.h"
-
-#define DC_LOGGER_INIT(logger)
-
-enum dcn32_clk_src_array_id {
-	DCN32_CLK_SRC_PLL0,
-	DCN32_CLK_SRC_PLL1,
-	DCN32_CLK_SRC_PLL2,
-	DCN32_CLK_SRC_PLL3,
-	DCN32_CLK_SRC_PLL4,
-	DCN32_CLK_SRC_TOTAL
-};
-
-/* begin *********************
- * macros to expend register list macro defined in HW object header file
- */
-
-/* DCN */
-#define BASE_INNER(seg) ctx->dcn_reg_offsets[seg]
-
-#define BASE(seg) BASE_INNER(seg)
-
-#define SR(reg_name)\
-		REG_STRUCT.reg_name = BASE(reg ## reg_name ## _BASE_IDX) +  \
-					reg ## reg_name
-#define SR_ARR(reg_name, id) \
-	REG_STRUCT[id].reg_name = BASE(reg##reg_name##_BASE_IDX) + reg##reg_name
-
-#define SR_ARR_INIT(reg_name, id, value) \
-	REG_STRUCT[id].reg_name = value
-
-#define SRI(reg_name, block, id)\
-	REG_STRUCT.reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define SRI_ARR(reg_name, block, id)\
-	REG_STRUCT[id].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define SR_ARR_I2C(reg_name, id) \
-	REG_STRUCT[id-1].reg_name = BASE(reg##reg_name##_BASE_IDX) + reg##reg_name
-
-#define SRI_ARR_I2C(reg_name, block, id)\
-	REG_STRUCT[id-1].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define SRI_ARR_ALPHABET(reg_name, block, index, id)\
-	REG_STRUCT[index].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define SRI2(reg_name, block, id)\
-	.reg_name = BASE(reg ## reg_name ## _BASE_IDX) +	\
-		reg ## reg_name
-#define SRI2_ARR(reg_name, block, id)\
-	REG_STRUCT[id].reg_name = BASE(reg ## reg_name ## _BASE_IDX) +	\
-		reg ## reg_name
-
-#define SRIR(var_name, reg_name, block, id)\
-	.var_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define SRII(reg_name, block, id)\
-	REG_STRUCT.reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define SRII_ARR_2(reg_name, block, id, inst)\
-	REG_STRUCT[inst].reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define SRII_MPC_RMU(reg_name, block, id)\
-	.RMU##_##reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define SRII_DWB(reg_name, temp_name, block, id)\
-	REG_STRUCT.reg_name[id] = BASE(reg ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## temp_name
-
-#define SF_DWB2(reg_name, block, id, field_name, post_fix)	\
-	.field_name = reg_name ## __ ## field_name ## post_fix
-
-#define DCCG_SRII(reg_name, block, id)\
-	REG_STRUCT.block ## _ ## reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define VUPDATE_SRII(reg_name, block, id)\
-	REG_STRUCT.reg_name[id] = BASE(reg ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
-		reg ## reg_name ## _ ## block ## id
-
-/* NBIO */
-#define NBIO_BASE_INNER(seg) ctx->nbio_reg_offsets[seg]
-
-#define NBIO_BASE(seg) \
-	NBIO_BASE_INNER(seg)
-
-#define NBIO_SR(reg_name)\
-	REG_STRUCT.reg_name = NBIO_BASE(regBIF_BX0_ ## reg_name ## _BASE_IDX) + \
-			regBIF_BX0_ ## reg_name
-#define NBIO_SR_ARR(reg_name, id)\
-	REG_STRUCT[id].reg_name = NBIO_BASE(regBIF_BX0_ ## reg_name ## _BASE_IDX) + \
-		regBIF_BX0_ ## reg_name
-
-#undef CTX
-#define CTX ctx
-#define REG(reg_name) \
-	(ctx->dcn_reg_offsets[reg ## reg_name ## _BASE_IDX] + reg ## reg_name)
-
-static struct bios_registers bios_regs;
-
-#define bios_regs_init() \
-		( \
-		NBIO_SR(BIOS_SCRATCH_3),\
-		NBIO_SR(BIOS_SCRATCH_6)\
-		)
-
-#define clk_src_regs_init(index, pllid)\
-	CS_COMMON_REG_LIST_DCN3_0_RI(index, pllid)
-
-static struct dce110_clk_src_regs clk_src_regs[5];
-
-static const struct dce110_clk_src_shift cs_shift = {
-		CS_COMMON_MASK_SH_LIST_DCN3_2(__SHIFT)
-};
-
-static const struct dce110_clk_src_mask cs_mask = {
-		CS_COMMON_MASK_SH_LIST_DCN3_2(_MASK)
-};
-
-#define abm_regs_init(id)\
-		ABM_DCN32_REG_LIST_RI(id)
-
-static struct dce_abm_registers abm_regs[4];
-
-static const struct dce_abm_shift abm_shift = {
-		ABM_MASK_SH_LIST_DCN32(__SHIFT)
-};
-
-static const struct dce_abm_mask abm_mask = {
-		ABM_MASK_SH_LIST_DCN32(_MASK)
-};
-
-#define audio_regs_init(id)\
-		AUD_COMMON_REG_LIST_RI(id)
-
-static struct dce_audio_registers audio_regs[5];
-
-#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
-		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
-
-static const struct dce_audio_shift audio_shift = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_audio_mask audio_mask = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
-};
-
-#define vpg_regs_init(id)\
-	VPG_DCN3_REG_LIST_RI(id)
-
-static struct dcn30_vpg_registers vpg_regs[10];
-
-static const struct dcn30_vpg_shift vpg_shift = {
-	DCN3_VPG_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn30_vpg_mask vpg_mask = {
-	DCN3_VPG_MASK_SH_LIST(_MASK)
-};
-
-#define afmt_regs_init(id)\
-	AFMT_DCN3_REG_LIST_RI(id)
-
-static struct dcn30_afmt_registers afmt_regs[6];
-
-static const struct dcn30_afmt_shift afmt_shift = {
-	DCN3_AFMT_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn30_afmt_mask afmt_mask = {
-	DCN3_AFMT_MASK_SH_LIST(_MASK)
-};
-
-#define apg_regs_init(id)\
-	APG_DCN31_REG_LIST_RI(id)
-
-static struct dcn31_apg_registers apg_regs[4];
-
-static const struct dcn31_apg_shift apg_shift = {
-	DCN31_APG_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_apg_mask apg_mask = {
-		DCN31_APG_MASK_SH_LIST(_MASK)
-};
-
-#define stream_enc_regs_init(id)\
-	SE_DCN32_REG_LIST_RI(id)
-
-static struct dcn10_stream_enc_registers stream_enc_regs[5];
-
-static const struct dcn10_stream_encoder_shift se_shift = {
-		SE_COMMON_MASK_SH_LIST_DCN32(__SHIFT)
-};
-
-static const struct dcn10_stream_encoder_mask se_mask = {
-		SE_COMMON_MASK_SH_LIST_DCN32(_MASK)
-};
-
-
-#define aux_regs_init(id)\
-	DCN2_AUX_REG_LIST_RI(id)
-
-static struct dcn10_link_enc_aux_registers link_enc_aux_regs[5];
-
-#define hpd_regs_init(id)\
-	HPD_REG_LIST_RI(id)
-
-static struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[5];
-
-#define link_regs_init(id, phyid)\
-	( \
-	LE_DCN31_REG_LIST_RI(id), \
-	UNIPHY_DCN2_REG_LIST_RI(id, phyid)\
-	)
-	/*DPCS_DCN31_REG_LIST(id),*/ \
-
-static struct dcn10_link_enc_registers link_enc_regs[5];
-
-static const struct dcn10_link_enc_shift le_shift = {
-	LINK_ENCODER_MASK_SH_LIST_DCN31(__SHIFT), \
-	//DPCS_DCN31_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn10_link_enc_mask le_mask = {
-	LINK_ENCODER_MASK_SH_LIST_DCN31(_MASK), \
-	//DPCS_DCN31_MASK_SH_LIST(_MASK)
-};
-
-#define hpo_dp_stream_encoder_reg_init(id)\
-	DCN3_1_HPO_DP_STREAM_ENC_REG_LIST_RI(id)
-
-static struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[4];
-
-static const struct dcn31_hpo_dp_stream_encoder_shift hpo_dp_se_shift = {
-	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_hpo_dp_stream_encoder_mask hpo_dp_se_mask = {
-	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(_MASK)
-};
-
-
-#define hpo_dp_link_encoder_reg_init(id)\
-	DCN3_1_HPO_DP_LINK_ENC_REG_LIST_RI(id)
-	/*DCN3_1_RDPCSTX_REG_LIST(0),*/
-	/*DCN3_1_RDPCSTX_REG_LIST(1),*/
-	/*DCN3_1_RDPCSTX_REG_LIST(2),*/
-	/*DCN3_1_RDPCSTX_REG_LIST(3),*/
-
-static struct dcn31_hpo_dp_link_encoder_registers hpo_dp_link_enc_regs[2];
-
-static const struct dcn31_hpo_dp_link_encoder_shift hpo_dp_le_shift = {
-	DCN3_2_HPO_DP_LINK_ENC_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_hpo_dp_link_encoder_mask hpo_dp_le_mask = {
-	DCN3_2_HPO_DP_LINK_ENC_MASK_SH_LIST(_MASK)
-};
-
-#define dpp_regs_init(id)\
-	DPP_REG_LIST_DCN30_COMMON_RI(id)
-
-static struct dcn3_dpp_registers dpp_regs[4];
-
-static const struct dcn3_dpp_shift tf_shift = {
-		DPP_REG_LIST_SH_MASK_DCN30_COMMON(__SHIFT)
-};
-
-static const struct dcn3_dpp_mask tf_mask = {
-		DPP_REG_LIST_SH_MASK_DCN30_COMMON(_MASK)
-};
-
-
-#define opp_regs_init(id)\
-	OPP_REG_LIST_DCN30_RI(id)
-
-static struct dcn20_opp_registers opp_regs[4];
-
-static const struct dcn20_opp_shift opp_shift = {
-	OPP_MASK_SH_LIST_DCN20(__SHIFT)
-};
-
-static const struct dcn20_opp_mask opp_mask = {
-	OPP_MASK_SH_LIST_DCN20(_MASK)
-};
-
-#define aux_engine_regs_init(id)\
-	( \
-	AUX_COMMON_REG_LIST0_RI(id), \
-	SR_ARR_INIT(AUXN_IMPCAL, id, 0), \
-	SR_ARR_INIT(AUXP_IMPCAL, id, 0), \
-	SR_ARR_INIT(AUX_RESET_MASK, id, DP_AUX0_AUX_CONTROL__AUX_RESET_MASK), \
-	SR_ARR_INIT(AUX_RESET_MASK, id, DP_AUX0_AUX_CONTROL__AUX_RESET_MASK)\
-	)
-
-static struct dce110_aux_registers aux_engine_regs[5];
-
-static const struct dce110_aux_registers_shift aux_shift = {
-	DCN_AUX_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce110_aux_registers_mask aux_mask = {
-	DCN_AUX_MASK_SH_LIST(_MASK)
-};
-
-#define dwbc_regs_dcn3_init(id)\
-	DWBC_COMMON_REG_LIST_DCN30_RI(id)
-
-static struct dcn30_dwbc_registers dwbc30_regs[1];
-
-static const struct dcn30_dwbc_shift dwbc30_shift = {
-	DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_dwbc_mask dwbc30_mask = {
-	DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define mcif_wb_regs_dcn3_init(id)\
-	MCIF_WB_COMMON_REG_LIST_DCN32_RI(id)
-
-static struct dcn30_mmhubbub_registers mcif_wb30_regs[1];
-
-static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
-	MCIF_WB_COMMON_MASK_SH_LIST_DCN32(__SHIFT)
-};
-
-static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
-	MCIF_WB_COMMON_MASK_SH_LIST_DCN32(_MASK)
-};
-
-#define dsc_regsDCN20_init(id)\
-	DSC_REG_LIST_DCN20_RI(id)
-
-static struct dcn20_dsc_registers dsc_regs[4];
-
-static const struct dcn20_dsc_shift dsc_shift = {
-	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
-};
-
-static const struct dcn20_dsc_mask dsc_mask = {
-	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
-};
-
-static struct dcn30_mpc_registers mpc_regs;
-
-#define dcn_mpc_regs_init() \
-	MPC_REG_LIST_DCN3_2_RI(0),\
-	MPC_REG_LIST_DCN3_2_RI(1),\
-	MPC_REG_LIST_DCN3_2_RI(2),\
-	MPC_REG_LIST_DCN3_2_RI(3),\
-	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(0),\
-	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(1),\
-	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(2),\
-	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(3),\
-	MPC_DWB_MUX_REG_LIST_DCN3_0_RI(0)
-
-static const struct dcn30_mpc_shift mpc_shift = {
-	MPC_COMMON_MASK_SH_LIST_DCN32(__SHIFT)
-};
-
-static const struct dcn30_mpc_mask mpc_mask = {
-	MPC_COMMON_MASK_SH_LIST_DCN32(_MASK)
-};
-
-#define optc_regs_init(id)\
-	OPTC_COMMON_REG_LIST_DCN3_2_RI(id)
-
-static struct dcn_optc_registers optc_regs[4];
-
-static const struct dcn_optc_shift optc_shift = {
-	OPTC_COMMON_MASK_SH_LIST_DCN3_2(__SHIFT)
-};
-
-static const struct dcn_optc_mask optc_mask = {
-	OPTC_COMMON_MASK_SH_LIST_DCN3_2(_MASK)
-};
-
-#define hubp_regs_init(id)\
-	HUBP_REG_LIST_DCN32_RI(id)
-
-static struct dcn_hubp2_registers hubp_regs[4];
-
-
-static const struct dcn_hubp2_shift hubp_shift = {
-		HUBP_MASK_SH_LIST_DCN32(__SHIFT)
-};
-
-static const struct dcn_hubp2_mask hubp_mask = {
-		HUBP_MASK_SH_LIST_DCN32(_MASK)
-};
-
-static struct dcn_hubbub_registers hubbub_reg;
-#define hubbub_reg_init()\
-		HUBBUB_REG_LIST_DCN32_RI(0)
-
-static const struct dcn_hubbub_shift hubbub_shift = {
-		HUBBUB_MASK_SH_LIST_DCN32(__SHIFT)
-};
-
-static const struct dcn_hubbub_mask hubbub_mask = {
-		HUBBUB_MASK_SH_LIST_DCN32(_MASK)
-};
-
-static struct dccg_registers dccg_regs;
-
-#define dccg_regs_init()\
-	DCCG_REG_LIST_DCN32_RI()
-
-static const struct dccg_shift dccg_shift = {
-		DCCG_MASK_SH_LIST_DCN32(__SHIFT)
-};
-
-static const struct dccg_mask dccg_mask = {
-		DCCG_MASK_SH_LIST_DCN32(_MASK)
-};
-
-
-#define SRII2(reg_name_pre, reg_name_post, id)\
-	.reg_name_pre ## _ ##  reg_name_post[id] = BASE(reg ## reg_name_pre \
-			## id ## _ ## reg_name_post ## _BASE_IDX) + \
-			reg ## reg_name_pre ## id ## _ ## reg_name_post
-
-
-#define HWSEQ_DCN32_REG_LIST()\
-	SR(DCHUBBUB_GLOBAL_TIMER_CNTL), \
-	SR(DIO_MEM_PWR_CTRL), \
-	SR(ODM_MEM_PWR_CTRL3), \
-	SR(MMHUBBUB_MEM_PWR_CNTL), \
-	SR(DCCG_GATE_DISABLE_CNTL), \
-	SR(DCCG_GATE_DISABLE_CNTL2), \
-	SR(DCFCLK_CNTL),\
-	SR(DC_MEM_GLOBAL_PWR_REQ_CNTL), \
-	SRII(PIXEL_RATE_CNTL, OTG, 0), \
-	SRII(PIXEL_RATE_CNTL, OTG, 1),\
-	SRII(PIXEL_RATE_CNTL, OTG, 2),\
-	SRII(PIXEL_RATE_CNTL, OTG, 3),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 0),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 1),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 2),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 3),\
-	SR(MICROSECOND_TIME_BASE_DIV), \
-	SR(MILLISECOND_TIME_BASE_DIV), \
-	SR(DISPCLK_FREQ_CHANGE_CNTL), \
-	SR(RBBMIF_TIMEOUT_DIS), \
-	SR(RBBMIF_TIMEOUT_DIS_2), \
-	SR(DCHUBBUB_CRC_CTRL), \
-	SR(DPP_TOP0_DPP_CRC_CTRL), \
-	SR(DPP_TOP0_DPP_CRC_VAL_B_A), \
-	SR(DPP_TOP0_DPP_CRC_VAL_R_G), \
-	SR(MPC_CRC_CTRL), \
-	SR(MPC_CRC_RESULT_GB), \
-	SR(MPC_CRC_RESULT_C), \
-	SR(MPC_CRC_RESULT_AR), \
-	SR(DOMAIN0_PG_CONFIG), \
-	SR(DOMAIN1_PG_CONFIG), \
-	SR(DOMAIN2_PG_CONFIG), \
-	SR(DOMAIN3_PG_CONFIG), \
-	SR(DOMAIN16_PG_CONFIG), \
-	SR(DOMAIN17_PG_CONFIG), \
-	SR(DOMAIN18_PG_CONFIG), \
-	SR(DOMAIN19_PG_CONFIG), \
-	SR(DOMAIN0_PG_STATUS), \
-	SR(DOMAIN1_PG_STATUS), \
-	SR(DOMAIN2_PG_STATUS), \
-	SR(DOMAIN3_PG_STATUS), \
-	SR(DOMAIN16_PG_STATUS), \
-	SR(DOMAIN17_PG_STATUS), \
-	SR(DOMAIN18_PG_STATUS), \
-	SR(DOMAIN19_PG_STATUS), \
-	SR(D1VGA_CONTROL), \
-	SR(D2VGA_CONTROL), \
-	SR(D3VGA_CONTROL), \
-	SR(D4VGA_CONTROL), \
-	SR(D5VGA_CONTROL), \
-	SR(D6VGA_CONTROL), \
-	SR(DC_IP_REQUEST_CNTL), \
-	SR(AZALIA_AUDIO_DTO), \
-	SR(AZALIA_CONTROLLER_CLOCK_GATING)
-
-static struct dce_hwseq_registers hwseq_reg;
-
-#define hwseq_reg_init()\
-	HWSEQ_DCN32_REG_LIST()
-
-#define HWSEQ_DCN32_MASK_SH_LIST(mask_sh)\
-	HWSEQ_DCN_MASK_SH_LIST(mask_sh), \
-	HWS_SF(, DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, mask_sh), \
-	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN0_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN1_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN2_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN3_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN16_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN17_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN18_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN19_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh), \
-	HWS_SF(, AZALIA_AUDIO_DTO, AZALIA_AUDIO_DTO_MODULE, mask_sh), \
-	HWS_SF(, HPO_TOP_CLOCK_CONTROL, HPO_HDMISTREAMCLK_G_GATE_DIS, mask_sh), \
-	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_UNASSIGNED_PWR_MODE, mask_sh), \
-	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_VBLANK_PWR_MODE, mask_sh), \
-	HWS_SF(, MMHUBBUB_MEM_PWR_CNTL, VGA_MEM_PWR_FORCE, mask_sh)
-
-static const struct dce_hwseq_shift hwseq_shift = {
-		HWSEQ_DCN32_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_hwseq_mask hwseq_mask = {
-		HWSEQ_DCN32_MASK_SH_LIST(_MASK)
-};
-#define vmid_regs_init(id)\
-		DCN20_VMID_REG_LIST_RI(id)
-
-static struct dcn_vmid_registers vmid_regs[16];
-
-static const struct dcn20_vmid_shift vmid_shifts = {
-		DCN20_VMID_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn20_vmid_mask vmid_masks = {
-		DCN20_VMID_MASK_SH_LIST(_MASK)
-};
-
-static const struct resource_caps res_cap_dcn32 = {
-	.num_timing_generator = 4,
-	.num_opp = 4,
-	.num_video_plane = 4,
-	.num_audio = 5,
-	.num_stream_encoder = 5,
-	.num_hpo_dp_stream_encoder = 4,
-	.num_hpo_dp_link_encoder = 2,
-	.num_pll = 5,
-	.num_dwb = 1,
-	.num_ddc = 5,
-	.num_vmid = 16,
-	.num_mpc_3dlut = 4,
-	.num_dsc = 4,
-};
-
-static const struct dc_plane_cap plane_cap = {
-	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.per_pixel_alpha = true,
-
-	.pixel_format_support = {
-			.argb8888 = true,
-			.nv12 = true,
-			.fp16 = true,
-			.p010 = true,
-			.ayuv = false,
-	},
-
-	.max_upscale_factor = {
-			.argb8888 = 16000,
-			.nv12 = 16000,
-			.fp16 = 16000
-	},
-
-	// 6:1 downscaling ratio: 1000/6 = 166.666
-	.max_downscale_factor = {
-			.argb8888 = 167,
-			.nv12 = 167,
-			.fp16 = 167
-	},
-	64,
-	64
-};
-
-static const struct dc_debug_options debug_defaults_drv = {
-	.disable_dmcu = true,
-	.force_abm_enable = false,
-	.timing_trace = false,
-	.clock_trace = true,
-	.disable_pplib_clock_request = false,
-	.pipe_split_policy = MPC_SPLIT_AVOID, // Due to CRB, no need to MPC split anymore
-	.force_single_disp_pipe_split = false,
-	.disable_dcc = DCC_ENABLE,
-	.vsr_support = true,
-	.performance_trace = false,
-	.max_downscale_src_width = 7680,/*upto 8K*/
-	.disable_pplib_wm_range = false,
-	.scl_reset_length10 = true,
-	.sanity_checks = false,
-	.underflow_assert_delay_us = 0xFFFFFFFF,
-	.dwb_fi_phase = -1, // -1 = disable,
-	.dmub_command_table = true,
-	.enable_mem_low_power = {
-		.bits = {
-			.vga = false,
-			.i2c = false,
-			.dmcu = false, // This is previously known to cause hang on S3 cycles if enabled
-			.dscl = false,
-			.cm = false,
-			.mpc = false,
-			.optc = true,
-		}
-	},
-	.use_max_lb = true,
-	.force_disable_subvp = false,
-	.exit_idle_opt_for_cursor_updates = true,
-	.using_dml2 = false,
-	.enable_single_display_2to1_odm_policy = true,
-
-	/* Must match enable_single_display_2to1_odm_policy to support dynamic ODM transitions*/
-	.enable_double_buffered_dsc_pg_support = true,
-	.enable_dp_dig_pixel_rate_div_policy = 1,
-	.allow_sw_cursor_fallback = false, // Linux can't do SW cursor "fallback"
-	.alloc_extra_way_for_cursor = true,
-	.min_prefetch_in_strobe_ns = 60000, // 60us
-	.disable_unbounded_requesting = false,
-	.override_dispclk_programming = true,
-	.disable_fpo_optimizations = false,
-	.fpo_vactive_margin_us = 2000, // 2000us
-	.disable_fpo_vactive = false,
-	.disable_boot_optimizations = false,
-	.disable_subvp_high_refresh = false,
-	.disable_dp_plus_plus_wa = true,
-	.fpo_vactive_min_active_margin_us = 200,
-	.fpo_vactive_max_blank_us = 1000,
-	.enable_legacy_fast_update = false,
-};
-
-static struct dce_aux *dcn32_aux_engine_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct aux_engine_dce110 *aux_engine =
-		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
-
-	if (!aux_engine)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT aux_engine_regs
-	aux_engine_regs_init(0),
-	aux_engine_regs_init(1),
-	aux_engine_regs_init(2),
-	aux_engine_regs_init(3),
-	aux_engine_regs_init(4);
-
-	dce110_aux_engine_construct(aux_engine, ctx, inst,
-				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
-				    &aux_engine_regs[inst],
-					&aux_mask,
-					&aux_shift,
-					ctx->dc->caps.extended_aux_timeout_support);
-
-	return &aux_engine->base;
-}
-#define i2c_inst_regs_init(id)\
-	I2C_HW_ENGINE_COMMON_REG_LIST_DCN30_RI(id)
-
-static struct dce_i2c_registers i2c_hw_regs[5];
-
-static const struct dce_i2c_shift i2c_shifts = {
-		I2C_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dce_i2c_mask i2c_masks = {
-		I2C_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static struct dce_i2c_hw *dcn32_i2c_hw_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_i2c_hw *dce_i2c_hw =
-		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
-
-	if (!dce_i2c_hw)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT i2c_hw_regs
-	i2c_inst_regs_init(1),
-	i2c_inst_regs_init(2),
-	i2c_inst_regs_init(3),
-	i2c_inst_regs_init(4),
-	i2c_inst_regs_init(5);
-
-	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
-				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
-
-	return dce_i2c_hw;
-}
-
-static struct clock_source *dcn32_clock_source_create(
-		struct dc_context *ctx,
-		struct dc_bios *bios,
-		enum clock_source_id id,
-		const struct dce110_clk_src_regs *regs,
-		bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src =
-		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dcn31_clk_src_construct(clk_src, ctx, bios, id,
-			regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-
-	kfree(clk_src);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
-static struct hubbub *dcn32_hubbub_create(struct dc_context *ctx)
-{
-	int i;
-
-	struct dcn20_hubbub *hubbub2 = kzalloc(sizeof(struct dcn20_hubbub),
-					  GFP_KERNEL);
-
-	if (!hubbub2)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT hubbub_reg
-	hubbub_reg_init();
-
-#undef REG_STRUCT
-#define REG_STRUCT vmid_regs
-	vmid_regs_init(0),
-	vmid_regs_init(1),
-	vmid_regs_init(2),
-	vmid_regs_init(3),
-	vmid_regs_init(4),
-	vmid_regs_init(5),
-	vmid_regs_init(6),
-	vmid_regs_init(7),
-	vmid_regs_init(8),
-	vmid_regs_init(9),
-	vmid_regs_init(10),
-	vmid_regs_init(11),
-	vmid_regs_init(12),
-	vmid_regs_init(13),
-	vmid_regs_init(14),
-	vmid_regs_init(15);
-
-	hubbub32_construct(hubbub2, ctx,
-			&hubbub_reg,
-			&hubbub_shift,
-			&hubbub_mask,
-			ctx->dc->dml.ip.det_buffer_size_kbytes,
-			ctx->dc->dml.ip.pixel_chunk_size_kbytes,
-			ctx->dc->dml.ip.config_return_buffer_size_in_kbytes);
-
-
-	for (i = 0; i < res_cap_dcn32.num_vmid; i++) {
-		struct dcn20_vmid *vmid = &hubbub2->vmid[i];
-
-		vmid->ctx = ctx;
-
-		vmid->regs = &vmid_regs[i];
-		vmid->shifts = &vmid_shifts;
-		vmid->masks = &vmid_masks;
-	}
-
-	return &hubbub2->base;
-}
-
-static struct hubp *dcn32_hubp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn20_hubp *hubp2 =
-		kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
-
-	if (!hubp2)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT hubp_regs
-	hubp_regs_init(0),
-	hubp_regs_init(1),
-	hubp_regs_init(2),
-	hubp_regs_init(3);
-
-	if (hubp32_construct(hubp2, ctx, inst,
-			&hubp_regs[inst], &hubp_shift, &hubp_mask))
-		return &hubp2->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(hubp2);
-	return NULL;
-}
-
-static void dcn32_dpp_destroy(struct dpp **dpp)
-{
-	kfree(TO_DCN30_DPP(*dpp));
-	*dpp = NULL;
-}
-
-static struct dpp *dcn32_dpp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn3_dpp *dpp3 =
-		kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
-
-	if (!dpp3)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT dpp_regs
-	dpp_regs_init(0),
-	dpp_regs_init(1),
-	dpp_regs_init(2),
-	dpp_regs_init(3);
-
-	if (dpp32_construct(dpp3, ctx, inst,
-			&dpp_regs[inst], &tf_shift, &tf_mask))
-		return &dpp3->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(dpp3);
-	return NULL;
-}
-
-static struct mpc *dcn32_mpc_create(
-		struct dc_context *ctx,
-		int num_mpcc,
-		int num_rmu)
-{
-	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc),
-					  GFP_KERNEL);
-
-	if (!mpc30)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT mpc_regs
-	dcn_mpc_regs_init();
-
-	dcn32_mpc_construct(mpc30, ctx,
-			&mpc_regs,
-			&mpc_shift,
-			&mpc_mask,
-			num_mpcc,
-			num_rmu);
-
-	return &mpc30->base;
-}
-
-static struct output_pixel_processor *dcn32_opp_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_opp *opp2 =
-		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
-
-	if (!opp2) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-#undef REG_STRUCT
-#define REG_STRUCT opp_regs
-	opp_regs_init(0),
-	opp_regs_init(1),
-	opp_regs_init(2),
-	opp_regs_init(3);
-
-	dcn20_opp_construct(opp2, ctx, inst,
-			&opp_regs[inst], &opp_shift, &opp_mask);
-	return &opp2->base;
-}
-
-
-static struct timing_generator *dcn32_timing_generator_create(
-		struct dc_context *ctx,
-		uint32_t instance)
-{
-	struct optc *tgn10 =
-		kzalloc(sizeof(struct optc), GFP_KERNEL);
-
-	if (!tgn10)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT optc_regs
-	optc_regs_init(0),
-	optc_regs_init(1),
-	optc_regs_init(2),
-	optc_regs_init(3);
-
-	tgn10->base.inst = instance;
-	tgn10->base.ctx = ctx;
-
-	tgn10->tg_regs = &optc_regs[instance];
-	tgn10->tg_shift = &optc_shift;
-	tgn10->tg_mask = &optc_mask;
-
-	dcn32_timing_generator_init(tgn10);
-
-	return &tgn10->base;
-}
-
-static const struct encoder_feature_support link_enc_feature = {
-		.max_hdmi_deep_color = COLOR_DEPTH_121212,
-		.max_hdmi_pixel_clock = 600000,
-		.hdmi_ycbcr420_supported = true,
-		.dp_ycbcr420_supported = true,
-		.fec_supported = true,
-		.flags.bits.IS_HBR2_CAPABLE = true,
-		.flags.bits.IS_HBR3_CAPABLE = true,
-		.flags.bits.IS_TPS3_CAPABLE = true,
-		.flags.bits.IS_TPS4_CAPABLE = true
-};
-
-static struct link_encoder *dcn32_link_encoder_create(
-	struct dc_context *ctx,
-	const struct encoder_init_data *enc_init_data)
-{
-	struct dcn20_link_encoder *enc20 =
-		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
-
-	if (!enc20)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT link_enc_aux_regs
-	aux_regs_init(0),
-	aux_regs_init(1),
-	aux_regs_init(2),
-	aux_regs_init(3),
-	aux_regs_init(4);
-
-#undef REG_STRUCT
-#define REG_STRUCT link_enc_hpd_regs
-	hpd_regs_init(0),
-	hpd_regs_init(1),
-	hpd_regs_init(2),
-	hpd_regs_init(3),
-	hpd_regs_init(4);
-
-#undef REG_STRUCT
-#define REG_STRUCT link_enc_regs
-	link_regs_init(0, A),
-	link_regs_init(1, B),
-	link_regs_init(2, C),
-	link_regs_init(3, D),
-	link_regs_init(4, E);
-
-	dcn32_link_encoder_construct(enc20,
-			enc_init_data,
-			&link_enc_feature,
-			&link_enc_regs[enc_init_data->transmitter],
-			&link_enc_aux_regs[enc_init_data->channel - 1],
-			&link_enc_hpd_regs[enc_init_data->hpd_source],
-			&le_shift,
-			&le_mask);
-
-	return &enc20->enc10.base;
-}
-
-struct panel_cntl *dcn32_panel_cntl_create(const struct panel_cntl_init_data *init_data)
-{
-	struct dcn31_panel_cntl *panel_cntl =
-		kzalloc(sizeof(struct dcn31_panel_cntl), GFP_KERNEL);
-
-	if (!panel_cntl)
-		return NULL;
-
-	dcn31_panel_cntl_construct(panel_cntl, init_data);
-
-	return &panel_cntl->base;
-}
-
-static void read_dce_straps(
-	struct dc_context *ctx,
-	struct resource_straps *straps)
-{
-	generic_reg_get(ctx, ctx->dcn_reg_offsets[regDC_PINSTRAPS_BASE_IDX] + regDC_PINSTRAPS,
-		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
-
-}
-
-static struct audio *dcn32_create_audio(
-		struct dc_context *ctx, unsigned int inst)
-{
-
-#undef REG_STRUCT
-#define REG_STRUCT audio_regs
-	audio_regs_init(0),
-	audio_regs_init(1),
-	audio_regs_init(2),
-	audio_regs_init(3),
-	audio_regs_init(4);
-
-	return dce_audio_create(ctx, inst,
-			&audio_regs[inst], &audio_shift, &audio_mask);
-}
-
-static struct vpg *dcn32_vpg_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn30_vpg *vpg3 = kzalloc(sizeof(struct dcn30_vpg), GFP_KERNEL);
-
-	if (!vpg3)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT vpg_regs
-	vpg_regs_init(0),
-	vpg_regs_init(1),
-	vpg_regs_init(2),
-	vpg_regs_init(3),
-	vpg_regs_init(4),
-	vpg_regs_init(5),
-	vpg_regs_init(6),
-	vpg_regs_init(7),
-	vpg_regs_init(8),
-	vpg_regs_init(9);
-
-	vpg3_construct(vpg3, ctx, inst,
-			&vpg_regs[inst],
-			&vpg_shift,
-			&vpg_mask);
-
-	return &vpg3->base;
-}
-
-static struct afmt *dcn32_afmt_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn30_afmt *afmt3 = kzalloc(sizeof(struct dcn30_afmt), GFP_KERNEL);
-
-	if (!afmt3)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT afmt_regs
-	afmt_regs_init(0),
-	afmt_regs_init(1),
-	afmt_regs_init(2),
-	afmt_regs_init(3),
-	afmt_regs_init(4),
-	afmt_regs_init(5);
-
-	afmt3_construct(afmt3, ctx, inst,
-			&afmt_regs[inst],
-			&afmt_shift,
-			&afmt_mask);
-
-	return &afmt3->base;
-}
-
-static struct apg *dcn31_apg_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn31_apg *apg31 = kzalloc(sizeof(struct dcn31_apg), GFP_KERNEL);
-
-	if (!apg31)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT apg_regs
-	apg_regs_init(0),
-	apg_regs_init(1),
-	apg_regs_init(2),
-	apg_regs_init(3);
-
-	apg31_construct(apg31, ctx, inst,
-			&apg_regs[inst],
-			&apg_shift,
-			&apg_mask);
-
-	return &apg31->base;
-}
-
-static struct stream_encoder *dcn32_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dcn10_stream_encoder *enc1;
-	struct vpg *vpg;
-	struct afmt *afmt;
-	int vpg_inst;
-	int afmt_inst;
-
-	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
-	if (eng_id <= ENGINE_ID_DIGF) {
-		vpg_inst = eng_id;
-		afmt_inst = eng_id;
-	} else
-		return NULL;
-
-	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
-	vpg = dcn32_vpg_create(ctx, vpg_inst);
-	afmt = dcn32_afmt_create(ctx, afmt_inst);
-
-	if (!enc1 || !vpg || !afmt) {
-		kfree(enc1);
-		kfree(vpg);
-		kfree(afmt);
-		return NULL;
-	}
-
-#undef REG_STRUCT
-#define REG_STRUCT stream_enc_regs
-	stream_enc_regs_init(0),
-	stream_enc_regs_init(1),
-	stream_enc_regs_init(2),
-	stream_enc_regs_init(3),
-	stream_enc_regs_init(4);
-
-	dcn32_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
-					eng_id, vpg, afmt,
-					&stream_enc_regs[eng_id],
-					&se_shift, &se_mask);
-
-	return &enc1->base;
-}
-
-static struct hpo_dp_stream_encoder *dcn32_hpo_dp_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dcn31_hpo_dp_stream_encoder *hpo_dp_enc31;
-	struct vpg *vpg;
-	struct apg *apg;
-	uint32_t hpo_dp_inst;
-	uint32_t vpg_inst;
-	uint32_t apg_inst;
-
-	ASSERT((eng_id >= ENGINE_ID_HPO_DP_0) && (eng_id <= ENGINE_ID_HPO_DP_3));
-	hpo_dp_inst = eng_id - ENGINE_ID_HPO_DP_0;
-
-	/* Mapping of VPG register blocks to HPO DP block instance:
-	 * VPG[6] -> HPO_DP[0]
-	 * VPG[7] -> HPO_DP[1]
-	 * VPG[8] -> HPO_DP[2]
-	 * VPG[9] -> HPO_DP[3]
-	 */
-	vpg_inst = hpo_dp_inst + 6;
-
-	/* Mapping of APG register blocks to HPO DP block instance:
-	 * APG[0] -> HPO_DP[0]
-	 * APG[1] -> HPO_DP[1]
-	 * APG[2] -> HPO_DP[2]
-	 * APG[3] -> HPO_DP[3]
-	 */
-	apg_inst = hpo_dp_inst;
-
-	/* allocate HPO stream encoder and create VPG sub-block */
-	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_stream_encoder), GFP_KERNEL);
-	vpg = dcn32_vpg_create(ctx, vpg_inst);
-	apg = dcn31_apg_create(ctx, apg_inst);
-
-	if (!hpo_dp_enc31 || !vpg || !apg) {
-		kfree(hpo_dp_enc31);
-		kfree(vpg);
-		kfree(apg);
-		return NULL;
-	}
-
-#undef REG_STRUCT
-#define REG_STRUCT hpo_dp_stream_enc_regs
-	hpo_dp_stream_encoder_reg_init(0),
-	hpo_dp_stream_encoder_reg_init(1),
-	hpo_dp_stream_encoder_reg_init(2),
-	hpo_dp_stream_encoder_reg_init(3);
-
-	dcn31_hpo_dp_stream_encoder_construct(hpo_dp_enc31, ctx, ctx->dc_bios,
-					hpo_dp_inst, eng_id, vpg, apg,
-					&hpo_dp_stream_enc_regs[hpo_dp_inst],
-					&hpo_dp_se_shift, &hpo_dp_se_mask);
-
-	return &hpo_dp_enc31->base;
-}
-
-static struct hpo_dp_link_encoder *dcn32_hpo_dp_link_encoder_create(
-	uint8_t inst,
-	struct dc_context *ctx)
-{
-	struct dcn31_hpo_dp_link_encoder *hpo_dp_enc31;
-
-	/* allocate HPO link encoder */
-	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
-
-#undef REG_STRUCT
-#define REG_STRUCT hpo_dp_link_enc_regs
-	hpo_dp_link_encoder_reg_init(0),
-	hpo_dp_link_encoder_reg_init(1);
-
-	hpo_dp_link_encoder32_construct(hpo_dp_enc31, ctx, inst,
-					&hpo_dp_link_enc_regs[inst],
-					&hpo_dp_le_shift, &hpo_dp_le_mask);
-
-	return &hpo_dp_enc31->base;
-}
-
-static struct dce_hwseq *dcn32_hwseq_create(
-	struct dc_context *ctx)
-{
-	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
-
-#undef REG_STRUCT
-#define REG_STRUCT hwseq_reg
-	hwseq_reg_init();
-
-	if (hws) {
-		hws->ctx = ctx;
-		hws->regs = &hwseq_reg;
-		hws->shifts = &hwseq_shift;
-		hws->masks = &hwseq_mask;
-	}
-	return hws;
-}
-static const struct resource_create_funcs res_create_funcs = {
-	.read_dce_straps = read_dce_straps,
-	.create_audio = dcn32_create_audio,
-	.create_stream_encoder = dcn32_stream_encoder_create,
-	.create_hpo_dp_stream_encoder = dcn32_hpo_dp_stream_encoder_create,
-	.create_hpo_dp_link_encoder = dcn32_hpo_dp_link_encoder_create,
-	.create_hwseq = dcn32_hwseq_create,
-};
-
-static void dcn32_resource_destruct(struct dcn32_resource_pool *pool)
-{
-	unsigned int i;
-
-	for (i = 0; i < pool->base.stream_enc_count; i++) {
-		if (pool->base.stream_enc[i] != NULL) {
-			if (pool->base.stream_enc[i]->vpg != NULL) {
-				kfree(DCN30_VPG_FROM_VPG(pool->base.stream_enc[i]->vpg));
-				pool->base.stream_enc[i]->vpg = NULL;
-			}
-			if (pool->base.stream_enc[i]->afmt != NULL) {
-				kfree(DCN30_AFMT_FROM_AFMT(pool->base.stream_enc[i]->afmt));
-				pool->base.stream_enc[i]->afmt = NULL;
-			}
-			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
-			pool->base.stream_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.hpo_dp_stream_enc_count; i++) {
-		if (pool->base.hpo_dp_stream_enc[i] != NULL) {
-			if (pool->base.hpo_dp_stream_enc[i]->vpg != NULL) {
-				kfree(DCN30_VPG_FROM_VPG(pool->base.hpo_dp_stream_enc[i]->vpg));
-				pool->base.hpo_dp_stream_enc[i]->vpg = NULL;
-			}
-			if (pool->base.hpo_dp_stream_enc[i]->apg != NULL) {
-				kfree(DCN31_APG_FROM_APG(pool->base.hpo_dp_stream_enc[i]->apg));
-				pool->base.hpo_dp_stream_enc[i]->apg = NULL;
-			}
-			kfree(DCN3_1_HPO_DP_STREAM_ENC_FROM_HPO_STREAM_ENC(pool->base.hpo_dp_stream_enc[i]));
-			pool->base.hpo_dp_stream_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.hpo_dp_link_enc_count; i++) {
-		if (pool->base.hpo_dp_link_enc[i] != NULL) {
-			kfree(DCN3_1_HPO_DP_LINK_ENC_FROM_HPO_LINK_ENC(pool->base.hpo_dp_link_enc[i]));
-			pool->base.hpo_dp_link_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		if (pool->base.dscs[i] != NULL)
-			dcn20_dsc_destroy(&pool->base.dscs[i]);
-	}
-
-	if (pool->base.mpc != NULL) {
-		kfree(TO_DCN20_MPC(pool->base.mpc));
-		pool->base.mpc = NULL;
-	}
-	if (pool->base.hubbub != NULL) {
-		kfree(TO_DCN20_HUBBUB(pool->base.hubbub));
-		pool->base.hubbub = NULL;
-	}
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		if (pool->base.dpps[i] != NULL)
-			dcn32_dpp_destroy(&pool->base.dpps[i]);
-
-		if (pool->base.ipps[i] != NULL)
-			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
-
-		if (pool->base.hubps[i] != NULL) {
-			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
-			pool->base.hubps[i] = NULL;
-		}
-
-		if (pool->base.irqs != NULL) {
-			dal_irq_service_destroy(&pool->base.irqs);
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		if (pool->base.engines[i] != NULL)
-			dce110_engine_destroy(&pool->base.engines[i]);
-		if (pool->base.hw_i2cs[i] != NULL) {
-			kfree(pool->base.hw_i2cs[i]);
-			pool->base.hw_i2cs[i] = NULL;
-		}
-		if (pool->base.sw_i2cs[i] != NULL) {
-			kfree(pool->base.sw_i2cs[i]);
-			pool->base.sw_i2cs[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
-		if (pool->base.opps[i] != NULL)
-			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		if (pool->base.timing_generators[i] != NULL)	{
-			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
-			pool->base.timing_generators[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
-		if (pool->base.dwbc[i] != NULL) {
-			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
-			pool->base.dwbc[i] = NULL;
-		}
-		if (pool->base.mcif_wb[i] != NULL) {
-			kfree(TO_DCN30_MMHUBBUB(pool->base.mcif_wb[i]));
-			pool->base.mcif_wb[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.audio_count; i++) {
-		if (pool->base.audios[i])
-			dce_aud_destroy(&pool->base.audios[i]);
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] != NULL) {
-			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
-			pool->base.clock_sources[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
-		if (pool->base.mpc_lut[i] != NULL) {
-			dc_3dlut_func_release(pool->base.mpc_lut[i]);
-			pool->base.mpc_lut[i] = NULL;
-		}
-		if (pool->base.mpc_shaper[i] != NULL) {
-			dc_transfer_func_release(pool->base.mpc_shaper[i]);
-			pool->base.mpc_shaper[i] = NULL;
-		}
-	}
-
-	if (pool->base.dp_clock_source != NULL) {
-		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
-		pool->base.dp_clock_source = NULL;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		if (pool->base.multiple_abms[i] != NULL)
-			dce_abm_destroy(&pool->base.multiple_abms[i]);
-	}
-
-	if (pool->base.psr != NULL)
-		dmub_psr_destroy(&pool->base.psr);
-
-	if (pool->base.dccg != NULL)
-		dcn_dccg_destroy(&pool->base.dccg);
-
-	if (pool->base.oem_device != NULL) {
-		struct dc *dc = pool->base.oem_device->ctx->dc;
-
-		dc->link_srv->destroy_ddc_service(&pool->base.oem_device);
-	}
-}
-
-
-static bool dcn32_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t dwb_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < dwb_count; i++) {
-		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc),
-						    GFP_KERNEL);
-
-		if (!dwbc30) {
-			dm_error("DC: failed to create dwbc30!\n");
-			return false;
-		}
-
-#undef REG_STRUCT
-#define REG_STRUCT dwbc30_regs
-		dwbc_regs_dcn3_init(0);
-
-		dcn30_dwbc_construct(dwbc30, ctx,
-				&dwbc30_regs[i],
-				&dwbc30_shift,
-				&dwbc30_mask,
-				i);
-
-		pool->dwbc[i] = &dwbc30->base;
-	}
-	return true;
-}
-
-static bool dcn32_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t dwb_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < dwb_count; i++) {
-		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub),
-						    GFP_KERNEL);
-
-		if (!mcif_wb30) {
-			dm_error("DC: failed to create mcif_wb30!\n");
-			return false;
-		}
-
-#undef REG_STRUCT
-#define REG_STRUCT mcif_wb30_regs
-		mcif_wb_regs_dcn3_init(0);
-
-		dcn32_mmhubbub_construct(mcif_wb30, ctx,
-				&mcif_wb30_regs[i],
-				&mcif_wb30_shift,
-				&mcif_wb30_mask,
-				i);
-
-		pool->mcif_wb[i] = &mcif_wb30->base;
-	}
-	return true;
-}
-
-static struct display_stream_compressor *dcn32_dsc_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_dsc *dsc =
-		kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
-
-	if (!dsc) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-#undef REG_STRUCT
-#define REG_STRUCT dsc_regs
-	dsc_regsDCN20_init(0),
-	dsc_regsDCN20_init(1),
-	dsc_regsDCN20_init(2),
-	dsc_regsDCN20_init(3);
-
-	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
-
-	dsc->max_image_width = 6016;
-
-	return &dsc->base;
-}
-
-static void dcn32_destroy_resource_pool(struct resource_pool **pool)
-{
-	struct dcn32_resource_pool *dcn32_pool = TO_DCN32_RES_POOL(*pool);
-
-	dcn32_resource_destruct(dcn32_pool);
-	kfree(dcn32_pool);
-	*pool = NULL;
-}
-
-bool dcn32_acquire_post_bldn_3dlut(
-		struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		int mpcc_id,
-		struct dc_3dlut **lut,
-		struct dc_transfer_func **shaper)
-{
-	bool ret = false;
-
-	ASSERT(*lut == NULL && *shaper == NULL);
-	*lut = NULL;
-	*shaper = NULL;
-
-	if (!res_ctx->is_mpc_3dlut_acquired[mpcc_id]) {
-		*lut = pool->mpc_lut[mpcc_id];
-		*shaper = pool->mpc_shaper[mpcc_id];
-		res_ctx->is_mpc_3dlut_acquired[mpcc_id] = true;
-		ret = true;
-	}
-	return ret;
-}
-
-bool dcn32_release_post_bldn_3dlut(
-		struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		struct dc_3dlut **lut,
-		struct dc_transfer_func **shaper)
-{
-	int i;
-	bool ret = false;
-
-	for (i = 0; i < pool->res_cap->num_mpc_3dlut; i++) {
-		if (pool->mpc_lut[i] == *lut && pool->mpc_shaper[i] == *shaper) {
-			res_ctx->is_mpc_3dlut_acquired[i] = false;
-			pool->mpc_lut[i]->state.raw = 0;
-			*lut = NULL;
-			*shaper = NULL;
-			ret = true;
-			break;
-		}
-	}
-	return ret;
-}
-
-static void dcn32_enable_phantom_plane(struct dc *dc,
-		struct dc_state *context,
-		struct dc_stream_state *phantom_stream,
-		unsigned int dc_pipe_idx)
-{
-	struct dc_plane_state *phantom_plane = NULL;
-	struct dc_plane_state *prev_phantom_plane = NULL;
-	struct pipe_ctx *curr_pipe = &context->res_ctx.pipe_ctx[dc_pipe_idx];
-
-	while (curr_pipe) {
-		if (curr_pipe->top_pipe && curr_pipe->top_pipe->plane_state == curr_pipe->plane_state)
-			phantom_plane = prev_phantom_plane;
-		else
-			phantom_plane = dc_create_plane_state(dc);
-
-		memcpy(&phantom_plane->address, &curr_pipe->plane_state->address, sizeof(phantom_plane->address));
-		memcpy(&phantom_plane->scaling_quality, &curr_pipe->plane_state->scaling_quality,
-				sizeof(phantom_plane->scaling_quality));
-		memcpy(&phantom_plane->src_rect, &curr_pipe->plane_state->src_rect, sizeof(phantom_plane->src_rect));
-		memcpy(&phantom_plane->dst_rect, &curr_pipe->plane_state->dst_rect, sizeof(phantom_plane->dst_rect));
-		memcpy(&phantom_plane->clip_rect, &curr_pipe->plane_state->clip_rect, sizeof(phantom_plane->clip_rect));
-		memcpy(&phantom_plane->plane_size, &curr_pipe->plane_state->plane_size,
-				sizeof(phantom_plane->plane_size));
-		memcpy(&phantom_plane->tiling_info, &curr_pipe->plane_state->tiling_info,
-				sizeof(phantom_plane->tiling_info));
-		memcpy(&phantom_plane->dcc, &curr_pipe->plane_state->dcc, sizeof(phantom_plane->dcc));
-		phantom_plane->format = curr_pipe->plane_state->format;
-		phantom_plane->rotation = curr_pipe->plane_state->rotation;
-		phantom_plane->visible = curr_pipe->plane_state->visible;
-
-		/* Shadow pipe has small viewport. */
-		phantom_plane->clip_rect.y = 0;
-		phantom_plane->clip_rect.height = phantom_stream->src.height;
-
-		phantom_plane->is_phantom = true;
-
-		dc_add_plane_to_context(dc, phantom_stream, phantom_plane, context);
-
-		curr_pipe = curr_pipe->bottom_pipe;
-		prev_phantom_plane = phantom_plane;
-	}
-}
-
-static struct dc_stream_state *dcn32_enable_phantom_stream(struct dc *dc,
-		struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		unsigned int pipe_cnt,
-		unsigned int dc_pipe_idx)
-{
-	struct dc_stream_state *phantom_stream = NULL;
-	struct pipe_ctx *ref_pipe = &context->res_ctx.pipe_ctx[dc_pipe_idx];
-
-	phantom_stream = dc_create_stream_for_sink(ref_pipe->stream->sink);
-	phantom_stream->signal = SIGNAL_TYPE_VIRTUAL;
-	phantom_stream->dpms_off = true;
-	phantom_stream->mall_stream_config.type = SUBVP_PHANTOM;
-	phantom_stream->mall_stream_config.paired_stream = ref_pipe->stream;
-	ref_pipe->stream->mall_stream_config.type = SUBVP_MAIN;
-	ref_pipe->stream->mall_stream_config.paired_stream = phantom_stream;
-
-	/* stream has limited viewport and small timing */
-	memcpy(&phantom_stream->timing, &ref_pipe->stream->timing, sizeof(phantom_stream->timing));
-	memcpy(&phantom_stream->src, &ref_pipe->stream->src, sizeof(phantom_stream->src));
-	memcpy(&phantom_stream->dst, &ref_pipe->stream->dst, sizeof(phantom_stream->dst));
-	DC_FP_START();
-	dcn32_set_phantom_stream_timing(dc, context, ref_pipe, phantom_stream, pipes, pipe_cnt, dc_pipe_idx);
-	DC_FP_END();
-
-	dc_add_stream_to_ctx(dc, context, phantom_stream);
-	return phantom_stream;
-}
-
-void dcn32_retain_phantom_pipes(struct dc *dc, struct dc_state *context)
-{
-	int i;
-	struct dc_plane_state *phantom_plane = NULL;
-	struct dc_stream_state *phantom_stream = NULL;
-
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-
-		if (resource_is_pipe_type(pipe, OTG_MASTER) &&
-				resource_is_pipe_type(pipe, DPP_PIPE) &&
-				pipe->stream->mall_stream_config.type == SUBVP_PHANTOM) {
-			phantom_plane = pipe->plane_state;
-			phantom_stream = pipe->stream;
-
-			dc_plane_state_retain(phantom_plane);
-			dc_stream_retain(phantom_stream);
-		}
-	}
-}
-
-// return true if removed piped from ctx, false otherwise
-bool dcn32_remove_phantom_pipes(struct dc *dc, struct dc_state *context, bool fast_update)
-{
-	int i;
-	bool removed_pipe = false;
-	struct dc_plane_state *phantom_plane = NULL;
-	struct dc_stream_state *phantom_stream = NULL;
-
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-		// build scaling params for phantom pipes
-		if (pipe->plane_state && pipe->stream && pipe->stream->mall_stream_config.type == SUBVP_PHANTOM) {
-			phantom_plane = pipe->plane_state;
-			phantom_stream = pipe->stream;
-
-			dc_rem_all_planes_for_stream(dc, pipe->stream, context);
-			dc_remove_stream_from_ctx(dc, context, pipe->stream);
-
-			/* Ref count is incremented on allocation and also when added to the context.
-			 * Therefore we must call release for the the phantom plane and stream once
-			 * they are removed from the ctx to finally decrement the refcount to 0 to free.
-			 */
-			dc_plane_state_release(phantom_plane);
-			dc_stream_release(phantom_stream);
-
-			removed_pipe = true;
-		}
-
-		/* For non-full updates, a shallow copy of the current state
-		 * is created. In this case we don't want to erase the current
-		 * state (there can be 2 HIRQL threads, one in flip, and one in
-		 * checkMPO) that can cause a race condition.
-		 *
-		 * This is just a workaround, needs a proper fix.
-		 */
-		if (!fast_update) {
-			// Clear all phantom stream info
-			if (pipe->stream) {
-				pipe->stream->mall_stream_config.type = SUBVP_NONE;
-				pipe->stream->mall_stream_config.paired_stream = NULL;
-			}
-
-			if (pipe->plane_state) {
-				pipe->plane_state->is_phantom = false;
-			}
-		}
-	}
-	return removed_pipe;
-}
-
-/* TODO: Input to this function should indicate which pipe indexes (or streams)
- * require a phantom pipe / stream
- */
-void dcn32_add_phantom_pipes(struct dc *dc, struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		unsigned int pipe_cnt,
-		unsigned int index)
-{
-	struct dc_stream_state *phantom_stream = NULL;
-	unsigned int i;
-
-	// The index of the DC pipe passed into this function is guarenteed to
-	// be a valid candidate for SubVP (i.e. has a plane, stream, doesn't
-	// already have phantom pipe assigned, etc.) by previous checks.
-	phantom_stream = dcn32_enable_phantom_stream(dc, context, pipes, pipe_cnt, index);
-	dcn32_enable_phantom_plane(dc, context, phantom_stream, index);
-
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-
-		// Build scaling params for phantom pipes which were newly added.
-		// We determine which phantom pipes were added by comparing with
-		// the phantom stream.
-		if (pipe->plane_state && pipe->stream && pipe->stream == phantom_stream &&
-				pipe->stream->mall_stream_config.type == SUBVP_PHANTOM) {
-			pipe->stream->use_dynamic_meta = false;
-			pipe->plane_state->flip_immediate = false;
-			if (!resource_build_scaling_params(pipe)) {
-				// Log / remove phantom pipes since failed to build scaling params
-			}
-		}
-	}
-}
-
-static bool dml1_validate(struct dc *dc, struct dc_state *context, bool fast_validate)
-{
-	bool out = false;
-
-	BW_VAL_TRACE_SETUP();
-
-	int vlevel = 0;
-	int pipe_cnt = 0;
-	display_e2e_pipe_params_st *pipes = kzalloc(dc->res_pool->pipe_count * sizeof(display_e2e_pipe_params_st), GFP_KERNEL);
-	struct mall_temp_config mall_temp_config;
-
-	/* To handle Freesync properly, setting FreeSync DML parameters
-	 * to its default state for the first stage of validation
-	 */
-	context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching = false;
-	context->bw_ctx.dml.soc.dram_clock_change_requirement_final = true;
-
-	DC_LOGGER_INIT(dc->ctx->logger);
-
-	/* For fast validation, there are situations where a shallow copy of
-	 * of the dc->current_state is created for the validation. In this case
-	 * we want to save and restore the mall config because we always
-	 * teardown subvp at the beginning of validation (and don't attempt
-	 * to add it back if it's fast validation). If we don't restore the
-	 * subvp config in cases of fast validation + shallow copy of the
-	 * dc->current_state, the dc->current_state will have a partially
-	 * removed subvp state when we did not intend to remove it.
-	 */
-	if (fast_validate) {
-		memset(&mall_temp_config, 0, sizeof(mall_temp_config));
-		dcn32_save_mall_state(dc, context, &mall_temp_config);
-	}
-
-	BW_VAL_TRACE_COUNT();
-
-	DC_FP_START();
-	out = dcn32_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, fast_validate);
-	DC_FP_END();
-
-	if (fast_validate)
-		dcn32_restore_mall_state(dc, context, &mall_temp_config);
-
-	if (pipe_cnt == 0)
-		goto validate_out;
-
-	if (!out)
-		goto validate_fail;
-
-	BW_VAL_TRACE_END_VOLTAGE_LEVEL();
-
-	if (fast_validate) {
-		BW_VAL_TRACE_SKIP(fast);
-		goto validate_out;
-	}
-
-	dc->res_pool->funcs->calculate_wm_and_dlg(dc, context, pipes, pipe_cnt, vlevel);
-
-	dcn32_override_min_req_memclk(dc, context);
-
-	BW_VAL_TRACE_END_WATERMARKS();
-
-	goto validate_out;
-
-validate_fail:
-	DC_LOG_WARNING("Mode Validation Warning: %s failed validation.\n",
-		dml_get_status_message(context->bw_ctx.dml.vba.ValidationStatus[context->bw_ctx.dml.vba.soc.num_states]));
-
-	BW_VAL_TRACE_SKIP(fail);
-	out = false;
-
-validate_out:
-	kfree(pipes);
-
-	BW_VAL_TRACE_FINISH();
-
-	return out;
-}
-
-bool dcn32_validate_bandwidth(struct dc *dc,
-		struct dc_state *context,
-		bool fast_validate)
-{
-	bool out = false;
-
-	if (dc->debug.using_dml2)
-		out = dml2_validate(dc, context, fast_validate);
-	else
-		out = dml1_validate(dc, context, fast_validate);
-	return out;
-}
-
-int dcn32_populate_dml_pipes_from_context(
-	struct dc *dc, struct dc_state *context,
-	display_e2e_pipe_params_st *pipes,
-	bool fast_validate)
-{
-	int i, pipe_cnt;
-	struct resource_context *res_ctx = &context->res_ctx;
-	struct pipe_ctx *pipe = NULL;
-	bool subvp_in_use = false;
-	struct dc_crtc_timing *timing;
-
-	dcn20_populate_dml_pipes_from_context(dc, context, pipes, fast_validate);
-
-	for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) {
-
-		if (!res_ctx->pipe_ctx[i].stream)
-			continue;
-		pipe = &res_ctx->pipe_ctx[i];
-		timing = &pipe->stream->timing;
-
-		pipes[pipe_cnt].pipe.src.gpuvm = true;
-		DC_FP_START();
-		dcn32_zero_pipe_dcc_fraction(pipes, pipe_cnt);
-		DC_FP_END();
-		pipes[pipe_cnt].pipe.dest.vfront_porch = timing->v_front_porch;
-		pipes[pipe_cnt].pipe.dest.odm_combine_policy = dm_odm_combine_policy_dal;
-		pipes[pipe_cnt].pipe.src.gpuvm_min_page_size_kbytes = 256; // according to spreadsheet
-		pipes[pipe_cnt].pipe.src.unbounded_req_mode = false;
-		pipes[pipe_cnt].pipe.scale_ratio_depth.lb_depth = dm_lb_19;
-
-		/* Only populate DML input with subvp info for full updates.
-		 * This is just a workaround -- needs a proper fix.
-		 */
-		if (!fast_validate) {
-			switch (pipe->stream->mall_stream_config.type) {
-			case SUBVP_MAIN:
-				pipes[pipe_cnt].pipe.src.use_mall_for_pstate_change = dm_use_mall_pstate_change_sub_viewport;
-				subvp_in_use = true;
-				break;
-			case SUBVP_PHANTOM:
-				pipes[pipe_cnt].pipe.src.use_mall_for_pstate_change = dm_use_mall_pstate_change_phantom_pipe;
-				pipes[pipe_cnt].pipe.src.use_mall_for_static_screen = dm_use_mall_static_screen_disable;
-				// Disallow unbounded req for SubVP according to DCHUB programming guide
-				pipes[pipe_cnt].pipe.src.unbounded_req_mode = false;
-				break;
-			case SUBVP_NONE:
-				pipes[pipe_cnt].pipe.src.use_mall_for_pstate_change = dm_use_mall_pstate_change_disable;
-				pipes[pipe_cnt].pipe.src.use_mall_for_static_screen = dm_use_mall_static_screen_disable;
-				break;
-			default:
-				break;
-			}
-		}
-
-		pipes[pipe_cnt].dout.dsc_input_bpc = 0;
-		if (pipes[pipe_cnt].dout.dsc_enable) {
-			switch (timing->display_color_depth) {
-			case COLOR_DEPTH_888:
-				pipes[pipe_cnt].dout.dsc_input_bpc = 8;
-				break;
-			case COLOR_DEPTH_101010:
-				pipes[pipe_cnt].dout.dsc_input_bpc = 10;
-				break;
-			case COLOR_DEPTH_121212:
-				pipes[pipe_cnt].dout.dsc_input_bpc = 12;
-				break;
-			default:
-				ASSERT(0);
-				break;
-			}
-		}
-
-
-		pipe_cnt++;
-	}
-
-	/* For DET allocation, we don't want to use DML policy (not optimal for utilizing all
-	 * the DET available for each pipe). Use the DET override input to maintain our driver
-	 * policy.
-	 */
-	dcn32_set_det_allocations(dc, context, pipes);
-
-	// In general cases we want to keep the dram clock change requirement
-	// (prefer configs that support MCLK switch). Only override to false
-	// for SubVP
-	if (context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching || subvp_in_use)
-		context->bw_ctx.dml.soc.dram_clock_change_requirement_final = false;
-	else
-		context->bw_ctx.dml.soc.dram_clock_change_requirement_final = true;
-
-	return pipe_cnt;
-}
-
-static struct dc_cap_funcs cap_funcs = {
-	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap,
-	.get_subvp_en = dcn32_subvp_in_use,
-};
-
-void dcn32_calculate_wm_and_dlg(struct dc *dc, struct dc_state *context,
-				display_e2e_pipe_params_st *pipes,
-				int pipe_cnt,
-				int vlevel)
-{
-    DC_FP_START();
-    dcn32_calculate_wm_and_dlg_fpu(dc, context, pipes, pipe_cnt, vlevel);
-    DC_FP_END();
-}
-
-static void dcn32_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
-{
-	DC_FP_START();
-	dcn32_update_bw_bounding_box_fpu(dc, bw_params);
-	DC_FP_END();
-}
-
-static struct resource_funcs dcn32_res_pool_funcs = {
-	.destroy = dcn32_destroy_resource_pool,
-	.link_enc_create = dcn32_link_encoder_create,
-	.link_enc_create_minimal = NULL,
-	.panel_cntl_create = dcn32_panel_cntl_create,
-	.validate_bandwidth = dcn32_validate_bandwidth,
-	.calculate_wm_and_dlg = dcn32_calculate_wm_and_dlg,
-	.populate_dml_pipes = dcn32_populate_dml_pipes_from_context,
-	.acquire_free_pipe_as_secondary_dpp_pipe = dcn32_acquire_free_pipe_as_secondary_dpp_pipe,
-	.acquire_free_pipe_as_secondary_opp_head = dcn32_acquire_free_pipe_as_secondary_opp_head,
-	.release_pipe = dcn20_release_pipe,
-	.add_stream_to_ctx = dcn30_add_stream_to_ctx,
-	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
-	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
-	.populate_dml_writeback_from_context = dcn30_populate_dml_writeback_from_context,
-	.set_mcif_arb_params = dcn30_set_mcif_arb_params,
-	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
-	.acquire_post_bldn_3dlut = dcn32_acquire_post_bldn_3dlut,
-	.release_post_bldn_3dlut = dcn32_release_post_bldn_3dlut,
-	.update_bw_bounding_box = dcn32_update_bw_bounding_box,
-	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
-	.update_soc_for_wm_a = dcn30_update_soc_for_wm_a,
-	.add_phantom_pipes = dcn32_add_phantom_pipes,
-	.remove_phantom_pipes = dcn32_remove_phantom_pipes,
-	.retain_phantom_pipes = dcn32_retain_phantom_pipes,
-	.save_mall_state = dcn32_save_mall_state,
-	.restore_mall_state = dcn32_restore_mall_state,
-};
-
-static uint32_t read_pipe_fuses(struct dc_context *ctx)
-{
-	uint32_t value = REG_READ(CC_DC_PIPE_DIS);
-	/* DCN32 support max 4 pipes */
-	value = value & 0xf;
-	return value;
-}
-
-
-static bool dcn32_resource_construct(
-	uint8_t num_virtual_links,
-	struct dc *dc,
-	struct dcn32_resource_pool *pool)
-{
-	int i, j;
-	struct dc_context *ctx = dc->ctx;
-	struct irq_service_init_data init_data;
-	struct ddc_service_init_data ddc_init_data = {0};
-	uint32_t pipe_fuses = 0;
-	uint32_t num_pipes  = 4;
-
-#undef REG_STRUCT
-#define REG_STRUCT bios_regs
-	bios_regs_init();
-
-#undef REG_STRUCT
-#define REG_STRUCT clk_src_regs
-	clk_src_regs_init(0, A),
-	clk_src_regs_init(1, B),
-	clk_src_regs_init(2, C),
-	clk_src_regs_init(3, D),
-	clk_src_regs_init(4, E);
-
-#undef REG_STRUCT
-#define REG_STRUCT abm_regs
-	abm_regs_init(0),
-	abm_regs_init(1),
-	abm_regs_init(2),
-	abm_regs_init(3);
-
-#undef REG_STRUCT
-#define REG_STRUCT dccg_regs
-	dccg_regs_init();
-
-	DC_FP_START();
-
-	ctx->dc_bios->regs = &bios_regs;
-
-	pool->base.res_cap = &res_cap_dcn32;
-	/* max number of pipes for ASIC before checking for pipe fuses */
-	num_pipes  = pool->base.res_cap->num_timing_generator;
-	pipe_fuses = read_pipe_fuses(ctx);
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++)
-		if (pipe_fuses & 1 << i)
-			num_pipes--;
-
-	if (pipe_fuses & 1)
-		ASSERT(0); //Unexpected - Pipe 0 should always be fully functional!
-
-	if (pipe_fuses & CC_DC_PIPE_DIS__DC_FULL_DIS_MASK)
-		ASSERT(0); //Entire DCN is harvested!
-
-	/* within dml lib, initial value is hard coded, if ASIC pipe is fused, the
-	 * value will be changed, update max_num_dpp and max_num_otg for dml.
-	 */
-	dcn3_2_ip.max_num_dpp = num_pipes;
-	dcn3_2_ip.max_num_otg = num_pipes;
-
-	pool->base.funcs = &dcn32_res_pool_funcs;
-
-	/*************************************************
-	 *  Resource + asic cap harcoding                *
-	 *************************************************/
-	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
-	pool->base.timing_generator_count = num_pipes;
-	pool->base.pipe_count = num_pipes;
-	pool->base.mpcc_count = num_pipes;
-	dc->caps.max_downscale_ratio = 600;
-	dc->caps.i2c_speed_in_khz = 100;
-	dc->caps.i2c_speed_in_khz_hdcp = 100; /*1.4 w/a applied by default*/
-	/* TODO: Bring max_cursor_size back to 256 after subvp cursor corruption is fixed*/
-	dc->caps.max_cursor_size = 64;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.dmdata_alloc_size = 2048;
-	dc->caps.mall_size_per_mem_channel = 4;
-	dc->caps.mall_size_total = 0;
-	dc->caps.cursor_cache_size = dc->caps.max_cursor_size * dc->caps.max_cursor_size * 8;
-
-	dc->caps.cache_line_size = 64;
-	dc->caps.cache_num_ways = 16;
-
-	/* Calculate the available MALL space */
-	dc->caps.max_cab_allocation_bytes = dcn32_calc_num_avail_chans_for_mall(
-		dc, dc->ctx->dc_bios->vram_info.num_chans) *
-		dc->caps.mall_size_per_mem_channel * 1024 * 1024;
-	dc->caps.mall_size_total = dc->caps.max_cab_allocation_bytes;
-
-	dc->caps.subvp_fw_processing_delay_us = 15;
-	dc->caps.subvp_drr_max_vblank_margin_us = 40;
-	dc->caps.subvp_prefetch_end_to_mall_start_us = 15;
-	dc->caps.subvp_swath_height_margin_lines = 16;
-	dc->caps.subvp_pstate_allow_width_us = 20;
-	dc->caps.subvp_vertical_int_margin_us = 30;
-	dc->caps.subvp_drr_vblank_start_margin_us = 100; // 100us margin
-
-	dc->caps.max_slave_planes = 2;
-	dc->caps.max_slave_yuv_planes = 2;
-	dc->caps.max_slave_rgb_planes = 2;
-	dc->caps.post_blend_color_processing = true;
-	dc->caps.force_dp_tps4_for_cp2520 = true;
-	if (dc->config.forceHBR2CP2520)
-		dc->caps.force_dp_tps4_for_cp2520 = false;
-	dc->caps.dp_hpo = true;
-	dc->caps.dp_hdmi21_pcon_support = true;
-	dc->caps.edp_dsc_support = true;
-	dc->caps.extended_aux_timeout_support = true;
-	dc->caps.dmcub_support = true;
-	dc->caps.seamless_odm = true;
-	dc->caps.max_v_total = (1 << 15) - 1;
-
-	/* Color pipeline capabilities */
-	dc->caps.color.dpp.dcn_arch = 1;
-	dc->caps.color.dpp.input_lut_shared = 0;
-	dc->caps.color.dpp.icsc = 1;
-	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
-	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
-	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
-	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
-	dc->caps.color.dpp.post_csc = 1;
-	dc->caps.color.dpp.gamma_corr = 1;
-	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
-
-	dc->caps.color.dpp.hw_3d_lut = 1;
-	dc->caps.color.dpp.ogam_ram = 0;  // no OGAM in DPP since DCN1
-	// no OGAM ROM on DCN2 and later ASICs
-	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
-	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
-	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
-	dc->caps.color.dpp.ocsc = 0;
-
-	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //4, configurable to be before or after BLND in MPCC
-	dc->caps.color.mpc.ogam_ram = 1;
-	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
-	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
-	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
-	dc->caps.color.mpc.ocsc = 1;
-
-	/* Use pipe context based otg sync logic */
-	dc->config.use_pipe_ctx_sync_logic = true;
-
-	dc->config.dc_mode_clk_limit_support = true;
-	/* read VBIOS LTTPR caps */
-	{
-		if (ctx->dc_bios->funcs->get_lttpr_caps) {
-			enum bp_result bp_query_result;
-			uint8_t is_vbios_lttpr_enable = 0;
-
-			bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
-			dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
-		}
-
-		/* interop bit is implicit */
-		{
-			dc->caps.vbios_lttpr_aware = true;
-		}
-	}
-
-	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
-		dc->debug = debug_defaults_drv;
-
-	// Init the vm_helper
-	if (dc->vm_helper)
-		vm_helper_init(dc->vm_helper, 16);
-
-	/*************************************************
-	 *  Create resources                             *
-	 *************************************************/
-
-	/* Clock Sources for Pixel Clock*/
-	pool->base.clock_sources[DCN32_CLK_SRC_PLL0] =
-			dcn32_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL0,
-				&clk_src_regs[0], false);
-	pool->base.clock_sources[DCN32_CLK_SRC_PLL1] =
-			dcn32_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL1,
-				&clk_src_regs[1], false);
-	pool->base.clock_sources[DCN32_CLK_SRC_PLL2] =
-			dcn32_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL2,
-				&clk_src_regs[2], false);
-	pool->base.clock_sources[DCN32_CLK_SRC_PLL3] =
-			dcn32_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL3,
-				&clk_src_regs[3], false);
-	pool->base.clock_sources[DCN32_CLK_SRC_PLL4] =
-			dcn32_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL4,
-				&clk_src_regs[4], false);
-
-	pool->base.clk_src_count = DCN32_CLK_SRC_TOTAL;
-
-	/* todo: not reuse phy_pll registers */
-	pool->base.dp_clock_source =
-			dcn32_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_ID_DP_DTO,
-				&clk_src_regs[0], true);
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-
-	/* DCCG */
-	pool->base.dccg = dccg32_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
-	if (pool->base.dccg == NULL) {
-		dm_error("DC: failed to create dccg!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	/* DML */
-	dml_init_instance(&dc->dml, &dcn3_2_soc, &dcn3_2_ip, DML_PROJECT_DCN32);
-
-	/* IRQ Service */
-	init_data.ctx = dc->ctx;
-	pool->base.irqs = dal_irq_service_dcn32_create(&init_data);
-	if (!pool->base.irqs)
-		goto create_fail;
-
-	/* HUBBUB */
-	pool->base.hubbub = dcn32_hubbub_create(ctx);
-	if (pool->base.hubbub == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create hubbub!\n");
-		goto create_fail;
-	}
-
-	/* HUBPs, DPPs, OPPs, TGs, ABMs */
-	for (i = 0, j = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-
-		/* if pipe is disabled, skip instance of HW pipe,
-		 * i.e, skip ASIC register instance
-		 */
-		if (pipe_fuses & 1 << i)
-			continue;
-
-		/* HUBPs */
-		pool->base.hubps[j] = dcn32_hubp_create(ctx, i);
-		if (pool->base.hubps[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create hubps!\n");
-			goto create_fail;
-		}
-
-		/* DPPs */
-		pool->base.dpps[j] = dcn32_dpp_create(ctx, i);
-		if (pool->base.dpps[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create dpps!\n");
-			goto create_fail;
-		}
-
-		/* OPPs */
-		pool->base.opps[j] = dcn32_opp_create(ctx, i);
-		if (pool->base.opps[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create output pixel processor!\n");
-			goto create_fail;
-		}
-
-		/* TGs */
-		pool->base.timing_generators[j] = dcn32_timing_generator_create(
-				ctx, i);
-		if (pool->base.timing_generators[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create tg!\n");
-			goto create_fail;
-		}
-
-		/* ABMs */
-		pool->base.multiple_abms[j] = dmub_abm_create(ctx,
-				&abm_regs[i],
-				&abm_shift,
-				&abm_mask);
-		if (pool->base.multiple_abms[j] == NULL) {
-			dm_error("DC: failed to create abm for pipe %d!\n", i);
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-
-		/* index for resource pool arrays for next valid pipe */
-		j++;
-	}
-
-	/* PSR */
-	pool->base.psr = dmub_psr_create(ctx);
-	if (pool->base.psr == NULL) {
-		dm_error("DC: failed to create psr obj!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	/* MPCCs */
-	pool->base.mpc = dcn32_mpc_create(ctx, pool->base.res_cap->num_timing_generator, pool->base.res_cap->num_mpc_3dlut);
-	if (pool->base.mpc == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mpc!\n");
-		goto create_fail;
-	}
-
-	/* DSCs */
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		pool->base.dscs[i] = dcn32_dsc_create(ctx, i);
-		if (pool->base.dscs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create display stream compressor %d!\n", i);
-			goto create_fail;
-		}
-	}
-
-	/* DWB */
-	if (!dcn32_dwbc_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create dwbc!\n");
-		goto create_fail;
-	}
-
-	/* MMHUBBUB */
-	if (!dcn32_mmhubbub_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mcif_wb!\n");
-		goto create_fail;
-	}
-
-	/* AUX and I2C */
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		pool->base.engines[i] = dcn32_aux_engine_create(ctx, i);
-		if (pool->base.engines[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create aux engine!!\n");
-			goto create_fail;
-		}
-		pool->base.hw_i2cs[i] = dcn32_i2c_hw_create(ctx, i);
-		if (pool->base.hw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create hw i2c!!\n");
-			goto create_fail;
-		}
-		pool->base.sw_i2cs[i] = NULL;
-	}
-
-	/* Audio, HWSeq, Stream Encoders including HPO and virtual, MPC 3D LUTs */
-	if (!resource_construct(num_virtual_links, dc, &pool->base,
-			&res_create_funcs))
-		goto create_fail;
-
-	/* HW Sequencer init functions and Plane caps */
-	dcn32_hw_sequencer_init_functions(dc);
-
-	dc->caps.max_planes =  pool->base.pipe_count;
-
-	for (i = 0; i < dc->caps.max_planes; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	dc->cap_funcs = cap_funcs;
-
-	if (dc->ctx->dc_bios->fw_info.oem_i2c_present) {
-		ddc_init_data.ctx = dc->ctx;
-		ddc_init_data.link = NULL;
-		ddc_init_data.id.id = dc->ctx->dc_bios->fw_info.oem_i2c_obj_id;
-		ddc_init_data.id.enum_id = 0;
-		ddc_init_data.id.type = OBJECT_TYPE_GENERIC;
-		pool->base.oem_device = dc->link_srv->create_ddc_service(&ddc_init_data);
-	} else {
-		pool->base.oem_device = NULL;
-	}
-
-	dc->dml2_options.dcn_pipe_count = pool->base.pipe_count;
-	dc->dml2_options.use_native_pstate_optimization = false;
-	dc->dml2_options.use_native_soc_bb_construction = true;
-	dc->dml2_options.minimize_dispclk_using_odm = true;
-
-	dc->dml2_options.callbacks.dc = dc;
-	dc->dml2_options.callbacks.build_scaling_params = &resource_build_scaling_params;
-	dc->dml2_options.callbacks.can_support_mclk_switch_using_fw_based_vblank_stretch = &dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch;
-	dc->dml2_options.callbacks.acquire_secondary_pipe_for_mpc_odm = &dc_resource_acquire_secondary_pipe_for_mpc_odm_legacy;
-	dc->dml2_options.callbacks.update_pipes_for_stream_with_slice_count = &resource_update_pipes_for_stream_with_slice_count;
-	dc->dml2_options.callbacks.update_pipes_for_plane_with_slice_count = &resource_update_pipes_for_plane_with_slice_count;
-	dc->dml2_options.callbacks.get_mpc_slice_index = &resource_get_mpc_slice_index;
-	dc->dml2_options.callbacks.get_odm_slice_index = &resource_get_odm_slice_index;
-	dc->dml2_options.callbacks.get_opp_head = &resource_get_opp_head;
-
-	dc->dml2_options.svp_pstate.callbacks.dc = dc;
-	dc->dml2_options.svp_pstate.callbacks.add_plane_to_context = &dc_add_plane_to_context;
-	dc->dml2_options.svp_pstate.callbacks.add_stream_to_ctx = &dc_add_stream_to_ctx;
-	dc->dml2_options.svp_pstate.callbacks.build_scaling_params = &resource_build_scaling_params;
-	dc->dml2_options.svp_pstate.callbacks.create_plane = &dc_create_plane_state;
-	dc->dml2_options.svp_pstate.callbacks.remove_plane_from_context = &dc_remove_plane_from_context;
-	dc->dml2_options.svp_pstate.callbacks.remove_stream_from_ctx = &dc_remove_stream_from_ctx;
-	dc->dml2_options.svp_pstate.callbacks.create_stream_for_sink = &dc_create_stream_for_sink;
-	dc->dml2_options.svp_pstate.callbacks.plane_state_release = &dc_plane_state_release;
-	dc->dml2_options.svp_pstate.callbacks.stream_release = &dc_stream_release;
-	dc->dml2_options.svp_pstate.callbacks.release_dsc = &dcn20_release_dsc;
-
-	dc->dml2_options.svp_pstate.subvp_fw_processing_delay_us = dc->caps.subvp_fw_processing_delay_us;
-	dc->dml2_options.svp_pstate.subvp_prefetch_end_to_mall_start_us = dc->caps.subvp_prefetch_end_to_mall_start_us;
-	dc->dml2_options.svp_pstate.subvp_pstate_allow_width_us = dc->caps.subvp_pstate_allow_width_us;
-	dc->dml2_options.svp_pstate.subvp_swath_height_margin_lines = dc->caps.subvp_swath_height_margin_lines;
-
-	dc->dml2_options.svp_pstate.force_disable_subvp = dc->debug.force_disable_subvp;
-	dc->dml2_options.svp_pstate.force_enable_subvp = dc->debug.force_subvp_mclk_switch;
-
-	dc->dml2_options.mall_cfg.cache_line_size_bytes = dc->caps.cache_line_size;
-	dc->dml2_options.mall_cfg.cache_num_ways = dc->caps.cache_num_ways;
-	dc->dml2_options.mall_cfg.max_cab_allocation_bytes = dc->caps.max_cab_allocation_bytes;
-	dc->dml2_options.mall_cfg.mblk_height_4bpe_pixels = DCN3_2_MBLK_HEIGHT_4BPE;
-	dc->dml2_options.mall_cfg.mblk_height_8bpe_pixels = DCN3_2_MBLK_HEIGHT_8BPE;
-	dc->dml2_options.mall_cfg.mblk_size_bytes = DCN3_2_MALL_MBLK_SIZE_BYTES;
-	dc->dml2_options.mall_cfg.mblk_width_pixels = DCN3_2_MBLK_WIDTH;
-
-	dc->dml2_options.max_segments_per_hubp = 18;
-	dc->dml2_options.det_segment_size = DCN3_2_DET_SEG_SIZE;
-	dc->dml2_options.map_dc_pipes_with_callbacks = true;
-
-	if (ASICREV_IS_GC_11_0_3(dc->ctx->asic_id.hw_internal_rev) && (dc->config.sdpif_request_limit_words_per_umc == 0))
-		dc->config.sdpif_request_limit_words_per_umc = 16;
-
-	DC_FP_END();
-
-	return true;
-
-create_fail:
-
-	DC_FP_END();
-
-	dcn32_resource_destruct(pool);
-
-	return false;
-}
-
-struct resource_pool *dcn32_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc)
-{
-	struct dcn32_resource_pool *pool =
-		kzalloc(sizeof(struct dcn32_resource_pool), GFP_KERNEL);
-
-	if (!pool)
-		return NULL;
-
-	if (dcn32_resource_construct(init_data->num_virtual_links, dc, pool))
-		return &pool->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(pool);
-	return NULL;
-}
-
-/*
- * Find the most optimal free pipe from res_ctx, which could be used as a
- * secondary dpp pipe for input opp head pipe.
- *
- * a free pipe - a pipe in input res_ctx not yet used for any streams or
- * planes.
- * secondary dpp pipe - a pipe gets inserted to a head OPP pipe's MPC blending
- * tree. This is typical used for rendering MPO planes or additional offset
- * areas in MPCC combine.
- *
- * Hardware Transition Minimization Algorithm for Finding a Secondary DPP Pipe
- * -------------------------------------------------------------------------
- *
- * PROBLEM:
- *
- * 1. There is a hardware limitation that a secondary DPP pipe cannot be
- * transferred from one MPC blending tree to the other in a single frame.
- * Otherwise it could cause glitches on the screen.
- *
- * For instance, we cannot transition from state 1 to state 2 in one frame. This
- * is because PIPE1 is transferred from PIPE0's MPC blending tree over to
- * PIPE2's MPC blending tree, which is not supported by hardware.
- * To support this transition we need to first remove PIPE1 from PIPE0's MPC
- * blending tree in one frame and then insert PIPE1 to PIPE2's MPC blending tree
- * in the next frame. This is not optimal as it will delay the flip for two
- * frames.
- *
- *	State 1:
- *	PIPE0 -- secondary DPP pipe --> (PIPE1)
- *	PIPE2 -- secondary DPP pipe --> NONE
- *
- *	State 2:
- *	PIPE0 -- secondary DPP pipe --> NONE
- *	PIPE2 -- secondary DPP pipe --> (PIPE1)
- *
- * 2. We want to in general minimize the unnecessary changes in pipe topology.
- * If a pipe is already added in current blending tree and there are no changes
- * to plane topology, we don't want to swap it with another free pipe
- * unnecessarily in every update. Powering up and down a pipe would require a
- * full update which delays the flip for 1 frame. If we use the original pipe
- * we don't have to toggle its power. So we can flip faster.
- */
-static int find_optimal_free_pipe_as_secondary_dpp_pipe(
-		const struct resource_context *cur_res_ctx,
-		struct resource_context *new_res_ctx,
-		const struct resource_pool *pool,
-		const struct pipe_ctx *new_opp_head)
-{
-	const struct pipe_ctx *cur_opp_head;
-	int free_pipe_idx;
-
-	cur_opp_head = &cur_res_ctx->pipe_ctx[new_opp_head->pipe_idx];
-	free_pipe_idx = resource_find_free_pipe_used_in_cur_mpc_blending_tree(
-			cur_res_ctx, new_res_ctx, cur_opp_head);
-
-	/* Up until here if we have not found a free secondary pipe, we will
-	 * need to wait for at least one frame to complete the transition
-	 * sequence.
-	 */
-	if (free_pipe_idx == FREE_PIPE_INDEX_NOT_FOUND)
-		free_pipe_idx = recource_find_free_pipe_not_used_in_cur_res_ctx(
-				cur_res_ctx, new_res_ctx, pool);
-
-	/* Up until here if we have not found a free secondary pipe, we will
-	 * need to wait for at least two frames to complete the transition
-	 * sequence. It really doesn't matter which pipe we decide take from
-	 * current enabled pipes. It won't save our frame time when we swap only
-	 * one pipe or more pipes.
-	 */
-	if (free_pipe_idx == FREE_PIPE_INDEX_NOT_FOUND)
-		free_pipe_idx = resource_find_free_pipe_used_as_cur_sec_dpp_in_mpcc_combine(
-				cur_res_ctx, new_res_ctx, pool);
-
-	if (free_pipe_idx == FREE_PIPE_INDEX_NOT_FOUND)
-		free_pipe_idx = resource_find_any_free_pipe(new_res_ctx, pool);
-
-	return free_pipe_idx;
-}
-
-static struct pipe_ctx *find_idle_secondary_pipe_check_mpo(
-		struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		const struct pipe_ctx *primary_pipe)
-{
-	int i;
-	struct pipe_ctx *secondary_pipe = NULL;
-	struct pipe_ctx *next_odm_mpo_pipe = NULL;
-	int primary_index, preferred_pipe_idx;
-	struct pipe_ctx *old_primary_pipe = NULL;
-
-	/*
-	 * Modified from find_idle_secondary_pipe
-	 * With windowed MPO and ODM, we want to avoid the case where we want a
-	 *  free pipe for the left side but the free pipe is being used on the
-	 *  right side.
-	 * Add check on current_state if the primary_pipe is the left side,
-	 *  to check the right side ( primary_pipe->next_odm_pipe ) to see if
-	 *  it is using a pipe for MPO ( primary_pipe->next_odm_pipe->bottom_pipe )
-	 * - If so, then don't use this pipe
-	 * EXCEPTION - 3 plane ( 2 MPO plane ) case
-	 * - in this case, the primary pipe has already gotten a free pipe for the
-	 *  MPO window in the left
-	 * - when it tries to get a free pipe for the MPO window on the right,
-	 *  it will see that it is already assigned to the right side
-	 *  ( primary_pipe->next_odm_pipe ).  But in this case, we want this
-	 *  free pipe, since it will be for the right side.  So add an
-	 *  additional condition, that skipping the free pipe on the right only
-	 *  applies if the primary pipe has no bottom pipe currently assigned
-	 */
-	if (primary_pipe) {
-		primary_index = primary_pipe->pipe_idx;
-		old_primary_pipe = &primary_pipe->stream->ctx->dc->current_state->res_ctx.pipe_ctx[primary_index];
-		if ((old_primary_pipe->next_odm_pipe) && (old_primary_pipe->next_odm_pipe->bottom_pipe)
-			&& (!primary_pipe->bottom_pipe))
-			next_odm_mpo_pipe = old_primary_pipe->next_odm_pipe->bottom_pipe;
-
-		preferred_pipe_idx = (pool->pipe_count - 1) - primary_pipe->pipe_idx;
-		if ((res_ctx->pipe_ctx[preferred_pipe_idx].stream == NULL) &&
-			!(next_odm_mpo_pipe && next_odm_mpo_pipe->pipe_idx == preferred_pipe_idx)) {
-			secondary_pipe = &res_ctx->pipe_ctx[preferred_pipe_idx];
-			secondary_pipe->pipe_idx = preferred_pipe_idx;
-		}
-	}
-
-	/*
-	 * search backwards for the second pipe to keep pipe
-	 * assignment more consistent
-	 */
-	if (!secondary_pipe)
-		for (i = pool->pipe_count - 1; i >= 0; i--) {
-			if ((res_ctx->pipe_ctx[i].stream == NULL) &&
-				!(next_odm_mpo_pipe && next_odm_mpo_pipe->pipe_idx == i)) {
-				secondary_pipe = &res_ctx->pipe_ctx[i];
-				secondary_pipe->pipe_idx = i;
-				break;
-			}
-		}
-
-	return secondary_pipe;
-}
-
-static struct pipe_ctx *dcn32_acquire_idle_pipe_for_head_pipe_in_layer(
-		struct dc_state *state,
-		const struct resource_pool *pool,
-		struct dc_stream_state *stream,
-		const struct pipe_ctx *head_pipe)
-{
-	struct resource_context *res_ctx = &state->res_ctx;
-	struct pipe_ctx *idle_pipe, *pipe;
-	struct resource_context *old_ctx = &stream->ctx->dc->current_state->res_ctx;
-	int head_index;
-
-	if (!head_pipe)
-		ASSERT(0);
-
-	/*
-	 * Modified from dcn20_acquire_idle_pipe_for_layer
-	 * Check if head_pipe in old_context already has bottom_pipe allocated.
-	 * - If so, check if that pipe is available in the current context.
-	 * --  If so, reuse pipe from old_context
-	 */
-	head_index = head_pipe->pipe_idx;
-	pipe = &old_ctx->pipe_ctx[head_index];
-	if (pipe->bottom_pipe && res_ctx->pipe_ctx[pipe->bottom_pipe->pipe_idx].stream == NULL) {
-		idle_pipe = &res_ctx->pipe_ctx[pipe->bottom_pipe->pipe_idx];
-		idle_pipe->pipe_idx = pipe->bottom_pipe->pipe_idx;
-	} else {
-		idle_pipe = find_idle_secondary_pipe_check_mpo(res_ctx, pool, head_pipe);
-		if (!idle_pipe)
-			return NULL;
-	}
-
-	idle_pipe->stream = head_pipe->stream;
-	idle_pipe->stream_res.tg = head_pipe->stream_res.tg;
-	idle_pipe->stream_res.opp = head_pipe->stream_res.opp;
-
-	idle_pipe->plane_res.hubp = pool->hubps[idle_pipe->pipe_idx];
-	idle_pipe->plane_res.ipp = pool->ipps[idle_pipe->pipe_idx];
-	idle_pipe->plane_res.dpp = pool->dpps[idle_pipe->pipe_idx];
-	idle_pipe->plane_res.mpcc_inst = pool->dpps[idle_pipe->pipe_idx]->inst;
-
-	return idle_pipe;
-}
-
-static int find_optimal_free_pipe_as_secondary_opp_head(
-		const struct resource_context *cur_res_ctx,
-		struct resource_context *new_res_ctx,
-		const struct resource_pool *pool,
-		const struct pipe_ctx *new_otg_master)
-{
-	const struct pipe_ctx *cur_otg_master;
-	int free_pipe_idx;
-
-	cur_otg_master =  &cur_res_ctx->pipe_ctx[new_otg_master->pipe_idx];
-	free_pipe_idx = resource_find_free_pipe_used_as_sec_opp_head_by_cur_otg_master(
-			cur_res_ctx, new_res_ctx, cur_otg_master);
-
-	/* Up until here if we have not found a free secondary pipe, we will
-	 * need to wait for at least one frame to complete the transition
-	 * sequence.
-	 */
-	if (free_pipe_idx == FREE_PIPE_INDEX_NOT_FOUND)
-		free_pipe_idx = recource_find_free_pipe_not_used_in_cur_res_ctx(
-				cur_res_ctx, new_res_ctx, pool);
-
-	if (free_pipe_idx == FREE_PIPE_INDEX_NOT_FOUND)
-		free_pipe_idx = resource_find_any_free_pipe(new_res_ctx, pool);
-
-	return free_pipe_idx;
-}
-
-struct pipe_ctx *dcn32_acquire_free_pipe_as_secondary_dpp_pipe(
-		const struct dc_state *cur_ctx,
-		struct dc_state *new_ctx,
-		const struct resource_pool *pool,
-		const struct pipe_ctx *opp_head_pipe)
-{
-
-	int free_pipe_idx;
-	struct pipe_ctx *free_pipe;
-
-	if (!opp_head_pipe->stream->ctx->dc->config.enable_windowed_mpo_odm)
-		return dcn32_acquire_idle_pipe_for_head_pipe_in_layer(
-				new_ctx, pool, opp_head_pipe->stream, opp_head_pipe);
-
-	free_pipe_idx = find_optimal_free_pipe_as_secondary_dpp_pipe(
-					&cur_ctx->res_ctx, &new_ctx->res_ctx,
-					pool, opp_head_pipe);
-	if (free_pipe_idx >= 0) {
-		free_pipe = &new_ctx->res_ctx.pipe_ctx[free_pipe_idx];
-		free_pipe->pipe_idx = free_pipe_idx;
-		free_pipe->stream = opp_head_pipe->stream;
-		free_pipe->stream_res.tg = opp_head_pipe->stream_res.tg;
-		free_pipe->stream_res.opp = opp_head_pipe->stream_res.opp;
-
-		free_pipe->plane_res.hubp = pool->hubps[free_pipe->pipe_idx];
-		free_pipe->plane_res.ipp = pool->ipps[free_pipe->pipe_idx];
-		free_pipe->plane_res.dpp = pool->dpps[free_pipe->pipe_idx];
-		free_pipe->plane_res.mpcc_inst =
-				pool->dpps[free_pipe->pipe_idx]->inst;
-	} else {
-		ASSERT(opp_head_pipe);
-		free_pipe = NULL;
-	}
-
-	return free_pipe;
-}
-
-struct pipe_ctx *dcn32_acquire_free_pipe_as_secondary_opp_head(
-		const struct dc_state *cur_ctx,
-		struct dc_state *new_ctx,
-		const struct resource_pool *pool,
-		const struct pipe_ctx *otg_master)
-{
-	int free_pipe_idx = find_optimal_free_pipe_as_secondary_opp_head(
-			&cur_ctx->res_ctx, &new_ctx->res_ctx,
-			pool, otg_master);
-	struct pipe_ctx *free_pipe;
-
-	if (free_pipe_idx >= 0) {
-		free_pipe = &new_ctx->res_ctx.pipe_ctx[free_pipe_idx];
-		free_pipe->pipe_idx = free_pipe_idx;
-		free_pipe->stream = otg_master->stream;
-		free_pipe->stream_res.tg = otg_master->stream_res.tg;
-		free_pipe->stream_res.dsc = NULL;
-		free_pipe->stream_res.opp = pool->opps[free_pipe_idx];
-		free_pipe->plane_res.mi = pool->mis[free_pipe_idx];
-		free_pipe->plane_res.hubp = pool->hubps[free_pipe_idx];
-		free_pipe->plane_res.ipp = pool->ipps[free_pipe_idx];
-		free_pipe->plane_res.xfm = pool->transforms[free_pipe_idx];
-		free_pipe->plane_res.dpp = pool->dpps[free_pipe_idx];
-		free_pipe->plane_res.mpcc_inst = pool->dpps[free_pipe_idx]->inst;
-		if (free_pipe->stream->timing.flags.DSC == 1) {
-			dcn20_acquire_dsc(free_pipe->stream->ctx->dc,
-					&new_ctx->res_ctx,
-					&free_pipe->stream_res.dsc,
-					free_pipe_idx);
-			ASSERT(free_pipe->stream_res.dsc);
-			if (free_pipe->stream_res.dsc == NULL) {
-				memset(free_pipe, 0, sizeof(*free_pipe));
-				free_pipe = NULL;
-			}
-		}
-	} else {
-		ASSERT(otg_master);
-		free_pipe = NULL;
-	}
-
-	return free_pipe;
-}
-
-unsigned int dcn32_calc_num_avail_chans_for_mall(struct dc *dc, int num_chans)
-{
-	/*
-	 * DCN32 and DCN321 SKUs may have different sizes for MALL
-	 *  but we may not be able to access all the MALL space.
-	 *  If the num_chans is power of 2, then we can access all
-	 *  of the available MALL space.  Otherwise, we can only
-	 *  access:
-	 *
-	 *  max_cab_size_in_bytes = total_cache_size_in_bytes *
-	 *    ((2^floor(log2(num_chans)))/num_chans)
-	 *
-	 * Calculating the MALL sizes for all available SKUs, we
-	 *  have come up with the follow simplified check.
-	 * - we have max_chans which provides the max MALL size.
-	 *  Each chans supports 4MB of MALL so:
-	 *
-	 *  total_cache_size_in_bytes = max_chans * 4 MB
-	 *
-	 * - we have avail_chans which shows the number of channels
-	 *  we can use if we can't access the entire MALL space.
-	 *  It is generally half of max_chans
-	 * - so we use the following checks:
-	 *
-	 *   if (num_chans == max_chans), return max_chans
-	 *   if (num_chans < max_chans), return avail_chans
-	 *
-	 * - exception is GC_11_0_0 where we can't access max_chans,
-	 *  so we define max_avail_chans as the maximum available
-	 *  MALL space
-	 *
-	 */
-	int gc_11_0_0_max_chans = 48;
-	int gc_11_0_0_max_avail_chans = 32;
-	int gc_11_0_0_avail_chans = 16;
-	int gc_11_0_3_max_chans = 16;
-	int gc_11_0_3_avail_chans = 8;
-	int gc_11_0_2_max_chans = 8;
-	int gc_11_0_2_avail_chans = 4;
-
-	if (ASICREV_IS_GC_11_0_0(dc->ctx->asic_id.hw_internal_rev)) {
-		return (num_chans == gc_11_0_0_max_chans) ?
-			gc_11_0_0_max_avail_chans : gc_11_0_0_avail_chans;
-	} else if (ASICREV_IS_GC_11_0_2(dc->ctx->asic_id.hw_internal_rev)) {
-		return (num_chans == gc_11_0_2_max_chans) ?
-			gc_11_0_2_max_chans : gc_11_0_2_avail_chans;
-	} else { // if (ASICREV_IS_GC_11_0_3(dc->ctx->asic_id.hw_internal_rev)) {
-		return (num_chans == gc_11_0_3_max_chans) ?
-			gc_11_0_3_max_chans : gc_11_0_3_avail_chans;
-	}
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.h
deleted file mode 100644
index b931008114c9..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.h
+++ /dev/null
@@ -1,1263 +0,0 @@
-/*
- * Copyright 2020 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#ifndef _DCN32_RESOURCE_H_
-#define _DCN32_RESOURCE_H_
-
-#include "core_types.h"
-
-#define DCN3_2_DEFAULT_DET_SIZE 256
-#define DCN3_2_MAX_DET_SIZE 1152
-#define DCN3_2_MIN_DET_SIZE 128
-#define DCN3_2_MIN_COMPBUF_SIZE_KB 128
-#define DCN3_2_DET_SEG_SIZE 64
-#define DCN3_2_MALL_MBLK_SIZE_BYTES 65536 // 64 * 1024
-#define DCN3_2_MBLK_WIDTH 128
-#define DCN3_2_MBLK_HEIGHT_4BPE 128
-#define DCN3_2_MBLK_HEIGHT_8BPE 64
-#define DCN3_2_DCFCLK_DS_INIT_KHZ 10000 // Choose 10Mhz for init DCFCLK DS freq
-#define SUBVP_HIGH_REFRESH_LIST_LEN 4
-#define DCN3_2_MAX_SUBVP_PIXEL_RATE_MHZ 1800
-#define DCN3_2_VMIN_DISPCLK_HZ 717000000
-
-#define TO_DCN32_RES_POOL(pool)\
-	container_of(pool, struct dcn32_resource_pool, base)
-
-extern struct _vcs_dpi_ip_params_st dcn3_2_ip;
-extern struct _vcs_dpi_soc_bounding_box_st dcn3_2_soc;
-
-struct subvp_high_refresh_list {
-	int min_refresh;
-	int max_refresh;
-	struct resolution {
-		int width;
-		int height;
-	} res[SUBVP_HIGH_REFRESH_LIST_LEN];
-};
-
-struct dcn32_resource_pool {
-	struct resource_pool base;
-};
-
-struct resource_pool *dcn32_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc);
-
-struct panel_cntl *dcn32_panel_cntl_create(
-		const struct panel_cntl_init_data *init_data);
-
-bool dcn32_acquire_post_bldn_3dlut(
-		struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		int mpcc_id,
-		struct dc_3dlut **lut,
-		struct dc_transfer_func **shaper);
-
-bool dcn32_release_post_bldn_3dlut(
-		struct resource_context *res_ctx,
-		const struct resource_pool *pool,
-		struct dc_3dlut **lut,
-		struct dc_transfer_func **shaper);
-
-bool dcn32_remove_phantom_pipes(struct dc *dc,
-		struct dc_state *context, bool fast_update);
-
-void dcn32_retain_phantom_pipes(struct dc *dc,
-		struct dc_state *context);
-
-void dcn32_add_phantom_pipes(struct dc *dc,
-		struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		unsigned int pipe_cnt,
-		unsigned int index);
-
-bool dcn32_validate_bandwidth(struct dc *dc,
-		struct dc_state *context,
-		bool fast_validate);
-
-int dcn32_populate_dml_pipes_from_context(
-	struct dc *dc, struct dc_state *context,
-	display_e2e_pipe_params_st *pipes,
-	bool fast_validate);
-
-void dcn32_calculate_wm_and_dlg(
-		struct dc *dc, struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		int pipe_cnt,
-		int vlevel);
-
-uint32_t dcn32_helper_mall_bytes_to_ways(
-		struct dc *dc,
-		uint32_t total_size_in_mall_bytes);
-
-uint32_t dcn32_helper_calculate_mall_bytes_for_cursor(
-		struct dc *dc,
-		struct pipe_ctx *pipe_ctx,
-		bool ignore_cursor_buf);
-
-uint32_t dcn32_helper_calculate_num_ways_for_subvp(
-		struct dc *dc,
-		struct dc_state *context);
-
-void dcn32_merge_pipes_for_subvp(struct dc *dc,
-		struct dc_state *context);
-
-bool dcn32_all_pipes_have_stream_and_plane(struct dc *dc,
-		struct dc_state *context);
-
-bool dcn32_subvp_in_use(struct dc *dc,
-		struct dc_state *context);
-
-bool dcn32_mpo_in_use(struct dc_state *context);
-
-bool dcn32_any_surfaces_rotated(struct dc *dc, struct dc_state *context);
-bool dcn32_is_center_timing(struct pipe_ctx *pipe);
-bool dcn32_is_psr_capable(struct pipe_ctx *pipe);
-
-struct pipe_ctx *dcn32_acquire_free_pipe_as_secondary_dpp_pipe(
-		const struct dc_state *cur_ctx,
-		struct dc_state *new_ctx,
-		const struct resource_pool *pool,
-		const struct pipe_ctx *opp_head_pipe);
-
-struct pipe_ctx *dcn32_acquire_free_pipe_as_secondary_opp_head(
-		const struct dc_state *cur_ctx,
-		struct dc_state *new_ctx,
-		const struct resource_pool *pool,
-		const struct pipe_ctx *otg_master);
-
-void dcn32_release_pipe(struct dc_state *context,
-			struct pipe_ctx *pipe,
-			const struct resource_pool *pool);
-
-void dcn32_determine_det_override(struct dc *dc,
-		struct dc_state *context,
-		display_e2e_pipe_params_st *pipes);
-
-void dcn32_set_det_allocations(struct dc *dc, struct dc_state *context,
-	display_e2e_pipe_params_st *pipes);
-
-void dcn32_save_mall_state(struct dc *dc,
-		struct dc_state *context,
-		struct mall_temp_config *temp_config);
-
-void dcn32_restore_mall_state(struct dc *dc,
-		struct dc_state *context,
-		struct mall_temp_config *temp_config);
-
-struct dc_stream_state *dcn32_can_support_mclk_switch_using_fw_based_vblank_stretch(struct dc *dc, const struct dc_state *context);
-
-bool dcn32_allow_subvp_with_active_margin(struct pipe_ctx *pipe);
-
-bool dcn32_allow_subvp_high_refresh_rate(struct dc *dc, struct dc_state *context, struct pipe_ctx *pipe);
-
-unsigned int dcn32_calc_num_avail_chans_for_mall(struct dc *dc, int num_chans);
-
-double dcn32_determine_max_vratio_prefetch(struct dc *dc, struct dc_state *context);
-
-bool dcn32_check_native_scaling_for_res(struct pipe_ctx *pipe, unsigned int width, unsigned int height);
-
-bool dcn32_subvp_drr_admissable(struct dc *dc, struct dc_state *context);
-
-bool dcn32_subvp_vblank_admissable(struct dc *dc, struct dc_state *context, int vlevel);
-
-/* definitions for run time init of reg offsets */
-
-/* CLK SRC */
-#define CS_COMMON_REG_LIST_DCN3_0_RI(index, pllid)                             \
-  SRI_ARR_ALPHABET(PIXCLK_RESYNC_CNTL, PHYPLL, index, pllid),                  \
-      SRII_ARR_2(PHASE, DP_DTO, 0, index),                                     \
-      SRII_ARR_2(PHASE, DP_DTO, 1, index),                                     \
-      SRII_ARR_2(PHASE, DP_DTO, 2, index),                                     \
-      SRII_ARR_2(PHASE, DP_DTO, 3, index),                                     \
-      SRII_ARR_2(MODULO, DP_DTO, 0, index),                                    \
-      SRII_ARR_2(MODULO, DP_DTO, 1, index),                                    \
-      SRII_ARR_2(MODULO, DP_DTO, 2, index),                                    \
-      SRII_ARR_2(MODULO, DP_DTO, 3, index),                                    \
-      SRII_ARR_2(PIXEL_RATE_CNTL, OTG, 0, index),                              \
-      SRII_ARR_2(PIXEL_RATE_CNTL, OTG, 1, index),                              \
-      SRII_ARR_2(PIXEL_RATE_CNTL, OTG, 2, index),                              \
-      SRII_ARR_2(PIXEL_RATE_CNTL, OTG, 3, index)
-
-/* ABM */
-#define ABM_DCN32_REG_LIST_RI(id)                                              \
-  SRI_ARR(DC_ABM1_HG_SAMPLE_RATE, ABM, id),                                    \
-      SRI_ARR(DC_ABM1_LS_SAMPLE_RATE, ABM, id),                                \
-      SRI_ARR(BL1_PWM_BL_UPDATE_SAMPLE_RATE, ABM, id),                         \
-      SRI_ARR(DC_ABM1_HG_MISC_CTRL, ABM, id),                                  \
-      SRI_ARR(DC_ABM1_IPCSC_COEFF_SEL, ABM, id),                               \
-      SRI_ARR(BL1_PWM_CURRENT_ABM_LEVEL, ABM, id),                             \
-      SRI_ARR(BL1_PWM_TARGET_ABM_LEVEL, ABM, id),                              \
-      SRI_ARR(BL1_PWM_USER_LEVEL, ABM, id),                                    \
-      SRI_ARR(DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES, ABM, id),                  \
-      SRI_ARR(DC_ABM1_HGLS_REG_READ_PROGRESS, ABM, id),                        \
-      SRI_ARR(DC_ABM1_ACE_OFFSET_SLOPE_0, ABM, id),                            \
-      SRI_ARR(DC_ABM1_ACE_THRES_12, ABM, id), NBIO_SR_ARR(BIOS_SCRATCH_2, id)
-
-/* Audio */
-#define AUD_COMMON_REG_LIST_RI(id)                                             \
-  SRI_ARR(AZALIA_F0_CODEC_ENDPOINT_INDEX, AZF0ENDPOINT, id),                   \
-      SRI_ARR(AZALIA_F0_CODEC_ENDPOINT_DATA, AZF0ENDPOINT, id),                \
-      SR_ARR(AZALIA_F0_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS, id),           \
-      SR_ARR(AZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES, id),     \
-      SR_ARR(AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES, id),             \
-      SR_ARR(DCCG_AUDIO_DTO_SOURCE, id), SR_ARR(DCCG_AUDIO_DTO0_MODULE, id),   \
-      SR_ARR(DCCG_AUDIO_DTO0_PHASE, id), SR_ARR(DCCG_AUDIO_DTO1_MODULE, id),   \
-      SR_ARR(DCCG_AUDIO_DTO1_PHASE, id)                                        \
-
-/* VPG */
-
-#define VPG_DCN3_REG_LIST_RI(id)                                               \
-  SRI_ARR(VPG_GENERIC_STATUS, VPG, id),                                        \
-      SRI_ARR(VPG_GENERIC_PACKET_ACCESS_CTRL, VPG, id),                        \
-      SRI_ARR(VPG_GENERIC_PACKET_DATA, VPG, id),                               \
-      SRI_ARR(VPG_GSP_FRAME_UPDATE_CTRL, VPG, id),                             \
-      SRI_ARR(VPG_GSP_IMMEDIATE_UPDATE_CTRL, VPG, id)
-
-/* AFMT */
-#define AFMT_DCN3_REG_LIST_RI(id)                                              \
-  SRI_ARR(AFMT_INFOFRAME_CONTROL0, AFMT, id),                                  \
-      SRI_ARR(AFMT_VBI_PACKET_CONTROL, AFMT, id),                              \
-      SRI_ARR(AFMT_AUDIO_PACKET_CONTROL, AFMT, id),                            \
-      SRI_ARR(AFMT_AUDIO_PACKET_CONTROL2, AFMT, id),                           \
-      SRI_ARR(AFMT_AUDIO_SRC_CONTROL, AFMT, id),                               \
-      SRI_ARR(AFMT_60958_0, AFMT, id), SRI_ARR(AFMT_60958_1, AFMT, id),        \
-      SRI_ARR(AFMT_60958_2, AFMT, id), SRI_ARR(AFMT_MEM_PWR, AFMT, id)
-
-/* APG */
-#define APG_DCN31_REG_LIST_RI(id)                                              \
-  SRI_ARR(APG_CONTROL, APG, id), SRI_ARR(APG_CONTROL2, APG, id),               \
-      SRI_ARR(APG_MEM_PWR, APG, id), SRI_ARR(APG_DBG_GEN_CONTROL, APG, id)
-
-/* Stream encoder */
-#define SE_DCN32_REG_LIST_RI(id)                                               \
-  SRI_ARR(AFMT_CNTL, DIG, id), SRI_ARR(DIG_FE_CNTL, DIG, id),                  \
-      SRI_ARR(HDMI_CONTROL, DIG, id), SRI_ARR(HDMI_DB_CONTROL, DIG, id),       \
-      SRI_ARR(HDMI_GC, DIG, id),                                               \
-      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL0, DIG, id),                          \
-      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL1, DIG, id),                          \
-      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL2, DIG, id),                          \
-      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL3, DIG, id),                          \
-      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL4, DIG, id),                          \
-      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL5, DIG, id),                          \
-      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL6, DIG, id),                          \
-      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL7, DIG, id),                          \
-      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL8, DIG, id),                          \
-      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL9, DIG, id),                          \
-      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL10, DIG, id),                         \
-      SRI_ARR(HDMI_INFOFRAME_CONTROL0, DIG, id),                               \
-      SRI_ARR(HDMI_INFOFRAME_CONTROL1, DIG, id),                               \
-      SRI_ARR(HDMI_VBI_PACKET_CONTROL, DIG, id),                               \
-      SRI_ARR(HDMI_AUDIO_PACKET_CONTROL, DIG, id),                             \
-      SRI_ARR(HDMI_ACR_PACKET_CONTROL, DIG, id),                               \
-      SRI_ARR(HDMI_ACR_32_0, DIG, id), SRI_ARR(HDMI_ACR_32_1, DIG, id),        \
-      SRI_ARR(HDMI_ACR_44_0, DIG, id), SRI_ARR(HDMI_ACR_44_1, DIG, id),        \
-      SRI_ARR(HDMI_ACR_48_0, DIG, id), SRI_ARR(HDMI_ACR_48_1, DIG, id),        \
-      SRI_ARR(DP_DB_CNTL, DP, id), SRI_ARR(DP_MSA_MISC, DP, id),               \
-      SRI_ARR(DP_MSA_VBID_MISC, DP, id), SRI_ARR(DP_MSA_COLORIMETRY, DP, id),  \
-      SRI_ARR(DP_MSA_TIMING_PARAM1, DP, id),                                   \
-      SRI_ARR(DP_MSA_TIMING_PARAM2, DP, id),                                   \
-      SRI_ARR(DP_MSA_TIMING_PARAM3, DP, id),                                   \
-      SRI_ARR(DP_MSA_TIMING_PARAM4, DP, id),                                   \
-      SRI_ARR(DP_MSE_RATE_CNTL, DP, id), SRI_ARR(DP_MSE_RATE_UPDATE, DP, id),  \
-      SRI_ARR(DP_PIXEL_FORMAT, DP, id), SRI_ARR(DP_SEC_CNTL, DP, id),          \
-      SRI_ARR(DP_SEC_CNTL1, DP, id), SRI_ARR(DP_SEC_CNTL2, DP, id),            \
-      SRI_ARR(DP_SEC_CNTL5, DP, id), SRI_ARR(DP_SEC_CNTL6, DP, id),            \
-      SRI_ARR(DP_STEER_FIFO, DP, id), SRI_ARR(DP_VID_M, DP, id),               \
-      SRI_ARR(DP_VID_N, DP, id), SRI_ARR(DP_VID_STREAM_CNTL, DP, id),          \
-      SRI_ARR(DP_VID_TIMING, DP, id), SRI_ARR(DP_SEC_AUD_N, DP, id),           \
-      SRI_ARR(DP_SEC_TIMESTAMP, DP, id), SRI_ARR(DP_DSC_CNTL, DP, id),         \
-      SRI_ARR(DP_SEC_METADATA_TRANSMISSION, DP, id),                           \
-      SRI_ARR(HDMI_METADATA_PACKET_CONTROL, DIG, id),                          \
-      SRI_ARR(DP_SEC_FRAMING4, DP, id), SRI_ARR(DP_GSP11_CNTL, DP, id),        \
-      SRI_ARR(DME_CONTROL, DME, id),                                           \
-      SRI_ARR(DP_SEC_METADATA_TRANSMISSION, DP, id),                           \
-      SRI_ARR(HDMI_METADATA_PACKET_CONTROL, DIG, id),                          \
-      SRI_ARR(DIG_FE_CNTL, DIG, id), SRI_ARR(DIG_CLOCK_PATTERN, DIG, id),      \
-      SRI_ARR(DIG_FIFO_CTRL0, DIG, id)
-
-/* Aux regs */
-
-#define AUX_REG_LIST_RI(id)                                                    \
-  SRI_ARR(AUX_CONTROL, DP_AUX, id), SRI_ARR(AUX_DPHY_RX_CONTROL0, DP_AUX, id), \
-      SRI_ARR(AUX_DPHY_RX_CONTROL1, DP_AUX, id)
-
-#define DCN2_AUX_REG_LIST_RI(id)                                               \
-  AUX_REG_LIST_RI(id), SRI_ARR(AUX_DPHY_TX_CONTROL, DP_AUX, id)
-
-/* HDP */
-#define HPD_REG_LIST_RI(id) SRI_ARR(DC_HPD_CONTROL, HPD, id)
-
-/* Link encoder */
-#define LE_DCN3_REG_LIST_RI(id)                                                \
-  SRI_ARR(DIG_BE_CNTL, DIG, id), SRI_ARR(DIG_BE_EN_CNTL, DIG, id),             \
-      SRI_ARR(TMDS_CTL_BITS, DIG, id),                                         \
-      SRI_ARR(TMDS_DCBALANCER_CONTROL, DIG, id), SRI_ARR(DP_CONFIG, DP, id),   \
-      SRI_ARR(DP_DPHY_CNTL, DP, id), SRI_ARR(DP_DPHY_PRBS_CNTL, DP, id),       \
-      SRI_ARR(DP_DPHY_SCRAM_CNTL, DP, id), SRI_ARR(DP_DPHY_SYM0, DP, id),      \
-      SRI_ARR(DP_DPHY_SYM1, DP, id), SRI_ARR(DP_DPHY_SYM2, DP, id),            \
-      SRI_ARR(DP_DPHY_TRAINING_PATTERN_SEL, DP, id),                           \
-      SRI_ARR(DP_LINK_CNTL, DP, id), SRI_ARR(DP_LINK_FRAMING_CNTL, DP, id),    \
-      SRI_ARR(DP_MSE_SAT0, DP, id), SRI_ARR(DP_MSE_SAT1, DP, id),              \
-      SRI_ARR(DP_MSE_SAT2, DP, id), SRI_ARR(DP_MSE_SAT_UPDATE, DP, id),        \
-      SRI_ARR(DP_SEC_CNTL, DP, id), SRI_ARR(DP_VID_STREAM_CNTL, DP, id),       \
-      SRI_ARR(DP_DPHY_FAST_TRAINING, DP, id), SRI_ARR(DP_SEC_CNTL1, DP, id),   \
-      SRI_ARR(DP_DPHY_BS_SR_SWAP_CNTL, DP, id),                                \
-      SRI_ARR(DP_DPHY_HBR2_PATTERN_CONTROL, DP, id)
-
-#define LE_DCN31_REG_LIST_RI(id)                                               \
-  LE_DCN3_REG_LIST_RI(id), SRI_ARR(DP_DPHY_INTERNAL_CTRL, DP, id),             \
-      SR_ARR(DIO_LINKA_CNTL, id), SR_ARR(DIO_LINKB_CNTL, id),                  \
-      SR_ARR(DIO_LINKC_CNTL, id), SR_ARR(DIO_LINKD_CNTL, id),                  \
-      SR_ARR(DIO_LINKE_CNTL, id), SR_ARR(DIO_LINKF_CNTL, id)
-
-#define UNIPHY_DCN2_REG_LIST_RI(id, phyid)                                     \
-  SRI_ARR_ALPHABET(CLOCK_ENABLE, SYMCLK, id, phyid),                           \
-      SRI_ARR_ALPHABET(CHANNEL_XBAR_CNTL, UNIPHY, id, phyid)
-
-/* HPO DP stream encoder */
-#define DCN3_1_HPO_DP_STREAM_ENC_REG_LIST_RI(id)                               \
-  SR_ARR(DP_STREAM_MAPPER_CONTROL0, id),                                       \
-      SR_ARR(DP_STREAM_MAPPER_CONTROL1, id),                                   \
-      SR_ARR(DP_STREAM_MAPPER_CONTROL2, id),                                   \
-      SR_ARR(DP_STREAM_MAPPER_CONTROL3, id),                                   \
-      SRI_ARR(DP_STREAM_ENC_CLOCK_CONTROL, DP_STREAM_ENC, id),                 \
-      SRI_ARR(DP_STREAM_ENC_INPUT_MUX_CONTROL, DP_STREAM_ENC, id),             \
-      SRI_ARR(DP_STREAM_ENC_AUDIO_CONTROL, DP_STREAM_ENC, id),                 \
-      SRI_ARR(DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0, DP_STREAM_ENC, id), \
-      SRI_ARR(DP_SYM32_ENC_CONTROL, DP_SYM32_ENC, id),                         \
-      SRI_ARR(DP_SYM32_ENC_VID_PIXEL_FORMAT, DP_SYM32_ENC, id),                \
-      SRI_ARR(DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL, DP_SYM32_ENC, id), \
-      SRI_ARR(DP_SYM32_ENC_VID_MSA0, DP_SYM32_ENC, id),                        \
-      SRI_ARR(DP_SYM32_ENC_VID_MSA1, DP_SYM32_ENC, id),                        \
-      SRI_ARR(DP_SYM32_ENC_VID_MSA2, DP_SYM32_ENC, id),                        \
-      SRI_ARR(DP_SYM32_ENC_VID_MSA3, DP_SYM32_ENC, id),                        \
-      SRI_ARR(DP_SYM32_ENC_VID_MSA4, DP_SYM32_ENC, id),                        \
-      SRI_ARR(DP_SYM32_ENC_VID_MSA5, DP_SYM32_ENC, id),                        \
-      SRI_ARR(DP_SYM32_ENC_VID_MSA6, DP_SYM32_ENC, id),                        \
-      SRI_ARR(DP_SYM32_ENC_VID_MSA7, DP_SYM32_ENC, id),                        \
-      SRI_ARR(DP_SYM32_ENC_VID_MSA8, DP_SYM32_ENC, id),                        \
-      SRI_ARR(DP_SYM32_ENC_VID_MSA_CONTROL, DP_SYM32_ENC, id),                 \
-      SRI_ARR(DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL, DP_SYM32_ENC, id),   \
-      SRI_ARR(DP_SYM32_ENC_VID_FIFO_CONTROL, DP_SYM32_ENC, id),                \
-      SRI_ARR(DP_SYM32_ENC_VID_STREAM_CONTROL, DP_SYM32_ENC, id),              \
-      SRI_ARR(DP_SYM32_ENC_VID_VBID_CONTROL, DP_SYM32_ENC, id),                \
-      SRI_ARR(DP_SYM32_ENC_SDP_CONTROL, DP_SYM32_ENC, id),                     \
-      SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL0, DP_SYM32_ENC, id),                \
-      SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL2, DP_SYM32_ENC, id),                \
-      SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL3, DP_SYM32_ENC, id),                \
-      SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL5, DP_SYM32_ENC, id),                \
-      SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL11, DP_SYM32_ENC, id),               \
-      SRI_ARR(DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL, DP_SYM32_ENC, id),     \
-      SRI_ARR(DP_SYM32_ENC_SDP_AUDIO_CONTROL0, DP_SYM32_ENC, id),              \
-      SRI_ARR(DP_SYM32_ENC_VID_CRC_CONTROL, DP_SYM32_ENC, id),                 \
-      SRI_ARR(DP_SYM32_ENC_HBLANK_CONTROL, DP_SYM32_ENC, id)
-
-/* HPO DP link encoder regs */
-#define DCN3_1_HPO_DP_LINK_ENC_REG_LIST_RI(id)                                 \
-  SRI_ARR(DP_LINK_ENC_CLOCK_CONTROL, DP_LINK_ENC, id),                         \
-      SRI_ARR(DP_DPHY_SYM32_CONTROL, DP_DPHY_SYM32, id),                       \
-      SRI_ARR(DP_DPHY_SYM32_STATUS, DP_DPHY_SYM32, id),                        \
-      SRI_ARR(DP_DPHY_SYM32_TP_CONFIG, DP_DPHY_SYM32, id),                     \
-      SRI_ARR(DP_DPHY_SYM32_TP_PRBS_SEED0, DP_DPHY_SYM32, id),                 \
-      SRI_ARR(DP_DPHY_SYM32_TP_PRBS_SEED1, DP_DPHY_SYM32, id),                 \
-      SRI_ARR(DP_DPHY_SYM32_TP_PRBS_SEED2, DP_DPHY_SYM32, id),                 \
-      SRI_ARR(DP_DPHY_SYM32_TP_PRBS_SEED3, DP_DPHY_SYM32, id),                 \
-      SRI_ARR(DP_DPHY_SYM32_TP_SQ_PULSE, DP_DPHY_SYM32, id),                   \
-      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM0, DP_DPHY_SYM32, id),                    \
-      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM1, DP_DPHY_SYM32, id),                    \
-      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM2, DP_DPHY_SYM32, id),                    \
-      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM3, DP_DPHY_SYM32, id),                    \
-      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM4, DP_DPHY_SYM32, id),                    \
-      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM5, DP_DPHY_SYM32, id),                    \
-      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM6, DP_DPHY_SYM32, id),                    \
-      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM7, DP_DPHY_SYM32, id),                    \
-      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM8, DP_DPHY_SYM32, id),                    \
-      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM9, DP_DPHY_SYM32, id),                    \
-      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM10, DP_DPHY_SYM32, id),                   \
-      SRI_ARR(DP_DPHY_SYM32_SAT_VC0, DP_DPHY_SYM32, id),                       \
-      SRI_ARR(DP_DPHY_SYM32_SAT_VC1, DP_DPHY_SYM32, id),                       \
-      SRI_ARR(DP_DPHY_SYM32_SAT_VC2, DP_DPHY_SYM32, id),                       \
-      SRI_ARR(DP_DPHY_SYM32_SAT_VC3, DP_DPHY_SYM32, id),                       \
-      SRI_ARR(DP_DPHY_SYM32_VC_RATE_CNTL0, DP_DPHY_SYM32, id),                 \
-      SRI_ARR(DP_DPHY_SYM32_VC_RATE_CNTL1, DP_DPHY_SYM32, id),                 \
-      SRI_ARR(DP_DPHY_SYM32_VC_RATE_CNTL2, DP_DPHY_SYM32, id),                 \
-      SRI_ARR(DP_DPHY_SYM32_VC_RATE_CNTL3, DP_DPHY_SYM32, id),                 \
-      SRI_ARR(DP_DPHY_SYM32_SAT_UPDATE, DP_DPHY_SYM32, id)
-
-/* DPP */
-#define DPP_REG_LIST_DCN30_COMMON_RI(id)                                       \
-  SRI_ARR(CM_DEALPHA, CM, id), SRI_ARR(CM_MEM_PWR_STATUS, CM, id),             \
-      SRI_ARR(CM_BIAS_CR_R, CM, id), SRI_ARR(CM_BIAS_Y_G_CB_B, CM, id),        \
-      SRI_ARR(PRE_DEGAM, CNVC_CFG, id), SRI_ARR(CM_GAMCOR_CONTROL, CM, id),    \
-      SRI_ARR(CM_GAMCOR_LUT_CONTROL, CM, id),                                  \
-      SRI_ARR(CM_GAMCOR_LUT_INDEX, CM, id),                                    \
-      SRI_ARR(CM_GAMCOR_LUT_INDEX, CM, id),                                    \
-      SRI_ARR(CM_GAMCOR_LUT_DATA, CM, id),                                     \
-      SRI_ARR(CM_GAMCOR_RAMB_START_CNTL_B, CM, id),                            \
-      SRI_ARR(CM_GAMCOR_RAMB_START_CNTL_G, CM, id),                            \
-      SRI_ARR(CM_GAMCOR_RAMB_START_CNTL_R, CM, id),                            \
-      SRI_ARR(CM_GAMCOR_RAMB_START_SLOPE_CNTL_B, CM, id),                      \
-      SRI_ARR(CM_GAMCOR_RAMB_START_SLOPE_CNTL_G, CM, id),                      \
-      SRI_ARR(CM_GAMCOR_RAMB_START_SLOPE_CNTL_R, CM, id),                      \
-      SRI_ARR(CM_GAMCOR_RAMB_END_CNTL1_B, CM, id),                             \
-      SRI_ARR(CM_GAMCOR_RAMB_END_CNTL2_B, CM, id),                             \
-      SRI_ARR(CM_GAMCOR_RAMB_END_CNTL1_G, CM, id),                             \
-      SRI_ARR(CM_GAMCOR_RAMB_END_CNTL2_G, CM, id),                             \
-      SRI_ARR(CM_GAMCOR_RAMB_END_CNTL1_R, CM, id),                             \
-      SRI_ARR(CM_GAMCOR_RAMB_END_CNTL2_R, CM, id),                             \
-      SRI_ARR(CM_GAMCOR_RAMB_REGION_0_1, CM, id),                              \
-      SRI_ARR(CM_GAMCOR_RAMB_REGION_32_33, CM, id),                            \
-      SRI_ARR(CM_GAMCOR_RAMB_OFFSET_B, CM, id),                                \
-      SRI_ARR(CM_GAMCOR_RAMB_OFFSET_G, CM, id),                                \
-      SRI_ARR(CM_GAMCOR_RAMB_OFFSET_R, CM, id),                                \
-      SRI_ARR(CM_GAMCOR_RAMB_START_BASE_CNTL_B, CM, id),                       \
-      SRI_ARR(CM_GAMCOR_RAMB_START_BASE_CNTL_G, CM, id),                       \
-      SRI_ARR(CM_GAMCOR_RAMB_START_BASE_CNTL_R, CM, id),                       \
-      SRI_ARR(CM_GAMCOR_RAMA_START_CNTL_B, CM, id),                            \
-      SRI_ARR(CM_GAMCOR_RAMA_START_CNTL_G, CM, id),                            \
-      SRI_ARR(CM_GAMCOR_RAMA_START_CNTL_R, CM, id),                            \
-      SRI_ARR(CM_GAMCOR_RAMA_START_SLOPE_CNTL_B, CM, id),                      \
-      SRI_ARR(CM_GAMCOR_RAMA_START_SLOPE_CNTL_G, CM, id),                      \
-      SRI_ARR(CM_GAMCOR_RAMA_START_SLOPE_CNTL_R, CM, id),                      \
-      SRI_ARR(CM_GAMCOR_RAMA_END_CNTL1_B, CM, id),                             \
-      SRI_ARR(CM_GAMCOR_RAMA_END_CNTL2_B, CM, id),                             \
-      SRI_ARR(CM_GAMCOR_RAMA_END_CNTL1_G, CM, id),                             \
-      SRI_ARR(CM_GAMCOR_RAMA_END_CNTL2_G, CM, id),                             \
-      SRI_ARR(CM_GAMCOR_RAMA_END_CNTL1_R, CM, id),                             \
-      SRI_ARR(CM_GAMCOR_RAMA_END_CNTL2_R, CM, id),                             \
-      SRI_ARR(CM_GAMCOR_RAMA_REGION_0_1, CM, id),                              \
-      SRI_ARR(CM_GAMCOR_RAMA_REGION_32_33, CM, id),                            \
-      SRI_ARR(CM_GAMCOR_RAMA_OFFSET_B, CM, id),                                \
-      SRI_ARR(CM_GAMCOR_RAMA_OFFSET_G, CM, id),                                \
-      SRI_ARR(CM_GAMCOR_RAMA_OFFSET_R, CM, id),                                \
-      SRI_ARR(CM_GAMCOR_RAMA_START_BASE_CNTL_B, CM, id),                       \
-      SRI_ARR(CM_GAMCOR_RAMA_START_BASE_CNTL_G, CM, id),                       \
-      SRI_ARR(CM_GAMCOR_RAMA_START_BASE_CNTL_R, CM, id),                       \
-      SRI_ARR(CM_GAMUT_REMAP_CONTROL, CM, id),                                 \
-      SRI_ARR(CM_GAMUT_REMAP_C11_C12, CM, id),                                 \
-      SRI_ARR(CM_GAMUT_REMAP_C13_C14, CM, id),                                 \
-      SRI_ARR(CM_GAMUT_REMAP_C21_C22, CM, id),                                 \
-      SRI_ARR(CM_GAMUT_REMAP_C23_C24, CM, id),                                 \
-      SRI_ARR(CM_GAMUT_REMAP_C31_C32, CM, id),                                 \
-      SRI_ARR(CM_GAMUT_REMAP_C33_C34, CM, id),                                 \
-      SRI_ARR(CM_GAMUT_REMAP_B_C11_C12, CM, id),                               \
-      SRI_ARR(CM_GAMUT_REMAP_B_C13_C14, CM, id),                               \
-      SRI_ARR(CM_GAMUT_REMAP_B_C21_C22, CM, id),                               \
-      SRI_ARR(CM_GAMUT_REMAP_B_C23_C24, CM, id),                               \
-      SRI_ARR(CM_GAMUT_REMAP_B_C31_C32, CM, id),                               \
-      SRI_ARR(CM_GAMUT_REMAP_B_C33_C34, CM, id),                               \
-      SRI_ARR(DSCL_EXT_OVERSCAN_LEFT_RIGHT, DSCL, id),                         \
-      SRI_ARR(DSCL_EXT_OVERSCAN_TOP_BOTTOM, DSCL, id),                         \
-      SRI_ARR(OTG_H_BLANK, DSCL, id), SRI_ARR(OTG_V_BLANK, DSCL, id),          \
-      SRI_ARR(SCL_MODE, DSCL, id), SRI_ARR(LB_DATA_FORMAT, DSCL, id),          \
-      SRI_ARR(LB_MEMORY_CTRL, DSCL, id), SRI_ARR(DSCL_AUTOCAL, DSCL, id),      \
-      SRI_ARR(DSCL_CONTROL, DSCL, id),                                         \
-      SRI_ARR(SCL_TAP_CONTROL, DSCL, id),                                      \
-      SRI_ARR(SCL_COEF_RAM_TAP_SELECT, DSCL, id),                              \
-      SRI_ARR(SCL_COEF_RAM_TAP_DATA, DSCL, id),                                \
-      SRI_ARR(DSCL_2TAP_CONTROL, DSCL, id), SRI_ARR(MPC_SIZE, DSCL, id),       \
-      SRI_ARR(SCL_HORZ_FILTER_SCALE_RATIO, DSCL, id),                          \
-      SRI_ARR(SCL_VERT_FILTER_SCALE_RATIO, DSCL, id),                          \
-      SRI_ARR(SCL_HORZ_FILTER_SCALE_RATIO_C, DSCL, id),                        \
-      SRI_ARR(SCL_VERT_FILTER_SCALE_RATIO_C, DSCL, id),                        \
-      SRI_ARR(SCL_HORZ_FILTER_INIT, DSCL, id),                                 \
-      SRI_ARR(SCL_HORZ_FILTER_INIT_C, DSCL, id),                               \
-      SRI_ARR(SCL_VERT_FILTER_INIT, DSCL, id),                                 \
-      SRI_ARR(SCL_VERT_FILTER_INIT_C, DSCL, id),                               \
-      SRI_ARR(RECOUT_START, DSCL, id), SRI_ARR(RECOUT_SIZE, DSCL, id),         \
-      SRI_ARR(PRE_DEALPHA, CNVC_CFG, id), SRI_ARR(PRE_REALPHA, CNVC_CFG, id),  \
-      SRI_ARR(PRE_CSC_MODE, CNVC_CFG, id),                                     \
-      SRI_ARR(PRE_CSC_C11_C12, CNVC_CFG, id),                                  \
-      SRI_ARR(PRE_CSC_C33_C34, CNVC_CFG, id),                                  \
-      SRI_ARR(PRE_CSC_B_C11_C12, CNVC_CFG, id),                                \
-      SRI_ARR(PRE_CSC_B_C33_C34, CNVC_CFG, id),                                \
-      SRI_ARR(CM_POST_CSC_CONTROL, CM, id),                                    \
-      SRI_ARR(CM_POST_CSC_C11_C12, CM, id),                                    \
-      SRI_ARR(CM_POST_CSC_C33_C34, CM, id),                                    \
-      SRI_ARR(CM_POST_CSC_B_C11_C12, CM, id),                                  \
-      SRI_ARR(CM_POST_CSC_B_C33_C34, CM, id),                                  \
-      SRI_ARR(CM_MEM_PWR_CTRL, CM, id), SRI_ARR(CM_CONTROL, CM, id),           \
-      SRI_ARR(FORMAT_CONTROL, CNVC_CFG, id),                                   \
-      SRI_ARR(CNVC_SURFACE_PIXEL_FORMAT, CNVC_CFG, id),                        \
-      SRI_ARR(CURSOR0_CONTROL, CNVC_CUR, id),                                  \
-      SRI_ARR(CURSOR0_COLOR0, CNVC_CUR, id),                                   \
-      SRI_ARR(CURSOR0_COLOR1, CNVC_CUR, id),                                   \
-      SRI_ARR(CURSOR0_FP_SCALE_BIAS, CNVC_CUR, id),                            \
-      SRI_ARR(DPP_CONTROL, DPP_TOP, id), SRI_ARR(CM_HDR_MULT_COEF, CM, id),    \
-      SRI_ARR(CURSOR_CONTROL, CURSOR0_, id),                                   \
-      SRI_ARR(ALPHA_2BIT_LUT, CNVC_CFG, id),                                   \
-      SRI_ARR(FCNV_FP_BIAS_R, CNVC_CFG, id),                                   \
-      SRI_ARR(FCNV_FP_BIAS_G, CNVC_CFG, id),                                   \
-      SRI_ARR(FCNV_FP_BIAS_B, CNVC_CFG, id),                                   \
-      SRI_ARR(FCNV_FP_SCALE_R, CNVC_CFG, id),                                  \
-      SRI_ARR(FCNV_FP_SCALE_G, CNVC_CFG, id),                                  \
-      SRI_ARR(FCNV_FP_SCALE_B, CNVC_CFG, id),                                  \
-      SRI_ARR(COLOR_KEYER_CONTROL, CNVC_CFG, id),                              \
-      SRI_ARR(COLOR_KEYER_ALPHA, CNVC_CFG, id),                                \
-      SRI_ARR(COLOR_KEYER_RED, CNVC_CFG, id),                                  \
-      SRI_ARR(COLOR_KEYER_GREEN, CNVC_CFG, id),                                \
-      SRI_ARR(COLOR_KEYER_BLUE, CNVC_CFG, id),                                 \
-      SRI_ARR(CURSOR_CONTROL, CURSOR0_, id),                                   \
-      SRI_ARR(OBUF_MEM_PWR_CTRL, DSCL, id),                                    \
-      SRI_ARR(DSCL_MEM_PWR_STATUS, DSCL, id),                                  \
-      SRI_ARR(DSCL_MEM_PWR_CTRL, DSCL, id)
-
-/* OPP */
-#define OPP_REG_LIST_DCN_RI(id)                                                \
-  SRI_ARR(FMT_BIT_DEPTH_CONTROL, FMT, id), SRI_ARR(FMT_CONTROL, FMT, id),      \
-      SRI_ARR(FMT_DITHER_RAND_R_SEED, FMT, id),                                \
-      SRI_ARR(FMT_DITHER_RAND_G_SEED, FMT, id),                                \
-      SRI_ARR(FMT_DITHER_RAND_B_SEED, FMT, id),                                \
-      SRI_ARR(FMT_CLAMP_CNTL, FMT, id),                                        \
-      SRI_ARR(FMT_DYNAMIC_EXP_CNTL, FMT, id),                                  \
-      SRI_ARR(FMT_MAP420_MEMORY_CONTROL, FMT, id),                             \
-      SRI_ARR(OPPBUF_CONTROL, OPPBUF, id),                                     \
-      SRI_ARR(OPPBUF_3D_PARAMETERS_0, OPPBUF, id),                             \
-      SRI_ARR(OPPBUF_3D_PARAMETERS_1, OPPBUF, id),                             \
-      SRI_ARR(OPP_PIPE_CONTROL, OPP_PIPE, id)                                  \
-
-#define OPP_REG_LIST_DCN10_RI(id) OPP_REG_LIST_DCN_RI(id)
-
-#define OPP_DPG_REG_LIST_RI(id)                                                \
-  SRI_ARR(DPG_CONTROL, DPG, id), SRI_ARR(DPG_DIMENSIONS, DPG, id),             \
-      SRI_ARR(DPG_OFFSET_SEGMENT, DPG, id), SRI_ARR(DPG_COLOUR_B_CB, DPG, id), \
-      SRI_ARR(DPG_COLOUR_G_Y, DPG, id), SRI_ARR(DPG_COLOUR_R_CR, DPG, id),     \
-      SRI_ARR(DPG_RAMP_CONTROL, DPG, id), SRI_ARR(DPG_STATUS, DPG, id)
-
-#define OPP_REG_LIST_DCN30_RI(id)                                              \
-  OPP_REG_LIST_DCN10_RI(id), OPP_DPG_REG_LIST_RI(id),                          \
-      SRI_ARR(FMT_422_CONTROL, FMT, id)
-
-/* Aux engine regs */
-#define AUX_COMMON_REG_LIST0_RI(id)                                            \
-  SRI_ARR(AUX_CONTROL, DP_AUX, id), SRI_ARR(AUX_ARB_CONTROL, DP_AUX, id),      \
-      SRI_ARR(AUX_SW_DATA, DP_AUX, id), SRI_ARR(AUX_SW_CONTROL, DP_AUX, id),   \
-      SRI_ARR(AUX_INTERRUPT_CONTROL, DP_AUX, id),                              \
-      SRI_ARR(AUX_DPHY_RX_CONTROL1, DP_AUX, id),                               \
-      SRI_ARR(AUX_SW_STATUS, DP_AUX, id)
-
-/* DWBC */
-#define DWBC_COMMON_REG_LIST_DCN30_RI(id)                                      \
-  SR_ARR(DWB_ENABLE_CLK_CTRL, id), SR_ARR(DWB_MEM_PWR_CTRL, id),               \
-      SR_ARR(FC_MODE_CTRL, id), SR_ARR(FC_FLOW_CTRL, id),                      \
-      SR_ARR(FC_WINDOW_START, id), SR_ARR(FC_WINDOW_SIZE, id),                 \
-      SR_ARR(FC_SOURCE_SIZE, id), SR_ARR(DWB_UPDATE_CTRL, id),                 \
-      SR_ARR(DWB_CRC_CTRL, id), SR_ARR(DWB_CRC_MASK_R_G, id),                  \
-      SR_ARR(DWB_CRC_MASK_B_A, id), SR_ARR(DWB_CRC_VAL_R_G, id),               \
-      SR_ARR(DWB_CRC_VAL_B_A, id), SR_ARR(DWB_OUT_CTRL, id),                   \
-      SR_ARR(DWB_MMHUBBUB_BACKPRESSURE_CNT_EN, id),                            \
-      SR_ARR(DWB_MMHUBBUB_BACKPRESSURE_CNT, id),                               \
-      SR_ARR(DWB_HOST_READ_CONTROL, id), SR_ARR(DWB_SOFT_RESET, id),           \
-      SR_ARR(DWB_HDR_MULT_COEF, id), SR_ARR(DWB_GAMUT_REMAP_MODE, id),         \
-      SR_ARR(DWB_GAMUT_REMAP_COEF_FORMAT, id),                                 \
-      SR_ARR(DWB_GAMUT_REMAPA_C11_C12, id),                                    \
-      SR_ARR(DWB_GAMUT_REMAPA_C13_C14, id),                                    \
-      SR_ARR(DWB_GAMUT_REMAPA_C21_C22, id),                                    \
-      SR_ARR(DWB_GAMUT_REMAPA_C23_C24, id),                                    \
-      SR_ARR(DWB_GAMUT_REMAPA_C31_C32, id),                                    \
-      SR_ARR(DWB_GAMUT_REMAPA_C33_C34, id),                                    \
-      SR_ARR(DWB_GAMUT_REMAPB_C11_C12, id),                                    \
-      SR_ARR(DWB_GAMUT_REMAPB_C13_C14, id),                                    \
-      SR_ARR(DWB_GAMUT_REMAPB_C21_C22, id),                                    \
-      SR_ARR(DWB_GAMUT_REMAPB_C23_C24, id),                                    \
-      SR_ARR(DWB_GAMUT_REMAPB_C31_C32, id),                                    \
-      SR_ARR(DWB_GAMUT_REMAPB_C33_C34, id), SR_ARR(DWB_OGAM_CONTROL, id),      \
-      SR_ARR(DWB_OGAM_LUT_INDEX, id), SR_ARR(DWB_OGAM_LUT_DATA, id),           \
-      SR_ARR(DWB_OGAM_LUT_CONTROL, id),                                        \
-      SR_ARR(DWB_OGAM_RAMA_START_CNTL_B, id),                                  \
-      SR_ARR(DWB_OGAM_RAMA_START_CNTL_G, id),                                  \
-      SR_ARR(DWB_OGAM_RAMA_START_CNTL_R, id),                                  \
-      SR_ARR(DWB_OGAM_RAMA_START_BASE_CNTL_B, id),                             \
-      SR_ARR(DWB_OGAM_RAMA_START_SLOPE_CNTL_B, id),                            \
-      SR_ARR(DWB_OGAM_RAMA_START_BASE_CNTL_G, id),                             \
-      SR_ARR(DWB_OGAM_RAMA_START_SLOPE_CNTL_G, id),                            \
-      SR_ARR(DWB_OGAM_RAMA_START_BASE_CNTL_R, id),                             \
-      SR_ARR(DWB_OGAM_RAMA_START_SLOPE_CNTL_R, id),                            \
-      SR_ARR(DWB_OGAM_RAMA_END_CNTL1_B, id),                                   \
-      SR_ARR(DWB_OGAM_RAMA_END_CNTL2_B, id),                                   \
-      SR_ARR(DWB_OGAM_RAMA_END_CNTL1_G, id),                                   \
-      SR_ARR(DWB_OGAM_RAMA_END_CNTL2_G, id),                                   \
-      SR_ARR(DWB_OGAM_RAMA_END_CNTL1_R, id),                                   \
-      SR_ARR(DWB_OGAM_RAMA_END_CNTL2_R, id),                                   \
-      SR_ARR(DWB_OGAM_RAMA_OFFSET_B, id), SR_ARR(DWB_OGAM_RAMA_OFFSET_G, id),  \
-      SR_ARR(DWB_OGAM_RAMA_OFFSET_R, id),                                      \
-      SR_ARR(DWB_OGAM_RAMA_REGION_0_1, id),                                    \
-      SR_ARR(DWB_OGAM_RAMA_REGION_2_3, id),                                    \
-      SR_ARR(DWB_OGAM_RAMA_REGION_4_5, id),                                    \
-      SR_ARR(DWB_OGAM_RAMA_REGION_6_7, id),                                    \
-      SR_ARR(DWB_OGAM_RAMA_REGION_8_9, id),                                    \
-      SR_ARR(DWB_OGAM_RAMA_REGION_10_11, id),                                  \
-      SR_ARR(DWB_OGAM_RAMA_REGION_12_13, id),                                  \
-      SR_ARR(DWB_OGAM_RAMA_REGION_14_15, id),                                  \
-      SR_ARR(DWB_OGAM_RAMA_REGION_16_17, id),                                  \
-      SR_ARR(DWB_OGAM_RAMA_REGION_18_19, id),                                  \
-      SR_ARR(DWB_OGAM_RAMA_REGION_20_21, id),                                  \
-      SR_ARR(DWB_OGAM_RAMA_REGION_22_23, id),                                  \
-      SR_ARR(DWB_OGAM_RAMA_REGION_24_25, id),                                  \
-      SR_ARR(DWB_OGAM_RAMA_REGION_26_27, id),                                  \
-      SR_ARR(DWB_OGAM_RAMA_REGION_28_29, id),                                  \
-      SR_ARR(DWB_OGAM_RAMA_REGION_30_31, id),                                  \
-      SR_ARR(DWB_OGAM_RAMA_REGION_32_33, id),                                  \
-      SR_ARR(DWB_OGAM_RAMB_START_CNTL_B, id),                                  \
-      SR_ARR(DWB_OGAM_RAMB_START_CNTL_G, id),                                  \
-      SR_ARR(DWB_OGAM_RAMB_START_CNTL_R, id),                                  \
-      SR_ARR(DWB_OGAM_RAMB_START_BASE_CNTL_B, id),                             \
-      SR_ARR(DWB_OGAM_RAMB_START_SLOPE_CNTL_B, id),                            \
-      SR_ARR(DWB_OGAM_RAMB_START_BASE_CNTL_G, id),                             \
-      SR_ARR(DWB_OGAM_RAMB_START_SLOPE_CNTL_G, id),                            \
-      SR_ARR(DWB_OGAM_RAMB_START_BASE_CNTL_R, id),                             \
-      SR_ARR(DWB_OGAM_RAMB_START_SLOPE_CNTL_R, id),                            \
-      SR_ARR(DWB_OGAM_RAMB_END_CNTL1_B, id),                                   \
-      SR_ARR(DWB_OGAM_RAMB_END_CNTL2_B, id),                                   \
-      SR_ARR(DWB_OGAM_RAMB_END_CNTL1_G, id),                                   \
-      SR_ARR(DWB_OGAM_RAMB_END_CNTL2_G, id),                                   \
-      SR_ARR(DWB_OGAM_RAMB_END_CNTL1_R, id),                                   \
-      SR_ARR(DWB_OGAM_RAMB_END_CNTL2_R, id),                                   \
-      SR_ARR(DWB_OGAM_RAMB_OFFSET_B, id), SR_ARR(DWB_OGAM_RAMB_OFFSET_G, id),  \
-      SR_ARR(DWB_OGAM_RAMB_OFFSET_R, id),                                      \
-      SR_ARR(DWB_OGAM_RAMB_REGION_0_1, id),                                    \
-      SR_ARR(DWB_OGAM_RAMB_REGION_2_3, id),                                    \
-      SR_ARR(DWB_OGAM_RAMB_REGION_4_5, id),                                    \
-      SR_ARR(DWB_OGAM_RAMB_REGION_6_7, id),                                    \
-      SR_ARR(DWB_OGAM_RAMB_REGION_8_9, id),                                    \
-      SR_ARR(DWB_OGAM_RAMB_REGION_10_11, id),                                  \
-      SR_ARR(DWB_OGAM_RAMB_REGION_12_13, id),                                  \
-      SR_ARR(DWB_OGAM_RAMB_REGION_14_15, id),                                  \
-      SR_ARR(DWB_OGAM_RAMB_REGION_16_17, id),                                  \
-      SR_ARR(DWB_OGAM_RAMB_REGION_18_19, id),                                  \
-      SR_ARR(DWB_OGAM_RAMB_REGION_20_21, id),                                  \
-      SR_ARR(DWB_OGAM_RAMB_REGION_22_23, id),                                  \
-      SR_ARR(DWB_OGAM_RAMB_REGION_24_25, id),                                  \
-      SR_ARR(DWB_OGAM_RAMB_REGION_26_27, id),                                  \
-      SR_ARR(DWB_OGAM_RAMB_REGION_28_29, id),                                  \
-      SR_ARR(DWB_OGAM_RAMB_REGION_30_31, id),                                  \
-      SR_ARR(DWB_OGAM_RAMB_REGION_32_33, id)
-
-/* MCIF */
-
-#define MCIF_WB_COMMON_REG_LIST_DCN32_RI(inst)                                 \
-  SRI2_ARR(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB, inst),                          \
-      SRI2_ARR(MCIF_WB_BUFMGR_STATUS, MCIF_WB, inst),                          \
-      SRI2_ARR(MCIF_WB_BUF_PITCH, MCIF_WB, inst),                              \
-      SRI2_ARR(MCIF_WB_BUF_1_STATUS, MCIF_WB, inst),                           \
-      SRI2_ARR(MCIF_WB_BUF_1_STATUS2, MCIF_WB, inst),                          \
-      SRI2_ARR(MCIF_WB_BUF_2_STATUS, MCIF_WB, inst),                           \
-      SRI2_ARR(MCIF_WB_BUF_2_STATUS2, MCIF_WB, inst),                          \
-      SRI2_ARR(MCIF_WB_BUF_3_STATUS, MCIF_WB, inst),                           \
-      SRI2_ARR(MCIF_WB_BUF_3_STATUS2, MCIF_WB, inst),                          \
-      SRI2_ARR(MCIF_WB_BUF_4_STATUS, MCIF_WB, inst),                           \
-      SRI2_ARR(MCIF_WB_BUF_4_STATUS2, MCIF_WB, inst),                          \
-      SRI2_ARR(MCIF_WB_ARBITRATION_CONTROL, MCIF_WB, inst),                    \
-      SRI2_ARR(MCIF_WB_SCLK_CHANGE, MCIF_WB, inst),                            \
-      SRI2_ARR(MCIF_WB_BUF_1_ADDR_Y, MCIF_WB, inst),                           \
-      SRI2_ARR(MCIF_WB_BUF_1_ADDR_C, MCIF_WB, inst),                           \
-      SRI2_ARR(MCIF_WB_BUF_2_ADDR_Y, MCIF_WB, inst),                           \
-      SRI2_ARR(MCIF_WB_BUF_2_ADDR_C, MCIF_WB, inst),                           \
-      SRI2_ARR(MCIF_WB_BUF_3_ADDR_Y, MCIF_WB, inst),                           \
-      SRI2_ARR(MCIF_WB_BUF_3_ADDR_C, MCIF_WB, inst),                           \
-      SRI2_ARR(MCIF_WB_BUF_4_ADDR_Y, MCIF_WB, inst),                           \
-      SRI2_ARR(MCIF_WB_BUF_4_ADDR_C, MCIF_WB, inst),                           \
-      SRI2_ARR(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB, inst),                     \
-      SRI2_ARR(MCIF_WB_NB_PSTATE_LATENCY_WATERMARK, MMHUBBUB, inst),           \
-      SRI2_ARR(MCIF_WB_NB_PSTATE_CONTROL, MCIF_WB, inst),                      \
-      SRI2_ARR(MCIF_WB_WATERMARK, MMHUBBUB, inst),                             \
-      SRI2_ARR(MCIF_WB_CLOCK_GATER_CONTROL, MCIF_WB, inst),                    \
-      SRI2_ARR(MCIF_WB_SELF_REFRESH_CONTROL, MCIF_WB, inst),                   \
-      SRI2_ARR(MULTI_LEVEL_QOS_CTRL, MCIF_WB, inst),                           \
-      SRI2_ARR(MCIF_WB_SECURITY_LEVEL, MCIF_WB, inst),                         \
-      SRI2_ARR(MCIF_WB_BUF_LUMA_SIZE, MCIF_WB, inst),                          \
-      SRI2_ARR(MCIF_WB_BUF_CHROMA_SIZE, MCIF_WB, inst),                        \
-      SRI2_ARR(MCIF_WB_BUF_1_ADDR_Y_HIGH, MCIF_WB, inst),                      \
-      SRI2_ARR(MCIF_WB_BUF_1_ADDR_C_HIGH, MCIF_WB, inst),                      \
-      SRI2_ARR(MCIF_WB_BUF_2_ADDR_Y_HIGH, MCIF_WB, inst),                      \
-      SRI2_ARR(MCIF_WB_BUF_2_ADDR_C_HIGH, MCIF_WB, inst),                      \
-      SRI2_ARR(MCIF_WB_BUF_3_ADDR_Y_HIGH, MCIF_WB, inst),                      \
-      SRI2_ARR(MCIF_WB_BUF_3_ADDR_C_HIGH, MCIF_WB, inst),                      \
-      SRI2_ARR(MCIF_WB_BUF_4_ADDR_Y_HIGH, MCIF_WB, inst),                      \
-      SRI2_ARR(MCIF_WB_BUF_4_ADDR_C_HIGH, MCIF_WB, inst),                      \
-      SRI2_ARR(MCIF_WB_BUF_1_RESOLUTION, MCIF_WB, inst),                       \
-      SRI2_ARR(MCIF_WB_BUF_2_RESOLUTION, MCIF_WB, inst),                       \
-      SRI2_ARR(MCIF_WB_BUF_3_RESOLUTION, MCIF_WB, inst),                       \
-      SRI2_ARR(MCIF_WB_BUF_4_RESOLUTION, MCIF_WB, inst),                       \
-      SRI2_ARR(MMHUBBUB_MEM_PWR_CNTL, MMHUBBUB, inst),                         \
-      SRI2_ARR(MMHUBBUB_WARMUP_ADDR_REGION, MMHUBBUB, inst),                   \
-      SRI2_ARR(MMHUBBUB_WARMUP_BASE_ADDR_HIGH, MMHUBBUB, inst),                \
-      SRI2_ARR(MMHUBBUB_WARMUP_BASE_ADDR_LOW, MMHUBBUB, inst),                 \
-      SRI2_ARR(MMHUBBUB_WARMUP_CONTROL_STATUS, MMHUBBUB, inst)
-
-/* DSC */
-
-#define DSC_REG_LIST_DCN20_RI(id)                                              \
-  SRI_ARR(DSC_TOP_CONTROL, DSC_TOP, id),                                       \
-      SRI_ARR(DSC_DEBUG_CONTROL, DSC_TOP, id),                                 \
-      SRI_ARR(DSCC_CONFIG0, DSCC, id), SRI_ARR(DSCC_CONFIG1, DSCC, id),        \
-      SRI_ARR(DSCC_STATUS, DSCC, id),                                          \
-      SRI_ARR(DSCC_INTERRUPT_CONTROL_STATUS, DSCC, id),                        \
-      SRI_ARR(DSCC_PPS_CONFIG0, DSCC, id),                                     \
-      SRI_ARR(DSCC_PPS_CONFIG1, DSCC, id),                                     \
-      SRI_ARR(DSCC_PPS_CONFIG2, DSCC, id),                                     \
-      SRI_ARR(DSCC_PPS_CONFIG3, DSCC, id),                                     \
-      SRI_ARR(DSCC_PPS_CONFIG4, DSCC, id),                                     \
-      SRI_ARR(DSCC_PPS_CONFIG5, DSCC, id),                                     \
-      SRI_ARR(DSCC_PPS_CONFIG6, DSCC, id),                                     \
-      SRI_ARR(DSCC_PPS_CONFIG7, DSCC, id),                                     \
-      SRI_ARR(DSCC_PPS_CONFIG8, DSCC, id),                                     \
-      SRI_ARR(DSCC_PPS_CONFIG9, DSCC, id),                                     \
-      SRI_ARR(DSCC_PPS_CONFIG10, DSCC, id),                                    \
-      SRI_ARR(DSCC_PPS_CONFIG11, DSCC, id),                                    \
-      SRI_ARR(DSCC_PPS_CONFIG12, DSCC, id),                                    \
-      SRI_ARR(DSCC_PPS_CONFIG13, DSCC, id),                                    \
-      SRI_ARR(DSCC_PPS_CONFIG14, DSCC, id),                                    \
-      SRI_ARR(DSCC_PPS_CONFIG15, DSCC, id),                                    \
-      SRI_ARR(DSCC_PPS_CONFIG16, DSCC, id),                                    \
-      SRI_ARR(DSCC_PPS_CONFIG17, DSCC, id),                                    \
-      SRI_ARR(DSCC_PPS_CONFIG18, DSCC, id),                                    \
-      SRI_ARR(DSCC_PPS_CONFIG19, DSCC, id),                                    \
-      SRI_ARR(DSCC_PPS_CONFIG20, DSCC, id),                                    \
-      SRI_ARR(DSCC_PPS_CONFIG21, DSCC, id),                                    \
-      SRI_ARR(DSCC_PPS_CONFIG22, DSCC, id),                                    \
-      SRI_ARR(DSCC_MEM_POWER_CONTROL, DSCC, id),                               \
-      SRI_ARR(DSCC_R_Y_SQUARED_ERROR_LOWER, DSCC, id),                         \
-      SRI_ARR(DSCC_R_Y_SQUARED_ERROR_UPPER, DSCC, id),                         \
-      SRI_ARR(DSCC_G_CB_SQUARED_ERROR_LOWER, DSCC, id),                        \
-      SRI_ARR(DSCC_G_CB_SQUARED_ERROR_UPPER, DSCC, id),                        \
-      SRI_ARR(DSCC_B_CR_SQUARED_ERROR_LOWER, DSCC, id),                        \
-      SRI_ARR(DSCC_B_CR_SQUARED_ERROR_UPPER, DSCC, id),                        \
-      SRI_ARR(DSCC_MAX_ABS_ERROR0, DSCC, id),                                  \
-      SRI_ARR(DSCC_MAX_ABS_ERROR1, DSCC, id),                                  \
-      SRI_ARR(DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL, DSCC, id),                 \
-      SRI_ARR(DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL, DSCC, id),                 \
-      SRI_ARR(DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL, DSCC, id),                 \
-      SRI_ARR(DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL, DSCC, id),                 \
-      SRI_ARR(DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL, DSCC, id),         \
-      SRI_ARR(DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL, DSCC, id),         \
-      SRI_ARR(DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL, DSCC, id),         \
-      SRI_ARR(DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL, DSCC, id),         \
-      SRI_ARR(DSCCIF_CONFIG0, DSCCIF, id),                                     \
-      SRI_ARR(DSCCIF_CONFIG1, DSCCIF, id),                                     \
-      SRI_ARR(DSCRM_DSC_FORWARD_CONFIG, DSCRM, id)
-
-/* MPC */
-
-#define MPC_DWB_MUX_REG_LIST_DCN3_0_RI(inst)                                   \
-  SRII_DWB(DWB_MUX, MUX, MPC_DWB, inst)
-
-#define MPC_OUT_MUX_COMMON_REG_LIST_DCN1_0_RI(inst)                            \
-  SRII(MUX, MPC_OUT, inst), VUPDATE_SRII(CUR, VUPDATE_LOCK_SET, inst)
-
-#define MPC_OUT_MUX_REG_LIST_DCN3_0_RI(inst)                                   \
-  MPC_OUT_MUX_COMMON_REG_LIST_DCN1_0_RI(inst), SRII(CSC_MODE, MPC_OUT, inst),  \
-      SRII(CSC_C11_C12_A, MPC_OUT, inst), SRII(CSC_C33_C34_A, MPC_OUT, inst),  \
-      SRII(CSC_C11_C12_B, MPC_OUT, inst), SRII(CSC_C33_C34_B, MPC_OUT, inst),  \
-      SRII(DENORM_CONTROL, MPC_OUT, inst),                                     \
-      SRII(DENORM_CLAMP_G_Y, MPC_OUT, inst),                                   \
-      SRII(DENORM_CLAMP_B_CB, MPC_OUT, inst), SR(MPC_OUT_CSC_COEF_FORMAT)
-
-#define MPC_COMMON_REG_LIST_DCN1_0_RI(inst)                                    \
-  SRII(MPCC_TOP_SEL, MPCC, inst), SRII(MPCC_BOT_SEL, MPCC, inst),              \
-      SRII(MPCC_CONTROL, MPCC, inst), SRII(MPCC_STATUS, MPCC, inst),           \
-      SRII(MPCC_OPP_ID, MPCC, inst), SRII(MPCC_BG_G_Y, MPCC, inst),            \
-      SRII(MPCC_BG_R_CR, MPCC, inst), SRII(MPCC_BG_B_CB, MPCC, inst),          \
-      SRII(MPCC_SM_CONTROL, MPCC, inst),                                       \
-      SRII(MPCC_UPDATE_LOCK_SEL, MPCC, inst)
-
-#define MPC_REG_LIST_DCN3_0_RI(inst)                                           \
-  MPC_COMMON_REG_LIST_DCN1_0_RI(inst), SRII(MPCC_TOP_GAIN, MPCC, inst),        \
-      SRII(MPCC_BOT_GAIN_INSIDE, MPCC, inst),                                  \
-      SRII(MPCC_BOT_GAIN_OUTSIDE, MPCC, inst),                                 \
-      SRII(MPCC_MEM_PWR_CTRL, MPCC, inst),                                     \
-      SRII(MPCC_OGAM_LUT_INDEX, MPCC_OGAM, inst),                              \
-      SRII(MPCC_OGAM_LUT_DATA, MPCC_OGAM, inst),                               \
-      SRII(MPCC_GAMUT_REMAP_COEF_FORMAT, MPCC_OGAM, inst),                     \
-      SRII(MPCC_GAMUT_REMAP_MODE, MPCC_OGAM, inst),                            \
-      SRII(MPC_GAMUT_REMAP_C11_C12_A, MPCC_OGAM, inst),                        \
-      SRII(MPC_GAMUT_REMAP_C33_C34_A, MPCC_OGAM, inst),                        \
-      SRII(MPC_GAMUT_REMAP_C11_C12_B, MPCC_OGAM, inst),                        \
-      SRII(MPC_GAMUT_REMAP_C33_C34_B, MPCC_OGAM, inst),                        \
-      SRII(MPCC_OGAM_RAMA_START_CNTL_B, MPCC_OGAM, inst),                      \
-      SRII(MPCC_OGAM_RAMA_START_CNTL_G, MPCC_OGAM, inst),                      \
-      SRII(MPCC_OGAM_RAMA_START_CNTL_R, MPCC_OGAM, inst),                      \
-      SRII(MPCC_OGAM_RAMA_START_SLOPE_CNTL_B, MPCC_OGAM, inst),                \
-      SRII(MPCC_OGAM_RAMA_START_SLOPE_CNTL_G, MPCC_OGAM, inst),                \
-      SRII(MPCC_OGAM_RAMA_START_SLOPE_CNTL_R, MPCC_OGAM, inst),                \
-      SRII(MPCC_OGAM_RAMA_END_CNTL1_B, MPCC_OGAM, inst),                       \
-      SRII(MPCC_OGAM_RAMA_END_CNTL2_B, MPCC_OGAM, inst),                       \
-      SRII(MPCC_OGAM_RAMA_END_CNTL1_G, MPCC_OGAM, inst),                       \
-      SRII(MPCC_OGAM_RAMA_END_CNTL2_G, MPCC_OGAM, inst),                       \
-      SRII(MPCC_OGAM_RAMA_END_CNTL1_R, MPCC_OGAM, inst),                       \
-      SRII(MPCC_OGAM_RAMA_END_CNTL2_R, MPCC_OGAM, inst),                       \
-      SRII(MPCC_OGAM_RAMA_REGION_0_1, MPCC_OGAM, inst),                        \
-      SRII(MPCC_OGAM_RAMA_REGION_32_33, MPCC_OGAM, inst),                      \
-      SRII(MPCC_OGAM_RAMA_OFFSET_B, MPCC_OGAM, inst),                          \
-      SRII(MPCC_OGAM_RAMA_OFFSET_G, MPCC_OGAM, inst),                          \
-      SRII(MPCC_OGAM_RAMA_OFFSET_R, MPCC_OGAM, inst),                          \
-      SRII(MPCC_OGAM_RAMA_START_BASE_CNTL_B, MPCC_OGAM, inst),                 \
-      SRII(MPCC_OGAM_RAMA_START_BASE_CNTL_G, MPCC_OGAM, inst),                 \
-      SRII(MPCC_OGAM_RAMA_START_BASE_CNTL_R, MPCC_OGAM, inst),                 \
-      SRII(MPCC_OGAM_RAMB_START_CNTL_B, MPCC_OGAM, inst),                      \
-      SRII(MPCC_OGAM_RAMB_START_CNTL_G, MPCC_OGAM, inst),                      \
-      SRII(MPCC_OGAM_RAMB_START_CNTL_R, MPCC_OGAM, inst),                      \
-      SRII(MPCC_OGAM_RAMB_START_SLOPE_CNTL_B, MPCC_OGAM, inst),                \
-      SRII(MPCC_OGAM_RAMB_START_SLOPE_CNTL_G, MPCC_OGAM, inst),                \
-      SRII(MPCC_OGAM_RAMB_START_SLOPE_CNTL_R, MPCC_OGAM, inst),                \
-      SRII(MPCC_OGAM_RAMB_END_CNTL1_B, MPCC_OGAM, inst),                       \
-      SRII(MPCC_OGAM_RAMB_END_CNTL2_B, MPCC_OGAM, inst),                       \
-      SRII(MPCC_OGAM_RAMB_END_CNTL1_G, MPCC_OGAM, inst),                       \
-      SRII(MPCC_OGAM_RAMB_END_CNTL2_G, MPCC_OGAM, inst),                       \
-      SRII(MPCC_OGAM_RAMB_END_CNTL1_R, MPCC_OGAM, inst),                       \
-      SRII(MPCC_OGAM_RAMB_END_CNTL2_R, MPCC_OGAM, inst),                       \
-      SRII(MPCC_OGAM_RAMB_REGION_0_1, MPCC_OGAM, inst),                        \
-      SRII(MPCC_OGAM_RAMB_REGION_32_33, MPCC_OGAM, inst),                      \
-      SRII(MPCC_OGAM_RAMB_OFFSET_B, MPCC_OGAM, inst),                          \
-      SRII(MPCC_OGAM_RAMB_OFFSET_G, MPCC_OGAM, inst),                          \
-      SRII(MPCC_OGAM_RAMB_OFFSET_R, MPCC_OGAM, inst),                          \
-      SRII(MPCC_OGAM_RAMB_START_BASE_CNTL_B, MPCC_OGAM, inst),                 \
-      SRII(MPCC_OGAM_RAMB_START_BASE_CNTL_G, MPCC_OGAM, inst),                 \
-      SRII(MPCC_OGAM_RAMB_START_BASE_CNTL_R, MPCC_OGAM, inst),                 \
-      SRII(MPCC_OGAM_CONTROL, MPCC_OGAM, inst),                                \
-      SRII(MPCC_OGAM_LUT_CONTROL, MPCC_OGAM, inst)
-
-#define MPC_REG_LIST_DCN3_2_RI(inst) \
-	MPC_REG_LIST_DCN3_0_RI(inst),\
-	SRII(MPCC_MOVABLE_CM_LOCATION_CONTROL, MPCC, inst),\
-	SRII(MPCC_MCM_SHAPER_CONTROL, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_OFFSET_R, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_OFFSET_G, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_OFFSET_B, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_SCALE_R, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_SCALE_G_B, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_LUT_INDEX, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_LUT_DATA, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_START_CNTL_B, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_START_CNTL_G, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_START_CNTL_R, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_END_CNTL_B, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_END_CNTL_G, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_END_CNTL_R, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_REGION_0_1, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_REGION_2_3, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_REGION_4_5, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_REGION_6_7, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_REGION_8_9, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_REGION_10_11, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_REGION_12_13, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_REGION_14_15, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_REGION_16_17, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_REGION_18_19, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_REGION_20_21, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_REGION_22_23, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_REGION_24_25, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_REGION_26_27, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_REGION_28_29, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_REGION_30_31, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMA_REGION_32_33, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_START_CNTL_B, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_START_CNTL_G, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_START_CNTL_R, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_END_CNTL_B, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_END_CNTL_G, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_END_CNTL_R, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_REGION_0_1, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_REGION_2_3, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_REGION_4_5, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_REGION_6_7, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_REGION_8_9, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_REGION_10_11, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_REGION_12_13, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_REGION_14_15, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_REGION_16_17, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_REGION_18_19, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_REGION_20_21, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_REGION_22_23, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_REGION_24_25, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_REGION_26_27, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_REGION_28_29, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_REGION_30_31, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_SHAPER_RAMB_REGION_32_33, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_3DLUT_MODE, MPCC_MCM, inst), /*TODO: may need to add other 3DLUT regs*/\
-	SRII(MPCC_MCM_3DLUT_INDEX, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_3DLUT_DATA, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_3DLUT_DATA_30BIT, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_3DLUT_READ_WRITE_CONTROL, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_3DLUT_OUT_NORM_FACTOR, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_3DLUT_OUT_OFFSET_R, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_3DLUT_OUT_OFFSET_G, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_3DLUT_OUT_OFFSET_B, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_CONTROL, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_LUT_INDEX, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_LUT_DATA, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_LUT_CONTROL, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_START_CNTL_B, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_START_CNTL_G, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_START_CNTL_R, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL1_B, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL2_B, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL1_G, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL2_G, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL1_R, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL2_R, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_OFFSET_B, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_OFFSET_G, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_OFFSET_R, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_REGION_0_1, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_REGION_2_3, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_REGION_4_5, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_REGION_6_7, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_REGION_8_9, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_REGION_10_11, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_REGION_12_13, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_REGION_14_15, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_REGION_16_17, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_REGION_18_19, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_REGION_20_21, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_REGION_22_23, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_REGION_24_25, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_REGION_26_27, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_REGION_28_29, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_REGION_30_31, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMA_REGION_32_33, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_START_CNTL_B, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_START_CNTL_G, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_START_CNTL_R, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL1_B, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL2_B, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL1_G, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL2_G, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL1_R, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL2_R, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_OFFSET_B, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_OFFSET_G, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_OFFSET_R, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_REGION_0_1, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_REGION_2_3, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_REGION_4_5, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_REGION_6_7, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_REGION_8_9, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_REGION_10_11, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_REGION_12_13, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_REGION_14_15, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_REGION_16_17, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_REGION_18_19, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_REGION_20_21, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_REGION_22_23, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_REGION_24_25, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_REGION_26_27, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_REGION_28_29, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_REGION_30_31, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_1DLUT_RAMB_REGION_32_33, MPCC_MCM, inst),\
-	SRII(MPCC_MCM_MEM_PWR_CTRL, MPCC_MCM, inst)
-/* OPTC */
-
-#define OPTC_COMMON_REG_LIST_DCN3_2_RI(inst)                                   \
-  SRI_ARR(OTG_VSTARTUP_PARAM, OTG, inst),                                      \
-      SRI_ARR(OTG_VUPDATE_PARAM, OTG, inst),                                   \
-      SRI_ARR(OTG_VREADY_PARAM, OTG, inst),                                    \
-      SRI_ARR(OTG_MASTER_UPDATE_LOCK, OTG, inst),                              \
-      SRI_ARR(OTG_GLOBAL_CONTROL0, OTG, inst),                                 \
-      SRI_ARR(OTG_GLOBAL_CONTROL1, OTG, inst),                                 \
-      SRI_ARR(OTG_GLOBAL_CONTROL2, OTG, inst),                                 \
-      SRI_ARR(OTG_GLOBAL_CONTROL4, OTG, inst),                                 \
-      SRI_ARR(OTG_DOUBLE_BUFFER_CONTROL, OTG, inst),                           \
-      SRI_ARR(OTG_H_TOTAL, OTG, inst),                                         \
-      SRI_ARR(OTG_H_BLANK_START_END, OTG, inst),                               \
-      SRI_ARR(OTG_H_SYNC_A, OTG, inst), SRI_ARR(OTG_H_SYNC_A_CNTL, OTG, inst), \
-      SRI_ARR(OTG_H_TIMING_CNTL, OTG, inst), SRI_ARR(OTG_V_TOTAL, OTG, inst),  \
-      SRI_ARR(OTG_V_BLANK_START_END, OTG, inst),                               \
-      SRI_ARR(OTG_V_SYNC_A, OTG, inst), SRI_ARR(OTG_V_SYNC_A_CNTL, OTG, inst), \
-      SRI_ARR(OTG_CONTROL, OTG, inst), SRI_ARR(OTG_STEREO_CONTROL, OTG, inst), \
-      SRI_ARR(OTG_3D_STRUCTURE_CONTROL, OTG, inst),                            \
-      SRI_ARR(OTG_STEREO_STATUS, OTG, inst),                                   \
-      SRI_ARR(OTG_V_TOTAL_MAX, OTG, inst),                                     \
-      SRI_ARR(OTG_V_TOTAL_MIN, OTG, inst),                                     \
-      SRI_ARR(OTG_V_TOTAL_CONTROL, OTG, inst),                                 \
-      SRI_ARR(OTG_TRIGA_CNTL, OTG, inst),                                      \
-      SRI_ARR(OTG_FORCE_COUNT_NOW_CNTL, OTG, inst),                            \
-      SRI_ARR(OTG_STATIC_SCREEN_CONTROL, OTG, inst),                           \
-      SRI_ARR(OTG_STATUS_FRAME_COUNT, OTG, inst),                              \
-      SRI_ARR(OTG_STATUS, OTG, inst), SRI_ARR(OTG_STATUS_POSITION, OTG, inst), \
-      SRI_ARR(OTG_NOM_VERT_POSITION, OTG, inst),                               \
-      SRI_ARR(OTG_M_CONST_DTO0, OTG, inst),                                    \
-      SRI_ARR(OTG_M_CONST_DTO1, OTG, inst),                                    \
-      SRI_ARR(OTG_CLOCK_CONTROL, OTG, inst),                                   \
-      SRI_ARR(OTG_VERTICAL_INTERRUPT0_CONTROL, OTG, inst),                     \
-      SRI_ARR(OTG_VERTICAL_INTERRUPT0_POSITION, OTG, inst),                    \
-      SRI_ARR(OTG_VERTICAL_INTERRUPT1_CONTROL, OTG, inst),                     \
-      SRI_ARR(OTG_VERTICAL_INTERRUPT1_POSITION, OTG, inst),                    \
-      SRI_ARR(OTG_VERTICAL_INTERRUPT2_CONTROL, OTG, inst),                     \
-      SRI_ARR(OTG_VERTICAL_INTERRUPT2_POSITION, OTG, inst),                    \
-      SRI_ARR(OPTC_INPUT_CLOCK_CONTROL, ODM, inst),                            \
-      SRI_ARR(OPTC_DATA_SOURCE_SELECT, ODM, inst),                             \
-      SRI_ARR(OPTC_INPUT_GLOBAL_CONTROL, ODM, inst),                           \
-      SRI_ARR(CONTROL, VTG, inst), SRI_ARR(OTG_VERT_SYNC_CONTROL, OTG, inst),  \
-      SRI_ARR(OTG_GSL_CONTROL, OTG, inst), SRI_ARR(OTG_CRC_CNTL, OTG, inst),   \
-      SRI_ARR(OTG_CRC0_DATA_RG, OTG, inst),                                    \
-      SRI_ARR(OTG_CRC0_DATA_B, OTG, inst),                                     \
-      SRI_ARR(OTG_CRC0_WINDOWA_X_CONTROL, OTG, inst),                          \
-      SRI_ARR(OTG_CRC0_WINDOWA_Y_CONTROL, OTG, inst),                          \
-      SRI_ARR(OTG_CRC0_WINDOWB_X_CONTROL, OTG, inst),                          \
-      SRI_ARR(OTG_CRC0_WINDOWB_Y_CONTROL, OTG, inst),                          \
-      SR_ARR(GSL_SOURCE_SELECT, inst),                                         \
-      SRI_ARR(OTG_TRIGA_MANUAL_TRIG, OTG, inst),                               \
-      SRI_ARR(OTG_GLOBAL_CONTROL1, OTG, inst),                                 \
-      SRI_ARR(OTG_GLOBAL_CONTROL2, OTG, inst),                                 \
-      SRI_ARR(OTG_GSL_WINDOW_X, OTG, inst),                                    \
-      SRI_ARR(OTG_GSL_WINDOW_Y, OTG, inst),                                    \
-      SRI_ARR(OTG_VUPDATE_KEEPOUT, OTG, inst),                                 \
-      SRI_ARR(OTG_DSC_START_POSITION, OTG, inst),                              \
-      SRI_ARR(OTG_DRR_TRIGGER_WINDOW, OTG, inst),                              \
-      SRI_ARR(OTG_DRR_V_TOTAL_CHANGE, OTG, inst),                              \
-      SRI_ARR(OPTC_DATA_FORMAT_CONTROL, ODM, inst),                            \
-      SRI_ARR(OPTC_BYTES_PER_PIXEL, ODM, inst),                                \
-      SRI_ARR(OPTC_WIDTH_CONTROL, ODM, inst),                                  \
-      SRI_ARR(OPTC_MEMORY_CONFIG, ODM, inst),                                  \
-      SRI_ARR(OTG_DRR_CONTROL, OTG, inst)
-
-/* HUBP */
-
-#define HUBP_REG_LIST_DCN_VM_RI(id)                                            \
-  SRI_ARR(NOM_PARAMETERS_0, HUBPREQ, id),                                      \
-      SRI_ARR(NOM_PARAMETERS_1, HUBPREQ, id),                                  \
-      SRI_ARR(NOM_PARAMETERS_2, HUBPREQ, id),                                  \
-      SRI_ARR(NOM_PARAMETERS_3, HUBPREQ, id),                                  \
-      SRI_ARR(DCN_VM_MX_L1_TLB_CNTL, HUBPREQ, id)
-#define HUBP_REG_LIST_DCN_RI(id)                                               \
-  SRI_ARR(DCHUBP_CNTL, HUBP, id), SRI_ARR(HUBPREQ_DEBUG_DB, HUBP, id),         \
-      SRI_ARR(HUBPREQ_DEBUG, HUBP, id), SRI_ARR(DCSURF_ADDR_CONFIG, HUBP, id), \
-      SRI_ARR(DCSURF_TILING_CONFIG, HUBP, id),                                 \
-      SRI_ARR(DCSURF_SURFACE_PITCH, HUBPREQ, id),                              \
-      SRI_ARR(DCSURF_SURFACE_PITCH_C, HUBPREQ, id),                            \
-      SRI_ARR(DCSURF_SURFACE_CONFIG, HUBP, id),                                \
-      SRI_ARR(DCSURF_FLIP_CONTROL, HUBPREQ, id),                               \
-      SRI_ARR(DCSURF_PRI_VIEWPORT_DIMENSION, HUBP, id),                        \
-      SRI_ARR(DCSURF_PRI_VIEWPORT_START, HUBP, id),                            \
-      SRI_ARR(DCSURF_SEC_VIEWPORT_DIMENSION, HUBP, id),                        \
-      SRI_ARR(DCSURF_SEC_VIEWPORT_START, HUBP, id),                            \
-      SRI_ARR(DCSURF_PRI_VIEWPORT_DIMENSION_C, HUBP, id),                      \
-      SRI_ARR(DCSURF_PRI_VIEWPORT_START_C, HUBP, id),                          \
-      SRI_ARR(DCSURF_SEC_VIEWPORT_DIMENSION_C, HUBP, id),                      \
-      SRI_ARR(DCSURF_SEC_VIEWPORT_START_C, HUBP, id),                          \
-      SRI_ARR(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH, HUBPREQ, id),               \
-      SRI_ARR(DCSURF_PRIMARY_SURFACE_ADDRESS, HUBPREQ, id),                    \
-      SRI_ARR(DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH, HUBPREQ, id),             \
-      SRI_ARR(DCSURF_SECONDARY_SURFACE_ADDRESS, HUBPREQ, id),                  \
-      SRI_ARR(DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH, HUBPREQ, id),          \
-      SRI_ARR(DCSURF_PRIMARY_META_SURFACE_ADDRESS, HUBPREQ, id),               \
-      SRI_ARR(DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH, HUBPREQ, id),        \
-      SRI_ARR(DCSURF_SECONDARY_META_SURFACE_ADDRESS, HUBPREQ, id),             \
-      SRI_ARR(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C, HUBPREQ, id),             \
-      SRI_ARR(DCSURF_PRIMARY_SURFACE_ADDRESS_C, HUBPREQ, id),                  \
-      SRI_ARR(DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C, HUBPREQ, id),           \
-      SRI_ARR(DCSURF_SECONDARY_SURFACE_ADDRESS_C, HUBPREQ, id),                \
-      SRI_ARR(DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C, HUBPREQ, id),        \
-      SRI_ARR(DCSURF_PRIMARY_META_SURFACE_ADDRESS_C, HUBPREQ, id),             \
-      SRI_ARR(DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C, HUBPREQ, id),      \
-      SRI_ARR(DCSURF_SECONDARY_META_SURFACE_ADDRESS_C, HUBPREQ, id),           \
-      SRI_ARR(DCSURF_SURFACE_INUSE, HUBPREQ, id),                              \
-      SRI_ARR(DCSURF_SURFACE_INUSE_HIGH, HUBPREQ, id),                         \
-      SRI_ARR(DCSURF_SURFACE_INUSE_C, HUBPREQ, id),                            \
-      SRI_ARR(DCSURF_SURFACE_INUSE_HIGH_C, HUBPREQ, id),                       \
-      SRI_ARR(DCSURF_SURFACE_EARLIEST_INUSE, HUBPREQ, id),                     \
-      SRI_ARR(DCSURF_SURFACE_EARLIEST_INUSE_HIGH, HUBPREQ, id),                \
-      SRI_ARR(DCSURF_SURFACE_EARLIEST_INUSE_C, HUBPREQ, id),                   \
-      SRI_ARR(DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C, HUBPREQ, id),              \
-      SRI_ARR(DCSURF_SURFACE_CONTROL, HUBPREQ, id),                            \
-      SRI_ARR(DCSURF_SURFACE_FLIP_INTERRUPT, HUBPREQ, id),                     \
-      SRI_ARR(HUBPRET_CONTROL, HUBPRET, id),                                   \
-      SRI_ARR(HUBPRET_READ_LINE_STATUS, HUBPRET, id),                          \
-      SRI_ARR(DCN_EXPANSION_MODE, HUBPREQ, id),                                \
-      SRI_ARR(DCHUBP_REQ_SIZE_CONFIG, HUBP, id),                               \
-      SRI_ARR(DCHUBP_REQ_SIZE_CONFIG_C, HUBP, id),                             \
-      SRI_ARR(BLANK_OFFSET_0, HUBPREQ, id),                                    \
-      SRI_ARR(BLANK_OFFSET_1, HUBPREQ, id),                                    \
-      SRI_ARR(DST_DIMENSIONS, HUBPREQ, id),                                    \
-      SRI_ARR(DST_AFTER_SCALER, HUBPREQ, id),                                  \
-      SRI_ARR(VBLANK_PARAMETERS_0, HUBPREQ, id),                               \
-      SRI_ARR(REF_FREQ_TO_PIX_FREQ, HUBPREQ, id),                              \
-      SRI_ARR(VBLANK_PARAMETERS_1, HUBPREQ, id),                               \
-      SRI_ARR(VBLANK_PARAMETERS_3, HUBPREQ, id),                               \
-      SRI_ARR(NOM_PARAMETERS_4, HUBPREQ, id),                                  \
-      SRI_ARR(NOM_PARAMETERS_5, HUBPREQ, id),                                  \
-      SRI_ARR(PER_LINE_DELIVERY_PRE, HUBPREQ, id),                             \
-      SRI_ARR(PER_LINE_DELIVERY, HUBPREQ, id),                                 \
-      SRI_ARR(VBLANK_PARAMETERS_2, HUBPREQ, id),                               \
-      SRI_ARR(VBLANK_PARAMETERS_4, HUBPREQ, id),                               \
-      SRI_ARR(NOM_PARAMETERS_6, HUBPREQ, id),                                  \
-      SRI_ARR(NOM_PARAMETERS_7, HUBPREQ, id),                                  \
-      SRI_ARR(DCN_TTU_QOS_WM, HUBPREQ, id),                                    \
-      SRI_ARR(DCN_GLOBAL_TTU_CNTL, HUBPREQ, id),                               \
-      SRI_ARR(DCN_SURF0_TTU_CNTL0, HUBPREQ, id),                               \
-      SRI_ARR(DCN_SURF0_TTU_CNTL1, HUBPREQ, id),                               \
-      SRI_ARR(DCN_SURF1_TTU_CNTL0, HUBPREQ, id),                               \
-      SRI_ARR(DCN_SURF1_TTU_CNTL1, HUBPREQ, id),                               \
-      SRI_ARR(DCN_CUR0_TTU_CNTL0, HUBPREQ, id),                                \
-      SRI_ARR(DCN_CUR0_TTU_CNTL1, HUBPREQ, id),                                \
-      SRI_ARR(HUBP_CLK_CNTL, HUBP, id)
-#define HUBP_REG_LIST_DCN2_COMMON_RI(id)                                       \
-  HUBP_REG_LIST_DCN_RI(id), HUBP_REG_LIST_DCN_VM_RI(id),                       \
-      SRI_ARR(PREFETCH_SETTINGS, HUBPREQ, id),                                 \
-      SRI_ARR(PREFETCH_SETTINGS_C, HUBPREQ, id),                               \
-      SRI_ARR(DCN_VM_SYSTEM_APERTURE_LOW_ADDR, HUBPREQ, id),                   \
-      SRI_ARR(DCN_VM_SYSTEM_APERTURE_HIGH_ADDR, HUBPREQ, id),                  \
-      SRI_ARR(CURSOR_SETTINGS, HUBPREQ, id),                                   \
-      SRI_ARR(CURSOR_SURFACE_ADDRESS_HIGH, CURSOR0_, id),                      \
-      SRI_ARR(CURSOR_SURFACE_ADDRESS, CURSOR0_, id),                           \
-      SRI_ARR(CURSOR_SIZE, CURSOR0_, id),                                      \
-      SRI_ARR(CURSOR_CONTROL, CURSOR0_, id),                                   \
-      SRI_ARR(CURSOR_POSITION, CURSOR0_, id),                                  \
-      SRI_ARR(CURSOR_HOT_SPOT, CURSOR0_, id),                                  \
-      SRI_ARR(CURSOR_DST_OFFSET, CURSOR0_, id),                                \
-      SRI_ARR(DMDATA_ADDRESS_HIGH, CURSOR0_, id),                              \
-      SRI_ARR(DMDATA_ADDRESS_LOW, CURSOR0_, id),                               \
-      SRI_ARR(DMDATA_CNTL, CURSOR0_, id),                                      \
-      SRI_ARR(DMDATA_SW_CNTL, CURSOR0_, id),                                   \
-      SRI_ARR(DMDATA_QOS_CNTL, CURSOR0_, id),                                  \
-      SRI_ARR(DMDATA_SW_DATA, CURSOR0_, id),                                   \
-      SRI_ARR(DMDATA_STATUS, CURSOR0_, id),                                    \
-      SRI_ARR(FLIP_PARAMETERS_0, HUBPREQ, id),                                 \
-      SRI_ARR(FLIP_PARAMETERS_1, HUBPREQ, id),                                 \
-      SRI_ARR(FLIP_PARAMETERS_2, HUBPREQ, id),                                 \
-      SRI_ARR(DCN_CUR1_TTU_CNTL0, HUBPREQ, id),                                \
-      SRI_ARR(DCN_CUR1_TTU_CNTL1, HUBPREQ, id),                                \
-      SRI_ARR(DCSURF_FLIP_CONTROL2, HUBPREQ, id),                              \
-      SRI_ARR(VMID_SETTINGS_0, HUBPREQ, id)
-#define HUBP_REG_LIST_DCN21_RI(id)                                             \
-  HUBP_REG_LIST_DCN2_COMMON_RI(id), SRI_ARR(FLIP_PARAMETERS_3, HUBPREQ, id),   \
-      SRI_ARR(FLIP_PARAMETERS_4, HUBPREQ, id),                                 \
-      SRI_ARR(FLIP_PARAMETERS_5, HUBPREQ, id),                                 \
-      SRI_ARR(FLIP_PARAMETERS_6, HUBPREQ, id),                                 \
-      SRI_ARR(VBLANK_PARAMETERS_5, HUBPREQ, id),                               \
-      SRI_ARR(VBLANK_PARAMETERS_6, HUBPREQ, id)
-#define HUBP_REG_LIST_DCN30_RI(id)                                             \
-  HUBP_REG_LIST_DCN21_RI(id), SRI_ARR(DCN_DMDATA_VM_CNTL, HUBPREQ, id)
-#define HUBP_REG_LIST_DCN32_RI(id)                                             \
-  HUBP_REG_LIST_DCN30_RI(id), SRI_ARR(DCHUBP_MALL_CONFIG, HUBP, id),           \
-      SRI_ARR(DCHUBP_VMPG_CONFIG, HUBP, id),                                   \
-      SRI_ARR(UCLK_PSTATE_FORCE, HUBPREQ, id)
-
-/* HUBBUB */
-
-#define HUBBUB_REG_LIST_DCN32_RI(id)                                           \
-  SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A),                                   \
-      SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B),                               \
-      SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C),                               \
-      SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D),                               \
-      SR(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL),                                  \
-      SR(DCHUBBUB_ARB_DRAM_STATE_CNTL), SR(DCHUBBUB_ARB_SAT_LEVEL),            \
-      SR(DCHUBBUB_ARB_DF_REQ_OUTSTAND), SR(DCHUBBUB_GLOBAL_TIMER_CNTL),        \
-      SR(DCHUBBUB_SOFT_RESET), SR(DCHUBBUB_CRC_CTRL),                          \
-      SR(DCN_VM_FB_LOCATION_BASE), SR(DCN_VM_FB_LOCATION_TOP),                 \
-      SR(DCN_VM_FB_OFFSET), SR(DCN_VM_AGP_BOT), SR(DCN_VM_AGP_TOP),            \
-      SR(DCN_VM_AGP_BASE), HUBBUB_SR_WATERMARK_REG_LIST(),                     \
-      SR(DCHUBBUB_ARB_FRAC_URG_BW_NOM_A), SR(DCHUBBUB_ARB_FRAC_URG_BW_NOM_B),  \
-      SR(DCHUBBUB_ARB_FRAC_URG_BW_NOM_C), SR(DCHUBBUB_ARB_FRAC_URG_BW_NOM_D),  \
-      SR(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_A),                                     \
-      SR(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_B),                                     \
-      SR(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_C),                                     \
-      SR(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_D),                                     \
-      SR(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_A),                            \
-      SR(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_B),                            \
-      SR(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_C),                            \
-      SR(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_D), SR(DCHUBBUB_DET0_CTRL),    \
-      SR(DCHUBBUB_DET1_CTRL), SR(DCHUBBUB_DET2_CTRL), SR(DCHUBBUB_DET3_CTRL),  \
-      SR(DCHUBBUB_COMPBUF_CTRL), SR(COMPBUF_RESERVED_SPACE),                   \
-      SR(DCHUBBUB_DEBUG_CTRL_0),                                               \
-      SR(DCHUBBUB_ARB_USR_RETRAINING_CNTL),                                    \
-      SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_A),                             \
-      SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_B),                             \
-      SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_C),                             \
-      SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_D),                             \
-      SR(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_A),                         \
-      SR(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_B),                         \
-      SR(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_C),                         \
-      SR(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_D),                         \
-      SR(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_A),                         \
-      SR(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_B),                         \
-      SR(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_C),                         \
-      SR(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_D),                         \
-      SR(DCHUBBUB_ARB_MALL_CNTL),                                              \
-      SR(DCN_VM_FAULT_ADDR_MSB), SR(DCN_VM_FAULT_ADDR_LSB),                    \
-      SR(DCN_VM_FAULT_CNTL), SR(DCN_VM_FAULT_STATUS),                          \
-      SR(SDPIF_REQUEST_RATE_LIMIT)
-
-/* DCCG */
-
-#define DCCG_REG_LIST_DCN32_RI()                                               \
-  SR(DPPCLK_DTO_CTRL), DCCG_SRII(DTO_PARAM, DPPCLK, 0),                        \
-      DCCG_SRII(DTO_PARAM, DPPCLK, 1), DCCG_SRII(DTO_PARAM, DPPCLK, 2),        \
-      DCCG_SRII(DTO_PARAM, DPPCLK, 3), DCCG_SRII(CLOCK_CNTL, HDMICHARCLK, 0),  \
-      SR(PHYASYMCLK_CLOCK_CNTL), SR(PHYBSYMCLK_CLOCK_CNTL),                    \
-      SR(PHYCSYMCLK_CLOCK_CNTL), SR(PHYDSYMCLK_CLOCK_CNTL),                    \
-      SR(PHYESYMCLK_CLOCK_CNTL), SR(DPSTREAMCLK_CNTL), SR(HDMISTREAMCLK_CNTL), \
-      SR(SYMCLK32_SE_CNTL), SR(SYMCLK32_LE_CNTL),                              \
-      DCCG_SRII(PIXEL_RATE_CNTL, OTG, 0), DCCG_SRII(PIXEL_RATE_CNTL, OTG, 1),  \
-      DCCG_SRII(PIXEL_RATE_CNTL, OTG, 2), DCCG_SRII(PIXEL_RATE_CNTL, OTG, 3),  \
-      DCCG_SRII(MODULO, DTBCLK_DTO, 0), DCCG_SRII(MODULO, DTBCLK_DTO, 1),      \
-      DCCG_SRII(MODULO, DTBCLK_DTO, 2), DCCG_SRII(MODULO, DTBCLK_DTO, 3),      \
-      DCCG_SRII(PHASE, DTBCLK_DTO, 0), DCCG_SRII(PHASE, DTBCLK_DTO, 1),        \
-      DCCG_SRII(PHASE, DTBCLK_DTO, 2), DCCG_SRII(PHASE, DTBCLK_DTO, 3),        \
-      SR(DCCG_AUDIO_DTBCLK_DTO_MODULO), SR(DCCG_AUDIO_DTBCLK_DTO_PHASE),       \
-      SR(OTG_PIXEL_RATE_DIV), SR(DTBCLK_P_CNTL),                               \
-      SR(DCCG_AUDIO_DTO_SOURCE), SR(DENTIST_DISPCLK_CNTL)
-
-/* VMID */
-#define DCN20_VMID_REG_LIST_RI(id)                                             \
-  SRI_ARR(CNTL, DCN_VM_CONTEXT, id),                                           \
-      SRI_ARR(PAGE_TABLE_BASE_ADDR_HI32, DCN_VM_CONTEXT, id),                  \
-      SRI_ARR(PAGE_TABLE_BASE_ADDR_LO32, DCN_VM_CONTEXT, id),                  \
-      SRI_ARR(PAGE_TABLE_START_ADDR_HI32, DCN_VM_CONTEXT, id),                 \
-      SRI_ARR(PAGE_TABLE_START_ADDR_LO32, DCN_VM_CONTEXT, id),                 \
-      SRI_ARR(PAGE_TABLE_END_ADDR_HI32, DCN_VM_CONTEXT, id),                   \
-      SRI_ARR(PAGE_TABLE_END_ADDR_LO32, DCN_VM_CONTEXT, id)
-
-/* I2C HW */
-
-#define I2C_HW_ENGINE_COMMON_REG_LIST_RI(id)                                   \
-      SRI_ARR_I2C(SETUP, DC_I2C_DDC, id), SRI_ARR_I2C(SPEED, DC_I2C_DDC, id),  \
-      SRI_ARR_I2C(HW_STATUS, DC_I2C_DDC, id),                                  \
-      SR_ARR_I2C(DC_I2C_ARBITRATION, id),                                      \
-      SR_ARR_I2C(DC_I2C_CONTROL, id), SR_ARR_I2C(DC_I2C_SW_STATUS, id),        \
-      SR_ARR_I2C(DC_I2C_TRANSACTION0, id), SR_ARR_I2C(DC_I2C_TRANSACTION1, id),\
-      SR_ARR_I2C(DC_I2C_TRANSACTION2, id), SR_ARR_I2C(DC_I2C_TRANSACTION3, id),\
-      SR_ARR_I2C(DC_I2C_DATA, id), SR_ARR_I2C(MICROSECOND_TIME_BASE_DIV, id)
-
-#define I2C_HW_ENGINE_COMMON_REG_LIST_DCN30_RI(id)                             \
-      I2C_HW_ENGINE_COMMON_REG_LIST_RI(id), SR_ARR_I2C(DIO_MEM_PWR_CTRL, id),  \
-      SR_ARR_I2C(DIO_MEM_PWR_STATUS, id)
-
-#endif /* _DCN32_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c
index bc5f0db23d0c..ef0a2b01734d 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c
@@ -24,7 +24,7 @@
  */
 
 // header file of functions being implemented
-#include "dcn32_resource.h"
+#include "dcn32/dcn32_resource.h"
 #include "dcn20/dcn20_resource.h"
 #include "dml/dcn32/display_mode_vba_util_32.h"
 #include "dml/dcn32/dcn32_fpu.h"
diff --git a/drivers/gpu/drm/amd/display/dc/dcn321/Makefile b/drivers/gpu/drm/amd/display/dc/dcn321/Makefile
index 0a199c83bb5b..c195c47f58b4 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn321/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn321/Makefile
@@ -10,7 +10,7 @@
 #
 # Makefile for dcn321.
 
-DCN321 = dcn321_resource.o dcn321_dio_link_encoder.o
+DCN321 = dcn321_dio_link_encoder.o
 
 AMD_DAL_DCN321 = $(addprefix $(AMDDALPATH)/dc/dcn321/,$(DCN321))
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c b/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c
deleted file mode 100644
index f7de3eca1225..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c
+++ /dev/null
@@ -1,2065 +0,0 @@
-// SPDX-License-Identifier: MIT
-/*
- * Copyright 2019 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#include "dm_services.h"
-#include "dc.h"
-
-#include "dcn32/dcn32_init.h"
-
-#include "resource.h"
-#include "include/irq_service_interface.h"
-#include "dcn32/dcn32_resource.h"
-#include "dcn321_resource.h"
-
-#include "dcn20/dcn20_resource.h"
-#include "dcn30/dcn30_resource.h"
-
-#include "dml/dcn321/dcn321_fpu.h"
-
-#include "dcn10/dcn10_ipp.h"
-#include "dcn30/dcn30_hubbub.h"
-#include "dcn31/dcn31_hubbub.h"
-#include "dcn32/dcn32_hubbub.h"
-#include "dcn32/dcn32_mpc.h"
-#include "dcn32/dcn32_hubp.h"
-#include "irq/dcn32/irq_service_dcn32.h"
-#include "dcn32/dcn32_dpp.h"
-#include "dcn32/dcn32_optc.h"
-#include "dcn20/dcn20_hwseq.h"
-#include "dcn30/dcn30_hwseq.h"
-#include "dce110/dce110_hwseq.h"
-#include "dcn30/dcn30_opp.h"
-#include "dcn20/dcn20_dsc.h"
-#include "dcn30/dcn30_vpg.h"
-#include "dcn30/dcn30_afmt.h"
-#include "dcn30/dcn30_dio_stream_encoder.h"
-#include "dcn32/dcn32_dio_stream_encoder.h"
-#include "dcn31/dcn31_hpo_dp_stream_encoder.h"
-#include "dcn31/dcn31_hpo_dp_link_encoder.h"
-#include "dcn32/dcn32_hpo_dp_link_encoder.h"
-#include "dcn31/dcn31_apg.h"
-#include "dcn31/dcn31_dio_link_encoder.h"
-#include "dcn32/dcn32_dio_link_encoder.h"
-#include "dcn321_dio_link_encoder.h"
-#include "dce/dce_clock_source.h"
-#include "dce/dce_audio.h"
-#include "dce/dce_hwseq.h"
-#include "clk_mgr.h"
-#include "virtual/virtual_stream_encoder.h"
-#include "dml/display_mode_vba.h"
-#include "dcn32/dcn32_dccg.h"
-#include "dcn10/dcn10_resource.h"
-#include "link.h"
-#include "dcn31/dcn31_panel_cntl.h"
-
-#include "dcn30/dcn30_dwb.h"
-#include "dcn32/dcn32_mmhubbub.h"
-
-#include "dcn/dcn_3_2_1_offset.h"
-#include "dcn/dcn_3_2_1_sh_mask.h"
-#include "nbio/nbio_4_3_0_offset.h"
-
-#include "reg_helper.h"
-#include "dce/dmub_abm.h"
-#include "dce/dmub_psr.h"
-#include "dce/dce_aux.h"
-#include "dce/dce_i2c.h"
-
-#include "dml/dcn30/display_mode_vba_30.h"
-#include "vm_helper.h"
-#include "dcn20/dcn20_vmid.h"
-
-#define DC_LOGGER_INIT(logger)
-
-enum dcn321_clk_src_array_id {
-	DCN321_CLK_SRC_PLL0,
-	DCN321_CLK_SRC_PLL1,
-	DCN321_CLK_SRC_PLL2,
-	DCN321_CLK_SRC_PLL3,
-	DCN321_CLK_SRC_PLL4,
-	DCN321_CLK_SRC_TOTAL
-};
-
-/* begin *********************
- * macros to expend register list macro defined in HW object header file
- */
-
-/* DCN */
-#define BASE_INNER(seg) ctx->dcn_reg_offsets[seg]
-
-#define BASE(seg) BASE_INNER(seg)
-
-#define SR(reg_name)\
-	REG_STRUCT.reg_name = BASE(reg ## reg_name ## _BASE_IDX) +  \
-		reg ## reg_name
-#define SR_ARR(reg_name, id)\
-	REG_STRUCT[id].reg_name = BASE(reg ## reg_name ## _BASE_IDX) +  \
-		reg ## reg_name
-#define SR_ARR_INIT(reg_name, id, value)\
-	REG_STRUCT[id].reg_name =  value
-
-#define SRI(reg_name, block, id)\
-	REG_STRUCT.reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define SRI_ARR(reg_name, block, id)\
-	REG_STRUCT[id].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define SR_ARR_I2C(reg_name, id) \
-	REG_STRUCT[id-1].reg_name = BASE(reg##reg_name##_BASE_IDX) + reg##reg_name
-
-#define SRI_ARR_I2C(reg_name, block, id)\
-	REG_STRUCT[id-1].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define SRI_ARR_ALPHABET(reg_name, block, index, id)\
-	REG_STRUCT[index].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define SRI2(reg_name, block, id)\
-	.reg_name = BASE(reg ## reg_name ## _BASE_IDX) + \
-		reg ## reg_name
-#define SRI2_ARR(reg_name, block, id)\
-	REG_STRUCT[id].reg_name = BASE(reg ## reg_name ## _BASE_IDX) + \
-		reg ## reg_name
-
-#define SRIR(var_name, reg_name, block, id)\
-	.var_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define SRII(reg_name, block, id)\
-	REG_STRUCT.reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define SRII_ARR_2(reg_name, block, id, inst)\
-	REG_STRUCT[inst].reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define SRII_MPC_RMU(reg_name, block, id)\
-	.RMU##_##reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define SRII_DWB(reg_name, temp_name, block, id)\
-	REG_STRUCT.reg_name[id] = BASE(reg ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## temp_name
-
-#define DCCG_SRII(reg_name, block, id)\
-	REG_STRUCT.block ## _ ## reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define SF_DWB2(reg_name, block, id, field_name, post_fix) \
-	.field_name = reg_name ## __ ## field_name ## post_fix
-
-#define VUPDATE_SRII(reg_name, block, id)\
-	REG_STRUCT.reg_name[id] = BASE(reg ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
-		reg ## reg_name ## _ ## block ## id
-
-/* NBIO */
-#define NBIO_BASE_INNER(seg) ctx->nbio_reg_offsets[seg]
-
-#define NBIO_BASE(seg) \
-	NBIO_BASE_INNER(seg)
-
-#define NBIO_SR(reg_name)\
-	REG_STRUCT.reg_name = NBIO_BASE(regBIF_BX0_ ## reg_name ## _BASE_IDX) + \
-		regBIF_BX0_ ## reg_name
-#define NBIO_SR_ARR(reg_name, id)\
-	REG_STRUCT[id].reg_name = NBIO_BASE(regBIF_BX0_ ## reg_name ## _BASE_IDX) + \
-		regBIF_BX0_ ## reg_name
-
-#define CTX ctx
-#define REG(reg_name) \
-	(ctx->dcn_reg_offsets[reg ## reg_name ## _BASE_IDX] + reg ## reg_name)
-
-static struct bios_registers bios_regs;
-
-#define bios_regs_init() \
-		( \
-		NBIO_SR(BIOS_SCRATCH_3),\
-		NBIO_SR(BIOS_SCRATCH_6)\
-		)
-
-#define clk_src_regs_init(index, pllid)\
-	CS_COMMON_REG_LIST_DCN3_0_RI(index, pllid)
-
-static struct dce110_clk_src_regs clk_src_regs[5];
-
-static const struct dce110_clk_src_shift cs_shift = {
-		CS_COMMON_MASK_SH_LIST_DCN3_2(__SHIFT)
-};
-
-static const struct dce110_clk_src_mask cs_mask = {
-		CS_COMMON_MASK_SH_LIST_DCN3_2(_MASK)
-};
-
-#define abm_regs_init(id)\
-		ABM_DCN32_REG_LIST_RI(id)
-
-static struct dce_abm_registers abm_regs[4];
-
-static const struct dce_abm_shift abm_shift = {
-		ABM_MASK_SH_LIST_DCN32(__SHIFT)
-};
-
-static const struct dce_abm_mask abm_mask = {
-		ABM_MASK_SH_LIST_DCN32(_MASK)
-};
-
-#define audio_regs_init(id)\
-		AUD_COMMON_REG_LIST_RI(id)
-
-static struct dce_audio_registers audio_regs[5];
-
-#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
-		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
-
-static const struct dce_audio_shift audio_shift = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_audio_mask audio_mask = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
-};
-
-#define vpg_regs_init(id)\
-	VPG_DCN3_REG_LIST_RI(id)
-
-static struct dcn30_vpg_registers vpg_regs[10];
-
-static const struct dcn30_vpg_shift vpg_shift = {
-	DCN3_VPG_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn30_vpg_mask vpg_mask = {
-	DCN3_VPG_MASK_SH_LIST(_MASK)
-};
-
-#define afmt_regs_init(id)\
-	AFMT_DCN3_REG_LIST_RI(id)
-
-static struct dcn30_afmt_registers afmt_regs[6];
-
-static const struct dcn30_afmt_shift afmt_shift = {
-	DCN3_AFMT_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn30_afmt_mask afmt_mask = {
-	DCN3_AFMT_MASK_SH_LIST(_MASK)
-};
-
-#define apg_regs_init(id)\
-	APG_DCN31_REG_LIST_RI(id)
-
-static struct dcn31_apg_registers apg_regs[4];
-
-static const struct dcn31_apg_shift apg_shift = {
-	DCN31_APG_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_apg_mask apg_mask = {
-		DCN31_APG_MASK_SH_LIST(_MASK)
-};
-
-#define stream_enc_regs_init(id)\
-	SE_DCN32_REG_LIST_RI(id)
-
-static struct dcn10_stream_enc_registers stream_enc_regs[5];
-
-static const struct dcn10_stream_encoder_shift se_shift = {
-		SE_COMMON_MASK_SH_LIST_DCN32(__SHIFT)
-};
-
-static const struct dcn10_stream_encoder_mask se_mask = {
-		SE_COMMON_MASK_SH_LIST_DCN32(_MASK)
-};
-
-
-#define aux_regs_init(id)\
-	DCN2_AUX_REG_LIST_RI(id)
-
-static struct dcn10_link_enc_aux_registers link_enc_aux_regs[5];
-
-#define hpd_regs_init(id)\
-	HPD_REG_LIST_RI(id)
-
-static struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[5];
-
-#define link_regs_init(id, phyid)\
-	( \
-	LE_DCN31_REG_LIST_RI(id), \
-	UNIPHY_DCN2_REG_LIST_RI(id, phyid)\
-	)
-	/*DPCS_DCN31_REG_LIST(id),*/ \
-
-static struct dcn10_link_enc_registers link_enc_regs[5];
-
-static const struct dcn10_link_enc_shift le_shift = {
-	LINK_ENCODER_MASK_SH_LIST_DCN31(__SHIFT), \
-//	DPCS_DCN31_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn10_link_enc_mask le_mask = {
-	LINK_ENCODER_MASK_SH_LIST_DCN31(_MASK), \
-//	DPCS_DCN31_MASK_SH_LIST(_MASK)
-};
-
-#define hpo_dp_stream_encoder_reg_init(id)\
-	DCN3_1_HPO_DP_STREAM_ENC_REG_LIST_RI(id)
-
-static struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[4];
-
-static const struct dcn31_hpo_dp_stream_encoder_shift hpo_dp_se_shift = {
-	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_hpo_dp_stream_encoder_mask hpo_dp_se_mask = {
-	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(_MASK)
-};
-
-
-#define hpo_dp_link_encoder_reg_init(id)\
-	DCN3_1_HPO_DP_LINK_ENC_REG_LIST_RI(id)
-	/*DCN3_1_RDPCSTX_REG_LIST(0),*/
-	/*DCN3_1_RDPCSTX_REG_LIST(1),*/
-	/*DCN3_1_RDPCSTX_REG_LIST(2),*/
-	/*DCN3_1_RDPCSTX_REG_LIST(3),*/
-
-static struct dcn31_hpo_dp_link_encoder_registers hpo_dp_link_enc_regs[2];
-
-static const struct dcn31_hpo_dp_link_encoder_shift hpo_dp_le_shift = {
-	DCN3_2_HPO_DP_LINK_ENC_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_hpo_dp_link_encoder_mask hpo_dp_le_mask = {
-	DCN3_2_HPO_DP_LINK_ENC_MASK_SH_LIST(_MASK)
-};
-
-#define dpp_regs_init(id)\
-	DPP_REG_LIST_DCN30_COMMON_RI(id)
-
-static struct dcn3_dpp_registers dpp_regs[4];
-
-static const struct dcn3_dpp_shift tf_shift = {
-		DPP_REG_LIST_SH_MASK_DCN30_COMMON(__SHIFT)
-};
-
-static const struct dcn3_dpp_mask tf_mask = {
-		DPP_REG_LIST_SH_MASK_DCN30_COMMON(_MASK)
-};
-
-
-#define opp_regs_init(id)\
-	OPP_REG_LIST_DCN30_RI(id)
-
-static struct dcn20_opp_registers opp_regs[4];
-
-static const struct dcn20_opp_shift opp_shift = {
-	OPP_MASK_SH_LIST_DCN20(__SHIFT)
-};
-
-static const struct dcn20_opp_mask opp_mask = {
-	OPP_MASK_SH_LIST_DCN20(_MASK)
-};
-
-#define aux_engine_regs_init(id) \
-	( \
-	AUX_COMMON_REG_LIST0_RI(id), SR_ARR_INIT(AUXN_IMPCAL, id, 0), \
-	SR_ARR_INIT(AUXP_IMPCAL, id, 0), \
-	SR_ARR_INIT(AUX_RESET_MASK, id, DP_AUX0_AUX_CONTROL__AUX_RESET_MASK), \
-	SR_ARR_INIT(AUX_RESET_MASK, id, DP_AUX0_AUX_CONTROL__AUX_RESET_MASK)\
-	)
-
-static struct dce110_aux_registers aux_engine_regs[5];
-
-static const struct dce110_aux_registers_shift aux_shift = {
-	DCN_AUX_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce110_aux_registers_mask aux_mask = {
-	DCN_AUX_MASK_SH_LIST(_MASK)
-};
-
-#define dwbc_regs_dcn3_init(id)\
-	DWBC_COMMON_REG_LIST_DCN30_RI(id)
-
-static struct dcn30_dwbc_registers dwbc30_regs[1];
-
-static const struct dcn30_dwbc_shift dwbc30_shift = {
-	DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dcn30_dwbc_mask dwbc30_mask = {
-	DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-#define mcif_wb_regs_dcn3_init(id)\
-	MCIF_WB_COMMON_REG_LIST_DCN32_RI(id)
-
-static struct dcn30_mmhubbub_registers mcif_wb30_regs[1];
-
-static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
-	MCIF_WB_COMMON_MASK_SH_LIST_DCN32(__SHIFT)
-};
-
-static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
-	MCIF_WB_COMMON_MASK_SH_LIST_DCN32(_MASK)
-};
-
-#define dsc_regsDCN20_init(id)\
-	DSC_REG_LIST_DCN20_RI(id)
-
-static struct dcn20_dsc_registers dsc_regs[4];
-
-static const struct dcn20_dsc_shift dsc_shift = {
-	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
-};
-
-static const struct dcn20_dsc_mask dsc_mask = {
-	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
-};
-
-static struct dcn30_mpc_registers mpc_regs;
-#define dcn_mpc_regs_init()\
-	MPC_REG_LIST_DCN3_2_RI(0),\
-	MPC_REG_LIST_DCN3_2_RI(1),\
-	MPC_REG_LIST_DCN3_2_RI(2),\
-	MPC_REG_LIST_DCN3_2_RI(3),\
-	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(0),\
-	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(1),\
-	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(2),\
-	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(3),\
-	MPC_DWB_MUX_REG_LIST_DCN3_0_RI(0)
-
-static const struct dcn30_mpc_shift mpc_shift = {
-	MPC_COMMON_MASK_SH_LIST_DCN32(__SHIFT)
-};
-
-static const struct dcn30_mpc_mask mpc_mask = {
-	MPC_COMMON_MASK_SH_LIST_DCN32(_MASK)
-};
-
-#define optc_regs_init(id)\
-	OPTC_COMMON_REG_LIST_DCN3_2_RI(id)
-
-static struct dcn_optc_registers optc_regs[4];
-
-static const struct dcn_optc_shift optc_shift = {
-	OPTC_COMMON_MASK_SH_LIST_DCN3_2(__SHIFT)
-};
-
-static const struct dcn_optc_mask optc_mask = {
-	OPTC_COMMON_MASK_SH_LIST_DCN3_2(_MASK)
-};
-
-#define hubp_regs_init(id) \
-	HUBP_REG_LIST_DCN32_RI(id)
-
-static struct dcn_hubp2_registers hubp_regs[4];
-
-static const struct dcn_hubp2_shift hubp_shift = {
-		HUBP_MASK_SH_LIST_DCN32(__SHIFT)
-};
-
-static const struct dcn_hubp2_mask hubp_mask = {
-		HUBP_MASK_SH_LIST_DCN32(_MASK)
-};
-
-static struct dcn_hubbub_registers hubbub_reg;
-#define hubbub_reg_init()\
-		HUBBUB_REG_LIST_DCN32_RI(0)
-
-static const struct dcn_hubbub_shift hubbub_shift = {
-		HUBBUB_MASK_SH_LIST_DCN32(__SHIFT)
-};
-
-static const struct dcn_hubbub_mask hubbub_mask = {
-		HUBBUB_MASK_SH_LIST_DCN32(_MASK)
-};
-
-static struct dccg_registers dccg_regs;
-
-#define dccg_regs_init()\
-	DCCG_REG_LIST_DCN32_RI()
-
-static const struct dccg_shift dccg_shift = {
-		DCCG_MASK_SH_LIST_DCN32(__SHIFT)
-};
-
-static const struct dccg_mask dccg_mask = {
-		DCCG_MASK_SH_LIST_DCN32(_MASK)
-};
-
-
-#define SRII2(reg_name_pre, reg_name_post, id)\
-	.reg_name_pre ## _ ##  reg_name_post[id] = BASE(reg ## reg_name_pre \
-			## id ## _ ## reg_name_post ## _BASE_IDX) + \
-			reg ## reg_name_pre ## id ## _ ## reg_name_post
-
-
-#define HWSEQ_DCN32_REG_LIST()\
-	SR(DCHUBBUB_GLOBAL_TIMER_CNTL), \
-	SR(DIO_MEM_PWR_CTRL), \
-	SR(ODM_MEM_PWR_CTRL3), \
-	SR(MMHUBBUB_MEM_PWR_CNTL), \
-	SR(DCCG_GATE_DISABLE_CNTL), \
-	SR(DCCG_GATE_DISABLE_CNTL2), \
-	SR(DCFCLK_CNTL),\
-	SR(DC_MEM_GLOBAL_PWR_REQ_CNTL), \
-	SRII(PIXEL_RATE_CNTL, OTG, 0), \
-	SRII(PIXEL_RATE_CNTL, OTG, 1),\
-	SRII(PIXEL_RATE_CNTL, OTG, 2),\
-	SRII(PIXEL_RATE_CNTL, OTG, 3),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 0),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 1),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 2),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 3),\
-	SR(MICROSECOND_TIME_BASE_DIV), \
-	SR(MILLISECOND_TIME_BASE_DIV), \
-	SR(DISPCLK_FREQ_CHANGE_CNTL), \
-	SR(RBBMIF_TIMEOUT_DIS), \
-	SR(RBBMIF_TIMEOUT_DIS_2), \
-	SR(DCHUBBUB_CRC_CTRL), \
-	SR(DPP_TOP0_DPP_CRC_CTRL), \
-	SR(DPP_TOP0_DPP_CRC_VAL_B_A), \
-	SR(DPP_TOP0_DPP_CRC_VAL_R_G), \
-	SR(MPC_CRC_CTRL), \
-	SR(MPC_CRC_RESULT_GB), \
-	SR(MPC_CRC_RESULT_C), \
-	SR(MPC_CRC_RESULT_AR), \
-	SR(DOMAIN0_PG_CONFIG), \
-	SR(DOMAIN1_PG_CONFIG), \
-	SR(DOMAIN2_PG_CONFIG), \
-	SR(DOMAIN3_PG_CONFIG), \
-	SR(DOMAIN16_PG_CONFIG), \
-	SR(DOMAIN17_PG_CONFIG), \
-	SR(DOMAIN18_PG_CONFIG), \
-	SR(DOMAIN19_PG_CONFIG), \
-	SR(DOMAIN0_PG_STATUS), \
-	SR(DOMAIN1_PG_STATUS), \
-	SR(DOMAIN2_PG_STATUS), \
-	SR(DOMAIN3_PG_STATUS), \
-	SR(DOMAIN16_PG_STATUS), \
-	SR(DOMAIN17_PG_STATUS), \
-	SR(DOMAIN18_PG_STATUS), \
-	SR(DOMAIN19_PG_STATUS), \
-	SR(D1VGA_CONTROL), \
-	SR(D2VGA_CONTROL), \
-	SR(D3VGA_CONTROL), \
-	SR(D4VGA_CONTROL), \
-	SR(D5VGA_CONTROL), \
-	SR(D6VGA_CONTROL), \
-	SR(DC_IP_REQUEST_CNTL), \
-	SR(AZALIA_AUDIO_DTO), \
-	SR(AZALIA_CONTROLLER_CLOCK_GATING)
-
-static struct dce_hwseq_registers hwseq_reg;
-
-#define hwseq_reg_init()\
-	HWSEQ_DCN32_REG_LIST()
-
-#define HWSEQ_DCN32_MASK_SH_LIST(mask_sh)\
-	HWSEQ_DCN_MASK_SH_LIST(mask_sh), \
-	HWS_SF(, DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, mask_sh), \
-	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN0_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN1_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN2_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN3_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN16_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN17_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN18_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN19_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh), \
-	HWS_SF(, AZALIA_AUDIO_DTO, AZALIA_AUDIO_DTO_MODULE, mask_sh), \
-	HWS_SF(, HPO_TOP_CLOCK_CONTROL, HPO_HDMISTREAMCLK_G_GATE_DIS, mask_sh), \
-	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_UNASSIGNED_PWR_MODE, mask_sh), \
-	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_VBLANK_PWR_MODE, mask_sh), \
-	HWS_SF(, MMHUBBUB_MEM_PWR_CNTL, VGA_MEM_PWR_FORCE, mask_sh)
-
-static const struct dce_hwseq_shift hwseq_shift = {
-		HWSEQ_DCN32_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_hwseq_mask hwseq_mask = {
-		HWSEQ_DCN32_MASK_SH_LIST(_MASK)
-};
-#define vmid_regs_init(id)\
-		DCN20_VMID_REG_LIST_RI(id)
-
-static struct dcn_vmid_registers vmid_regs[16];
-
-static const struct dcn20_vmid_shift vmid_shifts = {
-		DCN20_VMID_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn20_vmid_mask vmid_masks = {
-		DCN20_VMID_MASK_SH_LIST(_MASK)
-};
-
-static const struct resource_caps res_cap_dcn321 = {
-	.num_timing_generator = 4,
-	.num_opp = 4,
-	.num_video_plane = 4,
-	.num_audio = 5,
-	.num_stream_encoder = 5,
-	.num_hpo_dp_stream_encoder = 4,
-	.num_hpo_dp_link_encoder = 2,
-	.num_pll = 5,
-	.num_dwb = 1,
-	.num_ddc = 5,
-	.num_vmid = 16,
-	.num_mpc_3dlut = 4,
-	.num_dsc = 4,
-};
-
-static const struct dc_plane_cap plane_cap = {
-	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.per_pixel_alpha = true,
-
-	.pixel_format_support = {
-			.argb8888 = true,
-			.nv12 = true,
-			.fp16 = true,
-			.p010 = true,
-			.ayuv = false,
-	},
-
-	.max_upscale_factor = {
-			.argb8888 = 16000,
-			.nv12 = 16000,
-			.fp16 = 16000
-	},
-
-	// 6:1 downscaling ratio: 1000/6 = 166.666
-	.max_downscale_factor = {
-			.argb8888 = 167,
-			.nv12 = 167,
-			.fp16 = 167
-	},
-	64,
-	64
-};
-
-static const struct dc_debug_options debug_defaults_drv = {
-	.disable_dmcu = true,
-	.force_abm_enable = false,
-	.timing_trace = false,
-	.clock_trace = true,
-	.disable_pplib_clock_request = false,
-	.pipe_split_policy = MPC_SPLIT_AVOID,
-	.force_single_disp_pipe_split = false,
-	.disable_dcc = DCC_ENABLE,
-	.vsr_support = true,
-	.performance_trace = false,
-	.max_downscale_src_width = 7680,/*upto 8K*/
-	.disable_pplib_wm_range = false,
-	.scl_reset_length10 = true,
-	.sanity_checks = false,
-	.underflow_assert_delay_us = 0xFFFFFFFF,
-	.dwb_fi_phase = -1, // -1 = disable,
-	.dmub_command_table = true,
-	.enable_mem_low_power = {
-		.bits = {
-			.vga = false,
-			.i2c = false,
-			.dmcu = false, // This is previously known to cause hang on S3 cycles if enabled
-			.dscl = false,
-			.cm = false,
-			.mpc = false,
-			.optc = true,
-		}
-	},
-	.use_max_lb = true,
-	.force_disable_subvp = false,
-	.exit_idle_opt_for_cursor_updates = true,
-	.enable_single_display_2to1_odm_policy = true,
-
-	/*must match enable_single_display_2to1_odm_policy to support dynamic ODM transitions*/
-	.enable_double_buffered_dsc_pg_support = true,
-	.enable_dp_dig_pixel_rate_div_policy = 1,
-	.allow_sw_cursor_fallback = false, // Linux can't do SW cursor "fallback"
-	.alloc_extra_way_for_cursor = true,
-	.min_prefetch_in_strobe_ns = 60000, // 60us
-	.disable_unbounded_requesting = false,
-	.override_dispclk_programming = true,
-	.disable_fpo_optimizations = false,
-	.fpo_vactive_margin_us = 2000, // 2000us
-	.disable_fpo_vactive = false,
-	.disable_boot_optimizations = false,
-	.disable_subvp_high_refresh = false,
-	.fpo_vactive_min_active_margin_us = 200,
-	.fpo_vactive_max_blank_us = 1000,
-	.enable_legacy_fast_update = false,
-	.disable_dc_mode_overwrite = true,
-	.using_dml2 = false,
-};
-
-static struct dce_aux *dcn321_aux_engine_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct aux_engine_dce110 *aux_engine =
-		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
-
-	if (!aux_engine)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT aux_engine_regs
-	aux_engine_regs_init(0),
-	aux_engine_regs_init(1),
-	aux_engine_regs_init(2),
-	aux_engine_regs_init(3),
-	aux_engine_regs_init(4);
-
-	dce110_aux_engine_construct(aux_engine, ctx, inst,
-				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
-				    &aux_engine_regs[inst],
-					&aux_mask,
-					&aux_shift,
-					ctx->dc->caps.extended_aux_timeout_support);
-
-	return &aux_engine->base;
-}
-#define i2c_inst_regs_init(id)\
-	I2C_HW_ENGINE_COMMON_REG_LIST_DCN30_RI(id)
-
-static struct dce_i2c_registers i2c_hw_regs[5];
-
-static const struct dce_i2c_shift i2c_shifts = {
-		I2C_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
-};
-
-static const struct dce_i2c_mask i2c_masks = {
-		I2C_COMMON_MASK_SH_LIST_DCN30(_MASK)
-};
-
-static struct dce_i2c_hw *dcn321_i2c_hw_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_i2c_hw *dce_i2c_hw =
-		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
-
-	if (!dce_i2c_hw)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT i2c_hw_regs
-	i2c_inst_regs_init(1),
-	i2c_inst_regs_init(2),
-	i2c_inst_regs_init(3),
-	i2c_inst_regs_init(4),
-	i2c_inst_regs_init(5);
-
-	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
-				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
-
-	return dce_i2c_hw;
-}
-
-static struct clock_source *dcn321_clock_source_create(
-		struct dc_context *ctx,
-		struct dc_bios *bios,
-		enum clock_source_id id,
-		const struct dce110_clk_src_regs *regs,
-		bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src =
-		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dcn31_clk_src_construct(clk_src, ctx, bios, id,
-			regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-
-	kfree(clk_src);
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
-static struct hubbub *dcn321_hubbub_create(struct dc_context *ctx)
-{
-	int i;
-
-	struct dcn20_hubbub *hubbub2 = kzalloc(sizeof(struct dcn20_hubbub),
-					  GFP_KERNEL);
-
-	if (!hubbub2)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT hubbub_reg
-	hubbub_reg_init();
-
-#undef REG_STRUCT
-#define REG_STRUCT vmid_regs
-	vmid_regs_init(0),
-	vmid_regs_init(1),
-	vmid_regs_init(2),
-	vmid_regs_init(3),
-	vmid_regs_init(4),
-	vmid_regs_init(5),
-	vmid_regs_init(6),
-	vmid_regs_init(7),
-	vmid_regs_init(8),
-	vmid_regs_init(9),
-	vmid_regs_init(10),
-	vmid_regs_init(11),
-	vmid_regs_init(12),
-	vmid_regs_init(13),
-	vmid_regs_init(14),
-	vmid_regs_init(15);
-
-	hubbub32_construct(hubbub2, ctx,
-			&hubbub_reg,
-			&hubbub_shift,
-			&hubbub_mask,
-			ctx->dc->dml.ip.det_buffer_size_kbytes,
-			ctx->dc->dml.ip.pixel_chunk_size_kbytes,
-			ctx->dc->dml.ip.config_return_buffer_size_in_kbytes);
-
-
-	for (i = 0; i < res_cap_dcn321.num_vmid; i++) {
-		struct dcn20_vmid *vmid = &hubbub2->vmid[i];
-
-		vmid->ctx = ctx;
-
-		vmid->regs = &vmid_regs[i];
-		vmid->shifts = &vmid_shifts;
-		vmid->masks = &vmid_masks;
-	}
-
-	return &hubbub2->base;
-}
-
-static struct hubp *dcn321_hubp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn20_hubp *hubp2 =
-		kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
-
-	if (!hubp2)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT hubp_regs
-	hubp_regs_init(0),
-	hubp_regs_init(1),
-	hubp_regs_init(2),
-	hubp_regs_init(3);
-
-	if (hubp32_construct(hubp2, ctx, inst,
-			&hubp_regs[inst], &hubp_shift, &hubp_mask))
-		return &hubp2->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(hubp2);
-	return NULL;
-}
-
-static void dcn321_dpp_destroy(struct dpp **dpp)
-{
-	kfree(TO_DCN30_DPP(*dpp));
-	*dpp = NULL;
-}
-
-static struct dpp *dcn321_dpp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn3_dpp *dpp3 =
-		kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
-
-	if (!dpp3)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT dpp_regs
-	dpp_regs_init(0),
-	dpp_regs_init(1),
-	dpp_regs_init(2),
-	dpp_regs_init(3);
-
-	if (dpp32_construct(dpp3, ctx, inst,
-			&dpp_regs[inst], &tf_shift, &tf_mask))
-		return &dpp3->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(dpp3);
-	return NULL;
-}
-
-static struct mpc *dcn321_mpc_create(
-		struct dc_context *ctx,
-		int num_mpcc,
-		int num_rmu)
-{
-	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc),
-					  GFP_KERNEL);
-
-	if (!mpc30)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT mpc_regs
-	dcn_mpc_regs_init();
-
-	dcn32_mpc_construct(mpc30, ctx,
-			&mpc_regs,
-			&mpc_shift,
-			&mpc_mask,
-			num_mpcc,
-			num_rmu);
-
-	return &mpc30->base;
-}
-
-static struct output_pixel_processor *dcn321_opp_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_opp *opp2 =
-		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
-
-	if (!opp2) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-#undef REG_STRUCT
-#define REG_STRUCT opp_regs
-	opp_regs_init(0),
-	opp_regs_init(1),
-	opp_regs_init(2),
-	opp_regs_init(3);
-
-	dcn20_opp_construct(opp2, ctx, inst,
-			&opp_regs[inst], &opp_shift, &opp_mask);
-	return &opp2->base;
-}
-
-
-static struct timing_generator *dcn321_timing_generator_create(
-		struct dc_context *ctx,
-		uint32_t instance)
-{
-	struct optc *tgn10 =
-		kzalloc(sizeof(struct optc), GFP_KERNEL);
-
-	if (!tgn10)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT optc_regs
-	optc_regs_init(0),
-	optc_regs_init(1),
-	optc_regs_init(2),
-	optc_regs_init(3);
-
-	tgn10->base.inst = instance;
-	tgn10->base.ctx = ctx;
-
-	tgn10->tg_regs = &optc_regs[instance];
-	tgn10->tg_shift = &optc_shift;
-	tgn10->tg_mask = &optc_mask;
-
-	dcn32_timing_generator_init(tgn10);
-
-	return &tgn10->base;
-}
-
-static const struct encoder_feature_support link_enc_feature = {
-		.max_hdmi_deep_color = COLOR_DEPTH_121212,
-		.max_hdmi_pixel_clock = 600000,
-		.hdmi_ycbcr420_supported = true,
-		.dp_ycbcr420_supported = true,
-		.fec_supported = true,
-		.flags.bits.IS_HBR2_CAPABLE = true,
-		.flags.bits.IS_HBR3_CAPABLE = true,
-		.flags.bits.IS_TPS3_CAPABLE = true,
-		.flags.bits.IS_TPS4_CAPABLE = true
-};
-
-static struct link_encoder *dcn321_link_encoder_create(
-	struct dc_context *ctx,
-	const struct encoder_init_data *enc_init_data)
-{
-	struct dcn20_link_encoder *enc20 =
-		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
-
-	if (!enc20)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT link_enc_aux_regs
-	aux_regs_init(0),
-	aux_regs_init(1),
-	aux_regs_init(2),
-	aux_regs_init(3),
-	aux_regs_init(4);
-
-#undef REG_STRUCT
-#define REG_STRUCT link_enc_hpd_regs
-	hpd_regs_init(0),
-	hpd_regs_init(1),
-	hpd_regs_init(2),
-	hpd_regs_init(3),
-	hpd_regs_init(4);
-
-#undef REG_STRUCT
-#define REG_STRUCT link_enc_regs
-	link_regs_init(0, A),
-	link_regs_init(1, B),
-	link_regs_init(2, C),
-	link_regs_init(3, D),
-	link_regs_init(4, E);
-
-	dcn321_link_encoder_construct(enc20,
-			enc_init_data,
-			&link_enc_feature,
-			&link_enc_regs[enc_init_data->transmitter],
-			&link_enc_aux_regs[enc_init_data->channel - 1],
-			&link_enc_hpd_regs[enc_init_data->hpd_source],
-			&le_shift,
-			&le_mask);
-
-	return &enc20->enc10.base;
-}
-
-static void read_dce_straps(
-	struct dc_context *ctx,
-	struct resource_straps *straps)
-{
-	generic_reg_get(ctx, ctx->dcn_reg_offsets[regDC_PINSTRAPS_BASE_IDX] + regDC_PINSTRAPS,
-		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
-
-}
-
-static struct audio *dcn321_create_audio(
-		struct dc_context *ctx, unsigned int inst)
-{
-
-#undef REG_STRUCT
-#define REG_STRUCT audio_regs
-	audio_regs_init(0),
-	audio_regs_init(1),
-	audio_regs_init(2),
-	audio_regs_init(3),
-	audio_regs_init(4);
-
-	return dce_audio_create(ctx, inst,
-			&audio_regs[inst], &audio_shift, &audio_mask);
-}
-
-static struct vpg *dcn321_vpg_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn30_vpg *vpg3 = kzalloc(sizeof(struct dcn30_vpg), GFP_KERNEL);
-
-	if (!vpg3)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT vpg_regs
-	vpg_regs_init(0),
-	vpg_regs_init(1),
-	vpg_regs_init(2),
-	vpg_regs_init(3),
-	vpg_regs_init(4),
-	vpg_regs_init(5),
-	vpg_regs_init(6),
-	vpg_regs_init(7),
-	vpg_regs_init(8),
-	vpg_regs_init(9);
-
-	vpg3_construct(vpg3, ctx, inst,
-			&vpg_regs[inst],
-			&vpg_shift,
-			&vpg_mask);
-
-	return &vpg3->base;
-}
-
-static struct afmt *dcn321_afmt_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn30_afmt *afmt3 = kzalloc(sizeof(struct dcn30_afmt), GFP_KERNEL);
-
-	if (!afmt3)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT afmt_regs
-	afmt_regs_init(0),
-	afmt_regs_init(1),
-	afmt_regs_init(2),
-	afmt_regs_init(3),
-	afmt_regs_init(4),
-	afmt_regs_init(5);
-
-	afmt3_construct(afmt3, ctx, inst,
-			&afmt_regs[inst],
-			&afmt_shift,
-			&afmt_mask);
-
-	return &afmt3->base;
-}
-
-static struct apg *dcn321_apg_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn31_apg *apg31 = kzalloc(sizeof(struct dcn31_apg), GFP_KERNEL);
-
-	if (!apg31)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT apg_regs
-	apg_regs_init(0),
-	apg_regs_init(1),
-	apg_regs_init(2),
-	apg_regs_init(3);
-
-	apg31_construct(apg31, ctx, inst,
-			&apg_regs[inst],
-			&apg_shift,
-			&apg_mask);
-
-	return &apg31->base;
-}
-
-static struct stream_encoder *dcn321_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dcn10_stream_encoder *enc1;
-	struct vpg *vpg;
-	struct afmt *afmt;
-	int vpg_inst;
-	int afmt_inst;
-
-	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
-	if (eng_id <= ENGINE_ID_DIGF) {
-		vpg_inst = eng_id;
-		afmt_inst = eng_id;
-	} else
-		return NULL;
-
-	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
-	vpg = dcn321_vpg_create(ctx, vpg_inst);
-	afmt = dcn321_afmt_create(ctx, afmt_inst);
-
-	if (!enc1 || !vpg || !afmt) {
-		kfree(enc1);
-		kfree(vpg);
-		kfree(afmt);
-		return NULL;
-	}
-
-#undef REG_STRUCT
-#define REG_STRUCT stream_enc_regs
-	stream_enc_regs_init(0),
-	stream_enc_regs_init(1),
-	stream_enc_regs_init(2),
-	stream_enc_regs_init(3),
-	stream_enc_regs_init(4);
-
-	dcn32_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
-					eng_id, vpg, afmt,
-					&stream_enc_regs[eng_id],
-					&se_shift, &se_mask);
-
-	return &enc1->base;
-}
-
-static struct hpo_dp_stream_encoder *dcn321_hpo_dp_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dcn31_hpo_dp_stream_encoder *hpo_dp_enc31;
-	struct vpg *vpg;
-	struct apg *apg;
-	uint32_t hpo_dp_inst;
-	uint32_t vpg_inst;
-	uint32_t apg_inst;
-
-	ASSERT((eng_id >= ENGINE_ID_HPO_DP_0) && (eng_id <= ENGINE_ID_HPO_DP_3));
-	hpo_dp_inst = eng_id - ENGINE_ID_HPO_DP_0;
-
-	/* Mapping of VPG register blocks to HPO DP block instance:
-	 * VPG[6] -> HPO_DP[0]
-	 * VPG[7] -> HPO_DP[1]
-	 * VPG[8] -> HPO_DP[2]
-	 * VPG[9] -> HPO_DP[3]
-	 */
-	vpg_inst = hpo_dp_inst + 6;
-
-	/* Mapping of APG register blocks to HPO DP block instance:
-	 * APG[0] -> HPO_DP[0]
-	 * APG[1] -> HPO_DP[1]
-	 * APG[2] -> HPO_DP[2]
-	 * APG[3] -> HPO_DP[3]
-	 */
-	apg_inst = hpo_dp_inst;
-
-	/* allocate HPO stream encoder and create VPG sub-block */
-	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_stream_encoder), GFP_KERNEL);
-	vpg = dcn321_vpg_create(ctx, vpg_inst);
-	apg = dcn321_apg_create(ctx, apg_inst);
-
-	if (!hpo_dp_enc31 || !vpg || !apg) {
-		kfree(hpo_dp_enc31);
-		kfree(vpg);
-		kfree(apg);
-		return NULL;
-	}
-
-#undef REG_STRUCT
-#define REG_STRUCT hpo_dp_stream_enc_regs
-	hpo_dp_stream_encoder_reg_init(0),
-	hpo_dp_stream_encoder_reg_init(1),
-	hpo_dp_stream_encoder_reg_init(2),
-	hpo_dp_stream_encoder_reg_init(3);
-
-	dcn31_hpo_dp_stream_encoder_construct(hpo_dp_enc31, ctx, ctx->dc_bios,
-					hpo_dp_inst, eng_id, vpg, apg,
-					&hpo_dp_stream_enc_regs[hpo_dp_inst],
-					&hpo_dp_se_shift, &hpo_dp_se_mask);
-
-	return &hpo_dp_enc31->base;
-}
-
-static struct hpo_dp_link_encoder *dcn321_hpo_dp_link_encoder_create(
-	uint8_t inst,
-	struct dc_context *ctx)
-{
-	struct dcn31_hpo_dp_link_encoder *hpo_dp_enc31;
-
-	/* allocate HPO link encoder */
-	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
-
-#undef REG_STRUCT
-#define REG_STRUCT hpo_dp_link_enc_regs
-	hpo_dp_link_encoder_reg_init(0),
-	hpo_dp_link_encoder_reg_init(1);
-
-	hpo_dp_link_encoder32_construct(hpo_dp_enc31, ctx, inst,
-					&hpo_dp_link_enc_regs[inst],
-					&hpo_dp_le_shift, &hpo_dp_le_mask);
-
-	return &hpo_dp_enc31->base;
-}
-
-static struct dce_hwseq *dcn321_hwseq_create(
-	struct dc_context *ctx)
-{
-	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
-
-#undef REG_STRUCT
-#define REG_STRUCT hwseq_reg
-	hwseq_reg_init();
-
-	if (hws) {
-		hws->ctx = ctx;
-		hws->regs = &hwseq_reg;
-		hws->shifts = &hwseq_shift;
-		hws->masks = &hwseq_mask;
-	}
-	return hws;
-}
-static const struct resource_create_funcs res_create_funcs = {
-	.read_dce_straps = read_dce_straps,
-	.create_audio = dcn321_create_audio,
-	.create_stream_encoder = dcn321_stream_encoder_create,
-	.create_hpo_dp_stream_encoder = dcn321_hpo_dp_stream_encoder_create,
-	.create_hpo_dp_link_encoder = dcn321_hpo_dp_link_encoder_create,
-	.create_hwseq = dcn321_hwseq_create,
-};
-
-static void dcn321_resource_destruct(struct dcn321_resource_pool *pool)
-{
-	unsigned int i;
-
-	for (i = 0; i < pool->base.stream_enc_count; i++) {
-		if (pool->base.stream_enc[i] != NULL) {
-			if (pool->base.stream_enc[i]->vpg != NULL) {
-				kfree(DCN30_VPG_FROM_VPG(pool->base.stream_enc[i]->vpg));
-				pool->base.stream_enc[i]->vpg = NULL;
-			}
-			if (pool->base.stream_enc[i]->afmt != NULL) {
-				kfree(DCN30_AFMT_FROM_AFMT(pool->base.stream_enc[i]->afmt));
-				pool->base.stream_enc[i]->afmt = NULL;
-			}
-			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
-			pool->base.stream_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.hpo_dp_stream_enc_count; i++) {
-		if (pool->base.hpo_dp_stream_enc[i] != NULL) {
-			if (pool->base.hpo_dp_stream_enc[i]->vpg != NULL) {
-				kfree(DCN30_VPG_FROM_VPG(pool->base.hpo_dp_stream_enc[i]->vpg));
-				pool->base.hpo_dp_stream_enc[i]->vpg = NULL;
-			}
-			if (pool->base.hpo_dp_stream_enc[i]->apg != NULL) {
-				kfree(DCN31_APG_FROM_APG(pool->base.hpo_dp_stream_enc[i]->apg));
-				pool->base.hpo_dp_stream_enc[i]->apg = NULL;
-			}
-			kfree(DCN3_1_HPO_DP_STREAM_ENC_FROM_HPO_STREAM_ENC(pool->base.hpo_dp_stream_enc[i]));
-			pool->base.hpo_dp_stream_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.hpo_dp_link_enc_count; i++) {
-		if (pool->base.hpo_dp_link_enc[i] != NULL) {
-			kfree(DCN3_1_HPO_DP_LINK_ENC_FROM_HPO_LINK_ENC(pool->base.hpo_dp_link_enc[i]));
-			pool->base.hpo_dp_link_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		if (pool->base.dscs[i] != NULL)
-			dcn20_dsc_destroy(&pool->base.dscs[i]);
-	}
-
-	if (pool->base.mpc != NULL) {
-		kfree(TO_DCN20_MPC(pool->base.mpc));
-		pool->base.mpc = NULL;
-	}
-	if (pool->base.hubbub != NULL) {
-		kfree(TO_DCN20_HUBBUB(pool->base.hubbub));
-		pool->base.hubbub = NULL;
-	}
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		if (pool->base.dpps[i] != NULL)
-			dcn321_dpp_destroy(&pool->base.dpps[i]);
-
-		if (pool->base.ipps[i] != NULL)
-			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
-
-		if (pool->base.hubps[i] != NULL) {
-			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
-			pool->base.hubps[i] = NULL;
-		}
-
-		if (pool->base.irqs != NULL)
-			dal_irq_service_destroy(&pool->base.irqs);
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		if (pool->base.engines[i] != NULL)
-			dce110_engine_destroy(&pool->base.engines[i]);
-		if (pool->base.hw_i2cs[i] != NULL) {
-			kfree(pool->base.hw_i2cs[i]);
-			pool->base.hw_i2cs[i] = NULL;
-		}
-		if (pool->base.sw_i2cs[i] != NULL) {
-			kfree(pool->base.sw_i2cs[i]);
-			pool->base.sw_i2cs[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
-		if (pool->base.opps[i] != NULL)
-			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		if (pool->base.timing_generators[i] != NULL)	{
-			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
-			pool->base.timing_generators[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
-		if (pool->base.dwbc[i] != NULL) {
-			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
-			pool->base.dwbc[i] = NULL;
-		}
-		if (pool->base.mcif_wb[i] != NULL) {
-			kfree(TO_DCN30_MMHUBBUB(pool->base.mcif_wb[i]));
-			pool->base.mcif_wb[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.audio_count; i++) {
-		if (pool->base.audios[i])
-			dce_aud_destroy(&pool->base.audios[i]);
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] != NULL) {
-			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
-			pool->base.clock_sources[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
-		if (pool->base.mpc_lut[i] != NULL) {
-			dc_3dlut_func_release(pool->base.mpc_lut[i]);
-			pool->base.mpc_lut[i] = NULL;
-		}
-		if (pool->base.mpc_shaper[i] != NULL) {
-			dc_transfer_func_release(pool->base.mpc_shaper[i]);
-			pool->base.mpc_shaper[i] = NULL;
-		}
-	}
-
-	if (pool->base.dp_clock_source != NULL) {
-		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
-		pool->base.dp_clock_source = NULL;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		if (pool->base.multiple_abms[i] != NULL)
-			dce_abm_destroy(&pool->base.multiple_abms[i]);
-	}
-
-	if (pool->base.psr != NULL)
-		dmub_psr_destroy(&pool->base.psr);
-
-	if (pool->base.dccg != NULL)
-		dcn_dccg_destroy(&pool->base.dccg);
-
-	if (pool->base.oem_device != NULL) {
-		struct dc *dc = pool->base.oem_device->ctx->dc;
-
-		dc->link_srv->destroy_ddc_service(&pool->base.oem_device);
-	}
-}
-
-
-static bool dcn321_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t dwb_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < dwb_count; i++) {
-		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc),
-						    GFP_KERNEL);
-
-		if (!dwbc30) {
-			dm_error("DC: failed to create dwbc30!\n");
-			return false;
-		}
-
-#undef REG_STRUCT
-#define REG_STRUCT dwbc30_regs
-		dwbc_regs_dcn3_init(0);
-
-		dcn30_dwbc_construct(dwbc30, ctx,
-				&dwbc30_regs[i],
-				&dwbc30_shift,
-				&dwbc30_mask,
-				i);
-
-		pool->dwbc[i] = &dwbc30->base;
-	}
-	return true;
-}
-
-static bool dcn321_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t dwb_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < dwb_count; i++) {
-		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub),
-						    GFP_KERNEL);
-
-		if (!mcif_wb30) {
-			dm_error("DC: failed to create mcif_wb30!\n");
-			return false;
-		}
-
-#undef REG_STRUCT
-#define REG_STRUCT mcif_wb30_regs
-		mcif_wb_regs_dcn3_init(0);
-
-		dcn32_mmhubbub_construct(mcif_wb30, ctx,
-				&mcif_wb30_regs[i],
-				&mcif_wb30_shift,
-				&mcif_wb30_mask,
-				i);
-
-		pool->mcif_wb[i] = &mcif_wb30->base;
-	}
-	return true;
-}
-
-static struct display_stream_compressor *dcn321_dsc_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_dsc *dsc =
-		kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
-
-	if (!dsc) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-#undef REG_STRUCT
-#define REG_STRUCT dsc_regs
-	dsc_regsDCN20_init(0),
-	dsc_regsDCN20_init(1),
-	dsc_regsDCN20_init(2),
-	dsc_regsDCN20_init(3);
-
-	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
-
-	dsc->max_image_width = 6016;
-
-	return &dsc->base;
-}
-
-static void dcn321_destroy_resource_pool(struct resource_pool **pool)
-{
-	struct dcn321_resource_pool *dcn321_pool = TO_DCN321_RES_POOL(*pool);
-
-	dcn321_resource_destruct(dcn321_pool);
-	kfree(dcn321_pool);
-	*pool = NULL;
-}
-
-static struct dc_cap_funcs cap_funcs = {
-	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap,
-	.get_subvp_en = dcn32_subvp_in_use,
-};
-
-static void dcn321_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
-{
-	DC_FP_START();
-	dcn321_update_bw_bounding_box_fpu(dc, bw_params);
-	DC_FP_END();
-}
-
-static struct resource_funcs dcn321_res_pool_funcs = {
-	.destroy = dcn321_destroy_resource_pool,
-	.link_enc_create = dcn321_link_encoder_create,
-	.link_enc_create_minimal = NULL,
-	.panel_cntl_create = dcn32_panel_cntl_create,
-	.validate_bandwidth = dcn32_validate_bandwidth,
-	.calculate_wm_and_dlg = dcn32_calculate_wm_and_dlg,
-	.populate_dml_pipes = dcn32_populate_dml_pipes_from_context,
-	.acquire_free_pipe_as_secondary_dpp_pipe = dcn32_acquire_free_pipe_as_secondary_dpp_pipe,
-	.acquire_free_pipe_as_secondary_opp_head = dcn32_acquire_free_pipe_as_secondary_opp_head,
-	.release_pipe = dcn20_release_pipe,
-	.add_stream_to_ctx = dcn30_add_stream_to_ctx,
-	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
-	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
-	.populate_dml_writeback_from_context = dcn30_populate_dml_writeback_from_context,
-	.set_mcif_arb_params = dcn30_set_mcif_arb_params,
-	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
-	.acquire_post_bldn_3dlut = dcn32_acquire_post_bldn_3dlut,
-	.release_post_bldn_3dlut = dcn32_release_post_bldn_3dlut,
-	.update_bw_bounding_box = dcn321_update_bw_bounding_box,
-	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
-	.update_soc_for_wm_a = dcn30_update_soc_for_wm_a,
-	.add_phantom_pipes = dcn32_add_phantom_pipes,
-	.remove_phantom_pipes = dcn32_remove_phantom_pipes,
-	.retain_phantom_pipes = dcn32_retain_phantom_pipes,
-	.save_mall_state = dcn32_save_mall_state,
-	.restore_mall_state = dcn32_restore_mall_state,
-};
-
-static uint32_t read_pipe_fuses(struct dc_context *ctx)
-{
-	uint32_t value = REG_READ(CC_DC_PIPE_DIS);
-	/* DCN321 support max 4 pipes */
-	value = value & 0xf;
-	return value;
-}
-
-
-static bool dcn321_resource_construct(
-	uint8_t num_virtual_links,
-	struct dc *dc,
-	struct dcn321_resource_pool *pool)
-{
-	int i, j;
-	struct dc_context *ctx = dc->ctx;
-	struct irq_service_init_data init_data;
-	struct ddc_service_init_data ddc_init_data = {0};
-	uint32_t pipe_fuses = 0;
-	uint32_t num_pipes  = 4;
-
-#undef REG_STRUCT
-#define REG_STRUCT bios_regs
-	bios_regs_init();
-
-#undef REG_STRUCT
-#define REG_STRUCT clk_src_regs
-	clk_src_regs_init(0, A),
-	clk_src_regs_init(1, B),
-	clk_src_regs_init(2, C),
-	clk_src_regs_init(3, D),
-	clk_src_regs_init(4, E);
-
-#undef REG_STRUCT
-#define REG_STRUCT abm_regs
-	abm_regs_init(0),
-	abm_regs_init(1),
-	abm_regs_init(2),
-	abm_regs_init(3);
-
-#undef REG_STRUCT
-#define REG_STRUCT dccg_regs
-	dccg_regs_init();
-
-
-	ctx->dc_bios->regs = &bios_regs;
-
-	pool->base.res_cap = &res_cap_dcn321;
-	/* max number of pipes for ASIC before checking for pipe fuses */
-	num_pipes  = pool->base.res_cap->num_timing_generator;
-	pipe_fuses = read_pipe_fuses(ctx);
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++)
-		if (pipe_fuses & 1 << i)
-			num_pipes--;
-
-	if (pipe_fuses & 1)
-		ASSERT(0); //Unexpected - Pipe 0 should always be fully functional!
-
-	if (pipe_fuses & CC_DC_PIPE_DIS__DC_FULL_DIS_MASK)
-		ASSERT(0); //Entire DCN is harvested!
-
-	/* within dml lib, initial value is hard coded, if ASIC pipe is fused, the
-	 * value will be changed, update max_num_dpp and max_num_otg for dml.
-	 */
-	dcn3_21_ip.max_num_dpp = num_pipes;
-	dcn3_21_ip.max_num_otg = num_pipes;
-
-	pool->base.funcs = &dcn321_res_pool_funcs;
-
-	/*************************************************
-	 *  Resource + asic cap harcoding                *
-	 *************************************************/
-	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
-	pool->base.timing_generator_count = num_pipes;
-	pool->base.pipe_count = num_pipes;
-	pool->base.mpcc_count = num_pipes;
-	dc->caps.max_downscale_ratio = 600;
-	dc->caps.i2c_speed_in_khz = 100;
-	dc->caps.i2c_speed_in_khz_hdcp = 100; /*1.4 w/a applied by default*/
-	/* TODO: Bring max cursor size back to 256 after subvp cursor corruption is fixed*/
-	dc->caps.max_cursor_size = 64;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.dmdata_alloc_size = 2048;
-	dc->caps.mall_size_per_mem_channel = 4;
-	dc->caps.mall_size_total = 0;
-	dc->caps.cursor_cache_size = dc->caps.max_cursor_size * dc->caps.max_cursor_size * 8;
-	dc->caps.cache_line_size = 64;
-	dc->caps.cache_num_ways = 16;
-
-	/* Calculate the available MALL space */
-	dc->caps.max_cab_allocation_bytes = dcn32_calc_num_avail_chans_for_mall(
-		dc, dc->ctx->dc_bios->vram_info.num_chans) *
-		dc->caps.mall_size_per_mem_channel * 1024 * 1024;
-	dc->caps.mall_size_total = dc->caps.max_cab_allocation_bytes;
-
-	dc->caps.subvp_fw_processing_delay_us = 15;
-	dc->caps.subvp_drr_max_vblank_margin_us = 40;
-	dc->caps.subvp_prefetch_end_to_mall_start_us = 15;
-	dc->caps.subvp_swath_height_margin_lines = 16;
-	dc->caps.subvp_pstate_allow_width_us = 20;
-	dc->caps.subvp_vertical_int_margin_us = 30;
-	dc->caps.subvp_drr_vblank_start_margin_us = 100; // 100us margin
-	dc->caps.max_slave_planes = 2;
-	dc->caps.max_slave_yuv_planes = 2;
-	dc->caps.max_slave_rgb_planes = 2;
-	dc->caps.post_blend_color_processing = true;
-	dc->caps.force_dp_tps4_for_cp2520 = true;
-	dc->caps.dp_hpo = true;
-	dc->caps.dp_hdmi21_pcon_support = true;
-	dc->caps.edp_dsc_support = true;
-	dc->caps.extended_aux_timeout_support = true;
-	dc->caps.dmcub_support = true;
-	dc->caps.max_v_total = (1 << 15) - 1;
-
-	/* Color pipeline capabilities */
-	dc->caps.color.dpp.dcn_arch = 1;
-	dc->caps.color.dpp.input_lut_shared = 0;
-	dc->caps.color.dpp.icsc = 1;
-	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
-	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
-	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
-	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
-	dc->caps.color.dpp.post_csc = 1;
-	dc->caps.color.dpp.gamma_corr = 1;
-	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
-
-	dc->caps.color.dpp.hw_3d_lut = 1;
-	dc->caps.color.dpp.ogam_ram = 1;
-	// no OGAM ROM on DCN2 and later ASICs
-	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
-	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
-	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
-	dc->caps.color.dpp.ocsc = 0;
-
-	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //4, configurable to be before or after BLND in MPCC
-	dc->caps.color.mpc.ogam_ram = 1;
-	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
-	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
-	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
-	dc->caps.color.mpc.ocsc = 1;
-
-	dc->config.dc_mode_clk_limit_support = true;
-	/* read VBIOS LTTPR caps */
-	{
-		if (ctx->dc_bios->funcs->get_lttpr_caps) {
-			enum bp_result bp_query_result;
-			uint8_t is_vbios_lttpr_enable = 0;
-
-			bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
-			dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
-		}
-
-		/* interop bit is implicit */
-		{
-			dc->caps.vbios_lttpr_aware = true;
-		}
-	}
-
-	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
-		dc->debug = debug_defaults_drv;
-
-	// Init the vm_helper
-	if (dc->vm_helper)
-		vm_helper_init(dc->vm_helper, 16);
-
-	/*************************************************
-	 *  Create resources                             *
-	 *************************************************/
-
-	/* Clock Sources for Pixel Clock*/
-	pool->base.clock_sources[DCN321_CLK_SRC_PLL0] =
-			dcn321_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL0,
-				&clk_src_regs[0], false);
-	pool->base.clock_sources[DCN321_CLK_SRC_PLL1] =
-			dcn321_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL1,
-				&clk_src_regs[1], false);
-	pool->base.clock_sources[DCN321_CLK_SRC_PLL2] =
-			dcn321_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL2,
-				&clk_src_regs[2], false);
-	pool->base.clock_sources[DCN321_CLK_SRC_PLL3] =
-			dcn321_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL3,
-				&clk_src_regs[3], false);
-	pool->base.clock_sources[DCN321_CLK_SRC_PLL4] =
-			dcn321_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL4,
-				&clk_src_regs[4], false);
-
-	pool->base.clk_src_count = DCN321_CLK_SRC_TOTAL;
-
-	/* todo: not reuse phy_pll registers */
-	pool->base.dp_clock_source =
-			dcn321_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_ID_DP_DTO,
-				&clk_src_regs[0], true);
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-
-	/* DCCG */
-	pool->base.dccg = dccg32_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
-	if (pool->base.dccg == NULL) {
-		dm_error("DC: failed to create dccg!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	/* DML */
-	dml_init_instance(&dc->dml, &dcn3_21_soc, &dcn3_21_ip, DML_PROJECT_DCN32);
-
-	/* IRQ Service */
-	init_data.ctx = dc->ctx;
-	pool->base.irqs = dal_irq_service_dcn32_create(&init_data);
-	if (!pool->base.irqs)
-		goto create_fail;
-
-	/* HUBBUB */
-	pool->base.hubbub = dcn321_hubbub_create(ctx);
-	if (pool->base.hubbub == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create hubbub!\n");
-		goto create_fail;
-	}
-
-	/* HUBPs, DPPs, OPPs, TGs, ABMs */
-	for (i = 0, j = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-
-		/* if pipe is disabled, skip instance of HW pipe,
-		 * i.e, skip ASIC register instance
-		 */
-		if (pipe_fuses & 1 << i)
-			continue;
-
-		pool->base.hubps[j] = dcn321_hubp_create(ctx, i);
-		if (pool->base.hubps[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create hubps!\n");
-			goto create_fail;
-		}
-
-		pool->base.dpps[j] = dcn321_dpp_create(ctx, i);
-		if (pool->base.dpps[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create dpps!\n");
-			goto create_fail;
-		}
-
-		pool->base.opps[j] = dcn321_opp_create(ctx, i);
-		if (pool->base.opps[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create output pixel processor!\n");
-			goto create_fail;
-		}
-
-		pool->base.timing_generators[j] = dcn321_timing_generator_create(
-				ctx, i);
-		if (pool->base.timing_generators[j] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create tg!\n");
-			goto create_fail;
-		}
-
-		pool->base.multiple_abms[j] = dmub_abm_create(ctx,
-				&abm_regs[i],
-				&abm_shift,
-				&abm_mask);
-		if (pool->base.multiple_abms[j] == NULL) {
-			dm_error("DC: failed to create abm for pipe %d!\n", i);
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-
-		/* index for resource pool arrays for next valid pipe */
-		j++;
-	}
-
-	/* PSR */
-	pool->base.psr = dmub_psr_create(ctx);
-	if (pool->base.psr == NULL) {
-		dm_error("DC: failed to create psr obj!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	/* MPCCs */
-	pool->base.mpc = dcn321_mpc_create(ctx,  pool->base.res_cap->num_timing_generator, pool->base.res_cap->num_mpc_3dlut);
-	if (pool->base.mpc == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mpc!\n");
-		goto create_fail;
-	}
-
-	/* DSCs */
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		pool->base.dscs[i] = dcn321_dsc_create(ctx, i);
-		if (pool->base.dscs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create display stream compressor %d!\n", i);
-			goto create_fail;
-		}
-	}
-
-	/* DWB */
-	if (!dcn321_dwbc_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create dwbc!\n");
-		goto create_fail;
-	}
-
-	/* MMHUBBUB */
-	if (!dcn321_mmhubbub_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mcif_wb!\n");
-		goto create_fail;
-	}
-
-	/* AUX and I2C */
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		pool->base.engines[i] = dcn321_aux_engine_create(ctx, i);
-		if (pool->base.engines[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create aux engine!!\n");
-			goto create_fail;
-		}
-		pool->base.hw_i2cs[i] = dcn321_i2c_hw_create(ctx, i);
-		if (pool->base.hw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create hw i2c!!\n");
-			goto create_fail;
-		}
-		pool->base.sw_i2cs[i] = NULL;
-	}
-
-	/* Audio, HWSeq, Stream Encoders including HPO and virtual, MPC 3D LUTs */
-	if (!resource_construct(num_virtual_links, dc, &pool->base,
-			&res_create_funcs))
-		goto create_fail;
-
-	/* HW Sequencer init functions and Plane caps */
-	dcn32_hw_sequencer_init_functions(dc);
-
-	dc->caps.max_planes =  pool->base.pipe_count;
-
-	for (i = 0; i < dc->caps.max_planes; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	dc->cap_funcs = cap_funcs;
-
-	if (dc->ctx->dc_bios->fw_info.oem_i2c_present) {
-		ddc_init_data.ctx = dc->ctx;
-		ddc_init_data.link = NULL;
-		ddc_init_data.id.id = dc->ctx->dc_bios->fw_info.oem_i2c_obj_id;
-		ddc_init_data.id.enum_id = 0;
-		ddc_init_data.id.type = OBJECT_TYPE_GENERIC;
-		pool->base.oem_device = dc->link_srv->create_ddc_service(&ddc_init_data);
-	} else {
-		pool->base.oem_device = NULL;
-	}
-
-	dc->dml2_options.dcn_pipe_count = pool->base.pipe_count;
-	dc->dml2_options.use_native_pstate_optimization = false;
-	dc->dml2_options.use_native_soc_bb_construction = true;
-	dc->dml2_options.minimize_dispclk_using_odm = true;
-
-	dc->dml2_options.callbacks.dc = dc;
-	dc->dml2_options.callbacks.build_scaling_params = &resource_build_scaling_params;
-	dc->dml2_options.callbacks.can_support_mclk_switch_using_fw_based_vblank_stretch = &dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch;
-	dc->dml2_options.callbacks.acquire_secondary_pipe_for_mpc_odm = &dc_resource_acquire_secondary_pipe_for_mpc_odm_legacy;
-	dc->dml2_options.callbacks.update_pipes_for_stream_with_slice_count = &resource_update_pipes_for_stream_with_slice_count;
-	dc->dml2_options.callbacks.update_pipes_for_plane_with_slice_count = &resource_update_pipes_for_plane_with_slice_count;
-	dc->dml2_options.callbacks.get_mpc_slice_index = &resource_get_mpc_slice_index;
-	dc->dml2_options.callbacks.get_odm_slice_index = &resource_get_odm_slice_index;
-	dc->dml2_options.callbacks.get_opp_head = &resource_get_opp_head;
-
-	dc->dml2_options.svp_pstate.callbacks.dc = dc;
-	dc->dml2_options.svp_pstate.callbacks.add_plane_to_context = &dc_add_plane_to_context;
-	dc->dml2_options.svp_pstate.callbacks.add_stream_to_ctx = &dc_add_stream_to_ctx;
-	dc->dml2_options.svp_pstate.callbacks.build_scaling_params = &resource_build_scaling_params;
-	dc->dml2_options.svp_pstate.callbacks.create_plane = &dc_create_plane_state;
-	dc->dml2_options.svp_pstate.callbacks.remove_plane_from_context = &dc_remove_plane_from_context;
-	dc->dml2_options.svp_pstate.callbacks.remove_stream_from_ctx = &dc_remove_stream_from_ctx;
-	dc->dml2_options.svp_pstate.callbacks.create_stream_for_sink = &dc_create_stream_for_sink;
-	dc->dml2_options.svp_pstate.callbacks.plane_state_release = &dc_plane_state_release;
-	dc->dml2_options.svp_pstate.callbacks.stream_release = &dc_stream_release;
-	dc->dml2_options.svp_pstate.callbacks.release_dsc = &dcn20_release_dsc;
-
-	dc->dml2_options.svp_pstate.subvp_fw_processing_delay_us = dc->caps.subvp_fw_processing_delay_us;
-	dc->dml2_options.svp_pstate.subvp_prefetch_end_to_mall_start_us = dc->caps.subvp_prefetch_end_to_mall_start_us;
-	dc->dml2_options.svp_pstate.subvp_pstate_allow_width_us = dc->caps.subvp_pstate_allow_width_us;
-	dc->dml2_options.svp_pstate.subvp_swath_height_margin_lines = dc->caps.subvp_swath_height_margin_lines;
-
-	dc->dml2_options.svp_pstate.force_disable_subvp = dc->debug.force_disable_subvp;
-	dc->dml2_options.svp_pstate.force_enable_subvp = dc->debug.force_subvp_mclk_switch;
-
-	dc->dml2_options.mall_cfg.cache_line_size_bytes = dc->caps.cache_line_size;
-	dc->dml2_options.mall_cfg.cache_num_ways = dc->caps.cache_num_ways;
-	dc->dml2_options.mall_cfg.max_cab_allocation_bytes = dc->caps.max_cab_allocation_bytes;
-	dc->dml2_options.mall_cfg.mblk_height_4bpe_pixels = DCN3_2_MBLK_HEIGHT_4BPE;
-	dc->dml2_options.mall_cfg.mblk_height_8bpe_pixels = DCN3_2_MBLK_HEIGHT_8BPE;
-	dc->dml2_options.mall_cfg.mblk_size_bytes = DCN3_2_MALL_MBLK_SIZE_BYTES;
-	dc->dml2_options.mall_cfg.mblk_width_pixels = DCN3_2_MBLK_WIDTH;
-
-	dc->dml2_options.max_segments_per_hubp = 18;
-	dc->dml2_options.det_segment_size = DCN3_2_DET_SEG_SIZE;
-
-	return true;
-
-create_fail:
-
-	dcn321_resource_destruct(pool);
-
-	return false;
-}
-
-struct resource_pool *dcn321_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc)
-{
-	struct dcn321_resource_pool *pool =
-		kzalloc(sizeof(struct dcn321_resource_pool), GFP_KERNEL);
-
-	if (!pool)
-		return NULL;
-
-	if (dcn321_resource_construct(init_data->num_virtual_links, dc, pool))
-		return &pool->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(pool);
-	return NULL;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.h b/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.h
deleted file mode 100644
index 82cbf009f2d3..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright 2020 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#ifndef _DCN321_RESOURCE_H_
-#define _DCN321_RESOURCE_H_
-
-#include "core_types.h"
-
-#define TO_DCN321_RES_POOL(pool)\
-	container_of(pool, struct dcn321_resource_pool, base)
-
-extern struct _vcs_dpi_ip_params_st dcn3_21_ip;
-extern struct _vcs_dpi_soc_bounding_box_st dcn3_21_soc;
-
-struct dcn321_resource_pool {
-	struct resource_pool base;
-};
-
-struct resource_pool *dcn321_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc);
-
-#endif /* _DCN321_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn35/Makefile b/drivers/gpu/drm/amd/display/dc/dcn35/Makefile
index 20d0eef1a13b..719afb5a3b12 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn35/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn35/Makefile
@@ -10,7 +10,7 @@
 #
 # Makefile for DCN35.
 
-DCN35 = dcn35_resource.o dcn35_init.o dcn35_dio_stream_encoder.o \
+DCN35 = dcn35_init.o dcn35_dio_stream_encoder.o \
 	dcn35_dio_link_encoder.o dcn35_dccg.o dcn35_optc.o \
 	dcn35_dsc.o dcn35_hubp.o dcn35_hubbub.o \
 	dcn35_mmhubbub.o dcn35_opp.o dcn35_dpp.o dcn35_pg_cntl.o dcn35_dwb.o
diff --git a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dccg.c b/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dccg.c
index 479f3683c0b7..142efd390d86 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dccg.c
@@ -256,6 +256,21 @@ static void dccg35_set_dtbclk_dto(
 	if (params->ref_dtbclk_khz && req_dtbclk_khz) {
 		uint32_t modulo, phase;
 
+		switch (params->otg_inst) {
+		case 0:
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, DTBCLK_P0_GATE_DISABLE, 1);
+			break;
+		case 1:
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, DTBCLK_P1_GATE_DISABLE, 1);
+			break;
+		case 2:
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, DTBCLK_P2_GATE_DISABLE, 1);
+			break;
+		case 3:
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, DTBCLK_P3_GATE_DISABLE, 1);
+			break;
+		}
+
 		// phase / modulo = dtbclk / dtbclk ref
 		modulo = params->ref_dtbclk_khz * 1000;
 		phase = req_dtbclk_khz * 1000;
@@ -280,6 +295,21 @@ static void dccg35_set_dtbclk_dto(
 		REG_UPDATE(OTG_PIXEL_RATE_CNTL[params->otg_inst],
 				PIPE_DTO_SRC_SEL[params->otg_inst], 2);
 	} else {
+		switch (params->otg_inst) {
+		case 0:
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, DTBCLK_P0_GATE_DISABLE, 0);
+			break;
+		case 1:
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, DTBCLK_P1_GATE_DISABLE, 0);
+			break;
+		case 2:
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, DTBCLK_P2_GATE_DISABLE, 0);
+			break;
+		case 3:
+			REG_UPDATE(DCCG_GATE_DISABLE_CNTL5, DTBCLK_P3_GATE_DISABLE, 0);
+			break;
+		}
+
 		REG_UPDATE_2(OTG_PIXEL_RATE_CNTL[params->otg_inst],
 				DTBCLK_DTO_ENABLE[params->otg_inst], 0,
 				PIPE_DTO_SRC_SEL[params->otg_inst], params->is_hdmi ? 0 : 1);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dccg.h b/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dccg.h
index 423feb4c2f3f..bde48bee0119 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_dccg.h
@@ -34,6 +34,7 @@
 #define DCCG_REG_LIST_DCN35() \
 	DCCG_REG_LIST_DCN314(),\
 	SR(DPPCLK_CTRL),\
+	SR(DCCG_GATE_DISABLE_CNTL5),\
 	SR(DCCG_GATE_DISABLE_CNTL6),\
 	SR(DCCG_GLOBAL_FGCG_REP_CNTL),\
 	SR(SYMCLKA_CLOCK_ENABLE),\
@@ -174,7 +175,11 @@
 	DCCG_SF(SYMCLKB_CLOCK_ENABLE, SYMCLKB_FE_SRC_SEL, mask_sh),\
 	DCCG_SF(SYMCLKC_CLOCK_ENABLE, SYMCLKC_FE_SRC_SEL, mask_sh),\
 	DCCG_SF(SYMCLKD_CLOCK_ENABLE, SYMCLKD_FE_SRC_SEL, mask_sh),\
-	DCCG_SF(SYMCLKE_CLOCK_ENABLE, SYMCLKE_FE_SRC_SEL, mask_sh)
+	DCCG_SF(SYMCLKE_CLOCK_ENABLE, SYMCLKE_FE_SRC_SEL, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DTBCLK_P0_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DTBCLK_P1_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DTBCLK_P2_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL5, DTBCLK_P3_GATE_DISABLE, mask_sh),\
 
 struct dccg *dccg35_create(
 		struct dc_context *ctx,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_pg_cntl.c b/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_pg_cntl.c
index 46f71ff08fd1..0f60c40e1fc5 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_pg_cntl.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_pg_cntl.c
@@ -332,13 +332,6 @@ void pg_cntl35_io_clk_pg_control(struct pg_cntl *pg_cntl, bool power_on)
 	pg_cntl->pg_res_enable[PG_DCIO] = power_on;
 }
 
-void pg_cntl35_set_force_poweron_domain22(struct pg_cntl *pg_cntl, bool power_on)
-{
-	struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(pg_cntl);
-
-	REG_UPDATE(DOMAIN22_PG_CONFIG, DOMAIN_POWER_FORCEON, power_on ? 1 : 0);
-}
-
 static bool pg_cntl35_plane_otg_status(struct pg_cntl *pg_cntl)
 {
 	struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(pg_cntl);
@@ -508,8 +501,7 @@ static const struct pg_cntl_funcs pg_cntl35_funcs = {
 	.mpcc_pg_control = pg_cntl35_mpcc_pg_control,
 	.opp_pg_control = pg_cntl35_opp_pg_control,
 	.optc_pg_control = pg_cntl35_optc_pg_control,
-	.dwb_pg_control = pg_cntl35_dwb_pg_control,
-	.set_force_poweron_domain22 = pg_cntl35_set_force_poweron_domain22
+	.dwb_pg_control = pg_cntl35_dwb_pg_control
 };
 
 struct pg_cntl *pg_cntl35_create(
diff --git a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_pg_cntl.h b/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_pg_cntl.h
index 069dae08e222..3de240884d22 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_pg_cntl.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_pg_cntl.h
@@ -183,7 +183,6 @@ void pg_cntl35_optc_pg_control(struct pg_cntl *pg_cntl,
 	unsigned int optc_inst, bool power_on);
 void pg_cntl35_dwb_pg_control(struct pg_cntl *pg_cntl, bool power_on);
 void pg_cntl35_init_pg_status(struct pg_cntl *pg_cntl);
-void pg_cntl35_set_force_poweron_domain22(struct pg_cntl *pg_cntl, bool power_on);
 
 struct pg_cntl *pg_cntl35_create(
 	struct dc_context *ctx,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_resource.c b/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_resource.c
deleted file mode 100644
index c7e011d26d41..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_resource.c
+++ /dev/null
@@ -1,2141 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright 2023 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#include "dm_services.h"
-#include "dc.h"
-
-#include "dcn31/dcn31_init.h"
-#include "dcn35/dcn35_init.h"
-
-#include "resource.h"
-#include "include/irq_service_interface.h"
-#include "dcn35_resource.h"
-#include "dml2/dml2_wrapper.h"
-
-#include "dcn20/dcn20_resource.h"
-#include "dcn30/dcn30_resource.h"
-#include "dcn31/dcn31_resource.h"
-#include "dcn32/dcn32_resource.h"
-
-#include "dcn10/dcn10_ipp.h"
-#include "dcn30/dcn30_hubbub.h"
-#include "dcn31/dcn31_hubbub.h"
-#include "dcn35/dcn35_hubbub.h"
-#include "dcn32/dcn32_mpc.h"
-#include "dcn35/dcn35_hubp.h"
-#include "irq/dcn35/irq_service_dcn35.h"
-#include "dcn35/dcn35_dpp.h"
-#include "dcn35/dcn35_optc.h"
-#include "dcn20/dcn20_hwseq.h"
-#include "dcn30/dcn30_hwseq.h"
-#include "dce110/dce110_hwseq.h"
-#include "dcn35/dcn35_opp.h"
-#include "dcn35/dcn35_dsc.h"
-#include "dcn30/dcn30_vpg.h"
-#include "dcn30/dcn30_afmt.h"
-#include "dcn31/dcn31_dio_link_encoder.h"
-#include "dcn35/dcn35_dio_stream_encoder.h"
-#include "dcn31/dcn31_hpo_dp_stream_encoder.h"
-#include "dcn31/dcn31_hpo_dp_link_encoder.h"
-#include "dcn32/dcn32_hpo_dp_link_encoder.h"
-#include "link.h"
-#include "dcn31/dcn31_apg.h"
-#include "dcn32/dcn32_dio_link_encoder.h"
-#include "dcn31/dcn31_vpg.h"
-#include "dcn31/dcn31_afmt.h"
-#include "dce/dce_clock_source.h"
-#include "dce/dce_audio.h"
-#include "dce/dce_hwseq.h"
-#include "clk_mgr.h"
-#include "virtual/virtual_stream_encoder.h"
-#include "dce110/dce110_resource.h"
-#include "dml/display_mode_vba.h"
-#include "dcn35/dcn35_dccg.h"
-#include "dcn35/dcn35_pg_cntl.h"
-#include "dcn10/dcn10_resource.h"
-#include "dcn31/dcn31_panel_cntl.h"
-#include "dcn35/dcn35_hwseq.h"
-#include "dcn35_dio_link_encoder.h"
-#include "dml/dcn31/dcn31_fpu.h" /*todo*/
-#include "dml/dcn35/dcn35_fpu.h"
-#include "dcn35/dcn35_dwb.h"
-#include "dcn35/dcn35_mmhubbub.h"
-
-#include "dcn/dcn_3_5_0_offset.h"
-#include "dcn/dcn_3_5_0_sh_mask.h"
-#include "nbio/nbio_7_11_0_offset.h"
-#include "mmhub/mmhub_3_3_0_offset.h"
-#include "mmhub/mmhub_3_3_0_sh_mask.h"
-
-#define DSCC0_DSCC_CONFIG0__ICH_RESET_AT_END_OF_LINE__SHIFT                   0x0
-#define DSCC0_DSCC_CONFIG0__ICH_RESET_AT_END_OF_LINE_MASK                     0x0000000FL
-
-#include "reg_helper.h"
-#include "dce/dmub_abm.h"
-#include "dce/dmub_psr.h"
-#include "dce/dce_aux.h"
-#include "dce/dce_i2c.h"
-#include "dml/dcn31/display_mode_vba_31.h" /*temp*/
-#include "vm_helper.h"
-#include "dcn20/dcn20_vmid.h"
-
-#include "link_enc_cfg.h"
-#define DC_LOGGER_INIT(logger)
-
-enum dcn35_clk_src_array_id {
-	DCN35_CLK_SRC_PLL0,
-	DCN35_CLK_SRC_PLL1,
-	DCN35_CLK_SRC_PLL2,
-	DCN35_CLK_SRC_PLL3,
-	DCN35_CLK_SRC_PLL4,
-	DCN35_CLK_SRC_TOTAL
-};
-
-/* begin *********************
- * macros to expend register list macro defined in HW object header file
- */
-
-/* DCN */
-/* TODO awful hack. fixup dcn20_dwb.h */
-#undef BASE_INNER
-#define BASE_INNER(seg) ctx->dcn_reg_offsets[seg]
-
-#define BASE(seg) BASE_INNER(seg)
-
-#define SR(reg_name)\
-		REG_STRUCT.reg_name = BASE(reg ## reg_name ## _BASE_IDX) +  \
-					reg ## reg_name
-
-#define SR_ARR(reg_name, id) \
-	REG_STRUCT[id].reg_name = BASE(reg##reg_name##_BASE_IDX) + reg##reg_name
-
-#define SR_ARR_INIT(reg_name, id, value) \
-	REG_STRUCT[id].reg_name = value
-
-#define SRI(reg_name, block, id)\
-	REG_STRUCT.reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define SRI_ARR(reg_name, block, id)\
-	REG_STRUCT[id].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define SR_ARR_I2C(reg_name, id) \
-	REG_STRUCT[id-1].reg_name = BASE(reg##reg_name##_BASE_IDX) + reg##reg_name
-
-#define SRI_ARR_I2C(reg_name, block, id)\
-	REG_STRUCT[id-1].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define SRI_ARR_ALPHABET(reg_name, block, index, id)\
-	REG_STRUCT[index].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define SRI2(reg_name, block, id)\
-	.reg_name = BASE(reg ## reg_name ## _BASE_IDX) + \
-					reg ## reg_name
-
-#define SRI2_ARR(reg_name, block, id)\
-	REG_STRUCT[id].reg_name = BASE(reg ## reg_name ## _BASE_IDX) +	\
-		reg ## reg_name
-
-#define SRIR(var_name, reg_name, block, id)\
-	.var_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define SRII(reg_name, block, id)\
-	REG_STRUCT.reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define SRII_ARR_2(reg_name, block, id, inst)\
-	REG_STRUCT[inst].reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define SRII_MPC_RMU(reg_name, block, id)\
-	.RMU##_##reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
-
-#define SRII_DWB(reg_name, temp_name, block, id)\
-	REG_STRUCT.reg_name[id] = BASE(reg ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## temp_name
-
-#define SF_DWB2(reg_name, block, id, field_name, post_fix) \
-	.field_name = reg_name ## __ ## field_name ## post_fix
-
-#define DCCG_SRII(reg_name, block, id)\
-	REG_STRUCT.block ## _ ## reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-		reg ## block ## id ## _ ## reg_name
-
-#define VUPDATE_SRII(reg_name, block, id)\
-	REG_STRUCT.reg_name[id] = BASE(reg ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
-		reg ## reg_name ## _ ## block ## id
-
-/* NBIO */
-#define NBIO_BASE_INNER(seg) ctx->nbio_reg_offsets[seg]
-
-#define NBIO_BASE(seg) \
-	NBIO_BASE_INNER(seg)
-
-#define NBIO_SR(reg_name)\
-	REG_STRUCT.reg_name = NBIO_BASE(regBIF_BX2_ ## reg_name ## _BASE_IDX) + \
-				regBIF_BX2_ ## reg_name
-
-#define NBIO_SR_ARR(reg_name, id)\
-	REG_STRUCT[id].reg_name = NBIO_BASE(regBIF_BX2_ ## reg_name ## _BASE_IDX) + \
-		regBIF_BX2_ ## reg_name
-
-#define bios_regs_init() \
-		( \
-		NBIO_SR(BIOS_SCRATCH_3),\
-		NBIO_SR(BIOS_SCRATCH_6)\
-		)
-
-static struct bios_registers bios_regs;
-
-#define clk_src_regs_init(index, pllid)\
-	CS_COMMON_REG_LIST_DCN3_0_RI(index, pllid)
-
-static struct dce110_clk_src_regs clk_src_regs[5];
-
-static const struct dce110_clk_src_shift cs_shift = {
-		CS_COMMON_MASK_SH_LIST_DCN3_1_4(__SHIFT)
-};
-
-static const struct dce110_clk_src_mask cs_mask = {
-		CS_COMMON_MASK_SH_LIST_DCN3_1_4(_MASK)
-};
-
-#define abm_regs_init(id)\
-		ABM_DCN32_REG_LIST_RI(id)
-
-static struct dce_abm_registers abm_regs[4];
-
-static const struct dce_abm_shift abm_shift = {
-		ABM_MASK_SH_LIST_DCN35(__SHIFT)
-};
-
-static const struct dce_abm_mask abm_mask = {
-		ABM_MASK_SH_LIST_DCN35(_MASK)
-};
-
-#define audio_regs_init(id)\
-		AUD_COMMON_REG_LIST_RI(id)
-
-static struct dce_audio_registers audio_regs[7];
-
-
-#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
-		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
-		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
-
-static const struct dce_audio_shift audio_shift = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_audio_mask audio_mask = {
-		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
-};
-
-#define vpg_regs_init(id)\
-	VPG_DCN31_REG_LIST_RI(id)
-
-static struct dcn31_vpg_registers vpg_regs[10];
-
-static const struct dcn31_vpg_shift vpg_shift = {
-	DCN31_VPG_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_vpg_mask vpg_mask = {
-	DCN31_VPG_MASK_SH_LIST(_MASK)
-};
-
-#define afmt_regs_init(id)\
-	AFMT_DCN31_REG_LIST_RI(id)
-
-static struct dcn31_afmt_registers afmt_regs[6];
-
-static const struct dcn31_afmt_shift afmt_shift = {
-	DCN31_AFMT_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_afmt_mask afmt_mask = {
-	DCN31_AFMT_MASK_SH_LIST(_MASK)
-};
-
-#define apg_regs_init(id)\
-	APG_DCN31_REG_LIST_RI(id)
-
-static struct dcn31_apg_registers apg_regs[4];
-
-static const struct dcn31_apg_shift apg_shift = {
-	DCN31_APG_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_apg_mask apg_mask = {
-	DCN31_APG_MASK_SH_LIST(_MASK)
-};
-
-#define stream_enc_regs_init(id)\
-	SE_DCN35_REG_LIST_RI(id)
-
-static struct dcn10_stream_enc_registers stream_enc_regs[5];
-
-static const struct dcn10_stream_encoder_shift se_shift = {
-		SE_COMMON_MASK_SH_LIST_DCN35(__SHIFT)
-};
-
-static const struct dcn10_stream_encoder_mask se_mask = {
-		SE_COMMON_MASK_SH_LIST_DCN35(_MASK)
-};
-
-#define aux_regs_init(id)\
-	DCN2_AUX_REG_LIST_RI(id)
-
-static struct dcn10_link_enc_aux_registers link_enc_aux_regs[5];
-
-#define hpd_regs_init(id)\
-	HPD_REG_LIST_RI(id)
-
-static struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[5];
-
-
-static const struct dce110_aux_registers_shift aux_shift = {
-	DCN_AUX_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce110_aux_registers_mask aux_mask = {
-	DCN_AUX_MASK_SH_LIST(_MASK)
-};
-
-#define link_regs_init(id, phyid)\
-	( \
-	LE_DCN35_REG_LIST_RI(id), \
-	UNIPHY_DCN2_REG_LIST_RI(id, phyid)\
-	)
-
-static struct dcn10_link_enc_registers link_enc_regs[5];
-
-static const struct dcn10_link_enc_shift le_shift = {
-	LINK_ENCODER_MASK_SH_LIST_DCN35(__SHIFT), \
-	//DPCS_DCN31_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn10_link_enc_mask le_mask = {
-	LINK_ENCODER_MASK_SH_LIST_DCN35(_MASK), \
-	//DPCS_DCN31_MASK_SH_LIST(_MASK)
-};
-
-#define hpo_dp_stream_encoder_reg_init(id)\
-	DCN3_1_HPO_DP_STREAM_ENC_REG_LIST_RI(id)
-
-static struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[4];
-
-static const struct dcn31_hpo_dp_stream_encoder_shift hpo_dp_se_shift = {
-	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_hpo_dp_stream_encoder_mask hpo_dp_se_mask = {
-	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(_MASK)
-};
-
-#define hpo_dp_link_encoder_reg_init(id)\
-	DCN3_1_HPO_DP_LINK_ENC_REG_LIST_RI(id)
-	/*DCN3_1_RDPCSTX_REG_LIST(0),*/
-	/*DCN3_1_RDPCSTX_REG_LIST(1),*/
-	/*DCN3_1_RDPCSTX_REG_LIST(2),*/
-	/*DCN3_1_RDPCSTX_REG_LIST(3),*/
-
-static struct dcn31_hpo_dp_link_encoder_registers hpo_dp_link_enc_regs[2];
-
-static const struct dcn31_hpo_dp_link_encoder_shift hpo_dp_le_shift = {
-	DCN3_1_HPO_DP_LINK_ENC_COMMON_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn31_hpo_dp_link_encoder_mask hpo_dp_le_mask = {
-	DCN3_1_HPO_DP_LINK_ENC_COMMON_MASK_SH_LIST(_MASK)
-};
-
-#define dpp_regs_init(id)\
-	DPP_REG_LIST_DCN35_RI(id)
-
-static struct dcn3_dpp_registers dpp_regs[4];
-
-static const struct dcn35_dpp_shift tf_shift = {
-		DPP_REG_LIST_SH_MASK_DCN35(__SHIFT)
-};
-
-static const struct dcn35_dpp_mask tf_mask = {
-		DPP_REG_LIST_SH_MASK_DCN35(_MASK)
-};
-
-#define opp_regs_init(id)\
-	OPP_REG_LIST_DCN35_RI(id)
-
-static struct dcn35_opp_registers opp_regs[4];
-
-static const struct dcn35_opp_shift opp_shift = {
-	OPP_MASK_SH_LIST_DCN35(__SHIFT)
-};
-
-static const struct dcn35_opp_mask opp_mask = {
-	OPP_MASK_SH_LIST_DCN35(_MASK)
-};
-
-#define aux_engine_regs_init(id)\
-	( \
-	AUX_COMMON_REG_LIST0_RI(id), \
-	SR_ARR_INIT(AUXN_IMPCAL, id, 0), \
-	SR_ARR_INIT(AUXP_IMPCAL, id, 0), \
-	SR_ARR_INIT(AUX_RESET_MASK, id, DP_AUX0_AUX_CONTROL__AUX_RESET_MASK) \
-	)
-
-static struct dce110_aux_registers aux_engine_regs[5];
-
-#define dwbc_regs_dcn3_init(id)\
-	DWBC_COMMON_REG_LIST_DCN30_RI(id)
-
-static struct dcn30_dwbc_registers dwbc35_regs[1];
-
-static const struct dcn35_dwbc_shift dwbc35_shift = {
-	DWBC_COMMON_MASK_SH_LIST_DCN35(__SHIFT)
-};
-
-static const struct dcn35_dwbc_mask dwbc35_mask = {
-	DWBC_COMMON_MASK_SH_LIST_DCN35(_MASK)
-};
-
-#define mcif_wb_regs_dcn3_init(id)\
-	MCIF_WB_COMMON_REG_LIST_DCN3_5_RI(id)
-
-static struct dcn35_mmhubbub_registers mcif_wb35_regs[1];
-
-static const struct dcn35_mmhubbub_shift mcif_wb35_shift = {
-	MCIF_WB_COMMON_MASK_SH_LIST_DCN3_5(__SHIFT)
-};
-
-static const struct dcn35_mmhubbub_mask mcif_wb35_mask = {
-	MCIF_WB_COMMON_MASK_SH_LIST_DCN3_5(_MASK)
-};
-
-#define dsc_regsDCN35_init(id)\
-	DSC_REG_LIST_DCN20_RI(id)
-
-static struct dcn20_dsc_registers dsc_regs[4];
-
-static const struct dcn35_dsc_shift dsc_shift = {
-	DSC_REG_LIST_SH_MASK_DCN35(__SHIFT)
-};
-
-static const struct dcn35_dsc_mask dsc_mask = {
-	DSC_REG_LIST_SH_MASK_DCN35(_MASK)
-};
-
-static struct dcn30_mpc_registers mpc_regs;
-
-#define dcn_mpc_regs_init() \
-	MPC_REG_LIST_DCN3_2_RI(0),\
-	MPC_REG_LIST_DCN3_2_RI(1),\
-	MPC_REG_LIST_DCN3_2_RI(2),\
-	MPC_REG_LIST_DCN3_2_RI(3),\
-	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(0),\
-	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(1),\
-	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(2),\
-	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(3),\
-	MPC_DWB_MUX_REG_LIST_DCN3_0_RI(0)
-
-static const struct dcn30_mpc_shift mpc_shift = {
-	MPC_COMMON_MASK_SH_LIST_DCN32(__SHIFT)
-};
-
-static const struct dcn30_mpc_mask mpc_mask = {
-	MPC_COMMON_MASK_SH_LIST_DCN32(_MASK)
-};
-
-#define optc_regs_init(id)\
-	OPTC_COMMON_REG_LIST_DCN3_5_RI(id)
-
-static struct dcn_optc_registers optc_regs[4];
-
-static const struct dcn_optc_shift optc_shift = {
-	OPTC_COMMON_MASK_SH_LIST_DCN3_5(__SHIFT)
-};
-
-static const struct dcn_optc_mask optc_mask = {
-	OPTC_COMMON_MASK_SH_LIST_DCN3_5(_MASK)
-};
-
-#define hubp_regs_init(id)\
-	HUBP_REG_LIST_DCN30_RI(id)
-
-static struct dcn_hubp2_registers hubp_regs[4];
-
-
-static const struct dcn35_hubp2_shift hubp_shift = {
-		HUBP_MASK_SH_LIST_DCN35(__SHIFT)
-};
-
-static const struct dcn35_hubp2_mask hubp_mask = {
-		HUBP_MASK_SH_LIST_DCN35(_MASK)
-};
-
-static struct dcn_hubbub_registers hubbub_reg;
-
-#define hubbub_reg_init()\
-		HUBBUB_REG_LIST_DCN35(0)
-
-static const struct dcn_hubbub_shift hubbub_shift = {
-		HUBBUB_MASK_SH_LIST_DCN35(__SHIFT)
-};
-
-static const struct dcn_hubbub_mask hubbub_mask = {
-		HUBBUB_MASK_SH_LIST_DCN35(_MASK)
-};
-
-static struct dccg_registers dccg_regs;
-
-#define dccg_regs_init()\
-	DCCG_REG_LIST_DCN35()
-
-static const struct dccg_shift dccg_shift = {
-		DCCG_MASK_SH_LIST_DCN35(__SHIFT)
-};
-
-static const struct dccg_mask dccg_mask = {
-		DCCG_MASK_SH_LIST_DCN35(_MASK)
-};
-
-static struct pg_cntl_registers pg_cntl_regs;
-
-#define pg_cntl_dcn35_regs_init() \
-	PG_CNTL_REG_LIST_DCN35()
-
-static const struct pg_cntl_shift pg_cntl_shift = {
-		PG_CNTL_MASK_SH_LIST_DCN35(__SHIFT)
-};
-
-static const struct pg_cntl_mask pg_cntl_mask = {
-		PG_CNTL_MASK_SH_LIST_DCN35(_MASK)
-};
-
-#define SRII2(reg_name_pre, reg_name_post, id)\
-	.reg_name_pre ## _ ##  reg_name_post[id] = BASE(reg ## reg_name_pre \
-			## id ## _ ## reg_name_post ## _BASE_IDX) + \
-			reg ## reg_name_pre ## id ## _ ## reg_name_post
-
-static struct dce_hwseq_registers hwseq_reg;
-
-#define hwseq_reg_init()\
-	HWSEQ_DCN35_REG_LIST()
-
-#define HWSEQ_DCN35_MASK_SH_LIST(mask_sh)\
-	HWSEQ_DCN_MASK_SH_LIST(mask_sh), \
-	HWS_SF(, DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, mask_sh), \
-	HWS_SF(, DCHUBBUB_ARB_HOSTVM_CNTL, DISABLE_HOSTVM_FORCE_ALLOW_PSTATE, mask_sh), \
-	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN22_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN22_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN23_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN23_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN24_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN24_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN25_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
-	HWS_SF(, DOMAIN25_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
-	HWS_SF(, DOMAIN0_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN1_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN2_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN3_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN16_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN17_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN18_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN19_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN22_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN23_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN24_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DOMAIN25_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
-	HWS_SF(, DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh), \
-	HWS_SF(, AZALIA_AUDIO_DTO, AZALIA_AUDIO_DTO_MODULE, mask_sh), \
-	HWS_SF(, HPO_TOP_CLOCK_CONTROL, HPO_HDMISTREAMCLK_G_GATE_DIS, mask_sh), \
-	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_UNASSIGNED_PWR_MODE, mask_sh), \
-	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_VBLANK_PWR_MODE, mask_sh), \
-	HWS_SF(, DIO_MEM_PWR_CTRL, I2C_LIGHT_SLEEP_FORCE, mask_sh), \
-	HWS_SF(, HPO_TOP_HW_CONTROL, HPO_IO_EN, mask_sh),\
-	HWS_SF(, DMU_CLK_CNTL, DISPCLK_R_DMU_GATE_DIS, mask_sh),\
-	HWS_SF(, DMU_CLK_CNTL, DISPCLK_G_RBBMIF_GATE_DIS, mask_sh),\
-	HWS_SF(, DMU_CLK_CNTL, RBBMIF_FGCG_REP_DIS, mask_sh),\
-	HWS_SF(, DMU_CLK_CNTL, DPREFCLK_ALLOW_DS_CLKSTOP, mask_sh),\
-	HWS_SF(, DMU_CLK_CNTL, DISPCLK_ALLOW_DS_CLKSTOP, mask_sh),\
-	HWS_SF(, DMU_CLK_CNTL, DPPCLK_ALLOW_DS_CLKSTOP, mask_sh),\
-	HWS_SF(, DMU_CLK_CNTL, DTBCLK_ALLOW_DS_CLKSTOP, mask_sh),\
-	HWS_SF(, DMU_CLK_CNTL, DCFCLK_ALLOW_DS_CLKSTOP, mask_sh),\
-	HWS_SF(, DMU_CLK_CNTL, DPIACLK_ALLOW_DS_CLKSTOP, mask_sh),\
-	HWS_SF(, DMU_CLK_CNTL, LONO_FGCG_REP_DIS, mask_sh),\
-	HWS_SF(, DMU_CLK_CNTL, LONO_DISPCLK_GATE_DISABLE, mask_sh),\
-	HWS_SF(, DMU_CLK_CNTL, LONO_SOCCLK_GATE_DISABLE, mask_sh),\
-	HWS_SF(, DMU_CLK_CNTL, LONO_DMCUBCLK_GATE_DISABLE, mask_sh),\
-	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKA_FE_GATE_DISABLE, mask_sh), \
-	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKB_FE_GATE_DISABLE, mask_sh), \
-	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKC_FE_GATE_DISABLE, mask_sh), \
-	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKD_FE_GATE_DISABLE, mask_sh), \
-	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKE_FE_GATE_DISABLE, mask_sh), \
-	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, HDMICHARCLK0_GATE_DISABLE, mask_sh), \
-	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKA_GATE_DISABLE, mask_sh), \
-	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKB_GATE_DISABLE, mask_sh), \
-	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKC_GATE_DISABLE, mask_sh), \
-	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKD_GATE_DISABLE, mask_sh), \
-	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKE_GATE_DISABLE, mask_sh), \
-	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, PHYASYMCLK_ROOT_GATE_DISABLE, mask_sh), \
-	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, PHYBSYMCLK_ROOT_GATE_DISABLE, mask_sh), \
-	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, PHYCSYMCLK_ROOT_GATE_DISABLE, mask_sh), \
-	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, PHYDSYMCLK_ROOT_GATE_DISABLE, mask_sh), \
-	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, PHYESYMCLK_ROOT_GATE_DISABLE, mask_sh)
-
-static const struct dce_hwseq_shift hwseq_shift = {
-		HWSEQ_DCN35_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dce_hwseq_mask hwseq_mask = {
-		HWSEQ_DCN35_MASK_SH_LIST(_MASK)
-};
-
-#define vmid_regs_init(id)\
-		DCN20_VMID_REG_LIST_RI(id)
-
-static struct dcn_vmid_registers vmid_regs[16];
-
-static const struct dcn20_vmid_shift vmid_shifts = {
-		DCN20_VMID_MASK_SH_LIST(__SHIFT)
-};
-
-static const struct dcn20_vmid_mask vmid_masks = {
-		DCN20_VMID_MASK_SH_LIST(_MASK)
-};
-
-static const struct resource_caps res_cap_dcn35 = {
-	.num_timing_generator = 4,
-	.num_opp = 4,
-	.num_video_plane = 4,
-	.num_audio = 5,
-	.num_stream_encoder = 5,
-	.num_dig_link_enc = 5,
-	.num_hpo_dp_stream_encoder = 4,
-	.num_hpo_dp_link_encoder = 2,
-	.num_pll = 4,/*1 c10 edp, 3xc20 combo PHY*/
-	.num_dwb = 1,
-	.num_ddc = 5,
-	.num_vmid = 16,
-	.num_mpc_3dlut = 2,
-	.num_dsc = 4,
-};
-
-static const struct dc_plane_cap plane_cap = {
-	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
-	.per_pixel_alpha = true,
-
-	.pixel_format_support = {
-			.argb8888 = true,
-			.nv12 = true,
-			.fp16 = true,
-			.p010 = true,
-			.ayuv = false,
-	},
-
-	.max_upscale_factor = {
-			.argb8888 = 16000,
-			.nv12 = 16000,
-			.fp16 = 16000
-	},
-
-	// 6:1 downscaling ratio: 1000/6 = 166.666
-	.max_downscale_factor = {
-			.argb8888 = 167,
-			.nv12 = 167,
-			.fp16 = 167
-	},
-	64,
-	64
-};
-
-static const struct dc_debug_options debug_defaults_drv = {
-	.disable_dmcu = true,
-	.force_abm_enable = false,
-	.timing_trace = false,
-	.clock_trace = true,
-	.disable_pplib_clock_request = false,
-	.pipe_split_policy = MPC_SPLIT_AVOID,
-	.force_single_disp_pipe_split = false,
-	.disable_dcc = DCC_ENABLE,
-	.disable_dpp_power_gate = true,
-	.disable_hubp_power_gate = true,
-	.disable_clock_gate = true,
-	.disable_dsc_power_gate = true,
-	.vsr_support = true,
-	.performance_trace = false,
-	.max_downscale_src_width = 4096,/*upto true 4k*/
-	.disable_pplib_wm_range = false,
-	.scl_reset_length10 = true,
-	.sanity_checks = false,
-	.underflow_assert_delay_us = 0xFFFFFFFF,
-	.dwb_fi_phase = -1, // -1 = disable,
-	.dmub_command_table = true,
-	.pstate_enabled = true,
-	.use_max_lb = true,
-	.enable_mem_low_power = {
-		.bits = {
-			.vga = false,
-			.i2c = true,
-			.dmcu = false, // This is previously known to cause hang on S3 cycles if enabled
-			.dscl = true,
-			.cm = false,
-			.mpc = true,
-			.optc = true,
-			.vpg = true,
-			.afmt = true,
-		}
-	},
-	.root_clock_optimization = {
-		.bits = {
-			.dpp = true,
-			.dsc = true,/*dscclk and dsc pg*/
-			.hdmistream = true,
-			.hdmichar = true,
-			.dpstream = true,
-			.symclk32_se = true,
-			.symclk32_le = true,
-			.symclk_fe = true,
-			.physymclk = true,
-			.dpiasymclk = true,
-		}
-	},
-	.seamless_boot_odm_combine = DML_FAIL_SOURCE_PIXEL_FORMAT,
-	.enable_z9_disable_interface = true, /* Allow support for the PMFW interface for disable Z9*/
-	.using_dml2 = true,
-	.support_eDP1_5 = true,
-	.enable_hpo_pg_support = false,
-	.enable_legacy_fast_update = true,
-	.enable_single_display_2to1_odm_policy = false,
-	.disable_idle_power_optimizations = true,
-	.dmcub_emulation = false,
-	.disable_boot_optimizations = false,
-	.disable_unbounded_requesting = false,
-	.disable_mem_low_power = false,
-	//must match enable_single_display_2to1_odm_policy to support dynamic ODM transitions
-	.enable_double_buffered_dsc_pg_support = true,
-	.enable_dp_dig_pixel_rate_div_policy = 1,
-	.disable_z10 = false,
-	.ignore_pg = true,
-	.psp_disabled_wa = true,
-	.ips2_eval_delay_us = 200,
-	.ips2_entry_delay_us = 400
-};
-
-static const struct dc_panel_config panel_config_defaults = {
-	.psr = {
-		.disable_psr = false,
-		.disallow_psrsu = false,
-	},
-	.ilr = {
-		.optimize_edp_link_rate = true,
-	},
-};
-
-static void dcn35_dpp_destroy(struct dpp **dpp)
-{
-	kfree(TO_DCN20_DPP(*dpp));
-	*dpp = NULL;
-}
-
-static struct dpp *dcn35_dpp_create(struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn3_dpp *dpp = kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
-	bool success = (dpp != NULL);
-
-	if (!success)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT dpp_regs
-	dpp_regs_init(0),
-	dpp_regs_init(1),
-	dpp_regs_init(2),
-	dpp_regs_init(3);
-
-	success = dpp35_construct(dpp, ctx, inst, &dpp_regs[inst], &tf_shift,
-				  &tf_mask);
-	if (success) {
-		dpp35_set_fgcg(
-			dpp,
-			ctx->dc->debug.enable_fine_grain_clock_gating.bits.dpp);
-		return &dpp->base;
-	}
-
-	BREAK_TO_DEBUGGER();
-	kfree(dpp);
-	return NULL;
-}
-
-static struct output_pixel_processor *dcn35_opp_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_opp *opp =
-		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
-
-	if (!opp) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-#undef REG_STRUCT
-#define REG_STRUCT opp_regs
-	opp_regs_init(0),
-	opp_regs_init(1),
-	opp_regs_init(2),
-	opp_regs_init(3);
-
-	dcn35_opp_construct(opp, ctx, inst,
-			&opp_regs[inst], &opp_shift, &opp_mask);
-
-	dcn35_opp_set_fgcg(opp, ctx->dc->debug.enable_fine_grain_clock_gating.bits.opp);
-
-	return &opp->base;
-}
-
-static struct dce_aux *dcn31_aux_engine_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct aux_engine_dce110 *aux_engine =
-		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
-
-	if (!aux_engine)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT aux_engine_regs
-	aux_engine_regs_init(0),
-	aux_engine_regs_init(1),
-	aux_engine_regs_init(2),
-	aux_engine_regs_init(3),
-	aux_engine_regs_init(4);
-
-	dce110_aux_engine_construct(aux_engine, ctx, inst,
-				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
-				    &aux_engine_regs[inst],
-					&aux_mask,
-					&aux_shift,
-					ctx->dc->caps.extended_aux_timeout_support);
-
-	return &aux_engine->base;
-}
-
-#define i2c_inst_regs_init(id)\
-	I2C_HW_ENGINE_COMMON_REG_LIST_DCN30_RI(id)
-
-static struct dce_i2c_registers i2c_hw_regs[5];
-
-static const struct dce_i2c_shift i2c_shifts = {
-		I2C_COMMON_MASK_SH_LIST_DCN35(__SHIFT)
-};
-
-static const struct dce_i2c_mask i2c_masks = {
-		I2C_COMMON_MASK_SH_LIST_DCN35(_MASK)
-};
-
-/* ========================================================== */
-
-/*
- * DPIA index | Preferred Encoder     |    Host Router
- *   0        |      C                |       0
- *   1        |      First Available  |       0
- *   2        |      D                |       1
- *   3        |      First Available  |       1
- */
-/* ========================================================== */
-static const enum engine_id dpia_to_preferred_enc_id_table[] = {
-		ENGINE_ID_DIGC,
-		ENGINE_ID_DIGC,
-		ENGINE_ID_DIGD,
-		ENGINE_ID_DIGD
-};
-
-static enum engine_id dcn35_get_preferred_eng_id_dpia(unsigned int dpia_index)
-{
-	return dpia_to_preferred_enc_id_table[dpia_index];
-}
-
-static struct dce_i2c_hw *dcn31_i2c_hw_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dce_i2c_hw *dce_i2c_hw =
-		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
-
-	if (!dce_i2c_hw)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT i2c_hw_regs
-	i2c_inst_regs_init(1),
-	i2c_inst_regs_init(2),
-	i2c_inst_regs_init(3),
-	i2c_inst_regs_init(4),
-	i2c_inst_regs_init(5);
-
-	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
-				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
-
-	return dce_i2c_hw;
-}
-static struct mpc *dcn35_mpc_create(
-		struct dc_context *ctx,
-		int num_mpcc,
-		int num_rmu)
-{
-	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc), GFP_KERNEL);
-
-	if (!mpc30)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT mpc_regs
-	dcn_mpc_regs_init();
-
-	dcn32_mpc_construct(mpc30, ctx,
-			&mpc_regs,
-			&mpc_shift,
-			&mpc_mask,
-			num_mpcc,
-			num_rmu);
-
-	return &mpc30->base;
-}
-
-static struct hubbub *dcn35_hubbub_create(struct dc_context *ctx)
-{
-	int i;
-
-	struct dcn20_hubbub *hubbub3 = kzalloc(sizeof(struct dcn20_hubbub),
-					  GFP_KERNEL);
-
-	if (!hubbub3)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT hubbub_reg
-	hubbub_reg_init();
-
-#undef REG_STRUCT
-#define REG_STRUCT vmid_regs
-	vmid_regs_init(0),
-	vmid_regs_init(1),
-	vmid_regs_init(2),
-	vmid_regs_init(3),
-	vmid_regs_init(4),
-	vmid_regs_init(5),
-	vmid_regs_init(6),
-	vmid_regs_init(7),
-	vmid_regs_init(8),
-	vmid_regs_init(9),
-	vmid_regs_init(10),
-	vmid_regs_init(11),
-	vmid_regs_init(12),
-	vmid_regs_init(13),
-	vmid_regs_init(14),
-	vmid_regs_init(15);
-
-	hubbub35_construct(hubbub3, ctx,
-			&hubbub_reg,
-			&hubbub_shift,
-			&hubbub_mask,
-			384,/*ctx->dc->dml.ip.det_buffer_size_kbytes,*/
-			8, /*ctx->dc->dml.ip.pixel_chunk_size_kbytes,*/
-			1792 /*ctx->dc->dml.ip.config_return_buffer_size_in_kbytes*/);
-
-
-	for (i = 0; i < res_cap_dcn35.num_vmid; i++) {
-		struct dcn20_vmid *vmid = &hubbub3->vmid[i];
-
-		vmid->ctx = ctx;
-
-		vmid->regs = &vmid_regs[i];
-		vmid->shifts = &vmid_shifts;
-		vmid->masks = &vmid_masks;
-	}
-
-	return &hubbub3->base;
-}
-
-static struct timing_generator *dcn35_timing_generator_create(
-		struct dc_context *ctx,
-		uint32_t instance)
-{
-	struct optc *tgn10 =
-		kzalloc(sizeof(struct optc), GFP_KERNEL);
-
-	if (!tgn10)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT optc_regs
-	optc_regs_init(0),
-	optc_regs_init(1),
-	optc_regs_init(2),
-	optc_regs_init(3);
-
-	tgn10->base.inst = instance;
-	tgn10->base.ctx = ctx;
-
-	tgn10->tg_regs = &optc_regs[instance];
-	tgn10->tg_shift = &optc_shift;
-	tgn10->tg_mask = &optc_mask;
-
-	dcn35_timing_generator_init(tgn10);
-
-	return &tgn10->base;
-}
-
-static const struct encoder_feature_support link_enc_feature = {
-		.max_hdmi_deep_color = COLOR_DEPTH_121212,
-		.max_hdmi_pixel_clock = 600000,
-		.hdmi_ycbcr420_supported = true,
-		.dp_ycbcr420_supported = true,
-		.fec_supported = true,
-		.flags.bits.IS_HBR2_CAPABLE = true,
-		.flags.bits.IS_HBR3_CAPABLE = true,
-		.flags.bits.IS_TPS3_CAPABLE = true,
-		.flags.bits.IS_TPS4_CAPABLE = true
-};
-
-static struct link_encoder *dcn35_link_encoder_create(
-	struct dc_context *ctx,
-	const struct encoder_init_data *enc_init_data)
-{
-	struct dcn20_link_encoder *enc20 =
-		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
-
-	if (!enc20)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT link_enc_aux_regs
-	aux_regs_init(0),
-	aux_regs_init(1),
-	aux_regs_init(2),
-	aux_regs_init(3),
-	aux_regs_init(4);
-
-#undef REG_STRUCT
-#define REG_STRUCT link_enc_hpd_regs
-	hpd_regs_init(0),
-	hpd_regs_init(1),
-	hpd_regs_init(2),
-	hpd_regs_init(3),
-	hpd_regs_init(4);
-
-#undef REG_STRUCT
-#define REG_STRUCT link_enc_regs
-	link_regs_init(0, A),
-	link_regs_init(1, B),
-	link_regs_init(2, C),
-	link_regs_init(3, D),
-	link_regs_init(4, E);
-
-	dcn35_link_encoder_construct(enc20,
-			enc_init_data,
-			&link_enc_feature,
-			&link_enc_regs[enc_init_data->transmitter],
-			&link_enc_aux_regs[enc_init_data->channel - 1],
-			&link_enc_hpd_regs[enc_init_data->hpd_source],
-			&le_shift,
-			&le_mask);
-
-	return &enc20->enc10.base;
-}
-
-/* Create a minimal link encoder object not associated with a particular
- * physical connector.
- * resource_funcs.link_enc_create_minimal
- */
-static struct link_encoder *dcn31_link_enc_create_minimal(
-		struct dc_context *ctx, enum engine_id eng_id)
-{
-	struct dcn20_link_encoder *enc20;
-
-	if ((eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
-		return NULL;
-
-	enc20 = kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
-	if (!enc20)
-		return NULL;
-
-	dcn31_link_encoder_construct_minimal(
-			enc20,
-			ctx,
-			&link_enc_feature,
-			&link_enc_regs[eng_id - ENGINE_ID_DIGA],
-			eng_id);
-
-	return &enc20->enc10.base;
-}
-
-static struct panel_cntl *dcn31_panel_cntl_create(const struct panel_cntl_init_data *init_data)
-{
-	struct dcn31_panel_cntl *panel_cntl =
-		kzalloc(sizeof(struct dcn31_panel_cntl), GFP_KERNEL);
-
-	if (!panel_cntl)
-		return NULL;
-
-	dcn31_panel_cntl_construct(panel_cntl, init_data);
-
-	return &panel_cntl->base;
-}
-
-static void read_dce_straps(
-	struct dc_context *ctx,
-	struct resource_straps *straps)
-{
-	generic_reg_get(ctx, regDC_PINSTRAPS + BASE(regDC_PINSTRAPS_BASE_IDX),
-		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
-
-}
-
-static struct audio *dcn31_create_audio(
-		struct dc_context *ctx, unsigned int inst)
-{
-
-#undef REG_STRUCT
-#define REG_STRUCT audio_regs
-	audio_regs_init(0),
-	audio_regs_init(1),
-	audio_regs_init(2),
-	audio_regs_init(3),
-	audio_regs_init(4);
-	audio_regs_init(5);
-	audio_regs_init(6);
-
-	return dce_audio_create(ctx, inst,
-			&audio_regs[inst], &audio_shift, &audio_mask);
-}
-
-static struct vpg *dcn31_vpg_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn31_vpg *vpg31 = kzalloc(sizeof(struct dcn31_vpg), GFP_KERNEL);
-
-	if (!vpg31)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT vpg_regs
-	vpg_regs_init(0),
-	vpg_regs_init(1),
-	vpg_regs_init(2),
-	vpg_regs_init(3),
-	vpg_regs_init(4),
-	vpg_regs_init(5),
-	vpg_regs_init(6),
-	vpg_regs_init(7),
-	vpg_regs_init(8),
-	vpg_regs_init(9);
-
-	vpg31_construct(vpg31, ctx, inst,
-			&vpg_regs[inst],
-			&vpg_shift,
-			&vpg_mask);
-
-	return &vpg31->base;
-}
-
-static struct afmt *dcn31_afmt_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn31_afmt *afmt31 = kzalloc(sizeof(struct dcn31_afmt), GFP_KERNEL);
-
-	if (!afmt31)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT afmt_regs
-	afmt_regs_init(0),
-	afmt_regs_init(1),
-	afmt_regs_init(2),
-	afmt_regs_init(3),
-	afmt_regs_init(4),
-	afmt_regs_init(5);
-
-	afmt31_construct(afmt31, ctx, inst,
-			&afmt_regs[inst],
-			&afmt_shift,
-			&afmt_mask);
-
-	// Light sleep by default, no need to power down here
-
-	return &afmt31->base;
-}
-
-static struct apg *dcn31_apg_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn31_apg *apg31 = kzalloc(sizeof(struct dcn31_apg), GFP_KERNEL);
-
-	if (!apg31)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT apg_regs
-	apg_regs_init(0),
-	apg_regs_init(1),
-	apg_regs_init(2),
-	apg_regs_init(3);
-
-	apg31_construct(apg31, ctx, inst,
-			&apg_regs[inst],
-			&apg_shift,
-			&apg_mask);
-
-	return &apg31->base;
-}
-
-static struct stream_encoder *dcn35_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dcn10_stream_encoder *enc1;
-	struct vpg *vpg;
-	struct afmt *afmt;
-	int vpg_inst;
-	int afmt_inst;
-
-	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
-	if (eng_id <= ENGINE_ID_DIGF) {
-		vpg_inst = eng_id;
-		afmt_inst = eng_id;
-	} else
-		return NULL;
-
-	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
-	vpg = dcn31_vpg_create(ctx, vpg_inst);
-	afmt = dcn31_afmt_create(ctx, afmt_inst);
-
-	if (!enc1 || !vpg || !afmt) {
-		kfree(enc1);
-		kfree(vpg);
-		kfree(afmt);
-		return NULL;
-	}
-
-#undef REG_STRUCT
-#define REG_STRUCT stream_enc_regs
-	stream_enc_regs_init(0),
-	stream_enc_regs_init(1),
-	stream_enc_regs_init(2),
-	stream_enc_regs_init(3),
-	stream_enc_regs_init(4);
-
-	dcn35_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
-					eng_id, vpg, afmt,
-					&stream_enc_regs[eng_id],
-					&se_shift, &se_mask);
-
-	return &enc1->base;
-}
-
-static struct hpo_dp_stream_encoder *dcn31_hpo_dp_stream_encoder_create(
-	enum engine_id eng_id,
-	struct dc_context *ctx)
-{
-	struct dcn31_hpo_dp_stream_encoder *hpo_dp_enc31;
-	struct vpg *vpg;
-	struct apg *apg;
-	uint32_t hpo_dp_inst;
-	uint32_t vpg_inst;
-	uint32_t apg_inst;
-
-	ASSERT((eng_id >= ENGINE_ID_HPO_DP_0) && (eng_id <= ENGINE_ID_HPO_DP_3));
-	hpo_dp_inst = eng_id - ENGINE_ID_HPO_DP_0;
-
-	/* Mapping of VPG register blocks to HPO DP block instance:
-	 * VPG[6] -> HPO_DP[0]
-	 * VPG[7] -> HPO_DP[1]
-	 * VPG[8] -> HPO_DP[2]
-	 * VPG[9] -> HPO_DP[3]
-	 */
-	vpg_inst = hpo_dp_inst + 6;
-
-	/* Mapping of APG register blocks to HPO DP block instance:
-	 * APG[0] -> HPO_DP[0]
-	 * APG[1] -> HPO_DP[1]
-	 * APG[2] -> HPO_DP[2]
-	 * APG[3] -> HPO_DP[3]
-	 */
-	apg_inst = hpo_dp_inst;
-
-	/* allocate HPO stream encoder and create VPG sub-block */
-	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_stream_encoder), GFP_KERNEL);
-	vpg = dcn31_vpg_create(ctx, vpg_inst);
-	apg = dcn31_apg_create(ctx, apg_inst);
-
-	if (!hpo_dp_enc31 || !vpg || !apg) {
-		kfree(hpo_dp_enc31);
-		kfree(vpg);
-		kfree(apg);
-		return NULL;
-	}
-
-#undef REG_STRUCT
-#define REG_STRUCT hpo_dp_stream_enc_regs
-	hpo_dp_stream_encoder_reg_init(0),
-	hpo_dp_stream_encoder_reg_init(1),
-	hpo_dp_stream_encoder_reg_init(2),
-	hpo_dp_stream_encoder_reg_init(3);
-
-	dcn31_hpo_dp_stream_encoder_construct(hpo_dp_enc31, ctx, ctx->dc_bios,
-					hpo_dp_inst, eng_id, vpg, apg,
-					&hpo_dp_stream_enc_regs[hpo_dp_inst],
-					&hpo_dp_se_shift, &hpo_dp_se_mask);
-
-	return &hpo_dp_enc31->base;
-}
-
-static struct hpo_dp_link_encoder *dcn31_hpo_dp_link_encoder_create(
-	uint8_t inst,
-	struct dc_context *ctx)
-{
-	struct dcn31_hpo_dp_link_encoder *hpo_dp_enc31;
-
-	/* allocate HPO link encoder */
-	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
-
-#undef REG_STRUCT
-#define REG_STRUCT hpo_dp_link_enc_regs
-	hpo_dp_link_encoder_reg_init(0),
-	hpo_dp_link_encoder_reg_init(1);
-
-	hpo_dp_link_encoder31_construct(hpo_dp_enc31, ctx, inst,
-					&hpo_dp_link_enc_regs[inst],
-					&hpo_dp_le_shift, &hpo_dp_le_mask);
-
-	return &hpo_dp_enc31->base;
-}
-
-static struct dce_hwseq *dcn35_hwseq_create(
-	struct dc_context *ctx)
-{
-	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
-
-#undef REG_STRUCT
-#define REG_STRUCT hwseq_reg
-	hwseq_reg_init();
-
-	if (hws) {
-		hws->ctx = ctx;
-		hws->regs = &hwseq_reg;
-		hws->shifts = &hwseq_shift;
-		hws->masks = &hwseq_mask;
-	}
-	return hws;
-}
-static const struct resource_create_funcs res_create_funcs = {
-	.read_dce_straps = read_dce_straps,
-	.create_audio = dcn31_create_audio,
-	.create_stream_encoder = dcn35_stream_encoder_create,
-	.create_hpo_dp_stream_encoder = dcn31_hpo_dp_stream_encoder_create,
-	.create_hpo_dp_link_encoder = dcn31_hpo_dp_link_encoder_create,
-	.create_hwseq = dcn35_hwseq_create,
-};
-
-static void dcn35_resource_destruct(struct dcn35_resource_pool *pool)
-{
-	unsigned int i;
-
-	for (i = 0; i < pool->base.stream_enc_count; i++) {
-		if (pool->base.stream_enc[i] != NULL) {
-			if (pool->base.stream_enc[i]->vpg != NULL) {
-				kfree(DCN30_VPG_FROM_VPG(pool->base.stream_enc[i]->vpg));
-				pool->base.stream_enc[i]->vpg = NULL;
-			}
-			if (pool->base.stream_enc[i]->afmt != NULL) {
-				kfree(DCN30_AFMT_FROM_AFMT(pool->base.stream_enc[i]->afmt));
-				pool->base.stream_enc[i]->afmt = NULL;
-			}
-			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
-			pool->base.stream_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.hpo_dp_stream_enc_count; i++) {
-		if (pool->base.hpo_dp_stream_enc[i] != NULL) {
-			if (pool->base.hpo_dp_stream_enc[i]->vpg != NULL) {
-				kfree(DCN30_VPG_FROM_VPG(pool->base.hpo_dp_stream_enc[i]->vpg));
-				pool->base.hpo_dp_stream_enc[i]->vpg = NULL;
-			}
-			if (pool->base.hpo_dp_stream_enc[i]->apg != NULL) {
-				kfree(DCN31_APG_FROM_APG(pool->base.hpo_dp_stream_enc[i]->apg));
-				pool->base.hpo_dp_stream_enc[i]->apg = NULL;
-			}
-			kfree(DCN3_1_HPO_DP_STREAM_ENC_FROM_HPO_STREAM_ENC(pool->base.hpo_dp_stream_enc[i]));
-			pool->base.hpo_dp_stream_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.hpo_dp_link_enc_count; i++) {
-		if (pool->base.hpo_dp_link_enc[i] != NULL) {
-			kfree(DCN3_1_HPO_DP_LINK_ENC_FROM_HPO_LINK_ENC(pool->base.hpo_dp_link_enc[i]));
-			pool->base.hpo_dp_link_enc[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		if (pool->base.dscs[i] != NULL)
-			dcn20_dsc_destroy(&pool->base.dscs[i]);
-	}
-
-	if (pool->base.mpc != NULL) {
-		kfree(TO_DCN20_MPC(pool->base.mpc));
-		pool->base.mpc = NULL;
-	}
-	if (pool->base.hubbub != NULL) {
-		kfree(pool->base.hubbub);
-		pool->base.hubbub = NULL;
-	}
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		if (pool->base.dpps[i] != NULL)
-			dcn35_dpp_destroy(&pool->base.dpps[i]);
-
-		if (pool->base.ipps[i] != NULL)
-			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
-
-		if (pool->base.hubps[i] != NULL) {
-			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
-			pool->base.hubps[i] = NULL;
-		}
-
-		if (pool->base.irqs != NULL) {
-			dal_irq_service_destroy(&pool->base.irqs);
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		if (pool->base.engines[i] != NULL)
-			dce110_engine_destroy(&pool->base.engines[i]);
-		if (pool->base.hw_i2cs[i] != NULL) {
-			kfree(pool->base.hw_i2cs[i]);
-			pool->base.hw_i2cs[i] = NULL;
-		}
-		if (pool->base.sw_i2cs[i] != NULL) {
-			kfree(pool->base.sw_i2cs[i]);
-			pool->base.sw_i2cs[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
-		if (pool->base.opps[i] != NULL)
-			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		if (pool->base.timing_generators[i] != NULL)	{
-			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
-			pool->base.timing_generators[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
-		if (pool->base.dwbc[i] != NULL) {
-			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
-			pool->base.dwbc[i] = NULL;
-		}
-		if (pool->base.mcif_wb[i] != NULL) {
-			kfree(TO_DCN30_MMHUBBUB(pool->base.mcif_wb[i]));
-			pool->base.mcif_wb[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.audio_count; i++) {
-		if (pool->base.audios[i])
-			dce_aud_destroy(&pool->base.audios[i]);
-	}
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] != NULL) {
-			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
-			pool->base.clock_sources[i] = NULL;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
-		if (pool->base.mpc_lut[i] != NULL) {
-			dc_3dlut_func_release(pool->base.mpc_lut[i]);
-			pool->base.mpc_lut[i] = NULL;
-		}
-		if (pool->base.mpc_shaper[i] != NULL) {
-			dc_transfer_func_release(pool->base.mpc_shaper[i]);
-			pool->base.mpc_shaper[i] = NULL;
-		}
-	}
-
-	if (pool->base.dp_clock_source != NULL) {
-		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
-		pool->base.dp_clock_source = NULL;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		if (pool->base.multiple_abms[i] != NULL)
-			dce_abm_destroy(&pool->base.multiple_abms[i]);
-	}
-
-	if (pool->base.psr != NULL)
-		dmub_psr_destroy(&pool->base.psr);
-
-	if (pool->base.pg_cntl != NULL)
-		dcn_pg_cntl_destroy(&pool->base.pg_cntl);
-
-	if (pool->base.dccg != NULL)
-		dcn_dccg_destroy(&pool->base.dccg);
-}
-
-static struct hubp *dcn35_hubp_create(
-	struct dc_context *ctx,
-	uint32_t inst)
-{
-	struct dcn20_hubp *hubp2 =
-		kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
-
-	if (!hubp2)
-		return NULL;
-
-#undef REG_STRUCT
-#define REG_STRUCT hubp_regs
-	hubp_regs_init(0),
-	hubp_regs_init(1),
-	hubp_regs_init(2),
-	hubp_regs_init(3);
-
-	if (hubp35_construct(hubp2, ctx, inst,
-			&hubp_regs[inst], &hubp_shift, &hubp_mask))
-		return &hubp2->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(hubp2);
-	return NULL;
-}
-
-static void dcn35_dwbc_init(struct dcn30_dwbc *dwbc30, struct dc_context *ctx)
-{
-	dcn35_dwbc_set_fgcg(
-		dwbc30, ctx->dc->debug.enable_fine_grain_clock_gating.bits.dwb);
-}
-
-static bool dcn35_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t pipe_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < pipe_count; i++) {
-		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc),
-						    GFP_KERNEL);
-
-		if (!dwbc30) {
-			dm_error("DC: failed to create dwbc30!\n");
-			return false;
-		}
-
-#undef REG_STRUCT
-#define REG_STRUCT dwbc35_regs
-		dwbc_regs_dcn3_init(0);
-
-		dcn35_dwbc_construct(dwbc30, ctx,
-				&dwbc35_regs[i],
-				&dwbc35_shift,
-				&dwbc35_mask,
-				i);
-
-		pool->dwbc[i] = &dwbc30->base;
-
-		dcn35_dwbc_init(dwbc30, ctx);
-	}
-	return true;
-}
-
-static void dcn35_mmhubbub_init(struct dcn30_mmhubbub *mcif_wb30,
-				struct dc_context *ctx)
-{
-	dcn35_mmhubbub_set_fgcg(
-		mcif_wb30,
-		ctx->dc->debug.enable_fine_grain_clock_gating.bits.mmhubbub);
-}
-
-static bool dcn35_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
-{
-	int i;
-	uint32_t pipe_count = pool->res_cap->num_dwb;
-
-	for (i = 0; i < pipe_count; i++) {
-		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub),
-						    GFP_KERNEL);
-
-		if (!mcif_wb30) {
-			dm_error("DC: failed to create mcif_wb30!\n");
-			return false;
-		}
-
-#undef REG_STRUCT
-#define REG_STRUCT mcif_wb35_regs
-		mcif_wb_regs_dcn3_init(0);
-
-		dcn35_mmhubbub_construct(mcif_wb30, ctx,
-				&mcif_wb35_regs[i],
-				&mcif_wb35_shift,
-				&mcif_wb35_mask,
-				i);
-
-		dcn35_mmhubbub_init(mcif_wb30, ctx);
-
-		pool->mcif_wb[i] = &mcif_wb30->base;
-	}
-	return true;
-}
-
-static struct display_stream_compressor *dcn35_dsc_create(
-	struct dc_context *ctx, uint32_t inst)
-{
-	struct dcn20_dsc *dsc =
-		kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
-
-	if (!dsc) {
-		BREAK_TO_DEBUGGER();
-		return NULL;
-	}
-
-#undef REG_STRUCT
-#define REG_STRUCT dsc_regs
-	dsc_regsDCN35_init(0),
-	dsc_regsDCN35_init(1),
-	dsc_regsDCN35_init(2),
-	dsc_regsDCN35_init(3);
-
-	dsc35_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
-	dsc35_set_fgcg(dsc,
-		       ctx->dc->debug.enable_fine_grain_clock_gating.bits.dsc);
-	return &dsc->base;
-}
-
-static void dcn35_destroy_resource_pool(struct resource_pool **pool)
-{
-	struct dcn35_resource_pool *dcn35_pool = TO_DCN35_RES_POOL(*pool);
-
-	dcn35_resource_destruct(dcn35_pool);
-	kfree(dcn35_pool);
-	*pool = NULL;
-}
-
-static struct clock_source *dcn35_clock_source_create(
-		struct dc_context *ctx,
-		struct dc_bios *bios,
-		enum clock_source_id id,
-		const struct dce110_clk_src_regs *regs,
-		bool dp_clk_src)
-{
-	struct dce110_clk_src *clk_src =
-		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
-
-	if (!clk_src)
-		return NULL;
-
-	if (dcn31_clk_src_construct(clk_src, ctx, bios, id,
-			regs, &cs_shift, &cs_mask)) {
-		clk_src->base.dp_clk_src = dp_clk_src;
-		return &clk_src->base;
-	}
-
-	BREAK_TO_DEBUGGER();
-	return NULL;
-}
-
-static struct dc_cap_funcs cap_funcs = {
-	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
-};
-
-static void dcn35_get_panel_config_defaults(struct dc_panel_config *panel_config)
-{
-	*panel_config = panel_config_defaults;
-}
-
-
-static bool dcn35_validate_bandwidth(struct dc *dc,
-		struct dc_state *context,
-		bool fast_validate)
-{
-	bool out = false;
-
-	out = dml2_validate(dc, context, fast_validate);
-
-	return out;
-}
-
-
-static struct resource_funcs dcn35_res_pool_funcs = {
-	.destroy = dcn35_destroy_resource_pool,
-	.link_enc_create = dcn35_link_encoder_create,
-	.link_enc_create_minimal = dcn31_link_enc_create_minimal,
-	.link_encs_assign = link_enc_cfg_link_encs_assign,
-	.link_enc_unassign = link_enc_cfg_link_enc_unassign,
-	.panel_cntl_create = dcn31_panel_cntl_create,
-	.validate_bandwidth = dcn35_validate_bandwidth,
-	.calculate_wm_and_dlg = NULL,
-	.update_soc_for_wm_a = dcn31_update_soc_for_wm_a,
-	.populate_dml_pipes = dcn35_populate_dml_pipes_from_context_fpu,
-	.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
-	.release_pipe = dcn20_release_pipe,
-	.add_stream_to_ctx = dcn30_add_stream_to_ctx,
-	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
-	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
-	.populate_dml_writeback_from_context = dcn30_populate_dml_writeback_from_context,
-	.set_mcif_arb_params = dcn30_set_mcif_arb_params,
-	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
-	.acquire_post_bldn_3dlut = dcn30_acquire_post_bldn_3dlut,
-	.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
-	.update_bw_bounding_box = dcn35_update_bw_bounding_box_fpu,
-	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
-	.get_panel_config_defaults = dcn35_get_panel_config_defaults,
-	.get_preferred_eng_id_dpia = dcn35_get_preferred_eng_id_dpia,
-};
-
-static bool dcn35_resource_construct(
-	uint8_t num_virtual_links,
-	struct dc *dc,
-	struct dcn35_resource_pool *pool)
-{
-	int i;
-	struct dc_context *ctx = dc->ctx;
-	struct irq_service_init_data init_data;
-
-#undef REG_STRUCT
-#define REG_STRUCT bios_regs
-	bios_regs_init();
-
-#undef REG_STRUCT
-#define REG_STRUCT clk_src_regs
-	clk_src_regs_init(0, A),
-	clk_src_regs_init(1, B),
-	clk_src_regs_init(2, C),
-	clk_src_regs_init(3, D),
-	clk_src_regs_init(4, E);
-
-#undef REG_STRUCT
-#define REG_STRUCT abm_regs
-	abm_regs_init(0),
-	abm_regs_init(1),
-	abm_regs_init(2),
-	abm_regs_init(3);
-
-#undef REG_STRUCT
-#define REG_STRUCT dccg_regs
-	dccg_regs_init();
-
-	ctx->dc_bios->regs = &bios_regs;
-
-	pool->base.res_cap = &res_cap_dcn35;
-
-	pool->base.funcs = &dcn35_res_pool_funcs;
-
-	/*************************************************
-	 *  Resource + asic cap harcoding                *
-	 *************************************************/
-	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
-	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
-	pool->base.mpcc_count = pool->base.res_cap->num_timing_generator;
-	dc->caps.max_downscale_ratio = 600;
-	dc->caps.i2c_speed_in_khz = 100;
-	dc->caps.i2c_speed_in_khz_hdcp = 100;
-	dc->caps.max_cursor_size = 256;
-	dc->caps.min_horizontal_blanking_period = 80;
-	dc->caps.dmdata_alloc_size = 2048;
-	dc->caps.max_slave_planes = 2;
-	dc->caps.max_slave_yuv_planes = 2;
-	dc->caps.max_slave_rgb_planes = 2;
-	dc->caps.post_blend_color_processing = true;
-	dc->caps.force_dp_tps4_for_cp2520 = true;
-	if (dc->config.forceHBR2CP2520)
-		dc->caps.force_dp_tps4_for_cp2520 = false;
-	dc->caps.dp_hpo = true;
-	dc->caps.dp_hdmi21_pcon_support = true;
-
-	dc->caps.edp_dsc_support = true;
-	dc->caps.extended_aux_timeout_support = true;
-	dc->caps.dmcub_support = true;
-	dc->caps.is_apu = true;
-	dc->caps.seamless_odm = true;
-
-	dc->caps.zstate_support = true;
-	dc->caps.ips_support = true;
-	dc->caps.max_v_total = (1 << 15) - 1;
-
-	/* Color pipeline capabilities */
-	dc->caps.color.dpp.dcn_arch = 1;
-	dc->caps.color.dpp.input_lut_shared = 0;
-	dc->caps.color.dpp.icsc = 1;
-	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
-	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
-	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
-	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
-	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
-	dc->caps.color.dpp.post_csc = 1;
-	dc->caps.color.dpp.gamma_corr = 1;
-	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
-
-	dc->caps.color.dpp.hw_3d_lut = 1;
-	dc->caps.color.dpp.ogam_ram = 0;  // no OGAM in DPP since DCN1
-	// no OGAM ROM on DCN301
-	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
-	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
-	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
-	dc->caps.color.dpp.ocsc = 0;
-
-	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //2
-	dc->caps.color.mpc.ogam_ram = 1;
-	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
-	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
-	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
-	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
-	dc->caps.color.mpc.ocsc = 1;
-
-	/* max_disp_clock_khz_at_vmin is slightly lower than the STA value in order
-	 * to provide some margin.
-	 * It's expected for furture ASIC to have equal or higher value, in order to
-	 * have determinstic power improvement from generate to genration.
-	 * (i.e., we should not expect new ASIC generation with lower vmin rate)
-	 */
-	dc->caps.max_disp_clock_khz_at_vmin = 650000;
-
-	/* Use pipe context based otg sync logic */
-	dc->config.use_pipe_ctx_sync_logic = true;
-	dc->config.use_default_clock_table = false;
-	/* read VBIOS LTTPR caps */
-	{
-		if (ctx->dc_bios->funcs->get_lttpr_caps) {
-			enum bp_result bp_query_result;
-			uint8_t is_vbios_lttpr_enable = 0;
-
-			bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
-			dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
-		}
-
-		/* interop bit is implicit */
-		{
-			dc->caps.vbios_lttpr_aware = true;
-		}
-	}
-
-	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
-		dc->debug = debug_defaults_drv;
-
-	// Init the vm_helper
-	if (dc->vm_helper)
-		vm_helper_init(dc->vm_helper, 16);
-
-	/*************************************************
-	 *  Create resources                             *
-	 *************************************************/
-
-	/* Clock Sources for Pixel Clock*/
-	pool->base.clock_sources[DCN35_CLK_SRC_PLL0] =
-			dcn35_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL0,
-				&clk_src_regs[0], false);
-	pool->base.clock_sources[DCN35_CLK_SRC_PLL1] =
-			dcn35_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL1,
-				&clk_src_regs[1], false);
-	pool->base.clock_sources[DCN35_CLK_SRC_PLL2] =
-			dcn35_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL2,
-				&clk_src_regs[2], false);
-	pool->base.clock_sources[DCN35_CLK_SRC_PLL3] =
-			dcn35_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL3,
-				&clk_src_regs[3], false);
-	pool->base.clock_sources[DCN35_CLK_SRC_PLL4] =
-			dcn35_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_COMBO_PHY_PLL4,
-				&clk_src_regs[4], false);
-
-	pool->base.clk_src_count = DCN35_CLK_SRC_TOTAL;
-
-	/* todo: not reuse phy_pll registers */
-	pool->base.dp_clock_source =
-			dcn35_clock_source_create(ctx, ctx->dc_bios,
-				CLOCK_SOURCE_ID_DP_DTO,
-				&clk_src_regs[0], true);
-
-	for (i = 0; i < pool->base.clk_src_count; i++) {
-		if (pool->base.clock_sources[i] == NULL) {
-			dm_error("DC: failed to create clock sources!\n");
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-	/*temp till dml2 fully work without dml1*/
-	dml_init_instance(&dc->dml, &dcn3_5_soc, &dcn3_5_ip, DML_PROJECT_DCN31);
-
-	/* TODO: DCCG */
-	pool->base.dccg = dccg35_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
-	if (pool->base.dccg == NULL) {
-		dm_error("DC: failed to create dccg!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-#undef REG_STRUCT
-#define REG_STRUCT pg_cntl_regs
-	pg_cntl_dcn35_regs_init();
-
-	pool->base.pg_cntl = pg_cntl35_create(ctx, &pg_cntl_regs, &pg_cntl_shift, &pg_cntl_mask);
-	if (pool->base.pg_cntl == NULL) {
-		dm_error("DC: failed to create power gate control!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	/* TODO: IRQ */
-	init_data.ctx = dc->ctx;
-	pool->base.irqs = dal_irq_service_dcn35_create(&init_data);
-	if (!pool->base.irqs)
-		goto create_fail;
-
-	/* HUBBUB */
-	pool->base.hubbub = dcn35_hubbub_create(ctx);
-	if (pool->base.hubbub == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create hubbub!\n");
-		goto create_fail;
-	}
-
-	/* HUBPs, DPPs, OPPs and TGs */
-	for (i = 0; i < pool->base.pipe_count; i++) {
-		pool->base.hubps[i] = dcn35_hubp_create(ctx, i);
-		if (pool->base.hubps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create hubps!\n");
-			goto create_fail;
-		}
-
-		pool->base.dpps[i] = dcn35_dpp_create(ctx, i);
-		if (pool->base.dpps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create dpps!\n");
-			goto create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
-		pool->base.opps[i] = dcn35_opp_create(ctx, i);
-		if (pool->base.opps[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC: failed to create output pixel processor!\n");
-			goto create_fail;
-		}
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		pool->base.timing_generators[i] = dcn35_timing_generator_create(
-				ctx, i);
-		if (pool->base.timing_generators[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create tg!\n");
-			goto create_fail;
-		}
-	}
-	pool->base.timing_generator_count = i;
-
-	/* PSR */
-	pool->base.psr = dmub_psr_create(ctx);
-	if (pool->base.psr == NULL) {
-		dm_error("DC: failed to create psr obj!\n");
-		BREAK_TO_DEBUGGER();
-		goto create_fail;
-	}
-
-	/* ABM */
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
-		pool->base.multiple_abms[i] = dmub_abm_create(ctx,
-				&abm_regs[i],
-				&abm_shift,
-				&abm_mask);
-		if (pool->base.multiple_abms[i] == NULL) {
-			dm_error("DC: failed to create abm for pipe %d!\n", i);
-			BREAK_TO_DEBUGGER();
-			goto create_fail;
-		}
-	}
-
-	/* MPC and DSC */
-	pool->base.mpc = dcn35_mpc_create(ctx, pool->base.mpcc_count, pool->base.res_cap->num_mpc_3dlut);
-	if (pool->base.mpc == NULL) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mpc!\n");
-		goto create_fail;
-	}
-
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
-		pool->base.dscs[i] = dcn35_dsc_create(ctx, i);
-		if (pool->base.dscs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create display stream compressor %d!\n", i);
-			goto create_fail;
-		}
-	}
-
-	/* DWB and MMHUBBUB */
-	if (!dcn35_dwbc_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create dwbc!\n");
-		goto create_fail;
-	}
-
-	if (!dcn35_mmhubbub_create(ctx, &pool->base)) {
-		BREAK_TO_DEBUGGER();
-		dm_error("DC: failed to create mcif_wb!\n");
-		goto create_fail;
-	}
-
-	/* AUX and I2C */
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
-		pool->base.engines[i] = dcn31_aux_engine_create(ctx, i);
-		if (pool->base.engines[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create aux engine!!\n");
-			goto create_fail;
-		}
-		pool->base.hw_i2cs[i] = dcn31_i2c_hw_create(ctx, i);
-		if (pool->base.hw_i2cs[i] == NULL) {
-			BREAK_TO_DEBUGGER();
-			dm_error(
-				"DC:failed to create hw i2c!!\n");
-			goto create_fail;
-		}
-		pool->base.sw_i2cs[i] = NULL;
-	}
-
-	/* DCN3.5 has 6 DPIA */
-	pool->base.usb4_dpia_count = 4;
-	if (dc->debug.dpia_debug.bits.disable_dpia)
-		pool->base.usb4_dpia_count = 0;
-
-	/* Audio, Stream Encoders including HPO and virtual, MPC 3D LUTs */
-	if (!resource_construct(num_virtual_links, dc, &pool->base,
-			&res_create_funcs))
-		goto create_fail;
-
-	/* HW Sequencer and Plane caps */
-	dcn35_hw_sequencer_construct(dc);
-
-	dc->caps.max_planes =  pool->base.pipe_count;
-
-	for (i = 0; i < dc->caps.max_planes; ++i)
-		dc->caps.planes[i] = plane_cap;
-
-	dc->cap_funcs = cap_funcs;
-
-	dc->dcn_ip->max_num_dpp = pool->base.pipe_count;
-
-	dc->dml2_options.dcn_pipe_count = pool->base.pipe_count;
-	dc->dml2_options.use_native_pstate_optimization = true;
-	dc->dml2_options.use_native_soc_bb_construction = true;
-	if (dc->config.EnableMinDispClkODM)
-		dc->dml2_options.minimize_dispclk_using_odm = true;
-	dc->dml2_options.enable_windowed_mpo_odm = dc->config.enable_windowed_mpo_odm;
-
-	dc->dml2_options.callbacks.dc = dc;
-	dc->dml2_options.callbacks.build_scaling_params = &resource_build_scaling_params;
-	dc->dml2_options.callbacks.can_support_mclk_switch_using_fw_based_vblank_stretch = &dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch;
-	dc->dml2_options.callbacks.acquire_secondary_pipe_for_mpc_odm = &dc_resource_acquire_secondary_pipe_for_mpc_odm_legacy;
-	dc->dml2_options.callbacks.update_pipes_for_stream_with_slice_count = &resource_update_pipes_for_stream_with_slice_count;
-	dc->dml2_options.callbacks.update_pipes_for_plane_with_slice_count = &resource_update_pipes_for_plane_with_slice_count;
-	dc->dml2_options.callbacks.get_mpc_slice_index = &resource_get_mpc_slice_index;
-	dc->dml2_options.callbacks.get_odm_slice_index = &resource_get_odm_slice_index;
-	dc->dml2_options.callbacks.get_opp_head = &resource_get_opp_head;
-	dc->dml2_options.max_segments_per_hubp = 24;
-
-	dc->dml2_options.det_segment_size = DCN3_2_DET_SEG_SIZE;/*todo*/
-
-	if (dc->config.sdpif_request_limit_words_per_umc == 0)
-		dc->config.sdpif_request_limit_words_per_umc = 16;/*todo*/
-
-	return true;
-
-create_fail:
-
-	dcn35_resource_destruct(pool);
-
-	return false;
-}
-
-struct resource_pool *dcn35_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc)
-{
-	struct dcn35_resource_pool *pool =
-		kzalloc(sizeof(struct dcn35_resource_pool), GFP_KERNEL);
-
-	if (!pool)
-		return NULL;
-
-	if (dcn35_resource_construct(init_data->num_virtual_links, dc, pool))
-		return &pool->base;
-
-	BREAK_TO_DEBUGGER();
-	kfree(pool);
-	return NULL;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_resource.h b/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_resource.h
deleted file mode 100644
index 99aea102e3f7..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dcn35/dcn35_resource.h
+++ /dev/null
@@ -1,310 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright 2023 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#ifndef _DCN35_RESOURCE_H_
-#define _DCN35_RESOURCE_H_
-
-#include "core_types.h"
-
-#define DCN3_5_VMIN_DISPCLK_HZ 717000000
-#define TO_DCN35_RES_POOL(pool)\
-	container_of(pool, struct dcn35_resource_pool, base)
-
-extern struct _vcs_dpi_ip_params_st dcn3_5_ip;
-extern struct _vcs_dpi_soc_bounding_box_st dcn3_5_soc;
-
-struct dcn35_resource_pool {
-	struct resource_pool base;
-};
-
-struct resource_pool *dcn35_create_resource_pool(
-		const struct dc_init_data *init_data,
-		struct dc *dc);
-
-/* Defs for runtime init of registers */
-
-#define OPP_REG_LIST_DCN20_RI(id) \
-	OPP_REG_LIST_DCN10_RI(id), \
-	OPP_DPG_REG_LIST_RI(id), \
-	SRI_ARR(FMT_422_CONTROL, FMT, id), \
-	SRI_ARR(OPPBUF_CONTROL1, OPPBUF, id)
-
-#define OPP_REG_LIST_DCN35_RI(id) \
-	OPP_REG_LIST_DCN20_RI(id), \
-	SRI2_ARR(OPP_TOP_CLK_CONTROL, OPP, id)
-
-#define VPG_DCN31_REG_LIST_RI(id) \
-	SRI_ARR(VPG_GENERIC_STATUS, VPG, id), \
-	SRI_ARR(VPG_GENERIC_PACKET_ACCESS_CTRL, VPG, id), \
-	SRI_ARR(VPG_GENERIC_PACKET_DATA, VPG, id), \
-	SRI_ARR(VPG_GSP_FRAME_UPDATE_CTRL, VPG, id), \
-	SRI_ARR(VPG_GSP_IMMEDIATE_UPDATE_CTRL, VPG, id), \
-	SRI_ARR(VPG_MEM_PWR, VPG, id)
-
-#define AFMT_DCN31_REG_LIST_RI(id) \
-	SRI_ARR(AFMT_INFOFRAME_CONTROL0, AFMT, id), \
-	SRI_ARR(AFMT_VBI_PACKET_CONTROL, AFMT, id), \
-	SRI_ARR(AFMT_AUDIO_PACKET_CONTROL, AFMT, id), \
-	SRI_ARR(AFMT_AUDIO_PACKET_CONTROL2, AFMT, id), \
-	SRI_ARR(AFMT_AUDIO_SRC_CONTROL, AFMT, id), \
-	SRI_ARR(AFMT_60958_0, AFMT, id), \
-	SRI_ARR(AFMT_60958_1, AFMT, id), \
-	SRI_ARR(AFMT_60958_2, AFMT, id), \
-	SRI_ARR(AFMT_MEM_PWR, AFMT, id)
-
-/* Stream encoder */
-#define SE_DCN35_REG_LIST_RI(id) \
-	SRI_ARR(AFMT_CNTL, DIG, id), \
-	SRI_ARR(DIG_FE_CNTL, DIG, id), \
-	SRI_ARR(HDMI_CONTROL, DIG, id), \
-	SRI_ARR(HDMI_DB_CONTROL, DIG, id), \
-	SRI_ARR(HDMI_GC, DIG, id), \
-	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL0, DIG, id), \
-	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL1, DIG, id), \
-	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL2, DIG, id), \
-	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL3, DIG, id), \
-	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL4, DIG, id), \
-	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL5, DIG, id), \
-	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL6, DIG, id), \
-	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL7, DIG, id), \
-	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL8, DIG, id), \
-	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL9, DIG, id), \
-	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL10, DIG, id), \
-	SRI_ARR(HDMI_INFOFRAME_CONTROL0, DIG, id), \
-	SRI_ARR(HDMI_INFOFRAME_CONTROL1, DIG, id), \
-	SRI_ARR(HDMI_VBI_PACKET_CONTROL, DIG, id), \
-	SRI_ARR(HDMI_AUDIO_PACKET_CONTROL, DIG, id),\
-	SRI_ARR(HDMI_ACR_PACKET_CONTROL, DIG, id),\
-	SRI_ARR(HDMI_ACR_32_0, DIG, id),\
-	SRI_ARR(HDMI_ACR_32_1, DIG, id),\
-	SRI_ARR(HDMI_ACR_44_0, DIG, id),\
-	SRI_ARR(HDMI_ACR_44_1, DIG, id),\
-	SRI_ARR(HDMI_ACR_48_0, DIG, id),\
-	SRI_ARR(HDMI_ACR_48_1, DIG, id),\
-	SRI_ARR(DP_DB_CNTL, DP, id), \
-	SRI_ARR(DP_MSA_MISC, DP, id), \
-	SRI_ARR(DP_MSA_VBID_MISC, DP, id), \
-	SRI_ARR(DP_MSA_COLORIMETRY, DP, id), \
-	SRI_ARR(DP_MSA_TIMING_PARAM1, DP, id), \
-	SRI_ARR(DP_MSA_TIMING_PARAM2, DP, id), \
-	SRI_ARR(DP_MSA_TIMING_PARAM3, DP, id), \
-	SRI_ARR(DP_MSA_TIMING_PARAM4, DP, id), \
-	SRI_ARR(DP_MSE_RATE_CNTL, DP, id), \
-	SRI_ARR(DP_MSE_RATE_UPDATE, DP, id), \
-	SRI_ARR(DP_PIXEL_FORMAT, DP, id), \
-	SRI_ARR(DP_SEC_CNTL, DP, id), \
-	SRI_ARR(DP_SEC_CNTL1, DP, id), \
-	SRI_ARR(DP_SEC_CNTL2, DP, id), \
-	SRI_ARR(DP_SEC_CNTL5, DP, id), \
-	SRI_ARR(DP_SEC_CNTL6, DP, id), \
-	SRI_ARR(DP_STEER_FIFO, DP, id), \
-	SRI_ARR(DP_VID_M, DP, id), \
-	SRI_ARR(DP_VID_N, DP, id), \
-	SRI_ARR(DP_VID_STREAM_CNTL, DP, id), \
-	SRI_ARR(DP_VID_TIMING, DP, id), \
-	SRI_ARR(DP_SEC_AUD_N, DP, id), \
-	SRI_ARR(DP_SEC_TIMESTAMP, DP, id), \
-	SRI_ARR(DP_DSC_CNTL, DP, id), \
-	SRI_ARR(DP_SEC_METADATA_TRANSMISSION, DP, id), \
-	SRI_ARR(HDMI_METADATA_PACKET_CONTROL, DIG, id), \
-	SRI_ARR(DP_SEC_FRAMING4, DP, id), \
-	SRI_ARR(DP_GSP11_CNTL, DP, id), \
-	SRI_ARR(DME_CONTROL, DME, id),\
-	SRI_ARR(DP_SEC_METADATA_TRANSMISSION, DP, id), \
-	SRI_ARR(HDMI_METADATA_PACKET_CONTROL, DIG, id), \
-	SRI_ARR(DIG_FE_CNTL, DIG, id), \
-	SRI_ARR(DIG_FE_EN_CNTL, DIG, id), \
-	SRI_ARR(DIG_FE_CLK_CNTL, DIG, id), \
-	SRI_ARR(DIG_CLOCK_PATTERN, DIG, id), \
-	SRI_ARR(DIG_FIFO_CTRL0, DIG, id), \
-	SRI_ARR(STREAM_MAPPER_CONTROL, DIG, id)
-
-#define LE_DCN35_REG_LIST_RI(id)\
-	LE_DCN3_REG_LIST_RI(id),\
-	SRI_ARR(DP_DPHY_INTERNAL_CTRL, DP, id), \
-	SR_ARR(DIO_LINKA_CNTL, id), \
-	SR_ARR(DIO_LINKB_CNTL, id), \
-	SR_ARR(DIO_LINKC_CNTL, id), \
-	SR_ARR(DIO_LINKD_CNTL, id), \
-	SR_ARR(DIO_LINKE_CNTL, id), \
-	SR_ARR(DIO_LINKF_CNTL, id),\
-	SRI_ARR(DIG_BE_CLK_CNTL, DIG, id),\
-	SR_ARR(DIO_CLK_CNTL, id)
-
-#define MCIF_WB_COMMON_REG_LIST_DCN3_5_RI(inst)  \
-	MCIF_WB_COMMON_REG_LIST_DCN32_RI(inst), \
-		SRI2_ARR(MMHUBBUB_CLOCK_CNTL, MMHUBBUB, inst)
-
-#define HWSEQ_DCN35_REG_LIST()\
-	SR(DCHUBBUB_GLOBAL_TIMER_CNTL), \
-	SR(DCHUBBUB_ARB_HOSTVM_CNTL), \
-	SR(DIO_MEM_PWR_CTRL), \
-	SR(ODM_MEM_PWR_CTRL3), \
-	SR(MMHUBBUB_MEM_PWR_CNTL), \
-	SR(DCCG_GATE_DISABLE_CNTL), \
-	SR(DCCG_GATE_DISABLE_CNTL2), \
-	SR(DCCG_GATE_DISABLE_CNTL5), \
-	SR(DCFCLK_CNTL),\
-	SR(DC_MEM_GLOBAL_PWR_REQ_CNTL), \
-	SRII(PIXEL_RATE_CNTL, OTG, 0), \
-	SRII(PIXEL_RATE_CNTL, OTG, 1),\
-	SRII(PIXEL_RATE_CNTL, OTG, 2),\
-	SRII(PIXEL_RATE_CNTL, OTG, 3),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 0),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 1),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 2),\
-	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 3),\
-	SR(MICROSECOND_TIME_BASE_DIV), \
-	SR(MILLISECOND_TIME_BASE_DIV), \
-	SR(DISPCLK_FREQ_CHANGE_CNTL), \
-	SR(RBBMIF_TIMEOUT_DIS), \
-	SR(RBBMIF_TIMEOUT_DIS_2), \
-	SR(DCHUBBUB_CRC_CTRL), \
-	SR(DPP_TOP0_DPP_CRC_CTRL), \
-	SR(DPP_TOP0_DPP_CRC_VAL_B_A), \
-	SR(DPP_TOP0_DPP_CRC_VAL_R_G), \
-	SR(MPC_CRC_CTRL), \
-	SR(MPC_CRC_RESULT_GB), \
-	SR(MPC_CRC_RESULT_C), \
-	SR(MPC_CRC_RESULT_AR), \
-	SR(DOMAIN0_PG_CONFIG), \
-	SR(DOMAIN1_PG_CONFIG), \
-	SR(DOMAIN2_PG_CONFIG), \
-	SR(DOMAIN3_PG_CONFIG), \
-	SR(DOMAIN16_PG_CONFIG), \
-	SR(DOMAIN17_PG_CONFIG), \
-	SR(DOMAIN18_PG_CONFIG), \
-	SR(DOMAIN19_PG_CONFIG), \
-	SR(DOMAIN0_PG_STATUS), \
-	SR(DOMAIN1_PG_STATUS), \
-	SR(DOMAIN2_PG_STATUS), \
-	SR(DOMAIN3_PG_STATUS), \
-	SR(DOMAIN16_PG_STATUS), \
-	SR(DOMAIN17_PG_STATUS), \
-	SR(DOMAIN18_PG_STATUS), \
-	SR(DOMAIN19_PG_STATUS), \
-	SR(DC_IP_REQUEST_CNTL), \
-	SR(AZALIA_AUDIO_DTO), \
-	SR(AZALIA_CONTROLLER_CLOCK_GATING), \
-	SR(HPO_TOP_HW_CONTROL),\
-	SR(DMU_CLK_CNTL)
-
-/* OPTC */
-#define OPTC_COMMON_REG_LIST_DCN3_5_RI(inst)                                   \
-	SRI_ARR(OTG_VSTARTUP_PARAM, OTG, inst),\
-	SRI_ARR(OTG_VUPDATE_PARAM, OTG, inst),\
-	SRI_ARR(OTG_VREADY_PARAM, OTG, inst),\
-	SRI_ARR(OTG_MASTER_UPDATE_LOCK, OTG, inst),\
-	SRI_ARR(OTG_GLOBAL_CONTROL0, OTG, inst),\
-	SRI_ARR(OTG_GLOBAL_CONTROL1, OTG, inst),\
-	SRI_ARR(OTG_GLOBAL_CONTROL2, OTG, inst),\
-	SRI_ARR(OTG_GLOBAL_CONTROL4, OTG, inst),\
-	SRI_ARR(OTG_DOUBLE_BUFFER_CONTROL, OTG, inst),\
-	SRI_ARR(OTG_H_TOTAL, OTG, inst),\
-	SRI_ARR(OTG_H_BLANK_START_END, OTG, inst),\
-	SRI_ARR(OTG_H_SYNC_A, OTG, inst),\
-	SRI_ARR(OTG_H_SYNC_A_CNTL, OTG, inst),\
-	SRI_ARR(OTG_H_TIMING_CNTL, OTG, inst),\
-	SRI_ARR(OTG_V_TOTAL, OTG, inst),\
-	SRI_ARR(OTG_V_BLANK_START_END, OTG, inst),\
-	SRI_ARR(OTG_V_SYNC_A, OTG, inst),\
-	SRI_ARR(OTG_V_SYNC_A_CNTL, OTG, inst),\
-	SRI_ARR(OTG_CONTROL, OTG, inst),\
-	SRI_ARR(OTG_STEREO_CONTROL, OTG, inst),\
-	SRI_ARR(OTG_3D_STRUCTURE_CONTROL, OTG, inst),\
-	SRI_ARR(OTG_STEREO_STATUS, OTG, inst),\
-	SRI_ARR(OTG_V_TOTAL_MAX, OTG, inst),\
-	SRI_ARR(OTG_V_TOTAL_MIN, OTG, inst),\
-	SRI_ARR(OTG_V_TOTAL_CONTROL, OTG, inst),\
-	SRI_ARR(OTG_TRIGA_CNTL, OTG, inst),\
-	SRI_ARR(OTG_FORCE_COUNT_NOW_CNTL, OTG, inst),\
-	SRI_ARR(OTG_STATIC_SCREEN_CONTROL, OTG, inst),\
-	SRI_ARR(OTG_STATUS_FRAME_COUNT, OTG, inst),\
-	SRI_ARR(OTG_STATUS, OTG, inst),\
-	SRI_ARR(OTG_STATUS_POSITION, OTG, inst),\
-	SRI_ARR(OTG_NOM_VERT_POSITION, OTG, inst),\
-	SRI_ARR(OTG_M_CONST_DTO0, OTG, inst),\
-	SRI_ARR(OTG_M_CONST_DTO1, OTG, inst),\
-	SRI_ARR(OTG_CLOCK_CONTROL, OTG, inst),\
-	SRI_ARR(OTG_VERTICAL_INTERRUPT0_CONTROL, OTG, inst),\
-	SRI_ARR(OTG_VERTICAL_INTERRUPT0_POSITION, OTG, inst),\
-	SRI_ARR(OTG_VERTICAL_INTERRUPT1_CONTROL, OTG, inst),\
-	SRI_ARR(OTG_VERTICAL_INTERRUPT1_POSITION, OTG, inst),\
-	SRI_ARR(OTG_VERTICAL_INTERRUPT2_CONTROL, OTG, inst),\
-	SRI_ARR(OTG_VERTICAL_INTERRUPT2_POSITION, OTG, inst),\
-	SRI_ARR(OPTC_INPUT_CLOCK_CONTROL, ODM, inst),\
-	SRI_ARR(OPTC_DATA_SOURCE_SELECT, ODM, inst),\
-	SRI_ARR(OPTC_INPUT_GLOBAL_CONTROL, ODM, inst),\
-	SRI_ARR(CONTROL, VTG, inst),\
-	SRI_ARR(OTG_VERT_SYNC_CONTROL, OTG, inst),\
-	SRI_ARR(OTG_GSL_CONTROL, OTG, inst),\
-	SRI_ARR(OTG_CRC_CNTL, OTG, inst),\
-	SRI_ARR(OTG_CRC0_DATA_RG, OTG, inst),\
-	SRI_ARR(OTG_CRC0_DATA_B, OTG, inst),\
-	SRI_ARR(OTG_CRC1_DATA_RG, OTG, inst),\
-	SRI_ARR(OTG_CRC1_DATA_B, OTG, inst),\
-	SRI_ARR(OTG_CRC2_DATA_RG, OTG, inst),\
-	SRI_ARR(OTG_CRC2_DATA_B, OTG, inst),\
-	SRI_ARR(OTG_CRC3_DATA_RG, OTG, inst),\
-	SRI_ARR(OTG_CRC3_DATA_B, OTG, inst),\
-	SRI_ARR(OTG_CRC0_WINDOWA_X_CONTROL, OTG, inst),\
-	SRI_ARR(OTG_CRC0_WINDOWA_Y_CONTROL, OTG, inst),\
-	SRI_ARR(OTG_CRC0_WINDOWB_X_CONTROL, OTG, inst),\
-	SRI_ARR(OTG_CRC0_WINDOWB_Y_CONTROL, OTG, inst),\
-	SRI_ARR(OTG_CRC1_WINDOWA_X_CONTROL, OTG, inst),\
-	SRI_ARR(OTG_CRC1_WINDOWA_Y_CONTROL, OTG, inst),\
-	SRI_ARR(OTG_CRC1_WINDOWB_X_CONTROL, OTG, inst),\
-	SRI_ARR(OTG_CRC1_WINDOWB_Y_CONTROL, OTG, inst),\
-	SRI_ARR(OTG_CRC0_WINDOWA_X_CONTROL_READBACK, OTG, inst),\
-	SRI_ARR(OTG_CRC0_WINDOWA_Y_CONTROL_READBACK, OTG, inst),\
-	SRI_ARR(OTG_CRC0_WINDOWB_X_CONTROL_READBACK, OTG, inst),\
-	SRI_ARR(OTG_CRC0_WINDOWB_Y_CONTROL_READBACK, OTG, inst),\
-	SRI_ARR(OTG_CRC1_WINDOWA_X_CONTROL_READBACK, OTG, inst),\
-	SRI_ARR(OTG_CRC1_WINDOWA_Y_CONTROL_READBACK, OTG, inst),\
-	SRI_ARR(OTG_CRC1_WINDOWB_X_CONTROL_READBACK, OTG, inst),\
-	SRI_ARR(OTG_CRC1_WINDOWB_Y_CONTROL_READBACK, OTG, inst),\
-	SR_ARR(GSL_SOURCE_SELECT, inst),\
-	SRI_ARR(OTG_TRIGA_MANUAL_TRIG, OTG, inst),\
-	SRI_ARR(OTG_GLOBAL_CONTROL1, OTG, inst),\
-	SRI_ARR(OTG_GLOBAL_CONTROL2, OTG, inst),\
-	SRI_ARR(OTG_GSL_WINDOW_X, OTG, inst),\
-	SRI_ARR(OTG_GSL_WINDOW_Y, OTG, inst),\
-	SRI_ARR(OTG_VUPDATE_KEEPOUT, OTG, inst),\
-	SRI_ARR(OTG_DSC_START_POSITION, OTG, inst),\
-	SRI_ARR(OTG_DRR_TRIGGER_WINDOW, OTG, inst),\
-	SRI_ARR(OTG_DRR_V_TOTAL_CHANGE, OTG, inst),\
-	SRI_ARR(OPTC_DATA_FORMAT_CONTROL, ODM, inst),\
-	SRI_ARR(OPTC_BYTES_PER_PIXEL, ODM, inst),\
-	SRI_ARR(OPTC_WIDTH_CONTROL, ODM, inst),\
-	SRI_ARR(OPTC_MEMORY_CONFIG, ODM, inst),\
-	SRI_ARR(OTG_DRR_CONTROL, OTG, inst),\
-	SRI2_ARR(OPTC_CLOCK_CONTROL, OPTC, inst)
-
-/* DPP */
-#define DPP_REG_LIST_DCN35_RI(id)\
-	DPP_REG_LIST_DCN30_COMMON_RI(id)
-
-#endif /* _DCN35_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c
index 50b0434354f8..0c4a8fe8e5ca 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c
@@ -30,7 +30,7 @@
 #include "dcn_calc_auto.h"
 #include "dal_asic_id.h"
 #include "resource.h"
-#include "dcn10/dcn10_resource.h"
+#include "resource/dcn10/dcn10_resource.h"
 #include "dcn10/dcn10_hubbub.h"
 #include "dml/dml1_display_rq_dlg_calc.h"
 
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce/dce_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dce/dce_hwseq.h
index 44b4df6469d1..52f045cfd52a 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dce/dce_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dce/dce_hwseq.h
@@ -682,6 +682,7 @@ struct dce_hwseq_registers {
 	uint32_t DCHUBBUB_ARB_HOSTVM_CNTL;
 	uint32_t HPO_TOP_HW_CONTROL;
 	uint32_t DMU_CLK_CNTL;
+	uint32_t DCCG_GATE_DISABLE_CNTL4;
 	uint32_t DCCG_GATE_DISABLE_CNTL5;
 };
  /* set field name */
@@ -1199,7 +1200,19 @@ struct dce_hwseq_registers {
 	type PHYBSYMCLK_ROOT_GATE_DISABLE;\
 	type PHYCSYMCLK_ROOT_GATE_DISABLE;\
 	type PHYDSYMCLK_ROOT_GATE_DISABLE;\
-	type PHYESYMCLK_ROOT_GATE_DISABLE;
+	type PHYESYMCLK_ROOT_GATE_DISABLE;\
+	type DTBCLK_P0_GATE_DISABLE;\
+	type DTBCLK_P1_GATE_DISABLE;\
+	type DTBCLK_P2_GATE_DISABLE;\
+	type DTBCLK_P3_GATE_DISABLE;\
+	type DPSTREAMCLK0_GATE_DISABLE;\
+	type DPSTREAMCLK1_GATE_DISABLE;\
+	type DPSTREAMCLK2_GATE_DISABLE;\
+	type DPSTREAMCLK3_GATE_DISABLE;\
+	type DPIASYMCLK0_GATE_DISABLE;\
+	type DPIASYMCLK1_GATE_DISABLE;\
+	type DPIASYMCLK2_GATE_DISABLE;\
+	type DPIASYMCLK3_GATE_DISABLE;
 
 struct dce_hwseq_shift {
 	HWSEQ_REG_FIELD_LIST(uint8_t)
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
index 5a8258287438..39260371beb9 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
@@ -145,17 +145,36 @@ void dcn35_init_hw(struct dc *dc)
 		hws->funcs.bios_golden_init(dc);
 	}
 
-	REG_WRITE(DCCG_GATE_DISABLE_CNTL, 0);
-	REG_WRITE(DCCG_GATE_DISABLE_CNTL2,  0);
-
-	/* Disable gating for PHYASYMCLK. This will be enabled in dccg if needed */
-	REG_UPDATE_5(DCCG_GATE_DISABLE_CNTL2, PHYASYMCLK_ROOT_GATE_DISABLE, 1,
-			PHYBSYMCLK_ROOT_GATE_DISABLE, 1,
-			PHYCSYMCLK_ROOT_GATE_DISABLE, 1,
-			PHYDSYMCLK_ROOT_GATE_DISABLE, 1,
-			PHYESYMCLK_ROOT_GATE_DISABLE, 1);
+	if (!dc->debug.disable_clock_gate) {
+		REG_WRITE(DCCG_GATE_DISABLE_CNTL, 0);
+		REG_WRITE(DCCG_GATE_DISABLE_CNTL2,  0);
+
+		/* Disable gating for PHYASYMCLK. This will be enabled in dccg if needed */
+		REG_UPDATE_5(DCCG_GATE_DISABLE_CNTL2, PHYASYMCLK_ROOT_GATE_DISABLE, 1,
+				PHYBSYMCLK_ROOT_GATE_DISABLE, 1,
+				PHYCSYMCLK_ROOT_GATE_DISABLE, 1,
+				PHYDSYMCLK_ROOT_GATE_DISABLE, 1,
+				PHYESYMCLK_ROOT_GATE_DISABLE, 1);
+
+		REG_UPDATE_4(DCCG_GATE_DISABLE_CNTL4,
+				DPIASYMCLK0_GATE_DISABLE, 0,
+				DPIASYMCLK1_GATE_DISABLE, 0,
+				DPIASYMCLK2_GATE_DISABLE, 0,
+				DPIASYMCLK3_GATE_DISABLE, 0);
+
+		REG_WRITE(DCCG_GATE_DISABLE_CNTL5, 0xFFFFFFFF);
+		REG_UPDATE_4(DCCG_GATE_DISABLE_CNTL5,
+				DTBCLK_P0_GATE_DISABLE, 0,
+				DTBCLK_P1_GATE_DISABLE, 0,
+				DTBCLK_P2_GATE_DISABLE, 0,
+				DTBCLK_P3_GATE_DISABLE, 0);
+		REG_UPDATE_4(DCCG_GATE_DISABLE_CNTL5,
+				DPSTREAMCLK0_GATE_DISABLE, 0,
+				DPSTREAMCLK1_GATE_DISABLE, 0,
+				DPSTREAMCLK2_GATE_DISABLE, 0,
+				DPSTREAMCLK3_GATE_DISABLE, 0);
 
-	REG_WRITE(DCCG_GATE_DISABLE_CNTL5, 0x1f7c3fcf);
+	}
 
 	// Initialize the dccg
 	if (res_pool->dccg->funcs->dccg_init)
@@ -332,9 +351,6 @@ void dcn35_init_hw(struct dc *dc)
 	if (dc->res_pool->pg_cntl) {
 		if (dc->res_pool->pg_cntl->funcs->init_pg_status)
 			dc->res_pool->pg_cntl->funcs->init_pg_status(dc->res_pool->pg_cntl);
-
-		if (dc->res_pool->pg_cntl->funcs->set_force_poweron_domain22)
-			dc->res_pool->pg_cntl->funcs->set_force_poweron_domain22(dc->res_pool->pg_cntl, false);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/pg_cntl.h b/drivers/gpu/drm/amd/display/dc/inc/hw/pg_cntl.h
index b9812afb886b..00ea3864dd4d 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/pg_cntl.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/pg_cntl.h
@@ -47,8 +47,6 @@ struct pg_cntl_funcs {
 	void (*optc_pg_control)(struct pg_cntl *pg_cntl, unsigned int optc_inst, bool power_on);
 	void (*dwb_pg_control)(struct pg_cntl *pg_cntl, bool power_on);
 	void (*init_pg_status)(struct pg_cntl *pg_cntl);
-
-	void (*set_force_poweron_domain22)(struct pg_cntl *pg_cntl, bool power_on);
 };
 
 #endif //__DC_PG_CNTL_H__
diff --git a/drivers/gpu/drm/amd/display/dc/inc/resource.h b/drivers/gpu/drm/amd/display/dc/inc/resource.h
index 06ca8bfb91e7..0458d2d749f4 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/resource.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/resource.h
@@ -500,6 +500,18 @@ int recource_find_free_pipe_not_used_in_cur_res_ctx(
 		struct resource_context *new_res_ctx,
 		const struct resource_pool *pool);
 
+/*
+ * Look for a free pipe in new resource context that is used in current resource
+ * context as an OTG master pipe.
+ *
+ * return - FREE_PIPE_INDEX_NOT_FOUND if free pipe is not found, otherwise
+ * pipe idx of the free pipe
+ */
+int recource_find_free_pipe_used_as_otg_master_in_cur_res_ctx(
+		const struct resource_context *cur_res_ctx,
+		struct resource_context *new_res_ctx,
+		const struct resource_pool *pool);
+
 /*
  * Look for a free pipe in new resource context that is used as a secondary DPP
  * pipe in any MPCC combine in current resource context.
diff --git a/drivers/gpu/drm/amd/display/dc/resource/Makefile b/drivers/gpu/drm/amd/display/dc/resource/Makefile
new file mode 100644
index 000000000000..0a75ed8962a5
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/Makefile
@@ -0,0 +1,199 @@
+
+# Copyright 2022 Advanced Micro Devices, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# Makefile for the 'resource' sub-component of DAL.
+#
+
+
+###############################################################################
+#  DCE
+###############################################################################
+
+RESOURCE_DCE100 = dce100_resource.o
+
+AMD_DAL_RESOURCE_DCE100 = $(addprefix $(AMDDALPATH)/dc/resource/dce100/,$(RESOURCE_DCE100))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCE100)
+
+###############################################################################
+
+RESOURCE_DCE110 = dce110_resource.o
+
+AMD_DAL_RESOURCE_DCE110 = $(addprefix $(AMDDALPATH)/dc/resource/dce110/,$(RESOURCE_DCE110))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCE110)
+
+###############################################################################
+
+RESOURCE_DCE112 = dce112_resource.o
+
+AMD_DAL_RESOURCE_DCE112 = $(addprefix $(AMDDALPATH)/dc/resource/dce112/,$(RESOURCE_DCE112))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCE112)
+
+###############################################################################
+
+RESOURCE_DCE120 = dce120_resource.o
+
+AMD_DAL_RESOURCE_DCE120 = $(addprefix $(AMDDALPATH)/dc/resource/dce120/,$(RESOURCE_DCE120))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCE120)
+
+###############################################################################
+
+RESOURCE_DCE80 = dce80_resource.o
+
+AMD_DAL_RESOURCE_DCE80 = $(addprefix $(AMDDALPATH)/dc/resource/dce80/,$(RESOURCE_DCE80))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCE80)
+
+ifdef CONFIG_DRM_AMD_DC_FP
+###############################################################################
+# DCN
+###############################################################################
+
+RESOURCE_DCN10 = dcn10_resource.o
+
+AMD_DAL_RESOURCE_DCN10 = $(addprefix $(AMDDALPATH)/dc/resource/dcn10/,$(RESOURCE_DCN10))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCN10)
+
+###############################################################################
+
+RESOURCE_DCN20 = dcn20_resource.o
+
+AMD_DAL_RESOURCE_DCN20 = $(addprefix $(AMDDALPATH)/dc/resource/dcn20/,$(RESOURCE_DCN20))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCN20)
+
+###############################################################################
+
+RESOURCE_DCN201 = dcn201_resource.o
+
+AMD_DAL_RESOURCE_DCN201 = $(addprefix $(AMDDALPATH)/dc/resource/dcn201/,$(RESOURCE_DCN201))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCN201)
+
+###############################################################################
+
+RESOURCE_DCN21 = dcn21_resource.o
+
+AMD_DAL_RESOURCE_DCN21 = $(addprefix $(AMDDALPATH)/dc/resource/dcn21/,$(RESOURCE_DCN21))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCN21)
+
+###############################################################################
+
+###############################################################################
+
+###############################################################################
+
+RESOURCE_DCN30 = dcn30_resource.o
+
+AMD_DAL_RESOURCE_DCN30 = $(addprefix $(AMDDALPATH)/dc/resource/dcn30/,$(RESOURCE_DCN30))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCN30)
+
+###############################################################################
+
+RESOURCE_DCN301 = dcn301_resource.o
+
+AMD_DAL_RESOURCE_DCN301 = $(addprefix $(AMDDALPATH)/dc/resource/dcn301/,$(RESOURCE_DCN301))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCN301)
+
+###############################################################################
+
+RESOURCE_DCN302 = dcn302_resource.o
+
+AMD_DAL_RESOURCE_DCN302 = $(addprefix $(AMDDALPATH)/dc/resource/dcn302/,$(RESOURCE_DCN302))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCN302)
+
+###############################################################################
+
+RESOURCE_DCN303 = dcn303_resource.o
+
+AMD_DAL_RESOURCE_DCN303 = $(addprefix $(AMDDALPATH)/dc/resource/dcn303/,$(RESOURCE_DCN303))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCN303)
+
+###############################################################################
+
+RESOURCE_DCN31 = dcn31_resource.o
+
+AMD_DAL_RESOURCE_DCN31 = $(addprefix $(AMDDALPATH)/dc/resource/dcn31/,$(RESOURCE_DCN31))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCN31)
+
+###############################################################################
+
+RESOURCE_DCN314 = dcn314_resource.o
+
+AMD_DAL_RESOURCE_DCN314 = $(addprefix $(AMDDALPATH)/dc/resource/dcn314/,$(RESOURCE_DCN314))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCN314)
+
+###############################################################################
+
+RESOURCE_DCN315 = dcn315_resource.o
+
+AMD_DAL_RESOURCE_DCN315 = $(addprefix $(AMDDALPATH)/dc/resource/dcn315/,$(RESOURCE_DCN315))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCN315)
+
+###############################################################################
+
+RESOURCE_DCN316 = dcn316_resource.o
+
+AMD_DAL_RESOURCE_DCN316 = $(addprefix $(AMDDALPATH)/dc/resource/dcn316/,$(RESOURCE_DCN316))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCN316)
+
+###############################################################################
+
+RESOURCE_DCN32 = dcn32_resource.o
+
+AMD_DAL_RESOURCE_DCN32 = $(addprefix $(AMDDALPATH)/dc/resource/dcn32/,$(RESOURCE_DCN32))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCN32)
+
+###############################################################################
+
+RESOURCE_DCN321 = dcn321_resource.o
+
+AMD_DAL_RESOURCE_DCN321 = $(addprefix $(AMDDALPATH)/dc/resource/dcn321/,$(RESOURCE_DCN321))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCN321)
+
+###############################################################################
+
+RESOURCE_DCN35 = dcn35_resource.o
+
+AMD_DAL_RESOURCE_DCN35 = $(addprefix $(AMDDALPATH)/dc/resource/dcn35/,$(RESOURCE_DCN35))
+
+AMD_DISPLAY_FILES += $(AMD_DAL_RESOURCE_DCN35)
+
+###############################################################################
+
+###############################################################################
+
+endif
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c
new file mode 100644
index 000000000000..53a5f4cb648c
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c
@@ -0,0 +1,1179 @@
+/*
+ * Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dm_services.h"
+
+#include "link_encoder.h"
+#include "stream_encoder.h"
+
+#include "resource.h"
+#include "include/irq_service_interface.h"
+#include "virtual/virtual_stream_encoder.h"
+#include "dce110/dce110_resource.h"
+#include "dce110/dce110_timing_generator.h"
+#include "irq/dce110/irq_service_dce110.h"
+#include "dce/dce_link_encoder.h"
+#include "dce/dce_stream_encoder.h"
+#include "dce/dce_mem_input.h"
+#include "dce/dce_ipp.h"
+#include "dce/dce_transform.h"
+#include "dce/dce_opp.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_hwseq.h"
+#include "dce100/dce100_hwseq.h"
+#include "dce/dce_panel_cntl.h"
+
+#include "reg_helper.h"
+
+#include "dce/dce_10_0_d.h"
+#include "dce/dce_10_0_sh_mask.h"
+
+#include "dce/dce_dmcu.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_abm.h"
+#include "dce/dce_i2c.h"
+
+#include "dce100_resource.h"
+
+#ifndef mmMC_HUB_RDREQ_DMIF_LIMIT
+#include "gmc/gmc_8_2_d.h"
+#include "gmc/gmc_8_2_sh_mask.h"
+#endif
+
+#ifndef mmDP_DPHY_INTERNAL_CTRL
+	#define mmDP_DPHY_INTERNAL_CTRL 0x4aa7
+	#define mmDP0_DP_DPHY_INTERNAL_CTRL 0x4aa7
+	#define mmDP1_DP_DPHY_INTERNAL_CTRL 0x4ba7
+	#define mmDP2_DP_DPHY_INTERNAL_CTRL 0x4ca7
+	#define mmDP3_DP_DPHY_INTERNAL_CTRL 0x4da7
+	#define mmDP4_DP_DPHY_INTERNAL_CTRL 0x4ea7
+	#define mmDP5_DP_DPHY_INTERNAL_CTRL 0x4fa7
+	#define mmDP6_DP_DPHY_INTERNAL_CTRL 0x54a7
+	#define mmDP7_DP_DPHY_INTERNAL_CTRL 0x56a7
+	#define mmDP8_DP_DPHY_INTERNAL_CTRL 0x57a7
+#endif
+
+#ifndef mmBIOS_SCRATCH_2
+	#define mmBIOS_SCRATCH_2 0x05CB
+	#define mmBIOS_SCRATCH_3 0x05CC
+	#define mmBIOS_SCRATCH_6 0x05CF
+#endif
+
+#ifndef mmDP_DPHY_BS_SR_SWAP_CNTL
+	#define mmDP_DPHY_BS_SR_SWAP_CNTL                       0x4ADC
+	#define mmDP0_DP_DPHY_BS_SR_SWAP_CNTL                   0x4ADC
+	#define mmDP1_DP_DPHY_BS_SR_SWAP_CNTL                   0x4BDC
+	#define mmDP2_DP_DPHY_BS_SR_SWAP_CNTL                   0x4CDC
+	#define mmDP3_DP_DPHY_BS_SR_SWAP_CNTL                   0x4DDC
+	#define mmDP4_DP_DPHY_BS_SR_SWAP_CNTL                   0x4EDC
+	#define mmDP5_DP_DPHY_BS_SR_SWAP_CNTL                   0x4FDC
+	#define mmDP6_DP_DPHY_BS_SR_SWAP_CNTL                   0x54DC
+#endif
+
+#ifndef mmDP_DPHY_FAST_TRAINING
+	#define mmDP_DPHY_FAST_TRAINING                         0x4ABC
+	#define mmDP0_DP_DPHY_FAST_TRAINING                     0x4ABC
+	#define mmDP1_DP_DPHY_FAST_TRAINING                     0x4BBC
+	#define mmDP2_DP_DPHY_FAST_TRAINING                     0x4CBC
+	#define mmDP3_DP_DPHY_FAST_TRAINING                     0x4DBC
+	#define mmDP4_DP_DPHY_FAST_TRAINING                     0x4EBC
+	#define mmDP5_DP_DPHY_FAST_TRAINING                     0x4FBC
+	#define mmDP6_DP_DPHY_FAST_TRAINING                     0x54BC
+#endif
+
+static const struct dce110_timing_generator_offsets dce100_tg_offsets[] = {
+	{
+		.crtc = (mmCRTC0_CRTC_CONTROL - mmCRTC_CONTROL),
+		.dcp =  (mmDCP0_GRPH_CONTROL - mmGRPH_CONTROL),
+	},
+	{
+		.crtc = (mmCRTC1_CRTC_CONTROL - mmCRTC_CONTROL),
+		.dcp = (mmDCP1_GRPH_CONTROL - mmGRPH_CONTROL),
+	},
+	{
+		.crtc = (mmCRTC2_CRTC_CONTROL - mmCRTC_CONTROL),
+		.dcp = (mmDCP2_GRPH_CONTROL - mmGRPH_CONTROL),
+	},
+	{
+		.crtc = (mmCRTC3_CRTC_CONTROL - mmCRTC_CONTROL),
+		.dcp =  (mmDCP3_GRPH_CONTROL - mmGRPH_CONTROL),
+	},
+	{
+		.crtc = (mmCRTC4_CRTC_CONTROL - mmCRTC_CONTROL),
+		.dcp = (mmDCP4_GRPH_CONTROL - mmGRPH_CONTROL),
+	},
+	{
+		.crtc = (mmCRTC5_CRTC_CONTROL - mmCRTC_CONTROL),
+		.dcp = (mmDCP5_GRPH_CONTROL - mmGRPH_CONTROL),
+	}
+};
+
+/* set register offset */
+#define SR(reg_name)\
+	.reg_name = mm ## reg_name
+
+/* set register offset with instance */
+#define SRI(reg_name, block, id)\
+	.reg_name = mm ## block ## id ## _ ## reg_name
+
+#define ipp_regs(id)\
+[id] = {\
+		IPP_DCE100_REG_LIST_DCE_BASE(id)\
+}
+
+static const struct dce_ipp_registers ipp_regs[] = {
+		ipp_regs(0),
+		ipp_regs(1),
+		ipp_regs(2),
+		ipp_regs(3),
+		ipp_regs(4),
+		ipp_regs(5)
+};
+
+static const struct dce_ipp_shift ipp_shift = {
+		IPP_DCE100_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
+};
+
+static const struct dce_ipp_mask ipp_mask = {
+		IPP_DCE100_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
+};
+
+#define transform_regs(id)\
+[id] = {\
+		XFM_COMMON_REG_LIST_DCE100(id)\
+}
+
+static const struct dce_transform_registers xfm_regs[] = {
+		transform_regs(0),
+		transform_regs(1),
+		transform_regs(2),
+		transform_regs(3),
+		transform_regs(4),
+		transform_regs(5)
+};
+
+static const struct dce_transform_shift xfm_shift = {
+		XFM_COMMON_MASK_SH_LIST_DCE110(__SHIFT)
+};
+
+static const struct dce_transform_mask xfm_mask = {
+		XFM_COMMON_MASK_SH_LIST_DCE110(_MASK)
+};
+
+#define aux_regs(id)\
+[id] = {\
+	AUX_REG_LIST(id)\
+}
+
+static const struct dce110_link_enc_aux_registers link_enc_aux_regs[] = {
+		aux_regs(0),
+		aux_regs(1),
+		aux_regs(2),
+		aux_regs(3),
+		aux_regs(4),
+		aux_regs(5)
+};
+
+#define hpd_regs(id)\
+[id] = {\
+	HPD_REG_LIST(id)\
+}
+
+static const struct dce110_link_enc_hpd_registers link_enc_hpd_regs[] = {
+		hpd_regs(0),
+		hpd_regs(1),
+		hpd_regs(2),
+		hpd_regs(3),
+		hpd_regs(4),
+		hpd_regs(5)
+};
+
+#define link_regs(id)\
+[id] = {\
+	LE_DCE100_REG_LIST(id)\
+}
+
+static const struct dce110_link_enc_registers link_enc_regs[] = {
+	link_regs(0),
+	link_regs(1),
+	link_regs(2),
+	link_regs(3),
+	link_regs(4),
+	link_regs(5),
+	link_regs(6),
+};
+
+#define stream_enc_regs(id)\
+[id] = {\
+	SE_COMMON_REG_LIST_DCE_BASE(id),\
+	.AFMT_CNTL = 0,\
+}
+
+static const struct dce110_stream_enc_registers stream_enc_regs[] = {
+	stream_enc_regs(0),
+	stream_enc_regs(1),
+	stream_enc_regs(2),
+	stream_enc_regs(3),
+	stream_enc_regs(4),
+	stream_enc_regs(5),
+	stream_enc_regs(6)
+};
+
+static const struct dce_stream_encoder_shift se_shift = {
+		SE_COMMON_MASK_SH_LIST_DCE80_100(__SHIFT)
+};
+
+static const struct dce_stream_encoder_mask se_mask = {
+		SE_COMMON_MASK_SH_LIST_DCE80_100(_MASK)
+};
+
+static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
+	{ DCE_PANEL_CNTL_REG_LIST() }
+};
+
+static const struct dce_panel_cntl_shift panel_cntl_shift = {
+	DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_panel_cntl_mask panel_cntl_mask = {
+	DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
+};
+
+#define opp_regs(id)\
+[id] = {\
+	OPP_DCE_100_REG_LIST(id),\
+}
+
+static const struct dce_opp_registers opp_regs[] = {
+	opp_regs(0),
+	opp_regs(1),
+	opp_regs(2),
+	opp_regs(3),
+	opp_regs(4),
+	opp_regs(5)
+};
+
+static const struct dce_opp_shift opp_shift = {
+	OPP_COMMON_MASK_SH_LIST_DCE_100(__SHIFT)
+};
+
+static const struct dce_opp_mask opp_mask = {
+	OPP_COMMON_MASK_SH_LIST_DCE_100(_MASK)
+};
+#define aux_engine_regs(id)\
+[id] = {\
+	AUX_COMMON_REG_LIST(id), \
+	.AUX_RESET_MASK = 0 \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1),
+		aux_engine_regs(2),
+		aux_engine_regs(3),
+		aux_engine_regs(4),
+		aux_engine_regs(5)
+};
+
+#define audio_regs(id)\
+[id] = {\
+	AUD_COMMON_REG_LIST(id)\
+}
+
+static const struct dce_audio_registers audio_regs[] = {
+	audio_regs(0),
+	audio_regs(1),
+	audio_regs(2),
+	audio_regs(3),
+	audio_regs(4),
+	audio_regs(5),
+	audio_regs(6),
+};
+
+static const struct dce_audio_shift audio_shift = {
+		AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_audio_mask audio_mask = {
+		AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+#define clk_src_regs(id)\
+[id] = {\
+	CS_COMMON_REG_LIST_DCE_100_110(id),\
+}
+
+static const struct dce110_clk_src_regs clk_src_regs[] = {
+	clk_src_regs(0),
+	clk_src_regs(1),
+	clk_src_regs(2)
+};
+
+static const struct dce110_clk_src_shift cs_shift = {
+		CS_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
+};
+
+static const struct dce110_clk_src_mask cs_mask = {
+		CS_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
+};
+
+static const struct dce_dmcu_registers dmcu_regs = {
+		DMCU_DCE110_COMMON_REG_LIST()
+};
+
+static const struct dce_dmcu_shift dmcu_shift = {
+		DMCU_MASK_SH_LIST_DCE110(__SHIFT)
+};
+
+static const struct dce_dmcu_mask dmcu_mask = {
+		DMCU_MASK_SH_LIST_DCE110(_MASK)
+};
+
+static const struct dce_abm_registers abm_regs = {
+		ABM_DCE110_COMMON_REG_LIST()
+};
+
+static const struct dce_abm_shift abm_shift = {
+		ABM_MASK_SH_LIST_DCE110(__SHIFT)
+};
+
+static const struct dce_abm_mask abm_mask = {
+		ABM_MASK_SH_LIST_DCE110(_MASK)
+};
+
+#define DCFE_MEM_PWR_CTRL_REG_BASE 0x1b03
+
+static const struct bios_registers bios_regs = {
+	.BIOS_SCRATCH_3 = mmBIOS_SCRATCH_3,
+	.BIOS_SCRATCH_6 = mmBIOS_SCRATCH_6
+};
+
+static const struct resource_caps res_cap = {
+	.num_timing_generator = 6,
+	.num_audio = 6,
+	.num_stream_encoder = 6,
+	.num_pll = 3,
+	.num_ddc = 6,
+};
+
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCE_RGB,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = false,
+			.fp16 = true
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 1,
+			.fp16 = 1
+	},
+
+	.max_downscale_factor = {
+			.argb8888 = 250,
+			.nv12 = 1,
+			.fp16 = 1
+	}
+};
+
+static const struct dc_debug_options debug_defaults = {
+		.enable_legacy_fast_update = true,
+};
+
+#define CTX  ctx
+#define REG(reg) mm ## reg
+
+#ifndef mmCC_DC_HDMI_STRAPS
+#define mmCC_DC_HDMI_STRAPS 0x1918
+#define CC_DC_HDMI_STRAPS__HDMI_DISABLE_MASK 0x40
+#define CC_DC_HDMI_STRAPS__HDMI_DISABLE__SHIFT 0x6
+#define CC_DC_HDMI_STRAPS__AUDIO_STREAM_NUMBER_MASK 0x700
+#define CC_DC_HDMI_STRAPS__AUDIO_STREAM_NUMBER__SHIFT 0x8
+#endif
+
+static int map_transmitter_id_to_phy_instance(
+	enum transmitter transmitter)
+{
+	switch (transmitter) {
+	case TRANSMITTER_UNIPHY_A:
+		return 0;
+	case TRANSMITTER_UNIPHY_B:
+		return 1;
+	case TRANSMITTER_UNIPHY_C:
+		return 2;
+	case TRANSMITTER_UNIPHY_D:
+		return 3;
+	case TRANSMITTER_UNIPHY_E:
+		return 4;
+	case TRANSMITTER_UNIPHY_F:
+		return 5;
+	case TRANSMITTER_UNIPHY_G:
+		return 6;
+	default:
+		ASSERT(0);
+		return 0;
+	}
+}
+
+static void read_dce_straps(
+	struct dc_context *ctx,
+	struct resource_straps *straps)
+{
+	REG_GET_2(CC_DC_HDMI_STRAPS,
+			HDMI_DISABLE, &straps->hdmi_disable,
+			AUDIO_STREAM_NUMBER, &straps->audio_stream_number);
+
+	REG_GET(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO, &straps->dc_pinstraps_audio);
+}
+
+static struct audio *create_audio(
+		struct dc_context *ctx, unsigned int inst)
+{
+	return dce_audio_create(ctx, inst,
+			&audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+static struct timing_generator *dce100_timing_generator_create(
+		struct dc_context *ctx,
+		uint32_t instance,
+		const struct dce110_timing_generator_offsets *offsets)
+{
+	struct dce110_timing_generator *tg110 =
+		kzalloc(sizeof(struct dce110_timing_generator), GFP_KERNEL);
+
+	if (!tg110)
+		return NULL;
+
+	dce110_timing_generator_construct(tg110, ctx, instance, offsets);
+	return &tg110->base;
+}
+
+static struct stream_encoder *dce100_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dce110_stream_encoder *enc110 =
+		kzalloc(sizeof(struct dce110_stream_encoder), GFP_KERNEL);
+
+	if (!enc110)
+		return NULL;
+
+	dce110_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id,
+					&stream_enc_regs[eng_id], &se_shift, &se_mask);
+	return &enc110->base;
+}
+
+#define SRII(reg_name, block, id)\
+	.reg_name[id] = mm ## block ## id ## _ ## reg_name
+
+static const struct dce_hwseq_registers hwseq_reg = {
+		HWSEQ_DCE10_REG_LIST()
+};
+
+static const struct dce_hwseq_shift hwseq_shift = {
+		HWSEQ_DCE10_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+		HWSEQ_DCE10_MASK_SH_LIST(_MASK)
+};
+
+static struct dce_hwseq *dce100_hwseq_create(
+	struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &hwseq_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+	}
+	return hws;
+}
+
+static const struct resource_create_funcs res_create_funcs = {
+	.read_dce_straps = read_dce_straps,
+	.create_audio = create_audio,
+	.create_stream_encoder = dce100_stream_encoder_create,
+	.create_hwseq = dce100_hwseq_create,
+};
+
+#define mi_inst_regs(id) { \
+	MI_DCE8_REG_LIST(id), \
+	.MC_HUB_RDREQ_DMIF_LIMIT = mmMC_HUB_RDREQ_DMIF_LIMIT \
+}
+static const struct dce_mem_input_registers mi_regs[] = {
+		mi_inst_regs(0),
+		mi_inst_regs(1),
+		mi_inst_regs(2),
+		mi_inst_regs(3),
+		mi_inst_regs(4),
+		mi_inst_regs(5),
+};
+
+static const struct dce_mem_input_shift mi_shifts = {
+		MI_DCE8_MASK_SH_LIST(__SHIFT),
+		.ENABLE = MC_HUB_RDREQ_DMIF_LIMIT__ENABLE__SHIFT
+};
+
+static const struct dce_mem_input_mask mi_masks = {
+		MI_DCE8_MASK_SH_LIST(_MASK),
+		.ENABLE = MC_HUB_RDREQ_DMIF_LIMIT__ENABLE_MASK
+};
+
+static const struct dce110_aux_registers_shift aux_shift = {
+	DCE10_AUX_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+	DCE10_AUX_MASK_SH_LIST(_MASK)
+};
+
+static struct mem_input *dce100_mem_input_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_mem_input *dce_mi = kzalloc(sizeof(struct dce_mem_input),
+					       GFP_KERNEL);
+
+	if (!dce_mi) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dce_mem_input_construct(dce_mi, ctx, inst, &mi_regs[inst], &mi_shifts, &mi_masks);
+	dce_mi->wa.single_head_rdreq_dmif_limit = 2;
+	return &dce_mi->base;
+}
+
+static void dce100_transform_destroy(struct transform **xfm)
+{
+	kfree(TO_DCE_TRANSFORM(*xfm));
+	*xfm = NULL;
+}
+
+static struct transform *dce100_transform_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_transform *transform =
+		kzalloc(sizeof(struct dce_transform), GFP_KERNEL);
+
+	if (!transform)
+		return NULL;
+
+	dce_transform_construct(transform, ctx, inst,
+				&xfm_regs[inst], &xfm_shift, &xfm_mask);
+	return &transform->base;
+}
+
+static struct input_pixel_processor *dce100_ipp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dce_ipp *ipp = kzalloc(sizeof(struct dce_ipp), GFP_KERNEL);
+
+	if (!ipp) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dce_ipp_construct(ipp, ctx, inst,
+			&ipp_regs[inst], &ipp_shift, &ipp_mask);
+	return &ipp->base;
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+		.max_hdmi_deep_color = COLOR_DEPTH_121212,
+		.max_hdmi_pixel_clock = 300000,
+		.flags.bits.IS_HBR2_CAPABLE = true,
+		.flags.bits.IS_TPS3_CAPABLE = true
+};
+
+static struct link_encoder *dce100_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dce110_link_encoder *enc110 =
+		kzalloc(sizeof(struct dce110_link_encoder), GFP_KERNEL);
+	int link_regs_id;
+
+	if (!enc110)
+		return NULL;
+
+	link_regs_id =
+		map_transmitter_id_to_phy_instance(enc_init_data->transmitter);
+
+	dce110_link_encoder_construct(enc110,
+				      enc_init_data,
+				      &link_enc_feature,
+				      &link_enc_regs[link_regs_id],
+				      &link_enc_aux_regs[enc_init_data->channel - 1],
+				      &link_enc_hpd_regs[enc_init_data->hpd_source]);
+	return &enc110->base;
+}
+
+static struct panel_cntl *dce100_panel_cntl_create(const struct panel_cntl_init_data *init_data)
+{
+	struct dce_panel_cntl *panel_cntl =
+		kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
+
+	if (!panel_cntl)
+		return NULL;
+
+	dce_panel_cntl_construct(panel_cntl,
+			init_data,
+			&panel_cntl_regs[init_data->inst],
+			&panel_cntl_shift,
+			&panel_cntl_mask);
+
+	return &panel_cntl->base;
+}
+
+static struct output_pixel_processor *dce100_opp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce110_opp *opp =
+		kzalloc(sizeof(struct dce110_opp), GFP_KERNEL);
+
+	if (!opp)
+		return NULL;
+
+	dce110_opp_construct(opp,
+			     ctx, inst, &opp_regs[inst], &opp_shift, &opp_mask);
+	return &opp->base;
+}
+
+static struct dce_aux *dce100_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst],
+					&aux_mask,
+					&aux_shift,
+					ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
+
+static const struct dce_i2c_registers i2c_hw_regs[] = {
+		i2c_inst_regs(1),
+		i2c_inst_regs(2),
+		i2c_inst_regs(3),
+		i2c_inst_regs(4),
+		i2c_inst_regs(5),
+		i2c_inst_regs(6),
+};
+
+static const struct dce_i2c_shift i2c_shifts = {
+		I2C_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
+};
+
+static const struct dce_i2c_mask i2c_masks = {
+		I2C_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
+};
+
+static struct dce_i2c_hw *dce100_i2c_hw_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw =
+		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+	dce100_i2c_hw_construct(dce_i2c_hw, ctx, inst,
+				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+static struct clock_source *dce100_clock_source_create(
+	struct dc_context *ctx,
+	struct dc_bios *bios,
+	enum clock_source_id id,
+	const struct dce110_clk_src_regs *regs,
+	bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dce110_clk_src_construct(clk_src, ctx, bios, id,
+			regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	kfree(clk_src);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static void dce100_clock_source_destroy(struct clock_source **clk_src)
+{
+	kfree(TO_DCE110_CLK_SRC(*clk_src));
+	*clk_src = NULL;
+}
+
+static void dce100_resource_destruct(struct dce110_resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (pool->base.opps[i] != NULL)
+			dce110_opp_destroy(&pool->base.opps[i]);
+
+		if (pool->base.transforms[i] != NULL)
+			dce100_transform_destroy(&pool->base.transforms[i]);
+
+		if (pool->base.ipps[i] != NULL)
+			dce_ipp_destroy(&pool->base.ipps[i]);
+
+		if (pool->base.mis[i] != NULL) {
+			kfree(TO_DCE_MEM_INPUT(pool->base.mis[i]));
+			pool->base.mis[i] = NULL;
+		}
+
+		if (pool->base.timing_generators[i] != NULL)	{
+			kfree(DCE110TG_FROM_TG(pool->base.timing_generators[i]));
+			pool->base.timing_generators[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+		if (pool->base.hw_i2cs[i] != NULL) {
+			kfree(pool->base.hw_i2cs[i]);
+			pool->base.hw_i2cs[i] = NULL;
+		}
+		if (pool->base.sw_i2cs[i] != NULL) {
+			kfree(pool->base.sw_i2cs[i]);
+			pool->base.sw_i2cs[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.stream_enc_count; i++) {
+		if (pool->base.stream_enc[i] != NULL)
+			kfree(DCE110STRENC_FROM_STRENC(pool->base.stream_enc[i]));
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] != NULL)
+			dce100_clock_source_destroy(&pool->base.clock_sources[i]);
+	}
+
+	if (pool->base.dp_clock_source != NULL)
+		dce100_clock_source_destroy(&pool->base.dp_clock_source);
+
+	for (i = 0; i < pool->base.audio_count; i++)	{
+		if (pool->base.audios[i] != NULL)
+			dce_aud_destroy(&pool->base.audios[i]);
+	}
+
+	if (pool->base.abm != NULL)
+				dce_abm_destroy(&pool->base.abm);
+
+	if (pool->base.dmcu != NULL)
+			dce_dmcu_destroy(&pool->base.dmcu);
+
+	if (pool->base.irqs != NULL)
+		dal_irq_service_destroy(&pool->base.irqs);
+}
+
+static enum dc_status build_mapped_resource(
+		const struct dc  *dc,
+		struct dc_state *context,
+		struct dc_stream_state *stream)
+{
+	struct pipe_ctx *pipe_ctx = resource_get_otg_master_for_stream(&context->res_ctx, stream);
+
+	if (!pipe_ctx)
+		return DC_ERROR_UNEXPECTED;
+
+	dce110_resource_build_pipe_hw_param(pipe_ctx);
+
+	resource_build_info_frame(pipe_ctx);
+
+	return DC_OK;
+}
+
+static bool dce100_validate_bandwidth(
+	struct dc  *dc,
+	struct dc_state *context,
+	bool fast_validate)
+{
+	int i;
+	bool at_least_one_pipe = false;
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		if (context->res_ctx.pipe_ctx[i].stream)
+			at_least_one_pipe = true;
+	}
+
+	if (at_least_one_pipe) {
+		/* TODO implement when needed but for now hardcode max value*/
+		context->bw_ctx.bw.dce.dispclk_khz = 681000;
+		context->bw_ctx.bw.dce.yclk_khz = 250000 * MEMORY_TYPE_MULTIPLIER_CZ;
+	} else {
+		context->bw_ctx.bw.dce.dispclk_khz = 0;
+		context->bw_ctx.bw.dce.yclk_khz = 0;
+	}
+
+	return true;
+}
+
+static bool dce100_validate_surface_sets(
+		struct dc_state *context)
+{
+	int i;
+
+	for (i = 0; i < context->stream_count; i++) {
+		if (context->stream_status[i].plane_count == 0)
+			continue;
+
+		if (context->stream_status[i].plane_count > 1)
+			return false;
+
+		if (context->stream_status[i].plane_states[0]->format
+				>= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN)
+			return false;
+	}
+
+	return true;
+}
+
+static enum dc_status dce100_validate_global(
+		struct dc  *dc,
+		struct dc_state *context)
+{
+	if (!dce100_validate_surface_sets(context))
+		return DC_FAIL_SURFACE_VALIDATE;
+
+	return DC_OK;
+}
+
+enum dc_status dce100_add_stream_to_ctx(
+		struct dc *dc,
+		struct dc_state *new_ctx,
+		struct dc_stream_state *dc_stream)
+{
+	enum dc_status result = DC_ERROR_UNEXPECTED;
+
+	result = resource_map_pool_resources(dc, new_ctx, dc_stream);
+
+	if (result == DC_OK)
+		result = resource_map_clock_resources(dc, new_ctx, dc_stream);
+
+	if (result == DC_OK)
+		result = build_mapped_resource(dc, new_ctx, dc_stream);
+
+	return result;
+}
+
+static void dce100_destroy_resource_pool(struct resource_pool **pool)
+{
+	struct dce110_resource_pool *dce110_pool = TO_DCE110_RES_POOL(*pool);
+
+	dce100_resource_destruct(dce110_pool);
+	kfree(dce110_pool);
+	*pool = NULL;
+}
+
+enum dc_status dce100_validate_plane(const struct dc_plane_state *plane_state, struct dc_caps *caps)
+{
+
+	if (plane_state->format < SURFACE_PIXEL_FORMAT_VIDEO_BEGIN)
+		return DC_OK;
+
+	return DC_FAIL_SURFACE_VALIDATE;
+}
+
+struct stream_encoder *dce100_find_first_free_match_stream_enc_for_link(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		struct dc_stream_state *stream)
+{
+	int i;
+	int j = -1;
+	struct dc_link *link = stream->link;
+
+	for (i = 0; i < pool->stream_enc_count; i++) {
+		if (!res_ctx->is_stream_enc_acquired[i] &&
+				pool->stream_enc[i]) {
+			/* Store first available for MST second display
+			 * in daisy chain use case
+			 */
+			j = i;
+			if (pool->stream_enc[i]->id ==
+					link->link_enc->preferred_engine)
+				return pool->stream_enc[i];
+		}
+	}
+
+	/*
+	 * below can happen in cases when stream encoder is acquired:
+	 * 1) for second MST display in chain, so preferred engine already
+	 * acquired;
+	 * 2) for another link, which preferred engine already acquired by any
+	 * MST configuration.
+	 *
+	 * If signal is of DP type and preferred engine not found, return last available
+	 *
+	 * TODO - This is just a patch up and a generic solution is
+	 * required for non DP connectors.
+	 */
+
+	if (j >= 0 && link->connector_signal == SIGNAL_TYPE_DISPLAY_PORT)
+		return pool->stream_enc[j];
+
+	return NULL;
+}
+
+static const struct resource_funcs dce100_res_pool_funcs = {
+	.destroy = dce100_destroy_resource_pool,
+	.link_enc_create = dce100_link_encoder_create,
+	.panel_cntl_create = dce100_panel_cntl_create,
+	.validate_bandwidth = dce100_validate_bandwidth,
+	.validate_plane = dce100_validate_plane,
+	.add_stream_to_ctx = dce100_add_stream_to_ctx,
+	.validate_global = dce100_validate_global,
+	.find_first_free_match_stream_enc_for_link = dce100_find_first_free_match_stream_enc_for_link
+};
+
+static bool dce100_resource_construct(
+	uint8_t num_virtual_links,
+	struct dc  *dc,
+	struct dce110_resource_pool *pool)
+{
+	unsigned int i;
+	struct dc_context *ctx = dc->ctx;
+	struct dc_bios *bp;
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	pool->base.res_cap = &res_cap;
+	pool->base.funcs = &dce100_res_pool_funcs;
+	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+
+	bp = ctx->dc_bios;
+
+	if (bp->fw_info_valid && bp->fw_info.external_clock_source_frequency_for_dp != 0) {
+		pool->base.dp_clock_source =
+				dce100_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_EXTERNAL, NULL, true);
+
+		pool->base.clock_sources[0] =
+				dce100_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0, &clk_src_regs[0], false);
+		pool->base.clock_sources[1] =
+				dce100_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false);
+		pool->base.clock_sources[2] =
+				dce100_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[2], false);
+		pool->base.clk_src_count = 3;
+
+	} else {
+		pool->base.dp_clock_source =
+				dce100_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0, &clk_src_regs[0], true);
+
+		pool->base.clock_sources[0] =
+				dce100_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false);
+		pool->base.clock_sources[1] =
+				dce100_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[2], false);
+		pool->base.clk_src_count = 2;
+	}
+
+	if (pool->base.dp_clock_source == NULL) {
+		dm_error("DC: failed to create dp clock source!\n");
+		BREAK_TO_DEBUGGER();
+		goto res_create_fail;
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto res_create_fail;
+		}
+	}
+
+	pool->base.dmcu = dce_dmcu_create(ctx,
+			&dmcu_regs,
+			&dmcu_shift,
+			&dmcu_mask);
+	if (pool->base.dmcu == NULL) {
+		dm_error("DC: failed to create dmcu!\n");
+		BREAK_TO_DEBUGGER();
+		goto res_create_fail;
+	}
+
+	pool->base.abm = dce_abm_create(ctx,
+				&abm_regs,
+				&abm_shift,
+				&abm_mask);
+	if (pool->base.abm == NULL) {
+		dm_error("DC: failed to create abm!\n");
+		BREAK_TO_DEBUGGER();
+		goto res_create_fail;
+	}
+
+	{
+		struct irq_service_init_data init_data;
+		init_data.ctx = dc->ctx;
+		pool->base.irqs = dal_irq_service_dce110_create(&init_data);
+		if (!pool->base.irqs)
+			goto res_create_fail;
+	}
+
+	/*************************************************
+	*  Resource + asic cap harcoding                *
+	*************************************************/
+	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+	pool->base.pipe_count = res_cap.num_timing_generator;
+	pool->base.timing_generator_count = pool->base.res_cap->num_timing_generator;
+	dc->caps.max_downscale_ratio = 200;
+	dc->caps.i2c_speed_in_khz = 40;
+	dc->caps.i2c_speed_in_khz = 40;
+	dc->caps.max_cursor_size = 128;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.dual_link_dvi = true;
+	dc->caps.disable_dp_clk_share = true;
+	dc->caps.extended_aux_timeout_support = false;
+	dc->debug = debug_defaults;
+
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		pool->base.timing_generators[i] =
+			dce100_timing_generator_create(
+				ctx,
+				i,
+				&dce100_tg_offsets[i]);
+		if (pool->base.timing_generators[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.mis[i] = dce100_mem_input_create(ctx, i);
+		if (pool->base.mis[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create memory input!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.ipps[i] = dce100_ipp_create(ctx, i);
+		if (pool->base.ipps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create input pixel processor!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.transforms[i] = dce100_transform_create(ctx, i);
+		if (pool->base.transforms[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create transform!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.opps[i] = dce100_opp_create(ctx, i);
+		if (pool->base.opps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create output pixel processor!\n");
+			goto res_create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.engines[i] = dce100_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto res_create_fail;
+		}
+		pool->base.hw_i2cs[i] = dce100_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create i2c engine!!\n");
+			goto res_create_fail;
+		}
+		pool->base.sw_i2cs[i] = NULL;
+	}
+
+	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	if (!resource_construct(num_virtual_links, dc, &pool->base,
+			&res_create_funcs))
+		goto res_create_fail;
+
+	/* Create hardware sequencer */
+	dce100_hw_sequencer_construct(dc);
+	return true;
+
+res_create_fail:
+	dce100_resource_destruct(pool);
+
+	return false;
+}
+
+struct resource_pool *dce100_create_resource_pool(
+	uint8_t num_virtual_links,
+	struct dc  *dc)
+{
+	struct dce110_resource_pool *pool =
+		kzalloc(sizeof(struct dce110_resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dce100_resource_construct(num_virtual_links, dc, pool))
+		return &pool->base;
+
+	kfree(pool);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.h
new file mode 100644
index 000000000000..fecab7c560f5
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.h
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ *
+ */
+/*
+ * dce100_resource.h
+ *
+ *  Created on: 2016-01-20
+ *      Author: qyang
+ */
+
+#ifndef DCE100_RESOURCE_H_
+#define DCE100_RESOURCE_H_
+
+struct dc;
+struct resource_pool;
+struct dc_validation_set;
+
+struct resource_pool *dce100_create_resource_pool(
+	uint8_t num_virtual_links,
+	struct dc *dc);
+
+enum dc_status dce100_validate_plane(const struct dc_plane_state *plane_state, struct dc_caps *caps);
+
+enum dc_status dce100_add_stream_to_ctx(
+		struct dc *dc,
+		struct dc_state *new_ctx,
+		struct dc_stream_state *dc_stream);
+
+struct stream_encoder *dce100_find_first_free_match_stream_enc_for_link(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		struct dc_stream_state *stream);
+
+#endif /* DCE100_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.c
new file mode 100644
index 000000000000..fe518fd27b08
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.c
@@ -0,0 +1,1551 @@
+/*
+ * Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dm_services.h"
+
+#include "link_encoder.h"
+#include "stream_encoder.h"
+
+#include "resource.h"
+#include "dce110/dce110_resource.h"
+#include "include/irq_service_interface.h"
+#include "dce/dce_audio.h"
+#include "dce110/dce110_timing_generator.h"
+#include "irq/dce110/irq_service_dce110.h"
+#include "dce110/dce110_timing_generator_v.h"
+#include "dce/dce_link_encoder.h"
+#include "dce/dce_stream_encoder.h"
+#include "dce/dce_mem_input.h"
+#include "dce110/dce110_mem_input_v.h"
+#include "dce/dce_ipp.h"
+#include "dce/dce_transform.h"
+#include "dce110/dce110_transform_v.h"
+#include "dce/dce_opp.h"
+#include "dce110/dce110_opp_v.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_hwseq.h"
+#include "dce110/dce110_hwseq.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_abm.h"
+#include "dce/dce_dmcu.h"
+#include "dce/dce_i2c.h"
+#include "dce/dce_panel_cntl.h"
+
+#define DC_LOGGER \
+		dc->ctx->logger
+
+#include "dce110/dce110_compressor.h"
+
+#include "reg_helper.h"
+
+#include "dce/dce_11_0_d.h"
+#include "dce/dce_11_0_sh_mask.h"
+
+#ifndef mmMC_HUB_RDREQ_DMIF_LIMIT
+#include "gmc/gmc_8_2_d.h"
+#include "gmc/gmc_8_2_sh_mask.h"
+#endif
+
+#ifndef mmDP_DPHY_INTERNAL_CTRL
+	#define mmDP_DPHY_INTERNAL_CTRL 0x4aa7
+	#define mmDP0_DP_DPHY_INTERNAL_CTRL 0x4aa7
+	#define mmDP1_DP_DPHY_INTERNAL_CTRL 0x4ba7
+	#define mmDP2_DP_DPHY_INTERNAL_CTRL 0x4ca7
+	#define mmDP3_DP_DPHY_INTERNAL_CTRL 0x4da7
+	#define mmDP4_DP_DPHY_INTERNAL_CTRL 0x4ea7
+	#define mmDP5_DP_DPHY_INTERNAL_CTRL 0x4fa7
+	#define mmDP6_DP_DPHY_INTERNAL_CTRL 0x54a7
+	#define mmDP7_DP_DPHY_INTERNAL_CTRL 0x56a7
+	#define mmDP8_DP_DPHY_INTERNAL_CTRL 0x57a7
+#endif
+
+#ifndef mmBIOS_SCRATCH_2
+	#define mmBIOS_SCRATCH_2 0x05CB
+	#define mmBIOS_SCRATCH_3 0x05CC
+	#define mmBIOS_SCRATCH_6 0x05CF
+#endif
+
+#ifndef mmDP_DPHY_BS_SR_SWAP_CNTL
+	#define mmDP_DPHY_BS_SR_SWAP_CNTL                       0x4ADC
+	#define mmDP0_DP_DPHY_BS_SR_SWAP_CNTL                   0x4ADC
+	#define mmDP1_DP_DPHY_BS_SR_SWAP_CNTL                   0x4BDC
+	#define mmDP2_DP_DPHY_BS_SR_SWAP_CNTL                   0x4CDC
+	#define mmDP3_DP_DPHY_BS_SR_SWAP_CNTL                   0x4DDC
+	#define mmDP4_DP_DPHY_BS_SR_SWAP_CNTL                   0x4EDC
+	#define mmDP5_DP_DPHY_BS_SR_SWAP_CNTL                   0x4FDC
+	#define mmDP6_DP_DPHY_BS_SR_SWAP_CNTL                   0x54DC
+#endif
+
+#ifndef mmDP_DPHY_FAST_TRAINING
+	#define mmDP_DPHY_FAST_TRAINING                         0x4ABC
+	#define mmDP0_DP_DPHY_FAST_TRAINING                     0x4ABC
+	#define mmDP1_DP_DPHY_FAST_TRAINING                     0x4BBC
+	#define mmDP2_DP_DPHY_FAST_TRAINING                     0x4CBC
+	#define mmDP3_DP_DPHY_FAST_TRAINING                     0x4DBC
+	#define mmDP4_DP_DPHY_FAST_TRAINING                     0x4EBC
+	#define mmDP5_DP_DPHY_FAST_TRAINING                     0x4FBC
+	#define mmDP6_DP_DPHY_FAST_TRAINING                     0x54BC
+#endif
+
+#ifndef DPHY_RX_FAST_TRAINING_CAPABLE
+	#define DPHY_RX_FAST_TRAINING_CAPABLE 0x1
+#endif
+
+static const struct dce110_timing_generator_offsets dce110_tg_offsets[] = {
+	{
+		.crtc = (mmCRTC0_CRTC_CONTROL - mmCRTC_CONTROL),
+		.dcp =  (mmDCP0_GRPH_CONTROL - mmGRPH_CONTROL),
+	},
+	{
+		.crtc = (mmCRTC1_CRTC_CONTROL - mmCRTC_CONTROL),
+		.dcp = (mmDCP1_GRPH_CONTROL - mmGRPH_CONTROL),
+	},
+	{
+		.crtc = (mmCRTC2_CRTC_CONTROL - mmCRTC_CONTROL),
+		.dcp = (mmDCP2_GRPH_CONTROL - mmGRPH_CONTROL),
+	},
+	{
+		.crtc = (mmCRTC3_CRTC_CONTROL - mmCRTC_CONTROL),
+		.dcp =  (mmDCP3_GRPH_CONTROL - mmGRPH_CONTROL),
+	},
+	{
+		.crtc = (mmCRTC4_CRTC_CONTROL - mmCRTC_CONTROL),
+		.dcp = (mmDCP4_GRPH_CONTROL - mmGRPH_CONTROL),
+	},
+	{
+		.crtc = (mmCRTC5_CRTC_CONTROL - mmCRTC_CONTROL),
+		.dcp = (mmDCP5_GRPH_CONTROL - mmGRPH_CONTROL),
+	}
+};
+
+/* set register offset */
+#define SR(reg_name)\
+	.reg_name = mm ## reg_name
+
+/* set register offset with instance */
+#define SRI(reg_name, block, id)\
+	.reg_name = mm ## block ## id ## _ ## reg_name
+
+static const struct dce_dmcu_registers dmcu_regs = {
+		DMCU_DCE110_COMMON_REG_LIST()
+};
+
+static const struct dce_dmcu_shift dmcu_shift = {
+		DMCU_MASK_SH_LIST_DCE110(__SHIFT)
+};
+
+static const struct dce_dmcu_mask dmcu_mask = {
+		DMCU_MASK_SH_LIST_DCE110(_MASK)
+};
+
+static const struct dce_abm_registers abm_regs = {
+		ABM_DCE110_COMMON_REG_LIST()
+};
+
+static const struct dce_abm_shift abm_shift = {
+		ABM_MASK_SH_LIST_DCE110(__SHIFT)
+};
+
+static const struct dce_abm_mask abm_mask = {
+		ABM_MASK_SH_LIST_DCE110(_MASK)
+};
+
+#define ipp_regs(id)\
+[id] = {\
+		IPP_DCE110_REG_LIST_DCE_BASE(id)\
+}
+
+static const struct dce_ipp_registers ipp_regs[] = {
+		ipp_regs(0),
+		ipp_regs(1),
+		ipp_regs(2)
+};
+
+static const struct dce_ipp_shift ipp_shift = {
+		IPP_DCE100_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
+};
+
+static const struct dce_ipp_mask ipp_mask = {
+		IPP_DCE100_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
+};
+
+#define transform_regs(id)\
+[id] = {\
+		XFM_COMMON_REG_LIST_DCE110(id)\
+}
+
+static const struct dce_transform_registers xfm_regs[] = {
+		transform_regs(0),
+		transform_regs(1),
+		transform_regs(2)
+};
+
+static const struct dce_transform_shift xfm_shift = {
+		XFM_COMMON_MASK_SH_LIST_DCE110(__SHIFT)
+};
+
+static const struct dce_transform_mask xfm_mask = {
+		XFM_COMMON_MASK_SH_LIST_DCE110(_MASK)
+};
+
+#define aux_regs(id)\
+[id] = {\
+	AUX_REG_LIST(id)\
+}
+
+static const struct dce110_link_enc_aux_registers link_enc_aux_regs[] = {
+		aux_regs(0),
+		aux_regs(1),
+		aux_regs(2),
+		aux_regs(3),
+		aux_regs(4),
+		aux_regs(5)
+};
+
+#define hpd_regs(id)\
+[id] = {\
+	HPD_REG_LIST(id)\
+}
+
+static const struct dce110_link_enc_hpd_registers link_enc_hpd_regs[] = {
+		hpd_regs(0),
+		hpd_regs(1),
+		hpd_regs(2),
+		hpd_regs(3),
+		hpd_regs(4),
+		hpd_regs(5)
+};
+
+
+#define link_regs(id)\
+[id] = {\
+	LE_DCE110_REG_LIST(id)\
+}
+
+static const struct dce110_link_enc_registers link_enc_regs[] = {
+	link_regs(0),
+	link_regs(1),
+	link_regs(2),
+	link_regs(3),
+	link_regs(4),
+	link_regs(5),
+	link_regs(6),
+};
+
+#define stream_enc_regs(id)\
+[id] = {\
+	SE_COMMON_REG_LIST(id),\
+	.TMDS_CNTL = 0,\
+}
+
+static const struct dce110_stream_enc_registers stream_enc_regs[] = {
+	stream_enc_regs(0),
+	stream_enc_regs(1),
+	stream_enc_regs(2)
+};
+
+static const struct dce_stream_encoder_shift se_shift = {
+		SE_COMMON_MASK_SH_LIST_DCE110(__SHIFT)
+};
+
+static const struct dce_stream_encoder_mask se_mask = {
+		SE_COMMON_MASK_SH_LIST_DCE110(_MASK)
+};
+
+static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
+	{ DCE_PANEL_CNTL_REG_LIST() }
+};
+
+static const struct dce_panel_cntl_shift panel_cntl_shift = {
+	DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_panel_cntl_mask panel_cntl_mask = {
+	DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
+};
+
+static const struct dce110_aux_registers_shift aux_shift = {
+	DCE_AUX_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+	DCE_AUX_MASK_SH_LIST(_MASK)
+};
+
+#define opp_regs(id)\
+[id] = {\
+	OPP_DCE_110_REG_LIST(id),\
+}
+
+static const struct dce_opp_registers opp_regs[] = {
+	opp_regs(0),
+	opp_regs(1),
+	opp_regs(2),
+	opp_regs(3),
+	opp_regs(4),
+	opp_regs(5)
+};
+
+static const struct dce_opp_shift opp_shift = {
+	OPP_COMMON_MASK_SH_LIST_DCE_110(__SHIFT)
+};
+
+static const struct dce_opp_mask opp_mask = {
+	OPP_COMMON_MASK_SH_LIST_DCE_110(_MASK)
+};
+
+#define aux_engine_regs(id)\
+[id] = {\
+	AUX_COMMON_REG_LIST(id), \
+	.AUX_RESET_MASK = 0 \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1),
+		aux_engine_regs(2),
+		aux_engine_regs(3),
+		aux_engine_regs(4),
+		aux_engine_regs(5)
+};
+
+#define audio_regs(id)\
+[id] = {\
+	AUD_COMMON_REG_LIST(id)\
+}
+
+static const struct dce_audio_registers audio_regs[] = {
+	audio_regs(0),
+	audio_regs(1),
+	audio_regs(2),
+	audio_regs(3),
+	audio_regs(4),
+	audio_regs(5),
+	audio_regs(6),
+};
+
+static const struct dce_audio_shift audio_shift = {
+		AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_audio_mask audio_mask = {
+		AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+/* AG TBD Needs to be reduced back to 3 pipes once dce10 hw sequencer implemented. */
+
+
+#define clk_src_regs(id)\
+[id] = {\
+	CS_COMMON_REG_LIST_DCE_100_110(id),\
+}
+
+static const struct dce110_clk_src_regs clk_src_regs[] = {
+	clk_src_regs(0),
+	clk_src_regs(1),
+	clk_src_regs(2)
+};
+
+static const struct dce110_clk_src_shift cs_shift = {
+		CS_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
+};
+
+static const struct dce110_clk_src_mask cs_mask = {
+		CS_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
+};
+
+static const struct bios_registers bios_regs = {
+	.BIOS_SCRATCH_3 = mmBIOS_SCRATCH_3,
+	.BIOS_SCRATCH_6 = mmBIOS_SCRATCH_6
+};
+
+static const struct resource_caps carrizo_resource_cap = {
+		.num_timing_generator = 3,
+		.num_video_plane = 1,
+		.num_audio = 3,
+		.num_stream_encoder = 3,
+		.num_pll = 2,
+		.num_ddc = 3,
+};
+
+static const struct resource_caps stoney_resource_cap = {
+		.num_timing_generator = 2,
+		.num_video_plane = 1,
+		.num_audio = 3,
+		.num_stream_encoder = 3,
+		.num_pll = 2,
+		.num_ddc = 3,
+};
+
+static const struct dc_plane_cap plane_cap = {
+		.type = DC_PLANE_TYPE_DCE_RGB,
+		.per_pixel_alpha = 1,
+
+		.pixel_format_support = {
+				.argb8888 = true,
+				.nv12 = false,
+				.fp16 = true
+		},
+
+		.max_upscale_factor = {
+				.argb8888 = 16000,
+				.nv12 = 1,
+				.fp16 = 1
+		},
+
+		.max_downscale_factor = {
+				.argb8888 = 250,
+				.nv12 = 1,
+				.fp16 = 1
+		},
+		64,
+		64
+};
+
+static const struct dc_debug_options debug_defaults = {
+		.enable_legacy_fast_update = true,
+};
+
+static const struct dc_plane_cap underlay_plane_cap = {
+		.type = DC_PLANE_TYPE_DCE_UNDERLAY,
+		.per_pixel_alpha = 1,
+
+		.pixel_format_support = {
+				.argb8888 = false,
+				.nv12 = true,
+				.fp16 = false
+		},
+
+		.max_upscale_factor = {
+				.argb8888 = 1,
+				.nv12 = 16000,
+				.fp16 = 1
+		},
+
+		.max_downscale_factor = {
+				.argb8888 = 1,
+				.nv12 = 250,
+				.fp16 = 1
+		},
+		64,
+		64
+};
+
+#define CTX  ctx
+#define REG(reg) mm ## reg
+
+#ifndef mmCC_DC_HDMI_STRAPS
+#define mmCC_DC_HDMI_STRAPS 0x4819
+#define CC_DC_HDMI_STRAPS__HDMI_DISABLE_MASK 0x40
+#define CC_DC_HDMI_STRAPS__HDMI_DISABLE__SHIFT 0x6
+#define CC_DC_HDMI_STRAPS__AUDIO_STREAM_NUMBER_MASK 0x700
+#define CC_DC_HDMI_STRAPS__AUDIO_STREAM_NUMBER__SHIFT 0x8
+#endif
+
+static int map_transmitter_id_to_phy_instance(
+	enum transmitter transmitter)
+{
+	switch (transmitter) {
+	case TRANSMITTER_UNIPHY_A:
+		return 0;
+	case TRANSMITTER_UNIPHY_B:
+		return 1;
+	case TRANSMITTER_UNIPHY_C:
+		return 2;
+	case TRANSMITTER_UNIPHY_D:
+		return 3;
+	case TRANSMITTER_UNIPHY_E:
+		return 4;
+	case TRANSMITTER_UNIPHY_F:
+		return 5;
+	case TRANSMITTER_UNIPHY_G:
+		return 6;
+	default:
+		ASSERT(0);
+		return 0;
+	}
+}
+
+static void read_dce_straps(
+	struct dc_context *ctx,
+	struct resource_straps *straps)
+{
+	REG_GET_2(CC_DC_HDMI_STRAPS,
+			HDMI_DISABLE, &straps->hdmi_disable,
+			AUDIO_STREAM_NUMBER, &straps->audio_stream_number);
+
+	REG_GET(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO, &straps->dc_pinstraps_audio);
+}
+
+static struct audio *create_audio(
+		struct dc_context *ctx, unsigned int inst)
+{
+	return dce_audio_create(ctx, inst,
+			&audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+static struct timing_generator *dce110_timing_generator_create(
+		struct dc_context *ctx,
+		uint32_t instance,
+		const struct dce110_timing_generator_offsets *offsets)
+{
+	struct dce110_timing_generator *tg110 =
+		kzalloc(sizeof(struct dce110_timing_generator), GFP_KERNEL);
+
+	if (!tg110)
+		return NULL;
+
+	dce110_timing_generator_construct(tg110, ctx, instance, offsets);
+	return &tg110->base;
+}
+
+static struct stream_encoder *dce110_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dce110_stream_encoder *enc110 =
+		kzalloc(sizeof(struct dce110_stream_encoder), GFP_KERNEL);
+
+	if (!enc110)
+		return NULL;
+
+	dce110_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id,
+					&stream_enc_regs[eng_id],
+					&se_shift, &se_mask);
+	return &enc110->base;
+}
+
+#define SRII(reg_name, block, id)\
+	.reg_name[id] = mm ## block ## id ## _ ## reg_name
+
+static const struct dce_hwseq_registers hwseq_stoney_reg = {
+		HWSEQ_ST_REG_LIST()
+};
+
+static const struct dce_hwseq_registers hwseq_cz_reg = {
+		HWSEQ_CZ_REG_LIST()
+};
+
+static const struct dce_hwseq_shift hwseq_shift = {
+		HWSEQ_DCE11_MASK_SH_LIST(__SHIFT),
+};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+		HWSEQ_DCE11_MASK_SH_LIST(_MASK),
+};
+
+static struct dce_hwseq *dce110_hwseq_create(
+	struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = ASIC_REV_IS_STONEY(ctx->asic_id.hw_internal_rev) ?
+				&hwseq_stoney_reg : &hwseq_cz_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+		hws->wa.blnd_crtc_trigger = true;
+	}
+	return hws;
+}
+
+static const struct resource_create_funcs res_create_funcs = {
+	.read_dce_straps = read_dce_straps,
+	.create_audio = create_audio,
+	.create_stream_encoder = dce110_stream_encoder_create,
+	.create_hwseq = dce110_hwseq_create,
+};
+
+#define mi_inst_regs(id) { \
+	MI_DCE11_REG_LIST(id), \
+	.MC_HUB_RDREQ_DMIF_LIMIT = mmMC_HUB_RDREQ_DMIF_LIMIT \
+}
+static const struct dce_mem_input_registers mi_regs[] = {
+		mi_inst_regs(0),
+		mi_inst_regs(1),
+		mi_inst_regs(2),
+};
+
+static const struct dce_mem_input_shift mi_shifts = {
+		MI_DCE11_MASK_SH_LIST(__SHIFT),
+		.ENABLE = MC_HUB_RDREQ_DMIF_LIMIT__ENABLE__SHIFT
+};
+
+static const struct dce_mem_input_mask mi_masks = {
+		MI_DCE11_MASK_SH_LIST(_MASK),
+		.ENABLE = MC_HUB_RDREQ_DMIF_LIMIT__ENABLE_MASK
+};
+
+
+static struct mem_input *dce110_mem_input_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_mem_input *dce_mi = kzalloc(sizeof(struct dce_mem_input),
+					       GFP_KERNEL);
+
+	if (!dce_mi) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dce_mem_input_construct(dce_mi, ctx, inst, &mi_regs[inst], &mi_shifts, &mi_masks);
+	dce_mi->wa.single_head_rdreq_dmif_limit = 3;
+	return &dce_mi->base;
+}
+
+static void dce110_transform_destroy(struct transform **xfm)
+{
+	kfree(TO_DCE_TRANSFORM(*xfm));
+	*xfm = NULL;
+}
+
+static struct transform *dce110_transform_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_transform *transform =
+		kzalloc(sizeof(struct dce_transform), GFP_KERNEL);
+
+	if (!transform)
+		return NULL;
+
+	dce_transform_construct(transform, ctx, inst,
+				&xfm_regs[inst], &xfm_shift, &xfm_mask);
+	return &transform->base;
+}
+
+static struct input_pixel_processor *dce110_ipp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dce_ipp *ipp = kzalloc(sizeof(struct dce_ipp), GFP_KERNEL);
+
+	if (!ipp) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dce_ipp_construct(ipp, ctx, inst,
+			&ipp_regs[inst], &ipp_shift, &ipp_mask);
+	return &ipp->base;
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+		.max_hdmi_deep_color = COLOR_DEPTH_121212,
+		.max_hdmi_pixel_clock = 300000,
+		.flags.bits.IS_HBR2_CAPABLE = true,
+		.flags.bits.IS_TPS3_CAPABLE = true
+};
+
+static struct link_encoder *dce110_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dce110_link_encoder *enc110 =
+		kzalloc(sizeof(struct dce110_link_encoder), GFP_KERNEL);
+	int link_regs_id;
+
+	if (!enc110)
+		return NULL;
+
+	link_regs_id =
+		map_transmitter_id_to_phy_instance(enc_init_data->transmitter);
+
+	dce110_link_encoder_construct(enc110,
+				      enc_init_data,
+				      &link_enc_feature,
+				      &link_enc_regs[link_regs_id],
+				      &link_enc_aux_regs[enc_init_data->channel - 1],
+				      &link_enc_hpd_regs[enc_init_data->hpd_source]);
+	return &enc110->base;
+}
+
+static struct panel_cntl *dce110_panel_cntl_create(const struct panel_cntl_init_data *init_data)
+{
+	struct dce_panel_cntl *panel_cntl =
+		kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
+
+	if (!panel_cntl)
+		return NULL;
+
+	dce_panel_cntl_construct(panel_cntl,
+			init_data,
+			&panel_cntl_regs[init_data->inst],
+			&panel_cntl_shift,
+			&panel_cntl_mask);
+
+	return &panel_cntl->base;
+}
+
+static struct output_pixel_processor *dce110_opp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce110_opp *opp =
+		kzalloc(sizeof(struct dce110_opp), GFP_KERNEL);
+
+	if (!opp)
+		return NULL;
+
+	dce110_opp_construct(opp,
+			     ctx, inst, &opp_regs[inst], &opp_shift, &opp_mask);
+	return &opp->base;
+}
+
+static struct dce_aux *dce110_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst],
+					&aux_mask,
+					&aux_shift,
+					ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
+
+static const struct dce_i2c_registers i2c_hw_regs[] = {
+		i2c_inst_regs(1),
+		i2c_inst_regs(2),
+		i2c_inst_regs(3),
+		i2c_inst_regs(4),
+		i2c_inst_regs(5),
+		i2c_inst_regs(6),
+};
+
+static const struct dce_i2c_shift i2c_shifts = {
+		I2C_COMMON_MASK_SH_LIST_DCE110(__SHIFT)
+};
+
+static const struct dce_i2c_mask i2c_masks = {
+		I2C_COMMON_MASK_SH_LIST_DCE110(_MASK)
+};
+
+static struct dce_i2c_hw *dce110_i2c_hw_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw =
+		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+	dce100_i2c_hw_construct(dce_i2c_hw, ctx, inst,
+				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+static struct clock_source *dce110_clock_source_create(
+	struct dc_context *ctx,
+	struct dc_bios *bios,
+	enum clock_source_id id,
+	const struct dce110_clk_src_regs *regs,
+	bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dce110_clk_src_construct(clk_src, ctx, bios, id,
+			regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	kfree(clk_src);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static void dce110_clock_source_destroy(struct clock_source **clk_src)
+{
+	struct dce110_clk_src *dce110_clk_src;
+
+	if (!clk_src)
+		return;
+
+	dce110_clk_src = TO_DCE110_CLK_SRC(*clk_src);
+
+	kfree(dce110_clk_src->dp_ss_params);
+	kfree(dce110_clk_src->hdmi_ss_params);
+	kfree(dce110_clk_src->dvi_ss_params);
+
+	kfree(dce110_clk_src);
+	*clk_src = NULL;
+}
+
+static void dce110_resource_destruct(struct dce110_resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (pool->base.opps[i] != NULL)
+			dce110_opp_destroy(&pool->base.opps[i]);
+
+		if (pool->base.transforms[i] != NULL)
+			dce110_transform_destroy(&pool->base.transforms[i]);
+
+		if (pool->base.ipps[i] != NULL)
+			dce_ipp_destroy(&pool->base.ipps[i]);
+
+		if (pool->base.mis[i] != NULL) {
+			kfree(TO_DCE_MEM_INPUT(pool->base.mis[i]));
+			pool->base.mis[i] = NULL;
+		}
+
+		if (pool->base.timing_generators[i] != NULL)	{
+			kfree(DCE110TG_FROM_TG(pool->base.timing_generators[i]));
+			pool->base.timing_generators[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+		if (pool->base.hw_i2cs[i] != NULL) {
+			kfree(pool->base.hw_i2cs[i]);
+			pool->base.hw_i2cs[i] = NULL;
+		}
+		if (pool->base.sw_i2cs[i] != NULL) {
+			kfree(pool->base.sw_i2cs[i]);
+			pool->base.sw_i2cs[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.stream_enc_count; i++) {
+		if (pool->base.stream_enc[i] != NULL)
+			kfree(DCE110STRENC_FROM_STRENC(pool->base.stream_enc[i]));
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] != NULL) {
+			dce110_clock_source_destroy(&pool->base.clock_sources[i]);
+		}
+	}
+
+	if (pool->base.dp_clock_source != NULL)
+		dce110_clock_source_destroy(&pool->base.dp_clock_source);
+
+	for (i = 0; i < pool->base.audio_count; i++)	{
+		if (pool->base.audios[i] != NULL) {
+			dce_aud_destroy(&pool->base.audios[i]);
+		}
+	}
+
+	if (pool->base.abm != NULL)
+		dce_abm_destroy(&pool->base.abm);
+
+	if (pool->base.dmcu != NULL)
+		dce_dmcu_destroy(&pool->base.dmcu);
+
+	if (pool->base.irqs != NULL) {
+		dal_irq_service_destroy(&pool->base.irqs);
+	}
+}
+
+
+static void get_pixel_clock_parameters(
+	const struct pipe_ctx *pipe_ctx,
+	struct pixel_clk_params *pixel_clk_params)
+{
+	const struct dc_stream_state *stream = pipe_ctx->stream;
+
+	/*TODO: is this halved for YCbCr 420? in that case we might want to move
+	 * the pixel clock normalization for hdmi up to here instead of doing it
+	 * in pll_adjust_pix_clk
+	 */
+	pixel_clk_params->requested_pix_clk_100hz = stream->timing.pix_clk_100hz;
+	pixel_clk_params->encoder_object_id = stream->link->link_enc->id;
+	pixel_clk_params->signal_type = pipe_ctx->stream->signal;
+	pixel_clk_params->controller_id = pipe_ctx->stream_res.tg->inst + 1;
+	/* TODO: un-hardcode*/
+	pixel_clk_params->requested_sym_clk = LINK_RATE_LOW *
+						LINK_RATE_REF_FREQ_IN_KHZ;
+	pixel_clk_params->flags.ENABLE_SS = 0;
+	pixel_clk_params->color_depth =
+		stream->timing.display_color_depth;
+	pixel_clk_params->flags.DISPLAY_BLANKED = 1;
+	pixel_clk_params->flags.SUPPORT_YCBCR420 = (stream->timing.pixel_encoding ==
+			PIXEL_ENCODING_YCBCR420);
+	pixel_clk_params->pixel_encoding = stream->timing.pixel_encoding;
+	if (stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR422) {
+		pixel_clk_params->color_depth = COLOR_DEPTH_888;
+	}
+	if (stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR420) {
+		pixel_clk_params->requested_pix_clk_100hz  = pixel_clk_params->requested_pix_clk_100hz / 2;
+	}
+	if (stream->timing.timing_3d_format == TIMING_3D_FORMAT_HW_FRAME_PACKING)
+		pixel_clk_params->requested_pix_clk_100hz *= 2;
+
+}
+
+void dce110_resource_build_pipe_hw_param(struct pipe_ctx *pipe_ctx)
+{
+	get_pixel_clock_parameters(pipe_ctx, &pipe_ctx->stream_res.pix_clk_params);
+	pipe_ctx->clock_source->funcs->get_pix_clk_dividers(
+		pipe_ctx->clock_source,
+		&pipe_ctx->stream_res.pix_clk_params,
+		&pipe_ctx->pll_settings);
+	resource_build_bit_depth_reduction_params(pipe_ctx->stream,
+			&pipe_ctx->stream->bit_depth_params);
+	pipe_ctx->stream->clamping.pixel_encoding = pipe_ctx->stream->timing.pixel_encoding;
+}
+
+static bool is_surface_pixel_format_supported(struct pipe_ctx *pipe_ctx, unsigned int underlay_idx)
+{
+	if (pipe_ctx->pipe_idx != underlay_idx)
+		return true;
+	if (!pipe_ctx->plane_state)
+		return false;
+	if (pipe_ctx->plane_state->format < SURFACE_PIXEL_FORMAT_VIDEO_BEGIN)
+		return false;
+	return true;
+}
+
+static enum dc_status build_mapped_resource(
+		const struct dc *dc,
+		struct dc_state *context,
+		struct dc_stream_state *stream)
+{
+	struct pipe_ctx *pipe_ctx = resource_get_otg_master_for_stream(&context->res_ctx, stream);
+
+	if (!pipe_ctx)
+		return DC_ERROR_UNEXPECTED;
+
+	if (!is_surface_pixel_format_supported(pipe_ctx,
+			dc->res_pool->underlay_pipe_index))
+		return DC_SURFACE_PIXEL_FORMAT_UNSUPPORTED;
+
+	dce110_resource_build_pipe_hw_param(pipe_ctx);
+
+	/* TODO: validate audio ASIC caps, encoder */
+
+	resource_build_info_frame(pipe_ctx);
+
+	return DC_OK;
+}
+
+static bool dce110_validate_bandwidth(
+	struct dc *dc,
+	struct dc_state *context,
+	bool fast_validate)
+{
+	bool result = false;
+
+	DC_LOG_BANDWIDTH_CALCS(
+		"%s: start",
+		__func__);
+
+	if (bw_calcs(
+			dc->ctx,
+			dc->bw_dceip,
+			dc->bw_vbios,
+			context->res_ctx.pipe_ctx,
+			dc->res_pool->pipe_count,
+			&context->bw_ctx.bw.dce))
+		result =  true;
+
+	if (!result)
+		DC_LOG_BANDWIDTH_VALIDATION("%s: %dx%d@%d Bandwidth validation failed!\n",
+			__func__,
+			context->streams[0]->timing.h_addressable,
+			context->streams[0]->timing.v_addressable,
+			context->streams[0]->timing.pix_clk_100hz / 10);
+
+	if (memcmp(&dc->current_state->bw_ctx.bw.dce,
+			&context->bw_ctx.bw.dce, sizeof(context->bw_ctx.bw.dce))) {
+
+		DC_LOG_BANDWIDTH_CALCS(
+			"%s: finish,\n"
+			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
+			"stutMark_b: %d stutMark_a: %d\n"
+			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
+			"stutMark_b: %d stutMark_a: %d\n"
+			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
+			"stutMark_b: %d stutMark_a: %d stutter_mode_enable: %d\n"
+			"cstate: %d pstate: %d nbpstate: %d sync: %d dispclk: %d\n"
+			"sclk: %d sclk_sleep: %d yclk: %d blackout_recovery_time_us: %d\n"
+			,
+			__func__,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[0].b_mark,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[0].a_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[0].b_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[0].a_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[0].b_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[0].a_mark,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[1].b_mark,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[1].a_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[1].b_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[1].a_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[1].b_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[1].a_mark,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[2].b_mark,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[2].a_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[2].b_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[2].a_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[2].b_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[2].a_mark,
+			context->bw_ctx.bw.dce.stutter_mode_enable,
+			context->bw_ctx.bw.dce.cpuc_state_change_enable,
+			context->bw_ctx.bw.dce.cpup_state_change_enable,
+			context->bw_ctx.bw.dce.nbp_state_change_enable,
+			context->bw_ctx.bw.dce.all_displays_in_sync,
+			context->bw_ctx.bw.dce.dispclk_khz,
+			context->bw_ctx.bw.dce.sclk_khz,
+			context->bw_ctx.bw.dce.sclk_deep_sleep_khz,
+			context->bw_ctx.bw.dce.yclk_khz,
+			context->bw_ctx.bw.dce.blackout_recovery_time_us);
+	}
+	return result;
+}
+
+static enum dc_status dce110_validate_plane(const struct dc_plane_state *plane_state,
+					    struct dc_caps *caps)
+{
+	if (((plane_state->dst_rect.width * 2) < plane_state->src_rect.width) ||
+	    ((plane_state->dst_rect.height * 2) < plane_state->src_rect.height))
+		return DC_FAIL_SURFACE_VALIDATE;
+
+	return DC_OK;
+}
+
+static bool dce110_validate_surface_sets(
+		struct dc_state *context)
+{
+	int i, j;
+
+	for (i = 0; i < context->stream_count; i++) {
+		if (context->stream_status[i].plane_count == 0)
+			continue;
+
+		if (context->stream_status[i].plane_count > 2)
+			return false;
+
+		for (j = 0; j < context->stream_status[i].plane_count; j++) {
+			struct dc_plane_state *plane =
+				context->stream_status[i].plane_states[j];
+
+			/* underlay validation */
+			if (plane->format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN) {
+
+				if ((plane->src_rect.width > 1920 ||
+					plane->src_rect.height > 1080))
+					return false;
+
+				/* we don't have the logic to support underlay
+				 * only yet so block the use case where we get
+				 * NV12 plane as top layer
+				 */
+				if (j == 0)
+					return false;
+
+				/* irrespective of plane format,
+				 * stream should be RGB encoded
+				 */
+				if (context->streams[i]->timing.pixel_encoding
+						!= PIXEL_ENCODING_RGB)
+					return false;
+
+			}
+
+		}
+	}
+
+	return true;
+}
+
+static enum dc_status dce110_validate_global(
+		struct dc *dc,
+		struct dc_state *context)
+{
+	if (!dce110_validate_surface_sets(context))
+		return DC_FAIL_SURFACE_VALIDATE;
+
+	return DC_OK;
+}
+
+static enum dc_status dce110_add_stream_to_ctx(
+		struct dc *dc,
+		struct dc_state *new_ctx,
+		struct dc_stream_state *dc_stream)
+{
+	enum dc_status result = DC_ERROR_UNEXPECTED;
+
+	result = resource_map_pool_resources(dc, new_ctx, dc_stream);
+
+	if (result == DC_OK)
+		result = resource_map_clock_resources(dc, new_ctx, dc_stream);
+
+
+	if (result == DC_OK)
+		result = build_mapped_resource(dc, new_ctx, dc_stream);
+
+	return result;
+}
+
+static struct pipe_ctx *dce110_acquire_underlay(
+		const struct dc_state *cur_ctx,
+		struct dc_state *new_ctx,
+		const struct resource_pool *pool,
+		const struct pipe_ctx *opp_head_pipe)
+{
+	struct dc_stream_state *stream = opp_head_pipe->stream;
+	struct dc *dc = stream->ctx->dc;
+	struct dce_hwseq *hws = dc->hwseq;
+	struct resource_context *res_ctx = &new_ctx->res_ctx;
+	unsigned int underlay_idx = pool->underlay_pipe_index;
+	struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[underlay_idx];
+
+	if (res_ctx->pipe_ctx[underlay_idx].stream)
+		return NULL;
+
+	pipe_ctx->stream_res.tg = pool->timing_generators[underlay_idx];
+	pipe_ctx->plane_res.mi = pool->mis[underlay_idx];
+	/*pipe_ctx->plane_res.ipp = res_ctx->pool->ipps[underlay_idx];*/
+	pipe_ctx->plane_res.xfm = pool->transforms[underlay_idx];
+	pipe_ctx->stream_res.opp = pool->opps[underlay_idx];
+	pipe_ctx->pipe_idx = underlay_idx;
+
+	pipe_ctx->stream = stream;
+
+	if (!dc->current_state->res_ctx.pipe_ctx[underlay_idx].stream) {
+		struct tg_color black_color = {0};
+		struct dc_bios *dcb = dc->ctx->dc_bios;
+
+		hws->funcs.enable_display_power_gating(
+				dc,
+				pipe_ctx->stream_res.tg->inst,
+				dcb, PIPE_GATING_CONTROL_DISABLE);
+
+		/*
+		 * This is for powering on underlay, so crtc does not
+		 * need to be enabled
+		 */
+
+		pipe_ctx->stream_res.tg->funcs->program_timing(pipe_ctx->stream_res.tg,
+				&stream->timing,
+				0,
+				0,
+				0,
+				0,
+				pipe_ctx->stream->signal,
+				false);
+
+		pipe_ctx->stream_res.tg->funcs->enable_advanced_request(
+				pipe_ctx->stream_res.tg,
+				true,
+				&stream->timing);
+
+		pipe_ctx->plane_res.mi->funcs->allocate_mem_input(pipe_ctx->plane_res.mi,
+				stream->timing.h_total,
+				stream->timing.v_total,
+				stream->timing.pix_clk_100hz / 10,
+				new_ctx->stream_count);
+
+		color_space_to_black_color(dc,
+				COLOR_SPACE_YCBCR601, &black_color);
+		pipe_ctx->stream_res.tg->funcs->set_blank_color(
+				pipe_ctx->stream_res.tg,
+				&black_color);
+	}
+
+	return pipe_ctx;
+}
+
+static void dce110_destroy_resource_pool(struct resource_pool **pool)
+{
+	struct dce110_resource_pool *dce110_pool = TO_DCE110_RES_POOL(*pool);
+
+	dce110_resource_destruct(dce110_pool);
+	kfree(dce110_pool);
+	*pool = NULL;
+}
+
+struct stream_encoder *dce110_find_first_free_match_stream_enc_for_link(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		struct dc_stream_state *stream)
+{
+	int i;
+	int j = -1;
+	struct dc_link *link = stream->link;
+
+	for (i = 0; i < pool->stream_enc_count; i++) {
+		if (!res_ctx->is_stream_enc_acquired[i] &&
+				pool->stream_enc[i]) {
+			/* Store first available for MST second display
+			 * in daisy chain use case
+			 */
+			j = i;
+			if (pool->stream_enc[i]->id ==
+					link->link_enc->preferred_engine)
+				return pool->stream_enc[i];
+		}
+	}
+
+	/*
+	 * For CZ and later, we can allow DIG FE and BE to differ for all display types
+	 */
+
+	if (j >= 0)
+		return pool->stream_enc[j];
+
+	return NULL;
+}
+
+
+static const struct resource_funcs dce110_res_pool_funcs = {
+	.destroy = dce110_destroy_resource_pool,
+	.link_enc_create = dce110_link_encoder_create,
+	.panel_cntl_create = dce110_panel_cntl_create,
+	.validate_bandwidth = dce110_validate_bandwidth,
+	.validate_plane = dce110_validate_plane,
+	.acquire_free_pipe_as_secondary_dpp_pipe = dce110_acquire_underlay,
+	.add_stream_to_ctx = dce110_add_stream_to_ctx,
+	.validate_global = dce110_validate_global,
+	.find_first_free_match_stream_enc_for_link = dce110_find_first_free_match_stream_enc_for_link
+};
+
+static bool underlay_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	struct dce110_timing_generator *dce110_tgv = kzalloc(sizeof(*dce110_tgv),
+							     GFP_KERNEL);
+	struct dce_transform *dce110_xfmv = kzalloc(sizeof(*dce110_xfmv),
+						    GFP_KERNEL);
+	struct dce_mem_input *dce110_miv = kzalloc(sizeof(*dce110_miv),
+						   GFP_KERNEL);
+	struct dce110_opp *dce110_oppv = kzalloc(sizeof(*dce110_oppv),
+						 GFP_KERNEL);
+
+	if (!dce110_tgv || !dce110_xfmv || !dce110_miv || !dce110_oppv) {
+		kfree(dce110_tgv);
+		kfree(dce110_xfmv);
+		kfree(dce110_miv);
+		kfree(dce110_oppv);
+		return false;
+	}
+
+	dce110_opp_v_construct(dce110_oppv, ctx);
+
+	dce110_timing_generator_v_construct(dce110_tgv, ctx);
+	dce110_mem_input_v_construct(dce110_miv, ctx);
+	dce110_transform_v_construct(dce110_xfmv, ctx);
+
+	pool->opps[pool->pipe_count] = &dce110_oppv->base;
+	pool->timing_generators[pool->pipe_count] = &dce110_tgv->base;
+	pool->mis[pool->pipe_count] = &dce110_miv->base;
+	pool->transforms[pool->pipe_count] = &dce110_xfmv->base;
+	pool->pipe_count++;
+
+	/* update the public caps to indicate an underlay is available */
+	ctx->dc->caps.max_slave_planes = 1;
+	ctx->dc->caps.max_slave_yuv_planes = 1;
+	ctx->dc->caps.max_slave_rgb_planes = 0;
+
+	return true;
+}
+
+static void bw_calcs_data_update_from_pplib(struct dc *dc)
+{
+	struct dm_pp_clock_levels clks = {0};
+
+	/*do system clock*/
+	dm_pp_get_clock_levels_by_type(
+			dc->ctx,
+			DM_PP_CLOCK_TYPE_ENGINE_CLK,
+			&clks);
+	/* convert all the clock fro kHz to fix point mHz */
+	dc->bw_vbios->high_sclk = bw_frc_to_fixed(
+			clks.clocks_in_khz[clks.num_levels-1], 1000);
+	dc->bw_vbios->mid1_sclk  = bw_frc_to_fixed(
+			clks.clocks_in_khz[clks.num_levels/8], 1000);
+	dc->bw_vbios->mid2_sclk  = bw_frc_to_fixed(
+			clks.clocks_in_khz[clks.num_levels*2/8], 1000);
+	dc->bw_vbios->mid3_sclk  = bw_frc_to_fixed(
+			clks.clocks_in_khz[clks.num_levels*3/8], 1000);
+	dc->bw_vbios->mid4_sclk  = bw_frc_to_fixed(
+			clks.clocks_in_khz[clks.num_levels*4/8], 1000);
+	dc->bw_vbios->mid5_sclk  = bw_frc_to_fixed(
+			clks.clocks_in_khz[clks.num_levels*5/8], 1000);
+	dc->bw_vbios->mid6_sclk  = bw_frc_to_fixed(
+			clks.clocks_in_khz[clks.num_levels*6/8], 1000);
+	dc->bw_vbios->low_sclk  = bw_frc_to_fixed(
+			clks.clocks_in_khz[0], 1000);
+	dc->sclk_lvls = clks;
+
+	/*do display clock*/
+	dm_pp_get_clock_levels_by_type(
+			dc->ctx,
+			DM_PP_CLOCK_TYPE_DISPLAY_CLK,
+			&clks);
+	dc->bw_vbios->high_voltage_max_dispclk = bw_frc_to_fixed(
+			clks.clocks_in_khz[clks.num_levels-1], 1000);
+	dc->bw_vbios->mid_voltage_max_dispclk  = bw_frc_to_fixed(
+			clks.clocks_in_khz[clks.num_levels>>1], 1000);
+	dc->bw_vbios->low_voltage_max_dispclk  = bw_frc_to_fixed(
+			clks.clocks_in_khz[0], 1000);
+
+	/*do memory clock*/
+	dm_pp_get_clock_levels_by_type(
+			dc->ctx,
+			DM_PP_CLOCK_TYPE_MEMORY_CLK,
+			&clks);
+
+	dc->bw_vbios->low_yclk = bw_frc_to_fixed(
+		clks.clocks_in_khz[0] * MEMORY_TYPE_MULTIPLIER_CZ, 1000);
+	dc->bw_vbios->mid_yclk = bw_frc_to_fixed(
+		clks.clocks_in_khz[clks.num_levels>>1] * MEMORY_TYPE_MULTIPLIER_CZ,
+		1000);
+	dc->bw_vbios->high_yclk = bw_frc_to_fixed(
+		clks.clocks_in_khz[clks.num_levels-1] * MEMORY_TYPE_MULTIPLIER_CZ,
+		1000);
+}
+
+static const struct resource_caps *dce110_resource_cap(
+	struct hw_asic_id *asic_id)
+{
+	if (ASIC_REV_IS_STONEY(asic_id->hw_internal_rev))
+		return &stoney_resource_cap;
+	else
+		return &carrizo_resource_cap;
+}
+
+static bool dce110_resource_construct(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct dce110_resource_pool *pool,
+	struct hw_asic_id asic_id)
+{
+	unsigned int i;
+	struct dc_context *ctx = dc->ctx;
+	struct dc_bios *bp;
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	pool->base.res_cap = dce110_resource_cap(&ctx->asic_id);
+	pool->base.funcs = &dce110_res_pool_funcs;
+
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+
+	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
+	pool->base.underlay_pipe_index = pool->base.pipe_count;
+	pool->base.timing_generator_count = pool->base.res_cap->num_timing_generator;
+	dc->caps.max_downscale_ratio = 150;
+	dc->caps.i2c_speed_in_khz = 40;
+	dc->caps.i2c_speed_in_khz_hdcp = 40;
+	dc->caps.max_cursor_size = 128;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.is_apu = true;
+	dc->caps.extended_aux_timeout_support = false;
+	dc->debug = debug_defaults;
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	bp = ctx->dc_bios;
+
+	if (bp->fw_info_valid && bp->fw_info.external_clock_source_frequency_for_dp != 0) {
+		pool->base.dp_clock_source =
+				dce110_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_EXTERNAL, NULL, true);
+
+		pool->base.clock_sources[0] =
+				dce110_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0,
+						&clk_src_regs[0], false);
+		pool->base.clock_sources[1] =
+				dce110_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1,
+						&clk_src_regs[1], false);
+
+		pool->base.clk_src_count = 2;
+
+		/* TODO: find out if CZ support 3 PLLs */
+	}
+
+	if (pool->base.dp_clock_source == NULL) {
+		dm_error("DC: failed to create dp clock source!\n");
+		BREAK_TO_DEBUGGER();
+		goto res_create_fail;
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto res_create_fail;
+		}
+	}
+
+	pool->base.dmcu = dce_dmcu_create(ctx,
+			&dmcu_regs,
+			&dmcu_shift,
+			&dmcu_mask);
+	if (pool->base.dmcu == NULL) {
+		dm_error("DC: failed to create dmcu!\n");
+		BREAK_TO_DEBUGGER();
+		goto res_create_fail;
+	}
+
+	pool->base.abm = dce_abm_create(ctx,
+			&abm_regs,
+			&abm_shift,
+			&abm_mask);
+	if (pool->base.abm == NULL) {
+		dm_error("DC: failed to create abm!\n");
+		BREAK_TO_DEBUGGER();
+		goto res_create_fail;
+	}
+
+	{
+		struct irq_service_init_data init_data;
+		init_data.ctx = dc->ctx;
+		pool->base.irqs = dal_irq_service_dce110_create(&init_data);
+		if (!pool->base.irqs)
+			goto res_create_fail;
+	}
+
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		pool->base.timing_generators[i] = dce110_timing_generator_create(
+				ctx, i, &dce110_tg_offsets[i]);
+		if (pool->base.timing_generators[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.mis[i] = dce110_mem_input_create(ctx, i);
+		if (pool->base.mis[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create memory input!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.ipps[i] = dce110_ipp_create(ctx, i);
+		if (pool->base.ipps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create input pixel processor!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.transforms[i] = dce110_transform_create(ctx, i);
+		if (pool->base.transforms[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create transform!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.opps[i] = dce110_opp_create(ctx, i);
+		if (pool->base.opps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create output pixel processor!\n");
+			goto res_create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.engines[i] = dce110_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto res_create_fail;
+		}
+		pool->base.hw_i2cs[i] = dce110_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create i2c engine!!\n");
+			goto res_create_fail;
+		}
+		pool->base.sw_i2cs[i] = NULL;
+	}
+
+	if (dc->config.fbc_support)
+		dc->fbc_compressor = dce110_compressor_create(ctx);
+
+	if (!underlay_create(ctx, &pool->base))
+		goto res_create_fail;
+
+	if (!resource_construct(num_virtual_links, dc, &pool->base,
+			&res_create_funcs))
+		goto res_create_fail;
+
+	/* Create hardware sequencer */
+	dce110_hw_sequencer_construct(dc);
+
+	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < pool->base.underlay_pipe_index; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->caps.planes[pool->base.underlay_pipe_index] = underlay_plane_cap;
+
+	bw_calcs_init(dc->bw_dceip, dc->bw_vbios, dc->ctx->asic_id);
+
+	bw_calcs_data_update_from_pplib(dc);
+
+	return true;
+
+res_create_fail:
+	dce110_resource_destruct(pool);
+	return false;
+}
+
+struct resource_pool *dce110_create_resource_pool(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct hw_asic_id asic_id)
+{
+	struct dce110_resource_pool *pool =
+		kzalloc(sizeof(struct dce110_resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dce110_resource_construct(num_virtual_links, dc, pool, asic_id))
+		return &pool->base;
+
+	kfree(pool);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.h
new file mode 100644
index 000000000000..aa4531e0800e
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.h
@@ -0,0 +1,54 @@
+/*
+* Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_RESOURCE_DCE110_H__
+#define __DC_RESOURCE_DCE110_H__
+
+#include "core_types.h"
+
+struct dc;
+struct resource_pool;
+
+#define TO_DCE110_RES_POOL(pool)\
+	container_of(pool, struct dce110_resource_pool, base)
+
+struct dce110_resource_pool {
+	struct resource_pool base;
+};
+
+void dce110_resource_build_pipe_hw_param(struct pipe_ctx *pipe_ctx);
+
+struct resource_pool *dce110_create_resource_pool(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct hw_asic_id asic_id);
+
+struct stream_encoder *dce110_find_first_free_match_stream_enc_for_link(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		struct dc_stream_state *stream);
+
+#endif /* __DC_RESOURCE_DCE110_H__ */
+
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c
new file mode 100644
index 000000000000..d1edac46c9a0
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c
@@ -0,0 +1,1431 @@
+/*
+* Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dm_services.h"
+
+#include "link_encoder.h"
+#include "stream_encoder.h"
+
+#include "resource.h"
+#include "include/irq_service_interface.h"
+#include "dce110/dce110_resource.h"
+#include "dce110/dce110_timing_generator.h"
+
+#include "irq/dce110/irq_service_dce110.h"
+#include "dce/dce_mem_input.h"
+#include "dce/dce_transform.h"
+#include "dce/dce_link_encoder.h"
+#include "dce/dce_stream_encoder.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_opp.h"
+#include "dce/dce_ipp.h"
+#include "dce/dce_clock_source.h"
+
+#include "dce/dce_hwseq.h"
+#include "dce112/dce112_hwseq.h"
+#include "dce/dce_abm.h"
+#include "dce/dce_dmcu.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_i2c.h"
+#include "dce/dce_panel_cntl.h"
+
+#include "reg_helper.h"
+
+#include "dce/dce_11_2_d.h"
+#include "dce/dce_11_2_sh_mask.h"
+
+#include "dce100/dce100_resource.h"
+#include "dce112_resource.h"
+
+#define DC_LOGGER				\
+		dc->ctx->logger
+
+#ifndef mmDP_DPHY_INTERNAL_CTRL
+	#define mmDP_DPHY_INTERNAL_CTRL 0x4aa7
+	#define mmDP0_DP_DPHY_INTERNAL_CTRL 0x4aa7
+	#define mmDP1_DP_DPHY_INTERNAL_CTRL 0x4ba7
+	#define mmDP2_DP_DPHY_INTERNAL_CTRL 0x4ca7
+	#define mmDP3_DP_DPHY_INTERNAL_CTRL 0x4da7
+	#define mmDP4_DP_DPHY_INTERNAL_CTRL 0x4ea7
+	#define mmDP5_DP_DPHY_INTERNAL_CTRL 0x4fa7
+	#define mmDP6_DP_DPHY_INTERNAL_CTRL 0x54a7
+	#define mmDP7_DP_DPHY_INTERNAL_CTRL 0x56a7
+	#define mmDP8_DP_DPHY_INTERNAL_CTRL 0x57a7
+#endif
+
+#ifndef mmBIOS_SCRATCH_2
+	#define mmBIOS_SCRATCH_2 0x05CB
+	#define mmBIOS_SCRATCH_3 0x05CC
+	#define mmBIOS_SCRATCH_6 0x05CF
+#endif
+
+#ifndef mmDP_DPHY_BS_SR_SWAP_CNTL
+	#define mmDP_DPHY_BS_SR_SWAP_CNTL                       0x4ADC
+	#define mmDP0_DP_DPHY_BS_SR_SWAP_CNTL                   0x4ADC
+	#define mmDP1_DP_DPHY_BS_SR_SWAP_CNTL                   0x4BDC
+	#define mmDP2_DP_DPHY_BS_SR_SWAP_CNTL                   0x4CDC
+	#define mmDP3_DP_DPHY_BS_SR_SWAP_CNTL                   0x4DDC
+	#define mmDP4_DP_DPHY_BS_SR_SWAP_CNTL                   0x4EDC
+	#define mmDP5_DP_DPHY_BS_SR_SWAP_CNTL                   0x4FDC
+	#define mmDP6_DP_DPHY_BS_SR_SWAP_CNTL                   0x54DC
+#endif
+
+#ifndef mmDP_DPHY_FAST_TRAINING
+	#define mmDP_DPHY_FAST_TRAINING                         0x4ABC
+	#define mmDP0_DP_DPHY_FAST_TRAINING                     0x4ABC
+	#define mmDP1_DP_DPHY_FAST_TRAINING                     0x4BBC
+	#define mmDP2_DP_DPHY_FAST_TRAINING                     0x4CBC
+	#define mmDP3_DP_DPHY_FAST_TRAINING                     0x4DBC
+	#define mmDP4_DP_DPHY_FAST_TRAINING                     0x4EBC
+	#define mmDP5_DP_DPHY_FAST_TRAINING                     0x4FBC
+	#define mmDP6_DP_DPHY_FAST_TRAINING                     0x54BC
+#endif
+
+enum dce112_clk_src_array_id {
+	DCE112_CLK_SRC_PLL0,
+	DCE112_CLK_SRC_PLL1,
+	DCE112_CLK_SRC_PLL2,
+	DCE112_CLK_SRC_PLL3,
+	DCE112_CLK_SRC_PLL4,
+	DCE112_CLK_SRC_PLL5,
+
+	DCE112_CLK_SRC_TOTAL
+};
+
+static const struct dce110_timing_generator_offsets dce112_tg_offsets[] = {
+	{
+		.crtc = (mmCRTC0_CRTC_CONTROL - mmCRTC_CONTROL),
+		.dcp =  (mmDCP0_GRPH_CONTROL - mmGRPH_CONTROL),
+	},
+	{
+		.crtc = (mmCRTC1_CRTC_CONTROL - mmCRTC_CONTROL),
+		.dcp = (mmDCP1_GRPH_CONTROL - mmGRPH_CONTROL),
+	},
+	{
+		.crtc = (mmCRTC2_CRTC_CONTROL - mmCRTC_CONTROL),
+		.dcp = (mmDCP2_GRPH_CONTROL - mmGRPH_CONTROL),
+	},
+	{
+		.crtc = (mmCRTC3_CRTC_CONTROL - mmCRTC_CONTROL),
+		.dcp = (mmDCP3_GRPH_CONTROL - mmGRPH_CONTROL),
+	},
+	{
+		.crtc = (mmCRTC4_CRTC_CONTROL - mmCRTC_CONTROL),
+		.dcp = (mmDCP4_GRPH_CONTROL - mmGRPH_CONTROL),
+	},
+	{
+		.crtc = (mmCRTC5_CRTC_CONTROL - mmCRTC_CONTROL),
+		.dcp = (mmDCP5_GRPH_CONTROL - mmGRPH_CONTROL),
+	}
+};
+
+/* set register offset */
+#define SR(reg_name)\
+	.reg_name = mm ## reg_name
+
+/* set register offset with instance */
+#define SRI(reg_name, block, id)\
+	.reg_name = mm ## block ## id ## _ ## reg_name
+
+static const struct dce_dmcu_registers dmcu_regs = {
+		DMCU_DCE110_COMMON_REG_LIST()
+};
+
+static const struct dce_dmcu_shift dmcu_shift = {
+		DMCU_MASK_SH_LIST_DCE110(__SHIFT)
+};
+
+static const struct dce_dmcu_mask dmcu_mask = {
+		DMCU_MASK_SH_LIST_DCE110(_MASK)
+};
+
+static const struct dce_abm_registers abm_regs = {
+		ABM_DCE110_COMMON_REG_LIST()
+};
+
+static const struct dce_abm_shift abm_shift = {
+		ABM_MASK_SH_LIST_DCE110(__SHIFT)
+};
+
+static const struct dce_abm_mask abm_mask = {
+		ABM_MASK_SH_LIST_DCE110(_MASK)
+};
+
+static const struct dce110_aux_registers_shift aux_shift = {
+	DCE_AUX_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+	DCE_AUX_MASK_SH_LIST(_MASK)
+};
+
+#define ipp_regs(id)\
+[id] = {\
+		IPP_DCE110_REG_LIST_DCE_BASE(id)\
+}
+
+static const struct dce_ipp_registers ipp_regs[] = {
+		ipp_regs(0),
+		ipp_regs(1),
+		ipp_regs(2),
+		ipp_regs(3),
+		ipp_regs(4),
+		ipp_regs(5)
+};
+
+static const struct dce_ipp_shift ipp_shift = {
+		IPP_DCE100_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
+};
+
+static const struct dce_ipp_mask ipp_mask = {
+		IPP_DCE100_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
+};
+
+#define transform_regs(id)\
+[id] = {\
+		XFM_COMMON_REG_LIST_DCE110(id)\
+}
+
+static const struct dce_transform_registers xfm_regs[] = {
+		transform_regs(0),
+		transform_regs(1),
+		transform_regs(2),
+		transform_regs(3),
+		transform_regs(4),
+		transform_regs(5)
+};
+
+static const struct dce_transform_shift xfm_shift = {
+		XFM_COMMON_MASK_SH_LIST_DCE110(__SHIFT)
+};
+
+static const struct dce_transform_mask xfm_mask = {
+		XFM_COMMON_MASK_SH_LIST_DCE110(_MASK)
+};
+
+#define aux_regs(id)\
+[id] = {\
+	AUX_REG_LIST(id)\
+}
+
+static const struct dce110_link_enc_aux_registers link_enc_aux_regs[] = {
+		aux_regs(0),
+		aux_regs(1),
+		aux_regs(2),
+		aux_regs(3),
+		aux_regs(4),
+		aux_regs(5)
+};
+
+static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
+	{ DCE_PANEL_CNTL_REG_LIST() }
+};
+
+static const struct dce_panel_cntl_shift panel_cntl_shift = {
+	DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_panel_cntl_mask panel_cntl_mask = {
+	DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
+};
+
+#define hpd_regs(id)\
+[id] = {\
+	HPD_REG_LIST(id)\
+}
+
+static const struct dce110_link_enc_hpd_registers link_enc_hpd_regs[] = {
+		hpd_regs(0),
+		hpd_regs(1),
+		hpd_regs(2),
+		hpd_regs(3),
+		hpd_regs(4),
+		hpd_regs(5)
+};
+
+#define link_regs(id)\
+[id] = {\
+	LE_DCE110_REG_LIST(id)\
+}
+
+static const struct dce110_link_enc_registers link_enc_regs[] = {
+	link_regs(0),
+	link_regs(1),
+	link_regs(2),
+	link_regs(3),
+	link_regs(4),
+	link_regs(5),
+	link_regs(6),
+};
+
+#define stream_enc_regs(id)\
+[id] = {\
+	SE_COMMON_REG_LIST(id),\
+	.TMDS_CNTL = 0,\
+}
+
+static const struct dce110_stream_enc_registers stream_enc_regs[] = {
+	stream_enc_regs(0),
+	stream_enc_regs(1),
+	stream_enc_regs(2),
+	stream_enc_regs(3),
+	stream_enc_regs(4),
+	stream_enc_regs(5)
+};
+
+static const struct dce_stream_encoder_shift se_shift = {
+		SE_COMMON_MASK_SH_LIST_DCE112(__SHIFT)
+};
+
+static const struct dce_stream_encoder_mask se_mask = {
+		SE_COMMON_MASK_SH_LIST_DCE112(_MASK)
+};
+
+#define opp_regs(id)\
+[id] = {\
+	OPP_DCE_112_REG_LIST(id),\
+}
+
+static const struct dce_opp_registers opp_regs[] = {
+	opp_regs(0),
+	opp_regs(1),
+	opp_regs(2),
+	opp_regs(3),
+	opp_regs(4),
+	opp_regs(5)
+};
+
+static const struct dce_opp_shift opp_shift = {
+	OPP_COMMON_MASK_SH_LIST_DCE_112(__SHIFT)
+};
+
+static const struct dce_opp_mask opp_mask = {
+	OPP_COMMON_MASK_SH_LIST_DCE_112(_MASK)
+};
+
+#define aux_engine_regs(id)\
+[id] = {\
+	AUX_COMMON_REG_LIST(id), \
+	.AUX_RESET_MASK = 0 \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1),
+		aux_engine_regs(2),
+		aux_engine_regs(3),
+		aux_engine_regs(4),
+		aux_engine_regs(5)
+};
+
+#define audio_regs(id)\
+[id] = {\
+	AUD_COMMON_REG_LIST(id)\
+}
+
+static const struct dce_audio_registers audio_regs[] = {
+	audio_regs(0),
+	audio_regs(1),
+	audio_regs(2),
+	audio_regs(3),
+	audio_regs(4),
+	audio_regs(5)
+};
+
+static const struct dce_audio_shift audio_shift = {
+		AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_audio_mask audio_mask = {
+		AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+#define clk_src_regs(index, id)\
+[index] = {\
+	CS_COMMON_REG_LIST_DCE_112(id),\
+}
+
+static const struct dce110_clk_src_regs clk_src_regs[] = {
+	clk_src_regs(0, A),
+	clk_src_regs(1, B),
+	clk_src_regs(2, C),
+	clk_src_regs(3, D),
+	clk_src_regs(4, E),
+	clk_src_regs(5, F)
+};
+
+static const struct dce110_clk_src_shift cs_shift = {
+		CS_COMMON_MASK_SH_LIST_DCE_112(__SHIFT)
+};
+
+static const struct dce110_clk_src_mask cs_mask = {
+		CS_COMMON_MASK_SH_LIST_DCE_112(_MASK)
+};
+
+static const struct bios_registers bios_regs = {
+	.BIOS_SCRATCH_3 = mmBIOS_SCRATCH_3,
+	.BIOS_SCRATCH_6 = mmBIOS_SCRATCH_6
+};
+
+static const struct resource_caps polaris_10_resource_cap = {
+		.num_timing_generator = 6,
+		.num_audio = 6,
+		.num_stream_encoder = 6,
+		.num_pll = 8, /* why 8? 6 combo PHY PLL + 2 regular PLLs? */
+		.num_ddc = 6,
+};
+
+static const struct resource_caps polaris_11_resource_cap = {
+		.num_timing_generator = 5,
+		.num_audio = 5,
+		.num_stream_encoder = 5,
+		.num_pll = 8, /* why 8? 6 combo PHY PLL + 2 regular PLLs? */
+		.num_ddc = 5,
+};
+
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCE_RGB,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = false,
+			.fp16 = true
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 1,
+			.fp16 = 1
+	},
+
+	.max_downscale_factor = {
+			.argb8888 = 250,
+			.nv12 = 1,
+			.fp16 = 1
+	},
+	64,
+	64
+};
+
+static const struct dc_debug_options debug_defaults = {
+		.enable_legacy_fast_update = true,
+};
+
+#define CTX  ctx
+#define REG(reg) mm ## reg
+
+#ifndef mmCC_DC_HDMI_STRAPS
+#define mmCC_DC_HDMI_STRAPS 0x4819
+#define CC_DC_HDMI_STRAPS__HDMI_DISABLE_MASK 0x40
+#define CC_DC_HDMI_STRAPS__HDMI_DISABLE__SHIFT 0x6
+#define CC_DC_HDMI_STRAPS__AUDIO_STREAM_NUMBER_MASK 0x700
+#define CC_DC_HDMI_STRAPS__AUDIO_STREAM_NUMBER__SHIFT 0x8
+#endif
+
+static int map_transmitter_id_to_phy_instance(
+	enum transmitter transmitter)
+{
+	switch (transmitter) {
+	case TRANSMITTER_UNIPHY_A:
+		return 0;
+	case TRANSMITTER_UNIPHY_B:
+		return 1;
+	case TRANSMITTER_UNIPHY_C:
+		return 2;
+	case TRANSMITTER_UNIPHY_D:
+		return 3;
+	case TRANSMITTER_UNIPHY_E:
+		return 4;
+	case TRANSMITTER_UNIPHY_F:
+		return 5;
+	case TRANSMITTER_UNIPHY_G:
+		return 6;
+	default:
+		ASSERT(0);
+		return 0;
+	}
+}
+
+static void read_dce_straps(
+	struct dc_context *ctx,
+	struct resource_straps *straps)
+{
+	REG_GET_2(CC_DC_HDMI_STRAPS,
+			HDMI_DISABLE, &straps->hdmi_disable,
+			AUDIO_STREAM_NUMBER, &straps->audio_stream_number);
+
+	REG_GET(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO, &straps->dc_pinstraps_audio);
+}
+
+static struct audio *create_audio(
+		struct dc_context *ctx, unsigned int inst)
+{
+	return dce_audio_create(ctx, inst,
+			&audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+
+static struct timing_generator *dce112_timing_generator_create(
+		struct dc_context *ctx,
+		uint32_t instance,
+		const struct dce110_timing_generator_offsets *offsets)
+{
+	struct dce110_timing_generator *tg110 =
+		kzalloc(sizeof(struct dce110_timing_generator), GFP_KERNEL);
+
+	if (!tg110)
+		return NULL;
+
+	dce110_timing_generator_construct(tg110, ctx, instance, offsets);
+	return &tg110->base;
+}
+
+static struct stream_encoder *dce112_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dce110_stream_encoder *enc110 =
+		kzalloc(sizeof(struct dce110_stream_encoder), GFP_KERNEL);
+
+	if (!enc110)
+		return NULL;
+
+	dce110_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id,
+					&stream_enc_regs[eng_id],
+					&se_shift, &se_mask);
+	return &enc110->base;
+}
+
+#define SRII(reg_name, block, id)\
+	.reg_name[id] = mm ## block ## id ## _ ## reg_name
+
+static const struct dce_hwseq_registers hwseq_reg = {
+		HWSEQ_DCE112_REG_LIST()
+};
+
+static const struct dce_hwseq_shift hwseq_shift = {
+		HWSEQ_DCE112_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+		HWSEQ_DCE112_MASK_SH_LIST(_MASK)
+};
+
+static struct dce_hwseq *dce112_hwseq_create(
+	struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &hwseq_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+	}
+	return hws;
+}
+
+static const struct resource_create_funcs res_create_funcs = {
+	.read_dce_straps = read_dce_straps,
+	.create_audio = create_audio,
+	.create_stream_encoder = dce112_stream_encoder_create,
+	.create_hwseq = dce112_hwseq_create,
+};
+
+#define mi_inst_regs(id) { MI_DCE11_2_REG_LIST(id) }
+static const struct dce_mem_input_registers mi_regs[] = {
+		mi_inst_regs(0),
+		mi_inst_regs(1),
+		mi_inst_regs(2),
+		mi_inst_regs(3),
+		mi_inst_regs(4),
+		mi_inst_regs(5),
+};
+
+static const struct dce_mem_input_shift mi_shifts = {
+		MI_DCE11_2_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_mem_input_mask mi_masks = {
+		MI_DCE11_2_MASK_SH_LIST(_MASK)
+};
+
+static struct mem_input *dce112_mem_input_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_mem_input *dce_mi = kzalloc(sizeof(struct dce_mem_input),
+					       GFP_KERNEL);
+
+	if (!dce_mi) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dce112_mem_input_construct(dce_mi, ctx, inst, &mi_regs[inst], &mi_shifts, &mi_masks);
+	return &dce_mi->base;
+}
+
+static void dce112_transform_destroy(struct transform **xfm)
+{
+	kfree(TO_DCE_TRANSFORM(*xfm));
+	*xfm = NULL;
+}
+
+static struct transform *dce112_transform_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_transform *transform =
+		kzalloc(sizeof(struct dce_transform), GFP_KERNEL);
+
+	if (!transform)
+		return NULL;
+
+	dce_transform_construct(transform, ctx, inst,
+				&xfm_regs[inst], &xfm_shift, &xfm_mask);
+	transform->lb_memory_size = 0x1404; /*5124*/
+	return &transform->base;
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+		.max_hdmi_deep_color = COLOR_DEPTH_121212,
+		.max_hdmi_pixel_clock = 600000,
+		.hdmi_ycbcr420_supported = true,
+		.dp_ycbcr420_supported = false,
+		.flags.bits.IS_HBR2_CAPABLE = true,
+		.flags.bits.IS_HBR3_CAPABLE = true,
+		.flags.bits.IS_TPS3_CAPABLE = true,
+		.flags.bits.IS_TPS4_CAPABLE = true
+};
+
+static struct link_encoder *dce112_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dce110_link_encoder *enc110 =
+		kzalloc(sizeof(struct dce110_link_encoder), GFP_KERNEL);
+	int link_regs_id;
+
+	if (!enc110)
+		return NULL;
+
+	link_regs_id =
+		map_transmitter_id_to_phy_instance(enc_init_data->transmitter);
+
+	dce110_link_encoder_construct(enc110,
+				      enc_init_data,
+				      &link_enc_feature,
+				      &link_enc_regs[link_regs_id],
+				      &link_enc_aux_regs[enc_init_data->channel - 1],
+				      &link_enc_hpd_regs[enc_init_data->hpd_source]);
+	return &enc110->base;
+}
+
+static struct panel_cntl *dce112_panel_cntl_create(const struct panel_cntl_init_data *init_data)
+{
+	struct dce_panel_cntl *panel_cntl =
+		kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
+
+	if (!panel_cntl)
+		return NULL;
+
+	dce_panel_cntl_construct(panel_cntl,
+			init_data,
+			&panel_cntl_regs[init_data->inst],
+			&panel_cntl_shift,
+			&panel_cntl_mask);
+
+	return &panel_cntl->base;
+}
+
+static struct input_pixel_processor *dce112_ipp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dce_ipp *ipp = kzalloc(sizeof(struct dce_ipp), GFP_KERNEL);
+
+	if (!ipp) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dce_ipp_construct(ipp, ctx, inst,
+			&ipp_regs[inst], &ipp_shift, &ipp_mask);
+	return &ipp->base;
+}
+
+static struct output_pixel_processor *dce112_opp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce110_opp *opp =
+		kzalloc(sizeof(struct dce110_opp), GFP_KERNEL);
+
+	if (!opp)
+		return NULL;
+
+	dce110_opp_construct(opp,
+			     ctx, inst, &opp_regs[inst], &opp_shift, &opp_mask);
+	return &opp->base;
+}
+
+static struct dce_aux *dce112_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst],
+					&aux_mask,
+					&aux_shift,
+					ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
+
+static const struct dce_i2c_registers i2c_hw_regs[] = {
+		i2c_inst_regs(1),
+		i2c_inst_regs(2),
+		i2c_inst_regs(3),
+		i2c_inst_regs(4),
+		i2c_inst_regs(5),
+		i2c_inst_regs(6),
+};
+
+static const struct dce_i2c_shift i2c_shifts = {
+		I2C_COMMON_MASK_SH_LIST_DCE110(__SHIFT)
+};
+
+static const struct dce_i2c_mask i2c_masks = {
+		I2C_COMMON_MASK_SH_LIST_DCE110(_MASK)
+};
+
+static struct dce_i2c_hw *dce112_i2c_hw_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw =
+		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+	dce112_i2c_hw_construct(dce_i2c_hw, ctx, inst,
+				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+static struct clock_source *dce112_clock_source_create(
+	struct dc_context *ctx,
+	struct dc_bios *bios,
+	enum clock_source_id id,
+	const struct dce110_clk_src_regs *regs,
+	bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dce112_clk_src_construct(clk_src, ctx, bios, id,
+			regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	kfree(clk_src);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static void dce112_clock_source_destroy(struct clock_source **clk_src)
+{
+	kfree(TO_DCE110_CLK_SRC(*clk_src));
+	*clk_src = NULL;
+}
+
+static void dce112_resource_destruct(struct dce110_resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (pool->base.opps[i] != NULL)
+			dce110_opp_destroy(&pool->base.opps[i]);
+
+		if (pool->base.transforms[i] != NULL)
+			dce112_transform_destroy(&pool->base.transforms[i]);
+
+		if (pool->base.ipps[i] != NULL)
+			dce_ipp_destroy(&pool->base.ipps[i]);
+
+		if (pool->base.mis[i] != NULL) {
+			kfree(TO_DCE_MEM_INPUT(pool->base.mis[i]));
+			pool->base.mis[i] = NULL;
+		}
+
+		if (pool->base.timing_generators[i] != NULL) {
+			kfree(DCE110TG_FROM_TG(pool->base.timing_generators[i]));
+			pool->base.timing_generators[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+		if (pool->base.hw_i2cs[i] != NULL) {
+			kfree(pool->base.hw_i2cs[i]);
+			pool->base.hw_i2cs[i] = NULL;
+		}
+		if (pool->base.sw_i2cs[i] != NULL) {
+			kfree(pool->base.sw_i2cs[i]);
+			pool->base.sw_i2cs[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.stream_enc_count; i++) {
+		if (pool->base.stream_enc[i] != NULL)
+			kfree(DCE110STRENC_FROM_STRENC(pool->base.stream_enc[i]));
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] != NULL) {
+			dce112_clock_source_destroy(&pool->base.clock_sources[i]);
+		}
+	}
+
+	if (pool->base.dp_clock_source != NULL)
+		dce112_clock_source_destroy(&pool->base.dp_clock_source);
+
+	for (i = 0; i < pool->base.audio_count; i++)	{
+		if (pool->base.audios[i] != NULL) {
+			dce_aud_destroy(&pool->base.audios[i]);
+		}
+	}
+
+	if (pool->base.abm != NULL)
+		dce_abm_destroy(&pool->base.abm);
+
+	if (pool->base.dmcu != NULL)
+		dce_dmcu_destroy(&pool->base.dmcu);
+
+	if (pool->base.irqs != NULL) {
+		dal_irq_service_destroy(&pool->base.irqs);
+	}
+}
+
+static struct clock_source *find_matching_pll(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		const struct dc_stream_state *const stream)
+{
+	switch (stream->link->link_enc->transmitter) {
+	case TRANSMITTER_UNIPHY_A:
+		return pool->clock_sources[DCE112_CLK_SRC_PLL0];
+	case TRANSMITTER_UNIPHY_B:
+		return pool->clock_sources[DCE112_CLK_SRC_PLL1];
+	case TRANSMITTER_UNIPHY_C:
+		return pool->clock_sources[DCE112_CLK_SRC_PLL2];
+	case TRANSMITTER_UNIPHY_D:
+		return pool->clock_sources[DCE112_CLK_SRC_PLL3];
+	case TRANSMITTER_UNIPHY_E:
+		return pool->clock_sources[DCE112_CLK_SRC_PLL4];
+	case TRANSMITTER_UNIPHY_F:
+		return pool->clock_sources[DCE112_CLK_SRC_PLL5];
+	default:
+		return NULL;
+	}
+
+	return NULL;
+}
+
+static enum dc_status build_mapped_resource(
+		const struct dc *dc,
+		struct dc_state *context,
+		struct dc_stream_state *stream)
+{
+	struct pipe_ctx *pipe_ctx = resource_get_otg_master_for_stream(&context->res_ctx, stream);
+
+	if (!pipe_ctx)
+		return DC_ERROR_UNEXPECTED;
+
+	dce110_resource_build_pipe_hw_param(pipe_ctx);
+
+	resource_build_info_frame(pipe_ctx);
+
+	return DC_OK;
+}
+
+bool dce112_validate_bandwidth(
+	struct dc *dc,
+	struct dc_state *context,
+	bool fast_validate)
+{
+	bool result = false;
+
+	DC_LOG_BANDWIDTH_CALCS(
+		"%s: start",
+		__func__);
+
+	if (bw_calcs(
+			dc->ctx,
+			dc->bw_dceip,
+			dc->bw_vbios,
+			context->res_ctx.pipe_ctx,
+			dc->res_pool->pipe_count,
+			&context->bw_ctx.bw.dce))
+		result = true;
+
+	if (!result)
+		DC_LOG_BANDWIDTH_VALIDATION(
+			"%s: Bandwidth validation failed!",
+			__func__);
+
+	if (memcmp(&dc->current_state->bw_ctx.bw.dce,
+			&context->bw_ctx.bw.dce, sizeof(context->bw_ctx.bw.dce))) {
+
+		DC_LOG_BANDWIDTH_CALCS(
+			"%s: finish,\n"
+			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
+			"stutMark_b: %d stutMark_a: %d\n"
+			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
+			"stutMark_b: %d stutMark_a: %d\n"
+			"nbpMark_b: %d nbpMark_a: %d urgentMark_b: %d urgentMark_a: %d\n"
+			"stutMark_b: %d stutMark_a: %d stutter_mode_enable: %d\n"
+			"cstate: %d pstate: %d nbpstate: %d sync: %d dispclk: %d\n"
+			"sclk: %d sclk_sleep: %d yclk: %d blackout_recovery_time_us: %d\n"
+			,
+			__func__,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[0].b_mark,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[0].a_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[0].b_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[0].a_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[0].b_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[0].a_mark,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[1].b_mark,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[1].a_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[1].b_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[1].a_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[1].b_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[1].a_mark,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[2].b_mark,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[2].a_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[2].b_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[2].a_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[2].b_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[2].a_mark,
+			context->bw_ctx.bw.dce.stutter_mode_enable,
+			context->bw_ctx.bw.dce.cpuc_state_change_enable,
+			context->bw_ctx.bw.dce.cpup_state_change_enable,
+			context->bw_ctx.bw.dce.nbp_state_change_enable,
+			context->bw_ctx.bw.dce.all_displays_in_sync,
+			context->bw_ctx.bw.dce.dispclk_khz,
+			context->bw_ctx.bw.dce.sclk_khz,
+			context->bw_ctx.bw.dce.sclk_deep_sleep_khz,
+			context->bw_ctx.bw.dce.yclk_khz,
+			context->bw_ctx.bw.dce.blackout_recovery_time_us);
+	}
+	return result;
+}
+
+enum dc_status resource_map_phy_clock_resources(
+		const struct dc *dc,
+		struct dc_state *context,
+		struct dc_stream_state *stream)
+{
+
+	/* acquire new resources */
+	struct pipe_ctx *pipe_ctx = resource_get_otg_master_for_stream(
+			&context->res_ctx, stream);
+
+	if (!pipe_ctx)
+		return DC_ERROR_UNEXPECTED;
+
+	if (dc_is_dp_signal(pipe_ctx->stream->signal)
+		|| dc_is_virtual_signal(pipe_ctx->stream->signal))
+		pipe_ctx->clock_source =
+				dc->res_pool->dp_clock_source;
+	else {
+		if (stream && stream->link && stream->link->link_enc)
+			pipe_ctx->clock_source = find_matching_pll(
+				&context->res_ctx, dc->res_pool,
+				stream);
+	}
+
+	if (pipe_ctx->clock_source == NULL)
+		return DC_NO_CLOCK_SOURCE_RESOURCE;
+
+	resource_reference_clock_source(
+		&context->res_ctx,
+		dc->res_pool,
+		pipe_ctx->clock_source);
+
+	return DC_OK;
+}
+
+static bool dce112_validate_surface_sets(
+		struct dc_state *context)
+{
+	int i;
+
+	for (i = 0; i < context->stream_count; i++) {
+		if (context->stream_status[i].plane_count == 0)
+			continue;
+
+		if (context->stream_status[i].plane_count > 1)
+			return false;
+
+		if (context->stream_status[i].plane_states[0]->format
+				>= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN)
+			return false;
+	}
+
+	return true;
+}
+
+enum dc_status dce112_add_stream_to_ctx(
+		struct dc *dc,
+		struct dc_state *new_ctx,
+		struct dc_stream_state *dc_stream)
+{
+	enum dc_status result;
+
+	result = resource_map_pool_resources(dc, new_ctx, dc_stream);
+
+	if (result == DC_OK)
+		result = resource_map_phy_clock_resources(dc, new_ctx, dc_stream);
+
+
+	if (result == DC_OK)
+		result = build_mapped_resource(dc, new_ctx, dc_stream);
+
+	return result;
+}
+
+static enum dc_status dce112_validate_global(
+		struct dc *dc,
+		struct dc_state *context)
+{
+	if (!dce112_validate_surface_sets(context))
+		return DC_FAIL_SURFACE_VALIDATE;
+
+	return DC_OK;
+}
+
+static void dce112_destroy_resource_pool(struct resource_pool **pool)
+{
+	struct dce110_resource_pool *dce110_pool = TO_DCE110_RES_POOL(*pool);
+
+	dce112_resource_destruct(dce110_pool);
+	kfree(dce110_pool);
+	*pool = NULL;
+}
+
+static const struct resource_funcs dce112_res_pool_funcs = {
+	.destroy = dce112_destroy_resource_pool,
+	.link_enc_create = dce112_link_encoder_create,
+	.panel_cntl_create = dce112_panel_cntl_create,
+	.validate_bandwidth = dce112_validate_bandwidth,
+	.validate_plane = dce100_validate_plane,
+	.add_stream_to_ctx = dce112_add_stream_to_ctx,
+	.validate_global = dce112_validate_global,
+	.find_first_free_match_stream_enc_for_link = dce110_find_first_free_match_stream_enc_for_link
+};
+
+static void bw_calcs_data_update_from_pplib(struct dc *dc)
+{
+	struct dm_pp_clock_levels_with_latency eng_clks = {0};
+	struct dm_pp_clock_levels_with_latency mem_clks = {0};
+	struct dm_pp_wm_sets_with_clock_ranges clk_ranges = {0};
+	struct dm_pp_clock_levels clks = {0};
+	int memory_type_multiplier = MEMORY_TYPE_MULTIPLIER_CZ;
+
+	if (dc->bw_vbios && dc->bw_vbios->memory_type == bw_def_hbm)
+		memory_type_multiplier = MEMORY_TYPE_HBM;
+
+	/*do system clock  TODO PPLIB: after PPLIB implement,
+	 * then remove old way
+	 */
+	if (!dm_pp_get_clock_levels_by_type_with_latency(
+			dc->ctx,
+			DM_PP_CLOCK_TYPE_ENGINE_CLK,
+			&eng_clks)) {
+
+		/* This is only for temporary */
+		dm_pp_get_clock_levels_by_type(
+				dc->ctx,
+				DM_PP_CLOCK_TYPE_ENGINE_CLK,
+				&clks);
+		/* convert all the clock fro kHz to fix point mHz */
+		dc->bw_vbios->high_sclk = bw_frc_to_fixed(
+				clks.clocks_in_khz[clks.num_levels-1], 1000);
+		dc->bw_vbios->mid1_sclk  = bw_frc_to_fixed(
+				clks.clocks_in_khz[clks.num_levels/8], 1000);
+		dc->bw_vbios->mid2_sclk  = bw_frc_to_fixed(
+				clks.clocks_in_khz[clks.num_levels*2/8], 1000);
+		dc->bw_vbios->mid3_sclk  = bw_frc_to_fixed(
+				clks.clocks_in_khz[clks.num_levels*3/8], 1000);
+		dc->bw_vbios->mid4_sclk  = bw_frc_to_fixed(
+				clks.clocks_in_khz[clks.num_levels*4/8], 1000);
+		dc->bw_vbios->mid5_sclk  = bw_frc_to_fixed(
+				clks.clocks_in_khz[clks.num_levels*5/8], 1000);
+		dc->bw_vbios->mid6_sclk  = bw_frc_to_fixed(
+				clks.clocks_in_khz[clks.num_levels*6/8], 1000);
+		dc->bw_vbios->low_sclk  = bw_frc_to_fixed(
+				clks.clocks_in_khz[0], 1000);
+
+		/*do memory clock*/
+		dm_pp_get_clock_levels_by_type(
+				dc->ctx,
+				DM_PP_CLOCK_TYPE_MEMORY_CLK,
+				&clks);
+
+		dc->bw_vbios->low_yclk = bw_frc_to_fixed(
+			clks.clocks_in_khz[0] * memory_type_multiplier, 1000);
+		dc->bw_vbios->mid_yclk = bw_frc_to_fixed(
+			clks.clocks_in_khz[clks.num_levels>>1] * memory_type_multiplier,
+			1000);
+		dc->bw_vbios->high_yclk = bw_frc_to_fixed(
+			clks.clocks_in_khz[clks.num_levels-1] * memory_type_multiplier,
+			1000);
+
+		return;
+	}
+
+	/* convert all the clock fro kHz to fix point mHz  TODO: wloop data */
+	dc->bw_vbios->high_sclk = bw_frc_to_fixed(
+		eng_clks.data[eng_clks.num_levels-1].clocks_in_khz, 1000);
+	dc->bw_vbios->mid1_sclk  = bw_frc_to_fixed(
+		eng_clks.data[eng_clks.num_levels/8].clocks_in_khz, 1000);
+	dc->bw_vbios->mid2_sclk  = bw_frc_to_fixed(
+		eng_clks.data[eng_clks.num_levels*2/8].clocks_in_khz, 1000);
+	dc->bw_vbios->mid3_sclk  = bw_frc_to_fixed(
+		eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz, 1000);
+	dc->bw_vbios->mid4_sclk  = bw_frc_to_fixed(
+		eng_clks.data[eng_clks.num_levels*4/8].clocks_in_khz, 1000);
+	dc->bw_vbios->mid5_sclk  = bw_frc_to_fixed(
+		eng_clks.data[eng_clks.num_levels*5/8].clocks_in_khz, 1000);
+	dc->bw_vbios->mid6_sclk  = bw_frc_to_fixed(
+		eng_clks.data[eng_clks.num_levels*6/8].clocks_in_khz, 1000);
+	dc->bw_vbios->low_sclk  = bw_frc_to_fixed(
+			eng_clks.data[0].clocks_in_khz, 1000);
+
+	/*do memory clock*/
+	dm_pp_get_clock_levels_by_type_with_latency(
+			dc->ctx,
+			DM_PP_CLOCK_TYPE_MEMORY_CLK,
+			&mem_clks);
+
+	/* we don't need to call PPLIB for validation clock since they
+	 * also give us the highest sclk and highest mclk (UMA clock).
+	 * ALSO always convert UMA clock (from PPLIB)  to YCLK (HW formula):
+	 * YCLK = UMACLK*m_memoryTypeMultiplier
+	 */
+	dc->bw_vbios->low_yclk = bw_frc_to_fixed(
+		mem_clks.data[0].clocks_in_khz * memory_type_multiplier, 1000);
+	dc->bw_vbios->mid_yclk = bw_frc_to_fixed(
+		mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz * memory_type_multiplier,
+		1000);
+	dc->bw_vbios->high_yclk = bw_frc_to_fixed(
+		mem_clks.data[mem_clks.num_levels-1].clocks_in_khz * memory_type_multiplier,
+		1000);
+
+	/* Now notify PPLib/SMU about which Watermarks sets they should select
+	 * depending on DPM state they are in. And update BW MGR GFX Engine and
+	 * Memory clock member variables for Watermarks calculations for each
+	 * Watermark Set
+	 */
+	clk_ranges.num_wm_sets = 4;
+	clk_ranges.wm_clk_ranges[0].wm_set_id = WM_SET_A;
+	clk_ranges.wm_clk_ranges[0].wm_min_eng_clk_in_khz =
+			eng_clks.data[0].clocks_in_khz;
+	clk_ranges.wm_clk_ranges[0].wm_max_eng_clk_in_khz =
+			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz - 1;
+	clk_ranges.wm_clk_ranges[0].wm_min_mem_clk_in_khz =
+			mem_clks.data[0].clocks_in_khz;
+	clk_ranges.wm_clk_ranges[0].wm_max_mem_clk_in_khz =
+			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz - 1;
+
+	clk_ranges.wm_clk_ranges[1].wm_set_id = WM_SET_B;
+	clk_ranges.wm_clk_ranges[1].wm_min_eng_clk_in_khz =
+			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz;
+	/* 5 GHz instead of data[7].clockInKHz to cover Overdrive */
+	clk_ranges.wm_clk_ranges[1].wm_max_eng_clk_in_khz = 5000000;
+	clk_ranges.wm_clk_ranges[1].wm_min_mem_clk_in_khz =
+			mem_clks.data[0].clocks_in_khz;
+	clk_ranges.wm_clk_ranges[1].wm_max_mem_clk_in_khz =
+			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz - 1;
+
+	clk_ranges.wm_clk_ranges[2].wm_set_id = WM_SET_C;
+	clk_ranges.wm_clk_ranges[2].wm_min_eng_clk_in_khz =
+			eng_clks.data[0].clocks_in_khz;
+	clk_ranges.wm_clk_ranges[2].wm_max_eng_clk_in_khz =
+			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz - 1;
+	clk_ranges.wm_clk_ranges[2].wm_min_mem_clk_in_khz =
+			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz;
+	/* 5 GHz instead of data[2].clockInKHz to cover Overdrive */
+	clk_ranges.wm_clk_ranges[2].wm_max_mem_clk_in_khz = 5000000;
+
+	clk_ranges.wm_clk_ranges[3].wm_set_id = WM_SET_D;
+	clk_ranges.wm_clk_ranges[3].wm_min_eng_clk_in_khz =
+			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz;
+	/* 5 GHz instead of data[7].clockInKHz to cover Overdrive */
+	clk_ranges.wm_clk_ranges[3].wm_max_eng_clk_in_khz = 5000000;
+	clk_ranges.wm_clk_ranges[3].wm_min_mem_clk_in_khz =
+			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz;
+	/* 5 GHz instead of data[2].clockInKHz to cover Overdrive */
+	clk_ranges.wm_clk_ranges[3].wm_max_mem_clk_in_khz = 5000000;
+
+	/* Notify PP Lib/SMU which Watermarks to use for which clock ranges */
+	dm_pp_notify_wm_clock_changes(dc->ctx, &clk_ranges);
+}
+
+static const struct resource_caps *dce112_resource_cap(
+	struct hw_asic_id *asic_id)
+{
+	if (ASIC_REV_IS_POLARIS11_M(asic_id->hw_internal_rev) ||
+	    ASIC_REV_IS_POLARIS12_V(asic_id->hw_internal_rev))
+		return &polaris_11_resource_cap;
+	else
+		return &polaris_10_resource_cap;
+}
+
+static bool dce112_resource_construct(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct dce110_resource_pool *pool)
+{
+	unsigned int i;
+	struct dc_context *ctx = dc->ctx;
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	pool->base.res_cap = dce112_resource_cap(&ctx->asic_id);
+	pool->base.funcs = &dce112_res_pool_funcs;
+
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
+	pool->base.timing_generator_count = pool->base.res_cap->num_timing_generator;
+	dc->caps.max_downscale_ratio = 200;
+	dc->caps.i2c_speed_in_khz = 100;
+	dc->caps.i2c_speed_in_khz_hdcp = 100; /*1.4 w/a not applied by default*/
+	dc->caps.max_cursor_size = 128;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.dual_link_dvi = true;
+	dc->caps.extended_aux_timeout_support = false;
+	dc->debug = debug_defaults;
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	pool->base.clock_sources[DCE112_CLK_SRC_PLL0] =
+			dce112_clock_source_create(
+				ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL0,
+				&clk_src_regs[0], false);
+	pool->base.clock_sources[DCE112_CLK_SRC_PLL1] =
+			dce112_clock_source_create(
+				ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL1,
+				&clk_src_regs[1], false);
+	pool->base.clock_sources[DCE112_CLK_SRC_PLL2] =
+			dce112_clock_source_create(
+				ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL2,
+				&clk_src_regs[2], false);
+	pool->base.clock_sources[DCE112_CLK_SRC_PLL3] =
+			dce112_clock_source_create(
+				ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL3,
+				&clk_src_regs[3], false);
+	pool->base.clock_sources[DCE112_CLK_SRC_PLL4] =
+			dce112_clock_source_create(
+				ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL4,
+				&clk_src_regs[4], false);
+	pool->base.clock_sources[DCE112_CLK_SRC_PLL5] =
+			dce112_clock_source_create(
+				ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL5,
+				&clk_src_regs[5], false);
+	pool->base.clk_src_count = DCE112_CLK_SRC_TOTAL;
+
+	pool->base.dp_clock_source =  dce112_clock_source_create(
+		ctx, ctx->dc_bios,
+		CLOCK_SOURCE_ID_DP_DTO, &clk_src_regs[0], true);
+
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto res_create_fail;
+		}
+	}
+
+	pool->base.dmcu = dce_dmcu_create(ctx,
+			&dmcu_regs,
+			&dmcu_shift,
+			&dmcu_mask);
+	if (pool->base.dmcu == NULL) {
+		dm_error("DC: failed to create dmcu!\n");
+		BREAK_TO_DEBUGGER();
+		goto res_create_fail;
+	}
+
+	pool->base.abm = dce_abm_create(ctx,
+			&abm_regs,
+			&abm_shift,
+			&abm_mask);
+	if (pool->base.abm == NULL) {
+		dm_error("DC: failed to create abm!\n");
+		BREAK_TO_DEBUGGER();
+		goto res_create_fail;
+	}
+
+	{
+		struct irq_service_init_data init_data;
+		init_data.ctx = dc->ctx;
+		pool->base.irqs = dal_irq_service_dce110_create(&init_data);
+		if (!pool->base.irqs)
+			goto res_create_fail;
+	}
+
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		pool->base.timing_generators[i] =
+				dce112_timing_generator_create(
+					ctx,
+					i,
+					&dce112_tg_offsets[i]);
+		if (pool->base.timing_generators[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.mis[i] = dce112_mem_input_create(ctx, i);
+		if (pool->base.mis[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create memory input!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.ipps[i] = dce112_ipp_create(ctx, i);
+		if (pool->base.ipps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create input pixel processor!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.transforms[i] = dce112_transform_create(ctx, i);
+		if (pool->base.transforms[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create transform!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.opps[i] = dce112_opp_create(
+			ctx,
+			i);
+		if (pool->base.opps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create output pixel processor!\n");
+			goto res_create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.engines[i] = dce112_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto res_create_fail;
+		}
+		pool->base.hw_i2cs[i] = dce112_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create i2c engine!!\n");
+			goto res_create_fail;
+		}
+		pool->base.sw_i2cs[i] = NULL;
+	}
+
+	if (!resource_construct(num_virtual_links, dc, &pool->base,
+			  &res_create_funcs))
+		goto res_create_fail;
+
+	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	/* Create hardware sequencer */
+	dce112_hw_sequencer_construct(dc);
+
+	bw_calcs_init(dc->bw_dceip, dc->bw_vbios, dc->ctx->asic_id);
+
+	bw_calcs_data_update_from_pplib(dc);
+
+	return true;
+
+res_create_fail:
+	dce112_resource_destruct(pool);
+	return false;
+}
+
+struct resource_pool *dce112_create_resource_pool(
+	uint8_t num_virtual_links,
+	struct dc *dc)
+{
+	struct dce110_resource_pool *pool =
+		kzalloc(sizeof(struct dce110_resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dce112_resource_construct(num_virtual_links, dc, pool))
+		return &pool->base;
+
+	kfree(pool);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.h
new file mode 100644
index 000000000000..1f57ebc6f9b4
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.h
@@ -0,0 +1,57 @@
+/*
+* Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_RESOURCE_DCE112_H__
+#define __DC_RESOURCE_DCE112_H__
+
+#include "core_types.h"
+
+struct dc;
+struct resource_pool;
+
+struct resource_pool *dce112_create_resource_pool(
+	uint8_t num_virtual_links,
+	struct dc *dc);
+
+enum dc_status dce112_validate_with_context(
+		struct dc *dc,
+		const struct dc_validation_set set[],
+		int set_count,
+		struct dc_state *context,
+		struct dc_state *old_context);
+
+bool dce112_validate_bandwidth(
+	struct dc *dc,
+	struct dc_state *context,
+	bool fast_validate);
+
+enum dc_status dce112_add_stream_to_ctx(
+		struct dc *dc,
+		struct dc_state *new_ctx,
+		struct dc_stream_state *dc_stream);
+
+
+#endif /* __DC_RESOURCE_DCE112_H__ */
+
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.c
new file mode 100644
index 000000000000..20662edd0ae4
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.c
@@ -0,0 +1,1288 @@
+/*
+* Copyright 2012-15 Advanced Micro Devices, Inc.cls
+*
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dm_services.h"
+
+
+#include "stream_encoder.h"
+#include "resource.h"
+#include "include/irq_service_interface.h"
+#include "dce120_resource.h"
+
+#include "dce112/dce112_resource.h"
+
+#include "dce110/dce110_resource.h"
+#include "virtual/virtual_stream_encoder.h"
+#include "dce120/dce120_timing_generator.h"
+#include "irq/dce120/irq_service_dce120.h"
+#include "dce/dce_opp.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_ipp.h"
+#include "dce/dce_mem_input.h"
+#include "dce/dce_panel_cntl.h"
+
+#include "dce110/dce110_hwseq.h"
+#include "dce120/dce120_hwseq.h"
+#include "dce/dce_transform.h"
+#include "clk_mgr.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_link_encoder.h"
+#include "dce/dce_stream_encoder.h"
+#include "dce/dce_hwseq.h"
+#include "dce/dce_abm.h"
+#include "dce/dce_dmcu.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_i2c.h"
+
+#include "dce/dce_12_0_offset.h"
+#include "dce/dce_12_0_sh_mask.h"
+#include "soc15_hw_ip.h"
+#include "vega10_ip_offset.h"
+#include "nbio/nbio_6_1_offset.h"
+#include "mmhub/mmhub_1_0_offset.h"
+#include "mmhub/mmhub_1_0_sh_mask.h"
+#include "reg_helper.h"
+
+#include "dce100/dce100_resource.h"
+
+#ifndef mmDP0_DP_DPHY_INTERNAL_CTRL
+	#define mmDP0_DP_DPHY_INTERNAL_CTRL		0x210f
+	#define mmDP0_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
+	#define mmDP1_DP_DPHY_INTERNAL_CTRL		0x220f
+	#define mmDP1_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
+	#define mmDP2_DP_DPHY_INTERNAL_CTRL		0x230f
+	#define mmDP2_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
+	#define mmDP3_DP_DPHY_INTERNAL_CTRL		0x240f
+	#define mmDP3_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
+	#define mmDP4_DP_DPHY_INTERNAL_CTRL		0x250f
+	#define mmDP4_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
+	#define mmDP5_DP_DPHY_INTERNAL_CTRL		0x260f
+	#define mmDP5_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
+	#define mmDP6_DP_DPHY_INTERNAL_CTRL		0x270f
+	#define mmDP6_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
+#endif
+
+enum dce120_clk_src_array_id {
+	DCE120_CLK_SRC_PLL0,
+	DCE120_CLK_SRC_PLL1,
+	DCE120_CLK_SRC_PLL2,
+	DCE120_CLK_SRC_PLL3,
+	DCE120_CLK_SRC_PLL4,
+	DCE120_CLK_SRC_PLL5,
+
+	DCE120_CLK_SRC_TOTAL
+};
+
+static const struct dce110_timing_generator_offsets dce120_tg_offsets[] = {
+	{
+		.crtc = (mmCRTC0_CRTC_CONTROL - mmCRTC0_CRTC_CONTROL),
+	},
+	{
+		.crtc = (mmCRTC1_CRTC_CONTROL - mmCRTC0_CRTC_CONTROL),
+	},
+	{
+		.crtc = (mmCRTC2_CRTC_CONTROL - mmCRTC0_CRTC_CONTROL),
+	},
+	{
+		.crtc = (mmCRTC3_CRTC_CONTROL - mmCRTC0_CRTC_CONTROL),
+	},
+	{
+		.crtc = (mmCRTC4_CRTC_CONTROL - mmCRTC0_CRTC_CONTROL),
+	},
+	{
+		.crtc = (mmCRTC5_CRTC_CONTROL - mmCRTC0_CRTC_CONTROL),
+	}
+};
+
+/* begin *********************
+ * macros to expend register list macro defined in HW object header file */
+
+#define BASE_INNER(seg) \
+	DCE_BASE__INST0_SEG ## seg
+
+#define NBIO_BASE_INNER(seg) \
+	NBIF_BASE__INST0_SEG ## seg
+
+#define NBIO_BASE(seg) \
+	NBIO_BASE_INNER(seg)
+
+/* compile time expand base address. */
+#define BASE(seg) \
+	BASE_INNER(seg)
+
+#define SR(reg_name)\
+		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) +  \
+					mm ## reg_name
+
+#define SRI(reg_name, block, id)\
+	.reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+/* MMHUB */
+#define MMHUB_BASE_INNER(seg) \
+	MMHUB_BASE__INST0_SEG ## seg
+
+#define MMHUB_BASE(seg) \
+	MMHUB_BASE_INNER(seg)
+
+#define MMHUB_SR(reg_name)\
+		.reg_name = MMHUB_BASE(mm ## reg_name ## _BASE_IDX) +  \
+					mm ## reg_name
+
+/* macros to expend register list macro defined in HW object header file
+ * end *********************/
+
+
+static const struct dce_dmcu_registers dmcu_regs = {
+		DMCU_DCE110_COMMON_REG_LIST()
+};
+
+static const struct dce_dmcu_shift dmcu_shift = {
+		DMCU_MASK_SH_LIST_DCE110(__SHIFT)
+};
+
+static const struct dce_dmcu_mask dmcu_mask = {
+		DMCU_MASK_SH_LIST_DCE110(_MASK)
+};
+
+static const struct dce_abm_registers abm_regs = {
+		ABM_DCE110_COMMON_REG_LIST()
+};
+
+static const struct dce_abm_shift abm_shift = {
+		ABM_MASK_SH_LIST_DCE110(__SHIFT)
+};
+
+static const struct dce_abm_mask abm_mask = {
+		ABM_MASK_SH_LIST_DCE110(_MASK)
+};
+
+#define ipp_regs(id)\
+[id] = {\
+		IPP_DCE110_REG_LIST_DCE_BASE(id)\
+}
+
+static const struct dce_ipp_registers ipp_regs[] = {
+		ipp_regs(0),
+		ipp_regs(1),
+		ipp_regs(2),
+		ipp_regs(3),
+		ipp_regs(4),
+		ipp_regs(5)
+};
+
+static const struct dce_ipp_shift ipp_shift = {
+		IPP_DCE120_MASK_SH_LIST_SOC_BASE(__SHIFT)
+};
+
+static const struct dce_ipp_mask ipp_mask = {
+		IPP_DCE120_MASK_SH_LIST_SOC_BASE(_MASK)
+};
+
+#define transform_regs(id)\
+[id] = {\
+		XFM_COMMON_REG_LIST_DCE110(id)\
+}
+
+static const struct dce_transform_registers xfm_regs[] = {
+		transform_regs(0),
+		transform_regs(1),
+		transform_regs(2),
+		transform_regs(3),
+		transform_regs(4),
+		transform_regs(5)
+};
+
+static const struct dce_transform_shift xfm_shift = {
+		XFM_COMMON_MASK_SH_LIST_SOC_BASE(__SHIFT)
+};
+
+static const struct dce_transform_mask xfm_mask = {
+		XFM_COMMON_MASK_SH_LIST_SOC_BASE(_MASK)
+};
+
+#define aux_regs(id)\
+[id] = {\
+	AUX_REG_LIST(id)\
+}
+
+static const struct dce110_link_enc_aux_registers link_enc_aux_regs[] = {
+		aux_regs(0),
+		aux_regs(1),
+		aux_regs(2),
+		aux_regs(3),
+		aux_regs(4),
+		aux_regs(5)
+};
+
+#define hpd_regs(id)\
+[id] = {\
+	HPD_REG_LIST(id)\
+}
+
+static const struct dce110_link_enc_hpd_registers link_enc_hpd_regs[] = {
+		hpd_regs(0),
+		hpd_regs(1),
+		hpd_regs(2),
+		hpd_regs(3),
+		hpd_regs(4),
+		hpd_regs(5)
+};
+
+#define link_regs(id)\
+[id] = {\
+	LE_DCE120_REG_LIST(id), \
+	SRI(DP_DPHY_INTERNAL_CTRL, DP, id) \
+}
+
+static const struct dce110_link_enc_registers link_enc_regs[] = {
+	link_regs(0),
+	link_regs(1),
+	link_regs(2),
+	link_regs(3),
+	link_regs(4),
+	link_regs(5),
+	link_regs(6),
+};
+
+
+#define stream_enc_regs(id)\
+[id] = {\
+	SE_COMMON_REG_LIST(id),\
+	.TMDS_CNTL = 0,\
+}
+
+static const struct dce110_stream_enc_registers stream_enc_regs[] = {
+	stream_enc_regs(0),
+	stream_enc_regs(1),
+	stream_enc_regs(2),
+	stream_enc_regs(3),
+	stream_enc_regs(4),
+	stream_enc_regs(5)
+};
+
+static const struct dce_stream_encoder_shift se_shift = {
+		SE_COMMON_MASK_SH_LIST_DCE120(__SHIFT)
+};
+
+static const struct dce_stream_encoder_mask se_mask = {
+		SE_COMMON_MASK_SH_LIST_DCE120(_MASK)
+};
+
+static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
+	{ DCE_PANEL_CNTL_REG_LIST() }
+};
+
+static const struct dce_panel_cntl_shift panel_cntl_shift = {
+	DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_panel_cntl_mask panel_cntl_mask = {
+	DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
+};
+
+static const struct dce110_aux_registers_shift aux_shift = {
+	DCE12_AUX_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+	DCE12_AUX_MASK_SH_LIST(_MASK)
+};
+
+#define opp_regs(id)\
+[id] = {\
+	OPP_DCE_120_REG_LIST(id),\
+}
+
+static const struct dce_opp_registers opp_regs[] = {
+	opp_regs(0),
+	opp_regs(1),
+	opp_regs(2),
+	opp_regs(3),
+	opp_regs(4),
+	opp_regs(5)
+};
+
+static const struct dce_opp_shift opp_shift = {
+	OPP_COMMON_MASK_SH_LIST_DCE_120(__SHIFT)
+};
+
+static const struct dce_opp_mask opp_mask = {
+	OPP_COMMON_MASK_SH_LIST_DCE_120(_MASK)
+};
+ #define aux_engine_regs(id)\
+[id] = {\
+	AUX_COMMON_REG_LIST(id), \
+	.AUX_RESET_MASK = 0 \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1),
+		aux_engine_regs(2),
+		aux_engine_regs(3),
+		aux_engine_regs(4),
+		aux_engine_regs(5)
+};
+
+#define audio_regs(id)\
+[id] = {\
+	AUD_COMMON_REG_LIST(id)\
+}
+
+static const struct dce_audio_registers audio_regs[] = {
+	audio_regs(0),
+	audio_regs(1),
+	audio_regs(2),
+	audio_regs(3),
+	audio_regs(4),
+	audio_regs(5),
+	audio_regs(6),
+};
+
+#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
+		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
+
+static const struct dce_audio_shift audio_shift = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_audio_mask audio_mask = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+static int map_transmitter_id_to_phy_instance(
+	enum transmitter transmitter)
+{
+	switch (transmitter) {
+	case TRANSMITTER_UNIPHY_A:
+		return 0;
+	case TRANSMITTER_UNIPHY_B:
+		return 1;
+	case TRANSMITTER_UNIPHY_C:
+		return 2;
+	case TRANSMITTER_UNIPHY_D:
+		return 3;
+	case TRANSMITTER_UNIPHY_E:
+		return 4;
+	case TRANSMITTER_UNIPHY_F:
+		return 5;
+	case TRANSMITTER_UNIPHY_G:
+		return 6;
+	default:
+		ASSERT(0);
+		return 0;
+	}
+}
+
+#define clk_src_regs(index, id)\
+[index] = {\
+	CS_COMMON_REG_LIST_DCE_112(id),\
+}
+
+static const struct dce110_clk_src_regs clk_src_regs[] = {
+	clk_src_regs(0, A),
+	clk_src_regs(1, B),
+	clk_src_regs(2, C),
+	clk_src_regs(3, D),
+	clk_src_regs(4, E),
+	clk_src_regs(5, F)
+};
+
+static const struct dce110_clk_src_shift cs_shift = {
+		CS_COMMON_MASK_SH_LIST_DCE_112(__SHIFT)
+};
+
+static const struct dce110_clk_src_mask cs_mask = {
+		CS_COMMON_MASK_SH_LIST_DCE_112(_MASK)
+};
+
+static struct output_pixel_processor *dce120_opp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce110_opp *opp =
+		kzalloc(sizeof(struct dce110_opp), GFP_KERNEL);
+
+	if (!opp)
+		return NULL;
+
+	dce110_opp_construct(opp,
+			     ctx, inst, &opp_regs[inst], &opp_shift, &opp_mask);
+	return &opp->base;
+}
+static struct dce_aux *dce120_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst],
+					&aux_mask,
+					&aux_shift,
+					ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
+
+static const struct dce_i2c_registers i2c_hw_regs[] = {
+		i2c_inst_regs(1),
+		i2c_inst_regs(2),
+		i2c_inst_regs(3),
+		i2c_inst_regs(4),
+		i2c_inst_regs(5),
+		i2c_inst_regs(6),
+};
+
+static const struct dce_i2c_shift i2c_shifts = {
+		I2C_COMMON_MASK_SH_LIST_DCE110(__SHIFT)
+};
+
+static const struct dce_i2c_mask i2c_masks = {
+		I2C_COMMON_MASK_SH_LIST_DCE110(_MASK)
+};
+
+static struct dce_i2c_hw *dce120_i2c_hw_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw =
+		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+	dce112_i2c_hw_construct(dce_i2c_hw, ctx, inst,
+				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+static const struct bios_registers bios_regs = {
+	.BIOS_SCRATCH_3 = mmBIOS_SCRATCH_3 + NBIO_BASE(mmBIOS_SCRATCH_3_BASE_IDX),
+	.BIOS_SCRATCH_6 = mmBIOS_SCRATCH_6 + NBIO_BASE(mmBIOS_SCRATCH_6_BASE_IDX)
+};
+
+static const struct resource_caps res_cap = {
+		.num_timing_generator = 6,
+		.num_audio = 7,
+		.num_stream_encoder = 6,
+		.num_pll = 6,
+		.num_ddc = 6,
+};
+
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCE_RGB,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = false,
+			.fp16 = true
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 1,
+			.fp16 = 1
+	},
+
+	.max_downscale_factor = {
+			.argb8888 = 250,
+			.nv12 = 1,
+			.fp16 = 1
+	}
+};
+
+static const struct dc_debug_options debug_defaults = {
+		.disable_clock_gate = true,
+		.enable_legacy_fast_update = true,
+};
+
+static struct clock_source *dce120_clock_source_create(
+	struct dc_context *ctx,
+	struct dc_bios *bios,
+	enum clock_source_id id,
+	const struct dce110_clk_src_regs *regs,
+	bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(*clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dce112_clk_src_construct(clk_src, ctx, bios, id,
+				     regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	kfree(clk_src);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static void dce120_clock_source_destroy(struct clock_source **clk_src)
+{
+	kfree(TO_DCE110_CLK_SRC(*clk_src));
+	*clk_src = NULL;
+}
+
+
+static bool dce120_hw_sequencer_create(struct dc *dc)
+{
+	/* All registers used by dce11.2 match those in dce11 in offset and
+	 * structure
+	 */
+	dce120_hw_sequencer_construct(dc);
+
+	/*TODO	Move to separate file and Override what is needed */
+
+	return true;
+}
+
+static struct timing_generator *dce120_timing_generator_create(
+		struct dc_context *ctx,
+		uint32_t instance,
+		const struct dce110_timing_generator_offsets *offsets)
+{
+	struct dce110_timing_generator *tg110 =
+		kzalloc(sizeof(struct dce110_timing_generator), GFP_KERNEL);
+
+	if (!tg110)
+		return NULL;
+
+	dce120_timing_generator_construct(tg110, ctx, instance, offsets);
+	return &tg110->base;
+}
+
+static void dce120_transform_destroy(struct transform **xfm)
+{
+	kfree(TO_DCE_TRANSFORM(*xfm));
+	*xfm = NULL;
+}
+
+static void dce120_resource_destruct(struct dce110_resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (pool->base.opps[i] != NULL)
+			dce110_opp_destroy(&pool->base.opps[i]);
+
+		if (pool->base.transforms[i] != NULL)
+			dce120_transform_destroy(&pool->base.transforms[i]);
+
+		if (pool->base.ipps[i] != NULL)
+			dce_ipp_destroy(&pool->base.ipps[i]);
+
+		if (pool->base.mis[i] != NULL) {
+			kfree(TO_DCE_MEM_INPUT(pool->base.mis[i]));
+			pool->base.mis[i] = NULL;
+		}
+
+		if (pool->base.irqs != NULL) {
+			dal_irq_service_destroy(&pool->base.irqs);
+		}
+
+		if (pool->base.timing_generators[i] != NULL) {
+			kfree(DCE110TG_FROM_TG(pool->base.timing_generators[i]));
+			pool->base.timing_generators[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+		if (pool->base.hw_i2cs[i] != NULL) {
+			kfree(pool->base.hw_i2cs[i]);
+			pool->base.hw_i2cs[i] = NULL;
+		}
+		if (pool->base.sw_i2cs[i] != NULL) {
+			kfree(pool->base.sw_i2cs[i]);
+			pool->base.sw_i2cs[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.audio_count; i++) {
+		if (pool->base.audios[i])
+			dce_aud_destroy(&pool->base.audios[i]);
+	}
+
+	for (i = 0; i < pool->base.stream_enc_count; i++) {
+		if (pool->base.stream_enc[i] != NULL)
+			kfree(DCE110STRENC_FROM_STRENC(pool->base.stream_enc[i]));
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] != NULL)
+			dce120_clock_source_destroy(
+				&pool->base.clock_sources[i]);
+	}
+
+	if (pool->base.dp_clock_source != NULL)
+		dce120_clock_source_destroy(&pool->base.dp_clock_source);
+
+	if (pool->base.abm != NULL)
+		dce_abm_destroy(&pool->base.abm);
+
+	if (pool->base.dmcu != NULL)
+		dce_dmcu_destroy(&pool->base.dmcu);
+}
+
+static void read_dce_straps(
+	struct dc_context *ctx,
+	struct resource_straps *straps)
+{
+	uint32_t reg_val = dm_read_reg_soc15(ctx, mmCC_DC_MISC_STRAPS, 0);
+
+	straps->audio_stream_number = get_reg_field_value(reg_val,
+							  CC_DC_MISC_STRAPS,
+							  AUDIO_STREAM_NUMBER);
+	straps->hdmi_disable = get_reg_field_value(reg_val,
+						   CC_DC_MISC_STRAPS,
+						   HDMI_DISABLE);
+
+	reg_val = dm_read_reg_soc15(ctx, mmDC_PINSTRAPS, 0);
+	straps->dc_pinstraps_audio = get_reg_field_value(reg_val,
+							 DC_PINSTRAPS,
+							 DC_PINSTRAPS_AUDIO);
+}
+
+static struct audio *create_audio(
+		struct dc_context *ctx, unsigned int inst)
+{
+	return dce_audio_create(ctx, inst,
+			&audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+		.max_hdmi_deep_color = COLOR_DEPTH_121212,
+		.max_hdmi_pixel_clock = 600000,
+		.hdmi_ycbcr420_supported = true,
+		.dp_ycbcr420_supported = false,
+		.flags.bits.IS_HBR2_CAPABLE = true,
+		.flags.bits.IS_HBR3_CAPABLE = true,
+		.flags.bits.IS_TPS3_CAPABLE = true,
+		.flags.bits.IS_TPS4_CAPABLE = true,
+};
+
+static struct link_encoder *dce120_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dce110_link_encoder *enc110 =
+		kzalloc(sizeof(struct dce110_link_encoder), GFP_KERNEL);
+	int link_regs_id;
+
+	if (!enc110)
+		return NULL;
+
+	link_regs_id =
+		map_transmitter_id_to_phy_instance(enc_init_data->transmitter);
+
+	dce110_link_encoder_construct(enc110,
+				      enc_init_data,
+				      &link_enc_feature,
+				      &link_enc_regs[link_regs_id],
+				      &link_enc_aux_regs[enc_init_data->channel - 1],
+				      &link_enc_hpd_regs[enc_init_data->hpd_source]);
+
+	return &enc110->base;
+}
+
+static struct panel_cntl *dce120_panel_cntl_create(const struct panel_cntl_init_data *init_data)
+{
+	struct dce_panel_cntl *panel_cntl =
+		kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
+
+	if (!panel_cntl)
+		return NULL;
+
+	dce_panel_cntl_construct(panel_cntl,
+			init_data,
+			&panel_cntl_regs[init_data->inst],
+			&panel_cntl_shift,
+			&panel_cntl_mask);
+
+	return &panel_cntl->base;
+}
+
+static struct input_pixel_processor *dce120_ipp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dce_ipp *ipp = kzalloc(sizeof(struct dce_ipp), GFP_KERNEL);
+
+	if (!ipp) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dce_ipp_construct(ipp, ctx, inst,
+			&ipp_regs[inst], &ipp_shift, &ipp_mask);
+	return &ipp->base;
+}
+
+static struct stream_encoder *dce120_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dce110_stream_encoder *enc110 =
+		kzalloc(sizeof(struct dce110_stream_encoder), GFP_KERNEL);
+
+	if (!enc110)
+		return NULL;
+
+	dce110_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id,
+					&stream_enc_regs[eng_id],
+					&se_shift, &se_mask);
+	return &enc110->base;
+}
+
+#define SRII(reg_name, block, id)\
+	.reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+static const struct dce_hwseq_registers hwseq_reg = {
+		HWSEQ_DCE120_REG_LIST()
+};
+
+static const struct dce_hwseq_shift hwseq_shift = {
+		HWSEQ_DCE12_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+		HWSEQ_DCE12_MASK_SH_LIST(_MASK)
+};
+
+/* HWSEQ regs for VG20 */
+static const struct dce_hwseq_registers dce121_hwseq_reg = {
+		HWSEQ_VG20_REG_LIST()
+};
+
+static const struct dce_hwseq_shift dce121_hwseq_shift = {
+		HWSEQ_VG20_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_hwseq_mask dce121_hwseq_mask = {
+		HWSEQ_VG20_MASK_SH_LIST(_MASK)
+};
+
+static struct dce_hwseq *dce120_hwseq_create(
+	struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &hwseq_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+	}
+	return hws;
+}
+
+static struct dce_hwseq *dce121_hwseq_create(
+	struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &dce121_hwseq_reg;
+		hws->shifts = &dce121_hwseq_shift;
+		hws->masks = &dce121_hwseq_mask;
+	}
+	return hws;
+}
+
+static const struct resource_create_funcs res_create_funcs = {
+	.read_dce_straps = read_dce_straps,
+	.create_audio = create_audio,
+	.create_stream_encoder = dce120_stream_encoder_create,
+	.create_hwseq = dce120_hwseq_create,
+};
+
+static const struct resource_create_funcs dce121_res_create_funcs = {
+	.read_dce_straps = read_dce_straps,
+	.create_audio = create_audio,
+	.create_stream_encoder = dce120_stream_encoder_create,
+	.create_hwseq = dce121_hwseq_create,
+};
+
+
+#define mi_inst_regs(id) { MI_DCE12_REG_LIST(id) }
+static const struct dce_mem_input_registers mi_regs[] = {
+		mi_inst_regs(0),
+		mi_inst_regs(1),
+		mi_inst_regs(2),
+		mi_inst_regs(3),
+		mi_inst_regs(4),
+		mi_inst_regs(5),
+};
+
+static const struct dce_mem_input_shift mi_shifts = {
+		MI_DCE12_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_mem_input_mask mi_masks = {
+		MI_DCE12_MASK_SH_LIST(_MASK)
+};
+
+static struct mem_input *dce120_mem_input_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_mem_input *dce_mi = kzalloc(sizeof(struct dce_mem_input),
+					       GFP_KERNEL);
+
+	if (!dce_mi) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dce120_mem_input_construct(dce_mi, ctx, inst, &mi_regs[inst], &mi_shifts, &mi_masks);
+	return &dce_mi->base;
+}
+
+static struct transform *dce120_transform_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_transform *transform =
+		kzalloc(sizeof(struct dce_transform), GFP_KERNEL);
+
+	if (!transform)
+		return NULL;
+
+	dce_transform_construct(transform, ctx, inst,
+				&xfm_regs[inst], &xfm_shift, &xfm_mask);
+	transform->lb_memory_size = 0x1404; /*5124*/
+	return &transform->base;
+}
+
+static void dce120_destroy_resource_pool(struct resource_pool **pool)
+{
+	struct dce110_resource_pool *dce110_pool = TO_DCE110_RES_POOL(*pool);
+
+	dce120_resource_destruct(dce110_pool);
+	kfree(dce110_pool);
+	*pool = NULL;
+}
+
+static const struct resource_funcs dce120_res_pool_funcs = {
+	.destroy = dce120_destroy_resource_pool,
+	.link_enc_create = dce120_link_encoder_create,
+	.panel_cntl_create = dce120_panel_cntl_create,
+	.validate_bandwidth = dce112_validate_bandwidth,
+	.validate_plane = dce100_validate_plane,
+	.add_stream_to_ctx = dce112_add_stream_to_ctx,
+	.find_first_free_match_stream_enc_for_link = dce110_find_first_free_match_stream_enc_for_link
+};
+
+static void bw_calcs_data_update_from_pplib(struct dc *dc)
+{
+	struct dm_pp_clock_levels_with_latency eng_clks = {0};
+	struct dm_pp_clock_levels_with_latency mem_clks = {0};
+	struct dm_pp_wm_sets_with_clock_ranges clk_ranges = {0};
+	int i;
+	unsigned int clk;
+	unsigned int latency;
+	/*original logic in dal3*/
+	int memory_type_multiplier = MEMORY_TYPE_MULTIPLIER_CZ;
+
+	/*do system clock*/
+	if (!dm_pp_get_clock_levels_by_type_with_latency(
+				dc->ctx,
+				DM_PP_CLOCK_TYPE_ENGINE_CLK,
+				&eng_clks) || eng_clks.num_levels == 0) {
+
+		eng_clks.num_levels = 8;
+		clk = 300000;
+
+		for (i = 0; i < eng_clks.num_levels; i++) {
+			eng_clks.data[i].clocks_in_khz = clk;
+			clk += 100000;
+		}
+	}
+
+	/* convert all the clock fro kHz to fix point mHz  TODO: wloop data */
+	dc->bw_vbios->high_sclk = bw_frc_to_fixed(
+		eng_clks.data[eng_clks.num_levels-1].clocks_in_khz, 1000);
+	dc->bw_vbios->mid1_sclk  = bw_frc_to_fixed(
+		eng_clks.data[eng_clks.num_levels/8].clocks_in_khz, 1000);
+	dc->bw_vbios->mid2_sclk  = bw_frc_to_fixed(
+		eng_clks.data[eng_clks.num_levels*2/8].clocks_in_khz, 1000);
+	dc->bw_vbios->mid3_sclk  = bw_frc_to_fixed(
+		eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz, 1000);
+	dc->bw_vbios->mid4_sclk  = bw_frc_to_fixed(
+		eng_clks.data[eng_clks.num_levels*4/8].clocks_in_khz, 1000);
+	dc->bw_vbios->mid5_sclk  = bw_frc_to_fixed(
+		eng_clks.data[eng_clks.num_levels*5/8].clocks_in_khz, 1000);
+	dc->bw_vbios->mid6_sclk  = bw_frc_to_fixed(
+		eng_clks.data[eng_clks.num_levels*6/8].clocks_in_khz, 1000);
+	dc->bw_vbios->low_sclk  = bw_frc_to_fixed(
+			eng_clks.data[0].clocks_in_khz, 1000);
+
+	/*do memory clock*/
+	if (!dm_pp_get_clock_levels_by_type_with_latency(
+			dc->ctx,
+			DM_PP_CLOCK_TYPE_MEMORY_CLK,
+			&mem_clks) || mem_clks.num_levels == 0) {
+
+		mem_clks.num_levels = 3;
+		clk = 250000;
+		latency = 45;
+
+		for (i = 0; i < eng_clks.num_levels; i++) {
+			mem_clks.data[i].clocks_in_khz = clk;
+			mem_clks.data[i].latency_in_us = latency;
+			clk += 500000;
+			latency -= 5;
+		}
+
+	}
+
+	/* we don't need to call PPLIB for validation clock since they
+	 * also give us the highest sclk and highest mclk (UMA clock).
+	 * ALSO always convert UMA clock (from PPLIB)  to YCLK (HW formula):
+	 * YCLK = UMACLK*m_memoryTypeMultiplier
+	 */
+	if (dc->bw_vbios->memory_type == bw_def_hbm)
+		memory_type_multiplier = MEMORY_TYPE_HBM;
+
+	dc->bw_vbios->low_yclk = bw_frc_to_fixed(
+		mem_clks.data[0].clocks_in_khz * memory_type_multiplier, 1000);
+	dc->bw_vbios->mid_yclk = bw_frc_to_fixed(
+		mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz * memory_type_multiplier,
+		1000);
+	dc->bw_vbios->high_yclk = bw_frc_to_fixed(
+		mem_clks.data[mem_clks.num_levels-1].clocks_in_khz * memory_type_multiplier,
+		1000);
+
+	/* Now notify PPLib/SMU about which Watermarks sets they should select
+	 * depending on DPM state they are in. And update BW MGR GFX Engine and
+	 * Memory clock member variables for Watermarks calculations for each
+	 * Watermark Set
+	 */
+	clk_ranges.num_wm_sets = 4;
+	clk_ranges.wm_clk_ranges[0].wm_set_id = WM_SET_A;
+	clk_ranges.wm_clk_ranges[0].wm_min_eng_clk_in_khz =
+			eng_clks.data[0].clocks_in_khz;
+	clk_ranges.wm_clk_ranges[0].wm_max_eng_clk_in_khz =
+			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz - 1;
+	clk_ranges.wm_clk_ranges[0].wm_min_mem_clk_in_khz =
+			mem_clks.data[0].clocks_in_khz;
+	clk_ranges.wm_clk_ranges[0].wm_max_mem_clk_in_khz =
+			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz - 1;
+
+	clk_ranges.wm_clk_ranges[1].wm_set_id = WM_SET_B;
+	clk_ranges.wm_clk_ranges[1].wm_min_eng_clk_in_khz =
+			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz;
+	/* 5 GHz instead of data[7].clockInKHz to cover Overdrive */
+	clk_ranges.wm_clk_ranges[1].wm_max_eng_clk_in_khz = 5000000;
+	clk_ranges.wm_clk_ranges[1].wm_min_mem_clk_in_khz =
+			mem_clks.data[0].clocks_in_khz;
+	clk_ranges.wm_clk_ranges[1].wm_max_mem_clk_in_khz =
+			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz - 1;
+
+	clk_ranges.wm_clk_ranges[2].wm_set_id = WM_SET_C;
+	clk_ranges.wm_clk_ranges[2].wm_min_eng_clk_in_khz =
+			eng_clks.data[0].clocks_in_khz;
+	clk_ranges.wm_clk_ranges[2].wm_max_eng_clk_in_khz =
+			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz - 1;
+	clk_ranges.wm_clk_ranges[2].wm_min_mem_clk_in_khz =
+			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz;
+	/* 5 GHz instead of data[2].clockInKHz to cover Overdrive */
+	clk_ranges.wm_clk_ranges[2].wm_max_mem_clk_in_khz = 5000000;
+
+	clk_ranges.wm_clk_ranges[3].wm_set_id = WM_SET_D;
+	clk_ranges.wm_clk_ranges[3].wm_min_eng_clk_in_khz =
+			eng_clks.data[eng_clks.num_levels*3/8].clocks_in_khz;
+	/* 5 GHz instead of data[7].clockInKHz to cover Overdrive */
+	clk_ranges.wm_clk_ranges[3].wm_max_eng_clk_in_khz = 5000000;
+	clk_ranges.wm_clk_ranges[3].wm_min_mem_clk_in_khz =
+			mem_clks.data[mem_clks.num_levels>>1].clocks_in_khz;
+	/* 5 GHz instead of data[2].clockInKHz to cover Overdrive */
+	clk_ranges.wm_clk_ranges[3].wm_max_mem_clk_in_khz = 5000000;
+
+	/* Notify PP Lib/SMU which Watermarks to use for which clock ranges */
+	dm_pp_notify_wm_clock_changes(dc->ctx, &clk_ranges);
+}
+
+static uint32_t read_pipe_fuses(struct dc_context *ctx)
+{
+	uint32_t value = dm_read_reg_soc15(ctx, mmCC_DC_PIPE_DIS, 0);
+	/* VG20 support max 6 pipes */
+	value = value & 0x3f;
+	return value;
+}
+
+static bool dce120_resource_construct(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct dce110_resource_pool *pool)
+{
+	unsigned int i;
+	int j;
+	struct dc_context *ctx = dc->ctx;
+	struct irq_service_init_data irq_init_data;
+	static const struct resource_create_funcs *res_funcs;
+	bool is_vg20 = ASICREV_IS_VEGA20_P(ctx->asic_id.hw_internal_rev);
+	uint32_t pipe_fuses;
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	pool->base.res_cap = &res_cap;
+	pool->base.funcs = &dce120_res_pool_funcs;
+
+	/* TODO: Fill more data from GreenlandAsicCapability.cpp */
+	pool->base.pipe_count = res_cap.num_timing_generator;
+	pool->base.timing_generator_count = pool->base.res_cap->num_timing_generator;
+	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+
+	dc->caps.max_downscale_ratio = 200;
+	dc->caps.i2c_speed_in_khz = 100;
+	dc->caps.i2c_speed_in_khz_hdcp = 100; /*1.4 w/a not applied by default*/
+	dc->caps.max_cursor_size = 128;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.dual_link_dvi = true;
+	dc->caps.psp_setup_panel_mode = true;
+	dc->caps.extended_aux_timeout_support = false;
+	dc->debug = debug_defaults;
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	pool->base.clock_sources[DCE120_CLK_SRC_PLL0] =
+			dce120_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL0,
+				&clk_src_regs[0], false);
+	pool->base.clock_sources[DCE120_CLK_SRC_PLL1] =
+			dce120_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL1,
+				&clk_src_regs[1], false);
+	pool->base.clock_sources[DCE120_CLK_SRC_PLL2] =
+			dce120_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL2,
+				&clk_src_regs[2], false);
+	pool->base.clock_sources[DCE120_CLK_SRC_PLL3] =
+			dce120_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL3,
+				&clk_src_regs[3], false);
+	pool->base.clock_sources[DCE120_CLK_SRC_PLL4] =
+			dce120_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL4,
+				&clk_src_regs[4], false);
+	pool->base.clock_sources[DCE120_CLK_SRC_PLL5] =
+			dce120_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL5,
+				&clk_src_regs[5], false);
+	pool->base.clk_src_count = DCE120_CLK_SRC_TOTAL;
+
+	pool->base.dp_clock_source =
+			dce120_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_ID_DP_DTO,
+				&clk_src_regs[0], true);
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto clk_src_create_fail;
+		}
+	}
+
+	pool->base.dmcu = dce_dmcu_create(ctx,
+			&dmcu_regs,
+			&dmcu_shift,
+			&dmcu_mask);
+	if (pool->base.dmcu == NULL) {
+		dm_error("DC: failed to create dmcu!\n");
+		BREAK_TO_DEBUGGER();
+		goto res_create_fail;
+	}
+
+	pool->base.abm = dce_abm_create(ctx,
+			&abm_regs,
+			&abm_shift,
+			&abm_mask);
+	if (pool->base.abm == NULL) {
+		dm_error("DC: failed to create abm!\n");
+		BREAK_TO_DEBUGGER();
+		goto res_create_fail;
+	}
+
+
+	irq_init_data.ctx = dc->ctx;
+	pool->base.irqs = dal_irq_service_dce120_create(&irq_init_data);
+	if (!pool->base.irqs)
+		goto irqs_create_fail;
+
+	/* VG20: Pipe harvesting enabled, retrieve valid pipe fuses */
+	if (is_vg20)
+		pipe_fuses = read_pipe_fuses(ctx);
+
+	/* index to valid pipe resource */
+	j = 0;
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (is_vg20) {
+			if ((pipe_fuses & (1 << i)) != 0) {
+				dm_error("DC: skip invalid pipe %d!\n", i);
+				continue;
+			}
+		}
+
+		pool->base.timing_generators[j] =
+				dce120_timing_generator_create(
+					ctx,
+					i,
+					&dce120_tg_offsets[i]);
+		if (pool->base.timing_generators[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto controller_create_fail;
+		}
+
+		pool->base.mis[j] = dce120_mem_input_create(ctx, i);
+
+		if (pool->base.mis[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create memory input!\n");
+			goto controller_create_fail;
+		}
+
+		pool->base.ipps[j] = dce120_ipp_create(ctx, i);
+		if (pool->base.ipps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create input pixel processor!\n");
+			goto controller_create_fail;
+		}
+
+		pool->base.transforms[j] = dce120_transform_create(ctx, i);
+		if (pool->base.transforms[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create transform!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.opps[j] = dce120_opp_create(
+			ctx,
+			i);
+		if (pool->base.opps[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create output pixel processor!\n");
+		}
+
+		/* check next valid pipe */
+		j++;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.engines[i] = dce120_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto res_create_fail;
+		}
+		pool->base.hw_i2cs[i] = dce120_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create i2c engine!!\n");
+			goto res_create_fail;
+		}
+		pool->base.sw_i2cs[i] = NULL;
+	}
+
+	/* valid pipe num */
+	pool->base.pipe_count = j;
+	pool->base.timing_generator_count = j;
+
+	if (is_vg20)
+		res_funcs = &dce121_res_create_funcs;
+	else
+		res_funcs = &res_create_funcs;
+
+	if (!resource_construct(num_virtual_links, dc, &pool->base, res_funcs))
+		goto res_create_fail;
+
+	/* Create hardware sequencer */
+	if (!dce120_hw_sequencer_create(dc))
+		goto controller_create_fail;
+
+	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	bw_calcs_init(dc->bw_dceip, dc->bw_vbios, dc->ctx->asic_id);
+
+	bw_calcs_data_update_from_pplib(dc);
+
+	return true;
+
+irqs_create_fail:
+controller_create_fail:
+clk_src_create_fail:
+res_create_fail:
+
+	dce120_resource_destruct(pool);
+
+	return false;
+}
+
+struct resource_pool *dce120_create_resource_pool(
+	uint8_t num_virtual_links,
+	struct dc *dc)
+{
+	struct dce110_resource_pool *pool =
+		kzalloc(sizeof(struct dce110_resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dce120_resource_construct(num_virtual_links, dc, pool))
+		return &pool->base;
+
+	kfree(pool);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.h
new file mode 100644
index 000000000000..3d1f3cf012f4
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.h
@@ -0,0 +1,39 @@
+/*
+* Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_RESOURCE_DCE120_H__
+#define __DC_RESOURCE_DCE120_H__
+
+#include "core_types.h"
+
+struct dc;
+struct resource_pool;
+
+struct resource_pool *dce120_create_resource_pool(
+	uint8_t num_virtual_links,
+	struct dc *dc);
+
+#endif /* __DC_RESOURCE_DCE120_H__ */
+
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce80/CMakeLists.txt b/drivers/gpu/drm/amd/display/dc/resource/dce80/CMakeLists.txt
new file mode 100644
index 000000000000..19dd73bc9ab0
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce80/CMakeLists.txt
@@ -0,0 +1,4 @@
+dal3_subdirectory_sources(
+  dce80_resource.c
+  dce80_resource.h
+  )
\ No newline at end of file
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c
new file mode 100644
index 000000000000..35a2cce0c2b8
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c
@@ -0,0 +1,1544 @@
+/*
+ * Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dce/dce_8_0_d.h"
+#include "dce/dce_8_0_sh_mask.h"
+
+#include "dm_services.h"
+
+#include "link_encoder.h"
+#include "stream_encoder.h"
+
+#include "resource.h"
+#include "include/irq_service_interface.h"
+#include "irq/dce80/irq_service_dce80.h"
+#include "dce110/dce110_timing_generator.h"
+#include "dce110/dce110_resource.h"
+#include "dce80/dce80_timing_generator.h"
+#include "dce/dce_mem_input.h"
+#include "dce/dce_link_encoder.h"
+#include "dce/dce_stream_encoder.h"
+#include "dce/dce_ipp.h"
+#include "dce/dce_transform.h"
+#include "dce/dce_opp.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_hwseq.h"
+#include "dce80/dce80_hwseq.h"
+#include "dce100/dce100_resource.h"
+#include "dce/dce_panel_cntl.h"
+
+#include "reg_helper.h"
+
+#include "dce/dce_dmcu.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_abm.h"
+#include "dce/dce_i2c.h"
+/* TODO remove this include */
+
+#ifndef mmMC_HUB_RDREQ_DMIF_LIMIT
+#include "gmc/gmc_7_1_d.h"
+#include "gmc/gmc_7_1_sh_mask.h"
+#endif
+
+#include "dce80/dce80_resource.h"
+
+#ifndef mmDP_DPHY_INTERNAL_CTRL
+#define mmDP_DPHY_INTERNAL_CTRL                         0x1CDE
+#define mmDP0_DP_DPHY_INTERNAL_CTRL                     0x1CDE
+#define mmDP1_DP_DPHY_INTERNAL_CTRL                     0x1FDE
+#define mmDP2_DP_DPHY_INTERNAL_CTRL                     0x42DE
+#define mmDP3_DP_DPHY_INTERNAL_CTRL                     0x45DE
+#define mmDP4_DP_DPHY_INTERNAL_CTRL                     0x48DE
+#define mmDP5_DP_DPHY_INTERNAL_CTRL                     0x4BDE
+#define mmDP6_DP_DPHY_INTERNAL_CTRL                     0x4EDE
+#endif
+
+
+#ifndef mmBIOS_SCRATCH_2
+	#define mmBIOS_SCRATCH_2 0x05CB
+	#define mmBIOS_SCRATCH_3 0x05CC
+	#define mmBIOS_SCRATCH_6 0x05CF
+#endif
+
+#ifndef mmDP_DPHY_FAST_TRAINING
+	#define mmDP_DPHY_FAST_TRAINING                         0x1CCE
+	#define mmDP0_DP_DPHY_FAST_TRAINING                     0x1CCE
+	#define mmDP1_DP_DPHY_FAST_TRAINING                     0x1FCE
+	#define mmDP2_DP_DPHY_FAST_TRAINING                     0x42CE
+	#define mmDP3_DP_DPHY_FAST_TRAINING                     0x45CE
+	#define mmDP4_DP_DPHY_FAST_TRAINING                     0x48CE
+	#define mmDP5_DP_DPHY_FAST_TRAINING                     0x4BCE
+	#define mmDP6_DP_DPHY_FAST_TRAINING                     0x4ECE
+#endif
+
+
+#ifndef mmHPD_DC_HPD_CONTROL
+	#define mmHPD_DC_HPD_CONTROL                            0x189A
+	#define mmHPD0_DC_HPD_CONTROL                           0x189A
+	#define mmHPD1_DC_HPD_CONTROL                           0x18A2
+	#define mmHPD2_DC_HPD_CONTROL                           0x18AA
+	#define mmHPD3_DC_HPD_CONTROL                           0x18B2
+	#define mmHPD4_DC_HPD_CONTROL                           0x18BA
+	#define mmHPD5_DC_HPD_CONTROL                           0x18C2
+#endif
+
+#define DCE11_DIG_FE_CNTL 0x4a00
+#define DCE11_DIG_BE_CNTL 0x4a47
+#define DCE11_DP_SEC 0x4ac3
+
+static const struct dce110_timing_generator_offsets dce80_tg_offsets[] = {
+		{
+			.crtc = (mmCRTC0_CRTC_CONTROL - mmCRTC_CONTROL),
+			.dcp =  (mmGRPH_CONTROL - mmGRPH_CONTROL),
+			.dmif = (mmDMIF_PG0_DPG_WATERMARK_MASK_CONTROL
+					- mmDPG_WATERMARK_MASK_CONTROL),
+		},
+		{
+			.crtc = (mmCRTC1_CRTC_CONTROL - mmCRTC_CONTROL),
+			.dcp = (mmDCP1_GRPH_CONTROL - mmGRPH_CONTROL),
+			.dmif = (mmDMIF_PG1_DPG_WATERMARK_MASK_CONTROL
+					- mmDPG_WATERMARK_MASK_CONTROL),
+		},
+		{
+			.crtc = (mmCRTC2_CRTC_CONTROL - mmCRTC_CONTROL),
+			.dcp = (mmDCP2_GRPH_CONTROL - mmGRPH_CONTROL),
+			.dmif = (mmDMIF_PG2_DPG_WATERMARK_MASK_CONTROL
+					- mmDPG_WATERMARK_MASK_CONTROL),
+		},
+		{
+			.crtc = (mmCRTC3_CRTC_CONTROL - mmCRTC_CONTROL),
+			.dcp = (mmDCP3_GRPH_CONTROL - mmGRPH_CONTROL),
+			.dmif = (mmDMIF_PG3_DPG_WATERMARK_MASK_CONTROL
+					- mmDPG_WATERMARK_MASK_CONTROL),
+		},
+		{
+			.crtc = (mmCRTC4_CRTC_CONTROL - mmCRTC_CONTROL),
+			.dcp = (mmDCP4_GRPH_CONTROL - mmGRPH_CONTROL),
+			.dmif = (mmDMIF_PG4_DPG_WATERMARK_MASK_CONTROL
+					- mmDPG_WATERMARK_MASK_CONTROL),
+		},
+		{
+			.crtc = (mmCRTC5_CRTC_CONTROL - mmCRTC_CONTROL),
+			.dcp = (mmDCP5_GRPH_CONTROL - mmGRPH_CONTROL),
+			.dmif = (mmDMIF_PG5_DPG_WATERMARK_MASK_CONTROL
+					- mmDPG_WATERMARK_MASK_CONTROL),
+		}
+};
+
+/* set register offset */
+#define SR(reg_name)\
+	.reg_name = mm ## reg_name
+
+/* set register offset with instance */
+#define SRI(reg_name, block, id)\
+	.reg_name = mm ## block ## id ## _ ## reg_name
+
+#define ipp_regs(id)\
+[id] = {\
+		IPP_COMMON_REG_LIST_DCE_BASE(id)\
+}
+
+static const struct dce_ipp_registers ipp_regs[] = {
+		ipp_regs(0),
+		ipp_regs(1),
+		ipp_regs(2),
+		ipp_regs(3),
+		ipp_regs(4),
+		ipp_regs(5)
+};
+
+static const struct dce_ipp_shift ipp_shift = {
+		IPP_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
+};
+
+static const struct dce_ipp_mask ipp_mask = {
+		IPP_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
+};
+
+#define transform_regs(id)\
+[id] = {\
+		XFM_COMMON_REG_LIST_DCE80(id)\
+}
+
+static const struct dce_transform_registers xfm_regs[] = {
+		transform_regs(0),
+		transform_regs(1),
+		transform_regs(2),
+		transform_regs(3),
+		transform_regs(4),
+		transform_regs(5)
+};
+
+static const struct dce_transform_shift xfm_shift = {
+		XFM_COMMON_MASK_SH_LIST_DCE80(__SHIFT)
+};
+
+static const struct dce_transform_mask xfm_mask = {
+		XFM_COMMON_MASK_SH_LIST_DCE80(_MASK)
+};
+
+#define aux_regs(id)\
+[id] = {\
+	AUX_REG_LIST(id)\
+}
+
+static const struct dce110_link_enc_aux_registers link_enc_aux_regs[] = {
+	aux_regs(0),
+	aux_regs(1),
+	aux_regs(2),
+	aux_regs(3),
+	aux_regs(4),
+	aux_regs(5)
+};
+
+#define hpd_regs(id)\
+[id] = {\
+	HPD_REG_LIST(id)\
+}
+
+static const struct dce110_link_enc_hpd_registers link_enc_hpd_regs[] = {
+		hpd_regs(0),
+		hpd_regs(1),
+		hpd_regs(2),
+		hpd_regs(3),
+		hpd_regs(4),
+		hpd_regs(5)
+};
+
+#define link_regs(id)\
+[id] = {\
+	LE_DCE80_REG_LIST(id)\
+}
+
+static const struct dce110_link_enc_registers link_enc_regs[] = {
+	link_regs(0),
+	link_regs(1),
+	link_regs(2),
+	link_regs(3),
+	link_regs(4),
+	link_regs(5),
+	link_regs(6),
+};
+
+#define stream_enc_regs(id)\
+[id] = {\
+	SE_COMMON_REG_LIST_DCE_BASE(id),\
+	.AFMT_CNTL = 0,\
+}
+
+static const struct dce110_stream_enc_registers stream_enc_regs[] = {
+	stream_enc_regs(0),
+	stream_enc_regs(1),
+	stream_enc_regs(2),
+	stream_enc_regs(3),
+	stream_enc_regs(4),
+	stream_enc_regs(5),
+	stream_enc_regs(6)
+};
+
+static const struct dce_stream_encoder_shift se_shift = {
+		SE_COMMON_MASK_SH_LIST_DCE80_100(__SHIFT)
+};
+
+static const struct dce_stream_encoder_mask se_mask = {
+		SE_COMMON_MASK_SH_LIST_DCE80_100(_MASK)
+};
+
+static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
+	{ DCE_PANEL_CNTL_REG_LIST() }
+};
+
+static const struct dce_panel_cntl_shift panel_cntl_shift = {
+	DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_panel_cntl_mask panel_cntl_mask = {
+	DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
+};
+
+#define opp_regs(id)\
+[id] = {\
+	OPP_DCE_80_REG_LIST(id),\
+}
+
+static const struct dce_opp_registers opp_regs[] = {
+	opp_regs(0),
+	opp_regs(1),
+	opp_regs(2),
+	opp_regs(3),
+	opp_regs(4),
+	opp_regs(5)
+};
+
+static const struct dce_opp_shift opp_shift = {
+	OPP_COMMON_MASK_SH_LIST_DCE_80(__SHIFT)
+};
+
+static const struct dce_opp_mask opp_mask = {
+	OPP_COMMON_MASK_SH_LIST_DCE_80(_MASK)
+};
+
+static const struct dce110_aux_registers_shift aux_shift = {
+	DCE10_AUX_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+	DCE10_AUX_MASK_SH_LIST(_MASK)
+};
+
+#define aux_engine_regs(id)\
+[id] = {\
+	AUX_COMMON_REG_LIST(id), \
+	.AUX_RESET_MASK = 0 \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1),
+		aux_engine_regs(2),
+		aux_engine_regs(3),
+		aux_engine_regs(4),
+		aux_engine_regs(5)
+};
+
+#define audio_regs(id)\
+[id] = {\
+	AUD_COMMON_REG_LIST(id)\
+}
+
+static const struct dce_audio_registers audio_regs[] = {
+	audio_regs(0),
+	audio_regs(1),
+	audio_regs(2),
+	audio_regs(3),
+	audio_regs(4),
+	audio_regs(5),
+	audio_regs(6),
+};
+
+static const struct dce_audio_shift audio_shift = {
+		AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_audio_mask audio_mask = {
+		AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+#define clk_src_regs(id)\
+[id] = {\
+	CS_COMMON_REG_LIST_DCE_80(id),\
+}
+
+
+static const struct dce110_clk_src_regs clk_src_regs[] = {
+	clk_src_regs(0),
+	clk_src_regs(1),
+	clk_src_regs(2)
+};
+
+static const struct dce110_clk_src_shift cs_shift = {
+		CS_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
+};
+
+static const struct dce110_clk_src_mask cs_mask = {
+		CS_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
+};
+
+static const struct bios_registers bios_regs = {
+	.BIOS_SCRATCH_3 = mmBIOS_SCRATCH_3,
+	.BIOS_SCRATCH_6 = mmBIOS_SCRATCH_6
+};
+
+static const struct resource_caps res_cap = {
+		.num_timing_generator = 6,
+		.num_audio = 6,
+		.num_stream_encoder = 6,
+		.num_pll = 3,
+		.num_ddc = 6,
+};
+
+static const struct resource_caps res_cap_81 = {
+		.num_timing_generator = 4,
+		.num_audio = 7,
+		.num_stream_encoder = 7,
+		.num_pll = 3,
+		.num_ddc = 6,
+};
+
+static const struct resource_caps res_cap_83 = {
+		.num_timing_generator = 2,
+		.num_audio = 6,
+		.num_stream_encoder = 6,
+		.num_pll = 2,
+		.num_ddc = 2,
+};
+
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCE_RGB,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = false,
+			.fp16 = true
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 1,
+			.fp16 = 1
+	},
+
+	.max_downscale_factor = {
+			.argb8888 = 250,
+			.nv12 = 1,
+			.fp16 = 1
+	}
+};
+
+static const struct dc_debug_options debug_defaults = {
+		.enable_legacy_fast_update = true,
+};
+
+static const struct dce_dmcu_registers dmcu_regs = {
+		DMCU_DCE80_REG_LIST()
+};
+
+static const struct dce_dmcu_shift dmcu_shift = {
+		DMCU_MASK_SH_LIST_DCE80(__SHIFT)
+};
+
+static const struct dce_dmcu_mask dmcu_mask = {
+		DMCU_MASK_SH_LIST_DCE80(_MASK)
+};
+static const struct dce_abm_registers abm_regs = {
+		ABM_DCE110_COMMON_REG_LIST()
+};
+
+static const struct dce_abm_shift abm_shift = {
+		ABM_MASK_SH_LIST_DCE110(__SHIFT)
+};
+
+static const struct dce_abm_mask abm_mask = {
+		ABM_MASK_SH_LIST_DCE110(_MASK)
+};
+
+#define CTX  ctx
+#define REG(reg) mm ## reg
+
+#ifndef mmCC_DC_HDMI_STRAPS
+#define mmCC_DC_HDMI_STRAPS 0x1918
+#define CC_DC_HDMI_STRAPS__HDMI_DISABLE_MASK 0x40
+#define CC_DC_HDMI_STRAPS__HDMI_DISABLE__SHIFT 0x6
+#define CC_DC_HDMI_STRAPS__AUDIO_STREAM_NUMBER_MASK 0x700
+#define CC_DC_HDMI_STRAPS__AUDIO_STREAM_NUMBER__SHIFT 0x8
+#endif
+
+static int map_transmitter_id_to_phy_instance(
+	enum transmitter transmitter)
+{
+	switch (transmitter) {
+	case TRANSMITTER_UNIPHY_A:
+		return 0;
+	case TRANSMITTER_UNIPHY_B:
+		return 1;
+	case TRANSMITTER_UNIPHY_C:
+		return 2;
+	case TRANSMITTER_UNIPHY_D:
+		return 3;
+	case TRANSMITTER_UNIPHY_E:
+		return 4;
+	case TRANSMITTER_UNIPHY_F:
+		return 5;
+	case TRANSMITTER_UNIPHY_G:
+		return 6;
+	default:
+		ASSERT(0);
+		return 0;
+	}
+}
+
+static void read_dce_straps(
+	struct dc_context *ctx,
+	struct resource_straps *straps)
+{
+	REG_GET_2(CC_DC_HDMI_STRAPS,
+			HDMI_DISABLE, &straps->hdmi_disable,
+			AUDIO_STREAM_NUMBER, &straps->audio_stream_number);
+
+	REG_GET(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO, &straps->dc_pinstraps_audio);
+}
+
+static struct audio *create_audio(
+		struct dc_context *ctx, unsigned int inst)
+{
+	return dce_audio_create(ctx, inst,
+			&audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+static struct timing_generator *dce80_timing_generator_create(
+		struct dc_context *ctx,
+		uint32_t instance,
+		const struct dce110_timing_generator_offsets *offsets)
+{
+	struct dce110_timing_generator *tg110 =
+		kzalloc(sizeof(struct dce110_timing_generator), GFP_KERNEL);
+
+	if (!tg110)
+		return NULL;
+
+	dce80_timing_generator_construct(tg110, ctx, instance, offsets);
+	return &tg110->base;
+}
+
+static struct output_pixel_processor *dce80_opp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce110_opp *opp =
+		kzalloc(sizeof(struct dce110_opp), GFP_KERNEL);
+
+	if (!opp)
+		return NULL;
+
+	dce110_opp_construct(opp,
+			     ctx, inst, &opp_regs[inst], &opp_shift, &opp_mask);
+	return &opp->base;
+}
+
+static struct dce_aux *dce80_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst],
+					&aux_mask,
+					&aux_shift,
+					ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
+
+static const struct dce_i2c_registers i2c_hw_regs[] = {
+		i2c_inst_regs(1),
+		i2c_inst_regs(2),
+		i2c_inst_regs(3),
+		i2c_inst_regs(4),
+		i2c_inst_regs(5),
+		i2c_inst_regs(6),
+};
+
+static const struct dce_i2c_shift i2c_shifts = {
+		I2C_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(__SHIFT)
+};
+
+static const struct dce_i2c_mask i2c_masks = {
+		I2C_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK)
+};
+
+static struct dce_i2c_hw *dce80_i2c_hw_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw =
+		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+	dce_i2c_hw_construct(dce_i2c_hw, ctx, inst,
+				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+
+static struct dce_i2c_sw *dce80_i2c_sw_create(
+	struct dc_context *ctx)
+{
+	struct dce_i2c_sw *dce_i2c_sw =
+		kzalloc(sizeof(struct dce_i2c_sw), GFP_KERNEL);
+
+	if (!dce_i2c_sw)
+		return NULL;
+
+	dce_i2c_sw_construct(dce_i2c_sw, ctx);
+
+	return dce_i2c_sw;
+}
+static struct stream_encoder *dce80_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dce110_stream_encoder *enc110 =
+		kzalloc(sizeof(struct dce110_stream_encoder), GFP_KERNEL);
+
+	if (!enc110)
+		return NULL;
+
+	dce110_stream_encoder_construct(enc110, ctx, ctx->dc_bios, eng_id,
+					&stream_enc_regs[eng_id],
+					&se_shift, &se_mask);
+	return &enc110->base;
+}
+
+#define SRII(reg_name, block, id)\
+	.reg_name[id] = mm ## block ## id ## _ ## reg_name
+
+static const struct dce_hwseq_registers hwseq_reg = {
+		HWSEQ_DCE8_REG_LIST()
+};
+
+static const struct dce_hwseq_shift hwseq_shift = {
+		HWSEQ_DCE8_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+		HWSEQ_DCE8_MASK_SH_LIST(_MASK)
+};
+
+static struct dce_hwseq *dce80_hwseq_create(
+	struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &hwseq_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+	}
+	return hws;
+}
+
+static const struct resource_create_funcs res_create_funcs = {
+	.read_dce_straps = read_dce_straps,
+	.create_audio = create_audio,
+	.create_stream_encoder = dce80_stream_encoder_create,
+	.create_hwseq = dce80_hwseq_create,
+};
+
+#define mi_inst_regs(id) { \
+	MI_DCE8_REG_LIST(id), \
+	.MC_HUB_RDREQ_DMIF_LIMIT = mmMC_HUB_RDREQ_DMIF_LIMIT \
+}
+static const struct dce_mem_input_registers mi_regs[] = {
+		mi_inst_regs(0),
+		mi_inst_regs(1),
+		mi_inst_regs(2),
+		mi_inst_regs(3),
+		mi_inst_regs(4),
+		mi_inst_regs(5),
+};
+
+static const struct dce_mem_input_shift mi_shifts = {
+		MI_DCE8_MASK_SH_LIST(__SHIFT),
+		.ENABLE = MC_HUB_RDREQ_DMIF_LIMIT__ENABLE__SHIFT
+};
+
+static const struct dce_mem_input_mask mi_masks = {
+		MI_DCE8_MASK_SH_LIST(_MASK),
+		.ENABLE = MC_HUB_RDREQ_DMIF_LIMIT__ENABLE_MASK
+};
+
+static struct mem_input *dce80_mem_input_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_mem_input *dce_mi = kzalloc(sizeof(struct dce_mem_input),
+					       GFP_KERNEL);
+
+	if (!dce_mi) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dce_mem_input_construct(dce_mi, ctx, inst, &mi_regs[inst], &mi_shifts, &mi_masks);
+	dce_mi->wa.single_head_rdreq_dmif_limit = 2;
+	return &dce_mi->base;
+}
+
+static void dce80_transform_destroy(struct transform **xfm)
+{
+	kfree(TO_DCE_TRANSFORM(*xfm));
+	*xfm = NULL;
+}
+
+static struct transform *dce80_transform_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_transform *transform =
+		kzalloc(sizeof(struct dce_transform), GFP_KERNEL);
+
+	if (!transform)
+		return NULL;
+
+	dce_transform_construct(transform, ctx, inst,
+				&xfm_regs[inst], &xfm_shift, &xfm_mask);
+	transform->prescaler_on = false;
+	return &transform->base;
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+		.max_hdmi_deep_color = COLOR_DEPTH_121212,
+		.max_hdmi_pixel_clock = 297000,
+		.flags.bits.IS_HBR2_CAPABLE = true,
+		.flags.bits.IS_TPS3_CAPABLE = true
+};
+
+static struct link_encoder *dce80_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dce110_link_encoder *enc110 =
+		kzalloc(sizeof(struct dce110_link_encoder), GFP_KERNEL);
+	int link_regs_id;
+
+	if (!enc110)
+		return NULL;
+
+	link_regs_id =
+		map_transmitter_id_to_phy_instance(enc_init_data->transmitter);
+
+	dce110_link_encoder_construct(enc110,
+				      enc_init_data,
+				      &link_enc_feature,
+				      &link_enc_regs[link_regs_id],
+				      &link_enc_aux_regs[enc_init_data->channel - 1],
+				      &link_enc_hpd_regs[enc_init_data->hpd_source]);
+	return &enc110->base;
+}
+
+static struct panel_cntl *dce80_panel_cntl_create(const struct panel_cntl_init_data *init_data)
+{
+	struct dce_panel_cntl *panel_cntl =
+		kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
+
+	if (!panel_cntl)
+		return NULL;
+
+	dce_panel_cntl_construct(panel_cntl,
+			init_data,
+			&panel_cntl_regs[init_data->inst],
+			&panel_cntl_shift,
+			&panel_cntl_mask);
+
+	return &panel_cntl->base;
+}
+
+static struct clock_source *dce80_clock_source_create(
+	struct dc_context *ctx,
+	struct dc_bios *bios,
+	enum clock_source_id id,
+	const struct dce110_clk_src_regs *regs,
+	bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dce110_clk_src_construct(clk_src, ctx, bios, id,
+			regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	kfree(clk_src);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static void dce80_clock_source_destroy(struct clock_source **clk_src)
+{
+	kfree(TO_DCE110_CLK_SRC(*clk_src));
+	*clk_src = NULL;
+}
+
+static struct input_pixel_processor *dce80_ipp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dce_ipp *ipp = kzalloc(sizeof(struct dce_ipp), GFP_KERNEL);
+
+	if (!ipp) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dce_ipp_construct(ipp, ctx, inst,
+			&ipp_regs[inst], &ipp_shift, &ipp_mask);
+	return &ipp->base;
+}
+
+static void dce80_resource_destruct(struct dce110_resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (pool->base.opps[i] != NULL)
+			dce110_opp_destroy(&pool->base.opps[i]);
+
+		if (pool->base.transforms[i] != NULL)
+			dce80_transform_destroy(&pool->base.transforms[i]);
+
+		if (pool->base.ipps[i] != NULL)
+			dce_ipp_destroy(&pool->base.ipps[i]);
+
+		if (pool->base.mis[i] != NULL) {
+			kfree(TO_DCE_MEM_INPUT(pool->base.mis[i]));
+			pool->base.mis[i] = NULL;
+		}
+
+		if (pool->base.timing_generators[i] != NULL)	{
+			kfree(DCE110TG_FROM_TG(pool->base.timing_generators[i]));
+			pool->base.timing_generators[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+		if (pool->base.hw_i2cs[i] != NULL) {
+			kfree(pool->base.hw_i2cs[i]);
+			pool->base.hw_i2cs[i] = NULL;
+		}
+		if (pool->base.sw_i2cs[i] != NULL) {
+			kfree(pool->base.sw_i2cs[i]);
+			pool->base.sw_i2cs[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.stream_enc_count; i++) {
+		if (pool->base.stream_enc[i] != NULL)
+			kfree(DCE110STRENC_FROM_STRENC(pool->base.stream_enc[i]));
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] != NULL) {
+			dce80_clock_source_destroy(&pool->base.clock_sources[i]);
+		}
+	}
+
+	if (pool->base.abm != NULL)
+			dce_abm_destroy(&pool->base.abm);
+
+	if (pool->base.dmcu != NULL)
+			dce_dmcu_destroy(&pool->base.dmcu);
+
+	if (pool->base.dp_clock_source != NULL)
+		dce80_clock_source_destroy(&pool->base.dp_clock_source);
+
+	for (i = 0; i < pool->base.audio_count; i++)	{
+		if (pool->base.audios[i] != NULL) {
+			dce_aud_destroy(&pool->base.audios[i]);
+		}
+	}
+
+	if (pool->base.irqs != NULL) {
+		dal_irq_service_destroy(&pool->base.irqs);
+	}
+}
+
+static bool dce80_validate_bandwidth(
+	struct dc *dc,
+	struct dc_state *context,
+	bool fast_validate)
+{
+	int i;
+	bool at_least_one_pipe = false;
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		if (context->res_ctx.pipe_ctx[i].stream)
+			at_least_one_pipe = true;
+	}
+
+	if (at_least_one_pipe) {
+		/* TODO implement when needed but for now hardcode max value*/
+		context->bw_ctx.bw.dce.dispclk_khz = 681000;
+		context->bw_ctx.bw.dce.yclk_khz = 250000 * MEMORY_TYPE_MULTIPLIER_CZ;
+	} else {
+		context->bw_ctx.bw.dce.dispclk_khz = 0;
+		context->bw_ctx.bw.dce.yclk_khz = 0;
+	}
+
+	return true;
+}
+
+static bool dce80_validate_surface_sets(
+		struct dc_state *context)
+{
+	int i;
+
+	for (i = 0; i < context->stream_count; i++) {
+		if (context->stream_status[i].plane_count == 0)
+			continue;
+
+		if (context->stream_status[i].plane_count > 1)
+			return false;
+
+		if (context->stream_status[i].plane_states[0]->format
+				>= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN)
+			return false;
+	}
+
+	return true;
+}
+
+static enum dc_status dce80_validate_global(
+		struct dc *dc,
+		struct dc_state *context)
+{
+	if (!dce80_validate_surface_sets(context))
+		return DC_FAIL_SURFACE_VALIDATE;
+
+	return DC_OK;
+}
+
+static void dce80_destroy_resource_pool(struct resource_pool **pool)
+{
+	struct dce110_resource_pool *dce110_pool = TO_DCE110_RES_POOL(*pool);
+
+	dce80_resource_destruct(dce110_pool);
+	kfree(dce110_pool);
+	*pool = NULL;
+}
+
+static const struct resource_funcs dce80_res_pool_funcs = {
+	.destroy = dce80_destroy_resource_pool,
+	.link_enc_create = dce80_link_encoder_create,
+	.panel_cntl_create = dce80_panel_cntl_create,
+	.validate_bandwidth = dce80_validate_bandwidth,
+	.validate_plane = dce100_validate_plane,
+	.add_stream_to_ctx = dce100_add_stream_to_ctx,
+	.validate_global = dce80_validate_global,
+	.find_first_free_match_stream_enc_for_link = dce100_find_first_free_match_stream_enc_for_link
+};
+
+static bool dce80_construct(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct dce110_resource_pool *pool)
+{
+	unsigned int i;
+	struct dc_context *ctx = dc->ctx;
+	struct dc_bios *bp;
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	pool->base.res_cap = &res_cap;
+	pool->base.funcs = &dce80_res_pool_funcs;
+
+
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+	pool->base.pipe_count = res_cap.num_timing_generator;
+	pool->base.timing_generator_count = res_cap.num_timing_generator;
+	dc->caps.max_downscale_ratio = 200;
+	dc->caps.i2c_speed_in_khz = 40;
+	dc->caps.i2c_speed_in_khz_hdcp = 40;
+	dc->caps.max_cursor_size = 128;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.dual_link_dvi = true;
+	dc->caps.extended_aux_timeout_support = false;
+	dc->debug = debug_defaults;
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	bp = ctx->dc_bios;
+
+	if (bp->fw_info_valid && bp->fw_info.external_clock_source_frequency_for_dp != 0) {
+		pool->base.dp_clock_source =
+				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_EXTERNAL, NULL, true);
+
+		pool->base.clock_sources[0] =
+				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0, &clk_src_regs[0], false);
+		pool->base.clock_sources[1] =
+				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false);
+		pool->base.clock_sources[2] =
+				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[2], false);
+		pool->base.clk_src_count = 3;
+
+	} else {
+		pool->base.dp_clock_source =
+				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0, &clk_src_regs[0], true);
+
+		pool->base.clock_sources[0] =
+				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false);
+		pool->base.clock_sources[1] =
+				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[2], false);
+		pool->base.clk_src_count = 2;
+	}
+
+	if (pool->base.dp_clock_source == NULL) {
+		dm_error("DC: failed to create dp clock source!\n");
+		BREAK_TO_DEBUGGER();
+		goto res_create_fail;
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto res_create_fail;
+		}
+	}
+
+	pool->base.dmcu = dce_dmcu_create(ctx,
+			&dmcu_regs,
+			&dmcu_shift,
+			&dmcu_mask);
+	if (pool->base.dmcu == NULL) {
+		dm_error("DC: failed to create dmcu!\n");
+		BREAK_TO_DEBUGGER();
+		goto res_create_fail;
+	}
+
+	pool->base.abm = dce_abm_create(ctx,
+			&abm_regs,
+			&abm_shift,
+			&abm_mask);
+	if (pool->base.abm == NULL) {
+		dm_error("DC: failed to create abm!\n");
+		BREAK_TO_DEBUGGER();
+		goto res_create_fail;
+	}
+
+	{
+		struct irq_service_init_data init_data;
+		init_data.ctx = dc->ctx;
+		pool->base.irqs = dal_irq_service_dce80_create(&init_data);
+		if (!pool->base.irqs)
+			goto res_create_fail;
+	}
+
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		pool->base.timing_generators[i] = dce80_timing_generator_create(
+				ctx, i, &dce80_tg_offsets[i]);
+		if (pool->base.timing_generators[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.mis[i] = dce80_mem_input_create(ctx, i);
+		if (pool->base.mis[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create memory input!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.ipps[i] = dce80_ipp_create(ctx, i);
+		if (pool->base.ipps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create input pixel processor!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.transforms[i] = dce80_transform_create(ctx, i);
+		if (pool->base.transforms[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create transform!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.opps[i] = dce80_opp_create(ctx, i);
+		if (pool->base.opps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create output pixel processor!\n");
+			goto res_create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.engines[i] = dce80_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto res_create_fail;
+		}
+		pool->base.hw_i2cs[i] = dce80_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create i2c engine!!\n");
+			goto res_create_fail;
+		}
+		pool->base.sw_i2cs[i] = dce80_i2c_sw_create(ctx);
+		if (pool->base.sw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create sw i2c!!\n");
+			goto res_create_fail;
+		}
+	}
+
+	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->caps.disable_dp_clk_share = true;
+
+	if (!resource_construct(num_virtual_links, dc, &pool->base,
+			&res_create_funcs))
+		goto res_create_fail;
+
+	/* Create hardware sequencer */
+	dce80_hw_sequencer_construct(dc);
+
+	return true;
+
+res_create_fail:
+	dce80_resource_destruct(pool);
+	return false;
+}
+
+struct resource_pool *dce80_create_resource_pool(
+	uint8_t num_virtual_links,
+	struct dc *dc)
+{
+	struct dce110_resource_pool *pool =
+		kzalloc(sizeof(struct dce110_resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dce80_construct(num_virtual_links, dc, pool))
+		return &pool->base;
+
+	kfree(pool);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static bool dce81_construct(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct dce110_resource_pool *pool)
+{
+	unsigned int i;
+	struct dc_context *ctx = dc->ctx;
+	struct dc_bios *bp;
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	pool->base.res_cap = &res_cap_81;
+	pool->base.funcs = &dce80_res_pool_funcs;
+
+
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+	pool->base.pipe_count = res_cap_81.num_timing_generator;
+	pool->base.timing_generator_count = res_cap_81.num_timing_generator;
+	dc->caps.max_downscale_ratio = 200;
+	dc->caps.i2c_speed_in_khz = 40;
+	dc->caps.i2c_speed_in_khz_hdcp = 40;
+	dc->caps.max_cursor_size = 128;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.is_apu = true;
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	bp = ctx->dc_bios;
+
+	if (bp->fw_info_valid && bp->fw_info.external_clock_source_frequency_for_dp != 0) {
+		pool->base.dp_clock_source =
+				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_EXTERNAL, NULL, true);
+
+		pool->base.clock_sources[0] =
+				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0, &clk_src_regs[0], false);
+		pool->base.clock_sources[1] =
+				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false);
+		pool->base.clock_sources[2] =
+				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[2], false);
+		pool->base.clk_src_count = 3;
+
+	} else {
+		pool->base.dp_clock_source =
+				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0, &clk_src_regs[0], true);
+
+		pool->base.clock_sources[0] =
+				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false);
+		pool->base.clock_sources[1] =
+				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[2], false);
+		pool->base.clk_src_count = 2;
+	}
+
+	if (pool->base.dp_clock_source == NULL) {
+		dm_error("DC: failed to create dp clock source!\n");
+		BREAK_TO_DEBUGGER();
+		goto res_create_fail;
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto res_create_fail;
+		}
+	}
+
+	pool->base.dmcu = dce_dmcu_create(ctx,
+			&dmcu_regs,
+			&dmcu_shift,
+			&dmcu_mask);
+	if (pool->base.dmcu == NULL) {
+		dm_error("DC: failed to create dmcu!\n");
+		BREAK_TO_DEBUGGER();
+		goto res_create_fail;
+	}
+
+	pool->base.abm = dce_abm_create(ctx,
+			&abm_regs,
+			&abm_shift,
+			&abm_mask);
+	if (pool->base.abm == NULL) {
+		dm_error("DC: failed to create abm!\n");
+		BREAK_TO_DEBUGGER();
+		goto res_create_fail;
+	}
+
+	{
+		struct irq_service_init_data init_data;
+		init_data.ctx = dc->ctx;
+		pool->base.irqs = dal_irq_service_dce80_create(&init_data);
+		if (!pool->base.irqs)
+			goto res_create_fail;
+	}
+
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		pool->base.timing_generators[i] = dce80_timing_generator_create(
+				ctx, i, &dce80_tg_offsets[i]);
+		if (pool->base.timing_generators[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.mis[i] = dce80_mem_input_create(ctx, i);
+		if (pool->base.mis[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create memory input!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.ipps[i] = dce80_ipp_create(ctx, i);
+		if (pool->base.ipps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create input pixel processor!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.transforms[i] = dce80_transform_create(ctx, i);
+		if (pool->base.transforms[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create transform!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.opps[i] = dce80_opp_create(ctx, i);
+		if (pool->base.opps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create output pixel processor!\n");
+			goto res_create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.engines[i] = dce80_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto res_create_fail;
+		}
+		pool->base.hw_i2cs[i] = dce80_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create i2c engine!!\n");
+			goto res_create_fail;
+		}
+		pool->base.sw_i2cs[i] = dce80_i2c_sw_create(ctx);
+		if (pool->base.sw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create sw i2c!!\n");
+			goto res_create_fail;
+		}
+	}
+
+	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->caps.disable_dp_clk_share = true;
+
+	if (!resource_construct(num_virtual_links, dc, &pool->base,
+			&res_create_funcs))
+		goto res_create_fail;
+
+	/* Create hardware sequencer */
+	dce80_hw_sequencer_construct(dc);
+
+	return true;
+
+res_create_fail:
+	dce80_resource_destruct(pool);
+	return false;
+}
+
+struct resource_pool *dce81_create_resource_pool(
+	uint8_t num_virtual_links,
+	struct dc *dc)
+{
+	struct dce110_resource_pool *pool =
+		kzalloc(sizeof(struct dce110_resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dce81_construct(num_virtual_links, dc, pool))
+		return &pool->base;
+
+	kfree(pool);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static bool dce83_construct(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct dce110_resource_pool *pool)
+{
+	unsigned int i;
+	struct dc_context *ctx = dc->ctx;
+	struct dc_bios *bp;
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	pool->base.res_cap = &res_cap_83;
+	pool->base.funcs = &dce80_res_pool_funcs;
+
+
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+	pool->base.pipe_count = res_cap_83.num_timing_generator;
+	pool->base.timing_generator_count = res_cap_83.num_timing_generator;
+	dc->caps.max_downscale_ratio = 200;
+	dc->caps.i2c_speed_in_khz = 40;
+	dc->caps.i2c_speed_in_khz_hdcp = 40;
+	dc->caps.max_cursor_size = 128;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.is_apu = true;
+	dc->debug = debug_defaults;
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	bp = ctx->dc_bios;
+
+	if (bp->fw_info_valid && bp->fw_info.external_clock_source_frequency_for_dp != 0) {
+		pool->base.dp_clock_source =
+				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_EXTERNAL, NULL, true);
+
+		pool->base.clock_sources[0] =
+				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[0], false);
+		pool->base.clock_sources[1] =
+				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[1], false);
+		pool->base.clk_src_count = 2;
+
+	} else {
+		pool->base.dp_clock_source =
+				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[0], true);
+
+		pool->base.clock_sources[0] =
+				dce80_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[1], false);
+		pool->base.clk_src_count = 1;
+	}
+
+	if (pool->base.dp_clock_source == NULL) {
+		dm_error("DC: failed to create dp clock source!\n");
+		BREAK_TO_DEBUGGER();
+		goto res_create_fail;
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto res_create_fail;
+		}
+	}
+
+	pool->base.dmcu = dce_dmcu_create(ctx,
+			&dmcu_regs,
+			&dmcu_shift,
+			&dmcu_mask);
+	if (pool->base.dmcu == NULL) {
+		dm_error("DC: failed to create dmcu!\n");
+		BREAK_TO_DEBUGGER();
+		goto res_create_fail;
+	}
+
+	pool->base.abm = dce_abm_create(ctx,
+			&abm_regs,
+			&abm_shift,
+			&abm_mask);
+	if (pool->base.abm == NULL) {
+		dm_error("DC: failed to create abm!\n");
+		BREAK_TO_DEBUGGER();
+		goto res_create_fail;
+	}
+
+	{
+		struct irq_service_init_data init_data;
+		init_data.ctx = dc->ctx;
+		pool->base.irqs = dal_irq_service_dce80_create(&init_data);
+		if (!pool->base.irqs)
+			goto res_create_fail;
+	}
+
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		pool->base.timing_generators[i] = dce80_timing_generator_create(
+				ctx, i, &dce80_tg_offsets[i]);
+		if (pool->base.timing_generators[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.mis[i] = dce80_mem_input_create(ctx, i);
+		if (pool->base.mis[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create memory input!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.ipps[i] = dce80_ipp_create(ctx, i);
+		if (pool->base.ipps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create input pixel processor!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.transforms[i] = dce80_transform_create(ctx, i);
+		if (pool->base.transforms[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create transform!\n");
+			goto res_create_fail;
+		}
+
+		pool->base.opps[i] = dce80_opp_create(ctx, i);
+		if (pool->base.opps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create output pixel processor!\n");
+			goto res_create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.engines[i] = dce80_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto res_create_fail;
+		}
+		pool->base.hw_i2cs[i] = dce80_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create i2c engine!!\n");
+			goto res_create_fail;
+		}
+		pool->base.sw_i2cs[i] = dce80_i2c_sw_create(ctx);
+		if (pool->base.sw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create sw i2c!!\n");
+			goto res_create_fail;
+		}
+	}
+
+	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->caps.disable_dp_clk_share = true;
+
+	if (!resource_construct(num_virtual_links, dc, &pool->base,
+			&res_create_funcs))
+		goto res_create_fail;
+
+	/* Create hardware sequencer */
+	dce80_hw_sequencer_construct(dc);
+
+	return true;
+
+res_create_fail:
+	dce80_resource_destruct(pool);
+	return false;
+}
+
+struct resource_pool *dce83_create_resource_pool(
+	uint8_t num_virtual_links,
+	struct dc *dc)
+{
+	struct dce110_resource_pool *pool =
+		kzalloc(sizeof(struct dce110_resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dce83_construct(num_virtual_links, dc, pool))
+		return &pool->base;
+
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.h
new file mode 100644
index 000000000000..eff31ab83a39
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.h
@@ -0,0 +1,47 @@
+/*
+* Copyright 2012-15 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_RESOURCE_DCE80_H__
+#define __DC_RESOURCE_DCE80_H__
+
+#include "core_types.h"
+
+struct dc;
+struct resource_pool;
+
+struct resource_pool *dce80_create_resource_pool(
+	uint8_t num_virtual_links,
+	struct dc *dc);
+
+struct resource_pool *dce81_create_resource_pool(
+	uint8_t num_virtual_links,
+	struct dc *dc);
+
+struct resource_pool *dce83_create_resource_pool(
+	uint8_t num_virtual_links,
+	struct dc *dc);
+
+#endif /* __DC_RESOURCE_DCE80_H__ */
+
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.c
new file mode 100644
index 000000000000..af1b31f4e69a
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.c
@@ -0,0 +1,1689 @@
+/*
+* Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dm_services.h"
+#include "dc.h"
+
+#include "dcn10/dcn10_init.h"
+
+#include "resource.h"
+#include "include/irq_service_interface.h"
+#include "dcn10/dcn10_resource.h"
+#include "dcn10/dcn10_ipp.h"
+#include "dcn10/dcn10_mpc.h"
+
+#include "dcn10/dcn10_dwb.h"
+
+#include "irq/dcn10/irq_service_dcn10.h"
+#include "dcn10/dcn10_dpp.h"
+#include "dcn10/dcn10_optc.h"
+#include "dcn10/dcn10_hwseq.h"
+#include "dce110/dce110_hwseq.h"
+#include "dcn10/dcn10_opp.h"
+#include "dcn10/dcn10_link_encoder.h"
+#include "dcn10/dcn10_stream_encoder.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_hwseq.h"
+#include "virtual/virtual_stream_encoder.h"
+#include "dce110/dce110_resource.h"
+#include "dce112/dce112_resource.h"
+#include "dcn10/dcn10_hubp.h"
+#include "dcn10/dcn10_hubbub.h"
+#include "dce/dce_panel_cntl.h"
+
+#include "soc15_hw_ip.h"
+#include "vega10_ip_offset.h"
+
+#include "dcn/dcn_1_0_offset.h"
+#include "dcn/dcn_1_0_sh_mask.h"
+
+#include "nbio/nbio_7_0_offset.h"
+
+#include "mmhub/mmhub_9_1_offset.h"
+#include "mmhub/mmhub_9_1_sh_mask.h"
+
+#include "reg_helper.h"
+#include "dce/dce_abm.h"
+#include "dce/dce_dmcu.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_i2c.h"
+
+#ifndef mmDP0_DP_DPHY_INTERNAL_CTRL
+	#define mmDP0_DP_DPHY_INTERNAL_CTRL		0x210f
+	#define mmDP0_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
+	#define mmDP1_DP_DPHY_INTERNAL_CTRL		0x220f
+	#define mmDP1_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
+	#define mmDP2_DP_DPHY_INTERNAL_CTRL		0x230f
+	#define mmDP2_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
+	#define mmDP3_DP_DPHY_INTERNAL_CTRL		0x240f
+	#define mmDP3_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
+	#define mmDP4_DP_DPHY_INTERNAL_CTRL		0x250f
+	#define mmDP4_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
+	#define mmDP5_DP_DPHY_INTERNAL_CTRL		0x260f
+	#define mmDP5_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
+	#define mmDP6_DP_DPHY_INTERNAL_CTRL		0x270f
+	#define mmDP6_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
+#endif
+
+
+enum dcn10_clk_src_array_id {
+	DCN10_CLK_SRC_PLL0,
+	DCN10_CLK_SRC_PLL1,
+	DCN10_CLK_SRC_PLL2,
+	DCN10_CLK_SRC_PLL3,
+	DCN10_CLK_SRC_TOTAL,
+	DCN101_CLK_SRC_TOTAL = DCN10_CLK_SRC_PLL3
+};
+
+/* begin *********************
+ * macros to expend register list macro defined in HW object header file */
+
+/* DCN */
+#define BASE_INNER(seg) \
+	DCE_BASE__INST0_SEG ## seg
+
+#define BASE(seg) \
+	BASE_INNER(seg)
+
+#define SR(reg_name)\
+		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) +  \
+					mm ## reg_name
+
+#define SRI(reg_name, block, id)\
+	.reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+
+#define SRII(reg_name, block, id)\
+	.reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define VUPDATE_SRII(reg_name, block, id)\
+	.reg_name[id] = BASE(mm ## reg_name ## 0 ## _ ## block ## id ## _BASE_IDX) + \
+					mm ## reg_name ## 0 ## _ ## block ## id
+
+/* set field/register/bitfield name */
+#define SFRB(field_name, reg_name, bitfield, post_fix)\
+	.field_name = reg_name ## __ ## bitfield ## post_fix
+
+/* NBIO */
+#define NBIO_BASE_INNER(seg) \
+	NBIF_BASE__INST0_SEG ## seg
+
+#define NBIO_BASE(seg) \
+	NBIO_BASE_INNER(seg)
+
+#define NBIO_SR(reg_name)\
+		.reg_name = NBIO_BASE(mm ## reg_name ## _BASE_IDX) +  \
+					mm ## reg_name
+
+/* MMHUB */
+#define MMHUB_BASE_INNER(seg) \
+	MMHUB_BASE__INST0_SEG ## seg
+
+#define MMHUB_BASE(seg) \
+	MMHUB_BASE_INNER(seg)
+
+#define MMHUB_SR(reg_name)\
+		.reg_name = MMHUB_BASE(mm ## reg_name ## _BASE_IDX) +  \
+					mm ## reg_name
+
+/* macros to expend register list macro defined in HW object header file
+ * end *********************/
+
+
+static const struct dce_dmcu_registers dmcu_regs = {
+		DMCU_DCN10_REG_LIST()
+};
+
+static const struct dce_dmcu_shift dmcu_shift = {
+		DMCU_MASK_SH_LIST_DCN10(__SHIFT)
+};
+
+static const struct dce_dmcu_mask dmcu_mask = {
+		DMCU_MASK_SH_LIST_DCN10(_MASK)
+};
+
+static const struct dce_abm_registers abm_regs = {
+		ABM_DCN10_REG_LIST(0)
+};
+
+static const struct dce_abm_shift abm_shift = {
+		ABM_MASK_SH_LIST_DCN10(__SHIFT)
+};
+
+static const struct dce_abm_mask abm_mask = {
+		ABM_MASK_SH_LIST_DCN10(_MASK)
+};
+
+#define stream_enc_regs(id)\
+[id] = {\
+	SE_DCN_REG_LIST(id)\
+}
+
+static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
+	stream_enc_regs(0),
+	stream_enc_regs(1),
+	stream_enc_regs(2),
+	stream_enc_regs(3),
+};
+
+static const struct dcn10_stream_encoder_shift se_shift = {
+		SE_COMMON_MASK_SH_LIST_DCN10(__SHIFT)
+};
+
+static const struct dcn10_stream_encoder_mask se_mask = {
+		SE_COMMON_MASK_SH_LIST_DCN10(_MASK)
+};
+
+#define audio_regs(id)\
+[id] = {\
+		AUD_COMMON_REG_LIST(id)\
+}
+
+static const struct dce_audio_registers audio_regs[] = {
+	audio_regs(0),
+	audio_regs(1),
+	audio_regs(2),
+	audio_regs(3),
+};
+
+#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
+		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
+
+static const struct dce_audio_shift audio_shift = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_audio_mask audio_mask = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+#define aux_regs(id)\
+[id] = {\
+	AUX_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
+		aux_regs(0),
+		aux_regs(1),
+		aux_regs(2),
+		aux_regs(3)
+};
+
+#define hpd_regs(id)\
+[id] = {\
+	HPD_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
+		hpd_regs(0),
+		hpd_regs(1),
+		hpd_regs(2),
+		hpd_regs(3)
+};
+
+#define link_regs(id)\
+[id] = {\
+	LE_DCN10_REG_LIST(id), \
+	SRI(DP_DPHY_INTERNAL_CTRL, DP, id) \
+}
+
+static const struct dcn10_link_enc_registers link_enc_regs[] = {
+	link_regs(0),
+	link_regs(1),
+	link_regs(2),
+	link_regs(3)
+};
+
+static const struct dcn10_link_enc_shift le_shift = {
+		LINK_ENCODER_MASK_SH_LIST_DCN10(__SHIFT)
+};
+
+static const struct dcn10_link_enc_mask le_mask = {
+		LINK_ENCODER_MASK_SH_LIST_DCN10(_MASK)
+};
+
+static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
+	{ DCN_PANEL_CNTL_REG_LIST() }
+};
+
+static const struct dce_panel_cntl_shift panel_cntl_shift = {
+	DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_panel_cntl_mask panel_cntl_mask = {
+	DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
+};
+
+static const struct dce110_aux_registers_shift aux_shift = {
+	DCN10_AUX_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+	DCN10_AUX_MASK_SH_LIST(_MASK)
+};
+
+#define ipp_regs(id)\
+[id] = {\
+	IPP_REG_LIST_DCN10(id),\
+}
+
+static const struct dcn10_ipp_registers ipp_regs[] = {
+	ipp_regs(0),
+	ipp_regs(1),
+	ipp_regs(2),
+	ipp_regs(3),
+};
+
+static const struct dcn10_ipp_shift ipp_shift = {
+		IPP_MASK_SH_LIST_DCN10(__SHIFT)
+};
+
+static const struct dcn10_ipp_mask ipp_mask = {
+		IPP_MASK_SH_LIST_DCN10(_MASK),
+};
+
+#define opp_regs(id)\
+[id] = {\
+	OPP_REG_LIST_DCN10(id),\
+}
+
+static const struct dcn10_opp_registers opp_regs[] = {
+	opp_regs(0),
+	opp_regs(1),
+	opp_regs(2),
+	opp_regs(3),
+};
+
+static const struct dcn10_opp_shift opp_shift = {
+		OPP_MASK_SH_LIST_DCN10(__SHIFT)
+};
+
+static const struct dcn10_opp_mask opp_mask = {
+		OPP_MASK_SH_LIST_DCN10(_MASK),
+};
+
+#define aux_engine_regs(id)\
+[id] = {\
+	AUX_COMMON_REG_LIST(id), \
+	.AUX_RESET_MASK = 0 \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1),
+		aux_engine_regs(2),
+		aux_engine_regs(3),
+		aux_engine_regs(4),
+		aux_engine_regs(5)
+};
+
+#define tf_regs(id)\
+[id] = {\
+	TF_REG_LIST_DCN10(id),\
+}
+
+static const struct dcn_dpp_registers tf_regs[] = {
+	tf_regs(0),
+	tf_regs(1),
+	tf_regs(2),
+	tf_regs(3),
+};
+
+static const struct dcn_dpp_shift tf_shift = {
+	TF_REG_LIST_SH_MASK_DCN10(__SHIFT),
+	TF_DEBUG_REG_LIST_SH_DCN10
+
+};
+
+static const struct dcn_dpp_mask tf_mask = {
+	TF_REG_LIST_SH_MASK_DCN10(_MASK),
+	TF_DEBUG_REG_LIST_MASK_DCN10
+};
+
+static const struct dcn_mpc_registers mpc_regs = {
+		MPC_COMMON_REG_LIST_DCN1_0(0),
+		MPC_COMMON_REG_LIST_DCN1_0(1),
+		MPC_COMMON_REG_LIST_DCN1_0(2),
+		MPC_COMMON_REG_LIST_DCN1_0(3),
+		MPC_OUT_MUX_COMMON_REG_LIST_DCN1_0(0),
+		MPC_OUT_MUX_COMMON_REG_LIST_DCN1_0(1),
+		MPC_OUT_MUX_COMMON_REG_LIST_DCN1_0(2),
+		MPC_OUT_MUX_COMMON_REG_LIST_DCN1_0(3)
+};
+
+static const struct dcn_mpc_shift mpc_shift = {
+	MPC_COMMON_MASK_SH_LIST_DCN1_0(__SHIFT),\
+	SFRB(CUR_VUPDATE_LOCK_SET, CUR0_VUPDATE_LOCK_SET0, CUR0_VUPDATE_LOCK_SET, __SHIFT)
+};
+
+static const struct dcn_mpc_mask mpc_mask = {
+	MPC_COMMON_MASK_SH_LIST_DCN1_0(_MASK),\
+	SFRB(CUR_VUPDATE_LOCK_SET, CUR0_VUPDATE_LOCK_SET0, CUR0_VUPDATE_LOCK_SET, _MASK)
+};
+
+#define tg_regs(id)\
+[id] = {TG_COMMON_REG_LIST_DCN1_0(id)}
+
+static const struct dcn_optc_registers tg_regs[] = {
+	tg_regs(0),
+	tg_regs(1),
+	tg_regs(2),
+	tg_regs(3),
+};
+
+static const struct dcn_optc_shift tg_shift = {
+	TG_COMMON_MASK_SH_LIST_DCN1_0(__SHIFT)
+};
+
+static const struct dcn_optc_mask tg_mask = {
+	TG_COMMON_MASK_SH_LIST_DCN1_0(_MASK)
+};
+
+static const struct bios_registers bios_regs = {
+		NBIO_SR(BIOS_SCRATCH_3),
+		NBIO_SR(BIOS_SCRATCH_6)
+};
+
+#define hubp_regs(id)\
+[id] = {\
+	HUBP_REG_LIST_DCN10(id)\
+}
+
+static const struct dcn_mi_registers hubp_regs[] = {
+	hubp_regs(0),
+	hubp_regs(1),
+	hubp_regs(2),
+	hubp_regs(3),
+};
+
+static const struct dcn_mi_shift hubp_shift = {
+		HUBP_MASK_SH_LIST_DCN10(__SHIFT)
+};
+
+static const struct dcn_mi_mask hubp_mask = {
+		HUBP_MASK_SH_LIST_DCN10(_MASK)
+};
+
+static const struct dcn_hubbub_registers hubbub_reg = {
+		HUBBUB_REG_LIST_DCN10(0)
+};
+
+static const struct dcn_hubbub_shift hubbub_shift = {
+		HUBBUB_MASK_SH_LIST_DCN10(__SHIFT)
+};
+
+static const struct dcn_hubbub_mask hubbub_mask = {
+		HUBBUB_MASK_SH_LIST_DCN10(_MASK)
+};
+
+static int map_transmitter_id_to_phy_instance(
+	enum transmitter transmitter)
+{
+	switch (transmitter) {
+	case TRANSMITTER_UNIPHY_A:
+		return 0;
+	break;
+	case TRANSMITTER_UNIPHY_B:
+		return 1;
+	break;
+	case TRANSMITTER_UNIPHY_C:
+		return 2;
+	break;
+	case TRANSMITTER_UNIPHY_D:
+		return 3;
+	break;
+	default:
+		ASSERT(0);
+		return 0;
+	}
+}
+
+#define clk_src_regs(index, pllid)\
+[index] = {\
+	CS_COMMON_REG_LIST_DCN1_0(index, pllid),\
+}
+
+static const struct dce110_clk_src_regs clk_src_regs[] = {
+	clk_src_regs(0, A),
+	clk_src_regs(1, B),
+	clk_src_regs(2, C),
+	clk_src_regs(3, D)
+};
+
+static const struct dce110_clk_src_shift cs_shift = {
+		CS_COMMON_MASK_SH_LIST_DCN1_0(__SHIFT)
+};
+
+static const struct dce110_clk_src_mask cs_mask = {
+		CS_COMMON_MASK_SH_LIST_DCN1_0(_MASK)
+};
+
+static const struct resource_caps res_cap = {
+		.num_timing_generator = 4,
+		.num_opp = 4,
+		.num_video_plane = 4,
+		.num_audio = 4,
+		.num_stream_encoder = 4,
+		.num_pll = 4,
+		.num_ddc = 4,
+};
+
+static const struct resource_caps rv2_res_cap = {
+		.num_timing_generator = 3,
+		.num_opp = 3,
+		.num_video_plane = 3,
+		.num_audio = 3,
+		.num_stream_encoder = 3,
+		.num_pll = 3,
+		.num_ddc = 4,
+};
+
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
+	.per_pixel_alpha = true,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = true,
+			.fp16 = true,
+			.p010 = true
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 16000,
+			.fp16 = 1
+	},
+
+	.max_downscale_factor = {
+			.argb8888 = 250,
+			.nv12 = 250,
+			.fp16 = 1
+	}
+};
+
+static const struct dc_debug_options debug_defaults_drv = {
+		.sanity_checks = true,
+		.disable_dmcu = false,
+		.force_abm_enable = false,
+		.timing_trace = false,
+		.clock_trace = true,
+
+		/* raven smu dones't allow 0 disp clk,
+		 * smu min disp clk limit is 50Mhz
+		 * keep min disp clk 100Mhz avoid smu hang
+		 */
+		.min_disp_clk_khz = 100000,
+
+		.disable_pplib_clock_request = false,
+		.disable_pplib_wm_range = false,
+		.pplib_wm_report_mode = WM_REPORT_DEFAULT,
+		.pipe_split_policy = MPC_SPLIT_DYNAMIC,
+		.force_single_disp_pipe_split = true,
+		.disable_dcc = DCC_ENABLE,
+		.voltage_align_fclk = true,
+		.disable_stereo_support = true,
+		.vsr_support = true,
+		.performance_trace = false,
+		.az_endpoint_mute_only = true,
+		.recovery_enabled = false, /*enable this by default after testing.*/
+		.max_downscale_src_width = 3840,
+		.underflow_assert_delay_us = 0xFFFFFFFF,
+		.enable_legacy_fast_update = true,
+		.using_dml2 = false,
+};
+
+static const struct dc_debug_options debug_defaults_diags = {
+		.disable_dmcu = false,
+		.force_abm_enable = false,
+		.timing_trace = true,
+		.clock_trace = true,
+		.disable_stutter = true,
+		.disable_pplib_clock_request = true,
+		.disable_pplib_wm_range = true,
+		.underflow_assert_delay_us = 0xFFFFFFFF,
+};
+
+static void dcn10_dpp_destroy(struct dpp **dpp)
+{
+	kfree(TO_DCN10_DPP(*dpp));
+	*dpp = NULL;
+}
+
+static struct dpp *dcn10_dpp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn10_dpp *dpp =
+		kzalloc(sizeof(struct dcn10_dpp), GFP_KERNEL);
+
+	if (!dpp)
+		return NULL;
+
+	dpp1_construct(dpp, ctx, inst,
+		       &tf_regs[inst], &tf_shift, &tf_mask);
+	return &dpp->base;
+}
+
+static struct input_pixel_processor *dcn10_ipp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn10_ipp *ipp =
+		kzalloc(sizeof(struct dcn10_ipp), GFP_KERNEL);
+
+	if (!ipp) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dcn10_ipp_construct(ipp, ctx, inst,
+			&ipp_regs[inst], &ipp_shift, &ipp_mask);
+	return &ipp->base;
+}
+
+
+static struct output_pixel_processor *dcn10_opp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn10_opp *opp =
+		kzalloc(sizeof(struct dcn10_opp), GFP_KERNEL);
+
+	if (!opp) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dcn10_opp_construct(opp, ctx, inst,
+			&opp_regs[inst], &opp_shift, &opp_mask);
+	return &opp->base;
+}
+
+static struct dce_aux *dcn10_aux_engine_create(struct dc_context *ctx,
+					       uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst],
+					&aux_mask,
+					&aux_shift,
+					ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
+
+static const struct dce_i2c_registers i2c_hw_regs[] = {
+		i2c_inst_regs(1),
+		i2c_inst_regs(2),
+		i2c_inst_regs(3),
+		i2c_inst_regs(4),
+		i2c_inst_regs(5),
+		i2c_inst_regs(6),
+};
+
+static const struct dce_i2c_shift i2c_shifts = {
+		I2C_COMMON_MASK_SH_LIST_DCE110(__SHIFT)
+};
+
+static const struct dce_i2c_mask i2c_masks = {
+		I2C_COMMON_MASK_SH_LIST_DCE110(_MASK)
+};
+
+static struct dce_i2c_hw *dcn10_i2c_hw_create(struct dc_context *ctx,
+					      uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw =
+		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+	dcn1_i2c_hw_construct(dce_i2c_hw, ctx, inst,
+				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+static struct mpc *dcn10_mpc_create(struct dc_context *ctx)
+{
+	struct dcn10_mpc *mpc10 = kzalloc(sizeof(struct dcn10_mpc),
+					  GFP_KERNEL);
+
+	if (!mpc10)
+		return NULL;
+
+	dcn10_mpc_construct(mpc10, ctx,
+			&mpc_regs,
+			&mpc_shift,
+			&mpc_mask,
+			4);
+
+	return &mpc10->base;
+}
+
+static struct hubbub *dcn10_hubbub_create(struct dc_context *ctx)
+{
+	struct dcn10_hubbub *dcn10_hubbub = kzalloc(sizeof(struct dcn10_hubbub),
+					  GFP_KERNEL);
+
+	if (!dcn10_hubbub)
+		return NULL;
+
+	hubbub1_construct(&dcn10_hubbub->base, ctx,
+			&hubbub_reg,
+			&hubbub_shift,
+			&hubbub_mask);
+
+	return &dcn10_hubbub->base;
+}
+
+static struct timing_generator *dcn10_timing_generator_create(
+		struct dc_context *ctx,
+		uint32_t instance)
+{
+	struct optc *tgn10 =
+		kzalloc(sizeof(struct optc), GFP_KERNEL);
+
+	if (!tgn10)
+		return NULL;
+
+	tgn10->base.inst = instance;
+	tgn10->base.ctx = ctx;
+
+	tgn10->tg_regs = &tg_regs[instance];
+	tgn10->tg_shift = &tg_shift;
+	tgn10->tg_mask = &tg_mask;
+
+	dcn10_timing_generator_init(tgn10);
+
+	return &tgn10->base;
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+		.max_hdmi_deep_color = COLOR_DEPTH_121212,
+		.max_hdmi_pixel_clock = 600000,
+		.hdmi_ycbcr420_supported = true,
+		.dp_ycbcr420_supported = true,
+		.flags.bits.IS_HBR2_CAPABLE = true,
+		.flags.bits.IS_HBR3_CAPABLE = true,
+		.flags.bits.IS_TPS3_CAPABLE = true,
+		.flags.bits.IS_TPS4_CAPABLE = true
+};
+
+static struct link_encoder *dcn10_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dcn10_link_encoder *enc10 =
+		kzalloc(sizeof(struct dcn10_link_encoder), GFP_KERNEL);
+	int link_regs_id;
+
+	if (!enc10)
+		return NULL;
+
+	link_regs_id =
+		map_transmitter_id_to_phy_instance(enc_init_data->transmitter);
+
+	dcn10_link_encoder_construct(enc10,
+				      enc_init_data,
+				      &link_enc_feature,
+				      &link_enc_regs[link_regs_id],
+				      &link_enc_aux_regs[enc_init_data->channel - 1],
+				      &link_enc_hpd_regs[enc_init_data->hpd_source],
+				      &le_shift,
+				      &le_mask);
+
+	return &enc10->base;
+}
+
+static struct panel_cntl *dcn10_panel_cntl_create(const struct panel_cntl_init_data *init_data)
+{
+	struct dce_panel_cntl *panel_cntl =
+		kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
+
+	if (!panel_cntl)
+		return NULL;
+
+	dce_panel_cntl_construct(panel_cntl,
+			init_data,
+			&panel_cntl_regs[init_data->inst],
+			&panel_cntl_shift,
+			&panel_cntl_mask);
+
+	return &panel_cntl->base;
+}
+
+static struct clock_source *dcn10_clock_source_create(
+	struct dc_context *ctx,
+	struct dc_bios *bios,
+	enum clock_source_id id,
+	const struct dce110_clk_src_regs *regs,
+	bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dce112_clk_src_construct(clk_src, ctx, bios, id,
+			regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	kfree(clk_src);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static void read_dce_straps(
+	struct dc_context *ctx,
+	struct resource_straps *straps)
+{
+	generic_reg_get(ctx, mmDC_PINSTRAPS + BASE(mmDC_PINSTRAPS_BASE_IDX),
+		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
+}
+
+static struct audio *create_audio(
+		struct dc_context *ctx, unsigned int inst)
+{
+	return dce_audio_create(ctx, inst,
+			&audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+static struct stream_encoder *dcn10_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dcn10_stream_encoder *enc1 =
+		kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
+
+	if (!enc1)
+		return NULL;
+
+	dcn10_stream_encoder_construct(enc1, ctx, ctx->dc_bios, eng_id,
+					&stream_enc_regs[eng_id],
+					&se_shift, &se_mask);
+	return &enc1->base;
+}
+
+static const struct dce_hwseq_registers hwseq_reg = {
+		HWSEQ_DCN1_REG_LIST()
+};
+
+static const struct dce_hwseq_shift hwseq_shift = {
+		HWSEQ_DCN1_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+		HWSEQ_DCN1_MASK_SH_LIST(_MASK)
+};
+
+static struct dce_hwseq *dcn10_hwseq_create(
+	struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &hwseq_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+		hws->wa.DEGVIDCN10_253 = true;
+		hws->wa.false_optc_underflow = true;
+		hws->wa.DEGVIDCN10_254 = true;
+
+		if ((ctx->asic_id.chip_family == FAMILY_RV) &&
+			ASICREV_IS_RAVEN2(ctx->asic_id.hw_internal_rev))
+			switch (ctx->asic_id.pci_revision_id) {
+			case PRID_POLLOCK_94:
+			case PRID_POLLOCK_95:
+			case PRID_POLLOCK_E9:
+			case PRID_POLLOCK_EA:
+			case PRID_POLLOCK_EB:
+				hws->wa.wait_hubpret_read_start_during_mpo_transition = true;
+				break;
+			default:
+				hws->wa.wait_hubpret_read_start_during_mpo_transition = false;
+				break;
+			}
+	}
+	return hws;
+}
+
+static const struct resource_create_funcs res_create_funcs = {
+	.read_dce_straps = read_dce_straps,
+	.create_audio = create_audio,
+	.create_stream_encoder = dcn10_stream_encoder_create,
+	.create_hwseq = dcn10_hwseq_create,
+};
+
+static void dcn10_clock_source_destroy(struct clock_source **clk_src)
+{
+	kfree(TO_DCE110_CLK_SRC(*clk_src));
+	*clk_src = NULL;
+}
+
+static struct pp_smu_funcs *dcn10_pp_smu_create(struct dc_context *ctx)
+{
+	struct pp_smu_funcs *pp_smu = kzalloc(sizeof(*pp_smu), GFP_KERNEL);
+
+	if (!pp_smu)
+		return pp_smu;
+
+	dm_pp_get_funcs(ctx, pp_smu);
+	return pp_smu;
+}
+
+static void dcn10_resource_destruct(struct dcn10_resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->base.stream_enc_count; i++) {
+		if (pool->base.stream_enc[i] != NULL) {
+			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
+			pool->base.stream_enc[i] = NULL;
+		}
+	}
+
+	if (pool->base.mpc != NULL) {
+		kfree(TO_DCN10_MPC(pool->base.mpc));
+		pool->base.mpc = NULL;
+	}
+
+	kfree(pool->base.hubbub);
+	pool->base.hubbub = NULL;
+
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (pool->base.opps[i] != NULL)
+			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
+
+		if (pool->base.dpps[i] != NULL)
+			dcn10_dpp_destroy(&pool->base.dpps[i]);
+
+		if (pool->base.ipps[i] != NULL)
+			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
+
+		if (pool->base.hubps[i] != NULL) {
+			kfree(TO_DCN10_HUBP(pool->base.hubps[i]));
+			pool->base.hubps[i] = NULL;
+		}
+
+		if (pool->base.irqs != NULL) {
+			dal_irq_service_destroy(&pool->base.irqs);
+		}
+
+		if (pool->base.timing_generators[i] != NULL)	{
+			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
+			pool->base.timing_generators[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+		kfree(pool->base.hw_i2cs[i]);
+		pool->base.hw_i2cs[i] = NULL;
+		kfree(pool->base.sw_i2cs[i]);
+		pool->base.sw_i2cs[i] = NULL;
+	}
+
+	for (i = 0; i < pool->base.audio_count; i++) {
+		if (pool->base.audios[i])
+			dce_aud_destroy(&pool->base.audios[i]);
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] != NULL) {
+			dcn10_clock_source_destroy(&pool->base.clock_sources[i]);
+			pool->base.clock_sources[i] = NULL;
+		}
+	}
+
+	if (pool->base.dp_clock_source != NULL) {
+		dcn10_clock_source_destroy(&pool->base.dp_clock_source);
+		pool->base.dp_clock_source = NULL;
+	}
+
+	if (pool->base.abm != NULL)
+		dce_abm_destroy(&pool->base.abm);
+
+	if (pool->base.dmcu != NULL)
+		dce_dmcu_destroy(&pool->base.dmcu);
+
+	kfree(pool->base.pp_smu);
+}
+
+static struct hubp *dcn10_hubp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn10_hubp *hubp1 =
+		kzalloc(sizeof(struct dcn10_hubp), GFP_KERNEL);
+
+	if (!hubp1)
+		return NULL;
+
+	dcn10_hubp_construct(hubp1, ctx, inst,
+			     &hubp_regs[inst], &hubp_shift, &hubp_mask);
+	return &hubp1->base;
+}
+
+static void get_pixel_clock_parameters(
+	const struct pipe_ctx *pipe_ctx,
+	struct pixel_clk_params *pixel_clk_params)
+{
+	const struct dc_stream_state *stream = pipe_ctx->stream;
+	pixel_clk_params->requested_pix_clk_100hz = stream->timing.pix_clk_100hz;
+	pixel_clk_params->encoder_object_id = stream->link->link_enc->id;
+	pixel_clk_params->signal_type = pipe_ctx->stream->signal;
+	pixel_clk_params->controller_id = pipe_ctx->stream_res.tg->inst + 1;
+	/* TODO: un-hardcode*/
+	pixel_clk_params->requested_sym_clk = LINK_RATE_LOW *
+		LINK_RATE_REF_FREQ_IN_KHZ;
+	pixel_clk_params->flags.ENABLE_SS = 0;
+	pixel_clk_params->color_depth =
+		stream->timing.display_color_depth;
+	pixel_clk_params->flags.DISPLAY_BLANKED = 1;
+	pixel_clk_params->pixel_encoding = stream->timing.pixel_encoding;
+
+	if (stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR422)
+		pixel_clk_params->color_depth = COLOR_DEPTH_888;
+
+	if (stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR420)
+		pixel_clk_params->requested_pix_clk_100hz  /= 2;
+	if (stream->timing.timing_3d_format == TIMING_3D_FORMAT_HW_FRAME_PACKING)
+		pixel_clk_params->requested_pix_clk_100hz *= 2;
+
+}
+
+static void build_clamping_params(struct dc_stream_state *stream)
+{
+	stream->clamping.clamping_level = CLAMPING_FULL_RANGE;
+	stream->clamping.c_depth = stream->timing.display_color_depth;
+	stream->clamping.pixel_encoding = stream->timing.pixel_encoding;
+}
+
+static void build_pipe_hw_param(struct pipe_ctx *pipe_ctx)
+{
+
+	get_pixel_clock_parameters(pipe_ctx, &pipe_ctx->stream_res.pix_clk_params);
+
+	pipe_ctx->clock_source->funcs->get_pix_clk_dividers(
+		pipe_ctx->clock_source,
+		&pipe_ctx->stream_res.pix_clk_params,
+		&pipe_ctx->pll_settings);
+
+	pipe_ctx->stream->clamping.pixel_encoding = pipe_ctx->stream->timing.pixel_encoding;
+
+	resource_build_bit_depth_reduction_params(pipe_ctx->stream,
+					&pipe_ctx->stream->bit_depth_params);
+	build_clamping_params(pipe_ctx->stream);
+}
+
+static enum dc_status build_mapped_resource(
+		const struct dc *dc,
+		struct dc_state *context,
+		struct dc_stream_state *stream)
+{
+	struct pipe_ctx *pipe_ctx = resource_get_otg_master_for_stream(&context->res_ctx, stream);
+
+	if (!pipe_ctx)
+		return DC_ERROR_UNEXPECTED;
+
+	build_pipe_hw_param(pipe_ctx);
+	return DC_OK;
+}
+
+static enum dc_status dcn10_add_stream_to_ctx(
+		struct dc *dc,
+		struct dc_state *new_ctx,
+		struct dc_stream_state *dc_stream)
+{
+	enum dc_status result = DC_ERROR_UNEXPECTED;
+
+	result = resource_map_pool_resources(dc, new_ctx, dc_stream);
+
+	if (result == DC_OK)
+		result = resource_map_phy_clock_resources(dc, new_ctx, dc_stream);
+
+
+	if (result == DC_OK)
+		result = build_mapped_resource(dc, new_ctx, dc_stream);
+
+	return result;
+}
+
+static struct pipe_ctx *dcn10_acquire_free_pipe_for_layer(
+		const struct dc_state *cur_ctx,
+		struct dc_state *new_ctx,
+		const struct resource_pool *pool,
+		const struct pipe_ctx *opp_head_pipe)
+{
+	struct resource_context *res_ctx = &new_ctx->res_ctx;
+	struct pipe_ctx *head_pipe = resource_get_otg_master_for_stream(res_ctx, opp_head_pipe->stream);
+	struct pipe_ctx *idle_pipe = resource_find_free_secondary_pipe_legacy(res_ctx, pool, head_pipe);
+
+	if (!head_pipe) {
+		ASSERT(0);
+		return NULL;
+	}
+
+	if (!idle_pipe)
+		return NULL;
+
+	idle_pipe->stream = head_pipe->stream;
+	idle_pipe->stream_res.tg = head_pipe->stream_res.tg;
+	idle_pipe->stream_res.abm = head_pipe->stream_res.abm;
+	idle_pipe->stream_res.opp = head_pipe->stream_res.opp;
+
+	idle_pipe->plane_res.hubp = pool->hubps[idle_pipe->pipe_idx];
+	idle_pipe->plane_res.ipp = pool->ipps[idle_pipe->pipe_idx];
+	idle_pipe->plane_res.dpp = pool->dpps[idle_pipe->pipe_idx];
+	idle_pipe->plane_res.mpcc_inst = pool->dpps[idle_pipe->pipe_idx]->inst;
+
+	return idle_pipe;
+}
+
+static bool dcn10_get_dcc_compression_cap(const struct dc *dc,
+		const struct dc_dcc_surface_param *input,
+		struct dc_surface_dcc_cap *output)
+{
+	return dc->res_pool->hubbub->funcs->get_dcc_compression_cap(
+			dc->res_pool->hubbub,
+			input,
+			output);
+}
+
+static void dcn10_destroy_resource_pool(struct resource_pool **pool)
+{
+	struct dcn10_resource_pool *dcn10_pool = TO_DCN10_RES_POOL(*pool);
+
+	dcn10_resource_destruct(dcn10_pool);
+	kfree(dcn10_pool);
+	*pool = NULL;
+}
+
+static bool dcn10_validate_bandwidth(
+		struct dc *dc,
+		struct dc_state *context,
+		bool fast_validate)
+{
+	bool voltage_supported;
+
+	DC_FP_START();
+	voltage_supported = dcn_validate_bandwidth(dc, context, fast_validate);
+	DC_FP_END();
+
+	return voltage_supported;
+}
+
+static enum dc_status dcn10_validate_plane(const struct dc_plane_state *plane_state, struct dc_caps *caps)
+{
+	if (plane_state->format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN
+			&& caps->max_video_width != 0
+			&& plane_state->src_rect.width > caps->max_video_width)
+		return DC_FAIL_SURFACE_VALIDATE;
+
+	return DC_OK;
+}
+
+static enum dc_status dcn10_validate_global(struct dc *dc, struct dc_state *context)
+{
+	int i, j;
+	bool video_down_scaled = false;
+	bool video_large = false;
+	bool desktop_large = false;
+	bool dcc_disabled = false;
+	bool mpo_enabled = false;
+
+	for (i = 0; i < context->stream_count; i++) {
+		if (context->stream_status[i].plane_count == 0)
+			continue;
+
+		if (context->stream_status[i].plane_count > 2)
+			return DC_FAIL_UNSUPPORTED_1;
+
+		if (context->stream_status[i].plane_count > 1)
+			mpo_enabled = true;
+
+		for (j = 0; j < context->stream_status[i].plane_count; j++) {
+			struct dc_plane_state *plane =
+				context->stream_status[i].plane_states[j];
+
+
+			if (plane->format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN) {
+
+				if (plane->src_rect.width > plane->dst_rect.width ||
+						plane->src_rect.height > plane->dst_rect.height)
+					video_down_scaled = true;
+
+				if (plane->src_rect.width >= 3840)
+					video_large = true;
+
+			} else {
+				if (plane->src_rect.width >= 3840)
+					desktop_large = true;
+				if (!plane->dcc.enable)
+					dcc_disabled = true;
+			}
+		}
+	}
+
+	/* Disable MPO in multi-display configurations. */
+	if (context->stream_count > 1 && mpo_enabled)
+		return DC_FAIL_UNSUPPORTED_1;
+
+	/*
+	 * Workaround: On DCN10 there is UMC issue that causes underflow when
+	 * playing 4k video on 4k desktop with video downscaled and single channel
+	 * memory
+	 */
+	if (video_large && desktop_large && video_down_scaled && dcc_disabled &&
+			dc->dcn_soc->number_of_channels == 1)
+		return DC_FAIL_SURFACE_VALIDATE;
+
+	return DC_OK;
+}
+
+static enum dc_status dcn10_patch_unknown_plane_state(struct dc_plane_state *plane_state)
+{
+	enum surface_pixel_format surf_pix_format = plane_state->format;
+	unsigned int bpp = resource_pixel_format_to_bpp(surf_pix_format);
+
+	enum swizzle_mode_values swizzle = DC_SW_LINEAR;
+
+	if (bpp == 64)
+		swizzle = DC_SW_64KB_D;
+	else
+		swizzle = DC_SW_64KB_S;
+
+	plane_state->tiling_info.gfx9.swizzle = swizzle;
+	return DC_OK;
+}
+
+struct stream_encoder *dcn10_find_first_free_match_stream_enc_for_link(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		struct dc_stream_state *stream)
+{
+	int i;
+	int j = -1;
+	struct dc_link *link = stream->link;
+
+	for (i = 0; i < pool->stream_enc_count; i++) {
+		if (!res_ctx->is_stream_enc_acquired[i] &&
+				pool->stream_enc[i]) {
+			/* Store first available for MST second display
+			 * in daisy chain use case
+			 */
+			j = i;
+			if (link->ep_type == DISPLAY_ENDPOINT_PHY && pool->stream_enc[i]->id ==
+					link->link_enc->preferred_engine)
+				return pool->stream_enc[i];
+		}
+	}
+
+	/*
+	 * For CZ and later, we can allow DIG FE and BE to differ for all display types
+	 */
+
+	if (j >= 0)
+		return pool->stream_enc[j];
+
+	return NULL;
+}
+
+static const struct dc_cap_funcs cap_funcs = {
+	.get_dcc_compression_cap = dcn10_get_dcc_compression_cap
+};
+
+static const struct resource_funcs dcn10_res_pool_funcs = {
+	.destroy = dcn10_destroy_resource_pool,
+	.link_enc_create = dcn10_link_encoder_create,
+	.panel_cntl_create = dcn10_panel_cntl_create,
+	.validate_bandwidth = dcn10_validate_bandwidth,
+	.acquire_free_pipe_as_secondary_dpp_pipe = dcn10_acquire_free_pipe_for_layer,
+	.validate_plane = dcn10_validate_plane,
+	.validate_global = dcn10_validate_global,
+	.add_stream_to_ctx = dcn10_add_stream_to_ctx,
+	.patch_unknown_plane_state = dcn10_patch_unknown_plane_state,
+	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link
+};
+
+static uint32_t read_pipe_fuses(struct dc_context *ctx)
+{
+	uint32_t value = dm_read_reg_soc15(ctx, mmCC_DC_PIPE_DIS, 0);
+	/* RV1 support max 4 pipes */
+	value = value & 0xf;
+	return value;
+}
+
+static bool verify_clock_values(struct dm_pp_clock_levels_with_voltage *clks)
+{
+	int i;
+
+	if (clks->num_levels == 0)
+		return false;
+
+	for (i = 0; i < clks->num_levels; i++)
+		/* Ensure that the result is sane */
+		if (clks->data[i].clocks_in_khz == 0)
+			return false;
+
+	return true;
+}
+
+static bool dcn10_resource_construct(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct dcn10_resource_pool *pool)
+{
+	int i;
+	int j;
+	struct dc_context *ctx = dc->ctx;
+	uint32_t pipe_fuses = read_pipe_fuses(ctx);
+	struct dm_pp_clock_levels_with_voltage fclks = {0}, dcfclks = {0};
+	int min_fclk_khz, min_dcfclk_khz, socclk_khz;
+	bool res;
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	if (ctx->dce_version == DCN_VERSION_1_01)
+		pool->base.res_cap = &rv2_res_cap;
+	else
+		pool->base.res_cap = &res_cap;
+	pool->base.funcs = &dcn10_res_pool_funcs;
+
+	/*
+	 * TODO fill in from actual raven resource when we create
+	 * more than virtual encoder
+	 */
+
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+
+	/* max pipe num for ASIC before check pipe fuses */
+	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
+
+	if (dc->ctx->dce_version == DCN_VERSION_1_01)
+		pool->base.pipe_count = 3;
+	dc->caps.max_video_width = 3840;
+	dc->caps.max_downscale_ratio = 200;
+	dc->caps.i2c_speed_in_khz = 100;
+	dc->caps.i2c_speed_in_khz_hdcp = 100; /*1.4 w/a not applied by default*/
+	dc->caps.max_cursor_size = 256;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.max_slave_planes = 1;
+	dc->caps.max_slave_yuv_planes = 1;
+	dc->caps.max_slave_rgb_planes = 0;
+	dc->caps.is_apu = true;
+	dc->caps.post_blend_color_processing = false;
+	dc->caps.extended_aux_timeout_support = false;
+
+	/* Raven DP PHY HBR2 eye diagram pattern is not stable. Use TP4 */
+	dc->caps.force_dp_tps4_for_cp2520 = true;
+
+	/* Color pipeline capabilities */
+	dc->caps.color.dpp.dcn_arch = 1;
+	dc->caps.color.dpp.input_lut_shared = 1;
+	dc->caps.color.dpp.icsc = 1;
+	dc->caps.color.dpp.dgam_ram = 1;
+	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
+	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.dpp.dgam_rom_caps.pq = 0;
+	dc->caps.color.dpp.dgam_rom_caps.hlg = 0;
+	dc->caps.color.dpp.post_csc = 0;
+	dc->caps.color.dpp.gamma_corr = 0;
+	dc->caps.color.dpp.dgam_rom_for_yuv = 1;
+
+	dc->caps.color.dpp.hw_3d_lut = 0;
+	dc->caps.color.dpp.ogam_ram = 1; // RGAM on DCN1
+	dc->caps.color.dpp.ogam_rom_caps.srgb = 1;
+	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 1;
+	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
+	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
+	dc->caps.color.dpp.ocsc = 1;
+
+	/* no post-blend color operations */
+	dc->caps.color.mpc.gamut_remap = 0;
+	dc->caps.color.mpc.num_3dluts = 0;
+	dc->caps.color.mpc.shared_3d_lut = 0;
+	dc->caps.color.mpc.ogam_ram = 0;
+	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
+	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
+	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
+	dc->caps.color.mpc.ocsc = 0;
+
+	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
+		dc->debug = debug_defaults_drv;
+	else
+		dc->debug = debug_defaults_diags;
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	pool->base.clock_sources[DCN10_CLK_SRC_PLL0] =
+			dcn10_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL0,
+				&clk_src_regs[0], false);
+	pool->base.clock_sources[DCN10_CLK_SRC_PLL1] =
+			dcn10_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL1,
+				&clk_src_regs[1], false);
+	pool->base.clock_sources[DCN10_CLK_SRC_PLL2] =
+			dcn10_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL2,
+				&clk_src_regs[2], false);
+
+	if (dc->ctx->dce_version == DCN_VERSION_1_0) {
+		pool->base.clock_sources[DCN10_CLK_SRC_PLL3] =
+				dcn10_clock_source_create(ctx, ctx->dc_bios,
+					CLOCK_SOURCE_COMBO_PHY_PLL3,
+					&clk_src_regs[3], false);
+	}
+
+	pool->base.clk_src_count = DCN10_CLK_SRC_TOTAL;
+
+	if (dc->ctx->dce_version == DCN_VERSION_1_01)
+		pool->base.clk_src_count = DCN101_CLK_SRC_TOTAL;
+
+	pool->base.dp_clock_source =
+			dcn10_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_ID_DP_DTO,
+				/* todo: not reuse phy_pll registers */
+				&clk_src_regs[0], true);
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto fail;
+		}
+	}
+
+	pool->base.dmcu = dcn10_dmcu_create(ctx,
+			&dmcu_regs,
+			&dmcu_shift,
+			&dmcu_mask);
+	if (pool->base.dmcu == NULL) {
+		dm_error("DC: failed to create dmcu!\n");
+		BREAK_TO_DEBUGGER();
+		goto fail;
+	}
+
+	pool->base.abm = dce_abm_create(ctx,
+			&abm_regs,
+			&abm_shift,
+			&abm_mask);
+	if (pool->base.abm == NULL) {
+		dm_error("DC: failed to create abm!\n");
+		BREAK_TO_DEBUGGER();
+		goto fail;
+	}
+
+	dml_init_instance(&dc->dml, &dcn1_0_soc, &dcn1_0_ip, DML_PROJECT_RAVEN1);
+	memcpy(dc->dcn_ip, &dcn10_ip_defaults, sizeof(dcn10_ip_defaults));
+	memcpy(dc->dcn_soc, &dcn10_soc_defaults, sizeof(dcn10_soc_defaults));
+
+	DC_FP_START();
+	dcn10_resource_construct_fp(dc);
+	DC_FP_END();
+
+	if (!dc->config.is_vmin_only_asic)
+		if (ASICREV_IS_RAVEN2(dc->ctx->asic_id.hw_internal_rev))
+			switch (dc->ctx->asic_id.pci_revision_id) {
+			case PRID_DALI_DE:
+			case PRID_DALI_DF:
+			case PRID_DALI_E3:
+			case PRID_DALI_E4:
+			case PRID_POLLOCK_94:
+			case PRID_POLLOCK_95:
+			case PRID_POLLOCK_E9:
+			case PRID_POLLOCK_EA:
+			case PRID_POLLOCK_EB:
+				dc->config.is_vmin_only_asic = true;
+				break;
+			default:
+				break;
+			}
+
+	pool->base.pp_smu = dcn10_pp_smu_create(ctx);
+
+	/*
+	 * Right now SMU/PPLIB and DAL all have the AZ D3 force PME notification *
+	 * implemented. So AZ D3 should work.For issue 197007.                   *
+	 */
+	if (pool->base.pp_smu != NULL
+			&& pool->base.pp_smu->rv_funcs.set_pme_wa_enable != NULL)
+		dc->debug.az_endpoint_mute_only = false;
+
+
+	if (!dc->debug.disable_pplib_clock_request) {
+		/*
+		 * TODO: This is not the proper way to obtain
+		 * fabric_and_dram_bandwidth, should be min(fclk, memclk).
+		 */
+		res = dm_pp_get_clock_levels_by_type_with_voltage(
+				ctx, DM_PP_CLOCK_TYPE_FCLK, &fclks);
+
+		DC_FP_START();
+
+		if (res)
+			res = verify_clock_values(&fclks);
+
+		if (res)
+			dcn_bw_update_from_pplib_fclks(dc, &fclks);
+		else
+			BREAK_TO_DEBUGGER();
+
+		DC_FP_END();
+
+		res = dm_pp_get_clock_levels_by_type_with_voltage(
+			ctx, DM_PP_CLOCK_TYPE_DCFCLK, &dcfclks);
+
+		DC_FP_START();
+
+		if (res)
+			res = verify_clock_values(&dcfclks);
+
+		if (res)
+			dcn_bw_update_from_pplib_dcfclks(dc, &dcfclks);
+		else
+			BREAK_TO_DEBUGGER();
+
+		DC_FP_END();
+	}
+
+	dcn_bw_sync_calcs_and_dml(dc);
+	if (!dc->debug.disable_pplib_wm_range) {
+		dc->res_pool = &pool->base;
+		DC_FP_START();
+		dcn_get_soc_clks(
+			dc, &min_fclk_khz, &min_dcfclk_khz, &socclk_khz);
+		DC_FP_END();
+		dcn_bw_notify_pplib_of_wm_ranges(
+			dc, min_fclk_khz, min_dcfclk_khz, socclk_khz);
+	}
+
+	{
+		struct irq_service_init_data init_data;
+		init_data.ctx = dc->ctx;
+		pool->base.irqs = dal_irq_service_dcn10_create(&init_data);
+		if (!pool->base.irqs)
+			goto fail;
+	}
+
+	/* index to valid pipe resource  */
+	j = 0;
+	/* mem input -> ipp -> dpp -> opp -> TG */
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		/* if pipe is disabled, skip instance of HW pipe,
+		 * i.e, skip ASIC register instance
+		 */
+		if ((pipe_fuses & (1 << i)) != 0)
+			continue;
+
+		pool->base.hubps[j] = dcn10_hubp_create(ctx, i);
+		if (pool->base.hubps[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create memory input!\n");
+			goto fail;
+		}
+
+		pool->base.ipps[j] = dcn10_ipp_create(ctx, i);
+		if (pool->base.ipps[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create input pixel processor!\n");
+			goto fail;
+		}
+
+		pool->base.dpps[j] = dcn10_dpp_create(ctx, i);
+		if (pool->base.dpps[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create dpp!\n");
+			goto fail;
+		}
+
+		pool->base.opps[j] = dcn10_opp_create(ctx, i);
+		if (pool->base.opps[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create output pixel processor!\n");
+			goto fail;
+		}
+
+		pool->base.timing_generators[j] = dcn10_timing_generator_create(
+				ctx, i);
+		if (pool->base.timing_generators[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto fail;
+		}
+		/* check next valid pipe */
+		j++;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.engines[i] = dcn10_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto fail;
+		}
+		pool->base.hw_i2cs[i] = dcn10_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create hw i2c!!\n");
+			goto fail;
+		}
+		pool->base.sw_i2cs[i] = NULL;
+	}
+
+	/* valid pipe num */
+	pool->base.pipe_count = j;
+	pool->base.timing_generator_count = j;
+
+	/* within dml lib, it is hard code to 4. If ASIC pipe is fused,
+	 * the value may be changed
+	 */
+	dc->dml.ip.max_num_dpp = pool->base.pipe_count;
+	dc->dcn_ip->max_num_dpp = pool->base.pipe_count;
+
+	pool->base.mpc = dcn10_mpc_create(ctx);
+	if (pool->base.mpc == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mpc!\n");
+		goto fail;
+	}
+
+	pool->base.hubbub = dcn10_hubbub_create(ctx);
+	if (pool->base.hubbub == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create hubbub!\n");
+		goto fail;
+	}
+
+	if (!resource_construct(num_virtual_links, dc, &pool->base,
+			&res_create_funcs))
+		goto fail;
+
+	dcn10_hw_sequencer_construct(dc);
+	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->cap_funcs = cap_funcs;
+
+	return true;
+
+fail:
+
+	dcn10_resource_destruct(pool);
+
+	return false;
+}
+
+struct resource_pool *dcn10_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc)
+{
+	struct dcn10_resource_pool *pool =
+		kzalloc(sizeof(struct dcn10_resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dcn10_resource_construct(init_data->num_virtual_links, dc, pool))
+		return &pool->base;
+
+	kfree(pool);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.h
new file mode 100644
index 000000000000..bf8e33cd8147
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.h
@@ -0,0 +1,56 @@
+/*
+* Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_RESOURCE_DCN10_H__
+#define __DC_RESOURCE_DCN10_H__
+
+#include "core_types.h"
+#include "dml/dcn10/dcn10_fpu.h"
+
+#define TO_DCN10_RES_POOL(pool)\
+	container_of(pool, struct dcn10_resource_pool, base)
+
+struct dc;
+struct resource_pool;
+struct _vcs_dpi_display_pipe_params_st;
+
+extern struct _vcs_dpi_ip_params_st dcn1_0_ip;
+extern struct _vcs_dpi_soc_bounding_box_st dcn1_0_soc;
+
+struct dcn10_resource_pool {
+	struct resource_pool base;
+};
+struct resource_pool *dcn10_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc);
+
+struct stream_encoder *dcn10_find_first_free_match_stream_enc_for_link(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		struct dc_stream_state *stream);
+
+
+#endif /* __DC_RESOURCE_DCN10_H__ */
+
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c
new file mode 100644
index 000000000000..f04bb5b1471d
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c
@@ -0,0 +1,2789 @@
+/*
+* Copyright 2016 Advanced Micro Devices, Inc.
+ * Copyright 2019 Raptor Engineering, LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include <linux/slab.h>
+
+#include "dm_services.h"
+#include "dc.h"
+
+#include "dcn20/dcn20_init.h"
+
+#include "resource.h"
+#include "include/irq_service_interface.h"
+#include "dcn20/dcn20_resource.h"
+
+#include "dml/dcn20/dcn20_fpu.h"
+
+#include "dcn10/dcn10_hubp.h"
+#include "dcn10/dcn10_ipp.h"
+#include "dcn20/dcn20_hubbub.h"
+#include "dcn20/dcn20_mpc.h"
+#include "dcn20/dcn20_hubp.h"
+#include "irq/dcn20/irq_service_dcn20.h"
+#include "dcn20/dcn20_dpp.h"
+#include "dcn20/dcn20_optc.h"
+#include "dcn20/dcn20_hwseq.h"
+#include "dce110/dce110_hwseq.h"
+#include "dcn10/dcn10_resource.h"
+#include "dcn20/dcn20_opp.h"
+
+#include "dcn20/dcn20_dsc.h"
+
+#include "dcn20/dcn20_link_encoder.h"
+#include "dcn20/dcn20_stream_encoder.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_hwseq.h"
+#include "virtual/virtual_stream_encoder.h"
+#include "dce110/dce110_resource.h"
+#include "dml/display_mode_vba.h"
+#include "dcn20/dcn20_dccg.h"
+#include "dcn20/dcn20_vmid.h"
+#include "dce/dce_panel_cntl.h"
+
+#include "navi10_ip_offset.h"
+
+#include "dcn/dcn_2_0_0_offset.h"
+#include "dcn/dcn_2_0_0_sh_mask.h"
+#include "dpcs/dpcs_2_0_0_offset.h"
+#include "dpcs/dpcs_2_0_0_sh_mask.h"
+
+#include "nbio/nbio_2_3_offset.h"
+
+#include "dcn20/dcn20_dwb.h"
+#include "dcn20/dcn20_mmhubbub.h"
+
+#include "mmhub/mmhub_2_0_0_offset.h"
+#include "mmhub/mmhub_2_0_0_sh_mask.h"
+
+#include "reg_helper.h"
+#include "dce/dce_abm.h"
+#include "dce/dce_dmcu.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_i2c.h"
+#include "vm_helper.h"
+#include "link_enc_cfg.h"
+
+#include "amdgpu_socbb.h"
+
+#include "link.h"
+#define DC_LOGGER_INIT(logger)
+
+#ifndef mmDP0_DP_DPHY_INTERNAL_CTRL
+	#define mmDP0_DP_DPHY_INTERNAL_CTRL		0x210f
+	#define mmDP0_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
+	#define mmDP1_DP_DPHY_INTERNAL_CTRL		0x220f
+	#define mmDP1_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
+	#define mmDP2_DP_DPHY_INTERNAL_CTRL		0x230f
+	#define mmDP2_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
+	#define mmDP3_DP_DPHY_INTERNAL_CTRL		0x240f
+	#define mmDP3_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
+	#define mmDP4_DP_DPHY_INTERNAL_CTRL		0x250f
+	#define mmDP4_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
+	#define mmDP5_DP_DPHY_INTERNAL_CTRL		0x260f
+	#define mmDP5_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
+	#define mmDP6_DP_DPHY_INTERNAL_CTRL		0x270f
+	#define mmDP6_DP_DPHY_INTERNAL_CTRL_BASE_IDX	2
+#endif
+
+
+enum dcn20_clk_src_array_id {
+	DCN20_CLK_SRC_PLL0,
+	DCN20_CLK_SRC_PLL1,
+	DCN20_CLK_SRC_PLL2,
+	DCN20_CLK_SRC_PLL3,
+	DCN20_CLK_SRC_PLL4,
+	DCN20_CLK_SRC_PLL5,
+	DCN20_CLK_SRC_TOTAL
+};
+
+/* begin *********************
+ * macros to expend register list macro defined in HW object header file */
+
+/* DCN */
+#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
+
+#define BASE(seg) BASE_INNER(seg)
+
+#define SR(reg_name)\
+		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) +  \
+					mm ## reg_name
+
+#define SRI(reg_name, block, id)\
+	.reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define SRI2_DWB(reg_name, block, id)\
+	.reg_name = BASE(mm ## reg_name ## _BASE_IDX) + \
+					mm ## reg_name
+#define SF_DWB(reg_name, field_name, post_fix)\
+	.field_name = reg_name ## __ ## field_name ## post_fix
+
+#define SF_DWB2(reg_name, block, id, field_name, post_fix)	\
+	.field_name = reg_name ## __ ## field_name ## post_fix
+
+#define SRIR(var_name, reg_name, block, id)\
+	.var_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define SRII(reg_name, block, id)\
+	.reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define DCCG_SRII(reg_name, block, id)\
+	.block ## _ ## reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define VUPDATE_SRII(reg_name, block, id)\
+	.reg_name[id] = BASE(mm ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
+					mm ## reg_name ## _ ## block ## id
+
+/* NBIO */
+#define NBIO_BASE_INNER(seg) \
+	NBIO_BASE__INST0_SEG ## seg
+
+#define NBIO_BASE(seg) \
+	NBIO_BASE_INNER(seg)
+
+#define NBIO_SR(reg_name)\
+		.reg_name = NBIO_BASE(mm ## reg_name ## _BASE_IDX) + \
+					mm ## reg_name
+
+/* MMHUB */
+#define MMHUB_BASE_INNER(seg) \
+	MMHUB_BASE__INST0_SEG ## seg
+
+#define MMHUB_BASE(seg) \
+	MMHUB_BASE_INNER(seg)
+
+#define MMHUB_SR(reg_name)\
+		.reg_name = MMHUB_BASE(mmMM ## reg_name ## _BASE_IDX) + \
+					mmMM ## reg_name
+
+static const struct bios_registers bios_regs = {
+		NBIO_SR(BIOS_SCRATCH_3),
+		NBIO_SR(BIOS_SCRATCH_6)
+};
+
+#define clk_src_regs(index, pllid)\
+[index] = {\
+	CS_COMMON_REG_LIST_DCN2_0(index, pllid),\
+}
+
+static const struct dce110_clk_src_regs clk_src_regs[] = {
+	clk_src_regs(0, A),
+	clk_src_regs(1, B),
+	clk_src_regs(2, C),
+	clk_src_regs(3, D),
+	clk_src_regs(4, E),
+	clk_src_regs(5, F)
+};
+
+static const struct dce110_clk_src_shift cs_shift = {
+		CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
+};
+
+static const struct dce110_clk_src_mask cs_mask = {
+		CS_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
+};
+
+static const struct dce_dmcu_registers dmcu_regs = {
+		DMCU_DCN10_REG_LIST()
+};
+
+static const struct dce_dmcu_shift dmcu_shift = {
+		DMCU_MASK_SH_LIST_DCN10(__SHIFT)
+};
+
+static const struct dce_dmcu_mask dmcu_mask = {
+		DMCU_MASK_SH_LIST_DCN10(_MASK)
+};
+
+static const struct dce_abm_registers abm_regs = {
+		ABM_DCN20_REG_LIST()
+};
+
+static const struct dce_abm_shift abm_shift = {
+		ABM_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dce_abm_mask abm_mask = {
+		ABM_MASK_SH_LIST_DCN20(_MASK)
+};
+
+#define audio_regs(id)\
+[id] = {\
+		AUD_COMMON_REG_LIST(id)\
+}
+
+static const struct dce_audio_registers audio_regs[] = {
+	audio_regs(0),
+	audio_regs(1),
+	audio_regs(2),
+	audio_regs(3),
+	audio_regs(4),
+	audio_regs(5),
+	audio_regs(6),
+};
+
+#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
+		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
+
+static const struct dce_audio_shift audio_shift = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_audio_mask audio_mask = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+#define stream_enc_regs(id)\
+[id] = {\
+	SE_DCN2_REG_LIST(id)\
+}
+
+static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
+	stream_enc_regs(0),
+	stream_enc_regs(1),
+	stream_enc_regs(2),
+	stream_enc_regs(3),
+	stream_enc_regs(4),
+	stream_enc_regs(5),
+};
+
+static const struct dcn10_stream_encoder_shift se_shift = {
+		SE_COMMON_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn10_stream_encoder_mask se_mask = {
+		SE_COMMON_MASK_SH_LIST_DCN20(_MASK)
+};
+
+
+#define aux_regs(id)\
+[id] = {\
+	DCN2_AUX_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
+		aux_regs(0),
+		aux_regs(1),
+		aux_regs(2),
+		aux_regs(3),
+		aux_regs(4),
+		aux_regs(5)
+};
+
+#define hpd_regs(id)\
+[id] = {\
+	HPD_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
+		hpd_regs(0),
+		hpd_regs(1),
+		hpd_regs(2),
+		hpd_regs(3),
+		hpd_regs(4),
+		hpd_regs(5)
+};
+
+#define link_regs(id, phyid)\
+[id] = {\
+	LE_DCN10_REG_LIST(id), \
+	UNIPHY_DCN2_REG_LIST(phyid), \
+	DPCS_DCN2_REG_LIST(id), \
+	SRI(DP_DPHY_INTERNAL_CTRL, DP, id) \
+}
+
+static const struct dcn10_link_enc_registers link_enc_regs[] = {
+	link_regs(0, A),
+	link_regs(1, B),
+	link_regs(2, C),
+	link_regs(3, D),
+	link_regs(4, E),
+	link_regs(5, F)
+};
+
+static const struct dcn10_link_enc_shift le_shift = {
+	LINK_ENCODER_MASK_SH_LIST_DCN20(__SHIFT),\
+	DPCS_DCN2_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn10_link_enc_mask le_mask = {
+	LINK_ENCODER_MASK_SH_LIST_DCN20(_MASK),\
+	DPCS_DCN2_MASK_SH_LIST(_MASK)
+};
+
+static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
+	{ DCN_PANEL_CNTL_REG_LIST() }
+};
+
+static const struct dce_panel_cntl_shift panel_cntl_shift = {
+	DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_panel_cntl_mask panel_cntl_mask = {
+	DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
+};
+
+#define ipp_regs(id)\
+[id] = {\
+	IPP_REG_LIST_DCN20(id),\
+}
+
+static const struct dcn10_ipp_registers ipp_regs[] = {
+	ipp_regs(0),
+	ipp_regs(1),
+	ipp_regs(2),
+	ipp_regs(3),
+	ipp_regs(4),
+	ipp_regs(5),
+};
+
+static const struct dcn10_ipp_shift ipp_shift = {
+		IPP_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn10_ipp_mask ipp_mask = {
+		IPP_MASK_SH_LIST_DCN20(_MASK),
+};
+
+#define opp_regs(id)\
+[id] = {\
+	OPP_REG_LIST_DCN20(id),\
+}
+
+static const struct dcn20_opp_registers opp_regs[] = {
+	opp_regs(0),
+	opp_regs(1),
+	opp_regs(2),
+	opp_regs(3),
+	opp_regs(4),
+	opp_regs(5),
+};
+
+static const struct dcn20_opp_shift opp_shift = {
+		OPP_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn20_opp_mask opp_mask = {
+		OPP_MASK_SH_LIST_DCN20(_MASK)
+};
+
+#define aux_engine_regs(id)\
+[id] = {\
+	AUX_COMMON_REG_LIST0(id), \
+	.AUXN_IMPCAL = 0, \
+	.AUXP_IMPCAL = 0, \
+	.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1),
+		aux_engine_regs(2),
+		aux_engine_regs(3),
+		aux_engine_regs(4),
+		aux_engine_regs(5)
+};
+
+#define tf_regs(id)\
+[id] = {\
+	TF_REG_LIST_DCN20(id),\
+	TF_REG_LIST_DCN20_COMMON_APPEND(id),\
+}
+
+static const struct dcn2_dpp_registers tf_regs[] = {
+	tf_regs(0),
+	tf_regs(1),
+	tf_regs(2),
+	tf_regs(3),
+	tf_regs(4),
+	tf_regs(5),
+};
+
+static const struct dcn2_dpp_shift tf_shift = {
+		TF_REG_LIST_SH_MASK_DCN20(__SHIFT),
+		TF_DEBUG_REG_LIST_SH_DCN20
+};
+
+static const struct dcn2_dpp_mask tf_mask = {
+		TF_REG_LIST_SH_MASK_DCN20(_MASK),
+		TF_DEBUG_REG_LIST_MASK_DCN20
+};
+
+#define dwbc_regs_dcn2(id)\
+[id] = {\
+	DWBC_COMMON_REG_LIST_DCN2_0(id),\
+		}
+
+static const struct dcn20_dwbc_registers dwbc20_regs[] = {
+	dwbc_regs_dcn2(0),
+};
+
+static const struct dcn20_dwbc_shift dwbc20_shift = {
+	DWBC_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
+};
+
+static const struct dcn20_dwbc_mask dwbc20_mask = {
+	DWBC_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
+};
+
+#define mcif_wb_regs_dcn2(id)\
+[id] = {\
+	MCIF_WB_COMMON_REG_LIST_DCN2_0(id),\
+		}
+
+static const struct dcn20_mmhubbub_registers mcif_wb20_regs[] = {
+	mcif_wb_regs_dcn2(0),
+};
+
+static const struct dcn20_mmhubbub_shift mcif_wb20_shift = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
+};
+
+static const struct dcn20_mmhubbub_mask mcif_wb20_mask = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
+};
+
+static const struct dcn20_mpc_registers mpc_regs = {
+		MPC_REG_LIST_DCN2_0(0),
+		MPC_REG_LIST_DCN2_0(1),
+		MPC_REG_LIST_DCN2_0(2),
+		MPC_REG_LIST_DCN2_0(3),
+		MPC_REG_LIST_DCN2_0(4),
+		MPC_REG_LIST_DCN2_0(5),
+		MPC_OUT_MUX_REG_LIST_DCN2_0(0),
+		MPC_OUT_MUX_REG_LIST_DCN2_0(1),
+		MPC_OUT_MUX_REG_LIST_DCN2_0(2),
+		MPC_OUT_MUX_REG_LIST_DCN2_0(3),
+		MPC_OUT_MUX_REG_LIST_DCN2_0(4),
+		MPC_OUT_MUX_REG_LIST_DCN2_0(5),
+		MPC_DBG_REG_LIST_DCN2_0()
+};
+
+static const struct dcn20_mpc_shift mpc_shift = {
+	MPC_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT),
+	MPC_DEBUG_REG_LIST_SH_DCN20
+};
+
+static const struct dcn20_mpc_mask mpc_mask = {
+	MPC_COMMON_MASK_SH_LIST_DCN2_0(_MASK),
+	MPC_DEBUG_REG_LIST_MASK_DCN20
+};
+
+#define tg_regs(id)\
+[id] = {TG_COMMON_REG_LIST_DCN2_0(id)}
+
+
+static const struct dcn_optc_registers tg_regs[] = {
+	tg_regs(0),
+	tg_regs(1),
+	tg_regs(2),
+	tg_regs(3),
+	tg_regs(4),
+	tg_regs(5)
+};
+
+static const struct dcn_optc_shift tg_shift = {
+	TG_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
+};
+
+static const struct dcn_optc_mask tg_mask = {
+	TG_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
+};
+
+#define hubp_regs(id)\
+[id] = {\
+	HUBP_REG_LIST_DCN20(id)\
+}
+
+static const struct dcn_hubp2_registers hubp_regs[] = {
+		hubp_regs(0),
+		hubp_regs(1),
+		hubp_regs(2),
+		hubp_regs(3),
+		hubp_regs(4),
+		hubp_regs(5)
+};
+
+static const struct dcn_hubp2_shift hubp_shift = {
+		HUBP_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn_hubp2_mask hubp_mask = {
+		HUBP_MASK_SH_LIST_DCN20(_MASK)
+};
+
+static const struct dcn_hubbub_registers hubbub_reg = {
+		HUBBUB_REG_LIST_DCN20(0)
+};
+
+static const struct dcn_hubbub_shift hubbub_shift = {
+		HUBBUB_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn_hubbub_mask hubbub_mask = {
+		HUBBUB_MASK_SH_LIST_DCN20(_MASK)
+};
+
+#define vmid_regs(id)\
+[id] = {\
+		DCN20_VMID_REG_LIST(id)\
+}
+
+static const struct dcn_vmid_registers vmid_regs[] = {
+	vmid_regs(0),
+	vmid_regs(1),
+	vmid_regs(2),
+	vmid_regs(3),
+	vmid_regs(4),
+	vmid_regs(5),
+	vmid_regs(6),
+	vmid_regs(7),
+	vmid_regs(8),
+	vmid_regs(9),
+	vmid_regs(10),
+	vmid_regs(11),
+	vmid_regs(12),
+	vmid_regs(13),
+	vmid_regs(14),
+	vmid_regs(15)
+};
+
+static const struct dcn20_vmid_shift vmid_shifts = {
+		DCN20_VMID_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn20_vmid_mask vmid_masks = {
+		DCN20_VMID_MASK_SH_LIST(_MASK)
+};
+
+static const struct dce110_aux_registers_shift aux_shift = {
+		DCN_AUX_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+		DCN_AUX_MASK_SH_LIST(_MASK)
+};
+
+static int map_transmitter_id_to_phy_instance(
+	enum transmitter transmitter)
+{
+	switch (transmitter) {
+	case TRANSMITTER_UNIPHY_A:
+		return 0;
+	break;
+	case TRANSMITTER_UNIPHY_B:
+		return 1;
+	break;
+	case TRANSMITTER_UNIPHY_C:
+		return 2;
+	break;
+	case TRANSMITTER_UNIPHY_D:
+		return 3;
+	break;
+	case TRANSMITTER_UNIPHY_E:
+		return 4;
+	break;
+	case TRANSMITTER_UNIPHY_F:
+		return 5;
+	break;
+	default:
+		ASSERT(0);
+		return 0;
+	}
+}
+
+#define dsc_regsDCN20(id)\
+[id] = {\
+	DSC_REG_LIST_DCN20(id)\
+}
+
+static const struct dcn20_dsc_registers dsc_regs[] = {
+	dsc_regsDCN20(0),
+	dsc_regsDCN20(1),
+	dsc_regsDCN20(2),
+	dsc_regsDCN20(3),
+	dsc_regsDCN20(4),
+	dsc_regsDCN20(5)
+};
+
+static const struct dcn20_dsc_shift dsc_shift = {
+	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
+};
+
+static const struct dcn20_dsc_mask dsc_mask = {
+	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
+};
+
+static const struct dccg_registers dccg_regs = {
+		DCCG_REG_LIST_DCN2()
+};
+
+static const struct dccg_shift dccg_shift = {
+		DCCG_MASK_SH_LIST_DCN2(__SHIFT)
+};
+
+static const struct dccg_mask dccg_mask = {
+		DCCG_MASK_SH_LIST_DCN2(_MASK)
+};
+
+static const struct resource_caps res_cap_nv10 = {
+		.num_timing_generator = 6,
+		.num_opp = 6,
+		.num_video_plane = 6,
+		.num_audio = 7,
+		.num_stream_encoder = 6,
+		.num_pll = 6,
+		.num_dwb = 1,
+		.num_ddc = 6,
+		.num_vmid = 16,
+		.num_dsc = 6,
+};
+
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
+	.per_pixel_alpha = true,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = true,
+			.fp16 = true,
+			.p010 = true
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 16000,
+			.fp16 = 1
+	},
+
+	.max_downscale_factor = {
+			.argb8888 = 250,
+			.nv12 = 250,
+			.fp16 = 1
+	},
+	16,
+	16
+};
+static const struct resource_caps res_cap_nv14 = {
+		.num_timing_generator = 5,
+		.num_opp = 5,
+		.num_video_plane = 5,
+		.num_audio = 6,
+		.num_stream_encoder = 5,
+		.num_pll = 5,
+		.num_dwb = 1,
+		.num_ddc = 5,
+		.num_vmid = 16,
+		.num_dsc = 5,
+};
+
+static const struct dc_debug_options debug_defaults_drv = {
+		.disable_dmcu = false,
+		.force_abm_enable = false,
+		.timing_trace = false,
+		.clock_trace = true,
+		.disable_pplib_clock_request = true,
+		.pipe_split_policy = MPC_SPLIT_AVOID_MULT_DISP,
+		.force_single_disp_pipe_split = false,
+		.disable_dcc = DCC_ENABLE,
+		.vsr_support = true,
+		.performance_trace = false,
+		.max_downscale_src_width = 5120,/*upto 5K*/
+		.disable_pplib_wm_range = false,
+		.scl_reset_length10 = true,
+		.sanity_checks = false,
+		.underflow_assert_delay_us = 0xFFFFFFFF,
+		.enable_legacy_fast_update = true,
+		.using_dml2 = false,
+};
+
+void dcn20_dpp_destroy(struct dpp **dpp)
+{
+	kfree(TO_DCN20_DPP(*dpp));
+	*dpp = NULL;
+}
+
+struct dpp *dcn20_dpp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn20_dpp *dpp =
+		kzalloc(sizeof(struct dcn20_dpp), GFP_ATOMIC);
+
+	if (!dpp)
+		return NULL;
+
+	if (dpp2_construct(dpp, ctx, inst,
+			&tf_regs[inst], &tf_shift, &tf_mask))
+		return &dpp->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(dpp);
+	return NULL;
+}
+
+struct input_pixel_processor *dcn20_ipp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn10_ipp *ipp =
+		kzalloc(sizeof(struct dcn10_ipp), GFP_ATOMIC);
+
+	if (!ipp) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dcn20_ipp_construct(ipp, ctx, inst,
+			&ipp_regs[inst], &ipp_shift, &ipp_mask);
+	return &ipp->base;
+}
+
+
+struct output_pixel_processor *dcn20_opp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_opp *opp =
+		kzalloc(sizeof(struct dcn20_opp), GFP_ATOMIC);
+
+	if (!opp) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dcn20_opp_construct(opp, ctx, inst,
+			&opp_regs[inst], &opp_shift, &opp_mask);
+	return &opp->base;
+}
+
+struct dce_aux *dcn20_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_ATOMIC);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst],
+					&aux_mask,
+					&aux_shift,
+					ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
+
+static const struct dce_i2c_registers i2c_hw_regs[] = {
+		i2c_inst_regs(1),
+		i2c_inst_regs(2),
+		i2c_inst_regs(3),
+		i2c_inst_regs(4),
+		i2c_inst_regs(5),
+		i2c_inst_regs(6),
+};
+
+static const struct dce_i2c_shift i2c_shifts = {
+		I2C_COMMON_MASK_SH_LIST_DCN2(__SHIFT)
+};
+
+static const struct dce_i2c_mask i2c_masks = {
+		I2C_COMMON_MASK_SH_LIST_DCN2(_MASK)
+};
+
+struct dce_i2c_hw *dcn20_i2c_hw_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw =
+		kzalloc(sizeof(struct dce_i2c_hw), GFP_ATOMIC);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
+				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+struct mpc *dcn20_mpc_create(struct dc_context *ctx)
+{
+	struct dcn20_mpc *mpc20 = kzalloc(sizeof(struct dcn20_mpc),
+					  GFP_ATOMIC);
+
+	if (!mpc20)
+		return NULL;
+
+	dcn20_mpc_construct(mpc20, ctx,
+			&mpc_regs,
+			&mpc_shift,
+			&mpc_mask,
+			6);
+
+	return &mpc20->base;
+}
+
+struct hubbub *dcn20_hubbub_create(struct dc_context *ctx)
+{
+	int i;
+	struct dcn20_hubbub *hubbub = kzalloc(sizeof(struct dcn20_hubbub),
+					  GFP_ATOMIC);
+
+	if (!hubbub)
+		return NULL;
+
+	hubbub2_construct(hubbub, ctx,
+			&hubbub_reg,
+			&hubbub_shift,
+			&hubbub_mask);
+
+	for (i = 0; i < res_cap_nv10.num_vmid; i++) {
+		struct dcn20_vmid *vmid = &hubbub->vmid[i];
+
+		vmid->ctx = ctx;
+
+		vmid->regs = &vmid_regs[i];
+		vmid->shifts = &vmid_shifts;
+		vmid->masks = &vmid_masks;
+	}
+
+	return &hubbub->base;
+}
+
+struct timing_generator *dcn20_timing_generator_create(
+		struct dc_context *ctx,
+		uint32_t instance)
+{
+	struct optc *tgn10 =
+		kzalloc(sizeof(struct optc), GFP_ATOMIC);
+
+	if (!tgn10)
+		return NULL;
+
+	tgn10->base.inst = instance;
+	tgn10->base.ctx = ctx;
+
+	tgn10->tg_regs = &tg_regs[instance];
+	tgn10->tg_shift = &tg_shift;
+	tgn10->tg_mask = &tg_mask;
+
+	dcn20_timing_generator_init(tgn10);
+
+	return &tgn10->base;
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+		.max_hdmi_deep_color = COLOR_DEPTH_121212,
+		.max_hdmi_pixel_clock = 600000,
+		.hdmi_ycbcr420_supported = true,
+		.dp_ycbcr420_supported = true,
+		.fec_supported = true,
+		.flags.bits.IS_HBR2_CAPABLE = true,
+		.flags.bits.IS_HBR3_CAPABLE = true,
+		.flags.bits.IS_TPS3_CAPABLE = true,
+		.flags.bits.IS_TPS4_CAPABLE = true
+};
+
+struct link_encoder *dcn20_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dcn20_link_encoder *enc20 =
+		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+	int link_regs_id;
+
+	if (!enc20)
+		return NULL;
+
+	link_regs_id =
+		map_transmitter_id_to_phy_instance(enc_init_data->transmitter);
+
+	dcn20_link_encoder_construct(enc20,
+				      enc_init_data,
+				      &link_enc_feature,
+				      &link_enc_regs[link_regs_id],
+				      &link_enc_aux_regs[enc_init_data->channel - 1],
+				      &link_enc_hpd_regs[enc_init_data->hpd_source],
+				      &le_shift,
+				      &le_mask);
+
+	return &enc20->enc10.base;
+}
+
+static struct panel_cntl *dcn20_panel_cntl_create(const struct panel_cntl_init_data *init_data)
+{
+	struct dce_panel_cntl *panel_cntl =
+		kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
+
+	if (!panel_cntl)
+		return NULL;
+
+	dce_panel_cntl_construct(panel_cntl,
+			init_data,
+			&panel_cntl_regs[init_data->inst],
+			&panel_cntl_shift,
+			&panel_cntl_mask);
+
+	return &panel_cntl->base;
+}
+
+static struct clock_source *dcn20_clock_source_create(
+	struct dc_context *ctx,
+	struct dc_bios *bios,
+	enum clock_source_id id,
+	const struct dce110_clk_src_regs *regs,
+	bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(struct dce110_clk_src), GFP_ATOMIC);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dcn20_clk_src_construct(clk_src, ctx, bios, id,
+			regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	kfree(clk_src);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static void read_dce_straps(
+	struct dc_context *ctx,
+	struct resource_straps *straps)
+{
+	generic_reg_get(ctx, mmDC_PINSTRAPS + BASE(mmDC_PINSTRAPS_BASE_IDX),
+		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
+}
+
+static struct audio *dcn20_create_audio(
+		struct dc_context *ctx, unsigned int inst)
+{
+	return dce_audio_create(ctx, inst,
+			&audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+struct stream_encoder *dcn20_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dcn10_stream_encoder *enc1 =
+		kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
+
+	if (!enc1)
+		return NULL;
+
+	if (ASICREV_IS_NAVI14_M(ctx->asic_id.hw_internal_rev)) {
+		if (eng_id >= ENGINE_ID_DIGD)
+			eng_id++;
+	}
+
+	dcn20_stream_encoder_construct(enc1, ctx, ctx->dc_bios, eng_id,
+					&stream_enc_regs[eng_id],
+					&se_shift, &se_mask);
+
+	return &enc1->base;
+}
+
+static const struct dce_hwseq_registers hwseq_reg = {
+		HWSEQ_DCN2_REG_LIST()
+};
+
+static const struct dce_hwseq_shift hwseq_shift = {
+		HWSEQ_DCN2_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+		HWSEQ_DCN2_MASK_SH_LIST(_MASK)
+};
+
+struct dce_hwseq *dcn20_hwseq_create(
+	struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &hwseq_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+	}
+	return hws;
+}
+
+static const struct resource_create_funcs res_create_funcs = {
+	.read_dce_straps = read_dce_straps,
+	.create_audio = dcn20_create_audio,
+	.create_stream_encoder = dcn20_stream_encoder_create,
+	.create_hwseq = dcn20_hwseq_create,
+};
+
+static void dcn20_pp_smu_destroy(struct pp_smu_funcs **pp_smu);
+
+void dcn20_clock_source_destroy(struct clock_source **clk_src)
+{
+	kfree(TO_DCE110_CLK_SRC(*clk_src));
+	*clk_src = NULL;
+}
+
+
+struct display_stream_compressor *dcn20_dsc_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_dsc *dsc =
+		kzalloc(sizeof(struct dcn20_dsc), GFP_ATOMIC);
+
+	if (!dsc) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
+	return &dsc->base;
+}
+
+void dcn20_dsc_destroy(struct display_stream_compressor **dsc)
+{
+	kfree(container_of(*dsc, struct dcn20_dsc, base));
+	*dsc = NULL;
+}
+
+
+static void dcn20_resource_destruct(struct dcn20_resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->base.stream_enc_count; i++) {
+		if (pool->base.stream_enc[i] != NULL) {
+			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
+			pool->base.stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		if (pool->base.dscs[i] != NULL)
+			dcn20_dsc_destroy(&pool->base.dscs[i]);
+	}
+
+	if (pool->base.mpc != NULL) {
+		kfree(TO_DCN20_MPC(pool->base.mpc));
+		pool->base.mpc = NULL;
+	}
+	if (pool->base.hubbub != NULL) {
+		kfree(pool->base.hubbub);
+		pool->base.hubbub = NULL;
+	}
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (pool->base.dpps[i] != NULL)
+			dcn20_dpp_destroy(&pool->base.dpps[i]);
+
+		if (pool->base.ipps[i] != NULL)
+			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
+
+		if (pool->base.hubps[i] != NULL) {
+			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
+			pool->base.hubps[i] = NULL;
+		}
+
+		if (pool->base.irqs != NULL) {
+			dal_irq_service_destroy(&pool->base.irqs);
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+		if (pool->base.hw_i2cs[i] != NULL) {
+			kfree(pool->base.hw_i2cs[i]);
+			pool->base.hw_i2cs[i] = NULL;
+		}
+		if (pool->base.sw_i2cs[i] != NULL) {
+			kfree(pool->base.sw_i2cs[i]);
+			pool->base.sw_i2cs[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		if (pool->base.opps[i] != NULL)
+			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.timing_generators[i] != NULL)	{
+			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
+			pool->base.timing_generators[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+		if (pool->base.dwbc[i] != NULL) {
+			kfree(TO_DCN20_DWBC(pool->base.dwbc[i]));
+			pool->base.dwbc[i] = NULL;
+		}
+		if (pool->base.mcif_wb[i] != NULL) {
+			kfree(TO_DCN20_MMHUBBUB(pool->base.mcif_wb[i]));
+			pool->base.mcif_wb[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.audio_count; i++) {
+		if (pool->base.audios[i])
+			dce_aud_destroy(&pool->base.audios[i]);
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] != NULL) {
+			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
+			pool->base.clock_sources[i] = NULL;
+		}
+	}
+
+	if (pool->base.dp_clock_source != NULL) {
+		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
+		pool->base.dp_clock_source = NULL;
+	}
+
+
+	if (pool->base.abm != NULL)
+		dce_abm_destroy(&pool->base.abm);
+
+	if (pool->base.dmcu != NULL)
+		dce_dmcu_destroy(&pool->base.dmcu);
+
+	if (pool->base.dccg != NULL)
+		dcn_dccg_destroy(&pool->base.dccg);
+
+	if (pool->base.pp_smu != NULL)
+		dcn20_pp_smu_destroy(&pool->base.pp_smu);
+
+	if (pool->base.oem_device != NULL) {
+		struct dc *dc = pool->base.oem_device->ctx->dc;
+
+		dc->link_srv->destroy_ddc_service(&pool->base.oem_device);
+	}
+}
+
+struct hubp *dcn20_hubp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn20_hubp *hubp2 =
+		kzalloc(sizeof(struct dcn20_hubp), GFP_ATOMIC);
+
+	if (!hubp2)
+		return NULL;
+
+	if (hubp2_construct(hubp2, ctx, inst,
+			&hubp_regs[inst], &hubp_shift, &hubp_mask))
+		return &hubp2->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(hubp2);
+	return NULL;
+}
+
+static void get_pixel_clock_parameters(
+	struct pipe_ctx *pipe_ctx,
+	struct pixel_clk_params *pixel_clk_params)
+{
+	const struct dc_stream_state *stream = pipe_ctx->stream;
+	struct pipe_ctx *odm_pipe;
+	int opp_cnt = 1;
+	struct dc_link *link = stream->link;
+	struct link_encoder *link_enc = NULL;
+	struct dc *dc = pipe_ctx->stream->ctx->dc;
+	struct dce_hwseq *hws = dc->hwseq;
+
+	for (odm_pipe = pipe_ctx->next_odm_pipe; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe)
+		opp_cnt++;
+
+	pixel_clk_params->requested_pix_clk_100hz = stream->timing.pix_clk_100hz;
+
+	link_enc = link_enc_cfg_get_link_enc(link);
+	if (link_enc)
+		pixel_clk_params->encoder_object_id = link_enc->id;
+
+	pixel_clk_params->signal_type = pipe_ctx->stream->signal;
+	pixel_clk_params->controller_id = pipe_ctx->stream_res.tg->inst + 1;
+	/* TODO: un-hardcode*/
+	/* TODO - DP2.0 HW: calculate requested_sym_clk for UHBR rates */
+	pixel_clk_params->requested_sym_clk = LINK_RATE_LOW *
+		LINK_RATE_REF_FREQ_IN_KHZ;
+	pixel_clk_params->flags.ENABLE_SS = 0;
+	pixel_clk_params->color_depth =
+		stream->timing.display_color_depth;
+	pixel_clk_params->flags.DISPLAY_BLANKED = 1;
+	pixel_clk_params->pixel_encoding = stream->timing.pixel_encoding;
+
+	if (stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR422)
+		pixel_clk_params->color_depth = COLOR_DEPTH_888;
+
+	if (opp_cnt == 4)
+		pixel_clk_params->requested_pix_clk_100hz /= 4;
+	else if (optc2_is_two_pixels_per_containter(&stream->timing) || opp_cnt == 2)
+		pixel_clk_params->requested_pix_clk_100hz /= 2;
+	else if (hws->funcs.is_dp_dig_pixel_rate_div_policy) {
+		if (hws->funcs.is_dp_dig_pixel_rate_div_policy(pipe_ctx))
+			pixel_clk_params->requested_pix_clk_100hz /= 2;
+	}
+
+	if (stream->timing.timing_3d_format == TIMING_3D_FORMAT_HW_FRAME_PACKING)
+		pixel_clk_params->requested_pix_clk_100hz *= 2;
+
+}
+
+static void build_clamping_params(struct dc_stream_state *stream)
+{
+	stream->clamping.clamping_level = CLAMPING_FULL_RANGE;
+	stream->clamping.c_depth = stream->timing.display_color_depth;
+	stream->clamping.pixel_encoding = stream->timing.pixel_encoding;
+}
+
+static enum dc_status build_pipe_hw_param(struct pipe_ctx *pipe_ctx)
+{
+
+	get_pixel_clock_parameters(pipe_ctx, &pipe_ctx->stream_res.pix_clk_params);
+
+	pipe_ctx->clock_source->funcs->get_pix_clk_dividers(
+		pipe_ctx->clock_source,
+		&pipe_ctx->stream_res.pix_clk_params,
+		&pipe_ctx->pll_settings);
+
+	pipe_ctx->stream->clamping.pixel_encoding = pipe_ctx->stream->timing.pixel_encoding;
+
+	resource_build_bit_depth_reduction_params(pipe_ctx->stream,
+					&pipe_ctx->stream->bit_depth_params);
+	build_clamping_params(pipe_ctx->stream);
+
+	return DC_OK;
+}
+
+enum dc_status dcn20_build_mapped_resource(const struct dc *dc, struct dc_state *context, struct dc_stream_state *stream)
+{
+	enum dc_status status = DC_OK;
+	struct pipe_ctx *pipe_ctx = resource_get_otg_master_for_stream(&context->res_ctx, stream);
+
+	if (!pipe_ctx)
+		return DC_ERROR_UNEXPECTED;
+
+
+	status = build_pipe_hw_param(pipe_ctx);
+
+	return status;
+}
+
+
+void dcn20_acquire_dsc(const struct dc *dc,
+			struct resource_context *res_ctx,
+			struct display_stream_compressor **dsc,
+			int pipe_idx)
+{
+	int i;
+	const struct resource_pool *pool = dc->res_pool;
+	struct display_stream_compressor *dsc_old = dc->current_state->res_ctx.pipe_ctx[pipe_idx].stream_res.dsc;
+
+	ASSERT(*dsc == NULL); /* If this ASSERT fails, dsc was not released properly */
+	*dsc = NULL;
+
+	/* Always do 1-to-1 mapping when number of DSCs is same as number of pipes */
+	if (pool->res_cap->num_dsc == pool->res_cap->num_opp) {
+		*dsc = pool->dscs[pipe_idx];
+		res_ctx->is_dsc_acquired[pipe_idx] = true;
+		return;
+	}
+
+	/* Return old DSC to avoid the need for re-programming */
+	if (dsc_old && !res_ctx->is_dsc_acquired[dsc_old->inst]) {
+		*dsc = dsc_old;
+		res_ctx->is_dsc_acquired[dsc_old->inst] = true;
+		return ;
+	}
+
+	/* Find first free DSC */
+	for (i = 0; i < pool->res_cap->num_dsc; i++)
+		if (!res_ctx->is_dsc_acquired[i]) {
+			*dsc = pool->dscs[i];
+			res_ctx->is_dsc_acquired[i] = true;
+			break;
+		}
+}
+
+void dcn20_release_dsc(struct resource_context *res_ctx,
+			const struct resource_pool *pool,
+			struct display_stream_compressor **dsc)
+{
+	int i;
+
+	for (i = 0; i < pool->res_cap->num_dsc; i++)
+		if (pool->dscs[i] == *dsc) {
+			res_ctx->is_dsc_acquired[i] = false;
+			*dsc = NULL;
+			break;
+		}
+}
+
+
+
+enum dc_status dcn20_add_dsc_to_stream_resource(struct dc *dc,
+		struct dc_state *dc_ctx,
+		struct dc_stream_state *dc_stream)
+{
+	enum dc_status result = DC_OK;
+	int i;
+
+	/* Get a DSC if required and available */
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe_ctx = &dc_ctx->res_ctx.pipe_ctx[i];
+
+		if (pipe_ctx->top_pipe)
+			continue;
+
+		if (pipe_ctx->stream != dc_stream)
+			continue;
+
+		if (pipe_ctx->stream_res.dsc)
+			continue;
+
+		dcn20_acquire_dsc(dc, &dc_ctx->res_ctx, &pipe_ctx->stream_res.dsc, i);
+
+		/* The number of DSCs can be less than the number of pipes */
+		if (!pipe_ctx->stream_res.dsc) {
+			result = DC_NO_DSC_RESOURCE;
+		}
+
+		break;
+	}
+
+	return result;
+}
+
+
+static enum dc_status remove_dsc_from_stream_resource(struct dc *dc,
+		struct dc_state *new_ctx,
+		struct dc_stream_state *dc_stream)
+{
+	struct pipe_ctx *pipe_ctx = NULL;
+	int i;
+
+	for (i = 0; i < MAX_PIPES; i++) {
+		if (new_ctx->res_ctx.pipe_ctx[i].stream == dc_stream && !new_ctx->res_ctx.pipe_ctx[i].top_pipe) {
+			pipe_ctx = &new_ctx->res_ctx.pipe_ctx[i];
+
+			if (pipe_ctx->stream_res.dsc)
+				dcn20_release_dsc(&new_ctx->res_ctx, dc->res_pool, &pipe_ctx->stream_res.dsc);
+		}
+	}
+
+	if (!pipe_ctx)
+		return DC_ERROR_UNEXPECTED;
+	else
+		return DC_OK;
+}
+
+
+enum dc_status dcn20_add_stream_to_ctx(struct dc *dc, struct dc_state *new_ctx, struct dc_stream_state *dc_stream)
+{
+	enum dc_status result = DC_ERROR_UNEXPECTED;
+
+	result = resource_map_pool_resources(dc, new_ctx, dc_stream);
+
+	if (result == DC_OK)
+		result = resource_map_phy_clock_resources(dc, new_ctx, dc_stream);
+
+	/* Get a DSC if required and available */
+	if (result == DC_OK && dc_stream->timing.flags.DSC)
+		result = dcn20_add_dsc_to_stream_resource(dc, new_ctx, dc_stream);
+
+	if (result == DC_OK)
+		result = dcn20_build_mapped_resource(dc, new_ctx, dc_stream);
+
+	return result;
+}
+
+
+enum dc_status dcn20_remove_stream_from_ctx(struct dc *dc, struct dc_state *new_ctx, struct dc_stream_state *dc_stream)
+{
+	enum dc_status result = DC_OK;
+
+	result = remove_dsc_from_stream_resource(dc, new_ctx, dc_stream);
+
+	return result;
+}
+
+/**
+ * dcn20_split_stream_for_odm - Check if stream can be splited for ODM
+ *
+ * @dc: DC object with resource pool info required for pipe split
+ * @res_ctx: Persistent state of resources
+ * @prev_odm_pipe: Reference to the previous ODM pipe
+ * @next_odm_pipe: Reference to the next ODM pipe
+ *
+ * This function takes a logically active pipe and a logically free pipe and
+ * halves all the scaling parameters that need to be halved while populating
+ * the free pipe with the required resources and configuring the next/previous
+ * ODM pipe pointers.
+ *
+ * Return:
+ * Return true if split stream for ODM is possible, otherwise, return false.
+ */
+bool dcn20_split_stream_for_odm(
+		const struct dc *dc,
+		struct resource_context *res_ctx,
+		struct pipe_ctx *prev_odm_pipe,
+		struct pipe_ctx *next_odm_pipe)
+{
+	int pipe_idx = next_odm_pipe->pipe_idx;
+	const struct resource_pool *pool = dc->res_pool;
+
+	*next_odm_pipe = *prev_odm_pipe;
+
+	next_odm_pipe->pipe_idx = pipe_idx;
+	next_odm_pipe->plane_res.mi = pool->mis[next_odm_pipe->pipe_idx];
+	next_odm_pipe->plane_res.hubp = pool->hubps[next_odm_pipe->pipe_idx];
+	next_odm_pipe->plane_res.ipp = pool->ipps[next_odm_pipe->pipe_idx];
+	next_odm_pipe->plane_res.xfm = pool->transforms[next_odm_pipe->pipe_idx];
+	next_odm_pipe->plane_res.dpp = pool->dpps[next_odm_pipe->pipe_idx];
+	next_odm_pipe->plane_res.mpcc_inst = pool->dpps[next_odm_pipe->pipe_idx]->inst;
+	next_odm_pipe->stream_res.dsc = NULL;
+	if (prev_odm_pipe->next_odm_pipe && prev_odm_pipe->next_odm_pipe != next_odm_pipe) {
+		next_odm_pipe->next_odm_pipe = prev_odm_pipe->next_odm_pipe;
+		next_odm_pipe->next_odm_pipe->prev_odm_pipe = next_odm_pipe;
+	}
+	if (prev_odm_pipe->top_pipe && prev_odm_pipe->top_pipe->next_odm_pipe) {
+		prev_odm_pipe->top_pipe->next_odm_pipe->bottom_pipe = next_odm_pipe;
+		next_odm_pipe->top_pipe = prev_odm_pipe->top_pipe->next_odm_pipe;
+	}
+	if (prev_odm_pipe->bottom_pipe && prev_odm_pipe->bottom_pipe->next_odm_pipe) {
+		prev_odm_pipe->bottom_pipe->next_odm_pipe->top_pipe = next_odm_pipe;
+		next_odm_pipe->bottom_pipe = prev_odm_pipe->bottom_pipe->next_odm_pipe;
+	}
+	prev_odm_pipe->next_odm_pipe = next_odm_pipe;
+	next_odm_pipe->prev_odm_pipe = prev_odm_pipe;
+
+	if (prev_odm_pipe->plane_state) {
+		struct scaler_data *sd = &prev_odm_pipe->plane_res.scl_data;
+		int new_width;
+
+		/* HACTIVE halved for odm combine */
+		sd->h_active /= 2;
+		/* Calculate new vp and recout for left pipe */
+		/* Need at least 16 pixels width per side */
+		if (sd->recout.x + 16 >= sd->h_active)
+			return false;
+		new_width = sd->h_active - sd->recout.x;
+		sd->viewport.width -= dc_fixpt_floor(dc_fixpt_mul_int(
+				sd->ratios.horz, sd->recout.width - new_width));
+		sd->viewport_c.width -= dc_fixpt_floor(dc_fixpt_mul_int(
+				sd->ratios.horz_c, sd->recout.width - new_width));
+		sd->recout.width = new_width;
+
+		/* Calculate new vp and recout for right pipe */
+		sd = &next_odm_pipe->plane_res.scl_data;
+		/* HACTIVE halved for odm combine */
+		sd->h_active /= 2;
+		/* Need at least 16 pixels width per side */
+		if (new_width <= 16)
+			return false;
+		new_width = sd->recout.width + sd->recout.x - sd->h_active;
+		sd->viewport.width -= dc_fixpt_floor(dc_fixpt_mul_int(
+				sd->ratios.horz, sd->recout.width - new_width));
+		sd->viewport_c.width -= dc_fixpt_floor(dc_fixpt_mul_int(
+				sd->ratios.horz_c, sd->recout.width - new_width));
+		sd->recout.width = new_width;
+		sd->viewport.x += dc_fixpt_floor(dc_fixpt_mul_int(
+				sd->ratios.horz, sd->h_active - sd->recout.x));
+		sd->viewport_c.x += dc_fixpt_floor(dc_fixpt_mul_int(
+				sd->ratios.horz_c, sd->h_active - sd->recout.x));
+		sd->recout.x = 0;
+	}
+	if (!next_odm_pipe->top_pipe)
+		next_odm_pipe->stream_res.opp = pool->opps[next_odm_pipe->pipe_idx];
+	else
+		next_odm_pipe->stream_res.opp = next_odm_pipe->top_pipe->stream_res.opp;
+	if (next_odm_pipe->stream->timing.flags.DSC == 1 && !next_odm_pipe->top_pipe) {
+		dcn20_acquire_dsc(dc, res_ctx, &next_odm_pipe->stream_res.dsc, next_odm_pipe->pipe_idx);
+		ASSERT(next_odm_pipe->stream_res.dsc);
+		if (next_odm_pipe->stream_res.dsc == NULL)
+			return false;
+	}
+
+	return true;
+}
+
+void dcn20_split_stream_for_mpc(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		struct pipe_ctx *primary_pipe,
+		struct pipe_ctx *secondary_pipe)
+{
+	int pipe_idx = secondary_pipe->pipe_idx;
+	struct pipe_ctx *sec_bot_pipe = secondary_pipe->bottom_pipe;
+
+	*secondary_pipe = *primary_pipe;
+	secondary_pipe->bottom_pipe = sec_bot_pipe;
+
+	secondary_pipe->pipe_idx = pipe_idx;
+	secondary_pipe->plane_res.mi = pool->mis[secondary_pipe->pipe_idx];
+	secondary_pipe->plane_res.hubp = pool->hubps[secondary_pipe->pipe_idx];
+	secondary_pipe->plane_res.ipp = pool->ipps[secondary_pipe->pipe_idx];
+	secondary_pipe->plane_res.xfm = pool->transforms[secondary_pipe->pipe_idx];
+	secondary_pipe->plane_res.dpp = pool->dpps[secondary_pipe->pipe_idx];
+	secondary_pipe->plane_res.mpcc_inst = pool->dpps[secondary_pipe->pipe_idx]->inst;
+	secondary_pipe->stream_res.dsc = NULL;
+	if (primary_pipe->bottom_pipe && primary_pipe->bottom_pipe != secondary_pipe) {
+		ASSERT(!secondary_pipe->bottom_pipe);
+		secondary_pipe->bottom_pipe = primary_pipe->bottom_pipe;
+		secondary_pipe->bottom_pipe->top_pipe = secondary_pipe;
+	}
+	primary_pipe->bottom_pipe = secondary_pipe;
+	secondary_pipe->top_pipe = primary_pipe;
+
+	ASSERT(primary_pipe->plane_state);
+}
+
+unsigned int dcn20_calc_max_scaled_time(
+		unsigned int time_per_pixel,
+		enum mmhubbub_wbif_mode mode,
+		unsigned int urgent_watermark)
+{
+	unsigned int time_per_byte = 0;
+	unsigned int total_y_free_entry = 0x200; /* two memory piece for luma */
+	unsigned int total_c_free_entry = 0x140; /* two memory piece for chroma */
+	unsigned int small_free_entry, max_free_entry;
+	unsigned int buf_lh_capability;
+	unsigned int max_scaled_time;
+
+	if (mode == PACKED_444) /* packed mode */
+		time_per_byte = time_per_pixel/4;
+	else if (mode == PLANAR_420_8BPC)
+		time_per_byte  = time_per_pixel;
+	else if (mode == PLANAR_420_10BPC) /* p010 */
+		time_per_byte  = time_per_pixel * 819/1024;
+
+	if (time_per_byte == 0)
+		time_per_byte = 1;
+
+	small_free_entry  = (total_y_free_entry > total_c_free_entry) ? total_c_free_entry : total_y_free_entry;
+	max_free_entry    = (mode == PACKED_444) ? total_y_free_entry + total_c_free_entry : small_free_entry;
+	buf_lh_capability = max_free_entry*time_per_byte*32/16; /* there is 4bit fraction */
+	max_scaled_time   = buf_lh_capability - urgent_watermark;
+	return max_scaled_time;
+}
+
+void dcn20_set_mcif_arb_params(
+		struct dc *dc,
+		struct dc_state *context,
+		display_e2e_pipe_params_st *pipes,
+		int pipe_cnt)
+{
+	enum mmhubbub_wbif_mode wbif_mode;
+	struct mcif_arb_params *wb_arb_params;
+	int i, j, dwb_pipe;
+
+	/* Writeback MCIF_WB arbitration parameters */
+	dwb_pipe = 0;
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+
+		if (!context->res_ctx.pipe_ctx[i].stream)
+			continue;
+
+		for (j = 0; j < MAX_DWB_PIPES; j++) {
+			if (context->res_ctx.pipe_ctx[i].stream->writeback_info[j].wb_enabled == false)
+				continue;
+
+			//wb_arb_params = &context->res_ctx.pipe_ctx[i].stream->writeback_info[j].mcif_arb_params;
+			wb_arb_params = &context->bw_ctx.bw.dcn.bw_writeback.mcif_wb_arb[dwb_pipe];
+
+			if (context->res_ctx.pipe_ctx[i].stream->writeback_info[j].dwb_params.out_format == dwb_scaler_mode_yuv420) {
+				if (context->res_ctx.pipe_ctx[i].stream->writeback_info[j].dwb_params.output_depth == DWB_OUTPUT_PIXEL_DEPTH_8BPC)
+					wbif_mode = PLANAR_420_8BPC;
+				else
+					wbif_mode = PLANAR_420_10BPC;
+			} else
+				wbif_mode = PACKED_444;
+
+			DC_FP_START();
+			dcn20_fpu_set_wb_arb_params(wb_arb_params, context, pipes, pipe_cnt, i);
+			DC_FP_END();
+
+			wb_arb_params->slice_lines = 32;
+			wb_arb_params->arbitration_slice = 2;
+			wb_arb_params->max_scaled_time = dcn20_calc_max_scaled_time(wb_arb_params->time_per_pixel,
+				wbif_mode,
+				wb_arb_params->cli_watermark[0]); /* assume 4 watermark sets have the same value */
+
+			dwb_pipe++;
+
+			if (dwb_pipe >= MAX_DWB_PIPES)
+				return;
+		}
+		if (dwb_pipe >= MAX_DWB_PIPES)
+			return;
+	}
+}
+
+bool dcn20_validate_dsc(struct dc *dc, struct dc_state *new_ctx)
+{
+	int i;
+
+	/* Validate DSC config, dsc count validation is already done */
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe_ctx = &new_ctx->res_ctx.pipe_ctx[i];
+		struct dc_stream_state *stream = pipe_ctx->stream;
+		struct dsc_config dsc_cfg;
+		struct pipe_ctx *odm_pipe;
+		int opp_cnt = 1;
+
+		for (odm_pipe = pipe_ctx->next_odm_pipe; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe)
+			opp_cnt++;
+
+		/* Only need to validate top pipe */
+		if (pipe_ctx->top_pipe || pipe_ctx->prev_odm_pipe || !stream || !stream->timing.flags.DSC)
+			continue;
+
+		dsc_cfg.pic_width = (stream->timing.h_addressable + stream->timing.h_border_left
+				+ stream->timing.h_border_right) / opp_cnt;
+		dsc_cfg.pic_height = stream->timing.v_addressable + stream->timing.v_border_top
+				+ stream->timing.v_border_bottom;
+		dsc_cfg.pixel_encoding = stream->timing.pixel_encoding;
+		dsc_cfg.color_depth = stream->timing.display_color_depth;
+		dsc_cfg.is_odm = pipe_ctx->next_odm_pipe ? true : false;
+		dsc_cfg.dc_dsc_cfg = stream->timing.dsc_cfg;
+		dsc_cfg.dc_dsc_cfg.num_slices_h /= opp_cnt;
+
+		if (!pipe_ctx->stream_res.dsc->funcs->dsc_validate_stream(pipe_ctx->stream_res.dsc, &dsc_cfg))
+			return false;
+	}
+	return true;
+}
+
+struct pipe_ctx *dcn20_find_secondary_pipe(struct dc *dc,
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		const struct pipe_ctx *primary_pipe)
+{
+	struct pipe_ctx *secondary_pipe = NULL;
+
+	if (dc && primary_pipe) {
+		int j;
+		int preferred_pipe_idx = 0;
+
+		/* first check the prev dc state:
+		 * if this primary pipe has a bottom pipe in prev. state
+		 * and if the bottom pipe is still available (which it should be),
+		 * pick that pipe as secondary
+		 * Same logic applies for ODM pipes
+		 */
+		if (dc->current_state->res_ctx.pipe_ctx[primary_pipe->pipe_idx].next_odm_pipe) {
+			preferred_pipe_idx = dc->current_state->res_ctx.pipe_ctx[primary_pipe->pipe_idx].next_odm_pipe->pipe_idx;
+			if (res_ctx->pipe_ctx[preferred_pipe_idx].stream == NULL) {
+				secondary_pipe = &res_ctx->pipe_ctx[preferred_pipe_idx];
+				secondary_pipe->pipe_idx = preferred_pipe_idx;
+			}
+		}
+		if (secondary_pipe == NULL &&
+				dc->current_state->res_ctx.pipe_ctx[primary_pipe->pipe_idx].bottom_pipe) {
+			preferred_pipe_idx = dc->current_state->res_ctx.pipe_ctx[primary_pipe->pipe_idx].bottom_pipe->pipe_idx;
+			if (res_ctx->pipe_ctx[preferred_pipe_idx].stream == NULL) {
+				secondary_pipe = &res_ctx->pipe_ctx[preferred_pipe_idx];
+				secondary_pipe->pipe_idx = preferred_pipe_idx;
+			}
+		}
+
+		/*
+		 * if this primary pipe does not have a bottom pipe in prev. state
+		 * start backward and find a pipe that did not used to be a bottom pipe in
+		 * prev. dc state. This way we make sure we keep the same assignment as
+		 * last state and will not have to reprogram every pipe
+		 */
+		if (secondary_pipe == NULL) {
+			for (j = dc->res_pool->pipe_count - 1; j >= 0; j--) {
+				if (dc->current_state->res_ctx.pipe_ctx[j].top_pipe == NULL
+						&& dc->current_state->res_ctx.pipe_ctx[j].prev_odm_pipe == NULL) {
+					preferred_pipe_idx = j;
+
+					if (res_ctx->pipe_ctx[preferred_pipe_idx].stream == NULL) {
+						secondary_pipe = &res_ctx->pipe_ctx[preferred_pipe_idx];
+						secondary_pipe->pipe_idx = preferred_pipe_idx;
+						break;
+					}
+				}
+			}
+		}
+		/*
+		 * We should never hit this assert unless assignments are shuffled around
+		 * if this happens we will prob. hit a vsync tdr
+		 */
+		ASSERT(secondary_pipe);
+		/*
+		 * search backwards for the second pipe to keep pipe
+		 * assignment more consistent
+		 */
+		if (secondary_pipe == NULL) {
+			for (j = dc->res_pool->pipe_count - 1; j >= 0; j--) {
+				preferred_pipe_idx = j;
+
+				if (res_ctx->pipe_ctx[preferred_pipe_idx].stream == NULL) {
+					secondary_pipe = &res_ctx->pipe_ctx[preferred_pipe_idx];
+					secondary_pipe->pipe_idx = preferred_pipe_idx;
+					break;
+				}
+			}
+		}
+	}
+
+	return secondary_pipe;
+}
+
+void dcn20_merge_pipes_for_validate(
+		struct dc *dc,
+		struct dc_state *context)
+{
+	int i;
+
+	/* merge previously split odm pipes since mode support needs to make the decision */
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+		struct pipe_ctx *odm_pipe = pipe->next_odm_pipe;
+
+		if (pipe->prev_odm_pipe)
+			continue;
+
+		pipe->next_odm_pipe = NULL;
+		while (odm_pipe) {
+			struct pipe_ctx *next_odm_pipe = odm_pipe->next_odm_pipe;
+
+			odm_pipe->plane_state = NULL;
+			odm_pipe->stream = NULL;
+			odm_pipe->top_pipe = NULL;
+			odm_pipe->bottom_pipe = NULL;
+			odm_pipe->prev_odm_pipe = NULL;
+			odm_pipe->next_odm_pipe = NULL;
+			if (odm_pipe->stream_res.dsc)
+				dcn20_release_dsc(&context->res_ctx, dc->res_pool, &odm_pipe->stream_res.dsc);
+			/* Clear plane_res and stream_res */
+			memset(&odm_pipe->plane_res, 0, sizeof(odm_pipe->plane_res));
+			memset(&odm_pipe->stream_res, 0, sizeof(odm_pipe->stream_res));
+			odm_pipe = next_odm_pipe;
+		}
+		if (pipe->plane_state)
+			resource_build_scaling_params(pipe);
+	}
+
+	/* merge previously mpc split pipes since mode support needs to make the decision */
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+		struct pipe_ctx *hsplit_pipe = pipe->bottom_pipe;
+
+		if (!hsplit_pipe || hsplit_pipe->plane_state != pipe->plane_state)
+			continue;
+
+		pipe->bottom_pipe = hsplit_pipe->bottom_pipe;
+		if (hsplit_pipe->bottom_pipe)
+			hsplit_pipe->bottom_pipe->top_pipe = pipe;
+		hsplit_pipe->plane_state = NULL;
+		hsplit_pipe->stream = NULL;
+		hsplit_pipe->top_pipe = NULL;
+		hsplit_pipe->bottom_pipe = NULL;
+
+		/* Clear plane_res and stream_res */
+		memset(&hsplit_pipe->plane_res, 0, sizeof(hsplit_pipe->plane_res));
+		memset(&hsplit_pipe->stream_res, 0, sizeof(hsplit_pipe->stream_res));
+		if (pipe->plane_state)
+			resource_build_scaling_params(pipe);
+	}
+}
+
+int dcn20_validate_apply_pipe_split_flags(
+		struct dc *dc,
+		struct dc_state *context,
+		int vlevel,
+		int *split,
+		bool *merge)
+{
+	int i, pipe_idx, vlevel_split;
+	int plane_count = 0;
+	bool force_split = false;
+	bool avoid_split = dc->debug.pipe_split_policy == MPC_SPLIT_AVOID;
+	struct vba_vars_st *v = &context->bw_ctx.dml.vba;
+	int max_mpc_comb = v->maxMpcComb;
+
+	if (context->stream_count > 1) {
+		if (dc->debug.pipe_split_policy == MPC_SPLIT_AVOID_MULT_DISP)
+			avoid_split = true;
+	} else if (dc->debug.force_single_disp_pipe_split)
+			force_split = true;
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+
+		/**
+		 * Workaround for avoiding pipe-split in cases where we'd split
+		 * planes that are too small, resulting in splits that aren't
+		 * valid for the scaler.
+		 */
+		if (pipe->plane_state &&
+		    (pipe->plane_state->dst_rect.width <= 16 ||
+		     pipe->plane_state->dst_rect.height <= 16 ||
+		     pipe->plane_state->src_rect.width <= 16 ||
+		     pipe->plane_state->src_rect.height <= 16))
+			avoid_split = true;
+
+		/* TODO: fix dc bugs and remove this split threshold thing */
+		if (pipe->stream && !pipe->prev_odm_pipe &&
+				(!pipe->top_pipe || pipe->top_pipe->plane_state != pipe->plane_state))
+			++plane_count;
+	}
+	if (plane_count > dc->res_pool->pipe_count / 2)
+		avoid_split = true;
+
+	/* W/A: Mode timing with borders may not work well with pipe split, avoid for this corner case */
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+		struct dc_crtc_timing timing;
+
+		if (!pipe->stream)
+			continue;
+		else {
+			timing = pipe->stream->timing;
+			if (timing.h_border_left + timing.h_border_right
+					+ timing.v_border_top + timing.v_border_bottom > 0) {
+				avoid_split = true;
+				break;
+			}
+		}
+	}
+
+	/* Avoid split loop looks for lowest voltage level that allows most unsplit pipes possible */
+	if (avoid_split) {
+		for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
+			if (!context->res_ctx.pipe_ctx[i].stream)
+				continue;
+
+			for (vlevel_split = vlevel; vlevel <= context->bw_ctx.dml.soc.num_states; vlevel++)
+				if (v->NoOfDPP[vlevel][0][pipe_idx] == 1 &&
+						v->ModeSupport[vlevel][0])
+					break;
+			/* Impossible to not split this pipe */
+			if (vlevel > context->bw_ctx.dml.soc.num_states)
+				vlevel = vlevel_split;
+			else
+				max_mpc_comb = 0;
+			pipe_idx++;
+		}
+		v->maxMpcComb = max_mpc_comb;
+	}
+
+	/* Split loop sets which pipe should be split based on dml outputs and dc flags */
+	for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+		int pipe_plane = v->pipe_plane[pipe_idx];
+		bool split4mpc = context->stream_count == 1 && plane_count == 1
+				&& dc->config.enable_4to1MPC && dc->res_pool->pipe_count >= 4;
+
+		if (!context->res_ctx.pipe_ctx[i].stream)
+			continue;
+
+		if (split4mpc || v->NoOfDPP[vlevel][max_mpc_comb][pipe_plane] == 4)
+			split[i] = 4;
+		else if (force_split || v->NoOfDPP[vlevel][max_mpc_comb][pipe_plane] == 2)
+				split[i] = 2;
+
+		if ((pipe->stream->view_format ==
+				VIEW_3D_FORMAT_SIDE_BY_SIDE ||
+				pipe->stream->view_format ==
+				VIEW_3D_FORMAT_TOP_AND_BOTTOM) &&
+				(pipe->stream->timing.timing_3d_format ==
+				TIMING_3D_FORMAT_TOP_AND_BOTTOM ||
+				 pipe->stream->timing.timing_3d_format ==
+				TIMING_3D_FORMAT_SIDE_BY_SIDE))
+			split[i] = 2;
+		if (dc->debug.force_odm_combine & (1 << pipe->stream_res.tg->inst)) {
+			split[i] = 2;
+			v->ODMCombineEnablePerState[vlevel][pipe_plane] = dm_odm_combine_mode_2to1;
+		}
+		if (dc->debug.force_odm_combine_4to1 & (1 << pipe->stream_res.tg->inst)) {
+			split[i] = 4;
+			v->ODMCombineEnablePerState[vlevel][pipe_plane] = dm_odm_combine_mode_4to1;
+		}
+		/*420 format workaround*/
+		if (pipe->stream->timing.h_addressable > 7680 &&
+				pipe->stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR420) {
+			split[i] = 4;
+		}
+		v->ODMCombineEnabled[pipe_plane] =
+			v->ODMCombineEnablePerState[vlevel][pipe_plane];
+
+		if (v->ODMCombineEnabled[pipe_plane] == dm_odm_combine_mode_disabled) {
+			if (resource_get_mpc_slice_count(pipe) == 2) {
+				/*If need split for mpc but 2 way split already*/
+				if (split[i] == 4)
+					split[i] = 2; /* 2 -> 4 MPC */
+				else if (split[i] == 2)
+					split[i] = 0; /* 2 -> 2 MPC */
+				else if (pipe->top_pipe && pipe->top_pipe->plane_state == pipe->plane_state)
+					merge[i] = true; /* 2 -> 1 MPC */
+			} else if (resource_get_mpc_slice_count(pipe) == 4) {
+				/*If need split for mpc but 4 way split already*/
+				if (split[i] == 2 && ((pipe->top_pipe && !pipe->top_pipe->top_pipe)
+						|| !pipe->bottom_pipe)) {
+					merge[i] = true; /* 4 -> 2 MPC */
+				} else if (split[i] == 0 && pipe->top_pipe &&
+						pipe->top_pipe->plane_state == pipe->plane_state)
+					merge[i] = true; /* 4 -> 1 MPC */
+				split[i] = 0;
+			} else if (resource_get_odm_slice_count(pipe) > 1) {
+				/* ODM -> MPC transition */
+				if (pipe->prev_odm_pipe) {
+					split[i] = 0;
+					merge[i] = true;
+				}
+			}
+		} else {
+			if (resource_get_odm_slice_count(pipe) == 2) {
+				/*If need split for odm but 2 way split already*/
+				if (split[i] == 4)
+					split[i] = 2; /* 2 -> 4 ODM */
+				else if (split[i] == 2)
+					split[i] = 0; /* 2 -> 2 ODM */
+				else if (pipe->prev_odm_pipe) {
+					ASSERT(0); /* NOT expected yet */
+					merge[i] = true; /* exit ODM */
+				}
+			} else if (resource_get_odm_slice_count(pipe) == 4) {
+				/*If need split for odm but 4 way split already*/
+				if (split[i] == 2 && ((pipe->prev_odm_pipe && !pipe->prev_odm_pipe->prev_odm_pipe)
+						|| !pipe->next_odm_pipe)) {
+					merge[i] = true; /* 4 -> 2 ODM */
+				} else if (split[i] == 0 && pipe->prev_odm_pipe) {
+					ASSERT(0); /* NOT expected yet */
+					merge[i] = true; /* exit ODM */
+				}
+				split[i] = 0;
+			} else if (resource_get_mpc_slice_count(pipe) > 1) {
+				/* MPC -> ODM transition */
+				ASSERT(0); /* NOT expected yet */
+				if (pipe->top_pipe && pipe->top_pipe->plane_state == pipe->plane_state) {
+					split[i] = 0;
+					merge[i] = true;
+				}
+			}
+		}
+
+		/* Adjust dppclk when split is forced, do not bother with dispclk */
+		if (split[i] != 0 && v->NoOfDPP[vlevel][max_mpc_comb][pipe_idx] == 1) {
+			DC_FP_START();
+			dcn20_fpu_adjust_dppclk(v, vlevel, max_mpc_comb, pipe_idx, false);
+			DC_FP_END();
+		}
+		pipe_idx++;
+	}
+
+	return vlevel;
+}
+
+bool dcn20_fast_validate_bw(
+		struct dc *dc,
+		struct dc_state *context,
+		display_e2e_pipe_params_st *pipes,
+		int *pipe_cnt_out,
+		int *pipe_split_from,
+		int *vlevel_out,
+		bool fast_validate)
+{
+	bool out = false;
+	int split[MAX_PIPES] = { 0 };
+	int pipe_cnt, i, pipe_idx, vlevel;
+
+	ASSERT(pipes);
+	if (!pipes)
+		return false;
+
+	dcn20_merge_pipes_for_validate(dc, context);
+
+	DC_FP_START();
+	pipe_cnt = dc->res_pool->funcs->populate_dml_pipes(dc, context, pipes, fast_validate);
+	DC_FP_END();
+
+	*pipe_cnt_out = pipe_cnt;
+
+	if (!pipe_cnt) {
+		out = true;
+		goto validate_out;
+	}
+
+	vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
+
+	if (vlevel > context->bw_ctx.dml.soc.num_states)
+		goto validate_fail;
+
+	vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, vlevel, split, NULL);
+
+	/*initialize pipe_just_split_from to invalid idx*/
+	for (i = 0; i < MAX_PIPES; i++)
+		pipe_split_from[i] = -1;
+
+	for (i = 0, pipe_idx = -1; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+		struct pipe_ctx *hsplit_pipe = pipe->bottom_pipe;
+
+		if (!pipe->stream || pipe_split_from[i] >= 0)
+			continue;
+
+		pipe_idx++;
+
+		if (!pipe->top_pipe && !pipe->plane_state && context->bw_ctx.dml.vba.ODMCombineEnabled[pipe_idx]) {
+			hsplit_pipe = dcn20_find_secondary_pipe(dc, &context->res_ctx, dc->res_pool, pipe);
+			ASSERT(hsplit_pipe);
+			if (!dcn20_split_stream_for_odm(
+					dc, &context->res_ctx,
+					pipe, hsplit_pipe))
+				goto validate_fail;
+			pipe_split_from[hsplit_pipe->pipe_idx] = pipe_idx;
+			dcn20_build_mapped_resource(dc, context, pipe->stream);
+		}
+
+		if (!pipe->plane_state)
+			continue;
+		/* Skip 2nd half of already split pipe */
+		if (pipe->top_pipe && pipe->plane_state == pipe->top_pipe->plane_state)
+			continue;
+
+		/* We do not support mpo + odm at the moment */
+		if (hsplit_pipe && hsplit_pipe->plane_state != pipe->plane_state
+				&& context->bw_ctx.dml.vba.ODMCombineEnabled[pipe_idx])
+			goto validate_fail;
+
+		if (split[i] == 2) {
+			if (!hsplit_pipe || hsplit_pipe->plane_state != pipe->plane_state) {
+				/* pipe not split previously needs split */
+				hsplit_pipe = dcn20_find_secondary_pipe(dc, &context->res_ctx, dc->res_pool, pipe);
+				ASSERT(hsplit_pipe);
+				if (!hsplit_pipe) {
+					DC_FP_START();
+					dcn20_fpu_adjust_dppclk(&context->bw_ctx.dml.vba, vlevel, context->bw_ctx.dml.vba.maxMpcComb, pipe_idx, true);
+					DC_FP_END();
+					continue;
+				}
+				if (context->bw_ctx.dml.vba.ODMCombineEnabled[pipe_idx]) {
+					if (!dcn20_split_stream_for_odm(
+							dc, &context->res_ctx,
+							pipe, hsplit_pipe))
+						goto validate_fail;
+					dcn20_build_mapped_resource(dc, context, pipe->stream);
+				} else {
+					dcn20_split_stream_for_mpc(
+							&context->res_ctx, dc->res_pool,
+							pipe, hsplit_pipe);
+					resource_build_scaling_params(pipe);
+					resource_build_scaling_params(hsplit_pipe);
+				}
+				pipe_split_from[hsplit_pipe->pipe_idx] = pipe_idx;
+			}
+		} else if (hsplit_pipe && hsplit_pipe->plane_state == pipe->plane_state) {
+			/* merge should already have been done */
+			ASSERT(0);
+		}
+	}
+	/* Actual dsc count per stream dsc validation*/
+	if (!dcn20_validate_dsc(dc, context)) {
+		context->bw_ctx.dml.vba.ValidationStatus[context->bw_ctx.dml.vba.soc.num_states] =
+				DML_FAIL_DSC_VALIDATION_FAILURE;
+		goto validate_fail;
+	}
+
+	*vlevel_out = vlevel;
+
+	out = true;
+	goto validate_out;
+
+validate_fail:
+	out = false;
+
+validate_out:
+	return out;
+}
+
+bool dcn20_validate_bandwidth(struct dc *dc, struct dc_state *context,
+		bool fast_validate)
+{
+	bool voltage_supported;
+	display_e2e_pipe_params_st *pipes;
+
+	pipes = kcalloc(dc->res_pool->pipe_count, sizeof(display_e2e_pipe_params_st), GFP_KERNEL);
+	if (!pipes)
+		return false;
+
+	DC_FP_START();
+	voltage_supported = dcn20_validate_bandwidth_fp(dc, context, fast_validate, pipes);
+	DC_FP_END();
+
+	kfree(pipes);
+	return voltage_supported;
+}
+
+struct pipe_ctx *dcn20_acquire_free_pipe_for_layer(
+		const struct dc_state *cur_ctx,
+		struct dc_state *new_ctx,
+		const struct resource_pool *pool,
+		const struct pipe_ctx *opp_head)
+{
+	struct resource_context *res_ctx = &new_ctx->res_ctx;
+	struct pipe_ctx *otg_master = resource_get_otg_master_for_stream(res_ctx, opp_head->stream);
+	struct pipe_ctx *sec_dpp_pipe = resource_find_free_secondary_pipe_legacy(res_ctx, pool, otg_master);
+
+	ASSERT(otg_master);
+
+	if (!sec_dpp_pipe)
+		return NULL;
+
+	sec_dpp_pipe->stream = opp_head->stream;
+	sec_dpp_pipe->stream_res.tg = opp_head->stream_res.tg;
+	sec_dpp_pipe->stream_res.opp = opp_head->stream_res.opp;
+
+	sec_dpp_pipe->plane_res.hubp = pool->hubps[sec_dpp_pipe->pipe_idx];
+	sec_dpp_pipe->plane_res.ipp = pool->ipps[sec_dpp_pipe->pipe_idx];
+	sec_dpp_pipe->plane_res.dpp = pool->dpps[sec_dpp_pipe->pipe_idx];
+	sec_dpp_pipe->plane_res.mpcc_inst = pool->dpps[sec_dpp_pipe->pipe_idx]->inst;
+
+	return sec_dpp_pipe;
+}
+
+bool dcn20_get_dcc_compression_cap(const struct dc *dc,
+		const struct dc_dcc_surface_param *input,
+		struct dc_surface_dcc_cap *output)
+{
+	return dc->res_pool->hubbub->funcs->get_dcc_compression_cap(
+			dc->res_pool->hubbub,
+			input,
+			output);
+}
+
+static void dcn20_destroy_resource_pool(struct resource_pool **pool)
+{
+	struct dcn20_resource_pool *dcn20_pool = TO_DCN20_RES_POOL(*pool);
+
+	dcn20_resource_destruct(dcn20_pool);
+	kfree(dcn20_pool);
+	*pool = NULL;
+}
+
+
+static struct dc_cap_funcs cap_funcs = {
+	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
+};
+
+
+enum dc_status dcn20_patch_unknown_plane_state(struct dc_plane_state *plane_state)
+{
+	enum surface_pixel_format surf_pix_format = plane_state->format;
+	unsigned int bpp = resource_pixel_format_to_bpp(surf_pix_format);
+
+	plane_state->tiling_info.gfx9.swizzle = DC_SW_64KB_S;
+	if (bpp == 64)
+		plane_state->tiling_info.gfx9.swizzle = DC_SW_64KB_D;
+
+	return DC_OK;
+}
+
+void dcn20_release_pipe(struct dc_state *context,
+			struct pipe_ctx *pipe,
+			const struct resource_pool *pool)
+{
+	if (resource_is_pipe_type(pipe, OPP_HEAD) && pipe->stream_res.dsc)
+		dcn20_release_dsc(&context->res_ctx, pool, &pipe->stream_res.dsc);
+	memset(pipe, 0, sizeof(*pipe));
+}
+
+static const struct resource_funcs dcn20_res_pool_funcs = {
+	.destroy = dcn20_destroy_resource_pool,
+	.link_enc_create = dcn20_link_encoder_create,
+	.panel_cntl_create = dcn20_panel_cntl_create,
+	.validate_bandwidth = dcn20_validate_bandwidth,
+	.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
+	.release_pipe = dcn20_release_pipe,
+	.add_stream_to_ctx = dcn20_add_stream_to_ctx,
+	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
+	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
+	.populate_dml_writeback_from_context = dcn20_populate_dml_writeback_from_context,
+	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
+	.set_mcif_arb_params = dcn20_set_mcif_arb_params,
+	.populate_dml_pipes = dcn20_populate_dml_pipes_from_context,
+	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link
+};
+
+bool dcn20_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn20_dwbc *dwbc20 = kzalloc(sizeof(struct dcn20_dwbc),
+						    GFP_KERNEL);
+
+		if (!dwbc20) {
+			dm_error("DC: failed to create dwbc20!\n");
+			return false;
+		}
+		dcn20_dwbc_construct(dwbc20, ctx,
+				&dwbc20_regs[i],
+				&dwbc20_shift,
+				&dwbc20_mask,
+				i);
+		pool->dwbc[i] = &dwbc20->base;
+	}
+	return true;
+}
+
+bool dcn20_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	ASSERT(pipe_count > 0);
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn20_mmhubbub *mcif_wb20 = kzalloc(sizeof(struct dcn20_mmhubbub),
+						    GFP_KERNEL);
+
+		if (!mcif_wb20) {
+			dm_error("DC: failed to create mcif_wb20!\n");
+			return false;
+		}
+
+		dcn20_mmhubbub_construct(mcif_wb20, ctx,
+				&mcif_wb20_regs[i],
+				&mcif_wb20_shift,
+				&mcif_wb20_mask,
+				i);
+
+		pool->mcif_wb[i] = &mcif_wb20->base;
+	}
+	return true;
+}
+
+static struct pp_smu_funcs *dcn20_pp_smu_create(struct dc_context *ctx)
+{
+	struct pp_smu_funcs *pp_smu = kzalloc(sizeof(*pp_smu), GFP_ATOMIC);
+
+	if (!pp_smu)
+		return pp_smu;
+
+	dm_pp_get_funcs(ctx, pp_smu);
+
+	if (pp_smu->ctx.ver != PP_SMU_VER_NV)
+		pp_smu = memset(pp_smu, 0, sizeof(struct pp_smu_funcs));
+
+	return pp_smu;
+}
+
+static void dcn20_pp_smu_destroy(struct pp_smu_funcs **pp_smu)
+{
+	if (pp_smu && *pp_smu) {
+		kfree(*pp_smu);
+		*pp_smu = NULL;
+	}
+}
+
+static struct _vcs_dpi_soc_bounding_box_st *get_asic_rev_soc_bb(
+	uint32_t hw_internal_rev)
+{
+	if (ASICREV_IS_NAVI14_M(hw_internal_rev))
+		return &dcn2_0_nv14_soc;
+
+	if (ASICREV_IS_NAVI12_P(hw_internal_rev))
+		return &dcn2_0_nv12_soc;
+
+	return &dcn2_0_soc;
+}
+
+static struct _vcs_dpi_ip_params_st *get_asic_rev_ip_params(
+	uint32_t hw_internal_rev)
+{
+	/* NV14 */
+	if (ASICREV_IS_NAVI14_M(hw_internal_rev))
+		return &dcn2_0_nv14_ip;
+
+	/* NV12 and NV10 */
+	return &dcn2_0_ip;
+}
+
+static enum dml_project get_dml_project_version(uint32_t hw_internal_rev)
+{
+	return DML_PROJECT_NAVI10v2;
+}
+
+static bool init_soc_bounding_box(struct dc *dc,
+				  struct dcn20_resource_pool *pool)
+{
+	struct _vcs_dpi_soc_bounding_box_st *loaded_bb =
+			get_asic_rev_soc_bb(dc->ctx->asic_id.hw_internal_rev);
+	struct _vcs_dpi_ip_params_st *loaded_ip =
+			get_asic_rev_ip_params(dc->ctx->asic_id.hw_internal_rev);
+
+	DC_LOGGER_INIT(dc->ctx->logger);
+
+	if (pool->base.pp_smu) {
+		struct pp_smu_nv_clock_table max_clocks = {0};
+		unsigned int uclk_states[8] = {0};
+		unsigned int num_states = 0;
+		enum pp_smu_status status;
+		bool clock_limits_available = false;
+		bool uclk_states_available = false;
+
+		if (pool->base.pp_smu->nv_funcs.get_uclk_dpm_states) {
+			status = (pool->base.pp_smu->nv_funcs.get_uclk_dpm_states)
+				(&pool->base.pp_smu->nv_funcs.pp_smu, uclk_states, &num_states);
+
+			uclk_states_available = (status == PP_SMU_RESULT_OK);
+		}
+
+		if (pool->base.pp_smu->nv_funcs.get_maximum_sustainable_clocks) {
+			status = (*pool->base.pp_smu->nv_funcs.get_maximum_sustainable_clocks)
+					(&pool->base.pp_smu->nv_funcs.pp_smu, &max_clocks);
+			/* SMU cannot set DCF clock to anything equal to or higher than SOC clock
+			 */
+			if (max_clocks.dcfClockInKhz >= max_clocks.socClockInKhz)
+				max_clocks.dcfClockInKhz = max_clocks.socClockInKhz - 1000;
+			clock_limits_available = (status == PP_SMU_RESULT_OK);
+		}
+
+		if (clock_limits_available && uclk_states_available && num_states) {
+			DC_FP_START();
+			dcn20_update_bounding_box(dc, loaded_bb, &max_clocks, uclk_states, num_states);
+			DC_FP_END();
+		} else if (clock_limits_available) {
+			DC_FP_START();
+			dcn20_cap_soc_clocks(loaded_bb, max_clocks);
+			DC_FP_END();
+		}
+	}
+
+	loaded_ip->max_num_otg = pool->base.res_cap->num_timing_generator;
+	loaded_ip->max_num_dpp = pool->base.pipe_count;
+	DC_FP_START();
+	dcn20_patch_bounding_box(dc, loaded_bb);
+	DC_FP_END();
+	return true;
+}
+
+static bool dcn20_resource_construct(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct dcn20_resource_pool *pool)
+{
+	int i;
+	struct dc_context *ctx = dc->ctx;
+	struct irq_service_init_data init_data;
+	struct ddc_service_init_data ddc_init_data = {0};
+	struct _vcs_dpi_soc_bounding_box_st *loaded_bb =
+			get_asic_rev_soc_bb(ctx->asic_id.hw_internal_rev);
+	struct _vcs_dpi_ip_params_st *loaded_ip =
+			get_asic_rev_ip_params(ctx->asic_id.hw_internal_rev);
+	enum dml_project dml_project_version =
+			get_dml_project_version(ctx->asic_id.hw_internal_rev);
+
+	ctx->dc_bios->regs = &bios_regs;
+	pool->base.funcs = &dcn20_res_pool_funcs;
+
+	if (ASICREV_IS_NAVI14_M(ctx->asic_id.hw_internal_rev)) {
+		pool->base.res_cap = &res_cap_nv14;
+		pool->base.pipe_count = 5;
+		pool->base.mpcc_count = 5;
+	} else {
+		pool->base.res_cap = &res_cap_nv10;
+		pool->base.pipe_count = 6;
+		pool->base.mpcc_count = 6;
+	}
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+
+	dc->caps.max_downscale_ratio = 200;
+	dc->caps.i2c_speed_in_khz = 100;
+	dc->caps.i2c_speed_in_khz_hdcp = 100; /*1.4 w/a not applied by default*/
+	dc->caps.max_cursor_size = 256;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.dmdata_alloc_size = 2048;
+
+	dc->caps.max_slave_planes = 1;
+	dc->caps.max_slave_yuv_planes = 1;
+	dc->caps.max_slave_rgb_planes = 1;
+	dc->caps.post_blend_color_processing = true;
+	dc->caps.force_dp_tps4_for_cp2520 = true;
+	dc->caps.extended_aux_timeout_support = true;
+
+	/* Color pipeline capabilities */
+	dc->caps.color.dpp.dcn_arch = 1;
+	dc->caps.color.dpp.input_lut_shared = 0;
+	dc->caps.color.dpp.icsc = 1;
+	dc->caps.color.dpp.dgam_ram = 1;
+	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
+	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.dpp.dgam_rom_caps.pq = 0;
+	dc->caps.color.dpp.dgam_rom_caps.hlg = 0;
+	dc->caps.color.dpp.post_csc = 0;
+	dc->caps.color.dpp.gamma_corr = 0;
+	dc->caps.color.dpp.dgam_rom_for_yuv = 1;
+
+	dc->caps.color.dpp.hw_3d_lut = 1;
+	dc->caps.color.dpp.ogam_ram = 1;
+	// no OGAM ROM on DCN2, only MPC ROM
+	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
+	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
+	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
+	dc->caps.color.dpp.ocsc = 0;
+
+	dc->caps.color.mpc.gamut_remap = 0;
+	dc->caps.color.mpc.num_3dluts = 0;
+	dc->caps.color.mpc.shared_3d_lut = 0;
+	dc->caps.color.mpc.ogam_ram = 1;
+	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
+	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
+	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
+	dc->caps.color.mpc.ocsc = 1;
+
+	dc->caps.dp_hdmi21_pcon_support = true;
+
+	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
+		dc->debug = debug_defaults_drv;
+
+	//dcn2.0x
+	dc->work_arounds.dedcn20_305_wa = true;
+
+	// Init the vm_helper
+	if (dc->vm_helper)
+		vm_helper_init(dc->vm_helper, 16);
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	pool->base.clock_sources[DCN20_CLK_SRC_PLL0] =
+			dcn20_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL0,
+				&clk_src_regs[0], false);
+	pool->base.clock_sources[DCN20_CLK_SRC_PLL1] =
+			dcn20_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL1,
+				&clk_src_regs[1], false);
+	pool->base.clock_sources[DCN20_CLK_SRC_PLL2] =
+			dcn20_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL2,
+				&clk_src_regs[2], false);
+	pool->base.clock_sources[DCN20_CLK_SRC_PLL3] =
+			dcn20_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL3,
+				&clk_src_regs[3], false);
+	pool->base.clock_sources[DCN20_CLK_SRC_PLL4] =
+			dcn20_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL4,
+				&clk_src_regs[4], false);
+	pool->base.clock_sources[DCN20_CLK_SRC_PLL5] =
+			dcn20_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL5,
+				&clk_src_regs[5], false);
+	pool->base.clk_src_count = DCN20_CLK_SRC_TOTAL;
+	/* todo: not reuse phy_pll registers */
+	pool->base.dp_clock_source =
+			dcn20_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_ID_DP_DTO,
+				&clk_src_regs[0], true);
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	pool->base.dccg = dccg2_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
+	if (pool->base.dccg == NULL) {
+		dm_error("DC: failed to create dccg!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	pool->base.dmcu = dcn20_dmcu_create(ctx,
+			&dmcu_regs,
+			&dmcu_shift,
+			&dmcu_mask);
+	if (pool->base.dmcu == NULL) {
+		dm_error("DC: failed to create dmcu!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	pool->base.abm = dce_abm_create(ctx,
+			&abm_regs,
+			&abm_shift,
+			&abm_mask);
+	if (pool->base.abm == NULL) {
+		dm_error("DC: failed to create abm!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	pool->base.pp_smu = dcn20_pp_smu_create(ctx);
+
+
+	if (!init_soc_bounding_box(dc, pool)) {
+		dm_error("DC: failed to initialize soc bounding box!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	dml_init_instance(&dc->dml, loaded_bb, loaded_ip, dml_project_version);
+
+	if (!dc->debug.disable_pplib_wm_range) {
+		struct pp_smu_wm_range_sets ranges = {0};
+		int i = 0;
+
+		ranges.num_reader_wm_sets = 0;
+
+		if (loaded_bb->num_states == 1) {
+			ranges.reader_wm_sets[0].wm_inst = i;
+			ranges.reader_wm_sets[0].min_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
+			ranges.reader_wm_sets[0].max_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
+			ranges.reader_wm_sets[0].min_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
+			ranges.reader_wm_sets[0].max_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
+
+			ranges.num_reader_wm_sets = 1;
+		} else if (loaded_bb->num_states > 1) {
+			for (i = 0; i < 4 && i < loaded_bb->num_states; i++) {
+				ranges.reader_wm_sets[i].wm_inst = i;
+				ranges.reader_wm_sets[i].min_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
+				ranges.reader_wm_sets[i].max_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
+				DC_FP_START();
+				dcn20_fpu_set_wm_ranges(i, &ranges, loaded_bb);
+				DC_FP_END();
+
+				ranges.num_reader_wm_sets = i + 1;
+			}
+
+			ranges.reader_wm_sets[0].min_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
+			ranges.reader_wm_sets[ranges.num_reader_wm_sets - 1].max_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
+		}
+
+		ranges.num_writer_wm_sets = 1;
+
+		ranges.writer_wm_sets[0].wm_inst = 0;
+		ranges.writer_wm_sets[0].min_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
+		ranges.writer_wm_sets[0].max_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
+		ranges.writer_wm_sets[0].min_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
+		ranges.writer_wm_sets[0].max_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
+
+		/* Notify PP Lib/SMU which Watermarks to use for which clock ranges */
+		if (pool->base.pp_smu->nv_funcs.set_wm_ranges)
+			pool->base.pp_smu->nv_funcs.set_wm_ranges(&pool->base.pp_smu->nv_funcs.pp_smu, &ranges);
+	}
+
+	init_data.ctx = dc->ctx;
+	pool->base.irqs = dal_irq_service_dcn20_create(&init_data);
+	if (!pool->base.irqs)
+		goto create_fail;
+
+	/* mem input -> ipp -> dpp -> opp -> TG */
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		pool->base.hubps[i] = dcn20_hubp_create(ctx, i);
+		if (pool->base.hubps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create memory input!\n");
+			goto create_fail;
+		}
+
+		pool->base.ipps[i] = dcn20_ipp_create(ctx, i);
+		if (pool->base.ipps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create input pixel processor!\n");
+			goto create_fail;
+		}
+
+		pool->base.dpps[i] = dcn20_dpp_create(ctx, i);
+		if (pool->base.dpps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create dpps!\n");
+			goto create_fail;
+		}
+	}
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.engines[i] = dcn20_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto create_fail;
+		}
+		pool->base.hw_i2cs[i] = dcn20_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create hw i2c!!\n");
+			goto create_fail;
+		}
+		pool->base.sw_i2cs[i] = NULL;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		pool->base.opps[i] = dcn20_opp_create(ctx, i);
+		if (pool->base.opps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create output pixel processor!\n");
+			goto create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		pool->base.timing_generators[i] = dcn20_timing_generator_create(
+				ctx, i);
+		if (pool->base.timing_generators[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto create_fail;
+		}
+	}
+
+	pool->base.timing_generator_count = i;
+
+	pool->base.mpc = dcn20_mpc_create(ctx);
+	if (pool->base.mpc == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mpc!\n");
+		goto create_fail;
+	}
+
+	pool->base.hubbub = dcn20_hubbub_create(ctx);
+	if (pool->base.hubbub == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create hubbub!\n");
+		goto create_fail;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		pool->base.dscs[i] = dcn20_dsc_create(ctx, i);
+		if (pool->base.dscs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create display stream compressor %d!\n", i);
+			goto create_fail;
+		}
+	}
+
+	if (!dcn20_dwbc_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create dwbc!\n");
+		goto create_fail;
+	}
+	if (!dcn20_mmhubbub_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mcif_wb!\n");
+		goto create_fail;
+	}
+
+	if (!resource_construct(num_virtual_links, dc, &pool->base,
+			&res_create_funcs))
+		goto create_fail;
+
+	dcn20_hw_sequencer_construct(dc);
+
+	// IF NV12, set PG function pointer to NULL. It's not that
+	// PG isn't supported for NV12, it's that we don't want to
+	// program the registers because that will cause more power
+	// to be consumed. We could have created dcn20_init_hw to get
+	// the same effect by checking ASIC rev, but there was a
+	// request at some point to not check ASIC rev on hw sequencer.
+	if (ASICREV_IS_NAVI12_P(dc->ctx->asic_id.hw_internal_rev)) {
+		dc->hwseq->funcs.enable_power_gating_plane = NULL;
+		dc->debug.disable_dpp_power_gate = true;
+		dc->debug.disable_hubp_power_gate = true;
+	}
+
+
+	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->cap_funcs = cap_funcs;
+
+	if (dc->ctx->dc_bios->fw_info.oem_i2c_present) {
+		ddc_init_data.ctx = dc->ctx;
+		ddc_init_data.link = NULL;
+		ddc_init_data.id.id = dc->ctx->dc_bios->fw_info.oem_i2c_obj_id;
+		ddc_init_data.id.enum_id = 0;
+		ddc_init_data.id.type = OBJECT_TYPE_GENERIC;
+		pool->base.oem_device = dc->link_srv->create_ddc_service(&ddc_init_data);
+	} else {
+		pool->base.oem_device = NULL;
+	}
+
+	return true;
+
+create_fail:
+
+	dcn20_resource_destruct(pool);
+
+	return false;
+}
+
+struct resource_pool *dcn20_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc)
+{
+	struct dcn20_resource_pool *pool =
+		kzalloc(sizeof(struct dcn20_resource_pool), GFP_ATOMIC);
+
+	if (!pool)
+		return NULL;
+
+	if (dcn20_resource_construct(init_data->num_virtual_links, dc, pool))
+		return &pool->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(pool);
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.h
new file mode 100644
index 000000000000..37ecaccc5d12
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.h
@@ -0,0 +1,170 @@
+/*
+* Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_RESOURCE_DCN20_H__
+#define __DC_RESOURCE_DCN20_H__
+
+#include "core_types.h"
+#include "dml/dcn20/dcn20_fpu.h"
+
+#define TO_DCN20_RES_POOL(pool)\
+	container_of(pool, struct dcn20_resource_pool, base)
+
+struct dc;
+struct resource_pool;
+struct _vcs_dpi_display_pipe_params_st;
+
+extern struct _vcs_dpi_ip_params_st dcn2_0_ip;
+extern struct _vcs_dpi_ip_params_st dcn2_0_nv14_ip;
+extern struct _vcs_dpi_soc_bounding_box_st dcn2_0_soc;
+extern struct _vcs_dpi_soc_bounding_box_st dcn2_0_nv14_soc;
+extern struct _vcs_dpi_soc_bounding_box_st dcn2_0_nv12_soc;
+
+struct dcn20_resource_pool {
+	struct resource_pool base;
+};
+struct resource_pool *dcn20_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc);
+
+struct link_encoder *dcn20_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data);
+
+unsigned int dcn20_calc_max_scaled_time(
+		unsigned int time_per_pixel,
+		enum mmhubbub_wbif_mode mode,
+		unsigned int urgent_watermark);
+
+struct pipe_ctx *dcn20_acquire_free_pipe_for_layer(
+		const struct dc_state *cur_ctx,
+		struct dc_state *new_ctx,
+		const struct resource_pool *pool,
+		const struct pipe_ctx *opp_head_pipe);
+void dcn20_release_pipe(struct dc_state *context,
+			struct pipe_ctx *pipe,
+			const struct resource_pool *pool);
+struct stream_encoder *dcn20_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx);
+
+struct dce_hwseq *dcn20_hwseq_create(
+	struct dc_context *ctx);
+
+bool dcn20_get_dcc_compression_cap(const struct dc *dc,
+		const struct dc_dcc_surface_param *input,
+		struct dc_surface_dcc_cap *output);
+
+void dcn20_dpp_destroy(struct dpp **dpp);
+
+struct dpp *dcn20_dpp_create(
+	struct dc_context *ctx,
+	uint32_t inst);
+
+struct input_pixel_processor *dcn20_ipp_create(
+	struct dc_context *ctx, uint32_t inst);
+
+struct output_pixel_processor *dcn20_opp_create(
+	struct dc_context *ctx, uint32_t inst);
+
+struct dce_aux *dcn20_aux_engine_create(
+	struct dc_context *ctx, uint32_t inst);
+
+struct dce_i2c_hw *dcn20_i2c_hw_create(
+	struct dc_context *ctx,
+	uint32_t inst);
+
+void dcn20_clock_source_destroy(struct clock_source **clk_src);
+
+struct display_stream_compressor *dcn20_dsc_create(
+	struct dc_context *ctx, uint32_t inst);
+void dcn20_dsc_destroy(struct display_stream_compressor **dsc);
+
+struct hubp *dcn20_hubp_create(
+	struct dc_context *ctx,
+	uint32_t inst);
+struct timing_generator *dcn20_timing_generator_create(
+		struct dc_context *ctx,
+		uint32_t instance);
+struct mpc *dcn20_mpc_create(struct dc_context *ctx);
+struct hubbub *dcn20_hubbub_create(struct dc_context *ctx);
+
+bool dcn20_dwbc_create(struct dc_context *ctx, struct resource_pool *pool);
+bool dcn20_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool);
+
+void dcn20_set_mcif_arb_params(
+		struct dc *dc,
+		struct dc_state *context,
+		display_e2e_pipe_params_st *pipes,
+		int pipe_cnt);
+bool dcn20_validate_bandwidth(struct dc *dc, struct dc_state *context, bool fast_validate);
+void dcn20_merge_pipes_for_validate(
+		struct dc *dc,
+		struct dc_state *context);
+int dcn20_validate_apply_pipe_split_flags(
+		struct dc *dc,
+		struct dc_state *context,
+		int vlevel,
+		int *split,
+		bool *merge);
+void dcn20_release_dsc(struct resource_context *res_ctx,
+			const struct resource_pool *pool,
+			struct display_stream_compressor **dsc);
+bool dcn20_validate_dsc(struct dc *dc, struct dc_state *new_ctx);
+void dcn20_split_stream_for_mpc(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		struct pipe_ctx *primary_pipe,
+		struct pipe_ctx *secondary_pipe);
+bool dcn20_split_stream_for_odm(
+		const struct dc *dc,
+		struct resource_context *res_ctx,
+		struct pipe_ctx *prev_odm_pipe,
+		struct pipe_ctx *next_odm_pipe);
+void dcn20_acquire_dsc(const struct dc *dc,
+			struct resource_context *res_ctx,
+			struct display_stream_compressor **dsc,
+			int pipe_idx);
+struct pipe_ctx *dcn20_find_secondary_pipe(struct dc *dc,
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		const struct pipe_ctx *primary_pipe);
+bool dcn20_fast_validate_bw(
+		struct dc *dc,
+		struct dc_state *context,
+		display_e2e_pipe_params_st *pipes,
+		int *pipe_cnt_out,
+		int *pipe_split_from,
+		int *vlevel_out,
+		bool fast_validate);
+
+enum dc_status dcn20_build_mapped_resource(const struct dc *dc, struct dc_state *context, struct dc_stream_state *stream);
+enum dc_status dcn20_add_stream_to_ctx(struct dc *dc, struct dc_state *new_ctx, struct dc_stream_state *dc_stream);
+enum dc_status dcn20_add_dsc_to_stream_resource(struct dc *dc, struct dc_state *dc_ctx, struct dc_stream_state *dc_stream);
+enum dc_status dcn20_remove_stream_from_ctx(struct dc *dc, struct dc_state *new_ctx, struct dc_stream_state *dc_stream);
+enum dc_status dcn20_patch_unknown_plane_state(struct dc_plane_state *plane_state);
+
+#endif /* __DC_RESOURCE_DCN20_H__ */
+
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn201/dcn201_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn201/dcn201_resource.c
new file mode 100644
index 000000000000..914b234d7f6b
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn201/dcn201_resource.c
@@ -0,0 +1,1308 @@
+/*
+* Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dm_services.h"
+#include "dc.h"
+
+#include "dcn201/dcn201_init.h"
+#include "dml/dcn20/dcn20_fpu.h"
+#include "resource.h"
+#include "include/irq_service_interface.h"
+#include "dcn201_resource.h"
+
+#include "dcn20/dcn20_resource.h"
+
+#include "dcn10/dcn10_hubp.h"
+#include "dcn10/dcn10_ipp.h"
+#include "dcn201/dcn201_mpc.h"
+#include "dcn201/dcn201_hubp.h"
+#include "irq/dcn201/irq_service_dcn201.h"
+#include "dcn201/dcn201_dpp.h"
+#include "dcn201/dcn201_hubbub.h"
+#include "dcn201/dcn201_dccg.h"
+#include "dcn201/dcn201_optc.h"
+#include "dcn201/dcn201_hwseq.h"
+#include "dce110/dce110_hwseq.h"
+#include "dcn201/dcn201_opp.h"
+#include "dcn201/dcn201_link_encoder.h"
+#include "dcn20/dcn20_stream_encoder.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_hwseq.h"
+#include "virtual/virtual_stream_encoder.h"
+#include "dce110/dce110_resource.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_i2c.h"
+#include "dcn201/dcn201_hubbub.h"
+#include "dcn10/dcn10_resource.h"
+
+#include "cyan_skillfish_ip_offset.h"
+
+#include "dcn/dcn_2_0_3_offset.h"
+#include "dcn/dcn_2_0_3_sh_mask.h"
+#include "dpcs/dpcs_2_0_3_offset.h"
+#include "dpcs/dpcs_2_0_3_sh_mask.h"
+
+#include "mmhub/mmhub_2_0_0_offset.h"
+#include "mmhub/mmhub_2_0_0_sh_mask.h"
+#include "nbio/nbio_7_4_offset.h"
+
+#include "reg_helper.h"
+
+#define MIN_DISP_CLK_KHZ 100000
+#define MIN_DPP_CLK_KHZ 100000
+
+static struct _vcs_dpi_ip_params_st dcn201_ip = {
+	.gpuvm_enable = 0,
+	.hostvm_enable = 0,
+	.gpuvm_max_page_table_levels = 4,
+	.hostvm_max_page_table_levels = 4,
+	.hostvm_cached_page_table_levels = 0,
+	.pte_group_size_bytes = 2048,
+	.rob_buffer_size_kbytes = 168,
+	.det_buffer_size_kbytes = 164,
+	.dpte_buffer_size_in_pte_reqs_luma = 84,
+	.pde_proc_buffer_size_64k_reqs = 48,
+	.dpp_output_buffer_pixels = 2560,
+	.opp_output_buffer_lines = 1,
+	.pixel_chunk_size_kbytes = 8,
+	.pte_chunk_size_kbytes = 2,
+	.meta_chunk_size_kbytes = 2,
+	.writeback_chunk_size_kbytes = 2,
+	.line_buffer_size_bits = 789504,
+	.is_line_buffer_bpp_fixed = 0,
+	.line_buffer_fixed_bpp = 0,
+	.dcc_supported = true,
+	.max_line_buffer_lines = 12,
+	.writeback_luma_buffer_size_kbytes = 12,
+	.writeback_chroma_buffer_size_kbytes = 8,
+	.writeback_chroma_line_buffer_width_pixels = 4,
+	.writeback_max_hscl_ratio = 1,
+	.writeback_max_vscl_ratio = 1,
+	.writeback_min_hscl_ratio = 1,
+	.writeback_min_vscl_ratio = 1,
+	.writeback_max_hscl_taps = 12,
+	.writeback_max_vscl_taps = 12,
+	.writeback_line_buffer_luma_buffer_size = 0,
+	.writeback_line_buffer_chroma_buffer_size = 9600,
+	.cursor_buffer_size = 8,
+	.cursor_chunk_size = 2,
+	.max_num_otg = 2,
+	.max_num_dpp = 4,
+	.max_num_wb = 0,
+	.max_dchub_pscl_bw_pix_per_clk = 4,
+	.max_pscl_lb_bw_pix_per_clk = 2,
+	.max_lb_vscl_bw_pix_per_clk = 4,
+	.max_vscl_hscl_bw_pix_per_clk = 4,
+	.max_hscl_ratio = 8,
+	.max_vscl_ratio = 8,
+	.hscl_mults = 4,
+	.vscl_mults = 4,
+	.max_hscl_taps = 8,
+	.max_vscl_taps = 8,
+	.dispclk_ramp_margin_percent = 1,
+	.underscan_factor = 1.10,
+	.min_vblank_lines = 30,
+	.dppclk_delay_subtotal = 77,
+	.dppclk_delay_scl_lb_only = 16,
+	.dppclk_delay_scl = 50,
+	.dppclk_delay_cnvc_formatter = 8,
+	.dppclk_delay_cnvc_cursor = 6,
+	.dispclk_delay_subtotal = 87,
+	.dcfclk_cstate_latency = 10,
+	.max_inter_dcn_tile_repeaters = 8,
+	.number_of_cursors = 1,
+};
+
+static struct _vcs_dpi_soc_bounding_box_st dcn201_soc = {
+	.clock_limits = {
+			{
+				.state = 0,
+				.dscclk_mhz = 400.0,
+				.dcfclk_mhz = 1000.0,
+				.fabricclk_mhz = 200.0,
+				.dispclk_mhz = 300.0,
+				.dppclk_mhz = 300.0,
+				.phyclk_mhz = 810.0,
+				.socclk_mhz = 1254.0,
+				.dram_speed_mts = 2000.0,
+			},
+			{
+				.state = 1,
+				.dscclk_mhz = 400.0,
+				.dcfclk_mhz = 1000.0,
+				.fabricclk_mhz = 250.0,
+				.dispclk_mhz = 1200.0,
+				.dppclk_mhz = 1200.0,
+				.phyclk_mhz = 810.0,
+				.socclk_mhz = 1254.0,
+				.dram_speed_mts = 3600.0,
+			},
+			{
+				.state = 2,
+				.dscclk_mhz = 400.0,
+				.dcfclk_mhz = 1000.0,
+				.fabricclk_mhz = 750.0,
+				.dispclk_mhz = 1200.0,
+				.dppclk_mhz = 1200.0,
+				.phyclk_mhz = 810.0,
+				.socclk_mhz = 1254.0,
+				.dram_speed_mts = 6800.0,
+			},
+			{
+				.state = 3,
+				.dscclk_mhz = 400.0,
+				.dcfclk_mhz = 1000.0,
+				.fabricclk_mhz = 250.0,
+				.dispclk_mhz = 1200.0,
+				.dppclk_mhz = 1200.0,
+				.phyclk_mhz = 810.0,
+				.socclk_mhz = 1254.0,
+				.dram_speed_mts = 14000.0,
+			},
+			{
+				.state = 4,
+				.dscclk_mhz = 400.0,
+				.dcfclk_mhz = 1000.0,
+				.fabricclk_mhz = 750.0,
+				.dispclk_mhz = 1200.0,
+				.dppclk_mhz = 1200.0,
+				.phyclk_mhz = 810.0,
+				.socclk_mhz = 1254.0,
+				.dram_speed_mts = 14000.0,
+			}
+		},
+	.num_states = 4,
+	.sr_exit_time_us = 9.0,
+	.sr_enter_plus_exit_time_us = 11.0,
+	.urgent_latency_us = 4.0,
+	.urgent_latency_pixel_data_only_us = 4.0,
+	.urgent_latency_pixel_mixed_with_vm_data_us = 4.0,
+	.urgent_latency_vm_data_only_us = 4.0,
+	.urgent_out_of_order_return_per_channel_pixel_only_bytes = 256,
+	.urgent_out_of_order_return_per_channel_pixel_and_vm_bytes = 256,
+	.urgent_out_of_order_return_per_channel_vm_only_bytes = 256,
+	.pct_ideal_dram_sdp_bw_after_urgent_pixel_only = 80.0,
+	.pct_ideal_dram_sdp_bw_after_urgent_pixel_and_vm = 80.0,
+	.pct_ideal_dram_sdp_bw_after_urgent_vm_only = 80.0,
+	.max_avg_sdp_bw_use_normal_percent = 80.0,
+	.max_avg_dram_bw_use_normal_percent = 69.0,
+	.writeback_latency_us = 12.0,
+	.ideal_dram_bw_after_urgent_percent = 80.0,
+	.max_request_size_bytes = 256,
+	.dram_channel_width_bytes = 2,
+	.fabric_datapath_to_dcn_data_return_bytes = 64,
+	.dcn_downspread_percent = 0.3,
+	.downspread_percent = 0.3,
+	.dram_page_open_time_ns = 50.0,
+	.dram_rw_turnaround_time_ns = 17.5,
+	.dram_return_buffer_per_channel_bytes = 8192,
+	.round_trip_ping_latency_dcfclk_cycles = 128,
+	.urgent_out_of_order_return_per_channel_bytes = 256,
+	.channel_interleave_bytes = 256,
+	.num_banks = 8,
+	.num_chans = 16,
+	.vmm_page_size_bytes = 4096,
+	.dram_clock_change_latency_us = 250.0,
+	.writeback_dram_clock_change_latency_us = 23.0,
+	.return_bus_width_bytes = 64,
+	.dispclk_dppclk_vco_speed_mhz = 3000,
+	.use_urgent_burst_bw = 0,
+};
+
+enum dcn20_clk_src_array_id {
+	DCN20_CLK_SRC_PLL0,
+	DCN20_CLK_SRC_PLL1,
+	DCN20_CLK_SRC_TOTAL_DCN201
+};
+
+/* begin *********************
+ * macros to expend register list macro defined in HW object header file */
+
+/* DCN */
+
+#undef BASE_INNER
+#define BASE_INNER(seg) DMU_BASE__INST0_SEG ## seg
+
+#define BASE(seg) BASE_INNER(seg)
+
+#define SR(reg_name)\
+		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) +  \
+					mm ## reg_name
+
+#define SRI(reg_name, block, id)\
+	.reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define SRIR(var_name, reg_name, block, id)\
+	.var_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define SRII(reg_name, block, id)\
+	.reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define SRI_IX(reg_name, block, id)\
+	.reg_name = ix ## block ## id ## _ ## reg_name
+
+#define DCCG_SRII(reg_name, block, id)\
+	.block ## _ ## reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define VUPDATE_SRII(reg_name, block, id)\
+	.reg_name[id] = BASE(mm ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
+					mm ## reg_name ## _ ## block ## id
+
+/* NBIO */
+#define NBIO_BASE_INNER(seg) \
+	NBIO_BASE__INST0_SEG ## seg
+
+#define NBIO_BASE(seg) \
+	NBIO_BASE_INNER(seg)
+
+#define NBIO_SR(reg_name)\
+		.reg_name = NBIO_BASE(mm ## reg_name ## _BASE_IDX) + \
+					mm ## reg_name
+
+/* MMHUB */
+#define MMHUB_BASE_INNER(seg) \
+	MMHUB_BASE__INST0_SEG ## seg
+
+#define MMHUB_BASE(seg) \
+	MMHUB_BASE_INNER(seg)
+
+#define MMHUB_SR(reg_name)\
+		.reg_name = MMHUB_BASE(mmMM ## reg_name ## _BASE_IDX) + \
+					mmMM ## reg_name
+
+static const struct bios_registers bios_regs = {
+		NBIO_SR(BIOS_SCRATCH_3),
+		NBIO_SR(BIOS_SCRATCH_6)
+};
+
+#define clk_src_regs(index, pllid)\
+[index] = {\
+	CS_COMMON_REG_LIST_DCN201(index, pllid),\
+}
+
+static const struct dce110_clk_src_regs clk_src_regs[] = {
+	clk_src_regs(0, A),
+	clk_src_regs(1, B)
+};
+
+static const struct dce110_clk_src_shift cs_shift = {
+		CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
+};
+
+static const struct dce110_clk_src_mask cs_mask = {
+		CS_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
+};
+
+#define audio_regs(id)\
+[id] = {\
+		AUD_COMMON_REG_LIST(id)\
+}
+
+static const struct dce_audio_registers audio_regs[] = {
+	audio_regs(0),
+	audio_regs(1),
+};
+
+#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
+		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
+
+static const struct dce_audio_shift audio_shift = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_audio_mask audio_mask = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+#define stream_enc_regs(id)\
+[id] = {\
+	SE_DCN2_REG_LIST(id)\
+}
+
+static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
+	stream_enc_regs(0),
+	stream_enc_regs(1)
+};
+
+static const struct dcn10_stream_encoder_shift se_shift = {
+		SE_COMMON_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn10_stream_encoder_mask se_mask = {
+		SE_COMMON_MASK_SH_LIST_DCN20(_MASK)
+};
+
+static const struct dce110_aux_registers_shift aux_shift = {
+	DCN_AUX_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+	DCN_AUX_MASK_SH_LIST(_MASK)
+};
+
+#define aux_regs(id)\
+[id] = {\
+	DCN2_AUX_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
+		aux_regs(0),
+		aux_regs(1),
+};
+
+#define hpd_regs(id)\
+[id] = {\
+	HPD_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
+		hpd_regs(0),
+		hpd_regs(1),
+};
+
+#define link_regs(id, phyid)\
+[id] = {\
+	LE_DCN_COMMON_REG_LIST(id), \
+	UNIPHY_DCN2_REG_LIST(phyid) \
+}
+
+static const struct dcn10_link_enc_registers link_enc_regs[] = {
+	link_regs(0, A),
+	link_regs(1, B),
+};
+
+#define LINK_ENCODER_MASK_SH_LIST_DCN201(mask_sh)\
+	LINK_ENCODER_MASK_SH_LIST_DCN20(mask_sh)
+
+static const struct dcn10_link_enc_shift le_shift = {
+		LINK_ENCODER_MASK_SH_LIST_DCN201(__SHIFT)
+};
+
+static const struct dcn10_link_enc_mask le_mask = {
+		LINK_ENCODER_MASK_SH_LIST_DCN201(_MASK)
+};
+
+#define ipp_regs(id)\
+[id] = {\
+		IPP_REG_LIST_DCN201(id),\
+}
+
+static const struct dcn10_ipp_registers ipp_regs[] = {
+	ipp_regs(0),
+	ipp_regs(1),
+	ipp_regs(2),
+	ipp_regs(3),
+};
+
+static const struct dcn10_ipp_shift ipp_shift = {
+		IPP_MASK_SH_LIST_DCN201(__SHIFT)
+};
+
+static const struct dcn10_ipp_mask ipp_mask = {
+		IPP_MASK_SH_LIST_DCN201(_MASK)
+};
+
+#define opp_regs(id)\
+[id] = {\
+	OPP_REG_LIST_DCN201(id),\
+}
+
+static const struct dcn201_opp_registers opp_regs[] = {
+	opp_regs(0),
+	opp_regs(1),
+};
+
+static const struct dcn201_opp_shift opp_shift = {
+		OPP_MASK_SH_LIST_DCN201(__SHIFT)
+};
+
+static const struct dcn201_opp_mask opp_mask = {
+		OPP_MASK_SH_LIST_DCN201(_MASK)
+};
+
+#define aux_engine_regs(id)\
+[id] = {\
+	AUX_COMMON_REG_LIST0(id), \
+	.AUX_RESET_MASK = 0 \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1)
+};
+
+#define tf_regs(id)\
+[id] = {\
+	TF_REG_LIST_DCN201(id),\
+}
+
+static const struct dcn201_dpp_registers tf_regs[] = {
+	tf_regs(0),
+	tf_regs(1),
+	tf_regs(2),
+	tf_regs(3),
+};
+
+static const struct dcn201_dpp_shift tf_shift = {
+		TF_REG_LIST_SH_MASK_DCN201(__SHIFT)
+};
+
+static const struct dcn201_dpp_mask tf_mask = {
+		TF_REG_LIST_SH_MASK_DCN201(_MASK)
+};
+
+static const struct dcn201_mpc_registers mpc_regs = {
+		MPC_REG_LIST_DCN201(0),
+		MPC_REG_LIST_DCN201(1),
+		MPC_REG_LIST_DCN201(2),
+		MPC_REG_LIST_DCN201(3),
+		MPC_REG_LIST_DCN201(4),
+		MPC_OUT_MUX_REG_LIST_DCN201(0),
+		MPC_OUT_MUX_REG_LIST_DCN201(1),
+};
+
+static const struct dcn201_mpc_shift mpc_shift = {
+	MPC_COMMON_MASK_SH_LIST_DCN201(__SHIFT)
+};
+
+static const struct dcn201_mpc_mask mpc_mask = {
+	MPC_COMMON_MASK_SH_LIST_DCN201(_MASK)
+};
+
+#define tg_regs_dcn201(id)\
+[id] = {TG_COMMON_REG_LIST_DCN201(id)}
+
+static const struct dcn_optc_registers tg_regs[] = {
+	tg_regs_dcn201(0),
+	tg_regs_dcn201(1)
+};
+
+static const struct dcn_optc_shift tg_shift = {
+	TG_COMMON_MASK_SH_LIST_DCN201(__SHIFT)
+};
+
+static const struct dcn_optc_mask tg_mask = {
+	TG_COMMON_MASK_SH_LIST_DCN201(_MASK)
+};
+
+#define hubp_regsDCN201(id)\
+[id] = {\
+	HUBP_REG_LIST_DCN201(id)\
+}
+
+static const struct dcn201_hubp_registers hubp_regs[] = {
+		hubp_regsDCN201(0),
+		hubp_regsDCN201(1),
+		hubp_regsDCN201(2),
+		hubp_regsDCN201(3)
+};
+
+static const struct dcn201_hubp_shift hubp_shift = {
+		HUBP_MASK_SH_LIST_DCN201(__SHIFT)
+};
+
+static const struct dcn201_hubp_mask hubp_mask = {
+		HUBP_MASK_SH_LIST_DCN201(_MASK)
+};
+
+static const struct dcn_hubbub_registers hubbub_reg = {
+		HUBBUB_REG_LIST_DCN201(0)
+};
+
+static const struct dcn_hubbub_shift hubbub_shift = {
+		HUBBUB_MASK_SH_LIST_DCN201(__SHIFT)
+};
+
+static const struct dcn_hubbub_mask hubbub_mask = {
+		HUBBUB_MASK_SH_LIST_DCN201(_MASK)
+};
+
+
+static const struct dccg_registers dccg_regs = {
+		DCCG_COMMON_REG_LIST_DCN_BASE()
+};
+
+static const struct dccg_shift dccg_shift = {
+		DCCG_COMMON_MASK_SH_LIST_DCN_COMMON_BASE(__SHIFT)
+};
+
+static const struct dccg_mask dccg_mask = {
+		DCCG_COMMON_MASK_SH_LIST_DCN_COMMON_BASE(_MASK)
+};
+
+static const struct resource_caps res_cap_dnc201 = {
+		.num_timing_generator = 2,
+		.num_opp = 2,
+		.num_video_plane = 4,
+		.num_audio = 2,
+		.num_stream_encoder = 2,
+		.num_pll = 2,
+		.num_ddc = 2,
+};
+
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
+	.per_pixel_alpha = true,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = false,
+			.fp16 = true,
+			.p010 = false,
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 16000,
+			.fp16 = 1
+	},
+
+	.max_downscale_factor = {
+			.argb8888 = 250,
+			.nv12 = 250,
+			.fp16 = 250
+	},
+	64,
+	64
+};
+
+static const struct dc_debug_options debug_defaults_drv = {
+		.disable_dmcu = true,
+		.force_abm_enable = false,
+		.timing_trace = false,
+		.clock_trace = true,
+		.disable_pplib_clock_request = true,
+		.pipe_split_policy = MPC_SPLIT_DYNAMIC,
+		.force_single_disp_pipe_split = false,
+		.disable_dcc = DCC_ENABLE,
+		.vsr_support = true,
+		.performance_trace = false,
+		.az_endpoint_mute_only = true,
+		.max_downscale_src_width = 3840,
+		.disable_pplib_wm_range = true,
+		.scl_reset_length10 = true,
+		.sanity_checks = false,
+		.underflow_assert_delay_us = 0xFFFFFFFF,
+		.enable_tri_buf = false,
+		.enable_legacy_fast_update = true,
+		.using_dml2 = false,
+};
+
+static void dcn201_dpp_destroy(struct dpp **dpp)
+{
+	kfree(TO_DCN201_DPP(*dpp));
+	*dpp = NULL;
+}
+
+static struct dpp *dcn201_dpp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn201_dpp *dpp =
+		kzalloc(sizeof(struct dcn201_dpp), GFP_ATOMIC);
+
+	if (!dpp)
+		return NULL;
+
+	if (dpp201_construct(dpp, ctx, inst,
+			&tf_regs[inst], &tf_shift, &tf_mask))
+		return &dpp->base;
+
+	kfree(dpp);
+	return NULL;
+}
+
+static struct input_pixel_processor *dcn201_ipp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn10_ipp *ipp =
+		kzalloc(sizeof(struct dcn10_ipp), GFP_ATOMIC);
+
+	if (!ipp) {
+		return NULL;
+	}
+
+	dcn20_ipp_construct(ipp, ctx, inst,
+			&ipp_regs[inst], &ipp_shift, &ipp_mask);
+	return &ipp->base;
+}
+
+
+static struct output_pixel_processor *dcn201_opp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn201_opp *opp =
+		kzalloc(sizeof(struct dcn201_opp), GFP_ATOMIC);
+
+	if (!opp) {
+		return NULL;
+	}
+
+	dcn201_opp_construct(opp, ctx, inst,
+			&opp_regs[inst], &opp_shift, &opp_mask);
+	return &opp->base;
+}
+
+static struct dce_aux *dcn201_aux_engine_create(struct dc_context *ctx,
+						uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_ATOMIC);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst],
+					&aux_mask,
+					&aux_shift,
+					ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
+
+static const struct dce_i2c_registers i2c_hw_regs[] = {
+		i2c_inst_regs(1),
+		i2c_inst_regs(2),
+};
+
+static const struct dce_i2c_shift i2c_shifts = {
+		I2C_COMMON_MASK_SH_LIST_DCN2(__SHIFT)
+};
+
+static const struct dce_i2c_mask i2c_masks = {
+		I2C_COMMON_MASK_SH_LIST_DCN2(_MASK)
+};
+
+static struct dce_i2c_hw *dcn201_i2c_hw_create(struct dc_context *ctx,
+					       uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw =
+		kzalloc(sizeof(struct dce_i2c_hw), GFP_ATOMIC);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
+				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+
+static struct mpc *dcn201_mpc_create(struct dc_context *ctx, uint32_t num_mpcc)
+{
+	struct dcn201_mpc *mpc201 = kzalloc(sizeof(struct dcn201_mpc),
+					    GFP_ATOMIC);
+
+	if (!mpc201)
+		return NULL;
+
+	dcn201_mpc_construct(mpc201, ctx,
+			&mpc_regs,
+			&mpc_shift,
+			&mpc_mask,
+			num_mpcc);
+
+	return &mpc201->base;
+}
+
+static struct hubbub *dcn201_hubbub_create(struct dc_context *ctx)
+{
+	struct dcn20_hubbub *hubbub = kzalloc(sizeof(struct dcn20_hubbub),
+					  GFP_ATOMIC);
+
+	if (!hubbub)
+		return NULL;
+
+	hubbub201_construct(hubbub, ctx,
+			&hubbub_reg,
+			&hubbub_shift,
+			&hubbub_mask);
+
+	return &hubbub->base;
+}
+
+static struct timing_generator *dcn201_timing_generator_create(
+		struct dc_context *ctx,
+		uint32_t instance)
+{
+	struct optc *tgn10 =
+		kzalloc(sizeof(struct optc), GFP_ATOMIC);
+
+	if (!tgn10)
+		return NULL;
+
+	tgn10->base.inst = instance;
+	tgn10->base.ctx = ctx;
+
+	tgn10->tg_regs = &tg_regs[instance];
+	tgn10->tg_shift = &tg_shift;
+	tgn10->tg_mask = &tg_mask;
+
+	dcn201_timing_generator_init(tgn10);
+
+	return &tgn10->base;
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+		.max_hdmi_deep_color = COLOR_DEPTH_121212,
+		.max_hdmi_pixel_clock = 600000,
+		.hdmi_ycbcr420_supported = true,
+		.dp_ycbcr420_supported = true,
+		.fec_supported = true,
+		.flags.bits.IS_HBR2_CAPABLE = true,
+		.flags.bits.IS_HBR3_CAPABLE = true,
+		.flags.bits.IS_TPS3_CAPABLE = true,
+		.flags.bits.IS_TPS4_CAPABLE = true
+};
+
+static struct link_encoder *dcn201_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dcn20_link_encoder *enc20 =
+		kzalloc(sizeof(struct dcn20_link_encoder), GFP_ATOMIC);
+	struct dcn10_link_encoder *enc10 = &enc20->enc10;
+
+	if (!enc20)
+		return NULL;
+
+	dcn201_link_encoder_construct(enc20,
+			enc_init_data,
+			&link_enc_feature,
+			&link_enc_regs[enc_init_data->transmitter],
+			&link_enc_aux_regs[enc_init_data->channel - 1],
+			&link_enc_hpd_regs[enc_init_data->hpd_source],
+			&le_shift,
+			&le_mask);
+
+	return &enc10->base;
+}
+
+static struct clock_source *dcn201_clock_source_create(
+	struct dc_context *ctx,
+	struct dc_bios *bios,
+	enum clock_source_id id,
+	const struct dce110_clk_src_regs *regs,
+	bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(struct dce110_clk_src), GFP_ATOMIC);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dce112_clk_src_construct(clk_src, ctx, bios, id,
+			regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+	kfree(clk_src);
+	return NULL;
+}
+
+static void read_dce_straps(
+	struct dc_context *ctx,
+	struct resource_straps *straps)
+{
+	generic_reg_get(ctx, mmDC_PINSTRAPS + BASE(mmDC_PINSTRAPS_BASE_IDX),
+
+		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
+}
+
+static struct audio *dcn201_create_audio(
+		struct dc_context *ctx, unsigned int inst)
+{
+	return dce_audio_create(ctx, inst,
+			&audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+static struct stream_encoder *dcn201_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dcn10_stream_encoder *enc1 =
+		kzalloc(sizeof(struct dcn10_stream_encoder), GFP_ATOMIC);
+
+	if (!enc1)
+		return NULL;
+
+	dcn20_stream_encoder_construct(enc1, ctx, ctx->dc_bios, eng_id,
+					&stream_enc_regs[eng_id],
+					&se_shift, &se_mask);
+
+	return &enc1->base;
+}
+
+static const struct dce_hwseq_registers hwseq_reg = {
+		HWSEQ_DCN201_REG_LIST()
+};
+
+static const struct dce_hwseq_shift hwseq_shift = {
+		HWSEQ_DCN201_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+		HWSEQ_DCN201_MASK_SH_LIST(_MASK)
+};
+
+static struct dce_hwseq *dcn201_hwseq_create(
+	struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_ATOMIC);
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &hwseq_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+	}
+	return hws;
+}
+
+static const struct resource_create_funcs res_create_funcs = {
+	.read_dce_straps = read_dce_straps,
+	.create_audio = dcn201_create_audio,
+	.create_stream_encoder = dcn201_stream_encoder_create,
+	.create_hwseq = dcn201_hwseq_create,
+};
+
+static void dcn201_clock_source_destroy(struct clock_source **clk_src)
+{
+	kfree(TO_DCE110_CLK_SRC(*clk_src));
+	*clk_src = NULL;
+}
+
+static void dcn201_resource_destruct(struct dcn201_resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->base.stream_enc_count; i++) {
+		if (pool->base.stream_enc[i] != NULL) {
+			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
+			pool->base.stream_enc[i] = NULL;
+		}
+	}
+
+
+	if (pool->base.mpc != NULL) {
+		kfree(TO_DCN201_MPC(pool->base.mpc));
+		pool->base.mpc = NULL;
+	}
+
+	if (pool->base.hubbub != NULL) {
+		kfree(pool->base.hubbub);
+		pool->base.hubbub = NULL;
+	}
+
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (pool->base.dpps[i] != NULL)
+			dcn201_dpp_destroy(&pool->base.dpps[i]);
+
+		if (pool->base.ipps[i] != NULL)
+			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
+
+		if (pool->base.hubps[i] != NULL) {
+			kfree(TO_DCN10_HUBP(pool->base.hubps[i]));
+			pool->base.hubps[i] = NULL;
+		}
+
+		if (pool->base.irqs != NULL) {
+			dal_irq_service_destroy(&pool->base.irqs);
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		if (pool->base.opps[i] != NULL)
+			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.timing_generators[i] != NULL)	{
+			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
+			pool->base.timing_generators[i] = NULL;
+		}
+	}
+	for (i = 0; i < pool->base.audio_count; i++) {
+		if (pool->base.audios[i])
+			dce_aud_destroy(&pool->base.audios[i]);
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] != NULL) {
+			dcn201_clock_source_destroy(&pool->base.clock_sources[i]);
+			pool->base.clock_sources[i] = NULL;
+		}
+	}
+
+	if (pool->base.dp_clock_source != NULL) {
+		dcn201_clock_source_destroy(&pool->base.dp_clock_source);
+		pool->base.dp_clock_source = NULL;
+	}
+
+	if (pool->base.dccg != NULL)
+		dcn_dccg_destroy(&pool->base.dccg);
+}
+
+static struct hubp *dcn201_hubp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn201_hubp *hubp201 =
+		kzalloc(sizeof(struct dcn201_hubp), GFP_ATOMIC);
+
+	if (!hubp201)
+		return NULL;
+
+	if (dcn201_hubp_construct(hubp201, ctx, inst,
+			&hubp_regs[inst], &hubp_shift, &hubp_mask))
+		return &hubp201->base;
+
+	kfree(hubp201);
+	return NULL;
+}
+
+static struct pipe_ctx *dcn201_acquire_free_pipe_for_layer(
+		const struct dc_state *cur_ctx,
+		struct dc_state *new_ctx,
+		const struct resource_pool *pool,
+		const struct pipe_ctx *opp_head_pipe)
+{
+	struct resource_context *res_ctx = &new_ctx->res_ctx;
+	struct pipe_ctx *head_pipe = resource_get_otg_master_for_stream(res_ctx, opp_head_pipe->stream);
+	struct pipe_ctx *idle_pipe = resource_find_free_secondary_pipe_legacy(res_ctx, pool, head_pipe);
+
+	if (!head_pipe)
+		ASSERT(0);
+
+	if (!idle_pipe)
+		return NULL;
+
+	idle_pipe->stream = head_pipe->stream;
+	idle_pipe->stream_res.tg = head_pipe->stream_res.tg;
+	idle_pipe->stream_res.opp = head_pipe->stream_res.opp;
+
+	idle_pipe->plane_res.hubp = pool->hubps[idle_pipe->pipe_idx];
+	idle_pipe->plane_res.ipp = pool->ipps[idle_pipe->pipe_idx];
+	idle_pipe->plane_res.dpp = pool->dpps[idle_pipe->pipe_idx];
+	idle_pipe->plane_res.mpcc_inst = pool->dpps[idle_pipe->pipe_idx]->inst;
+
+	return idle_pipe;
+}
+
+static bool dcn201_get_dcc_compression_cap(const struct dc *dc,
+		const struct dc_dcc_surface_param *input,
+		struct dc_surface_dcc_cap *output)
+{
+	return dc->res_pool->hubbub->funcs->get_dcc_compression_cap(
+			dc->res_pool->hubbub,
+			input,
+			output);
+}
+
+static void dcn201_populate_dml_writeback_from_context(struct dc *dc,
+						       struct resource_context *res_ctx,
+						       display_e2e_pipe_params_st *pipes)
+{
+	DC_FP_START();
+	dcn201_populate_dml_writeback_from_context_fpu(dc, res_ctx, pipes);
+	DC_FP_END();
+}
+
+static void dcn201_destroy_resource_pool(struct resource_pool **pool)
+{
+	struct dcn201_resource_pool *dcn201_pool = TO_DCN201_RES_POOL(*pool);
+
+	dcn201_resource_destruct(dcn201_pool);
+	kfree(dcn201_pool);
+	*pool = NULL;
+}
+
+static void dcn201_link_init(struct dc_link *link)
+{
+	if (link->ctx->dc_bios->integrated_info)
+		link->dp_ss_off = !link->ctx->dc_bios->integrated_info->dp_ss_control;
+}
+
+static struct dc_cap_funcs cap_funcs = {
+	.get_dcc_compression_cap = dcn201_get_dcc_compression_cap,
+};
+
+static struct resource_funcs dcn201_res_pool_funcs = {
+	.link_init = dcn201_link_init,
+	.destroy = dcn201_destroy_resource_pool,
+	.link_enc_create = dcn201_link_encoder_create,
+	.panel_cntl_create = NULL,
+	.validate_bandwidth = dcn20_validate_bandwidth,
+	.populate_dml_pipes = dcn20_populate_dml_pipes_from_context,
+	.add_stream_to_ctx = dcn20_add_stream_to_ctx,
+	.add_dsc_to_stream_resource = NULL,
+	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
+	.acquire_free_pipe_as_secondary_dpp_pipe = dcn201_acquire_free_pipe_for_layer,
+	.release_pipe = dcn20_release_pipe,
+	.populate_dml_writeback_from_context = dcn201_populate_dml_writeback_from_context,
+	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
+	.set_mcif_arb_params = dcn20_set_mcif_arb_params,
+	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link
+};
+
+static bool dcn201_resource_construct(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct dcn201_resource_pool *pool)
+{
+	int i;
+	struct dc_context *ctx = dc->ctx;
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	pool->base.res_cap = &res_cap_dnc201;
+	pool->base.funcs = &dcn201_res_pool_funcs;
+
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+
+	pool->base.pipe_count = 4;
+	pool->base.mpcc_count = 5;
+	dc->caps.max_downscale_ratio = 200;
+	dc->caps.i2c_speed_in_khz = 100;
+	dc->caps.i2c_speed_in_khz_hdcp = 5; /*1.5 w/a applied by default*/
+	dc->caps.max_cursor_size = 256;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.dmdata_alloc_size = 2048;
+
+	dc->caps.max_slave_planes = 1;
+	dc->caps.max_slave_yuv_planes = 1;
+	dc->caps.max_slave_rgb_planes = 1;
+	dc->caps.post_blend_color_processing = true;
+	dc->caps.force_dp_tps4_for_cp2520 = true;
+	dc->caps.extended_aux_timeout_support = true;
+
+	/* Color pipeline capabilities */
+	dc->caps.color.dpp.dcn_arch = 1;
+	dc->caps.color.dpp.input_lut_shared = 0;
+	dc->caps.color.dpp.icsc = 1;
+	dc->caps.color.dpp.dgam_ram = 1;
+	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
+	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.dpp.dgam_rom_caps.pq = 0;
+	dc->caps.color.dpp.dgam_rom_caps.hlg = 0;
+	dc->caps.color.dpp.post_csc = 0;
+	dc->caps.color.dpp.gamma_corr = 0;
+	dc->caps.color.dpp.dgam_rom_for_yuv = 1;
+
+	dc->caps.color.dpp.hw_3d_lut = 1;
+	dc->caps.color.dpp.ogam_ram = 1;
+	// no OGAM ROM on DCN2
+	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
+	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
+	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
+	dc->caps.color.dpp.ocsc = 0;
+
+	dc->caps.color.mpc.gamut_remap = 0;
+	dc->caps.color.mpc.num_3dluts = 0;
+	dc->caps.color.mpc.shared_3d_lut = 0;
+	dc->caps.color.mpc.ogam_ram = 1;
+	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
+	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
+	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
+	dc->caps.color.mpc.ocsc = 1;
+
+	dc->debug = debug_defaults_drv;
+
+	/*a0 only, remove later*/
+	dc->work_arounds.no_connect_phy_config  = true;
+	dc->work_arounds.dedcn20_305_wa = true;
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	pool->base.clock_sources[DCN20_CLK_SRC_PLL0] =
+			dcn201_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL0,
+				&clk_src_regs[0], false);
+	pool->base.clock_sources[DCN20_CLK_SRC_PLL1] =
+			dcn201_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL1,
+				&clk_src_regs[1], false);
+
+	pool->base.clk_src_count = DCN20_CLK_SRC_TOTAL_DCN201;
+
+	/* todo: not reuse phy_pll registers */
+	pool->base.dp_clock_source =
+			dcn201_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_ID_DP_DTO,
+				&clk_src_regs[0], true);
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			goto create_fail;
+		}
+	}
+
+	pool->base.dccg = dccg201_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
+	if (pool->base.dccg == NULL) {
+		dm_error("DC: failed to create dccg!\n");
+		goto create_fail;
+	}
+
+	dcn201_ip.max_num_otg = pool->base.res_cap->num_timing_generator;
+	dcn201_ip.max_num_dpp = pool->base.pipe_count;
+	dml_init_instance(&dc->dml, &dcn201_soc, &dcn201_ip, DML_PROJECT_DCN201);
+	{
+		struct irq_service_init_data init_data;
+		init_data.ctx = dc->ctx;
+		pool->base.irqs = dal_irq_service_dcn201_create(&init_data);
+		if (!pool->base.irqs)
+			goto create_fail;
+	}
+
+	/* mem input -> ipp -> dpp -> opp -> TG */
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		pool->base.hubps[i] = dcn201_hubp_create(ctx, i);
+		if (pool->base.hubps[i] == NULL) {
+			dm_error(
+				"DC: failed to create memory input!\n");
+			goto create_fail;
+		}
+
+		pool->base.ipps[i] = dcn201_ipp_create(ctx, i);
+		if (pool->base.ipps[i] == NULL) {
+			dm_error(
+				"DC: failed to create input pixel processor!\n");
+			goto create_fail;
+		}
+
+		pool->base.dpps[i] = dcn201_dpp_create(ctx, i);
+		if (pool->base.dpps[i] == NULL) {
+			dm_error(
+				"DC: failed to create dpps!\n");
+			goto create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		pool->base.opps[i] = dcn201_opp_create(ctx, i);
+		if (pool->base.opps[i] == NULL) {
+			dm_error(
+				"DC: failed to create output pixel processor!\n");
+			goto create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.engines[i] = dcn201_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto create_fail;
+		}
+		pool->base.hw_i2cs[i] = dcn201_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			dm_error(
+				"DC:failed to create hw i2c!!\n");
+			goto create_fail;
+		}
+		pool->base.sw_i2cs[i] = NULL;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		pool->base.timing_generators[i] = dcn201_timing_generator_create(
+				ctx, i);
+		if (pool->base.timing_generators[i] == NULL) {
+			dm_error("DC: failed to create tg!\n");
+			goto create_fail;
+		}
+	}
+
+	pool->base.timing_generator_count = i;
+
+	pool->base.mpc = dcn201_mpc_create(ctx, pool->base.mpcc_count);
+	if (pool->base.mpc == NULL) {
+		dm_error("DC: failed to create mpc!\n");
+		goto create_fail;
+	}
+
+	pool->base.hubbub = dcn201_hubbub_create(ctx);
+	if (pool->base.hubbub == NULL) {
+		dm_error("DC: failed to create hubbub!\n");
+		goto create_fail;
+	}
+
+	if (!resource_construct(num_virtual_links, dc, &pool->base,
+			&res_create_funcs))
+		goto create_fail;
+
+	dcn201_hw_sequencer_construct(dc);
+
+	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->cap_funcs = cap_funcs;
+
+	return true;
+
+create_fail:
+
+	dcn201_resource_destruct(pool);
+
+	return false;
+}
+
+struct resource_pool *dcn201_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc)
+{
+	struct dcn201_resource_pool *pool =
+		kzalloc(sizeof(struct dcn201_resource_pool), GFP_ATOMIC);
+
+	if (!pool)
+		return NULL;
+
+	if (dcn201_resource_construct(init_data->num_virtual_links, dc, pool))
+		return &pool->base;
+
+	kfree(pool);
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn201/dcn201_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn201/dcn201_resource.h
new file mode 100644
index 000000000000..e0467d17d4ae
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn201/dcn201_resource.h
@@ -0,0 +1,50 @@
+/*
+* Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_RESOURCE_DCN201_H__
+#define __DC_RESOURCE_DCN201_H__
+
+#include "core_types.h"
+
+#define RRDPCS_PHY_DP_TX_PSTATE_POWER_UP    0x00000000
+#define RRDPCS_PHY_DP_TX_PSTATE_HOLD        0x00000001
+#define RRDPCS_PHY_DP_TX_PSTATE_HOLD_OFF    0x00000002
+#define RRDPCS_PHY_DP_TX_PSTATE_POWER_DOWN  0x00000003
+
+#define TO_DCN201_RES_POOL(pool)\
+	container_of(pool, struct dcn201_resource_pool, base)
+
+struct dc;
+struct resource_pool;
+struct _vcs_dpi_display_pipe_params_st;
+
+struct dcn201_resource_pool {
+	struct resource_pool base;
+};
+struct resource_pool *dcn201_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc);
+
+#endif /* __DC_RESOURCE_DCN201_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.c
new file mode 100644
index 000000000000..c07da45e1e2c
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.c
@@ -0,0 +1,1745 @@
+/*
+* Copyright 2018 Advanced Micro Devices, Inc.
+ * Copyright 2019 Raptor Engineering, LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include <linux/slab.h>
+
+#include "dm_services.h"
+#include "dc.h"
+
+#include "dcn21/dcn21_init.h"
+
+#include "resource.h"
+#include "include/irq_service_interface.h"
+#include "dcn20/dcn20_resource.h"
+#include "dcn21/dcn21_resource.h"
+
+#include "dml/dcn20/dcn20_fpu.h"
+
+#include "clk_mgr.h"
+#include "dcn10/dcn10_hubp.h"
+#include "dcn10/dcn10_ipp.h"
+#include "dcn20/dcn20_hubbub.h"
+#include "dcn20/dcn20_mpc.h"
+#include "dcn20/dcn20_hubp.h"
+#include "dcn21/dcn21_hubp.h"
+#include "irq/dcn21/irq_service_dcn21.h"
+#include "dcn20/dcn20_dpp.h"
+#include "dcn20/dcn20_optc.h"
+#include "dcn21/dcn21_hwseq.h"
+#include "dce110/dce110_hwseq.h"
+#include "dcn20/dcn20_opp.h"
+#include "dcn20/dcn20_dsc.h"
+#include "dcn21/dcn21_link_encoder.h"
+#include "dcn20/dcn20_stream_encoder.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_hwseq.h"
+#include "virtual/virtual_stream_encoder.h"
+#include "dml/display_mode_vba.h"
+#include "dcn20/dcn20_dccg.h"
+#include "dcn21/dcn21_dccg.h"
+#include "dcn21/dcn21_hubbub.h"
+#include "dcn10/dcn10_resource.h"
+#include "dce/dce_panel_cntl.h"
+
+#include "dcn20/dcn20_dwb.h"
+#include "dcn20/dcn20_mmhubbub.h"
+#include "dpcs/dpcs_2_1_0_offset.h"
+#include "dpcs/dpcs_2_1_0_sh_mask.h"
+
+#include "renoir_ip_offset.h"
+#include "dcn/dcn_2_1_0_offset.h"
+#include "dcn/dcn_2_1_0_sh_mask.h"
+
+#include "nbio/nbio_7_0_offset.h"
+
+#include "mmhub/mmhub_2_0_0_offset.h"
+#include "mmhub/mmhub_2_0_0_sh_mask.h"
+
+#include "reg_helper.h"
+#include "dce/dce_abm.h"
+#include "dce/dce_dmcu.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_i2c.h"
+#include "dcn21_resource.h"
+#include "vm_helper.h"
+#include "dcn20/dcn20_vmid.h"
+#include "dce/dmub_psr.h"
+#include "dce/dmub_abm.h"
+
+/* begin *********************
+ * macros to expend register list macro defined in HW object header file */
+
+/* DCN */
+#define BASE_INNER(seg) DMU_BASE__INST0_SEG ## seg
+
+#define BASE(seg) BASE_INNER(seg)
+
+#define SR(reg_name)\
+		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) +  \
+					mm ## reg_name
+
+#define SRI(reg_name, block, id)\
+	.reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define SRIR(var_name, reg_name, block, id)\
+	.var_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define SRII(reg_name, block, id)\
+	.reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define DCCG_SRII(reg_name, block, id)\
+	.block ## _ ## reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define VUPDATE_SRII(reg_name, block, id)\
+	.reg_name[id] = BASE(mm ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
+					mm ## reg_name ## _ ## block ## id
+
+/* NBIO */
+#define NBIO_BASE_INNER(seg) \
+	NBIF0_BASE__INST0_SEG ## seg
+
+#define NBIO_BASE(seg) \
+	NBIO_BASE_INNER(seg)
+
+#define NBIO_SR(reg_name)\
+		.reg_name = NBIO_BASE(mm ## reg_name ## _BASE_IDX) + \
+					mm ## reg_name
+
+/* MMHUB */
+#define MMHUB_BASE_INNER(seg) \
+	MMHUB_BASE__INST0_SEG ## seg
+
+#define MMHUB_BASE(seg) \
+	MMHUB_BASE_INNER(seg)
+
+#define MMHUB_SR(reg_name)\
+		.reg_name = MMHUB_BASE(mmMM ## reg_name ## _BASE_IDX) + \
+					mmMM ## reg_name
+
+#define clk_src_regs(index, pllid)\
+[index] = {\
+	CS_COMMON_REG_LIST_DCN2_1(index, pllid),\
+}
+
+static const struct dce110_clk_src_regs clk_src_regs[] = {
+	clk_src_regs(0, A),
+	clk_src_regs(1, B),
+	clk_src_regs(2, C),
+	clk_src_regs(3, D),
+	clk_src_regs(4, E),
+};
+
+static const struct dce110_clk_src_shift cs_shift = {
+		CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
+};
+
+static const struct dce110_clk_src_mask cs_mask = {
+		CS_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
+};
+
+static const struct bios_registers bios_regs = {
+		NBIO_SR(BIOS_SCRATCH_3),
+		NBIO_SR(BIOS_SCRATCH_6)
+};
+
+static const struct dce_dmcu_registers dmcu_regs = {
+		DMCU_DCN20_REG_LIST()
+};
+
+static const struct dce_dmcu_shift dmcu_shift = {
+		DMCU_MASK_SH_LIST_DCN10(__SHIFT)
+};
+
+static const struct dce_dmcu_mask dmcu_mask = {
+		DMCU_MASK_SH_LIST_DCN10(_MASK)
+};
+
+static const struct dce_abm_registers abm_regs = {
+		ABM_DCN20_REG_LIST()
+};
+
+static const struct dce_abm_shift abm_shift = {
+		ABM_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dce_abm_mask abm_mask = {
+		ABM_MASK_SH_LIST_DCN20(_MASK)
+};
+
+#define audio_regs(id)\
+[id] = {\
+		AUD_COMMON_REG_LIST(id)\
+}
+
+static const struct dce_audio_registers audio_regs[] = {
+	audio_regs(0),
+	audio_regs(1),
+	audio_regs(2),
+	audio_regs(3),
+	audio_regs(4),
+	audio_regs(5),
+};
+
+#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
+		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
+
+static const struct dce_audio_shift audio_shift = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_audio_mask audio_mask = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+static const struct dccg_registers dccg_regs = {
+		DCCG_COMMON_REG_LIST_DCN_BASE()
+};
+
+static const struct dccg_shift dccg_shift = {
+		DCCG_MASK_SH_LIST_DCN2_1(__SHIFT)
+};
+
+static const struct dccg_mask dccg_mask = {
+		DCCG_MASK_SH_LIST_DCN2_1(_MASK)
+};
+
+#define opp_regs(id)\
+[id] = {\
+	OPP_REG_LIST_DCN20(id),\
+}
+
+static const struct dcn20_opp_registers opp_regs[] = {
+	opp_regs(0),
+	opp_regs(1),
+	opp_regs(2),
+	opp_regs(3),
+	opp_regs(4),
+	opp_regs(5),
+};
+
+static const struct dcn20_opp_shift opp_shift = {
+		OPP_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn20_opp_mask opp_mask = {
+		OPP_MASK_SH_LIST_DCN20(_MASK)
+};
+
+#define tg_regs(id)\
+[id] = {TG_COMMON_REG_LIST_DCN2_0(id)}
+
+static const struct dcn_optc_registers tg_regs[] = {
+	tg_regs(0),
+	tg_regs(1),
+	tg_regs(2),
+	tg_regs(3)
+};
+
+static const struct dcn_optc_shift tg_shift = {
+	TG_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
+};
+
+static const struct dcn_optc_mask tg_mask = {
+	TG_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
+};
+
+static const struct dcn20_mpc_registers mpc_regs = {
+		MPC_REG_LIST_DCN2_0(0),
+		MPC_REG_LIST_DCN2_0(1),
+		MPC_REG_LIST_DCN2_0(2),
+		MPC_REG_LIST_DCN2_0(3),
+		MPC_REG_LIST_DCN2_0(4),
+		MPC_REG_LIST_DCN2_0(5),
+		MPC_OUT_MUX_REG_LIST_DCN2_0(0),
+		MPC_OUT_MUX_REG_LIST_DCN2_0(1),
+		MPC_OUT_MUX_REG_LIST_DCN2_0(2),
+		MPC_OUT_MUX_REG_LIST_DCN2_0(3),
+		MPC_DBG_REG_LIST_DCN2_0()
+};
+
+static const struct dcn20_mpc_shift mpc_shift = {
+	MPC_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT),
+	MPC_DEBUG_REG_LIST_SH_DCN20
+};
+
+static const struct dcn20_mpc_mask mpc_mask = {
+	MPC_COMMON_MASK_SH_LIST_DCN2_0(_MASK),
+	MPC_DEBUG_REG_LIST_MASK_DCN20
+};
+
+#define hubp_regs(id)\
+[id] = {\
+	HUBP_REG_LIST_DCN21(id)\
+}
+
+static const struct dcn_hubp2_registers hubp_regs[] = {
+		hubp_regs(0),
+		hubp_regs(1),
+		hubp_regs(2),
+		hubp_regs(3)
+};
+
+static const struct dcn_hubp2_shift hubp_shift = {
+		HUBP_MASK_SH_LIST_DCN21(__SHIFT)
+};
+
+static const struct dcn_hubp2_mask hubp_mask = {
+		HUBP_MASK_SH_LIST_DCN21(_MASK)
+};
+
+static const struct dcn_hubbub_registers hubbub_reg = {
+		HUBBUB_REG_LIST_DCN21()
+};
+
+static const struct dcn_hubbub_shift hubbub_shift = {
+		HUBBUB_MASK_SH_LIST_DCN21(__SHIFT)
+};
+
+static const struct dcn_hubbub_mask hubbub_mask = {
+		HUBBUB_MASK_SH_LIST_DCN21(_MASK)
+};
+
+
+#define vmid_regs(id)\
+[id] = {\
+		DCN20_VMID_REG_LIST(id)\
+}
+
+static const struct dcn_vmid_registers vmid_regs[] = {
+	vmid_regs(0),
+	vmid_regs(1),
+	vmid_regs(2),
+	vmid_regs(3),
+	vmid_regs(4),
+	vmid_regs(5),
+	vmid_regs(6),
+	vmid_regs(7),
+	vmid_regs(8),
+	vmid_regs(9),
+	vmid_regs(10),
+	vmid_regs(11),
+	vmid_regs(12),
+	vmid_regs(13),
+	vmid_regs(14),
+	vmid_regs(15)
+};
+
+static const struct dcn20_vmid_shift vmid_shifts = {
+		DCN20_VMID_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn20_vmid_mask vmid_masks = {
+		DCN20_VMID_MASK_SH_LIST(_MASK)
+};
+
+#define dsc_regsDCN20(id)\
+[id] = {\
+	DSC_REG_LIST_DCN20(id)\
+}
+
+static const struct dcn20_dsc_registers dsc_regs[] = {
+	dsc_regsDCN20(0),
+	dsc_regsDCN20(1),
+	dsc_regsDCN20(2),
+	dsc_regsDCN20(3),
+	dsc_regsDCN20(4),
+	dsc_regsDCN20(5)
+};
+
+static const struct dcn20_dsc_shift dsc_shift = {
+	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
+};
+
+static const struct dcn20_dsc_mask dsc_mask = {
+	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
+};
+
+#define ipp_regs(id)\
+[id] = {\
+	IPP_REG_LIST_DCN20(id),\
+}
+
+static const struct dcn10_ipp_registers ipp_regs[] = {
+	ipp_regs(0),
+	ipp_regs(1),
+	ipp_regs(2),
+	ipp_regs(3),
+};
+
+static const struct dcn10_ipp_shift ipp_shift = {
+		IPP_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn10_ipp_mask ipp_mask = {
+		IPP_MASK_SH_LIST_DCN20(_MASK),
+};
+
+#define opp_regs(id)\
+[id] = {\
+	OPP_REG_LIST_DCN20(id),\
+}
+
+
+#define aux_engine_regs(id)\
+[id] = {\
+	AUX_COMMON_REG_LIST0(id), \
+	.AUXN_IMPCAL = 0, \
+	.AUXP_IMPCAL = 0, \
+	.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1),
+		aux_engine_regs(2),
+		aux_engine_regs(3),
+		aux_engine_regs(4),
+};
+
+#define tf_regs(id)\
+[id] = {\
+	TF_REG_LIST_DCN20(id),\
+	TF_REG_LIST_DCN20_COMMON_APPEND(id),\
+}
+
+static const struct dcn2_dpp_registers tf_regs[] = {
+	tf_regs(0),
+	tf_regs(1),
+	tf_regs(2),
+	tf_regs(3),
+};
+
+static const struct dcn2_dpp_shift tf_shift = {
+		TF_REG_LIST_SH_MASK_DCN20(__SHIFT),
+		TF_DEBUG_REG_LIST_SH_DCN20
+};
+
+static const struct dcn2_dpp_mask tf_mask = {
+		TF_REG_LIST_SH_MASK_DCN20(_MASK),
+		TF_DEBUG_REG_LIST_MASK_DCN20
+};
+
+#define stream_enc_regs(id)\
+[id] = {\
+	SE_DCN2_REG_LIST(id)\
+}
+
+static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
+	stream_enc_regs(0),
+	stream_enc_regs(1),
+	stream_enc_regs(2),
+	stream_enc_regs(3),
+	stream_enc_regs(4),
+};
+
+static const struct dce110_aux_registers_shift aux_shift = {
+	DCN_AUX_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+	DCN_AUX_MASK_SH_LIST(_MASK)
+};
+
+static const struct dcn10_stream_encoder_shift se_shift = {
+		SE_COMMON_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn10_stream_encoder_mask se_mask = {
+		SE_COMMON_MASK_SH_LIST_DCN20(_MASK)
+};
+
+static void dcn21_pp_smu_destroy(struct pp_smu_funcs **pp_smu);
+
+static struct input_pixel_processor *dcn21_ipp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn10_ipp *ipp =
+		kzalloc(sizeof(struct dcn10_ipp), GFP_KERNEL);
+
+	if (!ipp) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dcn20_ipp_construct(ipp, ctx, inst,
+			&ipp_regs[inst], &ipp_shift, &ipp_mask);
+	return &ipp->base;
+}
+
+static struct dpp *dcn21_dpp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn20_dpp *dpp =
+		kzalloc(sizeof(struct dcn20_dpp), GFP_KERNEL);
+
+	if (!dpp)
+		return NULL;
+
+	if (dpp2_construct(dpp, ctx, inst,
+			&tf_regs[inst], &tf_shift, &tf_mask))
+		return &dpp->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(dpp);
+	return NULL;
+}
+
+static struct dce_aux *dcn21_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst],
+					&aux_mask,
+					&aux_shift,
+					ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+
+#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
+
+static const struct dce_i2c_registers i2c_hw_regs[] = {
+		i2c_inst_regs(1),
+		i2c_inst_regs(2),
+		i2c_inst_regs(3),
+		i2c_inst_regs(4),
+		i2c_inst_regs(5),
+};
+
+static const struct dce_i2c_shift i2c_shifts = {
+		I2C_COMMON_MASK_SH_LIST_DCN2(__SHIFT)
+};
+
+static const struct dce_i2c_mask i2c_masks = {
+		I2C_COMMON_MASK_SH_LIST_DCN2(_MASK)
+};
+
+static struct dce_i2c_hw *dcn21_i2c_hw_create(struct dc_context *ctx,
+					      uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw =
+		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
+				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+
+static const struct resource_caps res_cap_rn = {
+		.num_timing_generator = 4,
+		.num_opp = 4,
+		.num_video_plane = 4,
+		.num_audio = 4, // 4 audio endpoints.  4 audio streams
+		.num_stream_encoder = 5,
+		.num_pll = 5,  // maybe 3 because the last two used for USB-c
+		.num_dwb = 1,
+		.num_ddc = 5,
+		.num_vmid = 16,
+		.num_dsc = 3,
+};
+
+#ifdef DIAGS_BUILD
+static const struct resource_caps res_cap_rn_FPGA_4pipe = {
+		.num_timing_generator = 4,
+		.num_opp = 4,
+		.num_video_plane = 4,
+		.num_audio = 7,
+		.num_stream_encoder = 4,
+		.num_pll = 4,
+		.num_dwb = 1,
+		.num_ddc = 4,
+		.num_dsc = 0,
+};
+
+static const struct resource_caps res_cap_rn_FPGA_2pipe_dsc = {
+		.num_timing_generator = 2,
+		.num_opp = 2,
+		.num_video_plane = 2,
+		.num_audio = 7,
+		.num_stream_encoder = 2,
+		.num_pll = 4,
+		.num_dwb = 1,
+		.num_ddc = 4,
+		.num_dsc = 2,
+};
+#endif
+
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
+	.per_pixel_alpha = true,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = true,
+			.fp16 = true,
+			.p010 = true
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 16000,
+			.fp16 = 16000
+	},
+
+	.max_downscale_factor = {
+			.argb8888 = 250,
+			.nv12 = 250,
+			.fp16 = 250
+	},
+	64,
+	64
+};
+
+static const struct dc_debug_options debug_defaults_drv = {
+		.disable_dmcu = false,
+		.force_abm_enable = false,
+		.timing_trace = false,
+		.clock_trace = true,
+		.disable_pplib_clock_request = true,
+		.min_disp_clk_khz = 100000,
+		.pipe_split_policy = MPC_SPLIT_DYNAMIC,
+		.force_single_disp_pipe_split = false,
+		.disable_dcc = DCC_ENABLE,
+		.vsr_support = true,
+		.performance_trace = false,
+		.max_downscale_src_width = 4096,
+		.disable_pplib_wm_range = false,
+		.scl_reset_length10 = true,
+		.sanity_checks = true,
+		.disable_48mhz_pwrdwn = false,
+		.usbc_combo_phy_reset_wa = true,
+		.dmub_command_table = true,
+		.use_max_lb = true,
+		.enable_legacy_fast_update = true,
+		.using_dml2 = false,
+};
+
+static const struct dc_panel_config panel_config_defaults = {
+		.psr = {
+			.disable_psr = false,
+			.disallow_psrsu = false,
+			.disallow_replay = false,
+		},
+		.ilr = {
+			.optimize_edp_link_rate = true,
+		},
+};
+
+enum dcn20_clk_src_array_id {
+	DCN20_CLK_SRC_PLL0,
+	DCN20_CLK_SRC_PLL1,
+	DCN20_CLK_SRC_PLL2,
+	DCN20_CLK_SRC_PLL3,
+	DCN20_CLK_SRC_PLL4,
+	DCN20_CLK_SRC_TOTAL_DCN21
+};
+
+static void dcn21_resource_destruct(struct dcn21_resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->base.stream_enc_count; i++) {
+		if (pool->base.stream_enc[i] != NULL) {
+			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
+			pool->base.stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		if (pool->base.dscs[i] != NULL)
+			dcn20_dsc_destroy(&pool->base.dscs[i]);
+	}
+
+	if (pool->base.mpc != NULL) {
+		kfree(TO_DCN20_MPC(pool->base.mpc));
+		pool->base.mpc = NULL;
+	}
+	if (pool->base.hubbub != NULL) {
+		kfree(pool->base.hubbub);
+		pool->base.hubbub = NULL;
+	}
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (pool->base.dpps[i] != NULL)
+			dcn20_dpp_destroy(&pool->base.dpps[i]);
+
+		if (pool->base.ipps[i] != NULL)
+			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
+
+		if (pool->base.hubps[i] != NULL) {
+			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
+			pool->base.hubps[i] = NULL;
+		}
+
+		if (pool->base.irqs != NULL) {
+			dal_irq_service_destroy(&pool->base.irqs);
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+		if (pool->base.hw_i2cs[i] != NULL) {
+			kfree(pool->base.hw_i2cs[i]);
+			pool->base.hw_i2cs[i] = NULL;
+		}
+		if (pool->base.sw_i2cs[i] != NULL) {
+			kfree(pool->base.sw_i2cs[i]);
+			pool->base.sw_i2cs[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		if (pool->base.opps[i] != NULL)
+			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.timing_generators[i] != NULL)	{
+			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
+			pool->base.timing_generators[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+		if (pool->base.dwbc[i] != NULL) {
+			kfree(TO_DCN20_DWBC(pool->base.dwbc[i]));
+			pool->base.dwbc[i] = NULL;
+		}
+		if (pool->base.mcif_wb[i] != NULL) {
+			kfree(TO_DCN20_MMHUBBUB(pool->base.mcif_wb[i]));
+			pool->base.mcif_wb[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.audio_count; i++) {
+		if (pool->base.audios[i])
+			dce_aud_destroy(&pool->base.audios[i]);
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] != NULL) {
+			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
+			pool->base.clock_sources[i] = NULL;
+		}
+	}
+
+	if (pool->base.dp_clock_source != NULL) {
+		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
+		pool->base.dp_clock_source = NULL;
+	}
+
+	if (pool->base.abm != NULL) {
+		if (pool->base.abm->ctx->dc->config.disable_dmcu)
+			dmub_abm_destroy(&pool->base.abm);
+		else
+			dce_abm_destroy(&pool->base.abm);
+	}
+
+	if (pool->base.dmcu != NULL)
+		dce_dmcu_destroy(&pool->base.dmcu);
+
+	if (pool->base.psr != NULL)
+		dmub_psr_destroy(&pool->base.psr);
+
+	if (pool->base.dccg != NULL)
+		dcn_dccg_destroy(&pool->base.dccg);
+
+	if (pool->base.pp_smu != NULL)
+		dcn21_pp_smu_destroy(&pool->base.pp_smu);
+}
+
+bool dcn21_fast_validate_bw(struct dc *dc,
+			    struct dc_state *context,
+			    display_e2e_pipe_params_st *pipes,
+			    int *pipe_cnt_out,
+			    int *pipe_split_from,
+			    int *vlevel_out,
+			    bool fast_validate)
+{
+	bool out = false;
+	int split[MAX_PIPES] = { 0 };
+	int pipe_cnt, i, pipe_idx, vlevel;
+
+	ASSERT(pipes);
+	if (!pipes)
+		return false;
+
+	dcn20_merge_pipes_for_validate(dc, context);
+
+	DC_FP_START();
+	pipe_cnt = dc->res_pool->funcs->populate_dml_pipes(dc, context, pipes, fast_validate);
+	DC_FP_END();
+
+	*pipe_cnt_out = pipe_cnt;
+
+	if (!pipe_cnt) {
+		out = true;
+		goto validate_out;
+	}
+	/*
+	 * DML favors voltage over p-state, but we're more interested in
+	 * supporting p-state over voltage. We can't support p-state in
+	 * prefetch mode > 0 so try capping the prefetch mode to start.
+	 */
+	context->bw_ctx.dml.soc.allow_dram_self_refresh_or_dram_clock_change_in_vblank =
+				dm_allow_self_refresh_and_mclk_switch;
+	vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
+
+	if (vlevel > context->bw_ctx.dml.soc.num_states) {
+		/*
+		 * If mode is unsupported or there's still no p-state support then
+		 * fall back to favoring voltage.
+		 *
+		 * We don't actually support prefetch mode 2, so require that we
+		 * at least support prefetch mode 1.
+		 */
+		context->bw_ctx.dml.soc.allow_dram_self_refresh_or_dram_clock_change_in_vblank =
+					dm_allow_self_refresh;
+		vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
+		if (vlevel > context->bw_ctx.dml.soc.num_states)
+			goto validate_fail;
+	}
+
+	vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, vlevel, split, NULL);
+
+	for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+		struct pipe_ctx *mpo_pipe = pipe->bottom_pipe;
+		struct vba_vars_st *vba = &context->bw_ctx.dml.vba;
+
+		if (!pipe->stream)
+			continue;
+
+		/* We only support full screen mpo with ODM */
+		if (vba->ODMCombineEnabled[vba->pipe_plane[pipe_idx]] != dm_odm_combine_mode_disabled
+				&& pipe->plane_state && mpo_pipe
+				&& memcmp(&mpo_pipe->plane_state->clip_rect,
+						&pipe->stream->src,
+						sizeof(struct rect)) != 0) {
+			ASSERT(mpo_pipe->plane_state != pipe->plane_state);
+			goto validate_fail;
+		}
+		pipe_idx++;
+	}
+
+	/*initialize pipe_just_split_from to invalid idx*/
+	for (i = 0; i < MAX_PIPES; i++)
+		pipe_split_from[i] = -1;
+
+	for (i = 0, pipe_idx = -1; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+		struct pipe_ctx *hsplit_pipe = pipe->bottom_pipe;
+
+		if (!pipe->stream || pipe_split_from[i] >= 0)
+			continue;
+
+		pipe_idx++;
+
+		if (!pipe->top_pipe && !pipe->plane_state && context->bw_ctx.dml.vba.ODMCombineEnabled[pipe_idx]) {
+			hsplit_pipe = dcn20_find_secondary_pipe(dc, &context->res_ctx, dc->res_pool, pipe);
+			ASSERT(hsplit_pipe);
+			if (!dcn20_split_stream_for_odm(
+					dc, &context->res_ctx,
+					pipe, hsplit_pipe))
+				goto validate_fail;
+			pipe_split_from[hsplit_pipe->pipe_idx] = pipe_idx;
+			dcn20_build_mapped_resource(dc, context, pipe->stream);
+		}
+
+		if (!pipe->plane_state)
+			continue;
+		/* Skip 2nd half of already split pipe */
+		if (pipe->top_pipe && pipe->plane_state == pipe->top_pipe->plane_state)
+			continue;
+
+		if (split[i] == 2) {
+			if (!hsplit_pipe || hsplit_pipe->plane_state != pipe->plane_state) {
+				/* pipe not split previously needs split */
+				hsplit_pipe = dcn20_find_secondary_pipe(dc, &context->res_ctx, dc->res_pool, pipe);
+				ASSERT(hsplit_pipe);
+				if (!hsplit_pipe) {
+					DC_FP_START();
+					dcn20_fpu_adjust_dppclk(&context->bw_ctx.dml.vba, vlevel, context->bw_ctx.dml.vba.maxMpcComb, pipe_idx, true);
+					DC_FP_END();
+					continue;
+				}
+				if (context->bw_ctx.dml.vba.ODMCombineEnabled[pipe_idx]) {
+					if (!dcn20_split_stream_for_odm(
+							dc, &context->res_ctx,
+							pipe, hsplit_pipe))
+						goto validate_fail;
+					dcn20_build_mapped_resource(dc, context, pipe->stream);
+				} else {
+					dcn20_split_stream_for_mpc(
+							&context->res_ctx, dc->res_pool,
+							pipe, hsplit_pipe);
+					resource_build_scaling_params(pipe);
+					resource_build_scaling_params(hsplit_pipe);
+				}
+				pipe_split_from[hsplit_pipe->pipe_idx] = pipe_idx;
+			}
+		} else if (hsplit_pipe && hsplit_pipe->plane_state == pipe->plane_state) {
+			/* merge should already have been done */
+			ASSERT(0);
+		}
+	}
+	/* Actual dsc count per stream dsc validation*/
+	if (!dcn20_validate_dsc(dc, context)) {
+		context->bw_ctx.dml.vba.ValidationStatus[context->bw_ctx.dml.vba.soc.num_states] =
+				DML_FAIL_DSC_VALIDATION_FAILURE;
+		goto validate_fail;
+	}
+
+	*vlevel_out = vlevel;
+
+	out = true;
+	goto validate_out;
+
+validate_fail:
+	out = false;
+
+validate_out:
+	return out;
+}
+
+/*
+ * Some of the functions further below use the FPU, so we need to wrap this
+ * with DC_FP_START()/DC_FP_END(). Use the same approach as for
+ * dcn20_validate_bandwidth in dcn20_resource.c.
+ */
+static bool dcn21_validate_bandwidth(struct dc *dc, struct dc_state *context,
+		bool fast_validate)
+{
+	bool voltage_supported;
+	display_e2e_pipe_params_st *pipes;
+
+	pipes = kcalloc(dc->res_pool->pipe_count, sizeof(display_e2e_pipe_params_st), GFP_KERNEL);
+	if (!pipes)
+		return false;
+
+	DC_FP_START();
+	voltage_supported = dcn21_validate_bandwidth_fp(dc, context, fast_validate, pipes);
+	DC_FP_END();
+
+	kfree(pipes);
+	return voltage_supported;
+}
+
+static void dcn21_destroy_resource_pool(struct resource_pool **pool)
+{
+	struct dcn21_resource_pool *dcn21_pool = TO_DCN21_RES_POOL(*pool);
+
+	dcn21_resource_destruct(dcn21_pool);
+	kfree(dcn21_pool);
+	*pool = NULL;
+}
+
+static struct clock_source *dcn21_clock_source_create(
+		struct dc_context *ctx,
+		struct dc_bios *bios,
+		enum clock_source_id id,
+		const struct dce110_clk_src_regs *regs,
+		bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dcn20_clk_src_construct(clk_src, ctx, bios, id,
+			regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	kfree(clk_src);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static struct hubp *dcn21_hubp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn21_hubp *hubp21 =
+		kzalloc(sizeof(struct dcn21_hubp), GFP_KERNEL);
+
+	if (!hubp21)
+		return NULL;
+
+	if (hubp21_construct(hubp21, ctx, inst,
+			&hubp_regs[inst], &hubp_shift, &hubp_mask))
+		return &hubp21->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(hubp21);
+	return NULL;
+}
+
+static struct hubbub *dcn21_hubbub_create(struct dc_context *ctx)
+{
+	int i;
+
+	struct dcn20_hubbub *hubbub = kzalloc(sizeof(struct dcn20_hubbub),
+					  GFP_KERNEL);
+
+	if (!hubbub)
+		return NULL;
+
+	hubbub21_construct(hubbub, ctx,
+			&hubbub_reg,
+			&hubbub_shift,
+			&hubbub_mask);
+
+	for (i = 0; i < res_cap_rn.num_vmid; i++) {
+		struct dcn20_vmid *vmid = &hubbub->vmid[i];
+
+		vmid->ctx = ctx;
+
+		vmid->regs = &vmid_regs[i];
+		vmid->shifts = &vmid_shifts;
+		vmid->masks = &vmid_masks;
+	}
+	hubbub->num_vmid = res_cap_rn.num_vmid;
+
+	return &hubbub->base;
+}
+
+static struct output_pixel_processor *dcn21_opp_create(struct dc_context *ctx,
+						       uint32_t inst)
+{
+	struct dcn20_opp *opp =
+		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
+
+	if (!opp) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dcn20_opp_construct(opp, ctx, inst,
+			&opp_regs[inst], &opp_shift, &opp_mask);
+	return &opp->base;
+}
+
+static struct timing_generator *dcn21_timing_generator_create(struct dc_context *ctx,
+							      uint32_t instance)
+{
+	struct optc *tgn10 =
+		kzalloc(sizeof(struct optc), GFP_KERNEL);
+
+	if (!tgn10)
+		return NULL;
+
+	tgn10->base.inst = instance;
+	tgn10->base.ctx = ctx;
+
+	tgn10->tg_regs = &tg_regs[instance];
+	tgn10->tg_shift = &tg_shift;
+	tgn10->tg_mask = &tg_mask;
+
+	dcn20_timing_generator_init(tgn10);
+
+	return &tgn10->base;
+}
+
+static struct mpc *dcn21_mpc_create(struct dc_context *ctx)
+{
+	struct dcn20_mpc *mpc20 = kzalloc(sizeof(struct dcn20_mpc),
+					  GFP_KERNEL);
+
+	if (!mpc20)
+		return NULL;
+
+	dcn20_mpc_construct(mpc20, ctx,
+			&mpc_regs,
+			&mpc_shift,
+			&mpc_mask,
+			6);
+
+	return &mpc20->base;
+}
+
+static void read_dce_straps(
+	struct dc_context *ctx,
+	struct resource_straps *straps)
+{
+	generic_reg_get(ctx, mmDC_PINSTRAPS + BASE(mmDC_PINSTRAPS_BASE_IDX),
+		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
+
+}
+
+
+static struct display_stream_compressor *dcn21_dsc_create(struct dc_context *ctx,
+							  uint32_t inst)
+{
+	struct dcn20_dsc *dsc =
+		kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
+
+	if (!dsc) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
+	return &dsc->base;
+}
+
+static struct pp_smu_funcs *dcn21_pp_smu_create(struct dc_context *ctx)
+{
+	struct pp_smu_funcs *pp_smu = kzalloc(sizeof(*pp_smu), GFP_KERNEL);
+
+	if (!pp_smu)
+		return pp_smu;
+
+	dm_pp_get_funcs(ctx, pp_smu);
+
+	if (pp_smu->ctx.ver != PP_SMU_VER_RN)
+		pp_smu = memset(pp_smu, 0, sizeof(struct pp_smu_funcs));
+
+
+	return pp_smu;
+}
+
+static void dcn21_pp_smu_destroy(struct pp_smu_funcs **pp_smu)
+{
+	if (pp_smu && *pp_smu) {
+		kfree(*pp_smu);
+		*pp_smu = NULL;
+	}
+}
+
+static struct audio *dcn21_create_audio(
+		struct dc_context *ctx, unsigned int inst)
+{
+	return dce_audio_create(ctx, inst,
+			&audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+static struct dc_cap_funcs cap_funcs = {
+	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
+};
+
+static struct stream_encoder *dcn21_stream_encoder_create(enum engine_id eng_id,
+							  struct dc_context *ctx)
+{
+	struct dcn10_stream_encoder *enc1 =
+		kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
+
+	if (!enc1)
+		return NULL;
+
+	dcn20_stream_encoder_construct(enc1, ctx, ctx->dc_bios, eng_id,
+					&stream_enc_regs[eng_id],
+					&se_shift, &se_mask);
+
+	return &enc1->base;
+}
+
+static const struct dce_hwseq_registers hwseq_reg = {
+		HWSEQ_DCN21_REG_LIST()
+};
+
+static const struct dce_hwseq_shift hwseq_shift = {
+		HWSEQ_DCN21_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+		HWSEQ_DCN21_MASK_SH_LIST(_MASK)
+};
+
+static struct dce_hwseq *dcn21_hwseq_create(
+	struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &hwseq_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+		hws->wa.DEGVIDCN21 = true;
+		hws->wa.disallow_self_refresh_during_multi_plane_transition = true;
+	}
+	return hws;
+}
+
+static const struct resource_create_funcs res_create_funcs = {
+	.read_dce_straps = read_dce_straps,
+	.create_audio = dcn21_create_audio,
+	.create_stream_encoder = dcn21_stream_encoder_create,
+	.create_hwseq = dcn21_hwseq_create,
+};
+
+static const struct encoder_feature_support link_enc_feature = {
+		.max_hdmi_deep_color = COLOR_DEPTH_121212,
+		.max_hdmi_pixel_clock = 600000,
+		.hdmi_ycbcr420_supported = true,
+		.dp_ycbcr420_supported = true,
+		.fec_supported = true,
+		.flags.bits.IS_HBR2_CAPABLE = true,
+		.flags.bits.IS_HBR3_CAPABLE = true,
+		.flags.bits.IS_TPS3_CAPABLE = true,
+		.flags.bits.IS_TPS4_CAPABLE = true
+};
+
+
+#define link_regs(id, phyid)\
+[id] = {\
+	LE_DCN2_REG_LIST(id), \
+	UNIPHY_DCN2_REG_LIST(phyid), \
+	DPCS_DCN21_REG_LIST(id), \
+	SRI(DP_DPHY_INTERNAL_CTRL, DP, id) \
+}
+
+static const struct dcn10_link_enc_registers link_enc_regs[] = {
+	link_regs(0, A),
+	link_regs(1, B),
+	link_regs(2, C),
+	link_regs(3, D),
+	link_regs(4, E),
+};
+
+static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
+	{ DCN_PANEL_CNTL_REG_LIST() }
+};
+
+static const struct dce_panel_cntl_shift panel_cntl_shift = {
+	DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_panel_cntl_mask panel_cntl_mask = {
+	DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
+};
+
+#define aux_regs(id)\
+[id] = {\
+	DCN2_AUX_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
+		aux_regs(0),
+		aux_regs(1),
+		aux_regs(2),
+		aux_regs(3),
+		aux_regs(4)
+};
+
+#define hpd_regs(id)\
+[id] = {\
+	HPD_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
+		hpd_regs(0),
+		hpd_regs(1),
+		hpd_regs(2),
+		hpd_regs(3),
+		hpd_regs(4)
+};
+
+static const struct dcn10_link_enc_shift le_shift = {
+	LINK_ENCODER_MASK_SH_LIST_DCN20(__SHIFT),\
+	DPCS_DCN21_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn10_link_enc_mask le_mask = {
+	LINK_ENCODER_MASK_SH_LIST_DCN20(_MASK),\
+	DPCS_DCN21_MASK_SH_LIST(_MASK)
+};
+
+static int map_transmitter_id_to_phy_instance(
+	enum transmitter transmitter)
+{
+	switch (transmitter) {
+	case TRANSMITTER_UNIPHY_A:
+		return 0;
+	break;
+	case TRANSMITTER_UNIPHY_B:
+		return 1;
+	break;
+	case TRANSMITTER_UNIPHY_C:
+		return 2;
+	break;
+	case TRANSMITTER_UNIPHY_D:
+		return 3;
+	break;
+	case TRANSMITTER_UNIPHY_E:
+		return 4;
+	break;
+	default:
+		ASSERT(0);
+		return 0;
+	}
+}
+
+static struct link_encoder *dcn21_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dcn21_link_encoder *enc21 =
+		kzalloc(sizeof(struct dcn21_link_encoder), GFP_KERNEL);
+	int link_regs_id;
+
+	if (!enc21)
+		return NULL;
+
+	link_regs_id =
+		map_transmitter_id_to_phy_instance(enc_init_data->transmitter);
+
+	dcn21_link_encoder_construct(enc21,
+				      enc_init_data,
+				      &link_enc_feature,
+				      &link_enc_regs[link_regs_id],
+				      &link_enc_aux_regs[enc_init_data->channel - 1],
+				      &link_enc_hpd_regs[enc_init_data->hpd_source],
+				      &le_shift,
+				      &le_mask);
+
+	return &enc21->enc10.base;
+}
+
+static struct panel_cntl *dcn21_panel_cntl_create(const struct panel_cntl_init_data *init_data)
+{
+	struct dce_panel_cntl *panel_cntl =
+		kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
+
+	if (!panel_cntl)
+		return NULL;
+
+	dce_panel_cntl_construct(panel_cntl,
+			init_data,
+			&panel_cntl_regs[init_data->inst],
+			&panel_cntl_shift,
+			&panel_cntl_mask);
+
+	return &panel_cntl->base;
+}
+
+static void dcn21_get_panel_config_defaults(struct dc_panel_config *panel_config)
+{
+	*panel_config = panel_config_defaults;
+}
+
+#define CTX ctx
+
+#define REG(reg_name) \
+	(DCN_BASE.instance[0].segment[mm ## reg_name ## _BASE_IDX] + mm ## reg_name)
+
+static uint32_t read_pipe_fuses(struct dc_context *ctx)
+{
+	uint32_t value = REG_READ(CC_DC_PIPE_DIS);
+	/* RV1 support max 4 pipes */
+	value = value & 0xf;
+	return value;
+}
+
+static enum dc_status dcn21_patch_unknown_plane_state(struct dc_plane_state *plane_state)
+{
+	if (plane_state->ctx->dc->debug.disable_dcc == DCC_ENABLE) {
+		plane_state->dcc.enable = 1;
+		/* align to our worst case block width */
+		plane_state->dcc.meta_pitch = ((plane_state->src_rect.width + 1023) / 1024) * 1024;
+	}
+
+	return dcn20_patch_unknown_plane_state(plane_state);
+}
+
+static const struct resource_funcs dcn21_res_pool_funcs = {
+	.destroy = dcn21_destroy_resource_pool,
+	.link_enc_create = dcn21_link_encoder_create,
+	.panel_cntl_create = dcn21_panel_cntl_create,
+	.validate_bandwidth = dcn21_validate_bandwidth,
+	.populate_dml_pipes = dcn21_populate_dml_pipes_from_context,
+	.add_stream_to_ctx = dcn20_add_stream_to_ctx,
+	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
+	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
+	.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
+	.release_pipe = dcn20_release_pipe,
+	.populate_dml_writeback_from_context = dcn20_populate_dml_writeback_from_context,
+	.patch_unknown_plane_state = dcn21_patch_unknown_plane_state,
+	.set_mcif_arb_params = dcn20_set_mcif_arb_params,
+	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
+	.update_bw_bounding_box = dcn21_update_bw_bounding_box,
+	.get_panel_config_defaults = dcn21_get_panel_config_defaults,
+};
+
+static bool dcn21_resource_construct(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct dcn21_resource_pool *pool)
+{
+	int i, j;
+	struct dc_context *ctx = dc->ctx;
+	struct irq_service_init_data init_data;
+	uint32_t pipe_fuses = read_pipe_fuses(ctx);
+	uint32_t num_pipes;
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	pool->base.res_cap = &res_cap_rn;
+#ifdef DIAGS_BUILD
+	if (IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment))
+		//pool->base.res_cap = &res_cap_nv10_FPGA_2pipe_dsc;
+		pool->base.res_cap = &res_cap_rn_FPGA_4pipe;
+#endif
+
+	pool->base.funcs = &dcn21_res_pool_funcs;
+
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+
+	/* max pipe num for ASIC before check pipe fuses */
+	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
+
+	dc->caps.max_downscale_ratio = 200;
+	dc->caps.i2c_speed_in_khz = 100;
+	dc->caps.i2c_speed_in_khz_hdcp = 5; /*1.4 w/a applied by default*/
+	dc->caps.max_cursor_size = 256;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.dmdata_alloc_size = 2048;
+
+	dc->caps.max_slave_planes = 1;
+	dc->caps.max_slave_yuv_planes = 1;
+	dc->caps.max_slave_rgb_planes = 1;
+	dc->caps.post_blend_color_processing = true;
+	dc->caps.force_dp_tps4_for_cp2520 = true;
+	dc->caps.extended_aux_timeout_support = true;
+	dc->caps.dmcub_support = true;
+	dc->caps.is_apu = true;
+
+	/* Color pipeline capabilities */
+	dc->caps.color.dpp.dcn_arch = 1;
+	dc->caps.color.dpp.input_lut_shared = 0;
+	dc->caps.color.dpp.icsc = 1;
+	dc->caps.color.dpp.dgam_ram = 1;
+	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
+	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.dpp.dgam_rom_caps.pq = 0;
+	dc->caps.color.dpp.dgam_rom_caps.hlg = 0;
+	dc->caps.color.dpp.post_csc = 0;
+	dc->caps.color.dpp.gamma_corr = 0;
+	dc->caps.color.dpp.dgam_rom_for_yuv = 1;
+
+	dc->caps.color.dpp.hw_3d_lut = 1;
+	dc->caps.color.dpp.ogam_ram = 1;
+	// no OGAM ROM on DCN2
+	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
+	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
+	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
+	dc->caps.color.dpp.ocsc = 0;
+
+	dc->caps.color.mpc.gamut_remap = 0;
+	dc->caps.color.mpc.num_3dluts = 0;
+	dc->caps.color.mpc.shared_3d_lut = 0;
+	dc->caps.color.mpc.ogam_ram = 1;
+	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
+	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
+	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
+	dc->caps.color.mpc.ocsc = 1;
+
+	dc->caps.dp_hdmi21_pcon_support = true;
+
+	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
+		dc->debug = debug_defaults_drv;
+
+	// Init the vm_helper
+	if (dc->vm_helper)
+		vm_helper_init(dc->vm_helper, 16);
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	pool->base.clock_sources[DCN20_CLK_SRC_PLL0] =
+			dcn21_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL0,
+				&clk_src_regs[0], false);
+	pool->base.clock_sources[DCN20_CLK_SRC_PLL1] =
+			dcn21_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL1,
+				&clk_src_regs[1], false);
+	pool->base.clock_sources[DCN20_CLK_SRC_PLL2] =
+			dcn21_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL2,
+				&clk_src_regs[2], false);
+	pool->base.clock_sources[DCN20_CLK_SRC_PLL3] =
+			dcn21_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL3,
+				&clk_src_regs[3], false);
+	pool->base.clock_sources[DCN20_CLK_SRC_PLL4] =
+			dcn21_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL4,
+				&clk_src_regs[4], false);
+
+	pool->base.clk_src_count = DCN20_CLK_SRC_TOTAL_DCN21;
+
+	/* todo: not reuse phy_pll registers */
+	pool->base.dp_clock_source =
+			dcn21_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_ID_DP_DTO,
+				&clk_src_regs[0], true);
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	pool->base.dccg = dccg21_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
+	if (pool->base.dccg == NULL) {
+		dm_error("DC: failed to create dccg!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	if (!dc->config.disable_dmcu) {
+		pool->base.dmcu = dcn21_dmcu_create(ctx,
+				&dmcu_regs,
+				&dmcu_shift,
+				&dmcu_mask);
+		if (pool->base.dmcu == NULL) {
+			dm_error("DC: failed to create dmcu!\n");
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+
+		dc->debug.dmub_command_table = false;
+	}
+
+	if (dc->config.disable_dmcu) {
+		pool->base.psr = dmub_psr_create(ctx);
+
+		if (pool->base.psr == NULL) {
+			dm_error("DC: failed to create psr obj!\n");
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	if (dc->config.disable_dmcu)
+		pool->base.abm = dmub_abm_create(ctx,
+			&abm_regs,
+			&abm_shift,
+			&abm_mask);
+	else
+		pool->base.abm = dce_abm_create(ctx,
+			&abm_regs,
+			&abm_shift,
+			&abm_mask);
+
+	pool->base.pp_smu = dcn21_pp_smu_create(ctx);
+
+	num_pipes = dcn2_1_ip.max_num_dpp;
+
+	for (i = 0; i < dcn2_1_ip.max_num_dpp; i++)
+		if (pipe_fuses & 1 << i)
+			num_pipes--;
+	dcn2_1_ip.max_num_dpp = num_pipes;
+	dcn2_1_ip.max_num_otg = num_pipes;
+
+	dml_init_instance(&dc->dml, &dcn2_1_soc, &dcn2_1_ip, DML_PROJECT_DCN21);
+
+	init_data.ctx = dc->ctx;
+	pool->base.irqs = dal_irq_service_dcn21_create(&init_data);
+	if (!pool->base.irqs)
+		goto create_fail;
+
+	j = 0;
+	/* mem input -> ipp -> dpp -> opp -> TG */
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		/* if pipe is disabled, skip instance of HW pipe,
+		 * i.e, skip ASIC register instance
+		 */
+		if ((pipe_fuses & (1 << i)) != 0)
+			continue;
+
+		pool->base.hubps[j] = dcn21_hubp_create(ctx, i);
+		if (pool->base.hubps[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create memory input!\n");
+			goto create_fail;
+		}
+
+		pool->base.ipps[j] = dcn21_ipp_create(ctx, i);
+		if (pool->base.ipps[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create input pixel processor!\n");
+			goto create_fail;
+		}
+
+		pool->base.dpps[j] = dcn21_dpp_create(ctx, i);
+		if (pool->base.dpps[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create dpps!\n");
+			goto create_fail;
+		}
+
+		pool->base.opps[j] = dcn21_opp_create(ctx, i);
+		if (pool->base.opps[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create output pixel processor!\n");
+			goto create_fail;
+		}
+
+		pool->base.timing_generators[j] = dcn21_timing_generator_create(
+				ctx, i);
+		if (pool->base.timing_generators[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto create_fail;
+		}
+		j++;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.engines[i] = dcn21_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto create_fail;
+		}
+		pool->base.hw_i2cs[i] = dcn21_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create hw i2c!!\n");
+			goto create_fail;
+		}
+		pool->base.sw_i2cs[i] = NULL;
+	}
+
+	pool->base.timing_generator_count = j;
+	pool->base.pipe_count = j;
+	pool->base.mpcc_count = j;
+
+	pool->base.mpc = dcn21_mpc_create(ctx);
+	if (pool->base.mpc == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mpc!\n");
+		goto create_fail;
+	}
+
+	pool->base.hubbub = dcn21_hubbub_create(ctx);
+	if (pool->base.hubbub == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create hubbub!\n");
+		goto create_fail;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		pool->base.dscs[i] = dcn21_dsc_create(ctx, i);
+		if (pool->base.dscs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create display stream compressor %d!\n", i);
+			goto create_fail;
+		}
+	}
+
+	if (!dcn20_dwbc_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create dwbc!\n");
+		goto create_fail;
+	}
+	if (!dcn20_mmhubbub_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mcif_wb!\n");
+		goto create_fail;
+	}
+
+	if (!resource_construct(num_virtual_links, dc, &pool->base,
+			&res_create_funcs))
+		goto create_fail;
+
+	dcn21_hw_sequencer_construct(dc);
+
+	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->cap_funcs = cap_funcs;
+
+	return true;
+
+create_fail:
+
+	dcn21_resource_destruct(pool);
+
+	return false;
+}
+
+struct resource_pool *dcn21_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc)
+{
+	struct dcn21_resource_pool *pool =
+		kzalloc(sizeof(struct dcn21_resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dcn21_resource_construct(init_data->num_virtual_links, dc, pool))
+		return &pool->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(pool);
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.h
new file mode 100644
index 000000000000..f7ecc002c2f7
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.h
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef _DCN21_RESOURCE_H_
+#define _DCN21_RESOURCE_H_
+
+#include "core_types.h"
+
+#define TO_DCN21_RES_POOL(pool)\
+	container_of(pool, struct dcn21_resource_pool, base)
+
+struct dc;
+struct resource_pool;
+struct _vcs_dpi_display_pipe_params_st;
+
+extern struct _vcs_dpi_ip_params_st dcn2_1_ip;
+extern struct _vcs_dpi_soc_bounding_box_st dcn2_1_soc;
+
+struct dcn21_resource_pool {
+	struct resource_pool base;
+};
+struct resource_pool *dcn21_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc);
+bool dcn21_fast_validate_bw(
+		struct dc *dc,
+		struct dc_state *context,
+		display_e2e_pipe_params_st *pipes,
+		int *pipe_cnt_out,
+		int *pipe_split_from,
+		int *vlevel_out,
+		bool fast_validate);
+
+#endif /* _DCN21_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c
new file mode 100644
index 000000000000..2b6dcb489b90
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c
@@ -0,0 +1,2611 @@
+/*
+ * Copyright 2020 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+
+#include "dm_services.h"
+#include "dc.h"
+
+#include "dcn30/dcn30_init.h"
+
+#include "resource.h"
+#include "include/irq_service_interface.h"
+#include "dcn20/dcn20_resource.h"
+
+#include "dcn30_resource.h"
+
+#include "dcn10/dcn10_ipp.h"
+#include "dcn30/dcn30_hubbub.h"
+#include "dcn30/dcn30_mpc.h"
+#include "dcn30/dcn30_hubp.h"
+#include "irq/dcn30/irq_service_dcn30.h"
+#include "dcn30/dcn30_dpp.h"
+#include "dcn30/dcn30_optc.h"
+#include "dcn20/dcn20_hwseq.h"
+#include "dcn30/dcn30_hwseq.h"
+#include "dce110/dce110_hwseq.h"
+#include "dcn30/dcn30_opp.h"
+#include "dcn20/dcn20_dsc.h"
+#include "dcn30/dcn30_vpg.h"
+#include "dcn30/dcn30_afmt.h"
+#include "dcn30/dcn30_dio_stream_encoder.h"
+#include "dcn30/dcn30_dio_link_encoder.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_hwseq.h"
+#include "clk_mgr.h"
+#include "virtual/virtual_stream_encoder.h"
+#include "dce110/dce110_resource.h"
+#include "dml/display_mode_vba.h"
+#include "dcn30/dcn30_dccg.h"
+#include "dcn10/dcn10_resource.h"
+#include "link.h"
+#include "dce/dce_panel_cntl.h"
+
+#include "dcn30/dcn30_dwb.h"
+#include "dcn30/dcn30_mmhubbub.h"
+
+#include "sienna_cichlid_ip_offset.h"
+#include "dcn/dcn_3_0_0_offset.h"
+#include "dcn/dcn_3_0_0_sh_mask.h"
+
+#include "nbio/nbio_7_4_offset.h"
+
+#include "dpcs/dpcs_3_0_0_offset.h"
+#include "dpcs/dpcs_3_0_0_sh_mask.h"
+
+#include "mmhub/mmhub_2_0_0_offset.h"
+#include "mmhub/mmhub_2_0_0_sh_mask.h"
+
+#include "reg_helper.h"
+#include "dce/dmub_abm.h"
+#include "dce/dmub_psr.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_i2c.h"
+
+#include "dml/dcn30/dcn30_fpu.h"
+#include "dml/dcn30/display_mode_vba_30.h"
+#include "vm_helper.h"
+#include "dcn20/dcn20_vmid.h"
+#include "amdgpu_socbb.h"
+#include "dc_dmub_srv.h"
+
+#define DC_LOGGER \
+	dc->ctx->logger
+#define DC_LOGGER_INIT(logger)
+
+enum dcn30_clk_src_array_id {
+	DCN30_CLK_SRC_PLL0,
+	DCN30_CLK_SRC_PLL1,
+	DCN30_CLK_SRC_PLL2,
+	DCN30_CLK_SRC_PLL3,
+	DCN30_CLK_SRC_PLL4,
+	DCN30_CLK_SRC_PLL5,
+	DCN30_CLK_SRC_TOTAL
+};
+
+/* begin *********************
+ * macros to expend register list macro defined in HW object header file
+ */
+
+/* DCN */
+#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
+
+#define BASE(seg) BASE_INNER(seg)
+
+#define SR(reg_name)\
+		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) +  \
+					mm ## reg_name
+
+#define SRI(reg_name, block, id)\
+	.reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define SRI2(reg_name, block, id)\
+	.reg_name = BASE(mm ## reg_name ## _BASE_IDX) + \
+					mm ## reg_name
+
+#define SRIR(var_name, reg_name, block, id)\
+	.var_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define SRII(reg_name, block, id)\
+	.reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define SRII_MPC_RMU(reg_name, block, id)\
+	.RMU##_##reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define SRII_DWB(reg_name, temp_name, block, id)\
+	.reg_name[id] = BASE(mm ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## temp_name
+
+#define SF_DWB2(reg_name, block, id, field_name, post_fix)	\
+	.field_name = reg_name ## __ ## field_name ## post_fix
+
+#define DCCG_SRII(reg_name, block, id)\
+	.block ## _ ## reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define VUPDATE_SRII(reg_name, block, id)\
+	.reg_name[id] = BASE(mm ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
+					mm ## reg_name ## _ ## block ## id
+
+/* NBIO */
+#define NBIO_BASE_INNER(seg) \
+	NBIO_BASE__INST0_SEG ## seg
+
+#define NBIO_BASE(seg) \
+	NBIO_BASE_INNER(seg)
+
+#define NBIO_SR(reg_name)\
+		.reg_name = NBIO_BASE(mm ## reg_name ## _BASE_IDX) + \
+					mm ## reg_name
+
+/* MMHUB */
+#define MMHUB_BASE_INNER(seg) \
+	MMHUB_BASE__INST0_SEG ## seg
+
+#define MMHUB_BASE(seg) \
+	MMHUB_BASE_INNER(seg)
+
+#define MMHUB_SR(reg_name)\
+		.reg_name = MMHUB_BASE(mmMM ## reg_name ## _BASE_IDX) + \
+					mmMM ## reg_name
+
+/* CLOCK */
+#define CLK_BASE_INNER(seg) \
+	CLK_BASE__INST0_SEG ## seg
+
+#define CLK_BASE(seg) \
+	CLK_BASE_INNER(seg)
+
+#define CLK_SRI(reg_name, block, inst)\
+	.reg_name = CLK_BASE(mm ## block ## _ ## inst ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## _ ## inst ## _ ## reg_name
+
+
+static const struct bios_registers bios_regs = {
+		NBIO_SR(BIOS_SCRATCH_3),
+		NBIO_SR(BIOS_SCRATCH_6)
+};
+
+#define clk_src_regs(index, pllid)\
+[index] = {\
+	CS_COMMON_REG_LIST_DCN2_0(index, pllid),\
+}
+
+static const struct dce110_clk_src_regs clk_src_regs[] = {
+	clk_src_regs(0, A),
+	clk_src_regs(1, B),
+	clk_src_regs(2, C),
+	clk_src_regs(3, D),
+	clk_src_regs(4, E),
+	clk_src_regs(5, F)
+};
+
+static const struct dce110_clk_src_shift cs_shift = {
+		CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
+};
+
+static const struct dce110_clk_src_mask cs_mask = {
+		CS_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
+};
+
+#define abm_regs(id)\
+[id] = {\
+		ABM_DCN30_REG_LIST(id)\
+}
+
+static const struct dce_abm_registers abm_regs[] = {
+		abm_regs(0),
+		abm_regs(1),
+		abm_regs(2),
+		abm_regs(3),
+		abm_regs(4),
+		abm_regs(5),
+};
+
+static const struct dce_abm_shift abm_shift = {
+		ABM_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dce_abm_mask abm_mask = {
+		ABM_MASK_SH_LIST_DCN30(_MASK)
+};
+
+
+
+#define audio_regs(id)\
+[id] = {\
+		AUD_COMMON_REG_LIST(id)\
+}
+
+static const struct dce_audio_registers audio_regs[] = {
+	audio_regs(0),
+	audio_regs(1),
+	audio_regs(2),
+	audio_regs(3),
+	audio_regs(4),
+	audio_regs(5),
+	audio_regs(6)
+};
+
+#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
+		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
+
+static const struct dce_audio_shift audio_shift = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_audio_mask audio_mask = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+#define vpg_regs(id)\
+[id] = {\
+	VPG_DCN3_REG_LIST(id)\
+}
+
+static const struct dcn30_vpg_registers vpg_regs[] = {
+	vpg_regs(0),
+	vpg_regs(1),
+	vpg_regs(2),
+	vpg_regs(3),
+	vpg_regs(4),
+	vpg_regs(5),
+	vpg_regs(6),
+};
+
+static const struct dcn30_vpg_shift vpg_shift = {
+	DCN3_VPG_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn30_vpg_mask vpg_mask = {
+	DCN3_VPG_MASK_SH_LIST(_MASK)
+};
+
+#define afmt_regs(id)\
+[id] = {\
+	AFMT_DCN3_REG_LIST(id)\
+}
+
+static const struct dcn30_afmt_registers afmt_regs[] = {
+	afmt_regs(0),
+	afmt_regs(1),
+	afmt_regs(2),
+	afmt_regs(3),
+	afmt_regs(4),
+	afmt_regs(5),
+	afmt_regs(6),
+};
+
+static const struct dcn30_afmt_shift afmt_shift = {
+	DCN3_AFMT_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn30_afmt_mask afmt_mask = {
+	DCN3_AFMT_MASK_SH_LIST(_MASK)
+};
+
+#define stream_enc_regs(id)\
+[id] = {\
+	SE_DCN3_REG_LIST(id)\
+}
+
+static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
+	stream_enc_regs(0),
+	stream_enc_regs(1),
+	stream_enc_regs(2),
+	stream_enc_regs(3),
+	stream_enc_regs(4),
+	stream_enc_regs(5)
+};
+
+static const struct dcn10_stream_encoder_shift se_shift = {
+		SE_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn10_stream_encoder_mask se_mask = {
+		SE_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+
+#define aux_regs(id)\
+[id] = {\
+	DCN2_AUX_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
+		aux_regs(0),
+		aux_regs(1),
+		aux_regs(2),
+		aux_regs(3),
+		aux_regs(4),
+		aux_regs(5)
+};
+
+#define hpd_regs(id)\
+[id] = {\
+	HPD_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
+		hpd_regs(0),
+		hpd_regs(1),
+		hpd_regs(2),
+		hpd_regs(3),
+		hpd_regs(4),
+		hpd_regs(5)
+};
+
+#define link_regs(id, phyid)\
+[id] = {\
+	LE_DCN3_REG_LIST(id), \
+	UNIPHY_DCN2_REG_LIST(phyid), \
+	DPCS_DCN2_REG_LIST(id), \
+	SRI(DP_DPHY_INTERNAL_CTRL, DP, id) \
+}
+
+static const struct dce110_aux_registers_shift aux_shift = {
+	DCN_AUX_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+	DCN_AUX_MASK_SH_LIST(_MASK)
+};
+
+static const struct dcn10_link_enc_registers link_enc_regs[] = {
+	link_regs(0, A),
+	link_regs(1, B),
+	link_regs(2, C),
+	link_regs(3, D),
+	link_regs(4, E),
+	link_regs(5, F)
+};
+
+static const struct dcn10_link_enc_shift le_shift = {
+	LINK_ENCODER_MASK_SH_LIST_DCN30(__SHIFT),\
+	DPCS_DCN2_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn10_link_enc_mask le_mask = {
+	LINK_ENCODER_MASK_SH_LIST_DCN30(_MASK),\
+	DPCS_DCN2_MASK_SH_LIST(_MASK)
+};
+
+
+static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
+	{ DCN_PANEL_CNTL_REG_LIST() }
+};
+
+static const struct dce_panel_cntl_shift panel_cntl_shift = {
+	DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_panel_cntl_mask panel_cntl_mask = {
+	DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
+};
+
+#define dpp_regs(id)\
+[id] = {\
+	DPP_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn3_dpp_registers dpp_regs[] = {
+	dpp_regs(0),
+	dpp_regs(1),
+	dpp_regs(2),
+	dpp_regs(3),
+	dpp_regs(4),
+	dpp_regs(5),
+};
+
+static const struct dcn3_dpp_shift tf_shift = {
+		DPP_REG_LIST_SH_MASK_DCN30(__SHIFT)
+};
+
+static const struct dcn3_dpp_mask tf_mask = {
+		DPP_REG_LIST_SH_MASK_DCN30(_MASK)
+};
+
+#define opp_regs(id)\
+[id] = {\
+	OPP_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn20_opp_registers opp_regs[] = {
+	opp_regs(0),
+	opp_regs(1),
+	opp_regs(2),
+	opp_regs(3),
+	opp_regs(4),
+	opp_regs(5)
+};
+
+static const struct dcn20_opp_shift opp_shift = {
+	OPP_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn20_opp_mask opp_mask = {
+	OPP_MASK_SH_LIST_DCN20(_MASK)
+};
+
+#define aux_engine_regs(id)\
+[id] = {\
+	AUX_COMMON_REG_LIST0(id), \
+	.AUXN_IMPCAL = 0, \
+	.AUXP_IMPCAL = 0, \
+	.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1),
+		aux_engine_regs(2),
+		aux_engine_regs(3),
+		aux_engine_regs(4),
+		aux_engine_regs(5)
+};
+
+#define dwbc_regs_dcn3(id)\
+[id] = {\
+	DWBC_COMMON_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn30_dwbc_registers dwbc30_regs[] = {
+	dwbc_regs_dcn3(0),
+};
+
+static const struct dcn30_dwbc_shift dwbc30_shift = {
+	DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_dwbc_mask dwbc30_mask = {
+	DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define mcif_wb_regs_dcn3(id)\
+[id] = {\
+	MCIF_WB_COMMON_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn30_mmhubbub_registers mcif_wb30_regs[] = {
+	mcif_wb_regs_dcn3(0)
+};
+
+static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define dsc_regsDCN20(id)\
+[id] = {\
+	DSC_REG_LIST_DCN20(id)\
+}
+
+static const struct dcn20_dsc_registers dsc_regs[] = {
+	dsc_regsDCN20(0),
+	dsc_regsDCN20(1),
+	dsc_regsDCN20(2),
+	dsc_regsDCN20(3),
+	dsc_regsDCN20(4),
+	dsc_regsDCN20(5)
+};
+
+static const struct dcn20_dsc_shift dsc_shift = {
+	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
+};
+
+static const struct dcn20_dsc_mask dsc_mask = {
+	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
+};
+
+static const struct dcn30_mpc_registers mpc_regs = {
+		MPC_REG_LIST_DCN3_0(0),
+		MPC_REG_LIST_DCN3_0(1),
+		MPC_REG_LIST_DCN3_0(2),
+		MPC_REG_LIST_DCN3_0(3),
+		MPC_REG_LIST_DCN3_0(4),
+		MPC_REG_LIST_DCN3_0(5),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(0),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(1),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(2),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(3),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(4),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(5),
+		MPC_RMU_GLOBAL_REG_LIST_DCN3AG,
+		MPC_RMU_REG_LIST_DCN3AG(0),
+		MPC_RMU_REG_LIST_DCN3AG(1),
+		MPC_RMU_REG_LIST_DCN3AG(2),
+		MPC_DWB_MUX_REG_LIST_DCN3_0(0),
+};
+
+static const struct dcn30_mpc_shift mpc_shift = {
+	MPC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_mpc_mask mpc_mask = {
+	MPC_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define optc_regs(id)\
+[id] = {OPTC_COMMON_REG_LIST_DCN3_0(id)}
+
+
+static const struct dcn_optc_registers optc_regs[] = {
+	optc_regs(0),
+	optc_regs(1),
+	optc_regs(2),
+	optc_regs(3),
+	optc_regs(4),
+	optc_regs(5)
+};
+
+static const struct dcn_optc_shift optc_shift = {
+	OPTC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn_optc_mask optc_mask = {
+	OPTC_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define hubp_regs(id)\
+[id] = {\
+	HUBP_REG_LIST_DCN30(id)\
+}
+
+static const struct dcn_hubp2_registers hubp_regs[] = {
+		hubp_regs(0),
+		hubp_regs(1),
+		hubp_regs(2),
+		hubp_regs(3),
+		hubp_regs(4),
+		hubp_regs(5)
+};
+
+static const struct dcn_hubp2_shift hubp_shift = {
+		HUBP_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn_hubp2_mask hubp_mask = {
+		HUBP_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static const struct dcn_hubbub_registers hubbub_reg = {
+		HUBBUB_REG_LIST_DCN30(0)
+};
+
+static const struct dcn_hubbub_shift hubbub_shift = {
+		HUBBUB_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn_hubbub_mask hubbub_mask = {
+		HUBBUB_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static const struct dccg_registers dccg_regs = {
+		DCCG_REG_LIST_DCN30()
+};
+
+static const struct dccg_shift dccg_shift = {
+		DCCG_MASK_SH_LIST_DCN3(__SHIFT)
+};
+
+static const struct dccg_mask dccg_mask = {
+		DCCG_MASK_SH_LIST_DCN3(_MASK)
+};
+
+static const struct dce_hwseq_registers hwseq_reg = {
+		HWSEQ_DCN30_REG_LIST()
+};
+
+static const struct dce_hwseq_shift hwseq_shift = {
+		HWSEQ_DCN30_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+		HWSEQ_DCN30_MASK_SH_LIST(_MASK)
+};
+#define vmid_regs(id)\
+[id] = {\
+		DCN20_VMID_REG_LIST(id)\
+}
+
+static const struct dcn_vmid_registers vmid_regs[] = {
+	vmid_regs(0),
+	vmid_regs(1),
+	vmid_regs(2),
+	vmid_regs(3),
+	vmid_regs(4),
+	vmid_regs(5),
+	vmid_regs(6),
+	vmid_regs(7),
+	vmid_regs(8),
+	vmid_regs(9),
+	vmid_regs(10),
+	vmid_regs(11),
+	vmid_regs(12),
+	vmid_regs(13),
+	vmid_regs(14),
+	vmid_regs(15)
+};
+
+static const struct dcn20_vmid_shift vmid_shifts = {
+		DCN20_VMID_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn20_vmid_mask vmid_masks = {
+		DCN20_VMID_MASK_SH_LIST(_MASK)
+};
+
+static const struct resource_caps res_cap_dcn3 = {
+	.num_timing_generator = 6,
+	.num_opp = 6,
+	.num_video_plane = 6,
+	.num_audio = 6,
+	.num_stream_encoder = 6,
+	.num_pll = 6,
+	.num_dwb = 1,
+	.num_ddc = 6,
+	.num_vmid = 16,
+	.num_mpc_3dlut = 3,
+	.num_dsc = 6,
+};
+
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
+	.per_pixel_alpha = true,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = true,
+			.fp16 = true,
+			.p010 = true,
+			.ayuv = false,
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 16000,
+			.fp16 = 16000
+	},
+
+	/* 6:1 downscaling ratio: 1000/6 = 166.666 */
+	.max_downscale_factor = {
+			.argb8888 = 167,
+			.nv12 = 167,
+			.fp16 = 167
+	},
+	16,
+	16
+};
+
+static const struct dc_debug_options debug_defaults_drv = {
+	.disable_dmcu = true, //No DMCU on DCN30
+	.force_abm_enable = false,
+	.timing_trace = false,
+	.clock_trace = true,
+	.disable_pplib_clock_request = true,
+	.pipe_split_policy = MPC_SPLIT_DYNAMIC,
+	.force_single_disp_pipe_split = false,
+	.disable_dcc = DCC_ENABLE,
+	.vsr_support = true,
+	.performance_trace = false,
+	.max_downscale_src_width = 7680,/*upto 8K*/
+	.disable_pplib_wm_range = false,
+	.scl_reset_length10 = true,
+	.sanity_checks = false,
+	.underflow_assert_delay_us = 0xFFFFFFFF,
+	.dwb_fi_phase = -1, // -1 = disable,
+	.dmub_command_table = true,
+	.use_max_lb = true,
+	.exit_idle_opt_for_cursor_updates = true,
+	.enable_legacy_fast_update = false,
+	.using_dml2 = false,
+};
+
+static const struct dc_panel_config panel_config_defaults = {
+	.psr = {
+		.disable_psr = false,
+		.disallow_psrsu = false,
+		.disallow_replay = false,
+	},
+};
+
+static void dcn30_dpp_destroy(struct dpp **dpp)
+{
+	kfree(TO_DCN20_DPP(*dpp));
+	*dpp = NULL;
+}
+
+static struct dpp *dcn30_dpp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn3_dpp *dpp =
+		kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
+
+	if (!dpp)
+		return NULL;
+
+	if (dpp3_construct(dpp, ctx, inst,
+			&dpp_regs[inst], &tf_shift, &tf_mask))
+		return &dpp->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(dpp);
+	return NULL;
+}
+
+static struct output_pixel_processor *dcn30_opp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_opp *opp =
+		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
+
+	if (!opp) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dcn20_opp_construct(opp, ctx, inst,
+			&opp_regs[inst], &opp_shift, &opp_mask);
+	return &opp->base;
+}
+
+static struct dce_aux *dcn30_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst],
+					&aux_mask,
+					&aux_shift,
+					ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+
+#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST_DCN30(id) }
+
+static const struct dce_i2c_registers i2c_hw_regs[] = {
+		i2c_inst_regs(1),
+		i2c_inst_regs(2),
+		i2c_inst_regs(3),
+		i2c_inst_regs(4),
+		i2c_inst_regs(5),
+		i2c_inst_regs(6),
+};
+
+static const struct dce_i2c_shift i2c_shifts = {
+		I2C_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dce_i2c_mask i2c_masks = {
+		I2C_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static struct dce_i2c_hw *dcn30_i2c_hw_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw =
+		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
+				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+
+static struct mpc *dcn30_mpc_create(
+		struct dc_context *ctx,
+		int num_mpcc,
+		int num_rmu)
+{
+	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc),
+					  GFP_KERNEL);
+
+	if (!mpc30)
+		return NULL;
+
+	dcn30_mpc_construct(mpc30, ctx,
+			&mpc_regs,
+			&mpc_shift,
+			&mpc_mask,
+			num_mpcc,
+			num_rmu);
+
+	return &mpc30->base;
+}
+
+static struct hubbub *dcn30_hubbub_create(struct dc_context *ctx)
+{
+	int i;
+
+	struct dcn20_hubbub *hubbub3 = kzalloc(sizeof(struct dcn20_hubbub),
+					  GFP_KERNEL);
+
+	if (!hubbub3)
+		return NULL;
+
+	hubbub3_construct(hubbub3, ctx,
+			&hubbub_reg,
+			&hubbub_shift,
+			&hubbub_mask);
+
+
+	for (i = 0; i < res_cap_dcn3.num_vmid; i++) {
+		struct dcn20_vmid *vmid = &hubbub3->vmid[i];
+
+		vmid->ctx = ctx;
+
+		vmid->regs = &vmid_regs[i];
+		vmid->shifts = &vmid_shifts;
+		vmid->masks = &vmid_masks;
+	}
+
+	return &hubbub3->base;
+}
+
+static struct timing_generator *dcn30_timing_generator_create(
+		struct dc_context *ctx,
+		uint32_t instance)
+{
+	struct optc *tgn10 =
+		kzalloc(sizeof(struct optc), GFP_KERNEL);
+
+	if (!tgn10)
+		return NULL;
+
+	tgn10->base.inst = instance;
+	tgn10->base.ctx = ctx;
+
+	tgn10->tg_regs = &optc_regs[instance];
+	tgn10->tg_shift = &optc_shift;
+	tgn10->tg_mask = &optc_mask;
+
+	dcn30_timing_generator_init(tgn10);
+
+	return &tgn10->base;
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+		.max_hdmi_deep_color = COLOR_DEPTH_121212,
+		.max_hdmi_pixel_clock = 600000,
+		.hdmi_ycbcr420_supported = true,
+		.dp_ycbcr420_supported = true,
+		.fec_supported = true,
+		.flags.bits.IS_HBR2_CAPABLE = true,
+		.flags.bits.IS_HBR3_CAPABLE = true,
+		.flags.bits.IS_TPS3_CAPABLE = true,
+		.flags.bits.IS_TPS4_CAPABLE = true
+};
+
+static struct link_encoder *dcn30_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dcn20_link_encoder *enc20 =
+		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+
+	if (!enc20)
+		return NULL;
+
+	dcn30_link_encoder_construct(enc20,
+			enc_init_data,
+			&link_enc_feature,
+			&link_enc_regs[enc_init_data->transmitter],
+			&link_enc_aux_regs[enc_init_data->channel - 1],
+			&link_enc_hpd_regs[enc_init_data->hpd_source],
+			&le_shift,
+			&le_mask);
+
+	return &enc20->enc10.base;
+}
+
+static struct panel_cntl *dcn30_panel_cntl_create(const struct panel_cntl_init_data *init_data)
+{
+	struct dce_panel_cntl *panel_cntl =
+		kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
+
+	if (!panel_cntl)
+		return NULL;
+
+	dce_panel_cntl_construct(panel_cntl,
+			init_data,
+			&panel_cntl_regs[init_data->inst],
+			&panel_cntl_shift,
+			&panel_cntl_mask);
+
+	return &panel_cntl->base;
+}
+
+static void read_dce_straps(
+	struct dc_context *ctx,
+	struct resource_straps *straps)
+{
+	generic_reg_get(ctx, mmDC_PINSTRAPS + BASE(mmDC_PINSTRAPS_BASE_IDX),
+		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
+
+}
+
+static struct audio *dcn30_create_audio(
+		struct dc_context *ctx, unsigned int inst)
+{
+	return dce_audio_create(ctx, inst,
+			&audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+static struct vpg *dcn30_vpg_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn30_vpg *vpg3 = kzalloc(sizeof(struct dcn30_vpg), GFP_KERNEL);
+
+	if (!vpg3)
+		return NULL;
+
+	vpg3_construct(vpg3, ctx, inst,
+			&vpg_regs[inst],
+			&vpg_shift,
+			&vpg_mask);
+
+	return &vpg3->base;
+}
+
+static struct afmt *dcn30_afmt_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn30_afmt *afmt3 = kzalloc(sizeof(struct dcn30_afmt), GFP_KERNEL);
+
+	if (!afmt3)
+		return NULL;
+
+	afmt3_construct(afmt3, ctx, inst,
+			&afmt_regs[inst],
+			&afmt_shift,
+			&afmt_mask);
+
+	return &afmt3->base;
+}
+
+static struct stream_encoder *dcn30_stream_encoder_create(enum engine_id eng_id,
+							  struct dc_context *ctx)
+{
+	struct dcn10_stream_encoder *enc1;
+	struct vpg *vpg;
+	struct afmt *afmt;
+	int vpg_inst;
+	int afmt_inst;
+
+	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
+	if (eng_id <= ENGINE_ID_DIGF) {
+		vpg_inst = eng_id;
+		afmt_inst = eng_id;
+	} else
+		return NULL;
+
+	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
+	vpg = dcn30_vpg_create(ctx, vpg_inst);
+	afmt = dcn30_afmt_create(ctx, afmt_inst);
+
+	if (!enc1 || !vpg || !afmt) {
+		kfree(enc1);
+		kfree(vpg);
+		kfree(afmt);
+		return NULL;
+	}
+
+	dcn30_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
+					eng_id, vpg, afmt,
+					&stream_enc_regs[eng_id],
+					&se_shift, &se_mask);
+
+	return &enc1->base;
+}
+
+static struct dce_hwseq *dcn30_hwseq_create(struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &hwseq_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+	}
+	return hws;
+}
+static const struct resource_create_funcs res_create_funcs = {
+	.read_dce_straps = read_dce_straps,
+	.create_audio = dcn30_create_audio,
+	.create_stream_encoder = dcn30_stream_encoder_create,
+	.create_hwseq = dcn30_hwseq_create,
+};
+
+static void dcn30_resource_destruct(struct dcn30_resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->base.stream_enc_count; i++) {
+		if (pool->base.stream_enc[i] != NULL) {
+			if (pool->base.stream_enc[i]->vpg != NULL) {
+				kfree(DCN30_VPG_FROM_VPG(pool->base.stream_enc[i]->vpg));
+				pool->base.stream_enc[i]->vpg = NULL;
+			}
+			if (pool->base.stream_enc[i]->afmt != NULL) {
+				kfree(DCN30_AFMT_FROM_AFMT(pool->base.stream_enc[i]->afmt));
+				pool->base.stream_enc[i]->afmt = NULL;
+			}
+			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
+			pool->base.stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		if (pool->base.dscs[i] != NULL)
+			dcn20_dsc_destroy(&pool->base.dscs[i]);
+	}
+
+	if (pool->base.mpc != NULL) {
+		kfree(TO_DCN20_MPC(pool->base.mpc));
+		pool->base.mpc = NULL;
+	}
+	if (pool->base.hubbub != NULL) {
+		kfree(pool->base.hubbub);
+		pool->base.hubbub = NULL;
+	}
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (pool->base.dpps[i] != NULL)
+			dcn30_dpp_destroy(&pool->base.dpps[i]);
+
+		if (pool->base.ipps[i] != NULL)
+			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
+
+		if (pool->base.hubps[i] != NULL) {
+			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
+			pool->base.hubps[i] = NULL;
+		}
+
+		if (pool->base.irqs != NULL) {
+			dal_irq_service_destroy(&pool->base.irqs);
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+		if (pool->base.hw_i2cs[i] != NULL) {
+			kfree(pool->base.hw_i2cs[i]);
+			pool->base.hw_i2cs[i] = NULL;
+		}
+		if (pool->base.sw_i2cs[i] != NULL) {
+			kfree(pool->base.sw_i2cs[i]);
+			pool->base.sw_i2cs[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		if (pool->base.opps[i] != NULL)
+			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.timing_generators[i] != NULL)	{
+			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
+			pool->base.timing_generators[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+		if (pool->base.dwbc[i] != NULL) {
+			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
+			pool->base.dwbc[i] = NULL;
+		}
+		if (pool->base.mcif_wb[i] != NULL) {
+			kfree(TO_DCN30_MMHUBBUB(pool->base.mcif_wb[i]));
+			pool->base.mcif_wb[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.audio_count; i++) {
+		if (pool->base.audios[i])
+			dce_aud_destroy(&pool->base.audios[i]);
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] != NULL) {
+			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
+			pool->base.clock_sources[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+		if (pool->base.mpc_lut[i] != NULL) {
+			dc_3dlut_func_release(pool->base.mpc_lut[i]);
+			pool->base.mpc_lut[i] = NULL;
+		}
+		if (pool->base.mpc_shaper[i] != NULL) {
+			dc_transfer_func_release(pool->base.mpc_shaper[i]);
+			pool->base.mpc_shaper[i] = NULL;
+		}
+	}
+
+	if (pool->base.dp_clock_source != NULL) {
+		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
+		pool->base.dp_clock_source = NULL;
+	}
+
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (pool->base.multiple_abms[i] != NULL)
+			dce_abm_destroy(&pool->base.multiple_abms[i]);
+	}
+
+	if (pool->base.psr != NULL)
+		dmub_psr_destroy(&pool->base.psr);
+
+	if (pool->base.dccg != NULL)
+		dcn_dccg_destroy(&pool->base.dccg);
+
+	if (pool->base.oem_device != NULL) {
+		struct dc *dc = pool->base.oem_device->ctx->dc;
+
+		dc->link_srv->destroy_ddc_service(&pool->base.oem_device);
+	}
+}
+
+static struct hubp *dcn30_hubp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn20_hubp *hubp2 =
+		kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
+
+	if (!hubp2)
+		return NULL;
+
+	if (hubp3_construct(hubp2, ctx, inst,
+			&hubp_regs[inst], &hubp_shift, &hubp_mask))
+		return &hubp2->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(hubp2);
+	return NULL;
+}
+
+static bool dcn30_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc),
+						    GFP_KERNEL);
+
+		if (!dwbc30) {
+			dm_error("DC: failed to create dwbc30!\n");
+			return false;
+		}
+
+		dcn30_dwbc_construct(dwbc30, ctx,
+				&dwbc30_regs[i],
+				&dwbc30_shift,
+				&dwbc30_mask,
+				i);
+
+		pool->dwbc[i] = &dwbc30->base;
+	}
+	return true;
+}
+
+static bool dcn30_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub),
+						    GFP_KERNEL);
+
+		if (!mcif_wb30) {
+			dm_error("DC: failed to create mcif_wb30!\n");
+			return false;
+		}
+
+		dcn30_mmhubbub_construct(mcif_wb30, ctx,
+				&mcif_wb30_regs[i],
+				&mcif_wb30_shift,
+				&mcif_wb30_mask,
+				i);
+
+		pool->mcif_wb[i] = &mcif_wb30->base;
+	}
+	return true;
+}
+
+static struct display_stream_compressor *dcn30_dsc_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_dsc *dsc =
+		kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
+
+	if (!dsc) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
+	return &dsc->base;
+}
+
+enum dc_status dcn30_add_stream_to_ctx(struct dc *dc, struct dc_state *new_ctx, struct dc_stream_state *dc_stream)
+{
+
+	return dcn20_add_stream_to_ctx(dc, new_ctx, dc_stream);
+}
+
+static void dcn30_destroy_resource_pool(struct resource_pool **pool)
+{
+	struct dcn30_resource_pool *dcn30_pool = TO_DCN30_RES_POOL(*pool);
+
+	dcn30_resource_destruct(dcn30_pool);
+	kfree(dcn30_pool);
+	*pool = NULL;
+}
+
+static struct clock_source *dcn30_clock_source_create(
+		struct dc_context *ctx,
+		struct dc_bios *bios,
+		enum clock_source_id id,
+		const struct dce110_clk_src_regs *regs,
+		bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dcn3_clk_src_construct(clk_src, ctx, bios, id,
+			regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	kfree(clk_src);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+int dcn30_populate_dml_pipes_from_context(
+	struct dc *dc, struct dc_state *context,
+	display_e2e_pipe_params_st *pipes,
+	bool fast_validate)
+{
+	int i, pipe_cnt;
+	struct resource_context *res_ctx = &context->res_ctx;
+
+	DC_FP_START();
+	dcn20_populate_dml_pipes_from_context(dc, context, pipes, fast_validate);
+	DC_FP_END();
+
+	for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) {
+		if (!res_ctx->pipe_ctx[i].stream)
+			continue;
+
+		pipes[pipe_cnt++].pipe.scale_ratio_depth.lb_depth =
+			dm_lb_16;
+	}
+
+	return pipe_cnt;
+}
+
+void dcn30_populate_dml_writeback_from_context(
+	struct dc *dc, struct resource_context *res_ctx, display_e2e_pipe_params_st *pipes)
+{
+	DC_FP_START();
+	dcn30_fpu_populate_dml_writeback_from_context(dc, res_ctx, pipes);
+	DC_FP_END();
+}
+
+unsigned int dcn30_calc_max_scaled_time(
+		unsigned int time_per_pixel,
+		enum mmhubbub_wbif_mode mode,
+		unsigned int urgent_watermark)
+{
+	unsigned int time_per_byte = 0;
+	unsigned int total_free_entry = 0xb40;
+	unsigned int buf_lh_capability;
+	unsigned int max_scaled_time;
+
+	if (mode == PACKED_444) /* packed mode 32 bpp */
+		time_per_byte = time_per_pixel/4;
+	else if (mode == PACKED_444_FP16) /* packed mode 64 bpp */
+		time_per_byte = time_per_pixel/8;
+
+	if (time_per_byte == 0)
+		time_per_byte = 1;
+
+	buf_lh_capability = (total_free_entry*time_per_byte*32) >> 6; /* time_per_byte is in u6.6*/
+	max_scaled_time   = buf_lh_capability - urgent_watermark;
+	return max_scaled_time;
+}
+
+void dcn30_set_mcif_arb_params(
+		struct dc *dc,
+		struct dc_state *context,
+		display_e2e_pipe_params_st *pipes,
+		int pipe_cnt)
+{
+	enum mmhubbub_wbif_mode wbif_mode;
+	struct display_mode_lib *dml = &context->bw_ctx.dml;
+	struct mcif_arb_params *wb_arb_params;
+	int i, j, dwb_pipe;
+
+	/* Writeback MCIF_WB arbitration parameters */
+	dwb_pipe = 0;
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+
+		if (!context->res_ctx.pipe_ctx[i].stream)
+			continue;
+
+		for (j = 0; j < MAX_DWB_PIPES; j++) {
+			struct dc_writeback_info *writeback_info = &context->res_ctx.pipe_ctx[i].stream->writeback_info[j];
+
+			if (writeback_info->wb_enabled == false)
+				continue;
+
+			//wb_arb_params = &context->res_ctx.pipe_ctx[i].stream->writeback_info[j].mcif_arb_params;
+			wb_arb_params = &context->bw_ctx.bw.dcn.bw_writeback.mcif_wb_arb[dwb_pipe];
+
+			if (writeback_info->dwb_params.cnv_params.fc_out_format == DWB_OUT_FORMAT_64BPP_ARGB ||
+				writeback_info->dwb_params.cnv_params.fc_out_format == DWB_OUT_FORMAT_64BPP_RGBA)
+				wbif_mode = PACKED_444_FP16;
+			else
+				wbif_mode = PACKED_444;
+
+			DC_FP_START();
+			dcn30_fpu_set_mcif_arb_params(wb_arb_params, dml, pipes, pipe_cnt, j);
+			DC_FP_END();
+			wb_arb_params->time_per_pixel = (1000000 << 6) / context->res_ctx.pipe_ctx[i].stream->phy_pix_clk; /* time_per_pixel should be in u6.6 format */
+			wb_arb_params->slice_lines = 32;
+			wb_arb_params->arbitration_slice = 2; /* irrelevant since there is no YUV output */
+			wb_arb_params->max_scaled_time = dcn30_calc_max_scaled_time(wb_arb_params->time_per_pixel,
+					wbif_mode,
+					wb_arb_params->cli_watermark[0]); /* assume 4 watermark sets have the same value */
+
+			dwb_pipe++;
+
+			if (dwb_pipe >= MAX_DWB_PIPES)
+				return;
+		}
+		if (dwb_pipe >= MAX_DWB_PIPES)
+			return;
+	}
+
+}
+
+static struct dc_cap_funcs cap_funcs = {
+	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
+};
+
+bool dcn30_acquire_post_bldn_3dlut(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		int mpcc_id,
+		struct dc_3dlut **lut,
+		struct dc_transfer_func **shaper)
+{
+	int i;
+	bool ret = false;
+	union dc_3dlut_state *state;
+
+	ASSERT(*lut == NULL && *shaper == NULL);
+	*lut = NULL;
+	*shaper = NULL;
+
+	for (i = 0; i < pool->res_cap->num_mpc_3dlut; i++) {
+		if (!res_ctx->is_mpc_3dlut_acquired[i]) {
+			*lut = pool->mpc_lut[i];
+			*shaper = pool->mpc_shaper[i];
+			state = &pool->mpc_lut[i]->state;
+			res_ctx->is_mpc_3dlut_acquired[i] = true;
+			state->bits.rmu_idx_valid = 1;
+			state->bits.rmu_mux_num = i;
+			if (state->bits.rmu_mux_num == 0)
+				state->bits.mpc_rmu0_mux = mpcc_id;
+			else if (state->bits.rmu_mux_num == 1)
+				state->bits.mpc_rmu1_mux = mpcc_id;
+			else if (state->bits.rmu_mux_num == 2)
+				state->bits.mpc_rmu2_mux = mpcc_id;
+			ret = true;
+			break;
+		}
+	}
+	return ret;
+}
+
+bool dcn30_release_post_bldn_3dlut(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		struct dc_3dlut **lut,
+		struct dc_transfer_func **shaper)
+{
+	int i;
+	bool ret = false;
+
+	for (i = 0; i < pool->res_cap->num_mpc_3dlut; i++) {
+		if (pool->mpc_lut[i] == *lut && pool->mpc_shaper[i] == *shaper) {
+			res_ctx->is_mpc_3dlut_acquired[i] = false;
+			pool->mpc_lut[i]->state.raw = 0;
+			*lut = NULL;
+			*shaper = NULL;
+			ret = true;
+			break;
+		}
+	}
+	return ret;
+}
+
+static bool is_soc_bounding_box_valid(struct dc *dc)
+{
+	uint32_t hw_internal_rev = dc->ctx->asic_id.hw_internal_rev;
+
+	if (ASICREV_IS_SIENNA_CICHLID_P(hw_internal_rev))
+		return true;
+
+	return false;
+}
+
+static bool init_soc_bounding_box(struct dc *dc,
+				  struct dcn30_resource_pool *pool)
+{
+	struct _vcs_dpi_soc_bounding_box_st *loaded_bb = &dcn3_0_soc;
+	struct _vcs_dpi_ip_params_st *loaded_ip = &dcn3_0_ip;
+
+	DC_LOGGER_INIT(dc->ctx->logger);
+
+	if (!is_soc_bounding_box_valid(dc)) {
+		DC_LOG_ERROR("%s: not valid soc bounding box\n", __func__);
+		return false;
+	}
+
+	loaded_ip->max_num_otg = pool->base.res_cap->num_timing_generator;
+	loaded_ip->max_num_dpp = pool->base.pipe_count;
+	loaded_ip->clamp_min_dcfclk = dc->config.clamp_min_dcfclk;
+	dcn20_patch_bounding_box(dc, loaded_bb);
+	DC_FP_START();
+	patch_dcn30_soc_bounding_box(dc, &dcn3_0_soc);
+	DC_FP_END();
+
+	return true;
+}
+
+static bool dcn30_split_stream_for_mpc_or_odm(
+		const struct dc *dc,
+		struct resource_context *res_ctx,
+		struct pipe_ctx *pri_pipe,
+		struct pipe_ctx *sec_pipe,
+		bool odm)
+{
+	int pipe_idx = sec_pipe->pipe_idx;
+	const struct resource_pool *pool = dc->res_pool;
+
+	*sec_pipe = *pri_pipe;
+
+	sec_pipe->pipe_idx = pipe_idx;
+	sec_pipe->plane_res.mi = pool->mis[pipe_idx];
+	sec_pipe->plane_res.hubp = pool->hubps[pipe_idx];
+	sec_pipe->plane_res.ipp = pool->ipps[pipe_idx];
+	sec_pipe->plane_res.xfm = pool->transforms[pipe_idx];
+	sec_pipe->plane_res.dpp = pool->dpps[pipe_idx];
+	sec_pipe->plane_res.mpcc_inst = pool->dpps[pipe_idx]->inst;
+	sec_pipe->stream_res.dsc = NULL;
+	if (odm) {
+		if (pri_pipe->next_odm_pipe) {
+			ASSERT(pri_pipe->next_odm_pipe != sec_pipe);
+			sec_pipe->next_odm_pipe = pri_pipe->next_odm_pipe;
+			sec_pipe->next_odm_pipe->prev_odm_pipe = sec_pipe;
+		}
+		if (pri_pipe->top_pipe && pri_pipe->top_pipe->next_odm_pipe) {
+			pri_pipe->top_pipe->next_odm_pipe->bottom_pipe = sec_pipe;
+			sec_pipe->top_pipe = pri_pipe->top_pipe->next_odm_pipe;
+		}
+		if (pri_pipe->bottom_pipe && pri_pipe->bottom_pipe->next_odm_pipe) {
+			pri_pipe->bottom_pipe->next_odm_pipe->top_pipe = sec_pipe;
+			sec_pipe->bottom_pipe = pri_pipe->bottom_pipe->next_odm_pipe;
+		}
+		pri_pipe->next_odm_pipe = sec_pipe;
+		sec_pipe->prev_odm_pipe = pri_pipe;
+
+		if (!sec_pipe->top_pipe)
+			sec_pipe->stream_res.opp = pool->opps[pipe_idx];
+		else
+			sec_pipe->stream_res.opp = sec_pipe->top_pipe->stream_res.opp;
+		if (sec_pipe->stream->timing.flags.DSC == 1) {
+			dcn20_acquire_dsc(dc, res_ctx, &sec_pipe->stream_res.dsc, pipe_idx);
+			ASSERT(sec_pipe->stream_res.dsc);
+			if (sec_pipe->stream_res.dsc == NULL)
+				return false;
+		}
+	} else {
+		if (pri_pipe->bottom_pipe) {
+			ASSERT(pri_pipe->bottom_pipe != sec_pipe);
+			sec_pipe->bottom_pipe = pri_pipe->bottom_pipe;
+			sec_pipe->bottom_pipe->top_pipe = sec_pipe;
+		}
+		pri_pipe->bottom_pipe = sec_pipe;
+		sec_pipe->top_pipe = pri_pipe;
+
+		ASSERT(pri_pipe->plane_state);
+	}
+
+	return true;
+}
+
+static struct pipe_ctx *dcn30_find_split_pipe(
+		struct dc *dc,
+		struct dc_state *context,
+		int old_index)
+{
+	struct pipe_ctx *pipe = NULL;
+	int i;
+
+	if (old_index >= 0 && context->res_ctx.pipe_ctx[old_index].stream == NULL) {
+		pipe = &context->res_ctx.pipe_ctx[old_index];
+		pipe->pipe_idx = old_index;
+	}
+
+	if (!pipe)
+		for (i = dc->res_pool->pipe_count - 1; i >= 0; i--) {
+			if (dc->current_state->res_ctx.pipe_ctx[i].top_pipe == NULL
+					&& dc->current_state->res_ctx.pipe_ctx[i].prev_odm_pipe == NULL) {
+				if (context->res_ctx.pipe_ctx[i].stream == NULL) {
+					pipe = &context->res_ctx.pipe_ctx[i];
+					pipe->pipe_idx = i;
+					break;
+				}
+			}
+		}
+
+	/*
+	 * May need to fix pipes getting tossed from 1 opp to another on flip
+	 * Add for debugging transient underflow during topology updates:
+	 * ASSERT(pipe);
+	 */
+	if (!pipe)
+		for (i = dc->res_pool->pipe_count - 1; i >= 0; i--) {
+			if (context->res_ctx.pipe_ctx[i].stream == NULL) {
+				pipe = &context->res_ctx.pipe_ctx[i];
+				pipe->pipe_idx = i;
+				break;
+			}
+		}
+
+	return pipe;
+}
+
+noinline bool dcn30_internal_validate_bw(
+		struct dc *dc,
+		struct dc_state *context,
+		display_e2e_pipe_params_st *pipes,
+		int *pipe_cnt_out,
+		int *vlevel_out,
+		bool fast_validate,
+		bool allow_self_refresh_only)
+{
+	bool out = false;
+	bool repopulate_pipes = false;
+	int split[MAX_PIPES] = { 0 };
+	bool merge[MAX_PIPES] = { false };
+	bool newly_split[MAX_PIPES] = { false };
+	int pipe_cnt, i, pipe_idx, vlevel;
+	struct vba_vars_st *vba = &context->bw_ctx.dml.vba;
+
+	ASSERT(pipes);
+	if (!pipes)
+		return false;
+
+	context->bw_ctx.dml.vba.maxMpcComb = 0;
+	context->bw_ctx.dml.vba.VoltageLevel = 0;
+	context->bw_ctx.dml.vba.DRAMClockChangeSupport[0][0] = dm_dram_clock_change_vactive;
+	dc->res_pool->funcs->update_soc_for_wm_a(dc, context);
+	pipe_cnt = dc->res_pool->funcs->populate_dml_pipes(dc, context, pipes, fast_validate);
+
+	if (!pipe_cnt) {
+		out = true;
+		goto validate_out;
+	}
+
+	dml_log_pipe_params(&context->bw_ctx.dml, pipes, pipe_cnt);
+
+	if (!fast_validate || !allow_self_refresh_only) {
+		/*
+		 * DML favors voltage over p-state, but we're more interested in
+		 * supporting p-state over voltage. We can't support p-state in
+		 * prefetch mode > 0 so try capping the prefetch mode to start.
+		 */
+		context->bw_ctx.dml.soc.allow_dram_self_refresh_or_dram_clock_change_in_vblank =
+			dm_allow_self_refresh_and_mclk_switch;
+		vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
+		/* This may adjust vlevel and maxMpcComb */
+		if (vlevel < context->bw_ctx.dml.soc.num_states)
+			vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, vlevel, split, merge);
+	}
+	if (allow_self_refresh_only &&
+	    (fast_validate || vlevel == context->bw_ctx.dml.soc.num_states ||
+			vba->DRAMClockChangeSupport[vlevel][vba->maxMpcComb] == dm_dram_clock_change_unsupported)) {
+		/*
+		 * If mode is unsupported or there's still no p-state support
+		 * then fall back to favoring voltage.
+		 *
+		 * We don't actually support prefetch mode 2, so require that we
+		 * at least support prefetch mode 1.
+		 */
+		context->bw_ctx.dml.soc.allow_dram_self_refresh_or_dram_clock_change_in_vblank =
+			dm_allow_self_refresh;
+
+		vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
+		if (vlevel < context->bw_ctx.dml.soc.num_states) {
+			memset(split, 0, sizeof(split));
+			memset(merge, 0, sizeof(merge));
+			vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, vlevel, split, merge);
+		}
+	}
+
+	dml_log_mode_support_params(&context->bw_ctx.dml);
+
+	if (vlevel == context->bw_ctx.dml.soc.num_states)
+		goto validate_fail;
+
+	if (!dc->config.enable_windowed_mpo_odm) {
+		for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
+			struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+			struct pipe_ctx *mpo_pipe = pipe->bottom_pipe;
+
+			if (!pipe->stream)
+				continue;
+
+			/* We only support full screen mpo with ODM */
+			if (vba->ODMCombineEnabled[vba->pipe_plane[pipe_idx]] != dm_odm_combine_mode_disabled
+					&& pipe->plane_state && mpo_pipe
+					&& memcmp(&mpo_pipe->plane_state->clip_rect,
+							&pipe->stream->src,
+							sizeof(struct rect)) != 0) {
+				ASSERT(mpo_pipe->plane_state != pipe->plane_state);
+				goto validate_fail;
+			}
+			pipe_idx++;
+		}
+	}
+
+	/* merge pipes if necessary */
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+
+		/*skip pipes that don't need merging*/
+		if (!merge[i])
+			continue;
+
+		/* if ODM merge we ignore mpc tree, mpo pipes will have their own flags */
+		if (pipe->prev_odm_pipe) {
+			/*split off odm pipe*/
+			pipe->prev_odm_pipe->next_odm_pipe = pipe->next_odm_pipe;
+			if (pipe->next_odm_pipe)
+				pipe->next_odm_pipe->prev_odm_pipe = pipe->prev_odm_pipe;
+
+			pipe->bottom_pipe = NULL;
+			pipe->next_odm_pipe = NULL;
+			pipe->plane_state = NULL;
+			pipe->stream = NULL;
+			pipe->top_pipe = NULL;
+			pipe->prev_odm_pipe = NULL;
+			if (pipe->stream_res.dsc)
+				dcn20_release_dsc(&context->res_ctx, dc->res_pool, &pipe->stream_res.dsc);
+			memset(&pipe->plane_res, 0, sizeof(pipe->plane_res));
+			memset(&pipe->stream_res, 0, sizeof(pipe->stream_res));
+			repopulate_pipes = true;
+		} else if (pipe->top_pipe && pipe->top_pipe->plane_state == pipe->plane_state) {
+			struct pipe_ctx *top_pipe = pipe->top_pipe;
+			struct pipe_ctx *bottom_pipe = pipe->bottom_pipe;
+
+			top_pipe->bottom_pipe = bottom_pipe;
+			if (bottom_pipe)
+				bottom_pipe->top_pipe = top_pipe;
+
+			pipe->top_pipe = NULL;
+			pipe->bottom_pipe = NULL;
+			pipe->plane_state = NULL;
+			pipe->stream = NULL;
+			memset(&pipe->plane_res, 0, sizeof(pipe->plane_res));
+			memset(&pipe->stream_res, 0, sizeof(pipe->stream_res));
+			repopulate_pipes = true;
+		} else
+			ASSERT(0); /* Should never try to merge master pipe */
+
+	}
+
+	for (i = 0, pipe_idx = -1; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+		struct pipe_ctx *old_pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+		struct pipe_ctx *hsplit_pipe = NULL;
+		bool odm;
+		int old_index = -1;
+
+		if (!pipe->stream || newly_split[i])
+			continue;
+
+		pipe_idx++;
+		odm = vba->ODMCombineEnabled[vba->pipe_plane[pipe_idx]] != dm_odm_combine_mode_disabled;
+
+		if (!pipe->plane_state && !odm)
+			continue;
+
+		if (split[i]) {
+			if (odm) {
+				if (split[i] == 4 && old_pipe->next_odm_pipe && old_pipe->next_odm_pipe->next_odm_pipe)
+					old_index = old_pipe->next_odm_pipe->next_odm_pipe->pipe_idx;
+				else if (old_pipe->next_odm_pipe)
+					old_index = old_pipe->next_odm_pipe->pipe_idx;
+			} else {
+				if (split[i] == 4 && old_pipe->bottom_pipe && old_pipe->bottom_pipe->bottom_pipe &&
+						old_pipe->bottom_pipe->bottom_pipe->plane_state == old_pipe->plane_state)
+					old_index = old_pipe->bottom_pipe->bottom_pipe->pipe_idx;
+				else if (old_pipe->bottom_pipe &&
+						old_pipe->bottom_pipe->plane_state == old_pipe->plane_state)
+					old_index = old_pipe->bottom_pipe->pipe_idx;
+			}
+			hsplit_pipe = dcn30_find_split_pipe(dc, context, old_index);
+			ASSERT(hsplit_pipe);
+			if (!hsplit_pipe)
+				goto validate_fail;
+
+			if (!dcn30_split_stream_for_mpc_or_odm(
+					dc, &context->res_ctx,
+					pipe, hsplit_pipe, odm))
+				goto validate_fail;
+
+			newly_split[hsplit_pipe->pipe_idx] = true;
+			repopulate_pipes = true;
+		}
+		if (split[i] == 4) {
+			struct pipe_ctx *pipe_4to1;
+
+			if (odm && old_pipe->next_odm_pipe)
+				old_index = old_pipe->next_odm_pipe->pipe_idx;
+			else if (!odm && old_pipe->bottom_pipe &&
+						old_pipe->bottom_pipe->plane_state == old_pipe->plane_state)
+				old_index = old_pipe->bottom_pipe->pipe_idx;
+			else
+				old_index = -1;
+			pipe_4to1 = dcn30_find_split_pipe(dc, context, old_index);
+			ASSERT(pipe_4to1);
+			if (!pipe_4to1)
+				goto validate_fail;
+			if (!dcn30_split_stream_for_mpc_or_odm(
+					dc, &context->res_ctx,
+					pipe, pipe_4to1, odm))
+				goto validate_fail;
+			newly_split[pipe_4to1->pipe_idx] = true;
+
+			if (odm && old_pipe->next_odm_pipe && old_pipe->next_odm_pipe->next_odm_pipe
+					&& old_pipe->next_odm_pipe->next_odm_pipe->next_odm_pipe)
+				old_index = old_pipe->next_odm_pipe->next_odm_pipe->next_odm_pipe->pipe_idx;
+			else if (!odm && old_pipe->bottom_pipe && old_pipe->bottom_pipe->bottom_pipe &&
+					old_pipe->bottom_pipe->bottom_pipe->bottom_pipe &&
+					old_pipe->bottom_pipe->bottom_pipe->bottom_pipe->plane_state == old_pipe->plane_state)
+				old_index = old_pipe->bottom_pipe->bottom_pipe->bottom_pipe->pipe_idx;
+			else
+				old_index = -1;
+			pipe_4to1 = dcn30_find_split_pipe(dc, context, old_index);
+			ASSERT(pipe_4to1);
+			if (!pipe_4to1)
+				goto validate_fail;
+			if (!dcn30_split_stream_for_mpc_or_odm(
+					dc, &context->res_ctx,
+					hsplit_pipe, pipe_4to1, odm))
+				goto validate_fail;
+			newly_split[pipe_4to1->pipe_idx] = true;
+		}
+		if (odm)
+			dcn20_build_mapped_resource(dc, context, pipe->stream);
+	}
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+
+		if (pipe->plane_state) {
+			if (!resource_build_scaling_params(pipe))
+				goto validate_fail;
+		}
+	}
+
+	/* Actual dsc count per stream dsc validation*/
+	if (!dcn20_validate_dsc(dc, context)) {
+		vba->ValidationStatus[vba->soc.num_states] = DML_FAIL_DSC_VALIDATION_FAILURE;
+		goto validate_fail;
+	}
+
+	if (repopulate_pipes)
+		pipe_cnt = dc->res_pool->funcs->populate_dml_pipes(dc, context, pipes, fast_validate);
+	context->bw_ctx.dml.vba.VoltageLevel = vlevel;
+	*vlevel_out = vlevel;
+	*pipe_cnt_out = pipe_cnt;
+
+	out = true;
+	goto validate_out;
+
+validate_fail:
+	out = false;
+
+validate_out:
+	return out;
+}
+
+static int get_refresh_rate(struct dc_state *context)
+{
+	int refresh_rate = 0;
+	int h_v_total = 0;
+	struct dc_crtc_timing *timing = NULL;
+
+	if (context == NULL || context->streams[0] == NULL)
+		return 0;
+
+	/* check if refresh rate at least 120hz */
+	timing = &context->streams[0]->timing;
+	if (timing == NULL)
+		return 0;
+
+	h_v_total = timing->h_total * timing->v_total;
+	if (h_v_total == 0)
+		return 0;
+
+	refresh_rate = ((timing->pix_clk_100hz * 100) / (h_v_total)) + 1;
+	return refresh_rate;
+}
+
+#define MAX_STRETCHED_V_BLANK 500 // in micro-seconds
+/*
+ * Scaling factor for v_blank stretch calculations considering timing in
+ * micro-seconds and pixel clock in 100hz.
+ * Note: the parenthesis are necessary to ensure the correct order of
+ * operation where V_SCALE is used.
+ */
+#define V_SCALE (10000 / MAX_STRETCHED_V_BLANK)
+
+static int get_frame_rate_at_max_stretch_100hz(struct dc_state *context)
+{
+	struct dc_crtc_timing *timing = NULL;
+	uint32_t sec_per_100_lines;
+	uint32_t max_v_blank;
+	uint32_t curr_v_blank;
+	uint32_t v_stretch_max;
+	uint32_t stretched_frame_pix_cnt;
+	uint32_t scaled_stretched_frame_pix_cnt;
+	uint32_t scaled_refresh_rate;
+
+	if (context == NULL || context->streams[0] == NULL)
+		return 0;
+
+	/* check if refresh rate at least 120hz */
+	timing = &context->streams[0]->timing;
+	if (timing == NULL)
+		return 0;
+
+	sec_per_100_lines = timing->pix_clk_100hz / timing->h_total + 1;
+	max_v_blank = sec_per_100_lines / V_SCALE + 1;
+	curr_v_blank = timing->v_total - timing->v_addressable;
+	v_stretch_max = (max_v_blank > curr_v_blank) ? (max_v_blank - curr_v_blank) : (0);
+	stretched_frame_pix_cnt = (v_stretch_max + timing->v_total) * timing->h_total;
+	scaled_stretched_frame_pix_cnt = stretched_frame_pix_cnt / 10000;
+	scaled_refresh_rate = (timing->pix_clk_100hz) / scaled_stretched_frame_pix_cnt + 1;
+
+	return scaled_refresh_rate;
+}
+
+static bool is_refresh_rate_support_mclk_switch_using_fw_based_vblank_stretch(struct dc_state *context)
+{
+	int refresh_rate_max_stretch_100hz;
+	int min_refresh_100hz;
+
+	if (context == NULL || context->streams[0] == NULL)
+		return false;
+
+	refresh_rate_max_stretch_100hz = get_frame_rate_at_max_stretch_100hz(context);
+	min_refresh_100hz = context->streams[0]->timing.min_refresh_in_uhz / 10000;
+
+	if (refresh_rate_max_stretch_100hz < min_refresh_100hz)
+		return false;
+
+	return true;
+}
+
+bool dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch(struct dc *dc, struct dc_state *context)
+{
+	int refresh_rate = 0;
+	const int minimum_refreshrate_supported = 120;
+
+	if (context == NULL || context->streams[0] == NULL)
+		return false;
+
+	if (context->streams[0]->sink->edid_caps.panel_patch.disable_fams)
+		return false;
+
+	if (dc->debug.disable_fams)
+		return false;
+
+	if (!dc->caps.dmub_caps.mclk_sw)
+		return false;
+
+	if (context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching_shut_down)
+		return false;
+
+	/* more then 1 monitor connected */
+	if (context->stream_count != 1)
+		return false;
+
+	refresh_rate = get_refresh_rate(context);
+	if (refresh_rate < minimum_refreshrate_supported)
+		return false;
+
+	if (!is_refresh_rate_support_mclk_switch_using_fw_based_vblank_stretch(context))
+		return false;
+
+	if (!context->streams[0]->allow_freesync)
+		return false;
+
+	if (context->streams[0]->vrr_active_variable && dc->debug.disable_fams_gaming)
+		return false;
+
+	context->streams[0]->fpo_in_use = true;
+
+	return true;
+}
+
+/*
+ * set up FPO watermarks, pstate, dram latency
+ */
+void dcn30_setup_mclk_switch_using_fw_based_vblank_stretch(struct dc *dc, struct dc_state *context)
+{
+	ASSERT(dc != NULL && context != NULL);
+	if (dc == NULL || context == NULL)
+		return;
+
+	/* Set wm_a.pstate so high natural MCLK switches are impossible: 4 seconds */
+	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns = 4U * 1000U * 1000U * 1000U;
+}
+
+void dcn30_update_soc_for_wm_a(struct dc *dc, struct dc_state *context)
+{
+	DC_FP_START();
+	dcn30_fpu_update_soc_for_wm_a(dc, context);
+	DC_FP_END();
+}
+
+void dcn30_calculate_wm_and_dlg(
+		struct dc *dc, struct dc_state *context,
+		display_e2e_pipe_params_st *pipes,
+		int pipe_cnt,
+		int vlevel)
+{
+	DC_FP_START();
+	dcn30_fpu_calculate_wm_and_dlg(dc, context, pipes, pipe_cnt, vlevel);
+	DC_FP_END();
+}
+
+bool dcn30_validate_bandwidth(struct dc *dc,
+		struct dc_state *context,
+		bool fast_validate)
+{
+	bool out = false;
+
+	BW_VAL_TRACE_SETUP();
+
+	int vlevel = 0;
+	int pipe_cnt = 0;
+	display_e2e_pipe_params_st *pipes = kzalloc(dc->res_pool->pipe_count * sizeof(display_e2e_pipe_params_st), GFP_KERNEL);
+	DC_LOGGER_INIT(dc->ctx->logger);
+
+	BW_VAL_TRACE_COUNT();
+
+	DC_FP_START();
+	out = dcn30_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, fast_validate, true);
+	DC_FP_END();
+
+	if (pipe_cnt == 0)
+		goto validate_out;
+
+	if (!out)
+		goto validate_fail;
+
+	BW_VAL_TRACE_END_VOLTAGE_LEVEL();
+
+	if (fast_validate) {
+		BW_VAL_TRACE_SKIP(fast);
+		goto validate_out;
+	}
+
+	DC_FP_START();
+	if (dc->res_pool->funcs->calculate_wm_and_dlg)
+		dc->res_pool->funcs->calculate_wm_and_dlg(dc, context, pipes, pipe_cnt, vlevel);
+	DC_FP_END();
+
+	BW_VAL_TRACE_END_WATERMARKS();
+
+	goto validate_out;
+
+validate_fail:
+	DC_LOG_WARNING("Mode Validation Warning: %s failed validation.\n",
+		dml_get_status_message(context->bw_ctx.dml.vba.ValidationStatus[context->bw_ctx.dml.vba.soc.num_states]));
+
+	BW_VAL_TRACE_SKIP(fail);
+	out = false;
+
+validate_out:
+	kfree(pipes);
+
+	BW_VAL_TRACE_FINISH();
+
+	return out;
+}
+
+void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
+{
+	unsigned int i, j;
+	unsigned int num_states = 0;
+
+	unsigned int dcfclk_mhz[DC__VOLTAGE_STATES] = {0};
+	unsigned int dram_speed_mts[DC__VOLTAGE_STATES] = {0};
+	unsigned int optimal_uclk_for_dcfclk_sta_targets[DC__VOLTAGE_STATES] = {0};
+	unsigned int optimal_dcfclk_for_uclk[DC__VOLTAGE_STATES] = {0};
+
+	unsigned int dcfclk_sta_targets[DC__VOLTAGE_STATES] = {694, 875, 1000, 1200};
+	unsigned int num_dcfclk_sta_targets = 4;
+	unsigned int num_uclk_states;
+
+	struct dc_bounding_box_max_clk dcn30_bb_max_clk;
+
+	memset(&dcn30_bb_max_clk, 0, sizeof(dcn30_bb_max_clk));
+
+	if (dc->ctx->dc_bios->vram_info.num_chans)
+		dcn3_0_soc.num_chans = dc->ctx->dc_bios->vram_info.num_chans;
+
+	DC_FP_START();
+	dcn30_fpu_update_dram_channel_width_bytes(dc);
+	DC_FP_END();
+
+	if (bw_params->clk_table.entries[0].memclk_mhz) {
+
+		for (i = 0; i < MAX_NUM_DPM_LVL; i++) {
+			if (bw_params->clk_table.entries[i].dcfclk_mhz > dcn30_bb_max_clk.max_dcfclk_mhz)
+				dcn30_bb_max_clk.max_dcfclk_mhz = bw_params->clk_table.entries[i].dcfclk_mhz;
+			if (bw_params->clk_table.entries[i].dispclk_mhz > dcn30_bb_max_clk.max_dispclk_mhz)
+				dcn30_bb_max_clk.max_dispclk_mhz = bw_params->clk_table.entries[i].dispclk_mhz;
+			if (bw_params->clk_table.entries[i].dppclk_mhz > dcn30_bb_max_clk.max_dppclk_mhz)
+				dcn30_bb_max_clk.max_dppclk_mhz = bw_params->clk_table.entries[i].dppclk_mhz;
+			if (bw_params->clk_table.entries[i].phyclk_mhz > dcn30_bb_max_clk.max_phyclk_mhz)
+				dcn30_bb_max_clk.max_phyclk_mhz = bw_params->clk_table.entries[i].phyclk_mhz;
+		}
+
+		DC_FP_START();
+		dcn30_fpu_update_max_clk(&dcn30_bb_max_clk);
+		DC_FP_END();
+
+		if (dcn30_bb_max_clk.max_dcfclk_mhz > dcfclk_sta_targets[num_dcfclk_sta_targets-1]) {
+			// If max DCFCLK is greater than the max DCFCLK STA target, insert into the DCFCLK STA target array
+			dcfclk_sta_targets[num_dcfclk_sta_targets] = dcn30_bb_max_clk.max_dcfclk_mhz;
+			num_dcfclk_sta_targets++;
+		} else if (dcn30_bb_max_clk.max_dcfclk_mhz < dcfclk_sta_targets[num_dcfclk_sta_targets-1]) {
+			// If max DCFCLK is less than the max DCFCLK STA target, cap values and remove duplicates
+			for (i = 0; i < num_dcfclk_sta_targets; i++) {
+				if (dcfclk_sta_targets[i] > dcn30_bb_max_clk.max_dcfclk_mhz) {
+					dcfclk_sta_targets[i] = dcn30_bb_max_clk.max_dcfclk_mhz;
+					break;
+				}
+			}
+			// Update size of array since we "removed" duplicates
+			num_dcfclk_sta_targets = i + 1;
+		}
+
+		num_uclk_states = bw_params->clk_table.num_entries;
+
+		// Calculate optimal dcfclk for each uclk
+		for (i = 0; i < num_uclk_states; i++) {
+			DC_FP_START();
+			dcn30_fpu_get_optimal_dcfclk_fclk_for_uclk(bw_params->clk_table.entries[i].memclk_mhz * 16,
+					&optimal_dcfclk_for_uclk[i], NULL);
+			DC_FP_END();
+			if (optimal_dcfclk_for_uclk[i] < bw_params->clk_table.entries[0].dcfclk_mhz) {
+				optimal_dcfclk_for_uclk[i] = bw_params->clk_table.entries[0].dcfclk_mhz;
+			}
+		}
+
+		// Calculate optimal uclk for each dcfclk sta target
+		for (i = 0; i < num_dcfclk_sta_targets; i++) {
+			for (j = 0; j < num_uclk_states; j++) {
+				if (dcfclk_sta_targets[i] < optimal_dcfclk_for_uclk[j]) {
+					optimal_uclk_for_dcfclk_sta_targets[i] =
+							bw_params->clk_table.entries[j].memclk_mhz * 16;
+					break;
+				}
+			}
+		}
+
+		i = 0;
+		j = 0;
+		// create the final dcfclk and uclk table
+		while (i < num_dcfclk_sta_targets && j < num_uclk_states && num_states < DC__VOLTAGE_STATES) {
+			if (dcfclk_sta_targets[i] < optimal_dcfclk_for_uclk[j] && i < num_dcfclk_sta_targets) {
+				dcfclk_mhz[num_states] = dcfclk_sta_targets[i];
+				dram_speed_mts[num_states++] = optimal_uclk_for_dcfclk_sta_targets[i++];
+			} else {
+				if (j < num_uclk_states && optimal_dcfclk_for_uclk[j] <= dcn30_bb_max_clk.max_dcfclk_mhz) {
+					dcfclk_mhz[num_states] = optimal_dcfclk_for_uclk[j];
+					dram_speed_mts[num_states++] = bw_params->clk_table.entries[j++].memclk_mhz * 16;
+				} else {
+					j = num_uclk_states;
+				}
+			}
+		}
+
+		while (i < num_dcfclk_sta_targets && num_states < DC__VOLTAGE_STATES) {
+			dcfclk_mhz[num_states] = dcfclk_sta_targets[i];
+			dram_speed_mts[num_states++] = optimal_uclk_for_dcfclk_sta_targets[i++];
+		}
+
+		while (j < num_uclk_states && num_states < DC__VOLTAGE_STATES &&
+				optimal_dcfclk_for_uclk[j] <= dcn30_bb_max_clk.max_dcfclk_mhz) {
+			dcfclk_mhz[num_states] = optimal_dcfclk_for_uclk[j];
+			dram_speed_mts[num_states++] = bw_params->clk_table.entries[j++].memclk_mhz * 16;
+		}
+
+		dcn3_0_soc.num_states = num_states;
+		DC_FP_START();
+		dcn30_fpu_update_bw_bounding_box(dc, bw_params, &dcn30_bb_max_clk, dcfclk_mhz, dram_speed_mts);
+		DC_FP_END();
+	}
+}
+
+static void dcn30_get_panel_config_defaults(struct dc_panel_config *panel_config)
+{
+	*panel_config = panel_config_defaults;
+}
+
+static const struct resource_funcs dcn30_res_pool_funcs = {
+	.destroy = dcn30_destroy_resource_pool,
+	.link_enc_create = dcn30_link_encoder_create,
+	.panel_cntl_create = dcn30_panel_cntl_create,
+	.validate_bandwidth = dcn30_validate_bandwidth,
+	.calculate_wm_and_dlg = dcn30_calculate_wm_and_dlg,
+	.update_soc_for_wm_a = dcn30_update_soc_for_wm_a,
+	.populate_dml_pipes = dcn30_populate_dml_pipes_from_context,
+	.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
+	.release_pipe = dcn20_release_pipe,
+	.add_stream_to_ctx = dcn30_add_stream_to_ctx,
+	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
+	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
+	.populate_dml_writeback_from_context = dcn30_populate_dml_writeback_from_context,
+	.set_mcif_arb_params = dcn30_set_mcif_arb_params,
+	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
+	.acquire_post_bldn_3dlut = dcn30_acquire_post_bldn_3dlut,
+	.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
+	.update_bw_bounding_box = dcn30_update_bw_bounding_box,
+	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
+	.get_panel_config_defaults = dcn30_get_panel_config_defaults,
+};
+
+#define CTX ctx
+
+#define REG(reg_name) \
+	(DCN_BASE.instance[0].segment[mm ## reg_name ## _BASE_IDX] + mm ## reg_name)
+
+static uint32_t read_pipe_fuses(struct dc_context *ctx)
+{
+	uint32_t value = REG_READ(CC_DC_PIPE_DIS);
+	/* Support for max 6 pipes */
+	value = value & 0x3f;
+	return value;
+}
+
+static bool dcn30_resource_construct(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct dcn30_resource_pool *pool)
+{
+	int i;
+	struct dc_context *ctx = dc->ctx;
+	struct irq_service_init_data init_data;
+	struct ddc_service_init_data ddc_init_data = {0};
+	uint32_t pipe_fuses = read_pipe_fuses(ctx);
+	uint32_t num_pipes = 0;
+
+	if (!(pipe_fuses == 0 || pipe_fuses == 0x3e)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: Unexpected fuse recipe for navi2x !\n");
+		/* fault to single pipe */
+		pipe_fuses = 0x3e;
+	}
+
+	DC_FP_START();
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	pool->base.res_cap = &res_cap_dcn3;
+
+	pool->base.funcs = &dcn30_res_pool_funcs;
+
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
+	pool->base.mpcc_count = pool->base.res_cap->num_timing_generator;
+	dc->caps.max_downscale_ratio = 600;
+	dc->caps.i2c_speed_in_khz = 100;
+	dc->caps.i2c_speed_in_khz_hdcp = 100; /*1.4 w/a not applied by default*/
+	dc->caps.max_cursor_size = 256;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.dmdata_alloc_size = 2048;
+	dc->caps.mall_size_per_mem_channel = 8;
+	/* total size = mall per channel * num channels * 1024 * 1024 */
+	dc->caps.mall_size_total = dc->caps.mall_size_per_mem_channel * dc->ctx->dc_bios->vram_info.num_chans * 1048576;
+	dc->caps.cursor_cache_size = dc->caps.max_cursor_size * dc->caps.max_cursor_size * 8;
+
+	dc->caps.max_slave_planes = 2;
+	dc->caps.max_slave_yuv_planes = 2;
+	dc->caps.max_slave_rgb_planes = 2;
+	dc->caps.post_blend_color_processing = true;
+	dc->caps.force_dp_tps4_for_cp2520 = true;
+	dc->caps.extended_aux_timeout_support = true;
+	dc->caps.dmcub_support = true;
+
+	/* Color pipeline capabilities */
+	dc->caps.color.dpp.dcn_arch = 1;
+	dc->caps.color.dpp.input_lut_shared = 0;
+	dc->caps.color.dpp.icsc = 1;
+	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
+	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
+	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
+	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
+	dc->caps.color.dpp.post_csc = 1;
+	dc->caps.color.dpp.gamma_corr = 1;
+	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
+
+	dc->caps.color.dpp.hw_3d_lut = 1;
+	dc->caps.color.dpp.ogam_ram = 1;
+	// no OGAM ROM on DCN3
+	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
+	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
+	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
+	dc->caps.color.dpp.ocsc = 0;
+
+	dc->caps.color.mpc.gamut_remap = 1;
+	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //3
+	dc->caps.color.mpc.ogam_ram = 1;
+	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
+	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
+	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
+	dc->caps.color.mpc.ocsc = 1;
+
+	dc->caps.dp_hdmi21_pcon_support = true;
+	dc->caps.max_v_total = (1 << 15) - 1;
+
+	/* read VBIOS LTTPR caps */
+	{
+		if (ctx->dc_bios->funcs->get_lttpr_caps) {
+			enum bp_result bp_query_result;
+			uint8_t is_vbios_lttpr_enable = 0;
+
+			bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
+			dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
+		}
+
+		if (ctx->dc_bios->funcs->get_lttpr_interop) {
+			enum bp_result bp_query_result;
+			uint8_t is_vbios_interop_enabled = 0;
+
+			bp_query_result = ctx->dc_bios->funcs->get_lttpr_interop(ctx->dc_bios,
+					&is_vbios_interop_enabled);
+			dc->caps.vbios_lttpr_aware = (bp_query_result == BP_RESULT_OK) && !!is_vbios_interop_enabled;
+		}
+	}
+
+	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
+		dc->debug = debug_defaults_drv;
+
+	// Init the vm_helper
+	if (dc->vm_helper)
+		vm_helper_init(dc->vm_helper, 16);
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	/* Clock Sources for Pixel Clock*/
+	pool->base.clock_sources[DCN30_CLK_SRC_PLL0] =
+			dcn30_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL0,
+				&clk_src_regs[0], false);
+	pool->base.clock_sources[DCN30_CLK_SRC_PLL1] =
+			dcn30_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL1,
+				&clk_src_regs[1], false);
+	pool->base.clock_sources[DCN30_CLK_SRC_PLL2] =
+			dcn30_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL2,
+				&clk_src_regs[2], false);
+	pool->base.clock_sources[DCN30_CLK_SRC_PLL3] =
+			dcn30_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL3,
+				&clk_src_regs[3], false);
+	pool->base.clock_sources[DCN30_CLK_SRC_PLL4] =
+			dcn30_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL4,
+				&clk_src_regs[4], false);
+	pool->base.clock_sources[DCN30_CLK_SRC_PLL5] =
+			dcn30_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL5,
+				&clk_src_regs[5], false);
+
+	pool->base.clk_src_count = DCN30_CLK_SRC_TOTAL;
+
+	/* todo: not reuse phy_pll registers */
+	pool->base.dp_clock_source =
+			dcn30_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_ID_DP_DTO,
+				&clk_src_regs[0], true);
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	/* DCCG */
+	pool->base.dccg = dccg30_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
+	if (pool->base.dccg == NULL) {
+		dm_error("DC: failed to create dccg!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* PP Lib and SMU interfaces */
+	init_soc_bounding_box(dc, pool);
+
+	num_pipes = dcn3_0_ip.max_num_dpp;
+
+	for (i = 0; i < dcn3_0_ip.max_num_dpp; i++)
+		if (pipe_fuses & 1 << i)
+			num_pipes--;
+
+	dcn3_0_ip.max_num_dpp = num_pipes;
+	dcn3_0_ip.max_num_otg = num_pipes;
+
+	dml_init_instance(&dc->dml, &dcn3_0_soc, &dcn3_0_ip, DML_PROJECT_DCN30);
+
+	/* IRQ */
+	init_data.ctx = dc->ctx;
+	pool->base.irqs = dal_irq_service_dcn30_create(&init_data);
+	if (!pool->base.irqs)
+		goto create_fail;
+
+	/* HUBBUB */
+	pool->base.hubbub = dcn30_hubbub_create(ctx);
+	if (pool->base.hubbub == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create hubbub!\n");
+		goto create_fail;
+	}
+
+	/* HUBPs, DPPs, OPPs and TGs */
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		pool->base.hubps[i] = dcn30_hubp_create(ctx, i);
+		if (pool->base.hubps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create hubps!\n");
+			goto create_fail;
+		}
+
+		pool->base.dpps[i] = dcn30_dpp_create(ctx, i);
+		if (pool->base.dpps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create dpps!\n");
+			goto create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		pool->base.opps[i] = dcn30_opp_create(ctx, i);
+		if (pool->base.opps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create output pixel processor!\n");
+			goto create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		pool->base.timing_generators[i] = dcn30_timing_generator_create(
+				ctx, i);
+		if (pool->base.timing_generators[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto create_fail;
+		}
+	}
+	pool->base.timing_generator_count = i;
+	/* PSR */
+	pool->base.psr = dmub_psr_create(ctx);
+
+	if (pool->base.psr == NULL) {
+		dm_error("DC: failed to create PSR obj!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* ABM */
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		pool->base.multiple_abms[i] = dmub_abm_create(ctx,
+				&abm_regs[i],
+				&abm_shift,
+				&abm_mask);
+		if (pool->base.multiple_abms[i] == NULL) {
+			dm_error("DC: failed to create abm for pipe %d!\n", i);
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+	/* MPC and DSC */
+	pool->base.mpc = dcn30_mpc_create(ctx, pool->base.mpcc_count, pool->base.res_cap->num_mpc_3dlut);
+	if (pool->base.mpc == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mpc!\n");
+		goto create_fail;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		pool->base.dscs[i] = dcn30_dsc_create(ctx, i);
+		if (pool->base.dscs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create display stream compressor %d!\n", i);
+			goto create_fail;
+		}
+	}
+
+	/* DWB and MMHUBBUB */
+	if (!dcn30_dwbc_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create dwbc!\n");
+		goto create_fail;
+	}
+
+	if (!dcn30_mmhubbub_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mcif_wb!\n");
+		goto create_fail;
+	}
+
+	/* AUX and I2C */
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.engines[i] = dcn30_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto create_fail;
+		}
+		pool->base.hw_i2cs[i] = dcn30_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create hw i2c!!\n");
+			goto create_fail;
+		}
+		pool->base.sw_i2cs[i] = NULL;
+	}
+
+	/* Audio, Stream Encoders including DIG and virtual, MPC 3D LUTs */
+	if (!resource_construct(num_virtual_links, dc, &pool->base,
+			&res_create_funcs))
+		goto create_fail;
+
+	/* HW Sequencer and Plane caps */
+	dcn30_hw_sequencer_construct(dc);
+
+	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->cap_funcs = cap_funcs;
+
+	if (dc->ctx->dc_bios->fw_info.oem_i2c_present) {
+		ddc_init_data.ctx = dc->ctx;
+		ddc_init_data.link = NULL;
+		ddc_init_data.id.id = dc->ctx->dc_bios->fw_info.oem_i2c_obj_id;
+		ddc_init_data.id.enum_id = 0;
+		ddc_init_data.id.type = OBJECT_TYPE_GENERIC;
+		pool->base.oem_device = dc->link_srv->create_ddc_service(&ddc_init_data);
+	} else {
+		pool->base.oem_device = NULL;
+	}
+
+	DC_FP_END();
+
+	return true;
+
+create_fail:
+
+	DC_FP_END();
+	dcn30_resource_destruct(pool);
+
+	return false;
+}
+
+struct resource_pool *dcn30_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc)
+{
+	struct dcn30_resource_pool *pool =
+		kzalloc(sizeof(struct dcn30_resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dcn30_resource_construct(init_data->num_virtual_links, dc, pool))
+		return &pool->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(pool);
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.h
new file mode 100644
index 000000000000..8e6b8b7368fd
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.h
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2020 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef _DCN30_RESOURCE_H_
+#define _DCN30_RESOURCE_H_
+
+#include "core_types.h"
+
+#define TO_DCN30_RES_POOL(pool)\
+	container_of(pool, struct dcn30_resource_pool, base)
+
+struct dc;
+struct resource_pool;
+struct _vcs_dpi_display_pipe_params_st;
+
+extern struct _vcs_dpi_ip_params_st dcn3_0_ip;
+extern struct _vcs_dpi_soc_bounding_box_st dcn3_0_soc;
+
+struct dcn30_resource_pool {
+	struct resource_pool base;
+};
+struct resource_pool *dcn30_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc);
+
+void dcn30_set_mcif_arb_params(
+		struct dc *dc,
+		struct dc_state *context,
+		display_e2e_pipe_params_st *pipes,
+		int pipe_cnt);
+
+unsigned int dcn30_calc_max_scaled_time(
+		unsigned int time_per_pixel,
+		enum mmhubbub_wbif_mode mode,
+		unsigned int urgent_watermark);
+
+bool dcn30_validate_bandwidth(struct dc *dc, struct dc_state *context,
+		bool fast_validate);
+bool dcn30_internal_validate_bw(
+		struct dc *dc,
+		struct dc_state *context,
+		display_e2e_pipe_params_st *pipes,
+		int *pipe_cnt_out,
+		int *vlevel_out,
+		bool fast_validate,
+		bool allow_self_refresh_only);
+void dcn30_calculate_wm_and_dlg(
+		struct dc *dc, struct dc_state *context,
+		display_e2e_pipe_params_st *pipes,
+		int pipe_cnt,
+		int vlevel);
+void dcn30_update_soc_for_wm_a(struct dc *dc, struct dc_state *context);
+void dcn30_populate_dml_writeback_from_context(
+		struct dc *dc, struct resource_context *res_ctx, display_e2e_pipe_params_st *pipes);
+
+int dcn30_populate_dml_pipes_from_context(
+	struct dc *dc, struct dc_state *context,
+	display_e2e_pipe_params_st *pipes,
+	bool fast_validate);
+
+bool dcn30_acquire_post_bldn_3dlut(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		int mpcc_id,
+		struct dc_3dlut **lut,
+		struct dc_transfer_func **shaper);
+
+bool dcn30_release_post_bldn_3dlut(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		struct dc_3dlut **lut,
+		struct dc_transfer_func **shaper);
+
+enum dc_status dcn30_add_stream_to_ctx(
+		struct dc *dc,
+		struct dc_state *new_ctx,
+		struct dc_stream_state *dc_stream);
+
+void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params);
+
+bool dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch(struct dc *dc, struct dc_state *context);
+void dcn30_setup_mclk_switch_using_fw_based_vblank_stretch(struct dc *dc, struct dc_state *context);
+int dcn30_find_dummy_latency_index_for_fw_based_mclk_switch(struct dc *dc, struct dc_state *context,
+		display_e2e_pipe_params_st *pipes, int pipe_cnt, int vlevel);
+
+#endif /* _DCN30_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn301/dcn301_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn301/dcn301_resource.c
new file mode 100644
index 000000000000..511ff6b5b985
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn301/dcn301_resource.c
@@ -0,0 +1,1728 @@
+/*
+ * Copyright 2019-2021 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+
+#include "dm_services.h"
+#include "dc.h"
+
+#include "dcn301/dcn301_init.h"
+
+#include "resource.h"
+#include "include/irq_service_interface.h"
+#include "dcn30/dcn30_resource.h"
+#include "dcn301_resource.h"
+
+#include "dcn20/dcn20_resource.h"
+
+#include "dcn10/dcn10_ipp.h"
+#include "dcn301/dcn301_hubbub.h"
+#include "dcn30/dcn30_mpc.h"
+#include "dcn30/dcn30_hubp.h"
+#include "irq/dcn30/irq_service_dcn30.h"
+#include "dcn30/dcn30_dpp.h"
+#include "dcn301/dcn301_optc.h"
+#include "dcn20/dcn20_hwseq.h"
+#include "dcn30/dcn30_hwseq.h"
+#include "dce110/dce110_hwseq.h"
+#include "dcn30/dcn30_opp.h"
+#include "dcn20/dcn20_dsc.h"
+#include "dcn30/dcn30_vpg.h"
+#include "dcn30/dcn30_afmt.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_hwseq.h"
+#include "clk_mgr.h"
+#include "virtual/virtual_stream_encoder.h"
+#include "dce110/dce110_resource.h"
+#include "dml/display_mode_vba.h"
+#include "dcn301/dcn301_dccg.h"
+#include "dcn10/dcn10_resource.h"
+#include "dcn30/dcn30_dio_stream_encoder.h"
+#include "dcn301/dcn301_dio_link_encoder.h"
+#include "dcn301/dcn301_panel_cntl.h"
+
+#include "vangogh_ip_offset.h"
+
+#include "dcn30/dcn30_dwb.h"
+#include "dcn30/dcn30_mmhubbub.h"
+
+#include "dcn/dcn_3_0_1_offset.h"
+#include "dcn/dcn_3_0_1_sh_mask.h"
+
+#include "nbio/nbio_7_2_0_offset.h"
+
+#include "dpcs/dpcs_3_0_0_offset.h"
+#include "dpcs/dpcs_3_0_0_sh_mask.h"
+
+#include "reg_helper.h"
+#include "dce/dmub_abm.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_i2c.h"
+
+#include "dml/dcn30/dcn30_fpu.h"
+
+#include "dml/dcn30/display_mode_vba_30.h"
+#include "dml/dcn301/dcn301_fpu.h"
+#include "vm_helper.h"
+#include "dcn20/dcn20_vmid.h"
+#include "amdgpu_socbb.h"
+
+#define TO_DCN301_RES_POOL(pool)\
+	container_of(pool, struct dcn301_resource_pool, base)
+
+#define DC_LOGGER \
+	dc->ctx->logger
+#define DC_LOGGER_INIT(logger)
+
+enum dcn301_clk_src_array_id {
+	DCN301_CLK_SRC_PLL0,
+	DCN301_CLK_SRC_PLL1,
+	DCN301_CLK_SRC_PLL2,
+	DCN301_CLK_SRC_PLL3,
+	DCN301_CLK_SRC_TOTAL
+};
+
+/* begin *********************
+ * macros to expend register list macro defined in HW object header file
+ */
+
+/* DCN */
+#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
+
+#define BASE(seg) BASE_INNER(seg)
+
+#define SR(reg_name)\
+		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) +  \
+					mm ## reg_name
+
+#define SRI(reg_name, block, id)\
+	.reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define SRI2(reg_name, block, id)\
+	.reg_name = BASE(mm ## reg_name ## _BASE_IDX) + \
+					mm ## reg_name
+
+#define SRIR(var_name, reg_name, block, id)\
+	.var_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define SRII(reg_name, block, id)\
+	.reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define SRII2(reg_name_pre, reg_name_post, id)\
+	.reg_name_pre ## _ ##  reg_name_post[id] = BASE(mm ## reg_name_pre \
+			## id ## _ ## reg_name_post ## _BASE_IDX) + \
+			mm ## reg_name_pre ## id ## _ ## reg_name_post
+
+#define SRII_MPC_RMU(reg_name, block, id)\
+	.RMU##_##reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define SRII_DWB(reg_name, temp_name, block, id)\
+	.reg_name[id] = BASE(mm ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## temp_name
+
+#define SF_DWB2(reg_name, block, id, field_name, post_fix)	\
+	.field_name = reg_name ## __ ## field_name ## post_fix
+
+#define DCCG_SRII(reg_name, block, id)\
+	.block ## _ ## reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## id ## _ ## reg_name
+
+#define VUPDATE_SRII(reg_name, block, id)\
+	.reg_name[id] = BASE(mm ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
+					mm ## reg_name ## _ ## block ## id
+
+/* NBIO */
+#define NBIO_BASE_INNER(seg) \
+	NBIO_BASE__INST0_SEG ## seg
+
+#define NBIO_BASE(seg) \
+	NBIO_BASE_INNER(seg)
+
+#define NBIO_SR(reg_name)\
+		.reg_name = NBIO_BASE(regBIF_BX0_ ## reg_name ## _BASE_IDX) + \
+					regBIF_BX0_ ## reg_name
+
+/* MMHUB */
+#define MMHUB_BASE_INNER(seg) \
+	MMHUB_BASE__INST0_SEG ## seg
+
+#define MMHUB_BASE(seg) \
+	MMHUB_BASE_INNER(seg)
+
+#define MMHUB_SR(reg_name)\
+		.reg_name = MMHUB_BASE(regMM ## reg_name ## _BASE_IDX) + \
+					regMM ## reg_name
+
+/* CLOCK */
+#define CLK_BASE_INNER(seg) \
+	CLK_BASE__INST0_SEG ## seg
+
+#define CLK_BASE(seg) \
+	CLK_BASE_INNER(seg)
+
+#define CLK_SRI(reg_name, block, inst)\
+	.reg_name = CLK_BASE(mm ## block ## _ ## inst ## _ ## reg_name ## _BASE_IDX) + \
+					mm ## block ## _ ## inst ## _ ## reg_name
+
+static const struct bios_registers bios_regs = {
+		NBIO_SR(BIOS_SCRATCH_3),
+		NBIO_SR(BIOS_SCRATCH_6)
+};
+
+#define clk_src_regs(index, pllid)\
+[index] = {\
+	CS_COMMON_REG_LIST_DCN3_01(index, pllid),\
+}
+
+static const struct dce110_clk_src_regs clk_src_regs[] = {
+	clk_src_regs(0, A),
+	clk_src_regs(1, B),
+	clk_src_regs(2, C),
+	clk_src_regs(3, D)
+};
+
+static const struct dce110_clk_src_shift cs_shift = {
+		CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
+};
+
+static const struct dce110_clk_src_mask cs_mask = {
+		CS_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
+};
+
+#define abm_regs(id)\
+[id] = {\
+		ABM_DCN301_REG_LIST(id)\
+}
+
+static const struct dce_abm_registers abm_regs[] = {
+		abm_regs(0),
+		abm_regs(1),
+		abm_regs(2),
+		abm_regs(3),
+};
+
+static const struct dce_abm_shift abm_shift = {
+		ABM_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dce_abm_mask abm_mask = {
+		ABM_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define audio_regs(id)\
+[id] = {\
+		AUD_COMMON_REG_LIST(id)\
+}
+
+static const struct dce_audio_registers audio_regs[] = {
+	audio_regs(0),
+	audio_regs(1),
+	audio_regs(2),
+	audio_regs(3),
+	audio_regs(4),
+	audio_regs(5),
+	audio_regs(6)
+};
+
+#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
+		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
+
+static const struct dce_audio_shift audio_shift = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_audio_mask audio_mask = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+#define vpg_regs(id)\
+[id] = {\
+	VPG_DCN3_REG_LIST(id)\
+}
+
+static const struct dcn30_vpg_registers vpg_regs[] = {
+	vpg_regs(0),
+	vpg_regs(1),
+	vpg_regs(2),
+	vpg_regs(3),
+};
+
+static const struct dcn30_vpg_shift vpg_shift = {
+	DCN3_VPG_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn30_vpg_mask vpg_mask = {
+	DCN3_VPG_MASK_SH_LIST(_MASK)
+};
+
+#define afmt_regs(id)\
+[id] = {\
+	AFMT_DCN3_REG_LIST(id)\
+}
+
+static const struct dcn30_afmt_registers afmt_regs[] = {
+	afmt_regs(0),
+	afmt_regs(1),
+	afmt_regs(2),
+	afmt_regs(3),
+};
+
+static const struct dcn30_afmt_shift afmt_shift = {
+	DCN3_AFMT_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn30_afmt_mask afmt_mask = {
+	DCN3_AFMT_MASK_SH_LIST(_MASK)
+};
+
+#define stream_enc_regs(id)\
+[id] = {\
+	SE_DCN3_REG_LIST(id)\
+}
+
+static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
+	stream_enc_regs(0),
+	stream_enc_regs(1),
+	stream_enc_regs(2),
+	stream_enc_regs(3),
+};
+
+static const struct dcn10_stream_encoder_shift se_shift = {
+		SE_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn10_stream_encoder_mask se_mask = {
+		SE_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+
+#define aux_regs(id)\
+[id] = {\
+	DCN2_AUX_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
+		aux_regs(0),
+		aux_regs(1),
+		aux_regs(2),
+		aux_regs(3),
+};
+
+#define hpd_regs(id)\
+[id] = {\
+	HPD_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
+		hpd_regs(0),
+		hpd_regs(1),
+		hpd_regs(2),
+		hpd_regs(3),
+};
+
+
+#define link_regs(id, phyid)\
+[id] = {\
+	LE_DCN301_REG_LIST(id), \
+	UNIPHY_DCN2_REG_LIST(phyid), \
+	DPCS_DCN2_REG_LIST(id), \
+	SRI(DP_DPHY_INTERNAL_CTRL, DP, id) \
+}
+
+static const struct dce110_aux_registers_shift aux_shift = {
+	DCN_AUX_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+	DCN_AUX_MASK_SH_LIST(_MASK)
+};
+
+static const struct dcn10_link_enc_registers link_enc_regs[] = {
+	link_regs(0, A),
+	link_regs(1, B),
+	link_regs(2, C),
+	link_regs(3, D),
+};
+
+static const struct dcn10_link_enc_shift le_shift = {
+	LINK_ENCODER_MASK_SH_LIST_DCN301(__SHIFT),\
+	DPCS_DCN2_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn10_link_enc_mask le_mask = {
+	LINK_ENCODER_MASK_SH_LIST_DCN301(_MASK),\
+	DPCS_DCN2_MASK_SH_LIST(_MASK)
+};
+
+#define panel_cntl_regs(id)\
+[id] = {\
+	DCN301_PANEL_CNTL_REG_LIST(id),\
+}
+
+static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
+	panel_cntl_regs(0),
+	panel_cntl_regs(1),
+};
+
+static const struct dcn301_panel_cntl_shift panel_cntl_shift = {
+	DCN301_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn301_panel_cntl_mask panel_cntl_mask = {
+	DCN301_PANEL_CNTL_MASK_SH_LIST(_MASK)
+};
+
+#define dpp_regs(id)\
+[id] = {\
+	DPP_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn3_dpp_registers dpp_regs[] = {
+	dpp_regs(0),
+	dpp_regs(1),
+	dpp_regs(2),
+	dpp_regs(3),
+};
+
+static const struct dcn3_dpp_shift tf_shift = {
+		DPP_REG_LIST_SH_MASK_DCN30(__SHIFT)
+};
+
+static const struct dcn3_dpp_mask tf_mask = {
+		DPP_REG_LIST_SH_MASK_DCN30(_MASK)
+};
+
+#define opp_regs(id)\
+[id] = {\
+	OPP_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn20_opp_registers opp_regs[] = {
+	opp_regs(0),
+	opp_regs(1),
+	opp_regs(2),
+	opp_regs(3),
+};
+
+static const struct dcn20_opp_shift opp_shift = {
+	OPP_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn20_opp_mask opp_mask = {
+	OPP_MASK_SH_LIST_DCN20(_MASK)
+};
+
+#define aux_engine_regs(id)\
+[id] = {\
+	AUX_COMMON_REG_LIST0(id), \
+	.AUXN_IMPCAL = 0, \
+	.AUXP_IMPCAL = 0, \
+	.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1),
+		aux_engine_regs(2),
+		aux_engine_regs(3),
+};
+
+#define dwbc_regs_dcn3(id)\
+[id] = {\
+	DWBC_COMMON_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn30_dwbc_registers dwbc30_regs[] = {
+	dwbc_regs_dcn3(0),
+};
+
+static const struct dcn30_dwbc_shift dwbc30_shift = {
+	DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_dwbc_mask dwbc30_mask = {
+	DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define mcif_wb_regs_dcn3(id)\
+[id] = {\
+	MCIF_WB_COMMON_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn30_mmhubbub_registers mcif_wb30_regs[] = {
+	mcif_wb_regs_dcn3(0)
+};
+
+static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define dsc_regsDCN20(id)\
+[id] = {\
+	DSC_REG_LIST_DCN20(id)\
+}
+
+static const struct dcn20_dsc_registers dsc_regs[] = {
+	dsc_regsDCN20(0),
+	dsc_regsDCN20(1),
+	dsc_regsDCN20(2),
+};
+
+static const struct dcn20_dsc_shift dsc_shift = {
+	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
+};
+
+static const struct dcn20_dsc_mask dsc_mask = {
+	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
+};
+
+static const struct dcn30_mpc_registers mpc_regs = {
+		MPC_REG_LIST_DCN3_0(0),
+		MPC_REG_LIST_DCN3_0(1),
+		MPC_REG_LIST_DCN3_0(2),
+		MPC_REG_LIST_DCN3_0(3),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(0),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(1),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(2),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(3),
+		MPC_RMU_GLOBAL_REG_LIST_DCN3AG,
+		MPC_RMU_REG_LIST_DCN3AG(0),
+		MPC_RMU_REG_LIST_DCN3AG(1),
+		MPC_DWB_MUX_REG_LIST_DCN3_0(0),
+};
+
+static const struct dcn30_mpc_shift mpc_shift = {
+	MPC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_mpc_mask mpc_mask = {
+	MPC_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define optc_regs(id)\
+[id] = {OPTC_COMMON_REG_LIST_DCN3_0(id)}
+
+
+static const struct dcn_optc_registers optc_regs[] = {
+	optc_regs(0),
+	optc_regs(1),
+	optc_regs(2),
+	optc_regs(3),
+};
+
+static const struct dcn_optc_shift optc_shift = {
+	OPTC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn_optc_mask optc_mask = {
+	OPTC_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define hubp_regs(id)\
+[id] = {\
+	HUBP_REG_LIST_DCN30(id)\
+}
+
+static const struct dcn_hubp2_registers hubp_regs[] = {
+		hubp_regs(0),
+		hubp_regs(1),
+		hubp_regs(2),
+		hubp_regs(3),
+};
+
+static const struct dcn_hubp2_shift hubp_shift = {
+		HUBP_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn_hubp2_mask hubp_mask = {
+		HUBP_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static const struct dcn_hubbub_registers hubbub_reg = {
+		HUBBUB_REG_LIST_DCN301(0)
+};
+
+static const struct dcn_hubbub_shift hubbub_shift = {
+		HUBBUB_MASK_SH_LIST_DCN301(__SHIFT)
+};
+
+static const struct dcn_hubbub_mask hubbub_mask = {
+		HUBBUB_MASK_SH_LIST_DCN301(_MASK)
+};
+
+static const struct dccg_registers dccg_regs = {
+		DCCG_REG_LIST_DCN301()
+};
+
+static const struct dccg_shift dccg_shift = {
+		DCCG_MASK_SH_LIST_DCN301(__SHIFT)
+};
+
+static const struct dccg_mask dccg_mask = {
+		DCCG_MASK_SH_LIST_DCN301(_MASK)
+};
+
+static const struct dce_hwseq_registers hwseq_reg = {
+		HWSEQ_DCN301_REG_LIST()
+};
+
+static const struct dce_hwseq_shift hwseq_shift = {
+		HWSEQ_DCN301_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+		HWSEQ_DCN301_MASK_SH_LIST(_MASK)
+};
+#define vmid_regs(id)\
+[id] = {\
+		DCN20_VMID_REG_LIST(id)\
+}
+
+static const struct dcn_vmid_registers vmid_regs[] = {
+	vmid_regs(0),
+	vmid_regs(1),
+	vmid_regs(2),
+	vmid_regs(3),
+	vmid_regs(4),
+	vmid_regs(5),
+	vmid_regs(6),
+	vmid_regs(7),
+	vmid_regs(8),
+	vmid_regs(9),
+	vmid_regs(10),
+	vmid_regs(11),
+	vmid_regs(12),
+	vmid_regs(13),
+	vmid_regs(14),
+	vmid_regs(15)
+};
+
+static const struct dcn20_vmid_shift vmid_shifts = {
+		DCN20_VMID_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn20_vmid_mask vmid_masks = {
+		DCN20_VMID_MASK_SH_LIST(_MASK)
+};
+
+static struct resource_caps res_cap_dcn301 = {
+	.num_timing_generator = 4,
+	.num_opp = 4,
+	.num_video_plane = 4,
+	.num_audio = 4,
+	.num_stream_encoder = 4,
+	.num_pll = 4,
+	.num_dwb = 1,
+	.num_ddc = 4,
+	.num_vmid = 16,
+	.num_mpc_3dlut = 2,
+	.num_dsc = 3,
+};
+
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
+	.per_pixel_alpha = true,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = true,
+			.fp16 = true,
+			.p010 = true,
+			.ayuv = false,
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 16000,
+			.fp16 = 16000
+	},
+
+	/* 6:1 downscaling ratio: 1000/6 = 166.666 */
+	.max_downscale_factor = {
+			.argb8888 = 167,
+			.nv12 = 167,
+			.fp16 = 167 
+	},
+	64,
+	64
+};
+
+static const struct dc_debug_options debug_defaults_drv = {
+	.disable_dmcu = true,
+	.force_abm_enable = false,
+	.timing_trace = false,
+	.clock_trace = true,
+	.disable_dpp_power_gate = false,
+	.disable_hubp_power_gate = false,
+	.disable_clock_gate = true,
+	.disable_pplib_clock_request = true,
+	.disable_pplib_wm_range = true,
+	.pipe_split_policy = MPC_SPLIT_DYNAMIC,
+	.force_single_disp_pipe_split = false,
+	.disable_dcc = DCC_ENABLE,
+	.vsr_support = true,
+	.performance_trace = false,
+	.max_downscale_src_width = 7680,/*upto 8K*/
+	.scl_reset_length10 = true,
+	.sanity_checks = false,
+	.underflow_assert_delay_us = 0xFFFFFFFF,
+	.dwb_fi_phase = -1, // -1 = disable
+	.dmub_command_table = true,
+	.use_max_lb = false,
+	.exit_idle_opt_for_cursor_updates = true,
+	.using_dml2 = false,
+};
+
+static void dcn301_dpp_destroy(struct dpp **dpp)
+{
+	kfree(TO_DCN20_DPP(*dpp));
+	*dpp = NULL;
+}
+
+static struct dpp *dcn301_dpp_create(struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn3_dpp *dpp =
+		kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
+
+	if (!dpp)
+		return NULL;
+
+	if (dpp3_construct(dpp, ctx, inst,
+			&dpp_regs[inst], &tf_shift, &tf_mask))
+		return &dpp->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(dpp);
+	return NULL;
+}
+static struct output_pixel_processor *dcn301_opp_create(struct dc_context *ctx,
+							uint32_t inst)
+{
+	struct dcn20_opp *opp =
+		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
+
+	if (!opp) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dcn20_opp_construct(opp, ctx, inst,
+			&opp_regs[inst], &opp_shift, &opp_mask);
+	return &opp->base;
+}
+
+static struct dce_aux *dcn301_aux_engine_create(struct dc_context *ctx, uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst],
+					&aux_mask,
+					&aux_shift,
+					ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
+
+static const struct dce_i2c_registers i2c_hw_regs[] = {
+		i2c_inst_regs(1),
+		i2c_inst_regs(2),
+		i2c_inst_regs(3),
+		i2c_inst_regs(4),
+};
+
+static const struct dce_i2c_shift i2c_shifts = {
+		I2C_COMMON_MASK_SH_LIST_DCN2(__SHIFT)
+};
+
+static const struct dce_i2c_mask i2c_masks = {
+		I2C_COMMON_MASK_SH_LIST_DCN2(_MASK)
+};
+
+static struct dce_i2c_hw *dcn301_i2c_hw_create(struct dc_context *ctx, uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw =
+		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
+				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+static struct mpc *dcn301_mpc_create(
+		struct dc_context *ctx,
+		int num_mpcc,
+		int num_rmu)
+{
+	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc),
+					  GFP_KERNEL);
+
+	if (!mpc30)
+		return NULL;
+
+	dcn30_mpc_construct(mpc30, ctx,
+			&mpc_regs,
+			&mpc_shift,
+			&mpc_mask,
+			num_mpcc,
+			num_rmu);
+
+	return &mpc30->base;
+}
+
+static struct hubbub *dcn301_hubbub_create(struct dc_context *ctx)
+{
+	int i;
+
+	struct dcn20_hubbub *hubbub3 = kzalloc(sizeof(struct dcn20_hubbub),
+					  GFP_KERNEL);
+
+	if (!hubbub3)
+		return NULL;
+
+	hubbub301_construct(hubbub3, ctx,
+			&hubbub_reg,
+			&hubbub_shift,
+			&hubbub_mask);
+
+
+	for (i = 0; i < res_cap_dcn301.num_vmid; i++) {
+		struct dcn20_vmid *vmid = &hubbub3->vmid[i];
+
+		vmid->ctx = ctx;
+
+		vmid->regs = &vmid_regs[i];
+		vmid->shifts = &vmid_shifts;
+		vmid->masks = &vmid_masks;
+	}
+
+	hubbub3->num_vmid = res_cap_dcn301.num_vmid;
+
+	return &hubbub3->base;
+}
+
+static struct timing_generator *dcn301_timing_generator_create(
+	struct dc_context *ctx, uint32_t instance)
+{
+	struct optc *tgn10 =
+		kzalloc(sizeof(struct optc), GFP_KERNEL);
+
+	if (!tgn10)
+		return NULL;
+
+	tgn10->base.inst = instance;
+	tgn10->base.ctx = ctx;
+
+	tgn10->tg_regs = &optc_regs[instance];
+	tgn10->tg_shift = &optc_shift;
+	tgn10->tg_mask = &optc_mask;
+
+	dcn301_timing_generator_init(tgn10);
+
+	return &tgn10->base;
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+		.max_hdmi_deep_color = COLOR_DEPTH_121212,
+		.max_hdmi_pixel_clock = 600000,
+		.hdmi_ycbcr420_supported = true,
+		.dp_ycbcr420_supported = true,
+		.fec_supported = true,
+		.flags.bits.IS_HBR2_CAPABLE = true,
+		.flags.bits.IS_HBR3_CAPABLE = true,
+		.flags.bits.IS_TPS3_CAPABLE = true,
+		.flags.bits.IS_TPS4_CAPABLE = true
+};
+
+static struct link_encoder *dcn301_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dcn20_link_encoder *enc20 =
+		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+
+	if (!enc20)
+		return NULL;
+
+	dcn301_link_encoder_construct(enc20,
+			enc_init_data,
+			&link_enc_feature,
+			&link_enc_regs[enc_init_data->transmitter],
+			&link_enc_aux_regs[enc_init_data->channel - 1],
+			&link_enc_hpd_regs[enc_init_data->hpd_source],
+			&le_shift,
+			&le_mask);
+
+	return &enc20->enc10.base;
+}
+
+static struct panel_cntl *dcn301_panel_cntl_create(const struct panel_cntl_init_data *init_data)
+{
+	struct dcn301_panel_cntl *panel_cntl =
+		kzalloc(sizeof(struct dcn301_panel_cntl), GFP_KERNEL);
+
+	if (!panel_cntl)
+		return NULL;
+
+	dcn301_panel_cntl_construct(panel_cntl,
+			init_data,
+			&panel_cntl_regs[init_data->inst],
+			&panel_cntl_shift,
+			&panel_cntl_mask);
+
+	return &panel_cntl->base;
+}
+
+
+#define CTX ctx
+
+#define REG(reg_name) \
+	(DCN_BASE.instance[0].segment[mm ## reg_name ## _BASE_IDX] + mm ## reg_name)
+
+static uint32_t read_pipe_fuses(struct dc_context *ctx)
+{
+	uint32_t value = REG_READ(CC_DC_PIPE_DIS);
+	/* RV1 support max 4 pipes */
+	value = value & 0xf;
+	return value;
+}
+
+
+static void read_dce_straps(
+	struct dc_context *ctx,
+	struct resource_straps *straps)
+{
+	generic_reg_get(ctx, mmDC_PINSTRAPS + BASE(mmDC_PINSTRAPS_BASE_IDX),
+		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
+
+}
+
+static struct audio *dcn301_create_audio(
+		struct dc_context *ctx, unsigned int inst)
+{
+	return dce_audio_create(ctx, inst,
+			&audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+static struct vpg *dcn301_vpg_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn30_vpg *vpg3 = kzalloc(sizeof(struct dcn30_vpg), GFP_KERNEL);
+
+	if (!vpg3)
+		return NULL;
+
+	vpg3_construct(vpg3, ctx, inst,
+			&vpg_regs[inst],
+			&vpg_shift,
+			&vpg_mask);
+
+	return &vpg3->base;
+}
+
+static struct afmt *dcn301_afmt_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn30_afmt *afmt3 = kzalloc(sizeof(struct dcn30_afmt), GFP_KERNEL);
+
+	if (!afmt3)
+		return NULL;
+
+	afmt3_construct(afmt3, ctx, inst,
+			&afmt_regs[inst],
+			&afmt_shift,
+			&afmt_mask);
+
+	return &afmt3->base;
+}
+
+static struct stream_encoder *dcn301_stream_encoder_create(enum engine_id eng_id,
+							   struct dc_context *ctx)
+{
+	struct dcn10_stream_encoder *enc1;
+	struct vpg *vpg;
+	struct afmt *afmt;
+	int vpg_inst;
+	int afmt_inst;
+
+	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
+	if (eng_id <= ENGINE_ID_DIGF) {
+		vpg_inst = eng_id;
+		afmt_inst = eng_id;
+	} else
+		return NULL;
+
+	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
+	vpg = dcn301_vpg_create(ctx, vpg_inst);
+	afmt = dcn301_afmt_create(ctx, afmt_inst);
+
+	if (!enc1 || !vpg || !afmt) {
+		kfree(enc1);
+		kfree(vpg);
+		kfree(afmt);
+		return NULL;
+	}
+
+	dcn30_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
+					eng_id, vpg, afmt,
+					&stream_enc_regs[eng_id],
+					&se_shift, &se_mask);
+
+	return &enc1->base;
+}
+
+static struct dce_hwseq *dcn301_hwseq_create(struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &hwseq_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+	}
+	return hws;
+}
+static const struct resource_create_funcs res_create_funcs = {
+	.read_dce_straps = read_dce_straps,
+	.create_audio = dcn301_create_audio,
+	.create_stream_encoder = dcn301_stream_encoder_create,
+	.create_hwseq = dcn301_hwseq_create,
+};
+
+static void dcn301_destruct(struct dcn301_resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->base.stream_enc_count; i++) {
+		if (pool->base.stream_enc[i] != NULL) {
+			if (pool->base.stream_enc[i]->vpg != NULL) {
+				kfree(DCN30_VPG_FROM_VPG(pool->base.stream_enc[i]->vpg));
+				pool->base.stream_enc[i]->vpg = NULL;
+			}
+			if (pool->base.stream_enc[i]->afmt != NULL) {
+				kfree(DCN30_AFMT_FROM_AFMT(pool->base.stream_enc[i]->afmt));
+				pool->base.stream_enc[i]->afmt = NULL;
+			}
+			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
+			pool->base.stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		if (pool->base.dscs[i] != NULL)
+			dcn20_dsc_destroy(&pool->base.dscs[i]);
+	}
+
+	if (pool->base.mpc != NULL) {
+		kfree(TO_DCN20_MPC(pool->base.mpc));
+		pool->base.mpc = NULL;
+	}
+	if (pool->base.hubbub != NULL) {
+		kfree(pool->base.hubbub);
+		pool->base.hubbub = NULL;
+	}
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (pool->base.dpps[i] != NULL)
+			dcn301_dpp_destroy(&pool->base.dpps[i]);
+
+		if (pool->base.ipps[i] != NULL)
+			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
+
+		if (pool->base.hubps[i] != NULL) {
+			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
+			pool->base.hubps[i] = NULL;
+		}
+
+		if (pool->base.irqs != NULL) {
+			dal_irq_service_destroy(&pool->base.irqs);
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+		if (pool->base.hw_i2cs[i] != NULL) {
+			kfree(pool->base.hw_i2cs[i]);
+			pool->base.hw_i2cs[i] = NULL;
+		}
+		if (pool->base.sw_i2cs[i] != NULL) {
+			kfree(pool->base.sw_i2cs[i]);
+			pool->base.sw_i2cs[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		if (pool->base.opps[i] != NULL)
+			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.timing_generators[i] != NULL)	{
+			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
+			pool->base.timing_generators[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+		if (pool->base.dwbc[i] != NULL) {
+			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
+			pool->base.dwbc[i] = NULL;
+		}
+		if (pool->base.mcif_wb[i] != NULL) {
+			kfree(TO_DCN30_MMHUBBUB(pool->base.mcif_wb[i]));
+			pool->base.mcif_wb[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.audio_count; i++) {
+		if (pool->base.audios[i])
+			dce_aud_destroy(&pool->base.audios[i]);
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] != NULL) {
+			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
+			pool->base.clock_sources[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+		if (pool->base.mpc_lut[i] != NULL) {
+			dc_3dlut_func_release(pool->base.mpc_lut[i]);
+			pool->base.mpc_lut[i] = NULL;
+		}
+		if (pool->base.mpc_shaper[i] != NULL) {
+			dc_transfer_func_release(pool->base.mpc_shaper[i]);
+			pool->base.mpc_shaper[i] = NULL;
+		}
+	}
+
+	if (pool->base.dp_clock_source != NULL) {
+		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
+		pool->base.dp_clock_source = NULL;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.multiple_abms[i] != NULL)
+			dce_abm_destroy(&pool->base.multiple_abms[i]);
+	}
+
+	if (pool->base.dccg != NULL)
+		dcn_dccg_destroy(&pool->base.dccg);
+}
+
+static struct hubp *dcn301_hubp_create(struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_hubp *hubp2 =
+		kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
+
+	if (!hubp2)
+		return NULL;
+
+	if (hubp3_construct(hubp2, ctx, inst,
+			&hubp_regs[inst], &hubp_shift, &hubp_mask))
+		return &hubp2->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(hubp2);
+	return NULL;
+}
+
+static bool dcn301_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc),
+						    GFP_KERNEL);
+
+		if (!dwbc30) {
+			dm_error("DC: failed to create dwbc30!\n");
+			return false;
+		}
+
+		dcn30_dwbc_construct(dwbc30, ctx,
+				&dwbc30_regs[i],
+				&dwbc30_shift,
+				&dwbc30_mask,
+				i);
+
+		pool->dwbc[i] = &dwbc30->base;
+	}
+	return true;
+}
+
+static bool dcn301_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub),
+						    GFP_KERNEL);
+
+		if (!mcif_wb30) {
+			dm_error("DC: failed to create mcif_wb30!\n");
+			return false;
+		}
+
+		dcn30_mmhubbub_construct(mcif_wb30, ctx,
+				&mcif_wb30_regs[i],
+				&mcif_wb30_shift,
+				&mcif_wb30_mask,
+				i);
+
+		pool->mcif_wb[i] = &mcif_wb30->base;
+	}
+	return true;
+}
+
+static struct display_stream_compressor *dcn301_dsc_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_dsc *dsc =
+		kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
+
+	if (!dsc) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
+	return &dsc->base;
+}
+
+
+static void dcn301_destroy_resource_pool(struct resource_pool **pool)
+{
+	struct dcn301_resource_pool *dcn301_pool = TO_DCN301_RES_POOL(*pool);
+
+	dcn301_destruct(dcn301_pool);
+	kfree(dcn301_pool);
+	*pool = NULL;
+}
+
+static struct clock_source *dcn301_clock_source_create(
+		struct dc_context *ctx,
+		struct dc_bios *bios,
+		enum clock_source_id id,
+		const struct dce110_clk_src_regs *regs,
+		bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dcn301_clk_src_construct(clk_src, ctx, bios, id,
+			regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	kfree(clk_src);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static struct dc_cap_funcs cap_funcs = {
+	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
+};
+
+
+static bool is_soc_bounding_box_valid(struct dc *dc)
+{
+	uint32_t hw_internal_rev = dc->ctx->asic_id.hw_internal_rev;
+
+	if (ASICREV_IS_VANGOGH(hw_internal_rev))
+		return true;
+
+	return false;
+}
+
+static bool init_soc_bounding_box(struct dc *dc,
+				  struct dcn301_resource_pool *pool)
+{
+	struct _vcs_dpi_soc_bounding_box_st *loaded_bb = &dcn3_01_soc;
+	struct _vcs_dpi_ip_params_st *loaded_ip = &dcn3_01_ip;
+
+	DC_LOGGER_INIT(dc->ctx->logger);
+
+	if (!is_soc_bounding_box_valid(dc)) {
+		DC_LOG_ERROR("%s: not valid soc bounding box\n", __func__);
+		return false;
+	}
+
+	loaded_ip->max_num_otg = pool->base.res_cap->num_timing_generator;
+	loaded_ip->max_num_dpp = pool->base.pipe_count;
+	DC_FP_START();
+	dcn20_patch_bounding_box(dc, loaded_bb);
+	DC_FP_END();
+
+	if (dc->ctx->dc_bios->funcs->get_soc_bb_info) {
+		struct bp_soc_bb_info bb_info = {0};
+
+		if (dc->ctx->dc_bios->funcs->get_soc_bb_info(dc->ctx->dc_bios, &bb_info) == BP_RESULT_OK) {
+			DC_FP_START();
+			dcn301_fpu_init_soc_bounding_box(bb_info);
+			DC_FP_END();
+		}
+	}
+
+	return true;
+}
+
+
+static void set_wm_ranges(
+		struct pp_smu_funcs *pp_smu,
+		struct _vcs_dpi_soc_bounding_box_st *loaded_bb)
+{
+	struct pp_smu_wm_range_sets ranges = {0};
+	int i;
+
+	ranges.num_reader_wm_sets = 0;
+
+	if (loaded_bb->num_states == 1) {
+		ranges.reader_wm_sets[0].wm_inst = 0;
+		ranges.reader_wm_sets[0].min_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
+		ranges.reader_wm_sets[0].max_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
+		ranges.reader_wm_sets[0].min_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
+		ranges.reader_wm_sets[0].max_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
+
+		ranges.num_reader_wm_sets = 1;
+	} else if (loaded_bb->num_states > 1) {
+		for (i = 0; i < 4 && i < loaded_bb->num_states; i++) {
+			ranges.reader_wm_sets[i].wm_inst = i;
+			ranges.reader_wm_sets[i].min_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
+			ranges.reader_wm_sets[i].max_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
+			DC_FP_START();
+			dcn301_fpu_set_wm_ranges(i, &ranges, loaded_bb);
+			DC_FP_END();
+			ranges.num_reader_wm_sets = i + 1;
+		}
+
+		ranges.reader_wm_sets[0].min_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
+		ranges.reader_wm_sets[ranges.num_reader_wm_sets - 1].max_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
+	}
+
+	ranges.num_writer_wm_sets = 1;
+
+	ranges.writer_wm_sets[0].wm_inst = 0;
+	ranges.writer_wm_sets[0].min_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
+	ranges.writer_wm_sets[0].max_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
+	ranges.writer_wm_sets[0].min_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
+	ranges.writer_wm_sets[0].max_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
+
+	/* Notify PP Lib/SMU which Watermarks to use for which clock ranges */
+	pp_smu->nv_funcs.set_wm_ranges(&pp_smu->nv_funcs.pp_smu, &ranges);
+}
+
+static void dcn301_calculate_wm_and_dlg(
+		struct dc *dc, struct dc_state *context,
+		display_e2e_pipe_params_st *pipes,
+		int pipe_cnt,
+		int vlevel)
+{
+	DC_FP_START();
+	dcn301_calculate_wm_and_dlg_fp(dc, context, pipes, pipe_cnt, vlevel);
+	DC_FP_END();
+}
+
+static struct resource_funcs dcn301_res_pool_funcs = {
+	.destroy = dcn301_destroy_resource_pool,
+	.link_enc_create = dcn301_link_encoder_create,
+	.panel_cntl_create = dcn301_panel_cntl_create,
+	.validate_bandwidth = dcn30_validate_bandwidth,
+	.calculate_wm_and_dlg = dcn301_calculate_wm_and_dlg,
+	.update_soc_for_wm_a = dcn30_update_soc_for_wm_a,
+	.populate_dml_pipes = dcn30_populate_dml_pipes_from_context,
+	.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
+	.release_pipe = dcn20_release_pipe,
+	.add_stream_to_ctx = dcn30_add_stream_to_ctx,
+	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
+	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
+	.populate_dml_writeback_from_context = dcn30_populate_dml_writeback_from_context,
+	.set_mcif_arb_params = dcn30_set_mcif_arb_params,
+	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
+	.acquire_post_bldn_3dlut = dcn30_acquire_post_bldn_3dlut,
+	.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
+	.update_bw_bounding_box = dcn301_update_bw_bounding_box,
+	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state
+};
+
+static bool dcn301_resource_construct(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct dcn301_resource_pool *pool)
+{
+	int i, j;
+	struct dc_context *ctx = dc->ctx;
+	struct irq_service_init_data init_data;
+	uint32_t pipe_fuses = read_pipe_fuses(ctx);
+	uint32_t num_pipes = 0;
+
+	DC_LOGGER_INIT(dc->ctx->logger);
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	if (dc->ctx->asic_id.chip_id == DEVICE_ID_VGH_1435)
+		res_cap_dcn301.num_pll = 2;
+	pool->base.res_cap = &res_cap_dcn301;
+
+	pool->base.funcs = &dcn301_res_pool_funcs;
+
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
+	pool->base.mpcc_count = pool->base.res_cap->num_timing_generator;
+	dc->caps.max_downscale_ratio = 600;
+	dc->caps.i2c_speed_in_khz = 100;
+	dc->caps.i2c_speed_in_khz_hdcp = 5; /*1.4 w/a enabled by default*/
+	dc->caps.max_cursor_size = 256;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.dmdata_alloc_size = 2048;
+	dc->caps.max_slave_planes = 2;
+	dc->caps.max_slave_yuv_planes = 2;
+	dc->caps.max_slave_rgb_planes = 2;
+	dc->caps.is_apu = true;
+	dc->caps.post_blend_color_processing = true;
+	dc->caps.force_dp_tps4_for_cp2520 = true;
+	dc->caps.extended_aux_timeout_support = true;
+	dc->caps.dmcub_support = true;
+
+	/* Color pipeline capabilities */
+	dc->caps.color.dpp.dcn_arch = 1;
+	dc->caps.color.dpp.input_lut_shared = 0;
+	dc->caps.color.dpp.icsc = 1;
+	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
+	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
+	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
+	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
+	dc->caps.color.dpp.post_csc = 1;
+	dc->caps.color.dpp.gamma_corr = 1;
+	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
+
+	dc->caps.color.dpp.hw_3d_lut = 1;
+	dc->caps.color.dpp.ogam_ram = 1;
+	// no OGAM ROM on DCN301
+	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
+	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
+	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
+	dc->caps.color.dpp.ocsc = 0;
+
+	dc->caps.color.mpc.gamut_remap = 1;
+	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //2
+	dc->caps.color.mpc.ogam_ram = 1;
+	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
+	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
+	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
+	dc->caps.color.mpc.ocsc = 1;
+
+	dc->caps.dp_hdmi21_pcon_support = true;
+
+	/* read VBIOS LTTPR caps */
+	if (ctx->dc_bios->funcs->get_lttpr_caps) {
+		enum bp_result bp_query_result;
+		uint8_t is_vbios_lttpr_enable = 0;
+
+		bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
+		dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
+	}
+
+	if (ctx->dc_bios->funcs->get_lttpr_interop) {
+		enum bp_result bp_query_result;
+		uint8_t is_vbios_interop_enabled = 0;
+
+		bp_query_result = ctx->dc_bios->funcs->get_lttpr_interop(ctx->dc_bios, &is_vbios_interop_enabled);
+		dc->caps.vbios_lttpr_aware = (bp_query_result == BP_RESULT_OK) && !!is_vbios_interop_enabled;
+	}
+
+	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
+		dc->debug = debug_defaults_drv;
+
+	// Init the vm_helper
+	if (dc->vm_helper)
+		vm_helper_init(dc->vm_helper, 16);
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	/* Clock Sources for Pixel Clock*/
+	pool->base.clock_sources[DCN301_CLK_SRC_PLL0] =
+			dcn301_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL0,
+				&clk_src_regs[0], false);
+	pool->base.clock_sources[DCN301_CLK_SRC_PLL1] =
+			dcn301_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL1,
+				&clk_src_regs[1], false);
+	pool->base.clock_sources[DCN301_CLK_SRC_PLL2] =
+			dcn301_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL2,
+				&clk_src_regs[2], false);
+	pool->base.clock_sources[DCN301_CLK_SRC_PLL3] =
+			dcn301_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL3,
+				&clk_src_regs[3], false);
+
+	pool->base.clk_src_count = DCN301_CLK_SRC_TOTAL;
+
+	/* todo: not reuse phy_pll registers */
+	pool->base.dp_clock_source =
+			dcn301_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_ID_DP_DTO,
+				&clk_src_regs[0], true);
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	/* DCCG */
+	pool->base.dccg = dccg301_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
+	if (pool->base.dccg == NULL) {
+		dm_error("DC: failed to create dccg!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	init_soc_bounding_box(dc, pool);
+
+	if (!dc->debug.disable_pplib_wm_range && pool->base.pp_smu->nv_funcs.set_wm_ranges)
+		set_wm_ranges(pool->base.pp_smu, &dcn3_01_soc);
+
+	num_pipes = dcn3_01_ip.max_num_dpp;
+
+	for (i = 0; i < dcn3_01_ip.max_num_dpp; i++)
+		if (pipe_fuses & 1 << i)
+			num_pipes--;
+	dcn3_01_ip.max_num_dpp = num_pipes;
+	dcn3_01_ip.max_num_otg = num_pipes;
+
+
+	dml_init_instance(&dc->dml, &dcn3_01_soc, &dcn3_01_ip, DML_PROJECT_DCN30);
+
+	/* IRQ */
+	init_data.ctx = dc->ctx;
+	pool->base.irqs = dal_irq_service_dcn30_create(&init_data);
+	if (!pool->base.irqs)
+		goto create_fail;
+
+	/* HUBBUB */
+	pool->base.hubbub = dcn301_hubbub_create(ctx);
+	if (pool->base.hubbub == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create hubbub!\n");
+		goto create_fail;
+	}
+
+	j = 0;
+	/* HUBPs, DPPs, OPPs and TGs */
+	for (i = 0; i < pool->base.pipe_count; i++) {
+
+		/* if pipe is disabled, skip instance of HW pipe,
+		 * i.e, skip ASIC register instance
+		 */
+		if ((pipe_fuses & (1 << i)) != 0) {
+			DC_LOG_DEBUG("%s: fusing pipe %d\n", __func__, i);
+			continue;
+		}
+
+		pool->base.hubps[j] = dcn301_hubp_create(ctx, i);
+		if (pool->base.hubps[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create hubps!\n");
+			goto create_fail;
+		}
+
+		pool->base.dpps[j] = dcn301_dpp_create(ctx, i);
+		if (pool->base.dpps[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create dpps!\n");
+			goto create_fail;
+		}
+
+		pool->base.opps[j] = dcn301_opp_create(ctx, i);
+		if (pool->base.opps[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create output pixel processor!\n");
+			goto create_fail;
+		}
+
+		pool->base.timing_generators[j] = dcn301_timing_generator_create(ctx, i);
+		if (pool->base.timing_generators[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto create_fail;
+		}
+		j++;
+	}
+	pool->base.timing_generator_count = j;
+	pool->base.pipe_count = j;
+	pool->base.mpcc_count = j;
+
+	/* ABM (or ABMs for NV2x) */
+	/* TODO: */
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		pool->base.multiple_abms[i] = dmub_abm_create(ctx,
+				&abm_regs[i],
+				&abm_shift,
+				&abm_mask);
+		if (pool->base.multiple_abms[i] == NULL) {
+			dm_error("DC: failed to create abm for pipe %d!\n", i);
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	/* MPC and DSC */
+	pool->base.mpc = dcn301_mpc_create(ctx, pool->base.mpcc_count, pool->base.res_cap->num_mpc_3dlut);
+	if (pool->base.mpc == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mpc!\n");
+		goto create_fail;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		pool->base.dscs[i] = dcn301_dsc_create(ctx, i);
+		if (pool->base.dscs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create display stream compressor %d!\n", i);
+			goto create_fail;
+		}
+	}
+
+	/* DWB and MMHUBBUB */
+	if (!dcn301_dwbc_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create dwbc!\n");
+		goto create_fail;
+	}
+
+	if (!dcn301_mmhubbub_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mcif_wb!\n");
+		goto create_fail;
+	}
+
+	/* AUX and I2C */
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.engines[i] = dcn301_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto create_fail;
+		}
+		pool->base.hw_i2cs[i] = dcn301_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create hw i2c!!\n");
+			goto create_fail;
+		}
+		pool->base.sw_i2cs[i] = NULL;
+	}
+
+	/* Audio, Stream Encoders including HPO and virtual, MPC 3D LUTs */
+	if (!resource_construct(num_virtual_links, dc, &pool->base,
+			&res_create_funcs))
+		goto create_fail;
+
+	/* HW Sequencer and Plane caps */
+	dcn301_hw_sequencer_construct(dc);
+
+	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->cap_funcs = cap_funcs;
+
+	return true;
+
+create_fail:
+
+	dcn301_destruct(pool);
+
+	return false;
+}
+
+struct resource_pool *dcn301_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc)
+{
+	struct dcn301_resource_pool *pool =
+		kzalloc(sizeof(struct dcn301_resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dcn301_resource_construct(init_data->num_virtual_links, dc, pool))
+		return &pool->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(pool);
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn301/dcn301_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn301/dcn301_resource.h
new file mode 100644
index 000000000000..ae8672680cdd
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn301/dcn301_resource.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2020 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef _DCN301_RESOURCE_H_
+#define _DCN301_RESOURCE_H_
+
+#include "core_types.h"
+
+struct dc;
+struct resource_pool;
+struct _vcs_dpi_display_pipe_params_st;
+
+extern struct _vcs_dpi_ip_params_st dcn3_01_ip;
+extern struct _vcs_dpi_soc_bounding_box_st dcn3_01_soc;
+
+struct dcn301_resource_pool {
+	struct resource_pool base;
+};
+struct resource_pool *dcn301_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc);
+
+#endif /* _DCN301_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.c
new file mode 100644
index 000000000000..5791b5cc2875
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.c
@@ -0,0 +1,1518 @@
+/*
+ * Copyright 2020 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dcn302/dcn302_init.h"
+#include "dcn302_resource.h"
+#include "dcn302/dcn302_dccg.h"
+#include "irq/dcn302/irq_service_dcn302.h"
+
+#include "dcn30/dcn30_dio_link_encoder.h"
+#include "dcn30/dcn30_dio_stream_encoder.h"
+#include "dcn30/dcn30_dwb.h"
+#include "dcn30/dcn30_dpp.h"
+#include "dcn30/dcn30_hubbub.h"
+#include "dcn30/dcn30_hubp.h"
+#include "dcn30/dcn30_mmhubbub.h"
+#include "dcn30/dcn30_mpc.h"
+#include "dcn30/dcn30_opp.h"
+#include "dcn30/dcn30_optc.h"
+#include "dcn30/dcn30_resource.h"
+
+#include "dcn20/dcn20_dsc.h"
+#include "dcn20/dcn20_resource.h"
+
+#include "dml/dcn30/dcn30_fpu.h"
+
+#include "dcn10/dcn10_resource.h"
+
+#include "link.h"
+#include "dce/dce_abm.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_hwseq.h"
+#include "dce/dce_i2c_hw.h"
+#include "dce/dce_panel_cntl.h"
+#include "dce/dmub_abm.h"
+#include "dce/dmub_psr.h"
+#include "clk_mgr.h"
+
+#include "hw_sequencer_private.h"
+#include "reg_helper.h"
+#include "resource.h"
+#include "vm_helper.h"
+
+#include "dml/dcn302/dcn302_fpu.h"
+
+#include "dimgrey_cavefish_ip_offset.h"
+#include "dcn/dcn_3_0_2_offset.h"
+#include "dcn/dcn_3_0_2_sh_mask.h"
+#include "dpcs/dpcs_3_0_0_offset.h"
+#include "dpcs/dpcs_3_0_0_sh_mask.h"
+#include "nbio/nbio_7_4_offset.h"
+#include "amdgpu_socbb.h"
+
+#define DC_LOGGER \
+	dc->ctx->logger
+#define DC_LOGGER_INIT(logger)
+
+static const struct dc_debug_options debug_defaults_drv = {
+		.disable_dmcu = true,
+		.force_abm_enable = false,
+		.timing_trace = false,
+		.clock_trace = true,
+		.disable_pplib_clock_request = true,
+		.pipe_split_policy = MPC_SPLIT_DYNAMIC,
+		.force_single_disp_pipe_split = false,
+		.disable_dcc = DCC_ENABLE,
+		.vsr_support = true,
+		.performance_trace = false,
+		.max_downscale_src_width = 7680,/*upto 8K*/
+		.disable_pplib_wm_range = false,
+		.scl_reset_length10 = true,
+		.sanity_checks = false,
+		.underflow_assert_delay_us = 0xFFFFFFFF,
+		.dwb_fi_phase = -1, // -1 = disable,
+		.dmub_command_table = true,
+		.use_max_lb = true,
+		.exit_idle_opt_for_cursor_updates = true,
+		.enable_legacy_fast_update = false,
+		.using_dml2 = false,
+};
+
+static const struct dc_panel_config panel_config_defaults = {
+		.psr = {
+			.disable_psr = false,
+			.disallow_psrsu = false,
+			.disallow_replay = false,
+		},
+};
+
+enum dcn302_clk_src_array_id {
+	DCN302_CLK_SRC_PLL0,
+	DCN302_CLK_SRC_PLL1,
+	DCN302_CLK_SRC_PLL2,
+	DCN302_CLK_SRC_PLL3,
+	DCN302_CLK_SRC_PLL4,
+	DCN302_CLK_SRC_TOTAL
+};
+
+static const struct resource_caps res_cap_dcn302 = {
+		.num_timing_generator = 5,
+		.num_opp = 5,
+		.num_video_plane = 5,
+		.num_audio = 5,
+		.num_stream_encoder = 5,
+		.num_dwb = 1,
+		.num_ddc = 5,
+		.num_vmid = 16,
+		.num_mpc_3dlut = 2,
+		.num_dsc = 5,
+};
+
+static const struct dc_plane_cap plane_cap = {
+		.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
+		.per_pixel_alpha = true,
+		.pixel_format_support = {
+				.argb8888 = true,
+				.nv12 = true,
+				.fp16 = true,
+				.p010 = true,
+				.ayuv = false,
+		},
+		.max_upscale_factor = {
+				.argb8888 = 16000,
+				.nv12 = 16000,
+				.fp16 = 16000
+		},
+		/* 6:1 downscaling ratio: 1000/6 = 166.666 */
+		.max_downscale_factor = {
+				.argb8888 = 167,
+				.nv12 = 167,
+				.fp16 = 167
+		},
+		16,
+		16
+};
+
+/* NBIO */
+#define NBIO_BASE_INNER(seg) \
+		NBIO_BASE__INST0_SEG ## seg
+
+#define NBIO_BASE(seg) \
+		NBIO_BASE_INNER(seg)
+
+#define NBIO_SR(reg_name)\
+		.reg_name = NBIO_BASE(mm ## reg_name ## _BASE_IDX) + \
+		mm ## reg_name
+
+/* DCN */
+#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
+
+#define BASE(seg) BASE_INNER(seg)
+
+#define SR(reg_name)\
+		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) + mm ## reg_name
+
+#define SF(reg_name, field_name, post_fix)\
+		.field_name = reg_name ## __ ## field_name ## post_fix
+
+#define SRI(reg_name, block, id)\
+		.reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + mm ## block ## id ## _ ## reg_name
+
+#define SRI2(reg_name, block, id)\
+		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) + mm ## reg_name
+
+#define SRII(reg_name, block, id)\
+		.reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		mm ## block ## id ## _ ## reg_name
+
+#define DCCG_SRII(reg_name, block, id)\
+		.block ## _ ## reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		mm ## block ## id ## _ ## reg_name
+
+#define VUPDATE_SRII(reg_name, block, id)\
+		.reg_name[id] = BASE(mm ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
+		mm ## reg_name ## _ ## block ## id
+
+#define SRII_DWB(reg_name, temp_name, block, id)\
+		.reg_name[id] = BASE(mm ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
+		mm ## block ## id ## _ ## temp_name
+
+#define SF_DWB2(reg_name, block, id, field_name, post_fix)	\
+	.field_name = reg_name ## __ ## field_name ## post_fix
+
+#define SRII_MPC_RMU(reg_name, block, id)\
+		.RMU##_##reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		mm ## block ## id ## _ ## reg_name
+
+static const struct dcn_hubbub_registers hubbub_reg = {
+		HUBBUB_REG_LIST_DCN30(0)
+};
+
+static const struct dcn_hubbub_shift hubbub_shift = {
+		HUBBUB_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn_hubbub_mask hubbub_mask = {
+		HUBBUB_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define vmid_regs(id)\
+		[id] = { DCN20_VMID_REG_LIST(id) }
+
+static const struct dcn_vmid_registers vmid_regs[] = {
+		vmid_regs(0),
+		vmid_regs(1),
+		vmid_regs(2),
+		vmid_regs(3),
+		vmid_regs(4),
+		vmid_regs(5),
+		vmid_regs(6),
+		vmid_regs(7),
+		vmid_regs(8),
+		vmid_regs(9),
+		vmid_regs(10),
+		vmid_regs(11),
+		vmid_regs(12),
+		vmid_regs(13),
+		vmid_regs(14),
+		vmid_regs(15)
+};
+
+static const struct dcn20_vmid_shift vmid_shifts = {
+		DCN20_VMID_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn20_vmid_mask vmid_masks = {
+		DCN20_VMID_MASK_SH_LIST(_MASK)
+};
+
+static struct hubbub *dcn302_hubbub_create(struct dc_context *ctx)
+{
+	int i;
+
+	struct dcn20_hubbub *hubbub3 = kzalloc(sizeof(struct dcn20_hubbub), GFP_KERNEL);
+
+	if (!hubbub3)
+		return NULL;
+
+	hubbub3_construct(hubbub3, ctx, &hubbub_reg, &hubbub_shift, &hubbub_mask);
+
+	for (i = 0; i < res_cap_dcn302.num_vmid; i++) {
+		struct dcn20_vmid *vmid = &hubbub3->vmid[i];
+
+		vmid->ctx = ctx;
+
+		vmid->regs = &vmid_regs[i];
+		vmid->shifts = &vmid_shifts;
+		vmid->masks = &vmid_masks;
+	}
+
+	return &hubbub3->base;
+}
+
+#define vpg_regs(id)\
+		[id] = { VPG_DCN3_REG_LIST(id) }
+
+static const struct dcn30_vpg_registers vpg_regs[] = {
+		vpg_regs(0),
+		vpg_regs(1),
+		vpg_regs(2),
+		vpg_regs(3),
+		vpg_regs(4),
+		vpg_regs(5)
+};
+
+static const struct dcn30_vpg_shift vpg_shift = {
+		DCN3_VPG_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn30_vpg_mask vpg_mask = {
+		DCN3_VPG_MASK_SH_LIST(_MASK)
+};
+
+static struct vpg *dcn302_vpg_create(struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn30_vpg *vpg3 = kzalloc(sizeof(struct dcn30_vpg), GFP_KERNEL);
+
+	if (!vpg3)
+		return NULL;
+
+	vpg3_construct(vpg3, ctx, inst, &vpg_regs[inst], &vpg_shift, &vpg_mask);
+
+	return &vpg3->base;
+}
+
+#define afmt_regs(id)\
+		[id] = { AFMT_DCN3_REG_LIST(id) }
+
+static const struct dcn30_afmt_registers afmt_regs[] = {
+		afmt_regs(0),
+		afmt_regs(1),
+		afmt_regs(2),
+		afmt_regs(3),
+		afmt_regs(4),
+		afmt_regs(5)
+};
+
+static const struct dcn30_afmt_shift afmt_shift = {
+		DCN3_AFMT_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn30_afmt_mask afmt_mask = {
+		DCN3_AFMT_MASK_SH_LIST(_MASK)
+};
+
+static struct afmt *dcn302_afmt_create(struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn30_afmt *afmt3 = kzalloc(sizeof(struct dcn30_afmt), GFP_KERNEL);
+
+	if (!afmt3)
+		return NULL;
+
+	afmt3_construct(afmt3, ctx, inst, &afmt_regs[inst], &afmt_shift, &afmt_mask);
+
+	return &afmt3->base;
+}
+
+#define audio_regs(id)\
+		[id] = { AUD_COMMON_REG_LIST(id) }
+
+static const struct dce_audio_registers audio_regs[] = {
+		audio_regs(0),
+		audio_regs(1),
+		audio_regs(2),
+		audio_regs(3),
+		audio_regs(4),
+		audio_regs(5),
+		audio_regs(6)
+};
+
+#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
+		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
+
+static const struct dce_audio_shift audio_shift = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_audio_mask audio_mask = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+static struct audio *dcn302_create_audio(struct dc_context *ctx, unsigned int inst)
+{
+	return dce_audio_create(ctx, inst, &audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+#define stream_enc_regs(id)\
+		[id] = { SE_DCN3_REG_LIST(id) }
+
+static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
+		stream_enc_regs(0),
+		stream_enc_regs(1),
+		stream_enc_regs(2),
+		stream_enc_regs(3),
+		stream_enc_regs(4)
+};
+
+static const struct dcn10_stream_encoder_shift se_shift = {
+		SE_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn10_stream_encoder_mask se_mask = {
+		SE_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static struct stream_encoder *dcn302_stream_encoder_create(enum engine_id eng_id, struct dc_context *ctx)
+{
+	struct dcn10_stream_encoder *enc1;
+	struct vpg *vpg;
+	struct afmt *afmt;
+	int vpg_inst;
+	int afmt_inst;
+
+	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
+	if (eng_id <= ENGINE_ID_DIGE) {
+		vpg_inst = eng_id;
+		afmt_inst = eng_id;
+	} else
+		return NULL;
+
+	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
+	vpg = dcn302_vpg_create(ctx, vpg_inst);
+	afmt = dcn302_afmt_create(ctx, afmt_inst);
+
+	if (!enc1 || !vpg || !afmt) {
+		kfree(enc1);
+		kfree(vpg);
+		kfree(afmt);
+		return NULL;
+	}
+
+	dcn30_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios, eng_id, vpg, afmt, &stream_enc_regs[eng_id],
+			&se_shift, &se_mask);
+
+	return &enc1->base;
+}
+
+#define clk_src_regs(index, pllid)\
+		[index] = { CS_COMMON_REG_LIST_DCN3_02(index, pllid) }
+
+static const struct dce110_clk_src_regs clk_src_regs[] = {
+		clk_src_regs(0, A),
+		clk_src_regs(1, B),
+		clk_src_regs(2, C),
+		clk_src_regs(3, D),
+		clk_src_regs(4, E)
+};
+
+static const struct dce110_clk_src_shift cs_shift = {
+		CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
+};
+
+static const struct dce110_clk_src_mask cs_mask = {
+		CS_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
+};
+
+static struct clock_source *dcn302_clock_source_create(struct dc_context *ctx, struct dc_bios *bios,
+		enum clock_source_id id, const struct dce110_clk_src_regs *regs, bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src = kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dcn3_clk_src_construct(clk_src, ctx, bios, id, regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	kfree(clk_src);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static const struct dce_hwseq_registers hwseq_reg = {
+		HWSEQ_DCN302_REG_LIST()
+};
+
+static const struct dce_hwseq_shift hwseq_shift = {
+		HWSEQ_DCN302_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+		HWSEQ_DCN302_MASK_SH_LIST(_MASK)
+};
+
+static struct dce_hwseq *dcn302_hwseq_create(struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &hwseq_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+	}
+	return hws;
+}
+
+#define hubp_regs(id)\
+		[id] = { HUBP_REG_LIST_DCN30(id) }
+
+static const struct dcn_hubp2_registers hubp_regs[] = {
+		hubp_regs(0),
+		hubp_regs(1),
+		hubp_regs(2),
+		hubp_regs(3),
+		hubp_regs(4)
+};
+
+static const struct dcn_hubp2_shift hubp_shift = {
+		HUBP_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn_hubp2_mask hubp_mask = {
+		HUBP_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static struct hubp *dcn302_hubp_create(struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_hubp *hubp2 = kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
+
+	if (!hubp2)
+		return NULL;
+
+	if (hubp3_construct(hubp2, ctx, inst, &hubp_regs[inst], &hubp_shift, &hubp_mask))
+		return &hubp2->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(hubp2);
+	return NULL;
+}
+
+#define dpp_regs(id)\
+		[id] = { DPP_REG_LIST_DCN30(id) }
+
+static const struct dcn3_dpp_registers dpp_regs[] = {
+		dpp_regs(0),
+		dpp_regs(1),
+		dpp_regs(2),
+		dpp_regs(3),
+		dpp_regs(4)
+};
+
+static const struct dcn3_dpp_shift tf_shift = {
+		DPP_REG_LIST_SH_MASK_DCN30(__SHIFT)
+};
+
+static const struct dcn3_dpp_mask tf_mask = {
+		DPP_REG_LIST_SH_MASK_DCN30(_MASK)
+};
+
+static struct dpp *dcn302_dpp_create(struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn3_dpp *dpp = kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
+
+	if (!dpp)
+		return NULL;
+
+	if (dpp3_construct(dpp, ctx, inst, &dpp_regs[inst], &tf_shift, &tf_mask))
+		return &dpp->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(dpp);
+	return NULL;
+}
+
+#define opp_regs(id)\
+		[id] = { OPP_REG_LIST_DCN30(id) }
+
+static const struct dcn20_opp_registers opp_regs[] = {
+		opp_regs(0),
+		opp_regs(1),
+		opp_regs(2),
+		opp_regs(3),
+		opp_regs(4)
+};
+
+static const struct dcn20_opp_shift opp_shift = {
+		OPP_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn20_opp_mask opp_mask = {
+		OPP_MASK_SH_LIST_DCN20(_MASK)
+};
+
+static struct output_pixel_processor *dcn302_opp_create(struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_opp *opp = kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
+
+	if (!opp) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dcn20_opp_construct(opp, ctx, inst, &opp_regs[inst], &opp_shift, &opp_mask);
+	return &opp->base;
+}
+
+#define optc_regs(id)\
+		[id] = { OPTC_COMMON_REG_LIST_DCN3_0(id) }
+
+static const struct dcn_optc_registers optc_regs[] = {
+		optc_regs(0),
+		optc_regs(1),
+		optc_regs(2),
+		optc_regs(3),
+		optc_regs(4)
+};
+
+static const struct dcn_optc_shift optc_shift = {
+		OPTC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn_optc_mask optc_mask = {
+		OPTC_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static struct timing_generator *dcn302_timing_generator_create(struct dc_context *ctx, uint32_t instance)
+{
+	struct optc *tgn10 = kzalloc(sizeof(struct optc), GFP_KERNEL);
+
+	if (!tgn10)
+		return NULL;
+
+	tgn10->base.inst = instance;
+	tgn10->base.ctx = ctx;
+
+	tgn10->tg_regs = &optc_regs[instance];
+	tgn10->tg_shift = &optc_shift;
+	tgn10->tg_mask = &optc_mask;
+
+	dcn30_timing_generator_init(tgn10);
+
+	return &tgn10->base;
+}
+
+static const struct dcn30_mpc_registers mpc_regs = {
+		MPC_REG_LIST_DCN3_0(0),
+		MPC_REG_LIST_DCN3_0(1),
+		MPC_REG_LIST_DCN3_0(2),
+		MPC_REG_LIST_DCN3_0(3),
+		MPC_REG_LIST_DCN3_0(4),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(0),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(1),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(2),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(3),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(4),
+		MPC_RMU_GLOBAL_REG_LIST_DCN3AG,
+		MPC_RMU_REG_LIST_DCN3AG(0),
+		MPC_RMU_REG_LIST_DCN3AG(1),
+		MPC_RMU_REG_LIST_DCN3AG(2),
+		MPC_DWB_MUX_REG_LIST_DCN3_0(0),
+};
+
+static const struct dcn30_mpc_shift mpc_shift = {
+		MPC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_mpc_mask mpc_mask = {
+		MPC_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static struct mpc *dcn302_mpc_create(struct dc_context *ctx, int num_mpcc, int num_rmu)
+{
+	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc), GFP_KERNEL);
+
+	if (!mpc30)
+		return NULL;
+
+	dcn30_mpc_construct(mpc30, ctx, &mpc_regs, &mpc_shift, &mpc_mask, num_mpcc, num_rmu);
+
+	return &mpc30->base;
+}
+
+#define dsc_regsDCN20(id)\
+[id] = { DSC_REG_LIST_DCN20(id) }
+
+static const struct dcn20_dsc_registers dsc_regs[] = {
+		dsc_regsDCN20(0),
+		dsc_regsDCN20(1),
+		dsc_regsDCN20(2),
+		dsc_regsDCN20(3),
+		dsc_regsDCN20(4)
+};
+
+static const struct dcn20_dsc_shift dsc_shift = {
+		DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
+};
+
+static const struct dcn20_dsc_mask dsc_mask = {
+		DSC_REG_LIST_SH_MASK_DCN20(_MASK)
+};
+
+static struct display_stream_compressor *dcn302_dsc_create(struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_dsc *dsc = kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
+
+	if (!dsc) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
+	return &dsc->base;
+}
+
+#define dwbc_regs_dcn3(id)\
+[id] = { DWBC_COMMON_REG_LIST_DCN30(id) }
+
+static const struct dcn30_dwbc_registers dwbc30_regs[] = {
+		dwbc_regs_dcn3(0)
+};
+
+static const struct dcn30_dwbc_shift dwbc30_shift = {
+		DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_dwbc_mask dwbc30_mask = {
+		DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static bool dcn302_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc), GFP_KERNEL);
+
+		if (!dwbc30) {
+			dm_error("DC: failed to create dwbc30!\n");
+			return false;
+		}
+
+		dcn30_dwbc_construct(dwbc30, ctx, &dwbc30_regs[i], &dwbc30_shift, &dwbc30_mask, i);
+
+		pool->dwbc[i] = &dwbc30->base;
+	}
+	return true;
+}
+
+#define mcif_wb_regs_dcn3(id)\
+[id] = { MCIF_WB_COMMON_REG_LIST_DCN30(id) }
+
+static const struct dcn30_mmhubbub_registers mcif_wb30_regs[] = {
+		mcif_wb_regs_dcn3(0)
+};
+
+static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
+		MCIF_WB_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
+		MCIF_WB_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static bool dcn302_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub), GFP_KERNEL);
+
+		if (!mcif_wb30) {
+			dm_error("DC: failed to create mcif_wb30!\n");
+			return false;
+		}
+
+		dcn30_mmhubbub_construct(mcif_wb30, ctx, &mcif_wb30_regs[i], &mcif_wb30_shift, &mcif_wb30_mask, i);
+
+		pool->mcif_wb[i] = &mcif_wb30->base;
+	}
+	return true;
+}
+
+#define aux_engine_regs(id)\
+[id] = {\
+		AUX_COMMON_REG_LIST0(id), \
+		.AUXN_IMPCAL = 0, \
+		.AUXP_IMPCAL = 0, \
+		.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1),
+		aux_engine_regs(2),
+		aux_engine_regs(3),
+		aux_engine_regs(4)
+};
+
+static const struct dce110_aux_registers_shift aux_shift = {
+		DCN_AUX_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+		DCN_AUX_MASK_SH_LIST(_MASK)
+};
+
+static struct dce_aux *dcn302_aux_engine_create(struct dc_context *ctx, uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine = kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst, SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+			&aux_engine_regs[inst], &aux_mask, &aux_shift, ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+
+#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
+
+static const struct dce_i2c_registers i2c_hw_regs[] = {
+		i2c_inst_regs(1),
+		i2c_inst_regs(2),
+		i2c_inst_regs(3),
+		i2c_inst_regs(4),
+		i2c_inst_regs(5)
+};
+
+static const struct dce_i2c_shift i2c_shifts = {
+		I2C_COMMON_MASK_SH_LIST_DCN2(__SHIFT)
+};
+
+static const struct dce_i2c_mask i2c_masks = {
+		I2C_COMMON_MASK_SH_LIST_DCN2(_MASK)
+};
+
+static struct dce_i2c_hw *dcn302_i2c_hw_create(struct dc_context *ctx, uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw = kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst, &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+		.max_hdmi_deep_color = COLOR_DEPTH_121212,
+		.max_hdmi_pixel_clock = 600000,
+		.hdmi_ycbcr420_supported = true,
+		.dp_ycbcr420_supported = true,
+		.fec_supported = true,
+		.flags.bits.IS_HBR2_CAPABLE = true,
+		.flags.bits.IS_HBR3_CAPABLE = true,
+		.flags.bits.IS_TPS3_CAPABLE = true,
+		.flags.bits.IS_TPS4_CAPABLE = true
+};
+
+#define link_regs(id, phyid)\
+		[id] = {\
+				LE_DCN3_REG_LIST(id), \
+				UNIPHY_DCN2_REG_LIST(phyid), \
+				DPCS_DCN2_REG_LIST(id), \
+				SRI(DP_DPHY_INTERNAL_CTRL, DP, id) \
+		}
+
+static const struct dcn10_link_enc_registers link_enc_regs[] = {
+		link_regs(0, A),
+		link_regs(1, B),
+		link_regs(2, C),
+		link_regs(3, D),
+		link_regs(4, E)
+};
+
+static const struct dcn10_link_enc_shift le_shift = {
+		LINK_ENCODER_MASK_SH_LIST_DCN30(__SHIFT),
+		DPCS_DCN2_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn10_link_enc_mask le_mask = {
+		LINK_ENCODER_MASK_SH_LIST_DCN30(_MASK),
+		DPCS_DCN2_MASK_SH_LIST(_MASK)
+};
+
+#define aux_regs(id)\
+		[id] = { DCN2_AUX_REG_LIST(id) }
+
+static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
+		aux_regs(0),
+		aux_regs(1),
+		aux_regs(2),
+		aux_regs(3),
+		aux_regs(4)
+};
+
+#define hpd_regs(id)\
+		[id] = { HPD_REG_LIST(id) }
+
+static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
+		hpd_regs(0),
+		hpd_regs(1),
+		hpd_regs(2),
+		hpd_regs(3),
+		hpd_regs(4)
+};
+
+static struct link_encoder *dcn302_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dcn20_link_encoder *enc20 = kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+
+	if (!enc20)
+		return NULL;
+
+	dcn30_link_encoder_construct(enc20, enc_init_data, &link_enc_feature,
+			&link_enc_regs[enc_init_data->transmitter], &link_enc_aux_regs[enc_init_data->channel - 1],
+			&link_enc_hpd_regs[enc_init_data->hpd_source], &le_shift, &le_mask);
+
+	return &enc20->enc10.base;
+}
+
+static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
+		{ DCN_PANEL_CNTL_REG_LIST() }
+};
+
+static const struct dce_panel_cntl_shift panel_cntl_shift = {
+		DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_panel_cntl_mask panel_cntl_mask = {
+		DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
+};
+
+static struct panel_cntl *dcn302_panel_cntl_create(const struct panel_cntl_init_data *init_data)
+{
+	struct dce_panel_cntl *panel_cntl = kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
+
+	if (!panel_cntl)
+		return NULL;
+
+	dce_panel_cntl_construct(panel_cntl, init_data, &panel_cntl_regs[init_data->inst],
+			&panel_cntl_shift, &panel_cntl_mask);
+
+	return &panel_cntl->base;
+}
+
+static void read_dce_straps(struct dc_context *ctx, struct resource_straps *straps)
+{
+	generic_reg_get(ctx, mmDC_PINSTRAPS + BASE(mmDC_PINSTRAPS_BASE_IDX),
+			FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
+}
+
+static const struct resource_create_funcs res_create_funcs = {
+		.read_dce_straps = read_dce_straps,
+		.create_audio = dcn302_create_audio,
+		.create_stream_encoder = dcn302_stream_encoder_create,
+		.create_hwseq = dcn302_hwseq_create,
+};
+
+static bool is_soc_bounding_box_valid(struct dc *dc)
+{
+	uint32_t hw_internal_rev = dc->ctx->asic_id.hw_internal_rev;
+
+	if (ASICREV_IS_DIMGREY_CAVEFISH_P(hw_internal_rev))
+		return true;
+
+	return false;
+}
+
+static bool init_soc_bounding_box(struct dc *dc,  struct resource_pool *pool)
+{
+	struct _vcs_dpi_soc_bounding_box_st *loaded_bb = &dcn3_02_soc;
+	struct _vcs_dpi_ip_params_st *loaded_ip = &dcn3_02_ip;
+
+	DC_LOGGER_INIT(dc->ctx->logger);
+
+	if (!is_soc_bounding_box_valid(dc)) {
+		DC_LOG_ERROR("%s: not valid soc bounding box\n", __func__);
+		return false;
+	}
+
+	loaded_ip->max_num_otg = pool->pipe_count;
+	loaded_ip->max_num_dpp = pool->pipe_count;
+	loaded_ip->clamp_min_dcfclk = dc->config.clamp_min_dcfclk;
+	DC_FP_START();
+	dcn20_patch_bounding_box(dc, loaded_bb);
+	DC_FP_END();
+
+	if (dc->ctx->dc_bios->funcs->get_soc_bb_info) {
+		struct bp_soc_bb_info bb_info = { 0 };
+
+		if (dc->ctx->dc_bios->funcs->get_soc_bb_info(
+			    dc->ctx->dc_bios, &bb_info) == BP_RESULT_OK) {
+
+				DC_FP_START();
+				dcn302_fpu_init_soc_bounding_box(bb_info);
+				DC_FP_END();
+		}
+	}
+
+	return true;
+}
+
+static void dcn302_resource_destruct(struct resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->stream_enc_count; i++) {
+		if (pool->stream_enc[i] != NULL) {
+			if (pool->stream_enc[i]->vpg != NULL) {
+				kfree(DCN30_VPG_FROM_VPG(pool->stream_enc[i]->vpg));
+				pool->stream_enc[i]->vpg = NULL;
+			}
+			if (pool->stream_enc[i]->afmt != NULL) {
+				kfree(DCN30_AFMT_FROM_AFMT(pool->stream_enc[i]->afmt));
+				pool->stream_enc[i]->afmt = NULL;
+			}
+			kfree(DCN10STRENC_FROM_STRENC(pool->stream_enc[i]));
+			pool->stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->res_cap->num_dsc; i++) {
+		if (pool->dscs[i] != NULL)
+			dcn20_dsc_destroy(&pool->dscs[i]);
+	}
+
+	if (pool->mpc != NULL) {
+		kfree(TO_DCN20_MPC(pool->mpc));
+		pool->mpc = NULL;
+	}
+
+	if (pool->hubbub != NULL) {
+		kfree(pool->hubbub);
+		pool->hubbub = NULL;
+	}
+
+	for (i = 0; i < pool->pipe_count; i++) {
+		if (pool->dpps[i] != NULL) {
+			kfree(TO_DCN20_DPP(pool->dpps[i]));
+			pool->dpps[i] = NULL;
+		}
+
+		if (pool->hubps[i] != NULL) {
+			kfree(TO_DCN20_HUBP(pool->hubps[i]));
+			pool->hubps[i] = NULL;
+		}
+
+		if (pool->irqs != NULL)
+			dal_irq_service_destroy(&pool->irqs);
+	}
+
+	for (i = 0; i < pool->res_cap->num_ddc; i++) {
+		if (pool->engines[i] != NULL)
+			dce110_engine_destroy(&pool->engines[i]);
+		if (pool->hw_i2cs[i] != NULL) {
+			kfree(pool->hw_i2cs[i]);
+			pool->hw_i2cs[i] = NULL;
+		}
+		if (pool->sw_i2cs[i] != NULL) {
+			kfree(pool->sw_i2cs[i]);
+			pool->sw_i2cs[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->res_cap->num_opp; i++) {
+		if (pool->opps[i] != NULL)
+			pool->opps[i]->funcs->opp_destroy(&pool->opps[i]);
+	}
+
+	for (i = 0; i < pool->res_cap->num_timing_generator; i++) {
+		if (pool->timing_generators[i] != NULL)	{
+			kfree(DCN10TG_FROM_TG(pool->timing_generators[i]));
+			pool->timing_generators[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->res_cap->num_dwb; i++) {
+		if (pool->dwbc[i] != NULL) {
+			kfree(TO_DCN30_DWBC(pool->dwbc[i]));
+			pool->dwbc[i] = NULL;
+		}
+		if (pool->mcif_wb[i] != NULL) {
+			kfree(TO_DCN30_MMHUBBUB(pool->mcif_wb[i]));
+			pool->mcif_wb[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->audio_count; i++) {
+		if (pool->audios[i])
+			dce_aud_destroy(&pool->audios[i]);
+	}
+
+	for (i = 0; i < pool->clk_src_count; i++) {
+		if (pool->clock_sources[i] != NULL)
+			dcn20_clock_source_destroy(&pool->clock_sources[i]);
+	}
+
+	if (pool->dp_clock_source != NULL)
+		dcn20_clock_source_destroy(&pool->dp_clock_source);
+
+	for (i = 0; i < pool->res_cap->num_mpc_3dlut; i++) {
+		if (pool->mpc_lut[i] != NULL) {
+			dc_3dlut_func_release(pool->mpc_lut[i]);
+			pool->mpc_lut[i] = NULL;
+		}
+		if (pool->mpc_shaper[i] != NULL) {
+			dc_transfer_func_release(pool->mpc_shaper[i]);
+			pool->mpc_shaper[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->pipe_count; i++) {
+		if (pool->multiple_abms[i] != NULL)
+			dce_abm_destroy(&pool->multiple_abms[i]);
+	}
+
+	if (pool->psr != NULL)
+		dmub_psr_destroy(&pool->psr);
+
+	if (pool->dccg != NULL)
+		dcn_dccg_destroy(&pool->dccg);
+
+	if (pool->oem_device != NULL) {
+		struct dc *dc = pool->oem_device->ctx->dc;
+
+		dc->link_srv->destroy_ddc_service(&pool->oem_device);
+	}
+}
+
+static void dcn302_destroy_resource_pool(struct resource_pool **pool)
+{
+	dcn302_resource_destruct(*pool);
+	kfree(*pool);
+	*pool = NULL;
+}
+
+void dcn302_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
+{
+	DC_FP_START();
+	dcn302_fpu_update_bw_bounding_box(dc, bw_params);
+	DC_FP_END();
+}
+
+static void dcn302_get_panel_config_defaults(struct dc_panel_config *panel_config)
+{
+	*panel_config = panel_config_defaults;
+}
+
+static struct resource_funcs dcn302_res_pool_funcs = {
+		.destroy = dcn302_destroy_resource_pool,
+		.link_enc_create = dcn302_link_encoder_create,
+		.panel_cntl_create = dcn302_panel_cntl_create,
+		.validate_bandwidth = dcn30_validate_bandwidth,
+		.calculate_wm_and_dlg = dcn30_calculate_wm_and_dlg,
+		.update_soc_for_wm_a = dcn30_update_soc_for_wm_a,
+		.populate_dml_pipes = dcn30_populate_dml_pipes_from_context,
+		.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
+		.release_pipe = dcn20_release_pipe,
+		.add_stream_to_ctx = dcn30_add_stream_to_ctx,
+		.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
+		.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
+		.populate_dml_writeback_from_context = dcn30_populate_dml_writeback_from_context,
+		.set_mcif_arb_params = dcn30_set_mcif_arb_params,
+		.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
+		.acquire_post_bldn_3dlut = dcn30_acquire_post_bldn_3dlut,
+		.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
+		.update_bw_bounding_box = dcn302_update_bw_bounding_box,
+		.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
+		.get_panel_config_defaults = dcn302_get_panel_config_defaults,
+};
+
+static struct dc_cap_funcs cap_funcs = {
+		.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
+};
+
+static const struct bios_registers bios_regs = {
+		NBIO_SR(BIOS_SCRATCH_3),
+		NBIO_SR(BIOS_SCRATCH_6)
+};
+
+static const struct dccg_registers dccg_regs = {
+		DCCG_REG_LIST_DCN3_02()
+};
+
+static const struct dccg_shift dccg_shift = {
+		DCCG_MASK_SH_LIST_DCN3_02(__SHIFT)
+};
+
+static const struct dccg_mask dccg_mask = {
+		DCCG_MASK_SH_LIST_DCN3_02(_MASK)
+};
+
+#define abm_regs(id)\
+		[id] = { ABM_DCN302_REG_LIST(id) }
+
+static const struct dce_abm_registers abm_regs[] = {
+		abm_regs(0),
+		abm_regs(1),
+		abm_regs(2),
+		abm_regs(3),
+		abm_regs(4)
+};
+
+static const struct dce_abm_shift abm_shift = {
+		ABM_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dce_abm_mask abm_mask = {
+		ABM_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static bool dcn302_resource_construct(
+		uint8_t num_virtual_links,
+		struct dc *dc,
+		struct resource_pool *pool)
+{
+	int i;
+	struct dc_context *ctx = dc->ctx;
+	struct irq_service_init_data init_data;
+	struct ddc_service_init_data ddc_init_data = {0};
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	pool->res_cap = &res_cap_dcn302;
+
+	pool->funcs = &dcn302_res_pool_funcs;
+
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+	pool->underlay_pipe_index = NO_UNDERLAY_PIPE;
+	pool->pipe_count = pool->res_cap->num_timing_generator;
+	pool->mpcc_count = pool->res_cap->num_timing_generator;
+	dc->caps.max_downscale_ratio = 600;
+	dc->caps.i2c_speed_in_khz = 100;
+	dc->caps.i2c_speed_in_khz_hdcp = 5; /*1.4 w/a applied by derfault*/
+	dc->caps.max_cursor_size = 256;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.dmdata_alloc_size = 2048;
+	dc->caps.mall_size_per_mem_channel = 4;
+	/* total size = mall per channel * num channels * 1024 * 1024 */
+	dc->caps.mall_size_total = dc->caps.mall_size_per_mem_channel * dc->ctx->dc_bios->vram_info.num_chans * 1048576;
+	dc->caps.cursor_cache_size = dc->caps.max_cursor_size * dc->caps.max_cursor_size * 8;
+	dc->caps.max_slave_planes = 2;
+	dc->caps.max_slave_yuv_planes = 2;
+	dc->caps.max_slave_rgb_planes = 2;
+	dc->caps.post_blend_color_processing = true;
+	dc->caps.force_dp_tps4_for_cp2520 = true;
+	dc->caps.extended_aux_timeout_support = true;
+	dc->caps.dmcub_support = true;
+	dc->caps.max_v_total = (1 << 15) - 1;
+
+	/* Color pipeline capabilities */
+	dc->caps.color.dpp.dcn_arch = 1;
+	dc->caps.color.dpp.input_lut_shared = 0;
+	dc->caps.color.dpp.icsc = 1;
+	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
+	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
+	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
+	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
+	dc->caps.color.dpp.post_csc = 1;
+	dc->caps.color.dpp.gamma_corr = 1;
+	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
+
+	dc->caps.color.dpp.hw_3d_lut = 1;
+	dc->caps.color.dpp.ogam_ram = 1;
+	// no OGAM ROM on DCN3
+	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
+	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
+	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
+	dc->caps.color.dpp.ocsc = 0;
+
+	dc->caps.color.mpc.gamut_remap = 1;
+	dc->caps.color.mpc.num_3dluts = pool->res_cap->num_mpc_3dlut; //3
+	dc->caps.color.mpc.ogam_ram = 1;
+	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
+	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
+	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
+	dc->caps.color.mpc.ocsc = 1;
+
+	dc->caps.dp_hdmi21_pcon_support = true;
+
+	/* read VBIOS LTTPR caps */
+	if (ctx->dc_bios->funcs->get_lttpr_caps) {
+		enum bp_result bp_query_result;
+		uint8_t is_vbios_lttpr_enable = 0;
+
+		bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
+		dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
+	}
+
+	if (ctx->dc_bios->funcs->get_lttpr_interop) {
+		enum bp_result bp_query_result;
+		uint8_t is_vbios_interop_enabled = 0;
+
+		bp_query_result = ctx->dc_bios->funcs->get_lttpr_interop(ctx->dc_bios,
+				&is_vbios_interop_enabled);
+		dc->caps.vbios_lttpr_aware = (bp_query_result == BP_RESULT_OK) && !!is_vbios_interop_enabled;
+	}
+
+	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
+		dc->debug = debug_defaults_drv;
+
+	// Init the vm_helper
+	if (dc->vm_helper)
+		vm_helper_init(dc->vm_helper, 16);
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	/* Clock Sources for Pixel Clock*/
+	pool->clock_sources[DCN302_CLK_SRC_PLL0] =
+			dcn302_clock_source_create(ctx, ctx->dc_bios,
+					CLOCK_SOURCE_COMBO_PHY_PLL0,
+					&clk_src_regs[0], false);
+	pool->clock_sources[DCN302_CLK_SRC_PLL1] =
+			dcn302_clock_source_create(ctx, ctx->dc_bios,
+					CLOCK_SOURCE_COMBO_PHY_PLL1,
+					&clk_src_regs[1], false);
+	pool->clock_sources[DCN302_CLK_SRC_PLL2] =
+			dcn302_clock_source_create(ctx, ctx->dc_bios,
+					CLOCK_SOURCE_COMBO_PHY_PLL2,
+					&clk_src_regs[2], false);
+	pool->clock_sources[DCN302_CLK_SRC_PLL3] =
+			dcn302_clock_source_create(ctx, ctx->dc_bios,
+					CLOCK_SOURCE_COMBO_PHY_PLL3,
+					&clk_src_regs[3], false);
+	pool->clock_sources[DCN302_CLK_SRC_PLL4] =
+			dcn302_clock_source_create(ctx, ctx->dc_bios,
+					CLOCK_SOURCE_COMBO_PHY_PLL4,
+					&clk_src_regs[4], false);
+
+	pool->clk_src_count = DCN302_CLK_SRC_TOTAL;
+
+	/* todo: not reuse phy_pll registers */
+	pool->dp_clock_source =
+			dcn302_clock_source_create(ctx, ctx->dc_bios,
+					CLOCK_SOURCE_ID_DP_DTO,
+					&clk_src_regs[0], true);
+
+	for (i = 0; i < pool->clk_src_count; i++) {
+		if (pool->clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	/* DCCG */
+	pool->dccg = dccg30_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
+	if (pool->dccg == NULL) {
+		dm_error("DC: failed to create dccg!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* PP Lib and SMU interfaces */
+	init_soc_bounding_box(dc, pool);
+
+	/* DML */
+	dml_init_instance(&dc->dml, &dcn3_02_soc, &dcn3_02_ip, DML_PROJECT_DCN30);
+
+	/* IRQ */
+	init_data.ctx = dc->ctx;
+	pool->irqs = dal_irq_service_dcn302_create(&init_data);
+	if (!pool->irqs)
+		goto create_fail;
+
+	/* HUBBUB */
+	pool->hubbub = dcn302_hubbub_create(ctx);
+	if (pool->hubbub == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create hubbub!\n");
+		goto create_fail;
+	}
+
+	/* HUBPs, DPPs, OPPs and TGs */
+	for (i = 0; i < pool->pipe_count; i++) {
+		pool->hubps[i] = dcn302_hubp_create(ctx, i);
+		if (pool->hubps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create hubps!\n");
+			goto create_fail;
+		}
+
+		pool->dpps[i] = dcn302_dpp_create(ctx, i);
+		if (pool->dpps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create dpps!\n");
+			goto create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->res_cap->num_opp; i++) {
+		pool->opps[i] = dcn302_opp_create(ctx, i);
+		if (pool->opps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create output pixel processor!\n");
+			goto create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->res_cap->num_timing_generator; i++) {
+		pool->timing_generators[i] = dcn302_timing_generator_create(ctx, i);
+		if (pool->timing_generators[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto create_fail;
+		}
+	}
+	pool->timing_generator_count = i;
+
+	/* PSR */
+	pool->psr = dmub_psr_create(ctx);
+	if (pool->psr == NULL) {
+		dm_error("DC: failed to create psr!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* ABMs */
+	for (i = 0; i < pool->res_cap->num_timing_generator; i++) {
+		pool->multiple_abms[i] = dmub_abm_create(ctx, &abm_regs[i], &abm_shift, &abm_mask);
+		if (pool->multiple_abms[i] == NULL) {
+			dm_error("DC: failed to create abm for pipe %d!\n", i);
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	/* MPC and DSC */
+	pool->mpc = dcn302_mpc_create(ctx, pool->mpcc_count, pool->res_cap->num_mpc_3dlut);
+	if (pool->mpc == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mpc!\n");
+		goto create_fail;
+	}
+
+	for (i = 0; i < pool->res_cap->num_dsc; i++) {
+		pool->dscs[i] = dcn302_dsc_create(ctx, i);
+		if (pool->dscs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create display stream compressor %d!\n", i);
+			goto create_fail;
+		}
+	}
+
+	/* DWB and MMHUBBUB */
+	if (!dcn302_dwbc_create(ctx, pool)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create dwbc!\n");
+		goto create_fail;
+	}
+
+	if (!dcn302_mmhubbub_create(ctx, pool)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mcif_wb!\n");
+		goto create_fail;
+	}
+
+	/* AUX and I2C */
+	for (i = 0; i < pool->res_cap->num_ddc; i++) {
+		pool->engines[i] = dcn302_aux_engine_create(ctx, i);
+		if (pool->engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC:failed to create aux engine!!\n");
+			goto create_fail;
+		}
+		pool->hw_i2cs[i] = dcn302_i2c_hw_create(ctx, i);
+		if (pool->hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC:failed to create hw i2c!!\n");
+			goto create_fail;
+		}
+		pool->sw_i2cs[i] = NULL;
+	}
+
+	/* Audio, Stream Encoders including HPO and virtual, MPC 3D LUTs */
+	if (!resource_construct(num_virtual_links, dc, pool,
+			&res_create_funcs))
+		goto create_fail;
+
+	/* HW Sequencer and Plane caps */
+	dcn302_hw_sequencer_construct(dc);
+
+	dc->caps.max_planes =  pool->pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->cap_funcs = cap_funcs;
+
+	if (dc->ctx->dc_bios->fw_info.oem_i2c_present) {
+		ddc_init_data.ctx = dc->ctx;
+		ddc_init_data.link = NULL;
+		ddc_init_data.id.id = dc->ctx->dc_bios->fw_info.oem_i2c_obj_id;
+		ddc_init_data.id.enum_id = 0;
+		ddc_init_data.id.type = OBJECT_TYPE_GENERIC;
+		pool->oem_device = dc->link_srv->create_ddc_service(&ddc_init_data);
+	} else {
+		pool->oem_device = NULL;
+	}
+
+	return true;
+
+create_fail:
+
+	dcn302_resource_destruct(pool);
+
+	return false;
+}
+
+struct resource_pool *dcn302_create_resource_pool(const struct dc_init_data *init_data, struct dc *dc)
+{
+	struct resource_pool *pool = kzalloc(sizeof(struct resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dcn302_resource_construct(init_data->num_virtual_links, dc, pool))
+		return pool;
+
+	BREAK_TO_DEBUGGER();
+	kfree(pool);
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.h
new file mode 100644
index 000000000000..9f24e73b92b3
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2020 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef _DCN302_RESOURCE_H_
+#define _DCN302_RESOURCE_H_
+
+#include "core_types.h"
+
+extern struct _vcs_dpi_ip_params_st dcn3_02_ip;
+extern struct _vcs_dpi_soc_bounding_box_st dcn3_02_soc;
+
+struct resource_pool *dcn302_create_resource_pool(const struct dc_init_data *init_data, struct dc *dc);
+
+void dcn302_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params);
+
+#endif /* _DCN302_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c
new file mode 100644
index 000000000000..25cd6236b054
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c
@@ -0,0 +1,1448 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright (C) 2021 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ */
+
+#include "dcn303/dcn303_init.h"
+#include "dcn303_resource.h"
+#include "dcn303/dcn303_dccg.h"
+#include "irq/dcn303/irq_service_dcn303.h"
+
+#include "dcn30/dcn30_dio_link_encoder.h"
+#include "dcn30/dcn30_dio_stream_encoder.h"
+#include "dcn30/dcn30_dpp.h"
+#include "dcn30/dcn30_dwb.h"
+#include "dcn30/dcn30_hubbub.h"
+#include "dcn30/dcn30_hubp.h"
+#include "dcn30/dcn30_mmhubbub.h"
+#include "dcn30/dcn30_mpc.h"
+#include "dcn30/dcn30_opp.h"
+#include "dcn30/dcn30_optc.h"
+#include "dcn30/dcn30_resource.h"
+
+#include "dcn20/dcn20_dsc.h"
+#include "dcn20/dcn20_resource.h"
+
+#include "dml/dcn30/dcn30_fpu.h"
+
+#include "dcn10/dcn10_resource.h"
+
+#include "link.h"
+
+#include "dce/dce_abm.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_hwseq.h"
+#include "dce/dce_i2c_hw.h"
+#include "dce/dce_panel_cntl.h"
+#include "dce/dmub_abm.h"
+#include "dce/dmub_psr.h"
+#include "clk_mgr.h"
+
+#include "hw_sequencer_private.h"
+#include "reg_helper.h"
+#include "resource.h"
+#include "vm_helper.h"
+
+#include "sienna_cichlid_ip_offset.h"
+#include "dcn/dcn_3_0_3_offset.h"
+#include "dcn/dcn_3_0_3_sh_mask.h"
+#include "dpcs/dpcs_3_0_3_offset.h"
+#include "dpcs/dpcs_3_0_3_sh_mask.h"
+#include "nbio/nbio_2_3_offset.h"
+
+#include "dml/dcn303/dcn303_fpu.h"
+
+#define DC_LOGGER \
+	dc->ctx->logger
+#define DC_LOGGER_INIT(logger)
+
+
+static const struct dc_debug_options debug_defaults_drv = {
+		.disable_dmcu = true,
+		.force_abm_enable = false,
+		.timing_trace = false,
+		.clock_trace = true,
+		.disable_pplib_clock_request = true,
+		.pipe_split_policy = MPC_SPLIT_AVOID,
+		.force_single_disp_pipe_split = false,
+		.disable_dcc = DCC_ENABLE,
+		.vsr_support = true,
+		.performance_trace = false,
+		.max_downscale_src_width = 7680,/*upto 8K*/
+		.disable_pplib_wm_range = false,
+		.scl_reset_length10 = true,
+		.sanity_checks = false,
+		.underflow_assert_delay_us = 0xFFFFFFFF,
+		.dwb_fi_phase = -1, // -1 = disable,
+		.dmub_command_table = true,
+		.exit_idle_opt_for_cursor_updates = true,
+		.disable_idle_power_optimizations = false,
+		.using_dml2 = false,
+};
+
+static const struct dc_panel_config panel_config_defaults = {
+		.psr = {
+			.disable_psr = false,
+			.disallow_psrsu = false,
+			.disallow_replay = false,
+		},
+};
+
+enum dcn303_clk_src_array_id {
+	DCN303_CLK_SRC_PLL0,
+	DCN303_CLK_SRC_PLL1,
+	DCN303_CLK_SRC_TOTAL
+};
+
+static const struct resource_caps res_cap_dcn303 = {
+		.num_timing_generator = 2,
+		.num_opp = 2,
+		.num_video_plane = 2,
+		.num_audio = 2,
+		.num_stream_encoder = 2,
+		.num_dwb = 1,
+		.num_ddc = 2,
+		.num_vmid = 16,
+		.num_mpc_3dlut = 1,
+		.num_dsc = 2,
+};
+
+static const struct dc_plane_cap plane_cap = {
+		.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
+		.per_pixel_alpha = true,
+		.pixel_format_support = {
+				.argb8888 = true,
+				.nv12 = true,
+				.fp16 = true,
+				.p010 = true,
+				.ayuv = false,
+		},
+		.max_upscale_factor = {
+				.argb8888 = 16000,
+				.nv12 = 16000,
+				.fp16 = 16000
+		},
+		.max_downscale_factor = {
+				.argb8888 = 600,
+				.nv12 = 600,
+				.fp16 = 600
+		},
+		16,
+		16
+};
+
+/* NBIO */
+#define NBIO_BASE_INNER(seg) \
+		NBIO_BASE__INST0_SEG ## seg
+
+#define NBIO_BASE(seg) \
+		NBIO_BASE_INNER(seg)
+
+#define NBIO_SR(reg_name)\
+		.reg_name = NBIO_BASE(mm ## reg_name ## _BASE_IDX) + \
+		mm ## reg_name
+
+/* DCN */
+#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
+
+#define BASE(seg) BASE_INNER(seg)
+
+#define SR(reg_name)\
+		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) + mm ## reg_name
+
+#define SF(reg_name, field_name, post_fix)\
+		.field_name = reg_name ## __ ## field_name ## post_fix
+
+#define SRI(reg_name, block, id)\
+		.reg_name = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + mm ## block ## id ## _ ## reg_name
+
+#define SRI2(reg_name, block, id)\
+		.reg_name = BASE(mm ## reg_name ## _BASE_IDX) + mm ## reg_name
+
+#define SRII(reg_name, block, id)\
+		.reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		mm ## block ## id ## _ ## reg_name
+
+#define DCCG_SRII(reg_name, block, id)\
+		.block ## _ ## reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		mm ## block ## id ## _ ## reg_name
+
+#define VUPDATE_SRII(reg_name, block, id)\
+		.reg_name[id] = BASE(mm ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
+		mm ## reg_name ## _ ## block ## id
+
+#define SRII_DWB(reg_name, temp_name, block, id)\
+		.reg_name[id] = BASE(mm ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
+		mm ## block ## id ## _ ## temp_name
+
+#define SF_DWB2(reg_name, block, id, field_name, post_fix)	\
+	.field_name = reg_name ## __ ## field_name ## post_fix
+
+#define SRII_MPC_RMU(reg_name, block, id)\
+		.RMU##_##reg_name[id] = BASE(mm ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		mm ## block ## id ## _ ## reg_name
+
+static const struct dcn_hubbub_registers hubbub_reg = {
+		HUBBUB_REG_LIST_DCN30(0)
+};
+
+static const struct dcn_hubbub_shift hubbub_shift = {
+		HUBBUB_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn_hubbub_mask hubbub_mask = {
+		HUBBUB_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define vmid_regs(id)\
+		[id] = { DCN20_VMID_REG_LIST(id) }
+
+static const struct dcn_vmid_registers vmid_regs[] = {
+		vmid_regs(0),
+		vmid_regs(1),
+		vmid_regs(2),
+		vmid_regs(3),
+		vmid_regs(4),
+		vmid_regs(5),
+		vmid_regs(6),
+		vmid_regs(7),
+		vmid_regs(8),
+		vmid_regs(9),
+		vmid_regs(10),
+		vmid_regs(11),
+		vmid_regs(12),
+		vmid_regs(13),
+		vmid_regs(14),
+		vmid_regs(15)
+};
+
+static const struct dcn20_vmid_shift vmid_shifts = {
+		DCN20_VMID_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn20_vmid_mask vmid_masks = {
+		DCN20_VMID_MASK_SH_LIST(_MASK)
+};
+
+static struct hubbub *dcn303_hubbub_create(struct dc_context *ctx)
+{
+	int i;
+
+	struct dcn20_hubbub *hubbub3 = kzalloc(sizeof(struct dcn20_hubbub), GFP_KERNEL);
+
+	if (!hubbub3)
+		return NULL;
+
+	hubbub3_construct(hubbub3, ctx, &hubbub_reg, &hubbub_shift, &hubbub_mask);
+
+	for (i = 0; i < res_cap_dcn303.num_vmid; i++) {
+		struct dcn20_vmid *vmid = &hubbub3->vmid[i];
+
+		vmid->ctx = ctx;
+
+		vmid->regs = &vmid_regs[i];
+		vmid->shifts = &vmid_shifts;
+		vmid->masks = &vmid_masks;
+	}
+
+	return &hubbub3->base;
+}
+
+#define vpg_regs(id)\
+		[id] = { VPG_DCN3_REG_LIST(id) }
+
+static const struct dcn30_vpg_registers vpg_regs[] = {
+		vpg_regs(0),
+		vpg_regs(1),
+		vpg_regs(2)
+};
+
+static const struct dcn30_vpg_shift vpg_shift = {
+		DCN3_VPG_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn30_vpg_mask vpg_mask = {
+		DCN3_VPG_MASK_SH_LIST(_MASK)
+};
+
+static struct vpg *dcn303_vpg_create(struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn30_vpg *vpg3 = kzalloc(sizeof(struct dcn30_vpg), GFP_KERNEL);
+
+	if (!vpg3)
+		return NULL;
+
+	vpg3_construct(vpg3, ctx, inst, &vpg_regs[inst], &vpg_shift, &vpg_mask);
+
+	return &vpg3->base;
+}
+
+#define afmt_regs(id)\
+		[id] = { AFMT_DCN3_REG_LIST(id) }
+
+static const struct dcn30_afmt_registers afmt_regs[] = {
+		afmt_regs(0),
+		afmt_regs(1),
+		afmt_regs(2)
+};
+
+static const struct dcn30_afmt_shift afmt_shift = {
+		DCN3_AFMT_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn30_afmt_mask afmt_mask = {
+		DCN3_AFMT_MASK_SH_LIST(_MASK)
+};
+
+static struct afmt *dcn303_afmt_create(struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn30_afmt *afmt3 = kzalloc(sizeof(struct dcn30_afmt), GFP_KERNEL);
+
+	if (!afmt3)
+		return NULL;
+
+	afmt3_construct(afmt3, ctx, inst, &afmt_regs[inst], &afmt_shift, &afmt_mask);
+
+	return &afmt3->base;
+}
+
+#define audio_regs(id)\
+		[id] = { AUD_COMMON_REG_LIST(id) }
+
+static const struct dce_audio_registers audio_regs[] = {
+		audio_regs(0),
+		audio_regs(1),
+		audio_regs(2),
+		audio_regs(3),
+		audio_regs(4),
+		audio_regs(5),
+		audio_regs(6)
+};
+
+#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
+		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
+
+static const struct dce_audio_shift audio_shift = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_audio_mask audio_mask = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+static struct audio *dcn303_create_audio(struct dc_context *ctx, unsigned int inst)
+{
+	return dce_audio_create(ctx, inst, &audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+#define stream_enc_regs(id)\
+		[id] = { SE_DCN3_REG_LIST(id) }
+
+static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
+		stream_enc_regs(0),
+		stream_enc_regs(1)
+};
+
+static const struct dcn10_stream_encoder_shift se_shift = {
+		SE_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn10_stream_encoder_mask se_mask = {
+		SE_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static struct stream_encoder *dcn303_stream_encoder_create(enum engine_id eng_id, struct dc_context *ctx)
+{
+	struct dcn10_stream_encoder *enc1;
+	struct vpg *vpg;
+	struct afmt *afmt;
+	int vpg_inst;
+	int afmt_inst;
+
+	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
+	if (eng_id <= ENGINE_ID_DIGB) {
+		vpg_inst = eng_id;
+		afmt_inst = eng_id;
+	} else
+		return NULL;
+
+	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
+	vpg = dcn303_vpg_create(ctx, vpg_inst);
+	afmt = dcn303_afmt_create(ctx, afmt_inst);
+
+	if (!enc1 || !vpg || !afmt) {
+		kfree(enc1);
+		kfree(vpg);
+		kfree(afmt);
+		return NULL;
+	}
+
+	dcn30_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios, eng_id, vpg, afmt, &stream_enc_regs[eng_id],
+			&se_shift, &se_mask);
+
+	return &enc1->base;
+}
+
+#define clk_src_regs(index, pllid)\
+		[index] = { CS_COMMON_REG_LIST_DCN3_03(index, pllid) }
+
+static const struct dce110_clk_src_regs clk_src_regs[] = {
+		clk_src_regs(0, A),
+		clk_src_regs(1, B)
+};
+
+static const struct dce110_clk_src_shift cs_shift = {
+		CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
+};
+
+static const struct dce110_clk_src_mask cs_mask = {
+		CS_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
+};
+
+static struct clock_source *dcn303_clock_source_create(struct dc_context *ctx, struct dc_bios *bios,
+		enum clock_source_id id, const struct dce110_clk_src_regs *regs, bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src = kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dcn3_clk_src_construct(clk_src, ctx, bios, id, regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	kfree(clk_src);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static const struct dce_hwseq_registers hwseq_reg = {
+		HWSEQ_DCN303_REG_LIST()
+};
+
+static const struct dce_hwseq_shift hwseq_shift = {
+		HWSEQ_DCN303_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+		HWSEQ_DCN303_MASK_SH_LIST(_MASK)
+};
+
+static struct dce_hwseq *dcn303_hwseq_create(struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &hwseq_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+	}
+	return hws;
+}
+
+#define hubp_regs(id)\
+		[id] = { HUBP_REG_LIST_DCN30(id) }
+
+static const struct dcn_hubp2_registers hubp_regs[] = {
+		hubp_regs(0),
+		hubp_regs(1)
+};
+
+static const struct dcn_hubp2_shift hubp_shift = {
+		HUBP_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn_hubp2_mask hubp_mask = {
+		HUBP_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static struct hubp *dcn303_hubp_create(struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_hubp *hubp2 = kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
+
+	if (!hubp2)
+		return NULL;
+
+	if (hubp3_construct(hubp2, ctx, inst, &hubp_regs[inst], &hubp_shift, &hubp_mask))
+		return &hubp2->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(hubp2);
+	return NULL;
+}
+
+#define dpp_regs(id)\
+		[id] = { DPP_REG_LIST_DCN30(id) }
+
+static const struct dcn3_dpp_registers dpp_regs[] = {
+		dpp_regs(0),
+		dpp_regs(1)
+};
+
+static const struct dcn3_dpp_shift tf_shift = {
+		DPP_REG_LIST_SH_MASK_DCN30(__SHIFT)
+};
+
+static const struct dcn3_dpp_mask tf_mask = {
+		DPP_REG_LIST_SH_MASK_DCN30(_MASK)
+};
+
+static struct dpp *dcn303_dpp_create(struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn3_dpp *dpp = kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
+
+	if (!dpp)
+		return NULL;
+
+	if (dpp3_construct(dpp, ctx, inst, &dpp_regs[inst], &tf_shift, &tf_mask))
+		return &dpp->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(dpp);
+	return NULL;
+}
+
+#define opp_regs(id)\
+		[id] = { OPP_REG_LIST_DCN30(id) }
+
+static const struct dcn20_opp_registers opp_regs[] = {
+		opp_regs(0),
+		opp_regs(1)
+};
+
+static const struct dcn20_opp_shift opp_shift = {
+		OPP_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn20_opp_mask opp_mask = {
+		OPP_MASK_SH_LIST_DCN20(_MASK)
+};
+
+static struct output_pixel_processor *dcn303_opp_create(struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_opp *opp = kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
+
+	if (!opp) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dcn20_opp_construct(opp, ctx, inst, &opp_regs[inst], &opp_shift, &opp_mask);
+	return &opp->base;
+}
+
+#define optc_regs(id)\
+		[id] = { OPTC_COMMON_REG_LIST_DCN3_0(id) }
+
+static const struct dcn_optc_registers optc_regs[] = {
+		optc_regs(0),
+		optc_regs(1)
+};
+
+static const struct dcn_optc_shift optc_shift = {
+		OPTC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn_optc_mask optc_mask = {
+		OPTC_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static struct timing_generator *dcn303_timing_generator_create(struct dc_context *ctx, uint32_t instance)
+{
+	struct optc *tgn10 = kzalloc(sizeof(struct optc), GFP_KERNEL);
+
+	if (!tgn10)
+		return NULL;
+
+	tgn10->base.inst = instance;
+	tgn10->base.ctx = ctx;
+
+	tgn10->tg_regs = &optc_regs[instance];
+	tgn10->tg_shift = &optc_shift;
+	tgn10->tg_mask = &optc_mask;
+
+	dcn30_timing_generator_init(tgn10);
+
+	return &tgn10->base;
+}
+
+static const struct dcn30_mpc_registers mpc_regs = {
+		MPC_REG_LIST_DCN3_0(0),
+		MPC_REG_LIST_DCN3_0(1),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(0),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(1),
+		MPC_RMU_GLOBAL_REG_LIST_DCN3AG,
+		MPC_RMU_REG_LIST_DCN3AG(0),
+		MPC_DWB_MUX_REG_LIST_DCN3_0(0),
+};
+
+static const struct dcn30_mpc_shift mpc_shift = {
+		MPC_COMMON_MASK_SH_LIST_DCN303(__SHIFT)
+};
+
+static const struct dcn30_mpc_mask mpc_mask = {
+		MPC_COMMON_MASK_SH_LIST_DCN303(_MASK)
+};
+
+static struct mpc *dcn303_mpc_create(struct dc_context *ctx, int num_mpcc, int num_rmu)
+{
+	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc), GFP_KERNEL);
+
+	if (!mpc30)
+		return NULL;
+
+	dcn30_mpc_construct(mpc30, ctx, &mpc_regs, &mpc_shift, &mpc_mask, num_mpcc, num_rmu);
+
+	return &mpc30->base;
+}
+
+#define dsc_regsDCN20(id)\
+[id] = { DSC_REG_LIST_DCN20(id) }
+
+static const struct dcn20_dsc_registers dsc_regs[] = {
+		dsc_regsDCN20(0),
+		dsc_regsDCN20(1)
+};
+
+static const struct dcn20_dsc_shift dsc_shift = {
+		DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
+};
+
+static const struct dcn20_dsc_mask dsc_mask = {
+		DSC_REG_LIST_SH_MASK_DCN20(_MASK)
+};
+
+static struct display_stream_compressor *dcn303_dsc_create(struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_dsc *dsc = kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
+
+	if (!dsc) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
+	return &dsc->base;
+}
+
+#define dwbc_regs_dcn3(id)\
+[id] = { DWBC_COMMON_REG_LIST_DCN30(id) }
+
+static const struct dcn30_dwbc_registers dwbc30_regs[] = {
+		dwbc_regs_dcn3(0)
+};
+
+static const struct dcn30_dwbc_shift dwbc30_shift = {
+		DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_dwbc_mask dwbc30_mask = {
+		DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static bool dcn303_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc), GFP_KERNEL);
+
+		if (!dwbc30) {
+			dm_error("DC: failed to create dwbc30!\n");
+			return false;
+		}
+
+		dcn30_dwbc_construct(dwbc30, ctx, &dwbc30_regs[i], &dwbc30_shift, &dwbc30_mask, i);
+
+		pool->dwbc[i] = &dwbc30->base;
+	}
+	return true;
+}
+
+#define mcif_wb_regs_dcn3(id)\
+[id] = { MCIF_WB_COMMON_REG_LIST_DCN30(id) }
+
+static const struct dcn30_mmhubbub_registers mcif_wb30_regs[] = {
+		mcif_wb_regs_dcn3(0)
+};
+
+static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
+		MCIF_WB_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
+		MCIF_WB_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static bool dcn303_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub), GFP_KERNEL);
+
+		if (!mcif_wb30) {
+			dm_error("DC: failed to create mcif_wb30!\n");
+			return false;
+		}
+
+		dcn30_mmhubbub_construct(mcif_wb30, ctx, &mcif_wb30_regs[i], &mcif_wb30_shift, &mcif_wb30_mask, i);
+
+		pool->mcif_wb[i] = &mcif_wb30->base;
+	}
+	return true;
+}
+
+#define aux_engine_regs(id)\
+[id] = {\
+		AUX_COMMON_REG_LIST0(id), \
+		.AUXN_IMPCAL = 0, \
+		.AUXP_IMPCAL = 0, \
+		.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1)
+};
+
+static const struct dce110_aux_registers_shift aux_shift = {
+		DCN_AUX_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+		DCN_AUX_MASK_SH_LIST(_MASK)
+};
+
+static struct dce_aux *dcn303_aux_engine_create(struct dc_context *ctx, uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine = kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst, SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+			&aux_engine_regs[inst], &aux_mask, &aux_shift, ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+
+#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST(id) }
+
+static const struct dce_i2c_registers i2c_hw_regs[] = {
+		i2c_inst_regs(1),
+		i2c_inst_regs(2)
+};
+
+static const struct dce_i2c_shift i2c_shifts = {
+		I2C_COMMON_MASK_SH_LIST_DCN2(__SHIFT)
+};
+
+static const struct dce_i2c_mask i2c_masks = {
+		I2C_COMMON_MASK_SH_LIST_DCN2(_MASK)
+};
+
+static struct dce_i2c_hw *dcn303_i2c_hw_create(struct dc_context *ctx, uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw = kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst, &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+		.max_hdmi_deep_color = COLOR_DEPTH_121212,
+		.max_hdmi_pixel_clock = 600000,
+		.hdmi_ycbcr420_supported = true,
+		.dp_ycbcr420_supported = true,
+		.fec_supported = true,
+		.flags.bits.IS_HBR2_CAPABLE = true,
+		.flags.bits.IS_HBR3_CAPABLE = true,
+		.flags.bits.IS_TPS3_CAPABLE = true,
+		.flags.bits.IS_TPS4_CAPABLE = true
+};
+
+#define link_regs(id, phyid)\
+		[id] = {\
+				LE_DCN3_REG_LIST(id), \
+				UNIPHY_DCN2_REG_LIST(phyid), \
+				SRI(DP_DPHY_INTERNAL_CTRL, DP, id) \
+		}
+
+static const struct dcn10_link_enc_registers link_enc_regs[] = {
+		link_regs(0, A),
+		link_regs(1, B)
+};
+
+static const struct dcn10_link_enc_shift le_shift = {
+		LINK_ENCODER_MASK_SH_LIST_DCN30(__SHIFT),
+		DPCS_DCN2_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn10_link_enc_mask le_mask = {
+		LINK_ENCODER_MASK_SH_LIST_DCN30(_MASK),
+		DPCS_DCN2_MASK_SH_LIST(_MASK)
+};
+
+#define aux_regs(id)\
+		[id] = { DCN2_AUX_REG_LIST(id) }
+
+static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
+		aux_regs(0),
+		aux_regs(1)
+};
+
+#define hpd_regs(id)\
+		[id] = { HPD_REG_LIST(id) }
+
+static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
+		hpd_regs(0),
+		hpd_regs(1)
+};
+
+static struct link_encoder *dcn303_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dcn20_link_encoder *enc20 = kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+
+	if (!enc20)
+		return NULL;
+
+	dcn30_link_encoder_construct(enc20, enc_init_data, &link_enc_feature,
+			&link_enc_regs[enc_init_data->transmitter], &link_enc_aux_regs[enc_init_data->channel - 1],
+			&link_enc_hpd_regs[enc_init_data->hpd_source], &le_shift, &le_mask);
+
+	return &enc20->enc10.base;
+}
+
+static const struct dce_panel_cntl_registers panel_cntl_regs[] = {
+		{ DCN_PANEL_CNTL_REG_LIST() }
+};
+
+static const struct dce_panel_cntl_shift panel_cntl_shift = {
+		DCE_PANEL_CNTL_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_panel_cntl_mask panel_cntl_mask = {
+		DCE_PANEL_CNTL_MASK_SH_LIST(_MASK)
+};
+
+static struct panel_cntl *dcn303_panel_cntl_create(const struct panel_cntl_init_data *init_data)
+{
+	struct dce_panel_cntl *panel_cntl = kzalloc(sizeof(struct dce_panel_cntl), GFP_KERNEL);
+
+	if (!panel_cntl)
+		return NULL;
+
+	dce_panel_cntl_construct(panel_cntl, init_data, &panel_cntl_regs[init_data->inst],
+			&panel_cntl_shift, &panel_cntl_mask);
+
+	return &panel_cntl->base;
+}
+
+static void read_dce_straps(struct dc_context *ctx, struct resource_straps *straps)
+{
+	generic_reg_get(ctx, mmDC_PINSTRAPS + BASE(mmDC_PINSTRAPS_BASE_IDX),
+			FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
+}
+
+static const struct resource_create_funcs res_create_funcs = {
+		.read_dce_straps = read_dce_straps,
+		.create_audio = dcn303_create_audio,
+		.create_stream_encoder = dcn303_stream_encoder_create,
+		.create_hwseq = dcn303_hwseq_create,
+};
+
+static bool is_soc_bounding_box_valid(struct dc *dc)
+{
+	uint32_t hw_internal_rev = dc->ctx->asic_id.hw_internal_rev;
+
+	if (ASICREV_IS_BEIGE_GOBY_P(hw_internal_rev))
+		return true;
+
+	return false;
+}
+
+static bool init_soc_bounding_box(struct dc *dc,  struct resource_pool *pool)
+{
+	struct _vcs_dpi_soc_bounding_box_st *loaded_bb = &dcn3_03_soc;
+	struct _vcs_dpi_ip_params_st *loaded_ip = &dcn3_03_ip;
+
+	DC_LOGGER_INIT(dc->ctx->logger);
+
+	if (!is_soc_bounding_box_valid(dc)) {
+		DC_LOG_ERROR("%s: not valid soc bounding box/n", __func__);
+		return false;
+	}
+
+	loaded_ip->max_num_otg = pool->pipe_count;
+	loaded_ip->max_num_dpp = pool->pipe_count;
+	loaded_ip->clamp_min_dcfclk = dc->config.clamp_min_dcfclk;
+	DC_FP_START();
+	dcn20_patch_bounding_box(dc, loaded_bb);
+	DC_FP_END();
+
+	if (dc->ctx->dc_bios->funcs->get_soc_bb_info) {
+		struct bp_soc_bb_info bb_info = { 0 };
+
+		if (dc->ctx->dc_bios->funcs->get_soc_bb_info(
+			    dc->ctx->dc_bios, &bb_info) == BP_RESULT_OK) {
+					DC_FP_START();
+					dcn303_fpu_init_soc_bounding_box(bb_info);
+					DC_FP_END();
+		}
+	}
+
+	return true;
+}
+
+static void dcn303_resource_destruct(struct resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->stream_enc_count; i++) {
+		if (pool->stream_enc[i] != NULL) {
+			if (pool->stream_enc[i]->vpg != NULL) {
+				kfree(DCN30_VPG_FROM_VPG(pool->stream_enc[i]->vpg));
+				pool->stream_enc[i]->vpg = NULL;
+			}
+			if (pool->stream_enc[i]->afmt != NULL) {
+				kfree(DCN30_AFMT_FROM_AFMT(pool->stream_enc[i]->afmt));
+				pool->stream_enc[i]->afmt = NULL;
+			}
+			kfree(DCN10STRENC_FROM_STRENC(pool->stream_enc[i]));
+			pool->stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->res_cap->num_dsc; i++) {
+		if (pool->dscs[i] != NULL)
+			dcn20_dsc_destroy(&pool->dscs[i]);
+	}
+
+	if (pool->mpc != NULL) {
+		kfree(TO_DCN20_MPC(pool->mpc));
+		pool->mpc = NULL;
+	}
+
+	if (pool->hubbub != NULL) {
+		kfree(pool->hubbub);
+		pool->hubbub = NULL;
+	}
+
+	for (i = 0; i < pool->pipe_count; i++) {
+		if (pool->dpps[i] != NULL) {
+			kfree(TO_DCN20_DPP(pool->dpps[i]));
+			pool->dpps[i] = NULL;
+		}
+
+		if (pool->hubps[i] != NULL) {
+			kfree(TO_DCN20_HUBP(pool->hubps[i]));
+			pool->hubps[i] = NULL;
+		}
+
+		if (pool->irqs != NULL)
+			dal_irq_service_destroy(&pool->irqs);
+	}
+
+	for (i = 0; i < pool->res_cap->num_ddc; i++) {
+		if (pool->engines[i] != NULL)
+			dce110_engine_destroy(&pool->engines[i]);
+		if (pool->hw_i2cs[i] != NULL) {
+			kfree(pool->hw_i2cs[i]);
+			pool->hw_i2cs[i] = NULL;
+		}
+		if (pool->sw_i2cs[i] != NULL) {
+			kfree(pool->sw_i2cs[i]);
+			pool->sw_i2cs[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->res_cap->num_opp; i++) {
+		if (pool->opps[i] != NULL)
+			pool->opps[i]->funcs->opp_destroy(&pool->opps[i]);
+	}
+
+	for (i = 0; i < pool->res_cap->num_timing_generator; i++) {
+		if (pool->timing_generators[i] != NULL)	{
+			kfree(DCN10TG_FROM_TG(pool->timing_generators[i]));
+			pool->timing_generators[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->res_cap->num_dwb; i++) {
+		if (pool->dwbc[i] != NULL) {
+			kfree(TO_DCN30_DWBC(pool->dwbc[i]));
+			pool->dwbc[i] = NULL;
+		}
+		if (pool->mcif_wb[i] != NULL) {
+			kfree(TO_DCN30_MMHUBBUB(pool->mcif_wb[i]));
+			pool->mcif_wb[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->audio_count; i++) {
+		if (pool->audios[i])
+			dce_aud_destroy(&pool->audios[i]);
+	}
+
+	for (i = 0; i < pool->clk_src_count; i++) {
+		if (pool->clock_sources[i] != NULL)
+			dcn20_clock_source_destroy(&pool->clock_sources[i]);
+	}
+
+	if (pool->dp_clock_source != NULL)
+		dcn20_clock_source_destroy(&pool->dp_clock_source);
+
+	for (i = 0; i < pool->res_cap->num_mpc_3dlut; i++) {
+		if (pool->mpc_lut[i] != NULL) {
+			dc_3dlut_func_release(pool->mpc_lut[i]);
+			pool->mpc_lut[i] = NULL;
+		}
+		if (pool->mpc_shaper[i] != NULL) {
+			dc_transfer_func_release(pool->mpc_shaper[i]);
+			pool->mpc_shaper[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->pipe_count; i++) {
+		if (pool->multiple_abms[i] != NULL)
+			dce_abm_destroy(&pool->multiple_abms[i]);
+	}
+
+	if (pool->psr != NULL)
+		dmub_psr_destroy(&pool->psr);
+
+	if (pool->dccg != NULL)
+		dcn_dccg_destroy(&pool->dccg);
+
+	if (pool->oem_device != NULL) {
+		struct dc *dc = pool->oem_device->ctx->dc;
+
+		dc->link_srv->destroy_ddc_service(&pool->oem_device);
+	}
+}
+
+static void dcn303_destroy_resource_pool(struct resource_pool **pool)
+{
+	dcn303_resource_destruct(*pool);
+	kfree(*pool);
+	*pool = NULL;
+}
+
+static void dcn303_get_panel_config_defaults(struct dc_panel_config *panel_config)
+{
+	*panel_config = panel_config_defaults;
+}
+
+void dcn303_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
+{
+	DC_FP_START();
+	dcn303_fpu_update_bw_bounding_box(dc, bw_params);
+	DC_FP_END();
+}
+
+static struct resource_funcs dcn303_res_pool_funcs = {
+		.destroy = dcn303_destroy_resource_pool,
+		.link_enc_create = dcn303_link_encoder_create,
+		.panel_cntl_create = dcn303_panel_cntl_create,
+		.validate_bandwidth = dcn30_validate_bandwidth,
+		.calculate_wm_and_dlg = dcn30_calculate_wm_and_dlg,
+		.update_soc_for_wm_a = dcn30_update_soc_for_wm_a,
+		.populate_dml_pipes = dcn30_populate_dml_pipes_from_context,
+		.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
+		.release_pipe = dcn20_release_pipe,
+		.add_stream_to_ctx = dcn30_add_stream_to_ctx,
+		.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
+		.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
+		.populate_dml_writeback_from_context = dcn30_populate_dml_writeback_from_context,
+		.set_mcif_arb_params = dcn30_set_mcif_arb_params,
+		.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
+		.acquire_post_bldn_3dlut = dcn30_acquire_post_bldn_3dlut,
+		.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
+		.update_bw_bounding_box = dcn303_update_bw_bounding_box,
+		.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
+		.get_panel_config_defaults = dcn303_get_panel_config_defaults,
+};
+
+static struct dc_cap_funcs cap_funcs = {
+		.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
+};
+
+static const struct bios_registers bios_regs = {
+		NBIO_SR(BIOS_SCRATCH_3),
+		NBIO_SR(BIOS_SCRATCH_6)
+};
+
+static const struct dccg_registers dccg_regs = {
+		DCCG_REG_LIST_DCN3_03()
+};
+
+static const struct dccg_shift dccg_shift = {
+		DCCG_MASK_SH_LIST_DCN3_03(__SHIFT)
+};
+
+static const struct dccg_mask dccg_mask = {
+		DCCG_MASK_SH_LIST_DCN3_03(_MASK)
+};
+
+#define abm_regs(id)\
+		[id] = { ABM_DCN302_REG_LIST(id) }
+
+static const struct dce_abm_registers abm_regs[] = {
+		abm_regs(0),
+		abm_regs(1)
+};
+
+static const struct dce_abm_shift abm_shift = {
+		ABM_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dce_abm_mask abm_mask = {
+		ABM_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static bool dcn303_resource_construct(
+		uint8_t num_virtual_links,
+		struct dc *dc,
+		struct resource_pool *pool)
+{
+	int i;
+	struct dc_context *ctx = dc->ctx;
+	struct irq_service_init_data init_data;
+	struct ddc_service_init_data ddc_init_data;
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	pool->res_cap = &res_cap_dcn303;
+
+	pool->funcs = &dcn303_res_pool_funcs;
+
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+	pool->underlay_pipe_index = NO_UNDERLAY_PIPE;
+	pool->pipe_count = pool->res_cap->num_timing_generator;
+	pool->mpcc_count = pool->res_cap->num_timing_generator;
+	dc->caps.max_downscale_ratio = 600;
+	dc->caps.i2c_speed_in_khz = 100;
+	dc->caps.i2c_speed_in_khz_hdcp = 5; /*1.4 w/a applied by derfault*/
+	dc->caps.max_cursor_size = 256;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.dmdata_alloc_size = 2048;
+	dc->caps.mall_size_per_mem_channel = 4;
+	/* total size = mall per channel * num channels * 1024 * 1024 */
+	dc->caps.mall_size_total = dc->caps.mall_size_per_mem_channel *
+				   dc->ctx->dc_bios->vram_info.num_chans *
+				   1024 * 1024;
+	dc->caps.cursor_cache_size =
+		dc->caps.max_cursor_size * dc->caps.max_cursor_size * 8;
+	dc->caps.max_slave_planes = 1;
+	dc->caps.post_blend_color_processing = true;
+	dc->caps.force_dp_tps4_for_cp2520 = true;
+	dc->caps.extended_aux_timeout_support = true;
+	dc->caps.dmcub_support = true;
+	dc->caps.max_v_total = (1 << 15) - 1;
+
+	/* Color pipeline capabilities */
+	dc->caps.color.dpp.dcn_arch = 1;
+	dc->caps.color.dpp.input_lut_shared = 0;
+	dc->caps.color.dpp.icsc = 1;
+	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
+	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
+	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
+	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
+	dc->caps.color.dpp.post_csc = 1;
+	dc->caps.color.dpp.gamma_corr = 1;
+	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
+
+	dc->caps.color.dpp.hw_3d_lut = 1;
+	dc->caps.color.dpp.ogam_ram = 1;
+	// no OGAM ROM on DCN3
+	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
+	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
+	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
+	dc->caps.color.dpp.ocsc = 0;
+
+	dc->caps.color.mpc.gamut_remap = 1;
+	dc->caps.color.mpc.num_3dluts = pool->res_cap->num_mpc_3dlut; //3
+	dc->caps.color.mpc.ogam_ram = 1;
+	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
+	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
+	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
+	dc->caps.color.mpc.ocsc = 1;
+
+	dc->caps.dp_hdmi21_pcon_support = true;
+
+	dc->config.dc_mode_clk_limit_support = true;
+	/* read VBIOS LTTPR caps */
+	if (ctx->dc_bios->funcs->get_lttpr_caps) {
+		enum bp_result bp_query_result;
+		uint8_t is_vbios_lttpr_enable = 0;
+
+		bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
+		dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
+	}
+
+	if (ctx->dc_bios->funcs->get_lttpr_interop) {
+		enum bp_result bp_query_result;
+		uint8_t is_vbios_interop_enabled = 0;
+
+		bp_query_result = ctx->dc_bios->funcs->get_lttpr_interop(ctx->dc_bios, &is_vbios_interop_enabled);
+		dc->caps.vbios_lttpr_aware = (bp_query_result == BP_RESULT_OK) && !!is_vbios_interop_enabled;
+	}
+
+	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
+		dc->debug = debug_defaults_drv;
+
+	// Init the vm_helper
+	if (dc->vm_helper)
+		vm_helper_init(dc->vm_helper, 16);
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	/* Clock Sources for Pixel Clock*/
+	pool->clock_sources[DCN303_CLK_SRC_PLL0] =
+			dcn303_clock_source_create(ctx, ctx->dc_bios,
+					CLOCK_SOURCE_COMBO_PHY_PLL0,
+					&clk_src_regs[0], false);
+	pool->clock_sources[DCN303_CLK_SRC_PLL1] =
+			dcn303_clock_source_create(ctx, ctx->dc_bios,
+					CLOCK_SOURCE_COMBO_PHY_PLL1,
+					&clk_src_regs[1], false);
+
+	pool->clk_src_count = DCN303_CLK_SRC_TOTAL;
+
+	/* todo: not reuse phy_pll registers */
+	pool->dp_clock_source =
+			dcn303_clock_source_create(ctx, ctx->dc_bios,
+					CLOCK_SOURCE_ID_DP_DTO,
+					&clk_src_regs[0], true);
+
+	for (i = 0; i < pool->clk_src_count; i++) {
+		if (pool->clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	/* DCCG */
+	pool->dccg = dccg30_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
+	if (pool->dccg == NULL) {
+		dm_error("DC: failed to create dccg!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* PP Lib and SMU interfaces */
+	init_soc_bounding_box(dc, pool);
+
+	/* DML */
+	dml_init_instance(&dc->dml, &dcn3_03_soc, &dcn3_03_ip, DML_PROJECT_DCN30);
+
+	/* IRQ */
+	init_data.ctx = dc->ctx;
+	pool->irqs = dal_irq_service_dcn303_create(&init_data);
+	if (!pool->irqs)
+		goto create_fail;
+
+	/* HUBBUB */
+	pool->hubbub = dcn303_hubbub_create(ctx);
+	if (pool->hubbub == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create hubbub!\n");
+		goto create_fail;
+	}
+
+	/* HUBPs, DPPs, OPPs and TGs */
+	for (i = 0; i < pool->pipe_count; i++) {
+		pool->hubps[i] = dcn303_hubp_create(ctx, i);
+		if (pool->hubps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create hubps!\n");
+			goto create_fail;
+		}
+
+		pool->dpps[i] = dcn303_dpp_create(ctx, i);
+		if (pool->dpps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create dpps!\n");
+			goto create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->res_cap->num_opp; i++) {
+		pool->opps[i] = dcn303_opp_create(ctx, i);
+		if (pool->opps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create output pixel processor!\n");
+			goto create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->res_cap->num_timing_generator; i++) {
+		pool->timing_generators[i] = dcn303_timing_generator_create(ctx, i);
+		if (pool->timing_generators[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto create_fail;
+		}
+	}
+	pool->timing_generator_count = i;
+
+	/* PSR */
+	pool->psr = dmub_psr_create(ctx);
+	if (pool->psr == NULL) {
+		dm_error("DC: failed to create psr!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* ABM */
+	for (i = 0; i < pool->res_cap->num_timing_generator; i++) {
+		pool->multiple_abms[i] = dmub_abm_create(ctx, &abm_regs[i], &abm_shift, &abm_mask);
+		if (pool->multiple_abms[i] == NULL) {
+			dm_error("DC: failed to create abm for pipe %d!\n", i);
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	/* MPC and DSC */
+	pool->mpc = dcn303_mpc_create(ctx, pool->mpcc_count, pool->res_cap->num_mpc_3dlut);
+	if (pool->mpc == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mpc!\n");
+		goto create_fail;
+	}
+
+	for (i = 0; i < pool->res_cap->num_dsc; i++) {
+		pool->dscs[i] = dcn303_dsc_create(ctx, i);
+		if (pool->dscs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create display stream compressor %d!\n", i);
+			goto create_fail;
+		}
+	}
+
+	/* DWB and MMHUBBUB */
+	if (!dcn303_dwbc_create(ctx, pool)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create dwbc!\n");
+		goto create_fail;
+	}
+
+	if (!dcn303_mmhubbub_create(ctx, pool)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mcif_wb!\n");
+		goto create_fail;
+	}
+
+	/* AUX and I2C */
+	for (i = 0; i < pool->res_cap->num_ddc; i++) {
+		pool->engines[i] = dcn303_aux_engine_create(ctx, i);
+		if (pool->engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC:failed to create aux engine!!\n");
+			goto create_fail;
+		}
+		pool->hw_i2cs[i] = dcn303_i2c_hw_create(ctx, i);
+		if (pool->hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC:failed to create hw i2c!!\n");
+			goto create_fail;
+		}
+		pool->sw_i2cs[i] = NULL;
+	}
+
+	/* Audio, Stream Encoders including HPO and virtual, MPC 3D LUTs */
+	if (!resource_construct(num_virtual_links, dc, pool,
+			&res_create_funcs))
+		goto create_fail;
+
+	/* HW Sequencer and Plane caps */
+	dcn303_hw_sequencer_construct(dc);
+
+	dc->caps.max_planes =  pool->pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->cap_funcs = cap_funcs;
+
+	if (dc->ctx->dc_bios->fw_info.oem_i2c_present) {
+		ddc_init_data.ctx = dc->ctx;
+		ddc_init_data.link = NULL;
+		ddc_init_data.id.id = dc->ctx->dc_bios->fw_info.oem_i2c_obj_id;
+		ddc_init_data.id.enum_id = 0;
+		ddc_init_data.id.type = OBJECT_TYPE_GENERIC;
+		pool->oem_device = dc->link_srv->create_ddc_service(&ddc_init_data);
+	} else {
+		pool->oem_device = NULL;
+	}
+
+	return true;
+
+create_fail:
+
+	dcn303_resource_destruct(pool);
+
+	return false;
+}
+
+struct resource_pool *dcn303_create_resource_pool(const struct dc_init_data *init_data, struct dc *dc)
+{
+	struct resource_pool *pool = kzalloc(sizeof(struct resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dcn303_resource_construct(init_data->num_virtual_links, dc, pool))
+		return pool;
+
+	BREAK_TO_DEBUGGER();
+	kfree(pool);
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.h
new file mode 100644
index 000000000000..37cf1525820b
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.h
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright (C) 2021 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ */
+
+#ifndef _DCN303_RESOURCE_H_
+#define _DCN303_RESOURCE_H_
+
+#include "core_types.h"
+
+extern struct _vcs_dpi_ip_params_st dcn3_03_ip;
+extern struct _vcs_dpi_soc_bounding_box_st dcn3_03_soc;
+
+struct resource_pool *dcn303_create_resource_pool(const struct dc_init_data *init_data, struct dc *dc);
+
+void dcn303_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params);
+
+#endif /* _DCN303_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c
new file mode 100644
index 000000000000..31035fc3d868
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c
@@ -0,0 +1,2218 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+
+#include "dm_services.h"
+#include "dc.h"
+
+#include "dcn31/dcn31_init.h"
+
+#include "resource.h"
+#include "include/irq_service_interface.h"
+#include "dcn31_resource.h"
+
+#include "dcn20/dcn20_resource.h"
+#include "dcn30/dcn30_resource.h"
+
+#include "dml/dcn30/dcn30_fpu.h"
+
+#include "dcn10/dcn10_ipp.h"
+#include "dcn30/dcn30_hubbub.h"
+#include "dcn31/dcn31_hubbub.h"
+#include "dcn30/dcn30_mpc.h"
+#include "dcn31/dcn31_hubp.h"
+#include "irq/dcn31/irq_service_dcn31.h"
+#include "dcn30/dcn30_dpp.h"
+#include "dcn31/dcn31_optc.h"
+#include "dcn20/dcn20_hwseq.h"
+#include "dcn30/dcn30_hwseq.h"
+#include "dce110/dce110_hwseq.h"
+#include "dcn30/dcn30_opp.h"
+#include "dcn20/dcn20_dsc.h"
+#include "dcn30/dcn30_vpg.h"
+#include "dcn30/dcn30_afmt.h"
+#include "dcn30/dcn30_dio_stream_encoder.h"
+#include "dcn31/dcn31_hpo_dp_stream_encoder.h"
+#include "dcn31/dcn31_hpo_dp_link_encoder.h"
+#include "dcn31/dcn31_apg.h"
+#include "dcn31/dcn31_dio_link_encoder.h"
+#include "dcn31/dcn31_vpg.h"
+#include "dcn31/dcn31_afmt.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_hwseq.h"
+#include "clk_mgr.h"
+#include "virtual/virtual_stream_encoder.h"
+#include "dce110/dce110_resource.h"
+#include "dml/display_mode_vba.h"
+#include "dml/dcn31/dcn31_fpu.h"
+#include "dcn31/dcn31_dccg.h"
+#include "dcn10/dcn10_resource.h"
+#include "dcn31/dcn31_panel_cntl.h"
+
+#include "dcn30/dcn30_dwb.h"
+#include "dcn30/dcn30_mmhubbub.h"
+
+// TODO: change include headers /amd/include/asic_reg after upstream
+#include "yellow_carp_offset.h"
+#include "dcn/dcn_3_1_2_offset.h"
+#include "dcn/dcn_3_1_2_sh_mask.h"
+#include "nbio/nbio_7_2_0_offset.h"
+#include "dpcs/dpcs_4_2_0_offset.h"
+#include "dpcs/dpcs_4_2_0_sh_mask.h"
+#include "mmhub/mmhub_2_3_0_offset.h"
+#include "mmhub/mmhub_2_3_0_sh_mask.h"
+
+
+#define regDCHUBBUB_DEBUG_CTRL_0                                              0x04d6
+#define regDCHUBBUB_DEBUG_CTRL_0_BASE_IDX                                     2
+#define DCHUBBUB_DEBUG_CTRL_0__DET_DEPTH__SHIFT                               0x10
+#define DCHUBBUB_DEBUG_CTRL_0__DET_DEPTH_MASK                                 0x01FF0000L
+
+#include "reg_helper.h"
+#include "dce/dmub_abm.h"
+#include "dce/dmub_psr.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_i2c.h"
+#include "dce/dmub_replay.h"
+
+#include "dml/dcn30/display_mode_vba_30.h"
+#include "vm_helper.h"
+#include "dcn20/dcn20_vmid.h"
+
+#include "link_enc_cfg.h"
+
+#define DC_LOGGER \
+	dc->ctx->logger
+#define DC_LOGGER_INIT(logger)
+
+enum dcn31_clk_src_array_id {
+	DCN31_CLK_SRC_PLL0,
+	DCN31_CLK_SRC_PLL1,
+	DCN31_CLK_SRC_PLL2,
+	DCN31_CLK_SRC_PLL3,
+	DCN31_CLK_SRC_PLL4,
+	DCN30_CLK_SRC_TOTAL
+};
+
+/* begin *********************
+ * macros to expend register list macro defined in HW object header file
+ */
+
+/* DCN */
+#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
+
+#define BASE(seg) BASE_INNER(seg)
+
+#define SR(reg_name)\
+		.reg_name = BASE(reg ## reg_name ## _BASE_IDX) +  \
+					reg ## reg_name
+
+#define SRI(reg_name, block, id)\
+	.reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define SRI2(reg_name, block, id)\
+	.reg_name = BASE(reg ## reg_name ## _BASE_IDX) + \
+					reg ## reg_name
+
+#define SRIR(var_name, reg_name, block, id)\
+	.var_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define SRII(reg_name, block, id)\
+	.reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define SRII_MPC_RMU(reg_name, block, id)\
+	.RMU##_##reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define SRII_DWB(reg_name, temp_name, block, id)\
+	.reg_name[id] = BASE(reg ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## temp_name
+
+#define SF_DWB2(reg_name, block, id, field_name, post_fix)	\
+	.field_name = reg_name ## __ ## field_name ## post_fix
+
+#define DCCG_SRII(reg_name, block, id)\
+	.block ## _ ## reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define VUPDATE_SRII(reg_name, block, id)\
+	.reg_name[id] = BASE(reg ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
+					reg ## reg_name ## _ ## block ## id
+
+/* NBIO */
+#define NBIO_BASE_INNER(seg) \
+	NBIO_BASE__INST0_SEG ## seg
+
+#define NBIO_BASE(seg) \
+	NBIO_BASE_INNER(seg)
+
+#define NBIO_SR(reg_name)\
+		.reg_name = NBIO_BASE(regBIF_BX1_ ## reg_name ## _BASE_IDX) + \
+					regBIF_BX1_ ## reg_name
+
+/* MMHUB */
+#define MMHUB_BASE_INNER(seg) \
+	MMHUB_BASE__INST0_SEG ## seg
+
+#define MMHUB_BASE(seg) \
+	MMHUB_BASE_INNER(seg)
+
+#define MMHUB_SR(reg_name)\
+		.reg_name = MMHUB_BASE(mm ## reg_name ## _BASE_IDX) + \
+					mm ## reg_name
+
+/* CLOCK */
+#define CLK_BASE_INNER(seg) \
+	CLK_BASE__INST0_SEG ## seg
+
+#define CLK_BASE(seg) \
+	CLK_BASE_INNER(seg)
+
+#define CLK_SRI(reg_name, block, inst)\
+	.reg_name = CLK_BASE(reg ## block ## _ ## inst ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## _ ## inst ## _ ## reg_name
+
+
+static const struct bios_registers bios_regs = {
+		NBIO_SR(BIOS_SCRATCH_3),
+		NBIO_SR(BIOS_SCRATCH_6)
+};
+
+#define clk_src_regs(index, pllid)\
+[index] = {\
+	CS_COMMON_REG_LIST_DCN3_0(index, pllid),\
+}
+
+static const struct dce110_clk_src_regs clk_src_regs[] = {
+	clk_src_regs(0, A),
+	clk_src_regs(1, B),
+	clk_src_regs(2, C),
+	clk_src_regs(3, D),
+	clk_src_regs(4, E)
+};
+/*pll_id being rempped in dmub, in driver it is logical instance*/
+static const struct dce110_clk_src_regs clk_src_regs_b0[] = {
+	clk_src_regs(0, A),
+	clk_src_regs(1, B),
+	clk_src_regs(2, F),
+	clk_src_regs(3, G),
+	clk_src_regs(4, E)
+};
+
+static const struct dce110_clk_src_shift cs_shift = {
+		CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
+};
+
+static const struct dce110_clk_src_mask cs_mask = {
+		CS_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
+};
+
+#define abm_regs(id)\
+[id] = {\
+		ABM_DCN302_REG_LIST(id)\
+}
+
+static const struct dce_abm_registers abm_regs[] = {
+		abm_regs(0),
+		abm_regs(1),
+		abm_regs(2),
+		abm_regs(3),
+};
+
+static const struct dce_abm_shift abm_shift = {
+		ABM_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dce_abm_mask abm_mask = {
+		ABM_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define audio_regs(id)\
+[id] = {\
+		AUD_COMMON_REG_LIST(id)\
+}
+
+static const struct dce_audio_registers audio_regs[] = {
+	audio_regs(0),
+	audio_regs(1),
+	audio_regs(2),
+	audio_regs(3),
+	audio_regs(4),
+	audio_regs(5),
+	audio_regs(6)
+};
+
+#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
+		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
+
+static const struct dce_audio_shift audio_shift = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_audio_mask audio_mask = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+#define vpg_regs(id)\
+[id] = {\
+	VPG_DCN31_REG_LIST(id)\
+}
+
+static const struct dcn31_vpg_registers vpg_regs[] = {
+	vpg_regs(0),
+	vpg_regs(1),
+	vpg_regs(2),
+	vpg_regs(3),
+	vpg_regs(4),
+	vpg_regs(5),
+	vpg_regs(6),
+	vpg_regs(7),
+	vpg_regs(8),
+	vpg_regs(9),
+};
+
+static const struct dcn31_vpg_shift vpg_shift = {
+	DCN31_VPG_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_vpg_mask vpg_mask = {
+	DCN31_VPG_MASK_SH_LIST(_MASK)
+};
+
+#define afmt_regs(id)\
+[id] = {\
+	AFMT_DCN31_REG_LIST(id)\
+}
+
+static const struct dcn31_afmt_registers afmt_regs[] = {
+	afmt_regs(0),
+	afmt_regs(1),
+	afmt_regs(2),
+	afmt_regs(3),
+	afmt_regs(4),
+	afmt_regs(5)
+};
+
+static const struct dcn31_afmt_shift afmt_shift = {
+	DCN31_AFMT_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_afmt_mask afmt_mask = {
+	DCN31_AFMT_MASK_SH_LIST(_MASK)
+};
+
+#define apg_regs(id)\
+[id] = {\
+	APG_DCN31_REG_LIST(id)\
+}
+
+static const struct dcn31_apg_registers apg_regs[] = {
+	apg_regs(0),
+	apg_regs(1),
+	apg_regs(2),
+	apg_regs(3)
+};
+
+static const struct dcn31_apg_shift apg_shift = {
+	DCN31_APG_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_apg_mask apg_mask = {
+		DCN31_APG_MASK_SH_LIST(_MASK)
+};
+
+#define stream_enc_regs(id)\
+[id] = {\
+	SE_DCN3_REG_LIST(id)\
+}
+
+/* Some encoders won't be initialized here - but they're logical, not physical. */
+static const struct dcn10_stream_enc_registers stream_enc_regs[ENGINE_ID_COUNT] = {
+	stream_enc_regs(0),
+	stream_enc_regs(1),
+	stream_enc_regs(2),
+	stream_enc_regs(3),
+	stream_enc_regs(4)
+};
+
+static const struct dcn10_stream_encoder_shift se_shift = {
+		SE_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn10_stream_encoder_mask se_mask = {
+		SE_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+
+#define aux_regs(id)\
+[id] = {\
+	DCN2_AUX_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
+		aux_regs(0),
+		aux_regs(1),
+		aux_regs(2),
+		aux_regs(3),
+		aux_regs(4)
+};
+
+#define hpd_regs(id)\
+[id] = {\
+	HPD_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
+		hpd_regs(0),
+		hpd_regs(1),
+		hpd_regs(2),
+		hpd_regs(3),
+		hpd_regs(4)
+};
+
+#define link_regs(id, phyid)\
+[id] = {\
+	LE_DCN31_REG_LIST(id), \
+	UNIPHY_DCN2_REG_LIST(phyid), \
+	DPCS_DCN31_REG_LIST(id), \
+}
+
+static const struct dce110_aux_registers_shift aux_shift = {
+	DCN_AUX_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+	DCN_AUX_MASK_SH_LIST(_MASK)
+};
+
+static const struct dcn10_link_enc_registers link_enc_regs[] = {
+	link_regs(0, A),
+	link_regs(1, B),
+	link_regs(2, C),
+	link_regs(3, D),
+	link_regs(4, E)
+};
+
+static const struct dcn10_link_enc_shift le_shift = {
+	LINK_ENCODER_MASK_SH_LIST_DCN31(__SHIFT), \
+	DPCS_DCN31_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn10_link_enc_mask le_mask = {
+	LINK_ENCODER_MASK_SH_LIST_DCN31(_MASK), \
+	DPCS_DCN31_MASK_SH_LIST(_MASK)
+};
+
+#define hpo_dp_stream_encoder_reg_list(id)\
+[id] = {\
+	DCN3_1_HPO_DP_STREAM_ENC_REG_LIST(id)\
+}
+
+static const struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[] = {
+	hpo_dp_stream_encoder_reg_list(0),
+	hpo_dp_stream_encoder_reg_list(1),
+	hpo_dp_stream_encoder_reg_list(2),
+	hpo_dp_stream_encoder_reg_list(3),
+};
+
+static const struct dcn31_hpo_dp_stream_encoder_shift hpo_dp_se_shift = {
+	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_hpo_dp_stream_encoder_mask hpo_dp_se_mask = {
+	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(_MASK)
+};
+
+#define hpo_dp_link_encoder_reg_list(id)\
+[id] = {\
+	DCN3_1_HPO_DP_LINK_ENC_REG_LIST(id),\
+	DCN3_1_RDPCSTX_REG_LIST(0),\
+	DCN3_1_RDPCSTX_REG_LIST(1),\
+	DCN3_1_RDPCSTX_REG_LIST(2),\
+	DCN3_1_RDPCSTX_REG_LIST(3),\
+	DCN3_1_RDPCSTX_REG_LIST(4)\
+}
+
+static const struct dcn31_hpo_dp_link_encoder_registers hpo_dp_link_enc_regs[] = {
+	hpo_dp_link_encoder_reg_list(0),
+	hpo_dp_link_encoder_reg_list(1),
+};
+
+static const struct dcn31_hpo_dp_link_encoder_shift hpo_dp_le_shift = {
+	DCN3_1_HPO_DP_LINK_ENC_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_hpo_dp_link_encoder_mask hpo_dp_le_mask = {
+	DCN3_1_HPO_DP_LINK_ENC_MASK_SH_LIST(_MASK)
+};
+
+#define dpp_regs(id)\
+[id] = {\
+	DPP_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn3_dpp_registers dpp_regs[] = {
+	dpp_regs(0),
+	dpp_regs(1),
+	dpp_regs(2),
+	dpp_regs(3)
+};
+
+static const struct dcn3_dpp_shift tf_shift = {
+		DPP_REG_LIST_SH_MASK_DCN30(__SHIFT)
+};
+
+static const struct dcn3_dpp_mask tf_mask = {
+		DPP_REG_LIST_SH_MASK_DCN30(_MASK)
+};
+
+#define opp_regs(id)\
+[id] = {\
+	OPP_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn20_opp_registers opp_regs[] = {
+	opp_regs(0),
+	opp_regs(1),
+	opp_regs(2),
+	opp_regs(3)
+};
+
+static const struct dcn20_opp_shift opp_shift = {
+	OPP_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn20_opp_mask opp_mask = {
+	OPP_MASK_SH_LIST_DCN20(_MASK)
+};
+
+#define aux_engine_regs(id)\
+[id] = {\
+	AUX_COMMON_REG_LIST0(id), \
+	.AUXN_IMPCAL = 0, \
+	.AUXP_IMPCAL = 0, \
+	.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1),
+		aux_engine_regs(2),
+		aux_engine_regs(3),
+		aux_engine_regs(4)
+};
+
+#define dwbc_regs_dcn3(id)\
+[id] = {\
+	DWBC_COMMON_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn30_dwbc_registers dwbc30_regs[] = {
+	dwbc_regs_dcn3(0),
+};
+
+static const struct dcn30_dwbc_shift dwbc30_shift = {
+	DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_dwbc_mask dwbc30_mask = {
+	DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define mcif_wb_regs_dcn3(id)\
+[id] = {\
+	MCIF_WB_COMMON_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn30_mmhubbub_registers mcif_wb30_regs[] = {
+	mcif_wb_regs_dcn3(0)
+};
+
+static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define dsc_regsDCN20(id)\
+[id] = {\
+	DSC_REG_LIST_DCN20(id)\
+}
+
+static const struct dcn20_dsc_registers dsc_regs[] = {
+	dsc_regsDCN20(0),
+	dsc_regsDCN20(1),
+	dsc_regsDCN20(2)
+};
+
+static const struct dcn20_dsc_shift dsc_shift = {
+	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
+};
+
+static const struct dcn20_dsc_mask dsc_mask = {
+	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
+};
+
+static const struct dcn30_mpc_registers mpc_regs = {
+		MPC_REG_LIST_DCN3_0(0),
+		MPC_REG_LIST_DCN3_0(1),
+		MPC_REG_LIST_DCN3_0(2),
+		MPC_REG_LIST_DCN3_0(3),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(0),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(1),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(2),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(3),
+		MPC_RMU_GLOBAL_REG_LIST_DCN3AG,
+		MPC_RMU_REG_LIST_DCN3AG(0),
+		MPC_RMU_REG_LIST_DCN3AG(1),
+		//MPC_RMU_REG_LIST_DCN3AG(2),
+		MPC_DWB_MUX_REG_LIST_DCN3_0(0),
+};
+
+static const struct dcn30_mpc_shift mpc_shift = {
+	MPC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_mpc_mask mpc_mask = {
+	MPC_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define optc_regs(id)\
+[id] = {OPTC_COMMON_REG_LIST_DCN3_1(id)}
+
+static const struct dcn_optc_registers optc_regs[] = {
+	optc_regs(0),
+	optc_regs(1),
+	optc_regs(2),
+	optc_regs(3)
+};
+
+static const struct dcn_optc_shift optc_shift = {
+	OPTC_COMMON_MASK_SH_LIST_DCN3_1(__SHIFT)
+};
+
+static const struct dcn_optc_mask optc_mask = {
+	OPTC_COMMON_MASK_SH_LIST_DCN3_1(_MASK)
+};
+
+#define hubp_regs(id)\
+[id] = {\
+	HUBP_REG_LIST_DCN30(id)\
+}
+
+static const struct dcn_hubp2_registers hubp_regs[] = {
+		hubp_regs(0),
+		hubp_regs(1),
+		hubp_regs(2),
+		hubp_regs(3)
+};
+
+
+static const struct dcn_hubp2_shift hubp_shift = {
+		HUBP_MASK_SH_LIST_DCN31(__SHIFT)
+};
+
+static const struct dcn_hubp2_mask hubp_mask = {
+		HUBP_MASK_SH_LIST_DCN31(_MASK)
+};
+static const struct dcn_hubbub_registers hubbub_reg = {
+		HUBBUB_REG_LIST_DCN31(0)
+};
+
+static const struct dcn_hubbub_shift hubbub_shift = {
+		HUBBUB_MASK_SH_LIST_DCN31(__SHIFT)
+};
+
+static const struct dcn_hubbub_mask hubbub_mask = {
+		HUBBUB_MASK_SH_LIST_DCN31(_MASK)
+};
+
+static const struct dccg_registers dccg_regs = {
+		DCCG_REG_LIST_DCN31()
+};
+
+static const struct dccg_shift dccg_shift = {
+		DCCG_MASK_SH_LIST_DCN31(__SHIFT)
+};
+
+static const struct dccg_mask dccg_mask = {
+		DCCG_MASK_SH_LIST_DCN31(_MASK)
+};
+
+
+#define SRII2(reg_name_pre, reg_name_post, id)\
+	.reg_name_pre ## _ ##  reg_name_post[id] = BASE(reg ## reg_name_pre \
+			## id ## _ ## reg_name_post ## _BASE_IDX) + \
+			reg ## reg_name_pre ## id ## _ ## reg_name_post
+
+
+#define HWSEQ_DCN31_REG_LIST()\
+	SR(DCHUBBUB_GLOBAL_TIMER_CNTL), \
+	SR(DCHUBBUB_ARB_HOSTVM_CNTL), \
+	SR(DIO_MEM_PWR_CTRL), \
+	SR(ODM_MEM_PWR_CTRL3), \
+	SR(DMU_MEM_PWR_CNTL), \
+	SR(MMHUBBUB_MEM_PWR_CNTL), \
+	SR(DCCG_GATE_DISABLE_CNTL), \
+	SR(DCCG_GATE_DISABLE_CNTL2), \
+	SR(DCFCLK_CNTL),\
+	SR(DC_MEM_GLOBAL_PWR_REQ_CNTL), \
+	SRII(PIXEL_RATE_CNTL, OTG, 0), \
+	SRII(PIXEL_RATE_CNTL, OTG, 1),\
+	SRII(PIXEL_RATE_CNTL, OTG, 2),\
+	SRII(PIXEL_RATE_CNTL, OTG, 3),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 0),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 1),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 2),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 3),\
+	SR(MICROSECOND_TIME_BASE_DIV), \
+	SR(MILLISECOND_TIME_BASE_DIV), \
+	SR(DISPCLK_FREQ_CHANGE_CNTL), \
+	SR(RBBMIF_TIMEOUT_DIS), \
+	SR(RBBMIF_TIMEOUT_DIS_2), \
+	SR(DCHUBBUB_CRC_CTRL), \
+	SR(DPP_TOP0_DPP_CRC_CTRL), \
+	SR(DPP_TOP0_DPP_CRC_VAL_B_A), \
+	SR(DPP_TOP0_DPP_CRC_VAL_R_G), \
+	SR(MPC_CRC_CTRL), \
+	SR(MPC_CRC_RESULT_GB), \
+	SR(MPC_CRC_RESULT_C), \
+	SR(MPC_CRC_RESULT_AR), \
+	SR(DOMAIN0_PG_CONFIG), \
+	SR(DOMAIN1_PG_CONFIG), \
+	SR(DOMAIN2_PG_CONFIG), \
+	SR(DOMAIN3_PG_CONFIG), \
+	SR(DOMAIN16_PG_CONFIG), \
+	SR(DOMAIN17_PG_CONFIG), \
+	SR(DOMAIN18_PG_CONFIG), \
+	SR(DOMAIN0_PG_STATUS), \
+	SR(DOMAIN1_PG_STATUS), \
+	SR(DOMAIN2_PG_STATUS), \
+	SR(DOMAIN3_PG_STATUS), \
+	SR(DOMAIN16_PG_STATUS), \
+	SR(DOMAIN17_PG_STATUS), \
+	SR(DOMAIN18_PG_STATUS), \
+	SR(D1VGA_CONTROL), \
+	SR(D2VGA_CONTROL), \
+	SR(D3VGA_CONTROL), \
+	SR(D4VGA_CONTROL), \
+	SR(D5VGA_CONTROL), \
+	SR(D6VGA_CONTROL), \
+	SR(DC_IP_REQUEST_CNTL), \
+	SR(AZALIA_AUDIO_DTO), \
+	SR(AZALIA_CONTROLLER_CLOCK_GATING), \
+	SR(HPO_TOP_HW_CONTROL)
+
+static const struct dce_hwseq_registers hwseq_reg = {
+		HWSEQ_DCN31_REG_LIST()
+};
+
+#define HWSEQ_DCN31_MASK_SH_LIST(mask_sh)\
+	HWSEQ_DCN_MASK_SH_LIST(mask_sh), \
+	HWS_SF(, DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, mask_sh), \
+	HWS_SF(, DCHUBBUB_ARB_HOSTVM_CNTL, DISABLE_HOSTVM_FORCE_ALLOW_PSTATE, mask_sh), \
+	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN0_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN1_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN2_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN3_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN16_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN17_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN18_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh), \
+	HWS_SF(, AZALIA_AUDIO_DTO, AZALIA_AUDIO_DTO_MODULE, mask_sh), \
+	HWS_SF(, HPO_TOP_CLOCK_CONTROL, HPO_HDMISTREAMCLK_G_GATE_DIS, mask_sh), \
+	HWS_SF(, DMU_MEM_PWR_CNTL, DMCU_ERAM_MEM_PWR_FORCE, mask_sh), \
+	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_UNASSIGNED_PWR_MODE, mask_sh), \
+	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_VBLANK_PWR_MODE, mask_sh), \
+	HWS_SF(, MMHUBBUB_MEM_PWR_CNTL, VGA_MEM_PWR_FORCE, mask_sh), \
+	HWS_SF(, DIO_MEM_PWR_CTRL, I2C_LIGHT_SLEEP_FORCE, mask_sh), \
+	HWS_SF(, HPO_TOP_HW_CONTROL, HPO_IO_EN, mask_sh)
+
+static const struct dce_hwseq_shift hwseq_shift = {
+		HWSEQ_DCN31_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+		HWSEQ_DCN31_MASK_SH_LIST(_MASK)
+};
+#define vmid_regs(id)\
+[id] = {\
+		DCN20_VMID_REG_LIST(id)\
+}
+
+static const struct dcn_vmid_registers vmid_regs[] = {
+	vmid_regs(0),
+	vmid_regs(1),
+	vmid_regs(2),
+	vmid_regs(3),
+	vmid_regs(4),
+	vmid_regs(5),
+	vmid_regs(6),
+	vmid_regs(7),
+	vmid_regs(8),
+	vmid_regs(9),
+	vmid_regs(10),
+	vmid_regs(11),
+	vmid_regs(12),
+	vmid_regs(13),
+	vmid_regs(14),
+	vmid_regs(15)
+};
+
+static const struct dcn20_vmid_shift vmid_shifts = {
+		DCN20_VMID_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn20_vmid_mask vmid_masks = {
+		DCN20_VMID_MASK_SH_LIST(_MASK)
+};
+
+static const struct resource_caps res_cap_dcn31 = {
+	.num_timing_generator = 4,
+	.num_opp = 4,
+	.num_video_plane = 4,
+	.num_audio = 5,
+	.num_stream_encoder = 5,
+	.num_dig_link_enc = 5,
+	.num_hpo_dp_stream_encoder = 4,
+	.num_hpo_dp_link_encoder = 2,
+	.num_pll = 5,
+	.num_dwb = 1,
+	.num_ddc = 5,
+	.num_vmid = 16,
+	.num_mpc_3dlut = 2,
+	.num_dsc = 3,
+};
+
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
+	.per_pixel_alpha = true,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = true,
+			.fp16 = true,
+			.p010 = true,
+			.ayuv = false,
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 16000,
+			.fp16 = 16000
+	},
+
+	// 6:1 downscaling ratio: 1000/6 = 166.666
+	.max_downscale_factor = {
+			.argb8888 = 167,
+			.nv12 = 167,
+			.fp16 = 167
+	},
+	64,
+	64
+};
+
+static const struct dc_debug_options debug_defaults_drv = {
+	.disable_dmcu = true,
+	.force_abm_enable = false,
+	.timing_trace = false,
+	.clock_trace = true,
+	.disable_pplib_clock_request = false,
+	.pipe_split_policy = MPC_SPLIT_DYNAMIC,
+	.force_single_disp_pipe_split = false,
+	.disable_dcc = DCC_ENABLE,
+	.vsr_support = true,
+	.performance_trace = false,
+	.max_downscale_src_width = 4096,/*upto true 4K*/
+	.disable_pplib_wm_range = false,
+	.scl_reset_length10 = true,
+	.sanity_checks = true,
+	.underflow_assert_delay_us = 0xFFFFFFFF,
+	.dwb_fi_phase = -1, // -1 = disable,
+	.dmub_command_table = true,
+	.pstate_enabled = true,
+	.use_max_lb = true,
+	.enable_mem_low_power = {
+		.bits = {
+			.vga = true,
+			.i2c = true,
+			.dmcu = false, // This is previously known to cause hang on S3 cycles if enabled
+			.dscl = true,
+			.cm = true,
+			.mpc = true,
+			.optc = true,
+			.vpg = true,
+			.afmt = true,
+		}
+	},
+	.disable_z10 = true,
+	.enable_legacy_fast_update = true,
+	.enable_z9_disable_interface = true, /* Allow support for the PMFW interface for disable Z9*/
+	.dml_hostvm_override = DML_HOSTVM_OVERRIDE_FALSE,
+	.using_dml2 = false,
+};
+
+static const struct dc_panel_config panel_config_defaults = {
+	.psr = {
+		.disable_psr = false,
+		.disallow_psrsu = false,
+		.disallow_replay = false,
+	},
+	.ilr = {
+		.optimize_edp_link_rate = true,
+	},
+};
+
+static void dcn31_dpp_destroy(struct dpp **dpp)
+{
+	kfree(TO_DCN20_DPP(*dpp));
+	*dpp = NULL;
+}
+
+static struct dpp *dcn31_dpp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn3_dpp *dpp =
+		kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
+
+	if (!dpp)
+		return NULL;
+
+	if (dpp3_construct(dpp, ctx, inst,
+			&dpp_regs[inst], &tf_shift, &tf_mask))
+		return &dpp->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(dpp);
+	return NULL;
+}
+
+static struct output_pixel_processor *dcn31_opp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_opp *opp =
+		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
+
+	if (!opp) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dcn20_opp_construct(opp, ctx, inst,
+			&opp_regs[inst], &opp_shift, &opp_mask);
+	return &opp->base;
+}
+
+static struct dce_aux *dcn31_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst],
+					&aux_mask,
+					&aux_shift,
+					ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST_DCN30(id) }
+
+static const struct dce_i2c_registers i2c_hw_regs[] = {
+		i2c_inst_regs(1),
+		i2c_inst_regs(2),
+		i2c_inst_regs(3),
+		i2c_inst_regs(4),
+		i2c_inst_regs(5),
+};
+
+static const struct dce_i2c_shift i2c_shifts = {
+		I2C_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dce_i2c_mask i2c_masks = {
+		I2C_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static struct dce_i2c_hw *dcn31_i2c_hw_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw =
+		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
+				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+static struct mpc *dcn31_mpc_create(
+		struct dc_context *ctx,
+		int num_mpcc,
+		int num_rmu)
+{
+	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc),
+					  GFP_KERNEL);
+
+	if (!mpc30)
+		return NULL;
+
+	dcn30_mpc_construct(mpc30, ctx,
+			&mpc_regs,
+			&mpc_shift,
+			&mpc_mask,
+			num_mpcc,
+			num_rmu);
+
+	return &mpc30->base;
+}
+
+static struct hubbub *dcn31_hubbub_create(struct dc_context *ctx)
+{
+	int i;
+
+	struct dcn20_hubbub *hubbub3 = kzalloc(sizeof(struct dcn20_hubbub),
+					  GFP_KERNEL);
+
+	if (!hubbub3)
+		return NULL;
+
+	hubbub31_construct(hubbub3, ctx,
+			&hubbub_reg,
+			&hubbub_shift,
+			&hubbub_mask,
+			dcn3_1_ip.det_buffer_size_kbytes,
+			dcn3_1_ip.pixel_chunk_size_kbytes,
+			dcn3_1_ip.config_return_buffer_size_in_kbytes);
+
+
+	for (i = 0; i < res_cap_dcn31.num_vmid; i++) {
+		struct dcn20_vmid *vmid = &hubbub3->vmid[i];
+
+		vmid->ctx = ctx;
+
+		vmid->regs = &vmid_regs[i];
+		vmid->shifts = &vmid_shifts;
+		vmid->masks = &vmid_masks;
+	}
+
+	return &hubbub3->base;
+}
+
+static struct timing_generator *dcn31_timing_generator_create(
+		struct dc_context *ctx,
+		uint32_t instance)
+{
+	struct optc *tgn10 =
+		kzalloc(sizeof(struct optc), GFP_KERNEL);
+
+	if (!tgn10)
+		return NULL;
+
+	tgn10->base.inst = instance;
+	tgn10->base.ctx = ctx;
+
+	tgn10->tg_regs = &optc_regs[instance];
+	tgn10->tg_shift = &optc_shift;
+	tgn10->tg_mask = &optc_mask;
+
+	dcn31_timing_generator_init(tgn10);
+
+	return &tgn10->base;
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+		.max_hdmi_deep_color = COLOR_DEPTH_121212,
+		.max_hdmi_pixel_clock = 600000,
+		.hdmi_ycbcr420_supported = true,
+		.dp_ycbcr420_supported = true,
+		.fec_supported = true,
+		.flags.bits.IS_HBR2_CAPABLE = true,
+		.flags.bits.IS_HBR3_CAPABLE = true,
+		.flags.bits.IS_TPS3_CAPABLE = true,
+		.flags.bits.IS_TPS4_CAPABLE = true
+};
+
+static struct link_encoder *dcn31_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dcn20_link_encoder *enc20 =
+		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+
+	if (!enc20)
+		return NULL;
+
+	dcn31_link_encoder_construct(enc20,
+			enc_init_data,
+			&link_enc_feature,
+			&link_enc_regs[enc_init_data->transmitter],
+			&link_enc_aux_regs[enc_init_data->channel - 1],
+			&link_enc_hpd_regs[enc_init_data->hpd_source],
+			&le_shift,
+			&le_mask);
+
+	return &enc20->enc10.base;
+}
+
+/* Create a minimal link encoder object not associated with a particular
+ * physical connector.
+ * resource_funcs.link_enc_create_minimal
+ */
+static struct link_encoder *dcn31_link_enc_create_minimal(
+		struct dc_context *ctx, enum engine_id eng_id)
+{
+	struct dcn20_link_encoder *enc20;
+
+	if ((eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+		return NULL;
+
+	enc20 = kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+	if (!enc20)
+		return NULL;
+
+	dcn31_link_encoder_construct_minimal(
+			enc20,
+			ctx,
+			&link_enc_feature,
+			&link_enc_regs[eng_id - ENGINE_ID_DIGA],
+			eng_id);
+
+	return &enc20->enc10.base;
+}
+
+static struct panel_cntl *dcn31_panel_cntl_create(const struct panel_cntl_init_data *init_data)
+{
+	struct dcn31_panel_cntl *panel_cntl =
+		kzalloc(sizeof(struct dcn31_panel_cntl), GFP_KERNEL);
+
+	if (!panel_cntl)
+		return NULL;
+
+	dcn31_panel_cntl_construct(panel_cntl, init_data);
+
+	return &panel_cntl->base;
+}
+
+static void read_dce_straps(
+	struct dc_context *ctx,
+	struct resource_straps *straps)
+{
+	generic_reg_get(ctx, regDC_PINSTRAPS + BASE(regDC_PINSTRAPS_BASE_IDX),
+		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
+
+}
+
+static struct audio *dcn31_create_audio(
+		struct dc_context *ctx, unsigned int inst)
+{
+	return dce_audio_create(ctx, inst,
+			&audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+static struct vpg *dcn31_vpg_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn31_vpg *vpg31 = kzalloc(sizeof(struct dcn31_vpg), GFP_KERNEL);
+
+	if (!vpg31)
+		return NULL;
+
+	vpg31_construct(vpg31, ctx, inst,
+			&vpg_regs[inst],
+			&vpg_shift,
+			&vpg_mask);
+
+	return &vpg31->base;
+}
+
+static struct afmt *dcn31_afmt_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn31_afmt *afmt31 = kzalloc(sizeof(struct dcn31_afmt), GFP_KERNEL);
+
+	if (!afmt31)
+		return NULL;
+
+	afmt31_construct(afmt31, ctx, inst,
+			&afmt_regs[inst],
+			&afmt_shift,
+			&afmt_mask);
+
+	// Light sleep by default, no need to power down here
+
+	return &afmt31->base;
+}
+
+static struct apg *dcn31_apg_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn31_apg *apg31 = kzalloc(sizeof(struct dcn31_apg), GFP_KERNEL);
+
+	if (!apg31)
+		return NULL;
+
+	apg31_construct(apg31, ctx, inst,
+			&apg_regs[inst],
+			&apg_shift,
+			&apg_mask);
+
+	return &apg31->base;
+}
+
+static struct stream_encoder *dcn31_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dcn10_stream_encoder *enc1;
+	struct vpg *vpg;
+	struct afmt *afmt;
+	int vpg_inst;
+	int afmt_inst;
+
+	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
+	if (eng_id <= ENGINE_ID_DIGF) {
+		vpg_inst = eng_id;
+		afmt_inst = eng_id;
+	} else
+		return NULL;
+
+	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
+	vpg = dcn31_vpg_create(ctx, vpg_inst);
+	afmt = dcn31_afmt_create(ctx, afmt_inst);
+
+	if (!enc1 || !vpg || !afmt) {
+		kfree(enc1);
+		kfree(vpg);
+		kfree(afmt);
+		return NULL;
+	}
+
+	dcn30_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
+					eng_id, vpg, afmt,
+					&stream_enc_regs[eng_id],
+					&se_shift, &se_mask);
+
+	return &enc1->base;
+}
+
+static struct hpo_dp_stream_encoder *dcn31_hpo_dp_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dcn31_hpo_dp_stream_encoder *hpo_dp_enc31;
+	struct vpg *vpg;
+	struct apg *apg;
+	uint32_t hpo_dp_inst;
+	uint32_t vpg_inst;
+	uint32_t apg_inst;
+
+	ASSERT((eng_id >= ENGINE_ID_HPO_DP_0) && (eng_id <= ENGINE_ID_HPO_DP_3));
+	hpo_dp_inst = eng_id - ENGINE_ID_HPO_DP_0;
+
+	/* Mapping of VPG register blocks to HPO DP block instance:
+	 * VPG[6] -> HPO_DP[0]
+	 * VPG[7] -> HPO_DP[1]
+	 * VPG[8] -> HPO_DP[2]
+	 * VPG[9] -> HPO_DP[3]
+	 */
+	vpg_inst = hpo_dp_inst + 6;
+
+	/* Mapping of APG register blocks to HPO DP block instance:
+	 * APG[0] -> HPO_DP[0]
+	 * APG[1] -> HPO_DP[1]
+	 * APG[2] -> HPO_DP[2]
+	 * APG[3] -> HPO_DP[3]
+	 */
+	apg_inst = hpo_dp_inst;
+
+	/* allocate HPO stream encoder and create VPG sub-block */
+	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_stream_encoder), GFP_KERNEL);
+	vpg = dcn31_vpg_create(ctx, vpg_inst);
+	apg = dcn31_apg_create(ctx, apg_inst);
+
+	if (!hpo_dp_enc31 || !vpg || !apg) {
+		kfree(hpo_dp_enc31);
+		kfree(vpg);
+		kfree(apg);
+		return NULL;
+	}
+
+	dcn31_hpo_dp_stream_encoder_construct(hpo_dp_enc31, ctx, ctx->dc_bios,
+					hpo_dp_inst, eng_id, vpg, apg,
+					&hpo_dp_stream_enc_regs[hpo_dp_inst],
+					&hpo_dp_se_shift, &hpo_dp_se_mask);
+
+	return &hpo_dp_enc31->base;
+}
+
+static struct hpo_dp_link_encoder *dcn31_hpo_dp_link_encoder_create(
+	uint8_t inst,
+	struct dc_context *ctx)
+{
+	struct dcn31_hpo_dp_link_encoder *hpo_dp_enc31;
+
+	/* allocate HPO link encoder */
+	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
+
+	hpo_dp_link_encoder31_construct(hpo_dp_enc31, ctx, inst,
+					&hpo_dp_link_enc_regs[inst],
+					&hpo_dp_le_shift, &hpo_dp_le_mask);
+
+	return &hpo_dp_enc31->base;
+}
+
+static struct dce_hwseq *dcn31_hwseq_create(
+	struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &hwseq_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+	}
+	return hws;
+}
+static const struct resource_create_funcs res_create_funcs = {
+	.read_dce_straps = read_dce_straps,
+	.create_audio = dcn31_create_audio,
+	.create_stream_encoder = dcn31_stream_encoder_create,
+	.create_hpo_dp_stream_encoder = dcn31_hpo_dp_stream_encoder_create,
+	.create_hpo_dp_link_encoder = dcn31_hpo_dp_link_encoder_create,
+	.create_hwseq = dcn31_hwseq_create,
+};
+
+static void dcn31_resource_destruct(struct dcn31_resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->base.stream_enc_count; i++) {
+		if (pool->base.stream_enc[i] != NULL) {
+			if (pool->base.stream_enc[i]->vpg != NULL) {
+				kfree(DCN30_VPG_FROM_VPG(pool->base.stream_enc[i]->vpg));
+				pool->base.stream_enc[i]->vpg = NULL;
+			}
+			if (pool->base.stream_enc[i]->afmt != NULL) {
+				kfree(DCN30_AFMT_FROM_AFMT(pool->base.stream_enc[i]->afmt));
+				pool->base.stream_enc[i]->afmt = NULL;
+			}
+			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
+			pool->base.stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.hpo_dp_stream_enc_count; i++) {
+		if (pool->base.hpo_dp_stream_enc[i] != NULL) {
+			if (pool->base.hpo_dp_stream_enc[i]->vpg != NULL) {
+				kfree(DCN30_VPG_FROM_VPG(pool->base.hpo_dp_stream_enc[i]->vpg));
+				pool->base.hpo_dp_stream_enc[i]->vpg = NULL;
+			}
+			if (pool->base.hpo_dp_stream_enc[i]->apg != NULL) {
+				kfree(DCN31_APG_FROM_APG(pool->base.hpo_dp_stream_enc[i]->apg));
+				pool->base.hpo_dp_stream_enc[i]->apg = NULL;
+			}
+			kfree(DCN3_1_HPO_DP_STREAM_ENC_FROM_HPO_STREAM_ENC(pool->base.hpo_dp_stream_enc[i]));
+			pool->base.hpo_dp_stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.hpo_dp_link_enc_count; i++) {
+		if (pool->base.hpo_dp_link_enc[i] != NULL) {
+			kfree(DCN3_1_HPO_DP_LINK_ENC_FROM_HPO_LINK_ENC(pool->base.hpo_dp_link_enc[i]));
+			pool->base.hpo_dp_link_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		if (pool->base.dscs[i] != NULL)
+			dcn20_dsc_destroy(&pool->base.dscs[i]);
+	}
+
+	if (pool->base.mpc != NULL) {
+		kfree(TO_DCN20_MPC(pool->base.mpc));
+		pool->base.mpc = NULL;
+	}
+	if (pool->base.hubbub != NULL) {
+		kfree(pool->base.hubbub);
+		pool->base.hubbub = NULL;
+	}
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (pool->base.dpps[i] != NULL)
+			dcn31_dpp_destroy(&pool->base.dpps[i]);
+
+		if (pool->base.ipps[i] != NULL)
+			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
+
+		if (pool->base.hubps[i] != NULL) {
+			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
+			pool->base.hubps[i] = NULL;
+		}
+
+		if (pool->base.irqs != NULL) {
+			dal_irq_service_destroy(&pool->base.irqs);
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+		if (pool->base.hw_i2cs[i] != NULL) {
+			kfree(pool->base.hw_i2cs[i]);
+			pool->base.hw_i2cs[i] = NULL;
+		}
+		if (pool->base.sw_i2cs[i] != NULL) {
+			kfree(pool->base.sw_i2cs[i]);
+			pool->base.sw_i2cs[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		if (pool->base.opps[i] != NULL)
+			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.timing_generators[i] != NULL)	{
+			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
+			pool->base.timing_generators[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+		if (pool->base.dwbc[i] != NULL) {
+			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
+			pool->base.dwbc[i] = NULL;
+		}
+		if (pool->base.mcif_wb[i] != NULL) {
+			kfree(TO_DCN30_MMHUBBUB(pool->base.mcif_wb[i]));
+			pool->base.mcif_wb[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.audio_count; i++) {
+		if (pool->base.audios[i])
+			dce_aud_destroy(&pool->base.audios[i]);
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] != NULL) {
+			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
+			pool->base.clock_sources[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+		if (pool->base.mpc_lut[i] != NULL) {
+			dc_3dlut_func_release(pool->base.mpc_lut[i]);
+			pool->base.mpc_lut[i] = NULL;
+		}
+		if (pool->base.mpc_shaper[i] != NULL) {
+			dc_transfer_func_release(pool->base.mpc_shaper[i]);
+			pool->base.mpc_shaper[i] = NULL;
+		}
+	}
+
+	if (pool->base.dp_clock_source != NULL) {
+		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
+		pool->base.dp_clock_source = NULL;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.multiple_abms[i] != NULL)
+			dce_abm_destroy(&pool->base.multiple_abms[i]);
+	}
+
+	if (pool->base.psr != NULL)
+		dmub_psr_destroy(&pool->base.psr);
+
+	if (pool->base.replay != NULL)
+		dmub_replay_destroy(&pool->base.replay);
+
+	if (pool->base.dccg != NULL)
+		dcn_dccg_destroy(&pool->base.dccg);
+}
+
+static struct hubp *dcn31_hubp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn20_hubp *hubp2 =
+		kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
+
+	if (!hubp2)
+		return NULL;
+
+	if (hubp31_construct(hubp2, ctx, inst,
+			&hubp_regs[inst], &hubp_shift, &hubp_mask))
+		return &hubp2->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(hubp2);
+	return NULL;
+}
+
+static bool dcn31_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc),
+						    GFP_KERNEL);
+
+		if (!dwbc30) {
+			dm_error("DC: failed to create dwbc30!\n");
+			return false;
+		}
+
+		dcn30_dwbc_construct(dwbc30, ctx,
+				&dwbc30_regs[i],
+				&dwbc30_shift,
+				&dwbc30_mask,
+				i);
+
+		pool->dwbc[i] = &dwbc30->base;
+	}
+	return true;
+}
+
+static bool dcn31_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub),
+						    GFP_KERNEL);
+
+		if (!mcif_wb30) {
+			dm_error("DC: failed to create mcif_wb30!\n");
+			return false;
+		}
+
+		dcn30_mmhubbub_construct(mcif_wb30, ctx,
+				&mcif_wb30_regs[i],
+				&mcif_wb30_shift,
+				&mcif_wb30_mask,
+				i);
+
+		pool->mcif_wb[i] = &mcif_wb30->base;
+	}
+	return true;
+}
+
+static struct display_stream_compressor *dcn31_dsc_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_dsc *dsc =
+		kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
+
+	if (!dsc) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
+	return &dsc->base;
+}
+
+static void dcn31_destroy_resource_pool(struct resource_pool **pool)
+{
+	struct dcn31_resource_pool *dcn31_pool = TO_DCN31_RES_POOL(*pool);
+
+	dcn31_resource_destruct(dcn31_pool);
+	kfree(dcn31_pool);
+	*pool = NULL;
+}
+
+static struct clock_source *dcn31_clock_source_create(
+		struct dc_context *ctx,
+		struct dc_bios *bios,
+		enum clock_source_id id,
+		const struct dce110_clk_src_regs *regs,
+		bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dcn3_clk_src_construct(clk_src, ctx, bios, id,
+			regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	kfree(clk_src);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static bool is_dual_plane(enum surface_pixel_format format)
+{
+	return format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN || format == SURFACE_PIXEL_FORMAT_GRPH_RGBE_ALPHA;
+}
+
+int dcn31x_populate_dml_pipes_from_context(struct dc *dc,
+					  struct dc_state *context,
+					  display_e2e_pipe_params_st *pipes,
+					  bool fast_validate)
+{
+	uint32_t pipe_cnt;
+	int i;
+
+	dc_assert_fp_enabled();
+
+	pipe_cnt = dcn20_populate_dml_pipes_from_context(dc, context, pipes, fast_validate);
+
+	for (i = 0; i < pipe_cnt; i++) {
+		pipes[i].pipe.src.gpuvm = 1;
+		if (dc->debug.dml_hostvm_override == DML_HOSTVM_NO_OVERRIDE) {
+			//pipes[pipe_cnt].pipe.src.hostvm = dc->res_pool->hubbub->riommu_active;
+			pipes[i].pipe.src.hostvm = dc->vm_pa_config.is_hvm_enabled;
+		} else if (dc->debug.dml_hostvm_override == DML_HOSTVM_OVERRIDE_FALSE)
+			pipes[i].pipe.src.hostvm = false;
+		else if (dc->debug.dml_hostvm_override == DML_HOSTVM_OVERRIDE_TRUE)
+			pipes[i].pipe.src.hostvm = true;
+	}
+	return pipe_cnt;
+}
+
+int dcn31_populate_dml_pipes_from_context(
+	struct dc *dc, struct dc_state *context,
+	display_e2e_pipe_params_st *pipes,
+	bool fast_validate)
+{
+	int i, pipe_cnt;
+	struct resource_context *res_ctx = &context->res_ctx;
+	struct pipe_ctx *pipe;
+	bool upscaled = false;
+
+	DC_FP_START();
+	dcn31x_populate_dml_pipes_from_context(dc, context, pipes, fast_validate);
+	DC_FP_END();
+
+	for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) {
+		struct dc_crtc_timing *timing;
+
+		if (!res_ctx->pipe_ctx[i].stream)
+			continue;
+		pipe = &res_ctx->pipe_ctx[i];
+		timing = &pipe->stream->timing;
+		if (pipe->plane_state &&
+				(pipe->plane_state->src_rect.height < pipe->plane_state->dst_rect.height ||
+				pipe->plane_state->src_rect.width < pipe->plane_state->dst_rect.width))
+			upscaled = true;
+
+		/*
+		 * Immediate flip can be set dynamically after enabling the plane.
+		 * We need to require support for immediate flip or underflow can be
+		 * intermittently experienced depending on peak b/w requirements.
+		 */
+		pipes[pipe_cnt].pipe.src.immediate_flip = true;
+		pipes[pipe_cnt].pipe.src.unbounded_req_mode = false;
+		pipes[pipe_cnt].pipe.src.gpuvm = true;
+		pipes[pipe_cnt].pipe.dest.vfront_porch = timing->v_front_porch;
+		pipes[pipe_cnt].pipe.src.dcc_rate = 3;
+		pipes[pipe_cnt].dout.dsc_input_bpc = 0;
+		DC_FP_START();
+		dcn31_zero_pipe_dcc_fraction(pipes, pipe_cnt);
+		DC_FP_END();
+
+
+		if (pipes[pipe_cnt].dout.dsc_enable) {
+			switch (timing->display_color_depth) {
+			case COLOR_DEPTH_888:
+				pipes[pipe_cnt].dout.dsc_input_bpc = 8;
+				break;
+			case COLOR_DEPTH_101010:
+				pipes[pipe_cnt].dout.dsc_input_bpc = 10;
+				break;
+			case COLOR_DEPTH_121212:
+				pipes[pipe_cnt].dout.dsc_input_bpc = 12;
+				break;
+			default:
+				ASSERT(0);
+				break;
+			}
+		}
+
+		pipe_cnt++;
+	}
+	context->bw_ctx.dml.ip.det_buffer_size_kbytes = DCN3_1_DEFAULT_DET_SIZE;
+	dc->config.enable_4to1MPC = false;
+	if (pipe_cnt == 1 && pipe->plane_state && !dc->debug.disable_z9_mpc) {
+		if (is_dual_plane(pipe->plane_state->format)
+				&& pipe->plane_state->src_rect.width <= 1920 && pipe->plane_state->src_rect.height <= 1080) {
+			dc->config.enable_4to1MPC = true;
+		} else if (!is_dual_plane(pipe->plane_state->format) && pipe->plane_state->src_rect.width <= 5120) {
+			/* Limit to 5k max to avoid forced pipe split when there is not enough detile for swath */
+			context->bw_ctx.dml.ip.det_buffer_size_kbytes = 192;
+			pipes[0].pipe.src.unbounded_req_mode = true;
+		}
+	} else if (context->stream_count >= dc->debug.crb_alloc_policy_min_disp_count
+			&& dc->debug.crb_alloc_policy > DET_SIZE_DEFAULT) {
+		context->bw_ctx.dml.ip.det_buffer_size_kbytes = dc->debug.crb_alloc_policy * 64;
+	} else if (context->stream_count >= 3 && upscaled) {
+		context->bw_ctx.dml.ip.det_buffer_size_kbytes = 192;
+	}
+
+	return pipe_cnt;
+}
+
+void dcn31_calculate_wm_and_dlg(
+		struct dc *dc, struct dc_state *context,
+		display_e2e_pipe_params_st *pipes,
+		int pipe_cnt,
+		int vlevel)
+{
+	DC_FP_START();
+	dcn31_calculate_wm_and_dlg_fp(dc, context, pipes, pipe_cnt, vlevel);
+	DC_FP_END();
+}
+
+void
+dcn31_populate_dml_writeback_from_context(struct dc *dc,
+					  struct resource_context *res_ctx,
+					  display_e2e_pipe_params_st *pipes)
+{
+	DC_FP_START();
+	dcn30_populate_dml_writeback_from_context(dc, res_ctx, pipes);
+	DC_FP_END();
+}
+
+void
+dcn31_set_mcif_arb_params(struct dc *dc,
+			  struct dc_state *context,
+			  display_e2e_pipe_params_st *pipes,
+			  int pipe_cnt)
+{
+	DC_FP_START();
+	dcn30_set_mcif_arb_params(dc, context, pipes, pipe_cnt);
+	DC_FP_END();
+}
+
+bool dcn31_validate_bandwidth(struct dc *dc,
+		struct dc_state *context,
+		bool fast_validate)
+{
+	bool out = false;
+
+	BW_VAL_TRACE_SETUP();
+
+	int vlevel = 0;
+	int pipe_cnt = 0;
+	display_e2e_pipe_params_st *pipes = kzalloc(dc->res_pool->pipe_count * sizeof(display_e2e_pipe_params_st), GFP_KERNEL);
+	DC_LOGGER_INIT(dc->ctx->logger);
+
+	BW_VAL_TRACE_COUNT();
+
+	DC_FP_START();
+	out = dcn30_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, fast_validate, true);
+	DC_FP_END();
+
+	// Disable fast_validate to set min dcfclk in alculate_wm_and_dlg
+	if (pipe_cnt == 0)
+		fast_validate = false;
+
+	if (!out)
+		goto validate_fail;
+
+	BW_VAL_TRACE_END_VOLTAGE_LEVEL();
+
+	if (fast_validate) {
+		BW_VAL_TRACE_SKIP(fast);
+		goto validate_out;
+	}
+	if (dc->res_pool->funcs->calculate_wm_and_dlg)
+		dc->res_pool->funcs->calculate_wm_and_dlg(dc, context, pipes, pipe_cnt, vlevel);
+
+	BW_VAL_TRACE_END_WATERMARKS();
+
+	goto validate_out;
+
+validate_fail:
+	DC_LOG_WARNING("Mode Validation Warning: %s failed validation.\n",
+		dml_get_status_message(context->bw_ctx.dml.vba.ValidationStatus[context->bw_ctx.dml.vba.soc.num_states]));
+
+	BW_VAL_TRACE_SKIP(fail);
+	out = false;
+
+validate_out:
+	kfree(pipes);
+
+	BW_VAL_TRACE_FINISH();
+
+	return out;
+}
+
+static void dcn31_get_panel_config_defaults(struct dc_panel_config *panel_config)
+{
+	*panel_config = panel_config_defaults;
+}
+
+static struct dc_cap_funcs cap_funcs = {
+	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
+};
+
+static struct resource_funcs dcn31_res_pool_funcs = {
+	.destroy = dcn31_destroy_resource_pool,
+	.link_enc_create = dcn31_link_encoder_create,
+	.link_enc_create_minimal = dcn31_link_enc_create_minimal,
+	.link_encs_assign = link_enc_cfg_link_encs_assign,
+	.link_enc_unassign = link_enc_cfg_link_enc_unassign,
+	.panel_cntl_create = dcn31_panel_cntl_create,
+	.validate_bandwidth = dcn31_validate_bandwidth,
+	.calculate_wm_and_dlg = dcn31_calculate_wm_and_dlg,
+	.update_soc_for_wm_a = dcn31_update_soc_for_wm_a,
+	.populate_dml_pipes = dcn31_populate_dml_pipes_from_context,
+	.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
+	.release_pipe = dcn20_release_pipe,
+	.add_stream_to_ctx = dcn30_add_stream_to_ctx,
+	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
+	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
+	.populate_dml_writeback_from_context = dcn31_populate_dml_writeback_from_context,
+	.set_mcif_arb_params = dcn31_set_mcif_arb_params,
+	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
+	.acquire_post_bldn_3dlut = dcn30_acquire_post_bldn_3dlut,
+	.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
+	.update_bw_bounding_box = dcn31_update_bw_bounding_box,
+	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
+	.get_panel_config_defaults = dcn31_get_panel_config_defaults,
+};
+
+static struct clock_source *dcn30_clock_source_create(
+		struct dc_context *ctx,
+		struct dc_bios *bios,
+		enum clock_source_id id,
+		const struct dce110_clk_src_regs *regs,
+		bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dcn31_clk_src_construct(clk_src, ctx, bios, id,
+			regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static bool dcn31_resource_construct(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct dcn31_resource_pool *pool)
+{
+	int i;
+	struct dc_context *ctx = dc->ctx;
+	struct irq_service_init_data init_data;
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	pool->base.res_cap = &res_cap_dcn31;
+
+	pool->base.funcs = &dcn31_res_pool_funcs;
+
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
+	pool->base.mpcc_count = pool->base.res_cap->num_timing_generator;
+	dc->caps.max_downscale_ratio = 600;
+	dc->caps.i2c_speed_in_khz = 100;
+	dc->caps.i2c_speed_in_khz_hdcp = 5; /*1.4 w/a applied by default*/
+	dc->caps.max_cursor_size = 256;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.dmdata_alloc_size = 2048;
+
+	dc->caps.max_slave_planes = 2;
+	dc->caps.max_slave_yuv_planes = 2;
+	dc->caps.max_slave_rgb_planes = 2;
+	dc->caps.post_blend_color_processing = true;
+	dc->caps.force_dp_tps4_for_cp2520 = true;
+	if (dc->config.forceHBR2CP2520)
+		dc->caps.force_dp_tps4_for_cp2520 = false;
+	dc->caps.dp_hpo = true;
+	dc->caps.dp_hdmi21_pcon_support = true;
+	dc->caps.edp_dsc_support = true;
+	dc->caps.extended_aux_timeout_support = true;
+	dc->caps.dmcub_support = true;
+	dc->caps.is_apu = true;
+	dc->caps.zstate_support = true;
+
+	/* Color pipeline capabilities */
+	dc->caps.color.dpp.dcn_arch = 1;
+	dc->caps.color.dpp.input_lut_shared = 0;
+	dc->caps.color.dpp.icsc = 1;
+	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
+	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
+	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
+	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
+	dc->caps.color.dpp.post_csc = 1;
+	dc->caps.color.dpp.gamma_corr = 1;
+	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
+
+	dc->caps.color.dpp.hw_3d_lut = 1;
+	dc->caps.color.dpp.ogam_ram = 1;
+	// no OGAM ROM on DCN301
+	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
+	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
+	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
+	dc->caps.color.dpp.ocsc = 0;
+
+	dc->caps.color.mpc.gamut_remap = 1;
+	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //2
+	dc->caps.color.mpc.ogam_ram = 1;
+	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
+	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
+	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
+	dc->caps.color.mpc.ocsc = 1;
+
+	dc->config.use_old_fixed_vs_sequence = true;
+
+	/* Use pipe context based otg sync logic */
+	dc->config.use_pipe_ctx_sync_logic = true;
+
+	/* read VBIOS LTTPR caps */
+	{
+		if (ctx->dc_bios->funcs->get_lttpr_caps) {
+			enum bp_result bp_query_result;
+			uint8_t is_vbios_lttpr_enable = 0;
+
+			bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
+			dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
+		}
+
+		/* interop bit is implicit */
+		{
+			dc->caps.vbios_lttpr_aware = true;
+		}
+	}
+
+	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
+		dc->debug = debug_defaults_drv;
+
+	// Init the vm_helper
+	if (dc->vm_helper)
+		vm_helper_init(dc->vm_helper, 16);
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	/* Clock Sources for Pixel Clock*/
+	pool->base.clock_sources[DCN31_CLK_SRC_PLL0] =
+			dcn30_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL0,
+				&clk_src_regs[0], false);
+	pool->base.clock_sources[DCN31_CLK_SRC_PLL1] =
+			dcn30_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL1,
+				&clk_src_regs[1], false);
+	/*move phypllx_pixclk_resync to dmub next*/
+	if (dc->ctx->asic_id.hw_internal_rev == YELLOW_CARP_B0) {
+		pool->base.clock_sources[DCN31_CLK_SRC_PLL2] =
+			dcn30_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL2,
+				&clk_src_regs_b0[2], false);
+		pool->base.clock_sources[DCN31_CLK_SRC_PLL3] =
+			dcn30_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL3,
+				&clk_src_regs_b0[3], false);
+	} else {
+		pool->base.clock_sources[DCN31_CLK_SRC_PLL2] =
+			dcn30_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL2,
+				&clk_src_regs[2], false);
+		pool->base.clock_sources[DCN31_CLK_SRC_PLL3] =
+			dcn30_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL3,
+				&clk_src_regs[3], false);
+	}
+
+	pool->base.clock_sources[DCN31_CLK_SRC_PLL4] =
+			dcn30_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL4,
+				&clk_src_regs[4], false);
+
+	pool->base.clk_src_count = DCN30_CLK_SRC_TOTAL;
+
+	/* todo: not reuse phy_pll registers */
+	pool->base.dp_clock_source =
+			dcn31_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_ID_DP_DTO,
+				&clk_src_regs[0], true);
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	/* TODO: DCCG */
+	pool->base.dccg = dccg31_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
+	if (pool->base.dccg == NULL) {
+		dm_error("DC: failed to create dccg!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* TODO: IRQ */
+	init_data.ctx = dc->ctx;
+	pool->base.irqs = dal_irq_service_dcn31_create(&init_data);
+	if (!pool->base.irqs)
+		goto create_fail;
+
+	/* HUBBUB */
+	pool->base.hubbub = dcn31_hubbub_create(ctx);
+	if (pool->base.hubbub == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create hubbub!\n");
+		goto create_fail;
+	}
+
+	/* HUBPs, DPPs, OPPs and TGs */
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		pool->base.hubps[i] = dcn31_hubp_create(ctx, i);
+		if (pool->base.hubps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create hubps!\n");
+			goto create_fail;
+		}
+
+		pool->base.dpps[i] = dcn31_dpp_create(ctx, i);
+		if (pool->base.dpps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create dpps!\n");
+			goto create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		pool->base.opps[i] = dcn31_opp_create(ctx, i);
+		if (pool->base.opps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create output pixel processor!\n");
+			goto create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		pool->base.timing_generators[i] = dcn31_timing_generator_create(
+				ctx, i);
+		if (pool->base.timing_generators[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto create_fail;
+		}
+	}
+	pool->base.timing_generator_count = i;
+
+	/* PSR */
+	pool->base.psr = dmub_psr_create(ctx);
+	if (pool->base.psr == NULL) {
+		dm_error("DC: failed to create psr obj!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* Replay */
+	pool->base.replay = dmub_replay_create(ctx);
+	if (pool->base.replay == NULL) {
+		dm_error("DC: failed to create replay obj!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* ABM */
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		pool->base.multiple_abms[i] = dmub_abm_create(ctx,
+				&abm_regs[i],
+				&abm_shift,
+				&abm_mask);
+		if (pool->base.multiple_abms[i] == NULL) {
+			dm_error("DC: failed to create abm for pipe %d!\n", i);
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	/* MPC and DSC */
+	pool->base.mpc = dcn31_mpc_create(ctx, pool->base.mpcc_count, pool->base.res_cap->num_mpc_3dlut);
+	if (pool->base.mpc == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mpc!\n");
+		goto create_fail;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		pool->base.dscs[i] = dcn31_dsc_create(ctx, i);
+		if (pool->base.dscs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create display stream compressor %d!\n", i);
+			goto create_fail;
+		}
+	}
+
+	/* DWB and MMHUBBUB */
+	if (!dcn31_dwbc_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create dwbc!\n");
+		goto create_fail;
+	}
+
+	if (!dcn31_mmhubbub_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mcif_wb!\n");
+		goto create_fail;
+	}
+
+	/* AUX and I2C */
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.engines[i] = dcn31_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto create_fail;
+		}
+		pool->base.hw_i2cs[i] = dcn31_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create hw i2c!!\n");
+			goto create_fail;
+		}
+		pool->base.sw_i2cs[i] = NULL;
+	}
+
+	if (dc->ctx->asic_id.chip_family == FAMILY_YELLOW_CARP &&
+	    dc->ctx->asic_id.hw_internal_rev == YELLOW_CARP_B0 &&
+	    !dc->debug.dpia_debug.bits.disable_dpia) {
+		/* YELLOW CARP B0 has 4 DPIA's */
+		pool->base.usb4_dpia_count = 4;
+	}
+
+	if (dc->ctx->asic_id.chip_family == AMDGPU_FAMILY_GC_11_0_1)
+		pool->base.usb4_dpia_count = 4;
+
+	/* Audio, Stream Encoders including HPO and virtual, MPC 3D LUTs */
+	if (!resource_construct(num_virtual_links, dc, &pool->base,
+			&res_create_funcs))
+		goto create_fail;
+
+	/* HW Sequencer and Plane caps */
+	dcn31_hw_sequencer_construct(dc);
+
+	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->cap_funcs = cap_funcs;
+
+	dc->dcn_ip->max_num_dpp = dcn3_1_ip.max_num_dpp;
+
+	return true;
+
+create_fail:
+	dcn31_resource_destruct(pool);
+
+	return false;
+}
+
+struct resource_pool *dcn31_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc)
+{
+	struct dcn31_resource_pool *pool =
+		kzalloc(sizeof(struct dcn31_resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dcn31_resource_construct(init_data->num_virtual_links, dc, pool))
+		return &pool->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(pool);
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.h
new file mode 100644
index 000000000000..901436591ed4
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.h
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2020 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef _DCN31_RESOURCE_H_
+#define _DCN31_RESOURCE_H_
+
+#include "core_types.h"
+
+#define TO_DCN31_RES_POOL(pool)\
+	container_of(pool, struct dcn31_resource_pool, base)
+
+extern struct _vcs_dpi_ip_params_st dcn3_1_ip;
+
+struct dcn31_resource_pool {
+	struct resource_pool base;
+};
+
+bool dcn31_validate_bandwidth(struct dc *dc,
+		struct dc_state *context,
+		bool fast_validate);
+void dcn31_calculate_wm_and_dlg(
+		struct dc *dc, struct dc_state *context,
+		display_e2e_pipe_params_st *pipes,
+		int pipe_cnt,
+		int vlevel);
+int dcn31_populate_dml_pipes_from_context(
+	struct dc *dc, struct dc_state *context,
+	display_e2e_pipe_params_st *pipes,
+	bool fast_validate);
+void
+dcn31_populate_dml_writeback_from_context(struct dc *dc,
+					  struct resource_context *res_ctx,
+					  display_e2e_pipe_params_st *pipes);
+void
+dcn31_set_mcif_arb_params(struct dc *dc,
+			  struct dc_state *context,
+			  display_e2e_pipe_params_st *pipes,
+			  int pipe_cnt);
+
+struct resource_pool *dcn31_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc);
+
+/*temp: B0 specific before switch to dcn313 headers*/
+#ifndef regPHYPLLF_PIXCLK_RESYNC_CNTL
+#define regPHYPLLF_PIXCLK_RESYNC_CNTL 0x007e
+#define regPHYPLLF_PIXCLK_RESYNC_CNTL_BASE_IDX 1
+#define regPHYPLLG_PIXCLK_RESYNC_CNTL 0x005f
+#define regPHYPLLG_PIXCLK_RESYNC_CNTL_BASE_IDX 1
+
+//PHYPLLF_PIXCLK_RESYNC_CNTL
+#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_PIXCLK_RESYNC_ENABLE__SHIFT 0x0
+#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_DEEP_COLOR_DTO_ENABLE_STATUS__SHIFT 0x1
+#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_DCCG_DEEP_COLOR_CNTL__SHIFT 0x4
+#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_PIXCLK_ENABLE__SHIFT 0x8
+#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_PIXCLK_DOUBLE_RATE_ENABLE__SHIFT 0x9
+#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_PIXCLK_RESYNC_ENABLE_MASK 0x00000001L
+#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_DEEP_COLOR_DTO_ENABLE_STATUS_MASK 0x00000002L
+#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_DCCG_DEEP_COLOR_CNTL_MASK 0x00000030L
+#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_PIXCLK_ENABLE_MASK 0x00000100L
+#define PHYPLLF_PIXCLK_RESYNC_CNTL__PHYPLLF_PIXCLK_DOUBLE_RATE_ENABLE_MASK 0x00000200L
+
+//PHYPLLG_PIXCLK_RESYNC_CNTL
+#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_PIXCLK_RESYNC_ENABLE__SHIFT 0x0
+#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_DEEP_COLOR_DTO_ENABLE_STATUS__SHIFT 0x1
+#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_DCCG_DEEP_COLOR_CNTL__SHIFT 0x4
+#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_PIXCLK_ENABLE__SHIFT 0x8
+#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_PIXCLK_DOUBLE_RATE_ENABLE__SHIFT 0x9
+#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_PIXCLK_RESYNC_ENABLE_MASK 0x00000001L
+#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_DEEP_COLOR_DTO_ENABLE_STATUS_MASK 0x00000002L
+#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_DCCG_DEEP_COLOR_CNTL_MASK 0x00000030L
+#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_PIXCLK_ENABLE_MASK 0x00000100L
+#define PHYPLLG_PIXCLK_RESYNC_CNTL__PHYPLLG_PIXCLK_DOUBLE_RATE_ENABLE_MASK 0x00000200L
+#endif
+#endif /* _DCN31_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c
new file mode 100644
index 000000000000..677361d74a4e
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c
@@ -0,0 +1,2180 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+
+#include "dm_services.h"
+#include "dc.h"
+
+#include "dcn31/dcn31_init.h"
+#include "dcn314/dcn314_init.h"
+
+#include "resource.h"
+#include "include/irq_service_interface.h"
+#include "dcn314_resource.h"
+
+#include "dcn20/dcn20_resource.h"
+#include "dcn30/dcn30_resource.h"
+#include "dcn31/dcn31_resource.h"
+
+#include "dcn10/dcn10_ipp.h"
+#include "dcn30/dcn30_hubbub.h"
+#include "dcn31/dcn31_hubbub.h"
+#include "dcn30/dcn30_mpc.h"
+#include "dcn31/dcn31_hubp.h"
+#include "irq/dcn31/irq_service_dcn31.h"
+#include "irq/dcn314/irq_service_dcn314.h"
+#include "dcn30/dcn30_dpp.h"
+#include "dcn314/dcn314_optc.h"
+#include "dcn20/dcn20_hwseq.h"
+#include "dcn30/dcn30_hwseq.h"
+#include "dce110/dce110_hwseq.h"
+#include "dcn30/dcn30_opp.h"
+#include "dcn20/dcn20_dsc.h"
+#include "dcn30/dcn30_vpg.h"
+#include "dcn30/dcn30_afmt.h"
+#include "dcn31/dcn31_dio_link_encoder.h"
+#include "dcn314/dcn314_dio_stream_encoder.h"
+#include "dcn31/dcn31_hpo_dp_stream_encoder.h"
+#include "dcn31/dcn31_hpo_dp_link_encoder.h"
+#include "dcn31/dcn31_apg.h"
+#include "dcn31/dcn31_vpg.h"
+#include "dcn31/dcn31_afmt.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_hwseq.h"
+#include "clk_mgr.h"
+#include "virtual/virtual_stream_encoder.h"
+#include "dce110/dce110_resource.h"
+#include "dml/display_mode_vba.h"
+#include "dml/dcn31/dcn31_fpu.h"
+#include "dml/dcn314/dcn314_fpu.h"
+#include "dcn314/dcn314_dccg.h"
+#include "dcn10/dcn10_resource.h"
+#include "dcn31/dcn31_panel_cntl.h"
+#include "dcn314/dcn314_hwseq.h"
+
+#include "dcn30/dcn30_dwb.h"
+#include "dcn30/dcn30_mmhubbub.h"
+
+#include "dcn/dcn_3_1_4_offset.h"
+#include "dcn/dcn_3_1_4_sh_mask.h"
+#include "dpcs/dpcs_3_1_4_offset.h"
+#include "dpcs/dpcs_3_1_4_sh_mask.h"
+
+#define DCHUBBUB_DEBUG_CTRL_0__DET_DEPTH__SHIFT		0x10
+#define DCHUBBUB_DEBUG_CTRL_0__DET_DEPTH_MASK		0x01FF0000L
+
+#define DSCC0_DSCC_CONFIG0__ICH_RESET_AT_END_OF_LINE__SHIFT                   0x0
+#define DSCC0_DSCC_CONFIG0__ICH_RESET_AT_END_OF_LINE_MASK                     0x0000000FL
+
+#include "reg_helper.h"
+#include "dce/dmub_abm.h"
+#include "dce/dmub_psr.h"
+#include "dce/dmub_replay.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_i2c.h"
+#include "dml/dcn314/display_mode_vba_314.h"
+#include "vm_helper.h"
+#include "dcn20/dcn20_vmid.h"
+
+#include "link_enc_cfg.h"
+
+#define DCN_BASE__INST0_SEG1				0x000000C0
+#define DCN_BASE__INST0_SEG2				0x000034C0
+#define DCN_BASE__INST0_SEG3				0x00009000
+
+#define NBIO_BASE__INST0_SEG1				0x00000014
+
+#define MAX_INSTANCE					7
+#define MAX_SEGMENT					8
+
+#define regBIF_BX2_BIOS_SCRATCH_2			0x003a
+#define regBIF_BX2_BIOS_SCRATCH_2_BASE_IDX		1
+#define regBIF_BX2_BIOS_SCRATCH_3			0x003b
+#define regBIF_BX2_BIOS_SCRATCH_3_BASE_IDX		1
+#define regBIF_BX2_BIOS_SCRATCH_6			0x003e
+#define regBIF_BX2_BIOS_SCRATCH_6_BASE_IDX		1
+
+#define DC_LOGGER \
+	dc->ctx->logger
+#define DC_LOGGER_INIT(logger)
+
+enum dcn31_clk_src_array_id {
+	DCN31_CLK_SRC_PLL0,
+	DCN31_CLK_SRC_PLL1,
+	DCN31_CLK_SRC_PLL2,
+	DCN31_CLK_SRC_PLL3,
+	DCN31_CLK_SRC_PLL4,
+	DCN30_CLK_SRC_TOTAL
+};
+
+/* begin *********************
+ * macros to expend register list macro defined in HW object header file
+ */
+
+/* DCN */
+/* TODO awful hack. fixup dcn20_dwb.h */
+#undef BASE_INNER
+#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
+
+#define BASE(seg) BASE_INNER(seg)
+
+#define SR(reg_name)\
+		.reg_name = BASE(reg ## reg_name ## _BASE_IDX) +  \
+					reg ## reg_name
+
+#define SRI(reg_name, block, id)\
+	.reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define SRI2(reg_name, block, id)\
+	.reg_name = BASE(reg ## reg_name ## _BASE_IDX) + \
+					reg ## reg_name
+
+#define SRIR(var_name, reg_name, block, id)\
+	.var_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define SRII(reg_name, block, id)\
+	.reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define SRII_MPC_RMU(reg_name, block, id)\
+	.RMU##_##reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define SRII_DWB(reg_name, temp_name, block, id)\
+	.reg_name[id] = BASE(reg ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## temp_name
+
+#define SF_DWB2(reg_name, block, id, field_name, post_fix)	\
+	.field_name = reg_name ## __ ## field_name ## post_fix
+
+#define DCCG_SRII(reg_name, block, id)\
+	.block ## _ ## reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define VUPDATE_SRII(reg_name, block, id)\
+	.reg_name[id] = BASE(reg ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
+					reg ## reg_name ## _ ## block ## id
+
+/* NBIO */
+#define NBIO_BASE_INNER(seg) \
+	NBIO_BASE__INST0_SEG ## seg
+
+#define NBIO_BASE(seg) \
+	NBIO_BASE_INNER(seg)
+
+#define NBIO_SR(reg_name)\
+		.reg_name = NBIO_BASE(regBIF_BX2_ ## reg_name ## _BASE_IDX) + \
+					regBIF_BX2_ ## reg_name
+
+/* MMHUB */
+#define MMHUB_BASE_INNER(seg) \
+	MMHUB_BASE__INST0_SEG ## seg
+
+#define MMHUB_BASE(seg) \
+	MMHUB_BASE_INNER(seg)
+
+#define MMHUB_SR(reg_name)\
+		.reg_name = MMHUB_BASE(reg ## reg_name ## _BASE_IDX) + \
+					reg ## reg_name
+
+/* CLOCK */
+#define CLK_BASE_INNER(seg) \
+	CLK_BASE__INST0_SEG ## seg
+
+#define CLK_BASE(seg) \
+	CLK_BASE_INNER(seg)
+
+#define CLK_SRI(reg_name, block, inst)\
+	.reg_name = CLK_BASE(reg ## block ## _ ## inst ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## _ ## inst ## _ ## reg_name
+
+
+static const struct bios_registers bios_regs = {
+		NBIO_SR(BIOS_SCRATCH_3),
+		NBIO_SR(BIOS_SCRATCH_6)
+};
+
+#define clk_src_regs(index, pllid)\
+[index] = {\
+	CS_COMMON_REG_LIST_DCN3_0(index, pllid),\
+}
+
+static const struct dce110_clk_src_regs clk_src_regs[] = {
+	clk_src_regs(0, A),
+	clk_src_regs(1, B),
+	clk_src_regs(2, C),
+	clk_src_regs(3, D),
+	clk_src_regs(4, E)
+};
+
+static const struct dce110_clk_src_shift cs_shift = {
+		CS_COMMON_MASK_SH_LIST_DCN3_1_4(__SHIFT)
+};
+
+static const struct dce110_clk_src_mask cs_mask = {
+		CS_COMMON_MASK_SH_LIST_DCN3_1_4(_MASK)
+};
+
+#define abm_regs(id)\
+[id] = {\
+		ABM_DCN302_REG_LIST(id)\
+}
+
+static const struct dce_abm_registers abm_regs[] = {
+		abm_regs(0),
+		abm_regs(1),
+		abm_regs(2),
+		abm_regs(3),
+};
+
+static const struct dce_abm_shift abm_shift = {
+		ABM_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dce_abm_mask abm_mask = {
+		ABM_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define audio_regs(id)\
+[id] = {\
+		AUD_COMMON_REG_LIST(id)\
+}
+
+static const struct dce_audio_registers audio_regs[] = {
+	audio_regs(0),
+	audio_regs(1),
+	audio_regs(2),
+	audio_regs(3),
+	audio_regs(4),
+	audio_regs(5),
+	audio_regs(6)
+};
+
+#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
+		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
+
+static const struct dce_audio_shift audio_shift = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_audio_mask audio_mask = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+#define vpg_regs(id)\
+[id] = {\
+	VPG_DCN31_REG_LIST(id)\
+}
+
+static const struct dcn31_vpg_registers vpg_regs[] = {
+	vpg_regs(0),
+	vpg_regs(1),
+	vpg_regs(2),
+	vpg_regs(3),
+	vpg_regs(4),
+	vpg_regs(5),
+	vpg_regs(6),
+	vpg_regs(7),
+	vpg_regs(8),
+	vpg_regs(9),
+};
+
+static const struct dcn31_vpg_shift vpg_shift = {
+	DCN31_VPG_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_vpg_mask vpg_mask = {
+	DCN31_VPG_MASK_SH_LIST(_MASK)
+};
+
+#define afmt_regs(id)\
+[id] = {\
+	AFMT_DCN31_REG_LIST(id)\
+}
+
+static const struct dcn31_afmt_registers afmt_regs[] = {
+	afmt_regs(0),
+	afmt_regs(1),
+	afmt_regs(2),
+	afmt_regs(3),
+	afmt_regs(4),
+	afmt_regs(5)
+};
+
+static const struct dcn31_afmt_shift afmt_shift = {
+	DCN31_AFMT_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_afmt_mask afmt_mask = {
+	DCN31_AFMT_MASK_SH_LIST(_MASK)
+};
+
+#define apg_regs(id)\
+[id] = {\
+	APG_DCN31_REG_LIST(id)\
+}
+
+static const struct dcn31_apg_registers apg_regs[] = {
+	apg_regs(0),
+	apg_regs(1),
+	apg_regs(2),
+	apg_regs(3)
+};
+
+static const struct dcn31_apg_shift apg_shift = {
+	DCN31_APG_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_apg_mask apg_mask = {
+		DCN31_APG_MASK_SH_LIST(_MASK)
+};
+
+#define stream_enc_regs(id)\
+[id] = {\
+		SE_DCN314_REG_LIST(id)\
+}
+
+static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
+	stream_enc_regs(0),
+	stream_enc_regs(1),
+	stream_enc_regs(2),
+	stream_enc_regs(3),
+	stream_enc_regs(4)
+};
+
+static const struct dcn10_stream_encoder_shift se_shift = {
+		SE_COMMON_MASK_SH_LIST_DCN314(__SHIFT)
+};
+
+static const struct dcn10_stream_encoder_mask se_mask = {
+		SE_COMMON_MASK_SH_LIST_DCN314(_MASK)
+};
+
+
+#define aux_regs(id)\
+[id] = {\
+	DCN2_AUX_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
+		aux_regs(0),
+		aux_regs(1),
+		aux_regs(2),
+		aux_regs(3),
+		aux_regs(4)
+};
+
+#define hpd_regs(id)\
+[id] = {\
+	HPD_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
+		hpd_regs(0),
+		hpd_regs(1),
+		hpd_regs(2),
+		hpd_regs(3),
+		hpd_regs(4)
+};
+
+#define link_regs(id, phyid)\
+[id] = {\
+	LE_DCN31_REG_LIST(id), \
+	UNIPHY_DCN2_REG_LIST(phyid), \
+}
+
+static const struct dce110_aux_registers_shift aux_shift = {
+	DCN_AUX_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+	DCN_AUX_MASK_SH_LIST(_MASK)
+};
+
+static const struct dcn10_link_enc_registers link_enc_regs[] = {
+	link_regs(0, A),
+	link_regs(1, B),
+	link_regs(2, C),
+	link_regs(3, D),
+	link_regs(4, E)
+};
+
+static const struct dcn10_link_enc_shift le_shift = {
+	LINK_ENCODER_MASK_SH_LIST_DCN31(__SHIFT),
+	DPCS_DCN31_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn10_link_enc_mask le_mask = {
+	LINK_ENCODER_MASK_SH_LIST_DCN31(_MASK),
+	DPCS_DCN31_MASK_SH_LIST(_MASK)
+};
+
+#define hpo_dp_stream_encoder_reg_list(id)\
+[id] = {\
+	DCN3_1_HPO_DP_STREAM_ENC_REG_LIST(id)\
+}
+
+static const struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[] = {
+	hpo_dp_stream_encoder_reg_list(0),
+	hpo_dp_stream_encoder_reg_list(1),
+	hpo_dp_stream_encoder_reg_list(2),
+	hpo_dp_stream_encoder_reg_list(3)
+};
+
+static const struct dcn31_hpo_dp_stream_encoder_shift hpo_dp_se_shift = {
+	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_hpo_dp_stream_encoder_mask hpo_dp_se_mask = {
+	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(_MASK)
+};
+
+
+#define hpo_dp_link_encoder_reg_list(id)\
+[id] = {\
+	DCN3_1_HPO_DP_LINK_ENC_REG_LIST(id),\
+	DCN3_1_RDPCSTX_REG_LIST(0),\
+	DCN3_1_RDPCSTX_REG_LIST(1),\
+	DCN3_1_RDPCSTX_REG_LIST(2),\
+}
+
+static const struct dcn31_hpo_dp_link_encoder_registers hpo_dp_link_enc_regs[] = {
+	hpo_dp_link_encoder_reg_list(0),
+	hpo_dp_link_encoder_reg_list(1),
+};
+
+static const struct dcn31_hpo_dp_link_encoder_shift hpo_dp_le_shift = {
+	DCN3_1_HPO_DP_LINK_ENC_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_hpo_dp_link_encoder_mask hpo_dp_le_mask = {
+	DCN3_1_HPO_DP_LINK_ENC_MASK_SH_LIST(_MASK)
+};
+
+#define dpp_regs(id)\
+[id] = {\
+	DPP_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn3_dpp_registers dpp_regs[] = {
+	dpp_regs(0),
+	dpp_regs(1),
+	dpp_regs(2),
+	dpp_regs(3)
+};
+
+static const struct dcn3_dpp_shift tf_shift = {
+		DPP_REG_LIST_SH_MASK_DCN30(__SHIFT)
+};
+
+static const struct dcn3_dpp_mask tf_mask = {
+		DPP_REG_LIST_SH_MASK_DCN30(_MASK)
+};
+
+#define opp_regs(id)\
+[id] = {\
+	OPP_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn20_opp_registers opp_regs[] = {
+	opp_regs(0),
+	opp_regs(1),
+	opp_regs(2),
+	opp_regs(3)
+};
+
+static const struct dcn20_opp_shift opp_shift = {
+	OPP_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn20_opp_mask opp_mask = {
+	OPP_MASK_SH_LIST_DCN20(_MASK)
+};
+
+#define aux_engine_regs(id)\
+[id] = {\
+	AUX_COMMON_REG_LIST0(id), \
+	.AUXN_IMPCAL = 0, \
+	.AUXP_IMPCAL = 0, \
+	.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1),
+		aux_engine_regs(2),
+		aux_engine_regs(3),
+		aux_engine_regs(4)
+};
+
+#define dwbc_regs_dcn3(id)\
+[id] = {\
+	DWBC_COMMON_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn30_dwbc_registers dwbc30_regs[] = {
+	dwbc_regs_dcn3(0),
+};
+
+static const struct dcn30_dwbc_shift dwbc30_shift = {
+	DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_dwbc_mask dwbc30_mask = {
+	DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define mcif_wb_regs_dcn3(id)\
+[id] = {\
+	MCIF_WB_COMMON_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn30_mmhubbub_registers mcif_wb30_regs[] = {
+	mcif_wb_regs_dcn3(0)
+};
+
+static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define dsc_regsDCN314(id)\
+[id] = {\
+	DSC_REG_LIST_DCN20(id)\
+}
+
+static const struct dcn20_dsc_registers dsc_regs[] = {
+	dsc_regsDCN314(0),
+	dsc_regsDCN314(1),
+	dsc_regsDCN314(2),
+	dsc_regsDCN314(3)
+};
+
+static const struct dcn20_dsc_shift dsc_shift = {
+	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
+};
+
+static const struct dcn20_dsc_mask dsc_mask = {
+	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
+};
+
+static const struct dcn30_mpc_registers mpc_regs = {
+		MPC_REG_LIST_DCN3_0(0),
+		MPC_REG_LIST_DCN3_0(1),
+		MPC_REG_LIST_DCN3_0(2),
+		MPC_REG_LIST_DCN3_0(3),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(0),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(1),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(2),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(3),
+		MPC_RMU_GLOBAL_REG_LIST_DCN3AG,
+		MPC_RMU_REG_LIST_DCN3AG(0),
+		MPC_RMU_REG_LIST_DCN3AG(1),
+		//MPC_RMU_REG_LIST_DCN3AG(2),
+		MPC_DWB_MUX_REG_LIST_DCN3_0(0),
+};
+
+static const struct dcn30_mpc_shift mpc_shift = {
+	MPC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_mpc_mask mpc_mask = {
+	MPC_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define optc_regs(id)\
+[id] = {OPTC_COMMON_REG_LIST_DCN3_14(id)}
+
+static const struct dcn_optc_registers optc_regs[] = {
+	optc_regs(0),
+	optc_regs(1),
+	optc_regs(2),
+	optc_regs(3)
+};
+
+static const struct dcn_optc_shift optc_shift = {
+	OPTC_COMMON_MASK_SH_LIST_DCN3_14(__SHIFT)
+};
+
+static const struct dcn_optc_mask optc_mask = {
+	OPTC_COMMON_MASK_SH_LIST_DCN3_14(_MASK)
+};
+
+#define hubp_regs(id)\
+[id] = {\
+	HUBP_REG_LIST_DCN30(id)\
+}
+
+static const struct dcn_hubp2_registers hubp_regs[] = {
+		hubp_regs(0),
+		hubp_regs(1),
+		hubp_regs(2),
+		hubp_regs(3)
+};
+
+
+static const struct dcn_hubp2_shift hubp_shift = {
+		HUBP_MASK_SH_LIST_DCN31(__SHIFT)
+};
+
+static const struct dcn_hubp2_mask hubp_mask = {
+		HUBP_MASK_SH_LIST_DCN31(_MASK)
+};
+static const struct dcn_hubbub_registers hubbub_reg = {
+		HUBBUB_REG_LIST_DCN31(0)
+};
+
+static const struct dcn_hubbub_shift hubbub_shift = {
+		HUBBUB_MASK_SH_LIST_DCN31(__SHIFT)
+};
+
+static const struct dcn_hubbub_mask hubbub_mask = {
+		HUBBUB_MASK_SH_LIST_DCN31(_MASK)
+};
+
+static const struct dccg_registers dccg_regs = {
+		DCCG_REG_LIST_DCN314()
+};
+
+static const struct dccg_shift dccg_shift = {
+		DCCG_MASK_SH_LIST_DCN314(__SHIFT)
+};
+
+static const struct dccg_mask dccg_mask = {
+		DCCG_MASK_SH_LIST_DCN314(_MASK)
+};
+
+
+#define SRII2(reg_name_pre, reg_name_post, id)\
+	.reg_name_pre ## _ ##  reg_name_post[id] = BASE(reg ## reg_name_pre \
+			## id ## _ ## reg_name_post ## _BASE_IDX) + \
+			reg ## reg_name_pre ## id ## _ ## reg_name_post
+
+
+#define HWSEQ_DCN31_REG_LIST()\
+	SR(DCHUBBUB_GLOBAL_TIMER_CNTL), \
+	SR(DCHUBBUB_ARB_HOSTVM_CNTL), \
+	SR(DIO_MEM_PWR_CTRL), \
+	SR(ODM_MEM_PWR_CTRL3), \
+	SR(DMU_MEM_PWR_CNTL), \
+	SR(MMHUBBUB_MEM_PWR_CNTL), \
+	SR(DCCG_GATE_DISABLE_CNTL), \
+	SR(DCCG_GATE_DISABLE_CNTL2), \
+	SR(DCFCLK_CNTL),\
+	SR(DC_MEM_GLOBAL_PWR_REQ_CNTL), \
+	SRII(PIXEL_RATE_CNTL, OTG, 0), \
+	SRII(PIXEL_RATE_CNTL, OTG, 1),\
+	SRII(PIXEL_RATE_CNTL, OTG, 2),\
+	SRII(PIXEL_RATE_CNTL, OTG, 3),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 0),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 1),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 2),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 3),\
+	SR(MICROSECOND_TIME_BASE_DIV), \
+	SR(MILLISECOND_TIME_BASE_DIV), \
+	SR(DISPCLK_FREQ_CHANGE_CNTL), \
+	SR(RBBMIF_TIMEOUT_DIS), \
+	SR(RBBMIF_TIMEOUT_DIS_2), \
+	SR(DCHUBBUB_CRC_CTRL), \
+	SR(DPP_TOP0_DPP_CRC_CTRL), \
+	SR(DPP_TOP0_DPP_CRC_VAL_B_A), \
+	SR(DPP_TOP0_DPP_CRC_VAL_R_G), \
+	SR(MPC_CRC_CTRL), \
+	SR(MPC_CRC_RESULT_GB), \
+	SR(MPC_CRC_RESULT_C), \
+	SR(MPC_CRC_RESULT_AR), \
+	SR(DOMAIN0_PG_CONFIG), \
+	SR(DOMAIN1_PG_CONFIG), \
+	SR(DOMAIN2_PG_CONFIG), \
+	SR(DOMAIN3_PG_CONFIG), \
+	SR(DOMAIN16_PG_CONFIG), \
+	SR(DOMAIN17_PG_CONFIG), \
+	SR(DOMAIN18_PG_CONFIG), \
+	SR(DOMAIN19_PG_CONFIG), \
+	SR(DOMAIN0_PG_STATUS), \
+	SR(DOMAIN1_PG_STATUS), \
+	SR(DOMAIN2_PG_STATUS), \
+	SR(DOMAIN3_PG_STATUS), \
+	SR(DOMAIN16_PG_STATUS), \
+	SR(DOMAIN17_PG_STATUS), \
+	SR(DOMAIN18_PG_STATUS), \
+	SR(DOMAIN19_PG_STATUS), \
+	SR(D1VGA_CONTROL), \
+	SR(D2VGA_CONTROL), \
+	SR(D3VGA_CONTROL), \
+	SR(D4VGA_CONTROL), \
+	SR(D5VGA_CONTROL), \
+	SR(D6VGA_CONTROL), \
+	SR(DC_IP_REQUEST_CNTL), \
+	SR(AZALIA_AUDIO_DTO), \
+	SR(AZALIA_CONTROLLER_CLOCK_GATING), \
+	SR(HPO_TOP_HW_CONTROL)
+
+static const struct dce_hwseq_registers hwseq_reg = {
+		HWSEQ_DCN31_REG_LIST()
+};
+
+#define HWSEQ_DCN31_MASK_SH_LIST(mask_sh)\
+	HWSEQ_DCN_MASK_SH_LIST(mask_sh), \
+	HWS_SF(, DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, mask_sh), \
+	HWS_SF(, DCHUBBUB_ARB_HOSTVM_CNTL, DISABLE_HOSTVM_FORCE_ALLOW_PSTATE, mask_sh), \
+	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN0_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN1_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN2_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN3_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN16_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN17_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN18_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN19_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh), \
+	HWS_SF(, AZALIA_AUDIO_DTO, AZALIA_AUDIO_DTO_MODULE, mask_sh), \
+	HWS_SF(, HPO_TOP_CLOCK_CONTROL, HPO_HDMISTREAMCLK_G_GATE_DIS, mask_sh), \
+	HWS_SF(, DMU_MEM_PWR_CNTL, DMCU_ERAM_MEM_PWR_FORCE, mask_sh), \
+	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_UNASSIGNED_PWR_MODE, mask_sh), \
+	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_VBLANK_PWR_MODE, mask_sh), \
+	HWS_SF(, MMHUBBUB_MEM_PWR_CNTL, VGA_MEM_PWR_FORCE, mask_sh), \
+	HWS_SF(, DIO_MEM_PWR_CTRL, I2C_LIGHT_SLEEP_FORCE, mask_sh), \
+	HWS_SF(, HPO_TOP_HW_CONTROL, HPO_IO_EN, mask_sh)
+
+static const struct dce_hwseq_shift hwseq_shift = {
+		HWSEQ_DCN31_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+		HWSEQ_DCN31_MASK_SH_LIST(_MASK)
+};
+#define vmid_regs(id)\
+[id] = {\
+		DCN20_VMID_REG_LIST(id)\
+}
+
+static const struct dcn_vmid_registers vmid_regs[] = {
+	vmid_regs(0),
+	vmid_regs(1),
+	vmid_regs(2),
+	vmid_regs(3),
+	vmid_regs(4),
+	vmid_regs(5),
+	vmid_regs(6),
+	vmid_regs(7),
+	vmid_regs(8),
+	vmid_regs(9),
+	vmid_regs(10),
+	vmid_regs(11),
+	vmid_regs(12),
+	vmid_regs(13),
+	vmid_regs(14),
+	vmid_regs(15)
+};
+
+static const struct dcn20_vmid_shift vmid_shifts = {
+		DCN20_VMID_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn20_vmid_mask vmid_masks = {
+		DCN20_VMID_MASK_SH_LIST(_MASK)
+};
+
+static const struct resource_caps res_cap_dcn314 = {
+	.num_timing_generator = 4,
+	.num_opp = 4,
+	.num_video_plane = 4,
+	.num_audio = 5,
+	.num_stream_encoder = 5,
+	.num_dig_link_enc = 5,
+	.num_hpo_dp_stream_encoder = 4,
+	.num_hpo_dp_link_encoder = 2,
+	.num_pll = 5,
+	.num_dwb = 1,
+	.num_ddc = 5,
+	.num_vmid = 16,
+	.num_mpc_3dlut = 2,
+	.num_dsc = 4,
+};
+
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
+	.per_pixel_alpha = true,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = true,
+			.fp16 = true,
+			.p010 = true,
+			.ayuv = false,
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 16000,
+			.fp16 = 16000
+	},
+
+	// 6:1 downscaling ratio: 1000/6 = 166.666
+	// 4:1 downscaling ratio for ARGB888 to prevent underflow during P010 playback: 1000/4 = 250
+	.max_downscale_factor = {
+			.argb8888 = 250,
+			.nv12 = 167,
+			.fp16 = 167
+	},
+	64,
+	64
+};
+
+static const struct dc_debug_options debug_defaults_drv = {
+	.disable_z10 = false,
+	.enable_z9_disable_interface = true,
+	.minimum_z8_residency_time = 2000,
+	.psr_skip_crtc_disable = true,
+	.replay_skip_crtc_disabled = true,
+	.disable_dmcu = true,
+	.force_abm_enable = false,
+	.timing_trace = false,
+	.clock_trace = true,
+	.disable_dpp_power_gate = false,
+	.disable_hubp_power_gate = false,
+	.disable_pplib_clock_request = false,
+	.pipe_split_policy = MPC_SPLIT_DYNAMIC,
+	.force_single_disp_pipe_split = false,
+	.disable_dcc = DCC_ENABLE,
+	.vsr_support = true,
+	.performance_trace = false,
+	.max_downscale_src_width = 4096,/*upto true 4k*/
+	.disable_pplib_wm_range = false,
+	.scl_reset_length10 = true,
+	.sanity_checks = true,
+	.underflow_assert_delay_us = 0xFFFFFFFF,
+	.dwb_fi_phase = -1, // -1 = disable,
+	.dmub_command_table = true,
+	.pstate_enabled = true,
+	.use_max_lb = true,
+	.enable_mem_low_power = {
+		.bits = {
+			.vga = true,
+			.i2c = true,
+			.dmcu = false, // This is previously known to cause hang on S3 cycles if enabled
+			.dscl = true,
+			.cm = true,
+			.mpc = true,
+			.optc = true,
+			.vpg = true,
+			.afmt = true,
+		}
+	},
+
+	.root_clock_optimization = {
+			.bits = {
+					.dpp = true,
+					.dsc = true,
+					.hdmistream = true,
+					.hdmichar = true,
+					.dpstream = true,
+					.symclk32_se = false,
+					.symclk32_le = true,
+					.symclk_fe = true,
+					.physymclk = true,
+					.dpiasymclk = true,
+			}
+	},
+
+	.seamless_boot_odm_combine = true,
+	.using_dml2 = false,
+};
+
+static const struct dc_debug_options debug_defaults_diags = {
+	.disable_dmcu = true,
+	.force_abm_enable = false,
+	.timing_trace = true,
+	.clock_trace = true,
+	.disable_dpp_power_gate = true,
+	.disable_hubp_power_gate = true,
+	.disable_clock_gate = true,
+	.disable_pplib_clock_request = true,
+	.disable_pplib_wm_range = true,
+	.disable_stutter = false,
+	.scl_reset_length10 = true,
+	.dwb_fi_phase = -1, // -1 = disable
+	.dmub_command_table = true,
+	.enable_tri_buf = true,
+	.use_max_lb = true
+};
+
+static const struct dc_panel_config panel_config_defaults = {
+	.psr = {
+		.disable_psr = false,
+		.disallow_psrsu = false,
+		.disallow_replay = false,
+	},
+	.ilr = {
+		.optimize_edp_link_rate = true,
+	},
+};
+
+static void dcn31_dpp_destroy(struct dpp **dpp)
+{
+	kfree(TO_DCN20_DPP(*dpp));
+	*dpp = NULL;
+}
+
+static struct dpp *dcn31_dpp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn3_dpp *dpp =
+		kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
+
+	if (!dpp)
+		return NULL;
+
+	if (dpp3_construct(dpp, ctx, inst,
+			&dpp_regs[inst], &tf_shift, &tf_mask))
+		return &dpp->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(dpp);
+	return NULL;
+}
+
+static struct output_pixel_processor *dcn31_opp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_opp *opp =
+		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
+
+	if (!opp) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dcn20_opp_construct(opp, ctx, inst,
+			&opp_regs[inst], &opp_shift, &opp_mask);
+	return &opp->base;
+}
+
+static struct dce_aux *dcn31_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst],
+					&aux_mask,
+					&aux_shift,
+					ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST_DCN30(id) }
+
+static const struct dce_i2c_registers i2c_hw_regs[] = {
+		i2c_inst_regs(1),
+		i2c_inst_regs(2),
+		i2c_inst_regs(3),
+		i2c_inst_regs(4),
+		i2c_inst_regs(5),
+};
+
+static const struct dce_i2c_shift i2c_shifts = {
+		I2C_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dce_i2c_mask i2c_masks = {
+		I2C_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+/* ========================================================== */
+
+/*
+ * DPIA index | Preferred Encoder     |    Host Router
+ *   0        |      C                |       0
+ *   1        |      First Available  |       0
+ *   2        |      D                |       1
+ *   3        |      First Available  |       1
+ */
+/* ========================================================== */
+static const enum engine_id dpia_to_preferred_enc_id_table[] = {
+		ENGINE_ID_DIGC,
+		ENGINE_ID_DIGC,
+		ENGINE_ID_DIGD,
+		ENGINE_ID_DIGD
+};
+
+static enum engine_id dcn314_get_preferred_eng_id_dpia(unsigned int dpia_index)
+{
+	return dpia_to_preferred_enc_id_table[dpia_index];
+}
+
+static struct dce_i2c_hw *dcn31_i2c_hw_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw =
+		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
+				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+static struct mpc *dcn31_mpc_create(
+		struct dc_context *ctx,
+		int num_mpcc,
+		int num_rmu)
+{
+	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc),
+					  GFP_KERNEL);
+
+	if (!mpc30)
+		return NULL;
+
+	dcn30_mpc_construct(mpc30, ctx,
+			&mpc_regs,
+			&mpc_shift,
+			&mpc_mask,
+			num_mpcc,
+			num_rmu);
+
+	return &mpc30->base;
+}
+
+static struct hubbub *dcn31_hubbub_create(struct dc_context *ctx)
+{
+	int i;
+
+	struct dcn20_hubbub *hubbub3 = kzalloc(sizeof(struct dcn20_hubbub),
+					  GFP_KERNEL);
+
+	if (!hubbub3)
+		return NULL;
+
+	hubbub31_construct(hubbub3, ctx,
+			&hubbub_reg,
+			&hubbub_shift,
+			&hubbub_mask,
+			dcn3_14_ip.det_buffer_size_kbytes,
+			dcn3_14_ip.pixel_chunk_size_kbytes,
+			dcn3_14_ip.config_return_buffer_size_in_kbytes);
+
+
+	for (i = 0; i < res_cap_dcn314.num_vmid; i++) {
+		struct dcn20_vmid *vmid = &hubbub3->vmid[i];
+
+		vmid->ctx = ctx;
+
+		vmid->regs = &vmid_regs[i];
+		vmid->shifts = &vmid_shifts;
+		vmid->masks = &vmid_masks;
+	}
+
+	return &hubbub3->base;
+}
+
+static struct timing_generator *dcn31_timing_generator_create(
+		struct dc_context *ctx,
+		uint32_t instance)
+{
+	struct optc *tgn10 =
+		kzalloc(sizeof(struct optc), GFP_KERNEL);
+
+	if (!tgn10)
+		return NULL;
+
+	tgn10->base.inst = instance;
+	tgn10->base.ctx = ctx;
+
+	tgn10->tg_regs = &optc_regs[instance];
+	tgn10->tg_shift = &optc_shift;
+	tgn10->tg_mask = &optc_mask;
+
+	dcn314_timing_generator_init(tgn10);
+
+	return &tgn10->base;
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+		.max_hdmi_deep_color = COLOR_DEPTH_121212,
+		.max_hdmi_pixel_clock = 600000,
+		.hdmi_ycbcr420_supported = true,
+		.dp_ycbcr420_supported = true,
+		.fec_supported = true,
+		.flags.bits.IS_HBR2_CAPABLE = true,
+		.flags.bits.IS_HBR3_CAPABLE = true,
+		.flags.bits.IS_TPS3_CAPABLE = true,
+		.flags.bits.IS_TPS4_CAPABLE = true
+};
+
+static struct link_encoder *dcn31_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dcn20_link_encoder *enc20 =
+		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+
+	if (!enc20)
+		return NULL;
+
+	dcn31_link_encoder_construct(enc20,
+			enc_init_data,
+			&link_enc_feature,
+			&link_enc_regs[enc_init_data->transmitter],
+			&link_enc_aux_regs[enc_init_data->channel - 1],
+			&link_enc_hpd_regs[enc_init_data->hpd_source],
+			&le_shift,
+			&le_mask);
+
+	return &enc20->enc10.base;
+}
+
+/* Create a minimal link encoder object not associated with a particular
+ * physical connector.
+ * resource_funcs.link_enc_create_minimal
+ */
+static struct link_encoder *dcn31_link_enc_create_minimal(
+		struct dc_context *ctx, enum engine_id eng_id)
+{
+	struct dcn20_link_encoder *enc20;
+
+	if ((eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+		return NULL;
+
+	enc20 = kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+	if (!enc20)
+		return NULL;
+
+	dcn31_link_encoder_construct_minimal(
+			enc20,
+			ctx,
+			&link_enc_feature,
+			&link_enc_regs[eng_id - ENGINE_ID_DIGA],
+			eng_id);
+
+	return &enc20->enc10.base;
+}
+
+static struct panel_cntl *dcn31_panel_cntl_create(const struct panel_cntl_init_data *init_data)
+{
+	struct dcn31_panel_cntl *panel_cntl =
+		kzalloc(sizeof(struct dcn31_panel_cntl), GFP_KERNEL);
+
+	if (!panel_cntl)
+		return NULL;
+
+	dcn31_panel_cntl_construct(panel_cntl, init_data);
+
+	return &panel_cntl->base;
+}
+
+static void read_dce_straps(
+	struct dc_context *ctx,
+	struct resource_straps *straps)
+{
+	generic_reg_get(ctx, regDC_PINSTRAPS + BASE(regDC_PINSTRAPS_BASE_IDX),
+		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
+
+}
+
+static struct audio *dcn31_create_audio(
+		struct dc_context *ctx, unsigned int inst)
+{
+	return dce_audio_create(ctx, inst,
+			&audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+static struct vpg *dcn31_vpg_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn31_vpg *vpg31 = kzalloc(sizeof(struct dcn31_vpg), GFP_KERNEL);
+
+	if (!vpg31)
+		return NULL;
+
+	vpg31_construct(vpg31, ctx, inst,
+			&vpg_regs[inst],
+			&vpg_shift,
+			&vpg_mask);
+
+	return &vpg31->base;
+}
+
+static struct afmt *dcn31_afmt_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn31_afmt *afmt31 = kzalloc(sizeof(struct dcn31_afmt), GFP_KERNEL);
+
+	if (!afmt31)
+		return NULL;
+
+	afmt31_construct(afmt31, ctx, inst,
+			&afmt_regs[inst],
+			&afmt_shift,
+			&afmt_mask);
+
+	// Light sleep by default, no need to power down here
+
+	return &afmt31->base;
+}
+
+static struct apg *dcn31_apg_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn31_apg *apg31 = kzalloc(sizeof(struct dcn31_apg), GFP_KERNEL);
+
+	if (!apg31)
+		return NULL;
+
+	apg31_construct(apg31, ctx, inst,
+			&apg_regs[inst],
+			&apg_shift,
+			&apg_mask);
+
+	return &apg31->base;
+}
+
+static struct stream_encoder *dcn314_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dcn10_stream_encoder *enc1;
+	struct vpg *vpg;
+	struct afmt *afmt;
+	int vpg_inst;
+	int afmt_inst;
+
+	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
+	if (eng_id < ENGINE_ID_DIGF) {
+		vpg_inst = eng_id;
+		afmt_inst = eng_id;
+	} else
+		return NULL;
+
+	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
+	vpg = dcn31_vpg_create(ctx, vpg_inst);
+	afmt = dcn31_afmt_create(ctx, afmt_inst);
+
+	if (!enc1 || !vpg || !afmt) {
+		kfree(enc1);
+		kfree(vpg);
+		kfree(afmt);
+		return NULL;
+	}
+
+	dcn314_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
+					eng_id, vpg, afmt,
+					&stream_enc_regs[eng_id],
+					&se_shift, &se_mask);
+
+	return &enc1->base;
+}
+
+static struct hpo_dp_stream_encoder *dcn31_hpo_dp_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dcn31_hpo_dp_stream_encoder *hpo_dp_enc31;
+	struct vpg *vpg;
+	struct apg *apg;
+	uint32_t hpo_dp_inst;
+	uint32_t vpg_inst;
+	uint32_t apg_inst;
+
+	ASSERT((eng_id >= ENGINE_ID_HPO_DP_0) && (eng_id <= ENGINE_ID_HPO_DP_3));
+	hpo_dp_inst = eng_id - ENGINE_ID_HPO_DP_0;
+
+	/* Mapping of VPG register blocks to HPO DP block instance:
+	 * VPG[6] -> HPO_DP[0]
+	 * VPG[7] -> HPO_DP[1]
+	 * VPG[8] -> HPO_DP[2]
+	 * VPG[9] -> HPO_DP[3]
+	 */
+	//Uses offset index 5-8, but actually maps to vpg_inst 6-9
+	vpg_inst = hpo_dp_inst + 5;
+
+	/* Mapping of APG register blocks to HPO DP block instance:
+	 * APG[0] -> HPO_DP[0]
+	 * APG[1] -> HPO_DP[1]
+	 * APG[2] -> HPO_DP[2]
+	 * APG[3] -> HPO_DP[3]
+	 */
+	apg_inst = hpo_dp_inst;
+
+	/* allocate HPO stream encoder and create VPG sub-block */
+	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_stream_encoder), GFP_KERNEL);
+	vpg = dcn31_vpg_create(ctx, vpg_inst);
+	apg = dcn31_apg_create(ctx, apg_inst);
+
+	if (!hpo_dp_enc31 || !vpg || !apg) {
+		kfree(hpo_dp_enc31);
+		kfree(vpg);
+		kfree(apg);
+		return NULL;
+	}
+
+	dcn31_hpo_dp_stream_encoder_construct(hpo_dp_enc31, ctx, ctx->dc_bios,
+					hpo_dp_inst, eng_id, vpg, apg,
+					&hpo_dp_stream_enc_regs[hpo_dp_inst],
+					&hpo_dp_se_shift, &hpo_dp_se_mask);
+
+	return &hpo_dp_enc31->base;
+}
+
+static struct hpo_dp_link_encoder *dcn31_hpo_dp_link_encoder_create(
+	uint8_t inst,
+	struct dc_context *ctx)
+{
+	struct dcn31_hpo_dp_link_encoder *hpo_dp_enc31;
+
+	/* allocate HPO link encoder */
+	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
+
+	hpo_dp_link_encoder31_construct(hpo_dp_enc31, ctx, inst,
+					&hpo_dp_link_enc_regs[inst],
+					&hpo_dp_le_shift, &hpo_dp_le_mask);
+
+	return &hpo_dp_enc31->base;
+}
+
+static struct dce_hwseq *dcn314_hwseq_create(
+	struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &hwseq_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+	}
+	return hws;
+}
+static const struct resource_create_funcs res_create_funcs = {
+	.read_dce_straps = read_dce_straps,
+	.create_audio = dcn31_create_audio,
+	.create_stream_encoder = dcn314_stream_encoder_create,
+	.create_hpo_dp_stream_encoder = dcn31_hpo_dp_stream_encoder_create,
+	.create_hpo_dp_link_encoder = dcn31_hpo_dp_link_encoder_create,
+	.create_hwseq = dcn314_hwseq_create,
+};
+
+static void dcn314_resource_destruct(struct dcn314_resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->base.stream_enc_count; i++) {
+		if (pool->base.stream_enc[i] != NULL) {
+			if (pool->base.stream_enc[i]->vpg != NULL) {
+				kfree(DCN30_VPG_FROM_VPG(pool->base.stream_enc[i]->vpg));
+				pool->base.stream_enc[i]->vpg = NULL;
+			}
+			if (pool->base.stream_enc[i]->afmt != NULL) {
+				kfree(DCN30_AFMT_FROM_AFMT(pool->base.stream_enc[i]->afmt));
+				pool->base.stream_enc[i]->afmt = NULL;
+			}
+			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
+			pool->base.stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.hpo_dp_stream_enc_count; i++) {
+		if (pool->base.hpo_dp_stream_enc[i] != NULL) {
+			if (pool->base.hpo_dp_stream_enc[i]->vpg != NULL) {
+				kfree(DCN30_VPG_FROM_VPG(pool->base.hpo_dp_stream_enc[i]->vpg));
+				pool->base.hpo_dp_stream_enc[i]->vpg = NULL;
+			}
+			if (pool->base.hpo_dp_stream_enc[i]->apg != NULL) {
+				kfree(DCN31_APG_FROM_APG(pool->base.hpo_dp_stream_enc[i]->apg));
+				pool->base.hpo_dp_stream_enc[i]->apg = NULL;
+			}
+			kfree(DCN3_1_HPO_DP_STREAM_ENC_FROM_HPO_STREAM_ENC(pool->base.hpo_dp_stream_enc[i]));
+			pool->base.hpo_dp_stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.hpo_dp_link_enc_count; i++) {
+		if (pool->base.hpo_dp_link_enc[i] != NULL) {
+			kfree(DCN3_1_HPO_DP_LINK_ENC_FROM_HPO_LINK_ENC(pool->base.hpo_dp_link_enc[i]));
+			pool->base.hpo_dp_link_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		if (pool->base.dscs[i] != NULL)
+			dcn20_dsc_destroy(&pool->base.dscs[i]);
+	}
+
+	if (pool->base.mpc != NULL) {
+		kfree(TO_DCN20_MPC(pool->base.mpc));
+		pool->base.mpc = NULL;
+	}
+	if (pool->base.hubbub != NULL) {
+		kfree(pool->base.hubbub);
+		pool->base.hubbub = NULL;
+	}
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (pool->base.dpps[i] != NULL)
+			dcn31_dpp_destroy(&pool->base.dpps[i]);
+
+		if (pool->base.ipps[i] != NULL)
+			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
+
+		if (pool->base.hubps[i] != NULL) {
+			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
+			pool->base.hubps[i] = NULL;
+		}
+
+		if (pool->base.irqs != NULL)
+			dal_irq_service_destroy(&pool->base.irqs);
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+		if (pool->base.hw_i2cs[i] != NULL) {
+			kfree(pool->base.hw_i2cs[i]);
+			pool->base.hw_i2cs[i] = NULL;
+		}
+		if (pool->base.sw_i2cs[i] != NULL) {
+			kfree(pool->base.sw_i2cs[i]);
+			pool->base.sw_i2cs[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		if (pool->base.opps[i] != NULL)
+			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.timing_generators[i] != NULL)	{
+			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
+			pool->base.timing_generators[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+		if (pool->base.dwbc[i] != NULL) {
+			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
+			pool->base.dwbc[i] = NULL;
+		}
+		if (pool->base.mcif_wb[i] != NULL) {
+			kfree(TO_DCN30_MMHUBBUB(pool->base.mcif_wb[i]));
+			pool->base.mcif_wb[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.audio_count; i++) {
+		if (pool->base.audios[i])
+			dce_aud_destroy(&pool->base.audios[i]);
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] != NULL) {
+			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
+			pool->base.clock_sources[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+		if (pool->base.mpc_lut[i] != NULL) {
+			dc_3dlut_func_release(pool->base.mpc_lut[i]);
+			pool->base.mpc_lut[i] = NULL;
+		}
+		if (pool->base.mpc_shaper[i] != NULL) {
+			dc_transfer_func_release(pool->base.mpc_shaper[i]);
+			pool->base.mpc_shaper[i] = NULL;
+		}
+	}
+
+	if (pool->base.dp_clock_source != NULL) {
+		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
+		pool->base.dp_clock_source = NULL;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.multiple_abms[i] != NULL)
+			dce_abm_destroy(&pool->base.multiple_abms[i]);
+	}
+
+	if (pool->base.psr != NULL)
+		dmub_psr_destroy(&pool->base.psr);
+
+	if (pool->base.replay != NULL)
+		dmub_replay_destroy(&pool->base.replay);
+
+	if (pool->base.dccg != NULL)
+		dcn_dccg_destroy(&pool->base.dccg);
+}
+
+static struct hubp *dcn31_hubp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn20_hubp *hubp2 =
+		kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
+
+	if (!hubp2)
+		return NULL;
+
+	if (hubp31_construct(hubp2, ctx, inst,
+			&hubp_regs[inst], &hubp_shift, &hubp_mask))
+		return &hubp2->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(hubp2);
+	return NULL;
+}
+
+static bool dcn31_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc),
+						    GFP_KERNEL);
+
+		if (!dwbc30) {
+			dm_error("DC: failed to create dwbc30!\n");
+			return false;
+		}
+
+		dcn30_dwbc_construct(dwbc30, ctx,
+				&dwbc30_regs[i],
+				&dwbc30_shift,
+				&dwbc30_mask,
+				i);
+
+		pool->dwbc[i] = &dwbc30->base;
+	}
+	return true;
+}
+
+static bool dcn31_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub),
+						    GFP_KERNEL);
+
+		if (!mcif_wb30) {
+			dm_error("DC: failed to create mcif_wb30!\n");
+			return false;
+		}
+
+		dcn30_mmhubbub_construct(mcif_wb30, ctx,
+				&mcif_wb30_regs[i],
+				&mcif_wb30_shift,
+				&mcif_wb30_mask,
+				i);
+
+		pool->mcif_wb[i] = &mcif_wb30->base;
+	}
+	return true;
+}
+
+static struct display_stream_compressor *dcn314_dsc_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_dsc *dsc =
+		kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
+
+	if (!dsc) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
+	return &dsc->base;
+}
+
+static void dcn314_destroy_resource_pool(struct resource_pool **pool)
+{
+	struct dcn314_resource_pool *dcn314_pool = TO_DCN314_RES_POOL(*pool);
+
+	dcn314_resource_destruct(dcn314_pool);
+	kfree(dcn314_pool);
+	*pool = NULL;
+}
+
+static struct clock_source *dcn31_clock_source_create(
+		struct dc_context *ctx,
+		struct dc_bios *bios,
+		enum clock_source_id id,
+		const struct dce110_clk_src_regs *regs,
+		bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dcn31_clk_src_construct(clk_src, ctx, bios, id,
+			regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	BREAK_TO_DEBUGGER();
+	kfree(clk_src);
+	return NULL;
+}
+
+static int dcn314_populate_dml_pipes_from_context(
+	struct dc *dc, struct dc_state *context,
+	display_e2e_pipe_params_st *pipes,
+	bool fast_validate)
+{
+	int pipe_cnt;
+
+	DC_FP_START();
+	pipe_cnt = dcn314_populate_dml_pipes_from_context_fpu(dc, context, pipes, fast_validate);
+	DC_FP_END();
+
+	return pipe_cnt;
+}
+
+static struct dc_cap_funcs cap_funcs = {
+	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
+};
+
+static void dcn314_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
+{
+	DC_FP_START();
+	dcn314_update_bw_bounding_box_fpu(dc, bw_params);
+	DC_FP_END();
+}
+
+static void dcn314_get_panel_config_defaults(struct dc_panel_config *panel_config)
+{
+	*panel_config = panel_config_defaults;
+}
+
+static bool filter_modes_for_single_channel_workaround(struct dc *dc,
+		struct dc_state *context)
+{
+	// Filter 2K@240Hz+8K@24fps above combination timing if memory only has single dimm LPDDR
+	if (dc->clk_mgr->bw_params->vram_type == 34 &&
+	    dc->clk_mgr->bw_params->num_channels < 2 &&
+	    context->stream_count > 1) {
+		int total_phy_pix_clk = 0;
+
+		for (int i = 0; i < context->stream_count; i++)
+			if (context->res_ctx.pipe_ctx[i].stream)
+				total_phy_pix_clk += context->res_ctx.pipe_ctx[i].stream->phy_pix_clk;
+
+		if (total_phy_pix_clk >= (1148928+826260)) //2K@240Hz+8K@24fps
+			return true;
+	}
+	return false;
+}
+
+bool dcn314_validate_bandwidth(struct dc *dc,
+		struct dc_state *context,
+		bool fast_validate)
+{
+	bool out = false;
+
+	BW_VAL_TRACE_SETUP();
+
+	int vlevel = 0;
+	int pipe_cnt = 0;
+	display_e2e_pipe_params_st *pipes = kzalloc(dc->res_pool->pipe_count * sizeof(display_e2e_pipe_params_st), GFP_KERNEL);
+	DC_LOGGER_INIT(dc->ctx->logger);
+
+	BW_VAL_TRACE_COUNT();
+
+	if (filter_modes_for_single_channel_workaround(dc, context))
+		goto validate_fail;
+
+	DC_FP_START();
+	// do not support self refresh only
+	out = dcn30_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, fast_validate, false);
+	DC_FP_END();
+
+	// Disable fast_validate to set min dcfclk in calculate_wm_and_dlg
+	if (pipe_cnt == 0)
+		fast_validate = false;
+
+	if (!out)
+		goto validate_fail;
+
+	BW_VAL_TRACE_END_VOLTAGE_LEVEL();
+
+	if (fast_validate) {
+		BW_VAL_TRACE_SKIP(fast);
+		goto validate_out;
+	}
+	if (dc->res_pool->funcs->calculate_wm_and_dlg)
+		dc->res_pool->funcs->calculate_wm_and_dlg(dc, context, pipes, pipe_cnt, vlevel);
+
+	BW_VAL_TRACE_END_WATERMARKS();
+
+	goto validate_out;
+
+validate_fail:
+	DC_LOG_WARNING("Mode Validation Warning: %s failed validation.\n",
+		dml_get_status_message(context->bw_ctx.dml.vba.ValidationStatus[context->bw_ctx.dml.vba.soc.num_states]));
+
+	BW_VAL_TRACE_SKIP(fail);
+	out = false;
+
+validate_out:
+	kfree(pipes);
+
+	BW_VAL_TRACE_FINISH();
+
+	return out;
+}
+
+static struct resource_funcs dcn314_res_pool_funcs = {
+	.destroy = dcn314_destroy_resource_pool,
+	.link_enc_create = dcn31_link_encoder_create,
+	.link_enc_create_minimal = dcn31_link_enc_create_minimal,
+	.link_encs_assign = link_enc_cfg_link_encs_assign,
+	.link_enc_unassign = link_enc_cfg_link_enc_unassign,
+	.panel_cntl_create = dcn31_panel_cntl_create,
+	.validate_bandwidth = dcn314_validate_bandwidth,
+	.calculate_wm_and_dlg = dcn31_calculate_wm_and_dlg,
+	.update_soc_for_wm_a = dcn31_update_soc_for_wm_a,
+	.populate_dml_pipes = dcn314_populate_dml_pipes_from_context,
+	.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
+	.release_pipe = dcn20_release_pipe,
+	.add_stream_to_ctx = dcn30_add_stream_to_ctx,
+	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
+	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
+	.populate_dml_writeback_from_context = dcn30_populate_dml_writeback_from_context,
+	.set_mcif_arb_params = dcn30_set_mcif_arb_params,
+	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
+	.acquire_post_bldn_3dlut = dcn30_acquire_post_bldn_3dlut,
+	.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
+	.update_bw_bounding_box = dcn314_update_bw_bounding_box,
+	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
+	.get_panel_config_defaults = dcn314_get_panel_config_defaults,
+	.get_preferred_eng_id_dpia = dcn314_get_preferred_eng_id_dpia,
+};
+
+static struct clock_source *dcn30_clock_source_create(
+		struct dc_context *ctx,
+		struct dc_bios *bios,
+		enum clock_source_id id,
+		const struct dce110_clk_src_regs *regs,
+		bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dcn31_clk_src_construct(clk_src, ctx, bios, id,
+			regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	BREAK_TO_DEBUGGER();
+	kfree(clk_src);
+	return NULL;
+}
+
+static bool dcn314_resource_construct(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct dcn314_resource_pool *pool)
+{
+	int i;
+	struct dc_context *ctx = dc->ctx;
+	struct irq_service_init_data init_data;
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	pool->base.res_cap = &res_cap_dcn314;
+	pool->base.funcs = &dcn314_res_pool_funcs;
+
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
+	pool->base.mpcc_count = pool->base.res_cap->num_timing_generator;
+	dc->caps.max_downscale_ratio = 400;
+	dc->caps.i2c_speed_in_khz = 100;
+	dc->caps.i2c_speed_in_khz_hdcp = 100;
+	dc->caps.max_cursor_size = 256;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.dmdata_alloc_size = 2048;
+	dc->caps.max_slave_planes = 2;
+	dc->caps.max_slave_yuv_planes = 2;
+	dc->caps.max_slave_rgb_planes = 2;
+	dc->caps.post_blend_color_processing = true;
+	dc->caps.force_dp_tps4_for_cp2520 = true;
+	if (dc->config.forceHBR2CP2520)
+		dc->caps.force_dp_tps4_for_cp2520 = false;
+	dc->caps.dp_hpo = true;
+	dc->caps.dp_hdmi21_pcon_support = true;
+	dc->caps.edp_dsc_support = true;
+	dc->caps.extended_aux_timeout_support = true;
+	dc->caps.dmcub_support = true;
+	dc->caps.is_apu = true;
+	dc->caps.seamless_odm = true;
+
+	dc->caps.zstate_support = true;
+
+	/* Color pipeline capabilities */
+	dc->caps.color.dpp.dcn_arch = 1;
+	dc->caps.color.dpp.input_lut_shared = 0;
+	dc->caps.color.dpp.icsc = 1;
+	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
+	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
+	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
+	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
+	dc->caps.color.dpp.post_csc = 1;
+	dc->caps.color.dpp.gamma_corr = 1;
+	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
+
+	dc->caps.color.dpp.hw_3d_lut = 1;
+	dc->caps.color.dpp.ogam_ram = 1;
+	// no OGAM ROM on DCN301
+	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
+	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
+	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
+	dc->caps.color.dpp.ocsc = 0;
+
+	dc->caps.color.mpc.gamut_remap = 1;
+	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //2
+	dc->caps.color.mpc.ogam_ram = 1;
+	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
+	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
+	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
+	dc->caps.color.mpc.ocsc = 1;
+
+	dc->caps.max_disp_clock_khz_at_vmin = 650000;
+
+	/* Use pipe context based otg sync logic */
+	dc->config.use_pipe_ctx_sync_logic = true;
+
+	/* read VBIOS LTTPR caps */
+	{
+		if (ctx->dc_bios->funcs->get_lttpr_caps) {
+			enum bp_result bp_query_result;
+			uint8_t is_vbios_lttpr_enable = 0;
+
+			bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
+			dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
+		}
+
+		/* interop bit is implicit */
+		{
+			dc->caps.vbios_lttpr_aware = true;
+		}
+	}
+
+	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
+		dc->debug = debug_defaults_drv;
+	else
+		dc->debug = debug_defaults_diags;
+
+	/* Disable pipe power gating */
+	dc->debug.disable_dpp_power_gate = true;
+	dc->debug.disable_hubp_power_gate = true;
+
+	/* Disable root clock optimization */
+	dc->debug.root_clock_optimization.u32All = 0;
+
+	// Init the vm_helper
+	if (dc->vm_helper)
+		vm_helper_init(dc->vm_helper, 16);
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	/* Clock Sources for Pixel Clock*/
+	pool->base.clock_sources[DCN31_CLK_SRC_PLL0] =
+			dcn30_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL0,
+				&clk_src_regs[0], false);
+	pool->base.clock_sources[DCN31_CLK_SRC_PLL1] =
+			dcn30_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL1,
+				&clk_src_regs[1], false);
+	pool->base.clock_sources[DCN31_CLK_SRC_PLL2] =
+			dcn30_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL2,
+				&clk_src_regs[2], false);
+	pool->base.clock_sources[DCN31_CLK_SRC_PLL3] =
+			dcn30_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL3,
+				&clk_src_regs[3], false);
+	pool->base.clock_sources[DCN31_CLK_SRC_PLL4] =
+			dcn30_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL4,
+				&clk_src_regs[4], false);
+
+	pool->base.clk_src_count = DCN30_CLK_SRC_TOTAL;
+
+	/* todo: not reuse phy_pll registers */
+	pool->base.dp_clock_source =
+			dcn31_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_ID_DP_DTO,
+				&clk_src_regs[0], true);
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	pool->base.dccg = dccg314_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
+	if (pool->base.dccg == NULL) {
+		dm_error("DC: failed to create dccg!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	init_data.ctx = dc->ctx;
+	pool->base.irqs = dal_irq_service_dcn314_create(&init_data);
+	if (!pool->base.irqs)
+		goto create_fail;
+
+	/* HUBBUB */
+	pool->base.hubbub = dcn31_hubbub_create(ctx);
+	if (pool->base.hubbub == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create hubbub!\n");
+		goto create_fail;
+	}
+
+	/* HUBPs, DPPs, OPPs and TGs */
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		pool->base.hubps[i] = dcn31_hubp_create(ctx, i);
+		if (pool->base.hubps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create hubps!\n");
+			goto create_fail;
+		}
+
+		pool->base.dpps[i] = dcn31_dpp_create(ctx, i);
+		if (pool->base.dpps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create dpps!\n");
+			goto create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		pool->base.opps[i] = dcn31_opp_create(ctx, i);
+		if (pool->base.opps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create output pixel processor!\n");
+			goto create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		pool->base.timing_generators[i] = dcn31_timing_generator_create(
+				ctx, i);
+		if (pool->base.timing_generators[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto create_fail;
+		}
+	}
+	pool->base.timing_generator_count = i;
+
+	/* PSR */
+	pool->base.psr = dmub_psr_create(ctx);
+	if (pool->base.psr == NULL) {
+		dm_error("DC: failed to create psr obj!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* Replay */
+	pool->base.replay = dmub_replay_create(ctx);
+	if (pool->base.replay == NULL) {
+		dm_error("DC: failed to create replay obj!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* ABM */
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		pool->base.multiple_abms[i] = dmub_abm_create(ctx,
+				&abm_regs[i],
+				&abm_shift,
+				&abm_mask);
+		if (pool->base.multiple_abms[i] == NULL) {
+			dm_error("DC: failed to create abm for pipe %d!\n", i);
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	/* MPC and DSC */
+	pool->base.mpc = dcn31_mpc_create(ctx, pool->base.mpcc_count, pool->base.res_cap->num_mpc_3dlut);
+	if (pool->base.mpc == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mpc!\n");
+		goto create_fail;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		pool->base.dscs[i] = dcn314_dsc_create(ctx, i);
+		if (pool->base.dscs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create display stream compressor %d!\n", i);
+			goto create_fail;
+		}
+	}
+
+	/* DWB and MMHUBBUB */
+	if (!dcn31_dwbc_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create dwbc!\n");
+		goto create_fail;
+	}
+
+	if (!dcn31_mmhubbub_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mcif_wb!\n");
+		goto create_fail;
+	}
+
+	/* AUX and I2C */
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.engines[i] = dcn31_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto create_fail;
+		}
+		pool->base.hw_i2cs[i] = dcn31_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create hw i2c!!\n");
+			goto create_fail;
+		}
+		pool->base.sw_i2cs[i] = NULL;
+	}
+
+	/* DCN314 has 4 DPIA */
+	pool->base.usb4_dpia_count = 4;
+
+	/* Audio, Stream Encoders including HPO and virtual, MPC 3D LUTs */
+	if (!resource_construct(num_virtual_links, dc, &pool->base,
+			&res_create_funcs))
+		goto create_fail;
+
+	/* HW Sequencer and Plane caps */
+	dcn314_hw_sequencer_construct(dc);
+
+	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->cap_funcs = cap_funcs;
+
+	dc->dcn_ip->max_num_dpp = dcn3_14_ip.max_num_dpp;
+
+	return true;
+
+create_fail:
+
+	dcn314_resource_destruct(pool);
+
+	return false;
+}
+
+struct resource_pool *dcn314_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc)
+{
+	struct dcn314_resource_pool *pool =
+		kzalloc(sizeof(struct dcn314_resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dcn314_resource_construct(init_data->num_virtual_links, dc, pool))
+		return &pool->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(pool);
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.h
new file mode 100644
index 000000000000..49ffe71018df
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.h
@@ -0,0 +1,50 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef _DCN314_RESOURCE_H_
+#define _DCN314_RESOURCE_H_
+
+#include "core_types.h"
+
+extern struct _vcs_dpi_ip_params_st dcn3_14_ip;
+extern struct _vcs_dpi_soc_bounding_box_st dcn3_14_soc;
+
+#define TO_DCN314_RES_POOL(pool)\
+	container_of(pool, struct dcn314_resource_pool, base)
+
+struct dcn314_resource_pool {
+	struct resource_pool base;
+};
+
+bool dcn314_validate_bandwidth(struct dc *dc,
+		struct dc_state *context,
+		bool fast_validate);
+
+struct resource_pool *dcn314_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc);
+
+#endif /* _DCN314_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c
new file mode 100644
index 000000000000..cb8024eee8e4
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c
@@ -0,0 +1,2151 @@
+/*
+ * Copyright 2021 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+
+#include "dm_services.h"
+#include "dc.h"
+
+#include "dcn31/dcn31_init.h"
+
+#include "resource.h"
+#include "include/irq_service_interface.h"
+#include "dcn315_resource.h"
+
+#include "dcn20/dcn20_resource.h"
+#include "dcn30/dcn30_resource.h"
+#include "dcn31/dcn31_resource.h"
+
+#include "dcn10/dcn10_ipp.h"
+#include "dcn30/dcn30_hubbub.h"
+#include "dcn31/dcn31_hubbub.h"
+#include "dcn30/dcn30_mpc.h"
+#include "dcn31/dcn31_hubp.h"
+#include "irq/dcn315/irq_service_dcn315.h"
+#include "dcn30/dcn30_dpp.h"
+#include "dcn31/dcn31_optc.h"
+#include "dcn20/dcn20_hwseq.h"
+#include "dcn30/dcn30_hwseq.h"
+#include "dce110/dce110_hwseq.h"
+#include "dcn30/dcn30_opp.h"
+#include "dcn20/dcn20_dsc.h"
+#include "dcn30/dcn30_vpg.h"
+#include "dcn30/dcn30_afmt.h"
+#include "dcn30/dcn30_dio_stream_encoder.h"
+#include "dcn31/dcn31_hpo_dp_stream_encoder.h"
+#include "dcn31/dcn31_hpo_dp_link_encoder.h"
+#include "dcn31/dcn31_apg.h"
+#include "dcn31/dcn31_dio_link_encoder.h"
+#include "dcn31/dcn31_vpg.h"
+#include "dcn31/dcn31_afmt.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_hwseq.h"
+#include "clk_mgr.h"
+#include "virtual/virtual_stream_encoder.h"
+#include "dce110/dce110_resource.h"
+#include "dml/display_mode_vba.h"
+#include "dml/dcn31/dcn31_fpu.h"
+#include "dcn31/dcn31_dccg.h"
+#include "dcn10/dcn10_resource.h"
+#include "dcn31/dcn31_panel_cntl.h"
+
+#include "dcn30/dcn30_dwb.h"
+#include "dcn30/dcn30_mmhubbub.h"
+
+#include "dcn/dcn_3_1_5_offset.h"
+#include "dcn/dcn_3_1_5_sh_mask.h"
+#include "dpcs/dpcs_4_2_2_offset.h"
+#include "dpcs/dpcs_4_2_2_sh_mask.h"
+
+#define NBIO_BASE__INST0_SEG0                      0x00000000
+#define NBIO_BASE__INST0_SEG1                      0x00000014
+#define NBIO_BASE__INST0_SEG2                      0x00000D20
+#define NBIO_BASE__INST0_SEG3                      0x00010400
+#define NBIO_BASE__INST0_SEG4                      0x0241B000
+#define NBIO_BASE__INST0_SEG5                      0x04040000
+
+#define DPCS_BASE__INST0_SEG0                      0x00000012
+#define DPCS_BASE__INST0_SEG1                      0x000000C0
+#define DPCS_BASE__INST0_SEG2                      0x000034C0
+#define DPCS_BASE__INST0_SEG3                      0x00009000
+#define DPCS_BASE__INST0_SEG4                      0x02403C00
+#define DPCS_BASE__INST0_SEG5                      0
+
+#define DCN_BASE__INST0_SEG0                       0x00000012
+#define DCN_BASE__INST0_SEG1                       0x000000C0
+#define DCN_BASE__INST0_SEG2                       0x000034C0
+#define DCN_BASE__INST0_SEG3                       0x00009000
+#define DCN_BASE__INST0_SEG4                       0x02403C00
+#define DCN_BASE__INST0_SEG5                       0
+
+#define regBIF_BX_PF2_RSMU_INDEX                                                                        0x0000
+#define regBIF_BX_PF2_RSMU_INDEX_BASE_IDX                                                               1
+#define regBIF_BX_PF2_RSMU_DATA                                                                         0x0001
+#define regBIF_BX_PF2_RSMU_DATA_BASE_IDX                                                                1
+#define regBIF_BX2_BIOS_SCRATCH_6                                                                       0x003e
+#define regBIF_BX2_BIOS_SCRATCH_6_BASE_IDX                                                              1
+#define BIF_BX2_BIOS_SCRATCH_6__BIOS_SCRATCH_6__SHIFT                                                         0x0
+#define BIF_BX2_BIOS_SCRATCH_6__BIOS_SCRATCH_6_MASK                                                           0xFFFFFFFFL
+#define regBIF_BX2_BIOS_SCRATCH_2                                                                       0x003a
+#define regBIF_BX2_BIOS_SCRATCH_2_BASE_IDX                                                              1
+#define BIF_BX2_BIOS_SCRATCH_2__BIOS_SCRATCH_2__SHIFT                                                         0x0
+#define BIF_BX2_BIOS_SCRATCH_2__BIOS_SCRATCH_2_MASK                                                           0xFFFFFFFFL
+#define regBIF_BX2_BIOS_SCRATCH_3                                                                       0x003b
+#define regBIF_BX2_BIOS_SCRATCH_3_BASE_IDX                                                              1
+#define BIF_BX2_BIOS_SCRATCH_3__BIOS_SCRATCH_3__SHIFT                                                         0x0
+#define BIF_BX2_BIOS_SCRATCH_3__BIOS_SCRATCH_3_MASK                                                           0xFFFFFFFFL
+
+#define regDCHUBBUB_DEBUG_CTRL_0                                              0x04d6
+#define regDCHUBBUB_DEBUG_CTRL_0_BASE_IDX                                     2
+#define DCHUBBUB_DEBUG_CTRL_0__DET_DEPTH__SHIFT                               0x10
+#define DCHUBBUB_DEBUG_CTRL_0__DET_DEPTH_MASK                                 0x01FF0000L
+
+#include "reg_helper.h"
+#include "dce/dmub_abm.h"
+#include "dce/dmub_psr.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_i2c.h"
+
+#include "dml/dcn30/display_mode_vba_30.h"
+#include "vm_helper.h"
+#include "dcn20/dcn20_vmid.h"
+
+#include "link_enc_cfg.h"
+
+#define DCN3_15_MAX_DET_SIZE 384
+#define DCN3_15_CRB_SEGMENT_SIZE_KB 64
+#define DCN3_15_MAX_DET_SEGS (DCN3_15_MAX_DET_SIZE / DCN3_15_CRB_SEGMENT_SIZE_KB)
+/* Minimum 3 extra segments need to be in compbuf and claimable to guarantee seamless mpo transitions */
+#define MIN_RESERVED_DET_SEGS 3
+
+enum dcn31_clk_src_array_id {
+	DCN31_CLK_SRC_PLL0,
+	DCN31_CLK_SRC_PLL1,
+	DCN31_CLK_SRC_PLL2,
+	DCN31_CLK_SRC_PLL3,
+	DCN31_CLK_SRC_PLL4,
+	DCN30_CLK_SRC_TOTAL
+};
+
+/* begin *********************
+ * macros to expend register list macro defined in HW object header file
+ */
+
+/* DCN */
+#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
+
+#define BASE(seg) BASE_INNER(seg)
+
+#define SR(reg_name)\
+		.reg_name = BASE(reg ## reg_name ## _BASE_IDX) +  \
+					reg ## reg_name
+
+#define SRI(reg_name, block, id)\
+	.reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define SRI2(reg_name, block, id)\
+	.reg_name = BASE(reg ## reg_name ## _BASE_IDX) + \
+					reg ## reg_name
+
+#define SRIR(var_name, reg_name, block, id)\
+	.var_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define SRII(reg_name, block, id)\
+	.reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define SRII_MPC_RMU(reg_name, block, id)\
+	.RMU##_##reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define SRII_DWB(reg_name, temp_name, block, id)\
+	.reg_name[id] = BASE(reg ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## temp_name
+
+#define SF_DWB2(reg_name, block, id, field_name, post_fix)	\
+	.field_name = reg_name ## __ ## field_name ## post_fix
+
+#define DCCG_SRII(reg_name, block, id)\
+	.block ## _ ## reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define VUPDATE_SRII(reg_name, block, id)\
+	.reg_name[id] = BASE(reg ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
+					reg ## reg_name ## _ ## block ## id
+
+/* NBIO */
+#define NBIO_BASE_INNER(seg) \
+	NBIO_BASE__INST0_SEG ## seg
+
+#define NBIO_BASE(seg) \
+	NBIO_BASE_INNER(seg)
+
+#define NBIO_SR(reg_name)\
+		.reg_name = NBIO_BASE(regBIF_BX2_ ## reg_name ## _BASE_IDX) + \
+					regBIF_BX2_ ## reg_name
+
+static const struct bios_registers bios_regs = {
+		NBIO_SR(BIOS_SCRATCH_3),
+		NBIO_SR(BIOS_SCRATCH_6)
+};
+
+#define clk_src_regs(index, pllid)\
+[index] = {\
+	CS_COMMON_REG_LIST_DCN3_0(index, pllid),\
+}
+
+static const struct dce110_clk_src_regs clk_src_regs[] = {
+	clk_src_regs(0, A),
+	clk_src_regs(1, B),
+	clk_src_regs(2, C),
+	clk_src_regs(3, D),
+	clk_src_regs(4, E)
+};
+
+static const struct dce110_clk_src_shift cs_shift = {
+		CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
+};
+
+static const struct dce110_clk_src_mask cs_mask = {
+		CS_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
+};
+
+#define abm_regs(id)\
+[id] = {\
+		ABM_DCN302_REG_LIST(id)\
+}
+
+static const struct dce_abm_registers abm_regs[] = {
+		abm_regs(0),
+		abm_regs(1),
+		abm_regs(2),
+		abm_regs(3),
+};
+
+static const struct dce_abm_shift abm_shift = {
+		ABM_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dce_abm_mask abm_mask = {
+		ABM_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define audio_regs(id)\
+[id] = {\
+		AUD_COMMON_REG_LIST(id)\
+}
+
+static const struct dce_audio_registers audio_regs[] = {
+	audio_regs(0),
+	audio_regs(1),
+	audio_regs(2),
+	audio_regs(3),
+	audio_regs(4),
+	audio_regs(5),
+	audio_regs(6)
+};
+
+#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
+		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
+
+static const struct dce_audio_shift audio_shift = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_audio_mask audio_mask = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+#define vpg_regs(id)\
+[id] = {\
+	VPG_DCN31_REG_LIST(id)\
+}
+
+static const struct dcn31_vpg_registers vpg_regs[] = {
+	vpg_regs(0),
+	vpg_regs(1),
+	vpg_regs(2),
+	vpg_regs(3),
+	vpg_regs(4),
+	vpg_regs(5),
+	vpg_regs(6),
+	vpg_regs(7),
+	vpg_regs(8),
+	vpg_regs(9),
+};
+
+static const struct dcn31_vpg_shift vpg_shift = {
+	DCN31_VPG_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_vpg_mask vpg_mask = {
+	DCN31_VPG_MASK_SH_LIST(_MASK)
+};
+
+#define afmt_regs(id)\
+[id] = {\
+	AFMT_DCN31_REG_LIST(id)\
+}
+
+static const struct dcn31_afmt_registers afmt_regs[] = {
+	afmt_regs(0),
+	afmt_regs(1),
+	afmt_regs(2),
+	afmt_regs(3),
+	afmt_regs(4),
+	afmt_regs(5)
+};
+
+static const struct dcn31_afmt_shift afmt_shift = {
+	DCN31_AFMT_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_afmt_mask afmt_mask = {
+	DCN31_AFMT_MASK_SH_LIST(_MASK)
+};
+
+#define apg_regs(id)\
+[id] = {\
+	APG_DCN31_REG_LIST(id)\
+}
+
+static const struct dcn31_apg_registers apg_regs[] = {
+	apg_regs(0),
+	apg_regs(1),
+	apg_regs(2),
+	apg_regs(3)
+};
+
+static const struct dcn31_apg_shift apg_shift = {
+	DCN31_APG_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_apg_mask apg_mask = {
+		DCN31_APG_MASK_SH_LIST(_MASK)
+};
+
+#define stream_enc_regs(id)\
+[id] = {\
+	SE_DCN3_REG_LIST(id)\
+}
+
+static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
+	stream_enc_regs(0),
+	stream_enc_regs(1),
+	stream_enc_regs(2),
+	stream_enc_regs(3),
+	stream_enc_regs(4)
+};
+
+static const struct dcn10_stream_encoder_shift se_shift = {
+		SE_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn10_stream_encoder_mask se_mask = {
+		SE_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+
+#define aux_regs(id)\
+[id] = {\
+	DCN2_AUX_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
+		aux_regs(0),
+		aux_regs(1),
+		aux_regs(2),
+		aux_regs(3),
+		aux_regs(4)
+};
+
+#define hpd_regs(id)\
+[id] = {\
+	HPD_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
+		hpd_regs(0),
+		hpd_regs(1),
+		hpd_regs(2),
+		hpd_regs(3),
+		hpd_regs(4)
+};
+
+#define link_regs(id, phyid)\
+[id] = {\
+	LE_DCN31_REG_LIST(id), \
+	UNIPHY_DCN2_REG_LIST(phyid), \
+	DPCS_DCN31_REG_LIST(id), \
+}
+
+static const struct dce110_aux_registers_shift aux_shift = {
+	DCN_AUX_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+	DCN_AUX_MASK_SH_LIST(_MASK)
+};
+
+static const struct dcn10_link_enc_registers link_enc_regs[] = {
+	link_regs(0, A),
+	link_regs(1, B),
+	link_regs(2, C),
+	link_regs(3, D),
+	link_regs(4, E)
+};
+
+static const struct dcn10_link_enc_shift le_shift = {
+	LINK_ENCODER_MASK_SH_LIST_DCN31(__SHIFT), \
+	DPCS_DCN31_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn10_link_enc_mask le_mask = {
+	LINK_ENCODER_MASK_SH_LIST_DCN31(_MASK), \
+	DPCS_DCN31_MASK_SH_LIST(_MASK)
+};
+
+#define hpo_dp_stream_encoder_reg_list(id)\
+[id] = {\
+	DCN3_1_HPO_DP_STREAM_ENC_REG_LIST(id)\
+}
+
+static const struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[] = {
+	hpo_dp_stream_encoder_reg_list(0),
+	hpo_dp_stream_encoder_reg_list(1),
+	hpo_dp_stream_encoder_reg_list(2),
+	hpo_dp_stream_encoder_reg_list(3),
+};
+
+static const struct dcn31_hpo_dp_stream_encoder_shift hpo_dp_se_shift = {
+	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_hpo_dp_stream_encoder_mask hpo_dp_se_mask = {
+	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(_MASK)
+};
+
+
+#define hpo_dp_link_encoder_reg_list(id)\
+[id] = {\
+	DCN3_1_HPO_DP_LINK_ENC_REG_LIST(id),\
+	DCN3_1_RDPCSTX_REG_LIST(0),\
+	DCN3_1_RDPCSTX_REG_LIST(1),\
+	DCN3_1_RDPCSTX_REG_LIST(2),\
+	DCN3_1_RDPCSTX_REG_LIST(3),\
+	DCN3_1_RDPCSTX_REG_LIST(4)\
+}
+
+static const struct dcn31_hpo_dp_link_encoder_registers hpo_dp_link_enc_regs[] = {
+	hpo_dp_link_encoder_reg_list(0),
+	hpo_dp_link_encoder_reg_list(1),
+};
+
+static const struct dcn31_hpo_dp_link_encoder_shift hpo_dp_le_shift = {
+	DCN3_1_HPO_DP_LINK_ENC_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_hpo_dp_link_encoder_mask hpo_dp_le_mask = {
+	DCN3_1_HPO_DP_LINK_ENC_MASK_SH_LIST(_MASK)
+};
+
+#define dpp_regs(id)\
+[id] = {\
+	DPP_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn3_dpp_registers dpp_regs[] = {
+	dpp_regs(0),
+	dpp_regs(1),
+	dpp_regs(2),
+	dpp_regs(3)
+};
+
+static const struct dcn3_dpp_shift tf_shift = {
+		DPP_REG_LIST_SH_MASK_DCN30(__SHIFT)
+};
+
+static const struct dcn3_dpp_mask tf_mask = {
+		DPP_REG_LIST_SH_MASK_DCN30(_MASK)
+};
+
+#define opp_regs(id)\
+[id] = {\
+	OPP_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn20_opp_registers opp_regs[] = {
+	opp_regs(0),
+	opp_regs(1),
+	opp_regs(2),
+	opp_regs(3)
+};
+
+static const struct dcn20_opp_shift opp_shift = {
+	OPP_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn20_opp_mask opp_mask = {
+	OPP_MASK_SH_LIST_DCN20(_MASK)
+};
+
+#define aux_engine_regs(id)\
+[id] = {\
+	AUX_COMMON_REG_LIST0(id), \
+	.AUXN_IMPCAL = 0, \
+	.AUXP_IMPCAL = 0, \
+	.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1),
+		aux_engine_regs(2),
+		aux_engine_regs(3),
+		aux_engine_regs(4)
+};
+
+#define dwbc_regs_dcn3(id)\
+[id] = {\
+	DWBC_COMMON_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn30_dwbc_registers dwbc30_regs[] = {
+	dwbc_regs_dcn3(0),
+};
+
+static const struct dcn30_dwbc_shift dwbc30_shift = {
+	DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_dwbc_mask dwbc30_mask = {
+	DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define mcif_wb_regs_dcn3(id)\
+[id] = {\
+	MCIF_WB_COMMON_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn30_mmhubbub_registers mcif_wb30_regs[] = {
+	mcif_wb_regs_dcn3(0)
+};
+
+static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define dsc_regsDCN20(id)\
+[id] = {\
+	DSC_REG_LIST_DCN20(id)\
+}
+
+static const struct dcn20_dsc_registers dsc_regs[] = {
+	dsc_regsDCN20(0),
+	dsc_regsDCN20(1),
+	dsc_regsDCN20(2)
+};
+
+static const struct dcn20_dsc_shift dsc_shift = {
+	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
+};
+
+static const struct dcn20_dsc_mask dsc_mask = {
+	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
+};
+
+static const struct dcn30_mpc_registers mpc_regs = {
+		MPC_REG_LIST_DCN3_0(0),
+		MPC_REG_LIST_DCN3_0(1),
+		MPC_REG_LIST_DCN3_0(2),
+		MPC_REG_LIST_DCN3_0(3),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(0),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(1),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(2),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(3),
+		MPC_DWB_MUX_REG_LIST_DCN3_0(0),
+};
+
+static const struct dcn30_mpc_shift mpc_shift = {
+	MPC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_mpc_mask mpc_mask = {
+	MPC_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define optc_regs(id)\
+[id] = {OPTC_COMMON_REG_LIST_DCN3_1(id)}
+
+static const struct dcn_optc_registers optc_regs[] = {
+	optc_regs(0),
+	optc_regs(1),
+	optc_regs(2),
+	optc_regs(3)
+};
+
+static const struct dcn_optc_shift optc_shift = {
+	OPTC_COMMON_MASK_SH_LIST_DCN3_1(__SHIFT)
+};
+
+static const struct dcn_optc_mask optc_mask = {
+	OPTC_COMMON_MASK_SH_LIST_DCN3_1(_MASK)
+};
+
+#define hubp_regs(id)\
+[id] = {\
+	HUBP_REG_LIST_DCN30(id)\
+}
+
+static const struct dcn_hubp2_registers hubp_regs[] = {
+		hubp_regs(0),
+		hubp_regs(1),
+		hubp_regs(2),
+		hubp_regs(3)
+};
+
+
+static const struct dcn_hubp2_shift hubp_shift = {
+		HUBP_MASK_SH_LIST_DCN31(__SHIFT)
+};
+
+static const struct dcn_hubp2_mask hubp_mask = {
+		HUBP_MASK_SH_LIST_DCN31(_MASK)
+};
+static const struct dcn_hubbub_registers hubbub_reg = {
+		HUBBUB_REG_LIST_DCN31(0)
+};
+
+static const struct dcn_hubbub_shift hubbub_shift = {
+		HUBBUB_MASK_SH_LIST_DCN31(__SHIFT)
+};
+
+static const struct dcn_hubbub_mask hubbub_mask = {
+		HUBBUB_MASK_SH_LIST_DCN31(_MASK)
+};
+
+static const struct dccg_registers dccg_regs = {
+		DCCG_REG_LIST_DCN31()
+};
+
+static const struct dccg_shift dccg_shift = {
+		DCCG_MASK_SH_LIST_DCN31(__SHIFT)
+};
+
+static const struct dccg_mask dccg_mask = {
+		DCCG_MASK_SH_LIST_DCN31(_MASK)
+};
+
+
+#define SRII2(reg_name_pre, reg_name_post, id)\
+	.reg_name_pre ## _ ##  reg_name_post[id] = BASE(reg ## reg_name_pre \
+			## id ## _ ## reg_name_post ## _BASE_IDX) + \
+			reg ## reg_name_pre ## id ## _ ## reg_name_post
+
+
+#define HWSEQ_DCN31_REG_LIST()\
+	SR(DCHUBBUB_GLOBAL_TIMER_CNTL), \
+	SR(DCHUBBUB_ARB_HOSTVM_CNTL), \
+	SR(DIO_MEM_PWR_CTRL), \
+	SR(ODM_MEM_PWR_CTRL3), \
+	SR(DMU_MEM_PWR_CNTL), \
+	SR(MMHUBBUB_MEM_PWR_CNTL), \
+	SR(DCCG_GATE_DISABLE_CNTL), \
+	SR(DCCG_GATE_DISABLE_CNTL2), \
+	SR(DCFCLK_CNTL),\
+	SR(DC_MEM_GLOBAL_PWR_REQ_CNTL), \
+	SRII(PIXEL_RATE_CNTL, OTG, 0), \
+	SRII(PIXEL_RATE_CNTL, OTG, 1),\
+	SRII(PIXEL_RATE_CNTL, OTG, 2),\
+	SRII(PIXEL_RATE_CNTL, OTG, 3),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 0),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 1),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 2),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 3),\
+	SR(MICROSECOND_TIME_BASE_DIV), \
+	SR(MILLISECOND_TIME_BASE_DIV), \
+	SR(DISPCLK_FREQ_CHANGE_CNTL), \
+	SR(RBBMIF_TIMEOUT_DIS), \
+	SR(RBBMIF_TIMEOUT_DIS_2), \
+	SR(DCHUBBUB_CRC_CTRL), \
+	SR(DPP_TOP0_DPP_CRC_CTRL), \
+	SR(DPP_TOP0_DPP_CRC_VAL_B_A), \
+	SR(DPP_TOP0_DPP_CRC_VAL_R_G), \
+	SR(MPC_CRC_CTRL), \
+	SR(MPC_CRC_RESULT_GB), \
+	SR(MPC_CRC_RESULT_C), \
+	SR(MPC_CRC_RESULT_AR), \
+	SR(DOMAIN0_PG_CONFIG), \
+	SR(DOMAIN1_PG_CONFIG), \
+	SR(DOMAIN2_PG_CONFIG), \
+	SR(DOMAIN3_PG_CONFIG), \
+	SR(DOMAIN16_PG_CONFIG), \
+	SR(DOMAIN17_PG_CONFIG), \
+	SR(DOMAIN18_PG_CONFIG), \
+	SR(DOMAIN0_PG_STATUS), \
+	SR(DOMAIN1_PG_STATUS), \
+	SR(DOMAIN2_PG_STATUS), \
+	SR(DOMAIN3_PG_STATUS), \
+	SR(DOMAIN16_PG_STATUS), \
+	SR(DOMAIN17_PG_STATUS), \
+	SR(DOMAIN18_PG_STATUS), \
+	SR(D1VGA_CONTROL), \
+	SR(D2VGA_CONTROL), \
+	SR(D3VGA_CONTROL), \
+	SR(D4VGA_CONTROL), \
+	SR(D5VGA_CONTROL), \
+	SR(D6VGA_CONTROL), \
+	SR(DC_IP_REQUEST_CNTL), \
+	SR(AZALIA_AUDIO_DTO), \
+	SR(AZALIA_CONTROLLER_CLOCK_GATING), \
+	SR(HPO_TOP_HW_CONTROL)
+
+static const struct dce_hwseq_registers hwseq_reg = {
+		HWSEQ_DCN31_REG_LIST()
+};
+
+#define HWSEQ_DCN31_MASK_SH_LIST(mask_sh)\
+	HWSEQ_DCN_MASK_SH_LIST(mask_sh), \
+	HWS_SF(, DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, mask_sh), \
+	HWS_SF(, DCHUBBUB_ARB_HOSTVM_CNTL, DISABLE_HOSTVM_FORCE_ALLOW_PSTATE, mask_sh), \
+	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN0_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN1_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN2_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN3_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN16_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN17_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN18_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh), \
+	HWS_SF(, AZALIA_AUDIO_DTO, AZALIA_AUDIO_DTO_MODULE, mask_sh), \
+	HWS_SF(, HPO_TOP_CLOCK_CONTROL, HPO_HDMISTREAMCLK_G_GATE_DIS, mask_sh), \
+	HWS_SF(, DMU_MEM_PWR_CNTL, DMCU_ERAM_MEM_PWR_FORCE, mask_sh), \
+	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_UNASSIGNED_PWR_MODE, mask_sh), \
+	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_VBLANK_PWR_MODE, mask_sh), \
+	HWS_SF(, MMHUBBUB_MEM_PWR_CNTL, VGA_MEM_PWR_FORCE, mask_sh), \
+	HWS_SF(, DIO_MEM_PWR_CTRL, I2C_LIGHT_SLEEP_FORCE, mask_sh), \
+	HWS_SF(, HPO_TOP_HW_CONTROL, HPO_IO_EN, mask_sh)
+
+static const struct dce_hwseq_shift hwseq_shift = {
+		HWSEQ_DCN31_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+		HWSEQ_DCN31_MASK_SH_LIST(_MASK)
+};
+#define vmid_regs(id)\
+[id] = {\
+		DCN20_VMID_REG_LIST(id)\
+}
+
+static const struct dcn_vmid_registers vmid_regs[] = {
+	vmid_regs(0),
+	vmid_regs(1),
+	vmid_regs(2),
+	vmid_regs(3),
+	vmid_regs(4),
+	vmid_regs(5),
+	vmid_regs(6),
+	vmid_regs(7),
+	vmid_regs(8),
+	vmid_regs(9),
+	vmid_regs(10),
+	vmid_regs(11),
+	vmid_regs(12),
+	vmid_regs(13),
+	vmid_regs(14),
+	vmid_regs(15)
+};
+
+static const struct dcn20_vmid_shift vmid_shifts = {
+		DCN20_VMID_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn20_vmid_mask vmid_masks = {
+		DCN20_VMID_MASK_SH_LIST(_MASK)
+};
+
+static const struct resource_caps res_cap_dcn31 = {
+	.num_timing_generator = 4,
+	.num_opp = 4,
+	.num_video_plane = 4,
+	.num_audio = 5,
+	.num_stream_encoder = 5,
+	.num_dig_link_enc = 5,
+	.num_hpo_dp_stream_encoder = 4,
+	.num_hpo_dp_link_encoder = 2,
+	.num_pll = 5,
+	.num_dwb = 1,
+	.num_ddc = 5,
+	.num_vmid = 16,
+	.num_mpc_3dlut = 2,
+	.num_dsc = 3,
+};
+
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
+	.per_pixel_alpha = true,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = true,
+			.fp16 = true,
+			.p010 = true,
+			.ayuv = false,
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 16000,
+			.fp16 = 16000
+	},
+
+	// 6:1 downscaling ratio: 1000/6 = 166.666
+	.max_downscale_factor = {
+			.argb8888 = 167,
+			.nv12 = 167,
+			.fp16 = 167
+	},
+	64,
+	64
+};
+
+static const struct dc_debug_options debug_defaults_drv = {
+	.disable_z10 = true, /*hw not support it*/
+	.disable_dmcu = true,
+	.force_abm_enable = false,
+	.timing_trace = false,
+	.clock_trace = true,
+	.disable_pplib_clock_request = false,
+	.pipe_split_policy = MPC_SPLIT_DYNAMIC,
+	.force_single_disp_pipe_split = false,
+	.disable_dcc = DCC_ENABLE,
+	.vsr_support = true,
+	.performance_trace = false,
+	.max_downscale_src_width = 4096,/*upto true 4k*/
+	.disable_pplib_wm_range = false,
+	.scl_reset_length10 = true,
+	.sanity_checks = false,
+	.underflow_assert_delay_us = 0xFFFFFFFF,
+	.dwb_fi_phase = -1, // -1 = disable,
+	.dmub_command_table = true,
+	.pstate_enabled = true,
+	.use_max_lb = true,
+	.enable_mem_low_power = {
+		.bits = {
+			.vga = true,
+			.i2c = true,
+			.dmcu = false, // This is previously known to cause hang on S3 cycles if enabled
+			.dscl = true,
+			.cm = true,
+			.mpc = true,
+			.optc = true,
+			.vpg = true,
+			.afmt = true,
+		}
+	},
+	.enable_legacy_fast_update = true,
+	.psr_power_use_phy_fsm = 0,
+	.using_dml2 = false,
+};
+
+static const struct dc_panel_config panel_config_defaults = {
+	.psr = {
+		.disable_psr = false,
+		.disallow_psrsu = false,
+		.disallow_replay = false,
+	},
+	.ilr = {
+		.optimize_edp_link_rate = true,
+	},
+};
+
+static void dcn31_dpp_destroy(struct dpp **dpp)
+{
+	kfree(TO_DCN20_DPP(*dpp));
+	*dpp = NULL;
+}
+
+static struct dpp *dcn31_dpp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn3_dpp *dpp =
+		kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
+
+	if (!dpp)
+		return NULL;
+
+	if (dpp3_construct(dpp, ctx, inst,
+			&dpp_regs[inst], &tf_shift, &tf_mask))
+		return &dpp->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(dpp);
+	return NULL;
+}
+
+static struct output_pixel_processor *dcn31_opp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_opp *opp =
+		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
+
+	if (!opp) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dcn20_opp_construct(opp, ctx, inst,
+			&opp_regs[inst], &opp_shift, &opp_mask);
+	return &opp->base;
+}
+
+static struct dce_aux *dcn31_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst],
+					&aux_mask,
+					&aux_shift,
+					ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST_DCN30(id) }
+
+static const struct dce_i2c_registers i2c_hw_regs[] = {
+		i2c_inst_regs(1),
+		i2c_inst_regs(2),
+		i2c_inst_regs(3),
+		i2c_inst_regs(4),
+		i2c_inst_regs(5),
+};
+
+static const struct dce_i2c_shift i2c_shifts = {
+		I2C_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dce_i2c_mask i2c_masks = {
+		I2C_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static struct dce_i2c_hw *dcn31_i2c_hw_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw =
+		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
+				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+static struct mpc *dcn31_mpc_create(
+		struct dc_context *ctx,
+		int num_mpcc,
+		int num_rmu)
+{
+	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc),
+					  GFP_KERNEL);
+
+	if (!mpc30)
+		return NULL;
+
+	dcn30_mpc_construct(mpc30, ctx,
+			&mpc_regs,
+			&mpc_shift,
+			&mpc_mask,
+			num_mpcc,
+			num_rmu);
+
+	return &mpc30->base;
+}
+
+static struct hubbub *dcn31_hubbub_create(struct dc_context *ctx)
+{
+	int i;
+
+	struct dcn20_hubbub *hubbub3 = kzalloc(sizeof(struct dcn20_hubbub),
+					  GFP_KERNEL);
+
+	if (!hubbub3)
+		return NULL;
+
+	hubbub31_construct(hubbub3, ctx,
+			&hubbub_reg,
+			&hubbub_shift,
+			&hubbub_mask,
+			dcn3_15_ip.det_buffer_size_kbytes,
+			dcn3_15_ip.pixel_chunk_size_kbytes,
+			dcn3_15_ip.config_return_buffer_size_in_kbytes);
+
+
+	for (i = 0; i < res_cap_dcn31.num_vmid; i++) {
+		struct dcn20_vmid *vmid = &hubbub3->vmid[i];
+
+		vmid->ctx = ctx;
+
+		vmid->regs = &vmid_regs[i];
+		vmid->shifts = &vmid_shifts;
+		vmid->masks = &vmid_masks;
+	}
+
+	return &hubbub3->base;
+}
+
+static struct timing_generator *dcn31_timing_generator_create(
+		struct dc_context *ctx,
+		uint32_t instance)
+{
+	struct optc *tgn10 =
+		kzalloc(sizeof(struct optc), GFP_KERNEL);
+
+	if (!tgn10)
+		return NULL;
+
+	tgn10->base.inst = instance;
+	tgn10->base.ctx = ctx;
+
+	tgn10->tg_regs = &optc_regs[instance];
+	tgn10->tg_shift = &optc_shift;
+	tgn10->tg_mask = &optc_mask;
+
+	dcn31_timing_generator_init(tgn10);
+
+	return &tgn10->base;
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+		.max_hdmi_deep_color = COLOR_DEPTH_121212,
+		.max_hdmi_pixel_clock = 600000,
+		.hdmi_ycbcr420_supported = true,
+		.dp_ycbcr420_supported = true,
+		.fec_supported = true,
+		.flags.bits.IS_HBR2_CAPABLE = true,
+		.flags.bits.IS_HBR3_CAPABLE = true,
+		.flags.bits.IS_TPS3_CAPABLE = true,
+		.flags.bits.IS_TPS4_CAPABLE = true
+};
+
+static struct link_encoder *dcn31_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dcn20_link_encoder *enc20 =
+		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+
+	if (!enc20)
+		return NULL;
+
+	dcn31_link_encoder_construct(enc20,
+			enc_init_data,
+			&link_enc_feature,
+			&link_enc_regs[enc_init_data->transmitter],
+			&link_enc_aux_regs[enc_init_data->channel - 1],
+			&link_enc_hpd_regs[enc_init_data->hpd_source],
+			&le_shift,
+			&le_mask);
+
+	return &enc20->enc10.base;
+}
+
+/* Create a minimal link encoder object not associated with a particular
+ * physical connector.
+ * resource_funcs.link_enc_create_minimal
+ */
+static struct link_encoder *dcn31_link_enc_create_minimal(
+		struct dc_context *ctx, enum engine_id eng_id)
+{
+	struct dcn20_link_encoder *enc20;
+
+	if ((eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+		return NULL;
+
+	enc20 = kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+	if (!enc20)
+		return NULL;
+
+	dcn31_link_encoder_construct_minimal(
+			enc20,
+			ctx,
+			&link_enc_feature,
+			&link_enc_regs[eng_id - ENGINE_ID_DIGA],
+			eng_id);
+
+	return &enc20->enc10.base;
+}
+
+static struct panel_cntl *dcn31_panel_cntl_create(const struct panel_cntl_init_data *init_data)
+{
+	struct dcn31_panel_cntl *panel_cntl =
+		kzalloc(sizeof(struct dcn31_panel_cntl), GFP_KERNEL);
+
+	if (!panel_cntl)
+		return NULL;
+
+	dcn31_panel_cntl_construct(panel_cntl, init_data);
+
+	return &panel_cntl->base;
+}
+
+static void read_dce_straps(
+	struct dc_context *ctx,
+	struct resource_straps *straps)
+{
+	generic_reg_get(ctx, regDC_PINSTRAPS + BASE(regDC_PINSTRAPS_BASE_IDX),
+		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
+
+}
+
+static struct audio *dcn31_create_audio(
+		struct dc_context *ctx, unsigned int inst)
+{
+	return dce_audio_create(ctx, inst,
+			&audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+static struct vpg *dcn31_vpg_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn31_vpg *vpg31 = kzalloc(sizeof(struct dcn31_vpg), GFP_KERNEL);
+
+	if (!vpg31)
+		return NULL;
+
+	vpg31_construct(vpg31, ctx, inst,
+			&vpg_regs[inst],
+			&vpg_shift,
+			&vpg_mask);
+
+	return &vpg31->base;
+}
+
+static struct afmt *dcn31_afmt_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn31_afmt *afmt31 = kzalloc(sizeof(struct dcn31_afmt), GFP_KERNEL);
+
+	if (!afmt31)
+		return NULL;
+
+	afmt31_construct(afmt31, ctx, inst,
+			&afmt_regs[inst],
+			&afmt_shift,
+			&afmt_mask);
+
+	// Light sleep by default, no need to power down here
+
+	return &afmt31->base;
+}
+
+static struct apg *dcn31_apg_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn31_apg *apg31 = kzalloc(sizeof(struct dcn31_apg), GFP_KERNEL);
+
+	if (!apg31)
+		return NULL;
+
+	apg31_construct(apg31, ctx, inst,
+			&apg_regs[inst],
+			&apg_shift,
+			&apg_mask);
+
+	return &apg31->base;
+}
+
+static struct stream_encoder *dcn315_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dcn10_stream_encoder *enc1;
+	struct vpg *vpg;
+	struct afmt *afmt;
+	int vpg_inst;
+	int afmt_inst;
+
+	/*PHYB is wired off in HW, allow front end to remapping, otherwise needs more changes*/
+
+	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
+	if (eng_id <= ENGINE_ID_DIGF) {
+		vpg_inst = eng_id;
+		afmt_inst = eng_id;
+	} else
+		return NULL;
+
+	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
+	vpg = dcn31_vpg_create(ctx, vpg_inst);
+	afmt = dcn31_afmt_create(ctx, afmt_inst);
+
+	if (!enc1 || !vpg || !afmt) {
+		kfree(enc1);
+		kfree(vpg);
+		kfree(afmt);
+		return NULL;
+	}
+
+	dcn30_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
+					eng_id, vpg, afmt,
+					&stream_enc_regs[eng_id],
+					&se_shift, &se_mask);
+
+	return &enc1->base;
+}
+
+static struct hpo_dp_stream_encoder *dcn31_hpo_dp_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dcn31_hpo_dp_stream_encoder *hpo_dp_enc31;
+	struct vpg *vpg;
+	struct apg *apg;
+	uint32_t hpo_dp_inst;
+	uint32_t vpg_inst;
+	uint32_t apg_inst;
+
+	ASSERT((eng_id >= ENGINE_ID_HPO_DP_0) && (eng_id <= ENGINE_ID_HPO_DP_3));
+	hpo_dp_inst = eng_id - ENGINE_ID_HPO_DP_0;
+
+	/* Mapping of VPG register blocks to HPO DP block instance:
+	 * VPG[6] -> HPO_DP[0]
+	 * VPG[7] -> HPO_DP[1]
+	 * VPG[8] -> HPO_DP[2]
+	 * VPG[9] -> HPO_DP[3]
+	 */
+	vpg_inst = hpo_dp_inst + 6;
+
+	/* Mapping of APG register blocks to HPO DP block instance:
+	 * APG[0] -> HPO_DP[0]
+	 * APG[1] -> HPO_DP[1]
+	 * APG[2] -> HPO_DP[2]
+	 * APG[3] -> HPO_DP[3]
+	 */
+	apg_inst = hpo_dp_inst;
+
+	/* allocate HPO stream encoder and create VPG sub-block */
+	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_stream_encoder), GFP_KERNEL);
+	vpg = dcn31_vpg_create(ctx, vpg_inst);
+	apg = dcn31_apg_create(ctx, apg_inst);
+
+	if (!hpo_dp_enc31 || !vpg || !apg) {
+		kfree(hpo_dp_enc31);
+		kfree(vpg);
+		kfree(apg);
+		return NULL;
+	}
+
+	dcn31_hpo_dp_stream_encoder_construct(hpo_dp_enc31, ctx, ctx->dc_bios,
+					hpo_dp_inst, eng_id, vpg, apg,
+					&hpo_dp_stream_enc_regs[hpo_dp_inst],
+					&hpo_dp_se_shift, &hpo_dp_se_mask);
+
+	return &hpo_dp_enc31->base;
+}
+
+static struct hpo_dp_link_encoder *dcn31_hpo_dp_link_encoder_create(
+	uint8_t inst,
+	struct dc_context *ctx)
+{
+	struct dcn31_hpo_dp_link_encoder *hpo_dp_enc31;
+
+	/* allocate HPO link encoder */
+	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
+
+	hpo_dp_link_encoder31_construct(hpo_dp_enc31, ctx, inst,
+					&hpo_dp_link_enc_regs[inst],
+					&hpo_dp_le_shift, &hpo_dp_le_mask);
+
+	return &hpo_dp_enc31->base;
+}
+
+static struct dce_hwseq *dcn31_hwseq_create(
+	struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &hwseq_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+	}
+	return hws;
+}
+static const struct resource_create_funcs res_create_funcs = {
+	.read_dce_straps = read_dce_straps,
+	.create_audio = dcn31_create_audio,
+	.create_stream_encoder = dcn315_stream_encoder_create,
+	.create_hpo_dp_stream_encoder = dcn31_hpo_dp_stream_encoder_create,
+	.create_hpo_dp_link_encoder = dcn31_hpo_dp_link_encoder_create,
+	.create_hwseq = dcn31_hwseq_create,
+};
+
+static void dcn315_resource_destruct(struct dcn315_resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->base.stream_enc_count; i++) {
+		if (pool->base.stream_enc[i] != NULL) {
+			if (pool->base.stream_enc[i]->vpg != NULL) {
+				kfree(DCN30_VPG_FROM_VPG(pool->base.stream_enc[i]->vpg));
+				pool->base.stream_enc[i]->vpg = NULL;
+			}
+			if (pool->base.stream_enc[i]->afmt != NULL) {
+				kfree(DCN30_AFMT_FROM_AFMT(pool->base.stream_enc[i]->afmt));
+				pool->base.stream_enc[i]->afmt = NULL;
+			}
+			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
+			pool->base.stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.hpo_dp_stream_enc_count; i++) {
+		if (pool->base.hpo_dp_stream_enc[i] != NULL) {
+			if (pool->base.hpo_dp_stream_enc[i]->vpg != NULL) {
+				kfree(DCN30_VPG_FROM_VPG(pool->base.hpo_dp_stream_enc[i]->vpg));
+				pool->base.hpo_dp_stream_enc[i]->vpg = NULL;
+			}
+			if (pool->base.hpo_dp_stream_enc[i]->apg != NULL) {
+				kfree(DCN31_APG_FROM_APG(pool->base.hpo_dp_stream_enc[i]->apg));
+				pool->base.hpo_dp_stream_enc[i]->apg = NULL;
+			}
+			kfree(DCN3_1_HPO_DP_STREAM_ENC_FROM_HPO_STREAM_ENC(pool->base.hpo_dp_stream_enc[i]));
+			pool->base.hpo_dp_stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.hpo_dp_link_enc_count; i++) {
+		if (pool->base.hpo_dp_link_enc[i] != NULL) {
+			kfree(DCN3_1_HPO_DP_LINK_ENC_FROM_HPO_LINK_ENC(pool->base.hpo_dp_link_enc[i]));
+			pool->base.hpo_dp_link_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		if (pool->base.dscs[i] != NULL)
+			dcn20_dsc_destroy(&pool->base.dscs[i]);
+	}
+
+	if (pool->base.mpc != NULL) {
+		kfree(TO_DCN20_MPC(pool->base.mpc));
+		pool->base.mpc = NULL;
+	}
+	if (pool->base.hubbub != NULL) {
+		kfree(pool->base.hubbub);
+		pool->base.hubbub = NULL;
+	}
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (pool->base.dpps[i] != NULL)
+			dcn31_dpp_destroy(&pool->base.dpps[i]);
+
+		if (pool->base.ipps[i] != NULL)
+			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
+
+		if (pool->base.hubps[i] != NULL) {
+			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
+			pool->base.hubps[i] = NULL;
+		}
+
+		if (pool->base.irqs != NULL) {
+			dal_irq_service_destroy(&pool->base.irqs);
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+		if (pool->base.hw_i2cs[i] != NULL) {
+			kfree(pool->base.hw_i2cs[i]);
+			pool->base.hw_i2cs[i] = NULL;
+		}
+		if (pool->base.sw_i2cs[i] != NULL) {
+			kfree(pool->base.sw_i2cs[i]);
+			pool->base.sw_i2cs[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		if (pool->base.opps[i] != NULL)
+			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.timing_generators[i] != NULL)	{
+			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
+			pool->base.timing_generators[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+		if (pool->base.dwbc[i] != NULL) {
+			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
+			pool->base.dwbc[i] = NULL;
+		}
+		if (pool->base.mcif_wb[i] != NULL) {
+			kfree(TO_DCN30_MMHUBBUB(pool->base.mcif_wb[i]));
+			pool->base.mcif_wb[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.audio_count; i++) {
+		if (pool->base.audios[i])
+			dce_aud_destroy(&pool->base.audios[i]);
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] != NULL) {
+			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
+			pool->base.clock_sources[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+		if (pool->base.mpc_lut[i] != NULL) {
+			dc_3dlut_func_release(pool->base.mpc_lut[i]);
+			pool->base.mpc_lut[i] = NULL;
+		}
+		if (pool->base.mpc_shaper[i] != NULL) {
+			dc_transfer_func_release(pool->base.mpc_shaper[i]);
+			pool->base.mpc_shaper[i] = NULL;
+		}
+	}
+
+	if (pool->base.dp_clock_source != NULL) {
+		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
+		pool->base.dp_clock_source = NULL;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.multiple_abms[i] != NULL)
+			dce_abm_destroy(&pool->base.multiple_abms[i]);
+	}
+
+	if (pool->base.psr != NULL)
+		dmub_psr_destroy(&pool->base.psr);
+
+	if (pool->base.dccg != NULL)
+		dcn_dccg_destroy(&pool->base.dccg);
+}
+
+static struct hubp *dcn31_hubp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn20_hubp *hubp2 =
+		kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
+
+	if (!hubp2)
+		return NULL;
+
+	if (hubp31_construct(hubp2, ctx, inst,
+			&hubp_regs[inst], &hubp_shift, &hubp_mask))
+		return &hubp2->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(hubp2);
+	return NULL;
+}
+
+static bool dcn31_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc),
+						    GFP_KERNEL);
+
+		if (!dwbc30) {
+			dm_error("DC: failed to create dwbc30!\n");
+			return false;
+		}
+
+		dcn30_dwbc_construct(dwbc30, ctx,
+				&dwbc30_regs[i],
+				&dwbc30_shift,
+				&dwbc30_mask,
+				i);
+
+		pool->dwbc[i] = &dwbc30->base;
+	}
+	return true;
+}
+
+static bool dcn31_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub),
+						    GFP_KERNEL);
+
+		if (!mcif_wb30) {
+			dm_error("DC: failed to create mcif_wb30!\n");
+			return false;
+		}
+
+		dcn30_mmhubbub_construct(mcif_wb30, ctx,
+				&mcif_wb30_regs[i],
+				&mcif_wb30_shift,
+				&mcif_wb30_mask,
+				i);
+
+		pool->mcif_wb[i] = &mcif_wb30->base;
+	}
+	return true;
+}
+
+static struct display_stream_compressor *dcn31_dsc_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_dsc *dsc =
+		kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
+
+	if (!dsc) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
+	return &dsc->base;
+}
+
+static void dcn315_destroy_resource_pool(struct resource_pool **pool)
+{
+	struct dcn315_resource_pool *dcn31_pool = TO_DCN315_RES_POOL(*pool);
+
+	dcn315_resource_destruct(dcn31_pool);
+	kfree(dcn31_pool);
+	*pool = NULL;
+}
+
+static struct clock_source *dcn31_clock_source_create(
+		struct dc_context *ctx,
+		struct dc_bios *bios,
+		enum clock_source_id id,
+		const struct dce110_clk_src_regs *regs,
+		bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dcn31_clk_src_construct(clk_src, ctx, bios, id,
+			regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	kfree(clk_src);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static bool is_dual_plane(enum surface_pixel_format format)
+{
+	return format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN || format == SURFACE_PIXEL_FORMAT_GRPH_RGBE_ALPHA;
+}
+
+static int source_format_to_bpp (enum source_format_class SourcePixelFormat)
+{
+	if (SourcePixelFormat == dm_444_64)
+		return 8;
+	else if (SourcePixelFormat == dm_444_16)
+		return 2;
+	else if (SourcePixelFormat == dm_444_8)
+		return 1;
+	else if (SourcePixelFormat == dm_rgbe_alpha)
+		return 5;
+	else if (SourcePixelFormat == dm_420_8)
+		return 3;
+	else if (SourcePixelFormat == dm_420_12)
+		return 6;
+	else
+		return 4;
+}
+
+static bool allow_pixel_rate_crb(struct dc *dc, struct dc_state *context)
+{
+	int i;
+	struct resource_context *res_ctx = &context->res_ctx;
+
+	/*Don't apply for single stream*/
+	if (context->stream_count < 2)
+		return false;
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		if (!res_ctx->pipe_ctx[i].stream)
+			continue;
+
+		/*Don't apply if scaling*/
+		if (res_ctx->pipe_ctx[i].stream->src.width != res_ctx->pipe_ctx[i].stream->dst.width ||
+				res_ctx->pipe_ctx[i].stream->src.height != res_ctx->pipe_ctx[i].stream->dst.height ||
+				(res_ctx->pipe_ctx[i].plane_state && (res_ctx->pipe_ctx[i].plane_state->src_rect.width
+														!= res_ctx->pipe_ctx[i].plane_state->dst_rect.width ||
+					res_ctx->pipe_ctx[i].plane_state->src_rect.height
+														!= res_ctx->pipe_ctx[i].plane_state->dst_rect.height)))
+			return false;
+		/*Don't apply if MPO to avoid transition issues*/
+		if (res_ctx->pipe_ctx[i].top_pipe && res_ctx->pipe_ctx[i].top_pipe->plane_state != res_ctx->pipe_ctx[i].plane_state)
+			return false;
+	}
+	return true;
+}
+
+static int dcn315_populate_dml_pipes_from_context(
+	struct dc *dc, struct dc_state *context,
+	display_e2e_pipe_params_st *pipes,
+	bool fast_validate)
+{
+	int i, pipe_cnt, crb_idx, crb_pipes;
+	struct resource_context *res_ctx = &context->res_ctx;
+	struct pipe_ctx *pipe = NULL;
+	const int max_usable_det = context->bw_ctx.dml.ip.config_return_buffer_size_in_kbytes - DCN3_15_MIN_COMPBUF_SIZE_KB;
+	int remaining_det_segs = max_usable_det / DCN3_15_CRB_SEGMENT_SIZE_KB;
+	bool pixel_rate_crb = allow_pixel_rate_crb(dc, context);
+
+	DC_FP_START();
+	dcn31x_populate_dml_pipes_from_context(dc, context, pipes, fast_validate);
+	DC_FP_END();
+
+	for (i = 0, pipe_cnt = 0, crb_pipes = 0; i < dc->res_pool->pipe_count; i++) {
+		struct dc_crtc_timing *timing;
+
+		if (!res_ctx->pipe_ctx[i].stream)
+			continue;
+		pipe = &res_ctx->pipe_ctx[i];
+		timing = &pipe->stream->timing;
+
+		/*
+		 * Immediate flip can be set dynamically after enabling the plane.
+		 * We need to require support for immediate flip or underflow can be
+		 * intermittently experienced depending on peak b/w requirements.
+		 */
+		pipes[pipe_cnt].pipe.src.immediate_flip = true;
+
+		pipes[pipe_cnt].pipe.src.unbounded_req_mode = false;
+		pipes[pipe_cnt].pipe.dest.vfront_porch = timing->v_front_porch;
+		pipes[pipe_cnt].pipe.src.dcc_rate = 3;
+		pipes[pipe_cnt].dout.dsc_input_bpc = 0;
+		DC_FP_START();
+		dcn31_zero_pipe_dcc_fraction(pipes, pipe_cnt);
+		if (pixel_rate_crb && !pipe->top_pipe && !pipe->prev_odm_pipe) {
+			int bpp = source_format_to_bpp(pipes[pipe_cnt].pipe.src.source_format);
+			/* Ceil to crb segment size */
+			int approx_det_segs_required_for_pstate = dcn_get_approx_det_segs_required_for_pstate(
+					&context->bw_ctx.dml.soc, timing->pix_clk_100hz, bpp, DCN3_15_CRB_SEGMENT_SIZE_KB);
+
+			if (approx_det_segs_required_for_pstate <= 2 * DCN3_15_MAX_DET_SEGS) {
+				bool split_required = approx_det_segs_required_for_pstate > DCN3_15_MAX_DET_SEGS;
+				split_required = split_required || timing->pix_clk_100hz >= dcn_get_max_non_odm_pix_rate_100hz(&dc->dml.soc);
+				split_required = split_required || (pipe->plane_state && pipe->plane_state->src_rect.width > 5120);
+
+				/* Minimum 2 segments to allow mpc/odm combine if its used later */
+				if (approx_det_segs_required_for_pstate < 2)
+					approx_det_segs_required_for_pstate = 2;
+				if (split_required)
+					approx_det_segs_required_for_pstate += approx_det_segs_required_for_pstate % 2;
+				pipes[pipe_cnt].pipe.src.det_size_override = approx_det_segs_required_for_pstate;
+				remaining_det_segs -= approx_det_segs_required_for_pstate;
+			} else
+				remaining_det_segs = -1;
+			crb_pipes++;
+		}
+		DC_FP_END();
+
+		if (pipes[pipe_cnt].dout.dsc_enable) {
+			switch (timing->display_color_depth) {
+			case COLOR_DEPTH_888:
+				pipes[pipe_cnt].dout.dsc_input_bpc = 8;
+				break;
+			case COLOR_DEPTH_101010:
+				pipes[pipe_cnt].dout.dsc_input_bpc = 10;
+				break;
+			case COLOR_DEPTH_121212:
+				pipes[pipe_cnt].dout.dsc_input_bpc = 12;
+				break;
+			default:
+				ASSERT(0);
+				break;
+			}
+		}
+		pipe_cnt++;
+	}
+
+	/* Spread remaining unreserved crb evenly among all pipes*/
+	if (pixel_rate_crb) {
+		for (i = 0, pipe_cnt = 0, crb_idx = 0; i < dc->res_pool->pipe_count; i++) {
+			pipe = &res_ctx->pipe_ctx[i];
+			if (!pipe->stream)
+				continue;
+
+			/* Do not use asymetric crb if not enough for pstate support */
+			if (remaining_det_segs < 0) {
+				pipes[pipe_cnt].pipe.src.det_size_override = 0;
+				pipe_cnt++;
+				continue;
+			}
+
+			if (!pipe->top_pipe && !pipe->prev_odm_pipe) {
+				bool split_required = pipe->stream->timing.pix_clk_100hz >= dcn_get_max_non_odm_pix_rate_100hz(&dc->dml.soc)
+						|| (pipe->plane_state && pipe->plane_state->src_rect.width > 5120);
+
+				if (remaining_det_segs > MIN_RESERVED_DET_SEGS)
+					pipes[pipe_cnt].pipe.src.det_size_override += (remaining_det_segs - MIN_RESERVED_DET_SEGS) / crb_pipes +
+							(crb_idx < (remaining_det_segs - MIN_RESERVED_DET_SEGS) % crb_pipes ? 1 : 0);
+				if (pipes[pipe_cnt].pipe.src.det_size_override > 2 * DCN3_15_MAX_DET_SEGS) {
+					/* Clamp to 2 pipe split max det segments */
+					remaining_det_segs += pipes[pipe_cnt].pipe.src.det_size_override - 2 * (DCN3_15_MAX_DET_SEGS);
+					pipes[pipe_cnt].pipe.src.det_size_override = 2 * DCN3_15_MAX_DET_SEGS;
+				}
+				if (pipes[pipe_cnt].pipe.src.det_size_override > DCN3_15_MAX_DET_SEGS || split_required) {
+					/* If we are splitting we must have an even number of segments */
+					remaining_det_segs += pipes[pipe_cnt].pipe.src.det_size_override % 2;
+					pipes[pipe_cnt].pipe.src.det_size_override -= pipes[pipe_cnt].pipe.src.det_size_override % 2;
+				}
+				/* Convert segments into size for DML use */
+				pipes[pipe_cnt].pipe.src.det_size_override *= DCN3_15_CRB_SEGMENT_SIZE_KB;
+
+				crb_idx++;
+			}
+			pipe_cnt++;
+		}
+	}
+
+	if (pipe_cnt)
+		context->bw_ctx.dml.ip.det_buffer_size_kbytes =
+				(max_usable_det / DCN3_15_CRB_SEGMENT_SIZE_KB / pipe_cnt) * DCN3_15_CRB_SEGMENT_SIZE_KB;
+	if (context->bw_ctx.dml.ip.det_buffer_size_kbytes > DCN3_15_MAX_DET_SIZE)
+		context->bw_ctx.dml.ip.det_buffer_size_kbytes = DCN3_15_MAX_DET_SIZE;
+
+	dc->config.enable_4to1MPC = false;
+	if (pipe_cnt == 1 && pipe->plane_state && !dc->debug.disable_z9_mpc) {
+		if (is_dual_plane(pipe->plane_state->format)
+				&& pipe->plane_state->src_rect.width <= 1920 && pipe->plane_state->src_rect.height <= 1080) {
+			dc->config.enable_4to1MPC = true;
+			context->bw_ctx.dml.ip.det_buffer_size_kbytes =
+					(max_usable_det / DCN3_15_CRB_SEGMENT_SIZE_KB / 4) * DCN3_15_CRB_SEGMENT_SIZE_KB;
+		} else if (!is_dual_plane(pipe->plane_state->format)
+				&& pipe->plane_state->src_rect.width <= 5120
+				&& pipe->stream->timing.pix_clk_100hz < dcn_get_max_non_odm_pix_rate_100hz(&dc->dml.soc)) {
+			/* Limit to 5k max to avoid forced pipe split when there is not enough detile for swath */
+			context->bw_ctx.dml.ip.det_buffer_size_kbytes = 192;
+			pipes[0].pipe.src.unbounded_req_mode = true;
+		}
+	}
+
+	return pipe_cnt;
+}
+
+static void dcn315_get_panel_config_defaults(struct dc_panel_config *panel_config)
+{
+	*panel_config = panel_config_defaults;
+}
+
+static struct dc_cap_funcs cap_funcs = {
+	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
+};
+
+static struct resource_funcs dcn315_res_pool_funcs = {
+	.destroy = dcn315_destroy_resource_pool,
+	.link_enc_create = dcn31_link_encoder_create,
+	.link_enc_create_minimal = dcn31_link_enc_create_minimal,
+	.link_encs_assign = link_enc_cfg_link_encs_assign,
+	.link_enc_unassign = link_enc_cfg_link_enc_unassign,
+	.panel_cntl_create = dcn31_panel_cntl_create,
+	.validate_bandwidth = dcn31_validate_bandwidth,
+	.calculate_wm_and_dlg = dcn31_calculate_wm_and_dlg,
+	.update_soc_for_wm_a = dcn315_update_soc_for_wm_a,
+	.populate_dml_pipes = dcn315_populate_dml_pipes_from_context,
+	.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
+	.release_pipe = dcn20_release_pipe,
+	.add_stream_to_ctx = dcn30_add_stream_to_ctx,
+	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
+	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
+	.populate_dml_writeback_from_context = dcn31_populate_dml_writeback_from_context,
+	.set_mcif_arb_params = dcn31_set_mcif_arb_params,
+	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
+	.acquire_post_bldn_3dlut = dcn30_acquire_post_bldn_3dlut,
+	.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
+	.update_bw_bounding_box = dcn315_update_bw_bounding_box,
+	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
+	.get_panel_config_defaults = dcn315_get_panel_config_defaults,
+};
+
+static bool dcn315_resource_construct(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct dcn315_resource_pool *pool)
+{
+	int i;
+	struct dc_context *ctx = dc->ctx;
+	struct irq_service_init_data init_data;
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	pool->base.res_cap = &res_cap_dcn31;
+
+	pool->base.funcs = &dcn315_res_pool_funcs;
+
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
+	pool->base.mpcc_count = pool->base.res_cap->num_timing_generator;
+	dc->caps.max_downscale_ratio = 600;
+	dc->caps.i2c_speed_in_khz = 100;
+	dc->caps.i2c_speed_in_khz_hdcp = 100;
+	dc->caps.max_cursor_size = 256;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.dmdata_alloc_size = 2048;
+	dc->caps.max_slave_planes = 2;
+	dc->caps.max_slave_yuv_planes = 2;
+	dc->caps.max_slave_rgb_planes = 2;
+	dc->caps.post_blend_color_processing = true;
+	dc->caps.force_dp_tps4_for_cp2520 = true;
+	if (dc->config.forceHBR2CP2520)
+		dc->caps.force_dp_tps4_for_cp2520 = false;
+	dc->caps.dp_hpo = true;
+	dc->caps.dp_hdmi21_pcon_support = true;
+	dc->caps.edp_dsc_support = true;
+	dc->caps.extended_aux_timeout_support = true;
+	dc->caps.dmcub_support = true;
+	dc->caps.is_apu = true;
+
+	/* Color pipeline capabilities */
+	dc->caps.color.dpp.dcn_arch = 1;
+	dc->caps.color.dpp.input_lut_shared = 0;
+	dc->caps.color.dpp.icsc = 1;
+	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
+	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
+	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
+	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
+	dc->caps.color.dpp.post_csc = 1;
+	dc->caps.color.dpp.gamma_corr = 1;
+	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
+
+	dc->caps.color.dpp.hw_3d_lut = 1;
+	dc->caps.color.dpp.ogam_ram = 1;
+	// no OGAM ROM on DCN301
+	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
+	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
+	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
+	dc->caps.color.dpp.ocsc = 0;
+
+	dc->caps.color.mpc.gamut_remap = 1;
+	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //2
+	dc->caps.color.mpc.ogam_ram = 1;
+	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
+	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
+	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
+	dc->caps.color.mpc.ocsc = 1;
+
+	/* read VBIOS LTTPR caps */
+	{
+		if (ctx->dc_bios->funcs->get_lttpr_caps) {
+			enum bp_result bp_query_result;
+			uint8_t is_vbios_lttpr_enable = 0;
+
+			bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
+			dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
+		}
+
+		/* interop bit is implicit */
+		{
+			dc->caps.vbios_lttpr_aware = true;
+		}
+	}
+
+	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
+		dc->debug = debug_defaults_drv;
+
+	// Init the vm_helper
+	if (dc->vm_helper)
+		vm_helper_init(dc->vm_helper, 16);
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	/* Clock Sources for Pixel Clock*/
+	pool->base.clock_sources[DCN31_CLK_SRC_PLL0] =
+			dcn31_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL0,
+				&clk_src_regs[0], false);
+	pool->base.clock_sources[DCN31_CLK_SRC_PLL1] =
+			dcn31_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL1,
+				&clk_src_regs[1], false);
+	pool->base.clock_sources[DCN31_CLK_SRC_PLL2] =
+			dcn31_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL2,
+				&clk_src_regs[2], false);
+	pool->base.clock_sources[DCN31_CLK_SRC_PLL3] =
+			dcn31_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL3,
+				&clk_src_regs[3], false);
+	pool->base.clock_sources[DCN31_CLK_SRC_PLL4] =
+			dcn31_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL4,
+				&clk_src_regs[4], false);
+
+	pool->base.clk_src_count = DCN30_CLK_SRC_TOTAL;
+
+	/* todo: not reuse phy_pll registers */
+	pool->base.dp_clock_source =
+			dcn31_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_ID_DP_DTO,
+				&clk_src_regs[0], true);
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	/* TODO: DCCG */
+	pool->base.dccg = dccg31_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
+	if (pool->base.dccg == NULL) {
+		dm_error("DC: failed to create dccg!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* TODO: IRQ */
+	init_data.ctx = dc->ctx;
+	pool->base.irqs = dal_irq_service_dcn315_create(&init_data);
+	if (!pool->base.irqs)
+		goto create_fail;
+
+	/* HUBBUB */
+	pool->base.hubbub = dcn31_hubbub_create(ctx);
+	if (pool->base.hubbub == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create hubbub!\n");
+		goto create_fail;
+	}
+
+	/* HUBPs, DPPs, OPPs and TGs */
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		pool->base.hubps[i] = dcn31_hubp_create(ctx, i);
+		if (pool->base.hubps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create hubps!\n");
+			goto create_fail;
+		}
+
+		pool->base.dpps[i] = dcn31_dpp_create(ctx, i);
+		if (pool->base.dpps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create dpps!\n");
+			goto create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		pool->base.opps[i] = dcn31_opp_create(ctx, i);
+		if (pool->base.opps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create output pixel processor!\n");
+			goto create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		pool->base.timing_generators[i] = dcn31_timing_generator_create(
+				ctx, i);
+		if (pool->base.timing_generators[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto create_fail;
+		}
+	}
+	pool->base.timing_generator_count = i;
+
+	/* PSR */
+	pool->base.psr = dmub_psr_create(ctx);
+	if (pool->base.psr == NULL) {
+		dm_error("DC: failed to create psr obj!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* ABM */
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		pool->base.multiple_abms[i] = dmub_abm_create(ctx,
+				&abm_regs[i],
+				&abm_shift,
+				&abm_mask);
+		if (pool->base.multiple_abms[i] == NULL) {
+			dm_error("DC: failed to create abm for pipe %d!\n", i);
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	/* MPC and DSC */
+	pool->base.mpc = dcn31_mpc_create(ctx, pool->base.mpcc_count, pool->base.res_cap->num_mpc_3dlut);
+	if (pool->base.mpc == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mpc!\n");
+		goto create_fail;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		pool->base.dscs[i] = dcn31_dsc_create(ctx, i);
+		if (pool->base.dscs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create display stream compressor %d!\n", i);
+			goto create_fail;
+		}
+	}
+
+	/* DWB and MMHUBBUB */
+	if (!dcn31_dwbc_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create dwbc!\n");
+		goto create_fail;
+	}
+
+	if (!dcn31_mmhubbub_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mcif_wb!\n");
+		goto create_fail;
+	}
+
+	/* AUX and I2C */
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.engines[i] = dcn31_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto create_fail;
+		}
+		pool->base.hw_i2cs[i] = dcn31_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create hw i2c!!\n");
+			goto create_fail;
+		}
+		pool->base.sw_i2cs[i] = NULL;
+	}
+
+	/* Audio, Stream Encoders including HPO and virtual, MPC 3D LUTs */
+	if (!resource_construct(num_virtual_links, dc, &pool->base,
+			&res_create_funcs))
+		goto create_fail;
+
+	/* HW Sequencer and Plane caps */
+	dcn31_hw_sequencer_construct(dc);
+
+	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->cap_funcs = cap_funcs;
+
+	dc->dcn_ip->max_num_dpp = dcn3_15_ip.max_num_dpp;
+
+	return true;
+
+create_fail:
+
+	dcn315_resource_destruct(pool);
+
+	return false;
+}
+
+struct resource_pool *dcn315_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc)
+{
+	struct dcn315_resource_pool *pool =
+		kzalloc(sizeof(struct dcn315_resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dcn315_resource_construct(init_data->num_virtual_links, dc, pool))
+		return &pool->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(pool);
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.h
new file mode 100644
index 000000000000..22849eaa6f24
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.h
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2021 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef _DCN315_RESOURCE_H_
+#define _DCN315_RESOURCE_H_
+
+#include "core_types.h"
+
+#define TO_DCN315_RES_POOL(pool)\
+	container_of(pool, struct dcn315_resource_pool, base)
+
+extern struct _vcs_dpi_ip_params_st dcn3_15_ip;
+
+struct dcn315_resource_pool {
+	struct resource_pool base;
+};
+
+struct resource_pool *dcn315_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc);
+
+#endif /* _DCN315_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c
new file mode 100644
index 000000000000..b9753d4606f8
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c
@@ -0,0 +1,2038 @@
+/*
+ * Copyright 2021 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+
+#include "dm_services.h"
+#include "dc.h"
+
+#include "dcn31/dcn31_init.h"
+
+#include "resource.h"
+#include "include/irq_service_interface.h"
+#include "dcn316_resource.h"
+
+#include "dcn20/dcn20_resource.h"
+#include "dcn30/dcn30_resource.h"
+#include "dcn31/dcn31_resource.h"
+
+#include "dcn10/dcn10_ipp.h"
+#include "dcn30/dcn30_hubbub.h"
+#include "dcn31/dcn31_hubbub.h"
+#include "dcn30/dcn30_mpc.h"
+#include "dcn31/dcn31_hubp.h"
+#include "irq/dcn31/irq_service_dcn31.h"
+#include "dcn30/dcn30_dpp.h"
+#include "dcn31/dcn31_optc.h"
+#include "dcn20/dcn20_hwseq.h"
+#include "dcn30/dcn30_hwseq.h"
+#include "dce110/dce110_hwseq.h"
+#include "dcn30/dcn30_opp.h"
+#include "dcn20/dcn20_dsc.h"
+#include "dcn30/dcn30_vpg.h"
+#include "dcn30/dcn30_afmt.h"
+#include "dcn30/dcn30_dio_stream_encoder.h"
+#include "dcn31/dcn31_hpo_dp_stream_encoder.h"
+#include "dcn31/dcn31_hpo_dp_link_encoder.h"
+#include "dcn31/dcn31_apg.h"
+#include "dcn31/dcn31_dio_link_encoder.h"
+#include "dcn31/dcn31_vpg.h"
+#include "dcn31/dcn31_afmt.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_hwseq.h"
+#include "clk_mgr.h"
+#include "virtual/virtual_stream_encoder.h"
+#include "dce110/dce110_resource.h"
+#include "dml/display_mode_vba.h"
+#include "dml/dcn31/dcn31_fpu.h"
+#include "dcn31/dcn31_dccg.h"
+#include "dcn10/dcn10_resource.h"
+#include "dcn31/dcn31_panel_cntl.h"
+
+#include "dcn30/dcn30_dwb.h"
+#include "dcn30/dcn30_mmhubbub.h"
+
+#include "dcn/dcn_3_1_6_offset.h"
+#include "dcn/dcn_3_1_6_sh_mask.h"
+#include "dpcs/dpcs_4_2_3_offset.h"
+#include "dpcs/dpcs_4_2_3_sh_mask.h"
+
+#define regBIF_BX1_BIOS_SCRATCH_2                                                                       0x003a
+#define regBIF_BX1_BIOS_SCRATCH_2_BASE_IDX                                                              1
+#define regBIF_BX1_BIOS_SCRATCH_3                                                                       0x003b
+#define regBIF_BX1_BIOS_SCRATCH_3_BASE_IDX                                                              1
+#define regBIF_BX1_BIOS_SCRATCH_6                                                                       0x003e
+#define regBIF_BX1_BIOS_SCRATCH_6_BASE_IDX                                                              1
+
+#define regDCHUBBUB_DEBUG_CTRL_0                                              0x04d6
+#define regDCHUBBUB_DEBUG_CTRL_0_BASE_IDX                                     2
+#define DCHUBBUB_DEBUG_CTRL_0__DET_DEPTH__SHIFT                               0x10
+#define DCHUBBUB_DEBUG_CTRL_0__DET_DEPTH_MASK                                 0x01FF0000L
+
+#define DCN_BASE__INST0_SEG0                       0x00000012
+#define DCN_BASE__INST0_SEG1                       0x000000C0
+#define DCN_BASE__INST0_SEG2                       0x000034C0
+#define DCN_BASE__INST0_SEG3                       0x00009000
+#define DCN_BASE__INST0_SEG4                       0x02403C00
+#define DCN_BASE__INST0_SEG5                       0
+
+#define DPCS_BASE__INST0_SEG0                      0x00000012
+#define DPCS_BASE__INST0_SEG1                      0x000000C0
+#define DPCS_BASE__INST0_SEG2                      0x000034C0
+#define DPCS_BASE__INST0_SEG3                      0x00009000
+#define DPCS_BASE__INST0_SEG4                      0x02403C00
+#define DPCS_BASE__INST0_SEG5                      0
+
+#define NBIO_BASE__INST0_SEG0                      0x00000000
+#define NBIO_BASE__INST0_SEG1                      0x00000014
+#define NBIO_BASE__INST0_SEG2                      0x00000D20
+#define NBIO_BASE__INST0_SEG3                      0x00010400
+#define NBIO_BASE__INST0_SEG4                      0x0241B000
+#define NBIO_BASE__INST0_SEG5                      0x04040000
+
+#include "reg_helper.h"
+#include "dce/dmub_abm.h"
+#include "dce/dmub_psr.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_i2c.h"
+
+#include "dml/dcn30/display_mode_vba_30.h"
+#include "vm_helper.h"
+#include "dcn20/dcn20_vmid.h"
+
+#include "link_enc_cfg.h"
+
+#define DCN3_16_MAX_DET_SIZE 384
+#define DCN3_16_MIN_COMPBUF_SIZE_KB 128
+#define DCN3_16_CRB_SEGMENT_SIZE_KB 64
+
+enum dcn31_clk_src_array_id {
+	DCN31_CLK_SRC_PLL0,
+	DCN31_CLK_SRC_PLL1,
+	DCN31_CLK_SRC_PLL2,
+	DCN31_CLK_SRC_PLL3,
+	DCN31_CLK_SRC_PLL4,
+	DCN30_CLK_SRC_TOTAL
+};
+
+/* begin *********************
+ * macros to expend register list macro defined in HW object header file
+ */
+
+/* DCN */
+#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
+
+#define BASE(seg) BASE_INNER(seg)
+
+#define SR(reg_name)\
+		.reg_name = BASE(reg ## reg_name ## _BASE_IDX) +  \
+					reg ## reg_name
+
+#define SRI(reg_name, block, id)\
+	.reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define SRI2(reg_name, block, id)\
+	.reg_name = BASE(reg ## reg_name ## _BASE_IDX) + \
+					reg ## reg_name
+
+#define SRIR(var_name, reg_name, block, id)\
+	.var_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define SRII(reg_name, block, id)\
+	.reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define SRII_MPC_RMU(reg_name, block, id)\
+	.RMU##_##reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define SRII_DWB(reg_name, temp_name, block, id)\
+	.reg_name[id] = BASE(reg ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## temp_name
+
+#define SF_DWB2(reg_name, block, id, field_name, post_fix)	\
+	.field_name = reg_name ## __ ## field_name ## post_fix
+
+#define DCCG_SRII(reg_name, block, id)\
+	.block ## _ ## reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define VUPDATE_SRII(reg_name, block, id)\
+	.reg_name[id] = BASE(reg ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
+					reg ## reg_name ## _ ## block ## id
+
+/* NBIO */
+#define NBIO_BASE_INNER(seg) \
+	NBIO_BASE__INST0_SEG ## seg
+
+#define NBIO_BASE(seg) \
+	NBIO_BASE_INNER(seg)
+
+#define NBIO_SR(reg_name)\
+		.reg_name = NBIO_BASE(regBIF_BX1_ ## reg_name ## _BASE_IDX) + \
+					regBIF_BX1_ ## reg_name
+
+static const struct bios_registers bios_regs = {
+		NBIO_SR(BIOS_SCRATCH_3),
+		NBIO_SR(BIOS_SCRATCH_6)
+};
+
+#define clk_src_regs(index, pllid)\
+[index] = {\
+	CS_COMMON_REG_LIST_DCN3_0(index, pllid),\
+}
+
+static const struct dce110_clk_src_regs clk_src_regs[] = {
+	clk_src_regs(0, A),
+	clk_src_regs(1, B),
+	clk_src_regs(2, C),
+	clk_src_regs(3, D),
+	clk_src_regs(4, E)
+};
+
+static const struct dce110_clk_src_shift cs_shift = {
+		CS_COMMON_MASK_SH_LIST_DCN2_0(__SHIFT)
+};
+
+static const struct dce110_clk_src_mask cs_mask = {
+		CS_COMMON_MASK_SH_LIST_DCN2_0(_MASK)
+};
+
+#define abm_regs(id)\
+[id] = {\
+		ABM_DCN302_REG_LIST(id)\
+}
+
+static const struct dce_abm_registers abm_regs[] = {
+		abm_regs(0),
+		abm_regs(1),
+		abm_regs(2),
+		abm_regs(3),
+};
+
+static const struct dce_abm_shift abm_shift = {
+		ABM_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dce_abm_mask abm_mask = {
+		ABM_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define audio_regs(id)\
+[id] = {\
+		AUD_COMMON_REG_LIST(id)\
+}
+
+static const struct dce_audio_registers audio_regs[] = {
+	audio_regs(0),
+	audio_regs(1),
+	audio_regs(2),
+	audio_regs(3),
+	audio_regs(4),
+	audio_regs(5),
+	audio_regs(6)
+};
+
+#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
+		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
+
+static const struct dce_audio_shift audio_shift = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_audio_mask audio_mask = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+#define vpg_regs(id)\
+[id] = {\
+	VPG_DCN31_REG_LIST(id)\
+}
+
+static const struct dcn31_vpg_registers vpg_regs[] = {
+	vpg_regs(0),
+	vpg_regs(1),
+	vpg_regs(2),
+	vpg_regs(3),
+	vpg_regs(4),
+	vpg_regs(5),
+	vpg_regs(6),
+	vpg_regs(7),
+	vpg_regs(8),
+	vpg_regs(9),
+};
+
+static const struct dcn31_vpg_shift vpg_shift = {
+	DCN31_VPG_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_vpg_mask vpg_mask = {
+	DCN31_VPG_MASK_SH_LIST(_MASK)
+};
+
+#define afmt_regs(id)\
+[id] = {\
+	AFMT_DCN31_REG_LIST(id)\
+}
+
+static const struct dcn31_afmt_registers afmt_regs[] = {
+	afmt_regs(0),
+	afmt_regs(1),
+	afmt_regs(2),
+	afmt_regs(3),
+	afmt_regs(4),
+	afmt_regs(5)
+};
+
+static const struct dcn31_afmt_shift afmt_shift = {
+	DCN31_AFMT_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_afmt_mask afmt_mask = {
+	DCN31_AFMT_MASK_SH_LIST(_MASK)
+};
+
+
+#define apg_regs(id)\
+[id] = {\
+	APG_DCN31_REG_LIST(id)\
+}
+
+static const struct dcn31_apg_registers apg_regs[] = {
+	apg_regs(0),
+	apg_regs(1),
+	apg_regs(2),
+	apg_regs(3)
+};
+
+static const struct dcn31_apg_shift apg_shift = {
+	DCN31_APG_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_apg_mask apg_mask = {
+		DCN31_APG_MASK_SH_LIST(_MASK)
+};
+
+
+#define stream_enc_regs(id)\
+[id] = {\
+	SE_DCN3_REG_LIST(id)\
+}
+
+static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
+	stream_enc_regs(0),
+	stream_enc_regs(1),
+	stream_enc_regs(2),
+	stream_enc_regs(3),
+	stream_enc_regs(4)
+};
+
+static const struct dcn10_stream_encoder_shift se_shift = {
+		SE_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn10_stream_encoder_mask se_mask = {
+		SE_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+
+#define aux_regs(id)\
+[id] = {\
+	DCN2_AUX_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
+		aux_regs(0),
+		aux_regs(1),
+		aux_regs(2),
+		aux_regs(3),
+		aux_regs(4)
+};
+
+#define hpd_regs(id)\
+[id] = {\
+	HPD_REG_LIST(id)\
+}
+
+static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
+		hpd_regs(0),
+		hpd_regs(1),
+		hpd_regs(2),
+		hpd_regs(3),
+		hpd_regs(4)
+};
+
+#define link_regs(id, phyid)\
+[id] = {\
+	LE_DCN31_REG_LIST(id), \
+	UNIPHY_DCN2_REG_LIST(phyid), \
+	DPCS_DCN31_REG_LIST(id), \
+}
+
+static const struct dce110_aux_registers_shift aux_shift = {
+	DCN_AUX_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+	DCN_AUX_MASK_SH_LIST(_MASK)
+};
+
+static const struct dcn10_link_enc_registers link_enc_regs[] = {
+	link_regs(0, A),
+	link_regs(1, B),
+	link_regs(2, C),
+	link_regs(3, D),
+	link_regs(4, E)
+};
+
+static const struct dcn10_link_enc_shift le_shift = {
+	LINK_ENCODER_MASK_SH_LIST_DCN31(__SHIFT), \
+	DPCS_DCN31_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn10_link_enc_mask le_mask = {
+	LINK_ENCODER_MASK_SH_LIST_DCN31(_MASK), \
+	DPCS_DCN31_MASK_SH_LIST(_MASK)
+};
+
+
+
+#define hpo_dp_stream_encoder_reg_list(id)\
+[id] = {\
+	DCN3_1_HPO_DP_STREAM_ENC_REG_LIST(id)\
+}
+
+static const struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[] = {
+	hpo_dp_stream_encoder_reg_list(0),
+	hpo_dp_stream_encoder_reg_list(1),
+	hpo_dp_stream_encoder_reg_list(2),
+	hpo_dp_stream_encoder_reg_list(3),
+};
+
+static const struct dcn31_hpo_dp_stream_encoder_shift hpo_dp_se_shift = {
+	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_hpo_dp_stream_encoder_mask hpo_dp_se_mask = {
+	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(_MASK)
+};
+
+
+#define hpo_dp_link_encoder_reg_list(id)\
+[id] = {\
+	DCN3_1_HPO_DP_LINK_ENC_REG_LIST(id),\
+	DCN3_1_RDPCSTX_REG_LIST(0),\
+	DCN3_1_RDPCSTX_REG_LIST(1),\
+	DCN3_1_RDPCSTX_REG_LIST(2),\
+	DCN3_1_RDPCSTX_REG_LIST(3),\
+	DCN3_1_RDPCSTX_REG_LIST(4)\
+}
+
+static const struct dcn31_hpo_dp_link_encoder_registers hpo_dp_link_enc_regs[] = {
+	hpo_dp_link_encoder_reg_list(0),
+	hpo_dp_link_encoder_reg_list(1),
+};
+
+static const struct dcn31_hpo_dp_link_encoder_shift hpo_dp_le_shift = {
+	DCN3_1_HPO_DP_LINK_ENC_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_hpo_dp_link_encoder_mask hpo_dp_le_mask = {
+	DCN3_1_HPO_DP_LINK_ENC_MASK_SH_LIST(_MASK)
+};
+
+
+#define dpp_regs(id)\
+[id] = {\
+	DPP_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn3_dpp_registers dpp_regs[] = {
+	dpp_regs(0),
+	dpp_regs(1),
+	dpp_regs(2),
+	dpp_regs(3)
+};
+
+static const struct dcn3_dpp_shift tf_shift = {
+		DPP_REG_LIST_SH_MASK_DCN30(__SHIFT)
+};
+
+static const struct dcn3_dpp_mask tf_mask = {
+		DPP_REG_LIST_SH_MASK_DCN30(_MASK)
+};
+
+#define opp_regs(id)\
+[id] = {\
+	OPP_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn20_opp_registers opp_regs[] = {
+	opp_regs(0),
+	opp_regs(1),
+	opp_regs(2),
+	opp_regs(3)
+};
+
+static const struct dcn20_opp_shift opp_shift = {
+	OPP_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn20_opp_mask opp_mask = {
+	OPP_MASK_SH_LIST_DCN20(_MASK)
+};
+
+#define aux_engine_regs(id)\
+[id] = {\
+	AUX_COMMON_REG_LIST0(id), \
+	.AUXN_IMPCAL = 0, \
+	.AUXP_IMPCAL = 0, \
+	.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
+}
+
+static const struct dce110_aux_registers aux_engine_regs[] = {
+		aux_engine_regs(0),
+		aux_engine_regs(1),
+		aux_engine_regs(2),
+		aux_engine_regs(3),
+		aux_engine_regs(4)
+};
+
+#define dwbc_regs_dcn3(id)\
+[id] = {\
+	DWBC_COMMON_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn30_dwbc_registers dwbc30_regs[] = {
+	dwbc_regs_dcn3(0),
+};
+
+static const struct dcn30_dwbc_shift dwbc30_shift = {
+	DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_dwbc_mask dwbc30_mask = {
+	DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define mcif_wb_regs_dcn3(id)\
+[id] = {\
+	MCIF_WB_COMMON_REG_LIST_DCN30(id),\
+}
+
+static const struct dcn30_mmhubbub_registers mcif_wb30_regs[] = {
+	mcif_wb_regs_dcn3(0)
+};
+
+static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define dsc_regsDCN20(id)\
+[id] = {\
+	DSC_REG_LIST_DCN20(id)\
+}
+
+static const struct dcn20_dsc_registers dsc_regs[] = {
+	dsc_regsDCN20(0),
+	dsc_regsDCN20(1),
+	dsc_regsDCN20(2)
+};
+
+static const struct dcn20_dsc_shift dsc_shift = {
+	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
+};
+
+static const struct dcn20_dsc_mask dsc_mask = {
+	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
+};
+
+static const struct dcn30_mpc_registers mpc_regs = {
+		MPC_REG_LIST_DCN3_0(0),
+		MPC_REG_LIST_DCN3_0(1),
+		MPC_REG_LIST_DCN3_0(2),
+		MPC_REG_LIST_DCN3_0(3),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(0),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(1),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(2),
+		MPC_OUT_MUX_REG_LIST_DCN3_0(3),
+		MPC_RMU_GLOBAL_REG_LIST_DCN3AG,
+		MPC_RMU_REG_LIST_DCN3AG(0),
+		MPC_RMU_REG_LIST_DCN3AG(1),
+		//MPC_RMU_REG_LIST_DCN3AG(2),
+		MPC_DWB_MUX_REG_LIST_DCN3_0(0),
+};
+
+static const struct dcn30_mpc_shift mpc_shift = {
+	MPC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_mpc_mask mpc_mask = {
+	MPC_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define optc_regs(id)\
+[id] = {OPTC_COMMON_REG_LIST_DCN3_1(id)}
+
+static const struct dcn_optc_registers optc_regs[] = {
+	optc_regs(0),
+	optc_regs(1),
+	optc_regs(2),
+	optc_regs(3)
+};
+
+static const struct dcn_optc_shift optc_shift = {
+	OPTC_COMMON_MASK_SH_LIST_DCN3_1(__SHIFT)
+};
+
+static const struct dcn_optc_mask optc_mask = {
+	OPTC_COMMON_MASK_SH_LIST_DCN3_1(_MASK)
+};
+
+#define hubp_regs(id)\
+[id] = {\
+	HUBP_REG_LIST_DCN30(id)\
+}
+
+static const struct dcn_hubp2_registers hubp_regs[] = {
+		hubp_regs(0),
+		hubp_regs(1),
+		hubp_regs(2),
+		hubp_regs(3)
+};
+
+
+static const struct dcn_hubp2_shift hubp_shift = {
+		HUBP_MASK_SH_LIST_DCN31(__SHIFT)
+};
+
+static const struct dcn_hubp2_mask hubp_mask = {
+		HUBP_MASK_SH_LIST_DCN31(_MASK)
+};
+static const struct dcn_hubbub_registers hubbub_reg = {
+		HUBBUB_REG_LIST_DCN31(0)
+};
+
+static const struct dcn_hubbub_shift hubbub_shift = {
+		HUBBUB_MASK_SH_LIST_DCN31(__SHIFT)
+};
+
+static const struct dcn_hubbub_mask hubbub_mask = {
+		HUBBUB_MASK_SH_LIST_DCN31(_MASK)
+};
+
+static const struct dccg_registers dccg_regs = {
+		DCCG_REG_LIST_DCN31()
+};
+
+static const struct dccg_shift dccg_shift = {
+		DCCG_MASK_SH_LIST_DCN31(__SHIFT)
+};
+
+static const struct dccg_mask dccg_mask = {
+		DCCG_MASK_SH_LIST_DCN31(_MASK)
+};
+
+
+#define SRII2(reg_name_pre, reg_name_post, id)\
+	.reg_name_pre ## _ ##  reg_name_post[id] = BASE(reg ## reg_name_pre \
+			## id ## _ ## reg_name_post ## _BASE_IDX) + \
+			reg ## reg_name_pre ## id ## _ ## reg_name_post
+
+
+#define HWSEQ_DCN31_REG_LIST()\
+	SR(DCHUBBUB_GLOBAL_TIMER_CNTL), \
+	SR(DCHUBBUB_ARB_HOSTVM_CNTL), \
+	SR(DIO_MEM_PWR_CTRL), \
+	SR(ODM_MEM_PWR_CTRL3), \
+	SR(DMU_MEM_PWR_CNTL), \
+	SR(MMHUBBUB_MEM_PWR_CNTL), \
+	SR(DCCG_GATE_DISABLE_CNTL), \
+	SR(DCCG_GATE_DISABLE_CNTL2), \
+	SR(DCFCLK_CNTL),\
+	SR(DC_MEM_GLOBAL_PWR_REQ_CNTL), \
+	SRII(PIXEL_RATE_CNTL, OTG, 0), \
+	SRII(PIXEL_RATE_CNTL, OTG, 1),\
+	SRII(PIXEL_RATE_CNTL, OTG, 2),\
+	SRII(PIXEL_RATE_CNTL, OTG, 3),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 0),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 1),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 2),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 3),\
+	SR(MICROSECOND_TIME_BASE_DIV), \
+	SR(MILLISECOND_TIME_BASE_DIV), \
+	SR(DISPCLK_FREQ_CHANGE_CNTL), \
+	SR(RBBMIF_TIMEOUT_DIS), \
+	SR(RBBMIF_TIMEOUT_DIS_2), \
+	SR(DCHUBBUB_CRC_CTRL), \
+	SR(DPP_TOP0_DPP_CRC_CTRL), \
+	SR(DPP_TOP0_DPP_CRC_VAL_B_A), \
+	SR(DPP_TOP0_DPP_CRC_VAL_R_G), \
+	SR(MPC_CRC_CTRL), \
+	SR(MPC_CRC_RESULT_GB), \
+	SR(MPC_CRC_RESULT_C), \
+	SR(MPC_CRC_RESULT_AR), \
+	SR(DOMAIN0_PG_CONFIG), \
+	SR(DOMAIN1_PG_CONFIG), \
+	SR(DOMAIN2_PG_CONFIG), \
+	SR(DOMAIN3_PG_CONFIG), \
+	SR(DOMAIN16_PG_CONFIG), \
+	SR(DOMAIN17_PG_CONFIG), \
+	SR(DOMAIN18_PG_CONFIG), \
+	SR(DOMAIN0_PG_STATUS), \
+	SR(DOMAIN1_PG_STATUS), \
+	SR(DOMAIN2_PG_STATUS), \
+	SR(DOMAIN3_PG_STATUS), \
+	SR(DOMAIN16_PG_STATUS), \
+	SR(DOMAIN17_PG_STATUS), \
+	SR(DOMAIN18_PG_STATUS), \
+	SR(D1VGA_CONTROL), \
+	SR(D2VGA_CONTROL), \
+	SR(D3VGA_CONTROL), \
+	SR(D4VGA_CONTROL), \
+	SR(D5VGA_CONTROL), \
+	SR(D6VGA_CONTROL), \
+	SR(DC_IP_REQUEST_CNTL), \
+	SR(AZALIA_AUDIO_DTO), \
+	SR(AZALIA_CONTROLLER_CLOCK_GATING), \
+	SR(HPO_TOP_HW_CONTROL)
+
+static const struct dce_hwseq_registers hwseq_reg = {
+		HWSEQ_DCN31_REG_LIST()
+};
+
+#define HWSEQ_DCN31_MASK_SH_LIST(mask_sh)\
+	HWSEQ_DCN_MASK_SH_LIST(mask_sh), \
+	HWS_SF(, DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, mask_sh), \
+	HWS_SF(, DCHUBBUB_ARB_HOSTVM_CNTL, DISABLE_HOSTVM_FORCE_ALLOW_PSTATE, mask_sh), \
+	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN0_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN1_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN2_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN3_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN16_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN17_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN18_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh), \
+	HWS_SF(, AZALIA_AUDIO_DTO, AZALIA_AUDIO_DTO_MODULE, mask_sh), \
+	HWS_SF(, HPO_TOP_CLOCK_CONTROL, HPO_HDMISTREAMCLK_G_GATE_DIS, mask_sh), \
+	HWS_SF(, DMU_MEM_PWR_CNTL, DMCU_ERAM_MEM_PWR_FORCE, mask_sh), \
+	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_UNASSIGNED_PWR_MODE, mask_sh), \
+	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_VBLANK_PWR_MODE, mask_sh), \
+	HWS_SF(, MMHUBBUB_MEM_PWR_CNTL, VGA_MEM_PWR_FORCE, mask_sh), \
+	HWS_SF(, DIO_MEM_PWR_CTRL, I2C_LIGHT_SLEEP_FORCE, mask_sh), \
+	HWS_SF(, HPO_TOP_HW_CONTROL, HPO_IO_EN, mask_sh)
+
+static const struct dce_hwseq_shift hwseq_shift = {
+		HWSEQ_DCN31_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+		HWSEQ_DCN31_MASK_SH_LIST(_MASK)
+};
+#define vmid_regs(id)\
+[id] = {\
+		DCN20_VMID_REG_LIST(id)\
+}
+
+static const struct dcn_vmid_registers vmid_regs[] = {
+	vmid_regs(0),
+	vmid_regs(1),
+	vmid_regs(2),
+	vmid_regs(3),
+	vmid_regs(4),
+	vmid_regs(5),
+	vmid_regs(6),
+	vmid_regs(7),
+	vmid_regs(8),
+	vmid_regs(9),
+	vmid_regs(10),
+	vmid_regs(11),
+	vmid_regs(12),
+	vmid_regs(13),
+	vmid_regs(14),
+	vmid_regs(15)
+};
+
+static const struct dcn20_vmid_shift vmid_shifts = {
+		DCN20_VMID_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn20_vmid_mask vmid_masks = {
+		DCN20_VMID_MASK_SH_LIST(_MASK)
+};
+
+static const struct resource_caps res_cap_dcn31 = {
+	.num_timing_generator = 4,
+	.num_opp = 4,
+	.num_video_plane = 4,
+	.num_audio = 5,
+	.num_stream_encoder = 5,
+	.num_dig_link_enc = 5,
+	.num_hpo_dp_stream_encoder = 4,
+	.num_hpo_dp_link_encoder = 2,
+	.num_pll = 5,
+	.num_dwb = 1,
+	.num_ddc = 5,
+	.num_vmid = 16,
+	.num_mpc_3dlut = 2,
+	.num_dsc = 3,
+};
+
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
+	.per_pixel_alpha = true,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = true,
+			.fp16 = true,
+			.p010 = true,
+			.ayuv = false,
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 16000,
+			.fp16 = 16000
+	},
+
+	// 6:1 downscaling ratio: 1000/6 = 166.666
+	.max_downscale_factor = {
+			.argb8888 = 167,
+			.nv12 = 167,
+			.fp16 = 167
+	},
+	64,
+	64
+};
+
+static const struct dc_debug_options debug_defaults_drv = {
+	.disable_z10 = true, /*hw not support it*/
+	.disable_dmcu = true,
+	.force_abm_enable = false,
+	.timing_trace = false,
+	.clock_trace = true,
+	.disable_pplib_clock_request = false,
+	.pipe_split_policy = MPC_SPLIT_DYNAMIC,
+	.force_single_disp_pipe_split = false,
+	.disable_dcc = DCC_ENABLE,
+	.vsr_support = true,
+	.performance_trace = false,
+	.max_downscale_src_width = 4096,/*upto true 4k*/
+	.disable_pplib_wm_range = false,
+	.scl_reset_length10 = true,
+	.sanity_checks = false,
+	.underflow_assert_delay_us = 0xFFFFFFFF,
+	.dwb_fi_phase = -1, // -1 = disable,
+	.dmub_command_table = true,
+	.pstate_enabled = true,
+	.use_max_lb = true,
+	.enable_mem_low_power = {
+		.bits = {
+			.vga = true,
+			.i2c = true,
+			.dmcu = false, // This is previously known to cause hang on S3 cycles if enabled
+			.dscl = true,
+			.cm = true,
+			.mpc = true,
+			.optc = true,
+			.vpg = true,
+			.afmt = true,
+		}
+	},
+	.enable_legacy_fast_update = true,
+	.using_dml2 = false,
+};
+
+static const struct dc_panel_config panel_config_defaults = {
+	.psr = {
+		.disable_psr = false,
+		.disallow_psrsu = false,
+		.disallow_replay = false,
+	},
+	.ilr = {
+		.optimize_edp_link_rate = true,
+	},
+};
+
+static void dcn31_dpp_destroy(struct dpp **dpp)
+{
+	kfree(TO_DCN20_DPP(*dpp));
+	*dpp = NULL;
+}
+
+static struct dpp *dcn31_dpp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn3_dpp *dpp =
+		kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
+
+	if (!dpp)
+		return NULL;
+
+	if (dpp3_construct(dpp, ctx, inst,
+			&dpp_regs[inst], &tf_shift, &tf_mask))
+		return &dpp->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(dpp);
+	return NULL;
+}
+
+static struct output_pixel_processor *dcn31_opp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_opp *opp =
+		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
+
+	if (!opp) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dcn20_opp_construct(opp, ctx, inst,
+			&opp_regs[inst], &opp_shift, &opp_mask);
+	return &opp->base;
+}
+
+static struct dce_aux *dcn31_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst],
+					&aux_mask,
+					&aux_shift,
+					ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST_DCN30(id) }
+
+static const struct dce_i2c_registers i2c_hw_regs[] = {
+		i2c_inst_regs(1),
+		i2c_inst_regs(2),
+		i2c_inst_regs(3),
+		i2c_inst_regs(4),
+		i2c_inst_regs(5),
+};
+
+static const struct dce_i2c_shift i2c_shifts = {
+		I2C_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dce_i2c_mask i2c_masks = {
+		I2C_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static struct dce_i2c_hw *dcn31_i2c_hw_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw =
+		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
+				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+static struct mpc *dcn31_mpc_create(
+		struct dc_context *ctx,
+		int num_mpcc,
+		int num_rmu)
+{
+	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc),
+					  GFP_KERNEL);
+
+	if (!mpc30)
+		return NULL;
+
+	dcn30_mpc_construct(mpc30, ctx,
+			&mpc_regs,
+			&mpc_shift,
+			&mpc_mask,
+			num_mpcc,
+			num_rmu);
+
+	return &mpc30->base;
+}
+
+static struct hubbub *dcn31_hubbub_create(struct dc_context *ctx)
+{
+	int i;
+
+	struct dcn20_hubbub *hubbub3 = kzalloc(sizeof(struct dcn20_hubbub),
+					  GFP_KERNEL);
+
+	if (!hubbub3)
+		return NULL;
+
+	hubbub31_construct(hubbub3, ctx,
+			&hubbub_reg,
+			&hubbub_shift,
+			&hubbub_mask,
+			dcn3_16_ip.det_buffer_size_kbytes,
+			dcn3_16_ip.pixel_chunk_size_kbytes,
+			dcn3_16_ip.config_return_buffer_size_in_kbytes);
+
+
+	for (i = 0; i < res_cap_dcn31.num_vmid; i++) {
+		struct dcn20_vmid *vmid = &hubbub3->vmid[i];
+
+		vmid->ctx = ctx;
+
+		vmid->regs = &vmid_regs[i];
+		vmid->shifts = &vmid_shifts;
+		vmid->masks = &vmid_masks;
+	}
+
+	return &hubbub3->base;
+}
+
+static struct timing_generator *dcn31_timing_generator_create(
+		struct dc_context *ctx,
+		uint32_t instance)
+{
+	struct optc *tgn10 =
+		kzalloc(sizeof(struct optc), GFP_KERNEL);
+
+	if (!tgn10)
+		return NULL;
+
+	tgn10->base.inst = instance;
+	tgn10->base.ctx = ctx;
+
+	tgn10->tg_regs = &optc_regs[instance];
+	tgn10->tg_shift = &optc_shift;
+	tgn10->tg_mask = &optc_mask;
+
+	dcn31_timing_generator_init(tgn10);
+
+	return &tgn10->base;
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+		.max_hdmi_deep_color = COLOR_DEPTH_121212,
+		.max_hdmi_pixel_clock = 600000,
+		.hdmi_ycbcr420_supported = true,
+		.dp_ycbcr420_supported = true,
+		.fec_supported = true,
+		.flags.bits.IS_HBR2_CAPABLE = true,
+		.flags.bits.IS_HBR3_CAPABLE = true,
+		.flags.bits.IS_TPS3_CAPABLE = true,
+		.flags.bits.IS_TPS4_CAPABLE = true
+};
+
+static struct link_encoder *dcn31_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dcn20_link_encoder *enc20 =
+		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+
+	if (!enc20)
+		return NULL;
+
+	dcn31_link_encoder_construct(enc20,
+			enc_init_data,
+			&link_enc_feature,
+			&link_enc_regs[enc_init_data->transmitter],
+			&link_enc_aux_regs[enc_init_data->channel - 1],
+			&link_enc_hpd_regs[enc_init_data->hpd_source],
+			&le_shift,
+			&le_mask);
+
+	return &enc20->enc10.base;
+}
+
+/* Create a minimal link encoder object not associated with a particular
+ * physical connector.
+ * resource_funcs.link_enc_create_minimal
+ */
+static struct link_encoder *dcn31_link_enc_create_minimal(
+		struct dc_context *ctx, enum engine_id eng_id)
+{
+	struct dcn20_link_encoder *enc20;
+
+	if ((eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+		return NULL;
+
+	enc20 = kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+	if (!enc20)
+		return NULL;
+
+	dcn31_link_encoder_construct_minimal(
+			enc20,
+			ctx,
+			&link_enc_feature,
+			&link_enc_regs[eng_id - ENGINE_ID_DIGA],
+			eng_id);
+
+	return &enc20->enc10.base;
+}
+
+static struct panel_cntl *dcn31_panel_cntl_create(const struct panel_cntl_init_data *init_data)
+{
+	struct dcn31_panel_cntl *panel_cntl =
+		kzalloc(sizeof(struct dcn31_panel_cntl), GFP_KERNEL);
+
+	if (!panel_cntl)
+		return NULL;
+
+	dcn31_panel_cntl_construct(panel_cntl, init_data);
+
+	return &panel_cntl->base;
+}
+
+static void read_dce_straps(
+	struct dc_context *ctx,
+	struct resource_straps *straps)
+{
+	generic_reg_get(ctx, regDC_PINSTRAPS + BASE(regDC_PINSTRAPS_BASE_IDX),
+		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
+
+}
+
+static struct audio *dcn31_create_audio(
+		struct dc_context *ctx, unsigned int inst)
+{
+	return dce_audio_create(ctx, inst,
+			&audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+static struct vpg *dcn31_vpg_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn31_vpg *vpg31 = kzalloc(sizeof(struct dcn31_vpg), GFP_KERNEL);
+
+	if (!vpg31)
+		return NULL;
+
+	vpg31_construct(vpg31, ctx, inst,
+			&vpg_regs[inst],
+			&vpg_shift,
+			&vpg_mask);
+
+	return &vpg31->base;
+}
+
+static struct afmt *dcn31_afmt_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn31_afmt *afmt31 = kzalloc(sizeof(struct dcn31_afmt), GFP_KERNEL);
+
+	if (!afmt31)
+		return NULL;
+
+	afmt31_construct(afmt31, ctx, inst,
+			&afmt_regs[inst],
+			&afmt_shift,
+			&afmt_mask);
+
+	// Light sleep by default, no need to power down here
+
+	return &afmt31->base;
+}
+
+
+static struct apg *dcn31_apg_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn31_apg *apg31 = kzalloc(sizeof(struct dcn31_apg), GFP_KERNEL);
+
+	if (!apg31)
+		return NULL;
+
+	apg31_construct(apg31, ctx, inst,
+			&apg_regs[inst],
+			&apg_shift,
+			&apg_mask);
+
+	return &apg31->base;
+}
+
+
+static struct stream_encoder *dcn316_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dcn10_stream_encoder *enc1;
+	struct vpg *vpg;
+	struct afmt *afmt;
+	int vpg_inst;
+	int afmt_inst;
+
+	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
+	if (eng_id <= ENGINE_ID_DIGF) {
+		vpg_inst = eng_id;
+		afmt_inst = eng_id;
+	} else
+		return NULL;
+
+	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
+	vpg = dcn31_vpg_create(ctx, vpg_inst);
+	afmt = dcn31_afmt_create(ctx, afmt_inst);
+
+	if (!enc1 || !vpg || !afmt) {
+		kfree(enc1);
+		kfree(vpg);
+		kfree(afmt);
+		return NULL;
+	}
+
+	dcn30_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
+					eng_id, vpg, afmt,
+					&stream_enc_regs[eng_id],
+					&se_shift, &se_mask);
+
+	return &enc1->base;
+}
+
+
+static struct hpo_dp_stream_encoder *dcn31_hpo_dp_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dcn31_hpo_dp_stream_encoder *hpo_dp_enc31;
+	struct vpg *vpg;
+	struct apg *apg;
+	uint32_t hpo_dp_inst;
+	uint32_t vpg_inst;
+	uint32_t apg_inst;
+
+	ASSERT((eng_id >= ENGINE_ID_HPO_DP_0) && (eng_id <= ENGINE_ID_HPO_DP_3));
+	hpo_dp_inst = eng_id - ENGINE_ID_HPO_DP_0;
+
+	/* Mapping of VPG register blocks to HPO DP block instance:
+	 * VPG[6] -> HPO_DP[0]
+	 * VPG[7] -> HPO_DP[1]
+	 * VPG[8] -> HPO_DP[2]
+	 * VPG[9] -> HPO_DP[3]
+	 */
+	vpg_inst = hpo_dp_inst + 6;
+
+	/* Mapping of APG register blocks to HPO DP block instance:
+	 * APG[0] -> HPO_DP[0]
+	 * APG[1] -> HPO_DP[1]
+	 * APG[2] -> HPO_DP[2]
+	 * APG[3] -> HPO_DP[3]
+	 */
+	apg_inst = hpo_dp_inst;
+
+	/* allocate HPO stream encoder and create VPG sub-block */
+	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_stream_encoder), GFP_KERNEL);
+	vpg = dcn31_vpg_create(ctx, vpg_inst);
+	apg = dcn31_apg_create(ctx, apg_inst);
+
+	if (!hpo_dp_enc31 || !vpg || !apg) {
+		kfree(hpo_dp_enc31);
+		kfree(vpg);
+		kfree(apg);
+		return NULL;
+	}
+
+	dcn31_hpo_dp_stream_encoder_construct(hpo_dp_enc31, ctx, ctx->dc_bios,
+					hpo_dp_inst, eng_id, vpg, apg,
+					&hpo_dp_stream_enc_regs[hpo_dp_inst],
+					&hpo_dp_se_shift, &hpo_dp_se_mask);
+
+	return &hpo_dp_enc31->base;
+}
+
+static struct hpo_dp_link_encoder *dcn31_hpo_dp_link_encoder_create(
+	uint8_t inst,
+	struct dc_context *ctx)
+{
+	struct dcn31_hpo_dp_link_encoder *hpo_dp_enc31;
+
+	/* allocate HPO link encoder */
+	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
+
+	hpo_dp_link_encoder31_construct(hpo_dp_enc31, ctx, inst,
+					&hpo_dp_link_enc_regs[inst],
+					&hpo_dp_le_shift, &hpo_dp_le_mask);
+
+	return &hpo_dp_enc31->base;
+}
+
+
+static struct dce_hwseq *dcn31_hwseq_create(
+	struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &hwseq_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+	}
+	return hws;
+}
+static const struct resource_create_funcs res_create_funcs = {
+	.read_dce_straps = read_dce_straps,
+	.create_audio = dcn31_create_audio,
+	.create_stream_encoder = dcn316_stream_encoder_create,
+	.create_hpo_dp_stream_encoder = dcn31_hpo_dp_stream_encoder_create,
+	.create_hpo_dp_link_encoder = dcn31_hpo_dp_link_encoder_create,
+	.create_hwseq = dcn31_hwseq_create,
+};
+
+static void dcn316_resource_destruct(struct dcn316_resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->base.stream_enc_count; i++) {
+		if (pool->base.stream_enc[i] != NULL) {
+			if (pool->base.stream_enc[i]->vpg != NULL) {
+				kfree(DCN30_VPG_FROM_VPG(pool->base.stream_enc[i]->vpg));
+				pool->base.stream_enc[i]->vpg = NULL;
+			}
+			if (pool->base.stream_enc[i]->afmt != NULL) {
+				kfree(DCN30_AFMT_FROM_AFMT(pool->base.stream_enc[i]->afmt));
+				pool->base.stream_enc[i]->afmt = NULL;
+			}
+			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
+			pool->base.stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.hpo_dp_stream_enc_count; i++) {
+		if (pool->base.hpo_dp_stream_enc[i] != NULL) {
+			if (pool->base.hpo_dp_stream_enc[i]->vpg != NULL) {
+				kfree(DCN30_VPG_FROM_VPG(pool->base.hpo_dp_stream_enc[i]->vpg));
+				pool->base.hpo_dp_stream_enc[i]->vpg = NULL;
+			}
+			if (pool->base.hpo_dp_stream_enc[i]->apg != NULL) {
+				kfree(DCN31_APG_FROM_APG(pool->base.hpo_dp_stream_enc[i]->apg));
+				pool->base.hpo_dp_stream_enc[i]->apg = NULL;
+			}
+			kfree(DCN3_1_HPO_DP_STREAM_ENC_FROM_HPO_STREAM_ENC(pool->base.hpo_dp_stream_enc[i]));
+			pool->base.hpo_dp_stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.hpo_dp_link_enc_count; i++) {
+		if (pool->base.hpo_dp_link_enc[i] != NULL) {
+			kfree(DCN3_1_HPO_DP_LINK_ENC_FROM_HPO_LINK_ENC(pool->base.hpo_dp_link_enc[i]));
+			pool->base.hpo_dp_link_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		if (pool->base.dscs[i] != NULL)
+			dcn20_dsc_destroy(&pool->base.dscs[i]);
+	}
+
+	if (pool->base.mpc != NULL) {
+		kfree(TO_DCN20_MPC(pool->base.mpc));
+		pool->base.mpc = NULL;
+	}
+	if (pool->base.hubbub != NULL) {
+		kfree(pool->base.hubbub);
+		pool->base.hubbub = NULL;
+	}
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (pool->base.dpps[i] != NULL)
+			dcn31_dpp_destroy(&pool->base.dpps[i]);
+
+		if (pool->base.ipps[i] != NULL)
+			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
+
+		if (pool->base.hubps[i] != NULL) {
+			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
+			pool->base.hubps[i] = NULL;
+		}
+
+		if (pool->base.irqs != NULL) {
+			dal_irq_service_destroy(&pool->base.irqs);
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+		if (pool->base.hw_i2cs[i] != NULL) {
+			kfree(pool->base.hw_i2cs[i]);
+			pool->base.hw_i2cs[i] = NULL;
+		}
+		if (pool->base.sw_i2cs[i] != NULL) {
+			kfree(pool->base.sw_i2cs[i]);
+			pool->base.sw_i2cs[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		if (pool->base.opps[i] != NULL)
+			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.timing_generators[i] != NULL)	{
+			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
+			pool->base.timing_generators[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+		if (pool->base.dwbc[i] != NULL) {
+			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
+			pool->base.dwbc[i] = NULL;
+		}
+		if (pool->base.mcif_wb[i] != NULL) {
+			kfree(TO_DCN30_MMHUBBUB(pool->base.mcif_wb[i]));
+			pool->base.mcif_wb[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.audio_count; i++) {
+		if (pool->base.audios[i])
+			dce_aud_destroy(&pool->base.audios[i]);
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] != NULL) {
+			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
+			pool->base.clock_sources[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+		if (pool->base.mpc_lut[i] != NULL) {
+			dc_3dlut_func_release(pool->base.mpc_lut[i]);
+			pool->base.mpc_lut[i] = NULL;
+		}
+		if (pool->base.mpc_shaper[i] != NULL) {
+			dc_transfer_func_release(pool->base.mpc_shaper[i]);
+			pool->base.mpc_shaper[i] = NULL;
+		}
+	}
+
+	if (pool->base.dp_clock_source != NULL) {
+		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
+		pool->base.dp_clock_source = NULL;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.multiple_abms[i] != NULL)
+			dce_abm_destroy(&pool->base.multiple_abms[i]);
+	}
+
+	if (pool->base.psr != NULL)
+		dmub_psr_destroy(&pool->base.psr);
+
+	if (pool->base.dccg != NULL)
+		dcn_dccg_destroy(&pool->base.dccg);
+}
+
+static struct hubp *dcn31_hubp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn20_hubp *hubp2 =
+		kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
+
+	if (!hubp2)
+		return NULL;
+
+	if (hubp31_construct(hubp2, ctx, inst,
+			&hubp_regs[inst], &hubp_shift, &hubp_mask))
+		return &hubp2->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(hubp2);
+	return NULL;
+}
+
+static bool dcn31_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc),
+						    GFP_KERNEL);
+
+		if (!dwbc30) {
+			dm_error("DC: failed to create dwbc30!\n");
+			return false;
+		}
+
+		dcn30_dwbc_construct(dwbc30, ctx,
+				&dwbc30_regs[i],
+				&dwbc30_shift,
+				&dwbc30_mask,
+				i);
+
+		pool->dwbc[i] = &dwbc30->base;
+	}
+	return true;
+}
+
+static bool dcn31_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub),
+						    GFP_KERNEL);
+
+		if (!mcif_wb30) {
+			dm_error("DC: failed to create mcif_wb30!\n");
+			return false;
+		}
+
+		dcn30_mmhubbub_construct(mcif_wb30, ctx,
+				&mcif_wb30_regs[i],
+				&mcif_wb30_shift,
+				&mcif_wb30_mask,
+				i);
+
+		pool->mcif_wb[i] = &mcif_wb30->base;
+	}
+	return true;
+}
+
+static struct display_stream_compressor *dcn31_dsc_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_dsc *dsc =
+		kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
+
+	if (!dsc) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
+	return &dsc->base;
+}
+
+static void dcn316_destroy_resource_pool(struct resource_pool **pool)
+{
+	struct dcn316_resource_pool *dcn31_pool = TO_DCN316_RES_POOL(*pool);
+
+	dcn316_resource_destruct(dcn31_pool);
+	kfree(dcn31_pool);
+	*pool = NULL;
+}
+
+static struct clock_source *dcn31_clock_source_create(
+		struct dc_context *ctx,
+		struct dc_bios *bios,
+		enum clock_source_id id,
+		const struct dce110_clk_src_regs *regs,
+		bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dcn31_clk_src_construct(clk_src, ctx, bios, id,
+			regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	kfree(clk_src);
+
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static bool is_dual_plane(enum surface_pixel_format format)
+{
+	return format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN || format == SURFACE_PIXEL_FORMAT_GRPH_RGBE_ALPHA;
+}
+
+static int dcn316_populate_dml_pipes_from_context(
+	struct dc *dc, struct dc_state *context,
+	display_e2e_pipe_params_st *pipes,
+	bool fast_validate)
+{
+	int i, pipe_cnt;
+	struct resource_context *res_ctx = &context->res_ctx;
+	struct pipe_ctx *pipe;
+	const int max_usable_det = context->bw_ctx.dml.ip.config_return_buffer_size_in_kbytes - DCN3_16_MIN_COMPBUF_SIZE_KB;
+
+	DC_FP_START();
+	dcn31x_populate_dml_pipes_from_context(dc, context, pipes, fast_validate);
+	DC_FP_END();
+
+	for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) {
+		struct dc_crtc_timing *timing;
+
+		if (!res_ctx->pipe_ctx[i].stream)
+			continue;
+		pipe = &res_ctx->pipe_ctx[i];
+		timing = &pipe->stream->timing;
+
+		/*
+		 * Immediate flip can be set dynamically after enabling the plane.
+		 * We need to require support for immediate flip or underflow can be
+		 * intermittently experienced depending on peak b/w requirements.
+		 */
+		pipes[pipe_cnt].pipe.src.immediate_flip = true;
+
+		pipes[pipe_cnt].pipe.src.unbounded_req_mode = false;
+		pipes[pipe_cnt].pipe.dest.vfront_porch = timing->v_front_porch;
+		pipes[pipe_cnt].pipe.src.dcc_rate = 3;
+		pipes[pipe_cnt].dout.dsc_input_bpc = 0;
+		DC_FP_START();
+		dcn31_zero_pipe_dcc_fraction(pipes, pipe_cnt);
+		DC_FP_END();
+
+		if (pipes[pipe_cnt].dout.dsc_enable) {
+			switch (timing->display_color_depth) {
+			case COLOR_DEPTH_888:
+				pipes[pipe_cnt].dout.dsc_input_bpc = 8;
+				break;
+			case COLOR_DEPTH_101010:
+				pipes[pipe_cnt].dout.dsc_input_bpc = 10;
+				break;
+			case COLOR_DEPTH_121212:
+				pipes[pipe_cnt].dout.dsc_input_bpc = 12;
+				break;
+			default:
+				ASSERT(0);
+				break;
+			}
+		}
+
+		pipe_cnt++;
+	}
+
+	if (pipe_cnt)
+		context->bw_ctx.dml.ip.det_buffer_size_kbytes =
+				(max_usable_det / DCN3_16_CRB_SEGMENT_SIZE_KB / pipe_cnt) * DCN3_16_CRB_SEGMENT_SIZE_KB;
+	if (context->bw_ctx.dml.ip.det_buffer_size_kbytes > DCN3_16_MAX_DET_SIZE)
+		context->bw_ctx.dml.ip.det_buffer_size_kbytes = DCN3_16_MAX_DET_SIZE;
+	ASSERT(context->bw_ctx.dml.ip.det_buffer_size_kbytes >= DCN3_16_DEFAULT_DET_SIZE);
+	dc->config.enable_4to1MPC = false;
+	if (pipe_cnt == 1 && pipe->plane_state && !dc->debug.disable_z9_mpc) {
+		if (is_dual_plane(pipe->plane_state->format)
+				&& pipe->plane_state->src_rect.width <= 1920 && pipe->plane_state->src_rect.height <= 1080) {
+			dc->config.enable_4to1MPC = true;
+			context->bw_ctx.dml.ip.det_buffer_size_kbytes =
+					(max_usable_det / DCN3_16_CRB_SEGMENT_SIZE_KB / 4) * DCN3_16_CRB_SEGMENT_SIZE_KB;
+		} else if (!is_dual_plane(pipe->plane_state->format)) {
+			context->bw_ctx.dml.ip.det_buffer_size_kbytes = 192;
+			pipes[0].pipe.src.unbounded_req_mode = true;
+		}
+	}
+
+	return pipe_cnt;
+}
+
+static void dcn316_get_panel_config_defaults(struct dc_panel_config *panel_config)
+{
+	*panel_config = panel_config_defaults;
+}
+
+static struct dc_cap_funcs cap_funcs = {
+	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
+};
+
+static struct resource_funcs dcn316_res_pool_funcs = {
+	.destroy = dcn316_destroy_resource_pool,
+	.link_enc_create = dcn31_link_encoder_create,
+	.link_enc_create_minimal = dcn31_link_enc_create_minimal,
+	.link_encs_assign = link_enc_cfg_link_encs_assign,
+	.link_enc_unassign = link_enc_cfg_link_enc_unassign,
+	.panel_cntl_create = dcn31_panel_cntl_create,
+	.validate_bandwidth = dcn31_validate_bandwidth,
+	.calculate_wm_and_dlg = dcn31_calculate_wm_and_dlg,
+	.update_soc_for_wm_a = dcn31_update_soc_for_wm_a,
+	.populate_dml_pipes = dcn316_populate_dml_pipes_from_context,
+	.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
+	.release_pipe = dcn20_release_pipe,
+	.add_stream_to_ctx = dcn30_add_stream_to_ctx,
+	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
+	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
+	.populate_dml_writeback_from_context = dcn31_populate_dml_writeback_from_context,
+	.set_mcif_arb_params = dcn31_set_mcif_arb_params,
+	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
+	.acquire_post_bldn_3dlut = dcn30_acquire_post_bldn_3dlut,
+	.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
+	.update_bw_bounding_box = dcn316_update_bw_bounding_box,
+	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
+	.get_panel_config_defaults = dcn316_get_panel_config_defaults,
+};
+
+static bool dcn316_resource_construct(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct dcn316_resource_pool *pool)
+{
+	int i;
+	struct dc_context *ctx = dc->ctx;
+	struct irq_service_init_data init_data;
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	pool->base.res_cap = &res_cap_dcn31;
+
+	pool->base.funcs = &dcn316_res_pool_funcs;
+
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
+	pool->base.mpcc_count = pool->base.res_cap->num_timing_generator;
+	dc->caps.max_downscale_ratio = 600;
+	dc->caps.i2c_speed_in_khz = 100;
+	dc->caps.i2c_speed_in_khz_hdcp = 5; /*1.5 w/a applied by default*/
+	dc->caps.max_cursor_size = 256;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.dmdata_alloc_size = 2048;
+	dc->caps.max_slave_planes = 2;
+	dc->caps.max_slave_yuv_planes = 2;
+	dc->caps.max_slave_rgb_planes = 2;
+	dc->caps.post_blend_color_processing = true;
+	dc->caps.force_dp_tps4_for_cp2520 = true;
+	if (dc->config.forceHBR2CP2520)
+		dc->caps.force_dp_tps4_for_cp2520 = false;
+	dc->caps.dp_hpo = true;
+	dc->caps.dp_hdmi21_pcon_support = true;
+	dc->caps.edp_dsc_support = true;
+	dc->caps.extended_aux_timeout_support = true;
+	dc->caps.dmcub_support = true;
+	dc->caps.is_apu = true;
+
+	/* Color pipeline capabilities */
+	dc->caps.color.dpp.dcn_arch = 1;
+	dc->caps.color.dpp.input_lut_shared = 0;
+	dc->caps.color.dpp.icsc = 1;
+	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
+	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
+	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
+	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
+	dc->caps.color.dpp.post_csc = 1;
+	dc->caps.color.dpp.gamma_corr = 1;
+	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
+
+	dc->caps.color.dpp.hw_3d_lut = 1;
+	dc->caps.color.dpp.ogam_ram = 1;
+	// no OGAM ROM on DCN301
+	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
+	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
+	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
+	dc->caps.color.dpp.ocsc = 0;
+
+	dc->caps.color.mpc.gamut_remap = 1;
+	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //2
+	dc->caps.color.mpc.ogam_ram = 1;
+	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
+	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
+	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
+	dc->caps.color.mpc.ocsc = 1;
+
+	/* read VBIOS LTTPR caps */
+	{
+		if (ctx->dc_bios->funcs->get_lttpr_caps) {
+			enum bp_result bp_query_result;
+			uint8_t is_vbios_lttpr_enable = 0;
+
+			bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
+			dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
+		}
+
+		/* interop bit is implicit */
+		{
+			dc->caps.vbios_lttpr_aware = true;
+		}
+	}
+
+	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
+		dc->debug = debug_defaults_drv;
+
+	// Init the vm_helper
+	if (dc->vm_helper)
+		vm_helper_init(dc->vm_helper, 16);
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	/* Clock Sources for Pixel Clock*/
+	pool->base.clock_sources[DCN31_CLK_SRC_PLL0] =
+			dcn31_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL0,
+				&clk_src_regs[0], false);
+	pool->base.clock_sources[DCN31_CLK_SRC_PLL1] =
+			dcn31_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL1,
+				&clk_src_regs[1], false);
+	pool->base.clock_sources[DCN31_CLK_SRC_PLL2] =
+			dcn31_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL2,
+				&clk_src_regs[2], false);
+	pool->base.clock_sources[DCN31_CLK_SRC_PLL3] =
+			dcn31_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL3,
+				&clk_src_regs[3], false);
+	pool->base.clock_sources[DCN31_CLK_SRC_PLL4] =
+			dcn31_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL4,
+				&clk_src_regs[4], false);
+
+	pool->base.clk_src_count = DCN30_CLK_SRC_TOTAL;
+
+	/* todo: not reuse phy_pll registers */
+	pool->base.dp_clock_source =
+			dcn31_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_ID_DP_DTO,
+				&clk_src_regs[0], true);
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	/* TODO: DCCG */
+	pool->base.dccg = dccg31_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
+	if (pool->base.dccg == NULL) {
+		dm_error("DC: failed to create dccg!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* TODO: IRQ */
+	init_data.ctx = dc->ctx;
+	pool->base.irqs = dal_irq_service_dcn31_create(&init_data);
+	if (!pool->base.irqs)
+		goto create_fail;
+
+	/* HUBBUB */
+	pool->base.hubbub = dcn31_hubbub_create(ctx);
+	if (pool->base.hubbub == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create hubbub!\n");
+		goto create_fail;
+	}
+
+	/* HUBPs, DPPs, OPPs and TGs */
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		pool->base.hubps[i] = dcn31_hubp_create(ctx, i);
+		if (pool->base.hubps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create hubps!\n");
+			goto create_fail;
+		}
+
+		pool->base.dpps[i] = dcn31_dpp_create(ctx, i);
+		if (pool->base.dpps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create dpps!\n");
+			goto create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		pool->base.opps[i] = dcn31_opp_create(ctx, i);
+		if (pool->base.opps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create output pixel processor!\n");
+			goto create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		pool->base.timing_generators[i] = dcn31_timing_generator_create(
+				ctx, i);
+		if (pool->base.timing_generators[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto create_fail;
+		}
+	}
+	pool->base.timing_generator_count = i;
+
+	/* PSR */
+	pool->base.psr = dmub_psr_create(ctx);
+	if (pool->base.psr == NULL) {
+		dm_error("DC: failed to create psr obj!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* ABM */
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		pool->base.multiple_abms[i] = dmub_abm_create(ctx,
+				&abm_regs[i],
+				&abm_shift,
+				&abm_mask);
+		if (pool->base.multiple_abms[i] == NULL) {
+			dm_error("DC: failed to create abm for pipe %d!\n", i);
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	/* MPC and DSC */
+	pool->base.mpc = dcn31_mpc_create(ctx, pool->base.mpcc_count, pool->base.res_cap->num_mpc_3dlut);
+	if (pool->base.mpc == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mpc!\n");
+		goto create_fail;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		pool->base.dscs[i] = dcn31_dsc_create(ctx, i);
+		if (pool->base.dscs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create display stream compressor %d!\n", i);
+			goto create_fail;
+		}
+	}
+
+	/* DWB and MMHUBBUB */
+	if (!dcn31_dwbc_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create dwbc!\n");
+		goto create_fail;
+	}
+
+	if (!dcn31_mmhubbub_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mcif_wb!\n");
+		goto create_fail;
+	}
+
+	/* AUX and I2C */
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.engines[i] = dcn31_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto create_fail;
+		}
+		pool->base.hw_i2cs[i] = dcn31_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create hw i2c!!\n");
+			goto create_fail;
+		}
+		pool->base.sw_i2cs[i] = NULL;
+	}
+
+	/* Audio, Stream Encoders including HPO and virtual, MPC 3D LUTs */
+	if (!resource_construct(num_virtual_links, dc, &pool->base,
+			&res_create_funcs))
+		goto create_fail;
+
+	/* HW Sequencer and Plane caps */
+	dcn31_hw_sequencer_construct(dc);
+
+	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->cap_funcs = cap_funcs;
+
+	dc->dcn_ip->max_num_dpp = dcn3_16_ip.max_num_dpp;
+
+	return true;
+
+create_fail:
+
+	dcn316_resource_destruct(pool);
+
+	return false;
+}
+
+struct resource_pool *dcn316_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc)
+{
+	struct dcn316_resource_pool *pool =
+		kzalloc(sizeof(struct dcn316_resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dcn316_resource_construct(init_data->num_virtual_links, dc, pool))
+		return &pool->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(pool);
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.h
new file mode 100644
index 000000000000..aba6d634131b
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.h
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2021 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef _DCN316_RESOURCE_H_
+#define _DCN316_RESOURCE_H_
+
+#include "core_types.h"
+
+#define TO_DCN316_RES_POOL(pool)\
+	container_of(pool, struct dcn316_resource_pool, base)
+
+extern struct _vcs_dpi_ip_params_st dcn3_16_ip;
+
+struct dcn316_resource_pool {
+	struct resource_pool base;
+};
+
+struct resource_pool *dcn316_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc);
+
+#endif /* _DCN316_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c
new file mode 100644
index 000000000000..36e4c7bef403
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c
@@ -0,0 +1,2862 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dm_services.h"
+#include "dc.h"
+
+#include "dcn32/dcn32_init.h"
+
+#include "resource.h"
+#include "include/irq_service_interface.h"
+#include "dcn32_resource.h"
+
+#include "dcn20/dcn20_resource.h"
+#include "dcn30/dcn30_resource.h"
+
+#include "dcn10/dcn10_ipp.h"
+#include "dcn30/dcn30_hubbub.h"
+#include "dcn31/dcn31_hubbub.h"
+#include "dcn32/dcn32_hubbub.h"
+#include "dcn32/dcn32_mpc.h"
+#include "dcn32/dcn32_hubp.h"
+#include "irq/dcn32/irq_service_dcn32.h"
+#include "dcn32/dcn32_dpp.h"
+#include "dcn32/dcn32_optc.h"
+#include "dcn20/dcn20_hwseq.h"
+#include "dcn30/dcn30_hwseq.h"
+#include "dce110/dce110_hwseq.h"
+#include "dcn30/dcn30_opp.h"
+#include "dcn20/dcn20_dsc.h"
+#include "dcn30/dcn30_vpg.h"
+#include "dcn30/dcn30_afmt.h"
+#include "dcn30/dcn30_dio_stream_encoder.h"
+#include "dcn32/dcn32_dio_stream_encoder.h"
+#include "dcn31/dcn31_hpo_dp_stream_encoder.h"
+#include "dcn31/dcn31_hpo_dp_link_encoder.h"
+#include "dcn32/dcn32_hpo_dp_link_encoder.h"
+#include "dcn31/dcn31_apg.h"
+#include "dcn31/dcn31_dio_link_encoder.h"
+#include "dcn32/dcn32_dio_link_encoder.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_hwseq.h"
+#include "clk_mgr.h"
+#include "virtual/virtual_stream_encoder.h"
+#include "dml/display_mode_vba.h"
+#include "dcn32/dcn32_dccg.h"
+#include "dcn10/dcn10_resource.h"
+#include "link.h"
+#include "dcn31/dcn31_panel_cntl.h"
+
+#include "dcn30/dcn30_dwb.h"
+#include "dcn32/dcn32_mmhubbub.h"
+
+#include "dcn/dcn_3_2_0_offset.h"
+#include "dcn/dcn_3_2_0_sh_mask.h"
+#include "nbio/nbio_4_3_0_offset.h"
+
+#include "reg_helper.h"
+#include "dce/dmub_abm.h"
+#include "dce/dmub_psr.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_i2c.h"
+
+#include "dml/dcn30/display_mode_vba_30.h"
+#include "vm_helper.h"
+#include "dcn20/dcn20_vmid.h"
+#include "dml/dcn32/dcn32_fpu.h"
+
+#include "dml2/dml2_wrapper.h"
+
+#define DC_LOGGER_INIT(logger)
+
+enum dcn32_clk_src_array_id {
+	DCN32_CLK_SRC_PLL0,
+	DCN32_CLK_SRC_PLL1,
+	DCN32_CLK_SRC_PLL2,
+	DCN32_CLK_SRC_PLL3,
+	DCN32_CLK_SRC_PLL4,
+	DCN32_CLK_SRC_TOTAL
+};
+
+/* begin *********************
+ * macros to expend register list macro defined in HW object header file
+ */
+
+/* DCN */
+#define BASE_INNER(seg) ctx->dcn_reg_offsets[seg]
+
+#define BASE(seg) BASE_INNER(seg)
+
+#define SR(reg_name)\
+		REG_STRUCT.reg_name = BASE(reg ## reg_name ## _BASE_IDX) +  \
+					reg ## reg_name
+#define SR_ARR(reg_name, id) \
+	REG_STRUCT[id].reg_name = BASE(reg##reg_name##_BASE_IDX) + reg##reg_name
+
+#define SR_ARR_INIT(reg_name, id, value) \
+	REG_STRUCT[id].reg_name = value
+
+#define SRI(reg_name, block, id)\
+	REG_STRUCT.reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SRI_ARR(reg_name, block, id)\
+	REG_STRUCT[id].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SR_ARR_I2C(reg_name, id) \
+	REG_STRUCT[id-1].reg_name = BASE(reg##reg_name##_BASE_IDX) + reg##reg_name
+
+#define SRI_ARR_I2C(reg_name, block, id)\
+	REG_STRUCT[id-1].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SRI_ARR_ALPHABET(reg_name, block, index, id)\
+	REG_STRUCT[index].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SRI2(reg_name, block, id)\
+	.reg_name = BASE(reg ## reg_name ## _BASE_IDX) +	\
+		reg ## reg_name
+#define SRI2_ARR(reg_name, block, id)\
+	REG_STRUCT[id].reg_name = BASE(reg ## reg_name ## _BASE_IDX) +	\
+		reg ## reg_name
+
+#define SRIR(var_name, reg_name, block, id)\
+	.var_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SRII(reg_name, block, id)\
+	REG_STRUCT.reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define SRII_ARR_2(reg_name, block, id, inst)\
+	REG_STRUCT[inst].reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SRII_MPC_RMU(reg_name, block, id)\
+	.RMU##_##reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SRII_DWB(reg_name, temp_name, block, id)\
+	REG_STRUCT.reg_name[id] = BASE(reg ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## temp_name
+
+#define SF_DWB2(reg_name, block, id, field_name, post_fix)	\
+	.field_name = reg_name ## __ ## field_name ## post_fix
+
+#define DCCG_SRII(reg_name, block, id)\
+	REG_STRUCT.block ## _ ## reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define VUPDATE_SRII(reg_name, block, id)\
+	REG_STRUCT.reg_name[id] = BASE(reg ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
+		reg ## reg_name ## _ ## block ## id
+
+/* NBIO */
+#define NBIO_BASE_INNER(seg) ctx->nbio_reg_offsets[seg]
+
+#define NBIO_BASE(seg) \
+	NBIO_BASE_INNER(seg)
+
+#define NBIO_SR(reg_name)\
+	REG_STRUCT.reg_name = NBIO_BASE(regBIF_BX0_ ## reg_name ## _BASE_IDX) + \
+			regBIF_BX0_ ## reg_name
+#define NBIO_SR_ARR(reg_name, id)\
+	REG_STRUCT[id].reg_name = NBIO_BASE(regBIF_BX0_ ## reg_name ## _BASE_IDX) + \
+		regBIF_BX0_ ## reg_name
+
+#undef CTX
+#define CTX ctx
+#define REG(reg_name) \
+	(ctx->dcn_reg_offsets[reg ## reg_name ## _BASE_IDX] + reg ## reg_name)
+
+static struct bios_registers bios_regs;
+
+#define bios_regs_init() \
+		( \
+		NBIO_SR(BIOS_SCRATCH_3),\
+		NBIO_SR(BIOS_SCRATCH_6)\
+		)
+
+#define clk_src_regs_init(index, pllid)\
+	CS_COMMON_REG_LIST_DCN3_0_RI(index, pllid)
+
+static struct dce110_clk_src_regs clk_src_regs[5];
+
+static const struct dce110_clk_src_shift cs_shift = {
+		CS_COMMON_MASK_SH_LIST_DCN3_2(__SHIFT)
+};
+
+static const struct dce110_clk_src_mask cs_mask = {
+		CS_COMMON_MASK_SH_LIST_DCN3_2(_MASK)
+};
+
+#define abm_regs_init(id)\
+		ABM_DCN32_REG_LIST_RI(id)
+
+static struct dce_abm_registers abm_regs[4];
+
+static const struct dce_abm_shift abm_shift = {
+		ABM_MASK_SH_LIST_DCN32(__SHIFT)
+};
+
+static const struct dce_abm_mask abm_mask = {
+		ABM_MASK_SH_LIST_DCN32(_MASK)
+};
+
+#define audio_regs_init(id)\
+		AUD_COMMON_REG_LIST_RI(id)
+
+static struct dce_audio_registers audio_regs[5];
+
+#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
+		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
+
+static const struct dce_audio_shift audio_shift = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_audio_mask audio_mask = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+#define vpg_regs_init(id)\
+	VPG_DCN3_REG_LIST_RI(id)
+
+static struct dcn30_vpg_registers vpg_regs[10];
+
+static const struct dcn30_vpg_shift vpg_shift = {
+	DCN3_VPG_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn30_vpg_mask vpg_mask = {
+	DCN3_VPG_MASK_SH_LIST(_MASK)
+};
+
+#define afmt_regs_init(id)\
+	AFMT_DCN3_REG_LIST_RI(id)
+
+static struct dcn30_afmt_registers afmt_regs[6];
+
+static const struct dcn30_afmt_shift afmt_shift = {
+	DCN3_AFMT_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn30_afmt_mask afmt_mask = {
+	DCN3_AFMT_MASK_SH_LIST(_MASK)
+};
+
+#define apg_regs_init(id)\
+	APG_DCN31_REG_LIST_RI(id)
+
+static struct dcn31_apg_registers apg_regs[4];
+
+static const struct dcn31_apg_shift apg_shift = {
+	DCN31_APG_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_apg_mask apg_mask = {
+		DCN31_APG_MASK_SH_LIST(_MASK)
+};
+
+#define stream_enc_regs_init(id)\
+	SE_DCN32_REG_LIST_RI(id)
+
+static struct dcn10_stream_enc_registers stream_enc_regs[5];
+
+static const struct dcn10_stream_encoder_shift se_shift = {
+		SE_COMMON_MASK_SH_LIST_DCN32(__SHIFT)
+};
+
+static const struct dcn10_stream_encoder_mask se_mask = {
+		SE_COMMON_MASK_SH_LIST_DCN32(_MASK)
+};
+
+
+#define aux_regs_init(id)\
+	DCN2_AUX_REG_LIST_RI(id)
+
+static struct dcn10_link_enc_aux_registers link_enc_aux_regs[5];
+
+#define hpd_regs_init(id)\
+	HPD_REG_LIST_RI(id)
+
+static struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[5];
+
+#define link_regs_init(id, phyid)\
+	( \
+	LE_DCN31_REG_LIST_RI(id), \
+	UNIPHY_DCN2_REG_LIST_RI(id, phyid)\
+	)
+	/*DPCS_DCN31_REG_LIST(id),*/ \
+
+static struct dcn10_link_enc_registers link_enc_regs[5];
+
+static const struct dcn10_link_enc_shift le_shift = {
+	LINK_ENCODER_MASK_SH_LIST_DCN31(__SHIFT), \
+	//DPCS_DCN31_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn10_link_enc_mask le_mask = {
+	LINK_ENCODER_MASK_SH_LIST_DCN31(_MASK), \
+	//DPCS_DCN31_MASK_SH_LIST(_MASK)
+};
+
+#define hpo_dp_stream_encoder_reg_init(id)\
+	DCN3_1_HPO_DP_STREAM_ENC_REG_LIST_RI(id)
+
+static struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[4];
+
+static const struct dcn31_hpo_dp_stream_encoder_shift hpo_dp_se_shift = {
+	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_hpo_dp_stream_encoder_mask hpo_dp_se_mask = {
+	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(_MASK)
+};
+
+
+#define hpo_dp_link_encoder_reg_init(id)\
+	DCN3_1_HPO_DP_LINK_ENC_REG_LIST_RI(id)
+	/*DCN3_1_RDPCSTX_REG_LIST(0),*/
+	/*DCN3_1_RDPCSTX_REG_LIST(1),*/
+	/*DCN3_1_RDPCSTX_REG_LIST(2),*/
+	/*DCN3_1_RDPCSTX_REG_LIST(3),*/
+
+static struct dcn31_hpo_dp_link_encoder_registers hpo_dp_link_enc_regs[2];
+
+static const struct dcn31_hpo_dp_link_encoder_shift hpo_dp_le_shift = {
+	DCN3_2_HPO_DP_LINK_ENC_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_hpo_dp_link_encoder_mask hpo_dp_le_mask = {
+	DCN3_2_HPO_DP_LINK_ENC_MASK_SH_LIST(_MASK)
+};
+
+#define dpp_regs_init(id)\
+	DPP_REG_LIST_DCN30_COMMON_RI(id)
+
+static struct dcn3_dpp_registers dpp_regs[4];
+
+static const struct dcn3_dpp_shift tf_shift = {
+		DPP_REG_LIST_SH_MASK_DCN30_COMMON(__SHIFT)
+};
+
+static const struct dcn3_dpp_mask tf_mask = {
+		DPP_REG_LIST_SH_MASK_DCN30_COMMON(_MASK)
+};
+
+
+#define opp_regs_init(id)\
+	OPP_REG_LIST_DCN30_RI(id)
+
+static struct dcn20_opp_registers opp_regs[4];
+
+static const struct dcn20_opp_shift opp_shift = {
+	OPP_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn20_opp_mask opp_mask = {
+	OPP_MASK_SH_LIST_DCN20(_MASK)
+};
+
+#define aux_engine_regs_init(id)\
+	( \
+	AUX_COMMON_REG_LIST0_RI(id), \
+	SR_ARR_INIT(AUXN_IMPCAL, id, 0), \
+	SR_ARR_INIT(AUXP_IMPCAL, id, 0), \
+	SR_ARR_INIT(AUX_RESET_MASK, id, DP_AUX0_AUX_CONTROL__AUX_RESET_MASK), \
+	SR_ARR_INIT(AUX_RESET_MASK, id, DP_AUX0_AUX_CONTROL__AUX_RESET_MASK)\
+	)
+
+static struct dce110_aux_registers aux_engine_regs[5];
+
+static const struct dce110_aux_registers_shift aux_shift = {
+	DCN_AUX_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+	DCN_AUX_MASK_SH_LIST(_MASK)
+};
+
+#define dwbc_regs_dcn3_init(id)\
+	DWBC_COMMON_REG_LIST_DCN30_RI(id)
+
+static struct dcn30_dwbc_registers dwbc30_regs[1];
+
+static const struct dcn30_dwbc_shift dwbc30_shift = {
+	DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_dwbc_mask dwbc30_mask = {
+	DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define mcif_wb_regs_dcn3_init(id)\
+	MCIF_WB_COMMON_REG_LIST_DCN32_RI(id)
+
+static struct dcn30_mmhubbub_registers mcif_wb30_regs[1];
+
+static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN32(__SHIFT)
+};
+
+static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN32(_MASK)
+};
+
+#define dsc_regsDCN20_init(id)\
+	DSC_REG_LIST_DCN20_RI(id)
+
+static struct dcn20_dsc_registers dsc_regs[4];
+
+static const struct dcn20_dsc_shift dsc_shift = {
+	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
+};
+
+static const struct dcn20_dsc_mask dsc_mask = {
+	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
+};
+
+static struct dcn30_mpc_registers mpc_regs;
+
+#define dcn_mpc_regs_init() \
+	MPC_REG_LIST_DCN3_2_RI(0),\
+	MPC_REG_LIST_DCN3_2_RI(1),\
+	MPC_REG_LIST_DCN3_2_RI(2),\
+	MPC_REG_LIST_DCN3_2_RI(3),\
+	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(0),\
+	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(1),\
+	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(2),\
+	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(3),\
+	MPC_DWB_MUX_REG_LIST_DCN3_0_RI(0)
+
+static const struct dcn30_mpc_shift mpc_shift = {
+	MPC_COMMON_MASK_SH_LIST_DCN32(__SHIFT)
+};
+
+static const struct dcn30_mpc_mask mpc_mask = {
+	MPC_COMMON_MASK_SH_LIST_DCN32(_MASK)
+};
+
+#define optc_regs_init(id)\
+	OPTC_COMMON_REG_LIST_DCN3_2_RI(id)
+
+static struct dcn_optc_registers optc_regs[4];
+
+static const struct dcn_optc_shift optc_shift = {
+	OPTC_COMMON_MASK_SH_LIST_DCN3_2(__SHIFT)
+};
+
+static const struct dcn_optc_mask optc_mask = {
+	OPTC_COMMON_MASK_SH_LIST_DCN3_2(_MASK)
+};
+
+#define hubp_regs_init(id)\
+	HUBP_REG_LIST_DCN32_RI(id)
+
+static struct dcn_hubp2_registers hubp_regs[4];
+
+
+static const struct dcn_hubp2_shift hubp_shift = {
+		HUBP_MASK_SH_LIST_DCN32(__SHIFT)
+};
+
+static const struct dcn_hubp2_mask hubp_mask = {
+		HUBP_MASK_SH_LIST_DCN32(_MASK)
+};
+
+static struct dcn_hubbub_registers hubbub_reg;
+#define hubbub_reg_init()\
+		HUBBUB_REG_LIST_DCN32_RI(0)
+
+static const struct dcn_hubbub_shift hubbub_shift = {
+		HUBBUB_MASK_SH_LIST_DCN32(__SHIFT)
+};
+
+static const struct dcn_hubbub_mask hubbub_mask = {
+		HUBBUB_MASK_SH_LIST_DCN32(_MASK)
+};
+
+static struct dccg_registers dccg_regs;
+
+#define dccg_regs_init()\
+	DCCG_REG_LIST_DCN32_RI()
+
+static const struct dccg_shift dccg_shift = {
+		DCCG_MASK_SH_LIST_DCN32(__SHIFT)
+};
+
+static const struct dccg_mask dccg_mask = {
+		DCCG_MASK_SH_LIST_DCN32(_MASK)
+};
+
+
+#define SRII2(reg_name_pre, reg_name_post, id)\
+	.reg_name_pre ## _ ##  reg_name_post[id] = BASE(reg ## reg_name_pre \
+			## id ## _ ## reg_name_post ## _BASE_IDX) + \
+			reg ## reg_name_pre ## id ## _ ## reg_name_post
+
+
+#define HWSEQ_DCN32_REG_LIST()\
+	SR(DCHUBBUB_GLOBAL_TIMER_CNTL), \
+	SR(DIO_MEM_PWR_CTRL), \
+	SR(ODM_MEM_PWR_CTRL3), \
+	SR(MMHUBBUB_MEM_PWR_CNTL), \
+	SR(DCCG_GATE_DISABLE_CNTL), \
+	SR(DCCG_GATE_DISABLE_CNTL2), \
+	SR(DCFCLK_CNTL),\
+	SR(DC_MEM_GLOBAL_PWR_REQ_CNTL), \
+	SRII(PIXEL_RATE_CNTL, OTG, 0), \
+	SRII(PIXEL_RATE_CNTL, OTG, 1),\
+	SRII(PIXEL_RATE_CNTL, OTG, 2),\
+	SRII(PIXEL_RATE_CNTL, OTG, 3),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 0),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 1),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 2),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 3),\
+	SR(MICROSECOND_TIME_BASE_DIV), \
+	SR(MILLISECOND_TIME_BASE_DIV), \
+	SR(DISPCLK_FREQ_CHANGE_CNTL), \
+	SR(RBBMIF_TIMEOUT_DIS), \
+	SR(RBBMIF_TIMEOUT_DIS_2), \
+	SR(DCHUBBUB_CRC_CTRL), \
+	SR(DPP_TOP0_DPP_CRC_CTRL), \
+	SR(DPP_TOP0_DPP_CRC_VAL_B_A), \
+	SR(DPP_TOP0_DPP_CRC_VAL_R_G), \
+	SR(MPC_CRC_CTRL), \
+	SR(MPC_CRC_RESULT_GB), \
+	SR(MPC_CRC_RESULT_C), \
+	SR(MPC_CRC_RESULT_AR), \
+	SR(DOMAIN0_PG_CONFIG), \
+	SR(DOMAIN1_PG_CONFIG), \
+	SR(DOMAIN2_PG_CONFIG), \
+	SR(DOMAIN3_PG_CONFIG), \
+	SR(DOMAIN16_PG_CONFIG), \
+	SR(DOMAIN17_PG_CONFIG), \
+	SR(DOMAIN18_PG_CONFIG), \
+	SR(DOMAIN19_PG_CONFIG), \
+	SR(DOMAIN0_PG_STATUS), \
+	SR(DOMAIN1_PG_STATUS), \
+	SR(DOMAIN2_PG_STATUS), \
+	SR(DOMAIN3_PG_STATUS), \
+	SR(DOMAIN16_PG_STATUS), \
+	SR(DOMAIN17_PG_STATUS), \
+	SR(DOMAIN18_PG_STATUS), \
+	SR(DOMAIN19_PG_STATUS), \
+	SR(D1VGA_CONTROL), \
+	SR(D2VGA_CONTROL), \
+	SR(D3VGA_CONTROL), \
+	SR(D4VGA_CONTROL), \
+	SR(D5VGA_CONTROL), \
+	SR(D6VGA_CONTROL), \
+	SR(DC_IP_REQUEST_CNTL), \
+	SR(AZALIA_AUDIO_DTO), \
+	SR(AZALIA_CONTROLLER_CLOCK_GATING)
+
+static struct dce_hwseq_registers hwseq_reg;
+
+#define hwseq_reg_init()\
+	HWSEQ_DCN32_REG_LIST()
+
+#define HWSEQ_DCN32_MASK_SH_LIST(mask_sh)\
+	HWSEQ_DCN_MASK_SH_LIST(mask_sh), \
+	HWS_SF(, DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, mask_sh), \
+	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN0_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN1_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN2_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN3_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN16_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN17_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN18_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN19_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh), \
+	HWS_SF(, AZALIA_AUDIO_DTO, AZALIA_AUDIO_DTO_MODULE, mask_sh), \
+	HWS_SF(, HPO_TOP_CLOCK_CONTROL, HPO_HDMISTREAMCLK_G_GATE_DIS, mask_sh), \
+	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_UNASSIGNED_PWR_MODE, mask_sh), \
+	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_VBLANK_PWR_MODE, mask_sh), \
+	HWS_SF(, MMHUBBUB_MEM_PWR_CNTL, VGA_MEM_PWR_FORCE, mask_sh)
+
+static const struct dce_hwseq_shift hwseq_shift = {
+		HWSEQ_DCN32_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+		HWSEQ_DCN32_MASK_SH_LIST(_MASK)
+};
+#define vmid_regs_init(id)\
+		DCN20_VMID_REG_LIST_RI(id)
+
+static struct dcn_vmid_registers vmid_regs[16];
+
+static const struct dcn20_vmid_shift vmid_shifts = {
+		DCN20_VMID_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn20_vmid_mask vmid_masks = {
+		DCN20_VMID_MASK_SH_LIST(_MASK)
+};
+
+static const struct resource_caps res_cap_dcn32 = {
+	.num_timing_generator = 4,
+	.num_opp = 4,
+	.num_video_plane = 4,
+	.num_audio = 5,
+	.num_stream_encoder = 5,
+	.num_hpo_dp_stream_encoder = 4,
+	.num_hpo_dp_link_encoder = 2,
+	.num_pll = 5,
+	.num_dwb = 1,
+	.num_ddc = 5,
+	.num_vmid = 16,
+	.num_mpc_3dlut = 4,
+	.num_dsc = 4,
+};
+
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
+	.per_pixel_alpha = true,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = true,
+			.fp16 = true,
+			.p010 = true,
+			.ayuv = false,
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 16000,
+			.fp16 = 16000
+	},
+
+	// 6:1 downscaling ratio: 1000/6 = 166.666
+	.max_downscale_factor = {
+			.argb8888 = 167,
+			.nv12 = 167,
+			.fp16 = 167
+	},
+	64,
+	64
+};
+
+static const struct dc_debug_options debug_defaults_drv = {
+	.disable_dmcu = true,
+	.force_abm_enable = false,
+	.timing_trace = false,
+	.clock_trace = true,
+	.disable_pplib_clock_request = false,
+	.pipe_split_policy = MPC_SPLIT_AVOID, // Due to CRB, no need to MPC split anymore
+	.force_single_disp_pipe_split = false,
+	.disable_dcc = DCC_ENABLE,
+	.vsr_support = true,
+	.performance_trace = false,
+	.max_downscale_src_width = 7680,/*upto 8K*/
+	.disable_pplib_wm_range = false,
+	.scl_reset_length10 = true,
+	.sanity_checks = false,
+	.underflow_assert_delay_us = 0xFFFFFFFF,
+	.dwb_fi_phase = -1, // -1 = disable,
+	.dmub_command_table = true,
+	.enable_mem_low_power = {
+		.bits = {
+			.vga = false,
+			.i2c = false,
+			.dmcu = false, // This is previously known to cause hang on S3 cycles if enabled
+			.dscl = false,
+			.cm = false,
+			.mpc = false,
+			.optc = true,
+		}
+	},
+	.use_max_lb = true,
+	.force_disable_subvp = false,
+	.exit_idle_opt_for_cursor_updates = true,
+	.using_dml2 = false,
+	.enable_single_display_2to1_odm_policy = true,
+
+	/* Must match enable_single_display_2to1_odm_policy to support dynamic ODM transitions*/
+	.enable_double_buffered_dsc_pg_support = true,
+	.enable_dp_dig_pixel_rate_div_policy = 1,
+	.allow_sw_cursor_fallback = false, // Linux can't do SW cursor "fallback"
+	.alloc_extra_way_for_cursor = true,
+	.min_prefetch_in_strobe_ns = 60000, // 60us
+	.disable_unbounded_requesting = false,
+	.override_dispclk_programming = true,
+	.disable_fpo_optimizations = false,
+	.fpo_vactive_margin_us = 2000, // 2000us
+	.disable_fpo_vactive = false,
+	.disable_boot_optimizations = false,
+	.disable_subvp_high_refresh = false,
+	.disable_dp_plus_plus_wa = true,
+	.fpo_vactive_min_active_margin_us = 200,
+	.fpo_vactive_max_blank_us = 1000,
+	.enable_legacy_fast_update = false,
+};
+
+static struct dce_aux *dcn32_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT aux_engine_regs
+	aux_engine_regs_init(0),
+	aux_engine_regs_init(1),
+	aux_engine_regs_init(2),
+	aux_engine_regs_init(3),
+	aux_engine_regs_init(4);
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst],
+					&aux_mask,
+					&aux_shift,
+					ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+#define i2c_inst_regs_init(id)\
+	I2C_HW_ENGINE_COMMON_REG_LIST_DCN30_RI(id)
+
+static struct dce_i2c_registers i2c_hw_regs[5];
+
+static const struct dce_i2c_shift i2c_shifts = {
+		I2C_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dce_i2c_mask i2c_masks = {
+		I2C_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static struct dce_i2c_hw *dcn32_i2c_hw_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw =
+		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT i2c_hw_regs
+	i2c_inst_regs_init(1),
+	i2c_inst_regs_init(2),
+	i2c_inst_regs_init(3),
+	i2c_inst_regs_init(4),
+	i2c_inst_regs_init(5);
+
+	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
+				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+
+static struct clock_source *dcn32_clock_source_create(
+		struct dc_context *ctx,
+		struct dc_bios *bios,
+		enum clock_source_id id,
+		const struct dce110_clk_src_regs *regs,
+		bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dcn31_clk_src_construct(clk_src, ctx, bios, id,
+			regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	kfree(clk_src);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static struct hubbub *dcn32_hubbub_create(struct dc_context *ctx)
+{
+	int i;
+
+	struct dcn20_hubbub *hubbub2 = kzalloc(sizeof(struct dcn20_hubbub),
+					  GFP_KERNEL);
+
+	if (!hubbub2)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT hubbub_reg
+	hubbub_reg_init();
+
+#undef REG_STRUCT
+#define REG_STRUCT vmid_regs
+	vmid_regs_init(0),
+	vmid_regs_init(1),
+	vmid_regs_init(2),
+	vmid_regs_init(3),
+	vmid_regs_init(4),
+	vmid_regs_init(5),
+	vmid_regs_init(6),
+	vmid_regs_init(7),
+	vmid_regs_init(8),
+	vmid_regs_init(9),
+	vmid_regs_init(10),
+	vmid_regs_init(11),
+	vmid_regs_init(12),
+	vmid_regs_init(13),
+	vmid_regs_init(14),
+	vmid_regs_init(15);
+
+	hubbub32_construct(hubbub2, ctx,
+			&hubbub_reg,
+			&hubbub_shift,
+			&hubbub_mask,
+			ctx->dc->dml.ip.det_buffer_size_kbytes,
+			ctx->dc->dml.ip.pixel_chunk_size_kbytes,
+			ctx->dc->dml.ip.config_return_buffer_size_in_kbytes);
+
+
+	for (i = 0; i < res_cap_dcn32.num_vmid; i++) {
+		struct dcn20_vmid *vmid = &hubbub2->vmid[i];
+
+		vmid->ctx = ctx;
+
+		vmid->regs = &vmid_regs[i];
+		vmid->shifts = &vmid_shifts;
+		vmid->masks = &vmid_masks;
+	}
+
+	return &hubbub2->base;
+}
+
+static struct hubp *dcn32_hubp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn20_hubp *hubp2 =
+		kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
+
+	if (!hubp2)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT hubp_regs
+	hubp_regs_init(0),
+	hubp_regs_init(1),
+	hubp_regs_init(2),
+	hubp_regs_init(3);
+
+	if (hubp32_construct(hubp2, ctx, inst,
+			&hubp_regs[inst], &hubp_shift, &hubp_mask))
+		return &hubp2->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(hubp2);
+	return NULL;
+}
+
+static void dcn32_dpp_destroy(struct dpp **dpp)
+{
+	kfree(TO_DCN30_DPP(*dpp));
+	*dpp = NULL;
+}
+
+static struct dpp *dcn32_dpp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn3_dpp *dpp3 =
+		kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
+
+	if (!dpp3)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT dpp_regs
+	dpp_regs_init(0),
+	dpp_regs_init(1),
+	dpp_regs_init(2),
+	dpp_regs_init(3);
+
+	if (dpp32_construct(dpp3, ctx, inst,
+			&dpp_regs[inst], &tf_shift, &tf_mask))
+		return &dpp3->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(dpp3);
+	return NULL;
+}
+
+static struct mpc *dcn32_mpc_create(
+		struct dc_context *ctx,
+		int num_mpcc,
+		int num_rmu)
+{
+	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc),
+					  GFP_KERNEL);
+
+	if (!mpc30)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT mpc_regs
+	dcn_mpc_regs_init();
+
+	dcn32_mpc_construct(mpc30, ctx,
+			&mpc_regs,
+			&mpc_shift,
+			&mpc_mask,
+			num_mpcc,
+			num_rmu);
+
+	return &mpc30->base;
+}
+
+static struct output_pixel_processor *dcn32_opp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_opp *opp2 =
+		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
+
+	if (!opp2) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+#undef REG_STRUCT
+#define REG_STRUCT opp_regs
+	opp_regs_init(0),
+	opp_regs_init(1),
+	opp_regs_init(2),
+	opp_regs_init(3);
+
+	dcn20_opp_construct(opp2, ctx, inst,
+			&opp_regs[inst], &opp_shift, &opp_mask);
+	return &opp2->base;
+}
+
+
+static struct timing_generator *dcn32_timing_generator_create(
+		struct dc_context *ctx,
+		uint32_t instance)
+{
+	struct optc *tgn10 =
+		kzalloc(sizeof(struct optc), GFP_KERNEL);
+
+	if (!tgn10)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT optc_regs
+	optc_regs_init(0),
+	optc_regs_init(1),
+	optc_regs_init(2),
+	optc_regs_init(3);
+
+	tgn10->base.inst = instance;
+	tgn10->base.ctx = ctx;
+
+	tgn10->tg_regs = &optc_regs[instance];
+	tgn10->tg_shift = &optc_shift;
+	tgn10->tg_mask = &optc_mask;
+
+	dcn32_timing_generator_init(tgn10);
+
+	return &tgn10->base;
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+		.max_hdmi_deep_color = COLOR_DEPTH_121212,
+		.max_hdmi_pixel_clock = 600000,
+		.hdmi_ycbcr420_supported = true,
+		.dp_ycbcr420_supported = true,
+		.fec_supported = true,
+		.flags.bits.IS_HBR2_CAPABLE = true,
+		.flags.bits.IS_HBR3_CAPABLE = true,
+		.flags.bits.IS_TPS3_CAPABLE = true,
+		.flags.bits.IS_TPS4_CAPABLE = true
+};
+
+static struct link_encoder *dcn32_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dcn20_link_encoder *enc20 =
+		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+
+	if (!enc20)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT link_enc_aux_regs
+	aux_regs_init(0),
+	aux_regs_init(1),
+	aux_regs_init(2),
+	aux_regs_init(3),
+	aux_regs_init(4);
+
+#undef REG_STRUCT
+#define REG_STRUCT link_enc_hpd_regs
+	hpd_regs_init(0),
+	hpd_regs_init(1),
+	hpd_regs_init(2),
+	hpd_regs_init(3),
+	hpd_regs_init(4);
+
+#undef REG_STRUCT
+#define REG_STRUCT link_enc_regs
+	link_regs_init(0, A),
+	link_regs_init(1, B),
+	link_regs_init(2, C),
+	link_regs_init(3, D),
+	link_regs_init(4, E);
+
+	dcn32_link_encoder_construct(enc20,
+			enc_init_data,
+			&link_enc_feature,
+			&link_enc_regs[enc_init_data->transmitter],
+			&link_enc_aux_regs[enc_init_data->channel - 1],
+			&link_enc_hpd_regs[enc_init_data->hpd_source],
+			&le_shift,
+			&le_mask);
+
+	return &enc20->enc10.base;
+}
+
+struct panel_cntl *dcn32_panel_cntl_create(const struct panel_cntl_init_data *init_data)
+{
+	struct dcn31_panel_cntl *panel_cntl =
+		kzalloc(sizeof(struct dcn31_panel_cntl), GFP_KERNEL);
+
+	if (!panel_cntl)
+		return NULL;
+
+	dcn31_panel_cntl_construct(panel_cntl, init_data);
+
+	return &panel_cntl->base;
+}
+
+static void read_dce_straps(
+	struct dc_context *ctx,
+	struct resource_straps *straps)
+{
+	generic_reg_get(ctx, ctx->dcn_reg_offsets[regDC_PINSTRAPS_BASE_IDX] + regDC_PINSTRAPS,
+		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
+
+}
+
+static struct audio *dcn32_create_audio(
+		struct dc_context *ctx, unsigned int inst)
+{
+
+#undef REG_STRUCT
+#define REG_STRUCT audio_regs
+	audio_regs_init(0),
+	audio_regs_init(1),
+	audio_regs_init(2),
+	audio_regs_init(3),
+	audio_regs_init(4);
+
+	return dce_audio_create(ctx, inst,
+			&audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+static struct vpg *dcn32_vpg_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn30_vpg *vpg3 = kzalloc(sizeof(struct dcn30_vpg), GFP_KERNEL);
+
+	if (!vpg3)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT vpg_regs
+	vpg_regs_init(0),
+	vpg_regs_init(1),
+	vpg_regs_init(2),
+	vpg_regs_init(3),
+	vpg_regs_init(4),
+	vpg_regs_init(5),
+	vpg_regs_init(6),
+	vpg_regs_init(7),
+	vpg_regs_init(8),
+	vpg_regs_init(9);
+
+	vpg3_construct(vpg3, ctx, inst,
+			&vpg_regs[inst],
+			&vpg_shift,
+			&vpg_mask);
+
+	return &vpg3->base;
+}
+
+static struct afmt *dcn32_afmt_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn30_afmt *afmt3 = kzalloc(sizeof(struct dcn30_afmt), GFP_KERNEL);
+
+	if (!afmt3)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT afmt_regs
+	afmt_regs_init(0),
+	afmt_regs_init(1),
+	afmt_regs_init(2),
+	afmt_regs_init(3),
+	afmt_regs_init(4),
+	afmt_regs_init(5);
+
+	afmt3_construct(afmt3, ctx, inst,
+			&afmt_regs[inst],
+			&afmt_shift,
+			&afmt_mask);
+
+	return &afmt3->base;
+}
+
+static struct apg *dcn31_apg_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn31_apg *apg31 = kzalloc(sizeof(struct dcn31_apg), GFP_KERNEL);
+
+	if (!apg31)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT apg_regs
+	apg_regs_init(0),
+	apg_regs_init(1),
+	apg_regs_init(2),
+	apg_regs_init(3);
+
+	apg31_construct(apg31, ctx, inst,
+			&apg_regs[inst],
+			&apg_shift,
+			&apg_mask);
+
+	return &apg31->base;
+}
+
+static struct stream_encoder *dcn32_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dcn10_stream_encoder *enc1;
+	struct vpg *vpg;
+	struct afmt *afmt;
+	int vpg_inst;
+	int afmt_inst;
+
+	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
+	if (eng_id <= ENGINE_ID_DIGF) {
+		vpg_inst = eng_id;
+		afmt_inst = eng_id;
+	} else
+		return NULL;
+
+	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
+	vpg = dcn32_vpg_create(ctx, vpg_inst);
+	afmt = dcn32_afmt_create(ctx, afmt_inst);
+
+	if (!enc1 || !vpg || !afmt) {
+		kfree(enc1);
+		kfree(vpg);
+		kfree(afmt);
+		return NULL;
+	}
+
+#undef REG_STRUCT
+#define REG_STRUCT stream_enc_regs
+	stream_enc_regs_init(0),
+	stream_enc_regs_init(1),
+	stream_enc_regs_init(2),
+	stream_enc_regs_init(3),
+	stream_enc_regs_init(4);
+
+	dcn32_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
+					eng_id, vpg, afmt,
+					&stream_enc_regs[eng_id],
+					&se_shift, &se_mask);
+
+	return &enc1->base;
+}
+
+static struct hpo_dp_stream_encoder *dcn32_hpo_dp_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dcn31_hpo_dp_stream_encoder *hpo_dp_enc31;
+	struct vpg *vpg;
+	struct apg *apg;
+	uint32_t hpo_dp_inst;
+	uint32_t vpg_inst;
+	uint32_t apg_inst;
+
+	ASSERT((eng_id >= ENGINE_ID_HPO_DP_0) && (eng_id <= ENGINE_ID_HPO_DP_3));
+	hpo_dp_inst = eng_id - ENGINE_ID_HPO_DP_0;
+
+	/* Mapping of VPG register blocks to HPO DP block instance:
+	 * VPG[6] -> HPO_DP[0]
+	 * VPG[7] -> HPO_DP[1]
+	 * VPG[8] -> HPO_DP[2]
+	 * VPG[9] -> HPO_DP[3]
+	 */
+	vpg_inst = hpo_dp_inst + 6;
+
+	/* Mapping of APG register blocks to HPO DP block instance:
+	 * APG[0] -> HPO_DP[0]
+	 * APG[1] -> HPO_DP[1]
+	 * APG[2] -> HPO_DP[2]
+	 * APG[3] -> HPO_DP[3]
+	 */
+	apg_inst = hpo_dp_inst;
+
+	/* allocate HPO stream encoder and create VPG sub-block */
+	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_stream_encoder), GFP_KERNEL);
+	vpg = dcn32_vpg_create(ctx, vpg_inst);
+	apg = dcn31_apg_create(ctx, apg_inst);
+
+	if (!hpo_dp_enc31 || !vpg || !apg) {
+		kfree(hpo_dp_enc31);
+		kfree(vpg);
+		kfree(apg);
+		return NULL;
+	}
+
+#undef REG_STRUCT
+#define REG_STRUCT hpo_dp_stream_enc_regs
+	hpo_dp_stream_encoder_reg_init(0),
+	hpo_dp_stream_encoder_reg_init(1),
+	hpo_dp_stream_encoder_reg_init(2),
+	hpo_dp_stream_encoder_reg_init(3);
+
+	dcn31_hpo_dp_stream_encoder_construct(hpo_dp_enc31, ctx, ctx->dc_bios,
+					hpo_dp_inst, eng_id, vpg, apg,
+					&hpo_dp_stream_enc_regs[hpo_dp_inst],
+					&hpo_dp_se_shift, &hpo_dp_se_mask);
+
+	return &hpo_dp_enc31->base;
+}
+
+static struct hpo_dp_link_encoder *dcn32_hpo_dp_link_encoder_create(
+	uint8_t inst,
+	struct dc_context *ctx)
+{
+	struct dcn31_hpo_dp_link_encoder *hpo_dp_enc31;
+
+	/* allocate HPO link encoder */
+	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
+
+#undef REG_STRUCT
+#define REG_STRUCT hpo_dp_link_enc_regs
+	hpo_dp_link_encoder_reg_init(0),
+	hpo_dp_link_encoder_reg_init(1);
+
+	hpo_dp_link_encoder32_construct(hpo_dp_enc31, ctx, inst,
+					&hpo_dp_link_enc_regs[inst],
+					&hpo_dp_le_shift, &hpo_dp_le_mask);
+
+	return &hpo_dp_enc31->base;
+}
+
+static struct dce_hwseq *dcn32_hwseq_create(
+	struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+#undef REG_STRUCT
+#define REG_STRUCT hwseq_reg
+	hwseq_reg_init();
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &hwseq_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+	}
+	return hws;
+}
+static const struct resource_create_funcs res_create_funcs = {
+	.read_dce_straps = read_dce_straps,
+	.create_audio = dcn32_create_audio,
+	.create_stream_encoder = dcn32_stream_encoder_create,
+	.create_hpo_dp_stream_encoder = dcn32_hpo_dp_stream_encoder_create,
+	.create_hpo_dp_link_encoder = dcn32_hpo_dp_link_encoder_create,
+	.create_hwseq = dcn32_hwseq_create,
+};
+
+static void dcn32_resource_destruct(struct dcn32_resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->base.stream_enc_count; i++) {
+		if (pool->base.stream_enc[i] != NULL) {
+			if (pool->base.stream_enc[i]->vpg != NULL) {
+				kfree(DCN30_VPG_FROM_VPG(pool->base.stream_enc[i]->vpg));
+				pool->base.stream_enc[i]->vpg = NULL;
+			}
+			if (pool->base.stream_enc[i]->afmt != NULL) {
+				kfree(DCN30_AFMT_FROM_AFMT(pool->base.stream_enc[i]->afmt));
+				pool->base.stream_enc[i]->afmt = NULL;
+			}
+			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
+			pool->base.stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.hpo_dp_stream_enc_count; i++) {
+		if (pool->base.hpo_dp_stream_enc[i] != NULL) {
+			if (pool->base.hpo_dp_stream_enc[i]->vpg != NULL) {
+				kfree(DCN30_VPG_FROM_VPG(pool->base.hpo_dp_stream_enc[i]->vpg));
+				pool->base.hpo_dp_stream_enc[i]->vpg = NULL;
+			}
+			if (pool->base.hpo_dp_stream_enc[i]->apg != NULL) {
+				kfree(DCN31_APG_FROM_APG(pool->base.hpo_dp_stream_enc[i]->apg));
+				pool->base.hpo_dp_stream_enc[i]->apg = NULL;
+			}
+			kfree(DCN3_1_HPO_DP_STREAM_ENC_FROM_HPO_STREAM_ENC(pool->base.hpo_dp_stream_enc[i]));
+			pool->base.hpo_dp_stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.hpo_dp_link_enc_count; i++) {
+		if (pool->base.hpo_dp_link_enc[i] != NULL) {
+			kfree(DCN3_1_HPO_DP_LINK_ENC_FROM_HPO_LINK_ENC(pool->base.hpo_dp_link_enc[i]));
+			pool->base.hpo_dp_link_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		if (pool->base.dscs[i] != NULL)
+			dcn20_dsc_destroy(&pool->base.dscs[i]);
+	}
+
+	if (pool->base.mpc != NULL) {
+		kfree(TO_DCN20_MPC(pool->base.mpc));
+		pool->base.mpc = NULL;
+	}
+	if (pool->base.hubbub != NULL) {
+		kfree(TO_DCN20_HUBBUB(pool->base.hubbub));
+		pool->base.hubbub = NULL;
+	}
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (pool->base.dpps[i] != NULL)
+			dcn32_dpp_destroy(&pool->base.dpps[i]);
+
+		if (pool->base.ipps[i] != NULL)
+			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
+
+		if (pool->base.hubps[i] != NULL) {
+			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
+			pool->base.hubps[i] = NULL;
+		}
+
+		if (pool->base.irqs != NULL) {
+			dal_irq_service_destroy(&pool->base.irqs);
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+		if (pool->base.hw_i2cs[i] != NULL) {
+			kfree(pool->base.hw_i2cs[i]);
+			pool->base.hw_i2cs[i] = NULL;
+		}
+		if (pool->base.sw_i2cs[i] != NULL) {
+			kfree(pool->base.sw_i2cs[i]);
+			pool->base.sw_i2cs[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		if (pool->base.opps[i] != NULL)
+			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.timing_generators[i] != NULL)	{
+			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
+			pool->base.timing_generators[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+		if (pool->base.dwbc[i] != NULL) {
+			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
+			pool->base.dwbc[i] = NULL;
+		}
+		if (pool->base.mcif_wb[i] != NULL) {
+			kfree(TO_DCN30_MMHUBBUB(pool->base.mcif_wb[i]));
+			pool->base.mcif_wb[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.audio_count; i++) {
+		if (pool->base.audios[i])
+			dce_aud_destroy(&pool->base.audios[i]);
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] != NULL) {
+			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
+			pool->base.clock_sources[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+		if (pool->base.mpc_lut[i] != NULL) {
+			dc_3dlut_func_release(pool->base.mpc_lut[i]);
+			pool->base.mpc_lut[i] = NULL;
+		}
+		if (pool->base.mpc_shaper[i] != NULL) {
+			dc_transfer_func_release(pool->base.mpc_shaper[i]);
+			pool->base.mpc_shaper[i] = NULL;
+		}
+	}
+
+	if (pool->base.dp_clock_source != NULL) {
+		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
+		pool->base.dp_clock_source = NULL;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.multiple_abms[i] != NULL)
+			dce_abm_destroy(&pool->base.multiple_abms[i]);
+	}
+
+	if (pool->base.psr != NULL)
+		dmub_psr_destroy(&pool->base.psr);
+
+	if (pool->base.dccg != NULL)
+		dcn_dccg_destroy(&pool->base.dccg);
+
+	if (pool->base.oem_device != NULL) {
+		struct dc *dc = pool->base.oem_device->ctx->dc;
+
+		dc->link_srv->destroy_ddc_service(&pool->base.oem_device);
+	}
+}
+
+
+static bool dcn32_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t dwb_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < dwb_count; i++) {
+		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc),
+						    GFP_KERNEL);
+
+		if (!dwbc30) {
+			dm_error("DC: failed to create dwbc30!\n");
+			return false;
+		}
+
+#undef REG_STRUCT
+#define REG_STRUCT dwbc30_regs
+		dwbc_regs_dcn3_init(0);
+
+		dcn30_dwbc_construct(dwbc30, ctx,
+				&dwbc30_regs[i],
+				&dwbc30_shift,
+				&dwbc30_mask,
+				i);
+
+		pool->dwbc[i] = &dwbc30->base;
+	}
+	return true;
+}
+
+static bool dcn32_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t dwb_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < dwb_count; i++) {
+		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub),
+						    GFP_KERNEL);
+
+		if (!mcif_wb30) {
+			dm_error("DC: failed to create mcif_wb30!\n");
+			return false;
+		}
+
+#undef REG_STRUCT
+#define REG_STRUCT mcif_wb30_regs
+		mcif_wb_regs_dcn3_init(0);
+
+		dcn32_mmhubbub_construct(mcif_wb30, ctx,
+				&mcif_wb30_regs[i],
+				&mcif_wb30_shift,
+				&mcif_wb30_mask,
+				i);
+
+		pool->mcif_wb[i] = &mcif_wb30->base;
+	}
+	return true;
+}
+
+static struct display_stream_compressor *dcn32_dsc_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_dsc *dsc =
+		kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
+
+	if (!dsc) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+#undef REG_STRUCT
+#define REG_STRUCT dsc_regs
+	dsc_regsDCN20_init(0),
+	dsc_regsDCN20_init(1),
+	dsc_regsDCN20_init(2),
+	dsc_regsDCN20_init(3);
+
+	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
+
+	dsc->max_image_width = 6016;
+
+	return &dsc->base;
+}
+
+static void dcn32_destroy_resource_pool(struct resource_pool **pool)
+{
+	struct dcn32_resource_pool *dcn32_pool = TO_DCN32_RES_POOL(*pool);
+
+	dcn32_resource_destruct(dcn32_pool);
+	kfree(dcn32_pool);
+	*pool = NULL;
+}
+
+bool dcn32_acquire_post_bldn_3dlut(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		int mpcc_id,
+		struct dc_3dlut **lut,
+		struct dc_transfer_func **shaper)
+{
+	bool ret = false;
+
+	ASSERT(*lut == NULL && *shaper == NULL);
+	*lut = NULL;
+	*shaper = NULL;
+
+	if (!res_ctx->is_mpc_3dlut_acquired[mpcc_id]) {
+		*lut = pool->mpc_lut[mpcc_id];
+		*shaper = pool->mpc_shaper[mpcc_id];
+		res_ctx->is_mpc_3dlut_acquired[mpcc_id] = true;
+		ret = true;
+	}
+	return ret;
+}
+
+bool dcn32_release_post_bldn_3dlut(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		struct dc_3dlut **lut,
+		struct dc_transfer_func **shaper)
+{
+	int i;
+	bool ret = false;
+
+	for (i = 0; i < pool->res_cap->num_mpc_3dlut; i++) {
+		if (pool->mpc_lut[i] == *lut && pool->mpc_shaper[i] == *shaper) {
+			res_ctx->is_mpc_3dlut_acquired[i] = false;
+			pool->mpc_lut[i]->state.raw = 0;
+			*lut = NULL;
+			*shaper = NULL;
+			ret = true;
+			break;
+		}
+	}
+	return ret;
+}
+
+static void dcn32_enable_phantom_plane(struct dc *dc,
+		struct dc_state *context,
+		struct dc_stream_state *phantom_stream,
+		unsigned int dc_pipe_idx)
+{
+	struct dc_plane_state *phantom_plane = NULL;
+	struct dc_plane_state *prev_phantom_plane = NULL;
+	struct pipe_ctx *curr_pipe = &context->res_ctx.pipe_ctx[dc_pipe_idx];
+
+	while (curr_pipe) {
+		if (curr_pipe->top_pipe && curr_pipe->top_pipe->plane_state == curr_pipe->plane_state)
+			phantom_plane = prev_phantom_plane;
+		else
+			phantom_plane = dc_create_plane_state(dc);
+
+		memcpy(&phantom_plane->address, &curr_pipe->plane_state->address, sizeof(phantom_plane->address));
+		memcpy(&phantom_plane->scaling_quality, &curr_pipe->plane_state->scaling_quality,
+				sizeof(phantom_plane->scaling_quality));
+		memcpy(&phantom_plane->src_rect, &curr_pipe->plane_state->src_rect, sizeof(phantom_plane->src_rect));
+		memcpy(&phantom_plane->dst_rect, &curr_pipe->plane_state->dst_rect, sizeof(phantom_plane->dst_rect));
+		memcpy(&phantom_plane->clip_rect, &curr_pipe->plane_state->clip_rect, sizeof(phantom_plane->clip_rect));
+		memcpy(&phantom_plane->plane_size, &curr_pipe->plane_state->plane_size,
+				sizeof(phantom_plane->plane_size));
+		memcpy(&phantom_plane->tiling_info, &curr_pipe->plane_state->tiling_info,
+				sizeof(phantom_plane->tiling_info));
+		memcpy(&phantom_plane->dcc, &curr_pipe->plane_state->dcc, sizeof(phantom_plane->dcc));
+		phantom_plane->format = curr_pipe->plane_state->format;
+		phantom_plane->rotation = curr_pipe->plane_state->rotation;
+		phantom_plane->visible = curr_pipe->plane_state->visible;
+
+		/* Shadow pipe has small viewport. */
+		phantom_plane->clip_rect.y = 0;
+		phantom_plane->clip_rect.height = phantom_stream->src.height;
+
+		phantom_plane->is_phantom = true;
+
+		dc_add_plane_to_context(dc, phantom_stream, phantom_plane, context);
+
+		curr_pipe = curr_pipe->bottom_pipe;
+		prev_phantom_plane = phantom_plane;
+	}
+}
+
+static struct dc_stream_state *dcn32_enable_phantom_stream(struct dc *dc,
+		struct dc_state *context,
+		display_e2e_pipe_params_st *pipes,
+		unsigned int pipe_cnt,
+		unsigned int dc_pipe_idx)
+{
+	struct dc_stream_state *phantom_stream = NULL;
+	struct pipe_ctx *ref_pipe = &context->res_ctx.pipe_ctx[dc_pipe_idx];
+
+	phantom_stream = dc_create_stream_for_sink(ref_pipe->stream->sink);
+	phantom_stream->signal = SIGNAL_TYPE_VIRTUAL;
+	phantom_stream->dpms_off = true;
+	phantom_stream->mall_stream_config.type = SUBVP_PHANTOM;
+	phantom_stream->mall_stream_config.paired_stream = ref_pipe->stream;
+	ref_pipe->stream->mall_stream_config.type = SUBVP_MAIN;
+	ref_pipe->stream->mall_stream_config.paired_stream = phantom_stream;
+
+	/* stream has limited viewport and small timing */
+	memcpy(&phantom_stream->timing, &ref_pipe->stream->timing, sizeof(phantom_stream->timing));
+	memcpy(&phantom_stream->src, &ref_pipe->stream->src, sizeof(phantom_stream->src));
+	memcpy(&phantom_stream->dst, &ref_pipe->stream->dst, sizeof(phantom_stream->dst));
+	DC_FP_START();
+	dcn32_set_phantom_stream_timing(dc, context, ref_pipe, phantom_stream, pipes, pipe_cnt, dc_pipe_idx);
+	DC_FP_END();
+
+	dc_add_stream_to_ctx(dc, context, phantom_stream);
+	return phantom_stream;
+}
+
+void dcn32_retain_phantom_pipes(struct dc *dc, struct dc_state *context)
+{
+	int i;
+	struct dc_plane_state *phantom_plane = NULL;
+	struct dc_stream_state *phantom_stream = NULL;
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+
+		if (resource_is_pipe_type(pipe, OTG_MASTER) &&
+				resource_is_pipe_type(pipe, DPP_PIPE) &&
+				pipe->stream->mall_stream_config.type == SUBVP_PHANTOM) {
+			phantom_plane = pipe->plane_state;
+			phantom_stream = pipe->stream;
+
+			dc_plane_state_retain(phantom_plane);
+			dc_stream_retain(phantom_stream);
+		}
+	}
+}
+
+// return true if removed piped from ctx, false otherwise
+bool dcn32_remove_phantom_pipes(struct dc *dc, struct dc_state *context, bool fast_update)
+{
+	int i;
+	bool removed_pipe = false;
+	struct dc_plane_state *phantom_plane = NULL;
+	struct dc_stream_state *phantom_stream = NULL;
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+		// build scaling params for phantom pipes
+		if (pipe->plane_state && pipe->stream && pipe->stream->mall_stream_config.type == SUBVP_PHANTOM) {
+			phantom_plane = pipe->plane_state;
+			phantom_stream = pipe->stream;
+
+			dc_rem_all_planes_for_stream(dc, pipe->stream, context);
+			dc_remove_stream_from_ctx(dc, context, pipe->stream);
+
+			/* Ref count is incremented on allocation and also when added to the context.
+			 * Therefore we must call release for the the phantom plane and stream once
+			 * they are removed from the ctx to finally decrement the refcount to 0 to free.
+			 */
+			dc_plane_state_release(phantom_plane);
+			dc_stream_release(phantom_stream);
+
+			removed_pipe = true;
+		}
+
+		/* For non-full updates, a shallow copy of the current state
+		 * is created. In this case we don't want to erase the current
+		 * state (there can be 2 HIRQL threads, one in flip, and one in
+		 * checkMPO) that can cause a race condition.
+		 *
+		 * This is just a workaround, needs a proper fix.
+		 */
+		if (!fast_update) {
+			// Clear all phantom stream info
+			if (pipe->stream) {
+				pipe->stream->mall_stream_config.type = SUBVP_NONE;
+				pipe->stream->mall_stream_config.paired_stream = NULL;
+			}
+
+			if (pipe->plane_state) {
+				pipe->plane_state->is_phantom = false;
+			}
+		}
+	}
+	return removed_pipe;
+}
+
+/* TODO: Input to this function should indicate which pipe indexes (or streams)
+ * require a phantom pipe / stream
+ */
+void dcn32_add_phantom_pipes(struct dc *dc, struct dc_state *context,
+		display_e2e_pipe_params_st *pipes,
+		unsigned int pipe_cnt,
+		unsigned int index)
+{
+	struct dc_stream_state *phantom_stream = NULL;
+	unsigned int i;
+
+	// The index of the DC pipe passed into this function is guarenteed to
+	// be a valid candidate for SubVP (i.e. has a plane, stream, doesn't
+	// already have phantom pipe assigned, etc.) by previous checks.
+	phantom_stream = dcn32_enable_phantom_stream(dc, context, pipes, pipe_cnt, index);
+	dcn32_enable_phantom_plane(dc, context, phantom_stream, index);
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+
+		// Build scaling params for phantom pipes which were newly added.
+		// We determine which phantom pipes were added by comparing with
+		// the phantom stream.
+		if (pipe->plane_state && pipe->stream && pipe->stream == phantom_stream &&
+				pipe->stream->mall_stream_config.type == SUBVP_PHANTOM) {
+			pipe->stream->use_dynamic_meta = false;
+			pipe->plane_state->flip_immediate = false;
+			if (!resource_build_scaling_params(pipe)) {
+				// Log / remove phantom pipes since failed to build scaling params
+			}
+		}
+	}
+}
+
+static bool dml1_validate(struct dc *dc, struct dc_state *context, bool fast_validate)
+{
+	bool out = false;
+
+	BW_VAL_TRACE_SETUP();
+
+	int vlevel = 0;
+	int pipe_cnt = 0;
+	display_e2e_pipe_params_st *pipes = kzalloc(dc->res_pool->pipe_count * sizeof(display_e2e_pipe_params_st), GFP_KERNEL);
+	struct mall_temp_config mall_temp_config;
+
+	/* To handle Freesync properly, setting FreeSync DML parameters
+	 * to its default state for the first stage of validation
+	 */
+	context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching = false;
+	context->bw_ctx.dml.soc.dram_clock_change_requirement_final = true;
+
+	DC_LOGGER_INIT(dc->ctx->logger);
+
+	/* For fast validation, there are situations where a shallow copy of
+	 * of the dc->current_state is created for the validation. In this case
+	 * we want to save and restore the mall config because we always
+	 * teardown subvp at the beginning of validation (and don't attempt
+	 * to add it back if it's fast validation). If we don't restore the
+	 * subvp config in cases of fast validation + shallow copy of the
+	 * dc->current_state, the dc->current_state will have a partially
+	 * removed subvp state when we did not intend to remove it.
+	 */
+	if (fast_validate) {
+		memset(&mall_temp_config, 0, sizeof(mall_temp_config));
+		dcn32_save_mall_state(dc, context, &mall_temp_config);
+	}
+
+	BW_VAL_TRACE_COUNT();
+
+	DC_FP_START();
+	out = dcn32_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, fast_validate);
+	DC_FP_END();
+
+	if (fast_validate)
+		dcn32_restore_mall_state(dc, context, &mall_temp_config);
+
+	if (pipe_cnt == 0)
+		goto validate_out;
+
+	if (!out)
+		goto validate_fail;
+
+	BW_VAL_TRACE_END_VOLTAGE_LEVEL();
+
+	if (fast_validate) {
+		BW_VAL_TRACE_SKIP(fast);
+		goto validate_out;
+	}
+
+	dc->res_pool->funcs->calculate_wm_and_dlg(dc, context, pipes, pipe_cnt, vlevel);
+
+	dcn32_override_min_req_memclk(dc, context);
+
+	BW_VAL_TRACE_END_WATERMARKS();
+
+	goto validate_out;
+
+validate_fail:
+	DC_LOG_WARNING("Mode Validation Warning: %s failed validation.\n",
+		dml_get_status_message(context->bw_ctx.dml.vba.ValidationStatus[context->bw_ctx.dml.vba.soc.num_states]));
+
+	BW_VAL_TRACE_SKIP(fail);
+	out = false;
+
+validate_out:
+	kfree(pipes);
+
+	BW_VAL_TRACE_FINISH();
+
+	return out;
+}
+
+bool dcn32_validate_bandwidth(struct dc *dc,
+		struct dc_state *context,
+		bool fast_validate)
+{
+	bool out = false;
+
+	if (dc->debug.using_dml2)
+		out = dml2_validate(dc, context, fast_validate);
+	else
+		out = dml1_validate(dc, context, fast_validate);
+	return out;
+}
+
+int dcn32_populate_dml_pipes_from_context(
+	struct dc *dc, struct dc_state *context,
+	display_e2e_pipe_params_st *pipes,
+	bool fast_validate)
+{
+	int i, pipe_cnt;
+	struct resource_context *res_ctx = &context->res_ctx;
+	struct pipe_ctx *pipe = NULL;
+	bool subvp_in_use = false;
+	struct dc_crtc_timing *timing;
+
+	dcn20_populate_dml_pipes_from_context(dc, context, pipes, fast_validate);
+
+	for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) {
+
+		if (!res_ctx->pipe_ctx[i].stream)
+			continue;
+		pipe = &res_ctx->pipe_ctx[i];
+		timing = &pipe->stream->timing;
+
+		pipes[pipe_cnt].pipe.src.gpuvm = true;
+		DC_FP_START();
+		dcn32_zero_pipe_dcc_fraction(pipes, pipe_cnt);
+		DC_FP_END();
+		pipes[pipe_cnt].pipe.dest.vfront_porch = timing->v_front_porch;
+		pipes[pipe_cnt].pipe.dest.odm_combine_policy = dm_odm_combine_policy_dal;
+		pipes[pipe_cnt].pipe.src.gpuvm_min_page_size_kbytes = 256; // according to spreadsheet
+		pipes[pipe_cnt].pipe.src.unbounded_req_mode = false;
+		pipes[pipe_cnt].pipe.scale_ratio_depth.lb_depth = dm_lb_19;
+
+		/* Only populate DML input with subvp info for full updates.
+		 * This is just a workaround -- needs a proper fix.
+		 */
+		if (!fast_validate) {
+			switch (pipe->stream->mall_stream_config.type) {
+			case SUBVP_MAIN:
+				pipes[pipe_cnt].pipe.src.use_mall_for_pstate_change = dm_use_mall_pstate_change_sub_viewport;
+				subvp_in_use = true;
+				break;
+			case SUBVP_PHANTOM:
+				pipes[pipe_cnt].pipe.src.use_mall_for_pstate_change = dm_use_mall_pstate_change_phantom_pipe;
+				pipes[pipe_cnt].pipe.src.use_mall_for_static_screen = dm_use_mall_static_screen_disable;
+				// Disallow unbounded req for SubVP according to DCHUB programming guide
+				pipes[pipe_cnt].pipe.src.unbounded_req_mode = false;
+				break;
+			case SUBVP_NONE:
+				pipes[pipe_cnt].pipe.src.use_mall_for_pstate_change = dm_use_mall_pstate_change_disable;
+				pipes[pipe_cnt].pipe.src.use_mall_for_static_screen = dm_use_mall_static_screen_disable;
+				break;
+			default:
+				break;
+			}
+		}
+
+		pipes[pipe_cnt].dout.dsc_input_bpc = 0;
+		if (pipes[pipe_cnt].dout.dsc_enable) {
+			switch (timing->display_color_depth) {
+			case COLOR_DEPTH_888:
+				pipes[pipe_cnt].dout.dsc_input_bpc = 8;
+				break;
+			case COLOR_DEPTH_101010:
+				pipes[pipe_cnt].dout.dsc_input_bpc = 10;
+				break;
+			case COLOR_DEPTH_121212:
+				pipes[pipe_cnt].dout.dsc_input_bpc = 12;
+				break;
+			default:
+				ASSERT(0);
+				break;
+			}
+		}
+
+
+		pipe_cnt++;
+	}
+
+	/* For DET allocation, we don't want to use DML policy (not optimal for utilizing all
+	 * the DET available for each pipe). Use the DET override input to maintain our driver
+	 * policy.
+	 */
+	dcn32_set_det_allocations(dc, context, pipes);
+
+	// In general cases we want to keep the dram clock change requirement
+	// (prefer configs that support MCLK switch). Only override to false
+	// for SubVP
+	if (context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching || subvp_in_use)
+		context->bw_ctx.dml.soc.dram_clock_change_requirement_final = false;
+	else
+		context->bw_ctx.dml.soc.dram_clock_change_requirement_final = true;
+
+	return pipe_cnt;
+}
+
+static struct dc_cap_funcs cap_funcs = {
+	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap,
+	.get_subvp_en = dcn32_subvp_in_use,
+};
+
+void dcn32_calculate_wm_and_dlg(struct dc *dc, struct dc_state *context,
+				display_e2e_pipe_params_st *pipes,
+				int pipe_cnt,
+				int vlevel)
+{
+    DC_FP_START();
+    dcn32_calculate_wm_and_dlg_fpu(dc, context, pipes, pipe_cnt, vlevel);
+    DC_FP_END();
+}
+
+static void dcn32_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
+{
+	DC_FP_START();
+	dcn32_update_bw_bounding_box_fpu(dc, bw_params);
+	DC_FP_END();
+}
+
+static struct resource_funcs dcn32_res_pool_funcs = {
+	.destroy = dcn32_destroy_resource_pool,
+	.link_enc_create = dcn32_link_encoder_create,
+	.link_enc_create_minimal = NULL,
+	.panel_cntl_create = dcn32_panel_cntl_create,
+	.validate_bandwidth = dcn32_validate_bandwidth,
+	.calculate_wm_and_dlg = dcn32_calculate_wm_and_dlg,
+	.populate_dml_pipes = dcn32_populate_dml_pipes_from_context,
+	.acquire_free_pipe_as_secondary_dpp_pipe = dcn32_acquire_free_pipe_as_secondary_dpp_pipe,
+	.acquire_free_pipe_as_secondary_opp_head = dcn32_acquire_free_pipe_as_secondary_opp_head,
+	.release_pipe = dcn20_release_pipe,
+	.add_stream_to_ctx = dcn30_add_stream_to_ctx,
+	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
+	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
+	.populate_dml_writeback_from_context = dcn30_populate_dml_writeback_from_context,
+	.set_mcif_arb_params = dcn30_set_mcif_arb_params,
+	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
+	.acquire_post_bldn_3dlut = dcn32_acquire_post_bldn_3dlut,
+	.release_post_bldn_3dlut = dcn32_release_post_bldn_3dlut,
+	.update_bw_bounding_box = dcn32_update_bw_bounding_box,
+	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
+	.update_soc_for_wm_a = dcn30_update_soc_for_wm_a,
+	.add_phantom_pipes = dcn32_add_phantom_pipes,
+	.remove_phantom_pipes = dcn32_remove_phantom_pipes,
+	.retain_phantom_pipes = dcn32_retain_phantom_pipes,
+	.save_mall_state = dcn32_save_mall_state,
+	.restore_mall_state = dcn32_restore_mall_state,
+};
+
+static uint32_t read_pipe_fuses(struct dc_context *ctx)
+{
+	uint32_t value = REG_READ(CC_DC_PIPE_DIS);
+	/* DCN32 support max 4 pipes */
+	value = value & 0xf;
+	return value;
+}
+
+
+static bool dcn32_resource_construct(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct dcn32_resource_pool *pool)
+{
+	int i, j;
+	struct dc_context *ctx = dc->ctx;
+	struct irq_service_init_data init_data;
+	struct ddc_service_init_data ddc_init_data = {0};
+	uint32_t pipe_fuses = 0;
+	uint32_t num_pipes  = 4;
+
+#undef REG_STRUCT
+#define REG_STRUCT bios_regs
+	bios_regs_init();
+
+#undef REG_STRUCT
+#define REG_STRUCT clk_src_regs
+	clk_src_regs_init(0, A),
+	clk_src_regs_init(1, B),
+	clk_src_regs_init(2, C),
+	clk_src_regs_init(3, D),
+	clk_src_regs_init(4, E);
+
+#undef REG_STRUCT
+#define REG_STRUCT abm_regs
+	abm_regs_init(0),
+	abm_regs_init(1),
+	abm_regs_init(2),
+	abm_regs_init(3);
+
+#undef REG_STRUCT
+#define REG_STRUCT dccg_regs
+	dccg_regs_init();
+
+	DC_FP_START();
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	pool->base.res_cap = &res_cap_dcn32;
+	/* max number of pipes for ASIC before checking for pipe fuses */
+	num_pipes  = pool->base.res_cap->num_timing_generator;
+	pipe_fuses = read_pipe_fuses(ctx);
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++)
+		if (pipe_fuses & 1 << i)
+			num_pipes--;
+
+	if (pipe_fuses & 1)
+		ASSERT(0); //Unexpected - Pipe 0 should always be fully functional!
+
+	if (pipe_fuses & CC_DC_PIPE_DIS__DC_FULL_DIS_MASK)
+		ASSERT(0); //Entire DCN is harvested!
+
+	/* within dml lib, initial value is hard coded, if ASIC pipe is fused, the
+	 * value will be changed, update max_num_dpp and max_num_otg for dml.
+	 */
+	dcn3_2_ip.max_num_dpp = num_pipes;
+	dcn3_2_ip.max_num_otg = num_pipes;
+
+	pool->base.funcs = &dcn32_res_pool_funcs;
+
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+	pool->base.timing_generator_count = num_pipes;
+	pool->base.pipe_count = num_pipes;
+	pool->base.mpcc_count = num_pipes;
+	dc->caps.max_downscale_ratio = 600;
+	dc->caps.i2c_speed_in_khz = 100;
+	dc->caps.i2c_speed_in_khz_hdcp = 100; /*1.4 w/a applied by default*/
+	/* TODO: Bring max_cursor_size back to 256 after subvp cursor corruption is fixed*/
+	dc->caps.max_cursor_size = 64;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.dmdata_alloc_size = 2048;
+	dc->caps.mall_size_per_mem_channel = 4;
+	dc->caps.mall_size_total = 0;
+	dc->caps.cursor_cache_size = dc->caps.max_cursor_size * dc->caps.max_cursor_size * 8;
+
+	dc->caps.cache_line_size = 64;
+	dc->caps.cache_num_ways = 16;
+
+	/* Calculate the available MALL space */
+	dc->caps.max_cab_allocation_bytes = dcn32_calc_num_avail_chans_for_mall(
+		dc, dc->ctx->dc_bios->vram_info.num_chans) *
+		dc->caps.mall_size_per_mem_channel * 1024 * 1024;
+	dc->caps.mall_size_total = dc->caps.max_cab_allocation_bytes;
+
+	dc->caps.subvp_fw_processing_delay_us = 15;
+	dc->caps.subvp_drr_max_vblank_margin_us = 40;
+	dc->caps.subvp_prefetch_end_to_mall_start_us = 15;
+	dc->caps.subvp_swath_height_margin_lines = 16;
+	dc->caps.subvp_pstate_allow_width_us = 20;
+	dc->caps.subvp_vertical_int_margin_us = 30;
+	dc->caps.subvp_drr_vblank_start_margin_us = 100; // 100us margin
+
+	dc->caps.max_slave_planes = 2;
+	dc->caps.max_slave_yuv_planes = 2;
+	dc->caps.max_slave_rgb_planes = 2;
+	dc->caps.post_blend_color_processing = true;
+	dc->caps.force_dp_tps4_for_cp2520 = true;
+	if (dc->config.forceHBR2CP2520)
+		dc->caps.force_dp_tps4_for_cp2520 = false;
+	dc->caps.dp_hpo = true;
+	dc->caps.dp_hdmi21_pcon_support = true;
+	dc->caps.edp_dsc_support = true;
+	dc->caps.extended_aux_timeout_support = true;
+	dc->caps.dmcub_support = true;
+	dc->caps.seamless_odm = true;
+	dc->caps.max_v_total = (1 << 15) - 1;
+
+	/* Color pipeline capabilities */
+	dc->caps.color.dpp.dcn_arch = 1;
+	dc->caps.color.dpp.input_lut_shared = 0;
+	dc->caps.color.dpp.icsc = 1;
+	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
+	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
+	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
+	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
+	dc->caps.color.dpp.post_csc = 1;
+	dc->caps.color.dpp.gamma_corr = 1;
+	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
+
+	dc->caps.color.dpp.hw_3d_lut = 1;
+	dc->caps.color.dpp.ogam_ram = 0;  // no OGAM in DPP since DCN1
+	// no OGAM ROM on DCN2 and later ASICs
+	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
+	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
+	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
+	dc->caps.color.dpp.ocsc = 0;
+
+	dc->caps.color.mpc.gamut_remap = 1;
+	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //4, configurable to be before or after BLND in MPCC
+	dc->caps.color.mpc.ogam_ram = 1;
+	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
+	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
+	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
+	dc->caps.color.mpc.ocsc = 1;
+
+	/* Use pipe context based otg sync logic */
+	dc->config.use_pipe_ctx_sync_logic = true;
+
+	dc->config.dc_mode_clk_limit_support = true;
+	/* read VBIOS LTTPR caps */
+	{
+		if (ctx->dc_bios->funcs->get_lttpr_caps) {
+			enum bp_result bp_query_result;
+			uint8_t is_vbios_lttpr_enable = 0;
+
+			bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
+			dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
+		}
+
+		/* interop bit is implicit */
+		{
+			dc->caps.vbios_lttpr_aware = true;
+		}
+	}
+
+	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
+		dc->debug = debug_defaults_drv;
+
+	// Init the vm_helper
+	if (dc->vm_helper)
+		vm_helper_init(dc->vm_helper, 16);
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	/* Clock Sources for Pixel Clock*/
+	pool->base.clock_sources[DCN32_CLK_SRC_PLL0] =
+			dcn32_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL0,
+				&clk_src_regs[0], false);
+	pool->base.clock_sources[DCN32_CLK_SRC_PLL1] =
+			dcn32_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL1,
+				&clk_src_regs[1], false);
+	pool->base.clock_sources[DCN32_CLK_SRC_PLL2] =
+			dcn32_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL2,
+				&clk_src_regs[2], false);
+	pool->base.clock_sources[DCN32_CLK_SRC_PLL3] =
+			dcn32_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL3,
+				&clk_src_regs[3], false);
+	pool->base.clock_sources[DCN32_CLK_SRC_PLL4] =
+			dcn32_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL4,
+				&clk_src_regs[4], false);
+
+	pool->base.clk_src_count = DCN32_CLK_SRC_TOTAL;
+
+	/* todo: not reuse phy_pll registers */
+	pool->base.dp_clock_source =
+			dcn32_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_ID_DP_DTO,
+				&clk_src_regs[0], true);
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	/* DCCG */
+	pool->base.dccg = dccg32_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
+	if (pool->base.dccg == NULL) {
+		dm_error("DC: failed to create dccg!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* DML */
+	dml_init_instance(&dc->dml, &dcn3_2_soc, &dcn3_2_ip, DML_PROJECT_DCN32);
+
+	/* IRQ Service */
+	init_data.ctx = dc->ctx;
+	pool->base.irqs = dal_irq_service_dcn32_create(&init_data);
+	if (!pool->base.irqs)
+		goto create_fail;
+
+	/* HUBBUB */
+	pool->base.hubbub = dcn32_hubbub_create(ctx);
+	if (pool->base.hubbub == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create hubbub!\n");
+		goto create_fail;
+	}
+
+	/* HUBPs, DPPs, OPPs, TGs, ABMs */
+	for (i = 0, j = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+
+		/* if pipe is disabled, skip instance of HW pipe,
+		 * i.e, skip ASIC register instance
+		 */
+		if (pipe_fuses & 1 << i)
+			continue;
+
+		/* HUBPs */
+		pool->base.hubps[j] = dcn32_hubp_create(ctx, i);
+		if (pool->base.hubps[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create hubps!\n");
+			goto create_fail;
+		}
+
+		/* DPPs */
+		pool->base.dpps[j] = dcn32_dpp_create(ctx, i);
+		if (pool->base.dpps[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create dpps!\n");
+			goto create_fail;
+		}
+
+		/* OPPs */
+		pool->base.opps[j] = dcn32_opp_create(ctx, i);
+		if (pool->base.opps[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create output pixel processor!\n");
+			goto create_fail;
+		}
+
+		/* TGs */
+		pool->base.timing_generators[j] = dcn32_timing_generator_create(
+				ctx, i);
+		if (pool->base.timing_generators[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto create_fail;
+		}
+
+		/* ABMs */
+		pool->base.multiple_abms[j] = dmub_abm_create(ctx,
+				&abm_regs[i],
+				&abm_shift,
+				&abm_mask);
+		if (pool->base.multiple_abms[j] == NULL) {
+			dm_error("DC: failed to create abm for pipe %d!\n", i);
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+
+		/* index for resource pool arrays for next valid pipe */
+		j++;
+	}
+
+	/* PSR */
+	pool->base.psr = dmub_psr_create(ctx);
+	if (pool->base.psr == NULL) {
+		dm_error("DC: failed to create psr obj!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* MPCCs */
+	pool->base.mpc = dcn32_mpc_create(ctx, pool->base.res_cap->num_timing_generator, pool->base.res_cap->num_mpc_3dlut);
+	if (pool->base.mpc == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mpc!\n");
+		goto create_fail;
+	}
+
+	/* DSCs */
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		pool->base.dscs[i] = dcn32_dsc_create(ctx, i);
+		if (pool->base.dscs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create display stream compressor %d!\n", i);
+			goto create_fail;
+		}
+	}
+
+	/* DWB */
+	if (!dcn32_dwbc_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create dwbc!\n");
+		goto create_fail;
+	}
+
+	/* MMHUBBUB */
+	if (!dcn32_mmhubbub_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mcif_wb!\n");
+		goto create_fail;
+	}
+
+	/* AUX and I2C */
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.engines[i] = dcn32_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto create_fail;
+		}
+		pool->base.hw_i2cs[i] = dcn32_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create hw i2c!!\n");
+			goto create_fail;
+		}
+		pool->base.sw_i2cs[i] = NULL;
+	}
+
+	/* Audio, HWSeq, Stream Encoders including HPO and virtual, MPC 3D LUTs */
+	if (!resource_construct(num_virtual_links, dc, &pool->base,
+			&res_create_funcs))
+		goto create_fail;
+
+	/* HW Sequencer init functions and Plane caps */
+	dcn32_hw_sequencer_init_functions(dc);
+
+	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->cap_funcs = cap_funcs;
+
+	if (dc->ctx->dc_bios->fw_info.oem_i2c_present) {
+		ddc_init_data.ctx = dc->ctx;
+		ddc_init_data.link = NULL;
+		ddc_init_data.id.id = dc->ctx->dc_bios->fw_info.oem_i2c_obj_id;
+		ddc_init_data.id.enum_id = 0;
+		ddc_init_data.id.type = OBJECT_TYPE_GENERIC;
+		pool->base.oem_device = dc->link_srv->create_ddc_service(&ddc_init_data);
+	} else {
+		pool->base.oem_device = NULL;
+	}
+
+	dc->dml2_options.dcn_pipe_count = pool->base.pipe_count;
+	dc->dml2_options.use_native_pstate_optimization = false;
+	dc->dml2_options.use_native_soc_bb_construction = true;
+	dc->dml2_options.minimize_dispclk_using_odm = true;
+
+	dc->dml2_options.callbacks.dc = dc;
+	dc->dml2_options.callbacks.build_scaling_params = &resource_build_scaling_params;
+	dc->dml2_options.callbacks.can_support_mclk_switch_using_fw_based_vblank_stretch = &dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch;
+	dc->dml2_options.callbacks.acquire_secondary_pipe_for_mpc_odm = &dc_resource_acquire_secondary_pipe_for_mpc_odm_legacy;
+	dc->dml2_options.callbacks.update_pipes_for_stream_with_slice_count = &resource_update_pipes_for_stream_with_slice_count;
+	dc->dml2_options.callbacks.update_pipes_for_plane_with_slice_count = &resource_update_pipes_for_plane_with_slice_count;
+	dc->dml2_options.callbacks.get_mpc_slice_index = &resource_get_mpc_slice_index;
+	dc->dml2_options.callbacks.get_odm_slice_index = &resource_get_odm_slice_index;
+	dc->dml2_options.callbacks.get_opp_head = &resource_get_opp_head;
+
+	dc->dml2_options.svp_pstate.callbacks.dc = dc;
+	dc->dml2_options.svp_pstate.callbacks.add_plane_to_context = &dc_add_plane_to_context;
+	dc->dml2_options.svp_pstate.callbacks.add_stream_to_ctx = &dc_add_stream_to_ctx;
+	dc->dml2_options.svp_pstate.callbacks.build_scaling_params = &resource_build_scaling_params;
+	dc->dml2_options.svp_pstate.callbacks.create_plane = &dc_create_plane_state;
+	dc->dml2_options.svp_pstate.callbacks.remove_plane_from_context = &dc_remove_plane_from_context;
+	dc->dml2_options.svp_pstate.callbacks.remove_stream_from_ctx = &dc_remove_stream_from_ctx;
+	dc->dml2_options.svp_pstate.callbacks.create_stream_for_sink = &dc_create_stream_for_sink;
+	dc->dml2_options.svp_pstate.callbacks.plane_state_release = &dc_plane_state_release;
+	dc->dml2_options.svp_pstate.callbacks.stream_release = &dc_stream_release;
+	dc->dml2_options.svp_pstate.callbacks.release_dsc = &dcn20_release_dsc;
+
+	dc->dml2_options.svp_pstate.subvp_fw_processing_delay_us = dc->caps.subvp_fw_processing_delay_us;
+	dc->dml2_options.svp_pstate.subvp_prefetch_end_to_mall_start_us = dc->caps.subvp_prefetch_end_to_mall_start_us;
+	dc->dml2_options.svp_pstate.subvp_pstate_allow_width_us = dc->caps.subvp_pstate_allow_width_us;
+	dc->dml2_options.svp_pstate.subvp_swath_height_margin_lines = dc->caps.subvp_swath_height_margin_lines;
+
+	dc->dml2_options.svp_pstate.force_disable_subvp = dc->debug.force_disable_subvp;
+	dc->dml2_options.svp_pstate.force_enable_subvp = dc->debug.force_subvp_mclk_switch;
+
+	dc->dml2_options.mall_cfg.cache_line_size_bytes = dc->caps.cache_line_size;
+	dc->dml2_options.mall_cfg.cache_num_ways = dc->caps.cache_num_ways;
+	dc->dml2_options.mall_cfg.max_cab_allocation_bytes = dc->caps.max_cab_allocation_bytes;
+	dc->dml2_options.mall_cfg.mblk_height_4bpe_pixels = DCN3_2_MBLK_HEIGHT_4BPE;
+	dc->dml2_options.mall_cfg.mblk_height_8bpe_pixels = DCN3_2_MBLK_HEIGHT_8BPE;
+	dc->dml2_options.mall_cfg.mblk_size_bytes = DCN3_2_MALL_MBLK_SIZE_BYTES;
+	dc->dml2_options.mall_cfg.mblk_width_pixels = DCN3_2_MBLK_WIDTH;
+
+	dc->dml2_options.max_segments_per_hubp = 18;
+	dc->dml2_options.det_segment_size = DCN3_2_DET_SEG_SIZE;
+	dc->dml2_options.map_dc_pipes_with_callbacks = true;
+
+	if (ASICREV_IS_GC_11_0_3(dc->ctx->asic_id.hw_internal_rev) && (dc->config.sdpif_request_limit_words_per_umc == 0))
+		dc->config.sdpif_request_limit_words_per_umc = 16;
+
+	DC_FP_END();
+
+	return true;
+
+create_fail:
+
+	DC_FP_END();
+
+	dcn32_resource_destruct(pool);
+
+	return false;
+}
+
+struct resource_pool *dcn32_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc)
+{
+	struct dcn32_resource_pool *pool =
+		kzalloc(sizeof(struct dcn32_resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dcn32_resource_construct(init_data->num_virtual_links, dc, pool))
+		return &pool->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(pool);
+	return NULL;
+}
+
+/*
+ * Find the most optimal free pipe from res_ctx, which could be used as a
+ * secondary dpp pipe for input opp head pipe.
+ *
+ * a free pipe - a pipe in input res_ctx not yet used for any streams or
+ * planes.
+ * secondary dpp pipe - a pipe gets inserted to a head OPP pipe's MPC blending
+ * tree. This is typical used for rendering MPO planes or additional offset
+ * areas in MPCC combine.
+ *
+ * Hardware Transition Minimization Algorithm for Finding a Secondary DPP Pipe
+ * -------------------------------------------------------------------------
+ *
+ * PROBLEM:
+ *
+ * 1. There is a hardware limitation that a secondary DPP pipe cannot be
+ * transferred from one MPC blending tree to the other in a single frame.
+ * Otherwise it could cause glitches on the screen.
+ *
+ * For instance, we cannot transition from state 1 to state 2 in one frame. This
+ * is because PIPE1 is transferred from PIPE0's MPC blending tree over to
+ * PIPE2's MPC blending tree, which is not supported by hardware.
+ * To support this transition we need to first remove PIPE1 from PIPE0's MPC
+ * blending tree in one frame and then insert PIPE1 to PIPE2's MPC blending tree
+ * in the next frame. This is not optimal as it will delay the flip for two
+ * frames.
+ *
+ *	State 1:
+ *	PIPE0 -- secondary DPP pipe --> (PIPE1)
+ *	PIPE2 -- secondary DPP pipe --> NONE
+ *
+ *	State 2:
+ *	PIPE0 -- secondary DPP pipe --> NONE
+ *	PIPE2 -- secondary DPP pipe --> (PIPE1)
+ *
+ * 2. We want to in general minimize the unnecessary changes in pipe topology.
+ * If a pipe is already added in current blending tree and there are no changes
+ * to plane topology, we don't want to swap it with another free pipe
+ * unnecessarily in every update. Powering up and down a pipe would require a
+ * full update which delays the flip for 1 frame. If we use the original pipe
+ * we don't have to toggle its power. So we can flip faster.
+ */
+static int find_optimal_free_pipe_as_secondary_dpp_pipe(
+		const struct resource_context *cur_res_ctx,
+		struct resource_context *new_res_ctx,
+		const struct resource_pool *pool,
+		const struct pipe_ctx *new_opp_head)
+{
+	const struct pipe_ctx *cur_opp_head;
+	int free_pipe_idx;
+
+	cur_opp_head = &cur_res_ctx->pipe_ctx[new_opp_head->pipe_idx];
+	free_pipe_idx = resource_find_free_pipe_used_in_cur_mpc_blending_tree(
+			cur_res_ctx, new_res_ctx, cur_opp_head);
+
+	/* Up until here if we have not found a free secondary pipe, we will
+	 * need to wait for at least one frame to complete the transition
+	 * sequence.
+	 */
+	if (free_pipe_idx == FREE_PIPE_INDEX_NOT_FOUND)
+		free_pipe_idx = recource_find_free_pipe_not_used_in_cur_res_ctx(
+				cur_res_ctx, new_res_ctx, pool);
+
+	/* Up until here if we have not found a free secondary pipe, we will
+	 * need to wait for at least two frames to complete the transition
+	 * sequence. It really doesn't matter which pipe we decide take from
+	 * current enabled pipes. It won't save our frame time when we swap only
+	 * one pipe or more pipes.
+	 */
+	if (free_pipe_idx == FREE_PIPE_INDEX_NOT_FOUND)
+		free_pipe_idx = resource_find_free_pipe_used_as_cur_sec_dpp_in_mpcc_combine(
+				cur_res_ctx, new_res_ctx, pool);
+
+	if (free_pipe_idx == FREE_PIPE_INDEX_NOT_FOUND)
+		free_pipe_idx = resource_find_any_free_pipe(new_res_ctx, pool);
+
+	return free_pipe_idx;
+}
+
+static struct pipe_ctx *find_idle_secondary_pipe_check_mpo(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		const struct pipe_ctx *primary_pipe)
+{
+	int i;
+	struct pipe_ctx *secondary_pipe = NULL;
+	struct pipe_ctx *next_odm_mpo_pipe = NULL;
+	int primary_index, preferred_pipe_idx;
+	struct pipe_ctx *old_primary_pipe = NULL;
+
+	/*
+	 * Modified from find_idle_secondary_pipe
+	 * With windowed MPO and ODM, we want to avoid the case where we want a
+	 *  free pipe for the left side but the free pipe is being used on the
+	 *  right side.
+	 * Add check on current_state if the primary_pipe is the left side,
+	 *  to check the right side ( primary_pipe->next_odm_pipe ) to see if
+	 *  it is using a pipe for MPO ( primary_pipe->next_odm_pipe->bottom_pipe )
+	 * - If so, then don't use this pipe
+	 * EXCEPTION - 3 plane ( 2 MPO plane ) case
+	 * - in this case, the primary pipe has already gotten a free pipe for the
+	 *  MPO window in the left
+	 * - when it tries to get a free pipe for the MPO window on the right,
+	 *  it will see that it is already assigned to the right side
+	 *  ( primary_pipe->next_odm_pipe ).  But in this case, we want this
+	 *  free pipe, since it will be for the right side.  So add an
+	 *  additional condition, that skipping the free pipe on the right only
+	 *  applies if the primary pipe has no bottom pipe currently assigned
+	 */
+	if (primary_pipe) {
+		primary_index = primary_pipe->pipe_idx;
+		old_primary_pipe = &primary_pipe->stream->ctx->dc->current_state->res_ctx.pipe_ctx[primary_index];
+		if ((old_primary_pipe->next_odm_pipe) && (old_primary_pipe->next_odm_pipe->bottom_pipe)
+			&& (!primary_pipe->bottom_pipe))
+			next_odm_mpo_pipe = old_primary_pipe->next_odm_pipe->bottom_pipe;
+
+		preferred_pipe_idx = (pool->pipe_count - 1) - primary_pipe->pipe_idx;
+		if ((res_ctx->pipe_ctx[preferred_pipe_idx].stream == NULL) &&
+			!(next_odm_mpo_pipe && next_odm_mpo_pipe->pipe_idx == preferred_pipe_idx)) {
+			secondary_pipe = &res_ctx->pipe_ctx[preferred_pipe_idx];
+			secondary_pipe->pipe_idx = preferred_pipe_idx;
+		}
+	}
+
+	/*
+	 * search backwards for the second pipe to keep pipe
+	 * assignment more consistent
+	 */
+	if (!secondary_pipe)
+		for (i = pool->pipe_count - 1; i >= 0; i--) {
+			if ((res_ctx->pipe_ctx[i].stream == NULL) &&
+				!(next_odm_mpo_pipe && next_odm_mpo_pipe->pipe_idx == i)) {
+				secondary_pipe = &res_ctx->pipe_ctx[i];
+				secondary_pipe->pipe_idx = i;
+				break;
+			}
+		}
+
+	return secondary_pipe;
+}
+
+static struct pipe_ctx *dcn32_acquire_idle_pipe_for_head_pipe_in_layer(
+		struct dc_state *state,
+		const struct resource_pool *pool,
+		struct dc_stream_state *stream,
+		const struct pipe_ctx *head_pipe)
+{
+	struct resource_context *res_ctx = &state->res_ctx;
+	struct pipe_ctx *idle_pipe, *pipe;
+	struct resource_context *old_ctx = &stream->ctx->dc->current_state->res_ctx;
+	int head_index;
+
+	if (!head_pipe)
+		ASSERT(0);
+
+	/*
+	 * Modified from dcn20_acquire_idle_pipe_for_layer
+	 * Check if head_pipe in old_context already has bottom_pipe allocated.
+	 * - If so, check if that pipe is available in the current context.
+	 * --  If so, reuse pipe from old_context
+	 */
+	head_index = head_pipe->pipe_idx;
+	pipe = &old_ctx->pipe_ctx[head_index];
+	if (pipe->bottom_pipe && res_ctx->pipe_ctx[pipe->bottom_pipe->pipe_idx].stream == NULL) {
+		idle_pipe = &res_ctx->pipe_ctx[pipe->bottom_pipe->pipe_idx];
+		idle_pipe->pipe_idx = pipe->bottom_pipe->pipe_idx;
+	} else {
+		idle_pipe = find_idle_secondary_pipe_check_mpo(res_ctx, pool, head_pipe);
+		if (!idle_pipe)
+			return NULL;
+	}
+
+	idle_pipe->stream = head_pipe->stream;
+	idle_pipe->stream_res.tg = head_pipe->stream_res.tg;
+	idle_pipe->stream_res.opp = head_pipe->stream_res.opp;
+
+	idle_pipe->plane_res.hubp = pool->hubps[idle_pipe->pipe_idx];
+	idle_pipe->plane_res.ipp = pool->ipps[idle_pipe->pipe_idx];
+	idle_pipe->plane_res.dpp = pool->dpps[idle_pipe->pipe_idx];
+	idle_pipe->plane_res.mpcc_inst = pool->dpps[idle_pipe->pipe_idx]->inst;
+
+	return idle_pipe;
+}
+
+static int find_optimal_free_pipe_as_secondary_opp_head(
+		const struct resource_context *cur_res_ctx,
+		struct resource_context *new_res_ctx,
+		const struct resource_pool *pool,
+		const struct pipe_ctx *new_otg_master)
+{
+	const struct pipe_ctx *cur_otg_master;
+	int free_pipe_idx;
+
+	cur_otg_master =  &cur_res_ctx->pipe_ctx[new_otg_master->pipe_idx];
+	free_pipe_idx = resource_find_free_pipe_used_as_sec_opp_head_by_cur_otg_master(
+			cur_res_ctx, new_res_ctx, cur_otg_master);
+
+	/* Up until here if we have not found a free secondary pipe, we will
+	 * need to wait for at least one frame to complete the transition
+	 * sequence.
+	 */
+	if (free_pipe_idx == FREE_PIPE_INDEX_NOT_FOUND)
+		free_pipe_idx = recource_find_free_pipe_not_used_in_cur_res_ctx(
+				cur_res_ctx, new_res_ctx, pool);
+
+	if (free_pipe_idx == FREE_PIPE_INDEX_NOT_FOUND)
+		free_pipe_idx = resource_find_any_free_pipe(new_res_ctx, pool);
+
+	return free_pipe_idx;
+}
+
+struct pipe_ctx *dcn32_acquire_free_pipe_as_secondary_dpp_pipe(
+		const struct dc_state *cur_ctx,
+		struct dc_state *new_ctx,
+		const struct resource_pool *pool,
+		const struct pipe_ctx *opp_head_pipe)
+{
+
+	int free_pipe_idx;
+	struct pipe_ctx *free_pipe;
+
+	if (!opp_head_pipe->stream->ctx->dc->config.enable_windowed_mpo_odm)
+		return dcn32_acquire_idle_pipe_for_head_pipe_in_layer(
+				new_ctx, pool, opp_head_pipe->stream, opp_head_pipe);
+
+	free_pipe_idx = find_optimal_free_pipe_as_secondary_dpp_pipe(
+					&cur_ctx->res_ctx, &new_ctx->res_ctx,
+					pool, opp_head_pipe);
+	if (free_pipe_idx >= 0) {
+		free_pipe = &new_ctx->res_ctx.pipe_ctx[free_pipe_idx];
+		free_pipe->pipe_idx = free_pipe_idx;
+		free_pipe->stream = opp_head_pipe->stream;
+		free_pipe->stream_res.tg = opp_head_pipe->stream_res.tg;
+		free_pipe->stream_res.opp = opp_head_pipe->stream_res.opp;
+
+		free_pipe->plane_res.hubp = pool->hubps[free_pipe->pipe_idx];
+		free_pipe->plane_res.ipp = pool->ipps[free_pipe->pipe_idx];
+		free_pipe->plane_res.dpp = pool->dpps[free_pipe->pipe_idx];
+		free_pipe->plane_res.mpcc_inst =
+				pool->dpps[free_pipe->pipe_idx]->inst;
+	} else {
+		ASSERT(opp_head_pipe);
+		free_pipe = NULL;
+	}
+
+	return free_pipe;
+}
+
+struct pipe_ctx *dcn32_acquire_free_pipe_as_secondary_opp_head(
+		const struct dc_state *cur_ctx,
+		struct dc_state *new_ctx,
+		const struct resource_pool *pool,
+		const struct pipe_ctx *otg_master)
+{
+	int free_pipe_idx = find_optimal_free_pipe_as_secondary_opp_head(
+			&cur_ctx->res_ctx, &new_ctx->res_ctx,
+			pool, otg_master);
+	struct pipe_ctx *free_pipe;
+
+	if (free_pipe_idx >= 0) {
+		free_pipe = &new_ctx->res_ctx.pipe_ctx[free_pipe_idx];
+		free_pipe->pipe_idx = free_pipe_idx;
+		free_pipe->stream = otg_master->stream;
+		free_pipe->stream_res.tg = otg_master->stream_res.tg;
+		free_pipe->stream_res.dsc = NULL;
+		free_pipe->stream_res.opp = pool->opps[free_pipe_idx];
+		free_pipe->plane_res.mi = pool->mis[free_pipe_idx];
+		free_pipe->plane_res.hubp = pool->hubps[free_pipe_idx];
+		free_pipe->plane_res.ipp = pool->ipps[free_pipe_idx];
+		free_pipe->plane_res.xfm = pool->transforms[free_pipe_idx];
+		free_pipe->plane_res.dpp = pool->dpps[free_pipe_idx];
+		free_pipe->plane_res.mpcc_inst = pool->dpps[free_pipe_idx]->inst;
+		if (free_pipe->stream->timing.flags.DSC == 1) {
+			dcn20_acquire_dsc(free_pipe->stream->ctx->dc,
+					&new_ctx->res_ctx,
+					&free_pipe->stream_res.dsc,
+					free_pipe_idx);
+			ASSERT(free_pipe->stream_res.dsc);
+			if (free_pipe->stream_res.dsc == NULL) {
+				memset(free_pipe, 0, sizeof(*free_pipe));
+				free_pipe = NULL;
+			}
+		}
+	} else {
+		ASSERT(otg_master);
+		free_pipe = NULL;
+	}
+
+	return free_pipe;
+}
+
+unsigned int dcn32_calc_num_avail_chans_for_mall(struct dc *dc, int num_chans)
+{
+	/*
+	 * DCN32 and DCN321 SKUs may have different sizes for MALL
+	 *  but we may not be able to access all the MALL space.
+	 *  If the num_chans is power of 2, then we can access all
+	 *  of the available MALL space.  Otherwise, we can only
+	 *  access:
+	 *
+	 *  max_cab_size_in_bytes = total_cache_size_in_bytes *
+	 *    ((2^floor(log2(num_chans)))/num_chans)
+	 *
+	 * Calculating the MALL sizes for all available SKUs, we
+	 *  have come up with the follow simplified check.
+	 * - we have max_chans which provides the max MALL size.
+	 *  Each chans supports 4MB of MALL so:
+	 *
+	 *  total_cache_size_in_bytes = max_chans * 4 MB
+	 *
+	 * - we have avail_chans which shows the number of channels
+	 *  we can use if we can't access the entire MALL space.
+	 *  It is generally half of max_chans
+	 * - so we use the following checks:
+	 *
+	 *   if (num_chans == max_chans), return max_chans
+	 *   if (num_chans < max_chans), return avail_chans
+	 *
+	 * - exception is GC_11_0_0 where we can't access max_chans,
+	 *  so we define max_avail_chans as the maximum available
+	 *  MALL space
+	 *
+	 */
+	int gc_11_0_0_max_chans = 48;
+	int gc_11_0_0_max_avail_chans = 32;
+	int gc_11_0_0_avail_chans = 16;
+	int gc_11_0_3_max_chans = 16;
+	int gc_11_0_3_avail_chans = 8;
+	int gc_11_0_2_max_chans = 8;
+	int gc_11_0_2_avail_chans = 4;
+
+	if (ASICREV_IS_GC_11_0_0(dc->ctx->asic_id.hw_internal_rev)) {
+		return (num_chans == gc_11_0_0_max_chans) ?
+			gc_11_0_0_max_avail_chans : gc_11_0_0_avail_chans;
+	} else if (ASICREV_IS_GC_11_0_2(dc->ctx->asic_id.hw_internal_rev)) {
+		return (num_chans == gc_11_0_2_max_chans) ?
+			gc_11_0_2_max_chans : gc_11_0_2_avail_chans;
+	} else { // if (ASICREV_IS_GC_11_0_3(dc->ctx->asic_id.hw_internal_rev)) {
+		return (num_chans == gc_11_0_3_max_chans) ?
+			gc_11_0_3_max_chans : gc_11_0_3_avail_chans;
+	}
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.h
new file mode 100644
index 000000000000..b931008114c9
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.h
@@ -0,0 +1,1263 @@
+/*
+ * Copyright 2020 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef _DCN32_RESOURCE_H_
+#define _DCN32_RESOURCE_H_
+
+#include "core_types.h"
+
+#define DCN3_2_DEFAULT_DET_SIZE 256
+#define DCN3_2_MAX_DET_SIZE 1152
+#define DCN3_2_MIN_DET_SIZE 128
+#define DCN3_2_MIN_COMPBUF_SIZE_KB 128
+#define DCN3_2_DET_SEG_SIZE 64
+#define DCN3_2_MALL_MBLK_SIZE_BYTES 65536 // 64 * 1024
+#define DCN3_2_MBLK_WIDTH 128
+#define DCN3_2_MBLK_HEIGHT_4BPE 128
+#define DCN3_2_MBLK_HEIGHT_8BPE 64
+#define DCN3_2_DCFCLK_DS_INIT_KHZ 10000 // Choose 10Mhz for init DCFCLK DS freq
+#define SUBVP_HIGH_REFRESH_LIST_LEN 4
+#define DCN3_2_MAX_SUBVP_PIXEL_RATE_MHZ 1800
+#define DCN3_2_VMIN_DISPCLK_HZ 717000000
+
+#define TO_DCN32_RES_POOL(pool)\
+	container_of(pool, struct dcn32_resource_pool, base)
+
+extern struct _vcs_dpi_ip_params_st dcn3_2_ip;
+extern struct _vcs_dpi_soc_bounding_box_st dcn3_2_soc;
+
+struct subvp_high_refresh_list {
+	int min_refresh;
+	int max_refresh;
+	struct resolution {
+		int width;
+		int height;
+	} res[SUBVP_HIGH_REFRESH_LIST_LEN];
+};
+
+struct dcn32_resource_pool {
+	struct resource_pool base;
+};
+
+struct resource_pool *dcn32_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc);
+
+struct panel_cntl *dcn32_panel_cntl_create(
+		const struct panel_cntl_init_data *init_data);
+
+bool dcn32_acquire_post_bldn_3dlut(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		int mpcc_id,
+		struct dc_3dlut **lut,
+		struct dc_transfer_func **shaper);
+
+bool dcn32_release_post_bldn_3dlut(
+		struct resource_context *res_ctx,
+		const struct resource_pool *pool,
+		struct dc_3dlut **lut,
+		struct dc_transfer_func **shaper);
+
+bool dcn32_remove_phantom_pipes(struct dc *dc,
+		struct dc_state *context, bool fast_update);
+
+void dcn32_retain_phantom_pipes(struct dc *dc,
+		struct dc_state *context);
+
+void dcn32_add_phantom_pipes(struct dc *dc,
+		struct dc_state *context,
+		display_e2e_pipe_params_st *pipes,
+		unsigned int pipe_cnt,
+		unsigned int index);
+
+bool dcn32_validate_bandwidth(struct dc *dc,
+		struct dc_state *context,
+		bool fast_validate);
+
+int dcn32_populate_dml_pipes_from_context(
+	struct dc *dc, struct dc_state *context,
+	display_e2e_pipe_params_st *pipes,
+	bool fast_validate);
+
+void dcn32_calculate_wm_and_dlg(
+		struct dc *dc, struct dc_state *context,
+		display_e2e_pipe_params_st *pipes,
+		int pipe_cnt,
+		int vlevel);
+
+uint32_t dcn32_helper_mall_bytes_to_ways(
+		struct dc *dc,
+		uint32_t total_size_in_mall_bytes);
+
+uint32_t dcn32_helper_calculate_mall_bytes_for_cursor(
+		struct dc *dc,
+		struct pipe_ctx *pipe_ctx,
+		bool ignore_cursor_buf);
+
+uint32_t dcn32_helper_calculate_num_ways_for_subvp(
+		struct dc *dc,
+		struct dc_state *context);
+
+void dcn32_merge_pipes_for_subvp(struct dc *dc,
+		struct dc_state *context);
+
+bool dcn32_all_pipes_have_stream_and_plane(struct dc *dc,
+		struct dc_state *context);
+
+bool dcn32_subvp_in_use(struct dc *dc,
+		struct dc_state *context);
+
+bool dcn32_mpo_in_use(struct dc_state *context);
+
+bool dcn32_any_surfaces_rotated(struct dc *dc, struct dc_state *context);
+bool dcn32_is_center_timing(struct pipe_ctx *pipe);
+bool dcn32_is_psr_capable(struct pipe_ctx *pipe);
+
+struct pipe_ctx *dcn32_acquire_free_pipe_as_secondary_dpp_pipe(
+		const struct dc_state *cur_ctx,
+		struct dc_state *new_ctx,
+		const struct resource_pool *pool,
+		const struct pipe_ctx *opp_head_pipe);
+
+struct pipe_ctx *dcn32_acquire_free_pipe_as_secondary_opp_head(
+		const struct dc_state *cur_ctx,
+		struct dc_state *new_ctx,
+		const struct resource_pool *pool,
+		const struct pipe_ctx *otg_master);
+
+void dcn32_release_pipe(struct dc_state *context,
+			struct pipe_ctx *pipe,
+			const struct resource_pool *pool);
+
+void dcn32_determine_det_override(struct dc *dc,
+		struct dc_state *context,
+		display_e2e_pipe_params_st *pipes);
+
+void dcn32_set_det_allocations(struct dc *dc, struct dc_state *context,
+	display_e2e_pipe_params_st *pipes);
+
+void dcn32_save_mall_state(struct dc *dc,
+		struct dc_state *context,
+		struct mall_temp_config *temp_config);
+
+void dcn32_restore_mall_state(struct dc *dc,
+		struct dc_state *context,
+		struct mall_temp_config *temp_config);
+
+struct dc_stream_state *dcn32_can_support_mclk_switch_using_fw_based_vblank_stretch(struct dc *dc, const struct dc_state *context);
+
+bool dcn32_allow_subvp_with_active_margin(struct pipe_ctx *pipe);
+
+bool dcn32_allow_subvp_high_refresh_rate(struct dc *dc, struct dc_state *context, struct pipe_ctx *pipe);
+
+unsigned int dcn32_calc_num_avail_chans_for_mall(struct dc *dc, int num_chans);
+
+double dcn32_determine_max_vratio_prefetch(struct dc *dc, struct dc_state *context);
+
+bool dcn32_check_native_scaling_for_res(struct pipe_ctx *pipe, unsigned int width, unsigned int height);
+
+bool dcn32_subvp_drr_admissable(struct dc *dc, struct dc_state *context);
+
+bool dcn32_subvp_vblank_admissable(struct dc *dc, struct dc_state *context, int vlevel);
+
+/* definitions for run time init of reg offsets */
+
+/* CLK SRC */
+#define CS_COMMON_REG_LIST_DCN3_0_RI(index, pllid)                             \
+  SRI_ARR_ALPHABET(PIXCLK_RESYNC_CNTL, PHYPLL, index, pllid),                  \
+      SRII_ARR_2(PHASE, DP_DTO, 0, index),                                     \
+      SRII_ARR_2(PHASE, DP_DTO, 1, index),                                     \
+      SRII_ARR_2(PHASE, DP_DTO, 2, index),                                     \
+      SRII_ARR_2(PHASE, DP_DTO, 3, index),                                     \
+      SRII_ARR_2(MODULO, DP_DTO, 0, index),                                    \
+      SRII_ARR_2(MODULO, DP_DTO, 1, index),                                    \
+      SRII_ARR_2(MODULO, DP_DTO, 2, index),                                    \
+      SRII_ARR_2(MODULO, DP_DTO, 3, index),                                    \
+      SRII_ARR_2(PIXEL_RATE_CNTL, OTG, 0, index),                              \
+      SRII_ARR_2(PIXEL_RATE_CNTL, OTG, 1, index),                              \
+      SRII_ARR_2(PIXEL_RATE_CNTL, OTG, 2, index),                              \
+      SRII_ARR_2(PIXEL_RATE_CNTL, OTG, 3, index)
+
+/* ABM */
+#define ABM_DCN32_REG_LIST_RI(id)                                              \
+  SRI_ARR(DC_ABM1_HG_SAMPLE_RATE, ABM, id),                                    \
+      SRI_ARR(DC_ABM1_LS_SAMPLE_RATE, ABM, id),                                \
+      SRI_ARR(BL1_PWM_BL_UPDATE_SAMPLE_RATE, ABM, id),                         \
+      SRI_ARR(DC_ABM1_HG_MISC_CTRL, ABM, id),                                  \
+      SRI_ARR(DC_ABM1_IPCSC_COEFF_SEL, ABM, id),                               \
+      SRI_ARR(BL1_PWM_CURRENT_ABM_LEVEL, ABM, id),                             \
+      SRI_ARR(BL1_PWM_TARGET_ABM_LEVEL, ABM, id),                              \
+      SRI_ARR(BL1_PWM_USER_LEVEL, ABM, id),                                    \
+      SRI_ARR(DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES, ABM, id),                  \
+      SRI_ARR(DC_ABM1_HGLS_REG_READ_PROGRESS, ABM, id),                        \
+      SRI_ARR(DC_ABM1_ACE_OFFSET_SLOPE_0, ABM, id),                            \
+      SRI_ARR(DC_ABM1_ACE_THRES_12, ABM, id), NBIO_SR_ARR(BIOS_SCRATCH_2, id)
+
+/* Audio */
+#define AUD_COMMON_REG_LIST_RI(id)                                             \
+  SRI_ARR(AZALIA_F0_CODEC_ENDPOINT_INDEX, AZF0ENDPOINT, id),                   \
+      SRI_ARR(AZALIA_F0_CODEC_ENDPOINT_DATA, AZF0ENDPOINT, id),                \
+      SR_ARR(AZALIA_F0_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS, id),           \
+      SR_ARR(AZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES, id),     \
+      SR_ARR(AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES, id),             \
+      SR_ARR(DCCG_AUDIO_DTO_SOURCE, id), SR_ARR(DCCG_AUDIO_DTO0_MODULE, id),   \
+      SR_ARR(DCCG_AUDIO_DTO0_PHASE, id), SR_ARR(DCCG_AUDIO_DTO1_MODULE, id),   \
+      SR_ARR(DCCG_AUDIO_DTO1_PHASE, id)                                        \
+
+/* VPG */
+
+#define VPG_DCN3_REG_LIST_RI(id)                                               \
+  SRI_ARR(VPG_GENERIC_STATUS, VPG, id),                                        \
+      SRI_ARR(VPG_GENERIC_PACKET_ACCESS_CTRL, VPG, id),                        \
+      SRI_ARR(VPG_GENERIC_PACKET_DATA, VPG, id),                               \
+      SRI_ARR(VPG_GSP_FRAME_UPDATE_CTRL, VPG, id),                             \
+      SRI_ARR(VPG_GSP_IMMEDIATE_UPDATE_CTRL, VPG, id)
+
+/* AFMT */
+#define AFMT_DCN3_REG_LIST_RI(id)                                              \
+  SRI_ARR(AFMT_INFOFRAME_CONTROL0, AFMT, id),                                  \
+      SRI_ARR(AFMT_VBI_PACKET_CONTROL, AFMT, id),                              \
+      SRI_ARR(AFMT_AUDIO_PACKET_CONTROL, AFMT, id),                            \
+      SRI_ARR(AFMT_AUDIO_PACKET_CONTROL2, AFMT, id),                           \
+      SRI_ARR(AFMT_AUDIO_SRC_CONTROL, AFMT, id),                               \
+      SRI_ARR(AFMT_60958_0, AFMT, id), SRI_ARR(AFMT_60958_1, AFMT, id),        \
+      SRI_ARR(AFMT_60958_2, AFMT, id), SRI_ARR(AFMT_MEM_PWR, AFMT, id)
+
+/* APG */
+#define APG_DCN31_REG_LIST_RI(id)                                              \
+  SRI_ARR(APG_CONTROL, APG, id), SRI_ARR(APG_CONTROL2, APG, id),               \
+      SRI_ARR(APG_MEM_PWR, APG, id), SRI_ARR(APG_DBG_GEN_CONTROL, APG, id)
+
+/* Stream encoder */
+#define SE_DCN32_REG_LIST_RI(id)                                               \
+  SRI_ARR(AFMT_CNTL, DIG, id), SRI_ARR(DIG_FE_CNTL, DIG, id),                  \
+      SRI_ARR(HDMI_CONTROL, DIG, id), SRI_ARR(HDMI_DB_CONTROL, DIG, id),       \
+      SRI_ARR(HDMI_GC, DIG, id),                                               \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL0, DIG, id),                          \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL1, DIG, id),                          \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL2, DIG, id),                          \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL3, DIG, id),                          \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL4, DIG, id),                          \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL5, DIG, id),                          \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL6, DIG, id),                          \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL7, DIG, id),                          \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL8, DIG, id),                          \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL9, DIG, id),                          \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL10, DIG, id),                         \
+      SRI_ARR(HDMI_INFOFRAME_CONTROL0, DIG, id),                               \
+      SRI_ARR(HDMI_INFOFRAME_CONTROL1, DIG, id),                               \
+      SRI_ARR(HDMI_VBI_PACKET_CONTROL, DIG, id),                               \
+      SRI_ARR(HDMI_AUDIO_PACKET_CONTROL, DIG, id),                             \
+      SRI_ARR(HDMI_ACR_PACKET_CONTROL, DIG, id),                               \
+      SRI_ARR(HDMI_ACR_32_0, DIG, id), SRI_ARR(HDMI_ACR_32_1, DIG, id),        \
+      SRI_ARR(HDMI_ACR_44_0, DIG, id), SRI_ARR(HDMI_ACR_44_1, DIG, id),        \
+      SRI_ARR(HDMI_ACR_48_0, DIG, id), SRI_ARR(HDMI_ACR_48_1, DIG, id),        \
+      SRI_ARR(DP_DB_CNTL, DP, id), SRI_ARR(DP_MSA_MISC, DP, id),               \
+      SRI_ARR(DP_MSA_VBID_MISC, DP, id), SRI_ARR(DP_MSA_COLORIMETRY, DP, id),  \
+      SRI_ARR(DP_MSA_TIMING_PARAM1, DP, id),                                   \
+      SRI_ARR(DP_MSA_TIMING_PARAM2, DP, id),                                   \
+      SRI_ARR(DP_MSA_TIMING_PARAM3, DP, id),                                   \
+      SRI_ARR(DP_MSA_TIMING_PARAM4, DP, id),                                   \
+      SRI_ARR(DP_MSE_RATE_CNTL, DP, id), SRI_ARR(DP_MSE_RATE_UPDATE, DP, id),  \
+      SRI_ARR(DP_PIXEL_FORMAT, DP, id), SRI_ARR(DP_SEC_CNTL, DP, id),          \
+      SRI_ARR(DP_SEC_CNTL1, DP, id), SRI_ARR(DP_SEC_CNTL2, DP, id),            \
+      SRI_ARR(DP_SEC_CNTL5, DP, id), SRI_ARR(DP_SEC_CNTL6, DP, id),            \
+      SRI_ARR(DP_STEER_FIFO, DP, id), SRI_ARR(DP_VID_M, DP, id),               \
+      SRI_ARR(DP_VID_N, DP, id), SRI_ARR(DP_VID_STREAM_CNTL, DP, id),          \
+      SRI_ARR(DP_VID_TIMING, DP, id), SRI_ARR(DP_SEC_AUD_N, DP, id),           \
+      SRI_ARR(DP_SEC_TIMESTAMP, DP, id), SRI_ARR(DP_DSC_CNTL, DP, id),         \
+      SRI_ARR(DP_SEC_METADATA_TRANSMISSION, DP, id),                           \
+      SRI_ARR(HDMI_METADATA_PACKET_CONTROL, DIG, id),                          \
+      SRI_ARR(DP_SEC_FRAMING4, DP, id), SRI_ARR(DP_GSP11_CNTL, DP, id),        \
+      SRI_ARR(DME_CONTROL, DME, id),                                           \
+      SRI_ARR(DP_SEC_METADATA_TRANSMISSION, DP, id),                           \
+      SRI_ARR(HDMI_METADATA_PACKET_CONTROL, DIG, id),                          \
+      SRI_ARR(DIG_FE_CNTL, DIG, id), SRI_ARR(DIG_CLOCK_PATTERN, DIG, id),      \
+      SRI_ARR(DIG_FIFO_CTRL0, DIG, id)
+
+/* Aux regs */
+
+#define AUX_REG_LIST_RI(id)                                                    \
+  SRI_ARR(AUX_CONTROL, DP_AUX, id), SRI_ARR(AUX_DPHY_RX_CONTROL0, DP_AUX, id), \
+      SRI_ARR(AUX_DPHY_RX_CONTROL1, DP_AUX, id)
+
+#define DCN2_AUX_REG_LIST_RI(id)                                               \
+  AUX_REG_LIST_RI(id), SRI_ARR(AUX_DPHY_TX_CONTROL, DP_AUX, id)
+
+/* HDP */
+#define HPD_REG_LIST_RI(id) SRI_ARR(DC_HPD_CONTROL, HPD, id)
+
+/* Link encoder */
+#define LE_DCN3_REG_LIST_RI(id)                                                \
+  SRI_ARR(DIG_BE_CNTL, DIG, id), SRI_ARR(DIG_BE_EN_CNTL, DIG, id),             \
+      SRI_ARR(TMDS_CTL_BITS, DIG, id),                                         \
+      SRI_ARR(TMDS_DCBALANCER_CONTROL, DIG, id), SRI_ARR(DP_CONFIG, DP, id),   \
+      SRI_ARR(DP_DPHY_CNTL, DP, id), SRI_ARR(DP_DPHY_PRBS_CNTL, DP, id),       \
+      SRI_ARR(DP_DPHY_SCRAM_CNTL, DP, id), SRI_ARR(DP_DPHY_SYM0, DP, id),      \
+      SRI_ARR(DP_DPHY_SYM1, DP, id), SRI_ARR(DP_DPHY_SYM2, DP, id),            \
+      SRI_ARR(DP_DPHY_TRAINING_PATTERN_SEL, DP, id),                           \
+      SRI_ARR(DP_LINK_CNTL, DP, id), SRI_ARR(DP_LINK_FRAMING_CNTL, DP, id),    \
+      SRI_ARR(DP_MSE_SAT0, DP, id), SRI_ARR(DP_MSE_SAT1, DP, id),              \
+      SRI_ARR(DP_MSE_SAT2, DP, id), SRI_ARR(DP_MSE_SAT_UPDATE, DP, id),        \
+      SRI_ARR(DP_SEC_CNTL, DP, id), SRI_ARR(DP_VID_STREAM_CNTL, DP, id),       \
+      SRI_ARR(DP_DPHY_FAST_TRAINING, DP, id), SRI_ARR(DP_SEC_CNTL1, DP, id),   \
+      SRI_ARR(DP_DPHY_BS_SR_SWAP_CNTL, DP, id),                                \
+      SRI_ARR(DP_DPHY_HBR2_PATTERN_CONTROL, DP, id)
+
+#define LE_DCN31_REG_LIST_RI(id)                                               \
+  LE_DCN3_REG_LIST_RI(id), SRI_ARR(DP_DPHY_INTERNAL_CTRL, DP, id),             \
+      SR_ARR(DIO_LINKA_CNTL, id), SR_ARR(DIO_LINKB_CNTL, id),                  \
+      SR_ARR(DIO_LINKC_CNTL, id), SR_ARR(DIO_LINKD_CNTL, id),                  \
+      SR_ARR(DIO_LINKE_CNTL, id), SR_ARR(DIO_LINKF_CNTL, id)
+
+#define UNIPHY_DCN2_REG_LIST_RI(id, phyid)                                     \
+  SRI_ARR_ALPHABET(CLOCK_ENABLE, SYMCLK, id, phyid),                           \
+      SRI_ARR_ALPHABET(CHANNEL_XBAR_CNTL, UNIPHY, id, phyid)
+
+/* HPO DP stream encoder */
+#define DCN3_1_HPO_DP_STREAM_ENC_REG_LIST_RI(id)                               \
+  SR_ARR(DP_STREAM_MAPPER_CONTROL0, id),                                       \
+      SR_ARR(DP_STREAM_MAPPER_CONTROL1, id),                                   \
+      SR_ARR(DP_STREAM_MAPPER_CONTROL2, id),                                   \
+      SR_ARR(DP_STREAM_MAPPER_CONTROL3, id),                                   \
+      SRI_ARR(DP_STREAM_ENC_CLOCK_CONTROL, DP_STREAM_ENC, id),                 \
+      SRI_ARR(DP_STREAM_ENC_INPUT_MUX_CONTROL, DP_STREAM_ENC, id),             \
+      SRI_ARR(DP_STREAM_ENC_AUDIO_CONTROL, DP_STREAM_ENC, id),                 \
+      SRI_ARR(DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0, DP_STREAM_ENC, id), \
+      SRI_ARR(DP_SYM32_ENC_CONTROL, DP_SYM32_ENC, id),                         \
+      SRI_ARR(DP_SYM32_ENC_VID_PIXEL_FORMAT, DP_SYM32_ENC, id),                \
+      SRI_ARR(DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL, DP_SYM32_ENC, id), \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA0, DP_SYM32_ENC, id),                        \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA1, DP_SYM32_ENC, id),                        \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA2, DP_SYM32_ENC, id),                        \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA3, DP_SYM32_ENC, id),                        \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA4, DP_SYM32_ENC, id),                        \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA5, DP_SYM32_ENC, id),                        \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA6, DP_SYM32_ENC, id),                        \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA7, DP_SYM32_ENC, id),                        \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA8, DP_SYM32_ENC, id),                        \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA_CONTROL, DP_SYM32_ENC, id),                 \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL, DP_SYM32_ENC, id),   \
+      SRI_ARR(DP_SYM32_ENC_VID_FIFO_CONTROL, DP_SYM32_ENC, id),                \
+      SRI_ARR(DP_SYM32_ENC_VID_STREAM_CONTROL, DP_SYM32_ENC, id),              \
+      SRI_ARR(DP_SYM32_ENC_VID_VBID_CONTROL, DP_SYM32_ENC, id),                \
+      SRI_ARR(DP_SYM32_ENC_SDP_CONTROL, DP_SYM32_ENC, id),                     \
+      SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL0, DP_SYM32_ENC, id),                \
+      SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL2, DP_SYM32_ENC, id),                \
+      SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL3, DP_SYM32_ENC, id),                \
+      SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL5, DP_SYM32_ENC, id),                \
+      SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL11, DP_SYM32_ENC, id),               \
+      SRI_ARR(DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL, DP_SYM32_ENC, id),     \
+      SRI_ARR(DP_SYM32_ENC_SDP_AUDIO_CONTROL0, DP_SYM32_ENC, id),              \
+      SRI_ARR(DP_SYM32_ENC_VID_CRC_CONTROL, DP_SYM32_ENC, id),                 \
+      SRI_ARR(DP_SYM32_ENC_HBLANK_CONTROL, DP_SYM32_ENC, id)
+
+/* HPO DP link encoder regs */
+#define DCN3_1_HPO_DP_LINK_ENC_REG_LIST_RI(id)                                 \
+  SRI_ARR(DP_LINK_ENC_CLOCK_CONTROL, DP_LINK_ENC, id),                         \
+      SRI_ARR(DP_DPHY_SYM32_CONTROL, DP_DPHY_SYM32, id),                       \
+      SRI_ARR(DP_DPHY_SYM32_STATUS, DP_DPHY_SYM32, id),                        \
+      SRI_ARR(DP_DPHY_SYM32_TP_CONFIG, DP_DPHY_SYM32, id),                     \
+      SRI_ARR(DP_DPHY_SYM32_TP_PRBS_SEED0, DP_DPHY_SYM32, id),                 \
+      SRI_ARR(DP_DPHY_SYM32_TP_PRBS_SEED1, DP_DPHY_SYM32, id),                 \
+      SRI_ARR(DP_DPHY_SYM32_TP_PRBS_SEED2, DP_DPHY_SYM32, id),                 \
+      SRI_ARR(DP_DPHY_SYM32_TP_PRBS_SEED3, DP_DPHY_SYM32, id),                 \
+      SRI_ARR(DP_DPHY_SYM32_TP_SQ_PULSE, DP_DPHY_SYM32, id),                   \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM0, DP_DPHY_SYM32, id),                    \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM1, DP_DPHY_SYM32, id),                    \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM2, DP_DPHY_SYM32, id),                    \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM3, DP_DPHY_SYM32, id),                    \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM4, DP_DPHY_SYM32, id),                    \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM5, DP_DPHY_SYM32, id),                    \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM6, DP_DPHY_SYM32, id),                    \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM7, DP_DPHY_SYM32, id),                    \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM8, DP_DPHY_SYM32, id),                    \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM9, DP_DPHY_SYM32, id),                    \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM10, DP_DPHY_SYM32, id),                   \
+      SRI_ARR(DP_DPHY_SYM32_SAT_VC0, DP_DPHY_SYM32, id),                       \
+      SRI_ARR(DP_DPHY_SYM32_SAT_VC1, DP_DPHY_SYM32, id),                       \
+      SRI_ARR(DP_DPHY_SYM32_SAT_VC2, DP_DPHY_SYM32, id),                       \
+      SRI_ARR(DP_DPHY_SYM32_SAT_VC3, DP_DPHY_SYM32, id),                       \
+      SRI_ARR(DP_DPHY_SYM32_VC_RATE_CNTL0, DP_DPHY_SYM32, id),                 \
+      SRI_ARR(DP_DPHY_SYM32_VC_RATE_CNTL1, DP_DPHY_SYM32, id),                 \
+      SRI_ARR(DP_DPHY_SYM32_VC_RATE_CNTL2, DP_DPHY_SYM32, id),                 \
+      SRI_ARR(DP_DPHY_SYM32_VC_RATE_CNTL3, DP_DPHY_SYM32, id),                 \
+      SRI_ARR(DP_DPHY_SYM32_SAT_UPDATE, DP_DPHY_SYM32, id)
+
+/* DPP */
+#define DPP_REG_LIST_DCN30_COMMON_RI(id)                                       \
+  SRI_ARR(CM_DEALPHA, CM, id), SRI_ARR(CM_MEM_PWR_STATUS, CM, id),             \
+      SRI_ARR(CM_BIAS_CR_R, CM, id), SRI_ARR(CM_BIAS_Y_G_CB_B, CM, id),        \
+      SRI_ARR(PRE_DEGAM, CNVC_CFG, id), SRI_ARR(CM_GAMCOR_CONTROL, CM, id),    \
+      SRI_ARR(CM_GAMCOR_LUT_CONTROL, CM, id),                                  \
+      SRI_ARR(CM_GAMCOR_LUT_INDEX, CM, id),                                    \
+      SRI_ARR(CM_GAMCOR_LUT_INDEX, CM, id),                                    \
+      SRI_ARR(CM_GAMCOR_LUT_DATA, CM, id),                                     \
+      SRI_ARR(CM_GAMCOR_RAMB_START_CNTL_B, CM, id),                            \
+      SRI_ARR(CM_GAMCOR_RAMB_START_CNTL_G, CM, id),                            \
+      SRI_ARR(CM_GAMCOR_RAMB_START_CNTL_R, CM, id),                            \
+      SRI_ARR(CM_GAMCOR_RAMB_START_SLOPE_CNTL_B, CM, id),                      \
+      SRI_ARR(CM_GAMCOR_RAMB_START_SLOPE_CNTL_G, CM, id),                      \
+      SRI_ARR(CM_GAMCOR_RAMB_START_SLOPE_CNTL_R, CM, id),                      \
+      SRI_ARR(CM_GAMCOR_RAMB_END_CNTL1_B, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMB_END_CNTL2_B, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMB_END_CNTL1_G, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMB_END_CNTL2_G, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMB_END_CNTL1_R, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMB_END_CNTL2_R, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMB_REGION_0_1, CM, id),                              \
+      SRI_ARR(CM_GAMCOR_RAMB_REGION_32_33, CM, id),                            \
+      SRI_ARR(CM_GAMCOR_RAMB_OFFSET_B, CM, id),                                \
+      SRI_ARR(CM_GAMCOR_RAMB_OFFSET_G, CM, id),                                \
+      SRI_ARR(CM_GAMCOR_RAMB_OFFSET_R, CM, id),                                \
+      SRI_ARR(CM_GAMCOR_RAMB_START_BASE_CNTL_B, CM, id),                       \
+      SRI_ARR(CM_GAMCOR_RAMB_START_BASE_CNTL_G, CM, id),                       \
+      SRI_ARR(CM_GAMCOR_RAMB_START_BASE_CNTL_R, CM, id),                       \
+      SRI_ARR(CM_GAMCOR_RAMA_START_CNTL_B, CM, id),                            \
+      SRI_ARR(CM_GAMCOR_RAMA_START_CNTL_G, CM, id),                            \
+      SRI_ARR(CM_GAMCOR_RAMA_START_CNTL_R, CM, id),                            \
+      SRI_ARR(CM_GAMCOR_RAMA_START_SLOPE_CNTL_B, CM, id),                      \
+      SRI_ARR(CM_GAMCOR_RAMA_START_SLOPE_CNTL_G, CM, id),                      \
+      SRI_ARR(CM_GAMCOR_RAMA_START_SLOPE_CNTL_R, CM, id),                      \
+      SRI_ARR(CM_GAMCOR_RAMA_END_CNTL1_B, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMA_END_CNTL2_B, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMA_END_CNTL1_G, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMA_END_CNTL2_G, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMA_END_CNTL1_R, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMA_END_CNTL2_R, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMA_REGION_0_1, CM, id),                              \
+      SRI_ARR(CM_GAMCOR_RAMA_REGION_32_33, CM, id),                            \
+      SRI_ARR(CM_GAMCOR_RAMA_OFFSET_B, CM, id),                                \
+      SRI_ARR(CM_GAMCOR_RAMA_OFFSET_G, CM, id),                                \
+      SRI_ARR(CM_GAMCOR_RAMA_OFFSET_R, CM, id),                                \
+      SRI_ARR(CM_GAMCOR_RAMA_START_BASE_CNTL_B, CM, id),                       \
+      SRI_ARR(CM_GAMCOR_RAMA_START_BASE_CNTL_G, CM, id),                       \
+      SRI_ARR(CM_GAMCOR_RAMA_START_BASE_CNTL_R, CM, id),                       \
+      SRI_ARR(CM_GAMUT_REMAP_CONTROL, CM, id),                                 \
+      SRI_ARR(CM_GAMUT_REMAP_C11_C12, CM, id),                                 \
+      SRI_ARR(CM_GAMUT_REMAP_C13_C14, CM, id),                                 \
+      SRI_ARR(CM_GAMUT_REMAP_C21_C22, CM, id),                                 \
+      SRI_ARR(CM_GAMUT_REMAP_C23_C24, CM, id),                                 \
+      SRI_ARR(CM_GAMUT_REMAP_C31_C32, CM, id),                                 \
+      SRI_ARR(CM_GAMUT_REMAP_C33_C34, CM, id),                                 \
+      SRI_ARR(CM_GAMUT_REMAP_B_C11_C12, CM, id),                               \
+      SRI_ARR(CM_GAMUT_REMAP_B_C13_C14, CM, id),                               \
+      SRI_ARR(CM_GAMUT_REMAP_B_C21_C22, CM, id),                               \
+      SRI_ARR(CM_GAMUT_REMAP_B_C23_C24, CM, id),                               \
+      SRI_ARR(CM_GAMUT_REMAP_B_C31_C32, CM, id),                               \
+      SRI_ARR(CM_GAMUT_REMAP_B_C33_C34, CM, id),                               \
+      SRI_ARR(DSCL_EXT_OVERSCAN_LEFT_RIGHT, DSCL, id),                         \
+      SRI_ARR(DSCL_EXT_OVERSCAN_TOP_BOTTOM, DSCL, id),                         \
+      SRI_ARR(OTG_H_BLANK, DSCL, id), SRI_ARR(OTG_V_BLANK, DSCL, id),          \
+      SRI_ARR(SCL_MODE, DSCL, id), SRI_ARR(LB_DATA_FORMAT, DSCL, id),          \
+      SRI_ARR(LB_MEMORY_CTRL, DSCL, id), SRI_ARR(DSCL_AUTOCAL, DSCL, id),      \
+      SRI_ARR(DSCL_CONTROL, DSCL, id),                                         \
+      SRI_ARR(SCL_TAP_CONTROL, DSCL, id),                                      \
+      SRI_ARR(SCL_COEF_RAM_TAP_SELECT, DSCL, id),                              \
+      SRI_ARR(SCL_COEF_RAM_TAP_DATA, DSCL, id),                                \
+      SRI_ARR(DSCL_2TAP_CONTROL, DSCL, id), SRI_ARR(MPC_SIZE, DSCL, id),       \
+      SRI_ARR(SCL_HORZ_FILTER_SCALE_RATIO, DSCL, id),                          \
+      SRI_ARR(SCL_VERT_FILTER_SCALE_RATIO, DSCL, id),                          \
+      SRI_ARR(SCL_HORZ_FILTER_SCALE_RATIO_C, DSCL, id),                        \
+      SRI_ARR(SCL_VERT_FILTER_SCALE_RATIO_C, DSCL, id),                        \
+      SRI_ARR(SCL_HORZ_FILTER_INIT, DSCL, id),                                 \
+      SRI_ARR(SCL_HORZ_FILTER_INIT_C, DSCL, id),                               \
+      SRI_ARR(SCL_VERT_FILTER_INIT, DSCL, id),                                 \
+      SRI_ARR(SCL_VERT_FILTER_INIT_C, DSCL, id),                               \
+      SRI_ARR(RECOUT_START, DSCL, id), SRI_ARR(RECOUT_SIZE, DSCL, id),         \
+      SRI_ARR(PRE_DEALPHA, CNVC_CFG, id), SRI_ARR(PRE_REALPHA, CNVC_CFG, id),  \
+      SRI_ARR(PRE_CSC_MODE, CNVC_CFG, id),                                     \
+      SRI_ARR(PRE_CSC_C11_C12, CNVC_CFG, id),                                  \
+      SRI_ARR(PRE_CSC_C33_C34, CNVC_CFG, id),                                  \
+      SRI_ARR(PRE_CSC_B_C11_C12, CNVC_CFG, id),                                \
+      SRI_ARR(PRE_CSC_B_C33_C34, CNVC_CFG, id),                                \
+      SRI_ARR(CM_POST_CSC_CONTROL, CM, id),                                    \
+      SRI_ARR(CM_POST_CSC_C11_C12, CM, id),                                    \
+      SRI_ARR(CM_POST_CSC_C33_C34, CM, id),                                    \
+      SRI_ARR(CM_POST_CSC_B_C11_C12, CM, id),                                  \
+      SRI_ARR(CM_POST_CSC_B_C33_C34, CM, id),                                  \
+      SRI_ARR(CM_MEM_PWR_CTRL, CM, id), SRI_ARR(CM_CONTROL, CM, id),           \
+      SRI_ARR(FORMAT_CONTROL, CNVC_CFG, id),                                   \
+      SRI_ARR(CNVC_SURFACE_PIXEL_FORMAT, CNVC_CFG, id),                        \
+      SRI_ARR(CURSOR0_CONTROL, CNVC_CUR, id),                                  \
+      SRI_ARR(CURSOR0_COLOR0, CNVC_CUR, id),                                   \
+      SRI_ARR(CURSOR0_COLOR1, CNVC_CUR, id),                                   \
+      SRI_ARR(CURSOR0_FP_SCALE_BIAS, CNVC_CUR, id),                            \
+      SRI_ARR(DPP_CONTROL, DPP_TOP, id), SRI_ARR(CM_HDR_MULT_COEF, CM, id),    \
+      SRI_ARR(CURSOR_CONTROL, CURSOR0_, id),                                   \
+      SRI_ARR(ALPHA_2BIT_LUT, CNVC_CFG, id),                                   \
+      SRI_ARR(FCNV_FP_BIAS_R, CNVC_CFG, id),                                   \
+      SRI_ARR(FCNV_FP_BIAS_G, CNVC_CFG, id),                                   \
+      SRI_ARR(FCNV_FP_BIAS_B, CNVC_CFG, id),                                   \
+      SRI_ARR(FCNV_FP_SCALE_R, CNVC_CFG, id),                                  \
+      SRI_ARR(FCNV_FP_SCALE_G, CNVC_CFG, id),                                  \
+      SRI_ARR(FCNV_FP_SCALE_B, CNVC_CFG, id),                                  \
+      SRI_ARR(COLOR_KEYER_CONTROL, CNVC_CFG, id),                              \
+      SRI_ARR(COLOR_KEYER_ALPHA, CNVC_CFG, id),                                \
+      SRI_ARR(COLOR_KEYER_RED, CNVC_CFG, id),                                  \
+      SRI_ARR(COLOR_KEYER_GREEN, CNVC_CFG, id),                                \
+      SRI_ARR(COLOR_KEYER_BLUE, CNVC_CFG, id),                                 \
+      SRI_ARR(CURSOR_CONTROL, CURSOR0_, id),                                   \
+      SRI_ARR(OBUF_MEM_PWR_CTRL, DSCL, id),                                    \
+      SRI_ARR(DSCL_MEM_PWR_STATUS, DSCL, id),                                  \
+      SRI_ARR(DSCL_MEM_PWR_CTRL, DSCL, id)
+
+/* OPP */
+#define OPP_REG_LIST_DCN_RI(id)                                                \
+  SRI_ARR(FMT_BIT_DEPTH_CONTROL, FMT, id), SRI_ARR(FMT_CONTROL, FMT, id),      \
+      SRI_ARR(FMT_DITHER_RAND_R_SEED, FMT, id),                                \
+      SRI_ARR(FMT_DITHER_RAND_G_SEED, FMT, id),                                \
+      SRI_ARR(FMT_DITHER_RAND_B_SEED, FMT, id),                                \
+      SRI_ARR(FMT_CLAMP_CNTL, FMT, id),                                        \
+      SRI_ARR(FMT_DYNAMIC_EXP_CNTL, FMT, id),                                  \
+      SRI_ARR(FMT_MAP420_MEMORY_CONTROL, FMT, id),                             \
+      SRI_ARR(OPPBUF_CONTROL, OPPBUF, id),                                     \
+      SRI_ARR(OPPBUF_3D_PARAMETERS_0, OPPBUF, id),                             \
+      SRI_ARR(OPPBUF_3D_PARAMETERS_1, OPPBUF, id),                             \
+      SRI_ARR(OPP_PIPE_CONTROL, OPP_PIPE, id)                                  \
+
+#define OPP_REG_LIST_DCN10_RI(id) OPP_REG_LIST_DCN_RI(id)
+
+#define OPP_DPG_REG_LIST_RI(id)                                                \
+  SRI_ARR(DPG_CONTROL, DPG, id), SRI_ARR(DPG_DIMENSIONS, DPG, id),             \
+      SRI_ARR(DPG_OFFSET_SEGMENT, DPG, id), SRI_ARR(DPG_COLOUR_B_CB, DPG, id), \
+      SRI_ARR(DPG_COLOUR_G_Y, DPG, id), SRI_ARR(DPG_COLOUR_R_CR, DPG, id),     \
+      SRI_ARR(DPG_RAMP_CONTROL, DPG, id), SRI_ARR(DPG_STATUS, DPG, id)
+
+#define OPP_REG_LIST_DCN30_RI(id)                                              \
+  OPP_REG_LIST_DCN10_RI(id), OPP_DPG_REG_LIST_RI(id),                          \
+      SRI_ARR(FMT_422_CONTROL, FMT, id)
+
+/* Aux engine regs */
+#define AUX_COMMON_REG_LIST0_RI(id)                                            \
+  SRI_ARR(AUX_CONTROL, DP_AUX, id), SRI_ARR(AUX_ARB_CONTROL, DP_AUX, id),      \
+      SRI_ARR(AUX_SW_DATA, DP_AUX, id), SRI_ARR(AUX_SW_CONTROL, DP_AUX, id),   \
+      SRI_ARR(AUX_INTERRUPT_CONTROL, DP_AUX, id),                              \
+      SRI_ARR(AUX_DPHY_RX_CONTROL1, DP_AUX, id),                               \
+      SRI_ARR(AUX_SW_STATUS, DP_AUX, id)
+
+/* DWBC */
+#define DWBC_COMMON_REG_LIST_DCN30_RI(id)                                      \
+  SR_ARR(DWB_ENABLE_CLK_CTRL, id), SR_ARR(DWB_MEM_PWR_CTRL, id),               \
+      SR_ARR(FC_MODE_CTRL, id), SR_ARR(FC_FLOW_CTRL, id),                      \
+      SR_ARR(FC_WINDOW_START, id), SR_ARR(FC_WINDOW_SIZE, id),                 \
+      SR_ARR(FC_SOURCE_SIZE, id), SR_ARR(DWB_UPDATE_CTRL, id),                 \
+      SR_ARR(DWB_CRC_CTRL, id), SR_ARR(DWB_CRC_MASK_R_G, id),                  \
+      SR_ARR(DWB_CRC_MASK_B_A, id), SR_ARR(DWB_CRC_VAL_R_G, id),               \
+      SR_ARR(DWB_CRC_VAL_B_A, id), SR_ARR(DWB_OUT_CTRL, id),                   \
+      SR_ARR(DWB_MMHUBBUB_BACKPRESSURE_CNT_EN, id),                            \
+      SR_ARR(DWB_MMHUBBUB_BACKPRESSURE_CNT, id),                               \
+      SR_ARR(DWB_HOST_READ_CONTROL, id), SR_ARR(DWB_SOFT_RESET, id),           \
+      SR_ARR(DWB_HDR_MULT_COEF, id), SR_ARR(DWB_GAMUT_REMAP_MODE, id),         \
+      SR_ARR(DWB_GAMUT_REMAP_COEF_FORMAT, id),                                 \
+      SR_ARR(DWB_GAMUT_REMAPA_C11_C12, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPA_C13_C14, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPA_C21_C22, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPA_C23_C24, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPA_C31_C32, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPA_C33_C34, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPB_C11_C12, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPB_C13_C14, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPB_C21_C22, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPB_C23_C24, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPB_C31_C32, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPB_C33_C34, id), SR_ARR(DWB_OGAM_CONTROL, id),      \
+      SR_ARR(DWB_OGAM_LUT_INDEX, id), SR_ARR(DWB_OGAM_LUT_DATA, id),           \
+      SR_ARR(DWB_OGAM_LUT_CONTROL, id),                                        \
+      SR_ARR(DWB_OGAM_RAMA_START_CNTL_B, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_START_CNTL_G, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_START_CNTL_R, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_START_BASE_CNTL_B, id),                             \
+      SR_ARR(DWB_OGAM_RAMA_START_SLOPE_CNTL_B, id),                            \
+      SR_ARR(DWB_OGAM_RAMA_START_BASE_CNTL_G, id),                             \
+      SR_ARR(DWB_OGAM_RAMA_START_SLOPE_CNTL_G, id),                            \
+      SR_ARR(DWB_OGAM_RAMA_START_BASE_CNTL_R, id),                             \
+      SR_ARR(DWB_OGAM_RAMA_START_SLOPE_CNTL_R, id),                            \
+      SR_ARR(DWB_OGAM_RAMA_END_CNTL1_B, id),                                   \
+      SR_ARR(DWB_OGAM_RAMA_END_CNTL2_B, id),                                   \
+      SR_ARR(DWB_OGAM_RAMA_END_CNTL1_G, id),                                   \
+      SR_ARR(DWB_OGAM_RAMA_END_CNTL2_G, id),                                   \
+      SR_ARR(DWB_OGAM_RAMA_END_CNTL1_R, id),                                   \
+      SR_ARR(DWB_OGAM_RAMA_END_CNTL2_R, id),                                   \
+      SR_ARR(DWB_OGAM_RAMA_OFFSET_B, id), SR_ARR(DWB_OGAM_RAMA_OFFSET_G, id),  \
+      SR_ARR(DWB_OGAM_RAMA_OFFSET_R, id),                                      \
+      SR_ARR(DWB_OGAM_RAMA_REGION_0_1, id),                                    \
+      SR_ARR(DWB_OGAM_RAMA_REGION_2_3, id),                                    \
+      SR_ARR(DWB_OGAM_RAMA_REGION_4_5, id),                                    \
+      SR_ARR(DWB_OGAM_RAMA_REGION_6_7, id),                                    \
+      SR_ARR(DWB_OGAM_RAMA_REGION_8_9, id),                                    \
+      SR_ARR(DWB_OGAM_RAMA_REGION_10_11, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_12_13, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_14_15, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_16_17, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_18_19, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_20_21, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_22_23, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_24_25, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_26_27, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_28_29, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_30_31, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_32_33, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_START_CNTL_B, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_START_CNTL_G, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_START_CNTL_R, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_START_BASE_CNTL_B, id),                             \
+      SR_ARR(DWB_OGAM_RAMB_START_SLOPE_CNTL_B, id),                            \
+      SR_ARR(DWB_OGAM_RAMB_START_BASE_CNTL_G, id),                             \
+      SR_ARR(DWB_OGAM_RAMB_START_SLOPE_CNTL_G, id),                            \
+      SR_ARR(DWB_OGAM_RAMB_START_BASE_CNTL_R, id),                             \
+      SR_ARR(DWB_OGAM_RAMB_START_SLOPE_CNTL_R, id),                            \
+      SR_ARR(DWB_OGAM_RAMB_END_CNTL1_B, id),                                   \
+      SR_ARR(DWB_OGAM_RAMB_END_CNTL2_B, id),                                   \
+      SR_ARR(DWB_OGAM_RAMB_END_CNTL1_G, id),                                   \
+      SR_ARR(DWB_OGAM_RAMB_END_CNTL2_G, id),                                   \
+      SR_ARR(DWB_OGAM_RAMB_END_CNTL1_R, id),                                   \
+      SR_ARR(DWB_OGAM_RAMB_END_CNTL2_R, id),                                   \
+      SR_ARR(DWB_OGAM_RAMB_OFFSET_B, id), SR_ARR(DWB_OGAM_RAMB_OFFSET_G, id),  \
+      SR_ARR(DWB_OGAM_RAMB_OFFSET_R, id),                                      \
+      SR_ARR(DWB_OGAM_RAMB_REGION_0_1, id),                                    \
+      SR_ARR(DWB_OGAM_RAMB_REGION_2_3, id),                                    \
+      SR_ARR(DWB_OGAM_RAMB_REGION_4_5, id),                                    \
+      SR_ARR(DWB_OGAM_RAMB_REGION_6_7, id),                                    \
+      SR_ARR(DWB_OGAM_RAMB_REGION_8_9, id),                                    \
+      SR_ARR(DWB_OGAM_RAMB_REGION_10_11, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_12_13, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_14_15, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_16_17, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_18_19, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_20_21, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_22_23, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_24_25, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_26_27, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_28_29, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_30_31, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_32_33, id)
+
+/* MCIF */
+
+#define MCIF_WB_COMMON_REG_LIST_DCN32_RI(inst)                                 \
+  SRI2_ARR(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB, inst),                          \
+      SRI2_ARR(MCIF_WB_BUFMGR_STATUS, MCIF_WB, inst),                          \
+      SRI2_ARR(MCIF_WB_BUF_PITCH, MCIF_WB, inst),                              \
+      SRI2_ARR(MCIF_WB_BUF_1_STATUS, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_1_STATUS2, MCIF_WB, inst),                          \
+      SRI2_ARR(MCIF_WB_BUF_2_STATUS, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_2_STATUS2, MCIF_WB, inst),                          \
+      SRI2_ARR(MCIF_WB_BUF_3_STATUS, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_3_STATUS2, MCIF_WB, inst),                          \
+      SRI2_ARR(MCIF_WB_BUF_4_STATUS, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_4_STATUS2, MCIF_WB, inst),                          \
+      SRI2_ARR(MCIF_WB_ARBITRATION_CONTROL, MCIF_WB, inst),                    \
+      SRI2_ARR(MCIF_WB_SCLK_CHANGE, MCIF_WB, inst),                            \
+      SRI2_ARR(MCIF_WB_BUF_1_ADDR_Y, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_1_ADDR_C, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_2_ADDR_Y, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_2_ADDR_C, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_3_ADDR_Y, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_3_ADDR_C, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_4_ADDR_Y, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_4_ADDR_C, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB, inst),                     \
+      SRI2_ARR(MCIF_WB_NB_PSTATE_LATENCY_WATERMARK, MMHUBBUB, inst),           \
+      SRI2_ARR(MCIF_WB_NB_PSTATE_CONTROL, MCIF_WB, inst),                      \
+      SRI2_ARR(MCIF_WB_WATERMARK, MMHUBBUB, inst),                             \
+      SRI2_ARR(MCIF_WB_CLOCK_GATER_CONTROL, MCIF_WB, inst),                    \
+      SRI2_ARR(MCIF_WB_SELF_REFRESH_CONTROL, MCIF_WB, inst),                   \
+      SRI2_ARR(MULTI_LEVEL_QOS_CTRL, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_SECURITY_LEVEL, MCIF_WB, inst),                         \
+      SRI2_ARR(MCIF_WB_BUF_LUMA_SIZE, MCIF_WB, inst),                          \
+      SRI2_ARR(MCIF_WB_BUF_CHROMA_SIZE, MCIF_WB, inst),                        \
+      SRI2_ARR(MCIF_WB_BUF_1_ADDR_Y_HIGH, MCIF_WB, inst),                      \
+      SRI2_ARR(MCIF_WB_BUF_1_ADDR_C_HIGH, MCIF_WB, inst),                      \
+      SRI2_ARR(MCIF_WB_BUF_2_ADDR_Y_HIGH, MCIF_WB, inst),                      \
+      SRI2_ARR(MCIF_WB_BUF_2_ADDR_C_HIGH, MCIF_WB, inst),                      \
+      SRI2_ARR(MCIF_WB_BUF_3_ADDR_Y_HIGH, MCIF_WB, inst),                      \
+      SRI2_ARR(MCIF_WB_BUF_3_ADDR_C_HIGH, MCIF_WB, inst),                      \
+      SRI2_ARR(MCIF_WB_BUF_4_ADDR_Y_HIGH, MCIF_WB, inst),                      \
+      SRI2_ARR(MCIF_WB_BUF_4_ADDR_C_HIGH, MCIF_WB, inst),                      \
+      SRI2_ARR(MCIF_WB_BUF_1_RESOLUTION, MCIF_WB, inst),                       \
+      SRI2_ARR(MCIF_WB_BUF_2_RESOLUTION, MCIF_WB, inst),                       \
+      SRI2_ARR(MCIF_WB_BUF_3_RESOLUTION, MCIF_WB, inst),                       \
+      SRI2_ARR(MCIF_WB_BUF_4_RESOLUTION, MCIF_WB, inst),                       \
+      SRI2_ARR(MMHUBBUB_MEM_PWR_CNTL, MMHUBBUB, inst),                         \
+      SRI2_ARR(MMHUBBUB_WARMUP_ADDR_REGION, MMHUBBUB, inst),                   \
+      SRI2_ARR(MMHUBBUB_WARMUP_BASE_ADDR_HIGH, MMHUBBUB, inst),                \
+      SRI2_ARR(MMHUBBUB_WARMUP_BASE_ADDR_LOW, MMHUBBUB, inst),                 \
+      SRI2_ARR(MMHUBBUB_WARMUP_CONTROL_STATUS, MMHUBBUB, inst)
+
+/* DSC */
+
+#define DSC_REG_LIST_DCN20_RI(id)                                              \
+  SRI_ARR(DSC_TOP_CONTROL, DSC_TOP, id),                                       \
+      SRI_ARR(DSC_DEBUG_CONTROL, DSC_TOP, id),                                 \
+      SRI_ARR(DSCC_CONFIG0, DSCC, id), SRI_ARR(DSCC_CONFIG1, DSCC, id),        \
+      SRI_ARR(DSCC_STATUS, DSCC, id),                                          \
+      SRI_ARR(DSCC_INTERRUPT_CONTROL_STATUS, DSCC, id),                        \
+      SRI_ARR(DSCC_PPS_CONFIG0, DSCC, id),                                     \
+      SRI_ARR(DSCC_PPS_CONFIG1, DSCC, id),                                     \
+      SRI_ARR(DSCC_PPS_CONFIG2, DSCC, id),                                     \
+      SRI_ARR(DSCC_PPS_CONFIG3, DSCC, id),                                     \
+      SRI_ARR(DSCC_PPS_CONFIG4, DSCC, id),                                     \
+      SRI_ARR(DSCC_PPS_CONFIG5, DSCC, id),                                     \
+      SRI_ARR(DSCC_PPS_CONFIG6, DSCC, id),                                     \
+      SRI_ARR(DSCC_PPS_CONFIG7, DSCC, id),                                     \
+      SRI_ARR(DSCC_PPS_CONFIG8, DSCC, id),                                     \
+      SRI_ARR(DSCC_PPS_CONFIG9, DSCC, id),                                     \
+      SRI_ARR(DSCC_PPS_CONFIG10, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG11, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG12, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG13, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG14, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG15, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG16, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG17, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG18, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG19, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG20, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG21, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG22, DSCC, id),                                    \
+      SRI_ARR(DSCC_MEM_POWER_CONTROL, DSCC, id),                               \
+      SRI_ARR(DSCC_R_Y_SQUARED_ERROR_LOWER, DSCC, id),                         \
+      SRI_ARR(DSCC_R_Y_SQUARED_ERROR_UPPER, DSCC, id),                         \
+      SRI_ARR(DSCC_G_CB_SQUARED_ERROR_LOWER, DSCC, id),                        \
+      SRI_ARR(DSCC_G_CB_SQUARED_ERROR_UPPER, DSCC, id),                        \
+      SRI_ARR(DSCC_B_CR_SQUARED_ERROR_LOWER, DSCC, id),                        \
+      SRI_ARR(DSCC_B_CR_SQUARED_ERROR_UPPER, DSCC, id),                        \
+      SRI_ARR(DSCC_MAX_ABS_ERROR0, DSCC, id),                                  \
+      SRI_ARR(DSCC_MAX_ABS_ERROR1, DSCC, id),                                  \
+      SRI_ARR(DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL, DSCC, id),                 \
+      SRI_ARR(DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL, DSCC, id),                 \
+      SRI_ARR(DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL, DSCC, id),                 \
+      SRI_ARR(DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL, DSCC, id),                 \
+      SRI_ARR(DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL, DSCC, id),         \
+      SRI_ARR(DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL, DSCC, id),         \
+      SRI_ARR(DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL, DSCC, id),         \
+      SRI_ARR(DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL, DSCC, id),         \
+      SRI_ARR(DSCCIF_CONFIG0, DSCCIF, id),                                     \
+      SRI_ARR(DSCCIF_CONFIG1, DSCCIF, id),                                     \
+      SRI_ARR(DSCRM_DSC_FORWARD_CONFIG, DSCRM, id)
+
+/* MPC */
+
+#define MPC_DWB_MUX_REG_LIST_DCN3_0_RI(inst)                                   \
+  SRII_DWB(DWB_MUX, MUX, MPC_DWB, inst)
+
+#define MPC_OUT_MUX_COMMON_REG_LIST_DCN1_0_RI(inst)                            \
+  SRII(MUX, MPC_OUT, inst), VUPDATE_SRII(CUR, VUPDATE_LOCK_SET, inst)
+
+#define MPC_OUT_MUX_REG_LIST_DCN3_0_RI(inst)                                   \
+  MPC_OUT_MUX_COMMON_REG_LIST_DCN1_0_RI(inst), SRII(CSC_MODE, MPC_OUT, inst),  \
+      SRII(CSC_C11_C12_A, MPC_OUT, inst), SRII(CSC_C33_C34_A, MPC_OUT, inst),  \
+      SRII(CSC_C11_C12_B, MPC_OUT, inst), SRII(CSC_C33_C34_B, MPC_OUT, inst),  \
+      SRII(DENORM_CONTROL, MPC_OUT, inst),                                     \
+      SRII(DENORM_CLAMP_G_Y, MPC_OUT, inst),                                   \
+      SRII(DENORM_CLAMP_B_CB, MPC_OUT, inst), SR(MPC_OUT_CSC_COEF_FORMAT)
+
+#define MPC_COMMON_REG_LIST_DCN1_0_RI(inst)                                    \
+  SRII(MPCC_TOP_SEL, MPCC, inst), SRII(MPCC_BOT_SEL, MPCC, inst),              \
+      SRII(MPCC_CONTROL, MPCC, inst), SRII(MPCC_STATUS, MPCC, inst),           \
+      SRII(MPCC_OPP_ID, MPCC, inst), SRII(MPCC_BG_G_Y, MPCC, inst),            \
+      SRII(MPCC_BG_R_CR, MPCC, inst), SRII(MPCC_BG_B_CB, MPCC, inst),          \
+      SRII(MPCC_SM_CONTROL, MPCC, inst),                                       \
+      SRII(MPCC_UPDATE_LOCK_SEL, MPCC, inst)
+
+#define MPC_REG_LIST_DCN3_0_RI(inst)                                           \
+  MPC_COMMON_REG_LIST_DCN1_0_RI(inst), SRII(MPCC_TOP_GAIN, MPCC, inst),        \
+      SRII(MPCC_BOT_GAIN_INSIDE, MPCC, inst),                                  \
+      SRII(MPCC_BOT_GAIN_OUTSIDE, MPCC, inst),                                 \
+      SRII(MPCC_MEM_PWR_CTRL, MPCC, inst),                                     \
+      SRII(MPCC_OGAM_LUT_INDEX, MPCC_OGAM, inst),                              \
+      SRII(MPCC_OGAM_LUT_DATA, MPCC_OGAM, inst),                               \
+      SRII(MPCC_GAMUT_REMAP_COEF_FORMAT, MPCC_OGAM, inst),                     \
+      SRII(MPCC_GAMUT_REMAP_MODE, MPCC_OGAM, inst),                            \
+      SRII(MPC_GAMUT_REMAP_C11_C12_A, MPCC_OGAM, inst),                        \
+      SRII(MPC_GAMUT_REMAP_C33_C34_A, MPCC_OGAM, inst),                        \
+      SRII(MPC_GAMUT_REMAP_C11_C12_B, MPCC_OGAM, inst),                        \
+      SRII(MPC_GAMUT_REMAP_C33_C34_B, MPCC_OGAM, inst),                        \
+      SRII(MPCC_OGAM_RAMA_START_CNTL_B, MPCC_OGAM, inst),                      \
+      SRII(MPCC_OGAM_RAMA_START_CNTL_G, MPCC_OGAM, inst),                      \
+      SRII(MPCC_OGAM_RAMA_START_CNTL_R, MPCC_OGAM, inst),                      \
+      SRII(MPCC_OGAM_RAMA_START_SLOPE_CNTL_B, MPCC_OGAM, inst),                \
+      SRII(MPCC_OGAM_RAMA_START_SLOPE_CNTL_G, MPCC_OGAM, inst),                \
+      SRII(MPCC_OGAM_RAMA_START_SLOPE_CNTL_R, MPCC_OGAM, inst),                \
+      SRII(MPCC_OGAM_RAMA_END_CNTL1_B, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMA_END_CNTL2_B, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMA_END_CNTL1_G, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMA_END_CNTL2_G, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMA_END_CNTL1_R, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMA_END_CNTL2_R, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMA_REGION_0_1, MPCC_OGAM, inst),                        \
+      SRII(MPCC_OGAM_RAMA_REGION_32_33, MPCC_OGAM, inst),                      \
+      SRII(MPCC_OGAM_RAMA_OFFSET_B, MPCC_OGAM, inst),                          \
+      SRII(MPCC_OGAM_RAMA_OFFSET_G, MPCC_OGAM, inst),                          \
+      SRII(MPCC_OGAM_RAMA_OFFSET_R, MPCC_OGAM, inst),                          \
+      SRII(MPCC_OGAM_RAMA_START_BASE_CNTL_B, MPCC_OGAM, inst),                 \
+      SRII(MPCC_OGAM_RAMA_START_BASE_CNTL_G, MPCC_OGAM, inst),                 \
+      SRII(MPCC_OGAM_RAMA_START_BASE_CNTL_R, MPCC_OGAM, inst),                 \
+      SRII(MPCC_OGAM_RAMB_START_CNTL_B, MPCC_OGAM, inst),                      \
+      SRII(MPCC_OGAM_RAMB_START_CNTL_G, MPCC_OGAM, inst),                      \
+      SRII(MPCC_OGAM_RAMB_START_CNTL_R, MPCC_OGAM, inst),                      \
+      SRII(MPCC_OGAM_RAMB_START_SLOPE_CNTL_B, MPCC_OGAM, inst),                \
+      SRII(MPCC_OGAM_RAMB_START_SLOPE_CNTL_G, MPCC_OGAM, inst),                \
+      SRII(MPCC_OGAM_RAMB_START_SLOPE_CNTL_R, MPCC_OGAM, inst),                \
+      SRII(MPCC_OGAM_RAMB_END_CNTL1_B, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMB_END_CNTL2_B, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMB_END_CNTL1_G, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMB_END_CNTL2_G, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMB_END_CNTL1_R, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMB_END_CNTL2_R, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMB_REGION_0_1, MPCC_OGAM, inst),                        \
+      SRII(MPCC_OGAM_RAMB_REGION_32_33, MPCC_OGAM, inst),                      \
+      SRII(MPCC_OGAM_RAMB_OFFSET_B, MPCC_OGAM, inst),                          \
+      SRII(MPCC_OGAM_RAMB_OFFSET_G, MPCC_OGAM, inst),                          \
+      SRII(MPCC_OGAM_RAMB_OFFSET_R, MPCC_OGAM, inst),                          \
+      SRII(MPCC_OGAM_RAMB_START_BASE_CNTL_B, MPCC_OGAM, inst),                 \
+      SRII(MPCC_OGAM_RAMB_START_BASE_CNTL_G, MPCC_OGAM, inst),                 \
+      SRII(MPCC_OGAM_RAMB_START_BASE_CNTL_R, MPCC_OGAM, inst),                 \
+      SRII(MPCC_OGAM_CONTROL, MPCC_OGAM, inst),                                \
+      SRII(MPCC_OGAM_LUT_CONTROL, MPCC_OGAM, inst)
+
+#define MPC_REG_LIST_DCN3_2_RI(inst) \
+	MPC_REG_LIST_DCN3_0_RI(inst),\
+	SRII(MPCC_MOVABLE_CM_LOCATION_CONTROL, MPCC, inst),\
+	SRII(MPCC_MCM_SHAPER_CONTROL, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_OFFSET_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_OFFSET_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_OFFSET_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_SCALE_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_SCALE_G_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_LUT_INDEX, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_LUT_DATA, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_START_CNTL_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_START_CNTL_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_START_CNTL_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_END_CNTL_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_END_CNTL_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_END_CNTL_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_0_1, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_2_3, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_4_5, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_6_7, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_8_9, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_10_11, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_12_13, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_14_15, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_16_17, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_18_19, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_20_21, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_22_23, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_24_25, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_26_27, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_28_29, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_30_31, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_32_33, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_START_CNTL_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_START_CNTL_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_START_CNTL_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_END_CNTL_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_END_CNTL_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_END_CNTL_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_0_1, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_2_3, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_4_5, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_6_7, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_8_9, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_10_11, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_12_13, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_14_15, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_16_17, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_18_19, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_20_21, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_22_23, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_24_25, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_26_27, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_28_29, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_30_31, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_32_33, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_3DLUT_MODE, MPCC_MCM, inst), /*TODO: may need to add other 3DLUT regs*/\
+	SRII(MPCC_MCM_3DLUT_INDEX, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_3DLUT_DATA, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_3DLUT_DATA_30BIT, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_3DLUT_READ_WRITE_CONTROL, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_3DLUT_OUT_NORM_FACTOR, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_3DLUT_OUT_OFFSET_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_3DLUT_OUT_OFFSET_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_3DLUT_OUT_OFFSET_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_CONTROL, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_LUT_INDEX, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_LUT_DATA, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_LUT_CONTROL, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_START_CNTL_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_START_CNTL_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_START_CNTL_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL1_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL2_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL1_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL2_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL1_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL2_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_OFFSET_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_OFFSET_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_OFFSET_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_0_1, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_2_3, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_4_5, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_6_7, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_8_9, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_10_11, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_12_13, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_14_15, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_16_17, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_18_19, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_20_21, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_22_23, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_24_25, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_26_27, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_28_29, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_30_31, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_32_33, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_START_CNTL_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_START_CNTL_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_START_CNTL_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL1_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL2_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL1_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL2_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL1_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL2_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_OFFSET_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_OFFSET_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_OFFSET_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_0_1, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_2_3, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_4_5, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_6_7, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_8_9, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_10_11, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_12_13, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_14_15, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_16_17, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_18_19, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_20_21, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_22_23, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_24_25, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_26_27, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_28_29, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_30_31, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_32_33, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_MEM_PWR_CTRL, MPCC_MCM, inst)
+/* OPTC */
+
+#define OPTC_COMMON_REG_LIST_DCN3_2_RI(inst)                                   \
+  SRI_ARR(OTG_VSTARTUP_PARAM, OTG, inst),                                      \
+      SRI_ARR(OTG_VUPDATE_PARAM, OTG, inst),                                   \
+      SRI_ARR(OTG_VREADY_PARAM, OTG, inst),                                    \
+      SRI_ARR(OTG_MASTER_UPDATE_LOCK, OTG, inst),                              \
+      SRI_ARR(OTG_GLOBAL_CONTROL0, OTG, inst),                                 \
+      SRI_ARR(OTG_GLOBAL_CONTROL1, OTG, inst),                                 \
+      SRI_ARR(OTG_GLOBAL_CONTROL2, OTG, inst),                                 \
+      SRI_ARR(OTG_GLOBAL_CONTROL4, OTG, inst),                                 \
+      SRI_ARR(OTG_DOUBLE_BUFFER_CONTROL, OTG, inst),                           \
+      SRI_ARR(OTG_H_TOTAL, OTG, inst),                                         \
+      SRI_ARR(OTG_H_BLANK_START_END, OTG, inst),                               \
+      SRI_ARR(OTG_H_SYNC_A, OTG, inst), SRI_ARR(OTG_H_SYNC_A_CNTL, OTG, inst), \
+      SRI_ARR(OTG_H_TIMING_CNTL, OTG, inst), SRI_ARR(OTG_V_TOTAL, OTG, inst),  \
+      SRI_ARR(OTG_V_BLANK_START_END, OTG, inst),                               \
+      SRI_ARR(OTG_V_SYNC_A, OTG, inst), SRI_ARR(OTG_V_SYNC_A_CNTL, OTG, inst), \
+      SRI_ARR(OTG_CONTROL, OTG, inst), SRI_ARR(OTG_STEREO_CONTROL, OTG, inst), \
+      SRI_ARR(OTG_3D_STRUCTURE_CONTROL, OTG, inst),                            \
+      SRI_ARR(OTG_STEREO_STATUS, OTG, inst),                                   \
+      SRI_ARR(OTG_V_TOTAL_MAX, OTG, inst),                                     \
+      SRI_ARR(OTG_V_TOTAL_MIN, OTG, inst),                                     \
+      SRI_ARR(OTG_V_TOTAL_CONTROL, OTG, inst),                                 \
+      SRI_ARR(OTG_TRIGA_CNTL, OTG, inst),                                      \
+      SRI_ARR(OTG_FORCE_COUNT_NOW_CNTL, OTG, inst),                            \
+      SRI_ARR(OTG_STATIC_SCREEN_CONTROL, OTG, inst),                           \
+      SRI_ARR(OTG_STATUS_FRAME_COUNT, OTG, inst),                              \
+      SRI_ARR(OTG_STATUS, OTG, inst), SRI_ARR(OTG_STATUS_POSITION, OTG, inst), \
+      SRI_ARR(OTG_NOM_VERT_POSITION, OTG, inst),                               \
+      SRI_ARR(OTG_M_CONST_DTO0, OTG, inst),                                    \
+      SRI_ARR(OTG_M_CONST_DTO1, OTG, inst),                                    \
+      SRI_ARR(OTG_CLOCK_CONTROL, OTG, inst),                                   \
+      SRI_ARR(OTG_VERTICAL_INTERRUPT0_CONTROL, OTG, inst),                     \
+      SRI_ARR(OTG_VERTICAL_INTERRUPT0_POSITION, OTG, inst),                    \
+      SRI_ARR(OTG_VERTICAL_INTERRUPT1_CONTROL, OTG, inst),                     \
+      SRI_ARR(OTG_VERTICAL_INTERRUPT1_POSITION, OTG, inst),                    \
+      SRI_ARR(OTG_VERTICAL_INTERRUPT2_CONTROL, OTG, inst),                     \
+      SRI_ARR(OTG_VERTICAL_INTERRUPT2_POSITION, OTG, inst),                    \
+      SRI_ARR(OPTC_INPUT_CLOCK_CONTROL, ODM, inst),                            \
+      SRI_ARR(OPTC_DATA_SOURCE_SELECT, ODM, inst),                             \
+      SRI_ARR(OPTC_INPUT_GLOBAL_CONTROL, ODM, inst),                           \
+      SRI_ARR(CONTROL, VTG, inst), SRI_ARR(OTG_VERT_SYNC_CONTROL, OTG, inst),  \
+      SRI_ARR(OTG_GSL_CONTROL, OTG, inst), SRI_ARR(OTG_CRC_CNTL, OTG, inst),   \
+      SRI_ARR(OTG_CRC0_DATA_RG, OTG, inst),                                    \
+      SRI_ARR(OTG_CRC0_DATA_B, OTG, inst),                                     \
+      SRI_ARR(OTG_CRC0_WINDOWA_X_CONTROL, OTG, inst),                          \
+      SRI_ARR(OTG_CRC0_WINDOWA_Y_CONTROL, OTG, inst),                          \
+      SRI_ARR(OTG_CRC0_WINDOWB_X_CONTROL, OTG, inst),                          \
+      SRI_ARR(OTG_CRC0_WINDOWB_Y_CONTROL, OTG, inst),                          \
+      SR_ARR(GSL_SOURCE_SELECT, inst),                                         \
+      SRI_ARR(OTG_TRIGA_MANUAL_TRIG, OTG, inst),                               \
+      SRI_ARR(OTG_GLOBAL_CONTROL1, OTG, inst),                                 \
+      SRI_ARR(OTG_GLOBAL_CONTROL2, OTG, inst),                                 \
+      SRI_ARR(OTG_GSL_WINDOW_X, OTG, inst),                                    \
+      SRI_ARR(OTG_GSL_WINDOW_Y, OTG, inst),                                    \
+      SRI_ARR(OTG_VUPDATE_KEEPOUT, OTG, inst),                                 \
+      SRI_ARR(OTG_DSC_START_POSITION, OTG, inst),                              \
+      SRI_ARR(OTG_DRR_TRIGGER_WINDOW, OTG, inst),                              \
+      SRI_ARR(OTG_DRR_V_TOTAL_CHANGE, OTG, inst),                              \
+      SRI_ARR(OPTC_DATA_FORMAT_CONTROL, ODM, inst),                            \
+      SRI_ARR(OPTC_BYTES_PER_PIXEL, ODM, inst),                                \
+      SRI_ARR(OPTC_WIDTH_CONTROL, ODM, inst),                                  \
+      SRI_ARR(OPTC_MEMORY_CONFIG, ODM, inst),                                  \
+      SRI_ARR(OTG_DRR_CONTROL, OTG, inst)
+
+/* HUBP */
+
+#define HUBP_REG_LIST_DCN_VM_RI(id)                                            \
+  SRI_ARR(NOM_PARAMETERS_0, HUBPREQ, id),                                      \
+      SRI_ARR(NOM_PARAMETERS_1, HUBPREQ, id),                                  \
+      SRI_ARR(NOM_PARAMETERS_2, HUBPREQ, id),                                  \
+      SRI_ARR(NOM_PARAMETERS_3, HUBPREQ, id),                                  \
+      SRI_ARR(DCN_VM_MX_L1_TLB_CNTL, HUBPREQ, id)
+#define HUBP_REG_LIST_DCN_RI(id)                                               \
+  SRI_ARR(DCHUBP_CNTL, HUBP, id), SRI_ARR(HUBPREQ_DEBUG_DB, HUBP, id),         \
+      SRI_ARR(HUBPREQ_DEBUG, HUBP, id), SRI_ARR(DCSURF_ADDR_CONFIG, HUBP, id), \
+      SRI_ARR(DCSURF_TILING_CONFIG, HUBP, id),                                 \
+      SRI_ARR(DCSURF_SURFACE_PITCH, HUBPREQ, id),                              \
+      SRI_ARR(DCSURF_SURFACE_PITCH_C, HUBPREQ, id),                            \
+      SRI_ARR(DCSURF_SURFACE_CONFIG, HUBP, id),                                \
+      SRI_ARR(DCSURF_FLIP_CONTROL, HUBPREQ, id),                               \
+      SRI_ARR(DCSURF_PRI_VIEWPORT_DIMENSION, HUBP, id),                        \
+      SRI_ARR(DCSURF_PRI_VIEWPORT_START, HUBP, id),                            \
+      SRI_ARR(DCSURF_SEC_VIEWPORT_DIMENSION, HUBP, id),                        \
+      SRI_ARR(DCSURF_SEC_VIEWPORT_START, HUBP, id),                            \
+      SRI_ARR(DCSURF_PRI_VIEWPORT_DIMENSION_C, HUBP, id),                      \
+      SRI_ARR(DCSURF_PRI_VIEWPORT_START_C, HUBP, id),                          \
+      SRI_ARR(DCSURF_SEC_VIEWPORT_DIMENSION_C, HUBP, id),                      \
+      SRI_ARR(DCSURF_SEC_VIEWPORT_START_C, HUBP, id),                          \
+      SRI_ARR(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH, HUBPREQ, id),               \
+      SRI_ARR(DCSURF_PRIMARY_SURFACE_ADDRESS, HUBPREQ, id),                    \
+      SRI_ARR(DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH, HUBPREQ, id),             \
+      SRI_ARR(DCSURF_SECONDARY_SURFACE_ADDRESS, HUBPREQ, id),                  \
+      SRI_ARR(DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH, HUBPREQ, id),          \
+      SRI_ARR(DCSURF_PRIMARY_META_SURFACE_ADDRESS, HUBPREQ, id),               \
+      SRI_ARR(DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH, HUBPREQ, id),        \
+      SRI_ARR(DCSURF_SECONDARY_META_SURFACE_ADDRESS, HUBPREQ, id),             \
+      SRI_ARR(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C, HUBPREQ, id),             \
+      SRI_ARR(DCSURF_PRIMARY_SURFACE_ADDRESS_C, HUBPREQ, id),                  \
+      SRI_ARR(DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C, HUBPREQ, id),           \
+      SRI_ARR(DCSURF_SECONDARY_SURFACE_ADDRESS_C, HUBPREQ, id),                \
+      SRI_ARR(DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C, HUBPREQ, id),        \
+      SRI_ARR(DCSURF_PRIMARY_META_SURFACE_ADDRESS_C, HUBPREQ, id),             \
+      SRI_ARR(DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C, HUBPREQ, id),      \
+      SRI_ARR(DCSURF_SECONDARY_META_SURFACE_ADDRESS_C, HUBPREQ, id),           \
+      SRI_ARR(DCSURF_SURFACE_INUSE, HUBPREQ, id),                              \
+      SRI_ARR(DCSURF_SURFACE_INUSE_HIGH, HUBPREQ, id),                         \
+      SRI_ARR(DCSURF_SURFACE_INUSE_C, HUBPREQ, id),                            \
+      SRI_ARR(DCSURF_SURFACE_INUSE_HIGH_C, HUBPREQ, id),                       \
+      SRI_ARR(DCSURF_SURFACE_EARLIEST_INUSE, HUBPREQ, id),                     \
+      SRI_ARR(DCSURF_SURFACE_EARLIEST_INUSE_HIGH, HUBPREQ, id),                \
+      SRI_ARR(DCSURF_SURFACE_EARLIEST_INUSE_C, HUBPREQ, id),                   \
+      SRI_ARR(DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C, HUBPREQ, id),              \
+      SRI_ARR(DCSURF_SURFACE_CONTROL, HUBPREQ, id),                            \
+      SRI_ARR(DCSURF_SURFACE_FLIP_INTERRUPT, HUBPREQ, id),                     \
+      SRI_ARR(HUBPRET_CONTROL, HUBPRET, id),                                   \
+      SRI_ARR(HUBPRET_READ_LINE_STATUS, HUBPRET, id),                          \
+      SRI_ARR(DCN_EXPANSION_MODE, HUBPREQ, id),                                \
+      SRI_ARR(DCHUBP_REQ_SIZE_CONFIG, HUBP, id),                               \
+      SRI_ARR(DCHUBP_REQ_SIZE_CONFIG_C, HUBP, id),                             \
+      SRI_ARR(BLANK_OFFSET_0, HUBPREQ, id),                                    \
+      SRI_ARR(BLANK_OFFSET_1, HUBPREQ, id),                                    \
+      SRI_ARR(DST_DIMENSIONS, HUBPREQ, id),                                    \
+      SRI_ARR(DST_AFTER_SCALER, HUBPREQ, id),                                  \
+      SRI_ARR(VBLANK_PARAMETERS_0, HUBPREQ, id),                               \
+      SRI_ARR(REF_FREQ_TO_PIX_FREQ, HUBPREQ, id),                              \
+      SRI_ARR(VBLANK_PARAMETERS_1, HUBPREQ, id),                               \
+      SRI_ARR(VBLANK_PARAMETERS_3, HUBPREQ, id),                               \
+      SRI_ARR(NOM_PARAMETERS_4, HUBPREQ, id),                                  \
+      SRI_ARR(NOM_PARAMETERS_5, HUBPREQ, id),                                  \
+      SRI_ARR(PER_LINE_DELIVERY_PRE, HUBPREQ, id),                             \
+      SRI_ARR(PER_LINE_DELIVERY, HUBPREQ, id),                                 \
+      SRI_ARR(VBLANK_PARAMETERS_2, HUBPREQ, id),                               \
+      SRI_ARR(VBLANK_PARAMETERS_4, HUBPREQ, id),                               \
+      SRI_ARR(NOM_PARAMETERS_6, HUBPREQ, id),                                  \
+      SRI_ARR(NOM_PARAMETERS_7, HUBPREQ, id),                                  \
+      SRI_ARR(DCN_TTU_QOS_WM, HUBPREQ, id),                                    \
+      SRI_ARR(DCN_GLOBAL_TTU_CNTL, HUBPREQ, id),                               \
+      SRI_ARR(DCN_SURF0_TTU_CNTL0, HUBPREQ, id),                               \
+      SRI_ARR(DCN_SURF0_TTU_CNTL1, HUBPREQ, id),                               \
+      SRI_ARR(DCN_SURF1_TTU_CNTL0, HUBPREQ, id),                               \
+      SRI_ARR(DCN_SURF1_TTU_CNTL1, HUBPREQ, id),                               \
+      SRI_ARR(DCN_CUR0_TTU_CNTL0, HUBPREQ, id),                                \
+      SRI_ARR(DCN_CUR0_TTU_CNTL1, HUBPREQ, id),                                \
+      SRI_ARR(HUBP_CLK_CNTL, HUBP, id)
+#define HUBP_REG_LIST_DCN2_COMMON_RI(id)                                       \
+  HUBP_REG_LIST_DCN_RI(id), HUBP_REG_LIST_DCN_VM_RI(id),                       \
+      SRI_ARR(PREFETCH_SETTINGS, HUBPREQ, id),                                 \
+      SRI_ARR(PREFETCH_SETTINGS_C, HUBPREQ, id),                               \
+      SRI_ARR(DCN_VM_SYSTEM_APERTURE_LOW_ADDR, HUBPREQ, id),                   \
+      SRI_ARR(DCN_VM_SYSTEM_APERTURE_HIGH_ADDR, HUBPREQ, id),                  \
+      SRI_ARR(CURSOR_SETTINGS, HUBPREQ, id),                                   \
+      SRI_ARR(CURSOR_SURFACE_ADDRESS_HIGH, CURSOR0_, id),                      \
+      SRI_ARR(CURSOR_SURFACE_ADDRESS, CURSOR0_, id),                           \
+      SRI_ARR(CURSOR_SIZE, CURSOR0_, id),                                      \
+      SRI_ARR(CURSOR_CONTROL, CURSOR0_, id),                                   \
+      SRI_ARR(CURSOR_POSITION, CURSOR0_, id),                                  \
+      SRI_ARR(CURSOR_HOT_SPOT, CURSOR0_, id),                                  \
+      SRI_ARR(CURSOR_DST_OFFSET, CURSOR0_, id),                                \
+      SRI_ARR(DMDATA_ADDRESS_HIGH, CURSOR0_, id),                              \
+      SRI_ARR(DMDATA_ADDRESS_LOW, CURSOR0_, id),                               \
+      SRI_ARR(DMDATA_CNTL, CURSOR0_, id),                                      \
+      SRI_ARR(DMDATA_SW_CNTL, CURSOR0_, id),                                   \
+      SRI_ARR(DMDATA_QOS_CNTL, CURSOR0_, id),                                  \
+      SRI_ARR(DMDATA_SW_DATA, CURSOR0_, id),                                   \
+      SRI_ARR(DMDATA_STATUS, CURSOR0_, id),                                    \
+      SRI_ARR(FLIP_PARAMETERS_0, HUBPREQ, id),                                 \
+      SRI_ARR(FLIP_PARAMETERS_1, HUBPREQ, id),                                 \
+      SRI_ARR(FLIP_PARAMETERS_2, HUBPREQ, id),                                 \
+      SRI_ARR(DCN_CUR1_TTU_CNTL0, HUBPREQ, id),                                \
+      SRI_ARR(DCN_CUR1_TTU_CNTL1, HUBPREQ, id),                                \
+      SRI_ARR(DCSURF_FLIP_CONTROL2, HUBPREQ, id),                              \
+      SRI_ARR(VMID_SETTINGS_0, HUBPREQ, id)
+#define HUBP_REG_LIST_DCN21_RI(id)                                             \
+  HUBP_REG_LIST_DCN2_COMMON_RI(id), SRI_ARR(FLIP_PARAMETERS_3, HUBPREQ, id),   \
+      SRI_ARR(FLIP_PARAMETERS_4, HUBPREQ, id),                                 \
+      SRI_ARR(FLIP_PARAMETERS_5, HUBPREQ, id),                                 \
+      SRI_ARR(FLIP_PARAMETERS_6, HUBPREQ, id),                                 \
+      SRI_ARR(VBLANK_PARAMETERS_5, HUBPREQ, id),                               \
+      SRI_ARR(VBLANK_PARAMETERS_6, HUBPREQ, id)
+#define HUBP_REG_LIST_DCN30_RI(id)                                             \
+  HUBP_REG_LIST_DCN21_RI(id), SRI_ARR(DCN_DMDATA_VM_CNTL, HUBPREQ, id)
+#define HUBP_REG_LIST_DCN32_RI(id)                                             \
+  HUBP_REG_LIST_DCN30_RI(id), SRI_ARR(DCHUBP_MALL_CONFIG, HUBP, id),           \
+      SRI_ARR(DCHUBP_VMPG_CONFIG, HUBP, id),                                   \
+      SRI_ARR(UCLK_PSTATE_FORCE, HUBPREQ, id)
+
+/* HUBBUB */
+
+#define HUBBUB_REG_LIST_DCN32_RI(id)                                           \
+  SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A),                                   \
+      SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B),                               \
+      SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C),                               \
+      SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D),                               \
+      SR(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL),                                  \
+      SR(DCHUBBUB_ARB_DRAM_STATE_CNTL), SR(DCHUBBUB_ARB_SAT_LEVEL),            \
+      SR(DCHUBBUB_ARB_DF_REQ_OUTSTAND), SR(DCHUBBUB_GLOBAL_TIMER_CNTL),        \
+      SR(DCHUBBUB_SOFT_RESET), SR(DCHUBBUB_CRC_CTRL),                          \
+      SR(DCN_VM_FB_LOCATION_BASE), SR(DCN_VM_FB_LOCATION_TOP),                 \
+      SR(DCN_VM_FB_OFFSET), SR(DCN_VM_AGP_BOT), SR(DCN_VM_AGP_TOP),            \
+      SR(DCN_VM_AGP_BASE), HUBBUB_SR_WATERMARK_REG_LIST(),                     \
+      SR(DCHUBBUB_ARB_FRAC_URG_BW_NOM_A), SR(DCHUBBUB_ARB_FRAC_URG_BW_NOM_B),  \
+      SR(DCHUBBUB_ARB_FRAC_URG_BW_NOM_C), SR(DCHUBBUB_ARB_FRAC_URG_BW_NOM_D),  \
+      SR(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_A),                                     \
+      SR(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_B),                                     \
+      SR(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_C),                                     \
+      SR(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_D),                                     \
+      SR(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_A),                            \
+      SR(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_B),                            \
+      SR(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_C),                            \
+      SR(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_D), SR(DCHUBBUB_DET0_CTRL),    \
+      SR(DCHUBBUB_DET1_CTRL), SR(DCHUBBUB_DET2_CTRL), SR(DCHUBBUB_DET3_CTRL),  \
+      SR(DCHUBBUB_COMPBUF_CTRL), SR(COMPBUF_RESERVED_SPACE),                   \
+      SR(DCHUBBUB_DEBUG_CTRL_0),                                               \
+      SR(DCHUBBUB_ARB_USR_RETRAINING_CNTL),                                    \
+      SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_A),                             \
+      SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_B),                             \
+      SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_C),                             \
+      SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_D),                             \
+      SR(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_A),                         \
+      SR(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_B),                         \
+      SR(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_C),                         \
+      SR(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_D),                         \
+      SR(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_A),                         \
+      SR(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_B),                         \
+      SR(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_C),                         \
+      SR(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_D),                         \
+      SR(DCHUBBUB_ARB_MALL_CNTL),                                              \
+      SR(DCN_VM_FAULT_ADDR_MSB), SR(DCN_VM_FAULT_ADDR_LSB),                    \
+      SR(DCN_VM_FAULT_CNTL), SR(DCN_VM_FAULT_STATUS),                          \
+      SR(SDPIF_REQUEST_RATE_LIMIT)
+
+/* DCCG */
+
+#define DCCG_REG_LIST_DCN32_RI()                                               \
+  SR(DPPCLK_DTO_CTRL), DCCG_SRII(DTO_PARAM, DPPCLK, 0),                        \
+      DCCG_SRII(DTO_PARAM, DPPCLK, 1), DCCG_SRII(DTO_PARAM, DPPCLK, 2),        \
+      DCCG_SRII(DTO_PARAM, DPPCLK, 3), DCCG_SRII(CLOCK_CNTL, HDMICHARCLK, 0),  \
+      SR(PHYASYMCLK_CLOCK_CNTL), SR(PHYBSYMCLK_CLOCK_CNTL),                    \
+      SR(PHYCSYMCLK_CLOCK_CNTL), SR(PHYDSYMCLK_CLOCK_CNTL),                    \
+      SR(PHYESYMCLK_CLOCK_CNTL), SR(DPSTREAMCLK_CNTL), SR(HDMISTREAMCLK_CNTL), \
+      SR(SYMCLK32_SE_CNTL), SR(SYMCLK32_LE_CNTL),                              \
+      DCCG_SRII(PIXEL_RATE_CNTL, OTG, 0), DCCG_SRII(PIXEL_RATE_CNTL, OTG, 1),  \
+      DCCG_SRII(PIXEL_RATE_CNTL, OTG, 2), DCCG_SRII(PIXEL_RATE_CNTL, OTG, 3),  \
+      DCCG_SRII(MODULO, DTBCLK_DTO, 0), DCCG_SRII(MODULO, DTBCLK_DTO, 1),      \
+      DCCG_SRII(MODULO, DTBCLK_DTO, 2), DCCG_SRII(MODULO, DTBCLK_DTO, 3),      \
+      DCCG_SRII(PHASE, DTBCLK_DTO, 0), DCCG_SRII(PHASE, DTBCLK_DTO, 1),        \
+      DCCG_SRII(PHASE, DTBCLK_DTO, 2), DCCG_SRII(PHASE, DTBCLK_DTO, 3),        \
+      SR(DCCG_AUDIO_DTBCLK_DTO_MODULO), SR(DCCG_AUDIO_DTBCLK_DTO_PHASE),       \
+      SR(OTG_PIXEL_RATE_DIV), SR(DTBCLK_P_CNTL),                               \
+      SR(DCCG_AUDIO_DTO_SOURCE), SR(DENTIST_DISPCLK_CNTL)
+
+/* VMID */
+#define DCN20_VMID_REG_LIST_RI(id)                                             \
+  SRI_ARR(CNTL, DCN_VM_CONTEXT, id),                                           \
+      SRI_ARR(PAGE_TABLE_BASE_ADDR_HI32, DCN_VM_CONTEXT, id),                  \
+      SRI_ARR(PAGE_TABLE_BASE_ADDR_LO32, DCN_VM_CONTEXT, id),                  \
+      SRI_ARR(PAGE_TABLE_START_ADDR_HI32, DCN_VM_CONTEXT, id),                 \
+      SRI_ARR(PAGE_TABLE_START_ADDR_LO32, DCN_VM_CONTEXT, id),                 \
+      SRI_ARR(PAGE_TABLE_END_ADDR_HI32, DCN_VM_CONTEXT, id),                   \
+      SRI_ARR(PAGE_TABLE_END_ADDR_LO32, DCN_VM_CONTEXT, id)
+
+/* I2C HW */
+
+#define I2C_HW_ENGINE_COMMON_REG_LIST_RI(id)                                   \
+      SRI_ARR_I2C(SETUP, DC_I2C_DDC, id), SRI_ARR_I2C(SPEED, DC_I2C_DDC, id),  \
+      SRI_ARR_I2C(HW_STATUS, DC_I2C_DDC, id),                                  \
+      SR_ARR_I2C(DC_I2C_ARBITRATION, id),                                      \
+      SR_ARR_I2C(DC_I2C_CONTROL, id), SR_ARR_I2C(DC_I2C_SW_STATUS, id),        \
+      SR_ARR_I2C(DC_I2C_TRANSACTION0, id), SR_ARR_I2C(DC_I2C_TRANSACTION1, id),\
+      SR_ARR_I2C(DC_I2C_TRANSACTION2, id), SR_ARR_I2C(DC_I2C_TRANSACTION3, id),\
+      SR_ARR_I2C(DC_I2C_DATA, id), SR_ARR_I2C(MICROSECOND_TIME_BASE_DIV, id)
+
+#define I2C_HW_ENGINE_COMMON_REG_LIST_DCN30_RI(id)                             \
+      I2C_HW_ENGINE_COMMON_REG_LIST_RI(id), SR_ARR_I2C(DIO_MEM_PWR_CTRL, id),  \
+      SR_ARR_I2C(DIO_MEM_PWR_STATUS, id)
+
+#endif /* _DCN32_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c
new file mode 100644
index 000000000000..bedb70b98162
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c
@@ -0,0 +1,2065 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dm_services.h"
+#include "dc.h"
+
+#include "dcn32/dcn32_init.h"
+
+#include "resource.h"
+#include "include/irq_service_interface.h"
+#include "dcn32/dcn32_resource.h"
+#include "dcn321_resource.h"
+
+#include "dcn20/dcn20_resource.h"
+#include "dcn30/dcn30_resource.h"
+
+#include "dml/dcn321/dcn321_fpu.h"
+
+#include "dcn10/dcn10_ipp.h"
+#include "dcn30/dcn30_hubbub.h"
+#include "dcn31/dcn31_hubbub.h"
+#include "dcn32/dcn32_hubbub.h"
+#include "dcn32/dcn32_mpc.h"
+#include "dcn32/dcn32_hubp.h"
+#include "irq/dcn32/irq_service_dcn32.h"
+#include "dcn32/dcn32_dpp.h"
+#include "dcn32/dcn32_optc.h"
+#include "dcn20/dcn20_hwseq.h"
+#include "dcn30/dcn30_hwseq.h"
+#include "dce110/dce110_hwseq.h"
+#include "dcn30/dcn30_opp.h"
+#include "dcn20/dcn20_dsc.h"
+#include "dcn30/dcn30_vpg.h"
+#include "dcn30/dcn30_afmt.h"
+#include "dcn30/dcn30_dio_stream_encoder.h"
+#include "dcn32/dcn32_dio_stream_encoder.h"
+#include "dcn31/dcn31_hpo_dp_stream_encoder.h"
+#include "dcn31/dcn31_hpo_dp_link_encoder.h"
+#include "dcn32/dcn32_hpo_dp_link_encoder.h"
+#include "dcn31/dcn31_apg.h"
+#include "dcn31/dcn31_dio_link_encoder.h"
+#include "dcn32/dcn32_dio_link_encoder.h"
+#include "dcn321/dcn321_dio_link_encoder.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_hwseq.h"
+#include "clk_mgr.h"
+#include "virtual/virtual_stream_encoder.h"
+#include "dml/display_mode_vba.h"
+#include "dcn32/dcn32_dccg.h"
+#include "dcn10/dcn10_resource.h"
+#include "link.h"
+#include "dcn31/dcn31_panel_cntl.h"
+
+#include "dcn30/dcn30_dwb.h"
+#include "dcn32/dcn32_mmhubbub.h"
+
+#include "dcn/dcn_3_2_1_offset.h"
+#include "dcn/dcn_3_2_1_sh_mask.h"
+#include "nbio/nbio_4_3_0_offset.h"
+
+#include "reg_helper.h"
+#include "dce/dmub_abm.h"
+#include "dce/dmub_psr.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_i2c.h"
+
+#include "dml/dcn30/display_mode_vba_30.h"
+#include "vm_helper.h"
+#include "dcn20/dcn20_vmid.h"
+
+#define DC_LOGGER_INIT(logger)
+
+enum dcn321_clk_src_array_id {
+	DCN321_CLK_SRC_PLL0,
+	DCN321_CLK_SRC_PLL1,
+	DCN321_CLK_SRC_PLL2,
+	DCN321_CLK_SRC_PLL3,
+	DCN321_CLK_SRC_PLL4,
+	DCN321_CLK_SRC_TOTAL
+};
+
+/* begin *********************
+ * macros to expend register list macro defined in HW object header file
+ */
+
+/* DCN */
+#define BASE_INNER(seg) ctx->dcn_reg_offsets[seg]
+
+#define BASE(seg) BASE_INNER(seg)
+
+#define SR(reg_name)\
+	REG_STRUCT.reg_name = BASE(reg ## reg_name ## _BASE_IDX) +  \
+		reg ## reg_name
+#define SR_ARR(reg_name, id)\
+	REG_STRUCT[id].reg_name = BASE(reg ## reg_name ## _BASE_IDX) +  \
+		reg ## reg_name
+#define SR_ARR_INIT(reg_name, id, value)\
+	REG_STRUCT[id].reg_name =  value
+
+#define SRI(reg_name, block, id)\
+	REG_STRUCT.reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SRI_ARR(reg_name, block, id)\
+	REG_STRUCT[id].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SR_ARR_I2C(reg_name, id) \
+	REG_STRUCT[id-1].reg_name = BASE(reg##reg_name##_BASE_IDX) + reg##reg_name
+
+#define SRI_ARR_I2C(reg_name, block, id)\
+	REG_STRUCT[id-1].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SRI_ARR_ALPHABET(reg_name, block, index, id)\
+	REG_STRUCT[index].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SRI2(reg_name, block, id)\
+	.reg_name = BASE(reg ## reg_name ## _BASE_IDX) + \
+		reg ## reg_name
+#define SRI2_ARR(reg_name, block, id)\
+	REG_STRUCT[id].reg_name = BASE(reg ## reg_name ## _BASE_IDX) + \
+		reg ## reg_name
+
+#define SRIR(var_name, reg_name, block, id)\
+	.var_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SRII(reg_name, block, id)\
+	REG_STRUCT.reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SRII_ARR_2(reg_name, block, id, inst)\
+	REG_STRUCT[inst].reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SRII_MPC_RMU(reg_name, block, id)\
+	.RMU##_##reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SRII_DWB(reg_name, temp_name, block, id)\
+	REG_STRUCT.reg_name[id] = BASE(reg ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## temp_name
+
+#define DCCG_SRII(reg_name, block, id)\
+	REG_STRUCT.block ## _ ## reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SF_DWB2(reg_name, block, id, field_name, post_fix) \
+	.field_name = reg_name ## __ ## field_name ## post_fix
+
+#define VUPDATE_SRII(reg_name, block, id)\
+	REG_STRUCT.reg_name[id] = BASE(reg ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
+		reg ## reg_name ## _ ## block ## id
+
+/* NBIO */
+#define NBIO_BASE_INNER(seg) ctx->nbio_reg_offsets[seg]
+
+#define NBIO_BASE(seg) \
+	NBIO_BASE_INNER(seg)
+
+#define NBIO_SR(reg_name)\
+	REG_STRUCT.reg_name = NBIO_BASE(regBIF_BX0_ ## reg_name ## _BASE_IDX) + \
+		regBIF_BX0_ ## reg_name
+#define NBIO_SR_ARR(reg_name, id)\
+	REG_STRUCT[id].reg_name = NBIO_BASE(regBIF_BX0_ ## reg_name ## _BASE_IDX) + \
+		regBIF_BX0_ ## reg_name
+
+#define CTX ctx
+#define REG(reg_name) \
+	(ctx->dcn_reg_offsets[reg ## reg_name ## _BASE_IDX] + reg ## reg_name)
+
+static struct bios_registers bios_regs;
+
+#define bios_regs_init() \
+		( \
+		NBIO_SR(BIOS_SCRATCH_3),\
+		NBIO_SR(BIOS_SCRATCH_6)\
+		)
+
+#define clk_src_regs_init(index, pllid)\
+	CS_COMMON_REG_LIST_DCN3_0_RI(index, pllid)
+
+static struct dce110_clk_src_regs clk_src_regs[5];
+
+static const struct dce110_clk_src_shift cs_shift = {
+		CS_COMMON_MASK_SH_LIST_DCN3_2(__SHIFT)
+};
+
+static const struct dce110_clk_src_mask cs_mask = {
+		CS_COMMON_MASK_SH_LIST_DCN3_2(_MASK)
+};
+
+#define abm_regs_init(id)\
+		ABM_DCN32_REG_LIST_RI(id)
+
+static struct dce_abm_registers abm_regs[4];
+
+static const struct dce_abm_shift abm_shift = {
+		ABM_MASK_SH_LIST_DCN32(__SHIFT)
+};
+
+static const struct dce_abm_mask abm_mask = {
+		ABM_MASK_SH_LIST_DCN32(_MASK)
+};
+
+#define audio_regs_init(id)\
+		AUD_COMMON_REG_LIST_RI(id)
+
+static struct dce_audio_registers audio_regs[5];
+
+#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
+		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
+
+static const struct dce_audio_shift audio_shift = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_audio_mask audio_mask = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+#define vpg_regs_init(id)\
+	VPG_DCN3_REG_LIST_RI(id)
+
+static struct dcn30_vpg_registers vpg_regs[10];
+
+static const struct dcn30_vpg_shift vpg_shift = {
+	DCN3_VPG_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn30_vpg_mask vpg_mask = {
+	DCN3_VPG_MASK_SH_LIST(_MASK)
+};
+
+#define afmt_regs_init(id)\
+	AFMT_DCN3_REG_LIST_RI(id)
+
+static struct dcn30_afmt_registers afmt_regs[6];
+
+static const struct dcn30_afmt_shift afmt_shift = {
+	DCN3_AFMT_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn30_afmt_mask afmt_mask = {
+	DCN3_AFMT_MASK_SH_LIST(_MASK)
+};
+
+#define apg_regs_init(id)\
+	APG_DCN31_REG_LIST_RI(id)
+
+static struct dcn31_apg_registers apg_regs[4];
+
+static const struct dcn31_apg_shift apg_shift = {
+	DCN31_APG_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_apg_mask apg_mask = {
+		DCN31_APG_MASK_SH_LIST(_MASK)
+};
+
+#define stream_enc_regs_init(id)\
+	SE_DCN32_REG_LIST_RI(id)
+
+static struct dcn10_stream_enc_registers stream_enc_regs[5];
+
+static const struct dcn10_stream_encoder_shift se_shift = {
+		SE_COMMON_MASK_SH_LIST_DCN32(__SHIFT)
+};
+
+static const struct dcn10_stream_encoder_mask se_mask = {
+		SE_COMMON_MASK_SH_LIST_DCN32(_MASK)
+};
+
+
+#define aux_regs_init(id)\
+	DCN2_AUX_REG_LIST_RI(id)
+
+static struct dcn10_link_enc_aux_registers link_enc_aux_regs[5];
+
+#define hpd_regs_init(id)\
+	HPD_REG_LIST_RI(id)
+
+static struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[5];
+
+#define link_regs_init(id, phyid)\
+	( \
+	LE_DCN31_REG_LIST_RI(id), \
+	UNIPHY_DCN2_REG_LIST_RI(id, phyid)\
+	)
+	/*DPCS_DCN31_REG_LIST(id),*/ \
+
+static struct dcn10_link_enc_registers link_enc_regs[5];
+
+static const struct dcn10_link_enc_shift le_shift = {
+	LINK_ENCODER_MASK_SH_LIST_DCN31(__SHIFT), \
+//	DPCS_DCN31_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn10_link_enc_mask le_mask = {
+	LINK_ENCODER_MASK_SH_LIST_DCN31(_MASK), \
+//	DPCS_DCN31_MASK_SH_LIST(_MASK)
+};
+
+#define hpo_dp_stream_encoder_reg_init(id)\
+	DCN3_1_HPO_DP_STREAM_ENC_REG_LIST_RI(id)
+
+static struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[4];
+
+static const struct dcn31_hpo_dp_stream_encoder_shift hpo_dp_se_shift = {
+	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_hpo_dp_stream_encoder_mask hpo_dp_se_mask = {
+	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(_MASK)
+};
+
+
+#define hpo_dp_link_encoder_reg_init(id)\
+	DCN3_1_HPO_DP_LINK_ENC_REG_LIST_RI(id)
+	/*DCN3_1_RDPCSTX_REG_LIST(0),*/
+	/*DCN3_1_RDPCSTX_REG_LIST(1),*/
+	/*DCN3_1_RDPCSTX_REG_LIST(2),*/
+	/*DCN3_1_RDPCSTX_REG_LIST(3),*/
+
+static struct dcn31_hpo_dp_link_encoder_registers hpo_dp_link_enc_regs[2];
+
+static const struct dcn31_hpo_dp_link_encoder_shift hpo_dp_le_shift = {
+	DCN3_2_HPO_DP_LINK_ENC_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_hpo_dp_link_encoder_mask hpo_dp_le_mask = {
+	DCN3_2_HPO_DP_LINK_ENC_MASK_SH_LIST(_MASK)
+};
+
+#define dpp_regs_init(id)\
+	DPP_REG_LIST_DCN30_COMMON_RI(id)
+
+static struct dcn3_dpp_registers dpp_regs[4];
+
+static const struct dcn3_dpp_shift tf_shift = {
+		DPP_REG_LIST_SH_MASK_DCN30_COMMON(__SHIFT)
+};
+
+static const struct dcn3_dpp_mask tf_mask = {
+		DPP_REG_LIST_SH_MASK_DCN30_COMMON(_MASK)
+};
+
+
+#define opp_regs_init(id)\
+	OPP_REG_LIST_DCN30_RI(id)
+
+static struct dcn20_opp_registers opp_regs[4];
+
+static const struct dcn20_opp_shift opp_shift = {
+	OPP_MASK_SH_LIST_DCN20(__SHIFT)
+};
+
+static const struct dcn20_opp_mask opp_mask = {
+	OPP_MASK_SH_LIST_DCN20(_MASK)
+};
+
+#define aux_engine_regs_init(id) \
+	( \
+	AUX_COMMON_REG_LIST0_RI(id), SR_ARR_INIT(AUXN_IMPCAL, id, 0), \
+	SR_ARR_INIT(AUXP_IMPCAL, id, 0), \
+	SR_ARR_INIT(AUX_RESET_MASK, id, DP_AUX0_AUX_CONTROL__AUX_RESET_MASK), \
+	SR_ARR_INIT(AUX_RESET_MASK, id, DP_AUX0_AUX_CONTROL__AUX_RESET_MASK)\
+	)
+
+static struct dce110_aux_registers aux_engine_regs[5];
+
+static const struct dce110_aux_registers_shift aux_shift = {
+	DCN_AUX_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+	DCN_AUX_MASK_SH_LIST(_MASK)
+};
+
+#define dwbc_regs_dcn3_init(id)\
+	DWBC_COMMON_REG_LIST_DCN30_RI(id)
+
+static struct dcn30_dwbc_registers dwbc30_regs[1];
+
+static const struct dcn30_dwbc_shift dwbc30_shift = {
+	DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dcn30_dwbc_mask dwbc30_mask = {
+	DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+#define mcif_wb_regs_dcn3_init(id)\
+	MCIF_WB_COMMON_REG_LIST_DCN32_RI(id)
+
+static struct dcn30_mmhubbub_registers mcif_wb30_regs[1];
+
+static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN32(__SHIFT)
+};
+
+static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN32(_MASK)
+};
+
+#define dsc_regsDCN20_init(id)\
+	DSC_REG_LIST_DCN20_RI(id)
+
+static struct dcn20_dsc_registers dsc_regs[4];
+
+static const struct dcn20_dsc_shift dsc_shift = {
+	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
+};
+
+static const struct dcn20_dsc_mask dsc_mask = {
+	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
+};
+
+static struct dcn30_mpc_registers mpc_regs;
+#define dcn_mpc_regs_init()\
+	MPC_REG_LIST_DCN3_2_RI(0),\
+	MPC_REG_LIST_DCN3_2_RI(1),\
+	MPC_REG_LIST_DCN3_2_RI(2),\
+	MPC_REG_LIST_DCN3_2_RI(3),\
+	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(0),\
+	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(1),\
+	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(2),\
+	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(3),\
+	MPC_DWB_MUX_REG_LIST_DCN3_0_RI(0)
+
+static const struct dcn30_mpc_shift mpc_shift = {
+	MPC_COMMON_MASK_SH_LIST_DCN32(__SHIFT)
+};
+
+static const struct dcn30_mpc_mask mpc_mask = {
+	MPC_COMMON_MASK_SH_LIST_DCN32(_MASK)
+};
+
+#define optc_regs_init(id)\
+	OPTC_COMMON_REG_LIST_DCN3_2_RI(id)
+
+static struct dcn_optc_registers optc_regs[4];
+
+static const struct dcn_optc_shift optc_shift = {
+	OPTC_COMMON_MASK_SH_LIST_DCN3_2(__SHIFT)
+};
+
+static const struct dcn_optc_mask optc_mask = {
+	OPTC_COMMON_MASK_SH_LIST_DCN3_2(_MASK)
+};
+
+#define hubp_regs_init(id) \
+	HUBP_REG_LIST_DCN32_RI(id)
+
+static struct dcn_hubp2_registers hubp_regs[4];
+
+static const struct dcn_hubp2_shift hubp_shift = {
+		HUBP_MASK_SH_LIST_DCN32(__SHIFT)
+};
+
+static const struct dcn_hubp2_mask hubp_mask = {
+		HUBP_MASK_SH_LIST_DCN32(_MASK)
+};
+
+static struct dcn_hubbub_registers hubbub_reg;
+#define hubbub_reg_init()\
+		HUBBUB_REG_LIST_DCN32_RI(0)
+
+static const struct dcn_hubbub_shift hubbub_shift = {
+		HUBBUB_MASK_SH_LIST_DCN32(__SHIFT)
+};
+
+static const struct dcn_hubbub_mask hubbub_mask = {
+		HUBBUB_MASK_SH_LIST_DCN32(_MASK)
+};
+
+static struct dccg_registers dccg_regs;
+
+#define dccg_regs_init()\
+	DCCG_REG_LIST_DCN32_RI()
+
+static const struct dccg_shift dccg_shift = {
+		DCCG_MASK_SH_LIST_DCN32(__SHIFT)
+};
+
+static const struct dccg_mask dccg_mask = {
+		DCCG_MASK_SH_LIST_DCN32(_MASK)
+};
+
+
+#define SRII2(reg_name_pre, reg_name_post, id)\
+	.reg_name_pre ## _ ##  reg_name_post[id] = BASE(reg ## reg_name_pre \
+			## id ## _ ## reg_name_post ## _BASE_IDX) + \
+			reg ## reg_name_pre ## id ## _ ## reg_name_post
+
+
+#define HWSEQ_DCN32_REG_LIST()\
+	SR(DCHUBBUB_GLOBAL_TIMER_CNTL), \
+	SR(DIO_MEM_PWR_CTRL), \
+	SR(ODM_MEM_PWR_CTRL3), \
+	SR(MMHUBBUB_MEM_PWR_CNTL), \
+	SR(DCCG_GATE_DISABLE_CNTL), \
+	SR(DCCG_GATE_DISABLE_CNTL2), \
+	SR(DCFCLK_CNTL),\
+	SR(DC_MEM_GLOBAL_PWR_REQ_CNTL), \
+	SRII(PIXEL_RATE_CNTL, OTG, 0), \
+	SRII(PIXEL_RATE_CNTL, OTG, 1),\
+	SRII(PIXEL_RATE_CNTL, OTG, 2),\
+	SRII(PIXEL_RATE_CNTL, OTG, 3),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 0),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 1),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 2),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 3),\
+	SR(MICROSECOND_TIME_BASE_DIV), \
+	SR(MILLISECOND_TIME_BASE_DIV), \
+	SR(DISPCLK_FREQ_CHANGE_CNTL), \
+	SR(RBBMIF_TIMEOUT_DIS), \
+	SR(RBBMIF_TIMEOUT_DIS_2), \
+	SR(DCHUBBUB_CRC_CTRL), \
+	SR(DPP_TOP0_DPP_CRC_CTRL), \
+	SR(DPP_TOP0_DPP_CRC_VAL_B_A), \
+	SR(DPP_TOP0_DPP_CRC_VAL_R_G), \
+	SR(MPC_CRC_CTRL), \
+	SR(MPC_CRC_RESULT_GB), \
+	SR(MPC_CRC_RESULT_C), \
+	SR(MPC_CRC_RESULT_AR), \
+	SR(DOMAIN0_PG_CONFIG), \
+	SR(DOMAIN1_PG_CONFIG), \
+	SR(DOMAIN2_PG_CONFIG), \
+	SR(DOMAIN3_PG_CONFIG), \
+	SR(DOMAIN16_PG_CONFIG), \
+	SR(DOMAIN17_PG_CONFIG), \
+	SR(DOMAIN18_PG_CONFIG), \
+	SR(DOMAIN19_PG_CONFIG), \
+	SR(DOMAIN0_PG_STATUS), \
+	SR(DOMAIN1_PG_STATUS), \
+	SR(DOMAIN2_PG_STATUS), \
+	SR(DOMAIN3_PG_STATUS), \
+	SR(DOMAIN16_PG_STATUS), \
+	SR(DOMAIN17_PG_STATUS), \
+	SR(DOMAIN18_PG_STATUS), \
+	SR(DOMAIN19_PG_STATUS), \
+	SR(D1VGA_CONTROL), \
+	SR(D2VGA_CONTROL), \
+	SR(D3VGA_CONTROL), \
+	SR(D4VGA_CONTROL), \
+	SR(D5VGA_CONTROL), \
+	SR(D6VGA_CONTROL), \
+	SR(DC_IP_REQUEST_CNTL), \
+	SR(AZALIA_AUDIO_DTO), \
+	SR(AZALIA_CONTROLLER_CLOCK_GATING)
+
+static struct dce_hwseq_registers hwseq_reg;
+
+#define hwseq_reg_init()\
+	HWSEQ_DCN32_REG_LIST()
+
+#define HWSEQ_DCN32_MASK_SH_LIST(mask_sh)\
+	HWSEQ_DCN_MASK_SH_LIST(mask_sh), \
+	HWS_SF(, DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, mask_sh), \
+	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN0_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN1_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN2_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN3_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN16_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN17_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN18_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN19_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh), \
+	HWS_SF(, AZALIA_AUDIO_DTO, AZALIA_AUDIO_DTO_MODULE, mask_sh), \
+	HWS_SF(, HPO_TOP_CLOCK_CONTROL, HPO_HDMISTREAMCLK_G_GATE_DIS, mask_sh), \
+	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_UNASSIGNED_PWR_MODE, mask_sh), \
+	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_VBLANK_PWR_MODE, mask_sh), \
+	HWS_SF(, MMHUBBUB_MEM_PWR_CNTL, VGA_MEM_PWR_FORCE, mask_sh)
+
+static const struct dce_hwseq_shift hwseq_shift = {
+		HWSEQ_DCN32_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+		HWSEQ_DCN32_MASK_SH_LIST(_MASK)
+};
+#define vmid_regs_init(id)\
+		DCN20_VMID_REG_LIST_RI(id)
+
+static struct dcn_vmid_registers vmid_regs[16];
+
+static const struct dcn20_vmid_shift vmid_shifts = {
+		DCN20_VMID_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn20_vmid_mask vmid_masks = {
+		DCN20_VMID_MASK_SH_LIST(_MASK)
+};
+
+static const struct resource_caps res_cap_dcn321 = {
+	.num_timing_generator = 4,
+	.num_opp = 4,
+	.num_video_plane = 4,
+	.num_audio = 5,
+	.num_stream_encoder = 5,
+	.num_hpo_dp_stream_encoder = 4,
+	.num_hpo_dp_link_encoder = 2,
+	.num_pll = 5,
+	.num_dwb = 1,
+	.num_ddc = 5,
+	.num_vmid = 16,
+	.num_mpc_3dlut = 4,
+	.num_dsc = 4,
+};
+
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
+	.per_pixel_alpha = true,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = true,
+			.fp16 = true,
+			.p010 = true,
+			.ayuv = false,
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 16000,
+			.fp16 = 16000
+	},
+
+	// 6:1 downscaling ratio: 1000/6 = 166.666
+	.max_downscale_factor = {
+			.argb8888 = 167,
+			.nv12 = 167,
+			.fp16 = 167
+	},
+	64,
+	64
+};
+
+static const struct dc_debug_options debug_defaults_drv = {
+	.disable_dmcu = true,
+	.force_abm_enable = false,
+	.timing_trace = false,
+	.clock_trace = true,
+	.disable_pplib_clock_request = false,
+	.pipe_split_policy = MPC_SPLIT_AVOID,
+	.force_single_disp_pipe_split = false,
+	.disable_dcc = DCC_ENABLE,
+	.vsr_support = true,
+	.performance_trace = false,
+	.max_downscale_src_width = 7680,/*upto 8K*/
+	.disable_pplib_wm_range = false,
+	.scl_reset_length10 = true,
+	.sanity_checks = false,
+	.underflow_assert_delay_us = 0xFFFFFFFF,
+	.dwb_fi_phase = -1, // -1 = disable,
+	.dmub_command_table = true,
+	.enable_mem_low_power = {
+		.bits = {
+			.vga = false,
+			.i2c = false,
+			.dmcu = false, // This is previously known to cause hang on S3 cycles if enabled
+			.dscl = false,
+			.cm = false,
+			.mpc = false,
+			.optc = true,
+		}
+	},
+	.use_max_lb = true,
+	.force_disable_subvp = false,
+	.exit_idle_opt_for_cursor_updates = true,
+	.enable_single_display_2to1_odm_policy = true,
+
+	/*must match enable_single_display_2to1_odm_policy to support dynamic ODM transitions*/
+	.enable_double_buffered_dsc_pg_support = true,
+	.enable_dp_dig_pixel_rate_div_policy = 1,
+	.allow_sw_cursor_fallback = false, // Linux can't do SW cursor "fallback"
+	.alloc_extra_way_for_cursor = true,
+	.min_prefetch_in_strobe_ns = 60000, // 60us
+	.disable_unbounded_requesting = false,
+	.override_dispclk_programming = true,
+	.disable_fpo_optimizations = false,
+	.fpo_vactive_margin_us = 2000, // 2000us
+	.disable_fpo_vactive = false,
+	.disable_boot_optimizations = false,
+	.disable_subvp_high_refresh = false,
+	.fpo_vactive_min_active_margin_us = 200,
+	.fpo_vactive_max_blank_us = 1000,
+	.enable_legacy_fast_update = false,
+	.disable_dc_mode_overwrite = true,
+	.using_dml2 = false,
+};
+
+static struct dce_aux *dcn321_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT aux_engine_regs
+	aux_engine_regs_init(0),
+	aux_engine_regs_init(1),
+	aux_engine_regs_init(2),
+	aux_engine_regs_init(3),
+	aux_engine_regs_init(4);
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst],
+					&aux_mask,
+					&aux_shift,
+					ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+#define i2c_inst_regs_init(id)\
+	I2C_HW_ENGINE_COMMON_REG_LIST_DCN30_RI(id)
+
+static struct dce_i2c_registers i2c_hw_regs[5];
+
+static const struct dce_i2c_shift i2c_shifts = {
+		I2C_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
+};
+
+static const struct dce_i2c_mask i2c_masks = {
+		I2C_COMMON_MASK_SH_LIST_DCN30(_MASK)
+};
+
+static struct dce_i2c_hw *dcn321_i2c_hw_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw =
+		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT i2c_hw_regs
+	i2c_inst_regs_init(1),
+	i2c_inst_regs_init(2),
+	i2c_inst_regs_init(3),
+	i2c_inst_regs_init(4),
+	i2c_inst_regs_init(5);
+
+	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
+				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+
+static struct clock_source *dcn321_clock_source_create(
+		struct dc_context *ctx,
+		struct dc_bios *bios,
+		enum clock_source_id id,
+		const struct dce110_clk_src_regs *regs,
+		bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dcn31_clk_src_construct(clk_src, ctx, bios, id,
+			regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	kfree(clk_src);
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static struct hubbub *dcn321_hubbub_create(struct dc_context *ctx)
+{
+	int i;
+
+	struct dcn20_hubbub *hubbub2 = kzalloc(sizeof(struct dcn20_hubbub),
+					  GFP_KERNEL);
+
+	if (!hubbub2)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT hubbub_reg
+	hubbub_reg_init();
+
+#undef REG_STRUCT
+#define REG_STRUCT vmid_regs
+	vmid_regs_init(0),
+	vmid_regs_init(1),
+	vmid_regs_init(2),
+	vmid_regs_init(3),
+	vmid_regs_init(4),
+	vmid_regs_init(5),
+	vmid_regs_init(6),
+	vmid_regs_init(7),
+	vmid_regs_init(8),
+	vmid_regs_init(9),
+	vmid_regs_init(10),
+	vmid_regs_init(11),
+	vmid_regs_init(12),
+	vmid_regs_init(13),
+	vmid_regs_init(14),
+	vmid_regs_init(15);
+
+	hubbub32_construct(hubbub2, ctx,
+			&hubbub_reg,
+			&hubbub_shift,
+			&hubbub_mask,
+			ctx->dc->dml.ip.det_buffer_size_kbytes,
+			ctx->dc->dml.ip.pixel_chunk_size_kbytes,
+			ctx->dc->dml.ip.config_return_buffer_size_in_kbytes);
+
+
+	for (i = 0; i < res_cap_dcn321.num_vmid; i++) {
+		struct dcn20_vmid *vmid = &hubbub2->vmid[i];
+
+		vmid->ctx = ctx;
+
+		vmid->regs = &vmid_regs[i];
+		vmid->shifts = &vmid_shifts;
+		vmid->masks = &vmid_masks;
+	}
+
+	return &hubbub2->base;
+}
+
+static struct hubp *dcn321_hubp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn20_hubp *hubp2 =
+		kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
+
+	if (!hubp2)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT hubp_regs
+	hubp_regs_init(0),
+	hubp_regs_init(1),
+	hubp_regs_init(2),
+	hubp_regs_init(3);
+
+	if (hubp32_construct(hubp2, ctx, inst,
+			&hubp_regs[inst], &hubp_shift, &hubp_mask))
+		return &hubp2->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(hubp2);
+	return NULL;
+}
+
+static void dcn321_dpp_destroy(struct dpp **dpp)
+{
+	kfree(TO_DCN30_DPP(*dpp));
+	*dpp = NULL;
+}
+
+static struct dpp *dcn321_dpp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn3_dpp *dpp3 =
+		kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
+
+	if (!dpp3)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT dpp_regs
+	dpp_regs_init(0),
+	dpp_regs_init(1),
+	dpp_regs_init(2),
+	dpp_regs_init(3);
+
+	if (dpp32_construct(dpp3, ctx, inst,
+			&dpp_regs[inst], &tf_shift, &tf_mask))
+		return &dpp3->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(dpp3);
+	return NULL;
+}
+
+static struct mpc *dcn321_mpc_create(
+		struct dc_context *ctx,
+		int num_mpcc,
+		int num_rmu)
+{
+	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc),
+					  GFP_KERNEL);
+
+	if (!mpc30)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT mpc_regs
+	dcn_mpc_regs_init();
+
+	dcn32_mpc_construct(mpc30, ctx,
+			&mpc_regs,
+			&mpc_shift,
+			&mpc_mask,
+			num_mpcc,
+			num_rmu);
+
+	return &mpc30->base;
+}
+
+static struct output_pixel_processor *dcn321_opp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_opp *opp2 =
+		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
+
+	if (!opp2) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+#undef REG_STRUCT
+#define REG_STRUCT opp_regs
+	opp_regs_init(0),
+	opp_regs_init(1),
+	opp_regs_init(2),
+	opp_regs_init(3);
+
+	dcn20_opp_construct(opp2, ctx, inst,
+			&opp_regs[inst], &opp_shift, &opp_mask);
+	return &opp2->base;
+}
+
+
+static struct timing_generator *dcn321_timing_generator_create(
+		struct dc_context *ctx,
+		uint32_t instance)
+{
+	struct optc *tgn10 =
+		kzalloc(sizeof(struct optc), GFP_KERNEL);
+
+	if (!tgn10)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT optc_regs
+	optc_regs_init(0),
+	optc_regs_init(1),
+	optc_regs_init(2),
+	optc_regs_init(3);
+
+	tgn10->base.inst = instance;
+	tgn10->base.ctx = ctx;
+
+	tgn10->tg_regs = &optc_regs[instance];
+	tgn10->tg_shift = &optc_shift;
+	tgn10->tg_mask = &optc_mask;
+
+	dcn32_timing_generator_init(tgn10);
+
+	return &tgn10->base;
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+		.max_hdmi_deep_color = COLOR_DEPTH_121212,
+		.max_hdmi_pixel_clock = 600000,
+		.hdmi_ycbcr420_supported = true,
+		.dp_ycbcr420_supported = true,
+		.fec_supported = true,
+		.flags.bits.IS_HBR2_CAPABLE = true,
+		.flags.bits.IS_HBR3_CAPABLE = true,
+		.flags.bits.IS_TPS3_CAPABLE = true,
+		.flags.bits.IS_TPS4_CAPABLE = true
+};
+
+static struct link_encoder *dcn321_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dcn20_link_encoder *enc20 =
+		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+
+	if (!enc20)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT link_enc_aux_regs
+	aux_regs_init(0),
+	aux_regs_init(1),
+	aux_regs_init(2),
+	aux_regs_init(3),
+	aux_regs_init(4);
+
+#undef REG_STRUCT
+#define REG_STRUCT link_enc_hpd_regs
+	hpd_regs_init(0),
+	hpd_regs_init(1),
+	hpd_regs_init(2),
+	hpd_regs_init(3),
+	hpd_regs_init(4);
+
+#undef REG_STRUCT
+#define REG_STRUCT link_enc_regs
+	link_regs_init(0, A),
+	link_regs_init(1, B),
+	link_regs_init(2, C),
+	link_regs_init(3, D),
+	link_regs_init(4, E);
+
+	dcn321_link_encoder_construct(enc20,
+			enc_init_data,
+			&link_enc_feature,
+			&link_enc_regs[enc_init_data->transmitter],
+			&link_enc_aux_regs[enc_init_data->channel - 1],
+			&link_enc_hpd_regs[enc_init_data->hpd_source],
+			&le_shift,
+			&le_mask);
+
+	return &enc20->enc10.base;
+}
+
+static void read_dce_straps(
+	struct dc_context *ctx,
+	struct resource_straps *straps)
+{
+	generic_reg_get(ctx, ctx->dcn_reg_offsets[regDC_PINSTRAPS_BASE_IDX] + regDC_PINSTRAPS,
+		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
+
+}
+
+static struct audio *dcn321_create_audio(
+		struct dc_context *ctx, unsigned int inst)
+{
+
+#undef REG_STRUCT
+#define REG_STRUCT audio_regs
+	audio_regs_init(0),
+	audio_regs_init(1),
+	audio_regs_init(2),
+	audio_regs_init(3),
+	audio_regs_init(4);
+
+	return dce_audio_create(ctx, inst,
+			&audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+static struct vpg *dcn321_vpg_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn30_vpg *vpg3 = kzalloc(sizeof(struct dcn30_vpg), GFP_KERNEL);
+
+	if (!vpg3)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT vpg_regs
+	vpg_regs_init(0),
+	vpg_regs_init(1),
+	vpg_regs_init(2),
+	vpg_regs_init(3),
+	vpg_regs_init(4),
+	vpg_regs_init(5),
+	vpg_regs_init(6),
+	vpg_regs_init(7),
+	vpg_regs_init(8),
+	vpg_regs_init(9);
+
+	vpg3_construct(vpg3, ctx, inst,
+			&vpg_regs[inst],
+			&vpg_shift,
+			&vpg_mask);
+
+	return &vpg3->base;
+}
+
+static struct afmt *dcn321_afmt_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn30_afmt *afmt3 = kzalloc(sizeof(struct dcn30_afmt), GFP_KERNEL);
+
+	if (!afmt3)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT afmt_regs
+	afmt_regs_init(0),
+	afmt_regs_init(1),
+	afmt_regs_init(2),
+	afmt_regs_init(3),
+	afmt_regs_init(4),
+	afmt_regs_init(5);
+
+	afmt3_construct(afmt3, ctx, inst,
+			&afmt_regs[inst],
+			&afmt_shift,
+			&afmt_mask);
+
+	return &afmt3->base;
+}
+
+static struct apg *dcn321_apg_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn31_apg *apg31 = kzalloc(sizeof(struct dcn31_apg), GFP_KERNEL);
+
+	if (!apg31)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT apg_regs
+	apg_regs_init(0),
+	apg_regs_init(1),
+	apg_regs_init(2),
+	apg_regs_init(3);
+
+	apg31_construct(apg31, ctx, inst,
+			&apg_regs[inst],
+			&apg_shift,
+			&apg_mask);
+
+	return &apg31->base;
+}
+
+static struct stream_encoder *dcn321_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dcn10_stream_encoder *enc1;
+	struct vpg *vpg;
+	struct afmt *afmt;
+	int vpg_inst;
+	int afmt_inst;
+
+	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
+	if (eng_id <= ENGINE_ID_DIGF) {
+		vpg_inst = eng_id;
+		afmt_inst = eng_id;
+	} else
+		return NULL;
+
+	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
+	vpg = dcn321_vpg_create(ctx, vpg_inst);
+	afmt = dcn321_afmt_create(ctx, afmt_inst);
+
+	if (!enc1 || !vpg || !afmt) {
+		kfree(enc1);
+		kfree(vpg);
+		kfree(afmt);
+		return NULL;
+	}
+
+#undef REG_STRUCT
+#define REG_STRUCT stream_enc_regs
+	stream_enc_regs_init(0),
+	stream_enc_regs_init(1),
+	stream_enc_regs_init(2),
+	stream_enc_regs_init(3),
+	stream_enc_regs_init(4);
+
+	dcn32_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
+					eng_id, vpg, afmt,
+					&stream_enc_regs[eng_id],
+					&se_shift, &se_mask);
+
+	return &enc1->base;
+}
+
+static struct hpo_dp_stream_encoder *dcn321_hpo_dp_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dcn31_hpo_dp_stream_encoder *hpo_dp_enc31;
+	struct vpg *vpg;
+	struct apg *apg;
+	uint32_t hpo_dp_inst;
+	uint32_t vpg_inst;
+	uint32_t apg_inst;
+
+	ASSERT((eng_id >= ENGINE_ID_HPO_DP_0) && (eng_id <= ENGINE_ID_HPO_DP_3));
+	hpo_dp_inst = eng_id - ENGINE_ID_HPO_DP_0;
+
+	/* Mapping of VPG register blocks to HPO DP block instance:
+	 * VPG[6] -> HPO_DP[0]
+	 * VPG[7] -> HPO_DP[1]
+	 * VPG[8] -> HPO_DP[2]
+	 * VPG[9] -> HPO_DP[3]
+	 */
+	vpg_inst = hpo_dp_inst + 6;
+
+	/* Mapping of APG register blocks to HPO DP block instance:
+	 * APG[0] -> HPO_DP[0]
+	 * APG[1] -> HPO_DP[1]
+	 * APG[2] -> HPO_DP[2]
+	 * APG[3] -> HPO_DP[3]
+	 */
+	apg_inst = hpo_dp_inst;
+
+	/* allocate HPO stream encoder and create VPG sub-block */
+	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_stream_encoder), GFP_KERNEL);
+	vpg = dcn321_vpg_create(ctx, vpg_inst);
+	apg = dcn321_apg_create(ctx, apg_inst);
+
+	if (!hpo_dp_enc31 || !vpg || !apg) {
+		kfree(hpo_dp_enc31);
+		kfree(vpg);
+		kfree(apg);
+		return NULL;
+	}
+
+#undef REG_STRUCT
+#define REG_STRUCT hpo_dp_stream_enc_regs
+	hpo_dp_stream_encoder_reg_init(0),
+	hpo_dp_stream_encoder_reg_init(1),
+	hpo_dp_stream_encoder_reg_init(2),
+	hpo_dp_stream_encoder_reg_init(3);
+
+	dcn31_hpo_dp_stream_encoder_construct(hpo_dp_enc31, ctx, ctx->dc_bios,
+					hpo_dp_inst, eng_id, vpg, apg,
+					&hpo_dp_stream_enc_regs[hpo_dp_inst],
+					&hpo_dp_se_shift, &hpo_dp_se_mask);
+
+	return &hpo_dp_enc31->base;
+}
+
+static struct hpo_dp_link_encoder *dcn321_hpo_dp_link_encoder_create(
+	uint8_t inst,
+	struct dc_context *ctx)
+{
+	struct dcn31_hpo_dp_link_encoder *hpo_dp_enc31;
+
+	/* allocate HPO link encoder */
+	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
+
+#undef REG_STRUCT
+#define REG_STRUCT hpo_dp_link_enc_regs
+	hpo_dp_link_encoder_reg_init(0),
+	hpo_dp_link_encoder_reg_init(1);
+
+	hpo_dp_link_encoder32_construct(hpo_dp_enc31, ctx, inst,
+					&hpo_dp_link_enc_regs[inst],
+					&hpo_dp_le_shift, &hpo_dp_le_mask);
+
+	return &hpo_dp_enc31->base;
+}
+
+static struct dce_hwseq *dcn321_hwseq_create(
+	struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+#undef REG_STRUCT
+#define REG_STRUCT hwseq_reg
+	hwseq_reg_init();
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &hwseq_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+	}
+	return hws;
+}
+static const struct resource_create_funcs res_create_funcs = {
+	.read_dce_straps = read_dce_straps,
+	.create_audio = dcn321_create_audio,
+	.create_stream_encoder = dcn321_stream_encoder_create,
+	.create_hpo_dp_stream_encoder = dcn321_hpo_dp_stream_encoder_create,
+	.create_hpo_dp_link_encoder = dcn321_hpo_dp_link_encoder_create,
+	.create_hwseq = dcn321_hwseq_create,
+};
+
+static void dcn321_resource_destruct(struct dcn321_resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->base.stream_enc_count; i++) {
+		if (pool->base.stream_enc[i] != NULL) {
+			if (pool->base.stream_enc[i]->vpg != NULL) {
+				kfree(DCN30_VPG_FROM_VPG(pool->base.stream_enc[i]->vpg));
+				pool->base.stream_enc[i]->vpg = NULL;
+			}
+			if (pool->base.stream_enc[i]->afmt != NULL) {
+				kfree(DCN30_AFMT_FROM_AFMT(pool->base.stream_enc[i]->afmt));
+				pool->base.stream_enc[i]->afmt = NULL;
+			}
+			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
+			pool->base.stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.hpo_dp_stream_enc_count; i++) {
+		if (pool->base.hpo_dp_stream_enc[i] != NULL) {
+			if (pool->base.hpo_dp_stream_enc[i]->vpg != NULL) {
+				kfree(DCN30_VPG_FROM_VPG(pool->base.hpo_dp_stream_enc[i]->vpg));
+				pool->base.hpo_dp_stream_enc[i]->vpg = NULL;
+			}
+			if (pool->base.hpo_dp_stream_enc[i]->apg != NULL) {
+				kfree(DCN31_APG_FROM_APG(pool->base.hpo_dp_stream_enc[i]->apg));
+				pool->base.hpo_dp_stream_enc[i]->apg = NULL;
+			}
+			kfree(DCN3_1_HPO_DP_STREAM_ENC_FROM_HPO_STREAM_ENC(pool->base.hpo_dp_stream_enc[i]));
+			pool->base.hpo_dp_stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.hpo_dp_link_enc_count; i++) {
+		if (pool->base.hpo_dp_link_enc[i] != NULL) {
+			kfree(DCN3_1_HPO_DP_LINK_ENC_FROM_HPO_LINK_ENC(pool->base.hpo_dp_link_enc[i]));
+			pool->base.hpo_dp_link_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		if (pool->base.dscs[i] != NULL)
+			dcn20_dsc_destroy(&pool->base.dscs[i]);
+	}
+
+	if (pool->base.mpc != NULL) {
+		kfree(TO_DCN20_MPC(pool->base.mpc));
+		pool->base.mpc = NULL;
+	}
+	if (pool->base.hubbub != NULL) {
+		kfree(TO_DCN20_HUBBUB(pool->base.hubbub));
+		pool->base.hubbub = NULL;
+	}
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (pool->base.dpps[i] != NULL)
+			dcn321_dpp_destroy(&pool->base.dpps[i]);
+
+		if (pool->base.ipps[i] != NULL)
+			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
+
+		if (pool->base.hubps[i] != NULL) {
+			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
+			pool->base.hubps[i] = NULL;
+		}
+
+		if (pool->base.irqs != NULL)
+			dal_irq_service_destroy(&pool->base.irqs);
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+		if (pool->base.hw_i2cs[i] != NULL) {
+			kfree(pool->base.hw_i2cs[i]);
+			pool->base.hw_i2cs[i] = NULL;
+		}
+		if (pool->base.sw_i2cs[i] != NULL) {
+			kfree(pool->base.sw_i2cs[i]);
+			pool->base.sw_i2cs[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		if (pool->base.opps[i] != NULL)
+			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.timing_generators[i] != NULL)	{
+			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
+			pool->base.timing_generators[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+		if (pool->base.dwbc[i] != NULL) {
+			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
+			pool->base.dwbc[i] = NULL;
+		}
+		if (pool->base.mcif_wb[i] != NULL) {
+			kfree(TO_DCN30_MMHUBBUB(pool->base.mcif_wb[i]));
+			pool->base.mcif_wb[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.audio_count; i++) {
+		if (pool->base.audios[i])
+			dce_aud_destroy(&pool->base.audios[i]);
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] != NULL) {
+			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
+			pool->base.clock_sources[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+		if (pool->base.mpc_lut[i] != NULL) {
+			dc_3dlut_func_release(pool->base.mpc_lut[i]);
+			pool->base.mpc_lut[i] = NULL;
+		}
+		if (pool->base.mpc_shaper[i] != NULL) {
+			dc_transfer_func_release(pool->base.mpc_shaper[i]);
+			pool->base.mpc_shaper[i] = NULL;
+		}
+	}
+
+	if (pool->base.dp_clock_source != NULL) {
+		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
+		pool->base.dp_clock_source = NULL;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.multiple_abms[i] != NULL)
+			dce_abm_destroy(&pool->base.multiple_abms[i]);
+	}
+
+	if (pool->base.psr != NULL)
+		dmub_psr_destroy(&pool->base.psr);
+
+	if (pool->base.dccg != NULL)
+		dcn_dccg_destroy(&pool->base.dccg);
+
+	if (pool->base.oem_device != NULL) {
+		struct dc *dc = pool->base.oem_device->ctx->dc;
+
+		dc->link_srv->destroy_ddc_service(&pool->base.oem_device);
+	}
+}
+
+
+static bool dcn321_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t dwb_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < dwb_count; i++) {
+		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc),
+						    GFP_KERNEL);
+
+		if (!dwbc30) {
+			dm_error("DC: failed to create dwbc30!\n");
+			return false;
+		}
+
+#undef REG_STRUCT
+#define REG_STRUCT dwbc30_regs
+		dwbc_regs_dcn3_init(0);
+
+		dcn30_dwbc_construct(dwbc30, ctx,
+				&dwbc30_regs[i],
+				&dwbc30_shift,
+				&dwbc30_mask,
+				i);
+
+		pool->dwbc[i] = &dwbc30->base;
+	}
+	return true;
+}
+
+static bool dcn321_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t dwb_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < dwb_count; i++) {
+		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub),
+						    GFP_KERNEL);
+
+		if (!mcif_wb30) {
+			dm_error("DC: failed to create mcif_wb30!\n");
+			return false;
+		}
+
+#undef REG_STRUCT
+#define REG_STRUCT mcif_wb30_regs
+		mcif_wb_regs_dcn3_init(0);
+
+		dcn32_mmhubbub_construct(mcif_wb30, ctx,
+				&mcif_wb30_regs[i],
+				&mcif_wb30_shift,
+				&mcif_wb30_mask,
+				i);
+
+		pool->mcif_wb[i] = &mcif_wb30->base;
+	}
+	return true;
+}
+
+static struct display_stream_compressor *dcn321_dsc_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_dsc *dsc =
+		kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
+
+	if (!dsc) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+#undef REG_STRUCT
+#define REG_STRUCT dsc_regs
+	dsc_regsDCN20_init(0),
+	dsc_regsDCN20_init(1),
+	dsc_regsDCN20_init(2),
+	dsc_regsDCN20_init(3);
+
+	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
+
+	dsc->max_image_width = 6016;
+
+	return &dsc->base;
+}
+
+static void dcn321_destroy_resource_pool(struct resource_pool **pool)
+{
+	struct dcn321_resource_pool *dcn321_pool = TO_DCN321_RES_POOL(*pool);
+
+	dcn321_resource_destruct(dcn321_pool);
+	kfree(dcn321_pool);
+	*pool = NULL;
+}
+
+static struct dc_cap_funcs cap_funcs = {
+	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap,
+	.get_subvp_en = dcn32_subvp_in_use,
+};
+
+static void dcn321_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
+{
+	DC_FP_START();
+	dcn321_update_bw_bounding_box_fpu(dc, bw_params);
+	DC_FP_END();
+}
+
+static struct resource_funcs dcn321_res_pool_funcs = {
+	.destroy = dcn321_destroy_resource_pool,
+	.link_enc_create = dcn321_link_encoder_create,
+	.link_enc_create_minimal = NULL,
+	.panel_cntl_create = dcn32_panel_cntl_create,
+	.validate_bandwidth = dcn32_validate_bandwidth,
+	.calculate_wm_and_dlg = dcn32_calculate_wm_and_dlg,
+	.populate_dml_pipes = dcn32_populate_dml_pipes_from_context,
+	.acquire_free_pipe_as_secondary_dpp_pipe = dcn32_acquire_free_pipe_as_secondary_dpp_pipe,
+	.acquire_free_pipe_as_secondary_opp_head = dcn32_acquire_free_pipe_as_secondary_opp_head,
+	.release_pipe = dcn20_release_pipe,
+	.add_stream_to_ctx = dcn30_add_stream_to_ctx,
+	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
+	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
+	.populate_dml_writeback_from_context = dcn30_populate_dml_writeback_from_context,
+	.set_mcif_arb_params = dcn30_set_mcif_arb_params,
+	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
+	.acquire_post_bldn_3dlut = dcn32_acquire_post_bldn_3dlut,
+	.release_post_bldn_3dlut = dcn32_release_post_bldn_3dlut,
+	.update_bw_bounding_box = dcn321_update_bw_bounding_box,
+	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
+	.update_soc_for_wm_a = dcn30_update_soc_for_wm_a,
+	.add_phantom_pipes = dcn32_add_phantom_pipes,
+	.remove_phantom_pipes = dcn32_remove_phantom_pipes,
+	.retain_phantom_pipes = dcn32_retain_phantom_pipes,
+	.save_mall_state = dcn32_save_mall_state,
+	.restore_mall_state = dcn32_restore_mall_state,
+};
+
+static uint32_t read_pipe_fuses(struct dc_context *ctx)
+{
+	uint32_t value = REG_READ(CC_DC_PIPE_DIS);
+	/* DCN321 support max 4 pipes */
+	value = value & 0xf;
+	return value;
+}
+
+
+static bool dcn321_resource_construct(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct dcn321_resource_pool *pool)
+{
+	int i, j;
+	struct dc_context *ctx = dc->ctx;
+	struct irq_service_init_data init_data;
+	struct ddc_service_init_data ddc_init_data = {0};
+	uint32_t pipe_fuses = 0;
+	uint32_t num_pipes  = 4;
+
+#undef REG_STRUCT
+#define REG_STRUCT bios_regs
+	bios_regs_init();
+
+#undef REG_STRUCT
+#define REG_STRUCT clk_src_regs
+	clk_src_regs_init(0, A),
+	clk_src_regs_init(1, B),
+	clk_src_regs_init(2, C),
+	clk_src_regs_init(3, D),
+	clk_src_regs_init(4, E);
+
+#undef REG_STRUCT
+#define REG_STRUCT abm_regs
+	abm_regs_init(0),
+	abm_regs_init(1),
+	abm_regs_init(2),
+	abm_regs_init(3);
+
+#undef REG_STRUCT
+#define REG_STRUCT dccg_regs
+	dccg_regs_init();
+
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	pool->base.res_cap = &res_cap_dcn321;
+	/* max number of pipes for ASIC before checking for pipe fuses */
+	num_pipes  = pool->base.res_cap->num_timing_generator;
+	pipe_fuses = read_pipe_fuses(ctx);
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++)
+		if (pipe_fuses & 1 << i)
+			num_pipes--;
+
+	if (pipe_fuses & 1)
+		ASSERT(0); //Unexpected - Pipe 0 should always be fully functional!
+
+	if (pipe_fuses & CC_DC_PIPE_DIS__DC_FULL_DIS_MASK)
+		ASSERT(0); //Entire DCN is harvested!
+
+	/* within dml lib, initial value is hard coded, if ASIC pipe is fused, the
+	 * value will be changed, update max_num_dpp and max_num_otg for dml.
+	 */
+	dcn3_21_ip.max_num_dpp = num_pipes;
+	dcn3_21_ip.max_num_otg = num_pipes;
+
+	pool->base.funcs = &dcn321_res_pool_funcs;
+
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+	pool->base.timing_generator_count = num_pipes;
+	pool->base.pipe_count = num_pipes;
+	pool->base.mpcc_count = num_pipes;
+	dc->caps.max_downscale_ratio = 600;
+	dc->caps.i2c_speed_in_khz = 100;
+	dc->caps.i2c_speed_in_khz_hdcp = 100; /*1.4 w/a applied by default*/
+	/* TODO: Bring max cursor size back to 256 after subvp cursor corruption is fixed*/
+	dc->caps.max_cursor_size = 64;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.dmdata_alloc_size = 2048;
+	dc->caps.mall_size_per_mem_channel = 4;
+	dc->caps.mall_size_total = 0;
+	dc->caps.cursor_cache_size = dc->caps.max_cursor_size * dc->caps.max_cursor_size * 8;
+	dc->caps.cache_line_size = 64;
+	dc->caps.cache_num_ways = 16;
+
+	/* Calculate the available MALL space */
+	dc->caps.max_cab_allocation_bytes = dcn32_calc_num_avail_chans_for_mall(
+		dc, dc->ctx->dc_bios->vram_info.num_chans) *
+		dc->caps.mall_size_per_mem_channel * 1024 * 1024;
+	dc->caps.mall_size_total = dc->caps.max_cab_allocation_bytes;
+
+	dc->caps.subvp_fw_processing_delay_us = 15;
+	dc->caps.subvp_drr_max_vblank_margin_us = 40;
+	dc->caps.subvp_prefetch_end_to_mall_start_us = 15;
+	dc->caps.subvp_swath_height_margin_lines = 16;
+	dc->caps.subvp_pstate_allow_width_us = 20;
+	dc->caps.subvp_vertical_int_margin_us = 30;
+	dc->caps.subvp_drr_vblank_start_margin_us = 100; // 100us margin
+	dc->caps.max_slave_planes = 2;
+	dc->caps.max_slave_yuv_planes = 2;
+	dc->caps.max_slave_rgb_planes = 2;
+	dc->caps.post_blend_color_processing = true;
+	dc->caps.force_dp_tps4_for_cp2520 = true;
+	dc->caps.dp_hpo = true;
+	dc->caps.dp_hdmi21_pcon_support = true;
+	dc->caps.edp_dsc_support = true;
+	dc->caps.extended_aux_timeout_support = true;
+	dc->caps.dmcub_support = true;
+	dc->caps.max_v_total = (1 << 15) - 1;
+
+	/* Color pipeline capabilities */
+	dc->caps.color.dpp.dcn_arch = 1;
+	dc->caps.color.dpp.input_lut_shared = 0;
+	dc->caps.color.dpp.icsc = 1;
+	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
+	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
+	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
+	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
+	dc->caps.color.dpp.post_csc = 1;
+	dc->caps.color.dpp.gamma_corr = 1;
+	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
+
+	dc->caps.color.dpp.hw_3d_lut = 1;
+	dc->caps.color.dpp.ogam_ram = 1;
+	// no OGAM ROM on DCN2 and later ASICs
+	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
+	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
+	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
+	dc->caps.color.dpp.ocsc = 0;
+
+	dc->caps.color.mpc.gamut_remap = 1;
+	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //4, configurable to be before or after BLND in MPCC
+	dc->caps.color.mpc.ogam_ram = 1;
+	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
+	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
+	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
+	dc->caps.color.mpc.ocsc = 1;
+
+	dc->config.dc_mode_clk_limit_support = true;
+	/* read VBIOS LTTPR caps */
+	{
+		if (ctx->dc_bios->funcs->get_lttpr_caps) {
+			enum bp_result bp_query_result;
+			uint8_t is_vbios_lttpr_enable = 0;
+
+			bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
+			dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
+		}
+
+		/* interop bit is implicit */
+		{
+			dc->caps.vbios_lttpr_aware = true;
+		}
+	}
+
+	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
+		dc->debug = debug_defaults_drv;
+
+	// Init the vm_helper
+	if (dc->vm_helper)
+		vm_helper_init(dc->vm_helper, 16);
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	/* Clock Sources for Pixel Clock*/
+	pool->base.clock_sources[DCN321_CLK_SRC_PLL0] =
+			dcn321_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL0,
+				&clk_src_regs[0], false);
+	pool->base.clock_sources[DCN321_CLK_SRC_PLL1] =
+			dcn321_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL1,
+				&clk_src_regs[1], false);
+	pool->base.clock_sources[DCN321_CLK_SRC_PLL2] =
+			dcn321_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL2,
+				&clk_src_regs[2], false);
+	pool->base.clock_sources[DCN321_CLK_SRC_PLL3] =
+			dcn321_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL3,
+				&clk_src_regs[3], false);
+	pool->base.clock_sources[DCN321_CLK_SRC_PLL4] =
+			dcn321_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL4,
+				&clk_src_regs[4], false);
+
+	pool->base.clk_src_count = DCN321_CLK_SRC_TOTAL;
+
+	/* todo: not reuse phy_pll registers */
+	pool->base.dp_clock_source =
+			dcn321_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_ID_DP_DTO,
+				&clk_src_regs[0], true);
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	/* DCCG */
+	pool->base.dccg = dccg32_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
+	if (pool->base.dccg == NULL) {
+		dm_error("DC: failed to create dccg!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* DML */
+	dml_init_instance(&dc->dml, &dcn3_21_soc, &dcn3_21_ip, DML_PROJECT_DCN32);
+
+	/* IRQ Service */
+	init_data.ctx = dc->ctx;
+	pool->base.irqs = dal_irq_service_dcn32_create(&init_data);
+	if (!pool->base.irqs)
+		goto create_fail;
+
+	/* HUBBUB */
+	pool->base.hubbub = dcn321_hubbub_create(ctx);
+	if (pool->base.hubbub == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create hubbub!\n");
+		goto create_fail;
+	}
+
+	/* HUBPs, DPPs, OPPs, TGs, ABMs */
+	for (i = 0, j = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+
+		/* if pipe is disabled, skip instance of HW pipe,
+		 * i.e, skip ASIC register instance
+		 */
+		if (pipe_fuses & 1 << i)
+			continue;
+
+		pool->base.hubps[j] = dcn321_hubp_create(ctx, i);
+		if (pool->base.hubps[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create hubps!\n");
+			goto create_fail;
+		}
+
+		pool->base.dpps[j] = dcn321_dpp_create(ctx, i);
+		if (pool->base.dpps[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create dpps!\n");
+			goto create_fail;
+		}
+
+		pool->base.opps[j] = dcn321_opp_create(ctx, i);
+		if (pool->base.opps[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create output pixel processor!\n");
+			goto create_fail;
+		}
+
+		pool->base.timing_generators[j] = dcn321_timing_generator_create(
+				ctx, i);
+		if (pool->base.timing_generators[j] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto create_fail;
+		}
+
+		pool->base.multiple_abms[j] = dmub_abm_create(ctx,
+				&abm_regs[i],
+				&abm_shift,
+				&abm_mask);
+		if (pool->base.multiple_abms[j] == NULL) {
+			dm_error("DC: failed to create abm for pipe %d!\n", i);
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+
+		/* index for resource pool arrays for next valid pipe */
+		j++;
+	}
+
+	/* PSR */
+	pool->base.psr = dmub_psr_create(ctx);
+	if (pool->base.psr == NULL) {
+		dm_error("DC: failed to create psr obj!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* MPCCs */
+	pool->base.mpc = dcn321_mpc_create(ctx,  pool->base.res_cap->num_timing_generator, pool->base.res_cap->num_mpc_3dlut);
+	if (pool->base.mpc == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mpc!\n");
+		goto create_fail;
+	}
+
+	/* DSCs */
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		pool->base.dscs[i] = dcn321_dsc_create(ctx, i);
+		if (pool->base.dscs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create display stream compressor %d!\n", i);
+			goto create_fail;
+		}
+	}
+
+	/* DWB */
+	if (!dcn321_dwbc_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create dwbc!\n");
+		goto create_fail;
+	}
+
+	/* MMHUBBUB */
+	if (!dcn321_mmhubbub_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mcif_wb!\n");
+		goto create_fail;
+	}
+
+	/* AUX and I2C */
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.engines[i] = dcn321_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto create_fail;
+		}
+		pool->base.hw_i2cs[i] = dcn321_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create hw i2c!!\n");
+			goto create_fail;
+		}
+		pool->base.sw_i2cs[i] = NULL;
+	}
+
+	/* Audio, HWSeq, Stream Encoders including HPO and virtual, MPC 3D LUTs */
+	if (!resource_construct(num_virtual_links, dc, &pool->base,
+			&res_create_funcs))
+		goto create_fail;
+
+	/* HW Sequencer init functions and Plane caps */
+	dcn32_hw_sequencer_init_functions(dc);
+
+	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->cap_funcs = cap_funcs;
+
+	if (dc->ctx->dc_bios->fw_info.oem_i2c_present) {
+		ddc_init_data.ctx = dc->ctx;
+		ddc_init_data.link = NULL;
+		ddc_init_data.id.id = dc->ctx->dc_bios->fw_info.oem_i2c_obj_id;
+		ddc_init_data.id.enum_id = 0;
+		ddc_init_data.id.type = OBJECT_TYPE_GENERIC;
+		pool->base.oem_device = dc->link_srv->create_ddc_service(&ddc_init_data);
+	} else {
+		pool->base.oem_device = NULL;
+	}
+
+	dc->dml2_options.dcn_pipe_count = pool->base.pipe_count;
+	dc->dml2_options.use_native_pstate_optimization = false;
+	dc->dml2_options.use_native_soc_bb_construction = true;
+	dc->dml2_options.minimize_dispclk_using_odm = true;
+
+	dc->dml2_options.callbacks.dc = dc;
+	dc->dml2_options.callbacks.build_scaling_params = &resource_build_scaling_params;
+	dc->dml2_options.callbacks.can_support_mclk_switch_using_fw_based_vblank_stretch = &dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch;
+	dc->dml2_options.callbacks.acquire_secondary_pipe_for_mpc_odm = &dc_resource_acquire_secondary_pipe_for_mpc_odm_legacy;
+	dc->dml2_options.callbacks.update_pipes_for_stream_with_slice_count = &resource_update_pipes_for_stream_with_slice_count;
+	dc->dml2_options.callbacks.update_pipes_for_plane_with_slice_count = &resource_update_pipes_for_plane_with_slice_count;
+	dc->dml2_options.callbacks.get_mpc_slice_index = &resource_get_mpc_slice_index;
+	dc->dml2_options.callbacks.get_odm_slice_index = &resource_get_odm_slice_index;
+	dc->dml2_options.callbacks.get_opp_head = &resource_get_opp_head;
+
+	dc->dml2_options.svp_pstate.callbacks.dc = dc;
+	dc->dml2_options.svp_pstate.callbacks.add_plane_to_context = &dc_add_plane_to_context;
+	dc->dml2_options.svp_pstate.callbacks.add_stream_to_ctx = &dc_add_stream_to_ctx;
+	dc->dml2_options.svp_pstate.callbacks.build_scaling_params = &resource_build_scaling_params;
+	dc->dml2_options.svp_pstate.callbacks.create_plane = &dc_create_plane_state;
+	dc->dml2_options.svp_pstate.callbacks.remove_plane_from_context = &dc_remove_plane_from_context;
+	dc->dml2_options.svp_pstate.callbacks.remove_stream_from_ctx = &dc_remove_stream_from_ctx;
+	dc->dml2_options.svp_pstate.callbacks.create_stream_for_sink = &dc_create_stream_for_sink;
+	dc->dml2_options.svp_pstate.callbacks.plane_state_release = &dc_plane_state_release;
+	dc->dml2_options.svp_pstate.callbacks.stream_release = &dc_stream_release;
+	dc->dml2_options.svp_pstate.callbacks.release_dsc = &dcn20_release_dsc;
+
+	dc->dml2_options.svp_pstate.subvp_fw_processing_delay_us = dc->caps.subvp_fw_processing_delay_us;
+	dc->dml2_options.svp_pstate.subvp_prefetch_end_to_mall_start_us = dc->caps.subvp_prefetch_end_to_mall_start_us;
+	dc->dml2_options.svp_pstate.subvp_pstate_allow_width_us = dc->caps.subvp_pstate_allow_width_us;
+	dc->dml2_options.svp_pstate.subvp_swath_height_margin_lines = dc->caps.subvp_swath_height_margin_lines;
+
+	dc->dml2_options.svp_pstate.force_disable_subvp = dc->debug.force_disable_subvp;
+	dc->dml2_options.svp_pstate.force_enable_subvp = dc->debug.force_subvp_mclk_switch;
+
+	dc->dml2_options.mall_cfg.cache_line_size_bytes = dc->caps.cache_line_size;
+	dc->dml2_options.mall_cfg.cache_num_ways = dc->caps.cache_num_ways;
+	dc->dml2_options.mall_cfg.max_cab_allocation_bytes = dc->caps.max_cab_allocation_bytes;
+	dc->dml2_options.mall_cfg.mblk_height_4bpe_pixels = DCN3_2_MBLK_HEIGHT_4BPE;
+	dc->dml2_options.mall_cfg.mblk_height_8bpe_pixels = DCN3_2_MBLK_HEIGHT_8BPE;
+	dc->dml2_options.mall_cfg.mblk_size_bytes = DCN3_2_MALL_MBLK_SIZE_BYTES;
+	dc->dml2_options.mall_cfg.mblk_width_pixels = DCN3_2_MBLK_WIDTH;
+
+	dc->dml2_options.max_segments_per_hubp = 18;
+	dc->dml2_options.det_segment_size = DCN3_2_DET_SEG_SIZE;
+
+	return true;
+
+create_fail:
+
+	dcn321_resource_destruct(pool);
+
+	return false;
+}
+
+struct resource_pool *dcn321_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc)
+{
+	struct dcn321_resource_pool *pool =
+		kzalloc(sizeof(struct dcn321_resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dcn321_resource_construct(init_data->num_virtual_links, dc, pool))
+		return &pool->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(pool);
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.h
new file mode 100644
index 000000000000..82cbf009f2d3
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2020 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef _DCN321_RESOURCE_H_
+#define _DCN321_RESOURCE_H_
+
+#include "core_types.h"
+
+#define TO_DCN321_RES_POOL(pool)\
+	container_of(pool, struct dcn321_resource_pool, base)
+
+extern struct _vcs_dpi_ip_params_st dcn3_21_ip;
+extern struct _vcs_dpi_soc_bounding_box_st dcn3_21_soc;
+
+struct dcn321_resource_pool {
+	struct resource_pool base;
+};
+
+struct resource_pool *dcn321_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc);
+
+#endif /* _DCN321_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c
new file mode 100644
index 000000000000..5c935d94a95c
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c
@@ -0,0 +1,2153 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2023 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dm_services.h"
+#include "dc.h"
+
+#include "dcn31/dcn31_init.h"
+#include "dcn35/dcn35_init.h"
+
+#include "resource.h"
+#include "include/irq_service_interface.h"
+#include "dcn35_resource.h"
+#include "dml2/dml2_wrapper.h"
+
+#include "dcn20/dcn20_resource.h"
+#include "dcn30/dcn30_resource.h"
+#include "dcn31/dcn31_resource.h"
+#include "dcn32/dcn32_resource.h"
+
+#include "dcn10/dcn10_ipp.h"
+#include "dcn30/dcn30_hubbub.h"
+#include "dcn31/dcn31_hubbub.h"
+#include "dcn35/dcn35_hubbub.h"
+#include "dcn32/dcn32_mpc.h"
+#include "dcn35/dcn35_hubp.h"
+#include "irq/dcn35/irq_service_dcn35.h"
+#include "dcn35/dcn35_dpp.h"
+#include "dcn35/dcn35_optc.h"
+#include "dcn20/dcn20_hwseq.h"
+#include "dcn30/dcn30_hwseq.h"
+#include "dce110/dce110_hwseq.h"
+#include "dcn35/dcn35_opp.h"
+#include "dcn35/dcn35_dsc.h"
+#include "dcn30/dcn30_vpg.h"
+#include "dcn30/dcn30_afmt.h"
+#include "dcn31/dcn31_dio_link_encoder.h"
+#include "dcn35/dcn35_dio_stream_encoder.h"
+#include "dcn31/dcn31_hpo_dp_stream_encoder.h"
+#include "dcn31/dcn31_hpo_dp_link_encoder.h"
+#include "dcn32/dcn32_hpo_dp_link_encoder.h"
+#include "link.h"
+#include "dcn31/dcn31_apg.h"
+#include "dcn32/dcn32_dio_link_encoder.h"
+#include "dcn31/dcn31_vpg.h"
+#include "dcn31/dcn31_afmt.h"
+#include "dce/dce_clock_source.h"
+#include "dce/dce_audio.h"
+#include "dce/dce_hwseq.h"
+#include "clk_mgr.h"
+#include "virtual/virtual_stream_encoder.h"
+#include "dce110/dce110_resource.h"
+#include "dml/display_mode_vba.h"
+#include "dcn35/dcn35_dccg.h"
+#include "dcn35/dcn35_pg_cntl.h"
+#include "dcn10/dcn10_resource.h"
+#include "dcn31/dcn31_panel_cntl.h"
+#include "dcn35/dcn35_hwseq.h"
+#include "dcn35/dcn35_dio_link_encoder.h"
+#include "dml/dcn31/dcn31_fpu.h" /*todo*/
+#include "dml/dcn35/dcn35_fpu.h"
+#include "dcn35/dcn35_dwb.h"
+#include "dcn35/dcn35_mmhubbub.h"
+
+#include "dcn/dcn_3_5_0_offset.h"
+#include "dcn/dcn_3_5_0_sh_mask.h"
+#include "nbio/nbio_7_11_0_offset.h"
+#include "mmhub/mmhub_3_3_0_offset.h"
+#include "mmhub/mmhub_3_3_0_sh_mask.h"
+
+#define DSCC0_DSCC_CONFIG0__ICH_RESET_AT_END_OF_LINE__SHIFT                   0x0
+#define DSCC0_DSCC_CONFIG0__ICH_RESET_AT_END_OF_LINE_MASK                     0x0000000FL
+
+#include "reg_helper.h"
+#include "dce/dmub_abm.h"
+#include "dce/dmub_psr.h"
+#include "dce/dce_aux.h"
+#include "dce/dce_i2c.h"
+#include "dml/dcn31/display_mode_vba_31.h" /*temp*/
+#include "vm_helper.h"
+#include "dcn20/dcn20_vmid.h"
+
+#include "link_enc_cfg.h"
+#define DC_LOGGER_INIT(logger)
+
+enum dcn35_clk_src_array_id {
+	DCN35_CLK_SRC_PLL0,
+	DCN35_CLK_SRC_PLL1,
+	DCN35_CLK_SRC_PLL2,
+	DCN35_CLK_SRC_PLL3,
+	DCN35_CLK_SRC_PLL4,
+	DCN35_CLK_SRC_TOTAL
+};
+
+/* begin *********************
+ * macros to expend register list macro defined in HW object header file
+ */
+
+/* DCN */
+/* TODO awful hack. fixup dcn20_dwb.h */
+#undef BASE_INNER
+#define BASE_INNER(seg) ctx->dcn_reg_offsets[seg]
+
+#define BASE(seg) BASE_INNER(seg)
+
+#define SR(reg_name)\
+		REG_STRUCT.reg_name = BASE(reg ## reg_name ## _BASE_IDX) +  \
+					reg ## reg_name
+
+#define SR_ARR(reg_name, id) \
+	REG_STRUCT[id].reg_name = BASE(reg##reg_name##_BASE_IDX) + reg##reg_name
+
+#define SR_ARR_INIT(reg_name, id, value) \
+	REG_STRUCT[id].reg_name = value
+
+#define SRI(reg_name, block, id)\
+	REG_STRUCT.reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define SRI_ARR(reg_name, block, id)\
+	REG_STRUCT[id].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SR_ARR_I2C(reg_name, id) \
+	REG_STRUCT[id-1].reg_name = BASE(reg##reg_name##_BASE_IDX) + reg##reg_name
+
+#define SRI_ARR_I2C(reg_name, block, id)\
+	REG_STRUCT[id-1].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SRI_ARR_ALPHABET(reg_name, block, index, id)\
+	REG_STRUCT[index].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SRI2(reg_name, block, id)\
+	.reg_name = BASE(reg ## reg_name ## _BASE_IDX) + \
+					reg ## reg_name
+
+#define SRI2_ARR(reg_name, block, id)\
+	REG_STRUCT[id].reg_name = BASE(reg ## reg_name ## _BASE_IDX) +	\
+		reg ## reg_name
+
+#define SRIR(var_name, reg_name, block, id)\
+	.var_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define SRII(reg_name, block, id)\
+	REG_STRUCT.reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define SRII_ARR_2(reg_name, block, id, inst)\
+	REG_STRUCT[inst].reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SRII_MPC_RMU(reg_name, block, id)\
+	.RMU##_##reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+					reg ## block ## id ## _ ## reg_name
+
+#define SRII_DWB(reg_name, temp_name, block, id)\
+	REG_STRUCT.reg_name[id] = BASE(reg ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## temp_name
+
+#define SF_DWB2(reg_name, block, id, field_name, post_fix) \
+	.field_name = reg_name ## __ ## field_name ## post_fix
+
+#define DCCG_SRII(reg_name, block, id)\
+	REG_STRUCT.block ## _ ## reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define VUPDATE_SRII(reg_name, block, id)\
+	REG_STRUCT.reg_name[id] = BASE(reg ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
+		reg ## reg_name ## _ ## block ## id
+
+/* NBIO */
+#define NBIO_BASE_INNER(seg) ctx->nbio_reg_offsets[seg]
+
+#define NBIO_BASE(seg) \
+	NBIO_BASE_INNER(seg)
+
+#define NBIO_SR(reg_name)\
+	REG_STRUCT.reg_name = NBIO_BASE(regBIF_BX2_ ## reg_name ## _BASE_IDX) + \
+				regBIF_BX2_ ## reg_name
+
+#define NBIO_SR_ARR(reg_name, id)\
+	REG_STRUCT[id].reg_name = NBIO_BASE(regBIF_BX2_ ## reg_name ## _BASE_IDX) + \
+		regBIF_BX2_ ## reg_name
+
+#define bios_regs_init() \
+		( \
+		NBIO_SR(BIOS_SCRATCH_3),\
+		NBIO_SR(BIOS_SCRATCH_6)\
+		)
+
+static struct bios_registers bios_regs;
+
+#define clk_src_regs_init(index, pllid)\
+	CS_COMMON_REG_LIST_DCN3_0_RI(index, pllid)
+
+static struct dce110_clk_src_regs clk_src_regs[5];
+
+static const struct dce110_clk_src_shift cs_shift = {
+		CS_COMMON_MASK_SH_LIST_DCN3_1_4(__SHIFT)
+};
+
+static const struct dce110_clk_src_mask cs_mask = {
+		CS_COMMON_MASK_SH_LIST_DCN3_1_4(_MASK)
+};
+
+#define abm_regs_init(id)\
+		ABM_DCN32_REG_LIST_RI(id)
+
+static struct dce_abm_registers abm_regs[4];
+
+static const struct dce_abm_shift abm_shift = {
+		ABM_MASK_SH_LIST_DCN35(__SHIFT)
+};
+
+static const struct dce_abm_mask abm_mask = {
+		ABM_MASK_SH_LIST_DCN35(_MASK)
+};
+
+#define audio_regs_init(id)\
+		AUD_COMMON_REG_LIST_RI(id)
+
+static struct dce_audio_registers audio_regs[7];
+
+
+#define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
+		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA, AZALIA_ENDPOINT_REG_DATA, mask_sh),\
+		AUD_COMMON_MASK_SH_LIST_BASE(mask_sh)
+
+static const struct dce_audio_shift audio_shift = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_audio_mask audio_mask = {
+		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
+};
+
+#define vpg_regs_init(id)\
+	VPG_DCN31_REG_LIST_RI(id)
+
+static struct dcn31_vpg_registers vpg_regs[10];
+
+static const struct dcn31_vpg_shift vpg_shift = {
+	DCN31_VPG_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_vpg_mask vpg_mask = {
+	DCN31_VPG_MASK_SH_LIST(_MASK)
+};
+
+#define afmt_regs_init(id)\
+	AFMT_DCN31_REG_LIST_RI(id)
+
+static struct dcn31_afmt_registers afmt_regs[6];
+
+static const struct dcn31_afmt_shift afmt_shift = {
+	DCN31_AFMT_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_afmt_mask afmt_mask = {
+	DCN31_AFMT_MASK_SH_LIST(_MASK)
+};
+
+#define apg_regs_init(id)\
+	APG_DCN31_REG_LIST_RI(id)
+
+static struct dcn31_apg_registers apg_regs[4];
+
+static const struct dcn31_apg_shift apg_shift = {
+	DCN31_APG_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_apg_mask apg_mask = {
+	DCN31_APG_MASK_SH_LIST(_MASK)
+};
+
+#define stream_enc_regs_init(id)\
+	SE_DCN35_REG_LIST_RI(id)
+
+static struct dcn10_stream_enc_registers stream_enc_regs[5];
+
+static const struct dcn10_stream_encoder_shift se_shift = {
+		SE_COMMON_MASK_SH_LIST_DCN35(__SHIFT)
+};
+
+static const struct dcn10_stream_encoder_mask se_mask = {
+		SE_COMMON_MASK_SH_LIST_DCN35(_MASK)
+};
+
+#define aux_regs_init(id)\
+	DCN2_AUX_REG_LIST_RI(id)
+
+static struct dcn10_link_enc_aux_registers link_enc_aux_regs[5];
+
+#define hpd_regs_init(id)\
+	HPD_REG_LIST_RI(id)
+
+static struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[5];
+
+
+static const struct dce110_aux_registers_shift aux_shift = {
+	DCN_AUX_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce110_aux_registers_mask aux_mask = {
+	DCN_AUX_MASK_SH_LIST(_MASK)
+};
+
+#define link_regs_init(id, phyid)\
+	( \
+	LE_DCN35_REG_LIST_RI(id), \
+	UNIPHY_DCN2_REG_LIST_RI(id, phyid)\
+	)
+
+static struct dcn10_link_enc_registers link_enc_regs[5];
+
+static const struct dcn10_link_enc_shift le_shift = {
+	LINK_ENCODER_MASK_SH_LIST_DCN35(__SHIFT), \
+	//DPCS_DCN31_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn10_link_enc_mask le_mask = {
+	LINK_ENCODER_MASK_SH_LIST_DCN35(_MASK), \
+	//DPCS_DCN31_MASK_SH_LIST(_MASK)
+};
+
+#define hpo_dp_stream_encoder_reg_init(id)\
+	DCN3_1_HPO_DP_STREAM_ENC_REG_LIST_RI(id)
+
+static struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[4];
+
+static const struct dcn31_hpo_dp_stream_encoder_shift hpo_dp_se_shift = {
+	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_hpo_dp_stream_encoder_mask hpo_dp_se_mask = {
+	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(_MASK)
+};
+
+#define hpo_dp_link_encoder_reg_init(id)\
+	DCN3_1_HPO_DP_LINK_ENC_REG_LIST_RI(id)
+	/*DCN3_1_RDPCSTX_REG_LIST(0),*/
+	/*DCN3_1_RDPCSTX_REG_LIST(1),*/
+	/*DCN3_1_RDPCSTX_REG_LIST(2),*/
+	/*DCN3_1_RDPCSTX_REG_LIST(3),*/
+
+static struct dcn31_hpo_dp_link_encoder_registers hpo_dp_link_enc_regs[2];
+
+static const struct dcn31_hpo_dp_link_encoder_shift hpo_dp_le_shift = {
+	DCN3_1_HPO_DP_LINK_ENC_COMMON_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn31_hpo_dp_link_encoder_mask hpo_dp_le_mask = {
+	DCN3_1_HPO_DP_LINK_ENC_COMMON_MASK_SH_LIST(_MASK)
+};
+
+#define dpp_regs_init(id)\
+	DPP_REG_LIST_DCN35_RI(id)
+
+static struct dcn3_dpp_registers dpp_regs[4];
+
+static const struct dcn35_dpp_shift tf_shift = {
+		DPP_REG_LIST_SH_MASK_DCN35(__SHIFT)
+};
+
+static const struct dcn35_dpp_mask tf_mask = {
+		DPP_REG_LIST_SH_MASK_DCN35(_MASK)
+};
+
+#define opp_regs_init(id)\
+	OPP_REG_LIST_DCN35_RI(id)
+
+static struct dcn35_opp_registers opp_regs[4];
+
+static const struct dcn35_opp_shift opp_shift = {
+	OPP_MASK_SH_LIST_DCN35(__SHIFT)
+};
+
+static const struct dcn35_opp_mask opp_mask = {
+	OPP_MASK_SH_LIST_DCN35(_MASK)
+};
+
+#define aux_engine_regs_init(id)\
+	( \
+	AUX_COMMON_REG_LIST0_RI(id), \
+	SR_ARR_INIT(AUXN_IMPCAL, id, 0), \
+	SR_ARR_INIT(AUXP_IMPCAL, id, 0), \
+	SR_ARR_INIT(AUX_RESET_MASK, id, DP_AUX0_AUX_CONTROL__AUX_RESET_MASK) \
+	)
+
+static struct dce110_aux_registers aux_engine_regs[5];
+
+#define dwbc_regs_dcn3_init(id)\
+	DWBC_COMMON_REG_LIST_DCN30_RI(id)
+
+static struct dcn30_dwbc_registers dwbc35_regs[1];
+
+static const struct dcn35_dwbc_shift dwbc35_shift = {
+	DWBC_COMMON_MASK_SH_LIST_DCN35(__SHIFT)
+};
+
+static const struct dcn35_dwbc_mask dwbc35_mask = {
+	DWBC_COMMON_MASK_SH_LIST_DCN35(_MASK)
+};
+
+#define mcif_wb_regs_dcn3_init(id)\
+	MCIF_WB_COMMON_REG_LIST_DCN3_5_RI(id)
+
+static struct dcn35_mmhubbub_registers mcif_wb35_regs[1];
+
+static const struct dcn35_mmhubbub_shift mcif_wb35_shift = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN3_5(__SHIFT)
+};
+
+static const struct dcn35_mmhubbub_mask mcif_wb35_mask = {
+	MCIF_WB_COMMON_MASK_SH_LIST_DCN3_5(_MASK)
+};
+
+#define dsc_regsDCN35_init(id)\
+	DSC_REG_LIST_DCN20_RI(id)
+
+static struct dcn20_dsc_registers dsc_regs[4];
+
+static const struct dcn35_dsc_shift dsc_shift = {
+	DSC_REG_LIST_SH_MASK_DCN35(__SHIFT)
+};
+
+static const struct dcn35_dsc_mask dsc_mask = {
+	DSC_REG_LIST_SH_MASK_DCN35(_MASK)
+};
+
+static struct dcn30_mpc_registers mpc_regs;
+
+#define dcn_mpc_regs_init() \
+	MPC_REG_LIST_DCN3_2_RI(0),\
+	MPC_REG_LIST_DCN3_2_RI(1),\
+	MPC_REG_LIST_DCN3_2_RI(2),\
+	MPC_REG_LIST_DCN3_2_RI(3),\
+	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(0),\
+	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(1),\
+	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(2),\
+	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(3),\
+	MPC_DWB_MUX_REG_LIST_DCN3_0_RI(0)
+
+static const struct dcn30_mpc_shift mpc_shift = {
+	MPC_COMMON_MASK_SH_LIST_DCN32(__SHIFT)
+};
+
+static const struct dcn30_mpc_mask mpc_mask = {
+	MPC_COMMON_MASK_SH_LIST_DCN32(_MASK)
+};
+
+#define optc_regs_init(id)\
+	OPTC_COMMON_REG_LIST_DCN3_5_RI(id)
+
+static struct dcn_optc_registers optc_regs[4];
+
+static const struct dcn_optc_shift optc_shift = {
+	OPTC_COMMON_MASK_SH_LIST_DCN3_5(__SHIFT)
+};
+
+static const struct dcn_optc_mask optc_mask = {
+	OPTC_COMMON_MASK_SH_LIST_DCN3_5(_MASK)
+};
+
+#define hubp_regs_init(id)\
+	HUBP_REG_LIST_DCN30_RI(id)
+
+static struct dcn_hubp2_registers hubp_regs[4];
+
+
+static const struct dcn35_hubp2_shift hubp_shift = {
+		HUBP_MASK_SH_LIST_DCN35(__SHIFT)
+};
+
+static const struct dcn35_hubp2_mask hubp_mask = {
+		HUBP_MASK_SH_LIST_DCN35(_MASK)
+};
+
+static struct dcn_hubbub_registers hubbub_reg;
+
+#define hubbub_reg_init()\
+		HUBBUB_REG_LIST_DCN35(0)
+
+static const struct dcn_hubbub_shift hubbub_shift = {
+		HUBBUB_MASK_SH_LIST_DCN35(__SHIFT)
+};
+
+static const struct dcn_hubbub_mask hubbub_mask = {
+		HUBBUB_MASK_SH_LIST_DCN35(_MASK)
+};
+
+static struct dccg_registers dccg_regs;
+
+#define dccg_regs_init()\
+	DCCG_REG_LIST_DCN35()
+
+static const struct dccg_shift dccg_shift = {
+		DCCG_MASK_SH_LIST_DCN35(__SHIFT)
+};
+
+static const struct dccg_mask dccg_mask = {
+		DCCG_MASK_SH_LIST_DCN35(_MASK)
+};
+
+static struct pg_cntl_registers pg_cntl_regs;
+
+#define pg_cntl_dcn35_regs_init() \
+	PG_CNTL_REG_LIST_DCN35()
+
+static const struct pg_cntl_shift pg_cntl_shift = {
+		PG_CNTL_MASK_SH_LIST_DCN35(__SHIFT)
+};
+
+static const struct pg_cntl_mask pg_cntl_mask = {
+		PG_CNTL_MASK_SH_LIST_DCN35(_MASK)
+};
+
+#define SRII2(reg_name_pre, reg_name_post, id)\
+	.reg_name_pre ## _ ##  reg_name_post[id] = BASE(reg ## reg_name_pre \
+			## id ## _ ## reg_name_post ## _BASE_IDX) + \
+			reg ## reg_name_pre ## id ## _ ## reg_name_post
+
+static struct dce_hwseq_registers hwseq_reg;
+
+#define hwseq_reg_init()\
+	HWSEQ_DCN35_REG_LIST()
+
+#define HWSEQ_DCN35_MASK_SH_LIST(mask_sh)\
+	HWSEQ_DCN_MASK_SH_LIST(mask_sh), \
+	HWS_SF(, DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_REFDIV, mask_sh), \
+	HWS_SF(, DCHUBBUB_ARB_HOSTVM_CNTL, DISABLE_HOSTVM_FORCE_ALLOW_PSTATE, mask_sh), \
+	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN0_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN1_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN2_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN3_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN16_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN17_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN18_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN19_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN22_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN22_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN23_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN23_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN24_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN24_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN25_PG_CONFIG, DOMAIN_POWER_FORCEON, mask_sh), \
+	HWS_SF(, DOMAIN25_PG_CONFIG, DOMAIN_POWER_GATE, mask_sh), \
+	HWS_SF(, DOMAIN0_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN1_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN2_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN3_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN16_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN17_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN18_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN19_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN22_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN23_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN24_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DOMAIN25_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, mask_sh), \
+	HWS_SF(, DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh), \
+	HWS_SF(, AZALIA_AUDIO_DTO, AZALIA_AUDIO_DTO_MODULE, mask_sh), \
+	HWS_SF(, HPO_TOP_CLOCK_CONTROL, HPO_HDMISTREAMCLK_G_GATE_DIS, mask_sh), \
+	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_UNASSIGNED_PWR_MODE, mask_sh), \
+	HWS_SF(, ODM_MEM_PWR_CTRL3, ODM_MEM_VBLANK_PWR_MODE, mask_sh), \
+	HWS_SF(, DIO_MEM_PWR_CTRL, I2C_LIGHT_SLEEP_FORCE, mask_sh), \
+	HWS_SF(, HPO_TOP_HW_CONTROL, HPO_IO_EN, mask_sh),\
+	HWS_SF(, DMU_CLK_CNTL, DISPCLK_R_DMU_GATE_DIS, mask_sh),\
+	HWS_SF(, DMU_CLK_CNTL, DISPCLK_G_RBBMIF_GATE_DIS, mask_sh),\
+	HWS_SF(, DMU_CLK_CNTL, RBBMIF_FGCG_REP_DIS, mask_sh),\
+	HWS_SF(, DMU_CLK_CNTL, DPREFCLK_ALLOW_DS_CLKSTOP, mask_sh),\
+	HWS_SF(, DMU_CLK_CNTL, DISPCLK_ALLOW_DS_CLKSTOP, mask_sh),\
+	HWS_SF(, DMU_CLK_CNTL, DPPCLK_ALLOW_DS_CLKSTOP, mask_sh),\
+	HWS_SF(, DMU_CLK_CNTL, DTBCLK_ALLOW_DS_CLKSTOP, mask_sh),\
+	HWS_SF(, DMU_CLK_CNTL, DCFCLK_ALLOW_DS_CLKSTOP, mask_sh),\
+	HWS_SF(, DMU_CLK_CNTL, DPIACLK_ALLOW_DS_CLKSTOP, mask_sh),\
+	HWS_SF(, DMU_CLK_CNTL, LONO_FGCG_REP_DIS, mask_sh),\
+	HWS_SF(, DMU_CLK_CNTL, LONO_DISPCLK_GATE_DISABLE, mask_sh),\
+	HWS_SF(, DMU_CLK_CNTL, LONO_SOCCLK_GATE_DISABLE, mask_sh),\
+	HWS_SF(, DMU_CLK_CNTL, LONO_DMCUBCLK_GATE_DISABLE, mask_sh),\
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKA_FE_GATE_DISABLE, mask_sh), \
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKB_FE_GATE_DISABLE, mask_sh), \
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKC_FE_GATE_DISABLE, mask_sh), \
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKD_FE_GATE_DISABLE, mask_sh), \
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKE_FE_GATE_DISABLE, mask_sh), \
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, HDMICHARCLK0_GATE_DISABLE, mask_sh), \
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKA_GATE_DISABLE, mask_sh), \
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKB_GATE_DISABLE, mask_sh), \
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKC_GATE_DISABLE, mask_sh), \
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKD_GATE_DISABLE, mask_sh), \
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, SYMCLKE_GATE_DISABLE, mask_sh), \
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, PHYASYMCLK_ROOT_GATE_DISABLE, mask_sh), \
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, PHYBSYMCLK_ROOT_GATE_DISABLE, mask_sh), \
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, PHYCSYMCLK_ROOT_GATE_DISABLE, mask_sh), \
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, PHYDSYMCLK_ROOT_GATE_DISABLE, mask_sh), \
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL2, PHYESYMCLK_ROOT_GATE_DISABLE, mask_sh),\
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DTBCLK_P0_GATE_DISABLE, mask_sh),\
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DTBCLK_P1_GATE_DISABLE, mask_sh),\
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DTBCLK_P2_GATE_DISABLE, mask_sh),\
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DTBCLK_P3_GATE_DISABLE, mask_sh),\
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK0_GATE_DISABLE, mask_sh),\
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK1_GATE_DISABLE, mask_sh),\
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK2_GATE_DISABLE, mask_sh),\
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL5, DPSTREAMCLK3_GATE_DISABLE, mask_sh),\
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL4, DPIASYMCLK0_GATE_DISABLE, mask_sh),\
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL4, DPIASYMCLK1_GATE_DISABLE, mask_sh),\
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL4, DPIASYMCLK2_GATE_DISABLE, mask_sh),\
+	HWS_SF(, DCCG_GATE_DISABLE_CNTL4, DPIASYMCLK3_GATE_DISABLE, mask_sh)
+
+static const struct dce_hwseq_shift hwseq_shift = {
+		HWSEQ_DCN35_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dce_hwseq_mask hwseq_mask = {
+		HWSEQ_DCN35_MASK_SH_LIST(_MASK)
+};
+
+#define vmid_regs_init(id)\
+		DCN20_VMID_REG_LIST_RI(id)
+
+static struct dcn_vmid_registers vmid_regs[16];
+
+static const struct dcn20_vmid_shift vmid_shifts = {
+		DCN20_VMID_MASK_SH_LIST(__SHIFT)
+};
+
+static const struct dcn20_vmid_mask vmid_masks = {
+		DCN20_VMID_MASK_SH_LIST(_MASK)
+};
+
+static const struct resource_caps res_cap_dcn35 = {
+	.num_timing_generator = 4,
+	.num_opp = 4,
+	.num_video_plane = 4,
+	.num_audio = 5,
+	.num_stream_encoder = 5,
+	.num_dig_link_enc = 5,
+	.num_hpo_dp_stream_encoder = 4,
+	.num_hpo_dp_link_encoder = 2,
+	.num_pll = 4,/*1 c10 edp, 3xc20 combo PHY*/
+	.num_dwb = 1,
+	.num_ddc = 5,
+	.num_vmid = 16,
+	.num_mpc_3dlut = 2,
+	.num_dsc = 4,
+};
+
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
+	.per_pixel_alpha = true,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = true,
+			.fp16 = true,
+			.p010 = true,
+			.ayuv = false,
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 16000,
+			.fp16 = 16000
+	},
+
+	// 6:1 downscaling ratio: 1000/6 = 166.666
+	.max_downscale_factor = {
+			.argb8888 = 167,
+			.nv12 = 167,
+			.fp16 = 167
+	},
+	64,
+	64
+};
+
+static const struct dc_debug_options debug_defaults_drv = {
+	.disable_dmcu = true,
+	.force_abm_enable = false,
+	.timing_trace = false,
+	.clock_trace = true,
+	.disable_pplib_clock_request = false,
+	.pipe_split_policy = MPC_SPLIT_AVOID,
+	.force_single_disp_pipe_split = false,
+	.disable_dcc = DCC_ENABLE,
+	.disable_dpp_power_gate = true,
+	.disable_hubp_power_gate = true,
+	.disable_clock_gate = false,
+	.disable_dsc_power_gate = true,
+	.vsr_support = true,
+	.performance_trace = false,
+	.max_downscale_src_width = 4096,/*upto true 4k*/
+	.disable_pplib_wm_range = false,
+	.scl_reset_length10 = true,
+	.sanity_checks = false,
+	.underflow_assert_delay_us = 0xFFFFFFFF,
+	.dwb_fi_phase = -1, // -1 = disable,
+	.dmub_command_table = true,
+	.pstate_enabled = true,
+	.use_max_lb = true,
+	.enable_mem_low_power = {
+		.bits = {
+			.vga = false,
+			.i2c = true,
+			.dmcu = false, // This is previously known to cause hang on S3 cycles if enabled
+			.dscl = true,
+			.cm = true,
+			.mpc = true,
+			.optc = true,
+			.vpg = true,
+			.afmt = true,
+		}
+	},
+	.root_clock_optimization = {
+		.bits = {
+			.dpp = true,
+			.dsc = true,/*dscclk and dsc pg*/
+			.hdmistream = true,
+			.hdmichar = true,
+			.dpstream = true,
+			.symclk32_se = true,
+			.symclk32_le = true,
+			.symclk_fe = true,
+			.physymclk = true,
+			.dpiasymclk = true,
+		}
+	},
+	.seamless_boot_odm_combine = DML_FAIL_SOURCE_PIXEL_FORMAT,
+	.enable_z9_disable_interface = true, /* Allow support for the PMFW interface for disable Z9*/
+	.using_dml2 = true,
+	.support_eDP1_5 = true,
+	.enable_hpo_pg_support = false,
+	.enable_legacy_fast_update = true,
+	.enable_single_display_2to1_odm_policy = false,
+	.disable_idle_power_optimizations = true,
+	.dmcub_emulation = false,
+	.disable_boot_optimizations = false,
+	.disable_unbounded_requesting = false,
+	.disable_mem_low_power = false,
+	//must match enable_single_display_2to1_odm_policy to support dynamic ODM transitions
+	.enable_double_buffered_dsc_pg_support = true,
+	.enable_dp_dig_pixel_rate_div_policy = 1,
+	.disable_z10 = false,
+	.ignore_pg = true,
+	.psp_disabled_wa = true,
+	.ips2_eval_delay_us = 200,
+	.ips2_entry_delay_us = 400
+};
+
+static const struct dc_panel_config panel_config_defaults = {
+	.psr = {
+		.disable_psr = false,
+		.disallow_psrsu = false,
+	},
+	.ilr = {
+		.optimize_edp_link_rate = true,
+	},
+};
+
+static void dcn35_dpp_destroy(struct dpp **dpp)
+{
+	kfree(TO_DCN20_DPP(*dpp));
+	*dpp = NULL;
+}
+
+static struct dpp *dcn35_dpp_create(struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn3_dpp *dpp = kzalloc(sizeof(struct dcn3_dpp), GFP_KERNEL);
+	bool success = (dpp != NULL);
+
+	if (!success)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT dpp_regs
+	dpp_regs_init(0),
+	dpp_regs_init(1),
+	dpp_regs_init(2),
+	dpp_regs_init(3);
+
+	success = dpp35_construct(dpp, ctx, inst, &dpp_regs[inst], &tf_shift,
+				  &tf_mask);
+	if (success) {
+		dpp35_set_fgcg(
+			dpp,
+			ctx->dc->debug.enable_fine_grain_clock_gating.bits.dpp);
+		return &dpp->base;
+	}
+
+	BREAK_TO_DEBUGGER();
+	kfree(dpp);
+	return NULL;
+}
+
+static struct output_pixel_processor *dcn35_opp_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_opp *opp =
+		kzalloc(sizeof(struct dcn20_opp), GFP_KERNEL);
+
+	if (!opp) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+#undef REG_STRUCT
+#define REG_STRUCT opp_regs
+	opp_regs_init(0),
+	opp_regs_init(1),
+	opp_regs_init(2),
+	opp_regs_init(3);
+
+	dcn35_opp_construct(opp, ctx, inst,
+			&opp_regs[inst], &opp_shift, &opp_mask);
+
+	dcn35_opp_set_fgcg(opp, ctx->dc->debug.enable_fine_grain_clock_gating.bits.opp);
+
+	return &opp->base;
+}
+
+static struct dce_aux *dcn31_aux_engine_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct aux_engine_dce110 *aux_engine =
+		kzalloc(sizeof(struct aux_engine_dce110), GFP_KERNEL);
+
+	if (!aux_engine)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT aux_engine_regs
+	aux_engine_regs_init(0),
+	aux_engine_regs_init(1),
+	aux_engine_regs_init(2),
+	aux_engine_regs_init(3),
+	aux_engine_regs_init(4);
+
+	dce110_aux_engine_construct(aux_engine, ctx, inst,
+				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
+				    &aux_engine_regs[inst],
+					&aux_mask,
+					&aux_shift,
+					ctx->dc->caps.extended_aux_timeout_support);
+
+	return &aux_engine->base;
+}
+
+#define i2c_inst_regs_init(id)\
+	I2C_HW_ENGINE_COMMON_REG_LIST_DCN30_RI(id)
+
+static struct dce_i2c_registers i2c_hw_regs[5];
+
+static const struct dce_i2c_shift i2c_shifts = {
+		I2C_COMMON_MASK_SH_LIST_DCN35(__SHIFT)
+};
+
+static const struct dce_i2c_mask i2c_masks = {
+		I2C_COMMON_MASK_SH_LIST_DCN35(_MASK)
+};
+
+/* ========================================================== */
+
+/*
+ * DPIA index | Preferred Encoder     |    Host Router
+ *   0        |      C                |       0
+ *   1        |      First Available  |       0
+ *   2        |      D                |       1
+ *   3        |      First Available  |       1
+ */
+/* ========================================================== */
+static const enum engine_id dpia_to_preferred_enc_id_table[] = {
+		ENGINE_ID_DIGC,
+		ENGINE_ID_DIGC,
+		ENGINE_ID_DIGD,
+		ENGINE_ID_DIGD
+};
+
+static enum engine_id dcn35_get_preferred_eng_id_dpia(unsigned int dpia_index)
+{
+	return dpia_to_preferred_enc_id_table[dpia_index];
+}
+
+static struct dce_i2c_hw *dcn31_i2c_hw_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dce_i2c_hw *dce_i2c_hw =
+		kzalloc(sizeof(struct dce_i2c_hw), GFP_KERNEL);
+
+	if (!dce_i2c_hw)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT i2c_hw_regs
+	i2c_inst_regs_init(1),
+	i2c_inst_regs_init(2),
+	i2c_inst_regs_init(3),
+	i2c_inst_regs_init(4),
+	i2c_inst_regs_init(5);
+
+	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
+				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
+
+	return dce_i2c_hw;
+}
+static struct mpc *dcn35_mpc_create(
+		struct dc_context *ctx,
+		int num_mpcc,
+		int num_rmu)
+{
+	struct dcn30_mpc *mpc30 = kzalloc(sizeof(struct dcn30_mpc), GFP_KERNEL);
+
+	if (!mpc30)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT mpc_regs
+	dcn_mpc_regs_init();
+
+	dcn32_mpc_construct(mpc30, ctx,
+			&mpc_regs,
+			&mpc_shift,
+			&mpc_mask,
+			num_mpcc,
+			num_rmu);
+
+	return &mpc30->base;
+}
+
+static struct hubbub *dcn35_hubbub_create(struct dc_context *ctx)
+{
+	int i;
+
+	struct dcn20_hubbub *hubbub3 = kzalloc(sizeof(struct dcn20_hubbub),
+					  GFP_KERNEL);
+
+	if (!hubbub3)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT hubbub_reg
+	hubbub_reg_init();
+
+#undef REG_STRUCT
+#define REG_STRUCT vmid_regs
+	vmid_regs_init(0),
+	vmid_regs_init(1),
+	vmid_regs_init(2),
+	vmid_regs_init(3),
+	vmid_regs_init(4),
+	vmid_regs_init(5),
+	vmid_regs_init(6),
+	vmid_regs_init(7),
+	vmid_regs_init(8),
+	vmid_regs_init(9),
+	vmid_regs_init(10),
+	vmid_regs_init(11),
+	vmid_regs_init(12),
+	vmid_regs_init(13),
+	vmid_regs_init(14),
+	vmid_regs_init(15);
+
+	hubbub35_construct(hubbub3, ctx,
+			&hubbub_reg,
+			&hubbub_shift,
+			&hubbub_mask,
+			384,/*ctx->dc->dml.ip.det_buffer_size_kbytes,*/
+			8, /*ctx->dc->dml.ip.pixel_chunk_size_kbytes,*/
+			1792 /*ctx->dc->dml.ip.config_return_buffer_size_in_kbytes*/);
+
+
+	for (i = 0; i < res_cap_dcn35.num_vmid; i++) {
+		struct dcn20_vmid *vmid = &hubbub3->vmid[i];
+
+		vmid->ctx = ctx;
+
+		vmid->regs = &vmid_regs[i];
+		vmid->shifts = &vmid_shifts;
+		vmid->masks = &vmid_masks;
+	}
+
+	return &hubbub3->base;
+}
+
+static struct timing_generator *dcn35_timing_generator_create(
+		struct dc_context *ctx,
+		uint32_t instance)
+{
+	struct optc *tgn10 =
+		kzalloc(sizeof(struct optc), GFP_KERNEL);
+
+	if (!tgn10)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT optc_regs
+	optc_regs_init(0),
+	optc_regs_init(1),
+	optc_regs_init(2),
+	optc_regs_init(3);
+
+	tgn10->base.inst = instance;
+	tgn10->base.ctx = ctx;
+
+	tgn10->tg_regs = &optc_regs[instance];
+	tgn10->tg_shift = &optc_shift;
+	tgn10->tg_mask = &optc_mask;
+
+	dcn35_timing_generator_init(tgn10);
+
+	return &tgn10->base;
+}
+
+static const struct encoder_feature_support link_enc_feature = {
+		.max_hdmi_deep_color = COLOR_DEPTH_121212,
+		.max_hdmi_pixel_clock = 600000,
+		.hdmi_ycbcr420_supported = true,
+		.dp_ycbcr420_supported = true,
+		.fec_supported = true,
+		.flags.bits.IS_HBR2_CAPABLE = true,
+		.flags.bits.IS_HBR3_CAPABLE = true,
+		.flags.bits.IS_TPS3_CAPABLE = true,
+		.flags.bits.IS_TPS4_CAPABLE = true
+};
+
+static struct link_encoder *dcn35_link_encoder_create(
+	struct dc_context *ctx,
+	const struct encoder_init_data *enc_init_data)
+{
+	struct dcn20_link_encoder *enc20 =
+		kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+
+	if (!enc20)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT link_enc_aux_regs
+	aux_regs_init(0),
+	aux_regs_init(1),
+	aux_regs_init(2),
+	aux_regs_init(3),
+	aux_regs_init(4);
+
+#undef REG_STRUCT
+#define REG_STRUCT link_enc_hpd_regs
+	hpd_regs_init(0),
+	hpd_regs_init(1),
+	hpd_regs_init(2),
+	hpd_regs_init(3),
+	hpd_regs_init(4);
+
+#undef REG_STRUCT
+#define REG_STRUCT link_enc_regs
+	link_regs_init(0, A),
+	link_regs_init(1, B),
+	link_regs_init(2, C),
+	link_regs_init(3, D),
+	link_regs_init(4, E);
+
+	dcn35_link_encoder_construct(enc20,
+			enc_init_data,
+			&link_enc_feature,
+			&link_enc_regs[enc_init_data->transmitter],
+			&link_enc_aux_regs[enc_init_data->channel - 1],
+			&link_enc_hpd_regs[enc_init_data->hpd_source],
+			&le_shift,
+			&le_mask);
+
+	return &enc20->enc10.base;
+}
+
+/* Create a minimal link encoder object not associated with a particular
+ * physical connector.
+ * resource_funcs.link_enc_create_minimal
+ */
+static struct link_encoder *dcn31_link_enc_create_minimal(
+		struct dc_context *ctx, enum engine_id eng_id)
+{
+	struct dcn20_link_encoder *enc20;
+
+	if ((eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+		return NULL;
+
+	enc20 = kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
+	if (!enc20)
+		return NULL;
+
+	dcn31_link_encoder_construct_minimal(
+			enc20,
+			ctx,
+			&link_enc_feature,
+			&link_enc_regs[eng_id - ENGINE_ID_DIGA],
+			eng_id);
+
+	return &enc20->enc10.base;
+}
+
+static struct panel_cntl *dcn31_panel_cntl_create(const struct panel_cntl_init_data *init_data)
+{
+	struct dcn31_panel_cntl *panel_cntl =
+		kzalloc(sizeof(struct dcn31_panel_cntl), GFP_KERNEL);
+
+	if (!panel_cntl)
+		return NULL;
+
+	dcn31_panel_cntl_construct(panel_cntl, init_data);
+
+	return &panel_cntl->base;
+}
+
+static void read_dce_straps(
+	struct dc_context *ctx,
+	struct resource_straps *straps)
+{
+	generic_reg_get(ctx, regDC_PINSTRAPS + BASE(regDC_PINSTRAPS_BASE_IDX),
+		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
+
+}
+
+static struct audio *dcn31_create_audio(
+		struct dc_context *ctx, unsigned int inst)
+{
+
+#undef REG_STRUCT
+#define REG_STRUCT audio_regs
+	audio_regs_init(0),
+	audio_regs_init(1),
+	audio_regs_init(2),
+	audio_regs_init(3),
+	audio_regs_init(4);
+	audio_regs_init(5);
+	audio_regs_init(6);
+
+	return dce_audio_create(ctx, inst,
+			&audio_regs[inst], &audio_shift, &audio_mask);
+}
+
+static struct vpg *dcn31_vpg_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn31_vpg *vpg31 = kzalloc(sizeof(struct dcn31_vpg), GFP_KERNEL);
+
+	if (!vpg31)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT vpg_regs
+	vpg_regs_init(0),
+	vpg_regs_init(1),
+	vpg_regs_init(2),
+	vpg_regs_init(3),
+	vpg_regs_init(4),
+	vpg_regs_init(5),
+	vpg_regs_init(6),
+	vpg_regs_init(7),
+	vpg_regs_init(8),
+	vpg_regs_init(9);
+
+	vpg31_construct(vpg31, ctx, inst,
+			&vpg_regs[inst],
+			&vpg_shift,
+			&vpg_mask);
+
+	return &vpg31->base;
+}
+
+static struct afmt *dcn31_afmt_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn31_afmt *afmt31 = kzalloc(sizeof(struct dcn31_afmt), GFP_KERNEL);
+
+	if (!afmt31)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT afmt_regs
+	afmt_regs_init(0),
+	afmt_regs_init(1),
+	afmt_regs_init(2),
+	afmt_regs_init(3),
+	afmt_regs_init(4),
+	afmt_regs_init(5);
+
+	afmt31_construct(afmt31, ctx, inst,
+			&afmt_regs[inst],
+			&afmt_shift,
+			&afmt_mask);
+
+	// Light sleep by default, no need to power down here
+
+	return &afmt31->base;
+}
+
+static struct apg *dcn31_apg_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn31_apg *apg31 = kzalloc(sizeof(struct dcn31_apg), GFP_KERNEL);
+
+	if (!apg31)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT apg_regs
+	apg_regs_init(0),
+	apg_regs_init(1),
+	apg_regs_init(2),
+	apg_regs_init(3);
+
+	apg31_construct(apg31, ctx, inst,
+			&apg_regs[inst],
+			&apg_shift,
+			&apg_mask);
+
+	return &apg31->base;
+}
+
+static struct stream_encoder *dcn35_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dcn10_stream_encoder *enc1;
+	struct vpg *vpg;
+	struct afmt *afmt;
+	int vpg_inst;
+	int afmt_inst;
+
+	/* Mapping of VPG, AFMT, DME register blocks to DIO block instance */
+	if (eng_id <= ENGINE_ID_DIGF) {
+		vpg_inst = eng_id;
+		afmt_inst = eng_id;
+	} else
+		return NULL;
+
+	enc1 = kzalloc(sizeof(struct dcn10_stream_encoder), GFP_KERNEL);
+	vpg = dcn31_vpg_create(ctx, vpg_inst);
+	afmt = dcn31_afmt_create(ctx, afmt_inst);
+
+	if (!enc1 || !vpg || !afmt) {
+		kfree(enc1);
+		kfree(vpg);
+		kfree(afmt);
+		return NULL;
+	}
+
+#undef REG_STRUCT
+#define REG_STRUCT stream_enc_regs
+	stream_enc_regs_init(0),
+	stream_enc_regs_init(1),
+	stream_enc_regs_init(2),
+	stream_enc_regs_init(3),
+	stream_enc_regs_init(4);
+
+	dcn35_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
+					eng_id, vpg, afmt,
+					&stream_enc_regs[eng_id],
+					&se_shift, &se_mask);
+
+	return &enc1->base;
+}
+
+static struct hpo_dp_stream_encoder *dcn31_hpo_dp_stream_encoder_create(
+	enum engine_id eng_id,
+	struct dc_context *ctx)
+{
+	struct dcn31_hpo_dp_stream_encoder *hpo_dp_enc31;
+	struct vpg *vpg;
+	struct apg *apg;
+	uint32_t hpo_dp_inst;
+	uint32_t vpg_inst;
+	uint32_t apg_inst;
+
+	ASSERT((eng_id >= ENGINE_ID_HPO_DP_0) && (eng_id <= ENGINE_ID_HPO_DP_3));
+	hpo_dp_inst = eng_id - ENGINE_ID_HPO_DP_0;
+
+	/* Mapping of VPG register blocks to HPO DP block instance:
+	 * VPG[6] -> HPO_DP[0]
+	 * VPG[7] -> HPO_DP[1]
+	 * VPG[8] -> HPO_DP[2]
+	 * VPG[9] -> HPO_DP[3]
+	 */
+	vpg_inst = hpo_dp_inst + 6;
+
+	/* Mapping of APG register blocks to HPO DP block instance:
+	 * APG[0] -> HPO_DP[0]
+	 * APG[1] -> HPO_DP[1]
+	 * APG[2] -> HPO_DP[2]
+	 * APG[3] -> HPO_DP[3]
+	 */
+	apg_inst = hpo_dp_inst;
+
+	/* allocate HPO stream encoder and create VPG sub-block */
+	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_stream_encoder), GFP_KERNEL);
+	vpg = dcn31_vpg_create(ctx, vpg_inst);
+	apg = dcn31_apg_create(ctx, apg_inst);
+
+	if (!hpo_dp_enc31 || !vpg || !apg) {
+		kfree(hpo_dp_enc31);
+		kfree(vpg);
+		kfree(apg);
+		return NULL;
+	}
+
+#undef REG_STRUCT
+#define REG_STRUCT hpo_dp_stream_enc_regs
+	hpo_dp_stream_encoder_reg_init(0),
+	hpo_dp_stream_encoder_reg_init(1),
+	hpo_dp_stream_encoder_reg_init(2),
+	hpo_dp_stream_encoder_reg_init(3);
+
+	dcn31_hpo_dp_stream_encoder_construct(hpo_dp_enc31, ctx, ctx->dc_bios,
+					hpo_dp_inst, eng_id, vpg, apg,
+					&hpo_dp_stream_enc_regs[hpo_dp_inst],
+					&hpo_dp_se_shift, &hpo_dp_se_mask);
+
+	return &hpo_dp_enc31->base;
+}
+
+static struct hpo_dp_link_encoder *dcn31_hpo_dp_link_encoder_create(
+	uint8_t inst,
+	struct dc_context *ctx)
+{
+	struct dcn31_hpo_dp_link_encoder *hpo_dp_enc31;
+
+	/* allocate HPO link encoder */
+	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
+
+#undef REG_STRUCT
+#define REG_STRUCT hpo_dp_link_enc_regs
+	hpo_dp_link_encoder_reg_init(0),
+	hpo_dp_link_encoder_reg_init(1);
+
+	hpo_dp_link_encoder31_construct(hpo_dp_enc31, ctx, inst,
+					&hpo_dp_link_enc_regs[inst],
+					&hpo_dp_le_shift, &hpo_dp_le_mask);
+
+	return &hpo_dp_enc31->base;
+}
+
+static struct dce_hwseq *dcn35_hwseq_create(
+	struct dc_context *ctx)
+{
+	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
+
+#undef REG_STRUCT
+#define REG_STRUCT hwseq_reg
+	hwseq_reg_init();
+
+	if (hws) {
+		hws->ctx = ctx;
+		hws->regs = &hwseq_reg;
+		hws->shifts = &hwseq_shift;
+		hws->masks = &hwseq_mask;
+	}
+	return hws;
+}
+static const struct resource_create_funcs res_create_funcs = {
+	.read_dce_straps = read_dce_straps,
+	.create_audio = dcn31_create_audio,
+	.create_stream_encoder = dcn35_stream_encoder_create,
+	.create_hpo_dp_stream_encoder = dcn31_hpo_dp_stream_encoder_create,
+	.create_hpo_dp_link_encoder = dcn31_hpo_dp_link_encoder_create,
+	.create_hwseq = dcn35_hwseq_create,
+};
+
+static void dcn35_resource_destruct(struct dcn35_resource_pool *pool)
+{
+	unsigned int i;
+
+	for (i = 0; i < pool->base.stream_enc_count; i++) {
+		if (pool->base.stream_enc[i] != NULL) {
+			if (pool->base.stream_enc[i]->vpg != NULL) {
+				kfree(DCN30_VPG_FROM_VPG(pool->base.stream_enc[i]->vpg));
+				pool->base.stream_enc[i]->vpg = NULL;
+			}
+			if (pool->base.stream_enc[i]->afmt != NULL) {
+				kfree(DCN30_AFMT_FROM_AFMT(pool->base.stream_enc[i]->afmt));
+				pool->base.stream_enc[i]->afmt = NULL;
+			}
+			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
+			pool->base.stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.hpo_dp_stream_enc_count; i++) {
+		if (pool->base.hpo_dp_stream_enc[i] != NULL) {
+			if (pool->base.hpo_dp_stream_enc[i]->vpg != NULL) {
+				kfree(DCN30_VPG_FROM_VPG(pool->base.hpo_dp_stream_enc[i]->vpg));
+				pool->base.hpo_dp_stream_enc[i]->vpg = NULL;
+			}
+			if (pool->base.hpo_dp_stream_enc[i]->apg != NULL) {
+				kfree(DCN31_APG_FROM_APG(pool->base.hpo_dp_stream_enc[i]->apg));
+				pool->base.hpo_dp_stream_enc[i]->apg = NULL;
+			}
+			kfree(DCN3_1_HPO_DP_STREAM_ENC_FROM_HPO_STREAM_ENC(pool->base.hpo_dp_stream_enc[i]));
+			pool->base.hpo_dp_stream_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.hpo_dp_link_enc_count; i++) {
+		if (pool->base.hpo_dp_link_enc[i] != NULL) {
+			kfree(DCN3_1_HPO_DP_LINK_ENC_FROM_HPO_LINK_ENC(pool->base.hpo_dp_link_enc[i]));
+			pool->base.hpo_dp_link_enc[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		if (pool->base.dscs[i] != NULL)
+			dcn20_dsc_destroy(&pool->base.dscs[i]);
+	}
+
+	if (pool->base.mpc != NULL) {
+		kfree(TO_DCN20_MPC(pool->base.mpc));
+		pool->base.mpc = NULL;
+	}
+	if (pool->base.hubbub != NULL) {
+		kfree(pool->base.hubbub);
+		pool->base.hubbub = NULL;
+	}
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		if (pool->base.dpps[i] != NULL)
+			dcn35_dpp_destroy(&pool->base.dpps[i]);
+
+		if (pool->base.ipps[i] != NULL)
+			pool->base.ipps[i]->funcs->ipp_destroy(&pool->base.ipps[i]);
+
+		if (pool->base.hubps[i] != NULL) {
+			kfree(TO_DCN20_HUBP(pool->base.hubps[i]));
+			pool->base.hubps[i] = NULL;
+		}
+
+		if (pool->base.irqs != NULL) {
+			dal_irq_service_destroy(&pool->base.irqs);
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		if (pool->base.engines[i] != NULL)
+			dce110_engine_destroy(&pool->base.engines[i]);
+		if (pool->base.hw_i2cs[i] != NULL) {
+			kfree(pool->base.hw_i2cs[i]);
+			pool->base.hw_i2cs[i] = NULL;
+		}
+		if (pool->base.sw_i2cs[i] != NULL) {
+			kfree(pool->base.sw_i2cs[i]);
+			pool->base.sw_i2cs[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		if (pool->base.opps[i] != NULL)
+			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.timing_generators[i] != NULL)	{
+			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
+			pool->base.timing_generators[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+		if (pool->base.dwbc[i] != NULL) {
+			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
+			pool->base.dwbc[i] = NULL;
+		}
+		if (pool->base.mcif_wb[i] != NULL) {
+			kfree(TO_DCN30_MMHUBBUB(pool->base.mcif_wb[i]));
+			pool->base.mcif_wb[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.audio_count; i++) {
+		if (pool->base.audios[i])
+			dce_aud_destroy(&pool->base.audios[i]);
+	}
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] != NULL) {
+			dcn20_clock_source_destroy(&pool->base.clock_sources[i]);
+			pool->base.clock_sources[i] = NULL;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+		if (pool->base.mpc_lut[i] != NULL) {
+			dc_3dlut_func_release(pool->base.mpc_lut[i]);
+			pool->base.mpc_lut[i] = NULL;
+		}
+		if (pool->base.mpc_shaper[i] != NULL) {
+			dc_transfer_func_release(pool->base.mpc_shaper[i]);
+			pool->base.mpc_shaper[i] = NULL;
+		}
+	}
+
+	if (pool->base.dp_clock_source != NULL) {
+		dcn20_clock_source_destroy(&pool->base.dp_clock_source);
+		pool->base.dp_clock_source = NULL;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		if (pool->base.multiple_abms[i] != NULL)
+			dce_abm_destroy(&pool->base.multiple_abms[i]);
+	}
+
+	if (pool->base.psr != NULL)
+		dmub_psr_destroy(&pool->base.psr);
+
+	if (pool->base.pg_cntl != NULL)
+		dcn_pg_cntl_destroy(&pool->base.pg_cntl);
+
+	if (pool->base.dccg != NULL)
+		dcn_dccg_destroy(&pool->base.dccg);
+}
+
+static struct hubp *dcn35_hubp_create(
+	struct dc_context *ctx,
+	uint32_t inst)
+{
+	struct dcn20_hubp *hubp2 =
+		kzalloc(sizeof(struct dcn20_hubp), GFP_KERNEL);
+
+	if (!hubp2)
+		return NULL;
+
+#undef REG_STRUCT
+#define REG_STRUCT hubp_regs
+	hubp_regs_init(0),
+	hubp_regs_init(1),
+	hubp_regs_init(2),
+	hubp_regs_init(3);
+
+	if (hubp35_construct(hubp2, ctx, inst,
+			&hubp_regs[inst], &hubp_shift, &hubp_mask))
+		return &hubp2->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(hubp2);
+	return NULL;
+}
+
+static void dcn35_dwbc_init(struct dcn30_dwbc *dwbc30, struct dc_context *ctx)
+{
+	dcn35_dwbc_set_fgcg(
+		dwbc30, ctx->dc->debug.enable_fine_grain_clock_gating.bits.dwb);
+}
+
+static bool dcn35_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn30_dwbc *dwbc30 = kzalloc(sizeof(struct dcn30_dwbc),
+						    GFP_KERNEL);
+
+		if (!dwbc30) {
+			dm_error("DC: failed to create dwbc30!\n");
+			return false;
+		}
+
+#undef REG_STRUCT
+#define REG_STRUCT dwbc35_regs
+		dwbc_regs_dcn3_init(0);
+
+		dcn35_dwbc_construct(dwbc30, ctx,
+				&dwbc35_regs[i],
+				&dwbc35_shift,
+				&dwbc35_mask,
+				i);
+
+		pool->dwbc[i] = &dwbc30->base;
+
+		dcn35_dwbc_init(dwbc30, ctx);
+	}
+	return true;
+}
+
+static void dcn35_mmhubbub_init(struct dcn30_mmhubbub *mcif_wb30,
+				struct dc_context *ctx)
+{
+	dcn35_mmhubbub_set_fgcg(
+		mcif_wb30,
+		ctx->dc->debug.enable_fine_grain_clock_gating.bits.mmhubbub);
+}
+
+static bool dcn35_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
+{
+	int i;
+	uint32_t pipe_count = pool->res_cap->num_dwb;
+
+	for (i = 0; i < pipe_count; i++) {
+		struct dcn30_mmhubbub *mcif_wb30 = kzalloc(sizeof(struct dcn30_mmhubbub),
+						    GFP_KERNEL);
+
+		if (!mcif_wb30) {
+			dm_error("DC: failed to create mcif_wb30!\n");
+			return false;
+		}
+
+#undef REG_STRUCT
+#define REG_STRUCT mcif_wb35_regs
+		mcif_wb_regs_dcn3_init(0);
+
+		dcn35_mmhubbub_construct(mcif_wb30, ctx,
+				&mcif_wb35_regs[i],
+				&mcif_wb35_shift,
+				&mcif_wb35_mask,
+				i);
+
+		dcn35_mmhubbub_init(mcif_wb30, ctx);
+
+		pool->mcif_wb[i] = &mcif_wb30->base;
+	}
+	return true;
+}
+
+static struct display_stream_compressor *dcn35_dsc_create(
+	struct dc_context *ctx, uint32_t inst)
+{
+	struct dcn20_dsc *dsc =
+		kzalloc(sizeof(struct dcn20_dsc), GFP_KERNEL);
+
+	if (!dsc) {
+		BREAK_TO_DEBUGGER();
+		return NULL;
+	}
+
+#undef REG_STRUCT
+#define REG_STRUCT dsc_regs
+	dsc_regsDCN35_init(0),
+	dsc_regsDCN35_init(1),
+	dsc_regsDCN35_init(2),
+	dsc_regsDCN35_init(3);
+
+	dsc35_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
+	dsc35_set_fgcg(dsc,
+		       ctx->dc->debug.enable_fine_grain_clock_gating.bits.dsc);
+	return &dsc->base;
+}
+
+static void dcn35_destroy_resource_pool(struct resource_pool **pool)
+{
+	struct dcn35_resource_pool *dcn35_pool = TO_DCN35_RES_POOL(*pool);
+
+	dcn35_resource_destruct(dcn35_pool);
+	kfree(dcn35_pool);
+	*pool = NULL;
+}
+
+static struct clock_source *dcn35_clock_source_create(
+		struct dc_context *ctx,
+		struct dc_bios *bios,
+		enum clock_source_id id,
+		const struct dce110_clk_src_regs *regs,
+		bool dp_clk_src)
+{
+	struct dce110_clk_src *clk_src =
+		kzalloc(sizeof(struct dce110_clk_src), GFP_KERNEL);
+
+	if (!clk_src)
+		return NULL;
+
+	if (dcn31_clk_src_construct(clk_src, ctx, bios, id,
+			regs, &cs_shift, &cs_mask)) {
+		clk_src->base.dp_clk_src = dp_clk_src;
+		return &clk_src->base;
+	}
+
+	BREAK_TO_DEBUGGER();
+	return NULL;
+}
+
+static struct dc_cap_funcs cap_funcs = {
+	.get_dcc_compression_cap = dcn20_get_dcc_compression_cap
+};
+
+static void dcn35_get_panel_config_defaults(struct dc_panel_config *panel_config)
+{
+	*panel_config = panel_config_defaults;
+}
+
+
+static bool dcn35_validate_bandwidth(struct dc *dc,
+		struct dc_state *context,
+		bool fast_validate)
+{
+	bool out = false;
+
+	out = dml2_validate(dc, context, fast_validate);
+
+	return out;
+}
+
+
+static struct resource_funcs dcn35_res_pool_funcs = {
+	.destroy = dcn35_destroy_resource_pool,
+	.link_enc_create = dcn35_link_encoder_create,
+	.link_enc_create_minimal = dcn31_link_enc_create_minimal,
+	.link_encs_assign = link_enc_cfg_link_encs_assign,
+	.link_enc_unassign = link_enc_cfg_link_enc_unassign,
+	.panel_cntl_create = dcn31_panel_cntl_create,
+	.validate_bandwidth = dcn35_validate_bandwidth,
+	.calculate_wm_and_dlg = NULL,
+	.update_soc_for_wm_a = dcn31_update_soc_for_wm_a,
+	.populate_dml_pipes = dcn35_populate_dml_pipes_from_context_fpu,
+	.acquire_free_pipe_as_secondary_dpp_pipe = dcn20_acquire_free_pipe_for_layer,
+	.release_pipe = dcn20_release_pipe,
+	.add_stream_to_ctx = dcn30_add_stream_to_ctx,
+	.add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource,
+	.remove_stream_from_ctx = dcn20_remove_stream_from_ctx,
+	.populate_dml_writeback_from_context = dcn30_populate_dml_writeback_from_context,
+	.set_mcif_arb_params = dcn30_set_mcif_arb_params,
+	.find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link,
+	.acquire_post_bldn_3dlut = dcn30_acquire_post_bldn_3dlut,
+	.release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut,
+	.update_bw_bounding_box = dcn35_update_bw_bounding_box_fpu,
+	.patch_unknown_plane_state = dcn20_patch_unknown_plane_state,
+	.get_panel_config_defaults = dcn35_get_panel_config_defaults,
+	.get_preferred_eng_id_dpia = dcn35_get_preferred_eng_id_dpia,
+};
+
+static bool dcn35_resource_construct(
+	uint8_t num_virtual_links,
+	struct dc *dc,
+	struct dcn35_resource_pool *pool)
+{
+	int i;
+	struct dc_context *ctx = dc->ctx;
+	struct irq_service_init_data init_data;
+
+#undef REG_STRUCT
+#define REG_STRUCT bios_regs
+	bios_regs_init();
+
+#undef REG_STRUCT
+#define REG_STRUCT clk_src_regs
+	clk_src_regs_init(0, A),
+	clk_src_regs_init(1, B),
+	clk_src_regs_init(2, C),
+	clk_src_regs_init(3, D),
+	clk_src_regs_init(4, E);
+
+#undef REG_STRUCT
+#define REG_STRUCT abm_regs
+	abm_regs_init(0),
+	abm_regs_init(1),
+	abm_regs_init(2),
+	abm_regs_init(3);
+
+#undef REG_STRUCT
+#define REG_STRUCT dccg_regs
+	dccg_regs_init();
+
+	ctx->dc_bios->regs = &bios_regs;
+
+	pool->base.res_cap = &res_cap_dcn35;
+
+	pool->base.funcs = &dcn35_res_pool_funcs;
+
+	/*************************************************
+	 *  Resource + asic cap harcoding                *
+	 *************************************************/
+	pool->base.underlay_pipe_index = NO_UNDERLAY_PIPE;
+	pool->base.pipe_count = pool->base.res_cap->num_timing_generator;
+	pool->base.mpcc_count = pool->base.res_cap->num_timing_generator;
+	dc->caps.max_downscale_ratio = 600;
+	dc->caps.i2c_speed_in_khz = 100;
+	dc->caps.i2c_speed_in_khz_hdcp = 100;
+	dc->caps.max_cursor_size = 256;
+	dc->caps.min_horizontal_blanking_period = 80;
+	dc->caps.dmdata_alloc_size = 2048;
+	dc->caps.max_slave_planes = 2;
+	dc->caps.max_slave_yuv_planes = 2;
+	dc->caps.max_slave_rgb_planes = 2;
+	dc->caps.post_blend_color_processing = true;
+	dc->caps.force_dp_tps4_for_cp2520 = true;
+	if (dc->config.forceHBR2CP2520)
+		dc->caps.force_dp_tps4_for_cp2520 = false;
+	dc->caps.dp_hpo = true;
+	dc->caps.dp_hdmi21_pcon_support = true;
+
+	dc->caps.edp_dsc_support = true;
+	dc->caps.extended_aux_timeout_support = true;
+	dc->caps.dmcub_support = true;
+	dc->caps.is_apu = true;
+	dc->caps.seamless_odm = true;
+
+	dc->caps.zstate_support = true;
+	dc->caps.ips_support = true;
+	dc->caps.max_v_total = (1 << 15) - 1;
+
+	/* Color pipeline capabilities */
+	dc->caps.color.dpp.dcn_arch = 1;
+	dc->caps.color.dpp.input_lut_shared = 0;
+	dc->caps.color.dpp.icsc = 1;
+	dc->caps.color.dpp.dgam_ram = 0; // must use gamma_corr
+	dc->caps.color.dpp.dgam_rom_caps.srgb = 1;
+	dc->caps.color.dpp.dgam_rom_caps.bt2020 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.gamma2_2 = 1;
+	dc->caps.color.dpp.dgam_rom_caps.pq = 1;
+	dc->caps.color.dpp.dgam_rom_caps.hlg = 1;
+	dc->caps.color.dpp.post_csc = 1;
+	dc->caps.color.dpp.gamma_corr = 1;
+	dc->caps.color.dpp.dgam_rom_for_yuv = 0;
+
+	dc->caps.color.dpp.hw_3d_lut = 1;
+	dc->caps.color.dpp.ogam_ram = 0;  // no OGAM in DPP since DCN1
+	// no OGAM ROM on DCN301
+	dc->caps.color.dpp.ogam_rom_caps.srgb = 0;
+	dc->caps.color.dpp.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.dpp.ogam_rom_caps.pq = 0;
+	dc->caps.color.dpp.ogam_rom_caps.hlg = 0;
+	dc->caps.color.dpp.ocsc = 0;
+
+	dc->caps.color.mpc.gamut_remap = 1;
+	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //2
+	dc->caps.color.mpc.ogam_ram = 1;
+	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
+	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.gamma2_2 = 0;
+	dc->caps.color.mpc.ogam_rom_caps.pq = 0;
+	dc->caps.color.mpc.ogam_rom_caps.hlg = 0;
+	dc->caps.color.mpc.ocsc = 1;
+
+	/* max_disp_clock_khz_at_vmin is slightly lower than the STA value in order
+	 * to provide some margin.
+	 * It's expected for furture ASIC to have equal or higher value, in order to
+	 * have determinstic power improvement from generate to genration.
+	 * (i.e., we should not expect new ASIC generation with lower vmin rate)
+	 */
+	dc->caps.max_disp_clock_khz_at_vmin = 650000;
+
+	/* Use pipe context based otg sync logic */
+	dc->config.use_pipe_ctx_sync_logic = true;
+	dc->config.use_default_clock_table = false;
+	/* read VBIOS LTTPR caps */
+	{
+		if (ctx->dc_bios->funcs->get_lttpr_caps) {
+			enum bp_result bp_query_result;
+			uint8_t is_vbios_lttpr_enable = 0;
+
+			bp_query_result = ctx->dc_bios->funcs->get_lttpr_caps(ctx->dc_bios, &is_vbios_lttpr_enable);
+			dc->caps.vbios_lttpr_enable = (bp_query_result == BP_RESULT_OK) && !!is_vbios_lttpr_enable;
+		}
+
+		/* interop bit is implicit */
+		{
+			dc->caps.vbios_lttpr_aware = true;
+		}
+	}
+
+	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
+		dc->debug = debug_defaults_drv;
+
+	// Init the vm_helper
+	if (dc->vm_helper)
+		vm_helper_init(dc->vm_helper, 16);
+
+	/*************************************************
+	 *  Create resources                             *
+	 *************************************************/
+
+	/* Clock Sources for Pixel Clock*/
+	pool->base.clock_sources[DCN35_CLK_SRC_PLL0] =
+			dcn35_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL0,
+				&clk_src_regs[0], false);
+	pool->base.clock_sources[DCN35_CLK_SRC_PLL1] =
+			dcn35_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL1,
+				&clk_src_regs[1], false);
+	pool->base.clock_sources[DCN35_CLK_SRC_PLL2] =
+			dcn35_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL2,
+				&clk_src_regs[2], false);
+	pool->base.clock_sources[DCN35_CLK_SRC_PLL3] =
+			dcn35_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL3,
+				&clk_src_regs[3], false);
+	pool->base.clock_sources[DCN35_CLK_SRC_PLL4] =
+			dcn35_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_COMBO_PHY_PLL4,
+				&clk_src_regs[4], false);
+
+	pool->base.clk_src_count = DCN35_CLK_SRC_TOTAL;
+
+	/* todo: not reuse phy_pll registers */
+	pool->base.dp_clock_source =
+			dcn35_clock_source_create(ctx, ctx->dc_bios,
+				CLOCK_SOURCE_ID_DP_DTO,
+				&clk_src_regs[0], true);
+
+	for (i = 0; i < pool->base.clk_src_count; i++) {
+		if (pool->base.clock_sources[i] == NULL) {
+			dm_error("DC: failed to create clock sources!\n");
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+	/*temp till dml2 fully work without dml1*/
+	dml_init_instance(&dc->dml, &dcn3_5_soc, &dcn3_5_ip, DML_PROJECT_DCN31);
+
+	/* TODO: DCCG */
+	pool->base.dccg = dccg35_create(ctx, &dccg_regs, &dccg_shift, &dccg_mask);
+	if (pool->base.dccg == NULL) {
+		dm_error("DC: failed to create dccg!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+#undef REG_STRUCT
+#define REG_STRUCT pg_cntl_regs
+	pg_cntl_dcn35_regs_init();
+
+	pool->base.pg_cntl = pg_cntl35_create(ctx, &pg_cntl_regs, &pg_cntl_shift, &pg_cntl_mask);
+	if (pool->base.pg_cntl == NULL) {
+		dm_error("DC: failed to create power gate control!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* TODO: IRQ */
+	init_data.ctx = dc->ctx;
+	pool->base.irqs = dal_irq_service_dcn35_create(&init_data);
+	if (!pool->base.irqs)
+		goto create_fail;
+
+	/* HUBBUB */
+	pool->base.hubbub = dcn35_hubbub_create(ctx);
+	if (pool->base.hubbub == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create hubbub!\n");
+		goto create_fail;
+	}
+
+	/* HUBPs, DPPs, OPPs and TGs */
+	for (i = 0; i < pool->base.pipe_count; i++) {
+		pool->base.hubps[i] = dcn35_hubp_create(ctx, i);
+		if (pool->base.hubps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create hubps!\n");
+			goto create_fail;
+		}
+
+		pool->base.dpps[i] = dcn35_dpp_create(ctx, i);
+		if (pool->base.dpps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create dpps!\n");
+			goto create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+		pool->base.opps[i] = dcn35_opp_create(ctx, i);
+		if (pool->base.opps[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC: failed to create output pixel processor!\n");
+			goto create_fail;
+		}
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		pool->base.timing_generators[i] = dcn35_timing_generator_create(
+				ctx, i);
+		if (pool->base.timing_generators[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create tg!\n");
+			goto create_fail;
+		}
+	}
+	pool->base.timing_generator_count = i;
+
+	/* PSR */
+	pool->base.psr = dmub_psr_create(ctx);
+	if (pool->base.psr == NULL) {
+		dm_error("DC: failed to create psr obj!\n");
+		BREAK_TO_DEBUGGER();
+		goto create_fail;
+	}
+
+	/* ABM */
+	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+		pool->base.multiple_abms[i] = dmub_abm_create(ctx,
+				&abm_regs[i],
+				&abm_shift,
+				&abm_mask);
+		if (pool->base.multiple_abms[i] == NULL) {
+			dm_error("DC: failed to create abm for pipe %d!\n", i);
+			BREAK_TO_DEBUGGER();
+			goto create_fail;
+		}
+	}
+
+	/* MPC and DSC */
+	pool->base.mpc = dcn35_mpc_create(ctx, pool->base.mpcc_count, pool->base.res_cap->num_mpc_3dlut);
+	if (pool->base.mpc == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mpc!\n");
+		goto create_fail;
+	}
+
+	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+		pool->base.dscs[i] = dcn35_dsc_create(ctx, i);
+		if (pool->base.dscs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error("DC: failed to create display stream compressor %d!\n", i);
+			goto create_fail;
+		}
+	}
+
+	/* DWB and MMHUBBUB */
+	if (!dcn35_dwbc_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create dwbc!\n");
+		goto create_fail;
+	}
+
+	if (!dcn35_mmhubbub_create(ctx, &pool->base)) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create mcif_wb!\n");
+		goto create_fail;
+	}
+
+	/* AUX and I2C */
+	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+		pool->base.engines[i] = dcn31_aux_engine_create(ctx, i);
+		if (pool->base.engines[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create aux engine!!\n");
+			goto create_fail;
+		}
+		pool->base.hw_i2cs[i] = dcn31_i2c_hw_create(ctx, i);
+		if (pool->base.hw_i2cs[i] == NULL) {
+			BREAK_TO_DEBUGGER();
+			dm_error(
+				"DC:failed to create hw i2c!!\n");
+			goto create_fail;
+		}
+		pool->base.sw_i2cs[i] = NULL;
+	}
+
+	/* DCN3.5 has 6 DPIA */
+	pool->base.usb4_dpia_count = 4;
+	if (dc->debug.dpia_debug.bits.disable_dpia)
+		pool->base.usb4_dpia_count = 0;
+
+	/* Audio, Stream Encoders including HPO and virtual, MPC 3D LUTs */
+	if (!resource_construct(num_virtual_links, dc, &pool->base,
+			&res_create_funcs))
+		goto create_fail;
+
+	/* HW Sequencer and Plane caps */
+	dcn35_hw_sequencer_construct(dc);
+
+	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->cap_funcs = cap_funcs;
+
+	dc->dcn_ip->max_num_dpp = pool->base.pipe_count;
+
+	dc->dml2_options.dcn_pipe_count = pool->base.pipe_count;
+	dc->dml2_options.use_native_pstate_optimization = true;
+	dc->dml2_options.use_native_soc_bb_construction = true;
+	if (dc->config.EnableMinDispClkODM)
+		dc->dml2_options.minimize_dispclk_using_odm = true;
+	dc->dml2_options.enable_windowed_mpo_odm = dc->config.enable_windowed_mpo_odm;
+
+	dc->dml2_options.callbacks.dc = dc;
+	dc->dml2_options.callbacks.build_scaling_params = &resource_build_scaling_params;
+	dc->dml2_options.callbacks.can_support_mclk_switch_using_fw_based_vblank_stretch = &dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch;
+	dc->dml2_options.callbacks.acquire_secondary_pipe_for_mpc_odm = &dc_resource_acquire_secondary_pipe_for_mpc_odm_legacy;
+	dc->dml2_options.callbacks.update_pipes_for_stream_with_slice_count = &resource_update_pipes_for_stream_with_slice_count;
+	dc->dml2_options.callbacks.update_pipes_for_plane_with_slice_count = &resource_update_pipes_for_plane_with_slice_count;
+	dc->dml2_options.callbacks.get_mpc_slice_index = &resource_get_mpc_slice_index;
+	dc->dml2_options.callbacks.get_odm_slice_index = &resource_get_odm_slice_index;
+	dc->dml2_options.callbacks.get_opp_head = &resource_get_opp_head;
+	dc->dml2_options.max_segments_per_hubp = 24;
+
+	dc->dml2_options.det_segment_size = DCN3_2_DET_SEG_SIZE;/*todo*/
+
+	if (dc->config.sdpif_request_limit_words_per_umc == 0)
+		dc->config.sdpif_request_limit_words_per_umc = 16;/*todo*/
+
+	return true;
+
+create_fail:
+
+	dcn35_resource_destruct(pool);
+
+	return false;
+}
+
+struct resource_pool *dcn35_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc)
+{
+	struct dcn35_resource_pool *pool =
+		kzalloc(sizeof(struct dcn35_resource_pool), GFP_KERNEL);
+
+	if (!pool)
+		return NULL;
+
+	if (dcn35_resource_construct(init_data->num_virtual_links, dc, pool))
+		return &pool->base;
+
+	BREAK_TO_DEBUGGER();
+	kfree(pool);
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.h
new file mode 100644
index 000000000000..a51c4a9eaafe
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.h
@@ -0,0 +1,311 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2023 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef _DCN35_RESOURCE_H_
+#define _DCN35_RESOURCE_H_
+
+#include "core_types.h"
+
+#define DCN3_5_VMIN_DISPCLK_HZ 717000000
+#define TO_DCN35_RES_POOL(pool)\
+	container_of(pool, struct dcn35_resource_pool, base)
+
+extern struct _vcs_dpi_ip_params_st dcn3_5_ip;
+extern struct _vcs_dpi_soc_bounding_box_st dcn3_5_soc;
+
+struct dcn35_resource_pool {
+	struct resource_pool base;
+};
+
+struct resource_pool *dcn35_create_resource_pool(
+		const struct dc_init_data *init_data,
+		struct dc *dc);
+
+/* Defs for runtime init of registers */
+
+#define OPP_REG_LIST_DCN20_RI(id) \
+	OPP_REG_LIST_DCN10_RI(id), \
+	OPP_DPG_REG_LIST_RI(id), \
+	SRI_ARR(FMT_422_CONTROL, FMT, id), \
+	SRI_ARR(OPPBUF_CONTROL1, OPPBUF, id)
+
+#define OPP_REG_LIST_DCN35_RI(id) \
+	OPP_REG_LIST_DCN20_RI(id), \
+	SRI2_ARR(OPP_TOP_CLK_CONTROL, OPP, id)
+
+#define VPG_DCN31_REG_LIST_RI(id) \
+	SRI_ARR(VPG_GENERIC_STATUS, VPG, id), \
+	SRI_ARR(VPG_GENERIC_PACKET_ACCESS_CTRL, VPG, id), \
+	SRI_ARR(VPG_GENERIC_PACKET_DATA, VPG, id), \
+	SRI_ARR(VPG_GSP_FRAME_UPDATE_CTRL, VPG, id), \
+	SRI_ARR(VPG_GSP_IMMEDIATE_UPDATE_CTRL, VPG, id), \
+	SRI_ARR(VPG_MEM_PWR, VPG, id)
+
+#define AFMT_DCN31_REG_LIST_RI(id) \
+	SRI_ARR(AFMT_INFOFRAME_CONTROL0, AFMT, id), \
+	SRI_ARR(AFMT_VBI_PACKET_CONTROL, AFMT, id), \
+	SRI_ARR(AFMT_AUDIO_PACKET_CONTROL, AFMT, id), \
+	SRI_ARR(AFMT_AUDIO_PACKET_CONTROL2, AFMT, id), \
+	SRI_ARR(AFMT_AUDIO_SRC_CONTROL, AFMT, id), \
+	SRI_ARR(AFMT_60958_0, AFMT, id), \
+	SRI_ARR(AFMT_60958_1, AFMT, id), \
+	SRI_ARR(AFMT_60958_2, AFMT, id), \
+	SRI_ARR(AFMT_MEM_PWR, AFMT, id)
+
+/* Stream encoder */
+#define SE_DCN35_REG_LIST_RI(id) \
+	SRI_ARR(AFMT_CNTL, DIG, id), \
+	SRI_ARR(DIG_FE_CNTL, DIG, id), \
+	SRI_ARR(HDMI_CONTROL, DIG, id), \
+	SRI_ARR(HDMI_DB_CONTROL, DIG, id), \
+	SRI_ARR(HDMI_GC, DIG, id), \
+	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL0, DIG, id), \
+	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL1, DIG, id), \
+	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL2, DIG, id), \
+	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL3, DIG, id), \
+	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL4, DIG, id), \
+	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL5, DIG, id), \
+	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL6, DIG, id), \
+	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL7, DIG, id), \
+	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL8, DIG, id), \
+	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL9, DIG, id), \
+	SRI_ARR(HDMI_GENERIC_PACKET_CONTROL10, DIG, id), \
+	SRI_ARR(HDMI_INFOFRAME_CONTROL0, DIG, id), \
+	SRI_ARR(HDMI_INFOFRAME_CONTROL1, DIG, id), \
+	SRI_ARR(HDMI_VBI_PACKET_CONTROL, DIG, id), \
+	SRI_ARR(HDMI_AUDIO_PACKET_CONTROL, DIG, id),\
+	SRI_ARR(HDMI_ACR_PACKET_CONTROL, DIG, id),\
+	SRI_ARR(HDMI_ACR_32_0, DIG, id),\
+	SRI_ARR(HDMI_ACR_32_1, DIG, id),\
+	SRI_ARR(HDMI_ACR_44_0, DIG, id),\
+	SRI_ARR(HDMI_ACR_44_1, DIG, id),\
+	SRI_ARR(HDMI_ACR_48_0, DIG, id),\
+	SRI_ARR(HDMI_ACR_48_1, DIG, id),\
+	SRI_ARR(DP_DB_CNTL, DP, id), \
+	SRI_ARR(DP_MSA_MISC, DP, id), \
+	SRI_ARR(DP_MSA_VBID_MISC, DP, id), \
+	SRI_ARR(DP_MSA_COLORIMETRY, DP, id), \
+	SRI_ARR(DP_MSA_TIMING_PARAM1, DP, id), \
+	SRI_ARR(DP_MSA_TIMING_PARAM2, DP, id), \
+	SRI_ARR(DP_MSA_TIMING_PARAM3, DP, id), \
+	SRI_ARR(DP_MSA_TIMING_PARAM4, DP, id), \
+	SRI_ARR(DP_MSE_RATE_CNTL, DP, id), \
+	SRI_ARR(DP_MSE_RATE_UPDATE, DP, id), \
+	SRI_ARR(DP_PIXEL_FORMAT, DP, id), \
+	SRI_ARR(DP_SEC_CNTL, DP, id), \
+	SRI_ARR(DP_SEC_CNTL1, DP, id), \
+	SRI_ARR(DP_SEC_CNTL2, DP, id), \
+	SRI_ARR(DP_SEC_CNTL5, DP, id), \
+	SRI_ARR(DP_SEC_CNTL6, DP, id), \
+	SRI_ARR(DP_STEER_FIFO, DP, id), \
+	SRI_ARR(DP_VID_M, DP, id), \
+	SRI_ARR(DP_VID_N, DP, id), \
+	SRI_ARR(DP_VID_STREAM_CNTL, DP, id), \
+	SRI_ARR(DP_VID_TIMING, DP, id), \
+	SRI_ARR(DP_SEC_AUD_N, DP, id), \
+	SRI_ARR(DP_SEC_TIMESTAMP, DP, id), \
+	SRI_ARR(DP_DSC_CNTL, DP, id), \
+	SRI_ARR(DP_SEC_METADATA_TRANSMISSION, DP, id), \
+	SRI_ARR(HDMI_METADATA_PACKET_CONTROL, DIG, id), \
+	SRI_ARR(DP_SEC_FRAMING4, DP, id), \
+	SRI_ARR(DP_GSP11_CNTL, DP, id), \
+	SRI_ARR(DME_CONTROL, DME, id),\
+	SRI_ARR(DP_SEC_METADATA_TRANSMISSION, DP, id), \
+	SRI_ARR(HDMI_METADATA_PACKET_CONTROL, DIG, id), \
+	SRI_ARR(DIG_FE_CNTL, DIG, id), \
+	SRI_ARR(DIG_FE_EN_CNTL, DIG, id), \
+	SRI_ARR(DIG_FE_CLK_CNTL, DIG, id), \
+	SRI_ARR(DIG_CLOCK_PATTERN, DIG, id), \
+	SRI_ARR(DIG_FIFO_CTRL0, DIG, id), \
+	SRI_ARR(STREAM_MAPPER_CONTROL, DIG, id)
+
+#define LE_DCN35_REG_LIST_RI(id)\
+	LE_DCN3_REG_LIST_RI(id),\
+	SRI_ARR(DP_DPHY_INTERNAL_CTRL, DP, id), \
+	SR_ARR(DIO_LINKA_CNTL, id), \
+	SR_ARR(DIO_LINKB_CNTL, id), \
+	SR_ARR(DIO_LINKC_CNTL, id), \
+	SR_ARR(DIO_LINKD_CNTL, id), \
+	SR_ARR(DIO_LINKE_CNTL, id), \
+	SR_ARR(DIO_LINKF_CNTL, id),\
+	SRI_ARR(DIG_BE_CLK_CNTL, DIG, id),\
+	SR_ARR(DIO_CLK_CNTL, id)
+
+#define MCIF_WB_COMMON_REG_LIST_DCN3_5_RI(inst)  \
+	MCIF_WB_COMMON_REG_LIST_DCN32_RI(inst), \
+		SRI2_ARR(MMHUBBUB_CLOCK_CNTL, MMHUBBUB, inst)
+
+#define HWSEQ_DCN35_REG_LIST()\
+	SR(DCHUBBUB_GLOBAL_TIMER_CNTL), \
+	SR(DCHUBBUB_ARB_HOSTVM_CNTL), \
+	SR(DIO_MEM_PWR_CTRL), \
+	SR(ODM_MEM_PWR_CTRL3), \
+	SR(MMHUBBUB_MEM_PWR_CNTL), \
+	SR(DCCG_GATE_DISABLE_CNTL), \
+	SR(DCCG_GATE_DISABLE_CNTL2), \
+	SR(DCCG_GATE_DISABLE_CNTL4), \
+	SR(DCCG_GATE_DISABLE_CNTL5), \
+	SR(DCFCLK_CNTL),\
+	SR(DC_MEM_GLOBAL_PWR_REQ_CNTL), \
+	SRII(PIXEL_RATE_CNTL, OTG, 0), \
+	SRII(PIXEL_RATE_CNTL, OTG, 1),\
+	SRII(PIXEL_RATE_CNTL, OTG, 2),\
+	SRII(PIXEL_RATE_CNTL, OTG, 3),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 0),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 1),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 2),\
+	SRII(PHYPLL_PIXEL_RATE_CNTL, OTG, 3),\
+	SR(MICROSECOND_TIME_BASE_DIV), \
+	SR(MILLISECOND_TIME_BASE_DIV), \
+	SR(DISPCLK_FREQ_CHANGE_CNTL), \
+	SR(RBBMIF_TIMEOUT_DIS), \
+	SR(RBBMIF_TIMEOUT_DIS_2), \
+	SR(DCHUBBUB_CRC_CTRL), \
+	SR(DPP_TOP0_DPP_CRC_CTRL), \
+	SR(DPP_TOP0_DPP_CRC_VAL_B_A), \
+	SR(DPP_TOP0_DPP_CRC_VAL_R_G), \
+	SR(MPC_CRC_CTRL), \
+	SR(MPC_CRC_RESULT_GB), \
+	SR(MPC_CRC_RESULT_C), \
+	SR(MPC_CRC_RESULT_AR), \
+	SR(DOMAIN0_PG_CONFIG), \
+	SR(DOMAIN1_PG_CONFIG), \
+	SR(DOMAIN2_PG_CONFIG), \
+	SR(DOMAIN3_PG_CONFIG), \
+	SR(DOMAIN16_PG_CONFIG), \
+	SR(DOMAIN17_PG_CONFIG), \
+	SR(DOMAIN18_PG_CONFIG), \
+	SR(DOMAIN19_PG_CONFIG), \
+	SR(DOMAIN0_PG_STATUS), \
+	SR(DOMAIN1_PG_STATUS), \
+	SR(DOMAIN2_PG_STATUS), \
+	SR(DOMAIN3_PG_STATUS), \
+	SR(DOMAIN16_PG_STATUS), \
+	SR(DOMAIN17_PG_STATUS), \
+	SR(DOMAIN18_PG_STATUS), \
+	SR(DOMAIN19_PG_STATUS), \
+	SR(DC_IP_REQUEST_CNTL), \
+	SR(AZALIA_AUDIO_DTO), \
+	SR(AZALIA_CONTROLLER_CLOCK_GATING), \
+	SR(HPO_TOP_HW_CONTROL),\
+	SR(DMU_CLK_CNTL)
+
+/* OPTC */
+#define OPTC_COMMON_REG_LIST_DCN3_5_RI(inst)                                   \
+	SRI_ARR(OTG_VSTARTUP_PARAM, OTG, inst),\
+	SRI_ARR(OTG_VUPDATE_PARAM, OTG, inst),\
+	SRI_ARR(OTG_VREADY_PARAM, OTG, inst),\
+	SRI_ARR(OTG_MASTER_UPDATE_LOCK, OTG, inst),\
+	SRI_ARR(OTG_GLOBAL_CONTROL0, OTG, inst),\
+	SRI_ARR(OTG_GLOBAL_CONTROL1, OTG, inst),\
+	SRI_ARR(OTG_GLOBAL_CONTROL2, OTG, inst),\
+	SRI_ARR(OTG_GLOBAL_CONTROL4, OTG, inst),\
+	SRI_ARR(OTG_DOUBLE_BUFFER_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_H_TOTAL, OTG, inst),\
+	SRI_ARR(OTG_H_BLANK_START_END, OTG, inst),\
+	SRI_ARR(OTG_H_SYNC_A, OTG, inst),\
+	SRI_ARR(OTG_H_SYNC_A_CNTL, OTG, inst),\
+	SRI_ARR(OTG_H_TIMING_CNTL, OTG, inst),\
+	SRI_ARR(OTG_V_TOTAL, OTG, inst),\
+	SRI_ARR(OTG_V_BLANK_START_END, OTG, inst),\
+	SRI_ARR(OTG_V_SYNC_A, OTG, inst),\
+	SRI_ARR(OTG_V_SYNC_A_CNTL, OTG, inst),\
+	SRI_ARR(OTG_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_STEREO_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_3D_STRUCTURE_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_STEREO_STATUS, OTG, inst),\
+	SRI_ARR(OTG_V_TOTAL_MAX, OTG, inst),\
+	SRI_ARR(OTG_V_TOTAL_MIN, OTG, inst),\
+	SRI_ARR(OTG_V_TOTAL_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_TRIGA_CNTL, OTG, inst),\
+	SRI_ARR(OTG_FORCE_COUNT_NOW_CNTL, OTG, inst),\
+	SRI_ARR(OTG_STATIC_SCREEN_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_STATUS_FRAME_COUNT, OTG, inst),\
+	SRI_ARR(OTG_STATUS, OTG, inst),\
+	SRI_ARR(OTG_STATUS_POSITION, OTG, inst),\
+	SRI_ARR(OTG_NOM_VERT_POSITION, OTG, inst),\
+	SRI_ARR(OTG_M_CONST_DTO0, OTG, inst),\
+	SRI_ARR(OTG_M_CONST_DTO1, OTG, inst),\
+	SRI_ARR(OTG_CLOCK_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_VERTICAL_INTERRUPT0_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_VERTICAL_INTERRUPT0_POSITION, OTG, inst),\
+	SRI_ARR(OTG_VERTICAL_INTERRUPT1_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_VERTICAL_INTERRUPT1_POSITION, OTG, inst),\
+	SRI_ARR(OTG_VERTICAL_INTERRUPT2_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_VERTICAL_INTERRUPT2_POSITION, OTG, inst),\
+	SRI_ARR(OPTC_INPUT_CLOCK_CONTROL, ODM, inst),\
+	SRI_ARR(OPTC_DATA_SOURCE_SELECT, ODM, inst),\
+	SRI_ARR(OPTC_INPUT_GLOBAL_CONTROL, ODM, inst),\
+	SRI_ARR(CONTROL, VTG, inst),\
+	SRI_ARR(OTG_VERT_SYNC_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_GSL_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_CRC_CNTL, OTG, inst),\
+	SRI_ARR(OTG_CRC0_DATA_RG, OTG, inst),\
+	SRI_ARR(OTG_CRC0_DATA_B, OTG, inst),\
+	SRI_ARR(OTG_CRC1_DATA_RG, OTG, inst),\
+	SRI_ARR(OTG_CRC1_DATA_B, OTG, inst),\
+	SRI_ARR(OTG_CRC2_DATA_RG, OTG, inst),\
+	SRI_ARR(OTG_CRC2_DATA_B, OTG, inst),\
+	SRI_ARR(OTG_CRC3_DATA_RG, OTG, inst),\
+	SRI_ARR(OTG_CRC3_DATA_B, OTG, inst),\
+	SRI_ARR(OTG_CRC0_WINDOWA_X_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_CRC0_WINDOWA_Y_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_CRC0_WINDOWB_X_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_CRC0_WINDOWB_Y_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_CRC1_WINDOWA_X_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_CRC1_WINDOWA_Y_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_CRC1_WINDOWB_X_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_CRC1_WINDOWB_Y_CONTROL, OTG, inst),\
+	SRI_ARR(OTG_CRC0_WINDOWA_X_CONTROL_READBACK, OTG, inst),\
+	SRI_ARR(OTG_CRC0_WINDOWA_Y_CONTROL_READBACK, OTG, inst),\
+	SRI_ARR(OTG_CRC0_WINDOWB_X_CONTROL_READBACK, OTG, inst),\
+	SRI_ARR(OTG_CRC0_WINDOWB_Y_CONTROL_READBACK, OTG, inst),\
+	SRI_ARR(OTG_CRC1_WINDOWA_X_CONTROL_READBACK, OTG, inst),\
+	SRI_ARR(OTG_CRC1_WINDOWA_Y_CONTROL_READBACK, OTG, inst),\
+	SRI_ARR(OTG_CRC1_WINDOWB_X_CONTROL_READBACK, OTG, inst),\
+	SRI_ARR(OTG_CRC1_WINDOWB_Y_CONTROL_READBACK, OTG, inst),\
+	SR_ARR(GSL_SOURCE_SELECT, inst),\
+	SRI_ARR(OTG_TRIGA_MANUAL_TRIG, OTG, inst),\
+	SRI_ARR(OTG_GLOBAL_CONTROL1, OTG, inst),\
+	SRI_ARR(OTG_GLOBAL_CONTROL2, OTG, inst),\
+	SRI_ARR(OTG_GSL_WINDOW_X, OTG, inst),\
+	SRI_ARR(OTG_GSL_WINDOW_Y, OTG, inst),\
+	SRI_ARR(OTG_VUPDATE_KEEPOUT, OTG, inst),\
+	SRI_ARR(OTG_DSC_START_POSITION, OTG, inst),\
+	SRI_ARR(OTG_DRR_TRIGGER_WINDOW, OTG, inst),\
+	SRI_ARR(OTG_DRR_V_TOTAL_CHANGE, OTG, inst),\
+	SRI_ARR(OPTC_DATA_FORMAT_CONTROL, ODM, inst),\
+	SRI_ARR(OPTC_BYTES_PER_PIXEL, ODM, inst),\
+	SRI_ARR(OPTC_WIDTH_CONTROL, ODM, inst),\
+	SRI_ARR(OPTC_MEMORY_CONFIG, ODM, inst),\
+	SRI_ARR(OTG_DRR_CONTROL, OTG, inst),\
+	SRI2_ARR(OPTC_CLOCK_CONTROL, OPTC, inst)
+
+/* DPP */
+#define DPP_REG_LIST_DCN35_RI(id)\
+	DPP_REG_LIST_DCN30_COMMON_RI(id)
+
+#endif /* _DCN35_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
index ed4379c04715..55573083bc31 100644
--- a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
+++ b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
@@ -818,18 +818,54 @@ enum dmub_gpint_command {
 	 * RETURN: Lower 32-bit mask.
 	 */
 	DMUB_GPINT__UPDATE_TRACE_BUFFER_MASK = 101,
+
 	/**
-	 * DESC: Updates the trace buffer lower 32-bit mask.
+	 * DESC: Updates the trace buffer mask bit0~bit15.
 	 * ARGS: The new mask
 	 * RETURN: Lower 32-bit mask.
 	 */
 	DMUB_GPINT__SET_TRACE_BUFFER_MASK_WORD0 = 102,
+
 	/**
-	 * DESC: Updates the trace buffer mask bi0~bit15.
+	 * DESC: Updates the trace buffer mask bit16~bit31.
 	 * ARGS: The new mask
 	 * RETURN: Lower 32-bit mask.
 	 */
 	DMUB_GPINT__SET_TRACE_BUFFER_MASK_WORD1 = 103,
+
+	/**
+	 * DESC: Updates the trace buffer mask bit32~bit47.
+	 * ARGS: The new mask
+	 * RETURN: Lower 32-bit mask.
+	 */
+	DMUB_GPINT__SET_TRACE_BUFFER_MASK_WORD2 = 114,
+
+	/**
+	 * DESC: Updates the trace buffer mask bit48~bit63.
+	 * ARGS: The new mask
+	 * RETURN: Lower 32-bit mask.
+	 */
+	DMUB_GPINT__SET_TRACE_BUFFER_MASK_WORD3 = 115,
+
+	/**
+	 * DESC: Read the trace buffer mask bi0~bit15.
+	 */
+	DMUB_GPINT__GET_TRACE_BUFFER_MASK_WORD0 = 116,
+
+	/**
+	 * DESC: Read the trace buffer mask bit16~bit31.
+	 */
+	DMUB_GPINT__GET_TRACE_BUFFER_MASK_WORD1 = 117,
+
+	/**
+	 * DESC: Read the trace buffer mask bi32~bit47.
+	 */
+	DMUB_GPINT__GET_TRACE_BUFFER_MASK_WORD2 = 118,
+
+	/**
+	 * DESC: Updates the trace buffer mask bit32~bit63.
+	 */
+	DMUB_GPINT__GET_TRACE_BUFFER_MASK_WORD3 = 119,
 };
 
 /**
@@ -2840,6 +2876,10 @@ enum dmub_cmd_replay_type {
 	 * Set power opt and coasting vtotal.
 	 */
 	DMUB_CMD__REPLAY_SET_POWER_OPT_AND_COASTING_VTOTAL	= 4,
+	/**
+	 * Set disabled iiming sync.
+	 */
+	DMUB_CMD__REPLAY_SET_TIMING_SYNC_SUPPORTED	= 5,
 };
 
 /**
@@ -3002,6 +3042,27 @@ struct dmub_cmd_replay_set_power_opt_data {
 	uint32_t power_opt;
 };
 
+/**
+ * Data passed from driver to FW in a DMUB_CMD__REPLAY_SET_TIMING_SYNC_SUPPORTED command.
+ */
+struct dmub_cmd_replay_set_timing_sync_data {
+	/**
+	 * Panel Instance.
+	 * Panel isntance to identify which replay_state to use
+	 * Currently the support is only for 0 or 1
+	 */
+	uint8_t panel_inst;
+
+	/**
+	 * Explicit padding to 4 byte boundary.
+	 */
+	uint8_t pad[3];
+	/**
+	 * REPLAY set_timing_sync
+	 */
+	bool timing_sync_supported;
+};
+
 /**
  * Definition of a DMUB_CMD__SET_REPLAY_POWER_OPT command.
  */
@@ -3068,6 +3129,20 @@ struct dmub_rb_cmd_replay_set_power_opt_and_coasting_vtotal {
 	struct dmub_cmd_replay_set_coasting_vtotal_data replay_set_coasting_vtotal_data;
 };
 
+/**
+ * Definition of a DMUB_CMD__REPLAY_SET_TIMING_SYNC_SUPPORTED command.
+ */
+struct dmub_rb_cmd_replay_set_timing_sync {
+	/**
+	 * Command header.
+	 */
+	struct dmub_cmd_header header;
+	/**
+	 * Definition of DMUB_CMD__REPLAY_SET_TIMING_SYNC_SUPPORTED command.
+	 */
+	struct dmub_cmd_replay_set_timing_sync_data replay_set_timing_sync_data;
+};
+
 /**
  * Set of HW components that can be locked.
  *
@@ -4201,6 +4276,8 @@ union dmub_rb_cmd {
 	 * Definition of a DMUB_CMD__REPLAY_SET_POWER_OPT_AND_COASTING_VTOTAL command.
 	 */
 	struct dmub_rb_cmd_replay_set_power_opt_and_coasting_vtotal replay_set_power_opt_and_coasting_vtotal;
+
+	struct dmub_rb_cmd_replay_set_timing_sync replay_set_timing_sync;
 };
 
 /**
diff --git a/drivers/gpu/drm/amd/display/include/hdcp_msg_types.h b/drivers/gpu/drm/amd/display/include/hdcp_msg_types.h
index 42229b4effdc..eced9ad91f1d 100644
--- a/drivers/gpu/drm/amd/display/include/hdcp_msg_types.h
+++ b/drivers/gpu/drm/amd/display/include/hdcp_msg_types.h
@@ -69,6 +69,11 @@ enum hdcp_message_id {
 	HDCP_MESSAGE_ID_READ_RXSTATUS,
 	HDCP_MESSAGE_ID_WRITE_CONTENT_STREAM_TYPE,
 
+	/* PS175 chip */
+
+	HDCP_MESSAGE_ID_WRITE_PS175_CMD,
+	HDCP_MESSAGE_ID_READ_PS175_RSP,
+
 	HDCP_MESSAGE_ID_MAX
 };
 
diff --git a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c
index ccecddafeb05..47296d155c3a 100644
--- a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c
+++ b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c
@@ -693,10 +693,12 @@ static void build_vrr_infopacket_fs2_data(enum color_transfer_func app_tf,
 	if (app_tf != TRANSFER_FUNC_UNKNOWN) {
 		infopacket->valid = true;
 
-		if (app_tf != TRANSFER_FUNC_PQ2084) {
+		if (app_tf == TRANSFER_FUNC_PQ2084)
+			infopacket->sb[9] |= 0x20; // PB9 = [Bit 5 = PQ EOTF Active]
+		else {
 			infopacket->sb[6] |= 0x08;  // PB6 = [Bit 3 = Native Color Active]
 			if (app_tf == TRANSFER_FUNC_GAMMA_22)
-				infopacket->sb[9] |= 0x04;  // PB6 = [Bit 2 = Gamma 2.2 EOTF Active]
+				infopacket->sb[9] |= 0x04;  // PB9 = [Bit 2 = Gamma 2.2 EOTF Active]
 		}
 	}
 }
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_5_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_5_0_sh_mask.h
index b64664879211..fca72e2ec929 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_5_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_5_0_sh_mask.h
@@ -6220,12 +6220,20 @@
 #define DCCG_GATE_DISABLE_CNTL4__PHYD_REFCLK_ROOT_GATE_DISABLE__SHIFT                                         0x3
 #define DCCG_GATE_DISABLE_CNTL4__PHYE_REFCLK_ROOT_GATE_DISABLE__SHIFT                                         0x4
 #define DCCG_GATE_DISABLE_CNTL4__HDMICHARCLK0_ROOT_GATE_DISABLE__SHIFT                                        0x11
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK0_GATE_DISABLE__SHIFT                                              0x17
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK1_GATE_DISABLE__SHIFT                                              0x18
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK2_GATE_DISABLE__SHIFT                                              0x19
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK3_GATE_DISABLE__SHIFT                                              0x1a
 #define DCCG_GATE_DISABLE_CNTL4__PHYA_REFCLK_ROOT_GATE_DISABLE_MASK                                           0x00000001L
 #define DCCG_GATE_DISABLE_CNTL4__PHYB_REFCLK_ROOT_GATE_DISABLE_MASK                                           0x00000002L
 #define DCCG_GATE_DISABLE_CNTL4__PHYC_REFCLK_ROOT_GATE_DISABLE_MASK                                           0x00000004L
 #define DCCG_GATE_DISABLE_CNTL4__PHYD_REFCLK_ROOT_GATE_DISABLE_MASK                                           0x00000008L
 #define DCCG_GATE_DISABLE_CNTL4__PHYE_REFCLK_ROOT_GATE_DISABLE_MASK                                           0x00000010L
 #define DCCG_GATE_DISABLE_CNTL4__HDMICHARCLK0_ROOT_GATE_DISABLE_MASK                                          0x00020000L
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK0_GATE_DISABLE_MASK                                                0x00800000L
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK1_GATE_DISABLE_MASK                                                0x01000000L
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK2_GATE_DISABLE_MASK                                                0x02000000L
+#define DCCG_GATE_DISABLE_CNTL4__DPIASYMCLK3_GATE_DISABLE_MASK                                                0x04000000L
 #define DPSTREAMCLK_CNTL__DPSTREAMCLK0_SRC_SEL__SHIFT                                                         0x0
 #define DPSTREAMCLK_CNTL__DPSTREAMCLK0_EN__SHIFT                                                              0x3
 #define DPSTREAMCLK_CNTL__DPSTREAMCLK1_SRC_SEL__SHIFT                                                         0x4
diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
index cd3c40a86029..f9c438d16c56 100644
--- a/drivers/gpu/drm/amd/include/kgd_pp_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
@@ -421,7 +421,7 @@ struct amd_pm_funcs {
 	int (*set_hard_min_dcefclk_by_freq)(void *handle, uint32_t clock);
 	int (*set_hard_min_fclk_by_freq)(void *handle, uint32_t clock);
 	int (*set_min_deep_sleep_dcefclk)(void *handle, uint32_t clock);
-	int (*get_asic_baco_capability)(void *handle, bool *cap);
+	bool (*get_asic_baco_capability)(void *handle);
 	int (*get_asic_baco_state)(void *handle, int *state);
 	int (*set_asic_baco_state)(void *handle, int state);
 	int (*get_ppfeature_status)(void *handle, char *buf);
@@ -1127,4 +1127,19 @@ struct gpu_metrics_v3_0 {
 	/* Metrics table alpha filter time constant [us] */
 	uint32_t			time_filter_alphavalue;
 };
+
+struct amdgpu_pmmetrics_header {
+	uint16_t structure_size;
+	uint16_t pad;
+	uint32_t mp1_ip_discovery_version;
+	uint32_t pmfw_version;
+	uint32_t pmmetrics_version;
+};
+
+struct amdgpu_pm_metrics {
+	struct amdgpu_pmmetrics_header common_header;
+
+	uint8_t data[];
+};
+
 #endif
diff --git a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
index 08cb79401410..1ae3b81548fa 100644
--- a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
+++ b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
@@ -185,8 +185,7 @@ bool amdgpu_dpm_is_baco_supported(struct amdgpu_device *adev)
 {
 	const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
 	void *pp_handle = adev->powerplay.pp_handle;
-	bool baco_cap;
-	int ret = 0;
+	bool ret;
 
 	if (!pp_funcs || !pp_funcs->get_asic_baco_capability)
 		return false;
@@ -204,12 +203,11 @@ bool amdgpu_dpm_is_baco_supported(struct amdgpu_device *adev)
 
 	mutex_lock(&adev->pm.mutex);
 
-	ret = pp_funcs->get_asic_baco_capability(pp_handle,
-						 &baco_cap);
+	ret = pp_funcs->get_asic_baco_capability(pp_handle);
 
 	mutex_unlock(&adev->pm.mutex);
 
-	return ret ? false : baco_cap;
+	return ret;
 }
 
 int amdgpu_dpm_mode2_reset(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c
index 914c15387157..aed0e2cefbf9 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c
@@ -1371,21 +1371,18 @@ static int pp_set_active_display_count(void *handle, uint32_t count)
 	return phm_set_active_display_count(hwmgr, count);
 }
 
-static int pp_get_asic_baco_capability(void *handle, bool *cap)
+static bool pp_get_asic_baco_capability(void *handle)
 {
 	struct pp_hwmgr *hwmgr = handle;
 
-	*cap = false;
 	if (!hwmgr)
-		return -EINVAL;
+		return false;
 
 	if (!(hwmgr->not_vf && amdgpu_dpm) ||
 		!hwmgr->hwmgr_func->get_asic_baco_capability)
-		return 0;
+		return false;
 
-	hwmgr->hwmgr_func->get_asic_baco_capability(hwmgr, cap);
-
-	return 0;
+	return hwmgr->hwmgr_func->get_asic_baco_capability(hwmgr);
 }
 
 static int pp_get_asic_baco_state(void *handle, int *state)
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.c
index 044cda005aed..e8a9471c1898 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.c
@@ -33,21 +33,20 @@
 #include "smu/smu_7_1_2_d.h"
 #include "smu/smu_7_1_2_sh_mask.h"
 
-int smu7_baco_get_capability(struct pp_hwmgr *hwmgr, bool *cap)
+bool smu7_baco_get_capability(struct pp_hwmgr *hwmgr)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)(hwmgr->adev);
 	uint32_t reg;
 
-	*cap = false;
 	if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_BACO))
 		return 0;
 
 	reg = RREG32(mmCC_BIF_BX_FUSESTRAP0);
 
 	if (reg & CC_BIF_BX_FUSESTRAP0__STRAP_BIF_PX_CAPABLE_MASK)
-		*cap = true;
+		return true;
 
-	return 0;
+	return false;
 }
 
 int smu7_baco_get_state(struct pp_hwmgr *hwmgr, enum BACO_STATE *state)
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.h
index be0d98abb536..73a773f4ce2e 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.h
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_baco.h
@@ -25,7 +25,7 @@
 #include "hwmgr.h"
 #include "common_baco.h"
 
-extern int smu7_baco_get_capability(struct pp_hwmgr *hwmgr, bool *cap);
+extern bool smu7_baco_get_capability(struct pp_hwmgr *hwmgr);
 extern int smu7_baco_get_state(struct pp_hwmgr *hwmgr, enum BACO_STATE *state);
 extern int smu7_baco_set_state(struct pp_hwmgr *hwmgr, enum BACO_STATE state);
 
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.c
index de0a37f7c632..c66ef9741535 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.c
@@ -28,14 +28,13 @@
 #include "vega10_inc.h"
 #include "smu9_baco.h"
 
-int smu9_baco_get_capability(struct pp_hwmgr *hwmgr, bool *cap)
+bool smu9_baco_get_capability(struct pp_hwmgr *hwmgr)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)(hwmgr->adev);
 	uint32_t reg, data;
 
-	*cap = false;
 	if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_BACO))
-		return 0;
+		return false;
 
 	WREG32(0x12074, 0xFFF0003B);
 	data = RREG32(0x12075);
@@ -44,10 +43,10 @@ int smu9_baco_get_capability(struct pp_hwmgr *hwmgr, bool *cap)
 		reg = RREG32_SOC15(NBIF, 0, mmRCC_BIF_STRAP0);
 
 		if (reg & RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK)
-			*cap = true;
+			return true;
 	}
 
-	return 0;
+	return false;
 }
 
 int smu9_baco_get_state(struct pp_hwmgr *hwmgr, enum BACO_STATE *state)
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.h
index 84e90f801ac3..9ff7c2ea1b58 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.h
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu9_baco.h
@@ -25,7 +25,7 @@
 #include "hwmgr.h"
 #include "common_baco.h"
 
-extern int smu9_baco_get_capability(struct pp_hwmgr *hwmgr, bool *cap);
+extern bool smu9_baco_get_capability(struct pp_hwmgr *hwmgr);
 extern int smu9_baco_get_state(struct pp_hwmgr *hwmgr, enum BACO_STATE *state);
 
 #endif
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.c
index 994c0d374bfa..dad4c80aee58 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.c
@@ -36,23 +36,22 @@ static const struct soc15_baco_cmd_entry clean_baco_tbl[] = {
 	{CMD_WRITE, SOC15_REG_ENTRY(NBIF, 0, mmBIOS_SCRATCH_7), 0, 0, 0, 0},
 };
 
-int vega20_baco_get_capability(struct pp_hwmgr *hwmgr, bool *cap)
+bool vega20_baco_get_capability(struct pp_hwmgr *hwmgr)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)(hwmgr->adev);
 	uint32_t reg;
 
-	*cap = false;
 	if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_BACO))
-		return 0;
+		return false;
 
 	if (((RREG32(0x17569) & 0x20000000) >> 29) == 0x1) {
 		reg = RREG32_SOC15(NBIF, 0, mmRCC_BIF_STRAP0);
 
 		if (reg & RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK)
-			*cap = true;
+			return true;
 	}
 
-	return 0;
+	return false;
 }
 
 int vega20_baco_get_state(struct pp_hwmgr *hwmgr, enum BACO_STATE *state)
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.h
index f06471e712dc..bdad9c915631 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.h
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.h
@@ -25,7 +25,7 @@
 #include "hwmgr.h"
 #include "common_baco.h"
 
-extern int vega20_baco_get_capability(struct pp_hwmgr *hwmgr, bool *cap);
+extern bool vega20_baco_get_capability(struct pp_hwmgr *hwmgr);
 extern int vega20_baco_get_state(struct pp_hwmgr *hwmgr, enum BACO_STATE *state);
 extern int vega20_baco_set_state(struct pp_hwmgr *hwmgr, enum BACO_STATE state);
 extern int vega20_baco_apply_vdci_flush_workaround(struct pp_hwmgr *hwmgr);
diff --git a/drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h b/drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h
index 81650727a5de..6f536159df4d 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h
+++ b/drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h
@@ -351,7 +351,7 @@ struct pp_hwmgr_func {
 	int (*set_hard_min_fclk_by_freq)(struct pp_hwmgr *hwmgr, uint32_t clock);
 	int (*set_hard_min_gfxclk_by_freq)(struct pp_hwmgr *hwmgr, uint32_t clock);
 	int (*set_soft_max_gfxclk_by_freq)(struct pp_hwmgr *hwmgr, uint32_t clock);
-	int (*get_asic_baco_capability)(struct pp_hwmgr *hwmgr, bool *cap);
+	bool (*get_asic_baco_capability)(struct pp_hwmgr *hwmgr);
 	int (*get_asic_baco_state)(struct pp_hwmgr *hwmgr, enum BACO_STATE *state);
 	int (*set_asic_baco_state)(struct pp_hwmgr *hwmgr, enum BACO_STATE state);
 	int (*get_ppfeature_status)(struct pp_hwmgr *hwmgr, char *buf);
diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
index 1ead323f1c78..37c2605f9b35 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
@@ -3005,19 +3005,17 @@ static int smu_set_xgmi_pstate(void *handle,
 	return ret;
 }
 
-static int smu_get_baco_capability(void *handle, bool *cap)
+static bool smu_get_baco_capability(void *handle)
 {
 	struct smu_context *smu = handle;
 
-	*cap = false;
-
 	if (!smu->pm_enabled)
-		return 0;
+		return false;
 
-	if (smu->ppt_funcs && smu->ppt_funcs->baco_is_support)
-		*cap = smu->ppt_funcs->baco_is_support(smu);
+	if (!smu->ppt_funcs || !smu->ppt_funcs->baco_is_support)
+		return false;
 
-	return 0;
+	return smu->ppt_funcs->baco_is_support(smu);
 }
 
 static int smu_baco_set_state(void *handle, int state)
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h
index 95cb919718ae..48f5926d8153 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h
@@ -210,9 +210,6 @@ int smu_v13_0_set_azalia_d3_pme(struct smu_context *smu);
 int smu_v13_0_get_max_sustainable_clocks_by_dc(struct smu_context *smu,
 					       struct pp_smu_nv_clock_table *max_clocks);
 
-int smu_v13_0_baco_set_armd3_sequence(struct smu_context *smu,
-				      enum smu_baco_seq baco_seq);
-
 bool smu_v13_0_baco_is_support(struct smu_context *smu);
 
 enum smu_baco_state smu_v13_0_baco_get_state(struct smu_context *smu);
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
index cf1b84060bc3..86fc7273d588 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
@@ -2199,7 +2199,7 @@ int smu_v13_0_gfx_ulv_control(struct smu_context *smu,
 	return ret;
 }
 
-int smu_v13_0_baco_set_armd3_sequence(struct smu_context *smu,
+static int smu_v13_0_baco_set_armd3_sequence(struct smu_context *smu,
 				      enum smu_baco_seq baco_seq)
 {
 	struct smu_baco_context *smu_baco = &smu->smu_baco;
@@ -2283,22 +2283,40 @@ int smu_v13_0_baco_set_state(struct smu_context *smu,
 
 int smu_v13_0_baco_enter(struct smu_context *smu)
 {
-	int ret = 0;
-
-	ret = smu_v13_0_baco_set_state(smu,
-				       SMU_BACO_STATE_ENTER);
-	if (ret)
-		return ret;
+	struct smu_baco_context *smu_baco = &smu->smu_baco;
+	struct amdgpu_device *adev = smu->adev;
+	int ret;
 
-	msleep(10);
+	if (adev->in_runpm && smu_cmn_is_audio_func_enabled(adev)) {
+		return smu_v13_0_baco_set_armd3_sequence(smu,
+				(smu_baco->maco_support && amdgpu_runtime_pm != 1) ?
+					BACO_SEQ_BAMACO : BACO_SEQ_BACO);
+	} else {
+		ret = smu_v13_0_baco_set_state(smu, SMU_BACO_STATE_ENTER);
+		if (!ret)
+			usleep_range(10000, 11000);
 
-	return ret;
+		return ret;
+	}
 }
 
 int smu_v13_0_baco_exit(struct smu_context *smu)
 {
-	return smu_v13_0_baco_set_state(smu,
-					SMU_BACO_STATE_EXIT);
+	struct amdgpu_device *adev = smu->adev;
+	int ret;
+
+	if (adev->in_runpm && smu_cmn_is_audio_func_enabled(adev)) {
+		/* Wait for PMFW handling for the Dstate change */
+		usleep_range(10000, 11000);
+		ret = smu_v13_0_baco_set_armd3_sequence(smu, BACO_SEQ_ULPS);
+	} else {
+		ret = smu_v13_0_baco_set_state(smu, SMU_BACO_STATE_EXIT);
+	}
+
+	if (!ret)
+		adev->gfx.is_poweron = false;
+
+	return ret;
 }
 
 int smu_v13_0_set_gfx_power_up_by_imu(struct smu_context *smu)
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
index 82c4e1f1c6f0..a4debd7ab8bd 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
@@ -2558,38 +2558,6 @@ static int smu_v13_0_0_set_power_profile_mode(struct smu_context *smu,
 					       NULL);
 }
 
-static int smu_v13_0_0_baco_enter(struct smu_context *smu)
-{
-	struct smu_baco_context *smu_baco = &smu->smu_baco;
-	struct amdgpu_device *adev = smu->adev;
-
-	if (adev->in_runpm && smu_cmn_is_audio_func_enabled(adev))
-		return smu_v13_0_baco_set_armd3_sequence(smu,
-				(smu_baco->maco_support && amdgpu_runtime_pm != 1) ?
-					BACO_SEQ_BAMACO : BACO_SEQ_BACO);
-	else
-		return smu_v13_0_baco_enter(smu);
-}
-
-static int smu_v13_0_0_baco_exit(struct smu_context *smu)
-{
-	struct amdgpu_device *adev = smu->adev;
-	int ret;
-
-	if (adev->in_runpm && smu_cmn_is_audio_func_enabled(adev)) {
-		/* Wait for PMFW handling for the Dstate change */
-		usleep_range(10000, 11000);
-		ret = smu_v13_0_baco_set_armd3_sequence(smu, BACO_SEQ_ULPS);
-	} else {
-		ret = smu_v13_0_baco_exit(smu);
-	}
-
-	if (!ret)
-		adev->gfx.is_poweron = false;
-
-	return ret;
-}
-
 static bool smu_v13_0_0_is_mode1_reset_supported(struct smu_context *smu)
 {
 	struct amdgpu_device *adev = smu->adev;
@@ -3037,8 +3005,8 @@ static const struct pptable_funcs smu_v13_0_0_ppt_funcs = {
 	.baco_is_support = smu_v13_0_baco_is_support,
 	.baco_get_state = smu_v13_0_baco_get_state,
 	.baco_set_state = smu_v13_0_baco_set_state,
-	.baco_enter = smu_v13_0_0_baco_enter,
-	.baco_exit = smu_v13_0_0_baco_exit,
+	.baco_enter = smu_v13_0_baco_enter,
+	.baco_exit = smu_v13_0_baco_exit,
 	.mode1_reset_is_support = smu_v13_0_0_is_mode1_reset_supported,
 	.mode1_reset = smu_v13_0_0_mode1_reset,
 	.mode2_reset = smu_v13_0_0_mode2_reset,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
index 0e5a77c3c2e2..f723a4190ee5 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
@@ -2392,8 +2392,8 @@ static const struct mca_bank_ipid smu_v13_0_6_mca_ipid_table[AMDGPU_MCA_IP_COUNT
 
 static void mca_bank_entry_info_decode(struct mca_bank_entry *entry, struct mca_bank_info *info)
 {
-	uint64_t ipid = entry->regs[MCA_REG_IDX_IPID];
-	uint32_t insthi;
+	u64 ipid = entry->regs[MCA_REG_IDX_IPID];
+	u32 instidhi, instid;
 
 	/* NOTE: All MCA IPID register share the same format,
 	 * so the driver can share the MCMP1 register header file.
@@ -2402,9 +2402,15 @@ static void mca_bank_entry_info_decode(struct mca_bank_entry *entry, struct mca_
 	info->hwid = REG_GET_FIELD(ipid, MCMP1_IPIDT0, HardwareID);
 	info->mcatype = REG_GET_FIELD(ipid, MCMP1_IPIDT0, McaType);
 
-	insthi = REG_GET_FIELD(ipid, MCMP1_IPIDT0, InstanceIdHi);
-	info->aid = ((insthi >> 2) & 0x03);
-	info->socket_id = insthi & 0x03;
+	/*
+	 * Unfied DieID Format: SAASS. A:AID, S:Socket.
+	 * Unfied DieID[4] = InstanceId[0]
+	 * Unfied DieID[0:3] = InstanceIdHi[0:3]
+	 */
+	instidhi = REG_GET_FIELD(ipid, MCMP1_IPIDT0, InstanceIdHi);
+	instid = REG_GET_FIELD(ipid, MCMP1_IPIDT0, InstanceIdLo);
+	info->aid = ((instidhi >> 2) & 0x03);
+	info->socket_id = ((instid & 0x1) << 2) | (instidhi & 0x03);
 }
 
 static int mca_bank_read_reg(struct amdgpu_device *adev, enum amdgpu_mca_error_type type,
@@ -2578,6 +2584,7 @@ static bool mca_gfx_smu_bank_is_valid(const struct mca_ras_info *mca_ras, struct
 	uint32_t instlo;
 
 	instlo = REG_GET_FIELD(entry->regs[MCA_REG_IDX_IPID], MCMP1_IPIDT0, InstanceIdLo);
+	instlo &= GENMASK(31, 1);
 	switch (instlo) {
 	case 0x36430400: /* SMNAID XCD 0 */
 	case 0x38430400: /* SMNAID XCD 1 */
@@ -2596,6 +2603,7 @@ static bool mca_smu_bank_is_valid(const struct mca_ras_info *mca_ras, struct amd
 	uint32_t errcode, instlo;
 
 	instlo = REG_GET_FIELD(entry->regs[MCA_REG_IDX_IPID], MCMP1_IPIDT0, InstanceIdLo);
+	instlo &= GENMASK(31, 1);
 	if (instlo != 0x03b30400)
 		return false;
 
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
index 81eafed76045..f5596f031d00 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c
@@ -2515,38 +2515,6 @@ static int smu_v13_0_7_set_mp1_state(struct smu_context *smu,
 	return ret;
 }
 
-static int smu_v13_0_7_baco_enter(struct smu_context *smu)
-{
-	struct smu_baco_context *smu_baco = &smu->smu_baco;
-	struct amdgpu_device *adev = smu->adev;
-
-	if (adev->in_runpm && smu_cmn_is_audio_func_enabled(adev))
-		return smu_v13_0_baco_set_armd3_sequence(smu,
-				(smu_baco->maco_support && amdgpu_runtime_pm != 1) ?
-					BACO_SEQ_BAMACO : BACO_SEQ_BACO);
-	else
-		return smu_v13_0_baco_enter(smu);
-}
-
-static int smu_v13_0_7_baco_exit(struct smu_context *smu)
-{
-	struct amdgpu_device *adev = smu->adev;
-	int ret;
-
-	if (adev->in_runpm && smu_cmn_is_audio_func_enabled(adev)) {
-		/* Wait for PMFW handling for the Dstate change */
-		usleep_range(10000, 11000);
-		ret = smu_v13_0_baco_set_armd3_sequence(smu, BACO_SEQ_ULPS);
-	} else {
-		ret = smu_v13_0_baco_exit(smu);
-	}
-
-	if (!ret)
-		adev->gfx.is_poweron = false;
-
-	return ret;
-}
-
 static bool smu_v13_0_7_is_mode1_reset_supported(struct smu_context *smu)
 {
 	struct amdgpu_device *adev = smu->adev;
@@ -2628,8 +2596,8 @@ static const struct pptable_funcs smu_v13_0_7_ppt_funcs = {
 	.baco_is_support = smu_v13_0_baco_is_support,
 	.baco_get_state = smu_v13_0_baco_get_state,
 	.baco_set_state = smu_v13_0_baco_set_state,
-	.baco_enter = smu_v13_0_7_baco_enter,
-	.baco_exit = smu_v13_0_7_baco_exit,
+	.baco_enter = smu_v13_0_baco_enter,
+	.baco_exit = smu_v13_0_baco_exit,
 	.mode1_reset_is_support = smu_v13_0_7_is_mode1_reset_supported,
 	.mode1_reset = smu_v13_0_mode1_reset,
 	.set_mp1_state = smu_v13_0_7_set_mp1_state,
diff --git a/drivers/gpu/drm/armada/armada_crtc.c b/drivers/gpu/drm/armada/armada_crtc.c
index 15dd667aa2e7..52d2c942d3d2 100644
--- a/drivers/gpu/drm/armada/armada_crtc.c
+++ b/drivers/gpu/drm/armada/armada_crtc.c
@@ -1066,10 +1066,9 @@ static int armada_lcd_probe(struct platform_device *pdev)
 	return component_add(&pdev->dev, &armada_lcd_ops);
 }
 
-static int armada_lcd_remove(struct platform_device *pdev)
+static void armada_lcd_remove(struct platform_device *pdev)
 {
 	component_del(&pdev->dev, &armada_lcd_ops);
-	return 0;
 }
 
 static const struct of_device_id armada_lcd_of_match[] = {
@@ -1095,7 +1094,7 @@ MODULE_DEVICE_TABLE(platform, armada_lcd_platform_ids);
 
 struct platform_driver armada_lcd_platform_driver = {
 	.probe	= armada_lcd_probe,
-	.remove	= armada_lcd_remove,
+	.remove_new = armada_lcd_remove,
 	.driver = {
 		.name	= "armada-lcd",
 		.owner	=  THIS_MODULE,
diff --git a/drivers/gpu/drm/armada/armada_drv.c b/drivers/gpu/drm/armada/armada_drv.c
index fa1c67598706..e51ecc4f7ef4 100644
--- a/drivers/gpu/drm/armada/armada_drv.c
+++ b/drivers/gpu/drm/armada/armada_drv.c
@@ -226,10 +226,9 @@ static int armada_drm_probe(struct platform_device *pdev)
 					       match);
 }
 
-static int armada_drm_remove(struct platform_device *pdev)
+static void armada_drm_remove(struct platform_device *pdev)
 {
 	component_master_del(&pdev->dev, &armada_master_ops);
-	return 0;
 }
 
 static void armada_drm_shutdown(struct platform_device *pdev)
@@ -249,7 +248,7 @@ MODULE_DEVICE_TABLE(platform, armada_drm_platform_ids);
 
 static struct platform_driver armada_drm_platform_driver = {
 	.probe	= armada_drm_probe,
-	.remove	= armada_drm_remove,
+	.remove_new = armada_drm_remove,
 	.shutdown = armada_drm_shutdown,
 	.driver	= {
 		.name	= "armada-drm",
diff --git a/drivers/gpu/drm/ast/ast_drv.h b/drivers/gpu/drm/ast/ast_drv.h
index 2aee32344f4a..772f3b049c16 100644
--- a/drivers/gpu/drm/ast/ast_drv.h
+++ b/drivers/gpu/drm/ast/ast_drv.h
@@ -174,6 +174,17 @@ to_ast_sil164_connector(struct drm_connector *connector)
 	return container_of(connector, struct ast_sil164_connector, base);
 }
 
+struct ast_bmc_connector {
+	struct drm_connector base;
+	struct drm_connector *physical_connector;
+};
+
+static inline struct ast_bmc_connector *
+to_ast_bmc_connector(struct drm_connector *connector)
+{
+	return container_of(connector, struct ast_bmc_connector, base);
+}
+
 /*
  * Device
  */
@@ -218,7 +229,7 @@ struct ast_device {
 		} astdp;
 		struct {
 			struct drm_encoder encoder;
-			struct drm_connector connector;
+			struct ast_bmc_connector bmc_connector;
 		} bmc;
 	} output;
 
diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c
index cb9614984285..c20534d0ef7c 100644
--- a/drivers/gpu/drm/ast/ast_mode.c
+++ b/drivers/gpu/drm/ast/ast_mode.c
@@ -1767,6 +1767,30 @@ static const struct drm_encoder_funcs ast_bmc_encoder_funcs = {
 	.destroy = drm_encoder_cleanup,
 };
 
+static int ast_bmc_connector_helper_detect_ctx(struct drm_connector *connector,
+					       struct drm_modeset_acquire_ctx *ctx,
+					       bool force)
+{
+	struct ast_bmc_connector *bmc_connector = to_ast_bmc_connector(connector);
+	struct drm_connector *physical_connector = bmc_connector->physical_connector;
+
+	/*
+	 * Most user-space compositors cannot handle more than one connected
+	 * connector per CRTC. Hence, we only mark the BMC as connected if the
+	 * physical connector is disconnected. If the physical connector's status
+	 * is connected or unknown, the BMC remains disconnected. This has no
+	 * effect on the output of the BMC.
+	 *
+	 * FIXME: Remove this logic once user-space compositors can handle more
+	 *        than one connector per CRTC. The BMC should always be connected.
+	 */
+
+	if (physical_connector && physical_connector->status == connector_status_disconnected)
+		return connector_status_connected;
+
+	return connector_status_disconnected;
+}
+
 static int ast_bmc_connector_helper_get_modes(struct drm_connector *connector)
 {
 	return drm_add_modes_noedid(connector, 4096, 4096);
@@ -1774,6 +1798,7 @@ static int ast_bmc_connector_helper_get_modes(struct drm_connector *connector)
 
 static const struct drm_connector_helper_funcs ast_bmc_connector_helper_funcs = {
 	.get_modes = ast_bmc_connector_helper_get_modes,
+	.detect_ctx = ast_bmc_connector_helper_detect_ctx,
 };
 
 static const struct drm_connector_funcs ast_bmc_connector_funcs = {
@@ -1784,12 +1809,33 @@ static const struct drm_connector_funcs ast_bmc_connector_funcs = {
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 };
 
-static int ast_bmc_output_init(struct ast_device *ast)
+static int ast_bmc_connector_init(struct drm_device *dev,
+				  struct ast_bmc_connector *bmc_connector,
+				  struct drm_connector *physical_connector)
+{
+	struct drm_connector *connector = &bmc_connector->base;
+	int ret;
+
+	ret = drm_connector_init(dev, connector, &ast_bmc_connector_funcs,
+				 DRM_MODE_CONNECTOR_VIRTUAL);
+	if (ret)
+		return ret;
+
+	drm_connector_helper_add(connector, &ast_bmc_connector_helper_funcs);
+
+	bmc_connector->physical_connector = physical_connector;
+
+	return 0;
+}
+
+static int ast_bmc_output_init(struct ast_device *ast,
+			       struct drm_connector *physical_connector)
 {
 	struct drm_device *dev = &ast->base;
 	struct drm_crtc *crtc = &ast->crtc;
 	struct drm_encoder *encoder = &ast->output.bmc.encoder;
-	struct drm_connector *connector = &ast->output.bmc.connector;
+	struct ast_bmc_connector *bmc_connector = &ast->output.bmc.bmc_connector;
+	struct drm_connector *connector = &bmc_connector->base;
 	int ret;
 
 	ret = drm_encoder_init(dev, encoder,
@@ -1799,13 +1845,10 @@ static int ast_bmc_output_init(struct ast_device *ast)
 		return ret;
 	encoder->possible_crtcs = drm_crtc_mask(crtc);
 
-	ret = drm_connector_init(dev, connector, &ast_bmc_connector_funcs,
-				 DRM_MODE_CONNECTOR_VIRTUAL);
+	ret = ast_bmc_connector_init(dev, bmc_connector, physical_connector);
 	if (ret)
 		return ret;
 
-	drm_connector_helper_add(connector, &ast_bmc_connector_helper_funcs);
-
 	ret = drm_connector_attach_encoder(connector, encoder);
 	if (ret)
 		return ret;
@@ -1864,6 +1907,7 @@ static const struct drm_mode_config_funcs ast_mode_config_funcs = {
 int ast_mode_config_init(struct ast_device *ast)
 {
 	struct drm_device *dev = &ast->base;
+	struct drm_connector *physical_connector = NULL;
 	int ret;
 
 	ret = drmm_mode_config_init(dev);
@@ -1904,23 +1948,27 @@ int ast_mode_config_init(struct ast_device *ast)
 		ret = ast_vga_output_init(ast);
 		if (ret)
 			return ret;
+		physical_connector = &ast->output.vga.vga_connector.base;
 	}
 	if (ast->tx_chip_types & AST_TX_SIL164_BIT) {
 		ret = ast_sil164_output_init(ast);
 		if (ret)
 			return ret;
+		physical_connector = &ast->output.sil164.sil164_connector.base;
 	}
 	if (ast->tx_chip_types & AST_TX_DP501_BIT) {
 		ret = ast_dp501_output_init(ast);
 		if (ret)
 			return ret;
+		physical_connector = &ast->output.dp501.connector;
 	}
 	if (ast->tx_chip_types & AST_TX_ASTDP_BIT) {
 		ret = ast_astdp_output_init(ast);
 		if (ret)
 			return ret;
+		physical_connector = &ast->output.astdp.connector;
 	}
-	ret = ast_bmc_output_init(ast);
+	ret = ast_bmc_output_init(ast, physical_connector);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c
index 6af565ac307a..7d470527455b 100644
--- a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c
+++ b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c
@@ -2596,11 +2596,10 @@ static int cdns_mhdp_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int cdns_mhdp_remove(struct platform_device *pdev)
+static void cdns_mhdp_remove(struct platform_device *pdev)
 {
 	struct cdns_mhdp_device *mhdp = platform_get_drvdata(pdev);
 	unsigned long timeout = msecs_to_jiffies(100);
-	bool stop_fw = false;
 	int ret;
 
 	drm_bridge_remove(&mhdp->bridge);
@@ -2608,18 +2607,19 @@ static int cdns_mhdp_remove(struct platform_device *pdev)
 	ret = wait_event_timeout(mhdp->fw_load_wq,
 				 mhdp->hw_state == MHDP_HW_READY,
 				 timeout);
-	if (ret == 0)
-		dev_err(mhdp->dev, "%s: Timeout waiting for fw loading\n",
-			__func__);
-	else
-		stop_fw = true;
-
 	spin_lock(&mhdp->start_lock);
 	mhdp->hw_state = MHDP_HW_STOPPED;
 	spin_unlock(&mhdp->start_lock);
 
-	if (stop_fw)
+	if (ret == 0) {
+		dev_err(mhdp->dev, "%s: Timeout waiting for fw loading\n",
+			__func__);
+	} else {
 		ret = cdns_mhdp_set_firmware_active(mhdp, false);
+		if (ret)
+			dev_err(mhdp->dev, "Failed to stop firmware (%pe)\n",
+				ERR_PTR(ret));
+	}
 
 	phy_exit(mhdp->phy);
 
@@ -2634,8 +2634,6 @@ static int cdns_mhdp_remove(struct platform_device *pdev)
 	/* Ignoring mhdp->hdcp.check_work and mhdp->hdcp.prop_work here. */
 
 	clk_disable_unprepare(mhdp->clk);
-
-	return ret;
 }
 
 static const struct of_device_id mhdp_ids[] = {
@@ -2658,7 +2656,7 @@ static struct platform_driver mhdp_driver = {
 		.of_match_table	= mhdp_ids,
 	},
 	.probe	= cdns_mhdp_probe,
-	.remove	= cdns_mhdp_remove,
+	.remove_new = cdns_mhdp_remove,
 };
 module_platform_driver(mhdp_driver);
 
diff --git a/drivers/gpu/drm/bridge/ti-tpd12s015.c b/drivers/gpu/drm/bridge/ti-tpd12s015.c
index e0e015243a60..f9fb35683a27 100644
--- a/drivers/gpu/drm/bridge/ti-tpd12s015.c
+++ b/drivers/gpu/drm/bridge/ti-tpd12s015.c
@@ -179,13 +179,11 @@ static int tpd12s015_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int __exit tpd12s015_remove(struct platform_device *pdev)
+static void tpd12s015_remove(struct platform_device *pdev)
 {
 	struct tpd12s015_device *tpd = platform_get_drvdata(pdev);
 
 	drm_bridge_remove(&tpd->bridge);
-
-	return 0;
 }
 
 static const struct of_device_id tpd12s015_of_match[] = {
@@ -197,7 +195,7 @@ MODULE_DEVICE_TABLE(of, tpd12s015_of_match);
 
 static struct platform_driver tpd12s015_driver = {
 	.probe	= tpd12s015_probe,
-	.remove	= __exit_p(tpd12s015_remove),
+	.remove_new = tpd12s015_remove,
 	.driver	= {
 		.name	= "tpd12s015",
 		.of_match_table = tpd12s015_of_match,
diff --git a/drivers/gpu/drm/display/drm_dp_helper.c b/drivers/gpu/drm/display/drm_dp_helper.c
index f3680f4e6970..d72b6f9a352c 100644
--- a/drivers/gpu/drm/display/drm_dp_helper.c
+++ b/drivers/gpu/drm/display/drm_dp_helper.c
@@ -2245,6 +2245,8 @@ static const struct dpcd_quirk dpcd_quirk_list[] = {
 	{ OUI(0x00, 0x00, 0x00), DEVICE_ID('C', 'H', '7', '5', '1', '1'), false, BIT(DP_DPCD_QUIRK_NO_SINK_COUNT) },
 	/* Synaptics DP1.4 MST hubs can support DSC without virtual DPCD */
 	{ OUI(0x90, 0xCC, 0x24), DEVICE_ID_ANY, true, BIT(DP_DPCD_QUIRK_DSC_WITHOUT_VIRTUAL_DPCD) },
+	/* Synaptics DP1.4 MST hubs require DSC for some modes on which it applies HBLANK expansion. */
+	{ OUI(0x90, 0xCC, 0x24), DEVICE_ID_ANY, true, BIT(DP_DPCD_QUIRK_HBLANK_EXPANSION_REQUIRES_DSC) },
 	/* Apple MacBookPro 2017 15 inch eDP Retina panel reports too low DP_MAX_LINK_RATE */
 	{ OUI(0x00, 0x10, 0xfa), DEVICE_ID(101, 68, 21, 101, 98, 97), false, BIT(DP_DPCD_QUIRK_CAN_DO_MAX_LINK_RATE_3_24_GBPS) },
 };
@@ -2326,6 +2328,33 @@ int drm_dp_read_desc(struct drm_dp_aux *aux, struct drm_dp_desc *desc,
 }
 EXPORT_SYMBOL(drm_dp_read_desc);
 
+/**
+ * drm_dp_dsc_sink_bpp_incr() - Get bits per pixel increment
+ * @dsc_dpcd: DSC capabilities from DPCD
+ *
+ * Returns the bpp precision supported by the DP sink.
+ */
+u8 drm_dp_dsc_sink_bpp_incr(const u8 dsc_dpcd[DP_DSC_RECEIVER_CAP_SIZE])
+{
+	u8 bpp_increment_dpcd = dsc_dpcd[DP_DSC_BITS_PER_PIXEL_INC - DP_DSC_SUPPORT];
+
+	switch (bpp_increment_dpcd) {
+	case DP_DSC_BITS_PER_PIXEL_1_16:
+		return 16;
+	case DP_DSC_BITS_PER_PIXEL_1_8:
+		return 8;
+	case DP_DSC_BITS_PER_PIXEL_1_4:
+		return 4;
+	case DP_DSC_BITS_PER_PIXEL_1_2:
+		return 2;
+	case DP_DSC_BITS_PER_PIXEL_1_1:
+		return 1;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_dp_dsc_sink_bpp_incr);
+
 /**
  * drm_dp_dsc_sink_max_slice_count() - Get the max slice count
  * supported by the DSC sink.
@@ -3898,3 +3927,135 @@ int drm_panel_dp_aux_backlight(struct drm_panel *panel, struct drm_dp_aux *aux)
 EXPORT_SYMBOL(drm_panel_dp_aux_backlight);
 
 #endif
+
+/* See DP Standard v2.1 2.6.4.4.1.1, 2.8.4.4, 2.8.7 */
+static int drm_dp_link_symbol_cycles(int lane_count, int pixels, int bpp_x16,
+				     int symbol_size, bool is_mst)
+{
+	int cycles = DIV_ROUND_UP(pixels * bpp_x16, 16 * symbol_size * lane_count);
+	int align = is_mst ? 4 / lane_count : 1;
+
+	return ALIGN(cycles, align);
+}
+
+static int drm_dp_link_dsc_symbol_cycles(int lane_count, int pixels, int slice_count,
+					 int bpp_x16, int symbol_size, bool is_mst)
+{
+	int slice_pixels = DIV_ROUND_UP(pixels, slice_count);
+	int slice_data_cycles = drm_dp_link_symbol_cycles(lane_count, slice_pixels,
+							  bpp_x16, symbol_size, is_mst);
+	int slice_eoc_cycles = is_mst ? 4 / lane_count : 1;
+
+	return slice_count * (slice_data_cycles + slice_eoc_cycles);
+}
+
+/**
+ * drm_dp_bw_overhead - Calculate the BW overhead of a DP link stream
+ * @lane_count: DP link lane count
+ * @hactive: pixel count of the active period in one scanline of the stream
+ * @dsc_slice_count: DSC slice count if @flags/DRM_DP_LINK_BW_OVERHEAD_DSC is set
+ * @bpp_x16: bits per pixel in .4 binary fixed point
+ * @flags: DRM_DP_OVERHEAD_x flags
+ *
+ * Calculate the BW allocation overhead of a DP link stream, depending
+ * on the link's
+ * - @lane_count
+ * - SST/MST mode (@flags / %DRM_DP_OVERHEAD_MST)
+ * - symbol size (@flags / %DRM_DP_OVERHEAD_UHBR)
+ * - FEC mode (@flags / %DRM_DP_OVERHEAD_FEC)
+ * - SSC/REF_CLK mode (@flags / %DRM_DP_OVERHEAD_SSC_REF_CLK)
+ * as well as the stream's
+ * - @hactive timing
+ * - @bpp_x16 color depth
+ * - compression mode (@flags / %DRM_DP_OVERHEAD_DSC).
+ * Note that this overhead doesn't account for the 8b/10b, 128b/132b
+ * channel coding efficiency, for that see
+ * @drm_dp_link_bw_channel_coding_efficiency().
+ *
+ * Returns the overhead as 100% + overhead% in 1ppm units.
+ */
+int drm_dp_bw_overhead(int lane_count, int hactive,
+		       int dsc_slice_count,
+		       int bpp_x16, unsigned long flags)
+{
+	int symbol_size = flags & DRM_DP_BW_OVERHEAD_UHBR ? 32 : 8;
+	bool is_mst = flags & DRM_DP_BW_OVERHEAD_MST;
+	u32 overhead = 1000000;
+	int symbol_cycles;
+
+	/*
+	 * DP Standard v2.1 2.6.4.1
+	 * SSC downspread and ref clock variation margin:
+	 *   5300ppm + 300ppm ~ 0.6%
+	 */
+	if (flags & DRM_DP_BW_OVERHEAD_SSC_REF_CLK)
+		overhead += 6000;
+
+	/*
+	 * DP Standard v2.1 2.6.4.1.1, 3.5.1.5.4:
+	 * FEC symbol insertions for 8b/10b channel coding:
+	 * After each 250 data symbols on 2-4 lanes:
+	 *   250 LL + 5 FEC_PARITY_PH + 1 CD_ADJ   (256 byte FEC block)
+	 * After each 2 x 250 data symbols on 1 lane:
+	 *   2 * 250 LL + 11 FEC_PARITY_PH + 1 CD_ADJ (512 byte FEC block)
+	 * After 256 (2-4 lanes) or 128 (1 lane) FEC blocks:
+	 *   256 * 256 bytes + 1 FEC_PM
+	 * or
+	 *   128 * 512 bytes + 1 FEC_PM
+	 * (256 * 6 + 1) / (256 * 250) = 2.4015625 %
+	 */
+	if (flags & DRM_DP_BW_OVERHEAD_FEC)
+		overhead += 24016;
+
+	/*
+	 * DP Standard v2.1 2.7.9, 5.9.7
+	 * The FEC overhead for UHBR is accounted for in its 96.71% channel
+	 * coding efficiency.
+	 */
+	WARN_ON((flags & DRM_DP_BW_OVERHEAD_UHBR) &&
+		(flags & DRM_DP_BW_OVERHEAD_FEC));
+
+	if (flags & DRM_DP_BW_OVERHEAD_DSC)
+		symbol_cycles = drm_dp_link_dsc_symbol_cycles(lane_count, hactive,
+							      dsc_slice_count,
+							      bpp_x16, symbol_size,
+							      is_mst);
+	else
+		symbol_cycles = drm_dp_link_symbol_cycles(lane_count, hactive,
+							  bpp_x16, symbol_size,
+							  is_mst);
+
+	return DIV_ROUND_UP_ULL(mul_u32_u32(symbol_cycles * symbol_size * lane_count,
+					    overhead * 16),
+				hactive * bpp_x16);
+}
+EXPORT_SYMBOL(drm_dp_bw_overhead);
+
+/**
+ * drm_dp_bw_channel_coding_efficiency - Get a DP link's channel coding efficiency
+ * @is_uhbr: Whether the link has a 128b/132b channel coding
+ *
+ * Return the channel coding efficiency of the given DP link type, which is
+ * either 8b/10b or 128b/132b (aka UHBR). The corresponding overhead includes
+ * the 8b -> 10b, 128b -> 132b pixel data to link symbol conversion overhead
+ * and for 128b/132b any link or PHY level control symbol insertion overhead
+ * (LLCP, FEC, PHY sync, see DP Standard v2.1 3.5.2.18). For 8b/10b the
+ * corresponding FEC overhead is BW allocation specific, included in the value
+ * returned by drm_dp_bw_overhead().
+ *
+ * Returns the efficiency in the 100%/coding-overhead% ratio in
+ * 1ppm units.
+ */
+int drm_dp_bw_channel_coding_efficiency(bool is_uhbr)
+{
+	if (is_uhbr)
+		return 967100;
+	else
+		/*
+		 * Note that on 8b/10b MST the efficiency is only
+		 * 78.75% due to the 1 out of 64 MTPH packet overhead,
+		 * not accounted for here.
+		 */
+		return 800000;
+}
+EXPORT_SYMBOL(drm_dp_bw_channel_coding_efficiency);
diff --git a/drivers/gpu/drm/display/drm_dp_mst_topology.c b/drivers/gpu/drm/display/drm_dp_mst_topology.c
index 0e0d0e76de06..8ca01a6bf645 100644
--- a/drivers/gpu/drm/display/drm_dp_mst_topology.c
+++ b/drivers/gpu/drm/display/drm_dp_mst_topology.c
@@ -43,6 +43,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_fixed.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 
@@ -3578,16 +3579,26 @@ static int drm_dp_send_up_ack_reply(struct drm_dp_mst_topology_mgr *mgr,
  * value is in units of PBNs/(timeslots/1 MTP). This value can be used to
  * convert the number of PBNs required for a given stream to the number of
  * timeslots this stream requires in each MTP.
+ *
+ * Returns the BW / timeslot value in 20.12 fixed point format.
  */
-int drm_dp_get_vc_payload_bw(const struct drm_dp_mst_topology_mgr *mgr,
-			     int link_rate, int link_lane_count)
+fixed20_12 drm_dp_get_vc_payload_bw(const struct drm_dp_mst_topology_mgr *mgr,
+				    int link_rate, int link_lane_count)
 {
+	int ch_coding_efficiency =
+		drm_dp_bw_channel_coding_efficiency(drm_dp_is_uhbr_rate(link_rate));
+	fixed20_12 ret;
+
 	if (link_rate == 0 || link_lane_count == 0)
 		drm_dbg_kms(mgr->dev, "invalid link rate/lane count: (%d / %d)\n",
 			    link_rate, link_lane_count);
 
-	/* See DP v2.0 2.6.4.2, VCPayload_Bandwidth_for_OneTimeSlotPer_MTP_Allocation */
-	return link_rate * link_lane_count / 54000;
+	/* See DP v2.0 2.6.4.2, 2.7.6.3 VCPayload_Bandwidth_for_OneTimeSlotPer_MTP_Allocation */
+	ret.full = DIV_ROUND_DOWN_ULL(mul_u32_u32(link_rate * link_lane_count,
+						  ch_coding_efficiency),
+				      (1000000ULL * 8 * 5400) >> 12);
+
+	return ret;
 }
 EXPORT_SYMBOL(drm_dp_get_vc_payload_bw);
 
@@ -4335,7 +4346,7 @@ int drm_dp_atomic_find_time_slots(struct drm_atomic_state *state,
 		}
 	}
 
-	req_slots = DIV_ROUND_UP(pbn, topology_state->pbn_div);
+	req_slots = DIV_ROUND_UP(dfixed_const(pbn), topology_state->pbn_div.full);
 
 	drm_dbg_atomic(mgr->dev, "[CONNECTOR:%d:%s] [MST PORT:%p] TU %d -> %d\n",
 		       port->connector->base.id, port->connector->name,
@@ -4718,35 +4729,36 @@ EXPORT_SYMBOL(drm_dp_check_act_status);
 
 /**
  * drm_dp_calc_pbn_mode() - Calculate the PBN for a mode.
- * @clock: dot clock for the mode
- * @bpp: bpp for the mode.
- * @dsc: DSC mode. If true, bpp has units of 1/16 of a bit per pixel
+ * @clock: dot clock
+ * @bpp: bpp as .4 binary fixed point
  *
  * This uses the formula in the spec to calculate the PBN value for a mode.
  */
-int drm_dp_calc_pbn_mode(int clock, int bpp, bool dsc)
+int drm_dp_calc_pbn_mode(int clock, int bpp)
 {
 	/*
-	 * margin 5300ppm + 300ppm ~ 0.6% as per spec, factor is 1.006
 	 * The unit of 54/64Mbytes/sec is an arbitrary unit chosen based on
 	 * common multiplier to render an integer PBN for all link rate/lane
 	 * counts combinations
 	 * calculate
-	 * peak_kbps *= (1006/1000)
-	 * peak_kbps *= (64/54)
-	 * peak_kbps *= 8    convert to bytes
-	 *
-	 * If the bpp is in units of 1/16, further divide by 16. Put this
-	 * factor in the numerator rather than the denominator to avoid
-	 * integer overflow
+	 * peak_kbps = clock * bpp / 16
+	 * peak_kbps *= SSC overhead / 1000000
+	 * peak_kbps /= 8    convert to Kbytes
+	 * peak_kBps *= (64/54) / 1000    convert to PBN
 	 */
+	/*
+	 * TODO: Use the actual link and mode parameters to calculate
+	 * the overhead. For now it's assumed that these are
+	 * 4 link lanes, 4096 hactive pixels, which don't add any
+	 * significant data padding overhead and that there is no DSC
+	 * or FEC overhead.
+	 */
+	int overhead = drm_dp_bw_overhead(4, 4096, 0, bpp,
+					  DRM_DP_BW_OVERHEAD_MST |
+					  DRM_DP_BW_OVERHEAD_SSC_REF_CLK);
 
-	if (dsc)
-		return DIV_ROUND_UP_ULL(mul_u32_u32(clock * (bpp / 16), 64 * 1006),
-					8 * 54 * 1000 * 1000);
-
-	return DIV_ROUND_UP_ULL(mul_u32_u32(clock * bpp, 64 * 1006),
-				8 * 54 * 1000 * 1000);
+	return DIV64_U64_ROUND_UP(mul_u32_u32(clock * bpp, 64 * overhead >> 4),
+				  1000000ULL * 8 * 54 * 1000);
 }
 EXPORT_SYMBOL(drm_dp_calc_pbn_mode);
 
@@ -4871,7 +4883,8 @@ void drm_dp_mst_dump_topology(struct seq_file *m,
 	state = to_drm_dp_mst_topology_state(mgr->base.state);
 	seq_printf(m, "\n*** Atomic state info ***\n");
 	seq_printf(m, "payload_mask: %x, max_payloads: %d, start_slot: %u, pbn_div: %d\n",
-		   state->payload_mask, mgr->max_payloads, state->start_slot, state->pbn_div);
+		   state->payload_mask, mgr->max_payloads, state->start_slot,
+		   dfixed_trunc(state->pbn_div));
 
 	seq_printf(m, "\n| idx | port | vcpi | slots | pbn | dsc | status |     sink name     |\n");
 	for (i = 0; i < mgr->max_payloads; i++) {
@@ -5136,13 +5149,67 @@ static bool drm_dp_mst_port_downstream_of_branch(struct drm_dp_mst_port *port,
 	return false;
 }
 
+static bool
+drm_dp_mst_port_downstream_of_parent_locked(struct drm_dp_mst_topology_mgr *mgr,
+					    struct drm_dp_mst_port *port,
+					    struct drm_dp_mst_port *parent)
+{
+	if (!mgr->mst_primary)
+		return false;
+
+	port = drm_dp_mst_topology_get_port_validated_locked(mgr->mst_primary,
+							     port);
+	if (!port)
+		return false;
+
+	if (!parent)
+		return true;
+
+	parent = drm_dp_mst_topology_get_port_validated_locked(mgr->mst_primary,
+							       parent);
+	if (!parent)
+		return false;
+
+	if (!parent->mstb)
+		return false;
+
+	return drm_dp_mst_port_downstream_of_branch(port, parent->mstb);
+}
+
+/**
+ * drm_dp_mst_port_downstream_of_parent - check if a port is downstream of a parent port
+ * @mgr: MST topology manager
+ * @port: the port being looked up
+ * @parent: the parent port
+ *
+ * The function returns %true if @port is downstream of @parent. If @parent is
+ * %NULL - denoting the root port - the function returns %true if @port is in
+ * @mgr's topology.
+ */
+bool
+drm_dp_mst_port_downstream_of_parent(struct drm_dp_mst_topology_mgr *mgr,
+				     struct drm_dp_mst_port *port,
+				     struct drm_dp_mst_port *parent)
+{
+	bool ret;
+
+	mutex_lock(&mgr->lock);
+	ret = drm_dp_mst_port_downstream_of_parent_locked(mgr, port, parent);
+	mutex_unlock(&mgr->lock);
+
+	return ret;
+}
+EXPORT_SYMBOL(drm_dp_mst_port_downstream_of_parent);
+
 static int
 drm_dp_mst_atomic_check_port_bw_limit(struct drm_dp_mst_port *port,
-				      struct drm_dp_mst_topology_state *state);
+				      struct drm_dp_mst_topology_state *state,
+				      struct drm_dp_mst_port **failing_port);
 
 static int
 drm_dp_mst_atomic_check_mstb_bw_limit(struct drm_dp_mst_branch *mstb,
-				      struct drm_dp_mst_topology_state *state)
+				      struct drm_dp_mst_topology_state *state,
+				      struct drm_dp_mst_port **failing_port)
 {
 	struct drm_dp_mst_atomic_payload *payload;
 	struct drm_dp_mst_port *port;
@@ -5171,7 +5238,7 @@ drm_dp_mst_atomic_check_mstb_bw_limit(struct drm_dp_mst_branch *mstb,
 		drm_dbg_atomic(mstb->mgr->dev, "[MSTB:%p] Checking bandwidth limits\n", mstb);
 
 	list_for_each_entry(port, &mstb->ports, next) {
-		ret = drm_dp_mst_atomic_check_port_bw_limit(port, state);
+		ret = drm_dp_mst_atomic_check_port_bw_limit(port, state, failing_port);
 		if (ret < 0)
 			return ret;
 
@@ -5183,7 +5250,8 @@ drm_dp_mst_atomic_check_mstb_bw_limit(struct drm_dp_mst_branch *mstb,
 
 static int
 drm_dp_mst_atomic_check_port_bw_limit(struct drm_dp_mst_port *port,
-				      struct drm_dp_mst_topology_state *state)
+				      struct drm_dp_mst_topology_state *state,
+				      struct drm_dp_mst_port **failing_port)
 {
 	struct drm_dp_mst_atomic_payload *payload;
 	int pbn_used = 0;
@@ -5204,13 +5272,15 @@ drm_dp_mst_atomic_check_port_bw_limit(struct drm_dp_mst_port *port,
 			drm_dbg_atomic(port->mgr->dev,
 				       "[MSTB:%p] [MST PORT:%p] no BW available for the port\n",
 				       port->parent, port);
+			*failing_port = port;
 			return -EINVAL;
 		}
 
 		pbn_used = payload->pbn;
 	} else {
 		pbn_used = drm_dp_mst_atomic_check_mstb_bw_limit(port->mstb,
-								 state);
+								 state,
+								 failing_port);
 		if (pbn_used <= 0)
 			return pbn_used;
 	}
@@ -5219,6 +5289,7 @@ drm_dp_mst_atomic_check_port_bw_limit(struct drm_dp_mst_port *port,
 		drm_dbg_atomic(port->mgr->dev,
 			       "[MSTB:%p] [MST PORT:%p] required PBN of %d exceeds port limit of %d\n",
 			       port->parent, port, pbn_used, port->full_pbn);
+		*failing_port = port;
 		return -ENOSPC;
 	}
 
@@ -5271,10 +5342,10 @@ drm_dp_mst_atomic_check_payload_alloc_limits(struct drm_dp_mst_topology_mgr *mgr
 	}
 
 	if (!payload_count)
-		mst_state->pbn_div = 0;
+		mst_state->pbn_div.full = dfixed_const(0);
 
 	drm_dbg_atomic(mgr->dev, "[MST MGR:%p] mst state %p TU pbn_div=%d avail=%d used=%d\n",
-		       mgr, mst_state, mst_state->pbn_div, avail_slots,
+		       mgr, mst_state, dfixed_trunc(mst_state->pbn_div), avail_slots,
 		       mst_state->total_avail_slots - avail_slots);
 
 	return 0;
@@ -5396,20 +5467,82 @@ int drm_dp_mst_atomic_enable_dsc(struct drm_atomic_state *state,
 }
 EXPORT_SYMBOL(drm_dp_mst_atomic_enable_dsc);
 
+/**
+ * drm_dp_mst_atomic_check_mgr - Check the atomic state of an MST topology manager
+ * @state: The global atomic state
+ * @mgr: Manager to check
+ * @mst_state: The MST atomic state for @mgr
+ * @failing_port: Returns the port with a BW limitation
+ *
+ * Checks the given MST manager's topology state for an atomic update to ensure
+ * that it's valid. This includes checking whether there's enough bandwidth to
+ * support the new timeslot allocations in the atomic update.
+ *
+ * Any atomic drivers supporting DP MST must make sure to call this or
+ * the drm_dp_mst_atomic_check() function after checking the rest of their state
+ * in their &drm_mode_config_funcs.atomic_check() callback.
+ *
+ * See also:
+ * drm_dp_mst_atomic_check()
+ * drm_dp_atomic_find_time_slots()
+ * drm_dp_atomic_release_time_slots()
+ *
+ * Returns:
+ *   - 0 if the new state is valid
+ *   - %-ENOSPC, if the new state is invalid, because of BW limitation
+ *         @failing_port is set to:
+ *         - The non-root port where a BW limit check failed
+ *           with all the ports downstream of @failing_port passing
+ *           the BW limit check.
+ *           The returned port pointer is valid until at least
+ *           one payload downstream of it exists.
+ *         - %NULL if the BW limit check failed at the root port
+ *           with all the ports downstream of the root port passing
+ *           the BW limit check.
+ *   - %-EINVAL, if the new state is invalid, because the root port has
+ *     too many payloads.
+ */
+int drm_dp_mst_atomic_check_mgr(struct drm_atomic_state *state,
+				struct drm_dp_mst_topology_mgr *mgr,
+				struct drm_dp_mst_topology_state *mst_state,
+				struct drm_dp_mst_port **failing_port)
+{
+	int ret;
+
+	*failing_port = NULL;
+
+	if (!mgr->mst_state)
+		return 0;
+
+	mutex_lock(&mgr->lock);
+	ret = drm_dp_mst_atomic_check_mstb_bw_limit(mgr->mst_primary,
+						    mst_state,
+						    failing_port);
+	mutex_unlock(&mgr->lock);
+
+	if (ret < 0)
+		return ret;
+
+	return drm_dp_mst_atomic_check_payload_alloc_limits(mgr, mst_state);
+}
+EXPORT_SYMBOL(drm_dp_mst_atomic_check_mgr);
+
 /**
  * drm_dp_mst_atomic_check - Check that the new state of an MST topology in an
  * atomic update is valid
  * @state: Pointer to the new &struct drm_dp_mst_topology_state
  *
  * Checks the given topology state for an atomic update to ensure that it's
- * valid. This includes checking whether there's enough bandwidth to support
- * the new timeslot allocations in the atomic update.
+ * valid, calling drm_dp_mst_atomic_check_mgr() for all MST manager in the
+ * atomic state. This includes checking whether there's enough bandwidth to
+ * support the new timeslot allocations in the atomic update.
  *
  * Any atomic drivers supporting DP MST must make sure to call this after
  * checking the rest of their state in their
  * &drm_mode_config_funcs.atomic_check() callback.
  *
  * See also:
+ * drm_dp_mst_atomic_check_mgr()
  * drm_dp_atomic_find_time_slots()
  * drm_dp_atomic_release_time_slots()
  *
@@ -5424,21 +5557,11 @@ int drm_dp_mst_atomic_check(struct drm_atomic_state *state)
 	int i, ret = 0;
 
 	for_each_new_mst_mgr_in_state(state, mgr, mst_state, i) {
-		if (!mgr->mst_state)
-			continue;
+		struct drm_dp_mst_port *tmp_port;
 
-		ret = drm_dp_mst_atomic_check_payload_alloc_limits(mgr, mst_state);
+		ret = drm_dp_mst_atomic_check_mgr(state, mgr, mst_state, &tmp_port);
 		if (ret)
 			break;
-
-		mutex_lock(&mgr->lock);
-		ret = drm_dp_mst_atomic_check_mstb_bw_limit(mgr->mst_primary,
-							    mst_state);
-		mutex_unlock(&mgr->lock);
-		if (ret < 0)
-			break;
-		else
-			ret = 0;
 	}
 
 	return ret;
@@ -5894,6 +6017,7 @@ static bool drm_dp_mst_is_virtual_dpcd(struct drm_dp_mst_port *port)
 struct drm_dp_aux *drm_dp_mst_dsc_aux_for_port(struct drm_dp_mst_port *port)
 {
 	struct drm_dp_mst_port *immediate_upstream_port;
+	struct drm_dp_aux *immediate_upstream_aux;
 	struct drm_dp_mst_port *fec_port;
 	struct drm_dp_desc desc = {};
 	u8 endpoint_fec;
@@ -5958,21 +6082,25 @@ struct drm_dp_aux *drm_dp_mst_dsc_aux_for_port(struct drm_dp_mst_port *port)
 	 * - Port is on primary branch device
 	 * - Not a VGA adapter (DP_DWN_STRM_PORT_TYPE_ANALOG)
 	 */
-	if (drm_dp_read_desc(port->mgr->aux, &desc, true))
+	if (immediate_upstream_port)
+		immediate_upstream_aux = &immediate_upstream_port->aux;
+	else
+		immediate_upstream_aux = port->mgr->aux;
+
+	if (drm_dp_read_desc(immediate_upstream_aux, &desc, true))
 		return NULL;
 
-	if (drm_dp_has_quirk(&desc, DP_DPCD_QUIRK_DSC_WITHOUT_VIRTUAL_DPCD) &&
-	    port->mgr->dpcd[DP_DPCD_REV] >= DP_DPCD_REV_14 &&
-	    port->parent == port->mgr->mst_primary) {
+	if (drm_dp_has_quirk(&desc, DP_DPCD_QUIRK_DSC_WITHOUT_VIRTUAL_DPCD)) {
 		u8 dpcd_ext[DP_RECEIVER_CAP_SIZE];
 
-		if (drm_dp_read_dpcd_caps(port->mgr->aux, dpcd_ext) < 0)
+		if (drm_dp_read_dpcd_caps(immediate_upstream_aux, dpcd_ext) < 0)
 			return NULL;
 
-		if ((dpcd_ext[DP_DOWNSTREAMPORT_PRESENT] & DP_DWN_STRM_PORT_PRESENT) &&
+		if (dpcd_ext[DP_DPCD_REV] >= DP_DPCD_REV_14 &&
+		    ((dpcd_ext[DP_DOWNSTREAMPORT_PRESENT] & DP_DWN_STRM_PORT_PRESENT) &&
 		    ((dpcd_ext[DP_DOWNSTREAMPORT_PRESENT] & DP_DWN_STRM_PORT_TYPE_MASK)
-		     != DP_DWN_STRM_PORT_TYPE_ANALOG))
-			return port->mgr->aux;
+		     != DP_DWN_STRM_PORT_TYPE_ANALOG)))
+			return immediate_upstream_aux;
 	}
 
 	/*
diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c
index 2444fc33dd7c..c3f677130def 100644
--- a/drivers/gpu/drm/drm_atomic_helper.c
+++ b/drivers/gpu/drm/drm_atomic_helper.c
@@ -2382,10 +2382,10 @@ int drm_atomic_helper_setup_commit(struct drm_atomic_state *state,
 EXPORT_SYMBOL(drm_atomic_helper_setup_commit);
 
 /**
- * drm_atomic_helper_wait_for_dependencies - wait for required preceeding commits
+ * drm_atomic_helper_wait_for_dependencies - wait for required preceding commits
  * @old_state: atomic state object with old state structures
  *
- * This function waits for all preceeding commits that touch the same CRTC as
+ * This function waits for all preceding commits that touch the same CRTC as
  * @old_state to both be committed to the hardware (as signalled by
  * drm_atomic_helper_commit_hw_done()) and executed by the hardware (as signalled
  * by calling drm_crtc_send_vblank_event() on the &drm_crtc_state.event).
diff --git a/drivers/gpu/drm/drm_client.c b/drivers/gpu/drm/drm_client.c
index c3027115d055..9403b3f576f7 100644
--- a/drivers/gpu/drm/drm_client.c
+++ b/drivers/gpu/drm/drm_client.c
@@ -5,7 +5,6 @@
 
 #include <linux/iosys-map.h>
 #include <linux/list.h>
-#include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/seq_file.h>
 #include <linux/slab.h>
@@ -84,16 +83,13 @@ int drm_client_init(struct drm_device *dev, struct drm_client_dev *client,
 	if (!drm_core_check_feature(dev, DRIVER_MODESET) || !dev->driver->dumb_create)
 		return -EOPNOTSUPP;
 
-	if (funcs && !try_module_get(funcs->owner))
-		return -ENODEV;
-
 	client->dev = dev;
 	client->name = name;
 	client->funcs = funcs;
 
 	ret = drm_client_modeset_create(client);
 	if (ret)
-		goto err_put_module;
+		return ret;
 
 	ret = drm_client_open(client);
 	if (ret)
@@ -105,10 +101,6 @@ int drm_client_init(struct drm_device *dev, struct drm_client_dev *client,
 
 err_free:
 	drm_client_modeset_free(client);
-err_put_module:
-	if (funcs)
-		module_put(funcs->owner);
-
 	return ret;
 }
 EXPORT_SYMBOL(drm_client_init);
@@ -177,8 +169,6 @@ void drm_client_release(struct drm_client_dev *client)
 	drm_client_modeset_free(client);
 	drm_client_close(client);
 	drm_dev_put(dev);
-	if (client->funcs)
-		module_put(client->funcs->owner);
 }
 EXPORT_SYMBOL(drm_client_release);
 
diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c
index c3725086f413..b0516505f7ae 100644
--- a/drivers/gpu/drm/drm_connector.c
+++ b/drivers/gpu/drm/drm_connector.c
@@ -1198,6 +1198,12 @@ static const u32 dp_colorspaces =
  * 	drm_connector_set_path_property(), in the case of DP MST with the
  * 	path property the MST manager created. Userspace cannot change this
  * 	property.
+ *
+ * 	In the case of DP MST, the property has the format
+ * 	``mst:<parent>-<ports>`` where ``<parent>`` is the KMS object ID of the
+ * 	parent connector and ``<ports>`` is a hyphen-separated list of DP MST
+ * 	port numbers. Note, KMS object IDs are not guaranteed to be stable
+ * 	across reboots.
  * TILE:
  * 	Connector tile group property to indicate how a set of DRM connector
  * 	compose together into one logical screen. This is used by both high-res
diff --git a/drivers/gpu/drm/drm_crtc_internal.h b/drivers/gpu/drm/drm_crtc_internal.h
index 8556c3b3ff88..6b646e0783be 100644
--- a/drivers/gpu/drm/drm_crtc_internal.h
+++ b/drivers/gpu/drm/drm_crtc_internal.h
@@ -222,6 +222,8 @@ int drm_mode_addfb2_ioctl(struct drm_device *dev,
 			  void *data, struct drm_file *file_priv);
 int drm_mode_rmfb_ioctl(struct drm_device *dev,
 			void *data, struct drm_file *file_priv);
+int drm_mode_closefb_ioctl(struct drm_device *dev,
+			   void *data, struct drm_file *file_priv);
 int drm_mode_getfb(struct drm_device *dev,
 		   void *data, struct drm_file *file_priv);
 int drm_mode_getfb2_ioctl(struct drm_device *dev,
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index 39db08f803ea..cb4031d5dcbb 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -41,10 +41,12 @@
 #include <drm/drm_displayid.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_eld.h>
 #include <drm/drm_encoder.h>
 #include <drm/drm_print.h>
 
 #include "drm_crtc_internal.h"
+#include "drm_internal.h"
 
 static int oui(u8 first, u8 second, u8 third)
 {
@@ -5509,6 +5511,27 @@ static void clear_eld(struct drm_connector *connector)
 	connector->audio_latency[1] = 0;
 }
 
+/*
+ * Get 3-byte SAD buffer from struct cea_sad.
+ */
+void drm_edid_cta_sad_get(const struct cea_sad *cta_sad, u8 *sad)
+{
+	sad[0] = cta_sad->format << 3 | cta_sad->channels;
+	sad[1] = cta_sad->freq;
+	sad[2] = cta_sad->byte2;
+}
+
+/*
+ * Set struct cea_sad from 3-byte SAD buffer.
+ */
+void drm_edid_cta_sad_set(struct cea_sad *cta_sad, const u8 *sad)
+{
+	cta_sad->format = (sad[0] & 0x78) >> 3;
+	cta_sad->channels = sad[0] & 0x07;
+	cta_sad->freq = sad[1] & 0x7f;
+	cta_sad->byte2 = sad[2];
+}
+
 /*
  * drm_edid_to_eld - build ELD from EDID
  * @connector: connector corresponding to the HDMI/DP sink
@@ -5593,7 +5616,7 @@ static void drm_edid_to_eld(struct drm_connector *connector,
 }
 
 static int _drm_edid_to_sad(const struct drm_edid *drm_edid,
-			    struct cea_sad **sads)
+			    struct cea_sad **psads)
 {
 	const struct cea_db *db;
 	struct cea_db_iter iter;
@@ -5602,20 +5625,16 @@ static int _drm_edid_to_sad(const struct drm_edid *drm_edid,
 	cea_db_iter_edid_begin(drm_edid, &iter);
 	cea_db_iter_for_each(db, &iter) {
 		if (cea_db_tag(db) == CTA_DB_AUDIO) {
-			int j;
+			struct cea_sad *sads;
+			int i;
 
 			count = cea_db_payload_len(db) / 3; /* SAD is 3B */
-			*sads = kcalloc(count, sizeof(**sads), GFP_KERNEL);
-			if (!*sads)
+			sads = kcalloc(count, sizeof(*sads), GFP_KERNEL);
+			*psads = sads;
+			if (!sads)
 				return -ENOMEM;
-			for (j = 0; j < count; j++) {
-				const u8 *sad = &db->data[j * 3];
-
-				(*sads)[j].format = (sad[0] & 0x78) >> 3;
-				(*sads)[j].channels = sad[0] & 0x7;
-				(*sads)[j].freq = sad[1] & 0x7F;
-				(*sads)[j].byte2 = sad[2];
-			}
+			for (i = 0; i < count; i++)
+				drm_edid_cta_sad_set(&sads[i], &db->data[i * 3]);
 			break;
 		}
 	}
diff --git a/drivers/gpu/drm/drm_edid_load.c b/drivers/gpu/drm/drm_edid_load.c
index 5d9ef267ebb3..60fcb80bce61 100644
--- a/drivers/gpu/drm/drm_edid_load.c
+++ b/drivers/gpu/drm/drm_edid_load.c
@@ -23,22 +23,6 @@ module_param_string(edid_firmware, edid_firmware, sizeof(edid_firmware), 0644);
 MODULE_PARM_DESC(edid_firmware, "Do not probe monitor, use specified EDID blob "
 	"from built-in data or /lib/firmware instead. ");
 
-/* Use only for backward compatibility with drm_kms_helper.edid_firmware */
-int __drm_set_edid_firmware_path(const char *path)
-{
-	scnprintf(edid_firmware, sizeof(edid_firmware), "%s", path);
-
-	return 0;
-}
-EXPORT_SYMBOL(__drm_set_edid_firmware_path);
-
-/* Use only for backward compatibility with drm_kms_helper.edid_firmware */
-int __drm_get_edid_firmware_path(char *buf, size_t bufsize)
-{
-	return scnprintf(buf, bufsize, "%s", edid_firmware);
-}
-EXPORT_SYMBOL(__drm_get_edid_firmware_path);
-
 #define GENERIC_EDIDS 6
 static const char * const generic_edid_name[GENERIC_EDIDS] = {
 	"edid/800x600.bin",
diff --git a/drivers/gpu/drm/drm_eld.c b/drivers/gpu/drm/drm_eld.c
new file mode 100644
index 000000000000..5177991aa272
--- /dev/null
+++ b/drivers/gpu/drm/drm_eld.c
@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#include <drm/drm_edid.h>
+#include <drm/drm_eld.h>
+
+#include "drm_internal.h"
+
+/**
+ * drm_eld_sad_get - get SAD from ELD to struct cea_sad
+ * @eld: ELD buffer
+ * @sad_index: SAD index
+ * @cta_sad: destination struct cea_sad
+ *
+ * @return: 0 on success, or negative on errors
+ */
+int drm_eld_sad_get(const u8 *eld, int sad_index, struct cea_sad *cta_sad)
+{
+	const u8 *sad;
+
+	if (sad_index >= drm_eld_sad_count(eld))
+		return -EINVAL;
+
+	sad = eld + DRM_ELD_CEA_SAD(drm_eld_mnl(eld), sad_index);
+
+	drm_edid_cta_sad_set(cta_sad, sad);
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_eld_sad_get);
+
+/**
+ * drm_eld_sad_set - set SAD to ELD from struct cea_sad
+ * @eld: ELD buffer
+ * @sad_index: SAD index
+ * @cta_sad: source struct cea_sad
+ *
+ * @return: 0 on success, or negative on errors
+ */
+int drm_eld_sad_set(u8 *eld, int sad_index, const struct cea_sad *cta_sad)
+{
+	u8 *sad;
+
+	if (sad_index >= drm_eld_sad_count(eld))
+		return -EINVAL;
+
+	sad = eld + DRM_ELD_CEA_SAD(drm_eld_mnl(eld), sad_index);
+
+	drm_edid_cta_sad_get(cta_sad, sad);
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_eld_sad_set);
diff --git a/drivers/gpu/drm/drm_file.c b/drivers/gpu/drm/drm_file.c
index 446458aca8e9..5ddaffd32586 100644
--- a/drivers/gpu/drm/drm_file.c
+++ b/drivers/gpu/drm/drm_file.c
@@ -913,7 +913,7 @@ static void print_size(struct drm_printer *p, const char *stat,
 	unsigned u;
 
 	for (u = 0; u < ARRAY_SIZE(units) - 1; u++) {
-		if (sz < SZ_1K)
+		if (sz == 0 || !IS_ALIGNED(sz, SZ_1K))
 			break;
 		sz = div_u64(sz, SZ_1K);
 	}
diff --git a/drivers/gpu/drm/drm_flip_work.c b/drivers/gpu/drm/drm_flip_work.c
index 060b753881a2..8c6090a90d56 100644
--- a/drivers/gpu/drm/drm_flip_work.c
+++ b/drivers/gpu/drm/drm_flip_work.c
@@ -27,14 +27,12 @@
 #include <drm/drm_print.h>
 #include <drm/drm_util.h>
 
-/**
- * drm_flip_work_allocate_task - allocate a flip-work task
- * @data: data associated to the task
- * @flags: allocator flags
- *
- * Allocate a drm_flip_task object and attach private data to it.
- */
-struct drm_flip_task *drm_flip_work_allocate_task(void *data, gfp_t flags)
+struct drm_flip_task {
+	struct list_head node;
+	void *data;
+};
+
+static struct drm_flip_task *drm_flip_work_allocate_task(void *data, gfp_t flags)
 {
 	struct drm_flip_task *task;
 
@@ -44,18 +42,8 @@ struct drm_flip_task *drm_flip_work_allocate_task(void *data, gfp_t flags)
 
 	return task;
 }
-EXPORT_SYMBOL(drm_flip_work_allocate_task);
 
-/**
- * drm_flip_work_queue_task - queue a specific task
- * @work: the flip-work
- * @task: the task to handle
- *
- * Queues task, that will later be run (passed back to drm_flip_func_t
- * func) on a work queue after drm_flip_work_commit() is called.
- */
-void drm_flip_work_queue_task(struct drm_flip_work *work,
-			      struct drm_flip_task *task)
+static void drm_flip_work_queue_task(struct drm_flip_work *work, struct drm_flip_task *task)
 {
 	unsigned long flags;
 
@@ -63,7 +51,6 @@ void drm_flip_work_queue_task(struct drm_flip_work *work,
 	list_add_tail(&task->node, &work->queued);
 	spin_unlock_irqrestore(&work->lock, flags);
 }
-EXPORT_SYMBOL(drm_flip_work_queue_task);
 
 /**
  * drm_flip_work_queue - queue work
diff --git a/drivers/gpu/drm/drm_format_helper.c b/drivers/gpu/drm/drm_format_helper.c
index f93a4efcee90..b1be458ed4dd 100644
--- a/drivers/gpu/drm/drm_format_helper.c
+++ b/drivers/gpu/drm/drm_format_helper.c
@@ -20,6 +20,97 @@
 #include <drm/drm_print.h>
 #include <drm/drm_rect.h>
 
+/**
+ * drm_format_conv_state_init - Initialize format-conversion state
+ * @state: The state to initialize
+ *
+ * Clears all fields in struct drm_format_conv_state. The state will
+ * be empty with no preallocated resources.
+ */
+void drm_format_conv_state_init(struct drm_format_conv_state *state)
+{
+	state->tmp.mem = NULL;
+	state->tmp.size = 0;
+	state->tmp.preallocated = false;
+}
+EXPORT_SYMBOL(drm_format_conv_state_init);
+
+/**
+ * drm_format_conv_state_copy - Copy format-conversion state
+ * @state: Destination state
+ * @old_state: Source state
+ *
+ * Copies format-conversion state from @old_state to @state; except for
+ * temporary storage.
+ */
+void drm_format_conv_state_copy(struct drm_format_conv_state *state,
+				const struct drm_format_conv_state *old_state)
+{
+	/*
+	 * So far, there's only temporary storage here, which we don't
+	 * duplicate. Just clear the fields.
+	 */
+	state->tmp.mem = NULL;
+	state->tmp.size = 0;
+	state->tmp.preallocated = false;
+}
+EXPORT_SYMBOL(drm_format_conv_state_copy);
+
+/**
+ * drm_format_conv_state_reserve - Allocates storage for format conversion
+ * @state: The format-conversion state
+ * @new_size: The minimum allocation size
+ * @flags: Flags for kmalloc()
+ *
+ * Allocates at least @new_size bytes and returns a pointer to the memory
+ * range. After calling this function, previously returned memory blocks
+ * are invalid. It's best to collect all memory requirements of a format
+ * conversion and call this function once to allocate the range.
+ *
+ * Returns:
+ * A pointer to the allocated memory range, or NULL otherwise.
+ */
+void *drm_format_conv_state_reserve(struct drm_format_conv_state *state,
+				    size_t new_size, gfp_t flags)
+{
+	void *mem;
+
+	if (new_size <= state->tmp.size)
+		goto out;
+	else if (state->tmp.preallocated)
+		return NULL;
+
+	mem = krealloc(state->tmp.mem, new_size, flags);
+	if (!mem)
+		return NULL;
+
+	state->tmp.mem = mem;
+	state->tmp.size = new_size;
+
+out:
+	return state->tmp.mem;
+}
+EXPORT_SYMBOL(drm_format_conv_state_reserve);
+
+/**
+ * drm_format_conv_state_release - Releases an format-conversion storage
+ * @state: The format-conversion state
+ *
+ * Releases the memory range references by the format-conversion state.
+ * After this call, all pointers to the memory are invalid. Prefer
+ * drm_format_conv_state_init() for cleaning up and unloading a driver.
+ */
+void drm_format_conv_state_release(struct drm_format_conv_state *state)
+{
+	if (state->tmp.preallocated)
+		return;
+
+	kfree(state->tmp.mem);
+	state->tmp.mem = NULL;
+	state->tmp.size = 0;
+}
+EXPORT_SYMBOL(drm_format_conv_state_release);
+
 static unsigned int clip_offset(const struct drm_rect *clip, unsigned int pitch, unsigned int cpp)
 {
 	return clip->y1 * pitch + clip->x1 * cpp;
@@ -45,6 +136,7 @@ EXPORT_SYMBOL(drm_fb_clip_offset);
 static int __drm_fb_xfrm(void *dst, unsigned long dst_pitch, unsigned long dst_pixsize,
 			 const void *vaddr, const struct drm_framebuffer *fb,
 			 const struct drm_rect *clip, bool vaddr_cached_hint,
+			 struct drm_format_conv_state *state,
 			 void (*xfrm_line)(void *dbuf, const void *sbuf, unsigned int npixels))
 {
 	unsigned long linepixels = drm_rect_width(clip);
@@ -60,7 +152,7 @@ static int __drm_fb_xfrm(void *dst, unsigned long dst_pitch, unsigned long dst_p
 	 * one line at a time.
 	 */
 	if (!vaddr_cached_hint) {
-		stmp = kmalloc(sbuf_len, GFP_KERNEL);
+		stmp = drm_format_conv_state_reserve(state, sbuf_len, GFP_KERNEL);
 		if (!stmp)
 			return -ENOMEM;
 	}
@@ -79,8 +171,6 @@ static int __drm_fb_xfrm(void *dst, unsigned long dst_pitch, unsigned long dst_p
 		dst += dst_pitch;
 	}
 
-	kfree(stmp);
-
 	return 0;
 }
 
@@ -88,6 +178,7 @@ static int __drm_fb_xfrm(void *dst, unsigned long dst_pitch, unsigned long dst_p
 static int __drm_fb_xfrm_toio(void __iomem *dst, unsigned long dst_pitch, unsigned long dst_pixsize,
 			      const void *vaddr, const struct drm_framebuffer *fb,
 			      const struct drm_rect *clip, bool vaddr_cached_hint,
+			      struct drm_format_conv_state *state,
 			      void (*xfrm_line)(void *dbuf, const void *sbuf, unsigned int npixels))
 {
 	unsigned long linepixels = drm_rect_width(clip);
@@ -101,9 +192,9 @@ static int __drm_fb_xfrm_toio(void __iomem *dst, unsigned long dst_pitch, unsign
 	void *dbuf;
 
 	if (vaddr_cached_hint) {
-		dbuf = kmalloc(dbuf_len, GFP_KERNEL);
+		dbuf = drm_format_conv_state_reserve(state, dbuf_len, GFP_KERNEL);
 	} else {
-		dbuf = kmalloc(stmp_off + sbuf_len, GFP_KERNEL);
+		dbuf = drm_format_conv_state_reserve(state, stmp_off + sbuf_len, GFP_KERNEL);
 		stmp = dbuf + stmp_off;
 	}
 	if (!dbuf)
@@ -124,8 +215,6 @@ static int __drm_fb_xfrm_toio(void __iomem *dst, unsigned long dst_pitch, unsign
 		dst += dst_pitch;
 	}
 
-	kfree(dbuf);
-
 	return 0;
 }
 
@@ -134,6 +223,7 @@ static int drm_fb_xfrm(struct iosys_map *dst,
 		       const unsigned int *dst_pitch, const u8 *dst_pixsize,
 		       const struct iosys_map *src, const struct drm_framebuffer *fb,
 		       const struct drm_rect *clip, bool vaddr_cached_hint,
+		       struct drm_format_conv_state *state,
 		       void (*xfrm_line)(void *dbuf, const void *sbuf, unsigned int npixels))
 {
 	static const unsigned int default_dst_pitch[DRM_FORMAT_MAX_PLANES] = {
@@ -146,10 +236,12 @@ static int drm_fb_xfrm(struct iosys_map *dst,
 	/* TODO: handle src in I/O memory here */
 	if (dst[0].is_iomem)
 		return __drm_fb_xfrm_toio(dst[0].vaddr_iomem, dst_pitch[0], dst_pixsize[0],
-					  src[0].vaddr, fb, clip, vaddr_cached_hint, xfrm_line);
+					  src[0].vaddr, fb, clip, vaddr_cached_hint, state,
+					  xfrm_line);
 	else
 		return __drm_fb_xfrm(dst[0].vaddr, dst_pitch[0], dst_pixsize[0],
-				     src[0].vaddr, fb, clip, vaddr_cached_hint, xfrm_line);
+				     src[0].vaddr, fb, clip, vaddr_cached_hint, state,
+				     xfrm_line);
 }
 
 /**
@@ -235,6 +327,7 @@ static void drm_fb_swab32_line(void *dbuf, const void *sbuf, unsigned int pixels
  * @fb: DRM framebuffer
  * @clip: Clip rectangle area to copy
  * @cached: Source buffer is mapped cached (eg. not write-combined)
+ * @state: Transform and conversion state
  *
  * This function copies parts of a framebuffer to display memory and swaps per-pixel
  * bytes during the process. Destination and framebuffer formats must match. The
@@ -249,7 +342,8 @@ static void drm_fb_swab32_line(void *dbuf, const void *sbuf, unsigned int pixels
  */
 void drm_fb_swab(struct iosys_map *dst, const unsigned int *dst_pitch,
 		 const struct iosys_map *src, const struct drm_framebuffer *fb,
-		 const struct drm_rect *clip, bool cached)
+		 const struct drm_rect *clip, bool cached,
+		 struct drm_format_conv_state *state)
 {
 	const struct drm_format_info *format = fb->format;
 	u8 cpp = DIV_ROUND_UP(drm_format_info_bpp(format, 0), 8);
@@ -268,7 +362,7 @@ void drm_fb_swab(struct iosys_map *dst, const unsigned int *dst_pitch,
 		return;
 	}
 
-	drm_fb_xfrm(dst, dst_pitch, &cpp, src, fb, clip, cached, swab_line);
+	drm_fb_xfrm(dst, dst_pitch, &cpp, src, fb, clip, cached, state, swab_line);
 }
 EXPORT_SYMBOL(drm_fb_swab);
 
@@ -295,6 +389,7 @@ static void drm_fb_xrgb8888_to_rgb332_line(void *dbuf, const void *sbuf, unsigne
  * @src: Array of XRGB8888 source buffers
  * @fb: DRM framebuffer
  * @clip: Clip rectangle area to copy
+ * @state: Transform and conversion state
  *
  * This function copies parts of a framebuffer to display memory and converts the
  * color format during the process. Destination and framebuffer formats must match. The
@@ -309,13 +404,13 @@ static void drm_fb_xrgb8888_to_rgb332_line(void *dbuf, const void *sbuf, unsigne
  */
 void drm_fb_xrgb8888_to_rgb332(struct iosys_map *dst, const unsigned int *dst_pitch,
 			       const struct iosys_map *src, const struct drm_framebuffer *fb,
-			       const struct drm_rect *clip)
+			       const struct drm_rect *clip, struct drm_format_conv_state *state)
 {
 	static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
 		1,
 	};
 
-	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
+	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
 		    drm_fb_xrgb8888_to_rgb332_line);
 }
 EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb332);
@@ -364,6 +459,7 @@ static void drm_fb_xrgb8888_to_rgb565_swab_line(void *dbuf, const void *sbuf,
  * @src: Array of XRGB8888 source buffer
  * @fb: DRM framebuffer
  * @clip: Clip rectangle area to copy
+ * @state: Transform and conversion state
  * @swab: Swap bytes
  *
  * This function copies parts of a framebuffer to display memory and converts the
@@ -379,7 +475,8 @@ static void drm_fb_xrgb8888_to_rgb565_swab_line(void *dbuf, const void *sbuf,
  */
 void drm_fb_xrgb8888_to_rgb565(struct iosys_map *dst, const unsigned int *dst_pitch,
 			       const struct iosys_map *src, const struct drm_framebuffer *fb,
-			       const struct drm_rect *clip, bool swab)
+			       const struct drm_rect *clip, struct drm_format_conv_state *state,
+			       bool swab)
 {
 	static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
 		2,
@@ -392,7 +489,7 @@ void drm_fb_xrgb8888_to_rgb565(struct iosys_map *dst, const unsigned int *dst_pi
 	else
 		xfrm_line = drm_fb_xrgb8888_to_rgb565_line;
 
-	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, xfrm_line);
+	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state, xfrm_line);
 }
 EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb565);
 
@@ -421,6 +518,7 @@ static void drm_fb_xrgb8888_to_xrgb1555_line(void *dbuf, const void *sbuf, unsig
  * @src: Array of XRGB8888 source buffer
  * @fb: DRM framebuffer
  * @clip: Clip rectangle area to copy
+ * @state: Transform and conversion state
  *
  * This function copies parts of a framebuffer to display memory and converts
  * the color format during the process. The parameters @dst, @dst_pitch and
@@ -436,13 +534,13 @@ static void drm_fb_xrgb8888_to_xrgb1555_line(void *dbuf, const void *sbuf, unsig
  */
 void drm_fb_xrgb8888_to_xrgb1555(struct iosys_map *dst, const unsigned int *dst_pitch,
 				 const struct iosys_map *src, const struct drm_framebuffer *fb,
-				 const struct drm_rect *clip)
+				 const struct drm_rect *clip, struct drm_format_conv_state *state)
 {
 	static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
 		2,
 	};
 
-	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
+	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
 		    drm_fb_xrgb8888_to_xrgb1555_line);
 }
 EXPORT_SYMBOL(drm_fb_xrgb8888_to_xrgb1555);
@@ -473,6 +571,7 @@ static void drm_fb_xrgb8888_to_argb1555_line(void *dbuf, const void *sbuf, unsig
  * @src: Array of XRGB8888 source buffer
  * @fb: DRM framebuffer
  * @clip: Clip rectangle area to copy
+ * @state: Transform and conversion state
  *
  * This function copies parts of a framebuffer to display memory and converts
  * the color format during the process. The parameters @dst, @dst_pitch and
@@ -488,13 +587,13 @@ static void drm_fb_xrgb8888_to_argb1555_line(void *dbuf, const void *sbuf, unsig
  */
 void drm_fb_xrgb8888_to_argb1555(struct iosys_map *dst, const unsigned int *dst_pitch,
 				 const struct iosys_map *src, const struct drm_framebuffer *fb,
-				 const struct drm_rect *clip)
+				 const struct drm_rect *clip, struct drm_format_conv_state *state)
 {
 	static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
 		2,
 	};
 
-	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
+	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
 		    drm_fb_xrgb8888_to_argb1555_line);
 }
 EXPORT_SYMBOL(drm_fb_xrgb8888_to_argb1555);
@@ -525,6 +624,7 @@ static void drm_fb_xrgb8888_to_rgba5551_line(void *dbuf, const void *sbuf, unsig
  * @src: Array of XRGB8888 source buffer
  * @fb: DRM framebuffer
  * @clip: Clip rectangle area to copy
+ * @state: Transform and conversion state
  *
  * This function copies parts of a framebuffer to display memory and converts
  * the color format during the process. The parameters @dst, @dst_pitch and
@@ -540,13 +640,13 @@ static void drm_fb_xrgb8888_to_rgba5551_line(void *dbuf, const void *sbuf, unsig
  */
 void drm_fb_xrgb8888_to_rgba5551(struct iosys_map *dst, const unsigned int *dst_pitch,
 				 const struct iosys_map *src, const struct drm_framebuffer *fb,
-				 const struct drm_rect *clip)
+				 const struct drm_rect *clip, struct drm_format_conv_state *state)
 {
 	static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
 		2,
 	};
 
-	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
+	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
 		    drm_fb_xrgb8888_to_rgba5551_line);
 }
 EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgba5551);
@@ -575,6 +675,7 @@ static void drm_fb_xrgb8888_to_rgb888_line(void *dbuf, const void *sbuf, unsigne
  * @src: Array of XRGB8888 source buffers
  * @fb: DRM framebuffer
  * @clip: Clip rectangle area to copy
+ * @state: Transform and conversion state
  *
  * This function copies parts of a framebuffer to display memory and converts the
  * color format during the process. Destination and framebuffer formats must match. The
@@ -590,13 +691,13 @@ static void drm_fb_xrgb8888_to_rgb888_line(void *dbuf, const void *sbuf, unsigne
  */
 void drm_fb_xrgb8888_to_rgb888(struct iosys_map *dst, const unsigned int *dst_pitch,
 			       const struct iosys_map *src, const struct drm_framebuffer *fb,
-			       const struct drm_rect *clip)
+			       const struct drm_rect *clip, struct drm_format_conv_state *state)
 {
 	static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
 		3,
 	};
 
-	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
+	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
 		    drm_fb_xrgb8888_to_rgb888_line);
 }
 EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb888);
@@ -623,6 +724,7 @@ static void drm_fb_xrgb8888_to_argb8888_line(void *dbuf, const void *sbuf, unsig
  * @src: Array of XRGB8888 source buffer
  * @fb: DRM framebuffer
  * @clip: Clip rectangle area to copy
+ * @state: Transform and conversion state
  *
  * This function copies parts of a framebuffer to display memory and converts the
  * color format during the process. The parameters @dst, @dst_pitch and @src refer
@@ -638,13 +740,13 @@ static void drm_fb_xrgb8888_to_argb8888_line(void *dbuf, const void *sbuf, unsig
  */
 void drm_fb_xrgb8888_to_argb8888(struct iosys_map *dst, const unsigned int *dst_pitch,
 				 const struct iosys_map *src, const struct drm_framebuffer *fb,
-				 const struct drm_rect *clip)
+				 const struct drm_rect *clip, struct drm_format_conv_state *state)
 {
 	static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
 		4,
 	};
 
-	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
+	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
 		    drm_fb_xrgb8888_to_argb8888_line);
 }
 EXPORT_SYMBOL(drm_fb_xrgb8888_to_argb8888);
@@ -669,13 +771,14 @@ static void drm_fb_xrgb8888_to_abgr8888_line(void *dbuf, const void *sbuf, unsig
 static void drm_fb_xrgb8888_to_abgr8888(struct iosys_map *dst, const unsigned int *dst_pitch,
 					const struct iosys_map *src,
 					const struct drm_framebuffer *fb,
-					const struct drm_rect *clip)
+					const struct drm_rect *clip,
+					struct drm_format_conv_state *state)
 {
 	static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
 		4,
 	};
 
-	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
+	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
 		    drm_fb_xrgb8888_to_abgr8888_line);
 }
 
@@ -699,13 +802,14 @@ static void drm_fb_xrgb8888_to_xbgr8888_line(void *dbuf, const void *sbuf, unsig
 static void drm_fb_xrgb8888_to_xbgr8888(struct iosys_map *dst, const unsigned int *dst_pitch,
 					const struct iosys_map *src,
 					const struct drm_framebuffer *fb,
-					const struct drm_rect *clip)
+					const struct drm_rect *clip,
+					struct drm_format_conv_state *state)
 {
 	static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
 		4,
 	};
 
-	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
+	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
 		    drm_fb_xrgb8888_to_xbgr8888_line);
 }
 
@@ -735,6 +839,7 @@ static void drm_fb_xrgb8888_to_xrgb2101010_line(void *dbuf, const void *sbuf, un
  * @src: Array of XRGB8888 source buffers
  * @fb: DRM framebuffer
  * @clip: Clip rectangle area to copy
+ * @state: Transform and conversion state
  *
  * This function copies parts of a framebuffer to display memory and converts the
  * color format during the process. Destination and framebuffer formats must match. The
@@ -750,13 +855,14 @@ static void drm_fb_xrgb8888_to_xrgb2101010_line(void *dbuf, const void *sbuf, un
  */
 void drm_fb_xrgb8888_to_xrgb2101010(struct iosys_map *dst, const unsigned int *dst_pitch,
 				    const struct iosys_map *src, const struct drm_framebuffer *fb,
-				    const struct drm_rect *clip)
+				    const struct drm_rect *clip,
+				    struct drm_format_conv_state *state)
 {
 	static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
 		4,
 	};
 
-	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
+	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
 		    drm_fb_xrgb8888_to_xrgb2101010_line);
 }
 EXPORT_SYMBOL(drm_fb_xrgb8888_to_xrgb2101010);
@@ -788,6 +894,7 @@ static void drm_fb_xrgb8888_to_argb2101010_line(void *dbuf, const void *sbuf, un
  * @src: Array of XRGB8888 source buffers
  * @fb: DRM framebuffer
  * @clip: Clip rectangle area to copy
+ * @state: Transform and conversion state
  *
  * This function copies parts of a framebuffer to display memory and converts
  * the color format during the process. The parameters @dst, @dst_pitch and
@@ -803,13 +910,14 @@ static void drm_fb_xrgb8888_to_argb2101010_line(void *dbuf, const void *sbuf, un
  */
 void drm_fb_xrgb8888_to_argb2101010(struct iosys_map *dst, const unsigned int *dst_pitch,
 				    const struct iosys_map *src, const struct drm_framebuffer *fb,
-				    const struct drm_rect *clip)
+				    const struct drm_rect *clip,
+				    struct drm_format_conv_state *state)
 {
 	static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
 		4,
 	};
 
-	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
+	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
 		    drm_fb_xrgb8888_to_argb2101010_line);
 }
 EXPORT_SYMBOL(drm_fb_xrgb8888_to_argb2101010);
@@ -839,6 +947,7 @@ static void drm_fb_xrgb8888_to_gray8_line(void *dbuf, const void *sbuf, unsigned
  * @src: Array of XRGB8888 source buffers
  * @fb: DRM framebuffer
  * @clip: Clip rectangle area to copy
+ * @state: Transform and conversion state
  *
  * This function copies parts of a framebuffer to display memory and converts the
  * color format during the process. Destination and framebuffer formats must match. The
@@ -858,13 +967,13 @@ static void drm_fb_xrgb8888_to_gray8_line(void *dbuf, const void *sbuf, unsigned
  */
 void drm_fb_xrgb8888_to_gray8(struct iosys_map *dst, const unsigned int *dst_pitch,
 			      const struct iosys_map *src, const struct drm_framebuffer *fb,
-			      const struct drm_rect *clip)
+			      const struct drm_rect *clip, struct drm_format_conv_state *state)
 {
 	static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
 		1,
 	};
 
-	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
+	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, state,
 		    drm_fb_xrgb8888_to_gray8_line);
 }
 EXPORT_SYMBOL(drm_fb_xrgb8888_to_gray8);
@@ -878,6 +987,7 @@ EXPORT_SYMBOL(drm_fb_xrgb8888_to_gray8);
  * @src:	The framebuffer memory to copy from
  * @fb:		The framebuffer to copy from
  * @clip:	Clip rectangle area to copy
+ * @state: Transform and conversion state
  *
  * This function copies parts of a framebuffer to display memory. If the
  * formats of the display and the framebuffer mismatch, the blit function
@@ -896,7 +1006,7 @@ EXPORT_SYMBOL(drm_fb_xrgb8888_to_gray8);
  */
 int drm_fb_blit(struct iosys_map *dst, const unsigned int *dst_pitch, uint32_t dst_format,
 		const struct iosys_map *src, const struct drm_framebuffer *fb,
-		const struct drm_rect *clip)
+		const struct drm_rect *clip, struct drm_format_conv_state *state)
 {
 	uint32_t fb_format = fb->format->format;
 
@@ -904,44 +1014,44 @@ int drm_fb_blit(struct iosys_map *dst, const unsigned int *dst_pitch, uint32_t d
 		drm_fb_memcpy(dst, dst_pitch, src, fb, clip);
 		return 0;
 	} else if (fb_format == (dst_format | DRM_FORMAT_BIG_ENDIAN)) {
-		drm_fb_swab(dst, dst_pitch, src, fb, clip, false);
+		drm_fb_swab(dst, dst_pitch, src, fb, clip, false, state);
 		return 0;
 	} else if (fb_format == (dst_format & ~DRM_FORMAT_BIG_ENDIAN)) {
-		drm_fb_swab(dst, dst_pitch, src, fb, clip, false);
+		drm_fb_swab(dst, dst_pitch, src, fb, clip, false, state);
 		return 0;
 	} else if (fb_format == DRM_FORMAT_XRGB8888) {
 		if (dst_format == DRM_FORMAT_RGB565) {
-			drm_fb_xrgb8888_to_rgb565(dst, dst_pitch, src, fb, clip, false);
+			drm_fb_xrgb8888_to_rgb565(dst, dst_pitch, src, fb, clip, state, false);
 			return 0;
 		} else if (dst_format == DRM_FORMAT_XRGB1555) {
-			drm_fb_xrgb8888_to_xrgb1555(dst, dst_pitch, src, fb, clip);
+			drm_fb_xrgb8888_to_xrgb1555(dst, dst_pitch, src, fb, clip, state);
 			return 0;
 		} else if (dst_format == DRM_FORMAT_ARGB1555) {
-			drm_fb_xrgb8888_to_argb1555(dst, dst_pitch, src, fb, clip);
+			drm_fb_xrgb8888_to_argb1555(dst, dst_pitch, src, fb, clip, state);
 			return 0;
 		} else if (dst_format == DRM_FORMAT_RGBA5551) {
-			drm_fb_xrgb8888_to_rgba5551(dst, dst_pitch, src, fb, clip);
+			drm_fb_xrgb8888_to_rgba5551(dst, dst_pitch, src, fb, clip, state);
 			return 0;
 		} else if (dst_format == DRM_FORMAT_RGB888) {
-			drm_fb_xrgb8888_to_rgb888(dst, dst_pitch, src, fb, clip);
+			drm_fb_xrgb8888_to_rgb888(dst, dst_pitch, src, fb, clip, state);
 			return 0;
 		} else if (dst_format == DRM_FORMAT_ARGB8888) {
-			drm_fb_xrgb8888_to_argb8888(dst, dst_pitch, src, fb, clip);
+			drm_fb_xrgb8888_to_argb8888(dst, dst_pitch, src, fb, clip, state);
 			return 0;
 		} else if (dst_format == DRM_FORMAT_XBGR8888) {
-			drm_fb_xrgb8888_to_xbgr8888(dst, dst_pitch, src, fb, clip);
+			drm_fb_xrgb8888_to_xbgr8888(dst, dst_pitch, src, fb, clip, state);
 			return 0;
 		} else if (dst_format == DRM_FORMAT_ABGR8888) {
-			drm_fb_xrgb8888_to_abgr8888(dst, dst_pitch, src, fb, clip);
+			drm_fb_xrgb8888_to_abgr8888(dst, dst_pitch, src, fb, clip, state);
 			return 0;
 		} else if (dst_format == DRM_FORMAT_XRGB2101010) {
-			drm_fb_xrgb8888_to_xrgb2101010(dst, dst_pitch, src, fb, clip);
+			drm_fb_xrgb8888_to_xrgb2101010(dst, dst_pitch, src, fb, clip, state);
 			return 0;
 		} else if (dst_format == DRM_FORMAT_ARGB2101010) {
-			drm_fb_xrgb8888_to_argb2101010(dst, dst_pitch, src, fb, clip);
+			drm_fb_xrgb8888_to_argb2101010(dst, dst_pitch, src, fb, clip, state);
 			return 0;
 		} else if (dst_format == DRM_FORMAT_BGRX8888) {
-			drm_fb_swab(dst, dst_pitch, src, fb, clip, false);
+			drm_fb_swab(dst, dst_pitch, src, fb, clip, false, state);
 			return 0;
 		}
 	}
@@ -978,6 +1088,7 @@ static void drm_fb_gray8_to_mono_line(void *dbuf, const void *sbuf, unsigned int
  * @src: Array of XRGB8888 source buffers
  * @fb: DRM framebuffer
  * @clip: Clip rectangle area to copy
+ * @state: Transform and conversion state
  *
  * This function copies parts of a framebuffer to display memory and converts the
  * color format during the process. Destination and framebuffer formats must match. The
@@ -1002,7 +1113,7 @@ static void drm_fb_gray8_to_mono_line(void *dbuf, const void *sbuf, unsigned int
  */
 void drm_fb_xrgb8888_to_mono(struct iosys_map *dst, const unsigned int *dst_pitch,
 			     const struct iosys_map *src, const struct drm_framebuffer *fb,
-			     const struct drm_rect *clip)
+			     const struct drm_rect *clip, struct drm_format_conv_state *state)
 {
 	static const unsigned int default_dst_pitch[DRM_FORMAT_MAX_PLANES] = {
 		0, 0, 0, 0
@@ -1042,7 +1153,7 @@ void drm_fb_xrgb8888_to_mono(struct iosys_map *dst, const unsigned int *dst_pitc
 	 * Allocate a buffer to be used for both copying from the cma
 	 * memory and to store the intermediate grayscale line pixels.
 	 */
-	src32 = kmalloc(len_src32 + linepixels, GFP_KERNEL);
+	src32 = drm_format_conv_state_reserve(state, len_src32 + linepixels, GFP_KERNEL);
 	if (!src32)
 		return;
 
@@ -1056,8 +1167,6 @@ void drm_fb_xrgb8888_to_mono(struct iosys_map *dst, const unsigned int *dst_pitc
 		vaddr += fb->pitches[0];
 		mono += dst_pitch_0;
 	}
-
-	kfree(src32);
 }
 EXPORT_SYMBOL(drm_fb_xrgb8888_to_mono);
 
diff --git a/drivers/gpu/drm/drm_framebuffer.c b/drivers/gpu/drm/drm_framebuffer.c
index d3ba0698b84b..09e289fca5c3 100644
--- a/drivers/gpu/drm/drm_framebuffer.c
+++ b/drivers/gpu/drm/drm_framebuffer.c
@@ -394,6 +394,31 @@ static void drm_mode_rmfb_work_fn(struct work_struct *w)
 	}
 }
 
+static int drm_mode_closefb(struct drm_framebuffer *fb,
+			    struct drm_file *file_priv)
+{
+	struct drm_framebuffer *fbl;
+	bool found = false;
+
+	mutex_lock(&file_priv->fbs_lock);
+	list_for_each_entry(fbl, &file_priv->fbs, filp_head)
+		if (fb == fbl)
+			found = true;
+
+	if (!found) {
+		mutex_unlock(&file_priv->fbs_lock);
+		return -ENOENT;
+	}
+
+	list_del_init(&fb->filp_head);
+	mutex_unlock(&file_priv->fbs_lock);
+
+	/* Drop the reference that was stored in the fbs list */
+	drm_framebuffer_put(fb);
+
+	return 0;
+}
+
 /**
  * drm_mode_rmfb - remove an FB from the configuration
  * @dev: drm device
@@ -410,9 +435,8 @@ static void drm_mode_rmfb_work_fn(struct work_struct *w)
 int drm_mode_rmfb(struct drm_device *dev, u32 fb_id,
 		  struct drm_file *file_priv)
 {
-	struct drm_framebuffer *fb = NULL;
-	struct drm_framebuffer *fbl = NULL;
-	int found = 0;
+	struct drm_framebuffer *fb;
+	int ret;
 
 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
 		return -EOPNOTSUPP;
@@ -421,24 +445,13 @@ int drm_mode_rmfb(struct drm_device *dev, u32 fb_id,
 	if (!fb)
 		return -ENOENT;
 
-	mutex_lock(&file_priv->fbs_lock);
-	list_for_each_entry(fbl, &file_priv->fbs, filp_head)
-		if (fb == fbl)
-			found = 1;
-	if (!found) {
-		mutex_unlock(&file_priv->fbs_lock);
-		goto fail_unref;
+	ret = drm_mode_closefb(fb, file_priv);
+	if (ret != 0) {
+		drm_framebuffer_put(fb);
+		return ret;
 	}
 
-	list_del_init(&fb->filp_head);
-	mutex_unlock(&file_priv->fbs_lock);
-
-	/* drop the reference we picked up in framebuffer lookup */
-	drm_framebuffer_put(fb);
-
 	/*
-	 * we now own the reference that was stored in the fbs list
-	 *
 	 * drm_framebuffer_remove may fail with -EINTR on pending signals,
 	 * so run this in a separate stack as there's no way to correctly
 	 * handle this after the fb is already removed from the lookup table.
@@ -457,10 +470,6 @@ int drm_mode_rmfb(struct drm_device *dev, u32 fb_id,
 		drm_framebuffer_put(fb);
 
 	return 0;
-
-fail_unref:
-	drm_framebuffer_put(fb);
-	return -ENOENT;
 }
 
 int drm_mode_rmfb_ioctl(struct drm_device *dev,
@@ -471,6 +480,28 @@ int drm_mode_rmfb_ioctl(struct drm_device *dev,
 	return drm_mode_rmfb(dev, *fb_id, file_priv);
 }
 
+int drm_mode_closefb_ioctl(struct drm_device *dev,
+			   void *data, struct drm_file *file_priv)
+{
+	struct drm_mode_closefb *r = data;
+	struct drm_framebuffer *fb;
+	int ret;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EOPNOTSUPP;
+
+	if (r->pad)
+		return -EINVAL;
+
+	fb = drm_framebuffer_lookup(dev, file_priv, r->fb_id);
+	if (!fb)
+		return -ENOENT;
+
+	ret = drm_mode_closefb(fb, file_priv);
+	drm_framebuffer_put(fb);
+	return ret;
+}
+
 /**
  * drm_mode_getfb - get FB info
  * @dev: drm device for the ioctl
diff --git a/drivers/gpu/drm/drm_gem_atomic_helper.c b/drivers/gpu/drm/drm_gem_atomic_helper.c
index 5d4b9cd077f7..e440f458b663 100644
--- a/drivers/gpu/drm/drm_gem_atomic_helper.c
+++ b/drivers/gpu/drm/drm_gem_atomic_helper.c
@@ -218,7 +218,14 @@ void
 __drm_gem_duplicate_shadow_plane_state(struct drm_plane *plane,
 				       struct drm_shadow_plane_state *new_shadow_plane_state)
 {
+	struct drm_plane_state *plane_state = plane->state;
+	struct drm_shadow_plane_state *shadow_plane_state =
+		to_drm_shadow_plane_state(plane_state);
+
 	__drm_atomic_helper_plane_duplicate_state(plane, &new_shadow_plane_state->base);
+
+	drm_format_conv_state_copy(&shadow_plane_state->fmtcnv_state,
+				   &new_shadow_plane_state->fmtcnv_state);
 }
 EXPORT_SYMBOL(__drm_gem_duplicate_shadow_plane_state);
 
@@ -266,6 +273,7 @@ EXPORT_SYMBOL(drm_gem_duplicate_shadow_plane_state);
  */
 void __drm_gem_destroy_shadow_plane_state(struct drm_shadow_plane_state *shadow_plane_state)
 {
+	drm_format_conv_state_release(&shadow_plane_state->fmtcnv_state);
 	__drm_atomic_helper_plane_destroy_state(&shadow_plane_state->base);
 }
 EXPORT_SYMBOL(__drm_gem_destroy_shadow_plane_state);
@@ -302,6 +310,7 @@ void __drm_gem_reset_shadow_plane(struct drm_plane *plane,
 				  struct drm_shadow_plane_state *shadow_plane_state)
 {
 	__drm_atomic_helper_plane_reset(plane, &shadow_plane_state->base);
+	drm_format_conv_state_init(&shadow_plane_state->fmtcnv_state);
 }
 EXPORT_SYMBOL(__drm_gem_reset_shadow_plane);
 
diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c
index 08c088319652..54f5e8851de5 100644
--- a/drivers/gpu/drm/drm_gpuvm.c
+++ b/drivers/gpu/drm/drm_gpuvm.c
@@ -61,6 +61,42 @@
  * contained within struct drm_gpuva already. Hence, for inserting &drm_gpuva
  * entries from within dma-fence signalling critical sections it is enough to
  * pre-allocate the &drm_gpuva structures.
+ *
+ * &drm_gem_objects which are private to a single VM can share a common
+ * &dma_resv in order to improve locking efficiency (e.g. with &drm_exec).
+ * For this purpose drivers must pass a &drm_gem_object to drm_gpuvm_init(), in
+ * the following called 'resv object', which serves as the container of the
+ * GPUVM's shared &dma_resv. This resv object can be a driver specific
+ * &drm_gem_object, such as the &drm_gem_object containing the root page table,
+ * but it can also be a 'dummy' object, which can be allocated with
+ * drm_gpuvm_resv_object_alloc().
+ *
+ * In order to connect a struct drm_gpuva its backing &drm_gem_object each
+ * &drm_gem_object maintains a list of &drm_gpuvm_bo structures, and each
+ * &drm_gpuvm_bo contains a list of &drm_gpuva structures.
+ *
+ * A &drm_gpuvm_bo is an abstraction that represents a combination of a
+ * &drm_gpuvm and a &drm_gem_object. Every such combination should be unique.
+ * This is ensured by the API through drm_gpuvm_bo_obtain() and
+ * drm_gpuvm_bo_obtain_prealloc() which first look into the corresponding
+ * &drm_gem_object list of &drm_gpuvm_bos for an existing instance of this
+ * particular combination. If not existent a new instance is created and linked
+ * to the &drm_gem_object.
+ *
+ * &drm_gpuvm_bo structures, since unique for a given &drm_gpuvm, are also used
+ * as entry for the &drm_gpuvm's lists of external and evicted objects. Those
+ * lists are maintained in order to accelerate locking of dma-resv locks and
+ * validation of evicted objects bound in a &drm_gpuvm. For instance, all
+ * &drm_gem_object's &dma_resv of a given &drm_gpuvm can be locked by calling
+ * drm_gpuvm_exec_lock(). Once locked drivers can call drm_gpuvm_validate() in
+ * order to validate all evicted &drm_gem_objects. It is also possible to lock
+ * additional &drm_gem_objects by providing the corresponding parameters to
+ * drm_gpuvm_exec_lock() as well as open code the &drm_exec loop while making
+ * use of helper functions such as drm_gpuvm_prepare_range() or
+ * drm_gpuvm_prepare_objects().
+ *
+ * Every bound &drm_gem_object is treated as external object when its &dma_resv
+ * structure is different than the &drm_gpuvm's common &dma_resv structure.
  */
 
 /**
@@ -386,21 +422,42 @@
 /**
  * DOC: Locking
  *
- * Generally, the GPU VA manager does not take care of locking itself, it is
- * the drivers responsibility to take care about locking. Drivers might want to
- * protect the following operations: inserting, removing and iterating
- * &drm_gpuva objects as well as generating all kinds of operations, such as
- * split / merge or prefetch.
- *
- * The GPU VA manager also does not take care of the locking of the backing
- * &drm_gem_object buffers GPU VA lists by itself; drivers are responsible to
- * enforce mutual exclusion using either the GEMs dma_resv lock or alternatively
- * a driver specific external lock. For the latter see also
- * drm_gem_gpuva_set_lock().
- *
- * However, the GPU VA manager contains lockdep checks to ensure callers of its
- * API hold the corresponding lock whenever the &drm_gem_objects GPU VA list is
- * accessed by functions such as drm_gpuva_link() or drm_gpuva_unlink().
+ * In terms of managing &drm_gpuva entries DRM GPUVM does not take care of
+ * locking itself, it is the drivers responsibility to take care about locking.
+ * Drivers might want to protect the following operations: inserting, removing
+ * and iterating &drm_gpuva objects as well as generating all kinds of
+ * operations, such as split / merge or prefetch.
+ *
+ * DRM GPUVM also does not take care of the locking of the backing
+ * &drm_gem_object buffers GPU VA lists and &drm_gpuvm_bo abstractions by
+ * itself; drivers are responsible to enforce mutual exclusion using either the
+ * GEMs dma_resv lock or alternatively a driver specific external lock. For the
+ * latter see also drm_gem_gpuva_set_lock().
+ *
+ * However, DRM GPUVM contains lockdep checks to ensure callers of its API hold
+ * the corresponding lock whenever the &drm_gem_objects GPU VA list is accessed
+ * by functions such as drm_gpuva_link() or drm_gpuva_unlink(), but also
+ * drm_gpuvm_bo_obtain() and drm_gpuvm_bo_put().
+ *
+ * The latter is required since on creation and destruction of a &drm_gpuvm_bo
+ * the &drm_gpuvm_bo is attached / removed from the &drm_gem_objects gpuva list.
+ * Subsequent calls to drm_gpuvm_bo_obtain() for the same &drm_gpuvm and
+ * &drm_gem_object must be able to observe previous creations and destructions
+ * of &drm_gpuvm_bos in order to keep instances unique.
+ *
+ * The &drm_gpuvm's lists for keeping track of external and evicted objects are
+ * protected against concurrent insertion / removal and iteration internally.
+ *
+ * However, drivers still need ensure to protect concurrent calls to functions
+ * iterating those lists, namely drm_gpuvm_prepare_objects() and
+ * drm_gpuvm_validate().
+ *
+ * Alternatively, drivers can set the &DRM_GPUVM_RESV_PROTECTED flag to indicate
+ * that the corresponding &dma_resv locks are held in order to protect the
+ * lists. If &DRM_GPUVM_RESV_PROTECTED is set, internal locking is disabled and
+ * the corresponding lockdep checks are enabled. This is an optimization for
+ * drivers which are capable of taking the corresponding &dma_resv locks and
+ * hence do not require internal locking.
  */
 
 /**
@@ -430,6 +487,7 @@
  *	{
  *		struct drm_gpuva_ops *ops;
  *		struct drm_gpuva_op *op
+ *		struct drm_gpuvm_bo *vm_bo;
  *
  *		driver_lock_va_space();
  *		ops = drm_gpuvm_sm_map_ops_create(gpuvm, addr, range,
@@ -437,6 +495,10 @@
  *		if (IS_ERR(ops))
  *			return PTR_ERR(ops);
  *
+ *		vm_bo = drm_gpuvm_bo_obtain(gpuvm, obj);
+ *		if (IS_ERR(vm_bo))
+ *			return PTR_ERR(vm_bo);
+ *
  *		drm_gpuva_for_each_op(op, ops) {
  *			struct drm_gpuva *va;
  *
@@ -449,7 +511,7 @@
  *
  *				driver_vm_map();
  *				drm_gpuva_map(gpuvm, va, &op->map);
- *				drm_gpuva_link(va);
+ *				drm_gpuva_link(va, vm_bo);
  *
  *				break;
  *			case DRM_GPUVA_OP_REMAP: {
@@ -476,11 +538,11 @@
  *				driver_vm_remap();
  *				drm_gpuva_remap(prev, next, &op->remap);
  *
- *				drm_gpuva_unlink(va);
  *				if (prev)
- *					drm_gpuva_link(prev);
+ *					drm_gpuva_link(prev, va->vm_bo);
  *				if (next)
- *					drm_gpuva_link(next);
+ *					drm_gpuva_link(next, va->vm_bo);
+ *				drm_gpuva_unlink(va);
  *
  *				break;
  *			}
@@ -496,6 +558,7 @@
  *				break;
  *			}
  *		}
+ *		drm_gpuvm_bo_put(vm_bo);
  *		driver_unlock_va_space();
  *
  *		return 0;
@@ -505,6 +568,7 @@
  *
  *	struct driver_context {
  *		struct drm_gpuvm *gpuvm;
+ *		struct drm_gpuvm_bo *vm_bo;
  *		struct drm_gpuva *new_va;
  *		struct drm_gpuva *prev_va;
  *		struct drm_gpuva *next_va;
@@ -525,6 +589,7 @@
  *				  struct drm_gem_object *obj, u64 offset)
  *	{
  *		struct driver_context ctx;
+ *		struct drm_gpuvm_bo *vm_bo;
  *		struct drm_gpuva_ops *ops;
  *		struct drm_gpuva_op *op;
  *		int ret = 0;
@@ -534,16 +599,23 @@
  *		ctx.new_va = kzalloc(sizeof(*ctx.new_va), GFP_KERNEL);
  *		ctx.prev_va = kzalloc(sizeof(*ctx.prev_va), GFP_KERNEL);
  *		ctx.next_va = kzalloc(sizeof(*ctx.next_va), GFP_KERNEL);
- *		if (!ctx.new_va || !ctx.prev_va || !ctx.next_va) {
+ *		ctx.vm_bo = drm_gpuvm_bo_create(gpuvm, obj);
+ *		if (!ctx.new_va || !ctx.prev_va || !ctx.next_va || !vm_bo) {
  *			ret = -ENOMEM;
  *			goto out;
  *		}
  *
+ *		// Typically protected with a driver specific GEM gpuva lock
+ *		// used in the fence signaling path for drm_gpuva_link() and
+ *		// drm_gpuva_unlink(), hence pre-allocate.
+ *		ctx.vm_bo = drm_gpuvm_bo_obtain_prealloc(ctx.vm_bo);
+ *
  *		driver_lock_va_space();
  *		ret = drm_gpuvm_sm_map(gpuvm, &ctx, addr, range, obj, offset);
  *		driver_unlock_va_space();
  *
  *	out:
+ *		drm_gpuvm_bo_put(ctx.vm_bo);
  *		kfree(ctx.new_va);
  *		kfree(ctx.prev_va);
  *		kfree(ctx.next_va);
@@ -556,7 +628,7 @@
  *
  *		drm_gpuva_map(ctx->vm, ctx->new_va, &op->map);
  *
- *		drm_gpuva_link(ctx->new_va);
+ *		drm_gpuva_link(ctx->new_va, ctx->vm_bo);
  *
  *		// prevent the new GPUVA from being freed in
  *		// driver_mapping_create()
@@ -568,22 +640,23 @@
  *	int driver_gpuva_remap(struct drm_gpuva_op *op, void *__ctx)
  *	{
  *		struct driver_context *ctx = __ctx;
+ *		struct drm_gpuva *va = op->remap.unmap->va;
  *
  *		drm_gpuva_remap(ctx->prev_va, ctx->next_va, &op->remap);
  *
- *		drm_gpuva_unlink(op->remap.unmap->va);
- *		kfree(op->remap.unmap->va);
- *
  *		if (op->remap.prev) {
- *			drm_gpuva_link(ctx->prev_va);
+ *			drm_gpuva_link(ctx->prev_va, va->vm_bo);
  *			ctx->prev_va = NULL;
  *		}
  *
  *		if (op->remap.next) {
- *			drm_gpuva_link(ctx->next_va);
+ *			drm_gpuva_link(ctx->next_va, va->vm_bo);
  *			ctx->next_va = NULL;
  *		}
  *
+ *		drm_gpuva_unlink(va);
+ *		kfree(va);
+ *
  *		return 0;
  *	}
  *
@@ -597,6 +670,201 @@
  *	}
  */
 
+/**
+ * get_next_vm_bo_from_list() - get the next vm_bo element
+ * @__gpuvm: the &drm_gpuvm
+ * @__list_name: the name of the list we're iterating on
+ * @__local_list: a pointer to the local list used to store already iterated items
+ * @__prev_vm_bo: the previous element we got from get_next_vm_bo_from_list()
+ *
+ * This helper is here to provide lockless list iteration. Lockless as in, the
+ * iterator releases the lock immediately after picking the first element from
+ * the list, so list insertion deletion can happen concurrently.
+ *
+ * Elements popped from the original list are kept in a local list, so removal
+ * and is_empty checks can still happen while we're iterating the list.
+ */
+#define get_next_vm_bo_from_list(__gpuvm, __list_name, __local_list, __prev_vm_bo)	\
+	({										\
+		struct drm_gpuvm_bo *__vm_bo = NULL;					\
+											\
+		drm_gpuvm_bo_put(__prev_vm_bo);						\
+											\
+		spin_lock(&(__gpuvm)->__list_name.lock);				\
+		if (!(__gpuvm)->__list_name.local_list)					\
+			(__gpuvm)->__list_name.local_list = __local_list;		\
+		else									\
+			drm_WARN_ON((__gpuvm)->drm,					\
+				    (__gpuvm)->__list_name.local_list != __local_list);	\
+											\
+		while (!list_empty(&(__gpuvm)->__list_name.list)) {			\
+			__vm_bo = list_first_entry(&(__gpuvm)->__list_name.list,	\
+						   struct drm_gpuvm_bo,			\
+						   list.entry.__list_name);		\
+			if (kref_get_unless_zero(&__vm_bo->kref)) {			\
+				list_move_tail(&(__vm_bo)->list.entry.__list_name,	\
+					       __local_list);				\
+				break;							\
+			} else {							\
+				list_del_init(&(__vm_bo)->list.entry.__list_name);	\
+				__vm_bo = NULL;						\
+			}								\
+		}									\
+		spin_unlock(&(__gpuvm)->__list_name.lock);				\
+											\
+		__vm_bo;								\
+	})
+
+/**
+ * for_each_vm_bo_in_list() - internal vm_bo list iterator
+ * @__gpuvm: the &drm_gpuvm
+ * @__list_name: the name of the list we're iterating on
+ * @__local_list: a pointer to the local list used to store already iterated items
+ * @__vm_bo: the struct drm_gpuvm_bo to assign in each iteration step
+ *
+ * This helper is here to provide lockless list iteration. Lockless as in, the
+ * iterator releases the lock immediately after picking the first element from the
+ * list, hence list insertion and deletion can happen concurrently.
+ *
+ * It is not allowed to re-assign the vm_bo pointer from inside this loop.
+ *
+ * Typical use:
+ *
+ *	struct drm_gpuvm_bo *vm_bo;
+ *	LIST_HEAD(my_local_list);
+ *
+ *	ret = 0;
+ *	for_each_vm_bo_in_list(gpuvm, <list_name>, &my_local_list, vm_bo) {
+ *		ret = do_something_with_vm_bo(..., vm_bo);
+ *		if (ret)
+ *			break;
+ *	}
+ *	// Drop ref in case we break out of the loop.
+ *	drm_gpuvm_bo_put(vm_bo);
+ *	restore_vm_bo_list(gpuvm, <list_name>, &my_local_list);
+ *
+ *
+ * Only used for internal list iterations, not meant to be exposed to the outside
+ * world.
+ */
+#define for_each_vm_bo_in_list(__gpuvm, __list_name, __local_list, __vm_bo)	\
+	for (__vm_bo = get_next_vm_bo_from_list(__gpuvm, __list_name,		\
+						__local_list, NULL);		\
+	     __vm_bo;								\
+	     __vm_bo = get_next_vm_bo_from_list(__gpuvm, __list_name,		\
+						__local_list, __vm_bo))
+
+static void
+__restore_vm_bo_list(struct drm_gpuvm *gpuvm, spinlock_t *lock,
+		     struct list_head *list, struct list_head **local_list)
+{
+	/* Merge back the two lists, moving local list elements to the
+	 * head to preserve previous ordering, in case it matters.
+	 */
+	spin_lock(lock);
+	if (*local_list) {
+		list_splice(*local_list, list);
+		*local_list = NULL;
+	}
+	spin_unlock(lock);
+}
+
+/**
+ * restore_vm_bo_list() - move vm_bo elements back to their original list
+ * @__gpuvm: the &drm_gpuvm
+ * @__list_name: the name of the list we're iterating on
+ *
+ * When we're done iterating a vm_bo list, we should call restore_vm_bo_list()
+ * to restore the original state and let new iterations take place.
+ */
+#define restore_vm_bo_list(__gpuvm, __list_name)			\
+	__restore_vm_bo_list((__gpuvm), &(__gpuvm)->__list_name.lock,	\
+			     &(__gpuvm)->__list_name.list,		\
+			     &(__gpuvm)->__list_name.local_list)
+
+static void
+cond_spin_lock(spinlock_t *lock, bool cond)
+{
+	if (cond)
+		spin_lock(lock);
+}
+
+static void
+cond_spin_unlock(spinlock_t *lock, bool cond)
+{
+	if (cond)
+		spin_unlock(lock);
+}
+
+static void
+__drm_gpuvm_bo_list_add(struct drm_gpuvm *gpuvm, spinlock_t *lock,
+			struct list_head *entry, struct list_head *list)
+{
+	cond_spin_lock(lock, !!lock);
+	if (list_empty(entry))
+		list_add_tail(entry, list);
+	cond_spin_unlock(lock, !!lock);
+}
+
+/**
+ * drm_gpuvm_bo_list_add() - insert a vm_bo into the given list
+ * @__vm_bo: the &drm_gpuvm_bo
+ * @__list_name: the name of the list to insert into
+ * @__lock: whether to lock with the internal spinlock
+ *
+ * Inserts the given @__vm_bo into the list specified by @__list_name.
+ */
+#define drm_gpuvm_bo_list_add(__vm_bo, __list_name, __lock)			\
+	__drm_gpuvm_bo_list_add((__vm_bo)->vm,					\
+				__lock ? &(__vm_bo)->vm->__list_name.lock :	\
+					 NULL,					\
+				&(__vm_bo)->list.entry.__list_name,		\
+				&(__vm_bo)->vm->__list_name.list)
+
+static void
+__drm_gpuvm_bo_list_del(struct drm_gpuvm *gpuvm, spinlock_t *lock,
+			struct list_head *entry, bool init)
+{
+	cond_spin_lock(lock, !!lock);
+	if (init) {
+		if (!list_empty(entry))
+			list_del_init(entry);
+	} else {
+		list_del(entry);
+	}
+	cond_spin_unlock(lock, !!lock);
+}
+
+/**
+ * drm_gpuvm_bo_list_del_init() - remove a vm_bo from the given list
+ * @__vm_bo: the &drm_gpuvm_bo
+ * @__list_name: the name of the list to insert into
+ * @__lock: whether to lock with the internal spinlock
+ *
+ * Removes the given @__vm_bo from the list specified by @__list_name.
+ */
+#define drm_gpuvm_bo_list_del_init(__vm_bo, __list_name, __lock)		\
+	__drm_gpuvm_bo_list_del((__vm_bo)->vm,					\
+				__lock ? &(__vm_bo)->vm->__list_name.lock :	\
+					 NULL,					\
+				&(__vm_bo)->list.entry.__list_name,		\
+				true)
+
+/**
+ * drm_gpuvm_bo_list_del() - remove a vm_bo from the given list
+ * @__vm_bo: the &drm_gpuvm_bo
+ * @__list_name: the name of the list to insert into
+ * @__lock: whether to lock with the internal spinlock
+ *
+ * Removes the given @__vm_bo from the list specified by @__list_name.
+ */
+#define drm_gpuvm_bo_list_del(__vm_bo, __list_name, __lock)			\
+	__drm_gpuvm_bo_list_del((__vm_bo)->vm,					\
+				__lock ? &(__vm_bo)->vm->__list_name.lock :	\
+					 NULL,					\
+				&(__vm_bo)->list.entry.__list_name,		\
+				false)
+
 #define to_drm_gpuva(__node)	container_of((__node), struct drm_gpuva, rb.node)
 
 #define GPUVA_START(node) ((node)->va.addr)
@@ -618,8 +886,14 @@ drm_gpuvm_check_overflow(u64 addr, u64 range)
 {
 	u64 end;
 
-	return WARN(check_add_overflow(addr, range, &end),
-		    "GPUVA address limited to %zu bytes.\n", sizeof(end));
+	return check_add_overflow(addr, range, &end);
+}
+
+static bool
+drm_gpuvm_warn_check_overflow(struct drm_gpuvm *gpuvm, u64 addr, u64 range)
+{
+	return drm_WARN(gpuvm->drm, drm_gpuvm_check_overflow(addr, range),
+			"GPUVA address limited to %zu bytes.\n", sizeof(addr));
 }
 
 static bool
@@ -643,7 +917,18 @@ drm_gpuvm_in_kernel_node(struct drm_gpuvm *gpuvm, u64 addr, u64 range)
 	return krange && addr < kend && kstart < end;
 }
 
-static bool
+/**
+ * drm_gpuvm_range_valid() - checks whether the given range is valid for the
+ * given &drm_gpuvm
+ * @gpuvm: the GPUVM to check the range for
+ * @addr: the base address
+ * @range: the range starting from the base address
+ *
+ * Checks whether the range is within the GPUVM's managed boundaries.
+ *
+ * Returns: true for a valid range, false otherwise
+ */
+bool
 drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm,
 		      u64 addr, u64 range)
 {
@@ -651,11 +936,52 @@ drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm,
 	       drm_gpuvm_in_mm_range(gpuvm, addr, range) &&
 	       !drm_gpuvm_in_kernel_node(gpuvm, addr, range);
 }
+EXPORT_SYMBOL_GPL(drm_gpuvm_range_valid);
+
+static void
+drm_gpuvm_gem_object_free(struct drm_gem_object *obj)
+{
+	drm_gem_object_release(obj);
+	kfree(obj);
+}
+
+static const struct drm_gem_object_funcs drm_gpuvm_object_funcs = {
+	.free = drm_gpuvm_gem_object_free,
+};
+
+/**
+ * drm_gpuvm_resv_object_alloc() - allocate a dummy &drm_gem_object
+ * @drm: the drivers &drm_device
+ *
+ * Allocates a dummy &drm_gem_object which can be passed to drm_gpuvm_init() in
+ * order to serve as root GEM object providing the &drm_resv shared across
+ * &drm_gem_objects local to a single GPUVM.
+ *
+ * Returns: the &drm_gem_object on success, NULL on failure
+ */
+struct drm_gem_object *
+drm_gpuvm_resv_object_alloc(struct drm_device *drm)
+{
+	struct drm_gem_object *obj;
+
+	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
+	if (!obj)
+		return NULL;
+
+	obj->funcs = &drm_gpuvm_object_funcs;
+	drm_gem_private_object_init(drm, obj, 0);
+
+	return obj;
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_resv_object_alloc);
 
 /**
  * drm_gpuvm_init() - initialize a &drm_gpuvm
  * @gpuvm: pointer to the &drm_gpuvm to initialize
  * @name: the name of the GPU VA space
+ * @flags: the &drm_gpuvm_flags for this GPUVM
+ * @drm: the &drm_device this VM resides in
+ * @r_obj: the resv &drm_gem_object providing the GPUVM's common &dma_resv
  * @start_offset: the start offset of the GPU VA space
  * @range: the size of the GPU VA space
  * @reserve_offset: the start of the kernel reserved GPU VA area
@@ -668,8 +994,10 @@ drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm,
  * &name is expected to be managed by the surrounding driver structures.
  */
 void
-drm_gpuvm_init(struct drm_gpuvm *gpuvm,
-	       const char *name,
+drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name,
+	       enum drm_gpuvm_flags flags,
+	       struct drm_device *drm,
+	       struct drm_gem_object *r_obj,
 	       u64 start_offset, u64 range,
 	       u64 reserve_offset, u64 reserve_range,
 	       const struct drm_gpuvm_ops *ops)
@@ -677,45 +1005,676 @@ drm_gpuvm_init(struct drm_gpuvm *gpuvm,
 	gpuvm->rb.tree = RB_ROOT_CACHED;
 	INIT_LIST_HEAD(&gpuvm->rb.list);
 
-	drm_gpuvm_check_overflow(start_offset, range);
-	gpuvm->mm_start = start_offset;
-	gpuvm->mm_range = range;
+	INIT_LIST_HEAD(&gpuvm->extobj.list);
+	spin_lock_init(&gpuvm->extobj.lock);
+
+	INIT_LIST_HEAD(&gpuvm->evict.list);
+	spin_lock_init(&gpuvm->evict.lock);
+
+	kref_init(&gpuvm->kref);
 
 	gpuvm->name = name ? name : "unknown";
+	gpuvm->flags = flags;
 	gpuvm->ops = ops;
+	gpuvm->drm = drm;
+	gpuvm->r_obj = r_obj;
 
-	memset(&gpuvm->kernel_alloc_node, 0, sizeof(struct drm_gpuva));
+	drm_gem_object_get(r_obj);
+
+	drm_gpuvm_warn_check_overflow(gpuvm, start_offset, range);
+	gpuvm->mm_start = start_offset;
+	gpuvm->mm_range = range;
 
+	memset(&gpuvm->kernel_alloc_node, 0, sizeof(struct drm_gpuva));
 	if (reserve_range) {
 		gpuvm->kernel_alloc_node.va.addr = reserve_offset;
 		gpuvm->kernel_alloc_node.va.range = reserve_range;
 
-		if (likely(!drm_gpuvm_check_overflow(reserve_offset,
-						     reserve_range)))
+		if (likely(!drm_gpuvm_warn_check_overflow(gpuvm, reserve_offset,
+							  reserve_range)))
 			__drm_gpuva_insert(gpuvm, &gpuvm->kernel_alloc_node);
 	}
 }
 EXPORT_SYMBOL_GPL(drm_gpuvm_init);
 
+static void
+drm_gpuvm_fini(struct drm_gpuvm *gpuvm)
+{
+	gpuvm->name = NULL;
+
+	if (gpuvm->kernel_alloc_node.va.range)
+		__drm_gpuva_remove(&gpuvm->kernel_alloc_node);
+
+	drm_WARN(gpuvm->drm, !RB_EMPTY_ROOT(&gpuvm->rb.tree.rb_root),
+		 "GPUVA tree is not empty, potentially leaking memory.\n");
+
+	drm_WARN(gpuvm->drm, !list_empty(&gpuvm->extobj.list),
+		 "Extobj list should be empty.\n");
+	drm_WARN(gpuvm->drm, !list_empty(&gpuvm->evict.list),
+		 "Evict list should be empty.\n");
+
+	drm_gem_object_put(gpuvm->r_obj);
+}
+
+static void
+drm_gpuvm_free(struct kref *kref)
+{
+	struct drm_gpuvm *gpuvm = container_of(kref, struct drm_gpuvm, kref);
+
+	drm_gpuvm_fini(gpuvm);
+
+	if (drm_WARN_ON(gpuvm->drm, !gpuvm->ops->vm_free))
+		return;
+
+	gpuvm->ops->vm_free(gpuvm);
+}
+
 /**
- * drm_gpuvm_destroy() - cleanup a &drm_gpuvm
- * @gpuvm: pointer to the &drm_gpuvm to clean up
+ * drm_gpuvm_put() - drop a struct drm_gpuvm reference
+ * @gpuvm: the &drm_gpuvm to release the reference of
+ *
+ * This releases a reference to @gpuvm.
  *
- * Note that it is a bug to call this function on a manager that still
- * holds GPU VA mappings.
+ * This function may be called from atomic context.
  */
 void
-drm_gpuvm_destroy(struct drm_gpuvm *gpuvm)
+drm_gpuvm_put(struct drm_gpuvm *gpuvm)
 {
-	gpuvm->name = NULL;
+	if (gpuvm)
+		kref_put(&gpuvm->kref, drm_gpuvm_free);
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_put);
 
-	if (gpuvm->kernel_alloc_node.va.range)
-		__drm_gpuva_remove(&gpuvm->kernel_alloc_node);
+static int
+__drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm,
+			    struct drm_exec *exec,
+			    unsigned int num_fences)
+{
+	struct drm_gpuvm_bo *vm_bo;
+	LIST_HEAD(extobjs);
+	int ret = 0;
+
+	for_each_vm_bo_in_list(gpuvm, extobj, &extobjs, vm_bo) {
+		ret = drm_exec_prepare_obj(exec, vm_bo->obj, num_fences);
+		if (ret)
+			break;
+	}
+	/* Drop ref in case we break out of the loop. */
+	drm_gpuvm_bo_put(vm_bo);
+	restore_vm_bo_list(gpuvm, extobj);
+
+	return ret;
+}
+
+static int
+drm_gpuvm_prepare_objects_locked(struct drm_gpuvm *gpuvm,
+				 struct drm_exec *exec,
+				 unsigned int num_fences)
+{
+	struct drm_gpuvm_bo *vm_bo;
+	int ret = 0;
+
+	drm_gpuvm_resv_assert_held(gpuvm);
+	list_for_each_entry(vm_bo, &gpuvm->extobj.list, list.entry.extobj) {
+		ret = drm_exec_prepare_obj(exec, vm_bo->obj, num_fences);
+		if (ret)
+			break;
+
+		if (vm_bo->evicted)
+			drm_gpuvm_bo_list_add(vm_bo, evict, false);
+	}
+
+	return ret;
+}
+
+/**
+ * drm_gpuvm_prepare_objects() - prepare all assoiciated BOs
+ * @gpuvm: the &drm_gpuvm
+ * @exec: the &drm_exec locking context
+ * @num_fences: the amount of &dma_fences to reserve
+ *
+ * Calls drm_exec_prepare_obj() for all &drm_gem_objects the given
+ * &drm_gpuvm contains mappings of.
+ *
+ * Using this function directly, it is the drivers responsibility to call
+ * drm_exec_init() and drm_exec_fini() accordingly.
+ *
+ * Note: This function is safe against concurrent insertion and removal of
+ * external objects, however it is not safe against concurrent usage itself.
+ *
+ * Drivers need to make sure to protect this case with either an outer VM lock
+ * or by calling drm_gpuvm_prepare_vm() before this function within the
+ * drm_exec_until_all_locked() loop, such that the GPUVM's dma-resv lock ensures
+ * mutual exclusion.
+ *
+ * Returns: 0 on success, negative error code on failure.
+ */
+int
+drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm,
+			  struct drm_exec *exec,
+			  unsigned int num_fences)
+{
+	if (drm_gpuvm_resv_protected(gpuvm))
+		return drm_gpuvm_prepare_objects_locked(gpuvm, exec,
+							num_fences);
+	else
+		return __drm_gpuvm_prepare_objects(gpuvm, exec, num_fences);
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_objects);
+
+/**
+ * drm_gpuvm_prepare_range() - prepare all BOs mapped within a given range
+ * @gpuvm: the &drm_gpuvm
+ * @exec: the &drm_exec locking context
+ * @addr: the start address within the VA space
+ * @range: the range to iterate within the VA space
+ * @num_fences: the amount of &dma_fences to reserve
+ *
+ * Calls drm_exec_prepare_obj() for all &drm_gem_objects mapped between @addr
+ * and @addr + @range.
+ *
+ * Returns: 0 on success, negative error code on failure.
+ */
+int
+drm_gpuvm_prepare_range(struct drm_gpuvm *gpuvm, struct drm_exec *exec,
+			u64 addr, u64 range, unsigned int num_fences)
+{
+	struct drm_gpuva *va;
+	u64 end = addr + range;
+	int ret;
+
+	drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) {
+		struct drm_gem_object *obj = va->gem.obj;
+
+		ret = drm_exec_prepare_obj(exec, obj, num_fences);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_range);
+
+/**
+ * drm_gpuvm_exec_lock() - lock all dma-resv of all assoiciated BOs
+ * @vm_exec: the &drm_gpuvm_exec wrapper
+ *
+ * Acquires all dma-resv locks of all &drm_gem_objects the given
+ * &drm_gpuvm contains mappings of.
+ *
+ * Addionally, when calling this function with struct drm_gpuvm_exec::extra
+ * being set the driver receives the given @fn callback to lock additional
+ * dma-resv in the context of the &drm_gpuvm_exec instance. Typically, drivers
+ * would call drm_exec_prepare_obj() from within this callback.
+ *
+ * Returns: 0 on success, negative error code on failure.
+ */
+int
+drm_gpuvm_exec_lock(struct drm_gpuvm_exec *vm_exec)
+{
+	struct drm_gpuvm *gpuvm = vm_exec->vm;
+	struct drm_exec *exec = &vm_exec->exec;
+	unsigned int num_fences = vm_exec->num_fences;
+	int ret;
+
+	drm_exec_init(exec, vm_exec->flags);
+
+	drm_exec_until_all_locked(exec) {
+		ret = drm_gpuvm_prepare_vm(gpuvm, exec, num_fences);
+		drm_exec_retry_on_contention(exec);
+		if (ret)
+			goto err;
+
+		ret = drm_gpuvm_prepare_objects(gpuvm, exec, num_fences);
+		drm_exec_retry_on_contention(exec);
+		if (ret)
+			goto err;
+
+		if (vm_exec->extra.fn) {
+			ret = vm_exec->extra.fn(vm_exec);
+			drm_exec_retry_on_contention(exec);
+			if (ret)
+				goto err;
+		}
+	}
+
+	return 0;
+
+err:
+	drm_exec_fini(exec);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock);
 
-	WARN(!RB_EMPTY_ROOT(&gpuvm->rb.tree.rb_root),
-	     "GPUVA tree is not empty, potentially leaking memory.");
+static int
+fn_lock_array(struct drm_gpuvm_exec *vm_exec)
+{
+	struct {
+		struct drm_gem_object **objs;
+		unsigned int num_objs;
+	} *args = vm_exec->extra.priv;
+
+	return drm_exec_prepare_array(&vm_exec->exec, args->objs,
+				      args->num_objs, vm_exec->num_fences);
+}
+
+/**
+ * drm_gpuvm_exec_lock_array() - lock all dma-resv of all assoiciated BOs
+ * @vm_exec: the &drm_gpuvm_exec wrapper
+ * @objs: additional &drm_gem_objects to lock
+ * @num_objs: the number of additional &drm_gem_objects to lock
+ *
+ * Acquires all dma-resv locks of all &drm_gem_objects the given &drm_gpuvm
+ * contains mappings of, plus the ones given through @objs.
+ *
+ * Returns: 0 on success, negative error code on failure.
+ */
+int
+drm_gpuvm_exec_lock_array(struct drm_gpuvm_exec *vm_exec,
+			  struct drm_gem_object **objs,
+			  unsigned int num_objs)
+{
+	struct {
+		struct drm_gem_object **objs;
+		unsigned int num_objs;
+	} args;
+
+	args.objs = objs;
+	args.num_objs = num_objs;
+
+	vm_exec->extra.fn = fn_lock_array;
+	vm_exec->extra.priv = &args;
+
+	return drm_gpuvm_exec_lock(vm_exec);
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock_array);
+
+/**
+ * drm_gpuvm_exec_lock_range() - prepare all BOs mapped within a given range
+ * @vm_exec: the &drm_gpuvm_exec wrapper
+ * @addr: the start address within the VA space
+ * @range: the range to iterate within the VA space
+ *
+ * Acquires all dma-resv locks of all &drm_gem_objects mapped between @addr and
+ * @addr + @range.
+ *
+ * Returns: 0 on success, negative error code on failure.
+ */
+int
+drm_gpuvm_exec_lock_range(struct drm_gpuvm_exec *vm_exec,
+			  u64 addr, u64 range)
+{
+	struct drm_gpuvm *gpuvm = vm_exec->vm;
+	struct drm_exec *exec = &vm_exec->exec;
+	int ret;
+
+	drm_exec_init(exec, vm_exec->flags);
+
+	drm_exec_until_all_locked(exec) {
+		ret = drm_gpuvm_prepare_range(gpuvm, exec, addr, range,
+					      vm_exec->num_fences);
+		drm_exec_retry_on_contention(exec);
+		if (ret)
+			goto err;
+	}
+
+	return ret;
+
+err:
+	drm_exec_fini(exec);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock_range);
+
+static int
+__drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec)
+{
+	const struct drm_gpuvm_ops *ops = gpuvm->ops;
+	struct drm_gpuvm_bo *vm_bo;
+	LIST_HEAD(evict);
+	int ret = 0;
+
+	for_each_vm_bo_in_list(gpuvm, evict, &evict, vm_bo) {
+		ret = ops->vm_bo_validate(vm_bo, exec);
+		if (ret)
+			break;
+	}
+	/* Drop ref in case we break out of the loop. */
+	drm_gpuvm_bo_put(vm_bo);
+	restore_vm_bo_list(gpuvm, evict);
+
+	return ret;
+}
+
+static int
+drm_gpuvm_validate_locked(struct drm_gpuvm *gpuvm, struct drm_exec *exec)
+{
+	const struct drm_gpuvm_ops *ops = gpuvm->ops;
+	struct drm_gpuvm_bo *vm_bo, *next;
+	int ret = 0;
+
+	drm_gpuvm_resv_assert_held(gpuvm);
+
+	list_for_each_entry_safe(vm_bo, next, &gpuvm->evict.list,
+				 list.entry.evict) {
+		ret = ops->vm_bo_validate(vm_bo, exec);
+		if (ret)
+			break;
+
+		dma_resv_assert_held(vm_bo->obj->resv);
+		if (!vm_bo->evicted)
+			drm_gpuvm_bo_list_del_init(vm_bo, evict, false);
+	}
+
+	return ret;
+}
+
+/**
+ * drm_gpuvm_validate() - validate all BOs marked as evicted
+ * @gpuvm: the &drm_gpuvm to validate evicted BOs
+ * @exec: the &drm_exec instance used for locking the GPUVM
+ *
+ * Calls the &drm_gpuvm_ops::vm_bo_validate callback for all evicted buffer
+ * objects being mapped in the given &drm_gpuvm.
+ *
+ * Returns: 0 on success, negative error code on failure.
+ */
+int
+drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec)
+{
+	const struct drm_gpuvm_ops *ops = gpuvm->ops;
+
+	if (unlikely(!ops || !ops->vm_bo_validate))
+		return -EOPNOTSUPP;
+
+	if (drm_gpuvm_resv_protected(gpuvm))
+		return drm_gpuvm_validate_locked(gpuvm, exec);
+	else
+		return __drm_gpuvm_validate(gpuvm, exec);
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_validate);
+
+/**
+ * drm_gpuvm_resv_add_fence - add fence to private and all extobj
+ * dma-resv
+ * @gpuvm: the &drm_gpuvm to add a fence to
+ * @exec: the &drm_exec locking context
+ * @fence: fence to add
+ * @private_usage: private dma-resv usage
+ * @extobj_usage: extobj dma-resv usage
+ */
+void
+drm_gpuvm_resv_add_fence(struct drm_gpuvm *gpuvm,
+			 struct drm_exec *exec,
+			 struct dma_fence *fence,
+			 enum dma_resv_usage private_usage,
+			 enum dma_resv_usage extobj_usage)
+{
+	struct drm_gem_object *obj;
+	unsigned long index;
+
+	drm_exec_for_each_locked_object(exec, index, obj) {
+		dma_resv_assert_held(obj->resv);
+		dma_resv_add_fence(obj->resv, fence,
+				   drm_gpuvm_is_extobj(gpuvm, obj) ?
+				   extobj_usage : private_usage);
+	}
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_resv_add_fence);
+
+/**
+ * drm_gpuvm_bo_create() - create a new instance of struct drm_gpuvm_bo
+ * @gpuvm: The &drm_gpuvm the @obj is mapped in.
+ * @obj: The &drm_gem_object being mapped in the @gpuvm.
+ *
+ * If provided by the driver, this function uses the &drm_gpuvm_ops
+ * vm_bo_alloc() callback to allocate.
+ *
+ * Returns: a pointer to the &drm_gpuvm_bo on success, NULL on failure
+ */
+struct drm_gpuvm_bo *
+drm_gpuvm_bo_create(struct drm_gpuvm *gpuvm,
+		    struct drm_gem_object *obj)
+{
+	const struct drm_gpuvm_ops *ops = gpuvm->ops;
+	struct drm_gpuvm_bo *vm_bo;
+
+	if (ops && ops->vm_bo_alloc)
+		vm_bo = ops->vm_bo_alloc();
+	else
+		vm_bo = kzalloc(sizeof(*vm_bo), GFP_KERNEL);
+
+	if (unlikely(!vm_bo))
+		return NULL;
+
+	vm_bo->vm = drm_gpuvm_get(gpuvm);
+	vm_bo->obj = obj;
+	drm_gem_object_get(obj);
+
+	kref_init(&vm_bo->kref);
+	INIT_LIST_HEAD(&vm_bo->list.gpuva);
+	INIT_LIST_HEAD(&vm_bo->list.entry.gem);
+
+	INIT_LIST_HEAD(&vm_bo->list.entry.extobj);
+	INIT_LIST_HEAD(&vm_bo->list.entry.evict);
+
+	return vm_bo;
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_bo_create);
+
+static void
+drm_gpuvm_bo_destroy(struct kref *kref)
+{
+	struct drm_gpuvm_bo *vm_bo = container_of(kref, struct drm_gpuvm_bo,
+						  kref);
+	struct drm_gpuvm *gpuvm = vm_bo->vm;
+	const struct drm_gpuvm_ops *ops = gpuvm->ops;
+	struct drm_gem_object *obj = vm_bo->obj;
+	bool lock = !drm_gpuvm_resv_protected(gpuvm);
+
+	if (!lock)
+		drm_gpuvm_resv_assert_held(gpuvm);
+
+	drm_gpuvm_bo_list_del(vm_bo, extobj, lock);
+	drm_gpuvm_bo_list_del(vm_bo, evict, lock);
+
+	drm_gem_gpuva_assert_lock_held(obj);
+	list_del(&vm_bo->list.entry.gem);
+
+	if (ops && ops->vm_bo_free)
+		ops->vm_bo_free(vm_bo);
+	else
+		kfree(vm_bo);
+
+	drm_gpuvm_put(gpuvm);
+	drm_gem_object_put(obj);
+}
+
+/**
+ * drm_gpuvm_bo_put() - drop a struct drm_gpuvm_bo reference
+ * @vm_bo: the &drm_gpuvm_bo to release the reference of
+ *
+ * This releases a reference to @vm_bo.
+ *
+ * If the reference count drops to zero, the &gpuvm_bo is destroyed, which
+ * includes removing it from the GEMs gpuva list. Hence, if a call to this
+ * function can potentially let the reference count drop to zero the caller must
+ * hold the dma-resv or driver specific GEM gpuva lock.
+ *
+ * This function may only be called from non-atomic context.
+ */
+void
+drm_gpuvm_bo_put(struct drm_gpuvm_bo *vm_bo)
+{
+	might_sleep();
+
+	if (vm_bo)
+		kref_put(&vm_bo->kref, drm_gpuvm_bo_destroy);
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_bo_put);
+
+static struct drm_gpuvm_bo *
+__drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm,
+		    struct drm_gem_object *obj)
+{
+	struct drm_gpuvm_bo *vm_bo;
+
+	drm_gem_gpuva_assert_lock_held(obj);
+	drm_gem_for_each_gpuvm_bo(vm_bo, obj)
+		if (vm_bo->vm == gpuvm)
+			return vm_bo;
+
+	return NULL;
+}
+
+/**
+ * drm_gpuvm_bo_find() - find the &drm_gpuvm_bo for the given
+ * &drm_gpuvm and &drm_gem_object
+ * @gpuvm: The &drm_gpuvm the @obj is mapped in.
+ * @obj: The &drm_gem_object being mapped in the @gpuvm.
+ *
+ * Find the &drm_gpuvm_bo representing the combination of the given
+ * &drm_gpuvm and &drm_gem_object. If found, increases the reference
+ * count of the &drm_gpuvm_bo accordingly.
+ *
+ * Returns: a pointer to the &drm_gpuvm_bo on success, NULL on failure
+ */
+struct drm_gpuvm_bo *
+drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm,
+		  struct drm_gem_object *obj)
+{
+	struct drm_gpuvm_bo *vm_bo = __drm_gpuvm_bo_find(gpuvm, obj);
+
+	return vm_bo ? drm_gpuvm_bo_get(vm_bo) : NULL;
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_bo_find);
+
+/**
+ * drm_gpuvm_bo_obtain() - obtains and instance of the &drm_gpuvm_bo for the
+ * given &drm_gpuvm and &drm_gem_object
+ * @gpuvm: The &drm_gpuvm the @obj is mapped in.
+ * @obj: The &drm_gem_object being mapped in the @gpuvm.
+ *
+ * Find the &drm_gpuvm_bo representing the combination of the given
+ * &drm_gpuvm and &drm_gem_object. If found, increases the reference
+ * count of the &drm_gpuvm_bo accordingly. If not found, allocates a new
+ * &drm_gpuvm_bo.
+ *
+ * A new &drm_gpuvm_bo is added to the GEMs gpuva list.
+ *
+ * Returns: a pointer to the &drm_gpuvm_bo on success, an ERR_PTR on failure
+ */
+struct drm_gpuvm_bo *
+drm_gpuvm_bo_obtain(struct drm_gpuvm *gpuvm,
+		    struct drm_gem_object *obj)
+{
+	struct drm_gpuvm_bo *vm_bo;
+
+	vm_bo = drm_gpuvm_bo_find(gpuvm, obj);
+	if (vm_bo)
+		return vm_bo;
+
+	vm_bo = drm_gpuvm_bo_create(gpuvm, obj);
+	if (!vm_bo)
+		return ERR_PTR(-ENOMEM);
+
+	drm_gem_gpuva_assert_lock_held(obj);
+	list_add_tail(&vm_bo->list.entry.gem, &obj->gpuva.list);
+
+	return vm_bo;
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_bo_obtain);
+
+/**
+ * drm_gpuvm_bo_obtain_prealloc() - obtains and instance of the &drm_gpuvm_bo
+ * for the given &drm_gpuvm and &drm_gem_object
+ * @__vm_bo: A pre-allocated struct drm_gpuvm_bo.
+ *
+ * Find the &drm_gpuvm_bo representing the combination of the given
+ * &drm_gpuvm and &drm_gem_object. If found, increases the reference
+ * count of the found &drm_gpuvm_bo accordingly, while the @__vm_bo reference
+ * count is decreased. If not found @__vm_bo is returned without further
+ * increase of the reference count.
+ *
+ * A new &drm_gpuvm_bo is added to the GEMs gpuva list.
+ *
+ * Returns: a pointer to the found &drm_gpuvm_bo or @__vm_bo if no existing
+ * &drm_gpuvm_bo was found
+ */
+struct drm_gpuvm_bo *
+drm_gpuvm_bo_obtain_prealloc(struct drm_gpuvm_bo *__vm_bo)
+{
+	struct drm_gpuvm *gpuvm = __vm_bo->vm;
+	struct drm_gem_object *obj = __vm_bo->obj;
+	struct drm_gpuvm_bo *vm_bo;
+
+	vm_bo = drm_gpuvm_bo_find(gpuvm, obj);
+	if (vm_bo) {
+		drm_gpuvm_bo_put(__vm_bo);
+		return vm_bo;
+	}
+
+	drm_gem_gpuva_assert_lock_held(obj);
+	list_add_tail(&__vm_bo->list.entry.gem, &obj->gpuva.list);
+
+	return __vm_bo;
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_bo_obtain_prealloc);
+
+/**
+ * drm_gpuvm_bo_extobj_add() - adds the &drm_gpuvm_bo to its &drm_gpuvm's
+ * extobj list
+ * @vm_bo: The &drm_gpuvm_bo to add to its &drm_gpuvm's the extobj list.
+ *
+ * Adds the given @vm_bo to its &drm_gpuvm's extobj list if not on the list
+ * already and if the corresponding &drm_gem_object is an external object,
+ * actually.
+ */
+void
+drm_gpuvm_bo_extobj_add(struct drm_gpuvm_bo *vm_bo)
+{
+	struct drm_gpuvm *gpuvm = vm_bo->vm;
+	bool lock = !drm_gpuvm_resv_protected(gpuvm);
+
+	if (!lock)
+		drm_gpuvm_resv_assert_held(gpuvm);
+
+	if (drm_gpuvm_is_extobj(gpuvm, vm_bo->obj))
+		drm_gpuvm_bo_list_add(vm_bo, extobj, lock);
 }
-EXPORT_SYMBOL_GPL(drm_gpuvm_destroy);
+EXPORT_SYMBOL_GPL(drm_gpuvm_bo_extobj_add);
+
+/**
+ * drm_gpuvm_bo_evict() - add / remove a &drm_gpuvm_bo to / from the &drm_gpuvms
+ * evicted list
+ * @vm_bo: the &drm_gpuvm_bo to add or remove
+ * @evict: indicates whether the object is evicted
+ *
+ * Adds a &drm_gpuvm_bo to or removes it from the &drm_gpuvms evicted list.
+ */
+void
+drm_gpuvm_bo_evict(struct drm_gpuvm_bo *vm_bo, bool evict)
+{
+	struct drm_gpuvm *gpuvm = vm_bo->vm;
+	struct drm_gem_object *obj = vm_bo->obj;
+	bool lock = !drm_gpuvm_resv_protected(gpuvm);
+
+	dma_resv_assert_held(obj->resv);
+	vm_bo->evicted = evict;
+
+	/* Can't add external objects to the evicted list directly if not using
+	 * internal spinlocks, since in this case the evicted list is protected
+	 * with the VM's common dma-resv lock.
+	 */
+	if (drm_gpuvm_is_extobj(gpuvm, obj) && !lock)
+		return;
+
+	if (evict)
+		drm_gpuvm_bo_list_add(vm_bo, evict, lock);
+	else
+		drm_gpuvm_bo_list_del_init(vm_bo, evict, lock);
+}
+EXPORT_SYMBOL_GPL(drm_gpuvm_bo_evict);
 
 static int
 __drm_gpuva_insert(struct drm_gpuvm *gpuvm,
@@ -764,11 +1723,21 @@ drm_gpuva_insert(struct drm_gpuvm *gpuvm,
 {
 	u64 addr = va->va.addr;
 	u64 range = va->va.range;
+	int ret;
 
 	if (unlikely(!drm_gpuvm_range_valid(gpuvm, addr, range)))
 		return -EINVAL;
 
-	return __drm_gpuva_insert(gpuvm, va);
+	ret = __drm_gpuva_insert(gpuvm, va);
+	if (likely(!ret))
+		/* Take a reference of the GPUVM for the successfully inserted
+		 * drm_gpuva. We can't take the reference in
+		 * __drm_gpuva_insert() itself, since we don't want to increse
+		 * the reference count for the GPUVM's kernel_alloc_node.
+		 */
+		drm_gpuvm_get(gpuvm);
+
+	return ret;
 }
 EXPORT_SYMBOL_GPL(drm_gpuva_insert);
 
@@ -795,35 +1764,46 @@ drm_gpuva_remove(struct drm_gpuva *va)
 	struct drm_gpuvm *gpuvm = va->vm;
 
 	if (unlikely(va == &gpuvm->kernel_alloc_node)) {
-		WARN(1, "Can't destroy kernel reserved node.\n");
+		drm_WARN(gpuvm->drm, 1,
+			 "Can't destroy kernel reserved node.\n");
 		return;
 	}
 
 	__drm_gpuva_remove(va);
+	drm_gpuvm_put(va->vm);
 }
 EXPORT_SYMBOL_GPL(drm_gpuva_remove);
 
 /**
  * drm_gpuva_link() - link a &drm_gpuva
  * @va: the &drm_gpuva to link
+ * @vm_bo: the &drm_gpuvm_bo to add the &drm_gpuva to
  *
- * This adds the given &va to the GPU VA list of the &drm_gem_object it is
- * associated with.
+ * This adds the given &va to the GPU VA list of the &drm_gpuvm_bo and the
+ * &drm_gpuvm_bo to the &drm_gem_object it is associated with.
+ *
+ * For every &drm_gpuva entry added to the &drm_gpuvm_bo an additional
+ * reference of the latter is taken.
  *
  * This function expects the caller to protect the GEM's GPUVA list against
- * concurrent access using the GEMs dma_resv lock.
+ * concurrent access using either the GEMs dma_resv lock or a driver specific
+ * lock set through drm_gem_gpuva_set_lock().
  */
 void
-drm_gpuva_link(struct drm_gpuva *va)
+drm_gpuva_link(struct drm_gpuva *va, struct drm_gpuvm_bo *vm_bo)
 {
 	struct drm_gem_object *obj = va->gem.obj;
+	struct drm_gpuvm *gpuvm = va->vm;
 
 	if (unlikely(!obj))
 		return;
 
-	drm_gem_gpuva_assert_lock_held(obj);
+	drm_WARN_ON(gpuvm->drm, obj != vm_bo->obj);
+
+	va->vm_bo = drm_gpuvm_bo_get(vm_bo);
 
-	list_add_tail(&va->gem.entry, &obj->gpuva.list);
+	drm_gem_gpuva_assert_lock_held(obj);
+	list_add_tail(&va->gem.entry, &vm_bo->list.gpuva);
 }
 EXPORT_SYMBOL_GPL(drm_gpuva_link);
 
@@ -834,20 +1814,31 @@ EXPORT_SYMBOL_GPL(drm_gpuva_link);
  * This removes the given &va from the GPU VA list of the &drm_gem_object it is
  * associated with.
  *
+ * This removes the given &va from the GPU VA list of the &drm_gpuvm_bo and
+ * the &drm_gpuvm_bo from the &drm_gem_object it is associated with in case
+ * this call unlinks the last &drm_gpuva from the &drm_gpuvm_bo.
+ *
+ * For every &drm_gpuva entry removed from the &drm_gpuvm_bo a reference of
+ * the latter is dropped.
+ *
  * This function expects the caller to protect the GEM's GPUVA list against
- * concurrent access using the GEMs dma_resv lock.
+ * concurrent access using either the GEMs dma_resv lock or a driver specific
+ * lock set through drm_gem_gpuva_set_lock().
  */
 void
 drm_gpuva_unlink(struct drm_gpuva *va)
 {
 	struct drm_gem_object *obj = va->gem.obj;
+	struct drm_gpuvm_bo *vm_bo = va->vm_bo;
 
 	if (unlikely(!obj))
 		return;
 
 	drm_gem_gpuva_assert_lock_held(obj);
-
 	list_del_init(&va->gem.entry);
+
+	va->vm_bo = NULL;
+	drm_gpuvm_bo_put(vm_bo);
 }
 EXPORT_SYMBOL_GPL(drm_gpuva_unlink);
 
@@ -992,10 +1983,10 @@ drm_gpuva_remap(struct drm_gpuva *prev,
 		struct drm_gpuva *next,
 		struct drm_gpuva_op_remap *op)
 {
-	struct drm_gpuva *curr = op->unmap->va;
-	struct drm_gpuvm *gpuvm = curr->vm;
+	struct drm_gpuva *va = op->unmap->va;
+	struct drm_gpuvm *gpuvm = va->vm;
 
-	drm_gpuva_remove(curr);
+	drm_gpuva_remove(va);
 
 	if (op->prev) {
 		drm_gpuva_init_from_op(prev, op->prev);
@@ -1637,9 +2628,8 @@ drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm,
 EXPORT_SYMBOL_GPL(drm_gpuvm_prefetch_ops_create);
 
 /**
- * drm_gpuvm_gem_unmap_ops_create() - creates the &drm_gpuva_ops to unmap a GEM
- * @gpuvm: the &drm_gpuvm representing the GPU VA space
- * @obj: the &drm_gem_object to unmap
+ * drm_gpuvm_bo_unmap_ops_create() - creates the &drm_gpuva_ops to unmap a GEM
+ * @vm_bo: the &drm_gpuvm_bo abstraction
  *
  * This function creates a list of operations to perform unmapping for every
  * GPUVA attached to a GEM.
@@ -1656,15 +2646,14 @@ EXPORT_SYMBOL_GPL(drm_gpuvm_prefetch_ops_create);
  * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure
  */
 struct drm_gpuva_ops *
-drm_gpuvm_gem_unmap_ops_create(struct drm_gpuvm *gpuvm,
-			       struct drm_gem_object *obj)
+drm_gpuvm_bo_unmap_ops_create(struct drm_gpuvm_bo *vm_bo)
 {
 	struct drm_gpuva_ops *ops;
 	struct drm_gpuva_op *op;
 	struct drm_gpuva *va;
 	int ret;
 
-	drm_gem_gpuva_assert_lock_held(obj);
+	drm_gem_gpuva_assert_lock_held(vm_bo->obj);
 
 	ops = kzalloc(sizeof(*ops), GFP_KERNEL);
 	if (!ops)
@@ -1672,8 +2661,8 @@ drm_gpuvm_gem_unmap_ops_create(struct drm_gpuvm *gpuvm,
 
 	INIT_LIST_HEAD(&ops->list);
 
-	drm_gem_for_each_gpuva(va, obj) {
-		op = gpuva_op_alloc(gpuvm);
+	drm_gpuvm_bo_for_each_va(va, vm_bo) {
+		op = gpuva_op_alloc(vm_bo->vm);
 		if (!op) {
 			ret = -ENOMEM;
 			goto err_free_ops;
@@ -1687,10 +2676,10 @@ drm_gpuvm_gem_unmap_ops_create(struct drm_gpuvm *gpuvm,
 	return ops;
 
 err_free_ops:
-	drm_gpuva_ops_free(gpuvm, ops);
+	drm_gpuva_ops_free(vm_bo->vm, ops);
 	return ERR_PTR(ret);
 }
-EXPORT_SYMBOL_GPL(drm_gpuvm_gem_unmap_ops_create);
+EXPORT_SYMBOL_GPL(drm_gpuvm_bo_unmap_ops_create);
 
 /**
  * drm_gpuva_ops_free() - free the given &drm_gpuva_ops
diff --git a/drivers/gpu/drm/drm_internal.h b/drivers/gpu/drm/drm_internal.h
index 8462b657c375..b12c463bc460 100644
--- a/drivers/gpu/drm/drm_internal.h
+++ b/drivers/gpu/drm/drm_internal.h
@@ -22,6 +22,7 @@
  */
 
 #include <linux/kthread.h>
+#include <linux/types.h>
 
 #include <drm/drm_ioctl.h>
 #include <drm/drm_vblank.h>
@@ -31,6 +32,7 @@
 
 #define DRM_IF_VERSION(maj, min) (maj << 16 | min)
 
+struct cea_sad;
 struct dentry;
 struct dma_buf;
 struct iosys_map;
@@ -267,3 +269,7 @@ int drm_syncobj_query_ioctl(struct drm_device *dev, void *data,
 void drm_framebuffer_print_info(struct drm_printer *p, unsigned int indent,
 				const struct drm_framebuffer *fb);
 void drm_framebuffer_debugfs_init(struct drm_device *dev);
+
+/* drm_edid.c */
+void drm_edid_cta_sad_get(const struct cea_sad *cta_sad, u8 *sad);
+void drm_edid_cta_sad_set(struct cea_sad *cta_sad, const u8 *sad);
diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c
index 77590b0f38fa..44fda68c28ae 100644
--- a/drivers/gpu/drm/drm_ioctl.c
+++ b/drivers/gpu/drm/drm_ioctl.c
@@ -675,6 +675,7 @@ static const struct drm_ioctl_desc drm_ioctls[] = {
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_ADDFB, drm_mode_addfb_ioctl, 0),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_ADDFB2, drm_mode_addfb2_ioctl, 0),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_RMFB, drm_mode_rmfb_ioctl, 0),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_CLOSEFB, drm_mode_closefb_ioctl, 0),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_PAGE_FLIP, drm_mode_page_flip_ioctl, DRM_MASTER),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_DIRTYFB, drm_mode_dirtyfb_ioctl, DRM_MASTER),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_CREATE_DUMB, drm_mode_create_dumb_ioctl, 0),
diff --git a/drivers/gpu/drm/drm_kms_helper_common.c b/drivers/gpu/drm/drm_kms_helper_common.c
index 0bf0fc1abf54..0c7550c0462b 100644
--- a/drivers/gpu/drm/drm_kms_helper_common.c
+++ b/drivers/gpu/drm/drm_kms_helper_common.c
@@ -27,38 +27,6 @@
 
 #include <linux/module.h>
 
-#include <drm/drm_edid.h>
-#include <drm/drm_print.h>
-
-#include "drm_crtc_helper_internal.h"
-
 MODULE_AUTHOR("David Airlie, Jesse Barnes");
 MODULE_DESCRIPTION("DRM KMS helper");
 MODULE_LICENSE("GPL and additional rights");
-
-#if IS_ENABLED(CONFIG_DRM_LOAD_EDID_FIRMWARE)
-
-/* Backward compatibility for drm_kms_helper.edid_firmware */
-static int edid_firmware_set(const char *val, const struct kernel_param *kp)
-{
-	DRM_NOTE("drm_kms_helper.edid_firmware is deprecated, please use drm.edid_firmware instead.\n");
-
-	return __drm_set_edid_firmware_path(val);
-}
-
-static int edid_firmware_get(char *buffer, const struct kernel_param *kp)
-{
-	return __drm_get_edid_firmware_path(buffer, PAGE_SIZE);
-}
-
-static const struct kernel_param_ops edid_firmware_ops = {
-	.set = edid_firmware_set,
-	.get = edid_firmware_get,
-};
-
-module_param_cb(edid_firmware, &edid_firmware_ops, NULL, 0644);
-__MODULE_PARM_TYPE(edid_firmware, "charp");
-MODULE_PARM_DESC(edid_firmware,
-		 "DEPRECATED. Use drm.edid_firmware module parameter instead.");
-
-#endif
diff --git a/drivers/gpu/drm/drm_mipi_dbi.c b/drivers/gpu/drm/drm_mipi_dbi.c
index e90f0bf895b3..daac649aabdb 100644
--- a/drivers/gpu/drm/drm_mipi_dbi.c
+++ b/drivers/gpu/drm/drm_mipi_dbi.c
@@ -197,12 +197,14 @@ EXPORT_SYMBOL(mipi_dbi_command_stackbuf);
  * @fb: The source framebuffer
  * @clip: Clipping rectangle of the area to be copied
  * @swap: When true, swap MSB/LSB of 16-bit values
+ * @fmtcnv_state: Format-conversion state
  *
  * Returns:
  * Zero on success, negative error code on failure.
  */
 int mipi_dbi_buf_copy(void *dst, struct iosys_map *src, struct drm_framebuffer *fb,
-		      struct drm_rect *clip, bool swap)
+		      struct drm_rect *clip, bool swap,
+		      struct drm_format_conv_state *fmtcnv_state)
 {
 	struct drm_gem_object *gem = drm_gem_fb_get_obj(fb, 0);
 	struct iosys_map dst_map = IOSYS_MAP_INIT_VADDR(dst);
@@ -215,12 +217,13 @@ int mipi_dbi_buf_copy(void *dst, struct iosys_map *src, struct drm_framebuffer *
 	switch (fb->format->format) {
 	case DRM_FORMAT_RGB565:
 		if (swap)
-			drm_fb_swab(&dst_map, NULL, src, fb, clip, !gem->import_attach);
+			drm_fb_swab(&dst_map, NULL, src, fb, clip, !gem->import_attach,
+				    fmtcnv_state);
 		else
 			drm_fb_memcpy(&dst_map, NULL, src, fb, clip);
 		break;
 	case DRM_FORMAT_XRGB8888:
-		drm_fb_xrgb8888_to_rgb565(&dst_map, NULL, src, fb, clip, swap);
+		drm_fb_xrgb8888_to_rgb565(&dst_map, NULL, src, fb, clip, fmtcnv_state, swap);
 		break;
 	default:
 		drm_err_once(fb->dev, "Format is not supported: %p4cc\n",
@@ -252,7 +255,7 @@ static void mipi_dbi_set_window_address(struct mipi_dbi_dev *dbidev,
 }
 
 static void mipi_dbi_fb_dirty(struct iosys_map *src, struct drm_framebuffer *fb,
-			      struct drm_rect *rect)
+			      struct drm_rect *rect, struct drm_format_conv_state *fmtcnv_state)
 {
 	struct mipi_dbi_dev *dbidev = drm_to_mipi_dbi_dev(fb->dev);
 	unsigned int height = rect->y2 - rect->y1;
@@ -270,7 +273,7 @@ static void mipi_dbi_fb_dirty(struct iosys_map *src, struct drm_framebuffer *fb,
 	if (!dbi->dc || !full || swap ||
 	    fb->format->format == DRM_FORMAT_XRGB8888) {
 		tr = dbidev->tx_buf;
-		ret = mipi_dbi_buf_copy(tr, src, fb, rect, swap);
+		ret = mipi_dbi_buf_copy(tr, src, fb, rect, swap, fmtcnv_state);
 		if (ret)
 			goto err_msg;
 	} else {
@@ -332,7 +335,8 @@ void mipi_dbi_pipe_update(struct drm_simple_display_pipe *pipe,
 		return;
 
 	if (drm_atomic_helper_damage_merged(old_state, state, &rect))
-		mipi_dbi_fb_dirty(&shadow_plane_state->data[0], fb, &rect);
+		mipi_dbi_fb_dirty(&shadow_plane_state->data[0], fb, &rect,
+				  &shadow_plane_state->fmtcnv_state);
 
 	drm_dev_exit(idx);
 }
@@ -368,7 +372,8 @@ void mipi_dbi_enable_flush(struct mipi_dbi_dev *dbidev,
 	if (!drm_dev_enter(&dbidev->drm, &idx))
 		return;
 
-	mipi_dbi_fb_dirty(&shadow_plane_state->data[0], fb, &rect);
+	mipi_dbi_fb_dirty(&shadow_plane_state->data[0], fb, &rect,
+			  &shadow_plane_state->fmtcnv_state);
 	backlight_enable(dbidev->backlight);
 
 	drm_dev_exit(idx);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
index a8d3fa81e4ec..2ac87d60c222 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
@@ -29,6 +29,17 @@
  * DRM operations:
  */
 
+static struct device_node *etnaviv_of_first_available_node(void)
+{
+	struct device_node *np;
+
+	for_each_compatible_node(np, NULL, "vivante,gc") {
+		if (of_device_is_available(np))
+			return np;
+	}
+
+	return NULL;
+}
 
 static void load_gpu(struct drm_device *dev)
 {
@@ -79,7 +90,7 @@ static int etnaviv_open(struct drm_device *dev, struct drm_file *file)
 			drm_sched_entity_init(&ctx->sched_entity[i],
 					      DRM_SCHED_PRIORITY_NORMAL, &sched,
 					      1, NULL);
-			}
+		}
 	}
 
 	file->driver_priv = ctx;
@@ -233,11 +244,11 @@ static int show_each_gpu(struct seq_file *m, void *arg)
 }
 
 static struct drm_info_list etnaviv_debugfs_list[] = {
-		{"gpu", show_each_gpu, 0, etnaviv_gpu_debugfs},
-		{"gem", show_unlocked, 0, etnaviv_gem_show},
-		{ "mm", show_unlocked, 0, etnaviv_mm_show },
-		{"mmu", show_each_gpu, 0, etnaviv_mmu_show},
-		{"ring", show_each_gpu, 0, etnaviv_ring_show},
+	{"gpu", show_each_gpu, 0, etnaviv_gpu_debugfs},
+	{"gem", show_unlocked, 0, etnaviv_gem_show},
+	{ "mm", show_unlocked, 0, etnaviv_mm_show },
+	{"mmu", show_each_gpu, 0, etnaviv_mmu_show},
+	{"ring", show_each_gpu, 0, etnaviv_ring_show},
 };
 
 static void etnaviv_debugfs_init(struct drm_minor *minor)
@@ -597,11 +608,10 @@ static int etnaviv_pdev_probe(struct platform_device *pdev)
 			if (!of_device_is_available(core_node))
 				continue;
 
-			if (!first_node)
-				first_node = core_node;
-
 			drm_of_component_match_add(&pdev->dev, &match,
 						   component_compare_of, core_node);
+
+			of_node_put(core_node);
 		}
 	} else {
 		char **names = dev->platform_data;
@@ -634,32 +644,65 @@ static int etnaviv_pdev_probe(struct platform_device *pdev)
 	 * device as the GPU we found. This assumes that all Vivante
 	 * GPUs in the system share the same DMA constraints.
 	 */
-	if (first_node)
+	first_node = etnaviv_of_first_available_node();
+	if (first_node) {
 		of_dma_configure(&pdev->dev, first_node, true);
+		of_node_put(first_node);
+	}
 
 	return component_master_add_with_match(dev, &etnaviv_master_ops, match);
 }
 
-static int etnaviv_pdev_remove(struct platform_device *pdev)
+static void etnaviv_pdev_remove(struct platform_device *pdev)
 {
 	component_master_del(&pdev->dev, &etnaviv_master_ops);
-
-	return 0;
 }
 
 static struct platform_driver etnaviv_platform_driver = {
 	.probe      = etnaviv_pdev_probe,
-	.remove     = etnaviv_pdev_remove,
+	.remove_new = etnaviv_pdev_remove,
 	.driver     = {
 		.name   = "etnaviv",
 	},
 };
 
+static int etnaviv_create_platform_device(const char *name,
+					  struct platform_device **ppdev)
+{
+	struct platform_device *pdev;
+	int ret;
+
+	pdev = platform_device_alloc(name, PLATFORM_DEVID_NONE);
+	if (!pdev)
+		return -ENOMEM;
+
+	ret = platform_device_add(pdev);
+	if (ret) {
+		platform_device_put(pdev);
+		return ret;
+	}
+
+	*ppdev = pdev;
+
+	return 0;
+}
+
+static void etnaviv_destroy_platform_device(struct platform_device **ppdev)
+{
+	struct platform_device *pdev = *ppdev;
+
+	if (!pdev)
+		return;
+
+	platform_device_unregister(pdev);
+
+	*ppdev = NULL;
+}
+
 static struct platform_device *etnaviv_drm;
 
 static int __init etnaviv_init(void)
 {
-	struct platform_device *pdev;
 	int ret;
 	struct device_node *np;
 
@@ -677,27 +720,13 @@ static int __init etnaviv_init(void)
 	 * If the DT contains at least one available GPU device, instantiate
 	 * the DRM platform device.
 	 */
-	for_each_compatible_node(np, NULL, "vivante,gc") {
-		if (!of_device_is_available(np))
-			continue;
-
-		pdev = platform_device_alloc("etnaviv", PLATFORM_DEVID_NONE);
-		if (!pdev) {
-			ret = -ENOMEM;
-			of_node_put(np);
-			goto unregister_platform_driver;
-		}
+	np = etnaviv_of_first_available_node();
+	if (np) {
+		of_node_put(np);
 
-		ret = platform_device_add(pdev);
-		if (ret) {
-			platform_device_put(pdev);
-			of_node_put(np);
+		ret = etnaviv_create_platform_device("etnaviv", &etnaviv_drm);
+		if (ret)
 			goto unregister_platform_driver;
-		}
-
-		etnaviv_drm = pdev;
-		of_node_put(np);
-		break;
 	}
 
 	return 0;
@@ -712,7 +741,7 @@ module_init(etnaviv_init);
 
 static void __exit etnaviv_exit(void)
 {
-	platform_device_unregister(etnaviv_drm);
+	etnaviv_destroy_platform_device(&etnaviv_drm);
 	platform_driver_unregister(&etnaviv_platform_driver);
 	platform_driver_unregister(&etnaviv_gpu_driver);
 }
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
index b5f73502e3dd..71a6d2b1c80f 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
@@ -100,11 +100,10 @@ struct page **etnaviv_gem_get_pages(struct etnaviv_gem_object *etnaviv_obj)
 
 	if (!etnaviv_obj->sgt) {
 		struct drm_device *dev = etnaviv_obj->base.dev;
-		int npages = etnaviv_obj->base.size >> PAGE_SHIFT;
+		unsigned int npages = etnaviv_obj->base.size >> PAGE_SHIFT;
 		struct sg_table *sgt;
 
-		sgt = drm_prime_pages_to_sg(etnaviv_obj->base.dev,
-					    etnaviv_obj->pages, npages);
+		sgt = drm_prime_pages_to_sg(dev, etnaviv_obj->pages, npages);
 		if (IS_ERR(sgt)) {
 			dev_err(dev->dev, "failed to allocate sgt: %ld\n",
 				PTR_ERR(sgt));
@@ -542,7 +541,7 @@ static const struct drm_gem_object_funcs etnaviv_gem_object_funcs = {
 	.vm_ops = &vm_ops,
 };
 
-static int etnaviv_gem_new_impl(struct drm_device *dev, u32 size, u32 flags,
+static int etnaviv_gem_new_impl(struct drm_device *dev, u32 flags,
 	const struct etnaviv_gem_ops *ops, struct drm_gem_object **obj)
 {
 	struct etnaviv_gem_object *etnaviv_obj;
@@ -591,8 +590,7 @@ int etnaviv_gem_new_handle(struct drm_device *dev, struct drm_file *file,
 
 	size = PAGE_ALIGN(size);
 
-	ret = etnaviv_gem_new_impl(dev, size, flags,
-				   &etnaviv_gem_shmem_ops, &obj);
+	ret = etnaviv_gem_new_impl(dev, flags, &etnaviv_gem_shmem_ops, &obj);
 	if (ret)
 		goto fail;
 
@@ -627,7 +625,7 @@ int etnaviv_gem_new_private(struct drm_device *dev, size_t size, u32 flags,
 	struct drm_gem_object *obj;
 	int ret;
 
-	ret = etnaviv_gem_new_impl(dev, size, flags, ops, &obj);
+	ret = etnaviv_gem_new_impl(dev, flags, ops, &obj);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
index 2416c526f9b0..3d0f8d182506 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
@@ -535,7 +535,7 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data,
 
 	ret = drm_sched_job_init(&submit->sched_job,
 				 &ctx->sched_entity[args->pipe],
-				 submit->ctx);
+				 1, submit->ctx);
 	if (ret)
 		goto err_submit_put;
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
index 9276756e1397..e47e5562535a 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
@@ -513,8 +513,19 @@ static int etnaviv_hw_reset(struct etnaviv_gpu *gpu)
 	timeout = jiffies + msecs_to_jiffies(1000);
 
 	while (time_is_after_jiffies(timeout)) {
-		/* enable clock */
 		unsigned int fscale = 1 << (6 - gpu->freq_scale);
+		u32 pulse_eater = 0x01590880;
+
+		/* disable clock gating */
+		gpu_write_power(gpu, VIVS_PM_POWER_CONTROLS, 0x0);
+
+		/* disable pulse eater */
+		pulse_eater |= BIT(17);
+		gpu_write_power(gpu, VIVS_PM_PULSE_EATER, pulse_eater);
+		pulse_eater |= BIT(0);
+		gpu_write_power(gpu, VIVS_PM_PULSE_EATER, pulse_eater);
+
+		/* enable clock */
 		control = VIVS_HI_CLOCK_CONTROL_FSCALE_VAL(fscale);
 		etnaviv_gpu_load_clock(gpu, control);
 
@@ -1904,11 +1915,10 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int etnaviv_gpu_platform_remove(struct platform_device *pdev)
+static void etnaviv_gpu_platform_remove(struct platform_device *pdev)
 {
 	component_del(&pdev->dev, &gpu_ops);
 	pm_runtime_disable(&pdev->dev);
-	return 0;
 }
 
 static int etnaviv_gpu_rpm_suspend(struct device *dev)
@@ -1917,7 +1927,7 @@ static int etnaviv_gpu_rpm_suspend(struct device *dev)
 	u32 idle, mask;
 
 	/* If there are any jobs in the HW queue, we're not idle */
-	if (atomic_read(&gpu->sched.hw_rq_count))
+	if (atomic_read(&gpu->sched.credit_count))
 		return -EBUSY;
 
 	/* Check whether the hardware (except FE and MC) is idle */
@@ -1970,6 +1980,6 @@ struct platform_driver etnaviv_gpu_driver = {
 		.of_match_table = etnaviv_gpu_match,
 	},
 	.probe = etnaviv_gpu_platform_probe,
-	.remove = etnaviv_gpu_platform_remove,
+	.remove_new = etnaviv_gpu_platform_remove,
 	.id_table = gpu_ids,
 };
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c
index 4fa72567183a..1661d589bf3e 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c
@@ -70,7 +70,7 @@ static int etnaviv_context_map(struct etnaviv_iommu_context *context,
 }
 
 static int etnaviv_iommu_map(struct etnaviv_iommu_context *context, u32 iova,
-			     struct sg_table *sgt, unsigned len, int prot)
+			     struct sg_table *sgt, int prot)
 {	struct scatterlist *sg;
 	unsigned int da = iova;
 	unsigned int i;
@@ -314,7 +314,7 @@ int etnaviv_iommu_map_gem(struct etnaviv_iommu_context *context,
 		goto unlock;
 
 	mapping->iova = node->start;
-	ret = etnaviv_iommu_map(context, node->start, sgt, etnaviv_obj->base.size,
+	ret = etnaviv_iommu_map(context, node->start, sgt,
 				ETNAVIV_PROT_READ | ETNAVIV_PROT_WRITE);
 
 	if (ret < 0) {
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c b/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c
index bafdfe49c1d8..dc9dea664a28 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c
@@ -511,7 +511,7 @@ int etnaviv_pm_query_dom(struct etnaviv_gpu *gpu,
 
 	domain->id = domain->iter;
 	domain->nr_signals = dom->nr_signals;
-	strncpy(domain->name, dom->name, sizeof(domain->name));
+	strscpy_pad(domain->name, dom->name, sizeof(domain->name));
 
 	domain->iter++;
 	if (domain->iter == nr_domains)
@@ -540,7 +540,7 @@ int etnaviv_pm_query_sig(struct etnaviv_gpu *gpu,
 	sig = &dom->signal[signal->iter];
 
 	signal->id = signal->iter;
-	strncpy(signal->name, sig->name, sizeof(signal->name));
+	strscpy_pad(signal->name, sig->name, sizeof(signal->name));
 
 	signal->iter++;
 	if (signal->iter == dom->nr_signals)
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
index 9b79f218e21a..c4b04b0dee16 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
@@ -134,7 +134,7 @@ int etnaviv_sched_init(struct etnaviv_gpu *gpu)
 {
 	int ret;
 
-	ret = drm_sched_init(&gpu->sched, &etnaviv_sched_ops,
+	ret = drm_sched_init(&gpu->sched, &etnaviv_sched_ops, NULL,
 			     DRM_SCHED_PRIORITY_COUNT,
 			     etnaviv_hw_jobs_limit, etnaviv_job_hang_limit,
 			     msecs_to_jiffies(500), NULL, NULL,
diff --git a/drivers/gpu/drm/gud/gud_pipe.c b/drivers/gpu/drm/gud/gud_pipe.c
index a02f75be81f0..e163649816d5 100644
--- a/drivers/gpu/drm/gud/gud_pipe.c
+++ b/drivers/gpu/drm/gud/gud_pipe.c
@@ -51,7 +51,8 @@ static bool gud_is_big_endian(void)
 
 static size_t gud_xrgb8888_to_r124(u8 *dst, const struct drm_format_info *format,
 				   void *src, struct drm_framebuffer *fb,
-				   struct drm_rect *rect)
+				   struct drm_rect *rect,
+				   struct drm_format_conv_state *fmtcnv_state)
 {
 	unsigned int block_width = drm_format_info_block_width(format, 0);
 	unsigned int bits_per_pixel = 8 / block_width;
@@ -75,7 +76,7 @@ static size_t gud_xrgb8888_to_r124(u8 *dst, const struct drm_format_info *format
 
 	iosys_map_set_vaddr(&dst_map, buf);
 	iosys_map_set_vaddr(&vmap, src);
-	drm_fb_xrgb8888_to_gray8(&dst_map, NULL, &vmap, fb, rect);
+	drm_fb_xrgb8888_to_gray8(&dst_map, NULL, &vmap, fb, rect, fmtcnv_state);
 	pix8 = buf;
 
 	for (y = 0; y < height; y++) {
@@ -152,7 +153,8 @@ static size_t gud_xrgb8888_to_color(u8 *dst, const struct drm_format_info *forma
 static int gud_prep_flush(struct gud_device *gdrm, struct drm_framebuffer *fb,
 			  const struct iosys_map *src, bool cached_reads,
 			  const struct drm_format_info *format, struct drm_rect *rect,
-			  struct gud_set_buffer_req *req)
+			  struct gud_set_buffer_req *req,
+			  struct drm_format_conv_state *fmtcnv_state)
 {
 	u8 compression = gdrm->compression;
 	struct iosys_map dst;
@@ -178,23 +180,23 @@ static int gud_prep_flush(struct gud_device *gdrm, struct drm_framebuffer *fb,
 	 */
 	if (format != fb->format) {
 		if (format->format == GUD_DRM_FORMAT_R1) {
-			len = gud_xrgb8888_to_r124(buf, format, vaddr, fb, rect);
+			len = gud_xrgb8888_to_r124(buf, format, vaddr, fb, rect, fmtcnv_state);
 			if (!len)
 				return -ENOMEM;
 		} else if (format->format == DRM_FORMAT_R8) {
-			drm_fb_xrgb8888_to_gray8(&dst, NULL, src, fb, rect);
+			drm_fb_xrgb8888_to_gray8(&dst, NULL, src, fb, rect, fmtcnv_state);
 		} else if (format->format == DRM_FORMAT_RGB332) {
-			drm_fb_xrgb8888_to_rgb332(&dst, NULL, src, fb, rect);
+			drm_fb_xrgb8888_to_rgb332(&dst, NULL, src, fb, rect, fmtcnv_state);
 		} else if (format->format == DRM_FORMAT_RGB565) {
-			drm_fb_xrgb8888_to_rgb565(&dst, NULL, src, fb, rect,
+			drm_fb_xrgb8888_to_rgb565(&dst, NULL, src, fb, rect, fmtcnv_state,
 						  gud_is_big_endian());
 		} else if (format->format == DRM_FORMAT_RGB888) {
-			drm_fb_xrgb8888_to_rgb888(&dst, NULL, src, fb, rect);
+			drm_fb_xrgb8888_to_rgb888(&dst, NULL, src, fb, rect, fmtcnv_state);
 		} else {
 			len = gud_xrgb8888_to_color(buf, format, vaddr, fb, rect);
 		}
 	} else if (gud_is_big_endian() && format->cpp[0] > 1) {
-		drm_fb_swab(&dst, NULL, src, fb, rect, cached_reads);
+		drm_fb_swab(&dst, NULL, src, fb, rect, cached_reads, fmtcnv_state);
 	} else if (compression && cached_reads && pitch == fb->pitches[0]) {
 		/* can compress directly from the framebuffer */
 		buf = vaddr + rect->y1 * pitch;
@@ -266,7 +268,8 @@ static int gud_usb_bulk(struct gud_device *gdrm, size_t len)
 
 static int gud_flush_rect(struct gud_device *gdrm, struct drm_framebuffer *fb,
 			  const struct iosys_map *src, bool cached_reads,
-			  const struct drm_format_info *format, struct drm_rect *rect)
+			  const struct drm_format_info *format, struct drm_rect *rect,
+			  struct drm_format_conv_state *fmtcnv_state)
 {
 	struct gud_set_buffer_req req;
 	size_t len, trlen;
@@ -274,7 +277,7 @@ static int gud_flush_rect(struct gud_device *gdrm, struct drm_framebuffer *fb,
 
 	drm_dbg(&gdrm->drm, "Flushing [FB:%d] " DRM_RECT_FMT "\n", fb->base.id, DRM_RECT_ARG(rect));
 
-	ret = gud_prep_flush(gdrm, fb, src, cached_reads, format, rect, &req);
+	ret = gud_prep_flush(gdrm, fb, src, cached_reads, format, rect, &req, fmtcnv_state);
 	if (ret)
 		return ret;
 
@@ -318,6 +321,7 @@ static void gud_flush_damage(struct gud_device *gdrm, struct drm_framebuffer *fb
 			     const struct iosys_map *src, bool cached_reads,
 			     struct drm_rect *damage)
 {
+	struct drm_format_conv_state fmtcnv_state = DRM_FORMAT_CONV_STATE_INIT;
 	const struct drm_format_info *format;
 	unsigned int i, lines;
 	size_t pitch;
@@ -340,7 +344,7 @@ static void gud_flush_damage(struct gud_device *gdrm, struct drm_framebuffer *fb
 		rect.y1 += i * lines;
 		rect.y2 = min_t(u32, rect.y1 + lines, damage->y2);
 
-		ret = gud_flush_rect(gdrm, fb, src, cached_reads, format, &rect);
+		ret = gud_flush_rect(gdrm, fb, src, cached_reads, format, &rect, &fmtcnv_state);
 		if (ret) {
 			if (ret != -ENODEV && ret != -ECONNRESET &&
 			    ret != -ESHUTDOWN && ret != -EPROTO)
@@ -350,6 +354,8 @@ static void gud_flush_damage(struct gud_device *gdrm, struct drm_framebuffer *fb
 			break;
 		}
 	}
+
+	drm_format_conv_state_release(&fmtcnv_state);
 }
 
 void gud_flush_work(struct work_struct *work)
diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig
index ce397a8797f7..b5d6e3352071 100644
--- a/drivers/gpu/drm/i915/Kconfig
+++ b/drivers/gpu/drm/i915/Kconfig
@@ -94,7 +94,7 @@ config DRM_I915_CAPTURE_ERROR
 	  This option enables capturing the GPU state when a hang is detected.
 	  This information is vital for triaging hangs and assists in debugging.
 	  Please report any hang for triaging according to:
-	    https://gitlab.freedesktop.org/drm/intel/-/wikis/How-to-file-i915-bugs
+	    https://drm.pages.freedesktop.org/intel-docs/how-to-file-i915-bugs.html
 
 	  If in doubt, say "Y".
 
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 88b2bb005014..7e5d6a39d450 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -47,33 +47,34 @@ subdir-ccflags-y += -I$(srctree)/$(src)
 # Please keep these build lists sorted!
 
 # core driver code
-i915-y += i915_driver.o \
-	  i915_drm_client.o \
-	  i915_config.o \
-	  i915_getparam.o \
-	  i915_ioctl.o \
-	  i915_irq.o \
-	  i915_mitigations.o \
-	  i915_module.o \
-	  i915_params.o \
-	  i915_pci.o \
-	  i915_scatterlist.o \
-	  i915_suspend.o \
-	  i915_switcheroo.o \
-	  i915_sysfs.o \
-	  i915_utils.o \
-	  intel_clock_gating.o \
-	  intel_device_info.o \
-	  intel_memory_region.o \
-	  intel_pcode.o \
-	  intel_region_ttm.o \
-	  intel_runtime_pm.o \
-	  intel_sbi.o \
-	  intel_step.o \
-	  intel_uncore.o \
-	  intel_wakeref.o \
-	  vlv_sideband.o \
-	  vlv_suspend.o
+i915-y += \
+	i915_config.o \
+	i915_driver.o \
+	i915_drm_client.o \
+	i915_getparam.o \
+	i915_ioctl.o \
+	i915_irq.o \
+	i915_mitigations.o \
+	i915_module.o \
+	i915_params.o \
+	i915_pci.o \
+	i915_scatterlist.o \
+	i915_suspend.o \
+	i915_switcheroo.o \
+	i915_sysfs.o \
+	i915_utils.o \
+	intel_clock_gating.o \
+	intel_device_info.o \
+	intel_memory_region.o \
+	intel_pcode.o \
+	intel_region_ttm.o \
+	intel_runtime_pm.o \
+	intel_sbi.o \
+	intel_step.o \
+	intel_uncore.o \
+	intel_wakeref.o \
+	vlv_sideband.o \
+	vlv_suspend.o
 
 # core peripheral code
 i915-y += \
@@ -90,13 +91,13 @@ i915-y += \
 	i915_syncmap.o \
 	i915_user_extensions.o
 
-i915-$(CONFIG_COMPAT)   += i915_ioc32.o
+i915-$(CONFIG_COMPAT) += \
+	i915_ioc32.o
 i915-$(CONFIG_DEBUG_FS) += \
 	i915_debugfs.o \
-	i915_debugfs_params.o \
-	display/intel_display_debugfs.o \
-	display/intel_pipe_crc.o
-i915-$(CONFIG_PERF_EVENTS) += i915_pmu.o
+	i915_debugfs_params.o
+i915-$(CONFIG_PERF_EVENTS) += \
+	i915_pmu.o
 
 # "Graphics Technology" (aka we talk to the gpu)
 gt-y += \
@@ -153,7 +154,8 @@ gt-y += \
 	gt/sysfs_engines.o
 
 # x86 intel-gtt module support
-gt-$(CONFIG_X86) += gt/intel_ggtt_gmch.o
+gt-$(CONFIG_X86) += \
+	gt/intel_ggtt_gmch.o
 # autogenerated null render state
 gt-y += \
 	gt/gen6_renderstate.o \
@@ -172,9 +174,9 @@ gem-y += \
 	gem/i915_gem_domain.o \
 	gem/i915_gem_execbuffer.o \
 	gem/i915_gem_internal.o \
-	gem/i915_gem_object.o \
 	gem/i915_gem_lmem.o \
 	gem/i915_gem_mman.o \
+	gem/i915_gem_object.o \
 	gem/i915_gem_pages.o \
 	gem/i915_gem_phys.o \
 	gem/i915_gem_pm.o \
@@ -191,57 +193,61 @@ gem-y += \
 	gem/i915_gem_wait.o \
 	gem/i915_gemfs.o
 i915-y += \
-	  $(gem-y) \
-	  i915_active.o \
-	  i915_cmd_parser.o \
-	  i915_deps.o \
-	  i915_gem_evict.o \
-	  i915_gem_gtt.o \
-	  i915_gem_ww.o \
-	  i915_gem.o \
-	  i915_query.o \
-	  i915_request.o \
-	  i915_scheduler.o \
-	  i915_trace_points.o \
-	  i915_ttm_buddy_manager.o \
-	  i915_vma.o \
-	  i915_vma_resource.o
+	$(gem-y) \
+	i915_active.o \
+	i915_cmd_parser.o \
+	i915_deps.o \
+	i915_gem.o \
+	i915_gem_evict.o \
+	i915_gem_gtt.o \
+	i915_gem_ww.o \
+	i915_query.o \
+	i915_request.o \
+	i915_scheduler.o \
+	i915_trace_points.o \
+	i915_ttm_buddy_manager.o \
+	i915_vma.o \
+	i915_vma_resource.o
 
 # general-purpose microcontroller (GuC) support
 i915-y += \
-	  gt/uc/intel_gsc_fw.o \
-	  gt/uc/intel_gsc_proxy.o \
-	  gt/uc/intel_gsc_uc.o \
-	  gt/uc/intel_gsc_uc_debugfs.o \
-	  gt/uc/intel_gsc_uc_heci_cmd_submit.o \
-	  gt/uc/intel_guc.o \
-	  gt/uc/intel_guc_ads.o \
-	  gt/uc/intel_guc_capture.o \
-	  gt/uc/intel_guc_ct.o \
-	  gt/uc/intel_guc_debugfs.o \
-	  gt/uc/intel_guc_fw.o \
-	  gt/uc/intel_guc_hwconfig.o \
-	  gt/uc/intel_guc_log.o \
-	  gt/uc/intel_guc_log_debugfs.o \
-	  gt/uc/intel_guc_rc.o \
-	  gt/uc/intel_guc_slpc.o \
-	  gt/uc/intel_guc_submission.o \
-	  gt/uc/intel_huc.o \
-	  gt/uc/intel_huc_debugfs.o \
-	  gt/uc/intel_huc_fw.o \
-	  gt/uc/intel_uc.o \
-	  gt/uc/intel_uc_debugfs.o \
-	  gt/uc/intel_uc_fw.o
+	gt/uc/intel_gsc_fw.o \
+	gt/uc/intel_gsc_proxy.o \
+	gt/uc/intel_gsc_uc.o \
+	gt/uc/intel_gsc_uc_debugfs.o \
+	gt/uc/intel_gsc_uc_heci_cmd_submit.o\
+	gt/uc/intel_guc.o \
+	gt/uc/intel_guc_ads.o \
+	gt/uc/intel_guc_capture.o \
+	gt/uc/intel_guc_ct.o \
+	gt/uc/intel_guc_debugfs.o \
+	gt/uc/intel_guc_fw.o \
+	gt/uc/intel_guc_hwconfig.o \
+	gt/uc/intel_guc_log.o \
+	gt/uc/intel_guc_log_debugfs.o \
+	gt/uc/intel_guc_rc.o \
+	gt/uc/intel_guc_slpc.o \
+	gt/uc/intel_guc_submission.o \
+	gt/uc/intel_huc.o \
+	gt/uc/intel_huc_debugfs.o \
+	gt/uc/intel_huc_fw.o \
+	gt/uc/intel_uc.o \
+	gt/uc/intel_uc_debugfs.o \
+	gt/uc/intel_uc_fw.o
 
 # graphics system controller (GSC) support
-i915-y += gt/intel_gsc.o
+i915-y += \
+	gt/intel_gsc.o
 
 # graphics hardware monitoring (HWMON) support
-i915-$(CONFIG_HWMON) += i915_hwmon.o
+i915-$(CONFIG_HWMON) += \
+	i915_hwmon.o
 
 # modesetting core code
 i915-y += \
 	display/hsw_ips.o \
+	display/i9xx_plane.o \
+	display/i9xx_wm.o \
 	display/intel_atomic.o \
 	display/intel_atomic_plane.o \
 	display/intel_audio.o \
@@ -257,6 +263,7 @@ i915-y += \
 	display/intel_display.o \
 	display/intel_display_driver.o \
 	display/intel_display_irq.o \
+	display/intel_display_params.o \
 	display/intel_display_power.o \
 	display/intel_display_power_map.o \
 	display/intel_display_power_well.o \
@@ -270,6 +277,7 @@ i915-y += \
 	display/intel_dpt.o \
 	display/intel_drrs.o \
 	display/intel_dsb.o \
+	display/intel_dsb_buffer.o \
 	display/intel_fb.o \
 	display/intel_fb_pin.o \
 	display/intel_fbc.o \
@@ -287,8 +295,8 @@ i915-y += \
 	display/intel_load_detect.o \
 	display/intel_lpe_audio.o \
 	display/intel_modeset_lock.o \
-	display/intel_modeset_verify.o \
 	display/intel_modeset_setup.o \
+	display/intel_modeset_verify.o \
 	display/intel_overlay.o \
 	display/intel_pch_display.o \
 	display/intel_pch_refclk.o \
@@ -302,8 +310,6 @@ i915-y += \
 	display/intel_vblank.o \
 	display/intel_vga.o \
 	display/intel_wm.o \
-	display/i9xx_plane.o \
-	display/i9xx_wm.o \
 	display/skl_scaler.o \
 	display/skl_universal_plane.o \
 	display/skl_watermark.o
@@ -312,6 +318,10 @@ i915-$(CONFIG_ACPI) += \
 	display/intel_opregion.o
 i915-$(CONFIG_DRM_FBDEV_EMULATION) += \
 	display/intel_fbdev.o
+i915-$(CONFIG_DEBUG_FS) += \
+	display/intel_display_debugfs.o \
+	display/intel_display_debugfs_params.o \
+	display/intel_pipe_crc.o
 
 # modesetting output/encoder code
 i915-y += \
@@ -357,13 +367,14 @@ i915-y += \
 	display/vlv_dsi.o \
 	display/vlv_dsi_pll.o
 
-i915-y += i915_perf.o
+i915-y += \
+	i915_perf.o
 
 # Protected execution platform (PXP) support. Base support is required for HuC
 i915-y += \
 	pxp/intel_pxp.o \
-	pxp/intel_pxp_tee.o \
-	pxp/intel_pxp_huc.o
+	pxp/intel_pxp_huc.o \
+	pxp/intel_pxp_tee.o
 
 i915-$(CONFIG_DRM_I915_PXP) += \
 	pxp/intel_pxp_cmd.o \
@@ -374,11 +385,11 @@ i915-$(CONFIG_DRM_I915_PXP) += \
 	pxp/intel_pxp_session.o
 
 # Post-mortem debug and GPU hang state capture
-i915-$(CONFIG_DRM_I915_CAPTURE_ERROR) += i915_gpu_error.o
+i915-$(CONFIG_DRM_I915_CAPTURE_ERROR) += \
+	i915_gpu_error.o
 i915-$(CONFIG_DRM_I915_SELFTEST) += \
 	gem/selftests/i915_gem_client_blt.o \
 	gem/selftests/igt_gem_utils.o \
-	selftests/intel_scheduler_helpers.o \
 	selftests/i915_random.o \
 	selftests/i915_selftest.o \
 	selftests/igt_atomic.o \
@@ -387,10 +398,12 @@ i915-$(CONFIG_DRM_I915_SELFTEST) += \
 	selftests/igt_mmap.o \
 	selftests/igt_reset.o \
 	selftests/igt_spinner.o \
+	selftests/intel_scheduler_helpers.o \
 	selftests/librapl.o
 
 # virtual gpu code
-i915-y += i915_vgpu.o
+i915-y += \
+	i915_vgpu.o
 
 i915-$(CONFIG_DRM_I915_GVT) += \
 	intel_gvt.o \
diff --git a/drivers/gpu/drm/i915/display/hsw_ips.c b/drivers/gpu/drm/i915/display/hsw_ips.c
index 7dc38ac02092..611a7d6ef80c 100644
--- a/drivers/gpu/drm/i915/display/hsw_ips.c
+++ b/drivers/gpu/drm/i915/display/hsw_ips.c
@@ -193,7 +193,7 @@ bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state)
 	if (!hsw_crtc_supports_ips(crtc))
 		return false;
 
-	if (!i915->params.enable_ips)
+	if (!i915->display.params.enable_ips)
 		return false;
 
 	if (crtc_state->pipe_bpp > 24)
@@ -329,7 +329,7 @@ static int hsw_ips_debugfs_status_show(struct seq_file *m, void *unused)
 	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 
 	seq_printf(m, "Enabled by kernel parameter: %s\n",
-		   str_yes_no(i915->params.enable_ips));
+		   str_yes_no(i915->display.params.enable_ips));
 
 	if (DISPLAY_VER(i915) >= 8) {
 		seq_puts(m, "Currently: unknown\n");
diff --git a/drivers/gpu/drm/i915/display/i9xx_wm.c b/drivers/gpu/drm/i915/display/i9xx_wm.c
index af0c79a4c9a4..b37c0d02d500 100644
--- a/drivers/gpu/drm/i915/display/i9xx_wm.c
+++ b/drivers/gpu/drm/i915/display/i9xx_wm.c
@@ -2993,7 +2993,7 @@ static void ilk_wm_merge(struct drm_i915_private *dev_priv,
 
 	/* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
 	if (DISPLAY_VER(dev_priv) == 5 && HAS_FBC(dev_priv) &&
-	    dev_priv->params.enable_fbc && !merged->fbc_wm_enabled) {
+	    dev_priv->display.params.enable_fbc && !merged->fbc_wm_enabled) {
 		for (level = 2; level < num_levels; level++) {
 			struct intel_wm_level *wm = &merged->wm[level];
 
diff --git a/drivers/gpu/drm/i915/display/icl_dsi.c b/drivers/gpu/drm/i915/display/icl_dsi.c
index c4585e445198..481fcb650850 100644
--- a/drivers/gpu/drm/i915/display/icl_dsi.c
+++ b/drivers/gpu/drm/i915/display/icl_dsi.c
@@ -330,7 +330,7 @@ static int afe_clk(struct intel_encoder *encoder,
 	int bpp;
 
 	if (crtc_state->dsc.compression_enable)
-		bpp = crtc_state->dsc.compressed_bpp;
+		bpp = to_bpp_int(crtc_state->dsc.compressed_bpp_x16);
 	else
 		bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format);
 
@@ -860,7 +860,7 @@ gen11_dsi_set_transcoder_timings(struct intel_encoder *encoder,
 	 * compressed and non-compressed bpp.
 	 */
 	if (crtc_state->dsc.compression_enable) {
-		mul = crtc_state->dsc.compressed_bpp;
+		mul = to_bpp_int(crtc_state->dsc.compressed_bpp_x16);
 		div = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format);
 	}
 
@@ -884,7 +884,7 @@ gen11_dsi_set_transcoder_timings(struct intel_encoder *encoder,
 		int bpp, line_time_us, byte_clk_period_ns;
 
 		if (crtc_state->dsc.compression_enable)
-			bpp = crtc_state->dsc.compressed_bpp;
+			bpp = to_bpp_int(crtc_state->dsc.compressed_bpp_x16);
 		else
 			bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format);
 
@@ -1451,8 +1451,8 @@ static void gen11_dsi_get_timings(struct intel_encoder *encoder,
 	struct drm_display_mode *adjusted_mode =
 					&pipe_config->hw.adjusted_mode;
 
-	if (pipe_config->dsc.compressed_bpp) {
-		int div = pipe_config->dsc.compressed_bpp;
+	if (pipe_config->dsc.compressed_bpp_x16) {
+		int div = to_bpp_int(pipe_config->dsc.compressed_bpp_x16);
 		int mul = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format);
 
 		adjusted_mode->crtc_htotal =
diff --git a/drivers/gpu/drm/i915/display/intel_atomic.c b/drivers/gpu/drm/i915/display/intel_atomic.c
index 5d18145da279..ec0d5168b503 100644
--- a/drivers/gpu/drm/i915/display/intel_atomic.c
+++ b/drivers/gpu/drm/i915/display/intel_atomic.c
@@ -331,9 +331,6 @@ void intel_atomic_state_free(struct drm_atomic_state *_state)
 
 	drm_atomic_state_default_release(&state->base);
 	kfree(state->global_objs);
-
-	i915_sw_fence_fini(&state->commit_ready);
-
 	kfree(state);
 }
 
diff --git a/drivers/gpu/drm/i915/display/intel_atomic_plane.c b/drivers/gpu/drm/i915/display/intel_atomic_plane.c
index b1074350616c..06c2455bdd78 100644
--- a/drivers/gpu/drm/i915/display/intel_atomic_plane.c
+++ b/drivers/gpu/drm/i915/display/intel_atomic_plane.c
@@ -31,7 +31,10 @@
  * prepare/check/commit/cleanup steps.
  */
 
+#include <linux/dma-fence-chain.h>
+
 #include <drm/drm_atomic_helper.h>
+#include <drm/drm_gem_atomic_helper.h>
 #include <drm/drm_blend.h>
 #include <drm/drm_fourcc.h>
 
@@ -1012,6 +1015,41 @@ int intel_plane_check_src_coordinates(struct intel_plane_state *plane_state)
 	return 0;
 }
 
+static int add_dma_resv_fences(struct dma_resv *resv,
+			       struct drm_plane_state *new_plane_state)
+{
+	struct dma_fence *fence = dma_fence_get(new_plane_state->fence);
+	struct dma_fence *new;
+	int ret;
+
+	ret = dma_resv_get_singleton(resv, dma_resv_usage_rw(false), &new);
+	if (ret)
+		goto error;
+
+	if (new && fence) {
+		struct dma_fence_chain *chain = dma_fence_chain_alloc();
+
+		if (!chain) {
+			ret = -ENOMEM;
+			goto error;
+		}
+
+		dma_fence_chain_init(chain, fence, new, 1);
+		fence = &chain->base;
+
+	} else if (new) {
+		fence = new;
+	}
+
+	dma_fence_put(new_plane_state->fence);
+	new_plane_state->fence = fence;
+	return 0;
+
+error:
+	dma_fence_put(fence);
+	return ret;
+}
+
 /**
  * intel_prepare_plane_fb - Prepare fb for usage on plane
  * @_plane: drm plane to prepare for
@@ -1035,7 +1073,7 @@ intel_prepare_plane_fb(struct drm_plane *_plane,
 	struct intel_atomic_state *state =
 		to_intel_atomic_state(new_plane_state->uapi.state);
 	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
-	const struct intel_plane_state *old_plane_state =
+	struct intel_plane_state *old_plane_state =
 		intel_atomic_get_old_plane_state(state, plane);
 	struct drm_i915_gem_object *obj = intel_fb_obj(new_plane_state->hw.fb);
 	struct drm_i915_gem_object *old_obj = intel_fb_obj(old_plane_state->hw.fb);
@@ -1058,55 +1096,28 @@ intel_prepare_plane_fb(struct drm_plane *_plane,
 		 * can safely continue.
 		 */
 		if (new_crtc_state && intel_crtc_needs_modeset(new_crtc_state)) {
-			ret = i915_sw_fence_await_reservation(&state->commit_ready,
-							      old_obj->base.resv,
-							      false, 0,
-							      GFP_KERNEL);
+			ret = add_dma_resv_fences(intel_bo_to_drm_bo(old_obj)->resv,
+						  &new_plane_state->uapi);
 			if (ret < 0)
 				return ret;
 		}
 	}
 
-	if (new_plane_state->uapi.fence) { /* explicit fencing */
-		i915_gem_fence_wait_priority(new_plane_state->uapi.fence,
-					     &attr);
-		ret = i915_sw_fence_await_dma_fence(&state->commit_ready,
-						    new_plane_state->uapi.fence,
-						    i915_fence_timeout(dev_priv),
-						    GFP_KERNEL);
-		if (ret < 0)
-			return ret;
-	}
-
 	if (!obj)
 		return 0;
 
-
 	ret = intel_plane_pin_fb(new_plane_state);
 	if (ret)
 		return ret;
 
-	i915_gem_object_wait_priority(obj, 0, &attr);
+	ret = drm_gem_plane_helper_prepare_fb(&plane->base, &new_plane_state->uapi);
+	if (ret < 0)
+		goto unpin_fb;
 
-	if (!new_plane_state->uapi.fence) { /* implicit fencing */
-		struct dma_resv_iter cursor;
-		struct dma_fence *fence;
-
-		ret = i915_sw_fence_await_reservation(&state->commit_ready,
-						      obj->base.resv, false,
-						      i915_fence_timeout(dev_priv),
-						      GFP_KERNEL);
-		if (ret < 0)
-			goto unpin_fb;
+	if (new_plane_state->uapi.fence) {
+		i915_gem_fence_wait_priority(new_plane_state->uapi.fence,
+					     &attr);
 
-		dma_resv_iter_begin(&cursor, obj->base.resv,
-				    DMA_RESV_USAGE_WRITE);
-		dma_resv_for_each_fence_unlocked(&cursor, fence) {
-			intel_display_rps_boost_after_vblank(new_plane_state->hw.crtc,
-							     fence);
-		}
-		dma_resv_iter_end(&cursor);
-	} else {
 		intel_display_rps_boost_after_vblank(new_plane_state->hw.crtc,
 						     new_plane_state->uapi.fence);
 	}
diff --git a/drivers/gpu/drm/i915/display/intel_audio.c b/drivers/gpu/drm/i915/display/intel_audio.c
index 19605264a35c..07e0c73204f3 100644
--- a/drivers/gpu/drm/i915/display/intel_audio.c
+++ b/drivers/gpu/drm/i915/display/intel_audio.c
@@ -25,6 +25,7 @@
 #include <linux/kernel.h>
 
 #include <drm/drm_edid.h>
+#include <drm/drm_eld.h>
 #include <drm/i915_component.h>
 
 #include "i915_drv.h"
@@ -521,25 +522,25 @@ static unsigned int calc_hblank_early_prog(struct intel_encoder *encoder,
 	unsigned int link_clks_available, link_clks_required;
 	unsigned int tu_data, tu_line, link_clks_active;
 	unsigned int h_active, h_total, hblank_delta, pixel_clk;
-	unsigned int fec_coeff, cdclk, vdsc_bpp;
+	unsigned int fec_coeff, cdclk, vdsc_bppx16;
 	unsigned int link_clk, lanes;
 	unsigned int hblank_rise;
 
 	h_active = crtc_state->hw.adjusted_mode.crtc_hdisplay;
 	h_total = crtc_state->hw.adjusted_mode.crtc_htotal;
 	pixel_clk = crtc_state->hw.adjusted_mode.crtc_clock;
-	vdsc_bpp = crtc_state->dsc.compressed_bpp;
+	vdsc_bppx16 = crtc_state->dsc.compressed_bpp_x16;
 	cdclk = i915->display.cdclk.hw.cdclk;
 	/* fec= 0.972261, using rounding multiplier of 1000000 */
 	fec_coeff = 972261;
 	link_clk = crtc_state->port_clock;
 	lanes = crtc_state->lane_count;
 
-	drm_dbg_kms(&i915->drm, "h_active = %u link_clk = %u :"
-		    "lanes = %u vdsc_bpp = %u cdclk = %u\n",
-		    h_active, link_clk, lanes, vdsc_bpp, cdclk);
+	drm_dbg_kms(&i915->drm,
+		    "h_active = %u link_clk = %u : lanes = %u vdsc_bpp = " BPP_X16_FMT " cdclk = %u\n",
+		    h_active, link_clk, lanes, BPP_X16_ARGS(vdsc_bppx16), cdclk);
 
-	if (WARN_ON(!link_clk || !pixel_clk || !lanes || !vdsc_bpp || !cdclk))
+	if (WARN_ON(!link_clk || !pixel_clk || !lanes || !vdsc_bppx16 || !cdclk))
 		return 0;
 
 	link_clks_available = (h_total - h_active) * link_clk / pixel_clk - 28;
@@ -551,8 +552,8 @@ static unsigned int calc_hblank_early_prog(struct intel_encoder *encoder,
 		hblank_delta = DIV64_U64_ROUND_UP(mul_u32_u32(5 * (link_clk + cdclk), pixel_clk),
 						  mul_u32_u32(link_clk, cdclk));
 
-	tu_data = div64_u64(mul_u32_u32(pixel_clk * vdsc_bpp * 8, 1000000),
-			    mul_u32_u32(link_clk * lanes, fec_coeff));
+	tu_data = div64_u64(mul_u32_u32(pixel_clk * vdsc_bppx16 * 8, 1000000),
+			    mul_u32_u32(link_clk * lanes * 16, fec_coeff));
 	tu_line = div64_u64(h_active * mul_u32_u32(link_clk, fec_coeff),
 			    mul_u32_u32(64 * pixel_clk, 1000000));
 	link_clks_active  = (tu_line - 1) * 64 + tu_data;
diff --git a/drivers/gpu/drm/i915/display/intel_backlight.c b/drivers/gpu/drm/i915/display/intel_backlight.c
index 2e8f17c04522..612d4cd9dacb 100644
--- a/drivers/gpu/drm/i915/display/intel_backlight.c
+++ b/drivers/gpu/drm/i915/display/intel_backlight.c
@@ -88,10 +88,10 @@ u32 intel_backlight_invert_pwm_level(struct intel_connector *connector, u32 val)
 
 	drm_WARN_ON(&i915->drm, panel->backlight.pwm_level_max == 0);
 
-	if (i915->params.invert_brightness < 0)
+	if (i915->display.params.invert_brightness < 0)
 		return val;
 
-	if (i915->params.invert_brightness > 0 ||
+	if (i915->display.params.invert_brightness > 0 ||
 	    intel_has_quirk(i915, QUIRK_INVERT_BRIGHTNESS)) {
 		return panel->backlight.pwm_level_max - val + panel->backlight.pwm_level_min;
 	}
@@ -132,8 +132,9 @@ u32 intel_backlight_level_from_pwm(struct intel_connector *connector, u32 val)
 	drm_WARN_ON_ONCE(&i915->drm,
 			 panel->backlight.max == 0 || panel->backlight.pwm_level_max == 0);
 
-	if (i915->params.invert_brightness > 0 ||
-	    (i915->params.invert_brightness == 0 && intel_has_quirk(i915, QUIRK_INVERT_BRIGHTNESS)))
+	if (i915->display.params.invert_brightness > 0 ||
+	    (i915->display.params.invert_brightness == 0 &&
+	     intel_has_quirk(i915, QUIRK_INVERT_BRIGHTNESS)))
 		val = panel->backlight.pwm_level_max - (val - panel->backlight.pwm_level_min);
 
 	return scale(val, panel->backlight.pwm_level_min, panel->backlight.pwm_level_max,
diff --git a/drivers/gpu/drm/i915/display/intel_bios.c b/drivers/gpu/drm/i915/display/intel_bios.c
index 4e8f1e91bb08..2fd72b2fd109 100644
--- a/drivers/gpu/drm/i915/display/intel_bios.c
+++ b/drivers/gpu/drm/i915/display/intel_bios.c
@@ -1116,7 +1116,7 @@ parse_sdvo_panel_data(struct drm_i915_private *i915,
 	struct drm_display_mode *panel_fixed_mode;
 	int index;
 
-	index = i915->params.vbt_sdvo_panel_type;
+	index = i915->display.params.vbt_sdvo_panel_type;
 	if (index == -2) {
 		drm_dbg_kms(&i915->drm,
 			    "Ignore SDVO panel mode from BIOS VBT tables.\n");
@@ -1514,9 +1514,9 @@ parse_edp(struct drm_i915_private *i915,
 		u8 vswing;
 
 		/* Don't read from VBT if module parameter has valid value*/
-		if (i915->params.edp_vswing) {
+		if (i915->display.params.edp_vswing) {
 			panel->vbt.edp.low_vswing =
-				i915->params.edp_vswing == 1;
+				i915->display.params.edp_vswing == 1;
 		} else {
 			vswing = (edp->edp_vswing_preemph >> (panel_type * 4)) & 0xF;
 			panel->vbt.edp.low_vswing = vswing == 0;
@@ -2473,6 +2473,27 @@ static void sanitize_device_type(struct intel_bios_encoder_data *devdata,
 	devdata->child.device_type |= DEVICE_TYPE_NOT_HDMI_OUTPUT;
 }
 
+static void sanitize_hdmi_level_shift(struct intel_bios_encoder_data *devdata,
+				      enum port port)
+{
+	struct drm_i915_private *i915 = devdata->i915;
+
+	if (!intel_bios_encoder_supports_dvi(devdata))
+		return;
+
+	/*
+	 * Some BDW machines (eg. HP Pavilion 15-ab) shipped
+	 * with a HSW VBT where the level shifter value goes
+	 * up to 11, whereas the BDW max is 9.
+	 */
+	if (IS_BROADWELL(i915) && devdata->child.hdmi_level_shifter_value > 9) {
+		drm_dbg_kms(&i915->drm, "Bogus port %c VBT HDMI level shift %d, adjusting to %d\n",
+			    port_name(port), devdata->child.hdmi_level_shifter_value, 9);
+
+		devdata->child.hdmi_level_shifter_value = 9;
+	}
+}
+
 static bool
 intel_bios_encoder_supports_crt(const struct intel_bios_encoder_data *devdata)
 {
@@ -2652,6 +2673,7 @@ static void parse_ddi_port(struct intel_bios_encoder_data *devdata)
 	}
 
 	sanitize_device_type(devdata, port);
+	sanitize_hdmi_level_shift(devdata, port);
 }
 
 static bool has_ddi_port_info(struct drm_i915_private *i915)
@@ -3392,8 +3414,8 @@ static void fill_dsc(struct intel_crtc_state *crtc_state,
 
 	crtc_state->pipe_bpp = bpc * 3;
 
-	crtc_state->dsc.compressed_bpp = min(crtc_state->pipe_bpp,
-					     VBT_DSC_MAX_BPP(dsc->max_bpp));
+	crtc_state->dsc.compressed_bpp_x16 = to_bpp_x16(min(crtc_state->pipe_bpp,
+							    VBT_DSC_MAX_BPP(dsc->max_bpp)));
 
 	/*
 	 * FIXME: This is ugly, and slice count should take DSC engine
diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.c b/drivers/gpu/drm/i915/display/intel_cdclk.c
index c4839c67cb0f..b93d1ad7936d 100644
--- a/drivers/gpu/drm/i915/display/intel_cdclk.c
+++ b/drivers/gpu/drm/i915/display/intel_cdclk.c
@@ -2598,8 +2598,9 @@ static int intel_vdsc_min_cdclk(const struct intel_crtc_state *crtc_state)
 		 * => CDCLK >= compressed_bpp * Pixel clock  / 2 * Bigjoiner Interface bits
 		 */
 		int bigjoiner_interface_bits = DISPLAY_VER(i915) > 13 ? 36 : 24;
-		int min_cdclk_bj = (crtc_state->dsc.compressed_bpp * pixel_clock) /
-				   (2 * bigjoiner_interface_bits);
+		int min_cdclk_bj =
+			(to_bpp_int_roundup(crtc_state->dsc.compressed_bpp_x16) *
+			 pixel_clock) / (2 * bigjoiner_interface_bits);
 
 		min_cdclk = max(min_cdclk, min_cdclk_bj);
 	}
diff --git a/drivers/gpu/drm/i915/display/intel_crt.c b/drivers/gpu/drm/i915/display/intel_crt.c
index 913e5d230a4d..0e33a0523a75 100644
--- a/drivers/gpu/drm/i915/display/intel_crt.c
+++ b/drivers/gpu/drm/i915/display/intel_crt.c
@@ -841,7 +841,7 @@ intel_crt_detect(struct drm_connector *connector,
 	if (!intel_display_device_enabled(dev_priv))
 		return connector_status_disconnected;
 
-	if (dev_priv->params.load_detect_test) {
+	if (dev_priv->display.params.load_detect_test) {
 		wakeref = intel_display_power_get(dev_priv,
 						  intel_encoder->power_domain);
 		goto load_detect;
@@ -901,7 +901,7 @@ intel_crt_detect(struct drm_connector *connector,
 		else if (DISPLAY_VER(dev_priv) < 4)
 			status = intel_crt_load_detect(crt,
 				to_intel_crtc(connector->state->crtc)->pipe);
-		else if (dev_priv->params.load_detect_test)
+		else if (dev_priv->display.params.load_detect_test)
 			status = connector_status_disconnected;
 		else
 			status = connector_status_unknown;
diff --git a/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c b/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c
index 66fe880af8f3..2d15e82c0b3d 100644
--- a/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c
+++ b/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c
@@ -4,6 +4,7 @@
  */
 
 #include <drm/drm_edid.h>
+#include <drm/drm_eld.h>
 
 #include "i915_drv.h"
 #include "intel_crtc_state_dump.h"
diff --git a/drivers/gpu/drm/i915/display/intel_cx0_phy.c b/drivers/gpu/drm/i915/display/intel_cx0_phy.c
index d414f6b7f993..a8fa76580802 100644
--- a/drivers/gpu/drm/i915/display/intel_cx0_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_cx0_phy.c
@@ -31,7 +31,7 @@
 
 bool intel_is_c10phy(struct drm_i915_private *i915, enum phy phy)
 {
-	if (DISPLAY_VER_FULL(i915) == IP_VER(14, 0) && phy < PHY_C)
+	if ((IS_LUNARLAKE(i915) || IS_METEORLAKE(i915)) && phy < PHY_C)
 		return true;
 
 	return false;
@@ -206,6 +206,13 @@ static int __intel_cx0_read_once(struct drm_i915_private *i915, enum port port,
 
 	intel_clear_response_ready_flag(i915, port, lane);
 
+	/*
+	 * FIXME: Workaround to let HW to settle
+	 * down and let the message bus to end up
+	 * in a known state
+	 */
+	intel_cx0_bus_reset(i915, port, lane);
+
 	return REG_FIELD_GET(XELPDP_PORT_P2M_DATA_MASK, val);
 }
 
@@ -285,6 +292,13 @@ static int __intel_cx0_write_once(struct drm_i915_private *i915, enum port port,
 
 	intel_clear_response_ready_flag(i915, port, lane);
 
+	/*
+	 * FIXME: Workaround to let HW to settle
+	 * down and let the message bus to end up
+	 * in a known state
+	 */
+	intel_cx0_bus_reset(i915, port, lane);
+
 	return 0;
 }
 
@@ -1850,8 +1864,8 @@ static int intel_c10pll_calc_state(struct intel_crtc_state *crtc_state,
 	return -EINVAL;
 }
 
-void intel_c10pll_readout_hw_state(struct intel_encoder *encoder,
-				   struct intel_c10pll_state *pll_state)
+static void intel_c10pll_readout_hw_state(struct intel_encoder *encoder,
+					  struct intel_c10pll_state *pll_state)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	u8 lane = INTEL_CX0_LANE0;
@@ -2103,8 +2117,8 @@ static bool intel_c20_use_mplla(u32 clock)
 	return false;
 }
 
-void intel_c20pll_readout_hw_state(struct intel_encoder *encoder,
-				   struct intel_c20pll_state *pll_state)
+static void intel_c20pll_readout_hw_state(struct intel_encoder *encoder,
+					  struct intel_c20pll_state *pll_state)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	bool cntx;
@@ -2378,8 +2392,8 @@ static void intel_c20_pll_program(struct drm_i915_private *i915,
 		      BIT(0), cntx ? 0 : 1, MB_WRITE_COMMITTED);
 }
 
-int intel_c10pll_calc_port_clock(struct intel_encoder *encoder,
-				 const struct intel_c10pll_state *pll_state)
+static int intel_c10pll_calc_port_clock(struct intel_encoder *encoder,
+					const struct intel_c10pll_state *pll_state)
 {
 	unsigned int frac_quot = 0, frac_rem = 0, frac_den = 1;
 	unsigned int multiplier, tx_clk_div, hdmi_div, refclk = 38400;
@@ -2405,8 +2419,8 @@ int intel_c10pll_calc_port_clock(struct intel_encoder *encoder,
 	return tmpclk;
 }
 
-int intel_c20pll_calc_port_clock(struct intel_encoder *encoder,
-				 const struct intel_c20pll_state *pll_state)
+static int intel_c20pll_calc_port_clock(struct intel_encoder *encoder,
+					const struct intel_c20pll_state *pll_state)
 {
 	unsigned int frac, frac_en, frac_quot, frac_rem, frac_den;
 	unsigned int multiplier, refclk = 38400;
@@ -3003,17 +3017,110 @@ intel_mtl_port_pll_type(struct intel_encoder *encoder,
 		return ICL_PORT_DPLL_DEFAULT;
 }
 
-void intel_c10pll_state_verify(struct intel_atomic_state *state,
+static void intel_c10pll_state_verify(const struct intel_crtc_state *state,
+				      struct intel_crtc *crtc,
+				      struct intel_encoder *encoder,
+				      struct intel_c10pll_state *mpllb_hw_state)
+{
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	const struct intel_c10pll_state *mpllb_sw_state = &state->cx0pll_state.c10;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(mpllb_sw_state->pll); i++) {
+		u8 expected = mpllb_sw_state->pll[i];
+
+		I915_STATE_WARN(i915, mpllb_hw_state->pll[i] != expected,
+				"[CRTC:%d:%s] mismatch in C10MPLLB: Register[%d] (expected 0x%02x, found 0x%02x)",
+				crtc->base.base.id, crtc->base.name, i,
+				expected, mpllb_hw_state->pll[i]);
+	}
+
+	I915_STATE_WARN(i915, mpllb_hw_state->tx != mpllb_sw_state->tx,
+			"[CRTC:%d:%s] mismatch in C10MPLLB: Register TX0 (expected 0x%02x, found 0x%02x)",
+			crtc->base.base.id, crtc->base.name,
+			mpllb_sw_state->tx, mpllb_hw_state->tx);
+
+	I915_STATE_WARN(i915, mpllb_hw_state->cmn != mpllb_sw_state->cmn,
+			"[CRTC:%d:%s] mismatch in C10MPLLB: Register CMN0 (expected 0x%02x, found 0x%02x)",
+			crtc->base.base.id, crtc->base.name,
+			mpllb_sw_state->cmn, mpllb_hw_state->cmn);
+}
+
+void intel_cx0pll_readout_hw_state(struct intel_encoder *encoder,
+				   struct intel_cx0pll_state *pll_state)
+{
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	enum phy phy = intel_port_to_phy(i915, encoder->port);
+
+	if (intel_is_c10phy(i915, phy))
+		intel_c10pll_readout_hw_state(encoder, &pll_state->c10);
+	else
+		intel_c20pll_readout_hw_state(encoder, &pll_state->c20);
+}
+
+int intel_cx0pll_calc_port_clock(struct intel_encoder *encoder,
+				 const struct intel_cx0pll_state *pll_state)
+{
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	enum phy phy = intel_port_to_phy(i915, encoder->port);
+
+	if (intel_is_c10phy(i915, phy))
+		return intel_c10pll_calc_port_clock(encoder, &pll_state->c10);
+
+	return intel_c20pll_calc_port_clock(encoder, &pll_state->c20);
+}
+
+static void intel_c20pll_state_verify(const struct intel_crtc_state *state,
+				      struct intel_crtc *crtc,
+				      struct intel_encoder *encoder,
+				      struct intel_c20pll_state *mpll_hw_state)
+{
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	const struct intel_c20pll_state *mpll_sw_state = &state->cx0pll_state.c20;
+	bool use_mplla;
+	int i;
+
+	use_mplla = intel_c20_use_mplla(mpll_hw_state->clock);
+	if (use_mplla) {
+		for (i = 0; i < ARRAY_SIZE(mpll_sw_state->mplla); i++) {
+			I915_STATE_WARN(i915, mpll_hw_state->mplla[i] != mpll_sw_state->mplla[i],
+					"[CRTC:%d:%s] mismatch in C20MPLLA: Register[%d] (expected 0x%04x, found 0x%04x)",
+					crtc->base.base.id, crtc->base.name, i,
+					mpll_sw_state->mplla[i], mpll_hw_state->mplla[i]);
+		}
+	} else {
+		for (i = 0; i < ARRAY_SIZE(mpll_sw_state->mpllb); i++) {
+			I915_STATE_WARN(i915, mpll_hw_state->mpllb[i] != mpll_sw_state->mpllb[i],
+					"[CRTC:%d:%s] mismatch in C20MPLLB: Register[%d] (expected 0x%04x, found 0x%04x)",
+					crtc->base.base.id, crtc->base.name, i,
+					mpll_sw_state->mpllb[i], mpll_hw_state->mpllb[i]);
+		}
+	}
+
+	for (i = 0; i < ARRAY_SIZE(mpll_sw_state->tx); i++) {
+		I915_STATE_WARN(i915, mpll_hw_state->tx[i] != mpll_sw_state->tx[i],
+				"[CRTC:%d:%s] mismatch in C20: Register TX[%i] (expected 0x%04x, found 0x%04x)",
+				crtc->base.base.id, crtc->base.name, i,
+				mpll_sw_state->tx[i], mpll_hw_state->tx[i]);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(mpll_sw_state->cmn); i++) {
+		I915_STATE_WARN(i915, mpll_hw_state->cmn[i] != mpll_sw_state->cmn[i],
+				"[CRTC:%d:%s] mismatch in C20: Register CMN[%i] (expected 0x%04x, found 0x%04x)",
+				crtc->base.base.id, crtc->base.name, i,
+				mpll_sw_state->cmn[i], mpll_hw_state->cmn[i]);
+	}
+}
+
+void intel_cx0pll_state_verify(struct intel_atomic_state *state,
 			       struct intel_crtc *crtc)
 {
 	struct drm_i915_private *i915 = to_i915(state->base.dev);
 	const struct intel_crtc_state *new_crtc_state =
 		intel_atomic_get_new_crtc_state(state, crtc);
-	struct intel_c10pll_state mpllb_hw_state = {};
-	const struct intel_c10pll_state *mpllb_sw_state = &new_crtc_state->cx0pll_state.c10;
 	struct intel_encoder *encoder;
+	struct intel_cx0pll_state mpll_hw_state = {};
 	enum phy phy;
-	int i;
 
 	if (DISPLAY_VER(i915) < 14)
 		return;
@@ -3029,27 +3136,10 @@ void intel_c10pll_state_verify(struct intel_atomic_state *state,
 	encoder = intel_get_crtc_new_encoder(state, new_crtc_state);
 	phy = intel_port_to_phy(i915, encoder->port);
 
-	if (!intel_is_c10phy(i915, phy))
-		return;
-
-	intel_c10pll_readout_hw_state(encoder, &mpllb_hw_state);
+	intel_cx0pll_readout_hw_state(encoder, &mpll_hw_state);
 
-	for (i = 0; i < ARRAY_SIZE(mpllb_sw_state->pll); i++) {
-		u8 expected = mpllb_sw_state->pll[i];
-
-		I915_STATE_WARN(i915, mpllb_hw_state.pll[i] != expected,
-				"[CRTC:%d:%s] mismatch in C10MPLLB: Register[%d] (expected 0x%02x, found 0x%02x)",
-				crtc->base.base.id, crtc->base.name, i,
-				expected, mpllb_hw_state.pll[i]);
-	}
-
-	I915_STATE_WARN(i915, mpllb_hw_state.tx != mpllb_sw_state->tx,
-			"[CRTC:%d:%s] mismatch in C10MPLLB: Register TX0 (expected 0x%02x, found 0x%02x)",
-			crtc->base.base.id, crtc->base.name,
-			mpllb_sw_state->tx, mpllb_hw_state.tx);
-
-	I915_STATE_WARN(i915, mpllb_hw_state.cmn != mpllb_sw_state->cmn,
-			"[CRTC:%d:%s] mismatch in C10MPLLB: Register CMN0 (expected 0x%02x, found 0x%02x)",
-			crtc->base.base.id, crtc->base.name,
-			mpllb_sw_state->cmn, mpllb_hw_state.cmn);
+	if (intel_is_c10phy(i915, phy))
+		intel_c10pll_state_verify(new_crtc_state, crtc, encoder, &mpll_hw_state.c10);
+	else
+		intel_c20pll_state_verify(new_crtc_state, crtc, encoder, &mpll_hw_state.c20);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_cx0_phy.h b/drivers/gpu/drm/i915/display/intel_cx0_phy.h
index 0e0a38dac8cd..c6682677253a 100644
--- a/drivers/gpu/drm/i915/display/intel_cx0_phy.h
+++ b/drivers/gpu/drm/i915/display/intel_cx0_phy.h
@@ -16,6 +16,7 @@ struct drm_i915_private;
 struct intel_atomic_state;
 struct intel_c10pll_state;
 struct intel_c20pll_state;
+struct intel_cx0pll_state;
 struct intel_crtc;
 struct intel_crtc_state;
 struct intel_encoder;
@@ -28,20 +29,19 @@ void intel_mtl_pll_disable(struct intel_encoder *encoder);
 enum icl_port_dpll_id
 intel_mtl_port_pll_type(struct intel_encoder *encoder,
 			const struct intel_crtc_state *crtc_state);
-void intel_c10pll_readout_hw_state(struct intel_encoder *encoder, struct intel_c10pll_state *pll_state);
+
 int intel_cx0pll_calc_state(struct intel_crtc_state *crtc_state, struct intel_encoder *encoder);
+void intel_cx0pll_readout_hw_state(struct intel_encoder *encoder,
+				   struct intel_cx0pll_state *pll_state);
+int intel_cx0pll_calc_port_clock(struct intel_encoder *encoder,
+				 const struct intel_cx0pll_state *pll_state);
+
 void intel_c10pll_dump_hw_state(struct drm_i915_private *dev_priv,
 				const struct intel_c10pll_state *hw_state);
-int intel_c10pll_calc_port_clock(struct intel_encoder *encoder,
-				 const struct intel_c10pll_state *pll_state);
-void intel_c10pll_state_verify(struct intel_atomic_state *state,
+void intel_cx0pll_state_verify(struct intel_atomic_state *state,
 			       struct intel_crtc *crtc);
-void intel_c20pll_readout_hw_state(struct intel_encoder *encoder,
-				   struct intel_c20pll_state *pll_state);
 void intel_c20pll_dump_hw_state(struct drm_i915_private *i915,
 				const struct intel_c20pll_state *hw_state);
-int intel_c20pll_calc_port_clock(struct intel_encoder *encoder,
-				 const struct intel_c20pll_state *pll_state);
 void intel_cx0_phy_set_signal_levels(struct intel_encoder *encoder,
 				     const struct intel_crtc_state *crtc_state);
 int intel_cx0_phy_check_hdmi_link_rate(struct intel_hdmi *hdmi, int clock);
diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c
index 9151d5add960..f70af660dfcf 100644
--- a/drivers/gpu/drm/i915/display/intel_ddi.c
+++ b/drivers/gpu/drm/i915/display/intel_ddi.c
@@ -25,6 +25,7 @@
  *
  */
 
+#include <linux/iopoll.h>
 #include <linux/string_helpers.h>
 
 #include <drm/display/drm_scdc_helper.h>
@@ -2210,16 +2211,87 @@ static void intel_dp_sink_set_msa_timing_par_ignore_state(struct intel_dp *intel
 }
 
 static void intel_dp_sink_set_fec_ready(struct intel_dp *intel_dp,
-					const struct intel_crtc_state *crtc_state)
+					const struct intel_crtc_state *crtc_state,
+					bool enable)
 {
 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
 
 	if (!crtc_state->fec_enable)
 		return;
 
-	if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_FEC_CONFIGURATION, DP_FEC_READY) <= 0)
-		drm_dbg_kms(&i915->drm,
-			    "Failed to set FEC_READY in the sink\n");
+	if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_FEC_CONFIGURATION,
+			       enable ? DP_FEC_READY : 0) <= 0)
+		drm_dbg_kms(&i915->drm, "Failed to set FEC_READY to %s in the sink\n",
+			    enable ? "enabled" : "disabled");
+
+	if (enable &&
+	    drm_dp_dpcd_writeb(&intel_dp->aux, DP_FEC_STATUS,
+			       DP_FEC_DECODE_EN_DETECTED | DP_FEC_DECODE_DIS_DETECTED) <= 0)
+		drm_dbg_kms(&i915->drm, "Failed to clear FEC detected flags\n");
+}
+
+static int read_fec_detected_status(struct drm_dp_aux *aux)
+{
+	int ret;
+	u8 status;
+
+	ret = drm_dp_dpcd_readb(aux, DP_FEC_STATUS, &status);
+	if (ret < 0)
+		return ret;
+
+	return status;
+}
+
+static void wait_for_fec_detected(struct drm_dp_aux *aux, bool enabled)
+{
+	struct drm_i915_private *i915 = to_i915(aux->drm_dev);
+	int mask = enabled ? DP_FEC_DECODE_EN_DETECTED : DP_FEC_DECODE_DIS_DETECTED;
+	int status;
+	int err;
+
+	err = readx_poll_timeout(read_fec_detected_status, aux, status,
+				 status & mask || status < 0,
+				 10000, 200000);
+
+	if (!err && status >= 0)
+		return;
+
+	if (err == -ETIMEDOUT)
+		drm_dbg_kms(&i915->drm, "Timeout waiting for FEC %s to get detected\n",
+			    str_enabled_disabled(enabled));
+	else
+		drm_dbg_kms(&i915->drm, "FEC detected status read error: %d\n", status);
+}
+
+void intel_ddi_wait_for_fec_status(struct intel_encoder *encoder,
+				   const struct intel_crtc_state *crtc_state,
+				   bool enabled)
+{
+	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
+	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
+	int ret;
+
+	if (!crtc_state->fec_enable)
+		return;
+
+	if (enabled)
+		ret = intel_de_wait_for_set(i915, dp_tp_status_reg(encoder, crtc_state),
+					    DP_TP_STATUS_FEC_ENABLE_LIVE, 1);
+	else
+		ret = intel_de_wait_for_clear(i915, dp_tp_status_reg(encoder, crtc_state),
+					      DP_TP_STATUS_FEC_ENABLE_LIVE, 1);
+
+	if (ret)
+		drm_err(&i915->drm,
+			"Timeout waiting for FEC live state to get %s\n",
+			str_enabled_disabled(enabled));
+
+	/*
+	 * At least the Synoptics MST hub doesn't set the detected flag for
+	 * FEC decoding disabling so skip waiting for that.
+	 */
+	if (enabled)
+		wait_for_fec_detected(&intel_dp->aux, enabled);
 }
 
 static void intel_ddi_enable_fec(struct intel_encoder *encoder,
@@ -2234,8 +2306,8 @@ static void intel_ddi_enable_fec(struct intel_encoder *encoder,
 		     0, DP_TP_CTL_FEC_ENABLE);
 }
 
-static void intel_ddi_disable_fec_state(struct intel_encoder *encoder,
-					const struct intel_crtc_state *crtc_state)
+static void intel_ddi_disable_fec(struct intel_encoder *encoder,
+				  const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
@@ -2466,13 +2538,17 @@ static void mtl_ddi_pre_enable_dp(struct intel_atomic_state *state,
 		intel_dp_set_power(intel_dp, DP_SET_POWER_D0);
 
 	intel_dp_configure_protocol_converter(intel_dp, crtc_state);
-	intel_dp_sink_set_decompression_state(intel_dp, crtc_state, true);
+	if (!is_mst)
+		intel_dp_sink_enable_decompression(state,
+						   to_intel_connector(conn_state->connector),
+						   crtc_state);
+
 	/*
 	 * DDI FEC: "anticipates enabling FEC encoding sets the FEC_READY bit
 	 * in the FEC_CONFIGURATION register to 1 before initiating link
 	 * training
 	 */
-	intel_dp_sink_set_fec_ready(intel_dp, crtc_state);
+	intel_dp_sink_set_fec_ready(intel_dp, crtc_state, true);
 
 	intel_dp_check_frl_training(intel_dp);
 	intel_dp_pcon_dsc_configure(intel_dp, crtc_state);
@@ -2505,7 +2581,8 @@ static void mtl_ddi_pre_enable_dp(struct intel_atomic_state *state,
 	/* 6.o Configure and enable FEC if needed */
 	intel_ddi_enable_fec(encoder, crtc_state);
 
-	intel_dsc_dp_pps_write(encoder, crtc_state);
+	if (!is_mst)
+		intel_dsc_dp_pps_write(encoder, crtc_state);
 }
 
 static void tgl_ddi_pre_enable_dp(struct intel_atomic_state *state,
@@ -2616,13 +2693,16 @@ static void tgl_ddi_pre_enable_dp(struct intel_atomic_state *state,
 		intel_dp_set_power(intel_dp, DP_SET_POWER_D0);
 
 	intel_dp_configure_protocol_converter(intel_dp, crtc_state);
-	intel_dp_sink_set_decompression_state(intel_dp, crtc_state, true);
+	if (!is_mst)
+		intel_dp_sink_enable_decompression(state,
+						   to_intel_connector(conn_state->connector),
+						   crtc_state);
 	/*
 	 * DDI FEC: "anticipates enabling FEC encoding sets the FEC_READY bit
 	 * in the FEC_CONFIGURATION register to 1 before initiating link
 	 * training
 	 */
-	intel_dp_sink_set_fec_ready(intel_dp, crtc_state);
+	intel_dp_sink_set_fec_ready(intel_dp, crtc_state, true);
 
 	intel_dp_check_frl_training(intel_dp);
 	intel_dp_pcon_dsc_configure(intel_dp, crtc_state);
@@ -2643,7 +2723,8 @@ static void tgl_ddi_pre_enable_dp(struct intel_atomic_state *state,
 	/* 7.l Configure and enable FEC if needed */
 	intel_ddi_enable_fec(encoder, crtc_state);
 
-	intel_dsc_dp_pps_write(encoder, crtc_state);
+	if (!is_mst)
+		intel_dsc_dp_pps_write(encoder, crtc_state);
 }
 
 static void hsw_ddi_pre_enable_dp(struct intel_atomic_state *state,
@@ -2695,9 +2776,11 @@ static void hsw_ddi_pre_enable_dp(struct intel_atomic_state *state,
 	if (!is_mst)
 		intel_dp_set_power(intel_dp, DP_SET_POWER_D0);
 	intel_dp_configure_protocol_converter(intel_dp, crtc_state);
-	intel_dp_sink_set_decompression_state(intel_dp, crtc_state,
-					      true);
-	intel_dp_sink_set_fec_ready(intel_dp, crtc_state);
+	if (!is_mst)
+		intel_dp_sink_enable_decompression(state,
+						   to_intel_connector(conn_state->connector),
+						   crtc_state);
+	intel_dp_sink_set_fec_ready(intel_dp, crtc_state, true);
 	intel_dp_start_link_train(intel_dp, crtc_state);
 	if ((port != PORT_A || DISPLAY_VER(dev_priv) >= 9) &&
 	    !is_trans_port_sync_mode(crtc_state))
@@ -2705,10 +2788,10 @@ static void hsw_ddi_pre_enable_dp(struct intel_atomic_state *state,
 
 	intel_ddi_enable_fec(encoder, crtc_state);
 
-	if (!is_mst)
+	if (!is_mst) {
 		intel_ddi_enable_transcoder_clock(encoder, crtc_state);
-
-	intel_dsc_dp_pps_write(encoder, crtc_state);
+		intel_dsc_dp_pps_write(encoder, crtc_state);
+	}
 }
 
 static void intel_ddi_pre_enable_dp(struct intel_atomic_state *state,
@@ -2717,10 +2800,15 @@ static void intel_ddi_pre_enable_dp(struct intel_atomic_state *state,
 				    const struct drm_connector_state *conn_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 
-	if (HAS_DP20(dev_priv))
+	if (HAS_DP20(dev_priv)) {
 		intel_dp_128b132b_sdp_crc16(enc_to_intel_dp(encoder),
 					    crtc_state);
+		if (crtc_state->has_panel_replay)
+			drm_dp_dpcd_writeb(&intel_dp->aux, PANEL_REPLAY_CONFIG,
+					   DP_PANEL_REPLAY_ENABLE);
+	}
 
 	if (DISPLAY_VER(dev_priv) >= 14)
 		mtl_ddi_pre_enable_dp(state, encoder, crtc_state, conn_state);
@@ -2866,8 +2954,7 @@ static void disable_ddi_buf(struct intel_encoder *encoder,
 		intel_de_rmw(dev_priv, dp_tp_ctl_reg(encoder, crtc_state),
 			     DP_TP_CTL_ENABLE, 0);
 
-	/* Disable FEC in DP Sink */
-	intel_ddi_disable_fec_state(encoder, crtc_state);
+	intel_ddi_disable_fec(encoder, crtc_state);
 
 	if (wait)
 		intel_wait_ddi_buf_idle(dev_priv, port);
@@ -2882,10 +2969,12 @@ static void intel_disable_ddi_buf(struct intel_encoder *encoder,
 		mtl_disable_ddi_buf(encoder, crtc_state);
 
 		/* 3.f Disable DP_TP_CTL FEC Enable if it is needed */
-		intel_ddi_disable_fec_state(encoder, crtc_state);
+		intel_ddi_disable_fec(encoder, crtc_state);
 	} else {
 		disable_ddi_buf(encoder, crtc_state);
 	}
+
+	intel_ddi_wait_for_fec_status(encoder, crtc_state, false);
 }
 
 static void intel_ddi_post_disable_dp(struct intel_atomic_state *state,
@@ -2925,6 +3014,8 @@ static void intel_ddi_post_disable_dp(struct intel_atomic_state *state,
 
 	intel_disable_ddi_buf(encoder, old_crtc_state);
 
+	intel_dp_sink_set_fec_ready(intel_dp, old_crtc_state, false);
+
 	/*
 	 * From TGL spec: "If single stream or multi-stream master transcoder:
 	 * Configure Transcoder Clock select to direct no clock to the
@@ -3115,6 +3206,15 @@ static void intel_enable_ddi_dp(struct intel_atomic_state *state,
 	trans_port_sync_stop_link_train(state, encoder, crtc_state);
 }
 
+/* FIXME bad home for this function */
+i915_reg_t hsw_chicken_trans_reg(struct drm_i915_private *i915,
+				 enum transcoder cpu_transcoder)
+{
+	return DISPLAY_VER(i915) >= 14 ?
+		MTL_CHICKEN_TRANS(cpu_transcoder) :
+		CHICKEN_TRANS(cpu_transcoder);
+}
+
 static i915_reg_t
 gen9_chicken_trans_reg_by_port(struct drm_i915_private *dev_priv,
 			       enum port port)
@@ -3252,6 +3352,8 @@ static void intel_enable_ddi(struct intel_atomic_state *state,
 
 	intel_enable_transcoder(crtc_state);
 
+	intel_ddi_wait_for_fec_status(encoder, crtc_state, true);
+
 	intel_crtc_vblank_on(crtc_state);
 
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
@@ -3259,10 +3361,7 @@ static void intel_enable_ddi(struct intel_atomic_state *state,
 	else
 		intel_enable_ddi_dp(state, encoder, crtc_state, conn_state);
 
-	/* Enable hdcp if it's desired */
-	if (conn_state->content_protection ==
-	    DRM_MODE_CONTENT_PROTECTION_DESIRED)
-		intel_hdcp_enable(state, encoder, crtc_state, conn_state);
+	intel_hdcp_enable(state, encoder, crtc_state, conn_state);
 }
 
 static void intel_disable_ddi_dp(struct intel_atomic_state *state,
@@ -3271,6 +3370,8 @@ static void intel_disable_ddi_dp(struct intel_atomic_state *state,
 				 const struct drm_connector_state *old_conn_state)
 {
 	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
+	struct intel_connector *connector =
+		to_intel_connector(old_conn_state->connector);
 
 	intel_dp->link_trained = false;
 
@@ -3279,8 +3380,8 @@ static void intel_disable_ddi_dp(struct intel_atomic_state *state,
 	intel_psr_disable(intel_dp, old_crtc_state);
 	intel_edp_backlight_off(old_conn_state);
 	/* Disable the decompression in DP Sink */
-	intel_dp_sink_set_decompression_state(intel_dp, old_crtc_state,
-					      false);
+	intel_dp_sink_disable_decompression(state,
+					    connector, old_crtc_state);
 	/* Disable Ignore_MSA bit in DP Sink */
 	intel_dp_sink_set_msa_timing_par_ignore_state(intel_dp, old_crtc_state,
 						      false);
@@ -3854,18 +3955,13 @@ void intel_ddi_get_clock(struct intel_encoder *encoder,
 static void mtl_ddi_get_config(struct intel_encoder *encoder,
 			       struct intel_crtc_state *crtc_state)
 {
-	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 
 	if (intel_tc_port_in_tbt_alt_mode(dig_port)) {
 		crtc_state->port_clock = intel_mtl_tbt_calc_port_clock(encoder);
-	} else if (intel_is_c10phy(i915, phy)) {
-		intel_c10pll_readout_hw_state(encoder, &crtc_state->cx0pll_state.c10);
-		crtc_state->port_clock = intel_c10pll_calc_port_clock(encoder, &crtc_state->cx0pll_state.c10);
 	} else {
-		intel_c20pll_readout_hw_state(encoder, &crtc_state->cx0pll_state.c20);
-		crtc_state->port_clock = intel_c20pll_calc_port_clock(encoder, &crtc_state->cx0pll_state.c20);
+		intel_cx0pll_readout_hw_state(encoder, &crtc_state->cx0pll_state);
+		crtc_state->port_clock = intel_cx0pll_calc_port_clock(encoder, &crtc_state->cx0pll_state);
 	}
 
 	intel_ddi_get_config(encoder, crtc_state);
diff --git a/drivers/gpu/drm/i915/display/intel_ddi.h b/drivers/gpu/drm/i915/display/intel_ddi.h
index 4999c0ee229b..63853a1f6582 100644
--- a/drivers/gpu/drm/i915/display/intel_ddi.h
+++ b/drivers/gpu/drm/i915/display/intel_ddi.h
@@ -27,6 +27,8 @@ i915_reg_t dp_tp_ctl_reg(struct intel_encoder *encoder,
 			 const struct intel_crtc_state *crtc_state);
 i915_reg_t dp_tp_status_reg(struct intel_encoder *encoder,
 			    const struct intel_crtc_state *crtc_state);
+i915_reg_t hsw_chicken_trans_reg(struct drm_i915_private *i915,
+				 enum transcoder cpu_transcoder);
 void intel_ddi_fdi_post_disable(struct intel_atomic_state *state,
 				struct intel_encoder *intel_encoder,
 				const struct intel_crtc_state *old_crtc_state,
@@ -60,6 +62,9 @@ void intel_ddi_disable_transcoder_func(const struct intel_crtc_state *crtc_state
 void intel_ddi_enable_transcoder_clock(struct intel_encoder *encoder,
 				       const struct intel_crtc_state *crtc_state);
 void intel_ddi_disable_transcoder_clock(const  struct intel_crtc_state *crtc_state);
+void intel_ddi_wait_for_fec_status(struct intel_encoder *encoder,
+				   const struct intel_crtc_state *crtc_state,
+				   bool enabled);
 void intel_ddi_set_dp_msa(const struct intel_crtc_state *crtc_state,
 			  const struct drm_connector_state *conn_state);
 bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector);
diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c
index 28d85e1e858e..f8503d917c3d 100644
--- a/drivers/gpu/drm/i915/display/intel_display.c
+++ b/drivers/gpu/drm/i915/display/intel_display.c
@@ -48,6 +48,7 @@
 #include "g4x_dp.h"
 #include "g4x_hdmi.h"
 #include "hsw_ips.h"
+#include "i915_config.h"
 #include "i915_drv.h"
 #include "i915_reg.h"
 #include "i915_utils.h"
@@ -72,7 +73,6 @@
 #include "intel_dp.h"
 #include "intel_dp_link_training.h"
 #include "intel_dp_mst.h"
-#include "intel_dpio_phy.h"
 #include "intel_dpll.h"
 #include "intel_dpll_mgr.h"
 #include "intel_dpt.h"
@@ -397,7 +397,6 @@ void intel_enable_transcoder(const struct intel_crtc_state *new_crtc_state)
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
 	enum pipe pipe = crtc->pipe;
-	i915_reg_t reg;
 	u32 val;
 
 	drm_dbg_kms(&dev_priv->drm, "enabling pipe %c\n", pipe_name(pipe));
@@ -430,16 +429,16 @@ void intel_enable_transcoder(const struct intel_crtc_state *new_crtc_state)
 		intel_de_rmw(dev_priv, PIPE_ARB_CTL(pipe),
 			     0, PIPE_ARB_USE_PROG_SLOTS);
 
-	reg = TRANSCONF(cpu_transcoder);
-	val = intel_de_read(dev_priv, reg);
+	val = intel_de_read(dev_priv, TRANSCONF(cpu_transcoder));
 	if (val & TRANSCONF_ENABLE) {
 		/* we keep both pipes enabled on 830 */
 		drm_WARN_ON(&dev_priv->drm, !IS_I830(dev_priv));
 		return;
 	}
 
-	intel_de_write(dev_priv, reg, val | TRANSCONF_ENABLE);
-	intel_de_posting_read(dev_priv, reg);
+	intel_de_write(dev_priv, TRANSCONF(cpu_transcoder),
+		       val | TRANSCONF_ENABLE);
+	intel_de_posting_read(dev_priv, TRANSCONF(cpu_transcoder));
 
 	/*
 	 * Until the pipe starts PIPEDSL reads will return a stale value,
@@ -458,7 +457,6 @@ void intel_disable_transcoder(const struct intel_crtc_state *old_crtc_state)
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
 	enum pipe pipe = crtc->pipe;
-	i915_reg_t reg;
 	u32 val;
 
 	drm_dbg_kms(&dev_priv->drm, "disabling pipe %c\n", pipe_name(pipe));
@@ -469,8 +467,7 @@ void intel_disable_transcoder(const struct intel_crtc_state *old_crtc_state)
 	 */
 	assert_planes_disabled(crtc);
 
-	reg = TRANSCONF(cpu_transcoder);
-	val = intel_de_read(dev_priv, reg);
+	val = intel_de_read(dev_priv, TRANSCONF(cpu_transcoder));
 	if ((val & TRANSCONF_ENABLE) == 0)
 		return;
 
@@ -485,14 +482,12 @@ void intel_disable_transcoder(const struct intel_crtc_state *old_crtc_state)
 	if (!IS_I830(dev_priv))
 		val &= ~TRANSCONF_ENABLE;
 
-	if (DISPLAY_VER(dev_priv) >= 14)
-		intel_de_rmw(dev_priv, MTL_CHICKEN_TRANS(cpu_transcoder),
-			     FECSTALL_DIS_DPTSTREAM_DPTTG, 0);
-	else if (DISPLAY_VER(dev_priv) >= 12)
-		intel_de_rmw(dev_priv, CHICKEN_TRANS(cpu_transcoder),
+	intel_de_write(dev_priv, TRANSCONF(cpu_transcoder), val);
+
+	if (DISPLAY_VER(dev_priv) >= 12)
+		intel_de_rmw(dev_priv, hsw_chicken_trans_reg(dev_priv, cpu_transcoder),
 			     FECSTALL_DIS_DPTSTREAM_DPTTG, 0);
 
-	intel_de_write(dev_priv, reg, val);
 	if ((val & TRANSCONF_ENABLE) == 0)
 		intel_wait_for_pipe_off(old_crtc_state);
 }
@@ -1501,12 +1496,9 @@ static void hsw_set_linetime_wm(const struct intel_crtc_state *crtc_state)
 static void hsw_set_frame_start_delay(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	enum transcoder transcoder = crtc_state->cpu_transcoder;
-	i915_reg_t reg = DISPLAY_VER(dev_priv) >= 14 ? MTL_CHICKEN_TRANS(transcoder) :
-			 CHICKEN_TRANS(transcoder);
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 
-	intel_de_rmw(dev_priv, reg,
+	intel_de_rmw(i915, hsw_chicken_trans_reg(i915, crtc_state->cpu_transcoder),
 		     HSW_FRAME_START_DELAY_MASK,
 		     HSW_FRAME_START_DELAY(crtc_state->framestart_delay - 1));
 }
@@ -1784,31 +1776,31 @@ bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy)
 
 bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy)
 {
+	/*
+	 * DG2's "TC1", although TC-capable output, doesn't share the same flow
+	 * as other platforms on the display engine side and rather rely on the
+	 * SNPS PHY, that is programmed separately
+	 */
 	if (IS_DG2(dev_priv))
-		/* DG2's "TC1" output uses a SNPS PHY */
 		return false;
-	else if (IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER_FULL(dev_priv) == IP_VER(14, 0))
+
+	if (DISPLAY_VER(dev_priv) >= 13)
 		return phy >= PHY_F && phy <= PHY_I;
 	else if (IS_TIGERLAKE(dev_priv))
 		return phy >= PHY_D && phy <= PHY_I;
 	else if (IS_ICELAKE(dev_priv))
 		return phy >= PHY_C && phy <= PHY_F;
-	else
-		return false;
+
+	return false;
 }
 
 bool intel_phy_is_snps(struct drm_i915_private *dev_priv, enum phy phy)
 {
-	if (phy == PHY_NONE)
-		return false;
-	else if (IS_DG2(dev_priv))
-		/*
-		 * All four "combo" ports and the TC1 port (PHY E) use
-		 * Synopsis PHYs.
-		 */
-		return phy <= PHY_E;
-
-	return false;
+	/*
+	 * For DG2, and for DG2 only, all four "combo" ports and the TC1 port
+	 * (PHY E) use Synopsis PHYs. See intel_phy_is_tc().
+	 */
+	return IS_DG2(dev_priv) && phy > PHY_NONE && phy <= PHY_E;
 }
 
 enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port)
@@ -2397,15 +2389,15 @@ static void compute_m_n(u32 *ret_m, u32 *ret_n,
 }
 
 void
-intel_link_compute_m_n(u16 bits_per_pixel, int nlanes,
+intel_link_compute_m_n(u16 bits_per_pixel_x16, int nlanes,
 		       int pixel_clock, int link_clock,
-		       struct intel_link_m_n *m_n,
-		       bool fec_enable)
+		       int bw_overhead,
+		       struct intel_link_m_n *m_n)
 {
-	u32 data_clock = bits_per_pixel * pixel_clock;
-
-	if (fec_enable)
-		data_clock = intel_dp_mode_to_fec_clock(data_clock);
+	u32 link_symbol_clock = intel_dp_link_symbol_clock(link_clock);
+	u32 data_m = intel_dp_effective_data_rate(pixel_clock, bits_per_pixel_x16,
+						  bw_overhead);
+	u32 data_n = intel_dp_max_data_rate(link_clock, nlanes);
 
 	/*
 	 * Windows/BIOS uses fixed M/N values always. Follow suit.
@@ -2416,11 +2408,11 @@ intel_link_compute_m_n(u16 bits_per_pixel, int nlanes,
 	 */
 	m_n->tu = 64;
 	compute_m_n(&m_n->data_m, &m_n->data_n,
-		    data_clock, link_clock * nlanes * 8,
+		    data_m, data_n,
 		    0x8000000);
 
 	compute_m_n(&m_n->link_m, &m_n->link_n,
-		    pixel_clock, link_clock,
+		    pixel_clock, link_symbol_clock,
 		    0x80000);
 }
 
@@ -2838,67 +2830,6 @@ static void i9xx_get_pfit_config(struct intel_crtc_state *crtc_state)
 		intel_de_read(dev_priv, PFIT_PGM_RATIOS);
 }
 
-static void vlv_crtc_clock_get(struct intel_crtc *crtc,
-			       struct intel_crtc_state *pipe_config)
-{
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	enum pipe pipe = crtc->pipe;
-	struct dpll clock;
-	u32 mdiv;
-	int refclk = 100000;
-
-	/* In case of DSI, DPLL will not be used */
-	if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
-		return;
-
-	vlv_dpio_get(dev_priv);
-	mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
-	vlv_dpio_put(dev_priv);
-
-	clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
-	clock.m2 = mdiv & DPIO_M2DIV_MASK;
-	clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
-	clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
-	clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
-
-	pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
-}
-
-static void chv_crtc_clock_get(struct intel_crtc *crtc,
-			       struct intel_crtc_state *pipe_config)
-{
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	enum pipe pipe = crtc->pipe;
-	enum dpio_channel port = vlv_pipe_to_channel(pipe);
-	struct dpll clock;
-	u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
-	int refclk = 100000;
-
-	/* In case of DSI, DPLL will not be used */
-	if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
-		return;
-
-	vlv_dpio_get(dev_priv);
-	cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
-	pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
-	pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
-	pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
-	pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
-	vlv_dpio_put(dev_priv);
-
-	clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
-	clock.m2 = (pll_dw0 & 0xff) << 22;
-	if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
-		clock.m2 |= pll_dw2 & 0x3fffff;
-	clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
-	clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
-	clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
-
-	pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
-}
-
 static enum intel_output_format
 bdw_get_pipe_misc_output_format(struct intel_crtc *crtc)
 {
@@ -3790,9 +3721,7 @@ static bool hsw_get_pipe_config(struct intel_crtc *crtc,
 	}
 
 	if (!transcoder_is_dsi(pipe_config->cpu_transcoder)) {
-		tmp = intel_de_read(dev_priv, DISPLAY_VER(dev_priv) >= 14 ?
-				    MTL_CHICKEN_TRANS(pipe_config->cpu_transcoder) :
-				    CHICKEN_TRANS(pipe_config->cpu_transcoder));
+		tmp = intel_de_read(dev_priv, hsw_chicken_trans_reg(dev_priv, pipe_config->cpu_transcoder));
 
 		pipe_config->framestart_delay = REG_FIELD_GET(HSW_FRAME_START_DELAY_MASK, tmp) + 1;
 	} else {
@@ -3821,133 +3750,27 @@ bool intel_crtc_get_pipe_config(struct intel_crtc_state *crtc_state)
 	return true;
 }
 
-static int i9xx_pll_refclk(struct drm_device *dev,
-			   const struct intel_crtc_state *pipe_config)
-{
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	u32 dpll = pipe_config->dpll_hw_state.dpll;
-
-	if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
-		return dev_priv->display.vbt.lvds_ssc_freq;
-	else if (HAS_PCH_SPLIT(dev_priv))
-		return 120000;
-	else if (DISPLAY_VER(dev_priv) != 2)
-		return 96000;
-	else
-		return 48000;
-}
-
-/* Returns the clock of the currently programmed mode of the given pipe. */
-void i9xx_crtc_clock_get(struct intel_crtc *crtc,
-			 struct intel_crtc_state *pipe_config)
-{
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	u32 dpll = pipe_config->dpll_hw_state.dpll;
-	u32 fp;
-	struct dpll clock;
-	int port_clock;
-	int refclk = i9xx_pll_refclk(dev, pipe_config);
-
-	if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
-		fp = pipe_config->dpll_hw_state.fp0;
-	else
-		fp = pipe_config->dpll_hw_state.fp1;
-
-	clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
-	if (IS_PINEVIEW(dev_priv)) {
-		clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
-		clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
-	} else {
-		clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
-		clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
-	}
-
-	if (DISPLAY_VER(dev_priv) != 2) {
-		if (IS_PINEVIEW(dev_priv))
-			clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
-				DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
-		else
-			clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
-			       DPLL_FPA01_P1_POST_DIV_SHIFT);
-
-		switch (dpll & DPLL_MODE_MASK) {
-		case DPLLB_MODE_DAC_SERIAL:
-			clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
-				5 : 10;
-			break;
-		case DPLLB_MODE_LVDS:
-			clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
-				7 : 14;
-			break;
-		default:
-			drm_dbg_kms(&dev_priv->drm,
-				    "Unknown DPLL mode %08x in programmed "
-				    "mode\n", (int)(dpll & DPLL_MODE_MASK));
-			return;
-		}
-
-		if (IS_PINEVIEW(dev_priv))
-			port_clock = pnv_calc_dpll_params(refclk, &clock);
-		else
-			port_clock = i9xx_calc_dpll_params(refclk, &clock);
-	} else {
-		enum pipe lvds_pipe;
-
-		if (IS_I85X(dev_priv) &&
-		    intel_lvds_port_enabled(dev_priv, LVDS, &lvds_pipe) &&
-		    lvds_pipe == crtc->pipe) {
-			u32 lvds = intel_de_read(dev_priv, LVDS);
-
-			clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
-				       DPLL_FPA01_P1_POST_DIV_SHIFT);
-
-			if (lvds & LVDS_CLKB_POWER_UP)
-				clock.p2 = 7;
-			else
-				clock.p2 = 14;
-		} else {
-			if (dpll & PLL_P1_DIVIDE_BY_TWO)
-				clock.p1 = 2;
-			else {
-				clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
-					    DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
-			}
-			if (dpll & PLL_P2_DIVIDE_BY_4)
-				clock.p2 = 4;
-			else
-				clock.p2 = 2;
-		}
-
-		port_clock = i9xx_calc_dpll_params(refclk, &clock);
-	}
-
-	/*
-	 * This value includes pixel_multiplier. We will use
-	 * port_clock to compute adjusted_mode.crtc_clock in the
-	 * encoder's get_config() function.
-	 */
-	pipe_config->port_clock = port_clock;
-}
-
 int intel_dotclock_calculate(int link_freq,
 			     const struct intel_link_m_n *m_n)
 {
 	/*
-	 * The calculation for the data clock is:
+	 * The calculation for the data clock -> pixel clock is:
 	 * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
 	 * But we want to avoid losing precison if possible, so:
 	 * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
 	 *
-	 * and the link clock is simpler:
-	 * link_clock = (m * link_clock) / n
+	 * and for link freq (10kbs units) -> pixel clock it is:
+	 * link_symbol_clock = link_freq * 10 / link_symbol_size
+	 * pixel_clock = (m * link_symbol_clock) / n
+	 *    or for more precision:
+	 * pixel_clock = (m * link_freq * 10) / (n * link_symbol_size)
 	 */
 
 	if (!m_n->link_n)
 		return 0;
 
-	return DIV_ROUND_UP_ULL(mul_u32_u32(m_n->link_m, link_freq),
-				m_n->link_n);
+	return DIV_ROUND_UP_ULL(mul_u32_u32(m_n->link_m, link_freq * 10),
+				m_n->link_n * intel_dp_link_symbol_size(link_freq));
 }
 
 int intel_crtc_dotclock(const struct intel_crtc_state *pipe_config)
@@ -4679,6 +4502,7 @@ intel_modeset_pipe_config(struct intel_atomic_state *state,
 	if (ret)
 		return ret;
 
+	crtc_state->fec_enable = limits->force_fec_pipes & BIT(crtc->pipe);
 	crtc_state->max_link_bpp_x16 = limits->max_bpp_x16[crtc->pipe];
 
 	if (crtc_state->pipe_bpp > to_bpp_int(crtc_state->max_link_bpp_x16)) {
@@ -5414,7 +5238,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 
 	PIPE_CONF_CHECK_I(dsc.compression_enable);
 	PIPE_CONF_CHECK_I(dsc.dsc_split);
-	PIPE_CONF_CHECK_I(dsc.compressed_bpp);
+	PIPE_CONF_CHECK_I(dsc.compressed_bpp_x16);
 
 	PIPE_CONF_CHECK_BOOL(splitter.enable);
 	PIPE_CONF_CHECK_I(splitter.link_count);
@@ -5523,6 +5347,16 @@ int intel_modeset_pipes_in_mask_early(struct intel_atomic_state *state,
 	return 0;
 }
 
+static void
+intel_crtc_flag_modeset(struct intel_crtc_state *crtc_state)
+{
+	crtc_state->uapi.mode_changed = true;
+
+	crtc_state->update_pipe = false;
+	crtc_state->update_m_n = false;
+	crtc_state->update_lrr = false;
+}
+
 /**
  * intel_modeset_all_pipes_late - force a full modeset on all pipes
  * @state: intel atomic state
@@ -5556,9 +5390,8 @@ int intel_modeset_all_pipes_late(struct intel_atomic_state *state,
 		if (ret)
 			return ret;
 
-		crtc_state->update_pipe = false;
-		crtc_state->update_m_n = false;
-		crtc_state->update_lrr = false;
+		intel_crtc_flag_modeset(crtc_state);
+
 		crtc_state->update_planes |= crtc_state->active_planes;
 		crtc_state->async_flip_planes = 0;
 		crtc_state->do_async_flip = false;
@@ -5671,17 +5504,17 @@ static void intel_crtc_check_fastset(const struct intel_crtc_state *old_crtc_sta
 	else
 		new_crtc_state->uapi.mode_changed = false;
 
-	if (intel_crtc_needs_modeset(new_crtc_state) ||
-	    intel_compare_link_m_n(&old_crtc_state->dp_m_n,
+	if (intel_compare_link_m_n(&old_crtc_state->dp_m_n,
 				   &new_crtc_state->dp_m_n))
 		new_crtc_state->update_m_n = false;
 
-	if (intel_crtc_needs_modeset(new_crtc_state) ||
-	    (old_crtc_state->hw.adjusted_mode.crtc_vtotal == new_crtc_state->hw.adjusted_mode.crtc_vtotal &&
+	if ((old_crtc_state->hw.adjusted_mode.crtc_vtotal == new_crtc_state->hw.adjusted_mode.crtc_vtotal &&
 	     old_crtc_state->hw.adjusted_mode.crtc_vblank_end == new_crtc_state->hw.adjusted_mode.crtc_vblank_end))
 		new_crtc_state->update_lrr = false;
 
-	if (!intel_crtc_needs_modeset(new_crtc_state))
+	if (intel_crtc_needs_modeset(new_crtc_state))
+		intel_crtc_flag_modeset(new_crtc_state);
+	else
 		new_crtc_state->update_pipe = true;
 }
 
@@ -6453,15 +6286,14 @@ int intel_atomic_check(struct drm_device *dev,
 		if (!new_crtc_state->hw.enable || intel_crtc_needs_modeset(new_crtc_state))
 			continue;
 
+		if (intel_dp_mst_crtc_needs_modeset(state, crtc))
+			intel_crtc_flag_modeset(new_crtc_state);
+
 		if (intel_dp_mst_is_slave_trans(new_crtc_state)) {
 			enum transcoder master = new_crtc_state->mst_master_transcoder;
 
-			if (intel_cpu_transcoders_need_modeset(state, BIT(master))) {
-				new_crtc_state->uapi.mode_changed = true;
-				new_crtc_state->update_pipe = false;
-				new_crtc_state->update_m_n = false;
-				new_crtc_state->update_lrr = false;
-			}
+			if (intel_cpu_transcoders_need_modeset(state, BIT(master)))
+				intel_crtc_flag_modeset(new_crtc_state);
 		}
 
 		if (is_trans_port_sync_mode(new_crtc_state)) {
@@ -6470,21 +6302,13 @@ int intel_atomic_check(struct drm_device *dev,
 			if (new_crtc_state->master_transcoder != INVALID_TRANSCODER)
 				trans |= BIT(new_crtc_state->master_transcoder);
 
-			if (intel_cpu_transcoders_need_modeset(state, trans)) {
-				new_crtc_state->uapi.mode_changed = true;
-				new_crtc_state->update_pipe = false;
-				new_crtc_state->update_m_n = false;
-				new_crtc_state->update_lrr = false;
-			}
+			if (intel_cpu_transcoders_need_modeset(state, trans))
+				intel_crtc_flag_modeset(new_crtc_state);
 		}
 
 		if (new_crtc_state->bigjoiner_pipes) {
-			if (intel_pipes_need_modeset(state, new_crtc_state->bigjoiner_pipes)) {
-				new_crtc_state->uapi.mode_changed = true;
-				new_crtc_state->update_pipe = false;
-				new_crtc_state->update_m_n = false;
-				new_crtc_state->update_lrr = false;
-			}
+			if (intel_pipes_need_modeset(state, new_crtc_state->bigjoiner_pipes))
+				intel_crtc_flag_modeset(new_crtc_state);
 		}
 	}
 
@@ -6505,10 +6329,6 @@ int intel_atomic_check(struct drm_device *dev,
 		goto fail;
 	}
 
-	ret = drm_dp_mst_atomic_check(&state->base);
-	if (ret)
-		goto fail;
-
 	ret = intel_atomic_check_planes(state);
 	if (ret)
 		goto fail;
@@ -6744,8 +6564,8 @@ static void intel_enable_crtc(struct intel_atomic_state *state,
 	intel_crtc_enable_pipe_crc(crtc);
 }
 
-static void intel_update_crtc(struct intel_atomic_state *state,
-			      struct intel_crtc *crtc)
+static void intel_pre_update_crtc(struct intel_atomic_state *state,
+				  struct intel_crtc *crtc)
 {
 	struct drm_i915_private *i915 = to_i915(state->base.dev);
 	const struct intel_crtc_state *old_crtc_state =
@@ -6787,6 +6607,15 @@ static void intel_update_crtc(struct intel_atomic_state *state,
 		intel_color_commit_noarm(new_crtc_state);
 
 	intel_crtc_planes_update_noarm(state, crtc);
+}
+
+static void intel_update_crtc(struct intel_atomic_state *state,
+			      struct intel_crtc *crtc)
+{
+	const struct intel_crtc_state *old_crtc_state =
+		intel_atomic_get_old_crtc_state(state, crtc);
+	struct intel_crtc_state *new_crtc_state =
+		intel_atomic_get_new_crtc_state(state, crtc);
 
 	/* Perform vblank evasion around commit operation */
 	intel_pipe_update_start(state, crtc);
@@ -6815,7 +6644,7 @@ static void intel_update_crtc(struct intel_atomic_state *state,
 	 * valid pipe configuration from the BIOS we need to take care
 	 * of enabling them on the CRTC's first fastset.
 	 */
-	if (intel_crtc_needs_fastset(new_crtc_state) && !modeset &&
+	if (intel_crtc_needs_fastset(new_crtc_state) &&
 	    old_crtc_state->inherited)
 		intel_crtc_arm_fifo_underrun(crtc, new_crtc_state);
 }
@@ -6910,6 +6739,13 @@ static void intel_commit_modeset_enables(struct intel_atomic_state *state)
 			continue;
 
 		intel_enable_crtc(state, crtc);
+		intel_pre_update_crtc(state, crtc);
+	}
+
+	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
+		if (!new_crtc_state->hw.active)
+			continue;
+
 		intel_update_crtc(state, crtc);
 	}
 }
@@ -6947,6 +6783,15 @@ static void skl_commit_modeset_enables(struct intel_atomic_state *state)
 	 * So first lets enable all pipes that do not need a fullmodeset as
 	 * those don't have any external dependency.
 	 */
+	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
+		enum pipe pipe = crtc->pipe;
+
+		if ((update_pipes & BIT(pipe)) == 0)
+			continue;
+
+		intel_pre_update_crtc(state, crtc);
+	}
+
 	while (update_pipes) {
 		for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
 						    new_crtc_state, i) {
@@ -7017,6 +6862,15 @@ static void skl_commit_modeset_enables(struct intel_atomic_state *state)
 	/*
 	 * Finally we do the plane updates/etc. for all pipes that got enabled.
 	 */
+	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
+		enum pipe pipe = crtc->pipe;
+
+		if ((update_pipes & BIT(pipe)) == 0)
+			continue;
+
+		intel_pre_update_crtc(state, crtc);
+	}
+
 	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
 		enum pipe pipe = crtc->pipe;
 
@@ -7056,29 +6910,22 @@ void intel_atomic_helper_free_state_worker(struct work_struct *work)
 
 static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_state)
 {
-	struct wait_queue_entry wait_fence, wait_reset;
-	struct drm_i915_private *dev_priv = to_i915(intel_state->base.dev);
-
-	init_wait_entry(&wait_fence, 0);
-	init_wait_entry(&wait_reset, 0);
-	for (;;) {
-		prepare_to_wait(&intel_state->commit_ready.wait,
-				&wait_fence, TASK_UNINTERRUPTIBLE);
-		prepare_to_wait(bit_waitqueue(&to_gt(dev_priv)->reset.flags,
-					      I915_RESET_MODESET),
-				&wait_reset, TASK_UNINTERRUPTIBLE);
-
+	struct drm_i915_private *i915 = to_i915(intel_state->base.dev);
+	struct drm_plane *plane;
+	struct drm_plane_state *new_plane_state;
+	int ret, i;
 
-		if (i915_sw_fence_done(&intel_state->commit_ready) ||
-		    test_bit(I915_RESET_MODESET, &to_gt(dev_priv)->reset.flags))
-			break;
+	for_each_new_plane_in_state(&intel_state->base, plane, new_plane_state, i) {
+		if (new_plane_state->fence) {
+			ret = dma_fence_wait_timeout(new_plane_state->fence, false,
+						     i915_fence_timeout(i915));
+			if (ret <= 0)
+				break;
 
-		schedule();
+			dma_fence_put(new_plane_state->fence);
+			new_plane_state->fence = NULL;
+		}
 	}
-	finish_wait(&intel_state->commit_ready.wait, &wait_fence);
-	finish_wait(bit_waitqueue(&to_gt(dev_priv)->reset.flags,
-				  I915_RESET_MODESET),
-		    &wait_reset);
 }
 
 static void intel_atomic_cleanup_work(struct work_struct *work)
@@ -7370,32 +7217,6 @@ static void intel_atomic_commit_work(struct work_struct *work)
 	intel_atomic_commit_tail(state);
 }
 
-static int
-intel_atomic_commit_ready(struct i915_sw_fence *fence,
-			  enum i915_sw_fence_notify notify)
-{
-	struct intel_atomic_state *state =
-		container_of(fence, struct intel_atomic_state, commit_ready);
-
-	switch (notify) {
-	case FENCE_COMPLETE:
-		/* we do blocking waits in the worker, nothing to do here */
-		break;
-	case FENCE_FREE:
-		{
-			struct drm_i915_private *i915 = to_i915(state->base.dev);
-			struct intel_atomic_helper *helper =
-				&i915->display.atomic_helper;
-
-			if (llist_add(&state->freed, &helper->free_list))
-				queue_work(i915->unordered_wq, &helper->free_work);
-			break;
-		}
-	}
-
-	return NOTIFY_DONE;
-}
-
 static void intel_atomic_track_fbs(struct intel_atomic_state *state)
 {
 	struct intel_plane_state *old_plane_state, *new_plane_state;
@@ -7418,10 +7239,6 @@ int intel_atomic_commit(struct drm_device *dev, struct drm_atomic_state *_state,
 
 	state->wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
 
-	drm_atomic_state_get(&state->base);
-	i915_sw_fence_init(&state->commit_ready,
-			   intel_atomic_commit_ready);
-
 	/*
 	 * The intel_legacy_cursor_update() fast path takes care
 	 * of avoiding the vblank waits for simple cursor
@@ -7454,7 +7271,6 @@ int intel_atomic_commit(struct drm_device *dev, struct drm_atomic_state *_state,
 	if (ret) {
 		drm_dbg_atomic(&dev_priv->drm,
 			       "Preparing state failed with %i\n", ret);
-		i915_sw_fence_commit(&state->commit_ready);
 		intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
 		return ret;
 	}
@@ -7470,8 +7286,6 @@ int intel_atomic_commit(struct drm_device *dev, struct drm_atomic_state *_state,
 		struct intel_crtc *crtc;
 		int i;
 
-		i915_sw_fence_commit(&state->commit_ready);
-
 		for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
 			intel_color_cleanup_commit(new_crtc_state);
 
@@ -7485,7 +7299,6 @@ int intel_atomic_commit(struct drm_device *dev, struct drm_atomic_state *_state,
 	drm_atomic_state_get(&state->base);
 	INIT_WORK(&state->base.commit_work, intel_atomic_commit_work);
 
-	i915_sw_fence_commit(&state->commit_ready);
 	if (nonblock && state->modeset) {
 		queue_work(dev_priv->display.wq.modeset, &state->base.commit_work);
 	} else if (nonblock) {
diff --git a/drivers/gpu/drm/i915/display/intel_display.h b/drivers/gpu/drm/i915/display/intel_display.h
index 0e5dffe8f018..8548f49e3972 100644
--- a/drivers/gpu/drm/i915/display/intel_display.h
+++ b/drivers/gpu/drm/i915/display/intel_display.h
@@ -105,7 +105,6 @@ enum i9xx_plane_id {
 };
 
 #define plane_name(p) ((p) + 'A')
-#define sprite_name(p, s) ((p) * DISPLAY_RUNTIME_INFO(dev_priv)->num_sprites[(p)] + (s) + 'A')
 
 #define for_each_plane_id_on_crtc(__crtc, __p) \
 	for ((__p) = PLANE_PRIMARY; (__p) < I915_MAX_PLANES; (__p)++) \
@@ -395,8 +394,8 @@ u8 intel_calc_active_pipes(struct intel_atomic_state *state,
 			   u8 active_pipes);
 void intel_link_compute_m_n(u16 bpp, int nlanes,
 			    int pixel_clock, int link_clock,
-			    struct intel_link_m_n *m_n,
-			    bool fec_enable);
+			    int bw_overhead,
+			    struct intel_link_m_n *m_n);
 u32 intel_plane_fb_max_stride(struct drm_i915_private *dev_priv,
 			      u32 pixel_format, u64 modifier);
 enum drm_mode_status
@@ -482,8 +481,6 @@ void intel_cpu_transcoder_get_m1_n1(struct intel_crtc *crtc,
 void intel_cpu_transcoder_get_m2_n2(struct intel_crtc *crtc,
 				    enum transcoder cpu_transcoder,
 				    struct intel_link_m_n *m_n);
-void i9xx_crtc_clock_get(struct intel_crtc *crtc,
-			 struct intel_crtc_state *pipe_config);
 int intel_dotclock_calculate(int link_freq, const struct intel_link_m_n *m_n);
 int intel_crtc_dotclock(const struct intel_crtc_state *pipe_config);
 enum intel_display_power_domain intel_port_to_power_domain(struct intel_digital_port *dig_port);
@@ -552,7 +549,7 @@ bool assert_port_valid(struct drm_i915_private *i915, enum port port);
 	struct drm_device *drm = &(__i915)->drm;			\
 	int __ret_warn_on = !!(condition);				\
 	if (unlikely(__ret_warn_on))					\
-		if (!drm_WARN(drm, i915_modparams.verbose_state_checks, format)) \
+		if (!drm_WARN(drm, __i915->display.params.verbose_state_checks, format)) \
 			drm_err(drm, format);				\
 	unlikely(__ret_warn_on);					\
 })
diff --git a/drivers/gpu/drm/i915/display/intel_display_core.h b/drivers/gpu/drm/i915/display/intel_display_core.h
index ccfe27630fb6..7e82b87e9cde 100644
--- a/drivers/gpu/drm/i915/display/intel_display_core.h
+++ b/drivers/gpu/drm/i915/display/intel_display_core.h
@@ -19,6 +19,7 @@
 #include "intel_cdclk.h"
 #include "intel_display_device.h"
 #include "intel_display_limits.h"
+#include "intel_display_params.h"
 #include "intel_display_power.h"
 #include "intel_dpll_mgr.h"
 #include "intel_fbc.h"
@@ -347,15 +348,6 @@ struct intel_display {
 		struct intel_global_obj obj;
 	} dbuf;
 
-	struct {
-		wait_queue_head_t waitqueue;
-
-		/* mutex to protect pmdemand programming sequence */
-		struct mutex lock;
-
-		struct intel_global_obj obj;
-	} pmdemand;
-
 	struct {
 		/*
 		 * dkl.phy_lock protects against concurrent access of the
@@ -443,6 +435,15 @@ struct intel_display {
 		bool false_color;
 	} ips;
 
+	struct {
+		wait_queue_head_t waitqueue;
+
+		/* mutex to protect pmdemand programming sequence */
+		struct mutex lock;
+
+		struct intel_global_obj obj;
+	} pmdemand;
+
 	struct {
 		struct i915_power_domains domains;
 
@@ -520,6 +521,7 @@ struct intel_display {
 	struct intel_hotplug hotplug;
 	struct intel_opregion opregion;
 	struct intel_overlay *overlay;
+	struct intel_display_params params;
 	struct intel_vbt_data vbt;
 	struct intel_wm wm;
 };
diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs.c b/drivers/gpu/drm/i915/display/intel_display_debugfs.c
index 2836826f8c05..915420d0cef8 100644
--- a/drivers/gpu/drm/i915/display/intel_display_debugfs.c
+++ b/drivers/gpu/drm/i915/display/intel_display_debugfs.c
@@ -17,6 +17,7 @@
 #include "intel_de.h"
 #include "intel_crtc_state_dump.h"
 #include "intel_display_debugfs.h"
+#include "intel_display_debugfs_params.h"
 #include "intel_display_power.h"
 #include "intel_display_power_well.h"
 #include "intel_display_types.h"
@@ -641,6 +642,17 @@ static int i915_display_info(struct seq_file *m, void *unused)
 	return 0;
 }
 
+static int i915_display_capabilities(struct seq_file *m, void *unused)
+{
+	struct drm_i915_private *i915 = node_to_i915(m->private);
+	struct drm_printer p = drm_seq_file_printer(m);
+
+	intel_display_device_info_print(DISPLAY_INFO(i915),
+					DISPLAY_RUNTIME_INFO(i915), &p);
+
+	return 0;
+}
+
 static int i915_shared_dplls_info(struct seq_file *m, void *unused)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
@@ -1059,6 +1071,7 @@ static const struct drm_info_list intel_display_debugfs_list[] = {
 	{"i915_gem_framebuffer", i915_gem_framebuffer_info, 0},
 	{"i915_power_domain_info", i915_power_domain_info, 0},
 	{"i915_display_info", i915_display_info, 0},
+	{"i915_display_capabilities", i915_display_capabilities, 0},
 	{"i915_shared_dplls_info", i915_shared_dplls_info, 0},
 	{"i915_dp_mst_info", i915_dp_mst_info, 0},
 	{"i915_ddb_info", i915_ddb_info, 0},
@@ -1098,6 +1111,7 @@ void intel_display_debugfs_register(struct drm_i915_private *i915)
 	intel_hpd_debugfs_register(i915);
 	intel_psr_debugfs_register(i915);
 	intel_wm_debugfs_register(i915);
+	intel_display_debugfs_params(i915);
 }
 
 static int i915_panel_show(struct seq_file *m, void *data)
@@ -1242,6 +1256,8 @@ static int i915_dsc_fec_support_show(struct seq_file *m, void *data)
 								      DP_DSC_YCbCr420_Native)),
 			   str_yes_no(drm_dp_dsc_sink_supports_format(connector->dp.dsc_dpcd,
 								      DP_DSC_YCbCr444)));
+		seq_printf(m, "DSC_Sink_BPP_Precision: %d\n",
+			   drm_dp_dsc_sink_bpp_incr(connector->dp.dsc_dpcd));
 		seq_printf(m, "Force_DSC_Enable: %s\n",
 			   str_yes_no(intel_dp->force_dsc_en));
 		if (!intel_dp_is_edp(intel_dp))
@@ -1434,6 +1450,85 @@ static const struct file_operations i915_dsc_output_format_fops = {
 	.write = i915_dsc_output_format_write
 };
 
+static int i915_dsc_fractional_bpp_show(struct seq_file *m, void *data)
+{
+	struct drm_connector *connector = m->private;
+	struct drm_device *dev = connector->dev;
+	struct drm_crtc *crtc;
+	struct intel_dp *intel_dp;
+	struct intel_connector *intel_connector = to_intel_connector(connector);
+	struct intel_encoder *encoder = intel_attached_encoder(intel_connector);
+	int ret;
+
+	if (!encoder)
+		return -ENODEV;
+
+	ret = drm_modeset_lock_single_interruptible(&dev->mode_config.connection_mutex);
+	if (ret)
+		return ret;
+
+	crtc = connector->state->crtc;
+	if (connector->status != connector_status_connected || !crtc) {
+		ret = -ENODEV;
+		goto out;
+	}
+
+	intel_dp = intel_attached_dp(intel_connector);
+	seq_printf(m, "Force_DSC_Fractional_BPP_Enable: %s\n",
+		   str_yes_no(intel_dp->force_dsc_fractional_bpp_en));
+
+out:
+	drm_modeset_unlock(&dev->mode_config.connection_mutex);
+
+	return ret;
+}
+
+static ssize_t i915_dsc_fractional_bpp_write(struct file *file,
+					     const char __user *ubuf,
+					     size_t len, loff_t *offp)
+{
+	struct drm_connector *connector =
+		((struct seq_file *)file->private_data)->private;
+	struct intel_encoder *encoder = intel_attached_encoder(to_intel_connector(connector));
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
+	bool dsc_fractional_bpp_enable = false;
+	int ret;
+
+	if (len == 0)
+		return 0;
+
+	drm_dbg(&i915->drm,
+		"Copied %zu bytes from user to force fractional bpp for DSC\n", len);
+
+	ret = kstrtobool_from_user(ubuf, len, &dsc_fractional_bpp_enable);
+	if (ret < 0)
+		return ret;
+
+	drm_dbg(&i915->drm, "Got %s for DSC Fractional BPP Enable\n",
+		(dsc_fractional_bpp_enable) ? "true" : "false");
+	intel_dp->force_dsc_fractional_bpp_en = dsc_fractional_bpp_enable;
+
+	*offp += len;
+
+	return len;
+}
+
+static int i915_dsc_fractional_bpp_open(struct inode *inode,
+					struct file *file)
+{
+	return single_open(file, i915_dsc_fractional_bpp_show, inode->i_private);
+}
+
+static const struct file_operations i915_dsc_fractional_bpp_fops = {
+	.owner = THIS_MODULE,
+	.open = i915_dsc_fractional_bpp_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+	.write = i915_dsc_fractional_bpp_write
+};
+
 /*
  * Returns the Current CRTC's bpc.
  * Example usage: cat /sys/kernel/debug/dri/0/crtc-0/i915_current_bpc
@@ -1511,6 +1606,9 @@ void intel_connector_debugfs_add(struct intel_connector *intel_connector)
 
 		debugfs_create_file("i915_dsc_output_format", 0644, root,
 				    connector, &i915_dsc_output_format_fops);
+
+		debugfs_create_file("i915_dsc_fractional_bpp", 0644, root,
+				    connector, &i915_dsc_fractional_bpp_fops);
 	}
 
 	if (connector->connector_type == DRM_MODE_CONNECTOR_DSI ||
diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs_params.c b/drivers/gpu/drm/i915/display/intel_display_debugfs_params.c
new file mode 100644
index 000000000000..b7e68eb62452
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_display_debugfs_params.c
@@ -0,0 +1,176 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#include <linux/kernel.h>
+
+#include <drm/drm_drv.h>
+
+#include "intel_display_debugfs_params.h"
+#include "i915_drv.h"
+#include "intel_display_params.h"
+
+/* int param */
+static int intel_display_param_int_show(struct seq_file *m, void *data)
+{
+	int *value = m->private;
+
+	seq_printf(m, "%d\n", *value);
+
+	return 0;
+}
+
+static int intel_display_param_int_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, intel_display_param_int_show, inode->i_private);
+}
+
+static ssize_t intel_display_param_int_write(struct file *file,
+					     const char __user *ubuf, size_t len,
+					     loff_t *offp)
+{
+	struct seq_file *m = file->private_data;
+	int *value = m->private;
+	int ret;
+
+	ret = kstrtoint_from_user(ubuf, len, 0, value);
+	if (ret) {
+		/* support boolean values too */
+		bool b;
+
+		ret = kstrtobool_from_user(ubuf, len, &b);
+		if (!ret)
+			*value = b;
+	}
+
+	return ret ?: len;
+}
+
+static const struct file_operations intel_display_param_int_fops = {
+	.owner = THIS_MODULE,
+	.open = intel_display_param_int_open,
+	.read = seq_read,
+	.write = intel_display_param_int_write,
+	.llseek = default_llseek,
+	.release = single_release,
+};
+
+static const struct file_operations intel_display_param_int_fops_ro = {
+	.owner = THIS_MODULE,
+	.open = intel_display_param_int_open,
+	.read = seq_read,
+	.llseek = default_llseek,
+	.release = single_release,
+};
+
+/* unsigned int param */
+static int intel_display_param_uint_show(struct seq_file *m, void *data)
+{
+	unsigned int *value = m->private;
+
+	seq_printf(m, "%u\n", *value);
+
+	return 0;
+}
+
+static int intel_display_param_uint_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, intel_display_param_uint_show, inode->i_private);
+}
+
+static ssize_t intel_display_param_uint_write(struct file *file,
+					      const char __user *ubuf, size_t len,
+					      loff_t *offp)
+{
+	struct seq_file *m = file->private_data;
+	unsigned int *value = m->private;
+	int ret;
+
+	ret = kstrtouint_from_user(ubuf, len, 0, value);
+	if (ret) {
+		/* support boolean values too */
+		bool b;
+
+		ret = kstrtobool_from_user(ubuf, len, &b);
+		if (!ret)
+			*value = b;
+	}
+
+	return ret ?: len;
+}
+
+static const struct file_operations intel_display_param_uint_fops = {
+	.owner = THIS_MODULE,
+	.open = intel_display_param_uint_open,
+	.read = seq_read,
+	.write = intel_display_param_uint_write,
+	.llseek = default_llseek,
+	.release = single_release,
+};
+
+static const struct file_operations intel_display_param_uint_fops_ro = {
+	.owner = THIS_MODULE,
+	.open = intel_display_param_uint_open,
+	.read = seq_read,
+	.llseek = default_llseek,
+	.release = single_release,
+};
+
+#define RO(mode) (((mode) & 0222) == 0)
+
+__maybe_unused static struct dentry *
+intel_display_debugfs_create_int(const char *name, umode_t mode,
+			struct dentry *parent, int *value)
+{
+	return debugfs_create_file_unsafe(name, mode, parent, value,
+					  RO(mode) ? &intel_display_param_int_fops_ro :
+					  &intel_display_param_int_fops);
+}
+
+__maybe_unused static struct dentry *
+intel_display_debugfs_create_uint(const char *name, umode_t mode,
+			 struct dentry *parent, unsigned int *value)
+{
+	return debugfs_create_file_unsafe(name, mode, parent, value,
+					  RO(mode) ? &intel_display_param_uint_fops_ro :
+					  &intel_display_param_uint_fops);
+}
+
+#define _intel_display_param_create_file(parent, name, mode, valp)	\
+	do {								\
+		if (mode)						\
+			_Generic(valp,					\
+				 bool * : debugfs_create_bool,		\
+				 int * : intel_display_debugfs_create_int, \
+				 unsigned int * : intel_display_debugfs_create_uint, \
+				 unsigned long * : debugfs_create_ulong, \
+				 char ** : debugfs_create_str) \
+				(name, mode, parent, valp);		\
+	} while (0)
+
+/* add a subdirectory with files for each intel display param */
+void intel_display_debugfs_params(struct drm_i915_private *i915)
+{
+	struct drm_minor *minor = i915->drm.primary;
+	struct dentry *dir;
+	char dirname[16];
+
+	snprintf(dirname, sizeof(dirname), "%s_params", i915->drm.driver->name);
+	dir = debugfs_lookup(dirname, minor->debugfs_root);
+	if (!dir)
+		dir = debugfs_create_dir(dirname, minor->debugfs_root);
+	if (IS_ERR(dir))
+		return;
+
+	/*
+	 * Note: We could create files for params needing special handling
+	 * here. Set mode in params to 0 to skip the generic create file, or
+	 * just let the generic create file fail silently with -EEXIST.
+	 */
+
+#define REGISTER(T, x, unused, mode, ...) _intel_display_param_create_file( \
+		dir, #x, mode, &i915->display.params.x);
+	INTEL_DISPLAY_PARAMS_FOR_EACH(REGISTER);
+#undef REGISTER
+}
diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs_params.h b/drivers/gpu/drm/i915/display/intel_display_debugfs_params.h
new file mode 100644
index 000000000000..1e9945a4044c
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_display_debugfs_params.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#ifndef __INTEL_DISPLAY_DEBUGFS_PARAMS__
+#define __INTEL_DISPLAY_DEBUGFS_PARAMS__
+
+struct drm_i915_private;
+
+void intel_display_debugfs_params(struct drm_i915_private *i915);
+
+#endif /* __INTEL_DISPLAY_DEBUGFS_PARAMS__ */
diff --git a/drivers/gpu/drm/i915/display/intel_display_device.c b/drivers/gpu/drm/i915/display/intel_display_device.c
index 2b1ec23ba9c3..0b522c6a8d6f 100644
--- a/drivers/gpu/drm/i915/display/intel_display_device.c
+++ b/drivers/gpu/drm/i915/display/intel_display_device.c
@@ -12,6 +12,7 @@
 #include "intel_de.h"
 #include "intel_display.h"
 #include "intel_display_device.h"
+#include "intel_display_params.h"
 #include "intel_display_power.h"
 #include "intel_display_reg_defs.h"
 #include "intel_fbc.h"
@@ -937,6 +938,13 @@ void intel_display_device_probe(struct drm_i915_private *i915)
 		DISPLAY_RUNTIME_INFO(i915)->ip.rel = rel;
 		DISPLAY_RUNTIME_INFO(i915)->ip.step = step;
 	}
+
+	intel_display_params_copy(&i915->display.params);
+}
+
+void intel_display_device_remove(struct drm_i915_private *i915)
+{
+	intel_display_params_free(&i915->display.params);
 }
 
 static void __intel_display_device_info_runtime_init(struct drm_i915_private *i915)
@@ -1105,7 +1113,7 @@ void intel_display_device_info_runtime_init(struct drm_i915_private *i915)
 	}
 
 	/* Disable nuclear pageflip by default on pre-g4x */
-	if (!i915->params.nuclear_pageflip &&
+	if (!i915->display.params.nuclear_pageflip &&
 	    DISPLAY_VER(i915) < 5 && !IS_G4X(i915))
 		i915->drm.driver_features &= ~DRIVER_ATOMIC;
 }
@@ -1145,5 +1153,6 @@ bool intel_display_device_enabled(struct drm_i915_private *i915)
 	/* Only valid when HAS_DISPLAY() is true */
 	drm_WARN_ON(&i915->drm, !HAS_DISPLAY(i915));
 
-	return !i915->params.disable_display && !intel_opregion_headless_sku(i915);
+	return !i915->display.params.disable_display &&
+		!intel_opregion_headless_sku(i915);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_display_device.h b/drivers/gpu/drm/i915/display/intel_display_device.h
index 5b5c0e53307f..4299cc452e05 100644
--- a/drivers/gpu/drm/i915/display/intel_display_device.h
+++ b/drivers/gpu/drm/i915/display/intel_display_device.h
@@ -161,6 +161,7 @@ struct intel_display_device_info {
 
 bool intel_display_device_enabled(struct drm_i915_private *i915);
 void intel_display_device_probe(struct drm_i915_private *i915);
+void intel_display_device_remove(struct drm_i915_private *i915);
 void intel_display_device_info_runtime_init(struct drm_i915_private *i915);
 
 void intel_display_device_info_print(const struct intel_display_device_info *info,
diff --git a/drivers/gpu/drm/i915/display/intel_display_driver.c b/drivers/gpu/drm/i915/display/intel_display_driver.c
index 44b59ac301e6..62f7b10484be 100644
--- a/drivers/gpu/drm/i915/display/intel_display_driver.c
+++ b/drivers/gpu/drm/i915/display/intel_display_driver.c
@@ -181,6 +181,13 @@ void intel_display_driver_early_probe(struct drm_i915_private *i915)
 	if (!HAS_DISPLAY(i915))
 		return;
 
+	spin_lock_init(&i915->display.fb_tracking.lock);
+	mutex_init(&i915->display.backlight.lock);
+	mutex_init(&i915->display.audio.mutex);
+	mutex_init(&i915->display.wm.wm_mutex);
+	mutex_init(&i915->display.pps.mutex);
+	mutex_init(&i915->display.hdcp.hdcp_mutex);
+
 	intel_display_irq_init(i915);
 	intel_dkl_phy_init(i915);
 	intel_color_init_hooks(i915);
diff --git a/drivers/gpu/drm/i915/display/intel_display_params.c b/drivers/gpu/drm/i915/display/intel_display_params.c
new file mode 100644
index 000000000000..11e03cfb774d
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_display_params.c
@@ -0,0 +1,217 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#include "intel_display_params.h"
+#include "i915_drv.h"
+
+#define intel_display_param_named(name, T, perm, desc) \
+	module_param_named(name, intel_display_modparams.name, T, perm); \
+	MODULE_PARM_DESC(name, desc)
+#define intel_display_param_named_unsafe(name, T, perm, desc) \
+	module_param_named_unsafe(name, intel_display_modparams.name, T, perm); \
+	MODULE_PARM_DESC(name, desc)
+
+static struct intel_display_params intel_display_modparams __read_mostly = {
+#define MEMBER(T, member, value, ...) .member = (value),
+	INTEL_DISPLAY_PARAMS_FOR_EACH(MEMBER)
+#undef MEMBER
+};
+/*
+ * Note: As a rule, keep module parameter sysfs permissions read-only
+ * 0400. Runtime changes are only supported through i915 debugfs.
+ *
+ * For any exceptions requiring write access and runtime changes through module
+ * parameter sysfs, prevent debugfs file creation by setting the parameter's
+ * debugfs mode to 0.
+ */
+
+intel_display_param_named_unsafe(vbt_firmware, charp, 0400,
+	"Load VBT from specified file under /lib/firmware");
+
+intel_display_param_named_unsafe(lvds_channel_mode, int, 0400,
+	 "Specify LVDS channel mode "
+	 "(0=probe BIOS [default], 1=single-channel, 2=dual-channel)");
+
+intel_display_param_named_unsafe(panel_use_ssc, int, 0400,
+	"Use Spread Spectrum Clock with panels [LVDS/eDP] "
+	"(default: auto from VBT)");
+
+intel_display_param_named_unsafe(vbt_sdvo_panel_type, int, 0400,
+	"Override/Ignore selection of SDVO panel mode in the VBT "
+	"(-2=ignore, -1=auto [default], index in VBT BIOS table)");
+
+intel_display_param_named_unsafe(enable_dc, int, 0400,
+	"Enable power-saving display C-states. "
+	"(-1=auto [default]; 0=disable; 1=up to DC5; 2=up to DC6; "
+	"3=up to DC5 with DC3CO; 4=up to DC6 with DC3CO)");
+
+intel_display_param_named_unsafe(enable_dpt, bool, 0400,
+	"Enable display page table (DPT) (default: true)");
+
+intel_display_param_named_unsafe(enable_sagv, bool, 0400,
+	"Enable system agent voltage/frequency scaling (SAGV) (default: true)");
+
+intel_display_param_named_unsafe(disable_power_well, int, 0400,
+	"Disable display power wells when possible "
+	"(-1=auto [default], 0=power wells always on, 1=power wells disabled when possible)");
+
+intel_display_param_named_unsafe(enable_ips, bool, 0400, "Enable IPS (default: true)");
+
+intel_display_param_named_unsafe(invert_brightness, int, 0400,
+	"Invert backlight brightness "
+	"(-1 force normal, 0 machine defaults, 1 force inversion), please "
+	"report PCI device ID, subsystem vendor and subsystem device ID "
+	"to dri-devel@lists.freedesktop.org, if your machine needs it. "
+	"It will then be included in an upcoming module version.");
+
+/* WA to get away with the default setting in VBT for early platforms.Will be removed */
+intel_display_param_named_unsafe(edp_vswing, int, 0400,
+	"Ignore/Override vswing pre-emph table selection from VBT "
+	"(0=use value from vbt [default], 1=low power swing(200mV),"
+	"2=default swing(400mV))");
+
+intel_display_param_named(enable_dpcd_backlight, int, 0400,
+	"Enable support for DPCD backlight control"
+	"(-1=use per-VBT LFP backlight type setting [default], 0=disabled, 1=enable, 2=force VESA interface, 3=force Intel interface)");
+
+intel_display_param_named_unsafe(load_detect_test, bool, 0400,
+	"Force-enable the VGA load detect code for testing (default:false). "
+	"For developers only.");
+
+intel_display_param_named_unsafe(force_reset_modeset_test, bool, 0400,
+	"Force a modeset during gpu reset for testing (default:false). "
+	"For developers only.");
+
+intel_display_param_named(disable_display, bool, 0400,
+	"Disable display (default: false)");
+
+intel_display_param_named(verbose_state_checks, bool, 0400,
+	"Enable verbose logs (ie. WARN_ON()) in case of unexpected hw state conditions.");
+
+intel_display_param_named_unsafe(nuclear_pageflip, bool, 0400,
+	"Force enable atomic functionality on platforms that don't have full support yet.");
+
+intel_display_param_named_unsafe(enable_dp_mst, bool, 0400,
+	"Enable multi-stream transport (MST) for new DisplayPort sinks. (default: true)");
+
+intel_display_param_named_unsafe(enable_fbc, int, 0400,
+	"Enable frame buffer compression for power savings "
+	"(default: -1 (use per-chip default))");
+
+intel_display_param_named_unsafe(enable_psr, int, 0400,
+	"Enable PSR "
+	"(0=disabled, 1=enable up to PSR1, 2=enable up to PSR2) "
+	"Default: -1 (use per-chip default)");
+
+intel_display_param_named(psr_safest_params, bool, 0400,
+	"Replace PSR VBT parameters by the safest and not optimal ones. This "
+	"is helpful to detect if PSR issues are related to bad values set in "
+	" VBT. (0=use VBT parameters, 1=use safest parameters)"
+	"Default: 0");
+
+intel_display_param_named_unsafe(enable_psr2_sel_fetch, bool, 0400,
+	"Enable PSR2 selective fetch "
+	"(0=disabled, 1=enabled) "
+	"Default: 1");
+
+__maybe_unused
+static void _param_print_bool(struct drm_printer *p, const char *driver_name,
+			      const char *name, bool val)
+{
+	drm_printf(p, "%s.%s=%s\n", driver_name, name, str_yes_no(val));
+}
+
+__maybe_unused
+static void _param_print_int(struct drm_printer *p, const char *driver_name,
+			     const char *name, int val)
+{
+	drm_printf(p, "%s.%s=%d\n", driver_name, name, val);
+}
+
+__maybe_unused
+static void _param_print_uint(struct drm_printer *p, const char *driver_name,
+			      const char *name, unsigned int val)
+{
+	drm_printf(p, "%s.%s=%u\n", driver_name, name, val);
+}
+
+__maybe_unused
+static void _param_print_ulong(struct drm_printer *p, const char *driver_name,
+			       const char *name, unsigned long val)
+{
+	drm_printf(p, "%s.%s=%lu\n", driver_name, name, val);
+}
+
+__maybe_unused
+static void _param_print_charp(struct drm_printer *p, const char *driver_name,
+			       const char *name, const char *val)
+{
+	drm_printf(p, "%s.%s=%s\n", driver_name, name, val);
+}
+
+#define _param_print(p, driver_name, name, val)			\
+	_Generic(val,						\
+		 bool : _param_print_bool,			\
+		 int : _param_print_int,			\
+		 unsigned int : _param_print_uint,		\
+		 unsigned long : _param_print_ulong,		\
+		 char * : _param_print_charp)(p, driver_name, name, val)
+
+/**
+ * intel_display_params_dump - dump intel display modparams
+ * @i915: i915 device
+ * @p: the &drm_printer
+ *
+ * Pretty printer for i915 modparams.
+ */
+void intel_display_params_dump(struct drm_i915_private *i915, struct drm_printer *p)
+{
+#define PRINT(T, x, ...) _param_print(p, i915->drm.driver->name, #x, i915->display.params.x);
+	INTEL_DISPLAY_PARAMS_FOR_EACH(PRINT);
+#undef PRINT
+}
+
+__maybe_unused static void _param_dup_charp(char **valp)
+{
+	*valp = kstrdup(*valp ? *valp : "", GFP_ATOMIC);
+}
+
+__maybe_unused static void _param_nop(void *valp)
+{
+}
+
+#define _param_dup(valp)				\
+	_Generic(valp,					\
+		 char ** : _param_dup_charp,		\
+		 default : _param_nop)			\
+		(valp)
+
+void intel_display_params_copy(struct intel_display_params *dest)
+{
+	*dest = intel_display_modparams;
+#define DUP(T, x, ...) _param_dup(&dest->x);
+	INTEL_DISPLAY_PARAMS_FOR_EACH(DUP);
+#undef DUP
+}
+
+__maybe_unused static void _param_free_charp(char **valp)
+{
+	kfree(*valp);
+	*valp = NULL;
+}
+
+#define _param_free(valp)				\
+	_Generic(valp,					\
+		 char ** : _param_free_charp,		\
+		 default : _param_nop)			\
+		(valp)
+
+/* free the allocated members, *not* the passed in params itself */
+void intel_display_params_free(struct intel_display_params *params)
+{
+#define FREE(T, x, ...) _param_free(&params->x);
+	INTEL_DISPLAY_PARAMS_FOR_EACH(FREE);
+#undef FREE
+}
diff --git a/drivers/gpu/drm/i915/display/intel_display_params.h b/drivers/gpu/drm/i915/display/intel_display_params.h
new file mode 100644
index 000000000000..6206cc51df04
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_display_params.h
@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#ifndef _INTEL_DISPLAY_PARAMS_H_
+#define _INTEL_DISPLAY_PARAMS_H_
+
+#include <linux/types.h>
+
+struct drm_printer;
+struct drm_i915_private;
+
+/*
+ * Invoke param, a function-like macro, for each intel display param, with
+ * arguments:
+ *
+ * param(type, name, value, mode)
+ *
+ * type: parameter type, one of {bool, int, unsigned int, unsigned long, char *}
+ * name: name of the parameter
+ * value: initial/default value of the parameter
+ * mode: debugfs file permissions, one of {0400, 0600, 0}, use 0 to not create
+ *       debugfs file
+ */
+#define INTEL_DISPLAY_PARAMS_FOR_EACH(param) \
+	param(char *, vbt_firmware, NULL, 0400) \
+	param(int, lvds_channel_mode, 0, 0400) \
+	param(int, panel_use_ssc, -1, 0600) \
+	param(int, vbt_sdvo_panel_type, -1, 0400) \
+	param(int, enable_dc, -1, 0400) \
+	param(bool, enable_dpt, true, 0400) \
+	param(bool, enable_sagv, true, 0600) \
+	param(int, disable_power_well, -1, 0400) \
+	param(bool, enable_ips, true, 0600) \
+	param(int, invert_brightness, 0, 0600) \
+	param(int, edp_vswing, 0, 0400) \
+	param(int, enable_dpcd_backlight, -1, 0600) \
+	param(bool, load_detect_test, false, 0600) \
+	param(bool, force_reset_modeset_test, false, 0600) \
+	param(bool, disable_display, false, 0400) \
+	param(bool, verbose_state_checks, true, 0400) \
+	param(bool, nuclear_pageflip, false, 0400) \
+	param(bool, enable_dp_mst, true, 0600) \
+	param(int, enable_fbc, -1, 0600) \
+	param(int, enable_psr, -1, 0600) \
+	param(bool, psr_safest_params, false, 0400) \
+	param(bool, enable_psr2_sel_fetch, true, 0400) \
+
+#define MEMBER(T, member, ...) T member;
+struct intel_display_params {
+	INTEL_DISPLAY_PARAMS_FOR_EACH(MEMBER);
+};
+#undef MEMBER
+
+void intel_display_params_dump(struct drm_i915_private *i915,
+			       struct drm_printer *p);
+void intel_display_params_copy(struct intel_display_params *dest);
+void intel_display_params_free(struct intel_display_params *params);
+
+#endif
diff --git a/drivers/gpu/drm/i915/display/intel_display_power.c b/drivers/gpu/drm/i915/display/intel_display_power.c
index e25785ae1c20..e390595d7341 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power.c
@@ -967,7 +967,7 @@ static u32 get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
 		DISPLAY_VER(dev_priv) >= 11 ?
 	       DC_STATE_EN_DC9 : 0;
 
-	if (!dev_priv->params.disable_power_well)
+	if (!dev_priv->display.params.disable_power_well)
 		max_dc = 0;
 
 	if (enable_dc >= 0 && enable_dc <= max_dc) {
@@ -1016,11 +1016,11 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv)
 {
 	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 
-	dev_priv->params.disable_power_well =
+	dev_priv->display.params.disable_power_well =
 		sanitize_disable_power_well_option(dev_priv,
-						   dev_priv->params.disable_power_well);
+						   dev_priv->display.params.disable_power_well);
 	power_domains->allowed_dc_mask =
-		get_allowed_dc_mask(dev_priv, dev_priv->params.enable_dc);
+		get_allowed_dc_mask(dev_priv, dev_priv->display.params.enable_dc);
 
 	power_domains->target_dc_state =
 		sanitize_target_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
@@ -1950,7 +1950,7 @@ void intel_power_domains_init_hw(struct drm_i915_private *i915, bool resume)
 		intel_display_power_get(i915, POWER_DOMAIN_INIT);
 
 	/* Disable power support if the user asked so. */
-	if (!i915->params.disable_power_well) {
+	if (!i915->display.params.disable_power_well) {
 		drm_WARN_ON(&i915->drm, power_domains->disable_wakeref);
 		i915->display.power.domains.disable_wakeref = intel_display_power_get(i915,
 										      POWER_DOMAIN_INIT);
@@ -1977,7 +1977,7 @@ void intel_power_domains_driver_remove(struct drm_i915_private *i915)
 		fetch_and_zero(&i915->display.power.domains.init_wakeref);
 
 	/* Remove the refcount we took to keep power well support disabled. */
-	if (!i915->params.disable_power_well)
+	if (!i915->display.params.disable_power_well)
 		intel_display_power_put(i915, POWER_DOMAIN_INIT,
 					fetch_and_zero(&i915->display.power.domains.disable_wakeref));
 
@@ -2096,7 +2096,7 @@ void intel_power_domains_suspend(struct drm_i915_private *i915, bool s2idle)
 	 * Even if power well support was disabled we still want to disable
 	 * power wells if power domains must be deinitialized for suspend.
 	 */
-	if (!i915->params.disable_power_well)
+	if (!i915->display.params.disable_power_well)
 		intel_display_power_put(i915, POWER_DOMAIN_INIT,
 					fetch_and_zero(&i915->display.power.domains.disable_wakeref));
 
diff --git a/drivers/gpu/drm/i915/display/intel_display_power_well.c b/drivers/gpu/drm/i915/display/intel_display_power_well.c
index 07d650050099..47cd6bb04366 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power_well.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power_well.c
@@ -1400,20 +1400,16 @@ static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 {
 	enum i915_power_well_id id = i915_power_well_instance(power_well)->id;
 	enum dpio_phy phy;
-	enum pipe pipe;
 	u32 tmp;
 
 	drm_WARN_ON_ONCE(&dev_priv->drm,
 			 id != VLV_DISP_PW_DPIO_CMN_BC &&
 			 id != CHV_DISP_PW_DPIO_CMN_D);
 
-	if (id == VLV_DISP_PW_DPIO_CMN_BC) {
-		pipe = PIPE_A;
+	if (id == VLV_DISP_PW_DPIO_CMN_BC)
 		phy = DPIO_PHY0;
-	} else {
-		pipe = PIPE_C;
+	else
 		phy = DPIO_PHY1;
-	}
 
 	/* since ref/cri clock was enabled */
 	udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
@@ -1428,24 +1424,24 @@ static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 	vlv_dpio_get(dev_priv);
 
 	/* Enable dynamic power down */
-	tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW28);
+	tmp = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW28);
 	tmp |= DPIO_DYNPWRDOWNEN_CH0 | DPIO_CL1POWERDOWNEN |
 		DPIO_SUS_CLK_CONFIG_GATE_CLKREQ;
-	vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW28, tmp);
+	vlv_dpio_write(dev_priv, phy, CHV_CMN_DW28, tmp);
 
 	if (id == VLV_DISP_PW_DPIO_CMN_BC) {
-		tmp = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW6_CH1);
+		tmp = vlv_dpio_read(dev_priv, phy, _CHV_CMN_DW6_CH1);
 		tmp |= DPIO_DYNPWRDOWNEN_CH1;
-		vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW6_CH1, tmp);
+		vlv_dpio_write(dev_priv, phy, _CHV_CMN_DW6_CH1, tmp);
 	} else {
 		/*
 		 * Force the non-existing CL2 off. BXT does this
 		 * too, so maybe it saves some power even though
 		 * CL2 doesn't exist?
 		 */
-		tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW30);
+		tmp = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW30);
 		tmp |= DPIO_CL2_LDOFUSE_PWRENB;
-		vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW30, tmp);
+		vlv_dpio_write(dev_priv, phy, CHV_CMN_DW30, tmp);
 	}
 
 	vlv_dpio_put(dev_priv);
@@ -1499,7 +1495,6 @@ static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpio_phy phy,
 				     enum dpio_channel ch, bool override, unsigned int mask)
 {
-	enum pipe pipe = phy == DPIO_PHY0 ? PIPE_A : PIPE_C;
 	u32 reg, val, expected, actual;
 
 	/*
@@ -1518,7 +1513,7 @@ static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpi
 		reg = _CHV_CMN_DW6_CH1;
 
 	vlv_dpio_get(dev_priv);
-	val = vlv_dpio_read(dev_priv, pipe, reg);
+	val = vlv_dpio_read(dev_priv, phy, reg);
 	vlv_dpio_put(dev_priv);
 
 	/*
diff --git a/drivers/gpu/drm/i915/display/intel_display_reset.c b/drivers/gpu/drm/i915/display/intel_display_reset.c
index 17178d5d7788..c2c347b22448 100644
--- a/drivers/gpu/drm/i915/display/intel_display_reset.c
+++ b/drivers/gpu/drm/i915/display/intel_display_reset.c
@@ -29,7 +29,7 @@ void intel_display_reset_prepare(struct drm_i915_private *dev_priv)
 		return;
 
 	/* reset doesn't touch the display */
-	if (!dev_priv->params.force_reset_modeset_test &&
+	if (!dev_priv->display.params.force_reset_modeset_test &&
 	    !gpu_reset_clobbers_display(dev_priv))
 		return;
 
diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h
index 65ea37fe8cff..9a44350ba05d 100644
--- a/drivers/gpu/drm/i915/display/intel_display_types.h
+++ b/drivers/gpu/drm/i915/display/intel_display_types.h
@@ -624,6 +624,9 @@ struct intel_connector {
 		struct drm_dp_aux *dsc_decompression_aux;
 		u8 dsc_dpcd[DP_DSC_RECEIVER_CAP_SIZE];
 		u8 fec_capability;
+
+		u8 dsc_hblank_expansion_quirk:1;
+		u8 dsc_decompression_enabled:1;
 	} dp;
 
 	/* Work struct to schedule a uevent on link train failure */
@@ -676,8 +679,6 @@ struct intel_atomic_state {
 
 	bool rps_interactive;
 
-	struct i915_sw_fence commit_ready;
-
 	struct llist_node freed;
 };
 
@@ -1210,6 +1211,7 @@ struct intel_crtc_state {
 	bool has_psr2;
 	bool enable_psr2_sel_fetch;
 	bool req_psr2_sdp_prior_scanline;
+	bool has_panel_replay;
 	bool wm_level_disabled;
 	u32 dc3co_exitline;
 	u16 su_y_granularity;
@@ -1361,7 +1363,8 @@ struct intel_crtc_state {
 	struct {
 		bool compression_enable;
 		bool dsc_split;
-		u16 compressed_bpp;
+		/* Compressed Bpp in U6.4 format (first 4 bits for fractional part) */
+		u16 compressed_bpp_x16;
 		u8 slice_count;
 		struct drm_dsc_config config;
 	} dsc;
@@ -1707,9 +1710,13 @@ struct intel_psr {
 	bool irq_aux_error;
 	u16 su_w_granularity;
 	u16 su_y_granularity;
+	bool source_panel_replay_support;
+	bool sink_panel_replay_support;
+	bool panel_replay_enabled;
 	u32 dc3co_exitline;
 	u32 dc3co_exit_delay;
 	struct delayed_work dc3co_work;
+	u8 entry_setup_frames;
 };
 
 struct intel_dp {
@@ -1808,6 +1815,7 @@ struct intel_dp {
 	/* Display stream compression testing */
 	bool force_dsc_en;
 	int force_dsc_output_format;
+	bool force_dsc_fractional_bpp_en;
 	int force_dsc_bpc;
 
 	bool hobl_failed;
@@ -1992,17 +2000,6 @@ dp_to_lspcon(struct intel_dp *intel_dp)
 
 #define dp_to_i915(__intel_dp) to_i915(dp_to_dig_port(__intel_dp)->base.base.dev)
 
-#define CAN_PSR(intel_dp) ((intel_dp)->psr.sink_support && \
-			   (intel_dp)->psr.source_support)
-
-static inline bool intel_encoder_can_psr(struct intel_encoder *encoder)
-{
-	if (!intel_encoder_is_dp(encoder))
-		return false;
-
-	return CAN_PSR(enc_to_intel_dp(encoder));
-}
-
 static inline struct intel_digital_port *
 hdmi_to_dig_port(struct intel_hdmi *intel_hdmi)
 {
diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c
index 2c1034578984..1422c2370269 100644
--- a/drivers/gpu/drm/i915/display/intel_dp.c
+++ b/drivers/gpu/drm/i915/display/intel_dp.c
@@ -85,8 +85,8 @@
 #define DP_DSC_MAX_ENC_THROUGHPUT_0		340000
 #define DP_DSC_MAX_ENC_THROUGHPUT_1		400000
 
-/* DP DSC FEC Overhead factor = 1/(0.972261) */
-#define DP_DSC_FEC_OVERHEAD_FACTOR		972261
+/* DP DSC FEC Overhead factor in ppm = 1/(0.972261) = 1.028530 */
+#define DP_DSC_FEC_OVERHEAD_FACTOR		1028530
 
 /* Compliance test status bits  */
 #define INTEL_DP_RESOLUTION_SHIFT_MASK	0
@@ -124,7 +124,31 @@ static void intel_dp_unset_edid(struct intel_dp *intel_dp);
 /* Is link rate UHBR and thus 128b/132b? */
 bool intel_dp_is_uhbr(const struct intel_crtc_state *crtc_state)
 {
-	return crtc_state->port_clock >= 1000000;
+	return drm_dp_is_uhbr_rate(crtc_state->port_clock);
+}
+
+/**
+ * intel_dp_link_symbol_size - get the link symbol size for a given link rate
+ * @rate: link rate in 10kbit/s units
+ *
+ * Returns the link symbol size in bits/symbol units depending on the link
+ * rate -> channel coding.
+ */
+int intel_dp_link_symbol_size(int rate)
+{
+	return drm_dp_is_uhbr_rate(rate) ? 32 : 10;
+}
+
+/**
+ * intel_dp_link_symbol_clock - convert link rate to link symbol clock
+ * @rate: link rate in 10kbit/s units
+ *
+ * Returns the link symbol clock frequency in kHz units depending on the
+ * link rate and channel coding.
+ */
+int intel_dp_link_symbol_clock(int rate)
+{
+	return DIV_ROUND_CLOSEST(rate * 10, intel_dp_link_symbol_size(rate));
 }
 
 static void intel_dp_set_default_sink_rates(struct intel_dp *intel_dp)
@@ -331,6 +355,9 @@ int intel_dp_max_lane_count(struct intel_dp *intel_dp)
 /*
  * The required data bandwidth for a mode with given pixel clock and bpp. This
  * is the required net bandwidth independent of the data bandwidth efficiency.
+ *
+ * TODO: check if callers of this functions should use
+ * intel_dp_effective_data_rate() instead.
  */
 int
 intel_dp_link_required(int pixel_clock, int bpp)
@@ -339,6 +366,22 @@ intel_dp_link_required(int pixel_clock, int bpp)
 	return DIV_ROUND_UP(pixel_clock * bpp, 8);
 }
 
+/**
+ * intel_dp_effective_data_rate - Return the pixel data rate accounting for BW allocation overhead
+ * @pixel_clock: pixel clock in kHz
+ * @bpp_x16: bits per pixel .4 fixed point format
+ * @bw_overhead: BW allocation overhead in 1ppm units
+ *
+ * Return the effective pixel data rate in kB/sec units taking into account
+ * the provided SSC, FEC, DSC BW allocation overhead.
+ */
+int intel_dp_effective_data_rate(int pixel_clock, int bpp_x16,
+				 int bw_overhead)
+{
+	return DIV_ROUND_UP_ULL(mul_u32_u32(pixel_clock * bpp_x16, bw_overhead),
+				1000000 * 16 * 8);
+}
+
 /*
  * Given a link rate and lanes, get the data bandwidth.
  *
@@ -362,29 +405,27 @@ intel_dp_link_required(int pixel_clock, int bpp)
 int
 intel_dp_max_data_rate(int max_link_rate, int max_lanes)
 {
-	if (max_link_rate >= 1000000) {
-		/*
-		 * UHBR rates always use 128b/132b channel encoding, and have
-		 * 97.71% data bandwidth efficiency. Consider max_link_rate the
-		 * link bit rate in units of 10000 bps.
-		 */
-		int max_link_rate_kbps = max_link_rate * 10;
-
-		max_link_rate_kbps = DIV_ROUND_CLOSEST_ULL(mul_u32_u32(max_link_rate_kbps, 9671), 10000);
-		max_link_rate = max_link_rate_kbps / 8;
-	}
+	int ch_coding_efficiency =
+		drm_dp_bw_channel_coding_efficiency(drm_dp_is_uhbr_rate(max_link_rate));
+	int max_link_rate_kbps = max_link_rate * 10;
 
+	/*
+	 * UHBR rates always use 128b/132b channel encoding, and have
+	 * 97.71% data bandwidth efficiency. Consider max_link_rate the
+	 * link bit rate in units of 10000 bps.
+	 */
 	/*
 	 * Lower than UHBR rates always use 8b/10b channel encoding, and have
 	 * 80% data bandwidth efficiency for SST non-FEC. However, this turns
-	 * out to be a nop by coincidence, and can be skipped:
+	 * out to be a nop by coincidence:
 	 *
 	 *	int max_link_rate_kbps = max_link_rate * 10;
-	 *	max_link_rate_kbps = DIV_ROUND_CLOSEST_ULL(max_link_rate_kbps * 8, 10);
+	 *	max_link_rate_kbps = DIV_ROUND_DOWN_ULL(max_link_rate_kbps * 8, 10);
 	 *	max_link_rate = max_link_rate_kbps / 8;
 	 */
-
-	return max_link_rate * max_lanes;
+	return DIV_ROUND_DOWN_ULL(mul_u32_u32(max_link_rate_kbps * max_lanes,
+					      ch_coding_efficiency),
+				  1000000 * 8);
 }
 
 bool intel_dp_can_bigjoiner(struct intel_dp *intel_dp)
@@ -680,8 +721,22 @@ int intel_dp_get_link_train_fallback_values(struct intel_dp *intel_dp,
 
 u32 intel_dp_mode_to_fec_clock(u32 mode_clock)
 {
-	return div_u64(mul_u32_u32(mode_clock, 1000000U),
-		       DP_DSC_FEC_OVERHEAD_FACTOR);
+	return div_u64(mul_u32_u32(mode_clock, DP_DSC_FEC_OVERHEAD_FACTOR),
+		       1000000U);
+}
+
+int intel_dp_bw_fec_overhead(bool fec_enabled)
+{
+	/*
+	 * TODO: Calculate the actual overhead for a given mode.
+	 * The hard-coded 1/0.972261=2.853% overhead factor
+	 * corresponds (for instance) to the 8b/10b DP FEC 2.4% +
+	 * 0.453% DSC overhead. This is enough for a 3840 width mode,
+	 * which has a DSC overhead of up to ~0.2%, but may not be
+	 * enough for a 1024 width mode where this is ~0.8% (on a 4
+	 * lane DP link, with 2 DSC slices and 8 bpp color depth).
+	 */
+	return fec_enabled ? DP_DSC_FEC_OVERHEAD_FACTOR : 1000000;
 }
 
 static int
@@ -1369,9 +1424,9 @@ static bool intel_dp_source_supports_fec(struct intel_dp *intel_dp,
 	return false;
 }
 
-static bool intel_dp_supports_fec(struct intel_dp *intel_dp,
-				  const struct intel_connector *connector,
-				  const struct intel_crtc_state *pipe_config)
+bool intel_dp_supports_fec(struct intel_dp *intel_dp,
+			   const struct intel_connector *connector,
+			   const struct intel_crtc_state *pipe_config)
 {
 	return intel_dp_source_supports_fec(intel_dp, pipe_config) &&
 		drm_dp_sink_supports_fec(connector->dp.fec_capability);
@@ -1384,6 +1439,7 @@ static bool intel_dp_supports_dsc(const struct intel_connector *connector,
 		return false;
 
 	return intel_dsc_source_support(crtc_state) &&
+		connector->dp.dsc_decompression_aux &&
 		drm_dp_sink_supports_dsc(connector->dp.dsc_dpcd);
 }
 
@@ -1717,15 +1773,15 @@ static bool intel_dp_dsc_supports_format(const struct intel_connector *connector
 	return drm_dp_dsc_sink_supports_format(connector->dp.dsc_dpcd, sink_dsc_format);
 }
 
-static bool is_bw_sufficient_for_dsc_config(u16 compressed_bpp, u32 link_clock,
+static bool is_bw_sufficient_for_dsc_config(u16 compressed_bppx16, u32 link_clock,
 					    u32 lane_count, u32 mode_clock,
 					    enum intel_output_format output_format,
 					    int timeslots)
 {
 	u32 available_bw, required_bw;
 
-	available_bw = (link_clock * lane_count * timeslots)  / 8;
-	required_bw = compressed_bpp * (intel_dp_mode_to_fec_clock(mode_clock));
+	available_bw = (link_clock * lane_count * timeslots * 16)  / 8;
+	required_bw = compressed_bppx16 * (intel_dp_mode_to_fec_clock(mode_clock));
 
 	return available_bw > required_bw;
 }
@@ -1733,7 +1789,7 @@ static bool is_bw_sufficient_for_dsc_config(u16 compressed_bpp, u32 link_clock,
 static int dsc_compute_link_config(struct intel_dp *intel_dp,
 				   struct intel_crtc_state *pipe_config,
 				   struct link_config_limits *limits,
-				   u16 compressed_bpp,
+				   u16 compressed_bppx16,
 				   int timeslots)
 {
 	const struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
@@ -1748,8 +1804,8 @@ static int dsc_compute_link_config(struct intel_dp *intel_dp,
 		for (lane_count = limits->min_lane_count;
 		     lane_count <= limits->max_lane_count;
 		     lane_count <<= 1) {
-			if (!is_bw_sufficient_for_dsc_config(compressed_bpp, link_rate, lane_count,
-							     adjusted_mode->clock,
+			if (!is_bw_sufficient_for_dsc_config(compressed_bppx16, link_rate,
+							     lane_count, adjusted_mode->clock,
 							     pipe_config->output_format,
 							     timeslots))
 				continue;
@@ -1791,7 +1847,7 @@ u16 intel_dp_dsc_max_sink_compressed_bppx16(const struct intel_connector *connec
 	return 0;
 }
 
-static int dsc_sink_min_compressed_bpp(struct intel_crtc_state *pipe_config)
+int intel_dp_dsc_sink_min_compressed_bpp(struct intel_crtc_state *pipe_config)
 {
 	/* From Mandatory bit rate range Support Table 2-157 (DP v2.0) */
 	switch (pipe_config->output_format) {
@@ -1808,9 +1864,9 @@ static int dsc_sink_min_compressed_bpp(struct intel_crtc_state *pipe_config)
 	return 0;
 }
 
-static int dsc_sink_max_compressed_bpp(const struct intel_connector *connector,
-				       struct intel_crtc_state *pipe_config,
-				       int bpc)
+int intel_dp_dsc_sink_max_compressed_bpp(const struct intel_connector *connector,
+					 struct intel_crtc_state *pipe_config,
+					 int bpc)
 {
 	return intel_dp_dsc_max_sink_compressed_bppx16(connector,
 						       pipe_config, bpc) >> 4;
@@ -1862,10 +1918,11 @@ icl_dsc_compute_link_config(struct intel_dp *intel_dp,
 		ret = dsc_compute_link_config(intel_dp,
 					      pipe_config,
 					      limits,
-					      valid_dsc_bpp[i],
+					      valid_dsc_bpp[i] << 4,
 					      timeslots);
 		if (ret == 0) {
-			pipe_config->dsc.compressed_bpp = valid_dsc_bpp[i];
+			pipe_config->dsc.compressed_bpp_x16 =
+				to_bpp_x16(valid_dsc_bpp[i]);
 			return 0;
 		}
 	}
@@ -1881,6 +1938,7 @@ icl_dsc_compute_link_config(struct intel_dp *intel_dp,
  */
 static int
 xelpd_dsc_compute_link_config(struct intel_dp *intel_dp,
+			      const struct intel_connector *connector,
 			      struct intel_crtc_state *pipe_config,
 			      struct link_config_limits *limits,
 			      int dsc_max_bpp,
@@ -1888,22 +1946,38 @@ xelpd_dsc_compute_link_config(struct intel_dp *intel_dp,
 			      int pipe_bpp,
 			      int timeslots)
 {
-	u16 compressed_bpp;
+	u8 bppx16_incr = drm_dp_dsc_sink_bpp_incr(connector->dp.dsc_dpcd);
+	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
+	u16 compressed_bppx16;
+	u8 bppx16_step;
 	int ret;
 
-	/* Compressed BPP should be less than the Input DSC bpp */
-	dsc_max_bpp = min(dsc_max_bpp, pipe_bpp - 1);
+	if (DISPLAY_VER(i915) < 14 || bppx16_incr <= 1)
+		bppx16_step = 16;
+	else
+		bppx16_step = 16 / bppx16_incr;
 
-	for (compressed_bpp = dsc_max_bpp;
-	     compressed_bpp >= dsc_min_bpp;
-	     compressed_bpp--) {
+	/* Compressed BPP should be less than the Input DSC bpp */
+	dsc_max_bpp = min(dsc_max_bpp << 4, (pipe_bpp << 4) - bppx16_step);
+	dsc_min_bpp = dsc_min_bpp << 4;
+
+	for (compressed_bppx16 = dsc_max_bpp;
+	     compressed_bppx16 >= dsc_min_bpp;
+	     compressed_bppx16 -= bppx16_step) {
+		if (intel_dp->force_dsc_fractional_bpp_en &&
+		    !to_bpp_frac(compressed_bppx16))
+			continue;
 		ret = dsc_compute_link_config(intel_dp,
 					      pipe_config,
 					      limits,
-					      compressed_bpp,
+					      compressed_bppx16,
 					      timeslots);
 		if (ret == 0) {
-			pipe_config->dsc.compressed_bpp = compressed_bpp;
+			pipe_config->dsc.compressed_bpp_x16 = compressed_bppx16;
+			if (intel_dp->force_dsc_fractional_bpp_en &&
+			    to_bpp_frac(compressed_bppx16))
+				drm_dbg_kms(&i915->drm, "Forcing DSC fractional bpp\n");
+
 			return 0;
 		}
 	}
@@ -1924,12 +1998,14 @@ static int dsc_compute_compressed_bpp(struct intel_dp *intel_dp,
 	int dsc_joiner_max_bpp;
 
 	dsc_src_min_bpp = dsc_src_min_compressed_bpp();
-	dsc_sink_min_bpp = dsc_sink_min_compressed_bpp(pipe_config);
+	dsc_sink_min_bpp = intel_dp_dsc_sink_min_compressed_bpp(pipe_config);
 	dsc_min_bpp = max(dsc_src_min_bpp, dsc_sink_min_bpp);
 	dsc_min_bpp = max(dsc_min_bpp, to_bpp_int_roundup(limits->link.min_bpp_x16));
 
 	dsc_src_max_bpp = dsc_src_max_compressed_bpp(intel_dp);
-	dsc_sink_max_bpp = dsc_sink_max_compressed_bpp(connector, pipe_config, pipe_bpp / 3);
+	dsc_sink_max_bpp = intel_dp_dsc_sink_max_compressed_bpp(connector,
+								pipe_config,
+								pipe_bpp / 3);
 	dsc_max_bpp = dsc_sink_max_bpp ? min(dsc_sink_max_bpp, dsc_src_max_bpp) : dsc_src_max_bpp;
 
 	dsc_joiner_max_bpp = get_max_compressed_bpp_with_joiner(i915, adjusted_mode->clock,
@@ -1939,7 +2015,7 @@ static int dsc_compute_compressed_bpp(struct intel_dp *intel_dp,
 	dsc_max_bpp = min(dsc_max_bpp, to_bpp_int(limits->link.max_bpp_x16));
 
 	if (DISPLAY_VER(i915) >= 13)
-		return xelpd_dsc_compute_link_config(intel_dp, pipe_config, limits,
+		return xelpd_dsc_compute_link_config(intel_dp, connector, pipe_config, limits,
 						     dsc_max_bpp, dsc_min_bpp, pipe_bpp, timeslots);
 	return icl_dsc_compute_link_config(intel_dp, pipe_config, limits,
 					   dsc_max_bpp, dsc_min_bpp, pipe_bpp, timeslots);
@@ -2084,19 +2160,22 @@ static int intel_edp_dsc_compute_pipe_bpp(struct intel_dp *intel_dp,
 	pipe_config->lane_count = limits->max_lane_count;
 
 	dsc_src_min_bpp = dsc_src_min_compressed_bpp();
-	dsc_sink_min_bpp = dsc_sink_min_compressed_bpp(pipe_config);
+	dsc_sink_min_bpp = intel_dp_dsc_sink_min_compressed_bpp(pipe_config);
 	dsc_min_bpp = max(dsc_src_min_bpp, dsc_sink_min_bpp);
 	dsc_min_bpp = max(dsc_min_bpp, to_bpp_int_roundup(limits->link.min_bpp_x16));
 
 	dsc_src_max_bpp = dsc_src_max_compressed_bpp(intel_dp);
-	dsc_sink_max_bpp = dsc_sink_max_compressed_bpp(connector, pipe_config, pipe_bpp / 3);
+	dsc_sink_max_bpp = intel_dp_dsc_sink_max_compressed_bpp(connector,
+								pipe_config,
+								pipe_bpp / 3);
 	dsc_max_bpp = dsc_sink_max_bpp ? min(dsc_sink_max_bpp, dsc_src_max_bpp) : dsc_src_max_bpp;
 	dsc_max_bpp = min(dsc_max_bpp, to_bpp_int(limits->link.max_bpp_x16));
 
 	/* Compressed BPP should be less than the Input DSC bpp */
 	dsc_max_bpp = min(dsc_max_bpp, pipe_bpp - 1);
 
-	pipe_config->dsc.compressed_bpp = max(dsc_min_bpp, dsc_max_bpp);
+	pipe_config->dsc.compressed_bpp_x16 =
+		to_bpp_x16(max(dsc_min_bpp, dsc_max_bpp));
 
 	pipe_config->pipe_bpp = pipe_bpp;
 
@@ -2118,8 +2197,9 @@ int intel_dp_dsc_compute_config(struct intel_dp *intel_dp,
 		&pipe_config->hw.adjusted_mode;
 	int ret;
 
-	pipe_config->fec_enable = !intel_dp_is_edp(intel_dp) &&
-		intel_dp_supports_fec(intel_dp, connector, pipe_config);
+	pipe_config->fec_enable = pipe_config->fec_enable ||
+		(!intel_dp_is_edp(intel_dp) &&
+		 intel_dp_supports_fec(intel_dp, connector, pipe_config));
 
 	if (!intel_dp_supports_dsc(connector, pipe_config))
 		return -EINVAL;
@@ -2184,18 +2264,18 @@ int intel_dp_dsc_compute_config(struct intel_dp *intel_dp,
 	ret = intel_dp_dsc_compute_params(connector, pipe_config);
 	if (ret < 0) {
 		drm_dbg_kms(&dev_priv->drm,
-			    "Cannot compute valid DSC parameters for Input Bpp = %d "
-			    "Compressed BPP = %d\n",
+			    "Cannot compute valid DSC parameters for Input Bpp = %d"
+			    "Compressed BPP = " BPP_X16_FMT "\n",
 			    pipe_config->pipe_bpp,
-			    pipe_config->dsc.compressed_bpp);
+			    BPP_X16_ARGS(pipe_config->dsc.compressed_bpp_x16));
 		return ret;
 	}
 
 	pipe_config->dsc.compression_enable = true;
 	drm_dbg_kms(&dev_priv->drm, "DP DSC computed with Input Bpp = %d "
-		    "Compressed Bpp = %d Slice Count = %d\n",
+		    "Compressed Bpp = " BPP_X16_FMT " Slice Count = %d\n",
 		    pipe_config->pipe_bpp,
-		    pipe_config->dsc.compressed_bpp,
+		    BPP_X16_ARGS(pipe_config->dsc.compressed_bpp_x16),
 		    pipe_config->dsc.slice_count);
 
 	return 0;
@@ -2307,6 +2387,8 @@ intel_dp_compute_link_config(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
+	const struct intel_connector *connector =
+		to_intel_connector(conn_state->connector);
 	const struct drm_display_mode *adjusted_mode =
 		&pipe_config->hw.adjusted_mode;
 	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
@@ -2315,6 +2397,10 @@ intel_dp_compute_link_config(struct intel_encoder *encoder,
 	bool dsc_needed;
 	int ret = 0;
 
+	if (pipe_config->fec_enable &&
+	    !intel_dp_supports_fec(intel_dp, connector, pipe_config))
+		return -EINVAL;
+
 	if (intel_dp_need_bigjoiner(intel_dp, adjusted_mode->crtc_hdisplay,
 				    adjusted_mode->crtc_clock))
 		pipe_config->bigjoiner_pipes = GENMASK(crtc->pipe + 1, crtc->pipe);
@@ -2362,15 +2448,15 @@ intel_dp_compute_link_config(struct intel_encoder *encoder,
 
 	if (pipe_config->dsc.compression_enable) {
 		drm_dbg_kms(&i915->drm,
-			    "DP lane count %d clock %d Input bpp %d Compressed bpp %d\n",
+			    "DP lane count %d clock %d Input bpp %d Compressed bpp " BPP_X16_FMT "\n",
 			    pipe_config->lane_count, pipe_config->port_clock,
 			    pipe_config->pipe_bpp,
-			    pipe_config->dsc.compressed_bpp);
+			    BPP_X16_ARGS(pipe_config->dsc.compressed_bpp_x16));
 
 		drm_dbg_kms(&i915->drm,
 			    "DP link rate required %i available %i\n",
 			    intel_dp_link_required(adjusted_mode->crtc_clock,
-						   pipe_config->dsc.compressed_bpp),
+						   to_bpp_int_roundup(pipe_config->dsc.compressed_bpp_x16)),
 			    intel_dp_max_data_rate(pipe_config->port_clock,
 						   pipe_config->lane_count));
 	} else {
@@ -2439,12 +2525,22 @@ static void intel_dp_compute_vsc_colorimetry(const struct intel_crtc_state *crtc
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 
-	/*
-	 * Prepare VSC Header for SU as per DP 1.4 spec, Table 2-118
-	 * VSC SDP supporting 3D stereo, PSR2, and Pixel Encoding/
-	 * Colorimetry Format indication.
-	 */
-	vsc->revision = 0x5;
+	if (crtc_state->has_panel_replay) {
+		/*
+		 * Prepare VSC Header for SU as per DP 2.0 spec, Table 2-223
+		 * VSC SDP supporting 3D stereo, Panel Replay, and Pixel
+		 * Encoding/Colorimetry Format indication.
+		 */
+		vsc->revision = 0x7;
+	} else {
+		/*
+		 * Prepare VSC Header for SU as per DP 1.4 spec, Table 2-118
+		 * VSC SDP supporting 3D stereo, PSR2, and Pixel Encoding/
+		 * Colorimetry Format indication.
+		 */
+		vsc->revision = 0x5;
+	}
+
 	vsc->length = 0x13;
 
 	/* DP 1.4a spec, Table 2-120 */
@@ -2553,6 +2649,21 @@ void intel_dp_compute_psr_vsc_sdp(struct intel_dp *intel_dp,
 			vsc->revision = 0x4;
 			vsc->length = 0xe;
 		}
+	} else if (crtc_state->has_panel_replay) {
+		if (intel_dp->psr.colorimetry_support &&
+		    intel_dp_needs_vsc_sdp(crtc_state, conn_state)) {
+			/* [Panel Replay with colorimetry info] */
+			intel_dp_compute_vsc_colorimetry(crtc_state, conn_state,
+							 vsc);
+		} else {
+			/*
+			 * [Panel Replay without colorimetry info]
+			 * Prepare VSC Header for SU as per DP 2.0 spec, Table 2-223
+			 * VSC SDP supporting 3D stereo + Panel Replay.
+			 */
+			vsc->revision = 0x6;
+			vsc->length = 0x10;
+		}
 	} else {
 		/*
 		 * [PSR1]
@@ -2629,7 +2740,7 @@ static bool can_enable_drrs(struct intel_connector *connector,
 static void
 intel_dp_drrs_compute_config(struct intel_connector *connector,
 			     struct intel_crtc_state *pipe_config,
-			     int link_bpp)
+			     int link_bpp_x16)
 {
 	struct drm_i915_private *i915 = to_i915(connector->base.dev);
 	const struct drm_display_mode *downclock_mode =
@@ -2654,9 +2765,10 @@ intel_dp_drrs_compute_config(struct intel_connector *connector,
 	if (pipe_config->splitter.enable)
 		pixel_clock /= pipe_config->splitter.link_count;
 
-	intel_link_compute_m_n(link_bpp, pipe_config->lane_count, pixel_clock,
-			       pipe_config->port_clock, &pipe_config->dp_m2_n2,
-			       pipe_config->fec_enable);
+	intel_link_compute_m_n(link_bpp_x16, pipe_config->lane_count, pixel_clock,
+			       pipe_config->port_clock,
+			       intel_dp_bw_fec_overhead(pipe_config->fec_enable),
+			       &pipe_config->dp_m2_n2);
 
 	/* FIXME: abstract this better */
 	if (pipe_config->splitter.enable)
@@ -2757,7 +2869,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 	const struct drm_display_mode *fixed_mode;
 	struct intel_connector *connector = intel_dp->attached_connector;
-	int ret = 0, link_bpp;
+	int ret = 0, link_bpp_x16;
 
 	if (HAS_PCH_SPLIT(dev_priv) && !HAS_DDI(dev_priv) && encoder->port != PORT_A)
 		pipe_config->has_pch_encoder = true;
@@ -2806,10 +2918,10 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 		drm_dp_enhanced_frame_cap(intel_dp->dpcd);
 
 	if (pipe_config->dsc.compression_enable)
-		link_bpp = pipe_config->dsc.compressed_bpp;
+		link_bpp_x16 = pipe_config->dsc.compressed_bpp_x16;
 	else
-		link_bpp = intel_dp_output_bpp(pipe_config->output_format,
-					       pipe_config->pipe_bpp);
+		link_bpp_x16 = to_bpp_x16(intel_dp_output_bpp(pipe_config->output_format,
+							      pipe_config->pipe_bpp));
 
 	if (intel_dp->mso_link_count) {
 		int n = intel_dp->mso_link_count;
@@ -2833,12 +2945,12 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 
 	intel_dp_audio_compute_config(encoder, pipe_config, conn_state);
 
-	intel_link_compute_m_n(link_bpp,
+	intel_link_compute_m_n(link_bpp_x16,
 			       pipe_config->lane_count,
 			       adjusted_mode->crtc_clock,
 			       pipe_config->port_clock,
-			       &pipe_config->dp_m_n,
-			       pipe_config->fec_enable);
+			       intel_dp_bw_fec_overhead(pipe_config->fec_enable),
+			       &pipe_config->dp_m_n);
 
 	/* FIXME: abstract this better */
 	if (pipe_config->splitter.enable)
@@ -2849,7 +2961,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 
 	intel_vrr_compute_config(pipe_config, conn_state);
 	intel_psr_compute_config(intel_dp, pipe_config, conn_state);
-	intel_dp_drrs_compute_config(connector, pipe_config, link_bpp);
+	intel_dp_drrs_compute_config(connector, pipe_config, link_bpp_x16);
 	intel_dp_compute_vsc_sdp(intel_dp, pipe_config, conn_state);
 	intel_dp_compute_hdr_metadata_infoframe_sdp(intel_dp, pipe_config, conn_state);
 
@@ -2917,24 +3029,179 @@ static bool downstream_hpd_needs_d0(struct intel_dp *intel_dp)
 		intel_dp->downstream_ports[0] & DP_DS_PORT_HPD;
 }
 
-void intel_dp_sink_set_decompression_state(struct intel_dp *intel_dp,
-					   const struct intel_crtc_state *crtc_state,
-					   bool enable)
+static int
+write_dsc_decompression_flag(struct drm_dp_aux *aux, u8 flag, bool set)
 {
-	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
-	int ret;
+	int err;
+	u8 val;
 
-	if (!crtc_state->dsc.compression_enable)
-		return;
+	err = drm_dp_dpcd_readb(aux, DP_DSC_ENABLE, &val);
+	if (err < 0)
+		return err;
 
-	ret = drm_dp_dpcd_writeb(&intel_dp->aux, DP_DSC_ENABLE,
-				 enable ? DP_DECOMPRESSION_EN : 0);
-	if (ret < 0)
+	if (set)
+		val |= flag;
+	else
+		val &= ~flag;
+
+	return drm_dp_dpcd_writeb(aux, DP_DSC_ENABLE, val);
+}
+
+static void
+intel_dp_sink_set_dsc_decompression(struct intel_connector *connector,
+				    bool enable)
+{
+	struct drm_i915_private *i915 = to_i915(connector->base.dev);
+
+	if (write_dsc_decompression_flag(connector->dp.dsc_decompression_aux,
+					 DP_DECOMPRESSION_EN, enable) < 0)
 		drm_dbg_kms(&i915->drm,
 			    "Failed to %s sink decompression state\n",
 			    str_enable_disable(enable));
 }
 
+static void
+intel_dp_sink_set_dsc_passthrough(const struct intel_connector *connector,
+				  bool enable)
+{
+	struct drm_i915_private *i915 = to_i915(connector->base.dev);
+	struct drm_dp_aux *aux = connector->port ?
+				 connector->port->passthrough_aux : NULL;
+
+	if (!aux)
+		return;
+
+	if (write_dsc_decompression_flag(aux,
+					 DP_DSC_PASSTHROUGH_EN, enable) < 0)
+		drm_dbg_kms(&i915->drm,
+			    "Failed to %s sink compression passthrough state\n",
+			    str_enable_disable(enable));
+}
+
+static int intel_dp_dsc_aux_ref_count(struct intel_atomic_state *state,
+				      const struct intel_connector *connector,
+				      bool for_get_ref)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	struct drm_connector *_connector_iter;
+	struct drm_connector_state *old_conn_state;
+	struct drm_connector_state *new_conn_state;
+	int ref_count = 0;
+	int i;
+
+	/*
+	 * On SST the decompression AUX device won't be shared, each connector
+	 * uses for this its own AUX targeting the sink device.
+	 */
+	if (!connector->mst_port)
+		return connector->dp.dsc_decompression_enabled ? 1 : 0;
+
+	for_each_oldnew_connector_in_state(&state->base, _connector_iter,
+					   old_conn_state, new_conn_state, i) {
+		const struct intel_connector *
+			connector_iter = to_intel_connector(_connector_iter);
+
+		if (connector_iter->mst_port != connector->mst_port)
+			continue;
+
+		if (!connector_iter->dp.dsc_decompression_enabled)
+			continue;
+
+		drm_WARN_ON(&i915->drm,
+			    (for_get_ref && !new_conn_state->crtc) ||
+			    (!for_get_ref && !old_conn_state->crtc));
+
+		if (connector_iter->dp.dsc_decompression_aux ==
+		    connector->dp.dsc_decompression_aux)
+			ref_count++;
+	}
+
+	return ref_count;
+}
+
+static bool intel_dp_dsc_aux_get_ref(struct intel_atomic_state *state,
+				     struct intel_connector *connector)
+{
+	bool ret = intel_dp_dsc_aux_ref_count(state, connector, true) == 0;
+
+	connector->dp.dsc_decompression_enabled = true;
+
+	return ret;
+}
+
+static bool intel_dp_dsc_aux_put_ref(struct intel_atomic_state *state,
+				     struct intel_connector *connector)
+{
+	connector->dp.dsc_decompression_enabled = false;
+
+	return intel_dp_dsc_aux_ref_count(state, connector, false) == 0;
+}
+
+/**
+ * intel_dp_sink_enable_decompression - Enable DSC decompression in sink/last branch device
+ * @state: atomic state
+ * @connector: connector to enable the decompression for
+ * @new_crtc_state: new state for the CRTC driving @connector
+ *
+ * Enable the DSC decompression if required in the %DP_DSC_ENABLE DPCD
+ * register of the appropriate sink/branch device. On SST this is always the
+ * sink device, whereas on MST based on each device's DSC capabilities it's
+ * either the last branch device (enabling decompression in it) or both the
+ * last branch device (enabling passthrough in it) and the sink device
+ * (enabling decompression in it).
+ */
+void intel_dp_sink_enable_decompression(struct intel_atomic_state *state,
+					struct intel_connector *connector,
+					const struct intel_crtc_state *new_crtc_state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+
+	if (!new_crtc_state->dsc.compression_enable)
+		return;
+
+	if (drm_WARN_ON(&i915->drm,
+			!connector->dp.dsc_decompression_aux ||
+			connector->dp.dsc_decompression_enabled))
+		return;
+
+	if (!intel_dp_dsc_aux_get_ref(state, connector))
+		return;
+
+	intel_dp_sink_set_dsc_passthrough(connector, true);
+	intel_dp_sink_set_dsc_decompression(connector, true);
+}
+
+/**
+ * intel_dp_sink_disable_decompression - Disable DSC decompression in sink/last branch device
+ * @state: atomic state
+ * @connector: connector to disable the decompression for
+ * @old_crtc_state: old state for the CRTC driving @connector
+ *
+ * Disable the DSC decompression if required in the %DP_DSC_ENABLE DPCD
+ * register of the appropriate sink/branch device, corresponding to the
+ * sequence in intel_dp_sink_enable_decompression().
+ */
+void intel_dp_sink_disable_decompression(struct intel_atomic_state *state,
+					 struct intel_connector *connector,
+					 const struct intel_crtc_state *old_crtc_state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+
+	if (!old_crtc_state->dsc.compression_enable)
+		return;
+
+	if (drm_WARN_ON(&i915->drm,
+			!connector->dp.dsc_decompression_aux ||
+			!connector->dp.dsc_decompression_enabled))
+		return;
+
+	if (!intel_dp_dsc_aux_put_ref(state, connector))
+		return;
+
+	intel_dp_sink_set_dsc_decompression(connector, false);
+	intel_dp_sink_set_dsc_passthrough(connector, false);
+}
+
 static void
 intel_edp_init_source_oui(struct intel_dp *intel_dp, bool careful)
 {
@@ -3771,7 +4038,7 @@ intel_dp_can_mst(struct intel_dp *intel_dp)
 {
 	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
 
-	return i915->params.enable_dp_mst &&
+	return i915->display.params.enable_dp_mst &&
 		intel_dp_mst_source_support(intel_dp) &&
 		drm_dp_read_mst_cap(&intel_dp->aux, intel_dp->dpcd);
 }
@@ -3789,13 +4056,13 @@ intel_dp_configure_mst(struct intel_dp *intel_dp)
 		    encoder->base.base.id, encoder->base.name,
 		    str_yes_no(intel_dp_mst_source_support(intel_dp)),
 		    str_yes_no(sink_can_mst),
-		    str_yes_no(i915->params.enable_dp_mst));
+		    str_yes_no(i915->display.params.enable_dp_mst));
 
 	if (!intel_dp_mst_source_support(intel_dp))
 		return;
 
 	intel_dp->is_mst = sink_can_mst &&
-		i915->params.enable_dp_mst;
+		i915->display.params.enable_dp_mst;
 
 	drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr,
 					intel_dp->is_mst);
@@ -3865,11 +4132,16 @@ static ssize_t intel_dp_vsc_sdp_pack(const struct drm_dp_vsc_sdp *vsc,
 	sdp->sdp_header.HB2 = vsc->revision; /* Revision Number */
 	sdp->sdp_header.HB3 = vsc->length; /* Number of Valid Data Bytes */
 
+	if (vsc->revision == 0x6) {
+		sdp->db[0] = 1;
+		sdp->db[3] = 1;
+	}
+
 	/*
-	 * Only revision 0x5 supports Pixel Encoding/Colorimetry Format as
-	 * per DP 1.4a spec.
+	 * Revision 0x5 and revision 0x7 supports Pixel Encoding/Colorimetry
+	 * Format as per DP 1.4a spec and DP 2.0 respectively.
 	 */
-	if (vsc->revision != 0x5)
+	if (!(vsc->revision == 0x5 || vsc->revision == 0x7))
 		goto out;
 
 	/* VSC SDP Payload for DB16 through DB18 */
@@ -4049,7 +4321,10 @@ void intel_dp_set_infoframes(struct intel_encoder *encoder,
 			 VIDEO_DIP_ENABLE_SPD_HSW | VIDEO_DIP_ENABLE_DRM_GLK;
 	u32 val = intel_de_read(dev_priv, reg) & ~dip_enable;
 
-	/* TODO: Add DSC case (DIP_ENABLE_PPS) */
+	/* TODO: Sanitize DSC enabling wrt. intel_dsc_dp_pps_write(). */
+	if (!enable && HAS_DSC(dev_priv))
+		val &= ~VDIP_ENABLE_PPS;
+
 	/* When PSR is enabled, this routine doesn't disable VSC DIP */
 	if (!crtc_state->has_psr)
 		val &= ~VIDEO_DIP_ENABLE_VSC_HSW;
@@ -5409,6 +5684,7 @@ intel_dp_detect(struct drm_connector *connector,
 	if (status == connector_status_disconnected) {
 		memset(&intel_dp->compliance, 0, sizeof(intel_dp->compliance));
 		memset(intel_connector->dp.dsc_dpcd, 0, sizeof(intel_connector->dp.dsc_dpcd));
+		intel_dp->psr.sink_panel_replay_support = false;
 
 		if (intel_dp->is_mst) {
 			drm_dbg_kms(&dev_priv->drm,
@@ -6037,8 +6313,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
 	 * (eg. Acer Chromebook C710), so we'll check it only if multiple
 	 * ports are attempting to use the same AUX CH, according to VBT.
 	 */
-	if (intel_bios_dp_has_shared_aux_ch(encoder->devdata) &&
-	    !intel_digital_port_connected(encoder)) {
+	if (intel_bios_dp_has_shared_aux_ch(encoder->devdata)) {
 		/*
 		 * If this fails, presume the DPCD answer came
 		 * from some other port using the same AUX CH.
@@ -6046,10 +6321,27 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
 		 * FIXME maybe cleaner to check this before the
 		 * DPCD read? Would need sort out the VDD handling...
 		 */
-		drm_info(&dev_priv->drm,
-			 "[ENCODER:%d:%s] HPD is down, disabling eDP\n",
-			 encoder->base.base.id, encoder->base.name);
-		goto out_vdd_off;
+		if (!intel_digital_port_connected(encoder)) {
+			drm_info(&dev_priv->drm,
+				 "[ENCODER:%d:%s] HPD is down, disabling eDP\n",
+				 encoder->base.base.id, encoder->base.name);
+			goto out_vdd_off;
+		}
+
+		/*
+		 * Unfortunately even the HPD based detection fails on
+		 * eg. Asus B360M-A (CFL+CNP), so as a last resort fall
+		 * back to checking for a VGA branch device. Only do this
+		 * on known affected platforms to minimize false positives.
+		 */
+		if (DISPLAY_VER(dev_priv) == 9 && drm_dp_is_branch(intel_dp->dpcd) &&
+		    (intel_dp->dpcd[DP_DOWNSTREAMPORT_PRESENT] & DP_DWN_STRM_PORT_TYPE_MASK) ==
+		    DP_DWN_STRM_PORT_TYPE_ANALOG) {
+			drm_info(&dev_priv->drm,
+				 "[ENCODER:%d:%s] VGA converter detected, disabling eDP\n",
+				 encoder->base.base.id, encoder->base.name);
+			goto out_vdd_off;
+		}
 	}
 
 	mutex_lock(&dev_priv->drm.mode_config.mutex);
@@ -6238,16 +6530,6 @@ intel_dp_init_connector(struct intel_digital_port *dig_port,
 				    "HDCP init failed, skipping.\n");
 	}
 
-	/* For G4X desktop chip, PEG_BAND_GAP_DATA 3:0 must first be written
-	 * 0xd.  Failure to do so will result in spurious interrupts being
-	 * generated on the port when a cable is not attached.
-	 */
-	if (IS_G45(dev_priv)) {
-		u32 temp = intel_de_read(dev_priv, PEG_BAND_GAP_DATA);
-		intel_de_write(dev_priv, PEG_BAND_GAP_DATA,
-			       (temp & ~0xf) | 0xd);
-	}
-
 	intel_dp->frl.is_trained = false;
 	intel_dp->frl.trained_rate_gbps = 0;
 
diff --git a/drivers/gpu/drm/i915/display/intel_dp.h b/drivers/gpu/drm/i915/display/intel_dp.h
index 484aea215a25..05db46b111f2 100644
--- a/drivers/gpu/drm/i915/display/intel_dp.h
+++ b/drivers/gpu/drm/i915/display/intel_dp.h
@@ -57,9 +57,12 @@ int intel_dp_retrain_link(struct intel_encoder *encoder,
 void intel_dp_set_power(struct intel_dp *intel_dp, u8 mode);
 void intel_dp_configure_protocol_converter(struct intel_dp *intel_dp,
 					   const struct intel_crtc_state *crtc_state);
-void intel_dp_sink_set_decompression_state(struct intel_dp *intel_dp,
-					   const struct intel_crtc_state *crtc_state,
-					   bool enable);
+void intel_dp_sink_enable_decompression(struct intel_atomic_state *state,
+					struct intel_connector *connector,
+					const struct intel_crtc_state *new_crtc_state);
+void intel_dp_sink_disable_decompression(struct intel_atomic_state *state,
+					 struct intel_connector *connector,
+					 const struct intel_crtc_state *old_crtc_state);
 void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder);
 void intel_dp_encoder_shutdown(struct intel_encoder *intel_encoder);
 void intel_dp_encoder_flush_work(struct drm_encoder *encoder);
@@ -78,6 +81,8 @@ void intel_dp_audio_compute_config(struct intel_encoder *encoder,
 bool intel_dp_has_hdmi_sink(struct intel_dp *intel_dp);
 bool intel_dp_is_edp(struct intel_dp *intel_dp);
 bool intel_dp_is_uhbr(const struct intel_crtc_state *crtc_state);
+int intel_dp_link_symbol_size(int rate);
+int intel_dp_link_symbol_clock(int rate);
 bool intel_dp_is_port_edp(struct drm_i915_private *dev_priv, enum port port);
 enum irqreturn intel_dp_hpd_pulse(struct intel_digital_port *dig_port,
 				  bool long_hpd);
@@ -98,6 +103,8 @@ bool intel_dp_source_supports_tps4(struct drm_i915_private *i915);
 
 bool intel_dp_get_colorimetry_status(struct intel_dp *intel_dp);
 int intel_dp_link_required(int pixel_clock, int bpp);
+int intel_dp_effective_data_rate(int pixel_clock, int bpp_x16,
+				 int bw_overhead);
 int intel_dp_max_data_rate(int max_link_rate, int max_lanes);
 bool intel_dp_can_bigjoiner(struct intel_dp *intel_dp);
 bool intel_dp_needs_vsc_sdp(const struct intel_crtc_state *crtc_state,
@@ -125,6 +132,10 @@ u16 intel_dp_dsc_get_max_compressed_bpp(struct drm_i915_private *i915,
 					enum intel_output_format output_format,
 					u32 pipe_bpp,
 					u32 timeslots);
+int intel_dp_dsc_sink_min_compressed_bpp(struct intel_crtc_state *pipe_config);
+int intel_dp_dsc_sink_max_compressed_bpp(const struct intel_connector *connector,
+					 struct intel_crtc_state *pipe_config,
+					 int bpc);
 u8 intel_dp_dsc_get_slice_count(const struct intel_connector *connector,
 				int mode_clock, int mode_hdisplay,
 				bool bigjoiner);
@@ -136,7 +147,16 @@ static inline unsigned int intel_dp_unused_lane_mask(int lane_count)
 	return ~((1 << lane_count) - 1) & 0xf;
 }
 
+bool intel_dp_supports_fec(struct intel_dp *intel_dp,
+			   const struct intel_connector *connector,
+			   const struct intel_crtc_state *pipe_config);
 u32 intel_dp_mode_to_fec_clock(u32 mode_clock);
+int intel_dp_bw_fec_overhead(bool fec_enabled);
+
+bool intel_dp_supports_fec(struct intel_dp *intel_dp,
+			   const struct intel_connector *connector,
+			   const struct intel_crtc_state *pipe_config);
+
 u32 intel_dp_dsc_nearest_valid_bpp(struct drm_i915_private *i915, u32 bpp, u32 pipe_bpp);
 
 void intel_ddi_update_pipe(struct intel_atomic_state *state,
diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux.c b/drivers/gpu/drm/i915/display/intel_dp_aux.c
index 4431b6290c4c..2e2af71bcd5a 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_aux.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_aux.c
@@ -74,7 +74,7 @@ intel_dp_aux_wait_done(struct intel_dp *intel_dp)
 
 static u32 g4x_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
 {
-	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
 
 	if (index)
 		return 0;
@@ -83,12 +83,12 @@ static u32 g4x_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
 	 * The clock divider is based off the hrawclk, and would like to run at
 	 * 2MHz.  So, take the hrawclk value and divide by 2000 and use that
 	 */
-	return DIV_ROUND_CLOSEST(RUNTIME_INFO(dev_priv)->rawclk_freq, 2000);
+	return DIV_ROUND_CLOSEST(RUNTIME_INFO(i915)->rawclk_freq, 2000);
 }
 
 static u32 ilk_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
 {
-	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	u32 freq;
 
@@ -101,18 +101,18 @@ static u32 ilk_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
 	 * divide by 2000 and use that
 	 */
 	if (dig_port->aux_ch == AUX_CH_A)
-		freq = dev_priv->display.cdclk.hw.cdclk;
+		freq = i915->display.cdclk.hw.cdclk;
 	else
-		freq = RUNTIME_INFO(dev_priv)->rawclk_freq;
+		freq = RUNTIME_INFO(i915)->rawclk_freq;
 	return DIV_ROUND_CLOSEST(freq, 2000);
 }
 
 static u32 hsw_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
 {
-	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 
-	if (dig_port->aux_ch != AUX_CH_A && HAS_PCH_LPT_H(dev_priv)) {
+	if (dig_port->aux_ch != AUX_CH_A && HAS_PCH_LPT_H(i915)) {
 		/* Workaround for non-ULT HSW */
 		switch (index) {
 		case 0: return 63;
@@ -165,12 +165,11 @@ static u32 g4x_get_aux_send_ctl(struct intel_dp *intel_dp,
 				u32 aux_clock_divider)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	struct drm_i915_private *dev_priv =
-			to_i915(dig_port->base.base.dev);
+	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
 	u32 timeout;
 
 	/* Max timeout value on G4x-BDW: 1.6ms */
-	if (IS_BROADWELL(dev_priv))
+	if (IS_BROADWELL(i915))
 		timeout = DP_AUX_CH_CTL_TIME_OUT_600us;
 	else
 		timeout = DP_AUX_CH_CTL_TIME_OUT_400us;
@@ -229,8 +228,7 @@ intel_dp_aux_xfer(struct intel_dp *intel_dp,
 		  u32 aux_send_ctl_flags)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	struct drm_i915_private *i915 =
-			to_i915(dig_port->base.base.dev);
+	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
 	enum phy phy = intel_port_to_phy(i915, dig_port->base.port);
 	bool is_tc_port = intel_phy_is_tc(i915, phy);
 	i915_reg_t ch_ctl, ch_data[5];
@@ -531,9 +529,40 @@ intel_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg)
 	return ret;
 }
 
+static i915_reg_t vlv_aux_ctl_reg(struct intel_dp *intel_dp)
+{
+	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
+	enum aux_ch aux_ch = dig_port->aux_ch;
+
+	switch (aux_ch) {
+	case AUX_CH_B:
+	case AUX_CH_C:
+	case AUX_CH_D:
+		return VLV_DP_AUX_CH_CTL(aux_ch);
+	default:
+		MISSING_CASE(aux_ch);
+		return VLV_DP_AUX_CH_CTL(AUX_CH_B);
+	}
+}
+
+static i915_reg_t vlv_aux_data_reg(struct intel_dp *intel_dp, int index)
+{
+	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
+	enum aux_ch aux_ch = dig_port->aux_ch;
+
+	switch (aux_ch) {
+	case AUX_CH_B:
+	case AUX_CH_C:
+	case AUX_CH_D:
+		return VLV_DP_AUX_CH_DATA(aux_ch, index);
+	default:
+		MISSING_CASE(aux_ch);
+		return VLV_DP_AUX_CH_DATA(AUX_CH_B, index);
+	}
+}
+
 static i915_reg_t g4x_aux_ctl_reg(struct intel_dp *intel_dp)
 {
-	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum aux_ch aux_ch = dig_port->aux_ch;
 
@@ -550,7 +579,6 @@ static i915_reg_t g4x_aux_ctl_reg(struct intel_dp *intel_dp)
 
 static i915_reg_t g4x_aux_data_reg(struct intel_dp *intel_dp, int index)
 {
-	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum aux_ch aux_ch = dig_port->aux_ch;
 
@@ -567,7 +595,6 @@ static i915_reg_t g4x_aux_data_reg(struct intel_dp *intel_dp, int index)
 
 static i915_reg_t ilk_aux_ctl_reg(struct intel_dp *intel_dp)
 {
-	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum aux_ch aux_ch = dig_port->aux_ch;
 
@@ -586,7 +613,6 @@ static i915_reg_t ilk_aux_ctl_reg(struct intel_dp *intel_dp)
 
 static i915_reg_t ilk_aux_data_reg(struct intel_dp *intel_dp, int index)
 {
-	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum aux_ch aux_ch = dig_port->aux_ch;
 
@@ -605,7 +631,6 @@ static i915_reg_t ilk_aux_data_reg(struct intel_dp *intel_dp, int index)
 
 static i915_reg_t skl_aux_ctl_reg(struct intel_dp *intel_dp)
 {
-	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum aux_ch aux_ch = dig_port->aux_ch;
 
@@ -625,7 +650,6 @@ static i915_reg_t skl_aux_ctl_reg(struct intel_dp *intel_dp)
 
 static i915_reg_t skl_aux_data_reg(struct intel_dp *intel_dp, int index)
 {
-	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum aux_ch aux_ch = dig_port->aux_ch;
 
@@ -645,7 +669,6 @@ static i915_reg_t skl_aux_data_reg(struct intel_dp *intel_dp, int index)
 
 static i915_reg_t tgl_aux_ctl_reg(struct intel_dp *intel_dp)
 {
-	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum aux_ch aux_ch = dig_port->aux_ch;
 
@@ -668,7 +691,6 @@ static i915_reg_t tgl_aux_ctl_reg(struct intel_dp *intel_dp)
 
 static i915_reg_t tgl_aux_data_reg(struct intel_dp *intel_dp, int index)
 {
-	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum aux_ch aux_ch = dig_port->aux_ch;
 
@@ -691,7 +713,7 @@ static i915_reg_t tgl_aux_data_reg(struct intel_dp *intel_dp, int index)
 
 static i915_reg_t xelpdp_aux_ctl_reg(struct intel_dp *intel_dp)
 {
-	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum aux_ch aux_ch = dig_port->aux_ch;
 
@@ -702,16 +724,16 @@ static i915_reg_t xelpdp_aux_ctl_reg(struct intel_dp *intel_dp)
 	case AUX_CH_USBC2:
 	case AUX_CH_USBC3:
 	case AUX_CH_USBC4:
-		return XELPDP_DP_AUX_CH_CTL(dev_priv, aux_ch);
+		return XELPDP_DP_AUX_CH_CTL(i915, aux_ch);
 	default:
 		MISSING_CASE(aux_ch);
-		return XELPDP_DP_AUX_CH_CTL(dev_priv, AUX_CH_A);
+		return XELPDP_DP_AUX_CH_CTL(i915, AUX_CH_A);
 	}
 }
 
 static i915_reg_t xelpdp_aux_data_reg(struct intel_dp *intel_dp, int index)
 {
-	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum aux_ch aux_ch = dig_port->aux_ch;
 
@@ -722,10 +744,10 @@ static i915_reg_t xelpdp_aux_data_reg(struct intel_dp *intel_dp, int index)
 	case AUX_CH_USBC2:
 	case AUX_CH_USBC3:
 	case AUX_CH_USBC4:
-		return XELPDP_DP_AUX_CH_DATA(dev_priv, aux_ch, index);
+		return XELPDP_DP_AUX_CH_DATA(i915, aux_ch, index);
 	default:
 		MISSING_CASE(aux_ch);
-		return XELPDP_DP_AUX_CH_DATA(dev_priv, AUX_CH_A, index);
+		return XELPDP_DP_AUX_CH_DATA(i915, AUX_CH_A, index);
 	}
 }
 
@@ -739,49 +761,52 @@ void intel_dp_aux_fini(struct intel_dp *intel_dp)
 
 void intel_dp_aux_init(struct intel_dp *intel_dp)
 {
-	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	struct intel_encoder *encoder = &dig_port->base;
 	enum aux_ch aux_ch = dig_port->aux_ch;
 	char buf[AUX_CH_NAME_BUFSIZE];
 
-	if (DISPLAY_VER(dev_priv) >= 14) {
+	if (DISPLAY_VER(i915) >= 14) {
 		intel_dp->aux_ch_ctl_reg = xelpdp_aux_ctl_reg;
 		intel_dp->aux_ch_data_reg = xelpdp_aux_data_reg;
-	} else if (DISPLAY_VER(dev_priv) >= 12) {
+	} else if (DISPLAY_VER(i915) >= 12) {
 		intel_dp->aux_ch_ctl_reg = tgl_aux_ctl_reg;
 		intel_dp->aux_ch_data_reg = tgl_aux_data_reg;
-	} else if (DISPLAY_VER(dev_priv) >= 9) {
+	} else if (DISPLAY_VER(i915) >= 9) {
 		intel_dp->aux_ch_ctl_reg = skl_aux_ctl_reg;
 		intel_dp->aux_ch_data_reg = skl_aux_data_reg;
-	} else if (HAS_PCH_SPLIT(dev_priv)) {
+	} else if (HAS_PCH_SPLIT(i915)) {
 		intel_dp->aux_ch_ctl_reg = ilk_aux_ctl_reg;
 		intel_dp->aux_ch_data_reg = ilk_aux_data_reg;
+	} else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
+		intel_dp->aux_ch_ctl_reg = vlv_aux_ctl_reg;
+		intel_dp->aux_ch_data_reg = vlv_aux_data_reg;
 	} else {
 		intel_dp->aux_ch_ctl_reg = g4x_aux_ctl_reg;
 		intel_dp->aux_ch_data_reg = g4x_aux_data_reg;
 	}
 
-	if (DISPLAY_VER(dev_priv) >= 9)
+	if (DISPLAY_VER(i915) >= 9)
 		intel_dp->get_aux_clock_divider = skl_get_aux_clock_divider;
-	else if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
+	else if (IS_BROADWELL(i915) || IS_HASWELL(i915))
 		intel_dp->get_aux_clock_divider = hsw_get_aux_clock_divider;
-	else if (HAS_PCH_SPLIT(dev_priv))
+	else if (HAS_PCH_SPLIT(i915))
 		intel_dp->get_aux_clock_divider = ilk_get_aux_clock_divider;
 	else
 		intel_dp->get_aux_clock_divider = g4x_get_aux_clock_divider;
 
-	if (DISPLAY_VER(dev_priv) >= 9)
+	if (DISPLAY_VER(i915) >= 9)
 		intel_dp->get_aux_send_ctl = skl_get_aux_send_ctl;
 	else
 		intel_dp->get_aux_send_ctl = g4x_get_aux_send_ctl;
 
-	intel_dp->aux.drm_dev = &dev_priv->drm;
+	intel_dp->aux.drm_dev = &i915->drm;
 	drm_dp_aux_init(&intel_dp->aux);
 
 	/* Failure to allocate our preferred name is not critical */
 	intel_dp->aux.name = kasprintf(GFP_KERNEL, "AUX %s/%s",
-				       aux_ch_name(dev_priv, buf, sizeof(buf), aux_ch),
+				       aux_ch_name(i915, buf, sizeof(buf), aux_ch),
 				       encoder->base.name);
 
 	intel_dp->aux.transfer = intel_dp_aux_transfer;
diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c b/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c
index 26ea7e9f1b89..4f58efdc688a 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c
@@ -146,7 +146,7 @@ intel_dp_aux_supports_hdr_backlight(struct intel_connector *connector)
 	 * HDR static metadata we need to start maintaining table of
 	 * ranges for such panels.
 	 */
-	if (i915->params.enable_dpcd_backlight != INTEL_DP_AUX_BACKLIGHT_FORCE_INTEL &&
+	if (i915->display.params.enable_dpcd_backlight != INTEL_DP_AUX_BACKLIGHT_FORCE_INTEL &&
 	    !(connector->base.hdr_sink_metadata.hdmi_type1.metadata_type &
 	      BIT(HDMI_STATIC_METADATA_TYPE1))) {
 		drm_info(&i915->drm,
@@ -489,7 +489,7 @@ int intel_dp_aux_init_backlight_funcs(struct intel_connector *connector)
 	/* Check the VBT and user's module parameters to figure out which
 	 * interfaces to probe
 	 */
-	switch (i915->params.enable_dpcd_backlight) {
+	switch (i915->display.params.enable_dpcd_backlight) {
 	case INTEL_DP_AUX_BACKLIGHT_OFF:
 		return -ENODEV;
 	case INTEL_DP_AUX_BACKLIGHT_AUTO:
diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux_regs.h b/drivers/gpu/drm/i915/display/intel_dp_aux_regs.h
index 34f6e0a48ed2..e642445364d2 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_aux_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_dp_aux_regs.h
@@ -21,13 +21,14 @@
 #define __xe2lpd_aux_ch_idx(aux_ch)						\
 	(aux_ch >= AUX_CH_USBC1 ? aux_ch : AUX_CH_USBC4 + 1 + (aux_ch) - AUX_CH_A)
 
-/* TODO: Remove implicit dev_priv */
-#define _DPA_AUX_CH_CTL			(DISPLAY_MMIO_BASE(dev_priv) + 0x64010)
-#define _DPB_AUX_CH_CTL			(DISPLAY_MMIO_BASE(dev_priv) + 0x64110)
+#define _DPA_AUX_CH_CTL			0x64010
+#define _DPB_AUX_CH_CTL			0x64110
 #define _XELPDP_USBC1_AUX_CH_CTL	0x16f210
 #define _XELPDP_USBC2_AUX_CH_CTL	0x16f410
 #define DP_AUX_CH_CTL(aux_ch)		_MMIO_PORT(aux_ch, _DPA_AUX_CH_CTL,	\
 						   _DPB_AUX_CH_CTL)
+#define VLV_DP_AUX_CH_CTL(aux_ch)	_MMIO(VLV_DISPLAY_BASE + \
+					      _PORT(aux_ch, _DPA_AUX_CH_CTL, _DPB_AUX_CH_CTL))
 #define _XELPDP_DP_AUX_CH_CTL(aux_ch)						\
 		_MMIO(_PICK_EVEN_2RANGES(aux_ch, AUX_CH_USBC1,			\
 					 _DPA_AUX_CH_CTL, _DPB_AUX_CH_CTL,	\
@@ -69,13 +70,14 @@
 #define   DP_AUX_CH_CTL_SYNC_PULSE_SKL_MASK	REG_GENMASK(4, 0) /* skl+ */
 #define   DP_AUX_CH_CTL_SYNC_PULSE_SKL(c)	REG_FIELD_PREP(DP_AUX_CH_CTL_SYNC_PULSE_SKL_MASK, (c) - 1)
 
-/* TODO: Remove implicit dev_priv */
-#define _DPA_AUX_CH_DATA1		(DISPLAY_MMIO_BASE(dev_priv) + 0x64014)
-#define _DPB_AUX_CH_DATA1		(DISPLAY_MMIO_BASE(dev_priv) + 0x64114)
+#define _DPA_AUX_CH_DATA1		0x64014
+#define _DPB_AUX_CH_DATA1		0x64114
 #define _XELPDP_USBC1_AUX_CH_DATA1	0x16f214
 #define _XELPDP_USBC2_AUX_CH_DATA1	0x16f414
 #define DP_AUX_CH_DATA(aux_ch, i)	_MMIO(_PORT(aux_ch, _DPA_AUX_CH_DATA1,	\
 						    _DPB_AUX_CH_DATA1) + (i) * 4) /* 5 registers */
+#define VLV_DP_AUX_CH_DATA(aux_ch, i)	_MMIO(VLV_DISPLAY_BASE + _PORT(aux_ch, _DPA_AUX_CH_DATA1, \
+								       _DPB_AUX_CH_DATA1) + (i) * 4) /* 5 registers */
 #define _XELPDP_DP_AUX_CH_DATA(aux_ch, i)					\
 		_MMIO(_PICK_EVEN_2RANGES(aux_ch, AUX_CH_USBC1,			\
 					 _DPA_AUX_CH_DATA1, _DPB_AUX_CH_DATA1,	\
diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.c b/drivers/gpu/drm/i915/display/intel_dp_mst.c
index 7b4628f4f124..d3d53e1b4489 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_mst.c
@@ -26,6 +26,7 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_fixed.h>
 #include <drm/drm_probe_helper.h>
 
 #include "i915_drv.h"
@@ -43,6 +44,9 @@
 #include "intel_dpio_phy.h"
 #include "intel_hdcp.h"
 #include "intel_hotplug.h"
+#include "intel_link_bw.h"
+#include "intel_psr.h"
+#include "intel_vdsc.h"
 #include "skl_scaler.h"
 
 static int intel_dp_mst_check_constraints(struct drm_i915_private *i915, int bpp,
@@ -66,6 +70,73 @@ static int intel_dp_mst_check_constraints(struct drm_i915_private *i915, int bpp
 	return 0;
 }
 
+static int intel_dp_mst_bw_overhead(const struct intel_crtc_state *crtc_state,
+				    const struct intel_connector *connector,
+				    bool ssc, bool dsc, int bpp_x16)
+{
+	const struct drm_display_mode *adjusted_mode =
+		&crtc_state->hw.adjusted_mode;
+	unsigned long flags = DRM_DP_BW_OVERHEAD_MST;
+	int dsc_slice_count = 0;
+	int overhead;
+
+	flags |= intel_dp_is_uhbr(crtc_state) ? DRM_DP_BW_OVERHEAD_UHBR : 0;
+	flags |= ssc ? DRM_DP_BW_OVERHEAD_SSC_REF_CLK : 0;
+	flags |= crtc_state->fec_enable ? DRM_DP_BW_OVERHEAD_FEC : 0;
+
+	if (dsc) {
+		flags |= DRM_DP_BW_OVERHEAD_DSC;
+		/* TODO: add support for bigjoiner */
+		dsc_slice_count = intel_dp_dsc_get_slice_count(connector,
+							       adjusted_mode->clock,
+							       adjusted_mode->hdisplay,
+							       false);
+	}
+
+	overhead = drm_dp_bw_overhead(crtc_state->lane_count,
+				      adjusted_mode->hdisplay,
+				      dsc_slice_count,
+				      bpp_x16,
+				      flags);
+
+	/*
+	 * TODO: clarify whether a minimum required by the fixed FEC overhead
+	 * in the bspec audio programming sequence is required here.
+	 */
+	return max(overhead, intel_dp_bw_fec_overhead(crtc_state->fec_enable));
+}
+
+static void intel_dp_mst_compute_m_n(const struct intel_crtc_state *crtc_state,
+				     const struct intel_connector *connector,
+				     int overhead,
+				     int bpp_x16,
+				     struct intel_link_m_n *m_n)
+{
+	const struct drm_display_mode *adjusted_mode =
+		&crtc_state->hw.adjusted_mode;
+
+	/* TODO: Check WA 14013163432 to set data M/N for full BW utilization. */
+	intel_link_compute_m_n(bpp_x16, crtc_state->lane_count,
+			       adjusted_mode->crtc_clock,
+			       crtc_state->port_clock,
+			       overhead,
+			       m_n);
+
+	m_n->tu = DIV_ROUND_UP_ULL(mul_u32_u32(m_n->data_m, 64), m_n->data_n);
+}
+
+static int intel_dp_mst_calc_pbn(int pixel_clock, int bpp_x16, int bw_overhead)
+{
+	int effective_data_rate =
+		intel_dp_effective_data_rate(pixel_clock, bpp_x16, bw_overhead);
+
+	/*
+	 * TODO: Use drm_dp_calc_pbn_mode() instead, once it's converted
+	 * to calculate PBN with the BW overhead passed to it.
+	 */
+	return DIV_ROUND_UP(effective_data_rate * 64, 54 * 1000);
+}
+
 static int intel_dp_mst_find_vcpi_slots_for_bpp(struct intel_encoder *encoder,
 						struct intel_crtc_state *crtc_state,
 						int max_bpp,
@@ -94,20 +165,67 @@ static int intel_dp_mst_find_vcpi_slots_for_bpp(struct intel_encoder *encoder,
 	crtc_state->lane_count = limits->max_lane_count;
 	crtc_state->port_clock = limits->max_rate;
 
+	if (dsc) {
+		if (!intel_dp_supports_fec(intel_dp, connector, crtc_state))
+			return -EINVAL;
+
+		crtc_state->fec_enable = !intel_dp_is_uhbr(crtc_state);
+	}
+
 	mst_state->pbn_div = drm_dp_get_vc_payload_bw(&intel_dp->mst_mgr,
 						      crtc_state->port_clock,
 						      crtc_state->lane_count);
 
+	drm_dbg_kms(&i915->drm, "Looking for slots in range min bpp %d max bpp %d\n",
+		    min_bpp, max_bpp);
+
 	for (bpp = max_bpp; bpp >= min_bpp; bpp -= step) {
+		int local_bw_overhead;
+		int remote_bw_overhead;
+		int link_bpp_x16;
+		int remote_tu;
+
 		drm_dbg_kms(&i915->drm, "Trying bpp %d\n", bpp);
 
 		ret = intel_dp_mst_check_constraints(i915, bpp, adjusted_mode, crtc_state, dsc);
 		if (ret)
 			continue;
 
-		crtc_state->pbn = drm_dp_calc_pbn_mode(adjusted_mode->crtc_clock,
-						       dsc ? bpp << 4 : bpp,
-						       dsc);
+		link_bpp_x16 = to_bpp_x16(dsc ? bpp :
+					  intel_dp_output_bpp(crtc_state->output_format, bpp));
+
+		local_bw_overhead = intel_dp_mst_bw_overhead(crtc_state, connector,
+							     false, dsc, link_bpp_x16);
+		remote_bw_overhead = intel_dp_mst_bw_overhead(crtc_state, connector,
+							      true, dsc, link_bpp_x16);
+
+		intel_dp_mst_compute_m_n(crtc_state, connector,
+					 local_bw_overhead,
+					 link_bpp_x16,
+					 &crtc_state->dp_m_n);
+
+		/*
+		 * The TU size programmed to the HW determines which slots in
+		 * an MTP frame are used for this stream, which needs to match
+		 * the payload size programmed to the first downstream branch
+		 * device's payload table.
+		 *
+		 * Note that atm the payload's PBN value DRM core sends via
+		 * the ALLOCATE_PAYLOAD side-band message matches the payload
+		 * size (which it calculates from the PBN value) it programs
+		 * to the first branch device's payload table. The allocation
+		 * in the payload table could be reduced though (to
+		 * crtc_state->dp_m_n.tu), provided that the driver doesn't
+		 * enable SSC on the corresponding link.
+		 */
+		crtc_state->pbn = intel_dp_mst_calc_pbn(adjusted_mode->crtc_clock,
+							link_bpp_x16,
+							remote_bw_overhead);
+
+		remote_tu = DIV_ROUND_UP(dfixed_const(crtc_state->pbn), mst_state->pbn_div.full);
+
+		drm_WARN_ON(&i915->drm, remote_tu < crtc_state->dp_m_n.tu);
+		crtc_state->dp_m_n.tu = remote_tu;
 
 		slots = drm_dp_atomic_find_time_slots(state, &intel_dp->mst_mgr,
 						      connector->port,
@@ -116,13 +234,9 @@ static int intel_dp_mst_find_vcpi_slots_for_bpp(struct intel_encoder *encoder,
 			return slots;
 
 		if (slots >= 0) {
-			ret = drm_dp_mst_atomic_check(state);
-			/*
-			 * If we got slots >= 0 and we can fit those based on check
-			 * then we can exit the loop. Otherwise keep trying.
-			 */
-			if (!ret)
-				break;
+			drm_WARN_ON(&i915->drm, slots != crtc_state->dp_m_n.tu);
+
+			break;
 		}
 	}
 
@@ -137,7 +251,7 @@ static int intel_dp_mst_find_vcpi_slots_for_bpp(struct intel_encoder *encoder,
 		if (!dsc)
 			crtc_state->pipe_bpp = bpp;
 		else
-			crtc_state->dsc.compressed_bpp = bpp;
+			crtc_state->dsc.compressed_bpp_x16 = to_bpp_x16(bpp);
 		drm_dbg_kms(&i915->drm, "Got %d slots for pipe bpp %d dsc %d\n", slots, bpp, dsc);
 	}
 
@@ -149,10 +263,7 @@ static int intel_dp_mst_compute_link_config(struct intel_encoder *encoder,
 					    struct drm_connector_state *conn_state,
 					    struct link_config_limits *limits)
 {
-	const struct drm_display_mode *adjusted_mode =
-		&crtc_state->hw.adjusted_mode;
 	int slots = -EINVAL;
-	int link_bpp;
 
 	/*
 	 * FIXME: allocate the BW according to link_bpp, which in the case of
@@ -167,16 +278,6 @@ static int intel_dp_mst_compute_link_config(struct intel_encoder *encoder,
 	if (slots < 0)
 		return slots;
 
-	link_bpp = intel_dp_output_bpp(crtc_state->output_format, crtc_state->pipe_bpp);
-
-	intel_link_compute_m_n(link_bpp,
-			       crtc_state->lane_count,
-			       adjusted_mode->crtc_clock,
-			       crtc_state->port_clock,
-			       &crtc_state->dp_m_n,
-			       crtc_state->fec_enable);
-	crtc_state->dp_m_n.tu = slots;
-
 	return 0;
 }
 
@@ -188,15 +289,12 @@ static int intel_dp_dsc_mst_compute_link_config(struct intel_encoder *encoder,
 	struct intel_connector *connector =
 		to_intel_connector(conn_state->connector);
 	struct drm_i915_private *i915 = to_i915(connector->base.dev);
-	const struct drm_display_mode *adjusted_mode =
-		&crtc_state->hw.adjusted_mode;
 	int slots = -EINVAL;
 	int i, num_bpc;
 	u8 dsc_bpc[3] = {};
 	int min_bpp, max_bpp, sink_min_bpp, sink_max_bpp;
 	u8 dsc_max_bpc;
-	bool need_timeslot_recalc = false;
-	u32 last_compressed_bpp;
+	int min_compressed_bpp, max_compressed_bpp;
 
 	/* Max DSC Input BPC for ICL is 10 and for TGL+ is 12 */
 	if (DISPLAY_VER(i915) >= 12)
@@ -232,45 +330,31 @@ static int intel_dp_dsc_mst_compute_link_config(struct intel_encoder *encoder,
 	if (max_bpp > sink_max_bpp)
 		max_bpp = sink_max_bpp;
 
-	min_bpp = max(min_bpp, to_bpp_int_roundup(limits->link.min_bpp_x16));
-	max_bpp = min(max_bpp, to_bpp_int(limits->link.max_bpp_x16));
-
-	slots = intel_dp_mst_find_vcpi_slots_for_bpp(encoder, crtc_state, max_bpp,
-						     min_bpp, limits,
-						     conn_state, 2 * 3, true);
+	max_compressed_bpp = intel_dp_dsc_sink_max_compressed_bpp(connector,
+								  crtc_state,
+								  max_bpp / 3);
+	max_compressed_bpp = min(max_compressed_bpp,
+				 to_bpp_int(limits->link.max_bpp_x16));
 
-	if (slots < 0)
-		return slots;
+	min_compressed_bpp = intel_dp_dsc_sink_min_compressed_bpp(crtc_state);
+	min_compressed_bpp = max(min_compressed_bpp,
+				 to_bpp_int_roundup(limits->link.min_bpp_x16));
 
-	last_compressed_bpp = crtc_state->dsc.compressed_bpp;
+	drm_dbg_kms(&i915->drm, "DSC Sink supported compressed min bpp %d compressed max bpp %d\n",
+		    min_compressed_bpp, max_compressed_bpp);
 
-	crtc_state->dsc.compressed_bpp = intel_dp_dsc_nearest_valid_bpp(i915,
-									last_compressed_bpp,
-									crtc_state->pipe_bpp);
+	/* Align compressed bpps according to our own constraints */
+	max_compressed_bpp = intel_dp_dsc_nearest_valid_bpp(i915, max_compressed_bpp,
+							    crtc_state->pipe_bpp);
+	min_compressed_bpp = intel_dp_dsc_nearest_valid_bpp(i915, min_compressed_bpp,
+							    crtc_state->pipe_bpp);
 
-	if (crtc_state->dsc.compressed_bpp != last_compressed_bpp)
-		need_timeslot_recalc = true;
+	slots = intel_dp_mst_find_vcpi_slots_for_bpp(encoder, crtc_state, max_compressed_bpp,
+						     min_compressed_bpp, limits,
+						     conn_state, 1, true);
 
-	/*
-	 * Apparently some MST hubs dislike if vcpi slots are not matching precisely
-	 * the actual compressed bpp we use.
-	 */
-	if (need_timeslot_recalc) {
-		slots = intel_dp_mst_find_vcpi_slots_for_bpp(encoder, crtc_state,
-							     crtc_state->dsc.compressed_bpp,
-							     crtc_state->dsc.compressed_bpp,
-							     limits, conn_state, 2 * 3, true);
-		if (slots < 0)
-			return slots;
-	}
-
-	intel_link_compute_m_n(crtc_state->dsc.compressed_bpp,
-			       crtc_state->lane_count,
-			       adjusted_mode->crtc_clock,
-			       crtc_state->port_clock,
-			       &crtc_state->dp_m_n,
-			       crtc_state->fec_enable);
-	crtc_state->dp_m_n.tu = slots;
+	if (slots < 0)
+		return slots;
 
 	return 0;
 }
@@ -297,8 +381,103 @@ static int intel_dp_mst_update_slots(struct intel_encoder *encoder,
 	return 0;
 }
 
+static bool
+intel_dp_mst_dsc_source_support(const struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
+
+	/*
+	 * FIXME: Enabling DSC on ICL results in blank screen and FIFO pipe /
+	 * transcoder underruns, re-enable DSC after fixing this issue.
+	 */
+	return DISPLAY_VER(i915) >= 12 && intel_dsc_source_support(crtc_state);
+}
+
+static int mode_hblank_period_ns(const struct drm_display_mode *mode)
+{
+	return DIV_ROUND_CLOSEST_ULL(mul_u32_u32(mode->htotal - mode->hdisplay,
+						 NSEC_PER_SEC / 1000),
+				     mode->crtc_clock);
+}
+
+static bool
+hblank_expansion_quirk_needs_dsc(const struct intel_connector *connector,
+				 const struct intel_crtc_state *crtc_state)
+{
+	const struct drm_display_mode *adjusted_mode =
+		&crtc_state->hw.adjusted_mode;
+
+	if (!connector->dp.dsc_hblank_expansion_quirk)
+		return false;
+
+	if (mode_hblank_period_ns(adjusted_mode) > 300)
+		return false;
+
+	return true;
+}
+
+static bool
+adjust_limits_for_dsc_hblank_expansion_quirk(const struct intel_connector *connector,
+					     const struct intel_crtc_state *crtc_state,
+					     struct link_config_limits *limits,
+					     bool dsc)
+{
+	struct drm_i915_private *i915 = to_i915(connector->base.dev);
+	const struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	int min_bpp_x16 = limits->link.min_bpp_x16;
+
+	if (!hblank_expansion_quirk_needs_dsc(connector, crtc_state))
+		return true;
+
+	if (!dsc) {
+		if (intel_dp_mst_dsc_source_support(crtc_state)) {
+			drm_dbg_kms(&i915->drm,
+				    "[CRTC:%d:%s][CONNECTOR:%d:%s] DSC needed by hblank expansion quirk\n",
+				    crtc->base.base.id, crtc->base.name,
+				    connector->base.base.id, connector->base.name);
+			return false;
+		}
+
+		drm_dbg_kms(&i915->drm,
+			    "[CRTC:%d:%s][CONNECTOR:%d:%s] Increasing link min bpp to 24 due to hblank expansion quirk\n",
+			    crtc->base.base.id, crtc->base.name,
+			    connector->base.base.id, connector->base.name);
+
+		if (limits->link.max_bpp_x16 < to_bpp_x16(24))
+			return false;
+
+		limits->link.min_bpp_x16 = to_bpp_x16(24);
+
+		return true;
+	}
+
+	drm_WARN_ON(&i915->drm, limits->min_rate != limits->max_rate);
+
+	if (limits->max_rate < 540000)
+		min_bpp_x16 = to_bpp_x16(13);
+	else if (limits->max_rate < 810000)
+		min_bpp_x16 = to_bpp_x16(10);
+
+	if (limits->link.min_bpp_x16 >= min_bpp_x16)
+		return true;
+
+	drm_dbg_kms(&i915->drm,
+		    "[CRTC:%d:%s][CONNECTOR:%d:%s] Increasing link min bpp to " BPP_X16_FMT " in DSC mode due to hblank expansion quirk\n",
+		    crtc->base.base.id, crtc->base.name,
+		    connector->base.base.id, connector->base.name,
+		    BPP_X16_ARGS(min_bpp_x16));
+
+	if (limits->link.max_bpp_x16 < min_bpp_x16)
+		return false;
+
+	limits->link.min_bpp_x16 = min_bpp_x16;
+
+	return true;
+}
+
 static bool
 intel_dp_mst_compute_config_limits(struct intel_dp *intel_dp,
+				   const struct intel_connector *connector,
 				   struct intel_crtc_state *crtc_state,
 				   bool dsc,
 				   struct link_config_limits *limits)
@@ -326,10 +505,16 @@ intel_dp_mst_compute_config_limits(struct intel_dp *intel_dp,
 
 	intel_dp_adjust_compliance_config(intel_dp, crtc_state, limits);
 
-	return intel_dp_compute_config_link_bpp_limits(intel_dp,
-						       crtc_state,
-						       dsc,
-						       limits);
+	if (!intel_dp_compute_config_link_bpp_limits(intel_dp,
+						     crtc_state,
+						     dsc,
+						     limits))
+		return false;
+
+	return adjust_limits_for_dsc_hblank_expansion_quirk(connector,
+							    crtc_state,
+							    limits,
+							    dsc);
 }
 
 static int intel_dp_mst_compute_config(struct intel_encoder *encoder,
@@ -339,12 +524,18 @@ static int intel_dp_mst_compute_config(struct intel_encoder *encoder,
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder);
 	struct intel_dp *intel_dp = &intel_mst->primary->dp;
+	const struct intel_connector *connector =
+		to_intel_connector(conn_state->connector);
 	const struct drm_display_mode *adjusted_mode =
 		&pipe_config->hw.adjusted_mode;
 	struct link_config_limits limits;
 	bool dsc_needed;
 	int ret = 0;
 
+	if (pipe_config->fec_enable &&
+	    !intel_dp_supports_fec(intel_dp, connector, pipe_config))
+		return -EINVAL;
+
 	if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN)
 		return -EINVAL;
 
@@ -354,6 +545,7 @@ static int intel_dp_mst_compute_config(struct intel_encoder *encoder,
 
 	dsc_needed = intel_dp->force_dsc_en ||
 		     !intel_dp_mst_compute_config_limits(intel_dp,
+							 connector,
 							 pipe_config,
 							 false,
 							 &limits);
@@ -375,7 +567,11 @@ static int intel_dp_mst_compute_config(struct intel_encoder *encoder,
 			    str_yes_no(ret),
 			    str_yes_no(intel_dp->force_dsc_en));
 
+		if (!intel_dp_mst_dsc_source_support(pipe_config))
+			return -EINVAL;
+
 		if (!intel_dp_mst_compute_config_limits(intel_dp,
+							connector,
 							pipe_config,
 							true,
 							&limits))
@@ -420,6 +616,8 @@ static int intel_dp_mst_compute_config(struct intel_encoder *encoder,
 
 	intel_ddi_compute_min_voltage_level(dev_priv, pipe_config);
 
+	intel_psr_compute_config(intel_dp, pipe_config, conn_state);
+
 	return 0;
 }
 
@@ -459,6 +657,130 @@ intel_dp_mst_transcoder_mask(struct intel_atomic_state *state,
 	return transcoders;
 }
 
+static u8 get_pipes_downstream_of_mst_port(struct intel_atomic_state *state,
+					   struct drm_dp_mst_topology_mgr *mst_mgr,
+					   struct drm_dp_mst_port *parent_port)
+{
+	const struct intel_digital_connector_state *conn_state;
+	struct intel_connector *connector;
+	u8 mask = 0;
+	int i;
+
+	for_each_new_intel_connector_in_state(state, connector, conn_state, i) {
+		if (!conn_state->base.crtc)
+			continue;
+
+		if (&connector->mst_port->mst_mgr != mst_mgr)
+			continue;
+
+		if (connector->port != parent_port &&
+		    !drm_dp_mst_port_downstream_of_parent(mst_mgr,
+							  connector->port,
+							  parent_port))
+			continue;
+
+		mask |= BIT(to_intel_crtc(conn_state->base.crtc)->pipe);
+	}
+
+	return mask;
+}
+
+static int intel_dp_mst_check_fec_change(struct intel_atomic_state *state,
+					 struct drm_dp_mst_topology_mgr *mst_mgr,
+					 struct intel_link_bw_limits *limits)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	struct intel_crtc *crtc;
+	u8 mst_pipe_mask;
+	u8 fec_pipe_mask = 0;
+	int ret;
+
+	mst_pipe_mask = get_pipes_downstream_of_mst_port(state, mst_mgr, NULL);
+
+	for_each_intel_crtc_in_pipe_mask(&i915->drm, crtc, mst_pipe_mask) {
+		struct intel_crtc_state *crtc_state =
+			intel_atomic_get_new_crtc_state(state, crtc);
+
+		/* Atomic connector check should've added all the MST CRTCs. */
+		if (drm_WARN_ON(&i915->drm, !crtc_state))
+			return -EINVAL;
+
+		if (crtc_state->fec_enable)
+			fec_pipe_mask |= BIT(crtc->pipe);
+	}
+
+	if (!fec_pipe_mask || mst_pipe_mask == fec_pipe_mask)
+		return 0;
+
+	limits->force_fec_pipes |= mst_pipe_mask;
+
+	ret = intel_modeset_pipes_in_mask_early(state, "MST FEC",
+						mst_pipe_mask);
+
+	return ret ? : -EAGAIN;
+}
+
+static int intel_dp_mst_check_bw(struct intel_atomic_state *state,
+				 struct drm_dp_mst_topology_mgr *mst_mgr,
+				 struct drm_dp_mst_topology_state *mst_state,
+				 struct intel_link_bw_limits *limits)
+{
+	struct drm_dp_mst_port *mst_port;
+	u8 mst_port_pipes;
+	int ret;
+
+	ret = drm_dp_mst_atomic_check_mgr(&state->base, mst_mgr, mst_state, &mst_port);
+	if (ret != -ENOSPC)
+		return ret;
+
+	mst_port_pipes = get_pipes_downstream_of_mst_port(state, mst_mgr, mst_port);
+
+	ret = intel_link_bw_reduce_bpp(state, limits,
+				       mst_port_pipes, "MST link BW");
+
+	return ret ? : -EAGAIN;
+}
+
+/**
+ * intel_dp_mst_atomic_check_link - check all modeset MST link configuration
+ * @state: intel atomic state
+ * @limits: link BW limits
+ *
+ * Check the link configuration for all modeset MST outputs. If the
+ * configuration is invalid @limits will be updated if possible to
+ * reduce the total BW, after which the configuration for all CRTCs in
+ * @state must be recomputed with the updated @limits.
+ *
+ * Returns:
+ *   - 0 if the confugration is valid
+ *   - %-EAGAIN, if the configuration is invalid and @limits got updated
+ *     with fallback values with which the configuration of all CRTCs in
+ *     @state must be recomputed
+ *   - Other negative error, if the configuration is invalid without a
+ *     fallback possibility, or the check failed for another reason
+ */
+int intel_dp_mst_atomic_check_link(struct intel_atomic_state *state,
+				   struct intel_link_bw_limits *limits)
+{
+	struct drm_dp_mst_topology_mgr *mgr;
+	struct drm_dp_mst_topology_state *mst_state;
+	int ret;
+	int i;
+
+	for_each_new_mst_mgr_in_state(&state->base, mgr, mst_state, i) {
+		ret = intel_dp_mst_check_fec_change(state, mgr, limits);
+		if (ret)
+			return ret;
+
+		ret = intel_dp_mst_check_bw(state, mgr, mst_state,
+					    limits);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
 static int intel_dp_mst_compute_config_late(struct intel_encoder *encoder,
 					    struct intel_crtc_state *crtc_state,
 					    struct drm_connector_state *conn_state)
@@ -479,19 +801,23 @@ static int intel_dp_mst_compute_config_late(struct intel_encoder *encoder,
  * that shares the same MST stream as mode changed,
  * intel_modeset_pipe_config()+intel_crtc_check_fastset() will take care to do
  * a fastset when possible.
+ *
+ * On TGL+ this is required since each stream go through a master transcoder,
+ * so if the master transcoder needs modeset, all other streams in the
+ * topology need a modeset. All platforms need to add the atomic state
+ * for all streams in the topology, since a modeset on one may require
+ * changing the MST link BW usage of the others, which in turn needs a
+ * recomputation of the corresponding CRTC states.
  */
 static int
-intel_dp_mst_atomic_master_trans_check(struct intel_connector *connector,
-				       struct intel_atomic_state *state)
+intel_dp_mst_atomic_topology_check(struct intel_connector *connector,
+				   struct intel_atomic_state *state)
 {
 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 	struct drm_connector_list_iter connector_list_iter;
 	struct intel_connector *connector_iter;
 	int ret = 0;
 
-	if (DISPLAY_VER(dev_priv) < 12)
-		return  0;
-
 	if (!intel_connector_needs_modeset(state, &connector->base))
 		return 0;
 
@@ -545,7 +871,7 @@ intel_dp_mst_atomic_check(struct drm_connector *connector,
 	if (ret)
 		return ret;
 
-	ret = intel_dp_mst_atomic_master_trans_check(intel_connector, state);
+	ret = intel_dp_mst_atomic_topology_check(intel_connector, state);
 	if (ret)
 		return ret;
 
@@ -587,10 +913,6 @@ static void intel_mst_disable_dp(struct intel_atomic_state *state,
 	struct intel_dp *intel_dp = &dig_port->dp;
 	struct intel_connector *connector =
 		to_intel_connector(old_conn_state->connector);
-	struct drm_dp_mst_topology_state *new_mst_state =
-		drm_atomic_get_new_mst_topology_state(&state->base, &intel_dp->mst_mgr);
-	struct drm_dp_mst_atomic_payload *new_payload =
-		drm_atomic_get_mst_payload_state(new_mst_state, connector->port);
 	struct drm_i915_private *i915 = to_i915(connector->base.dev);
 
 	drm_dbg_kms(&i915->drm, "active links %d\n",
@@ -598,9 +920,9 @@ static void intel_mst_disable_dp(struct intel_atomic_state *state,
 
 	intel_hdcp_disable(intel_mst->connector);
 
-	drm_dp_remove_payload_part1(&intel_dp->mst_mgr, new_mst_state, new_payload);
-
 	intel_audio_codec_disable(encoder, old_crtc_state, old_conn_state);
+
+	intel_dp_sink_disable_decompression(state, connector, old_crtc_state);
 }
 
 static void intel_mst_post_disable_dp(struct intel_atomic_state *state,
@@ -634,6 +956,8 @@ static void intel_mst_post_disable_dp(struct intel_atomic_state *state,
 
 	intel_disable_transcoder(old_crtc_state);
 
+	drm_dp_remove_payload_part1(&intel_dp->mst_mgr, new_mst_state, new_payload);
+
 	clear_act_sent(encoder, old_crtc_state);
 
 	intel_de_rmw(dev_priv, TRANS_DDI_FUNC_CTL(old_crtc_state->cpu_transcoder),
@@ -646,6 +970,8 @@ static void intel_mst_post_disable_dp(struct intel_atomic_state *state,
 
 	intel_ddi_disable_transcoder_func(old_crtc_state);
 
+	intel_dsc_disable(old_crtc_state);
+
 	if (DISPLAY_VER(dev_priv) >= 9)
 		skl_scaler_disable(old_crtc_state);
 	else
@@ -662,9 +988,8 @@ static void intel_mst_post_disable_dp(struct intel_atomic_state *state,
 	 * BSpec 4287: disable DIP after the transcoder is disabled and before
 	 * the transcoder clock select is set to none.
 	 */
-	if (last_mst_stream)
-		intel_dp_set_infoframes(&dig_port->base, false,
-					old_crtc_state, NULL);
+	intel_dp_set_infoframes(&dig_port->base, false,
+				old_crtc_state, NULL);
 	/*
 	 * From TGL spec: "If multi-stream slave transcoder: Configure
 	 * Transcoder Clock Select to direct no clock to the transcoder"
@@ -754,6 +1079,8 @@ static void intel_mst_pre_enable_dp(struct intel_atomic_state *state,
 
 	drm_dp_send_power_updown_phy(&intel_dp->mst_mgr, connector->port, true);
 
+	intel_dp_sink_enable_decompression(state, connector, pipe_config);
+
 	if (first_mst_stream)
 		dig_port->base.pre_enable(state, &dig_port->base,
 						pipe_config, NULL);
@@ -776,6 +1103,7 @@ static void intel_mst_pre_enable_dp(struct intel_atomic_state *state,
 	if (DISPLAY_VER(dev_priv) < 12 || !first_mst_stream)
 		intel_ddi_enable_transcoder_clock(encoder, pipe_config);
 
+	intel_dsc_dp_pps_write(&dig_port->base, pipe_config);
 	intel_ddi_set_dp_msa(pipe_config, conn_state);
 }
 
@@ -792,11 +1120,10 @@ static void intel_mst_enable_dp(struct intel_atomic_state *state,
 	struct drm_dp_mst_topology_state *mst_state =
 		drm_atomic_get_new_mst_topology_state(&state->base, &intel_dp->mst_mgr);
 	enum transcoder trans = pipe_config->cpu_transcoder;
+	bool first_mst_stream = intel_dp->active_mst_links == 1;
 
 	drm_WARN_ON(&dev_priv->drm, pipe_config->has_pch_encoder);
 
-	clear_act_sent(encoder, pipe_config);
-
 	if (intel_dp_is_uhbr(pipe_config)) {
 		const struct drm_display_mode *adjusted_mode =
 			&pipe_config->hw.adjusted_mode;
@@ -810,6 +1137,8 @@ static void intel_mst_enable_dp(struct intel_atomic_state *state,
 
 	intel_ddi_enable_transcoder_func(encoder, pipe_config);
 
+	clear_act_sent(encoder, pipe_config);
+
 	intel_de_rmw(dev_priv, TRANS_DDI_FUNC_CTL(trans), 0,
 		     TRANS_DDI_DP_VC_PAYLOAD_ALLOC);
 
@@ -818,15 +1147,16 @@ static void intel_mst_enable_dp(struct intel_atomic_state *state,
 
 	wait_for_act_sent(encoder, pipe_config);
 
+	if (first_mst_stream)
+		intel_ddi_wait_for_fec_status(encoder, pipe_config, true);
+
 	drm_dp_add_payload_part2(&intel_dp->mst_mgr, &state->base,
 				 drm_atomic_get_mst_payload_state(mst_state, connector->port));
 
-	if (DISPLAY_VER(dev_priv) >= 14 && pipe_config->fec_enable)
-		intel_de_rmw(dev_priv, MTL_CHICKEN_TRANS(trans), 0,
-			     FECSTALL_DIS_DPTSTREAM_DPTTG);
-	else if (DISPLAY_VER(dev_priv) >= 12 && pipe_config->fec_enable)
-		intel_de_rmw(dev_priv, CHICKEN_TRANS(trans), 0,
-			     FECSTALL_DIS_DPTSTREAM_DPTTG);
+	if (DISPLAY_VER(dev_priv) >= 12)
+		intel_de_rmw(dev_priv, hsw_chicken_trans_reg(dev_priv, trans),
+			     FECSTALL_DIS_DPTSTREAM_DPTTG,
+			     pipe_config->fec_enable ? FECSTALL_DIS_DPTSTREAM_DPTTG : 0);
 
 	intel_audio_sdp_split_update(pipe_config);
 
@@ -836,10 +1166,7 @@ static void intel_mst_enable_dp(struct intel_atomic_state *state,
 
 	intel_audio_codec_enable(encoder, pipe_config, conn_state);
 
-	/* Enable hdcp if it's desired */
-	if (conn_state->content_protection ==
-	    DRM_MODE_CONTENT_PROTECTION_DESIRED)
-		intel_hdcp_enable(state, encoder, pipe_config, conn_state);
+	intel_hdcp_enable(state, encoder, pipe_config, conn_state);
 }
 
 static bool intel_dp_mst_enc_get_hw_state(struct intel_encoder *encoder,
@@ -974,8 +1301,20 @@ intel_dp_mst_mode_valid_ctx(struct drm_connector *connector,
 	if (ret)
 		return ret;
 
+	/*
+	 * TODO:
+	 * - Also check if compression would allow for the mode
+	 * - Calculate the overhead using drm_dp_bw_overhead() /
+	 *   drm_dp_bw_channel_coding_efficiency(), similarly to the
+	 *   compute config code, as drm_dp_calc_pbn_mode() doesn't
+	 *   account with all the overheads.
+	 * - Check here and during compute config the BW reported by
+	 *   DFP_Link_Available_Payload_Bandwidth_Number (or the
+	 *   corresponding link capabilities of the sink) in case the
+	 *   stream is uncompressed for it by the last branch device.
+	 */
 	if (mode_rate > max_rate || mode->clock > max_dotclk ||
-	    drm_dp_calc_pbn_mode(mode->clock, min_bpp, false) > port->full_pbn) {
+	    drm_dp_calc_pbn_mode(mode->clock, min_bpp << 4) > port->full_pbn) {
 		*status = MODE_CLOCK_HIGH;
 		return 0;
 	}
@@ -1139,6 +1478,36 @@ intel_dp_mst_read_decompression_port_dsc_caps(struct intel_dp *intel_dp,
 	intel_dp_get_dsc_sink_cap(dpcd_caps[DP_DPCD_REV], connector);
 }
 
+static bool detect_dsc_hblank_expansion_quirk(const struct intel_connector *connector)
+{
+	struct drm_i915_private *i915 = to_i915(connector->base.dev);
+	struct drm_dp_desc desc;
+	u8 dpcd[DP_RECEIVER_CAP_SIZE];
+
+	if (!connector->dp.dsc_decompression_aux)
+		return false;
+
+	if (drm_dp_read_desc(connector->dp.dsc_decompression_aux,
+			     &desc, true) < 0)
+		return false;
+
+	if (!drm_dp_has_quirk(&desc,
+			      DP_DPCD_QUIRK_HBLANK_EXPANSION_REQUIRES_DSC))
+		return false;
+
+	if (drm_dp_read_dpcd_caps(connector->dp.dsc_decompression_aux, dpcd) < 0)
+		return false;
+
+	if (!(dpcd[DP_RECEIVE_PORT_0_CAP_0] & DP_HBLANK_EXPANSION_CAPABLE))
+		return false;
+
+	drm_dbg_kms(&i915->drm,
+		    "[CONNECTOR:%d:%s] DSC HBLANK expansion quirk detected\n",
+		    connector->base.base.id, connector->base.name);
+
+	return true;
+}
+
 static struct drm_connector *intel_dp_add_mst_connector(struct drm_dp_mst_topology_mgr *mgr,
 							struct drm_dp_mst_port *port,
 							const char *pathprop)
@@ -1161,6 +1530,11 @@ static struct drm_connector *intel_dp_add_mst_connector(struct drm_dp_mst_topolo
 	intel_connector->port = port;
 	drm_dp_mst_get_port_malloc(port);
 
+	intel_connector->dp.dsc_decompression_aux = drm_dp_mst_dsc_aux_for_port(port);
+	intel_dp_mst_read_decompression_port_dsc_caps(intel_dp, intel_connector);
+	intel_connector->dp.dsc_hblank_expansion_quirk =
+		detect_dsc_hblank_expansion_quirk(intel_connector);
+
 	connector = &intel_connector->base;
 	ret = drm_connector_init(dev, connector, &intel_dp_mst_connector_funcs,
 				 DRM_MODE_CONNECTOR_DisplayPort);
@@ -1172,14 +1546,6 @@ static struct drm_connector *intel_dp_add_mst_connector(struct drm_dp_mst_topolo
 
 	drm_connector_helper_add(connector, &intel_dp_mst_connector_helper_funcs);
 
-	/*
-	 * TODO: set the AUX for the actual MST port decompressing the stream.
-	 * At the moment the driver only supports enabling this globally in the
-	 * first downstream MST branch, via intel_dp's (root port) AUX.
-	 */
-	intel_connector->dp.dsc_decompression_aux = &intel_dp->aux;
-	intel_dp_mst_read_decompression_port_dsc_caps(intel_dp, intel_connector);
-
 	for_each_pipe(dev_priv, pipe) {
 		struct drm_encoder *enc =
 			&intel_dp->mst_encoders[pipe]->base.base;
@@ -1407,3 +1773,91 @@ int intel_dp_mst_add_topology_state_for_crtc(struct intel_atomic_state *state,
 
 	return 0;
 }
+
+static struct intel_connector *
+get_connector_in_state_for_crtc(struct intel_atomic_state *state,
+				const struct intel_crtc *crtc)
+{
+	struct drm_connector_state *old_conn_state;
+	struct drm_connector_state *new_conn_state;
+	struct drm_connector *_connector;
+	int i;
+
+	for_each_oldnew_connector_in_state(&state->base, _connector,
+					   old_conn_state, new_conn_state, i) {
+		struct intel_connector *connector =
+			to_intel_connector(_connector);
+
+		if (old_conn_state->crtc == &crtc->base ||
+		    new_conn_state->crtc == &crtc->base)
+			return connector;
+	}
+
+	return NULL;
+}
+
+/**
+ * intel_dp_mst_crtc_needs_modeset - check if changes in topology need to modeset the given CRTC
+ * @state: atomic state
+ * @crtc: CRTC for which to check the modeset requirement
+ *
+ * Check if any change in a MST topology requires a forced modeset on @crtc in
+ * this topology. One such change is enabling/disabling the DSC decompression
+ * state in the first branch device's UFP DPCD as required by one CRTC, while
+ * the other @crtc in the same topology is still active, requiring a full modeset
+ * on @crtc.
+ */
+bool intel_dp_mst_crtc_needs_modeset(struct intel_atomic_state *state,
+				     struct intel_crtc *crtc)
+{
+	const struct intel_connector *crtc_connector;
+	const struct drm_connector_state *conn_state;
+	const struct drm_connector *_connector;
+	int i;
+
+	if (!intel_crtc_has_type(intel_atomic_get_new_crtc_state(state, crtc),
+				 INTEL_OUTPUT_DP_MST))
+		return false;
+
+	crtc_connector = get_connector_in_state_for_crtc(state, crtc);
+
+	if (!crtc_connector)
+		/* None of the connectors in the topology needs modeset */
+		return false;
+
+	for_each_new_connector_in_state(&state->base, _connector, conn_state, i) {
+		const struct intel_connector *connector =
+			to_intel_connector(_connector);
+		const struct intel_crtc_state *new_crtc_state;
+		const struct intel_crtc_state *old_crtc_state;
+		struct intel_crtc *crtc_iter;
+
+		if (connector->mst_port != crtc_connector->mst_port ||
+		    !conn_state->crtc)
+			continue;
+
+		crtc_iter = to_intel_crtc(conn_state->crtc);
+
+		new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc_iter);
+		old_crtc_state = intel_atomic_get_old_crtc_state(state, crtc_iter);
+
+		if (!intel_crtc_needs_modeset(new_crtc_state))
+			continue;
+
+		if (old_crtc_state->dsc.compression_enable ==
+		    new_crtc_state->dsc.compression_enable)
+			continue;
+		/*
+		 * Toggling the decompression flag because of this stream in
+		 * the first downstream branch device's UFP DPCD may reset the
+		 * whole branch device. To avoid the reset while other streams
+		 * are also active modeset the whole MST topology in this
+		 * case.
+		 */
+		if (connector->dp.dsc_decompression_aux ==
+		    &connector->mst_port->aux)
+			return true;
+	}
+
+	return false;
+}
diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.h b/drivers/gpu/drm/i915/display/intel_dp_mst.h
index f1815bb72267..8ca1d599091c 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_mst.h
+++ b/drivers/gpu/drm/i915/display/intel_dp_mst.h
@@ -13,6 +13,7 @@ struct intel_crtc;
 struct intel_crtc_state;
 struct intel_digital_port;
 struct intel_dp;
+struct intel_link_bw_limits;
 
 int intel_dp_mst_encoder_init(struct intel_digital_port *dig_port, int conn_id);
 void intel_dp_mst_encoder_cleanup(struct intel_digital_port *dig_port);
@@ -22,5 +23,9 @@ bool intel_dp_mst_is_slave_trans(const struct intel_crtc_state *crtc_state);
 bool intel_dp_mst_source_support(struct intel_dp *intel_dp);
 int intel_dp_mst_add_topology_state_for_crtc(struct intel_atomic_state *state,
 					     struct intel_crtc *crtc);
+int intel_dp_mst_atomic_check_link(struct intel_atomic_state *state,
+				   struct intel_link_bw_limits *limits);
+bool intel_dp_mst_crtc_needs_modeset(struct intel_atomic_state *state,
+				     struct intel_crtc *crtc);
 
 #endif /* __INTEL_DP_MST_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_dpio_phy.c b/drivers/gpu/drm/i915/display/intel_dpio_phy.c
index 62b93d097e44..4ca910874a4f 100644
--- a/drivers/gpu/drm/i915/display/intel_dpio_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_dpio_phy.c
@@ -666,6 +666,20 @@ enum dpio_phy vlv_dig_port_to_phy(struct intel_digital_port *dig_port)
 	}
 }
 
+enum dpio_phy vlv_pipe_to_phy(enum pipe pipe)
+{
+	switch (pipe) {
+	default:
+		MISSING_CASE(pipe);
+		fallthrough;
+	case PIPE_A:
+	case PIPE_B:
+		return DPIO_PHY0;
+	case PIPE_C:
+		return DPIO_PHY1;
+	}
+}
+
 enum dpio_channel vlv_pipe_to_channel(enum pipe pipe)
 {
 	switch (pipe) {
@@ -689,50 +703,50 @@ void chv_set_phy_signal_level(struct intel_encoder *encoder,
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	enum dpio_channel ch = vlv_dig_port_to_channel(dig_port);
-	enum pipe pipe = crtc->pipe;
+	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
 	u32 val;
 	int i;
 
 	vlv_dpio_get(dev_priv);
 
 	/* Clear calc init */
-	val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch));
+	val = vlv_dpio_read(dev_priv, phy, VLV_PCS01_DW10(ch));
 	val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3);
 	val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK);
 	val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5;
-	vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS01_DW10(ch), val);
 
 	if (crtc_state->lane_count > 2) {
-		val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch));
+		val = vlv_dpio_read(dev_priv, phy, VLV_PCS23_DW10(ch));
 		val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3);
 		val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK);
 		val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5;
-		vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val);
+		vlv_dpio_write(dev_priv, phy, VLV_PCS23_DW10(ch), val);
 	}
 
-	val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW9(ch));
+	val = vlv_dpio_read(dev_priv, phy, VLV_PCS01_DW9(ch));
 	val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK);
 	val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000;
-	vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW9(ch), val);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS01_DW9(ch), val);
 
 	if (crtc_state->lane_count > 2) {
-		val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW9(ch));
+		val = vlv_dpio_read(dev_priv, phy, VLV_PCS23_DW9(ch));
 		val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK);
 		val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000;
-		vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW9(ch), val);
+		vlv_dpio_write(dev_priv, phy, VLV_PCS23_DW9(ch), val);
 	}
 
 	/* Program swing deemph */
 	for (i = 0; i < crtc_state->lane_count; i++) {
-		val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW4(ch, i));
+		val = vlv_dpio_read(dev_priv, phy, CHV_TX_DW4(ch, i));
 		val &= ~DPIO_SWING_DEEMPH9P5_MASK;
 		val |= deemph_reg_value << DPIO_SWING_DEEMPH9P5_SHIFT;
-		vlv_dpio_write(dev_priv, pipe, CHV_TX_DW4(ch, i), val);
+		vlv_dpio_write(dev_priv, phy, CHV_TX_DW4(ch, i), val);
 	}
 
 	/* Program swing margin */
 	for (i = 0; i < crtc_state->lane_count; i++) {
-		val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i));
+		val = vlv_dpio_read(dev_priv, phy, CHV_TX_DW2(ch, i));
 
 		val &= ~DPIO_SWING_MARGIN000_MASK;
 		val |= margin_reg_value << DPIO_SWING_MARGIN000_SHIFT;
@@ -745,7 +759,7 @@ void chv_set_phy_signal_level(struct intel_encoder *encoder,
 		val &= ~(0xff << DPIO_UNIQ_TRANS_SCALE_SHIFT);
 		val |= 0x9a << DPIO_UNIQ_TRANS_SCALE_SHIFT;
 
-		vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val);
+		vlv_dpio_write(dev_priv, phy, CHV_TX_DW2(ch, i), val);
 	}
 
 	/*
@@ -755,23 +769,23 @@ void chv_set_phy_signal_level(struct intel_encoder *encoder,
 	 * 27 for ch0 and ch1.
 	 */
 	for (i = 0; i < crtc_state->lane_count; i++) {
-		val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW3(ch, i));
+		val = vlv_dpio_read(dev_priv, phy, CHV_TX_DW3(ch, i));
 		if (uniq_trans_scale)
 			val |= DPIO_TX_UNIQ_TRANS_SCALE_EN;
 		else
 			val &= ~DPIO_TX_UNIQ_TRANS_SCALE_EN;
-		vlv_dpio_write(dev_priv, pipe, CHV_TX_DW3(ch, i), val);
+		vlv_dpio_write(dev_priv, phy, CHV_TX_DW3(ch, i), val);
 	}
 
 	/* Start swing calculation */
-	val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch));
+	val = vlv_dpio_read(dev_priv, phy, VLV_PCS01_DW10(ch));
 	val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3;
-	vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS01_DW10(ch), val);
 
 	if (crtc_state->lane_count > 2) {
-		val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch));
+		val = vlv_dpio_read(dev_priv, phy, VLV_PCS23_DW10(ch));
 		val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3;
-		vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val);
+		vlv_dpio_write(dev_priv, phy, VLV_PCS23_DW10(ch), val);
 	}
 
 	vlv_dpio_put(dev_priv);
@@ -782,43 +796,43 @@ void chv_data_lane_soft_reset(struct intel_encoder *encoder,
 			      bool reset)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	enum dpio_channel ch = vlv_dig_port_to_channel(enc_to_dig_port(encoder));
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	enum pipe pipe = crtc->pipe;
+	enum dpio_channel ch = vlv_dig_port_to_channel(enc_to_dig_port(encoder));
+	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
 	u32 val;
 
-	val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch));
+	val = vlv_dpio_read(dev_priv, phy, VLV_PCS01_DW0(ch));
 	if (reset)
 		val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
 	else
 		val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET;
-	vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS01_DW0(ch), val);
 
 	if (crtc_state->lane_count > 2) {
-		val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch));
+		val = vlv_dpio_read(dev_priv, phy, VLV_PCS23_DW0(ch));
 		if (reset)
 			val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
 		else
 			val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET;
-		vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val);
+		vlv_dpio_write(dev_priv, phy, VLV_PCS23_DW0(ch), val);
 	}
 
-	val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch));
+	val = vlv_dpio_read(dev_priv, phy, VLV_PCS01_DW1(ch));
 	val |= CHV_PCS_REQ_SOFTRESET_EN;
 	if (reset)
 		val &= ~DPIO_PCS_CLK_SOFT_RESET;
 	else
 		val |= DPIO_PCS_CLK_SOFT_RESET;
-	vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS01_DW1(ch), val);
 
 	if (crtc_state->lane_count > 2) {
-		val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch));
+		val = vlv_dpio_read(dev_priv, phy, VLV_PCS23_DW1(ch));
 		val |= CHV_PCS_REQ_SOFTRESET_EN;
 		if (reset)
 			val &= ~DPIO_PCS_CLK_SOFT_RESET;
 		else
 			val |= DPIO_PCS_CLK_SOFT_RESET;
-		vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val);
+		vlv_dpio_write(dev_priv, phy, VLV_PCS23_DW1(ch), val);
 	}
 }
 
@@ -829,6 +843,7 @@ void chv_phy_pre_pll_enable(struct intel_encoder *encoder,
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	enum dpio_channel ch = vlv_dig_port_to_channel(dig_port);
+	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
 	enum pipe pipe = crtc->pipe;
 	unsigned int lane_mask =
 		intel_dp_unused_lane_mask(crtc_state->lane_count);
@@ -851,40 +866,40 @@ void chv_phy_pre_pll_enable(struct intel_encoder *encoder,
 
 	/* program left/right clock distribution */
 	if (pipe != PIPE_B) {
-		val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0);
+		val = vlv_dpio_read(dev_priv, phy, _CHV_CMN_DW5_CH0);
 		val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK);
 		if (ch == DPIO_CH0)
 			val |= CHV_BUFLEFTENA1_FORCE;
 		if (ch == DPIO_CH1)
 			val |= CHV_BUFRIGHTENA1_FORCE;
-		vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val);
+		vlv_dpio_write(dev_priv, phy, _CHV_CMN_DW5_CH0, val);
 	} else {
-		val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1);
+		val = vlv_dpio_read(dev_priv, phy, _CHV_CMN_DW1_CH1);
 		val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK);
 		if (ch == DPIO_CH0)
 			val |= CHV_BUFLEFTENA2_FORCE;
 		if (ch == DPIO_CH1)
 			val |= CHV_BUFRIGHTENA2_FORCE;
-		vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val);
+		vlv_dpio_write(dev_priv, phy, _CHV_CMN_DW1_CH1, val);
 	}
 
 	/* program clock channel usage */
-	val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW8(ch));
+	val = vlv_dpio_read(dev_priv, phy, VLV_PCS01_DW8(ch));
 	val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE;
 	if (pipe != PIPE_B)
 		val &= ~CHV_PCS_USEDCLKCHANNEL;
 	else
 		val |= CHV_PCS_USEDCLKCHANNEL;
-	vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW8(ch), val);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS01_DW8(ch), val);
 
 	if (crtc_state->lane_count > 2) {
-		val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW8(ch));
+		val = vlv_dpio_read(dev_priv, phy, VLV_PCS23_DW8(ch));
 		val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE;
 		if (pipe != PIPE_B)
 			val &= ~CHV_PCS_USEDCLKCHANNEL;
 		else
 			val |= CHV_PCS_USEDCLKCHANNEL;
-		vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW8(ch), val);
+		vlv_dpio_write(dev_priv, phy, VLV_PCS23_DW8(ch), val);
 	}
 
 	/*
@@ -892,12 +907,12 @@ void chv_phy_pre_pll_enable(struct intel_encoder *encoder,
 	 * matches the pipe, but here we need to
 	 * pick the CL based on the port.
 	 */
-	val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW19(ch));
+	val = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW19(ch));
 	if (pipe != PIPE_B)
 		val &= ~CHV_CMN_USEDCLKCHANNEL;
 	else
 		val |= CHV_CMN_USEDCLKCHANNEL;
-	vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW19(ch), val);
+	vlv_dpio_write(dev_priv, phy, CHV_CMN_DW19(ch), val);
 
 	vlv_dpio_put(dev_priv);
 }
@@ -910,21 +925,21 @@ void chv_phy_pre_encoder_enable(struct intel_encoder *encoder,
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	enum dpio_channel ch = vlv_dig_port_to_channel(dig_port);
-	enum pipe pipe = crtc->pipe;
+	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
 	int data, i, stagger;
 	u32 val;
 
 	vlv_dpio_get(dev_priv);
 
 	/* allow hardware to manage TX FIFO reset source */
-	val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW11(ch));
+	val = vlv_dpio_read(dev_priv, phy, VLV_PCS01_DW11(ch));
 	val &= ~DPIO_LANEDESKEW_STRAP_OVRD;
-	vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS01_DW11(ch), val);
 
 	if (crtc_state->lane_count > 2) {
-		val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch));
+		val = vlv_dpio_read(dev_priv, phy, VLV_PCS23_DW11(ch));
 		val &= ~DPIO_LANEDESKEW_STRAP_OVRD;
-		vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val);
+		vlv_dpio_write(dev_priv, phy, VLV_PCS23_DW11(ch), val);
 	}
 
 	/* Program Tx lane latency optimal setting*/
@@ -934,7 +949,7 @@ void chv_phy_pre_encoder_enable(struct intel_encoder *encoder,
 			data = 0x0;
 		else
 			data = (i == 1) ? 0x0 : 0x1;
-		vlv_dpio_write(dev_priv, pipe, CHV_TX_DW14(ch, i),
+		vlv_dpio_write(dev_priv, phy, CHV_TX_DW14(ch, i),
 				data << DPIO_UPAR_SHIFT);
 	}
 
@@ -950,17 +965,17 @@ void chv_phy_pre_encoder_enable(struct intel_encoder *encoder,
 	else
 		stagger = 0x2;
 
-	val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW11(ch));
+	val = vlv_dpio_read(dev_priv, phy, VLV_PCS01_DW11(ch));
 	val |= DPIO_TX2_STAGGER_MASK(0x1f);
-	vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS01_DW11(ch), val);
 
 	if (crtc_state->lane_count > 2) {
-		val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch));
+		val = vlv_dpio_read(dev_priv, phy, VLV_PCS23_DW11(ch));
 		val |= DPIO_TX2_STAGGER_MASK(0x1f);
-		vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val);
+		vlv_dpio_write(dev_priv, phy, VLV_PCS23_DW11(ch), val);
 	}
 
-	vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW12(ch),
+	vlv_dpio_write(dev_priv, phy, VLV_PCS01_DW12(ch),
 		       DPIO_LANESTAGGER_STRAP(stagger) |
 		       DPIO_LANESTAGGER_STRAP_OVRD |
 		       DPIO_TX1_STAGGER_MASK(0x1f) |
@@ -968,7 +983,7 @@ void chv_phy_pre_encoder_enable(struct intel_encoder *encoder,
 		       DPIO_TX2_STAGGER_MULT(0));
 
 	if (crtc_state->lane_count > 2) {
-		vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW12(ch),
+		vlv_dpio_write(dev_priv, phy, VLV_PCS23_DW12(ch),
 			       DPIO_LANESTAGGER_STRAP(stagger) |
 			       DPIO_LANESTAGGER_STRAP_OVRD |
 			       DPIO_TX1_STAGGER_MASK(0x1f) |
@@ -998,19 +1013,20 @@ void chv_phy_post_pll_disable(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	enum pipe pipe = to_intel_crtc(old_crtc_state->uapi.crtc)->pipe;
+	enum dpio_phy phy = vlv_pipe_to_phy(pipe);
 	u32 val;
 
 	vlv_dpio_get(dev_priv);
 
 	/* disable left/right clock distribution */
 	if (pipe != PIPE_B) {
-		val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0);
+		val = vlv_dpio_read(dev_priv, phy, _CHV_CMN_DW5_CH0);
 		val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK);
-		vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val);
+		vlv_dpio_write(dev_priv, phy, _CHV_CMN_DW5_CH0, val);
 	} else {
-		val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1);
+		val = vlv_dpio_read(dev_priv, phy, _CHV_CMN_DW1_CH1);
 		val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK);
-		vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val);
+		vlv_dpio_write(dev_priv, phy, _CHV_CMN_DW1_CH1, val);
 	}
 
 	vlv_dpio_put(dev_priv);
@@ -1036,22 +1052,22 @@ void vlv_set_phy_signal_level(struct intel_encoder *encoder,
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	enum dpio_channel port = vlv_dig_port_to_channel(dig_port);
-	enum pipe pipe = crtc->pipe;
+	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
 
 	vlv_dpio_get(dev_priv);
 
-	vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), 0x00000000);
-	vlv_dpio_write(dev_priv, pipe, VLV_TX_DW4(port), demph_reg_value);
-	vlv_dpio_write(dev_priv, pipe, VLV_TX_DW2(port),
+	vlv_dpio_write(dev_priv, phy, VLV_TX_DW5(port), 0x00000000);
+	vlv_dpio_write(dev_priv, phy, VLV_TX_DW4(port), demph_reg_value);
+	vlv_dpio_write(dev_priv, phy, VLV_TX_DW2(port),
 			 uniqtranscale_reg_value);
-	vlv_dpio_write(dev_priv, pipe, VLV_TX_DW3(port), 0x0C782040);
+	vlv_dpio_write(dev_priv, phy, VLV_TX_DW3(port), 0x0C782040);
 
 	if (tx3_demph)
-		vlv_dpio_write(dev_priv, pipe, VLV_TX3_DW4(port), tx3_demph);
+		vlv_dpio_write(dev_priv, phy, VLV_TX3_DW4(port), tx3_demph);
 
-	vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW11(port), 0x00030000);
-	vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW9(port), preemph_reg_value);
-	vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), DPIO_TX_OCALINIT_EN);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW11(port), 0x00030000);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW9(port), preemph_reg_value);
+	vlv_dpio_write(dev_priv, phy, VLV_TX_DW5(port), DPIO_TX_OCALINIT_EN);
 
 	vlv_dpio_put(dev_priv);
 }
@@ -1063,24 +1079,24 @@ void vlv_phy_pre_pll_enable(struct intel_encoder *encoder,
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	enum dpio_channel port = vlv_dig_port_to_channel(dig_port);
-	enum pipe pipe = crtc->pipe;
+	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
 
 	/* Program Tx lane resets to default */
 	vlv_dpio_get(dev_priv);
 
-	vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW0(port),
+	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW0(port),
 			 DPIO_PCS_TX_LANE2_RESET |
 			 DPIO_PCS_TX_LANE1_RESET);
-	vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW1(port),
+	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW1(port),
 			 DPIO_PCS_CLK_CRI_RXEB_EIOS_EN |
 			 DPIO_PCS_CLK_CRI_RXDIGFILTSG_EN |
 			 (1<<DPIO_PCS_CLK_DATAWIDTH_SHIFT) |
 				 DPIO_PCS_CLK_SOFT_RESET);
 
 	/* Fix up inter-pair skew failure */
-	vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW12(port), 0x00750f00);
-	vlv_dpio_write(dev_priv, pipe, VLV_TX_DW11(port), 0x00001500);
-	vlv_dpio_write(dev_priv, pipe, VLV_TX_DW14(port), 0x40400000);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW12(port), 0x00750f00);
+	vlv_dpio_write(dev_priv, phy, VLV_TX_DW11(port), 0x00001500);
+	vlv_dpio_write(dev_priv, phy, VLV_TX_DW14(port), 0x40400000);
 
 	vlv_dpio_put(dev_priv);
 }
@@ -1094,23 +1110,24 @@ void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder,
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	enum dpio_channel port = vlv_dig_port_to_channel(dig_port);
 	enum pipe pipe = crtc->pipe;
+	enum dpio_phy phy = vlv_pipe_to_phy(pipe);
 	u32 val;
 
 	vlv_dpio_get(dev_priv);
 
 	/* Enable clock channels for this port */
-	val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW8(port));
+	val = vlv_dpio_read(dev_priv, phy, VLV_PCS01_DW8(port));
 	val = 0;
 	if (pipe)
 		val |= (1<<21);
 	else
 		val &= ~(1<<21);
 	val |= 0x001000c4;
-	vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW8(port), val);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW8(port), val);
 
 	/* Program lane clock */
-	vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW14(port), 0x00760018);
-	vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW23(port), 0x00400888);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW14(port), 0x00760018);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW23(port), 0x00400888);
 
 	vlv_dpio_put(dev_priv);
 }
@@ -1122,10 +1139,10 @@ void vlv_phy_reset_lanes(struct intel_encoder *encoder,
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
 	enum dpio_channel port = vlv_dig_port_to_channel(dig_port);
-	enum pipe pipe = crtc->pipe;
+	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
 
 	vlv_dpio_get(dev_priv);
-	vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW0(port), 0x00000000);
-	vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW1(port), 0x00e00060);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW0(port), 0x00000000);
+	vlv_dpio_write(dev_priv, phy, VLV_PCS_DW1(port), 0x00e00060);
 	vlv_dpio_put(dev_priv);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_dpio_phy.h b/drivers/gpu/drm/i915/display/intel_dpio_phy.h
index 4d43dbbdf81c..9adc4e8c1738 100644
--- a/drivers/gpu/drm/i915/display/intel_dpio_phy.h
+++ b/drivers/gpu/drm/i915/display/intel_dpio_phy.h
@@ -44,6 +44,7 @@ u8 bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder);
 
 enum dpio_channel vlv_dig_port_to_channel(struct intel_digital_port *dig_port);
 enum dpio_phy vlv_dig_port_to_phy(struct intel_digital_port *dig_port);
+enum dpio_phy vlv_pipe_to_phy(enum pipe pipe);
 enum dpio_channel vlv_pipe_to_channel(enum pipe pipe);
 
 void chv_set_phy_signal_level(struct intel_encoder *encoder,
@@ -116,6 +117,10 @@ static inline enum dpio_phy vlv_dig_port_to_phy(struct intel_digital_port *dig_p
 {
 	return DPIO_PHY0;
 }
+static inline enum dpio_phy vlv_pipe_to_phy(enum pipe pipe)
+{
+	return DPIO_PHY0;
+}
 static inline enum dpio_channel vlv_pipe_to_channel(enum pipe pipe)
 {
 	return DPIO_CH0;
diff --git a/drivers/gpu/drm/i915/display/intel_dpll.c b/drivers/gpu/drm/i915/display/intel_dpll.c
index d41c1dc9f66c..3038655377ea 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll.c
+++ b/drivers/gpu/drm/i915/display/intel_dpll.c
@@ -16,6 +16,7 @@
 #include "intel_dpio_phy.h"
 #include "intel_dpll.h"
 #include "intel_lvds.h"
+#include "intel_lvds_regs.h"
 #include "intel_panel.h"
 #include "intel_pps.h"
 #include "intel_snps_phy.h"
@@ -311,7 +312,7 @@ static const struct intel_limit intel_limits_bxt = {
  * divided-down version of it.
  */
 /* m1 is reserved as 0 in Pineview, n is a ring counter */
-int pnv_calc_dpll_params(int refclk, struct dpll *clock)
+static int pnv_calc_dpll_params(int refclk, struct dpll *clock)
 {
 	clock->m = clock->m2 + 2;
 	clock->p = clock->p1 * clock->p2;
@@ -342,7 +343,7 @@ int i9xx_calc_dpll_params(int refclk, struct dpll *clock)
 	return clock->dot;
 }
 
-int vlv_calc_dpll_params(int refclk, struct dpll *clock)
+static int vlv_calc_dpll_params(int refclk, struct dpll *clock)
 {
 	clock->m = clock->m1 * clock->m2;
 	clock->p = clock->p1 * clock->p2 * 5;
@@ -368,6 +369,176 @@ int chv_calc_dpll_params(int refclk, struct dpll *clock)
 	return clock->dot;
 }
 
+static int i9xx_pll_refclk(struct drm_device *dev,
+			   const struct intel_crtc_state *pipe_config)
+{
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	u32 dpll = pipe_config->dpll_hw_state.dpll;
+
+	if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
+		return dev_priv->display.vbt.lvds_ssc_freq;
+	else if (HAS_PCH_SPLIT(dev_priv))
+		return 120000;
+	else if (DISPLAY_VER(dev_priv) != 2)
+		return 96000;
+	else
+		return 48000;
+}
+
+/* Returns the clock of the currently programmed mode of the given pipe. */
+void i9xx_crtc_clock_get(struct intel_crtc *crtc,
+			 struct intel_crtc_state *pipe_config)
+{
+	struct drm_device *dev = crtc->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	u32 dpll = pipe_config->dpll_hw_state.dpll;
+	u32 fp;
+	struct dpll clock;
+	int port_clock;
+	int refclk = i9xx_pll_refclk(dev, pipe_config);
+
+	if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
+		fp = pipe_config->dpll_hw_state.fp0;
+	else
+		fp = pipe_config->dpll_hw_state.fp1;
+
+	clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
+	if (IS_PINEVIEW(dev_priv)) {
+		clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
+		clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
+	} else {
+		clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
+		clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
+	}
+
+	if (DISPLAY_VER(dev_priv) != 2) {
+		if (IS_PINEVIEW(dev_priv))
+			clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
+				DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
+		else
+			clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
+			       DPLL_FPA01_P1_POST_DIV_SHIFT);
+
+		switch (dpll & DPLL_MODE_MASK) {
+		case DPLLB_MODE_DAC_SERIAL:
+			clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
+				5 : 10;
+			break;
+		case DPLLB_MODE_LVDS:
+			clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
+				7 : 14;
+			break;
+		default:
+			drm_dbg_kms(&dev_priv->drm,
+				    "Unknown DPLL mode %08x in programmed "
+				    "mode\n", (int)(dpll & DPLL_MODE_MASK));
+			return;
+		}
+
+		if (IS_PINEVIEW(dev_priv))
+			port_clock = pnv_calc_dpll_params(refclk, &clock);
+		else
+			port_clock = i9xx_calc_dpll_params(refclk, &clock);
+	} else {
+		enum pipe lvds_pipe;
+
+		if (IS_I85X(dev_priv) &&
+		    intel_lvds_port_enabled(dev_priv, LVDS, &lvds_pipe) &&
+		    lvds_pipe == crtc->pipe) {
+			u32 lvds = intel_de_read(dev_priv, LVDS);
+
+			clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
+				       DPLL_FPA01_P1_POST_DIV_SHIFT);
+
+			if (lvds & LVDS_CLKB_POWER_UP)
+				clock.p2 = 7;
+			else
+				clock.p2 = 14;
+		} else {
+			if (dpll & PLL_P1_DIVIDE_BY_TWO)
+				clock.p1 = 2;
+			else {
+				clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
+					    DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
+			}
+			if (dpll & PLL_P2_DIVIDE_BY_4)
+				clock.p2 = 4;
+			else
+				clock.p2 = 2;
+		}
+
+		port_clock = i9xx_calc_dpll_params(refclk, &clock);
+	}
+
+	/*
+	 * This value includes pixel_multiplier. We will use
+	 * port_clock to compute adjusted_mode.crtc_clock in the
+	 * encoder's get_config() function.
+	 */
+	pipe_config->port_clock = port_clock;
+}
+
+void vlv_crtc_clock_get(struct intel_crtc *crtc,
+			struct intel_crtc_state *pipe_config)
+{
+	struct drm_device *dev = crtc->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
+	struct dpll clock;
+	u32 mdiv;
+	int refclk = 100000;
+
+	/* In case of DSI, DPLL will not be used */
+	if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
+		return;
+
+	vlv_dpio_get(dev_priv);
+	mdiv = vlv_dpio_read(dev_priv, phy, VLV_PLL_DW3(crtc->pipe));
+	vlv_dpio_put(dev_priv);
+
+	clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
+	clock.m2 = mdiv & DPIO_M2DIV_MASK;
+	clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
+	clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
+	clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
+
+	pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
+}
+
+void chv_crtc_clock_get(struct intel_crtc *crtc,
+			struct intel_crtc_state *pipe_config)
+{
+	struct drm_device *dev = crtc->base.dev;
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	enum dpio_channel port = vlv_pipe_to_channel(crtc->pipe);
+	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
+	struct dpll clock;
+	u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
+	int refclk = 100000;
+
+	/* In case of DSI, DPLL will not be used */
+	if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
+		return;
+
+	vlv_dpio_get(dev_priv);
+	cmn_dw13 = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW13(port));
+	pll_dw0 = vlv_dpio_read(dev_priv, phy, CHV_PLL_DW0(port));
+	pll_dw1 = vlv_dpio_read(dev_priv, phy, CHV_PLL_DW1(port));
+	pll_dw2 = vlv_dpio_read(dev_priv, phy, CHV_PLL_DW2(port));
+	pll_dw3 = vlv_dpio_read(dev_priv, phy, CHV_PLL_DW3(port));
+	vlv_dpio_put(dev_priv);
+
+	clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
+	clock.m2 = (pll_dw0 & 0xff) << 22;
+	if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
+		clock.m2 |= pll_dw2 & 0x3fffff;
+	clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
+	clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
+	clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
+
+	pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
+}
+
 /*
  * Returns whether the given set of divisors are valid for a given refclk with
  * the given connectors.
@@ -1003,12 +1174,10 @@ static int dg2_crtc_compute_clock(struct intel_atomic_state *state,
 static int mtl_crtc_compute_clock(struct intel_atomic_state *state,
 				  struct intel_crtc *crtc)
 {
-	struct drm_i915_private *i915 = to_i915(state->base.dev);
 	struct intel_crtc_state *crtc_state =
 		intel_atomic_get_new_crtc_state(state, crtc);
 	struct intel_encoder *encoder =
 		intel_get_crtc_new_encoder(state, crtc_state);
-	enum phy phy = intel_port_to_phy(i915, encoder->port);
 	int ret;
 
 	ret = intel_cx0pll_calc_state(crtc_state, encoder);
@@ -1016,10 +1185,7 @@ static int mtl_crtc_compute_clock(struct intel_atomic_state *state,
 		return ret;
 
 	/* TODO: Do the readback via intel_compute_shared_dplls() */
-	if (intel_is_c10phy(i915, phy))
-		crtc_state->port_clock = intel_c10pll_calc_port_clock(encoder, &crtc_state->cx0pll_state.c10);
-	else
-		crtc_state->port_clock = intel_c20pll_calc_port_clock(encoder, &crtc_state->cx0pll_state.c20);
+	crtc_state->port_clock = intel_cx0pll_calc_port_clock(encoder, &crtc_state->cx0pll_state);
 
 	crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state);
 
@@ -1645,7 +1811,7 @@ void i9xx_enable_pll(const struct intel_crtc_state *crtc_state)
 }
 
 static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv,
-				 enum pipe pipe)
+				 enum dpio_phy phy)
 {
 	u32 reg_val;
 
@@ -1653,30 +1819,31 @@ static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv,
 	 * PLLB opamp always calibrates to max value of 0x3f, force enable it
 	 * and set it to a reasonable value instead.
 	 */
-	reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
+	reg_val = vlv_dpio_read(dev_priv, phy, VLV_PLL_DW9(1));
 	reg_val &= 0xffffff00;
 	reg_val |= 0x00000030;
-	vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
+	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW9(1), reg_val);
 
-	reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
+	reg_val = vlv_dpio_read(dev_priv, phy, VLV_REF_DW13);
 	reg_val &= 0x00ffffff;
 	reg_val |= 0x8c000000;
-	vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
+	vlv_dpio_write(dev_priv, phy, VLV_REF_DW13, reg_val);
 
-	reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
+	reg_val = vlv_dpio_read(dev_priv, phy, VLV_PLL_DW9(1));
 	reg_val &= 0xffffff00;
-	vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
+	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW9(1), reg_val);
 
-	reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
+	reg_val = vlv_dpio_read(dev_priv, phy, VLV_REF_DW13);
 	reg_val &= 0x00ffffff;
 	reg_val |= 0xb0000000;
-	vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
+	vlv_dpio_write(dev_priv, phy, VLV_REF_DW13, reg_val);
 }
 
 static void vlv_prepare_pll(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
 	enum pipe pipe = crtc->pipe;
 	u32 mdiv;
 	u32 bestn, bestm1, bestm2, bestp1, bestp2;
@@ -1694,18 +1861,18 @@ static void vlv_prepare_pll(const struct intel_crtc_state *crtc_state)
 
 	/* PLL B needs special handling */
 	if (pipe == PIPE_B)
-		vlv_pllb_recal_opamp(dev_priv, pipe);
+		vlv_pllb_recal_opamp(dev_priv, phy);
 
 	/* Set up Tx target for periodic Rcomp update */
-	vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9_BCAST, 0x0100000f);
+	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW9_BCAST, 0x0100000f);
 
 	/* Disable target IRef on PLL */
-	reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW8(pipe));
+	reg_val = vlv_dpio_read(dev_priv, phy, VLV_PLL_DW8(pipe));
 	reg_val &= 0x00ffffff;
-	vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW8(pipe), reg_val);
+	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW8(pipe), reg_val);
 
 	/* Disable fast lock */
-	vlv_dpio_write(dev_priv, pipe, VLV_CMN_DW0, 0x610);
+	vlv_dpio_write(dev_priv, phy, VLV_CMN_DW0, 0x610);
 
 	/* Set idtafcrecal before PLL is enabled */
 	mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK));
@@ -1719,46 +1886,46 @@ static void vlv_prepare_pll(const struct intel_crtc_state *crtc_state)
 	 * Note: don't use the DAC post divider as it seems unstable.
 	 */
 	mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT);
-	vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
+	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW3(pipe), mdiv);
 
 	mdiv |= DPIO_ENABLE_CALIBRATION;
-	vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
+	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW3(pipe), mdiv);
 
 	/* Set HBR and RBR LPF coefficients */
 	if (crtc_state->port_clock == 162000 ||
 	    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG) ||
 	    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
-		vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
+		vlv_dpio_write(dev_priv, phy, VLV_PLL_DW10(pipe),
 				 0x009f0003);
 	else
-		vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
+		vlv_dpio_write(dev_priv, phy, VLV_PLL_DW10(pipe),
 				 0x00d0000f);
 
 	if (intel_crtc_has_dp_encoder(crtc_state)) {
 		/* Use SSC source */
 		if (pipe == PIPE_A)
-			vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
+			vlv_dpio_write(dev_priv, phy, VLV_PLL_DW5(pipe),
 					 0x0df40000);
 		else
-			vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
+			vlv_dpio_write(dev_priv, phy, VLV_PLL_DW5(pipe),
 					 0x0df70000);
 	} else { /* HDMI or VGA */
 		/* Use bend source */
 		if (pipe == PIPE_A)
-			vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
+			vlv_dpio_write(dev_priv, phy, VLV_PLL_DW5(pipe),
 					 0x0df70000);
 		else
-			vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
+			vlv_dpio_write(dev_priv, phy, VLV_PLL_DW5(pipe),
 					 0x0df40000);
 	}
 
-	coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe));
+	coreclk = vlv_dpio_read(dev_priv, phy, VLV_PLL_DW7(pipe));
 	coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
 	if (intel_crtc_has_dp_encoder(crtc_state))
 		coreclk |= 0x01000000;
-	vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk);
+	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW7(pipe), coreclk);
 
-	vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW11(pipe), 0x87871000);
+	vlv_dpio_write(dev_priv, phy, VLV_PLL_DW11(pipe), 0x87871000);
 
 	vlv_dpio_put(dev_priv);
 }
@@ -1809,6 +1976,7 @@ static void chv_prepare_pll(const struct intel_crtc_state *crtc_state)
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum pipe pipe = crtc->pipe;
 	enum dpio_channel port = vlv_pipe_to_channel(pipe);
+	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
 	u32 loopfilter, tribuf_calcntr;
 	u32 bestm2, bestp1, bestp2, bestm2_frac;
 	u32 dpio_val;
@@ -1825,39 +1993,39 @@ static void chv_prepare_pll(const struct intel_crtc_state *crtc_state)
 	vlv_dpio_get(dev_priv);
 
 	/* p1 and p2 divider */
-	vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW13(port),
+	vlv_dpio_write(dev_priv, phy, CHV_CMN_DW13(port),
 			5 << DPIO_CHV_S1_DIV_SHIFT |
 			bestp1 << DPIO_CHV_P1_DIV_SHIFT |
 			bestp2 << DPIO_CHV_P2_DIV_SHIFT |
 			1 << DPIO_CHV_K_DIV_SHIFT);
 
 	/* Feedback post-divider - m2 */
-	vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW0(port), bestm2);
+	vlv_dpio_write(dev_priv, phy, CHV_PLL_DW0(port), bestm2);
 
 	/* Feedback refclk divider - n and m1 */
-	vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW1(port),
+	vlv_dpio_write(dev_priv, phy, CHV_PLL_DW1(port),
 			DPIO_CHV_M1_DIV_BY_2 |
 			1 << DPIO_CHV_N_DIV_SHIFT);
 
 	/* M2 fraction division */
-	vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac);
+	vlv_dpio_write(dev_priv, phy, CHV_PLL_DW2(port), bestm2_frac);
 
 	/* M2 fraction division enable */
-	dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
+	dpio_val = vlv_dpio_read(dev_priv, phy, CHV_PLL_DW3(port));
 	dpio_val &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
 	dpio_val |= (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT);
 	if (bestm2_frac)
 		dpio_val |= DPIO_CHV_FRAC_DIV_EN;
-	vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW3(port), dpio_val);
+	vlv_dpio_write(dev_priv, phy, CHV_PLL_DW3(port), dpio_val);
 
 	/* Program digital lock detect threshold */
-	dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW9(port));
+	dpio_val = vlv_dpio_read(dev_priv, phy, CHV_PLL_DW9(port));
 	dpio_val &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
 					DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
 	dpio_val |= (0x5 << DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT);
 	if (!bestm2_frac)
 		dpio_val |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
-	vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW9(port), dpio_val);
+	vlv_dpio_write(dev_priv, phy, CHV_PLL_DW9(port), dpio_val);
 
 	/* Loop filter */
 	if (vco == 5400000) {
@@ -1882,16 +2050,16 @@ static void chv_prepare_pll(const struct intel_crtc_state *crtc_state)
 		loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
 		tribuf_calcntr = 0;
 	}
-	vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW6(port), loopfilter);
+	vlv_dpio_write(dev_priv, phy, CHV_PLL_DW6(port), loopfilter);
 
-	dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW8(port));
+	dpio_val = vlv_dpio_read(dev_priv, phy, CHV_PLL_DW8(port));
 	dpio_val &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
 	dpio_val |= (tribuf_calcntr << DPIO_CHV_TDC_TARGET_CNT_SHIFT);
-	vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW8(port), dpio_val);
+	vlv_dpio_write(dev_priv, phy, CHV_PLL_DW8(port), dpio_val);
 
 	/* AFC Recal */
-	vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port),
-			vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)) |
+	vlv_dpio_write(dev_priv, phy, CHV_CMN_DW14(port),
+			vlv_dpio_read(dev_priv, phy, CHV_CMN_DW14(port)) |
 			DPIO_AFC_RECAL);
 
 	vlv_dpio_put(dev_priv);
@@ -1903,14 +2071,15 @@ static void _chv_enable_pll(const struct intel_crtc_state *crtc_state)
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum pipe pipe = crtc->pipe;
 	enum dpio_channel port = vlv_pipe_to_channel(pipe);
+	enum dpio_phy phy = vlv_pipe_to_phy(crtc->pipe);
 	u32 tmp;
 
 	vlv_dpio_get(dev_priv);
 
 	/* Enable back the 10bit clock to display controller */
-	tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
+	tmp = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW14(port));
 	tmp |= DPIO_DCLKP_EN;
-	vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp);
+	vlv_dpio_write(dev_priv, phy, CHV_CMN_DW14(port), tmp);
 
 	vlv_dpio_put(dev_priv);
 
@@ -2031,6 +2200,7 @@ void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
 void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
 {
 	enum dpio_channel port = vlv_pipe_to_channel(pipe);
+	enum dpio_phy phy = vlv_pipe_to_phy(pipe);
 	u32 val;
 
 	/* Make sure the pipe isn't still relying on us */
@@ -2047,9 +2217,9 @@ void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
 	vlv_dpio_get(dev_priv);
 
 	/* Disable 10bit clock to display controller */
-	val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
+	val = vlv_dpio_read(dev_priv, phy, CHV_CMN_DW14(port));
 	val &= ~DPIO_DCLKP_EN;
-	vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
+	vlv_dpio_write(dev_priv, phy, CHV_CMN_DW14(port), val);
 
 	vlv_dpio_put(dev_priv);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_dpll.h b/drivers/gpu/drm/i915/display/intel_dpll.h
index bbc30542f29f..ac01bb19cc6c 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll.h
+++ b/drivers/gpu/drm/i915/display/intel_dpll.h
@@ -20,8 +20,6 @@ int intel_dpll_crtc_compute_clock(struct intel_atomic_state *state,
 				  struct intel_crtc *crtc);
 int intel_dpll_crtc_get_shared_dpll(struct intel_atomic_state *state,
 				    struct intel_crtc *crtc);
-int vlv_calc_dpll_params(int refclk, struct dpll *clock);
-int pnv_calc_dpll_params(int refclk, struct dpll *clock);
 int i9xx_calc_dpll_params(int refclk, struct dpll *clock);
 u32 i9xx_dpll_compute_fp(const struct dpll *dpll);
 void vlv_compute_dpll(struct intel_crtc_state *crtc_state);
@@ -41,6 +39,13 @@ bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state,
 			struct dpll *best_clock);
 int chv_calc_dpll_params(int refclk, struct dpll *pll_clock);
 
+void i9xx_crtc_clock_get(struct intel_crtc *crtc,
+			 struct intel_crtc_state *pipe_config);
+void vlv_crtc_clock_get(struct intel_crtc *crtc,
+			struct intel_crtc_state *pipe_config);
+void chv_crtc_clock_get(struct intel_crtc *crtc,
+			struct intel_crtc_state *pipe_config);
+
 void assert_pll_enabled(struct drm_i915_private *i915, enum pipe pipe);
 void assert_pll_disabled(struct drm_i915_private *i915, enum pipe pipe);
 
diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
index 399653a20f98..7958d0bd851e 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
+++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
@@ -219,6 +219,26 @@ intel_tc_pll_enable_reg(struct drm_i915_private *i915,
 	return MG_PLL_ENABLE(tc_port);
 }
 
+static void _intel_enable_shared_dpll(struct drm_i915_private *i915,
+				      struct intel_shared_dpll *pll)
+{
+	if (pll->info->power_domain)
+		pll->wakeref = intel_display_power_get(i915, pll->info->power_domain);
+
+	pll->info->funcs->enable(i915, pll);
+	pll->on = true;
+}
+
+static void _intel_disable_shared_dpll(struct drm_i915_private *i915,
+				       struct intel_shared_dpll *pll)
+{
+	pll->info->funcs->disable(i915, pll);
+	pll->on = false;
+
+	if (pll->info->power_domain)
+		intel_display_power_put(i915, pll->info->power_domain, pll->wakeref);
+}
+
 /**
  * intel_enable_shared_dpll - enable a CRTC's shared DPLL
  * @crtc_state: CRTC, and its state, which has a shared DPLL
@@ -258,8 +278,8 @@ void intel_enable_shared_dpll(const struct intel_crtc_state *crtc_state)
 	drm_WARN_ON(&i915->drm, pll->on);
 
 	drm_dbg_kms(&i915->drm, "enabling %s\n", pll->info->name);
-	pll->info->funcs->enable(i915, pll);
-	pll->on = true;
+
+	_intel_enable_shared_dpll(i915, pll);
 
 out:
 	mutex_unlock(&i915->display.dpll.lock);
@@ -304,8 +324,8 @@ void intel_disable_shared_dpll(const struct intel_crtc_state *crtc_state)
 		goto out;
 
 	drm_dbg_kms(&i915->drm, "disabling %s\n", pll->info->name);
-	pll->info->funcs->disable(i915, pll);
-	pll->on = false;
+
+	_intel_disable_shared_dpll(i915, pll);
 
 out:
 	mutex_unlock(&i915->display.dpll.lock);
@@ -631,9 +651,9 @@ static const struct intel_shared_dpll_funcs ibx_pch_dpll_funcs = {
 };
 
 static const struct dpll_info pch_plls[] = {
-	{ "PCH DPLL A", &ibx_pch_dpll_funcs, DPLL_ID_PCH_PLL_A, 0 },
-	{ "PCH DPLL B", &ibx_pch_dpll_funcs, DPLL_ID_PCH_PLL_B, 0 },
-	{ },
+	{ .name = "PCH DPLL A", .funcs = &ibx_pch_dpll_funcs, .id = DPLL_ID_PCH_PLL_A, },
+	{ .name = "PCH DPLL B", .funcs = &ibx_pch_dpll_funcs, .id = DPLL_ID_PCH_PLL_B, },
+	{}
 };
 
 static const struct intel_dpll_mgr pch_pll_mgr = {
@@ -1239,13 +1259,16 @@ static const struct intel_shared_dpll_funcs hsw_ddi_lcpll_funcs = {
 };
 
 static const struct dpll_info hsw_plls[] = {
-	{ "WRPLL 1",    &hsw_ddi_wrpll_funcs, DPLL_ID_WRPLL1,     0 },
-	{ "WRPLL 2",    &hsw_ddi_wrpll_funcs, DPLL_ID_WRPLL2,     0 },
-	{ "SPLL",       &hsw_ddi_spll_funcs,  DPLL_ID_SPLL,       0 },
-	{ "LCPLL 810",  &hsw_ddi_lcpll_funcs, DPLL_ID_LCPLL_810,  INTEL_DPLL_ALWAYS_ON },
-	{ "LCPLL 1350", &hsw_ddi_lcpll_funcs, DPLL_ID_LCPLL_1350, INTEL_DPLL_ALWAYS_ON },
-	{ "LCPLL 2700", &hsw_ddi_lcpll_funcs, DPLL_ID_LCPLL_2700, INTEL_DPLL_ALWAYS_ON },
-	{ },
+	{ .name = "WRPLL 1", .funcs = &hsw_ddi_wrpll_funcs, .id = DPLL_ID_WRPLL1, },
+	{ .name = "WRPLL 2", .funcs = &hsw_ddi_wrpll_funcs, .id = DPLL_ID_WRPLL2, },
+	{ .name = "SPLL", .funcs = &hsw_ddi_spll_funcs, .id = DPLL_ID_SPLL, },
+	{ .name = "LCPLL 810", .funcs = &hsw_ddi_lcpll_funcs, .id = DPLL_ID_LCPLL_810,
+	  .flags = INTEL_DPLL_ALWAYS_ON, },
+	{ .name = "LCPLL 1350", .funcs = &hsw_ddi_lcpll_funcs, .id = DPLL_ID_LCPLL_1350,
+	  .flags = INTEL_DPLL_ALWAYS_ON, },
+	{ .name = "LCPLL 2700", .funcs = &hsw_ddi_lcpll_funcs, .id = DPLL_ID_LCPLL_2700,
+	  .flags = INTEL_DPLL_ALWAYS_ON, },
+	{}
 };
 
 static const struct intel_dpll_mgr hsw_pll_mgr = {
@@ -1921,11 +1944,12 @@ static const struct intel_shared_dpll_funcs skl_ddi_dpll0_funcs = {
 };
 
 static const struct dpll_info skl_plls[] = {
-	{ "DPLL 0", &skl_ddi_dpll0_funcs, DPLL_ID_SKL_DPLL0, INTEL_DPLL_ALWAYS_ON },
-	{ "DPLL 1", &skl_ddi_pll_funcs,   DPLL_ID_SKL_DPLL1, 0 },
-	{ "DPLL 2", &skl_ddi_pll_funcs,   DPLL_ID_SKL_DPLL2, 0 },
-	{ "DPLL 3", &skl_ddi_pll_funcs,   DPLL_ID_SKL_DPLL3, 0 },
-	{ },
+	{ .name = "DPLL 0", .funcs = &skl_ddi_dpll0_funcs, .id = DPLL_ID_SKL_DPLL0,
+	  .flags = INTEL_DPLL_ALWAYS_ON, },
+	{ .name = "DPLL 1", .funcs = &skl_ddi_pll_funcs, .id = DPLL_ID_SKL_DPLL1, },
+	{ .name = "DPLL 2", .funcs = &skl_ddi_pll_funcs, .id = DPLL_ID_SKL_DPLL2, },
+	{ .name = "DPLL 3", .funcs = &skl_ddi_pll_funcs, .id = DPLL_ID_SKL_DPLL3, },
+	{}
 };
 
 static const struct intel_dpll_mgr skl_pll_mgr = {
@@ -2376,10 +2400,10 @@ static const struct intel_shared_dpll_funcs bxt_ddi_pll_funcs = {
 };
 
 static const struct dpll_info bxt_plls[] = {
-	{ "PORT PLL A", &bxt_ddi_pll_funcs, DPLL_ID_SKL_DPLL0, 0 },
-	{ "PORT PLL B", &bxt_ddi_pll_funcs, DPLL_ID_SKL_DPLL1, 0 },
-	{ "PORT PLL C", &bxt_ddi_pll_funcs, DPLL_ID_SKL_DPLL2, 0 },
-	{ },
+	{ .name = "PORT PLL A", .funcs = &bxt_ddi_pll_funcs, .id = DPLL_ID_SKL_DPLL0, },
+	{ .name = "PORT PLL B", .funcs = &bxt_ddi_pll_funcs, .id = DPLL_ID_SKL_DPLL1, },
+	{ .name = "PORT PLL C", .funcs = &bxt_ddi_pll_funcs, .id = DPLL_ID_SKL_DPLL2, },
+	{}
 };
 
 static const struct intel_dpll_mgr bxt_pll_mgr = {
@@ -3834,18 +3858,6 @@ static void combo_pll_enable(struct drm_i915_private *i915,
 {
 	i915_reg_t enable_reg = intel_combo_pll_enable_reg(i915, pll);
 
-	if ((IS_JASPERLAKE(i915) || IS_ELKHARTLAKE(i915)) &&
-	    pll->info->id == DPLL_ID_EHL_DPLL4) {
-
-		/*
-		 * We need to disable DC states when this DPLL is enabled.
-		 * This can be done by taking a reference on DPLL4 power
-		 * domain.
-		 */
-		pll->wakeref = intel_display_power_get(i915,
-						       POWER_DOMAIN_DC_OFF);
-	}
-
 	icl_pll_power_enable(i915, pll, enable_reg);
 
 	icl_dpll_write(i915, pll);
@@ -3941,11 +3953,6 @@ static void combo_pll_disable(struct drm_i915_private *i915,
 	i915_reg_t enable_reg = intel_combo_pll_enable_reg(i915, pll);
 
 	icl_pll_disable(i915, pll, enable_reg);
-
-	if ((IS_JASPERLAKE(i915) || IS_ELKHARTLAKE(i915)) &&
-	    pll->info->id == DPLL_ID_EHL_DPLL4)
-		intel_display_power_put(i915, POWER_DOMAIN_DC_OFF,
-					pll->wakeref);
 }
 
 static void tbt_pll_disable(struct drm_i915_private *i915,
@@ -4014,14 +4021,14 @@ static const struct intel_shared_dpll_funcs mg_pll_funcs = {
 };
 
 static const struct dpll_info icl_plls[] = {
-	{ "DPLL 0",   &combo_pll_funcs, DPLL_ID_ICL_DPLL0,  0 },
-	{ "DPLL 1",   &combo_pll_funcs, DPLL_ID_ICL_DPLL1,  0 },
-	{ "TBT PLL",  &tbt_pll_funcs, DPLL_ID_ICL_TBTPLL, 0 },
-	{ "MG PLL 1", &mg_pll_funcs, DPLL_ID_ICL_MGPLL1, 0 },
-	{ "MG PLL 2", &mg_pll_funcs, DPLL_ID_ICL_MGPLL2, 0 },
-	{ "MG PLL 3", &mg_pll_funcs, DPLL_ID_ICL_MGPLL3, 0 },
-	{ "MG PLL 4", &mg_pll_funcs, DPLL_ID_ICL_MGPLL4, 0 },
-	{ },
+	{ .name = "DPLL 0", .funcs = &combo_pll_funcs, .id = DPLL_ID_ICL_DPLL0, },
+	{ .name = "DPLL 1", .funcs = &combo_pll_funcs, .id = DPLL_ID_ICL_DPLL1, },
+	{ .name = "TBT PLL", .funcs = &tbt_pll_funcs, .id = DPLL_ID_ICL_TBTPLL, },
+	{ .name = "MG PLL 1", .funcs = &mg_pll_funcs, .id = DPLL_ID_ICL_MGPLL1, },
+	{ .name = "MG PLL 2", .funcs = &mg_pll_funcs, .id = DPLL_ID_ICL_MGPLL2, },
+	{ .name = "MG PLL 3", .funcs = &mg_pll_funcs, .id = DPLL_ID_ICL_MGPLL3, },
+	{ .name = "MG PLL 4", .funcs = &mg_pll_funcs, .id = DPLL_ID_ICL_MGPLL4, },
+	{}
 };
 
 static const struct intel_dpll_mgr icl_pll_mgr = {
@@ -4035,10 +4042,11 @@ static const struct intel_dpll_mgr icl_pll_mgr = {
 };
 
 static const struct dpll_info ehl_plls[] = {
-	{ "DPLL 0", &combo_pll_funcs, DPLL_ID_ICL_DPLL0, 0 },
-	{ "DPLL 1", &combo_pll_funcs, DPLL_ID_ICL_DPLL1, 0 },
-	{ "DPLL 4", &combo_pll_funcs, DPLL_ID_EHL_DPLL4, 0 },
-	{ },
+	{ .name = "DPLL 0", .funcs = &combo_pll_funcs, .id = DPLL_ID_ICL_DPLL0, },
+	{ .name = "DPLL 1", .funcs = &combo_pll_funcs, .id = DPLL_ID_ICL_DPLL1, },
+	{ .name = "DPLL 4", .funcs = &combo_pll_funcs, .id = DPLL_ID_EHL_DPLL4,
+	  .power_domain = POWER_DOMAIN_DC_OFF, },
+	{}
 };
 
 static const struct intel_dpll_mgr ehl_pll_mgr = {
@@ -4058,16 +4066,16 @@ static const struct intel_shared_dpll_funcs dkl_pll_funcs = {
 };
 
 static const struct dpll_info tgl_plls[] = {
-	{ "DPLL 0", &combo_pll_funcs, DPLL_ID_ICL_DPLL0,  0 },
-	{ "DPLL 1", &combo_pll_funcs, DPLL_ID_ICL_DPLL1,  0 },
-	{ "TBT PLL",  &tbt_pll_funcs, DPLL_ID_ICL_TBTPLL, 0 },
-	{ "TC PLL 1", &dkl_pll_funcs, DPLL_ID_ICL_MGPLL1, 0 },
-	{ "TC PLL 2", &dkl_pll_funcs, DPLL_ID_ICL_MGPLL2, 0 },
-	{ "TC PLL 3", &dkl_pll_funcs, DPLL_ID_ICL_MGPLL3, 0 },
-	{ "TC PLL 4", &dkl_pll_funcs, DPLL_ID_ICL_MGPLL4, 0 },
-	{ "TC PLL 5", &dkl_pll_funcs, DPLL_ID_TGL_MGPLL5, 0 },
-	{ "TC PLL 6", &dkl_pll_funcs, DPLL_ID_TGL_MGPLL6, 0 },
-	{ },
+	{ .name = "DPLL 0", .funcs = &combo_pll_funcs, .id = DPLL_ID_ICL_DPLL0, },
+	{ .name = "DPLL 1", .funcs = &combo_pll_funcs, .id = DPLL_ID_ICL_DPLL1, },
+	{ .name = "TBT PLL", .funcs = &tbt_pll_funcs, .id = DPLL_ID_ICL_TBTPLL, },
+	{ .name = "TC PLL 1", .funcs = &dkl_pll_funcs, .id = DPLL_ID_ICL_MGPLL1, },
+	{ .name = "TC PLL 2", .funcs = &dkl_pll_funcs, .id = DPLL_ID_ICL_MGPLL2, },
+	{ .name = "TC PLL 3", .funcs = &dkl_pll_funcs, .id = DPLL_ID_ICL_MGPLL3, },
+	{ .name = "TC PLL 4", .funcs = &dkl_pll_funcs, .id = DPLL_ID_ICL_MGPLL4, },
+	{ .name = "TC PLL 5", .funcs = &dkl_pll_funcs, .id = DPLL_ID_TGL_MGPLL5, },
+	{ .name = "TC PLL 6", .funcs = &dkl_pll_funcs, .id = DPLL_ID_TGL_MGPLL6, },
+	{}
 };
 
 static const struct intel_dpll_mgr tgl_pll_mgr = {
@@ -4081,10 +4089,10 @@ static const struct intel_dpll_mgr tgl_pll_mgr = {
 };
 
 static const struct dpll_info rkl_plls[] = {
-	{ "DPLL 0", &combo_pll_funcs, DPLL_ID_ICL_DPLL0, 0 },
-	{ "DPLL 1", &combo_pll_funcs, DPLL_ID_ICL_DPLL1, 0 },
-	{ "DPLL 4", &combo_pll_funcs, DPLL_ID_EHL_DPLL4, 0 },
-	{ },
+	{ .name = "DPLL 0", .funcs = &combo_pll_funcs, .id = DPLL_ID_ICL_DPLL0, },
+	{ .name = "DPLL 1", .funcs = &combo_pll_funcs, .id = DPLL_ID_ICL_DPLL1, },
+	{ .name = "DPLL 4", .funcs = &combo_pll_funcs, .id = DPLL_ID_EHL_DPLL4, },
+	{}
 };
 
 static const struct intel_dpll_mgr rkl_pll_mgr = {
@@ -4097,11 +4105,11 @@ static const struct intel_dpll_mgr rkl_pll_mgr = {
 };
 
 static const struct dpll_info dg1_plls[] = {
-	{ "DPLL 0", &combo_pll_funcs, DPLL_ID_DG1_DPLL0, 0 },
-	{ "DPLL 1", &combo_pll_funcs, DPLL_ID_DG1_DPLL1, 0 },
-	{ "DPLL 2", &combo_pll_funcs, DPLL_ID_DG1_DPLL2, 0 },
-	{ "DPLL 3", &combo_pll_funcs, DPLL_ID_DG1_DPLL3, 0 },
-	{ },
+	{ .name = "DPLL 0", .funcs = &combo_pll_funcs, .id = DPLL_ID_DG1_DPLL0, },
+	{ .name = "DPLL 1", .funcs = &combo_pll_funcs, .id = DPLL_ID_DG1_DPLL1, },
+	{ .name = "DPLL 2", .funcs = &combo_pll_funcs, .id = DPLL_ID_DG1_DPLL2, },
+	{ .name = "DPLL 3", .funcs = &combo_pll_funcs, .id = DPLL_ID_DG1_DPLL3, },
+	{}
 };
 
 static const struct intel_dpll_mgr dg1_pll_mgr = {
@@ -4114,11 +4122,11 @@ static const struct intel_dpll_mgr dg1_pll_mgr = {
 };
 
 static const struct dpll_info adls_plls[] = {
-	{ "DPLL 0", &combo_pll_funcs, DPLL_ID_ICL_DPLL0, 0 },
-	{ "DPLL 1", &combo_pll_funcs, DPLL_ID_ICL_DPLL1, 0 },
-	{ "DPLL 2", &combo_pll_funcs, DPLL_ID_DG1_DPLL2, 0 },
-	{ "DPLL 3", &combo_pll_funcs, DPLL_ID_DG1_DPLL3, 0 },
-	{ },
+	{ .name = "DPLL 0", .funcs = &combo_pll_funcs, .id = DPLL_ID_ICL_DPLL0, },
+	{ .name = "DPLL 1", .funcs = &combo_pll_funcs, .id = DPLL_ID_ICL_DPLL1, },
+	{ .name = "DPLL 2", .funcs = &combo_pll_funcs, .id = DPLL_ID_DG1_DPLL2, },
+	{ .name = "DPLL 3", .funcs = &combo_pll_funcs, .id = DPLL_ID_DG1_DPLL3, },
+	{}
 };
 
 static const struct intel_dpll_mgr adls_pll_mgr = {
@@ -4131,14 +4139,14 @@ static const struct intel_dpll_mgr adls_pll_mgr = {
 };
 
 static const struct dpll_info adlp_plls[] = {
-	{ "DPLL 0", &combo_pll_funcs, DPLL_ID_ICL_DPLL0,  0 },
-	{ "DPLL 1", &combo_pll_funcs, DPLL_ID_ICL_DPLL1,  0 },
-	{ "TBT PLL",  &tbt_pll_funcs, DPLL_ID_ICL_TBTPLL, 0 },
-	{ "TC PLL 1", &dkl_pll_funcs, DPLL_ID_ICL_MGPLL1, 0 },
-	{ "TC PLL 2", &dkl_pll_funcs, DPLL_ID_ICL_MGPLL2, 0 },
-	{ "TC PLL 3", &dkl_pll_funcs, DPLL_ID_ICL_MGPLL3, 0 },
-	{ "TC PLL 4", &dkl_pll_funcs, DPLL_ID_ICL_MGPLL4, 0 },
-	{ },
+	{ .name = "DPLL 0", .funcs = &combo_pll_funcs, .id = DPLL_ID_ICL_DPLL0, },
+	{ .name = "DPLL 1", .funcs = &combo_pll_funcs, .id = DPLL_ID_ICL_DPLL1, },
+	{ .name = "TBT PLL", .funcs = &tbt_pll_funcs, .id = DPLL_ID_ICL_TBTPLL, },
+	{ .name = "TC PLL 1", .funcs = &dkl_pll_funcs, .id = DPLL_ID_ICL_MGPLL1, },
+	{ .name = "TC PLL 2", .funcs = &dkl_pll_funcs, .id = DPLL_ID_ICL_MGPLL2, },
+	{ .name = "TC PLL 3", .funcs = &dkl_pll_funcs, .id = DPLL_ID_ICL_MGPLL3, },
+	{ .name = "TC PLL 4", .funcs = &dkl_pll_funcs, .id = DPLL_ID_ICL_MGPLL4, },
+	{}
 };
 
 static const struct intel_dpll_mgr adlp_pll_mgr = {
@@ -4365,12 +4373,8 @@ static void readout_dpll_hw_state(struct drm_i915_private *i915,
 
 	pll->on = intel_dpll_get_hw_state(i915, pll, &pll->state.hw_state);
 
-	if ((IS_JASPERLAKE(i915) || IS_ELKHARTLAKE(i915)) &&
-	    pll->on &&
-	    pll->info->id == DPLL_ID_EHL_DPLL4) {
-		pll->wakeref = intel_display_power_get(i915,
-						       POWER_DOMAIN_DC_OFF);
-	}
+	if (pll->on && pll->info->power_domain)
+		pll->wakeref = intel_display_power_get(i915, pll->info->power_domain);
 
 	pll->state.pipe_mask = 0;
 	for_each_intel_crtc(&i915->drm, crtc) {
@@ -4417,8 +4421,7 @@ static void sanitize_dpll_state(struct drm_i915_private *i915,
 		    "%s enabled but not in use, disabling\n",
 		    pll->info->name);
 
-	pll->info->funcs->disable(i915, pll);
-	pll->on = false;
+	_intel_disable_shared_dpll(i915, pll);
 }
 
 void intel_dpll_sanitize_state(struct drm_i915_private *i915)
diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.h b/drivers/gpu/drm/i915/display/intel_dpll_mgr.h
index dd4796a61751..2e7ea0d8d3ff 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.h
+++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.h
@@ -27,6 +27,7 @@
 
 #include <linux/types.h>
 
+#include "intel_display_power.h"
 #include "intel_wakeref.h"
 
 #define for_each_shared_dpll(__i915, __pll, __i) \
@@ -270,6 +271,11 @@ struct dpll_info {
 	 */
 	enum intel_dpll_id id;
 
+	/**
+	 * @power_domain: extra power domain required by the DPLL
+	 */
+	enum intel_display_power_domain power_domain;
+
 #define INTEL_DPLL_ALWAYS_ON	(1 << 0)
 	/**
 	 * @flags:
diff --git a/drivers/gpu/drm/i915/display/intel_dpt.c b/drivers/gpu/drm/i915/display/intel_dpt.c
index 48582b31b7f7..2b067cb952f0 100644
--- a/drivers/gpu/drm/i915/display/intel_dpt.c
+++ b/drivers/gpu/drm/i915/display/intel_dpt.c
@@ -332,11 +332,13 @@ void intel_dpt_configure(struct intel_crtc *crtc)
 
 			intel_de_rmw(i915, PLANE_CHICKEN(pipe, plane_id),
 				     PLANE_CHICKEN_DISABLE_DPT,
-				     i915->params.enable_dpt ? 0 : PLANE_CHICKEN_DISABLE_DPT);
+				     i915->display.params.enable_dpt ? 0 :
+				     PLANE_CHICKEN_DISABLE_DPT);
 		}
 	} else if (DISPLAY_VER(i915) == 13) {
 		intel_de_rmw(i915, CHICKEN_MISC_2,
 			     CHICKEN_MISC_DISABLE_DPT,
-			     i915->params.enable_dpt ? 0 : CHICKEN_MISC_DISABLE_DPT);
+			     i915->display.params.enable_dpt ? 0 :
+			     CHICKEN_MISC_DISABLE_DPT);
 	}
 }
diff --git a/drivers/gpu/drm/i915/display/intel_dsb.c b/drivers/gpu/drm/i915/display/intel_dsb.c
index 78b6fe24dcd8..9598d50f68f2 100644
--- a/drivers/gpu/drm/i915/display/intel_dsb.c
+++ b/drivers/gpu/drm/i915/display/intel_dsb.c
@@ -4,9 +4,6 @@
  *
  */
 
-#include "gem/i915_gem_internal.h"
-#include "gem/i915_gem_lmem.h"
-
 #include "i915_drv.h"
 #include "i915_irq.h"
 #include "i915_reg.h"
@@ -14,12 +11,13 @@
 #include "intel_de.h"
 #include "intel_display_types.h"
 #include "intel_dsb.h"
+#include "intel_dsb_buffer.h"
 #include "intel_dsb_regs.h"
 #include "intel_vblank.h"
 #include "intel_vrr.h"
 #include "skl_watermark.h"
 
-struct i915_vma;
+#define CACHELINE_BYTES 64
 
 enum dsb_id {
 	INVALID_DSB = -1,
@@ -32,8 +30,7 @@ enum dsb_id {
 struct intel_dsb {
 	enum dsb_id id;
 
-	u32 *cmd_buf;
-	struct i915_vma *vma;
+	struct intel_dsb_buffer dsb_buf;
 	struct intel_crtc *crtc;
 
 	/*
@@ -109,15 +106,17 @@ static void intel_dsb_dump(struct intel_dsb *dsb)
 {
 	struct intel_crtc *crtc = dsb->crtc;
 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
-	const u32 *buf = dsb->cmd_buf;
 	int i;
 
 	drm_dbg_kms(&i915->drm, "[CRTC:%d:%s] DSB %d commands {\n",
 		    crtc->base.base.id, crtc->base.name, dsb->id);
 	for (i = 0; i < ALIGN(dsb->free_pos, 64 / 4); i += 4)
 		drm_dbg_kms(&i915->drm,
-			    " 0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n",
-			    i * 4, buf[i], buf[i+1], buf[i+2], buf[i+3]);
+			    " 0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i * 4,
+			    intel_dsb_buffer_read(&dsb->dsb_buf, i),
+			    intel_dsb_buffer_read(&dsb->dsb_buf, i + 1),
+			    intel_dsb_buffer_read(&dsb->dsb_buf, i + 2),
+			    intel_dsb_buffer_read(&dsb->dsb_buf, i + 3));
 	drm_dbg_kms(&i915->drm, "}\n");
 }
 
@@ -129,8 +128,6 @@ static bool is_dsb_busy(struct drm_i915_private *i915, enum pipe pipe,
 
 static void intel_dsb_emit(struct intel_dsb *dsb, u32 ldw, u32 udw)
 {
-	u32 *buf = dsb->cmd_buf;
-
 	if (!assert_dsb_has_room(dsb))
 		return;
 
@@ -139,14 +136,13 @@ static void intel_dsb_emit(struct intel_dsb *dsb, u32 ldw, u32 udw)
 
 	dsb->ins_start_offset = dsb->free_pos;
 
-	buf[dsb->free_pos++] = ldw;
-	buf[dsb->free_pos++] = udw;
+	intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos++, ldw);
+	intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos++, udw);
 }
 
 static bool intel_dsb_prev_ins_is_write(struct intel_dsb *dsb,
 					u32 opcode, i915_reg_t reg)
 {
-	const u32 *buf = dsb->cmd_buf;
 	u32 prev_opcode, prev_reg;
 
 	/*
@@ -157,8 +153,10 @@ static bool intel_dsb_prev_ins_is_write(struct intel_dsb *dsb,
 	if (dsb->free_pos == 0)
 		return false;
 
-	prev_opcode = buf[dsb->ins_start_offset + 1] & ~DSB_REG_VALUE_MASK;
-	prev_reg = buf[dsb->ins_start_offset + 1] & DSB_REG_VALUE_MASK;
+	prev_opcode = intel_dsb_buffer_read(&dsb->dsb_buf,
+					    dsb->ins_start_offset + 1) & ~DSB_REG_VALUE_MASK;
+	prev_reg =  intel_dsb_buffer_read(&dsb->dsb_buf,
+					  dsb->ins_start_offset + 1) & DSB_REG_VALUE_MASK;
 
 	return prev_opcode == opcode && prev_reg == i915_mmio_reg_offset(reg);
 }
@@ -191,6 +189,8 @@ static bool intel_dsb_prev_ins_is_indexed_write(struct intel_dsb *dsb, i915_reg_
 void intel_dsb_reg_write(struct intel_dsb *dsb,
 			 i915_reg_t reg, u32 val)
 {
+	u32 old_val;
+
 	/*
 	 * For example the buffer will look like below for 3 dwords for auto
 	 * increment register:
@@ -214,31 +214,32 @@ void intel_dsb_reg_write(struct intel_dsb *dsb,
 			       (DSB_BYTE_EN << DSB_BYTE_EN_SHIFT) |
 			       i915_mmio_reg_offset(reg));
 	} else {
-		u32 *buf = dsb->cmd_buf;
-
 		if (!assert_dsb_has_room(dsb))
 			return;
 
 		/* convert to indexed write? */
 		if (intel_dsb_prev_ins_is_mmio_write(dsb, reg)) {
-			u32 prev_val = buf[dsb->ins_start_offset + 0];
+			u32 prev_val = intel_dsb_buffer_read(&dsb->dsb_buf,
+							     dsb->ins_start_offset + 0);
 
-			buf[dsb->ins_start_offset + 0] = 1; /* count */
-			buf[dsb->ins_start_offset + 1] =
-				(DSB_OPCODE_INDEXED_WRITE << DSB_OPCODE_SHIFT) |
-				i915_mmio_reg_offset(reg);
-			buf[dsb->ins_start_offset + 2] = prev_val;
+			intel_dsb_buffer_write(&dsb->dsb_buf,
+					       dsb->ins_start_offset + 0, 1); /* count */
+			intel_dsb_buffer_write(&dsb->dsb_buf, dsb->ins_start_offset + 1,
+					       (DSB_OPCODE_INDEXED_WRITE << DSB_OPCODE_SHIFT) |
+					       i915_mmio_reg_offset(reg));
+			intel_dsb_buffer_write(&dsb->dsb_buf, dsb->ins_start_offset + 2, prev_val);
 
 			dsb->free_pos++;
 		}
 
-		buf[dsb->free_pos++] = val;
+		intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos++, val);
 		/* Update the count */
-		buf[dsb->ins_start_offset]++;
+		old_val = intel_dsb_buffer_read(&dsb->dsb_buf, dsb->ins_start_offset);
+		intel_dsb_buffer_write(&dsb->dsb_buf, dsb->ins_start_offset, old_val + 1);
 
 		/* if number of data words is odd, then the last dword should be 0.*/
 		if (dsb->free_pos & 0x1)
-			buf[dsb->free_pos] = 0;
+			intel_dsb_buffer_write(&dsb->dsb_buf, dsb->free_pos, 0);
 	}
 }
 
@@ -297,8 +298,8 @@ static void intel_dsb_align_tail(struct intel_dsb *dsb)
 	aligned_tail = ALIGN(tail, CACHELINE_BYTES);
 
 	if (aligned_tail > tail)
-		memset(&dsb->cmd_buf[dsb->free_pos], 0,
-		       aligned_tail - tail);
+		intel_dsb_buffer_memset(&dsb->dsb_buf, dsb->free_pos, 0,
+					aligned_tail - tail);
 
 	dsb->free_pos = aligned_tail / 4;
 }
@@ -317,7 +318,7 @@ void intel_dsb_finish(struct intel_dsb *dsb)
 
 	intel_dsb_align_tail(dsb);
 
-	i915_gem_object_flush_map(dsb->vma->obj);
+	intel_dsb_buffer_flush_map(&dsb->dsb_buf);
 }
 
 static int intel_dsb_dewake_scanline(const struct intel_crtc_state *crtc_state)
@@ -361,7 +362,7 @@ static void _intel_dsb_commit(struct intel_dsb *dsb, u32 ctrl,
 			  ctrl | DSB_ENABLE);
 
 	intel_de_write_fw(dev_priv, DSB_HEAD(pipe, dsb->id),
-			  i915_ggtt_offset(dsb->vma));
+			  intel_dsb_buffer_ggtt_offset(&dsb->dsb_buf));
 
 	if (dewake_scanline >= 0) {
 		int diff, hw_dewake_scanline;
@@ -383,7 +384,7 @@ static void _intel_dsb_commit(struct intel_dsb *dsb, u32 ctrl,
 	}
 
 	intel_de_write_fw(dev_priv, DSB_TAIL(pipe, dsb->id),
-			  i915_ggtt_offset(dsb->vma) + tail);
+			  intel_dsb_buffer_ggtt_offset(&dsb->dsb_buf) + tail);
 }
 
 /**
@@ -408,7 +409,7 @@ void intel_dsb_wait(struct intel_dsb *dsb)
 	enum pipe pipe = crtc->pipe;
 
 	if (wait_for(!is_dsb_busy(dev_priv, pipe, dsb->id), 1)) {
-		u32 offset = i915_ggtt_offset(dsb->vma);
+		u32 offset = intel_dsb_buffer_ggtt_offset(&dsb->dsb_buf);
 
 		intel_de_write_fw(dev_priv, DSB_CTRL(pipe, dsb->id),
 				  DSB_ENABLE | DSB_HALT);
@@ -445,12 +446,9 @@ struct intel_dsb *intel_dsb_prepare(const struct intel_crtc_state *crtc_state,
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
-	struct drm_i915_gem_object *obj;
 	intel_wakeref_t wakeref;
 	struct intel_dsb *dsb;
-	struct i915_vma *vma;
 	unsigned int size;
-	u32 *buf;
 
 	if (!HAS_DSB(i915))
 		return NULL;
@@ -464,37 +462,13 @@ struct intel_dsb *intel_dsb_prepare(const struct intel_crtc_state *crtc_state,
 	/* ~1 qword per instruction, full cachelines */
 	size = ALIGN(max_cmds * 8, CACHELINE_BYTES);
 
-	if (HAS_LMEM(i915)) {
-		obj = i915_gem_object_create_lmem(i915, PAGE_ALIGN(size),
-						  I915_BO_ALLOC_CONTIGUOUS);
-		if (IS_ERR(obj))
-			goto out_put_rpm;
-	} else {
-		obj = i915_gem_object_create_internal(i915, PAGE_ALIGN(size));
-		if (IS_ERR(obj))
-			goto out_put_rpm;
-
-		i915_gem_object_set_cache_coherency(obj, I915_CACHE_NONE);
-	}
-
-	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
-	if (IS_ERR(vma)) {
-		i915_gem_object_put(obj);
-		goto out_put_rpm;
-	}
-
-	buf = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WC);
-	if (IS_ERR(buf)) {
-		i915_vma_unpin_and_release(&vma, I915_VMA_RELEASE_MAP);
+	if (!intel_dsb_buffer_create(crtc, &dsb->dsb_buf, size))
 		goto out_put_rpm;
-	}
 
 	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 
 	dsb->id = DSB1;
-	dsb->vma = vma;
 	dsb->crtc = crtc;
-	dsb->cmd_buf = buf;
 	dsb->size = size / 4; /* in dwords */
 	dsb->free_pos = 0;
 	dsb->ins_start_offset = 0;
@@ -522,6 +496,6 @@ struct intel_dsb *intel_dsb_prepare(const struct intel_crtc_state *crtc_state,
  */
 void intel_dsb_cleanup(struct intel_dsb *dsb)
 {
-	i915_vma_unpin_and_release(&dsb->vma, I915_VMA_RELEASE_MAP);
+	intel_dsb_buffer_cleanup(&dsb->dsb_buf);
 	kfree(dsb);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_dsb_buffer.c b/drivers/gpu/drm/i915/display/intel_dsb_buffer.c
new file mode 100644
index 000000000000..c77d48bda26a
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_dsb_buffer.c
@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2023, Intel Corporation.
+ */
+
+#include "gem/i915_gem_internal.h"
+#include "gem/i915_gem_lmem.h"
+#include "i915_drv.h"
+#include "i915_vma.h"
+#include "intel_display_types.h"
+#include "intel_dsb_buffer.h"
+
+u32 intel_dsb_buffer_ggtt_offset(struct intel_dsb_buffer *dsb_buf)
+{
+	return i915_ggtt_offset(dsb_buf->vma);
+}
+
+void intel_dsb_buffer_write(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val)
+{
+	dsb_buf->cmd_buf[idx] = val;
+}
+
+u32 intel_dsb_buffer_read(struct intel_dsb_buffer *dsb_buf, u32 idx)
+{
+	return dsb_buf->cmd_buf[idx];
+}
+
+void intel_dsb_buffer_memset(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val, size_t size)
+{
+	WARN_ON(idx > (dsb_buf->buf_size - size) / sizeof(*dsb_buf->cmd_buf));
+
+	memset(&dsb_buf->cmd_buf[idx], val, size);
+}
+
+bool intel_dsb_buffer_create(struct intel_crtc *crtc, struct intel_dsb_buffer *dsb_buf, size_t size)
+{
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	struct drm_i915_gem_object *obj;
+	struct i915_vma *vma;
+	u32 *buf;
+
+	if (HAS_LMEM(i915)) {
+		obj = i915_gem_object_create_lmem(i915, PAGE_ALIGN(size),
+						  I915_BO_ALLOC_CONTIGUOUS);
+		if (IS_ERR(obj))
+			return false;
+	} else {
+		obj = i915_gem_object_create_internal(i915, PAGE_ALIGN(size));
+		if (IS_ERR(obj))
+			return false;
+
+		i915_gem_object_set_cache_coherency(obj, I915_CACHE_NONE);
+	}
+
+	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
+	if (IS_ERR(vma)) {
+		i915_gem_object_put(obj);
+		return false;
+	}
+
+	buf = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WC);
+	if (IS_ERR(buf)) {
+		i915_vma_unpin_and_release(&vma, I915_VMA_RELEASE_MAP);
+		return false;
+	}
+
+	dsb_buf->vma = vma;
+	dsb_buf->cmd_buf = buf;
+	dsb_buf->buf_size = size;
+
+	return true;
+}
+
+void intel_dsb_buffer_cleanup(struct intel_dsb_buffer *dsb_buf)
+{
+	i915_vma_unpin_and_release(&dsb_buf->vma, I915_VMA_RELEASE_MAP);
+}
+
+void intel_dsb_buffer_flush_map(struct intel_dsb_buffer *dsb_buf)
+{
+	i915_gem_object_flush_map(dsb_buf->vma->obj);
+}
diff --git a/drivers/gpu/drm/i915/display/intel_dsb_buffer.h b/drivers/gpu/drm/i915/display/intel_dsb_buffer.h
new file mode 100644
index 000000000000..425acd393905
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_dsb_buffer.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2023 Intel Corporation
+ */
+
+#ifndef _INTEL_DSB_BUFFER_H
+#define _INTEL_DSB_BUFFER_H
+
+#include <linux/types.h>
+
+struct intel_crtc;
+struct i915_vma;
+
+struct intel_dsb_buffer {
+	u32 *cmd_buf;
+	struct i915_vma *vma;
+	size_t buf_size;
+};
+
+u32 intel_dsb_buffer_ggtt_offset(struct intel_dsb_buffer *dsb_buf);
+void intel_dsb_buffer_write(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val);
+u32 intel_dsb_buffer_read(struct intel_dsb_buffer *dsb_buf, u32 idx);
+void intel_dsb_buffer_memset(struct intel_dsb_buffer *dsb_buf, u32 idx, u32 val, size_t size);
+bool intel_dsb_buffer_create(struct intel_crtc *crtc, struct intel_dsb_buffer *dsb_buf,
+			     size_t size);
+void intel_dsb_buffer_cleanup(struct intel_dsb_buffer *dsb_buf);
+void intel_dsb_buffer_flush_map(struct intel_dsb_buffer *dsb_buf);
+
+#endif
diff --git a/drivers/gpu/drm/i915/display/intel_fb.c b/drivers/gpu/drm/i915/display/intel_fb.c
index 19b35ece31f1..c1777ea35761 100644
--- a/drivers/gpu/drm/i915/display/intel_fb.c
+++ b/drivers/gpu/drm/i915/display/intel_fb.c
@@ -764,7 +764,7 @@ bool intel_fb_modifier_uses_dpt(struct drm_i915_private *i915, u64 modifier)
 
 bool intel_fb_uses_dpt(const struct drm_framebuffer *fb)
 {
-	return fb && to_i915(fb->dev)->params.enable_dpt &&
+	return fb && to_i915(fb->dev)->display.params.enable_dpt &&
 		intel_fb_modifier_uses_dpt(to_i915(fb->dev), fb->modifier);
 }
 
@@ -1930,10 +1930,10 @@ static int intel_user_framebuffer_dirty(struct drm_framebuffer *fb,
 	if (!atomic_read(&front->bits))
 		return 0;
 
-	if (dma_resv_test_signaled(obj->base.resv, dma_resv_usage_rw(false)))
+	if (dma_resv_test_signaled(intel_bo_to_drm_bo(obj)->resv, dma_resv_usage_rw(false)))
 		goto flush;
 
-	ret = dma_resv_get_singleton(obj->base.resv, dma_resv_usage_rw(false),
+	ret = dma_resv_get_singleton(intel_bo_to_drm_bo(obj)->resv, dma_resv_usage_rw(false),
 				     &fence);
 	if (ret || !fence)
 		goto flush;
diff --git a/drivers/gpu/drm/i915/display/intel_fbc.c b/drivers/gpu/drm/i915/display/intel_fbc.c
index 4820d21cc942..63f389a1707d 100644
--- a/drivers/gpu/drm/i915/display/intel_fbc.c
+++ b/drivers/gpu/drm/i915/display/intel_fbc.c
@@ -608,6 +608,7 @@ static u32 ivb_dpfc_ctl(struct intel_fbc *fbc)
 static void ivb_fbc_activate(struct intel_fbc *fbc)
 {
 	struct drm_i915_private *i915 = fbc->i915;
+	u32 dpfc_ctl;
 
 	if (DISPLAY_VER(i915) >= 10)
 		glk_fbc_program_cfb_stride(fbc);
@@ -617,8 +618,13 @@ static void ivb_fbc_activate(struct intel_fbc *fbc)
 	if (intel_gt_support_legacy_fencing(to_gt(i915)))
 		snb_fbc_program_fence(fbc);
 
+	/* wa_14019417088 Alternative WA*/
+	dpfc_ctl = ivb_dpfc_ctl(fbc);
+	if (DISPLAY_VER(i915) >= 20)
+		intel_de_write(i915, ILK_DPFC_CONTROL(fbc->id), dpfc_ctl);
+
 	intel_de_write(i915, ILK_DPFC_CONTROL(fbc->id),
-		       DPFC_CTL_EN | ivb_dpfc_ctl(fbc));
+		       DPFC_CTL_EN | dpfc_ctl);
 }
 
 static bool ivb_fbc_is_compressing(struct intel_fbc *fbc)
@@ -1022,10 +1028,13 @@ static bool intel_fbc_hw_tracking_covers_screen(const struct intel_plane_state *
 	struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
 	unsigned int effective_w, effective_h, max_w, max_h;
 
-	if (DISPLAY_VER(i915) >= 10) {
+	if (DISPLAY_VER(i915) >= 11) {
+		max_w = 8192;
+		max_h = 4096;
+	} else if (DISPLAY_VER(i915) >= 10) {
 		max_w = 5120;
 		max_h = 4096;
-	} else if (DISPLAY_VER(i915) >= 8 || IS_HASWELL(i915)) {
+	} else if (DISPLAY_VER(i915) >= 7) {
 		max_w = 4096;
 		max_h = 4096;
 	} else if (IS_G4X(i915) || DISPLAY_VER(i915) >= 5) {
@@ -1044,6 +1053,31 @@ static bool intel_fbc_hw_tracking_covers_screen(const struct intel_plane_state *
 	return effective_w <= max_w && effective_h <= max_h;
 }
 
+static bool intel_fbc_plane_size_valid(const struct intel_plane_state *plane_state)
+{
+	struct drm_i915_private *i915 = to_i915(plane_state->uapi.plane->dev);
+	unsigned int w, h, max_w, max_h;
+
+	if (DISPLAY_VER(i915) >= 10) {
+		max_w = 5120;
+		max_h = 4096;
+	} else if (DISPLAY_VER(i915) >= 8 || IS_HASWELL(i915)) {
+		max_w = 4096;
+		max_h = 4096;
+	} else if (IS_G4X(i915) || DISPLAY_VER(i915) >= 5) {
+		max_w = 4096;
+		max_h = 2048;
+	} else {
+		max_w = 2048;
+		max_h = 1536;
+	}
+
+	w = drm_rect_width(&plane_state->uapi.src) >> 16;
+	h = drm_rect_height(&plane_state->uapi.src) >> 16;
+
+	return w <= max_w && h <= max_h;
+}
+
 static bool i8xx_fbc_tiling_valid(const struct intel_plane_state *plane_state)
 {
 	const struct drm_framebuffer *fb = plane_state->hw.fb;
@@ -1174,7 +1208,7 @@ static int intel_fbc_check_plane(struct intel_atomic_state *state,
 		return 0;
 	}
 
-	if (!i915->params.enable_fbc) {
+	if (!i915->display.params.enable_fbc) {
 		plane_state->no_fbc_reason = "disabled per module param or by default";
 		return 0;
 	}
@@ -1241,11 +1275,16 @@ static int intel_fbc_check_plane(struct intel_atomic_state *state,
 		return 0;
 	}
 
-	if (!intel_fbc_hw_tracking_covers_screen(plane_state)) {
+	if (!intel_fbc_plane_size_valid(plane_state)) {
 		plane_state->no_fbc_reason = "plane size too big";
 		return 0;
 	}
 
+	if (!intel_fbc_hw_tracking_covers_screen(plane_state)) {
+		plane_state->no_fbc_reason = "surface size too big";
+		return 0;
+	}
+
 	/*
 	 * Work around a problem on GEN9+ HW, where enabling FBC on a plane
 	 * having a Y offset that isn't divisible by 4 causes FIFO underrun
@@ -1751,8 +1790,8 @@ void intel_fbc_handle_fifo_underrun_irq(struct drm_i915_private *i915)
  */
 static int intel_sanitize_fbc_option(struct drm_i915_private *i915)
 {
-	if (i915->params.enable_fbc >= 0)
-		return !!i915->params.enable_fbc;
+	if (i915->display.params.enable_fbc >= 0)
+		return !!i915->display.params.enable_fbc;
 
 	if (!HAS_FBC(i915))
 		return 0;
@@ -1824,9 +1863,9 @@ void intel_fbc_init(struct drm_i915_private *i915)
 	if (need_fbc_vtd_wa(i915))
 		DISPLAY_RUNTIME_INFO(i915)->fbc_mask = 0;
 
-	i915->params.enable_fbc = intel_sanitize_fbc_option(i915);
+	i915->display.params.enable_fbc = intel_sanitize_fbc_option(i915);
 	drm_dbg_kms(&i915->drm, "Sanitized enable_fbc value: %d\n",
-		    i915->params.enable_fbc);
+		    i915->display.params.enable_fbc);
 
 	for_each_fbc_id(i915, fbc_id)
 		i915->display.fbc[fbc_id] = intel_fbc_create(i915, fbc_id);
diff --git a/drivers/gpu/drm/i915/display/intel_fdi.c b/drivers/gpu/drm/i915/display/intel_fdi.c
index e6429dfebe15..295a0f24ebbf 100644
--- a/drivers/gpu/drm/i915/display/intel_fdi.c
+++ b/drivers/gpu/drm/i915/display/intel_fdi.c
@@ -10,6 +10,7 @@
 #include "intel_crtc.h"
 #include "intel_ddi.h"
 #include "intel_de.h"
+#include "intel_dp.h"
 #include "intel_display_types.h"
 #include "intel_fdi.h"
 #include "intel_fdi_regs.h"
@@ -338,8 +339,11 @@ int ilk_fdi_compute_config(struct intel_crtc *crtc,
 
 	pipe_config->fdi_lanes = lane;
 
-	intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock,
-			       link_bw, &pipe_config->fdi_m_n, false);
+	intel_link_compute_m_n(to_bpp_x16(pipe_config->pipe_bpp),
+			       lane, fdi_dotclock,
+			       link_bw,
+			       intel_dp_bw_fec_overhead(false),
+			       &pipe_config->fdi_m_n);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/display/intel_frontbuffer.c b/drivers/gpu/drm/i915/display/intel_frontbuffer.c
index ec46716b2f49..2ea37c0414a9 100644
--- a/drivers/gpu/drm/i915/display/intel_frontbuffer.c
+++ b/drivers/gpu/drm/i915/display/intel_frontbuffer.c
@@ -265,8 +265,6 @@ static void frontbuffer_release(struct kref *ref)
 	spin_unlock(&intel_bo_to_i915(obj)->display.fb_tracking.lock);
 
 	i915_active_fini(&front->write);
-
-	i915_gem_object_put(obj);
 	kfree_rcu(front, rcu);
 }
 
diff --git a/drivers/gpu/drm/i915/display/intel_hdcp.c b/drivers/gpu/drm/i915/display/intel_hdcp.c
index c89da3568ebd..39b3f7c0c77c 100644
--- a/drivers/gpu/drm/i915/display/intel_hdcp.c
+++ b/drivers/gpu/drm/i915/display/intel_hdcp.c
@@ -923,7 +923,7 @@ static int _intel_hdcp_disable(struct intel_connector *connector)
 	return 0;
 }
 
-static int _intel_hdcp_enable(struct intel_connector *connector)
+static int intel_hdcp1_enable(struct intel_connector *connector)
 {
 	struct drm_i915_private *i915 = to_i915(connector->base.dev);
 	struct intel_hdcp *hdcp = &connector->hdcp;
@@ -1058,7 +1058,7 @@ static int intel_hdcp_check_link(struct intel_connector *connector)
 		goto out;
 	}
 
-	ret = _intel_hdcp_enable(connector);
+	ret = intel_hdcp1_enable(connector);
 	if (ret) {
 		drm_err(&i915->drm, "Failed to enable hdcp (%d)\n", ret);
 		intel_hdcp_update_value(connector,
@@ -2324,10 +2324,10 @@ intel_hdcp_set_streams(struct intel_digital_port *dig_port,
 	return 0;
 }
 
-int intel_hdcp_enable(struct intel_atomic_state *state,
-		      struct intel_encoder *encoder,
-		      const struct intel_crtc_state *pipe_config,
-		      const struct drm_connector_state *conn_state)
+static int _intel_hdcp_enable(struct intel_atomic_state *state,
+			      struct intel_encoder *encoder,
+			      const struct intel_crtc_state *pipe_config,
+			      const struct drm_connector_state *conn_state)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	struct intel_connector *connector =
@@ -2388,7 +2388,7 @@ int intel_hdcp_enable(struct intel_atomic_state *state,
 	 */
 	if (ret && intel_hdcp_capable(connector) &&
 	    hdcp->content_type != DRM_MODE_HDCP_CONTENT_TYPE1) {
-		ret = _intel_hdcp_enable(connector);
+		ret = intel_hdcp1_enable(connector);
 	}
 
 	if (!ret) {
@@ -2404,6 +2404,27 @@ int intel_hdcp_enable(struct intel_atomic_state *state,
 	return ret;
 }
 
+void intel_hdcp_enable(struct intel_atomic_state *state,
+		       struct intel_encoder *encoder,
+		       const struct intel_crtc_state *crtc_state,
+		       const struct drm_connector_state *conn_state)
+{
+	struct intel_connector *connector =
+		to_intel_connector(conn_state->connector);
+	struct intel_hdcp *hdcp = &connector->hdcp;
+
+	/*
+	 * Enable hdcp if it's desired or if userspace is enabled and
+	 * driver set its state to undesired
+	 */
+	if (conn_state->content_protection ==
+	    DRM_MODE_CONTENT_PROTECTION_DESIRED ||
+	    (conn_state->content_protection ==
+	    DRM_MODE_CONTENT_PROTECTION_ENABLED && hdcp->value ==
+	    DRM_MODE_CONTENT_PROTECTION_UNDESIRED))
+		_intel_hdcp_enable(state, encoder, crtc_state, conn_state);
+}
+
 int intel_hdcp_disable(struct intel_connector *connector)
 {
 	struct intel_digital_port *dig_port = intel_attached_dig_port(connector);
@@ -2491,7 +2512,7 @@ void intel_hdcp_update_pipe(struct intel_atomic_state *state,
 	}
 
 	if (desired_and_not_enabled || content_protection_type_changed)
-		intel_hdcp_enable(state, encoder, crtc_state, conn_state);
+		_intel_hdcp_enable(state, encoder, crtc_state, conn_state);
 }
 
 void intel_hdcp_component_fini(struct drm_i915_private *i915)
diff --git a/drivers/gpu/drm/i915/display/intel_hdcp.h b/drivers/gpu/drm/i915/display/intel_hdcp.h
index 5997c52a0958..a9c784fd9ba5 100644
--- a/drivers/gpu/drm/i915/display/intel_hdcp.h
+++ b/drivers/gpu/drm/i915/display/intel_hdcp.h
@@ -28,10 +28,10 @@ void intel_hdcp_atomic_check(struct drm_connector *connector,
 int intel_hdcp_init(struct intel_connector *connector,
 		    struct intel_digital_port *dig_port,
 		    const struct intel_hdcp_shim *hdcp_shim);
-int intel_hdcp_enable(struct intel_atomic_state *state,
-		      struct intel_encoder *encoder,
-		      const struct intel_crtc_state *pipe_config,
-		      const struct drm_connector_state *conn_state);
+void intel_hdcp_enable(struct intel_atomic_state *state,
+		       struct intel_encoder *encoder,
+		       const struct intel_crtc_state *pipe_config,
+		       const struct drm_connector_state *conn_state);
 int intel_hdcp_disable(struct intel_connector *connector);
 void intel_hdcp_update_pipe(struct intel_atomic_state *state,
 			    struct intel_encoder *encoder,
diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.c b/drivers/gpu/drm/i915/display/intel_hdmi.c
index ac315f8e7820..ab18cfc19c0a 100644
--- a/drivers/gpu/drm/i915/display/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/display/intel_hdmi.c
@@ -3030,16 +3030,6 @@ void intel_hdmi_init_connector(struct intel_digital_port *dig_port,
 				    "HDCP init failed, skipping.\n");
 	}
 
-	/* For G4X desktop chip, PEG_BAND_GAP_DATA 3:0 must first be written
-	 * 0xd.  Failure to do so will result in spurious interrupts being
-	 * generated on the port when a cable is not attached.
-	 */
-	if (IS_G45(dev_priv)) {
-		u32 temp = intel_de_read(dev_priv, PEG_BAND_GAP_DATA);
-		intel_de_write(dev_priv, PEG_BAND_GAP_DATA,
-		               (temp & ~0xf) | 0xd);
-	}
-
 	cec_fill_conn_info_from_drm(&conn_info, connector);
 
 	intel_hdmi->cec_notifier =
diff --git a/drivers/gpu/drm/i915/display/intel_hotplug_irq.c b/drivers/gpu/drm/i915/display/intel_hotplug_irq.c
index f07047e9cb30..04f62f27ad74 100644
--- a/drivers/gpu/drm/i915/display/intel_hotplug_irq.c
+++ b/drivers/gpu/drm/i915/display/intel_hotplug_irq.c
@@ -1361,11 +1361,24 @@ static void bxt_hpd_irq_setup(struct drm_i915_private *dev_priv)
 	bxt_hpd_detection_setup(dev_priv);
 }
 
+static void g45_hpd_peg_band_gap_wa(struct drm_i915_private *i915)
+{
+	/*
+	 * For G4X desktop chip, PEG_BAND_GAP_DATA 3:0 must first be written
+	 * 0xd.  Failure to do so will result in spurious interrupts being
+	 * generated on the port when a cable is not attached.
+	 */
+	intel_de_rmw(i915, PEG_BAND_GAP_DATA, 0xf, 0xd);
+}
+
 static void i915_hpd_enable_detection(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	u32 hotplug_en = hpd_mask_i915[encoder->hpd_pin];
 
+	if (IS_G45(i915))
+		g45_hpd_peg_band_gap_wa(i915);
+
 	/* HPD sense and interrupt enable are one and the same */
 	i915_hotplug_interrupt_update(i915, hotplug_en, hotplug_en);
 }
@@ -1389,6 +1402,9 @@ static void i915_hpd_irq_setup(struct drm_i915_private *dev_priv)
 		hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
 	hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
 
+	if (IS_G45(dev_priv))
+		g45_hpd_peg_band_gap_wa(dev_priv);
+
 	/* Ignore TV since it's buggy */
 	i915_hotplug_interrupt_update_locked(dev_priv,
 					     HOTPLUG_INT_EN_MASK |
diff --git a/drivers/gpu/drm/i915/display/intel_link_bw.c b/drivers/gpu/drm/i915/display/intel_link_bw.c
index c5eb5f242536..9c6d35a405a1 100644
--- a/drivers/gpu/drm/i915/display/intel_link_bw.c
+++ b/drivers/gpu/drm/i915/display/intel_link_bw.c
@@ -7,6 +7,7 @@
 
 #include "intel_atomic.h"
 #include "intel_display_types.h"
+#include "intel_dp_mst.h"
 #include "intel_fdi.h"
 #include "intel_link_bw.h"
 
@@ -21,6 +22,7 @@ void intel_link_bw_init_limits(struct drm_i915_private *i915, struct intel_link_
 {
 	enum pipe pipe;
 
+	limits->force_fec_pipes = 0;
 	limits->bpp_limit_reached_pipes = 0;
 	for_each_pipe(i915, pipe)
 		limits->max_bpp_x16[pipe] = INT_MAX;
@@ -53,11 +55,11 @@ int intel_link_bw_reduce_bpp(struct intel_atomic_state *state,
 	struct drm_i915_private *i915 = to_i915(state->base.dev);
 	enum pipe max_bpp_pipe = INVALID_PIPE;
 	struct intel_crtc *crtc;
-	int max_bpp = 0;
+	int max_bpp_x16 = 0;
 
 	for_each_intel_crtc_in_pipe_mask(&i915->drm, crtc, pipe_mask) {
 		struct intel_crtc_state *crtc_state;
-		int link_bpp;
+		int link_bpp_x16;
 
 		if (limits->bpp_limit_reached_pipes & BIT(crtc->pipe))
 			continue;
@@ -68,7 +70,7 @@ int intel_link_bw_reduce_bpp(struct intel_atomic_state *state,
 			return PTR_ERR(crtc_state);
 
 		if (crtc_state->dsc.compression_enable)
-			link_bpp = crtc_state->dsc.compressed_bpp;
+			link_bpp_x16 = crtc_state->dsc.compressed_bpp_x16;
 		else
 			/*
 			 * TODO: for YUV420 the actual link bpp is only half
@@ -76,10 +78,10 @@ int intel_link_bw_reduce_bpp(struct intel_atomic_state *state,
 			 * is based on the pipe bpp value, set the actual link bpp
 			 * limit here once the MST BW allocation is fixed.
 			 */
-			link_bpp = crtc_state->pipe_bpp;
+			link_bpp_x16 = to_bpp_x16(crtc_state->pipe_bpp);
 
-		if (link_bpp > max_bpp) {
-			max_bpp = link_bpp;
+		if (link_bpp_x16 > max_bpp_x16) {
+			max_bpp_x16 = link_bpp_x16;
 			max_bpp_pipe = crtc->pipe;
 		}
 	}
@@ -87,7 +89,7 @@ int intel_link_bw_reduce_bpp(struct intel_atomic_state *state,
 	if (max_bpp_pipe == INVALID_PIPE)
 		return -ENOSPC;
 
-	limits->max_bpp_x16[max_bpp_pipe] = to_bpp_x16(max_bpp) - 1;
+	limits->max_bpp_x16[max_bpp_pipe] = max_bpp_x16 - 1;
 
 	return intel_modeset_pipes_in_mask_early(state, reason,
 						 BIT(max_bpp_pipe));
@@ -143,6 +145,10 @@ static int check_all_link_config(struct intel_atomic_state *state,
 	/* TODO: Check additional shared display link configurations like MST */
 	int ret;
 
+	ret = intel_dp_mst_atomic_check_link(state, limits);
+	if (ret)
+		return ret;
+
 	ret = intel_fdi_atomic_check_link(state, limits);
 	if (ret)
 		return ret;
@@ -158,6 +164,12 @@ assert_link_limit_change_valid(struct drm_i915_private *i915,
 	bool bpps_changed = false;
 	enum pipe pipe;
 
+	/* FEC can't be forced off after it was forced on. */
+	if (drm_WARN_ON(&i915->drm,
+			(old_limits->force_fec_pipes & new_limits->force_fec_pipes) !=
+			old_limits->force_fec_pipes))
+		return false;
+
 	for_each_pipe(i915, pipe) {
 		/* The bpp limit can only decrease. */
 		if (drm_WARN_ON(&i915->drm,
@@ -172,7 +184,9 @@ assert_link_limit_change_valid(struct drm_i915_private *i915,
 
 	/* At least one limit must change. */
 	if (drm_WARN_ON(&i915->drm,
-			!bpps_changed))
+			!bpps_changed &&
+			new_limits->force_fec_pipes ==
+			old_limits->force_fec_pipes))
 		return false;
 
 	return true;
diff --git a/drivers/gpu/drm/i915/display/intel_link_bw.h b/drivers/gpu/drm/i915/display/intel_link_bw.h
index e07df22a779a..2cf57307cc24 100644
--- a/drivers/gpu/drm/i915/display/intel_link_bw.h
+++ b/drivers/gpu/drm/i915/display/intel_link_bw.h
@@ -16,6 +16,7 @@ struct intel_atomic_state;
 struct intel_crtc_state;
 
 struct intel_link_bw_limits {
+	u8 force_fec_pipes;
 	u8 bpp_limit_reached_pipes;
 	/* in 1/16 bpp units */
 	int max_bpp_x16[I915_MAX_PIPES];
diff --git a/drivers/gpu/drm/i915/display/intel_lvds.c b/drivers/gpu/drm/i915/display/intel_lvds.c
index 2a4ca7e65775..4b114fde57b1 100644
--- a/drivers/gpu/drm/i915/display/intel_lvds.c
+++ b/drivers/gpu/drm/i915/display/intel_lvds.c
@@ -794,8 +794,8 @@ static bool compute_is_dual_link_lvds(struct intel_lvds_encoder *lvds_encoder)
 	unsigned int val;
 
 	/* use the module option value if specified */
-	if (i915->params.lvds_channel_mode > 0)
-		return i915->params.lvds_channel_mode == 2;
+	if (i915->display.params.lvds_channel_mode > 0)
+		return i915->display.params.lvds_channel_mode == 2;
 
 	/* single channel LVDS is limited to 112 MHz */
 	if (fixed_mode->clock > 112999)
diff --git a/drivers/gpu/drm/i915/display/intel_modeset_setup.c b/drivers/gpu/drm/i915/display/intel_modeset_setup.c
index b8f43efb0ab5..94eece7f63be 100644
--- a/drivers/gpu/drm/i915/display/intel_modeset_setup.c
+++ b/drivers/gpu/drm/i915/display/intel_modeset_setup.c
@@ -318,6 +318,12 @@ static void intel_modeset_update_connector_atomic_state(struct drm_i915_private
 			const struct intel_crtc_state *crtc_state =
 				to_intel_crtc_state(crtc->base.state);
 
+			if (crtc_state->dsc.compression_enable) {
+				drm_WARN_ON(&i915->drm, !connector->dp.dsc_decompression_aux);
+				connector->dp.dsc_decompression_enabled = true;
+			} else {
+				connector->dp.dsc_decompression_enabled = false;
+			}
 			conn_state->max_bpc = (crtc_state->pipe_bpp ?: 24) / 3;
 		}
 	}
diff --git a/drivers/gpu/drm/i915/display/intel_modeset_verify.c b/drivers/gpu/drm/i915/display/intel_modeset_verify.c
index 5e1c2c780412..076298a8d405 100644
--- a/drivers/gpu/drm/i915/display/intel_modeset_verify.c
+++ b/drivers/gpu/drm/i915/display/intel_modeset_verify.c
@@ -244,7 +244,7 @@ void intel_modeset_verify_crtc(struct intel_atomic_state *state,
 	verify_crtc_state(state, crtc);
 	intel_shared_dpll_state_verify(state, crtc);
 	intel_mpllb_state_verify(state, crtc);
-	intel_c10pll_state_verify(state, crtc);
+	intel_cx0pll_state_verify(state, crtc);
 }
 
 void intel_modeset_verify_disabled(struct intel_atomic_state *state)
diff --git a/drivers/gpu/drm/i915/display/intel_opregion.c b/drivers/gpu/drm/i915/display/intel_opregion.c
index 84078fb82b2f..1ce785db6a5e 100644
--- a/drivers/gpu/drm/i915/display/intel_opregion.c
+++ b/drivers/gpu/drm/i915/display/intel_opregion.c
@@ -841,7 +841,7 @@ static int intel_load_vbt_firmware(struct drm_i915_private *dev_priv)
 {
 	struct intel_opregion *opregion = &dev_priv->display.opregion;
 	const struct firmware *fw = NULL;
-	const char *name = dev_priv->params.vbt_firmware;
+	const char *name = dev_priv->display.params.vbt_firmware;
 	int ret;
 
 	if (!name || !*name)
diff --git a/drivers/gpu/drm/i915/display/intel_panel.c b/drivers/gpu/drm/i915/display/intel_panel.c
index 483beedac5b8..0d8e5320a4f8 100644
--- a/drivers/gpu/drm/i915/display/intel_panel.c
+++ b/drivers/gpu/drm/i915/display/intel_panel.c
@@ -46,8 +46,8 @@
 
 bool intel_panel_use_ssc(struct drm_i915_private *i915)
 {
-	if (i915->params.panel_use_ssc >= 0)
-		return i915->params.panel_use_ssc != 0;
+	if (i915->display.params.panel_use_ssc >= 0)
+		return i915->display.params.panel_use_ssc != 0;
 	return i915->display.vbt.lvds_use_ssc &&
 		!intel_has_quirk(i915, QUIRK_LVDS_SSC_DISABLE);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_pch_display.c b/drivers/gpu/drm/i915/display/intel_pch_display.c
index 866786e6b32f..baf679759e00 100644
--- a/drivers/gpu/drm/i915/display/intel_pch_display.c
+++ b/drivers/gpu/drm/i915/display/intel_pch_display.c
@@ -8,6 +8,7 @@
 #include "intel_crt.h"
 #include "intel_de.h"
 #include "intel_display_types.h"
+#include "intel_dpll.h"
 #include "intel_fdi.h"
 #include "intel_fdi_regs.h"
 #include "intel_lvds.h"
diff --git a/drivers/gpu/drm/i915/display/intel_pps.c b/drivers/gpu/drm/i915/display/intel_pps.c
index 73f0f1714b37..a8fa3a20990e 100644
--- a/drivers/gpu/drm/i915/display/intel_pps.c
+++ b/drivers/gpu/drm/i915/display/intel_pps.c
@@ -90,7 +90,7 @@ vlv_power_sequencer_kick(struct intel_dp *intel_dp)
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum pipe pipe = intel_dp->pps.pps_pipe;
 	bool pll_enabled, release_cl_override = false;
-	enum dpio_phy phy = DPIO_PHY(pipe);
+	enum dpio_phy phy = vlv_pipe_to_phy(pipe);
 	enum dpio_channel ch = vlv_pipe_to_channel(pipe);
 	u32 DP;
 
diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c
index 4f1f31fc9529..8d180132a74b 100644
--- a/drivers/gpu/drm/i915/display/intel_psr.c
+++ b/drivers/gpu/drm/i915/display/intel_psr.c
@@ -29,6 +29,7 @@
 #include "i915_reg.h"
 #include "intel_atomic.h"
 #include "intel_crtc.h"
+#include "intel_ddi.h"
 #include "intel_de.h"
 #include "intel_display_types.h"
 #include "intel_dp.h"
@@ -172,6 +173,15 @@
  * irrelevant for normal operation.
  */
 
+bool intel_encoder_can_psr(struct intel_encoder *encoder)
+{
+	if (intel_encoder_is_dp(encoder) || encoder->type == INTEL_OUTPUT_DP_MST)
+		return CAN_PSR(enc_to_intel_dp(encoder)) ||
+		       CAN_PANEL_REPLAY(enc_to_intel_dp(encoder));
+	else
+		return false;
+}
+
 static bool psr_global_enabled(struct intel_dp *intel_dp)
 {
 	struct intel_connector *connector = intel_dp->attached_connector;
@@ -179,9 +189,9 @@ static bool psr_global_enabled(struct intel_dp *intel_dp)
 
 	switch (intel_dp->psr.debug & I915_PSR_DEBUG_MODE_MASK) {
 	case I915_PSR_DEBUG_DEFAULT:
-		if (i915->params.enable_psr == -1)
+		if (i915->display.params.enable_psr == -1)
 			return connector->panel.vbt.psr.enable;
-		return i915->params.enable_psr;
+		return i915->display.params.enable_psr;
 	case I915_PSR_DEBUG_DISABLE:
 		return false;
 	default:
@@ -198,7 +208,7 @@ static bool psr2_global_enabled(struct intel_dp *intel_dp)
 	case I915_PSR_DEBUG_FORCE_PSR1:
 		return false;
 	default:
-		if (i915->params.enable_psr == 1)
+		if (i915->display.params.enable_psr == 1)
 			return false;
 		return true;
 	}
@@ -474,27 +484,41 @@ static void intel_dp_get_su_granularity(struct intel_dp *intel_dp)
 	intel_dp->psr.su_y_granularity = y;
 }
 
-void intel_psr_init_dpcd(struct intel_dp *intel_dp)
+static void _panel_replay_init_dpcd(struct intel_dp *intel_dp)
 {
-	struct drm_i915_private *dev_priv =
-		to_i915(dp_to_dig_port(intel_dp)->base.base.dev);
+	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
+	u8 pr_dpcd = 0;
 
-	drm_dp_dpcd_read(&intel_dp->aux, DP_PSR_SUPPORT, intel_dp->psr_dpcd,
-			 sizeof(intel_dp->psr_dpcd));
+	intel_dp->psr.sink_panel_replay_support = false;
+	drm_dp_dpcd_readb(&intel_dp->aux, DP_PANEL_REPLAY_CAP, &pr_dpcd);
 
-	if (!intel_dp->psr_dpcd[0])
+	if (!(pr_dpcd & DP_PANEL_REPLAY_SUPPORT)) {
+		drm_dbg_kms(&i915->drm,
+			    "Panel replay is not supported by panel\n");
 		return;
-	drm_dbg_kms(&dev_priv->drm, "eDP panel supports PSR version %x\n",
+	}
+
+	drm_dbg_kms(&i915->drm,
+		    "Panel replay is supported by panel\n");
+	intel_dp->psr.sink_panel_replay_support = true;
+}
+
+static void _psr_init_dpcd(struct intel_dp *intel_dp)
+{
+	struct drm_i915_private *i915 =
+		to_i915(dp_to_dig_port(intel_dp)->base.base.dev);
+
+	drm_dbg_kms(&i915->drm, "eDP panel supports PSR version %x\n",
 		    intel_dp->psr_dpcd[0]);
 
 	if (drm_dp_has_quirk(&intel_dp->desc, DP_DPCD_QUIRK_NO_PSR)) {
-		drm_dbg_kms(&dev_priv->drm,
+		drm_dbg_kms(&i915->drm,
 			    "PSR support not currently available for this panel\n");
 		return;
 	}
 
 	if (!(intel_dp->edp_dpcd[1] & DP_EDP_SET_POWER_CAP)) {
-		drm_dbg_kms(&dev_priv->drm,
+		drm_dbg_kms(&i915->drm,
 			    "Panel lacks power state control, PSR cannot be enabled\n");
 		return;
 	}
@@ -503,8 +527,8 @@ void intel_psr_init_dpcd(struct intel_dp *intel_dp)
 	intel_dp->psr.sink_sync_latency =
 		intel_dp_get_sink_sync_latency(intel_dp);
 
-	if (DISPLAY_VER(dev_priv) >= 9 &&
-	    (intel_dp->psr_dpcd[0] == DP_PSR2_WITH_Y_COORD_IS_SUPPORTED)) {
+	if (DISPLAY_VER(i915) >= 9 &&
+	    intel_dp->psr_dpcd[0] == DP_PSR2_WITH_Y_COORD_IS_SUPPORTED) {
 		bool y_req = intel_dp->psr_dpcd[1] &
 			     DP_PSR2_SU_Y_COORDINATE_REQUIRED;
 		bool alpm = intel_dp_get_alpm_status(intel_dp);
@@ -521,14 +545,25 @@ void intel_psr_init_dpcd(struct intel_dp *intel_dp)
 		 * GTC first.
 		 */
 		intel_dp->psr.sink_psr2_support = y_req && alpm;
-		drm_dbg_kms(&dev_priv->drm, "PSR2 %ssupported\n",
+		drm_dbg_kms(&i915->drm, "PSR2 %ssupported\n",
 			    intel_dp->psr.sink_psr2_support ? "" : "not ");
+	}
+}
 
-		if (intel_dp->psr.sink_psr2_support) {
-			intel_dp->psr.colorimetry_support =
-				intel_dp_get_colorimetry_status(intel_dp);
-			intel_dp_get_su_granularity(intel_dp);
-		}
+void intel_psr_init_dpcd(struct intel_dp *intel_dp)
+{
+	_panel_replay_init_dpcd(intel_dp);
+
+	drm_dp_dpcd_read(&intel_dp->aux, DP_PSR_SUPPORT, intel_dp->psr_dpcd,
+			 sizeof(intel_dp->psr_dpcd));
+
+	if (intel_dp->psr_dpcd[0])
+		_psr_init_dpcd(intel_dp);
+
+	if (intel_dp->psr.sink_psr2_support) {
+		intel_dp->psr.colorimetry_support =
+			intel_dp_get_colorimetry_status(intel_dp);
+		intel_dp_get_su_granularity(intel_dp);
 	}
 }
 
@@ -574,8 +609,11 @@ static void intel_psr_enable_sink(struct intel_dp *intel_dp)
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 	u8 dpcd_val = DP_PSR_ENABLE;
 
-	/* Enable ALPM at sink for psr2 */
+	if (intel_dp->psr.panel_replay_enabled)
+		return;
+
 	if (intel_dp->psr.psr2_enabled) {
+		/* Enable ALPM at sink for psr2 */
 		drm_dp_dpcd_writeb(&intel_dp->aux, DP_RECEIVER_ALPM_CONFIG,
 				   DP_ALPM_ENABLE |
 				   DP_ALPM_LOCK_ERROR_IRQ_HPD_ENABLE);
@@ -592,6 +630,9 @@ static void intel_psr_enable_sink(struct intel_dp *intel_dp)
 	if (intel_dp->psr.req_psr2_sdp_prior_scanline)
 		dpcd_val |= DP_PSR_SU_REGION_SCANLINE_CAPTURE;
 
+	if (intel_dp->psr.entry_setup_frames > 0)
+		dpcd_val |= DP_PSR_FRAME_CAPTURE;
+
 	drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, dpcd_val);
 
 	drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
@@ -606,7 +647,7 @@ static u32 intel_psr1_get_tp_time(struct intel_dp *intel_dp)
 	if (DISPLAY_VER(dev_priv) >= 11)
 		val |= EDP_PSR_TP4_TIME_0us;
 
-	if (dev_priv->params.psr_safest_params) {
+	if (dev_priv->display.params.psr_safest_params) {
 		val |= EDP_PSR_TP1_TIME_2500us;
 		val |= EDP_PSR_TP2_TP3_TIME_2500us;
 		goto check_tp3_sel;
@@ -690,6 +731,9 @@ static void hsw_activate_psr1(struct intel_dp *intel_dp)
 	if (DISPLAY_VER(dev_priv) >= 8)
 		val |= EDP_PSR_CRC_ENABLE;
 
+	if (DISPLAY_VER(dev_priv) >= 20)
+		val |= LNL_EDP_PSR_ENTRY_SETUP_FRAMES(intel_dp->psr.entry_setup_frames);
+
 	intel_de_rmw(dev_priv, psr_ctl_reg(dev_priv, cpu_transcoder),
 		     ~EDP_PSR_RESTORE_PSR_ACTIVE_CTX_MASK, val);
 }
@@ -700,7 +744,7 @@ static u32 intel_psr2_get_tp_time(struct intel_dp *intel_dp)
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 	u32 val = 0;
 
-	if (dev_priv->params.psr_safest_params)
+	if (dev_priv->display.params.psr_safest_params)
 		return EDP_PSR2_TP2_TIME_2500us;
 
 	if (connector->panel.vbt.psr.psr2_tp2_tp3_wakeup_time_us >= 0 &&
@@ -727,11 +771,38 @@ static int psr2_block_count(struct intel_dp *intel_dp)
 	return psr2_block_count_lines(intel_dp) / 4;
 }
 
+static u8 frames_before_su_entry(struct intel_dp *intel_dp)
+{
+	u8 frames_before_su_entry;
+
+	frames_before_su_entry = max_t(u8,
+				       intel_dp->psr.sink_sync_latency + 1,
+				       2);
+
+	/* Entry setup frames must be at least 1 less than frames before SU entry */
+	if (intel_dp->psr.entry_setup_frames >= frames_before_su_entry)
+		frames_before_su_entry = intel_dp->psr.entry_setup_frames + 1;
+
+	return frames_before_su_entry;
+}
+
+static void dg2_activate_panel_replay(struct intel_dp *intel_dp)
+{
+	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+
+	intel_de_rmw(dev_priv, PSR2_MAN_TRK_CTL(intel_dp->psr.transcoder),
+		     0, ADLP_PSR2_MAN_TRK_CTL_SF_CONTINUOS_FULL_FRAME);
+
+	intel_de_rmw(dev_priv, TRANS_DP2_CTL(intel_dp->psr.transcoder), 0,
+		     TRANS_DP2_PANEL_REPLAY_ENABLE);
+}
+
 static void hsw_activate_psr2(struct intel_dp *intel_dp)
 {
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
 	u32 val = EDP_PSR2_ENABLE;
+	u32 psr_val = 0;
 
 	val |= EDP_PSR2_IDLE_FRAMES(psr_compute_idle_frames(intel_dp));
 
@@ -741,7 +812,8 @@ static void hsw_activate_psr2(struct intel_dp *intel_dp)
 	if (DISPLAY_VER(dev_priv) >= 10 && DISPLAY_VER(dev_priv) <= 12)
 		val |= EDP_Y_COORDINATE_ENABLE;
 
-	val |= EDP_PSR2_FRAME_BEFORE_SU(max_t(u8, intel_dp->psr.sink_sync_latency + 1, 2));
+	val |= EDP_PSR2_FRAME_BEFORE_SU(frames_before_su_entry(intel_dp));
+
 	val |= intel_psr2_get_tp_time(intel_dp);
 
 	if (DISPLAY_VER(dev_priv) >= 12) {
@@ -785,6 +857,9 @@ static void hsw_activate_psr2(struct intel_dp *intel_dp)
 	if (intel_dp->psr.req_psr2_sdp_prior_scanline)
 		val |= EDP_PSR2_SU_SDP_SCANLINE;
 
+	if (DISPLAY_VER(dev_priv) >= 20)
+		psr_val |= LNL_EDP_PSR_ENTRY_SETUP_FRAMES(intel_dp->psr.entry_setup_frames);
+
 	if (intel_dp->psr.psr2_sel_fetch_enabled) {
 		u32 tmp;
 
@@ -798,7 +873,7 @@ static void hsw_activate_psr2(struct intel_dp *intel_dp)
 	 * PSR2 HW is incorrectly using EDP_PSR_TP1_TP3_SEL and BSpec is
 	 * recommending keep this bit unset while PSR2 is enabled.
 	 */
-	intel_de_write(dev_priv, psr_ctl_reg(dev_priv, cpu_transcoder), 0);
+	intel_de_write(dev_priv, psr_ctl_reg(dev_priv, cpu_transcoder), psr_val);
 
 	intel_de_write(dev_priv, EDP_PSR2_CTL(cpu_transcoder), val);
 }
@@ -943,7 +1018,7 @@ static bool intel_psr2_sel_fetch_config_valid(struct intel_dp *intel_dp,
 {
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 
-	if (!dev_priv->params.enable_psr2_sel_fetch &&
+	if (!dev_priv->display.params.enable_psr2_sel_fetch &&
 	    intel_dp->psr.debug != I915_PSR_DEBUG_ENABLE_SEL_FETCH) {
 		drm_dbg_kms(&dev_priv->drm,
 			    "PSR2 sel fetch not enabled, disabled by parameter\n");
@@ -1056,7 +1131,7 @@ static bool _compute_psr2_wake_times(struct intel_dp *intel_dp,
 	    fast_wake_lines > max_wake_lines)
 		return false;
 
-	if (i915->params.psr_safest_params)
+	if (i915->display.params.psr_safest_params)
 		io_wake_lines = fast_wake_lines = max_wake_lines;
 
 	/* According to Bspec lower limit should be set as 7 lines. */
@@ -1066,6 +1141,39 @@ static bool _compute_psr2_wake_times(struct intel_dp *intel_dp,
 	return true;
 }
 
+static int intel_psr_entry_setup_frames(struct intel_dp *intel_dp,
+					const struct drm_display_mode *adjusted_mode)
+{
+	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
+	int psr_setup_time = drm_dp_psr_setup_time(intel_dp->psr_dpcd);
+	int entry_setup_frames = 0;
+
+	if (psr_setup_time < 0) {
+		drm_dbg_kms(&i915->drm,
+			    "PSR condition failed: Invalid PSR setup time (0x%02x)\n",
+			    intel_dp->psr_dpcd[1]);
+		return -ETIME;
+	}
+
+	if (intel_usecs_to_scanlines(adjusted_mode, psr_setup_time) >
+	    adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vdisplay - 1) {
+		if (DISPLAY_VER(i915) >= 20) {
+			/* setup entry frames can be up to 3 frames */
+			entry_setup_frames = 1;
+			drm_dbg_kms(&i915->drm,
+				    "PSR setup entry frames %d\n",
+				    entry_setup_frames);
+		} else {
+			drm_dbg_kms(&i915->drm,
+				    "PSR condition failed: PSR setup time (%d us) too long\n",
+				    psr_setup_time);
+			return -ETIME;
+		}
+	}
+
+	return entry_setup_frames;
+}
+
 static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
 				    struct intel_crtc_state *crtc_state)
 {
@@ -1206,24 +1314,42 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
 	return false;
 }
 
-void intel_psr_compute_config(struct intel_dp *intel_dp,
-			      struct intel_crtc_state *crtc_state,
-			      struct drm_connector_state *conn_state)
+static bool _psr_compute_config(struct intel_dp *intel_dp,
+				struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
-	const struct drm_display_mode *adjusted_mode =
-		&crtc_state->hw.adjusted_mode;
-	int psr_setup_time;
+	const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
+	u8 entry_setup_frames;
 
 	/*
 	 * Current PSR panels don't work reliably with VRR enabled
 	 * So if VRR is enabled, do not enable PSR.
 	 */
 	if (crtc_state->vrr.enable)
-		return;
+		return false;
 
 	if (!CAN_PSR(intel_dp))
-		return;
+		return false;
+
+	entry_setup_frames = intel_psr_entry_setup_frames(intel_dp, adjusted_mode);
+
+	if (entry_setup_frames >= 0) {
+		intel_dp->psr.entry_setup_frames = entry_setup_frames;
+	} else {
+		drm_dbg_kms(&dev_priv->drm,
+			    "PSR condition failed: PSR setup timing not met\n");
+		return false;
+	}
+
+	return true;
+}
+
+void intel_psr_compute_config(struct intel_dp *intel_dp,
+			      struct intel_crtc_state *crtc_state,
+			      struct drm_connector_state *conn_state)
+{
+	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+	const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
 
 	if (!psr_global_enabled(intel_dp)) {
 		drm_dbg_kms(&dev_priv->drm, "PSR disabled by flag\n");
@@ -1242,23 +1368,11 @@ void intel_psr_compute_config(struct intel_dp *intel_dp,
 		return;
 	}
 
-	psr_setup_time = drm_dp_psr_setup_time(intel_dp->psr_dpcd);
-	if (psr_setup_time < 0) {
-		drm_dbg_kms(&dev_priv->drm,
-			    "PSR condition failed: Invalid PSR setup time (0x%02x)\n",
-			    intel_dp->psr_dpcd[1]);
-		return;
-	}
-
-	if (intel_usecs_to_scanlines(adjusted_mode, psr_setup_time) >
-	    adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vdisplay - 1) {
-		drm_dbg_kms(&dev_priv->drm,
-			    "PSR condition failed: PSR setup time (%d us) too long\n",
-			    psr_setup_time);
-		return;
-	}
+	if (CAN_PANEL_REPLAY(intel_dp))
+		crtc_state->has_panel_replay = true;
+	else
+		crtc_state->has_psr = _psr_compute_config(intel_dp, crtc_state);
 
-	crtc_state->has_psr = true;
 	crtc_state->has_psr2 = intel_psr2_config_valid(intel_dp, crtc_state);
 
 	crtc_state->infoframes.enable |= intel_hdmi_infoframe_enable(DP_SDP_VSC);
@@ -1279,18 +1393,23 @@ void intel_psr_get_config(struct intel_encoder *encoder,
 		return;
 
 	intel_dp = &dig_port->dp;
-	if (!CAN_PSR(intel_dp))
+	if (!(CAN_PSR(intel_dp) || CAN_PANEL_REPLAY(intel_dp)))
 		return;
 
 	mutex_lock(&intel_dp->psr.lock);
 	if (!intel_dp->psr.enabled)
 		goto unlock;
 
-	/*
-	 * Not possible to read EDP_PSR/PSR2_CTL registers as it is
-	 * enabled/disabled because of frontbuffer tracking and others.
-	 */
-	pipe_config->has_psr = true;
+	if (intel_dp->psr.panel_replay_enabled) {
+		pipe_config->has_panel_replay = true;
+	} else {
+		/*
+		 * Not possible to read EDP_PSR/PSR2_CTL registers as it is
+		 * enabled/disabled because of frontbuffer tracking and others.
+		 */
+		pipe_config->has_psr = true;
+	}
+
 	pipe_config->has_psr2 = intel_dp->psr.psr2_enabled;
 	pipe_config->infoframes.enable |= intel_hdmi_infoframe_enable(DP_SDP_VSC);
 
@@ -1327,8 +1446,10 @@ static void intel_psr_activate(struct intel_dp *intel_dp)
 
 	lockdep_assert_held(&intel_dp->psr.lock);
 
-	/* psr1 and psr2 are mutually exclusive.*/
-	if (intel_dp->psr.psr2_enabled)
+	/* psr1, psr2 and panel-replay are mutually exclusive.*/
+	if (intel_dp->psr.panel_replay_enabled)
+		dg2_activate_panel_replay(intel_dp);
+	else if (intel_dp->psr.psr2_enabled)
 		hsw_activate_psr2(intel_dp);
 	else
 		hsw_activate_psr1(intel_dp);
@@ -1452,12 +1573,10 @@ static void intel_psr_enable_source(struct intel_dp *intel_dp,
 		 * All supported adlp panels have 1-based X granularity, this may
 		 * cause issues if non-supported panels are used.
 		 */
-		if (IS_DISPLAY_IP_STEP(dev_priv, IP_VER(14, 0), STEP_A0, STEP_B0))
-			intel_de_rmw(dev_priv, MTL_CHICKEN_TRANS(cpu_transcoder), 0,
-				     ADLP_1_BASED_X_GRANULARITY);
-		else if (IS_ALDERLAKE_P(dev_priv))
-			intel_de_rmw(dev_priv, CHICKEN_TRANS(cpu_transcoder), 0,
-				     ADLP_1_BASED_X_GRANULARITY);
+		if (IS_DISPLAY_IP_STEP(dev_priv, IP_VER(14, 0), STEP_A0, STEP_B0) ||
+		    IS_ALDERLAKE_P(dev_priv))
+			intel_de_rmw(dev_priv, hsw_chicken_trans_reg(dev_priv, cpu_transcoder),
+				     0, ADLP_1_BASED_X_GRANULARITY);
 
 		/* Wa_16012604467:adlp,mtl[a0,b0] */
 		if (IS_DISPLAY_IP_STEP(dev_priv, IP_VER(14, 0), STEP_A0, STEP_B0))
@@ -1508,6 +1627,7 @@ static void intel_psr_enable_locked(struct intel_dp *intel_dp,
 	drm_WARN_ON(&dev_priv->drm, intel_dp->psr.enabled);
 
 	intel_dp->psr.psr2_enabled = crtc_state->has_psr2;
+	intel_dp->psr.panel_replay_enabled = crtc_state->has_panel_replay;
 	intel_dp->psr.busy_frontbuffer_bits = 0;
 	intel_dp->psr.pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe;
 	intel_dp->psr.transcoder = crtc_state->cpu_transcoder;
@@ -1523,8 +1643,12 @@ static void intel_psr_enable_locked(struct intel_dp *intel_dp,
 	if (!psr_interrupt_error_check(intel_dp))
 		return;
 
-	drm_dbg_kms(&dev_priv->drm, "Enabling PSR%s\n",
-		    intel_dp->psr.psr2_enabled ? "2" : "1");
+	if (intel_dp->psr.panel_replay_enabled)
+		drm_dbg_kms(&dev_priv->drm, "Enabling Panel Replay\n");
+	else
+		drm_dbg_kms(&dev_priv->drm, "Enabling PSR%s\n",
+			    intel_dp->psr.psr2_enabled ? "2" : "1");
+
 	intel_write_dp_vsc_sdp(encoder, crtc_state, &crtc_state->psr_vsc);
 	intel_snps_phy_update_psr_power_state(dev_priv, phy, true);
 	intel_psr_enable_sink(intel_dp);
@@ -1553,7 +1677,10 @@ static void intel_psr_exit(struct intel_dp *intel_dp)
 		return;
 	}
 
-	if (intel_dp->psr.psr2_enabled) {
+	if (intel_dp->psr.panel_replay_enabled) {
+		intel_de_rmw(dev_priv, TRANS_DP2_CTL(intel_dp->psr.transcoder),
+			     TRANS_DP2_PANEL_REPLAY_ENABLE, 0);
+	} else if (intel_dp->psr.psr2_enabled) {
 		tgl_disallow_dc3co_on_psr2_exit(intel_dp);
 
 		val = intel_de_rmw(dev_priv, EDP_PSR2_CTL(cpu_transcoder),
@@ -1602,8 +1729,11 @@ static void intel_psr_disable_locked(struct intel_dp *intel_dp)
 	if (!intel_dp->psr.enabled)
 		return;
 
-	drm_dbg_kms(&dev_priv->drm, "Disabling PSR%s\n",
-		    intel_dp->psr.psr2_enabled ? "2" : "1");
+	if (intel_dp->psr.panel_replay_enabled)
+		drm_dbg_kms(&dev_priv->drm, "Disabling Panel Replay\n");
+	else
+		drm_dbg_kms(&dev_priv->drm, "Disabling PSR%s\n",
+			    intel_dp->psr.psr2_enabled ? "2" : "1");
 
 	intel_psr_exit(intel_dp);
 	intel_psr_wait_exit_locked(intel_dp);
@@ -1636,6 +1766,7 @@ static void intel_psr_disable_locked(struct intel_dp *intel_dp)
 		drm_dp_dpcd_writeb(&intel_dp->aux, DP_RECEIVER_ALPM_CONFIG, 0);
 
 	intel_dp->psr.enabled = false;
+	intel_dp->psr.panel_replay_enabled = false;
 	intel_dp->psr.psr2_enabled = false;
 	intel_dp->psr.psr2_sel_fetch_enabled = false;
 	intel_dp->psr.psr2_sel_fetch_cff_enabled = false;
@@ -2207,7 +2338,7 @@ void intel_psr_post_plane_update(struct intel_atomic_state *state,
 		intel_atomic_get_new_crtc_state(state, crtc);
 	struct intel_encoder *encoder;
 
-	if (!crtc_state->has_psr)
+	if (!(crtc_state->has_psr || crtc_state->has_panel_replay))
 		return;
 
 	for_each_intel_encoder_mask_with_psr(state->base.dev, encoder,
@@ -2693,9 +2824,12 @@ void intel_psr_init(struct intel_dp *intel_dp)
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 
-	if (!HAS_PSR(dev_priv))
+	if (!(HAS_PSR(dev_priv) || HAS_DP20(dev_priv)))
 		return;
 
+	if (!intel_dp_is_edp(intel_dp))
+		intel_psr_init_dpcd(intel_dp);
+
 	/*
 	 * HSW spec explicitly says PSR is tied to port A.
 	 * BDW+ platforms have a instance of PSR registers per transcoder but
@@ -2711,7 +2845,10 @@ void intel_psr_init(struct intel_dp *intel_dp)
 		return;
 	}
 
-	intel_dp->psr.source_support = true;
+	if (HAS_DP20(dev_priv) && !intel_dp_is_edp(intel_dp))
+		intel_dp->psr.source_panel_replay_support = true;
+	else
+		intel_dp->psr.source_support = true;
 
 	/* Set link_standby x link_off defaults */
 	if (DISPLAY_VER(dev_priv) < 12)
@@ -2728,12 +2865,19 @@ static int psr_get_status_and_error_status(struct intel_dp *intel_dp,
 {
 	struct drm_dp_aux *aux = &intel_dp->aux;
 	int ret;
+	unsigned int offset;
 
-	ret = drm_dp_dpcd_readb(aux, DP_PSR_STATUS, status);
+	offset = intel_dp->psr.panel_replay_enabled ?
+		 DP_SINK_DEVICE_PR_AND_FRAME_LOCK_STATUS : DP_PSR_STATUS;
+
+	ret = drm_dp_dpcd_readb(aux, offset, status);
 	if (ret != 1)
 		return ret;
 
-	ret = drm_dp_dpcd_readb(aux, DP_PSR_ERROR_STATUS, error_status);
+	offset = intel_dp->psr.panel_replay_enabled ?
+		 DP_PANEL_REPLAY_ERROR_STATUS : DP_PSR_ERROR_STATUS;
+
+	ret = drm_dp_dpcd_readb(aux, offset, error_status);
 	if (ret != 1)
 		return ret;
 
@@ -2954,7 +3098,7 @@ psr_source_status(struct intel_dp *intel_dp, struct seq_file *m)
 			status = live_status[status_val];
 	}
 
-	seq_printf(m, "Source PSR status: %s [0x%08x]\n", status, val);
+	seq_printf(m, "Source PSR/PanelReplay status: %s [0x%08x]\n", status, val);
 }
 
 static int intel_psr_status(struct seq_file *m, struct intel_dp *intel_dp)
@@ -2967,18 +3111,22 @@ static int intel_psr_status(struct seq_file *m, struct intel_dp *intel_dp)
 	bool enabled;
 	u32 val;
 
-	seq_printf(m, "Sink support: %s", str_yes_no(psr->sink_support));
+	seq_printf(m, "Sink support: PSR = %s",
+		   str_yes_no(psr->sink_support));
+
 	if (psr->sink_support)
 		seq_printf(m, " [0x%02x]", intel_dp->psr_dpcd[0]);
-	seq_puts(m, "\n");
+	seq_printf(m, ", Panel Replay = %s\n", str_yes_no(psr->sink_panel_replay_support));
 
-	if (!psr->sink_support)
+	if (!(psr->sink_support || psr->sink_panel_replay_support))
 		return 0;
 
 	wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
 	mutex_lock(&psr->lock);
 
-	if (psr->enabled)
+	if (psr->panel_replay_enabled)
+		status = "Panel Replay Enabled";
+	else if (psr->enabled)
 		status = psr->psr2_enabled ? "PSR2 enabled" : "PSR1 enabled";
 	else
 		status = "disabled";
@@ -2991,14 +3139,17 @@ static int intel_psr_status(struct seq_file *m, struct intel_dp *intel_dp)
 		goto unlock;
 	}
 
-	if (psr->psr2_enabled) {
+	if (psr->panel_replay_enabled) {
+		val = intel_de_read(dev_priv, TRANS_DP2_CTL(cpu_transcoder));
+		enabled = val & TRANS_DP2_PANEL_REPLAY_ENABLE;
+	} else if (psr->psr2_enabled) {
 		val = intel_de_read(dev_priv, EDP_PSR2_CTL(cpu_transcoder));
 		enabled = val & EDP_PSR2_ENABLE;
 	} else {
 		val = intel_de_read(dev_priv, psr_ctl_reg(dev_priv, cpu_transcoder));
 		enabled = val & EDP_PSR_ENABLE;
 	}
-	seq_printf(m, "Source PSR ctl: %s [0x%08x]\n",
+	seq_printf(m, "Source PSR/PanelReplay ctl: %s [0x%08x]\n",
 		   str_enabled_disabled(enabled), val);
 	psr_source_status(intel_dp, m);
 	seq_printf(m, "Busy frontbuffer bits: 0x%08x\n",
@@ -3136,6 +3287,16 @@ void intel_psr_debugfs_register(struct drm_i915_private *i915)
 			    i915, &i915_edp_psr_status_fops);
 }
 
+static const char *psr_mode_str(struct intel_dp *intel_dp)
+{
+	if (intel_dp->psr.panel_replay_enabled)
+		return "PANEL-REPLAY";
+	else if (intel_dp->psr.enabled)
+		return "PSR";
+
+	return "unknown";
+}
+
 static int i915_psr_sink_status_show(struct seq_file *m, void *data)
 {
 	struct intel_connector *connector = m->private;
@@ -3150,12 +3311,19 @@ static int i915_psr_sink_status_show(struct seq_file *m, void *data)
 		"reserved",
 		"sink internal error",
 	};
+	static const char * const panel_replay_status[] = {
+		"Sink device frame is locked to the Source device",
+		"Sink device is coasting, using the VTotal target",
+		"Sink device is governing the frame rate (frame rate unlock is granted)",
+		"Sink device in the process of re-locking with the Source device",
+	};
 	const char *str;
 	int ret;
 	u8 status, error_status;
+	u32 idx;
 
-	if (!CAN_PSR(intel_dp)) {
-		seq_puts(m, "PSR Unsupported\n");
+	if (!(CAN_PSR(intel_dp) || CAN_PANEL_REPLAY(intel_dp))) {
+		seq_puts(m, "PSR/Panel-Replay Unsupported\n");
 		return -ENODEV;
 	}
 
@@ -3166,15 +3334,20 @@ static int i915_psr_sink_status_show(struct seq_file *m, void *data)
 	if (ret)
 		return ret;
 
-	status &= DP_PSR_SINK_STATE_MASK;
-	if (status < ARRAY_SIZE(sink_status))
-		str = sink_status[status];
-	else
-		str = "unknown";
+	str = "unknown";
+	if (intel_dp->psr.panel_replay_enabled) {
+		idx = (status & DP_SINK_FRAME_LOCKED_MASK) >> DP_SINK_FRAME_LOCKED_SHIFT;
+		if (idx < ARRAY_SIZE(panel_replay_status))
+			str = panel_replay_status[idx];
+	} else if (intel_dp->psr.enabled) {
+		idx = status & DP_PSR_SINK_STATE_MASK;
+		if (idx < ARRAY_SIZE(sink_status))
+			str = sink_status[idx];
+	}
 
-	seq_printf(m, "Sink PSR status: 0x%x [%s]\n", status, str);
+	seq_printf(m, "Sink %s status: 0x%x [%s]\n", psr_mode_str(intel_dp), status, str);
 
-	seq_printf(m, "Sink PSR error status: 0x%x", error_status);
+	seq_printf(m, "Sink %s error status: 0x%x", psr_mode_str(intel_dp), error_status);
 
 	if (error_status & (DP_PSR_RFB_STORAGE_ERROR |
 			    DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR |
@@ -3183,11 +3356,11 @@ static int i915_psr_sink_status_show(struct seq_file *m, void *data)
 	else
 		seq_puts(m, "\n");
 	if (error_status & DP_PSR_RFB_STORAGE_ERROR)
-		seq_puts(m, "\tPSR RFB storage error\n");
+		seq_printf(m, "\t%s RFB storage error\n", psr_mode_str(intel_dp));
 	if (error_status & DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR)
-		seq_puts(m, "\tPSR VSC SDP uncorrectable error\n");
+		seq_printf(m, "\t%s VSC SDP uncorrectable error\n", psr_mode_str(intel_dp));
 	if (error_status & DP_PSR_LINK_CRC_ERROR)
-		seq_puts(m, "\tPSR Link CRC error\n");
+		seq_printf(m, "\t%s Link CRC error\n", psr_mode_str(intel_dp));
 
 	return ret;
 }
@@ -3207,13 +3380,16 @@ void intel_psr_connector_debugfs_add(struct intel_connector *connector)
 	struct drm_i915_private *i915 = to_i915(connector->base.dev);
 	struct dentry *root = connector->base.debugfs_entry;
 
-	if (connector->base.connector_type != DRM_MODE_CONNECTOR_eDP)
-		return;
+	if (connector->base.connector_type != DRM_MODE_CONNECTOR_eDP) {
+		if (!(HAS_DP20(i915) &&
+		      connector->base.connector_type == DRM_MODE_CONNECTOR_DisplayPort))
+			return;
+	}
 
 	debugfs_create_file("i915_psr_sink_status", 0444, root,
 			    connector, &i915_psr_sink_status_fops);
 
-	if (HAS_PSR(i915))
+	if (HAS_PSR(i915) || HAS_DP20(i915))
 		debugfs_create_file("i915_psr_status", 0444, root,
 				    connector, &i915_psr_status_fops);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_psr.h b/drivers/gpu/drm/i915/display/intel_psr.h
index bf35f42df6bc..6a1f4573852b 100644
--- a/drivers/gpu/drm/i915/display/intel_psr.h
+++ b/drivers/gpu/drm/i915/display/intel_psr.h
@@ -21,6 +21,13 @@ struct intel_encoder;
 struct intel_plane;
 struct intel_plane_state;
 
+#define CAN_PSR(intel_dp) ((intel_dp)->psr.sink_support && \
+			   (intel_dp)->psr.source_support)
+
+#define CAN_PANEL_REPLAY(intel_dp) ((intel_dp)->psr.sink_panel_replay_support && \
+				    (intel_dp)->psr.source_panel_replay_support)
+
+bool intel_encoder_can_psr(struct intel_encoder *encoder);
 void intel_psr_init_dpcd(struct intel_dp *intel_dp);
 void intel_psr_pre_plane_update(struct intel_atomic_state *state,
 				struct intel_crtc *crtc);
diff --git a/drivers/gpu/drm/i915/display/intel_psr_regs.h b/drivers/gpu/drm/i915/display/intel_psr_regs.h
index d39951383c92..efe4306b37e0 100644
--- a/drivers/gpu/drm/i915/display/intel_psr_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_psr_regs.h
@@ -35,6 +35,8 @@
 #define   EDP_PSR_MIN_LINK_ENTRY_TIME_0_LINES	REG_FIELD_PREP(EDP_PSR_MIN_LINK_ENTRY_TIME_MASK, 3)
 #define   EDP_PSR_MAX_SLEEP_TIME_MASK		REG_GENMASK(24, 20)
 #define   EDP_PSR_MAX_SLEEP_TIME(x)		REG_FIELD_PREP(EDP_PSR_MAX_SLEEP_TIME_MASK, (x))
+#define   LNL_EDP_PSR_ENTRY_SETUP_FRAMES_MASK	REG_GENMASK(17, 16)
+#define   LNL_EDP_PSR_ENTRY_SETUP_FRAMES(x)	REG_FIELD_PREP(LNL_EDP_PSR_ENTRY_SETUP_FRAMES_MASK, (x))
 #define   EDP_PSR_SKIP_AUX_EXIT			REG_BIT(12)
 #define   EDP_PSR_TP_MASK			REG_BIT(11)
 #define   EDP_PSR_TP_TP1_TP2			REG_FIELD_PREP(EDP_PSR_TP_MASK, 0)
diff --git a/drivers/gpu/drm/i915/display/intel_qp_tables.c b/drivers/gpu/drm/i915/display/intel_qp_tables.c
index 543cdc46aa1d..600c815e37e4 100644
--- a/drivers/gpu/drm/i915/display/intel_qp_tables.c
+++ b/drivers/gpu/drm/i915/display/intel_qp_tables.c
@@ -34,9 +34,6 @@
  * These qp tables are as per the C model
  * and it has the rows pointing to bpps which increment
  * in steps of 0.5
- * We do not support fractional bpps as of today,
- * hence we would skip the fractional bpps during
- * our references for qp calclulations.
  */
 static const u8 rc_range_minqp444_8bpc[DSC_NUM_BUF_RANGES][RC_RANGE_QP444_8BPC_MAX_NUM_BPP] = {
 	{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
diff --git a/drivers/gpu/drm/i915/display/intel_sdvo.c b/drivers/gpu/drm/i915/display/intel_sdvo.c
index a636f42ceae5..3eac559043d7 100644
--- a/drivers/gpu/drm/i915/display/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/display/intel_sdvo.c
@@ -35,6 +35,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_eld.h>
 
 #include "i915_drv.h"
 #include "i915_reg.h"
diff --git a/drivers/gpu/drm/i915/display/intel_sprite.c b/drivers/gpu/drm/i915/display/intel_sprite.c
index 1fb16510f750..d7b440c8caef 100644
--- a/drivers/gpu/drm/i915/display/intel_sprite.c
+++ b/drivers/gpu/drm/i915/display/intel_sprite.c
@@ -48,6 +48,11 @@
 #include "intel_frontbuffer.h"
 #include "intel_sprite.h"
 
+static char sprite_name(struct drm_i915_private *i915, enum pipe pipe, int sprite)
+{
+	return pipe * DISPLAY_RUNTIME_INFO(i915)->num_sprites[pipe] + sprite + 'A';
+}
+
 static void i9xx_plane_linear_gamma(u16 gamma[8])
 {
 	/* The points are not evenly spaced. */
@@ -1636,7 +1641,7 @@ intel_sprite_plane_create(struct drm_i915_private *dev_priv,
 				       0, plane_funcs,
 				       formats, num_formats, modifiers,
 				       DRM_PLANE_TYPE_OVERLAY,
-				       "sprite %c", sprite_name(pipe, sprite));
+				       "sprite %c", sprite_name(dev_priv, pipe, sprite));
 	kfree(modifiers);
 
 	if (ret)
diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.c b/drivers/gpu/drm/i915/display/intel_vdsc.c
index 6757dbae9ee5..5f2fb702e367 100644
--- a/drivers/gpu/drm/i915/display/intel_vdsc.c
+++ b/drivers/gpu/drm/i915/display/intel_vdsc.c
@@ -77,8 +77,8 @@ intel_vdsc_set_min_max_qp(struct drm_dsc_config *vdsc_cfg, int buf,
 static void
 calculate_rc_params(struct drm_dsc_config *vdsc_cfg)
 {
+	int bpp = to_bpp_int(vdsc_cfg->bits_per_pixel);
 	int bpc = vdsc_cfg->bits_per_component;
-	int bpp = vdsc_cfg->bits_per_pixel >> 4;
 	int qp_bpc_modifier = (bpc - 8) * 2;
 	int uncompressed_bpg_rate;
 	int first_line_bpg_offset;
@@ -148,7 +148,13 @@ calculate_rc_params(struct drm_dsc_config *vdsc_cfg)
 		static const s8 ofs_und8[] = {
 			10, 8, 6, 4, 2, 0, -2, -4, -6, -8, -10, -10, -12, -12, -12
 		};
-
+		/*
+		 * For 420 format since bits_per_pixel (bpp) is set to target bpp * 2,
+		 * QP table values for target bpp 4.0 to 4.4375 (rounded to 4.0) are
+		 * actually for bpp 8 to 8.875 (rounded to 4.0 * 2 i.e 8).
+		 * Similarly values for target bpp 4.5 to 4.8375 (rounded to 4.5)
+		 * are for bpp 9 to 9.875 (rounded to 4.5 * 2 i.e 9), and so on.
+		 */
 		bpp_i  = bpp - 8;
 		for (buf_i = 0; buf_i < DSC_NUM_BUF_RANGES; buf_i++) {
 			u8 range_bpg_offset;
@@ -178,6 +184,9 @@ calculate_rc_params(struct drm_dsc_config *vdsc_cfg)
 				range_bpg_offset & DSC_RANGE_BPG_OFFSET_MASK;
 		}
 	} else {
+		/* fractional bpp part * 10000 (for precision up to 4 decimal places) */
+		int fractional_bits = to_bpp_frac(vdsc_cfg->bits_per_pixel);
+
 		static const s8 ofs_und6[] = {
 			0, -2, -2, -4, -6, -6, -8, -8, -8, -10, -10, -12, -12, -12, -12
 		};
@@ -191,7 +200,14 @@ calculate_rc_params(struct drm_dsc_config *vdsc_cfg)
 			10, 8, 6, 4, 2, 0, -2, -4, -6, -8, -10, -10, -12, -12, -12
 		};
 
-		bpp_i  = (2 * (bpp - 6));
+		/*
+		 * QP table rows have values in increment of 0.5.
+		 * So 6.0 bpp to 6.4375 will have index 0, 6.5 to 6.9375 will have index 1,
+		 * and so on.
+		 * 0.5 fractional part with 4 decimal precision becomes 5000
+		 */
+		bpp_i  = ((bpp - 6) + (fractional_bits < 5000 ? 0 : 1));
+
 		for (buf_i = 0; buf_i < DSC_NUM_BUF_RANGES; buf_i++) {
 			u8 range_bpg_offset;
 
@@ -248,7 +264,7 @@ int intel_dsc_compute_params(struct intel_crtc_state *pipe_config)
 	struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	struct drm_dsc_config *vdsc_cfg = &pipe_config->dsc.config;
-	u16 compressed_bpp = pipe_config->dsc.compressed_bpp;
+	u16 compressed_bpp = to_bpp_int(pipe_config->dsc.compressed_bpp_x16);
 	int err;
 	int ret;
 
@@ -279,8 +295,7 @@ int intel_dsc_compute_params(struct intel_crtc_state *pipe_config)
 	/* Gen 11 does not support VBR */
 	vdsc_cfg->vbr_enable = false;
 
-	/* Gen 11 only supports integral values of bpp */
-	vdsc_cfg->bits_per_pixel = compressed_bpp << 4;
+	vdsc_cfg->bits_per_pixel = pipe_config->dsc.compressed_bpp_x16;
 
 	/*
 	 * According to DSC 1.2 specs in Section 4.1 if native_420 is set
@@ -874,7 +889,7 @@ static void intel_dsc_get_pps_config(struct intel_crtc_state *crtc_state)
 	if (vdsc_cfg->native_420)
 		vdsc_cfg->bits_per_pixel >>= 1;
 
-	crtc_state->dsc.compressed_bpp = vdsc_cfg->bits_per_pixel >> 4;
+	crtc_state->dsc.compressed_bpp_x16 = vdsc_cfg->bits_per_pixel;
 
 	/* PPS 2 */
 	pps_temp = intel_dsc_pps_read_and_verify(crtc_state, 2);
diff --git a/drivers/gpu/drm/i915/display/skl_universal_plane.c b/drivers/gpu/drm/i915/display/skl_universal_plane.c
index 245a64332cc7..39499a0ec6c0 100644
--- a/drivers/gpu/drm/i915/display/skl_universal_plane.c
+++ b/drivers/gpu/drm/i915/display/skl_universal_plane.c
@@ -21,7 +21,6 @@
 #include "skl_scaler.h"
 #include "skl_universal_plane.h"
 #include "skl_watermark.h"
-#include "gt/intel_gt.h"
 #include "pxp/intel_pxp.h"
 
 static const u32 skl_plane_formats[] = {
@@ -1855,16 +1854,19 @@ static bool skl_fb_scalable(const struct drm_framebuffer *fb)
 	}
 }
 
-static bool bo_has_valid_encryption(struct drm_i915_gem_object *obj)
+static void check_protection(struct intel_plane_state *plane_state)
 {
-	struct drm_i915_private *i915 = to_i915(obj->base.dev);
+	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
+	struct drm_i915_private *i915 = to_i915(plane->base.dev);
+	const struct drm_framebuffer *fb = plane_state->hw.fb;
+	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
 
-	return intel_pxp_key_check(i915->pxp, obj, false) == 0;
-}
+	if (DISPLAY_VER(i915) < 11)
+		return;
 
-static bool pxp_is_borked(struct drm_i915_gem_object *obj)
-{
-	return i915_gem_object_is_protected(obj) && !bo_has_valid_encryption(obj);
+	plane_state->decrypt = intel_pxp_key_check(i915->pxp, obj, false) == 0;
+	plane_state->force_black = i915_gem_object_is_protected(obj) &&
+		!plane_state->decrypt;
 }
 
 static int skl_plane_check(struct intel_crtc_state *crtc_state,
@@ -1911,10 +1913,7 @@ static int skl_plane_check(struct intel_crtc_state *crtc_state,
 	if (ret)
 		return ret;
 
-	if (DISPLAY_VER(dev_priv) >= 11) {
-		plane_state->decrypt = bo_has_valid_encryption(intel_fb_obj(fb));
-		plane_state->force_black = pxp_is_borked(intel_fb_obj(fb));
-	}
+	check_protection(plane_state);
 
 	/* HW only has 8 bits pixel precision, disable plane if invisible */
 	if (!(plane_state->hw.alpha >> 8))
@@ -2489,7 +2488,7 @@ skl_get_initial_plane_config(struct intel_crtc *crtc,
 		goto error;
 	}
 
-	if (!dev_priv->params.enable_dpt &&
+	if (!dev_priv->display.params.enable_dpt &&
 	    intel_fb_modifier_uses_dpt(dev_priv, fb->modifier)) {
 		drm_dbg_kms(&dev_priv->drm, "DPT disabled, skipping initial FB\n");
 		goto error;
diff --git a/drivers/gpu/drm/i915/display/skl_watermark.c b/drivers/gpu/drm/i915/display/skl_watermark.c
index 99b8ccdc3dfa..56588d6e24ae 100644
--- a/drivers/gpu/drm/i915/display/skl_watermark.c
+++ b/drivers/gpu/drm/i915/display/skl_watermark.c
@@ -412,7 +412,7 @@ static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 
-	if (!i915->params.enable_sagv)
+	if (!i915->display.params.enable_sagv)
 		return false;
 
 	if (DISPLAY_VER(i915) >= 12)
@@ -3702,7 +3702,8 @@ static int intel_sagv_status_show(struct seq_file *m, void *unused)
 	};
 
 	seq_printf(m, "SAGV available: %s\n", str_yes_no(intel_has_sagv(i915)));
-	seq_printf(m, "SAGV modparam: %s\n", str_enabled_disabled(i915->params.enable_sagv));
+	seq_printf(m, "SAGV modparam: %s\n",
+		   str_enabled_disabled(i915->display.params.enable_sagv));
 	seq_printf(m, "SAGV status: %s\n", sagv_status[i915->display.sagv.status]);
 	seq_printf(m, "SAGV block time: %d usec\n", i915->display.sagv.block_time_us);
 
diff --git a/drivers/gpu/drm/i915/display/vlv_dsi.c b/drivers/gpu/drm/i915/display/vlv_dsi.c
index 55da627a8b8d..bda49734ca33 100644
--- a/drivers/gpu/drm/i915/display/vlv_dsi.c
+++ b/drivers/gpu/drm/i915/display/vlv_dsi.c
@@ -561,6 +561,12 @@ static void glk_dsi_clear_device_ready(struct intel_encoder *encoder)
 	glk_dsi_disable_mipi_io(encoder);
 }
 
+static i915_reg_t port_ctrl_reg(struct drm_i915_private *i915, enum port port)
+{
+	return IS_GEMINILAKE(i915) || IS_BROXTON(i915) ?
+		BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(port);
+}
+
 static void vlv_dsi_clear_device_ready(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
@@ -570,7 +576,7 @@ static void vlv_dsi_clear_device_ready(struct intel_encoder *encoder)
 	drm_dbg_kms(&dev_priv->drm, "\n");
 	for_each_dsi_port(port, intel_dsi->ports) {
 		/* Common bit for both MIPI Port A & MIPI Port C on VLV/CHV */
-		i915_reg_t port_ctrl = IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv) ?
+		i915_reg_t port_ctrl = IS_BROXTON(dev_priv) ?
 			BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(PORT_A);
 
 		intel_de_write(dev_priv, MIPI_DEVICE_READY(port),
@@ -589,7 +595,7 @@ static void vlv_dsi_clear_device_ready(struct intel_encoder *encoder)
 		 * On VLV/CHV, wait till Clock lanes are in LP-00 state for MIPI
 		 * Port A only. MIPI Port C has no similar bit for checking.
 		 */
-		if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv) || port == PORT_A) &&
+		if ((IS_BROXTON(dev_priv) || port == PORT_A) &&
 		    intel_de_wait_for_clear(dev_priv, port_ctrl,
 					    AFE_LATCHOUT, 30))
 			drm_err(&dev_priv->drm, "DSI LP not going Low\n");
@@ -627,8 +633,7 @@ static void intel_dsi_port_enable(struct intel_encoder *encoder,
 	}
 
 	for_each_dsi_port(port, intel_dsi->ports) {
-		i915_reg_t port_ctrl = IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv) ?
-			BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(port);
+		i915_reg_t port_ctrl = port_ctrl_reg(dev_priv, port);
 		u32 temp;
 
 		temp = intel_de_read(dev_priv, port_ctrl);
@@ -664,8 +669,7 @@ static void intel_dsi_port_disable(struct intel_encoder *encoder)
 	enum port port;
 
 	for_each_dsi_port(port, intel_dsi->ports) {
-		i915_reg_t port_ctrl = IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv) ?
-			BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(port);
+		i915_reg_t port_ctrl = port_ctrl_reg(dev_priv, port);
 
 		/* de-assert ip_tg_enable signal */
 		intel_de_rmw(dev_priv, port_ctrl, DPI_ENABLE, 0);
@@ -955,9 +959,8 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder,
 
 	/* XXX: this only works for one DSI output */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		i915_reg_t ctrl_reg = IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv) ?
-			BXT_MIPI_PORT_CTRL(port) : MIPI_PORT_CTRL(port);
-		bool enabled = intel_de_read(dev_priv, ctrl_reg) & DPI_ENABLE;
+		i915_reg_t port_ctrl = port_ctrl_reg(dev_priv, port);
+		bool enabled = intel_de_read(dev_priv, port_ctrl) & DPI_ENABLE;
 
 		/*
 		 * Due to some hardware limitations on VLV/CHV, the DPI enable
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_frontbuffer.h b/drivers/gpu/drm/i915/gem/i915_gem_object_frontbuffer.h
index e5e870b6f186..9fbf14867a2a 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object_frontbuffer.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object_frontbuffer.h
@@ -89,6 +89,7 @@ i915_gem_object_set_frontbuffer(struct drm_i915_gem_object *obj,
 
 	if (!front) {
 		RCU_INIT_POINTER(obj->frontbuffer, NULL);
+		drm_gem_object_put(intel_bo_to_drm_bo(obj));
 	} else if (rcu_access_pointer(obj->frontbuffer)) {
 		cur = rcu_dereference_protected(obj->frontbuffer, true);
 		kref_get(&cur->ref);
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.h b/drivers/gpu/drm/i915/gt/intel_engine_pm.h
index d68675925b79..1d97c435a015 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_pm.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.h
@@ -10,6 +10,7 @@
 #include "i915_request.h"
 #include "intel_engine_types.h"
 #include "intel_wakeref.h"
+#include "intel_gt.h"
 #include "intel_gt_pm.h"
 
 static inline bool
diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c
index ed32bf5b1546..ba1186fc524f 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt.c
@@ -982,8 +982,6 @@ int intel_gt_probe_all(struct drm_i915_private *i915)
 
 err:
 	i915_probe_error(i915, "Failed to initialize %s! (%d)\n", gtdef->name, ret);
-	intel_gt_release_all(i915);
-
 	return ret;
 }
 
@@ -1002,15 +1000,6 @@ int intel_gt_tiles_init(struct drm_i915_private *i915)
 	return 0;
 }
 
-void intel_gt_release_all(struct drm_i915_private *i915)
-{
-	struct intel_gt *gt;
-	unsigned int id;
-
-	for_each_gt(gt, i915, id)
-		i915->gt[id] = NULL;
-}
-
 void intel_gt_info_print(const struct intel_gt_info *info,
 			 struct drm_printer *p)
 {
diff --git a/drivers/gpu/drm/i915/gt/intel_gt.h b/drivers/gpu/drm/i915/gt/intel_gt.h
index 970bedf6b78a..e1f13735f530 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt.h
+++ b/drivers/gpu/drm/i915/gt/intel_gt.h
@@ -167,6 +167,20 @@ void intel_gt_release_all(struct drm_i915_private *i915);
 	     (id__)++) \
 		for_each_if(((gt__) = (i915__)->gt[(id__)]))
 
+/* Simple iterator over all initialised engines */
+#define for_each_engine(engine__, gt__, id__) \
+	for ((id__) = 0; \
+	     (id__) < I915_NUM_ENGINES; \
+	     (id__)++) \
+		for_each_if ((engine__) = (gt__)->engine[(id__)])
+
+/* Iterator over subset of engines selected by mask */
+#define for_each_engine_masked(engine__, gt__, mask__, tmp__) \
+	for ((tmp__) = (mask__) & (gt__)->info.engine_mask; \
+	     (tmp__) ? \
+	     ((engine__) = (gt__)->engine[__mask_next_bit(tmp__)]), 1 : \
+	     0;)
+
 void intel_gt_info_print(const struct intel_gt_info *info,
 			 struct drm_printer *p);
 
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_engines_debugfs.c b/drivers/gpu/drm/i915/gt/intel_gt_engines_debugfs.c
index 8f9b874fdc9c..3aa1d014c14d 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_engines_debugfs.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_engines_debugfs.c
@@ -6,8 +6,8 @@
 
 #include <drm/drm_print.h>
 
-#include "i915_drv.h" /* for_each_engine! */
 #include "intel_engine.h"
+#include "intel_gt.h"
 #include "intel_gt_debugfs.h"
 #include "intel_gt_engines_debugfs.h"
 
diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c
index 05f9348b7a9d..d4a3f3e093b0 100644
--- a/drivers/gpu/drm/i915/gvt/cmd_parser.c
+++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c
@@ -3047,7 +3047,7 @@ static int shadow_indirect_ctx(struct intel_shadow_wa_ctx *wa_ctx)
 
 static int combine_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
 {
-	u32 per_ctx_start[CACHELINE_DWORDS] = {0};
+	u32 per_ctx_start[CACHELINE_DWORDS] = {};
 	unsigned char *bb_start_sva;
 
 	if (!wa_ctx->per_ctx.valid)
diff --git a/drivers/gpu/drm/i915/gvt/fb_decoder.c b/drivers/gpu/drm/i915/gvt/fb_decoder.c
index 835c3fde8a20..313efdabee57 100644
--- a/drivers/gpu/drm/i915/gvt/fb_decoder.c
+++ b/drivers/gpu/drm/i915/gvt/fb_decoder.c
@@ -56,7 +56,7 @@ static const struct pixel_format bdw_pixel_formats[] = {
 	{DRM_FORMAT_XBGR8888, 32, "32-bit RGBX (8:8:8:8 MSB-X:B:G:R)"},
 
 	/* non-supported format has bpp default to 0 */
-	{0, 0, NULL},
+	{}
 };
 
 static const struct pixel_format skl_pixel_formats[] = {
@@ -76,7 +76,7 @@ static const struct pixel_format skl_pixel_formats[] = {
 	{DRM_FORMAT_XRGB2101010, 32, "32-bit BGRX (2:10:10:10 MSB-X:R:G:B)"},
 
 	/* non-supported format has bpp default to 0 */
-	{0, 0, NULL},
+	{}
 };
 
 static int bdw_format_to_drm(int format)
@@ -293,7 +293,7 @@ static const struct cursor_mode_format cursor_pixel_formats[] = {
 	{DRM_FORMAT_ARGB8888, 32, 64, 64, "64x64 32bpp ARGB"},
 
 	/* non-supported format has bpp default to 0 */
-	{0, 0, 0, 0, NULL},
+	{}
 };
 
 static int cursor_mode_to_drm(int mode)
diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c
index a9f7fa9b90bd..90f6c1ece57d 100644
--- a/drivers/gpu/drm/i915/gvt/handlers.c
+++ b/drivers/gpu/drm/i915/gvt/handlers.c
@@ -538,7 +538,7 @@ static u32 bxt_vgpu_get_dp_bitrate(struct intel_vgpu *vgpu, enum port port)
 	int refclk = vgpu->gvt->gt->i915->display.dpll.ref_clks.nssc;
 	enum dpio_phy phy = DPIO_PHY0;
 	enum dpio_channel ch = DPIO_CH0;
-	struct dpll clock = {0};
+	struct dpll clock = {};
 	u32 temp;
 
 	/* Port to PHY mapping is fixed, see bxt_ddi_phy_info{} */
@@ -2576,7 +2576,6 @@ static int init_bdw_mmio_info(struct intel_gvt *gvt)
 
 static int init_skl_mmio_info(struct intel_gvt *gvt)
 {
-	struct drm_i915_private *dev_priv = gvt->gt->i915;
 	int ret;
 
 	MMIO_DH(FORCEWAKE_RENDER_GEN9, D_SKL_PLUS, NULL, mul_force_wake_write);
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index e9b79c2c37d8..bfe92d2402ea 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -32,6 +32,8 @@
 
 #include <drm/drm_debugfs.h>
 
+#include "display/intel_display_params.h"
+
 #include "gem/i915_gem_context.h"
 #include "gt/intel_gt.h"
 #include "gt/intel_gt_buffer_pool.h"
@@ -67,13 +69,13 @@ static int i915_capabilities(struct seq_file *m, void *data)
 	seq_printf(m, "pch: %d\n", INTEL_PCH_TYPE(i915));
 
 	intel_device_info_print(INTEL_INFO(i915), RUNTIME_INFO(i915), &p);
-	intel_display_device_info_print(DISPLAY_INFO(i915), DISPLAY_RUNTIME_INFO(i915), &p);
 	i915_print_iommu_status(i915, &p);
 	intel_gt_info_print(&to_gt(i915)->info, &p);
 	intel_driver_caps_print(&i915->caps, &p);
 
 	kernel_param_lock(THIS_MODULE);
 	i915_params_dump(&i915->params, &p);
+	intel_display_params_dump(i915, &p);
 	kernel_param_unlock(THIS_MODULE);
 
 	return 0;
diff --git a/drivers/gpu/drm/i915/i915_driver.c b/drivers/gpu/drm/i915/i915_driver.c
index 8a0e2c745e1f..2a1faf403965 100644
--- a/drivers/gpu/drm/i915/i915_driver.c
+++ b/drivers/gpu/drm/i915/i915_driver.c
@@ -231,16 +231,10 @@ static int i915_driver_early_probe(struct drm_i915_private *dev_priv)
 
 	spin_lock_init(&dev_priv->irq_lock);
 	spin_lock_init(&dev_priv->gpu_error.lock);
-	mutex_init(&dev_priv->display.backlight.lock);
 
 	mutex_init(&dev_priv->sb_lock);
 	cpu_latency_qos_add_request(&dev_priv->sb_qos, PM_QOS_DEFAULT_VALUE);
 
-	mutex_init(&dev_priv->display.audio.mutex);
-	mutex_init(&dev_priv->display.wm.wm_mutex);
-	mutex_init(&dev_priv->display.pps.mutex);
-	mutex_init(&dev_priv->display.hdcp.hdcp_mutex);
-
 	i915_memcpy_init_early(dev_priv);
 	intel_runtime_pm_init_early(&dev_priv->runtime_pm);
 
@@ -782,7 +776,7 @@ int i915_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	ret = i915_driver_mmio_probe(i915);
 	if (ret < 0)
-		goto out_tiles_cleanup;
+		goto out_runtime_pm_put;
 
 	ret = i915_driver_hw_probe(i915);
 	if (ret < 0)
@@ -842,8 +836,6 @@ int i915_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	i915_ggtt_driver_late_release(i915);
 out_cleanup_mmio:
 	i915_driver_mmio_release(i915);
-out_tiles_cleanup:
-	intel_gt_release_all(i915);
 out_runtime_pm_put:
 	enable_rpm_wakeref_asserts(&i915->runtime_pm);
 	i915_driver_late_release(i915);
@@ -909,6 +901,8 @@ static void i915_driver_release(struct drm_device *dev)
 	intel_runtime_pm_driver_release(rpm);
 
 	i915_driver_late_release(dev_priv);
+
+	intel_display_device_remove(dev_priv);
 }
 
 static int i915_driver_open(struct drm_device *dev, struct drm_file *file)
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index dd452c220df7..861567362abd 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -396,20 +396,6 @@ static inline struct intel_gt *to_gt(const struct drm_i915_private *i915)
 	return i915->gt[0];
 }
 
-/* Simple iterator over all initialised engines */
-#define for_each_engine(engine__, gt__, id__) \
-	for ((id__) = 0; \
-	     (id__) < I915_NUM_ENGINES; \
-	     (id__)++) \
-		for_each_if ((engine__) = (gt__)->engine[(id__)])
-
-/* Iterator over subset of engines selected by mask */
-#define for_each_engine_masked(engine__, gt__, mask__, tmp__) \
-	for ((tmp__) = (mask__) & (gt__)->info.engine_mask; \
-	     (tmp__) ? \
-	     ((engine__) = (gt__)->engine[__mask_next_bit(tmp__)]), 1 : \
-	     0;)
-
 #define rb_to_uabi_engine(rb) \
 	rb_entry_safe(rb, struct intel_engine_cs, uabi_node)
 
@@ -418,11 +404,6 @@ static inline struct intel_gt *to_gt(const struct drm_i915_private *i915)
 	     (engine__); \
 	     (engine__) = rb_to_uabi_engine(rb_next(&(engine__)->uabi_node)))
 
-#define for_each_uabi_class_engine(engine__, class__, i915__) \
-	for ((engine__) = intel_engine_lookup_user((i915__), (class__), 0); \
-	     (engine__) && (engine__)->uabi_class == (class__); \
-	     (engine__) = rb_to_uabi_engine(rb_next(&(engine__)->uabi_node)))
-
 #define INTEL_INFO(i915)	((i915)->__info)
 #define RUNTIME_INFO(i915)	(&(i915)->__runtime)
 #define DRIVER_CAPS(i915)	(&(i915)->caps)
@@ -575,6 +556,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
 #define IS_DG2(i915)	IS_PLATFORM(i915, INTEL_DG2)
 #define IS_PONTEVECCHIO(i915) IS_PLATFORM(i915, INTEL_PONTEVECCHIO)
 #define IS_METEORLAKE(i915) IS_PLATFORM(i915, INTEL_METEORLAKE)
+#define IS_LUNARLAKE(i915) 0
 
 #define IS_DG2_G10(i915) \
 	IS_SUBPLATFORM(i915, INTEL_DG2, INTEL_SUBPLATFORM_G10)
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index c166ad5e187a..92758b6b41f0 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -1306,8 +1306,6 @@ void i915_gem_init_early(struct drm_i915_private *dev_priv)
 {
 	i915_gem_init__mm(dev_priv);
 	i915_gem_init__contexts(dev_priv);
-
-	spin_lock_init(&dev_priv->display.fb_tracking.lock);
 }
 
 void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index b4e31e59c799..0971f4976324 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -660,6 +660,7 @@ static void err_print_params(struct drm_i915_error_state_buf *m,
 	struct drm_printer p = i915_error_printer(m);
 
 	i915_params_dump(params, &p);
+	intel_display_params_dump(m->i915, &p);
 }
 
 static void err_print_pciid(struct drm_i915_error_state_buf *m,
@@ -1027,6 +1028,7 @@ static void i915_vma_coredump_free(struct i915_vma_coredump *vma)
 static void cleanup_params(struct i915_gpu_coredump *error)
 {
 	i915_params_free(&error->params);
+	intel_display_params_free(&error->display_params);
 }
 
 static void cleanup_uc(struct intel_uc_coredump *uc)
@@ -1988,6 +1990,7 @@ static void capture_gen(struct i915_gpu_coredump *error)
 	error->suspend_count = i915->suspend_count;
 
 	i915_params_copy(&error->params, &i915->params);
+	intel_display_params_copy(&error->display_params);
 	memcpy(&error->device_info,
 	       INTEL_INFO(i915),
 	       sizeof(error->device_info));
@@ -2174,7 +2177,7 @@ void i915_error_state_store(struct i915_gpu_coredump *error)
 	    ktime_get_real_seconds() - DRIVER_TIMESTAMP < DAY_AS_SECONDS(180)) {
 		pr_info("GPU hangs can indicate a bug anywhere in the entire gfx stack, including userspace.\n");
 		pr_info("Please file a _new_ bug report at https://gitlab.freedesktop.org/drm/intel/issues/new.\n");
-		pr_info("Please see https://gitlab.freedesktop.org/drm/intel/-/wikis/How-to-file-i915-bugs for details.\n");
+		pr_info("Please see https://drm.pages.freedesktop.org/intel-docs/how-to-file-i915-bugs.html for details.\n");
 		pr_info("drm/i915 developers can then reassign to the right component if it's not a kernel issue.\n");
 		pr_info("The GPU crash dump is required to analyze GPU hangs, so please always attach it.\n");
 		pr_info("GPU crash dump saved to /sys/class/drm/card%d/error\n",
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.h b/drivers/gpu/drm/i915/i915_gpu_error.h
index 9f5971f5e980..4ce227f7e1e1 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.h
+++ b/drivers/gpu/drm/i915/i915_gpu_error.h
@@ -15,6 +15,7 @@
 #include <drm/drm_mm.h>
 
 #include "display/intel_display_device.h"
+#include "display/intel_display_params.h"
 #include "gt/intel_engine.h"
 #include "gt/intel_gt_types.h"
 #include "gt/uc/intel_uc_fw.h"
@@ -214,6 +215,7 @@ struct i915_gpu_coredump {
 	struct intel_display_runtime_info display_runtime_info;
 	struct intel_driver_caps driver_caps;
 	struct i915_params params;
+	struct intel_display_params display_params;
 
 	struct intel_overlay_error_state *overlay;
 
diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c
index 036c4c3ed6ed..de43048543e8 100644
--- a/drivers/gpu/drm/i915/i915_params.c
+++ b/drivers/gpu/drm/i915/i915_params.c
@@ -67,33 +67,9 @@ i915_param_named(modeset, int, 0400,
 	"Use kernel modesetting [KMS] (0=disable, "
 	"1=on, -1=force vga console preference [default])");
 
-i915_param_named_unsafe(enable_dc, int, 0400,
-	"Enable power-saving display C-states. "
-	"(-1=auto [default]; 0=disable; 1=up to DC5; 2=up to DC6; "
-	"3=up to DC5 with DC3CO; 4=up to DC6 with DC3CO)");
-
-i915_param_named_unsafe(enable_fbc, int, 0400,
-	"Enable frame buffer compression for power savings "
-	"(default: -1 (use per-chip default))");
-
-i915_param_named_unsafe(lvds_channel_mode, int, 0400,
-	 "Specify LVDS channel mode "
-	 "(0=probe BIOS [default], 1=single-channel, 2=dual-channel)");
-
-i915_param_named_unsafe(panel_use_ssc, int, 0400,
-	"Use Spread Spectrum Clock with panels [LVDS/eDP] "
-	"(default: auto from VBT)");
-
-i915_param_named_unsafe(vbt_sdvo_panel_type, int, 0400,
-	"Override/Ignore selection of SDVO panel mode in the VBT "
-	"(-2=ignore, -1=auto [default], index in VBT BIOS table)");
-
 i915_param_named_unsafe(reset, uint, 0400,
 	"Attempt GPU resets (0=disabled, 1=full gpu reset, 2=engine reset [default])");
 
-i915_param_named_unsafe(vbt_firmware, charp, 0400,
-	"Load VBT from specified file under /lib/firmware");
-
 #if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR)
 i915_param_named(error_capture, bool, 0400,
 	"Record the GPU state following a hang. "
@@ -106,55 +82,10 @@ i915_param_named_unsafe(enable_hangcheck, bool, 0400,
 	"WARNING: Disabling this can cause system wide hangs. "
 	"(default: true)");
 
-i915_param_named_unsafe(enable_psr, int, 0400,
-	"Enable PSR "
-	"(0=disabled, 1=enable up to PSR1, 2=enable up to PSR2) "
-	"Default: -1 (use per-chip default)");
-
-i915_param_named(psr_safest_params, bool, 0400,
-	"Replace PSR VBT parameters by the safest and not optimal ones. This "
-	"is helpful to detect if PSR issues are related to bad values set in "
-	" VBT. (0=use VBT parameters, 1=use safest parameters)");
-
-i915_param_named_unsafe(enable_psr2_sel_fetch, bool, 0400,
-	"Enable PSR2 selective fetch "
-	"(0=disabled, 1=enabled) "
-	"Default: 0");
-
-i915_param_named_unsafe(enable_sagv, bool, 0600,
-	"Enable system agent voltage/frequency scaling (SAGV) (default: true)");
-
 i915_param_named_unsafe(force_probe, charp, 0400,
 	"Force probe options for specified supported devices. "
 	"See CONFIG_DRM_I915_FORCE_PROBE for details.");
 
-i915_param_named_unsafe(disable_power_well, int, 0400,
-	"Disable display power wells when possible "
-	"(-1=auto [default], 0=power wells always on, 1=power wells disabled when possible)");
-
-i915_param_named_unsafe(enable_ips, int, 0400, "Enable IPS (default: true)");
-
-i915_param_named_unsafe(enable_dpt, bool, 0400,
-	"Enable display page table (DPT) (default: true)");
-
-i915_param_named_unsafe(load_detect_test, bool, 0400,
-	"Force-enable the VGA load detect code for testing (default:false). "
-	"For developers only.");
-
-i915_param_named_unsafe(force_reset_modeset_test, bool, 0400,
-	"Force a modeset during gpu reset for testing (default:false). "
-	"For developers only.");
-
-i915_param_named_unsafe(invert_brightness, int, 0400,
-	"Invert backlight brightness "
-	"(-1 force normal, 0 machine defaults, 1 force inversion), please "
-	"report PCI device ID, subsystem vendor and subsystem device ID "
-	"to dri-devel@lists.freedesktop.org, if your machine needs it. "
-	"It will then be included in an upcoming module version.");
-
-i915_param_named(disable_display, bool, 0400,
-	"Disable display (default: false)");
-
 i915_param_named(memtest, bool, 0400,
 	"Perform a read/write test of all device memory on module load (default: off)");
 
@@ -162,19 +93,6 @@ i915_param_named(mmio_debug, int, 0400,
 	"Enable the MMIO debug code for the first N failures (default: off). "
 	"This may negatively affect performance.");
 
-/* Special case writable file */
-i915_param_named(verbose_state_checks, bool, 0600,
-	"Enable verbose logs (ie. WARN_ON()) in case of unexpected hw state conditions.");
-
-i915_param_named_unsafe(nuclear_pageflip, bool, 0400,
-	"Force enable atomic functionality on platforms that don't have full support yet.");
-
-/* WA to get away with the default setting in VBT for early platforms.Will be removed */
-i915_param_named_unsafe(edp_vswing, int, 0400,
-	"Ignore/Override vswing pre-emph table selection from VBT "
-	"(0=use value from vbt [default], 1=low power swing(200mV),"
-	"2=default swing(400mV))");
-
 i915_param_named_unsafe(enable_guc, int, 0400,
 	"Enable GuC load for GuC submission and/or HuC load. "
 	"Required functionality can be selected using bitmask values. "
@@ -196,18 +114,11 @@ i915_param_named_unsafe(dmc_firmware_path, charp, 0400,
 i915_param_named_unsafe(gsc_firmware_path, charp, 0400,
 	"GSC firmware path to use instead of the default one");
 
-i915_param_named_unsafe(enable_dp_mst, bool, 0400,
-	"Enable multi-stream transport (MST) for new DisplayPort sinks. (default: true)");
-
 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG)
 i915_param_named_unsafe(inject_probe_failure, uint, 0400,
 	"Force an error after a number of failure check points (0:disabled (default), N:force failure at the Nth failure check point)");
 #endif
 
-i915_param_named(enable_dpcd_backlight, int, 0400,
-	"Enable support for DPCD backlight control"
-	"(-1=use per-VBT LFP backlight type setting [default], 0=disabled, 1=enable, 2=force VESA interface, 3=force Intel interface)");
-
 #if IS_ENABLED(CONFIG_DRM_I915_GVT)
 i915_param_named(enable_gvt, bool, 0400,
 	"Enable support for Intel GVT-g graphics virtualization host support(default:false)");
diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h
index d5194b039aab..1315d7fac850 100644
--- a/drivers/gpu/drm/i915/i915_params.h
+++ b/drivers/gpu/drm/i915/i915_params.h
@@ -46,21 +46,7 @@ struct drm_printer;
  *       debugfs file
  */
 #define I915_PARAMS_FOR_EACH(param) \
-	param(char *, vbt_firmware, NULL, 0400) \
 	param(int, modeset, -1, 0400) \
-	param(int, lvds_channel_mode, 0, 0400) \
-	param(int, panel_use_ssc, -1, 0600) \
-	param(int, vbt_sdvo_panel_type, -1, 0400) \
-	param(int, enable_dc, -1, 0400) \
-	param(int, enable_fbc, -1, 0600) \
-	param(int, enable_psr, -1, 0600) \
-	param(bool, enable_dpt, true, 0400) \
-	param(bool, psr_safest_params, false, 0400) \
-	param(bool, enable_psr2_sel_fetch, true, 0400) \
-	param(bool, enable_sagv, true, 0600) \
-	param(int, disable_power_well, -1, 0400) \
-	param(int, enable_ips, 1, 0600) \
-	param(int, invert_brightness, 0, 0600) \
 	param(int, enable_guc, -1, 0400) \
 	param(int, guc_log_level, -1, 0400) \
 	param(char *, guc_firmware_path, NULL, 0400) \
@@ -69,23 +55,15 @@ struct drm_printer;
 	param(char *, gsc_firmware_path, NULL, 0400) \
 	param(bool, memtest, false, 0400) \
 	param(int, mmio_debug, -IS_ENABLED(CONFIG_DRM_I915_DEBUG_MMIO), 0600) \
-	param(int, edp_vswing, 0, 0400) \
 	param(unsigned int, reset, 3, 0600) \
 	param(unsigned int, inject_probe_failure, 0, 0) \
-	param(int, enable_dpcd_backlight, -1, 0600) \
 	param(char *, force_probe, CONFIG_DRM_I915_FORCE_PROBE, 0400) \
 	param(unsigned int, request_timeout_ms, CONFIG_DRM_I915_REQUEST_TIMEOUT, CONFIG_DRM_I915_REQUEST_TIMEOUT ? 0600 : 0) \
 	param(unsigned int, lmem_size, 0, 0400) \
 	param(unsigned int, lmem_bar_size, 0, 0400) \
 	/* leave bools at the end to not create holes */ \
 	param(bool, enable_hangcheck, true, 0600) \
-	param(bool, load_detect_test, false, 0600) \
-	param(bool, force_reset_modeset_test, false, 0600) \
 	param(bool, error_capture, true, IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) ? 0600 : 0) \
-	param(bool, disable_display, false, 0400) \
-	param(bool, verbose_state_checks, true, 0) \
-	param(bool, nuclear_pageflip, false, 0400) \
-	param(bool, enable_dp_mst, true, 0600) \
 	param(bool, enable_gvt, false, IS_ENABLED(CONFIG_DRM_I915_GVT) ? 0400 : 0)
 
 #define MEMBER(T, member, ...) T member;
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 135e8d8dbdf0..27dc903f0553 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -195,8 +195,6 @@
 #define  DPIO_SFR_BYPASS		(1 << 1)
 #define  DPIO_CMNRST			(1 << 0)
 
-#define DPIO_PHY(pipe)			((pipe) >> 1)
-
 /*
  * Per pipe/PLL DPIO regs
  */
diff --git a/drivers/gpu/drm/i915/i915_utils.h b/drivers/gpu/drm/i915/i915_utils.h
index c61066498bf2..f98577967b7f 100644
--- a/drivers/gpu/drm/i915/i915_utils.h
+++ b/drivers/gpu/drm/i915/i915_utils.h
@@ -40,7 +40,7 @@
 struct drm_i915_private;
 struct timer_list;
 
-#define FDO_BUG_URL "https://gitlab.freedesktop.org/drm/intel/-/wikis/How-to-file-i915-bugs"
+#define FDO_BUG_URL "https://drm.pages.freedesktop.org/intel-docs/how-to-file-i915-bugs.html"
 
 #define MISSING_CASE(x) WARN(1, "Missing case (%s == %ld)\n", \
 			     __stringify(x), (long)(x))
diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.h b/drivers/gpu/drm/i915/intel_runtime_pm.h
index f79cda7a2503..be43614c73fd 100644
--- a/drivers/gpu/drm/i915/intel_runtime_pm.h
+++ b/drivers/gpu/drm/i915/intel_runtime_pm.h
@@ -11,8 +11,6 @@
 
 #include "intel_wakeref.h"
 
-#include "i915_utils.h"
-
 struct device;
 struct drm_i915_private;
 struct drm_printer;
diff --git a/drivers/gpu/drm/i915/selftests/intel_uncore.c b/drivers/gpu/drm/i915/selftests/intel_uncore.c
index 03ea75cd84dd..4f98aa8a861e 100644
--- a/drivers/gpu/drm/i915/selftests/intel_uncore.c
+++ b/drivers/gpu/drm/i915/selftests/intel_uncore.c
@@ -24,6 +24,8 @@
 
 #include "../i915_selftest.h"
 
+#include "gt/intel_gt.h"
+
 static int intel_fw_table_check(const struct intel_forcewake_range *ranges,
 				unsigned int num_ranges,
 				bool is_watertight)
diff --git a/drivers/gpu/drm/i915/soc/intel_gmch.c b/drivers/gpu/drm/i915/soc/intel_gmch.c
index f32e9f78770a..40874ebfb64c 100644
--- a/drivers/gpu/drm/i915/soc/intel_gmch.c
+++ b/drivers/gpu/drm/i915/soc/intel_gmch.c
@@ -33,18 +33,22 @@ int intel_gmch_bridge_setup(struct drm_i915_private *i915)
 					i915->gmch.pdev);
 }
 
+static int mchbar_reg(struct drm_i915_private *i915)
+{
+	return GRAPHICS_VER(i915) >= 4 ? MCHBAR_I965 : MCHBAR_I915;
+}
+
 /* Allocate space for the MCH regs if needed, return nonzero on error */
 static int
 intel_alloc_mchbar_resource(struct drm_i915_private *i915)
 {
-	int reg = GRAPHICS_VER(i915) >= 4 ? MCHBAR_I965 : MCHBAR_I915;
 	u32 temp_lo, temp_hi = 0;
 	u64 mchbar_addr;
 	int ret;
 
 	if (GRAPHICS_VER(i915) >= 4)
-		pci_read_config_dword(i915->gmch.pdev, reg + 4, &temp_hi);
-	pci_read_config_dword(i915->gmch.pdev, reg, &temp_lo);
+		pci_read_config_dword(i915->gmch.pdev, mchbar_reg(i915) + 4, &temp_hi);
+	pci_read_config_dword(i915->gmch.pdev, mchbar_reg(i915), &temp_lo);
 	mchbar_addr = ((u64)temp_hi << 32) | temp_lo;
 
 	/* If ACPI doesn't have it, assume we need to allocate it ourselves */
@@ -68,10 +72,10 @@ intel_alloc_mchbar_resource(struct drm_i915_private *i915)
 	}
 
 	if (GRAPHICS_VER(i915) >= 4)
-		pci_write_config_dword(i915->gmch.pdev, reg + 4,
+		pci_write_config_dword(i915->gmch.pdev, mchbar_reg(i915) + 4,
 				       upper_32_bits(i915->gmch.mch_res.start));
 
-	pci_write_config_dword(i915->gmch.pdev, reg,
+	pci_write_config_dword(i915->gmch.pdev, mchbar_reg(i915),
 			       lower_32_bits(i915->gmch.mch_res.start));
 	return 0;
 }
@@ -79,7 +83,6 @@ intel_alloc_mchbar_resource(struct drm_i915_private *i915)
 /* Setup MCHBAR if possible, return true if we should disable it again */
 void intel_gmch_bar_setup(struct drm_i915_private *i915)
 {
-	int mchbar_reg = GRAPHICS_VER(i915) >= 4 ? MCHBAR_I965 : MCHBAR_I915;
 	u32 temp;
 	bool enabled;
 
@@ -92,7 +95,7 @@ void intel_gmch_bar_setup(struct drm_i915_private *i915)
 		pci_read_config_dword(i915->gmch.pdev, DEVEN, &temp);
 		enabled = !!(temp & DEVEN_MCHBAR_EN);
 	} else {
-		pci_read_config_dword(i915->gmch.pdev, mchbar_reg, &temp);
+		pci_read_config_dword(i915->gmch.pdev, mchbar_reg(i915), &temp);
 		enabled = temp & 1;
 	}
 
@@ -110,15 +113,13 @@ void intel_gmch_bar_setup(struct drm_i915_private *i915)
 		pci_write_config_dword(i915->gmch.pdev, DEVEN,
 				       temp | DEVEN_MCHBAR_EN);
 	} else {
-		pci_read_config_dword(i915->gmch.pdev, mchbar_reg, &temp);
-		pci_write_config_dword(i915->gmch.pdev, mchbar_reg, temp | 1);
+		pci_read_config_dword(i915->gmch.pdev, mchbar_reg(i915), &temp);
+		pci_write_config_dword(i915->gmch.pdev, mchbar_reg(i915), temp | 1);
 	}
 }
 
 void intel_gmch_bar_teardown(struct drm_i915_private *i915)
 {
-	int mchbar_reg = GRAPHICS_VER(i915) >= 4 ? MCHBAR_I965 : MCHBAR_I915;
-
 	if (i915->gmch.mchbar_need_disable) {
 		if (IS_I915G(i915) || IS_I915GM(i915)) {
 			u32 deven_val;
@@ -131,10 +132,10 @@ void intel_gmch_bar_teardown(struct drm_i915_private *i915)
 		} else {
 			u32 mchbar_val;
 
-			pci_read_config_dword(i915->gmch.pdev, mchbar_reg,
+			pci_read_config_dword(i915->gmch.pdev, mchbar_reg(i915),
 					      &mchbar_val);
 			mchbar_val &= ~1;
-			pci_write_config_dword(i915->gmch.pdev, mchbar_reg,
+			pci_write_config_dword(i915->gmch.pdev, mchbar_reg(i915),
 					       mchbar_val);
 		}
 	}
diff --git a/drivers/gpu/drm/i915/vlv_sideband.c b/drivers/gpu/drm/i915/vlv_sideband.c
index b98dec3ad817..63a795876309 100644
--- a/drivers/gpu/drm/i915/vlv_sideband.c
+++ b/drivers/gpu/drm/i915/vlv_sideband.c
@@ -227,9 +227,9 @@ static u32 vlv_dpio_phy_iosf_port(struct drm_i915_private *i915, enum dpio_phy p
 		return IOSF_PORT_DPIO;
 }
 
-u32 vlv_dpio_read(struct drm_i915_private *i915, enum pipe pipe, int reg)
+u32 vlv_dpio_read(struct drm_i915_private *i915, enum dpio_phy phy, int reg)
 {
-	u32 port = vlv_dpio_phy_iosf_port(i915, DPIO_PHY(pipe));
+	u32 port = vlv_dpio_phy_iosf_port(i915, phy);
 	u32 val = 0;
 
 	vlv_sideband_rw(i915, DPIO_DEVFN, port, SB_MRD_NP, reg, &val);
@@ -239,16 +239,16 @@ u32 vlv_dpio_read(struct drm_i915_private *i915, enum pipe pipe, int reg)
 	 * so ideally we should check the register offset instead...
 	 */
 	drm_WARN(&i915->drm, val == 0xffffffff,
-		 "DPIO read pipe %c reg 0x%x == 0x%x\n",
-		 pipe_name(pipe), reg, val);
+		 "DPIO PHY%d read reg 0x%x == 0x%x\n",
+		 phy, reg, val);
 
 	return val;
 }
 
 void vlv_dpio_write(struct drm_i915_private *i915,
-		    enum pipe pipe, int reg, u32 val)
+		    enum dpio_phy phy, int reg, u32 val)
 {
-	u32 port = vlv_dpio_phy_iosf_port(i915, DPIO_PHY(pipe));
+	u32 port = vlv_dpio_phy_iosf_port(i915, phy);
 
 	vlv_sideband_rw(i915, DPIO_DEVFN, port, SB_MWR_NP, reg, &val);
 }
diff --git a/drivers/gpu/drm/i915/vlv_sideband.h b/drivers/gpu/drm/i915/vlv_sideband.h
index 9ce283d96b80..4073966860e2 100644
--- a/drivers/gpu/drm/i915/vlv_sideband.h
+++ b/drivers/gpu/drm/i915/vlv_sideband.h
@@ -11,7 +11,7 @@
 
 #include "vlv_sideband_reg.h"
 
-enum pipe;
+enum dpio_phy;
 struct drm_i915_private;
 
 enum {
@@ -75,9 +75,9 @@ static inline void vlv_dpio_get(struct drm_i915_private *i915)
 	vlv_iosf_sb_get(i915, BIT(VLV_IOSF_SB_DPIO));
 }
 
-u32 vlv_dpio_read(struct drm_i915_private *i915, enum pipe pipe, int reg);
+u32 vlv_dpio_read(struct drm_i915_private *i915, enum dpio_phy phy, int reg);
 void vlv_dpio_write(struct drm_i915_private *i915,
-		    enum pipe pipe, int reg, u32 val);
+		    enum dpio_phy phy, int reg, u32 val);
 
 static inline void vlv_dpio_put(struct drm_i915_private *i915)
 {
diff --git a/drivers/gpu/drm/imx/dcss/dcss-drv.c b/drivers/gpu/drm/imx/dcss/dcss-drv.c
index b61cec0cc79d..ad5f29ea8f6a 100644
--- a/drivers/gpu/drm/imx/dcss/dcss-drv.c
+++ b/drivers/gpu/drm/imx/dcss/dcss-drv.c
@@ -80,7 +80,7 @@ static int dcss_drv_platform_probe(struct platform_device *pdev)
 	return err;
 }
 
-static int dcss_drv_platform_remove(struct platform_device *pdev)
+static void dcss_drv_platform_remove(struct platform_device *pdev)
 {
 	struct dcss_drv *mdrv = dev_get_drvdata(&pdev->dev);
 
@@ -88,8 +88,6 @@ static int dcss_drv_platform_remove(struct platform_device *pdev)
 	dcss_dev_destroy(mdrv->dcss);
 
 	kfree(mdrv);
-
-	return 0;
 }
 
 static void dcss_drv_platform_shutdown(struct platform_device *pdev)
@@ -120,7 +118,7 @@ MODULE_DEVICE_TABLE(of, dcss_of_match);
 
 static struct platform_driver dcss_platform_driver = {
 	.probe	= dcss_drv_platform_probe,
-	.remove	= dcss_drv_platform_remove,
+	.remove_new = dcss_drv_platform_remove,
 	.shutdown = dcss_drv_platform_shutdown,
 	.driver	= {
 		.name = "imx-dcss",
diff --git a/drivers/gpu/drm/imx/lcdc/imx-lcdc.c b/drivers/gpu/drm/imx/lcdc/imx-lcdc.c
index 22b65f4a0e30..0902983374d0 100644
--- a/drivers/gpu/drm/imx/lcdc/imx-lcdc.c
+++ b/drivers/gpu/drm/imx/lcdc/imx-lcdc.c
@@ -515,14 +515,12 @@ static int imx_lcdc_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int imx_lcdc_remove(struct platform_device *pdev)
+static void imx_lcdc_remove(struct platform_device *pdev)
 {
 	struct drm_device *drm = platform_get_drvdata(pdev);
 
 	drm_dev_unregister(drm);
 	drm_atomic_helper_shutdown(drm);
-
-	return 0;
 }
 
 static void imx_lcdc_shutdown(struct platform_device *pdev)
@@ -536,7 +534,7 @@ static struct platform_driver imx_lcdc_driver = {
 		.of_match_table = imx_lcdc_of_dev_id,
 	},
 	.probe = imx_lcdc_probe,
-	.remove = imx_lcdc_remove,
+	.remove_new = imx_lcdc_remove,
 	.shutdown = imx_lcdc_shutdown,
 };
 module_platform_driver(imx_lcdc_driver);
diff --git a/drivers/gpu/drm/kmb/kmb_drv.c b/drivers/gpu/drm/kmb/kmb_drv.c
index 24035b53441c..169b83987ce2 100644
--- a/drivers/gpu/drm/kmb/kmb_drv.c
+++ b/drivers/gpu/drm/kmb/kmb_drv.c
@@ -448,7 +448,7 @@ static const struct drm_driver kmb_driver = {
 	.minor = DRIVER_MINOR,
 };
 
-static int kmb_remove(struct platform_device *pdev)
+static void kmb_remove(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct drm_device *drm = dev_get_drvdata(dev);
@@ -473,7 +473,6 @@ static int kmb_remove(struct platform_device *pdev)
 	/* Unregister DSI host */
 	kmb_dsi_host_unregister(kmb->kmb_dsi);
 	drm_atomic_helper_shutdown(drm);
-	return 0;
 }
 
 static int kmb_probe(struct platform_device *pdev)
@@ -621,7 +620,7 @@ static SIMPLE_DEV_PM_OPS(kmb_pm_ops, kmb_pm_suspend, kmb_pm_resume);
 
 static struct platform_driver kmb_platform_driver = {
 	.probe = kmb_probe,
-	.remove = kmb_remove,
+	.remove_new = kmb_remove,
 	.driver = {
 		.name = "kmb-drm",
 		.pm = &kmb_pm_ops,
diff --git a/drivers/gpu/drm/lima/lima_device.c b/drivers/gpu/drm/lima/lima_device.c
index 02cef0cea657..0bf7105c8748 100644
--- a/drivers/gpu/drm/lima/lima_device.c
+++ b/drivers/gpu/drm/lima/lima_device.c
@@ -514,7 +514,7 @@ int lima_device_suspend(struct device *dev)
 
 	/* check any task running */
 	for (i = 0; i < lima_pipe_num; i++) {
-		if (atomic_read(&ldev->pipe[i].base.hw_rq_count))
+		if (atomic_read(&ldev->pipe[i].base.credit_count))
 			return -EBUSY;
 	}
 
diff --git a/drivers/gpu/drm/lima/lima_sched.c b/drivers/gpu/drm/lima/lima_sched.c
index 295f0353a02e..c3bf8cda8498 100644
--- a/drivers/gpu/drm/lima/lima_sched.c
+++ b/drivers/gpu/drm/lima/lima_sched.c
@@ -123,7 +123,7 @@ int lima_sched_task_init(struct lima_sched_task *task,
 	for (i = 0; i < num_bos; i++)
 		drm_gem_object_get(&bos[i]->base.base);
 
-	err = drm_sched_job_init(&task->base, &context->base, vm);
+	err = drm_sched_job_init(&task->base, &context->base, 1, vm);
 	if (err) {
 		kfree(task->bos);
 		return err;
@@ -488,7 +488,7 @@ int lima_sched_pipe_init(struct lima_sched_pipe *pipe, const char *name)
 
 	INIT_WORK(&pipe->recover_work, lima_sched_recover_work);
 
-	return drm_sched_init(&pipe->base, &lima_sched_ops,
+	return drm_sched_init(&pipe->base, &lima_sched_ops, NULL,
 			      DRM_SCHED_PRIORITY_COUNT,
 			      1,
 			      lima_job_hang_limit,
diff --git a/drivers/gpu/drm/mediatek/mtk_disp_ovl_adaptor.c b/drivers/gpu/drm/mediatek/mtk_disp_ovl_adaptor.c
index 6bf6367853fb..3fdef3ad4ffd 100644
--- a/drivers/gpu/drm/mediatek/mtk_disp_ovl_adaptor.c
+++ b/drivers/gpu/drm/mediatek/mtk_disp_ovl_adaptor.c
@@ -531,16 +531,15 @@ static int mtk_disp_ovl_adaptor_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int mtk_disp_ovl_adaptor_remove(struct platform_device *pdev)
+static void mtk_disp_ovl_adaptor_remove(struct platform_device *pdev)
 {
 	component_master_del(&pdev->dev, &mtk_disp_ovl_adaptor_master_ops);
 	pm_runtime_disable(&pdev->dev);
-	return 0;
 }
 
 struct platform_driver mtk_disp_ovl_adaptor_driver = {
 	.probe		= mtk_disp_ovl_adaptor_probe,
-	.remove		= mtk_disp_ovl_adaptor_remove,
+	.remove_new	= mtk_disp_ovl_adaptor_remove,
 	.driver		= {
 		.name	= "mediatek-disp-ovl-adaptor",
 		.owner	= THIS_MODULE,
diff --git a/drivers/gpu/drm/mediatek/mtk_ethdr.c b/drivers/gpu/drm/mediatek/mtk_ethdr.c
index db7ac666ec5e..6a5d0c345aab 100644
--- a/drivers/gpu/drm/mediatek/mtk_ethdr.c
+++ b/drivers/gpu/drm/mediatek/mtk_ethdr.c
@@ -346,10 +346,9 @@ static int mtk_ethdr_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int mtk_ethdr_remove(struct platform_device *pdev)
+static void mtk_ethdr_remove(struct platform_device *pdev)
 {
 	component_del(&pdev->dev, &mtk_ethdr_component_ops);
-	return 0;
 }
 
 static const struct of_device_id mtk_ethdr_driver_dt_match[] = {
@@ -361,7 +360,7 @@ MODULE_DEVICE_TABLE(of, mtk_ethdr_driver_dt_match);
 
 struct platform_driver mtk_ethdr_driver = {
 	.probe		= mtk_ethdr_probe,
-	.remove		= mtk_ethdr_remove,
+	.remove_new	= mtk_ethdr_remove,
 	.driver		= {
 		.name	= "mediatek-disp-ethdr",
 		.owner	= THIS_MODULE,
diff --git a/drivers/gpu/drm/meson/meson_dw_mipi_dsi.c b/drivers/gpu/drm/meson/meson_dw_mipi_dsi.c
index e5fe4e994f43..a6bc1bdb3d0d 100644
--- a/drivers/gpu/drm/meson/meson_dw_mipi_dsi.c
+++ b/drivers/gpu/drm/meson/meson_dw_mipi_dsi.c
@@ -323,13 +323,11 @@ static int meson_dw_mipi_dsi_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int meson_dw_mipi_dsi_remove(struct platform_device *pdev)
+static void meson_dw_mipi_dsi_remove(struct platform_device *pdev)
 {
 	struct meson_dw_mipi_dsi *mipi_dsi = platform_get_drvdata(pdev);
 
 	dw_mipi_dsi_remove(mipi_dsi->dmd);
-
-	return 0;
 }
 
 static const struct of_device_id meson_dw_mipi_dsi_of_table[] = {
@@ -340,7 +338,7 @@ MODULE_DEVICE_TABLE(of, meson_dw_mipi_dsi_of_table);
 
 static struct platform_driver meson_dw_mipi_dsi_platform_driver = {
 	.probe		= meson_dw_mipi_dsi_probe,
-	.remove		= meson_dw_mipi_dsi_remove,
+	.remove_new	= meson_dw_mipi_dsi_remove,
 	.driver		= {
 		.name		= DRIVER_NAME,
 		.of_match_table	= meson_dw_mipi_dsi_of_table,
diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c
index 41b13dec9bef..f62ab5257e66 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_device.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_device.c
@@ -841,7 +841,8 @@ static void suspend_scheduler(struct msm_gpu *gpu)
 	 */
 	for (i = 0; i < gpu->nr_rings; i++) {
 		struct drm_gpu_scheduler *sched = &gpu->rb[i]->sched;
-		kthread_park(sched->thread);
+
+		drm_sched_wqueue_stop(sched);
 	}
 }
 
@@ -851,7 +852,8 @@ static void resume_scheduler(struct msm_gpu *gpu)
 
 	for (i = 0; i < gpu->nr_rings; i++) {
 		struct drm_gpu_scheduler *sched = &gpu->rb[i]->sched;
-		kthread_unpark(sched->thread);
+
+		drm_sched_wqueue_start(sched);
 	}
 }
 
diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c
index 99744de6c05a..c002cabe7b9c 100644
--- a/drivers/gpu/drm/msm/msm_gem_submit.c
+++ b/drivers/gpu/drm/msm/msm_gem_submit.c
@@ -48,7 +48,7 @@ static struct msm_gem_submit *submit_create(struct drm_device *dev,
 		return ERR_PTR(ret);
 	}
 
-	ret = drm_sched_job_init(&submit->base, queue->entity, queue);
+	ret = drm_sched_job_init(&submit->base, queue->entity, 1, queue);
 	if (ret) {
 		kfree(submit->hw_fence);
 		kfree(submit);
diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.c b/drivers/gpu/drm/msm/msm_ringbuffer.c
index 95257ab0185d..4968568e3b54 100644
--- a/drivers/gpu/drm/msm/msm_ringbuffer.c
+++ b/drivers/gpu/drm/msm/msm_ringbuffer.c
@@ -94,7 +94,7 @@ struct msm_ringbuffer *msm_ringbuffer_new(struct msm_gpu *gpu, int id,
 	 /* currently managing hangcheck ourselves: */
 	sched_timeout = MAX_SCHEDULE_TIMEOUT;
 
-	ret = drm_sched_init(&ring->sched, &msm_sched_ops,
+	ret = drm_sched_init(&ring->sched, &msm_sched_ops, NULL,
 			     DRM_SCHED_PRIORITY_COUNT,
 			     num_hw_submissions, 0, sched_timeout,
 			     NULL, NULL, to_msm_bo(ring->bo)->name, gpu->dev->dev);
diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c
index 7840b6428afb..11fe75b68e95 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/disp.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c
@@ -38,7 +38,9 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_eld.h>
 #include <drm/drm_fb_helper.h>
+#include <drm/drm_fixed.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_vblank.h>
 
@@ -945,7 +947,8 @@ nv50_msto_prepare(struct drm_atomic_state *state,
 	if (ret == 0) {
 		nvif_outp_dp_mst_vcpi(&mstm->outp->outp, msto->head->base.index,
 				      payload->vc_start_slot, payload->time_slots,
-				      payload->pbn, payload->time_slots * mst_state->pbn_div);
+				      payload->pbn,
+				      payload->time_slots * dfixed_trunc(mst_state->pbn_div));
 	} else {
 		nvif_outp_dp_mst_vcpi(&mstm->outp->outp, msto->head->base.index, 0, 0, 0, 0);
 	}
@@ -982,15 +985,14 @@ nv50_msto_atomic_check(struct drm_encoder *encoder,
 		const int clock = crtc_state->adjusted_mode.clock;
 
 		asyh->or.bpc = connector->display_info.bpc;
-		asyh->dp.pbn = drm_dp_calc_pbn_mode(clock, asyh->or.bpc * 3,
-						    false);
+		asyh->dp.pbn = drm_dp_calc_pbn_mode(clock, asyh->or.bpc * 3 << 4);
 	}
 
 	mst_state = drm_atomic_get_mst_topology_state(state, &mstm->mgr);
 	if (IS_ERR(mst_state))
 		return PTR_ERR(mst_state);
 
-	if (!mst_state->pbn_div) {
+	if (!mst_state->pbn_div.full) {
 		struct nouveau_encoder *outp = mstc->mstm->outp;
 
 		mst_state->pbn_div = drm_dp_get_vc_payload_bw(&mstm->mgr,
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index 0f3bd187ede6..7afad86da64b 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -148,10 +148,17 @@ nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
 	 * If nouveau_bo_new() allocated this buffer, the GEM object was never
 	 * initialized, so don't attempt to release it.
 	 */
-	if (bo->base.dev)
+	if (bo->base.dev) {
+		/* Gem objects not being shared with other VMs get their
+		 * dma_resv from a root GEM object.
+		 */
+		if (nvbo->no_share)
+			drm_gem_object_put(nvbo->r_obj);
+
 		drm_gem_object_release(&bo->base);
-	else
+	} else {
 		dma_resv_fini(&bo->base._resv);
+	}
 
 	kfree(nvbo);
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.h b/drivers/gpu/drm/nouveau/nouveau_bo.h
index 07f671cf895e..70c551921a9e 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.h
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.h
@@ -26,6 +26,11 @@ struct nouveau_bo {
 	struct list_head entry;
 	int pbbo_index;
 	bool validate_mapped;
+
+	/* Root GEM object we derive the dma_resv of in case this BO is not
+	 * shared between VMs.
+	 */
+	struct drm_gem_object *r_obj;
 	bool no_share;
 
 	/* GPU address space is independent of CPU word size */
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
index 50589f982d1a..f603eaef1560 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
@@ -190,6 +190,8 @@ nouveau_cli_work_queue(struct nouveau_cli *cli, struct dma_fence *fence,
 static void
 nouveau_cli_fini(struct nouveau_cli *cli)
 {
+	struct nouveau_uvmm *uvmm = nouveau_cli_uvmm_locked(cli);
+
 	/* All our channels are dead now, which means all the fences they
 	 * own are signalled, and all callback functions have been called.
 	 *
@@ -199,7 +201,8 @@ nouveau_cli_fini(struct nouveau_cli *cli)
 	WARN_ON(!list_empty(&cli->worker));
 
 	usif_client_fini(cli);
-	nouveau_uvmm_fini(&cli->uvmm);
+	if (uvmm)
+		nouveau_uvmm_fini(uvmm);
 	nouveau_sched_entity_fini(&cli->sched_entity);
 	nouveau_vmm_fini(&cli->svm);
 	nouveau_vmm_fini(&cli->vmm);
diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h
index e73a233c6572..7f7051df84a6 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drv.h
+++ b/drivers/gpu/drm/nouveau/nouveau_drv.h
@@ -93,7 +93,10 @@ struct nouveau_cli {
 	struct nvif_mmu mmu;
 	struct nouveau_vmm vmm;
 	struct nouveau_vmm svm;
-	struct nouveau_uvmm uvmm;
+	struct {
+		struct nouveau_uvmm *ptr;
+		bool disabled;
+	} uvmm;
 
 	struct nouveau_sched_entity sched_entity;
 
@@ -121,10 +124,7 @@ struct nouveau_cli_work {
 static inline struct nouveau_uvmm *
 nouveau_cli_uvmm(struct nouveau_cli *cli)
 {
-	if (!cli || !cli->uvmm.vmm.cli)
-		return NULL;
-
-	return &cli->uvmm;
+	return cli ? cli->uvmm.ptr : NULL;
 }
 
 static inline struct nouveau_uvmm *
diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c
index a0d303e5ce3d..49c2bcbef129 100644
--- a/drivers/gpu/drm/nouveau/nouveau_gem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_gem.c
@@ -111,7 +111,8 @@ nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv)
 	if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50)
 		return 0;
 
-	if (nvbo->no_share && uvmm && &uvmm->resv != nvbo->bo.base.resv)
+	if (nvbo->no_share && uvmm &&
+	    drm_gpuvm_resv(&uvmm->base) != nvbo->bo.base.resv)
 		return -EPERM;
 
 	ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
@@ -245,7 +246,7 @@ nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain,
 		if (unlikely(!uvmm))
 			return -EINVAL;
 
-		resv = &uvmm->resv;
+		resv = drm_gpuvm_resv(&uvmm->base);
 	}
 
 	if (!(domain & (NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART)))
@@ -288,6 +289,11 @@ nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain,
 	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA)
 		nvbo->valid_domains &= domain;
 
+	if (nvbo->no_share) {
+		nvbo->r_obj = drm_gpuvm_resv_obj(&uvmm->base);
+		drm_gem_object_get(nvbo->r_obj);
+	}
+
 	*pnvbo = nvbo;
 	return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_platform.c b/drivers/gpu/drm/nouveau/nouveau_platform.c
index 23cd43a7fd19..bf2dc7567ea4 100644
--- a/drivers/gpu/drm/nouveau/nouveau_platform.c
+++ b/drivers/gpu/drm/nouveau/nouveau_platform.c
@@ -43,11 +43,10 @@ static int nouveau_platform_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int nouveau_platform_remove(struct platform_device *pdev)
+static void nouveau_platform_remove(struct platform_device *pdev)
 {
 	struct drm_device *dev = platform_get_drvdata(pdev);
 	nouveau_drm_device_remove(dev);
-	return 0;
 }
 
 #if IS_ENABLED(CONFIG_OF)
@@ -93,5 +92,5 @@ struct platform_driver nouveau_platform_driver = {
 		.of_match_table = of_match_ptr(nouveau_platform_match),
 	},
 	.probe = nouveau_platform_probe,
-	.remove = nouveau_platform_remove,
+	.remove_new = nouveau_platform_remove,
 };
diff --git a/drivers/gpu/drm/nouveau/nouveau_sched.c b/drivers/gpu/drm/nouveau/nouveau_sched.c
index 7c376c4ccdcf..e03fd2bc8a11 100644
--- a/drivers/gpu/drm/nouveau/nouveau_sched.c
+++ b/drivers/gpu/drm/nouveau/nouveau_sched.c
@@ -89,7 +89,7 @@ nouveau_job_init(struct nouveau_job *job,
 
 	}
 
-	ret = drm_sched_job_init(&job->base, &entity->base, NULL);
+	ret = drm_sched_job_init(&job->base, &entity->base, 1, NULL);
 	if (ret)
 		goto err_free_chains;
 
@@ -435,7 +435,7 @@ int nouveau_sched_init(struct nouveau_drm *drm)
 	if (!drm->sched_wq)
 		return -ENOMEM;
 
-	return drm_sched_init(sched, &nouveau_sched_ops,
+	return drm_sched_init(sched, &nouveau_sched_ops, NULL,
 			      DRM_SCHED_PRIORITY_COUNT,
 			      NOUVEAU_SCHED_HW_SUBMISSIONS, 0, job_hang_limit,
 			      NULL, NULL, "nouveau_sched", drm->dev->dev);
diff --git a/drivers/gpu/drm/nouveau/nouveau_uvmm.c b/drivers/gpu/drm/nouveau/nouveau_uvmm.c
index 5cf892c50f43..eda7bb8624f1 100644
--- a/drivers/gpu/drm/nouveau/nouveau_uvmm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_uvmm.c
@@ -62,6 +62,8 @@ struct bind_job_op {
 	enum vm_bind_op op;
 	u32 flags;
 
+	struct drm_gpuvm_bo *vm_bo;
+
 	struct {
 		u64 addr;
 		u64 range;
@@ -929,25 +931,13 @@ nouveau_uvmm_sm_unmap_cleanup(struct nouveau_uvmm *uvmm,
 static int
 nouveau_uvmm_validate_range(struct nouveau_uvmm *uvmm, u64 addr, u64 range)
 {
-	u64 end = addr + range;
-	u64 kernel_managed_end = uvmm->kernel_managed_addr +
-				 uvmm->kernel_managed_size;
-
 	if (addr & ~PAGE_MASK)
 		return -EINVAL;
 
 	if (range & ~PAGE_MASK)
 		return -EINVAL;
 
-	if (end <= addr)
-		return -EINVAL;
-
-	if (addr < NOUVEAU_VA_SPACE_START ||
-	    end > NOUVEAU_VA_SPACE_END)
-		return -EINVAL;
-
-	if (addr < kernel_managed_end &&
-	    end > uvmm->kernel_managed_addr)
+	if (!drm_gpuvm_range_valid(&uvmm->base, addr, range))
 		return -EINVAL;
 
 	return 0;
@@ -1113,22 +1103,28 @@ bind_validate_region(struct nouveau_job *job)
 }
 
 static void
-bind_link_gpuvas(struct drm_gpuva_ops *ops, struct nouveau_uvma_prealloc *new)
+bind_link_gpuvas(struct bind_job_op *bop)
 {
+	struct nouveau_uvma_prealloc *new = &bop->new;
+	struct drm_gpuvm_bo *vm_bo = bop->vm_bo;
+	struct drm_gpuva_ops *ops = bop->ops;
 	struct drm_gpuva_op *op;
 
 	drm_gpuva_for_each_op(op, ops) {
 		switch (op->op) {
 		case DRM_GPUVA_OP_MAP:
-			drm_gpuva_link(&new->map->va);
+			drm_gpuva_link(&new->map->va, vm_bo);
 			break;
-		case DRM_GPUVA_OP_REMAP:
+		case DRM_GPUVA_OP_REMAP: {
+			struct drm_gpuva *va = op->remap.unmap->va;
+
 			if (op->remap.prev)
-				drm_gpuva_link(&new->prev->va);
+				drm_gpuva_link(&new->prev->va, va->vm_bo);
 			if (op->remap.next)
-				drm_gpuva_link(&new->next->va);
-			drm_gpuva_unlink(op->remap.unmap->va);
+				drm_gpuva_link(&new->next->va, va->vm_bo);
+			drm_gpuva_unlink(va);
 			break;
+		}
 		case DRM_GPUVA_OP_UNMAP:
 			drm_gpuva_unlink(op->unmap.va);
 			break;
@@ -1150,10 +1146,17 @@ nouveau_uvmm_bind_job_submit(struct nouveau_job *job)
 
 	list_for_each_op(op, &bind_job->ops) {
 		if (op->op == OP_MAP) {
-			op->gem.obj = drm_gem_object_lookup(job->file_priv,
-							    op->gem.handle);
-			if (!op->gem.obj)
+			struct drm_gem_object *obj = op->gem.obj =
+				drm_gem_object_lookup(job->file_priv,
+						      op->gem.handle);
+			if (!obj)
 				return -ENOENT;
+
+			dma_resv_lock(obj->resv, NULL);
+			op->vm_bo = drm_gpuvm_bo_obtain(&uvmm->base, obj);
+			dma_resv_unlock(obj->resv);
+			if (IS_ERR(op->vm_bo))
+				return PTR_ERR(op->vm_bo);
 		}
 
 		ret = bind_validate_op(job, op);
@@ -1364,7 +1367,7 @@ nouveau_uvmm_bind_job_submit(struct nouveau_job *job)
 		case OP_UNMAP_SPARSE:
 		case OP_MAP:
 		case OP_UNMAP:
-			bind_link_gpuvas(op->ops, &op->new);
+			bind_link_gpuvas(op);
 			break;
 		default:
 			break;
@@ -1511,6 +1514,12 @@ nouveau_uvmm_bind_job_free_work_fn(struct work_struct *work)
 		if (!IS_ERR_OR_NULL(op->ops))
 			drm_gpuva_ops_free(&uvmm->base, op->ops);
 
+		if (!IS_ERR_OR_NULL(op->vm_bo)) {
+			dma_resv_lock(obj->resv, NULL);
+			drm_gpuvm_bo_put(op->vm_bo);
+			dma_resv_unlock(obj->resv);
+		}
+
 		if (obj)
 			drm_gem_object_put(obj);
 	}
@@ -1648,18 +1657,6 @@ nouveau_uvmm_bind_job_init(struct nouveau_uvmm_bind_job **pjob,
 	return ret;
 }
 
-int
-nouveau_uvmm_ioctl_vm_init(struct drm_device *dev,
-			   void *data,
-			   struct drm_file *file_priv)
-{
-	struct nouveau_cli *cli = nouveau_cli(file_priv);
-	struct drm_nouveau_vm_init *init = data;
-
-	return nouveau_uvmm_init(&cli->uvmm, cli, init->kernel_managed_addr,
-				 init->kernel_managed_size);
-}
-
 static int
 nouveau_uvmm_vm_bind(struct nouveau_uvmm_bind_job_args *args)
 {
@@ -1776,15 +1773,18 @@ void
 nouveau_uvmm_bo_map_all(struct nouveau_bo *nvbo, struct nouveau_mem *mem)
 {
 	struct drm_gem_object *obj = &nvbo->bo.base;
+	struct drm_gpuvm_bo *vm_bo;
 	struct drm_gpuva *va;
 
 	dma_resv_assert_held(obj->resv);
 
-	drm_gem_for_each_gpuva(va, obj) {
-		struct nouveau_uvma *uvma = uvma_from_va(va);
+	drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
+		drm_gpuvm_bo_for_each_va(va, vm_bo) {
+			struct nouveau_uvma *uvma = uvma_from_va(va);
 
-		nouveau_uvma_map(uvma, mem);
-		drm_gpuva_invalidate(va, false);
+			nouveau_uvma_map(uvma, mem);
+			drm_gpuva_invalidate(va, false);
+		}
 	}
 }
 
@@ -1792,29 +1792,53 @@ void
 nouveau_uvmm_bo_unmap_all(struct nouveau_bo *nvbo)
 {
 	struct drm_gem_object *obj = &nvbo->bo.base;
+	struct drm_gpuvm_bo *vm_bo;
 	struct drm_gpuva *va;
 
 	dma_resv_assert_held(obj->resv);
 
-	drm_gem_for_each_gpuva(va, obj) {
-		struct nouveau_uvma *uvma = uvma_from_va(va);
+	drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
+		drm_gpuvm_bo_for_each_va(va, vm_bo) {
+			struct nouveau_uvma *uvma = uvma_from_va(va);
 
-		nouveau_uvma_unmap(uvma);
-		drm_gpuva_invalidate(va, true);
+			nouveau_uvma_unmap(uvma);
+			drm_gpuva_invalidate(va, true);
+		}
 	}
 }
 
+static void
+nouveau_uvmm_free(struct drm_gpuvm *gpuvm)
+{
+	struct nouveau_uvmm *uvmm = uvmm_from_gpuvm(gpuvm);
+
+	kfree(uvmm);
+}
+
+static const struct drm_gpuvm_ops gpuvm_ops = {
+	.vm_free = nouveau_uvmm_free,
+};
+
 int
-nouveau_uvmm_init(struct nouveau_uvmm *uvmm, struct nouveau_cli *cli,
-		  u64 kernel_managed_addr, u64 kernel_managed_size)
+nouveau_uvmm_ioctl_vm_init(struct drm_device *dev,
+			   void *data,
+			   struct drm_file *file_priv)
 {
+	struct nouveau_uvmm *uvmm;
+	struct nouveau_cli *cli = nouveau_cli(file_priv);
+	struct drm_device *drm = cli->drm->dev;
+	struct drm_gem_object *r_obj;
+	struct drm_nouveau_vm_init *init = data;
+	u64 kernel_managed_end;
 	int ret;
-	u64 kernel_managed_end = kernel_managed_addr + kernel_managed_size;
 
-	mutex_init(&uvmm->mutex);
-	dma_resv_init(&uvmm->resv);
-	mt_init_flags(&uvmm->region_mt, MT_FLAGS_LOCK_EXTERN);
-	mt_set_external_lock(&uvmm->region_mt, &uvmm->mutex);
+	if (check_add_overflow(init->kernel_managed_addr,
+			       init->kernel_managed_size,
+			       &kernel_managed_end))
+		return -EINVAL;
+
+	if (kernel_managed_end > NOUVEAU_VA_SPACE_END)
+		return -EINVAL;
 
 	mutex_lock(&cli->mutex);
 
@@ -1823,39 +1847,48 @@ nouveau_uvmm_init(struct nouveau_uvmm *uvmm, struct nouveau_cli *cli,
 		goto out_unlock;
 	}
 
-	if (kernel_managed_end <= kernel_managed_addr) {
-		ret = -EINVAL;
+	uvmm = kzalloc(sizeof(*uvmm), GFP_KERNEL);
+	if (!uvmm) {
+		ret = -ENOMEM;
 		goto out_unlock;
 	}
 
-	if (kernel_managed_end > NOUVEAU_VA_SPACE_END) {
-		ret = -EINVAL;
+	r_obj = drm_gpuvm_resv_object_alloc(drm);
+	if (!r_obj) {
+		kfree(uvmm);
+		ret = -ENOMEM;
 		goto out_unlock;
 	}
 
-	uvmm->kernel_managed_addr = kernel_managed_addr;
-	uvmm->kernel_managed_size = kernel_managed_size;
+	mutex_init(&uvmm->mutex);
+	mt_init_flags(&uvmm->region_mt, MT_FLAGS_LOCK_EXTERN);
+	mt_set_external_lock(&uvmm->region_mt, &uvmm->mutex);
 
-	drm_gpuvm_init(&uvmm->base, cli->name,
+	drm_gpuvm_init(&uvmm->base, cli->name, 0, drm, r_obj,
 		       NOUVEAU_VA_SPACE_START,
 		       NOUVEAU_VA_SPACE_END,
-		       kernel_managed_addr, kernel_managed_size,
-		       NULL);
+		       init->kernel_managed_addr,
+		       init->kernel_managed_size,
+		       &gpuvm_ops);
+	/* GPUVM takes care from here on. */
+	drm_gem_object_put(r_obj);
 
 	ret = nvif_vmm_ctor(&cli->mmu, "uvmm",
 			    cli->vmm.vmm.object.oclass, RAW,
-			    kernel_managed_addr, kernel_managed_size,
-			    NULL, 0, &cli->uvmm.vmm.vmm);
+			    init->kernel_managed_addr,
+			    init->kernel_managed_size,
+			    NULL, 0, &uvmm->vmm.vmm);
 	if (ret)
-		goto out_free_gpuva_mgr;
+		goto out_gpuvm_fini;
 
-	cli->uvmm.vmm.cli = cli;
+	uvmm->vmm.cli = cli;
+	cli->uvmm.ptr = uvmm;
 	mutex_unlock(&cli->mutex);
 
 	return 0;
 
-out_free_gpuva_mgr:
-	drm_gpuvm_destroy(&uvmm->base);
+out_gpuvm_fini:
+	drm_gpuvm_put(&uvmm->base);
 out_unlock:
 	mutex_unlock(&cli->mutex);
 	return ret;
@@ -1870,9 +1903,6 @@ nouveau_uvmm_fini(struct nouveau_uvmm *uvmm)
 	struct nouveau_sched_entity *entity = &cli->sched_entity;
 	struct drm_gpuva *va, *next;
 
-	if (!cli)
-		return;
-
 	rmb(); /* for list_empty to work without lock */
 	wait_event(entity->job.wq, list_empty(&entity->job.list.head));
 
@@ -1910,8 +1940,6 @@ nouveau_uvmm_fini(struct nouveau_uvmm *uvmm)
 
 	mutex_lock(&cli->mutex);
 	nouveau_vmm_fini(&uvmm->vmm);
-	drm_gpuvm_destroy(&uvmm->base);
+	drm_gpuvm_put(&uvmm->base);
 	mutex_unlock(&cli->mutex);
-
-	dma_resv_fini(&uvmm->resv);
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_uvmm.h b/drivers/gpu/drm/nouveau/nouveau_uvmm.h
index a308c59760a5..f0a6d98ace4f 100644
--- a/drivers/gpu/drm/nouveau/nouveau_uvmm.h
+++ b/drivers/gpu/drm/nouveau/nouveau_uvmm.h
@@ -12,12 +12,6 @@ struct nouveau_uvmm {
 	struct nouveau_vmm vmm;
 	struct maple_tree region_mt;
 	struct mutex mutex;
-	struct dma_resv resv;
-
-	u64 kernel_managed_addr;
-	u64 kernel_managed_size;
-
-	bool disabled;
 };
 
 struct nouveau_uvma_region {
@@ -82,8 +76,6 @@ struct nouveau_uvmm_bind_job_args {
 
 #define to_uvmm_bind_job(job) container_of((job), struct nouveau_uvmm_bind_job, base)
 
-int nouveau_uvmm_init(struct nouveau_uvmm *uvmm, struct nouveau_cli *cli,
-		      u64 kernel_managed_addr, u64 kernel_managed_size);
 void nouveau_uvmm_fini(struct nouveau_uvmm *uvmm);
 
 void nouveau_uvmm_bo_map_all(struct nouveau_bo *nvbov, struct nouveau_mem *mem);
diff --git a/drivers/gpu/drm/nouveau/nv04_fence.c b/drivers/gpu/drm/nouveau/nv04_fence.c
index 5b71a5a5cd85..cdbc75e3d1f6 100644
--- a/drivers/gpu/drm/nouveau/nv04_fence.c
+++ b/drivers/gpu/drm/nouveau/nv04_fence.c
@@ -39,7 +39,7 @@ struct nv04_fence_priv {
 static int
 nv04_fence_emit(struct nouveau_fence *fence)
 {
-	struct nvif_push *push = fence->channel->chan.push;
+	struct nvif_push *push = unrcu_pointer(fence->channel)->chan.push;
 	int ret = PUSH_WAIT(push, 2);
 	if (ret == 0) {
 		PUSH_NVSQ(push, NV_SW, 0x0150, fence->base.seqno);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/r535.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/r535.c
index 3adbb05ff587..d088e636edc3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/r535.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/r535.c
@@ -539,7 +539,7 @@ r535_fifo_runl_ctor(struct nvkm_fifo *fifo)
 	struct nvkm_runl *runl;
 	struct nvkm_engn *engn;
 	u32 cgids = 2048;
-	u32 chids = 2048 / CHID_PER_USERD;
+	u32 chids = 2048;
 	int ret;
 	NV2080_CTRL_FIFO_GET_DEVICE_INFO_TABLE_PARAMS *ctrl;
 
diff --git a/drivers/gpu/drm/omapdrm/omap_drv.c b/drivers/gpu/drm/omapdrm/omap_drv.c
index b2835b3ea6f5..6598c9c08ba1 100644
--- a/drivers/gpu/drm/omapdrm/omap_drv.c
+++ b/drivers/gpu/drm/omapdrm/omap_drv.c
@@ -69,7 +69,6 @@ static void omap_atomic_commit_tail(struct drm_atomic_state *old_state)
 {
 	struct drm_device *dev = old_state->dev;
 	struct omap_drm_private *priv = dev->dev_private;
-	bool fence_cookie = dma_fence_begin_signalling();
 
 	dispc_runtime_get(priv->dispc);
 
@@ -92,6 +91,8 @@ static void omap_atomic_commit_tail(struct drm_atomic_state *old_state)
 		omap_atomic_wait_for_completion(dev, old_state);
 
 		drm_atomic_helper_commit_planes(dev, old_state, 0);
+
+		drm_atomic_helper_commit_hw_done(old_state);
 	} else {
 		/*
 		 * OMAP3 DSS seems to have issues with the work-around above,
@@ -101,11 +102,9 @@ static void omap_atomic_commit_tail(struct drm_atomic_state *old_state)
 		drm_atomic_helper_commit_planes(dev, old_state, 0);
 
 		drm_atomic_helper_commit_modeset_enables(dev, old_state);
-	}
 
-	drm_atomic_helper_commit_hw_done(old_state);
-
-	dma_fence_end_signalling(fence_cookie);
+		drm_atomic_helper_commit_hw_done(old_state);
+	}
 
 	/*
 	 * Wait for completion of the page flips to ensure that old buffers
diff --git a/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c b/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c
index 9323e7b9e384..be8f48e3c1db 100644
--- a/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c
+++ b/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c
@@ -1709,6 +1709,7 @@ static const struct panel_desc auo_b101uan08_3_desc = {
 	.mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_SYNC_PULSE |
 		      MIPI_DSI_MODE_LPM,
 	.init_cmds = auo_b101uan08_3_init_cmd,
+	.lp11_before_reset = true,
 };
 
 static const struct drm_display_mode boe_tv105wum_nw0_default_mode = {
@@ -1766,11 +1767,11 @@ static const struct panel_desc starry_qfh032011_53g_desc = {
 };
 
 static const struct drm_display_mode starry_himax83102_j02_default_mode = {
-	.clock = 161600,
+	.clock = 162850,
 	.hdisplay = 1200,
-	.hsync_start = 1200 + 40,
-	.hsync_end = 1200 + 40 + 20,
-	.htotal = 1200 + 40 + 20 + 40,
+	.hsync_start = 1200 + 50,
+	.hsync_end = 1200 + 50 + 20,
+	.htotal = 1200 + 50 + 20 + 50,
 	.vdisplay = 1920,
 	.vsync_start = 1920 + 116,
 	.vsync_end = 1920 + 116 + 8,
diff --git a/drivers/gpu/drm/panel/panel-edp.c b/drivers/gpu/drm/panel/panel-edp.c
index 95c8472d878a..f22677373171 100644
--- a/drivers/gpu/drm/panel/panel-edp.c
+++ b/drivers/gpu/drm/panel/panel-edp.c
@@ -973,6 +973,8 @@ static const struct panel_desc auo_b116xak01 = {
 	},
 	.delay = {
 		.hpd_absent = 200,
+		.unprepare = 500,
+		.enable = 50,
 	},
 };
 
@@ -1801,6 +1803,12 @@ static const struct panel_delay delay_200_500_e50 = {
 	.enable = 50,
 };
 
+static const struct panel_delay delay_200_500_e80 = {
+	.hpd_absent = 200,
+	.unprepare = 500,
+	.enable = 80,
+};
+
 static const struct panel_delay delay_200_500_e80_d50 = {
 	.hpd_absent = 200,
 	.unprepare = 500,
@@ -1820,6 +1828,19 @@ static const struct panel_delay delay_200_500_e200 = {
 	.enable = 200,
 };
 
+static const struct panel_delay delay_200_500_e200_d10 = {
+	.hpd_absent = 200,
+	.unprepare = 500,
+	.enable = 200,
+	.disable = 10,
+};
+
+static const struct panel_delay delay_200_150_e200 = {
+	.hpd_absent = 200,
+	.unprepare = 150,
+	.enable = 200,
+};
+
 #define EDP_PANEL_ENTRY(vend_chr_0, vend_chr_1, vend_chr_2, product_id, _delay, _name) \
 { \
 	.name = _name, \
@@ -1840,34 +1861,69 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x145c, &delay_200_500_e50, "B116XAB01.4"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x1e9b, &delay_200_500_e50, "B133UAN02.1"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x1ea5, &delay_200_500_e50, "B116XAK01.6"),
-	EDP_PANEL_ENTRY('A', 'U', 'O', 0x405c, &auo_b116xak01.delay, "B116XAK01"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0x208d, &delay_200_500_e50, "B140HTN02.1"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0x235c, &delay_200_500_e50, "B116XTN02.3"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0x239b, &delay_200_500_e50, "B116XAN06.1"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0x255c, &delay_200_500_e50, "B116XTN02.5"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0x403d, &delay_200_500_e50, "B140HAN04.0"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0x405c, &auo_b116xak01.delay, "B116XAK01.0"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x582d, &delay_200_500_e50, "B133UAN01.0"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x615c, &delay_200_500_e50, "B116XAN06.1"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0x635c, &delay_200_500_e50, "B116XAN06.3"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0x639c, &delay_200_500_e50, "B140HAK02.7"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x8594, &delay_200_500_e50, "B133UAN01.0"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0xf390, &delay_200_500_e50, "B140XTN07.7"),
 
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0715, &delay_200_150_e200, "NT116WHM-N21"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0731, &delay_200_500_e80, "NT116WHM-N42"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0741, &delay_200_500_e200, "NT116WHM-N44"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0786, &delay_200_500_p2e80, "NV116WHM-T01"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x07d1, &boe_nv133fhm_n61.delay, "NV133FHM-N61"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x07f6, &delay_200_500_e200, "NT140FHM-N44"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x082d, &boe_nv133fhm_n61.delay, "NV133FHM-N62"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x08b2, &delay_200_500_e200, "NT140WHM-N49"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x09c3, &delay_200_500_e50, "NT116WHM-N21,836X2"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x094b, &delay_200_500_e50, "NT116WHM-N21"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0951, &delay_200_500_e80, "NV116WHM-N47"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x095f, &delay_200_500_e50, "NE135FBM-N41 v8.1"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0979, &delay_200_500_e50, "NV116WHM-N49 V8.0"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x098d, &boe_nv110wtm_n61.delay, "NV110WTM-N61"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x09ae, &delay_200_500_e200, "NT140FHM-N45"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x09dd, &delay_200_500_e50, "NT116WHM-N21"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0a5d, &delay_200_500_e50, "NV116WHM-N45"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0ac5, &delay_200_500_e50, "NV116WHM-N4C"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b43, &delay_200_500_e200, "NV140FHM-T09"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b56, &delay_200_500_e80, "NT140FHM-N47"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0c20, &delay_200_500_e80, "NT140FHM-N47"),
 
+	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1132, &delay_200_500_e80_d50, "N116BGE-EA2"),
+	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1138, &innolux_n116bca_ea1.delay, "N116BCA-EA1-RC4"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1139, &delay_200_500_e80_d50, "N116BGE-EA2"),
+	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1145, &delay_200_500_e80_d50, "N116BCN-EB1"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x114c, &innolux_n116bca_ea1.delay, "N116BCA-EA1"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1152, &delay_200_500_e80_d50, "N116BCN-EA1"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1153, &delay_200_500_e80_d50, "N116BGE-EA2"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1154, &delay_200_500_e80_d50, "N116BCA-EA2"),
+	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1157, &delay_200_500_e80_d50, "N116BGE-EA2"),
+	EDP_PANEL_ENTRY('C', 'M', 'N', 0x115b, &delay_200_500_e80_d50, "N116BCN-EB1"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1247, &delay_200_500_e80_d50, "N120ACA-EA1"),
+	EDP_PANEL_ENTRY('C', 'M', 'N', 0x142b, &delay_200_500_e80_d50, "N140HCA-EAC"),
+	EDP_PANEL_ENTRY('C', 'M', 'N', 0x144f, &delay_200_500_e80_d50, "N140HGA-EA1"),
+	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1468, &delay_200_500_e80, "N140HGA-EA1"),
+	EDP_PANEL_ENTRY('C', 'M', 'N', 0x14e5, &delay_200_500_e80_d50, "N140HGA-EA1"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x14d4, &delay_200_500_e80_d50, "N140HCA-EAC"),
+	EDP_PANEL_ENTRY('C', 'M', 'N', 0x14d6, &delay_200_500_e80_d50, "N140BGA-EA4"),
+
+	EDP_PANEL_ENTRY('H', 'K', 'C', 0x2d5c, &delay_200_500_e200, "MB116AN01-2"),
 
+	EDP_PANEL_ENTRY('I', 'V', 'O', 0x048e, &delay_200_500_e200_d10, "M116NWR6 R5"),
 	EDP_PANEL_ENTRY('I', 'V', 'O', 0x057d, &delay_200_500_e200, "R140NWF5 RH"),
 	EDP_PANEL_ENTRY('I', 'V', 'O', 0x854a, &delay_200_500_p2e100, "M133NW4J"),
 	EDP_PANEL_ENTRY('I', 'V', 'O', 0x854b, &delay_200_500_p2e100, "R133NW4K-R0"),
+	EDP_PANEL_ENTRY('I', 'V', 'O', 0x8c4d, &delay_200_150_e200, "R140NWFM R1"),
 
 	EDP_PANEL_ENTRY('K', 'D', 'B', 0x0624, &kingdisplay_kd116n21_30nv_a010.delay, "116N21-30NV-A010"),
+	EDP_PANEL_ENTRY('K', 'D', 'C', 0x0809, &delay_200_500_e50, "KD116N2930A15"),
 	EDP_PANEL_ENTRY('K', 'D', 'B', 0x1120, &delay_200_500_e80_d50, "116N29-30NK-C007"),
 
 	EDP_PANEL_ENTRY('S', 'H', 'P', 0x1511, &delay_200_500_e50, "LQ140M1JW48"),
diff --git a/drivers/gpu/drm/panel/panel-elida-kd35t133.c b/drivers/gpu/drm/panel/panel-elida-kd35t133.c
index e7be15b68102..00791ea81e90 100644
--- a/drivers/gpu/drm/panel/panel-elida-kd35t133.c
+++ b/drivers/gpu/drm/panel/panel-elida-kd35t133.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * Elida kd35t133 5.5" MIPI-DSI panel driver
+ * Elida kd35t133 3.5" MIPI-DSI panel driver
  * Copyright (C) 2020 Theobroma Systems Design und Consulting GmbH
  *
  * based on
@@ -43,7 +43,6 @@ struct kd35t133 {
 	struct regulator *vdd;
 	struct regulator *iovcc;
 	enum drm_panel_orientation orientation;
-	bool prepared;
 };
 
 static inline struct kd35t133 *panel_to_kd35t133(struct drm_panel *panel)
@@ -91,9 +90,6 @@ static int kd35t133_unprepare(struct drm_panel *panel)
 	struct mipi_dsi_device *dsi = to_mipi_dsi_device(ctx->dev);
 	int ret;
 
-	if (!ctx->prepared)
-		return 0;
-
 	ret = mipi_dsi_dcs_set_display_off(dsi);
 	if (ret < 0)
 		dev_err(ctx->dev, "failed to set display off: %d\n", ret);
@@ -104,11 +100,11 @@ static int kd35t133_unprepare(struct drm_panel *panel)
 		return ret;
 	}
 
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+
 	regulator_disable(ctx->iovcc);
 	regulator_disable(ctx->vdd);
 
-	ctx->prepared = false;
-
 	return 0;
 }
 
@@ -118,9 +114,6 @@ static int kd35t133_prepare(struct drm_panel *panel)
 	struct mipi_dsi_device *dsi = to_mipi_dsi_device(ctx->dev);
 	int ret;
 
-	if (ctx->prepared)
-		return 0;
-
 	dev_dbg(ctx->dev, "Resetting the panel\n");
 	ret = regulator_enable(ctx->vdd);
 	if (ret < 0) {
@@ -164,8 +157,6 @@ static int kd35t133_prepare(struct drm_panel *panel)
 
 	msleep(50);
 
-	ctx->prepared = true;
-
 	return 0;
 
 disable_iovcc:
@@ -209,11 +200,6 @@ static int kd35t133_get_modes(struct drm_panel *panel,
 	connector->display_info.width_mm = mode->width_mm;
 	connector->display_info.height_mm = mode->height_mm;
 	drm_mode_probed_add(connector, mode);
-	/*
-	 * TODO: Remove once all drm drivers call
-	 * drm_connector_set_orientation_from_panel()
-	 */
-	drm_connector_set_panel_orientation(connector, ctx->orientation);
 
 	return 1;
 }
@@ -299,27 +285,11 @@ static int kd35t133_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static void kd35t133_shutdown(struct mipi_dsi_device *dsi)
-{
-	struct kd35t133 *ctx = mipi_dsi_get_drvdata(dsi);
-	int ret;
-
-	ret = drm_panel_unprepare(&ctx->panel);
-	if (ret < 0)
-		dev_err(&dsi->dev, "Failed to unprepare panel: %d\n", ret);
-
-	ret = drm_panel_disable(&ctx->panel);
-	if (ret < 0)
-		dev_err(&dsi->dev, "Failed to disable panel: %d\n", ret);
-}
-
 static void kd35t133_remove(struct mipi_dsi_device *dsi)
 {
 	struct kd35t133 *ctx = mipi_dsi_get_drvdata(dsi);
 	int ret;
 
-	kd35t133_shutdown(dsi);
-
 	ret = mipi_dsi_detach(dsi);
 	if (ret < 0)
 		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
@@ -340,7 +310,6 @@ static struct mipi_dsi_driver kd35t133_driver = {
 	},
 	.probe	= kd35t133_probe,
 	.remove = kd35t133_remove,
-	.shutdown = kd35t133_shutdown,
 };
 module_mipi_dsi_driver(kd35t133_driver);
 
diff --git a/drivers/gpu/drm/panel/panel-newvision-nv3051d.c b/drivers/gpu/drm/panel/panel-newvision-nv3051d.c
index 79de6c886292..94d89ffd596b 100644
--- a/drivers/gpu/drm/panel/panel-newvision-nv3051d.c
+++ b/drivers/gpu/drm/panel/panel-newvision-nv3051d.c
@@ -28,6 +28,7 @@ struct nv3051d_panel_info {
 	unsigned int num_modes;
 	u16 width_mm, height_mm;
 	u32 bus_flags;
+	u32 mode_flags;
 };
 
 struct panel_nv3051d {
@@ -261,6 +262,8 @@ static int panel_nv3051d_unprepare(struct drm_panel *panel)
 
 	usleep_range(10000, 15000);
 
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+
 	regulator_disable(ctx->vdd);
 
 	return 0;
@@ -385,15 +388,7 @@ static int panel_nv3051d_probe(struct mipi_dsi_device *dsi)
 
 	dsi->lanes = 4;
 	dsi->format = MIPI_DSI_FMT_RGB888;
-	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST |
-			  MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_NO_EOT_PACKET;
-
-	/*
-	 * The panel in the RG351V is identical to the 353P, except it
-	 * requires MIPI_DSI_CLOCK_NON_CONTINUOUS to operate correctly.
-	 */
-	if (of_device_is_compatible(dev->of_node, "anbernic,rg351v-panel"))
-		dsi->mode_flags |= MIPI_DSI_CLOCK_NON_CONTINUOUS;
+	dsi->mode_flags = ctx->panel_info->mode_flags;
 
 	drm_panel_init(&ctx->panel, &dsi->dev, &panel_nv3051d_funcs,
 		       DRM_MODE_CONNECTOR_DSI);
@@ -481,16 +476,56 @@ static const struct drm_display_mode nv3051d_rgxx3_modes[] = {
 	},
 };
 
-static const struct nv3051d_panel_info nv3051d_rgxx3_info = {
+static const struct drm_display_mode nv3051d_rk2023_modes[] = {
+	{
+		.hdisplay       = 640,
+		.hsync_start    = 640 + 40,
+		.hsync_end      = 640 + 40 + 2,
+		.htotal         = 640 + 40 + 2 + 80,
+		.vdisplay       = 480,
+		.vsync_start    = 480 + 18,
+		.vsync_end      = 480 + 18 + 2,
+		.vtotal         = 480 + 18 + 2 + 4,
+		.clock          = 24150,
+		.flags          = DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC,
+	},
+};
+
+static const struct nv3051d_panel_info nv3051d_rg351v_info = {
 	.display_modes = nv3051d_rgxx3_modes,
 	.num_modes = ARRAY_SIZE(nv3051d_rgxx3_modes),
 	.width_mm = 70,
 	.height_mm = 57,
 	.bus_flags = DRM_BUS_FLAG_DE_LOW | DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE,
+	.mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST |
+		      MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_NO_EOT_PACKET |
+		      MIPI_DSI_CLOCK_NON_CONTINUOUS,
+};
+
+static const struct nv3051d_panel_info nv3051d_rg353p_info = {
+	.display_modes = nv3051d_rgxx3_modes,
+	.num_modes = ARRAY_SIZE(nv3051d_rgxx3_modes),
+	.width_mm = 70,
+	.height_mm = 57,
+	.bus_flags = DRM_BUS_FLAG_DE_LOW | DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE,
+	.mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST |
+		      MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_NO_EOT_PACKET,
+};
+
+static const struct nv3051d_panel_info nv3051d_rk2023_info = {
+	.display_modes = nv3051d_rk2023_modes,
+	.num_modes = ARRAY_SIZE(nv3051d_rk2023_modes),
+	.width_mm = 70,
+	.height_mm = 57,
+	.bus_flags = DRM_BUS_FLAG_DE_LOW | DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE,
+	.mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST |
+		      MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_NO_EOT_PACKET,
 };
 
 static const struct of_device_id newvision_nv3051d_of_match[] = {
-	{ .compatible = "newvision,nv3051d", .data = &nv3051d_rgxx3_info },
+	{ .compatible = "anbernic,rg351v-panel", .data = &nv3051d_rg351v_info },
+	{ .compatible = "anbernic,rg353p-panel", .data = &nv3051d_rg353p_info },
+	{ .compatible = "powkiddy,rk2023-panel", .data = &nv3051d_rk2023_info },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, newvision_nv3051d_of_match);
diff --git a/drivers/gpu/drm/panel/panel-novatek-nt35510.c b/drivers/gpu/drm/panel/panel-novatek-nt35510.c
index d6dceb858008..83a9cf53d269 100644
--- a/drivers/gpu/drm/panel/panel-novatek-nt35510.c
+++ b/drivers/gpu/drm/panel/panel-novatek-nt35510.c
@@ -1023,7 +1023,7 @@ static const struct nt35510_config nt35510_hydis_hva40wv1 = {
 		.hdisplay = 480,
 		.hsync_start = 480 + 2, /* HFP = 2 */
 		.hsync_end = 480 + 2 + 0, /* HSync = 0 */
-		.htotal = 480 + 2 + 0 + 5, /* HFP = 5 */
+		.htotal = 480 + 2 + 0 + 5, /* HBP = 5 */
 		.vdisplay = 800,
 		.vsync_start = 800 + 2, /* VFP = 2 */
 		.vsync_end = 800 + 2 + 0, /* VSync = 0 */
diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c
index 6cd32b909087..9367a4572dcf 100644
--- a/drivers/gpu/drm/panel/panel-simple.c
+++ b/drivers/gpu/drm/panel/panel-simple.c
@@ -2379,13 +2379,13 @@ static const struct panel_desc innolux_g070y2_t02 = {
 static const struct display_timing innolux_g101ice_l01_timing = {
 	.pixelclock = { 60400000, 71100000, 74700000 },
 	.hactive = { 1280, 1280, 1280 },
-	.hfront_porch = { 41, 80, 100 },
-	.hback_porch = { 40, 79, 99 },
-	.hsync_len = { 1, 1, 1 },
+	.hfront_porch = { 30, 60, 70 },
+	.hback_porch = { 30, 60, 70 },
+	.hsync_len = { 22, 40, 60 },
 	.vactive = { 800, 800, 800 },
-	.vfront_porch = { 5, 11, 14 },
-	.vback_porch = { 4, 11, 14 },
-	.vsync_len = { 1, 1, 1 },
+	.vfront_porch = { 3, 8, 14 },
+	.vback_porch = { 3, 8, 14 },
+	.vsync_len = { 4, 7, 12 },
 	.flags = DISPLAY_FLAGS_DE_HIGH,
 };
 
@@ -2402,6 +2402,7 @@ static const struct panel_desc innolux_g101ice_l01 = {
 		.disable = 200,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH,
 	.connector_type = DRM_MODE_CONNECTOR_LVDS,
 };
 
diff --git a/drivers/gpu/drm/panfrost/panfrost_device.c b/drivers/gpu/drm/panfrost/panfrost_device.c
index 28f7046e1b1a..c90ad5ee34e7 100644
--- a/drivers/gpu/drm/panfrost/panfrost_device.c
+++ b/drivers/gpu/drm/panfrost/panfrost_device.c
@@ -403,7 +403,7 @@ void panfrost_device_reset(struct panfrost_device *pfdev)
 	panfrost_job_enable_interrupts(pfdev);
 }
 
-static int panfrost_device_resume(struct device *dev)
+static int panfrost_device_runtime_resume(struct device *dev)
 {
 	struct panfrost_device *pfdev = dev_get_drvdata(dev);
 
@@ -413,7 +413,7 @@ static int panfrost_device_resume(struct device *dev)
 	return 0;
 }
 
-static int panfrost_device_suspend(struct device *dev)
+static int panfrost_device_runtime_suspend(struct device *dev)
 {
 	struct panfrost_device *pfdev = dev_get_drvdata(dev);
 
@@ -426,5 +426,75 @@ static int panfrost_device_suspend(struct device *dev)
 	return 0;
 }
 
-EXPORT_GPL_RUNTIME_DEV_PM_OPS(panfrost_pm_ops, panfrost_device_suspend,
-			      panfrost_device_resume, NULL);
+static int panfrost_device_resume(struct device *dev)
+{
+	struct panfrost_device *pfdev = dev_get_drvdata(dev);
+	int ret;
+
+	if (pfdev->comp->pm_features & BIT(GPU_PM_VREG_OFF)) {
+		unsigned long freq = pfdev->pfdevfreq.fast_rate;
+		struct dev_pm_opp *opp;
+
+		opp = dev_pm_opp_find_freq_ceil(dev, &freq);
+		if (IS_ERR(opp))
+			return PTR_ERR(opp);
+		dev_pm_opp_set_opp(dev, opp);
+		dev_pm_opp_put(opp);
+	}
+
+	if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS)) {
+		ret = clk_enable(pfdev->clock);
+		if (ret)
+			goto err_clk;
+
+		if (pfdev->bus_clock) {
+			ret = clk_enable(pfdev->bus_clock);
+			if (ret)
+				goto err_bus_clk;
+		}
+	}
+
+	ret = pm_runtime_force_resume(dev);
+	if (ret)
+		goto err_resume;
+
+	return 0;
+
+err_resume:
+	if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS) && pfdev->bus_clock)
+		clk_disable(pfdev->bus_clock);
+err_bus_clk:
+	if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS))
+		clk_disable(pfdev->clock);
+err_clk:
+	if (pfdev->comp->pm_features & BIT(GPU_PM_VREG_OFF))
+		dev_pm_opp_set_opp(dev, NULL);
+	return ret;
+}
+
+static int panfrost_device_suspend(struct device *dev)
+{
+	struct panfrost_device *pfdev = dev_get_drvdata(dev);
+	int ret;
+
+	ret = pm_runtime_force_suspend(dev);
+	if (ret)
+		return ret;
+
+	if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS)) {
+		if (pfdev->bus_clock)
+			clk_disable(pfdev->bus_clock);
+
+		clk_disable(pfdev->clock);
+	}
+
+	if (pfdev->comp->pm_features & BIT(GPU_PM_VREG_OFF))
+		dev_pm_opp_set_opp(dev, NULL);
+
+	return 0;
+}
+
+EXPORT_GPL_DEV_PM_OPS(panfrost_pm_ops) = {
+	RUNTIME_PM_OPS(panfrost_device_runtime_suspend, panfrost_device_runtime_resume, NULL)
+	SYSTEM_SLEEP_PM_OPS(panfrost_device_suspend, panfrost_device_resume)
+};
diff --git a/drivers/gpu/drm/panfrost/panfrost_device.h b/drivers/gpu/drm/panfrost/panfrost_device.h
index 1ef38f60d5dc..0fc558db6bfd 100644
--- a/drivers/gpu/drm/panfrost/panfrost_device.h
+++ b/drivers/gpu/drm/panfrost/panfrost_device.h
@@ -25,6 +25,16 @@ struct panfrost_perfcnt;
 #define NUM_JOB_SLOTS 3
 #define MAX_PM_DOMAINS 5
 
+/**
+ * enum panfrost_gpu_pm - Supported kernel power management features
+ * @GPU_PM_CLK_DIS:  Allow disabling clocks during system suspend
+ * @GPU_PM_VREG_OFF: Allow turning off regulators during system suspend
+ */
+enum panfrost_gpu_pm {
+	GPU_PM_CLK_DIS,
+	GPU_PM_VREG_OFF,
+};
+
 struct panfrost_features {
 	u16 id;
 	u16 revision;
@@ -75,6 +85,9 @@ struct panfrost_compatible {
 
 	/* Vendor implementation quirks callback */
 	void (*vendor_quirk)(struct panfrost_device *pfdev);
+
+	/* Allowed PM features */
+	u8 pm_features;
 };
 
 struct panfrost_device {
diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c
index 7cabf4e3d1f2..a926d71e8131 100644
--- a/drivers/gpu/drm/panfrost/panfrost_drv.c
+++ b/drivers/gpu/drm/panfrost/panfrost_drv.c
@@ -274,7 +274,7 @@ static int panfrost_ioctl_submit(struct drm_device *dev, void *data,
 
 	ret = drm_sched_job_init(&job->base,
 				 &file_priv->sched_entity[slot],
-				 NULL);
+				 1, NULL);
 	if (ret)
 		goto out_put_job;
 
@@ -734,6 +734,7 @@ static const struct panfrost_compatible mediatek_mt8183_b_data = {
 	.supply_names = mediatek_mt8183_b_supplies,
 	.num_pm_domains = ARRAY_SIZE(mediatek_mt8183_pm_domains),
 	.pm_domain_names = mediatek_mt8183_pm_domains,
+	.pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF),
 };
 
 static const char * const mediatek_mt8186_pm_domains[] = { "core0", "core1" };
@@ -742,6 +743,7 @@ static const struct panfrost_compatible mediatek_mt8186_data = {
 	.supply_names = mediatek_mt8183_b_supplies,
 	.num_pm_domains = ARRAY_SIZE(mediatek_mt8186_pm_domains),
 	.pm_domain_names = mediatek_mt8186_pm_domains,
+	.pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF),
 };
 
 static const char * const mediatek_mt8192_supplies[] = { "mali", NULL };
@@ -752,6 +754,7 @@ static const struct panfrost_compatible mediatek_mt8192_data = {
 	.supply_names = mediatek_mt8192_supplies,
 	.num_pm_domains = ARRAY_SIZE(mediatek_mt8192_pm_domains),
 	.pm_domain_names = mediatek_mt8192_pm_domains,
+	.pm_features = BIT(GPU_PM_CLK_DIS) | BIT(GPU_PM_VREG_OFF),
 };
 
 static const struct of_device_id dt_match[] = {
diff --git a/drivers/gpu/drm/panfrost/panfrost_dump.c b/drivers/gpu/drm/panfrost/panfrost_dump.c
index e7942ac449c6..47751302f1bc 100644
--- a/drivers/gpu/drm/panfrost/panfrost_dump.c
+++ b/drivers/gpu/drm/panfrost/panfrost_dump.c
@@ -220,16 +220,8 @@ void panfrost_core_dump(struct panfrost_job *job)
 
 		iter.hdr->bomap.data[0] = bomap - bomap_start;
 
-		for_each_sgtable_page(bo->base.sgt, &page_iter, 0) {
-			struct page *page = sg_page_iter_page(&page_iter);
-
-			if (!IS_ERR(page)) {
-				*bomap++ = page_to_phys(page);
-			} else {
-				dev_err(pfdev->dev, "Panfrost Dump: wrong page\n");
-				*bomap++ = 0;
-			}
-		}
+		for_each_sgtable_page(bo->base.sgt, &page_iter, 0)
+			*bomap++ = page_to_phys(sg_page_iter_page(&page_iter));
 
 		iter.hdr->bomap.iova = mapping->mmnode.start << PAGE_SHIFT;
 
diff --git a/drivers/gpu/drm/panfrost/panfrost_gpu.c b/drivers/gpu/drm/panfrost/panfrost_gpu.c
index f0be7e19b13e..09f5e1563ebd 100644
--- a/drivers/gpu/drm/panfrost/panfrost_gpu.c
+++ b/drivers/gpu/drm/panfrost/panfrost_gpu.c
@@ -60,14 +60,21 @@ int panfrost_gpu_soft_reset(struct panfrost_device *pfdev)
 
 	gpu_write(pfdev, GPU_INT_MASK, 0);
 	gpu_write(pfdev, GPU_INT_CLEAR, GPU_IRQ_RESET_COMPLETED);
-	gpu_write(pfdev, GPU_CMD, GPU_CMD_SOFT_RESET);
 
+	gpu_write(pfdev, GPU_CMD, GPU_CMD_SOFT_RESET);
 	ret = readl_relaxed_poll_timeout(pfdev->iomem + GPU_INT_RAWSTAT,
-		val, val & GPU_IRQ_RESET_COMPLETED, 100, 10000);
+		val, val & GPU_IRQ_RESET_COMPLETED, 10, 10000);
 
 	if (ret) {
-		dev_err(pfdev->dev, "gpu soft reset timed out\n");
-		return ret;
+		dev_err(pfdev->dev, "gpu soft reset timed out, attempting hard reset\n");
+
+		gpu_write(pfdev, GPU_CMD, GPU_CMD_HARD_RESET);
+		ret = readl_relaxed_poll_timeout(pfdev->iomem + GPU_INT_RAWSTAT, val,
+						 val & GPU_IRQ_RESET_COMPLETED, 100, 10000);
+		if (ret) {
+			dev_err(pfdev->dev, "gpu hard reset timed out\n");
+			return ret;
+		}
 	}
 
 	gpu_write(pfdev, GPU_INT_CLEAR, GPU_IRQ_MASK_ALL);
@@ -362,32 +369,42 @@ unsigned long long panfrost_cycle_counter_read(struct panfrost_device *pfdev)
 	return ((u64)hi << 32) | lo;
 }
 
+static u64 panfrost_get_core_mask(struct panfrost_device *pfdev)
+{
+	u64 core_mask;
+
+	if (pfdev->features.l2_present == 1)
+		return U64_MAX;
+
+	/*
+	 * Only support one core group now.
+	 * ~(l2_present - 1) unsets all bits in l2_present except
+	 * the bottom bit. (l2_present - 2) has all the bits in
+	 * the first core group set. AND them together to generate
+	 * a mask of cores in the first core group.
+	 */
+	core_mask = ~(pfdev->features.l2_present - 1) &
+		     (pfdev->features.l2_present - 2);
+	dev_info_once(pfdev->dev, "using only 1st core group (%lu cores from %lu)\n",
+		      hweight64(core_mask),
+		      hweight64(pfdev->features.shader_present));
+
+	return core_mask;
+}
+
 void panfrost_gpu_power_on(struct panfrost_device *pfdev)
 {
 	int ret;
 	u32 val;
-	u64 core_mask = U64_MAX;
+	u64 core_mask;
 
 	panfrost_gpu_init_quirks(pfdev);
+	core_mask = panfrost_get_core_mask(pfdev);
 
-	if (pfdev->features.l2_present != 1) {
-		/*
-		 * Only support one core group now.
-		 * ~(l2_present - 1) unsets all bits in l2_present except
-		 * the bottom bit. (l2_present - 2) has all the bits in
-		 * the first core group set. AND them together to generate
-		 * a mask of cores in the first core group.
-		 */
-		core_mask = ~(pfdev->features.l2_present - 1) &
-			     (pfdev->features.l2_present - 2);
-		dev_info_once(pfdev->dev, "using only 1st core group (%lu cores from %lu)\n",
-			      hweight64(core_mask),
-			      hweight64(pfdev->features.shader_present));
-	}
 	gpu_write(pfdev, L2_PWRON_LO, pfdev->features.l2_present & core_mask);
 	ret = readl_relaxed_poll_timeout(pfdev->iomem + L2_READY_LO,
 		val, val == (pfdev->features.l2_present & core_mask),
-		100, 20000);
+		10, 20000);
 	if (ret)
 		dev_err(pfdev->dev, "error powering up gpu L2");
 
@@ -395,22 +412,40 @@ void panfrost_gpu_power_on(struct panfrost_device *pfdev)
 		  pfdev->features.shader_present & core_mask);
 	ret = readl_relaxed_poll_timeout(pfdev->iomem + SHADER_READY_LO,
 		val, val == (pfdev->features.shader_present & core_mask),
-		100, 20000);
+		10, 20000);
 	if (ret)
 		dev_err(pfdev->dev, "error powering up gpu shader");
 
 	gpu_write(pfdev, TILER_PWRON_LO, pfdev->features.tiler_present);
 	ret = readl_relaxed_poll_timeout(pfdev->iomem + TILER_READY_LO,
-		val, val == pfdev->features.tiler_present, 100, 1000);
+		val, val == pfdev->features.tiler_present, 10, 1000);
 	if (ret)
 		dev_err(pfdev->dev, "error powering up gpu tiler");
 }
 
 void panfrost_gpu_power_off(struct panfrost_device *pfdev)
 {
-	gpu_write(pfdev, TILER_PWROFF_LO, 0);
-	gpu_write(pfdev, SHADER_PWROFF_LO, 0);
-	gpu_write(pfdev, L2_PWROFF_LO, 0);
+	u64 core_mask = panfrost_get_core_mask(pfdev);
+	int ret;
+	u32 val;
+
+	gpu_write(pfdev, SHADER_PWROFF_LO, pfdev->features.shader_present & core_mask);
+	ret = readl_relaxed_poll_timeout(pfdev->iomem + SHADER_PWRTRANS_LO,
+					 val, !val, 1, 1000);
+	if (ret)
+		dev_err(pfdev->dev, "shader power transition timeout");
+
+	gpu_write(pfdev, TILER_PWROFF_LO, pfdev->features.tiler_present);
+	ret = readl_relaxed_poll_timeout(pfdev->iomem + TILER_PWRTRANS_LO,
+					 val, !val, 1, 1000);
+	if (ret)
+		dev_err(pfdev->dev, "tiler power transition timeout");
+
+	gpu_write(pfdev, L2_PWROFF_LO, pfdev->features.l2_present & core_mask);
+	ret = readl_poll_timeout(pfdev->iomem + L2_PWRTRANS_LO,
+				 val, !val, 0, 1000);
+	if (ret)
+		dev_err(pfdev->dev, "l2 power transition timeout");
 }
 
 int panfrost_gpu_init(struct panfrost_device *pfdev)
diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panfrost/panfrost_job.c
index ecd2e035147f..f9446e197428 100644
--- a/drivers/gpu/drm/panfrost/panfrost_job.c
+++ b/drivers/gpu/drm/panfrost/panfrost_job.c
@@ -852,7 +852,7 @@ int panfrost_job_init(struct panfrost_device *pfdev)
 		js->queue[j].fence_context = dma_fence_context_alloc(1);
 
 		ret = drm_sched_init(&js->queue[j].sched,
-				     &panfrost_sched_ops,
+				     &panfrost_sched_ops, NULL,
 				     DRM_SCHED_PRIORITY_COUNT,
 				     nentries, 0,
 				     msecs_to_jiffies(JOB_TIMEOUT_MS),
@@ -963,7 +963,7 @@ int panfrost_job_is_idle(struct panfrost_device *pfdev)
 
 	for (i = 0; i < NUM_JOB_SLOTS; i++) {
 		/* If there are any jobs in the HW queue, we're not idle */
-		if (atomic_read(&js->queue[i].sched.hw_rq_count))
+		if (atomic_read(&js->queue[i].sched.credit_count))
 			return false;
 	}
 
diff --git a/drivers/gpu/drm/panfrost/panfrost_regs.h b/drivers/gpu/drm/panfrost/panfrost_regs.h
index 55ec807550b3..c25743b05c55 100644
--- a/drivers/gpu/drm/panfrost/panfrost_regs.h
+++ b/drivers/gpu/drm/panfrost/panfrost_regs.h
@@ -44,6 +44,7 @@
 	 GPU_IRQ_MULTIPLE_FAULT)
 #define GPU_CMD				0x30
 #define   GPU_CMD_SOFT_RESET		0x01
+#define   GPU_CMD_HARD_RESET		0x02
 #define   GPU_CMD_PERFCNT_CLEAR		0x03
 #define   GPU_CMD_PERFCNT_SAMPLE	0x04
 #define   GPU_CMD_CYCLE_COUNT_START	0x05
diff --git a/drivers/gpu/drm/radeon/clearstate_evergreen.h b/drivers/gpu/drm/radeon/clearstate_evergreen.h
index 63a1ffbb3ced..3b645558f133 100644
--- a/drivers/gpu/drm/radeon/clearstate_evergreen.h
+++ b/drivers/gpu/drm/radeon/clearstate_evergreen.h
@@ -1049,7 +1049,7 @@ static const struct cs_extent_def SECT_CONTEXT_defs[] =
     {SECT_CONTEXT_def_5, 0x0000a29e, 5 },
     {SECT_CONTEXT_def_6, 0x0000a2a5, 56 },
     {SECT_CONTEXT_def_7, 0x0000a2de, 290 },
-    { 0, 0, 0 }
+    { NULL, 0, 0 }
 };
 static const u32 SECT_CLEAR_def_1[] =
 {
@@ -1060,7 +1060,7 @@ static const u32 SECT_CLEAR_def_1[] =
 static const struct cs_extent_def SECT_CLEAR_defs[] =
 {
     {SECT_CLEAR_def_1, 0x0000ffc0, 3 },
-    { 0, 0, 0 }
+    { NULL, 0, 0 }
 };
 static const u32 SECT_CTRLCONST_def_1[] =
 {
@@ -1070,11 +1070,11 @@ static const u32 SECT_CTRLCONST_def_1[] =
 static const struct cs_extent_def SECT_CTRLCONST_defs[] =
 {
     {SECT_CTRLCONST_def_1, 0x0000f3fc, 2 },
-    { 0, 0, 0 }
+    { NULL, 0, 0 }
 };
 static const struct cs_section_def evergreen_cs_data[] = {
     { SECT_CONTEXT_defs, SECT_CONTEXT },
     { SECT_CLEAR_defs, SECT_CLEAR },
     { SECT_CTRLCONST_defs, SECT_CTRLCONST },
-    { 0, SECT_NONE }
+    { NULL, SECT_NONE }
 };
diff --git a/drivers/gpu/drm/radeon/radeon_audio.c b/drivers/gpu/drm/radeon/radeon_audio.c
index d6ccaf24ee0c..279bf130a18c 100644
--- a/drivers/gpu/drm/radeon/radeon_audio.c
+++ b/drivers/gpu/drm/radeon/radeon_audio.c
@@ -26,6 +26,7 @@
 #include <linux/component.h>
 
 #include <drm/drm_crtc.h>
+#include <drm/drm_eld.h>
 #include "dce6_afmt.h"
 #include "evergreen_hdmi.h"
 #include "radeon.h"
diff --git a/drivers/gpu/drm/rockchip/rk3066_hdmi.c b/drivers/gpu/drm/rockchip/rk3066_hdmi.c
index fa6e592e0276..0e7aae341960 100644
--- a/drivers/gpu/drm/rockchip/rk3066_hdmi.c
+++ b/drivers/gpu/drm/rockchip/rk3066_hdmi.c
@@ -55,7 +55,6 @@ struct rk3066_hdmi {
 	unsigned int tmdsclk;
 
 	struct hdmi_data_info hdmi_data;
-	struct drm_display_mode previous_mode;
 };
 
 static struct rk3066_hdmi *encoder_to_rk3066_hdmi(struct drm_encoder *encoder)
@@ -387,21 +386,21 @@ static int rk3066_hdmi_setup(struct rk3066_hdmi *hdmi,
 	return 0;
 }
 
-static void
-rk3066_hdmi_encoder_mode_set(struct drm_encoder *encoder,
-			     struct drm_display_mode *mode,
-			     struct drm_display_mode *adj_mode)
+static void rk3066_hdmi_encoder_enable(struct drm_encoder *encoder,
+				       struct drm_atomic_state *state)
 {
 	struct rk3066_hdmi *hdmi = encoder_to_rk3066_hdmi(encoder);
+	struct drm_connector_state *conn_state;
+	struct drm_crtc_state *crtc_state;
+	int mux, val;
 
-	/* Store the display mode for plugin/DPMS poweron events. */
-	drm_mode_copy(&hdmi->previous_mode, adj_mode);
-}
+	conn_state = drm_atomic_get_new_connector_state(state, &hdmi->connector);
+	if (WARN_ON(!conn_state))
+		return;
 
-static void rk3066_hdmi_encoder_enable(struct drm_encoder *encoder)
-{
-	struct rk3066_hdmi *hdmi = encoder_to_rk3066_hdmi(encoder);
-	int mux, val;
+	crtc_state = drm_atomic_get_new_crtc_state(state, conn_state->crtc);
+	if (WARN_ON(!crtc_state))
+		return;
 
 	mux = drm_of_encoder_active_endpoint_id(hdmi->dev->of_node, encoder);
 	if (mux)
@@ -414,10 +413,11 @@ static void rk3066_hdmi_encoder_enable(struct drm_encoder *encoder)
 	DRM_DEV_DEBUG(hdmi->dev, "hdmi encoder enable select: vop%s\n",
 		      (mux) ? "1" : "0");
 
-	rk3066_hdmi_setup(hdmi, &hdmi->previous_mode);
+	rk3066_hdmi_setup(hdmi, &crtc_state->adjusted_mode);
 }
 
-static void rk3066_hdmi_encoder_disable(struct drm_encoder *encoder)
+static void rk3066_hdmi_encoder_disable(struct drm_encoder *encoder,
+					struct drm_atomic_state *state)
 {
 	struct rk3066_hdmi *hdmi = encoder_to_rk3066_hdmi(encoder);
 
@@ -434,14 +434,6 @@ static void rk3066_hdmi_encoder_disable(struct drm_encoder *encoder)
 	rk3066_hdmi_set_power_mode(hdmi, HDMI_SYS_POWER_MODE_A);
 }
 
-static bool
-rk3066_hdmi_encoder_mode_fixup(struct drm_encoder *encoder,
-			       const struct drm_display_mode *mode,
-			       struct drm_display_mode *adj_mode)
-{
-	return true;
-}
-
 static int
 rk3066_hdmi_encoder_atomic_check(struct drm_encoder *encoder,
 				 struct drm_crtc_state *crtc_state,
@@ -457,11 +449,9 @@ rk3066_hdmi_encoder_atomic_check(struct drm_encoder *encoder,
 
 static const
 struct drm_encoder_helper_funcs rk3066_hdmi_encoder_helper_funcs = {
-	.enable       = rk3066_hdmi_encoder_enable,
-	.disable      = rk3066_hdmi_encoder_disable,
-	.mode_fixup   = rk3066_hdmi_encoder_mode_fixup,
-	.mode_set     = rk3066_hdmi_encoder_mode_set,
-	.atomic_check = rk3066_hdmi_encoder_atomic_check,
+	.atomic_check   = rk3066_hdmi_encoder_atomic_check,
+	.atomic_enable  = rk3066_hdmi_encoder_enable,
+	.atomic_disable = rk3066_hdmi_encoder_disable,
 };
 
 static enum drm_connector_status
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
index 066299894d04..a13473b2d54c 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
@@ -247,14 +247,22 @@ static inline void vop_cfg_done(struct vop *vop)
 	VOP_REG_SET(vop, common, cfg_done, 1);
 }
 
-static bool has_rb_swapped(uint32_t format)
+static bool has_rb_swapped(uint32_t version, uint32_t format)
 {
 	switch (format) {
 	case DRM_FORMAT_XBGR8888:
 	case DRM_FORMAT_ABGR8888:
-	case DRM_FORMAT_BGR888:
 	case DRM_FORMAT_BGR565:
 		return true;
+	/*
+	 * full framework (IP version 3.x) only need rb swapped for RGB888 and
+	 * little framework (IP version 2.x) only need rb swapped for BGR888,
+	 * check for 3.x to also only rb swap BGR888 for unknown vop version
+	 */
+	case DRM_FORMAT_RGB888:
+		return VOP_MAJOR(version) == 3;
+	case DRM_FORMAT_BGR888:
+		return VOP_MAJOR(version) != 3;
 	default:
 		return false;
 	}
@@ -1030,7 +1038,7 @@ static void vop_plane_atomic_update(struct drm_plane *plane,
 	VOP_WIN_SET(vop, win, dsp_info, dsp_info);
 	VOP_WIN_SET(vop, win, dsp_st, dsp_st);
 
-	rb_swap = has_rb_swapped(fb->format->format);
+	rb_swap = has_rb_swapped(vop->data->version, fb->format->format);
 	VOP_WIN_SET(vop, win, rb_swap, rb_swap);
 
 	/*
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
index 6862fb146ace..312da5783362 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
@@ -325,11 +325,14 @@ static enum vop2_data_format vop2_convert_format(u32 format)
 	case DRM_FORMAT_NV16:
 	case DRM_FORMAT_NV61:
 		return VOP2_FMT_YUV422SP;
+	case DRM_FORMAT_NV20:
 	case DRM_FORMAT_Y210:
 		return VOP2_FMT_YUV422SP_10;
 	case DRM_FORMAT_NV24:
 	case DRM_FORMAT_NV42:
 		return VOP2_FMT_YUV444SP;
+	case DRM_FORMAT_NV30:
+		return VOP2_FMT_YUV444SP_10;
 	case DRM_FORMAT_YUYV:
 	case DRM_FORMAT_YVYU:
 		return VOP2_FMT_VYUY422;
@@ -414,6 +417,8 @@ static bool vop2_win_uv_swap(u32 format)
 	case DRM_FORMAT_NV16:
 	case DRM_FORMAT_NV24:
 	case DRM_FORMAT_NV15:
+	case DRM_FORMAT_NV20:
+	case DRM_FORMAT_NV30:
 	case DRM_FORMAT_YUYV:
 	case DRM_FORMAT_UYVY:
 		return true;
diff --git a/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c b/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c
index 22288ad7f326..2c45d81983a5 100644
--- a/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c
+++ b/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c
@@ -48,8 +48,10 @@ static const uint32_t formats_rk356x_esmart[] = {
 	DRM_FORMAT_NV15, /* yuv420_10bit linear mode, 2 plane, no padding */
 	DRM_FORMAT_NV16, /* yuv422_8bit linear mode, 2 plane */
 	DRM_FORMAT_NV61, /* yuv422_8bit linear mode, 2 plane */
+	DRM_FORMAT_NV20, /* yuv422_10bit linear mode, 2 plane, no padding */
 	DRM_FORMAT_NV24, /* yuv444_8bit linear mode, 2 plane */
 	DRM_FORMAT_NV42, /* yuv444_8bit linear mode, 2 plane */
+	DRM_FORMAT_NV30, /* yuv444_10bit linear mode, 2 plane, no padding */
 	DRM_FORMAT_YVYU, /* yuv422_8bit[YVYU] linear mode */
 	DRM_FORMAT_VYUY, /* yuv422_8bit[VYUY] linear mode */
 };
diff --git a/drivers/gpu/drm/scheduler/gpu_scheduler_trace.h b/drivers/gpu/drm/scheduler/gpu_scheduler_trace.h
index 3143ecaaff86..f8ed093b7356 100644
--- a/drivers/gpu/drm/scheduler/gpu_scheduler_trace.h
+++ b/drivers/gpu/drm/scheduler/gpu_scheduler_trace.h
@@ -51,7 +51,7 @@ DECLARE_EVENT_CLASS(drm_sched_job,
 			   __assign_str(name, sched_job->sched->name);
 			   __entry->job_count = spsc_queue_count(&entity->job_queue);
 			   __entry->hw_job_count = atomic_read(
-				   &sched_job->sched->hw_rq_count);
+				   &sched_job->sched->credit_count);
 			   ),
 	    TP_printk("entity=%p, id=%llu, fence=%p, ring=%s, job count:%u, hw job count:%d",
 		      __entry->entity, __entry->id,
diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c
index 409e4256f6e7..4d42b1e4daa6 100644
--- a/drivers/gpu/drm/scheduler/sched_entity.c
+++ b/drivers/gpu/drm/scheduler/sched_entity.c
@@ -370,7 +370,7 @@ static void drm_sched_entity_wakeup(struct dma_fence *f,
 		container_of(cb, struct drm_sched_entity, cb);
 
 	drm_sched_entity_clear_dep(f, cb);
-	drm_sched_wakeup_if_can_queue(entity->rq->sched);
+	drm_sched_wakeup(entity->rq->sched, entity);
 }
 
 /**
@@ -602,7 +602,7 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job)
 		if (drm_sched_policy == DRM_SCHED_POLICY_FIFO)
 			drm_sched_rq_update_fifo(entity, submit_ts);
 
-		drm_sched_wakeup_if_can_queue(entity->rq->sched);
+		drm_sched_wakeup(entity->rq->sched, entity);
 	}
 }
 EXPORT_SYMBOL(drm_sched_entity_push_job);
diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c
index 99797a8c836a..044a8c4875ba 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -48,7 +48,30 @@
  * through the jobs entity pointer.
  */
 
-#include <linux/kthread.h>
+/**
+ * DOC: Flow Control
+ *
+ * The DRM GPU scheduler provides a flow control mechanism to regulate the rate
+ * in which the jobs fetched from scheduler entities are executed.
+ *
+ * In this context the &drm_gpu_scheduler keeps track of a driver specified
+ * credit limit representing the capacity of this scheduler and a credit count;
+ * every &drm_sched_job carries a driver specified number of credits.
+ *
+ * Once a job is executed (but not yet finished), the job's credits contribute
+ * to the scheduler's credit count until the job is finished. If by executing
+ * one more job the scheduler's credit count would exceed the scheduler's
+ * credit limit, the job won't be executed. Instead, the scheduler will wait
+ * until the credit count has decreased enough to not overflow its credit limit.
+ * This implies waiting for previously executed jobs.
+ *
+ * Optionally, drivers may register a callback (update_job_credits) provided by
+ * struct drm_sched_backend_ops to update the job's credits dynamically. The
+ * scheduler executes this callback every time the scheduler considers a job for
+ * execution and subsequently checks whether the job fits the scheduler's credit
+ * limit.
+ */
+
 #include <linux/wait.h>
 #include <linux/sched.h>
 #include <linux/completion.h>
@@ -76,6 +99,51 @@ int drm_sched_policy = DRM_SCHED_POLICY_FIFO;
 MODULE_PARM_DESC(sched_policy, "Specify the scheduling policy for entities on a run-queue, " __stringify(DRM_SCHED_POLICY_RR) " = Round Robin, " __stringify(DRM_SCHED_POLICY_FIFO) " = FIFO (default).");
 module_param_named(sched_policy, drm_sched_policy, int, 0444);
 
+static u32 drm_sched_available_credits(struct drm_gpu_scheduler *sched)
+{
+	u32 credits;
+
+	drm_WARN_ON(sched, check_sub_overflow(sched->credit_limit,
+					      atomic_read(&sched->credit_count),
+					      &credits));
+
+	return credits;
+}
+
+/**
+ * drm_sched_can_queue -- Can we queue more to the hardware?
+ * @sched: scheduler instance
+ * @entity: the scheduler entity
+ *
+ * Return true if we can push at least one more job from @entity, false
+ * otherwise.
+ */
+static bool drm_sched_can_queue(struct drm_gpu_scheduler *sched,
+				struct drm_sched_entity *entity)
+{
+	struct drm_sched_job *s_job;
+
+	s_job = to_drm_sched_job(spsc_queue_peek(&entity->job_queue));
+	if (!s_job)
+		return false;
+
+	if (sched->ops->update_job_credits) {
+		s_job->credits = sched->ops->update_job_credits(s_job);
+
+		drm_WARN(sched, !s_job->credits,
+			 "Jobs with zero credits bypass job-flow control.\n");
+	}
+
+	/* If a job exceeds the credit limit, truncate it to the credit limit
+	 * itself to guarantee forward progress.
+	 */
+	if (drm_WARN(sched, s_job->credits > sched->credit_limit,
+		     "Jobs may not exceed the credit limit, truncate.\n"))
+		s_job->credits = sched->credit_limit;
+
+	return drm_sched_available_credits(sched) >= s_job->credits;
+}
+
 static __always_inline bool drm_sched_entity_compare_before(struct rb_node *a,
 							    const struct rb_node *b)
 {
@@ -187,12 +255,18 @@ void drm_sched_rq_remove_entity(struct drm_sched_rq *rq,
 /**
  * drm_sched_rq_select_entity_rr - Select an entity which could provide a job to run
  *
+ * @sched: the gpu scheduler
  * @rq: scheduler run queue to check.
  *
- * Try to find a ready entity, returns NULL if none found.
+ * Try to find the next ready entity.
+ *
+ * Return an entity if one is found; return an error-pointer (!NULL) if an
+ * entity was ready, but the scheduler had insufficient credits to accommodate
+ * its job; return NULL, if no ready entity was found.
  */
 static struct drm_sched_entity *
-drm_sched_rq_select_entity_rr(struct drm_sched_rq *rq)
+drm_sched_rq_select_entity_rr(struct drm_gpu_scheduler *sched,
+			      struct drm_sched_rq *rq)
 {
 	struct drm_sched_entity *entity;
 
@@ -202,6 +276,14 @@ drm_sched_rq_select_entity_rr(struct drm_sched_rq *rq)
 	if (entity) {
 		list_for_each_entry_continue(entity, &rq->entities, list) {
 			if (drm_sched_entity_is_ready(entity)) {
+				/* If we can't queue yet, preserve the current
+				 * entity in terms of fairness.
+				 */
+				if (!drm_sched_can_queue(sched, entity)) {
+					spin_unlock(&rq->lock);
+					return ERR_PTR(-ENOSPC);
+				}
+
 				rq->current_entity = entity;
 				reinit_completion(&entity->entity_idle);
 				spin_unlock(&rq->lock);
@@ -211,8 +293,15 @@ drm_sched_rq_select_entity_rr(struct drm_sched_rq *rq)
 	}
 
 	list_for_each_entry(entity, &rq->entities, list) {
-
 		if (drm_sched_entity_is_ready(entity)) {
+			/* If we can't queue yet, preserve the current entity in
+			 * terms of fairness.
+			 */
+			if (!drm_sched_can_queue(sched, entity)) {
+				spin_unlock(&rq->lock);
+				return ERR_PTR(-ENOSPC);
+			}
+
 			rq->current_entity = entity;
 			reinit_completion(&entity->entity_idle);
 			spin_unlock(&rq->lock);
@@ -231,12 +320,18 @@ drm_sched_rq_select_entity_rr(struct drm_sched_rq *rq)
 /**
  * drm_sched_rq_select_entity_fifo - Select an entity which provides a job to run
  *
+ * @sched: the gpu scheduler
  * @rq: scheduler run queue to check.
  *
- * Find oldest waiting ready entity, returns NULL if none found.
+ * Find oldest waiting ready entity.
+ *
+ * Return an entity if one is found; return an error-pointer (!NULL) if an
+ * entity was ready, but the scheduler had insufficient credits to accommodate
+ * its job; return NULL, if no ready entity was found.
  */
 static struct drm_sched_entity *
-drm_sched_rq_select_entity_fifo(struct drm_sched_rq *rq)
+drm_sched_rq_select_entity_fifo(struct drm_gpu_scheduler *sched,
+				struct drm_sched_rq *rq)
 {
 	struct rb_node *rb;
 
@@ -246,6 +341,14 @@ drm_sched_rq_select_entity_fifo(struct drm_sched_rq *rq)
 
 		entity = rb_entry(rb, struct drm_sched_entity, rb_tree_node);
 		if (drm_sched_entity_is_ready(entity)) {
+			/* If we can't queue yet, preserve the current entity in
+			 * terms of fairness.
+			 */
+			if (!drm_sched_can_queue(sched, entity)) {
+				spin_unlock(&rq->lock);
+				return ERR_PTR(-ENOSPC);
+			}
+
 			rq->current_entity = entity;
 			reinit_completion(&entity->entity_idle);
 			break;
@@ -256,6 +359,42 @@ drm_sched_rq_select_entity_fifo(struct drm_sched_rq *rq)
 	return rb ? rb_entry(rb, struct drm_sched_entity, rb_tree_node) : NULL;
 }
 
+/**
+ * drm_sched_run_job_queue - enqueue run-job work
+ * @sched: scheduler instance
+ */
+static void drm_sched_run_job_queue(struct drm_gpu_scheduler *sched)
+{
+	if (!READ_ONCE(sched->pause_submit))
+		queue_work(sched->submit_wq, &sched->work_run_job);
+}
+
+/**
+ * __drm_sched_run_free_queue - enqueue free-job work
+ * @sched: scheduler instance
+ */
+static void __drm_sched_run_free_queue(struct drm_gpu_scheduler *sched)
+{
+	if (!READ_ONCE(sched->pause_submit))
+		queue_work(sched->submit_wq, &sched->work_free_job);
+}
+
+/**
+ * drm_sched_run_free_queue - enqueue free-job work if ready
+ * @sched: scheduler instance
+ */
+static void drm_sched_run_free_queue(struct drm_gpu_scheduler *sched)
+{
+	struct drm_sched_job *job;
+
+	spin_lock(&sched->job_list_lock);
+	job = list_first_entry_or_null(&sched->pending_list,
+				       struct drm_sched_job, list);
+	if (job && dma_fence_is_signaled(&job->s_fence->finished))
+		__drm_sched_run_free_queue(sched);
+	spin_unlock(&sched->job_list_lock);
+}
+
 /**
  * drm_sched_job_done - complete a job
  * @s_job: pointer to the job which is done
@@ -267,7 +406,7 @@ static void drm_sched_job_done(struct drm_sched_job *s_job, int result)
 	struct drm_sched_fence *s_fence = s_job->s_fence;
 	struct drm_gpu_scheduler *sched = s_fence->sched;
 
-	atomic_dec(&sched->hw_rq_count);
+	atomic_sub(s_job->credits, &sched->credit_count);
 	atomic_dec(sched->score);
 
 	trace_drm_sched_process_job(s_fence);
@@ -275,7 +414,7 @@ static void drm_sched_job_done(struct drm_sched_job *s_job, int result)
 	dma_fence_get(&s_fence->finished);
 	drm_sched_fence_finished(s_fence, result);
 	dma_fence_put(&s_fence->finished);
-	wake_up_interruptible(&sched->wake_up_worker);
+	__drm_sched_run_free_queue(sched);
 }
 
 /**
@@ -299,10 +438,35 @@ static void drm_sched_job_done_cb(struct dma_fence *f, struct dma_fence_cb *cb)
  */
 static void drm_sched_start_timeout(struct drm_gpu_scheduler *sched)
 {
+	lockdep_assert_held(&sched->job_list_lock);
+
 	if (sched->timeout != MAX_SCHEDULE_TIMEOUT &&
 	    !list_empty(&sched->pending_list))
-		queue_delayed_work(sched->timeout_wq, &sched->work_tdr, sched->timeout);
+		mod_delayed_work(sched->timeout_wq, &sched->work_tdr, sched->timeout);
+}
+
+static void drm_sched_start_timeout_unlocked(struct drm_gpu_scheduler *sched)
+{
+	spin_lock(&sched->job_list_lock);
+	drm_sched_start_timeout(sched);
+	spin_unlock(&sched->job_list_lock);
+}
+
+/**
+ * drm_sched_tdr_queue_imm: - immediately start job timeout handler
+ *
+ * @sched: scheduler for which the timeout handling should be started.
+ *
+ * Start timeout handling immediately for the named scheduler.
+ */
+void drm_sched_tdr_queue_imm(struct drm_gpu_scheduler *sched)
+{
+	spin_lock(&sched->job_list_lock);
+	sched->timeout = 0;
+	drm_sched_start_timeout(sched);
+	spin_unlock(&sched->job_list_lock);
 }
+EXPORT_SYMBOL(drm_sched_tdr_queue_imm);
 
 /**
  * drm_sched_fault - immediately start timeout handler
@@ -388,7 +552,7 @@ static void drm_sched_job_timedout(struct work_struct *work)
 
 	sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
 
-	/* Protects against concurrent deletion in drm_sched_get_cleanup_job */
+	/* Protects against concurrent deletion in drm_sched_get_finished_job */
 	spin_lock(&sched->job_list_lock);
 	job = list_first_entry_or_null(&sched->pending_list,
 				       struct drm_sched_job, list);
@@ -416,11 +580,8 @@ static void drm_sched_job_timedout(struct work_struct *work)
 		spin_unlock(&sched->job_list_lock);
 	}
 
-	if (status != DRM_GPU_SCHED_STAT_ENODEV) {
-		spin_lock(&sched->job_list_lock);
-		drm_sched_start_timeout(sched);
-		spin_unlock(&sched->job_list_lock);
-	}
+	if (status != DRM_GPU_SCHED_STAT_ENODEV)
+		drm_sched_start_timeout_unlocked(sched);
 }
 
 /**
@@ -439,13 +600,13 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
 {
 	struct drm_sched_job *s_job, *tmp;
 
-	kthread_park(sched->thread);
+	drm_sched_wqueue_stop(sched);
 
 	/*
 	 * Reinsert back the bad job here - now it's safe as
-	 * drm_sched_get_cleanup_job cannot race against us and release the
+	 * drm_sched_get_finished_job cannot race against us and release the
 	 * bad job at this point - we parked (waited for) any in progress
-	 * (earlier) cleanups and drm_sched_get_cleanup_job will not be called
+	 * (earlier) cleanups and drm_sched_get_finished_job will not be called
 	 * now until the scheduler thread is unparked.
 	 */
 	if (bad && bad->sched == sched)
@@ -468,7 +629,7 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
 					      &s_job->cb)) {
 			dma_fence_put(s_job->s_fence->parent);
 			s_job->s_fence->parent = NULL;
-			atomic_dec(&sched->hw_rq_count);
+			atomic_sub(s_job->credits, &sched->credit_count);
 		} else {
 			/*
 			 * remove job from pending_list.
@@ -529,7 +690,7 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery)
 	list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {
 		struct dma_fence *fence = s_job->s_fence->parent;
 
-		atomic_inc(&sched->hw_rq_count);
+		atomic_add(s_job->credits, &sched->credit_count);
 
 		if (!full_recovery)
 			continue;
@@ -546,13 +707,10 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery)
 			drm_sched_job_done(s_job, -ECANCELED);
 	}
 
-	if (full_recovery) {
-		spin_lock(&sched->job_list_lock);
-		drm_sched_start_timeout(sched);
-		spin_unlock(&sched->job_list_lock);
-	}
+	if (full_recovery)
+		drm_sched_start_timeout_unlocked(sched);
 
-	kthread_unpark(sched->thread);
+	drm_sched_wqueue_start(sched);
 }
 EXPORT_SYMBOL(drm_sched_start);
 
@@ -613,6 +771,8 @@ EXPORT_SYMBOL(drm_sched_resubmit_jobs);
  * drm_sched_job_init - init a scheduler job
  * @job: scheduler job to init
  * @entity: scheduler entity to use
+ * @credits: the number of credits this job contributes to the schedulers
+ * credit limit
  * @owner: job owner for debugging
  *
  * Refer to drm_sched_entity_push_job() documentation
@@ -630,7 +790,7 @@ EXPORT_SYMBOL(drm_sched_resubmit_jobs);
  */
 int drm_sched_job_init(struct drm_sched_job *job,
 		       struct drm_sched_entity *entity,
-		       void *owner)
+		       u32 credits, void *owner)
 {
 	if (!entity->rq) {
 		/* This will most likely be followed by missing frames
@@ -641,7 +801,13 @@ int drm_sched_job_init(struct drm_sched_job *job,
 		return -ENOENT;
 	}
 
+	if (unlikely(!credits)) {
+		pr_err("*ERROR* %s: credits cannot be 0!\n", __func__);
+		return -EINVAL;
+	}
+
 	job->entity = entity;
+	job->credits = credits;
 	job->s_fence = drm_sched_fence_alloc(entity, owner);
 	if (!job->s_fence)
 		return -ENOMEM;
@@ -854,27 +1020,18 @@ void drm_sched_job_cleanup(struct drm_sched_job *job)
 EXPORT_SYMBOL(drm_sched_job_cleanup);
 
 /**
- * drm_sched_can_queue -- Can we queue more to the hardware?
- * @sched: scheduler instance
- *
- * Return true if we can push more jobs to the hw, otherwise false.
- */
-static bool drm_sched_can_queue(struct drm_gpu_scheduler *sched)
-{
-	return atomic_read(&sched->hw_rq_count) <
-		sched->hw_submission_limit;
-}
-
-/**
- * drm_sched_wakeup_if_can_queue - Wake up the scheduler
+ * drm_sched_wakeup - Wake up the scheduler if it is ready to queue
  * @sched: scheduler instance
+ * @entity: the scheduler entity
  *
  * Wake up the scheduler if we can queue jobs.
  */
-void drm_sched_wakeup_if_can_queue(struct drm_gpu_scheduler *sched)
+void drm_sched_wakeup(struct drm_gpu_scheduler *sched,
+		      struct drm_sched_entity *entity)
 {
-	if (drm_sched_can_queue(sched))
-		wake_up_interruptible(&sched->wake_up_worker);
+	if (drm_sched_entity_is_ready(entity))
+		if (drm_sched_can_queue(sched, entity))
+			drm_sched_run_job_queue(sched);
 }
 
 /**
@@ -882,7 +1039,11 @@ void drm_sched_wakeup_if_can_queue(struct drm_gpu_scheduler *sched)
  *
  * @sched: scheduler instance
  *
- * Returns the entity to process or NULL if none are found.
+ * Return an entity to process or NULL if none are found.
+ *
+ * Note, that we break out of the for-loop when "entity" is non-null, which can
+ * also be an error-pointer--this assures we don't process lower priority
+ * run-queues. See comments in the respectively called functions.
  */
 static struct drm_sched_entity *
 drm_sched_select_entity(struct drm_gpu_scheduler *sched)
@@ -890,23 +1051,20 @@ drm_sched_select_entity(struct drm_gpu_scheduler *sched)
 	struct drm_sched_entity *entity;
 	int i;
 
-	if (!drm_sched_can_queue(sched))
-		return NULL;
-
 	/* Kernel run queue has higher priority than normal run queue*/
 	for (i = sched->num_rqs - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) {
 		entity = drm_sched_policy == DRM_SCHED_POLICY_FIFO ?
-			drm_sched_rq_select_entity_fifo(sched->sched_rq[i]) :
-			drm_sched_rq_select_entity_rr(sched->sched_rq[i]);
+			drm_sched_rq_select_entity_fifo(sched, sched->sched_rq[i]) :
+			drm_sched_rq_select_entity_rr(sched, sched->sched_rq[i]);
 		if (entity)
 			break;
 	}
 
-	return entity;
+	return IS_ERR(entity) ? NULL : entity;
 }
 
 /**
- * drm_sched_get_cleanup_job - fetch the next finished job to be destroyed
+ * drm_sched_get_finished_job - fetch the next finished job to be destroyed
  *
  * @sched: scheduler instance
  *
@@ -914,7 +1072,7 @@ drm_sched_select_entity(struct drm_gpu_scheduler *sched)
  * ready for it to be destroyed.
  */
 static struct drm_sched_job *
-drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched)
+drm_sched_get_finished_job(struct drm_gpu_scheduler *sched)
 {
 	struct drm_sched_job *job, *next;
 
@@ -934,8 +1092,10 @@ drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched)
 						typeof(*next), list);
 
 		if (next) {
-			next->s_fence->scheduled.timestamp =
-				dma_fence_timestamp(&job->s_fence->finished);
+			if (test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT,
+				     &next->s_fence->scheduled.flags))
+				next->s_fence->scheduled.timestamp =
+					dma_fence_timestamp(&job->s_fence->finished);
 			/* start TO timer for next job */
 			drm_sched_start_timeout(sched);
 		}
@@ -985,91 +1145,82 @@ drm_sched_pick_best(struct drm_gpu_scheduler **sched_list,
 EXPORT_SYMBOL(drm_sched_pick_best);
 
 /**
- * drm_sched_blocked - check if the scheduler is blocked
- *
- * @sched: scheduler instance
+ * drm_sched_free_job_work - worker to call free_job
  *
- * Returns true if blocked, otherwise false.
+ * @w: free job work
  */
-static bool drm_sched_blocked(struct drm_gpu_scheduler *sched)
+static void drm_sched_free_job_work(struct work_struct *w)
 {
-	if (kthread_should_park()) {
-		kthread_parkme();
-		return true;
-	}
+	struct drm_gpu_scheduler *sched =
+		container_of(w, struct drm_gpu_scheduler, work_free_job);
+	struct drm_sched_job *job;
+
+	if (READ_ONCE(sched->pause_submit))
+		return;
 
-	return false;
+	job = drm_sched_get_finished_job(sched);
+	if (job)
+		sched->ops->free_job(job);
+
+	drm_sched_run_free_queue(sched);
+	drm_sched_run_job_queue(sched);
 }
 
 /**
- * drm_sched_main - main scheduler thread
- *
- * @param: scheduler instance
+ * drm_sched_run_job_work - worker to call run_job
  *
- * Returns 0.
+ * @w: run job work
  */
-static int drm_sched_main(void *param)
+static void drm_sched_run_job_work(struct work_struct *w)
 {
-	struct drm_gpu_scheduler *sched = (struct drm_gpu_scheduler *)param;
+	struct drm_gpu_scheduler *sched =
+		container_of(w, struct drm_gpu_scheduler, work_run_job);
+	struct drm_sched_entity *entity;
+	struct dma_fence *fence;
+	struct drm_sched_fence *s_fence;
+	struct drm_sched_job *sched_job;
 	int r;
 
-	sched_set_fifo_low(current);
-
-	while (!kthread_should_stop()) {
-		struct drm_sched_entity *entity = NULL;
-		struct drm_sched_fence *s_fence;
-		struct drm_sched_job *sched_job;
-		struct dma_fence *fence;
-		struct drm_sched_job *cleanup_job = NULL;
-
-		wait_event_interruptible(sched->wake_up_worker,
-					 (cleanup_job = drm_sched_get_cleanup_job(sched)) ||
-					 (!drm_sched_blocked(sched) &&
-					  (entity = drm_sched_select_entity(sched))) ||
-					 kthread_should_stop());
-
-		if (cleanup_job)
-			sched->ops->free_job(cleanup_job);
-
-		if (!entity)
-			continue;
-
-		sched_job = drm_sched_entity_pop_job(entity);
+	if (READ_ONCE(sched->pause_submit))
+		return;
 
-		if (!sched_job) {
-			complete_all(&entity->entity_idle);
-			continue;
-		}
+	entity = drm_sched_select_entity(sched);
+	if (!entity)
+		return;
 
-		s_fence = sched_job->s_fence;
+	sched_job = drm_sched_entity_pop_job(entity);
+	if (!sched_job) {
+		complete_all(&entity->entity_idle);
+		return;	/* No more work */
+	}
 
-		atomic_inc(&sched->hw_rq_count);
-		drm_sched_job_begin(sched_job);
+	s_fence = sched_job->s_fence;
 
-		trace_drm_run_job(sched_job, entity);
-		fence = sched->ops->run_job(sched_job);
-		complete_all(&entity->entity_idle);
-		drm_sched_fence_scheduled(s_fence, fence);
+	atomic_add(sched_job->credits, &sched->credit_count);
+	drm_sched_job_begin(sched_job);
 
-		if (!IS_ERR_OR_NULL(fence)) {
-			/* Drop for original kref_init of the fence */
-			dma_fence_put(fence);
+	trace_drm_run_job(sched_job, entity);
+	fence = sched->ops->run_job(sched_job);
+	complete_all(&entity->entity_idle);
+	drm_sched_fence_scheduled(s_fence, fence);
 
-			r = dma_fence_add_callback(fence, &sched_job->cb,
-						   drm_sched_job_done_cb);
-			if (r == -ENOENT)
-				drm_sched_job_done(sched_job, fence->error);
-			else if (r)
-				DRM_DEV_ERROR(sched->dev, "fence add callback failed (%d)\n",
-					  r);
-		} else {
-			drm_sched_job_done(sched_job, IS_ERR(fence) ?
-					   PTR_ERR(fence) : 0);
-		}
+	if (!IS_ERR_OR_NULL(fence)) {
+		/* Drop for original kref_init of the fence */
+		dma_fence_put(fence);
 
-		wake_up(&sched->job_scheduled);
+		r = dma_fence_add_callback(fence, &sched_job->cb,
+					   drm_sched_job_done_cb);
+		if (r == -ENOENT)
+			drm_sched_job_done(sched_job, fence->error);
+		else if (r)
+			DRM_DEV_ERROR(sched->dev, "fence add callback failed (%d)\n", r);
+	} else {
+		drm_sched_job_done(sched_job, IS_ERR(fence) ?
+				   PTR_ERR(fence) : 0);
 	}
-	return 0;
+
+	wake_up(&sched->job_scheduled);
+	drm_sched_run_job_queue(sched);
 }
 
 /**
@@ -1077,8 +1228,10 @@ static int drm_sched_main(void *param)
  *
  * @sched: scheduler instance
  * @ops: backend operations for this scheduler
+ * @submit_wq: workqueue to use for submission. If NULL, an ordered wq is
+ *	       allocated and used
  * @num_rqs: number of runqueues, one for each priority, up to DRM_SCHED_PRIORITY_COUNT
- * @hw_submission: number of hw submissions that can be in flight
+ * @credit_limit: the number of credits this scheduler can hold from all jobs
  * @hang_limit: number of times to allow a job to hang before dropping it
  * @timeout: timeout value in jiffies for the scheduler
  * @timeout_wq: workqueue to use for timeout work. If NULL, the system_wq is
@@ -1091,14 +1244,15 @@ static int drm_sched_main(void *param)
  */
 int drm_sched_init(struct drm_gpu_scheduler *sched,
 		   const struct drm_sched_backend_ops *ops,
-		   u32 num_rqs, uint32_t hw_submission, unsigned int hang_limit,
+		   struct workqueue_struct *submit_wq,
+		   u32 num_rqs, u32 credit_limit, unsigned int hang_limit,
 		   long timeout, struct workqueue_struct *timeout_wq,
 		   atomic_t *score, const char *name, struct device *dev)
 {
 	int i, ret;
 
 	sched->ops = ops;
-	sched->hw_submission_limit = hw_submission;
+	sched->credit_limit = credit_limit;
 	sched->name = name;
 	sched->timeout = timeout;
 	sched->timeout_wq = timeout_wq ? : system_wq;
@@ -1121,14 +1275,22 @@ int drm_sched_init(struct drm_gpu_scheduler *sched,
 		return 0;
 	}
 
+	if (submit_wq) {
+		sched->submit_wq = submit_wq;
+		sched->own_submit_wq = false;
+	} else {
+		sched->submit_wq = alloc_ordered_workqueue(name, 0);
+		if (!sched->submit_wq)
+			return -ENOMEM;
+
+		sched->own_submit_wq = true;
+	}
+	ret = -ENOMEM;
 	sched->sched_rq = kmalloc_array(num_rqs, sizeof(*sched->sched_rq),
 					GFP_KERNEL | __GFP_ZERO);
-	if (!sched->sched_rq) {
-		drm_err(sched, "%s: out of memory for sched_rq\n", __func__);
-		return -ENOMEM;
-	}
+	if (!sched->sched_rq)
+		goto Out_free;
 	sched->num_rqs = num_rqs;
-	ret = -ENOMEM;
 	for (i = DRM_SCHED_PRIORITY_MIN; i < sched->num_rqs; i++) {
 		sched->sched_rq[i] = kzalloc(sizeof(*sched->sched_rq[i]), GFP_KERNEL);
 		if (!sched->sched_rq[i])
@@ -1136,31 +1298,27 @@ int drm_sched_init(struct drm_gpu_scheduler *sched,
 		drm_sched_rq_init(sched, sched->sched_rq[i]);
 	}
 
-	init_waitqueue_head(&sched->wake_up_worker);
 	init_waitqueue_head(&sched->job_scheduled);
 	INIT_LIST_HEAD(&sched->pending_list);
 	spin_lock_init(&sched->job_list_lock);
-	atomic_set(&sched->hw_rq_count, 0);
+	atomic_set(&sched->credit_count, 0);
 	INIT_DELAYED_WORK(&sched->work_tdr, drm_sched_job_timedout);
+	INIT_WORK(&sched->work_run_job, drm_sched_run_job_work);
+	INIT_WORK(&sched->work_free_job, drm_sched_free_job_work);
 	atomic_set(&sched->_score, 0);
 	atomic64_set(&sched->job_id_count, 0);
-
-	/* Each scheduler will run on a seperate kernel thread */
-	sched->thread = kthread_run(drm_sched_main, sched, sched->name);
-	if (IS_ERR(sched->thread)) {
-		ret = PTR_ERR(sched->thread);
-		sched->thread = NULL;
-		DRM_DEV_ERROR(sched->dev, "Failed to create scheduler for %s.\n", name);
-		goto Out_unroll;
-	}
+	sched->pause_submit = false;
 
 	sched->ready = true;
 	return 0;
 Out_unroll:
 	for (--i ; i >= DRM_SCHED_PRIORITY_MIN; i--)
 		kfree(sched->sched_rq[i]);
+Out_free:
 	kfree(sched->sched_rq);
 	sched->sched_rq = NULL;
+	if (sched->own_submit_wq)
+		destroy_workqueue(sched->submit_wq);
 	drm_err(sched, "%s: Failed to setup GPU scheduler--out of memory\n", __func__);
 	return ret;
 }
@@ -1178,8 +1336,7 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched)
 	struct drm_sched_entity *s_entity;
 	int i;
 
-	if (sched->thread)
-		kthread_stop(sched->thread);
+	drm_sched_wqueue_stop(sched);
 
 	for (i = sched->num_rqs - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) {
 		struct drm_sched_rq *rq = sched->sched_rq[i];
@@ -1202,6 +1359,8 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched)
 	/* Confirm no work left behind accessing device structures */
 	cancel_delayed_work_sync(&sched->work_tdr);
 
+	if (sched->own_submit_wq)
+		destroy_workqueue(sched->submit_wq);
 	sched->ready = false;
 	kfree(sched->sched_rq);
 	sched->sched_rq = NULL;
@@ -1252,3 +1411,42 @@ void drm_sched_increase_karma(struct drm_sched_job *bad)
 	}
 }
 EXPORT_SYMBOL(drm_sched_increase_karma);
+
+/**
+ * drm_sched_wqueue_ready - Is the scheduler ready for submission
+ *
+ * @sched: scheduler instance
+ *
+ * Returns true if submission is ready
+ */
+bool drm_sched_wqueue_ready(struct drm_gpu_scheduler *sched)
+{
+	return sched->ready;
+}
+EXPORT_SYMBOL(drm_sched_wqueue_ready);
+
+/**
+ * drm_sched_wqueue_stop - stop scheduler submission
+ *
+ * @sched: scheduler instance
+ */
+void drm_sched_wqueue_stop(struct drm_gpu_scheduler *sched)
+{
+	WRITE_ONCE(sched->pause_submit, true);
+	cancel_work_sync(&sched->work_run_job);
+	cancel_work_sync(&sched->work_free_job);
+}
+EXPORT_SYMBOL(drm_sched_wqueue_stop);
+
+/**
+ * drm_sched_wqueue_start - start scheduler submission
+ *
+ * @sched: scheduler instance
+ */
+void drm_sched_wqueue_start(struct drm_gpu_scheduler *sched)
+{
+	WRITE_ONCE(sched->pause_submit, false);
+	queue_work(sched->submit_wq, &sched->work_run_job);
+	queue_work(sched->submit_wq, &sched->work_free_job);
+}
+EXPORT_SYMBOL(drm_sched_wqueue_start);
diff --git a/drivers/gpu/drm/solomon/ssd130x.c b/drivers/gpu/drm/solomon/ssd130x.c
index e0174f82e353..bef293922b98 100644
--- a/drivers/gpu/drm/solomon/ssd130x.c
+++ b/drivers/gpu/drm/solomon/ssd130x.c
@@ -808,7 +808,8 @@ static void ssd132x_clear_screen(struct ssd130x_device *ssd130x, u8 *data_array)
 static int ssd130x_fb_blit_rect(struct drm_framebuffer *fb,
 				const struct iosys_map *vmap,
 				struct drm_rect *rect,
-				u8 *buf, u8 *data_array)
+				u8 *buf, u8 *data_array,
+				struct drm_format_conv_state *fmtcnv_state)
 {
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(fb->dev);
 	struct iosys_map dst;
@@ -826,7 +827,7 @@ static int ssd130x_fb_blit_rect(struct drm_framebuffer *fb,
 		return ret;
 
 	iosys_map_set_vaddr(&dst, buf);
-	drm_fb_xrgb8888_to_mono(&dst, &dst_pitch, vmap, fb, rect);
+	drm_fb_xrgb8888_to_mono(&dst, &dst_pitch, vmap, fb, rect, fmtcnv_state);
 
 	drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE);
 
@@ -838,7 +839,8 @@ static int ssd130x_fb_blit_rect(struct drm_framebuffer *fb,
 static int ssd132x_fb_blit_rect(struct drm_framebuffer *fb,
 				const struct iosys_map *vmap,
 				struct drm_rect *rect, u8 *buf,
-				u8 *data_array)
+				u8 *data_array,
+				struct drm_format_conv_state *fmtcnv_state)
 {
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(fb->dev);
 	unsigned int dst_pitch = drm_rect_width(rect);
@@ -855,7 +857,7 @@ static int ssd132x_fb_blit_rect(struct drm_framebuffer *fb,
 		return ret;
 
 	iosys_map_set_vaddr(&dst, buf);
-	drm_fb_xrgb8888_to_gray8(&dst, &dst_pitch, vmap, fb, rect);
+	drm_fb_xrgb8888_to_gray8(&dst, &dst_pitch, vmap, fb, rect, fmtcnv_state);
 
 	drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE);
 
@@ -871,6 +873,7 @@ static int ssd130x_primary_plane_atomic_check(struct drm_plane *plane,
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
 	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
 	struct ssd130x_plane_state *ssd130x_state = to_ssd130x_plane_state(plane_state);
+	struct drm_shadow_plane_state *shadow_plane_state = &ssd130x_state->base;
 	struct drm_crtc *crtc = plane_state->crtc;
 	struct drm_crtc_state *crtc_state = NULL;
 	const struct drm_format_info *fi;
@@ -895,6 +898,16 @@ static int ssd130x_primary_plane_atomic_check(struct drm_plane *plane,
 
 	pitch = drm_format_info_min_pitch(fi, 0, ssd130x->width);
 
+	if (plane_state->fb->format != fi) {
+		void *buf;
+
+		/* format conversion necessary; reserve buffer */
+		buf = drm_format_conv_state_reserve(&shadow_plane_state->fmtcnv_state,
+						    pitch, GFP_KERNEL);
+		if (!buf)
+			return -ENOMEM;
+	}
+
 	ssd130x_state->buffer = kcalloc(pitch, ssd130x->height, GFP_KERNEL);
 	if (!ssd130x_state->buffer)
 		return -ENOMEM;
@@ -909,6 +922,7 @@ static int ssd132x_primary_plane_atomic_check(struct drm_plane *plane,
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
 	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
 	struct ssd130x_plane_state *ssd130x_state = to_ssd130x_plane_state(plane_state);
+	struct drm_shadow_plane_state *shadow_plane_state = &ssd130x_state->base;
 	struct drm_crtc *crtc = plane_state->crtc;
 	struct drm_crtc_state *crtc_state = NULL;
 	const struct drm_format_info *fi;
@@ -933,6 +947,16 @@ static int ssd132x_primary_plane_atomic_check(struct drm_plane *plane,
 
 	pitch = drm_format_info_min_pitch(fi, 0, ssd130x->width);
 
+	if (plane_state->fb->format != fi) {
+		void *buf;
+
+		/* format conversion necessary; reserve buffer */
+		buf = drm_format_conv_state_reserve(&shadow_plane_state->fmtcnv_state,
+						    pitch, GFP_KERNEL);
+		if (!buf)
+			return -ENOMEM;
+	}
+
 	ssd130x_state->buffer = kcalloc(pitch, ssd130x->height, GFP_KERNEL);
 	if (!ssd130x_state->buffer)
 		return -ENOMEM;
@@ -968,7 +992,8 @@ static void ssd130x_primary_plane_atomic_update(struct drm_plane *plane,
 
 		ssd130x_fb_blit_rect(fb, &shadow_plane_state->data[0], &dst_clip,
 				     ssd130x_plane_state->buffer,
-				     ssd130x_crtc_state->data_array);
+				     ssd130x_crtc_state->data_array,
+				     &shadow_plane_state->fmtcnv_state);
 	}
 
 	drm_dev_exit(idx);
@@ -1002,7 +1027,8 @@ static void ssd132x_primary_plane_atomic_update(struct drm_plane *plane,
 
 		ssd132x_fb_blit_rect(fb, &shadow_plane_state->data[0], &dst_clip,
 				     ssd130x_plane_state->buffer,
-				     ssd130x_crtc_state->data_array);
+				     ssd130x_crtc_state->data_array,
+				     &shadow_plane_state->fmtcnv_state);
 	}
 
 	drm_dev_exit(idx);
diff --git a/drivers/gpu/drm/sprd/sprd_dpu.c b/drivers/gpu/drm/sprd/sprd_dpu.c
index 48183bbd0590..deb3bb96e2a8 100644
--- a/drivers/gpu/drm/sprd/sprd_dpu.c
+++ b/drivers/gpu/drm/sprd/sprd_dpu.c
@@ -859,16 +859,14 @@ static int sprd_dpu_probe(struct platform_device *pdev)
 	return component_add(&pdev->dev, &dpu_component_ops);
 }
 
-static int sprd_dpu_remove(struct platform_device *pdev)
+static void sprd_dpu_remove(struct platform_device *pdev)
 {
 	component_del(&pdev->dev, &dpu_component_ops);
-
-	return 0;
 }
 
 struct platform_driver sprd_dpu_driver = {
 	.probe = sprd_dpu_probe,
-	.remove = sprd_dpu_remove,
+	.remove_new = sprd_dpu_remove,
 	.driver = {
 		.name = "sprd-dpu-drv",
 		.of_match_table = dpu_match_table,
diff --git a/drivers/gpu/drm/sprd/sprd_drm.c b/drivers/gpu/drm/sprd/sprd_drm.c
index 0aa39156f2fa..a74cd0caf645 100644
--- a/drivers/gpu/drm/sprd/sprd_drm.c
+++ b/drivers/gpu/drm/sprd/sprd_drm.c
@@ -138,10 +138,9 @@ static int sprd_drm_probe(struct platform_device *pdev)
 	return drm_of_component_probe(&pdev->dev, component_compare_of, &drm_component_ops);
 }
 
-static int sprd_drm_remove(struct platform_device *pdev)
+static void sprd_drm_remove(struct platform_device *pdev)
 {
 	component_master_del(&pdev->dev, &drm_component_ops);
-	return 0;
 }
 
 static void sprd_drm_shutdown(struct platform_device *pdev)
@@ -164,7 +163,7 @@ MODULE_DEVICE_TABLE(of, drm_match_table);
 
 static struct platform_driver sprd_drm_driver = {
 	.probe = sprd_drm_probe,
-	.remove = sprd_drm_remove,
+	.remove_new = sprd_drm_remove,
 	.shutdown = sprd_drm_shutdown,
 	.driver = {
 		.name = "sprd-drm-drv",
diff --git a/drivers/gpu/drm/sprd/sprd_dsi.c b/drivers/gpu/drm/sprd/sprd_dsi.c
index d7b143a75601..0b69c140eab3 100644
--- a/drivers/gpu/drm/sprd/sprd_dsi.c
+++ b/drivers/gpu/drm/sprd/sprd_dsi.c
@@ -1051,18 +1051,16 @@ static int sprd_dsi_probe(struct platform_device *pdev)
 	return mipi_dsi_host_register(&dsi->host);
 }
 
-static int sprd_dsi_remove(struct platform_device *pdev)
+static void sprd_dsi_remove(struct platform_device *pdev)
 {
 	struct sprd_dsi *dsi = dev_get_drvdata(&pdev->dev);
 
 	mipi_dsi_host_unregister(&dsi->host);
-
-	return 0;
 }
 
 struct platform_driver sprd_dsi_driver = {
 	.probe = sprd_dsi_probe,
-	.remove = sprd_dsi_remove,
+	.remove_new = sprd_dsi_remove,
 	.driver = {
 		.name = "sprd-dsi-drv",
 		.of_match_table = dsi_match_table,
diff --git a/drivers/gpu/drm/tegra/hdmi.c b/drivers/gpu/drm/tegra/hdmi.c
index 0ba3ca3ac509..a1fcee665023 100644
--- a/drivers/gpu/drm/tegra/hdmi.c
+++ b/drivers/gpu/drm/tegra/hdmi.c
@@ -24,6 +24,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_debugfs.h>
+#include <drm/drm_eld.h>
 #include <drm/drm_file.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_probe_helper.h>
diff --git a/drivers/gpu/drm/tegra/sor.c b/drivers/gpu/drm/tegra/sor.c
index d5a3d3f4fece..83341576630d 100644
--- a/drivers/gpu/drm/tegra/sor.c
+++ b/drivers/gpu/drm/tegra/sor.c
@@ -20,6 +20,7 @@
 #include <drm/display/drm_scdc_helper.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_debugfs.h>
+#include <drm/drm_eld.h>
 #include <drm/drm_file.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_simple_kms_helper.h>
diff --git a/drivers/gpu/drm/tests/Makefile b/drivers/gpu/drm/tests/Makefile
index ba7baa622675..2645af241ff0 100644
--- a/drivers/gpu/drm/tests/Makefile
+++ b/drivers/gpu/drm/tests/Makefile
@@ -9,6 +9,7 @@ obj-$(CONFIG_DRM_KUNIT_TEST) += \
 	drm_connector_test.o \
 	drm_damage_helper_test.o \
 	drm_dp_mst_helper_test.o \
+	drm_exec_test.o \
 	drm_format_helper_test.o \
 	drm_format_test.o \
 	drm_framebuffer_test.o \
@@ -17,7 +18,6 @@ obj-$(CONFIG_DRM_KUNIT_TEST) += \
 	drm_modes_test.o \
 	drm_plane_helper_test.o \
 	drm_probe_helper_test.o \
-	drm_rect_test.o	\
-	drm_exec_test.o
+	drm_rect_test.o
 
 CFLAGS_drm_mm_test.o := $(DISABLE_STRUCTLEAK_PLUGIN)
diff --git a/drivers/gpu/drm/tests/drm_buddy_test.c b/drivers/gpu/drm/tests/drm_buddy_test.c
index 09ee6f6af896..ea2af6bd9abe 100644
--- a/drivers/gpu/drm/tests/drm_buddy_test.c
+++ b/drivers/gpu/drm/tests/drm_buddy_test.c
@@ -13,315 +13,11 @@
 
 #include "../lib/drm_random.h"
 
-#define TIMEOUT(name__)								\
-	unsigned long name__ = jiffies + MAX_SCHEDULE_TIMEOUT
-
-static unsigned int random_seed;
-
 static inline u64 get_size(int order, u64 chunk_size)
 {
 	return (1 << order) * chunk_size;
 }
 
-__printf(2, 3)
-static bool __timeout(unsigned long timeout, const char *fmt, ...)
-{
-	va_list va;
-
-	if (!signal_pending(current)) {
-		cond_resched();
-		if (time_before(jiffies, timeout))
-			return false;
-	}
-
-	if (fmt) {
-		va_start(va, fmt);
-		vprintk(fmt, va);
-		va_end(va);
-	}
-
-	return true;
-}
-
-static void __dump_block(struct kunit *test, struct drm_buddy *mm,
-			 struct drm_buddy_block *block, bool buddy)
-{
-	kunit_err(test, "block info: header=%llx, state=%u, order=%d, offset=%llx size=%llx root=%d buddy=%d\n",
-		  block->header, drm_buddy_block_state(block),
-			  drm_buddy_block_order(block), drm_buddy_block_offset(block),
-			  drm_buddy_block_size(mm, block), !block->parent, buddy);
-}
-
-static void dump_block(struct kunit *test, struct drm_buddy *mm,
-		       struct drm_buddy_block *block)
-{
-	struct drm_buddy_block *buddy;
-
-	__dump_block(test, mm, block, false);
-
-	buddy = drm_get_buddy(block);
-	if (buddy)
-		__dump_block(test, mm, buddy, true);
-}
-
-static int check_block(struct kunit *test, struct drm_buddy *mm,
-		       struct drm_buddy_block *block)
-{
-	struct drm_buddy_block *buddy;
-	unsigned int block_state;
-	u64 block_size;
-	u64 offset;
-	int err = 0;
-
-	block_state = drm_buddy_block_state(block);
-
-	if (block_state != DRM_BUDDY_ALLOCATED &&
-	    block_state != DRM_BUDDY_FREE && block_state != DRM_BUDDY_SPLIT) {
-		kunit_err(test, "block state mismatch\n");
-		err = -EINVAL;
-	}
-
-	block_size = drm_buddy_block_size(mm, block);
-	offset = drm_buddy_block_offset(block);
-
-	if (block_size < mm->chunk_size) {
-		kunit_err(test, "block size smaller than min size\n");
-		err = -EINVAL;
-	}
-
-	/* We can't use is_power_of_2() for a u64 on 32-bit systems. */
-	if (block_size & (block_size - 1)) {
-		kunit_err(test, "block size not power of two\n");
-		err = -EINVAL;
-	}
-
-	if (!IS_ALIGNED(block_size, mm->chunk_size)) {
-		kunit_err(test, "block size not aligned to min size\n");
-		err = -EINVAL;
-	}
-
-	if (!IS_ALIGNED(offset, mm->chunk_size)) {
-		kunit_err(test, "block offset not aligned to min size\n");
-		err = -EINVAL;
-	}
-
-	if (!IS_ALIGNED(offset, block_size)) {
-		kunit_err(test, "block offset not aligned to block size\n");
-		err = -EINVAL;
-	}
-
-	buddy = drm_get_buddy(block);
-
-	if (!buddy && block->parent) {
-		kunit_err(test, "buddy has gone fishing\n");
-		err = -EINVAL;
-	}
-
-	if (buddy) {
-		if (drm_buddy_block_offset(buddy) != (offset ^ block_size)) {
-			kunit_err(test, "buddy has wrong offset\n");
-			err = -EINVAL;
-		}
-
-		if (drm_buddy_block_size(mm, buddy) != block_size) {
-			kunit_err(test, "buddy size mismatch\n");
-			err = -EINVAL;
-		}
-
-		if (drm_buddy_block_state(buddy) == block_state &&
-		    block_state == DRM_BUDDY_FREE) {
-			kunit_err(test, "block and its buddy are free\n");
-			err = -EINVAL;
-		}
-	}
-
-	return err;
-}
-
-static int check_blocks(struct kunit *test, struct drm_buddy *mm,
-			struct list_head *blocks, u64 expected_size, bool is_contiguous)
-{
-	struct drm_buddy_block *block;
-	struct drm_buddy_block *prev;
-	u64 total;
-	int err = 0;
-
-	block = NULL;
-	prev = NULL;
-	total = 0;
-
-	list_for_each_entry(block, blocks, link) {
-		err = check_block(test, mm, block);
-
-		if (!drm_buddy_block_is_allocated(block)) {
-			kunit_err(test, "block not allocated\n");
-			err = -EINVAL;
-		}
-
-		if (is_contiguous && prev) {
-			u64 prev_block_size;
-			u64 prev_offset;
-			u64 offset;
-
-			prev_offset = drm_buddy_block_offset(prev);
-			prev_block_size = drm_buddy_block_size(mm, prev);
-			offset = drm_buddy_block_offset(block);
-
-			if (offset != (prev_offset + prev_block_size)) {
-				kunit_err(test, "block offset mismatch\n");
-				err = -EINVAL;
-			}
-		}
-
-		if (err)
-			break;
-
-		total += drm_buddy_block_size(mm, block);
-		prev = block;
-	}
-
-	if (!err) {
-		if (total != expected_size) {
-			kunit_err(test, "size mismatch, expected=%llx, found=%llx\n",
-				  expected_size, total);
-			err = -EINVAL;
-		}
-		return err;
-	}
-
-	if (prev) {
-		kunit_err(test, "prev block, dump:\n");
-		dump_block(test, mm, prev);
-	}
-
-	kunit_err(test, "bad block, dump:\n");
-	dump_block(test, mm, block);
-
-	return err;
-}
-
-static int check_mm(struct kunit *test, struct drm_buddy *mm)
-{
-	struct drm_buddy_block *root;
-	struct drm_buddy_block *prev;
-	unsigned int i;
-	u64 total;
-	int err = 0;
-
-	if (!mm->n_roots) {
-		kunit_err(test, "n_roots is zero\n");
-		return -EINVAL;
-	}
-
-	if (mm->n_roots != hweight64(mm->size)) {
-		kunit_err(test, "n_roots mismatch, n_roots=%u, expected=%lu\n",
-			  mm->n_roots, hweight64(mm->size));
-		return -EINVAL;
-	}
-
-	root = NULL;
-	prev = NULL;
-	total = 0;
-
-	for (i = 0; i < mm->n_roots; ++i) {
-		struct drm_buddy_block *block;
-		unsigned int order;
-
-		root = mm->roots[i];
-		if (!root) {
-			kunit_err(test, "root(%u) is NULL\n", i);
-			err = -EINVAL;
-			break;
-		}
-
-		err = check_block(test, mm, root);
-
-		if (!drm_buddy_block_is_free(root)) {
-			kunit_err(test, "root not free\n");
-			err = -EINVAL;
-		}
-
-		order = drm_buddy_block_order(root);
-
-		if (!i) {
-			if (order != mm->max_order) {
-				kunit_err(test, "max order root missing\n");
-				err = -EINVAL;
-			}
-		}
-
-		if (prev) {
-			u64 prev_block_size;
-			u64 prev_offset;
-			u64 offset;
-
-			prev_offset = drm_buddy_block_offset(prev);
-			prev_block_size = drm_buddy_block_size(mm, prev);
-			offset = drm_buddy_block_offset(root);
-
-			if (offset != (prev_offset + prev_block_size)) {
-				kunit_err(test, "root offset mismatch\n");
-				err = -EINVAL;
-			}
-		}
-
-		block = list_first_entry_or_null(&mm->free_list[order],
-						 struct drm_buddy_block, link);
-		if (block != root) {
-			kunit_err(test, "root mismatch at order=%u\n", order);
-			err = -EINVAL;
-		}
-
-		if (err)
-			break;
-
-		prev = root;
-		total += drm_buddy_block_size(mm, root);
-	}
-
-	if (!err) {
-		if (total != mm->size) {
-			kunit_err(test, "expected mm size=%llx, found=%llx\n",
-				  mm->size, total);
-			err = -EINVAL;
-		}
-		return err;
-	}
-
-	if (prev) {
-		kunit_err(test, "prev root(%u), dump:\n", i - 1);
-		dump_block(test, mm, prev);
-	}
-
-	if (root) {
-		kunit_err(test, "bad root(%u), dump:\n", i);
-		dump_block(test, mm, root);
-	}
-
-	return err;
-}
-
-static void mm_config(u64 *size, u64 *chunk_size)
-{
-	DRM_RND_STATE(prng, random_seed);
-	u32 s, ms;
-
-	/* Nothing fancy, just try to get an interesting bit pattern */
-
-	prandom_seed_state(&prng, random_seed);
-
-	/* Let size be a random number of pages up to 8 GB (2M pages) */
-	s = 1 + drm_prandom_u32_max_state((BIT(33 - 12)) - 1, &prng);
-	/* Let the chunk size be a random power of 2 less than size */
-	ms = BIT(drm_prandom_u32_max_state(ilog2(s), &prng));
-	/* Round size down to the chunk size */
-	s &= -ms;
-
-	/* Convert from pages to bytes */
-	*chunk_size = (u64)ms << 12;
-	*size = (u64)s << 12;
-}
-
 static void drm_test_buddy_alloc_pathological(struct kunit *test)
 {
 	u64 mm_size, size, start = 0;
@@ -403,96 +99,6 @@ static void drm_test_buddy_alloc_pathological(struct kunit *test)
 	drm_buddy_fini(&mm);
 }
 
-static void drm_test_buddy_alloc_smoke(struct kunit *test)
-{
-	u64 mm_size, chunk_size, start = 0;
-	unsigned long flags = 0;
-	struct drm_buddy mm;
-	int *order;
-	int i;
-
-	DRM_RND_STATE(prng, random_seed);
-	TIMEOUT(end_time);
-
-	mm_config(&mm_size, &chunk_size);
-
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, chunk_size),
-			       "buddy_init failed\n");
-
-	order = drm_random_order(mm.max_order + 1, &prng);
-	KUNIT_ASSERT_TRUE(test, order);
-
-	for (i = 0; i <= mm.max_order; ++i) {
-		struct drm_buddy_block *block;
-		int max_order = order[i];
-		bool timeout = false;
-		LIST_HEAD(blocks);
-		u64 total, size;
-		LIST_HEAD(tmp);
-		int order, err;
-
-		KUNIT_ASSERT_FALSE_MSG(test, check_mm(test, &mm),
-				       "pre-mm check failed, abort\n");
-
-		order = max_order;
-		total = 0;
-
-		do {
-retry:
-			size = get_size(order, chunk_size);
-			err = drm_buddy_alloc_blocks(&mm, start, mm_size, size, size, &tmp, flags);
-			if (err) {
-				if (err == -ENOMEM) {
-					KUNIT_FAIL(test, "buddy_alloc hit -ENOMEM with order=%d\n",
-						   order);
-				} else {
-					if (order--) {
-						err = 0;
-						goto retry;
-					}
-
-					KUNIT_FAIL(test, "buddy_alloc with order=%d failed\n",
-						   order);
-				}
-
-				break;
-			}
-
-			block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
-			KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
-
-			list_move_tail(&block->link, &blocks);
-			KUNIT_EXPECT_EQ_MSG(test, drm_buddy_block_order(block), order,
-					    "buddy_alloc order mismatch\n");
-
-			total += drm_buddy_block_size(&mm, block);
-
-			if (__timeout(end_time, NULL)) {
-				timeout = true;
-				break;
-			}
-		} while (total < mm.size);
-
-		if (!err)
-			err = check_blocks(test, &mm, &blocks, total, false);
-
-		drm_buddy_free_list(&mm, &blocks);
-
-		if (!err) {
-			KUNIT_EXPECT_FALSE_MSG(test, check_mm(test, &mm),
-					       "post-mm check failed\n");
-		}
-
-		if (err || timeout)
-			break;
-
-		cond_resched();
-	}
-
-	kfree(order);
-	drm_buddy_fini(&mm);
-}
-
 static void drm_test_buddy_alloc_pessimistic(struct kunit *test)
 {
 	u64 mm_size, size, start = 0;
@@ -634,64 +240,6 @@ static void drm_test_buddy_alloc_optimistic(struct kunit *test)
 	drm_buddy_fini(&mm);
 }
 
-static void drm_test_buddy_alloc_range(struct kunit *test)
-{
-	unsigned long flags = DRM_BUDDY_RANGE_ALLOCATION;
-	u64 offset, size, rem, chunk_size, end;
-	unsigned long page_num;
-	struct drm_buddy mm;
-	LIST_HEAD(blocks);
-
-	mm_config(&size, &chunk_size);
-
-	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, size, chunk_size),
-			       "buddy_init failed");
-
-	KUNIT_ASSERT_FALSE_MSG(test, check_mm(test, &mm),
-			       "pre-mm check failed, abort!");
-
-	rem = mm.size;
-	offset = 0;
-
-	for_each_prime_number_from(page_num, 1, ULONG_MAX - 1) {
-		struct drm_buddy_block *block;
-		LIST_HEAD(tmp);
-
-		size = min(page_num * mm.chunk_size, rem);
-		end = offset + size;
-
-		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, offset, end,
-								    size, mm.chunk_size,
-									&tmp, flags),
-				"alloc_range with offset=%llx, size=%llx failed\n", offset, size);
-
-		block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
-		KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_range has no blocks\n");
-
-		KUNIT_ASSERT_EQ_MSG(test, drm_buddy_block_offset(block), offset,
-				    "alloc_range start offset mismatch, found=%llx, expected=%llx\n",
-							drm_buddy_block_offset(block), offset);
-
-		KUNIT_ASSERT_FALSE(test, check_blocks(test, &mm, &tmp, size, true));
-
-		list_splice_tail(&tmp, &blocks);
-
-		offset += size;
-
-		rem -= size;
-		if (!rem)
-			break;
-
-		cond_resched();
-	}
-
-	drm_buddy_free_list(&mm, &blocks);
-
-	KUNIT_EXPECT_FALSE_MSG(test, check_mm(test, &mm), "post-mm check failed\n");
-
-	drm_buddy_fini(&mm);
-}
-
 static void drm_test_buddy_alloc_limit(struct kunit *test)
 {
 	u64 size = U64_MAX, start = 0;
@@ -727,29 +275,16 @@ static void drm_test_buddy_alloc_limit(struct kunit *test)
 	drm_buddy_fini(&mm);
 }
 
-static int drm_buddy_suite_init(struct kunit_suite *suite)
-{
-	while (!random_seed)
-		random_seed = get_random_u32();
-
-	kunit_info(suite, "Testing DRM buddy manager, with random_seed=0x%x\n", random_seed);
-
-	return 0;
-}
-
 static struct kunit_case drm_buddy_tests[] = {
 	KUNIT_CASE(drm_test_buddy_alloc_limit),
-	KUNIT_CASE(drm_test_buddy_alloc_range),
 	KUNIT_CASE(drm_test_buddy_alloc_optimistic),
 	KUNIT_CASE(drm_test_buddy_alloc_pessimistic),
-	KUNIT_CASE(drm_test_buddy_alloc_smoke),
 	KUNIT_CASE(drm_test_buddy_alloc_pathological),
 	{}
 };
 
 static struct kunit_suite drm_buddy_test_suite = {
 	.name = "drm_buddy",
-	.suite_init = drm_buddy_suite_init,
 	.test_cases = drm_buddy_tests,
 };
 
diff --git a/drivers/gpu/drm/tests/drm_dp_mst_helper_test.c b/drivers/gpu/drm/tests/drm_dp_mst_helper_test.c
index 545beea33e8c..d916e548fcb1 100644
--- a/drivers/gpu/drm/tests/drm_dp_mst_helper_test.c
+++ b/drivers/gpu/drm/tests/drm_dp_mst_helper_test.c
@@ -42,13 +42,13 @@ static const struct drm_dp_mst_calc_pbn_mode_test drm_dp_mst_calc_pbn_mode_cases
 		.clock = 332880,
 		.bpp = 24,
 		.dsc = true,
-		.expected = 50
+		.expected = 1191
 	},
 	{
 		.clock = 324540,
 		.bpp = 24,
 		.dsc = true,
-		.expected = 49
+		.expected = 1161
 	},
 };
 
@@ -56,7 +56,7 @@ static void drm_test_dp_mst_calc_pbn_mode(struct kunit *test)
 {
 	const struct drm_dp_mst_calc_pbn_mode_test *params = test->param_value;
 
-	KUNIT_EXPECT_EQ(test, drm_dp_calc_pbn_mode(params->clock, params->bpp, params->dsc),
+	KUNIT_EXPECT_EQ(test, drm_dp_calc_pbn_mode(params->clock, params->bpp << 4),
 			params->expected);
 }
 
@@ -68,6 +68,152 @@ static void dp_mst_calc_pbn_mode_desc(const struct drm_dp_mst_calc_pbn_mode_test
 KUNIT_ARRAY_PARAM(drm_dp_mst_calc_pbn_mode, drm_dp_mst_calc_pbn_mode_cases,
 		  dp_mst_calc_pbn_mode_desc);
 
+struct drm_dp_mst_calc_pbn_div_test {
+	int link_rate;
+	int lane_count;
+	fixed20_12 expected;
+};
+
+#define fp_init(__int, __frac) { \
+	.full = (__int) * (1 << 12) + \
+		(__frac) * (1 << 12) / 100000 \
+}
+
+static const struct drm_dp_mst_calc_pbn_div_test drm_dp_mst_calc_pbn_div_dp1_4_cases[] = {
+	/*
+	 * UHBR rates (DP Standard v2.1 2.7.6.3, specifying the rounded to
+	 *             closest value to 2 decimal places):
+	 * .expected = .link_rate * .lane_count * 0.9671 / 8 / 54 / 100
+	 * DP1.4 rates (DP Standard v2.1 2.6.4.2):
+	 * .expected = .link_rate * .lane_count * 0.8000 / 8 / 54 / 100
+	 *
+	 * truncated to 5 decimal places.
+	 */
+	{
+		.link_rate = 2000000,
+		.lane_count = 4,
+		.expected = fp_init(179,  9259),  /* 179.09259 */
+	},
+	{
+		.link_rate = 2000000,
+		.lane_count = 2,
+		.expected = fp_init(89, 54629),
+	},
+	{
+		.link_rate = 2000000,
+		.lane_count = 1,
+		.expected = fp_init(44, 77314),
+	},
+	{
+		.link_rate = 1350000,
+		.lane_count = 4,
+		.expected = fp_init(120, 88750),
+	},
+	{
+		.link_rate = 1350000,
+		.lane_count = 2,
+		.expected = fp_init(60, 44375),
+	},
+	{
+		.link_rate = 1350000,
+		.lane_count = 1,
+		.expected = fp_init(30, 22187),
+	},
+	{
+		.link_rate = 1000000,
+		.lane_count = 4,
+		.expected = fp_init(89, 54629),
+	},
+	{
+		.link_rate = 1000000,
+		.lane_count = 2,
+		.expected = fp_init(44, 77314),
+	},
+	{
+		.link_rate = 1000000,
+		.lane_count = 1,
+		.expected = fp_init(22, 38657),
+	},
+	{
+		.link_rate = 810000,
+		.lane_count = 4,
+		.expected = fp_init(60, 0),
+	},
+	{
+		.link_rate = 810000,
+		.lane_count = 2,
+		.expected = fp_init(30, 0),
+	},
+	{
+		.link_rate = 810000,
+		.lane_count = 1,
+		.expected = fp_init(15, 0),
+	},
+	{
+		.link_rate = 540000,
+		.lane_count = 4,
+		.expected = fp_init(40, 0),
+	},
+	{
+		.link_rate = 540000,
+		.lane_count = 2,
+		.expected = fp_init(20, 0),
+	},
+	{
+		.link_rate = 540000,
+		.lane_count = 1,
+		.expected = fp_init(10, 0),
+	},
+	{
+		.link_rate = 270000,
+		.lane_count = 4,
+		.expected = fp_init(20, 0),
+	},
+	{
+		.link_rate = 270000,
+		.lane_count = 2,
+		.expected = fp_init(10, 0),
+	},
+	{
+		.link_rate = 270000,
+		.lane_count = 1,
+		.expected = fp_init(5, 0),
+	},
+	{
+		.link_rate = 162000,
+		.lane_count = 4,
+		.expected = fp_init(12, 0),
+	},
+	{
+		.link_rate = 162000,
+		.lane_count = 2,
+		.expected = fp_init(6, 0),
+	},
+	{
+		.link_rate = 162000,
+		.lane_count = 1,
+		.expected = fp_init(3, 0),
+	},
+};
+
+static void drm_test_dp_mst_calc_pbn_div(struct kunit *test)
+{
+	const struct drm_dp_mst_calc_pbn_div_test *params = test->param_value;
+	/* mgr->dev is only needed by drm_dbg_kms(), but it's not called for the test cases. */
+	struct drm_dp_mst_topology_mgr *mgr = test->priv;
+
+	KUNIT_EXPECT_EQ(test, drm_dp_get_vc_payload_bw(mgr, params->link_rate, params->lane_count).full,
+			params->expected.full);
+}
+
+static void dp_mst_calc_pbn_div_desc(const struct drm_dp_mst_calc_pbn_div_test *t, char *desc)
+{
+	sprintf(desc, "Link rate %d lane count %d", t->link_rate, t->lane_count);
+}
+
+KUNIT_ARRAY_PARAM(drm_dp_mst_calc_pbn_div, drm_dp_mst_calc_pbn_div_dp1_4_cases,
+		  dp_mst_calc_pbn_div_desc);
+
 static u8 data[] = { 0xff, 0x00, 0xdd };
 
 struct drm_dp_mst_sideband_msg_req_test {
@@ -416,13 +562,27 @@ KUNIT_ARRAY_PARAM(drm_dp_mst_sideband_msg_req, drm_dp_mst_sideband_msg_req_cases
 
 static struct kunit_case drm_dp_mst_helper_tests[] = {
 	KUNIT_CASE_PARAM(drm_test_dp_mst_calc_pbn_mode, drm_dp_mst_calc_pbn_mode_gen_params),
+	KUNIT_CASE_PARAM(drm_test_dp_mst_calc_pbn_div, drm_dp_mst_calc_pbn_div_gen_params),
 	KUNIT_CASE_PARAM(drm_test_dp_mst_sideband_msg_req_decode,
 			 drm_dp_mst_sideband_msg_req_gen_params),
 	{ }
 };
 
+static int drm_dp_mst_helper_tests_init(struct kunit *test)
+{
+	struct drm_dp_mst_topology_mgr *mgr;
+
+	mgr = kunit_kzalloc(test, sizeof(*mgr), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, mgr);
+
+	test->priv = mgr;
+
+	return 0;
+}
+
 static struct kunit_suite drm_dp_mst_helper_test_suite = {
 	.name = "drm_dp_mst_helper",
+	.init = drm_dp_mst_helper_tests_init,
 	.test_cases = drm_dp_mst_helper_tests,
 };
 
diff --git a/drivers/gpu/drm/tests/drm_format_helper_test.c b/drivers/gpu/drm/tests/drm_format_helper_test.c
index f6408e56f786..08992636ec05 100644
--- a/drivers/gpu/drm/tests/drm_format_helper_test.c
+++ b/drivers/gpu/drm/tests/drm_format_helper_test.c
@@ -20,6 +20,10 @@
 
 #define TEST_USE_DEFAULT_PITCH 0
 
+static unsigned char fmtcnv_state_mem[PAGE_SIZE];
+static struct drm_format_conv_state fmtcnv_state =
+	DRM_FORMAT_CONV_STATE_INIT_PREALLOCATED(fmtcnv_state_mem, sizeof(fmtcnv_state_mem));
+
 struct convert_to_gray8_result {
 	unsigned int dst_pitch;
 	const u8 expected[TEST_BUF_SIZE];
@@ -630,8 +634,7 @@ static void drm_test_fb_xrgb8888_to_gray8(struct kunit *test)
 	const unsigned int *dst_pitch = (result->dst_pitch == TEST_USE_DEFAULT_PITCH) ?
 		NULL : &result->dst_pitch;
 
-	drm_fb_xrgb8888_to_gray8(&dst, dst_pitch, &src, &fb, &params->clip);
-
+	drm_fb_xrgb8888_to_gray8(&dst, dst_pitch, &src, &fb, &params->clip, &fmtcnv_state);
 	KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size);
 }
 
@@ -664,7 +667,7 @@ static void drm_test_fb_xrgb8888_to_rgb332(struct kunit *test)
 	const unsigned int *dst_pitch = (result->dst_pitch == TEST_USE_DEFAULT_PITCH) ?
 		NULL : &result->dst_pitch;
 
-	drm_fb_xrgb8888_to_rgb332(&dst, dst_pitch, &src, &fb, &params->clip);
+	drm_fb_xrgb8888_to_rgb332(&dst, dst_pitch, &src, &fb, &params->clip, &fmtcnv_state);
 	KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size);
 }
 
@@ -697,12 +700,14 @@ static void drm_test_fb_xrgb8888_to_rgb565(struct kunit *test)
 	const unsigned int *dst_pitch = (result->dst_pitch == TEST_USE_DEFAULT_PITCH) ?
 		NULL : &result->dst_pitch;
 
-	drm_fb_xrgb8888_to_rgb565(&dst, dst_pitch, &src, &fb, &params->clip, false);
+	drm_fb_xrgb8888_to_rgb565(&dst, dst_pitch, &src, &fb, &params->clip,
+				  &fmtcnv_state, false);
 	buf = le16buf_to_cpu(test, (__force const __le16 *)buf, dst_size / sizeof(__le16));
 	KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size);
 
 	buf = dst.vaddr; /* restore original value of buf */
-	drm_fb_xrgb8888_to_rgb565(&dst, &result->dst_pitch, &src, &fb, &params->clip, true);
+	drm_fb_xrgb8888_to_rgb565(&dst, &result->dst_pitch, &src, &fb, &params->clip,
+				  &fmtcnv_state, true);
 	buf = le16buf_to_cpu(test, (__force const __le16 *)buf, dst_size / sizeof(__le16));
 	KUNIT_EXPECT_MEMEQ(test, buf, result->expected_swab, dst_size);
 
@@ -711,7 +716,8 @@ static void drm_test_fb_xrgb8888_to_rgb565(struct kunit *test)
 
 	int blit_result = 0;
 
-	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_RGB565, &src, &fb, &params->clip);
+	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_RGB565, &src, &fb, &params->clip,
+				  &fmtcnv_state);
 
 	buf = le16buf_to_cpu(test, (__force const __le16 *)buf, dst_size / sizeof(__le16));
 
@@ -748,7 +754,7 @@ static void drm_test_fb_xrgb8888_to_xrgb1555(struct kunit *test)
 	const unsigned int *dst_pitch = (result->dst_pitch == TEST_USE_DEFAULT_PITCH) ?
 		NULL : &result->dst_pitch;
 
-	drm_fb_xrgb8888_to_xrgb1555(&dst, dst_pitch, &src, &fb, &params->clip);
+	drm_fb_xrgb8888_to_xrgb1555(&dst, dst_pitch, &src, &fb, &params->clip, &fmtcnv_state);
 	buf = le16buf_to_cpu(test, (__force const __le16 *)buf, dst_size / sizeof(__le16));
 	KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size);
 
@@ -757,7 +763,8 @@ static void drm_test_fb_xrgb8888_to_xrgb1555(struct kunit *test)
 
 	int blit_result = 0;
 
-	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_XRGB1555, &src, &fb, &params->clip);
+	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_XRGB1555, &src, &fb, &params->clip,
+				  &fmtcnv_state);
 
 	buf = le16buf_to_cpu(test, (__force const __le16 *)buf, dst_size / sizeof(__le16));
 
@@ -794,7 +801,7 @@ static void drm_test_fb_xrgb8888_to_argb1555(struct kunit *test)
 	const unsigned int *dst_pitch = (result->dst_pitch == TEST_USE_DEFAULT_PITCH) ?
 		NULL : &result->dst_pitch;
 
-	drm_fb_xrgb8888_to_argb1555(&dst, dst_pitch, &src, &fb, &params->clip);
+	drm_fb_xrgb8888_to_argb1555(&dst, dst_pitch, &src, &fb, &params->clip, &fmtcnv_state);
 	buf = le16buf_to_cpu(test, (__force const __le16 *)buf, dst_size / sizeof(__le16));
 	KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size);
 
@@ -803,7 +810,8 @@ static void drm_test_fb_xrgb8888_to_argb1555(struct kunit *test)
 
 	int blit_result = 0;
 
-	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_ARGB1555, &src, &fb, &params->clip);
+	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_ARGB1555, &src, &fb, &params->clip,
+				  &fmtcnv_state);
 
 	buf = le16buf_to_cpu(test, (__force const __le16 *)buf, dst_size / sizeof(__le16));
 
@@ -840,7 +848,7 @@ static void drm_test_fb_xrgb8888_to_rgba5551(struct kunit *test)
 	const unsigned int *dst_pitch = (result->dst_pitch == TEST_USE_DEFAULT_PITCH) ?
 		NULL : &result->dst_pitch;
 
-	drm_fb_xrgb8888_to_rgba5551(&dst, dst_pitch, &src, &fb, &params->clip);
+	drm_fb_xrgb8888_to_rgba5551(&dst, dst_pitch, &src, &fb, &params->clip, &fmtcnv_state);
 	buf = le16buf_to_cpu(test, (__force const __le16 *)buf, dst_size / sizeof(__le16));
 	KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size);
 
@@ -849,7 +857,8 @@ static void drm_test_fb_xrgb8888_to_rgba5551(struct kunit *test)
 
 	int blit_result = 0;
 
-	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_RGBA5551, &src, &fb, &params->clip);
+	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_RGBA5551, &src, &fb, &params->clip,
+				  &fmtcnv_state);
 
 	buf = le16buf_to_cpu(test, (__force const __le16 *)buf, dst_size / sizeof(__le16));
 
@@ -890,7 +899,7 @@ static void drm_test_fb_xrgb8888_to_rgb888(struct kunit *test)
 	const unsigned int *dst_pitch = (result->dst_pitch == TEST_USE_DEFAULT_PITCH) ?
 		NULL : &result->dst_pitch;
 
-	drm_fb_xrgb8888_to_rgb888(&dst, dst_pitch, &src, &fb, &params->clip);
+	drm_fb_xrgb8888_to_rgb888(&dst, dst_pitch, &src, &fb, &params->clip, &fmtcnv_state);
 	KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size);
 
 	buf = dst.vaddr; /* restore original value of buf */
@@ -898,7 +907,8 @@ static void drm_test_fb_xrgb8888_to_rgb888(struct kunit *test)
 
 	int blit_result = 0;
 
-	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_RGB888, &src, &fb, &params->clip);
+	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_RGB888, &src, &fb, &params->clip,
+				  &fmtcnv_state);
 
 	KUNIT_EXPECT_FALSE(test, blit_result);
 	KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size);
@@ -933,7 +943,7 @@ static void drm_test_fb_xrgb8888_to_argb8888(struct kunit *test)
 	const unsigned int *dst_pitch = (result->dst_pitch == TEST_USE_DEFAULT_PITCH) ?
 		NULL : &result->dst_pitch;
 
-	drm_fb_xrgb8888_to_argb8888(&dst, dst_pitch, &src, &fb, &params->clip);
+	drm_fb_xrgb8888_to_argb8888(&dst, dst_pitch, &src, &fb, &params->clip, &fmtcnv_state);
 	buf = le32buf_to_cpu(test, (__force const __le32 *)buf, dst_size / sizeof(u32));
 	KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size);
 
@@ -942,7 +952,8 @@ static void drm_test_fb_xrgb8888_to_argb8888(struct kunit *test)
 
 	int blit_result = 0;
 
-	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_ARGB8888, &src, &fb, &params->clip);
+	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_ARGB8888, &src, &fb, &params->clip,
+				  &fmtcnv_state);
 
 	buf = le32buf_to_cpu(test, (__force const __le32 *)buf, dst_size / sizeof(u32));
 
@@ -979,7 +990,7 @@ static void drm_test_fb_xrgb8888_to_xrgb2101010(struct kunit *test)
 	const unsigned int *dst_pitch = (result->dst_pitch == TEST_USE_DEFAULT_PITCH) ?
 		NULL : &result->dst_pitch;
 
-	drm_fb_xrgb8888_to_xrgb2101010(&dst, dst_pitch, &src, &fb, &params->clip);
+	drm_fb_xrgb8888_to_xrgb2101010(&dst, dst_pitch, &src, &fb, &params->clip, &fmtcnv_state);
 	buf = le32buf_to_cpu(test, buf, dst_size / sizeof(u32));
 	KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size);
 
@@ -989,7 +1000,7 @@ static void drm_test_fb_xrgb8888_to_xrgb2101010(struct kunit *test)
 	int blit_result = 0;
 
 	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_XRGB2101010, &src, &fb,
-				  &params->clip);
+				  &params->clip, &fmtcnv_state);
 
 	KUNIT_EXPECT_FALSE(test, blit_result);
 	KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size);
@@ -1024,7 +1035,7 @@ static void drm_test_fb_xrgb8888_to_argb2101010(struct kunit *test)
 	const unsigned int *dst_pitch = (result->dst_pitch == TEST_USE_DEFAULT_PITCH) ?
 		NULL : &result->dst_pitch;
 
-	drm_fb_xrgb8888_to_argb2101010(&dst, dst_pitch, &src, &fb, &params->clip);
+	drm_fb_xrgb8888_to_argb2101010(&dst, dst_pitch, &src, &fb, &params->clip, &fmtcnv_state);
 	buf = le32buf_to_cpu(test, (__force const __le32 *)buf, dst_size / sizeof(u32));
 	KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size);
 
@@ -1034,7 +1045,7 @@ static void drm_test_fb_xrgb8888_to_argb2101010(struct kunit *test)
 	int blit_result = 0;
 
 	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_ARGB2101010, &src, &fb,
-				  &params->clip);
+				  &params->clip, &fmtcnv_state);
 
 	buf = le32buf_to_cpu(test, (__force const __le32 *)buf, dst_size / sizeof(u32));
 
@@ -1071,7 +1082,7 @@ static void drm_test_fb_xrgb8888_to_mono(struct kunit *test)
 	const unsigned int *dst_pitch = (result->dst_pitch == TEST_USE_DEFAULT_PITCH) ?
 		NULL : &result->dst_pitch;
 
-	drm_fb_xrgb8888_to_mono(&dst, dst_pitch, &src, &fb, &params->clip);
+	drm_fb_xrgb8888_to_mono(&dst, dst_pitch, &src, &fb, &params->clip, &fmtcnv_state);
 	KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size);
 }
 
@@ -1104,7 +1115,7 @@ static void drm_test_fb_swab(struct kunit *test)
 	const unsigned int *dst_pitch = (result->dst_pitch == TEST_USE_DEFAULT_PITCH) ?
 		NULL : &result->dst_pitch;
 
-	drm_fb_swab(&dst, dst_pitch, &src, &fb, &params->clip, false);
+	drm_fb_swab(&dst, dst_pitch, &src, &fb, &params->clip, false, &fmtcnv_state);
 	buf = le32buf_to_cpu(test, (__force const __le32 *)buf, dst_size / sizeof(u32));
 	KUNIT_EXPECT_MEMEQ(test, buf, result->expected, dst_size);
 
@@ -1114,7 +1125,7 @@ static void drm_test_fb_swab(struct kunit *test)
 	int blit_result;
 
 	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_XRGB8888 | DRM_FORMAT_BIG_ENDIAN,
-				  &src, &fb, &params->clip);
+				  &src, &fb, &params->clip, &fmtcnv_state);
 	buf = le32buf_to_cpu(test, (__force const __le32 *)buf, dst_size / sizeof(u32));
 
 	KUNIT_EXPECT_FALSE(test, blit_result);
@@ -1123,7 +1134,8 @@ static void drm_test_fb_swab(struct kunit *test)
 	buf = dst.vaddr;
 	memset(buf, 0, dst_size);
 
-	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_BGRX8888, &src, &fb, &params->clip);
+	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_BGRX8888, &src, &fb, &params->clip,
+				  &fmtcnv_state);
 	buf = le32buf_to_cpu(test, (__force const __le32 *)buf, dst_size / sizeof(u32));
 
 	KUNIT_EXPECT_FALSE(test, blit_result);
@@ -1137,7 +1149,8 @@ static void drm_test_fb_swab(struct kunit *test)
 	mock_format.format |= DRM_FORMAT_BIG_ENDIAN;
 	fb.format = &mock_format;
 
-	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_XRGB8888, &src, &fb, &params->clip);
+	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_XRGB8888, &src, &fb, &params->clip,
+				  &fmtcnv_state);
 	buf = le32buf_to_cpu(test, (__force const __le32 *)buf, dst_size / sizeof(u32));
 
 	KUNIT_EXPECT_FALSE(test, blit_result);
@@ -1175,7 +1188,8 @@ static void drm_test_fb_xrgb8888_to_abgr8888(struct kunit *test)
 
 	int blit_result = 0;
 
-	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_ABGR8888, &src, &fb, &params->clip);
+	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_ABGR8888, &src, &fb, &params->clip,
+				  &fmtcnv_state);
 
 	buf = le32buf_to_cpu(test, (__force const __le32 *)buf, dst_size / sizeof(u32));
 
@@ -1214,7 +1228,8 @@ static void drm_test_fb_xrgb8888_to_xbgr8888(struct kunit *test)
 
 	int blit_result = 0;
 
-	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_XBGR8888, &src, &fb, &params->clip);
+	blit_result = drm_fb_blit(&dst, dst_pitch, DRM_FORMAT_XBGR8888, &src, &fb, &params->clip,
+				  &fmtcnv_state);
 
 	buf = le32buf_to_cpu(test, (__force const __le32 *)buf, dst_size / sizeof(u32));
 
@@ -1817,7 +1832,8 @@ static void drm_test_fb_memcpy(struct kunit *test)
 
 	int blit_result;
 
-	blit_result = drm_fb_blit(dst, dst_pitches, params->format, src, &fb, &params->clip);
+	blit_result = drm_fb_blit(dst, dst_pitches, params->format, src, &fb, &params->clip,
+				  &fmtcnv_state);
 
 	KUNIT_EXPECT_FALSE(test, blit_result);
 	for (size_t i = 0; i < fb.format->num_planes; i++) {
diff --git a/drivers/gpu/drm/tests/drm_mm_test.c b/drivers/gpu/drm/tests/drm_mm_test.c
index 05d5e7af6d25..4e9247cf9977 100644
--- a/drivers/gpu/drm/tests/drm_mm_test.c
+++ b/drivers/gpu/drm/tests/drm_mm_test.c
@@ -17,10 +17,6 @@
 
 #include "../lib/drm_random.h"
 
-static unsigned int random_seed;
-static unsigned int max_iterations = 8192;
-static unsigned int max_prime = 128;
-
 enum {
 	BEST,
 	BOTTOMUP,
@@ -37,10 +33,6 @@ static const struct insert_mode {
 	[TOPDOWN] = { "top-down", DRM_MM_INSERT_HIGH },
 	[EVICT] = { "evict", DRM_MM_INSERT_EVICT },
 	{}
-}, evict_modes[] = {
-	{ "bottom-up", DRM_MM_INSERT_LOW },
-	{ "top-down", DRM_MM_INSERT_HIGH },
-	{}
 };
 
 static bool assert_no_holes(struct kunit *test, const struct drm_mm *mm)
@@ -97,57 +89,6 @@ static bool assert_one_hole(struct kunit *test, const struct drm_mm *mm, u64 sta
 	return ok;
 }
 
-static bool assert_continuous(struct kunit *test, const struct drm_mm *mm, u64 size)
-{
-	struct drm_mm_node *node, *check, *found;
-	unsigned long n;
-	u64 addr;
-
-	if (!assert_no_holes(test, mm))
-		return false;
-
-	n = 0;
-	addr = 0;
-	drm_mm_for_each_node(node, mm) {
-		if (node->start != addr) {
-			KUNIT_FAIL(test, "node[%ld] list out of order, expected %llx found %llx\n",
-				   n, addr, node->start);
-			return false;
-		}
-
-		if (node->size != size) {
-			KUNIT_FAIL(test, "node[%ld].size incorrect, expected %llx, found %llx\n",
-				   n, size, node->size);
-			return false;
-		}
-
-		if (drm_mm_hole_follows(node)) {
-			KUNIT_FAIL(test, "node[%ld] is followed by a hole!\n", n);
-			return false;
-		}
-
-		found = NULL;
-		drm_mm_for_each_node_in_range(check, mm, addr, addr + size) {
-			if (node != check) {
-				KUNIT_FAIL(test,
-					   "lookup return wrong node, expected start %llx, found %llx\n",
-					   node->start, check->start);
-				return false;
-			}
-			found = check;
-		}
-		if (!found) {
-			KUNIT_FAIL(test, "lookup failed for node %llx + %llx\n", addr, size);
-			return false;
-		}
-
-		addr += size;
-		n++;
-	}
-
-	return true;
-}
-
 static u64 misalignment(struct drm_mm_node *node, u64 alignment)
 {
 	u64 rem;
@@ -270,215 +211,6 @@ static void drm_test_mm_debug(struct kunit *test)
 			       nodes[0].start, nodes[0].size);
 }
 
-static struct drm_mm_node *set_node(struct drm_mm_node *node,
-				    u64 start, u64 size)
-{
-	node->start = start;
-	node->size = size;
-	return node;
-}
-
-static bool expect_reserve_fail(struct kunit *test, struct drm_mm *mm, struct drm_mm_node *node)
-{
-	int err;
-
-	err = drm_mm_reserve_node(mm, node);
-	if (likely(err == -ENOSPC))
-		return true;
-
-	if (!err) {
-		KUNIT_FAIL(test, "impossible reserve succeeded, node %llu + %llu\n",
-			   node->start, node->size);
-		drm_mm_remove_node(node);
-	} else {
-		KUNIT_FAIL(test,
-			   "impossible reserve failed with wrong error %d [expected %d], node %llu + %llu\n",
-		       err, -ENOSPC, node->start, node->size);
-	}
-	return false;
-}
-
-static bool noinline_for_stack check_reserve_boundaries(struct kunit *test, struct drm_mm *mm,
-							unsigned int count,
-							u64 size)
-{
-	const struct boundary {
-		u64 start, size;
-		const char *name;
-	} boundaries[] = {
-#define B(st, sz) { (st), (sz), "{ " #st ", " #sz "}" }
-		B(0, 0),
-		B(-size, 0),
-		B(size, 0),
-		B(size * count, 0),
-		B(-size, size),
-		B(-size, -size),
-		B(-size, 2 * size),
-		B(0, -size),
-		B(size, -size),
-		B(count * size, size),
-		B(count * size, -size),
-		B(count * size, count * size),
-		B(count * size, -count * size),
-		B(count * size, -(count + 1) * size),
-		B((count + 1) * size, size),
-		B((count + 1) * size, -size),
-		B((count + 1) * size, -2 * size),
-#undef B
-	};
-	struct drm_mm_node tmp = {};
-	int n;
-
-	for (n = 0; n < ARRAY_SIZE(boundaries); n++) {
-		if (!expect_reserve_fail(test, mm, set_node(&tmp, boundaries[n].start,
-							    boundaries[n].size))) {
-			KUNIT_FAIL(test, "boundary[%d:%s] failed, count=%u, size=%lld\n",
-				   n, boundaries[n].name, count, size);
-			return false;
-		}
-	}
-
-	return true;
-}
-
-static int __drm_test_mm_reserve(struct kunit *test, unsigned int count, u64 size)
-{
-	DRM_RND_STATE(prng, random_seed);
-	struct drm_mm mm;
-	struct drm_mm_node tmp, *nodes, *node, *next;
-	unsigned int *order, n, m, o = 0;
-	int ret, err;
-
-	/* For exercising drm_mm_reserve_node(), we want to check that
-	 * reservations outside of the drm_mm range are rejected, and to
-	 * overlapping and otherwise already occupied ranges. Afterwards,
-	 * the tree and nodes should be intact.
-	 */
-
-	DRM_MM_BUG_ON(!count);
-	DRM_MM_BUG_ON(!size);
-
-	ret = -ENOMEM;
-	order = drm_random_order(count, &prng);
-	if (!order)
-		goto err;
-
-	nodes = vzalloc(array_size(count, sizeof(*nodes)));
-	KUNIT_ASSERT_TRUE(test, nodes);
-
-	ret = -EINVAL;
-	drm_mm_init(&mm, 0, count * size);
-
-	if (!check_reserve_boundaries(test, &mm, count, size))
-		goto out;
-
-	for (n = 0; n < count; n++) {
-		nodes[n].start = order[n] * size;
-		nodes[n].size = size;
-
-		err = drm_mm_reserve_node(&mm, &nodes[n]);
-		if (err) {
-			KUNIT_FAIL(test, "reserve failed, step %d, start %llu\n",
-				   n, nodes[n].start);
-			ret = err;
-			goto out;
-		}
-
-		if (!drm_mm_node_allocated(&nodes[n])) {
-			KUNIT_FAIL(test, "reserved node not allocated! step %d, start %llu\n",
-				   n, nodes[n].start);
-			goto out;
-		}
-
-		if (!expect_reserve_fail(test, &mm, &nodes[n]))
-			goto out;
-	}
-
-	/* After random insertion the nodes should be in order */
-	if (!assert_continuous(test, &mm, size))
-		goto out;
-
-	/* Repeated use should then fail */
-	drm_random_reorder(order, count, &prng);
-	for (n = 0; n < count; n++) {
-		if (!expect_reserve_fail(test, &mm, set_node(&tmp, order[n] * size, 1)))
-			goto out;
-
-		/* Remove and reinsert should work */
-		drm_mm_remove_node(&nodes[order[n]]);
-		err = drm_mm_reserve_node(&mm, &nodes[order[n]]);
-		if (err) {
-			KUNIT_FAIL(test, "reserve failed, step %d, start %llu\n",
-				   n, nodes[n].start);
-			ret = err;
-			goto out;
-		}
-	}
-
-	if (!assert_continuous(test, &mm, size))
-		goto out;
-
-	/* Overlapping use should then fail */
-	for (n = 0; n < count; n++) {
-		if (!expect_reserve_fail(test, &mm, set_node(&tmp, 0, size * count)))
-			goto out;
-	}
-	for (n = 0; n < count; n++) {
-		if (!expect_reserve_fail(test, &mm, set_node(&tmp, size * n, size * (count - n))))
-			goto out;
-	}
-
-	/* Remove several, reinsert, check full */
-	for_each_prime_number(n, min(max_prime, count)) {
-		for (m = 0; m < n; m++) {
-			node = &nodes[order[(o + m) % count]];
-			drm_mm_remove_node(node);
-		}
-
-		for (m = 0; m < n; m++) {
-			node = &nodes[order[(o + m) % count]];
-			err = drm_mm_reserve_node(&mm, node);
-			if (err) {
-				KUNIT_FAIL(test, "reserve failed, step %d/%d, start %llu\n",
-					   m, n, node->start);
-				ret = err;
-				goto out;
-			}
-		}
-
-		o += n;
-
-		if (!assert_continuous(test, &mm, size))
-			goto out;
-	}
-
-	ret = 0;
-out:
-	drm_mm_for_each_node_safe(node, next, &mm)
-		drm_mm_remove_node(node);
-	drm_mm_takedown(&mm);
-	vfree(nodes);
-	kfree(order);
-err:
-	return ret;
-}
-
-static void drm_test_mm_reserve(struct kunit *test)
-{
-	const unsigned int count = min_t(unsigned int, BIT(10), max_iterations);
-	int n;
-
-	for_each_prime_number_from(n, 1, 54) {
-		u64 size = BIT_ULL(n);
-
-		KUNIT_ASSERT_FALSE(test, __drm_test_mm_reserve(test, count, size - 1));
-		KUNIT_ASSERT_FALSE(test, __drm_test_mm_reserve(test, count, size));
-		KUNIT_ASSERT_FALSE(test, __drm_test_mm_reserve(test, count, size + 1));
-
-		cond_resched();
-	}
-}
-
 static bool expect_insert(struct kunit *test, struct drm_mm *mm,
 			  struct drm_mm_node *node, u64 size, u64 alignment, unsigned long color,
 			const struct insert_mode *mode)
@@ -503,1754 +235,118 @@ static bool expect_insert(struct kunit *test, struct drm_mm *mm,
 	return true;
 }
 
-static bool expect_insert_fail(struct kunit *test, struct drm_mm *mm, u64 size)
-{
-	struct drm_mm_node tmp = {};
-	int err;
-
-	err = drm_mm_insert_node(mm, &tmp, size);
-	if (likely(err == -ENOSPC))
-		return true;
-
-	if (!err) {
-		KUNIT_FAIL(test, "impossible insert succeeded, node %llu + %llu\n",
-			   tmp.start, tmp.size);
-		drm_mm_remove_node(&tmp);
-	} else {
-		KUNIT_FAIL(test,
-			   "impossible insert failed with wrong error %d [expected %d], size %llu\n",
-			   err, -ENOSPC, size);
-	}
-	return false;
-}
-
-static int __drm_test_mm_insert(struct kunit *test, unsigned int count, u64 size, bool replace)
+static void drm_test_mm_align_pot(struct kunit *test, int max)
 {
-	DRM_RND_STATE(prng, random_seed);
-	const struct insert_mode *mode;
 	struct drm_mm mm;
-	struct drm_mm_node *nodes, *node, *next;
-	unsigned int *order, n, m, o = 0;
-	int ret;
-
-	/* Fill a range with lots of nodes, check it doesn't fail too early */
-
-	DRM_MM_BUG_ON(!count);
-	DRM_MM_BUG_ON(!size);
-
-	ret = -ENOMEM;
-	nodes = vmalloc(array_size(count, sizeof(*nodes)));
-	KUNIT_ASSERT_TRUE(test, nodes);
-
-	order = drm_random_order(count, &prng);
-	if (!order)
-		goto err_nodes;
-
-	ret = -EINVAL;
-	drm_mm_init(&mm, 0, count * size);
-
-	for (mode = insert_modes; mode->name; mode++) {
-		for (n = 0; n < count; n++) {
-			struct drm_mm_node tmp;
-
-			node = replace ? &tmp : &nodes[n];
-			memset(node, 0, sizeof(*node));
-			if (!expect_insert(test, &mm, node, size, 0, n, mode)) {
-				KUNIT_FAIL(test, "%s insert failed, size %llu step %d\n",
-					   mode->name, size, n);
-				goto out;
-			}
-
-			if (replace) {
-				drm_mm_replace_node(&tmp, &nodes[n]);
-				if (drm_mm_node_allocated(&tmp)) {
-					KUNIT_FAIL(test,
-						   "replaced old-node still allocated! step %d\n",
-						   n);
-					goto out;
-				}
-
-				if (!assert_node(test, &nodes[n], &mm, size, 0, n)) {
-					KUNIT_FAIL(test,
-						   "replaced node did not inherit parameters, size %llu step %d\n",
-						   size, n);
-					goto out;
-				}
-
-				if (tmp.start != nodes[n].start) {
-					KUNIT_FAIL(test,
-						   "replaced node mismatch location expected [%llx + %llx], found [%llx + %llx]\n",
-						   tmp.start, size, nodes[n].start, nodes[n].size);
-					goto out;
-				}
-			}
-		}
-
-		/* After random insertion the nodes should be in order */
-		if (!assert_continuous(test, &mm, size))
-			goto out;
-
-		/* Repeated use should then fail */
-		if (!expect_insert_fail(test, &mm, size))
-			goto out;
+	struct drm_mm_node *node, *next;
+	int bit;
 
-		/* Remove one and reinsert, as the only hole it should refill itself */
-		for (n = 0; n < count; n++) {
-			u64 addr = nodes[n].start;
+	/* Check that we can align to the full u64 address space */
 
-			drm_mm_remove_node(&nodes[n]);
-			if (!expect_insert(test, &mm, &nodes[n], size, 0, n, mode)) {
-				KUNIT_FAIL(test, "%s reinsert failed, size %llu step %d\n",
-					   mode->name, size, n);
-				goto out;
-			}
+	drm_mm_init(&mm, 1, U64_MAX - 2);
 
-			if (nodes[n].start != addr) {
-				KUNIT_FAIL(test,
-					   "%s reinsert node moved, step %d, expected %llx, found %llx\n",
-					   mode->name, n, addr, nodes[n].start);
-				goto out;
-			}
+	for (bit = max - 1; bit; bit--) {
+		u64 align, size;
 
-			if (!assert_continuous(test, &mm, size))
-				goto out;
+		node = kzalloc(sizeof(*node), GFP_KERNEL);
+		if (!node) {
+			KUNIT_FAIL(test, "failed to allocate node");
+			goto out;
 		}
 
-		/* Remove several, reinsert, check full */
-		for_each_prime_number(n, min(max_prime, count)) {
-			for (m = 0; m < n; m++) {
-				node = &nodes[order[(o + m) % count]];
-				drm_mm_remove_node(node);
-			}
-
-			for (m = 0; m < n; m++) {
-				node = &nodes[order[(o + m) % count]];
-				if (!expect_insert(test, &mm, node, size, 0, n, mode)) {
-					KUNIT_FAIL(test,
-						   "%s multiple reinsert failed, size %llu step %d\n",
-							   mode->name, size, n);
-					goto out;
-				}
-			}
-
-			o += n;
-
-			if (!assert_continuous(test, &mm, size))
-				goto out;
-
-			if (!expect_insert_fail(test, &mm, size))
-				goto out;
+		align = BIT_ULL(bit);
+		size = BIT_ULL(bit - 1) + 1;
+		if (!expect_insert(test, &mm, node, size, align, bit, &insert_modes[0])) {
+			KUNIT_FAIL(test, "insert failed with alignment=%llx [%d]", align, bit);
+			goto out;
 		}
 
-		drm_mm_for_each_node_safe(node, next, &mm)
-			drm_mm_remove_node(node);
-		DRM_MM_BUG_ON(!drm_mm_clean(&mm));
-
 		cond_resched();
 	}
 
-	ret = 0;
 out:
-	drm_mm_for_each_node_safe(node, next, &mm)
+	drm_mm_for_each_node_safe(node, next, &mm) {
 		drm_mm_remove_node(node);
-	drm_mm_takedown(&mm);
-	kfree(order);
-err_nodes:
-	vfree(nodes);
-	return ret;
-}
-
-static void drm_test_mm_insert(struct kunit *test)
-{
-	const unsigned int count = min_t(unsigned int, BIT(10), max_iterations);
-	unsigned int n;
-
-	for_each_prime_number_from(n, 1, 54) {
-		u64 size = BIT_ULL(n);
-
-		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert(test, count, size - 1, false));
-		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert(test, count, size, false));
-		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert(test, count, size + 1, false));
-
-		cond_resched();
+		kfree(node);
 	}
+	drm_mm_takedown(&mm);
 }
 
-static void drm_test_mm_replace(struct kunit *test)
+static void drm_test_mm_align32(struct kunit *test)
 {
-	const unsigned int count = min_t(unsigned int, BIT(10), max_iterations);
-	unsigned int n;
-
-	/* Reuse __drm_test_mm_insert to exercise replacement by inserting a dummy node,
-	 * then replacing it with the intended node. We want to check that
-	 * the tree is intact and all the information we need is carried
-	 * across to the target node.
-	 */
-
-	for_each_prime_number_from(n, 1, 54) {
-		u64 size = BIT_ULL(n);
-
-		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert(test, count, size - 1, true));
-		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert(test, count, size, true));
-		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert(test, count, size + 1, true));
-
-		cond_resched();
-	}
+	drm_test_mm_align_pot(test, 32);
 }
 
-static bool expect_insert_in_range(struct kunit *test, struct drm_mm *mm, struct drm_mm_node *node,
-				   u64 size, u64 alignment, unsigned long color,
-				   u64 range_start, u64 range_end, const struct insert_mode *mode)
+static void drm_test_mm_align64(struct kunit *test)
 {
-	int err;
-
-	err = drm_mm_insert_node_in_range(mm, node,
-					  size, alignment, color,
-					  range_start, range_end,
-					  mode->mode);
-	if (err) {
-		KUNIT_FAIL(test,
-			   "insert (size=%llu, alignment=%llu, color=%lu, mode=%s) nto range [%llx, %llx] failed with err=%d\n",
-				   size, alignment, color, mode->name,
-				   range_start, range_end, err);
-		return false;
-	}
-
-	if (!assert_node(test, node, mm, size, alignment, color)) {
-		drm_mm_remove_node(node);
-		return false;
-	}
-
-	return true;
+	drm_test_mm_align_pot(test, 64);
 }
 
-static bool expect_insert_in_range_fail(struct kunit *test, struct drm_mm *mm,
-					u64 size, u64 range_start, u64 range_end)
+static void drm_test_mm_once(struct kunit *test, unsigned int mode)
 {
-	struct drm_mm_node tmp = {};
-	int err;
+	struct drm_mm mm;
+	struct drm_mm_node rsvd_lo, rsvd_hi, node;
 
-	err = drm_mm_insert_node_in_range(mm, &tmp, size, 0, 0, range_start, range_end,
-					  0);
-	if (likely(err == -ENOSPC))
-		return true;
+	drm_mm_init(&mm, 0, 7);
 
-	if (!err) {
-		KUNIT_FAIL(test,
-			   "impossible insert succeeded, node %llx + %llu, range [%llx, %llx]\n",
-				   tmp.start, tmp.size, range_start, range_end);
-		drm_mm_remove_node(&tmp);
-	} else {
-		KUNIT_FAIL(test,
-			   "impossible insert failed with wrong error %d [expected %d], size %llu, range [%llx, %llx]\n",
-				   err, -ENOSPC, size, range_start, range_end);
+	memset(&rsvd_lo, 0, sizeof(rsvd_lo));
+	rsvd_lo.start = 1;
+	rsvd_lo.size = 1;
+	if (drm_mm_reserve_node(&mm, &rsvd_lo)) {
+		KUNIT_FAIL(test, "Could not reserve low node\n");
+		goto err;
 	}
 
-	return false;
-}
-
-static bool assert_contiguous_in_range(struct kunit *test, struct drm_mm *mm,
-				       u64 size, u64 start, u64 end)
-{
-	struct drm_mm_node *node;
-	unsigned int n;
-
-	if (!expect_insert_in_range_fail(test, mm, size, start, end))
-		return false;
-
-	n = div64_u64(start + size - 1, size);
-	drm_mm_for_each_node(node, mm) {
-		if (node->start < start || node->start + node->size > end) {
-			KUNIT_FAIL(test,
-				   "node %d out of range, address [%llx + %llu], range [%llx, %llx]\n",
-					   n, node->start, node->start + node->size, start, end);
-			return false;
-		}
-
-		if (node->start != n * size) {
-			KUNIT_FAIL(test, "node %d out of order, expected start %llx, found %llx\n",
-				   n, n * size, node->start);
-			return false;
-		}
-
-		if (node->size != size) {
-			KUNIT_FAIL(test, "node %d has wrong size, expected size %llx, found %llx\n",
-				   n, size, node->size);
-			return false;
-		}
-
-		if (drm_mm_hole_follows(node) && drm_mm_hole_node_end(node) < end) {
-			KUNIT_FAIL(test, "node %d is followed by a hole!\n", n);
-			return false;
-		}
-
-		n++;
+	memset(&rsvd_hi, 0, sizeof(rsvd_hi));
+	rsvd_hi.start = 5;
+	rsvd_hi.size = 1;
+	if (drm_mm_reserve_node(&mm, &rsvd_hi)) {
+		KUNIT_FAIL(test, "Could not reserve low node\n");
+		goto err_lo;
 	}
 
-	if (start > 0) {
-		node = __drm_mm_interval_first(mm, 0, start - 1);
-		if (drm_mm_node_allocated(node)) {
-			KUNIT_FAIL(test, "node before start: node=%llx+%llu, start=%llx\n",
-				   node->start, node->size, start);
-			return false;
-		}
+	if (!drm_mm_hole_follows(&rsvd_lo) || !drm_mm_hole_follows(&rsvd_hi)) {
+		KUNIT_FAIL(test, "Expected a hole after lo and high nodes!\n");
+		goto err_hi;
 	}
 
-	if (end < U64_MAX) {
-		node = __drm_mm_interval_first(mm, end, U64_MAX);
-		if (drm_mm_node_allocated(node)) {
-			KUNIT_FAIL(test, "node after end: node=%llx+%llu, end=%llx\n",
-				   node->start, node->size, end);
-			return false;
-		}
+	memset(&node, 0, sizeof(node));
+	if (drm_mm_insert_node_generic(&mm, &node, 2, 0, 0, mode)) {
+		KUNIT_FAIL(test, "Could not insert the node into the available hole!\n");
+		goto err_hi;
 	}
 
-	return true;
+	drm_mm_remove_node(&node);
+err_hi:
+	drm_mm_remove_node(&rsvd_hi);
+err_lo:
+	drm_mm_remove_node(&rsvd_lo);
+err:
+	drm_mm_takedown(&mm);
 }
 
-static int __drm_test_mm_insert_range(struct kunit *test, unsigned int count, u64 size,
-				      u64 start, u64 end)
+static void drm_test_mm_lowest(struct kunit *test)
 {
-	const struct insert_mode *mode;
-	struct drm_mm mm;
-	struct drm_mm_node *nodes, *node, *next;
-	unsigned int n, start_n, end_n;
-	int ret;
-
-	DRM_MM_BUG_ON(!count);
-	DRM_MM_BUG_ON(!size);
-	DRM_MM_BUG_ON(end <= start);
-
-	/* Very similar to __drm_test_mm_insert(), but now instead of populating the
-	 * full range of the drm_mm, we try to fill a small portion of it.
-	 */
-
-	ret = -ENOMEM;
-	nodes = vzalloc(array_size(count, sizeof(*nodes)));
-	KUNIT_ASSERT_TRUE(test, nodes);
-
-	ret = -EINVAL;
-	drm_mm_init(&mm, 0, count * size);
-
-	start_n = div64_u64(start + size - 1, size);
-	end_n = div64_u64(end - size, size);
-
-	for (mode = insert_modes; mode->name; mode++) {
-		for (n = start_n; n <= end_n; n++) {
-			if (!expect_insert_in_range(test, &mm, &nodes[n], size, size, n,
-						    start, end, mode)) {
-				KUNIT_FAIL(test,
-					   "%s insert failed, size %llu, step %d [%d, %d], range [%llx, %llx]\n",
-						   mode->name, size, n, start_n, end_n, start, end);
-				goto out;
-			}
-		}
-
-		if (!assert_contiguous_in_range(test, &mm, size, start, end)) {
-			KUNIT_FAIL(test,
-				   "%s: range [%llx, %llx] not full after initialisation, size=%llu\n",
-				   mode->name, start, end, size);
-			goto out;
-		}
-
-		/* Remove one and reinsert, it should refill itself */
-		for (n = start_n; n <= end_n; n++) {
-			u64 addr = nodes[n].start;
-
-			drm_mm_remove_node(&nodes[n]);
-			if (!expect_insert_in_range(test, &mm, &nodes[n], size, size, n,
-						    start, end, mode)) {
-				KUNIT_FAIL(test, "%s reinsert failed, step %d\n", mode->name, n);
-				goto out;
-			}
-
-			if (nodes[n].start != addr) {
-				KUNIT_FAIL(test,
-					   "%s reinsert node moved, step %d, expected %llx, found %llx\n",
-					   mode->name, n, addr, nodes[n].start);
-				goto out;
-			}
-		}
-
-		if (!assert_contiguous_in_range(test, &mm, size, start, end)) {
-			KUNIT_FAIL(test,
-				   "%s: range [%llx, %llx] not full after reinsertion, size=%llu\n",
-				   mode->name, start, end, size);
-			goto out;
-		}
-
-		drm_mm_for_each_node_safe(node, next, &mm)
-			drm_mm_remove_node(node);
-		DRM_MM_BUG_ON(!drm_mm_clean(&mm));
-
-		cond_resched();
-	}
-
-	ret = 0;
-out:
-	drm_mm_for_each_node_safe(node, next, &mm)
-		drm_mm_remove_node(node);
-	drm_mm_takedown(&mm);
-	vfree(nodes);
-	return ret;
-}
-
-static int insert_outside_range(struct kunit *test)
-{
-	struct drm_mm mm;
-	const unsigned int start = 1024;
-	const unsigned int end = 2048;
-	const unsigned int size = end - start;
-
-	drm_mm_init(&mm, start, size);
-
-	if (!expect_insert_in_range_fail(test, &mm, 1, 0, start))
-		return -EINVAL;
-
-	if (!expect_insert_in_range_fail(test, &mm, size,
-					 start - size / 2, start + (size + 1) / 2))
-		return -EINVAL;
-
-	if (!expect_insert_in_range_fail(test, &mm, size,
-					 end - (size + 1) / 2, end + size / 2))
-		return -EINVAL;
-
-	if (!expect_insert_in_range_fail(test, &mm, 1, end, end + size))
-		return -EINVAL;
-
-	drm_mm_takedown(&mm);
-	return 0;
-}
-
-static void drm_test_mm_insert_range(struct kunit *test)
-{
-	const unsigned int count = min_t(unsigned int, BIT(13), max_iterations);
-	unsigned int n;
-
-	/* Check that requests outside the bounds of drm_mm are rejected. */
-	KUNIT_ASSERT_FALSE(test, insert_outside_range(test));
-
-	for_each_prime_number_from(n, 1, 50) {
-		const u64 size = BIT_ULL(n);
-		const u64 max = count * size;
-
-		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert_range(test, count, size, 0, max));
-		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert_range(test, count, size, 1, max));
-		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert_range(test, count, size, 0, max - 1));
-		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert_range(test, count, size, 0, max / 2));
-		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert_range(test, count, size,
-								    max / 2, max));
-		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert_range(test, count, size,
-								    max / 4 + 1, 3 * max / 4 - 1));
-
-		cond_resched();
-	}
-}
-
-static int prepare_frag(struct kunit *test, struct drm_mm *mm, struct drm_mm_node *nodes,
-			unsigned int num_insert, const struct insert_mode *mode)
-{
-	unsigned int size = 4096;
-	unsigned int i;
-
-	for (i = 0; i < num_insert; i++) {
-		if (!expect_insert(test, mm, &nodes[i], size, 0, i, mode) != 0) {
-			KUNIT_FAIL(test, "%s insert failed\n", mode->name);
-			return -EINVAL;
-		}
-	}
-
-	/* introduce fragmentation by freeing every other node */
-	for (i = 0; i < num_insert; i++) {
-		if (i % 2 == 0)
-			drm_mm_remove_node(&nodes[i]);
-	}
-
-	return 0;
-}
-
-static u64 get_insert_time(struct kunit *test, struct drm_mm *mm,
-			   unsigned int num_insert, struct drm_mm_node *nodes,
-			   const struct insert_mode *mode)
-{
-	unsigned int size = 8192;
-	ktime_t start;
-	unsigned int i;
-
-	start = ktime_get();
-	for (i = 0; i < num_insert; i++) {
-		if (!expect_insert(test, mm, &nodes[i], size, 0, i, mode) != 0) {
-			KUNIT_FAIL(test, "%s insert failed\n", mode->name);
-			return 0;
-		}
-	}
-
-	return ktime_to_ns(ktime_sub(ktime_get(), start));
-}
-
-static void drm_test_mm_frag(struct kunit *test)
-{
-	struct drm_mm mm;
-	const struct insert_mode *mode;
-	struct drm_mm_node *nodes, *node, *next;
-	unsigned int insert_size = 10000;
-	unsigned int scale_factor = 4;
-
-	/* We need 4 * insert_size nodes to hold intermediate allocated
-	 * drm_mm nodes.
-	 * 1 times for prepare_frag()
-	 * 1 times for get_insert_time()
-	 * 2 times for get_insert_time()
-	 */
-	nodes = vzalloc(array_size(insert_size * 4, sizeof(*nodes)));
-	KUNIT_ASSERT_TRUE(test, nodes);
-
-	/* For BOTTOMUP and TOPDOWN, we first fragment the
-	 * address space using prepare_frag() and then try to verify
-	 * that insertions scale quadratically from 10k to 20k insertions
-	 */
-	drm_mm_init(&mm, 1, U64_MAX - 2);
-	for (mode = insert_modes; mode->name; mode++) {
-		u64 insert_time1, insert_time2;
-
-		if (mode->mode != DRM_MM_INSERT_LOW &&
-		    mode->mode != DRM_MM_INSERT_HIGH)
-			continue;
-
-		if (prepare_frag(test, &mm, nodes, insert_size, mode))
-			goto err;
-
-		insert_time1 = get_insert_time(test, &mm, insert_size,
-					       nodes + insert_size, mode);
-		if (insert_time1 == 0)
-			goto err;
-
-		insert_time2 = get_insert_time(test, &mm, (insert_size * 2),
-					       nodes + insert_size * 2, mode);
-		if (insert_time2 == 0)
-			goto err;
-
-		kunit_info(test, "%s fragmented insert of %u and %u insertions took %llu and %llu nsecs\n",
-			   mode->name, insert_size, insert_size * 2, insert_time1, insert_time2);
-
-		if (insert_time2 > (scale_factor * insert_time1)) {
-			KUNIT_FAIL(test, "%s fragmented insert took %llu nsecs more\n",
-				   mode->name, insert_time2 - (scale_factor * insert_time1));
-			goto err;
-		}
-
-		drm_mm_for_each_node_safe(node, next, &mm)
-			drm_mm_remove_node(node);
-	}
-
-err:
-	drm_mm_for_each_node_safe(node, next, &mm)
-		drm_mm_remove_node(node);
-	drm_mm_takedown(&mm);
-	vfree(nodes);
-}
-
-static void drm_test_mm_align(struct kunit *test)
-{
-	const struct insert_mode *mode;
-	const unsigned int max_count = min(8192u, max_prime);
-	struct drm_mm mm;
-	struct drm_mm_node *nodes, *node, *next;
-	unsigned int prime;
-
-	/* For each of the possible insertion modes, we pick a few
-	 * arbitrary alignments and check that the inserted node
-	 * meets our requirements.
-	 */
-
-	nodes = vzalloc(array_size(max_count, sizeof(*nodes)));
-	KUNIT_ASSERT_TRUE(test, nodes);
-
-	drm_mm_init(&mm, 1, U64_MAX - 2);
-
-	for (mode = insert_modes; mode->name; mode++) {
-		unsigned int i = 0;
-
-		for_each_prime_number_from(prime, 1, max_count) {
-			u64 size = next_prime_number(prime);
-
-			if (!expect_insert(test, &mm, &nodes[i], size, prime, i, mode)) {
-				KUNIT_FAIL(test, "%s insert failed with alignment=%d",
-					   mode->name, prime);
-				goto out;
-			}
-
-			i++;
-		}
-
-		drm_mm_for_each_node_safe(node, next, &mm)
-			drm_mm_remove_node(node);
-		DRM_MM_BUG_ON(!drm_mm_clean(&mm));
-
-		cond_resched();
-	}
-
-out:
-	drm_mm_for_each_node_safe(node, next, &mm)
-		drm_mm_remove_node(node);
-	drm_mm_takedown(&mm);
-	vfree(nodes);
-}
-
-static void drm_test_mm_align_pot(struct kunit *test, int max)
-{
-	struct drm_mm mm;
-	struct drm_mm_node *node, *next;
-	int bit;
-
-	/* Check that we can align to the full u64 address space */
-
-	drm_mm_init(&mm, 1, U64_MAX - 2);
-
-	for (bit = max - 1; bit; bit--) {
-		u64 align, size;
-
-		node = kzalloc(sizeof(*node), GFP_KERNEL);
-		if (!node) {
-			KUNIT_FAIL(test, "failed to allocate node");
-			goto out;
-		}
-
-		align = BIT_ULL(bit);
-		size = BIT_ULL(bit - 1) + 1;
-		if (!expect_insert(test, &mm, node, size, align, bit, &insert_modes[0])) {
-			KUNIT_FAIL(test, "insert failed with alignment=%llx [%d]", align, bit);
-			goto out;
-		}
-
-		cond_resched();
-	}
-
-out:
-	drm_mm_for_each_node_safe(node, next, &mm) {
-		drm_mm_remove_node(node);
-		kfree(node);
-	}
-	drm_mm_takedown(&mm);
-}
-
-static void drm_test_mm_align32(struct kunit *test)
-{
-	drm_test_mm_align_pot(test, 32);
-}
-
-static void drm_test_mm_align64(struct kunit *test)
-{
-	drm_test_mm_align_pot(test, 64);
-}
-
-static void show_scan(struct kunit *test, const struct drm_mm_scan *scan)
-{
-	kunit_info(test, "scan: hit [%llx, %llx], size=%lld, align=%lld, color=%ld\n",
-		   scan->hit_start, scan->hit_end, scan->size, scan->alignment, scan->color);
-}
-
-static void show_holes(struct kunit *test, const struct drm_mm *mm, int count)
-{
-	u64 hole_start, hole_end;
-	struct drm_mm_node *hole;
-
-	drm_mm_for_each_hole(hole, mm, hole_start, hole_end) {
-		struct drm_mm_node *next = list_next_entry(hole, node_list);
-		const char *node1 = NULL, *node2 = NULL;
-
-		if (drm_mm_node_allocated(hole))
-			node1 = kasprintf(GFP_KERNEL, "[%llx + %lld, color=%ld], ",
-					  hole->start, hole->size, hole->color);
-
-		if (drm_mm_node_allocated(next))
-			node2 = kasprintf(GFP_KERNEL, ", [%llx + %lld, color=%ld]",
-					  next->start, next->size, next->color);
-
-		kunit_info(test, "%sHole [%llx - %llx, size %lld]%s\n", node1,
-			   hole_start, hole_end, hole_end - hole_start, node2);
-
-		kfree(node2);
-		kfree(node1);
-
-		if (!--count)
-			break;
-	}
-}
-
-struct evict_node {
-	struct drm_mm_node node;
-	struct list_head link;
-};
-
-static bool evict_nodes(struct kunit *test, struct drm_mm_scan *scan,
-			struct evict_node *nodes, unsigned int *order, unsigned int count,
-			bool use_color, struct list_head *evict_list)
-{
-	struct evict_node *e, *en;
-	unsigned int i;
-
-	for (i = 0; i < count; i++) {
-		e = &nodes[order ? order[i] : i];
-		list_add(&e->link, evict_list);
-		if (drm_mm_scan_add_block(scan, &e->node))
-			break;
-	}
-	list_for_each_entry_safe(e, en, evict_list, link) {
-		if (!drm_mm_scan_remove_block(scan, &e->node))
-			list_del(&e->link);
-	}
-	if (list_empty(evict_list)) {
-		KUNIT_FAIL(test,
-			   "Failed to find eviction: size=%lld [avail=%d], align=%lld (color=%lu)\n",
-			   scan->size, count, scan->alignment, scan->color);
-		return false;
-	}
-
-	list_for_each_entry(e, evict_list, link)
-		drm_mm_remove_node(&e->node);
-
-	if (use_color) {
-		struct drm_mm_node *node;
-
-		while ((node = drm_mm_scan_color_evict(scan))) {
-			e = container_of(node, typeof(*e), node);
-			drm_mm_remove_node(&e->node);
-			list_add(&e->link, evict_list);
-		}
-	} else {
-		if (drm_mm_scan_color_evict(scan)) {
-			KUNIT_FAIL(test,
-				   "drm_mm_scan_color_evict unexpectedly reported overlapping nodes!\n");
-			return false;
-		}
-	}
-
-	return true;
-}
-
-static bool evict_nothing(struct kunit *test, struct drm_mm *mm,
-			  unsigned int total_size, struct evict_node *nodes)
-{
-	struct drm_mm_scan scan;
-	LIST_HEAD(evict_list);
-	struct evict_node *e;
-	struct drm_mm_node *node;
-	unsigned int n;
-
-	drm_mm_scan_init(&scan, mm, 1, 0, 0, 0);
-	for (n = 0; n < total_size; n++) {
-		e = &nodes[n];
-		list_add(&e->link, &evict_list);
-		drm_mm_scan_add_block(&scan, &e->node);
-	}
-	list_for_each_entry(e, &evict_list, link)
-		drm_mm_scan_remove_block(&scan, &e->node);
-
-	for (n = 0; n < total_size; n++) {
-		e = &nodes[n];
-
-		if (!drm_mm_node_allocated(&e->node)) {
-			KUNIT_FAIL(test, "node[%d] no longer allocated!\n", n);
-			return false;
-		}
-
-		e->link.next = NULL;
-	}
-
-	drm_mm_for_each_node(node, mm) {
-		e = container_of(node, typeof(*e), node);
-		e->link.next = &e->link;
-	}
-
-	for (n = 0; n < total_size; n++) {
-		e = &nodes[n];
-
-		if (!e->link.next) {
-			KUNIT_FAIL(test, "node[%d] no longer connected!\n", n);
-			return false;
-		}
-	}
-
-	return assert_continuous(test, mm, nodes[0].node.size);
-}
-
-static bool evict_everything(struct kunit *test, struct drm_mm *mm,
-			     unsigned int total_size, struct evict_node *nodes)
-{
-	struct drm_mm_scan scan;
-	LIST_HEAD(evict_list);
-	struct evict_node *e;
-	unsigned int n;
-	int err;
-
-	drm_mm_scan_init(&scan, mm, total_size, 0, 0, 0);
-	for (n = 0; n < total_size; n++) {
-		e = &nodes[n];
-		list_add(&e->link, &evict_list);
-		if (drm_mm_scan_add_block(&scan, &e->node))
-			break;
-	}
-
-	err = 0;
-	list_for_each_entry(e, &evict_list, link) {
-		if (!drm_mm_scan_remove_block(&scan, &e->node)) {
-			if (!err) {
-				KUNIT_FAIL(test, "Node %lld not marked for eviction!\n",
-					   e->node.start);
-				err = -EINVAL;
-			}
-		}
-	}
-	if (err)
-		return false;
-
-	list_for_each_entry(e, &evict_list, link)
-		drm_mm_remove_node(&e->node);
-
-	if (!assert_one_hole(test, mm, 0, total_size))
-		return false;
-
-	list_for_each_entry(e, &evict_list, link) {
-		err = drm_mm_reserve_node(mm, &e->node);
-		if (err) {
-			KUNIT_FAIL(test, "Failed to reinsert node after eviction: start=%llx\n",
-				   e->node.start);
-			return false;
-		}
-	}
-
-	return assert_continuous(test, mm, nodes[0].node.size);
-}
-
-static int evict_something(struct kunit *test, struct drm_mm *mm,
-			   u64 range_start, u64 range_end, struct evict_node *nodes,
-			   unsigned int *order, unsigned int count, unsigned int size,
-			   unsigned int alignment, const struct insert_mode *mode)
-{
-	struct drm_mm_scan scan;
-	LIST_HEAD(evict_list);
-	struct evict_node *e;
-	struct drm_mm_node tmp;
-	int err;
-
-	drm_mm_scan_init_with_range(&scan, mm, size, alignment, 0, range_start,
-				    range_end, mode->mode);
-	if (!evict_nodes(test, &scan, nodes, order, count, false, &evict_list))
-		return -EINVAL;
-
-	memset(&tmp, 0, sizeof(tmp));
-	err = drm_mm_insert_node_generic(mm, &tmp, size, alignment, 0,
-					 DRM_MM_INSERT_EVICT);
-	if (err) {
-		KUNIT_FAIL(test, "Failed to insert into eviction hole: size=%d, align=%d\n",
-			   size, alignment);
-		show_scan(test, &scan);
-		show_holes(test, mm, 3);
-		return err;
-	}
-
-	if (tmp.start < range_start || tmp.start + tmp.size > range_end) {
-		KUNIT_FAIL(test,
-			   "Inserted [address=%llu + %llu] did not fit into the request range [%llu, %llu]\n",
-			   tmp.start, tmp.size, range_start, range_end);
-		err = -EINVAL;
-	}
-
-	if (!assert_node(test, &tmp, mm, size, alignment, 0) ||
-	    drm_mm_hole_follows(&tmp)) {
-		KUNIT_FAIL(test,
-			   "Inserted did not fill the eviction hole: size=%lld [%d], align=%d [rem=%lld], start=%llx, hole-follows?=%d\n",
-			   tmp.size, size, alignment, misalignment(&tmp, alignment),
-			   tmp.start, drm_mm_hole_follows(&tmp));
-		err = -EINVAL;
-	}
-
-	drm_mm_remove_node(&tmp);
-	if (err)
-		return err;
-
-	list_for_each_entry(e, &evict_list, link) {
-		err = drm_mm_reserve_node(mm, &e->node);
-		if (err) {
-			KUNIT_FAIL(test, "Failed to reinsert node after eviction: start=%llx\n",
-				   e->node.start);
-			return err;
-		}
-	}
-
-	if (!assert_continuous(test, mm, nodes[0].node.size)) {
-		KUNIT_FAIL(test, "range is no longer continuous\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static void drm_test_mm_evict(struct kunit *test)
-{
-	DRM_RND_STATE(prng, random_seed);
-	const unsigned int size = 8192;
-	const struct insert_mode *mode;
-	struct drm_mm mm;
-	struct evict_node *nodes;
-	struct drm_mm_node *node, *next;
-	unsigned int *order, n;
-
-	/* Here we populate a full drm_mm and then try and insert a new node
-	 * by evicting other nodes in a random order. The drm_mm_scan should
-	 * pick the first matching hole it finds from the random list. We
-	 * repeat that for different allocation strategies, alignments and
-	 * sizes to try and stress the hole finder.
-	 */
-
-	nodes = vzalloc(array_size(size, sizeof(*nodes)));
-	KUNIT_ASSERT_TRUE(test, nodes);
-
-	order = drm_random_order(size, &prng);
-	if (!order)
-		goto err_nodes;
-
-	drm_mm_init(&mm, 0, size);
-	for (n = 0; n < size; n++) {
-		if (drm_mm_insert_node(&mm, &nodes[n].node, 1)) {
-			KUNIT_FAIL(test, "insert failed, step %d\n", n);
-			goto out;
-		}
-	}
-
-	/* First check that using the scanner doesn't break the mm */
-	if (!evict_nothing(test, &mm, size, nodes)) {
-		KUNIT_FAIL(test, "evict_nothing() failed\n");
-		goto out;
-	}
-	if (!evict_everything(test, &mm, size, nodes)) {
-		KUNIT_FAIL(test, "evict_everything() failed\n");
-		goto out;
-	}
-
-	for (mode = evict_modes; mode->name; mode++) {
-		for (n = 1; n <= size; n <<= 1) {
-			drm_random_reorder(order, size, &prng);
-			if (evict_something(test, &mm, 0, U64_MAX, nodes, order, size, n, 1,
-					    mode)) {
-				KUNIT_FAIL(test, "%s evict_something(size=%u) failed\n",
-					   mode->name, n);
-				goto out;
-			}
-		}
-
-		for (n = 1; n < size; n <<= 1) {
-			drm_random_reorder(order, size, &prng);
-			if (evict_something(test, &mm, 0, U64_MAX, nodes, order, size,
-					    size / 2, n, mode)) {
-				KUNIT_FAIL(test,
-					   "%s evict_something(size=%u, alignment=%u) failed\n",
-					   mode->name, size / 2, n);
-				goto out;
-			}
-		}
-
-		for_each_prime_number_from(n, 1, min(size, max_prime)) {
-			unsigned int nsize = (size - n + 1) / 2;
-
-			DRM_MM_BUG_ON(!nsize);
-
-			drm_random_reorder(order, size, &prng);
-			if (evict_something(test, &mm, 0, U64_MAX, nodes, order, size,
-					    nsize, n, mode)) {
-				KUNIT_FAIL(test,
-					   "%s evict_something(size=%u, alignment=%u) failed\n",
-					   mode->name, nsize, n);
-				goto out;
-			}
-		}
-
-		cond_resched();
-	}
-
-out:
-	drm_mm_for_each_node_safe(node, next, &mm)
-		drm_mm_remove_node(node);
-	drm_mm_takedown(&mm);
-	kfree(order);
-err_nodes:
-	vfree(nodes);
-}
-
-static void drm_test_mm_evict_range(struct kunit *test)
-{
-	DRM_RND_STATE(prng, random_seed);
-	const unsigned int size = 8192;
-	const unsigned int range_size = size / 2;
-	const unsigned int range_start = size / 4;
-	const unsigned int range_end = range_start + range_size;
-	const struct insert_mode *mode;
-	struct drm_mm mm;
-	struct evict_node *nodes;
-	struct drm_mm_node *node, *next;
-	unsigned int *order, n;
-
-	/* Like drm_test_mm_evict() but now we are limiting the search to a
-	 * small portion of the full drm_mm.
-	 */
-
-	nodes = vzalloc(array_size(size, sizeof(*nodes)));
-	KUNIT_ASSERT_TRUE(test, nodes);
-
-	order = drm_random_order(size, &prng);
-	if (!order)
-		goto err_nodes;
-
-	drm_mm_init(&mm, 0, size);
-	for (n = 0; n < size; n++) {
-		if (drm_mm_insert_node(&mm, &nodes[n].node, 1)) {
-			KUNIT_FAIL(test, "insert failed, step %d\n", n);
-			goto out;
-		}
-	}
-
-	for (mode = evict_modes; mode->name; mode++) {
-		for (n = 1; n <= range_size; n <<= 1) {
-			drm_random_reorder(order, size, &prng);
-			if (evict_something(test, &mm, range_start, range_end, nodes,
-					    order, size, n, 1, mode)) {
-				KUNIT_FAIL(test,
-					   "%s evict_something(size=%u) failed with range [%u, %u]\n",
-					   mode->name, n, range_start, range_end);
-				goto out;
-			}
-		}
-
-		for (n = 1; n <= range_size; n <<= 1) {
-			drm_random_reorder(order, size, &prng);
-			if (evict_something(test, &mm, range_start, range_end, nodes,
-					    order, size, range_size / 2, n, mode)) {
-				KUNIT_FAIL(test,
-					   "%s evict_something(size=%u, alignment=%u) failed with range [%u, %u]\n",
-					   mode->name, range_size / 2, n, range_start, range_end);
-				goto out;
-			}
-		}
-
-		for_each_prime_number_from(n, 1, min(range_size, max_prime)) {
-			unsigned int nsize = (range_size - n + 1) / 2;
-
-			DRM_MM_BUG_ON(!nsize);
-
-			drm_random_reorder(order, size, &prng);
-			if (evict_something(test, &mm, range_start, range_end, nodes,
-					    order, size, nsize, n, mode)) {
-				KUNIT_FAIL(test,
-					   "%s evict_something(size=%u, alignment=%u) failed with range [%u, %u]\n",
-					   mode->name, nsize, n, range_start, range_end);
-				goto out;
-			}
-		}
-
-		cond_resched();
-	}
-
-out:
-	drm_mm_for_each_node_safe(node, next, &mm)
-		drm_mm_remove_node(node);
-	drm_mm_takedown(&mm);
-	kfree(order);
-err_nodes:
-	vfree(nodes);
-}
-
-static unsigned int node_index(const struct drm_mm_node *node)
-{
-	return div64_u64(node->start, node->size);
-}
-
-static void drm_test_mm_topdown(struct kunit *test)
-{
-	const struct insert_mode *topdown = &insert_modes[TOPDOWN];
-
-	DRM_RND_STATE(prng, random_seed);
-	const unsigned int count = 8192;
-	unsigned int size;
-	unsigned long *bitmap;
-	struct drm_mm mm;
-	struct drm_mm_node *nodes, *node, *next;
-	unsigned int *order, n, m, o = 0;
-
-	/* When allocating top-down, we expect to be returned a node
-	 * from a suitable hole at the top of the drm_mm. We check that
-	 * the returned node does match the highest available slot.
-	 */
-
-	nodes = vzalloc(array_size(count, sizeof(*nodes)));
-	KUNIT_ASSERT_TRUE(test, nodes);
-
-	bitmap = bitmap_zalloc(count, GFP_KERNEL);
-	if (!bitmap)
-		goto err_nodes;
-
-	order = drm_random_order(count, &prng);
-	if (!order)
-		goto err_bitmap;
-
-	for (size = 1; size <= 64; size <<= 1) {
-		drm_mm_init(&mm, 0, size * count);
-		for (n = 0; n < count; n++) {
-			if (!expect_insert(test, &mm, &nodes[n], size, 0, n, topdown)) {
-				KUNIT_FAIL(test, "insert failed, size %u step %d\n", size, n);
-				goto out;
-			}
-
-			if (drm_mm_hole_follows(&nodes[n])) {
-				KUNIT_FAIL(test,
-					   "hole after topdown insert %d, start=%llx\n, size=%u",
-					   n, nodes[n].start, size);
-				goto out;
-			}
-
-			if (!assert_one_hole(test, &mm, 0, size * (count - n - 1)))
-				goto out;
-		}
-
-		if (!assert_continuous(test, &mm, size))
-			goto out;
-
-		drm_random_reorder(order, count, &prng);
-		for_each_prime_number_from(n, 1, min(count, max_prime)) {
-			for (m = 0; m < n; m++) {
-				node = &nodes[order[(o + m) % count]];
-				drm_mm_remove_node(node);
-				__set_bit(node_index(node), bitmap);
-			}
-
-			for (m = 0; m < n; m++) {
-				unsigned int last;
-
-				node = &nodes[order[(o + m) % count]];
-				if (!expect_insert(test, &mm, node, size, 0, 0, topdown)) {
-					KUNIT_FAIL(test, "insert failed, step %d/%d\n", m, n);
-					goto out;
-				}
-
-				if (drm_mm_hole_follows(node)) {
-					KUNIT_FAIL(test,
-						   "hole after topdown insert %d/%d, start=%llx\n",
-						   m, n, node->start);
-					goto out;
-				}
-
-				last = find_last_bit(bitmap, count);
-				if (node_index(node) != last) {
-					KUNIT_FAIL(test,
-						   "node %d/%d, size %d, not inserted into upmost hole, expected %d, found %d\n",
-						   m, n, size, last, node_index(node));
-					goto out;
-				}
-
-				__clear_bit(last, bitmap);
-			}
-
-			DRM_MM_BUG_ON(find_first_bit(bitmap, count) != count);
-
-			o += n;
-		}
-
-		drm_mm_for_each_node_safe(node, next, &mm)
-			drm_mm_remove_node(node);
-		DRM_MM_BUG_ON(!drm_mm_clean(&mm));
-		cond_resched();
-	}
-
-out:
-	drm_mm_for_each_node_safe(node, next, &mm)
-		drm_mm_remove_node(node);
-	drm_mm_takedown(&mm);
-	kfree(order);
-err_bitmap:
-	bitmap_free(bitmap);
-err_nodes:
-	vfree(nodes);
-}
-
-static void drm_test_mm_bottomup(struct kunit *test)
-{
-	const struct insert_mode *bottomup = &insert_modes[BOTTOMUP];
-
-	DRM_RND_STATE(prng, random_seed);
-	const unsigned int count = 8192;
-	unsigned int size;
-	unsigned long *bitmap;
-	struct drm_mm mm;
-	struct drm_mm_node *nodes, *node, *next;
-	unsigned int *order, n, m, o = 0;
-
-	/* Like drm_test_mm_topdown, but instead of searching for the last hole,
-	 * we search for the first.
-	 */
-
-	nodes = vzalloc(array_size(count, sizeof(*nodes)));
-	KUNIT_ASSERT_TRUE(test, nodes);
-
-	bitmap = bitmap_zalloc(count, GFP_KERNEL);
-	if (!bitmap)
-		goto err_nodes;
-
-	order = drm_random_order(count, &prng);
-	if (!order)
-		goto err_bitmap;
-
-	for (size = 1; size <= 64; size <<= 1) {
-		drm_mm_init(&mm, 0, size * count);
-		for (n = 0; n < count; n++) {
-			if (!expect_insert(test, &mm, &nodes[n], size, 0, n, bottomup)) {
-				KUNIT_FAIL(test,
-					   "bottomup insert failed, size %u step %d\n", size, n);
-				goto out;
-			}
-
-			if (!assert_one_hole(test, &mm, size * (n + 1), size * count))
-				goto out;
-		}
-
-		if (!assert_continuous(test, &mm, size))
-			goto out;
-
-		drm_random_reorder(order, count, &prng);
-		for_each_prime_number_from(n, 1, min(count, max_prime)) {
-			for (m = 0; m < n; m++) {
-				node = &nodes[order[(o + m) % count]];
-				drm_mm_remove_node(node);
-				__set_bit(node_index(node), bitmap);
-			}
-
-			for (m = 0; m < n; m++) {
-				unsigned int first;
-
-				node = &nodes[order[(o + m) % count]];
-				if (!expect_insert(test, &mm, node, size, 0, 0, bottomup)) {
-					KUNIT_FAIL(test, "insert failed, step %d/%d\n", m, n);
-					goto out;
-				}
-
-				first = find_first_bit(bitmap, count);
-				if (node_index(node) != first) {
-					KUNIT_FAIL(test,
-						   "node %d/%d not inserted into bottom hole, expected %d, found %d\n",
-						   m, n, first, node_index(node));
-					goto out;
-				}
-				__clear_bit(first, bitmap);
-			}
-
-			DRM_MM_BUG_ON(find_first_bit(bitmap, count) != count);
-
-			o += n;
-		}
-
-		drm_mm_for_each_node_safe(node, next, &mm)
-			drm_mm_remove_node(node);
-		DRM_MM_BUG_ON(!drm_mm_clean(&mm));
-		cond_resched();
-	}
-
-out:
-	drm_mm_for_each_node_safe(node, next, &mm)
-		drm_mm_remove_node(node);
-	drm_mm_takedown(&mm);
-	kfree(order);
-err_bitmap:
-	bitmap_free(bitmap);
-err_nodes:
-	vfree(nodes);
-}
-
-static void drm_test_mm_once(struct kunit *test, unsigned int mode)
-{
-	struct drm_mm mm;
-	struct drm_mm_node rsvd_lo, rsvd_hi, node;
-
-	drm_mm_init(&mm, 0, 7);
-
-	memset(&rsvd_lo, 0, sizeof(rsvd_lo));
-	rsvd_lo.start = 1;
-	rsvd_lo.size = 1;
-	if (drm_mm_reserve_node(&mm, &rsvd_lo)) {
-		KUNIT_FAIL(test, "Could not reserve low node\n");
-		goto err;
-	}
-
-	memset(&rsvd_hi, 0, sizeof(rsvd_hi));
-	rsvd_hi.start = 5;
-	rsvd_hi.size = 1;
-	if (drm_mm_reserve_node(&mm, &rsvd_hi)) {
-		KUNIT_FAIL(test, "Could not reserve low node\n");
-		goto err_lo;
-	}
-
-	if (!drm_mm_hole_follows(&rsvd_lo) || !drm_mm_hole_follows(&rsvd_hi)) {
-		KUNIT_FAIL(test, "Expected a hole after lo and high nodes!\n");
-		goto err_hi;
-	}
-
-	memset(&node, 0, sizeof(node));
-	if (drm_mm_insert_node_generic(&mm, &node, 2, 0, 0, mode)) {
-		KUNIT_FAIL(test, "Could not insert the node into the available hole!\n");
-		goto err_hi;
-	}
-
-	drm_mm_remove_node(&node);
-err_hi:
-	drm_mm_remove_node(&rsvd_hi);
-err_lo:
-	drm_mm_remove_node(&rsvd_lo);
-err:
-	drm_mm_takedown(&mm);
-}
-
-static void drm_test_mm_lowest(struct kunit *test)
-{
-	drm_test_mm_once(test, DRM_MM_INSERT_LOW);
-}
+	drm_test_mm_once(test, DRM_MM_INSERT_LOW);
+}
 
 static void drm_test_mm_highest(struct kunit *test)
 {
 	drm_test_mm_once(test, DRM_MM_INSERT_HIGH);
 }
 
-static void separate_adjacent_colors(const struct drm_mm_node *node,
-				     unsigned long color, u64 *start, u64 *end)
-{
-	if (drm_mm_node_allocated(node) && node->color != color)
-		++*start;
-
-	node = list_next_entry(node, node_list);
-	if (drm_mm_node_allocated(node) && node->color != color)
-		--*end;
-}
-
-static bool colors_abutt(struct kunit *test, const struct drm_mm_node *node)
-{
-	if (!drm_mm_hole_follows(node) &&
-	    drm_mm_node_allocated(list_next_entry(node, node_list))) {
-		KUNIT_FAIL(test, "colors abutt; %ld [%llx + %llx] is next to %ld [%llx + %llx]!\n",
-			   node->color, node->start, node->size,
-		       list_next_entry(node, node_list)->color,
-		       list_next_entry(node, node_list)->start,
-		       list_next_entry(node, node_list)->size);
-		return true;
-	}
-
-	return false;
-}
-
-static void drm_test_mm_color(struct kunit *test)
-{
-	const unsigned int count = min(4096u, max_iterations);
-	const struct insert_mode *mode;
-	struct drm_mm mm;
-	struct drm_mm_node *node, *nn;
-	unsigned int n;
-
-	/* Color adjustment complicates everything. First we just check
-	 * that when we insert a node we apply any color_adjustment callback.
-	 * The callback we use should ensure that there is a gap between
-	 * any two nodes, and so after each insertion we check that those
-	 * holes are inserted and that they are preserved.
-	 */
-
-	drm_mm_init(&mm, 0, U64_MAX);
-
-	for (n = 1; n <= count; n++) {
-		node = kzalloc(sizeof(*node), GFP_KERNEL);
-		if (!node)
-			goto out;
-
-		if (!expect_insert(test, &mm, node, n, 0, n, &insert_modes[0])) {
-			KUNIT_FAIL(test, "insert failed, step %d\n", n);
-			kfree(node);
-			goto out;
-		}
-	}
-
-	drm_mm_for_each_node_safe(node, nn, &mm) {
-		if (node->color != node->size) {
-			KUNIT_FAIL(test, "invalid color stored: expected %lld, found %ld\n",
-				   node->size, node->color);
-
-			goto out;
-		}
-
-		drm_mm_remove_node(node);
-		kfree(node);
-	}
-
-	/* Now, let's start experimenting with applying a color callback */
-	mm.color_adjust = separate_adjacent_colors;
-	for (mode = insert_modes; mode->name; mode++) {
-		u64 last;
-
-		node = kzalloc(sizeof(*node), GFP_KERNEL);
-		if (!node)
-			goto out;
-
-		node->size = 1 + 2 * count;
-		node->color = node->size;
-
-		if (drm_mm_reserve_node(&mm, node)) {
-			KUNIT_FAIL(test, "initial reserve failed!\n");
-			goto out;
-		}
-
-		last = node->start + node->size;
-
-		for (n = 1; n <= count; n++) {
-			int rem;
-
-			node = kzalloc(sizeof(*node), GFP_KERNEL);
-			if (!node)
-				goto out;
-
-			node->start = last;
-			node->size = n + count;
-			node->color = node->size;
-
-			if (drm_mm_reserve_node(&mm, node) != -ENOSPC) {
-				KUNIT_FAIL(test, "reserve %d did not report color overlap!", n);
-				goto out;
-			}
-
-			node->start += n + 1;
-			rem = misalignment(node, n + count);
-			node->start += n + count - rem;
-
-			if (drm_mm_reserve_node(&mm, node)) {
-				KUNIT_FAIL(test, "reserve %d failed", n);
-				goto out;
-			}
-
-			last = node->start + node->size;
-		}
-
-		for (n = 1; n <= count; n++) {
-			node = kzalloc(sizeof(*node), GFP_KERNEL);
-			if (!node)
-				goto out;
-
-			if (!expect_insert(test, &mm, node, n, n, n, mode)) {
-				KUNIT_FAIL(test, "%s insert failed, step %d\n", mode->name, n);
-				kfree(node);
-				goto out;
-			}
-		}
-
-		drm_mm_for_each_node_safe(node, nn, &mm) {
-			u64 rem;
-
-			if (node->color != node->size) {
-				KUNIT_FAIL(test,
-					   "%s invalid color stored: expected %lld, found %ld\n",
-					   mode->name, node->size, node->color);
-
-				goto out;
-			}
-
-			if (colors_abutt(test, node))
-				goto out;
-
-			div64_u64_rem(node->start, node->size, &rem);
-			if (rem) {
-				KUNIT_FAIL(test,
-					   "%s colored node misaligned, start=%llx expected alignment=%lld [rem=%lld]\n",
-					   mode->name, node->start, node->size, rem);
-				goto out;
-			}
-
-			drm_mm_remove_node(node);
-			kfree(node);
-		}
-
-		cond_resched();
-	}
-
-out:
-	drm_mm_for_each_node_safe(node, nn, &mm) {
-		drm_mm_remove_node(node);
-		kfree(node);
-	}
-	drm_mm_takedown(&mm);
-}
-
-static int evict_color(struct kunit *test, struct drm_mm *mm, u64 range_start,
-		       u64 range_end, struct evict_node *nodes, unsigned int *order,
-		unsigned int count, unsigned int size, unsigned int alignment,
-		unsigned long color, const struct insert_mode *mode)
-{
-	struct drm_mm_scan scan;
-	LIST_HEAD(evict_list);
-	struct evict_node *e;
-	struct drm_mm_node tmp;
-	int err;
-
-	drm_mm_scan_init_with_range(&scan, mm, size, alignment, color, range_start,
-				    range_end, mode->mode);
-	if (!evict_nodes(test, &scan, nodes, order, count, true, &evict_list))
-		return -EINVAL;
-
-	memset(&tmp, 0, sizeof(tmp));
-	err = drm_mm_insert_node_generic(mm, &tmp, size, alignment, color,
-					 DRM_MM_INSERT_EVICT);
-	if (err) {
-		KUNIT_FAIL(test,
-			   "Failed to insert into eviction hole: size=%d, align=%d, color=%lu, err=%d\n",
-			   size, alignment, color, err);
-		show_scan(test, &scan);
-		show_holes(test, mm, 3);
-		return err;
-	}
-
-	if (tmp.start < range_start || tmp.start + tmp.size > range_end) {
-		KUNIT_FAIL(test,
-			   "Inserted [address=%llu + %llu] did not fit into the request range [%llu, %llu]\n",
-			   tmp.start, tmp.size, range_start, range_end);
-		err = -EINVAL;
-	}
-
-	if (colors_abutt(test, &tmp))
-		err = -EINVAL;
-
-	if (!assert_node(test, &tmp, mm, size, alignment, color)) {
-		KUNIT_FAIL(test,
-			   "Inserted did not fit the eviction hole: size=%lld [%d], align=%d [rem=%lld], start=%llx\n",
-			   tmp.size, size, alignment, misalignment(&tmp, alignment), tmp.start);
-		err = -EINVAL;
-	}
-
-	drm_mm_remove_node(&tmp);
-	if (err)
-		return err;
-
-	list_for_each_entry(e, &evict_list, link) {
-		err = drm_mm_reserve_node(mm, &e->node);
-		if (err) {
-			KUNIT_FAIL(test, "Failed to reinsert node after eviction: start=%llx\n",
-				   e->node.start);
-			return err;
-		}
-	}
-
-	cond_resched();
-	return 0;
-}
-
-static void drm_test_mm_color_evict(struct kunit *test)
-{
-	DRM_RND_STATE(prng, random_seed);
-	const unsigned int total_size = min(8192u, max_iterations);
-	const struct insert_mode *mode;
-	unsigned long color = 0;
-	struct drm_mm mm;
-	struct evict_node *nodes;
-	struct drm_mm_node *node, *next;
-	unsigned int *order, n;
-
-	/* Check that the drm_mm_scan also honours color adjustment when
-	 * choosing its victims to create a hole. Our color_adjust does not
-	 * allow two nodes to be placed together without an intervening hole
-	 * enlarging the set of victims that must be evicted.
-	 */
-
-	nodes = vzalloc(array_size(total_size, sizeof(*nodes)));
-	KUNIT_ASSERT_TRUE(test, nodes);
-
-	order = drm_random_order(total_size, &prng);
-	if (!order)
-		goto err_nodes;
-
-	drm_mm_init(&mm, 0, 2 * total_size - 1);
-	mm.color_adjust = separate_adjacent_colors;
-	for (n = 0; n < total_size; n++) {
-		if (!expect_insert(test, &mm, &nodes[n].node,
-				   1, 0, color++,
-				   &insert_modes[0])) {
-			KUNIT_FAIL(test, "insert failed, step %d\n", n);
-			goto out;
-		}
-	}
-
-	for (mode = evict_modes; mode->name; mode++) {
-		for (n = 1; n <= total_size; n <<= 1) {
-			drm_random_reorder(order, total_size, &prng);
-			if (evict_color(test, &mm, 0, U64_MAX, nodes, order, total_size,
-					n, 1, color++, mode)) {
-				KUNIT_FAIL(test, "%s evict_color(size=%u) failed\n", mode->name, n);
-				goto out;
-			}
-		}
-
-		for (n = 1; n < total_size; n <<= 1) {
-			drm_random_reorder(order, total_size, &prng);
-			if (evict_color(test, &mm, 0, U64_MAX, nodes, order, total_size,
-					total_size / 2, n, color++, mode)) {
-				KUNIT_FAIL(test, "%s evict_color(size=%u, alignment=%u) failed\n",
-					   mode->name, total_size / 2, n);
-				goto out;
-			}
-		}
-
-		for_each_prime_number_from(n, 1, min(total_size, max_prime)) {
-			unsigned int nsize = (total_size - n + 1) / 2;
-
-			DRM_MM_BUG_ON(!nsize);
-
-			drm_random_reorder(order, total_size, &prng);
-			if (evict_color(test, &mm, 0, U64_MAX, nodes, order, total_size,
-					nsize, n, color++, mode)) {
-				KUNIT_FAIL(test, "%s evict_color(size=%u, alignment=%u) failed\n",
-					   mode->name, nsize, n);
-				goto out;
-			}
-		}
-
-		cond_resched();
-	}
-
-out:
-	drm_mm_for_each_node_safe(node, next, &mm)
-		drm_mm_remove_node(node);
-	drm_mm_takedown(&mm);
-	kfree(order);
-err_nodes:
-	vfree(nodes);
-}
-
-static void drm_test_mm_color_evict_range(struct kunit *test)
-{
-	DRM_RND_STATE(prng, random_seed);
-	const unsigned int total_size = 8192;
-	const unsigned int range_size = total_size / 2;
-	const unsigned int range_start = total_size / 4;
-	const unsigned int range_end = range_start + range_size;
-	const struct insert_mode *mode;
-	unsigned long color = 0;
-	struct drm_mm mm;
-	struct evict_node *nodes;
-	struct drm_mm_node *node, *next;
-	unsigned int *order, n;
-
-	/* Like drm_test_mm_color_evict(), but limited to small portion of the full
-	 * drm_mm range.
-	 */
-
-	nodes = vzalloc(array_size(total_size, sizeof(*nodes)));
-	KUNIT_ASSERT_TRUE(test, nodes);
-
-	order = drm_random_order(total_size, &prng);
-	if (!order)
-		goto err_nodes;
-
-	drm_mm_init(&mm, 0, 2 * total_size - 1);
-	mm.color_adjust = separate_adjacent_colors;
-	for (n = 0; n < total_size; n++) {
-		if (!expect_insert(test, &mm, &nodes[n].node,
-				   1, 0, color++,
-				   &insert_modes[0])) {
-			KUNIT_FAIL(test, "insert failed, step %d\n", n);
-			goto out;
-		}
-	}
-
-	for (mode = evict_modes; mode->name; mode++) {
-		for (n = 1; n <= range_size; n <<= 1) {
-			drm_random_reorder(order, range_size, &prng);
-			if (evict_color(test, &mm, range_start, range_end, nodes, order,
-					total_size, n, 1, color++, mode)) {
-				KUNIT_FAIL(test,
-					   "%s evict_color(size=%u) failed for range [%x, %x]\n",
-						mode->name, n, range_start, range_end);
-				goto out;
-			}
-		}
-
-		for (n = 1; n < range_size; n <<= 1) {
-			drm_random_reorder(order, total_size, &prng);
-			if (evict_color(test, &mm, range_start, range_end, nodes, order,
-					total_size, range_size / 2, n, color++, mode)) {
-				KUNIT_FAIL(test,
-					   "%s evict_color(size=%u, alignment=%u) failed for range [%x, %x]\n",
-					   mode->name, total_size / 2, n, range_start, range_end);
-				goto out;
-			}
-		}
-
-		for_each_prime_number_from(n, 1, min(range_size, max_prime)) {
-			unsigned int nsize = (range_size - n + 1) / 2;
-
-			DRM_MM_BUG_ON(!nsize);
-
-			drm_random_reorder(order, total_size, &prng);
-			if (evict_color(test, &mm, range_start, range_end, nodes, order,
-					total_size, nsize, n, color++, mode)) {
-				KUNIT_FAIL(test,
-					   "%s evict_color(size=%u, alignment=%u) failed for range [%x, %x]\n",
-					   mode->name, nsize, n, range_start, range_end);
-				goto out;
-			}
-		}
-
-		cond_resched();
-	}
-
-out:
-	drm_mm_for_each_node_safe(node, next, &mm)
-		drm_mm_remove_node(node);
-	drm_mm_takedown(&mm);
-	kfree(order);
-err_nodes:
-	vfree(nodes);
-}
-
-static int drm_mm_suite_init(struct kunit_suite *suite)
-{
-	while (!random_seed)
-		random_seed = get_random_u32();
-
-	kunit_info(suite,
-		   "Testing DRM range manager, with random_seed=0x%x max_iterations=%u max_prime=%u\n",
-		   random_seed, max_iterations, max_prime);
-
-	return 0;
-}
-
-module_param(random_seed, uint, 0400);
-module_param(max_iterations, uint, 0400);
-module_param(max_prime, uint, 0400);
-
 static struct kunit_case drm_mm_tests[] = {
 	KUNIT_CASE(drm_test_mm_init),
 	KUNIT_CASE(drm_test_mm_debug),
-	KUNIT_CASE(drm_test_mm_reserve),
-	KUNIT_CASE(drm_test_mm_insert),
-	KUNIT_CASE(drm_test_mm_replace),
-	KUNIT_CASE(drm_test_mm_insert_range),
-	KUNIT_CASE(drm_test_mm_frag),
-	KUNIT_CASE(drm_test_mm_align),
 	KUNIT_CASE(drm_test_mm_align32),
 	KUNIT_CASE(drm_test_mm_align64),
-	KUNIT_CASE(drm_test_mm_evict),
-	KUNIT_CASE(drm_test_mm_evict_range),
-	KUNIT_CASE(drm_test_mm_topdown),
-	KUNIT_CASE(drm_test_mm_bottomup),
 	KUNIT_CASE(drm_test_mm_lowest),
 	KUNIT_CASE(drm_test_mm_highest),
-	KUNIT_CASE(drm_test_mm_color),
-	KUNIT_CASE(drm_test_mm_color_evict),
-	KUNIT_CASE(drm_test_mm_color_evict_range),
 	{}
 };
 
 static struct kunit_suite drm_mm_test_suite = {
 	.name = "drm_mm",
-	.suite_init = drm_mm_suite_init,
 	.test_cases = drm_mm_tests,
 };
 
diff --git a/drivers/gpu/drm/tidss/tidss_kms.c b/drivers/gpu/drm/tidss/tidss_kms.c
index c979ad1af236..d096d8d2bc8f 100644
--- a/drivers/gpu/drm/tidss/tidss_kms.c
+++ b/drivers/gpu/drm/tidss/tidss_kms.c
@@ -4,8 +4,6 @@
  * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
  */
 
-#include <linux/dma-fence.h>
-
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
@@ -25,7 +23,6 @@ static void tidss_atomic_commit_tail(struct drm_atomic_state *old_state)
 {
 	struct drm_device *ddev = old_state->dev;
 	struct tidss_device *tidss = to_tidss(ddev);
-	bool fence_cookie = dma_fence_begin_signalling();
 
 	dev_dbg(ddev->dev, "%s\n", __func__);
 
@@ -36,7 +33,6 @@ static void tidss_atomic_commit_tail(struct drm_atomic_state *old_state)
 	drm_atomic_helper_commit_modeset_enables(ddev, old_state);
 
 	drm_atomic_helper_commit_hw_done(old_state);
-	dma_fence_end_signalling(fence_cookie);
 	drm_atomic_helper_wait_for_flip_done(ddev, old_state);
 
 	drm_atomic_helper_cleanup_planes(ddev, old_state);
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_drv.c b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
index 8ebd7134ee21..2f6eaac7f659 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_drv.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
@@ -138,7 +138,7 @@ static int tilcdc_irq_install(struct drm_device *dev, unsigned int irq)
 	if (ret)
 		return ret;
 
-	priv->irq_enabled = false;
+	priv->irq_enabled = true;
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/tiny/arcpgu.c b/drivers/gpu/drm/tiny/arcpgu.c
index e5b10e41554a..4f8f3172379e 100644
--- a/drivers/gpu/drm/tiny/arcpgu.c
+++ b/drivers/gpu/drm/tiny/arcpgu.c
@@ -404,14 +404,12 @@ static int arcpgu_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int arcpgu_remove(struct platform_device *pdev)
+static void arcpgu_remove(struct platform_device *pdev)
 {
 	struct drm_device *drm = platform_get_drvdata(pdev);
 
 	drm_dev_unregister(drm);
 	arcpgu_unload(drm);
-
-	return 0;
 }
 
 static const struct of_device_id arcpgu_of_table[] = {
@@ -423,7 +421,7 @@ MODULE_DEVICE_TABLE(of, arcpgu_of_table);
 
 static struct platform_driver arcpgu_platform_driver = {
 	.probe = arcpgu_probe,
-	.remove = arcpgu_remove,
+	.remove_new = arcpgu_remove,
 	.driver = {
 		   .name = "arcpgu",
 		   .of_match_table = arcpgu_of_table,
diff --git a/drivers/gpu/drm/tiny/cirrus.c b/drivers/gpu/drm/tiny/cirrus.c
index c5c34cd2edc1..4e3a152f897a 100644
--- a/drivers/gpu/drm/tiny/cirrus.c
+++ b/drivers/gpu/drm/tiny/cirrus.c
@@ -411,7 +411,8 @@ static void cirrus_primary_plane_helper_atomic_update(struct drm_plane *plane,
 		unsigned int offset = drm_fb_clip_offset(pitch, format, &damage);
 		struct iosys_map dst = IOSYS_MAP_INIT_OFFSET(&vaddr, offset);
 
-		drm_fb_blit(&dst, &pitch, format->format, shadow_plane_state->data, fb, &damage);
+		drm_fb_blit(&dst, &pitch, format->format, shadow_plane_state->data, fb,
+			    &damage, &shadow_plane_state->fmtcnv_state);
 	}
 
 	drm_dev_exit(idx);
diff --git a/drivers/gpu/drm/tiny/ili9225.c b/drivers/gpu/drm/tiny/ili9225.c
index 4ceb68ffac4b..dd8b0a181be9 100644
--- a/drivers/gpu/drm/tiny/ili9225.c
+++ b/drivers/gpu/drm/tiny/ili9225.c
@@ -78,7 +78,7 @@ static inline int ili9225_command(struct mipi_dbi *dbi, u8 cmd, u16 data)
 }
 
 static void ili9225_fb_dirty(struct iosys_map *src, struct drm_framebuffer *fb,
-			     struct drm_rect *rect)
+			     struct drm_rect *rect, struct drm_format_conv_state *fmtcnv_state)
 {
 	struct mipi_dbi_dev *dbidev = drm_to_mipi_dbi_dev(fb->dev);
 	unsigned int height = rect->y2 - rect->y1;
@@ -98,7 +98,7 @@ static void ili9225_fb_dirty(struct iosys_map *src, struct drm_framebuffer *fb,
 	if (!dbi->dc || !full || swap ||
 	    fb->format->format == DRM_FORMAT_XRGB8888) {
 		tr = dbidev->tx_buf;
-		ret = mipi_dbi_buf_copy(tr, src, fb, rect, swap);
+		ret = mipi_dbi_buf_copy(tr, src, fb, rect, swap, fmtcnv_state);
 		if (ret)
 			goto err_msg;
 	} else {
@@ -171,7 +171,8 @@ static void ili9225_pipe_update(struct drm_simple_display_pipe *pipe,
 		return;
 
 	if (drm_atomic_helper_damage_merged(old_state, state, &rect))
-		ili9225_fb_dirty(&shadow_plane_state->data[0], fb, &rect);
+		ili9225_fb_dirty(&shadow_plane_state->data[0], fb, &rect,
+				 &shadow_plane_state->fmtcnv_state);
 
 	drm_dev_exit(idx);
 }
@@ -281,7 +282,8 @@ static void ili9225_pipe_enable(struct drm_simple_display_pipe *pipe,
 
 	ili9225_command(dbi, ILI9225_DISPLAY_CONTROL_1, 0x1017);
 
-	ili9225_fb_dirty(&shadow_plane_state->data[0], fb, &rect);
+	ili9225_fb_dirty(&shadow_plane_state->data[0], fb, &rect,
+			 &shadow_plane_state->fmtcnv_state);
 
 out_exit:
 	drm_dev_exit(idx);
diff --git a/drivers/gpu/drm/tiny/ofdrm.c b/drivers/gpu/drm/tiny/ofdrm.c
index 2d999a0facde..05a72473cfc6 100644
--- a/drivers/gpu/drm/tiny/ofdrm.c
+++ b/drivers/gpu/drm/tiny/ofdrm.c
@@ -758,7 +758,11 @@ static const uint64_t ofdrm_primary_plane_format_modifiers[] = {
 static int ofdrm_primary_plane_helper_atomic_check(struct drm_plane *plane,
 						   struct drm_atomic_state *new_state)
 {
+	struct drm_device *dev = plane->dev;
+	struct ofdrm_device *odev = ofdrm_device_of_dev(dev);
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(new_state, plane);
+	struct drm_shadow_plane_state *new_shadow_plane_state =
+		to_drm_shadow_plane_state(new_plane_state);
 	struct drm_framebuffer *new_fb = new_plane_state->fb;
 	struct drm_crtc *new_crtc = new_plane_state->crtc;
 	struct drm_crtc_state *new_crtc_state = NULL;
@@ -777,6 +781,16 @@ static int ofdrm_primary_plane_helper_atomic_check(struct drm_plane *plane,
 	else if (!new_plane_state->visible)
 		return 0;
 
+	if (new_fb->format != odev->format) {
+		void *buf;
+
+		/* format conversion necessary; reserve buffer */
+		buf = drm_format_conv_state_reserve(&new_shadow_plane_state->fmtcnv_state,
+						    odev->pitch, GFP_KERNEL);
+		if (!buf)
+			return -ENOMEM;
+	}
+
 	new_crtc_state = drm_atomic_get_new_crtc_state(new_state, new_plane_state->crtc);
 
 	new_ofdrm_crtc_state = to_ofdrm_crtc_state(new_crtc_state);
@@ -817,7 +831,7 @@ static void ofdrm_primary_plane_helper_atomic_update(struct drm_plane *plane,
 
 		iosys_map_incr(&dst, drm_fb_clip_offset(dst_pitch, dst_format, &dst_clip));
 		drm_fb_blit(&dst, &dst_pitch, dst_format->format, shadow_plane_state->data, fb,
-			    &damage);
+			    &damage, &shadow_plane_state->fmtcnv_state);
 	}
 
 	drm_dev_exit(idx);
diff --git a/drivers/gpu/drm/tiny/repaper.c b/drivers/gpu/drm/tiny/repaper.c
index 73dd4f4289c2..8fd6758f5725 100644
--- a/drivers/gpu/drm/tiny/repaper.c
+++ b/drivers/gpu/drm/tiny/repaper.c
@@ -509,7 +509,8 @@ static void repaper_get_temperature(struct repaper_epd *epd)
 	epd->factored_stage_time = epd->stage_time * factor10x / 10;
 }
 
-static int repaper_fb_dirty(struct drm_framebuffer *fb)
+static int repaper_fb_dirty(struct drm_framebuffer *fb,
+			    struct drm_format_conv_state *fmtcnv_state)
 {
 	struct drm_gem_dma_object *dma_obj = drm_fb_dma_get_gem_obj(fb, 0);
 	struct repaper_epd *epd = drm_to_epd(fb->dev);
@@ -545,7 +546,7 @@ static int repaper_fb_dirty(struct drm_framebuffer *fb)
 
 	iosys_map_set_vaddr(&dst, buf);
 	iosys_map_set_vaddr(&vmap, dma_obj->vaddr);
-	drm_fb_xrgb8888_to_mono(&dst, &dst_pitch, &vmap, fb, &clip);
+	drm_fb_xrgb8888_to_mono(&dst, &dst_pitch, &vmap, fb, &clip, fmtcnv_state);
 
 	drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE);
 
@@ -830,13 +831,16 @@ static void repaper_pipe_update(struct drm_simple_display_pipe *pipe,
 				struct drm_plane_state *old_state)
 {
 	struct drm_plane_state *state = pipe->plane.state;
+	struct drm_format_conv_state fmtcnv_state = DRM_FORMAT_CONV_STATE_INIT;
 	struct drm_rect rect;
 
 	if (!pipe->crtc.state->active)
 		return;
 
 	if (drm_atomic_helper_damage_merged(old_state, state, &rect))
-		repaper_fb_dirty(state->fb);
+		repaper_fb_dirty(state->fb, &fmtcnv_state);
+
+	drm_format_conv_state_release(&fmtcnv_state);
 }
 
 static const struct drm_simple_display_pipe_funcs repaper_pipe_funcs = {
diff --git a/drivers/gpu/drm/tiny/simpledrm.c b/drivers/gpu/drm/tiny/simpledrm.c
index 5fefc895bca2..34bbbd7b53dd 100644
--- a/drivers/gpu/drm/tiny/simpledrm.c
+++ b/drivers/gpu/drm/tiny/simpledrm.c
@@ -19,6 +19,7 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_fbdev_generic.h>
 #include <drm/drm_format_helper.h>
+#include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_gem_shmem_helper.h>
@@ -579,6 +580,44 @@ static const uint64_t simpledrm_primary_plane_format_modifiers[] = {
 	DRM_FORMAT_MOD_INVALID
 };
 
+static int simpledrm_primary_plane_helper_atomic_check(struct drm_plane *plane,
+						       struct drm_atomic_state *state)
+{
+	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, plane);
+	struct drm_shadow_plane_state *new_shadow_plane_state =
+		to_drm_shadow_plane_state(new_plane_state);
+	struct drm_framebuffer *new_fb = new_plane_state->fb;
+	struct drm_crtc *new_crtc = new_plane_state->crtc;
+	struct drm_crtc_state *new_crtc_state = NULL;
+	struct drm_device *dev = plane->dev;
+	struct simpledrm_device *sdev = simpledrm_device_of_dev(dev);
+	int ret;
+
+	if (new_crtc)
+		new_crtc_state = drm_atomic_get_new_crtc_state(state, new_crtc);
+
+	ret = drm_atomic_helper_check_plane_state(new_plane_state, new_crtc_state,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  false, false);
+	if (ret)
+		return ret;
+	else if (!new_plane_state->visible)
+		return 0;
+
+	if (new_fb->format != sdev->format) {
+		void *buf;
+
+		/* format conversion necessary; reserve buffer */
+		buf = drm_format_conv_state_reserve(&new_shadow_plane_state->fmtcnv_state,
+						    sdev->pitch, GFP_KERNEL);
+		if (!buf)
+			return -ENOMEM;
+	}
+
+	return 0;
+}
+
 static void simpledrm_primary_plane_helper_atomic_update(struct drm_plane *plane,
 							 struct drm_atomic_state *state)
 {
@@ -609,7 +648,7 @@ static void simpledrm_primary_plane_helper_atomic_update(struct drm_plane *plane
 
 		iosys_map_incr(&dst, drm_fb_clip_offset(sdev->pitch, sdev->format, &dst_clip));
 		drm_fb_blit(&dst, &sdev->pitch, sdev->format->format, shadow_plane_state->data,
-			    fb, &damage);
+			    fb, &damage, &shadow_plane_state->fmtcnv_state);
 	}
 
 	drm_dev_exit(idx);
@@ -635,7 +674,7 @@ static void simpledrm_primary_plane_helper_atomic_disable(struct drm_plane *plan
 
 static const struct drm_plane_helper_funcs simpledrm_primary_plane_helper_funcs = {
 	DRM_GEM_SHADOW_PLANE_HELPER_FUNCS,
-	.atomic_check = drm_plane_helper_atomic_check,
+	.atomic_check = simpledrm_primary_plane_helper_atomic_check,
 	.atomic_update = simpledrm_primary_plane_helper_atomic_update,
 	.atomic_disable = simpledrm_primary_plane_helper_atomic_disable,
 };
diff --git a/drivers/gpu/drm/tiny/st7586.c b/drivers/gpu/drm/tiny/st7586.c
index 3cf4eec16a81..7336fa1ddaed 100644
--- a/drivers/gpu/drm/tiny/st7586.c
+++ b/drivers/gpu/drm/tiny/st7586.c
@@ -64,7 +64,8 @@ static const u8 st7586_lookup[] = { 0x7, 0x4, 0x2, 0x0 };
 
 static void st7586_xrgb8888_to_gray332(u8 *dst, void *vaddr,
 				       struct drm_framebuffer *fb,
-				       struct drm_rect *clip)
+				       struct drm_rect *clip,
+				       struct drm_format_conv_state *fmtcnv_state)
 {
 	size_t len = (clip->x2 - clip->x1) * (clip->y2 - clip->y1);
 	unsigned int x, y;
@@ -77,7 +78,7 @@ static void st7586_xrgb8888_to_gray332(u8 *dst, void *vaddr,
 
 	iosys_map_set_vaddr(&dst_map, buf);
 	iosys_map_set_vaddr(&vmap, vaddr);
-	drm_fb_xrgb8888_to_gray8(&dst_map, NULL, &vmap, fb, clip);
+	drm_fb_xrgb8888_to_gray8(&dst_map, NULL, &vmap, fb, clip, fmtcnv_state);
 	src = buf;
 
 	for (y = clip->y1; y < clip->y2; y++) {
@@ -93,7 +94,7 @@ static void st7586_xrgb8888_to_gray332(u8 *dst, void *vaddr,
 }
 
 static int st7586_buf_copy(void *dst, struct iosys_map *src, struct drm_framebuffer *fb,
-			   struct drm_rect *clip)
+			   struct drm_rect *clip, struct drm_format_conv_state *fmtcnv_state)
 {
 	int ret;
 
@@ -101,7 +102,7 @@ static int st7586_buf_copy(void *dst, struct iosys_map *src, struct drm_framebuf
 	if (ret)
 		return ret;
 
-	st7586_xrgb8888_to_gray332(dst, src->vaddr, fb, clip);
+	st7586_xrgb8888_to_gray332(dst, src->vaddr, fb, clip, fmtcnv_state);
 
 	drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE);
 
@@ -109,7 +110,7 @@ static int st7586_buf_copy(void *dst, struct iosys_map *src, struct drm_framebuf
 }
 
 static void st7586_fb_dirty(struct iosys_map *src, struct drm_framebuffer *fb,
-			    struct drm_rect *rect)
+			    struct drm_rect *rect, struct drm_format_conv_state *fmtcnv_state)
 {
 	struct mipi_dbi_dev *dbidev = drm_to_mipi_dbi_dev(fb->dev);
 	struct mipi_dbi *dbi = &dbidev->dbi;
@@ -121,7 +122,7 @@ static void st7586_fb_dirty(struct iosys_map *src, struct drm_framebuffer *fb,
 
 	DRM_DEBUG_KMS("Flushing [FB:%d] " DRM_RECT_FMT "\n", fb->base.id, DRM_RECT_ARG(rect));
 
-	ret = st7586_buf_copy(dbidev->tx_buf, src, fb, rect);
+	ret = st7586_buf_copy(dbidev->tx_buf, src, fb, rect, fmtcnv_state);
 	if (ret)
 		goto err_msg;
 
@@ -160,7 +161,8 @@ static void st7586_pipe_update(struct drm_simple_display_pipe *pipe,
 		return;
 
 	if (drm_atomic_helper_damage_merged(old_state, state, &rect))
-		st7586_fb_dirty(&shadow_plane_state->data[0], fb, &rect);
+		st7586_fb_dirty(&shadow_plane_state->data[0], fb, &rect,
+				&shadow_plane_state->fmtcnv_state);
 
 	drm_dev_exit(idx);
 }
@@ -238,7 +240,8 @@ static void st7586_pipe_enable(struct drm_simple_display_pipe *pipe,
 
 	msleep(100);
 
-	st7586_fb_dirty(&shadow_plane_state->data[0], fb, &rect);
+	st7586_fb_dirty(&shadow_plane_state->data[0], fb, &rect,
+			&shadow_plane_state->fmtcnv_state);
 
 	mipi_dbi_command(dbi, MIPI_DCS_SET_DISPLAY_ON);
 out_exit:
diff --git a/drivers/gpu/drm/v3d/Makefile b/drivers/gpu/drm/v3d/Makefile
index e8b314137020..4b21b20e4998 100644
--- a/drivers/gpu/drm/v3d/Makefile
+++ b/drivers/gpu/drm/v3d/Makefile
@@ -11,7 +11,8 @@ v3d-y := \
 	v3d_mmu.o \
 	v3d_perfmon.o \
 	v3d_trace_points.o \
-	v3d_sched.o
+	v3d_sched.o \
+	v3d_sysfs.o
 
 v3d-$(CONFIG_DEBUG_FS) += v3d_debugfs.o
 
diff --git a/drivers/gpu/drm/v3d/v3d_debugfs.c b/drivers/gpu/drm/v3d/v3d_debugfs.c
index 330669f51fa7..f843a50d5dce 100644
--- a/drivers/gpu/drm/v3d/v3d_debugfs.c
+++ b/drivers/gpu/drm/v3d/v3d_debugfs.c
@@ -12,69 +12,83 @@
 #include "v3d_drv.h"
 #include "v3d_regs.h"
 
-#define REGDEF(reg) { reg, #reg }
+#define REGDEF(min_ver, max_ver, reg) { min_ver, max_ver, reg, #reg }
 struct v3d_reg_def {
+	u32 min_ver;
+	u32 max_ver;
 	u32 reg;
 	const char *name;
 };
 
 static const struct v3d_reg_def v3d_hub_reg_defs[] = {
-	REGDEF(V3D_HUB_AXICFG),
-	REGDEF(V3D_HUB_UIFCFG),
-	REGDEF(V3D_HUB_IDENT0),
-	REGDEF(V3D_HUB_IDENT1),
-	REGDEF(V3D_HUB_IDENT2),
-	REGDEF(V3D_HUB_IDENT3),
-	REGDEF(V3D_HUB_INT_STS),
-	REGDEF(V3D_HUB_INT_MSK_STS),
-
-	REGDEF(V3D_MMU_CTL),
-	REGDEF(V3D_MMU_VIO_ADDR),
-	REGDEF(V3D_MMU_VIO_ID),
-	REGDEF(V3D_MMU_DEBUG_INFO),
+	REGDEF(33, 42, V3D_HUB_AXICFG),
+	REGDEF(33, 71, V3D_HUB_UIFCFG),
+	REGDEF(33, 71, V3D_HUB_IDENT0),
+	REGDEF(33, 71, V3D_HUB_IDENT1),
+	REGDEF(33, 71, V3D_HUB_IDENT2),
+	REGDEF(33, 71, V3D_HUB_IDENT3),
+	REGDEF(33, 71, V3D_HUB_INT_STS),
+	REGDEF(33, 71, V3D_HUB_INT_MSK_STS),
+
+	REGDEF(33, 71, V3D_MMU_CTL),
+	REGDEF(33, 71, V3D_MMU_VIO_ADDR),
+	REGDEF(33, 71, V3D_MMU_VIO_ID),
+	REGDEF(33, 71, V3D_MMU_DEBUG_INFO),
+
+	REGDEF(71, 71, V3D_GMP_STATUS(71)),
+	REGDEF(71, 71, V3D_GMP_CFG(71)),
+	REGDEF(71, 71, V3D_GMP_VIO_ADDR(71)),
 };
 
 static const struct v3d_reg_def v3d_gca_reg_defs[] = {
-	REGDEF(V3D_GCA_SAFE_SHUTDOWN),
-	REGDEF(V3D_GCA_SAFE_SHUTDOWN_ACK),
+	REGDEF(33, 33, V3D_GCA_SAFE_SHUTDOWN),
+	REGDEF(33, 33, V3D_GCA_SAFE_SHUTDOWN_ACK),
 };
 
 static const struct v3d_reg_def v3d_core_reg_defs[] = {
-	REGDEF(V3D_CTL_IDENT0),
-	REGDEF(V3D_CTL_IDENT1),
-	REGDEF(V3D_CTL_IDENT2),
-	REGDEF(V3D_CTL_MISCCFG),
-	REGDEF(V3D_CTL_INT_STS),
-	REGDEF(V3D_CTL_INT_MSK_STS),
-	REGDEF(V3D_CLE_CT0CS),
-	REGDEF(V3D_CLE_CT0CA),
-	REGDEF(V3D_CLE_CT0EA),
-	REGDEF(V3D_CLE_CT1CS),
-	REGDEF(V3D_CLE_CT1CA),
-	REGDEF(V3D_CLE_CT1EA),
-
-	REGDEF(V3D_PTB_BPCA),
-	REGDEF(V3D_PTB_BPCS),
-
-	REGDEF(V3D_GMP_STATUS),
-	REGDEF(V3D_GMP_CFG),
-	REGDEF(V3D_GMP_VIO_ADDR),
-
-	REGDEF(V3D_ERR_FDBGO),
-	REGDEF(V3D_ERR_FDBGB),
-	REGDEF(V3D_ERR_FDBGS),
-	REGDEF(V3D_ERR_STAT),
+	REGDEF(33, 71, V3D_CTL_IDENT0),
+	REGDEF(33, 71, V3D_CTL_IDENT1),
+	REGDEF(33, 71, V3D_CTL_IDENT2),
+	REGDEF(33, 71, V3D_CTL_MISCCFG),
+	REGDEF(33, 71, V3D_CTL_INT_STS),
+	REGDEF(33, 71, V3D_CTL_INT_MSK_STS),
+	REGDEF(33, 71, V3D_CLE_CT0CS),
+	REGDEF(33, 71, V3D_CLE_CT0CA),
+	REGDEF(33, 71, V3D_CLE_CT0EA),
+	REGDEF(33, 71, V3D_CLE_CT1CS),
+	REGDEF(33, 71, V3D_CLE_CT1CA),
+	REGDEF(33, 71, V3D_CLE_CT1EA),
+
+	REGDEF(33, 71, V3D_PTB_BPCA),
+	REGDEF(33, 71, V3D_PTB_BPCS),
+
+	REGDEF(33, 41, V3D_GMP_STATUS(33)),
+	REGDEF(33, 41, V3D_GMP_CFG(33)),
+	REGDEF(33, 41, V3D_GMP_VIO_ADDR(33)),
+
+	REGDEF(33, 71, V3D_ERR_FDBGO),
+	REGDEF(33, 71, V3D_ERR_FDBGB),
+	REGDEF(33, 71, V3D_ERR_FDBGS),
+	REGDEF(33, 71, V3D_ERR_STAT),
 };
 
 static const struct v3d_reg_def v3d_csd_reg_defs[] = {
-	REGDEF(V3D_CSD_STATUS),
-	REGDEF(V3D_CSD_CURRENT_CFG0),
-	REGDEF(V3D_CSD_CURRENT_CFG1),
-	REGDEF(V3D_CSD_CURRENT_CFG2),
-	REGDEF(V3D_CSD_CURRENT_CFG3),
-	REGDEF(V3D_CSD_CURRENT_CFG4),
-	REGDEF(V3D_CSD_CURRENT_CFG5),
-	REGDEF(V3D_CSD_CURRENT_CFG6),
+	REGDEF(41, 71, V3D_CSD_STATUS),
+	REGDEF(41, 41, V3D_CSD_CURRENT_CFG0(41)),
+	REGDEF(41, 41, V3D_CSD_CURRENT_CFG1(41)),
+	REGDEF(41, 41, V3D_CSD_CURRENT_CFG2(41)),
+	REGDEF(41, 41, V3D_CSD_CURRENT_CFG3(41)),
+	REGDEF(41, 41, V3D_CSD_CURRENT_CFG4(41)),
+	REGDEF(41, 41, V3D_CSD_CURRENT_CFG5(41)),
+	REGDEF(41, 41, V3D_CSD_CURRENT_CFG6(41)),
+	REGDEF(71, 71, V3D_CSD_CURRENT_CFG0(71)),
+	REGDEF(71, 71, V3D_CSD_CURRENT_CFG1(71)),
+	REGDEF(71, 71, V3D_CSD_CURRENT_CFG2(71)),
+	REGDEF(71, 71, V3D_CSD_CURRENT_CFG3(71)),
+	REGDEF(71, 71, V3D_CSD_CURRENT_CFG4(71)),
+	REGDEF(71, 71, V3D_CSD_CURRENT_CFG5(71)),
+	REGDEF(71, 71, V3D_CSD_CURRENT_CFG6(71)),
+	REGDEF(71, 71, V3D_V7_CSD_CURRENT_CFG7),
 };
 
 static int v3d_v3d_debugfs_regs(struct seq_file *m, void *unused)
@@ -85,38 +99,41 @@ static int v3d_v3d_debugfs_regs(struct seq_file *m, void *unused)
 	int i, core;
 
 	for (i = 0; i < ARRAY_SIZE(v3d_hub_reg_defs); i++) {
-		seq_printf(m, "%s (0x%04x): 0x%08x\n",
-			   v3d_hub_reg_defs[i].name, v3d_hub_reg_defs[i].reg,
-			   V3D_READ(v3d_hub_reg_defs[i].reg));
+		const struct v3d_reg_def *def = &v3d_hub_reg_defs[i];
+
+		if (v3d->ver >= def->min_ver && v3d->ver <= def->max_ver) {
+			seq_printf(m, "%s (0x%04x): 0x%08x\n",
+				   def->name, def->reg, V3D_READ(def->reg));
+		}
 	}
 
-	if (v3d->ver < 41) {
-		for (i = 0; i < ARRAY_SIZE(v3d_gca_reg_defs); i++) {
+	for (i = 0; i < ARRAY_SIZE(v3d_gca_reg_defs); i++) {
+		const struct v3d_reg_def *def = &v3d_gca_reg_defs[i];
+
+		if (v3d->ver >= def->min_ver && v3d->ver <= def->max_ver) {
 			seq_printf(m, "%s (0x%04x): 0x%08x\n",
-				   v3d_gca_reg_defs[i].name,
-				   v3d_gca_reg_defs[i].reg,
-				   V3D_GCA_READ(v3d_gca_reg_defs[i].reg));
+				   def->name, def->reg, V3D_GCA_READ(def->reg));
 		}
 	}
 
 	for (core = 0; core < v3d->cores; core++) {
 		for (i = 0; i < ARRAY_SIZE(v3d_core_reg_defs); i++) {
-			seq_printf(m, "core %d %s (0x%04x): 0x%08x\n",
-				   core,
-				   v3d_core_reg_defs[i].name,
-				   v3d_core_reg_defs[i].reg,
-				   V3D_CORE_READ(core,
-						 v3d_core_reg_defs[i].reg));
+			const struct v3d_reg_def *def = &v3d_core_reg_defs[i];
+
+			if (v3d->ver >= def->min_ver && v3d->ver <= def->max_ver) {
+				seq_printf(m, "core %d %s (0x%04x): 0x%08x\n",
+					   core, def->name, def->reg,
+					   V3D_CORE_READ(core, def->reg));
+			}
 		}
 
-		if (v3d_has_csd(v3d)) {
-			for (i = 0; i < ARRAY_SIZE(v3d_csd_reg_defs); i++) {
+		for (i = 0; i < ARRAY_SIZE(v3d_csd_reg_defs); i++) {
+			const struct v3d_reg_def *def = &v3d_csd_reg_defs[i];
+
+			if (v3d->ver >= def->min_ver && v3d->ver <= def->max_ver) {
 				seq_printf(m, "core %d %s (0x%04x): 0x%08x\n",
-					   core,
-					   v3d_csd_reg_defs[i].name,
-					   v3d_csd_reg_defs[i].reg,
-					   V3D_CORE_READ(core,
-							 v3d_csd_reg_defs[i].reg));
+					   core, def->name, def->reg,
+					   V3D_CORE_READ(core, def->reg));
 			}
 		}
 	}
@@ -147,8 +164,10 @@ static int v3d_v3d_debugfs_ident(struct seq_file *m, void *unused)
 		   str_yes_no(ident2 & V3D_HUB_IDENT2_WITH_MMU));
 	seq_printf(m, "TFU:        %s\n",
 		   str_yes_no(ident1 & V3D_HUB_IDENT1_WITH_TFU));
-	seq_printf(m, "TSY:        %s\n",
-		   str_yes_no(ident1 & V3D_HUB_IDENT1_WITH_TSY));
+	if (v3d->ver <= 42) {
+		seq_printf(m, "TSY:        %s\n",
+			   str_yes_no(ident1 & V3D_HUB_IDENT1_WITH_TSY));
+	}
 	seq_printf(m, "MSO:        %s\n",
 		   str_yes_no(ident1 & V3D_HUB_IDENT1_WITH_MSO));
 	seq_printf(m, "L3C:        %s (%dkb)\n",
@@ -177,10 +196,14 @@ static int v3d_v3d_debugfs_ident(struct seq_file *m, void *unused)
 		seq_printf(m, "  QPUs:         %d\n", nslc * qups);
 		seq_printf(m, "  Semaphores:   %d\n",
 			   V3D_GET_FIELD(ident1, V3D_IDENT1_NSEM));
-		seq_printf(m, "  BCG int:      %d\n",
-			   (ident2 & V3D_IDENT2_BCG_INT) != 0);
-		seq_printf(m, "  Override TMU: %d\n",
-			   (misccfg & V3D_MISCCFG_OVRTMUOUT) != 0);
+		if (v3d->ver <= 42) {
+			seq_printf(m, "  BCG int:      %d\n",
+				   (ident2 & V3D_IDENT2_BCG_INT) != 0);
+		}
+		if (v3d->ver < 40) {
+			seq_printf(m, "  Override TMU: %d\n",
+				   (misccfg & V3D_MISCCFG_OVRTMUOUT) != 0);
+		}
 	}
 
 	return 0;
@@ -212,14 +235,15 @@ static int v3d_measure_clock(struct seq_file *m, void *unused)
 	int measure_ms = 1000;
 
 	if (v3d->ver >= 40) {
+		int cycle_count_reg = V3D_PCTR_CYCLE_COUNT(v3d->ver);
 		V3D_CORE_WRITE(core, V3D_V4_PCTR_0_SRC_0_3,
-			       V3D_SET_FIELD(V3D_PCTR_CYCLE_COUNT,
+			       V3D_SET_FIELD(cycle_count_reg,
 					     V3D_PCTR_S0));
 		V3D_CORE_WRITE(core, V3D_V4_PCTR_0_CLR, 1);
 		V3D_CORE_WRITE(core, V3D_V4_PCTR_0_EN, 1);
 	} else {
 		V3D_CORE_WRITE(core, V3D_V3_PCTR_0_PCTRS0,
-			       V3D_PCTR_CYCLE_COUNT);
+			       V3D_PCTR_CYCLE_COUNT(v3d->ver));
 		V3D_CORE_WRITE(core, V3D_V3_PCTR_0_CLR, 1);
 		V3D_CORE_WRITE(core, V3D_V3_PCTR_0_EN,
 			       V3D_V3_PCTR_0_EN_ENABLE |
diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c
index ffbbe9d527d3..44a1ca57d6a4 100644
--- a/drivers/gpu/drm/v3d/v3d_drv.c
+++ b/drivers/gpu/drm/v3d/v3d_drv.c
@@ -19,6 +19,7 @@
 #include <linux/module.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
+#include <linux/sched/clock.h>
 #include <linux/reset.h>
 
 #include <drm/drm_drv.h>
@@ -111,6 +112,10 @@ v3d_open(struct drm_device *dev, struct drm_file *file)
 	v3d_priv->v3d = v3d;
 
 	for (i = 0; i < V3D_MAX_QUEUES; i++) {
+		v3d_priv->enabled_ns[i] = 0;
+		v3d_priv->start_ns[i] = 0;
+		v3d_priv->jobs_sent[i] = 0;
+
 		sched = &v3d->queue[i].sched;
 		drm_sched_entity_init(&v3d_priv->sched_entity[i],
 				      DRM_SCHED_PRIORITY_NORMAL, &sched,
@@ -136,7 +141,35 @@ v3d_postclose(struct drm_device *dev, struct drm_file *file)
 	kfree(v3d_priv);
 }
 
-DEFINE_DRM_GEM_FOPS(v3d_drm_fops);
+static void v3d_show_fdinfo(struct drm_printer *p, struct drm_file *file)
+{
+	struct v3d_file_priv *file_priv = file->driver_priv;
+	u64 timestamp = local_clock();
+	enum v3d_queue queue;
+
+	for (queue = 0; queue < V3D_MAX_QUEUES; queue++) {
+		/* Note that, in case of a GPU reset, the time spent during an
+		 * attempt of executing the job is not computed in the runtime.
+		 */
+		drm_printf(p, "drm-engine-%s: \t%llu ns\n",
+			   v3d_queue_to_string(queue),
+			   file_priv->start_ns[queue] ? file_priv->enabled_ns[queue]
+						      + timestamp - file_priv->start_ns[queue]
+						      : file_priv->enabled_ns[queue]);
+
+		/* Note that we only count jobs that completed. Therefore, jobs
+		 * that were resubmitted due to a GPU reset are not computed.
+		 */
+		drm_printf(p, "v3d-jobs-%s: \t%llu jobs\n",
+			   v3d_queue_to_string(queue), file_priv->jobs_sent[queue]);
+	}
+}
+
+static const struct file_operations v3d_drm_fops = {
+	.owner = THIS_MODULE,
+	DRM_GEM_FOPS,
+	.show_fdinfo = drm_show_fdinfo,
+};
 
 /* DRM_AUTH is required on SUBMIT_CL for now, while we don't have GMP
  * protection between clients.  Note that render nodes would be
@@ -176,6 +209,7 @@ static const struct drm_driver v3d_drm_driver = {
 	.ioctls = v3d_drm_ioctls,
 	.num_ioctls = ARRAY_SIZE(v3d_drm_ioctls),
 	.fops = &v3d_drm_fops,
+	.show_fdinfo = v3d_show_fdinfo,
 
 	.name = DRIVER_NAME,
 	.desc = DRIVER_DESC,
@@ -187,6 +221,7 @@ static const struct drm_driver v3d_drm_driver = {
 
 static const struct of_device_id v3d_of_match[] = {
 	{ .compatible = "brcm,2711-v3d" },
+	{ .compatible = "brcm,2712-v3d" },
 	{ .compatible = "brcm,7268-v3d" },
 	{ .compatible = "brcm,7278-v3d" },
 	{},
@@ -281,8 +316,14 @@ static int v3d_platform_drm_probe(struct platform_device *pdev)
 	if (ret)
 		goto irq_disable;
 
+	ret = v3d_sysfs_init(dev);
+	if (ret)
+		goto drm_unregister;
+
 	return 0;
 
+drm_unregister:
+	drm_dev_unregister(drm);
 irq_disable:
 	v3d_irq_disable(v3d);
 gem_destroy:
@@ -296,6 +337,9 @@ static void v3d_platform_drm_remove(struct platform_device *pdev)
 {
 	struct drm_device *drm = platform_get_drvdata(pdev);
 	struct v3d_dev *v3d = to_v3d_dev(drm);
+	struct device *dev = &pdev->dev;
+
+	v3d_sysfs_destroy(dev);
 
 	drm_dev_unregister(drm);
 
diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h
index 106454f28956..4c59fefaa0b4 100644
--- a/drivers/gpu/drm/v3d/v3d_drv.h
+++ b/drivers/gpu/drm/v3d/v3d_drv.h
@@ -21,11 +21,27 @@ struct reset_control;
 
 #define V3D_MAX_QUEUES (V3D_CACHE_CLEAN + 1)
 
+static inline char *v3d_queue_to_string(enum v3d_queue queue)
+{
+	switch (queue) {
+	case V3D_BIN: return "bin";
+	case V3D_RENDER: return "render";
+	case V3D_TFU: return "tfu";
+	case V3D_CSD: return "csd";
+	case V3D_CACHE_CLEAN: return "cache_clean";
+	}
+	return "UNKNOWN";
+}
+
 struct v3d_queue_state {
 	struct drm_gpu_scheduler sched;
 
 	u64 fence_context;
 	u64 emit_seqno;
+
+	u64 start_ns;
+	u64 enabled_ns;
+	u64 jobs_sent;
 };
 
 /* Performance monitor object. The perform lifetime is controlled by userspace
@@ -167,6 +183,12 @@ struct v3d_file_priv {
 	} perfmon;
 
 	struct drm_sched_entity sched_entity[V3D_MAX_QUEUES];
+
+	u64 start_ns[V3D_MAX_QUEUES];
+
+	u64 enabled_ns[V3D_MAX_QUEUES];
+
+	u64 jobs_sent[V3D_MAX_QUEUES];
 };
 
 struct v3d_bo {
@@ -238,6 +260,11 @@ struct v3d_job {
 	 */
 	struct v3d_perfmon *perfmon;
 
+	/* File descriptor of the process that submitted the job that could be used
+	 * for collecting stats by process of GPU usage.
+	 */
+	struct drm_file *file;
+
 	/* Callback for the freeing of the job on refcount going to 0. */
 	void (*free)(struct kref *ref);
 };
@@ -418,3 +445,7 @@ int v3d_perfmon_destroy_ioctl(struct drm_device *dev, void *data,
 			      struct drm_file *file_priv);
 int v3d_perfmon_get_values_ioctl(struct drm_device *dev, void *data,
 				 struct drm_file *file_priv);
+
+/* v3d_sysfs.c */
+int v3d_sysfs_init(struct device *dev);
+void v3d_sysfs_destroy(struct device *dev);
diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c
index 2e94ce788c71..9d2ac23c29e3 100644
--- a/drivers/gpu/drm/v3d/v3d_gem.c
+++ b/drivers/gpu/drm/v3d/v3d_gem.c
@@ -47,9 +47,9 @@ v3d_init_hw_state(struct v3d_dev *v3d)
 static void
 v3d_idle_axi(struct v3d_dev *v3d, int core)
 {
-	V3D_CORE_WRITE(core, V3D_GMP_CFG, V3D_GMP_CFG_STOP_REQ);
+	V3D_CORE_WRITE(core, V3D_GMP_CFG(v3d->ver), V3D_GMP_CFG_STOP_REQ);
 
-	if (wait_for((V3D_CORE_READ(core, V3D_GMP_STATUS) &
+	if (wait_for((V3D_CORE_READ(core, V3D_GMP_STATUS(v3d->ver)) &
 		      (V3D_GMP_STATUS_RD_COUNT_MASK |
 		       V3D_GMP_STATUS_WR_COUNT_MASK |
 		       V3D_GMP_STATUS_CFG_BUSY)) == 0, 100)) {
@@ -415,9 +415,10 @@ v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv,
 	job = *container;
 	job->v3d = v3d;
 	job->free = free;
+	job->file = file_priv;
 
 	ret = drm_sched_job_init(&job->base, &v3d_priv->sched_entity[queue],
-				 v3d_priv);
+				 1, v3d_priv);
 	if (ret)
 		goto fail;
 
@@ -1013,8 +1014,12 @@ v3d_gem_init(struct drm_device *dev)
 	u32 pt_size = 4096 * 1024;
 	int ret, i;
 
-	for (i = 0; i < V3D_MAX_QUEUES; i++)
+	for (i = 0; i < V3D_MAX_QUEUES; i++) {
 		v3d->queue[i].fence_context = dma_fence_context_alloc(1);
+		v3d->queue[i].start_ns = 0;
+		v3d->queue[i].enabled_ns = 0;
+		v3d->queue[i].jobs_sent = 0;
+	}
 
 	spin_lock_init(&v3d->mm_lock);
 	spin_lock_init(&v3d->job_lock);
@@ -1072,6 +1077,8 @@ v3d_gem_destroy(struct drm_device *dev)
 	 */
 	WARN_ON(v3d->bin_job);
 	WARN_ON(v3d->render_job);
+	WARN_ON(v3d->tfu_job);
+	WARN_ON(v3d->csd_job);
 
 	drm_mm_takedown(&v3d->mm);
 
diff --git a/drivers/gpu/drm/v3d/v3d_irq.c b/drivers/gpu/drm/v3d/v3d_irq.c
index e714d5318f30..afc76390a197 100644
--- a/drivers/gpu/drm/v3d/v3d_irq.c
+++ b/drivers/gpu/drm/v3d/v3d_irq.c
@@ -14,21 +14,23 @@
  */
 
 #include <linux/platform_device.h>
+#include <linux/sched/clock.h>
 
 #include "v3d_drv.h"
 #include "v3d_regs.h"
 #include "v3d_trace.h"
 
-#define V3D_CORE_IRQS ((u32)(V3D_INT_OUTOMEM |	\
-			     V3D_INT_FLDONE |	\
-			     V3D_INT_FRDONE |	\
-			     V3D_INT_CSDDONE |	\
-			     V3D_INT_GMPV))
+#define V3D_CORE_IRQS(ver) ((u32)(V3D_INT_OUTOMEM |	\
+				  V3D_INT_FLDONE |	\
+				  V3D_INT_FRDONE |	\
+				  V3D_INT_CSDDONE(ver) |	\
+				  (ver < 71 ? V3D_INT_GMPV : 0)))
 
-#define V3D_HUB_IRQS ((u32)(V3D_HUB_INT_MMU_WRV |	\
-			    V3D_HUB_INT_MMU_PTI |	\
-			    V3D_HUB_INT_MMU_CAP |	\
-			    V3D_HUB_INT_TFUC))
+#define V3D_HUB_IRQS(ver) ((u32)(V3D_HUB_INT_MMU_WRV |	\
+				 V3D_HUB_INT_MMU_PTI |	\
+				 V3D_HUB_INT_MMU_CAP |	\
+				 V3D_HUB_INT_TFUC |		\
+				 (ver >= 71 ? V3D_V7_HUB_INT_GMPV : 0)))
 
 static irqreturn_t
 v3d_hub_irq(int irq, void *arg);
@@ -100,6 +102,18 @@ v3d_irq(int irq, void *arg)
 	if (intsts & V3D_INT_FLDONE) {
 		struct v3d_fence *fence =
 			to_v3d_fence(v3d->bin_job->base.irq_fence);
+		struct v3d_file_priv *file = v3d->bin_job->base.file->driver_priv;
+		u64 runtime = local_clock() - file->start_ns[V3D_BIN];
+
+		file->enabled_ns[V3D_BIN] += local_clock() - file->start_ns[V3D_BIN];
+		file->jobs_sent[V3D_BIN]++;
+		v3d->queue[V3D_BIN].jobs_sent++;
+
+		file->start_ns[V3D_BIN] = 0;
+		v3d->queue[V3D_BIN].start_ns = 0;
+
+		file->enabled_ns[V3D_BIN] += runtime;
+		v3d->queue[V3D_BIN].enabled_ns += runtime;
 
 		trace_v3d_bcl_irq(&v3d->drm, fence->seqno);
 		dma_fence_signal(&fence->base);
@@ -109,15 +123,39 @@ v3d_irq(int irq, void *arg)
 	if (intsts & V3D_INT_FRDONE) {
 		struct v3d_fence *fence =
 			to_v3d_fence(v3d->render_job->base.irq_fence);
+		struct v3d_file_priv *file = v3d->render_job->base.file->driver_priv;
+		u64 runtime = local_clock() - file->start_ns[V3D_RENDER];
+
+		file->enabled_ns[V3D_RENDER] += local_clock() - file->start_ns[V3D_RENDER];
+		file->jobs_sent[V3D_RENDER]++;
+		v3d->queue[V3D_RENDER].jobs_sent++;
+
+		file->start_ns[V3D_RENDER] = 0;
+		v3d->queue[V3D_RENDER].start_ns = 0;
+
+		file->enabled_ns[V3D_RENDER] += runtime;
+		v3d->queue[V3D_RENDER].enabled_ns += runtime;
 
 		trace_v3d_rcl_irq(&v3d->drm, fence->seqno);
 		dma_fence_signal(&fence->base);
 		status = IRQ_HANDLED;
 	}
 
-	if (intsts & V3D_INT_CSDDONE) {
+	if (intsts & V3D_INT_CSDDONE(v3d->ver)) {
 		struct v3d_fence *fence =
 			to_v3d_fence(v3d->csd_job->base.irq_fence);
+		struct v3d_file_priv *file = v3d->csd_job->base.file->driver_priv;
+		u64 runtime = local_clock() - file->start_ns[V3D_CSD];
+
+		file->enabled_ns[V3D_CSD] += local_clock() - file->start_ns[V3D_CSD];
+		file->jobs_sent[V3D_CSD]++;
+		v3d->queue[V3D_CSD].jobs_sent++;
+
+		file->start_ns[V3D_CSD] = 0;
+		v3d->queue[V3D_CSD].start_ns = 0;
+
+		file->enabled_ns[V3D_CSD] += runtime;
+		v3d->queue[V3D_CSD].enabled_ns += runtime;
 
 		trace_v3d_csd_irq(&v3d->drm, fence->seqno);
 		dma_fence_signal(&fence->base);
@@ -127,7 +165,7 @@ v3d_irq(int irq, void *arg)
 	/* We shouldn't be triggering these if we have GMP in
 	 * always-allowed mode.
 	 */
-	if (intsts & V3D_INT_GMPV)
+	if (v3d->ver < 71 && (intsts & V3D_INT_GMPV))
 		dev_err(v3d->drm.dev, "GMP violation\n");
 
 	/* V3D 4.2 wires the hub and core IRQs together, so if we &
@@ -154,6 +192,18 @@ v3d_hub_irq(int irq, void *arg)
 	if (intsts & V3D_HUB_INT_TFUC) {
 		struct v3d_fence *fence =
 			to_v3d_fence(v3d->tfu_job->base.irq_fence);
+		struct v3d_file_priv *file = v3d->tfu_job->base.file->driver_priv;
+		u64 runtime = local_clock() - file->start_ns[V3D_TFU];
+
+		file->enabled_ns[V3D_TFU] += local_clock() - file->start_ns[V3D_TFU];
+		file->jobs_sent[V3D_TFU]++;
+		v3d->queue[V3D_TFU].jobs_sent++;
+
+		file->start_ns[V3D_TFU] = 0;
+		v3d->queue[V3D_TFU].start_ns = 0;
+
+		file->enabled_ns[V3D_TFU] += runtime;
+		v3d->queue[V3D_TFU].enabled_ns += runtime;
 
 		trace_v3d_tfu_irq(&v3d->drm, fence->seqno);
 		dma_fence_signal(&fence->base);
@@ -197,6 +247,11 @@ v3d_hub_irq(int irq, void *arg)
 		status = IRQ_HANDLED;
 	}
 
+	if (v3d->ver >= 71 && (intsts & V3D_V7_HUB_INT_GMPV)) {
+		dev_err(v3d->drm.dev, "GMP Violation\n");
+		status = IRQ_HANDLED;
+	}
+
 	return status;
 }
 
@@ -211,8 +266,8 @@ v3d_irq_init(struct v3d_dev *v3d)
 	 * for us.
 	 */
 	for (core = 0; core < v3d->cores; core++)
-		V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS);
-	V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS);
+		V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS(v3d->ver));
+	V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS(v3d->ver));
 
 	irq1 = platform_get_irq_optional(v3d_to_pdev(v3d), 1);
 	if (irq1 == -EPROBE_DEFER)
@@ -256,12 +311,12 @@ v3d_irq_enable(struct v3d_dev *v3d)
 
 	/* Enable our set of interrupts, masking out any others. */
 	for (core = 0; core < v3d->cores; core++) {
-		V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~V3D_CORE_IRQS);
-		V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_CLR, V3D_CORE_IRQS);
+		V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~V3D_CORE_IRQS(v3d->ver));
+		V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_CLR, V3D_CORE_IRQS(v3d->ver));
 	}
 
-	V3D_WRITE(V3D_HUB_INT_MSK_SET, ~V3D_HUB_IRQS);
-	V3D_WRITE(V3D_HUB_INT_MSK_CLR, V3D_HUB_IRQS);
+	V3D_WRITE(V3D_HUB_INT_MSK_SET, ~V3D_HUB_IRQS(v3d->ver));
+	V3D_WRITE(V3D_HUB_INT_MSK_CLR, V3D_HUB_IRQS(v3d->ver));
 }
 
 void
@@ -276,8 +331,8 @@ v3d_irq_disable(struct v3d_dev *v3d)
 
 	/* Clear any pending interrupts we might have left. */
 	for (core = 0; core < v3d->cores; core++)
-		V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS);
-	V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS);
+		V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS(v3d->ver));
+	V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS(v3d->ver));
 
 	cancel_work_sync(&v3d->overflow_mem_work);
 }
diff --git a/drivers/gpu/drm/v3d/v3d_regs.h b/drivers/gpu/drm/v3d/v3d_regs.h
index 3663e0d6bf76..1b1a62ad9585 100644
--- a/drivers/gpu/drm/v3d/v3d_regs.h
+++ b/drivers/gpu/drm/v3d/v3d_regs.h
@@ -57,6 +57,7 @@
 #define V3D_HUB_INT_MSK_STS                            0x0005c
 #define V3D_HUB_INT_MSK_SET                            0x00060
 #define V3D_HUB_INT_MSK_CLR                            0x00064
+# define V3D_V7_HUB_INT_GMPV                           BIT(6)
 # define V3D_HUB_INT_MMU_WRV                           BIT(5)
 # define V3D_HUB_INT_MMU_PTI                           BIT(4)
 # define V3D_HUB_INT_MMU_CAP                           BIT(3)
@@ -64,6 +65,7 @@
 # define V3D_HUB_INT_TFUC                              BIT(1)
 # define V3D_HUB_INT_TFUF                              BIT(0)
 
+/* GCA registers only exist in V3D < 41 */
 #define V3D_GCA_CACHE_CTRL                             0x0000c
 # define V3D_GCA_CACHE_CTRL_FLUSH                      BIT(0)
 
@@ -86,7 +88,8 @@
 # define V3D_TOP_GR_BRIDGE_SW_INIT_1                   0x0000c
 # define V3D_TOP_GR_BRIDGE_SW_INIT_1_V3D_CLK_108_SW_INIT BIT(0)
 
-#define V3D_TFU_CS                                     0x00400
+#define V3D_TFU_CS(ver) ((ver >= 71) ? 0x00700 : 0x00400)
+
 /* Stops current job, empties input fifo. */
 # define V3D_TFU_CS_TFURST                             BIT(31)
 # define V3D_TFU_CS_CVTCT_MASK                         V3D_MASK(23, 16)
@@ -95,7 +98,7 @@
 # define V3D_TFU_CS_NFREE_SHIFT                        8
 # define V3D_TFU_CS_BUSY                               BIT(0)
 
-#define V3D_TFU_SU                                     0x00404
+#define V3D_TFU_SU(ver) ((ver >= 71) ? 0x00704 : 0x00404)
 /* Interrupt when FINTTHR input slots are free (0 = disabled) */
 # define V3D_TFU_SU_FINTTHR_MASK                       V3D_MASK(13, 8)
 # define V3D_TFU_SU_FINTTHR_SHIFT                      8
@@ -106,39 +109,42 @@
 # define V3D_TFU_SU_THROTTLE_MASK                      V3D_MASK(1, 0)
 # define V3D_TFU_SU_THROTTLE_SHIFT                     0
 
-#define V3D_TFU_ICFG                                   0x00408
+#define V3D_TFU_ICFG(ver) ((ver >= 71) ? 0x00708 : 0x00408)
 /* Interrupt when the conversion is complete. */
 # define V3D_TFU_ICFG_IOC                              BIT(0)
 
 /* Input Image Address */
-#define V3D_TFU_IIA                                    0x0040c
+#define V3D_TFU_IIA(ver) ((ver >= 71) ? 0x0070c : 0x0040c)
 /* Input Chroma Address */
-#define V3D_TFU_ICA                                    0x00410
+#define V3D_TFU_ICA(ver) ((ver >= 71) ? 0x00710 : 0x00410)
 /* Input Image Stride */
-#define V3D_TFU_IIS                                    0x00414
+#define V3D_TFU_IIS(ver) ((ver >= 71) ? 0x00714 : 0x00414)
 /* Input Image U-Plane Address */
-#define V3D_TFU_IUA                                    0x00418
+#define V3D_TFU_IUA(ver) ((ver >= 71) ? 0x00718 : 0x00418)
+/* Image output config (VD 7.x only) */
+#define V3D_V7_TFU_IOC                                 0x0071c
 /* Output Image Address */
-#define V3D_TFU_IOA                                    0x0041c
+#define V3D_TFU_IOA(ver) ((ver >= 71) ? 0x00720 : 0x0041c)
 /* Image Output Size */
-#define V3D_TFU_IOS                                    0x00420
+#define V3D_TFU_IOS(ver) ((ver >= 71) ? 0x00724 : 0x00420)
 /* TFU YUV Coefficient 0 */
-#define V3D_TFU_COEF0                                  0x00424
-/* Use these regs instead of the defaults. */
+#define V3D_TFU_COEF0(ver) ((ver >= 71) ? 0x00728 : 0x00424)
+/* Use these regs instead of the defaults (V3D 4.x only) */
 # define V3D_TFU_COEF0_USECOEF                         BIT(31)
 /* TFU YUV Coefficient 1 */
-#define V3D_TFU_COEF1                                  0x00428
+#define V3D_TFU_COEF1(ver) ((ver >= 71) ? 0x0072c : 0x00428)
 /* TFU YUV Coefficient 2 */
-#define V3D_TFU_COEF2                                  0x0042c
+#define V3D_TFU_COEF2(ver) ((ver >= 71) ? 0x00730 : 0x0042c)
 /* TFU YUV Coefficient 3 */
-#define V3D_TFU_COEF3                                  0x00430
+#define V3D_TFU_COEF3(ver) ((ver >= 71) ? 0x00734 : 0x00430)
 
+/* V3D 4.x only */
 #define V3D_TFU_CRC                                    0x00434
 
 /* Per-MMU registers. */
 
 #define V3D_MMUC_CONTROL                               0x01000
-# define V3D_MMUC_CONTROL_CLEAR                        BIT(3)
+#define V3D_MMUC_CONTROL_CLEAR(ver) ((ver >= 71) ? BIT(11) : BIT(3))
 # define V3D_MMUC_CONTROL_FLUSHING                     BIT(2)
 # define V3D_MMUC_CONTROL_FLUSH                        BIT(1)
 # define V3D_MMUC_CONTROL_ENABLE                       BIT(0)
@@ -246,7 +252,6 @@
 
 #define V3D_CTL_L2TCACTL                               0x00030
 # define V3D_L2TCACTL_TMUWCF                           BIT(8)
-# define V3D_L2TCACTL_L2T_NO_WM                        BIT(4)
 /* Invalidates cache lines. */
 # define V3D_L2TCACTL_FLM_FLUSH                        0
 /* Removes cachelines without writing dirty lines back. */
@@ -267,8 +272,8 @@
 #define V3D_CTL_INT_MSK_CLR                            0x00064
 # define V3D_INT_QPU_MASK                              V3D_MASK(27, 16)
 # define V3D_INT_QPU_SHIFT                             16
-# define V3D_INT_CSDDONE                               BIT(7)
-# define V3D_INT_PCTR                                  BIT(6)
+#define V3D_INT_CSDDONE(ver) ((ver >= 71) ? BIT(6) : BIT(7))
+#define V3D_INT_PCTR(ver) ((ver >= 71) ? BIT(5) : BIT(6))
 # define V3D_INT_GMPV                                  BIT(5)
 # define V3D_INT_TRFB                                  BIT(4)
 # define V3D_INT_SPILLUSE                              BIT(3)
@@ -350,21 +355,25 @@
 #define V3D_V4_PCTR_0_SRC_X(x)                         (V3D_V4_PCTR_0_SRC_0_3 + \
 							4 * (x))
 # define V3D_PCTR_S0_MASK                              V3D_MASK(6, 0)
+# define V3D_V7_PCTR_S0_MASK                           V3D_MASK(7, 0)
 # define V3D_PCTR_S0_SHIFT                             0
 # define V3D_PCTR_S1_MASK                              V3D_MASK(14, 8)
+# define V3D_V7_PCTR_S1_MASK                           V3D_MASK(15, 8)
 # define V3D_PCTR_S1_SHIFT                             8
 # define V3D_PCTR_S2_MASK                              V3D_MASK(22, 16)
+# define V3D_V7_PCTR_S2_MASK                           V3D_MASK(23, 16)
 # define V3D_PCTR_S2_SHIFT                             16
 # define V3D_PCTR_S3_MASK                              V3D_MASK(30, 24)
+# define V3D_V7_PCTR_S3_MASK                           V3D_MASK(31, 24)
 # define V3D_PCTR_S3_SHIFT                             24
-# define V3D_PCTR_CYCLE_COUNT                          32
+#define V3D_PCTR_CYCLE_COUNT(ver) ((ver >= 71) ? 0 : 32)
 
 /* Output values of the counters. */
 #define V3D_PCTR_0_PCTR0                               0x00680
 #define V3D_PCTR_0_PCTR31                              0x006fc
 #define V3D_PCTR_0_PCTRX(x)                            (V3D_PCTR_0_PCTR0 + \
 							4 * (x))
-#define V3D_GMP_STATUS                                 0x00800
+#define V3D_GMP_STATUS(ver) ((ver >= 71) ? 0x00600 : 0x00800)
 # define V3D_GMP_STATUS_GMPRST                         BIT(31)
 # define V3D_GMP_STATUS_WR_COUNT_MASK                  V3D_MASK(30, 24)
 # define V3D_GMP_STATUS_WR_COUNT_SHIFT                 24
@@ -377,13 +386,13 @@
 # define V3D_GMP_STATUS_INVPROT                        BIT(1)
 # define V3D_GMP_STATUS_VIO                            BIT(0)
 
-#define V3D_GMP_CFG                                    0x00804
+#define V3D_GMP_CFG(ver) ((ver >= 71) ? 0x00604 : 0x00804)
 # define V3D_GMP_CFG_LBURSTEN                          BIT(3)
 # define V3D_GMP_CFG_PGCRSEN                           BIT()
 # define V3D_GMP_CFG_STOP_REQ                          BIT(1)
 # define V3D_GMP_CFG_PROT_ENABLE                       BIT(0)
 
-#define V3D_GMP_VIO_ADDR                               0x00808
+#define V3D_GMP_VIO_ADDR(ver) ((ver >= 71) ? 0x00608 : 0x00808)
 #define V3D_GMP_VIO_TYPE                               0x0080c
 #define V3D_GMP_TABLE_ADDR                             0x00810
 #define V3D_GMP_CLEAR_LOAD                             0x00814
@@ -398,25 +407,25 @@
 # define V3D_CSD_STATUS_HAVE_CURRENT_DISPATCH          BIT(1)
 # define V3D_CSD_STATUS_HAVE_QUEUED_DISPATCH           BIT(0)
 
-#define V3D_CSD_QUEUED_CFG0                            0x00904
+#define V3D_CSD_QUEUED_CFG0(ver) ((ver >= 71) ? 0x00930 : 0x00904)
 # define V3D_CSD_QUEUED_CFG0_NUM_WGS_X_MASK            V3D_MASK(31, 16)
 # define V3D_CSD_QUEUED_CFG0_NUM_WGS_X_SHIFT           16
 # define V3D_CSD_QUEUED_CFG0_WG_X_OFFSET_MASK          V3D_MASK(15, 0)
 # define V3D_CSD_QUEUED_CFG0_WG_X_OFFSET_SHIFT         0
 
-#define V3D_CSD_QUEUED_CFG1                            0x00908
+#define V3D_CSD_QUEUED_CFG1(ver) ((ver >= 71) ? 0x00934 : 0x00908)
 # define V3D_CSD_QUEUED_CFG1_NUM_WGS_Y_MASK            V3D_MASK(31, 16)
 # define V3D_CSD_QUEUED_CFG1_NUM_WGS_Y_SHIFT           16
 # define V3D_CSD_QUEUED_CFG1_WG_Y_OFFSET_MASK          V3D_MASK(15, 0)
 # define V3D_CSD_QUEUED_CFG1_WG_Y_OFFSET_SHIFT         0
 
-#define V3D_CSD_QUEUED_CFG2                            0x0090c
+#define V3D_CSD_QUEUED_CFG2(ver) ((ver >= 71) ? 0x00938 : 0x0090c)
 # define V3D_CSD_QUEUED_CFG2_NUM_WGS_Z_MASK            V3D_MASK(31, 16)
 # define V3D_CSD_QUEUED_CFG2_NUM_WGS_Z_SHIFT           16
 # define V3D_CSD_QUEUED_CFG2_WG_Z_OFFSET_MASK          V3D_MASK(15, 0)
 # define V3D_CSD_QUEUED_CFG2_WG_Z_OFFSET_SHIFT         0
 
-#define V3D_CSD_QUEUED_CFG3                            0x00910
+#define V3D_CSD_QUEUED_CFG3(ver) ((ver >= 71) ? 0x0093c : 0x00910)
 # define V3D_CSD_QUEUED_CFG3_OVERLAP_WITH_PREV         BIT(26)
 # define V3D_CSD_QUEUED_CFG3_MAX_SG_ID_MASK            V3D_MASK(25, 20)
 # define V3D_CSD_QUEUED_CFG3_MAX_SG_ID_SHIFT           20
@@ -428,23 +437,28 @@
 # define V3D_CSD_QUEUED_CFG3_WG_SIZE_SHIFT             0
 
 /* Number of batches, minus 1 */
-#define V3D_CSD_QUEUED_CFG4                            0x00914
+#define V3D_CSD_QUEUED_CFG4(ver) ((ver >= 71) ? 0x00940 : 0x00914)
 
 /* Shader address, pnan, singleseg, threading, like a shader record. */
-#define V3D_CSD_QUEUED_CFG5                            0x00918
+#define V3D_CSD_QUEUED_CFG5(ver) ((ver >= 71) ? 0x00944 : 0x00918)
 
 /* Uniforms address (4 byte aligned) */
-#define V3D_CSD_QUEUED_CFG6                            0x0091c
-
-#define V3D_CSD_CURRENT_CFG0                          0x00920
-#define V3D_CSD_CURRENT_CFG1                          0x00924
-#define V3D_CSD_CURRENT_CFG2                          0x00928
-#define V3D_CSD_CURRENT_CFG3                          0x0092c
-#define V3D_CSD_CURRENT_CFG4                          0x00930
-#define V3D_CSD_CURRENT_CFG5                          0x00934
-#define V3D_CSD_CURRENT_CFG6                          0x00938
-
-#define V3D_CSD_CURRENT_ID0                            0x0093c
+#define V3D_CSD_QUEUED_CFG6(ver) ((ver >= 71) ? 0x00948 : 0x0091c)
+
+/* V3D 7.x+ only */
+#define V3D_V7_CSD_QUEUED_CFG7                         0x0094c
+
+#define V3D_CSD_CURRENT_CFG0(ver) ((ver >= 71) ? 0x00958 : 0x00920)
+#define V3D_CSD_CURRENT_CFG1(ver) ((ver >= 71) ? 0x0095c : 0x00924)
+#define V3D_CSD_CURRENT_CFG2(ver) ((ver >= 71) ? 0x00960 : 0x00928)
+#define V3D_CSD_CURRENT_CFG3(ver) ((ver >= 71) ? 0x00964 : 0x0092c)
+#define V3D_CSD_CURRENT_CFG4(ver) ((ver >= 71) ? 0x00968 : 0x00930)
+#define V3D_CSD_CURRENT_CFG5(ver) ((ver >= 71) ? 0x0096c : 0x00934)
+#define V3D_CSD_CURRENT_CFG6(ver) ((ver >= 71) ? 0x00970 : 0x00938)
+/* V3D 7.x+ only */
+#define V3D_V7_CSD_CURRENT_CFG7                        0x00974
+
+#define V3D_CSD_CURRENT_ID0(ver) ((ver >= 71) ? 0x00978 : 0x0093c)
 # define V3D_CSD_CURRENT_ID0_WG_X_MASK                 V3D_MASK(31, 16)
 # define V3D_CSD_CURRENT_ID0_WG_X_SHIFT                16
 # define V3D_CSD_CURRENT_ID0_WG_IN_SG_MASK             V3D_MASK(11, 8)
@@ -452,7 +466,7 @@
 # define V3D_CSD_CURRENT_ID0_L_IDX_MASK                V3D_MASK(7, 0)
 # define V3D_CSD_CURRENT_ID0_L_IDX_SHIFT               0
 
-#define V3D_CSD_CURRENT_ID1                            0x00940
+#define V3D_CSD_CURRENT_ID1(ver) ((ver >= 71) ? 0x0097c : 0x00940)
 # define V3D_CSD_CURRENT_ID0_WG_Z_MASK                 V3D_MASK(31, 16)
 # define V3D_CSD_CURRENT_ID0_WG_Z_SHIFT                16
 # define V3D_CSD_CURRENT_ID0_WG_Y_MASK                 V3D_MASK(15, 0)
diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c
index 038e1ae589c7..fccbea2a5f2e 100644
--- a/drivers/gpu/drm/v3d/v3d_sched.c
+++ b/drivers/gpu/drm/v3d/v3d_sched.c
@@ -18,6 +18,7 @@
  * semaphores to interlock between them.
  */
 
+#include <linux/sched/clock.h>
 #include <linux/kthread.h>
 
 #include "v3d_drv.h"
@@ -76,6 +77,7 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job)
 {
 	struct v3d_bin_job *job = to_bin_job(sched_job);
 	struct v3d_dev *v3d = job->base.v3d;
+	struct v3d_file_priv *file = job->base.file->driver_priv;
 	struct drm_device *dev = &v3d->drm;
 	struct dma_fence *fence;
 	unsigned long irqflags;
@@ -107,6 +109,9 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job)
 	trace_v3d_submit_cl(dev, false, to_v3d_fence(fence)->seqno,
 			    job->start, job->end);
 
+	file->start_ns[V3D_BIN] = local_clock();
+	v3d->queue[V3D_BIN].start_ns = file->start_ns[V3D_BIN];
+
 	v3d_switch_perfmon(v3d, &job->base);
 
 	/* Set the current and end address of the control list.
@@ -131,6 +136,7 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job)
 {
 	struct v3d_render_job *job = to_render_job(sched_job);
 	struct v3d_dev *v3d = job->base.v3d;
+	struct v3d_file_priv *file = job->base.file->driver_priv;
 	struct drm_device *dev = &v3d->drm;
 	struct dma_fence *fence;
 
@@ -158,6 +164,9 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job)
 	trace_v3d_submit_cl(dev, true, to_v3d_fence(fence)->seqno,
 			    job->start, job->end);
 
+	file->start_ns[V3D_RENDER] = local_clock();
+	v3d->queue[V3D_RENDER].start_ns = file->start_ns[V3D_RENDER];
+
 	v3d_switch_perfmon(v3d, &job->base);
 
 	/* XXX: Set the QCFG */
@@ -176,6 +185,7 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job)
 {
 	struct v3d_tfu_job *job = to_tfu_job(sched_job);
 	struct v3d_dev *v3d = job->base.v3d;
+	struct v3d_file_priv *file = job->base.file->driver_priv;
 	struct drm_device *dev = &v3d->drm;
 	struct dma_fence *fence;
 
@@ -190,20 +200,25 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job)
 
 	trace_v3d_submit_tfu(dev, to_v3d_fence(fence)->seqno);
 
-	V3D_WRITE(V3D_TFU_IIA, job->args.iia);
-	V3D_WRITE(V3D_TFU_IIS, job->args.iis);
-	V3D_WRITE(V3D_TFU_ICA, job->args.ica);
-	V3D_WRITE(V3D_TFU_IUA, job->args.iua);
-	V3D_WRITE(V3D_TFU_IOA, job->args.ioa);
-	V3D_WRITE(V3D_TFU_IOS, job->args.ios);
-	V3D_WRITE(V3D_TFU_COEF0, job->args.coef[0]);
-	if (job->args.coef[0] & V3D_TFU_COEF0_USECOEF) {
-		V3D_WRITE(V3D_TFU_COEF1, job->args.coef[1]);
-		V3D_WRITE(V3D_TFU_COEF2, job->args.coef[2]);
-		V3D_WRITE(V3D_TFU_COEF3, job->args.coef[3]);
+	file->start_ns[V3D_TFU] = local_clock();
+	v3d->queue[V3D_TFU].start_ns = file->start_ns[V3D_TFU];
+
+	V3D_WRITE(V3D_TFU_IIA(v3d->ver), job->args.iia);
+	V3D_WRITE(V3D_TFU_IIS(v3d->ver), job->args.iis);
+	V3D_WRITE(V3D_TFU_ICA(v3d->ver), job->args.ica);
+	V3D_WRITE(V3D_TFU_IUA(v3d->ver), job->args.iua);
+	V3D_WRITE(V3D_TFU_IOA(v3d->ver), job->args.ioa);
+	if (v3d->ver >= 71)
+		V3D_WRITE(V3D_V7_TFU_IOC, job->args.v71.ioc);
+	V3D_WRITE(V3D_TFU_IOS(v3d->ver), job->args.ios);
+	V3D_WRITE(V3D_TFU_COEF0(v3d->ver), job->args.coef[0]);
+	if (v3d->ver >= 71 || (job->args.coef[0] & V3D_TFU_COEF0_USECOEF)) {
+		V3D_WRITE(V3D_TFU_COEF1(v3d->ver), job->args.coef[1]);
+		V3D_WRITE(V3D_TFU_COEF2(v3d->ver), job->args.coef[2]);
+		V3D_WRITE(V3D_TFU_COEF3(v3d->ver), job->args.coef[3]);
 	}
 	/* ICFG kicks off the job. */
-	V3D_WRITE(V3D_TFU_ICFG, job->args.icfg | V3D_TFU_ICFG_IOC);
+	V3D_WRITE(V3D_TFU_ICFG(v3d->ver), job->args.icfg | V3D_TFU_ICFG_IOC);
 
 	return fence;
 }
@@ -213,9 +228,10 @@ v3d_csd_job_run(struct drm_sched_job *sched_job)
 {
 	struct v3d_csd_job *job = to_csd_job(sched_job);
 	struct v3d_dev *v3d = job->base.v3d;
+	struct v3d_file_priv *file = job->base.file->driver_priv;
 	struct drm_device *dev = &v3d->drm;
 	struct dma_fence *fence;
-	int i;
+	int i, csd_cfg0_reg, csd_cfg_reg_count;
 
 	v3d->csd_job = job;
 
@@ -231,12 +247,17 @@ v3d_csd_job_run(struct drm_sched_job *sched_job)
 
 	trace_v3d_submit_csd(dev, to_v3d_fence(fence)->seqno);
 
+	file->start_ns[V3D_CSD] = local_clock();
+	v3d->queue[V3D_CSD].start_ns = file->start_ns[V3D_CSD];
+
 	v3d_switch_perfmon(v3d, &job->base);
 
-	for (i = 1; i <= 6; i++)
-		V3D_CORE_WRITE(0, V3D_CSD_QUEUED_CFG0 + 4 * i, job->args.cfg[i]);
+	csd_cfg0_reg = V3D_CSD_QUEUED_CFG0(v3d->ver);
+	csd_cfg_reg_count = v3d->ver < 71 ? 6 : 7;
+	for (i = 1; i <= csd_cfg_reg_count; i++)
+		V3D_CORE_WRITE(0, csd_cfg0_reg + 4 * i, job->args.cfg[i]);
 	/* CFG0 write kicks off the job. */
-	V3D_CORE_WRITE(0, V3D_CSD_QUEUED_CFG0, job->args.cfg[0]);
+	V3D_CORE_WRITE(0, csd_cfg0_reg, job->args.cfg[0]);
 
 	return fence;
 }
@@ -246,9 +267,25 @@ v3d_cache_clean_job_run(struct drm_sched_job *sched_job)
 {
 	struct v3d_job *job = to_v3d_job(sched_job);
 	struct v3d_dev *v3d = job->v3d;
+	struct v3d_file_priv *file = job->file->driver_priv;
+	u64 runtime;
+
+	file->start_ns[V3D_CACHE_CLEAN] = local_clock();
+	v3d->queue[V3D_CACHE_CLEAN].start_ns = file->start_ns[V3D_CACHE_CLEAN];
 
 	v3d_clean_caches(v3d);
 
+	runtime = local_clock() - file->start_ns[V3D_CACHE_CLEAN];
+
+	file->enabled_ns[V3D_CACHE_CLEAN] += runtime;
+	v3d->queue[V3D_CACHE_CLEAN].enabled_ns += runtime;
+
+	file->jobs_sent[V3D_CACHE_CLEAN]++;
+	v3d->queue[V3D_CACHE_CLEAN].jobs_sent++;
+
+	file->start_ns[V3D_CACHE_CLEAN] = 0;
+	v3d->queue[V3D_CACHE_CLEAN].start_ns = 0;
+
 	return NULL;
 }
 
@@ -336,7 +373,7 @@ v3d_csd_job_timedout(struct drm_sched_job *sched_job)
 {
 	struct v3d_csd_job *job = to_csd_job(sched_job);
 	struct v3d_dev *v3d = job->base.v3d;
-	u32 batches = V3D_CORE_READ(0, V3D_CSD_CURRENT_CFG4);
+	u32 batches = V3D_CORE_READ(0, V3D_CSD_CURRENT_CFG4(v3d->ver));
 
 	/* If we've made progress, skip reset and let the timer get
 	 * rearmed.
@@ -388,7 +425,7 @@ v3d_sched_init(struct v3d_dev *v3d)
 	int ret;
 
 	ret = drm_sched_init(&v3d->queue[V3D_BIN].sched,
-			     &v3d_bin_sched_ops,
+			     &v3d_bin_sched_ops, NULL,
 			     DRM_SCHED_PRIORITY_COUNT,
 			     hw_jobs_limit, job_hang_limit,
 			     msecs_to_jiffies(hang_limit_ms), NULL,
@@ -397,7 +434,7 @@ v3d_sched_init(struct v3d_dev *v3d)
 		return ret;
 
 	ret = drm_sched_init(&v3d->queue[V3D_RENDER].sched,
-			     &v3d_render_sched_ops,
+			     &v3d_render_sched_ops, NULL,
 			     DRM_SCHED_PRIORITY_COUNT,
 			     hw_jobs_limit, job_hang_limit,
 			     msecs_to_jiffies(hang_limit_ms), NULL,
@@ -406,7 +443,7 @@ v3d_sched_init(struct v3d_dev *v3d)
 		goto fail;
 
 	ret = drm_sched_init(&v3d->queue[V3D_TFU].sched,
-			     &v3d_tfu_sched_ops,
+			     &v3d_tfu_sched_ops, NULL,
 			     DRM_SCHED_PRIORITY_COUNT,
 			     hw_jobs_limit, job_hang_limit,
 			     msecs_to_jiffies(hang_limit_ms), NULL,
@@ -416,7 +453,7 @@ v3d_sched_init(struct v3d_dev *v3d)
 
 	if (v3d_has_csd(v3d)) {
 		ret = drm_sched_init(&v3d->queue[V3D_CSD].sched,
-				     &v3d_csd_sched_ops,
+				     &v3d_csd_sched_ops, NULL,
 				     DRM_SCHED_PRIORITY_COUNT,
 				     hw_jobs_limit, job_hang_limit,
 				     msecs_to_jiffies(hang_limit_ms), NULL,
@@ -425,7 +462,7 @@ v3d_sched_init(struct v3d_dev *v3d)
 			goto fail;
 
 		ret = drm_sched_init(&v3d->queue[V3D_CACHE_CLEAN].sched,
-				     &v3d_cache_clean_sched_ops,
+				     &v3d_cache_clean_sched_ops, NULL,
 				     DRM_SCHED_PRIORITY_COUNT,
 				     hw_jobs_limit, job_hang_limit,
 				     msecs_to_jiffies(hang_limit_ms), NULL,
diff --git a/drivers/gpu/drm/v3d/v3d_sysfs.c b/drivers/gpu/drm/v3d/v3d_sysfs.c
new file mode 100644
index 000000000000..d106845ba890
--- /dev/null
+++ b/drivers/gpu/drm/v3d/v3d_sysfs.c
@@ -0,0 +1,69 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2023 Igalia S.L.
+ */
+
+#include <linux/sched/clock.h>
+#include <linux/sysfs.h>
+
+#include "v3d_drv.h"
+
+static ssize_t
+gpu_stats_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+	struct v3d_dev *v3d = to_v3d_dev(drm);
+	enum v3d_queue queue;
+	u64 timestamp = local_clock();
+	u64 active_runtime;
+	ssize_t len = 0;
+
+	len += sysfs_emit(buf, "queue\ttimestamp\tjobs\truntime\n");
+
+	for (queue = 0; queue < V3D_MAX_QUEUES; queue++) {
+		if (v3d->queue[queue].start_ns)
+			active_runtime = timestamp - v3d->queue[queue].start_ns;
+		else
+			active_runtime = 0;
+
+		/* Each line will display the queue name, timestamp, the number
+		 * of jobs sent to that queue and the runtime, as can be seem here:
+		 *
+		 * queue	timestamp	jobs	runtime
+		 * bin		239043069420	22620	17438164056
+		 * render	239043069420	22619	27284814161
+		 * tfu		239043069420	8763	394592566
+		 * csd		239043069420	3168	10787905530
+		 * cache_clean	239043069420	6127	237375940
+		 */
+		len += sysfs_emit_at(buf, len, "%s\t%llu\t%llu\t%llu\n",
+				     v3d_queue_to_string(queue),
+				     timestamp,
+				     v3d->queue[queue].jobs_sent,
+				     v3d->queue[queue].enabled_ns + active_runtime);
+	}
+
+	return len;
+}
+static DEVICE_ATTR_RO(gpu_stats);
+
+static struct attribute *v3d_sysfs_entries[] = {
+	&dev_attr_gpu_stats.attr,
+	NULL,
+};
+
+static struct attribute_group v3d_sysfs_attr_group = {
+	.attrs = v3d_sysfs_entries,
+};
+
+int
+v3d_sysfs_init(struct device *dev)
+{
+	return sysfs_create_group(&dev->kobj, &v3d_sysfs_attr_group);
+}
+
+void
+v3d_sysfs_destroy(struct device *dev)
+{
+	return sysfs_remove_group(&dev->kobj, &v3d_sysfs_attr_group);
+}
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h b/drivers/gpu/drm/virtio/virtgpu_drv.h
index 96365a772f77..bb7d86a0c6a1 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.h
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.h
@@ -58,6 +58,9 @@
 #define MAX_CAPSET_ID 63
 #define MAX_RINGS 64
 
+/* See virtio_gpu_ctx_create. One additional character for NULL terminator. */
+#define DEBUG_NAME_MAX_LEN 65
+
 struct virtio_gpu_object_params {
 	unsigned long size;
 	bool dumb;
@@ -274,6 +277,8 @@ struct virtio_gpu_fpriv {
 	uint64_t base_fence_ctx;
 	uint64_t ring_idx_mask;
 	struct mutex context_lock;
+	char debug_name[DEBUG_NAME_MAX_LEN];
+	bool explicit_debug_name;
 };
 
 /* virtgpu_ioctl.c */
diff --git a/drivers/gpu/drm/virtio/virtgpu_ioctl.c b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
index b24b11f25197..e4f76f315550 100644
--- a/drivers/gpu/drm/virtio/virtgpu_ioctl.c
+++ b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
@@ -42,12 +42,19 @@
 static void virtio_gpu_create_context_locked(struct virtio_gpu_device *vgdev,
 					     struct virtio_gpu_fpriv *vfpriv)
 {
-	char dbgname[TASK_COMM_LEN];
+	if (vfpriv->explicit_debug_name) {
+		virtio_gpu_cmd_context_create(vgdev, vfpriv->ctx_id,
+					      vfpriv->context_init,
+					      strlen(vfpriv->debug_name),
+					      vfpriv->debug_name);
+	} else {
+		char dbgname[TASK_COMM_LEN];
 
-	get_task_comm(dbgname, current);
-	virtio_gpu_cmd_context_create(vgdev, vfpriv->ctx_id,
-				      vfpriv->context_init, strlen(dbgname),
-				      dbgname);
+		get_task_comm(dbgname, current);
+		virtio_gpu_cmd_context_create(vgdev, vfpriv->ctx_id,
+					      vfpriv->context_init, strlen(dbgname),
+					      dbgname);
+	}
 
 	vfpriv->context_created = true;
 }
@@ -107,6 +114,9 @@ static int virtio_gpu_getparam_ioctl(struct drm_device *dev, void *data,
 	case VIRTGPU_PARAM_SUPPORTED_CAPSET_IDs:
 		value = vgdev->capset_id_mask;
 		break;
+	case VIRTGPU_PARAM_EXPLICIT_DEBUG_NAME:
+		value = vgdev->has_context_init ? 1 : 0;
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -565,8 +575,8 @@ static int virtio_gpu_context_init_ioctl(struct drm_device *dev,
 					 void *data, struct drm_file *file)
 {
 	int ret = 0;
-	uint32_t num_params, i, param, value;
-	uint64_t valid_ring_mask;
+	uint32_t num_params, i;
+	uint64_t valid_ring_mask, param, value;
 	size_t len;
 	struct drm_virtgpu_context_set_param *ctx_set_params = NULL;
 	struct virtio_gpu_device *vgdev = dev->dev_private;
@@ -580,7 +590,7 @@ static int virtio_gpu_context_init_ioctl(struct drm_device *dev,
 		return -EINVAL;
 
 	/* Number of unique parameters supported at this time. */
-	if (num_params > 3)
+	if (num_params > 4)
 		return -EINVAL;
 
 	ctx_set_params = memdup_user(u64_to_user_ptr(args->ctx_set_params),
@@ -642,6 +652,21 @@ static int virtio_gpu_context_init_ioctl(struct drm_device *dev,
 
 			vfpriv->ring_idx_mask = value;
 			break;
+		case VIRTGPU_CONTEXT_PARAM_DEBUG_NAME:
+			if (vfpriv->explicit_debug_name) {
+				ret = -EINVAL;
+				goto out_unlock;
+			}
+
+			ret = strncpy_from_user(vfpriv->debug_name,
+						u64_to_user_ptr(value),
+						DEBUG_NAME_MAX_LEN - 1);
+			if (ret < 0)
+				goto out_unlock;
+
+			vfpriv->explicit_debug_name = true;
+			ret = 0;
+			break;
 		default:
 			ret = -EINVAL;
 			goto out_unlock;
diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig
index 4ce74af79657..03e7fc3d05a5 100644
--- a/drivers/hid/Kconfig
+++ b/drivers/hid/Kconfig
@@ -1296,6 +1296,15 @@ config HID_ALPS
 	Say Y here if you have a Alps touchpads over i2c-hid or usbhid
 	and want support for its special functionalities.
 
+config HID_MCP2200
+	tristate "Microchip MCP2200 HID USB-to-GPIO bridge"
+	depends on USB_HID && GPIOLIB
+	help
+	  Provides GPIO functionality over USB-HID through MCP2200 device.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called hid-mcp2200.ko.
+
 config HID_MCP2221
 	tristate "Microchip MCP2221 HID USB-to-I2C/SMbus host support"
 	depends on USB_HID && I2C
diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile
index 8a06d0f840bc..082a728eac60 100644
--- a/drivers/hid/Makefile
+++ b/drivers/hid/Makefile
@@ -79,6 +79,7 @@ obj-$(CONFIG_HID_LOGITECH_HIDPP)	+= hid-logitech-hidpp.o
 obj-$(CONFIG_HID_MACALLY)	+= hid-macally.o
 obj-$(CONFIG_HID_MAGICMOUSE)	+= hid-magicmouse.o
 obj-$(CONFIG_HID_MALTRON)	+= hid-maltron.o
+obj-$(CONFIG_HID_MCP2200)	+= hid-mcp2200.o
 obj-$(CONFIG_HID_MCP2221)	+= hid-mcp2221.o
 obj-$(CONFIG_HID_MAYFLASH)	+= hid-mf.o
 obj-$(CONFIG_HID_MEGAWORLD_FF)	+= hid-megaworld.o
diff --git a/drivers/hid/hid-apple.c b/drivers/hid/hid-apple.c
index 3ca45975c686..d9e9829b2200 100644
--- a/drivers/hid/hid-apple.c
+++ b/drivers/hid/hid-apple.c
@@ -345,6 +345,8 @@ static const struct apple_non_apple_keyboard non_apple_keyboards[] = {
 	{ "AONE" },
 	{ "GANSS" },
 	{ "Hailuck" },
+	{ "Jamesdonkey" },
+	{ "A3R" },
 };
 
 static bool apple_is_non_apple_keyboard(struct hid_device *hdev)
diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c
index 8992e3c1e769..e0181218ad85 100644
--- a/drivers/hid/hid-core.c
+++ b/drivers/hid/hid-core.c
@@ -702,15 +702,22 @@ static void hid_close_report(struct hid_device *device)
  * Free a device structure, all reports, and all fields.
  */
 
-static void hid_device_release(struct device *dev)
+void hiddev_free(struct kref *ref)
 {
-	struct hid_device *hid = to_hid_device(dev);
+	struct hid_device *hid = container_of(ref, struct hid_device, ref);
 
 	hid_close_report(hid);
 	kfree(hid->dev_rdesc);
 	kfree(hid);
 }
 
+static void hid_device_release(struct device *dev)
+{
+	struct hid_device *hid = to_hid_device(dev);
+
+	kref_put(&hid->ref, hiddev_free);
+}
+
 /*
  * Fetch a report description item from the data stream. We support long
  * items, though they are not used yet.
@@ -2846,6 +2853,7 @@ struct hid_device *hid_allocate_device(void)
 	spin_lock_init(&hdev->debug_list_lock);
 	sema_init(&hdev->driver_input_lock, 1);
 	mutex_init(&hdev->ll_open_lock);
+	kref_init(&hdev->ref);
 
 	hid_bpf_device_init(hdev);
 
diff --git a/drivers/hid/hid-debug.c b/drivers/hid/hid-debug.c
index e7ef1ea107c9..7dd83ec74f8a 100644
--- a/drivers/hid/hid-debug.c
+++ b/drivers/hid/hid-debug.c
@@ -1135,6 +1135,7 @@ static int hid_debug_events_open(struct inode *inode, struct file *file)
 		goto out;
 	}
 	list->hdev = (struct hid_device *) inode->i_private;
+	kref_get(&list->hdev->ref);
 	file->private_data = list;
 	mutex_init(&list->read_mutex);
 
@@ -1227,6 +1228,8 @@ static int hid_debug_events_release(struct inode *inode, struct file *file)
 	list_del(&list->node);
 	spin_unlock_irqrestore(&list->hdev->debug_list_lock, flags);
 	kfifo_free(&list->hid_debug_fifo);
+
+	kref_put(&list->hdev->ref, hiddev_free);
 	kfree(list);
 
 	return 0;
diff --git a/drivers/hid/hid-glorious.c b/drivers/hid/hid-glorious.c
index 558eb08c19ef..281b3a7187ce 100644
--- a/drivers/hid/hid-glorious.c
+++ b/drivers/hid/hid-glorious.c
@@ -21,6 +21,10 @@ MODULE_DESCRIPTION("HID driver for Glorious PC Gaming Race mice");
  * Glorious Model O and O- specify the const flag in the consumer input
  * report descriptor, which leads to inputs being ignored. Fix this
  * by patching the descriptor.
+ *
+ * Glorious Model I incorrectly specifes the Usage Minimum for its
+ * keyboard HID report, causing keycodes to be misinterpreted.
+ * Fix this by setting Usage Minimum to 0 in that report.
  */
 static __u8 *glorious_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 		unsigned int *rsize)
@@ -32,6 +36,10 @@ static __u8 *glorious_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 		rdesc[85] = rdesc[113] = rdesc[141] = \
 			HID_MAIN_ITEM_VARIABLE | HID_MAIN_ITEM_RELATIVE;
 	}
+	if (*rsize == 156 && rdesc[41] == 1) {
+		hid_info(hdev, "patching Glorious Model I keyboard report descriptor\n");
+		rdesc[41] = 0;
+	}
 	return rdesc;
 }
 
@@ -44,6 +52,8 @@ static void glorious_update_name(struct hid_device *hdev)
 		model = "Model O"; break;
 	case USB_DEVICE_ID_GLORIOUS_MODEL_D:
 		model = "Model D"; break;
+	case USB_DEVICE_ID_GLORIOUS_MODEL_I:
+		model = "Model I"; break;
 	}
 
 	snprintf(hdev->name, sizeof(hdev->name), "%s %s", "Glorious", model);
@@ -66,10 +76,12 @@ static int glorious_probe(struct hid_device *hdev,
 }
 
 static const struct hid_device_id glorious_devices[] = {
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GLORIOUS,
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SINOWEALTH,
 		USB_DEVICE_ID_GLORIOUS_MODEL_O) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GLORIOUS,
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SINOWEALTH,
 		USB_DEVICE_ID_GLORIOUS_MODEL_D) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LAVIEW,
+		USB_DEVICE_ID_GLORIOUS_MODEL_I) },
 	{ }
 };
 MODULE_DEVICE_TABLE(hid, glorious_devices);
diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h
index f7973ccd84a2..b14bfa59e924 100644
--- a/drivers/hid/hid-ids.h
+++ b/drivers/hid/hid-ids.h
@@ -511,10 +511,6 @@
 #define USB_DEVICE_ID_GENERAL_TOUCH_WIN8_PIT_010A 0x010a
 #define USB_DEVICE_ID_GENERAL_TOUCH_WIN8_PIT_E100 0xe100
 
-#define USB_VENDOR_ID_GLORIOUS  0x258a
-#define USB_DEVICE_ID_GLORIOUS_MODEL_D 0x0033
-#define USB_DEVICE_ID_GLORIOUS_MODEL_O 0x0036
-
 #define I2C_VENDOR_ID_GOODIX		0x27c6
 #define I2C_DEVICE_ID_GOODIX_01F0	0x01f0
 
@@ -745,6 +741,9 @@
 #define USB_VENDOR_ID_LABTEC		0x1020
 #define USB_DEVICE_ID_LABTEC_WIRELESS_KEYBOARD	0x0006
 
+#define USB_VENDOR_ID_LAVIEW		0x22D4
+#define USB_DEVICE_ID_GLORIOUS_MODEL_I	0x1503
+
 #define USB_VENDOR_ID_LCPOWER		0x1241
 #define USB_DEVICE_ID_LCPOWER_LC1000	0xf767
 
@@ -869,7 +868,6 @@
 #define USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_2		0xc534
 #define USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_LIGHTSPEED_1	0xc539
 #define USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_LIGHTSPEED_1_1	0xc53f
-#define USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_LIGHTSPEED_1_2	0xc547
 #define USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_POWERPLAY	0xc53a
 #define USB_DEVICE_ID_SPACETRAVELLER	0xc623
 #define USB_DEVICE_ID_SPACENAVIGATOR	0xc626
@@ -917,6 +915,7 @@
 #define USB_DEVICE_ID_PICK16F1454	0x0042
 #define USB_DEVICE_ID_PICK16F1454_V2	0xf2f7
 #define USB_DEVICE_ID_LUXAFOR		0xf372
+#define USB_DEVICE_ID_MCP2200		0x00df
 #define USB_DEVICE_ID_MCP2221		0x00dd
 
 #define USB_VENDOR_ID_MICROSOFT		0x045e
@@ -1160,6 +1159,10 @@
 #define USB_VENDOR_ID_SIGMATEL		0x066F
 #define USB_DEVICE_ID_SIGMATEL_STMP3780	0x3780
 
+#define USB_VENDOR_ID_SINOWEALTH  0x258a
+#define USB_DEVICE_ID_GLORIOUS_MODEL_D 0x0033
+#define USB_DEVICE_ID_GLORIOUS_MODEL_O 0x0036
+
 #define USB_VENDOR_ID_SIS_TOUCH		0x0457
 #define USB_DEVICE_ID_SIS9200_TOUCH	0x9200
 #define USB_DEVICE_ID_SIS817_TOUCH	0x0817
diff --git a/drivers/hid/hid-logitech-dj.c b/drivers/hid/hid-logitech-dj.c
index 8afe3be683ba..e6a8b6d8eab7 100644
--- a/drivers/hid/hid-logitech-dj.c
+++ b/drivers/hid/hid-logitech-dj.c
@@ -1695,12 +1695,11 @@ static int logi_dj_raw_event(struct hid_device *hdev,
 		}
 		/*
 		 * Mouse-only receivers send unnumbered mouse data. The 27 MHz
-		 * receiver uses 6 byte packets, the nano receiver 8 bytes,
-		 * the lightspeed receiver (Pro X Superlight) 13 bytes.
+		 * receiver uses 6 byte packets, the nano receiver 8 bytes.
 		 */
 		if (djrcv_dev->unnumbered_application == HID_GD_MOUSE &&
-		    size <= 13){
-			u8 mouse_report[14];
+		    size <= 8) {
+			u8 mouse_report[9];
 
 			/* Prepend report id */
 			mouse_report[0] = REPORT_TYPE_MOUSE;
@@ -1984,10 +1983,6 @@ static const struct hid_device_id logi_dj_receivers[] = {
 	  HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
 		USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_LIGHTSPEED_1_1),
 	 .driver_data = recvr_type_gaming_hidpp},
-	{ /* Logitech lightspeed receiver (0xc547) */
-	  HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH,
-		USB_DEVICE_ID_LOGITECH_NANO_RECEIVER_LIGHTSPEED_1_2),
-	 .driver_data = recvr_type_gaming_hidpp},
 
 	{ /* Logitech 27 MHz HID++ 1.0 receiver (0xc513) */
 	  HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_MX3000_RECEIVER),
diff --git a/drivers/hid/hid-mcp2200.c b/drivers/hid/hid-mcp2200.c
new file mode 100644
index 000000000000..bf57f7f6caa0
--- /dev/null
+++ b/drivers/hid/hid-mcp2200.c
@@ -0,0 +1,392 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * MCP2200 - Microchip USB to GPIO bridge
+ *
+ * Copyright (c) 2023, Johannes Roith <johannes@gnu-linux.rocks>
+ *
+ * Datasheet: https://ww1.microchip.com/downloads/en/DeviceDoc/22228A.pdf
+ * App Note for HID: https://ww1.microchip.com/downloads/en/DeviceDoc/93066A.pdf
+ */
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/gpio/driver.h>
+#include <linux/hid.h>
+#include <linux/hidraw.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include "hid-ids.h"
+
+/* Commands codes in a raw output report */
+#define SET_CLEAR_OUTPUTS	0x08
+#define CONFIGURE		0x10
+#define READ_EE			0x20
+#define WRITE_EE		0x40
+#define READ_ALL		0x80
+
+/* MCP GPIO direction encoding */
+enum MCP_IO_DIR {
+	MCP2200_DIR_OUT = 0x00,
+	MCP2200_DIR_IN  = 0x01,
+};
+
+/* Altternative pin assignments */
+#define TXLED		2
+#define RXLED		3
+#define USBCFG		6
+#define SSPND		7
+#define MCP_NGPIO	8
+
+/* CMD to set or clear a GPIO output */
+struct mcp_set_clear_outputs {
+	u8 cmd;
+	u8 dummys1[10];
+	u8 set_bmap;
+	u8 clear_bmap;
+	u8 dummys2[3];
+} __packed;
+
+/* CMD to configure the IOs */
+struct mcp_configure {
+	u8 cmd;
+	u8 dummys1[3];
+	u8 io_bmap;
+	u8 config_alt_pins;
+	u8 io_default_val_bmap;
+	u8 config_alt_options;
+	u8 baud_h;
+	u8 baud_l;
+	u8 dummys2[6];
+} __packed;
+
+/* CMD to read all parameters */
+struct mcp_read_all {
+	u8 cmd;
+	u8 dummys[15];
+} __packed;
+
+/* Response to the read all cmd */
+struct mcp_read_all_resp {
+	u8 cmd;
+	u8 eep_addr;
+	u8 dummy;
+	u8 eep_val;
+	u8 io_bmap;
+	u8 config_alt_pins;
+	u8 io_default_val_bmap;
+	u8 config_alt_options;
+	u8 baud_h;
+	u8 baud_l;
+	u8 io_port_val_bmap;
+	u8 dummys[5];
+} __packed;
+
+struct mcp2200 {
+	struct hid_device *hdev;
+	struct mutex lock;
+	struct completion wait_in_report;
+	u8 gpio_dir;
+	u8 gpio_val;
+	u8 gpio_inval;
+	u8 baud_h;
+	u8 baud_l;
+	u8 config_alt_pins;
+	u8 gpio_reset_val;
+	u8 config_alt_options;
+	int status;
+	struct gpio_chip gc;
+	u8 hid_report[16];
+};
+
+/* this executes the READ_ALL cmd */
+static int mcp_cmd_read_all(struct mcp2200 *mcp)
+{
+	struct mcp_read_all *read_all;
+	int len, t;
+
+	reinit_completion(&mcp->wait_in_report);
+
+	mutex_lock(&mcp->lock);
+
+	read_all = (struct mcp_read_all *) mcp->hid_report;
+	read_all->cmd = READ_ALL;
+	len = hid_hw_output_report(mcp->hdev, (u8 *) read_all,
+				   sizeof(struct mcp_read_all));
+
+	mutex_unlock(&mcp->lock);
+
+	if (len != sizeof(struct mcp_read_all))
+		return -EINVAL;
+
+	t = wait_for_completion_timeout(&mcp->wait_in_report,
+					msecs_to_jiffies(4000));
+	if (!t)
+		return -ETIMEDOUT;
+
+	/* return status, negative value if wrong response was received */
+	return mcp->status;
+}
+
+static void mcp_set_multiple(struct gpio_chip *gc, unsigned long *mask,
+			     unsigned long *bits)
+{
+	struct mcp2200 *mcp = gpiochip_get_data(gc);
+	u8 value;
+	int status;
+	struct mcp_set_clear_outputs *cmd;
+
+	mutex_lock(&mcp->lock);
+	cmd = (struct mcp_set_clear_outputs *) mcp->hid_report;
+
+	value = mcp->gpio_val & ~*mask;
+	value |= (*mask & *bits);
+
+	cmd->cmd = SET_CLEAR_OUTPUTS;
+	cmd->set_bmap = value;
+	cmd->clear_bmap = ~(value);
+
+	status = hid_hw_output_report(mcp->hdev, (u8 *) cmd,
+		       sizeof(struct mcp_set_clear_outputs));
+
+	if (status == sizeof(struct mcp_set_clear_outputs))
+		mcp->gpio_val = value;
+
+	mutex_unlock(&mcp->lock);
+}
+
+static void mcp_set(struct gpio_chip *gc, unsigned int gpio_nr, int value)
+{
+	unsigned long mask = 1 << gpio_nr;
+	unsigned long bmap_value = value << gpio_nr;
+
+	mcp_set_multiple(gc, &mask, &bmap_value);
+}
+
+static int mcp_get_multiple(struct gpio_chip *gc, unsigned long *mask,
+		unsigned long *bits)
+{
+	u32 val;
+	struct mcp2200 *mcp = gpiochip_get_data(gc);
+	int status;
+
+	status = mcp_cmd_read_all(mcp);
+	if (status)
+		return status;
+
+	val = mcp->gpio_inval;
+	*bits = (val & *mask);
+	return 0;
+}
+
+static int mcp_get(struct gpio_chip *gc, unsigned int gpio_nr)
+{
+	unsigned long mask = 0, bits = 0;
+
+	mask = (1 << gpio_nr);
+	mcp_get_multiple(gc, &mask, &bits);
+	return bits > 0;
+}
+
+static int mcp_get_direction(struct gpio_chip *gc, unsigned int gpio_nr)
+{
+	struct mcp2200 *mcp = gpiochip_get_data(gc);
+
+	return (mcp->gpio_dir & (MCP2200_DIR_IN << gpio_nr))
+		? GPIO_LINE_DIRECTION_IN : GPIO_LINE_DIRECTION_OUT;
+}
+
+static int mcp_set_direction(struct gpio_chip *gc, unsigned int gpio_nr,
+			     enum MCP_IO_DIR io_direction)
+{
+	struct mcp2200 *mcp = gpiochip_get_data(gc);
+	struct mcp_configure *conf;
+	int status;
+	/* after the configure cmd we will need to set the outputs again */
+	unsigned long mask = ~(mcp->gpio_dir); /* only set outputs */
+	unsigned long bits = mcp->gpio_val;
+	/* Offsets of alternative pins in config_alt_pins, 0 is not used */
+	u8 alt_pin_conf[8] = {SSPND, USBCFG, 0, 0, 0, 0, RXLED, TXLED};
+	u8 config_alt_pins = mcp->config_alt_pins;
+
+	/* Read in the reset baudrate first, we need it later */
+	status = mcp_cmd_read_all(mcp);
+	if (status != 0)
+		return status;
+
+	mutex_lock(&mcp->lock);
+	conf = (struct mcp_configure  *) mcp->hid_report;
+
+	/* configure will reset the chip! */
+	conf->cmd = CONFIGURE;
+	conf->io_bmap = (mcp->gpio_dir & ~(1 << gpio_nr))
+		| (io_direction << gpio_nr);
+	/* Don't overwrite the reset parameters */
+	conf->baud_h = mcp->baud_h;
+	conf->baud_l = mcp->baud_l;
+	conf->config_alt_options = mcp->config_alt_options;
+	conf->io_default_val_bmap = mcp->gpio_reset_val;
+	/* Adjust alt. func if necessary */
+	if (alt_pin_conf[gpio_nr])
+		config_alt_pins &= ~(1 << alt_pin_conf[gpio_nr]);
+	conf->config_alt_pins = config_alt_pins;
+
+	status = hid_hw_output_report(mcp->hdev, (u8 *) conf,
+				      sizeof(struct mcp_set_clear_outputs));
+
+	if (status == sizeof(struct mcp_set_clear_outputs)) {
+		mcp->gpio_dir = conf->io_bmap;
+		mcp->config_alt_pins = config_alt_pins;
+	} else {
+		mutex_unlock(&mcp->lock);
+		return -EIO;
+	}
+
+	mutex_unlock(&mcp->lock);
+
+	/* Configure CMD will clear all IOs -> rewrite them */
+	mcp_set_multiple(gc, &mask, &bits);
+	return 0;
+}
+
+static int mcp_direction_input(struct gpio_chip *gc, unsigned int gpio_nr)
+{
+	return mcp_set_direction(gc, gpio_nr, MCP2200_DIR_IN);
+}
+
+static int mcp_direction_output(struct gpio_chip *gc, unsigned int gpio_nr,
+				int value)
+{
+	int ret;
+	unsigned long mask, bmap_value;
+
+	mask = 1 << gpio_nr;
+	bmap_value = value << gpio_nr;
+
+	ret = mcp_set_direction(gc, gpio_nr, MCP2200_DIR_OUT);
+	if (!ret)
+		mcp_set_multiple(gc, &mask, &bmap_value);
+	return ret;
+}
+
+static const struct gpio_chip template_chip = {
+	.label			= "mcp2200",
+	.owner			= THIS_MODULE,
+	.get_direction		= mcp_get_direction,
+	.direction_input	= mcp_direction_input,
+	.direction_output	= mcp_direction_output,
+	.set			= mcp_set,
+	.set_multiple		= mcp_set_multiple,
+	.get			= mcp_get,
+	.get_multiple		= mcp_get_multiple,
+	.base			= -1,
+	.ngpio			= MCP_NGPIO,
+	.can_sleep		= true,
+};
+
+/*
+ * MCP2200 uses interrupt endpoint for input reports. This function
+ * is called by HID layer when it receives i/p report from mcp2200,
+ * which is actually a response to the previously sent command.
+ */
+static int mcp2200_raw_event(struct hid_device *hdev, struct hid_report *report,
+		u8 *data, int size)
+{
+	struct mcp2200 *mcp = hid_get_drvdata(hdev);
+	struct mcp_read_all_resp *all_resp;
+
+	switch (data[0]) {
+	case READ_ALL:
+		all_resp = (struct mcp_read_all_resp *) data;
+		mcp->status = 0;
+		mcp->gpio_inval = all_resp->io_port_val_bmap;
+		mcp->baud_h = all_resp->baud_h;
+		mcp->baud_l = all_resp->baud_l;
+		mcp->gpio_reset_val = all_resp->io_default_val_bmap;
+		mcp->config_alt_pins = all_resp->config_alt_pins;
+		mcp->config_alt_options = all_resp->config_alt_options;
+		break;
+	default:
+		mcp->status = -EIO;
+		break;
+	}
+
+	complete(&mcp->wait_in_report);
+	return 0;
+}
+
+static int mcp2200_probe(struct hid_device *hdev, const struct hid_device_id *id)
+{
+	int ret;
+	struct mcp2200 *mcp;
+
+	mcp = devm_kzalloc(&hdev->dev, sizeof(*mcp), GFP_KERNEL);
+	if (!mcp)
+		return -ENOMEM;
+
+	ret = hid_parse(hdev);
+	if (ret) {
+		hid_err(hdev, "can't parse reports\n");
+		return ret;
+	}
+
+	ret = hid_hw_start(hdev, 0);
+	if (ret) {
+		hid_err(hdev, "can't start hardware\n");
+		return ret;
+	}
+
+	hid_info(hdev, "USB HID v%x.%02x Device [%s] on %s\n", hdev->version >> 8,
+			hdev->version & 0xff, hdev->name, hdev->phys);
+
+	ret = hid_hw_open(hdev);
+	if (ret) {
+		hid_err(hdev, "can't open device\n");
+		hid_hw_stop(hdev);
+		return ret;
+	}
+
+	mutex_init(&mcp->lock);
+	init_completion(&mcp->wait_in_report);
+	hid_set_drvdata(hdev, mcp);
+	mcp->hdev = hdev;
+
+	mcp->gc = template_chip;
+	mcp->gc.parent = &hdev->dev;
+
+	ret = devm_gpiochip_add_data(&hdev->dev, &mcp->gc, mcp);
+	if (ret < 0) {
+		hid_err(hdev, "Unable to register gpiochip\n");
+		hid_hw_close(hdev);
+		hid_hw_stop(hdev);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void mcp2200_remove(struct hid_device *hdev)
+{
+	hid_hw_close(hdev);
+	hid_hw_stop(hdev);
+}
+
+static const struct hid_device_id mcp2200_devices[] = {
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROCHIP, USB_DEVICE_ID_MCP2200) },
+	{ }
+};
+MODULE_DEVICE_TABLE(hid, mcp2200_devices);
+
+static struct hid_driver mcp2200_driver = {
+	.name		= "mcp2200",
+	.id_table	= mcp2200_devices,
+	.probe		= mcp2200_probe,
+	.remove		= mcp2200_remove,
+	.raw_event	= mcp2200_raw_event,
+};
+
+/* Register with HID core */
+module_hid_driver(mcp2200_driver);
+
+MODULE_AUTHOR("Johannes Roith <johannes@gnu-linux.rocks>");
+MODULE_DESCRIPTION("MCP2200 Microchip HID USB to GPIO bridge");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hid/hid-mcp2221.c b/drivers/hid/hid-mcp2221.c
index 72883e0ce757..f9cceaeffd08 100644
--- a/drivers/hid/hid-mcp2221.c
+++ b/drivers/hid/hid-mcp2221.c
@@ -49,6 +49,7 @@ enum {
 	MCP2221_I2C_MASK_ADDR_NACK = 0x40,
 	MCP2221_I2C_WRADDRL_SEND = 0x21,
 	MCP2221_I2C_ADDR_NACK = 0x25,
+	MCP2221_I2C_READ_PARTIAL = 0x54,
 	MCP2221_I2C_READ_COMPL = 0x55,
 	MCP2221_ALT_F_NOT_GPIOV = 0xEE,
 	MCP2221_ALT_F_NOT_GPIOD = 0xEF,
@@ -187,6 +188,25 @@ static int mcp_cancel_last_cmd(struct mcp2221 *mcp)
 	return mcp_send_data_req_status(mcp, mcp->txbuf, 8);
 }
 
+/* Check if the last command succeeded or failed and return the result.
+ * If the command did fail, cancel that command which will free the i2c bus.
+ */
+static int mcp_chk_last_cmd_status_free_bus(struct mcp2221 *mcp)
+{
+	int ret;
+
+	ret = mcp_chk_last_cmd_status(mcp);
+	if (ret) {
+		/* The last command was a failure.
+		 * Send a cancel which will also free the bus.
+		 */
+		usleep_range(980, 1000);
+		mcp_cancel_last_cmd(mcp);
+	}
+
+	return ret;
+}
+
 static int mcp_set_i2c_speed(struct mcp2221 *mcp)
 {
 	int ret;
@@ -241,7 +261,7 @@ static int mcp_i2c_write(struct mcp2221 *mcp,
 		usleep_range(980, 1000);
 
 		if (last_status) {
-			ret = mcp_chk_last_cmd_status(mcp);
+			ret = mcp_chk_last_cmd_status_free_bus(mcp);
 			if (ret)
 				return ret;
 		}
@@ -278,6 +298,7 @@ static int mcp_i2c_smbus_read(struct mcp2221 *mcp,
 {
 	int ret;
 	u16 total_len;
+	int retries = 0;
 
 	mcp->txbuf[0] = type;
 	if (msg) {
@@ -301,20 +322,31 @@ static int mcp_i2c_smbus_read(struct mcp2221 *mcp,
 	mcp->rxbuf_idx = 0;
 
 	do {
+		/* Wait for the data to be read by the device */
+		usleep_range(980, 1000);
+
 		memset(mcp->txbuf, 0, 4);
 		mcp->txbuf[0] = MCP2221_I2C_GET_DATA;
 
 		ret = mcp_send_data_req_status(mcp, mcp->txbuf, 1);
-		if (ret)
-			return ret;
-
-		ret = mcp_chk_last_cmd_status(mcp);
-		if (ret)
-			return ret;
-
-		usleep_range(980, 1000);
+		if (ret) {
+			if (retries < 5) {
+				/* The data wasn't ready to read.
+				 * Wait a bit longer and try again.
+				 */
+				usleep_range(90, 100);
+				retries++;
+			} else {
+				return ret;
+			}
+		} else {
+			retries = 0;
+		}
 	} while (mcp->rxbuf_idx < total_len);
 
+	usleep_range(980, 1000);
+	ret = mcp_chk_last_cmd_status_free_bus(mcp);
+
 	return ret;
 }
 
@@ -328,11 +360,6 @@ static int mcp_i2c_xfer(struct i2c_adapter *adapter,
 
 	mutex_lock(&mcp->lock);
 
-	/* Setting speed before every transaction is required for mcp2221 */
-	ret = mcp_set_i2c_speed(mcp);
-	if (ret)
-		goto exit;
-
 	if (num == 1) {
 		if (msgs->flags & I2C_M_RD) {
 			ret = mcp_i2c_smbus_read(mcp, msgs, MCP2221_I2C_RD_DATA,
@@ -417,9 +444,7 @@ static int mcp_smbus_write(struct mcp2221 *mcp, u16 addr,
 	if (last_status) {
 		usleep_range(980, 1000);
 
-		ret = mcp_chk_last_cmd_status(mcp);
-		if (ret)
-			return ret;
+		ret = mcp_chk_last_cmd_status_free_bus(mcp);
 	}
 
 	return ret;
@@ -437,10 +462,6 @@ static int mcp_smbus_xfer(struct i2c_adapter *adapter, u16 addr,
 
 	mutex_lock(&mcp->lock);
 
-	ret = mcp_set_i2c_speed(mcp);
-	if (ret)
-		goto exit;
-
 	switch (size) {
 
 	case I2C_SMBUS_QUICK:
@@ -791,7 +812,8 @@ static int mcp2221_raw_event(struct hid_device *hdev,
 				mcp->status = -EIO;
 				break;
 			}
-			if (data[2] == MCP2221_I2C_READ_COMPL) {
+			if (data[2] == MCP2221_I2C_READ_COMPL ||
+			    data[2] == MCP2221_I2C_READ_PARTIAL) {
 				buf = mcp->rxbuf;
 				memcpy(&buf[mcp->rxbuf_idx], &data[4], data[3]);
 				mcp->rxbuf_idx = mcp->rxbuf_idx + data[3];
@@ -1142,27 +1164,35 @@ static int mcp2221_probe(struct hid_device *hdev,
 	if (ret)
 		return ret;
 
+	hid_device_io_start(hdev);
+
 	/* Set I2C bus clock diviser */
 	if (i2c_clk_freq > 400)
 		i2c_clk_freq = 400;
 	if (i2c_clk_freq < 50)
 		i2c_clk_freq = 50;
 	mcp->cur_i2c_clk_div = (12000000 / (i2c_clk_freq * 1000)) - 3;
+	ret = mcp_set_i2c_speed(mcp);
+	if (ret) {
+		hid_err(hdev, "can't set i2c speed: %d\n", ret);
+		return ret;
+	}
 
 	mcp->adapter.owner = THIS_MODULE;
 	mcp->adapter.class = I2C_CLASS_HWMON;
 	mcp->adapter.algo = &mcp_i2c_algo;
 	mcp->adapter.retries = 1;
 	mcp->adapter.dev.parent = &hdev->dev;
+	ACPI_COMPANION_SET(&mcp->adapter.dev, ACPI_COMPANION(hdev->dev.parent));
 	snprintf(mcp->adapter.name, sizeof(mcp->adapter.name),
 			"MCP2221 usb-i2c bridge");
 
+	i2c_set_adapdata(&mcp->adapter, mcp);
 	ret = devm_i2c_add_adapter(&hdev->dev, &mcp->adapter);
 	if (ret) {
 		hid_err(hdev, "can't add usb-i2c adapter: %d\n", ret);
 		return ret;
 	}
-	i2c_set_adapdata(&mcp->adapter, mcp);
 
 #if IS_REACHABLE(CONFIG_GPIOLIB)
 	/* Setup GPIO chip */
diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c
index 5a48fcaa32f0..ea472923fab0 100644
--- a/drivers/hid/hid-quirks.c
+++ b/drivers/hid/hid-quirks.c
@@ -33,6 +33,7 @@ static const struct hid_device_id hid_quirks[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_AKAI, USB_DEVICE_ID_AKAI_MPKMINI2), HID_QUIRK_NO_INIT_REPORTS },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ALPS, USB_DEVICE_ID_IBM_GAMEPAD), HID_QUIRK_BADPAD },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_AMI, USB_DEVICE_ID_AMI_VIRT_KEYBOARD_AND_MOUSE), HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_REVB_ANSI), HID_QUIRK_ALWAYS_POLL },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_2PORTKVM), HID_QUIRK_NOGET },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_4PORTKVMC), HID_QUIRK_NOGET },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_4PORTKVM), HID_QUIRK_NOGET },
diff --git a/drivers/hsi/controllers/omap_ssi_core.c b/drivers/hsi/controllers/omap_ssi_core.c
index 4c1a00f9929e..6802efb4d6cd 100644
--- a/drivers/hsi/controllers/omap_ssi_core.c
+++ b/drivers/hsi/controllers/omap_ssi_core.c
@@ -355,7 +355,7 @@ static int ssi_add_controller(struct hsi_controller *ssi,
 	if (!omap_ssi)
 		return -ENOMEM;
 
-	err = ida_simple_get(&platform_omap_ssi_ida, 0, 0, GFP_KERNEL);
+	err = ida_alloc(&platform_omap_ssi_ida, GFP_KERNEL);
 	if (err < 0)
 		return err;
 	ssi->id = err;
@@ -417,7 +417,7 @@ static int ssi_add_controller(struct hsi_controller *ssi,
 	return 0;
 
 out_err:
-	ida_simple_remove(&platform_omap_ssi_ida, ssi->id);
+	ida_free(&platform_omap_ssi_ida, ssi->id);
 	return err;
 }
 
@@ -451,7 +451,7 @@ static void ssi_remove_controller(struct hsi_controller *ssi)
 	tasklet_kill(&omap_ssi->gdd_tasklet);
 	hsi_unregister_controller(ssi);
 	clk_notifier_unregister(omap_ssi->fck, &omap_ssi->fck_nb);
-	ida_simple_remove(&platform_omap_ssi_ida, id);
+	ida_free(&platform_omap_ssi_ida, id);
 }
 
 static inline int ssi_of_get_available_ports_count(const struct device_node *np)
diff --git a/drivers/hwmon/aspeed-pwm-tacho.c b/drivers/hwmon/aspeed-pwm-tacho.c
index 997df4b40509..f6e1e55e8292 100644
--- a/drivers/hwmon/aspeed-pwm-tacho.c
+++ b/drivers/hwmon/aspeed-pwm-tacho.c
@@ -166,6 +166,8 @@
 
 #define MAX_CDEV_NAME_LEN 16
 
+#define MAX_ASPEED_FAN_TACH_CHANNELS 16
+
 struct aspeed_cooling_device {
 	char name[16];
 	struct aspeed_pwm_tacho_data *priv;
@@ -181,7 +183,7 @@ struct aspeed_pwm_tacho_data {
 	struct reset_control *rst;
 	unsigned long clk_freq;
 	bool pwm_present[8];
-	bool fan_tach_present[16];
+	bool fan_tach_present[MAX_ASPEED_FAN_TACH_CHANNELS];
 	u8 type_pwm_clock_unit[3];
 	u8 type_pwm_clock_division_h[3];
 	u8 type_pwm_clock_division_l[3];
@@ -190,7 +192,7 @@ struct aspeed_pwm_tacho_data {
 	u16 type_fan_tach_unit[3];
 	u8 pwm_port_type[8];
 	u8 pwm_port_fan_ctrl[8];
-	u8 fan_tach_ch_source[16];
+	u8 fan_tach_ch_source[MAX_ASPEED_FAN_TACH_CHANNELS];
 	struct aspeed_cooling_device *cdev[8];
 	const struct attribute_group *groups[3];
 };
@@ -737,20 +739,27 @@ static void aspeed_create_pwm_port(struct aspeed_pwm_tacho_data *priv,
 	aspeed_set_pwm_port_fan_ctrl(priv, pwm_port, INIT_FAN_CTRL);
 }
 
-static void aspeed_create_fan_tach_channel(struct aspeed_pwm_tacho_data *priv,
-					   u8 *fan_tach_ch,
-					   int count,
-					   u8 pwm_source)
+static int aspeed_create_fan_tach_channel(struct device *dev,
+					  struct aspeed_pwm_tacho_data *priv,
+					  u8 *fan_tach_ch,
+					  int count,
+					  u8 pwm_source)
 {
 	u8 val, index;
 
 	for (val = 0; val < count; val++) {
 		index = fan_tach_ch[val];
+		if (index >= MAX_ASPEED_FAN_TACH_CHANNELS) {
+			dev_err(dev, "Invalid Fan Tach input channel %u\n.", index);
+			return -EINVAL;
+		}
 		aspeed_set_fan_tach_ch_enable(priv->regmap, index, true);
 		priv->fan_tach_present[index] = true;
 		priv->fan_tach_ch_source[index] = pwm_source;
 		aspeed_set_fan_tach_ch_source(priv->regmap, index, pwm_source);
 	}
+
+	return 0;
 }
 
 static int
@@ -874,7 +883,10 @@ static int aspeed_create_fan(struct device *dev,
 					fan_tach_ch, count);
 	if (ret)
 		return ret;
-	aspeed_create_fan_tach_channel(priv, fan_tach_ch, count, pwm_port);
+
+	ret = aspeed_create_fan_tach_channel(dev, priv, fan_tach_ch, count, pwm_port);
+	if (ret)
+		return ret;
 
 	return 0;
 }
diff --git a/drivers/hwmon/emc1403.c b/drivers/hwmon/emc1403.c
index bb7c859e799d..1332e4ac078c 100644
--- a/drivers/hwmon/emc1403.c
+++ b/drivers/hwmon/emc1403.c
@@ -346,6 +346,9 @@ static int emc1403_detect(struct i2c_client *client,
 	case 0x27:
 		strscpy(info->type, "emc1424", I2C_NAME_SIZE);
 		break;
+	case 0x60:
+		strscpy(info->type, "emc1442", I2C_NAME_SIZE);
+		break;
 	default:
 		return -ENODEV;
 	}
@@ -430,7 +433,7 @@ static int emc1403_probe(struct i2c_client *client)
 }
 
 static const unsigned short emc1403_address_list[] = {
-	0x18, 0x1c, 0x29, 0x4c, 0x4d, 0x5c, I2C_CLIENT_END
+	0x18, 0x1c, 0x29, 0x3c, 0x4c, 0x4d, 0x5c, I2C_CLIENT_END
 };
 
 /* Last digit of chip name indicates number of channels */
@@ -444,6 +447,7 @@ static const struct i2c_device_id emc1403_idtable[] = {
 	{ "emc1422", emc1402 },
 	{ "emc1423", emc1403 },
 	{ "emc1424", emc1404 },
+	{ "emc1442", emc1402 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, emc1403_idtable);
diff --git a/drivers/hwmon/ltc2991.c b/drivers/hwmon/ltc2991.c
index bd63c61129a9..80a6e391f266 100644
--- a/drivers/hwmon/ltc2991.c
+++ b/drivers/hwmon/ltc2991.c
@@ -54,7 +54,6 @@
 #define LTC2991_VCC_CH_NR		0
 
 struct ltc2991_state {
-	struct device		*dev;
 	struct regmap		*regmap;
 	u32			r_sense_uohm[LTC2991_MAX_CHANNEL];
 	bool			temp_en[LTC2991_MAX_CHANNEL];
@@ -283,19 +282,19 @@ static const struct regmap_config ltc2991_regmap_config = {
 	.max_register = 0x1D,
 };
 
-static int ltc2991_init(struct ltc2991_state *st)
+static int ltc2991_init(struct ltc2991_state *st, struct device *dev)
 {
 	struct fwnode_handle *child;
 	int ret;
 	u32 val, addr;
 	u8 v5_v8_reg_data = 0, v1_v4_reg_data = 0;
 
-	ret = devm_regulator_get_enable(st->dev, "vcc");
+	ret = devm_regulator_get_enable(dev, "vcc");
 	if (ret)
-		return dev_err_probe(st->dev, ret,
+		return dev_err_probe(dev, ret,
 				     "failed to enable regulator\n");
 
-	device_for_each_child_node(st->dev, child) {
+	device_for_each_child_node(dev, child) {
 		ret = fwnode_property_read_u32(child, "reg", &addr);
 		if (ret < 0) {
 			fwnode_handle_put(child);
@@ -312,7 +311,7 @@ static int ltc2991_init(struct ltc2991_state *st)
 					       &val);
 		if (!ret) {
 			if (!val)
-				return dev_err_probe(st->dev, -EINVAL,
+				return dev_err_probe(dev, -EINVAL,
 						     "shunt resistor value cannot be zero\n");
 
 			st->r_sense_uohm[addr] = val;
@@ -361,19 +360,19 @@ static int ltc2991_init(struct ltc2991_state *st)
 
 	ret = regmap_write(st->regmap, LTC2991_V5_V8_CTRL, v5_v8_reg_data);
 	if (ret)
-		return dev_err_probe(st->dev, ret,
+		return dev_err_probe(dev, ret,
 				     "Error: Failed to set V5-V8 CTRL reg.\n");
 
 	ret = regmap_write(st->regmap, LTC2991_V1_V4_CTRL, v1_v4_reg_data);
 	if (ret)
-		return dev_err_probe(st->dev, ret,
+		return dev_err_probe(dev, ret,
 				     "Error: Failed to set V1-V4 CTRL reg.\n");
 
 	ret = regmap_write(st->regmap, LTC2991_PWM_TH_LSB_T_INT,
 			   LTC2991_REPEAT_ACQ_EN);
 	if (ret)
-		return dev_err_probe(st->dev, ret,
-				     "Error: Failed to set contiuous mode.\n");
+		return dev_err_probe(dev, ret,
+				     "Error: Failed to set continuous mode.\n");
 
 	/* Enable all channels and trigger conversions */
 	return regmap_write(st->regmap, LTC2991_CH_EN_TRIGGER,
@@ -392,12 +391,11 @@ static int ltc2991_i2c_probe(struct i2c_client *client)
 	if (!st)
 		return -ENOMEM;
 
-	st->dev = &client->dev;
 	st->regmap = devm_regmap_init_i2c(client, &ltc2991_regmap_config);
 	if (IS_ERR(st->regmap))
 		return PTR_ERR(st->regmap);
 
-	ret = ltc2991_init(st);
+	ret = ltc2991_init(st, &client->dev);
 	if (ret)
 		return ret;
 
diff --git a/drivers/hwmon/max31827.c b/drivers/hwmon/max31827.c
index fd1fed1a797c..a1ce65145669 100644
--- a/drivers/hwmon/max31827.c
+++ b/drivers/hwmon/max31827.c
@@ -12,6 +12,7 @@
 #include <linux/i2c.h>
 #include <linux/mutex.h>
 #include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
 
 #define MAX31827_T_REG			0x0
 #define MAX31827_CONFIGURATION_REG	0x2
diff --git a/drivers/hwmon/max6650.c b/drivers/hwmon/max6650.c
index cc8428a3045d..9649c6611d5f 100644
--- a/drivers/hwmon/max6650.c
+++ b/drivers/hwmon/max6650.c
@@ -26,7 +26,7 @@
 #include <linux/hwmon.h>
 #include <linux/hwmon-sysfs.h>
 #include <linux/err.h>
-#include <linux/of_device.h>
+#include <linux/of.h>
 #include <linux/thermal.h>
 
 /*
@@ -763,8 +763,6 @@ static int max6650_probe(struct i2c_client *client)
 {
 	struct thermal_cooling_device *cooling_dev;
 	struct device *dev = &client->dev;
-	const struct of_device_id *of_id =
-		of_match_device(of_match_ptr(max6650_dt_match), dev);
 	struct max6650_data *data;
 	struct device *hwmon_dev;
 	int err;
@@ -776,8 +774,8 @@ static int max6650_probe(struct i2c_client *client)
 	data->client = client;
 	i2c_set_clientdata(client, data);
 	mutex_init(&data->update_lock);
-	data->nr_fans = of_id ? (int)(uintptr_t)of_id->data :
-				i2c_match_id(max6650_id, client)->driver_data;
+
+	data->nr_fans = (uintptr_t)i2c_get_match_data(client);
 
 	/*
 	 * Initialize the max6650 chip
diff --git a/drivers/hwmon/nct6775-core.c b/drivers/hwmon/nct6775-core.c
index d928eb8ae5a3..7534d5c657bd 100644
--- a/drivers/hwmon/nct6775-core.c
+++ b/drivers/hwmon/nct6775-core.c
@@ -63,19 +63,19 @@
 
 /* used to set data->name = nct6775_device_names[data->sio_kind] */
 static const char * const nct6775_device_names[] = {
-	"nct6106",
-	"nct6116",
-	"nct6775",
-	"nct6776",
-	"nct6779",
-	"nct6791",
-	"nct6792",
-	"nct6793",
-	"nct6795",
-	"nct6796",
-	"nct6797",
-	"nct6798",
-	"nct6799",
+	[nct6106] = "nct6106",
+	[nct6116] = "nct6116",
+	[nct6775] = "nct6775",
+	[nct6776] = "nct6776",
+	[nct6779] = "nct6779",
+	[nct6791] = "nct6791",
+	[nct6792] = "nct6792",
+	[nct6793] = "nct6793",
+	[nct6795] = "nct6795",
+	[nct6796] = "nct6796",
+	[nct6797] = "nct6797",
+	[nct6798] = "nct6798",
+	[nct6799] = "nct6799",
 };
 
 /* Common and NCT6775 specific data */
diff --git a/drivers/hwmon/nct6775-i2c.c b/drivers/hwmon/nct6775-i2c.c
index 87a4fc78c571..aff69fa50461 100644
--- a/drivers/hwmon/nct6775-i2c.c
+++ b/drivers/hwmon/nct6775-i2c.c
@@ -21,7 +21,7 @@
 #include <linux/hwmon.h>
 #include <linux/hwmon-sysfs.h>
 #include <linux/err.h>
-#include <linux/of_device.h>
+#include <linux/of.h>
 #include <linux/regmap.h>
 #include "nct6775.h"
 
@@ -155,23 +155,13 @@ static const struct regmap_config nct6775_i2c_regmap_config = {
 static int nct6775_i2c_probe(struct i2c_client *client)
 {
 	struct nct6775_data *data;
-	const struct of_device_id *of_id;
-	const struct i2c_device_id *i2c_id;
 	struct device *dev = &client->dev;
 
-	of_id = of_match_device(nct6775_i2c_of_match, dev);
-	i2c_id = i2c_match_id(nct6775_i2c_id, client);
-
-	if (of_id && (unsigned long)of_id->data != i2c_id->driver_data)
-		dev_notice(dev, "Device mismatch: %s in device tree, %s detected\n",
-			   of_id->name, i2c_id->name);
-
 	data = devm_kzalloc(&client->dev, sizeof(*data), GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
 
-	data->kind = i2c_id->driver_data;
-
+	data->kind = (enum kinds)(uintptr_t)i2c_get_match_data(client);
 	data->read_only = true;
 	data->driver_data = client;
 	data->driver_init = nct6775_i2c_probe_init;
diff --git a/drivers/hwmon/nct6775-platform.c b/drivers/hwmon/nct6775-platform.c
index 0adeeab7ee03..9aa4dcf4a6f3 100644
--- a/drivers/hwmon/nct6775-platform.c
+++ b/drivers/hwmon/nct6775-platform.c
@@ -23,19 +23,19 @@
 enum sensor_access { access_direct, access_asuswmi };
 
 static const char * const nct6775_sio_names[] __initconst = {
-	"NCT6106D",
-	"NCT6116D",
-	"NCT6775F",
-	"NCT6776D/F",
-	"NCT6779D",
-	"NCT6791D",
-	"NCT6792D",
-	"NCT6793D",
-	"NCT6795D",
-	"NCT6796D",
-	"NCT6797D",
-	"NCT6798D",
-	"NCT6796D-S/NCT6799D-R",
+	[nct6106] = "NCT6106D",
+	[nct6116] = "NCT6116D",
+	[nct6775] = "NCT6775F",
+	[nct6776] = "NCT6776D/F",
+	[nct6779] = "NCT6779D",
+	[nct6791] = "NCT6791D",
+	[nct6792] = "NCT6792D",
+	[nct6793] = "NCT6793D",
+	[nct6795] = "NCT6795D",
+	[nct6796] = "NCT6796D",
+	[nct6797] = "NCT6797D",
+	[nct6798] = "NCT6798D",
+	[nct6799] = "NCT6796D-S/NCT6799D-R",
 };
 
 static unsigned short force_id;
diff --git a/drivers/hwmon/nct6775.h b/drivers/hwmon/nct6775.h
index 296eff99d003..d31e7a030216 100644
--- a/drivers/hwmon/nct6775.h
+++ b/drivers/hwmon/nct6775.h
@@ -4,7 +4,7 @@
 
 #include <linux/types.h>
 
-enum kinds { nct6106, nct6116, nct6775, nct6776, nct6779, nct6791, nct6792,
+enum kinds { nct6106 = 1, nct6116, nct6775, nct6776, nct6779, nct6791, nct6792,
 	     nct6793, nct6795, nct6796, nct6797, nct6798, nct6799 };
 enum pwm_enable { off, manual, thermal_cruise, speed_cruise, sf3, sf4 };
 
diff --git a/drivers/hwmon/npcm750-pwm-fan.c b/drivers/hwmon/npcm750-pwm-fan.c
index 4702e4edc662..d9733da8ea34 100644
--- a/drivers/hwmon/npcm750-pwm-fan.c
+++ b/drivers/hwmon/npcm750-pwm-fan.c
@@ -46,9 +46,9 @@
 #define NPCM7XX_PWM_CTRL_CH3_EN_BIT		BIT(16)
 
 /* Define the maximum PWM channel number */
-#define NPCM7XX_PWM_MAX_CHN_NUM			8
+#define NPCM7XX_PWM_MAX_CHN_NUM			12
 #define NPCM7XX_PWM_MAX_CHN_NUM_IN_A_MODULE	4
-#define NPCM7XX_PWM_MAX_MODULES                 2
+#define NPCM7XX_PWM_MAX_MODULES                 3
 
 /* Define the Counter Register, value = 100 for match 100% */
 #define NPCM7XX_PWM_COUNTER_DEFAULT_NUM		255
@@ -171,6 +171,10 @@
 #define FAN_PREPARE_TO_GET_FIRST_CAPTURE	0x01
 #define FAN_ENOUGH_SAMPLE			0x02
 
+struct npcm_hwmon_info {
+	u32 pwm_max_channel;
+};
+
 struct npcm7xx_fan_dev {
 	u8 fan_st_flg;
 	u8 fan_pls_per_rev;
@@ -204,6 +208,7 @@ struct npcm7xx_pwm_fan_data {
 	struct timer_list fan_timer;
 	struct npcm7xx_fan_dev fan_dev[NPCM7XX_FAN_MAX_CHN_NUM];
 	struct npcm7xx_cooling_device *cdev[NPCM7XX_PWM_MAX_CHN_NUM];
+	const struct npcm_hwmon_info *info;
 	u8 fan_select;
 };
 
@@ -542,7 +547,7 @@ static umode_t npcm7xx_pwm_is_visible(const void *_data, u32 attr, int channel)
 {
 	const struct npcm7xx_pwm_fan_data *data = _data;
 
-	if (!data->pwm_present[channel])
+	if (!data->pwm_present[channel] || channel >= data->info->pwm_max_channel)
 		return 0;
 
 	switch (attr) {
@@ -638,6 +643,10 @@ static const struct hwmon_channel_info * const npcm7xx_info[] = {
 			   HWMON_PWM_INPUT,
 			   HWMON_PWM_INPUT,
 			   HWMON_PWM_INPUT,
+			   HWMON_PWM_INPUT,
+			   HWMON_PWM_INPUT,
+			   HWMON_PWM_INPUT,
+			   HWMON_PWM_INPUT,
 			   HWMON_PWM_INPUT),
 	HWMON_CHANNEL_INFO(fan,
 			   HWMON_F_INPUT,
@@ -670,6 +679,14 @@ static const struct hwmon_chip_info npcm7xx_chip_info = {
 	.info = npcm7xx_info,
 };
 
+static const struct npcm_hwmon_info npxm7xx_hwmon_info = {
+	.pwm_max_channel = 8,
+};
+
+static const struct npcm_hwmon_info npxm8xx_hwmon_info = {
+	.pwm_max_channel = 12,
+};
+
 static u32 npcm7xx_pwm_init(struct npcm7xx_pwm_fan_data *data)
 {
 	int m, ch;
@@ -925,6 +942,10 @@ static int npcm7xx_pwm_fan_probe(struct platform_device *pdev)
 	if (!data)
 		return -ENOMEM;
 
+	data->info = device_get_match_data(dev);
+	if (!data->info)
+		return -EINVAL;
+
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pwm");
 	if (!res) {
 		dev_err(dev, "pwm resource not found\n");
@@ -1017,7 +1038,8 @@ static int npcm7xx_pwm_fan_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id of_pwm_fan_match_table[] = {
-	{ .compatible = "nuvoton,npcm750-pwm-fan", },
+	{ .compatible = "nuvoton,npcm750-pwm-fan", .data = &npxm7xx_hwmon_info},
+	{ .compatible = "nuvoton,npcm845-pwm-fan", .data = &npxm8xx_hwmon_info},
 	{},
 };
 MODULE_DEVICE_TABLE(of, of_pwm_fan_match_table);
diff --git a/drivers/hwmon/pmbus/Kconfig b/drivers/hwmon/pmbus/Kconfig
index b4e93bd5835e..17c98fca5bb6 100644
--- a/drivers/hwmon/pmbus/Kconfig
+++ b/drivers/hwmon/pmbus/Kconfig
@@ -333,6 +333,15 @@ config SENSORS_MP5023
 	  This driver can also be built as a module. If so, the module will
 	  be called mp5023.
 
+config SENSORS_MP5990
+	tristate "MPS MP5990"
+	help
+	  If you say yes here you get hardware monitoring support for MPS
+	  MP5990.
+
+	  This driver can also be built as a module. If so, the module will
+	  be called mp5990.
+
 config SENSORS_MPQ7932_REGULATOR
 	bool "Regulator support for MPQ7932"
 	depends on SENSORS_MPQ7932 && REGULATOR
diff --git a/drivers/hwmon/pmbus/Makefile b/drivers/hwmon/pmbus/Makefile
index 84ee960a6c2d..212d9ca0acc9 100644
--- a/drivers/hwmon/pmbus/Makefile
+++ b/drivers/hwmon/pmbus/Makefile
@@ -35,6 +35,7 @@ obj-$(CONFIG_SENSORS_MAX8688)	+= max8688.o
 obj-$(CONFIG_SENSORS_MP2888)	+= mp2888.o
 obj-$(CONFIG_SENSORS_MP2975)	+= mp2975.o
 obj-$(CONFIG_SENSORS_MP5023)	+= mp5023.o
+obj-$(CONFIG_SENSORS_MP5990)	+= mp5990.o
 obj-$(CONFIG_SENSORS_MPQ7932)	+= mpq7932.o
 obj-$(CONFIG_SENSORS_PLI1209BC)	+= pli1209bc.o
 obj-$(CONFIG_SENSORS_PM6764TR)	+= pm6764tr.o
diff --git a/drivers/hwmon/pmbus/lm25066.c b/drivers/hwmon/pmbus/lm25066.c
index 929fa6d34efd..bd43457094a6 100644
--- a/drivers/hwmon/pmbus/lm25066.c
+++ b/drivers/hwmon/pmbus/lm25066.c
@@ -14,10 +14,10 @@
 #include <linux/slab.h>
 #include <linux/i2c.h>
 #include <linux/log2.h>
-#include <linux/of_device.h>
+#include <linux/of.h>
 #include "pmbus.h"
 
-enum chips { lm25056, lm25066, lm5064, lm5066, lm5066i };
+enum chips { lm25056 = 1, lm25066, lm5064, lm5066, lm5066i };
 
 #define LM25066_READ_VAUX		0xd0
 #define LM25066_MFR_READ_IIN		0xd1
@@ -468,8 +468,6 @@ static int lm25066_probe(struct i2c_client *client)
 	struct lm25066_data *data;
 	struct pmbus_driver_info *info;
 	const struct __coeff *coeff;
-	const struct of_device_id *of_id;
-	const struct i2c_device_id *i2c_id;
 
 	if (!i2c_check_functionality(client->adapter,
 				     I2C_FUNC_SMBUS_READ_BYTE_DATA))
@@ -484,14 +482,8 @@ static int lm25066_probe(struct i2c_client *client)
 	if (config < 0)
 		return config;
 
-	i2c_id = i2c_match_id(lm25066_id, client);
+	data->id = (enum chips)i2c_get_match_data(client);
 
-	of_id = of_match_device(lm25066_of_match, &client->dev);
-	if (of_id && (unsigned long)of_id->data != i2c_id->driver_data)
-		dev_notice(&client->dev, "Device mismatch: %s in device tree, %s detected\n",
-			   of_id->name, i2c_id->name);
-
-	data->id = i2c_id->driver_data;
 	info = &data->info;
 
 	info->pages = 1;
diff --git a/drivers/hwmon/pmbus/mp5990.c b/drivers/hwmon/pmbus/mp5990.c
new file mode 100644
index 000000000000..1dfbab25a064
--- /dev/null
+++ b/drivers/hwmon/pmbus/mp5990.c
@@ -0,0 +1,179 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Driver for MPS MP5990 Hot-Swap Controller
+ */
+
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include "pmbus.h"
+
+#define MP5990_EFUSE_CFG	(0xC4)
+#define MP5990_VOUT_FORMAT	BIT(9)
+
+struct mp5990_data {
+	struct pmbus_driver_info info;
+	u8 vout_mode;
+	u8 vout_linear_exponent;
+};
+
+#define to_mp5990_data(x)  container_of(x, struct mp5990_data, info)
+
+static int mp5990_read_byte_data(struct i2c_client *client, int page, int reg)
+{
+	const struct pmbus_driver_info *info = pmbus_get_driver_info(client);
+	struct mp5990_data *data = to_mp5990_data(info);
+
+	switch (reg) {
+	case PMBUS_VOUT_MODE:
+		if (data->vout_mode == linear) {
+			/*
+			 * The VOUT format used by the chip is linear11,
+			 * not linear16. Report that VOUT is in linear mode
+			 * and return exponent value extracted while probing
+			 * the chip.
+			 */
+			return data->vout_linear_exponent;
+		}
+
+		/*
+		 * The datasheet does not support the VOUT command,
+		 * but the device responds with a default value of 0x17.
+		 * In the standard, 0x17 represents linear mode.
+		 * Therefore, we should report that VOUT is in direct
+		 * format when the chip is configured for it.
+		 */
+		return PB_VOUT_MODE_DIRECT;
+
+	default:
+		return -ENODATA;
+	}
+}
+
+static int mp5990_read_word_data(struct i2c_client *client, int page,
+				 int phase, int reg)
+{
+	const struct pmbus_driver_info *info = pmbus_get_driver_info(client);
+	struct mp5990_data *data = to_mp5990_data(info);
+	int ret;
+	s32 mantissa;
+
+	switch (reg) {
+	case PMBUS_READ_VOUT:
+		ret = pmbus_read_word_data(client, page, phase, reg);
+		if (ret < 0)
+			return ret;
+		/*
+		 * Because the VOUT format used by the chip is linear11 and not
+		 * linear16, we disregard bits[15:11]. The exponent is reported
+		 * as part of the VOUT_MODE command.
+		 */
+		if (data->vout_mode == linear) {
+			mantissa = ((s16)((ret & 0x7ff) << 5)) >> 5;
+			ret = mantissa;
+		}
+		break;
+	default:
+		return -ENODATA;
+	}
+
+	return ret;
+}
+
+static struct pmbus_driver_info mp5990_info = {
+	.pages = 1,
+	.format[PSC_VOLTAGE_IN] = direct,
+	.format[PSC_VOLTAGE_OUT] = direct,
+	.format[PSC_CURRENT_OUT] = direct,
+	.format[PSC_POWER] = direct,
+	.format[PSC_TEMPERATURE] = direct,
+	.m[PSC_VOLTAGE_IN] = 32,
+	.b[PSC_VOLTAGE_IN] = 0,
+	.R[PSC_VOLTAGE_IN] = 0,
+	.m[PSC_VOLTAGE_OUT] = 32,
+	.b[PSC_VOLTAGE_OUT] = 0,
+	.R[PSC_VOLTAGE_OUT] = 0,
+	.m[PSC_CURRENT_OUT] = 16,
+	.b[PSC_CURRENT_OUT] = 0,
+	.R[PSC_CURRENT_OUT] = 0,
+	.m[PSC_POWER] = 1,
+	.b[PSC_POWER] = 0,
+	.R[PSC_POWER] = 0,
+	.m[PSC_TEMPERATURE] = 1,
+	.b[PSC_TEMPERATURE] = 0,
+	.R[PSC_TEMPERATURE] = 0,
+	.func[0] =
+		PMBUS_HAVE_VIN | PMBUS_HAVE_VOUT | PMBUS_HAVE_PIN |
+		PMBUS_HAVE_TEMP | PMBUS_HAVE_IOUT |
+		PMBUS_HAVE_STATUS_INPUT | PMBUS_HAVE_STATUS_TEMP,
+	.read_byte_data = mp5990_read_byte_data,
+	.read_word_data = mp5990_read_word_data,
+};
+
+static int mp5990_probe(struct i2c_client *client)
+{
+	struct pmbus_driver_info *info;
+	struct mp5990_data *data;
+	int ret;
+
+	data = devm_kzalloc(&client->dev, sizeof(struct mp5990_data),
+			    GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	memcpy(&data->info, &mp5990_info, sizeof(*info));
+	info = &data->info;
+
+	/* Read Vout Config */
+	ret = i2c_smbus_read_word_data(client, MP5990_EFUSE_CFG);
+	if (ret < 0) {
+		dev_err(&client->dev, "Can't get vout mode.");
+		return ret;
+	}
+
+	/*
+	 * EFUSE_CFG (0xC4) bit9=1 is linear mode, bit=0 is direct mode.
+	 */
+	if (ret & MP5990_VOUT_FORMAT) {
+		data->vout_mode = linear;
+		data->info.format[PSC_VOLTAGE_IN] = linear;
+		data->info.format[PSC_VOLTAGE_OUT] = linear;
+		data->info.format[PSC_CURRENT_OUT] = linear;
+		data->info.format[PSC_POWER] = linear;
+		ret = i2c_smbus_read_word_data(client, PMBUS_READ_VOUT);
+		if (ret < 0) {
+			dev_err(&client->dev, "Can't get vout exponent.");
+			return ret;
+		}
+		data->vout_linear_exponent = (u8)((ret >> 11) & 0x1f);
+	} else {
+		data->vout_mode = direct;
+	}
+	return pmbus_do_probe(client, info);
+}
+
+static const struct of_device_id mp5990_of_match[] = {
+	{ .compatible = "mps,mp5990" },
+	{}
+};
+
+static const struct i2c_device_id mp5990_id[] = {
+	{"mp5990", 0},
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, mp5990_id);
+
+static struct i2c_driver mp5990_driver = {
+	.driver = {
+		   .name = "mp5990",
+		   .of_match_table = mp5990_of_match,
+	},
+	.probe = mp5990_probe,
+	.id_table = mp5990_id,
+};
+module_i2c_driver(mp5990_driver);
+
+MODULE_AUTHOR("Peter Yin <peter.yin@quantatw.com>");
+MODULE_DESCRIPTION("PMBus driver for MP5990 HSC");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS(PMBUS);
diff --git a/drivers/hwmon/sht4x.c b/drivers/hwmon/sht4x.c
index 7ee797410458..4883755d4b1e 100644
--- a/drivers/hwmon/sht4x.c
+++ b/drivers/hwmon/sht4x.c
@@ -49,6 +49,7 @@ DECLARE_CRC8_TABLE(sht4x_crc8_table);
  * struct sht4x_data - All the data required to operate an SHT4X chip
  * @client: the i2c client associated with the SHT4X
  * @lock: a mutex that is used to prevent parallel access to the i2c client
+ * @valid: validity of fields below
  * @update_interval: the minimum poll interval
  * @last_updated: the previous time that the SHT4X was polled
  * @temperature: the latest temperature value received from the SHT4X
@@ -66,7 +67,7 @@ struct sht4x_data {
 
 /**
  * sht4x_read_values() - read and parse the raw data from the SHT4X
- * @sht4x_data: the struct sht4x_data to use for the lock
+ * @data: the struct sht4x_data to use for the lock
  * Return: 0 if successful, -ERRNO if not
  */
 static int sht4x_read_values(struct sht4x_data *data)
diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c
index 9fabe00a40d6..d7f0e231feb9 100644
--- a/drivers/hwtracing/coresight/coresight-core.c
+++ b/drivers/hwtracing/coresight/coresight-core.c
@@ -1093,6 +1093,7 @@ static int coresight_validate_source(struct coresight_device *csdev,
 
 	if (subtype != CORESIGHT_DEV_SUBTYPE_SOURCE_PROC &&
 	    subtype != CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE &&
+	    subtype != CORESIGHT_DEV_SUBTYPE_SOURCE_TPDM &&
 	    subtype != CORESIGHT_DEV_SUBTYPE_SOURCE_OTHERS) {
 		dev_err(&csdev->dev, "wrong device subtype in %s\n", function);
 		return -EINVAL;
@@ -1162,6 +1163,7 @@ int coresight_enable(struct coresight_device *csdev)
 		per_cpu(tracer_path, cpu) = path;
 		break;
 	case CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE:
+	case CORESIGHT_DEV_SUBTYPE_SOURCE_TPDM:
 	case CORESIGHT_DEV_SUBTYPE_SOURCE_OTHERS:
 		/*
 		 * Use the hash of source's device name as ID
@@ -1212,6 +1214,7 @@ void coresight_disable(struct coresight_device *csdev)
 		per_cpu(tracer_path, cpu) = NULL;
 		break;
 	case CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE:
+	case CORESIGHT_DEV_SUBTYPE_SOURCE_TPDM:
 	case CORESIGHT_DEV_SUBTYPE_SOURCE_OTHERS:
 		hash = hashlen_hash(hashlen_string(NULL, dev_name(&csdev->dev)));
 		/* Find the path by the hash. */
diff --git a/drivers/hwtracing/coresight/coresight-dummy.c b/drivers/hwtracing/coresight/coresight-dummy.c
index e4deafae7bc2..ac70c0b491be 100644
--- a/drivers/hwtracing/coresight/coresight-dummy.c
+++ b/drivers/hwtracing/coresight/coresight-dummy.c
@@ -122,14 +122,13 @@ static int dummy_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int dummy_remove(struct platform_device *pdev)
+static void dummy_remove(struct platform_device *pdev)
 {
 	struct dummy_drvdata *drvdata = platform_get_drvdata(pdev);
 	struct device *dev = &pdev->dev;
 
 	pm_runtime_disable(dev);
 	coresight_unregister(drvdata->csdev);
-	return 0;
 }
 
 static const struct of_device_id dummy_match[] = {
@@ -140,7 +139,7 @@ static const struct of_device_id dummy_match[] = {
 
 static struct platform_driver dummy_driver = {
 	.probe	= dummy_probe,
-	.remove	= dummy_remove,
+	.remove_new = dummy_remove,
 	.driver	= {
 		.name   = "coresight-dummy",
 		.of_match_table = dummy_match,
diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c
index 5ca6278baff4..a52cfcce25d6 100644
--- a/drivers/hwtracing/coresight/coresight-etm-perf.c
+++ b/drivers/hwtracing/coresight/coresight-etm-perf.c
@@ -68,6 +68,7 @@ PMU_FORMAT_ATTR(preset,		"config:0-3");
 PMU_FORMAT_ATTR(sinkid,		"config2:0-31");
 /* config ID - set if a system configuration is selected */
 PMU_FORMAT_ATTR(configid,	"config2:32-63");
+PMU_FORMAT_ATTR(cc_threshold,	"config3:0-11");
 
 
 /*
@@ -101,6 +102,7 @@ static struct attribute *etm_config_formats_attr[] = {
 	&format_attr_preset.attr,
 	&format_attr_configid.attr,
 	&format_attr_branch_broadcast.attr,
+	&format_attr_cc_threshold.attr,
 	NULL,
 };
 
@@ -493,7 +495,7 @@ static void etm_event_start(struct perf_event *event, int flags)
 		goto fail_end_stop;
 
 	/* Finally enable the tracer */
-	if (coresight_enable_source(csdev, CS_MODE_PERF, event))
+	if (source_ops(csdev)->enable(csdev, event, CS_MODE_PERF))
 		goto fail_disable_path;
 
 	/*
@@ -587,7 +589,7 @@ static void etm_event_stop(struct perf_event *event, int mode)
 		return;
 
 	/* stop tracer */
-	coresight_disable_source(csdev, event);
+	source_ops(csdev)->disable(csdev, event);
 
 	/* tell the core */
 	event->hw.state = PERF_HES_STOPPED;
diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c
index 77b0271ce6eb..ce1995a2827f 100644
--- a/drivers/hwtracing/coresight/coresight-etm4x-core.c
+++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c
@@ -644,7 +644,7 @@ static int etm4_parse_event_config(struct coresight_device *csdev,
 	struct etmv4_config *config = &drvdata->config;
 	struct perf_event_attr *attr = &event->attr;
 	unsigned long cfg_hash;
-	int preset;
+	int preset, cc_threshold;
 
 	/* Clear configuration from previous run */
 	memset(config, 0, sizeof(struct etmv4_config));
@@ -667,7 +667,12 @@ static int etm4_parse_event_config(struct coresight_device *csdev,
 	if (attr->config & BIT(ETM_OPT_CYCACC)) {
 		config->cfg |= TRCCONFIGR_CCI;
 		/* TRM: Must program this for cycacc to work */
-		config->ccctlr = ETM_CYC_THRESHOLD_DEFAULT;
+		cc_threshold = attr->config3 & ETM_CYC_THRESHOLD_MASK;
+		if (!cc_threshold)
+			cc_threshold = ETM_CYC_THRESHOLD_DEFAULT;
+		if (cc_threshold < drvdata->ccitmin)
+			cc_threshold = drvdata->ccitmin;
+		config->ccctlr = cc_threshold;
 	}
 	if (attr->config & BIT(ETM_OPT_TS)) {
 		/*
@@ -1150,6 +1155,41 @@ static void cpu_detect_trace_filtering(struct etmv4_drvdata *drvdata)
 	drvdata->trfcr = trfcr;
 }
 
+/*
+ * The following errata on applicable cpu ranges, affect the CCITMIN filed
+ * in TCRIDR3 register. Software read for the field returns 0x100 limiting
+ * the cycle threshold granularity, whereas the right value should have
+ * been 0x4, which is well supported in the hardware.
+ */
+static struct midr_range etm_wrong_ccitmin_cpus[] = {
+	/* Erratum #1490853 - Cortex-A76 */
+	MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 4, 0),
+	/* Erratum #1490853 - Neoverse-N1 */
+	MIDR_RANGE(MIDR_NEOVERSE_N1, 0, 0, 4, 0),
+	/* Erratum #1491015 - Cortex-A77 */
+	MIDR_RANGE(MIDR_CORTEX_A77, 0, 0, 1, 0),
+	/* Erratum #1502854 - Cortex-X1 */
+	MIDR_REV(MIDR_CORTEX_X1, 0, 0),
+	/* Erratum #1619801 - Neoverse-V1 */
+	MIDR_REV(MIDR_NEOVERSE_V1, 0, 0),
+	{},
+};
+
+static void etm4_fixup_wrong_ccitmin(struct etmv4_drvdata *drvdata)
+{
+	/*
+	 * Erratum affected cpus will read 256 as the minimum
+	 * instruction trace cycle counting threshold whereas
+	 * the correct value should be 4 instead. Override the
+	 * recorded value for 'drvdata->ccitmin' to workaround
+	 * this problem.
+	 */
+	if (is_midr_in_range_list(read_cpuid_id(), etm_wrong_ccitmin_cpus)) {
+		if (drvdata->ccitmin == 256)
+			drvdata->ccitmin = 4;
+	}
+}
+
 static void etm4_init_arch_data(void *info)
 {
 	u32 etmidr0;
@@ -1214,6 +1254,8 @@ static void etm4_init_arch_data(void *info)
 	etmidr3 = etm4x_relaxed_read32(csa, TRCIDR3);
 	/* CCITMIN, bits[11:0] minimum threshold value that can be programmed */
 	drvdata->ccitmin = FIELD_GET(TRCIDR3_CCITMIN_MASK, etmidr3);
+	etm4_fixup_wrong_ccitmin(drvdata);
+
 	/* EXLEVEL_S, bits[19:16] Secure state instruction tracing */
 	drvdata->s_ex_level = FIELD_GET(TRCIDR3_EXLEVEL_S_MASK, etmidr3);
 	drvdata->config.s_ex_level = drvdata->s_ex_level;
@@ -2224,7 +2266,7 @@ static void clear_etmdrvdata(void *info)
 	per_cpu(delayed_probe, cpu) = NULL;
 }
 
-static void __exit etm4_remove_dev(struct etmv4_drvdata *drvdata)
+static void etm4_remove_dev(struct etmv4_drvdata *drvdata)
 {
 	bool had_delayed_probe;
 	/*
@@ -2253,7 +2295,7 @@ static void __exit etm4_remove_dev(struct etmv4_drvdata *drvdata)
 	}
 }
 
-static void __exit etm4_remove_amba(struct amba_device *adev)
+static void etm4_remove_amba(struct amba_device *adev)
 {
 	struct etmv4_drvdata *drvdata = dev_get_drvdata(&adev->dev);
 
@@ -2261,7 +2303,7 @@ static void __exit etm4_remove_amba(struct amba_device *adev)
 		etm4_remove_dev(drvdata);
 }
 
-static int __exit etm4_remove_platform_dev(struct platform_device *pdev)
+static void etm4_remove_platform_dev(struct platform_device *pdev)
 {
 	struct etmv4_drvdata *drvdata = dev_get_drvdata(&pdev->dev);
 
@@ -2271,8 +2313,6 @@ static int __exit etm4_remove_platform_dev(struct platform_device *pdev)
 
 	if (drvdata && !IS_ERR_OR_NULL(drvdata->pclk))
 		clk_put(drvdata->pclk);
-
-	return 0;
 }
 
 static const struct amba_id etm4_ids[] = {
@@ -2358,7 +2398,7 @@ MODULE_DEVICE_TABLE(acpi, etm4x_acpi_ids);
 
 static struct platform_driver etm4_platform_driver = {
 	.probe		= etm4_probe_platform_dev,
-	.remove		= etm4_remove_platform_dev,
+	.remove_new	= etm4_remove_platform_dev,
 	.driver			= {
 		.name			= "coresight-etm4x",
 		.of_match_table		= etm4_sysreg_match,
diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h
index 20e2e4cb7614..da17b6c49b0f 100644
--- a/drivers/hwtracing/coresight/coresight-etm4x.h
+++ b/drivers/hwtracing/coresight/coresight-etm4x.h
@@ -1036,7 +1036,7 @@ struct etmv4_drvdata {
 	u8				ctxid_size;
 	u8				vmid_size;
 	u8				ccsize;
-	u8				ccitmin;
+	u16				ccitmin;
 	u8				s_ex_level;
 	u8				ns_ex_level;
 	u8				q_support;
diff --git a/drivers/hwtracing/coresight/coresight-funnel.c b/drivers/hwtracing/coresight/coresight-funnel.c
index b8e150e45b27..a5b1fc787766 100644
--- a/drivers/hwtracing/coresight/coresight-funnel.c
+++ b/drivers/hwtracing/coresight/coresight-funnel.c
@@ -335,11 +335,10 @@ static int static_funnel_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int static_funnel_remove(struct platform_device *pdev)
+static void static_funnel_remove(struct platform_device *pdev)
 {
 	funnel_remove(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
-	return 0;
 }
 
 static const struct of_device_id static_funnel_match[] = {
@@ -360,7 +359,7 @@ MODULE_DEVICE_TABLE(acpi, static_funnel_ids);
 
 static struct platform_driver static_funnel_driver = {
 	.probe          = static_funnel_probe,
-	.remove          = static_funnel_remove,
+	.remove_new      = static_funnel_remove,
 	.driver         = {
 		.name   = "coresight-static-funnel",
 		/* THIS_MODULE is taken care of by platform_driver_register() */
diff --git a/drivers/hwtracing/coresight/coresight-replicator.c b/drivers/hwtracing/coresight/coresight-replicator.c
index b6be73034996..91d93060dda5 100644
--- a/drivers/hwtracing/coresight/coresight-replicator.c
+++ b/drivers/hwtracing/coresight/coresight-replicator.c
@@ -320,11 +320,10 @@ static int static_replicator_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int static_replicator_remove(struct platform_device *pdev)
+static void static_replicator_remove(struct platform_device *pdev)
 {
 	replicator_remove(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
-	return 0;
 }
 
 #ifdef CONFIG_PM
@@ -373,7 +372,7 @@ MODULE_DEVICE_TABLE(acpi, static_replicator_acpi_ids);
 
 static struct platform_driver static_replicator_driver = {
 	.probe          = static_replicator_probe,
-	.remove         = static_replicator_remove,
+	.remove_new     = static_replicator_remove,
 	.driver         = {
 		.name   = "coresight-static-replicator",
 		/* THIS_MODULE is taken care of by platform_driver_register() */
diff --git a/drivers/hwtracing/coresight/coresight-tmc-core.c b/drivers/hwtracing/coresight/coresight-tmc-core.c
index c106d142e632..7ec5365e2b64 100644
--- a/drivers/hwtracing/coresight/coresight-tmc-core.c
+++ b/drivers/hwtracing/coresight/coresight-tmc-core.c
@@ -10,6 +10,7 @@
 #include <linux/device.h>
 #include <linux/idr.h>
 #include <linux/io.h>
+#include <linux/iommu.h>
 #include <linux/err.h>
 #include <linux/fs.h>
 #include <linux/miscdevice.h>
@@ -344,7 +345,14 @@ static const struct attribute_group coresight_tmc_mgmt_group = {
 	.name = "mgmt",
 };
 
-static const struct attribute_group *coresight_tmc_groups[] = {
+static const struct attribute_group *coresight_etf_groups[] = {
+	&coresight_tmc_group,
+	&coresight_tmc_mgmt_group,
+	NULL,
+};
+
+static const struct attribute_group *coresight_etr_groups[] = {
+	&coresight_etr_group,
 	&coresight_tmc_group,
 	&coresight_tmc_mgmt_group,
 	NULL,
@@ -465,6 +473,7 @@ static int tmc_probe(struct amba_device *adev, const struct amba_id *id)
 	drvdata->memwidth = tmc_get_memwidth(devid);
 	/* This device is not associated with a session */
 	drvdata->pid = -1;
+	drvdata->etr_mode = ETR_MODE_AUTO;
 
 	if (drvdata->config_type == TMC_CONFIG_TYPE_ETR) {
 		drvdata->size = tmc_etr_get_default_buffer_size(dev);
@@ -474,16 +483,17 @@ static int tmc_probe(struct amba_device *adev, const struct amba_id *id)
 	}
 
 	desc.dev = dev;
-	desc.groups = coresight_tmc_groups;
 
 	switch (drvdata->config_type) {
 	case TMC_CONFIG_TYPE_ETB:
+		desc.groups = coresight_etf_groups;
 		desc.type = CORESIGHT_DEV_TYPE_SINK;
 		desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_BUFFER;
 		desc.ops = &tmc_etb_cs_ops;
 		dev_list = &etb_devs;
 		break;
 	case TMC_CONFIG_TYPE_ETR:
+		desc.groups = coresight_etr_groups;
 		desc.type = CORESIGHT_DEV_TYPE_SINK;
 		desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_SYSMEM;
 		desc.ops = &tmc_etr_cs_ops;
@@ -496,6 +506,7 @@ static int tmc_probe(struct amba_device *adev, const struct amba_id *id)
 		dev_list = &etr_devs;
 		break;
 	case TMC_CONFIG_TYPE_ETF:
+		desc.groups = coresight_etf_groups;
 		desc.type = CORESIGHT_DEV_TYPE_LINKSINK;
 		desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_BUFFER;
 		desc.subtype.link_subtype = CORESIGHT_DEV_SUBTYPE_LINK_FIFO;
diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c
index 8311e1028ddb..af02ba5d5f15 100644
--- a/drivers/hwtracing/coresight/coresight-tmc-etr.c
+++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c
@@ -26,6 +26,12 @@ struct etr_flat_buf {
 	size_t		size;
 };
 
+struct etr_buf_hw {
+	bool	has_iommu;
+	bool	has_etr_sg;
+	bool	has_catu;
+};
+
 /*
  * etr_perf_buffer - Perf buffer used for ETR
  * @drvdata		- The ETR drvdaga this buffer has been allocated for.
@@ -830,6 +836,22 @@ static inline int tmc_etr_mode_alloc_buf(int mode,
 	}
 }
 
+static void get_etr_buf_hw(struct device *dev, struct etr_buf_hw *buf_hw)
+{
+	struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
+
+	buf_hw->has_iommu = iommu_get_domain_for_dev(dev->parent);
+	buf_hw->has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG);
+	buf_hw->has_catu = !!tmc_etr_get_catu_device(drvdata);
+}
+
+static bool etr_can_use_flat_mode(struct etr_buf_hw *buf_hw, ssize_t etr_buf_size)
+{
+	bool has_sg = buf_hw->has_catu || buf_hw->has_etr_sg;
+
+	return !has_sg || buf_hw->has_iommu || etr_buf_size < SZ_1M;
+}
+
 /*
  * tmc_alloc_etr_buf: Allocate a buffer use by ETR.
  * @drvdata	: ETR device details.
@@ -843,23 +865,22 @@ static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,
 					 int node, void **pages)
 {
 	int rc = -ENOMEM;
-	bool has_etr_sg, has_iommu;
-	bool has_sg, has_catu;
 	struct etr_buf *etr_buf;
+	struct etr_buf_hw buf_hw;
 	struct device *dev = &drvdata->csdev->dev;
 
-	has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG);
-	has_iommu = iommu_get_domain_for_dev(dev->parent);
-	has_catu = !!tmc_etr_get_catu_device(drvdata);
-
-	has_sg = has_catu || has_etr_sg;
-
+	get_etr_buf_hw(dev, &buf_hw);
 	etr_buf = kzalloc(sizeof(*etr_buf), GFP_KERNEL);
 	if (!etr_buf)
 		return ERR_PTR(-ENOMEM);
 
 	etr_buf->size = size;
 
+	/* If there is user directive for buffer mode, try that first */
+	if (drvdata->etr_mode != ETR_MODE_AUTO)
+		rc = tmc_etr_mode_alloc_buf(drvdata->etr_mode, drvdata,
+					    etr_buf, node, pages);
+
 	/*
 	 * If we have to use an existing list of pages, we cannot reliably
 	 * use a contiguous DMA memory (even if we have an IOMMU). Otherwise,
@@ -872,14 +893,13 @@ static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,
 	 * Fallback to available mechanisms.
 	 *
 	 */
-	if (!pages &&
-	    (!has_sg || has_iommu || size < SZ_1M))
+	if (rc && !pages && etr_can_use_flat_mode(&buf_hw, size))
 		rc = tmc_etr_mode_alloc_buf(ETR_MODE_FLAT, drvdata,
 					    etr_buf, node, pages);
-	if (rc && has_etr_sg)
+	if (rc && buf_hw.has_etr_sg)
 		rc = tmc_etr_mode_alloc_buf(ETR_MODE_ETR_SG, drvdata,
 					    etr_buf, node, pages);
-	if (rc && has_catu)
+	if (rc && buf_hw.has_catu)
 		rc = tmc_etr_mode_alloc_buf(ETR_MODE_CATU, drvdata,
 					    etr_buf, node, pages);
 	if (rc) {
@@ -1804,3 +1824,70 @@ int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)
 
 	return 0;
 }
+
+static const char *const buf_modes_str[] = {
+	[ETR_MODE_FLAT]		= "flat",
+	[ETR_MODE_ETR_SG]	= "tmc-sg",
+	[ETR_MODE_CATU]		= "catu",
+	[ETR_MODE_AUTO]		= "auto",
+};
+
+static ssize_t buf_modes_available_show(struct device *dev,
+					    struct device_attribute *attr, char *buf)
+{
+	struct etr_buf_hw buf_hw;
+	ssize_t size = 0;
+
+	get_etr_buf_hw(dev, &buf_hw);
+	size += sysfs_emit(buf, "%s ", buf_modes_str[ETR_MODE_AUTO]);
+	size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_FLAT]);
+	if (buf_hw.has_etr_sg)
+		size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_ETR_SG]);
+
+	if (buf_hw.has_catu)
+		size += sysfs_emit_at(buf, size, "%s ", buf_modes_str[ETR_MODE_CATU]);
+
+	size += sysfs_emit_at(buf, size, "\n");
+	return size;
+}
+static DEVICE_ATTR_RO(buf_modes_available);
+
+static ssize_t buf_mode_preferred_show(struct device *dev,
+					 struct device_attribute *attr, char *buf)
+{
+	struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
+
+	return sysfs_emit(buf, "%s\n", buf_modes_str[drvdata->etr_mode]);
+}
+
+static ssize_t buf_mode_preferred_store(struct device *dev,
+					  struct device_attribute *attr,
+					  const char *buf, size_t size)
+{
+	struct tmc_drvdata *drvdata = dev_get_drvdata(dev->parent);
+	struct etr_buf_hw buf_hw;
+
+	get_etr_buf_hw(dev, &buf_hw);
+	if (sysfs_streq(buf, buf_modes_str[ETR_MODE_FLAT]))
+		drvdata->etr_mode = ETR_MODE_FLAT;
+	else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_ETR_SG]) && buf_hw.has_etr_sg)
+		drvdata->etr_mode = ETR_MODE_ETR_SG;
+	else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_CATU]) && buf_hw.has_catu)
+		drvdata->etr_mode = ETR_MODE_CATU;
+	else if (sysfs_streq(buf, buf_modes_str[ETR_MODE_AUTO]))
+		drvdata->etr_mode = ETR_MODE_AUTO;
+	else
+		return -EINVAL;
+	return size;
+}
+static DEVICE_ATTR_RW(buf_mode_preferred);
+
+static struct attribute *coresight_etr_attrs[] = {
+	&dev_attr_buf_modes_available.attr,
+	&dev_attr_buf_mode_preferred.attr,
+	NULL,
+};
+
+const struct attribute_group coresight_etr_group = {
+	.attrs = coresight_etr_attrs,
+};
diff --git a/drivers/hwtracing/coresight/coresight-tmc.h b/drivers/hwtracing/coresight/coresight-tmc.h
index 0ee48c5ba764..8dcb426ac3e7 100644
--- a/drivers/hwtracing/coresight/coresight-tmc.h
+++ b/drivers/hwtracing/coresight/coresight-tmc.h
@@ -135,6 +135,7 @@ enum etr_mode {
 	ETR_MODE_FLAT,		/* Uses contiguous flat buffer */
 	ETR_MODE_ETR_SG,	/* Uses in-built TMC ETR SG mechanism */
 	ETR_MODE_CATU,		/* Use SG mechanism in CATU */
+	ETR_MODE_AUTO,		/* Use the default mechanism */
 };
 
 struct etr_buf_operations;
@@ -207,6 +208,7 @@ struct tmc_drvdata {
 	enum tmc_mem_intf_width	memwidth;
 	u32			trigger_cntr;
 	u32			etr_caps;
+	enum etr_mode		etr_mode;
 	struct idr		idr;
 	struct mutex		idr_mutex;
 	struct etr_buf		*sysfs_buf;
@@ -334,5 +336,6 @@ void tmc_etr_set_catu_ops(const struct etr_buf_operations *catu);
 void tmc_etr_remove_catu_ops(void);
 struct etr_buf *tmc_etr_get_buffer(struct coresight_device *csdev,
 				   enum cs_mode mode, void *data);
+extern const struct attribute_group coresight_etr_group;
 
 #endif
diff --git a/drivers/hwtracing/coresight/coresight-tpda.c b/drivers/hwtracing/coresight/coresight-tpda.c
index 8d2b9d29237d..5f82737c37bb 100644
--- a/drivers/hwtracing/coresight/coresight-tpda.c
+++ b/drivers/hwtracing/coresight/coresight-tpda.c
@@ -21,6 +21,80 @@
 
 DEFINE_CORESIGHT_DEVLIST(tpda_devs, "tpda");
 
+static bool coresight_device_is_tpdm(struct coresight_device *csdev)
+{
+	return (csdev->type == CORESIGHT_DEV_TYPE_SOURCE) &&
+	       (csdev->subtype.source_subtype ==
+			CORESIGHT_DEV_SUBTYPE_SOURCE_TPDM);
+}
+
+/*
+ * Read the DSB element size from the TPDM device
+ * Returns
+ *    The dsb element size read from the devicetree if available.
+ *    0 - Otherwise, with a warning once.
+ */
+static int tpdm_read_dsb_element_size(struct coresight_device *csdev)
+{
+	int rc = 0;
+	u8 size = 0;
+
+	rc = fwnode_property_read_u8(dev_fwnode(csdev->dev.parent),
+			"qcom,dsb-element-size", &size);
+	if (rc)
+		dev_warn_once(&csdev->dev,
+			"Failed to read TPDM DSB Element size: %d\n", rc);
+
+	return size;
+}
+
+/*
+ * Search and read element data size from the TPDM node in
+ * the devicetree. Each input port of TPDA is connected to
+ * a TPDM. Different TPDM supports different types of dataset,
+ * and some may support more than one type of dataset.
+ * Parameter "inport" is used to pass in the input port number
+ * of TPDA, and it is set to -1 in the recursize call.
+ */
+static int tpda_get_element_size(struct coresight_device *csdev,
+				 int inport)
+{
+	int dsb_size = -ENOENT;
+	int i, size;
+	struct coresight_device *in;
+
+	for (i = 0; i < csdev->pdata->nr_inconns; i++) {
+		in = csdev->pdata->in_conns[i]->src_dev;
+		if (!in)
+			continue;
+
+		/* Ignore the paths that do not match port */
+		if (inport > 0 &&
+		    csdev->pdata->in_conns[i]->dest_port != inport)
+			continue;
+
+		if (coresight_device_is_tpdm(in)) {
+			size = tpdm_read_dsb_element_size(in);
+		} else {
+			/* Recurse down the path */
+			size = tpda_get_element_size(in, -1);
+		}
+
+		if (size < 0)
+			return size;
+
+		if (dsb_size < 0) {
+			/* Found a size, save it. */
+			dsb_size = size;
+		} else {
+			/* Found duplicate TPDMs */
+			return -EEXIST;
+		}
+	}
+
+	return dsb_size;
+}
+
 /* Settings pre enabling port control register */
 static void tpda_enable_pre_port(struct tpda_drvdata *drvdata)
 {
@@ -32,26 +106,55 @@ static void tpda_enable_pre_port(struct tpda_drvdata *drvdata)
 	writel_relaxed(val, drvdata->base + TPDA_CR);
 }
 
-static void tpda_enable_port(struct tpda_drvdata *drvdata, int port)
+static int tpda_enable_port(struct tpda_drvdata *drvdata, int port)
 {
 	u32 val;
+	int size;
 
 	val = readl_relaxed(drvdata->base + TPDA_Pn_CR(port));
+	/*
+	 * Configure aggregator port n DSB data set element size
+	 * Set the bit to 0 if the size is 32
+	 * Set the bit to 1 if the size is 64
+	 */
+	size = tpda_get_element_size(drvdata->csdev, port);
+	switch (size) {
+	case 32:
+		val &= ~TPDA_Pn_CR_DSBSIZE;
+		break;
+	case 64:
+		val |= TPDA_Pn_CR_DSBSIZE;
+		break;
+	case 0:
+		return -EEXIST;
+	case -EEXIST:
+		dev_warn_once(&drvdata->csdev->dev,
+			"Detected multiple TPDMs on port %d", -EEXIST);
+		return -EEXIST;
+	default:
+		return -EINVAL;
+	}
+
 	/* Enable the port */
 	val |= TPDA_Pn_CR_ENA;
 	writel_relaxed(val, drvdata->base + TPDA_Pn_CR(port));
+
+	return 0;
 }
 
-static void __tpda_enable(struct tpda_drvdata *drvdata, int port)
+static int __tpda_enable(struct tpda_drvdata *drvdata, int port)
 {
+	int ret;
+
 	CS_UNLOCK(drvdata->base);
 
 	if (!drvdata->csdev->enable)
 		tpda_enable_pre_port(drvdata);
 
-	tpda_enable_port(drvdata, port);
-
+	ret = tpda_enable_port(drvdata, port);
 	CS_LOCK(drvdata->base);
+
+	return ret;
 }
 
 static int tpda_enable(struct coresight_device *csdev,
@@ -59,16 +162,19 @@ static int tpda_enable(struct coresight_device *csdev,
 		       struct coresight_connection *out)
 {
 	struct tpda_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
+	int ret = 0;
 
 	spin_lock(&drvdata->spinlock);
-	if (atomic_read(&in->dest_refcnt) == 0)
-		__tpda_enable(drvdata, in->dest_port);
+	if (atomic_read(&in->dest_refcnt) == 0) {
+		ret = __tpda_enable(drvdata, in->dest_port);
+		if (!ret) {
+			atomic_inc(&in->dest_refcnt);
+			dev_dbg(drvdata->dev, "TPDA inport %d enabled.\n", in->dest_port);
+		}
+	}
 
-	atomic_inc(&in->dest_refcnt);
 	spin_unlock(&drvdata->spinlock);
-
-	dev_dbg(drvdata->dev, "TPDA inport %d enabled.\n", in->dest_port);
-	return 0;
+	return ret;
 }
 
 static void __tpda_disable(struct tpda_drvdata *drvdata, int port)
diff --git a/drivers/hwtracing/coresight/coresight-tpda.h b/drivers/hwtracing/coresight/coresight-tpda.h
index 0399678df312..b3b38fd41b64 100644
--- a/drivers/hwtracing/coresight/coresight-tpda.h
+++ b/drivers/hwtracing/coresight/coresight-tpda.h
@@ -10,6 +10,8 @@
 #define TPDA_Pn_CR(n)		(0x004 + (n * 4))
 /* Aggregator port enable bit */
 #define TPDA_Pn_CR_ENA		BIT(0)
+/* Aggregator port DSB data set element size bit */
+#define TPDA_Pn_CR_DSBSIZE		BIT(8)
 
 #define TPDA_MAX_INPORTS	32
 
diff --git a/drivers/hwtracing/coresight/coresight-tpdm.c b/drivers/hwtracing/coresight/coresight-tpdm.c
index f4854af0431e..97654aa4b772 100644
--- a/drivers/hwtracing/coresight/coresight-tpdm.c
+++ b/drivers/hwtracing/coresight/coresight-tpdm.c
@@ -4,6 +4,7 @@
  */
 
 #include <linux/amba/bus.h>
+#include <linux/bitfield.h>
 #include <linux/bitmap.h>
 #include <linux/coresight.h>
 #include <linux/coresight-pmu.h>
@@ -20,23 +21,265 @@
 
 DEFINE_CORESIGHT_DEVLIST(tpdm_devs, "tpdm");
 
-static void tpdm_enable_dsb(struct tpdm_drvdata *drvdata)
+/* Read dataset array member with the index number */
+static ssize_t tpdm_simple_dataset_show(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct tpdm_drvdata *drvdata = dev_get_drvdata(dev->parent);
+	struct tpdm_dataset_attribute *tpdm_attr =
+		container_of(attr, struct tpdm_dataset_attribute, attr);
+
+	switch (tpdm_attr->mem) {
+	case DSB_EDGE_CTRL:
+		if (tpdm_attr->idx >= TPDM_DSB_MAX_EDCR)
+			return -EINVAL;
+		return sysfs_emit(buf, "0x%x\n",
+			drvdata->dsb->edge_ctrl[tpdm_attr->idx]);
+	case DSB_EDGE_CTRL_MASK:
+		if (tpdm_attr->idx >= TPDM_DSB_MAX_EDCMR)
+			return -EINVAL;
+		return sysfs_emit(buf, "0x%x\n",
+			drvdata->dsb->edge_ctrl_mask[tpdm_attr->idx]);
+	case DSB_TRIG_PATT:
+		if (tpdm_attr->idx >= TPDM_DSB_MAX_PATT)
+			return -EINVAL;
+		return sysfs_emit(buf, "0x%x\n",
+			drvdata->dsb->trig_patt[tpdm_attr->idx]);
+	case DSB_TRIG_PATT_MASK:
+		if (tpdm_attr->idx >= TPDM_DSB_MAX_PATT)
+			return -EINVAL;
+		return sysfs_emit(buf, "0x%x\n",
+			drvdata->dsb->trig_patt_mask[tpdm_attr->idx]);
+	case DSB_PATT:
+		if (tpdm_attr->idx >= TPDM_DSB_MAX_PATT)
+			return -EINVAL;
+		return sysfs_emit(buf, "0x%x\n",
+			drvdata->dsb->patt_val[tpdm_attr->idx]);
+	case DSB_PATT_MASK:
+		if (tpdm_attr->idx >= TPDM_DSB_MAX_PATT)
+			return -EINVAL;
+		return sysfs_emit(buf, "0x%x\n",
+			drvdata->dsb->patt_mask[tpdm_attr->idx]);
+	case DSB_MSR:
+		if (tpdm_attr->idx >= drvdata->dsb_msr_num)
+			return -EINVAL;
+		return sysfs_emit(buf, "0x%x\n",
+				drvdata->dsb->msr[tpdm_attr->idx]);
+	}
+	return -EINVAL;
+}
+
+/* Write dataset array member with the index number */
+static ssize_t tpdm_simple_dataset_store(struct device *dev,
+					 struct device_attribute *attr,
+					 const char *buf,
+					 size_t size)
+{
+	unsigned long val;
+	ssize_t ret = size;
+
+	struct tpdm_drvdata *drvdata = dev_get_drvdata(dev->parent);
+	struct tpdm_dataset_attribute *tpdm_attr =
+		container_of(attr, struct tpdm_dataset_attribute, attr);
+
+	if (kstrtoul(buf, 0, &val))
+		return -EINVAL;
+
+	spin_lock(&drvdata->spinlock);
+	switch (tpdm_attr->mem) {
+	case DSB_TRIG_PATT:
+		if (tpdm_attr->idx < TPDM_DSB_MAX_PATT)
+			drvdata->dsb->trig_patt[tpdm_attr->idx] = val;
+		else
+			ret = -EINVAL;
+		break;
+	case DSB_TRIG_PATT_MASK:
+		if (tpdm_attr->idx < TPDM_DSB_MAX_PATT)
+			drvdata->dsb->trig_patt_mask[tpdm_attr->idx] = val;
+		else
+			ret = -EINVAL;
+		break;
+	case DSB_PATT:
+		if (tpdm_attr->idx < TPDM_DSB_MAX_PATT)
+			drvdata->dsb->patt_val[tpdm_attr->idx] = val;
+		else
+			ret = -EINVAL;
+		break;
+	case DSB_PATT_MASK:
+		if (tpdm_attr->idx < TPDM_DSB_MAX_PATT)
+			drvdata->dsb->patt_mask[tpdm_attr->idx] = val;
+		else
+			ret = -EINVAL;
+		break;
+	case DSB_MSR:
+		if (tpdm_attr->idx < drvdata->dsb_msr_num)
+			drvdata->dsb->msr[tpdm_attr->idx] = val;
+		else
+			ret = -EINVAL;
+		break;
+	default:
+		ret = -EINVAL;
+	}
+	spin_unlock(&drvdata->spinlock);
+
+	return ret;
+}
+
+static bool tpdm_has_dsb_dataset(struct tpdm_drvdata *drvdata)
+{
+	return (drvdata->datasets & TPDM_PIDR0_DS_DSB);
+}
+
+static umode_t tpdm_dsb_is_visible(struct kobject *kobj,
+				   struct attribute *attr, int n)
+{
+	struct device *dev = kobj_to_dev(kobj);
+	struct tpdm_drvdata *drvdata = dev_get_drvdata(dev->parent);
+
+	if (drvdata && tpdm_has_dsb_dataset(drvdata))
+		return attr->mode;
+
+	return 0;
+}
+
+static umode_t tpdm_dsb_msr_is_visible(struct kobject *kobj,
+				       struct attribute *attr, int n)
+{
+	struct device *dev = kobj_to_dev(kobj);
+	struct tpdm_drvdata *drvdata = dev_get_drvdata(dev->parent);
+	struct device_attribute *dev_attr =
+		container_of(attr, struct device_attribute, attr);
+	struct tpdm_dataset_attribute *tpdm_attr =
+		container_of(dev_attr, struct tpdm_dataset_attribute, attr);
+
+	if (tpdm_attr->idx < drvdata->dsb_msr_num)
+		return attr->mode;
+
+	return 0;
+}
+
+static void tpdm_reset_datasets(struct tpdm_drvdata *drvdata)
+{
+	if (tpdm_has_dsb_dataset(drvdata)) {
+		memset(drvdata->dsb, 0, sizeof(struct dsb_dataset));
+
+		drvdata->dsb->trig_ts = true;
+		drvdata->dsb->trig_type = false;
+	}
+}
+
+static void set_dsb_mode(struct tpdm_drvdata *drvdata, u32 *val)
+{
+	u32 mode;
+
+	/* Set the test accurate mode */
+	mode = TPDM_DSB_MODE_TEST(drvdata->dsb->mode);
+	*val &= ~TPDM_DSB_CR_TEST_MODE;
+	*val |= FIELD_PREP(TPDM_DSB_CR_TEST_MODE, mode);
+
+	/* Set the byte lane for high-performance mode */
+	mode = TPDM_DSB_MODE_HPBYTESEL(drvdata->dsb->mode);
+	*val &= ~TPDM_DSB_CR_HPSEL;
+	*val |= FIELD_PREP(TPDM_DSB_CR_HPSEL, mode);
+
+	/* Set the performance mode */
+	if (drvdata->dsb->mode & TPDM_DSB_MODE_PERF)
+		*val |= TPDM_DSB_CR_MODE;
+	else
+		*val &= ~TPDM_DSB_CR_MODE;
+}
+
+static void set_dsb_tier(struct tpdm_drvdata *drvdata)
 {
 	u32 val;
 
-	/* Set the enable bit of DSB control register to 1 */
+	val = readl_relaxed(drvdata->base + TPDM_DSB_TIER);
+
+	/* Clear all relevant fields */
+	val &= ~(TPDM_DSB_TIER_PATT_TSENAB | TPDM_DSB_TIER_PATT_TYPE |
+		 TPDM_DSB_TIER_XTRIG_TSENAB);
+
+	/* Set pattern timestamp type and enablement */
+	if (drvdata->dsb->patt_ts) {
+		val |= TPDM_DSB_TIER_PATT_TSENAB;
+		if (drvdata->dsb->patt_type)
+			val |= TPDM_DSB_TIER_PATT_TYPE;
+		else
+			val &= ~TPDM_DSB_TIER_PATT_TYPE;
+	} else {
+		val &= ~TPDM_DSB_TIER_PATT_TSENAB;
+	}
+
+	/* Set trigger timestamp */
+	if (drvdata->dsb->trig_ts)
+		val |= TPDM_DSB_TIER_XTRIG_TSENAB;
+	else
+		val &= ~TPDM_DSB_TIER_XTRIG_TSENAB;
+
+	writel_relaxed(val, drvdata->base + TPDM_DSB_TIER);
+}
+
+static void set_dsb_msr(struct tpdm_drvdata *drvdata)
+{
+	int i;
+
+	for (i = 0; i < drvdata->dsb_msr_num; i++)
+		writel_relaxed(drvdata->dsb->msr[i],
+			   drvdata->base + TPDM_DSB_MSR(i));
+}
+
+static void tpdm_enable_dsb(struct tpdm_drvdata *drvdata)
+{
+	u32 val, i;
+
+	for (i = 0; i < TPDM_DSB_MAX_EDCR; i++)
+		writel_relaxed(drvdata->dsb->edge_ctrl[i],
+			   drvdata->base + TPDM_DSB_EDCR(i));
+	for (i = 0; i < TPDM_DSB_MAX_EDCMR; i++)
+		writel_relaxed(drvdata->dsb->edge_ctrl_mask[i],
+			   drvdata->base + TPDM_DSB_EDCMR(i));
+	for (i = 0; i < TPDM_DSB_MAX_PATT; i++) {
+		writel_relaxed(drvdata->dsb->patt_val[i],
+			   drvdata->base + TPDM_DSB_TPR(i));
+		writel_relaxed(drvdata->dsb->patt_mask[i],
+			   drvdata->base + TPDM_DSB_TPMR(i));
+		writel_relaxed(drvdata->dsb->trig_patt[i],
+			   drvdata->base + TPDM_DSB_XPR(i));
+		writel_relaxed(drvdata->dsb->trig_patt_mask[i],
+			   drvdata->base + TPDM_DSB_XPMR(i));
+	}
+
+	set_dsb_tier(drvdata);
+
+	set_dsb_msr(drvdata);
+
 	val = readl_relaxed(drvdata->base + TPDM_DSB_CR);
+	/* Set the mode of DSB dataset */
+	set_dsb_mode(drvdata, &val);
+	/* Set trigger type */
+	if (drvdata->dsb->trig_type)
+		val |= TPDM_DSB_CR_TRIG_TYPE;
+	else
+		val &= ~TPDM_DSB_CR_TRIG_TYPE;
+	/* Set the enable bit of DSB control register to 1 */
 	val |= TPDM_DSB_CR_ENA;
 	writel_relaxed(val, drvdata->base + TPDM_DSB_CR);
 }
 
-/* TPDM enable operations */
+/*
+ * TPDM enable operations
+ * The TPDM or Monitor serves as data collection component for various
+ * dataset types. It covers Basic Counts(BC), Tenure Counts(TC),
+ * Continuous Multi-Bit(CMB), Multi-lane CMB(MCMB) and Discrete Single
+ * Bit(DSB). This function will initialize the configuration according
+ * to the dataset type supported by the TPDM.
+ */
 static void __tpdm_enable(struct tpdm_drvdata *drvdata)
 {
 	CS_UNLOCK(drvdata->base);
 
-	/* Check if DSB datasets is present for TPDM. */
-	if (drvdata->datasets & TPDM_PIDR0_DS_DSB)
+	if (tpdm_has_dsb_dataset(drvdata))
 		tpdm_enable_dsb(drvdata);
 
 	CS_LOCK(drvdata->base);
@@ -76,8 +319,7 @@ static void __tpdm_disable(struct tpdm_drvdata *drvdata)
 {
 	CS_UNLOCK(drvdata->base);
 
-	/* Check if DSB datasets is present for TPDM. */
-	if (drvdata->datasets & TPDM_PIDR0_DS_DSB)
+	if (tpdm_has_dsb_dataset(drvdata))
 		tpdm_disable_dsb(drvdata);
 
 	CS_LOCK(drvdata->base);
@@ -110,16 +352,45 @@ static const struct coresight_ops tpdm_cs_ops = {
 	.source_ops	= &tpdm_source_ops,
 };
 
-static void tpdm_init_default_data(struct tpdm_drvdata *drvdata)
+static int tpdm_datasets_setup(struct tpdm_drvdata *drvdata)
 {
 	u32 pidr;
 
-	CS_UNLOCK(drvdata->base);
 	/*  Get the datasets present on the TPDM. */
 	pidr = readl_relaxed(drvdata->base + CORESIGHT_PERIPHIDR0);
 	drvdata->datasets |= pidr & GENMASK(TPDM_DATASETS - 1, 0);
-	CS_LOCK(drvdata->base);
+
+	if (tpdm_has_dsb_dataset(drvdata) && (!drvdata->dsb)) {
+		drvdata->dsb = devm_kzalloc(drvdata->dev,
+						sizeof(*drvdata->dsb), GFP_KERNEL);
+		if (!drvdata->dsb)
+			return -ENOMEM;
+	}
+	tpdm_reset_datasets(drvdata);
+
+	return 0;
+}
+
+static ssize_t reset_dataset_store(struct device *dev,
+				   struct device_attribute *attr,
+				   const char *buf,
+				   size_t size)
+{
+	int ret = 0;
+	unsigned long val;
+	struct tpdm_drvdata *drvdata = dev_get_drvdata(dev->parent);
+
+	ret = kstrtoul(buf, 0, &val);
+	if (ret || val != 1)
+		return -EINVAL;
+
+	spin_lock(&drvdata->spinlock);
+	tpdm_reset_datasets(drvdata);
+	spin_unlock(&drvdata->spinlock);
+
+	return size;
 }
+static DEVICE_ATTR_WO(reset_dataset);
 
 /*
  * value 1: 64 bits test data
@@ -161,6 +432,7 @@ static ssize_t integration_test_store(struct device *dev,
 static DEVICE_ATTR_WO(integration_test);
 
 static struct attribute *tpdm_attrs[] = {
+	&dev_attr_reset_dataset.attr,
 	&dev_attr_integration_test.attr,
 	NULL,
 };
@@ -169,8 +441,421 @@ static struct attribute_group tpdm_attr_grp = {
 	.attrs = tpdm_attrs,
 };
 
+static ssize_t dsb_mode_show(struct device *dev,
+			     struct device_attribute *attr,
+			     char *buf)
+{
+	struct tpdm_drvdata *drvdata = dev_get_drvdata(dev->parent);
+
+	return sysfs_emit(buf, "%x\n", drvdata->dsb->mode);
+}
+
+static ssize_t dsb_mode_store(struct device *dev,
+			      struct device_attribute *attr,
+			      const char *buf,
+			      size_t size)
+{
+	struct tpdm_drvdata *drvdata = dev_get_drvdata(dev->parent);
+	unsigned long val;
+
+	if ((kstrtoul(buf, 0, &val)) || (val < 0) ||
+			(val & ~TPDM_DSB_MODE_MASK))
+		return -EINVAL;
+
+	spin_lock(&drvdata->spinlock);
+	drvdata->dsb->mode = val & TPDM_DSB_MODE_MASK;
+	spin_unlock(&drvdata->spinlock);
+	return size;
+}
+static DEVICE_ATTR_RW(dsb_mode);
+
+static ssize_t ctrl_idx_show(struct device *dev,
+			     struct device_attribute *attr,
+			     char *buf)
+{
+	struct tpdm_drvdata *drvdata = dev_get_drvdata(dev->parent);
+
+	return sysfs_emit(buf, "%u\n",
+			(unsigned int)drvdata->dsb->edge_ctrl_idx);
+}
+
+/*
+ * The EDCR registers can include up to 16 32-bit registers, and each
+ * one can be configured to control up to 16 edge detections(2 bits
+ * control one edge detection). So a total 256 edge detections can be
+ * configured. This function provides a way to set the index number of
+ * the edge detection which needs to be configured.
+ */
+static ssize_t ctrl_idx_store(struct device *dev,
+			      struct device_attribute *attr,
+			      const char *buf,
+			      size_t size)
+{
+	struct tpdm_drvdata *drvdata = dev_get_drvdata(dev->parent);
+	unsigned long val;
+
+	if ((kstrtoul(buf, 0, &val)) || (val >= TPDM_DSB_MAX_LINES))
+		return -EINVAL;
+
+	spin_lock(&drvdata->spinlock);
+	drvdata->dsb->edge_ctrl_idx = val;
+	spin_unlock(&drvdata->spinlock);
+
+	return size;
+}
+static DEVICE_ATTR_RW(ctrl_idx);
+
+/*
+ * This function is used to control the edge detection according
+ * to the index number that has been set.
+ * "edge_ctrl" should be one of the following values.
+ * 0 - Rising edge detection
+ * 1 - Falling edge detection
+ * 2 - Rising and falling edge detection (toggle detection)
+ */
+static ssize_t ctrl_val_store(struct device *dev,
+			      struct device_attribute *attr,
+			      const char *buf,
+			      size_t size)
+{
+	struct tpdm_drvdata *drvdata = dev_get_drvdata(dev->parent);
+	unsigned long val, edge_ctrl;
+	int reg;
+
+	if ((kstrtoul(buf, 0, &edge_ctrl)) || (edge_ctrl > 0x2))
+		return -EINVAL;
+
+	spin_lock(&drvdata->spinlock);
+	/*
+	 * There are 2 bit per DSB Edge Control line.
+	 * Thus we have 16 lines in a 32bit word.
+	 */
+	reg = EDCR_TO_WORD_IDX(drvdata->dsb->edge_ctrl_idx);
+	val = drvdata->dsb->edge_ctrl[reg];
+	val &= ~EDCR_TO_WORD_MASK(drvdata->dsb->edge_ctrl_idx);
+	val |= EDCR_TO_WORD_VAL(edge_ctrl, drvdata->dsb->edge_ctrl_idx);
+	drvdata->dsb->edge_ctrl[reg] = val;
+	spin_unlock(&drvdata->spinlock);
+
+	return size;
+}
+static DEVICE_ATTR_WO(ctrl_val);
+
+static ssize_t ctrl_mask_store(struct device *dev,
+			       struct device_attribute *attr,
+			       const char *buf,
+			       size_t size)
+{
+	struct tpdm_drvdata *drvdata = dev_get_drvdata(dev->parent);
+	unsigned long val;
+	u32 set;
+	int reg;
+
+	if ((kstrtoul(buf, 0, &val)) || (val & ~1UL))
+		return -EINVAL;
+
+	spin_lock(&drvdata->spinlock);
+	/*
+	 * There is 1 bit per DSB Edge Control Mark line.
+	 * Thus we have 32 lines in a 32bit word.
+	 */
+	reg = EDCMR_TO_WORD_IDX(drvdata->dsb->edge_ctrl_idx);
+	set = drvdata->dsb->edge_ctrl_mask[reg];
+	if (val)
+		set |= BIT(EDCMR_TO_WORD_SHIFT(drvdata->dsb->edge_ctrl_idx));
+	else
+		set &= ~BIT(EDCMR_TO_WORD_SHIFT(drvdata->dsb->edge_ctrl_idx));
+	drvdata->dsb->edge_ctrl_mask[reg] = set;
+	spin_unlock(&drvdata->spinlock);
+
+	return size;
+}
+static DEVICE_ATTR_WO(ctrl_mask);
+
+static ssize_t enable_ts_show(struct device *dev,
+			      struct device_attribute *attr,
+			      char *buf)
+{
+	struct tpdm_drvdata *drvdata = dev_get_drvdata(dev->parent);
+
+	return sysfs_emit(buf, "%u\n",
+			 (unsigned int)drvdata->dsb->patt_ts);
+}
+
+/*
+ * value 1: Enable/Disable DSB pattern timestamp
+ */
+static ssize_t enable_ts_store(struct device *dev,
+			       struct device_attribute *attr,
+			       const char *buf,
+			       size_t size)
+{
+	struct tpdm_drvdata *drvdata = dev_get_drvdata(dev->parent);
+	unsigned long val;
+
+	if ((kstrtoul(buf, 0, &val)) || (val & ~1UL))
+		return -EINVAL;
+
+	spin_lock(&drvdata->spinlock);
+	drvdata->dsb->patt_ts = !!val;
+	spin_unlock(&drvdata->spinlock);
+	return size;
+}
+static DEVICE_ATTR_RW(enable_ts);
+
+static ssize_t set_type_show(struct device *dev,
+			     struct device_attribute *attr,
+			     char *buf)
+{
+	struct tpdm_drvdata *drvdata = dev_get_drvdata(dev->parent);
+
+	return sysfs_emit(buf, "%u\n",
+			 (unsigned int)drvdata->dsb->patt_type);
+}
+
+/*
+ * value 1: Set DSB pattern type
+ */
+static ssize_t set_type_store(struct device *dev,
+			      struct device_attribute *attr,
+			      const char *buf, size_t size)
+{
+	struct tpdm_drvdata *drvdata = dev_get_drvdata(dev->parent);
+	unsigned long val;
+
+	if ((kstrtoul(buf, 0, &val)) || (val & ~1UL))
+		return -EINVAL;
+
+	spin_lock(&drvdata->spinlock);
+	drvdata->dsb->patt_type = val;
+	spin_unlock(&drvdata->spinlock);
+	return size;
+}
+static DEVICE_ATTR_RW(set_type);
+
+static ssize_t dsb_trig_type_show(struct device *dev,
+				  struct device_attribute *attr, char *buf)
+{
+	struct tpdm_drvdata *drvdata = dev_get_drvdata(dev->parent);
+
+	return sysfs_emit(buf, "%u\n",
+			 (unsigned int)drvdata->dsb->trig_type);
+}
+
+/*
+ * Trigger type (boolean):
+ * false - Disable trigger type.
+ * true  - Enable trigger type.
+ */
+static ssize_t dsb_trig_type_store(struct device *dev,
+				   struct device_attribute *attr,
+				   const char *buf,
+				   size_t size)
+{
+	struct tpdm_drvdata *drvdata = dev_get_drvdata(dev->parent);
+	unsigned long val;
+
+	if ((kstrtoul(buf, 0, &val)) || (val & ~1UL))
+		return -EINVAL;
+
+	spin_lock(&drvdata->spinlock);
+	if (val)
+		drvdata->dsb->trig_type = true;
+	else
+		drvdata->dsb->trig_type = false;
+	spin_unlock(&drvdata->spinlock);
+	return size;
+}
+static DEVICE_ATTR_RW(dsb_trig_type);
+
+static ssize_t dsb_trig_ts_show(struct device *dev,
+				struct device_attribute *attr,
+				char *buf)
+{
+	struct tpdm_drvdata *drvdata = dev_get_drvdata(dev->parent);
+
+	return sysfs_emit(buf, "%u\n",
+			 (unsigned int)drvdata->dsb->trig_ts);
+}
+
+/*
+ * Trigger timestamp (boolean):
+ * false - Disable trigger timestamp.
+ * true  - Enable trigger timestamp.
+ */
+static ssize_t dsb_trig_ts_store(struct device *dev,
+				 struct device_attribute *attr,
+				 const char *buf,
+				 size_t size)
+{
+	struct tpdm_drvdata *drvdata = dev_get_drvdata(dev->parent);
+	unsigned long val;
+
+	if ((kstrtoul(buf, 0, &val)) || (val & ~1UL))
+		return -EINVAL;
+
+	spin_lock(&drvdata->spinlock);
+	if (val)
+		drvdata->dsb->trig_ts = true;
+	else
+		drvdata->dsb->trig_ts = false;
+	spin_unlock(&drvdata->spinlock);
+	return size;
+}
+static DEVICE_ATTR_RW(dsb_trig_ts);
+
+static struct attribute *tpdm_dsb_edge_attrs[] = {
+	&dev_attr_ctrl_idx.attr,
+	&dev_attr_ctrl_val.attr,
+	&dev_attr_ctrl_mask.attr,
+	DSB_EDGE_CTRL_ATTR(0),
+	DSB_EDGE_CTRL_ATTR(1),
+	DSB_EDGE_CTRL_ATTR(2),
+	DSB_EDGE_CTRL_ATTR(3),
+	DSB_EDGE_CTRL_ATTR(4),
+	DSB_EDGE_CTRL_ATTR(5),
+	DSB_EDGE_CTRL_ATTR(6),
+	DSB_EDGE_CTRL_ATTR(7),
+	DSB_EDGE_CTRL_ATTR(8),
+	DSB_EDGE_CTRL_ATTR(9),
+	DSB_EDGE_CTRL_ATTR(10),
+	DSB_EDGE_CTRL_ATTR(11),
+	DSB_EDGE_CTRL_ATTR(12),
+	DSB_EDGE_CTRL_ATTR(13),
+	DSB_EDGE_CTRL_ATTR(14),
+	DSB_EDGE_CTRL_ATTR(15),
+	DSB_EDGE_CTRL_MASK_ATTR(0),
+	DSB_EDGE_CTRL_MASK_ATTR(1),
+	DSB_EDGE_CTRL_MASK_ATTR(2),
+	DSB_EDGE_CTRL_MASK_ATTR(3),
+	DSB_EDGE_CTRL_MASK_ATTR(4),
+	DSB_EDGE_CTRL_MASK_ATTR(5),
+	DSB_EDGE_CTRL_MASK_ATTR(6),
+	DSB_EDGE_CTRL_MASK_ATTR(7),
+	NULL,
+};
+
+static struct attribute *tpdm_dsb_trig_patt_attrs[] = {
+	DSB_TRIG_PATT_ATTR(0),
+	DSB_TRIG_PATT_ATTR(1),
+	DSB_TRIG_PATT_ATTR(2),
+	DSB_TRIG_PATT_ATTR(3),
+	DSB_TRIG_PATT_ATTR(4),
+	DSB_TRIG_PATT_ATTR(5),
+	DSB_TRIG_PATT_ATTR(6),
+	DSB_TRIG_PATT_ATTR(7),
+	DSB_TRIG_PATT_MASK_ATTR(0),
+	DSB_TRIG_PATT_MASK_ATTR(1),
+	DSB_TRIG_PATT_MASK_ATTR(2),
+	DSB_TRIG_PATT_MASK_ATTR(3),
+	DSB_TRIG_PATT_MASK_ATTR(4),
+	DSB_TRIG_PATT_MASK_ATTR(5),
+	DSB_TRIG_PATT_MASK_ATTR(6),
+	DSB_TRIG_PATT_MASK_ATTR(7),
+	NULL,
+};
+
+static struct attribute *tpdm_dsb_patt_attrs[] = {
+	DSB_PATT_ATTR(0),
+	DSB_PATT_ATTR(1),
+	DSB_PATT_ATTR(2),
+	DSB_PATT_ATTR(3),
+	DSB_PATT_ATTR(4),
+	DSB_PATT_ATTR(5),
+	DSB_PATT_ATTR(6),
+	DSB_PATT_ATTR(7),
+	DSB_PATT_MASK_ATTR(0),
+	DSB_PATT_MASK_ATTR(1),
+	DSB_PATT_MASK_ATTR(2),
+	DSB_PATT_MASK_ATTR(3),
+	DSB_PATT_MASK_ATTR(4),
+	DSB_PATT_MASK_ATTR(5),
+	DSB_PATT_MASK_ATTR(6),
+	DSB_PATT_MASK_ATTR(7),
+	&dev_attr_enable_ts.attr,
+	&dev_attr_set_type.attr,
+	NULL,
+};
+
+static struct attribute *tpdm_dsb_msr_attrs[] = {
+	DSB_MSR_ATTR(0),
+	DSB_MSR_ATTR(1),
+	DSB_MSR_ATTR(2),
+	DSB_MSR_ATTR(3),
+	DSB_MSR_ATTR(4),
+	DSB_MSR_ATTR(5),
+	DSB_MSR_ATTR(6),
+	DSB_MSR_ATTR(7),
+	DSB_MSR_ATTR(8),
+	DSB_MSR_ATTR(9),
+	DSB_MSR_ATTR(10),
+	DSB_MSR_ATTR(11),
+	DSB_MSR_ATTR(12),
+	DSB_MSR_ATTR(13),
+	DSB_MSR_ATTR(14),
+	DSB_MSR_ATTR(15),
+	DSB_MSR_ATTR(16),
+	DSB_MSR_ATTR(17),
+	DSB_MSR_ATTR(18),
+	DSB_MSR_ATTR(19),
+	DSB_MSR_ATTR(20),
+	DSB_MSR_ATTR(21),
+	DSB_MSR_ATTR(22),
+	DSB_MSR_ATTR(23),
+	DSB_MSR_ATTR(24),
+	DSB_MSR_ATTR(25),
+	DSB_MSR_ATTR(26),
+	DSB_MSR_ATTR(27),
+	DSB_MSR_ATTR(28),
+	DSB_MSR_ATTR(29),
+	DSB_MSR_ATTR(30),
+	DSB_MSR_ATTR(31),
+	NULL,
+};
+
+static struct attribute *tpdm_dsb_attrs[] = {
+	&dev_attr_dsb_mode.attr,
+	&dev_attr_dsb_trig_ts.attr,
+	&dev_attr_dsb_trig_type.attr,
+	NULL,
+};
+
+static struct attribute_group tpdm_dsb_attr_grp = {
+	.attrs = tpdm_dsb_attrs,
+	.is_visible = tpdm_dsb_is_visible,
+};
+
+static struct attribute_group tpdm_dsb_edge_grp = {
+	.attrs = tpdm_dsb_edge_attrs,
+	.is_visible = tpdm_dsb_is_visible,
+	.name = "dsb_edge",
+};
+
+static struct attribute_group tpdm_dsb_trig_patt_grp = {
+	.attrs = tpdm_dsb_trig_patt_attrs,
+	.is_visible = tpdm_dsb_is_visible,
+	.name = "dsb_trig_patt",
+};
+
+static struct attribute_group tpdm_dsb_patt_grp = {
+	.attrs = tpdm_dsb_patt_attrs,
+	.is_visible = tpdm_dsb_is_visible,
+	.name = "dsb_patt",
+};
+
+static struct attribute_group tpdm_dsb_msr_grp = {
+	.attrs = tpdm_dsb_msr_attrs,
+	.is_visible = tpdm_dsb_msr_is_visible,
+	.name = "dsb_msr",
+};
+
 static const struct attribute_group *tpdm_attr_grps[] = {
 	&tpdm_attr_grp,
+	&tpdm_dsb_attr_grp,
+	&tpdm_dsb_edge_grp,
+	&tpdm_dsb_trig_patt_grp,
+	&tpdm_dsb_patt_grp,
+	&tpdm_dsb_msr_grp,
 	NULL,
 };
 
@@ -181,6 +866,7 @@ static int tpdm_probe(struct amba_device *adev, const struct amba_id *id)
 	struct coresight_platform_data *pdata;
 	struct tpdm_drvdata *drvdata;
 	struct coresight_desc desc = { 0 };
+	int ret;
 
 	pdata = coresight_get_platform_data(dev);
 	if (IS_ERR(pdata))
@@ -200,12 +886,20 @@ static int tpdm_probe(struct amba_device *adev, const struct amba_id *id)
 
 	drvdata->base = base;
 
+	ret = tpdm_datasets_setup(drvdata);
+	if (ret)
+		return ret;
+
+	if (drvdata && tpdm_has_dsb_dataset(drvdata))
+		of_property_read_u32(drvdata->dev->of_node,
+			   "qcom,dsb-msrs-num", &drvdata->dsb_msr_num);
+
 	/* Set up coresight component description */
 	desc.name = coresight_alloc_device_name(&tpdm_devs, dev);
 	if (!desc.name)
 		return -ENOMEM;
 	desc.type = CORESIGHT_DEV_TYPE_SOURCE;
-	desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_OTHERS;
+	desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_TPDM;
 	desc.ops = &tpdm_cs_ops;
 	desc.pdata = adev->dev.platform_data;
 	desc.dev = &adev->dev;
@@ -216,7 +910,7 @@ static int tpdm_probe(struct amba_device *adev, const struct amba_id *id)
 		return PTR_ERR(drvdata->csdev);
 
 	spin_lock_init(&drvdata->spinlock);
-	tpdm_init_default_data(drvdata);
+
 	/* Decrease pm refcount when probe is done.*/
 	pm_runtime_put(&adev->dev);
 
diff --git a/drivers/hwtracing/coresight/coresight-tpdm.h b/drivers/hwtracing/coresight/coresight-tpdm.h
index 543854043a2d..4115b2a17b8d 100644
--- a/drivers/hwtracing/coresight/coresight-tpdm.h
+++ b/drivers/hwtracing/coresight/coresight-tpdm.h
@@ -11,8 +11,52 @@
 
 /* DSB Subunit Registers */
 #define TPDM_DSB_CR		(0x780)
+#define TPDM_DSB_TIER		(0x784)
+#define TPDM_DSB_TPR(n)		(0x788 + (n * 4))
+#define TPDM_DSB_TPMR(n)	(0x7A8 + (n * 4))
+#define TPDM_DSB_XPR(n)		(0x7C8 + (n * 4))
+#define TPDM_DSB_XPMR(n)	(0x7E8 + (n * 4))
+#define TPDM_DSB_EDCR(n)	(0x808 + (n * 4))
+#define TPDM_DSB_EDCMR(n)	(0x848 + (n * 4))
+#define TPDM_DSB_MSR(n)		(0x980 + (n * 4))
+
 /* Enable bit for DSB subunit */
 #define TPDM_DSB_CR_ENA		BIT(0)
+/* Enable bit for DSB subunit perfmance mode */
+#define TPDM_DSB_CR_MODE		BIT(1)
+/* Enable bit for DSB subunit trigger type */
+#define TPDM_DSB_CR_TRIG_TYPE		BIT(12)
+/* Data bits for DSB high performace mode */
+#define TPDM_DSB_CR_HPSEL		GENMASK(6, 2)
+/* Data bits for DSB test mode */
+#define TPDM_DSB_CR_TEST_MODE		GENMASK(10, 9)
+
+/* Enable bit for DSB subunit pattern timestamp */
+#define TPDM_DSB_TIER_PATT_TSENAB		BIT(0)
+/* Enable bit for DSB subunit trigger timestamp */
+#define TPDM_DSB_TIER_XTRIG_TSENAB		BIT(1)
+/* Bit for DSB subunit pattern type */
+#define TPDM_DSB_TIER_PATT_TYPE			BIT(2)
+
+/* DSB programming modes */
+/* DSB mode bits mask */
+#define TPDM_DSB_MODE_MASK			GENMASK(8, 0)
+/* Test mode control bit*/
+#define TPDM_DSB_MODE_TEST(val)	(val & GENMASK(1, 0))
+/* Performance mode */
+#define TPDM_DSB_MODE_PERF		BIT(3)
+/* High performance mode */
+#define TPDM_DSB_MODE_HPBYTESEL(val)	(val & GENMASK(8, 4))
+
+#define EDCRS_PER_WORD			16
+#define EDCR_TO_WORD_IDX(r)		((r) / EDCRS_PER_WORD)
+#define EDCR_TO_WORD_SHIFT(r)		((r % EDCRS_PER_WORD) * 2)
+#define EDCR_TO_WORD_VAL(val, r)	(val << EDCR_TO_WORD_SHIFT(r))
+#define EDCR_TO_WORD_MASK(r)		EDCR_TO_WORD_VAL(0x3, r)
+
+#define EDCMRS_PER_WORD				32
+#define EDCMR_TO_WORD_IDX(r)		((r) / EDCMRS_PER_WORD)
+#define EDCMR_TO_WORD_SHIFT(r)		((r) % EDCMRS_PER_WORD)
 
 /* TPDM integration test registers */
 #define TPDM_ITATBCNTRL		(0xEF0)
@@ -40,6 +84,95 @@
 #define TPDM_PIDR0_DS_IMPDEF	BIT(0)
 #define TPDM_PIDR0_DS_DSB	BIT(1)
 
+#define TPDM_DSB_MAX_LINES	256
+/* MAX number of EDCR registers */
+#define TPDM_DSB_MAX_EDCR	16
+/* MAX number of EDCMR registers */
+#define TPDM_DSB_MAX_EDCMR	8
+/* MAX number of DSB pattern */
+#define TPDM_DSB_MAX_PATT	8
+/* MAX number of DSB MSR */
+#define TPDM_DSB_MAX_MSR 32
+
+#define tpdm_simple_dataset_ro(name, mem, idx)			\
+	(&((struct tpdm_dataset_attribute[]) {			\
+	   {								\
+		__ATTR(name, 0444, tpdm_simple_dataset_show, NULL),	\
+		mem,							\
+		idx,							\
+	   }								\
+	})[0].attr.attr)
+
+#define tpdm_simple_dataset_rw(name, mem, idx)			\
+	(&((struct tpdm_dataset_attribute[]) {			\
+	   {								\
+		__ATTR(name, 0644, tpdm_simple_dataset_show,		\
+		tpdm_simple_dataset_store),		\
+		mem,							\
+		idx,							\
+	   }								\
+	})[0].attr.attr)
+
+#define DSB_EDGE_CTRL_ATTR(nr)					\
+		tpdm_simple_dataset_ro(edcr##nr,		\
+		DSB_EDGE_CTRL, nr)
+
+#define DSB_EDGE_CTRL_MASK_ATTR(nr)				\
+		tpdm_simple_dataset_ro(edcmr##nr,		\
+		DSB_EDGE_CTRL_MASK, nr)
+
+#define DSB_TRIG_PATT_ATTR(nr)					\
+		tpdm_simple_dataset_rw(xpr##nr,			\
+		DSB_TRIG_PATT, nr)
+
+#define DSB_TRIG_PATT_MASK_ATTR(nr)				\
+		tpdm_simple_dataset_rw(xpmr##nr,		\
+		DSB_TRIG_PATT_MASK, nr)
+
+#define DSB_PATT_ATTR(nr)					\
+		tpdm_simple_dataset_rw(tpr##nr,			\
+		DSB_PATT, nr)
+
+#define DSB_PATT_MASK_ATTR(nr)					\
+		tpdm_simple_dataset_rw(tpmr##nr,		\
+		DSB_PATT_MASK, nr)
+
+#define DSB_MSR_ATTR(nr)					\
+		tpdm_simple_dataset_rw(msr##nr,			\
+		DSB_MSR, nr)
+
+/**
+ * struct dsb_dataset - specifics associated to dsb dataset
+ * @mode:             DSB programming mode
+ * @edge_ctrl_idx     Index number of the edge control
+ * @edge_ctrl:        Save value for edge control
+ * @edge_ctrl_mask:   Save value for edge control mask
+ * @patt_val:         Save value for pattern
+ * @patt_mask:        Save value for pattern mask
+ * @trig_patt:        Save value for trigger pattern
+ * @trig_patt_mask:   Save value for trigger pattern mask
+ * @msr               Save value for MSR
+ * @patt_ts:          Enable/Disable pattern timestamp
+ * @patt_type:        Set pattern type
+ * @trig_ts:          Enable/Disable trigger timestamp.
+ * @trig_type:        Enable/Disable trigger type.
+ */
+struct dsb_dataset {
+	u32			mode;
+	u32			edge_ctrl_idx;
+	u32			edge_ctrl[TPDM_DSB_MAX_EDCR];
+	u32			edge_ctrl_mask[TPDM_DSB_MAX_EDCMR];
+	u32			patt_val[TPDM_DSB_MAX_PATT];
+	u32			patt_mask[TPDM_DSB_MAX_PATT];
+	u32			trig_patt[TPDM_DSB_MAX_PATT];
+	u32			trig_patt_mask[TPDM_DSB_MAX_PATT];
+	u32			msr[TPDM_DSB_MAX_MSR];
+	bool			patt_ts;
+	bool			patt_type;
+	bool			trig_ts;
+	bool			trig_type;
+};
+
 /**
  * struct tpdm_drvdata - specifics associated to an TPDM component
  * @base:       memory mapped base address for this component.
@@ -48,6 +181,8 @@
  * @spinlock:   lock for the drvdata value.
  * @enable:     enable status of the component.
  * @datasets:   The datasets types present of the TPDM.
+ * @dsb         Specifics associated to TPDM DSB.
+ * @dsb_msr_num Number of MSR supported by DSB TPDM
  */
 
 struct tpdm_drvdata {
@@ -57,6 +192,32 @@ struct tpdm_drvdata {
 	spinlock_t		spinlock;
 	bool			enable;
 	unsigned long		datasets;
+	struct dsb_dataset	*dsb;
+	u32			dsb_msr_num;
+};
+
+/* Enumerate members of various datasets */
+enum dataset_mem {
+	DSB_EDGE_CTRL,
+	DSB_EDGE_CTRL_MASK,
+	DSB_TRIG_PATT,
+	DSB_TRIG_PATT_MASK,
+	DSB_PATT,
+	DSB_PATT_MASK,
+	DSB_MSR,
+};
+
+/**
+ * struct tpdm_dataset_attribute - Record the member variables and
+ * index number of datasets that need to be operated by sysfs file
+ * @attr:       The device attribute
+ * @mem:        The member in the dataset data structure
+ * @idx:        The index number of the array data
+ */
+struct tpdm_dataset_attribute {
+	struct device_attribute attr;
+	enum dataset_mem mem;
+	u32 idx;
 };
 
 #endif  /* _CORESIGHT_CORESIGHT_TPDM_H */
diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c
index e20c1c6acc73..6136776482e6 100644
--- a/drivers/hwtracing/coresight/coresight-trbe.c
+++ b/drivers/hwtracing/coresight/coresight-trbe.c
@@ -1253,8 +1253,18 @@ static void arm_trbe_register_coresight_cpu(struct trbe_drvdata *drvdata, int cp
 	desc.name = devm_kasprintf(dev, GFP_KERNEL, "trbe%d", cpu);
 	if (!desc.name)
 		goto cpu_clear;
-
-	desc.pdata = coresight_get_platform_data(dev);
+	/*
+	 * TRBE coresight devices do not need regular connections
+	 * information, as the paths get built between all percpu
+	 * source and their respective percpu sink devices. Though
+	 * coresight_register() expect device connections via the
+	 * platform_data, which TRBE devices do not have. As they
+	 * are not real ACPI devices, coresight_get_platform_data()
+	 * ends up failing. Instead let's allocate a dummy zeroed
+	 * coresight_platform_data structure and assign that back
+	 * into the device for that purpose.
+	 */
+	desc.pdata = devm_kzalloc(dev, sizeof(*desc.pdata), GFP_KERNEL);
 	if (IS_ERR(desc.pdata))
 		goto cpu_clear;
 
@@ -1520,14 +1530,13 @@ static int arm_trbe_device_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int arm_trbe_device_remove(struct platform_device *pdev)
+static void arm_trbe_device_remove(struct platform_device *pdev)
 {
 	struct trbe_drvdata *drvdata = platform_get_drvdata(pdev);
 
 	arm_trbe_remove_cpuhp(drvdata);
 	arm_trbe_remove_coresight(drvdata);
 	arm_trbe_remove_irq(drvdata);
-	return 0;
 }
 
 static const struct of_device_id arm_trbe_of_match[] = {
@@ -1536,14 +1545,23 @@ static const struct of_device_id arm_trbe_of_match[] = {
 };
 MODULE_DEVICE_TABLE(of, arm_trbe_of_match);
 
+#ifdef CONFIG_ACPI
+static const struct platform_device_id arm_trbe_acpi_match[] = {
+	{ ARMV8_TRBE_PDEV_NAME, 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(platform, arm_trbe_acpi_match);
+#endif
+
 static struct platform_driver arm_trbe_driver = {
+	.id_table = ACPI_PTR(arm_trbe_acpi_match),
 	.driver	= {
 		.name = DRVNAME,
 		.of_match_table = of_match_ptr(arm_trbe_of_match),
 		.suppress_bind_attrs = true,
 	},
 	.probe	= arm_trbe_device_probe,
-	.remove	= arm_trbe_device_remove,
+	.remove_new = arm_trbe_device_remove,
 };
 
 static int __init arm_trbe_init(void)
diff --git a/drivers/hwtracing/coresight/coresight-trbe.h b/drivers/hwtracing/coresight/coresight-trbe.h
index e915e749be55..45202c48acce 100644
--- a/drivers/hwtracing/coresight/coresight-trbe.h
+++ b/drivers/hwtracing/coresight/coresight-trbe.h
@@ -7,11 +7,13 @@
  *
  * Author: Anshuman Khandual <anshuman.khandual@arm.com>
  */
+#include <linux/acpi.h>
 #include <linux/coresight.h>
 #include <linux/device.h>
 #include <linux/irq.h>
 #include <linux/kernel.h>
 #include <linux/of.h>
+#include <linux/perf/arm_pmu.h>
 #include <linux/platform_device.h>
 #include <linux/smp.h>
 
diff --git a/drivers/hwtracing/coresight/ultrasoc-smb.c b/drivers/hwtracing/coresight/ultrasoc-smb.c
index e9a32a97fbee..10e886455b8b 100644
--- a/drivers/hwtracing/coresight/ultrasoc-smb.c
+++ b/drivers/hwtracing/coresight/ultrasoc-smb.c
@@ -97,27 +97,19 @@ static int smb_open(struct inode *inode, struct file *file)
 {
 	struct smb_drv_data *drvdata = container_of(file->private_data,
 					struct smb_drv_data, miscdev);
-	int ret = 0;
 
-	mutex_lock(&drvdata->mutex);
+	guard(spinlock)(&drvdata->spinlock);
 
-	if (drvdata->reading) {
-		ret = -EBUSY;
-		goto out;
-	}
+	if (drvdata->reading)
+		return -EBUSY;
 
-	if (atomic_read(&drvdata->csdev->refcnt)) {
-		ret = -EBUSY;
-		goto out;
-	}
+	if (atomic_read(&drvdata->csdev->refcnt))
+		return -EBUSY;
 
 	smb_update_data_size(drvdata);
-
 	drvdata->reading = true;
-out:
-	mutex_unlock(&drvdata->mutex);
 
-	return ret;
+	return 0;
 }
 
 static ssize_t smb_read(struct file *file, char __user *data, size_t len,
@@ -132,10 +124,8 @@ static ssize_t smb_read(struct file *file, char __user *data, size_t len,
 	if (!len)
 		return 0;
 
-	mutex_lock(&drvdata->mutex);
-
 	if (!sdb->data_size)
-		goto out;
+		return 0;
 
 	to_copy = min(sdb->data_size, len);
 
@@ -145,20 +135,15 @@ static ssize_t smb_read(struct file *file, char __user *data, size_t len,
 
 	if (copy_to_user(data, sdb->buf_base + sdb->buf_rdptr, to_copy)) {
 		dev_dbg(dev, "Failed to copy data to user\n");
-		to_copy = -EFAULT;
-		goto out;
+		return -EFAULT;
 	}
 
 	*ppos += to_copy;
-
 	smb_update_read_ptr(drvdata, to_copy);
-
-	dev_dbg(dev, "%zu bytes copied\n", to_copy);
-out:
 	if (!sdb->data_size)
 		smb_reset_buffer(drvdata);
-	mutex_unlock(&drvdata->mutex);
 
+	dev_dbg(dev, "%zu bytes copied\n", to_copy);
 	return to_copy;
 }
 
@@ -167,9 +152,8 @@ static int smb_release(struct inode *inode, struct file *file)
 	struct smb_drv_data *drvdata = container_of(file->private_data,
 					struct smb_drv_data, miscdev);
 
-	mutex_lock(&drvdata->mutex);
+	guard(spinlock)(&drvdata->spinlock);
 	drvdata->reading = false;
-	mutex_unlock(&drvdata->mutex);
 
 	return 0;
 }
@@ -262,19 +246,15 @@ static int smb_enable(struct coresight_device *csdev, enum cs_mode mode,
 	struct smb_drv_data *drvdata = dev_get_drvdata(csdev->dev.parent);
 	int ret = 0;
 
-	mutex_lock(&drvdata->mutex);
+	guard(spinlock)(&drvdata->spinlock);
 
 	/* Do nothing, the trace data is reading by other interface now */
-	if (drvdata->reading) {
-		ret = -EBUSY;
-		goto out;
-	}
+	if (drvdata->reading)
+		return -EBUSY;
 
 	/* Do nothing, the SMB is already enabled as other mode */
-	if (drvdata->mode != CS_MODE_DISABLED && drvdata->mode != mode) {
-		ret = -EBUSY;
-		goto out;
-	}
+	if (drvdata->mode != CS_MODE_DISABLED && drvdata->mode != mode)
+		return -EBUSY;
 
 	switch (mode) {
 	case CS_MODE_SYSFS:
@@ -288,13 +268,10 @@ static int smb_enable(struct coresight_device *csdev, enum cs_mode mode,
 	}
 
 	if (ret)
-		goto out;
+		return ret;
 
 	atomic_inc(&csdev->refcnt);
-
 	dev_dbg(&csdev->dev, "Ultrasoc SMB enabled\n");
-out:
-	mutex_unlock(&drvdata->mutex);
 
 	return ret;
 }
@@ -302,19 +279,14 @@ static int smb_enable(struct coresight_device *csdev, enum cs_mode mode,
 static int smb_disable(struct coresight_device *csdev)
 {
 	struct smb_drv_data *drvdata = dev_get_drvdata(csdev->dev.parent);
-	int ret = 0;
 
-	mutex_lock(&drvdata->mutex);
+	guard(spinlock)(&drvdata->spinlock);
 
-	if (drvdata->reading) {
-		ret = -EBUSY;
-		goto out;
-	}
+	if (drvdata->reading)
+		return -EBUSY;
 
-	if (atomic_dec_return(&csdev->refcnt)) {
-		ret = -EBUSY;
-		goto out;
-	}
+	if (atomic_dec_return(&csdev->refcnt))
+		return -EBUSY;
 
 	/* Complain if we (somehow) got out of sync */
 	WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED);
@@ -324,12 +296,9 @@ static int smb_disable(struct coresight_device *csdev)
 	/* Dissociate from the target process. */
 	drvdata->pid = -1;
 	drvdata->mode = CS_MODE_DISABLED;
-
 	dev_dbg(&csdev->dev, "Ultrasoc SMB disabled\n");
-out:
-	mutex_unlock(&drvdata->mutex);
 
-	return ret;
+	return 0;
 }
 
 static void *smb_alloc_buffer(struct coresight_device *csdev,
@@ -402,17 +371,17 @@ static unsigned long smb_update_buffer(struct coresight_device *csdev,
 	struct smb_drv_data *drvdata = dev_get_drvdata(csdev->dev.parent);
 	struct smb_data_buffer *sdb = &drvdata->sdb;
 	struct cs_buffers *buf = sink_config;
-	unsigned long data_size = 0;
+	unsigned long data_size;
 	bool lost = false;
 
 	if (!buf)
 		return 0;
 
-	mutex_lock(&drvdata->mutex);
+	guard(spinlock)(&drvdata->spinlock);
 
 	/* Don't do anything if another tracer is using this sink. */
 	if (atomic_read(&csdev->refcnt) != 1)
-		goto out;
+		return 0;
 
 	smb_disable_hw(drvdata);
 	smb_update_data_size(drvdata);
@@ -431,8 +400,6 @@ static unsigned long smb_update_buffer(struct coresight_device *csdev,
 	smb_sync_perf_buffer(drvdata, buf, handle->head);
 	if (!buf->snapshot && lost)
 		perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
-out:
-	mutex_unlock(&drvdata->mutex);
 
 	return data_size;
 }
@@ -484,7 +451,6 @@ static int smb_init_data_buffer(struct platform_device *pdev,
 static void smb_init_hw(struct smb_drv_data *drvdata)
 {
 	smb_disable_hw(drvdata);
-	smb_reset_buffer(drvdata);
 
 	writel(SMB_LB_CFG_LO_DEFAULT, drvdata->base + SMB_LB_CFG_LO_REG);
 	writel(SMB_LB_CFG_HI_DEFAULT, drvdata->base + SMB_LB_CFG_HI_REG);
@@ -590,38 +556,32 @@ static int smb_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	mutex_init(&drvdata->mutex);
+	ret = smb_config_inport(dev, true);
+	if (ret)
+		return ret;
+
+	smb_reset_buffer(drvdata);
+	platform_set_drvdata(pdev, drvdata);
+	spin_lock_init(&drvdata->spinlock);
 	drvdata->pid = -1;
 
 	ret = smb_register_sink(pdev, drvdata);
 	if (ret) {
+		smb_config_inport(&pdev->dev, false);
 		dev_err(dev, "Failed to register SMB sink\n");
 		return ret;
 	}
 
-	ret = smb_config_inport(dev, true);
-	if (ret) {
-		smb_unregister_sink(drvdata);
-		return ret;
-	}
-
-	platform_set_drvdata(pdev, drvdata);
-
 	return 0;
 }
 
-static int smb_remove(struct platform_device *pdev)
+static void smb_remove(struct platform_device *pdev)
 {
 	struct smb_drv_data *drvdata = platform_get_drvdata(pdev);
-	int ret;
-
-	ret = smb_config_inport(&pdev->dev, false);
-	if (ret)
-		return ret;
 
 	smb_unregister_sink(drvdata);
 
-	return 0;
+	smb_config_inport(&pdev->dev, false);
 }
 
 #ifdef CONFIG_ACPI
@@ -639,7 +599,7 @@ static struct platform_driver smb_driver = {
 		.suppress_bind_attrs = true,
 	},
 	.probe = smb_probe,
-	.remove = smb_remove,
+	.remove_new = smb_remove,
 };
 module_platform_driver(smb_driver);
 
diff --git a/drivers/hwtracing/coresight/ultrasoc-smb.h b/drivers/hwtracing/coresight/ultrasoc-smb.h
index d2e14e8d2c8a..82a44c14a882 100644
--- a/drivers/hwtracing/coresight/ultrasoc-smb.h
+++ b/drivers/hwtracing/coresight/ultrasoc-smb.h
@@ -8,7 +8,7 @@
 #define _ULTRASOC_SMB_H
 
 #include <linux/miscdevice.h>
-#include <linux/mutex.h>
+#include <linux/spinlock.h>
 
 /* Offset of SMB global registers */
 #define SMB_GLB_CFG_REG		0x00
@@ -105,7 +105,7 @@ struct smb_data_buffer {
  * @csdev:	Component vitals needed by the framework.
  * @sdb:	Data buffer for SMB.
  * @miscdev:	Specifics to handle "/dev/xyz.smb" entry.
- * @mutex:	Control data access to one at a time.
+ * @spinlock:	Control data access to one at a time.
  * @reading:	Synchronise user space access to SMB buffer.
  * @pid:	Process ID of the process being monitored by the
  *		session that is using this component.
@@ -116,7 +116,7 @@ struct smb_drv_data {
 	struct coresight_device	*csdev;
 	struct smb_data_buffer sdb;
 	struct miscdevice miscdev;
-	struct mutex mutex;
+	spinlock_t spinlock;
 	bool reading;
 	pid_t pid;
 	enum cs_mode mode;
diff --git a/drivers/hwtracing/ptt/hisi_ptt.c b/drivers/hwtracing/ptt/hisi_ptt.c
index 49ea1b0f7489..c1b5fd2b8974 100644
--- a/drivers/hwtracing/ptt/hisi_ptt.c
+++ b/drivers/hwtracing/ptt/hisi_ptt.c
@@ -183,6 +183,10 @@ static void hisi_ptt_wait_dma_reset_done(struct hisi_ptt *hisi_ptt)
 static void hisi_ptt_trace_end(struct hisi_ptt *hisi_ptt)
 {
 	writel(0, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL);
+
+	/* Mask the interrupt on the end */
+	writel(HISI_PTT_TRACE_INT_MASK_ALL, hisi_ptt->iobase + HISI_PTT_TRACE_INT_MASK);
+
 	hisi_ptt->trace_ctrl.started = false;
 }
 
@@ -270,15 +274,14 @@ static int hisi_ptt_update_aux(struct hisi_ptt *hisi_ptt, int index, bool stop)
 	buf->pos += size;
 
 	/*
-	 * Just commit the traced data if we're going to stop. Otherwise if the
-	 * resident AUX buffer cannot contain the data of next trace buffer,
-	 * apply a new one.
+	 * Always commit the data to the AUX buffer in time to make sure
+	 * userspace got enough time to consume the data.
+	 *
+	 * If we're not going to stop, apply a new one and check whether
+	 * there's enough room for the next trace.
 	 */
-	if (stop) {
-		perf_aux_output_end(handle, buf->pos);
-	} else if (buf->length - buf->pos < HISI_PTT_TRACE_BUF_SIZE) {
-		perf_aux_output_end(handle, buf->pos);
-
+	perf_aux_output_end(handle, size);
+	if (!stop) {
 		buf = perf_aux_output_begin(handle, event);
 		if (!buf)
 			return -EINVAL;
@@ -342,9 +345,9 @@ static int hisi_ptt_register_irq(struct hisi_ptt *hisi_ptt)
 		return ret;
 
 	hisi_ptt->trace_irq = pci_irq_vector(pdev, HISI_PTT_TRACE_DMA_IRQ);
-	ret = devm_request_threaded_irq(&pdev->dev, hisi_ptt->trace_irq,
-					NULL, hisi_ptt_isr, 0,
-					DRV_NAME, hisi_ptt);
+	ret = devm_request_irq(&pdev->dev, hisi_ptt->trace_irq, hisi_ptt_isr,
+				IRQF_NOBALANCING | IRQF_NO_THREAD, DRV_NAME,
+				hisi_ptt);
 	if (ret) {
 		pci_err(pdev, "failed to request irq %d, ret = %d\n",
 			hisi_ptt->trace_irq, ret);
@@ -1000,6 +1003,9 @@ static int hisi_ptt_pmu_event_init(struct perf_event *event)
 		return -EOPNOTSUPP;
 	}
 
+	if (event->attach_state & PERF_ATTACH_TASK)
+		return -EOPNOTSUPP;
+
 	if (event->attr.type != hisi_ptt->hisi_ptt_pmu.type)
 		return -ENOENT;
 
@@ -1178,6 +1184,10 @@ static void hisi_ptt_pmu_del(struct perf_event *event, int flags)
 	hisi_ptt_pmu_stop(event, PERF_EF_UPDATE);
 }
 
+static void hisi_ptt_pmu_read(struct perf_event *event)
+{
+}
+
 static void hisi_ptt_remove_cpuhp_instance(void *hotplug_node)
 {
 	cpuhp_state_remove_instance_nocalls(hisi_ptt_pmu_online, hotplug_node);
@@ -1221,6 +1231,7 @@ static int hisi_ptt_register_pmu(struct hisi_ptt *hisi_ptt)
 		.stop		= hisi_ptt_pmu_stop,
 		.add		= hisi_ptt_pmu_add,
 		.del		= hisi_ptt_pmu_del,
+		.read		= hisi_ptt_pmu_read,
 	};
 
 	reg = readl(hisi_ptt->iobase + HISI_PTT_LOCATION);
diff --git a/drivers/hwtracing/ptt/hisi_ptt.h b/drivers/hwtracing/ptt/hisi_ptt.h
index e17f045d7e72..46030aa88081 100644
--- a/drivers/hwtracing/ptt/hisi_ptt.h
+++ b/drivers/hwtracing/ptt/hisi_ptt.h
@@ -47,6 +47,7 @@
 #define HISI_PTT_TRACE_INT_STAT		0x0890
 #define   HISI_PTT_TRACE_INT_STAT_MASK	GENMASK(3, 0)
 #define HISI_PTT_TRACE_INT_MASK		0x0894
+#define   HISI_PTT_TRACE_INT_MASK_ALL	GENMASK(3, 0)
 #define HISI_PTT_TUNING_INT_STAT	0x0898
 #define   HISI_PTT_TUNING_INT_STAT_MASK	BIT(0)
 #define HISI_PTT_TRACE_WR_STS		0x08a0
diff --git a/drivers/i3c/master.c b/drivers/i3c/master.c
index 95caa162706f..8b729ebae2a6 100644
--- a/drivers/i3c/master.c
+++ b/drivers/i3c/master.c
@@ -1130,8 +1130,16 @@ static int i3c_master_getmxds_locked(struct i3c_master_controller *master,
 
 	i3c_ccc_cmd_init(&cmd, true, I3C_CCC_GETMXDS, &dest, 1);
 	ret = i3c_master_send_ccc_cmd_locked(master, &cmd);
-	if (ret)
-		goto out;
+	if (ret) {
+		/*
+		 * Retry when the device does not support max read turnaround
+		 * while expecting shorter length from this CCC command.
+		 */
+		dest.payload.len -= 3;
+		ret = i3c_master_send_ccc_cmd_locked(master, &cmd);
+		if (ret)
+			goto out;
+	}
 
 	if (dest.payload.len != 2 && dest.payload.len != 5) {
 		ret = -EIO;
diff --git a/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c b/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c
index 2b2323aa6714..638b054d6c92 100644
--- a/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c
+++ b/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c
@@ -298,7 +298,7 @@ static int hci_cmd_v1_daa(struct i3c_hci *hci)
 	unsigned int dcr, bcr;
 	DECLARE_COMPLETION_ONSTACK(done);
 
-	xfer = hci_alloc_xfer(2);
+	xfer = hci_alloc_xfer(1);
 	if (!xfer)
 		return -ENOMEM;
 
@@ -339,12 +339,13 @@ static int hci_cmd_v1_daa(struct i3c_hci *hci)
 			ret = -ETIME;
 			break;
 		}
-		if (RESP_STATUS(xfer[0].response) == RESP_ERR_NACK &&
+		if ((RESP_STATUS(xfer->response) == RESP_ERR_ADDR_HEADER ||
+		     RESP_STATUS(xfer->response) == RESP_ERR_NACK) &&
 		    RESP_DATA_LENGTH(xfer->response) == 1) {
 			ret = 0;  /* no more devices to be assigned */
 			break;
 		}
-		if (RESP_STATUS(xfer[0].response) != RESP_SUCCESS) {
+		if (RESP_STATUS(xfer->response) != RESP_SUCCESS) {
 			ret = -EIO;
 			break;
 		}
diff --git a/drivers/i3c/master/mipi-i3c-hci/core.c b/drivers/i3c/master/mipi-i3c-hci/core.c
index 1ae56a5699c6..d7e966a25583 100644
--- a/drivers/i3c/master/mipi-i3c-hci/core.c
+++ b/drivers/i3c/master/mipi-i3c-hci/core.c
@@ -245,7 +245,14 @@ static int i3c_hci_send_ccc_cmd(struct i3c_master_controller *m,
 		if (ccc->rnw)
 			ccc->dests[i - prefixed].payload.len =
 				RESP_DATA_LENGTH(xfer[i].response);
-		if (RESP_STATUS(xfer[i].response) != RESP_SUCCESS) {
+		switch (RESP_STATUS(xfer[i].response)) {
+		case RESP_SUCCESS:
+			continue;
+		case RESP_ERR_ADDR_HEADER:
+		case RESP_ERR_NACK:
+			ccc->err = I3C_ERROR_M2;
+			fallthrough;
+		default:
 			ret = -EIO;
 			goto out;
 		}
@@ -269,6 +276,34 @@ static int i3c_hci_daa(struct i3c_master_controller *m)
 	return hci->cmd->perform_daa(hci);
 }
 
+static int i3c_hci_alloc_safe_xfer_buf(struct i3c_hci *hci,
+				       struct hci_xfer *xfer)
+{
+	if (hci->io != &mipi_i3c_hci_dma ||
+	    xfer->data == NULL || !is_vmalloc_addr(xfer->data))
+		return 0;
+
+	if (xfer->rnw)
+		xfer->bounce_buf = kzalloc(xfer->data_len, GFP_KERNEL);
+	else
+		xfer->bounce_buf = kmemdup(xfer->data,
+					   xfer->data_len, GFP_KERNEL);
+
+	return xfer->bounce_buf == NULL ? -ENOMEM : 0;
+}
+
+static void i3c_hci_free_safe_xfer_buf(struct i3c_hci *hci,
+				       struct hci_xfer *xfer)
+{
+	if (hci->io != &mipi_i3c_hci_dma || xfer->bounce_buf == NULL)
+		return;
+
+	if (xfer->rnw)
+		memcpy(xfer->data, xfer->bounce_buf, xfer->data_len);
+
+	kfree(xfer->bounce_buf);
+}
+
 static int i3c_hci_priv_xfers(struct i3c_dev_desc *dev,
 			      struct i3c_priv_xfer *i3c_xfers,
 			      int nxfers)
@@ -302,6 +337,9 @@ static int i3c_hci_priv_xfers(struct i3c_dev_desc *dev,
 		}
 		hci->cmd->prep_i3c_xfer(hci, dev, &xfer[i]);
 		xfer[i].cmd_desc[0] |= CMD_0_ROC;
+		ret = i3c_hci_alloc_safe_xfer_buf(hci, &xfer[i]);
+		if (ret)
+			goto out;
 	}
 	last = i - 1;
 	xfer[last].cmd_desc[0] |= CMD_0_TOC;
@@ -325,6 +363,9 @@ static int i3c_hci_priv_xfers(struct i3c_dev_desc *dev,
 	}
 
 out:
+	for (i = 0; i < nxfers; i++)
+		i3c_hci_free_safe_xfer_buf(hci, &xfer[i]);
+
 	hci_free_xfer(xfer, nxfers);
 	return ret;
 }
@@ -350,6 +391,9 @@ static int i3c_hci_i2c_xfers(struct i2c_dev_desc *dev,
 		xfer[i].rnw = i2c_xfers[i].flags & I2C_M_RD;
 		hci->cmd->prep_i2c_xfer(hci, dev, &xfer[i]);
 		xfer[i].cmd_desc[0] |= CMD_0_ROC;
+		ret = i3c_hci_alloc_safe_xfer_buf(hci, &xfer[i]);
+		if (ret)
+			goto out;
 	}
 	last = i - 1;
 	xfer[last].cmd_desc[0] |= CMD_0_TOC;
@@ -371,6 +415,9 @@ static int i3c_hci_i2c_xfers(struct i2c_dev_desc *dev,
 	}
 
 out:
+	for (i = 0; i < nxfers; i++)
+		i3c_hci_free_safe_xfer_buf(hci, &xfer[i]);
+
 	hci_free_xfer(xfer, nxfers);
 	return ret;
 }
diff --git a/drivers/i3c/master/mipi-i3c-hci/dma.c b/drivers/i3c/master/mipi-i3c-hci/dma.c
index c805a8497319..4e01a95cc4d0 100644
--- a/drivers/i3c/master/mipi-i3c-hci/dma.c
+++ b/drivers/i3c/master/mipi-i3c-hci/dma.c
@@ -362,6 +362,7 @@ static int hci_dma_queue_xfer(struct i3c_hci *hci,
 	struct hci_rh_data *rh;
 	unsigned int i, ring, enqueue_ptr;
 	u32 op1_val, op2_val;
+	void *buf;
 
 	/* For now we only use ring 0 */
 	ring = 0;
@@ -390,9 +391,10 @@ static int hci_dma_queue_xfer(struct i3c_hci *hci,
 
 		/* 2nd and 3rd words of Data Buffer Descriptor Structure */
 		if (xfer->data) {
+			buf = xfer->bounce_buf ? xfer->bounce_buf : xfer->data;
 			xfer->data_dma =
 				dma_map_single(&hci->master.dev,
-					       xfer->data,
+					       buf,
 					       xfer->data_len,
 					       xfer->rnw ?
 						  DMA_FROM_DEVICE :
diff --git a/drivers/i3c/master/mipi-i3c-hci/hci.h b/drivers/i3c/master/mipi-i3c-hci/hci.h
index f109923f6c3f..f94d95e024be 100644
--- a/drivers/i3c/master/mipi-i3c-hci/hci.h
+++ b/drivers/i3c/master/mipi-i3c-hci/hci.h
@@ -90,6 +90,7 @@ struct hci_xfer {
 		struct {
 			/* DMA specific */
 			dma_addr_t data_dma;
+			void *bounce_buf;
 			int ring_number;
 			int ring_entry;
 		};
diff --git a/drivers/iio/accel/kionix-kx022a.c b/drivers/iio/accel/kionix-kx022a.c
index 60864be3a667..53d59a04ae15 100644
--- a/drivers/iio/accel/kionix-kx022a.c
+++ b/drivers/iio/accel/kionix-kx022a.c
@@ -393,17 +393,17 @@ static const unsigned int kx022a_odrs[] = {
  *	(range / 2^bits) * g = (range / 2^bits) * 9.80665 m/s^2
  *	=> KX022A uses 16 bit (HiRes mode - assume the low 8 bits are zeroed
  *	in low-power mode(?) )
- *	=> +/-2G  => 4 / 2^16 * 9,80665 * 10^6 (to scale to micro)
- *	=> +/-2G  - 598.550415
- *	   +/-4G  - 1197.10083
- *	   +/-8G  - 2394.20166
- *	   +/-16G - 4788.40332
+ *	=> +/-2G  => 4 / 2^16 * 9,80665
+ *	=> +/-2G  - 0.000598550415
+ *	   +/-4G  - 0.00119710083
+ *	   +/-8G  - 0.00239420166
+ *	   +/-16G - 0.00478840332
  */
 static const int kx022a_scale_table[][2] = {
-	{ 598, 550415 },
-	{ 1197, 100830 },
-	{ 2394, 201660 },
-	{ 4788, 403320 },
+	{ 0, 598550 },
+	{ 0, 1197101 },
+	{ 0, 2394202 },
+	{ 0, 4788403 },
 };
 
 static int kx022a_read_avail(struct iio_dev *indio_dev,
@@ -422,7 +422,7 @@ static int kx022a_read_avail(struct iio_dev *indio_dev,
 		*vals = (const int *)kx022a_scale_table;
 		*length = ARRAY_SIZE(kx022a_scale_table) *
 			  ARRAY_SIZE(kx022a_scale_table[0]);
-		*type = IIO_VAL_INT_PLUS_MICRO;
+		*type = IIO_VAL_INT_PLUS_NANO;
 		return IIO_AVAIL_LIST;
 	default:
 		return -EINVAL;
@@ -485,6 +485,20 @@ static int kx022a_turn_on_unlock(struct kx022a_data *data)
 	return ret;
 }
 
+static int kx022a_write_raw_get_fmt(struct iio_dev *idev,
+				    struct iio_chan_spec const *chan,
+				    long mask)
+{
+	switch (mask) {
+	case IIO_CHAN_INFO_SCALE:
+		return IIO_VAL_INT_PLUS_NANO;
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		return IIO_VAL_INT_PLUS_MICRO;
+	default:
+		return -EINVAL;
+	}
+}
+
 static int kx022a_write_raw(struct iio_dev *idev,
 			    struct iio_chan_spec const *chan,
 			    int val, int val2, long mask)
@@ -629,7 +643,7 @@ static int kx022a_read_raw(struct iio_dev *idev,
 
 		kx022a_reg2scale(regval, val, val2);
 
-		return IIO_VAL_INT_PLUS_MICRO;
+		return IIO_VAL_INT_PLUS_NANO;
 	}
 
 	return -EINVAL;
@@ -856,6 +870,7 @@ static int kx022a_fifo_flush(struct iio_dev *idev, unsigned int samples)
 static const struct iio_info kx022a_info = {
 	.read_raw = &kx022a_read_raw,
 	.write_raw = &kx022a_write_raw,
+	.write_raw_get_fmt = &kx022a_write_raw_get_fmt,
 	.read_avail = &kx022a_read_avail,
 
 	.validate_trigger	= iio_validate_own_trigger,
diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig
index 35f9867da12c..1e2b7a2c67c6 100644
--- a/drivers/iio/adc/Kconfig
+++ b/drivers/iio/adc/Kconfig
@@ -745,6 +745,17 @@ config MAX1363
 	  To compile this driver as a module, choose M here: the module will be
 	  called max1363.
 
+config MAX34408
+	tristate "Maxim max34408/max344089 ADC driver"
+	depends on I2C
+	help
+	  Say yes here to build support for Maxim max34408/max34409 current sense
+	  monitor with 8-bits ADC interface with overcurrent delay/threshold and
+	  shutdown delay.
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called max34408.
+
 config MAX77541_ADC
 	tristate "Analog Devices MAX77541 ADC driver"
 	depends on MFD_MAX77541
diff --git a/drivers/iio/adc/Makefile b/drivers/iio/adc/Makefile
index bee11d442af4..c0803383a7cc 100644
--- a/drivers/iio/adc/Makefile
+++ b/drivers/iio/adc/Makefile
@@ -68,6 +68,7 @@ obj-$(CONFIG_MAX11205) += max11205.o
 obj-$(CONFIG_MAX11410) += max11410.o
 obj-$(CONFIG_MAX1241) += max1241.o
 obj-$(CONFIG_MAX1363) += max1363.o
+obj-$(CONFIG_MAX34408) += max34408.o
 obj-$(CONFIG_MAX77541_ADC) += max77541-adc.o
 obj-$(CONFIG_MAX9611) += max9611.o
 obj-$(CONFIG_MCP320X) += mcp320x.o
diff --git a/drivers/iio/adc/max34408.c b/drivers/iio/adc/max34408.c
new file mode 100644
index 000000000000..6c2ea2bc52c6
--- /dev/null
+++ b/drivers/iio/adc/max34408.c
@@ -0,0 +1,276 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * IIO driver for Maxim MAX34409/34408 ADC, 4-Channels/2-Channels, 8bits, I2C
+ *
+ * Datasheet: https://www.analog.com/media/en/technical-documentation/data-sheets/MAX34408-MAX34409.pdf
+ *
+ * TODO: ALERT interrupt, Overcurrent delay, Shutdown delay
+ */
+
+#include <linux/bitfield.h>
+#include <linux/init.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+
+#include <linux/iio/iio.h>
+#include <linux/iio/types.h>
+
+#define MAX34408_STATUS_REG		0x0
+#define MAX34408_CONTROL_REG		0x1
+#define MAX34408_OCDELAY_REG		0x2
+#define MAX34408_SDDELAY_REG		0x3
+
+#define MAX34408_ADC1_REG		0x4
+#define MAX34408_ADC2_REG		0x5
+/* ADC3 & ADC4 always returns 0x0 on 34408 */
+#define MAX34409_ADC3_REG		0x6
+#define MAX34409_ADC4_REG		0x7
+
+#define MAX34408_OCT1_REG		0x8
+#define MAX34408_OCT2_REG		0x9
+#define MAX34409_OCT3_REG		0xA
+#define MAX34409_OCT4_REG		0xB
+
+#define MAX34408_DID_REG		0xC
+#define MAX34408_DCYY_REG		0xD
+#define MAX34408_DCWW_REG		0xE
+
+/* Bit masks for status register */
+#define MAX34408_STATUS_OC_MSK		GENMASK(1, 0)
+#define MAX34409_STATUS_OC_MSK		GENMASK(3, 0)
+#define MAX34408_STATUS_SHTDN		BIT(4)
+#define MAX34408_STATUS_ENA		BIT(5)
+
+/* Bit masks for control register */
+#define MAX34408_CONTROL_AVG0		BIT(0)
+#define MAX34408_CONTROL_AVG1		BIT(1)
+#define MAX34408_CONTROL_AVG2		BIT(2)
+#define MAX34408_CONTROL_ALERT		BIT(3)
+
+#define MAX34408_DEFAULT_AVG		0x4
+
+/* Bit masks for over current delay */
+#define MAX34408_OCDELAY_OCD_MSK	GENMASK(6, 0)
+#define MAX34408_OCDELAY_RESET		BIT(7)
+
+/* Bit masks for shutdown delay */
+#define MAX34408_SDDELAY_SHD_MSK	GENMASK(6, 0)
+#define MAX34408_SDDELAY_RESET		BIT(7)
+
+#define MAX34408_DEFAULT_RSENSE		1000
+
+/**
+ * struct max34408_data - max34408/max34409 specific data.
+ * @regmap:	device register map.
+ * @dev:	max34408 device.
+ * @lock:	lock for protecting access to device hardware registers, mostly
+ *		for read modify write cycles for control registers.
+ * @input_rsense:	Rsense values in uOhm, will be overwritten by
+ *			values from channel nodes.
+ */
+struct max34408_data {
+	struct regmap *regmap;
+	struct device *dev;
+	struct mutex lock;
+	u32 input_rsense[4];
+};
+
+static const struct regmap_config max34408_regmap_config = {
+	.reg_bits	= 8,
+	.val_bits	= 8,
+	.max_register	= MAX34408_DCWW_REG,
+};
+
+struct max34408_adc_model_data {
+	const char *model_name;
+	const struct iio_chan_spec *channels;
+	const int num_channels;
+};
+
+#define MAX34008_CHANNEL(_index, _address)			\
+	{							\
+		.type = IIO_CURRENT,				\
+		.info_mask_separate	= BIT(IIO_CHAN_INFO_RAW) | \
+					  BIT(IIO_CHAN_INFO_SCALE) | \
+					  BIT(IIO_CHAN_INFO_OFFSET), \
+		.channel = (_index),				\
+		.address = (_address),				\
+		.indexed = 1,					\
+	}
+
+static const struct iio_chan_spec max34408_channels[] = {
+	MAX34008_CHANNEL(0, MAX34408_ADC1_REG),
+	MAX34008_CHANNEL(1, MAX34408_ADC2_REG),
+};
+
+static const struct iio_chan_spec max34409_channels[] = {
+	MAX34008_CHANNEL(0, MAX34408_ADC1_REG),
+	MAX34008_CHANNEL(1, MAX34408_ADC2_REG),
+	MAX34008_CHANNEL(2, MAX34409_ADC3_REG),
+	MAX34008_CHANNEL(3, MAX34409_ADC4_REG),
+};
+
+static int max34408_read_adc_avg(struct max34408_data *max34408,
+				 const struct iio_chan_spec *chan, int *val)
+{
+	unsigned int ctrl;
+	int rc;
+
+	guard(mutex)(&max34408->lock);
+	rc = regmap_read(max34408->regmap, MAX34408_CONTROL_REG, (u32 *)&ctrl);
+	if (rc)
+		return rc;
+
+	/* set averaging (0b100) default values*/
+	rc = regmap_write(max34408->regmap, MAX34408_CONTROL_REG,
+			  MAX34408_DEFAULT_AVG);
+	if (rc) {
+		dev_err(max34408->dev,
+			"Error (%d) writing control register\n", rc);
+		return rc;
+	}
+
+	rc = regmap_read(max34408->regmap, chan->address, val);
+	if (rc)
+		return rc;
+
+	/* back to old values */
+	rc = regmap_write(max34408->regmap, MAX34408_CONTROL_REG, ctrl);
+	if (rc)
+		dev_err(max34408->dev,
+			"Error (%d) writing control register\n", rc);
+
+	return rc;
+}
+
+static int max34408_read_raw(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     int *val, int *val2, long mask)
+{
+	struct max34408_data *max34408 = iio_priv(indio_dev);
+	int rc;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		rc = max34408_read_adc_avg(max34408, chan, val);
+		if (rc)
+			return rc;
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		/*
+		 * calcluate current for 8bit ADC with Rsense
+		 * value.
+		 * 10 mV * 1000 / Rsense uOhm = max current
+		 * (max current * adc val * 1000) / (2^8 - 1) mA
+		 */
+		*val = 10000 / max34408->input_rsense[chan->channel];
+		*val2 = 8;
+		return IIO_VAL_FRACTIONAL_LOG2;
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info max34408_info = {
+	.read_raw	= max34408_read_raw,
+};
+
+static const struct max34408_adc_model_data max34408_model_data = {
+	.model_name = "max34408",
+	.channels = max34408_channels,
+	.num_channels = 2,
+};
+
+static const struct max34408_adc_model_data max34409_model_data = {
+	.model_name = "max34409",
+	.channels = max34409_channels,
+	.num_channels = 4,
+};
+
+static int max34408_probe(struct i2c_client *client)
+{
+	const struct max34408_adc_model_data *model_data;
+	struct device *dev = &client->dev;
+	struct max34408_data *max34408;
+	struct fwnode_handle *node;
+	struct iio_dev *indio_dev;
+	struct regmap *regmap;
+	int rc, i = 0;
+
+	model_data = i2c_get_match_data(client);
+	if (!model_data)
+		return -EINVAL;
+
+	regmap = devm_regmap_init_i2c(client, &max34408_regmap_config);
+	if (IS_ERR(regmap)) {
+		dev_err_probe(dev, PTR_ERR(regmap),
+			      "regmap_init failed\n");
+		return PTR_ERR(regmap);
+	}
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*max34408));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	max34408 = iio_priv(indio_dev);
+	max34408->regmap = regmap;
+	max34408->dev = dev;
+	mutex_init(&max34408->lock);
+
+	device_for_each_child_node(dev, node) {
+		fwnode_property_read_u32(node, "maxim,rsense-val-micro-ohms",
+					 &max34408->input_rsense[i]);
+		i++;
+	}
+
+	/* disable ALERT and averaging */
+	rc = regmap_write(max34408->regmap, MAX34408_CONTROL_REG, 0x0);
+	if (rc)
+		return rc;
+
+	indio_dev->channels = model_data->channels;
+	indio_dev->num_channels = model_data->num_channels;
+	indio_dev->name = model_data->model_name;
+
+	indio_dev->info = &max34408_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+
+	return devm_iio_device_register(dev, indio_dev);
+}
+
+static const struct of_device_id max34408_of_match[] = {
+	{
+		.compatible = "maxim,max34408",
+		.data = &max34408_model_data,
+	},
+	{
+		.compatible = "maxim,max34409",
+		.data = &max34409_model_data,
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, max34408_of_match);
+
+static const struct i2c_device_id max34408_id[] = {
+	{ "max34408", (kernel_ulong_t)&max34408_model_data },
+	{ "max34409", (kernel_ulong_t)&max34409_model_data },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, max34408_id);
+
+static struct i2c_driver max34408_driver = {
+	.driver = {
+		.name   = "max34408",
+		.of_match_table = max34408_of_match,
+	},
+	.probe = max34408_probe,
+	.id_table = max34408_id,
+};
+module_i2c_driver(max34408_driver);
+
+MODULE_AUTHOR("Ivan Mikhaylov <fr0st61te@gmail.com>");
+MODULE_DESCRIPTION("Maxim MAX34408/34409 ADC driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/common/ms_sensors/ms_sensors_i2c.c b/drivers/iio/common/ms_sensors/ms_sensors_i2c.c
index 6633b35a94e6..9c9bc77003c7 100644
--- a/drivers/iio/common/ms_sensors/ms_sensors_i2c.c
+++ b/drivers/iio/common/ms_sensors/ms_sensors_i2c.c
@@ -15,8 +15,8 @@
 /* Conversion times in us */
 static const u16 ms_sensors_ht_t_conversion_time[] = { 50000, 25000,
 						       13000, 7000 };
-static const u16 ms_sensors_ht_h_conversion_time[] = { 16000, 3000,
-						       5000, 8000 };
+static const u16 ms_sensors_ht_h_conversion_time[] = { 16000, 5000,
+						       3000, 8000 };
 static const u16 ms_sensors_tp_conversion_time[] = { 500, 1100, 2100,
 						     4100, 8220, 16440 };
 
diff --git a/drivers/iio/frequency/adf4377.c b/drivers/iio/frequency/adf4377.c
index 26abecbd51e0..9284c13f1abb 100644
--- a/drivers/iio/frequency/adf4377.c
+++ b/drivers/iio/frequency/adf4377.c
@@ -870,7 +870,6 @@ static const struct iio_chan_spec adf4377_channels[] = {
 static int adf4377_properties_parse(struct adf4377_state *st)
 {
 	struct spi_device *spi = st->spi;
-	const char *str;
 	int ret;
 
 	st->clkin = devm_clk_get_enabled(&spi->dev, "ref_in");
@@ -896,16 +895,13 @@ static int adf4377_properties_parse(struct adf4377_state *st)
 		return dev_err_probe(&spi->dev, PTR_ERR(st->gpio_enclk2),
 				     "failed to get the CE GPIO\n");
 
-	ret = device_property_read_string(&spi->dev, "adi,muxout-select", &str);
-	if (ret) {
-		st->muxout_select = ADF4377_MUXOUT_HIGH_Z;
-	} else {
-		ret = match_string(adf4377_muxout_modes, ARRAY_SIZE(adf4377_muxout_modes), str);
-		if (ret < 0)
-			return ret;
-
+	ret = device_property_match_property_string(&spi->dev, "adi,muxout-select",
+						    adf4377_muxout_modes,
+						    ARRAY_SIZE(adf4377_muxout_modes));
+	if (ret >= 0)
 		st->muxout_select = ret;
-	}
+	else
+		st->muxout_select = ADF4377_MUXOUT_HIGH_Z;
 
 	return 0;
 }
diff --git a/drivers/iio/frequency/admv1014.c b/drivers/iio/frequency/admv1014.c
index bb5e1feef42b..b46b73b89eb7 100644
--- a/drivers/iio/frequency/admv1014.c
+++ b/drivers/iio/frequency/admv1014.c
@@ -710,7 +710,6 @@ static int admv1014_init(struct admv1014_state *st)
 
 static int admv1014_properties_parse(struct admv1014_state *st)
 {
-	const char *str;
 	unsigned int i;
 	struct spi_device *spi = st->spi;
 	int ret;
@@ -719,27 +718,21 @@ static int admv1014_properties_parse(struct admv1014_state *st)
 
 	st->p1db_comp = device_property_read_bool(&spi->dev, "adi,p1db-compensation-enable");
 
-	ret = device_property_read_string(&spi->dev, "adi,input-mode", &str);
-	if (ret) {
-		st->input_mode = ADMV1014_IQ_MODE;
-	} else {
-		ret = match_string(input_mode_names, ARRAY_SIZE(input_mode_names), str);
-		if (ret < 0)
-			return ret;
-
+	ret = device_property_match_property_string(&spi->dev, "adi,input-mode",
+						    input_mode_names,
+						    ARRAY_SIZE(input_mode_names));
+	if (ret >= 0)
 		st->input_mode = ret;
-	}
-
-	ret = device_property_read_string(&spi->dev, "adi,quad-se-mode", &str);
-	if (ret) {
-		st->quad_se_mode = ADMV1014_SE_MODE_POS;
-	} else {
-		ret = match_string(quad_se_mode_names, ARRAY_SIZE(quad_se_mode_names), str);
-		if (ret < 0)
-			return ret;
+	else
+		st->input_mode = ADMV1014_IQ_MODE;
 
+	ret = device_property_match_property_string(&spi->dev, "adi,quad-se-mode",
+						    quad_se_mode_names,
+						    ARRAY_SIZE(quad_se_mode_names));
+	if (ret >= 0)
 		st->quad_se_mode = ADMV1014_SE_MODE_POS + (ret * 3);
-	}
+	else
+		st->quad_se_mode = ADMV1014_SE_MODE_POS;
 
 	for (i = 0; i < ADMV1014_NUM_REGULATORS; ++i)
 		st->regulators[i].supply = admv1014_reg_name[i];
diff --git a/drivers/iio/imu/Kconfig b/drivers/iio/imu/Kconfig
index c2f97629e9cd..52a155ff3250 100644
--- a/drivers/iio/imu/Kconfig
+++ b/drivers/iio/imu/Kconfig
@@ -53,6 +53,7 @@ config ADIS16480
 	  ADIS16485, ADIS16488 inertial sensors.
 
 source "drivers/iio/imu/bmi160/Kconfig"
+source "drivers/iio/imu/bmi323/Kconfig"
 source "drivers/iio/imu/bno055/Kconfig"
 
 config FXOS8700
diff --git a/drivers/iio/imu/Makefile b/drivers/iio/imu/Makefile
index 6eb612034722..7e2d7d5c3b7b 100644
--- a/drivers/iio/imu/Makefile
+++ b/drivers/iio/imu/Makefile
@@ -15,6 +15,7 @@ adis_lib-$(CONFIG_IIO_ADIS_LIB_BUFFER) += adis_buffer.o
 obj-$(CONFIG_IIO_ADIS_LIB) += adis_lib.o
 
 obj-y += bmi160/
+obj-y += bmi323/
 obj-y += bno055/
 
 obj-$(CONFIG_FXOS8700) += fxos8700_core.o
diff --git a/drivers/iio/imu/adis.c b/drivers/iio/imu/adis.c
index bc40240b29e2..495caf4ce87a 100644
--- a/drivers/iio/imu/adis.c
+++ b/drivers/iio/imu/adis.c
@@ -44,8 +44,6 @@ int __adis_write_reg(struct adis *adis, unsigned int reg, unsigned int value,
 			.cs_change = 1,
 			.delay.value = adis->data->write_delay,
 			.delay.unit = SPI_DELAY_UNIT_USECS,
-			.cs_change_delay.value = adis->data->cs_change_delay,
-			.cs_change_delay.unit = SPI_DELAY_UNIT_USECS,
 		}, {
 			.tx_buf = adis->tx + 2,
 			.bits_per_word = 8,
@@ -53,8 +51,6 @@ int __adis_write_reg(struct adis *adis, unsigned int reg, unsigned int value,
 			.cs_change = 1,
 			.delay.value = adis->data->write_delay,
 			.delay.unit = SPI_DELAY_UNIT_USECS,
-			.cs_change_delay.value = adis->data->cs_change_delay,
-			.cs_change_delay.unit = SPI_DELAY_UNIT_USECS,
 		}, {
 			.tx_buf = adis->tx + 4,
 			.bits_per_word = 8,
@@ -62,8 +58,6 @@ int __adis_write_reg(struct adis *adis, unsigned int reg, unsigned int value,
 			.cs_change = 1,
 			.delay.value = adis->data->write_delay,
 			.delay.unit = SPI_DELAY_UNIT_USECS,
-			.cs_change_delay.value = adis->data->cs_change_delay,
-			.cs_change_delay.unit = SPI_DELAY_UNIT_USECS,
 		}, {
 			.tx_buf = adis->tx + 6,
 			.bits_per_word = 8,
@@ -144,8 +138,6 @@ int __adis_read_reg(struct adis *adis, unsigned int reg, unsigned int *val,
 			.cs_change = 1,
 			.delay.value = adis->data->write_delay,
 			.delay.unit = SPI_DELAY_UNIT_USECS,
-			.cs_change_delay.value = adis->data->cs_change_delay,
-			.cs_change_delay.unit = SPI_DELAY_UNIT_USECS,
 		}, {
 			.tx_buf = adis->tx + 2,
 			.bits_per_word = 8,
@@ -153,8 +145,6 @@ int __adis_read_reg(struct adis *adis, unsigned int reg, unsigned int *val,
 			.cs_change = 1,
 			.delay.value = adis->data->read_delay,
 			.delay.unit = SPI_DELAY_UNIT_USECS,
-			.cs_change_delay.value = adis->data->cs_change_delay,
-			.cs_change_delay.unit = SPI_DELAY_UNIT_USECS,
 		}, {
 			.tx_buf = adis->tx + 4,
 			.rx_buf = adis->rx,
@@ -163,8 +153,6 @@ int __adis_read_reg(struct adis *adis, unsigned int reg, unsigned int *val,
 			.cs_change = 1,
 			.delay.value = adis->data->read_delay,
 			.delay.unit = SPI_DELAY_UNIT_USECS,
-			.cs_change_delay.value = adis->data->cs_change_delay,
-			.cs_change_delay.unit = SPI_DELAY_UNIT_USECS,
 		}, {
 			.rx_buf = adis->rx + 2,
 			.bits_per_word = 8,
@@ -524,6 +512,12 @@ int adis_init(struct adis *adis, struct iio_dev *indio_dev,
 	}
 
 	mutex_init(&adis->state_lock);
+
+	if (!spi->cs_inactive.value) {
+		spi->cs_inactive.value = data->cs_change_delay;
+		spi->cs_inactive.unit = SPI_DELAY_UNIT_USECS;
+	}
+
 	adis->spi = spi;
 	adis->data = data;
 	iio_device_set_drvdata(indio_dev, adis);
diff --git a/drivers/iio/imu/bmi323/Kconfig b/drivers/iio/imu/bmi323/Kconfig
new file mode 100644
index 000000000000..ab37b285393c
--- /dev/null
+++ b/drivers/iio/imu/bmi323/Kconfig
@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# BMI323 IMU driver
+#
+
+config BMI323
+	tristate
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
+
+config BMI323_I2C
+	tristate "Bosch BMI323 I2C driver"
+	depends on I2C
+	select BMI323
+	select REGMAP_I2C
+	help
+	  Enable support for the Bosch BMI323 6-Axis IMU connected to I2C
+	  interface.
+
+	  This driver can also be built as a module. If so, the module will be
+	  called bmi323_i2c.
+
+config BMI323_SPI
+	tristate "Bosch BMI323 SPI driver"
+	depends on SPI
+	select BMI323
+	select REGMAP_SPI
+	help
+	  Enable support for the Bosch BMI323 6-Axis IMU connected to SPI
+	  interface.
+
+	  This driver can also be built as a module. If so, the module will be
+	  called bmi323_spi.
diff --git a/drivers/iio/imu/bmi323/Makefile b/drivers/iio/imu/bmi323/Makefile
new file mode 100644
index 000000000000..a6a6dc0207c9
--- /dev/null
+++ b/drivers/iio/imu/bmi323/Makefile
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for Bosch BMI323 IMU
+#
+obj-$(CONFIG_BMI323) += bmi323_core.o
+obj-$(CONFIG_BMI323_I2C) += bmi323_i2c.o
+obj-$(CONFIG_BMI323_SPI) += bmi323_spi.o
diff --git a/drivers/iio/imu/bmi323/bmi323.h b/drivers/iio/imu/bmi323/bmi323.h
new file mode 100644
index 000000000000..dff126d41658
--- /dev/null
+++ b/drivers/iio/imu/bmi323/bmi323.h
@@ -0,0 +1,209 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * IIO driver for Bosch BMI323 6-Axis IMU
+ *
+ * Copyright (C) 2023, Jagath Jog J <jagathjog1996@gmail.com>
+ */
+
+#ifndef _BMI323_H_
+#define _BMI323_H_
+
+#include <linux/bits.h>
+#include <linux/regmap.h>
+#include <linux/units.h>
+
+#define BMI323_I2C_DUMMY			2
+#define BMI323_SPI_DUMMY			1
+
+/* Register map */
+
+#define BMI323_CHIP_ID_REG			0x00
+#define BMI323_CHIP_ID_VAL			0x0043
+#define BMI323_CHIP_ID_MSK			GENMASK(7, 0)
+#define BMI323_ERR_REG				0x01
+#define BMI323_STATUS_REG			0x02
+#define BMI323_STATUS_POR_MSK			BIT(0)
+
+/* Accelero/Gyro/Temp data registers */
+#define BMI323_ACCEL_X_REG			0x03
+#define BMI323_GYRO_X_REG			0x06
+#define BMI323_TEMP_REG				0x09
+#define BMI323_ALL_CHAN_MSK			GENMASK(5, 0)
+
+/* Status registers */
+#define BMI323_STATUS_INT1_REG			0x0D
+#define BMI323_STATUS_INT2_REG			0x0E
+#define BMI323_STATUS_NOMOTION_MSK		BIT(0)
+#define BMI323_STATUS_MOTION_MSK		BIT(1)
+#define BMI323_STATUS_STP_WTR_MSK		BIT(5)
+#define BMI323_STATUS_TAP_MSK			BIT(8)
+#define BMI323_STATUS_ERROR_MSK			BIT(10)
+#define BMI323_STATUS_TMP_DRDY_MSK		BIT(11)
+#define BMI323_STATUS_GYR_DRDY_MSK		BIT(12)
+#define BMI323_STATUS_ACC_DRDY_MSK		BIT(13)
+#define BMI323_STATUS_ACC_GYR_DRDY_MSK		GENMASK(13, 12)
+#define BMI323_STATUS_FIFO_WTRMRK_MSK		BIT(14)
+#define BMI323_STATUS_FIFO_FULL_MSK		BIT(15)
+
+/* Feature registers */
+#define BMI323_FEAT_IO0_REG			0x10
+#define BMI323_FEAT_IO0_XYZ_NOMOTION_MSK	GENMASK(2, 0)
+#define BMI323_FEAT_IO0_XYZ_MOTION_MSK		GENMASK(5, 3)
+#define BMI323_FEAT_XYZ_MSK			GENMASK(2, 0)
+#define BMI323_FEAT_IO0_STP_CNT_MSK		BIT(9)
+#define BMI323_FEAT_IO0_S_TAP_MSK		BIT(12)
+#define BMI323_FEAT_IO0_D_TAP_MSK		BIT(13)
+#define BMI323_FEAT_IO1_REG			0x11
+#define BMI323_FEAT_IO1_ERR_MSK			GENMASK(3, 0)
+#define BMI323_FEAT_IO2_REG			0x12
+#define BMI323_FEAT_IO_STATUS_REG		0x14
+#define BMI323_FEAT_IO_STATUS_MSK		BIT(0)
+#define BMI323_FEAT_ENG_POLL			2000
+#define BMI323_FEAT_ENG_TIMEOUT			10000
+
+/* FIFO registers */
+#define BMI323_FIFO_FILL_LEVEL_REG		0x15
+#define BMI323_FIFO_DATA_REG			0x16
+
+/* Accelero/Gyro config registers */
+#define BMI323_ACC_CONF_REG			0x20
+#define BMI323_GYRO_CONF_REG			0x21
+#define BMI323_ACC_GYRO_CONF_MODE_MSK		GENMASK(14, 12)
+#define BMI323_ACC_GYRO_CONF_ODR_MSK		GENMASK(3, 0)
+#define BMI323_ACC_GYRO_CONF_SCL_MSK		GENMASK(6, 4)
+#define BMI323_ACC_GYRO_CONF_BW_MSK		BIT(7)
+#define BMI323_ACC_GYRO_CONF_AVG_MSK		GENMASK(10, 8)
+
+/* FIFO registers */
+#define BMI323_FIFO_WTRMRK_REG			0x35
+#define BMI323_FIFO_CONF_REG			0x36
+#define BMI323_FIFO_CONF_STP_FUL_MSK		BIT(0)
+#define BMI323_FIFO_CONF_ACC_GYR_EN_MSK		GENMASK(10, 9)
+#define BMI323_FIFO_ACC_GYR_MSK			GENMASK(1, 0)
+#define BMI323_FIFO_CTRL_REG			0x37
+#define BMI323_FIFO_FLUSH_MSK			BIT(0)
+
+/* Interrupt pin config registers */
+#define BMI323_IO_INT_CTR_REG			0x38
+#define BMI323_IO_INT1_LVL_MSK			BIT(0)
+#define BMI323_IO_INT1_OD_MSK			BIT(1)
+#define BMI323_IO_INT1_OP_EN_MSK		BIT(2)
+#define BMI323_IO_INT1_LVL_OD_OP_MSK		GENMASK(2, 0)
+#define BMI323_IO_INT2_LVL_MSK			BIT(8)
+#define BMI323_IO_INT2_OD_MSK			BIT(9)
+#define BMI323_IO_INT2_OP_EN_MSK		BIT(10)
+#define BMI323_IO_INT2_LVL_OD_OP_MSK		GENMASK(10, 8)
+#define BMI323_IO_INT_CONF_REG			0x39
+#define BMI323_IO_INT_LTCH_MSK			BIT(0)
+#define BMI323_INT_MAP1_REG			0x3A
+#define BMI323_INT_MAP2_REG			0x3B
+#define BMI323_NOMOTION_MSK			GENMASK(1, 0)
+#define BMI323_MOTION_MSK			GENMASK(3, 2)
+#define BMI323_STEP_CNT_MSK			GENMASK(11, 10)
+#define BMI323_TAP_MSK				GENMASK(1, 0)
+#define BMI323_TMP_DRDY_MSK			GENMASK(7, 6)
+#define BMI323_GYR_DRDY_MSK			GENMASK(9, 8)
+#define BMI323_ACC_DRDY_MSK			GENMASK(11, 10)
+#define BMI323_FIFO_WTRMRK_MSK			GENMASK(13, 12)
+#define BMI323_FIFO_FULL_MSK			GENMASK(15, 14)
+
+/* Feature registers */
+#define BMI323_FEAT_CTRL_REG			0x40
+#define BMI323_FEAT_ENG_EN_MSK			BIT(0)
+#define BMI323_FEAT_DATA_ADDR			0x41
+#define BMI323_FEAT_DATA_TX			0x42
+#define BMI323_FEAT_DATA_STATUS			0x43
+#define BMI323_FEAT_DATA_TX_RDY_MSK		BIT(1)
+#define BMI323_FEAT_EVNT_EXT_REG		0x47
+#define BMI323_FEAT_EVNT_EXT_S_MSK		BIT(3)
+#define BMI323_FEAT_EVNT_EXT_D_MSK		BIT(4)
+
+#define BMI323_CMD_REG				0x7E
+#define BMI323_RST_VAL				0xDEAF
+#define BMI323_CFG_RES_REG			0x7F
+
+/* Extended registers */
+#define BMI323_GEN_SET1_REG			0x02
+#define BMI323_GEN_SET1_MODE_MSK		BIT(0)
+#define BMI323_GEN_HOLD_DUR_MSK			GENMASK(4, 1)
+
+/* Any Motion/No Motion config registers */
+#define BMI323_ANYMO1_REG			0x05
+#define BMI323_NOMO1_REG			0x08
+#define BMI323_MO2_OFFSET			0x01
+#define BMI323_MO3_OFFSET			0x02
+#define BMI323_MO1_REF_UP_MSK			BIT(12)
+#define BMI323_MO1_SLOPE_TH_MSK			GENMASK(11, 0)
+#define BMI323_MO2_HYSTR_MSK			GENMASK(9, 0)
+#define BMI323_MO3_DURA_MSK			GENMASK(12, 0)
+
+/* Step counter config registers */
+#define BMI323_STEP_SC1_REG			0x10
+#define BMI323_STEP_SC1_WTRMRK_MSK		GENMASK(9, 0)
+#define BMI323_STEP_SC1_RST_CNT_MSK		BIT(10)
+#define BMI323_STEP_SC1_REG			0x10
+#define BMI323_STEP_LEN				2
+
+/* Tap gesture config registers */
+#define BMI323_TAP1_REG				0x1E
+#define BMI323_TAP1_AXIS_SEL_MSK		GENMASK(1, 0)
+#define BMI323_AXIS_XYZ_MSK			GENMASK(1, 0)
+#define BMI323_TAP1_TIMOUT_MSK			BIT(2)
+#define BMI323_TAP1_MAX_PEAKS_MSK		GENMASK(5, 3)
+#define BMI323_TAP1_MODE_MSK			GENMASK(7, 6)
+#define BMI323_TAP2_REG				0x1F
+#define BMI323_TAP2_THRES_MSK			GENMASK(9, 0)
+#define BMI323_TAP2_MAX_DUR_MSK			GENMASK(15, 10)
+#define BMI323_TAP3_REG				0x20
+#define BMI323_TAP3_QUIET_TIM_MSK		GENMASK(15, 12)
+#define BMI323_TAP3_QT_BW_TAP_MSK		GENMASK(11, 8)
+#define BMI323_TAP3_QT_AFT_GES_MSK		GENMASK(15, 12)
+
+#define BMI323_MOTION_THRES_SCALE		512
+#define BMI323_MOTION_HYSTR_SCALE		512
+#define BMI323_MOTION_DURAT_SCALE		50
+#define BMI323_TAP_THRES_SCALE			512
+#define BMI323_DUR_BW_TAP_SCALE			200
+#define BMI323_QUITE_TIM_GES_SCALE		25
+#define BMI323_MAX_GES_DUR_SCALE		25
+
+/*
+ * The formula to calculate temperature in C.
+ * See datasheet section 6.1.1, Register Map Overview
+ *
+ * T_C = (temp_raw / 512) + 23
+ */
+#define BMI323_TEMP_OFFSET			11776
+#define BMI323_TEMP_SCALE			1953125
+
+/*
+ * The BMI323 features a FIFO with a capacity of 2048 bytes. Each frame
+ * consists of accelerometer (X, Y, Z) data and gyroscope (X, Y, Z) data,
+ * totaling 6 words or 12 bytes. The FIFO buffer can hold a total of
+ * 170 frames.
+ *
+ * If a watermark interrupt is configured for 170 frames, the interrupt will
+ * trigger when the FIFO reaches 169 frames, so limit the maximum watermark
+ * level to 169 frames. In terms of data, 169 frames would equal 1014 bytes,
+ * which is approximately 2 frames before the FIFO reaches its full capacity.
+ * See datasheet section 5.7.3 FIFO Buffer Interrupts
+ */
+#define BMI323_BYTES_PER_SAMPLE			2
+#define BMI323_FIFO_LENGTH_IN_BYTES		2048
+#define BMI323_FIFO_FRAME_LENGTH		6
+#define BMI323_FIFO_FULL_IN_FRAMES		\
+	((BMI323_FIFO_LENGTH_IN_BYTES /		\
+	(BMI323_BYTES_PER_SAMPLE * BMI323_FIFO_FRAME_LENGTH)) - 1)
+#define BMI323_FIFO_FULL_IN_WORDS		\
+	(BMI323_FIFO_FULL_IN_FRAMES * BMI323_FIFO_FRAME_LENGTH)
+
+#define BMI323_INT_MICRO_TO_RAW(val, val2, scale) ((val) * (scale) + \
+						  ((val2) * (scale)) / MEGA)
+
+#define BMI323_RAW_TO_MICRO(raw, scale) ((((raw) % (scale)) * MEGA) / scale)
+
+struct device;
+int bmi323_core_probe(struct device *dev);
+extern const struct regmap_config bmi323_regmap_config;
+
+#endif
diff --git a/drivers/iio/imu/bmi323/bmi323_core.c b/drivers/iio/imu/bmi323/bmi323_core.c
new file mode 100644
index 000000000000..183af482828f
--- /dev/null
+++ b/drivers/iio/imu/bmi323/bmi323_core.c
@@ -0,0 +1,2139 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * IIO core driver for Bosch BMI323 6-Axis IMU.
+ *
+ * Copyright (C) 2023, Jagath Jog J <jagathjog1996@gmail.com>
+ *
+ * Datasheet: https://www.bosch-sensortec.com/media/boschsensortec/downloads/datasheets/bst-bmi323-ds000.pdf
+ */
+
+#include <linux/bitfield.h>
+#include <linux/cleanup.h>
+#include <linux/device.h>
+#include <linux/interrupt.h>
+#include <linux/minmax.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/units.h>
+
+#include <asm/unaligned.h>
+
+#include <linux/iio/buffer.h>
+#include <linux/iio/events.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+#include <linux/iio/trigger.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
+
+#include "bmi323.h"
+
+enum bmi323_sensor_type {
+	BMI323_ACCEL,
+	BMI323_GYRO,
+	BMI323_SENSORS_CNT,
+};
+
+enum bmi323_opr_mode {
+	ACC_GYRO_MODE_DISABLE = 0x00,
+	GYRO_DRIVE_MODE_ENABLED = 0x01,
+	ACC_GYRO_MODE_DUTYCYCLE = 0x03,
+	ACC_GYRO_MODE_CONTINOUS = 0x04,
+	ACC_GYRO_MODE_HIGH_PERF = 0x07,
+};
+
+enum bmi323_state {
+	BMI323_IDLE,
+	BMI323_BUFFER_DRDY_TRIGGERED,
+	BMI323_BUFFER_FIFO,
+};
+
+enum bmi323_irq_pin {
+	BMI323_IRQ_DISABLED,
+	BMI323_IRQ_INT1,
+	BMI323_IRQ_INT2,
+};
+
+enum bmi323_3db_bw {
+	BMI323_BW_ODR_BY_2,
+	BMI323_BW_ODR_BY_4,
+};
+
+enum bmi323_scan {
+	BMI323_ACCEL_X,
+	BMI323_ACCEL_Y,
+	BMI323_ACCEL_Z,
+	BMI323_GYRO_X,
+	BMI323_GYRO_Y,
+	BMI323_GYRO_Z,
+	BMI323_CHAN_MAX
+};
+
+struct bmi323_hw {
+	u8 data;
+	u8 config;
+	const int (*scale_table)[2];
+	int scale_table_len;
+};
+
+/*
+ * The accelerometer supports +-2G/4G/8G/16G ranges, and the resolution of
+ * each sample is 16 bits, signed.
+ * At +-8G the scale can calculated by
+ * ((8 + 8) * 9.80665 / (2^16 - 1)) * 10^6 = 2394.23819 scale in micro
+ *
+ */
+static const int bmi323_accel_scale[][2] = {
+	{ 0, 598 },
+	{ 0, 1197 },
+	{ 0, 2394 },
+	{ 0, 4788 },
+};
+
+static const int bmi323_gyro_scale[][2] = {
+	{ 0, 66 },
+	{ 0, 133 },
+	{ 0, 266 },
+	{ 0, 532 },
+	{ 0, 1065 },
+};
+
+static const int bmi323_accel_gyro_avrg[] = {0, 2, 4, 8, 16, 32, 64};
+
+static const struct bmi323_hw bmi323_hw[2] = {
+	[BMI323_ACCEL] = {
+		.data = BMI323_ACCEL_X_REG,
+		.config = BMI323_ACC_CONF_REG,
+		.scale_table = bmi323_accel_scale,
+		.scale_table_len = ARRAY_SIZE(bmi323_accel_scale),
+	},
+	[BMI323_GYRO] = {
+		.data = BMI323_GYRO_X_REG,
+		.config = BMI323_GYRO_CONF_REG,
+		.scale_table = bmi323_gyro_scale,
+		.scale_table_len = ARRAY_SIZE(bmi323_gyro_scale),
+	},
+};
+
+struct bmi323_data {
+	struct device *dev;
+	struct regmap *regmap;
+	struct iio_mount_matrix orientation;
+	enum bmi323_irq_pin irq_pin;
+	struct iio_trigger *trig;
+	bool drdy_trigger_enabled;
+	enum bmi323_state state;
+	s64 fifo_tstamp, old_fifo_tstamp;
+	u32 odrns[BMI323_SENSORS_CNT];
+	u32 odrhz[BMI323_SENSORS_CNT];
+	unsigned int feature_events;
+
+	/*
+	 * Lock to protect the members of device's private data from concurrent
+	 * access and also to serialize the access of extended registers.
+	 * See bmi323_write_ext_reg(..) for more info.
+	 */
+	struct mutex mutex;
+	int watermark;
+	__le16 fifo_buff[BMI323_FIFO_FULL_IN_WORDS] __aligned(IIO_DMA_MINALIGN);
+	struct {
+		__le16 channels[BMI323_CHAN_MAX];
+		s64 ts __aligned(8);
+	} buffer;
+	__le16 steps_count[BMI323_STEP_LEN];
+};
+
+static const struct iio_mount_matrix *
+bmi323_get_mount_matrix(const struct iio_dev *idev,
+			const struct iio_chan_spec *chan)
+{
+	struct bmi323_data *data = iio_priv(idev);
+
+	return &data->orientation;
+}
+
+static const struct iio_chan_spec_ext_info bmi323_ext_info[] = {
+	IIO_MOUNT_MATRIX(IIO_SHARED_BY_TYPE, bmi323_get_mount_matrix),
+	{ }
+};
+
+static const struct iio_event_spec bmi323_step_wtrmrk_event = {
+	.type = IIO_EV_TYPE_CHANGE,
+	.dir = IIO_EV_DIR_NONE,
+	.mask_shared_by_type = BIT(IIO_EV_INFO_ENABLE) |
+			       BIT(IIO_EV_INFO_VALUE),
+};
+
+static const struct iio_event_spec bmi323_accel_event[] = {
+	{
+		.type = IIO_EV_TYPE_MAG,
+		.dir = IIO_EV_DIR_FALLING,
+		.mask_shared_by_type = BIT(IIO_EV_INFO_VALUE) |
+				       BIT(IIO_EV_INFO_PERIOD) |
+				       BIT(IIO_EV_INFO_HYSTERESIS) |
+				       BIT(IIO_EV_INFO_ENABLE),
+	},
+	{
+		.type = IIO_EV_TYPE_MAG,
+		.dir = IIO_EV_DIR_RISING,
+		.mask_shared_by_type = BIT(IIO_EV_INFO_VALUE) |
+				       BIT(IIO_EV_INFO_PERIOD) |
+				       BIT(IIO_EV_INFO_HYSTERESIS) |
+				       BIT(IIO_EV_INFO_ENABLE),
+	},
+	{
+		.type = IIO_EV_TYPE_GESTURE,
+		.dir = IIO_EV_DIR_SINGLETAP,
+		.mask_shared_by_type = BIT(IIO_EV_INFO_ENABLE) |
+				       BIT(IIO_EV_INFO_VALUE) |
+				       BIT(IIO_EV_INFO_RESET_TIMEOUT),
+	},
+	{
+		.type = IIO_EV_TYPE_GESTURE,
+		.dir = IIO_EV_DIR_DOUBLETAP,
+		.mask_shared_by_type = BIT(IIO_EV_INFO_ENABLE) |
+				       BIT(IIO_EV_INFO_VALUE) |
+				       BIT(IIO_EV_INFO_RESET_TIMEOUT) |
+				       BIT(IIO_EV_INFO_TAP2_MIN_DELAY),
+	},
+};
+
+#define BMI323_ACCEL_CHANNEL(_type, _axis, _index) {			\
+	.type = _type,							\
+	.modified = 1,							\
+	.channel2 = IIO_MOD_##_axis,					\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),			\
+	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SAMP_FREQ) |	\
+				    BIT(IIO_CHAN_INFO_SCALE) |		\
+				    BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), \
+	.info_mask_shared_by_type_available =				\
+				    BIT(IIO_CHAN_INFO_SAMP_FREQ) |	\
+				    BIT(IIO_CHAN_INFO_SCALE) |		\
+				    BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), \
+	.scan_index = _index,						\
+	.scan_type = {							\
+		.sign = 's',						\
+		.realbits = 16,						\
+		.storagebits = 16,					\
+		.endianness = IIO_LE,					\
+	},								\
+	.ext_info = bmi323_ext_info,					\
+	.event_spec = bmi323_accel_event,				\
+	.num_event_specs = ARRAY_SIZE(bmi323_accel_event),		\
+}
+
+#define BMI323_GYRO_CHANNEL(_type, _axis, _index) {			\
+	.type = _type,							\
+	.modified = 1,							\
+	.channel2 = IIO_MOD_##_axis,					\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),			\
+	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SAMP_FREQ) |	\
+				    BIT(IIO_CHAN_INFO_SCALE) |		\
+				    BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), \
+	.info_mask_shared_by_type_available =				\
+				    BIT(IIO_CHAN_INFO_SAMP_FREQ) |	\
+				    BIT(IIO_CHAN_INFO_SCALE) |		\
+				    BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), \
+	.scan_index = _index,						\
+	.scan_type = {							\
+		.sign = 's',						\
+		.realbits = 16,						\
+		.storagebits = 16,					\
+		.endianness = IIO_LE,					\
+	},								\
+	.ext_info = bmi323_ext_info,					\
+}
+
+static const struct iio_chan_spec bmi323_channels[] = {
+	BMI323_ACCEL_CHANNEL(IIO_ACCEL, X, BMI323_ACCEL_X),
+	BMI323_ACCEL_CHANNEL(IIO_ACCEL, Y, BMI323_ACCEL_Y),
+	BMI323_ACCEL_CHANNEL(IIO_ACCEL, Z, BMI323_ACCEL_Z),
+	BMI323_GYRO_CHANNEL(IIO_ANGL_VEL, X, BMI323_GYRO_X),
+	BMI323_GYRO_CHANNEL(IIO_ANGL_VEL, Y, BMI323_GYRO_Y),
+	BMI323_GYRO_CHANNEL(IIO_ANGL_VEL, Z, BMI323_GYRO_Z),
+	{
+		.type = IIO_TEMP,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_OFFSET) |
+				      BIT(IIO_CHAN_INFO_SCALE),
+		.scan_index = -1,
+	},
+	{
+		.type = IIO_STEPS,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED) |
+				      BIT(IIO_CHAN_INFO_ENABLE),
+		.scan_index = -1,
+		.event_spec = &bmi323_step_wtrmrk_event,
+		.num_event_specs = 1,
+
+	},
+	IIO_CHAN_SOFT_TIMESTAMP(BMI323_CHAN_MAX),
+};
+
+static const int bmi323_acc_gyro_odr[][2] = {
+	{ 0, 781250 },
+	{ 1, 562500 },
+	{ 3, 125000 },
+	{ 6, 250000 },
+	{ 12, 500000 },
+	{ 25, 0 },
+	{ 50, 0 },
+	{ 100, 0 },
+	{ 200, 0 },
+	{ 400, 0 },
+	{ 800, 0 },
+};
+
+static const int bmi323_acc_gyro_odrns[] = {
+	1280 * MEGA,
+	640 * MEGA,
+	320 * MEGA,
+	160 * MEGA,
+	80 * MEGA,
+	40 * MEGA,
+	20 * MEGA,
+	10 * MEGA,
+	5 * MEGA,
+	2500 * KILO,
+	1250 * KILO,
+};
+
+static enum bmi323_sensor_type bmi323_iio_to_sensor(enum iio_chan_type iio_type)
+{
+	switch (iio_type) {
+	case IIO_ACCEL:
+		return BMI323_ACCEL;
+	case IIO_ANGL_VEL:
+		return BMI323_GYRO;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int bmi323_set_mode(struct bmi323_data *data,
+			   enum bmi323_sensor_type sensor,
+			   enum bmi323_opr_mode mode)
+{
+	guard(mutex)(&data->mutex);
+	return regmap_update_bits(data->regmap, bmi323_hw[sensor].config,
+				  BMI323_ACC_GYRO_CONF_MODE_MSK,
+				  FIELD_PREP(BMI323_ACC_GYRO_CONF_MODE_MSK,
+					     mode));
+}
+
+/*
+ * When writing data to extended register there must be no communication to
+ * any other register before write transaction is complete.
+ * See datasheet section 6.2 Extended Register Map Description.
+ */
+static int bmi323_write_ext_reg(struct bmi323_data *data, unsigned int ext_addr,
+				unsigned int ext_data)
+{
+	int ret, feature_status;
+
+	ret = regmap_read(data->regmap, BMI323_FEAT_DATA_STATUS,
+			  &feature_status);
+	if (ret)
+		return ret;
+
+	if (!FIELD_GET(BMI323_FEAT_DATA_TX_RDY_MSK, feature_status))
+		return -EBUSY;
+
+	ret = regmap_write(data->regmap, BMI323_FEAT_DATA_ADDR, ext_addr);
+	if (ret)
+		return ret;
+
+	return regmap_write(data->regmap, BMI323_FEAT_DATA_TX, ext_data);
+}
+
+/*
+ * When reading data from extended register there must be no communication to
+ * any other register before read transaction is complete.
+ * See datasheet section 6.2 Extended Register Map Description.
+ */
+static int bmi323_read_ext_reg(struct bmi323_data *data, unsigned int ext_addr,
+			       unsigned int *ext_data)
+{
+	int ret, feature_status;
+
+	ret = regmap_read(data->regmap, BMI323_FEAT_DATA_STATUS,
+			  &feature_status);
+	if (ret)
+		return ret;
+
+	if (!FIELD_GET(BMI323_FEAT_DATA_TX_RDY_MSK, feature_status))
+		return -EBUSY;
+
+	ret = regmap_write(data->regmap, BMI323_FEAT_DATA_ADDR, ext_addr);
+	if (ret)
+		return ret;
+
+	return regmap_read(data->regmap, BMI323_FEAT_DATA_TX, ext_data);
+}
+
+static int bmi323_update_ext_reg(struct bmi323_data *data,
+				 unsigned int ext_addr,
+				 unsigned int mask, unsigned int ext_data)
+{
+	unsigned int value;
+	int ret;
+
+	ret = bmi323_read_ext_reg(data, ext_addr, &value);
+	if (ret)
+		return ret;
+
+	set_mask_bits(&value, mask, ext_data);
+
+	return bmi323_write_ext_reg(data, ext_addr, value);
+}
+
+static int bmi323_get_error_status(struct bmi323_data *data)
+{
+	int error, ret;
+
+	guard(mutex)(&data->mutex);
+	ret = regmap_read(data->regmap, BMI323_ERR_REG, &error);
+	if (ret)
+		return ret;
+
+	if (error)
+		dev_err(data->dev, "Sensor error 0x%x\n", error);
+
+	return error;
+}
+
+static int bmi323_feature_engine_events(struct bmi323_data *data,
+					const unsigned int event_mask,
+					bool state)
+{
+	unsigned int value;
+	int ret;
+
+	ret = regmap_read(data->regmap, BMI323_FEAT_IO0_REG, &value);
+	if (ret)
+		return ret;
+
+	/* Register must be cleared before changing an active config */
+	ret = regmap_write(data->regmap, BMI323_FEAT_IO0_REG, 0);
+	if (ret)
+		return ret;
+
+	if (state)
+		value |= event_mask;
+	else
+		value &= ~event_mask;
+
+	ret = regmap_write(data->regmap, BMI323_FEAT_IO0_REG, value);
+	if (ret)
+		return ret;
+
+	return regmap_write(data->regmap, BMI323_FEAT_IO_STATUS_REG,
+			    BMI323_FEAT_IO_STATUS_MSK);
+}
+
+static int bmi323_step_wtrmrk_en(struct bmi323_data *data, int state)
+{
+	enum bmi323_irq_pin step_irq;
+	int ret;
+
+	guard(mutex)(&data->mutex);
+	if (!FIELD_GET(BMI323_FEAT_IO0_STP_CNT_MSK, data->feature_events))
+		return -EINVAL;
+
+	if (state)
+		step_irq = data->irq_pin;
+	else
+		step_irq = BMI323_IRQ_DISABLED;
+
+	ret = bmi323_update_ext_reg(data, BMI323_STEP_SC1_REG,
+				    BMI323_STEP_SC1_WTRMRK_MSK,
+				    FIELD_PREP(BMI323_STEP_SC1_WTRMRK_MSK,
+					       state ? 1 : 0));
+	if (ret)
+		return ret;
+
+	return regmap_update_bits(data->regmap, BMI323_INT_MAP1_REG,
+				  BMI323_STEP_CNT_MSK,
+				  FIELD_PREP(BMI323_STEP_CNT_MSK, step_irq));
+}
+
+static int bmi323_motion_config_reg(enum iio_event_direction dir)
+{
+	switch (dir) {
+	case IIO_EV_DIR_RISING:
+		return BMI323_ANYMO1_REG;
+	case IIO_EV_DIR_FALLING:
+		return BMI323_NOMO1_REG;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int bmi323_motion_event_en(struct bmi323_data *data,
+				  enum iio_event_direction dir, int state)
+{
+	unsigned int state_value = state ? BMI323_FEAT_XYZ_MSK : 0;
+	int config, ret, msk, raw, field_value;
+	enum bmi323_irq_pin motion_irq;
+	int irq_msk, irq_field_val;
+
+	if (state)
+		motion_irq = data->irq_pin;
+	else
+		motion_irq = BMI323_IRQ_DISABLED;
+
+	switch (dir) {
+	case IIO_EV_DIR_RISING:
+		msk = BMI323_FEAT_IO0_XYZ_MOTION_MSK;
+		raw = 512;
+		config = BMI323_ANYMO1_REG;
+		irq_msk = BMI323_MOTION_MSK;
+		irq_field_val = FIELD_PREP(BMI323_MOTION_MSK, motion_irq);
+		field_value = FIELD_PREP(BMI323_FEAT_IO0_XYZ_MOTION_MSK,
+					 state_value);
+		break;
+	case IIO_EV_DIR_FALLING:
+		msk = BMI323_FEAT_IO0_XYZ_NOMOTION_MSK;
+		raw = 0;
+		config = BMI323_NOMO1_REG;
+		irq_msk = BMI323_NOMOTION_MSK;
+		irq_field_val = FIELD_PREP(BMI323_NOMOTION_MSK, motion_irq);
+		field_value = FIELD_PREP(BMI323_FEAT_IO0_XYZ_NOMOTION_MSK,
+					 state_value);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	guard(mutex)(&data->mutex);
+	ret = bmi323_feature_engine_events(data, msk, state);
+	if (ret)
+		return ret;
+
+	ret = bmi323_update_ext_reg(data, config,
+				    BMI323_MO1_REF_UP_MSK,
+				    FIELD_PREP(BMI323_MO1_REF_UP_MSK, 0));
+	if (ret)
+		return ret;
+
+	/* Set initial value to avoid interrupts while enabling*/
+	ret = bmi323_update_ext_reg(data, config,
+				    BMI323_MO1_SLOPE_TH_MSK,
+				    FIELD_PREP(BMI323_MO1_SLOPE_TH_MSK, raw));
+	if (ret)
+		return ret;
+
+	ret = regmap_update_bits(data->regmap, BMI323_INT_MAP1_REG, irq_msk,
+				 irq_field_val);
+	if (ret)
+		return ret;
+
+	set_mask_bits(&data->feature_events, msk, field_value);
+
+	return 0;
+}
+
+static int bmi323_tap_event_en(struct bmi323_data *data,
+			       enum iio_event_direction dir, int state)
+{
+	enum bmi323_irq_pin tap_irq;
+	int ret, tap_enabled;
+
+	guard(mutex)(&data->mutex);
+
+	if (data->odrhz[BMI323_ACCEL] < 200) {
+		dev_err(data->dev, "Invalid accelerometer parameter\n");
+		return -EINVAL;
+	}
+
+	switch (dir) {
+	case IIO_EV_DIR_SINGLETAP:
+		ret = bmi323_feature_engine_events(data,
+						   BMI323_FEAT_IO0_S_TAP_MSK,
+						   state);
+		if (ret)
+			return ret;
+
+		set_mask_bits(&data->feature_events, BMI323_FEAT_IO0_S_TAP_MSK,
+			      FIELD_PREP(BMI323_FEAT_IO0_S_TAP_MSK, state));
+		break;
+	case IIO_EV_DIR_DOUBLETAP:
+		ret = bmi323_feature_engine_events(data,
+						   BMI323_FEAT_IO0_D_TAP_MSK,
+						   state);
+		if (ret)
+			return ret;
+
+		set_mask_bits(&data->feature_events, BMI323_FEAT_IO0_D_TAP_MSK,
+			      FIELD_PREP(BMI323_FEAT_IO0_D_TAP_MSK, state));
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	tap_enabled = FIELD_GET(BMI323_FEAT_IO0_S_TAP_MSK |
+				BMI323_FEAT_IO0_D_TAP_MSK,
+				data->feature_events);
+
+	if (tap_enabled)
+		tap_irq = data->irq_pin;
+	else
+		tap_irq = BMI323_IRQ_DISABLED;
+
+	ret = regmap_update_bits(data->regmap, BMI323_INT_MAP2_REG,
+				 BMI323_TAP_MSK,
+				 FIELD_PREP(BMI323_TAP_MSK, tap_irq));
+	if (ret)
+		return ret;
+
+	if (!state)
+		return 0;
+
+	ret = bmi323_update_ext_reg(data, BMI323_TAP1_REG,
+				    BMI323_TAP1_MAX_PEAKS_MSK,
+				    FIELD_PREP(BMI323_TAP1_MAX_PEAKS_MSK,
+					       0x04));
+	if (ret)
+		return ret;
+
+	ret = bmi323_update_ext_reg(data, BMI323_TAP1_REG,
+				    BMI323_TAP1_AXIS_SEL_MSK,
+				    FIELD_PREP(BMI323_TAP1_AXIS_SEL_MSK,
+					       BMI323_AXIS_XYZ_MSK));
+	if (ret)
+		return ret;
+
+	return bmi323_update_ext_reg(data, BMI323_TAP1_REG,
+				     BMI323_TAP1_TIMOUT_MSK,
+				     FIELD_PREP(BMI323_TAP1_TIMOUT_MSK,
+						0));
+}
+
+static ssize_t in_accel_gesture_tap_wait_dur_show(struct device *dev,
+						  struct device_attribute *attr,
+						  char *buf)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct bmi323_data *data = iio_priv(indio_dev);
+	unsigned int reg_value, raw;
+	int ret, val[2];
+
+	scoped_guard(mutex, &data->mutex) {
+		ret = bmi323_read_ext_reg(data, BMI323_TAP2_REG, &reg_value);
+		if (ret)
+			return ret;
+	}
+
+	raw = FIELD_GET(BMI323_TAP2_MAX_DUR_MSK, reg_value);
+	val[0] = raw / BMI323_MAX_GES_DUR_SCALE;
+	val[1] = BMI323_RAW_TO_MICRO(raw, BMI323_MAX_GES_DUR_SCALE);
+
+	return iio_format_value(buf, IIO_VAL_INT_PLUS_MICRO, ARRAY_SIZE(val),
+				val);
+}
+
+static ssize_t in_accel_gesture_tap_wait_dur_store(struct device *dev,
+						   struct device_attribute *attr,
+						   const char *buf, size_t len)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct bmi323_data *data = iio_priv(indio_dev);
+	int ret, val_int, val_fract, raw;
+
+	ret = iio_str_to_fixpoint(buf, 100000, &val_int, &val_fract);
+	if (ret)
+		return ret;
+
+	raw = BMI323_INT_MICRO_TO_RAW(val_int, val_fract,
+				      BMI323_MAX_GES_DUR_SCALE);
+	if (!in_range(raw, 0, 64))
+		return -EINVAL;
+
+	guard(mutex)(&data->mutex);
+	ret = bmi323_update_ext_reg(data, BMI323_TAP2_REG,
+				    BMI323_TAP2_MAX_DUR_MSK,
+				    FIELD_PREP(BMI323_TAP2_MAX_DUR_MSK, raw));
+	if (ret)
+		return ret;
+
+	return len;
+}
+
+/*
+ * Maximum duration from first tap within the second tap is expected to happen.
+ * This timeout is applicable only if gesture_tap_wait_timeout is enabled.
+ */
+static IIO_DEVICE_ATTR_RW(in_accel_gesture_tap_wait_dur, 0);
+
+static ssize_t in_accel_gesture_tap_wait_timeout_show(struct device *dev,
+						      struct device_attribute *attr,
+						      char *buf)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct bmi323_data *data = iio_priv(indio_dev);
+	unsigned int reg_value, raw;
+	int ret;
+
+	scoped_guard(mutex, &data->mutex) {
+		ret = bmi323_read_ext_reg(data, BMI323_TAP1_REG, &reg_value);
+		if (ret)
+			return ret;
+	}
+
+	raw = FIELD_GET(BMI323_TAP1_TIMOUT_MSK, reg_value);
+
+	return iio_format_value(buf, IIO_VAL_INT, 1, &raw);
+}
+
+static ssize_t in_accel_gesture_tap_wait_timeout_store(struct device *dev,
+						       struct device_attribute *attr,
+						       const char *buf,
+						       size_t len)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct bmi323_data *data = iio_priv(indio_dev);
+	bool val;
+	int ret;
+
+	ret = kstrtobool(buf, &val);
+	if (ret)
+		return ret;
+
+	guard(mutex)(&data->mutex);
+	ret = bmi323_update_ext_reg(data, BMI323_TAP1_REG,
+				    BMI323_TAP1_TIMOUT_MSK,
+				    FIELD_PREP(BMI323_TAP1_TIMOUT_MSK, val));
+	if (ret)
+		return ret;
+
+	return len;
+}
+
+/* Enable/disable gesture confirmation with wait time */
+static IIO_DEVICE_ATTR_RW(in_accel_gesture_tap_wait_timeout, 0);
+
+static IIO_CONST_ATTR(in_accel_gesture_tap_wait_dur_available,
+		      "[0.0 0.04 2.52]");
+
+static IIO_CONST_ATTR(in_accel_gesture_doubletap_tap2_min_delay_available,
+		      "[0.005 0.005 0.075]");
+
+static IIO_CONST_ATTR(in_accel_gesture_tap_reset_timeout_available,
+		      "[0.04 0.04 0.6]");
+
+static IIO_CONST_ATTR(in_accel_gesture_tap_value_available, "[0.0 0.002 1.99]");
+
+static IIO_CONST_ATTR(in_accel_mag_value_available, "[0.0 0.002 7.99]");
+
+static IIO_CONST_ATTR(in_accel_mag_period_available, "[0.0 0.02 162.0]");
+
+static IIO_CONST_ATTR(in_accel_mag_hysteresis_available, "[0.0 0.002 1.99]");
+
+static struct attribute *bmi323_event_attributes[] = {
+	&iio_const_attr_in_accel_gesture_tap_value_available.dev_attr.attr,
+	&iio_const_attr_in_accel_gesture_tap_reset_timeout_available.dev_attr.attr,
+	&iio_const_attr_in_accel_gesture_doubletap_tap2_min_delay_available.dev_attr.attr,
+	&iio_const_attr_in_accel_gesture_tap_wait_dur_available.dev_attr.attr,
+	&iio_dev_attr_in_accel_gesture_tap_wait_timeout.dev_attr.attr,
+	&iio_dev_attr_in_accel_gesture_tap_wait_dur.dev_attr.attr,
+	&iio_const_attr_in_accel_mag_value_available.dev_attr.attr,
+	&iio_const_attr_in_accel_mag_period_available.dev_attr.attr,
+	&iio_const_attr_in_accel_mag_hysteresis_available.dev_attr.attr,
+	NULL
+};
+
+static const struct attribute_group bmi323_event_attribute_group = {
+	.attrs = bmi323_event_attributes,
+};
+
+static int bmi323_write_event_config(struct iio_dev *indio_dev,
+				     const struct iio_chan_spec *chan,
+				     enum iio_event_type type,
+				     enum iio_event_direction dir, int state)
+{
+	struct bmi323_data *data = iio_priv(indio_dev);
+
+	switch (type) {
+	case IIO_EV_TYPE_MAG:
+		return bmi323_motion_event_en(data, dir, state);
+	case IIO_EV_TYPE_GESTURE:
+		return bmi323_tap_event_en(data, dir, state);
+	case IIO_EV_TYPE_CHANGE:
+		return bmi323_step_wtrmrk_en(data, state);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int bmi323_read_event_config(struct iio_dev *indio_dev,
+				    const struct iio_chan_spec *chan,
+				    enum iio_event_type type,
+				    enum iio_event_direction dir)
+{
+	struct bmi323_data *data = iio_priv(indio_dev);
+	int ret, value, reg_val;
+
+	guard(mutex)(&data->mutex);
+
+	switch (chan->type) {
+	case IIO_ACCEL:
+		switch (dir) {
+		case IIO_EV_DIR_SINGLETAP:
+			ret = FIELD_GET(BMI323_FEAT_IO0_S_TAP_MSK,
+					data->feature_events);
+			break;
+		case IIO_EV_DIR_DOUBLETAP:
+			ret = FIELD_GET(BMI323_FEAT_IO0_D_TAP_MSK,
+					data->feature_events);
+			break;
+		case IIO_EV_DIR_RISING:
+			value = FIELD_GET(BMI323_FEAT_IO0_XYZ_MOTION_MSK,
+					  data->feature_events);
+			ret = value ? 1 : 0;
+			break;
+		case IIO_EV_DIR_FALLING:
+			value = FIELD_GET(BMI323_FEAT_IO0_XYZ_NOMOTION_MSK,
+					  data->feature_events);
+			ret = value ? 1 : 0;
+			break;
+		default:
+			ret = -EINVAL;
+			break;
+		}
+		return ret;
+	case IIO_STEPS:
+		ret = regmap_read(data->regmap, BMI323_INT_MAP1_REG, &reg_val);
+		if (ret)
+			return ret;
+
+		return FIELD_GET(BMI323_STEP_CNT_MSK, reg_val) ? 1 : 0;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int bmi323_write_event_value(struct iio_dev *indio_dev,
+				    const struct iio_chan_spec *chan,
+				    enum iio_event_type type,
+				    enum iio_event_direction dir,
+				    enum iio_event_info info,
+				    int val, int val2)
+{
+	struct bmi323_data *data = iio_priv(indio_dev);
+	unsigned int raw;
+	int reg;
+
+	guard(mutex)(&data->mutex);
+
+	switch (type) {
+	case IIO_EV_TYPE_GESTURE:
+		switch (info) {
+		case IIO_EV_INFO_VALUE:
+			if (!in_range(val, 0, 2))
+				return -EINVAL;
+
+			raw = BMI323_INT_MICRO_TO_RAW(val, val2,
+						      BMI323_TAP_THRES_SCALE);
+
+			return bmi323_update_ext_reg(data, BMI323_TAP2_REG,
+						     BMI323_TAP2_THRES_MSK,
+						     FIELD_PREP(BMI323_TAP2_THRES_MSK,
+								raw));
+		case IIO_EV_INFO_RESET_TIMEOUT:
+			if (val || !in_range(val2, 40000, 560001))
+				return -EINVAL;
+
+			raw = BMI323_INT_MICRO_TO_RAW(val, val2,
+						      BMI323_QUITE_TIM_GES_SCALE);
+
+			return bmi323_update_ext_reg(data, BMI323_TAP3_REG,
+						     BMI323_TAP3_QT_AFT_GES_MSK,
+						     FIELD_PREP(BMI323_TAP3_QT_AFT_GES_MSK,
+								raw));
+		case IIO_EV_INFO_TAP2_MIN_DELAY:
+			if (val || !in_range(val2, 5000, 70001))
+				return -EINVAL;
+
+			raw = BMI323_INT_MICRO_TO_RAW(val, val2,
+						      BMI323_DUR_BW_TAP_SCALE);
+
+			return bmi323_update_ext_reg(data, BMI323_TAP3_REG,
+						     BMI323_TAP3_QT_BW_TAP_MSK,
+						     FIELD_PREP(BMI323_TAP3_QT_BW_TAP_MSK,
+								raw));
+		default:
+			return -EINVAL;
+		}
+	case IIO_EV_TYPE_MAG:
+		reg = bmi323_motion_config_reg(dir);
+		if (reg < 0)
+			return -EINVAL;
+
+		switch (info) {
+		case IIO_EV_INFO_VALUE:
+			if (!in_range(val, 0, 8))
+				return -EINVAL;
+
+			raw = BMI323_INT_MICRO_TO_RAW(val, val2,
+						      BMI323_MOTION_THRES_SCALE);
+
+			return bmi323_update_ext_reg(data, reg,
+						     BMI323_MO1_SLOPE_TH_MSK,
+						     FIELD_PREP(BMI323_MO1_SLOPE_TH_MSK,
+								raw));
+		case IIO_EV_INFO_PERIOD:
+			if (!in_range(val, 0, 163))
+				return -EINVAL;
+
+			raw = BMI323_INT_MICRO_TO_RAW(val, val2,
+						      BMI323_MOTION_DURAT_SCALE);
+
+			return bmi323_update_ext_reg(data,
+						     reg + BMI323_MO3_OFFSET,
+						     BMI323_MO3_DURA_MSK,
+						     FIELD_PREP(BMI323_MO3_DURA_MSK,
+								raw));
+		case IIO_EV_INFO_HYSTERESIS:
+			if (!in_range(val, 0, 2))
+				return -EINVAL;
+
+			raw = BMI323_INT_MICRO_TO_RAW(val, val2,
+						      BMI323_MOTION_HYSTR_SCALE);
+
+			return bmi323_update_ext_reg(data,
+						     reg + BMI323_MO2_OFFSET,
+						     BMI323_MO2_HYSTR_MSK,
+						     FIELD_PREP(BMI323_MO2_HYSTR_MSK,
+								raw));
+		default:
+			return -EINVAL;
+		}
+	case IIO_EV_TYPE_CHANGE:
+		if (!in_range(val, 0, 20461))
+			return -EINVAL;
+
+		raw = val / 20;
+		return bmi323_update_ext_reg(data, BMI323_STEP_SC1_REG,
+					     BMI323_STEP_SC1_WTRMRK_MSK,
+					     FIELD_PREP(BMI323_STEP_SC1_WTRMRK_MSK,
+							raw));
+	default:
+		return -EINVAL;
+	}
+}
+
+static int bmi323_read_event_value(struct iio_dev *indio_dev,
+				   const struct iio_chan_spec *chan,
+				   enum iio_event_type type,
+				   enum iio_event_direction dir,
+				   enum iio_event_info info,
+				   int *val, int *val2)
+{
+	struct bmi323_data *data = iio_priv(indio_dev);
+	unsigned int raw, reg_value;
+	int ret, reg;
+
+	guard(mutex)(&data->mutex);
+
+	switch (type) {
+	case IIO_EV_TYPE_GESTURE:
+		switch (info) {
+		case IIO_EV_INFO_VALUE:
+			ret = bmi323_read_ext_reg(data, BMI323_TAP2_REG,
+						  &reg_value);
+			if (ret)
+				return ret;
+
+			raw = FIELD_GET(BMI323_TAP2_THRES_MSK, reg_value);
+			*val = raw / BMI323_TAP_THRES_SCALE;
+			*val2 = BMI323_RAW_TO_MICRO(raw, BMI323_TAP_THRES_SCALE);
+			return IIO_VAL_INT_PLUS_MICRO;
+		case IIO_EV_INFO_RESET_TIMEOUT:
+			ret = bmi323_read_ext_reg(data, BMI323_TAP3_REG,
+						  &reg_value);
+			if (ret)
+				return ret;
+
+			raw = FIELD_GET(BMI323_TAP3_QT_AFT_GES_MSK, reg_value);
+			*val = 0;
+			*val2 = BMI323_RAW_TO_MICRO(raw,
+						    BMI323_QUITE_TIM_GES_SCALE);
+			return IIO_VAL_INT_PLUS_MICRO;
+		case IIO_EV_INFO_TAP2_MIN_DELAY:
+			ret = bmi323_read_ext_reg(data, BMI323_TAP3_REG,
+						  &reg_value);
+			if (ret)
+				return ret;
+
+			raw = FIELD_GET(BMI323_TAP3_QT_BW_TAP_MSK, reg_value);
+			*val = 0;
+			*val2 = BMI323_RAW_TO_MICRO(raw,
+						    BMI323_DUR_BW_TAP_SCALE);
+			return IIO_VAL_INT_PLUS_MICRO;
+		default:
+			return -EINVAL;
+		}
+	case IIO_EV_TYPE_MAG:
+		reg = bmi323_motion_config_reg(dir);
+		if (reg < 0)
+			return -EINVAL;
+
+		switch (info) {
+		case IIO_EV_INFO_VALUE:
+			ret = bmi323_read_ext_reg(data, reg, &reg_value);
+			if (ret)
+				return ret;
+
+			raw = FIELD_GET(BMI323_MO1_SLOPE_TH_MSK, reg_value);
+			*val = raw / BMI323_MOTION_THRES_SCALE;
+			*val2 = BMI323_RAW_TO_MICRO(raw,
+						    BMI323_MOTION_THRES_SCALE);
+			return IIO_VAL_INT_PLUS_MICRO;
+		case IIO_EV_INFO_PERIOD:
+			ret = bmi323_read_ext_reg(data,
+						  reg + BMI323_MO3_OFFSET,
+						  &reg_value);
+			if (ret)
+				return ret;
+
+			raw = FIELD_GET(BMI323_MO3_DURA_MSK, reg_value);
+			*val = raw / BMI323_MOTION_DURAT_SCALE;
+			*val2 = BMI323_RAW_TO_MICRO(raw,
+						    BMI323_MOTION_DURAT_SCALE);
+			return IIO_VAL_INT_PLUS_MICRO;
+		case IIO_EV_INFO_HYSTERESIS:
+			ret = bmi323_read_ext_reg(data,
+						  reg + BMI323_MO2_OFFSET,
+						  &reg_value);
+			if (ret)
+				return ret;
+
+			raw = FIELD_GET(BMI323_MO2_HYSTR_MSK, reg_value);
+			*val = raw / BMI323_MOTION_HYSTR_SCALE;
+			*val2 = BMI323_RAW_TO_MICRO(raw,
+						    BMI323_MOTION_HYSTR_SCALE);
+			return IIO_VAL_INT_PLUS_MICRO;
+		default:
+			return -EINVAL;
+		}
+	case IIO_EV_TYPE_CHANGE:
+		ret = bmi323_read_ext_reg(data, BMI323_STEP_SC1_REG,
+					  &reg_value);
+		if (ret)
+			return ret;
+
+		raw = FIELD_GET(BMI323_STEP_SC1_WTRMRK_MSK, reg_value);
+		*val = raw * 20;
+		return IIO_VAL_INT;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int __bmi323_fifo_flush(struct iio_dev *indio_dev)
+{
+	struct bmi323_data *data = iio_priv(indio_dev);
+	int i, ret, fifo_lvl, frame_count, bit, index;
+	__le16 *frame, *pchannels;
+	u64 sample_period;
+	s64 tstamp;
+
+	guard(mutex)(&data->mutex);
+	ret = regmap_read(data->regmap, BMI323_FIFO_FILL_LEVEL_REG, &fifo_lvl);
+	if (ret)
+		return ret;
+
+	fifo_lvl = min(fifo_lvl, BMI323_FIFO_FULL_IN_WORDS);
+
+	frame_count = fifo_lvl / BMI323_FIFO_FRAME_LENGTH;
+	if (!frame_count)
+		return -EINVAL;
+
+	if (fifo_lvl % BMI323_FIFO_FRAME_LENGTH)
+		dev_warn(data->dev, "Bad FIFO alignment\n");
+
+	/*
+	 * Approximate timestamps for each of the sample based on the sampling
+	 * frequency, timestamp for last sample and number of samples.
+	 */
+	if (data->old_fifo_tstamp) {
+		sample_period = data->fifo_tstamp - data->old_fifo_tstamp;
+		do_div(sample_period, frame_count);
+	} else {
+		sample_period = data->odrns[BMI323_ACCEL];
+	}
+
+	tstamp = data->fifo_tstamp - (frame_count - 1) * sample_period;
+
+	ret = regmap_noinc_read(data->regmap, BMI323_FIFO_DATA_REG,
+				&data->fifo_buff[0],
+				fifo_lvl * BMI323_BYTES_PER_SAMPLE);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < frame_count; i++) {
+		frame = &data->fifo_buff[i * BMI323_FIFO_FRAME_LENGTH];
+		pchannels = &data->buffer.channels[0];
+
+		index = 0;
+		for_each_set_bit(bit, indio_dev->active_scan_mask,
+				 BMI323_CHAN_MAX)
+			pchannels[index++] = frame[bit];
+
+		iio_push_to_buffers_with_timestamp(indio_dev, &data->buffer,
+						   tstamp);
+
+		tstamp += sample_period;
+	}
+
+	return frame_count;
+}
+
+static int bmi323_set_watermark(struct iio_dev *indio_dev, unsigned int val)
+{
+	struct bmi323_data *data = iio_priv(indio_dev);
+
+	val = min(val, (u32)BMI323_FIFO_FULL_IN_FRAMES);
+
+	guard(mutex)(&data->mutex);
+	data->watermark = val;
+
+	return 0;
+}
+
+static int bmi323_fifo_disable(struct bmi323_data *data)
+{
+	int ret;
+
+	guard(mutex)(&data->mutex);
+	ret = regmap_write(data->regmap, BMI323_FIFO_CONF_REG, 0);
+	if (ret)
+		return ret;
+
+	ret = regmap_update_bits(data->regmap, BMI323_INT_MAP2_REG,
+				 BMI323_FIFO_WTRMRK_MSK,
+				 FIELD_PREP(BMI323_FIFO_WTRMRK_MSK, 0));
+	if (ret)
+		return ret;
+
+	data->fifo_tstamp = 0;
+	data->state = BMI323_IDLE;
+
+	return 0;
+}
+
+static int bmi323_buffer_predisable(struct iio_dev *indio_dev)
+{
+	struct bmi323_data *data = iio_priv(indio_dev);
+
+	if (iio_device_get_current_mode(indio_dev) == INDIO_BUFFER_TRIGGERED)
+		return 0;
+
+	return bmi323_fifo_disable(data);
+}
+
+static int bmi323_update_watermark(struct bmi323_data *data)
+{
+	int wtrmrk;
+
+	wtrmrk = data->watermark * BMI323_FIFO_FRAME_LENGTH;
+
+	return regmap_write(data->regmap, BMI323_FIFO_WTRMRK_REG, wtrmrk);
+}
+
+static int bmi323_fifo_enable(struct bmi323_data *data)
+{
+	int ret;
+
+	guard(mutex)(&data->mutex);
+	ret = regmap_update_bits(data->regmap, BMI323_FIFO_CONF_REG,
+				 BMI323_FIFO_CONF_ACC_GYR_EN_MSK,
+				 FIELD_PREP(BMI323_FIFO_CONF_ACC_GYR_EN_MSK,
+					    BMI323_FIFO_ACC_GYR_MSK));
+	if (ret)
+		return ret;
+
+	ret = regmap_update_bits(data->regmap, BMI323_INT_MAP2_REG,
+				 BMI323_FIFO_WTRMRK_MSK,
+				 FIELD_PREP(BMI323_FIFO_WTRMRK_MSK,
+					    data->irq_pin));
+	if (ret)
+		return ret;
+
+	ret = bmi323_update_watermark(data);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(data->regmap, BMI323_FIFO_CTRL_REG,
+			   BMI323_FIFO_FLUSH_MSK);
+	if (ret)
+		return ret;
+
+	data->state = BMI323_BUFFER_FIFO;
+
+	return 0;
+}
+
+static int bmi323_buffer_preenable(struct iio_dev *indio_dev)
+{
+	struct bmi323_data *data = iio_priv(indio_dev);
+
+	guard(mutex)(&data->mutex);
+	/*
+	 * When the ODR of the accelerometer and gyroscope do not match, the
+	 * maximum ODR value between the accelerometer and gyroscope is used
+	 * for FIFO and the signal with lower ODR will insert dummy frame.
+	 * So allow buffer read only when ODR's of accelero and gyro are equal.
+	 * See datasheet section 5.7 "FIFO Data Buffering".
+	 */
+	if (data->odrns[BMI323_ACCEL] != data->odrns[BMI323_GYRO]) {
+		dev_err(data->dev, "Accelero and Gyro ODR doesn't match\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int bmi323_buffer_postenable(struct iio_dev *indio_dev)
+{
+	struct bmi323_data *data = iio_priv(indio_dev);
+
+	if (iio_device_get_current_mode(indio_dev) == INDIO_BUFFER_TRIGGERED)
+		return 0;
+
+	return bmi323_fifo_enable(data);
+}
+
+static ssize_t hwfifo_watermark_show(struct device *dev,
+				     struct device_attribute *attr, char *buf)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct bmi323_data *data = iio_priv(indio_dev);
+	int wm;
+
+	scoped_guard(mutex, &data->mutex)
+		wm = data->watermark;
+
+	return sysfs_emit(buf, "%d\n", wm);
+}
+static IIO_DEVICE_ATTR_RO(hwfifo_watermark, 0);
+
+static ssize_t hwfifo_enabled_show(struct device *dev,
+				   struct device_attribute *attr,
+				   char *buf)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct bmi323_data *data = iio_priv(indio_dev);
+	bool state;
+
+	scoped_guard(mutex, &data->mutex)
+		state = data->state == BMI323_BUFFER_FIFO;
+
+	return sysfs_emit(buf, "%d\n", state);
+}
+static IIO_DEVICE_ATTR_RO(hwfifo_enabled, 0);
+
+static const struct iio_dev_attr *bmi323_fifo_attributes[] = {
+	&iio_dev_attr_hwfifo_watermark,
+	&iio_dev_attr_hwfifo_enabled,
+	NULL
+};
+
+static const struct iio_buffer_setup_ops bmi323_buffer_ops = {
+	.preenable = bmi323_buffer_preenable,
+	.postenable = bmi323_buffer_postenable,
+	.predisable = bmi323_buffer_predisable,
+};
+
+static irqreturn_t bmi323_irq_thread_handler(int irq, void *private)
+{
+	struct iio_dev *indio_dev = private;
+	struct bmi323_data *data = iio_priv(indio_dev);
+	unsigned int status_addr, status, feature_event;
+	s64 timestamp = iio_get_time_ns(indio_dev);
+	int ret;
+
+	if (data->irq_pin == BMI323_IRQ_INT1)
+		status_addr = BMI323_STATUS_INT1_REG;
+	else
+		status_addr = BMI323_STATUS_INT2_REG;
+
+	scoped_guard(mutex, &data->mutex) {
+		ret = regmap_read(data->regmap, status_addr, &status);
+		if (ret)
+			return IRQ_NONE;
+	}
+
+	if (!status || FIELD_GET(BMI323_STATUS_ERROR_MSK, status))
+		return IRQ_NONE;
+
+	if (FIELD_GET(BMI323_STATUS_FIFO_WTRMRK_MSK, status)) {
+		data->old_fifo_tstamp = data->fifo_tstamp;
+		data->fifo_tstamp = iio_get_time_ns(indio_dev);
+		ret = __bmi323_fifo_flush(indio_dev);
+		if (ret < 0)
+			return IRQ_NONE;
+	}
+
+	if (FIELD_GET(BMI323_STATUS_ACC_GYR_DRDY_MSK, status))
+		iio_trigger_poll_nested(data->trig);
+
+	if (FIELD_GET(BMI323_STATUS_MOTION_MSK, status))
+		iio_push_event(indio_dev, IIO_MOD_EVENT_CODE(IIO_ACCEL, 0,
+							     IIO_MOD_X_OR_Y_OR_Z,
+							     IIO_EV_TYPE_MAG,
+							     IIO_EV_DIR_RISING),
+			       timestamp);
+
+	if (FIELD_GET(BMI323_STATUS_NOMOTION_MSK, status))
+		iio_push_event(indio_dev, IIO_MOD_EVENT_CODE(IIO_ACCEL, 0,
+							     IIO_MOD_X_OR_Y_OR_Z,
+							     IIO_EV_TYPE_MAG,
+							     IIO_EV_DIR_FALLING),
+			       timestamp);
+
+	if (FIELD_GET(BMI323_STATUS_STP_WTR_MSK, status))
+		iio_push_event(indio_dev, IIO_MOD_EVENT_CODE(IIO_STEPS, 0,
+							     IIO_NO_MOD,
+							     IIO_EV_TYPE_CHANGE,
+							     IIO_EV_DIR_NONE),
+			       timestamp);
+
+	if (FIELD_GET(BMI323_STATUS_TAP_MSK, status)) {
+		scoped_guard(mutex, &data->mutex) {
+			ret = regmap_read(data->regmap,
+					  BMI323_FEAT_EVNT_EXT_REG,
+					  &feature_event);
+			if (ret)
+				return IRQ_NONE;
+		}
+
+		if (FIELD_GET(BMI323_FEAT_EVNT_EXT_S_MSK, feature_event)) {
+			iio_push_event(indio_dev,
+				       IIO_MOD_EVENT_CODE(IIO_ACCEL, 0,
+							  IIO_MOD_X_OR_Y_OR_Z,
+							  IIO_EV_TYPE_GESTURE,
+							  IIO_EV_DIR_SINGLETAP),
+				       timestamp);
+		}
+
+		if (FIELD_GET(BMI323_FEAT_EVNT_EXT_D_MSK, feature_event))
+			iio_push_event(indio_dev,
+				       IIO_MOD_EVENT_CODE(IIO_ACCEL, 0,
+							  IIO_MOD_X_OR_Y_OR_Z,
+							  IIO_EV_TYPE_GESTURE,
+							  IIO_EV_DIR_DOUBLETAP),
+				       timestamp);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int bmi323_set_drdy_irq(struct bmi323_data *data,
+			       enum bmi323_irq_pin irq_pin)
+{
+	int ret;
+
+	ret = regmap_update_bits(data->regmap, BMI323_INT_MAP2_REG,
+				 BMI323_GYR_DRDY_MSK,
+				 FIELD_PREP(BMI323_GYR_DRDY_MSK, irq_pin));
+	if (ret)
+		return ret;
+
+	return regmap_update_bits(data->regmap, BMI323_INT_MAP2_REG,
+				  BMI323_ACC_DRDY_MSK,
+				  FIELD_PREP(BMI323_ACC_DRDY_MSK, irq_pin));
+}
+
+static int bmi323_data_rdy_trigger_set_state(struct iio_trigger *trig,
+					     bool state)
+{
+	struct bmi323_data *data = iio_trigger_get_drvdata(trig);
+	enum bmi323_irq_pin irq_pin;
+
+	guard(mutex)(&data->mutex);
+
+	if (data->state == BMI323_BUFFER_FIFO) {
+		dev_warn(data->dev, "Can't set trigger when FIFO enabled\n");
+		return -EBUSY;
+	}
+
+	if (state) {
+		data->state = BMI323_BUFFER_DRDY_TRIGGERED;
+		irq_pin = data->irq_pin;
+	} else {
+		data->state = BMI323_IDLE;
+		irq_pin = BMI323_IRQ_DISABLED;
+	}
+
+	return bmi323_set_drdy_irq(data, irq_pin);
+}
+
+static const struct iio_trigger_ops bmi323_trigger_ops = {
+	.set_trigger_state = &bmi323_data_rdy_trigger_set_state,
+};
+
+static irqreturn_t bmi323_trigger_handler(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct bmi323_data *data = iio_priv(indio_dev);
+	int ret, bit, index = 0;
+
+	/* Lock to protect the data->buffer */
+	guard(mutex)(&data->mutex);
+
+	if (*indio_dev->active_scan_mask == BMI323_ALL_CHAN_MSK) {
+		ret = regmap_bulk_read(data->regmap, BMI323_ACCEL_X_REG,
+				       &data->buffer.channels,
+				       ARRAY_SIZE(data->buffer.channels));
+		if (ret)
+			return IRQ_NONE;
+	} else {
+		for_each_set_bit(bit, indio_dev->active_scan_mask,
+				 BMI323_CHAN_MAX) {
+			ret = regmap_raw_read(data->regmap,
+					      BMI323_ACCEL_X_REG + bit,
+					      &data->buffer.channels[index++],
+					      BMI323_BYTES_PER_SAMPLE);
+			if (ret)
+				return IRQ_NONE;
+		}
+	}
+
+	iio_push_to_buffers_with_timestamp(indio_dev, &data->buffer,
+					   iio_get_time_ns(indio_dev));
+
+	iio_trigger_notify_done(indio_dev->trig);
+
+	return IRQ_HANDLED;
+}
+
+static int bmi323_set_average(struct bmi323_data *data,
+			      enum bmi323_sensor_type sensor, int avg)
+{
+	int raw = ARRAY_SIZE(bmi323_accel_gyro_avrg);
+
+	while (raw--)
+		if (avg == bmi323_accel_gyro_avrg[raw])
+			break;
+	if (raw < 0)
+		return -EINVAL;
+
+	guard(mutex)(&data->mutex);
+	return regmap_update_bits(data->regmap, bmi323_hw[sensor].config,
+				 BMI323_ACC_GYRO_CONF_AVG_MSK,
+				 FIELD_PREP(BMI323_ACC_GYRO_CONF_AVG_MSK,
+					    raw));
+}
+
+static int bmi323_get_average(struct bmi323_data *data,
+			      enum bmi323_sensor_type sensor, int *avg)
+{
+	int ret, value, raw;
+
+	scoped_guard(mutex, &data->mutex) {
+		ret = regmap_read(data->regmap, bmi323_hw[sensor].config, &value);
+		if (ret)
+			return ret;
+	}
+
+	raw = FIELD_GET(BMI323_ACC_GYRO_CONF_AVG_MSK, value);
+	*avg = bmi323_accel_gyro_avrg[raw];
+
+	return IIO_VAL_INT;
+}
+
+static int bmi323_enable_steps(struct bmi323_data *data, int val)
+{
+	int ret;
+
+	guard(mutex)(&data->mutex);
+	if (data->odrhz[BMI323_ACCEL] < 200) {
+		dev_err(data->dev, "Invalid accelerometer parameter\n");
+		return -EINVAL;
+	}
+
+	ret = bmi323_feature_engine_events(data, BMI323_FEAT_IO0_STP_CNT_MSK,
+					   val ? 1 : 0);
+	if (ret)
+		return ret;
+
+	set_mask_bits(&data->feature_events, BMI323_FEAT_IO0_STP_CNT_MSK,
+		      FIELD_PREP(BMI323_FEAT_IO0_STP_CNT_MSK, val ? 1 : 0));
+
+	return 0;
+}
+
+static int bmi323_read_steps(struct bmi323_data *data, int *val)
+{
+	int ret;
+
+	guard(mutex)(&data->mutex);
+	if (!FIELD_GET(BMI323_FEAT_IO0_STP_CNT_MSK, data->feature_events))
+		return -EINVAL;
+
+	ret = regmap_bulk_read(data->regmap, BMI323_FEAT_IO2_REG,
+			       data->steps_count,
+			       ARRAY_SIZE(data->steps_count));
+	if (ret)
+		return ret;
+
+	*val = get_unaligned_le32(data->steps_count);
+
+	return IIO_VAL_INT;
+}
+
+static int bmi323_read_axis(struct bmi323_data *data,
+			    struct iio_chan_spec const *chan, int *val)
+{
+	enum bmi323_sensor_type sensor;
+	unsigned int value;
+	u8 addr;
+	int ret;
+
+	ret = bmi323_get_error_status(data);
+	if (ret)
+		return -EINVAL;
+
+	sensor = bmi323_iio_to_sensor(chan->type);
+	addr = bmi323_hw[sensor].data + (chan->channel2 - IIO_MOD_X);
+
+	scoped_guard(mutex, &data->mutex) {
+		ret = regmap_read(data->regmap, addr, &value);
+		if (ret)
+			return ret;
+	}
+
+	*val = sign_extend32(value, chan->scan_type.realbits - 1);
+
+	return IIO_VAL_INT;
+}
+
+static int bmi323_get_temp_data(struct bmi323_data *data, int *val)
+{
+	unsigned int value;
+	int ret;
+
+	ret = bmi323_get_error_status(data);
+	if (ret)
+		return -EINVAL;
+
+	scoped_guard(mutex, &data->mutex) {
+		ret = regmap_read(data->regmap, BMI323_TEMP_REG, &value);
+		if (ret)
+			return ret;
+	}
+
+	*val = sign_extend32(value, 15);
+
+	return IIO_VAL_INT;
+}
+
+static int bmi323_get_odr(struct bmi323_data *data,
+			  enum bmi323_sensor_type sensor, int *odr, int *uodr)
+{
+	int ret, value, odr_raw;
+
+	scoped_guard(mutex, &data->mutex) {
+		ret = regmap_read(data->regmap, bmi323_hw[sensor].config, &value);
+		if (ret)
+			return ret;
+	}
+
+	odr_raw = FIELD_GET(BMI323_ACC_GYRO_CONF_ODR_MSK, value);
+	*odr = bmi323_acc_gyro_odr[odr_raw - 1][0];
+	*uodr = bmi323_acc_gyro_odr[odr_raw - 1][1];
+
+	return IIO_VAL_INT_PLUS_MICRO;
+}
+
+static int bmi323_configure_power_mode(struct bmi323_data *data,
+				       enum bmi323_sensor_type sensor,
+				       int odr_index)
+{
+	enum bmi323_opr_mode mode;
+
+	if (bmi323_acc_gyro_odr[odr_index][0] > 25)
+		mode = ACC_GYRO_MODE_CONTINOUS;
+	else
+		mode = ACC_GYRO_MODE_DUTYCYCLE;
+
+	return bmi323_set_mode(data, sensor, mode);
+}
+
+static int bmi323_set_odr(struct bmi323_data *data,
+			  enum bmi323_sensor_type sensor, int odr, int uodr)
+{
+	int odr_raw, ret;
+
+	odr_raw = ARRAY_SIZE(bmi323_acc_gyro_odr);
+
+	while (odr_raw--)
+		if (odr == bmi323_acc_gyro_odr[odr_raw][0] &&
+		    uodr == bmi323_acc_gyro_odr[odr_raw][1])
+			break;
+	if (odr_raw < 0)
+		return -EINVAL;
+
+	ret = bmi323_configure_power_mode(data, sensor, odr_raw);
+	if (ret)
+		return -EINVAL;
+
+	guard(mutex)(&data->mutex);
+	data->odrhz[sensor] = bmi323_acc_gyro_odr[odr_raw][0];
+	data->odrns[sensor] = bmi323_acc_gyro_odrns[odr_raw];
+
+	odr_raw++;
+
+	return regmap_update_bits(data->regmap, bmi323_hw[sensor].config,
+				  BMI323_ACC_GYRO_CONF_ODR_MSK,
+				  FIELD_PREP(BMI323_ACC_GYRO_CONF_ODR_MSK,
+					     odr_raw));
+}
+
+static int bmi323_get_scale(struct bmi323_data *data,
+			    enum bmi323_sensor_type sensor, int *val2)
+{
+	int ret, value, scale_raw;
+
+	scoped_guard(mutex, &data->mutex) {
+		ret = regmap_read(data->regmap, bmi323_hw[sensor].config,
+				  &value);
+		if (ret)
+			return ret;
+	}
+
+	scale_raw = FIELD_GET(BMI323_ACC_GYRO_CONF_SCL_MSK, value);
+	*val2 = bmi323_hw[sensor].scale_table[scale_raw][1];
+
+	return IIO_VAL_INT_PLUS_MICRO;
+}
+
+static int bmi323_set_scale(struct bmi323_data *data,
+			    enum bmi323_sensor_type sensor, int val, int val2)
+{
+	int scale_raw;
+
+	scale_raw = bmi323_hw[sensor].scale_table_len;
+
+	while (scale_raw--)
+		if (val == bmi323_hw[sensor].scale_table[scale_raw][0] &&
+		    val2 == bmi323_hw[sensor].scale_table[scale_raw][1])
+			break;
+	if (scale_raw < 0)
+		return -EINVAL;
+
+	guard(mutex)(&data->mutex);
+	return regmap_update_bits(data->regmap, bmi323_hw[sensor].config,
+				  BMI323_ACC_GYRO_CONF_SCL_MSK,
+				  FIELD_PREP(BMI323_ACC_GYRO_CONF_SCL_MSK,
+					     scale_raw));
+}
+
+static int bmi323_read_avail(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     const int **vals, int *type, int *length,
+			     long mask)
+{
+	enum bmi323_sensor_type sensor;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		*type = IIO_VAL_INT_PLUS_MICRO;
+		*vals = (const int *)bmi323_acc_gyro_odr;
+		*length = ARRAY_SIZE(bmi323_acc_gyro_odr) * 2;
+		return IIO_AVAIL_LIST;
+	case IIO_CHAN_INFO_SCALE:
+		sensor = bmi323_iio_to_sensor(chan->type);
+		*type = IIO_VAL_INT_PLUS_MICRO;
+		*vals = (const int *)bmi323_hw[sensor].scale_table;
+		*length = bmi323_hw[sensor].scale_table_len * 2;
+		return IIO_AVAIL_LIST;
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
+		*type = IIO_VAL_INT;
+		*vals = (const int *)bmi323_accel_gyro_avrg;
+		*length = ARRAY_SIZE(bmi323_accel_gyro_avrg);
+		return IIO_AVAIL_LIST;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int bmi323_write_raw(struct iio_dev *indio_dev,
+			    struct iio_chan_spec const *chan, int val,
+			    int val2, long mask)
+{
+	struct bmi323_data *data = iio_priv(indio_dev);
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		ret = iio_device_claim_direct_mode(indio_dev);
+		if (ret)
+			return ret;
+
+		ret = bmi323_set_odr(data, bmi323_iio_to_sensor(chan->type),
+				     val, val2);
+		iio_device_release_direct_mode(indio_dev);
+		return ret;
+	case IIO_CHAN_INFO_SCALE:
+		ret = iio_device_claim_direct_mode(indio_dev);
+		if (ret)
+			return ret;
+
+		ret = bmi323_set_scale(data, bmi323_iio_to_sensor(chan->type),
+				       val, val2);
+		iio_device_release_direct_mode(indio_dev);
+		return ret;
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
+		ret = iio_device_claim_direct_mode(indio_dev);
+		if (ret)
+			return ret;
+
+		ret = bmi323_set_average(data, bmi323_iio_to_sensor(chan->type),
+					 val);
+
+		iio_device_release_direct_mode(indio_dev);
+		return ret;
+	case IIO_CHAN_INFO_ENABLE:
+		return bmi323_enable_steps(data, val);
+	case IIO_CHAN_INFO_PROCESSED:
+		scoped_guard(mutex, &data->mutex) {
+			if (val || !FIELD_GET(BMI323_FEAT_IO0_STP_CNT_MSK,
+					      data->feature_events))
+				return -EINVAL;
+
+			/* Clear step counter value */
+			ret = bmi323_update_ext_reg(data, BMI323_STEP_SC1_REG,
+						    BMI323_STEP_SC1_RST_CNT_MSK,
+						    FIELD_PREP(BMI323_STEP_SC1_RST_CNT_MSK,
+							       1));
+		}
+		return ret;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int bmi323_read_raw(struct iio_dev *indio_dev,
+			   struct iio_chan_spec const *chan, int *val,
+			   int *val2, long mask)
+{
+	struct bmi323_data *data = iio_priv(indio_dev);
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_PROCESSED:
+		return bmi323_read_steps(data, val);
+	case IIO_CHAN_INFO_RAW:
+		switch (chan->type) {
+		case IIO_ACCEL:
+		case IIO_ANGL_VEL:
+			ret = iio_device_claim_direct_mode(indio_dev);
+			if (ret)
+				return ret;
+
+			ret = bmi323_read_axis(data, chan, val);
+
+			iio_device_release_direct_mode(indio_dev);
+			return ret;
+		case IIO_TEMP:
+			return bmi323_get_temp_data(data, val);
+		default:
+			return -EINVAL;
+		}
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		return bmi323_get_odr(data, bmi323_iio_to_sensor(chan->type),
+				      val, val2);
+	case IIO_CHAN_INFO_SCALE:
+		switch (chan->type) {
+		case IIO_ACCEL:
+		case IIO_ANGL_VEL:
+			*val = 0;
+			return bmi323_get_scale(data,
+						bmi323_iio_to_sensor(chan->type),
+						val2);
+		case IIO_TEMP:
+			*val = BMI323_TEMP_SCALE / MEGA;
+			*val2 = BMI323_TEMP_SCALE % MEGA;
+			return IIO_VAL_INT_PLUS_MICRO;
+		default:
+			return -EINVAL;
+		}
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
+		return bmi323_get_average(data,
+					  bmi323_iio_to_sensor(chan->type),
+					  val);
+	case IIO_CHAN_INFO_OFFSET:
+		switch (chan->type) {
+		case IIO_TEMP:
+			*val = BMI323_TEMP_OFFSET;
+			return IIO_VAL_INT;
+		default:
+			return -EINVAL;
+		}
+	case IIO_CHAN_INFO_ENABLE:
+		scoped_guard(mutex, &data->mutex)
+			*val = FIELD_GET(BMI323_FEAT_IO0_STP_CNT_MSK,
+					 data->feature_events);
+		return IIO_VAL_INT;
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info bmi323_info = {
+	.read_raw = bmi323_read_raw,
+	.write_raw = bmi323_write_raw,
+	.read_avail = bmi323_read_avail,
+	.hwfifo_set_watermark = bmi323_set_watermark,
+	.write_event_config = bmi323_write_event_config,
+	.read_event_config = bmi323_read_event_config,
+	.write_event_value = bmi323_write_event_value,
+	.read_event_value = bmi323_read_event_value,
+	.event_attrs = &bmi323_event_attribute_group,
+};
+
+#define BMI323_SCAN_MASK_ACCEL_3AXIS		\
+	(BIT(BMI323_ACCEL_X) | BIT(BMI323_ACCEL_Y) | BIT(BMI323_ACCEL_Z))
+
+#define BMI323_SCAN_MASK_GYRO_3AXIS		\
+	(BIT(BMI323_GYRO_X) | BIT(BMI323_GYRO_Y) | BIT(BMI323_GYRO_Z))
+
+static const unsigned long bmi323_avail_scan_masks[] = {
+	/* 3-axis accel */
+	BMI323_SCAN_MASK_ACCEL_3AXIS,
+	/* 3-axis gyro */
+	BMI323_SCAN_MASK_GYRO_3AXIS,
+	/* 3-axis accel + 3-axis gyro */
+	BMI323_SCAN_MASK_ACCEL_3AXIS | BMI323_SCAN_MASK_GYRO_3AXIS,
+	0
+};
+
+static int bmi323_int_pin_config(struct bmi323_data *data,
+				 enum bmi323_irq_pin irq_pin,
+				 bool active_high, bool open_drain, bool latch)
+{
+	unsigned int mask, field_value;
+	int ret;
+
+	ret = regmap_update_bits(data->regmap, BMI323_IO_INT_CONF_REG,
+				 BMI323_IO_INT_LTCH_MSK,
+				 FIELD_PREP(BMI323_IO_INT_LTCH_MSK, latch));
+	if (ret)
+		return ret;
+
+	ret = bmi323_update_ext_reg(data, BMI323_GEN_SET1_REG,
+				    BMI323_GEN_HOLD_DUR_MSK,
+				    FIELD_PREP(BMI323_GEN_HOLD_DUR_MSK, 0));
+	if (ret)
+		return ret;
+
+	switch (irq_pin) {
+	case BMI323_IRQ_INT1:
+		mask = BMI323_IO_INT1_LVL_OD_OP_MSK;
+
+		field_value = FIELD_PREP(BMI323_IO_INT1_LVL_MSK, active_high) |
+			      FIELD_PREP(BMI323_IO_INT1_OD_MSK, open_drain) |
+			      FIELD_PREP(BMI323_IO_INT1_OP_EN_MSK, 1);
+		break;
+	case BMI323_IRQ_INT2:
+		mask = BMI323_IO_INT2_LVL_OD_OP_MSK;
+
+		field_value = FIELD_PREP(BMI323_IO_INT2_LVL_MSK, active_high) |
+			      FIELD_PREP(BMI323_IO_INT2_OD_MSK, open_drain) |
+			      FIELD_PREP(BMI323_IO_INT2_OP_EN_MSK, 1);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return regmap_update_bits(data->regmap, BMI323_IO_INT_CTR_REG, mask,
+				  field_value);
+}
+
+static int bmi323_trigger_probe(struct bmi323_data *data,
+				struct iio_dev *indio_dev)
+{
+	bool open_drain, active_high, latch;
+	struct fwnode_handle *fwnode;
+	enum bmi323_irq_pin irq_pin;
+	int ret, irq, irq_type;
+	struct irq_data *desc;
+
+	fwnode = dev_fwnode(data->dev);
+	if (!fwnode)
+		return -ENODEV;
+
+	irq = fwnode_irq_get_byname(fwnode, "INT1");
+	if (irq > 0) {
+		irq_pin = BMI323_IRQ_INT1;
+	} else {
+		irq = fwnode_irq_get_byname(fwnode, "INT2");
+		if (irq < 0)
+			return 0;
+
+		irq_pin = BMI323_IRQ_INT2;
+	}
+
+	desc = irq_get_irq_data(irq);
+	if (!desc)
+		return dev_err_probe(data->dev, -EINVAL,
+				     "Could not find IRQ %d\n", irq);
+
+	irq_type = irqd_get_trigger_type(desc);
+	switch (irq_type) {
+	case IRQF_TRIGGER_RISING:
+		latch = false;
+		active_high = true;
+		break;
+	case IRQF_TRIGGER_HIGH:
+		latch = true;
+		active_high = true;
+		break;
+	case IRQF_TRIGGER_FALLING:
+		latch = false;
+		active_high = false;
+		break;
+	case IRQF_TRIGGER_LOW:
+		latch = true;
+		active_high = false;
+		break;
+	default:
+		return dev_err_probe(data->dev, -EINVAL,
+				     "Invalid interrupt type 0x%x specified\n",
+				     irq_type);
+	}
+
+	open_drain = fwnode_property_read_bool(fwnode, "drive-open-drain");
+
+	ret = bmi323_int_pin_config(data, irq_pin, active_high, open_drain,
+				    latch);
+	if (ret)
+		return dev_err_probe(data->dev, ret,
+				     "Failed to configure irq line\n");
+
+	data->trig = devm_iio_trigger_alloc(data->dev, "%s-trig-%d",
+					    indio_dev->name, irq_pin);
+	if (!data->trig)
+		return -ENOMEM;
+
+	data->trig->ops = &bmi323_trigger_ops;
+	iio_trigger_set_drvdata(data->trig, data);
+
+	ret = devm_request_threaded_irq(data->dev, irq, NULL,
+					bmi323_irq_thread_handler,
+					IRQF_ONESHOT, "bmi323-int", indio_dev);
+	if (ret)
+		return dev_err_probe(data->dev, ret, "Failed to request IRQ\n");
+
+	ret = devm_iio_trigger_register(data->dev, data->trig);
+	if (ret)
+		return dev_err_probe(data->dev, ret,
+				     "Trigger registration failed\n");
+
+	data->irq_pin = irq_pin;
+
+	return 0;
+}
+
+static int bmi323_feature_engine_enable(struct bmi323_data *data, bool en)
+{
+	unsigned int feature_status;
+	int ret;
+
+	if (!en)
+		return regmap_write(data->regmap, BMI323_FEAT_CTRL_REG, 0);
+
+	ret = regmap_write(data->regmap, BMI323_FEAT_IO2_REG, 0x012c);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(data->regmap, BMI323_FEAT_IO_STATUS_REG,
+			   BMI323_FEAT_IO_STATUS_MSK);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(data->regmap, BMI323_FEAT_CTRL_REG,
+			   BMI323_FEAT_ENG_EN_MSK);
+	if (ret)
+		return ret;
+
+	/*
+	 * It takes around 4 msec to enable the Feature engine, so check
+	 * the status of the feature engine every 2 msec for a maximum
+	 * of 5 trials.
+	 */
+	ret = regmap_read_poll_timeout(data->regmap, BMI323_FEAT_IO1_REG,
+				       feature_status,
+				       FIELD_GET(BMI323_FEAT_IO1_ERR_MSK,
+						 feature_status) == 1,
+				       BMI323_FEAT_ENG_POLL,
+				       BMI323_FEAT_ENG_TIMEOUT);
+	if (ret)
+		return dev_err_probe(data->dev, -EINVAL,
+				"Failed to enable feature engine\n");
+
+	return 0;
+}
+
+static void bmi323_disable(void *data_ptr)
+{
+	struct bmi323_data *data = data_ptr;
+
+	bmi323_set_mode(data, BMI323_ACCEL, ACC_GYRO_MODE_DISABLE);
+	bmi323_set_mode(data, BMI323_GYRO, ACC_GYRO_MODE_DISABLE);
+}
+
+static int bmi323_set_bw(struct bmi323_data *data,
+			 enum bmi323_sensor_type sensor, enum bmi323_3db_bw bw)
+{
+	return regmap_update_bits(data->regmap, bmi323_hw[sensor].config,
+				  BMI323_ACC_GYRO_CONF_BW_MSK,
+				  FIELD_PREP(BMI323_ACC_GYRO_CONF_BW_MSK, bw));
+}
+
+static int bmi323_init(struct bmi323_data *data)
+{
+	int ret, val;
+
+	/*
+	 * Perform soft reset to make sure the device is in a known state after
+	 * start up. A delay of 1.5 ms is required after reset.
+	 * See datasheet section 5.17 "Soft Reset".
+	 */
+	ret = regmap_write(data->regmap, BMI323_CMD_REG, BMI323_RST_VAL);
+	if (ret)
+		return ret;
+
+	usleep_range(1500, 2000);
+
+	/*
+	 * Dummy read is required to enable SPI interface after reset.
+	 * See datasheet section 7.2.1 "Protocol Selection".
+	 */
+	regmap_read(data->regmap, BMI323_CHIP_ID_REG, &val);
+
+	ret = regmap_read(data->regmap, BMI323_STATUS_REG, &val);
+	if (ret)
+		return ret;
+
+	if (!FIELD_GET(BMI323_STATUS_POR_MSK, val))
+		return dev_err_probe(data->dev, -EINVAL,
+				     "Sensor initialization error\n");
+
+	ret = regmap_read(data->regmap, BMI323_CHIP_ID_REG, &val);
+	if (ret)
+		return ret;
+
+	if (FIELD_GET(BMI323_CHIP_ID_MSK, val) != BMI323_CHIP_ID_VAL)
+		return dev_err_probe(data->dev, -EINVAL, "Chip ID mismatch\n");
+
+	ret = bmi323_feature_engine_enable(data, true);
+	if (ret)
+		return ret;
+
+	ret = regmap_read(data->regmap, BMI323_ERR_REG, &val);
+	if (ret)
+		return ret;
+
+	if (val)
+		return dev_err_probe(data->dev, -EINVAL,
+				     "Sensor power error = 0x%x\n", val);
+
+	/*
+	 * Set the Bandwidth coefficient which defines the 3 dB cutoff
+	 * frequency in relation to the ODR.
+	 */
+	ret = bmi323_set_bw(data, BMI323_ACCEL, BMI323_BW_ODR_BY_2);
+	if (ret)
+		return ret;
+
+	ret = bmi323_set_bw(data, BMI323_GYRO, BMI323_BW_ODR_BY_2);
+	if (ret)
+		return ret;
+
+	ret = bmi323_set_odr(data, BMI323_ACCEL, 25, 0);
+	if (ret)
+		return ret;
+
+	ret = bmi323_set_odr(data, BMI323_GYRO, 25, 0);
+	if (ret)
+		return ret;
+
+	return devm_add_action_or_reset(data->dev, bmi323_disable, data);
+}
+
+int bmi323_core_probe(struct device *dev)
+{
+	static const char * const regulator_names[] = { "vdd", "vddio" };
+	struct iio_dev *indio_dev;
+	struct bmi323_data *data;
+	struct regmap *regmap;
+	int ret;
+
+	regmap = dev_get_regmap(dev, NULL);
+	if (!regmap)
+		return dev_err_probe(dev, -ENODEV, "Failed to get regmap\n");
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*data));
+	if (!indio_dev)
+		return dev_err_probe(dev, -ENOMEM,
+				     "Failed to allocate device\n");
+
+	ret = devm_regulator_bulk_get_enable(dev, ARRAY_SIZE(regulator_names),
+					     regulator_names);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to enable regulators\n");
+
+	data = iio_priv(indio_dev);
+	data->dev = dev;
+	data->regmap = regmap;
+	mutex_init(&data->mutex);
+
+	ret = bmi323_init(data);
+	if (ret)
+		return -EINVAL;
+
+	ret = iio_read_mount_matrix(dev, &data->orientation);
+	if (ret)
+		return ret;
+
+	indio_dev->name = "bmi323-imu";
+	indio_dev->info = &bmi323_info;
+	indio_dev->channels = bmi323_channels;
+	indio_dev->num_channels = ARRAY_SIZE(bmi323_channels);
+	indio_dev->available_scan_masks = bmi323_avail_scan_masks;
+	indio_dev->modes = INDIO_DIRECT_MODE | INDIO_BUFFER_SOFTWARE;
+	dev_set_drvdata(data->dev, indio_dev);
+
+	ret = bmi323_trigger_probe(data, indio_dev);
+	if (ret)
+		return -EINVAL;
+
+	ret = devm_iio_triggered_buffer_setup_ext(data->dev, indio_dev,
+						  &iio_pollfunc_store_time,
+						  bmi323_trigger_handler,
+						  IIO_BUFFER_DIRECTION_IN,
+						  &bmi323_buffer_ops,
+						  bmi323_fifo_attributes);
+	if (ret)
+		return dev_err_probe(data->dev, ret,
+				     "Failed to setup trigger buffer\n");
+
+	ret = devm_iio_device_register(data->dev, indio_dev);
+	if (ret)
+		return dev_err_probe(data->dev, ret,
+				     "Unable to register iio device\n");
+
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(bmi323_core_probe, IIO_BMI323);
+
+MODULE_DESCRIPTION("Bosch BMI323 IMU driver");
+MODULE_AUTHOR("Jagath Jog J <jagathjog1996@gmail.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/imu/bmi323/bmi323_i2c.c b/drivers/iio/imu/bmi323/bmi323_i2c.c
new file mode 100644
index 000000000000..0008e186367d
--- /dev/null
+++ b/drivers/iio/imu/bmi323/bmi323_i2c.c
@@ -0,0 +1,121 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * I2C driver for Bosch BMI323 6-Axis IMU.
+ *
+ * Copyright (C) 2023, Jagath Jog J <jagathjog1996@gmail.com>
+ */
+
+#include <linux/i2c.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+
+#include "bmi323.h"
+
+struct bmi323_i2c_priv {
+	struct i2c_client *i2c;
+	u8 i2c_rx_buffer[BMI323_FIFO_LENGTH_IN_BYTES + BMI323_I2C_DUMMY];
+};
+
+/*
+ * From BMI323 datasheet section 4: Notes on the Serial Interface Support.
+ * Each I2C register read operation requires to read two dummy bytes before
+ * the actual payload.
+ */
+static int bmi323_regmap_i2c_read(void *context, const void *reg_buf,
+				  size_t reg_size, void *val_buf,
+				  size_t val_size)
+{
+	struct bmi323_i2c_priv *priv = context;
+	struct i2c_msg msgs[2];
+	int ret;
+
+	msgs[0].addr = priv->i2c->addr;
+	msgs[0].flags = priv->i2c->flags;
+	msgs[0].len = reg_size;
+	msgs[0].buf = (u8 *)reg_buf;
+
+	msgs[1].addr = priv->i2c->addr;
+	msgs[1].len = val_size + BMI323_I2C_DUMMY;
+	msgs[1].buf = priv->i2c_rx_buffer;
+	msgs[1].flags = priv->i2c->flags | I2C_M_RD;
+
+	ret = i2c_transfer(priv->i2c->adapter, msgs, ARRAY_SIZE(msgs));
+	if (ret < 0)
+		return -EIO;
+
+	memcpy(val_buf, priv->i2c_rx_buffer + BMI323_I2C_DUMMY, val_size);
+
+	return 0;
+}
+
+static int bmi323_regmap_i2c_write(void *context, const void *data,
+				   size_t count)
+{
+	struct bmi323_i2c_priv *priv = context;
+	u8 reg;
+
+	reg = *(u8 *)data;
+	return i2c_smbus_write_i2c_block_data(priv->i2c, reg,
+					      count - sizeof(u8),
+					      data + sizeof(u8));
+}
+
+static struct regmap_bus bmi323_regmap_bus = {
+	.read = bmi323_regmap_i2c_read,
+	.write = bmi323_regmap_i2c_write,
+};
+
+const struct regmap_config bmi323_i2c_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 16,
+	.max_register = BMI323_CFG_RES_REG,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
+};
+
+static int bmi323_i2c_probe(struct i2c_client *i2c)
+{
+	struct device *dev = &i2c->dev;
+	struct bmi323_i2c_priv *priv;
+	struct regmap *regmap;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->i2c = i2c;
+	regmap = devm_regmap_init(dev, &bmi323_regmap_bus, priv,
+				  &bmi323_i2c_regmap_config);
+	if (IS_ERR(regmap))
+		return dev_err_probe(dev, PTR_ERR(regmap),
+				     "Failed to initialize I2C Regmap\n");
+
+	return bmi323_core_probe(dev);
+}
+
+static const struct i2c_device_id bmi323_i2c_ids[] = {
+	{ "bmi323" },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, bmi323_i2c_ids);
+
+static const struct of_device_id bmi323_of_i2c_match[] = {
+	{ .compatible = "bosch,bmi323" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, bmi323_of_i2c_match);
+
+static struct i2c_driver bmi323_i2c_driver = {
+	.driver = {
+		.name = "bmi323",
+		.of_match_table = bmi323_of_i2c_match,
+	},
+	.probe = bmi323_i2c_probe,
+	.id_table = bmi323_i2c_ids,
+};
+module_i2c_driver(bmi323_i2c_driver);
+
+MODULE_DESCRIPTION("Bosch BMI323 IMU driver");
+MODULE_AUTHOR("Jagath Jog J <jagathjog1996@gmail.com>");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS(IIO_BMI323);
diff --git a/drivers/iio/imu/bmi323/bmi323_spi.c b/drivers/iio/imu/bmi323/bmi323_spi.c
new file mode 100644
index 000000000000..6dc3352dd714
--- /dev/null
+++ b/drivers/iio/imu/bmi323/bmi323_spi.c
@@ -0,0 +1,92 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * SPI driver for Bosch BMI323 6-Axis IMU.
+ *
+ * Copyright (C) 2023, Jagath Jog J <jagathjog1996@gmail.com>
+ */
+
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/spi/spi.h>
+
+#include "bmi323.h"
+
+/*
+ * From BMI323 datasheet section 4: Notes on the Serial Interface Support.
+ * Each SPI register read operation requires to read one dummy byte before
+ * the actual payload.
+ */
+static int bmi323_regmap_spi_read(void *context, const void *reg_buf,
+				  size_t reg_size, void *val_buf,
+				  size_t val_size)
+{
+	struct spi_device *spi = context;
+
+	return spi_write_then_read(spi, reg_buf, reg_size, val_buf, val_size);
+}
+
+static int bmi323_regmap_spi_write(void *context, const void *data,
+				   size_t count)
+{
+	struct spi_device *spi = context;
+	u8 *data_buff = (u8 *)data;
+
+	data_buff[1] = data_buff[0];
+	return spi_write(spi, data_buff + 1, count - 1);
+}
+
+static struct regmap_bus bmi323_regmap_bus = {
+	.read = bmi323_regmap_spi_read,
+	.write = bmi323_regmap_spi_write,
+};
+
+const struct regmap_config bmi323_spi_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 16,
+	.pad_bits = 8,
+	.read_flag_mask = BIT(7),
+	.max_register = BMI323_CFG_RES_REG,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
+};
+
+static int bmi323_spi_probe(struct spi_device *spi)
+{
+	struct device *dev = &spi->dev;
+	struct regmap *regmap;
+
+	regmap = devm_regmap_init(dev, &bmi323_regmap_bus, dev,
+				  &bmi323_spi_regmap_config);
+	if (IS_ERR(regmap))
+		return dev_err_probe(dev, PTR_ERR(regmap),
+				     "Failed to initialize SPI Regmap\n");
+
+	return bmi323_core_probe(dev);
+}
+
+static const struct spi_device_id bmi323_spi_ids[] = {
+	{ "bmi323" },
+	{ }
+};
+MODULE_DEVICE_TABLE(spi, bmi323_spi_ids);
+
+static const struct of_device_id bmi323_of_spi_match[] = {
+	{ .compatible = "bosch,bmi323" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, bmi323_of_spi_match);
+
+static struct spi_driver bmi323_spi_driver = {
+	.driver = {
+		.name = "bmi323",
+		.of_match_table = bmi323_of_spi_match,
+	},
+	.probe = bmi323_spi_probe,
+	.id_table = bmi323_spi_ids,
+};
+module_spi_driver(bmi323_spi_driver);
+
+MODULE_DESCRIPTION("Bosch BMI323 IMU driver");
+MODULE_AUTHOR("Jagath Jog J <jagathjog1996@gmail.com>");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS(IIO_BMI323);
diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c
index b1e4fde27d25..f67bd5a39beb 100644
--- a/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c
+++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_accel.c
@@ -137,10 +137,7 @@ static int inv_icm42600_accel_update_scan_mode(struct iio_dev *indio_dev,
 out_unlock:
 	mutex_unlock(&st->lock);
 	/* sleep maximum required time */
-	if (sleep_accel > sleep_temp)
-		sleep = sleep_accel;
-	else
-		sleep = sleep_temp;
+	sleep = max(sleep_accel, sleep_temp);
 	if (sleep)
 		msleep(sleep);
 	return ret;
diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c
index 6ef1df9d60b7..b52f328fd26c 100644
--- a/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c
+++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_buffer.c
@@ -424,10 +424,7 @@ static int inv_icm42600_buffer_postdisable(struct iio_dev *indio_dev)
 	mutex_unlock(&st->lock);
 
 	/* sleep maximum required time */
-	if (sleep_sensor > sleep_temp)
-		sleep = sleep_sensor;
-	else
-		sleep = sleep_temp;
+	sleep = max(sleep_sensor, sleep_temp);
 	if (sleep)
 		msleep(sleep);
 
diff --git a/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c b/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c
index 3bf946e56e1d..3df0a715e885 100644
--- a/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c
+++ b/drivers/iio/imu/inv_icm42600/inv_icm42600_gyro.c
@@ -137,10 +137,7 @@ static int inv_icm42600_gyro_update_scan_mode(struct iio_dev *indio_dev,
 out_unlock:
 	mutex_unlock(&st->lock);
 	/* sleep maximum required time */
-	if (sleep_gyro > sleep_temp)
-		sleep = sleep_gyro;
-	else
-		sleep = sleep_temp;
+	sleep = max(sleep_gyro, sleep_temp);
 	if (sleep)
 		msleep(sleep);
 	return ret;
diff --git a/drivers/iio/industrialio-buffer.c b/drivers/iio/industrialio-buffer.c
index 176d31d9f9d8..09c41e9ccf87 100644
--- a/drivers/iio/industrialio-buffer.c
+++ b/drivers/iio/industrialio-buffer.c
@@ -413,6 +413,22 @@ static const unsigned long *iio_scan_mask_match(const unsigned long *av_masks,
 {
 	if (bitmap_empty(mask, masklength))
 		return NULL;
+	/*
+	 * The condition here do not handle multi-long masks correctly.
+	 * It only checks the first long to be zero, and will use such mask
+	 * as a terminator even if there was bits set after the first long.
+	 *
+	 * Correct check would require using:
+	 * while (!bitmap_empty(av_masks, masklength))
+	 * instead. This is potentially hazardous because the
+	 * avaliable_scan_masks is a zero terminated array of longs - and
+	 * using the proper bitmap_empty() check for multi-long wide masks
+	 * would require the array to be terminated with multiple zero longs -
+	 * which is not such an usual pattern.
+	 *
+	 * As writing of this no multi-long wide masks were found in-tree, so
+	 * the simple while (*av_masks) check is working.
+	 */
 	while (*av_masks) {
 		if (strict) {
 			if (bitmap_equal(mask, av_masks, masklength))
diff --git a/drivers/iio/industrialio-core.c b/drivers/iio/industrialio-core.c
index c77745b594bd..34e1f8d0071c 100644
--- a/drivers/iio/industrialio-core.c
+++ b/drivers/iio/industrialio-core.c
@@ -1896,6 +1896,66 @@ static int iio_check_extended_name(const struct iio_dev *indio_dev)
 
 static const struct iio_buffer_setup_ops noop_ring_setup_ops;
 
+static void iio_sanity_check_avail_scan_masks(struct iio_dev *indio_dev)
+{
+	unsigned int num_masks, masklength, longs_per_mask;
+	const unsigned long *av_masks;
+	int i;
+
+	av_masks = indio_dev->available_scan_masks;
+	masklength = indio_dev->masklength;
+	longs_per_mask = BITS_TO_LONGS(masklength);
+
+	/*
+	 * The code determining how many available_scan_masks is in the array
+	 * will be assuming the end of masks when first long with all bits
+	 * zeroed is encountered. This is incorrect for masks where mask
+	 * consists of more than one long, and where some of the available masks
+	 * has long worth of bits zeroed (but has subsequent bit(s) set). This
+	 * is a safety measure against bug where array of masks is terminated by
+	 * a single zero while mask width is greater than width of a long.
+	 */
+	if (longs_per_mask > 1)
+		dev_warn(indio_dev->dev.parent,
+			 "multi long available scan masks not fully supported\n");
+
+	if (bitmap_empty(av_masks, masklength))
+		dev_warn(indio_dev->dev.parent, "empty scan mask\n");
+
+	for (num_masks = 0; *av_masks; num_masks++)
+		av_masks += longs_per_mask;
+
+	if (num_masks < 2)
+		return;
+
+	av_masks = indio_dev->available_scan_masks;
+
+	/*
+	 * Go through all the masks from first to one before the last, and see
+	 * that no mask found later from the available_scan_masks array is a
+	 * subset of mask found earlier. If this happens, then the mask found
+	 * later will never get used because scanning the array is stopped when
+	 * the first suitable mask is found. Drivers should order the array of
+	 * available masks in the order of preference (presumably the least
+	 * costy to access masks first).
+	 */
+	for (i = 0; i < num_masks - 1; i++) {
+		const unsigned long *mask1;
+		int j;
+
+		mask1 = av_masks + i * longs_per_mask;
+		for (j = i + 1; j < num_masks; j++) {
+			const unsigned long *mask2;
+
+			mask2 = av_masks + j * longs_per_mask;
+			if (bitmap_subset(mask2, mask1, masklength))
+				dev_warn(indio_dev->dev.parent,
+					 "available_scan_mask %d subset of %d. Never used\n",
+					 j, i);
+		}
+	}
+}
+
 int __iio_device_register(struct iio_dev *indio_dev, struct module *this_mod)
 {
 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
@@ -1934,6 +1994,9 @@ int __iio_device_register(struct iio_dev *indio_dev, struct module *this_mod)
 		goto error_unreg_debugfs;
 	}
 
+	if (indio_dev->available_scan_masks)
+		iio_sanity_check_avail_scan_masks(indio_dev);
+
 	ret = iio_device_register_sysfs(indio_dev);
 	if (ret) {
 		dev_err(indio_dev->dev.parent,
diff --git a/drivers/iio/light/rohm-bu27008.c b/drivers/iio/light/rohm-bu27008.c
index 6a6d77805091..0f010eff1981 100644
--- a/drivers/iio/light/rohm-bu27008.c
+++ b/drivers/iio/light/rohm-bu27008.c
@@ -130,6 +130,7 @@
  * @BU27008_BLUE:	Blue channel. Via data2 (when used).
  * @BU27008_CLEAR:	Clear channel. Via data2 or data3 (when used).
  * @BU27008_IR:		IR channel. Via data3 (when used).
+ * @BU27008_LUX:	Illuminance channel, computed using RGB and IR.
  * @BU27008_NUM_CHANS:	Number of channel types.
  */
 enum bu27008_chan_type {
@@ -138,6 +139,7 @@ enum bu27008_chan_type {
 	BU27008_BLUE,
 	BU27008_CLEAR,
 	BU27008_IR,
+	BU27008_LUX,
 	BU27008_NUM_CHANS
 };
 
@@ -172,6 +174,8 @@ static const unsigned long bu27008_scan_masks[] = {
 	ALWAYS_SCANNABLE | BIT(BU27008_CLEAR) | BIT(BU27008_IR),
 	/* buffer is R, G, B, IR */
 	ALWAYS_SCANNABLE | BIT(BU27008_BLUE) | BIT(BU27008_IR),
+	/* buffer is R, G, B, IR, LUX */
+	ALWAYS_SCANNABLE | BIT(BU27008_BLUE) | BIT(BU27008_IR) | BIT(BU27008_LUX),
 	0
 };
 
@@ -331,6 +335,19 @@ static const struct iio_chan_spec bu27008_channels[] = {
 	 * Hence we don't advertise available ones either.
 	 */
 	BU27008_CHAN(IR, DATA3, 0),
+	{
+		.type = IIO_LIGHT,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_SCALE),
+		.channel = BU27008_LUX,
+		.scan_index = BU27008_LUX,
+		.scan_type = {
+			.sign = 'u',
+			.realbits = 64,
+			.storagebits = 64,
+			.endianness = IIO_CPU,
+		},
+	},
 	IIO_CHAN_SOFT_TIMESTAMP(BU27008_NUM_CHANS),
 };
 
@@ -1004,6 +1021,169 @@ static int bu27008_read_one(struct bu27008_data *data, struct iio_dev *idev,
 	return ret;
 }
 
+#define BU27008_LUX_DATA_RED	0
+#define BU27008_LUX_DATA_GREEN	1
+#define BU27008_LUX_DATA_BLUE	2
+#define BU27008_LUX_DATA_IR	3
+#define LUX_DATA_SIZE (BU27008_NUM_HW_CHANS * sizeof(__le16))
+
+static int bu27008_read_lux_chans(struct bu27008_data *data, unsigned int time,
+				  __le16 *chan_data)
+{
+	int ret, chan_sel, tmpret, valid;
+
+	chan_sel = BU27008_BLUE2_IR3 << (ffs(data->cd->chan_sel_mask) - 1);
+
+	ret = regmap_update_bits(data->regmap, data->cd->chan_sel_reg,
+				 data->cd->chan_sel_mask, chan_sel);
+	if (ret)
+		return ret;
+
+	ret = bu27008_meas_set(data, true);
+	if (ret)
+		return ret;
+
+	msleep(time / USEC_PER_MSEC);
+
+	ret = regmap_read_poll_timeout(data->regmap, data->cd->valid_reg,
+				       valid, (valid & BU27008_MASK_VALID),
+				       BU27008_VALID_RESULT_WAIT_QUANTA_US,
+				       BU27008_MAX_VALID_RESULT_WAIT_US);
+	if (ret)
+		goto out;
+
+	ret = regmap_bulk_read(data->regmap, BU27008_REG_DATA0_LO, chan_data,
+			       LUX_DATA_SIZE);
+	if (ret)
+		goto out;
+out:
+	tmpret = bu27008_meas_set(data, false);
+	if (tmpret)
+		dev_warn(data->dev, "Stopping measurement failed\n");
+
+	return ret;
+}
+
+/*
+ * Following equation for computing lux out of register values was given by
+ * ROHM HW colleagues;
+ *
+ * Red = RedData*1024 / Gain * 20 / meas_mode
+ * Green = GreenData* 1024 / Gain * 20 / meas_mode
+ * Blue = BlueData* 1024 / Gain * 20 / meas_mode
+ * IR = IrData* 1024 / Gain * 20 / meas_mode
+ *
+ * where meas_mode is the integration time in mS / 10
+ *
+ * IRratio = (IR > 0.18 * Green) ? 0 : 1
+ *
+ * Lx = max(c1*Red + c2*Green + c3*Blue,0)
+ *
+ * for
+ * IRratio 0: c1 = -0.00002237, c2 = 0.0003219, c3 = -0.000120371
+ * IRratio 1: c1 = -0.00001074, c2 = 0.000305415, c3 = -0.000129367
+ */
+
+/*
+ * The max chan data is 0xffff. When we multiply it by 1024 * 20, we'll get
+ * 0x4FFFB000 which still fits in 32-bit integer. This won't overflow.
+ */
+#define NORM_CHAN_DATA_FOR_LX_CALC(chan, gain, time) (le16_to_cpu(chan) * \
+				   1024 * 20 / (gain) / (time))
+static u64 bu27008_calc_nlux(struct bu27008_data *data, __le16 *lux_data,
+		unsigned int gain, unsigned int gain_ir, unsigned int time)
+{
+	unsigned int red, green, blue, ir;
+	s64 c1, c2, c3, nlux;
+
+	time /= 10000;
+	ir = NORM_CHAN_DATA_FOR_LX_CALC(lux_data[BU27008_LUX_DATA_IR], gain_ir, time);
+	red = NORM_CHAN_DATA_FOR_LX_CALC(lux_data[BU27008_LUX_DATA_RED], gain, time);
+	green = NORM_CHAN_DATA_FOR_LX_CALC(lux_data[BU27008_LUX_DATA_GREEN], gain, time);
+	blue = NORM_CHAN_DATA_FOR_LX_CALC(lux_data[BU27008_LUX_DATA_BLUE], gain, time);
+
+	if ((u64)ir * 100LLU > (u64)green * 18LLU) {
+		c1 = -22370;
+		c2 = 321900;
+		c3 = -120371;
+	} else {
+		c1 = -10740;
+		c2 = 305415;
+		c3 = -129367;
+	}
+	nlux = c1 * red + c2 * green + c3 * blue;
+
+	return max_t(s64, 0, nlux);
+}
+
+static int bu27008_get_time_n_gains(struct bu27008_data *data,
+		unsigned int *gain, unsigned int *gain_ir, unsigned int *time)
+{
+	int ret;
+
+	ret = bu27008_get_gain(data, &data->gts, gain);
+	if (ret < 0)
+		return ret;
+
+	ret = bu27008_get_gain(data, &data->gts_ir, gain_ir);
+	if (ret < 0)
+		return ret;
+
+	ret = bu27008_get_int_time_us(data);
+	if (ret < 0)
+		return ret;
+
+	/* Max integration time is 400000. Fits in signed int. */
+	*time = ret;
+
+	return 0;
+}
+
+struct bu27008_buf {
+	__le16 chan[BU27008_NUM_HW_CHANS];
+	u64 lux __aligned(8);
+	s64 ts __aligned(8);
+};
+
+static int bu27008_buffer_fill_lux(struct bu27008_data *data,
+				   struct bu27008_buf *raw)
+{
+	unsigned int gain, gain_ir, time;
+	int ret;
+
+	ret = bu27008_get_time_n_gains(data, &gain, &gain_ir, &time);
+	if (ret)
+		return ret;
+
+	raw->lux = bu27008_calc_nlux(data, raw->chan, gain, gain_ir, time);
+
+	return 0;
+}
+
+static int bu27008_read_lux(struct bu27008_data *data, struct iio_dev *idev,
+			    struct iio_chan_spec const *chan,
+			    int *val, int *val2)
+{
+	__le16 lux_data[BU27008_NUM_HW_CHANS];
+	unsigned int gain, gain_ir, time;
+	u64 nlux;
+	int ret;
+
+	ret = bu27008_get_time_n_gains(data, &gain, &gain_ir, &time);
+	if (ret)
+		return ret;
+
+	ret = bu27008_read_lux_chans(data, time, lux_data);
+	if (ret)
+		return ret;
+
+	nlux = bu27008_calc_nlux(data, lux_data, gain, gain_ir, time);
+	*val = (int)nlux;
+	*val2 = nlux >> 32LLU;
+
+	return IIO_VAL_INT_64;
+}
+
 static int bu27008_read_raw(struct iio_dev *idev,
 			   struct iio_chan_spec const *chan,
 			   int *val, int *val2, long mask)
@@ -1018,7 +1198,10 @@ static int bu27008_read_raw(struct iio_dev *idev,
 			return -EBUSY;
 
 		mutex_lock(&data->mutex);
-		ret = bu27008_read_one(data, idev, chan, val, val2);
+		if (chan->type == IIO_LIGHT)
+			ret = bu27008_read_lux(data, idev, chan, val, val2);
+		else
+			ret = bu27008_read_one(data, idev, chan, val, val2);
 		mutex_unlock(&data->mutex);
 
 		iio_device_release_direct_mode(idev);
@@ -1026,6 +1209,11 @@ static int bu27008_read_raw(struct iio_dev *idev,
 		return ret;
 
 	case IIO_CHAN_INFO_SCALE:
+		if (chan->type == IIO_LIGHT) {
+			*val = 0;
+			*val2 = 1;
+			return IIO_VAL_INT_PLUS_NANO;
+		}
 		ret = bu27008_get_scale(data, chan->scan_index == BU27008_IR,
 					val, val2);
 		if (ret)
@@ -1236,10 +1424,7 @@ static irqreturn_t bu27008_trigger_handler(int irq, void *p)
 	struct iio_poll_func *pf = p;
 	struct iio_dev *idev = pf->indio_dev;
 	struct bu27008_data *data = iio_priv(idev);
-	struct {
-		__le16 chan[BU27008_NUM_HW_CHANS];
-		s64 ts __aligned(8);
-	} raw;
+	struct bu27008_buf raw;
 	int ret, dummy;
 
 	memset(&raw, 0, sizeof(raw));
@@ -1257,6 +1442,12 @@ static irqreturn_t bu27008_trigger_handler(int irq, void *p)
 	if (ret < 0)
 		goto err_read;
 
+	if (test_bit(BU27008_LUX, idev->active_scan_mask)) {
+		ret = bu27008_buffer_fill_lux(data, &raw);
+		if (ret)
+			goto err_read;
+	}
+
 	iio_push_to_buffers_with_timestamp(idev, &raw, pf->timestamp);
 err_read:
 	iio_trigger_notify_done(idev->trig);
diff --git a/drivers/iio/magnetometer/tmag5273.c b/drivers/iio/magnetometer/tmag5273.c
index c5e5c4ad681e..bcd989d65059 100644
--- a/drivers/iio/magnetometer/tmag5273.c
+++ b/drivers/iio/magnetometer/tmag5273.c
@@ -497,17 +497,13 @@ static int tmag5273_set_operating_mode(struct tmag5273_data *data,
 static void tmag5273_read_device_property(struct tmag5273_data *data)
 {
 	struct device *dev = data->dev;
-	const char *str;
 	int ret;
 
 	data->angle_measurement = TMAG5273_ANGLE_EN_X_Y;
 
-	ret = device_property_read_string(dev, "ti,angle-measurement", &str);
-	if (ret)
-		return;
-
-	ret = match_string(tmag5273_angle_names,
-			   ARRAY_SIZE(tmag5273_angle_names), str);
+	ret = device_property_match_property_string(dev, "ti,angle-measurement",
+						    tmag5273_angle_names,
+						    ARRAY_SIZE(tmag5273_angle_names));
 	if (ret >= 0)
 		data->angle_measurement = ret;
 }
diff --git a/drivers/iio/pressure/bmp280-core.c b/drivers/iio/pressure/bmp280-core.c
index a2ef1373a274..fe8734468ed3 100644
--- a/drivers/iio/pressure/bmp280-core.c
+++ b/drivers/iio/pressure/bmp280-core.c
@@ -13,6 +13,7 @@
  * https://www.bosch-sensortec.com/media/boschsensortec/downloads/datasheets/bst-bmp280-ds001.pdf
  * https://www.bosch-sensortec.com/media/boschsensortec/downloads/datasheets/bst-bme280-ds002.pdf
  * https://www.bosch-sensortec.com/media/boschsensortec/downloads/datasheets/bst-bmp388-ds001.pdf
+ * https://www.bosch-sensortec.com/media/boschsensortec/downloads/datasheets/bst-bmp390-ds002.pdf
  * https://www.bosch-sensortec.com/media/boschsensortec/downloads/datasheets/bst-bmp581-ds004.pdf
  *
  * Notice:
@@ -794,10 +795,12 @@ static int bmp280_chip_config(struct bmp280_data *data)
 }
 
 static const int bmp280_oversampling_avail[] = { 1, 2, 4, 8, 16 };
+static const u8 bmp280_chip_ids[] = { BMP280_CHIP_ID };
 
 const struct bmp280_chip_info bmp280_chip_info = {
 	.id_reg = BMP280_REG_ID,
-	.chip_id = BMP280_CHIP_ID,
+	.chip_id = bmp280_chip_ids,
+	.num_chip_id = ARRAY_SIZE(bmp280_chip_ids),
 	.regmap_config = &bmp280_regmap_config,
 	.start_up_time = 2000,
 	.channels = bmp280_channels,
@@ -846,9 +849,12 @@ static int bme280_chip_config(struct bmp280_data *data)
 	return bmp280_chip_config(data);
 }
 
+static const u8 bme280_chip_ids[] = { BME280_CHIP_ID };
+
 const struct bmp280_chip_info bme280_chip_info = {
 	.id_reg = BMP280_REG_ID,
-	.chip_id = BME280_CHIP_ID,
+	.chip_id = bme280_chip_ids,
+	.num_chip_id = ARRAY_SIZE(bme280_chip_ids),
 	.regmap_config = &bmp280_regmap_config,
 	.start_up_time = 2000,
 	.channels = bmp280_channels,
@@ -920,7 +926,7 @@ static int bmp380_cmd(struct bmp280_data *data, u8 cmd)
 }
 
 /*
- * Returns temperature in Celsius dregrees, resolution is 0.01º C. Output value of
+ * Returns temperature in Celsius degrees, resolution is 0.01º C. Output value of
  * "5123" equals 51.2º C. t_fine carries fine temperature as global value.
  *
  * Taken from datasheet, Section Appendix 9, "Compensation formula" and repo
@@ -1220,10 +1226,12 @@ static int bmp380_chip_config(struct bmp280_data *data)
 
 static const int bmp380_oversampling_avail[] = { 1, 2, 4, 8, 16, 32 };
 static const int bmp380_iir_filter_coeffs_avail[] = { 1, 2, 4, 8, 16, 32, 64, 128};
+static const u8 bmp380_chip_ids[] = { BMP380_CHIP_ID, BMP390_CHIP_ID };
 
 const struct bmp280_chip_info bmp380_chip_info = {
 	.id_reg = BMP380_REG_ID,
-	.chip_id = BMP380_CHIP_ID,
+	.chip_id = bmp380_chip_ids,
+	.num_chip_id = ARRAY_SIZE(bmp380_chip_ids),
 	.regmap_config = &bmp380_regmap_config,
 	.start_up_time = 2000,
 	.channels = bmp380_channels,
@@ -1385,7 +1393,7 @@ static int bmp580_read_temp(struct bmp280_data *data, int *val, int *val2)
 
 	/*
 	 * Temperature is returned in Celsius degrees in fractional
-	 * form down 2^16. We reescale by x1000 to return milli Celsius
+	 * form down 2^16. We rescale by x1000 to return milli Celsius
 	 * to respect IIO ABI.
 	 */
 	*val = raw_temp * 1000;
@@ -1412,7 +1420,7 @@ static int bmp580_read_press(struct bmp280_data *data, int *val, int *val2)
 	}
 	/*
 	 * Pressure is returned in Pascals in fractional form down 2^16.
-	 * We reescale /1000 to convert to kilopascal to respect IIO ABI.
+	 * We rescale /1000 to convert to kilopascal to respect IIO ABI.
 	 */
 	*val = raw_press;
 	*val2 = 64000; /* 2^6 * 1000 */
@@ -1720,10 +1728,12 @@ static int bmp580_chip_config(struct bmp280_data *data)
 }
 
 static const int bmp580_oversampling_avail[] = { 1, 2, 4, 8, 16, 32, 64, 128 };
+static const u8 bmp580_chip_ids[] = { BMP580_CHIP_ID, BMP580_CHIP_ID_ALT };
 
 const struct bmp280_chip_info bmp580_chip_info = {
 	.id_reg = BMP580_REG_CHIP_ID,
-	.chip_id = BMP580_CHIP_ID,
+	.chip_id = bmp580_chip_ids,
+	.num_chip_id = ARRAY_SIZE(bmp580_chip_ids),
 	.regmap_config = &bmp580_regmap_config,
 	.start_up_time = 2000,
 	.channels = bmp380_channels,
@@ -1983,10 +1993,12 @@ static int bmp180_chip_config(struct bmp280_data *data)
 
 static const int bmp180_oversampling_temp_avail[] = { 1 };
 static const int bmp180_oversampling_press_avail[] = { 1, 2, 4, 8 };
+static const u8 bmp180_chip_ids[] = { BMP180_CHIP_ID };
 
 const struct bmp280_chip_info bmp180_chip_info = {
 	.id_reg = BMP280_REG_ID,
-	.chip_id = BMP180_CHIP_ID,
+	.chip_id = bmp180_chip_ids,
+	.num_chip_id = ARRAY_SIZE(bmp180_chip_ids),
 	.regmap_config = &bmp180_regmap_config,
 	.start_up_time = 2000,
 	.channels = bmp280_channels,
@@ -2077,6 +2089,7 @@ int bmp280_common_probe(struct device *dev,
 	struct bmp280_data *data;
 	struct gpio_desc *gpiod;
 	unsigned int chip_id;
+	unsigned int i;
 	int ret;
 
 	indio_dev = devm_iio_device_alloc(dev, sizeof(*data));
@@ -2142,12 +2155,17 @@ int bmp280_common_probe(struct device *dev,
 	ret = regmap_read(regmap, data->chip_info->id_reg, &chip_id);
 	if (ret < 0)
 		return ret;
-	if (chip_id != data->chip_info->chip_id) {
-		dev_err(dev, "bad chip id: expected %x got %x\n",
-			data->chip_info->chip_id, chip_id);
-		return -EINVAL;
+
+	for (i = 0; i < data->chip_info->num_chip_id; i++) {
+		if (chip_id == data->chip_info->chip_id[i]) {
+			dev_info(dev, "0x%x is a known chip id for %s\n", chip_id, name);
+			break;
+		}
 	}
 
+	if (i == data->chip_info->num_chip_id)
+		dev_warn(dev, "bad chip id: 0x%x is not a known chip id\n", chip_id);
+
 	if (data->chip_info->preinit) {
 		ret = data->chip_info->preinit(data);
 		if (ret)
diff --git a/drivers/iio/pressure/bmp280-i2c.c b/drivers/iio/pressure/bmp280-i2c.c
index dbe630ad05b5..34e3bc758493 100644
--- a/drivers/iio/pressure/bmp280-i2c.c
+++ b/drivers/iio/pressure/bmp280-i2c.c
@@ -7,13 +7,11 @@
 
 static int bmp280_i2c_probe(struct i2c_client *client)
 {
-	struct regmap *regmap;
-	const struct bmp280_chip_info *chip_info;
 	const struct i2c_device_id *id = i2c_client_get_device_id(client);
+	const struct bmp280_chip_info *chip_info;
+	struct regmap *regmap;
 
-	chip_info = device_get_match_data(&client->dev);
-	if (!chip_info)
-		chip_info = (const struct bmp280_chip_info *) id->driver_data;
+	chip_info = i2c_get_match_data(client);
 
 	regmap = devm_regmap_init_i2c(client, chip_info->regmap_config);
 	if (IS_ERR(regmap)) {
diff --git a/drivers/iio/pressure/bmp280-spi.c b/drivers/iio/pressure/bmp280-spi.c
index 1dff9bb7c4e9..433d6fac83c4 100644
--- a/drivers/iio/pressure/bmp280-spi.c
+++ b/drivers/iio/pressure/bmp280-spi.c
@@ -14,8 +14,7 @@
 static int bmp280_regmap_spi_write(void *context, const void *data,
                                    size_t count)
 {
-	struct device *dev = context;
-	struct spi_device *spi = to_spi_device(dev);
+	struct spi_device *spi = to_spi_device(context);
 	u8 buf[2];
 
 	memcpy(buf, data, 2);
@@ -31,8 +30,7 @@ static int bmp280_regmap_spi_write(void *context, const void *data,
 static int bmp280_regmap_spi_read(void *context, const void *reg,
                                   size_t reg_size, void *val, size_t val_size)
 {
-	struct device *dev = context;
-	struct spi_device *spi = to_spi_device(dev);
+	struct spi_device *spi = to_spi_device(context);
 
 	return spi_write_then_read(spi, reg, reg_size, val, val_size);
 }
@@ -58,9 +56,7 @@ static int bmp280_spi_probe(struct spi_device *spi)
 		return ret;
 	}
 
-	chip_info = device_get_match_data(&spi->dev);
-	if (!chip_info)
-		chip_info = (const struct bmp280_chip_info *) id->driver_data;
+	chip_info = spi_get_device_match_data(spi);
 
 	regmap = devm_regmap_init(&spi->dev,
 				  &bmp280_regmap_bus,
diff --git a/drivers/iio/pressure/bmp280.h b/drivers/iio/pressure/bmp280.h
index 5c0563ce7572..4012387d7956 100644
--- a/drivers/iio/pressure/bmp280.h
+++ b/drivers/iio/pressure/bmp280.h
@@ -292,6 +292,7 @@
 #define BMP580_CHIP_ID_ALT		0x51
 #define BMP180_CHIP_ID			0x55
 #define BMP280_CHIP_ID			0x58
+#define BMP390_CHIP_ID			0x60
 #define BME280_CHIP_ID			0x60
 #define BMP280_SOFT_RESET_VAL		0xB6
 
@@ -410,7 +411,7 @@ struct bmp280_data {
 		__le16 bmp280_cal_buf[BMP280_CONTIGUOUS_CALIB_REGS / 2];
 		__be16 bmp180_cal_buf[BMP180_REG_CALIB_COUNT / 2];
 		u8 bmp380_cal_buf[BMP380_CALIB_REG_COUNT];
-		/* Miscellaneous, endianess-aware data buffers */
+		/* Miscellaneous, endianness-aware data buffers */
 		__le16 le16;
 		__be16 be16;
 	} __aligned(IIO_DMA_MINALIGN);
@@ -418,7 +419,8 @@ struct bmp280_data {
 
 struct bmp280_chip_info {
 	unsigned int id_reg;
-	const unsigned int chip_id;
+	const u8 *chip_id;
+	int num_chip_id;
 
 	const struct regmap_config *regmap_config;
 
diff --git a/drivers/iio/proximity/sx9324.c b/drivers/iio/proximity/sx9324.c
index 438f9c9aba6e..ac2ed2da21cc 100644
--- a/drivers/iio/proximity/sx9324.c
+++ b/drivers/iio/proximity/sx9324.c
@@ -888,7 +888,6 @@ sx9324_get_default_reg(struct device *dev, int idx,
 	char prop[] = SX9324_PROXRAW_DEF;
 	u32 start = 0, raw = 0, pos = 0;
 	int ret, count, ph, pin;
-	const char *res;
 
 	memcpy(reg_def, &sx9324_default_regs[idx], sizeof(*reg_def));
 
@@ -915,24 +914,21 @@ sx9324_get_default_reg(struct device *dev, int idx,
 		reg_def->def = raw;
 		break;
 	case SX9324_REG_AFE_CTRL0:
-		ret = device_property_read_string(dev,
-				"semtech,cs-idle-sleep", &res);
-		if (!ret)
-			ret = match_string(sx9324_csidle, ARRAY_SIZE(sx9324_csidle), res);
+		ret = device_property_match_property_string(dev, "semtech,cs-idle-sleep",
+							    sx9324_csidle,
+							    ARRAY_SIZE(sx9324_csidle));
 		if (ret >= 0) {
 			reg_def->def &= ~SX9324_REG_AFE_CTRL0_CSIDLE_MASK;
 			reg_def->def |= ret << SX9324_REG_AFE_CTRL0_CSIDLE_SHIFT;
 		}
 
-		ret = device_property_read_string(dev,
-				"semtech,int-comp-resistor", &res);
-		if (ret)
-			break;
-		ret = match_string(sx9324_rints, ARRAY_SIZE(sx9324_rints), res);
-		if (ret < 0)
-			break;
-		reg_def->def &= ~SX9324_REG_AFE_CTRL0_RINT_MASK;
-		reg_def->def |= ret << SX9324_REG_AFE_CTRL0_RINT_SHIFT;
+		ret = device_property_match_property_string(dev, "semtech,int-comp-resistor",
+							    sx9324_rints,
+							    ARRAY_SIZE(sx9324_rints));
+		if (ret >= 0) {
+			reg_def->def &= ~SX9324_REG_AFE_CTRL0_RINT_MASK;
+			reg_def->def |= ret << SX9324_REG_AFE_CTRL0_RINT_SHIFT;
+		}
 		break;
 	case SX9324_REG_AFE_CTRL4:
 	case SX9324_REG_AFE_CTRL7:
diff --git a/drivers/iio/resolver/ad2s1210.c b/drivers/iio/resolver/ad2s1210.c
index 1bd1b950e7cc..a414eef12e5e 100644
--- a/drivers/iio/resolver/ad2s1210.c
+++ b/drivers/iio/resolver/ad2s1210.c
@@ -141,7 +141,7 @@ struct ad2s1210_state {
 	struct spi_device *sdev;
 	/** GPIO pin connected to SAMPLE line. */
 	struct gpio_desc *sample_gpio;
-	/** GPIO pins connected to A0 and A1 lines. */
+	/** GPIO pins connected to A0 and A1 lines (optional). */
 	struct gpio_descs *mode_gpios;
 	/** Used to access config registers. */
 	struct regmap *regmap;
@@ -149,6 +149,8 @@ struct ad2s1210_state {
 	unsigned long clkin_hz;
 	/** Available raw hysteresis values based on resolution. */
 	int hysteresis_available[2];
+	/* adi,fixed-mode property - only valid when mode_gpios == NULL. */
+	enum ad2s1210_mode fixed_mode;
 	/** The selected resolution */
 	enum ad2s1210_resolution resolution;
 	/** Copy of fault register from the previous read. */
@@ -175,6 +177,9 @@ static int ad2s1210_set_mode(struct ad2s1210_state *st, enum ad2s1210_mode mode)
 	struct gpio_descs *gpios = st->mode_gpios;
 	DECLARE_BITMAP(bitmap, 2);
 
+	if (!gpios)
+		return mode == st->fixed_mode ? 0 : -EOPNOTSUPP;
+
 	bitmap[0] = mode;
 
 	return gpiod_set_array_value(gpios->ndescs, gpios->desc, gpios->info,
@@ -276,7 +281,8 @@ static int ad2s1210_regmap_reg_read(void *context, unsigned int reg,
 	 * parity error. The fault register is read-only and the D7 bit means
 	 * something else there.
 	 */
-	if (reg != AD2S1210_REG_FAULT && st->rx[1] & AD2S1210_ADDRESS_DATA)
+	if ((reg > AD2S1210_REG_VELOCITY_LSB && reg != AD2S1210_REG_FAULT)
+	     && st->rx[1] & AD2S1210_ADDRESS_DATA)
 		return -EBADMSG;
 
 	*val = st->rx[1];
@@ -450,21 +456,53 @@ static int ad2s1210_single_conversion(struct iio_dev *indio_dev,
 	ad2s1210_toggle_sample_line(st);
 	timestamp = iio_get_time_ns(indio_dev);
 
-	switch (chan->type) {
-	case IIO_ANGL:
-		ret = ad2s1210_set_mode(st, MOD_POS);
-		break;
-	case IIO_ANGL_VEL:
-		ret = ad2s1210_set_mode(st, MOD_VEL);
-		break;
-	default:
-		return -EINVAL;
+	if (st->fixed_mode == MOD_CONFIG) {
+		unsigned int reg_val;
+
+		switch (chan->type) {
+		case IIO_ANGL:
+			ret = regmap_bulk_read(st->regmap,
+					       AD2S1210_REG_POSITION_MSB,
+					       &st->sample.raw, 2);
+			if (ret < 0)
+				return ret;
+
+			break;
+		case IIO_ANGL_VEL:
+			ret = regmap_bulk_read(st->regmap,
+					       AD2S1210_REG_VELOCITY_MSB,
+					       &st->sample.raw, 2);
+			if (ret < 0)
+				return ret;
+
+			break;
+		default:
+			return -EINVAL;
+		}
+
+		ret = regmap_read(st->regmap, AD2S1210_REG_FAULT, &reg_val);
+		if (ret < 0)
+			return ret;
+
+		st->sample.fault = reg_val;
+	} else {
+		switch (chan->type) {
+		case IIO_ANGL:
+			ret = ad2s1210_set_mode(st, MOD_POS);
+			break;
+		case IIO_ANGL_VEL:
+			ret = ad2s1210_set_mode(st, MOD_VEL);
+			break;
+		default:
+			return -EINVAL;
+		}
+		if (ret < 0)
+			return ret;
+
+		ret = spi_read(st->sdev, &st->sample, 3);
+		if (ret < 0)
+			return ret;
 	}
-	if (ret < 0)
-		return ret;
-	ret = spi_read(st->sdev, &st->sample, 3);
-	if (ret < 0)
-		return ret;
 
 	switch (chan->type) {
 	case IIO_ANGL:
@@ -1252,27 +1290,53 @@ static irqreturn_t ad2s1210_trigger_handler(int irq, void *p)
 	ad2s1210_toggle_sample_line(st);
 
 	if (test_bit(0, indio_dev->active_scan_mask)) {
-		ret = ad2s1210_set_mode(st, MOD_POS);
-		if (ret < 0)
-			goto error_ret;
-
-		ret = spi_read(st->sdev, &st->sample, 3);
-		if (ret < 0)
-			goto error_ret;
+		if (st->fixed_mode == MOD_CONFIG) {
+			ret = regmap_bulk_read(st->regmap,
+					       AD2S1210_REG_POSITION_MSB,
+					       &st->sample.raw, 2);
+			if (ret < 0)
+				goto error_ret;
+		} else {
+			ret = ad2s1210_set_mode(st, MOD_POS);
+			if (ret < 0)
+				goto error_ret;
+
+			ret = spi_read(st->sdev, &st->sample, 3);
+			if (ret < 0)
+				goto error_ret;
+		}
 
 		memcpy(&st->scan.chan[chan++], &st->sample.raw, 2);
 	}
 
 	if (test_bit(1, indio_dev->active_scan_mask)) {
-		ret = ad2s1210_set_mode(st, MOD_VEL);
-		if (ret < 0)
-			goto error_ret;
+		if (st->fixed_mode == MOD_CONFIG) {
+			ret = regmap_bulk_read(st->regmap,
+					       AD2S1210_REG_VELOCITY_MSB,
+					       &st->sample.raw, 2);
+			if (ret < 0)
+				goto error_ret;
+		} else {
+			ret = ad2s1210_set_mode(st, MOD_VEL);
+			if (ret < 0)
+				goto error_ret;
+
+			ret = spi_read(st->sdev, &st->sample, 3);
+			if (ret < 0)
+				goto error_ret;
+		}
 
-		ret = spi_read(st->sdev, &st->sample, 3);
+		memcpy(&st->scan.chan[chan++], &st->sample.raw, 2);
+	}
+
+	if (st->fixed_mode == MOD_CONFIG) {
+		unsigned int reg_val;
+
+		ret = regmap_read(st->regmap, AD2S1210_REG_FAULT, &reg_val);
 		if (ret < 0)
-			goto error_ret;
+			return ret;
 
-		memcpy(&st->scan.chan[chan++], &st->sample.raw, 2);
+		st->sample.fault = reg_val;
 	}
 
 	ad2s1210_push_events(indio_dev, st->sample.fault, pf->timestamp);
@@ -1299,9 +1363,24 @@ static const struct iio_info ad2s1210_info = {
 static int ad2s1210_setup_properties(struct ad2s1210_state *st)
 {
 	struct device *dev = &st->sdev->dev;
+	const char *str_val;
 	u32 val;
 	int ret;
 
+	ret = device_property_read_string(dev, "adi,fixed-mode", &str_val);
+	if (ret == -EINVAL)
+		st->fixed_mode = -1;
+	else if (ret < 0)
+		return dev_err_probe(dev, ret,
+			"failed to read adi,fixed-mode property\n");
+	else {
+		if (strcmp(str_val, "config"))
+			return dev_err_probe(dev, -EINVAL,
+				"only adi,fixed-mode=\"config\" is supported\n");
+
+		st->fixed_mode = MOD_CONFIG;
+	}
+
 	ret = device_property_read_u32(dev, "assigned-resolution-bits", &val);
 	if (ret < 0)
 		return dev_err_probe(dev, ret,
@@ -1347,6 +1426,7 @@ static int ad2s1210_setup_gpios(struct ad2s1210_state *st)
 {
 	struct device *dev = &st->sdev->dev;
 	struct gpio_descs *resolution_gpios;
+	struct gpio_desc *reset_gpio;
 	DECLARE_BITMAP(bitmap, 2);
 	int ret;
 
@@ -1357,12 +1437,21 @@ static int ad2s1210_setup_gpios(struct ad2s1210_state *st)
 				     "failed to request sample GPIO\n");
 
 	/* both pins high means that we start in config mode */
-	st->mode_gpios = devm_gpiod_get_array(dev, "mode", GPIOD_OUT_HIGH);
+	st->mode_gpios = devm_gpiod_get_array_optional(dev, "mode",
+						       GPIOD_OUT_HIGH);
 	if (IS_ERR(st->mode_gpios))
 		return dev_err_probe(dev, PTR_ERR(st->mode_gpios),
 				     "failed to request mode GPIOs\n");
 
-	if (st->mode_gpios->ndescs != 2)
+	if (!st->mode_gpios && st->fixed_mode == -1)
+		return dev_err_probe(dev, -EINVAL,
+			"must specify either adi,fixed-mode or mode-gpios\n");
+
+	if (st->mode_gpios && st->fixed_mode != -1)
+		return dev_err_probe(dev, -EINVAL,
+			"must specify only one of adi,fixed-mode or mode-gpios\n");
+
+	if (st->mode_gpios && st->mode_gpios->ndescs != 2)
 		return dev_err_probe(dev, -EINVAL,
 				     "requires exactly 2 mode-gpios\n");
 
@@ -1393,6 +1482,17 @@ static int ad2s1210_setup_gpios(struct ad2s1210_state *st)
 					     "failed to set resolution gpios\n");
 	}
 
+	/* If the optional reset GPIO is present, toggle it to do a hard reset. */
+	reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(reset_gpio),
+				     "failed to request reset GPIO\n");
+
+	if (reset_gpio) {
+		udelay(10);
+		gpiod_set_value(reset_gpio, 0);
+	}
+
 	return 0;
 }
 
diff --git a/drivers/iio/temperature/Kconfig b/drivers/iio/temperature/Kconfig
index ed384f33e0c7..ea2ce364b2e9 100644
--- a/drivers/iio/temperature/Kconfig
+++ b/drivers/iio/temperature/Kconfig
@@ -158,4 +158,14 @@ config MAX31865
 	  This driver can also be build as a module. If so, the module
 	  will be called max31865.
 
+config MCP9600
+	tristate "MCP9600 thermocouple EMF converter"
+	depends on I2C
+	help
+	  If you say yes here you get support for MCP9600
+	  thermocouple EMF converter connected via I2C.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called mcp9600.
+
 endmenu
diff --git a/drivers/iio/temperature/Makefile b/drivers/iio/temperature/Makefile
index dfec8c6d3019..9330d4a39598 100644
--- a/drivers/iio/temperature/Makefile
+++ b/drivers/iio/temperature/Makefile
@@ -10,6 +10,7 @@ obj-$(CONFIG_MAXIM_THERMOCOUPLE) += maxim_thermocouple.o
 obj-$(CONFIG_MAX30208) += max30208.o
 obj-$(CONFIG_MAX31856) += max31856.o
 obj-$(CONFIG_MAX31865) += max31865.o
+obj-$(CONFIG_MCP9600) += mcp9600.o
 obj-$(CONFIG_MLX90614) += mlx90614.o
 obj-$(CONFIG_MLX90632) += mlx90632.o
 obj-$(CONFIG_TMP006) += tmp006.o
diff --git a/drivers/iio/temperature/mcp9600.c b/drivers/iio/temperature/mcp9600.c
new file mode 100644
index 000000000000..46845804292b
--- /dev/null
+++ b/drivers/iio/temperature/mcp9600.c
@@ -0,0 +1,139 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * mcp9600.c - Support for Microchip MCP9600 thermocouple EMF converter
+ *
+ * Copyright (c) 2022 Andrew Hepp
+ * Author: <andrew.hepp@ahepp.dev>
+ */
+
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+
+#include <linux/iio/iio.h>
+
+/* MCP9600 registers */
+#define MCP9600_HOT_JUNCTION 0x0
+#define MCP9600_COLD_JUNCTION 0x2
+#define MCP9600_DEVICE_ID 0x20
+
+/* MCP9600 device id value */
+#define MCP9600_DEVICE_ID_MCP9600 0x40
+
+static const struct iio_chan_spec mcp9600_channels[] = {
+	{
+		.type = IIO_TEMP,
+		.address = MCP9600_HOT_JUNCTION,
+		.info_mask_separate =
+			BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE),
+	},
+	{
+		.type = IIO_TEMP,
+		.address = MCP9600_COLD_JUNCTION,
+		.channel2 = IIO_MOD_TEMP_AMBIENT,
+		.modified = 1,
+		.info_mask_separate =
+			BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE),
+	},
+};
+
+struct mcp9600_data {
+	struct i2c_client *client;
+};
+
+static int mcp9600_read(struct mcp9600_data *data,
+			struct iio_chan_spec const *chan, int *val)
+{
+	int ret;
+
+	ret = i2c_smbus_read_word_swapped(data->client, chan->address);
+
+	if (ret < 0)
+		return ret;
+	*val = ret;
+
+	return 0;
+}
+
+static int mcp9600_read_raw(struct iio_dev *indio_dev,
+			    struct iio_chan_spec const *chan, int *val,
+			    int *val2, long mask)
+{
+	struct mcp9600_data *data = iio_priv(indio_dev);
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = mcp9600_read(data, chan, val);
+		if (ret)
+			return ret;
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		*val = 62;
+		*val2 = 500000;
+		return IIO_VAL_INT_PLUS_MICRO;
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info mcp9600_info = {
+	.read_raw = mcp9600_read_raw,
+};
+
+static int mcp9600_probe(struct i2c_client *client)
+{
+	struct iio_dev *indio_dev;
+	struct mcp9600_data *data;
+	int ret;
+
+	ret = i2c_smbus_read_byte_data(client, MCP9600_DEVICE_ID);
+	if (ret < 0)
+		return dev_err_probe(&client->dev, ret, "Failed to read device ID\n");
+	if (ret != MCP9600_DEVICE_ID_MCP9600)
+		dev_warn(&client->dev, "Expected ID %x, got %x\n",
+				MCP9600_DEVICE_ID_MCP9600, ret);
+
+	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	data = iio_priv(indio_dev);
+	data->client = client;
+
+	indio_dev->info = &mcp9600_info;
+	indio_dev->name = "mcp9600";
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = mcp9600_channels;
+	indio_dev->num_channels = ARRAY_SIZE(mcp9600_channels);
+
+	return devm_iio_device_register(&client->dev, indio_dev);
+}
+
+static const struct i2c_device_id mcp9600_id[] = {
+	{ "mcp9600" },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, mcp9600_id);
+
+static const struct of_device_id mcp9600_of_match[] = {
+	{ .compatible = "microchip,mcp9600" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, mcp9600_of_match);
+
+static struct i2c_driver mcp9600_driver = {
+	.driver = {
+		.name = "mcp9600",
+		.of_match_table = mcp9600_of_match,
+	},
+	.probe = mcp9600_probe,
+	.id_table = mcp9600_id
+};
+module_i2c_driver(mcp9600_driver);
+
+MODULE_AUTHOR("Andrew Hepp <andrew.hepp@ahepp.dev>");
+MODULE_DESCRIPTION("Microchip MCP9600 thermocouple EMF converter driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 8a6da87f464b..94a7f3b0c71c 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -1971,7 +1971,7 @@ int ib_get_eth_speed(struct ib_device *dev, u32 port_num, u16 *speed, u8 *width)
 	int rc;
 	u32 netdev_speed;
 	struct net_device *netdev;
-	struct ethtool_link_ksettings lksettings;
+	struct ethtool_link_ksettings lksettings = {};
 
 	if (rdma_port_get_link_layer(dev, port_num) != IB_LINK_LAYER_ETHERNET)
 		return -EINVAL;
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index faa88d12ee86..b2467de721dc 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -1184,7 +1184,8 @@ static struct bnxt_re_qp *bnxt_re_create_shadow_qp
 }
 
 static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp,
-				struct ib_qp_init_attr *init_attr)
+				struct ib_qp_init_attr *init_attr,
+				struct bnxt_re_ucontext *uctx)
 {
 	struct bnxt_qplib_dev_attr *dev_attr;
 	struct bnxt_qplib_qp *qplqp;
@@ -1213,7 +1214,7 @@ static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp,
 		/* Allocate 1 more than what's provided so posting max doesn't
 		 * mean empty.
 		 */
-		entries = roundup_pow_of_two(init_attr->cap.max_recv_wr + 1);
+		entries = bnxt_re_init_depth(init_attr->cap.max_recv_wr + 1, uctx);
 		rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1);
 		rq->q_full_delta = 0;
 		rq->sg_info.pgsize = PAGE_SIZE;
@@ -1243,7 +1244,7 @@ static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp)
 
 static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
 				struct ib_qp_init_attr *init_attr,
-				struct ib_udata *udata)
+				struct bnxt_re_ucontext *uctx)
 {
 	struct bnxt_qplib_dev_attr *dev_attr;
 	struct bnxt_qplib_qp *qplqp;
@@ -1272,7 +1273,7 @@ static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
 	/* Allocate 128 + 1 more than what's provided */
 	diff = (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) ?
 		0 : BNXT_QPLIB_RESERVED_QP_WRS;
-	entries = roundup_pow_of_two(entries + diff + 1);
+	entries = bnxt_re_init_depth(entries + diff + 1, uctx);
 	sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1);
 	sq->q_full_delta = diff + 1;
 	/*
@@ -1288,7 +1289,8 @@ static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
 }
 
 static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp,
-				       struct ib_qp_init_attr *init_attr)
+				       struct ib_qp_init_attr *init_attr,
+				       struct bnxt_re_ucontext *uctx)
 {
 	struct bnxt_qplib_dev_attr *dev_attr;
 	struct bnxt_qplib_qp *qplqp;
@@ -1300,7 +1302,7 @@ static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp,
 	dev_attr = &rdev->dev_attr;
 
 	if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) {
-		entries = roundup_pow_of_two(init_attr->cap.max_send_wr + 1);
+		entries = bnxt_re_init_depth(init_attr->cap.max_send_wr + 1, uctx);
 		qplqp->sq.max_wqe = min_t(u32, entries,
 					  dev_attr->max_qp_wqes + 1);
 		qplqp->sq.q_full_delta = qplqp->sq.max_wqe -
@@ -1338,6 +1340,7 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
 				struct ib_udata *udata)
 {
 	struct bnxt_qplib_dev_attr *dev_attr;
+	struct bnxt_re_ucontext *uctx;
 	struct bnxt_qplib_qp *qplqp;
 	struct bnxt_re_dev *rdev;
 	struct bnxt_re_cq *cq;
@@ -1347,6 +1350,7 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
 	qplqp = &qp->qplib_qp;
 	dev_attr = &rdev->dev_attr;
 
+	uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
 	/* Setup misc params */
 	ether_addr_copy(qplqp->smac, rdev->netdev->dev_addr);
 	qplqp->pd = &pd->qplib_pd;
@@ -1388,18 +1392,18 @@ static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
 	}
 
 	/* Setup RQ/SRQ */
-	rc = bnxt_re_init_rq_attr(qp, init_attr);
+	rc = bnxt_re_init_rq_attr(qp, init_attr, uctx);
 	if (rc)
 		goto out;
 	if (init_attr->qp_type == IB_QPT_GSI)
 		bnxt_re_adjust_gsi_rq_attr(qp);
 
 	/* Setup SQ */
-	rc = bnxt_re_init_sq_attr(qp, init_attr, udata);
+	rc = bnxt_re_init_sq_attr(qp, init_attr, uctx);
 	if (rc)
 		goto out;
 	if (init_attr->qp_type == IB_QPT_GSI)
-		bnxt_re_adjust_gsi_sq_attr(qp, init_attr);
+		bnxt_re_adjust_gsi_sq_attr(qp, init_attr, uctx);
 
 	if (udata) /* This will update DPI and qp_handle */
 		rc = bnxt_re_init_user_qp(rdev, pd, qp, udata);
@@ -1715,6 +1719,7 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq,
 {
 	struct bnxt_qplib_dev_attr *dev_attr;
 	struct bnxt_qplib_nq *nq = NULL;
+	struct bnxt_re_ucontext *uctx;
 	struct bnxt_re_dev *rdev;
 	struct bnxt_re_srq *srq;
 	struct bnxt_re_pd *pd;
@@ -1739,13 +1744,14 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq,
 		goto exit;
 	}
 
+	uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
 	srq->rdev = rdev;
 	srq->qplib_srq.pd = &pd->qplib_pd;
 	srq->qplib_srq.dpi = &rdev->dpi_privileged;
 	/* Allocate 1 more than what's provided so posting max doesn't
 	 * mean empty
 	 */
-	entries = roundup_pow_of_two(srq_init_attr->attr.max_wr + 1);
+	entries = bnxt_re_init_depth(srq_init_attr->attr.max_wr + 1, uctx);
 	if (entries > dev_attr->max_srq_wqes + 1)
 		entries = dev_attr->max_srq_wqes + 1;
 	srq->qplib_srq.max_wqe = entries;
@@ -2103,6 +2109,9 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
 		qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic;
 	}
 	if (qp_attr_mask & IB_QP_CAP) {
+		struct bnxt_re_ucontext *uctx =
+			rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
+
 		qp->qplib_qp.modify_flags |=
 				CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE |
 				CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE |
@@ -2119,7 +2128,7 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
 				  "Create QP failed - max exceeded");
 			return -EINVAL;
 		}
-		entries = roundup_pow_of_two(qp_attr->cap.max_send_wr);
+		entries = bnxt_re_init_depth(qp_attr->cap.max_send_wr, uctx);
 		qp->qplib_qp.sq.max_wqe = min_t(u32, entries,
 						dev_attr->max_qp_wqes + 1);
 		qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe -
@@ -2132,7 +2141,7 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
 		qp->qplib_qp.sq.q_full_delta -= 1;
 		qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge;
 		if (qp->qplib_qp.rq.max_wqe) {
-			entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr);
+			entries = bnxt_re_init_depth(qp_attr->cap.max_recv_wr, uctx);
 			qp->qplib_qp.rq.max_wqe =
 				min_t(u32, entries, dev_attr->max_qp_wqes + 1);
 			qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe -
@@ -2920,9 +2929,11 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
 int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 		      struct ib_udata *udata)
 {
+	struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq);
 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibcq->device, ibdev);
+	struct bnxt_re_ucontext *uctx =
+		rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
 	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
-	struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq);
 	int rc, entries;
 	int cqe = attr->cqe;
 	struct bnxt_qplib_nq *nq = NULL;
@@ -2941,7 +2952,7 @@ int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	cq->rdev = rdev;
 	cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq);
 
-	entries = roundup_pow_of_two(cqe + 1);
+	entries = bnxt_re_init_depth(cqe + 1, uctx);
 	if (entries > dev_attr->max_cq_wqes + 1)
 		entries = dev_attr->max_cq_wqes + 1;
 
@@ -2949,8 +2960,6 @@ int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	cq->qplib_cq.sg_info.pgshft = PAGE_SHIFT;
 	if (udata) {
 		struct bnxt_re_cq_req req;
-		struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context(
-			udata, struct bnxt_re_ucontext, ib_uctx);
 		if (ib_copy_from_udata(&req, udata, sizeof(req))) {
 			rc = -EFAULT;
 			goto fail;
@@ -3072,12 +3081,11 @@ int bnxt_re_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
 		return -EINVAL;
 	}
 
-	entries = roundup_pow_of_two(cqe + 1);
+	uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
+	entries = bnxt_re_init_depth(cqe + 1, uctx);
 	if (entries > dev_attr->max_cq_wqes + 1)
 		entries = dev_attr->max_cq_wqes + 1;
 
-	uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext,
-					 ib_uctx);
 	/* uverbs consumer */
 	if (ib_copy_from_udata(&req, udata, sizeof(req))) {
 		rc = -EFAULT;
@@ -4108,6 +4116,7 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata)
 	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
 	struct bnxt_re_user_mmap_entry *entry;
 	struct bnxt_re_uctx_resp resp = {};
+	struct bnxt_re_uctx_req ureq = {};
 	u32 chip_met_rev_num = 0;
 	int rc;
 
@@ -4157,6 +4166,16 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata)
 	if (rdev->pacing.dbr_pacing)
 		resp.comp_mask |= BNXT_RE_UCNTX_CMASK_DBR_PACING_ENABLED;
 
+	if (udata->inlen >= sizeof(ureq)) {
+		rc = ib_copy_from_udata(&ureq, udata, min(udata->inlen, sizeof(ureq)));
+		if (rc)
+			goto cfail;
+		if (ureq.comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT) {
+			resp.comp_mask |= BNXT_RE_UCNTX_CMASK_POW2_DISABLED;
+			uctx->cmask |= BNXT_RE_UCNTX_CMASK_POW2_DISABLED;
+		}
+	}
+
 	rc = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp)));
 	if (rc) {
 		ibdev_err(ibdev, "Failed to copy user context");
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
index 84715b7e7a4e..98baea98fc17 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
@@ -140,6 +140,7 @@ struct bnxt_re_ucontext {
 	void			*shpg;
 	spinlock_t		sh_lock;	/* protect shpg */
 	struct rdma_user_mmap_entry *shpage_mmap;
+	u64 cmask;
 };
 
 enum bnxt_re_mmap_flag {
@@ -167,6 +168,12 @@ static inline u16 bnxt_re_get_rwqe_size(int nsge)
 	return sizeof(struct rq_wqe_hdr) + (nsge * sizeof(struct sq_sge));
 }
 
+static inline u32 bnxt_re_init_depth(u32 ent, struct bnxt_re_ucontext *uctx)
+{
+	return uctx ? (uctx->cmask & BNXT_RE_UCNTX_CMASK_POW2_DISABLED) ?
+		ent : roundup_pow_of_two(ent) : ent;
+}
+
 int bnxt_re_query_device(struct ib_device *ibdev,
 			 struct ib_device_attr *ib_attr,
 			 struct ib_udata *udata);
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
index abbabea7f5fa..b821c37dffd9 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
@@ -237,18 +237,15 @@ static void clean_nq(struct bnxt_qplib_nq *nq, struct bnxt_qplib_cq *cq)
 	struct bnxt_qplib_hwq *hwq = &nq->hwq;
 	struct nq_base *nqe, **nq_ptr;
 	int budget = nq->budget;
-	u32 sw_cons, raw_cons;
 	uintptr_t q_handle;
 	u16 type;
 
 	spin_lock_bh(&hwq->lock);
 	/* Service the NQ until empty */
-	raw_cons = hwq->cons;
 	while (budget--) {
-		sw_cons = HWQ_CMP(raw_cons, hwq);
 		nq_ptr = (struct nq_base **)hwq->pbl_ptr;
-		nqe = &nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)];
-		if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements))
+		nqe = &nq_ptr[NQE_PG(hwq->cons)][NQE_IDX(hwq->cons)];
+		if (!NQE_CMP_VALID(nqe, nq->nq_db.dbinfo.flags))
 			break;
 
 		/*
@@ -276,7 +273,8 @@ static void clean_nq(struct bnxt_qplib_nq *nq, struct bnxt_qplib_cq *cq)
 		default:
 			break;
 		}
-		raw_cons++;
+		bnxt_qplib_hwq_incr_cons(hwq->max_elements, &hwq->cons,
+					 1, &nq->nq_db.dbinfo.flags);
 	}
 	spin_unlock_bh(&hwq->lock);
 }
@@ -302,18 +300,16 @@ static void bnxt_qplib_service_nq(struct tasklet_struct *t)
 	struct bnxt_qplib_hwq *hwq = &nq->hwq;
 	struct bnxt_qplib_cq *cq;
 	int budget = nq->budget;
-	u32 sw_cons, raw_cons;
 	struct nq_base *nqe;
 	uintptr_t q_handle;
+	u32 hw_polled = 0;
 	u16 type;
 
 	spin_lock_bh(&hwq->lock);
 	/* Service the NQ until empty */
-	raw_cons = hwq->cons;
 	while (budget--) {
-		sw_cons = HWQ_CMP(raw_cons, hwq);
-		nqe = bnxt_qplib_get_qe(hwq, sw_cons, NULL);
-		if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements))
+		nqe = bnxt_qplib_get_qe(hwq, hwq->cons, NULL);
+		if (!NQE_CMP_VALID(nqe, nq->nq_db.dbinfo.flags))
 			break;
 
 		/*
@@ -372,12 +368,12 @@ static void bnxt_qplib_service_nq(struct tasklet_struct *t)
 				 "nqe with type = 0x%x not handled\n", type);
 			break;
 		}
-		raw_cons++;
+		hw_polled++;
+		bnxt_qplib_hwq_incr_cons(hwq->max_elements, &hwq->cons,
+					 1, &nq->nq_db.dbinfo.flags);
 	}
-	if (hwq->cons != raw_cons) {
-		hwq->cons = raw_cons;
+	if (hw_polled)
 		bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, nq->res->cctx, true);
-	}
 	spin_unlock_bh(&hwq->lock);
 }
 
@@ -505,6 +501,7 @@ static int bnxt_qplib_map_nq_db(struct bnxt_qplib_nq *nq,  u32 reg_offt)
 	pdev = nq->pdev;
 	nq_db = &nq->nq_db;
 
+	nq_db->dbinfo.flags = 0;
 	nq_db->reg.bar_id = NQ_CONS_PCI_BAR_REGION;
 	nq_db->reg.bar_base = pci_resource_start(pdev, nq_db->reg.bar_id);
 	if (!nq_db->reg.bar_base) {
@@ -649,7 +646,7 @@ int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
 		rc = -ENOMEM;
 		goto fail;
 	}
-
+	srq->dbinfo.flags = 0;
 	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
 				 CMDQ_BASE_OPCODE_CREATE_SRQ,
 				 sizeof(req));
@@ -703,13 +700,9 @@ int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res,
 			  struct bnxt_qplib_srq *srq)
 {
 	struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
-	u32 sw_prod, sw_cons, count = 0;
-
-	sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq);
-	sw_cons = HWQ_CMP(srq_hwq->cons, srq_hwq);
+	u32 count;
 
-	count = sw_prod > sw_cons ? sw_prod - sw_cons :
-				    srq_hwq->max_elements - sw_cons + sw_prod;
+	count = __bnxt_qplib_get_avail(srq_hwq);
 	if (count > srq->threshold) {
 		srq->arm_req = false;
 		bnxt_qplib_srq_arm_db(&srq->dbinfo, srq->threshold);
@@ -761,7 +754,7 @@ int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq,
 	struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
 	struct rq_wqe *srqe;
 	struct sq_sge *hw_sge;
-	u32 sw_prod, sw_cons, count = 0;
+	u32 count = 0;
 	int i, next;
 
 	spin_lock(&srq_hwq->lock);
@@ -775,8 +768,7 @@ int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq,
 	srq->start_idx = srq->swq[next].next_idx;
 	spin_unlock(&srq_hwq->lock);
 
-	sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq);
-	srqe = bnxt_qplib_get_qe(srq_hwq, sw_prod, NULL);
+	srqe = bnxt_qplib_get_qe(srq_hwq, srq_hwq->prod, NULL);
 	memset(srqe, 0, srq->wqe_size);
 	/* Calculate wqe_size16 and data_len */
 	for (i = 0, hw_sge = (struct sq_sge *)srqe->data;
@@ -792,17 +784,10 @@ int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq,
 	srqe->wr_id[0] = cpu_to_le32((u32)next);
 	srq->swq[next].wr_id = wqe->wr_id;
 
-	srq_hwq->prod++;
+	bnxt_qplib_hwq_incr_prod(&srq->dbinfo, srq_hwq, srq->dbinfo.max_slot);
 
 	spin_lock(&srq_hwq->lock);
-	sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq);
-	/* retaining srq_hwq->cons for this logic
-	 * actually the lock is only required to
-	 * read srq_hwq->cons.
-	 */
-	sw_cons = HWQ_CMP(srq_hwq->cons, srq_hwq);
-	count = sw_prod > sw_cons ? sw_prod - sw_cons :
-				    srq_hwq->max_elements - sw_cons + sw_prod;
+	count = __bnxt_qplib_get_avail(srq_hwq);
 	spin_unlock(&srq_hwq->lock);
 	/* Ring DB */
 	bnxt_qplib_ring_prod_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ);
@@ -849,6 +834,7 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 	u32 tbl_indx;
 	int rc;
 
+	sq->dbinfo.flags = 0;
 	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
 				 CMDQ_BASE_OPCODE_CREATE_QP1,
 				 sizeof(req));
@@ -885,6 +871,7 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 
 	/* RQ */
 	if (rq->max_wqe) {
+		rq->dbinfo.flags = 0;
 		hwq_attr.res = res;
 		hwq_attr.sginfo = &rq->sg_info;
 		hwq_attr.stride = sizeof(struct sq_sge);
@@ -992,6 +979,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 	u32 tbl_indx;
 	u16 nsge;
 
+	sq->dbinfo.flags = 0;
 	bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req,
 				 CMDQ_BASE_OPCODE_CREATE_QP,
 				 sizeof(req));
@@ -1040,6 +1028,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 
 	/* RQ */
 	if (!qp->srq) {
+		rq->dbinfo.flags = 0;
 		hwq_attr.res = res;
 		hwq_attr.sginfo = &rq->sg_info;
 		hwq_attr.stride = sizeof(struct sq_sge);
@@ -1454,12 +1443,15 @@ int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp)
 {
 	struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
+	u32 peek_flags, peek_cons;
 	struct cq_base *hw_cqe;
 	int i;
 
+	peek_flags = cq->dbinfo.flags;
+	peek_cons = cq_hwq->cons;
 	for (i = 0; i < cq_hwq->max_elements; i++) {
-		hw_cqe = bnxt_qplib_get_qe(cq_hwq, i, NULL);
-		if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq->max_elements))
+		hw_cqe = bnxt_qplib_get_qe(cq_hwq, peek_cons, NULL);
+		if (!CQE_CMP_VALID(hw_cqe, peek_flags))
 			continue;
 		/*
 		 * The valid test of the entry must be done first before
@@ -1489,6 +1481,8 @@ static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp)
 		default:
 			break;
 		}
+		bnxt_qplib_hwq_incr_cons(cq_hwq->max_elements, &peek_cons,
+					 1, &peek_flags);
 	}
 }
 
@@ -1961,7 +1955,7 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
 	bnxt_qplib_fill_psn_search(qp, wqe, swq);
 queue_err:
 	bnxt_qplib_swq_mod_start(sq, wqe_idx);
-	bnxt_qplib_hwq_incr_prod(hwq, swq->slots);
+	bnxt_qplib_hwq_incr_prod(&sq->dbinfo, hwq, swq->slots);
 	qp->wqe_cnt++;
 done:
 	if (sch_handler) {
@@ -2049,7 +2043,7 @@ int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
 	base_hdr->wr_id[0] = cpu_to_le32(wqe_idx);
 queue_err:
 	bnxt_qplib_swq_mod_start(rq, wqe_idx);
-	bnxt_qplib_hwq_incr_prod(hwq, swq->slots);
+	bnxt_qplib_hwq_incr_prod(&rq->dbinfo, hwq, swq->slots);
 done:
 	if (sch_handler) {
 		nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
@@ -2086,6 +2080,7 @@ int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
 		return -EINVAL;
 	}
 
+	cq->dbinfo.flags = 0;
 	hwq_attr.res = res;
 	hwq_attr.depth = cq->max_wqe;
 	hwq_attr.stride = sizeof(struct cq_base);
@@ -2101,7 +2096,7 @@ int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
 
 	req.dpi = cpu_to_le32(cq->dpi->dpi);
 	req.cq_handle = cpu_to_le64(cq->cq_handle);
-	req.cq_size = cpu_to_le32(cq->hwq.max_elements);
+	req.cq_size = cpu_to_le32(cq->max_wqe);
 	pbl = &cq->hwq.pbl[PBL_LVL_0];
 	pg_sz_lvl = (bnxt_qplib_base_pg_size(&cq->hwq) <<
 		     CMDQ_CREATE_CQ_PG_SIZE_SFT);
@@ -2144,6 +2139,8 @@ void bnxt_qplib_resize_cq_complete(struct bnxt_qplib_res *res,
 {
 	bnxt_qplib_free_hwq(res, &cq->hwq);
 	memcpy(&cq->hwq, &cq->resize_hwq, sizeof(cq->hwq));
+       /* Reset only the cons bit in the flags */
+	cq->dbinfo.flags &= ~(1UL << BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT);
 }
 
 int bnxt_qplib_resize_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq,
@@ -2240,7 +2237,8 @@ static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp,
 		cqe++;
 		(*budget)--;
 skip_compl:
-		bnxt_qplib_hwq_incr_cons(&sq->hwq, sq->swq[last].slots);
+		bnxt_qplib_hwq_incr_cons(sq->hwq.max_elements, &sq->hwq.cons,
+					 sq->swq[last].slots, &sq->dbinfo.flags);
 		sq->swq_last = sq->swq[last].next_idx;
 	}
 	*pcqe = cqe;
@@ -2287,7 +2285,8 @@ static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp,
 		cqe->wr_id = rq->swq[last].wr_id;
 		cqe++;
 		(*budget)--;
-		bnxt_qplib_hwq_incr_cons(&rq->hwq, rq->swq[last].slots);
+		bnxt_qplib_hwq_incr_cons(rq->hwq.max_elements, &rq->hwq.cons,
+					 rq->swq[last].slots, &rq->dbinfo.flags);
 		rq->swq_last = rq->swq[last].next_idx;
 	}
 	*pcqe = cqe;
@@ -2316,7 +2315,7 @@ void bnxt_qplib_mark_qp_error(void *qp_handle)
 static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq,
 		     u32 cq_cons, u32 swq_last, u32 cqe_sq_cons)
 {
-	u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx;
+	u32 peek_sw_cq_cons, peek_sq_cons_idx, peek_flags;
 	struct bnxt_qplib_q *sq = &qp->sq;
 	struct cq_req *peek_req_hwcqe;
 	struct bnxt_qplib_qp *peek_qp;
@@ -2347,16 +2346,14 @@ static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq,
 	}
 	if (sq->condition) {
 		/* Peek at the completions */
-		peek_raw_cq_cons = cq->hwq.cons;
+		peek_flags = cq->dbinfo.flags;
 		peek_sw_cq_cons = cq_cons;
 		i = cq->hwq.max_elements;
 		while (i--) {
-			peek_sw_cq_cons = HWQ_CMP((peek_sw_cq_cons), &cq->hwq);
 			peek_hwcqe = bnxt_qplib_get_qe(&cq->hwq,
 						       peek_sw_cq_cons, NULL);
 			/* If the next hwcqe is VALID */
-			if (CQE_CMP_VALID(peek_hwcqe, peek_raw_cq_cons,
-					  cq->hwq.max_elements)) {
+			if (CQE_CMP_VALID(peek_hwcqe, peek_flags)) {
 			/*
 			 * The valid test of the entry must be done first before
 			 * reading any further.
@@ -2399,8 +2396,9 @@ static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq,
 				rc = -EINVAL;
 				goto out;
 			}
-			peek_sw_cq_cons++;
-			peek_raw_cq_cons++;
+			bnxt_qplib_hwq_incr_cons(cq->hwq.max_elements,
+						 &peek_sw_cq_cons,
+						 1, &peek_flags);
 		}
 		dev_err(&cq->hwq.pdev->dev,
 			"Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x\n",
@@ -2487,7 +2485,8 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
 			}
 		}
 skip:
-		bnxt_qplib_hwq_incr_cons(&sq->hwq, swq->slots);
+		bnxt_qplib_hwq_incr_cons(sq->hwq.max_elements, &sq->hwq.cons,
+					 swq->slots, &sq->dbinfo.flags);
 		sq->swq_last = swq->next_idx;
 		if (sq->single)
 			break;
@@ -2514,7 +2513,8 @@ static void bnxt_qplib_release_srqe(struct bnxt_qplib_srq *srq, u32 tag)
 	srq->swq[srq->last_idx].next_idx = (int)tag;
 	srq->last_idx = (int)tag;
 	srq->swq[srq->last_idx].next_idx = -1;
-	srq->hwq.cons++; /* Support for SRQE counter */
+	bnxt_qplib_hwq_incr_cons(srq->hwq.max_elements, &srq->hwq.cons,
+				 srq->dbinfo.max_slot, &srq->dbinfo.flags);
 	spin_unlock(&srq->hwq.lock);
 }
 
@@ -2583,7 +2583,8 @@ static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,
 		cqe->wr_id = swq->wr_id;
 		cqe++;
 		(*budget)--;
-		bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots);
+		bnxt_qplib_hwq_incr_cons(rq->hwq.max_elements, &rq->hwq.cons,
+					 swq->slots, &rq->dbinfo.flags);
 		rq->swq_last = swq->next_idx;
 		*pcqe = cqe;
 
@@ -2669,7 +2670,8 @@ static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
 		cqe->wr_id = swq->wr_id;
 		cqe++;
 		(*budget)--;
-		bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots);
+		bnxt_qplib_hwq_incr_cons(rq->hwq.max_elements, &rq->hwq.cons,
+					 swq->slots, &rq->dbinfo.flags);
 		rq->swq_last = swq->next_idx;
 		*pcqe = cqe;
 
@@ -2686,14 +2688,11 @@ static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
 bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq)
 {
 	struct cq_base *hw_cqe;
-	u32 sw_cons, raw_cons;
 	bool rc = true;
 
-	raw_cons = cq->hwq.cons;
-	sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
-	hw_cqe = bnxt_qplib_get_qe(&cq->hwq, sw_cons, NULL);
+	hw_cqe = bnxt_qplib_get_qe(&cq->hwq, cq->hwq.cons, NULL);
 	 /* Check for Valid bit. If the CQE is valid, return false */
-	rc = !CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements);
+	rc = !CQE_CMP_VALID(hw_cqe, cq->dbinfo.flags);
 	return rc;
 }
 
@@ -2775,7 +2774,8 @@ static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,
 		cqe->wr_id = swq->wr_id;
 		cqe++;
 		(*budget)--;
-		bnxt_qplib_hwq_incr_cons(&rq->hwq, swq->slots);
+		bnxt_qplib_hwq_incr_cons(rq->hwq.max_elements, &rq->hwq.cons,
+					 swq->slots, &rq->dbinfo.flags);
 		rq->swq_last = swq->next_idx;
 		*pcqe = cqe;
 
@@ -2848,7 +2848,8 @@ static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,
 			cqe++;
 			(*budget)--;
 		}
-		bnxt_qplib_hwq_incr_cons(&sq->hwq, sq->swq[swq_last].slots);
+		bnxt_qplib_hwq_incr_cons(sq->hwq.max_elements, &sq->hwq.cons,
+					 sq->swq[swq_last].slots, &sq->dbinfo.flags);
 		sq->swq_last = sq->swq[swq_last].next_idx;
 	}
 	*pcqe = cqe;
@@ -2933,19 +2934,17 @@ int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
 		       int num_cqes, struct bnxt_qplib_qp **lib_qp)
 {
 	struct cq_base *hw_cqe;
-	u32 sw_cons, raw_cons;
 	int budget, rc = 0;
+	u32 hw_polled = 0;
 	u8 type;
 
-	raw_cons = cq->hwq.cons;
 	budget = num_cqes;
 
 	while (budget) {
-		sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
-		hw_cqe = bnxt_qplib_get_qe(&cq->hwq, sw_cons, NULL);
+		hw_cqe = bnxt_qplib_get_qe(&cq->hwq, cq->hwq.cons, NULL);
 
 		/* Check for Valid bit */
-		if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements))
+		if (!CQE_CMP_VALID(hw_cqe, cq->dbinfo.flags))
 			break;
 
 		/*
@@ -2960,7 +2959,7 @@ int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
 			rc = bnxt_qplib_cq_process_req(cq,
 						       (struct cq_req *)hw_cqe,
 						       &cqe, &budget,
-						       sw_cons, lib_qp);
+						       cq->hwq.cons, lib_qp);
 			break;
 		case CQ_BASE_CQE_TYPE_RES_RC:
 			rc = bnxt_qplib_cq_process_res_rc(cq,
@@ -3006,12 +3005,13 @@ int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
 				dev_err(&cq->hwq.pdev->dev,
 					"process_cqe error rc = 0x%x\n", rc);
 		}
-		raw_cons++;
+		hw_polled++;
+		bnxt_qplib_hwq_incr_cons(cq->hwq.max_elements, &cq->hwq.cons,
+					 1, &cq->dbinfo.flags);
+
 	}
-	if (cq->hwq.cons != raw_cons) {
-		cq->hwq.cons = raw_cons;
+	if (hw_polled)
 		bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ);
-	}
 exit:
 	return num_cqes - budget;
 }
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.h b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
index 404b851091ca..23c27cb42978 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
@@ -348,9 +348,21 @@ struct bnxt_qplib_qp {
 #define CQE_IDX(x)		((x) & CQE_MAX_IDX_PER_PG)
 
 #define ROCE_CQE_CMP_V			0
-#define CQE_CMP_VALID(hdr, raw_cons, cp_bit)			\
+#define CQE_CMP_VALID(hdr, pass)			\
 	(!!((hdr)->cqe_type_toggle & CQ_BASE_TOGGLE) ==		\
-	   !((raw_cons) & (cp_bit)))
+	   !((pass) & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK))
+
+static inline u32 __bnxt_qplib_get_avail(struct bnxt_qplib_hwq *hwq)
+{
+	int cons, prod, avail;
+
+	cons = hwq->cons;
+	prod = hwq->prod;
+	avail = cons - prod;
+	if (cons <= prod)
+		avail += hwq->depth;
+	return avail;
+}
 
 static inline bool bnxt_qplib_queue_full(struct bnxt_qplib_q *que,
 					 u8 slots)
@@ -443,9 +455,9 @@ struct bnxt_qplib_cq {
 #define NQE_PG(x)		(((x) & ~NQE_MAX_IDX_PER_PG) / NQE_CNT_PER_PG)
 #define NQE_IDX(x)		((x) & NQE_MAX_IDX_PER_PG)
 
-#define NQE_CMP_VALID(hdr, raw_cons, cp_bit)			\
+#define NQE_CMP_VALID(hdr, pass)			\
 	(!!(le32_to_cpu((hdr)->info63_v[0]) & NQ_BASE_V) ==	\
-	   !((raw_cons) & (cp_bit)))
+	   !((pass) & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK))
 
 #define BNXT_QPLIB_NQE_MAX_CNT		(128 * 1024)
 
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
index e47b4ca64d33..15e6d2b80c70 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
@@ -734,17 +734,15 @@ static void bnxt_qplib_service_creq(struct tasklet_struct *t)
 	u32 type, budget = CREQ_ENTRY_POLL_BUDGET;
 	struct bnxt_qplib_hwq *hwq = &creq->hwq;
 	struct creq_base *creqe;
-	u32 sw_cons, raw_cons;
 	unsigned long flags;
 	u32 num_wakeup = 0;
+	u32 hw_polled = 0;
 
 	/* Service the CREQ until budget is over */
 	spin_lock_irqsave(&hwq->lock, flags);
-	raw_cons = hwq->cons;
 	while (budget > 0) {
-		sw_cons = HWQ_CMP(raw_cons, hwq);
-		creqe = bnxt_qplib_get_qe(hwq, sw_cons, NULL);
-		if (!CREQ_CMP_VALID(creqe, raw_cons, hwq->max_elements))
+		creqe = bnxt_qplib_get_qe(hwq, hwq->cons, NULL);
+		if (!CREQ_CMP_VALID(creqe, creq->creq_db.dbinfo.flags))
 			break;
 		/* The valid test of the entry must be done first before
 		 * reading any further.
@@ -775,15 +773,15 @@ static void bnxt_qplib_service_creq(struct tasklet_struct *t)
 					 type);
 			break;
 		}
-		raw_cons++;
 		budget--;
+		hw_polled++;
+		bnxt_qplib_hwq_incr_cons(hwq->max_elements, &hwq->cons,
+					 1, &creq->creq_db.dbinfo.flags);
 	}
 
-	if (hwq->cons != raw_cons) {
-		hwq->cons = raw_cons;
+	if (hw_polled)
 		bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo,
 				      rcfw->res->cctx, true);
-	}
 	spin_unlock_irqrestore(&hwq->lock, flags);
 	if (num_wakeup)
 		wake_up_nr(&rcfw->cmdq.waitq, num_wakeup);
@@ -1113,6 +1111,7 @@ static int bnxt_qplib_map_creq_db(struct bnxt_qplib_rcfw *rcfw, u32 reg_offt)
 	pdev = rcfw->pdev;
 	creq_db = &rcfw->creq.creq_db;
 
+	creq_db->dbinfo.flags = 0;
 	creq_db->reg.bar_id = RCFW_COMM_CONS_PCI_BAR_REGION;
 	creq_db->reg.bar_base = pci_resource_start(pdev, creq_db->reg.bar_id);
 	if (!creq_db->reg.bar_id)
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h
index 7b31bee3e000..45996e60a0d0 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h
@@ -141,9 +141,9 @@ struct bnxt_qplib_crsbe {
 /* Allocate 1 per QP for async error notification for now */
 #define BNXT_QPLIB_CREQE_MAX_CNT	(64 * 1024)
 #define BNXT_QPLIB_CREQE_UNITS		16	/* 16-Bytes per prod unit */
-#define CREQ_CMP_VALID(hdr, raw_cons, cp_bit)			\
+#define CREQ_CMP_VALID(hdr, pass)			\
 	(!!((hdr)->v & CREQ_BASE_V) ==				\
-	   !((raw_cons) & (cp_bit)))
+	   !((pass) & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK))
 #define CREQ_ENTRY_POLL_BUDGET		0x100
 
 /* HWQ */
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c
index 157db6b7e119..ae2bde34e785 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_res.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c
@@ -343,7 +343,7 @@ int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq,
 	hwq->cons = 0;
 	hwq->pdev = pdev;
 	hwq->depth = hwq_attr->depth;
-	hwq->max_elements = depth;
+	hwq->max_elements = hwq->depth;
 	hwq->element_size = stride;
 	hwq->qe_ppg = pg_size / stride;
 	/* For direct access to the elements */
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.h b/drivers/infiniband/hw/bnxt_re/qplib_res.h
index 5949f004f785..3e3383b8a913 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_res.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_res.h
@@ -186,6 +186,14 @@ struct bnxt_qplib_db_info {
 	struct bnxt_qplib_hwq	*hwq;
 	u32			xid;
 	u32			max_slot;
+	u32                     flags;
+};
+
+enum bnxt_qplib_db_info_flags_mask {
+	BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT        = 0x0UL,
+	BNXT_QPLIB_FLAG_EPOCH_PROD_SHIFT        = 0x1UL,
+	BNXT_QPLIB_FLAG_EPOCH_CONS_MASK         = 0x1UL,
+	BNXT_QPLIB_FLAG_EPOCH_PROD_MASK         = 0x2UL,
 };
 
 /* Tables */
@@ -396,24 +404,34 @@ void bnxt_qplib_unmap_db_bar(struct bnxt_qplib_res *res);
 
 int bnxt_qplib_determine_atomics(struct pci_dev *dev);
 
-static inline void bnxt_qplib_hwq_incr_prod(struct bnxt_qplib_hwq *hwq, u32 cnt)
+static inline void bnxt_qplib_hwq_incr_prod(struct bnxt_qplib_db_info *dbinfo,
+					    struct bnxt_qplib_hwq *hwq, u32 cnt)
 {
-	hwq->prod = (hwq->prod + cnt) % hwq->depth;
+	/* move prod and update toggle/epoch if wrap around */
+	hwq->prod += cnt;
+	if (hwq->prod >= hwq->depth) {
+		hwq->prod %= hwq->depth;
+		dbinfo->flags ^= 1UL << BNXT_QPLIB_FLAG_EPOCH_PROD_SHIFT;
+	}
 }
 
-static inline void bnxt_qplib_hwq_incr_cons(struct bnxt_qplib_hwq *hwq,
-					    u32 cnt)
+static inline void bnxt_qplib_hwq_incr_cons(u32 max_elements, u32 *cons, u32 cnt,
+					    u32 *dbinfo_flags)
 {
-	hwq->cons = (hwq->cons + cnt) % hwq->depth;
+	/* move cons and update toggle/epoch if wrap around */
+	*cons += cnt;
+	if (*cons >= max_elements) {
+		*cons %= max_elements;
+		*dbinfo_flags ^= 1UL << BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT;
+	}
 }
 
 static inline void bnxt_qplib_ring_db32(struct bnxt_qplib_db_info *info,
 					bool arm)
 {
-	u32 key;
+	u32 key = 0;
 
-	key = info->hwq->cons & (info->hwq->max_elements - 1);
-	key |= (CMPL_DOORBELL_IDX_VALID |
+	key |= info->hwq->cons | (CMPL_DOORBELL_IDX_VALID |
 		(CMPL_DOORBELL_KEY_CMPL & CMPL_DOORBELL_KEY_MASK));
 	if (!arm)
 		key |= CMPL_DOORBELL_MASK;
@@ -427,8 +445,7 @@ static inline void bnxt_qplib_ring_db(struct bnxt_qplib_db_info *info,
 
 	key = (info->xid & DBC_DBC_XID_MASK) | DBC_DBC_PATH_ROCE | type;
 	key <<= 32;
-	key |= (info->hwq->cons & (info->hwq->max_elements - 1)) &
-		DBC_DBC_INDEX_MASK;
+	key |= (info->hwq->cons & DBC_DBC_INDEX_MASK);
 	writeq(key, info->db);
 }
 
diff --git a/drivers/infiniband/hw/hfi1/user_exp_rcv.c b/drivers/infiniband/hw/hfi1/user_exp_rcv.c
index 6419872f95cf..cf2d29098406 100644
--- a/drivers/infiniband/hw/hfi1/user_exp_rcv.c
+++ b/drivers/infiniband/hw/hfi1/user_exp_rcv.c
@@ -491,8 +491,8 @@ int hfi1_user_exp_rcv_clear(struct hfi1_filedata *fd,
 	if (unlikely(tinfo->tidcnt > fd->tid_used))
 		return -EINVAL;
 
-	tidinfo = memdup_user(u64_to_user_ptr(tinfo->tidlist),
-			      sizeof(tidinfo[0]) * tinfo->tidcnt);
+	tidinfo = memdup_array_user(u64_to_user_ptr(tinfo->tidlist),
+				    tinfo->tidcnt, sizeof(tidinfo[0]));
 	if (IS_ERR(tidinfo))
 		return PTR_ERR(tidinfo);
 
diff --git a/drivers/infiniband/hw/hfi1/user_sdma.c b/drivers/infiniband/hw/hfi1/user_sdma.c
index 6a4aa59c0cbc..b72625283fcf 100644
--- a/drivers/infiniband/hw/hfi1/user_sdma.c
+++ b/drivers/infiniband/hw/hfi1/user_sdma.c
@@ -494,8 +494,8 @@ int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,
 		 * equal to the pkt count. However, there is no way to
 		 * tell at this point.
 		 */
-		tmp = memdup_user(iovec[idx].iov_base,
-				  ntids * sizeof(*req->tids));
+		tmp = memdup_array_user(iovec[idx].iov_base,
+					ntids, sizeof(*req->tids));
 		if (IS_ERR(tmp)) {
 			ret = PTR_ERR(tmp);
 			SDMA_DBG(req, "Failed to copy %d TIDs (%d)",
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
index 0cd2612a4987..2bca9560f32d 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
@@ -4760,10 +4760,15 @@ static int check_cong_type(struct ib_qp *ibqp,
 		cong_alg->wnd_mode_sel = WND_LIMIT;
 		break;
 	default:
-		ibdev_err(&hr_dev->ib_dev,
-			  "error type(%u) for congestion selection.\n",
-			  hr_dev->caps.cong_type);
-		return -EINVAL;
+		ibdev_warn(&hr_dev->ib_dev,
+			   "invalid type(%u) for congestion selection.\n",
+			   hr_dev->caps.cong_type);
+		hr_dev->caps.cong_type = CONG_TYPE_DCQCN;
+		cong_alg->alg_sel = CONG_DCQCN;
+		cong_alg->alg_sub_sel = UNSUPPORT_CONG_LEVEL;
+		cong_alg->dip_vld = DIP_INVALID;
+		cong_alg->wnd_mode_sel = WND_LIMIT;
+		break;
 	}
 
 	return 0;
diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c
index 8fa7e4a18e73..74f6bc7b7ad1 100644
--- a/drivers/infiniband/hw/irdma/hw.c
+++ b/drivers/infiniband/hw/irdma/hw.c
@@ -321,7 +321,11 @@ static void irdma_process_aeq(struct irdma_pci_f *rf)
 			break;
 		case IRDMA_AE_QP_SUSPEND_COMPLETE:
 			if (iwqp->iwdev->vsi.tc_change_pending) {
-				atomic_dec(&iwqp->sc_qp.vsi->qp_suspend_reqs);
+				if (!atomic_dec_return(&qp->vsi->qp_suspend_reqs))
+					wake_up(&iwqp->iwdev->suspend_wq);
+			}
+			if (iwqp->suspend_pending) {
+				iwqp->suspend_pending = false;
 				wake_up(&iwqp->iwdev->suspend_wq);
 			}
 			break;
diff --git a/drivers/infiniband/hw/irdma/main.c b/drivers/infiniband/hw/irdma/main.c
index 9ac48b4dab41..3f13200ff71b 100644
--- a/drivers/infiniband/hw/irdma/main.c
+++ b/drivers/infiniband/hw/irdma/main.c
@@ -48,7 +48,7 @@ static void irdma_prep_tc_change(struct irdma_device *iwdev)
 	/* Wait for all qp's to suspend */
 	wait_event_timeout(iwdev->suspend_wq,
 			   !atomic_read(&iwdev->vsi.qp_suspend_reqs),
-			   IRDMA_EVENT_TIMEOUT);
+			   msecs_to_jiffies(IRDMA_EVENT_TIMEOUT_MS));
 	irdma_ws_reset(&iwdev->vsi);
 }
 
diff --git a/drivers/infiniband/hw/irdma/main.h b/drivers/infiniband/hw/irdma/main.h
index d66d87bb8bc4..b65bc2ea542f 100644
--- a/drivers/infiniband/hw/irdma/main.h
+++ b/drivers/infiniband/hw/irdma/main.h
@@ -78,7 +78,7 @@ extern struct auxiliary_driver i40iw_auxiliary_drv;
 
 #define MAX_DPC_ITERATIONS	128
 
-#define IRDMA_EVENT_TIMEOUT		50000
+#define IRDMA_EVENT_TIMEOUT_MS		5000
 #define IRDMA_VCHNL_EVENT_TIMEOUT	100000
 #define IRDMA_RST_TIMEOUT_HZ		4
 
diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c
index 916bfe2a91eb..0422787592d8 100644
--- a/drivers/infiniband/hw/irdma/utils.c
+++ b/drivers/infiniband/hw/irdma/utils.c
@@ -1393,17 +1393,12 @@ int irdma_ieq_check_mpacrc(struct shash_desc *desc, void *addr, u32 len,
 			   u32 val)
 {
 	u32 crc = 0;
-	int ret;
-	int ret_code = 0;
 
-	crypto_shash_init(desc);
-	ret = crypto_shash_update(desc, addr, len);
-	if (!ret)
-		crypto_shash_final(desc, (u8 *)&crc);
+	crypto_shash_digest(desc, addr, len, (u8 *)&crc);
 	if (crc != val)
-		ret_code = -EINVAL;
+		return -EINVAL;
 
-	return ret_code;
+	return 0;
 }
 
 /**
diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c
index 2138f0a2ff85..5fa88e6cca4e 100644
--- a/drivers/infiniband/hw/irdma/verbs.c
+++ b/drivers/infiniband/hw/irdma/verbs.c
@@ -1157,6 +1157,21 @@ static u8 irdma_roce_get_vlan_prio(const struct ib_gid_attr *attr, u8 prio)
 	return prio;
 }
 
+static int irdma_wait_for_suspend(struct irdma_qp *iwqp)
+{
+	if (!wait_event_timeout(iwqp->iwdev->suspend_wq,
+				!iwqp->suspend_pending,
+				msecs_to_jiffies(IRDMA_EVENT_TIMEOUT_MS))) {
+		iwqp->suspend_pending = false;
+		ibdev_warn(&iwqp->iwdev->ibdev,
+			   "modify_qp timed out waiting for suspend. qp_id = %d, last_ae = 0x%x\n",
+			   iwqp->ibqp.qp_num, iwqp->last_aeq);
+		return -EBUSY;
+	}
+
+	return 0;
+}
+
 /**
  * irdma_modify_qp_roce - modify qp request
  * @ibqp: qp's pointer for modify
@@ -1420,17 +1435,11 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 
 			info.next_iwarp_state = IRDMA_QP_STATE_SQD;
 			issue_modify_qp = 1;
+			iwqp->suspend_pending = true;
 			break;
 		case IB_QPS_SQE:
 		case IB_QPS_ERR:
 		case IB_QPS_RESET:
-			if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) {
-				spin_unlock_irqrestore(&iwqp->lock, flags);
-				info.next_iwarp_state = IRDMA_QP_STATE_SQD;
-				irdma_hw_modify_qp(iwdev, iwqp, &info, true);
-				spin_lock_irqsave(&iwqp->lock, flags);
-			}
-
 			if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) {
 				spin_unlock_irqrestore(&iwqp->lock, flags);
 				if (udata && udata->inlen) {
@@ -1467,6 +1476,11 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 			ctx_info->rem_endpoint_idx = udp_info->arp_idx;
 			if (irdma_hw_modify_qp(iwdev, iwqp, &info, true))
 				return -EINVAL;
+			if (info.next_iwarp_state == IRDMA_QP_STATE_SQD) {
+				ret = irdma_wait_for_suspend(iwqp);
+				if (ret)
+					return ret;
+			}
 			spin_lock_irqsave(&iwqp->lock, flags);
 			if (iwqp->iwarp_state == info.curr_iwarp_state) {
 				iwqp->iwarp_state = info.next_iwarp_state;
diff --git a/drivers/infiniband/hw/irdma/verbs.h b/drivers/infiniband/hw/irdma/verbs.h
index c42ac22de00e..cfa140b36395 100644
--- a/drivers/infiniband/hw/irdma/verbs.h
+++ b/drivers/infiniband/hw/irdma/verbs.h
@@ -198,6 +198,7 @@ struct irdma_qp {
 	u8 flush_issued : 1;
 	u8 sig_all : 1;
 	u8 pau_mode : 1;
+	u8 suspend_pending : 1;
 	u8 rsvd : 1;
 	u8 iwarp_state;
 	u16 term_sq_flush_code;
diff --git a/drivers/infiniband/sw/siw/siw.h b/drivers/infiniband/sw/siw/siw.h
index cec5cccd2e75..849e496c7e67 100644
--- a/drivers/infiniband/sw/siw/siw.h
+++ b/drivers/infiniband/sw/siw/siw.h
@@ -121,11 +121,10 @@ struct siw_page_chunk {
 };
 
 struct siw_umem {
+	struct ib_umem *base_mem;
 	struct siw_page_chunk *page_chunk;
 	int num_pages;
-	bool writable;
 	u64 fp_addr; /* First page base address */
-	struct mm_struct *owning_mm;
 };
 
 struct siw_pble {
diff --git a/drivers/infiniband/sw/siw/siw_mem.c b/drivers/infiniband/sw/siw/siw_mem.c
index c5f7f1669d09..2110ceb0603c 100644
--- a/drivers/infiniband/sw/siw/siw_mem.c
+++ b/drivers/infiniband/sw/siw/siw_mem.c
@@ -5,6 +5,7 @@
 
 #include <linux/gfp.h>
 #include <rdma/ib_verbs.h>
+#include <rdma/ib_umem.h>
 #include <linux/dma-mapping.h>
 #include <linux/slab.h>
 #include <linux/sched/mm.h>
@@ -60,28 +61,17 @@ struct siw_mem *siw_mem_id2obj(struct siw_device *sdev, int stag_index)
 	return NULL;
 }
 
-static void siw_free_plist(struct siw_page_chunk *chunk, int num_pages,
-			   bool dirty)
+void siw_umem_release(struct siw_umem *umem)
 {
-	unpin_user_pages_dirty_lock(chunk->plist, num_pages, dirty);
-}
-
-void siw_umem_release(struct siw_umem *umem, bool dirty)
-{
-	struct mm_struct *mm_s = umem->owning_mm;
 	int i, num_pages = umem->num_pages;
 
-	for (i = 0; num_pages; i++) {
-		int to_free = min_t(int, PAGES_PER_CHUNK, num_pages);
+	if (umem->base_mem)
+		ib_umem_release(umem->base_mem);
 
-		siw_free_plist(&umem->page_chunk[i], to_free,
-			       umem->writable && dirty);
+	for (i = 0; num_pages > 0; i++) {
 		kfree(umem->page_chunk[i].plist);
-		num_pages -= to_free;
+		num_pages -= PAGES_PER_CHUNK;
 	}
-	atomic64_sub(umem->num_pages, &mm_s->pinned_vm);
-
-	mmdrop(mm_s);
 	kfree(umem->page_chunk);
 	kfree(umem);
 }
@@ -145,7 +135,7 @@ void siw_free_mem(struct kref *ref)
 
 	if (!mem->is_mw && mem->mem_obj) {
 		if (mem->is_pbl == 0)
-			siw_umem_release(mem->umem, true);
+			siw_umem_release(mem->umem);
 		else
 			kfree(mem->pbl);
 	}
@@ -362,18 +352,16 @@ struct siw_pbl *siw_pbl_alloc(u32 num_buf)
 	return pbl;
 }
 
-struct siw_umem *siw_umem_get(u64 start, u64 len, bool writable)
+struct siw_umem *siw_umem_get(struct ib_device *base_dev, u64 start,
+			      u64 len, int rights)
 {
 	struct siw_umem *umem;
-	struct mm_struct *mm_s;
+	struct ib_umem *base_mem;
+	struct sg_page_iter sg_iter;
+	struct sg_table *sgt;
 	u64 first_page_va;
-	unsigned long mlock_limit;
-	unsigned int foll_flags = FOLL_LONGTERM;
 	int num_pages, num_chunks, i, rv = 0;
 
-	if (!can_do_mlock())
-		return ERR_PTR(-EPERM);
-
 	if (!len)
 		return ERR_PTR(-EINVAL);
 
@@ -385,65 +373,50 @@ struct siw_umem *siw_umem_get(u64 start, u64 len, bool writable)
 	if (!umem)
 		return ERR_PTR(-ENOMEM);
 
-	mm_s = current->mm;
-	umem->owning_mm = mm_s;
-	umem->writable = writable;
-
-	mmgrab(mm_s);
-
-	if (writable)
-		foll_flags |= FOLL_WRITE;
-
-	mmap_read_lock(mm_s);
-
-	mlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-
-	if (atomic64_add_return(num_pages, &mm_s->pinned_vm) > mlock_limit) {
-		rv = -ENOMEM;
-		goto out_sem_up;
-	}
-	umem->fp_addr = first_page_va;
-
 	umem->page_chunk =
 		kcalloc(num_chunks, sizeof(struct siw_page_chunk), GFP_KERNEL);
 	if (!umem->page_chunk) {
 		rv = -ENOMEM;
-		goto out_sem_up;
+		goto err_out;
 	}
-	for (i = 0; num_pages; i++) {
+	base_mem = ib_umem_get(base_dev, start, len, rights);
+	if (IS_ERR(base_mem)) {
+		rv = PTR_ERR(base_mem);
+		siw_dbg(base_dev, "Cannot pin user memory: %d\n", rv);
+		goto err_out;
+	}
+	umem->fp_addr = first_page_va;
+	umem->base_mem = base_mem;
+
+	sgt = &base_mem->sgt_append.sgt;
+	__sg_page_iter_start(&sg_iter, sgt->sgl, sgt->orig_nents, 0);
+
+	if (!__sg_page_iter_next(&sg_iter)) {
+		rv = -EINVAL;
+		goto err_out;
+	}
+	for (i = 0; num_pages > 0; i++) {
 		int nents = min_t(int, num_pages, PAGES_PER_CHUNK);
 		struct page **plist =
 			kcalloc(nents, sizeof(struct page *), GFP_KERNEL);
 
 		if (!plist) {
 			rv = -ENOMEM;
-			goto out_sem_up;
+			goto err_out;
 		}
 		umem->page_chunk[i].plist = plist;
-		while (nents) {
-			rv = pin_user_pages(first_page_va, nents, foll_flags,
-					    plist);
-			if (rv < 0)
-				goto out_sem_up;
-
-			umem->num_pages += rv;
-			first_page_va += rv * PAGE_SIZE;
-			plist += rv;
-			nents -= rv;
-			num_pages -= rv;
+		while (nents--) {
+			*plist = sg_page_iter_page(&sg_iter);
+			umem->num_pages++;
+			num_pages--;
+			plist++;
+			if (!__sg_page_iter_next(&sg_iter))
+				break;
 		}
 	}
-out_sem_up:
-	mmap_read_unlock(mm_s);
-
-	if (rv > 0)
-		return umem;
-
-	/* Adjust accounting for pages not pinned */
-	if (num_pages)
-		atomic64_sub(num_pages, &mm_s->pinned_vm);
-
-	siw_umem_release(umem, false);
+	return umem;
+err_out:
+	siw_umem_release(umem);
 
 	return ERR_PTR(rv);
 }
diff --git a/drivers/infiniband/sw/siw/siw_mem.h b/drivers/infiniband/sw/siw/siw_mem.h
index a2835284fe5b..e74cfcd6dbc1 100644
--- a/drivers/infiniband/sw/siw/siw_mem.h
+++ b/drivers/infiniband/sw/siw/siw_mem.h
@@ -6,8 +6,9 @@
 #ifndef _SIW_MEM_H
 #define _SIW_MEM_H
 
-struct siw_umem *siw_umem_get(u64 start, u64 len, bool writable);
-void siw_umem_release(struct siw_umem *umem, bool dirty);
+struct siw_umem *siw_umem_get(struct ib_device *base_dave, u64 start,
+			      u64 len, int rights);
+void siw_umem_release(struct siw_umem *umem);
 struct siw_pbl *siw_pbl_alloc(u32 num_buf);
 dma_addr_t siw_pbl_get_buffer(struct siw_pbl *pbl, u64 off, int *len, int *idx);
 struct siw_mem *siw_mem_id2obj(struct siw_device *sdev, int stag_index);
diff --git a/drivers/infiniband/sw/siw/siw_qp_tx.c b/drivers/infiniband/sw/siw/siw_qp_tx.c
index b2c06100cf01..64c113dc8a99 100644
--- a/drivers/infiniband/sw/siw/siw_qp_tx.c
+++ b/drivers/infiniband/sw/siw/siw_qp_tx.c
@@ -249,14 +249,10 @@ static int siw_qp_prepare_tx(struct siw_iwarp_tx *c_tx)
 		/*
 		 * Do complete CRC if enabled and short packet
 		 */
-		if (c_tx->mpa_crc_hd) {
-			crypto_shash_init(c_tx->mpa_crc_hd);
-			if (crypto_shash_update(c_tx->mpa_crc_hd,
-						(u8 *)&c_tx->pkt,
-						c_tx->ctrl_len))
-				return -EINVAL;
-			crypto_shash_final(c_tx->mpa_crc_hd, (u8 *)crc);
-		}
+		if (c_tx->mpa_crc_hd &&
+		    crypto_shash_digest(c_tx->mpa_crc_hd, (u8 *)&c_tx->pkt,
+					c_tx->ctrl_len, (u8 *)crc) != 0)
+			return -EINVAL;
 		c_tx->ctrl_len += MPA_CRC_SIZE;
 
 		return PKT_COMPLETE;
diff --git a/drivers/infiniband/sw/siw/siw_verbs.c b/drivers/infiniband/sw/siw/siw_verbs.c
index c5c27db9c2fe..27f7dda89e49 100644
--- a/drivers/infiniband/sw/siw/siw_verbs.c
+++ b/drivers/infiniband/sw/siw/siw_verbs.c
@@ -1321,8 +1321,6 @@ struct ib_mr *siw_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
 	struct siw_umem *umem = NULL;
 	struct siw_ureq_reg_mr ureq;
 	struct siw_device *sdev = to_siw_dev(pd->device);
-
-	unsigned long mem_limit = rlimit(RLIMIT_MEMLOCK);
 	int rv;
 
 	siw_dbg_pd(pd, "start: 0x%pK, va: 0x%pK, len: %llu\n",
@@ -1338,20 +1336,7 @@ struct ib_mr *siw_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
 		rv = -EINVAL;
 		goto err_out;
 	}
-	if (mem_limit != RLIM_INFINITY) {
-		unsigned long num_pages =
-			(PAGE_ALIGN(len + (start & ~PAGE_MASK))) >> PAGE_SHIFT;
-		mem_limit >>= PAGE_SHIFT;
-
-		if (num_pages > mem_limit - current->mm->locked_vm) {
-			siw_dbg_pd(pd, "pages req %lu, max %lu, lock %lu\n",
-				   num_pages, mem_limit,
-				   current->mm->locked_vm);
-			rv = -ENOMEM;
-			goto err_out;
-		}
-	}
-	umem = siw_umem_get(start, len, ib_access_writable(rights));
+	umem = siw_umem_get(pd->device, start, len, rights);
 	if (IS_ERR(umem)) {
 		rv = PTR_ERR(umem);
 		siw_dbg_pd(pd, "getting user memory failed: %d\n", rv);
@@ -1404,7 +1389,7 @@ struct ib_mr *siw_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
 		kfree_rcu(mr, rcu);
 	} else {
 		if (umem)
-			siw_umem_release(umem, false);
+			siw_umem_release(umem);
 	}
 	return ERR_PTR(rv);
 }
diff --git a/drivers/input/mouse/synaptics.c b/drivers/input/mouse/synaptics.c
index 22d16d80efb9..7a303a9d6bf7 100644
--- a/drivers/input/mouse/synaptics.c
+++ b/drivers/input/mouse/synaptics.c
@@ -183,6 +183,7 @@ static const char * const smbus_pnp_ids[] = {
 	"LEN009b", /* T580 */
 	"LEN0402", /* X1 Extreme Gen 2 / P1 Gen 2 */
 	"LEN040f", /* P1 Gen 3 */
+	"LEN0411", /* L14 Gen 1 */
 	"LEN200f", /* T450s */
 	"LEN2044", /* L470  */
 	"LEN2054", /* E480 */
diff --git a/drivers/input/rmi4/rmi_spi.c b/drivers/input/rmi4/rmi_spi.c
index 852aeb0b2c07..07c866f42296 100644
--- a/drivers/input/rmi4/rmi_spi.c
+++ b/drivers/input/rmi4/rmi_spi.c
@@ -375,7 +375,7 @@ static int rmi_spi_probe(struct spi_device *spi)
 	struct rmi_device_platform_data *spi_pdata = spi->dev.platform_data;
 	int error;
 
-	if (spi->master->flags & SPI_MASTER_HALF_DUPLEX)
+	if (spi->master->flags & SPI_CONTROLLER_HALF_DUPLEX)
 		return -EINVAL;
 
 	rmi_spi = devm_kzalloc(&spi->dev, sizeof(struct rmi_spi_xport),
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index fcc987f5d4ed..b9a0523cbb0a 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -3357,7 +3357,7 @@ static void irq_remapping_prepare_irte(struct amd_ir_data *data,
 
 	data->irq_2_irte.devid = devid;
 	data->irq_2_irte.index = index + sub_handle;
-	iommu->irte_ops->prepare(data->entry, apic->delivery_mode,
+	iommu->irte_ops->prepare(data->entry, APIC_DELIVERY_MODE_FIXED,
 				 apic->dest_mode_logical, irq_cfg->vector,
 				 irq_cfg->dest_apicid, devid);
 
@@ -3634,7 +3634,7 @@ int amd_iommu_deactivate_guest_mode(void *data)
 
 	entry->lo.fields_remap.valid       = valid;
 	entry->lo.fields_remap.dm          = apic->dest_mode_logical;
-	entry->lo.fields_remap.int_type    = apic->delivery_mode;
+	entry->lo.fields_remap.int_type    = APIC_DELIVERY_MODE_FIXED;
 	entry->hi.fields.vector            = cfg->vector;
 	entry->lo.fields_remap.destination =
 				APICID_TO_IRTE_DEST_LO(cfg->dest_apicid);
diff --git a/drivers/iommu/intel/irq_remapping.c b/drivers/iommu/intel/irq_remapping.c
index 29b9e55dcf26..566297bc87dd 100644
--- a/drivers/iommu/intel/irq_remapping.c
+++ b/drivers/iommu/intel/irq_remapping.c
@@ -1112,7 +1112,7 @@ static void prepare_irte(struct irte *irte, int vector, unsigned int dest)
 	 * irq migration in the presence of interrupt-remapping.
 	*/
 	irte->trigger_mode = 0;
-	irte->dlvry_mode = apic->delivery_mode;
+	irte->dlvry_mode = APIC_DELIVERY_MODE_FIXED;
 	irte->vector = vector;
 	irte->dest_id = IRTE_DEST(dest);
 	irte->redir_hint = 1;
diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h
index 05be59ae21b2..6ae2329052c9 100644
--- a/drivers/md/bcache/bcache.h
+++ b/drivers/md/bcache/bcache.h
@@ -265,6 +265,7 @@ struct bcache_device {
 #define BCACHE_DEV_WB_RUNNING		3
 #define BCACHE_DEV_RATE_DW_RUNNING	4
 	int			nr_stripes;
+#define BCH_MIN_STRIPE_SZ		((4 << 20) >> SECTOR_SHIFT)
 	unsigned int		stripe_size;
 	atomic_t		*stripe_sectors_dirty;
 	unsigned long		*full_dirty_stripes;
diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c
index ae5cbb55861f..1b0892fe7ddb 100644
--- a/drivers/md/bcache/btree.c
+++ b/drivers/md/bcache/btree.c
@@ -293,16 +293,16 @@ static void btree_complete_write(struct btree *b, struct btree_write *w)
 	w->journal	= NULL;
 }
 
-static void btree_node_write_unlock(struct closure *cl)
+static CLOSURE_CALLBACK(btree_node_write_unlock)
 {
-	struct btree *b = container_of(cl, struct btree, io);
+	closure_type(b, struct btree, io);
 
 	up(&b->io_mutex);
 }
 
-static void __btree_node_write_done(struct closure *cl)
+static CLOSURE_CALLBACK(__btree_node_write_done)
 {
-	struct btree *b = container_of(cl, struct btree, io);
+	closure_type(b, struct btree, io);
 	struct btree_write *w = btree_prev_write(b);
 
 	bch_bbio_free(b->bio, b->c);
@@ -315,12 +315,12 @@ static void __btree_node_write_done(struct closure *cl)
 	closure_return_with_destructor(cl, btree_node_write_unlock);
 }
 
-static void btree_node_write_done(struct closure *cl)
+static CLOSURE_CALLBACK(btree_node_write_done)
 {
-	struct btree *b = container_of(cl, struct btree, io);
+	closure_type(b, struct btree, io);
 
 	bio_free_pages(b->bio);
-	__btree_node_write_done(cl);
+	__btree_node_write_done(&cl->work);
 }
 
 static void btree_node_write_endio(struct bio *bio)
@@ -1000,6 +1000,9 @@ static struct btree *mca_alloc(struct cache_set *c, struct btree_op *op,
  *
  * The btree node will have either a read or a write lock held, depending on
  * level and op->lock.
+ *
+ * Note: Only error code or btree pointer will be returned, it is unncessary
+ *       for callers to check NULL pointer.
  */
 struct btree *bch_btree_node_get(struct cache_set *c, struct btree_op *op,
 				 struct bkey *k, int level, bool write,
@@ -1111,6 +1114,10 @@ static void btree_node_free(struct btree *b)
 	mutex_unlock(&b->c->bucket_lock);
 }
 
+/*
+ * Only error code or btree pointer will be returned, it is unncessary for
+ * callers to check NULL pointer.
+ */
 struct btree *__bch_btree_node_alloc(struct cache_set *c, struct btree_op *op,
 				     int level, bool wait,
 				     struct btree *parent)
@@ -1368,7 +1375,7 @@ static int btree_gc_coalesce(struct btree *b, struct btree_op *op,
 	memset(new_nodes, 0, sizeof(new_nodes));
 	closure_init_stack(&cl);
 
-	while (nodes < GC_MERGE_NODES && !IS_ERR(r[nodes].b))
+	while (nodes < GC_MERGE_NODES && !IS_ERR_OR_NULL(r[nodes].b))
 		keys += r[nodes++].keys;
 
 	blocks = btree_default_blocks(b->c) * 2 / 3;
@@ -1532,6 +1539,8 @@ static int btree_gc_rewrite_node(struct btree *b, struct btree_op *op,
 		return 0;
 
 	n = btree_node_alloc_replacement(replace, NULL);
+	if (IS_ERR(n))
+		return 0;
 
 	/* recheck reserve after allocating replacement node */
 	if (btree_check_reserve(b, NULL)) {
diff --git a/drivers/md/bcache/journal.c b/drivers/md/bcache/journal.c
index c182c21de2e8..7ff14bd2feb8 100644
--- a/drivers/md/bcache/journal.c
+++ b/drivers/md/bcache/journal.c
@@ -723,11 +723,11 @@ static void journal_write_endio(struct bio *bio)
 	closure_put(&w->c->journal.io);
 }
 
-static void journal_write(struct closure *cl);
+static CLOSURE_CALLBACK(journal_write);
 
-static void journal_write_done(struct closure *cl)
+static CLOSURE_CALLBACK(journal_write_done)
 {
-	struct journal *j = container_of(cl, struct journal, io);
+	closure_type(j, struct journal, io);
 	struct journal_write *w = (j->cur == j->w)
 		? &j->w[1]
 		: &j->w[0];
@@ -736,19 +736,19 @@ static void journal_write_done(struct closure *cl)
 	continue_at_nobarrier(cl, journal_write, bch_journal_wq);
 }
 
-static void journal_write_unlock(struct closure *cl)
+static CLOSURE_CALLBACK(journal_write_unlock)
 	__releases(&c->journal.lock)
 {
-	struct cache_set *c = container_of(cl, struct cache_set, journal.io);
+	closure_type(c, struct cache_set, journal.io);
 
 	c->journal.io_in_flight = 0;
 	spin_unlock(&c->journal.lock);
 }
 
-static void journal_write_unlocked(struct closure *cl)
+static CLOSURE_CALLBACK(journal_write_unlocked)
 	__releases(c->journal.lock)
 {
-	struct cache_set *c = container_of(cl, struct cache_set, journal.io);
+	closure_type(c, struct cache_set, journal.io);
 	struct cache *ca = c->cache;
 	struct journal_write *w = c->journal.cur;
 	struct bkey *k = &c->journal.key;
@@ -823,12 +823,12 @@ static void journal_write_unlocked(struct closure *cl)
 	continue_at(cl, journal_write_done, NULL);
 }
 
-static void journal_write(struct closure *cl)
+static CLOSURE_CALLBACK(journal_write)
 {
-	struct cache_set *c = container_of(cl, struct cache_set, journal.io);
+	closure_type(c, struct cache_set, journal.io);
 
 	spin_lock(&c->journal.lock);
-	journal_write_unlocked(cl);
+	journal_write_unlocked(&cl->work);
 }
 
 static void journal_try_write(struct cache_set *c)
diff --git a/drivers/md/bcache/movinggc.c b/drivers/md/bcache/movinggc.c
index 9f32901fdad1..ebd500bdf0b2 100644
--- a/drivers/md/bcache/movinggc.c
+++ b/drivers/md/bcache/movinggc.c
@@ -35,16 +35,16 @@ static bool moving_pred(struct keybuf *buf, struct bkey *k)
 
 /* Moving GC - IO loop */
 
-static void moving_io_destructor(struct closure *cl)
+static CLOSURE_CALLBACK(moving_io_destructor)
 {
-	struct moving_io *io = container_of(cl, struct moving_io, cl);
+	closure_type(io, struct moving_io, cl);
 
 	kfree(io);
 }
 
-static void write_moving_finish(struct closure *cl)
+static CLOSURE_CALLBACK(write_moving_finish)
 {
-	struct moving_io *io = container_of(cl, struct moving_io, cl);
+	closure_type(io, struct moving_io, cl);
 	struct bio *bio = &io->bio.bio;
 
 	bio_free_pages(bio);
@@ -89,9 +89,9 @@ static void moving_init(struct moving_io *io)
 	bch_bio_map(bio, NULL);
 }
 
-static void write_moving(struct closure *cl)
+static CLOSURE_CALLBACK(write_moving)
 {
-	struct moving_io *io = container_of(cl, struct moving_io, cl);
+	closure_type(io, struct moving_io, cl);
 	struct data_insert_op *op = &io->op;
 
 	if (!op->status) {
@@ -113,9 +113,9 @@ static void write_moving(struct closure *cl)
 	continue_at(cl, write_moving_finish, op->wq);
 }
 
-static void read_moving_submit(struct closure *cl)
+static CLOSURE_CALLBACK(read_moving_submit)
 {
-	struct moving_io *io = container_of(cl, struct moving_io, cl);
+	closure_type(io, struct moving_io, cl);
 	struct bio *bio = &io->bio.bio;
 
 	bch_submit_bbio(bio, io->op.c, &io->w->key, 0);
diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c
index a9b1f3896249..83d112bd2b1c 100644
--- a/drivers/md/bcache/request.c
+++ b/drivers/md/bcache/request.c
@@ -25,7 +25,7 @@
 
 struct kmem_cache *bch_search_cache;
 
-static void bch_data_insert_start(struct closure *cl);
+static CLOSURE_CALLBACK(bch_data_insert_start);
 
 static unsigned int cache_mode(struct cached_dev *dc)
 {
@@ -55,9 +55,9 @@ static void bio_csum(struct bio *bio, struct bkey *k)
 
 /* Insert data into cache */
 
-static void bch_data_insert_keys(struct closure *cl)
+static CLOSURE_CALLBACK(bch_data_insert_keys)
 {
-	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
+	closure_type(op, struct data_insert_op, cl);
 	atomic_t *journal_ref = NULL;
 	struct bkey *replace_key = op->replace ? &op->replace_key : NULL;
 	int ret;
@@ -136,9 +136,9 @@ static void bch_data_invalidate(struct closure *cl)
 	continue_at(cl, bch_data_insert_keys, op->wq);
 }
 
-static void bch_data_insert_error(struct closure *cl)
+static CLOSURE_CALLBACK(bch_data_insert_error)
 {
-	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
+	closure_type(op, struct data_insert_op, cl);
 
 	/*
 	 * Our data write just errored, which means we've got a bunch of keys to
@@ -163,7 +163,7 @@ static void bch_data_insert_error(struct closure *cl)
 
 	op->insert_keys.top = dst;
 
-	bch_data_insert_keys(cl);
+	bch_data_insert_keys(&cl->work);
 }
 
 static void bch_data_insert_endio(struct bio *bio)
@@ -184,9 +184,9 @@ static void bch_data_insert_endio(struct bio *bio)
 	bch_bbio_endio(op->c, bio, bio->bi_status, "writing data to cache");
 }
 
-static void bch_data_insert_start(struct closure *cl)
+static CLOSURE_CALLBACK(bch_data_insert_start)
 {
-	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
+	closure_type(op, struct data_insert_op, cl);
 	struct bio *bio = op->bio, *n;
 
 	if (op->bypass)
@@ -305,16 +305,16 @@ static void bch_data_insert_start(struct closure *cl)
  * If op->bypass is true, instead of inserting the data it invalidates the
  * region of the cache represented by op->bio and op->inode.
  */
-void bch_data_insert(struct closure *cl)
+CLOSURE_CALLBACK(bch_data_insert)
 {
-	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
+	closure_type(op, struct data_insert_op, cl);
 
 	trace_bcache_write(op->c, op->inode, op->bio,
 			   op->writeback, op->bypass);
 
 	bch_keylist_init(&op->insert_keys);
 	bio_get(op->bio);
-	bch_data_insert_start(cl);
+	bch_data_insert_start(&cl->work);
 }
 
 /*
@@ -575,9 +575,9 @@ static int cache_lookup_fn(struct btree_op *op, struct btree *b, struct bkey *k)
 	return n == bio ? MAP_DONE : MAP_CONTINUE;
 }
 
-static void cache_lookup(struct closure *cl)
+static CLOSURE_CALLBACK(cache_lookup)
 {
-	struct search *s = container_of(cl, struct search, iop.cl);
+	closure_type(s, struct search, iop.cl);
 	struct bio *bio = &s->bio.bio;
 	struct cached_dev *dc;
 	int ret;
@@ -698,9 +698,9 @@ static void do_bio_hook(struct search *s,
 	bio_cnt_set(bio, 3);
 }
 
-static void search_free(struct closure *cl)
+static CLOSURE_CALLBACK(search_free)
 {
-	struct search *s = container_of(cl, struct search, cl);
+	closure_type(s, struct search, cl);
 
 	atomic_dec(&s->iop.c->search_inflight);
 
@@ -749,20 +749,20 @@ static inline struct search *search_alloc(struct bio *bio,
 
 /* Cached devices */
 
-static void cached_dev_bio_complete(struct closure *cl)
+static CLOSURE_CALLBACK(cached_dev_bio_complete)
 {
-	struct search *s = container_of(cl, struct search, cl);
+	closure_type(s, struct search, cl);
 	struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
 
 	cached_dev_put(dc);
-	search_free(cl);
+	search_free(&cl->work);
 }
 
 /* Process reads */
 
-static void cached_dev_read_error_done(struct closure *cl)
+static CLOSURE_CALLBACK(cached_dev_read_error_done)
 {
-	struct search *s = container_of(cl, struct search, cl);
+	closure_type(s, struct search, cl);
 
 	if (s->iop.replace_collision)
 		bch_mark_cache_miss_collision(s->iop.c, s->d);
@@ -770,12 +770,12 @@ static void cached_dev_read_error_done(struct closure *cl)
 	if (s->iop.bio)
 		bio_free_pages(s->iop.bio);
 
-	cached_dev_bio_complete(cl);
+	cached_dev_bio_complete(&cl->work);
 }
 
-static void cached_dev_read_error(struct closure *cl)
+static CLOSURE_CALLBACK(cached_dev_read_error)
 {
-	struct search *s = container_of(cl, struct search, cl);
+	closure_type(s, struct search, cl);
 	struct bio *bio = &s->bio.bio;
 
 	/*
@@ -801,9 +801,9 @@ static void cached_dev_read_error(struct closure *cl)
 	continue_at(cl, cached_dev_read_error_done, NULL);
 }
 
-static void cached_dev_cache_miss_done(struct closure *cl)
+static CLOSURE_CALLBACK(cached_dev_cache_miss_done)
 {
-	struct search *s = container_of(cl, struct search, cl);
+	closure_type(s, struct search, cl);
 	struct bcache_device *d = s->d;
 
 	if (s->iop.replace_collision)
@@ -812,13 +812,13 @@ static void cached_dev_cache_miss_done(struct closure *cl)
 	if (s->iop.bio)
 		bio_free_pages(s->iop.bio);
 
-	cached_dev_bio_complete(cl);
+	cached_dev_bio_complete(&cl->work);
 	closure_put(&d->cl);
 }
 
-static void cached_dev_read_done(struct closure *cl)
+static CLOSURE_CALLBACK(cached_dev_read_done)
 {
-	struct search *s = container_of(cl, struct search, cl);
+	closure_type(s, struct search, cl);
 	struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
 
 	/*
@@ -858,9 +858,9 @@ static void cached_dev_read_done(struct closure *cl)
 	continue_at(cl, cached_dev_cache_miss_done, NULL);
 }
 
-static void cached_dev_read_done_bh(struct closure *cl)
+static CLOSURE_CALLBACK(cached_dev_read_done_bh)
 {
-	struct search *s = container_of(cl, struct search, cl);
+	closure_type(s, struct search, cl);
 	struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
 
 	bch_mark_cache_accounting(s->iop.c, s->d,
@@ -955,13 +955,13 @@ static void cached_dev_read(struct cached_dev *dc, struct search *s)
 
 /* Process writes */
 
-static void cached_dev_write_complete(struct closure *cl)
+static CLOSURE_CALLBACK(cached_dev_write_complete)
 {
-	struct search *s = container_of(cl, struct search, cl);
+	closure_type(s, struct search, cl);
 	struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
 
 	up_read_non_owner(&dc->writeback_lock);
-	cached_dev_bio_complete(cl);
+	cached_dev_bio_complete(&cl->work);
 }
 
 static void cached_dev_write(struct cached_dev *dc, struct search *s)
@@ -1048,9 +1048,9 @@ static void cached_dev_write(struct cached_dev *dc, struct search *s)
 	continue_at(cl, cached_dev_write_complete, NULL);
 }
 
-static void cached_dev_nodata(struct closure *cl)
+static CLOSURE_CALLBACK(cached_dev_nodata)
 {
-	struct search *s = container_of(cl, struct search, cl);
+	closure_type(s, struct search, cl);
 	struct bio *bio = &s->bio.bio;
 
 	if (s->iop.flush_journal)
@@ -1265,9 +1265,9 @@ static int flash_dev_cache_miss(struct btree *b, struct search *s,
 	return MAP_CONTINUE;
 }
 
-static void flash_dev_nodata(struct closure *cl)
+static CLOSURE_CALLBACK(flash_dev_nodata)
 {
-	struct search *s = container_of(cl, struct search, cl);
+	closure_type(s, struct search, cl);
 
 	if (s->iop.flush_journal)
 		bch_journal_meta(s->iop.c, cl);
diff --git a/drivers/md/bcache/request.h b/drivers/md/bcache/request.h
index 38ab4856eaab..46bbef00aebb 100644
--- a/drivers/md/bcache/request.h
+++ b/drivers/md/bcache/request.h
@@ -34,7 +34,7 @@ struct data_insert_op {
 };
 
 unsigned int bch_get_congested(const struct cache_set *c);
-void bch_data_insert(struct closure *cl);
+CLOSURE_CALLBACK(bch_data_insert);
 
 void bch_cached_dev_request_init(struct cached_dev *dc);
 void cached_dev_submit_bio(struct bio *bio);
diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c
index 8bd899766372..1402096b8076 100644
--- a/drivers/md/bcache/super.c
+++ b/drivers/md/bcache/super.c
@@ -327,9 +327,9 @@ static void __write_super(struct cache_sb *sb, struct cache_sb_disk *out,
 	submit_bio(bio);
 }
 
-static void bch_write_bdev_super_unlock(struct closure *cl)
+static CLOSURE_CALLBACK(bch_write_bdev_super_unlock)
 {
-	struct cached_dev *dc = container_of(cl, struct cached_dev, sb_write);
+	closure_type(dc, struct cached_dev, sb_write);
 
 	up(&dc->sb_write_mutex);
 }
@@ -363,9 +363,9 @@ static void write_super_endio(struct bio *bio)
 	closure_put(&ca->set->sb_write);
 }
 
-static void bcache_write_super_unlock(struct closure *cl)
+static CLOSURE_CALLBACK(bcache_write_super_unlock)
 {
-	struct cache_set *c = container_of(cl, struct cache_set, sb_write);
+	closure_type(c, struct cache_set, sb_write);
 
 	up(&c->sb_write_mutex);
 }
@@ -407,9 +407,9 @@ static void uuid_endio(struct bio *bio)
 	closure_put(cl);
 }
 
-static void uuid_io_unlock(struct closure *cl)
+static CLOSURE_CALLBACK(uuid_io_unlock)
 {
-	struct cache_set *c = container_of(cl, struct cache_set, uuid_write);
+	closure_type(c, struct cache_set, uuid_write);
 
 	up(&c->uuid_write_mutex);
 }
@@ -905,6 +905,8 @@ static int bcache_device_init(struct bcache_device *d, unsigned int block_size,
 
 	if (!d->stripe_size)
 		d->stripe_size = 1 << 31;
+	else if (d->stripe_size < BCH_MIN_STRIPE_SZ)
+		d->stripe_size = roundup(BCH_MIN_STRIPE_SZ, d->stripe_size);
 
 	n = DIV_ROUND_UP_ULL(sectors, d->stripe_size);
 	if (!n || n > max_stripes) {
@@ -1342,9 +1344,9 @@ void bch_cached_dev_release(struct kobject *kobj)
 	module_put(THIS_MODULE);
 }
 
-static void cached_dev_free(struct closure *cl)
+static CLOSURE_CALLBACK(cached_dev_free)
 {
-	struct cached_dev *dc = container_of(cl, struct cached_dev, disk.cl);
+	closure_type(dc, struct cached_dev, disk.cl);
 
 	if (test_and_clear_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags))
 		cancel_writeback_rate_update_dwork(dc);
@@ -1376,9 +1378,9 @@ static void cached_dev_free(struct closure *cl)
 	kobject_put(&dc->disk.kobj);
 }
 
-static void cached_dev_flush(struct closure *cl)
+static CLOSURE_CALLBACK(cached_dev_flush)
 {
-	struct cached_dev *dc = container_of(cl, struct cached_dev, disk.cl);
+	closure_type(dc, struct cached_dev, disk.cl);
 	struct bcache_device *d = &dc->disk;
 
 	mutex_lock(&bch_register_lock);
@@ -1497,9 +1499,9 @@ void bch_flash_dev_release(struct kobject *kobj)
 	kfree(d);
 }
 
-static void flash_dev_free(struct closure *cl)
+static CLOSURE_CALLBACK(flash_dev_free)
 {
-	struct bcache_device *d = container_of(cl, struct bcache_device, cl);
+	closure_type(d, struct bcache_device, cl);
 
 	mutex_lock(&bch_register_lock);
 	atomic_long_sub(bcache_dev_sectors_dirty(d),
@@ -1510,9 +1512,9 @@ static void flash_dev_free(struct closure *cl)
 	kobject_put(&d->kobj);
 }
 
-static void flash_dev_flush(struct closure *cl)
+static CLOSURE_CALLBACK(flash_dev_flush)
 {
-	struct bcache_device *d = container_of(cl, struct bcache_device, cl);
+	closure_type(d, struct bcache_device, cl);
 
 	mutex_lock(&bch_register_lock);
 	bcache_device_unlink(d);
@@ -1668,9 +1670,9 @@ void bch_cache_set_release(struct kobject *kobj)
 	module_put(THIS_MODULE);
 }
 
-static void cache_set_free(struct closure *cl)
+static CLOSURE_CALLBACK(cache_set_free)
 {
-	struct cache_set *c = container_of(cl, struct cache_set, cl);
+	closure_type(c, struct cache_set, cl);
 	struct cache *ca;
 
 	debugfs_remove(c->debug);
@@ -1709,9 +1711,9 @@ static void cache_set_free(struct closure *cl)
 	kobject_put(&c->kobj);
 }
 
-static void cache_set_flush(struct closure *cl)
+static CLOSURE_CALLBACK(cache_set_flush)
 {
-	struct cache_set *c = container_of(cl, struct cache_set, caching);
+	closure_type(c, struct cache_set, caching);
 	struct cache *ca = c->cache;
 	struct btree *b;
 
@@ -1806,9 +1808,9 @@ static void conditional_stop_bcache_device(struct cache_set *c,
 	}
 }
 
-static void __cache_set_unregister(struct closure *cl)
+static CLOSURE_CALLBACK(__cache_set_unregister)
 {
-	struct cache_set *c = container_of(cl, struct cache_set, caching);
+	closure_type(c, struct cache_set, caching);
 	struct cached_dev *dc;
 	struct bcache_device *d;
 	size_t i;
@@ -2016,7 +2018,7 @@ static int run_cache_set(struct cache_set *c)
 		c->root = bch_btree_node_get(c, NULL, k,
 					     j->btree_level,
 					     true, NULL);
-		if (IS_ERR_OR_NULL(c->root))
+		if (IS_ERR(c->root))
 			goto err;
 
 		list_del_init(&c->root->list);
diff --git a/drivers/md/bcache/sysfs.c b/drivers/md/bcache/sysfs.c
index 45d8af755de6..a438efb66069 100644
--- a/drivers/md/bcache/sysfs.c
+++ b/drivers/md/bcache/sysfs.c
@@ -1104,7 +1104,7 @@ SHOW(__bch_cache)
 			sum += INITIAL_PRIO - cached[i];
 
 		if (n)
-			do_div(sum, n);
+			sum = div64_u64(sum, n);
 
 		for (i = 0; i < ARRAY_SIZE(q); i++)
 			q[i] = INITIAL_PRIO - cached[n * (i + 1) /
diff --git a/drivers/md/bcache/writeback.c b/drivers/md/bcache/writeback.c
index 24c049067f61..8827a6f130ad 100644
--- a/drivers/md/bcache/writeback.c
+++ b/drivers/md/bcache/writeback.c
@@ -341,16 +341,16 @@ static void dirty_init(struct keybuf_key *w)
 	bch_bio_map(bio, NULL);
 }
 
-static void dirty_io_destructor(struct closure *cl)
+static CLOSURE_CALLBACK(dirty_io_destructor)
 {
-	struct dirty_io *io = container_of(cl, struct dirty_io, cl);
+	closure_type(io, struct dirty_io, cl);
 
 	kfree(io);
 }
 
-static void write_dirty_finish(struct closure *cl)
+static CLOSURE_CALLBACK(write_dirty_finish)
 {
-	struct dirty_io *io = container_of(cl, struct dirty_io, cl);
+	closure_type(io, struct dirty_io, cl);
 	struct keybuf_key *w = io->bio.bi_private;
 	struct cached_dev *dc = io->dc;
 
@@ -400,9 +400,9 @@ static void dirty_endio(struct bio *bio)
 	closure_put(&io->cl);
 }
 
-static void write_dirty(struct closure *cl)
+static CLOSURE_CALLBACK(write_dirty)
 {
-	struct dirty_io *io = container_of(cl, struct dirty_io, cl);
+	closure_type(io, struct dirty_io, cl);
 	struct keybuf_key *w = io->bio.bi_private;
 	struct cached_dev *dc = io->dc;
 
@@ -462,9 +462,9 @@ static void read_dirty_endio(struct bio *bio)
 	dirty_endio(bio);
 }
 
-static void read_dirty_submit(struct closure *cl)
+static CLOSURE_CALLBACK(read_dirty_submit)
 {
-	struct dirty_io *io = container_of(cl, struct dirty_io, cl);
+	closure_type(io, struct dirty_io, cl);
 
 	closure_bio_submit(io->dc->disk.c, &io->bio, cl);
 
@@ -913,7 +913,7 @@ static int bch_dirty_init_thread(void *arg)
 	int cur_idx, prev_idx, skip_nr;
 
 	k = p = NULL;
-	cur_idx = prev_idx = 0;
+	prev_idx = 0;
 
 	bch_btree_iter_init(&c->root->keys, &iter, NULL);
 	k = bch_btree_iter_next_filter(&iter, &c->root->keys, bch_ptr_bad);
@@ -977,24 +977,35 @@ static int bch_btre_dirty_init_thread_nr(void)
 void bch_sectors_dirty_init(struct bcache_device *d)
 {
 	int i;
+	struct btree *b = NULL;
 	struct bkey *k = NULL;
 	struct btree_iter iter;
 	struct sectors_dirty_init op;
 	struct cache_set *c = d->c;
 	struct bch_dirty_init_state state;
 
+retry_lock:
+	b = c->root;
+	rw_lock(0, b, b->level);
+	if (b != c->root) {
+		rw_unlock(0, b);
+		goto retry_lock;
+	}
+
 	/* Just count root keys if no leaf node */
-	rw_lock(0, c->root, c->root->level);
 	if (c->root->level == 0) {
 		bch_btree_op_init(&op.op, -1);
 		op.inode = d->id;
 		op.count = 0;
 
 		for_each_key_filter(&c->root->keys,
-				    k, &iter, bch_ptr_invalid)
+				    k, &iter, bch_ptr_invalid) {
+			if (KEY_INODE(k) != op.inode)
+				continue;
 			sectors_dirty_init_fn(&op.op, c->root, k);
+		}
 
-		rw_unlock(0, c->root);
+		rw_unlock(0, b);
 		return;
 	}
 
@@ -1014,23 +1025,24 @@ void bch_sectors_dirty_init(struct bcache_device *d)
 		if (atomic_read(&state.enough))
 			break;
 
+		atomic_inc(&state.started);
 		state.infos[i].state = &state;
 		state.infos[i].thread =
 			kthread_run(bch_dirty_init_thread, &state.infos[i],
 				    "bch_dirtcnt[%d]", i);
 		if (IS_ERR(state.infos[i].thread)) {
 			pr_err("fails to run thread bch_dirty_init[%d]\n", i);
+			atomic_dec(&state.started);
 			for (--i; i >= 0; i--)
 				kthread_stop(state.infos[i].thread);
 			goto out;
 		}
-		atomic_inc(&state.started);
 	}
 
 out:
 	/* Must wait for all threads to stop. */
 	wait_event(state.wait, atomic_read(&state.started) == 0);
-	rw_unlock(0, c->root);
+	rw_unlock(0, b);
 }
 
 void bch_cached_dev_writeback_init(struct cached_dev *dc)
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 23c32cd1f1d8..8dcabf84d866 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -2675,7 +2675,7 @@ static int lock_fs(struct mapped_device *md)
 
 	WARN_ON(test_bit(DMF_FROZEN, &md->flags));
 
-	r = freeze_bdev(md->disk->part0);
+	r = bdev_freeze(md->disk->part0);
 	if (!r)
 		set_bit(DMF_FROZEN, &md->flags);
 	return r;
@@ -2685,7 +2685,7 @@ static void unlock_fs(struct mapped_device *md)
 {
 	if (!test_bit(DMF_FROZEN, &md->flags))
 		return;
-	thaw_bdev(md->disk->part0);
+	bdev_thaw(md->disk->part0);
 	clear_bit(DMF_FROZEN, &md->flags);
 }
 
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 4ee4593c874a..c94373d64f2c 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -8666,7 +8666,8 @@ static void md_end_clone_io(struct bio *bio)
 	struct bio *orig_bio = md_io_clone->orig_bio;
 	struct mddev *mddev = md_io_clone->mddev;
 
-	orig_bio->bi_status = bio->bi_status;
+	if (bio->bi_status && !orig_bio->bi_status)
+		orig_bio->bi_status = bio->bi_status;
 
 	if (md_io_clone->start_time)
 		bio_end_io_acct(orig_bio, md_io_clone->start_time);
diff --git a/drivers/media/pci/mgb4/Kconfig b/drivers/media/pci/mgb4/Kconfig
index 13fad15a434c..f2a05a1c8ffa 100644
--- a/drivers/media/pci/mgb4/Kconfig
+++ b/drivers/media/pci/mgb4/Kconfig
@@ -2,6 +2,7 @@
 config VIDEO_MGB4
 	tristate "Digiteq Automotive MGB4 support"
 	depends on VIDEO_DEV && PCI && I2C && DMADEVICES && SPI && MTD && IIO
+	depends on COMMON_CLK
 	select VIDEOBUF2_DMA_SG
 	select IIO_BUFFER
 	select IIO_TRIGGERED_BUFFER
diff --git a/drivers/media/pci/mgb4/mgb4_core.c b/drivers/media/pci/mgb4/mgb4_core.c
index 3efb33fbf40c..5bfb8a06202e 100644
--- a/drivers/media/pci/mgb4/mgb4_core.c
+++ b/drivers/media/pci/mgb4/mgb4_core.c
@@ -42,6 +42,10 @@
 
 #define MGB4_USER_IRQS 16
 
+#define DIGITEQ_VID 0x1ed8
+#define T100_DID    0x0101
+#define T200_DID    0x0201
+
 ATTRIBUTE_GROUPS(mgb4_pci);
 
 static int flashid;
@@ -151,7 +155,7 @@ static struct spi_master *get_spi_adap(struct platform_device *pdev)
 	return dev ? container_of(dev, struct spi_master, dev) : NULL;
 }
 
-static int init_spi(struct mgb4_dev *mgbdev)
+static int init_spi(struct mgb4_dev *mgbdev, u32 devid)
 {
 	struct resource spi_resources[] = {
 		{
@@ -213,8 +217,13 @@ static int init_spi(struct mgb4_dev *mgbdev)
 	snprintf(mgbdev->fw_part_name, sizeof(mgbdev->fw_part_name),
 		 "mgb4-fw.%d", flashid);
 	mgbdev->partitions[0].name = mgbdev->fw_part_name;
-	mgbdev->partitions[0].size = 0x400000;
-	mgbdev->partitions[0].offset = 0x400000;
+	if (devid == T200_DID) {
+		mgbdev->partitions[0].size = 0x950000;
+		mgbdev->partitions[0].offset = 0x1000000;
+	} else {
+		mgbdev->partitions[0].size = 0x400000;
+		mgbdev->partitions[0].offset = 0x400000;
+	}
 	mgbdev->partitions[0].mask_flags = 0;
 
 	snprintf(mgbdev->data_part_name, sizeof(mgbdev->data_part_name),
@@ -551,7 +560,7 @@ static int mgb4_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		goto err_video_regs;
 
 	/* SPI FLASH */
-	rv = init_spi(mgbdev);
+	rv = init_spi(mgbdev, id->device);
 	if (rv < 0)
 		goto err_cmt_regs;
 
@@ -666,7 +675,8 @@ static void mgb4_remove(struct pci_dev *pdev)
 }
 
 static const struct pci_device_id mgb4_pci_ids[] = {
-	{ PCI_DEVICE(0x1ed8, 0x0101), },
+	{ PCI_DEVICE(DIGITEQ_VID, T100_DID), },
+	{ PCI_DEVICE(DIGITEQ_VID, T200_DID), },
 	{ 0, }
 };
 MODULE_DEVICE_TABLE(pci, mgb4_pci_ids);
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_pipe.c b/drivers/media/platform/renesas/vsp1/vsp1_pipe.c
index f8093ba9539e..68d05243c3ee 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_pipe.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_pipe.c
@@ -373,7 +373,7 @@ int vsp1_pipeline_stop(struct vsp1_pipeline *pipe)
 			   (7 << VI6_DPR_SMPPT_TGW_SHIFT) |
 			   (VI6_DPR_NODE_UNUSED << VI6_DPR_SMPPT_PT_SHIFT));
 
-	v4l2_subdev_call(&pipe->output->entity.subdev, video, s_stream, 0);
+	vsp1_wpf_stop(pipe->output);
 
 	return ret;
 }
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_rpf.c b/drivers/media/platform/renesas/vsp1/vsp1_rpf.c
index 3b17f5fa4067..ea12c3f12c92 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_rpf.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_rpf.c
@@ -43,14 +43,6 @@ static inline void vsp1_rpf_write(struct vsp1_rwpf *rpf,
 			       data);
 }
 
-/* -----------------------------------------------------------------------------
- * V4L2 Subdevice Operations
- */
-
-static const struct v4l2_subdev_ops rpf_ops = {
-	.pad    = &vsp1_rwpf_pad_ops,
-};
-
 /* -----------------------------------------------------------------------------
  * VSP1 Entity Operations
  */
@@ -411,7 +403,7 @@ struct vsp1_rwpf *vsp1_rpf_create(struct vsp1_device *vsp1, unsigned int index)
 	rpf->entity.index = index;
 
 	sprintf(name, "rpf.%u", index);
-	ret = vsp1_entity_init(vsp1, &rpf->entity, name, 2, &rpf_ops,
+	ret = vsp1_entity_init(vsp1, &rpf->entity, name, 2, &vsp1_rwpf_subdev_ops,
 			       MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER);
 	if (ret < 0)
 		return ERR_PTR(ret);
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_rwpf.c b/drivers/media/platform/renesas/vsp1/vsp1_rwpf.c
index 22a82d218152..e0f87c8103ca 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_rwpf.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_rwpf.c
@@ -24,7 +24,7 @@ struct v4l2_rect *vsp1_rwpf_get_crop(struct vsp1_rwpf *rwpf,
 }
 
 /* -----------------------------------------------------------------------------
- * V4L2 Subdevice Pad Operations
+ * V4L2 Subdevice Operations
  */
 
 static int vsp1_rwpf_enum_mbus_code(struct v4l2_subdev *subdev,
@@ -243,7 +243,7 @@ static int vsp1_rwpf_set_selection(struct v4l2_subdev *subdev,
 	return ret;
 }
 
-const struct v4l2_subdev_pad_ops vsp1_rwpf_pad_ops = {
+static const struct v4l2_subdev_pad_ops vsp1_rwpf_pad_ops = {
 	.init_cfg = vsp1_entity_init_cfg,
 	.enum_mbus_code = vsp1_rwpf_enum_mbus_code,
 	.enum_frame_size = vsp1_rwpf_enum_frame_size,
@@ -253,6 +253,10 @@ const struct v4l2_subdev_pad_ops vsp1_rwpf_pad_ops = {
 	.set_selection = vsp1_rwpf_set_selection,
 };
 
+const struct v4l2_subdev_ops vsp1_rwpf_subdev_ops = {
+	.pad    = &vsp1_rwpf_pad_ops,
+};
+
 /* -----------------------------------------------------------------------------
  * Controls
  */
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_rwpf.h b/drivers/media/platform/renesas/vsp1/vsp1_rwpf.h
index eac5c04c2239..e0d212c70b2f 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_rwpf.h
+++ b/drivers/media/platform/renesas/vsp1/vsp1_rwpf.h
@@ -79,9 +79,11 @@ static inline struct vsp1_rwpf *entity_to_rwpf(struct vsp1_entity *entity)
 struct vsp1_rwpf *vsp1_rpf_create(struct vsp1_device *vsp1, unsigned int index);
 struct vsp1_rwpf *vsp1_wpf_create(struct vsp1_device *vsp1, unsigned int index);
 
+void vsp1_wpf_stop(struct vsp1_rwpf *wpf);
+
 int vsp1_rwpf_init_ctrls(struct vsp1_rwpf *rwpf, unsigned int ncontrols);
 
-extern const struct v4l2_subdev_pad_ops vsp1_rwpf_pad_ops;
+extern const struct v4l2_subdev_ops vsp1_rwpf_subdev_ops;
 
 struct v4l2_rect *vsp1_rwpf_get_crop(struct vsp1_rwpf *rwpf,
 				     struct v4l2_subdev_state *sd_state);
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_wpf.c b/drivers/media/platform/renesas/vsp1/vsp1_wpf.c
index d0074ca00920..cab4445eca69 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_wpf.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_wpf.c
@@ -186,17 +186,13 @@ static int wpf_init_controls(struct vsp1_rwpf *wpf)
 }
 
 /* -----------------------------------------------------------------------------
- * V4L2 Subdevice Core Operations
+ * VSP1 Entity Operations
  */
 
-static int wpf_s_stream(struct v4l2_subdev *subdev, int enable)
+void vsp1_wpf_stop(struct vsp1_rwpf *wpf)
 {
-	struct vsp1_rwpf *wpf = to_rwpf(subdev);
 	struct vsp1_device *vsp1 = wpf->entity.vsp1;
 
-	if (enable)
-		return 0;
-
 	/*
 	 * Write to registers directly when stopping the stream as there will be
 	 * no pipeline run to apply the display list.
@@ -204,27 +200,8 @@ static int wpf_s_stream(struct v4l2_subdev *subdev, int enable)
 	vsp1_write(vsp1, VI6_WPF_IRQ_ENB(wpf->entity.index), 0);
 	vsp1_write(vsp1, wpf->entity.index * VI6_WPF_OFFSET +
 		   VI6_WPF_SRCRPF, 0);
-
-	return 0;
 }
 
-/* -----------------------------------------------------------------------------
- * V4L2 Subdevice Operations
- */
-
-static const struct v4l2_subdev_video_ops wpf_video_ops = {
-	.s_stream = wpf_s_stream,
-};
-
-static const struct v4l2_subdev_ops wpf_ops = {
-	.video	= &wpf_video_ops,
-	.pad    = &vsp1_rwpf_pad_ops,
-};
-
-/* -----------------------------------------------------------------------------
- * VSP1 Entity Operations
- */
-
 static void vsp1_wpf_destroy(struct vsp1_entity *entity)
 {
 	struct vsp1_rwpf *wpf = entity_to_rwpf(entity);
@@ -583,7 +560,7 @@ struct vsp1_rwpf *vsp1_wpf_create(struct vsp1_device *vsp1, unsigned int index)
 	wpf->entity.index = index;
 
 	sprintf(name, "wpf.%u", index);
-	ret = vsp1_entity_init(vsp1, &wpf->entity, name, 2, &wpf_ops,
+	ret = vsp1_entity_init(vsp1, &wpf->entity, name, 2, &vsp1_rwpf_subdev_ops,
 			       MEDIA_ENT_F_PROC_VIDEO_PIXEL_FORMATTER);
 	if (ret < 0)
 		return ERR_PTR(ret);
diff --git a/drivers/memory/tegra/tegra186.c b/drivers/memory/tegra/tegra186.c
index 533f85a4b2bd..d1f1dfb42716 100644
--- a/drivers/memory/tegra/tegra186.c
+++ b/drivers/memory/tegra/tegra186.c
@@ -75,6 +75,9 @@ static void tegra186_mc_client_sid_override(struct tegra_mc *mc,
 {
 	u32 value, old;
 
+	if (client->regs.sid.security == 0 && client->regs.sid.override == 0)
+		return;
+
 	value = readl(mc->regs + client->regs.sid.security);
 	if ((value & MC_SID_STREAMID_SECURITY_OVERRIDE) == 0) {
 		/*
@@ -136,9 +139,23 @@ static int tegra186_mc_probe_device(struct tegra_mc *mc, struct device *dev)
 	return 0;
 }
 
+static int tegra186_mc_resume(struct tegra_mc *mc)
+{
+	unsigned int i;
+
+	for (i = 0; i < mc->soc->num_clients; i++) {
+		const struct tegra_mc_client *client = &mc->soc->clients[i];
+
+		tegra186_mc_client_sid_override(mc, client, client->sid);
+	}
+
+	return 0;
+}
+
 const struct tegra_mc_ops tegra186_mc_ops = {
 	.probe = tegra186_mc_probe,
 	.remove = tegra186_mc_remove,
+	.resume = tegra186_mc_resume,
 	.probe_device = tegra186_mc_probe_device,
 	.handle_irq = tegra30_mc_handle_irq,
 };
diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c
index 152dfe593c43..f9a5cffa64b1 100644
--- a/drivers/mmc/core/block.c
+++ b/drivers/mmc/core/block.c
@@ -1482,6 +1482,8 @@ static void mmc_blk_cqe_complete_rq(struct mmc_queue *mq, struct request *req)
 			blk_mq_requeue_request(req, true);
 		else
 			__blk_mq_end_request(req, BLK_STS_OK);
+	} else if (mq->in_recovery) {
+		blk_mq_requeue_request(req, true);
 	} else {
 		blk_mq_end_request(req, BLK_STS_OK);
 	}
diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c
index 3d3e0ca52614..a8c17b4cd737 100644
--- a/drivers/mmc/core/core.c
+++ b/drivers/mmc/core/core.c
@@ -551,7 +551,9 @@ int mmc_cqe_recovery(struct mmc_host *host)
 	cmd.flags        = MMC_RSP_R1B | MMC_CMD_AC;
 	cmd.flags       &= ~MMC_RSP_CRC; /* Ignore CRC */
 	cmd.busy_timeout = MMC_CQE_RECOVERY_TIMEOUT;
-	mmc_wait_for_cmd(host, &cmd, 0);
+	mmc_wait_for_cmd(host, &cmd, MMC_CMD_RETRIES);
+
+	mmc_poll_for_busy(host->card, MMC_CQE_RECOVERY_TIMEOUT, true, MMC_BUSY_IO);
 
 	memset(&cmd, 0, sizeof(cmd));
 	cmd.opcode       = MMC_CMDQ_TASK_MGMT;
@@ -559,10 +561,13 @@ int mmc_cqe_recovery(struct mmc_host *host)
 	cmd.flags        = MMC_RSP_R1B | MMC_CMD_AC;
 	cmd.flags       &= ~MMC_RSP_CRC; /* Ignore CRC */
 	cmd.busy_timeout = MMC_CQE_RECOVERY_TIMEOUT;
-	err = mmc_wait_for_cmd(host, &cmd, 0);
+	err = mmc_wait_for_cmd(host, &cmd, MMC_CMD_RETRIES);
 
 	host->cqe_ops->cqe_recovery_finish(host);
 
+	if (err)
+		err = mmc_wait_for_cmd(host, &cmd, MMC_CMD_RETRIES);
+
 	mmc_retune_release(host);
 
 	return err;
diff --git a/drivers/mmc/core/mmc.c b/drivers/mmc/core/mmc.c
index 705942edacc6..7b996db570c9 100644
--- a/drivers/mmc/core/mmc.c
+++ b/drivers/mmc/core/mmc.c
@@ -613,11 +613,6 @@ static int mmc_decode_ext_csd(struct mmc_card *card, u8 *ext_csd)
 		} else {
 			card->ext_csd.data_tag_unit_size = 0;
 		}
-
-		card->ext_csd.max_packed_writes =
-			ext_csd[EXT_CSD_MAX_PACKED_WRITES];
-		card->ext_csd.max_packed_reads =
-			ext_csd[EXT_CSD_MAX_PACKED_READS];
 	} else {
 		card->ext_csd.data_sector_size = 512;
 	}
diff --git a/drivers/mmc/host/cqhci-core.c b/drivers/mmc/host/cqhci-core.c
index b3d7d6d8d654..41e94cd14109 100644
--- a/drivers/mmc/host/cqhci-core.c
+++ b/drivers/mmc/host/cqhci-core.c
@@ -942,8 +942,8 @@ static bool cqhci_clear_all_tasks(struct mmc_host *mmc, unsigned int timeout)
 	ret = cqhci_tasks_cleared(cq_host);
 
 	if (!ret)
-		pr_debug("%s: cqhci: Failed to clear tasks\n",
-			 mmc_hostname(mmc));
+		pr_warn("%s: cqhci: Failed to clear tasks\n",
+			mmc_hostname(mmc));
 
 	return ret;
 }
@@ -976,7 +976,7 @@ static bool cqhci_halt(struct mmc_host *mmc, unsigned int timeout)
 	ret = cqhci_halted(cq_host);
 
 	if (!ret)
-		pr_debug("%s: cqhci: Failed to halt\n", mmc_hostname(mmc));
+		pr_warn("%s: cqhci: Failed to halt\n", mmc_hostname(mmc));
 
 	return ret;
 }
@@ -984,10 +984,10 @@ static bool cqhci_halt(struct mmc_host *mmc, unsigned int timeout)
 /*
  * After halting we expect to be able to use the command line. We interpret the
  * failure to halt to mean the data lines might still be in use (and the upper
- * layers will need to send a STOP command), so we set the timeout based on a
- * generous command timeout.
+ * layers will need to send a STOP command), however failing to halt complicates
+ * the recovery, so set a timeout that would reasonably allow I/O to complete.
  */
-#define CQHCI_START_HALT_TIMEOUT	5
+#define CQHCI_START_HALT_TIMEOUT	500
 
 static void cqhci_recovery_start(struct mmc_host *mmc)
 {
@@ -1075,28 +1075,28 @@ static void cqhci_recovery_finish(struct mmc_host *mmc)
 
 	ok = cqhci_halt(mmc, CQHCI_FINISH_HALT_TIMEOUT);
 
-	if (!cqhci_clear_all_tasks(mmc, CQHCI_CLEAR_TIMEOUT))
-		ok = false;
-
 	/*
 	 * The specification contradicts itself, by saying that tasks cannot be
 	 * cleared if CQHCI does not halt, but if CQHCI does not halt, it should
 	 * be disabled/re-enabled, but not to disable before clearing tasks.
 	 * Have a go anyway.
 	 */
-	if (!ok) {
-		pr_debug("%s: cqhci: disable / re-enable\n", mmc_hostname(mmc));
-		cqcfg = cqhci_readl(cq_host, CQHCI_CFG);
-		cqcfg &= ~CQHCI_ENABLE;
-		cqhci_writel(cq_host, cqcfg, CQHCI_CFG);
-		cqcfg |= CQHCI_ENABLE;
-		cqhci_writel(cq_host, cqcfg, CQHCI_CFG);
-		/* Be sure that there are no tasks */
-		ok = cqhci_halt(mmc, CQHCI_FINISH_HALT_TIMEOUT);
-		if (!cqhci_clear_all_tasks(mmc, CQHCI_CLEAR_TIMEOUT))
-			ok = false;
-		WARN_ON(!ok);
-	}
+	if (!cqhci_clear_all_tasks(mmc, CQHCI_CLEAR_TIMEOUT))
+		ok = false;
+
+	/* Disable to make sure tasks really are cleared */
+	cqcfg = cqhci_readl(cq_host, CQHCI_CFG);
+	cqcfg &= ~CQHCI_ENABLE;
+	cqhci_writel(cq_host, cqcfg, CQHCI_CFG);
+
+	cqcfg = cqhci_readl(cq_host, CQHCI_CFG);
+	cqcfg |= CQHCI_ENABLE;
+	cqhci_writel(cq_host, cqcfg, CQHCI_CFG);
+
+	cqhci_halt(mmc, CQHCI_FINISH_HALT_TIMEOUT);
+
+	if (!ok)
+		cqhci_clear_all_tasks(mmc, CQHCI_CLEAR_TIMEOUT);
 
 	cqhci_recover_mrqs(cq_host);
 
diff --git a/drivers/mmc/host/meson-mx-sdhc-mmc.c b/drivers/mmc/host/meson-mx-sdhc-mmc.c
index 528ec8166e7c..1ed9731e77ef 100644
--- a/drivers/mmc/host/meson-mx-sdhc-mmc.c
+++ b/drivers/mmc/host/meson-mx-sdhc-mmc.c
@@ -269,7 +269,7 @@ static int meson_mx_sdhc_enable_clks(struct mmc_host *mmc)
 static int meson_mx_sdhc_set_clk(struct mmc_host *mmc, struct mmc_ios *ios)
 {
 	struct meson_mx_sdhc_host *host = mmc_priv(mmc);
-	u32 rx_clk_phase;
+	u32 val, rx_clk_phase;
 	int ret;
 
 	meson_mx_sdhc_disable_clks(mmc);
@@ -290,27 +290,11 @@ static int meson_mx_sdhc_set_clk(struct mmc_host *mmc, struct mmc_ios *ios)
 		mmc->actual_clock = clk_get_rate(host->sd_clk);
 
 		/*
-		 * according to Amlogic the following latching points are
-		 * selected with empirical values, there is no (known) formula
-		 * to calculate these.
+		 * Phase 90 should work in most cases. For data transmission,
+		 * meson_mx_sdhc_execute_tuning() will find a accurate value
 		 */
-		if (mmc->actual_clock > 100000000) {
-			rx_clk_phase = 1;
-		} else if (mmc->actual_clock > 45000000) {
-			if (ios->signal_voltage == MMC_SIGNAL_VOLTAGE_330)
-				rx_clk_phase = 15;
-			else
-				rx_clk_phase = 11;
-		} else if (mmc->actual_clock >= 25000000) {
-			rx_clk_phase = 15;
-		} else if (mmc->actual_clock > 5000000) {
-			rx_clk_phase = 23;
-		} else if (mmc->actual_clock > 1000000) {
-			rx_clk_phase = 55;
-		} else {
-			rx_clk_phase = 1061;
-		}
-
+		regmap_read(host->regmap, MESON_SDHC_CLKC, &val);
+		rx_clk_phase = FIELD_GET(MESON_SDHC_CLKC_CLK_DIV, val) / 4;
 		regmap_update_bits(host->regmap, MESON_SDHC_CLK2,
 				   MESON_SDHC_CLK2_RX_CLK_PHASE,
 				   FIELD_PREP(MESON_SDHC_CLK2_RX_CLK_PHASE,
diff --git a/drivers/mmc/host/mmc_spi.c b/drivers/mmc/host/mmc_spi.c
index cc333ad67cac..b0cccef4cfbf 100644
--- a/drivers/mmc/host/mmc_spi.c
+++ b/drivers/mmc/host/mmc_spi.c
@@ -1322,7 +1322,7 @@ static int mmc_spi_probe(struct spi_device *spi)
 	/* We rely on full duplex transfers, mostly to reduce
 	 * per-transfer overheads (by making fewer transfers).
 	 */
-	if (spi->master->flags & SPI_MASTER_HALF_DUPLEX)
+	if (spi->master->flags & SPI_CONTROLLER_HALF_DUPLEX)
 		return -EINVAL;
 
 	/* MMC and SD specs only seem to care that sampling is on the
diff --git a/drivers/mmc/host/mmci.c b/drivers/mmc/host/mmci.c
index e967cca7a16f..b790c3c3c8f9 100644
--- a/drivers/mmc/host/mmci.c
+++ b/drivers/mmc/host/mmci.c
@@ -273,6 +273,7 @@ static struct variant_data variant_stm32_sdmmc = {
 	.datactrl_mask_sdio	= MCI_DPSM_ST_SDIOEN,
 	.stm32_idmabsize_mask	= GENMASK(12, 5),
 	.stm32_idmabsize_align	= BIT(5),
+	.supports_sdio_irq	= true,
 	.busy_timeout		= true,
 	.busy_detect		= true,
 	.busy_detect_flag	= MCI_STM32_BUSYD0,
@@ -300,6 +301,7 @@ static struct variant_data variant_stm32_sdmmcv2 = {
 	.datactrl_mask_sdio	= MCI_DPSM_ST_SDIOEN,
 	.stm32_idmabsize_mask	= GENMASK(16, 5),
 	.stm32_idmabsize_align	= BIT(5),
+	.supports_sdio_irq	= true,
 	.dma_lli		= true,
 	.busy_timeout		= true,
 	.busy_detect		= true,
@@ -328,6 +330,7 @@ static struct variant_data variant_stm32_sdmmcv3 = {
 	.datactrl_mask_sdio	= MCI_DPSM_ST_SDIOEN,
 	.stm32_idmabsize_mask	= GENMASK(16, 6),
 	.stm32_idmabsize_align	= BIT(6),
+	.supports_sdio_irq	= true,
 	.dma_lli		= true,
 	.busy_timeout		= true,
 	.busy_detect		= true,
@@ -421,8 +424,9 @@ void mmci_write_pwrreg(struct mmci_host *host, u32 pwr)
  */
 static void mmci_write_datactrlreg(struct mmci_host *host, u32 datactrl)
 {
-	/* Keep busy mode in DPSM if enabled */
-	datactrl |= host->datactrl_reg & host->variant->busy_dpsm_flag;
+	/* Keep busy mode in DPSM and SDIO mask if enabled */
+	datactrl |= host->datactrl_reg & (host->variant->busy_dpsm_flag |
+					  host->variant->datactrl_mask_sdio);
 
 	if (host->datactrl_reg != datactrl) {
 		host->datactrl_reg = datactrl;
@@ -1762,6 +1766,25 @@ static irqreturn_t mmci_pio_irq(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
+static void mmci_write_sdio_irq_bit(struct mmci_host *host, int enable)
+{
+	void __iomem *base = host->base;
+	u32 mask = readl_relaxed(base + MMCIMASK0);
+
+	if (enable)
+		writel_relaxed(mask | MCI_ST_SDIOITMASK, base + MMCIMASK0);
+	else
+		writel_relaxed(mask & ~MCI_ST_SDIOITMASK, base + MMCIMASK0);
+}
+
+static void mmci_signal_sdio_irq(struct mmci_host *host, u32 status)
+{
+	if (status & MCI_ST_SDIOIT) {
+		mmci_write_sdio_irq_bit(host, 0);
+		sdio_signal_irq(host->mmc);
+	}
+}
+
 /*
  * Handle completion of command and data transfers.
  */
@@ -1806,6 +1829,9 @@ static irqreturn_t mmci_irq(int irq, void *dev_id)
 			mmci_data_irq(host, host->data, status);
 		}
 
+		if (host->variant->supports_sdio_irq)
+			mmci_signal_sdio_irq(host, status);
+
 		/*
 		 * Busy detection has been handled by mmci_cmd_irq() above.
 		 * Clear the status bit to prevent polling in IRQ context.
@@ -2042,6 +2068,35 @@ static int mmci_sig_volt_switch(struct mmc_host *mmc, struct mmc_ios *ios)
 	return ret;
 }
 
+static void mmci_enable_sdio_irq(struct mmc_host *mmc, int enable)
+{
+	struct mmci_host *host = mmc_priv(mmc);
+	unsigned long flags;
+
+	if (enable)
+		/* Keep the SDIO mode bit if SDIO irqs are enabled */
+		pm_runtime_get_sync(mmc_dev(mmc));
+
+	spin_lock_irqsave(&host->lock, flags);
+	mmci_write_sdio_irq_bit(host, enable);
+	spin_unlock_irqrestore(&host->lock, flags);
+
+	if (!enable) {
+		pm_runtime_mark_last_busy(mmc_dev(mmc));
+		pm_runtime_put_autosuspend(mmc_dev(mmc));
+	}
+}
+
+static void mmci_ack_sdio_irq(struct mmc_host *mmc)
+{
+	struct mmci_host *host = mmc_priv(mmc);
+	unsigned long flags;
+
+	spin_lock_irqsave(&host->lock, flags);
+	mmci_write_sdio_irq_bit(host, 1);
+	spin_unlock_irqrestore(&host->lock, flags);
+}
+
 static struct mmc_host_ops mmci_ops = {
 	.request	= mmci_request,
 	.pre_req	= mmci_pre_request,
@@ -2317,6 +2372,16 @@ static int mmci_probe(struct amba_device *dev,
 		mmc->caps |= MMC_CAP_WAIT_WHILE_BUSY;
 	}
 
+	if (variant->supports_sdio_irq && host->mmc->caps & MMC_CAP_SDIO_IRQ) {
+		mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD;
+
+		mmci_ops.enable_sdio_irq = mmci_enable_sdio_irq;
+		mmci_ops.ack_sdio_irq	= mmci_ack_sdio_irq;
+
+		mmci_write_datactrlreg(host,
+				       host->variant->datactrl_mask_sdio);
+	}
+
 	/* Variants with mandatory busy timeout in HW needs R1B responses. */
 	if (variant->busy_timeout)
 		mmc->caps |= MMC_CAP_NEED_RSP_BUSY;
diff --git a/drivers/mmc/host/mmci.h b/drivers/mmc/host/mmci.h
index 34d9897c289b..a5eb4ced4d5d 100644
--- a/drivers/mmc/host/mmci.h
+++ b/drivers/mmc/host/mmci.h
@@ -331,6 +331,7 @@ enum mmci_busy_state {
  *	       register.
  * @opendrain: bitmask identifying the OPENDRAIN bit inside MMCIPOWER register
  * @dma_lli: true if variant has dma link list feature.
+ * @supports_sdio_irq: allow SD I/O card to interrupt the host
  * @stm32_idmabsize_mask: stm32 sdmmc idma buffer size.
  * @dma_flow_controller: use peripheral as flow controller for DMA.
  */
@@ -377,6 +378,7 @@ struct variant_data {
 	u32			start_err;
 	u32			opendrain;
 	u8			dma_lli:1;
+	bool			supports_sdio_irq;
 	u32			stm32_idmabsize_mask;
 	u32			stm32_idmabsize_align;
 	bool			dma_flow_controller;
diff --git a/drivers/mmc/host/sdhci-of-dwcmshc.c b/drivers/mmc/host/sdhci-of-dwcmshc.c
index 3a3bae6948a8..0eb72544c09e 100644
--- a/drivers/mmc/host/sdhci-of-dwcmshc.c
+++ b/drivers/mmc/host/sdhci-of-dwcmshc.c
@@ -8,6 +8,7 @@
  */
 
 #include <linux/acpi.h>
+#include <linux/bitfield.h>
 #include <linux/clk.h>
 #include <linux/dma-mapping.h>
 #include <linux/iopoll.h>
@@ -35,6 +36,21 @@
 #define DWCMSHC_CARD_IS_EMMC		BIT(0)
 #define DWCMSHC_ENHANCED_STROBE		BIT(8)
 #define DWCMSHC_EMMC_ATCTRL		0x40
+/* Tuning and auto-tuning fields in AT_CTRL_R control register */
+#define AT_CTRL_AT_EN			BIT(0) /* autotuning is enabled */
+#define AT_CTRL_CI_SEL			BIT(1) /* interval to drive center phase select */
+#define AT_CTRL_SWIN_TH_EN		BIT(2) /* sampling window threshold enable */
+#define AT_CTRL_RPT_TUNE_ERR		BIT(3) /* enable reporting framing errors */
+#define AT_CTRL_SW_TUNE_EN		BIT(4) /* enable software managed tuning */
+#define AT_CTRL_WIN_EDGE_SEL_MASK	GENMASK(11, 8) /* bits [11:8] */
+#define AT_CTRL_WIN_EDGE_SEL		0xf /* sampling window edge select */
+#define AT_CTRL_TUNE_CLK_STOP_EN	BIT(16) /* clocks stopped during phase code change */
+#define AT_CTRL_PRE_CHANGE_DLY_MASK	GENMASK(18, 17) /* bits [18:17] */
+#define AT_CTRL_PRE_CHANGE_DLY		0x1  /* 2-cycle latency */
+#define AT_CTRL_POST_CHANGE_DLY_MASK	GENMASK(20, 19) /* bits [20:19] */
+#define AT_CTRL_POST_CHANGE_DLY		0x3  /* 4-cycle latency */
+#define AT_CTRL_SWIN_TH_VAL_MASK	GENMASK(31, 24) /* bits [31:24] */
+#define AT_CTRL_SWIN_TH_VAL		0x9  /* sampling window threshold */
 
 /* Rockchip specific Registers */
 #define DWCMSHC_EMMC_DLL_CTRL		0x800
@@ -72,6 +88,82 @@
 	(((x) & DWCMSHC_EMMC_DLL_TIMEOUT) == 0))
 #define RK35xx_MAX_CLKS 3
 
+/* PHY register area pointer */
+#define DWC_MSHC_PTR_PHY_R	0x300
+
+/* PHY general configuration */
+#define PHY_CNFG_R		(DWC_MSHC_PTR_PHY_R + 0x00)
+#define PHY_CNFG_RSTN_DEASSERT	0x1  /* Deassert PHY reset */
+#define PHY_CNFG_PAD_SP_MASK	GENMASK(19, 16) /* bits [19:16] */
+#define PHY_CNFG_PAD_SP		0x0c /* PMOS TX drive strength */
+#define PHY_CNFG_PAD_SN_MASK	GENMASK(23, 20) /* bits [23:20] */
+#define PHY_CNFG_PAD_SN		0x0c /* NMOS TX drive strength */
+
+/* PHY command/response pad settings */
+#define PHY_CMDPAD_CNFG_R	(DWC_MSHC_PTR_PHY_R + 0x04)
+
+/* PHY data pad settings */
+#define PHY_DATAPAD_CNFG_R	(DWC_MSHC_PTR_PHY_R + 0x06)
+
+/* PHY clock pad settings */
+#define PHY_CLKPAD_CNFG_R	(DWC_MSHC_PTR_PHY_R + 0x08)
+
+/* PHY strobe pad settings */
+#define PHY_STBPAD_CNFG_R	(DWC_MSHC_PTR_PHY_R + 0x0a)
+
+/* PHY reset pad settings */
+#define PHY_RSTNPAD_CNFG_R	(DWC_MSHC_PTR_PHY_R + 0x0c)
+
+/* Bitfields are common for all pad settings */
+#define PHY_PAD_RXSEL_1V8		0x1 /* Receiver type select for 1.8V */
+#define PHY_PAD_RXSEL_3V3		0x2 /* Receiver type select for 3.3V */
+
+#define PHY_PAD_WEAKPULL_MASK		GENMASK(4, 3) /* bits [4:3] */
+#define PHY_PAD_WEAKPULL_PULLUP		0x1 /* Weak pull up enabled */
+#define PHY_PAD_WEAKPULL_PULLDOWN	0x2 /* Weak pull down enabled */
+
+#define PHY_PAD_TXSLEW_CTRL_P_MASK	GENMASK(8, 5) /* bits [8:5] */
+#define PHY_PAD_TXSLEW_CTRL_P		0x3 /* Slew control for P-Type pad TX */
+#define PHY_PAD_TXSLEW_CTRL_N_MASK	GENMASK(12, 9) /* bits [12:9] */
+#define PHY_PAD_TXSLEW_CTRL_N		0x3 /* Slew control for N-Type pad TX */
+
+/* PHY CLK delay line settings */
+#define PHY_SDCLKDL_CNFG_R		(DWC_MSHC_PTR_PHY_R + 0x1d)
+#define PHY_SDCLKDL_CNFG_UPDATE	BIT(4) /* set before writing to SDCLKDL_DC */
+
+/* PHY CLK delay line delay code */
+#define PHY_SDCLKDL_DC_R		(DWC_MSHC_PTR_PHY_R + 0x1e)
+#define PHY_SDCLKDL_DC_INITIAL		0x40 /* initial delay code */
+#define PHY_SDCLKDL_DC_DEFAULT		0x32 /* default delay code */
+#define PHY_SDCLKDL_DC_HS400		0x18 /* delay code for HS400 mode */
+
+/* PHY drift_cclk_rx delay line configuration setting */
+#define PHY_ATDL_CNFG_R			(DWC_MSHC_PTR_PHY_R + 0x21)
+#define PHY_ATDL_CNFG_INPSEL_MASK	GENMASK(3, 2) /* bits [3:2] */
+#define PHY_ATDL_CNFG_INPSEL		0x3 /* delay line input source */
+
+/* PHY DLL control settings */
+#define PHY_DLL_CTRL_R			(DWC_MSHC_PTR_PHY_R + 0x24)
+#define PHY_DLL_CTRL_DISABLE		0x0 /* PHY DLL is enabled */
+#define PHY_DLL_CTRL_ENABLE		0x1 /* PHY DLL is disabled */
+
+/* PHY DLL  configuration register 1 */
+#define PHY_DLL_CNFG1_R			(DWC_MSHC_PTR_PHY_R + 0x25)
+#define PHY_DLL_CNFG1_SLVDLY_MASK	GENMASK(5, 4) /* bits [5:4] */
+#define PHY_DLL_CNFG1_SLVDLY		0x2 /* DLL slave update delay input */
+#define PHY_DLL_CNFG1_WAITCYCLE		0x5 /* DLL wait cycle input */
+
+/* PHY DLL configuration register 2 */
+#define PHY_DLL_CNFG2_R			(DWC_MSHC_PTR_PHY_R + 0x26)
+#define PHY_DLL_CNFG2_JUMPSTEP		0xa /* DLL jump step input */
+
+/* PHY DLL master and slave delay line configuration settings */
+#define PHY_DLLDL_CNFG_R		(DWC_MSHC_PTR_PHY_R + 0x28)
+#define PHY_DLLDL_CNFG_SLV_INPSEL_MASK	GENMASK(6, 5) /* bits [6:5] */
+#define PHY_DLLDL_CNFG_SLV_INPSEL	0x3 /* clock source select for slave DL */
+
+#define FLAG_IO_FIXED_1V8	BIT(0)
+
 #define BOUNDARY_OK(addr, len) \
 	((addr | (SZ_128M - 1)) == ((addr + len - 1) | (SZ_128M - 1)))
 
@@ -92,6 +184,8 @@ struct dwcmshc_priv {
 	struct clk	*bus_clk;
 	int vendor_specific_area1; /* P_VENDOR_SPECIFIC_AREA reg */
 	void *priv; /* pointer to SoC private stuff */
+	u16 delay_line;
+	u16 flags;
 };
 
 /*
@@ -157,6 +251,127 @@ static void dwcmshc_request(struct mmc_host *mmc, struct mmc_request *mrq)
 	sdhci_request(mmc, mrq);
 }
 
+static void dwcmshc_phy_1_8v_init(struct sdhci_host *host)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct dwcmshc_priv *priv = sdhci_pltfm_priv(pltfm_host);
+	u32 val;
+
+	/* deassert phy reset & set tx drive strength */
+	val = PHY_CNFG_RSTN_DEASSERT;
+	val |= FIELD_PREP(PHY_CNFG_PAD_SP_MASK, PHY_CNFG_PAD_SP);
+	val |= FIELD_PREP(PHY_CNFG_PAD_SN_MASK, PHY_CNFG_PAD_SN);
+	sdhci_writel(host, val, PHY_CNFG_R);
+
+	/* disable delay line */
+	sdhci_writeb(host, PHY_SDCLKDL_CNFG_UPDATE, PHY_SDCLKDL_CNFG_R);
+
+	/* set delay line */
+	sdhci_writeb(host, priv->delay_line, PHY_SDCLKDL_DC_R);
+	sdhci_writeb(host, PHY_DLL_CNFG2_JUMPSTEP, PHY_DLL_CNFG2_R);
+
+	/* enable delay lane */
+	val = sdhci_readb(host, PHY_SDCLKDL_CNFG_R);
+	val &= ~(PHY_SDCLKDL_CNFG_UPDATE);
+	sdhci_writeb(host, val, PHY_SDCLKDL_CNFG_R);
+
+	/* configure phy pads */
+	val = PHY_PAD_RXSEL_1V8;
+	val |= FIELD_PREP(PHY_PAD_WEAKPULL_MASK, PHY_PAD_WEAKPULL_PULLUP);
+	val |= FIELD_PREP(PHY_PAD_TXSLEW_CTRL_P_MASK, PHY_PAD_TXSLEW_CTRL_P);
+	val |= FIELD_PREP(PHY_PAD_TXSLEW_CTRL_N_MASK, PHY_PAD_TXSLEW_CTRL_N);
+	sdhci_writew(host, val, PHY_CMDPAD_CNFG_R);
+	sdhci_writew(host, val, PHY_DATAPAD_CNFG_R);
+	sdhci_writew(host, val, PHY_RSTNPAD_CNFG_R);
+
+	val = FIELD_PREP(PHY_PAD_TXSLEW_CTRL_P_MASK, PHY_PAD_TXSLEW_CTRL_P);
+	val |= FIELD_PREP(PHY_PAD_TXSLEW_CTRL_N_MASK, PHY_PAD_TXSLEW_CTRL_N);
+	sdhci_writew(host, val, PHY_CLKPAD_CNFG_R);
+
+	val = PHY_PAD_RXSEL_1V8;
+	val |= FIELD_PREP(PHY_PAD_WEAKPULL_MASK, PHY_PAD_WEAKPULL_PULLDOWN);
+	val |= FIELD_PREP(PHY_PAD_TXSLEW_CTRL_P_MASK, PHY_PAD_TXSLEW_CTRL_P);
+	val |= FIELD_PREP(PHY_PAD_TXSLEW_CTRL_N_MASK, PHY_PAD_TXSLEW_CTRL_N);
+	sdhci_writew(host, val, PHY_STBPAD_CNFG_R);
+
+	/* enable data strobe mode */
+	sdhci_writeb(host, FIELD_PREP(PHY_DLLDL_CNFG_SLV_INPSEL_MASK, PHY_DLLDL_CNFG_SLV_INPSEL),
+		     PHY_DLLDL_CNFG_R);
+
+	/* enable phy dll */
+	sdhci_writeb(host, PHY_DLL_CTRL_ENABLE, PHY_DLL_CTRL_R);
+}
+
+static void dwcmshc_phy_3_3v_init(struct sdhci_host *host)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct dwcmshc_priv *priv = sdhci_pltfm_priv(pltfm_host);
+	u32 val;
+
+	/* deassert phy reset & set tx drive strength */
+	val = PHY_CNFG_RSTN_DEASSERT;
+	val |= FIELD_PREP(PHY_CNFG_PAD_SP_MASK, PHY_CNFG_PAD_SP);
+	val |= FIELD_PREP(PHY_CNFG_PAD_SN_MASK, PHY_CNFG_PAD_SN);
+	sdhci_writel(host, val, PHY_CNFG_R);
+
+	/* disable delay line */
+	sdhci_writeb(host, PHY_SDCLKDL_CNFG_UPDATE, PHY_SDCLKDL_CNFG_R);
+
+	/* set delay line */
+	sdhci_writeb(host, priv->delay_line, PHY_SDCLKDL_DC_R);
+	sdhci_writeb(host, PHY_DLL_CNFG2_JUMPSTEP, PHY_DLL_CNFG2_R);
+
+	/* enable delay lane */
+	val = sdhci_readb(host, PHY_SDCLKDL_CNFG_R);
+	val &= ~(PHY_SDCLKDL_CNFG_UPDATE);
+	sdhci_writeb(host, val, PHY_SDCLKDL_CNFG_R);
+
+	/* configure phy pads */
+	val = PHY_PAD_RXSEL_3V3;
+	val |= FIELD_PREP(PHY_PAD_WEAKPULL_MASK, PHY_PAD_WEAKPULL_PULLUP);
+	val |= FIELD_PREP(PHY_PAD_TXSLEW_CTRL_P_MASK, PHY_PAD_TXSLEW_CTRL_P);
+	val |= FIELD_PREP(PHY_PAD_TXSLEW_CTRL_N_MASK, PHY_PAD_TXSLEW_CTRL_N);
+	sdhci_writew(host, val, PHY_CMDPAD_CNFG_R);
+	sdhci_writew(host, val, PHY_DATAPAD_CNFG_R);
+	sdhci_writew(host, val, PHY_RSTNPAD_CNFG_R);
+
+	val = FIELD_PREP(PHY_PAD_TXSLEW_CTRL_P_MASK, PHY_PAD_TXSLEW_CTRL_P);
+	val |= FIELD_PREP(PHY_PAD_TXSLEW_CTRL_N_MASK, PHY_PAD_TXSLEW_CTRL_N);
+	sdhci_writew(host, val, PHY_CLKPAD_CNFG_R);
+
+	val = PHY_PAD_RXSEL_3V3;
+	val |= FIELD_PREP(PHY_PAD_WEAKPULL_MASK, PHY_PAD_WEAKPULL_PULLDOWN);
+	val |= FIELD_PREP(PHY_PAD_TXSLEW_CTRL_P_MASK, PHY_PAD_TXSLEW_CTRL_P);
+	val |= FIELD_PREP(PHY_PAD_TXSLEW_CTRL_N_MASK, PHY_PAD_TXSLEW_CTRL_N);
+	sdhci_writew(host, val, PHY_STBPAD_CNFG_R);
+
+	/* enable phy dll */
+	sdhci_writeb(host, PHY_DLL_CTRL_ENABLE, PHY_DLL_CTRL_R);
+}
+
+static void th1520_sdhci_set_phy(struct sdhci_host *host)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct dwcmshc_priv *priv = sdhci_pltfm_priv(pltfm_host);
+	u32 emmc_caps = MMC_CAP2_NO_SD | MMC_CAP2_NO_SDIO;
+	u16 emmc_ctrl;
+
+	/* Before power on, set PHY configs */
+	if (priv->flags & FLAG_IO_FIXED_1V8)
+		dwcmshc_phy_1_8v_init(host);
+	else
+		dwcmshc_phy_3_3v_init(host);
+
+	if ((host->mmc->caps2 & emmc_caps) == emmc_caps) {
+		emmc_ctrl = sdhci_readw(host, priv->vendor_specific_area1 + DWCMSHC_EMMC_CONTROL);
+		emmc_ctrl |= DWCMSHC_CARD_IS_EMMC;
+		sdhci_writew(host, emmc_ctrl, priv->vendor_specific_area1 + DWCMSHC_EMMC_CONTROL);
+	}
+
+	sdhci_writeb(host, FIELD_PREP(PHY_DLL_CNFG1_SLVDLY_MASK, PHY_DLL_CNFG1_SLVDLY) |
+		     PHY_DLL_CNFG1_WAITCYCLE, PHY_DLL_CNFG1_R);
+}
+
 static void dwcmshc_set_uhs_signaling(struct sdhci_host *host,
 				      unsigned int timing)
 {
@@ -189,9 +404,25 @@ static void dwcmshc_set_uhs_signaling(struct sdhci_host *host,
 		ctrl_2 |= DWCMSHC_CTRL_HS400;
 	}
 
+	if (priv->flags & FLAG_IO_FIXED_1V8)
+		ctrl_2 |= SDHCI_CTRL_VDD_180;
 	sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
 }
 
+static void th1520_set_uhs_signaling(struct sdhci_host *host,
+				     unsigned int timing)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct dwcmshc_priv *priv = sdhci_pltfm_priv(pltfm_host);
+
+	dwcmshc_set_uhs_signaling(host, timing);
+	if (timing == MMC_TIMING_MMC_HS400)
+		priv->delay_line = PHY_SDCLKDL_DC_HS400;
+	else
+		sdhci_writeb(host, 0, PHY_DLLDL_CNFG_R);
+	th1520_sdhci_set_phy(host);
+}
+
 static void dwcmshc_hs400_enhanced_strobe(struct mmc_host *mmc,
 					  struct mmc_ios *ios)
 {
@@ -338,6 +569,79 @@ static void rk35xx_sdhci_reset(struct sdhci_host *host, u8 mask)
 	sdhci_reset(host, mask);
 }
 
+static int th1520_execute_tuning(struct sdhci_host *host, u32 opcode)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct dwcmshc_priv *priv = sdhci_pltfm_priv(pltfm_host);
+	u32 val = 0;
+
+	if (host->flags & SDHCI_HS400_TUNING)
+		return 0;
+
+	sdhci_writeb(host, FIELD_PREP(PHY_ATDL_CNFG_INPSEL_MASK, PHY_ATDL_CNFG_INPSEL),
+		     PHY_ATDL_CNFG_R);
+	val = sdhci_readl(host, priv->vendor_specific_area1 + DWCMSHC_EMMC_ATCTRL);
+
+	/*
+	 * configure tuning settings:
+	 *  - center phase select code driven in block gap interval
+	 *  - disable reporting of framing errors
+	 *  - disable software managed tuning
+	 *  - disable user selection of sampling window edges,
+	 *    instead tuning calculated edges are used
+	 */
+	val &= ~(AT_CTRL_CI_SEL | AT_CTRL_RPT_TUNE_ERR | AT_CTRL_SW_TUNE_EN |
+		 FIELD_PREP(AT_CTRL_WIN_EDGE_SEL_MASK, AT_CTRL_WIN_EDGE_SEL));
+
+	/*
+	 * configure tuning settings:
+	 *  - enable auto-tuning
+	 *  - enable sampling window threshold
+	 *  - stop clocks during phase code change
+	 *  - set max latency in cycles between tx and rx clocks
+	 *  - set max latency in cycles to switch output phase
+	 *  - set max sampling window threshold value
+	 */
+	val |= AT_CTRL_AT_EN | AT_CTRL_SWIN_TH_EN | AT_CTRL_TUNE_CLK_STOP_EN;
+	val |= FIELD_PREP(AT_CTRL_PRE_CHANGE_DLY_MASK, AT_CTRL_PRE_CHANGE_DLY);
+	val |= FIELD_PREP(AT_CTRL_POST_CHANGE_DLY_MASK, AT_CTRL_POST_CHANGE_DLY);
+	val |= FIELD_PREP(AT_CTRL_SWIN_TH_VAL_MASK, AT_CTRL_SWIN_TH_VAL);
+
+	sdhci_writel(host, val, priv->vendor_specific_area1 + DWCMSHC_EMMC_ATCTRL);
+	val = sdhci_readl(host, priv->vendor_specific_area1 + DWCMSHC_EMMC_ATCTRL);
+
+	/* perform tuning */
+	sdhci_start_tuning(host);
+	host->tuning_err = __sdhci_execute_tuning(host, opcode);
+	if (host->tuning_err) {
+		/* disable auto-tuning upon tuning error */
+		val &= ~AT_CTRL_AT_EN;
+		sdhci_writel(host, val, priv->vendor_specific_area1 + DWCMSHC_EMMC_ATCTRL);
+		dev_err(mmc_dev(host->mmc), "tuning failed: %d\n", host->tuning_err);
+		return -EIO;
+	}
+	sdhci_end_tuning(host);
+
+	return 0;
+}
+
+static void th1520_sdhci_reset(struct sdhci_host *host, u8 mask)
+{
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct dwcmshc_priv *priv = sdhci_pltfm_priv(pltfm_host);
+	u16 ctrl_2;
+
+	sdhci_reset(host, mask);
+
+	if (priv->flags & FLAG_IO_FIXED_1V8) {
+		ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2);
+		if (!(ctrl_2 & SDHCI_CTRL_VDD_180)) {
+			ctrl_2 |= SDHCI_CTRL_VDD_180;
+			sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2);
+		}
+	}
+}
+
 static const struct sdhci_ops sdhci_dwcmshc_ops = {
 	.set_clock		= sdhci_set_clock,
 	.set_bus_width		= sdhci_set_bus_width,
@@ -356,6 +660,17 @@ static const struct sdhci_ops sdhci_dwcmshc_rk35xx_ops = {
 	.adma_write_desc	= dwcmshc_adma_write_desc,
 };
 
+static const struct sdhci_ops sdhci_dwcmshc_th1520_ops = {
+	.set_clock		= sdhci_set_clock,
+	.set_bus_width		= sdhci_set_bus_width,
+	.set_uhs_signaling	= th1520_set_uhs_signaling,
+	.get_max_clock		= dwcmshc_get_max_clock,
+	.reset			= th1520_sdhci_reset,
+	.adma_write_desc	= dwcmshc_adma_write_desc,
+	.voltage_switch		= dwcmshc_phy_1_8v_init,
+	.platform_execute_tuning = &th1520_execute_tuning,
+};
+
 static const struct sdhci_pltfm_data sdhci_dwcmshc_pdata = {
 	.ops = &sdhci_dwcmshc_ops,
 	.quirks = SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
@@ -379,6 +694,12 @@ static const struct sdhci_pltfm_data sdhci_dwcmshc_rk35xx_pdata = {
 		   SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN,
 };
 
+static const struct sdhci_pltfm_data sdhci_dwcmshc_th1520_pdata = {
+	.ops = &sdhci_dwcmshc_th1520_ops,
+	.quirks = SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
+	.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN,
+};
+
 static int dwcmshc_rk35xx_init(struct sdhci_host *host, struct dwcmshc_priv *dwc_priv)
 {
 	int err;
@@ -447,6 +768,10 @@ static const struct of_device_id sdhci_dwcmshc_dt_ids[] = {
 		.compatible = "snps,dwcmshc-sdhci",
 		.data = &sdhci_dwcmshc_pdata,
 	},
+	{
+		.compatible = "thead,th1520-dwcmshc",
+		.data = &sdhci_dwcmshc_th1520_pdata,
+	},
 	{},
 };
 MODULE_DEVICE_TABLE(of, sdhci_dwcmshc_dt_ids);
@@ -542,6 +867,30 @@ static int dwcmshc_probe(struct platform_device *pdev)
 			goto err_clk;
 	}
 
+	if (pltfm_data == &sdhci_dwcmshc_th1520_pdata) {
+		priv->delay_line = PHY_SDCLKDL_DC_DEFAULT;
+
+		if ((device_property_read_bool(dev, "mmc-ddr-1_8v")) |
+		    (device_property_read_bool(dev, "mmc-hs200-1_8v")) |
+		    (device_property_read_bool(dev, "mmc-hs400-1_8v")))
+			priv->flags |= FLAG_IO_FIXED_1V8;
+		else
+			priv->flags &= ~FLAG_IO_FIXED_1V8;
+
+		/*
+		 * start_signal_voltage_switch() will try 3.3V first
+		 * then 1.8V. Use SDHCI_SIGNALING_180 rather than
+		 * SDHCI_SIGNALING_330 to avoid setting voltage to 3.3V
+		 * in sdhci_start_signal_voltage_switch().
+		 */
+		if (priv->flags & FLAG_IO_FIXED_1V8) {
+			host->flags &= ~SDHCI_SIGNALING_330;
+			host->flags |=  SDHCI_SIGNALING_180;
+		}
+
+		sdhci_enable_v4_mode(host);
+	}
+
 #ifdef CONFIG_ACPI
 	if (pltfm_data == &sdhci_dwcmshc_bf3_pdata)
 		sdhci_enable_v4_mode(host);
diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c
index ff41aa56564e..c79f73459915 100644
--- a/drivers/mmc/host/sdhci.c
+++ b/drivers/mmc/host/sdhci.c
@@ -2841,7 +2841,7 @@ void sdhci_send_tuning(struct sdhci_host *host, u32 opcode)
 }
 EXPORT_SYMBOL_GPL(sdhci_send_tuning);
 
-static int __sdhci_execute_tuning(struct sdhci_host *host, u32 opcode)
+int __sdhci_execute_tuning(struct sdhci_host *host, u32 opcode)
 {
 	int i;
 
@@ -2879,6 +2879,7 @@ static int __sdhci_execute_tuning(struct sdhci_host *host, u32 opcode)
 	sdhci_reset_tuning(host);
 	return -EAGAIN;
 }
+EXPORT_SYMBOL_GPL(__sdhci_execute_tuning);
 
 int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode)
 {
diff --git a/drivers/mmc/host/sdhci.h b/drivers/mmc/host/sdhci.h
index f219bdea8f28..a20864fc0641 100644
--- a/drivers/mmc/host/sdhci.h
+++ b/drivers/mmc/host/sdhci.h
@@ -793,6 +793,7 @@ void sdhci_set_bus_width(struct sdhci_host *host, int width);
 void sdhci_reset(struct sdhci_host *host, u8 mask);
 void sdhci_set_uhs_signaling(struct sdhci_host *host, unsigned timing);
 int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode);
+int __sdhci_execute_tuning(struct sdhci_host *host, u32 opcode);
 void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios);
 int sdhci_start_signal_voltage_switch(struct mmc_host *mmc,
 				      struct mmc_ios *ios);
diff --git a/drivers/mtd/nand/raw/brcmnand/bcm63138_nand.c b/drivers/mtd/nand/raw/brcmnand/bcm63138_nand.c
index 9596629000f4..968c5b674b08 100644
--- a/drivers/mtd/nand/raw/brcmnand/bcm63138_nand.c
+++ b/drivers/mtd/nand/raw/brcmnand/bcm63138_nand.c
@@ -85,7 +85,7 @@ MODULE_DEVICE_TABLE(of, bcm63138_nand_of_match);
 
 static struct platform_driver bcm63138_nand_driver = {
 	.probe			= bcm63138_nand_probe,
-	.remove			= brcmnand_remove,
+	.remove_new		= brcmnand_remove,
 	.driver = {
 		.name		= "bcm63138_nand",
 		.pm		= &brcmnand_pm_ops,
diff --git a/drivers/mtd/nand/raw/brcmnand/bcm6368_nand.c b/drivers/mtd/nand/raw/brcmnand/bcm6368_nand.c
index a06cd87f839a..05b7b653bdf3 100644
--- a/drivers/mtd/nand/raw/brcmnand/bcm6368_nand.c
+++ b/drivers/mtd/nand/raw/brcmnand/bcm6368_nand.c
@@ -117,7 +117,7 @@ MODULE_DEVICE_TABLE(of, bcm6368_nand_of_match);
 
 static struct platform_driver bcm6368_nand_driver = {
 	.probe			= bcm6368_nand_probe,
-	.remove			= brcmnand_remove,
+	.remove_new		= brcmnand_remove,
 	.driver = {
 		.name		= "bcm6368_nand",
 		.pm		= &brcmnand_pm_ops,
diff --git a/drivers/mtd/nand/raw/brcmnand/bcma_nand.c b/drivers/mtd/nand/raw/brcmnand/bcma_nand.c
index dd27977919fb..4e7e435ba339 100644
--- a/drivers/mtd/nand/raw/brcmnand/bcma_nand.c
+++ b/drivers/mtd/nand/raw/brcmnand/bcma_nand.c
@@ -119,7 +119,7 @@ static int brcmnand_bcma_nand_probe(struct platform_device *pdev)
 
 static struct platform_driver brcmnand_bcma_nand_driver = {
 	.probe			= brcmnand_bcma_nand_probe,
-	.remove			= brcmnand_remove,
+	.remove_new		= brcmnand_remove,
 	.driver = {
 		.name		= "bcma_brcmnand",
 		.pm		= &brcmnand_pm_ops,
diff --git a/drivers/mtd/nand/raw/brcmnand/brcmnand.c b/drivers/mtd/nand/raw/brcmnand/brcmnand.c
index 440bef477930..30fc399f346e 100644
--- a/drivers/mtd/nand/raw/brcmnand/brcmnand.c
+++ b/drivers/mtd/nand/raw/brcmnand/brcmnand.c
@@ -3299,7 +3299,7 @@ int brcmnand_probe(struct platform_device *pdev, struct brcmnand_soc *soc)
 }
 EXPORT_SYMBOL_GPL(brcmnand_probe);
 
-int brcmnand_remove(struct platform_device *pdev)
+void brcmnand_remove(struct platform_device *pdev)
 {
 	struct brcmnand_controller *ctrl = dev_get_drvdata(&pdev->dev);
 	struct brcmnand_host *host;
@@ -3316,8 +3316,6 @@ int brcmnand_remove(struct platform_device *pdev)
 	clk_disable_unprepare(ctrl->clk);
 
 	dev_set_drvdata(&pdev->dev, NULL);
-
-	return 0;
 }
 EXPORT_SYMBOL_GPL(brcmnand_remove);
 
diff --git a/drivers/mtd/nand/raw/brcmnand/brcmnand.h b/drivers/mtd/nand/raw/brcmnand/brcmnand.h
index f1f93d85f50d..928114c0be5e 100644
--- a/drivers/mtd/nand/raw/brcmnand/brcmnand.h
+++ b/drivers/mtd/nand/raw/brcmnand/brcmnand.h
@@ -88,7 +88,7 @@ static inline void brcmnand_soc_write(struct brcmnand_soc *soc, u32 val,
 }
 
 int brcmnand_probe(struct platform_device *pdev, struct brcmnand_soc *soc);
-int brcmnand_remove(struct platform_device *pdev);
+void brcmnand_remove(struct platform_device *pdev);
 
 extern const struct dev_pm_ops brcmnand_pm_ops;
 
diff --git a/drivers/mtd/nand/raw/brcmnand/brcmstb_nand.c b/drivers/mtd/nand/raw/brcmnand/brcmstb_nand.c
index 950923d977b7..558f083b92e9 100644
--- a/drivers/mtd/nand/raw/brcmnand/brcmstb_nand.c
+++ b/drivers/mtd/nand/raw/brcmnand/brcmstb_nand.c
@@ -23,7 +23,7 @@ static int brcmstb_nand_probe(struct platform_device *pdev)
 
 static struct platform_driver brcmstb_nand_driver = {
 	.probe			= brcmstb_nand_probe,
-	.remove			= brcmnand_remove,
+	.remove_new		= brcmnand_remove,
 	.driver = {
 		.name		= "brcmstb_nand",
 		.pm		= &brcmnand_pm_ops,
diff --git a/drivers/mtd/nand/raw/brcmnand/iproc_nand.c b/drivers/mtd/nand/raw/brcmnand/iproc_nand.c
index 089c70fc6edf..bf46c8b85898 100644
--- a/drivers/mtd/nand/raw/brcmnand/iproc_nand.c
+++ b/drivers/mtd/nand/raw/brcmnand/iproc_nand.c
@@ -134,7 +134,7 @@ MODULE_DEVICE_TABLE(of, iproc_nand_of_match);
 
 static struct platform_driver iproc_nand_driver = {
 	.probe			= iproc_nand_probe,
-	.remove			= brcmnand_remove,
+	.remove_new		= brcmnand_remove,
 	.driver = {
 		.name		= "iproc_nand",
 		.pm		= &brcmnand_pm_ops,
diff --git a/drivers/mtd/nand/raw/fsl_ifc_nand.c b/drivers/mtd/nand/raw/fsl_ifc_nand.c
index 20bb1e0cb5eb..f0e2318ce088 100644
--- a/drivers/mtd/nand/raw/fsl_ifc_nand.c
+++ b/drivers/mtd/nand/raw/fsl_ifc_nand.c
@@ -21,7 +21,7 @@
 
 #define ERR_BYTE		0xFF /* Value returned for read
 					bytes when read failed	*/
-#define IFC_TIMEOUT_MSECS	500  /* Maximum number of mSecs to wait
+#define IFC_TIMEOUT_MSECS	1000 /* Maximum timeout to wait
 					for IFC NAND Machine	*/
 
 struct fsl_ifc_ctrl;
diff --git a/drivers/mtd/nand/raw/meson_nand.c b/drivers/mtd/nand/raw/meson_nand.c
index 71ec4052e52a..7e16a13fb438 100644
--- a/drivers/mtd/nand/raw/meson_nand.c
+++ b/drivers/mtd/nand/raw/meson_nand.c
@@ -90,6 +90,8 @@
 
 /* eMMC clock register, misc control */
 #define CLK_SELECT_NAND		BIT(31)
+#define CLK_ALWAYS_ON_NAND	BIT(24)
+#define CLK_SELECT_FIX_PLL2	BIT(6)
 
 #define NFC_CLK_CYCLE		6
 
@@ -1154,7 +1156,7 @@ static int meson_nfc_clk_init(struct meson_nfc *nfc)
 		return PTR_ERR(nfc->nand_clk);
 
 	/* init SD_EMMC_CLOCK to sane defaults w/min clock rate */
-	writel(CLK_SELECT_NAND | readl(nfc->reg_clk),
+	writel(CLK_ALWAYS_ON_NAND | CLK_SELECT_NAND | CLK_SELECT_FIX_PLL2,
 	       nfc->reg_clk);
 
 	ret = clk_prepare_enable(nfc->core_clk);
diff --git a/drivers/mtd/nand/raw/txx9ndfmc.c b/drivers/mtd/nand/raw/txx9ndfmc.c
index eddcc0728a67..37f79c019a72 100644
--- a/drivers/mtd/nand/raw/txx9ndfmc.c
+++ b/drivers/mtd/nand/raw/txx9ndfmc.c
@@ -276,7 +276,7 @@ static const struct nand_controller_ops txx9ndfmc_controller_ops = {
 	.attach_chip = txx9ndfmc_attach_chip,
 };
 
-static int __init txx9ndfmc_probe(struct platform_device *dev)
+static int txx9ndfmc_probe(struct platform_device *dev)
 {
 	struct txx9ndfmc_platform_data *plat = dev_get_platdata(&dev->dev);
 	int hold, spw;
@@ -369,13 +369,11 @@ static int __init txx9ndfmc_probe(struct platform_device *dev)
 	return 0;
 }
 
-static int __exit txx9ndfmc_remove(struct platform_device *dev)
+static void txx9ndfmc_remove(struct platform_device *dev)
 {
 	struct txx9ndfmc_drvdata *drvdata = platform_get_drvdata(dev);
 	int ret, i;
 
-	if (!drvdata)
-		return 0;
 	for (i = 0; i < MAX_TXX9NDFMC_DEV; i++) {
 		struct mtd_info *mtd = drvdata->mtds[i];
 		struct nand_chip *chip;
@@ -392,7 +390,6 @@ static int __exit txx9ndfmc_remove(struct platform_device *dev)
 		kfree(txx9_priv->mtdname);
 		kfree(txx9_priv);
 	}
-	return 0;
 }
 
 #ifdef CONFIG_PM
@@ -407,14 +404,14 @@ static int txx9ndfmc_resume(struct platform_device *dev)
 #endif
 
 static struct platform_driver txx9ndfmc_driver = {
-	.remove		= __exit_p(txx9ndfmc_remove),
+	.probe		= txx9ndfmc_probe,
+	.remove_new	= txx9ndfmc_remove,
 	.resume		= txx9ndfmc_resume,
 	.driver		= {
 		.name	= "txx9ndfmc",
 	},
 };
-
-module_platform_driver_probe(txx9ndfmc_driver, txx9ndfmc_probe);
+module_platform_driver(txx9ndfmc_driver);
 
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("TXx9 SoC NAND flash controller driver");
diff --git a/drivers/mtd/nand/spi/gigadevice.c b/drivers/mtd/nand/spi/gigadevice.c
index 987710e09441..6023cba748bb 100644
--- a/drivers/mtd/nand/spi/gigadevice.c
+++ b/drivers/mtd/nand/spi/gigadevice.c
@@ -186,7 +186,7 @@ static int gd5fxgq4uexxg_ecc_get_status(struct spinand_device *spinand,
 {
 	u8 status2;
 	struct spi_mem_op op = SPINAND_GET_FEATURE_OP(GD5FXGQXXEXXG_REG_STATUS2,
-						      &status2);
+						      spinand->scratchbuf);
 	int ret;
 
 	switch (status & STATUS_ECC_MASK) {
@@ -207,6 +207,7 @@ static int gd5fxgq4uexxg_ecc_get_status(struct spinand_device *spinand,
 		 * report the maximum of 4 in this case
 		 */
 		/* bits sorted this way (3...0): ECCS1,ECCS0,ECCSE1,ECCSE0 */
+		status2 = *(spinand->scratchbuf);
 		return ((status & STATUS_ECC_MASK) >> 2) |
 			((status2 & STATUS_ECC_MASK) >> 4);
 
@@ -228,7 +229,7 @@ static int gd5fxgq5xexxg_ecc_get_status(struct spinand_device *spinand,
 {
 	u8 status2;
 	struct spi_mem_op op = SPINAND_GET_FEATURE_OP(GD5FXGQXXEXXG_REG_STATUS2,
-						      &status2);
+						      spinand->scratchbuf);
 	int ret;
 
 	switch (status & STATUS_ECC_MASK) {
@@ -248,6 +249,7 @@ static int gd5fxgq5xexxg_ecc_get_status(struct spinand_device *spinand,
 		 * 1 ... 4 bits are flipped (and corrected)
 		 */
 		/* bits sorted this way (1...0): ECCSE1, ECCSE0 */
+		status2 = *(spinand->scratchbuf);
 		return ((status2 & STATUS_ECC_MASK) >> 4) + 1;
 
 	case STATUS_ECC_UNCOR_ERROR:
diff --git a/drivers/mtd/spi-nor/atmel.c b/drivers/mtd/spi-nor/atmel.c
index e13b8d2dd50a..45d1153a04a0 100644
--- a/drivers/mtd/spi-nor/atmel.c
+++ b/drivers/mtd/spi-nor/atmel.c
@@ -16,12 +16,12 @@
  * is to unlock the whole flash array on startup. Therefore, we have to support
  * exactly this operation.
  */
-static int at25fs_nor_lock(struct spi_nor *nor, loff_t ofs, uint64_t len)
+static int at25fs_nor_lock(struct spi_nor *nor, loff_t ofs, u64 len)
 {
 	return -EOPNOTSUPP;
 }
 
-static int at25fs_nor_unlock(struct spi_nor *nor, loff_t ofs, uint64_t len)
+static int at25fs_nor_unlock(struct spi_nor *nor, loff_t ofs, u64 len)
 {
 	int ret;
 
@@ -37,7 +37,7 @@ static int at25fs_nor_unlock(struct spi_nor *nor, loff_t ofs, uint64_t len)
 	return ret;
 }
 
-static int at25fs_nor_is_locked(struct spi_nor *nor, loff_t ofs, uint64_t len)
+static int at25fs_nor_is_locked(struct spi_nor *nor, loff_t ofs, u64 len)
 {
 	return -EOPNOTSUPP;
 }
@@ -69,7 +69,7 @@ static const struct spi_nor_fixups at25fs_nor_fixups = {
  * Return: 0 on success, -error otherwise.
  */
 static int atmel_nor_set_global_protection(struct spi_nor *nor, loff_t ofs,
-					   uint64_t len, bool is_protect)
+					   u64 len, bool is_protect)
 {
 	int ret;
 	u8 sr;
@@ -118,20 +118,18 @@ static int atmel_nor_set_global_protection(struct spi_nor *nor, loff_t ofs,
 	return spi_nor_write_sr(nor, nor->bouncebuf, 1);
 }
 
-static int atmel_nor_global_protect(struct spi_nor *nor, loff_t ofs,
-				    uint64_t len)
+static int atmel_nor_global_protect(struct spi_nor *nor, loff_t ofs, u64 len)
 {
 	return atmel_nor_set_global_protection(nor, ofs, len, true);
 }
 
-static int atmel_nor_global_unprotect(struct spi_nor *nor, loff_t ofs,
-				      uint64_t len)
+static int atmel_nor_global_unprotect(struct spi_nor *nor, loff_t ofs, u64 len)
 {
 	return atmel_nor_set_global_protection(nor, ofs, len, false);
 }
 
 static int atmel_nor_is_global_protected(struct spi_nor *nor, loff_t ofs,
-					 uint64_t len)
+					 u64 len)
 {
 	int ret;
 
diff --git a/drivers/mtd/spi-nor/core.c b/drivers/mtd/spi-nor/core.c
index 1c443fe568cf..25a64c65717d 100644
--- a/drivers/mtd/spi-nor/core.c
+++ b/drivers/mtd/spi-nor/core.c
@@ -1799,8 +1799,7 @@ static int spi_nor_erase_multi_sectors(struct spi_nor *nor, u64 addr, u32 len)
 static int spi_nor_erase(struct mtd_info *mtd, struct erase_info *instr)
 {
 	struct spi_nor *nor = mtd_to_spi_nor(mtd);
-	u32 addr, len;
-	uint32_t rem;
+	u32 addr, len, rem;
 	int ret;
 
 	dev_dbg(nor->dev, "at 0x%llx, len %lld\n", (long long)instr->addr,
@@ -2146,7 +2145,7 @@ static int spi_nor_write(struct mtd_info *mtd, loff_t to, size_t len,
 		if (is_power_of_2(page_size)) {
 			page_offset = addr & (page_size - 1);
 		} else {
-			uint64_t aux = addr;
+			u64 aux = addr;
 
 			page_offset = do_div(aux, page_size);
 		}
diff --git a/drivers/mtd/spi-nor/core.h b/drivers/mtd/spi-nor/core.h
index 93cd2fc3606d..a456042379ee 100644
--- a/drivers/mtd/spi-nor/core.h
+++ b/drivers/mtd/spi-nor/core.h
@@ -293,9 +293,9 @@ struct spi_nor_erase_map {
  * @is_locked:	check if a region of the SPI NOR is completely locked
  */
 struct spi_nor_locking_ops {
-	int (*lock)(struct spi_nor *nor, loff_t ofs, uint64_t len);
-	int (*unlock)(struct spi_nor *nor, loff_t ofs, uint64_t len);
-	int (*is_locked)(struct spi_nor *nor, loff_t ofs, uint64_t len);
+	int (*lock)(struct spi_nor *nor, loff_t ofs, u64 len);
+	int (*unlock)(struct spi_nor *nor, loff_t ofs, u64 len);
+	int (*is_locked)(struct spi_nor *nor, loff_t ofs, u64 len);
 };
 
 /**
diff --git a/drivers/mtd/spi-nor/sst.c b/drivers/mtd/spi-nor/sst.c
index 44d2a546bf17..180b7390690c 100644
--- a/drivers/mtd/spi-nor/sst.c
+++ b/drivers/mtd/spi-nor/sst.c
@@ -13,12 +13,12 @@
 
 #define SST26VF_CR_BPNV		BIT(3)
 
-static int sst26vf_nor_lock(struct spi_nor *nor, loff_t ofs, uint64_t len)
+static int sst26vf_nor_lock(struct spi_nor *nor, loff_t ofs, u64 len)
 {
 	return -EOPNOTSUPP;
 }
 
-static int sst26vf_nor_unlock(struct spi_nor *nor, loff_t ofs, uint64_t len)
+static int sst26vf_nor_unlock(struct spi_nor *nor, loff_t ofs, u64 len)
 {
 	int ret;
 
@@ -38,7 +38,7 @@ static int sst26vf_nor_unlock(struct spi_nor *nor, loff_t ofs, uint64_t len)
 	return spi_nor_global_block_unlock(nor);
 }
 
-static int sst26vf_nor_is_locked(struct spi_nor *nor, loff_t ofs, uint64_t len)
+static int sst26vf_nor_is_locked(struct spi_nor *nor, loff_t ofs, u64 len)
 {
 	return -EOPNOTSUPP;
 }
diff --git a/drivers/mtd/spi-nor/swp.c b/drivers/mtd/spi-nor/swp.c
index 585813310ee1..e48c3cff247a 100644
--- a/drivers/mtd/spi-nor/swp.c
+++ b/drivers/mtd/spi-nor/swp.c
@@ -53,7 +53,7 @@ static u64 spi_nor_get_min_prot_length_sr(struct spi_nor *nor)
 }
 
 static void spi_nor_get_locked_range_sr(struct spi_nor *nor, u8 sr, loff_t *ofs,
-					uint64_t *len)
+					u64 *len)
 {
 	struct mtd_info *mtd = &nor->mtd;
 	u64 min_prot_len;
@@ -90,10 +90,10 @@ static void spi_nor_get_locked_range_sr(struct spi_nor *nor, u8 sr, loff_t *ofs,
  * (if @locked is false); false otherwise.
  */
 static bool spi_nor_check_lock_status_sr(struct spi_nor *nor, loff_t ofs,
-					 uint64_t len, u8 sr, bool locked)
+					 u64 len, u8 sr, bool locked)
 {
 	loff_t lock_offs, lock_offs_max, offs_max;
-	uint64_t lock_len;
+	u64 lock_len;
 
 	if (!len)
 		return true;
@@ -111,14 +111,13 @@ static bool spi_nor_check_lock_status_sr(struct spi_nor *nor, loff_t ofs,
 		return (ofs >= lock_offs_max) || (offs_max <= lock_offs);
 }
 
-static bool spi_nor_is_locked_sr(struct spi_nor *nor, loff_t ofs, uint64_t len,
-				 u8 sr)
+static bool spi_nor_is_locked_sr(struct spi_nor *nor, loff_t ofs, u64 len, u8 sr)
 {
 	return spi_nor_check_lock_status_sr(nor, ofs, len, sr, true);
 }
 
-static bool spi_nor_is_unlocked_sr(struct spi_nor *nor, loff_t ofs,
-				   uint64_t len, u8 sr)
+static bool spi_nor_is_unlocked_sr(struct spi_nor *nor, loff_t ofs, u64 len,
+				   u8 sr)
 {
 	return spi_nor_check_lock_status_sr(nor, ofs, len, sr, false);
 }
@@ -156,7 +155,7 @@ static bool spi_nor_is_unlocked_sr(struct spi_nor *nor, loff_t ofs,
  *
  * Returns negative on errors, 0 on success.
  */
-static int spi_nor_sr_lock(struct spi_nor *nor, loff_t ofs, uint64_t len)
+static int spi_nor_sr_lock(struct spi_nor *nor, loff_t ofs, u64 len)
 {
 	struct mtd_info *mtd = &nor->mtd;
 	u64 min_prot_len;
@@ -246,7 +245,7 @@ static int spi_nor_sr_lock(struct spi_nor *nor, loff_t ofs, uint64_t len)
  *
  * Returns negative on errors, 0 on success.
  */
-static int spi_nor_sr_unlock(struct spi_nor *nor, loff_t ofs, uint64_t len)
+static int spi_nor_sr_unlock(struct spi_nor *nor, loff_t ofs, u64 len)
 {
 	struct mtd_info *mtd = &nor->mtd;
 	u64 min_prot_len;
@@ -331,7 +330,7 @@ static int spi_nor_sr_unlock(struct spi_nor *nor, loff_t ofs, uint64_t len)
  * Returns 1 if entire region is locked, 0 if any portion is unlocked, and
  * negative on errors.
  */
-static int spi_nor_sr_is_locked(struct spi_nor *nor, loff_t ofs, uint64_t len)
+static int spi_nor_sr_is_locked(struct spi_nor *nor, loff_t ofs, u64 len)
 {
 	int ret;
 
@@ -353,7 +352,7 @@ void spi_nor_init_default_locking_ops(struct spi_nor *nor)
 	nor->params->locking_ops = &spi_nor_sr_locking_ops;
 }
 
-static int spi_nor_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
+static int spi_nor_lock(struct mtd_info *mtd, loff_t ofs, u64 len)
 {
 	struct spi_nor *nor = mtd_to_spi_nor(mtd);
 	int ret;
@@ -368,7 +367,7 @@ static int spi_nor_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
 	return ret;
 }
 
-static int spi_nor_unlock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
+static int spi_nor_unlock(struct mtd_info *mtd, loff_t ofs, u64 len)
 {
 	struct spi_nor *nor = mtd_to_spi_nor(mtd);
 	int ret;
@@ -383,7 +382,7 @@ static int spi_nor_unlock(struct mtd_info *mtd, loff_t ofs, uint64_t len)
 	return ret;
 }
 
-static int spi_nor_is_locked(struct mtd_info *mtd, loff_t ofs, uint64_t len)
+static int spi_nor_is_locked(struct mtd_info *mtd, loff_t ofs, u64 len)
 {
 	struct spi_nor *nor = mtd_to_spi_nor(mtd);
 	int ret;
diff --git a/drivers/net/bonding/bond_alb.c b/drivers/net/bonding/bond_alb.c
index dc2c7b979656..7edf0fd58c34 100644
--- a/drivers/net/bonding/bond_alb.c
+++ b/drivers/net/bonding/bond_alb.c
@@ -985,7 +985,8 @@ static int alb_upper_dev_walk(struct net_device *upper,
 	if (netif_is_macvlan(upper) && !strict_match) {
 		tags = bond_verify_device_path(bond->dev, upper, 0);
 		if (IS_ERR_OR_NULL(tags))
-			BUG();
+			return -ENOMEM;
+
 		alb_send_lp_vid(slave, upper->dev_addr,
 				tags[0].vlan_proto, tags[0].vlan_id);
 		kfree(tags);
diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c
index 8e6cc0e133b7..d987432cee3b 100644
--- a/drivers/net/bonding/bond_main.c
+++ b/drivers/net/bonding/bond_main.c
@@ -2973,8 +2973,11 @@ struct bond_vlan_tag *bond_verify_device_path(struct net_device *start_dev,
 
 	if (start_dev == end_dev) {
 		tags = kcalloc(level + 1, sizeof(*tags), GFP_ATOMIC);
-		if (!tags)
+		if (!tags) {
+			net_err_ratelimited("%s: %s: Failed to allocate tags\n",
+					    __func__, start_dev->name);
 			return ERR_PTR(-ENOMEM);
+		}
 		tags[level].vlan_proto = BOND_VLAN_PROTO_NONE;
 		return tags;
 	}
@@ -5755,10 +5758,8 @@ static int bond_ethtool_get_ts_info(struct net_device *bond_dev,
 {
 	struct bonding *bond = netdev_priv(bond_dev);
 	struct ethtool_ts_info ts_info;
-	const struct ethtool_ops *ops;
 	struct net_device *real_dev;
 	bool sw_tx_support = false;
-	struct phy_device *phydev;
 	struct list_head *iter;
 	struct slave *slave;
 	int ret = 0;
@@ -5769,29 +5770,12 @@ static int bond_ethtool_get_ts_info(struct net_device *bond_dev,
 	rcu_read_unlock();
 
 	if (real_dev) {
-		ops = real_dev->ethtool_ops;
-		phydev = real_dev->phydev;
-
-		if (phy_has_tsinfo(phydev)) {
-			ret = phy_ts_info(phydev, info);
-			goto out;
-		} else if (ops->get_ts_info) {
-			ret = ops->get_ts_info(real_dev, info);
-			goto out;
-		}
+		ret = ethtool_get_ts_info_by_layer(real_dev, info);
 	} else {
 		/* Check if all slaves support software tx timestamping */
 		rcu_read_lock();
 		bond_for_each_slave_rcu(bond, slave, iter) {
-			ret = -1;
-			ops = slave->dev->ethtool_ops;
-			phydev = slave->dev->phydev;
-
-			if (phy_has_tsinfo(phydev))
-				ret = phy_ts_info(phydev, &ts_info);
-			else if (ops->get_ts_info)
-				ret = ops->get_ts_info(slave->dev, &ts_info);
-
+			ret = ethtool_get_ts_info_by_layer(slave->dev, &ts_info);
 			if (!ret && (ts_info.so_timestamping & SOF_TIMESTAMPING_TX_SOFTWARE)) {
 				sw_tx_support = true;
 				continue;
@@ -5803,15 +5787,9 @@ static int bond_ethtool_get_ts_info(struct net_device *bond_dev,
 		rcu_read_unlock();
 	}
 
-	ret = 0;
-	info->so_timestamping = SOF_TIMESTAMPING_RX_SOFTWARE |
-				SOF_TIMESTAMPING_SOFTWARE;
 	if (sw_tx_support)
 		info->so_timestamping |= SOF_TIMESTAMPING_TX_SOFTWARE;
 
-	info->phc_index = -1;
-
-out:
 	dev_put(real_dev);
 	return ret;
 }
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
index d0359b569afe..e6ac1bd21bb3 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
@@ -331,16 +331,16 @@ static void bnxt_sched_reset_rxr(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
 }
 
 void bnxt_sched_reset_txr(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
-			  int idx)
+			  u16 curr)
 {
 	struct bnxt_napi *bnapi = txr->bnapi;
 
 	if (bnapi->tx_fault)
 		return;
 
-	netdev_err(bp->dev, "Invalid Tx completion (ring:%d tx_pkts:%d cons:%u prod:%u i:%d)",
-		   txr->txq_index, bnapi->tx_pkts,
-		   txr->tx_cons, txr->tx_prod, idx);
+	netdev_err(bp->dev, "Invalid Tx completion (ring:%d tx_hw_cons:%u cons:%u prod:%u curr:%u)",
+		   txr->txq_index, txr->tx_hw_cons,
+		   txr->tx_cons, txr->tx_prod, curr);
 	WARN_ON_ONCE(1);
 	bnapi->tx_fault = 1;
 	bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT);
@@ -381,6 +381,8 @@ static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
 static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
 			     u16 prod)
 {
+	/* Sync BD data before updating doorbell */
+	wmb();
 	bnxt_db_write(bp, &txr->tx_db, prod);
 	txr->kick_pending = 0;
 }
@@ -388,7 +390,7 @@ static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 {
 	struct bnxt *bp = netdev_priv(dev);
-	struct tx_bd *txbd;
+	struct tx_bd *txbd, *txbd0;
 	struct tx_bd_ext *txbd1;
 	struct netdev_queue *txq;
 	int i;
@@ -432,8 +434,6 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 
 	txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 
-	txbd->tx_bd_opaque = prod;
-
 	tx_buf = &txr->tx_buf_ring[prod];
 	tx_buf->skb = skb;
 	tx_buf->nr_frags = last_frag;
@@ -519,7 +519,9 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 
 		txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
 		txbd->tx_bd_haddr = txr->data_mapping;
+		txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 2);
 		prod = NEXT_TX(prod);
+		tx_push->tx_bd_opaque = txbd->tx_bd_opaque;
 		txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
 		memcpy(txbd, tx_push1, sizeof(*txbd));
 		prod = NEXT_TX(prod);
@@ -562,6 +564,7 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 		((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
 
 	txbd->tx_bd_haddr = cpu_to_le64(mapping);
+	txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 2 + last_frag);
 
 	prod = NEXT_TX(prod);
 	txbd1 = (struct tx_bd_ext *)
@@ -601,6 +604,7 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 	txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
 	txbd1->tx_bd_cfa_action =
 			cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
+	txbd0 = txbd;
 	for (i = 0; i < last_frag; i++) {
 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
 
@@ -632,16 +636,17 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 
 	skb_tx_timestamp(skb);
 
-	/* Sync BD data before updating doorbell */
-	wmb();
-
 	prod = NEXT_TX(prod);
 	WRITE_ONCE(txr->tx_prod, prod);
 
-	if (!netdev_xmit_more() || netif_xmit_stopped(txq))
+	if (!netdev_xmit_more() || netif_xmit_stopped(txq)) {
 		bnxt_txr_db_kick(bp, txr, prod);
-	else
+	} else {
+		if (free_size >= bp->tx_wake_thresh)
+			txbd0->tx_bd_len_flags_type |=
+				cpu_to_le32(TX_BD_FLAGS_NO_CMPL);
 		txr->kick_pending = 1;
+	}
 
 tx_done:
 
@@ -686,17 +691,17 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
 	return NETDEV_TX_OK;
 }
 
-static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
+static void __bnxt_tx_int(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
+			  int budget)
 {
-	struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 	struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
-	u16 cons = txr->tx_cons;
 	struct pci_dev *pdev = bp->pdev;
-	int nr_pkts = bnapi->tx_pkts;
-	int i;
+	u16 hw_cons = txr->tx_hw_cons;
 	unsigned int tx_bytes = 0;
+	u16 cons = txr->tx_cons;
+	int tx_pkts = 0;
 
-	for (i = 0; i < nr_pkts; i++) {
+	while (cons != hw_cons) {
 		struct bnxt_sw_tx_bd *tx_buf;
 		struct sk_buff *skb;
 		int j, last;
@@ -707,10 +712,11 @@ static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
 		tx_buf->skb = NULL;
 
 		if (unlikely(!skb)) {
-			bnxt_sched_reset_txr(bp, txr, i);
+			bnxt_sched_reset_txr(bp, txr, cons);
 			return;
 		}
 
+		tx_pkts++;
 		tx_bytes += skb->len;
 
 		if (tx_buf->is_push) {
@@ -747,14 +753,25 @@ static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
 		dev_consume_skb_any(skb);
 	}
 
-	bnapi->tx_pkts = 0;
 	WRITE_ONCE(txr->tx_cons, cons);
 
-	__netif_txq_completed_wake(txq, nr_pkts, tx_bytes,
+	__netif_txq_completed_wake(txq, tx_pkts, tx_bytes,
 				   bnxt_tx_avail(bp, txr), bp->tx_wake_thresh,
 				   READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING);
 }
 
+static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
+{
+	struct bnxt_tx_ring_info *txr;
+	int i;
+
+	bnxt_for_each_napi_tx(i, bnapi, txr) {
+		if (txr->tx_hw_cons != txr->tx_cons)
+			__bnxt_tx_int(bp, txr, budget);
+	}
+	bnapi->events &= ~BNXT_TX_CMP_EVENT;
+}
+
 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
 					 struct bnxt_rx_ring_info *rxr,
 					 unsigned int *offset,
@@ -2588,7 +2605,6 @@ static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
 	struct bnxt_napi *bnapi = cpr->bnapi;
 	u32 raw_cons = cpr->cp_raw_cons;
 	u32 cons;
-	int tx_pkts = 0;
 	int rx_pkts = 0;
 	u8 event = 0;
 	struct tx_cmp *txcmp;
@@ -2609,9 +2625,17 @@ static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
 		 */
 		dma_rmb();
 		if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
-			tx_pkts++;
+			u32 opaque = txcmp->tx_cmp_opaque;
+			struct bnxt_tx_ring_info *txr;
+			u16 tx_freed;
+
+			txr = bnapi->tx_ring[TX_OPAQUE_RING(opaque)];
+			event |= BNXT_TX_CMP_EVENT;
+			txr->tx_hw_cons = TX_OPAQUE_PROD(bp, opaque);
+			tx_freed = (txr->tx_hw_cons - txr->tx_cons) &
+				   bp->tx_ring_mask;
 			/* return full budget so NAPI will complete. */
-			if (unlikely(tx_pkts >= bp->tx_wake_thresh)) {
+			if (unlikely(tx_freed >= bp->tx_wake_thresh)) {
 				rx_pkts = budget;
 				raw_cons = NEXT_RAW_CMP(raw_cons);
 				if (budget)
@@ -2655,7 +2679,7 @@ static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
 		xdp_do_flush();
 
 	if (event & BNXT_TX_EVENT) {
-		struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
+		struct bnxt_tx_ring_info *txr = bnapi->tx_ring[0];
 		u16 prod = txr->tx_prod;
 
 		/* Sync BD data before updating doorbell */
@@ -2665,7 +2689,6 @@ static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
 	}
 
 	cpr->cp_raw_cons = raw_cons;
-	bnapi->tx_pkts += tx_pkts;
 	bnapi->events |= event;
 	return rx_pkts;
 }
@@ -2673,7 +2696,7 @@ static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi,
 				  int budget)
 {
-	if (bnapi->tx_pkts && !bnapi->tx_fault)
+	if ((bnapi->events & BNXT_TX_CMP_EVENT) && !bnapi->tx_fault)
 		bnapi->tx_int(bp, bnapi, budget);
 
 	if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) {
@@ -2686,7 +2709,7 @@ static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi,
 
 		bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
 	}
-	bnapi->events = 0;
+	bnapi->events &= BNXT_TX_CMP_EVENT;
 }
 
 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
@@ -2826,14 +2849,11 @@ static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
 	int i, work_done = 0;
 
-	for (i = 0; i < 2; i++) {
-		struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
+	for (i = 0; i < cpr->cp_ring_count; i++) {
+		struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[i];
 
-		if (cpr2) {
-			work_done += __bnxt_poll_work(bp, cpr2,
-						      budget - work_done);
-			cpr->has_more_work |= cpr2->has_more_work;
-		}
+		work_done += __bnxt_poll_work(bp, cpr2, budget - work_done);
+		cpr->has_more_work |= cpr2->has_more_work;
 	}
 	return work_done;
 }
@@ -2844,11 +2864,11 @@ static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
 	int i;
 
-	for (i = 0; i < 2; i++) {
-		struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
+	for (i = 0; i < cpr->cp_ring_count; i++) {
+		struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[i];
 		struct bnxt_db_info *db;
 
-		if (cpr2 && cpr2->had_work_done) {
+		if (cpr2->had_work_done) {
 			db = &cpr2->cp_db;
 			bnxt_writeq(bp, db->db_key64 | dbr_type |
 				    RING_CMP(cpr2->cp_raw_cons), db->doorbell);
@@ -2901,13 +2921,16 @@ static int bnxt_poll_p5(struct napi_struct *napi, int budget)
 
 		if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
 			u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
+			u32 cq_type = BNXT_NQ_HDL_TYPE(idx);
 			struct bnxt_cp_ring_info *cpr2;
 
 			/* No more budget for RX work */
-			if (budget && work_done >= budget && idx == BNXT_RX_HDL)
+			if (budget && work_done >= budget &&
+			    cq_type == BNXT_NQ_HDL_TYPE_RX)
 				break;
 
-			cpr2 = cpr->cp_ring_arr[idx];
+			idx = BNXT_NQ_HDL_IDX(idx);
+			cpr2 = &cpr->cp_ring_arr[idx];
 			work_done += __bnxt_poll_work(bp, cpr2,
 						      budget - work_done);
 			cpr->has_more_work |= cpr2->has_more_work;
@@ -2922,8 +2945,9 @@ static int bnxt_poll_p5(struct napi_struct *napi, int budget)
 		BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons);
 	}
 poll_done:
-	cpr_rx = cpr->cp_ring_arr[BNXT_RX_HDL];
-	if (cpr_rx && (bp->flags & BNXT_FLAG_DIM)) {
+	cpr_rx = &cpr->cp_ring_arr[0];
+	if (cpr_rx->cp_ring_type == BNXT_NQ_HDL_TYPE_RX &&
+	    (bp->flags & BNXT_FLAG_DIM)) {
 		struct dim_sample dim_sample = {};
 
 		dim_update_sample(cpr->event_ctr,
@@ -3395,6 +3419,15 @@ static void bnxt_free_tx_rings(struct bnxt *bp)
 	}
 }
 
+#define BNXT_TC_TO_RING_BASE(bp, tc)	\
+	((tc) * (bp)->tx_nr_rings_per_tc)
+
+#define BNXT_RING_TO_TC_OFF(bp, tx)	\
+	((tx) % (bp)->tx_nr_rings_per_tc)
+
+#define BNXT_RING_TO_TC(bp, tx)		\
+	((tx) / (bp)->tx_nr_rings_per_tc)
+
 static int bnxt_alloc_tx_rings(struct bnxt *bp)
 {
 	int i, j, rc;
@@ -3450,7 +3483,7 @@ static int bnxt_alloc_tx_rings(struct bnxt *bp)
 		spin_lock_init(&txr->xdp_tx_lock);
 		if (i < bp->tx_nr_rings_xdp)
 			continue;
-		if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
+		if (BNXT_RING_TO_TC_OFF(bp, i) == (bp->tx_nr_rings_per_tc - 1))
 			j++;
 	}
 	return 0;
@@ -3533,36 +3566,33 @@ static void bnxt_free_cp_rings(struct bnxt *bp)
 
 		bnxt_free_ring(bp, &ring->ring_mem);
 
-		for (j = 0; j < 2; j++) {
-			struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
+		if (!cpr->cp_ring_arr)
+			continue;
 
-			if (cpr2) {
-				ring = &cpr2->cp_ring_struct;
-				bnxt_free_ring(bp, &ring->ring_mem);
-				bnxt_free_cp_arrays(cpr2);
-				kfree(cpr2);
-				cpr->cp_ring_arr[j] = NULL;
-			}
+		for (j = 0; j < cpr->cp_ring_count; j++) {
+			struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
+
+			ring = &cpr2->cp_ring_struct;
+			bnxt_free_ring(bp, &ring->ring_mem);
+			bnxt_free_cp_arrays(cpr2);
 		}
+		kfree(cpr->cp_ring_arr);
+		cpr->cp_ring_arr = NULL;
+		cpr->cp_ring_count = 0;
 	}
 }
 
-static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
+static int bnxt_alloc_cp_sub_ring(struct bnxt *bp,
+				  struct bnxt_cp_ring_info *cpr)
 {
 	struct bnxt_ring_mem_info *rmem;
 	struct bnxt_ring_struct *ring;
-	struct bnxt_cp_ring_info *cpr;
 	int rc;
 
-	cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
-	if (!cpr)
-		return NULL;
-
 	rc = bnxt_alloc_cp_arrays(cpr, bp->cp_nr_pages);
 	if (rc) {
 		bnxt_free_cp_arrays(cpr);
-		kfree(cpr);
-		return NULL;
+		return -ENOMEM;
 	}
 	ring = &cpr->cp_ring_struct;
 	rmem = &ring->ring_mem;
@@ -3575,23 +3605,26 @@ static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
 	if (rc) {
 		bnxt_free_ring(bp, rmem);
 		bnxt_free_cp_arrays(cpr);
-		kfree(cpr);
-		cpr = NULL;
 	}
-	return cpr;
+	return rc;
 }
 
 static int bnxt_alloc_cp_rings(struct bnxt *bp)
 {
 	bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
-	int i, rc, ulp_base_vec, ulp_msix;
+	int i, j, rc, ulp_base_vec, ulp_msix;
+	int tcs = netdev_get_num_tc(bp->dev);
 
+	if (!tcs)
+		tcs = 1;
 	ulp_msix = bnxt_get_ulp_msix_num(bp);
 	ulp_base_vec = bnxt_get_ulp_msix_base(bp);
-	for (i = 0; i < bp->cp_nr_rings; i++) {
+	for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
 		struct bnxt_napi *bnapi = bp->bnapi[i];
-		struct bnxt_cp_ring_info *cpr;
+		struct bnxt_cp_ring_info *cpr, *cpr2;
 		struct bnxt_ring_struct *ring;
+		int cp_count = 0, k;
+		int rx = 0, tx = 0;
 
 		if (!bnapi)
 			continue;
@@ -3613,31 +3646,51 @@ static int bnxt_alloc_cp_rings(struct bnxt *bp)
 			continue;
 
 		if (i < bp->rx_nr_rings) {
-			struct bnxt_cp_ring_info *cpr2 =
-				bnxt_alloc_cp_sub_ring(bp);
-
-			cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
-			if (!cpr2)
-				return -ENOMEM;
-			cpr2->bnapi = bnapi;
+			cp_count++;
+			rx = 1;
+		}
+		if (i < bp->tx_nr_rings_xdp) {
+			cp_count++;
+			tx = 1;
+		} else if ((sh && i < bp->tx_nr_rings) ||
+			 (!sh && i >= bp->rx_nr_rings)) {
+			cp_count += tcs;
+			tx = 1;
 		}
-		if ((sh && i < bp->tx_nr_rings) ||
-		    (!sh && i >= bp->rx_nr_rings)) {
-			struct bnxt_cp_ring_info *cpr2 =
-				bnxt_alloc_cp_sub_ring(bp);
 
-			cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
-			if (!cpr2)
-				return -ENOMEM;
+		cpr->cp_ring_arr = kcalloc(cp_count, sizeof(*cpr),
+					   GFP_KERNEL);
+		if (!cpr->cp_ring_arr)
+			return -ENOMEM;
+		cpr->cp_ring_count = cp_count;
+
+		for (k = 0; k < cp_count; k++) {
+			cpr2 = &cpr->cp_ring_arr[k];
+			rc = bnxt_alloc_cp_sub_ring(bp, cpr2);
+			if (rc)
+				return rc;
 			cpr2->bnapi = bnapi;
+			cpr2->cp_idx = k;
+			if (!k && rx) {
+				bp->rx_ring[i].rx_cpr = cpr2;
+				cpr2->cp_ring_type = BNXT_NQ_HDL_TYPE_RX;
+			} else {
+				int n, tc = k - rx;
+
+				n = BNXT_TC_TO_RING_BASE(bp, tc) + j;
+				bp->tx_ring[n].tx_cpr = cpr2;
+				cpr2->cp_ring_type = BNXT_NQ_HDL_TYPE_TX;
+			}
 		}
+		if (tx)
+			j++;
 	}
 	return 0;
 }
 
 static void bnxt_init_ring_struct(struct bnxt *bp)
 {
-	int i;
+	int i, j;
 
 	for (i = 0; i < bp->cp_nr_rings; i++) {
 		struct bnxt_napi *bnapi = bp->bnapi[i];
@@ -3682,18 +3735,16 @@ static void bnxt_init_ring_struct(struct bnxt *bp)
 		rmem->vmem = (void **)&rxr->rx_agg_ring;
 
 skip_rx:
-		txr = bnapi->tx_ring;
-		if (!txr)
-			continue;
-
-		ring = &txr->tx_ring_struct;
-		rmem = &ring->ring_mem;
-		rmem->nr_pages = bp->tx_nr_pages;
-		rmem->page_size = HW_RXBD_RING_SIZE;
-		rmem->pg_arr = (void **)txr->tx_desc_ring;
-		rmem->dma_arr = txr->tx_desc_mapping;
-		rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
-		rmem->vmem = (void **)&txr->tx_buf_ring;
+		bnxt_for_each_napi_tx(j, bnapi, txr) {
+			ring = &txr->tx_ring_struct;
+			rmem = &ring->ring_mem;
+			rmem->nr_pages = bp->tx_nr_pages;
+			rmem->page_size = HW_TXBD_RING_SIZE;
+			rmem->pg_arr = (void **)txr->tx_desc_ring;
+			rmem->dma_arr = txr->tx_desc_mapping;
+			rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
+			rmem->vmem = (void **)&txr->tx_buf_ring;
+		}
 	}
 }
 
@@ -3814,11 +3865,10 @@ static void bnxt_init_cp_rings(struct bnxt *bp)
 		ring->fw_ring_id = INVALID_HW_RING_ID;
 		cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
 		cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
-		for (j = 0; j < 2; j++) {
-			struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
-
-			if (!cpr2)
-				continue;
+		if (!cpr->cp_ring_arr)
+			continue;
+		for (j = 0; j < cpr->cp_ring_count; j++) {
+			struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
 
 			ring = &cpr2->cp_ring_struct;
 			ring->fw_ring_id = INVALID_HW_RING_ID;
@@ -4493,7 +4543,7 @@ static int bnxt_alloc_stats(struct bnxt *bp)
 
 static void bnxt_clear_ring_indices(struct bnxt *bp)
 {
-	int i;
+	int i, j;
 
 	if (!bp->bnapi)
 		return;
@@ -4510,10 +4560,10 @@ static void bnxt_clear_ring_indices(struct bnxt *bp)
 		cpr = &bnapi->cp_ring;
 		cpr->cp_raw_cons = 0;
 
-		txr = bnapi->tx_ring;
-		if (txr) {
+		bnxt_for_each_napi_tx(j, bnapi, txr) {
 			txr->tx_prod = 0;
 			txr->tx_cons = 0;
+			txr->tx_hw_cons = 0;
 		}
 
 		rxr = bnapi->rx_ring;
@@ -4523,6 +4573,7 @@ static void bnxt_clear_ring_indices(struct bnxt *bp)
 			rxr->rx_sw_agg_prod = 0;
 			rxr->rx_next_cons = 0;
 		}
+		bnapi->events = 0;
 	}
 }
 
@@ -4649,6 +4700,8 @@ static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
 					BNXT_RMEM_RING_PTE_FLAG;
 				rxr->rx_agg_ring_struct.ring_mem.flags =
 					BNXT_RMEM_RING_PTE_FLAG;
+			} else {
+				rxr->rx_cpr =  &bp->bnapi[i]->cp_ring;
 			}
 			rxr->bnapi = bp->bnapi[i];
 			bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
@@ -4671,22 +4724,33 @@ static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
 		else
 			j = bp->rx_nr_rings;
 
-		for (i = 0; i < bp->tx_nr_rings; i++, j++) {
+		for (i = 0; i < bp->tx_nr_rings; i++) {
 			struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
+			struct bnxt_napi *bnapi2;
 
 			if (bp->flags & BNXT_FLAG_CHIP_P5)
 				txr->tx_ring_struct.ring_mem.flags =
 					BNXT_RMEM_RING_PTE_FLAG;
-			txr->bnapi = bp->bnapi[j];
-			bp->bnapi[j]->tx_ring = txr;
 			bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
 			if (i >= bp->tx_nr_rings_xdp) {
+				int k = j + BNXT_RING_TO_TC_OFF(bp, i);
+
+				bnapi2 = bp->bnapi[k];
 				txr->txq_index = i - bp->tx_nr_rings_xdp;
-				bp->bnapi[j]->tx_int = bnxt_tx_int;
+				txr->tx_napi_idx =
+					BNXT_RING_TO_TC(bp, txr->txq_index);
+				bnapi2->tx_ring[txr->tx_napi_idx] = txr;
+				bnapi2->tx_int = bnxt_tx_int;
 			} else {
-				bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
-				bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
+				bnapi2 = bp->bnapi[j];
+				bnapi2->flags |= BNXT_NAPI_FLAG_XDP;
+				bnapi2->tx_ring[0] = txr;
+				bnapi2->tx_int = bnxt_tx_int_xdp;
+				j++;
 			}
+			txr->bnapi = bnapi2;
+			if (!(bp->flags & BNXT_FLAG_CHIP_P5))
+				txr->tx_cpr = &bnapi2->cp_ring;
 		}
 
 		rc = bnxt_alloc_stats(bp);
@@ -5237,28 +5301,18 @@ static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
 
 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
 {
-	if (bp->flags & BNXT_FLAG_CHIP_P5) {
-		struct bnxt_napi *bnapi = rxr->bnapi;
-		struct bnxt_cp_ring_info *cpr;
-
-		cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
-		return cpr->cp_ring_struct.fw_ring_id;
-	} else {
+	if (bp->flags & BNXT_FLAG_CHIP_P5)
+		return rxr->rx_cpr->cp_ring_struct.fw_ring_id;
+	else
 		return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
-	}
 }
 
 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
 {
-	if (bp->flags & BNXT_FLAG_CHIP_P5) {
-		struct bnxt_napi *bnapi = txr->bnapi;
-		struct bnxt_cp_ring_info *cpr;
-
-		cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
-		return cpr->cp_ring_struct.fw_ring_id;
-	} else {
+	if (bp->flags & BNXT_FLAG_CHIP_P5)
+		return txr->tx_cpr->cp_ring_struct.fw_ring_id;
+	else
 		return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
-	}
 }
 
 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp)
@@ -6017,14 +6071,12 @@ static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
 		u32 map_idx;
 
 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
+			struct bnxt_cp_ring_info *cpr2 = txr->tx_cpr;
 			struct bnxt_napi *bnapi = txr->bnapi;
-			struct bnxt_cp_ring_info *cpr, *cpr2;
 			u32 type2 = HWRM_RING_ALLOC_CMPL;
 
-			cpr = &bnapi->cp_ring;
-			cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
 			ring = &cpr2->cp_ring_struct;
-			ring->handle = BNXT_TX_HDL;
+			ring->handle = BNXT_SET_NQ_HDL(cpr2);
 			map_idx = bnapi->index;
 			rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
 			if (rc)
@@ -6057,13 +6109,11 @@ static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
 			bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
 		bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
-			struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
+			struct bnxt_cp_ring_info *cpr2 = rxr->rx_cpr;
 			u32 type2 = HWRM_RING_ALLOC_CMPL;
-			struct bnxt_cp_ring_info *cpr2;
 
-			cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
 			ring = &cpr2->cp_ring_struct;
-			ring->handle = BNXT_RX_HDL;
+			ring->handle = BNXT_SET_NQ_HDL(cpr2);
 			rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
 			if (rc)
 				goto err_out;
@@ -6208,18 +6258,16 @@ static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
 		struct bnxt_ring_struct *ring;
 		int j;
 
-		for (j = 0; j < 2; j++) {
-			struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
-
-			if (cpr2) {
-				ring = &cpr2->cp_ring_struct;
-				if (ring->fw_ring_id == INVALID_HW_RING_ID)
-					continue;
-				hwrm_ring_free_send_msg(bp, ring,
-					RING_FREE_REQ_RING_TYPE_L2_CMPL,
-					INVALID_HW_RING_ID);
-				ring->fw_ring_id = INVALID_HW_RING_ID;
-			}
+		for (j = 0; j < cpr->cp_ring_count && cpr->cp_ring_arr; j++) {
+			struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
+
+			ring = &cpr2->cp_ring_struct;
+			if (ring->fw_ring_id == INVALID_HW_RING_ID)
+				continue;
+			hwrm_ring_free_send_msg(bp, ring,
+						RING_FREE_REQ_RING_TYPE_L2_CMPL,
+						INVALID_HW_RING_ID);
+			ring->fw_ring_id = INVALID_HW_RING_ID;
 		}
 		ring = &cpr->cp_ring_struct;
 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
@@ -6274,7 +6322,8 @@ static int bnxt_hwrm_get_rings(struct bnxt *bp)
 			if (bp->flags & BNXT_FLAG_AGG_RINGS)
 				rx >>= 1;
 			if (cp < (rx + tx)) {
-				bnxt_trim_rings(bp, &rx, &tx, cp, false);
+				rx = cp / 2;
+				tx = rx;
 				if (bp->flags & BNXT_FLAG_AGG_RINGS)
 					rx <<= 1;
 				hw_resc->resv_rx_rings = rx;
@@ -6575,6 +6624,7 @@ static int __bnxt_reserve_rings(struct bnxt *bp)
 	int grp, rx_rings, rc;
 	int vnic = 1, stat;
 	bool sh = false;
+	int tx_cp;
 
 	if (!bnxt_need_reserve_rings(bp))
 		return 0;
@@ -6624,7 +6674,8 @@ static int __bnxt_reserve_rings(struct bnxt *bp)
 	rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
 		rx = rx_rings << 1;
-	cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
+	tx_cp = bnxt_num_tx_to_cp(bp, tx);
+	cp = sh ? max_t(int, tx_cp, rx_rings) : tx_cp + rx_rings;
 	bp->tx_nr_rings = tx;
 
 	/* If we cannot reserve all the RX rings, reset the RSS map only
@@ -6887,10 +6938,40 @@ int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
 	return hwrm_req_send(bp, req_rx);
 }
 
+static int
+bnxt_hwrm_set_rx_coal(struct bnxt *bp, struct bnxt_napi *bnapi,
+		      struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
+{
+	u16 ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
+
+	req->ring_id = cpu_to_le16(ring_id);
+	return hwrm_req_send(bp, req);
+}
+
+static int
+bnxt_hwrm_set_tx_coal(struct bnxt *bp, struct bnxt_napi *bnapi,
+		      struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
+{
+	struct bnxt_tx_ring_info *txr;
+	int i, rc;
+
+	bnxt_for_each_napi_tx(i, bnapi, txr) {
+		u16 ring_id;
+
+		ring_id = bnxt_cp_ring_for_tx(bp, txr);
+		req->ring_id = cpu_to_le16(ring_id);
+		rc = hwrm_req_send(bp, req);
+		if (rc)
+			return rc;
+		if (!(bp->flags & BNXT_FLAG_CHIP_P5))
+			return 0;
+	}
+	return 0;
+}
+
 int bnxt_hwrm_set_coal(struct bnxt *bp)
 {
-	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx, *req_tx,
-							   *req;
+	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx, *req_tx;
 	int i, rc;
 
 	rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
@@ -6911,29 +6992,19 @@ int bnxt_hwrm_set_coal(struct bnxt *bp)
 	for (i = 0; i < bp->cp_nr_rings; i++) {
 		struct bnxt_napi *bnapi = bp->bnapi[i];
 		struct bnxt_coal *hw_coal;
-		u16 ring_id;
-
-		req = req_rx;
-		if (!bnapi->rx_ring) {
-			ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
-			req = req_tx;
-		} else {
-			ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
-		}
-		req->ring_id = cpu_to_le16(ring_id);
 
-		rc = hwrm_req_send(bp, req);
+		if (!bnapi->rx_ring)
+			rc = bnxt_hwrm_set_tx_coal(bp, bnapi, req_tx);
+		else
+			rc = bnxt_hwrm_set_rx_coal(bp, bnapi, req_rx);
 		if (rc)
 			break;
 
 		if (!(bp->flags & BNXT_FLAG_CHIP_P5))
 			continue;
 
-		if (bnapi->rx_ring && bnapi->tx_ring) {
-			req = req_tx;
-			ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
-			req->ring_id = cpu_to_le16(ring_id);
-			rc = hwrm_req_send(bp, req);
+		if (bnapi->rx_ring && bnapi->tx_ring[0]) {
+			rc = bnxt_hwrm_set_tx_coal(bp, bnapi, req_tx);
 			if (rc)
 				break;
 		}
@@ -9031,8 +9102,8 @@ static int bnxt_set_real_num_queues(struct bnxt *bp)
 	return rc;
 }
 
-static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
-			   bool shared)
+static int __bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
+			     bool shared)
 {
 	int _rx = *rx, _tx = *tx;
 
@@ -9055,6 +9126,46 @@ static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
 	return 0;
 }
 
+static int __bnxt_num_tx_to_cp(struct bnxt *bp, int tx, int tx_sets, int tx_xdp)
+{
+	return (tx - tx_xdp) / tx_sets + tx_xdp;
+}
+
+int bnxt_num_tx_to_cp(struct bnxt *bp, int tx)
+{
+	int tcs = netdev_get_num_tc(bp->dev);
+
+	if (!tcs)
+		tcs = 1;
+	return __bnxt_num_tx_to_cp(bp, tx, tcs, bp->tx_nr_rings_xdp);
+}
+
+static int bnxt_num_cp_to_tx(struct bnxt *bp, int tx_cp)
+{
+	int tcs = netdev_get_num_tc(bp->dev);
+
+	return (tx_cp - bp->tx_nr_rings_xdp) * tcs +
+	       bp->tx_nr_rings_xdp;
+}
+
+static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
+			   bool sh)
+{
+	int tx_cp = bnxt_num_tx_to_cp(bp, *tx);
+
+	if (tx_cp != *tx) {
+		int tx_saved = tx_cp, rc;
+
+		rc = __bnxt_trim_rings(bp, rx, &tx_cp, max, sh);
+		if (rc)
+			return rc;
+		if (tx_cp != tx_saved)
+			*tx = bnxt_num_cp_to_tx(bp, tx_cp);
+		return 0;
+	}
+	return __bnxt_trim_rings(bp, rx, tx, max, sh);
+}
+
 static void bnxt_setup_msix(struct bnxt *bp)
 {
 	const int len = sizeof(bp->irq_tbl[0].name);
@@ -9067,7 +9178,7 @@ static void bnxt_setup_msix(struct bnxt *bp)
 
 		for (i = 0; i < tcs; i++) {
 			count = bp->tx_nr_rings_per_tc;
-			off = i * count;
+			off = BNXT_TC_TO_RING_BASE(bp, i);
 			netdev_set_tc_queue(dev, i, count, off);
 		}
 	}
@@ -9217,7 +9328,7 @@ static int bnxt_get_num_msix(struct bnxt *bp)
 
 static int bnxt_init_msix(struct bnxt *bp)
 {
-	int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
+	int i, total_vecs, max, rc = 0, min = 1, ulp_msix, tx_cp;
 	struct msix_entry *msix_ent;
 
 	total_vecs = bnxt_get_num_msix(bp);
@@ -9259,9 +9370,10 @@ static int bnxt_init_msix(struct bnxt *bp)
 		if (rc)
 			goto msix_setup_exit;
 
+		tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings);
 		bp->cp_nr_rings = (min == 1) ?
-				  max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
-				  bp->tx_nr_rings + bp->rx_nr_rings;
+				  max_t(int, tx_cp, bp->rx_nr_rings) :
+				  tx_cp + bp->rx_nr_rings;
 
 	} else {
 		rc = -ENOMEM;
@@ -9527,8 +9639,6 @@ static void bnxt_enable_napi(struct bnxt *bp)
 		cpr = &bnapi->cp_ring;
 		bnapi->in_reset = false;
 
-		bnapi->tx_pkts = 0;
-
 		if (bnapi->rx_ring) {
 			INIT_WORK(&cpr->dim.work, bnxt_dim_work);
 			cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
@@ -11559,15 +11669,13 @@ static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
 
 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
 {
-	struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
-	int i = bnapi->index;
-
-	if (!txr)
-		return;
+	struct bnxt_tx_ring_info *txr;
+	int i = bnapi->index, j;
 
-	netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
-		    i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
-		    txr->tx_cons);
+	bnxt_for_each_napi_tx(j, bnapi, txr)
+		netdev_info(bnapi->bp->dev, "[%d.%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
+			    i, j, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
+			    txr->tx_cons);
 }
 
 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
@@ -11997,12 +12105,11 @@ static void bnxt_chk_missed_irq(struct bnxt *bp)
 			continue;
 
 		cpr = &bnapi->cp_ring;
-		for (j = 0; j < 2; j++) {
-			struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
+		for (j = 0; j < cpr->cp_ring_count; j++) {
+			struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
 			u32 val[2];
 
-			if (!cpr2 || cpr2->has_more_work ||
-			    !bnxt_has_work(bp, cpr2))
+			if (cpr2->has_more_work || !bnxt_has_work(bp, cpr2))
 				continue;
 
 			if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
@@ -12161,23 +12268,27 @@ static void bnxt_sp_task(struct work_struct *work)
 	clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
 }
 
+static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
+				int *max_cp);
+
 /* Under rtnl_lock */
 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
 		     int tx_xdp)
 {
-	int max_rx, max_tx, tx_sets = 1;
+	int max_rx, max_tx, max_cp, tx_sets = 1, tx_cp;
 	int tx_rings_needed, stats;
 	int rx_rings = rx;
-	int cp, vnics, rc;
+	int cp, vnics;
 
 	if (tcs)
 		tx_sets = tcs;
 
-	rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
-	if (rc)
-		return rc;
+	if (bp->flags & BNXT_FLAG_AGG_RINGS)
+		rx_rings <<= 1;
+
+	_bnxt_get_max_rings(bp, &max_rx, &max_tx, &max_cp);
 
-	if (max_rx < rx)
+	if (max_rx < rx_rings)
 		return -ENOMEM;
 
 	tx_rings_needed = tx * tx_sets + tx_xdp;
@@ -12186,11 +12297,12 @@ int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
 
 	vnics = 1;
 	if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
-		vnics += rx_rings;
+		vnics += rx;
 
-	if (bp->flags & BNXT_FLAG_AGG_RINGS)
-		rx_rings <<= 1;
-	cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
+	tx_cp = __bnxt_num_tx_to_cp(bp, tx_rings_needed, tx_sets, tx_xdp);
+	cp = sh ? max_t(int, tx_cp, rx) : tx_cp + rx;
+	if (max_cp < cp)
+		return -ENOMEM;
 	stats = cp;
 	if (BNXT_NEW_RM(bp)) {
 		cp += bnxt_get_ulp_msix_num(bp);
@@ -12824,7 +12936,7 @@ int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
 {
 	struct bnxt *bp = netdev_priv(dev);
 	bool sh = false;
-	int rc;
+	int rc, tx_cp;
 
 	if (tc > bp->max_tc) {
 		netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
@@ -12855,8 +12967,9 @@ int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
 		netdev_reset_tc(dev);
 	}
 	bp->tx_nr_rings += bp->tx_nr_rings_xdp;
-	bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
-			       bp->tx_nr_rings + bp->rx_nr_rings;
+	tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings);
+	bp->cp_nr_rings = sh ? max_t(int, tx_cp, bp->rx_nr_rings) :
+			       tx_cp + bp->rx_nr_rings;
 
 	if (netif_running(bp->dev))
 		return bnxt_open_nic(bp, true, false);
@@ -13335,7 +13448,10 @@ static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
 		*max_rx >>= 1;
 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
-		bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
+		if (*max_cp < (*max_rx + *max_tx)) {
+			*max_rx = *max_cp / 2;
+			*max_tx = *max_rx;
+		}
 		/* On P5 chips, max_cp output param should be available NQs */
 		*max_cp = max_irq;
 	}
@@ -13636,7 +13752,8 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	}
 
 	max_irqs = bnxt_get_max_irq(pdev);
-	dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
+	dev = alloc_etherdev_mqs(sizeof(*bp), max_irqs * BNXT_MAX_QUEUE,
+				 max_irqs);
 	if (!dev)
 		return -ENOMEM;
 
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
index e702dbc3e6b1..4ce993943924 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
@@ -61,6 +61,24 @@ struct tx_bd {
 	__le64 tx_bd_haddr;
 } __packed;
 
+#define TX_OPAQUE_IDX_MASK	0x0000ffff
+#define TX_OPAQUE_BDS_MASK	0x00ff0000
+#define TX_OPAQUE_BDS_SHIFT	16
+#define TX_OPAQUE_RING_MASK	0xff000000
+#define TX_OPAQUE_RING_SHIFT	24
+
+#define SET_TX_OPAQUE(bp, txr, idx, bds)				\
+	(((txr)->tx_napi_idx << TX_OPAQUE_RING_SHIFT) |			\
+	 ((bds) << TX_OPAQUE_BDS_SHIFT) | ((idx) & (bp)->tx_ring_mask))
+
+#define TX_OPAQUE_IDX(opq)	((opq) & TX_OPAQUE_IDX_MASK)
+#define TX_OPAQUE_RING(opq)	(((opq) & TX_OPAQUE_RING_MASK) >>	\
+				 TX_OPAQUE_RING_SHIFT)
+#define TX_OPAQUE_BDS(opq)	(((opq) & TX_OPAQUE_BDS_MASK) >>	\
+				 TX_OPAQUE_BDS_SHIFT)
+#define TX_OPAQUE_PROD(bp, opq)	((TX_OPAQUE_IDX(opq) + TX_OPAQUE_BDS(opq)) &\
+				 (bp)->tx_ring_mask)
+
 struct tx_bd_ext {
 	__le32 tx_bd_hsize_lflags;
 	#define TX_BD_FLAGS_TCP_UDP_CHKSUM			(1 << 0)
@@ -530,6 +548,19 @@ struct nqe_cn {
 	__le32	cq_handle_high;
 };
 
+#define BNXT_NQ_HDL_IDX_MASK	0x00ffffff
+#define BNXT_NQ_HDL_TYPE_MASK	0xff000000
+#define BNXT_NQ_HDL_TYPE_SHIFT	24
+#define BNXT_NQ_HDL_TYPE_RX	0x00
+#define BNXT_NQ_HDL_TYPE_TX	0x01
+
+#define BNXT_NQ_HDL_IDX(hdl)	((hdl) & BNXT_NQ_HDL_IDX_MASK)
+#define BNXT_NQ_HDL_TYPE(hdl)	(((hdl) & BNXT_NQ_HDL_TYPE_MASK) >>	\
+				 BNXT_NQ_HDL_TYPE_SHIFT)
+
+#define BNXT_SET_NQ_HDL(cpr)						\
+	(((cpr)->cp_ring_type << BNXT_NQ_HDL_TYPE_SHIFT) | (cpr)->cp_idx)
+
 #define DB_IDX_MASK						0xffffff
 #define DB_IDX_VALID						(0x1 << 26)
 #define DB_IRQ_DIS						(0x1 << 27)
@@ -702,6 +733,7 @@ struct nqe_cn {
 #define BNXT_AGG_EVENT		2
 #define BNXT_TX_EVENT		4
 #define BNXT_REDIRECT_EVENT	8
+#define BNXT_TX_CMP_EVENT	0x10
 
 struct bnxt_sw_tx_bd {
 	union {
@@ -792,9 +824,12 @@ struct bnxt_db_info {
 
 struct bnxt_tx_ring_info {
 	struct bnxt_napi	*bnapi;
+	struct bnxt_cp_ring_info	*tx_cpr;
 	u16			tx_prod;
 	u16			tx_cons;
+	u16			tx_hw_cons;
 	u16			txq_index;
+	u8			tx_napi_idx;
 	u8			kick_pending;
 	struct bnxt_db_info	tx_db;
 
@@ -901,6 +936,7 @@ struct bnxt_tpa_idx_map {
 
 struct bnxt_rx_ring_info {
 	struct bnxt_napi	*bnapi;
+	struct bnxt_cp_ring_info	*rx_cpr;
 	u16			rx_prod;
 	u16			rx_agg_prod;
 	u16			rx_sw_agg_prod;
@@ -980,6 +1016,8 @@ struct bnxt_cp_ring_info {
 
 	u8			had_work_done:1;
 	u8			has_more_work:1;
+	u8			cp_ring_type;
+	u8			cp_idx;
 
 	u32			last_cp_raw_cons;
 
@@ -1004,11 +1042,18 @@ struct bnxt_cp_ring_info {
 
 	struct bnxt_ring_struct	cp_ring_struct;
 
-	struct bnxt_cp_ring_info *cp_ring_arr[2];
-#define BNXT_RX_HDL	0
-#define BNXT_TX_HDL	1
+	int			cp_ring_count;
+	struct bnxt_cp_ring_info *cp_ring_arr;
 };
 
+#define BNXT_MAX_QUEUE		8
+#define BNXT_MAX_TXR_PER_NAPI	BNXT_MAX_QUEUE
+
+#define bnxt_for_each_napi_tx(iter, bnapi, txr)		\
+	for (iter = 0, txr = (bnapi)->tx_ring[0]; txr;	\
+	     txr = (iter < BNXT_MAX_TXR_PER_NAPI - 1) ?	\
+	     (bnapi)->tx_ring[++iter] : NULL)
+
 struct bnxt_napi {
 	struct napi_struct	napi;
 	struct bnxt		*bp;
@@ -1016,11 +1061,10 @@ struct bnxt_napi {
 	int			index;
 	struct bnxt_cp_ring_info	cp_ring;
 	struct bnxt_rx_ring_info	*rx_ring;
-	struct bnxt_tx_ring_info	*tx_ring;
+	struct bnxt_tx_ring_info	*tx_ring[BNXT_MAX_TXR_PER_NAPI];
 
 	void			(*tx_int)(struct bnxt *, struct bnxt_napi *,
 					  int budget);
-	int			tx_pkts;
 	u8			events;
 	u8			tx_fault:1;
 
@@ -1355,8 +1399,6 @@ struct bnxt_link_info {
 	(PORT_PHY_CFG_REQ_FLAGS_FEC_CLAUSE74_DISABLE |		\
 	 BNXT_FEC_RS_OFF(link_info))
 
-#define BNXT_MAX_QUEUE	8
-
 struct bnxt_queue_info {
 	u8	queue_id;
 	u8	queue_profile;
@@ -2351,6 +2393,7 @@ int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings);
 int bnxt_nq_rings_in_use(struct bnxt *bp);
 int bnxt_hwrm_set_coal(struct bnxt *);
 void bnxt_free_ctx_mem(struct bnxt *bp);
+int bnxt_num_tx_to_cp(struct bnxt *bp, int tx);
 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp);
 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp);
 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp);
@@ -2360,7 +2403,7 @@ int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init);
 void bnxt_tx_disable(struct bnxt *bp);
 void bnxt_tx_enable(struct bnxt *bp);
 void bnxt_sched_reset_txr(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
-			  int idx);
+			  u16 curr);
 void bnxt_report_link(struct bnxt *bp);
 int bnxt_update_link(struct bnxt *bp, bool chng_link_state);
 int bnxt_hwrm_set_pause(struct bnxt *);
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
index f3f384773ac0..585044310141 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
@@ -528,7 +528,8 @@ static int bnxt_get_num_ring_stats(struct bnxt *bp)
 	     bnxt_get_num_tpa_ring_stats(bp);
 	tx = NUM_RING_TX_HW_STATS;
 	cmn = NUM_RING_CMN_SW_STATS;
-	return rx * bp->rx_nr_rings + tx * bp->tx_nr_rings +
+	return rx * bp->rx_nr_rings +
+	       tx * (bp->tx_nr_rings_xdp + bp->tx_nr_rings_per_tc) +
 	       cmn * bp->cp_nr_rings;
 }
 
@@ -923,6 +924,7 @@ static int bnxt_set_channels(struct net_device *dev,
 	bool sh = false;
 	int tx_xdp = 0;
 	int rc = 0;
+	int tx_cp;
 
 	if (channel->other_count)
 		return -EINVAL;
@@ -994,8 +996,9 @@ static int bnxt_set_channels(struct net_device *dev,
 	if (tcs > 1)
 		bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tcs + tx_xdp;
 
-	bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
-			       bp->tx_nr_rings + bp->rx_nr_rings;
+	tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings);
+	bp->cp_nr_rings = sh ? max_t(int, tx_cp, bp->rx_nr_rings) :
+			       tx_cp + bp->rx_nr_rings;
 
 	/* After changing number of rx channels, update NTUPLE feature. */
 	netdev_update_features(dev);
@@ -3941,7 +3944,7 @@ static int bnxt_run_loopback(struct bnxt *bp)
 
 	cpr = &rxr->bnapi->cp_ring;
 	if (bp->flags & BNXT_FLAG_CHIP_P5)
-		cpr = cpr->cp_ring_arr[BNXT_RX_HDL];
+		cpr = rxr->rx_cpr;
 	pkt_size = min(bp->dev->mtu + ETH_HLEN, bp->rx_copy_thresh);
 	skb = netdev_alloc_skb(bp->dev, pkt_size);
 	if (!skb)
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
index 96f5ca778c67..9d428eb3fdb9 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
@@ -52,7 +52,7 @@ struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp,
 		((num_frags + 1) << TX_BD_FLAGS_BD_CNT_SHIFT) |
 		bnxt_lhint_arr[len >> 9];
 	txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
-	txbd->tx_bd_opaque = prod;
+	txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 1 + num_frags);
 	txbd->tx_bd_haddr = cpu_to_le64(mapping);
 
 	/* now let us fill up the frags into the next buffers */
@@ -127,19 +127,19 @@ static void __bnxt_xmit_xdp_redirect(struct bnxt *bp,
 
 void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
 {
-	struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
+	struct bnxt_tx_ring_info *txr = bnapi->tx_ring[0];
 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
+	u16 tx_hw_cons = txr->tx_hw_cons;
 	bool rx_doorbell_needed = false;
-	int nr_pkts = bnapi->tx_pkts;
 	struct bnxt_sw_tx_bd *tx_buf;
 	u16 tx_cons = txr->tx_cons;
 	u16 last_tx_cons = tx_cons;
-	int i, j, frags;
+	int j, frags;
 
 	if (!budget)
 		return;
 
-	for (i = 0; i < nr_pkts; i++) {
+	while (tx_cons != tx_hw_cons) {
 		tx_buf = &txr->tx_buf_ring[tx_cons];
 
 		if (tx_buf->action == XDP_REDIRECT) {
@@ -164,13 +164,13 @@ void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
 				page_pool_recycle_direct(rxr->page_pool, tx_buf->page);
 			}
 		} else {
-			bnxt_sched_reset_txr(bp, txr, i);
+			bnxt_sched_reset_txr(bp, txr, tx_cons);
 			return;
 		}
 		tx_cons = NEXT_TX(tx_cons);
 	}
 
-	bnapi->tx_pkts = 0;
+	bnapi->events &= ~BNXT_TX_CMP_EVENT;
 	WRITE_ONCE(txr->tx_cons, tx_cons);
 	if (rx_doorbell_needed) {
 		tx_buf = &txr->tx_buf_ring[last_tx_cons];
@@ -249,7 +249,7 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons,
 	pdev = bp->pdev;
 	offset = bp->rx_offset;
 
-	txr = rxr->bnapi->tx_ring;
+	txr = rxr->bnapi->tx_ring[0];
 	/* BNXT_RX_PAGE_MODE(bp) when XDP enabled */
 	orig_data = xdp.data;
 
@@ -275,7 +275,7 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons,
 	case XDP_TX:
 		rx_buf = &rxr->rx_buf_ring[cons];
 		mapping = rx_buf->mapping - bp->rx_dma_offset;
-		*event = 0;
+		*event &= BNXT_TX_CMP_EVENT;
 
 		if (unlikely(xdp_buff_has_frags(&xdp))) {
 			struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(&xdp);
@@ -398,7 +398,7 @@ int bnxt_xdp_xmit(struct net_device *dev, int num_frames,
 static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog)
 {
 	struct net_device *dev = bp->dev;
-	int tx_xdp = 0, rc, tc;
+	int tx_xdp = 0, tx_cp, rc, tc;
 	struct bpf_prog *old;
 
 	if (prog && !prog->aux->xdp_has_frags &&
@@ -446,7 +446,8 @@ static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog)
 	}
 	bp->tx_nr_rings_xdp = tx_xdp;
 	bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc + tx_xdp;
-	bp->cp_nr_rings = max_t(int, bp->tx_nr_rings, bp->rx_nr_rings);
+	tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings);
+	bp->cp_nr_rings = max_t(int, tx_cp, bp->rx_nr_rings);
 	bnxt_set_tpa_flags(bp);
 	bnxt_set_ring_params(bp);
 
diff --git a/drivers/net/ethernet/cadence/macb.h b/drivers/net/ethernet/cadence/macb.h
index 78c972bb1d96..aa5700ac9c00 100644
--- a/drivers/net/ethernet/cadence/macb.h
+++ b/drivers/net/ethernet/cadence/macb.h
@@ -1165,9 +1165,10 @@ struct macb_ptp_info {
 	int (*get_ts_info)(struct net_device *dev,
 			   struct ethtool_ts_info *info);
 	int (*get_hwtst)(struct net_device *netdev,
-			 struct ifreq *ifr);
+			 struct kernel_hwtstamp_config *tstamp_config);
 	int (*set_hwtst)(struct net_device *netdev,
-			 struct ifreq *ifr, int cmd);
+			 struct kernel_hwtstamp_config *tstamp_config,
+			 struct netlink_ext_ack *extack);
 };
 
 struct macb_pm_data {
@@ -1314,7 +1315,7 @@ struct macb {
 	struct ptp_clock *ptp_clock;
 	struct ptp_clock_info ptp_clock_info;
 	struct tsu_incr tsu_incr;
-	struct hwtstamp_config tstamp_config;
+	struct kernel_hwtstamp_config tstamp_config;
 
 	/* RX queue filer rule set*/
 	struct ethtool_rx_fs_list rx_fs_list;
@@ -1363,8 +1364,12 @@ static inline void gem_ptp_do_rxstamp(struct macb *bp, struct sk_buff *skb, stru
 
 	gem_ptp_rxstamp(bp, skb, desc);
 }
-int gem_get_hwtst(struct net_device *dev, struct ifreq *rq);
-int gem_set_hwtst(struct net_device *dev, struct ifreq *ifr, int cmd);
+
+int gem_get_hwtst(struct net_device *dev,
+		  struct kernel_hwtstamp_config *tstamp_config);
+int gem_set_hwtst(struct net_device *dev,
+		  struct kernel_hwtstamp_config *tstamp_config,
+		  struct netlink_ext_ack *extack);
 #else
 static inline void gem_ptp_init(struct net_device *ndev) { }
 static inline void gem_ptp_remove(struct net_device *ndev) { }
diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c
index cebae0f418f2..898debfd4db3 100644
--- a/drivers/net/ethernet/cadence/macb_main.c
+++ b/drivers/net/ethernet/cadence/macb_main.c
@@ -3773,18 +3773,38 @@ static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
 	if (!netif_running(dev))
 		return -EINVAL;
 
-	if (bp->ptp_info) {
-		switch (cmd) {
-		case SIOCSHWTSTAMP:
-			return bp->ptp_info->set_hwtst(dev, rq, cmd);
-		case SIOCGHWTSTAMP:
-			return bp->ptp_info->get_hwtst(dev, rq);
-		}
-	}
-
 	return phylink_mii_ioctl(bp->phylink, rq, cmd);
 }
 
+static int macb_hwtstamp_get(struct net_device *dev,
+			     struct kernel_hwtstamp_config *cfg)
+{
+	struct macb *bp = netdev_priv(dev);
+
+	if (!netif_running(dev))
+		return -EINVAL;
+
+	if (!bp->ptp_info)
+		return -EOPNOTSUPP;
+
+	return bp->ptp_info->get_hwtst(dev, cfg);
+}
+
+static int macb_hwtstamp_set(struct net_device *dev,
+			     struct kernel_hwtstamp_config *cfg,
+			     struct netlink_ext_ack *extack)
+{
+	struct macb *bp = netdev_priv(dev);
+
+	if (!netif_running(dev))
+		return -EINVAL;
+
+	if (!bp->ptp_info)
+		return -EOPNOTSUPP;
+
+	return bp->ptp_info->set_hwtst(dev, cfg, extack);
+}
+
 static inline void macb_set_txcsum_feature(struct macb *bp,
 					   netdev_features_t features)
 {
@@ -3884,6 +3904,8 @@ static const struct net_device_ops macb_netdev_ops = {
 #endif
 	.ndo_set_features	= macb_set_features,
 	.ndo_features_check	= macb_features_check,
+	.ndo_hwtstamp_set	= macb_hwtstamp_set,
+	.ndo_hwtstamp_get	= macb_hwtstamp_get,
 };
 
 /* Configure peripheral capabilities according to device tree
@@ -4539,6 +4561,8 @@ static const struct net_device_ops at91ether_netdev_ops = {
 #ifdef CONFIG_NET_POLL_CONTROLLER
 	.ndo_poll_controller	= at91ether_poll_controller,
 #endif
+	.ndo_hwtstamp_set	= macb_hwtstamp_set,
+	.ndo_hwtstamp_get	= macb_hwtstamp_get,
 };
 
 static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk,
diff --git a/drivers/net/ethernet/cadence/macb_ptp.c b/drivers/net/ethernet/cadence/macb_ptp.c
index 51d26fa190d7..a63bf29c4fa8 100644
--- a/drivers/net/ethernet/cadence/macb_ptp.c
+++ b/drivers/net/ethernet/cadence/macb_ptp.c
@@ -374,19 +374,16 @@ static int gem_ptp_set_ts_mode(struct macb *bp,
 	return 0;
 }
 
-int gem_get_hwtst(struct net_device *dev, struct ifreq *rq)
+int gem_get_hwtst(struct net_device *dev,
+		  struct kernel_hwtstamp_config *tstamp_config)
 {
-	struct hwtstamp_config *tstamp_config;
 	struct macb *bp = netdev_priv(dev);
 
-	tstamp_config = &bp->tstamp_config;
+	*tstamp_config = bp->tstamp_config;
 	if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0)
 		return -EOPNOTSUPP;
 
-	if (copy_to_user(rq->ifr_data, tstamp_config, sizeof(*tstamp_config)))
-		return -EFAULT;
-	else
-		return 0;
+	return 0;
 }
 
 static void gem_ptp_set_one_step_sync(struct macb *bp, u8 enable)
@@ -401,22 +398,18 @@ static void gem_ptp_set_one_step_sync(struct macb *bp, u8 enable)
 		macb_writel(bp, NCR, reg_val & ~MACB_BIT(OSSMODE));
 }
 
-int gem_set_hwtst(struct net_device *dev, struct ifreq *ifr, int cmd)
+int gem_set_hwtst(struct net_device *dev,
+		  struct kernel_hwtstamp_config *tstamp_config,
+		  struct netlink_ext_ack *extack)
 {
 	enum macb_bd_control tx_bd_control = TSTAMP_DISABLED;
 	enum macb_bd_control rx_bd_control = TSTAMP_DISABLED;
-	struct hwtstamp_config *tstamp_config;
 	struct macb *bp = netdev_priv(dev);
 	u32 regval;
 
-	tstamp_config = &bp->tstamp_config;
 	if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0)
 		return -EOPNOTSUPP;
 
-	if (copy_from_user(tstamp_config, ifr->ifr_data,
-			   sizeof(*tstamp_config)))
-		return -EFAULT;
-
 	switch (tstamp_config->tx_type) {
 	case HWTSTAMP_TX_OFF:
 		break;
@@ -463,12 +456,11 @@ int gem_set_hwtst(struct net_device *dev, struct ifreq *ifr, int cmd)
 		return -ERANGE;
 	}
 
+	bp->tstamp_config = *tstamp_config;
+
 	if (gem_ptp_set_ts_mode(bp, tx_bd_control, rx_bd_control) != 0)
 		return -ERANGE;
 
-	if (copy_to_user(ifr->ifr_data, tstamp_config, sizeof(*tstamp_config)))
-		return -EFAULT;
-	else
-		return 0;
+	return 0;
 }
 
diff --git a/drivers/net/ethernet/google/gve/gve_dqo.h b/drivers/net/ethernet/google/gve/gve_dqo.h
index 1eb4d5fd8561..c36b93f0de15 100644
--- a/drivers/net/ethernet/google/gve/gve_dqo.h
+++ b/drivers/net/ethernet/google/gve/gve_dqo.h
@@ -33,6 +33,9 @@
 #define GVE_DEALLOCATE_COMPL_TIMEOUT 60
 
 netdev_tx_t gve_tx_dqo(struct sk_buff *skb, struct net_device *dev);
+netdev_features_t gve_features_check_dqo(struct sk_buff *skb,
+					 struct net_device *dev,
+					 netdev_features_t features);
 bool gve_tx_poll_dqo(struct gve_notify_block *block, bool do_clean);
 int gve_rx_poll_dqo(struct gve_notify_block *block, int budget);
 int gve_tx_alloc_rings_dqo(struct gve_priv *priv);
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index 2d42e733837b..cc169748ffa2 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -79,6 +79,18 @@ static int gve_verify_driver_compatibility(struct gve_priv *priv)
 	return err;
 }
 
+static netdev_features_t gve_features_check(struct sk_buff *skb,
+					    struct net_device *dev,
+					    netdev_features_t features)
+{
+	struct gve_priv *priv = netdev_priv(dev);
+
+	if (!gve_is_gqi(priv))
+		return gve_features_check_dqo(skb, dev, features);
+
+	return features;
+}
+
 static netdev_tx_t gve_start_xmit(struct sk_buff *skb, struct net_device *dev)
 {
 	struct gve_priv *priv = netdev_priv(dev);
@@ -1879,6 +1891,7 @@ static int gve_set_features(struct net_device *netdev,
 
 static const struct net_device_ops gve_netdev_ops = {
 	.ndo_start_xmit		=	gve_start_xmit,
+	.ndo_features_check	=	gve_features_check,
 	.ndo_open		=	gve_open,
 	.ndo_stop		=	gve_close,
 	.ndo_get_stats64	=	gve_get_stats,
diff --git a/drivers/net/ethernet/google/gve/gve_tx_dqo.c b/drivers/net/ethernet/google/gve/gve_tx_dqo.c
index 1e19b834a613..f59c4710f118 100644
--- a/drivers/net/ethernet/google/gve/gve_tx_dqo.c
+++ b/drivers/net/ethernet/google/gve/gve_tx_dqo.c
@@ -843,6 +843,16 @@ static bool gve_can_send_tso(const struct sk_buff *skb)
 	return true;
 }
 
+netdev_features_t gve_features_check_dqo(struct sk_buff *skb,
+					 struct net_device *dev,
+					 netdev_features_t features)
+{
+	if (skb_is_gso(skb) && !gve_can_send_tso(skb))
+		return features & ~NETIF_F_GSO_MASK;
+
+	return features;
+}
+
 /* Attempt to transmit specified SKB.
  *
  * Returns 0 if the SKB was transmitted or dropped.
@@ -854,11 +864,10 @@ static int gve_try_tx_skb(struct gve_priv *priv, struct gve_tx_ring *tx,
 	int num_buffer_descs;
 	int total_num_descs;
 
-	if (tx->dqo.qpl) {
-		if (skb_is_gso(skb))
-			if (unlikely(ipv6_hopopt_jumbo_remove(skb)))
-				goto drop;
+	if (skb_is_gso(skb) && unlikely(ipv6_hopopt_jumbo_remove(skb)))
+		goto drop;
 
+	if (tx->dqo.qpl) {
 		/* We do not need to verify the number of buffers used per
 		 * packet or per segment in case of TSO as with 2K size buffers
 		 * none of the TX packet rules would be violated.
@@ -868,24 +877,8 @@ static int gve_try_tx_skb(struct gve_priv *priv, struct gve_tx_ring *tx,
 		 */
 		num_buffer_descs = DIV_ROUND_UP(skb->len, GVE_TX_BUF_SIZE_DQO);
 	} else {
-		if (skb_is_gso(skb)) {
-			/* If TSO doesn't meet HW requirements, attempt to linearize the
-			 * packet.
-			 */
-			if (unlikely(!gve_can_send_tso(skb) &&
-				     skb_linearize(skb) < 0)) {
-				net_err_ratelimited("%s: Failed to transmit TSO packet\n",
-						    priv->dev->name);
-				goto drop;
-			}
-
-			if (unlikely(ipv6_hopopt_jumbo_remove(skb)))
-				goto drop;
-
-			num_buffer_descs = gve_num_buffer_descs_needed(skb);
-		} else {
-			num_buffer_descs = gve_num_buffer_descs_needed(skb);
-
+		num_buffer_descs = gve_num_buffer_descs_needed(skb);
+		if (!skb_is_gso(skb)) {
 			if (unlikely(num_buffer_descs > GVE_TX_MAX_DATA_DESCS)) {
 				if (unlikely(skb_linearize(skb) < 0))
 					goto drop;
diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h
index 1bf424ac3145..6022944d04f8 100644
--- a/drivers/net/ethernet/intel/i40e/i40e.h
+++ b/drivers/net/ethernet/intel/i40e/i40e.h
@@ -24,6 +24,7 @@
 #define I40E_MAX_VEB			16
 
 #define I40E_MAX_NUM_DESCRIPTORS	4096
+#define I40E_MAX_NUM_DESCRIPTORS_XL710	8160
 #define I40E_MAX_CSR_SPACE		(4 * 1024 * 1024 - 64 * 1024)
 #define I40E_DEFAULT_NUM_DESCRIPTORS	512
 #define I40E_REQ_DESCRIPTOR_MULTIPLE	32
@@ -33,11 +34,11 @@
 #define I40E_MIN_VSI_ALLOC		83 /* LAN, ATR, FCOE, 64 VF */
 /* max 16 qps */
 #define i40e_default_queues_per_vmdq(pf) \
-		(((pf)->hw_features & I40E_HW_RSS_AQ_CAPABLE) ? 4 : 1)
+	(test_bit(I40E_HW_CAP_RSS_AQ, (pf)->hw.caps) ? 4 : 1)
 #define I40E_DEFAULT_QUEUES_PER_VF	4
 #define I40E_MAX_VF_QUEUES		16
 #define i40e_pf_get_max_q_per_tc(pf) \
-		(((pf)->hw_features & I40E_HW_128_QP_RSS_CAPABLE) ? 128 : 64)
+	(test_bit(I40E_HW_CAP_128_QP_RSS, (pf)->hw.caps) ? 128 : 64)
 #define I40E_FDIR_RING_COUNT		32
 #define I40E_MAX_AQ_BUF_SIZE		4096
 #define I40E_AQ_LEN			256
@@ -78,7 +79,7 @@
 #define I40E_MAX_BW_INACTIVE_ACCUM	4 /* accumulate 4 credits max */
 
 /* driver state flags */
-enum i40e_state_t {
+enum i40e_state {
 	__I40E_TESTING,
 	__I40E_CONFIG_BUSY,
 	__I40E_CONFIG_DONE,
@@ -126,7 +127,7 @@ enum i40e_state_t {
 	BIT_ULL(__I40E_PF_RESET_AND_REBUILD_REQUESTED)
 
 /* VSI state flags */
-enum i40e_vsi_state_t {
+enum i40e_vsi_state {
 	__I40E_VSI_DOWN,
 	__I40E_VSI_NEEDS_RESTART,
 	__I40E_VSI_SYNCING_FILTERS,
@@ -138,6 +139,60 @@ enum i40e_vsi_state_t {
 	__I40E_VSI_STATE_SIZE__,
 };
 
+enum i40e_pf_flags {
+	I40E_FLAG_MSI_ENA,
+	I40E_FLAG_MSIX_ENA,
+	I40E_FLAG_RSS_ENA,
+	I40E_FLAG_VMDQ_ENA,
+	I40E_FLAG_SRIOV_ENA,
+	I40E_FLAG_DCB_CAPABLE,
+	I40E_FLAG_DCB_ENA,
+	I40E_FLAG_FD_SB_ENA,
+	I40E_FLAG_FD_ATR_ENA,
+	I40E_FLAG_MFP_ENA,
+	I40E_FLAG_HW_ATR_EVICT_ENA,
+	I40E_FLAG_VEB_MODE_ENA,
+	I40E_FLAG_VEB_STATS_ENA,
+	I40E_FLAG_LINK_POLLING_ENA,
+	I40E_FLAG_TRUE_PROMISC_ENA,
+	I40E_FLAG_LEGACY_RX_ENA,
+	I40E_FLAG_PTP_ENA,
+	I40E_FLAG_IWARP_ENA,
+	I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA,
+	I40E_FLAG_SOURCE_PRUNING_DIS,
+	I40E_FLAG_TC_MQPRIO_ENA,
+	I40E_FLAG_FD_SB_INACTIVE,
+	I40E_FLAG_FD_SB_TO_CLOUD_FILTER,
+	I40E_FLAG_FW_LLDP_DIS,
+	I40E_FLAG_RS_FEC,
+	I40E_FLAG_BASE_R_FEC,
+	/* TOTAL_PORT_SHUTDOWN_ENA
+	 * Allows to physically disable the link on the NIC's port.
+	 * If enabled, (after link down request from the OS)
+	 * no link, traffic or led activity is possible on that port.
+	 *
+	 * If I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA is set, the
+	 * I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA must be explicitly forced
+	 * to true and cannot be disabled by system admin at that time.
+	 * The functionalities are exclusive in terms of configuration, but
+	 * they also have similar behavior (allowing to disable physical
+	 * link of the port), with following differences:
+	 * - LINK_DOWN_ON_CLOSE_ENA is configurable at host OS run-time and
+	 *   is supported by whole family of 7xx Intel Ethernet Controllers
+	 * - TOTAL_PORT_SHUTDOWN_ENA may be enabled only before OS loads
+	 *   (in BIOS) only if motherboard's BIOS and NIC's FW has support of it
+	 * - when LINK_DOWN_ON_CLOSE_ENABLED is used, the link is being brought
+	 *   down by sending phy_type=0 to NIC's FW
+	 * - when TOTAL_PORT_SHUTDOWN_ENA is used, phy_type is not altered,
+	 *   instead the link is being brought down by clearing
+	 *   bit (I40E_AQ_PHY_ENABLE_LINK) in abilities field of
+	 *   i40e_aq_set_phy_config structure
+	 */
+	I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA,
+	I40E_FLAG_VF_VLAN_PRUNING_ENA,
+	I40E_PF_FLAGS_NBITS,		/* must be last */
+};
+
 enum i40e_interrupt_policy {
 	I40E_INTERRUPT_BEST_CASE,
 	I40E_INTERRUPT_MEDIUM,
@@ -480,78 +535,7 @@ struct i40e_pf {
 	struct timer_list service_timer;
 	struct work_struct service_task;
 
-	u32 hw_features;
-#define I40E_HW_RSS_AQ_CAPABLE			BIT(0)
-#define I40E_HW_128_QP_RSS_CAPABLE		BIT(1)
-#define I40E_HW_ATR_EVICT_CAPABLE		BIT(2)
-#define I40E_HW_WB_ON_ITR_CAPABLE		BIT(3)
-#define I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE	BIT(4)
-#define I40E_HW_NO_PCI_LINK_CHECK		BIT(5)
-#define I40E_HW_100M_SGMII_CAPABLE		BIT(6)
-#define I40E_HW_NO_DCB_SUPPORT			BIT(7)
-#define I40E_HW_USE_SET_LLDP_MIB		BIT(8)
-#define I40E_HW_GENEVE_OFFLOAD_CAPABLE		BIT(9)
-#define I40E_HW_PTP_L4_CAPABLE			BIT(10)
-#define I40E_HW_WOL_MC_MAGIC_PKT_WAKE		BIT(11)
-#define I40E_HW_HAVE_CRT_RETIMER		BIT(13)
-#define I40E_HW_OUTER_UDP_CSUM_CAPABLE		BIT(14)
-#define I40E_HW_PHY_CONTROLS_LEDS		BIT(15)
-#define I40E_HW_STOP_FW_LLDP			BIT(16)
-#define I40E_HW_PORT_ID_VALID			BIT(17)
-#define I40E_HW_RESTART_AUTONEG			BIT(18)
-
-	u32 flags;
-#define I40E_FLAG_RX_CSUM_ENABLED		BIT(0)
-#define I40E_FLAG_MSI_ENABLED			BIT(1)
-#define I40E_FLAG_MSIX_ENABLED			BIT(2)
-#define I40E_FLAG_RSS_ENABLED			BIT(3)
-#define I40E_FLAG_VMDQ_ENABLED			BIT(4)
-#define I40E_FLAG_SRIOV_ENABLED			BIT(5)
-#define I40E_FLAG_DCB_CAPABLE			BIT(6)
-#define I40E_FLAG_DCB_ENABLED			BIT(7)
-#define I40E_FLAG_FD_SB_ENABLED			BIT(8)
-#define I40E_FLAG_FD_ATR_ENABLED		BIT(9)
-#define I40E_FLAG_MFP_ENABLED			BIT(10)
-#define I40E_FLAG_HW_ATR_EVICT_ENABLED		BIT(11)
-#define I40E_FLAG_VEB_MODE_ENABLED		BIT(12)
-#define I40E_FLAG_VEB_STATS_ENABLED		BIT(13)
-#define I40E_FLAG_LINK_POLLING_ENABLED		BIT(14)
-#define I40E_FLAG_TRUE_PROMISC_SUPPORT		BIT(15)
-#define I40E_FLAG_LEGACY_RX			BIT(16)
-#define I40E_FLAG_PTP				BIT(17)
-#define I40E_FLAG_IWARP_ENABLED			BIT(18)
-#define I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED	BIT(19)
-#define I40E_FLAG_SOURCE_PRUNING_DISABLED       BIT(20)
-#define I40E_FLAG_TC_MQPRIO			BIT(21)
-#define I40E_FLAG_FD_SB_INACTIVE		BIT(22)
-#define I40E_FLAG_FD_SB_TO_CLOUD_FILTER		BIT(23)
-#define I40E_FLAG_DISABLE_FW_LLDP		BIT(24)
-#define I40E_FLAG_RS_FEC			BIT(25)
-#define I40E_FLAG_BASE_R_FEC			BIT(26)
-/* TOTAL_PORT_SHUTDOWN
- * Allows to physically disable the link on the NIC's port.
- * If enabled, (after link down request from the OS)
- * no link, traffic or led activity is possible on that port.
- *
- * If I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED is set, the
- * I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED must be explicitly forced to true
- * and cannot be disabled by system admin at that time.
- * The functionalities are exclusive in terms of configuration, but they also
- * have similar behavior (allowing to disable physical link of the port),
- * with following differences:
- * - LINK_DOWN_ON_CLOSE_ENABLED is configurable at host OS run-time and is
- *   supported by whole family of 7xx Intel Ethernet Controllers
- * - TOTAL_PORT_SHUTDOWN may be enabled only before OS loads (in BIOS)
- *   only if motherboard's BIOS and NIC's FW has support of it
- * - when LINK_DOWN_ON_CLOSE_ENABLED is used, the link is being brought down
- *   by sending phy_type=0 to NIC's FW
- * - when TOTAL_PORT_SHUTDOWN is used, phy_type is not altered, instead
- *   the link is being brought down by clearing bit (I40E_AQ_PHY_ENABLE_LINK)
- *   in abilities field of i40e_aq_set_phy_config structure
- */
-#define I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED	BIT(27)
-#define I40E_FLAG_VF_VLAN_PRUNING		BIT(28)
-
+	DECLARE_BITMAP(flags, I40E_PF_FLAGS_NBITS);
 	struct i40e_client_instance *cinst;
 	bool stat_offsets_loaded;
 	struct i40e_hw_port_stats stats;
@@ -1267,7 +1251,7 @@ struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr);
 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi);
 static inline bool i40e_is_sw_dcb(struct i40e_pf *pf)
 {
-	return !!(pf->flags & I40E_FLAG_DISABLE_FW_LLDP);
+	return test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags);
 }
 
 #ifdef CONFIG_I40E_DCB
@@ -1301,7 +1285,7 @@ int i40e_set_partition_bw_setting(struct i40e_pf *pf);
 int i40e_commit_partition_bw_setting(struct i40e_pf *pf);
 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup);
 
-void i40e_set_fec_in_flags(u8 fec_cfg, u32 *flags);
+void i40e_set_fec_in_flags(u8 fec_cfg, unsigned long *flags);
 
 static inline bool i40e_enabled_xdp_vsi(struct i40e_vsi *vsi)
 {
@@ -1321,13 +1305,13 @@ int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,
  * i40e_is_tc_mqprio_enabled - check if TC MQPRIO is enabled on PF
  * @pf: pointer to a pf.
  *
- * Check and return value of flag I40E_FLAG_TC_MQPRIO.
+ * Check and return state of flag I40E_FLAG_TC_MQPRIO.
  *
- * Return: I40E_FLAG_TC_MQPRIO set state.
+ * Return: true/false if I40E_FLAG_TC_MQPRIO is set or not
  **/
-static inline u32 i40e_is_tc_mqprio_enabled(struct i40e_pf *pf)
+static inline bool i40e_is_tc_mqprio_enabled(struct i40e_pf *pf)
 {
-	return pf->flags & I40E_FLAG_TC_MQPRIO;
+	return test_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);
 }
 
 /**
diff --git a/drivers/net/ethernet/intel/i40e/i40e_adminq.c b/drivers/net/ethernet/intel/i40e/i40e_adminq.c
index 9ce6e633cc2f..896c43905309 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_adminq.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_adminq.c
@@ -17,29 +17,16 @@ static void i40e_resume_aq(struct i40e_hw *hw);
 static void i40e_adminq_init_regs(struct i40e_hw *hw)
 {
 	/* set head and tail registers in our local struct */
-	if (i40e_is_vf(hw)) {
-		hw->aq.asq.tail = I40E_VF_ATQT1;
-		hw->aq.asq.head = I40E_VF_ATQH1;
-		hw->aq.asq.len  = I40E_VF_ATQLEN1;
-		hw->aq.asq.bal  = I40E_VF_ATQBAL1;
-		hw->aq.asq.bah  = I40E_VF_ATQBAH1;
-		hw->aq.arq.tail = I40E_VF_ARQT1;
-		hw->aq.arq.head = I40E_VF_ARQH1;
-		hw->aq.arq.len  = I40E_VF_ARQLEN1;
-		hw->aq.arq.bal  = I40E_VF_ARQBAL1;
-		hw->aq.arq.bah  = I40E_VF_ARQBAH1;
-	} else {
-		hw->aq.asq.tail = I40E_PF_ATQT;
-		hw->aq.asq.head = I40E_PF_ATQH;
-		hw->aq.asq.len  = I40E_PF_ATQLEN;
-		hw->aq.asq.bal  = I40E_PF_ATQBAL;
-		hw->aq.asq.bah  = I40E_PF_ATQBAH;
-		hw->aq.arq.tail = I40E_PF_ARQT;
-		hw->aq.arq.head = I40E_PF_ARQH;
-		hw->aq.arq.len  = I40E_PF_ARQLEN;
-		hw->aq.arq.bal  = I40E_PF_ARQBAL;
-		hw->aq.arq.bah  = I40E_PF_ARQBAH;
-	}
+	hw->aq.asq.tail = I40E_PF_ATQT;
+	hw->aq.asq.head = I40E_PF_ATQH;
+	hw->aq.asq.len  = I40E_PF_ATQLEN;
+	hw->aq.asq.bal  = I40E_PF_ATQBAL;
+	hw->aq.asq.bah  = I40E_PF_ATQBAH;
+	hw->aq.arq.tail = I40E_PF_ARQT;
+	hw->aq.arq.head = I40E_PF_ARQH;
+	hw->aq.arq.len  = I40E_PF_ARQLEN;
+	hw->aq.arq.bal  = I40E_PF_ARQBAL;
+	hw->aq.arq.bah  = I40E_PF_ARQBAH;
 }
 
 /**
@@ -503,44 +490,76 @@ static int i40e_shutdown_arq(struct i40e_hw *hw)
 }
 
 /**
- *  i40e_set_hw_flags - set HW flags
+ *  i40e_set_hw_caps - set HW flags
  *  @hw: pointer to the hardware structure
  **/
-static void i40e_set_hw_flags(struct i40e_hw *hw)
+static void i40e_set_hw_caps(struct i40e_hw *hw)
 {
-	struct i40e_adminq_info *aq = &hw->aq;
-
-	hw->flags = 0;
+	bitmap_zero(hw->caps, I40E_HW_CAPS_NBITS);
 
 	switch (hw->mac.type) {
 	case I40E_MAC_XL710:
-		if (aq->api_maj_ver > 1 ||
-		    (aq->api_maj_ver == 1 &&
-		     aq->api_min_ver >= I40E_MINOR_VER_GET_LINK_INFO_XL710)) {
-			hw->flags |= I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE;
-			hw->flags |= I40E_HW_FLAG_FW_LLDP_STOPPABLE;
+		if (i40e_is_aq_api_ver_ge(hw, 1,
+					  I40E_MINOR_VER_GET_LINK_INFO_XL710)) {
+			set_bit(I40E_HW_CAP_AQ_PHY_ACCESS, hw->caps);
+			set_bit(I40E_HW_CAP_FW_LLDP_STOPPABLE, hw->caps);
 			/* The ability to RX (not drop) 802.1ad frames */
-			hw->flags |= I40E_HW_FLAG_802_1AD_CAPABLE;
+			set_bit(I40E_HW_CAP_802_1AD, hw->caps);
+		}
+		if (i40e_is_aq_api_ver_ge(hw, 1, 5)) {
+			/* Supported in FW API version higher than 1.4 */
+			set_bit(I40E_HW_CAP_GENEVE_OFFLOAD, hw->caps);
+		}
+		if (i40e_is_fw_ver_lt(hw, 4, 33)) {
+			set_bit(I40E_HW_CAP_RESTART_AUTONEG, hw->caps);
+			/* No DCB support  for FW < v4.33 */
+			set_bit(I40E_HW_CAP_NO_DCB_SUPPORT, hw->caps);
+		}
+		if (i40e_is_fw_ver_lt(hw, 4, 3)) {
+			/* Disable FW LLDP if FW < v4.3 */
+			set_bit(I40E_HW_CAP_STOP_FW_LLDP, hw->caps);
+		}
+		if (i40e_is_fw_ver_ge(hw, 4, 40)) {
+			/* Use the FW Set LLDP MIB API if FW >= v4.40 */
+			set_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, hw->caps);
+		}
+		if (i40e_is_fw_ver_ge(hw, 6, 0)) {
+			/* Enable PTP L4 if FW > v6.0 */
+			set_bit(I40E_HW_CAP_PTP_L4, hw->caps);
 		}
 		break;
 	case I40E_MAC_X722:
-		hw->flags |= I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE |
-			     I40E_HW_FLAG_NVM_READ_REQUIRES_LOCK;
+		set_bit(I40E_HW_CAP_AQ_SRCTL_ACCESS_ENABLE, hw->caps);
+		set_bit(I40E_HW_CAP_NVM_READ_REQUIRES_LOCK, hw->caps);
+		set_bit(I40E_HW_CAP_RSS_AQ, hw->caps);
+		set_bit(I40E_HW_CAP_128_QP_RSS, hw->caps);
+		set_bit(I40E_HW_CAP_ATR_EVICT, hw->caps);
+		set_bit(I40E_HW_CAP_WB_ON_ITR, hw->caps);
+		set_bit(I40E_HW_CAP_MULTI_TCP_UDP_RSS_PCTYPE, hw->caps);
+		set_bit(I40E_HW_CAP_NO_PCI_LINK_CHECK, hw->caps);
+		set_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, hw->caps);
+		set_bit(I40E_HW_CAP_GENEVE_OFFLOAD, hw->caps);
+		set_bit(I40E_HW_CAP_PTP_L4, hw->caps);
+		set_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, hw->caps);
+		set_bit(I40E_HW_CAP_OUTER_UDP_CSUM, hw->caps);
+
+		if (rd32(hw, I40E_GLQF_FDEVICTENA(1)) !=
+		    I40E_FDEVICT_PCTYPE_DEFAULT) {
+			hw_warn(hw, "FD EVICT PCTYPES are not right, disable FD HW EVICT\n");
+			clear_bit(I40E_HW_CAP_ATR_EVICT, hw->caps);
+		}
 
-		if (aq->api_maj_ver > 1 ||
-		    (aq->api_maj_ver == 1 &&
-		     aq->api_min_ver >= I40E_MINOR_VER_FW_LLDP_STOPPABLE_X722))
-			hw->flags |= I40E_HW_FLAG_FW_LLDP_STOPPABLE;
+		if (i40e_is_aq_api_ver_ge(hw, 1,
+					  I40E_MINOR_VER_FW_LLDP_STOPPABLE_X722))
+			set_bit(I40E_HW_CAP_FW_LLDP_STOPPABLE, hw->caps);
 
-		if (aq->api_maj_ver > 1 ||
-		    (aq->api_maj_ver == 1 &&
-		     aq->api_min_ver >= I40E_MINOR_VER_GET_LINK_INFO_X722))
-			hw->flags |= I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE;
+		if (i40e_is_aq_api_ver_ge(hw, 1,
+					  I40E_MINOR_VER_GET_LINK_INFO_X722))
+			set_bit(I40E_HW_CAP_AQ_PHY_ACCESS, hw->caps);
 
-		if (aq->api_maj_ver > 1 ||
-		    (aq->api_maj_ver == 1 &&
-		     aq->api_min_ver >= I40E_MINOR_VER_FW_REQUEST_FEC_X722))
-			hw->flags |= I40E_HW_FLAG_X722_FEC_REQUEST_CAPABLE;
+		if (i40e_is_aq_api_ver_ge(hw, 1,
+					  I40E_MINOR_VER_FW_REQUEST_FEC_X722))
+			set_bit(I40E_HW_CAP_X722_FEC_REQUEST, hw->caps);
 
 		fallthrough;
 	default:
@@ -548,22 +567,18 @@ static void i40e_set_hw_flags(struct i40e_hw *hw)
 	}
 
 	/* Newer versions of firmware require lock when reading the NVM */
-	if (aq->api_maj_ver > 1 ||
-	    (aq->api_maj_ver == 1 &&
-	     aq->api_min_ver >= 5))
-		hw->flags |= I40E_HW_FLAG_NVM_READ_REQUIRES_LOCK;
-
-	if (aq->api_maj_ver > 1 ||
-	    (aq->api_maj_ver == 1 &&
-	     aq->api_min_ver >= 8)) {
-		hw->flags |= I40E_HW_FLAG_FW_LLDP_PERSISTENT;
-		hw->flags |= I40E_HW_FLAG_DROP_MODE;
-	}
+	if (i40e_is_aq_api_ver_ge(hw, 1, 5))
+		set_bit(I40E_HW_CAP_NVM_READ_REQUIRES_LOCK, hw->caps);
+
+	/* The ability to RX (not drop) 802.1ad frames was added in API 1.7 */
+	if (i40e_is_aq_api_ver_ge(hw, 1, 7))
+		set_bit(I40E_HW_CAP_802_1AD, hw->caps);
 
-	if (aq->api_maj_ver > 1 ||
-	    (aq->api_maj_ver == 1 &&
-	     aq->api_min_ver >= 9))
-		hw->flags |= I40E_HW_FLAG_AQ_PHY_ACCESS_EXTENDED;
+	if (i40e_is_aq_api_ver_ge(hw, 1, 8))
+		set_bit(I40E_HW_CAP_FW_LLDP_PERSISTENT, hw->caps);
+
+	if (i40e_is_aq_api_ver_ge(hw, 1, 9))
+		set_bit(I40E_HW_CAP_AQ_PHY_ACCESS_EXTENDED, hw->caps);
 }
 
 /**
@@ -633,7 +648,7 @@ int i40e_init_adminq(struct i40e_hw *hw)
 	/* Some features were introduced in different FW API version
 	 * for different MAC type.
 	 */
-	i40e_set_hw_flags(hw);
+	i40e_set_hw_caps(hw);
 
 	/* get the NVM version info */
 	i40e_read_nvm_word(hw, I40E_SR_NVM_DEV_STARTER_VERSION,
@@ -648,25 +663,7 @@ int i40e_init_adminq(struct i40e_hw *hw)
 			   &oem_lo);
 	hw->nvm.oem_ver = ((u32)oem_hi << 16) | oem_lo;
 
-	if (hw->mac.type == I40E_MAC_XL710 &&
-	    hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
-	    hw->aq.api_min_ver >= I40E_MINOR_VER_GET_LINK_INFO_XL710) {
-		hw->flags |= I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE;
-		hw->flags |= I40E_HW_FLAG_FW_LLDP_STOPPABLE;
-	}
-	if (hw->mac.type == I40E_MAC_X722 &&
-	    hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
-	    hw->aq.api_min_ver >= I40E_MINOR_VER_FW_LLDP_STOPPABLE_X722) {
-		hw->flags |= I40E_HW_FLAG_FW_LLDP_STOPPABLE;
-	}
-
-	/* The ability to RX (not drop) 802.1ad frames was added in API 1.7 */
-	if (hw->aq.api_maj_ver > 1 ||
-	    (hw->aq.api_maj_ver == 1 &&
-	     hw->aq.api_min_ver >= 7))
-		hw->flags |= I40E_HW_FLAG_802_1AD_CAPABLE;
-
-	if (hw->aq.api_maj_ver > I40E_FW_API_VERSION_MAJOR) {
+	if (i40e_is_aq_api_ver_ge(hw, I40E_FW_API_VERSION_MAJOR + 1, 0)) {
 		ret_code = -EIO;
 		goto init_adminq_free_arq;
 	}
diff --git a/drivers/net/ethernet/intel/i40e/i40e_common.c b/drivers/net/ethernet/intel/i40e/i40e_common.c
index d7e24d661724..e171f4814e21 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_common.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_common.c
@@ -1374,8 +1374,8 @@ i40e_aq_get_phy_capabilities(struct i40e_hw *hw,
 
 	if (report_init) {
 		if (hw->mac.type ==  I40E_MAC_XL710 &&
-		    hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
-		    hw->aq.api_min_ver >= I40E_MINOR_VER_GET_LINK_INFO_XL710) {
+		    i40e_is_aq_api_ver_ge(hw, I40E_FW_API_VERSION_MAJOR,
+					  I40E_MINOR_VER_GET_LINK_INFO_XL710)) {
 			status = i40e_aq_get_link_info(hw, true, NULL, NULL);
 		} else {
 			hw->phy.phy_types = le32_to_cpu(abilities->phy_type);
@@ -1645,12 +1645,11 @@ int i40e_aq_get_link_info(struct i40e_hw *hw,
 	else
 		hw_link_info->lse_enable = false;
 
-	if ((hw->mac.type == I40E_MAC_XL710) &&
-	    (hw->aq.fw_maj_ver < 4 || (hw->aq.fw_maj_ver == 4 &&
-	     hw->aq.fw_min_ver < 40)) && hw_link_info->phy_type == 0xE)
+	if (hw->mac.type == I40E_MAC_XL710 && i40e_is_fw_ver_lt(hw, 4, 40) &&
+	    hw_link_info->phy_type == 0xE)
 		hw_link_info->phy_type = I40E_PHY_TYPE_10GBASE_SFPP_CU;
 
-	if (hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE &&
+	if (test_bit(I40E_HW_CAP_AQ_PHY_ACCESS, hw->caps) &&
 	    hw->mac.type != I40E_MAC_X722) {
 		__le32 tmp;
 
@@ -1749,21 +1748,6 @@ int i40e_aq_set_phy_debug(struct i40e_hw *hw, u8 cmd_flags,
 	return status;
 }
 
-/**
- * i40e_is_aq_api_ver_ge
- * @aq: pointer to AdminQ info containing HW API version to compare
- * @maj: API major value
- * @min: API minor value
- *
- * Assert whether current HW API version is greater/equal than provided.
- **/
-static bool i40e_is_aq_api_ver_ge(struct i40e_adminq_info *aq, u16 maj,
-				  u16 min)
-{
-	return (aq->api_maj_ver > maj ||
-		(aq->api_maj_ver == maj && aq->api_min_ver >= min));
-}
-
 /**
  * i40e_aq_add_vsi
  * @hw: pointer to the hw struct
@@ -1890,14 +1874,14 @@ int i40e_aq_set_vsi_unicast_promiscuous(struct i40e_hw *hw,
 
 	if (set) {
 		flags |= I40E_AQC_SET_VSI_PROMISC_UNICAST;
-		if (rx_only_promisc && i40e_is_aq_api_ver_ge(&hw->aq, 1, 5))
+		if (rx_only_promisc && i40e_is_aq_api_ver_ge(hw, 1, 5))
 			flags |= I40E_AQC_SET_VSI_PROMISC_RX_ONLY;
 	}
 
 	cmd->promiscuous_flags = cpu_to_le16(flags);
 
 	cmd->valid_flags = cpu_to_le16(I40E_AQC_SET_VSI_PROMISC_UNICAST);
-	if (i40e_is_aq_api_ver_ge(&hw->aq, 1, 5))
+	if (i40e_is_aq_api_ver_ge(hw, 1, 5))
 		cmd->valid_flags |=
 			cpu_to_le16(I40E_AQC_SET_VSI_PROMISC_RX_ONLY);
 
@@ -2000,13 +1984,13 @@ int i40e_aq_set_vsi_uc_promisc_on_vlan(struct i40e_hw *hw,
 
 	if (enable) {
 		flags |= I40E_AQC_SET_VSI_PROMISC_UNICAST;
-		if (i40e_is_aq_api_ver_ge(&hw->aq, 1, 5))
+		if (i40e_is_aq_api_ver_ge(hw, 1, 5))
 			flags |= I40E_AQC_SET_VSI_PROMISC_RX_ONLY;
 	}
 
 	cmd->promiscuous_flags = cpu_to_le16(flags);
 	cmd->valid_flags = cpu_to_le16(I40E_AQC_SET_VSI_PROMISC_UNICAST);
-	if (i40e_is_aq_api_ver_ge(&hw->aq, 1, 5))
+	if (i40e_is_aq_api_ver_ge(hw, 1, 5))
 		cmd->valid_flags |=
 			cpu_to_le16(I40E_AQC_SET_VSI_PROMISC_RX_ONLY);
 	cmd->seid = cpu_to_le16(seid);
@@ -2253,7 +2237,7 @@ int i40e_aq_set_switch_config(struct i40e_hw *hw,
 	scfg->flags = cpu_to_le16(flags);
 	scfg->valid_flags = cpu_to_le16(valid_flags);
 	scfg->mode = mode;
-	if (hw->flags & I40E_HW_FLAG_802_1AD_CAPABLE) {
+	if (test_bit(I40E_HW_CAP_802_1AD, hw->caps)) {
 		scfg->switch_tag = cpu_to_le16(hw->switch_tag);
 		scfg->first_tag = cpu_to_le16(hw->first_tag);
 		scfg->second_tag = cpu_to_le16(hw->second_tag);
@@ -3637,7 +3621,7 @@ i40e_aq_restore_lldp(struct i40e_hw *hw, u8 *setting, bool restore,
 		(struct i40e_aqc_lldp_restore *)&desc.params.raw;
 	int status;
 
-	if (!(hw->flags & I40E_HW_FLAG_FW_LLDP_PERSISTENT)) {
+	if (!test_bit(I40E_HW_CAP_FW_LLDP_PERSISTENT, hw->caps)) {
 		i40e_debug(hw, I40E_DEBUG_ALL,
 			   "Restore LLDP not supported by current FW version.\n");
 		return -ENODEV;
@@ -3680,7 +3664,7 @@ int i40e_aq_stop_lldp(struct i40e_hw *hw, bool shutdown_agent,
 		cmd->command |= I40E_AQ_LLDP_AGENT_SHUTDOWN;
 
 	if (persist) {
-		if (hw->flags & I40E_HW_FLAG_FW_LLDP_PERSISTENT)
+		if (test_bit(I40E_HW_CAP_FW_LLDP_PERSISTENT, hw->caps))
 			cmd->command |= I40E_AQ_LLDP_AGENT_STOP_PERSIST;
 		else
 			i40e_debug(hw, I40E_DEBUG_ALL,
@@ -3713,7 +3697,7 @@ int i40e_aq_start_lldp(struct i40e_hw *hw, bool persist,
 	cmd->command = I40E_AQ_LLDP_AGENT_START;
 
 	if (persist) {
-		if (hw->flags & I40E_HW_FLAG_FW_LLDP_PERSISTENT)
+		if (test_bit(I40E_HW_CAP_FW_LLDP_PERSISTENT, hw->caps))
 			cmd->command |= I40E_AQ_LLDP_AGENT_START_PERSIST;
 		else
 			i40e_debug(hw, I40E_DEBUG_ALL,
@@ -3741,7 +3725,7 @@ i40e_aq_set_dcb_parameters(struct i40e_hw *hw, bool dcb_enable,
 		(struct i40e_aqc_set_dcb_parameters *)&desc.params.raw;
 	int status;
 
-	if (!(hw->flags & I40E_HW_FLAG_FW_LLDP_STOPPABLE))
+	if (!test_bit(I40E_HW_CAP_FW_LLDP_STOPPABLE, hw->caps))
 		return -ENODEV;
 
 	i40e_fill_default_direct_cmd_desc(&desc,
@@ -5043,7 +5027,7 @@ static int i40e_led_get_reg(struct i40e_hw *hw, u16 led_addr,
 	u32 i;
 
 	*reg_val = 0;
-	if (hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE) {
+	if (test_bit(I40E_HW_CAP_AQ_PHY_ACCESS, hw->caps)) {
 		status =
 		       i40e_aq_get_phy_register(hw,
 						I40E_AQ_PHY_REG_ACCESS_EXTERNAL,
@@ -5076,7 +5060,7 @@ static int i40e_led_set_reg(struct i40e_hw *hw, u16 led_addr,
 	int status;
 	u32 i;
 
-	if (hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE) {
+	if (test_bit(I40E_HW_CAP_AQ_PHY_ACCESS, hw->caps)) {
 		status =
 		       i40e_aq_set_phy_register(hw,
 						I40E_AQ_PHY_REG_ACCESS_EXTERNAL,
@@ -5115,7 +5099,7 @@ int i40e_led_get_phy(struct i40e_hw *hw, u16 *led_addr,
 	u8 port_num;
 	u32 i;
 
-	if (hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE) {
+	if (test_bit(I40E_HW_CAP_AQ_PHY_ACCESS, hw->caps)) {
 		status =
 		      i40e_aq_get_phy_register(hw,
 					       I40E_AQ_PHY_REG_ACCESS_EXTERNAL,
@@ -5238,14 +5222,14 @@ int i40e_aq_rx_ctl_read_register(struct i40e_hw *hw,
  **/
 u32 i40e_read_rx_ctl(struct i40e_hw *hw, u32 reg_addr)
 {
-	bool use_register;
+	bool use_register = false;
 	int status = 0;
 	int retry = 5;
 	u32 val = 0;
 
-	use_register = (((hw->aq.api_maj_ver == 1) &&
-			(hw->aq.api_min_ver < 5)) ||
-			(hw->mac.type == I40E_MAC_X722));
+	if (i40e_is_aq_api_ver_lt(hw, 1, 5) || hw->mac.type == I40E_MAC_X722)
+		use_register = true;
+
 	if (!use_register) {
 do_retry:
 		status = i40e_aq_rx_ctl_read_register(hw, reg_addr, &val, NULL);
@@ -5300,13 +5284,13 @@ int i40e_aq_rx_ctl_write_register(struct i40e_hw *hw,
  **/
 void i40e_write_rx_ctl(struct i40e_hw *hw, u32 reg_addr, u32 reg_val)
 {
-	bool use_register;
+	bool use_register = false;
 	int status = 0;
 	int retry = 5;
 
-	use_register = (((hw->aq.api_maj_ver == 1) &&
-			(hw->aq.api_min_ver < 5)) ||
-			(hw->mac.type == I40E_MAC_X722));
+	if (i40e_is_aq_api_ver_lt(hw, 1, 5) || hw->mac.type == I40E_MAC_X722)
+		use_register = true;
+
 	if (!use_register) {
 do_retry:
 		status = i40e_aq_rx_ctl_write_register(hw, reg_addr,
@@ -5335,7 +5319,7 @@ static void i40e_mdio_if_number_selection(struct i40e_hw *hw, bool set_mdio,
 					  struct i40e_aqc_phy_register_access *cmd)
 {
 	if (set_mdio && cmd->phy_interface == I40E_AQ_PHY_REG_ACCESS_EXTERNAL) {
-		if (hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_EXTENDED)
+		if (test_bit(I40E_HW_CAP_AQ_PHY_ACCESS_EXTENDED, hw->caps))
 			cmd->cmd_flags |=
 				I40E_AQ_PHY_REG_ACCESS_SET_MDIO_IF_NUMBER |
 				((mdio_num <<
diff --git a/drivers/net/ethernet/intel/i40e/i40e_dcb.c b/drivers/net/ethernet/intel/i40e/i40e_dcb.c
index 68602fc375f6..498728e16a37 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_dcb.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_dcb.c
@@ -804,14 +804,11 @@ int i40e_get_dcb_config(struct i40e_hw *hw)
 	int ret = 0;
 
 	/* If Firmware version < v4.33 on X710/XL710, IEEE only */
-	if ((hw->mac.type == I40E_MAC_XL710) &&
-	    (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) ||
-	      (hw->aq.fw_maj_ver < 4)))
+	if (hw->mac.type == I40E_MAC_XL710 && i40e_is_fw_ver_lt(hw, 4, 33))
 		return i40e_get_ieee_dcb_config(hw);
 
 	/* If Firmware version == v4.33 on X710/XL710, use old CEE struct */
-	if ((hw->mac.type == I40E_MAC_XL710) &&
-	    ((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver == 33))) {
+	if (hw->mac.type == I40E_MAC_XL710 && i40e_is_fw_ver_eq(hw, 4, 33)) {
 		ret = i40e_aq_get_cee_dcb_config(hw, &cee_v1_cfg,
 						 sizeof(cee_v1_cfg), NULL);
 		if (!ret) {
@@ -877,7 +874,7 @@ int i40e_init_dcb(struct i40e_hw *hw, bool enable_mib_change)
 		return -EOPNOTSUPP;
 
 	/* Read LLDP NVM area */
-	if (hw->flags & I40E_HW_FLAG_FW_LLDP_PERSISTENT) {
+	if (test_bit(I40E_HW_CAP_FW_LLDP_PERSISTENT, hw->caps)) {
 		u8 offset = 0;
 
 		if (hw->mac.type == I40E_MAC_XL710)
diff --git a/drivers/net/ethernet/intel/i40e/i40e_dcb_nl.c b/drivers/net/ethernet/intel/i40e/i40e_dcb_nl.c
index 077a95dad32c..4721845fda6e 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_dcb_nl.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_dcb_nl.c
@@ -310,8 +310,8 @@ static u8 i40e_dcbnl_getstate(struct net_device *netdev)
 	struct i40e_pf *pf = i40e_netdev_to_pf(netdev);
 
 	dev_dbg(&pf->pdev->dev, "DCB state=%d\n",
-		!!(pf->flags & I40E_FLAG_DCB_ENABLED));
-	return !!(pf->flags & I40E_FLAG_DCB_ENABLED);
+		test_bit(I40E_FLAG_DCB_ENA, pf->flags) ? 1 : 0);
+	return test_bit(I40E_FLAG_DCB_ENA, pf->flags) ? 1 : 0;
 }
 
 /**
@@ -331,19 +331,19 @@ static u8 i40e_dcbnl_setstate(struct net_device *netdev, u8 state)
 		return ret;
 
 	dev_dbg(&pf->pdev->dev, "new state=%d current state=%d\n",
-		state, (pf->flags & I40E_FLAG_DCB_ENABLED) ? 1 : 0);
+		state, test_bit(I40E_FLAG_DCB_ENA, pf->flags) ? 1 : 0);
 	/* Nothing to do */
-	if (!state == !(pf->flags & I40E_FLAG_DCB_ENABLED))
+	if (!state == !test_bit(I40E_FLAG_DCB_ENA, pf->flags))
 		return ret;
 
 	if (i40e_is_sw_dcb(pf)) {
 		if (state) {
-			pf->flags |= I40E_FLAG_DCB_ENABLED;
+			set_bit(I40E_FLAG_DCB_ENA, pf->flags);
 			memcpy(&pf->hw.desired_dcbx_config,
 			       &pf->hw.local_dcbx_config,
 			       sizeof(struct i40e_dcbx_config));
 		} else {
-			pf->flags &= ~I40E_FLAG_DCB_ENABLED;
+			clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
 		}
 	} else {
 		/* Cannot directly manipulate FW LLDP Agent */
@@ -653,7 +653,7 @@ static u8 i40e_dcbnl_get_cap(struct net_device *netdev, int capid, u8 *cap)
 {
 	struct i40e_pf *pf = i40e_netdev_to_pf(netdev);
 
-	if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
+	if (!test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))
 		return I40E_DCBNL_STATUS_ERROR;
 
 	switch (capid) {
@@ -693,7 +693,7 @@ static int i40e_dcbnl_getnumtcs(struct net_device *netdev, int tcid, u8 *num)
 {
 	struct i40e_pf *pf = i40e_netdev_to_pf(netdev);
 
-	if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
+	if (!test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))
 		return -EINVAL;
 
 	*num = I40E_MAX_TRAFFIC_CLASS;
@@ -827,15 +827,12 @@ static void i40e_dcbnl_get_perm_hw_addr(struct net_device *dev,
 					u8 *perm_addr)
 {
 	struct i40e_pf *pf = i40e_netdev_to_pf(dev);
-	int i, j;
+	int i;
 
 	memset(perm_addr, 0xff, MAX_ADDR_LEN);
 
 	for (i = 0; i < dev->addr_len; i++)
 		perm_addr[i] = pf->hw.mac.perm_addr[i];
-
-	for (j = 0; j < dev->addr_len; j++, i++)
-		perm_addr[i] = pf->hw.mac.san_addr[j];
 }
 
 static const struct dcbnl_rtnl_ops dcbnl_ops = {
@@ -891,11 +888,11 @@ void i40e_dcbnl_set_all(struct i40e_vsi *vsi)
 		return;
 
 	/* DCB not enabled */
-	if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
+	if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags))
 		return;
 
 	/* MFP mode but not an iSCSI PF so return */
-	if ((pf->flags & I40E_FLAG_MFP_ENABLED) && !(hw->func_caps.iscsi))
+	if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) && !(hw->func_caps.iscsi))
 		return;
 
 	dcbxcfg = &hw->local_dcbx_config;
@@ -1002,7 +999,7 @@ void i40e_dcbnl_flush_apps(struct i40e_pf *pf,
 	int i;
 
 	/* MFP mode but not an iSCSI PF so return */
-	if ((pf->flags & I40E_FLAG_MFP_ENABLED) && !(pf->hw.func_caps.iscsi))
+	if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) && !(pf->hw.func_caps.iscsi))
 		return;
 
 	for (i = 0; i < old_cfg->numapps; i++) {
@@ -1025,7 +1022,7 @@ void i40e_dcbnl_setup(struct i40e_vsi *vsi)
 	struct i40e_pf *pf = i40e_netdev_to_pf(dev);
 
 	/* Not DCB capable */
-	if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
+	if (!test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))
 		return;
 
 	dev->dcbnl_ops = &dcbnl_ops;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_debug.h b/drivers/net/ethernet/intel/i40e/i40e_debug.h
index 27ebc72d8bfe..e9871dfb32bd 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_debug.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_debug.h
@@ -37,6 +37,7 @@ struct i40e_hw;
 struct device *i40e_hw_to_dev(struct i40e_hw *hw);
 
 #define hw_dbg(hw, S, A...) dev_dbg(i40e_hw_to_dev(hw), S, ##A)
+#define hw_warn(hw, S, A...) dev_warn(i40e_hw_to_dev(hw), S, ##A)
 
 #define i40e_debug(h, m, s, ...)				\
 do {								\
diff --git a/drivers/net/ethernet/intel/i40e/i40e_debugfs.c b/drivers/net/ethernet/intel/i40e/i40e_debugfs.c
index 999c9708def5..88240571721a 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_debugfs.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_debugfs.c
@@ -147,9 +147,8 @@ static void i40e_dbg_dump_vsi_seid(struct i40e_pf *pf, int seid)
 			 "    state[%d] = %08lx\n",
 			 i, vsi->state[i]);
 	if (vsi == pf->vsi[pf->lan_vsi])
-		dev_info(&pf->pdev->dev, "    MAC address: %pM SAN MAC: %pM Port MAC: %pM\n",
+		dev_info(&pf->pdev->dev, "    MAC address: %pM Port MAC: %pM\n",
 			 pf->hw.mac.addr,
-			 pf->hw.mac.san_addr,
 			 pf->hw.mac.port_addr);
 	hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
 		dev_info(&pf->pdev->dev,
@@ -820,8 +819,8 @@ static ssize_t i40e_dbg_command_write(struct file *filp,
 		/* By default we are in VEPA mode, if this is the first VF/VMDq
 		 * VSI to be added switch to VEB mode.
 		 */
-		if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
-			pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
+		if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {
+			set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
 			i40e_do_reset_safe(pf, I40E_PF_RESET_FLAG);
 		}
 
diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
index fd7163128c4d..eb9a7b32af73 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
@@ -430,35 +430,35 @@ static const char i40e_gstrings_test[][ETH_GSTRING_LEN] = {
 
 struct i40e_priv_flags {
 	char flag_string[ETH_GSTRING_LEN];
-	u64 flag;
+	u8 bitno;
 	bool read_only;
 };
 
-#define I40E_PRIV_FLAG(_name, _flag, _read_only) { \
+#define I40E_PRIV_FLAG(_name, _bitno, _read_only) { \
 	.flag_string = _name, \
-	.flag = _flag, \
+	.bitno = _bitno, \
 	.read_only = _read_only, \
 }
 
 static const struct i40e_priv_flags i40e_gstrings_priv_flags[] = {
 	/* NOTE: MFP setting cannot be changed */
-	I40E_PRIV_FLAG("MFP", I40E_FLAG_MFP_ENABLED, 1),
+	I40E_PRIV_FLAG("MFP", I40E_FLAG_MFP_ENA, 1),
 	I40E_PRIV_FLAG("total-port-shutdown",
-		       I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED, 1),
-	I40E_PRIV_FLAG("LinkPolling", I40E_FLAG_LINK_POLLING_ENABLED, 0),
-	I40E_PRIV_FLAG("flow-director-atr", I40E_FLAG_FD_ATR_ENABLED, 0),
-	I40E_PRIV_FLAG("veb-stats", I40E_FLAG_VEB_STATS_ENABLED, 0),
-	I40E_PRIV_FLAG("hw-atr-eviction", I40E_FLAG_HW_ATR_EVICT_ENABLED, 0),
+		       I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, 1),
+	I40E_PRIV_FLAG("LinkPolling", I40E_FLAG_LINK_POLLING_ENA, 0),
+	I40E_PRIV_FLAG("flow-director-atr", I40E_FLAG_FD_ATR_ENA, 0),
+	I40E_PRIV_FLAG("veb-stats", I40E_FLAG_VEB_STATS_ENA, 0),
+	I40E_PRIV_FLAG("hw-atr-eviction", I40E_FLAG_HW_ATR_EVICT_ENA, 0),
 	I40E_PRIV_FLAG("link-down-on-close",
-		       I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED, 0),
-	I40E_PRIV_FLAG("legacy-rx", I40E_FLAG_LEGACY_RX, 0),
+		       I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, 0),
+	I40E_PRIV_FLAG("legacy-rx", I40E_FLAG_LEGACY_RX_ENA, 0),
 	I40E_PRIV_FLAG("disable-source-pruning",
-		       I40E_FLAG_SOURCE_PRUNING_DISABLED, 0),
-	I40E_PRIV_FLAG("disable-fw-lldp", I40E_FLAG_DISABLE_FW_LLDP, 0),
+		       I40E_FLAG_SOURCE_PRUNING_DIS, 0),
+	I40E_PRIV_FLAG("disable-fw-lldp", I40E_FLAG_FW_LLDP_DIS, 0),
 	I40E_PRIV_FLAG("rs-fec", I40E_FLAG_RS_FEC, 0),
 	I40E_PRIV_FLAG("base-r-fec", I40E_FLAG_BASE_R_FEC, 0),
 	I40E_PRIV_FLAG("vf-vlan-pruning",
-		       I40E_FLAG_VF_VLAN_PRUNING, 0),
+		       I40E_FLAG_VF_VLAN_PRUNING_ENA, 0),
 };
 
 #define I40E_PRIV_FLAGS_STR_LEN ARRAY_SIZE(i40e_gstrings_priv_flags)
@@ -466,7 +466,7 @@ static const struct i40e_priv_flags i40e_gstrings_priv_flags[] = {
 /* Private flags with a global effect, restricted to PF 0 */
 static const struct i40e_priv_flags i40e_gl_gstrings_priv_flags[] = {
 	I40E_PRIV_FLAG("vf-true-promisc-support",
-		       I40E_FLAG_TRUE_PROMISC_SUPPORT, 0),
+		       I40E_FLAG_TRUE_PROMISC_ENA, 0),
 };
 
 #define I40E_GL_PRIV_FLAGS_STR_LEN ARRAY_SIZE(i40e_gl_gstrings_priv_flags)
@@ -502,7 +502,7 @@ static void i40e_phy_type_to_ethtool(struct i40e_pf *pf,
 		if (hw_link_info->requested_speeds & I40E_LINK_SPEED_1GB)
 			ethtool_link_ksettings_add_link_mode(ks, advertising,
 							     1000baseT_Full);
-		if (pf->hw_features & I40E_HW_100M_SGMII_CAPABLE) {
+		if (test_bit(I40E_HW_CAP_100M_SGMII, pf->hw.caps)) {
 			ethtool_link_ksettings_add_link_mode(ks, supported,
 							     100baseT_Full);
 			ethtool_link_ksettings_add_link_mode(ks, advertising,
@@ -601,7 +601,7 @@ static void i40e_phy_type_to_ethtool(struct i40e_pf *pf,
 							     10000baseKX4_Full);
 	}
 	if (phy_types & I40E_CAP_PHY_TYPE_10GBASE_KR &&
-	    !(pf->hw_features & I40E_HW_HAVE_CRT_RETIMER)) {
+	    !test_bit(I40E_HW_CAP_CRT_RETIMER, pf->hw.caps)) {
 		ethtool_link_ksettings_add_link_mode(ks, supported,
 						     10000baseKR_Full);
 		if (hw_link_info->requested_speeds & I40E_LINK_SPEED_10GB)
@@ -609,7 +609,7 @@ static void i40e_phy_type_to_ethtool(struct i40e_pf *pf,
 							     10000baseKR_Full);
 	}
 	if (phy_types & I40E_CAP_PHY_TYPE_1000BASE_KX &&
-	    !(pf->hw_features & I40E_HW_HAVE_CRT_RETIMER)) {
+	    !test_bit(I40E_HW_CAP_CRT_RETIMER, pf->hw.caps)) {
 		ethtool_link_ksettings_add_link_mode(ks, supported,
 						     1000baseKX_Full);
 		if (hw_link_info->requested_speeds & I40E_LINK_SPEED_1GB)
@@ -917,7 +917,7 @@ static void i40e_get_settings_link_up(struct i40e_hw *hw,
 		if (hw_link_info->requested_speeds & I40E_LINK_SPEED_1GB)
 			ethtool_link_ksettings_add_link_mode(ks, advertising,
 							     1000baseT_Full);
-		if (pf->hw_features & I40E_HW_100M_SGMII_CAPABLE) {
+		if (test_bit(I40E_HW_CAP_100M_SGMII, pf->hw.caps)) {
 			ethtool_link_ksettings_add_link_mode(ks, supported,
 							     100baseT_Full);
 			if (hw_link_info->requested_speeds &
@@ -1488,12 +1488,8 @@ static int i40e_set_fec_cfg(struct net_device *netdev, u8 fec_cfg)
 	struct i40e_pf *pf = np->vsi->back;
 	struct i40e_hw *hw = &pf->hw;
 	int status = 0;
-	u32 flags = 0;
 	int err = 0;
 
-	flags = READ_ONCE(pf->flags);
-	i40e_set_fec_in_flags(fec_cfg, &flags);
-
 	/* Get the current phy config */
 	memset(&abilities, 0, sizeof(abilities));
 	status = i40e_aq_get_phy_capabilities(hw, false, false, &abilities,
@@ -1525,7 +1521,7 @@ static int i40e_set_fec_cfg(struct net_device *netdev, u8 fec_cfg)
 			err = -EAGAIN;
 			goto done;
 		}
-		pf->flags = flags;
+		i40e_set_fec_in_flags(fec_cfg, pf->flags);
 		status = i40e_update_link_info(hw);
 		if (status)
 			/* debug level message only due to relation to the link
@@ -1599,7 +1595,7 @@ static int i40e_set_fec_param(struct net_device *netdev,
 		return -EPERM;
 
 	if (hw->mac.type == I40E_MAC_X722 &&
-	    !(hw->flags & I40E_HW_FLAG_X722_FEC_REQUEST_CAPABLE)) {
+	    !test_bit(I40E_HW_CAP_X722_FEC_REQUEST, hw->caps)) {
 		netdev_err(netdev, "Setting FEC encoding not supported by firmware. Please update the NVM image.\n");
 		return -EOPNOTSUPP;
 	}
@@ -2015,6 +2011,18 @@ static void i40e_get_drvinfo(struct net_device *netdev,
 		drvinfo->n_priv_flags += I40E_GL_PRIV_FLAGS_STR_LEN;
 }
 
+static u32 i40e_get_max_num_descriptors(struct i40e_pf *pf)
+{
+	struct i40e_hw *hw = &pf->hw;
+
+	switch (hw->mac.type) {
+	case I40E_MAC_XL710:
+		return I40E_MAX_NUM_DESCRIPTORS_XL710;
+	default:
+		return I40E_MAX_NUM_DESCRIPTORS;
+	}
+}
+
 static void i40e_get_ringparam(struct net_device *netdev,
 			       struct ethtool_ringparam *ring,
 			       struct kernel_ethtool_ringparam *kernel_ring,
@@ -2024,8 +2032,8 @@ static void i40e_get_ringparam(struct net_device *netdev,
 	struct i40e_pf *pf = np->vsi->back;
 	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
 
-	ring->rx_max_pending = I40E_MAX_NUM_DESCRIPTORS;
-	ring->tx_max_pending = I40E_MAX_NUM_DESCRIPTORS;
+	ring->rx_max_pending = i40e_get_max_num_descriptors(pf);
+	ring->tx_max_pending = i40e_get_max_num_descriptors(pf);
 	ring->rx_mini_max_pending = 0;
 	ring->rx_jumbo_max_pending = 0;
 	ring->rx_pending = vsi->rx_rings[0]->count;
@@ -2050,12 +2058,12 @@ static int i40e_set_ringparam(struct net_device *netdev,
 			      struct kernel_ethtool_ringparam *kernel_ring,
 			      struct netlink_ext_ack *extack)
 {
+	u32 new_rx_count, new_tx_count, max_num_descriptors;
 	struct i40e_ring *tx_rings = NULL, *rx_rings = NULL;
 	struct i40e_netdev_priv *np = netdev_priv(netdev);
 	struct i40e_hw *hw = &np->vsi->back->hw;
 	struct i40e_vsi *vsi = np->vsi;
 	struct i40e_pf *pf = vsi->back;
-	u32 new_rx_count, new_tx_count;
 	u16 tx_alloc_queue_pairs;
 	int timeout = 50;
 	int i, err = 0;
@@ -2063,14 +2071,15 @@ static int i40e_set_ringparam(struct net_device *netdev,
 	if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending))
 		return -EINVAL;
 
-	if (ring->tx_pending > I40E_MAX_NUM_DESCRIPTORS ||
+	max_num_descriptors = i40e_get_max_num_descriptors(pf);
+	if (ring->tx_pending > max_num_descriptors ||
 	    ring->tx_pending < I40E_MIN_NUM_DESCRIPTORS ||
-	    ring->rx_pending > I40E_MAX_NUM_DESCRIPTORS ||
+	    ring->rx_pending > max_num_descriptors ||
 	    ring->rx_pending < I40E_MIN_NUM_DESCRIPTORS) {
 		netdev_info(netdev,
 			    "Descriptors requested (Tx: %d / Rx: %d) out of range [%d-%d]\n",
 			    ring->tx_pending, ring->rx_pending,
-			    I40E_MIN_NUM_DESCRIPTORS, I40E_MAX_NUM_DESCRIPTORS);
+			    I40E_MIN_NUM_DESCRIPTORS, max_num_descriptors);
 		return -EINVAL;
 	}
 
@@ -2419,7 +2428,7 @@ static void i40e_get_ethtool_stats(struct net_device *netdev,
 
 	veb_stats = ((pf->lan_veb != I40E_NO_VEB) &&
 		     (pf->lan_veb < I40E_MAX_VEB) &&
-		     (pf->flags & I40E_FLAG_VEB_STATS_ENABLED));
+		     test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags));
 
 	if (veb_stats) {
 		veb = pf->veb[pf->lan_veb];
@@ -2548,7 +2557,7 @@ static int i40e_get_ts_info(struct net_device *dev,
 	struct i40e_pf *pf = i40e_netdev_to_pf(dev);
 
 	/* only report HW timestamping if PTP is enabled */
-	if (!(pf->flags & I40E_FLAG_PTP))
+	if (!test_bit(I40E_FLAG_PTP_ENA, pf->flags))
 		return ethtool_op_get_ts_info(dev, info);
 
 	info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE |
@@ -2570,7 +2579,7 @@ static int i40e_get_ts_info(struct net_device *dev,
 			   BIT(HWTSTAMP_FILTER_PTP_V2_L2_SYNC) |
 			   BIT(HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ);
 
-	if (pf->hw_features & I40E_HW_PTP_L4_CAPABLE)
+	if (test_bit(I40E_HW_CAP_PTP_L4, pf->hw.caps))
 		info->rx_filters |= BIT(HWTSTAMP_FILTER_PTP_V1_L4_SYNC) |
 				    BIT(HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ) |
 				    BIT(HWTSTAMP_FILTER_PTP_V2_EVENT) |
@@ -2819,10 +2828,10 @@ static int i40e_set_phys_id(struct net_device *netdev,
 
 	switch (state) {
 	case ETHTOOL_ID_ACTIVE:
-		if (!(pf->hw_features & I40E_HW_PHY_CONTROLS_LEDS)) {
+		if (!test_bit(I40E_HW_CAP_PHY_CONTROLS_LEDS, pf->hw.caps)) {
 			pf->led_status = i40e_led_get(hw);
 		} else {
-			if (!(hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE))
+			if (!test_bit(I40E_HW_CAP_AQ_PHY_ACCESS, hw->caps))
 				i40e_aq_set_phy_debug(hw, I40E_PHY_DEBUG_ALL,
 						      NULL);
 			ret = i40e_led_get_phy(hw, &temp_status,
@@ -2831,25 +2840,25 @@ static int i40e_set_phys_id(struct net_device *netdev,
 		}
 		return blink_freq;
 	case ETHTOOL_ID_ON:
-		if (!(pf->hw_features & I40E_HW_PHY_CONTROLS_LEDS))
+		if (!test_bit(I40E_HW_CAP_PHY_CONTROLS_LEDS, pf->hw.caps))
 			i40e_led_set(hw, 0xf, false);
 		else
 			ret = i40e_led_set_phy(hw, true, pf->led_status, 0);
 		break;
 	case ETHTOOL_ID_OFF:
-		if (!(pf->hw_features & I40E_HW_PHY_CONTROLS_LEDS))
+		if (!test_bit(I40E_HW_CAP_PHY_CONTROLS_LEDS, pf->hw.caps))
 			i40e_led_set(hw, 0x0, false);
 		else
 			ret = i40e_led_set_phy(hw, false, pf->led_status, 0);
 		break;
 	case ETHTOOL_ID_INACTIVE:
-		if (!(pf->hw_features & I40E_HW_PHY_CONTROLS_LEDS)) {
+		if (!test_bit(I40E_HW_CAP_PHY_CONTROLS_LEDS, pf->hw.caps)) {
 			i40e_led_set(hw, pf->led_status, false);
 		} else {
 			ret = i40e_led_set_phy(hw, false, pf->led_status,
 					       (pf->phy_led_val |
 					       I40E_PHY_LED_MODE_ORIG));
-			if (!(hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE))
+			if (!test_bit(I40E_HW_CAP_AQ_PHY_ACCESS, hw->caps))
 				i40e_aq_set_phy_debug(hw, 0, NULL);
 		}
 		break;
@@ -3628,7 +3637,7 @@ static int i40e_set_rss_hash_opt(struct i40e_pf *pf, struct ethtool_rxnfc *nfc)
 
 	bitmap_zero(flow_pctypes, FLOW_PCTYPES_SIZE);
 
-	if (pf->flags & I40E_FLAG_MFP_ENABLED) {
+	if (test_bit(I40E_FLAG_MFP_ENA, pf->flags)) {
 		dev_err(&pf->pdev->dev,
 			"Change of RSS hash input set is not supported when MFP mode is enabled\n");
 		return -EOPNOTSUPP;
@@ -3644,19 +3653,22 @@ static int i40e_set_rss_hash_opt(struct i40e_pf *pf, struct ethtool_rxnfc *nfc)
 	switch (nfc->flow_type) {
 	case TCP_V4_FLOW:
 		set_bit(I40E_FILTER_PCTYPE_NONF_IPV4_TCP, flow_pctypes);
-		if (pf->hw_features & I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE)
+		if (test_bit(I40E_HW_CAP_MULTI_TCP_UDP_RSS_PCTYPE,
+			     pf->hw.caps))
 			set_bit(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK,
 				flow_pctypes);
 		break;
 	case TCP_V6_FLOW:
 		set_bit(I40E_FILTER_PCTYPE_NONF_IPV6_TCP, flow_pctypes);
-		if (pf->hw_features & I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE)
+		if (test_bit(I40E_HW_CAP_MULTI_TCP_UDP_RSS_PCTYPE,
+			     pf->hw.caps))
 			set_bit(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK,
 				flow_pctypes);
 		break;
 	case UDP_V4_FLOW:
 		set_bit(I40E_FILTER_PCTYPE_NONF_IPV4_UDP, flow_pctypes);
-		if (pf->hw_features & I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE) {
+		if (test_bit(I40E_HW_CAP_MULTI_TCP_UDP_RSS_PCTYPE,
+			     pf->hw.caps)) {
 			set_bit(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP,
 				flow_pctypes);
 			set_bit(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP,
@@ -3666,7 +3678,8 @@ static int i40e_set_rss_hash_opt(struct i40e_pf *pf, struct ethtool_rxnfc *nfc)
 		break;
 	case UDP_V6_FLOW:
 		set_bit(I40E_FILTER_PCTYPE_NONF_IPV6_UDP, flow_pctypes);
-		if (pf->hw_features & I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE) {
+		if (test_bit(I40E_HW_CAP_MULTI_TCP_UDP_RSS_PCTYPE,
+			     pf->hw.caps)) {
 			set_bit(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP,
 				flow_pctypes);
 			set_bit(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP,
@@ -4644,7 +4657,7 @@ static int i40e_check_fdir_input_set(struct i40e_vsi *vsi,
 	 * main port cannot change them when in MFP mode as this would impact
 	 * any filters on the other ports.
 	 */
-	if (pf->flags & I40E_FLAG_MFP_ENABLED) {
+	if (test_bit(I40E_FLAG_MFP_ENA, pf->flags)) {
 		netif_err(pf, drv, vsi->netdev, "Cannot change Flow Director input sets while MFP is enabled\n");
 		return -EOPNOTSUPP;
 	}
@@ -4804,7 +4817,7 @@ static int i40e_add_fdir_ethtool(struct i40e_vsi *vsi,
 		return -EINVAL;
 	pf = vsi->back;
 
-	if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
+	if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))
 		return -EOPNOTSUPP;
 
 	if (test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
@@ -5001,7 +5014,7 @@ static void i40e_get_channels(struct net_device *dev,
 	ch->max_combined = i40e_max_channels(vsi);
 
 	/* report info for other vector */
-	ch->other_count = (pf->flags & I40E_FLAG_FD_SB_ENABLED) ? 1 : 0;
+	ch->other_count = test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) ? 1 : 0;
 	ch->max_other = ch->other_count;
 
 	/* Note: This code assumes DCB is disabled for now. */
@@ -5044,7 +5057,7 @@ static int i40e_set_channels(struct net_device *dev,
 		return -EINVAL;
 
 	/* verify other_count has not changed */
-	if (ch->other_count != ((pf->flags & I40E_FLAG_FD_SB_ENABLED) ? 1 : 0))
+	if (ch->other_count != (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) ? 1 : 0))
 		return -EINVAL;
 
 	/* verify the number of channels does not exceed hardware limits */
@@ -5215,11 +5228,11 @@ static u32 i40e_get_priv_flags(struct net_device *dev)
 	u32 i, j, ret_flags = 0;
 
 	for (i = 0; i < I40E_PRIV_FLAGS_STR_LEN; i++) {
-		const struct i40e_priv_flags *priv_flags;
+		const struct i40e_priv_flags *priv_flag;
 
-		priv_flags = &i40e_gstrings_priv_flags[i];
+		priv_flag = &i40e_gstrings_priv_flags[i];
 
-		if (priv_flags->flag & pf->flags)
+		if (test_bit(priv_flag->bitno, pf->flags))
 			ret_flags |= BIT(i);
 	}
 
@@ -5227,11 +5240,11 @@ static u32 i40e_get_priv_flags(struct net_device *dev)
 		return ret_flags;
 
 	for (j = 0; j < I40E_GL_PRIV_FLAGS_STR_LEN; j++) {
-		const struct i40e_priv_flags *priv_flags;
+		const struct i40e_priv_flags *priv_flag;
 
-		priv_flags = &i40e_gl_gstrings_priv_flags[j];
+		priv_flag = &i40e_gl_gstrings_priv_flags[j];
 
-		if (priv_flags->flag & pf->flags)
+		if (test_bit(priv_flag->bitno, pf->flags))
 			ret_flags |= BIT(i + j);
 	}
 
@@ -5245,8 +5258,10 @@ static u32 i40e_get_priv_flags(struct net_device *dev)
  **/
 static int i40e_set_priv_flags(struct net_device *dev, u32 flags)
 {
+	DECLARE_BITMAP(changed_flags, I40E_PF_FLAGS_NBITS);
+	DECLARE_BITMAP(orig_flags, I40E_PF_FLAGS_NBITS);
+	DECLARE_BITMAP(new_flags, I40E_PF_FLAGS_NBITS);
 	struct i40e_netdev_priv *np = netdev_priv(dev);
-	u64 orig_flags, new_flags, changed_flags;
 	enum i40e_admin_queue_err adq_err;
 	struct i40e_vsi *vsi = np->vsi;
 	struct i40e_pf *pf = vsi->back;
@@ -5254,51 +5269,57 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags)
 	int status;
 	u32 i, j;
 
-	orig_flags = READ_ONCE(pf->flags);
-	new_flags = orig_flags;
+	bitmap_copy(orig_flags, pf->flags, I40E_PF_FLAGS_NBITS);
+	bitmap_copy(new_flags, pf->flags, I40E_PF_FLAGS_NBITS);
 
 	for (i = 0; i < I40E_PRIV_FLAGS_STR_LEN; i++) {
-		const struct i40e_priv_flags *priv_flags;
-
-		priv_flags = &i40e_gstrings_priv_flags[i];
+		const struct i40e_priv_flags *priv_flag;
+		bool new_val;
 
-		if (flags & BIT(i))
-			new_flags |= priv_flags->flag;
-		else
-			new_flags &= ~(priv_flags->flag);
+		priv_flag = &i40e_gstrings_priv_flags[i];
+		new_val = (flags & BIT(i)) ? true : false;
 
 		/* If this is a read-only flag, it can't be changed */
-		if (priv_flags->read_only &&
-		    ((orig_flags ^ new_flags) & ~BIT(i)))
+		if (priv_flag->read_only &&
+		    test_bit(priv_flag->bitno, orig_flags) != new_val)
 			return -EOPNOTSUPP;
+
+		if (new_val)
+			set_bit(priv_flag->bitno, new_flags);
+		else
+			clear_bit(priv_flag->bitno, new_flags);
 	}
 
 	if (pf->hw.pf_id != 0)
 		goto flags_complete;
 
 	for (j = 0; j < I40E_GL_PRIV_FLAGS_STR_LEN; j++) {
-		const struct i40e_priv_flags *priv_flags;
+		const struct i40e_priv_flags *priv_flag;
+		bool new_val;
 
-		priv_flags = &i40e_gl_gstrings_priv_flags[j];
-
-		if (flags & BIT(i + j))
-			new_flags |= priv_flags->flag;
-		else
-			new_flags &= ~(priv_flags->flag);
+		priv_flag = &i40e_gl_gstrings_priv_flags[j];
+		new_val = (flags & BIT(i + j)) ? true : false;
 
 		/* If this is a read-only flag, it can't be changed */
-		if (priv_flags->read_only &&
-		    ((orig_flags ^ new_flags) & ~BIT(i)))
+		if (priv_flag->read_only &&
+		    test_bit(priv_flag->bitno, orig_flags) != new_val)
 			return -EOPNOTSUPP;
+
+		if (new_val)
+			set_bit(priv_flag->bitno, new_flags);
+		else
+			clear_bit(priv_flag->bitno, new_flags);
 	}
 
 flags_complete:
-	changed_flags = orig_flags ^ new_flags;
+	bitmap_xor(changed_flags, pf->flags, orig_flags, I40E_PF_FLAGS_NBITS);
 
-	if (changed_flags & I40E_FLAG_DISABLE_FW_LLDP)
+	if (test_bit(I40E_FLAG_FW_LLDP_DIS, changed_flags))
 		reset_needed = I40E_PF_RESET_AND_REBUILD_FLAG;
-	if (changed_flags & (I40E_FLAG_VEB_STATS_ENABLED |
-	    I40E_FLAG_LEGACY_RX | I40E_FLAG_SOURCE_PRUNING_DISABLED))
+
+	if (test_bit(I40E_FLAG_VEB_STATS_ENA, changed_flags) ||
+	    test_bit(I40E_FLAG_LEGACY_RX_ENA, changed_flags) ||
+	    test_bit(I40E_FLAG_SOURCE_PRUNING_DIS, changed_flags))
 		reset_needed = BIT(__I40E_PF_RESET_REQUESTED);
 
 	/* Before we finalize any flag changes, we need to perform some
@@ -5306,8 +5327,8 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags)
 	 */
 
 	/* ATR eviction is not supported on all devices */
-	if ((new_flags & I40E_FLAG_HW_ATR_EVICT_ENABLED) &&
-	    !(pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE))
+	if (test_bit(I40E_FLAG_HW_ATR_EVICT_ENA, new_flags) &&
+	    !test_bit(I40E_HW_CAP_ATR_EVICT, pf->hw.caps))
 		return -EOPNOTSUPP;
 
 	/* If the driver detected FW LLDP was disabled on init, this flag could
@@ -5318,15 +5339,14 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags)
 	 * disable LLDP, however we _must_ not allow the user to enable/disable
 	 * LLDP with this flag on unsupported FW versions.
 	 */
-	if (changed_flags & I40E_FLAG_DISABLE_FW_LLDP) {
-		if (!(pf->hw.flags & I40E_HW_FLAG_FW_LLDP_STOPPABLE)) {
-			dev_warn(&pf->pdev->dev,
-				 "Device does not support changing FW LLDP\n");
-			return -EOPNOTSUPP;
-		}
+	if (test_bit(I40E_FLAG_FW_LLDP_DIS, changed_flags) &&
+	    !test_bit(I40E_HW_CAP_FW_LLDP_STOPPABLE, pf->hw.caps)) {
+		dev_warn(&pf->pdev->dev,
+			 "Device does not support changing FW LLDP\n");
+		return -EOPNOTSUPP;
 	}
 
-	if (changed_flags & I40E_FLAG_RS_FEC &&
+	if (test_bit(I40E_FLAG_RS_FEC, changed_flags) &&
 	    pf->hw.device_id != I40E_DEV_ID_25G_SFP28 &&
 	    pf->hw.device_id != I40E_DEV_ID_25G_B) {
 		dev_warn(&pf->pdev->dev,
@@ -5334,7 +5354,7 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags)
 		return -EOPNOTSUPP;
 	}
 
-	if (changed_flags & I40E_FLAG_BASE_R_FEC &&
+	if (test_bit(I40E_FLAG_BASE_R_FEC, changed_flags) &&
 	    pf->hw.device_id != I40E_DEV_ID_25G_SFP28 &&
 	    pf->hw.device_id != I40E_DEV_ID_25G_B &&
 	    pf->hw.device_id != I40E_DEV_ID_KX_X722) {
@@ -5349,17 +5369,17 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags)
 	 */
 
 	/* Flush current ATR settings if ATR was disabled */
-	if ((changed_flags & I40E_FLAG_FD_ATR_ENABLED) &&
-	    !(new_flags & I40E_FLAG_FD_ATR_ENABLED)) {
+	if (test_bit(I40E_FLAG_FD_ATR_ENA, changed_flags) &&
+	    !test_bit(I40E_FLAG_FD_ATR_ENA, new_flags)) {
 		set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
 		set_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
 	}
 
-	if (changed_flags & I40E_FLAG_TRUE_PROMISC_SUPPORT) {
+	if (test_bit(I40E_FLAG_TRUE_PROMISC_ENA, changed_flags)) {
 		u16 sw_flags = 0, valid_flags = 0;
 		int ret;
 
-		if (!(new_flags & I40E_FLAG_TRUE_PROMISC_SUPPORT))
+		if (!test_bit(I40E_FLAG_TRUE_PROMISC_ENA, new_flags))
 			sw_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
 		valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
 		ret = i40e_aq_set_switch_config(&pf->hw, sw_flags, valid_flags,
@@ -5374,17 +5394,17 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags)
 		}
 	}
 
-	if ((changed_flags & I40E_FLAG_RS_FEC) ||
-	    (changed_flags & I40E_FLAG_BASE_R_FEC)) {
+	if (test_bit(I40E_FLAG_RS_FEC, changed_flags) ||
+	    test_bit(I40E_FLAG_BASE_R_FEC, changed_flags)) {
 		u8 fec_cfg = 0;
 
-		if (new_flags & I40E_FLAG_RS_FEC &&
-		    new_flags & I40E_FLAG_BASE_R_FEC) {
+		if (test_bit(I40E_FLAG_RS_FEC, new_flags) &&
+		    test_bit(I40E_FLAG_BASE_R_FEC, new_flags)) {
 			fec_cfg = I40E_AQ_SET_FEC_AUTO;
-		} else if (new_flags & I40E_FLAG_RS_FEC) {
+		} else if (test_bit(I40E_FLAG_RS_FEC, new_flags)) {
 			fec_cfg = (I40E_AQ_SET_FEC_REQUEST_RS |
 				   I40E_AQ_SET_FEC_ABILITY_RS);
-		} else if (new_flags & I40E_FLAG_BASE_R_FEC) {
+		} else if (test_bit(I40E_FLAG_BASE_R_FEC, new_flags)) {
 			fec_cfg = (I40E_AQ_SET_FEC_REQUEST_KR |
 				   I40E_AQ_SET_FEC_ABILITY_KR);
 		}
@@ -5392,35 +5412,35 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags)
 			dev_warn(&pf->pdev->dev, "Cannot change FEC config\n");
 	}
 
-	if ((changed_flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED) &&
-	    (orig_flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED)) {
+	if (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, changed_flags) &&
+	    test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, orig_flags)) {
 		dev_err(&pf->pdev->dev,
 			"Setting link-down-on-close not supported on this port (because total-port-shutdown is enabled)\n");
 		return -EOPNOTSUPP;
 	}
 
-	if ((changed_flags & I40E_FLAG_VF_VLAN_PRUNING) &&
+	if (test_bit(I40E_FLAG_VF_VLAN_PRUNING_ENA, changed_flags) &&
 	    pf->num_alloc_vfs) {
 		dev_warn(&pf->pdev->dev,
 			 "Changing vf-vlan-pruning flag while VF(s) are active is not supported\n");
 		return -EOPNOTSUPP;
 	}
 
-	if ((changed_flags & I40E_FLAG_LEGACY_RX) &&
+	if (test_bit(I40E_FLAG_LEGACY_RX_ENA, changed_flags) &&
 	    I40E_2K_TOO_SMALL_WITH_PADDING) {
 		dev_warn(&pf->pdev->dev,
 			 "2k Rx buffer is too small to fit standard MTU and skb_shared_info\n");
 		return -EOPNOTSUPP;
 	}
 
-	if ((changed_flags & new_flags &
-	     I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED) &&
-	    (new_flags & I40E_FLAG_MFP_ENABLED))
+	if (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, changed_flags) &&
+	    test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, new_flags) &&
+	    test_bit(I40E_FLAG_MFP_ENA, new_flags))
 		dev_warn(&pf->pdev->dev,
 			 "Turning on link-down-on-close flag may affect other partitions\n");
 
-	if (changed_flags & I40E_FLAG_DISABLE_FW_LLDP) {
-		if (new_flags & I40E_FLAG_DISABLE_FW_LLDP) {
+	if (test_bit(I40E_FLAG_FW_LLDP_DIS, changed_flags)) {
+		if (test_bit(I40E_FLAG_FW_LLDP_DIS, new_flags)) {
 #ifdef CONFIG_I40E_DCB
 			i40e_dcb_sw_default_config(pf);
 #endif /* CONFIG_I40E_DCB */
@@ -5461,7 +5481,7 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags)
 	 * initialization or (b) while holding the RTNL lock, we don't need
 	 * anything fancy here.
 	 */
-	pf->flags = new_flags;
+	bitmap_copy(pf->flags, new_flags, I40E_PF_FLAGS_NBITS);
 
 	/* Issue reset to cause things to take effect, as additional bits
 	 * are added we will need to create a mask of bits requiring reset
@@ -5491,7 +5511,7 @@ static int i40e_get_module_info(struct net_device *netdev,
 	int status;
 
 	/* Check if firmware supports reading module EEPROM. */
-	if (!(hw->flags & I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE)) {
+	if (!test_bit(I40E_HW_CAP_AQ_PHY_ACCESS, hw->caps)) {
 		netdev_err(vsi->netdev, "Module EEPROM memory read not supported. Please update the NVM image.\n");
 		return -EINVAL;
 	}
@@ -5571,8 +5591,8 @@ static int i40e_get_module_info(struct net_device *netdev,
 		modinfo->eeprom_len = I40E_MODULE_QSFP_MAX_LEN;
 		break;
 	default:
-		netdev_err(vsi->netdev, "Module type unrecognized\n");
-		return -EINVAL;
+		netdev_dbg(vsi->netdev, "SFP module type unrecognized or no SFP connector used.\n");
+		return -EOPNOTSUPP;
 	}
 	return 0;
 }
diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c
index f7a332e51524..7ded598a68e6 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_main.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
@@ -371,7 +371,7 @@ static void i40e_tx_timeout(struct net_device *netdev, unsigned int txqueue)
 	if (tx_ring) {
 		head = i40e_get_head(tx_ring);
 		/* Read interrupt register */
-		if (pf->flags & I40E_FLAG_MSIX_ENABLED)
+		if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
 			val = rd32(&pf->hw,
 			     I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
 						tx_ring->vsi->base_vector - 1));
@@ -1209,13 +1209,13 @@ static void i40e_update_pf_stats(struct i40e_pf *pf)
 			   pf->stat_offsets_loaded,
 			   &osd->rx_lpi_count, &nsd->rx_lpi_count);
 
-	if (pf->flags & I40E_FLAG_FD_SB_ENABLED &&
+	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&
 	    !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
 		nsd->fd_sb_status = true;
 	else
 		nsd->fd_sb_status = false;
 
-	if (pf->flags & I40E_FLAG_FD_ATR_ENABLED &&
+	if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
 	    !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
 		nsd->fd_atr_status = true;
 	else
@@ -1485,7 +1485,7 @@ static s16 i40e_get_vf_new_vlan(struct i40e_vsi *vsi,
 		return pvid;
 
 	is_any = (trusted ||
-		  !(pf->flags & I40E_FLAG_VF_VLAN_PRUNING));
+		  !test_bit(I40E_FLAG_VF_VLAN_PRUNING_ENA, pf->flags));
 
 	if ((vlan_filters && f->vlan == I40E_VLAN_ANY) ||
 	    (!is_any && !vlan_filters && f->vlan == I40E_VLAN_ANY) ||
@@ -1890,7 +1890,7 @@ static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
 	u8 *lut;
 	int ret;
 
-	if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE))
+	if (!test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))
 		return 0;
 	if (!vsi->rss_size)
 		vsi->rss_size = min_t(int, pf->alloc_rss_size,
@@ -2045,7 +2045,7 @@ static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
 		 */
 		if (vsi->req_queue_pairs > 0)
 			vsi->num_queue_pairs = vsi->req_queue_pairs;
-		else if (pf->flags & I40E_FLAG_MSIX_ENABLED)
+		else if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
 			vsi->num_queue_pairs = pf->num_lan_msix;
 		else
 			vsi->num_queue_pairs = 1;
@@ -2058,7 +2058,7 @@ static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
 	else
 		num_tc_qps = vsi->alloc_queue_pairs;
 
-	if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
+	if (enabled_tc && test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) {
 		/* Find numtc from enabled TC bitmap */
 		for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
 			if (enabled_tc & BIT(i)) /* TC is enabled */
@@ -2077,7 +2077,7 @@ static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
 	vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
 
 	/* Do not allow use more TC queue pairs than MSI-X vectors exist */
-	if (pf->flags & I40E_FLAG_MSIX_ENABLED)
+	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
 		num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix);
 
 	/* Setup queue offset/count for all TCs for given VSI */
@@ -2089,8 +2089,10 @@ static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
 
 			switch (vsi->type) {
 			case I40E_VSI_MAIN:
-				if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED |
-				    I40E_FLAG_FD_ATR_ENABLED)) ||
+				if ((!test_bit(I40E_FLAG_FD_SB_ENA,
+					       pf->flags) &&
+				     !test_bit(I40E_FLAG_FD_ATR_ENA,
+					       pf->flags)) ||
 				    vsi->tc_config.enabled_tc != 1) {
 					qcount = min_t(int, pf->alloc_rss_size,
 						       num_tc_qps);
@@ -2476,7 +2478,7 @@ static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc)
 
 	if (vsi->type == I40E_VSI_MAIN &&
 	    pf->lan_veb != I40E_NO_VEB &&
-	    !(pf->flags & I40E_FLAG_MFP_ENABLED)) {
+	    !test_bit(I40E_FLAG_MFP_ENA, pf->flags)) {
 		/* set defport ON for Main VSI instead of true promisc
 		 * this way we will get all unicast/multicast and VLAN
 		 * promisc behavior but will not get VF or VMDq traffic
@@ -2907,7 +2909,7 @@ static void i40e_sync_filters_subtask(struct i40e_pf *pf)
  */
 static u16 i40e_calculate_vsi_rx_buf_len(struct i40e_vsi *vsi)
 {
-	if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
+	if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags))
 		return SKB_WITH_OVERHEAD(I40E_RXBUFFER_2048);
 
 	return PAGE_SIZE < 8192 ? I40E_RXBUFFER_3072 : I40E_RXBUFFER_2048;
@@ -3462,7 +3464,7 @@ static int i40e_configure_tx_ring(struct i40e_ring *ring)
 		ring->xsk_pool = i40e_xsk_pool(ring);
 
 	/* some ATR related tx ring init */
-	if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
+	if (test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags)) {
 		ring->atr_sample_rate = vsi->back->atr_sample_rate;
 		ring->atr_count = 0;
 	} else {
@@ -3478,9 +3480,11 @@ static int i40e_configure_tx_ring(struct i40e_ring *ring)
 	tx_ctx.new_context = 1;
 	tx_ctx.base = (ring->dma / 128);
 	tx_ctx.qlen = ring->count;
-	tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
-					       I40E_FLAG_FD_ATR_ENABLED));
-	tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
+	if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags) ||
+	    test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags))
+		tx_ctx.fd_ena = 1;
+	if (test_bit(I40E_FLAG_PTP_ENA, vsi->back->flags))
+		tx_ctx.timesync_ena = 1;
 	/* FDIR VSI tx ring can still use RS bit and writebacks */
 	if (vsi->type != I40E_VSI_FDIR)
 		tx_ctx.head_wb_ena = 1;
@@ -3663,7 +3667,7 @@ static int i40e_configure_rx_ring(struct i40e_ring *ring)
 	}
 
 	/* configure Rx buffer alignment */
-	if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) {
+	if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags)) {
 		if (I40E_2K_TOO_SMALL_WITH_PADDING) {
 			dev_info(&vsi->back->pdev->dev,
 				 "2k Rx buffer is too small to fit standard MTU and skb_shared_info\n");
@@ -3761,7 +3765,7 @@ static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
 	u16 qoffset, qcount;
 	int i, n;
 
-	if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
+	if (!test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) {
 		/* Reset the TC information */
 		for (i = 0; i < vsi->num_queue_pairs; i++) {
 			rx_ring = vsi->rx_rings[i];
@@ -3828,7 +3832,7 @@ static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
 	struct i40e_pf *pf = vsi->back;
 	struct hlist_node *node;
 
-	if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
+	if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))
 		return;
 
 	/* Reset FDir counters as we're replaying all existing filters */
@@ -3966,10 +3970,10 @@ static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
 	      I40E_PFINT_ICR0_ENA_VFLR_MASK          |
 	      I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
 
-	if (pf->flags & I40E_FLAG_IWARP_ENABLED)
+	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags))
 		val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
 
-	if (pf->flags & I40E_FLAG_PTP)
+	if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))
 		val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
 
 	wr32(hw, I40E_PFINT_ICR0_ENA, val);
@@ -4205,7 +4209,7 @@ static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
 	}
 
 	/* disable each interrupt */
-	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
+	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
 		for (i = vsi->base_vector;
 		     i < (vsi->num_q_vectors + vsi->base_vector); i++)
 			wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
@@ -4231,7 +4235,7 @@ static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
 	struct i40e_pf *pf = vsi->back;
 	int i;
 
-	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
+	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
 		for (i = 0; i < vsi->num_q_vectors; i++)
 			i40e_irq_dynamic_enable(vsi, i);
 	} else {
@@ -4252,7 +4256,7 @@ static void i40e_free_misc_vector(struct i40e_pf *pf)
 	wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
 	i40e_flush(&pf->hw);
 
-	if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) {
+	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags) && pf->msix_entries) {
 		free_irq(pf->msix_entries[0].vector, pf);
 		clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
 	}
@@ -4287,7 +4291,7 @@ static irqreturn_t i40e_intr(int irq, void *data)
 	    (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
 		pf->sw_int_count++;
 
-	if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
+	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) &&
 	    (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
 		ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
 		dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
@@ -4480,7 +4484,7 @@ static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
 	i += tx_ring->count;
 	tx_ring->next_to_clean = i;
 
-	if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED)
+	if (test_bit(I40E_FLAG_MSIX_ENA, vsi->back->flags))
 		i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
 
 	return budget > 0;
@@ -4593,9 +4597,9 @@ static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
 	struct i40e_pf *pf = vsi->back;
 	int err;
 
-	if (pf->flags & I40E_FLAG_MSIX_ENABLED)
+	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
 		err = i40e_vsi_request_irq_msix(vsi, basename);
-	else if (pf->flags & I40E_FLAG_MSI_ENABLED)
+	else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags))
 		err = request_irq(pf->pdev->irq, i40e_intr, 0,
 				  pf->int_name, pf);
 	else
@@ -4627,7 +4631,7 @@ static void i40e_netpoll(struct net_device *netdev)
 	if (test_bit(__I40E_VSI_DOWN, vsi->state))
 		return;
 
-	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
+	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
 		for (i = 0; i < vsi->num_q_vectors; i++)
 			i40e_msix_clean_rings(0, vsi->q_vectors[i]);
 	} else {
@@ -4967,7 +4971,7 @@ static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
 	u32 val, qp;
 	int i;
 
-	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
+	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
 		if (!vsi->q_vectors)
 			return;
 
@@ -5129,16 +5133,17 @@ static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
 {
 	/* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
-	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
+	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
 		pci_disable_msix(pf->pdev);
 		kfree(pf->msix_entries);
 		pf->msix_entries = NULL;
 		kfree(pf->irq_pile);
 		pf->irq_pile = NULL;
-	} else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
+	} else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags)) {
 		pci_disable_msi(pf->pdev);
 	}
-	pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
+	clear_bit(I40E_FLAG_MSI_ENA, pf->flags);
+	clear_bit(I40E_FLAG_MSIX_ENA, pf->flags);
 }
 
 /**
@@ -5478,11 +5483,11 @@ static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
 		return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc;
 
 	/* If neither MQPRIO nor DCB is enabled, then always use single TC */
-	if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
+	if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags))
 		return 1;
 
 	/* SFP mode will be enabled for all TCs on port */
-	if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
+	if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))
 		return i40e_dcb_get_num_tc(dcbcfg);
 
 	/* MFP mode return count of enabled TCs for this PF */
@@ -5512,11 +5517,11 @@ static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
 	/* If neither MQPRIO nor DCB is enabled for this PF then just return
 	 * default TC
 	 */
-	if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
+	if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags))
 		return I40E_DEFAULT_TRAFFIC_CLASS;
 
 	/* SFP mode we want PF to be enabled for all TCs */
-	if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
+	if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))
 		return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
 
 	/* MFP enabled and iSCSI PF type */
@@ -5605,7 +5610,7 @@ static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
 	/* There is no need to reset BW when mqprio mode is on.  */
 	if (i40e_is_tc_mqprio_enabled(pf))
 		return 0;
-	if (!vsi->mqprio_qopt.qopt.hw && !(pf->flags & I40E_FLAG_DCB_ENABLED)) {
+	if (!vsi->mqprio_qopt.qopt.hw && !test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
 		ret = i40e_set_bw_limit(vsi, vsi->seid, 0);
 		if (ret)
 			dev_info(&pf->pdev->dev,
@@ -5858,7 +5863,7 @@ static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
 		}
 		vsi->reconfig_rss = false;
 	}
-	if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
+	if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) {
 		ctxt.info.valid_sections |=
 				cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
 		ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
@@ -6271,7 +6276,7 @@ static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid,
 	if (ch->type == I40E_VSI_VMDQ2)
 		ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
 
-	if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) {
+	if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {
 		ctxt.info.valid_sections |=
 		     cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
 		ctxt.info.switch_id =
@@ -6576,8 +6581,8 @@ int i40e_create_queue_channel(struct i40e_vsi *vsi,
 	 * VSI to be added switch to VEB mode.
 	 */
 
-	if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
-		pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
+	if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {
+		set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
 
 		if (vsi->type == I40E_VSI_MAIN) {
 			if (i40e_is_tc_mqprio_enabled(pf))
@@ -6988,9 +6993,9 @@ int i40e_hw_dcb_config(struct i40e_pf *pf, struct i40e_dcbx_config *new_cfg)
 	if (need_reconfig) {
 		/* Enable DCB tagging only when more than one TC */
 		if (new_numtc > 1)
-			pf->flags |= I40E_FLAG_DCB_ENABLED;
+			set_bit(I40E_FLAG_DCB_ENA, pf->flags);
 		else
-			pf->flags &= ~I40E_FLAG_DCB_ENABLED;
+			clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
 
 		set_bit(__I40E_PORT_SUSPENDED, pf->state);
 		/* Reconfiguration needed quiesce all VSIs */
@@ -7080,7 +7085,7 @@ int i40e_hw_dcb_config(struct i40e_pf *pf, struct i40e_dcbx_config *new_cfg)
 			set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
 		}
 		/* registers are set, lets apply */
-		if (pf->hw_features & I40E_HW_USE_SET_LLDP_MIB)
+		if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps))
 			ret = i40e_hw_set_dcb_config(pf, new_cfg);
 	}
 
@@ -7101,7 +7106,7 @@ int i40e_dcb_sw_default_config(struct i40e_pf *pf)
 	struct i40e_hw *hw = &pf->hw;
 	int err;
 
-	if (pf->hw_features & I40E_HW_USE_SET_LLDP_MIB) {
+	if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps)) {
 		/* Update the local cached instance with TC0 ETS */
 		memset(&pf->tmp_cfg, 0, sizeof(struct i40e_dcbx_config));
 		pf->tmp_cfg.etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;
@@ -7162,12 +7167,12 @@ static int i40e_init_pf_dcb(struct i40e_pf *pf)
 	/* Do not enable DCB for SW1 and SW2 images even if the FW is capable
 	 * Also do not enable DCBx if FW LLDP agent is disabled
 	 */
-	if (pf->hw_features & I40E_HW_NO_DCB_SUPPORT) {
+	if (test_bit(I40E_HW_CAP_NO_DCB_SUPPORT, pf->hw.caps)) {
 		dev_info(&pf->pdev->dev, "DCB is not supported.\n");
 		err = -EOPNOTSUPP;
 		goto out;
 	}
-	if (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) {
+	if (test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)) {
 		dev_info(&pf->pdev->dev, "FW LLDP is disabled, attempting SW DCB\n");
 		err = i40e_dcb_sw_default_config(pf);
 		if (err) {
@@ -7178,8 +7183,8 @@ static int i40e_init_pf_dcb(struct i40e_pf *pf)
 		pf->dcbx_cap = DCB_CAP_DCBX_HOST |
 			       DCB_CAP_DCBX_VER_IEEE;
 		/* at init capable but disabled */
-		pf->flags |= I40E_FLAG_DCB_CAPABLE;
-		pf->flags &= ~I40E_FLAG_DCB_ENABLED;
+		set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
+		clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
 		goto out;
 	}
 	err = i40e_init_dcb(hw, true);
@@ -7194,20 +7199,20 @@ static int i40e_init_pf_dcb(struct i40e_pf *pf)
 			pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
 				       DCB_CAP_DCBX_VER_IEEE;
 
-			pf->flags |= I40E_FLAG_DCB_CAPABLE;
+			set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
 			/* Enable DCB tagging only when more than one TC
 			 * or explicitly disable if only one TC
 			 */
 			if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
-				pf->flags |= I40E_FLAG_DCB_ENABLED;
+				set_bit(I40E_FLAG_DCB_ENA, pf->flags);
 			else
-				pf->flags &= ~I40E_FLAG_DCB_ENABLED;
+				clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
 			dev_dbg(&pf->pdev->dev,
 				"DCBX offload is supported for this PF.\n");
 		}
 	} else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) {
 		dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n");
-		pf->flags |= I40E_FLAG_DISABLE_FW_LLDP;
+		set_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags);
 	} else {
 		dev_info(&pf->pdev->dev,
 			 "Query for DCB configuration failed, err %pe aq_err %s\n",
@@ -7367,7 +7372,7 @@ static int i40e_up_complete(struct i40e_vsi *vsi)
 	struct i40e_pf *pf = vsi->back;
 	int err;
 
-	if (pf->flags & I40E_FLAG_MSIX_ENABLED)
+	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
 		i40e_vsi_configure_msix(vsi);
 	else
 		i40e_configure_msi_and_legacy(vsi);
@@ -7471,7 +7476,7 @@ static int i40e_force_link_state(struct i40e_pf *pf, bool is_up)
 	 * and its speed values are OK, no need for a flap
 	 * if non_zero_phy_type was set, still need to force up
 	 */
-	if (pf->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED)
+	if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags))
 		non_zero_phy_type = true;
 	else if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0)
 		return 0;
@@ -7487,7 +7492,7 @@ static int i40e_force_link_state(struct i40e_pf *pf, bool is_up)
 		non_zero_phy_type ? (u8)((mask >> 32) & 0xff) : 0;
 	/* Copy the old settings, except of phy_type */
 	config.abilities = abilities.abilities;
-	if (pf->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED) {
+	if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags)) {
 		if (is_up)
 			config.abilities |= I40E_AQ_PHY_ENABLE_LINK;
 		else
@@ -7537,8 +7542,8 @@ int i40e_up(struct i40e_vsi *vsi)
 	int err;
 
 	if (vsi->type == I40E_VSI_MAIN &&
-	    (vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED ||
-	     vsi->back->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED))
+	    (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) ||
+	     test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags)))
 		i40e_force_link_state(vsi->back, true);
 
 	err = i40e_vsi_configure(vsi);
@@ -7566,8 +7571,8 @@ void i40e_down(struct i40e_vsi *vsi)
 	i40e_vsi_disable_irq(vsi);
 	i40e_vsi_stop_rings(vsi);
 	if (vsi->type == I40E_VSI_MAIN &&
-	   (vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED ||
-	    vsi->back->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED))
+	   (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) ||
+	    test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags)))
 		i40e_force_link_state(vsi->back, false);
 	i40e_napi_disable_all(vsi);
 
@@ -7973,7 +7978,7 @@ static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev)
 	struct i40e_fwd_adapter *fwd;
 	int avail_macvlan, ret;
 
-	if ((pf->flags & I40E_FLAG_DCB_ENABLED)) {
+	if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
 		netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n");
 		return ERR_PTR(-EINVAL);
 	}
@@ -8168,23 +8173,23 @@ static int i40e_setup_tc(struct net_device *netdev, void *type_data)
 	hw = mqprio_qopt->qopt.hw;
 	mode = mqprio_qopt->mode;
 	if (!hw) {
-		pf->flags &= ~I40E_FLAG_TC_MQPRIO;
+		clear_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);
 		memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
 		goto config_tc;
 	}
 
 	/* Check if MFP enabled */
-	if (pf->flags & I40E_FLAG_MFP_ENABLED) {
+	if (test_bit(I40E_FLAG_MFP_ENA, pf->flags)) {
 		netdev_info(netdev,
 			    "Configuring TC not supported in MFP mode\n");
 		return ret;
 	}
 	switch (mode) {
 	case TC_MQPRIO_MODE_DCB:
-		pf->flags &= ~I40E_FLAG_TC_MQPRIO;
+		clear_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);
 
 		/* Check if DCB enabled to continue */
-		if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
+		if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
 			netdev_info(netdev,
 				    "DCB is not enabled for adapter\n");
 			return ret;
@@ -8198,20 +8203,20 @@ static int i40e_setup_tc(struct net_device *netdev, void *type_data)
 		}
 		break;
 	case TC_MQPRIO_MODE_CHANNEL:
-		if (pf->flags & I40E_FLAG_DCB_ENABLED) {
+		if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
 			netdev_info(netdev,
 				    "Full offload of TC Mqprio options is not supported when DCB is enabled\n");
 			return ret;
 		}
-		if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
+		if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
 			return ret;
 		ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt);
 		if (ret)
 			return ret;
 		memcpy(&vsi->mqprio_qopt, mqprio_qopt,
 		       sizeof(*mqprio_qopt));
-		pf->flags |= I40E_FLAG_TC_MQPRIO;
-		pf->flags &= ~I40E_FLAG_DCB_ENABLED;
+		set_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);
+		clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
 		break;
 	default:
 		return -EINVAL;
@@ -8795,11 +8800,11 @@ static int i40e_configure_clsflower(struct i40e_vsi *vsi,
 		return -EINVAL;
 	}
 
-	if (vsi->back->flags & I40E_FLAG_FD_SB_ENABLED) {
+	if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags)) {
 		dev_err(&vsi->back->pdev->dev,
 			"Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n");
-		vsi->back->flags &= ~I40E_FLAG_FD_SB_ENABLED;
-		vsi->back->flags |= I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
+		clear_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags);
+		clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, vsi->back->flags);
 	}
 
 	filter = kzalloc(sizeof(*filter), GFP_KERNEL);
@@ -8895,11 +8900,11 @@ static int i40e_delete_clsflower(struct i40e_vsi *vsi,
 
 	pf->num_cloud_filters--;
 	if (!pf->num_cloud_filters)
-		if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
-		    !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
-			pf->flags |= I40E_FLAG_FD_SB_ENABLED;
-			pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
-			pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
+		if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) &&
+		    !test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) {
+			set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
+			clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags);
+			clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
 		}
 	return 0;
 }
@@ -9200,11 +9205,11 @@ static void i40e_cloud_filter_exit(struct i40e_pf *pf)
 	}
 	pf->num_cloud_filters = 0;
 
-	if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
-	    !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
-		pf->flags |= I40E_FLAG_FD_SB_ENABLED;
-		pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
-		pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
+	if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) &&
+	    !test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) {
+		set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
+		clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags);
+		clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
 	}
 }
 
@@ -9292,7 +9297,7 @@ void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
 		i40e_prep_for_reset(pf);
 		i40e_reset_and_rebuild(pf, true, lock_acquired);
 		dev_info(&pf->pdev->dev,
-			 pf->flags & I40E_FLAG_DISABLE_FW_LLDP ?
+			 test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ?
 			 "FW LLDP is disabled\n" :
 			 "FW LLDP is enabled\n");
 
@@ -9407,12 +9412,12 @@ static int i40e_handle_lldp_event(struct i40e_pf *pf,
 	if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
 	    (hw->phy.link_info.link_speed &
 	     ~(I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB)) &&
-	     !(pf->flags & I40E_FLAG_DCB_CAPABLE))
+	     !test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))
 		/* let firmware decide if the DCB should be disabled */
-		pf->flags |= I40E_FLAG_DCB_CAPABLE;
+		set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
 
 	/* Not DCB capable or capability disabled */
-	if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
+	if (!test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))
 		return ret;
 
 	/* Ignore if event is not for Nearest Bridge */
@@ -9448,7 +9453,7 @@ static int i40e_handle_lldp_event(struct i40e_pf *pf,
 		     (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {
 			dev_warn(&pf->pdev->dev,
 				 "DCB is not supported for X710-T*L 2.5/5G speeds\n");
-			pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
+			clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
 		} else {
 			dev_info(&pf->pdev->dev,
 				 "Failed querying DCB configuration data from firmware, err %pe aq_err %s\n",
@@ -9476,9 +9481,9 @@ static int i40e_handle_lldp_event(struct i40e_pf *pf,
 
 	/* Enable DCB tagging only when more than one TC */
 	if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
-		pf->flags |= I40E_FLAG_DCB_ENABLED;
+		set_bit(I40E_FLAG_DCB_ENA, pf->flags);
 	else
-		pf->flags &= ~I40E_FLAG_DCB_ENABLED;
+		clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
 
 	set_bit(__I40E_PORT_SUSPENDED, pf->state);
 	/* Reconfiguration needed quiesce all VSIs */
@@ -9610,7 +9615,7 @@ u32 i40e_get_global_fd_count(struct i40e_pf *pf)
 static void i40e_reenable_fdir_sb(struct i40e_pf *pf)
 {
 	if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
-		if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
+		if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&
 		    (I40E_DEBUG_FD & pf->hw.debug_mask))
 			dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
 }
@@ -9631,7 +9636,7 @@ static void i40e_reenable_fdir_atr(struct i40e_pf *pf)
 					I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
 					I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
 
-		if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
+		if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
 		    (I40E_DEBUG_FD & pf->hw.debug_mask))
 			dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
 	}
@@ -9946,7 +9951,7 @@ static void i40e_link_event(struct i40e_pf *pf)
 	if (pf->vf)
 		i40e_vc_notify_link_state(pf);
 
-	if (pf->flags & I40E_FLAG_PTP)
+	if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))
 		i40e_ptp_set_increment(pf);
 #ifdef CONFIG_I40E_DCB
 	if (new_link == old_link)
@@ -9963,13 +9968,13 @@ static void i40e_link_event(struct i40e_pf *pf)
 		memset(&pf->tmp_cfg, 0, sizeof(pf->tmp_cfg));
 		err = i40e_dcb_sw_default_config(pf);
 		if (err) {
-			pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
-				       I40E_FLAG_DCB_ENABLED);
+			clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
+			clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
 		} else {
 			pf->dcbx_cap = DCB_CAP_DCBX_HOST |
 				       DCB_CAP_DCBX_VER_IEEE;
-			pf->flags |= I40E_FLAG_DCB_CAPABLE;
-			pf->flags &= ~I40E_FLAG_DCB_ENABLED;
+			set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
+			clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
 		}
 	}
 #endif /* CONFIG_I40E_DCB */
@@ -9994,7 +9999,7 @@ static void i40e_watchdog_subtask(struct i40e_pf *pf)
 		return;
 	pf->service_timer_previous = jiffies;
 
-	if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) ||
+	if (test_bit(I40E_FLAG_LINK_POLLING_ENA, pf->flags) ||
 	    test_bit(__I40E_TEMP_LINK_POLLING, pf->state))
 		i40e_link_event(pf);
 
@@ -10005,7 +10010,7 @@ static void i40e_watchdog_subtask(struct i40e_pf *pf)
 		if (pf->vsi[i] && pf->vsi[i]->netdev)
 			i40e_update_stats(pf->vsi[i]);
 
-	if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) {
+	if (test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags)) {
 		/* Update the stats for the active switching components */
 		for (i = 0; i < I40E_MAX_VEB; i++)
 			if (pf->veb[i])
@@ -10094,7 +10099,7 @@ static void i40e_handle_link_event(struct i40e_pf *pf,
 		if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
 		    (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
 		    (!(status->link_info & I40E_AQ_LINK_UP)) &&
-		    (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) {
+		    (!test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags))) {
 			dev_err(&pf->pdev->dev,
 				"Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n");
 			dev_err(&pf->pdev->dev,
@@ -10400,7 +10405,7 @@ static int i40e_reconstitute_veb(struct i40e_veb *veb)
 	if (ret)
 		goto end_reconstitute;
 
-	if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
+	if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags))
 		veb->bridge_mode = BRIDGE_MODE_VEB;
 	else
 		veb->bridge_mode = BRIDGE_MODE_VEPA;
@@ -10545,7 +10550,7 @@ static void i40e_fdir_sb_setup(struct i40e_pf *pf)
 			wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
 	}
 
-	if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
+	if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))
 		return;
 
 	/* find existing VSI and see if it needs configuring */
@@ -10557,8 +10562,8 @@ static void i40e_fdir_sb_setup(struct i40e_pf *pf)
 				     pf->vsi[pf->lan_vsi]->seid, 0);
 		if (!vsi) {
 			dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
-			pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
-			pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
+			clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
+			set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
 			return;
 		}
 	}
@@ -10936,14 +10941,14 @@ static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
 			i40e_aq_set_dcb_parameters(hw, false, NULL);
 			dev_warn(&pf->pdev->dev,
 				 "DCB is not supported for X710-T*L 2.5/5G speeds\n");
-			pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
+			clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
 		} else {
 			i40e_aq_set_dcb_parameters(hw, true, NULL);
 			ret = i40e_init_pf_dcb(pf);
 			if (ret) {
 				dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n",
 					 ret);
-				pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
+				clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
 				/* Continue without DCB enabled */
 			}
 		}
@@ -11064,7 +11069,7 @@ static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
 		wr32(hw, I40E_REG_MSS, val);
 	}
 
-	if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
+	if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) {
 		msleep(75);
 		ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
 		if (ret)
@@ -11074,7 +11079,7 @@ static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
 					     pf->hw.aq.asq_last_status));
 	}
 	/* reinit the misc interrupt */
-	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
+	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
 		ret = i40e_setup_misc_vector(pf);
 		if (ret)
 			goto end_unlock;
@@ -11349,7 +11354,7 @@ static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
 		if (!vsi->num_rx_desc)
 			vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
 						 I40E_REQ_DESCRIPTOR_MULTIPLE);
-		if (pf->flags & I40E_FLAG_MSIX_ENABLED)
+		if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
 			vsi->num_q_vectors = pf->num_lan_msix;
 		else
 			vsi->num_q_vectors = 1;
@@ -11667,7 +11672,7 @@ static int i40e_alloc_rings(struct i40e_vsi *vsi)
 		ring->count = vsi->num_tx_desc;
 		ring->size = 0;
 		ring->dcb_tc = 0;
-		if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
+		if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps))
 			ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
 		ring->itr_setting = pf->tx_itr_default;
 		WRITE_ONCE(vsi->tx_rings[i], ring++);
@@ -11684,7 +11689,7 @@ static int i40e_alloc_rings(struct i40e_vsi *vsi)
 		ring->count = vsi->num_tx_desc;
 		ring->size = 0;
 		ring->dcb_tc = 0;
-		if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
+		if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps))
 			ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
 		set_ring_xdp(ring);
 		ring->itr_setting = pf->tx_itr_default;
@@ -11748,7 +11753,7 @@ static int i40e_init_msix(struct i40e_pf *pf)
 	int v_actual;
 	int iwarp_requested = 0;
 
-	if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
+	if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
 		return -ENODEV;
 
 	/* The number of vectors we'll request will be comprised of:
@@ -11787,7 +11792,7 @@ static int i40e_init_msix(struct i40e_pf *pf)
 	vectors_left -= pf->num_lan_msix;
 
 	/* reserve one vector for sideband flow director */
-	if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
+	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
 		if (vectors_left) {
 			pf->num_fdsb_msix = 1;
 			v_budget++;
@@ -11798,7 +11803,7 @@ static int i40e_init_msix(struct i40e_pf *pf)
 	}
 
 	/* can we reserve enough for iWARP? */
-	if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
+	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
 		iwarp_requested = pf->num_iwarp_msix;
 
 		if (!vectors_left)
@@ -11810,7 +11815,7 @@ static int i40e_init_msix(struct i40e_pf *pf)
 	}
 
 	/* any vectors left over go for VMDq support */
-	if (pf->flags & I40E_FLAG_VMDQ_ENABLED) {
+	if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags)) {
 		if (!vectors_left) {
 			pf->num_vmdq_msix = 0;
 			pf->num_vmdq_qps = 0;
@@ -11867,7 +11872,7 @@ static int i40e_init_msix(struct i40e_pf *pf)
 	v_actual = i40e_reserve_msix_vectors(pf, v_budget);
 
 	if (v_actual < I40E_MIN_MSIX) {
-		pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
+		clear_bit(I40E_FLAG_MSIX_ENA, pf->flags);
 		kfree(pf->msix_entries);
 		pf->msix_entries = NULL;
 		pci_disable_msix(pf->pdev);
@@ -11905,7 +11910,7 @@ static int i40e_init_msix(struct i40e_pf *pf)
 			pf->num_lan_msix = 1;
 			break;
 		case 3:
-			if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
+			if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
 				pf->num_lan_msix = 1;
 				pf->num_iwarp_msix = 1;
 			} else {
@@ -11913,7 +11918,7 @@ static int i40e_init_msix(struct i40e_pf *pf)
 			}
 			break;
 		default:
-			if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
+			if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
 				pf->num_iwarp_msix = min_t(int, (vec / 3),
 						 iwarp_requested);
 				pf->num_vmdq_vsis = min_t(int, (vec / 3),
@@ -11922,7 +11927,7 @@ static int i40e_init_msix(struct i40e_pf *pf)
 				pf->num_vmdq_vsis = min_t(int, (vec / 2),
 						  I40E_DEFAULT_NUM_VMDQ_VSI);
 			}
-			if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
+			if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
 				pf->num_fdsb_msix = 1;
 				vec--;
 			}
@@ -11934,22 +11939,20 @@ static int i40e_init_msix(struct i40e_pf *pf)
 		}
 	}
 
-	if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
-	    (pf->num_fdsb_msix == 0)) {
+	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) && pf->num_fdsb_msix == 0) {
 		dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
-		pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
-		pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
+		clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
+		set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
 	}
-	if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
-	    (pf->num_vmdq_msix == 0)) {
+	if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) && pf->num_vmdq_msix == 0) {
 		dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
-		pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
+		clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
 	}
 
-	if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
-	    (pf->num_iwarp_msix == 0)) {
+	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) &&
+	    pf->num_iwarp_msix == 0) {
 		dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
-		pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
+		clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
 	}
 	i40e_debug(&pf->hw, I40E_DEBUG_INIT,
 		   "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
@@ -12003,7 +12006,7 @@ static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
 	int err, v_idx, num_q_vectors;
 
 	/* if not MSIX, give the one vector only to the LAN VSI */
-	if (pf->flags & I40E_FLAG_MSIX_ENABLED)
+	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
 		num_q_vectors = vsi->num_q_vectors;
 	else if (vsi == pf->vsi[pf->lan_vsi])
 		num_q_vectors = 1;
@@ -12034,38 +12037,39 @@ static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
 	int vectors = 0;
 	ssize_t size;
 
-	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
+	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
 		vectors = i40e_init_msix(pf);
 		if (vectors < 0) {
-			pf->flags &= ~(I40E_FLAG_MSIX_ENABLED	|
-				       I40E_FLAG_IWARP_ENABLED	|
-				       I40E_FLAG_RSS_ENABLED	|
-				       I40E_FLAG_DCB_CAPABLE	|
-				       I40E_FLAG_DCB_ENABLED	|
-				       I40E_FLAG_SRIOV_ENABLED	|
-				       I40E_FLAG_FD_SB_ENABLED	|
-				       I40E_FLAG_FD_ATR_ENABLED	|
-				       I40E_FLAG_VMDQ_ENABLED);
-			pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
+			clear_bit(I40E_FLAG_MSIX_ENA, pf->flags);
+			clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
+			clear_bit(I40E_FLAG_RSS_ENA, pf->flags);
+			clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
+			clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
+			clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags);
+			clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
+			clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);
+			clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
+			set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
 
 			/* rework the queue expectations without MSIX */
 			i40e_determine_queue_usage(pf);
 		}
 	}
 
-	if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
-	    (pf->flags & I40E_FLAG_MSI_ENABLED)) {
+	if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&
+	    test_bit(I40E_FLAG_MSI_ENA, pf->flags)) {
 		dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
 		vectors = pci_enable_msi(pf->pdev);
 		if (vectors < 0) {
 			dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
 				 vectors);
-			pf->flags &= ~I40E_FLAG_MSI_ENABLED;
+			clear_bit(I40E_FLAG_MSI_ENA, pf->flags);
 		}
 		vectors = 1;  /* one MSI or Legacy vector */
 	}
 
-	if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
+	if (!test_bit(I40E_FLAG_MSI_ENA, pf->flags) &&
+	    !test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
 		dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
 
 	/* set up vector assignment tracking */
@@ -12098,7 +12102,8 @@ static int i40e_restore_interrupt_scheme(struct i40e_pf *pf)
 	 * scheme. We need to re-enabled them here in order to attempt to
 	 * re-acquire the MSI or MSI-X vectors
 	 */
-	pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
+	set_bit(I40E_FLAG_MSI_ENA, pf->flags);
+	set_bit(I40E_FLAG_MSIX_ENA, pf->flags);
 
 	err = i40e_init_interrupt_scheme(pf);
 	if (err)
@@ -12120,7 +12125,7 @@ static int i40e_restore_interrupt_scheme(struct i40e_pf *pf)
 	if (err)
 		goto err_unwind;
 
-	if (pf->flags & I40E_FLAG_IWARP_ENABLED)
+	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags))
 		i40e_client_update_msix_info(pf);
 
 	return 0;
@@ -12148,7 +12153,7 @@ static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf)
 {
 	int err;
 
-	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
+	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
 		err = i40e_setup_misc_vector(pf);
 
 		if (err) {
@@ -12158,7 +12163,7 @@ static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf)
 			return err;
 		}
 	} else {
-		u32 flags = pf->flags & I40E_FLAG_MSI_ENABLED ? 0 : IRQF_SHARED;
+		u32 flags = test_bit(I40E_FLAG_MSI_ENA, pf->flags) ? 0 : IRQF_SHARED;
 
 		err = request_irq(pf->pdev->irq, i40e_intr, flags,
 				  pf->int_name, pf);
@@ -12362,7 +12367,7 @@ int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
 {
 	struct i40e_pf *pf = vsi->back;
 
-	if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
+	if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))
 		return i40e_config_rss_aq(vsi, seed, lut, lut_size);
 	else
 		return i40e_config_rss_reg(vsi, seed, lut, lut_size);
@@ -12381,7 +12386,7 @@ int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
 {
 	struct i40e_pf *pf = vsi->back;
 
-	if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
+	if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))
 		return i40e_get_rss_aq(vsi, seed, lut, lut_size);
 	else
 		return i40e_get_rss_reg(vsi, seed, lut, lut_size);
@@ -12484,7 +12489,7 @@ int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
 	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
 	int new_rss_size;
 
-	if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
+	if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags))
 		return 0;
 
 	queue_count = min_t(int, queue_count, num_online_cpus());
@@ -12717,9 +12722,9 @@ static int i40e_sw_init(struct i40e_pf *pf)
 	u16 pow;
 
 	/* Set default capability flags */
-	pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
-		    I40E_FLAG_MSI_ENABLED     |
-		    I40E_FLAG_MSIX_ENABLED;
+	bitmap_zero(pf->flags, I40E_PF_FLAGS_NBITS);
+	set_bit(I40E_FLAG_MSI_ENA, pf->flags);
+	set_bit(I40E_FLAG_MSIX_ENA, pf->flags);
 
 	/* Set default ITR */
 	pf->rx_itr_default = I40E_ITR_RX_DEF;
@@ -12739,14 +12744,14 @@ static int i40e_sw_init(struct i40e_pf *pf)
 	pf->rss_size_max = min_t(int, pf->rss_size_max, pow);
 
 	if (pf->hw.func_caps.rss) {
-		pf->flags |= I40E_FLAG_RSS_ENABLED;
+		set_bit(I40E_FLAG_RSS_ENA, pf->flags);
 		pf->alloc_rss_size = min_t(int, pf->rss_size_max,
 					   num_online_cpus());
 	}
 
 	/* MFP mode enabled */
 	if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
-		pf->flags |= I40E_FLAG_MFP_ENABLED;
+		set_bit(I40E_FLAG_MFP_ENA, pf->flags);
 		dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
 		if (i40e_get_partition_bw_setting(pf)) {
 			dev_warn(&pf->pdev->dev,
@@ -12763,84 +12768,32 @@ static int i40e_sw_init(struct i40e_pf *pf)
 
 	if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
 	    (pf->hw.func_caps.fd_filters_best_effort > 0)) {
-		pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
+		set_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);
 		pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
-		if (pf->flags & I40E_FLAG_MFP_ENABLED &&
+		if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) &&
 		    pf->hw.num_partitions > 1)
 			dev_info(&pf->pdev->dev,
 				 "Flow Director Sideband mode Disabled in MFP mode\n");
 		else
-			pf->flags |= I40E_FLAG_FD_SB_ENABLED;
+			set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
 		pf->fdir_pf_filter_count =
 				 pf->hw.func_caps.fd_filters_guaranteed;
 		pf->hw.fdir_shared_filter_count =
 				 pf->hw.func_caps.fd_filters_best_effort;
 	}
 
-	if (pf->hw.mac.type == I40E_MAC_X722) {
-		pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE |
-				    I40E_HW_128_QP_RSS_CAPABLE |
-				    I40E_HW_ATR_EVICT_CAPABLE |
-				    I40E_HW_WB_ON_ITR_CAPABLE |
-				    I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE |
-				    I40E_HW_NO_PCI_LINK_CHECK |
-				    I40E_HW_USE_SET_LLDP_MIB |
-				    I40E_HW_GENEVE_OFFLOAD_CAPABLE |
-				    I40E_HW_PTP_L4_CAPABLE |
-				    I40E_HW_WOL_MC_MAGIC_PKT_WAKE |
-				    I40E_HW_OUTER_UDP_CSUM_CAPABLE);
-
-#define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03
-		if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) !=
-		    I40E_FDEVICT_PCTYPE_DEFAULT) {
-			dev_warn(&pf->pdev->dev,
-				 "FD EVICT PCTYPES are not right, disable FD HW EVICT\n");
-			pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE;
-		}
-	} else if ((pf->hw.aq.api_maj_ver > 1) ||
-		   ((pf->hw.aq.api_maj_ver == 1) &&
-		    (pf->hw.aq.api_min_ver > 4))) {
-		/* Supported in FW API version higher than 1.4 */
-		pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE;
-	}
-
 	/* Enable HW ATR eviction if possible */
-	if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE)
-		pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED;
-
-	if ((pf->hw.mac.type == I40E_MAC_XL710) &&
-	    (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) ||
-	    (pf->hw.aq.fw_maj_ver < 4))) {
-		pf->hw_features |= I40E_HW_RESTART_AUTONEG;
-		/* No DCB support  for FW < v4.33 */
-		pf->hw_features |= I40E_HW_NO_DCB_SUPPORT;
-	}
-
-	/* Disable FW LLDP if FW < v4.3 */
-	if ((pf->hw.mac.type == I40E_MAC_XL710) &&
-	    (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
-	    (pf->hw.aq.fw_maj_ver < 4)))
-		pf->hw_features |= I40E_HW_STOP_FW_LLDP;
-
-	/* Use the FW Set LLDP MIB API if FW > v4.40 */
-	if ((pf->hw.mac.type == I40E_MAC_XL710) &&
-	    (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) ||
-	    (pf->hw.aq.fw_maj_ver >= 5)))
-		pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB;
-
-	/* Enable PTP L4 if FW > v6.0 */
-	if (pf->hw.mac.type == I40E_MAC_XL710 &&
-	    pf->hw.aq.fw_maj_ver >= 6)
-		pf->hw_features |= I40E_HW_PTP_L4_CAPABLE;
+	if (test_bit(I40E_HW_CAP_ATR_EVICT, pf->hw.caps))
+		set_bit(I40E_FLAG_HW_ATR_EVICT_ENA, pf->flags);
 
 	if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) {
 		pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
-		pf->flags |= I40E_FLAG_VMDQ_ENABLED;
+		set_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
 		pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
 	}
 
 	if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) {
-		pf->flags |= I40E_FLAG_IWARP_ENABLED;
+		set_bit(I40E_FLAG_IWARP_ENA, pf->flags);
 		/* IWARP needs one extra vector for CQP just like MISC.*/
 		pf->num_iwarp_msix = (int)num_online_cpus() + 1;
 	}
@@ -12850,14 +12803,13 @@ static int i40e_sw_init(struct i40e_pf *pf)
 	 * if NPAR is functioning so unset this hw flag in this case.
 	 */
 	if (pf->hw.mac.type == I40E_MAC_XL710 &&
-	    pf->hw.func_caps.npar_enable &&
-	    (pf->hw.flags & I40E_HW_FLAG_FW_LLDP_STOPPABLE))
-		pf->hw.flags &= ~I40E_HW_FLAG_FW_LLDP_STOPPABLE;
+	    pf->hw.func_caps.npar_enable)
+		clear_bit(I40E_HW_CAP_FW_LLDP_STOPPABLE, pf->hw.caps);
 
 #ifdef CONFIG_PCI_IOV
 	if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
 		pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
-		pf->flags |= I40E_FLAG_SRIOV_ENABLED;
+		set_bit(I40E_FLAG_SRIOV_ENA, pf->flags);
 		pf->num_req_vfs = min_t(int,
 					pf->hw.func_caps.num_vfs,
 					I40E_MAX_VF_COUNT);
@@ -12868,7 +12820,7 @@ static int i40e_sw_init(struct i40e_pf *pf)
 	pf->lan_vsi = I40E_NO_VSI;
 
 	/* By default FW has this off for performance reasons */
-	pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED;
+	clear_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags);
 
 	/* set up queue assignment tracking */
 	size = sizeof(struct i40e_lump_tracking)
@@ -12887,8 +12839,8 @@ static int i40e_sw_init(struct i40e_pf *pf)
 		/* Link down on close must be on when total port shutdown
 		 * is enabled for a given port
 		 */
-		pf->flags |= (I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED |
-			      I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED);
+		set_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags);
+		set_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags);
 		dev_info(&pf->pdev->dev,
 			 "total-port-shutdown was enabled, link-down-on-close is forced on\n");
 	}
@@ -12914,31 +12866,31 @@ bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
 	 */
 	if (features & NETIF_F_NTUPLE) {
 		/* Enable filters and mark for reset */
-		if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
+		if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))
 			need_reset = true;
 		/* enable FD_SB only if there is MSI-X vector and no cloud
 		 * filters exist
 		 */
 		if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) {
-			pf->flags |= I40E_FLAG_FD_SB_ENABLED;
-			pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
+			set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
+			clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
 		}
 	} else {
 		/* turn off filters, mark for reset and clear SW filter list */
-		if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
+		if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
 			need_reset = true;
 			i40e_fdir_filter_exit(pf);
 		}
-		pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
+		clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
 		clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state);
-		pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
+		set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
 
 		/* reset fd counters */
 		pf->fd_add_err = 0;
 		pf->fd_atr_cnt = 0;
 		/* if ATR was auto disabled it can be re-enabled. */
 		if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
-			if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
+			if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
 			    (I40E_DEBUG_FD & pf->hw.debug_mask))
 				dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
 	}
@@ -13087,7 +13039,7 @@ static int i40e_get_phys_port_id(struct net_device *netdev,
 	struct i40e_pf *pf = np->vsi->back;
 	struct i40e_hw *hw = &pf->hw;
 
-	if (!(pf->hw_features & I40E_HW_PORT_ID_VALID))
+	if (!test_bit(I40E_HW_CAP_PORT_ID_VALID, pf->hw.caps))
 		return -EOPNOTSUPP;
 
 	ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
@@ -13116,7 +13068,7 @@ static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
 	struct i40e_pf *pf = np->vsi->back;
 	int err = 0;
 
-	if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
+	if (!test_bit(I40E_FLAG_SRIOV_ENA, pf->flags))
 		return -EOPNOTSUPP;
 
 	if (vid) {
@@ -13216,9 +13168,9 @@ static int i40e_ndo_bridge_setlink(struct net_device *dev,
 			veb->bridge_mode = mode;
 			/* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
 			if (mode == BRIDGE_MODE_VEB)
-				pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
+				set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
 			else
-				pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
+				clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
 			i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
 			break;
 		}
@@ -13552,7 +13504,7 @@ static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair)
 	struct i40e_hw *hw = &pf->hw;
 
 	/* All rings in a qp belong to the same qvector. */
-	if (pf->flags & I40E_FLAG_MSIX_ENABLED)
+	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
 		i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx);
 	else
 		i40e_irq_dynamic_enable_icr0(pf);
@@ -13577,7 +13529,7 @@ static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair)
 	 *
 	 * All rings in a qp belong to the same qvector.
 	 */
-	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
+	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
 		u32 intpf = vsi->base_vector + rxr->q_vector->v_idx;
 
 		wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0);
@@ -13762,7 +13714,7 @@ static int i40e_config_netdev(struct i40e_vsi *vsi)
 			  NETIF_F_RXCSUM		|
 			  0;
 
-	if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE))
+	if (!test_bit(I40E_HW_CAP_OUTER_UDP_CSUM, pf->hw.caps))
 		netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
 
 	netdev->udp_tunnel_nic_info = &pf->udp_tunnel_nic;
@@ -13798,7 +13750,7 @@ static int i40e_config_netdev(struct i40e_vsi *vsi)
 		      NETIF_F_HW_VLAN_CTAG_TX	|
 		      NETIF_F_HW_VLAN_CTAG_RX;
 
-	if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
+	if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))
 		hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC;
 
 	netdev->hw_features |= hw_features | NETIF_F_LOOPBACK;
@@ -13989,7 +13941,7 @@ static int i40e_add_vsi(struct i40e_vsi *vsi)
 		 * negative logic - if it's set, we need to fiddle with
 		 * the VSI to disable source pruning.
 		 */
-		if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) {
+		if (test_bit(I40E_FLAG_SOURCE_PRUNING_DIS, pf->flags)) {
 			memset(&ctxt, 0, sizeof(ctxt));
 			ctxt.seid = pf->main_vsi_seid;
 			ctxt.pf_num = pf->hw.pf_id;
@@ -14011,7 +13963,7 @@ static int i40e_add_vsi(struct i40e_vsi *vsi)
 		}
 
 		/* MFP mode setup queue map and update VSI */
-		if ((pf->flags & I40E_FLAG_MFP_ENABLED) &&
+		if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) &&
 		    !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
 			memset(&ctxt, 0, sizeof(ctxt));
 			ctxt.seid = pf->main_vsi_seid;
@@ -14059,7 +14011,7 @@ static int i40e_add_vsi(struct i40e_vsi *vsi)
 		ctxt.uplink_seid = vsi->uplink_seid;
 		ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
 		ctxt.flags = I40E_AQ_VSI_TYPE_PF;
-		if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) &&
+		if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags) &&
 		    (i40e_is_vsi_uplink_mode_veb(vsi))) {
 			ctxt.info.valid_sections |=
 			     cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
@@ -14107,7 +14059,7 @@ static int i40e_add_vsi(struct i40e_vsi *vsi)
 				cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
 		}
 
-		if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
+		if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) {
 			ctxt.info.valid_sections |=
 				cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
 			ctxt.info.queueing_opt_flags |=
@@ -14323,7 +14275,7 @@ static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
 	/* In Legacy mode, we do not have to get any other vector since we
 	 * piggyback on the misc/ICR0 for queue interrupts.
 	*/
-	if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
+	if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
 		return ret;
 	if (vsi->num_q_vectors)
 		vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
@@ -14490,9 +14442,9 @@ struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
 			 * already enabled, in which case we can't force VEPA
 			 * mode.
 			 */
-			if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
+			if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {
 				veb->bridge_mode = BRIDGE_MODE_VEPA;
-				pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
+				clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
 			}
 			i40e_config_bridge_mode(veb);
 		}
@@ -14588,12 +14540,16 @@ struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
 		break;
 	}
 
-	if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) &&
-	    (vsi->type == I40E_VSI_VMDQ2)) {
+	if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps) &&
+	    vsi->type == I40E_VSI_VMDQ2) {
 		ret = i40e_vsi_config_rss(vsi);
+		if (ret)
+			goto err_config;
 	}
 	return vsi;
 
+err_config:
+	i40e_vsi_clear_rings(vsi);
 err_rings:
 	i40e_vsi_free_q_vectors(vsi);
 err_msix:
@@ -14831,7 +14787,7 @@ void i40e_veb_release(struct i40e_veb *veb)
 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
 {
 	struct i40e_pf *pf = veb->pf;
-	bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED);
+	bool enable_stats = !!test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags);
 	int ret;
 
 	ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid,
@@ -15131,7 +15087,7 @@ static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acqui
 	*/
 
 	if ((pf->hw.pf_id == 0) &&
-	    !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) {
+	    !test_bit(I40E_FLAG_TRUE_PROMISC_ENA, pf->flags)) {
 		flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
 		pf->last_sw_conf_flags = flags;
 	}
@@ -15198,7 +15154,7 @@ static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acqui
 	/* enable RSS in the HW, even for only one queue, as the stack can use
 	 * the hash
 	 */
-	if ((pf->flags & I40E_FLAG_RSS_ENABLED))
+	if (test_bit(I40E_FLAG_RSS_ENA, pf->flags))
 		i40e_pf_config_rss(pf);
 
 	/* fill in link information and enable LSE reporting */
@@ -15240,42 +15196,42 @@ static void i40e_determine_queue_usage(struct i40e_pf *pf)
 	queues_left = pf->hw.func_caps.num_tx_qp;
 
 	if ((queues_left == 1) ||
-	    !(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
+	    !test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
 		/* one qp for PF, no queues for anything else */
 		queues_left = 0;
 		pf->alloc_rss_size = pf->num_lan_qps = 1;
 
 		/* make sure all the fancies are disabled */
-		pf->flags &= ~(I40E_FLAG_RSS_ENABLED	|
-			       I40E_FLAG_IWARP_ENABLED	|
-			       I40E_FLAG_FD_SB_ENABLED	|
-			       I40E_FLAG_FD_ATR_ENABLED	|
-			       I40E_FLAG_DCB_CAPABLE	|
-			       I40E_FLAG_DCB_ENABLED	|
-			       I40E_FLAG_SRIOV_ENABLED	|
-			       I40E_FLAG_VMDQ_ENABLED);
-		pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
-	} else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED |
-				  I40E_FLAG_FD_SB_ENABLED |
-				  I40E_FLAG_FD_ATR_ENABLED |
-				  I40E_FLAG_DCB_CAPABLE))) {
+		clear_bit(I40E_FLAG_RSS_ENA, pf->flags);
+		clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
+		clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
+		clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);
+		clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
+		clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
+		clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags);
+		clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
+		set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
+	} else if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags) &&
+		   !test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&
+		   !test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
+		   !test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags)) {
 		/* one qp for PF */
 		pf->alloc_rss_size = pf->num_lan_qps = 1;
 		queues_left -= pf->num_lan_qps;
 
-		pf->flags &= ~(I40E_FLAG_RSS_ENABLED	|
-			       I40E_FLAG_IWARP_ENABLED	|
-			       I40E_FLAG_FD_SB_ENABLED	|
-			       I40E_FLAG_FD_ATR_ENABLED	|
-			       I40E_FLAG_DCB_ENABLED	|
-			       I40E_FLAG_VMDQ_ENABLED);
-		pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
+		clear_bit(I40E_FLAG_RSS_ENA, pf->flags);
+		clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
+		clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
+		clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);
+		clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
+		clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
+		set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
 	} else {
 		/* Not enough queues for all TCs */
-		if ((pf->flags & I40E_FLAG_DCB_CAPABLE) &&
-		    (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
-			pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
-					I40E_FLAG_DCB_ENABLED);
+		if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags) &&
+		    queues_left < I40E_MAX_TRAFFIC_CLASS) {
+			clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
+			clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
 			dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
 		}
 
@@ -15288,24 +15244,24 @@ static void i40e_determine_queue_usage(struct i40e_pf *pf)
 		queues_left -= pf->num_lan_qps;
 	}
 
-	if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
+	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
 		if (queues_left > 1) {
 			queues_left -= 1; /* save 1 queue for FD */
 		} else {
-			pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
-			pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
+			clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
+			set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
 			dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
 		}
 	}
 
-	if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
+	if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&
 	    pf->num_vf_qps && pf->num_req_vfs && queues_left) {
 		pf->num_req_vfs = min_t(int, pf->num_req_vfs,
 					(queues_left / pf->num_vf_qps));
 		queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
 	}
 
-	if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
+	if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) &&
 	    pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
 		pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
 					  (queues_left / pf->num_vmdq_qps));
@@ -15316,7 +15272,7 @@ static void i40e_determine_queue_usage(struct i40e_pf *pf)
 	dev_dbg(&pf->pdev->dev,
 		"qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
 		pf->hw.func_caps.num_tx_qp,
-		!!(pf->flags & I40E_FLAG_FD_SB_ENABLED),
+		!!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags),
 		pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
 		pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
 		queues_left);
@@ -15340,7 +15296,8 @@ static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
 	settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
 
 	/* Flow Director is enabled */
-	if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED))
+	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) ||
+	    test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags))
 		settings->enable_fdir = true;
 
 	/* Ethtype and MACVLAN filters enabled for PF */
@@ -15372,21 +15329,21 @@ static void i40e_print_features(struct i40e_pf *pf)
 	i += scnprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",
 		      pf->hw.func_caps.num_vsis,
 		      pf->vsi[pf->lan_vsi]->num_queue_pairs);
-	if (pf->flags & I40E_FLAG_RSS_ENABLED)
+	if (test_bit(I40E_FLAG_RSS_ENA, pf->flags))
 		i += scnprintf(&buf[i], REMAIN(i), " RSS");
-	if (pf->flags & I40E_FLAG_FD_ATR_ENABLED)
+	if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags))
 		i += scnprintf(&buf[i], REMAIN(i), " FD_ATR");
-	if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
+	if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
 		i += scnprintf(&buf[i], REMAIN(i), " FD_SB");
 		i += scnprintf(&buf[i], REMAIN(i), " NTUPLE");
 	}
-	if (pf->flags & I40E_FLAG_DCB_CAPABLE)
+	if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))
 		i += scnprintf(&buf[i], REMAIN(i), " DCB");
 	i += scnprintf(&buf[i], REMAIN(i), " VxLAN");
 	i += scnprintf(&buf[i], REMAIN(i), " Geneve");
-	if (pf->flags & I40E_FLAG_PTP)
+	if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))
 		i += scnprintf(&buf[i], REMAIN(i), " PTP");
-	if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
+	if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags))
 		i += scnprintf(&buf[i], REMAIN(i), " VEB");
 	else
 		i += scnprintf(&buf[i], REMAIN(i), " VEPA");
@@ -15417,22 +15374,26 @@ static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
  * @fec_cfg: FEC option to set in flags
  * @flags: ptr to flags in which we set FEC option
  **/
-void i40e_set_fec_in_flags(u8 fec_cfg, u32 *flags)
+void i40e_set_fec_in_flags(u8 fec_cfg, unsigned long *flags)
 {
-	if (fec_cfg & I40E_AQ_SET_FEC_AUTO)
-		*flags |= I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC;
+	if (fec_cfg & I40E_AQ_SET_FEC_AUTO) {
+		set_bit(I40E_FLAG_RS_FEC, flags);
+		set_bit(I40E_FLAG_BASE_R_FEC, flags);
+	}
 	if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) ||
 	    (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) {
-		*flags |= I40E_FLAG_RS_FEC;
-		*flags &= ~I40E_FLAG_BASE_R_FEC;
+		set_bit(I40E_FLAG_RS_FEC, flags);
+		clear_bit(I40E_FLAG_BASE_R_FEC, flags);
 	}
 	if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) ||
 	    (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) {
-		*flags |= I40E_FLAG_BASE_R_FEC;
-		*flags &= ~I40E_FLAG_RS_FEC;
+		set_bit(I40E_FLAG_BASE_R_FEC, flags);
+		clear_bit(I40E_FLAG_RS_FEC, flags);
+	}
+	if (fec_cfg == 0) {
+		clear_bit(I40E_FLAG_RS_FEC, flags);
+		clear_bit(I40E_FLAG_BASE_R_FEC, flags);
 	}
-	if (fec_cfg == 0)
-		*flags &= ~(I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC);
 }
 
 /**
@@ -15858,15 +15819,15 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		 hw->vendor_id, hw->device_id, hw->subsystem_vendor_id,
 		 hw->subsystem_device_id);
 
-	if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
-	    hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw))
+	if (i40e_is_aq_api_ver_ge(hw, I40E_FW_API_VERSION_MAJOR,
+				  I40E_FW_MINOR_VERSION(hw) + 1))
 		dev_dbg(&pdev->dev,
 			"The driver for the device detected a newer version of the NVM image v%u.%u than v%u.%u.\n",
 			 hw->aq.api_maj_ver,
 			 hw->aq.api_min_ver,
 			 I40E_FW_API_VERSION_MAJOR,
 			 I40E_FW_MINOR_VERSION(hw));
-	else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4)
+	else if (i40e_is_aq_api_ver_lt(hw, 1, 4))
 		dev_info(&pdev->dev,
 			 "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n",
 			 hw->aq.api_maj_ver,
@@ -15913,7 +15874,7 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	 * Ignore error return codes because if it was already disabled via
 	 * hardware settings this will fail
 	 */
-	if (pf->hw_features & I40E_HW_STOP_FW_LLDP) {
+	if (test_bit(I40E_HW_CAP_STOP_FW_LLDP, pf->hw.caps)) {
 		dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
 		i40e_aq_stop_lldp(hw, true, false, NULL);
 	}
@@ -15930,7 +15891,7 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
 	i40e_get_port_mac_addr(hw, hw->mac.port_addr);
 	if (is_valid_ether_addr(hw->mac.port_addr))
-		pf->hw_features |= I40E_HW_PORT_ID_VALID;
+		set_bit(I40E_HW_CAP_PORT_ID_VALID, pf->hw.caps);
 
 	i40e_ptp_alloc_pins(pf);
 	pci_set_drvdata(pdev, pf);
@@ -15940,10 +15901,10 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	status = i40e_get_fw_lldp_status(&pf->hw, &lldp_status);
 	(!status &&
 	 lldp_status == I40E_GET_FW_LLDP_STATUS_ENABLED) ?
-		(pf->flags &= ~I40E_FLAG_DISABLE_FW_LLDP) :
-		(pf->flags |= I40E_FLAG_DISABLE_FW_LLDP);
+		(clear_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)) :
+		(set_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags));
 	dev_info(&pdev->dev,
-		 (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) ?
+		 test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ?
 			"FW LLDP is disabled\n" :
 			"FW LLDP is enabled\n");
 
@@ -15953,7 +15914,8 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	err = i40e_init_pf_dcb(pf);
 	if (err) {
 		dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
-		pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED);
+		clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
+		clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
 		/* Continue without DCB enabled */
 	}
 #endif /* CONFIG_I40E_DCB */
@@ -16021,11 +15983,11 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 #ifdef CONFIG_PCI_IOV
 	/* prep for VF support */
-	if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
-	    (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
+	if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&
+	    test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&
 	    !test_bit(__I40E_BAD_EEPROM, pf->state)) {
 		if (pci_num_vf(pdev))
-			pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
+			set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
 	}
 #endif
 	err = i40e_setup_pf_switch(pf, false, false);
@@ -16066,7 +16028,7 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		wr32(hw, I40E_REG_MSS, val);
 	}
 
-	if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
+	if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) {
 		msleep(75);
 		err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
 		if (err)
@@ -16086,7 +16048,7 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	 * the misc functionality and queue processing is combined in
 	 * the same vector and that gets setup at open.
 	 */
-	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
+	if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
 		err = i40e_setup_misc_vector(pf);
 		if (err) {
 			dev_info(&pdev->dev,
@@ -16099,8 +16061,8 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 #ifdef CONFIG_PCI_IOV
 	/* prep for VF support */
-	if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
-	    (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
+	if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&
+	    test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&
 	    !test_bit(__I40E_BAD_EEPROM, pf->state)) {
 		/* disable link interrupts for VFs */
 		val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
@@ -16120,7 +16082,7 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	}
 #endif /* CONFIG_PCI_IOV */
 
-	if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
+	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
 		pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
 						      pf->num_iwarp_msix,
 						      I40E_IWARP_IRQ_PILE_ID);
@@ -16128,7 +16090,7 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 			dev_info(&pdev->dev,
 				 "failed to get tracking for %d vectors for IWARP err=%d\n",
 				 pf->num_iwarp_msix, pf->iwarp_base_vector);
-			pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
+			clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
 		}
 	}
 
@@ -16142,7 +16104,7 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		  round_jiffies(jiffies + pf->service_timer_period));
 
 	/* add this PF to client device list and launch a client service task */
-	if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
+	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
 		err = i40e_lan_add_device(pf);
 		if (err)
 			dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
@@ -16155,7 +16117,7 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	 * and will report PCI Gen 1 x 1 by default so don't bother
 	 * checking them.
 	 */
-	if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) {
+	if (!test_bit(I40E_HW_CAP_NO_PCI_LINK_CHECK, pf->hw.caps)) {
 		char speed[PCI_SPEED_SIZE] = "Unknown";
 		char width[PCI_WIDTH_SIZE] = "Unknown";
 
@@ -16209,7 +16171,7 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
 
 	/* set the FEC config due to the board capabilities */
-	i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, &pf->flags);
+	i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, pf->flags);
 
 	/* get the supported phy types from the fw */
 	err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
@@ -16236,10 +16198,10 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 						       pf->main_vsi_seid);
 
 	if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
-		(pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
-		pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS;
+	    (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
+		set_bit(I40E_HW_CAP_PHY_CONTROLS_LEDS, pf->hw.caps);
 	if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
-		pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER;
+		set_bit(I40E_HW_CAP_CRT_RETIMER, pf->hw.caps);
 	/* print a string summarizing features */
 	i40e_print_features(pf);
 
@@ -16308,10 +16270,10 @@ static void i40e_remove(struct pci_dev *pdev)
 		usleep_range(1000, 2000);
 	set_bit(__I40E_IN_REMOVE, pf->state);
 
-	if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
+	if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags)) {
 		set_bit(__I40E_VF_RESETS_DISABLED, pf->state);
 		i40e_free_vfs(pf);
-		pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
+		clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags);
 	}
 	/* no more scheduling of any task */
 	set_bit(__I40E_SUSPENDED, pf->state);
@@ -16366,7 +16328,7 @@ static void i40e_remove(struct pci_dev *pdev)
 	i40e_cloud_filter_exit(pf);
 
 	/* remove attached clients */
-	if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
+	if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
 		ret_code = i40e_lan_del_device(pf);
 		if (ret_code)
 			dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
@@ -16385,7 +16347,7 @@ static void i40e_remove(struct pci_dev *pdev)
 unmap:
 	/* Free MSI/legacy interrupt 0 when in recovery mode. */
 	if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
-	    !(pf->flags & I40E_FLAG_MSIX_ENABLED))
+	    !test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
 		free_irq(pf->pdev->irq, pf);
 
 	/* shutdown the adminq */
@@ -16601,7 +16563,8 @@ static void i40e_shutdown(struct pci_dev *pdev)
 	 */
 	i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
 
-	if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
+	if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) &&
+	    pf->wol_en)
 		i40e_enable_mc_magic_wake(pf);
 
 	i40e_prep_for_reset(pf);
@@ -16613,7 +16576,7 @@ static void i40e_shutdown(struct pci_dev *pdev)
 
 	/* Free MSI/legacy interrupt 0 when in recovery mode. */
 	if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
-	    !(pf->flags & I40E_FLAG_MSIX_ENABLED))
+	    !test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
 		free_irq(pf->pdev->irq, pf);
 
 	/* Since we're going to destroy queues during the
@@ -16654,7 +16617,8 @@ static int __maybe_unused i40e_suspend(struct device *dev)
 	 */
 	i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
 
-	if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
+	if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) &&
+	    pf->wol_en)
 		i40e_enable_mc_magic_wake(pf);
 
 	/* Since we're going to destroy queues during the
diff --git a/drivers/net/ethernet/intel/i40e/i40e_nvm.c b/drivers/net/ethernet/intel/i40e/i40e_nvm.c
index 77cdbfc19d47..62eb34871094 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_nvm.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_nvm.c
@@ -291,7 +291,7 @@ static int i40e_read_nvm_word_aq(struct i40e_hw *hw, u16 offset,
 static int __i40e_read_nvm_word(struct i40e_hw *hw,
 				u16 offset, u16 *data)
 {
-	if (hw->flags & I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE)
+	if (test_bit(I40E_HW_CAP_AQ_SRCTL_ACCESS_ENABLE, hw->caps))
 		return i40e_read_nvm_word_aq(hw, offset, data);
 
 	return i40e_read_nvm_word_srctl(hw, offset, data);
@@ -310,14 +310,14 @@ int i40e_read_nvm_word(struct i40e_hw *hw, u16 offset,
 {
 	int ret_code = 0;
 
-	if (hw->flags & I40E_HW_FLAG_NVM_READ_REQUIRES_LOCK)
+	if (test_bit(I40E_HW_CAP_NVM_READ_REQUIRES_LOCK, hw->caps))
 		ret_code = i40e_acquire_nvm(hw, I40E_RESOURCE_READ);
 	if (ret_code)
 		return ret_code;
 
 	ret_code = __i40e_read_nvm_word(hw, offset, data);
 
-	if (hw->flags & I40E_HW_FLAG_NVM_READ_REQUIRES_LOCK)
+	if (test_bit(I40E_HW_CAP_NVM_READ_REQUIRES_LOCK, hw->caps))
 		i40e_release_nvm(hw);
 
 	return ret_code;
@@ -499,7 +499,7 @@ static int __i40e_read_nvm_buffer(struct i40e_hw *hw,
 				  u16 offset, u16 *words,
 				  u16 *data)
 {
-	if (hw->flags & I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE)
+	if (test_bit(I40E_HW_CAP_AQ_SRCTL_ACCESS_ENABLE, hw->caps))
 		return i40e_read_nvm_buffer_aq(hw, offset, words, data);
 
 	return i40e_read_nvm_buffer_srctl(hw, offset, words, data);
@@ -521,7 +521,7 @@ int i40e_read_nvm_buffer(struct i40e_hw *hw, u16 offset,
 {
 	int ret_code = 0;
 
-	if (hw->flags & I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE) {
+	if (test_bit(I40E_HW_CAP_AQ_SRCTL_ACCESS_ENABLE, hw->caps)) {
 		ret_code = i40e_acquire_nvm(hw, I40E_RESOURCE_READ);
 		if (!ret_code) {
 			ret_code = i40e_read_nvm_buffer_aq(hw, offset, words,
diff --git a/drivers/net/ethernet/intel/i40e/i40e_prototype.h b/drivers/net/ethernet/intel/i40e/i40e_prototype.h
index 001162042050..af4269330581 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_prototype.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_prototype.h
@@ -501,4 +501,74 @@ i40e_add_pinfo_to_list(struct i40e_hw *hw,
 /* i40e_ddp */
 int i40e_ddp_flash(struct net_device *netdev, struct ethtool_flash *flash);
 
+/* Firmware and AdminQ version check helpers */
+
+/**
+ * i40e_is_aq_api_ver_ge
+ * @hw: pointer to i40e_hw structure
+ * @maj: API major value to compare
+ * @min: API minor value to compare
+ *
+ * Assert whether current HW API version is greater/equal than provided.
+ **/
+static inline bool i40e_is_aq_api_ver_ge(struct i40e_hw *hw, u16 maj, u16 min)
+{
+	return (hw->aq.api_maj_ver > maj ||
+		(hw->aq.api_maj_ver == maj && hw->aq.api_min_ver >= min));
+}
+
+/**
+ * i40e_is_aq_api_ver_lt
+ * @hw: pointer to i40e_hw structure
+ * @maj: API major value to compare
+ * @min: API minor value to compare
+ *
+ * Assert whether current HW API version is less than provided.
+ **/
+static inline bool i40e_is_aq_api_ver_lt(struct i40e_hw *hw, u16 maj, u16 min)
+{
+	return !i40e_is_aq_api_ver_ge(hw, maj, min);
+}
+
+/**
+ * i40e_is_fw_ver_ge
+ * @hw: pointer to i40e_hw structure
+ * @maj: API major value to compare
+ * @min: API minor value to compare
+ *
+ * Assert whether current firmware version is greater/equal than provided.
+ **/
+static inline bool i40e_is_fw_ver_ge(struct i40e_hw *hw, u16 maj, u16 min)
+{
+	return (hw->aq.fw_maj_ver > maj ||
+		(hw->aq.fw_maj_ver == maj && hw->aq.fw_min_ver >= min));
+}
+
+/**
+ * i40e_is_fw_ver_lt
+ * @hw: pointer to i40e_hw structure
+ * @maj: API major value to compare
+ * @min: API minor value to compare
+ *
+ * Assert whether current firmware version is less than provided.
+ **/
+static inline bool i40e_is_fw_ver_lt(struct i40e_hw *hw, u16 maj, u16 min)
+{
+	return !i40e_is_fw_ver_ge(hw, maj, min);
+}
+
+/**
+ * i40e_is_fw_ver_eq
+ * @hw: pointer to i40e_hw structure
+ * @maj: API major value to compare
+ * @min: API minor value to compare
+ *
+ * Assert whether current firmware version is equal to provided.
+ **/
+static inline bool i40e_is_fw_ver_eq(struct i40e_hw *hw, u16 maj, u16 min)
+{
+	return (hw->aq.fw_maj_ver > maj ||
+		(hw->aq.fw_maj_ver == maj && hw->aq.fw_min_ver == min));
+}
+
 #endif /* _I40E_PROTOTYPE_H_ */
diff --git a/drivers/net/ethernet/intel/i40e/i40e_ptp.c b/drivers/net/ethernet/intel/i40e/i40e_ptp.c
index 20b77398f060..1cf993a79438 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_ptp.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_ptp.c
@@ -35,7 +35,7 @@ enum i40e_ptp_pin {
 	GPIO_4
 };
 
-enum i40e_can_set_pins_t {
+enum i40e_can_set_pins {
 	CANT_DO_PINS = -1,
 	CAN_SET_PINS,
 	CAN_DO_PINS
@@ -193,7 +193,7 @@ static bool i40e_is_ptp_pin_dev(struct i40e_hw *hw)
  * return CAN_DO_PINS if pins can be manipulated within a NIC or
  * return CANT_DO_PINS otherwise.
  **/
-static enum i40e_can_set_pins_t i40e_can_set_pins(struct i40e_pf *pf)
+static enum i40e_can_set_pins i40e_can_set_pins(struct i40e_pf *pf)
 {
 	if (!i40e_is_ptp_pin_dev(&pf->hw)) {
 		dev_warn(&pf->pdev->dev,
@@ -680,7 +680,7 @@ void i40e_ptp_rx_hang(struct i40e_pf *pf)
 	 * configured. We don't want to spuriously warn about Rx timestamp
 	 * hangs if we don't care about the timestamps.
 	 */
-	if (!(pf->flags & I40E_FLAG_PTP) || !pf->ptp_rx)
+	if (!test_bit(I40E_FLAG_PTP_ENA, pf->flags) || !pf->ptp_rx)
 		return;
 
 	spin_lock_bh(&pf->ptp_rx_lock);
@@ -733,7 +733,7 @@ void i40e_ptp_tx_hang(struct i40e_pf *pf)
 {
 	struct sk_buff *skb;
 
-	if (!(pf->flags & I40E_FLAG_PTP) || !pf->ptp_tx)
+	if (!test_bit(I40E_FLAG_PTP_ENA, pf->flags) || !pf->ptp_tx)
 		return;
 
 	/* Nothing to do if we're not already waiting for a timestamp */
@@ -771,7 +771,7 @@ void i40e_ptp_tx_hwtstamp(struct i40e_pf *pf)
 	u32 hi, lo;
 	u64 ns;
 
-	if (!(pf->flags & I40E_FLAG_PTP) || !pf->ptp_tx)
+	if (!test_bit(I40E_FLAG_PTP_ENA, pf->flags) || !pf->ptp_tx)
 		return;
 
 	/* don't attempt to timestamp if we don't have an skb */
@@ -818,7 +818,7 @@ void i40e_ptp_rx_hwtstamp(struct i40e_pf *pf, struct sk_buff *skb, u8 index)
 	/* Since we cannot turn off the Rx timestamp logic if the device is
 	 * doing Tx timestamping, check if Rx timestamping is configured.
 	 */
-	if (!(pf->flags & I40E_FLAG_PTP) || !pf->ptp_rx)
+	if (!test_bit(I40E_FLAG_PTP_ENA, pf->flags) || !pf->ptp_rx)
 		return;
 
 	hw = &pf->hw;
@@ -924,7 +924,7 @@ int i40e_ptp_get_ts_config(struct i40e_pf *pf, struct ifreq *ifr)
 {
 	struct hwtstamp_config *config = &pf->tstamp_config;
 
-	if (!(pf->flags & I40E_FLAG_PTP))
+	if (!test_bit(I40E_FLAG_PTP_ENA, pf->flags))
 		return -EOPNOTSUPP;
 
 	return copy_to_user(ifr->ifr_data, config, sizeof(*config)) ?
@@ -1071,7 +1071,7 @@ static void i40e_ptp_set_pins_hw(struct i40e_pf *pf)
 static int i40e_ptp_set_pins(struct i40e_pf *pf,
 			     struct i40e_ptp_pins_settings *pins)
 {
-	enum i40e_can_set_pins_t pin_caps = i40e_can_set_pins(pf);
+	enum i40e_can_set_pins pin_caps = i40e_can_set_pins(pf);
 	int i = 0;
 
 	if (pin_caps == CANT_DO_PINS)
@@ -1211,7 +1211,7 @@ static int i40e_ptp_set_timestamp_mode(struct i40e_pf *pf,
 	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
-		if (!(pf->hw_features & I40E_HW_PTP_L4_CAPABLE))
+		if (!test_bit(I40E_HW_CAP_PTP_L4, pf->hw.caps))
 			return -ERANGE;
 		pf->ptp_rx = true;
 		tsyntype = I40E_PRTTSYN_CTL1_V1MESSTYPE0_MASK |
@@ -1225,7 +1225,7 @@ static int i40e_ptp_set_timestamp_mode(struct i40e_pf *pf,
 	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
-		if (!(pf->hw_features & I40E_HW_PTP_L4_CAPABLE))
+		if (!test_bit(I40E_HW_CAP_PTP_L4, pf->hw.caps))
 			return -ERANGE;
 		fallthrough;
 	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
@@ -1234,7 +1234,7 @@ static int i40e_ptp_set_timestamp_mode(struct i40e_pf *pf,
 		pf->ptp_rx = true;
 		tsyntype = I40E_PRTTSYN_CTL1_V2MESSTYPE0_MASK |
 			   I40E_PRTTSYN_CTL1_TSYNTYPE_V2;
-		if (pf->hw_features & I40E_HW_PTP_L4_CAPABLE) {
+		if (test_bit(I40E_HW_CAP_PTP_L4, pf->hw.caps)) {
 			tsyntype |= I40E_PRTTSYN_CTL1_UDP_ENA_MASK;
 			config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
 		} else {
@@ -1308,7 +1308,7 @@ int i40e_ptp_set_ts_config(struct i40e_pf *pf, struct ifreq *ifr)
 	struct hwtstamp_config config;
 	int err;
 
-	if (!(pf->flags & I40E_FLAG_PTP))
+	if (!test_bit(I40E_FLAG_PTP_ENA, pf->flags))
 		return -EOPNOTSUPP;
 
 	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
@@ -1426,7 +1426,7 @@ static long i40e_ptp_create_clock(struct i40e_pf *pf)
 void i40e_ptp_save_hw_time(struct i40e_pf *pf)
 {
 	/* don't try to access the PTP clock if it's not enabled */
-	if (!(pf->flags & I40E_FLAG_PTP))
+	if (!test_bit(I40E_FLAG_PTP_ENA, pf->flags))
 		return;
 
 	i40e_ptp_gettimex(&pf->ptp_caps, &pf->ptp_prev_hw_time, NULL);
@@ -1483,7 +1483,7 @@ void i40e_ptp_init(struct i40e_pf *pf)
 	pf_id = (rd32(hw, I40E_PRTTSYN_CTL0) & I40E_PRTTSYN_CTL0_PF_ID_MASK) >>
 		I40E_PRTTSYN_CTL0_PF_ID_SHIFT;
 	if (hw->pf_id != pf_id) {
-		pf->flags &= ~I40E_FLAG_PTP;
+		clear_bit(I40E_FLAG_PTP_ENA, pf->flags);
 		dev_info(&pf->pdev->dev, "%s: PTP not supported on %s\n",
 			 __func__,
 			 netdev->name);
@@ -1504,7 +1504,7 @@ void i40e_ptp_init(struct i40e_pf *pf)
 
 		if (pf->hw.debug_mask & I40E_DEBUG_LAN)
 			dev_info(&pf->pdev->dev, "PHC enabled\n");
-		pf->flags |= I40E_FLAG_PTP;
+		set_bit(I40E_FLAG_PTP_ENA, pf->flags);
 
 		/* Ensure the clocks are running. */
 		regval = rd32(hw, I40E_PRTTSYN_CTL0);
@@ -1539,7 +1539,7 @@ void i40e_ptp_stop(struct i40e_pf *pf)
 	struct i40e_hw *hw = &pf->hw;
 	u32 regval;
 
-	pf->flags &= ~I40E_FLAG_PTP;
+	clear_bit(I40E_FLAG_PTP_ENA, pf->flags);
 	pf->ptp_tx = false;
 	pf->ptp_rx = false;
 
diff --git a/drivers/net/ethernet/intel/i40e/i40e_register.h b/drivers/net/ethernet/intel/i40e/i40e_register.h
index f408fcf23ce8..d561687303ea 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_register.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_register.h
@@ -899,6 +899,7 @@
 #define I40E_GLQF_ORT_FLX_PAYLOAD_SHIFT 7
 #define I40E_GLQF_ORT_FLX_PAYLOAD_MASK I40E_MASK(0x1, I40E_GLQF_ORT_FLX_PAYLOAD_SHIFT)
 #define I40E_GLQF_FDEVICTENA(_i) (0x00270384 + ((_i) * 4)) /* _i=0...1 */ /* Reset: CORER */
+#define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03
 /* Redefined for X722 family */
 #define I40E_GLGEN_STAT_CLEAR 0x00390004 /* Reset: CORER */
 #endif /* _I40E_REGISTER_H_ */
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
index dd410b15000f..b82df5bdfac0 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
@@ -464,7 +464,7 @@ static int i40e_add_del_fdir_tcp(struct i40e_vsi *vsi,
 			       &pf->fd_tcp6_filter_cnt);
 
 	if (add) {
-		if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
+		if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
 		    I40E_DEBUG_FD & pf->hw.debug_mask)
 			dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 flow being applied\n");
 		set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
@@ -734,7 +734,7 @@ static void i40e_fd_handle_status(struct i40e_ring *rx_ring, u64 qword0_raw,
 		 * FD ATR/SB and then re-enable it when there is room.
 		 */
 		if (fcnt_prog >= (fcnt_avail - I40E_FDIR_BUFFER_FULL_MARGIN)) {
-			if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
+			if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&
 			    !test_and_set_bit(__I40E_FD_SB_AUTO_DISABLED,
 					      pf->state))
 				if (I40E_DEBUG_FD & pf->hw.debug_mask)
@@ -1071,7 +1071,7 @@ static void i40e_enable_wb_on_itr(struct i40e_vsi *vsi,
 	if (q_vector->arm_wb_state)
 		return;
 
-	if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) {
+	if (test_bit(I40E_FLAG_MSIX_ENA, vsi->back->flags)) {
 		val = I40E_PFINT_DYN_CTLN_WB_ON_ITR_MASK |
 		      I40E_PFINT_DYN_CTLN_ITR_INDX_MASK; /* set noitr */
 
@@ -1095,7 +1095,7 @@ static void i40e_enable_wb_on_itr(struct i40e_vsi *vsi,
  **/
 void i40e_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector)
 {
-	if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) {
+	if (test_bit(I40E_FLAG_MSIX_ENA, vsi->back->flags)) {
 		u32 val = I40E_PFINT_DYN_CTLN_INTENA_MASK |
 			  I40E_PFINT_DYN_CTLN_ITR_INDX_MASK | /* set noitr */
 			  I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK |
@@ -2699,7 +2699,7 @@ static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
 	u32 intval;
 
 	/* If we don't have MSIX, then we only need to re-enable icr0 */
-	if (!(vsi->back->flags & I40E_FLAG_MSIX_ENABLED)) {
+	if (!test_bit(I40E_FLAG_MSIX_ENA, vsi->back->flags)) {
 		i40e_irq_dynamic_enable_icr0(vsi->back);
 		return;
 	}
@@ -2888,7 +2888,7 @@ static void i40e_atr(struct i40e_ring *tx_ring, struct sk_buff *skb,
 	u16 i;
 
 	/* make sure ATR is enabled */
-	if (!(pf->flags & I40E_FLAG_FD_ATR_ENABLED))
+	if (!test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags))
 		return;
 
 	if (test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
@@ -2933,7 +2933,7 @@ static void i40e_atr(struct i40e_ring *tx_ring, struct sk_buff *skb,
 	/* Due to lack of space, no more new filters can be programmed */
 	if (th->syn && test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
 		return;
-	if (pf->flags & I40E_FLAG_HW_ATR_EVICT_ENABLED) {
+	if (test_bit(I40E_FLAG_HW_ATR_EVICT_ENA, pf->flags)) {
 		/* HW ATR eviction will take care of removing filters on FIN
 		 * and RST packets.
 		 */
@@ -2995,7 +2995,7 @@ static void i40e_atr(struct i40e_ring *tx_ring, struct sk_buff *skb,
 			I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
 			I40E_TXD_FLTR_QW1_CNTINDEX_MASK;
 
-	if (pf->flags & I40E_FLAG_HW_ATR_EVICT_ENABLED)
+	if (test_bit(I40E_FLAG_HW_ATR_EVICT_ENA, pf->flags))
 		dtype_cmd |= I40E_TXD_FLTR_QW1_ATR_MASK;
 
 	fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype);
@@ -3053,7 +3053,7 @@ static inline int i40e_tx_prepare_vlan_flags(struct sk_buff *skb,
 		tx_flags |= I40E_TX_FLAGS_SW_VLAN;
 	}
 
-	if (!(tx_ring->vsi->back->flags & I40E_FLAG_DCB_ENABLED))
+	if (!test_bit(I40E_FLAG_DCB_ENA, tx_ring->vsi->back->flags))
 		goto out;
 
 	/* Insert 802.1p priority into VLAN header */
@@ -3229,7 +3229,7 @@ static int i40e_tsyn(struct i40e_ring *tx_ring, struct sk_buff *skb,
 	 * we are not already transmitting a packet to be timestamped
 	 */
 	pf = i40e_netdev_to_pf(tx_ring->netdev);
-	if (!(pf->flags & I40E_FLAG_PTP))
+	if (!test_bit(I40E_FLAG_PTP_ENA, pf->flags))
 		return 0;
 
 	if (pf->ptp_tx &&
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.h b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
index 421fe5675584..abf15067eb5d 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
@@ -58,7 +58,7 @@ static inline u16 i40e_intrl_usec_to_reg(int intrl)
  * mentioning ITR_INDX, ITR_NONE cannot be used as an index 'n' into any
  * register but instead is a special value meaning "don't update" ITR0/1/2.
  */
-enum i40e_dyn_idx_t {
+enum i40e_dyn_idx {
 	I40E_IDX_ITR0 = 0,
 	I40E_IDX_ITR1 = 1,
 	I40E_IDX_ITR2 = 2,
@@ -92,8 +92,8 @@ enum i40e_dyn_idx_t {
 	BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP))
 
 #define i40e_pf_get_default_rss_hena(pf) \
-	(((pf)->hw_features & I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE) ? \
-	  I40E_DEFAULT_RSS_HENA_EXPANDED : I40E_DEFAULT_RSS_HENA)
+	(test_bit(I40E_HW_CAP_MULTI_TCP_UDP_RSS_PCTYPE, (pf)->hw.caps) ? \
+	 I40E_DEFAULT_RSS_HENA_EXPANDED : I40E_DEFAULT_RSS_HENA)
 
 /* Supported Rx Buffer Sizes (a multiple of 128) */
 #define I40E_RXBUFFER_256   256
@@ -306,7 +306,7 @@ struct i40e_rx_queue_stats {
 	u64 page_busy_count;
 };
 
-enum i40e_ring_state_t {
+enum i40e_ring_state {
 	__I40E_TX_FDIR_INIT_DONE,
 	__I40E_TX_XPS_INIT_DONE,
 	__I40E_RING_STATE_NBITS /* must be last */
diff --git a/drivers/net/ethernet/intel/i40e/i40e_type.h b/drivers/net/ethernet/intel/i40e/i40e_type.h
index aff6dc6afbe2..de69c2e22448 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_type.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_type.h
@@ -64,9 +64,7 @@ typedef void (*I40E_ADMINQ_CALLBACK)(struct i40e_hw *, struct i40e_aq_desc *);
 enum i40e_mac_type {
 	I40E_MAC_UNKNOWN = 0,
 	I40E_MAC_XL710,
-	I40E_MAC_VF,
 	I40E_MAC_X722,
-	I40E_MAC_X722_VF,
 	I40E_MAC_GENERIC,
 };
 
@@ -272,9 +270,7 @@ struct i40e_mac_info {
 	enum i40e_mac_type type;
 	u8 addr[ETH_ALEN];
 	u8 perm_addr[ETH_ALEN];
-	u8 san_addr[ETH_ALEN];
 	u8 port_addr[ETH_ALEN];
-	u16 max_fcoeq;
 };
 
 enum i40e_aq_resources_ids {
@@ -482,6 +478,36 @@ struct i40e_dcbx_config {
 	struct i40e_dcb_app_priority_table app[I40E_DCBX_MAX_APPS];
 };
 
+enum i40e_hw_flags {
+	I40E_HW_CAP_AQ_SRCTL_ACCESS_ENABLE,
+	I40E_HW_CAP_802_1AD,
+	I40E_HW_CAP_AQ_PHY_ACCESS,
+	I40E_HW_CAP_NVM_READ_REQUIRES_LOCK,
+	I40E_HW_CAP_FW_LLDP_STOPPABLE,
+	I40E_HW_CAP_FW_LLDP_PERSISTENT,
+	I40E_HW_CAP_AQ_PHY_ACCESS_EXTENDED,
+	I40E_HW_CAP_X722_FEC_REQUEST,
+	I40E_HW_CAP_RSS_AQ,
+	I40E_HW_CAP_128_QP_RSS,
+	I40E_HW_CAP_ATR_EVICT,
+	I40E_HW_CAP_WB_ON_ITR,
+	I40E_HW_CAP_MULTI_TCP_UDP_RSS_PCTYPE,
+	I40E_HW_CAP_NO_PCI_LINK_CHECK,
+	I40E_HW_CAP_100M_SGMII,
+	I40E_HW_CAP_NO_DCB_SUPPORT,
+	I40E_HW_CAP_USE_SET_LLDP_MIB,
+	I40E_HW_CAP_GENEVE_OFFLOAD,
+	I40E_HW_CAP_PTP_L4,
+	I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE,
+	I40E_HW_CAP_CRT_RETIMER,
+	I40E_HW_CAP_OUTER_UDP_CSUM,
+	I40E_HW_CAP_PHY_CONTROLS_LEDS,
+	I40E_HW_CAP_STOP_FW_LLDP,
+	I40E_HW_CAP_PORT_ID_VALID,
+	I40E_HW_CAP_RESTART_AUTONEG,
+	I40E_HW_CAPS_NBITS,
+};
+
 /* Port hardware description */
 struct i40e_hw {
 	u8 __iomem *hw_addr;
@@ -546,16 +572,7 @@ struct i40e_hw {
 	struct i40e_dcbx_config remote_dcbx_config; /* Peer Cfg */
 	struct i40e_dcbx_config desired_dcbx_config; /* CEE Desired Cfg */
 
-#define I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE BIT_ULL(0)
-#define I40E_HW_FLAG_802_1AD_CAPABLE        BIT_ULL(1)
-#define I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE  BIT_ULL(2)
-#define I40E_HW_FLAG_NVM_READ_REQUIRES_LOCK BIT_ULL(3)
-#define I40E_HW_FLAG_FW_LLDP_STOPPABLE      BIT_ULL(4)
-#define I40E_HW_FLAG_FW_LLDP_PERSISTENT     BIT_ULL(5)
-#define I40E_HW_FLAG_AQ_PHY_ACCESS_EXTENDED BIT_ULL(6)
-#define I40E_HW_FLAG_DROP_MODE              BIT_ULL(7)
-#define I40E_HW_FLAG_X722_FEC_REQUEST_CAPABLE BIT_ULL(8)
-	u64 flags;
+	DECLARE_BITMAP(caps, I40E_HW_CAPS_NBITS);
 
 	/* Used in set switch config AQ command */
 	u16 switch_tag;
@@ -567,12 +584,6 @@ struct i40e_hw {
 	char err_str[16];
 };
 
-static inline bool i40e_is_vf(struct i40e_hw *hw)
-{
-	return (hw->mac.type == I40E_MAC_VF ||
-		hw->mac.type == I40E_MAC_X722_VF);
-}
-
 struct i40e_driver_version {
 	u8 major_version;
 	u8 minor_version;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
index 08d7edccfb8d..af47ba51e28a 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
@@ -1808,7 +1808,7 @@ int i40e_alloc_vfs(struct i40e_pf *pf, u16 num_alloc_vfs)
 	if (pci_num_vf(pf->pdev) != num_alloc_vfs) {
 		ret = pci_enable_sriov(pf->pdev, num_alloc_vfs);
 		if (ret) {
-			pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
+			clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
 			pf->num_alloc_vfs = 0;
 			goto err_iov;
 		}
@@ -1919,8 +1919,8 @@ int i40e_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
 	}
 
 	if (num_vfs) {
-		if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
-			pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
+		if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {
+			set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
 			i40e_do_reset_safe(pf, I40E_PF_RESET_AND_REBUILD_FLAG);
 		}
 		ret = i40e_pci_sriov_enable(pdev, num_vfs);
@@ -1929,7 +1929,7 @@ int i40e_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
 
 	if (!pci_vfs_assigned(pf->pdev)) {
 		i40e_free_vfs(pf);
-		pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
+		clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
 		i40e_do_reset_safe(pf, I40E_PF_RESET_AND_REBUILD_FLAG);
 	} else {
 		dev_warn(&pdev->dev, "Unable to free VFs because some are assigned to VMs.\n");
@@ -2137,14 +2137,14 @@ static int i40e_vc_get_vf_resources_msg(struct i40e_vf *vf, u8 *msg)
 	if (vf->driver_caps & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
 		vfres->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_RSS_PF;
 	} else {
-		if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) &&
+		if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps) &&
 		    (vf->driver_caps & VIRTCHNL_VF_OFFLOAD_RSS_AQ))
 			vfres->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_RSS_AQ;
 		else
 			vfres->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_RSS_REG;
 	}
 
-	if (pf->hw_features & I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE) {
+	if (test_bit(I40E_HW_CAP_MULTI_TCP_UDP_RSS_PCTYPE, pf->hw.caps)) {
 		if (vf->driver_caps & VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2)
 			vfres->vf_cap_flags |=
 				VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2;
@@ -2153,12 +2153,12 @@ static int i40e_vc_get_vf_resources_msg(struct i40e_vf *vf, u8 *msg)
 	if (vf->driver_caps & VIRTCHNL_VF_OFFLOAD_ENCAP)
 		vfres->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_ENCAP;
 
-	if ((pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE) &&
+	if (test_bit(I40E_HW_CAP_OUTER_UDP_CSUM, pf->hw.caps) &&
 	    (vf->driver_caps & VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM))
 		vfres->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM;
 
 	if (vf->driver_caps & VIRTCHNL_VF_OFFLOAD_RX_POLLING) {
-		if (pf->flags & I40E_FLAG_MFP_ENABLED) {
+		if (test_bit(I40E_FLAG_MFP_ENA, pf->flags)) {
 			dev_err(&pf->pdev->dev,
 				"VF %d requested polling mode: this feature is supported only when the device is running in single function per port (SFP) mode\n",
 				 vf->vf_id);
@@ -2168,7 +2168,7 @@ static int i40e_vc_get_vf_resources_msg(struct i40e_vf *vf, u8 *msg)
 		vfres->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_RX_POLLING;
 	}
 
-	if (pf->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) {
+	if (test_bit(I40E_HW_CAP_WB_ON_ITR, pf->hw.caps)) {
 		if (vf->driver_caps & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR)
 			vfres->vf_cap_flags |=
 					VIRTCHNL_VF_OFFLOAD_WB_ON_ITR;
@@ -4841,7 +4841,7 @@ int i40e_ndo_set_vf_trust(struct net_device *netdev, int vf_id, bool setting)
 		goto out;
 	}
 
-	if (pf->flags & I40E_FLAG_MFP_ENABLED) {
+	if (test_bit(I40E_FLAG_MFP_ENA, pf->flags)) {
 		dev_err(&pf->pdev->dev, "Trusted VF not supported in MFP mode.\n");
 		ret = -EINVAL;
 		goto out;
diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h
index 351e0d36df44..cd7dcd0fa7f2 100644
--- a/drivers/net/ethernet/intel/ice/ice.h
+++ b/drivers/net/ethernet/intel/ice/ice.h
@@ -522,11 +522,18 @@ enum ice_misc_thread_tasks {
 	ICE_MISC_THREAD_NBITS		/* must be last */
 };
 
-struct ice_switchdev_info {
+struct ice_eswitch {
 	struct ice_vsi *control_vsi;
 	struct ice_vsi *uplink_vsi;
 	struct ice_esw_br_offloads *br_offloads;
+	struct xarray reprs;
 	bool is_running;
+	/* struct to allow cp queues management optimization */
+	struct {
+		int to_reach;
+		int value;
+		bool is_reaching;
+	} qs;
 };
 
 struct ice_agg_node {
@@ -637,7 +644,7 @@ struct ice_pf {
 	struct ice_link_default_override_tlv link_dflt_override;
 	struct ice_lag *lag; /* Link Aggregation information */
 
-	struct ice_switchdev_info switchdev;
+	struct ice_eswitch eswitch;
 	struct ice_esw_br_port *br_port;
 
 #define ICE_INVALID_AGG_NODE_ID		0
@@ -846,7 +853,7 @@ static inline struct ice_vsi *ice_find_vsi(struct ice_pf *pf, u16 vsi_num)
  */
 static inline bool ice_is_switchdev_running(struct ice_pf *pf)
 {
-	return pf->switchdev.is_running;
+	return pf->eswitch.is_running;
 }
 
 #define ICE_FD_STAT_CTR_BLOCK_COUNT	256
diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c
index 80dc5445b50d..f4e24d11ebd0 100644
--- a/drivers/net/ethernet/intel/ice/ice_devlink.c
+++ b/drivers/net/ethernet/intel/ice/ice_devlink.c
@@ -810,6 +810,10 @@ static void ice_traverse_tx_tree(struct devlink *devlink, struct ice_sched_node
 	struct ice_vf *vf;
 	int i;
 
+	if (node->rate_node)
+		/* already added, skip to the next */
+		goto traverse_children;
+
 	if (node->parent == tc_node) {
 		/* create root node */
 		rate_node = devl_rate_node_create(devlink, node, node->name, NULL);
@@ -831,6 +835,7 @@ static void ice_traverse_tx_tree(struct devlink *devlink, struct ice_sched_node
 	if (rate_node && !IS_ERR(rate_node))
 		node->rate_node = rate_node;
 
+traverse_children:
 	for (i = 0; i < node->num_children; i++)
 		ice_traverse_tx_tree(devlink, node->children[i], tc_node, pf);
 }
@@ -861,6 +866,30 @@ int ice_devlink_rate_init_tx_topology(struct devlink *devlink, struct ice_vsi *v
 	return 0;
 }
 
+static void ice_clear_rate_nodes(struct ice_sched_node *node)
+{
+	node->rate_node = NULL;
+
+	for (int i = 0; i < node->num_children; i++)
+		ice_clear_rate_nodes(node->children[i]);
+}
+
+/**
+ * ice_devlink_rate_clear_tx_topology - clear node->rate_node
+ * @vsi: main vsi struct
+ *
+ * Clear rate_node to cleanup creation of Tx topology.
+ *
+ */
+void ice_devlink_rate_clear_tx_topology(struct ice_vsi *vsi)
+{
+	struct ice_port_info *pi = vsi->port_info;
+
+	mutex_lock(&pi->sched_lock);
+	ice_clear_rate_nodes(pi->root->children[0]);
+	mutex_unlock(&pi->sched_lock);
+}
+
 /**
  * ice_set_object_tx_share - sets node scheduling parameter
  * @pi: devlink struct instance
diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.h b/drivers/net/ethernet/intel/ice/ice_devlink.h
index 6ec96779f52e..d291c0e2e17b 100644
--- a/drivers/net/ethernet/intel/ice/ice_devlink.h
+++ b/drivers/net/ethernet/intel/ice/ice_devlink.h
@@ -20,5 +20,6 @@ void ice_devlink_destroy_regions(struct ice_pf *pf);
 
 int ice_devlink_rate_init_tx_topology(struct devlink *devlink, struct ice_vsi *vsi);
 void ice_tear_down_devlink_rate_tree(struct ice_pf *pf);
+void ice_devlink_rate_clear_tx_topology(struct ice_vsi *vsi);
 
 #endif /* _ICE_DEVLINK_H_ */
diff --git a/drivers/net/ethernet/intel/ice/ice_eswitch.c b/drivers/net/ethernet/intel/ice/ice_eswitch.c
index a655d499abfa..3f80e2081e5d 100644
--- a/drivers/net/ethernet/intel/ice/ice_eswitch.c
+++ b/drivers/net/ethernet/intel/ice/ice_eswitch.c
@@ -11,17 +11,34 @@
 #include "ice_tc_lib.h"
 
 /**
- * ice_eswitch_add_vf_sp_rule - add adv rule with VF's VSI index
+ * ice_eswitch_del_sp_rules - delete adv rules added on PRs
+ * @pf: pointer to the PF struct
+ *
+ * Delete all advanced rules that were used to forward packets with the
+ * device's VSI index to the corresponding eswitch ctrl VSI queue.
+ */
+static void ice_eswitch_del_sp_rules(struct ice_pf *pf)
+{
+	struct ice_repr *repr;
+	unsigned long id;
+
+	xa_for_each(&pf->eswitch.reprs, id, repr) {
+		if (repr->sp_rule.rid)
+			ice_rem_adv_rule_by_id(&pf->hw, &repr->sp_rule);
+	}
+}
+
+/**
+ * ice_eswitch_add_sp_rule - add adv rule with device's VSI index
  * @pf: pointer to PF struct
- * @vf: pointer to VF struct
+ * @repr: pointer to the repr struct
  *
  * This function adds advanced rule that forwards packets with
- * VF's VSI index to the corresponding switchdev ctrl VSI queue.
+ * device's VSI index to the corresponding eswitch ctrl VSI queue.
  */
-static int
-ice_eswitch_add_vf_sp_rule(struct ice_pf *pf, struct ice_vf *vf)
+static int ice_eswitch_add_sp_rule(struct ice_pf *pf, struct ice_repr *repr)
 {
-	struct ice_vsi *ctrl_vsi = pf->switchdev.control_vsi;
+	struct ice_vsi *ctrl_vsi = pf->eswitch.control_vsi;
 	struct ice_adv_rule_info rule_info = { 0 };
 	struct ice_adv_lkup_elem *list;
 	struct ice_hw *hw = &pf->hw;
@@ -38,39 +55,42 @@ ice_eswitch_add_vf_sp_rule(struct ice_pf *pf, struct ice_vf *vf)
 	rule_info.sw_act.vsi_handle = ctrl_vsi->idx;
 	rule_info.sw_act.fltr_act = ICE_FWD_TO_Q;
 	rule_info.sw_act.fwd_id.q_id = hw->func_caps.common_cap.rxq_first_id +
-				       ctrl_vsi->rxq_map[vf->vf_id];
+				       ctrl_vsi->rxq_map[repr->q_id];
 	rule_info.flags_info.act |= ICE_SINGLE_ACT_LB_ENABLE;
 	rule_info.flags_info.act_valid = true;
 	rule_info.tun_type = ICE_SW_TUN_AND_NON_TUN;
-	rule_info.src_vsi = vf->lan_vsi_idx;
+	rule_info.src_vsi = repr->src_vsi->idx;
 
 	err = ice_add_adv_rule(hw, list, lkups_cnt, &rule_info,
-			       &vf->repr->sp_rule);
+			       &repr->sp_rule);
 	if (err)
-		dev_err(ice_pf_to_dev(pf), "Unable to add VF slow-path rule in switchdev mode for VF %d",
-			vf->vf_id);
+		dev_err(ice_pf_to_dev(pf), "Unable to add slow-path rule for eswitch for PR %d",
+			repr->id);
 
 	kfree(list);
 	return err;
 }
 
-/**
- * ice_eswitch_del_vf_sp_rule - delete adv rule with VF's VSI index
- * @vf: pointer to the VF struct
- *
- * Delete the advanced rule that was used to forward packets with the VF's VSI
- * index to the corresponding switchdev ctrl VSI queue.
- */
-static void ice_eswitch_del_vf_sp_rule(struct ice_vf *vf)
+static int
+ice_eswitch_add_sp_rules(struct ice_pf *pf)
 {
-	if (!vf->repr)
-		return;
+	struct ice_repr *repr;
+	unsigned long id;
+	int err;
 
-	ice_rem_adv_rule_by_id(&vf->pf->hw, &vf->repr->sp_rule);
+	xa_for_each(&pf->eswitch.reprs, id, repr) {
+		err = ice_eswitch_add_sp_rule(pf, repr);
+		if (err) {
+			ice_eswitch_del_sp_rules(pf);
+			return err;
+		}
+	}
+
+	return 0;
 }
 
 /**
- * ice_eswitch_setup_env - configure switchdev HW filters
+ * ice_eswitch_setup_env - configure eswitch HW filters
  * @pf: pointer to PF struct
  *
  * This function adds HW filters configuration specific for switchdev
@@ -78,18 +98,18 @@ static void ice_eswitch_del_vf_sp_rule(struct ice_vf *vf)
  */
 static int ice_eswitch_setup_env(struct ice_pf *pf)
 {
-	struct ice_vsi *uplink_vsi = pf->switchdev.uplink_vsi;
-	struct net_device *uplink_netdev = uplink_vsi->netdev;
-	struct ice_vsi *ctrl_vsi = pf->switchdev.control_vsi;
+	struct ice_vsi *uplink_vsi = pf->eswitch.uplink_vsi;
+	struct ice_vsi *ctrl_vsi = pf->eswitch.control_vsi;
+	struct net_device *netdev = uplink_vsi->netdev;
 	struct ice_vsi_vlan_ops *vlan_ops;
 	bool rule_added = false;
 
 	ice_remove_vsi_fltr(&pf->hw, uplink_vsi->idx);
 
-	netif_addr_lock_bh(uplink_netdev);
-	__dev_uc_unsync(uplink_netdev, NULL);
-	__dev_mc_unsync(uplink_netdev, NULL);
-	netif_addr_unlock_bh(uplink_netdev);
+	netif_addr_lock_bh(netdev);
+	__dev_uc_unsync(netdev, NULL);
+	__dev_mc_unsync(netdev, NULL);
+	netif_addr_unlock_bh(netdev);
 
 	if (ice_vsi_add_vlan_zero(uplink_vsi))
 		goto err_def_rx;
@@ -132,19 +152,20 @@ static int ice_eswitch_setup_env(struct ice_pf *pf)
 }
 
 /**
- * ice_eswitch_remap_rings_to_vectors - reconfigure rings of switchdev ctrl VSI
- * @pf: pointer to PF struct
+ * ice_eswitch_remap_rings_to_vectors - reconfigure rings of eswitch ctrl VSI
+ * @eswitch: pointer to eswitch struct
  *
- * In switchdev number of allocated Tx/Rx rings is equal.
+ * In eswitch number of allocated Tx/Rx rings is equal.
  *
  * This function fills q_vectors structures associated with representor and
  * move each ring pairs to port representor netdevs. Each port representor
  * will have dedicated 1 Tx/Rx ring pair, so number of rings pair is equal to
  * number of VFs.
  */
-static void ice_eswitch_remap_rings_to_vectors(struct ice_pf *pf)
+static void ice_eswitch_remap_rings_to_vectors(struct ice_eswitch *eswitch)
 {
-	struct ice_vsi *vsi = pf->switchdev.control_vsi;
+	struct ice_vsi *vsi = eswitch->control_vsi;
+	unsigned long repr_id = 0;
 	int q_id;
 
 	ice_for_each_txq(vsi, q_id) {
@@ -152,13 +173,14 @@ static void ice_eswitch_remap_rings_to_vectors(struct ice_pf *pf)
 		struct ice_tx_ring *tx_ring;
 		struct ice_rx_ring *rx_ring;
 		struct ice_repr *repr;
-		struct ice_vf *vf;
 
-		vf = ice_get_vf_by_id(pf, q_id);
-		if (WARN_ON(!vf))
-			continue;
+		repr = xa_find(&eswitch->reprs, &repr_id, U32_MAX,
+			       XA_PRESENT);
+		if (!repr)
+			break;
 
-		repr = vf->repr;
+		repr_id += 1;
+		repr->q_id = q_id;
 		q_vector = repr->q_vector;
 		tx_ring = vsi->tx_rings[q_id];
 		rx_ring = vsi->rx_rings[q_id];
@@ -181,136 +203,96 @@ static void ice_eswitch_remap_rings_to_vectors(struct ice_pf *pf)
 		rx_ring->q_vector = q_vector;
 		rx_ring->next = NULL;
 		rx_ring->netdev = repr->netdev;
-
-		ice_put_vf(vf);
 	}
 }
 
 /**
- * ice_eswitch_release_reprs - clear PR VSIs configuration
+ * ice_eswitch_release_repr - clear PR VSI configuration
  * @pf: poiner to PF struct
- * @ctrl_vsi: pointer to switchdev control VSI
+ * @repr: pointer to PR
  */
 static void
-ice_eswitch_release_reprs(struct ice_pf *pf, struct ice_vsi *ctrl_vsi)
+ice_eswitch_release_repr(struct ice_pf *pf, struct ice_repr *repr)
 {
-	struct ice_vf *vf;
-	unsigned int bkt;
+	struct ice_vsi *vsi = repr->src_vsi;
 
-	lockdep_assert_held(&pf->vfs.table_lock);
-
-	ice_for_each_vf(pf, bkt, vf) {
-		struct ice_vsi *vsi = vf->repr->src_vsi;
-
-		/* Skip VFs that aren't configured */
-		if (!vf->repr->dst)
-			continue;
+	/* Skip representors that aren't configured */
+	if (!repr->dst)
+		return;
 
-		ice_vsi_update_security(vsi, ice_vsi_ctx_set_antispoof);
-		metadata_dst_free(vf->repr->dst);
-		vf->repr->dst = NULL;
-		ice_eswitch_del_vf_sp_rule(vf);
-		ice_fltr_add_mac_and_broadcast(vsi, vf->hw_lan_addr,
-					       ICE_FWD_TO_VSI);
+	ice_vsi_update_security(vsi, ice_vsi_ctx_set_antispoof);
+	metadata_dst_free(repr->dst);
+	repr->dst = NULL;
+	ice_fltr_add_mac_and_broadcast(vsi, repr->parent_mac,
+				       ICE_FWD_TO_VSI);
 
-		netif_napi_del(&vf->repr->q_vector->napi);
-	}
+	netif_napi_del(&repr->q_vector->napi);
 }
 
 /**
- * ice_eswitch_setup_reprs - configure port reprs to run in switchdev mode
+ * ice_eswitch_setup_repr - configure PR to run in switchdev mode
  * @pf: pointer to PF struct
+ * @repr: pointer to PR struct
  */
-static int ice_eswitch_setup_reprs(struct ice_pf *pf)
+static int ice_eswitch_setup_repr(struct ice_pf *pf, struct ice_repr *repr)
 {
-	struct ice_vsi *ctrl_vsi = pf->switchdev.control_vsi;
-	int max_vsi_num = 0;
-	struct ice_vf *vf;
-	unsigned int bkt;
-
-	lockdep_assert_held(&pf->vfs.table_lock);
-
-	ice_for_each_vf(pf, bkt, vf) {
-		struct ice_vsi *vsi = vf->repr->src_vsi;
-
-		ice_remove_vsi_fltr(&pf->hw, vsi->idx);
-		vf->repr->dst = metadata_dst_alloc(0, METADATA_HW_PORT_MUX,
-						   GFP_KERNEL);
-		if (!vf->repr->dst) {
-			ice_fltr_add_mac_and_broadcast(vsi, vf->hw_lan_addr,
-						       ICE_FWD_TO_VSI);
-			goto err;
-		}
-
-		if (ice_eswitch_add_vf_sp_rule(pf, vf)) {
-			ice_fltr_add_mac_and_broadcast(vsi, vf->hw_lan_addr,
-						       ICE_FWD_TO_VSI);
-			goto err;
-		}
-
-		if (ice_vsi_update_security(vsi, ice_vsi_ctx_clear_antispoof)) {
-			ice_fltr_add_mac_and_broadcast(vsi, vf->hw_lan_addr,
-						       ICE_FWD_TO_VSI);
-			ice_eswitch_del_vf_sp_rule(vf);
-			metadata_dst_free(vf->repr->dst);
-			vf->repr->dst = NULL;
-			goto err;
-		}
+	struct ice_vsi *ctrl_vsi = pf->eswitch.control_vsi;
+	struct ice_vsi *vsi = repr->src_vsi;
+	struct metadata_dst *dst;
 
-		if (ice_vsi_add_vlan_zero(vsi)) {
-			ice_fltr_add_mac_and_broadcast(vsi, vf->hw_lan_addr,
-						       ICE_FWD_TO_VSI);
-			ice_eswitch_del_vf_sp_rule(vf);
-			metadata_dst_free(vf->repr->dst);
-			vf->repr->dst = NULL;
-			ice_vsi_update_security(vsi, ice_vsi_ctx_set_antispoof);
-			goto err;
-		}
+	ice_remove_vsi_fltr(&pf->hw, vsi->idx);
+	repr->dst = metadata_dst_alloc(0, METADATA_HW_PORT_MUX,
+				       GFP_KERNEL);
+	if (!repr->dst)
+		goto err_add_mac_fltr;
 
-		if (max_vsi_num < vsi->vsi_num)
-			max_vsi_num = vsi->vsi_num;
+	if (ice_vsi_update_security(vsi, ice_vsi_ctx_clear_antispoof))
+		goto err_dst_free;
 
-		netif_napi_add(vf->repr->netdev, &vf->repr->q_vector->napi,
-			       ice_napi_poll);
+	if (ice_vsi_add_vlan_zero(vsi))
+		goto err_update_security;
 
-		netif_keep_dst(vf->repr->netdev);
-	}
+	netif_napi_add(repr->netdev, &repr->q_vector->napi,
+		       ice_napi_poll);
 
-	ice_for_each_vf(pf, bkt, vf) {
-		struct ice_repr *repr = vf->repr;
-		struct ice_vsi *vsi = repr->src_vsi;
-		struct metadata_dst *dst;
+	netif_keep_dst(repr->netdev);
 
-		dst = repr->dst;
-		dst->u.port_info.port_id = vsi->vsi_num;
-		dst->u.port_info.lower_dev = repr->netdev;
-		ice_repr_set_traffic_vsi(repr, ctrl_vsi);
-	}
+	dst = repr->dst;
+	dst->u.port_info.port_id = vsi->vsi_num;
+	dst->u.port_info.lower_dev = repr->netdev;
+	ice_repr_set_traffic_vsi(repr, ctrl_vsi);
 
 	return 0;
 
-err:
-	ice_eswitch_release_reprs(pf, ctrl_vsi);
+err_update_security:
+	ice_vsi_update_security(vsi, ice_vsi_ctx_set_antispoof);
+err_dst_free:
+	metadata_dst_free(repr->dst);
+	repr->dst = NULL;
+err_add_mac_fltr:
+	ice_fltr_add_mac_and_broadcast(vsi, repr->parent_mac, ICE_FWD_TO_VSI);
 
 	return -ENODEV;
 }
 
 /**
- * ice_eswitch_update_repr - reconfigure VF port representor
- * @vsi: VF VSI for which port representor is configured
+ * ice_eswitch_update_repr - reconfigure port representor
+ * @repr_id: representor ID
+ * @vsi: VSI for which port representor is configured
  */
-void ice_eswitch_update_repr(struct ice_vsi *vsi)
+void ice_eswitch_update_repr(unsigned long repr_id, struct ice_vsi *vsi)
 {
 	struct ice_pf *pf = vsi->back;
 	struct ice_repr *repr;
-	struct ice_vf *vf;
 	int ret;
 
 	if (!ice_is_switchdev_running(pf))
 		return;
 
-	vf = vsi->vf;
-	repr = vf->repr;
+	repr = xa_load(&pf->eswitch.reprs, repr_id);
+	if (!repr)
+		return;
+
 	repr->src_vsi = vsi;
 	repr->dst->u.port_info.port_id = vsi->vsi_num;
 
@@ -319,9 +301,10 @@ void ice_eswitch_update_repr(struct ice_vsi *vsi)
 
 	ret = ice_vsi_update_security(vsi, ice_vsi_ctx_clear_antispoof);
 	if (ret) {
-		ice_fltr_add_mac_and_broadcast(vsi, vf->hw_lan_addr, ICE_FWD_TO_VSI);
-		dev_err(ice_pf_to_dev(pf), "Failed to update VF %d port representor",
-			vsi->vf->vf_id);
+		ice_fltr_add_mac_and_broadcast(vsi, repr->parent_mac,
+					       ICE_FWD_TO_VSI);
+		dev_err(ice_pf_to_dev(pf), "Failed to update VSI of port representor %d",
+			repr->id);
 	}
 }
 
@@ -353,13 +336,13 @@ ice_eswitch_port_start_xmit(struct sk_buff *skb, struct net_device *netdev)
 	skb_dst_drop(skb);
 	dst_hold((struct dst_entry *)repr->dst);
 	skb_dst_set(skb, (struct dst_entry *)repr->dst);
-	skb->queue_mapping = repr->vf->vf_id;
+	skb->queue_mapping = repr->q_id;
 
 	return ice_start_xmit(skb, netdev);
 }
 
 /**
- * ice_eswitch_set_target_vsi - set switchdev context in Tx context descriptor
+ * ice_eswitch_set_target_vsi - set eswitch context in Tx context descriptor
  * @skb: pointer to send buffer
  * @off: pointer to offload struct
  */
@@ -382,7 +365,7 @@ ice_eswitch_set_target_vsi(struct sk_buff *skb,
 }
 
 /**
- * ice_eswitch_release_env - clear switchdev HW filters
+ * ice_eswitch_release_env - clear eswitch HW filters
  * @pf: pointer to PF struct
  *
  * This function removes HW filters configuration specific for switchdev
@@ -390,8 +373,8 @@ ice_eswitch_set_target_vsi(struct sk_buff *skb,
  */
 static void ice_eswitch_release_env(struct ice_pf *pf)
 {
-	struct ice_vsi *uplink_vsi = pf->switchdev.uplink_vsi;
-	struct ice_vsi *ctrl_vsi = pf->switchdev.control_vsi;
+	struct ice_vsi *uplink_vsi = pf->eswitch.uplink_vsi;
+	struct ice_vsi *ctrl_vsi = pf->eswitch.control_vsi;
 	struct ice_vsi_vlan_ops *vlan_ops;
 
 	vlan_ops = ice_get_compat_vsi_vlan_ops(uplink_vsi);
@@ -407,7 +390,7 @@ static void ice_eswitch_release_env(struct ice_pf *pf)
 }
 
 /**
- * ice_eswitch_vsi_setup - configure switchdev control VSI
+ * ice_eswitch_vsi_setup - configure eswitch control VSI
  * @pf: pointer to PF structure
  * @pi: pointer to port_info structure
  */
@@ -423,49 +406,30 @@ ice_eswitch_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi)
 	return ice_vsi_setup(pf, &params);
 }
 
-/**
- * ice_eswitch_napi_del - remove NAPI handle for all port representors
- * @pf: pointer to PF structure
- */
-static void ice_eswitch_napi_del(struct ice_pf *pf)
-{
-	struct ice_vf *vf;
-	unsigned int bkt;
-
-	lockdep_assert_held(&pf->vfs.table_lock);
-
-	ice_for_each_vf(pf, bkt, vf)
-		netif_napi_del(&vf->repr->q_vector->napi);
-}
-
 /**
  * ice_eswitch_napi_enable - enable NAPI for all port representors
- * @pf: pointer to PF structure
+ * @reprs: xarray of reprs
  */
-static void ice_eswitch_napi_enable(struct ice_pf *pf)
+static void ice_eswitch_napi_enable(struct xarray *reprs)
 {
-	struct ice_vf *vf;
-	unsigned int bkt;
-
-	lockdep_assert_held(&pf->vfs.table_lock);
+	struct ice_repr *repr;
+	unsigned long id;
 
-	ice_for_each_vf(pf, bkt, vf)
-		napi_enable(&vf->repr->q_vector->napi);
+	xa_for_each(reprs, id, repr)
+		napi_enable(&repr->q_vector->napi);
 }
 
 /**
  * ice_eswitch_napi_disable - disable NAPI for all port representors
- * @pf: pointer to PF structure
+ * @reprs: xarray of reprs
  */
-static void ice_eswitch_napi_disable(struct ice_pf *pf)
+static void ice_eswitch_napi_disable(struct xarray *reprs)
 {
-	struct ice_vf *vf;
-	unsigned int bkt;
-
-	lockdep_assert_held(&pf->vfs.table_lock);
+	struct ice_repr *repr;
+	unsigned long id;
 
-	ice_for_each_vf(pf, bkt, vf)
-		napi_disable(&vf->repr->q_vector->napi);
+	xa_for_each(reprs, id, repr)
+		napi_disable(&repr->q_vector->napi);
 }
 
 /**
@@ -486,39 +450,26 @@ static int ice_eswitch_enable_switchdev(struct ice_pf *pf)
 		return -EINVAL;
 	}
 
-	pf->switchdev.control_vsi = ice_eswitch_vsi_setup(pf, pf->hw.port_info);
-	if (!pf->switchdev.control_vsi)
+	pf->eswitch.control_vsi = ice_eswitch_vsi_setup(pf, pf->hw.port_info);
+	if (!pf->eswitch.control_vsi)
 		return -ENODEV;
 
-	ctrl_vsi = pf->switchdev.control_vsi;
-	pf->switchdev.uplink_vsi = uplink_vsi;
+	ctrl_vsi = pf->eswitch.control_vsi;
+	/* cp VSI is createad with 1 queue as default */
+	pf->eswitch.qs.value = 1;
+	pf->eswitch.uplink_vsi = uplink_vsi;
 
 	if (ice_eswitch_setup_env(pf))
 		goto err_vsi;
 
-	if (ice_repr_add_for_all_vfs(pf))
-		goto err_repr_add;
-
-	if (ice_eswitch_setup_reprs(pf))
-		goto err_setup_reprs;
-
-	ice_eswitch_remap_rings_to_vectors(pf);
-
-	if (ice_vsi_open(ctrl_vsi))
-		goto err_setup_reprs;
-
 	if (ice_eswitch_br_offloads_init(pf))
 		goto err_br_offloads;
 
-	ice_eswitch_napi_enable(pf);
+	pf->eswitch.is_running = true;
 
 	return 0;
 
 err_br_offloads:
-	ice_vsi_close(ctrl_vsi);
-err_setup_reprs:
-	ice_repr_rem_from_all_vfs(pf);
-err_repr_add:
 	ice_eswitch_release_env(pf);
 err_vsi:
 	ice_vsi_release(ctrl_vsi);
@@ -526,19 +477,19 @@ static int ice_eswitch_enable_switchdev(struct ice_pf *pf)
 }
 
 /**
- * ice_eswitch_disable_switchdev - disable switchdev resources
+ * ice_eswitch_disable_switchdev - disable eswitch resources
  * @pf: pointer to PF structure
  */
 static void ice_eswitch_disable_switchdev(struct ice_pf *pf)
 {
-	struct ice_vsi *ctrl_vsi = pf->switchdev.control_vsi;
+	struct ice_vsi *ctrl_vsi = pf->eswitch.control_vsi;
 
-	ice_eswitch_napi_disable(pf);
 	ice_eswitch_br_offloads_deinit(pf);
 	ice_eswitch_release_env(pf);
-	ice_eswitch_release_reprs(pf, ctrl_vsi);
 	ice_vsi_release(ctrl_vsi);
-	ice_repr_rem_from_all_vfs(pf);
+
+	pf->eswitch.is_running = false;
+	pf->eswitch.qs.is_reaching = false;
 }
 
 /**
@@ -566,6 +517,7 @@ ice_eswitch_mode_set(struct devlink *devlink, u16 mode,
 	case DEVLINK_ESWITCH_MODE_LEGACY:
 		dev_info(ice_pf_to_dev(pf), "PF %d changed eswitch mode to legacy",
 			 pf->hw.pf_id);
+		xa_destroy(&pf->eswitch.reprs);
 		NL_SET_ERR_MSG_MOD(extack, "Changed eswitch mode to legacy");
 		break;
 	case DEVLINK_ESWITCH_MODE_SWITCHDEV:
@@ -578,6 +530,7 @@ ice_eswitch_mode_set(struct devlink *devlink, u16 mode,
 
 		dev_info(ice_pf_to_dev(pf), "PF %d changed eswitch mode to switchdev",
 			 pf->hw.pf_id);
+		xa_init_flags(&pf->eswitch.reprs, XA_FLAGS_ALLOC);
 		NL_SET_ERR_MSG_MOD(extack, "Changed eswitch mode to switchdev");
 		break;
 	}
@@ -616,74 +569,166 @@ bool ice_is_eswitch_mode_switchdev(struct ice_pf *pf)
 }
 
 /**
- * ice_eswitch_release - cleanup eswitch
+ * ice_eswitch_start_all_tx_queues - start Tx queues of all port representors
  * @pf: pointer to PF structure
  */
-void ice_eswitch_release(struct ice_pf *pf)
+static void ice_eswitch_start_all_tx_queues(struct ice_pf *pf)
 {
-	if (pf->eswitch_mode == DEVLINK_ESWITCH_MODE_LEGACY)
+	struct ice_repr *repr;
+	unsigned long id;
+
+	if (test_bit(ICE_DOWN, pf->state))
 		return;
 
-	ice_eswitch_disable_switchdev(pf);
-	pf->switchdev.is_running = false;
+	xa_for_each(&pf->eswitch.reprs, id, repr)
+		ice_repr_start_tx_queues(repr);
 }
 
 /**
- * ice_eswitch_configure - configure eswitch
+ * ice_eswitch_stop_all_tx_queues - stop Tx queues of all port representors
  * @pf: pointer to PF structure
  */
-int ice_eswitch_configure(struct ice_pf *pf)
+void ice_eswitch_stop_all_tx_queues(struct ice_pf *pf)
 {
-	int status;
-
-	if (pf->eswitch_mode == DEVLINK_ESWITCH_MODE_LEGACY || pf->switchdev.is_running)
-		return 0;
+	struct ice_repr *repr;
+	unsigned long id;
 
-	status = ice_eswitch_enable_switchdev(pf);
-	if (status)
-		return status;
+	if (test_bit(ICE_DOWN, pf->state))
+		return;
 
-	pf->switchdev.is_running = true;
-	return 0;
+	xa_for_each(&pf->eswitch.reprs, id, repr)
+		ice_repr_stop_tx_queues(repr);
 }
 
-/**
- * ice_eswitch_start_all_tx_queues - start Tx queues of all port representors
- * @pf: pointer to PF structure
- */
-static void ice_eswitch_start_all_tx_queues(struct ice_pf *pf)
+static void ice_eswitch_stop_reprs(struct ice_pf *pf)
 {
-	struct ice_vf *vf;
-	unsigned int bkt;
+	ice_eswitch_del_sp_rules(pf);
+	ice_eswitch_stop_all_tx_queues(pf);
+	ice_eswitch_napi_disable(&pf->eswitch.reprs);
+}
 
-	lockdep_assert_held(&pf->vfs.table_lock);
+static void ice_eswitch_start_reprs(struct ice_pf *pf)
+{
+	ice_eswitch_napi_enable(&pf->eswitch.reprs);
+	ice_eswitch_start_all_tx_queues(pf);
+	ice_eswitch_add_sp_rules(pf);
+}
 
-	if (test_bit(ICE_DOWN, pf->state))
-		return;
+static void
+ice_eswitch_cp_change_queues(struct ice_eswitch *eswitch, int change)
+{
+	struct ice_vsi *cp = eswitch->control_vsi;
+	int queues = 0;
+
+	if (eswitch->qs.is_reaching) {
+		if (eswitch->qs.to_reach >= eswitch->qs.value + change) {
+			queues = eswitch->qs.to_reach;
+			eswitch->qs.is_reaching = false;
+		} else {
+			queues = 0;
+		}
+	} else if ((change > 0 && cp->alloc_txq <= eswitch->qs.value) ||
+		   change < 0) {
+		queues = cp->alloc_txq + change;
+	}
 
-	ice_for_each_vf(pf, bkt, vf) {
-		if (vf->repr)
-			ice_repr_start_tx_queues(vf->repr);
+	if (queues) {
+		cp->req_txq = queues;
+		cp->req_rxq = queues;
+		ice_vsi_close(cp);
+		ice_vsi_rebuild(cp, ICE_VSI_FLAG_NO_INIT);
+		ice_vsi_open(cp);
+	} else if (!change) {
+		/* change == 0 means that VSI wasn't open, open it here */
+		ice_vsi_open(cp);
 	}
+
+	eswitch->qs.value += change;
+	ice_eswitch_remap_rings_to_vectors(eswitch);
 }
 
-/**
- * ice_eswitch_stop_all_tx_queues - stop Tx queues of all port representors
- * @pf: pointer to PF structure
- */
-void ice_eswitch_stop_all_tx_queues(struct ice_pf *pf)
+int
+ice_eswitch_attach(struct ice_pf *pf, struct ice_vf *vf)
 {
-	struct ice_vf *vf;
-	unsigned int bkt;
+	struct ice_repr *repr;
+	int change = 1;
+	int err;
 
-	lockdep_assert_held(&pf->vfs.table_lock);
+	if (pf->eswitch_mode == DEVLINK_ESWITCH_MODE_LEGACY)
+		return 0;
 
-	if (test_bit(ICE_DOWN, pf->state))
+	if (xa_empty(&pf->eswitch.reprs)) {
+		err = ice_eswitch_enable_switchdev(pf);
+		if (err)
+			return err;
+		/* Control plane VSI is created with 1 queue as default */
+		pf->eswitch.qs.to_reach -= 1;
+		change = 0;
+	}
+
+	ice_eswitch_stop_reprs(pf);
+
+	repr = ice_repr_add_vf(vf);
+	if (IS_ERR(repr))
+		goto err_create_repr;
+
+	err = ice_eswitch_setup_repr(pf, repr);
+	if (err)
+		goto err_setup_repr;
+
+	err = xa_alloc(&pf->eswitch.reprs, &repr->id, repr,
+		       XA_LIMIT(1, INT_MAX), GFP_KERNEL);
+	if (err)
+		goto err_xa_alloc;
+
+	vf->repr_id = repr->id;
+
+	ice_eswitch_cp_change_queues(&pf->eswitch, change);
+	ice_eswitch_start_reprs(pf);
+
+	return 0;
+
+err_xa_alloc:
+	ice_eswitch_release_repr(pf, repr);
+err_setup_repr:
+	ice_repr_rem_vf(repr);
+err_create_repr:
+	if (xa_empty(&pf->eswitch.reprs))
+		ice_eswitch_disable_switchdev(pf);
+	ice_eswitch_start_reprs(pf);
+
+	return err;
+}
+
+void ice_eswitch_detach(struct ice_pf *pf, struct ice_vf *vf)
+{
+	struct ice_repr *repr = xa_load(&pf->eswitch.reprs, vf->repr_id);
+	struct devlink *devlink = priv_to_devlink(pf);
+
+	if (!repr)
 		return;
 
-	ice_for_each_vf(pf, bkt, vf) {
-		if (vf->repr)
-			ice_repr_stop_tx_queues(vf->repr);
+	ice_eswitch_stop_reprs(pf);
+	xa_erase(&pf->eswitch.reprs, repr->id);
+
+	if (xa_empty(&pf->eswitch.reprs))
+		ice_eswitch_disable_switchdev(pf);
+	else
+		ice_eswitch_cp_change_queues(&pf->eswitch, -1);
+
+	ice_eswitch_release_repr(pf, repr);
+	ice_repr_rem_vf(repr);
+
+	if (xa_empty(&pf->eswitch.reprs)) {
+		/* since all port representors are destroyed, there is
+		 * no point in keeping the nodes
+		 */
+		ice_devlink_rate_clear_tx_topology(ice_get_main_vsi(pf));
+		devl_lock(devlink);
+		devl_rate_nodes_destroy(devlink);
+		devl_unlock(devlink);
+	} else {
+		ice_eswitch_start_reprs(pf);
 	}
 }
 
@@ -693,30 +738,35 @@ void ice_eswitch_stop_all_tx_queues(struct ice_pf *pf)
  */
 int ice_eswitch_rebuild(struct ice_pf *pf)
 {
-	struct ice_vsi *ctrl_vsi = pf->switchdev.control_vsi;
-	int status;
-
-	ice_eswitch_napi_disable(pf);
-	ice_eswitch_napi_del(pf);
-
-	status = ice_eswitch_setup_env(pf);
-	if (status)
-		return status;
+	struct ice_repr *repr;
+	unsigned long id;
+	int err;
 
-	status = ice_eswitch_setup_reprs(pf);
-	if (status)
-		return status;
+	if (!ice_is_switchdev_running(pf))
+		return 0;
 
-	ice_eswitch_remap_rings_to_vectors(pf);
+	err = ice_vsi_rebuild(pf->eswitch.control_vsi, ICE_VSI_FLAG_INIT);
+	if (err)
+		return err;
 
-	ice_replay_tc_fltrs(pf);
+	xa_for_each(&pf->eswitch.reprs, id, repr)
+		ice_eswitch_detach(pf, repr->vf);
 
-	status = ice_vsi_open(ctrl_vsi);
-	if (status)
-		return status;
+	return 0;
+}
 
-	ice_eswitch_napi_enable(pf);
-	ice_eswitch_start_all_tx_queues(pf);
+/**
+ * ice_eswitch_reserve_cp_queues - reserve control plane VSI queues
+ * @pf: pointer to PF structure
+ * @change: how many more (or less) queues is needed
+ *
+ * Remember to call ice_eswitch_attach/detach() the "change" times.
+ */
+void ice_eswitch_reserve_cp_queues(struct ice_pf *pf, int change)
+{
+	if (pf->eswitch.qs.value + change < 0)
+		return;
 
-	return 0;
+	pf->eswitch.qs.to_reach = pf->eswitch.qs.value + change;
+	pf->eswitch.qs.is_reaching = true;
 }
diff --git a/drivers/net/ethernet/intel/ice/ice_eswitch.h b/drivers/net/ethernet/intel/ice/ice_eswitch.h
index b18bf83a2f5b..1a288a03a79a 100644
--- a/drivers/net/ethernet/intel/ice/ice_eswitch.h
+++ b/drivers/net/ethernet/intel/ice/ice_eswitch.h
@@ -7,8 +7,9 @@
 #include <net/devlink.h>
 
 #ifdef CONFIG_ICE_SWITCHDEV
-void ice_eswitch_release(struct ice_pf *pf);
-int ice_eswitch_configure(struct ice_pf *pf);
+void ice_eswitch_detach(struct ice_pf *pf, struct ice_vf *vf);
+int
+ice_eswitch_attach(struct ice_pf *pf, struct ice_vf *vf);
 int ice_eswitch_rebuild(struct ice_pf *pf);
 
 int ice_eswitch_mode_get(struct devlink *devlink, u16 *mode);
@@ -17,7 +18,7 @@ ice_eswitch_mode_set(struct devlink *devlink, u16 mode,
 		     struct netlink_ext_ack *extack);
 bool ice_is_eswitch_mode_switchdev(struct ice_pf *pf);
 
-void ice_eswitch_update_repr(struct ice_vsi *vsi);
+void ice_eswitch_update_repr(unsigned long repr_id, struct ice_vsi *vsi);
 
 void ice_eswitch_stop_all_tx_queues(struct ice_pf *pf);
 
@@ -25,8 +26,15 @@ void ice_eswitch_set_target_vsi(struct sk_buff *skb,
 				struct ice_tx_offload_params *off);
 netdev_tx_t
 ice_eswitch_port_start_xmit(struct sk_buff *skb, struct net_device *netdev);
+void ice_eswitch_reserve_cp_queues(struct ice_pf *pf, int change);
 #else /* CONFIG_ICE_SWITCHDEV */
-static inline void ice_eswitch_release(struct ice_pf *pf) { }
+static inline void ice_eswitch_detach(struct ice_pf *pf, struct ice_vf *vf) { }
+
+static inline int
+ice_eswitch_attach(struct ice_pf *pf, struct ice_vf *vf)
+{
+	return -EOPNOTSUPP;
+}
 
 static inline void ice_eswitch_stop_all_tx_queues(struct ice_pf *pf) { }
 
@@ -34,7 +42,8 @@ static inline void
 ice_eswitch_set_target_vsi(struct sk_buff *skb,
 			   struct ice_tx_offload_params *off) { }
 
-static inline void ice_eswitch_update_repr(struct ice_vsi *vsi) { }
+static inline void
+ice_eswitch_update_repr(unsigned long repr_id, struct ice_vsi *vsi) { }
 
 static inline int ice_eswitch_configure(struct ice_pf *pf)
 {
@@ -68,5 +77,8 @@ ice_eswitch_port_start_xmit(struct sk_buff *skb, struct net_device *netdev)
 {
 	return NETDEV_TX_BUSY;
 }
+
+static inline void
+ice_eswitch_reserve_cp_queues(struct ice_pf *pf, int change) { }
 #endif /* CONFIG_ICE_SWITCHDEV */
 #endif /* _ICE_ESWITCH_H_ */
diff --git a/drivers/net/ethernet/intel/ice/ice_eswitch_br.c b/drivers/net/ethernet/intel/ice/ice_eswitch_br.c
index 6ae0269bdf73..ac5beecd028b 100644
--- a/drivers/net/ethernet/intel/ice/ice_eswitch_br.c
+++ b/drivers/net/ethernet/intel/ice/ice_eswitch_br.c
@@ -893,10 +893,14 @@ ice_eswitch_br_port_deinit(struct ice_esw_br *bridge,
 			ice_eswitch_br_fdb_entry_delete(bridge, fdb_entry);
 	}
 
-	if (br_port->type == ICE_ESWITCH_BR_UPLINK_PORT && vsi->back)
+	if (br_port->type == ICE_ESWITCH_BR_UPLINK_PORT && vsi->back) {
 		vsi->back->br_port = NULL;
-	else if (vsi->vf && vsi->vf->repr)
-		vsi->vf->repr->br_port = NULL;
+	} else {
+		struct ice_repr *repr = ice_repr_get_by_vsi(vsi);
+
+		if (repr)
+			repr->br_port = NULL;
+	}
 
 	xa_erase(&bridge->ports, br_port->vsi_idx);
 	ice_eswitch_br_port_vlans_flush(br_port);
@@ -947,7 +951,7 @@ ice_eswitch_br_vf_repr_port_init(struct ice_esw_br *bridge,
 static int
 ice_eswitch_br_uplink_port_init(struct ice_esw_br *bridge, struct ice_pf *pf)
 {
-	struct ice_vsi *vsi = pf->switchdev.uplink_vsi;
+	struct ice_vsi *vsi = pf->eswitch.uplink_vsi;
 	struct ice_esw_br_port *br_port;
 	int err;
 
@@ -1185,7 +1189,7 @@ ice_eswitch_br_port_event(struct notifier_block *nb,
 static void
 ice_eswitch_br_offloads_dealloc(struct ice_pf *pf)
 {
-	struct ice_esw_br_offloads *br_offloads = pf->switchdev.br_offloads;
+	struct ice_esw_br_offloads *br_offloads = pf->eswitch.br_offloads;
 
 	ASSERT_RTNL();
 
@@ -1194,7 +1198,7 @@ ice_eswitch_br_offloads_dealloc(struct ice_pf *pf)
 
 	ice_eswitch_br_deinit(br_offloads, br_offloads->bridge);
 
-	pf->switchdev.br_offloads = NULL;
+	pf->eswitch.br_offloads = NULL;
 	kfree(br_offloads);
 }
 
@@ -1205,14 +1209,14 @@ ice_eswitch_br_offloads_alloc(struct ice_pf *pf)
 
 	ASSERT_RTNL();
 
-	if (pf->switchdev.br_offloads)
+	if (pf->eswitch.br_offloads)
 		return ERR_PTR(-EEXIST);
 
 	br_offloads = kzalloc(sizeof(*br_offloads), GFP_KERNEL);
 	if (!br_offloads)
 		return ERR_PTR(-ENOMEM);
 
-	pf->switchdev.br_offloads = br_offloads;
+	pf->eswitch.br_offloads = br_offloads;
 	br_offloads->pf = pf;
 
 	return br_offloads;
@@ -1223,7 +1227,7 @@ ice_eswitch_br_offloads_deinit(struct ice_pf *pf)
 {
 	struct ice_esw_br_offloads *br_offloads;
 
-	br_offloads = pf->switchdev.br_offloads;
+	br_offloads = pf->eswitch.br_offloads;
 	if (!br_offloads)
 		return;
 
diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c
index 4b1e56396293..d826b5afa143 100644
--- a/drivers/net/ethernet/intel/ice/ice_lib.c
+++ b/drivers/net/ethernet/intel/ice/ice_lib.c
@@ -212,11 +212,18 @@ static void ice_vsi_set_num_qs(struct ice_vsi *vsi)
 						 vsi->alloc_txq));
 		break;
 	case ICE_VSI_SWITCHDEV_CTRL:
-		/* The number of queues for ctrl VSI is equal to number of VFs.
+		/* The number of queues for ctrl VSI is equal to number of PRs
 		 * Each ring is associated to the corresponding VF_PR netdev.
+		 * Tx and Rx rings are always equal
 		 */
-		vsi->alloc_txq = ice_get_num_vfs(pf);
-		vsi->alloc_rxq = vsi->alloc_txq;
+		if (vsi->req_txq && vsi->req_rxq) {
+			vsi->alloc_txq = vsi->req_txq;
+			vsi->alloc_rxq = vsi->req_rxq;
+		} else {
+			vsi->alloc_txq = 1;
+			vsi->alloc_rxq = 1;
+		}
+
 		vsi->num_q_vectors = 1;
 		break;
 	case ICE_VSI_VF:
@@ -519,16 +526,14 @@ static irqreturn_t ice_eswitch_msix_clean_rings(int __always_unused irq, void *d
 {
 	struct ice_q_vector *q_vector = (struct ice_q_vector *)data;
 	struct ice_pf *pf = q_vector->vsi->back;
-	struct ice_vf *vf;
-	unsigned int bkt;
+	struct ice_repr *repr;
+	unsigned long id;
 
 	if (!q_vector->tx.tx_ring && !q_vector->rx.rx_ring)
 		return IRQ_HANDLED;
 
-	rcu_read_lock();
-	ice_for_each_vf_rcu(pf, bkt, vf)
-		napi_schedule(&vf->repr->q_vector->napi);
-	rcu_read_unlock();
+	xa_for_each(&pf->eswitch.reprs, id, repr)
+		napi_schedule(&repr->q_vector->napi);
 
 	return IRQ_HANDLED;
 }
@@ -3071,27 +3076,26 @@ ice_vsi_rebuild_set_coalesce(struct ice_vsi *vsi,
 }
 
 /**
- * ice_vsi_realloc_stat_arrays - Frees unused stat structures
+ * ice_vsi_realloc_stat_arrays - Frees unused stat structures or alloc new ones
  * @vsi: VSI pointer
- * @prev_txq: Number of Tx rings before ring reallocation
- * @prev_rxq: Number of Rx rings before ring reallocation
  */
-static void
-ice_vsi_realloc_stat_arrays(struct ice_vsi *vsi, int prev_txq, int prev_rxq)
+static int
+ice_vsi_realloc_stat_arrays(struct ice_vsi *vsi)
 {
+	u16 req_txq = vsi->req_txq ? vsi->req_txq : vsi->alloc_txq;
+	u16 req_rxq = vsi->req_rxq ? vsi->req_rxq : vsi->alloc_rxq;
+	struct ice_ring_stats **tx_ring_stats;
+	struct ice_ring_stats **rx_ring_stats;
 	struct ice_vsi_stats *vsi_stat;
 	struct ice_pf *pf = vsi->back;
+	u16 prev_txq = vsi->alloc_txq;
+	u16 prev_rxq = vsi->alloc_rxq;
 	int i;
 
-	if (!prev_txq || !prev_rxq)
-		return;
-	if (vsi->type == ICE_VSI_CHNL)
-		return;
-
 	vsi_stat = pf->vsi_stats[vsi->idx];
 
-	if (vsi->num_txq < prev_txq) {
-		for (i = vsi->num_txq; i < prev_txq; i++) {
+	if (req_txq < prev_txq) {
+		for (i = req_txq; i < prev_txq; i++) {
 			if (vsi_stat->tx_ring_stats[i]) {
 				kfree_rcu(vsi_stat->tx_ring_stats[i], rcu);
 				WRITE_ONCE(vsi_stat->tx_ring_stats[i], NULL);
@@ -3099,14 +3103,36 @@ ice_vsi_realloc_stat_arrays(struct ice_vsi *vsi, int prev_txq, int prev_rxq)
 		}
 	}
 
-	if (vsi->num_rxq < prev_rxq) {
-		for (i = vsi->num_rxq; i < prev_rxq; i++) {
+	tx_ring_stats = vsi_stat->rx_ring_stats;
+	vsi_stat->tx_ring_stats =
+		krealloc_array(vsi_stat->tx_ring_stats, req_txq,
+			       sizeof(*vsi_stat->tx_ring_stats),
+			       GFP_KERNEL | __GFP_ZERO);
+	if (!vsi_stat->tx_ring_stats) {
+		vsi_stat->tx_ring_stats = tx_ring_stats;
+		return -ENOMEM;
+	}
+
+	if (req_rxq < prev_rxq) {
+		for (i = req_rxq; i < prev_rxq; i++) {
 			if (vsi_stat->rx_ring_stats[i]) {
 				kfree_rcu(vsi_stat->rx_ring_stats[i], rcu);
 				WRITE_ONCE(vsi_stat->rx_ring_stats[i], NULL);
 			}
 		}
 	}
+
+	rx_ring_stats = vsi_stat->rx_ring_stats;
+	vsi_stat->rx_ring_stats =
+		krealloc_array(vsi_stat->rx_ring_stats, req_rxq,
+			       sizeof(*vsi_stat->rx_ring_stats),
+			       GFP_KERNEL | __GFP_ZERO);
+	if (!vsi_stat->rx_ring_stats) {
+		vsi_stat->rx_ring_stats = rx_ring_stats;
+		return -ENOMEM;
+	}
+
+	return 0;
 }
 
 /**
@@ -3123,9 +3149,9 @@ int ice_vsi_rebuild(struct ice_vsi *vsi, u32 vsi_flags)
 {
 	struct ice_vsi_cfg_params params = {};
 	struct ice_coalesce_stored *coalesce;
-	int ret, prev_txq, prev_rxq;
 	int prev_num_q_vectors = 0;
 	struct ice_pf *pf;
+	int ret;
 
 	if (!vsi)
 		return -EINVAL;
@@ -3144,8 +3170,9 @@ int ice_vsi_rebuild(struct ice_vsi *vsi, u32 vsi_flags)
 
 	prev_num_q_vectors = ice_vsi_rebuild_get_coalesce(vsi, coalesce);
 
-	prev_txq = vsi->num_txq;
-	prev_rxq = vsi->num_rxq;
+	ret = ice_vsi_realloc_stat_arrays(vsi);
+	if (ret)
+		goto err_vsi_cfg;
 
 	ice_vsi_decfg(vsi);
 	ret = ice_vsi_cfg_def(vsi, &params);
@@ -3163,8 +3190,6 @@ int ice_vsi_rebuild(struct ice_vsi *vsi, u32 vsi_flags)
 		return ice_schedule_reset(pf, ICE_RESET_PFR);
 	}
 
-	ice_vsi_realloc_stat_arrays(vsi, prev_txq, prev_rxq);
-
 	ice_vsi_rebuild_set_coalesce(vsi, coalesce, prev_num_q_vectors);
 	kfree(coalesce);
 
diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c
index 6607fa6fe556..10822011de22 100644
--- a/drivers/net/ethernet/intel/ice/ice_main.c
+++ b/drivers/net/ethernet/intel/ice/ice_main.c
@@ -4702,6 +4702,8 @@ static void ice_deinit_features(struct ice_pf *pf)
 		ice_ptp_release(pf);
 	if (test_bit(ICE_FLAG_DPLL, pf->flags))
 		ice_dpll_deinit(pf);
+	if (pf->eswitch_mode == DEVLINK_ESWITCH_MODE_SWITCHDEV)
+		xa_destroy(&pf->eswitch.reprs);
 }
 
 static void ice_init_wakeup(struct ice_pf *pf)
@@ -7410,9 +7412,9 @@ static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type)
 			ice_ptp_cfg_timestamp(pf, true);
 	}
 
-	err = ice_vsi_rebuild_by_type(pf, ICE_VSI_SWITCHDEV_CTRL);
+	err = ice_eswitch_rebuild(pf);
 	if (err) {
-		dev_err(dev, "Switchdev CTRL VSI rebuild failed: %d\n", err);
+		dev_err(dev, "Switchdev rebuild failed: %d\n", err);
 		goto err_vsi_rebuild;
 	}
 
diff --git a/drivers/net/ethernet/intel/ice/ice_repr.c b/drivers/net/ethernet/intel/ice/ice_repr.c
index c686ac0935eb..5f30fb131f74 100644
--- a/drivers/net/ethernet/intel/ice/ice_repr.c
+++ b/drivers/net/ethernet/intel/ice/ice_repr.c
@@ -14,7 +14,7 @@
  */
 static int ice_repr_get_sw_port_id(struct ice_repr *repr)
 {
-	return repr->vf->pf->hw.port_info->lport;
+	return repr->src_vsi->back->hw.port_info->lport;
 }
 
 /**
@@ -35,7 +35,7 @@ ice_repr_get_phys_port_name(struct net_device *netdev, char *buf, size_t len)
 		return -EOPNOTSUPP;
 
 	res = snprintf(buf, len, "pf%dvfr%d", ice_repr_get_sw_port_id(repr),
-		       repr->vf->vf_id);
+		       repr->id);
 	if (res <= 0)
 		return -EOPNOTSUPP;
 	return 0;
@@ -278,25 +278,67 @@ ice_repr_reg_netdev(struct net_device *netdev)
 	return register_netdev(netdev);
 }
 
+static void ice_repr_remove_node(struct devlink_port *devlink_port)
+{
+	devl_lock(devlink_port->devlink);
+	devl_rate_leaf_destroy(devlink_port);
+	devl_unlock(devlink_port->devlink);
+}
+
 /**
- * ice_repr_add - add representor for VF
- * @vf: pointer to VF structure
+ * ice_repr_rem - remove representor from VF
+ * @repr: pointer to representor structure
  */
-static int ice_repr_add(struct ice_vf *vf)
+static void ice_repr_rem(struct ice_repr *repr)
+{
+	kfree(repr->q_vector);
+	free_netdev(repr->netdev);
+	kfree(repr);
+}
+
+/**
+ * ice_repr_rem_vf - remove representor from VF
+ * @repr: pointer to representor structure
+ */
+void ice_repr_rem_vf(struct ice_repr *repr)
+{
+	ice_repr_remove_node(&repr->vf->devlink_port);
+	unregister_netdev(repr->netdev);
+	ice_devlink_destroy_vf_port(repr->vf);
+	ice_virtchnl_set_dflt_ops(repr->vf);
+	ice_repr_rem(repr);
+}
+
+static void ice_repr_set_tx_topology(struct ice_pf *pf)
+{
+	struct devlink *devlink;
+
+	/* only export if ADQ and DCB disabled and eswitch enabled*/
+	if (ice_is_adq_active(pf) || ice_is_dcb_active(pf) ||
+	    !ice_is_switchdev_running(pf))
+		return;
+
+	devlink = priv_to_devlink(pf);
+	ice_devlink_rate_init_tx_topology(devlink, ice_get_main_vsi(pf));
+}
+
+/**
+ * ice_repr_add - add representor for generic VSI
+ * @pf: pointer to PF structure
+ * @src_vsi: pointer to VSI structure of device to represent
+ * @parent_mac: device MAC address
+ */
+static struct ice_repr *
+ice_repr_add(struct ice_pf *pf, struct ice_vsi *src_vsi, const u8 *parent_mac)
 {
 	struct ice_q_vector *q_vector;
 	struct ice_netdev_priv *np;
 	struct ice_repr *repr;
-	struct ice_vsi *vsi;
 	int err;
 
-	vsi = ice_get_vf_vsi(vf);
-	if (!vsi)
-		return -EINVAL;
-
 	repr = kzalloc(sizeof(*repr), GFP_KERNEL);
 	if (!repr)
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 
 	repr->netdev = alloc_etherdev(sizeof(struct ice_netdev_priv));
 	if (!repr->netdev) {
@@ -304,9 +346,7 @@ static int ice_repr_add(struct ice_vf *vf)
 		goto err_alloc;
 	}
 
-	repr->src_vsi = vsi;
-	repr->vf = vf;
-	vf->repr = repr;
+	repr->src_vsi = src_vsi;
 	np = netdev_priv(repr->netdev);
 	np->repr = repr;
 
@@ -316,10 +356,40 @@ static int ice_repr_add(struct ice_vf *vf)
 		goto err_alloc_q_vector;
 	}
 	repr->q_vector = q_vector;
+	repr->q_id = repr->id;
+
+	ether_addr_copy(repr->parent_mac, parent_mac);
+
+	return repr;
+
+err_alloc_q_vector:
+	free_netdev(repr->netdev);
+err_alloc:
+	kfree(repr);
+	return ERR_PTR(err);
+}
+
+struct ice_repr *ice_repr_add_vf(struct ice_vf *vf)
+{
+	struct ice_repr *repr;
+	struct ice_vsi *vsi;
+	int err;
+
+	vsi = ice_get_vf_vsi(vf);
+	if (!vsi)
+		return ERR_PTR(-ENOENT);
 
 	err = ice_devlink_create_vf_port(vf);
 	if (err)
-		goto err_devlink;
+		return ERR_PTR(err);
+
+	repr = ice_repr_add(vf->pf, vsi, vf->hw_lan_addr);
+	if (IS_ERR(repr)) {
+		err = PTR_ERR(repr);
+		goto err_repr_add;
+	}
+
+	repr->vf = vf;
 
 	repr->netdev->min_mtu = ETH_MIN_MTU;
 	repr->netdev->max_mtu = ICE_MAX_MTU;
@@ -331,100 +401,23 @@ static int ice_repr_add(struct ice_vf *vf)
 		goto err_netdev;
 
 	ice_virtchnl_set_repr_ops(vf);
+	ice_repr_set_tx_topology(vf->pf);
 
-	return 0;
+	return repr;
 
 err_netdev:
+	ice_repr_rem(repr);
+err_repr_add:
 	ice_devlink_destroy_vf_port(vf);
-err_devlink:
-	kfree(repr->q_vector);
-	vf->repr->q_vector = NULL;
-err_alloc_q_vector:
-	free_netdev(repr->netdev);
-	repr->netdev = NULL;
-err_alloc:
-	kfree(repr);
-	vf->repr = NULL;
-	return err;
-}
-
-/**
- * ice_repr_rem - remove representor from VF
- * @vf: pointer to VF structure
- */
-static void ice_repr_rem(struct ice_vf *vf)
-{
-	if (!vf->repr)
-		return;
-
-	kfree(vf->repr->q_vector);
-	vf->repr->q_vector = NULL;
-	unregister_netdev(vf->repr->netdev);
-	ice_devlink_destroy_vf_port(vf);
-	free_netdev(vf->repr->netdev);
-	vf->repr->netdev = NULL;
-	kfree(vf->repr);
-	vf->repr = NULL;
-
-	ice_virtchnl_set_dflt_ops(vf);
-}
-
-/**
- * ice_repr_rem_from_all_vfs - remove port representor for all VFs
- * @pf: pointer to PF structure
- */
-void ice_repr_rem_from_all_vfs(struct ice_pf *pf)
-{
-	struct devlink *devlink;
-	struct ice_vf *vf;
-	unsigned int bkt;
-
-	lockdep_assert_held(&pf->vfs.table_lock);
-
-	ice_for_each_vf(pf, bkt, vf)
-		ice_repr_rem(vf);
-
-	/* since all port representors are destroyed, there is
-	 * no point in keeping the nodes
-	 */
-	devlink = priv_to_devlink(pf);
-	devl_lock(devlink);
-	devl_rate_nodes_destroy(devlink);
-	devl_unlock(devlink);
+	return ERR_PTR(err);
 }
 
-/**
- * ice_repr_add_for_all_vfs - add port representor for all VFs
- * @pf: pointer to PF structure
- */
-int ice_repr_add_for_all_vfs(struct ice_pf *pf)
+struct ice_repr *ice_repr_get_by_vsi(struct ice_vsi *vsi)
 {
-	struct devlink *devlink;
-	struct ice_vf *vf;
-	unsigned int bkt;
-	int err;
-
-	lockdep_assert_held(&pf->vfs.table_lock);
-
-	ice_for_each_vf(pf, bkt, vf) {
-		err = ice_repr_add(vf);
-		if (err)
-			goto err;
-	}
-
-	/* only export if ADQ and DCB disabled */
-	if (ice_is_adq_active(pf) || ice_is_dcb_active(pf))
-		return 0;
-
-	devlink = priv_to_devlink(pf);
-	ice_devlink_rate_init_tx_topology(devlink, ice_get_main_vsi(pf));
-
-	return 0;
-
-err:
-	ice_repr_rem_from_all_vfs(pf);
+	if (!vsi->vf)
+		return NULL;
 
-	return err;
+	return xa_load(&vsi->back->eswitch.reprs, vsi->vf->repr_id);
 }
 
 /**
diff --git a/drivers/net/ethernet/intel/ice/ice_repr.h b/drivers/net/ethernet/intel/ice/ice_repr.h
index e1ee2d2c1d2d..f9aede315716 100644
--- a/drivers/net/ethernet/intel/ice/ice_repr.h
+++ b/drivers/net/ethernet/intel/ice/ice_repr.h
@@ -13,14 +13,17 @@ struct ice_repr {
 	struct net_device *netdev;
 	struct metadata_dst *dst;
 	struct ice_esw_br_port *br_port;
+	int q_id;
+	u32 id;
+	u8 parent_mac[ETH_ALEN];
 #ifdef CONFIG_ICE_SWITCHDEV
 	/* info about slow path rule */
 	struct ice_rule_query_data sp_rule;
 #endif
 };
 
-int ice_repr_add_for_all_vfs(struct ice_pf *pf);
-void ice_repr_rem_from_all_vfs(struct ice_pf *pf);
+struct ice_repr *ice_repr_add_vf(struct ice_vf *vf);
+void ice_repr_rem_vf(struct ice_repr *repr);
 
 void ice_repr_start_tx_queues(struct ice_repr *repr);
 void ice_repr_stop_tx_queues(struct ice_repr *repr);
@@ -29,4 +32,6 @@ void ice_repr_set_traffic_vsi(struct ice_repr *repr, struct ice_vsi *vsi);
 
 struct ice_repr *ice_netdev_to_repr(struct net_device *netdev);
 bool ice_is_port_repr_netdev(const struct net_device *netdev);
+
+struct ice_repr *ice_repr_get_by_vsi(struct ice_vsi *vsi);
 #endif
diff --git a/drivers/net/ethernet/intel/ice/ice_sriov.c b/drivers/net/ethernet/intel/ice/ice_sriov.c
index 2a5e6616cc0a..5a45bd5ce6ad 100644
--- a/drivers/net/ethernet/intel/ice/ice_sriov.c
+++ b/drivers/net/ethernet/intel/ice/ice_sriov.c
@@ -172,13 +172,14 @@ void ice_free_vfs(struct ice_pf *pf)
 	else
 		dev_warn(dev, "VFs are assigned - not disabling SR-IOV\n");
 
-	mutex_lock(&vfs->table_lock);
+	ice_eswitch_reserve_cp_queues(pf, -ice_get_num_vfs(pf));
 
-	ice_eswitch_release(pf);
+	mutex_lock(&vfs->table_lock);
 
 	ice_for_each_vf(pf, bkt, vf) {
 		mutex_lock(&vf->cfg_lock);
 
+		ice_eswitch_detach(pf, vf);
 		ice_dis_vf_qs(vf);
 
 		if (test_bit(ICE_VF_STATE_INIT, vf->vf_states)) {
@@ -614,6 +615,14 @@ static int ice_start_vfs(struct ice_pf *pf)
 			goto teardown;
 		}
 
+		retval = ice_eswitch_attach(pf, vf);
+		if (retval) {
+			dev_err(ice_pf_to_dev(pf), "Failed to attach VF %d to eswitch, error %d",
+				vf->vf_id, retval);
+			ice_vf_vsi_release(vf);
+			goto teardown;
+		}
+
 		set_bit(ICE_VF_STATE_INIT, vf->vf_states);
 		ice_ena_vf_mappings(vf);
 		wr32(hw, VFGEN_RSTAT(vf->vf_id), VIRTCHNL_VFR_VFACTIVE);
@@ -923,6 +932,7 @@ static int ice_ena_vfs(struct ice_pf *pf, u16 num_vfs)
 		goto err_unroll_sriov;
 	}
 
+	ice_eswitch_reserve_cp_queues(pf, num_vfs);
 	ret = ice_start_vfs(pf);
 	if (ret) {
 		dev_err(dev, "Failed to start %d VFs, err %d\n", num_vfs, ret);
@@ -932,12 +942,6 @@ static int ice_ena_vfs(struct ice_pf *pf, u16 num_vfs)
 
 	clear_bit(ICE_VF_DIS, pf->state);
 
-	ret = ice_eswitch_configure(pf);
-	if (ret) {
-		dev_err(dev, "Failed to configure eswitch, err %d\n", ret);
-		goto err_unroll_sriov;
-	}
-
 	/* rearm global interrupts */
 	if (test_and_clear_bit(ICE_OICR_INTR_DIS, pf->state))
 		ice_irq_dynamic_ena(hw, NULL, NULL);
diff --git a/drivers/net/ethernet/intel/ice/ice_tc_lib.c b/drivers/net/ethernet/intel/ice/ice_tc_lib.c
index dd03cb69ad26..08d3bbf4b44c 100644
--- a/drivers/net/ethernet/intel/ice/ice_tc_lib.c
+++ b/drivers/net/ethernet/intel/ice/ice_tc_lib.c
@@ -653,7 +653,7 @@ static int ice_tc_setup_redirect_action(struct net_device *filter_dev,
 		   ice_tc_is_dev_uplink(target_dev)) {
 		repr = ice_netdev_to_repr(filter_dev);
 
-		fltr->dest_vsi = repr->src_vsi->back->switchdev.uplink_vsi;
+		fltr->dest_vsi = repr->src_vsi->back->eswitch.uplink_vsi;
 		fltr->direction = ICE_ESWITCH_FLTR_EGRESS;
 	} else if (ice_tc_is_dev_uplink(filter_dev) &&
 		   ice_is_port_repr_netdev(target_dev)) {
@@ -765,7 +765,7 @@ ice_eswitch_add_tc_fltr(struct ice_vsi *vsi, struct ice_tc_flower_fltr *fltr)
 		rule_info.sw_act.src = hw->pf_id;
 		rule_info.flags_info.act = ICE_SINGLE_ACT_LB_ENABLE;
 	} else if (fltr->direction == ICE_ESWITCH_FLTR_EGRESS &&
-		   fltr->dest_vsi == vsi->back->switchdev.uplink_vsi) {
+		   fltr->dest_vsi == vsi->back->eswitch.uplink_vsi) {
 		/* VF to Uplink */
 		rule_info.sw_act.flag |= ICE_FLTR_TX;
 		rule_info.sw_act.src = vsi->idx;
diff --git a/drivers/net/ethernet/intel/ice/ice_vf_lib.c b/drivers/net/ethernet/intel/ice/ice_vf_lib.c
index aca1f2ea5034..d2a99a20c4ad 100644
--- a/drivers/net/ethernet/intel/ice/ice_vf_lib.c
+++ b/drivers/net/ethernet/intel/ice/ice_vf_lib.c
@@ -760,6 +760,7 @@ void ice_reset_all_vfs(struct ice_pf *pf)
 	ice_for_each_vf(pf, bkt, vf) {
 		mutex_lock(&vf->cfg_lock);
 
+		ice_eswitch_detach(pf, vf);
 		vf->driver_caps = 0;
 		ice_vc_set_default_allowlist(vf);
 
@@ -775,13 +776,11 @@ void ice_reset_all_vfs(struct ice_pf *pf)
 		ice_vf_rebuild_vsi(vf);
 		ice_vf_post_vsi_rebuild(vf);
 
+		ice_eswitch_attach(pf, vf);
+
 		mutex_unlock(&vf->cfg_lock);
 	}
 
-	if (ice_is_eswitch_mode_switchdev(pf))
-		if (ice_eswitch_rebuild(pf))
-			dev_warn(dev, "eswitch rebuild failed\n");
-
 	ice_flush(hw);
 	clear_bit(ICE_VF_DIS, pf->state);
 
@@ -928,7 +927,7 @@ int ice_reset_vf(struct ice_vf *vf, u32 flags)
 		goto out_unlock;
 	}
 
-	ice_eswitch_update_repr(vsi);
+	ice_eswitch_update_repr(vf->repr_id, vsi);
 
 	/* if the VF has been reset allow it to come up again */
 	ice_mbx_clear_malvf(&vf->mbx_info);
diff --git a/drivers/net/ethernet/intel/ice/ice_vf_lib.h b/drivers/net/ethernet/intel/ice/ice_vf_lib.h
index 93c774f2f437..35866553f288 100644
--- a/drivers/net/ethernet/intel/ice/ice_vf_lib.h
+++ b/drivers/net/ethernet/intel/ice/ice_vf_lib.h
@@ -130,7 +130,7 @@ struct ice_vf {
 	struct ice_mdd_vf_events mdd_tx_events;
 	DECLARE_BITMAP(opcodes_allowlist, VIRTCHNL_OP_MAX);
 
-	struct ice_repr *repr;
+	unsigned long repr_id;
 	const struct ice_virtchnl_ops *virtchnl_ops;
 	const struct ice_vf_ops *vf_ops;
 
diff --git a/drivers/net/ethernet/intel/igc/igc.h b/drivers/net/ethernet/intel/igc/igc.h
index f48f82d5e274..ac7c861e83a0 100644
--- a/drivers/net/ethernet/intel/igc/igc.h
+++ b/drivers/net/ethernet/intel/igc/igc.h
@@ -81,6 +81,21 @@ struct igc_tx_timestamp_request {
 	u32 flags;             /* flags that should be added to the tx_buffer */
 };
 
+struct igc_inline_rx_tstamps {
+	/* Timestamps are saved in little endian at the beginning of the packet
+	 * buffer following the layout:
+	 *
+	 * DWORD: | 0              | 1              | 2              | 3              |
+	 * Field: | Timer1 SYSTIML | Timer1 SYSTIMH | Timer0 SYSTIML | Timer0 SYSTIMH |
+	 *
+	 * SYSTIML holds the nanoseconds part while SYSTIMH holds the seconds
+	 * part of the timestamp.
+	 *
+	 */
+	__le32 timer1[2];
+	__le32 timer0[2];
+};
+
 struct igc_ring_container {
 	struct igc_ring *ring;          /* pointer to linked list of rings */
 	unsigned int total_bytes;       /* total bytes processed this int */
@@ -261,6 +276,8 @@ struct igc_adapter {
 	unsigned int ptp_flags;
 	/* System time value lock */
 	spinlock_t tmreg_lock;
+	/* Free-running timer lock */
+	spinlock_t free_timer_lock;
 	struct cyclecounter cc;
 	struct timecounter tc;
 	struct timespec64 prev_ptp_time; /* Pre-reset PTP clock */
@@ -469,6 +486,8 @@ enum igc_tx_flags {
 	IGC_TX_FLAGS_TSTAMP_1	= 0x100,
 	IGC_TX_FLAGS_TSTAMP_2	= 0x200,
 	IGC_TX_FLAGS_TSTAMP_3	= 0x400,
+
+	IGC_TX_FLAGS_TSTAMP_TIMER_1 = 0x800,
 };
 
 enum igc_boards {
@@ -531,7 +550,7 @@ struct igc_rx_buffer {
 struct igc_xdp_buff {
 	struct xdp_buff xdp;
 	union igc_adv_rx_desc *rx_desc;
-	ktime_t rx_ts; /* data indication bit IGC_RXDADV_STAT_TSIP */
+	struct igc_inline_rx_tstamps *rx_ts; /* data indication bit IGC_RXDADV_STAT_TSIP */
 };
 
 struct igc_q_vector {
diff --git a/drivers/net/ethernet/intel/igc/igc_base.h b/drivers/net/ethernet/intel/igc/igc_base.h
index f7d6491d4c60..bf8cdfbba9ff 100644
--- a/drivers/net/ethernet/intel/igc/igc_base.h
+++ b/drivers/net/ethernet/intel/igc/igc_base.h
@@ -37,6 +37,10 @@ struct igc_adv_tx_context_desc {
 #define IGC_ADVTXD_TSTAMP_REG_1	0x00010000 /* Select register 1 for timestamp */
 #define IGC_ADVTXD_TSTAMP_REG_2	0x00020000 /* Select register 2 for timestamp */
 #define IGC_ADVTXD_TSTAMP_REG_3	0x00030000 /* Select register 3 for timestamp */
+#define IGC_ADVTXD_TSTAMP_TIMER_1	0x00010000 /* Select timer 1 for timestamp */
+#define IGC_ADVTXD_TSTAMP_TIMER_2	0x00020000 /* Select timer 2 for timestamp */
+#define IGC_ADVTXD_TSTAMP_TIMER_3	0x00030000 /* Select timer 3 for timestamp */
+
 #define IGC_ADVTXD_DTYP_CTXT	0x00200000 /* Advanced Context Descriptor */
 #define IGC_ADVTXD_DTYP_DATA	0x00300000 /* Advanced Data Descriptor */
 #define IGC_ADVTXD_DCMD_EOP	0x01000000 /* End of Packet */
diff --git a/drivers/net/ethernet/intel/igc/igc_defines.h b/drivers/net/ethernet/intel/igc/igc_defines.h
index b3037016f31d..5f92b3c7c3d4 100644
--- a/drivers/net/ethernet/intel/igc/igc_defines.h
+++ b/drivers/net/ethernet/intel/igc/igc_defines.h
@@ -317,6 +317,8 @@
 #define IGC_TXD_CMD_TSE		0x04000000 /* TCP Seg enable */
 #define IGC_TXD_EXTCMD_TSTAMP	0x00000010 /* IEEE1588 Timestamp packet */
 
+#define IGC_TXD_PTP2_TIMER_1	0x00000020
+
 /* IPSec Encrypt Enable */
 #define IGC_ADVTXD_L4LEN_SHIFT	8  /* Adv ctxt L4LEN shift */
 #define IGC_ADVTXD_MSS_SHIFT	16 /* Adv ctxt MSS shift */
diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c
index e9bb403bbacf..61db1d3bfa0b 100644
--- a/drivers/net/ethernet/intel/igc/igc_main.c
+++ b/drivers/net/ethernet/intel/igc/igc_main.c
@@ -1299,14 +1299,16 @@ static void igc_tx_olinfo_status(struct igc_ring *tx_ring,
 	u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT;
 
 	/* insert L4 checksum */
-	olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) *
-			  ((IGC_TXD_POPTS_TXSM << 8) /
-			  IGC_TX_FLAGS_CSUM);
+	olinfo_status |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_CSUM,
+				      (IGC_TXD_POPTS_TXSM << 8));
 
 	/* insert IPv4 checksum */
-	olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) *
-			  (((IGC_TXD_POPTS_IXSM << 8)) /
-			  IGC_TX_FLAGS_IPV4);
+	olinfo_status |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_IPV4,
+				      (IGC_TXD_POPTS_IXSM << 8));
+
+	/* Use the second timer (free running, in general) for the timestamp */
+	olinfo_status |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_TIMER_1,
+				      IGC_TXD_PTP2_TIMER_1);
 
 	tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
 }
@@ -1651,6 +1653,8 @@ static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb,
 		if (igc_request_tx_tstamp(adapter, skb, &tstamp_flags)) {
 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
 			tx_flags |= IGC_TX_FLAGS_TSTAMP | tstamp_flags;
+			if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP_USE_CYCLES)
+				tx_flags |= IGC_TX_FLAGS_TSTAMP_TIMER_1;
 		} else {
 			adapter->tx_hwtstamp_skipped++;
 		}
@@ -1963,9 +1967,9 @@ static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring,
 
 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring,
 					 struct igc_rx_buffer *rx_buffer,
-					 struct xdp_buff *xdp,
-					 ktime_t timestamp)
+					 struct igc_xdp_buff *ctx)
 {
+	struct xdp_buff *xdp = &ctx->xdp;
 	unsigned int metasize = xdp->data - xdp->data_meta;
 	unsigned int size = xdp->data_end - xdp->data;
 	unsigned int truesize = igc_get_rx_frame_truesize(rx_ring, size);
@@ -1982,8 +1986,10 @@ static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring,
 	if (unlikely(!skb))
 		return NULL;
 
-	if (timestamp)
-		skb_hwtstamps(skb)->hwtstamp = timestamp;
+	if (ctx->rx_ts) {
+		skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP_NETDEV;
+		skb_hwtstamps(skb)->netdev_data = ctx->rx_ts;
+	}
 
 	/* Determine available headroom for copy */
 	headlen = size;
@@ -2583,11 +2589,10 @@ static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget)
 	int xdp_status = 0, rx_buffer_pgcnt;
 
 	while (likely(total_packets < budget)) {
-		union igc_adv_rx_desc *rx_desc;
+		struct igc_xdp_buff ctx = { .rx_ts = NULL };
 		struct igc_rx_buffer *rx_buffer;
+		union igc_adv_rx_desc *rx_desc;
 		unsigned int size, truesize;
-		struct igc_xdp_buff ctx;
-		ktime_t timestamp = 0;
 		int pkt_offset = 0;
 		void *pktbuf;
 
@@ -2614,9 +2619,7 @@ static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget)
 		pktbuf = page_address(rx_buffer->page) + rx_buffer->page_offset;
 
 		if (igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP)) {
-			timestamp = igc_ptp_rx_pktstamp(q_vector->adapter,
-							pktbuf);
-			ctx.rx_ts = timestamp;
+			ctx.rx_ts = pktbuf;
 			pkt_offset = IGC_TS_HDR_LEN;
 			size -= IGC_TS_HDR_LEN;
 		}
@@ -2653,8 +2656,7 @@ static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget)
 		else if (ring_uses_build_skb(rx_ring))
 			skb = igc_build_skb(rx_ring, rx_buffer, &ctx.xdp);
 		else
-			skb = igc_construct_skb(rx_ring, rx_buffer, &ctx.xdp,
-						timestamp);
+			skb = igc_construct_skb(rx_ring, rx_buffer, &ctx);
 
 		/* exit if we failed to retrieve a buffer */
 		if (!skb) {
@@ -2803,9 +2805,7 @@ static int igc_clean_rx_irq_zc(struct igc_q_vector *q_vector, const int budget)
 		ctx->rx_desc = desc;
 
 		if (igc_test_staterr(desc, IGC_RXDADV_STAT_TSIP)) {
-			timestamp = igc_ptp_rx_pktstamp(q_vector->adapter,
-							bi->xdp->data);
-			ctx->rx_ts = timestamp;
+			ctx->rx_ts = bi->xdp->data;
 
 			bi->xdp->data += IGC_TS_HDR_LEN;
 
@@ -6562,6 +6562,24 @@ int igc_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags)
 	return 0;
 }
 
+static ktime_t igc_get_tstamp(struct net_device *dev,
+			      const struct skb_shared_hwtstamps *hwtstamps,
+			      bool cycles)
+{
+	struct igc_adapter *adapter = netdev_priv(dev);
+	struct igc_inline_rx_tstamps *tstamp;
+	ktime_t timestamp;
+
+	tstamp = hwtstamps->netdev_data;
+
+	if (cycles)
+		timestamp = igc_ptp_rx_pktstamp(adapter, tstamp->timer1);
+	else
+		timestamp = igc_ptp_rx_pktstamp(adapter, tstamp->timer0);
+
+	return timestamp;
+}
+
 static const struct net_device_ops igc_netdev_ops = {
 	.ndo_open		= igc_open,
 	.ndo_stop		= igc_close,
@@ -6579,6 +6597,7 @@ static const struct net_device_ops igc_netdev_ops = {
 	.ndo_bpf		= igc_bpf,
 	.ndo_xdp_xmit		= igc_xdp_xmit,
 	.ndo_xsk_wakeup		= igc_xsk_wakeup,
+	.ndo_get_tstamp		= igc_get_tstamp,
 };
 
 /* PCIe configuration access */
@@ -6682,9 +6701,11 @@ static int igc_xdp_rx_hash(const struct xdp_md *_ctx, u32 *hash,
 static int igc_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp)
 {
 	const struct igc_xdp_buff *ctx = (void *)_ctx;
+	struct igc_adapter *adapter = netdev_priv(ctx->xdp.rxq->dev);
+	struct igc_inline_rx_tstamps *tstamp = ctx->rx_ts;
 
 	if (igc_test_staterr(ctx->rx_desc, IGC_RXDADV_STAT_TSIP)) {
-		*timestamp = ctx->rx_ts;
+		*timestamp = igc_ptp_rx_pktstamp(adapter, tstamp->timer0);
 
 		return 0;
 	}
diff --git a/drivers/net/ethernet/intel/igc/igc_ptp.c b/drivers/net/ethernet/intel/igc/igc_ptp.c
index 928f38792203..885faaa7b9de 100644
--- a/drivers/net/ethernet/intel/igc/igc_ptp.c
+++ b/drivers/net/ethernet/intel/igc/igc_ptp.c
@@ -459,12 +459,10 @@ static int igc_ptp_systim_to_hwtstamp(struct igc_adapter *adapter,
 /**
  * igc_ptp_rx_pktstamp - Retrieve timestamp from Rx packet buffer
  * @adapter: Pointer to adapter the packet buffer belongs to
- * @buf: Pointer to packet buffer
+ * @buf: Pointer to start of timestamp in HW format (2 32-bit words)
  *
- * This function retrieves the timestamp saved in the beginning of packet
- * buffer. While two timestamps are available, one in timer0 reference and the
- * other in timer1 reference, this function considers only the timestamp in
- * timer0 reference.
+ * This function retrieves and converts the timestamp stored at @buf
+ * to ktime_t, adjusting for hardware latencies.
  *
  * Returns timestamp value.
  */
@@ -474,17 +472,8 @@ ktime_t igc_ptp_rx_pktstamp(struct igc_adapter *adapter, __le32 *buf)
 	u32 secs, nsecs;
 	int adjust;
 
-	/* Timestamps are saved in little endian at the beginning of the packet
-	 * buffer following the layout:
-	 *
-	 * DWORD: | 0              | 1              | 2              | 3              |
-	 * Field: | Timer1 SYSTIML | Timer1 SYSTIMH | Timer0 SYSTIML | Timer0 SYSTIMH |
-	 *
-	 * SYSTIML holds the nanoseconds part while SYSTIMH holds the seconds
-	 * part of the timestamp.
-	 */
-	nsecs = le32_to_cpu(buf[2]);
-	secs = le32_to_cpu(buf[3]);
+	nsecs = le32_to_cpu(buf[0]);
+	secs = le32_to_cpu(buf[1]);
 
 	timestamp = ktime_set(secs, nsecs);
 
@@ -542,10 +531,11 @@ static void igc_ptp_enable_rx_timestamp(struct igc_adapter *adapter)
 
 	for (i = 0; i < adapter->num_rx_queues; i++) {
 		val = rd32(IGC_SRRCTL(i));
-		/* FIXME: For now, only support retrieving RX timestamps from
-		 * timer 0.
+		/* Enable retrieving timestamps from timer 0, the
+		 * "adjustable clock" and timer 1 the "free running
+		 * clock".
 		 */
-		val |= IGC_SRRCTL_TIMER1SEL(0) | IGC_SRRCTL_TIMER0SEL(0) |
+		val |= IGC_SRRCTL_TIMER1SEL(1) | IGC_SRRCTL_TIMER0SEL(0) |
 		       IGC_SRRCTL_TIMESTAMP;
 		wr32(IGC_SRRCTL(i), val);
 	}
@@ -1035,6 +1025,26 @@ static int igc_ptp_getcrosststamp(struct ptp_clock_info *ptp,
 					     adapter, &adapter->snapshot, cts);
 }
 
+static int igc_ptp_getcyclesx64(struct ptp_clock_info *ptp,
+				struct timespec64 *ts,
+				struct ptp_system_timestamp *sts)
+{
+	struct igc_adapter *igc = container_of(ptp, struct igc_adapter, ptp_caps);
+	struct igc_hw *hw = &igc->hw;
+	unsigned long flags;
+
+	spin_lock_irqsave(&igc->free_timer_lock, flags);
+
+	ptp_read_system_prets(sts);
+	ts->tv_nsec = rd32(IGC_SYSTIML_1);
+	ts->tv_sec = rd32(IGC_SYSTIMH_1);
+	ptp_read_system_postts(sts);
+
+	spin_unlock_irqrestore(&igc->free_timer_lock, flags);
+
+	return 0;
+}
+
 /**
  * igc_ptp_init - Initialize PTP functionality
  * @adapter: Board private structure
@@ -1088,6 +1098,7 @@ void igc_ptp_init(struct igc_adapter *adapter)
 		adapter->ptp_caps.adjfine = igc_ptp_adjfine_i225;
 		adapter->ptp_caps.adjtime = igc_ptp_adjtime_i225;
 		adapter->ptp_caps.gettimex64 = igc_ptp_gettimex64_i225;
+		adapter->ptp_caps.getcyclesx64 = igc_ptp_getcyclesx64;
 		adapter->ptp_caps.settime64 = igc_ptp_settime_i225;
 		adapter->ptp_caps.enable = igc_ptp_feature_enable_i225;
 		adapter->ptp_caps.pps = 1;
@@ -1108,6 +1119,7 @@ void igc_ptp_init(struct igc_adapter *adapter)
 	}
 
 	spin_lock_init(&adapter->ptp_tx_lock);
+	spin_lock_init(&adapter->free_timer_lock);
 	spin_lock_init(&adapter->tmreg_lock);
 
 	adapter->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
diff --git a/drivers/net/ethernet/intel/igc/igc_regs.h b/drivers/net/ethernet/intel/igc/igc_regs.h
index 20e17f5fbce3..d38c87d7e5e8 100644
--- a/drivers/net/ethernet/intel/igc/igc_regs.h
+++ b/drivers/net/ethernet/intel/igc/igc_regs.h
@@ -243,6 +243,11 @@
 #define IGC_SYSTIMR	0x0B6F8  /* System time register Residue */
 #define IGC_TIMINCA	0x0B608  /* Increment attributes register - RW */
 
+#define IGC_SYSTIML_1	0x0B688  /* System time register Low - RO (timer 1) */
+#define IGC_SYSTIMH_1	0x0B68C  /* System time register High - RO (timer 1) */
+#define IGC_SYSTIMR_1	0x0B684  /* System time register Residue (timer 1) */
+#define IGC_TIMINCA_1	0x0B690  /* Increment attributes register - RW (timer 1) */
+
 /* TX Timestamp Low */
 #define IGC_TXSTMPL_0		0x0B618
 #define IGC_TXSTMPL_1		0x0B698
diff --git a/drivers/net/ethernet/marvell/octeon_ep/Makefile b/drivers/net/ethernet/marvell/octeon_ep/Makefile
index 2026c8118158..02a4a21bc298 100644
--- a/drivers/net/ethernet/marvell/octeon_ep/Makefile
+++ b/drivers/net/ethernet/marvell/octeon_ep/Makefile
@@ -6,4 +6,5 @@
 obj-$(CONFIG_OCTEON_EP) += octeon_ep.o
 
 octeon_ep-y := octep_main.o octep_cn9k_pf.o octep_tx.o octep_rx.o \
-	       octep_ethtool.o octep_ctrl_mbox.o octep_ctrl_net.o
+	       octep_ethtool.o octep_ctrl_mbox.o octep_ctrl_net.o \
+	       octep_cnxk_pf.o
diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c b/drivers/net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c
new file mode 100644
index 000000000000..abb03e9119e7
--- /dev/null
+++ b/drivers/net/ethernet/marvell/octeon_ep/octep_cnxk_pf.c
@@ -0,0 +1,886 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Marvell Octeon EP (EndPoint) Ethernet Driver
+ *
+ * Copyright (C) 2020 Marvell.
+ *
+ */
+
+#include <linux/pci.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+
+#include "octep_config.h"
+#include "octep_main.h"
+#include "octep_regs_cnxk_pf.h"
+
+/* We will support 128 pf's in control mbox */
+#define CTRL_MBOX_MAX_PF	128
+#define CTRL_MBOX_SZ		((size_t)(0x400000 / CTRL_MBOX_MAX_PF))
+
+/* Names of Hardware non-queue generic interrupts */
+static char *cnxk_non_ioq_msix_names[] = {
+	"epf_ire_rint",
+	"epf_ore_rint",
+	"epf_vfire_rint",
+	"epf_rsvd0",
+	"epf_vfore_rint",
+	"epf_rsvd1",
+	"epf_mbox_rint",
+	"epf_rsvd2_0",
+	"epf_rsvd2_1",
+	"epf_dma_rint",
+	"epf_dma_vf_rint",
+	"epf_rsvd3",
+	"epf_pp_vf_rint",
+	"epf_rsvd3",
+	"epf_misc_rint",
+	"epf_rsvd5",
+	/* Next 16 are for OEI_RINT */
+	"epf_oei_rint0",
+	"epf_oei_rint1",
+	"epf_oei_rint2",
+	"epf_oei_rint3",
+	"epf_oei_rint4",
+	"epf_oei_rint5",
+	"epf_oei_rint6",
+	"epf_oei_rint7",
+	"epf_oei_rint8",
+	"epf_oei_rint9",
+	"epf_oei_rint10",
+	"epf_oei_rint11",
+	"epf_oei_rint12",
+	"epf_oei_rint13",
+	"epf_oei_rint14",
+	"epf_oei_rint15",
+	/* IOQ interrupt */
+	"octeon_ep"
+};
+
+/* Dump useful hardware CSRs for debug purpose */
+static void cnxk_dump_regs(struct octep_device *oct, int qno)
+{
+	struct device *dev = &oct->pdev->dev;
+
+	dev_info(dev, "IQ-%d register dump\n", qno);
+	dev_info(dev, "R[%d]_IN_INSTR_DBELL[0x%llx]: 0x%016llx\n",
+		 qno, CNXK_SDP_R_IN_INSTR_DBELL(qno),
+		 octep_read_csr64(oct, CNXK_SDP_R_IN_INSTR_DBELL(qno)));
+	dev_info(dev, "R[%d]_IN_CONTROL[0x%llx]: 0x%016llx\n",
+		 qno, CNXK_SDP_R_IN_CONTROL(qno),
+		 octep_read_csr64(oct, CNXK_SDP_R_IN_CONTROL(qno)));
+	dev_info(dev, "R[%d]_IN_ENABLE[0x%llx]: 0x%016llx\n",
+		 qno, CNXK_SDP_R_IN_ENABLE(qno),
+		 octep_read_csr64(oct, CNXK_SDP_R_IN_ENABLE(qno)));
+	dev_info(dev, "R[%d]_IN_INSTR_BADDR[0x%llx]: 0x%016llx\n",
+		 qno, CNXK_SDP_R_IN_INSTR_BADDR(qno),
+		 octep_read_csr64(oct, CNXK_SDP_R_IN_INSTR_BADDR(qno)));
+	dev_info(dev, "R[%d]_IN_INSTR_RSIZE[0x%llx]: 0x%016llx\n",
+		 qno, CNXK_SDP_R_IN_INSTR_RSIZE(qno),
+		 octep_read_csr64(oct, CNXK_SDP_R_IN_INSTR_RSIZE(qno)));
+	dev_info(dev, "R[%d]_IN_CNTS[0x%llx]: 0x%016llx\n",
+		 qno, CNXK_SDP_R_IN_CNTS(qno),
+		 octep_read_csr64(oct, CNXK_SDP_R_IN_CNTS(qno)));
+	dev_info(dev, "R[%d]_IN_INT_LEVELS[0x%llx]: 0x%016llx\n",
+		 qno, CNXK_SDP_R_IN_INT_LEVELS(qno),
+		 octep_read_csr64(oct, CNXK_SDP_R_IN_INT_LEVELS(qno)));
+	dev_info(dev, "R[%d]_IN_PKT_CNT[0x%llx]: 0x%016llx\n",
+		 qno, CNXK_SDP_R_IN_PKT_CNT(qno),
+		 octep_read_csr64(oct, CNXK_SDP_R_IN_PKT_CNT(qno)));
+	dev_info(dev, "R[%d]_IN_BYTE_CNT[0x%llx]: 0x%016llx\n",
+		 qno, CNXK_SDP_R_IN_BYTE_CNT(qno),
+		 octep_read_csr64(oct, CNXK_SDP_R_IN_BYTE_CNT(qno)));
+
+	dev_info(dev, "OQ-%d register dump\n", qno);
+	dev_info(dev, "R[%d]_OUT_SLIST_DBELL[0x%llx]: 0x%016llx\n",
+		 qno, CNXK_SDP_R_OUT_SLIST_DBELL(qno),
+		 octep_read_csr64(oct, CNXK_SDP_R_OUT_SLIST_DBELL(qno)));
+	dev_info(dev, "R[%d]_OUT_CONTROL[0x%llx]: 0x%016llx\n",
+		 qno, CNXK_SDP_R_OUT_CONTROL(qno),
+		 octep_read_csr64(oct, CNXK_SDP_R_OUT_CONTROL(qno)));
+	dev_info(dev, "R[%d]_OUT_ENABLE[0x%llx]: 0x%016llx\n",
+		 qno, CNXK_SDP_R_OUT_ENABLE(qno),
+		 octep_read_csr64(oct, CNXK_SDP_R_OUT_ENABLE(qno)));
+	dev_info(dev, "R[%d]_OUT_SLIST_BADDR[0x%llx]: 0x%016llx\n",
+		 qno, CNXK_SDP_R_OUT_SLIST_BADDR(qno),
+		 octep_read_csr64(oct, CNXK_SDP_R_OUT_SLIST_BADDR(qno)));
+	dev_info(dev, "R[%d]_OUT_SLIST_RSIZE[0x%llx]: 0x%016llx\n",
+		 qno, CNXK_SDP_R_OUT_SLIST_RSIZE(qno),
+		 octep_read_csr64(oct, CNXK_SDP_R_OUT_SLIST_RSIZE(qno)));
+	dev_info(dev, "R[%d]_OUT_CNTS[0x%llx]: 0x%016llx\n",
+		 qno, CNXK_SDP_R_OUT_CNTS(qno),
+		 octep_read_csr64(oct, CNXK_SDP_R_OUT_CNTS(qno)));
+	dev_info(dev, "R[%d]_OUT_INT_LEVELS[0x%llx]: 0x%016llx\n",
+		 qno, CNXK_SDP_R_OUT_INT_LEVELS(qno),
+		 octep_read_csr64(oct, CNXK_SDP_R_OUT_INT_LEVELS(qno)));
+	dev_info(dev, "R[%d]_OUT_PKT_CNT[0x%llx]: 0x%016llx\n",
+		 qno, CNXK_SDP_R_OUT_PKT_CNT(qno),
+		 octep_read_csr64(oct, CNXK_SDP_R_OUT_PKT_CNT(qno)));
+	dev_info(dev, "R[%d]_OUT_BYTE_CNT[0x%llx]: 0x%016llx\n",
+		 qno, CNXK_SDP_R_OUT_BYTE_CNT(qno),
+		 octep_read_csr64(oct, CNXK_SDP_R_OUT_BYTE_CNT(qno)));
+	dev_info(dev, "R[%d]_ERR_TYPE[0x%llx]: 0x%016llx\n",
+		 qno, CNXK_SDP_R_ERR_TYPE(qno),
+		 octep_read_csr64(oct, CNXK_SDP_R_ERR_TYPE(qno)));
+}
+
+/* Reset Hardware Tx queue */
+static int cnxk_reset_iq(struct octep_device *oct, int q_no)
+{
+	struct octep_config *conf = oct->conf;
+	u64 val = 0ULL;
+
+	dev_dbg(&oct->pdev->dev, "Reset PF IQ-%d\n", q_no);
+
+	/* Get absolute queue number */
+	q_no += conf->pf_ring_cfg.srn;
+
+	/* Disable the Tx/Instruction Ring */
+	octep_write_csr64(oct, CNXK_SDP_R_IN_ENABLE(q_no), val);
+
+	/* clear the Instruction Ring packet/byte counts and doorbell CSRs */
+	octep_write_csr64(oct, CNXK_SDP_R_IN_CNTS(q_no), val);
+	octep_write_csr64(oct, CNXK_SDP_R_IN_INT_LEVELS(q_no), val);
+	octep_write_csr64(oct, CNXK_SDP_R_IN_PKT_CNT(q_no), val);
+	octep_write_csr64(oct, CNXK_SDP_R_IN_BYTE_CNT(q_no), val);
+	octep_write_csr64(oct, CNXK_SDP_R_IN_INSTR_BADDR(q_no), val);
+	octep_write_csr64(oct, CNXK_SDP_R_IN_INSTR_RSIZE(q_no), val);
+
+	val = 0xFFFFFFFF;
+	octep_write_csr64(oct, CNXK_SDP_R_IN_INSTR_DBELL(q_no), val);
+
+	return 0;
+}
+
+/* Reset Hardware Rx queue */
+static void cnxk_reset_oq(struct octep_device *oct, int q_no)
+{
+	u64 val = 0ULL;
+
+	q_no += CFG_GET_PORTS_PF_SRN(oct->conf);
+
+	/* Disable Output (Rx) Ring */
+	octep_write_csr64(oct, CNXK_SDP_R_OUT_ENABLE(q_no), val);
+	octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_BADDR(q_no), val);
+	octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_RSIZE(q_no), val);
+	octep_write_csr64(oct, CNXK_SDP_R_OUT_INT_LEVELS(q_no), val);
+
+	/* Clear count CSRs */
+	val = octep_read_csr(oct, CNXK_SDP_R_OUT_CNTS(q_no));
+	octep_write_csr(oct, CNXK_SDP_R_OUT_CNTS(q_no), val);
+
+	octep_write_csr64(oct, CNXK_SDP_R_OUT_PKT_CNT(q_no), 0xFFFFFFFFFULL);
+	octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_DBELL(q_no), 0xFFFFFFFF);
+}
+
+/* Reset all hardware Tx/Rx queues */
+static void octep_reset_io_queues_cnxk_pf(struct octep_device *oct)
+{
+	struct pci_dev *pdev = oct->pdev;
+	int q;
+
+	dev_dbg(&pdev->dev, "Reset OCTEP_CNXK PF IO Queues\n");
+
+	for (q = 0; q < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); q++) {
+		cnxk_reset_iq(oct, q);
+		cnxk_reset_oq(oct, q);
+	}
+}
+
+/* Initialize windowed addresses to access some hardware registers */
+static void octep_setup_pci_window_regs_cnxk_pf(struct octep_device *oct)
+{
+	u8 __iomem *bar0_pciaddr = oct->mmio[0].hw_addr;
+
+	oct->pci_win_regs.pci_win_wr_addr = (u8 __iomem *)(bar0_pciaddr + CNXK_SDP_WIN_WR_ADDR64);
+	oct->pci_win_regs.pci_win_rd_addr = (u8 __iomem *)(bar0_pciaddr + CNXK_SDP_WIN_RD_ADDR64);
+	oct->pci_win_regs.pci_win_wr_data = (u8 __iomem *)(bar0_pciaddr + CNXK_SDP_WIN_WR_DATA64);
+	oct->pci_win_regs.pci_win_rd_data = (u8 __iomem *)(bar0_pciaddr + CNXK_SDP_WIN_RD_DATA64);
+}
+
+/* Configure Hardware mapping: inform hardware which rings belong to PF. */
+static void octep_configure_ring_mapping_cnxk_pf(struct octep_device *oct)
+{
+	struct octep_config *conf = oct->conf;
+	struct pci_dev *pdev = oct->pdev;
+	u64 pf_srn = CFG_GET_PORTS_PF_SRN(oct->conf);
+	int q;
+
+	for (q = 0; q < CFG_GET_PORTS_ACTIVE_IO_RINGS(conf); q++) {
+		u64 regval = 0;
+
+		if (oct->pcie_port)
+			regval = 8 << CNXK_SDP_FUNC_SEL_EPF_BIT_POS;
+
+		octep_write_csr64(oct, CNXK_SDP_EPVF_RING(pf_srn + q), regval);
+
+		regval = octep_read_csr64(oct, CNXK_SDP_EPVF_RING(pf_srn + q));
+		dev_dbg(&pdev->dev, "Write SDP_EPVF_RING[0x%llx] = 0x%llx\n",
+			CNXK_SDP_EPVF_RING(pf_srn + q), regval);
+	}
+}
+
+/* Initialize configuration limits and initial active config */
+static void octep_init_config_cnxk_pf(struct octep_device *oct)
+{
+	struct octep_config *conf = oct->conf;
+	struct pci_dev *pdev = oct->pdev;
+	u8 link = 0;
+	u64 val;
+	int pos;
+
+	/* Read ring configuration:
+	 * PF ring count, number of VFs and rings per VF supported
+	 */
+	val = octep_read_csr64(oct, CNXK_SDP_EPF_RINFO);
+	dev_info(&pdev->dev, "SDP_EPF_RINFO[0x%x]:0x%llx\n", CNXK_SDP_EPF_RINFO, val);
+	conf->sriov_cfg.max_rings_per_vf = CNXK_SDP_EPF_RINFO_RPVF(val);
+	conf->sriov_cfg.active_rings_per_vf = conf->sriov_cfg.max_rings_per_vf;
+	conf->sriov_cfg.max_vfs = CNXK_SDP_EPF_RINFO_NVFS(val);
+	conf->sriov_cfg.active_vfs = conf->sriov_cfg.max_vfs;
+	conf->sriov_cfg.vf_srn = CNXK_SDP_EPF_RINFO_SRN(val);
+
+	val = octep_read_csr64(oct, CNXK_SDP_MAC_PF_RING_CTL(oct->pcie_port));
+	dev_info(&pdev->dev, "SDP_MAC_PF_RING_CTL[%d]:0x%llx\n", oct->pcie_port, val);
+	conf->pf_ring_cfg.srn =  CNXK_SDP_MAC_PF_RING_CTL_SRN(val);
+	conf->pf_ring_cfg.max_io_rings = CNXK_SDP_MAC_PF_RING_CTL_RPPF(val);
+	conf->pf_ring_cfg.active_io_rings = conf->pf_ring_cfg.max_io_rings;
+	dev_info(&pdev->dev, "pf_srn=%u rpvf=%u nvfs=%u rppf=%u\n",
+		 conf->pf_ring_cfg.srn, conf->sriov_cfg.active_rings_per_vf,
+		 conf->sriov_cfg.active_vfs, conf->pf_ring_cfg.active_io_rings);
+
+	conf->iq.num_descs = OCTEP_IQ_MAX_DESCRIPTORS;
+	conf->iq.instr_type = OCTEP_64BYTE_INSTR;
+	conf->iq.db_min = OCTEP_DB_MIN;
+	conf->iq.intr_threshold = OCTEP_IQ_INTR_THRESHOLD;
+
+	conf->oq.num_descs = OCTEP_OQ_MAX_DESCRIPTORS;
+	conf->oq.buf_size = OCTEP_OQ_BUF_SIZE;
+	conf->oq.refill_threshold = OCTEP_OQ_REFILL_THRESHOLD;
+	conf->oq.oq_intr_pkt = OCTEP_OQ_INTR_PKT_THRESHOLD;
+	conf->oq.oq_intr_time = OCTEP_OQ_INTR_TIME_THRESHOLD;
+
+	conf->msix_cfg.non_ioq_msix = CNXK_NUM_NON_IOQ_INTR;
+	conf->msix_cfg.ioq_msix = conf->pf_ring_cfg.active_io_rings;
+	conf->msix_cfg.non_ioq_msix_names = cnxk_non_ioq_msix_names;
+
+	pos = pci_find_ext_capability(oct->pdev, PCI_EXT_CAP_ID_SRIOV);
+	if (pos) {
+		pci_read_config_byte(oct->pdev,
+				     pos + PCI_SRIOV_FUNC_LINK,
+				     &link);
+		link = PCI_DEVFN(PCI_SLOT(oct->pdev->devfn), link);
+	}
+	conf->ctrl_mbox_cfg.barmem_addr = (void __iomem *)oct->mmio[2].hw_addr +
+					   CNXK_PEM_BAR4_INDEX_OFFSET +
+					   (link * CTRL_MBOX_SZ);
+
+	conf->fw_info.hb_interval = OCTEP_DEFAULT_FW_HB_INTERVAL;
+	conf->fw_info.hb_miss_count = OCTEP_DEFAULT_FW_HB_MISS_COUNT;
+}
+
+/* Setup registers for a hardware Tx Queue  */
+static void octep_setup_iq_regs_cnxk_pf(struct octep_device *oct, int iq_no)
+{
+	struct octep_iq *iq = oct->iq[iq_no];
+	u32 reset_instr_cnt;
+	u64 reg_val;
+
+	iq_no += CFG_GET_PORTS_PF_SRN(oct->conf);
+	reg_val = octep_read_csr64(oct, CNXK_SDP_R_IN_CONTROL(iq_no));
+
+	/* wait for IDLE to set to 1 */
+	if (!(reg_val & CNXK_R_IN_CTL_IDLE)) {
+		do {
+			reg_val = octep_read_csr64(oct, CNXK_SDP_R_IN_CONTROL(iq_no));
+		} while (!(reg_val & CNXK_R_IN_CTL_IDLE));
+	}
+
+	reg_val |= CNXK_R_IN_CTL_RDSIZE;
+	reg_val |= CNXK_R_IN_CTL_IS_64B;
+	reg_val |= CNXK_R_IN_CTL_ESR;
+	octep_write_csr64(oct, CNXK_SDP_R_IN_CONTROL(iq_no), reg_val);
+
+	/* Write the start of the input queue's ring and its size  */
+	octep_write_csr64(oct, CNXK_SDP_R_IN_INSTR_BADDR(iq_no),
+			  iq->desc_ring_dma);
+	octep_write_csr64(oct, CNXK_SDP_R_IN_INSTR_RSIZE(iq_no),
+			  iq->max_count);
+
+	/* Remember the doorbell & instruction count register addr
+	 * for this queue
+	 */
+	iq->doorbell_reg = oct->mmio[0].hw_addr +
+			   CNXK_SDP_R_IN_INSTR_DBELL(iq_no);
+	iq->inst_cnt_reg = oct->mmio[0].hw_addr +
+			   CNXK_SDP_R_IN_CNTS(iq_no);
+	iq->intr_lvl_reg = oct->mmio[0].hw_addr +
+			   CNXK_SDP_R_IN_INT_LEVELS(iq_no);
+
+	/* Store the current instruction counter (used in flush_iq calculation) */
+	reset_instr_cnt = readl(iq->inst_cnt_reg);
+	writel(reset_instr_cnt, iq->inst_cnt_reg);
+
+	/* INTR_THRESHOLD is set to max(FFFFFFFF) to disable the INTR */
+	reg_val = CFG_GET_IQ_INTR_THRESHOLD(oct->conf) & 0xffffffff;
+	octep_write_csr64(oct, CNXK_SDP_R_IN_INT_LEVELS(iq_no), reg_val);
+}
+
+/* Setup registers for a hardware Rx Queue  */
+static void octep_setup_oq_regs_cnxk_pf(struct octep_device *oct, int oq_no)
+{
+	u64 reg_val;
+	u64 oq_ctl = 0ULL;
+	u32 time_threshold = 0;
+	struct octep_oq *oq = oct->oq[oq_no];
+
+	oq_no += CFG_GET_PORTS_PF_SRN(oct->conf);
+	reg_val = octep_read_csr64(oct, CNXK_SDP_R_OUT_CONTROL(oq_no));
+
+	/* wait for IDLE to set to 1 */
+	if (!(reg_val & CNXK_R_OUT_CTL_IDLE)) {
+		do {
+			reg_val = octep_read_csr64(oct, CNXK_SDP_R_OUT_CONTROL(oq_no));
+		} while (!(reg_val & CNXK_R_OUT_CTL_IDLE));
+	}
+
+	reg_val &= ~(CNXK_R_OUT_CTL_IMODE);
+	reg_val &= ~(CNXK_R_OUT_CTL_ROR_P);
+	reg_val &= ~(CNXK_R_OUT_CTL_NSR_P);
+	reg_val &= ~(CNXK_R_OUT_CTL_ROR_I);
+	reg_val &= ~(CNXK_R_OUT_CTL_NSR_I);
+	reg_val &= ~(CNXK_R_OUT_CTL_ES_I);
+	reg_val &= ~(CNXK_R_OUT_CTL_ROR_D);
+	reg_val &= ~(CNXK_R_OUT_CTL_NSR_D);
+	reg_val &= ~(CNXK_R_OUT_CTL_ES_D);
+	reg_val |= (CNXK_R_OUT_CTL_ES_P);
+
+	octep_write_csr64(oct, CNXK_SDP_R_OUT_CONTROL(oq_no), reg_val);
+	octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_BADDR(oq_no),
+			  oq->desc_ring_dma);
+	octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_RSIZE(oq_no),
+			  oq->max_count);
+
+	oq_ctl = octep_read_csr64(oct, CNXK_SDP_R_OUT_CONTROL(oq_no));
+
+	/* Clear the ISIZE and BSIZE (22-0) */
+	oq_ctl &= ~0x7fffffULL;
+
+	/* Populate the BSIZE (15-0) */
+	oq_ctl |= (oq->buffer_size & 0xffff);
+	octep_write_csr64(oct, CNXK_SDP_R_OUT_CONTROL(oq_no), oq_ctl);
+
+	/* Get the mapped address of the pkt_sent and pkts_credit regs */
+	oq->pkts_sent_reg = oct->mmio[0].hw_addr + CNXK_SDP_R_OUT_CNTS(oq_no);
+	oq->pkts_credit_reg = oct->mmio[0].hw_addr +
+			      CNXK_SDP_R_OUT_SLIST_DBELL(oq_no);
+
+	time_threshold = CFG_GET_OQ_INTR_TIME(oct->conf);
+	reg_val = ((u64)time_threshold << 32) |
+		  CFG_GET_OQ_INTR_PKT(oct->conf);
+	octep_write_csr64(oct, CNXK_SDP_R_OUT_INT_LEVELS(oq_no), reg_val);
+}
+
+/* Setup registers for a PF mailbox */
+static void octep_setup_mbox_regs_cnxk_pf(struct octep_device *oct, int q_no)
+{
+	struct octep_mbox *mbox = oct->mbox[q_no];
+
+	mbox->q_no = q_no;
+
+	/* PF mbox interrupt reg */
+	mbox->mbox_int_reg = oct->mmio[0].hw_addr + CNXK_SDP_EPF_MBOX_RINT(0);
+
+	/* PF to VF DATA reg. PF writes into this reg */
+	mbox->mbox_write_reg = oct->mmio[0].hw_addr + CNXK_SDP_R_MBOX_PF_VF_DATA(q_no);
+
+	/* VF to PF DATA reg. PF reads from this reg */
+	mbox->mbox_read_reg = oct->mmio[0].hw_addr + CNXK_SDP_R_MBOX_VF_PF_DATA(q_no);
+}
+
+/* Poll OEI events like heartbeat */
+static void octep_poll_oei_cnxk_pf(struct octep_device *oct)
+{
+	u64 reg0;
+
+	/* Check for OEI INTR */
+	reg0 = octep_read_csr64(oct, CNXK_SDP_EPF_OEI_RINT);
+	if (reg0) {
+		octep_write_csr64(oct, CNXK_SDP_EPF_OEI_RINT, reg0);
+		if (reg0 & CNXK_SDP_EPF_OEI_RINT_DATA_BIT_MBOX)
+			queue_work(octep_wq, &oct->ctrl_mbox_task);
+		if (reg0 & CNXK_SDP_EPF_OEI_RINT_DATA_BIT_HBEAT)
+			atomic_set(&oct->hb_miss_cnt, 0);
+	}
+}
+
+/* OEI interrupt handler */
+static irqreturn_t octep_oei_intr_handler_cnxk_pf(void *dev)
+{
+	struct octep_device *oct = (struct octep_device *)dev;
+
+	octep_poll_oei_cnxk_pf(oct);
+	return IRQ_HANDLED;
+}
+
+/* Process non-ioq interrupts required to keep pf interface running.
+ * OEI_RINT is needed for control mailbox
+ * MBOX_RINT is needed for pfvf mailbox
+ */
+static void octep_poll_non_ioq_interrupts_cnxk_pf(struct octep_device *oct)
+{
+	octep_poll_oei_cnxk_pf(oct);
+}
+
+/* Interrupt handler for input ring error interrupts. */
+static irqreturn_t octep_ire_intr_handler_cnxk_pf(void *dev)
+{
+	struct octep_device *oct = (struct octep_device *)dev;
+	struct pci_dev *pdev = oct->pdev;
+	u64 reg_val = 0;
+	int i = 0;
+
+	/* Check for IRERR INTR */
+	reg_val = octep_read_csr64(oct, CNXK_SDP_EPF_IRERR_RINT);
+	if (reg_val) {
+		dev_info(&pdev->dev,
+			 "received IRERR_RINT intr: 0x%llx\n", reg_val);
+		octep_write_csr64(oct, CNXK_SDP_EPF_IRERR_RINT, reg_val);
+
+		for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
+			reg_val = octep_read_csr64(oct,
+						   CNXK_SDP_R_ERR_TYPE(i));
+			if (reg_val) {
+				dev_info(&pdev->dev,
+					 "Received err type on IQ-%d: 0x%llx\n",
+					 i, reg_val);
+				octep_write_csr64(oct, CNXK_SDP_R_ERR_TYPE(i),
+						  reg_val);
+			}
+		}
+	}
+	return IRQ_HANDLED;
+}
+
+/* Interrupt handler for output ring error interrupts. */
+static irqreturn_t octep_ore_intr_handler_cnxk_pf(void *dev)
+{
+	struct octep_device *oct = (struct octep_device *)dev;
+	struct pci_dev *pdev = oct->pdev;
+	u64 reg_val = 0;
+	int i = 0;
+
+	/* Check for ORERR INTR */
+	reg_val = octep_read_csr64(oct, CNXK_SDP_EPF_ORERR_RINT);
+	if (reg_val) {
+		dev_info(&pdev->dev,
+			 "Received ORERR_RINT intr: 0x%llx\n", reg_val);
+		octep_write_csr64(oct, CNXK_SDP_EPF_ORERR_RINT, reg_val);
+		for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
+			reg_val = octep_read_csr64(oct, CNXK_SDP_R_ERR_TYPE(i));
+			if (reg_val) {
+				dev_info(&pdev->dev,
+					 "Received err type on OQ-%d: 0x%llx\n",
+					 i, reg_val);
+				octep_write_csr64(oct, CNXK_SDP_R_ERR_TYPE(i),
+						  reg_val);
+			}
+		}
+	}
+	return IRQ_HANDLED;
+}
+
+/* Interrupt handler for vf input ring error interrupts. */
+static irqreturn_t octep_vfire_intr_handler_cnxk_pf(void *dev)
+{
+	struct octep_device *oct = (struct octep_device *)dev;
+	struct pci_dev *pdev = oct->pdev;
+	u64 reg_val = 0;
+
+	/* Check for VFIRE INTR */
+	reg_val = octep_read_csr64(oct, CNXK_SDP_EPF_VFIRE_RINT(0));
+	if (reg_val) {
+		dev_info(&pdev->dev,
+			 "Received VFIRE_RINT intr: 0x%llx\n", reg_val);
+		octep_write_csr64(oct, CNXK_SDP_EPF_VFIRE_RINT(0), reg_val);
+	}
+	return IRQ_HANDLED;
+}
+
+/* Interrupt handler for vf output ring error interrupts. */
+static irqreturn_t octep_vfore_intr_handler_cnxk_pf(void *dev)
+{
+	struct octep_device *oct = (struct octep_device *)dev;
+	struct pci_dev *pdev = oct->pdev;
+	u64 reg_val = 0;
+
+	/* Check for VFORE INTR */
+	reg_val = octep_read_csr64(oct, CNXK_SDP_EPF_VFORE_RINT(0));
+	if (reg_val) {
+		dev_info(&pdev->dev,
+			 "Received VFORE_RINT intr: 0x%llx\n", reg_val);
+		octep_write_csr64(oct, CNXK_SDP_EPF_VFORE_RINT(0), reg_val);
+	}
+	return IRQ_HANDLED;
+}
+
+/* Interrupt handler for dpi dma related interrupts. */
+static irqreturn_t octep_dma_intr_handler_cnxk_pf(void *dev)
+{
+	struct octep_device *oct = (struct octep_device *)dev;
+	u64 reg_val = 0;
+
+	/* Check for DMA INTR */
+	reg_val = octep_read_csr64(oct, CNXK_SDP_EPF_DMA_RINT);
+	if (reg_val)
+		octep_write_csr64(oct, CNXK_SDP_EPF_DMA_RINT, reg_val);
+
+	return IRQ_HANDLED;
+}
+
+/* Interrupt handler for dpi dma transaction error interrupts for VFs  */
+static irqreturn_t octep_dma_vf_intr_handler_cnxk_pf(void *dev)
+{
+	struct octep_device *oct = (struct octep_device *)dev;
+	struct pci_dev *pdev = oct->pdev;
+	u64 reg_val = 0;
+
+	/* Check for DMA VF INTR */
+	reg_val = octep_read_csr64(oct, CNXK_SDP_EPF_DMA_VF_RINT(0));
+	if (reg_val) {
+		dev_info(&pdev->dev,
+			 "Received DMA_VF_RINT intr: 0x%llx\n", reg_val);
+		octep_write_csr64(oct, CNXK_SDP_EPF_DMA_VF_RINT(0), reg_val);
+	}
+	return IRQ_HANDLED;
+}
+
+/* Interrupt handler for pp transaction error interrupts for VFs  */
+static irqreturn_t octep_pp_vf_intr_handler_cnxk_pf(void *dev)
+{
+	struct octep_device *oct = (struct octep_device *)dev;
+	struct pci_dev *pdev = oct->pdev;
+	u64 reg_val = 0;
+
+	/* Check for PPVF INTR */
+	reg_val = octep_read_csr64(oct, CNXK_SDP_EPF_PP_VF_RINT(0));
+	if (reg_val) {
+		dev_info(&pdev->dev,
+			 "Received PP_VF_RINT intr: 0x%llx\n", reg_val);
+		octep_write_csr64(oct, CNXK_SDP_EPF_PP_VF_RINT(0), reg_val);
+	}
+	return IRQ_HANDLED;
+}
+
+/* Interrupt handler for mac related interrupts. */
+static irqreturn_t octep_misc_intr_handler_cnxk_pf(void *dev)
+{
+	struct octep_device *oct = (struct octep_device *)dev;
+	struct pci_dev *pdev = oct->pdev;
+	u64 reg_val = 0;
+
+	/* Check for MISC INTR */
+	reg_val = octep_read_csr64(oct, CNXK_SDP_EPF_MISC_RINT);
+	if (reg_val) {
+		dev_info(&pdev->dev,
+			 "Received MISC_RINT intr: 0x%llx\n", reg_val);
+		octep_write_csr64(oct, CNXK_SDP_EPF_MISC_RINT, reg_val);
+	}
+	return IRQ_HANDLED;
+}
+
+/* Interrupts handler for all reserved interrupts. */
+static irqreturn_t octep_rsvd_intr_handler_cnxk_pf(void *dev)
+{
+	struct octep_device *oct = (struct octep_device *)dev;
+	struct pci_dev *pdev = oct->pdev;
+
+	dev_info(&pdev->dev, "Reserved interrupts raised; Ignore\n");
+	return IRQ_HANDLED;
+}
+
+/* Tx/Rx queue interrupt handler */
+static irqreturn_t octep_ioq_intr_handler_cnxk_pf(void *data)
+{
+	struct octep_ioq_vector *vector = (struct octep_ioq_vector *)data;
+	struct octep_oq *oq = vector->oq;
+
+	napi_schedule_irqoff(oq->napi);
+	return IRQ_HANDLED;
+}
+
+/* soft reset */
+static int octep_soft_reset_cnxk_pf(struct octep_device *oct)
+{
+	dev_info(&oct->pdev->dev, "CNXKXX: Doing soft reset\n");
+
+	octep_write_csr64(oct, CNXK_SDP_WIN_WR_MASK_REG, 0xFF);
+
+	/* Firmware status CSR is supposed to be cleared by
+	 * core domain reset, but due to a hw bug, it is not.
+	 * Set it to RUNNING right before reset so that it is not
+	 * left in READY (1) state after a reset.  This is required
+	 * in addition to the early setting to handle the case where
+	 * the OcteonTX is unexpectedly reset, reboots, and then
+	 * the module is removed.
+	 */
+	OCTEP_PCI_WIN_WRITE(oct, CNXK_PEMX_PFX_CSX_PFCFGX(0, 0, CNXK_PCIEEP_VSECST_CTL),
+			    FW_STATUS_RUNNING);
+
+	/* Set chip domain reset bit */
+	OCTEP_PCI_WIN_WRITE(oct, CNXK_RST_CHIP_DOMAIN_W1S, 1);
+	/* Wait till Octeon resets. */
+	mdelay(10);
+	/* restore the  reset value */
+	octep_write_csr64(oct, CNXK_SDP_WIN_WR_MASK_REG, 0xFF);
+
+	return 0;
+}
+
+/* Re-initialize Octeon hardware registers */
+static void octep_reinit_regs_cnxk_pf(struct octep_device *oct)
+{
+	u32 i;
+
+	for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++)
+		oct->hw_ops.setup_iq_regs(oct, i);
+
+	for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++)
+		oct->hw_ops.setup_oq_regs(oct, i);
+
+	oct->hw_ops.enable_interrupts(oct);
+	oct->hw_ops.enable_io_queues(oct);
+
+	for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++)
+		writel(oct->oq[i]->max_count, oct->oq[i]->pkts_credit_reg);
+}
+
+/* Enable all interrupts */
+static void octep_enable_interrupts_cnxk_pf(struct octep_device *oct)
+{
+	u64 intr_mask = 0ULL;
+	int srn, num_rings, i;
+
+	srn = CFG_GET_PORTS_PF_SRN(oct->conf);
+	num_rings = CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf);
+
+	for (i = 0; i < num_rings; i++)
+		intr_mask |= (0x1ULL << (srn + i));
+
+	octep_write_csr64(oct, CNXK_SDP_EPF_IRERR_RINT_ENA_W1S, intr_mask);
+	octep_write_csr64(oct, CNXK_SDP_EPF_ORERR_RINT_ENA_W1S, intr_mask);
+	octep_write_csr64(oct, CNXK_SDP_EPF_OEI_RINT_ENA_W1S, -1ULL);
+
+	octep_write_csr64(oct, CNXK_SDP_EPF_VFIRE_RINT_ENA_W1S(0), -1ULL);
+	octep_write_csr64(oct, CNXK_SDP_EPF_VFORE_RINT_ENA_W1S(0), -1ULL);
+
+	octep_write_csr64(oct, CNXK_SDP_EPF_MISC_RINT_ENA_W1S, intr_mask);
+	octep_write_csr64(oct, CNXK_SDP_EPF_DMA_RINT_ENA_W1S, intr_mask);
+
+	octep_write_csr64(oct, CNXK_SDP_EPF_DMA_VF_RINT_ENA_W1S(0), -1ULL);
+	octep_write_csr64(oct, CNXK_SDP_EPF_PP_VF_RINT_ENA_W1S(0), -1ULL);
+}
+
+/* Disable all interrupts */
+static void octep_disable_interrupts_cnxk_pf(struct octep_device *oct)
+{
+	u64 intr_mask = 0ULL;
+	int srn, num_rings, i;
+
+	srn = CFG_GET_PORTS_PF_SRN(oct->conf);
+	num_rings = CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf);
+
+	for (i = 0; i < num_rings; i++)
+		intr_mask |= (0x1ULL << (srn + i));
+
+	octep_write_csr64(oct, CNXK_SDP_EPF_IRERR_RINT_ENA_W1C, intr_mask);
+	octep_write_csr64(oct, CNXK_SDP_EPF_ORERR_RINT_ENA_W1C, intr_mask);
+	octep_write_csr64(oct, CNXK_SDP_EPF_OEI_RINT_ENA_W1C, -1ULL);
+
+	octep_write_csr64(oct, CNXK_SDP_EPF_VFIRE_RINT_ENA_W1C(0), -1ULL);
+	octep_write_csr64(oct, CNXK_SDP_EPF_VFORE_RINT_ENA_W1C(0), -1ULL);
+
+	octep_write_csr64(oct, CNXK_SDP_EPF_MISC_RINT_ENA_W1C, intr_mask);
+	octep_write_csr64(oct, CNXK_SDP_EPF_DMA_RINT_ENA_W1C, intr_mask);
+
+	octep_write_csr64(oct, CNXK_SDP_EPF_DMA_VF_RINT_ENA_W1C(0), -1ULL);
+	octep_write_csr64(oct, CNXK_SDP_EPF_PP_VF_RINT_ENA_W1C(0), -1ULL);
+}
+
+/* Get new Octeon Read Index: index of descriptor that Octeon reads next. */
+static u32 octep_update_iq_read_index_cnxk_pf(struct octep_iq *iq)
+{
+	u32 pkt_in_done = readl(iq->inst_cnt_reg);
+	u32 last_done, new_idx;
+
+	last_done = pkt_in_done - iq->pkt_in_done;
+	iq->pkt_in_done = pkt_in_done;
+
+	new_idx = (iq->octep_read_index + last_done) % iq->max_count;
+
+	return new_idx;
+}
+
+/* Enable a hardware Tx Queue */
+static void octep_enable_iq_cnxk_pf(struct octep_device *oct, int iq_no)
+{
+	u64 loop = HZ;
+	u64 reg_val;
+
+	iq_no += CFG_GET_PORTS_PF_SRN(oct->conf);
+
+	octep_write_csr64(oct, CNXK_SDP_R_IN_INSTR_DBELL(iq_no), 0xFFFFFFFF);
+
+	while (octep_read_csr64(oct, CNXK_SDP_R_IN_INSTR_DBELL(iq_no)) &&
+	       loop--) {
+		schedule_timeout_interruptible(1);
+	}
+
+	reg_val = octep_read_csr64(oct,  CNXK_SDP_R_IN_INT_LEVELS(iq_no));
+	reg_val |= (0x1ULL << 62);
+	octep_write_csr64(oct, CNXK_SDP_R_IN_INT_LEVELS(iq_no), reg_val);
+
+	reg_val = octep_read_csr64(oct, CNXK_SDP_R_IN_ENABLE(iq_no));
+	reg_val |= 0x1ULL;
+	octep_write_csr64(oct, CNXK_SDP_R_IN_ENABLE(iq_no), reg_val);
+}
+
+/* Enable a hardware Rx Queue */
+static void octep_enable_oq_cnxk_pf(struct octep_device *oct, int oq_no)
+{
+	u64 reg_val = 0ULL;
+
+	oq_no += CFG_GET_PORTS_PF_SRN(oct->conf);
+
+	reg_val = octep_read_csr64(oct,  CNXK_SDP_R_OUT_INT_LEVELS(oq_no));
+	reg_val |= (0x1ULL << 62);
+	octep_write_csr64(oct, CNXK_SDP_R_OUT_INT_LEVELS(oq_no), reg_val);
+
+	octep_write_csr64(oct, CNXK_SDP_R_OUT_SLIST_DBELL(oq_no), 0xFFFFFFFF);
+
+	reg_val = octep_read_csr64(oct, CNXK_SDP_R_OUT_ENABLE(oq_no));
+	reg_val |= 0x1ULL;
+	octep_write_csr64(oct, CNXK_SDP_R_OUT_ENABLE(oq_no), reg_val);
+}
+
+/* Enable all hardware Tx/Rx Queues assined to PF */
+static void octep_enable_io_queues_cnxk_pf(struct octep_device *oct)
+{
+	u8 q;
+
+	for (q = 0; q < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); q++) {
+		octep_enable_iq_cnxk_pf(oct, q);
+		octep_enable_oq_cnxk_pf(oct, q);
+	}
+}
+
+/* Disable a hardware Tx Queue assined to PF */
+static void octep_disable_iq_cnxk_pf(struct octep_device *oct, int iq_no)
+{
+	u64 reg_val = 0ULL;
+
+	iq_no += CFG_GET_PORTS_PF_SRN(oct->conf);
+
+	reg_val = octep_read_csr64(oct, CNXK_SDP_R_IN_ENABLE(iq_no));
+	reg_val &= ~0x1ULL;
+	octep_write_csr64(oct, CNXK_SDP_R_IN_ENABLE(iq_no), reg_val);
+}
+
+/* Disable a hardware Rx Queue assined to PF */
+static void octep_disable_oq_cnxk_pf(struct octep_device *oct, int oq_no)
+{
+	u64 reg_val = 0ULL;
+
+	oq_no += CFG_GET_PORTS_PF_SRN(oct->conf);
+	reg_val = octep_read_csr64(oct, CNXK_SDP_R_OUT_ENABLE(oq_no));
+	reg_val &= ~0x1ULL;
+	octep_write_csr64(oct, CNXK_SDP_R_OUT_ENABLE(oq_no), reg_val);
+}
+
+/* Disable all hardware Tx/Rx Queues assined to PF */
+static void octep_disable_io_queues_cnxk_pf(struct octep_device *oct)
+{
+	int q = 0;
+
+	for (q = 0; q < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); q++) {
+		octep_disable_iq_cnxk_pf(oct, q);
+		octep_disable_oq_cnxk_pf(oct, q);
+	}
+}
+
+/* Dump hardware registers (including Tx/Rx queues) for debugging. */
+static void octep_dump_registers_cnxk_pf(struct octep_device *oct)
+{
+	u8 srn, num_rings, q;
+
+	srn = CFG_GET_PORTS_PF_SRN(oct->conf);
+	num_rings = CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf);
+
+	for (q = srn; q < srn + num_rings; q++)
+		cnxk_dump_regs(oct, q);
+}
+
+/**
+ * octep_device_setup_cnxk_pf() - Setup Octeon device.
+ *
+ * @oct: Octeon device private data structure.
+ *
+ * - initialize hardware operations.
+ * - get target side pcie port number for the device.
+ * - setup window access to hardware registers.
+ * - set initial configuration and max limits.
+ * - setup hardware mapping of rings to the PF device.
+ */
+void octep_device_setup_cnxk_pf(struct octep_device *oct)
+{
+	oct->hw_ops.setup_iq_regs = octep_setup_iq_regs_cnxk_pf;
+	oct->hw_ops.setup_oq_regs = octep_setup_oq_regs_cnxk_pf;
+	oct->hw_ops.setup_mbox_regs = octep_setup_mbox_regs_cnxk_pf;
+
+	oct->hw_ops.oei_intr_handler = octep_oei_intr_handler_cnxk_pf;
+	oct->hw_ops.ire_intr_handler = octep_ire_intr_handler_cnxk_pf;
+	oct->hw_ops.ore_intr_handler = octep_ore_intr_handler_cnxk_pf;
+	oct->hw_ops.vfire_intr_handler = octep_vfire_intr_handler_cnxk_pf;
+	oct->hw_ops.vfore_intr_handler = octep_vfore_intr_handler_cnxk_pf;
+	oct->hw_ops.dma_intr_handler = octep_dma_intr_handler_cnxk_pf;
+	oct->hw_ops.dma_vf_intr_handler = octep_dma_vf_intr_handler_cnxk_pf;
+	oct->hw_ops.pp_vf_intr_handler = octep_pp_vf_intr_handler_cnxk_pf;
+	oct->hw_ops.misc_intr_handler = octep_misc_intr_handler_cnxk_pf;
+	oct->hw_ops.rsvd_intr_handler = octep_rsvd_intr_handler_cnxk_pf;
+	oct->hw_ops.ioq_intr_handler = octep_ioq_intr_handler_cnxk_pf;
+	oct->hw_ops.soft_reset = octep_soft_reset_cnxk_pf;
+	oct->hw_ops.reinit_regs = octep_reinit_regs_cnxk_pf;
+
+	oct->hw_ops.enable_interrupts = octep_enable_interrupts_cnxk_pf;
+	oct->hw_ops.disable_interrupts = octep_disable_interrupts_cnxk_pf;
+	oct->hw_ops.poll_non_ioq_interrupts = octep_poll_non_ioq_interrupts_cnxk_pf;
+
+	oct->hw_ops.update_iq_read_idx = octep_update_iq_read_index_cnxk_pf;
+
+	oct->hw_ops.enable_iq = octep_enable_iq_cnxk_pf;
+	oct->hw_ops.enable_oq = octep_enable_oq_cnxk_pf;
+	oct->hw_ops.enable_io_queues = octep_enable_io_queues_cnxk_pf;
+
+	oct->hw_ops.disable_iq = octep_disable_iq_cnxk_pf;
+	oct->hw_ops.disable_oq = octep_disable_oq_cnxk_pf;
+	oct->hw_ops.disable_io_queues = octep_disable_io_queues_cnxk_pf;
+	oct->hw_ops.reset_io_queues = octep_reset_io_queues_cnxk_pf;
+
+	oct->hw_ops.dump_registers = octep_dump_registers_cnxk_pf;
+
+	octep_setup_pci_window_regs_cnxk_pf(oct);
+
+	oct->pcie_port = octep_read_csr64(oct, CNXK_SDP_MAC_NUMBER) & 0xff;
+	dev_info(&oct->pdev->dev,
+		 "Octeon device using PCIE Port %d\n", oct->pcie_port);
+
+	octep_init_config_cnxk_pf(oct);
+	octep_configure_ring_mapping_cnxk_pf(oct);
+
+	/* Firmware status CSR is supposed to be cleared by
+	 * core domain reset, but due to IPBUPEM-38842, it is not.
+	 * Set it to RUNNING early in boot, so that unexpected resets
+	 * leave it in a state that is not READY (1).
+	 */
+	OCTEP_PCI_WIN_WRITE(oct, CNXK_PEMX_PFX_CSX_PFCFGX(0, 0, CNXK_PCIEEP_VSECST_CTL),
+			    FW_STATUS_RUNNING);
+}
diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_config.h b/drivers/net/ethernet/marvell/octeon_ep/octep_config.h
index 1622a6ebf036..91cfa19c65b9 100644
--- a/drivers/net/ethernet/marvell/octeon_ep/octep_config.h
+++ b/drivers/net/ethernet/marvell/octeon_ep/octep_config.h
@@ -13,9 +13,10 @@
 #define OCTEP_64BYTE_INSTR  64
 
 /* Tx Queue: maximum descriptors per ring */
+/* This needs to be a power of 2 */
 #define OCTEP_IQ_MAX_DESCRIPTORS    1024
 /* Minimum input (Tx) requests to be enqueued to ring doorbell */
-#define OCTEP_DB_MIN                1
+#define OCTEP_DB_MIN                8
 /* Packet threshold for Tx queue interrupt */
 #define OCTEP_IQ_INTR_THRESHOLD     0x0
 
diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_main.c b/drivers/net/ethernet/marvell/octeon_ep/octep_main.c
index 552970c7dec0..2da00a701df2 100644
--- a/drivers/net/ethernet/marvell/octeon_ep/octep_main.c
+++ b/drivers/net/ethernet/marvell/octeon_ep/octep_main.c
@@ -24,6 +24,10 @@ struct workqueue_struct *octep_wq;
 static const struct pci_device_id octep_pci_id_tbl[] = {
 	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN93_PF)},
 	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF95N_PF)},
+	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN10KA_PF)},
+	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF10KA_PF)},
+	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF10KB_PF)},
+	{PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN10KB_PF)},
 	{0, },
 };
 MODULE_DEVICE_TABLE(pci, octep_pci_id_tbl);
@@ -777,17 +781,24 @@ static int octep_stop(struct net_device *netdev)
  */
 static inline int octep_iq_full_check(struct octep_iq *iq)
 {
-	if (likely((iq->max_count - atomic_read(&iq->instr_pending)) >=
+	if (likely((IQ_INSTR_SPACE(iq)) >
 		   OCTEP_WAKE_QUEUE_THRESHOLD))
 		return 0;
 
 	/* Stop the queue if unable to send */
 	netif_stop_subqueue(iq->netdev, iq->q_no);
 
+	/* Allow for pending updates in write index
+	 * from iq_process_completion in other cpus
+	 * to reflect, in case queue gets free
+	 * entries.
+	 */
+	smp_mb();
+
 	/* check again and restart the queue, in case NAPI has just freed
 	 * enough Tx ring entries.
 	 */
-	if (unlikely((iq->max_count - atomic_read(&iq->instr_pending)) >=
+	if (unlikely(IQ_INSTR_SPACE(iq) >
 		     OCTEP_WAKE_QUEUE_THRESHOLD)) {
 		netif_start_subqueue(iq->netdev, iq->q_no);
 		iq->stats.restart_cnt++;
@@ -818,8 +829,12 @@ static netdev_tx_t octep_start_xmit(struct sk_buff *skb,
 	struct octep_iq *iq;
 	skb_frag_t *frag;
 	u16 nr_frags, si;
+	int xmit_more;
 	u16 q_no, wi;
 
+	if (skb_put_padto(skb, ETH_ZLEN))
+		return NETDEV_TX_OK;
+
 	q_no = skb_get_queue_mapping(skb);
 	if (q_no >= oct->num_iqs) {
 		netdev_err(netdev, "Invalid Tx skb->queue_mapping=%d\n", q_no);
@@ -827,10 +842,6 @@ static netdev_tx_t octep_start_xmit(struct sk_buff *skb,
 	}
 
 	iq = oct->iq[q_no];
-	if (octep_iq_full_check(iq)) {
-		iq->stats.tx_busy++;
-		return NETDEV_TX_BUSY;
-	}
 
 	shinfo = skb_shinfo(skb);
 	nr_frags = shinfo->nr_frags;
@@ -869,9 +880,6 @@ static netdev_tx_t octep_start_xmit(struct sk_buff *skb,
 		if (dma_mapping_error(iq->dev, dma))
 			goto dma_map_err;
 
-		dma_sync_single_for_cpu(iq->dev, tx_buffer->sglist_dma,
-					OCTEP_SGLIST_SIZE_PER_PKT,
-					DMA_TO_DEVICE);
 		memset(sglist, 0, OCTEP_SGLIST_SIZE_PER_PKT);
 		sglist[0].len[3] = len;
 		sglist[0].dma_ptr[0] = dma;
@@ -891,26 +899,33 @@ static netdev_tx_t octep_start_xmit(struct sk_buff *skb,
 			frag++;
 			si++;
 		}
-		dma_sync_single_for_device(iq->dev, tx_buffer->sglist_dma,
-					   OCTEP_SGLIST_SIZE_PER_PKT,
-					   DMA_TO_DEVICE);
-
 		hw_desc->dptr = tx_buffer->sglist_dma;
 	}
 
-	netdev_tx_sent_queue(iq->netdev_q, skb->len);
+	xmit_more = netdev_xmit_more();
+
+	__netdev_tx_sent_queue(iq->netdev_q, skb->len, xmit_more);
+
 	skb_tx_timestamp(skb);
-	atomic_inc(&iq->instr_pending);
+	iq->fill_cnt++;
 	wi++;
-	if (wi == iq->max_count)
-		wi = 0;
-	iq->host_write_index = wi;
+	iq->host_write_index = wi & iq->ring_size_mask;
+
+	/* octep_iq_full_check stops the queue and returns
+	 * true if so, in case the queue has become full
+	 * by inserting current packet. If so, we can
+	 * go ahead and ring doorbell.
+	 */
+	if (!octep_iq_full_check(iq) && xmit_more &&
+	    iq->fill_cnt < iq->fill_threshold)
+		return NETDEV_TX_OK;
+
 	/* Flush the hw descriptor before writing to doorbell */
 	wmb();
-
-	/* Ring Doorbell to notify the NIC there is a new packet */
-	writel(1, iq->doorbell_reg);
-	iq->stats.instr_posted++;
+	/* Ring Doorbell to notify the NIC of new packets */
+	writel(iq->fill_cnt, iq->doorbell_reg);
+	iq->stats.instr_posted += iq->fill_cnt;
+	iq->fill_cnt = 0;
 	return NETDEV_TX_OK;
 
 dma_map_sg_err:
@@ -1136,6 +1151,14 @@ static const char *octep_devid_to_str(struct octep_device *oct)
 		return "CN93XX";
 	case OCTEP_PCI_DEVICE_ID_CNF95N_PF:
 		return "CNF95N";
+	case OCTEP_PCI_DEVICE_ID_CN10KA_PF:
+		return "CN10KA";
+	case OCTEP_PCI_DEVICE_ID_CNF10KA_PF:
+		return "CNF10KA";
+	case OCTEP_PCI_DEVICE_ID_CNF10KB_PF:
+		return "CNF10KB";
+	case OCTEP_PCI_DEVICE_ID_CN10KB_PF:
+		return "CN10KB";
 	default:
 		return "Unsupported";
 	}
@@ -1181,6 +1204,14 @@ int octep_device_setup(struct octep_device *oct)
 			 OCTEP_MINOR_REV(oct));
 		octep_device_setup_cn93_pf(oct);
 		break;
+	case OCTEP_PCI_DEVICE_ID_CNF10KA_PF:
+	case OCTEP_PCI_DEVICE_ID_CN10KA_PF:
+	case OCTEP_PCI_DEVICE_ID_CNF10KB_PF:
+	case OCTEP_PCI_DEVICE_ID_CN10KB_PF:
+		dev_info(&pdev->dev, "Setting up OCTEON %s PF PASS%d.%d\n",
+			 octep_devid_to_str(oct), OCTEP_MAJOR_REV(oct), OCTEP_MINOR_REV(oct));
+		octep_device_setup_cnxk_pf(oct);
+		break;
 	default:
 		dev_err(&pdev->dev,
 			"%s: unsupported device\n", __func__);
diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_main.h b/drivers/net/ethernet/marvell/octeon_ep/octep_main.h
index 6df902ebb7f3..e2fe8b28eb0e 100644
--- a/drivers/net/ethernet/marvell/octeon_ep/octep_main.h
+++ b/drivers/net/ethernet/marvell/octeon_ep/octep_main.h
@@ -23,6 +23,11 @@
 
 #define  OCTEP_PCI_DEVICE_ID_CNF95N_PF 0xB400    //95N PF
 
+#define  OCTEP_PCI_DEVICE_ID_CN10KA_PF  0xB900   //CN10KA PF
+#define  OCTEP_PCI_DEVICE_ID_CNF10KA_PF 0xBA00   //CNF10KA PF
+#define  OCTEP_PCI_DEVICE_ID_CNF10KB_PF 0xBC00   //CNF10KB PF
+#define  OCTEP_PCI_DEVICE_ID_CN10KB_PF  0xBD00   //CN10KB PF
+
 #define  OCTEP_MAX_QUEUES   63
 #define  OCTEP_MAX_IQ       OCTEP_MAX_QUEUES
 #define  OCTEP_MAX_OQ       OCTEP_MAX_QUEUES
@@ -40,6 +45,15 @@
 #define  OCTEP_OQ_INTR_RESEND_BIT  59
 
 #define  OCTEP_MMIO_REGIONS     3
+
+#define  IQ_INSTR_PENDING(iq)  ({ typeof(iq) iq__ = (iq); \
+				  ((iq__)->host_write_index - (iq__)->flush_index) & \
+				  (iq__)->ring_size_mask; \
+				})
+#define  IQ_INSTR_SPACE(iq)    ({ typeof(iq) iq_ = (iq); \
+				  (iq_)->max_count - IQ_INSTR_PENDING(iq_); \
+				})
+
 /* PCI address space mapping information.
  * Each of the 3 address spaces given by BAR0, BAR2 and BAR4 of
  * Octeon gets mapped to different physical address spaces in
@@ -377,6 +391,7 @@ int octep_setup_oqs(struct octep_device *oct);
 void octep_free_oqs(struct octep_device *oct);
 void octep_oq_dbell_init(struct octep_device *oct);
 void octep_device_setup_cn93_pf(struct octep_device *oct);
+void octep_device_setup_cnxk_pf(struct octep_device *oct);
 int octep_iq_process_completions(struct octep_iq *iq, u16 budget);
 int octep_oq_process_rx(struct octep_oq *oq, int budget);
 void octep_set_ethtool_ops(struct net_device *netdev);
diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cnxk_pf.h b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cnxk_pf.h
new file mode 100644
index 000000000000..abe02df8af11
--- /dev/null
+++ b/drivers/net/ethernet/marvell/octeon_ep/octep_regs_cnxk_pf.h
@@ -0,0 +1,400 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Marvell Octeon EP (EndPoint) Ethernet Driver
+ *
+ * Copyright (C) 2020 Marvell.
+ *
+ */
+
+#ifndef _OCTEP_REGS_CNXK_PF_H_
+#define _OCTEP_REGS_CNXK_PF_H_
+
+/* ############################ RST ######################### */
+#define    CNXK_RST_BOOT               0x000087E006001600ULL
+#define    CNXK_RST_CHIP_DOMAIN_W1S    0x000087E006001810ULL
+#define    CNXK_RST_CORE_DOMAIN_W1S    0x000087E006001820ULL
+#define    CNXK_RST_CORE_DOMAIN_W1C    0x000087E006001828ULL
+
+#define     CNXK_CONFIG_XPANSION_BAR             0x38
+#define     CNXK_CONFIG_PCIE_CAP                 0x70
+#define     CNXK_CONFIG_PCIE_DEVCAP              0x74
+#define     CNXK_CONFIG_PCIE_DEVCTL              0x78
+#define     CNXK_CONFIG_PCIE_LINKCAP             0x7C
+#define     CNXK_CONFIG_PCIE_LINKCTL             0x80
+#define     CNXK_CONFIG_PCIE_SLOTCAP             0x84
+#define     CNXK_CONFIG_PCIE_SLOTCTL             0x88
+
+#define     CNXK_PCIE_SRIOV_FDL                  0x188      /* 0x98 */
+#define     CNXK_PCIE_SRIOV_FDL_BIT_POS          0x10
+#define     CNXK_PCIE_SRIOV_FDL_MASK             0xFF
+
+#define     CNXK_CONFIG_PCIE_FLTMSK              0x720
+
+/* ################# Offsets of RING, EPF, MAC ######################### */
+#define    CNXK_RING_OFFSET                      (0x1ULL << 17)
+#define    CNXK_EPF_OFFSET                       (0x1ULL << 25)
+#define    CNXK_MAC_OFFSET                       (0x1ULL << 4)
+#define    CNXK_BIT_ARRAY_OFFSET                 (0x1ULL << 4)
+#define    CNXK_EPVF_RING_OFFSET                 (0x1ULL << 4)
+
+/* ################# Scratch Registers ######################### */
+#define    CNXK_SDP_EPF_SCRATCH                  0x209E0
+
+/* ################# Window Registers ######################### */
+#define    CNXK_SDP_WIN_WR_ADDR64                0x20000
+#define    CNXK_SDP_WIN_RD_ADDR64                0x20010
+#define    CNXK_SDP_WIN_WR_DATA64                0x20020
+#define    CNXK_SDP_WIN_WR_MASK_REG              0x20030
+#define    CNXK_SDP_WIN_RD_DATA64                0x20040
+
+#define    CNXK_SDP_MAC_NUMBER                   0x2C100
+
+/* ################# Global Previliged registers ######################### */
+#define    CNXK_SDP_EPF_RINFO                    0x209F0
+
+#define    CNXK_SDP_EPF_RINFO_SRN(val)           ((val) & 0x7F)
+#define    CNXK_SDP_EPF_RINFO_RPVF(val)          (((val) >> 32) & 0xF)
+#define    CNXK_SDP_EPF_RINFO_NVFS(val)          (((val) >> 48) & 0x7F)
+
+/* SDP Function select */
+#define    CNXK_SDP_FUNC_SEL_EPF_BIT_POS         7
+#define    CNXK_SDP_FUNC_SEL_FUNC_BIT_POS        0
+
+/* ##### RING IN (Into device from PCI: Tx Ring) REGISTERS #### */
+#define    CNXK_SDP_R_IN_CONTROL_START           0x10000
+#define    CNXK_SDP_R_IN_ENABLE_START            0x10010
+#define    CNXK_SDP_R_IN_INSTR_BADDR_START       0x10020
+#define    CNXK_SDP_R_IN_INSTR_RSIZE_START       0x10030
+#define    CNXK_SDP_R_IN_INSTR_DBELL_START       0x10040
+#define    CNXK_SDP_R_IN_CNTS_START              0x10050
+#define    CNXK_SDP_R_IN_INT_LEVELS_START        0x10060
+#define    CNXK_SDP_R_IN_PKT_CNT_START           0x10080
+#define    CNXK_SDP_R_IN_BYTE_CNT_START          0x10090
+
+#define    CNXK_SDP_R_IN_CONTROL(ring)		\
+	(CNXK_SDP_R_IN_CONTROL_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_IN_ENABLE(ring)		\
+	(CNXK_SDP_R_IN_ENABLE_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_IN_INSTR_BADDR(ring)	\
+	(CNXK_SDP_R_IN_INSTR_BADDR_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_IN_INSTR_RSIZE(ring)	\
+	(CNXK_SDP_R_IN_INSTR_RSIZE_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_IN_INSTR_DBELL(ring)	\
+	(CNXK_SDP_R_IN_INSTR_DBELL_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_IN_CNTS(ring)		\
+	(CNXK_SDP_R_IN_CNTS_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_IN_INT_LEVELS(ring)	\
+	(CNXK_SDP_R_IN_INT_LEVELS_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_IN_PKT_CNT(ring)		\
+	(CNXK_SDP_R_IN_PKT_CNT_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_IN_BYTE_CNT(ring)		\
+	(CNXK_SDP_R_IN_BYTE_CNT_START + ((ring) * CNXK_RING_OFFSET))
+
+/* Rings per Virtual Function */
+#define    CNXK_R_IN_CTL_RPVF_MASK	(0xF)
+#define    CNXK_R_IN_CTL_RPVF_POS	(48)
+
+/* Number of instructions to be read in one MAC read request.
+ * setting to Max value(4)
+ */
+#define    CNXK_R_IN_CTL_IDLE                    (0x1ULL << 28)
+#define    CNXK_R_IN_CTL_RDSIZE                  (0x3ULL << 25)
+#define    CNXK_R_IN_CTL_IS_64B                  (0x1ULL << 24)
+#define    CNXK_R_IN_CTL_D_NSR                   (0x1ULL << 8)
+#define    CNXK_R_IN_CTL_D_ESR                   (0x1ULL << 6)
+#define    CNXK_R_IN_CTL_D_ROR                   (0x1ULL << 5)
+#define    CNXK_R_IN_CTL_NSR                     (0x1ULL << 3)
+#define    CNXK_R_IN_CTL_ESR                     (0x1ULL << 1)
+#define    CNXK_R_IN_CTL_ROR                     (0x1ULL << 0)
+
+#define    CNXK_R_IN_CTL_MASK  (CNXK_R_IN_CTL_RDSIZE | CNXK_R_IN_CTL_IS_64B)
+
+/* ##### RING OUT (out from device to PCI host: Rx Ring) REGISTERS #### */
+#define    CNXK_SDP_R_OUT_CNTS_START              0x10100
+#define    CNXK_SDP_R_OUT_INT_LEVELS_START        0x10110
+#define    CNXK_SDP_R_OUT_SLIST_BADDR_START       0x10120
+#define    CNXK_SDP_R_OUT_SLIST_RSIZE_START       0x10130
+#define    CNXK_SDP_R_OUT_SLIST_DBELL_START       0x10140
+#define    CNXK_SDP_R_OUT_CONTROL_START           0x10150
+#define    CNXK_SDP_R_OUT_WMARK_START             0x10160
+#define    CNXK_SDP_R_OUT_ENABLE_START            0x10170
+#define    CNXK_SDP_R_OUT_PKT_CNT_START           0x10180
+#define    CNXK_SDP_R_OUT_BYTE_CNT_START          0x10190
+
+#define    CNXK_SDP_R_OUT_CONTROL(ring)          \
+	(CNXK_SDP_R_OUT_CONTROL_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_OUT_ENABLE(ring)          \
+	(CNXK_SDP_R_OUT_ENABLE_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_OUT_SLIST_BADDR(ring)          \
+	(CNXK_SDP_R_OUT_SLIST_BADDR_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_OUT_SLIST_RSIZE(ring)          \
+	(CNXK_SDP_R_OUT_SLIST_RSIZE_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_OUT_SLIST_DBELL(ring)          \
+	(CNXK_SDP_R_OUT_SLIST_DBELL_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_OUT_CNTS(ring)          \
+	(CNXK_SDP_R_OUT_CNTS_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_OUT_INT_LEVELS(ring)          \
+	(CNXK_SDP_R_OUT_INT_LEVELS_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_OUT_PKT_CNT(ring)          \
+	(CNXK_SDP_R_OUT_PKT_CNT_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_OUT_BYTE_CNT(ring)          \
+	(CNXK_SDP_R_OUT_BYTE_CNT_START + ((ring) * CNXK_RING_OFFSET))
+
+/*------------------ R_OUT Masks ----------------*/
+#define    CNXK_R_OUT_INT_LEVELS_BMODE            BIT_ULL(63)
+#define    CNXK_R_OUT_INT_LEVELS_TIMET            (32)
+
+#define    CNXK_R_OUT_CTL_IDLE                    BIT_ULL(40)
+#define    CNXK_R_OUT_CTL_ES_I                    BIT_ULL(34)
+#define    CNXK_R_OUT_CTL_NSR_I                   BIT_ULL(33)
+#define    CNXK_R_OUT_CTL_ROR_I                   BIT_ULL(32)
+#define    CNXK_R_OUT_CTL_ES_D                    BIT_ULL(30)
+#define    CNXK_R_OUT_CTL_NSR_D                   BIT_ULL(29)
+#define    CNXK_R_OUT_CTL_ROR_D                   BIT_ULL(28)
+#define    CNXK_R_OUT_CTL_ES_P                    BIT_ULL(26)
+#define    CNXK_R_OUT_CTL_NSR_P                   BIT_ULL(25)
+#define    CNXK_R_OUT_CTL_ROR_P                   BIT_ULL(24)
+#define    CNXK_R_OUT_CTL_IMODE                   BIT_ULL(23)
+
+/* ############### Interrupt Moderation Registers ############### */
+#define CNXK_SDP_R_IN_INT_MDRT_CTL0_START         0x10280
+#define CNXK_SDP_R_IN_INT_MDRT_CTL1_START         0x102A0
+#define CNXK_SDP_R_IN_INT_MDRT_DBG_START          0x102C0
+
+#define CNXK_SDP_R_OUT_INT_MDRT_CTL0_START        0x10380
+#define CNXK_SDP_R_OUT_INT_MDRT_CTL1_START        0x103A0
+#define CNXK_SDP_R_OUT_INT_MDRT_DBG_START         0x103C0
+
+#define CNXK_SDP_R_OUT_CNTS_ISM_START             0x10510
+#define CNXK_SDP_R_IN_CNTS_ISM_START              0x10520
+
+#define    CNXK_SDP_R_IN_INT_MDRT_CTL0(ring)		\
+	(CNXK_SDP_R_IN_INT_MDRT_CTL0_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_IN_INT_MDRT_CTL1(ring)		\
+	(CNXK_SDP_R_IN_INT_MDRT_CTL1_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_IN_INT_MDRT_DBG(ring)		\
+	(CNXK_SDP_R_IN_INT_MDRT_DBG_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_OUT_INT_MDRT_CTL0(ring)		\
+	(CNXK_SDP_R_OUT_INT_MDRT_CTL0_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_OUT_INT_MDRT_CTL1(ring)		\
+	(CNXK_SDP_R_OUT_INT_MDRT_CTL1_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_OUT_INT_MDRT_DBG(ring)		\
+	(CNXK_SDP_R_OUT_INT_MDRT_DBG_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_OUT_CNTS_ISM(ring)          \
+	(CNXK_SDP_R_OUT_CNTS_ISM_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_IN_CNTS_ISM(ring)          \
+	(CNXK_SDP_R_IN_CNTS_ISM_START + ((ring) * CNXK_RING_OFFSET))
+
+/* ##################### Mail Box Registers ########################## */
+/* INT register for VF. when a MBOX write from PF happed to a VF,
+ * corresponding bit will be set in this register as well as in
+ * PF_VF_INT register.
+ *
+ * This is a RO register, the int can be cleared by writing 1 to PF_VF_INT
+ */
+/* Basically first 3 are from PF to VF. The last one is data from VF to PF */
+#define    CNXK_SDP_R_MBOX_PF_VF_DATA_START       0x10210
+#define    CNXK_SDP_R_MBOX_PF_VF_INT_START        0x10220
+#define    CNXK_SDP_R_MBOX_VF_PF_DATA_START       0x10230
+
+#define    CNXK_SDP_R_MBOX_PF_VF_DATA(ring)		\
+	(CNXK_SDP_R_MBOX_PF_VF_DATA_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_MBOX_PF_VF_INT(ring)		\
+	(CNXK_SDP_R_MBOX_PF_VF_INT_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_MBOX_VF_PF_DATA(ring)		\
+	(CNXK_SDP_R_MBOX_VF_PF_DATA_START + ((ring) * CNXK_RING_OFFSET))
+
+/* ##################### Interrupt Registers ########################## */
+#define	   CNXK_SDP_R_ERR_TYPE_START	          0x10400
+
+#define    CNXK_SDP_R_ERR_TYPE(ring)		\
+	(CNXK_SDP_R_ERR_TYPE_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_MBOX_ISM_START              0x10500
+#define    CNXK_SDP_R_OUT_CNTS_ISM_START          0x10510
+#define    CNXK_SDP_R_IN_CNTS_ISM_START           0x10520
+
+#define    CNXK_SDP_R_MBOX_ISM(ring)		\
+	(CNXK_SDP_R_MBOX_ISM_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_OUT_CNTS_ISM(ring)	\
+	(CNXK_SDP_R_OUT_CNTS_ISM_START + ((ring) * CNXK_RING_OFFSET))
+
+#define    CNXK_SDP_R_IN_CNTS_ISM(ring)		\
+	(CNXK_SDP_R_IN_CNTS_ISM_START + ((ring) * CNXK_RING_OFFSET))
+
+#define	   CNXK_SDP_EPF_MBOX_RINT_START	          0x20100
+#define	   CNXK_SDP_EPF_MBOX_RINT_W1S_START	  0x20120
+#define	   CNXK_SDP_EPF_MBOX_RINT_ENA_W1C_START   0x20140
+#define	   CNXK_SDP_EPF_MBOX_RINT_ENA_W1S_START   0x20160
+
+#define	   CNXK_SDP_EPF_VFIRE_RINT_START          0x20180
+#define	   CNXK_SDP_EPF_VFIRE_RINT_W1S_START      0x201A0
+#define	   CNXK_SDP_EPF_VFIRE_RINT_ENA_W1C_START  0x201C0
+#define	   CNXK_SDP_EPF_VFIRE_RINT_ENA_W1S_START  0x201E0
+
+#define	   CNXK_SDP_EPF_IRERR_RINT                0x20200
+#define	   CNXK_SDP_EPF_IRERR_RINT_W1S            0x20210
+#define	   CNXK_SDP_EPF_IRERR_RINT_ENA_W1C        0x20220
+#define	   CNXK_SDP_EPF_IRERR_RINT_ENA_W1S        0x20230
+
+#define	   CNXK_SDP_EPF_VFORE_RINT_START          0x20240
+#define	   CNXK_SDP_EPF_VFORE_RINT_W1S_START      0x20260
+#define	   CNXK_SDP_EPF_VFORE_RINT_ENA_W1C_START  0x20280
+#define	   CNXK_SDP_EPF_VFORE_RINT_ENA_W1S_START  0x202A0
+
+#define	   CNXK_SDP_EPF_ORERR_RINT                0x20320
+#define	   CNXK_SDP_EPF_ORERR_RINT_W1S            0x20330
+#define	   CNXK_SDP_EPF_ORERR_RINT_ENA_W1C        0x20340
+#define	   CNXK_SDP_EPF_ORERR_RINT_ENA_W1S        0x20350
+
+#define	   CNXK_SDP_EPF_OEI_RINT                  0x20400
+#define	   CNXK_SDP_EPF_OEI_RINT_W1S              0x20500
+#define	   CNXK_SDP_EPF_OEI_RINT_ENA_W1C          0x20600
+#define	   CNXK_SDP_EPF_OEI_RINT_ENA_W1S          0x20700
+
+#define	   CNXK_SDP_EPF_DMA_RINT                  0x20800
+#define	   CNXK_SDP_EPF_DMA_RINT_W1S              0x20810
+#define	   CNXK_SDP_EPF_DMA_RINT_ENA_W1C          0x20820
+#define	   CNXK_SDP_EPF_DMA_RINT_ENA_W1S          0x20830
+
+#define	   CNXK_SDP_EPF_DMA_INT_LEVEL_START	    0x20840
+#define	   CNXK_SDP_EPF_DMA_CNT_START	            0x20860
+#define	   CNXK_SDP_EPF_DMA_TIM_START	            0x20880
+
+#define	   CNXK_SDP_EPF_MISC_RINT                 0x208A0
+#define	   CNXK_SDP_EPF_MISC_RINT_W1S	            0x208B0
+#define	   CNXK_SDP_EPF_MISC_RINT_ENA_W1C         0x208C0
+#define	   CNXK_SDP_EPF_MISC_RINT_ENA_W1S         0x208D0
+
+#define	   CNXK_SDP_EPF_DMA_VF_RINT_START           0x208E0
+#define	   CNXK_SDP_EPF_DMA_VF_RINT_W1S_START       0x20900
+#define	   CNXK_SDP_EPF_DMA_VF_RINT_ENA_W1C_START   0x20920
+#define	   CNXK_SDP_EPF_DMA_VF_RINT_ENA_W1S_START   0x20940
+
+#define	   CNXK_SDP_EPF_PP_VF_RINT_START            0x20960
+#define	   CNXK_SDP_EPF_PP_VF_RINT_W1S_START        0x20980
+#define	   CNXK_SDP_EPF_PP_VF_RINT_ENA_W1C_START    0x209A0
+#define	   CNXK_SDP_EPF_PP_VF_RINT_ENA_W1S_START    0x209C0
+
+#define	   CNXK_SDP_EPF_MBOX_RINT(index)		\
+		(CNXK_SDP_EPF_MBOX_RINT_START + ((index) * CNXK_BIT_ARRAY_OFFSET))
+#define	   CNXK_SDP_EPF_MBOX_RINT_W1S(index)		\
+		(CNXK_SDP_EPF_MBOX_RINT_W1S_START + ((index) * CNXK_BIT_ARRAY_OFFSET))
+#define	   CNXK_SDP_EPF_MBOX_RINT_ENA_W1C(index)	\
+		(CNXK_SDP_EPF_MBOX_RINT_ENA_W1C_START + ((index) * CNXK_BIT_ARRAY_OFFSET))
+#define	   CNXK_SDP_EPF_MBOX_RINT_ENA_W1S(index)	\
+		(CNXK_SDP_EPF_MBOX_RINT_ENA_W1S_START + ((index) * CNXK_BIT_ARRAY_OFFSET))
+
+#define	   CNXK_SDP_EPF_VFIRE_RINT(index)		\
+		(CNXK_SDP_EPF_VFIRE_RINT_START + ((index) * CNXK_BIT_ARRAY_OFFSET))
+#define	   CNXK_SDP_EPF_VFIRE_RINT_W1S(index)		\
+		(CNXK_SDP_EPF_VFIRE_RINT_W1S_START + ((index) * CNXK_BIT_ARRAY_OFFSET))
+#define	   CNXK_SDP_EPF_VFIRE_RINT_ENA_W1C(index)	\
+		(CNXK_SDP_EPF_VFIRE_RINT_ENA_W1C_START + ((index) * CNXK_BIT_ARRAY_OFFSET))
+#define	   CNXK_SDP_EPF_VFIRE_RINT_ENA_W1S(index)	\
+		(CNXK_SDP_EPF_VFIRE_RINT_ENA_W1S_START + ((index) * CNXK_BIT_ARRAY_OFFSET))
+
+#define	   CNXK_SDP_EPF_VFORE_RINT(index)		\
+		(CNXK_SDP_EPF_VFORE_RINT_START + ((index) * CNXK_BIT_ARRAY_OFFSET))
+#define	   CNXK_SDP_EPF_VFORE_RINT_W1S(index)		\
+		(CNXK_SDP_EPF_VFORE_RINT_W1S_START + ((index) * CNXK_BIT_ARRAY_OFFSET))
+#define	   CNXK_SDP_EPF_VFORE_RINT_ENA_W1C(index)	\
+		(CNXK_SDP_EPF_VFORE_RINT_ENA_W1C_START + ((index) * CNXK_BIT_ARRAY_OFFSET))
+#define	   CNXK_SDP_EPF_VFORE_RINT_ENA_W1S(index)	\
+		(CNXK_SDP_EPF_VFORE_RINT_ENA_W1S_START + ((index) * CNXK_BIT_ARRAY_OFFSET))
+
+#define	   CNXK_SDP_EPF_DMA_VF_RINT(index)		\
+		(CNXK_SDP_EPF_DMA_VF_RINT_START + ((index) + CNXK_BIT_ARRAY_OFFSET))
+#define	   CNXK_SDP_EPF_DMA_VF_RINT_W1S(index)		\
+		(CNXK_SDP_EPF_DMA_VF_RINT_W1S_START + ((index) + CNXK_BIT_ARRAY_OFFSET))
+#define	   CNXK_SDP_EPF_DMA_VF_RINT_ENA_W1C(index)	\
+		(CNXK_SDP_EPF_DMA_VF_RINT_ENA_W1C_START + ((index) + CNXK_BIT_ARRAY_OFFSET))
+#define	   CNXK_SDP_EPF_DMA_VF_RINT_ENA_W1S(index)	\
+		(CNXK_SDP_EPF_DMA_VF_RINT_ENA_W1S_START + ((index) + CNXK_BIT_ARRAY_OFFSET))
+
+#define	   CNXK_SDP_EPF_PP_VF_RINT(index)		\
+		(CNXK_SDP_EPF_PP_VF_RINT_START + ((index) + CNXK_BIT_ARRAY_OFFSET))
+#define	   CNXK_SDP_EPF_PP_VF_RINT_W1S(index)		\
+		(CNXK_SDP_EPF_PP_VF_RINT_W1S_START + ((index) + CNXK_BIT_ARRAY_OFFSET))
+#define	   CNXK_SDP_EPF_PP_VF_RINT_ENA_W1C(index)	\
+		(CNXK_SDP_EPF_PP_VF_RINT_ENA_W1C_START + ((index) + CNXK_BIT_ARRAY_OFFSET))
+#define	   CNXK_SDP_EPF_PP_VF_RINT_ENA_W1S(index)	\
+		(CNXK_SDP_EPF_PP_VF_RINT_ENA_W1S_START + ((index) + CNXK_BIT_ARRAY_OFFSET))
+
+/*------------------ Interrupt Masks ----------------*/
+#define	   CNXK_INTR_R_SEND_ISM       BIT_ULL(63)
+#define	   CNXK_INTR_R_OUT_INT        BIT_ULL(62)
+#define    CNXK_INTR_R_IN_INT         BIT_ULL(61)
+#define    CNXK_INTR_R_MBOX_INT       BIT_ULL(60)
+#define    CNXK_INTR_R_RESEND         BIT_ULL(59)
+#define    CNXK_INTR_R_CLR_TIM        BIT_ULL(58)
+
+/* ####################### Ring Mapping Registers ################################## */
+#define    CNXK_SDP_EPVF_RING_START          0x26000
+#define    CNXK_SDP_IN_RING_TB_MAP_START     0x28000
+#define    CNXK_SDP_IN_RATE_LIMIT_START      0x2A000
+#define    CNXK_SDP_MAC_PF_RING_CTL_START    0x2C000
+
+#define	   CNXK_SDP_EPVF_RING(ring)		\
+		(CNXK_SDP_EPVF_RING_START + ((ring) * CNXK_EPVF_RING_OFFSET))
+#define	   CNXK_SDP_IN_RING_TB_MAP(ring)	\
+		(CNXK_SDP_N_RING_TB_MAP_START + ((ring) * CNXK_EPVF_RING_OFFSET))
+#define	   CNXK_SDP_IN_RATE_LIMIT(ring)		\
+		(CNXK_SDP_IN_RATE_LIMIT_START + ((ring) * CNXK_EPVF_RING_OFFSET))
+#define	   CNXK_SDP_MAC_PF_RING_CTL(mac)	\
+		(CNXK_SDP_MAC_PF_RING_CTL_START + ((mac) * CNXK_MAC_OFFSET))
+
+#define    CNXK_SDP_MAC_PF_RING_CTL_NPFS(val)  ((val) & 0x3)
+#define    CNXK_SDP_MAC_PF_RING_CTL_SRN(val)   (((val) >> 8) & 0x7F)
+#define    CNXK_SDP_MAC_PF_RING_CTL_RPPF(val)  (((val) >> 16) & 0x3F)
+
+/* Number of non-queue interrupts in CNXKxx */
+#define    CNXK_NUM_NON_IOQ_INTR    32
+
+/* bit 0 for control mbox interrupt */
+#define CNXK_SDP_EPF_OEI_RINT_DATA_BIT_MBOX	BIT_ULL(0)
+/* bit 1 for firmware heartbeat interrupt */
+#define CNXK_SDP_EPF_OEI_RINT_DATA_BIT_HBEAT	BIT_ULL(1)
+#define FW_STATUS_RUNNING      2ULL
+#define CNXK_PEMX_PFX_CSX_PFCFGX(pem, pf, offset)      ({ typeof(offset) _off = (offset); \
+							  ((0x8e0000008000 | \
+							    (uint64_t)(pem) << 36 \
+							    | (pf) << 18 \
+							    | ((_off >> 16) & 1) << 16 \
+							    | (_off >> 3) << 3) \
+							   + (((_off >> 2) & 1) << 2)); \
+							})
+
+/* Register defines for use with CNXK_PEMX_PFX_CSX_PFCFGX */
+#define CNXK_PCIEEP_VSECST_CTL  0x418
+
+#define CNXK_PEM_BAR4_INDEX		7
+#define CNXK_PEM_BAR4_INDEX_SIZE	0x400000ULL
+#define CNXK_PEM_BAR4_INDEX_OFFSET	(CNXK_PEM_BAR4_INDEX * CNXK_PEM_BAR4_INDEX_SIZE)
+
+#endif /* _OCTEP_REGS_CNXK_PF_H_ */
diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_tx.c b/drivers/net/ethernet/marvell/octeon_ep/octep_tx.c
index d0adb82d65c3..06851b78aa28 100644
--- a/drivers/net/ethernet/marvell/octeon_ep/octep_tx.c
+++ b/drivers/net/ethernet/marvell/octeon_ep/octep_tx.c
@@ -21,7 +21,6 @@ static void octep_iq_reset_indices(struct octep_iq *iq)
 	iq->flush_index = 0;
 	iq->pkts_processed = 0;
 	iq->pkt_in_done = 0;
-	atomic_set(&iq->instr_pending, 0);
 }
 
 /**
@@ -82,7 +81,6 @@ int octep_iq_process_completions(struct octep_iq *iq, u16 budget)
 	}
 
 	iq->pkts_processed += compl_pkts;
-	atomic_sub(compl_pkts, &iq->instr_pending);
 	iq->stats.instr_completed += compl_pkts;
 	iq->stats.bytes_sent += compl_bytes;
 	iq->stats.sgentry_sent += compl_sg;
@@ -91,7 +89,7 @@ int octep_iq_process_completions(struct octep_iq *iq, u16 budget)
 	netdev_tx_completed_queue(iq->netdev_q, compl_pkts, compl_bytes);
 
 	if (unlikely(__netif_subqueue_stopped(iq->netdev, iq->q_no)) &&
-	    ((iq->max_count - atomic_read(&iq->instr_pending)) >
+	    (IQ_INSTR_SPACE(iq) >
 	     OCTEP_WAKE_QUEUE_THRESHOLD))
 		netif_wake_subqueue(iq->netdev, iq->q_no);
 	return !budget;
@@ -144,7 +142,6 @@ static void octep_iq_free_pending(struct octep_iq *iq)
 		dev_kfree_skb_any(skb);
 	}
 
-	atomic_set(&iq->instr_pending, 0);
 	iq->flush_index = fi;
 	netdev_tx_reset_queue(netdev_get_tx_queue(iq->netdev, iq->q_no));
 }
diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_tx.h b/drivers/net/ethernet/marvell/octeon_ep/octep_tx.h
index 86c98b13fc44..1ba4ff65e54d 100644
--- a/drivers/net/ethernet/marvell/octeon_ep/octep_tx.h
+++ b/drivers/net/ethernet/marvell/octeon_ep/octep_tx.h
@@ -172,9 +172,6 @@ struct octep_iq {
 	/* Statistics for this input queue. */
 	struct octep_iq_stats stats;
 
-	/* This field keeps track of the instructions pending in this queue. */
-	atomic_t instr_pending;
-
 	/* Pointer to the Virtual Base addr of the input ring. */
 	struct octep_tx_desc_hw *desc_ring;
 
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
index 91b99fd70361..ba95ac913274 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
@@ -1934,6 +1934,8 @@ int otx2_stop(struct net_device *netdev)
 	/* Clear RSS enable flag */
 	rss = &pf->hw.rss_info;
 	rss->enable = false;
+	if (!netif_is_rxfh_configured(netdev))
+		kfree(rss->rss_ctx[DEFAULT_RSS_CONTEXT_GROUP]);
 
 	/* Cleanup Queue IRQ */
 	vec = pci_irq_vector(pf->pdev,
diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
index 3cf6589cfdac..a6e91573f8da 100644
--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
@@ -1159,15 +1159,18 @@ static int mtk_init_fq_dma(struct mtk_eth *eth)
 	phy_ring_tail = eth->phy_scratch_ring + soc->txrx.txd_size * (cnt - 1);
 
 	for (i = 0; i < cnt; i++) {
+		dma_addr_t addr = dma_addr + i * MTK_QDMA_PAGE_SIZE;
 		struct mtk_tx_dma_v2 *txd;
 
 		txd = eth->scratch_ring + i * soc->txrx.txd_size;
-		txd->txd1 = dma_addr + i * MTK_QDMA_PAGE_SIZE;
+		txd->txd1 = addr;
 		if (i < cnt - 1)
 			txd->txd2 = eth->phy_scratch_ring +
 				    (i + 1) * soc->txrx.txd_size;
 
 		txd->txd3 = TX_DMA_PLEN0(MTK_QDMA_PAGE_SIZE);
+		if (MTK_HAS_CAPS(soc->caps, MTK_36BIT_DMA))
+			txd->txd3 |= TX_DMA_PREP_ADDR64(addr);
 		txd->txd4 = 0;
 		if (mtk_is_netsys_v2_or_greater(eth)) {
 			txd->txd5 = 0;
diff --git a/drivers/net/ethernet/mediatek/mtk_wed.c b/drivers/net/ethernet/mediatek/mtk_wed.c
index 9a6744c0d458..c895e265ae0e 100644
--- a/drivers/net/ethernet/mediatek/mtk_wed.c
+++ b/drivers/net/ethernet/mediatek/mtk_wed.c
@@ -670,7 +670,7 @@ mtk_wed_tx_buffer_alloc(struct mtk_wed_device *dev)
 		void *buf;
 		int s;
 
-		page = __dev_alloc_pages(GFP_KERNEL, 0);
+		page = __dev_alloc_page(GFP_KERNEL);
 		if (!page)
 			return -ENOMEM;
 
@@ -691,10 +691,11 @@ mtk_wed_tx_buffer_alloc(struct mtk_wed_device *dev)
 
 		for (s = 0; s < MTK_WED_BUF_PER_PAGE; s++) {
 			struct mtk_wdma_desc *desc = desc_ptr;
+			u32 ctrl;
 
 			desc->buf0 = cpu_to_le32(buf_phys);
 			if (!mtk_wed_is_v3_or_greater(dev->hw)) {
-				u32 txd_size, ctrl;
+				u32 txd_size;
 
 				txd_size = dev->wlan.init_buf(buf, buf_phys,
 							      token++);
@@ -708,11 +709,11 @@ mtk_wed_tx_buffer_alloc(struct mtk_wed_device *dev)
 					ctrl |= MTK_WDMA_DESC_CTRL_LAST_SEG0 |
 						FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN1_V2,
 							   MTK_WED_BUF_SIZE - txd_size);
-				desc->ctrl = cpu_to_le32(ctrl);
 				desc->info = 0;
 			} else {
-				desc->ctrl = cpu_to_le32(token << 16);
+				ctrl = token << 16 | TX_DMA_PREP_ADDR64(buf_phys);
 			}
+			desc->ctrl = cpu_to_le32(ctrl);
 
 			desc_ptr += desc_size;
 			buf += MTK_WED_BUF_SIZE;
@@ -811,6 +812,7 @@ mtk_wed_hwrro_buffer_alloc(struct mtk_wed_device *dev)
 		buf_phys = page_phys;
 		for (s = 0; s < MTK_WED_RX_BUF_PER_PAGE; s++) {
 			desc->buf0 = cpu_to_le32(buf_phys);
+			desc->token = cpu_to_le32(RX_DMA_PREP_ADDR64(buf_phys));
 			buf_phys += MTK_WED_PAGE_BUF_SIZE;
 			desc++;
 		}
diff --git a/drivers/net/ethernet/mediatek/mtk_wed_wo.c b/drivers/net/ethernet/mediatek/mtk_wed_wo.c
index 3bd51a3d6650..7ffbd4fca881 100644
--- a/drivers/net/ethernet/mediatek/mtk_wed_wo.c
+++ b/drivers/net/ethernet/mediatek/mtk_wed_wo.c
@@ -142,7 +142,8 @@ mtk_wed_wo_queue_refill(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q,
 		dma_addr_t addr;
 		void *buf;
 
-		buf = page_frag_alloc(&q->cache, q->buf_size, GFP_ATOMIC);
+		buf = page_frag_alloc(&q->cache, q->buf_size,
+				      GFP_ATOMIC | GFP_DMA32);
 		if (!buf)
 			break;
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/crdump.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/crdump.c
index 28d02749d3c4..7659ad21e6e5 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/diag/crdump.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/crdump.c
@@ -55,7 +55,10 @@ int mlx5_crdump_collect(struct mlx5_core_dev *dev, u32 *cr_data)
 	ret = mlx5_vsc_sem_set_space(dev, MLX5_SEMAPHORE_SW_RESET,
 				     MLX5_VSC_LOCK);
 	if (ret) {
-		mlx5_core_warn(dev, "Failed to lock SW reset semaphore\n");
+		if (ret == -EBUSY)
+			mlx5_core_info(dev, "SW reset semaphore is already in use\n");
+		else
+			mlx5_core_warn(dev, "Failed to lock SW reset semaphore\n");
 		goto unlock_gw;
 	}
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c
index b12fe3c5a258..a55452c69f06 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c
@@ -147,6 +147,20 @@ mlx5e_rep_setup_tc_cls_flower(struct mlx5e_priv *priv,
 	}
 }
 
+static void mlx5e_tc_stats_matchall(struct mlx5e_priv *priv,
+				    struct tc_cls_matchall_offload *ma)
+{
+	struct mlx5e_rep_priv *rpriv = priv->ppriv;
+	u64 dbytes;
+	u64 dpkts;
+
+	dpkts = priv->stats.rep_stats.vport_rx_packets - rpriv->prev_vf_vport_stats.rx_packets;
+	dbytes = priv->stats.rep_stats.vport_rx_bytes - rpriv->prev_vf_vport_stats.rx_bytes;
+	mlx5e_stats_copy_rep_stats(&rpriv->prev_vf_vport_stats, &priv->stats.rep_stats);
+	flow_stats_update(&ma->stats, dbytes, dpkts, 0, jiffies,
+			  FLOW_ACTION_HW_STATS_DELAYED);
+}
+
 static
 int mlx5e_rep_setup_tc_cls_matchall(struct mlx5e_priv *priv,
 				    struct tc_cls_matchall_offload *ma)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/pedit.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/pedit.c
index 368a95fa77d3..b14cd62edffc 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/pedit.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/pedit.c
@@ -48,7 +48,8 @@ mlx5e_tc_act_pedit_parse_action(struct mlx5e_priv *priv,
 				struct pedit_headers_action *hdrs,
 				struct netlink_ext_ack *extack)
 {
-	u8 cmd = (act->id == FLOW_ACTION_MANGLE) ? 0 : 1;
+	u8 cmd = (act->id == FLOW_ACTION_MANGLE) ? TCA_PEDIT_KEY_EX_CMD_SET :
+						   TCA_PEDIT_KEY_EX_CMD_ADD;
 	u8 htype = act->mangle.htype;
 	int err = -EOPNOTSUPP;
 	u32 mask, val, offset;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index ea58c6917433..3aecdf099a2f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -5244,7 +5244,6 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev)
 
 	netdev->gso_partial_features             |= NETIF_F_GSO_UDP_L4;
 	netdev->hw_features                      |= NETIF_F_GSO_UDP_L4;
-	netdev->features                         |= NETIF_F_GSO_UDP_L4;
 
 	mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled);
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
index 3ab682bbcf86..fe0726c7b847 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
@@ -112,8 +112,18 @@ static const struct counter_desc vport_rep_stats_desc[] = {
 			     tx_vport_rdma_multicast_bytes) },
 };
 
+static const struct counter_desc vport_rep_loopback_stats_desc[] = {
+	{ MLX5E_DECLARE_STAT(struct mlx5e_rep_stats,
+			     vport_loopback_packets) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_rep_stats,
+			     vport_loopback_bytes) },
+};
+
 #define NUM_VPORT_REP_SW_COUNTERS ARRAY_SIZE(sw_rep_stats_desc)
 #define NUM_VPORT_REP_HW_COUNTERS ARRAY_SIZE(vport_rep_stats_desc)
+#define NUM_VPORT_REP_LOOPBACK_COUNTERS(dev) \
+	(MLX5_CAP_GEN(dev, vport_counter_local_loopback) ? \
+	 ARRAY_SIZE(vport_rep_loopback_stats_desc) : 0)
 
 static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(sw_rep)
 {
@@ -157,7 +167,8 @@ static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(sw_rep)
 
 static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(vport_rep)
 {
-	return NUM_VPORT_REP_HW_COUNTERS;
+	return NUM_VPORT_REP_HW_COUNTERS +
+	       NUM_VPORT_REP_LOOPBACK_COUNTERS(priv->mdev);
 }
 
 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(vport_rep)
@@ -166,6 +177,9 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(vport_rep)
 
 	for (i = 0; i < NUM_VPORT_REP_HW_COUNTERS; i++)
 		strcpy(data + (idx++) * ETH_GSTRING_LEN, vport_rep_stats_desc[i].format);
+	for (i = 0; i < NUM_VPORT_REP_LOOPBACK_COUNTERS(priv->mdev); i++)
+		strcpy(data + (idx++) * ETH_GSTRING_LEN,
+		       vport_rep_loopback_stats_desc[i].format);
 	return idx;
 }
 
@@ -176,6 +190,9 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(vport_rep)
 	for (i = 0; i < NUM_VPORT_REP_HW_COUNTERS; i++)
 		data[idx++] = MLX5E_READ_CTR64_CPU(&priv->stats.rep_stats,
 						   vport_rep_stats_desc, i);
+	for (i = 0; i < NUM_VPORT_REP_LOOPBACK_COUNTERS(priv->mdev); i++)
+		data[idx++] = MLX5E_READ_CTR64_CPU(&priv->stats.rep_stats,
+						   vport_rep_loopback_stats_desc, i);
 	return idx;
 }
 
@@ -247,6 +264,13 @@ static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(vport_rep)
 	rep_stats->tx_vport_rdma_multicast_bytes =
 		MLX5_GET_CTR(out, received_ib_multicast.octets);
 
+	if (MLX5_CAP_GEN(priv->mdev, vport_counter_local_loopback)) {
+		rep_stats->vport_loopback_packets =
+			MLX5_GET_CTR(out, local_loopback.packets);
+		rep_stats->vport_loopback_bytes =
+			MLX5_GET_CTR(out, local_loopback.octets);
+	}
+
 out:
 	kvfree(out);
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
index 477c547dcc04..12b3607afecd 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
@@ -476,6 +476,8 @@ struct mlx5e_rep_stats {
 	u64 tx_vport_rdma_multicast_packets;
 	u64 rx_vport_rdma_multicast_bytes;
 	u64 tx_vport_rdma_multicast_bytes;
+	u64 vport_loopback_packets;
+	u64 vport_loopback_bytes;
 };
 
 struct mlx5e_stats {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
index 7ca9e5b86778..9dca2809b46f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
@@ -3206,10 +3206,10 @@ static int offload_pedit_fields(struct mlx5e_priv *priv,
 	headers_c = mlx5e_get_match_headers_criteria(*action_flags, &parse_attr->spec);
 	headers_v = mlx5e_get_match_headers_value(*action_flags, &parse_attr->spec);
 
-	set_masks = &hdrs[0].masks;
-	add_masks = &hdrs[1].masks;
-	set_vals = &hdrs[0].vals;
-	add_vals = &hdrs[1].vals;
+	set_masks = &hdrs[TCA_PEDIT_KEY_EX_CMD_SET].masks;
+	add_masks = &hdrs[TCA_PEDIT_KEY_EX_CMD_ADD].masks;
+	set_vals = &hdrs[TCA_PEDIT_KEY_EX_CMD_SET].vals;
+	add_vals = &hdrs[TCA_PEDIT_KEY_EX_CMD_ADD].vals;
 
 	for (i = 0; i < ARRAY_SIZE(fields); i++) {
 		bool skip;
@@ -5011,22 +5011,6 @@ int mlx5e_tc_delete_matchall(struct mlx5e_priv *priv,
 	return apply_police_params(priv, 0, extack);
 }
 
-void mlx5e_tc_stats_matchall(struct mlx5e_priv *priv,
-			     struct tc_cls_matchall_offload *ma)
-{
-	struct mlx5e_rep_priv *rpriv = priv->ppriv;
-	struct rtnl_link_stats64 cur_stats;
-	u64 dbytes;
-	u64 dpkts;
-
-	mlx5e_stats_copy_rep_stats(&cur_stats, &priv->stats.rep_stats);
-	dpkts = cur_stats.rx_packets - rpriv->prev_vf_vport_stats.rx_packets;
-	dbytes = cur_stats.rx_bytes - rpriv->prev_vf_vport_stats.rx_bytes;
-	rpriv->prev_vf_vport_stats = cur_stats;
-	flow_stats_update(&ma->stats, dbytes, dpkts, 0, jiffies,
-			  FLOW_ACTION_HW_STATS_DELAYED);
-}
-
 static void mlx5e_tc_hairpin_update_dead_peer(struct mlx5e_priv *priv,
 					      struct mlx5e_priv *peer_priv)
 {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h
index adb39e30f90f..c24bda56b2b5 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h
@@ -203,8 +203,6 @@ int mlx5e_tc_configure_matchall(struct mlx5e_priv *priv,
 				struct tc_cls_matchall_offload *f);
 int mlx5e_tc_delete_matchall(struct mlx5e_priv *priv,
 			     struct tc_cls_matchall_offload *f);
-void mlx5e_tc_stats_matchall(struct mlx5e_priv *priv,
-			     struct tc_cls_matchall_offload *ma);
 
 struct mlx5e_encap_entry;
 void mlx5e_tc_encap_flows_add(struct mlx5e_priv *priv,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h
index 4aed1768b85f..78eb6b7097e1 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h
@@ -181,7 +181,7 @@ struct mlx5_flow_rule {
 
 struct mlx5_flow_handle {
 	int num_rules;
-	struct mlx5_flow_rule *rule[];
+	struct mlx5_flow_rule *rule[] __counted_by(num_rules);
 };
 
 /* Type of children is mlx5_flow_group */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c
index 17fe30a4c06c..0c26d707eed2 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c
@@ -539,7 +539,7 @@ struct mlx5_fc_bulk {
 	u32 base_id;
 	int bulk_len;
 	unsigned long *bitmask;
-	struct mlx5_fc fcs[];
+	struct mlx5_fc fcs[] __counted_by(bulk_len);
 };
 
 static void mlx5_fc_init(struct mlx5_fc *counter, struct mlx5_fc_bulk *bulk,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c
index b568988e92e3..4b8cb120362b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c
@@ -325,6 +325,25 @@ static void mlx5_fw_live_patch_event(struct work_struct *work)
 		mlx5_core_err(dev, "Failed to reload FW tracer\n");
 }
 
+static const struct pci_device_id mgt_ifc_device_ids[] = {
+	{ PCI_VDEVICE(MELLANOX, 0xc2d2) }, /* BlueField1 MGT interface device ID */
+	{ PCI_VDEVICE(MELLANOX, 0xc2d3) }, /* BlueField2 MGT interface device ID */
+	{ PCI_VDEVICE(MELLANOX, 0xc2d4) }, /* BlueField3-Lx MGT interface device ID */
+	{ PCI_VDEVICE(MELLANOX, 0xc2d5) }, /* BlueField3 MGT interface device ID */
+	{ PCI_VDEVICE(MELLANOX, 0xc2d6) }, /* BlueField4 MGT interface device ID */
+};
+
+static bool mlx5_is_mgt_ifc_pci_device(struct mlx5_core_dev *dev, u16 dev_id)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(mgt_ifc_device_ids); ++i)
+		if (mgt_ifc_device_ids[i].device == dev_id)
+			return true;
+
+	return false;
+}
+
 static int mlx5_check_dev_ids(struct mlx5_core_dev *dev, u16 dev_id)
 {
 	struct pci_bus *bridge_bus = dev->pdev->bus;
@@ -339,10 +358,15 @@ static int mlx5_check_dev_ids(struct mlx5_core_dev *dev, u16 dev_id)
 		err = pci_read_config_word(sdev, PCI_DEVICE_ID, &sdev_id);
 		if (err)
 			return pcibios_err_to_errno(err);
-		if (sdev_id != dev_id) {
-			mlx5_core_warn(dev, "unrecognized dev_id (0x%x)\n", sdev_id);
-			return -EPERM;
-		}
+
+		if (sdev_id == dev_id)
+			continue;
+
+		if (mlx5_is_mgt_ifc_pci_device(dev, sdev_id))
+			continue;
+
+		mlx5_core_warn(dev, "unrecognized dev_id (0x%x)\n", sdev_id);
+		return -EPERM;
 	}
 	return 0;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
index 0c83ef174275..0361741632a6 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
@@ -266,9 +266,6 @@ static int mlx5_ptp_settime_real_time(struct mlx5_core_dev *mdev,
 {
 	u32 in[MLX5_ST_SZ_DW(mtutc_reg)] = {};
 
-	if (!mlx5_modify_mtutc_allowed(mdev))
-		return 0;
-
 	if (ts->tv_sec < 0 || ts->tv_sec > U32_MAX ||
 	    ts->tv_nsec < 0 || ts->tv_nsec > NSEC_PER_SEC)
 		return -EINVAL;
@@ -286,12 +283,15 @@ static int mlx5_ptp_settime(struct ptp_clock_info *ptp, const struct timespec64
 	struct mlx5_timer *timer = &clock->timer;
 	struct mlx5_core_dev *mdev;
 	unsigned long flags;
-	int err;
 
 	mdev = container_of(clock, struct mlx5_core_dev, clock);
-	err = mlx5_ptp_settime_real_time(mdev, ts);
-	if (err)
-		return err;
+
+	if (mlx5_modify_mtutc_allowed(mdev)) {
+		int err = mlx5_ptp_settime_real_time(mdev, ts);
+
+		if (err)
+			return err;
+	}
 
 	write_seqlock_irqsave(&clock->lock, flags);
 	timecounter_init(&timer->tc, &timer->cycles, timespec64_to_ns(ts));
@@ -341,9 +341,6 @@ static int mlx5_ptp_adjtime_real_time(struct mlx5_core_dev *mdev, s64 delta)
 {
 	u32 in[MLX5_ST_SZ_DW(mtutc_reg)] = {};
 
-	if (!mlx5_modify_mtutc_allowed(mdev))
-		return 0;
-
 	/* HW time adjustment range is checked. If out of range, settime instead */
 	if (!mlx5_is_mtutc_time_adj_cap(mdev, delta)) {
 		struct timespec64 ts;
@@ -367,13 +364,16 @@ static int mlx5_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
 	struct mlx5_timer *timer = &clock->timer;
 	struct mlx5_core_dev *mdev;
 	unsigned long flags;
-	int err;
 
 	mdev = container_of(clock, struct mlx5_core_dev, clock);
 
-	err = mlx5_ptp_adjtime_real_time(mdev, delta);
-	if (err)
-		return err;
+	if (mlx5_modify_mtutc_allowed(mdev)) {
+		int err = mlx5_ptp_adjtime_real_time(mdev, delta);
+
+		if (err)
+			return err;
+	}
+
 	write_seqlock_irqsave(&clock->lock, flags);
 	timecounter_adjtime(&timer->tc, delta);
 	mlx5_update_clock_info_page(mdev);
@@ -396,15 +396,14 @@ static int mlx5_ptp_freq_adj_real_time(struct mlx5_core_dev *mdev, long scaled_p
 {
 	u32 in[MLX5_ST_SZ_DW(mtutc_reg)] = {};
 
-	if (!mlx5_modify_mtutc_allowed(mdev))
-		return 0;
-
 	MLX5_SET(mtutc_reg, in, operation, MLX5_MTUTC_OPERATION_ADJUST_FREQ_UTC);
 
-	if (MLX5_CAP_MCAM_FEATURE(mdev, mtutc_freq_adj_units)) {
+	if (MLX5_CAP_MCAM_FEATURE(mdev, mtutc_freq_adj_units) &&
+	    scaled_ppm <= S32_MAX && scaled_ppm >= S32_MIN) {
+		/* HW scaled_ppm support on mlx5 devices only supports a 32-bit value */
 		MLX5_SET(mtutc_reg, in, freq_adj_units,
 			 MLX5_MTUTC_FREQ_ADJ_UNITS_SCALED_PPM);
-		MLX5_SET(mtutc_reg, in, freq_adjustment, scaled_ppm);
+		MLX5_SET(mtutc_reg, in, freq_adjustment, (s32)scaled_ppm);
 	} else {
 		MLX5_SET(mtutc_reg, in, freq_adj_units, MLX5_MTUTC_FREQ_ADJ_UNITS_PPB);
 		MLX5_SET(mtutc_reg, in, freq_adjustment, scaled_ppm_to_ppb(scaled_ppm));
@@ -420,13 +419,15 @@ static int mlx5_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm)
 	struct mlx5_core_dev *mdev;
 	unsigned long flags;
 	u32 mult;
-	int err;
 
 	mdev = container_of(clock, struct mlx5_core_dev, clock);
 
-	err = mlx5_ptp_freq_adj_real_time(mdev, scaled_ppm);
-	if (err)
-		return err;
+	if (mlx5_modify_mtutc_allowed(mdev)) {
+		int err = mlx5_ptp_freq_adj_real_time(mdev, scaled_ppm);
+
+		if (err)
+			return err;
+	}
 
 	mult = (u32)adjust_by_scaled_ppm(timer->nominal_c_mult, scaled_ppm);
 
@@ -1004,14 +1005,38 @@ static void mlx5_init_clock_info(struct mlx5_core_dev *mdev)
 	info->frac = timer->tc.frac;
 }
 
+static void mlx5_init_timer_max_freq_adjustment(struct mlx5_core_dev *mdev)
+{
+	struct mlx5_clock *clock = &mdev->clock;
+	u32 out[MLX5_ST_SZ_DW(mtutc_reg)] = {};
+	u32 in[MLX5_ST_SZ_DW(mtutc_reg)] = {};
+	u8 log_max_freq_adjustment = 0;
+	int err;
+
+	err = mlx5_core_access_reg(mdev, in, sizeof(in), out, sizeof(out),
+				   MLX5_REG_MTUTC, 0, 0);
+	if (!err)
+		log_max_freq_adjustment =
+			MLX5_GET(mtutc_reg, out, log_max_freq_adjustment);
+
+	if (log_max_freq_adjustment)
+		clock->ptp_info.max_adj =
+			min(S32_MAX, 1 << log_max_freq_adjustment);
+}
+
 static void mlx5_init_timer_clock(struct mlx5_core_dev *mdev)
 {
 	struct mlx5_clock *clock = &mdev->clock;
 
+	/* Configure the PHC */
+	clock->ptp_info = mlx5_ptp_clock_info;
+
+	if (MLX5_CAP_MCAM_REG(mdev, mtutc))
+		mlx5_init_timer_max_freq_adjustment(mdev);
+
 	mlx5_timecounter_init(mdev);
 	mlx5_init_clock_info(mdev);
 	mlx5_init_overflow_period(clock);
-	clock->ptp_info = mlx5_ptp_clock_info;
 
 	if (mlx5_real_time_mode(mdev)) {
 		struct timespec64 ts;
@@ -1042,11 +1067,10 @@ void mlx5_init_clock(struct mlx5_core_dev *mdev)
 	}
 
 	seqlock_init(&clock->lock);
-	mlx5_init_timer_clock(mdev);
 	INIT_WORK(&clock->pps_info.out_work, mlx5_pps_out);
 
-	/* Configure the PHC */
-	clock->ptp_info = mlx5_ptp_clock_info;
+	/* Initialize the device clock */
+	mlx5_init_timer_clock(mdev);
 
 	/* Initialize 1PPS data structures */
 	mlx5_init_pps(mdev);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index a17152c1cbb2..bccf6e53556c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -219,7 +219,6 @@ static void mlx5_set_driver_version(struct mlx5_core_dev *dev)
 	int driver_ver_sz = MLX5_FLD_SZ_BYTES(set_driver_version_in,
 					      driver_version);
 	u8 in[MLX5_ST_SZ_BYTES(set_driver_version_in)] = {};
-	int remaining_size = driver_ver_sz;
 	char *string;
 
 	if (!MLX5_CAP_GEN(dev, driver_version))
@@ -227,22 +226,9 @@ static void mlx5_set_driver_version(struct mlx5_core_dev *dev)
 
 	string = MLX5_ADDR_OF(set_driver_version_in, in, driver_version);
 
-	strncpy(string, "Linux", remaining_size);
-
-	remaining_size = max_t(int, 0, driver_ver_sz - strlen(string));
-	strncat(string, ",", remaining_size);
-
-	remaining_size = max_t(int, 0, driver_ver_sz - strlen(string));
-	strncat(string, KBUILD_MODNAME, remaining_size);
-
-	remaining_size = max_t(int, 0, driver_ver_sz - strlen(string));
-	strncat(string, ",", remaining_size);
-
-	remaining_size = max_t(int, 0, driver_ver_sz - strlen(string));
-
-	snprintf(string + strlen(string), remaining_size, "%u.%u.%u",
-		LINUX_VERSION_MAJOR, LINUX_VERSION_PATCHLEVEL,
-		LINUX_VERSION_SUBLEVEL);
+	snprintf(string, driver_ver_sz, "Linux,%s,%u.%u.%u",
+		 KBUILD_MODNAME, LINUX_VERSION_MAJOR,
+		 LINUX_VERSION_PATCHLEVEL, LINUX_VERSION_SUBLEVEL);
 
 	/*Send the command*/
 	MLX5_SET(set_driver_version_in, in, opcode,
diff --git a/drivers/net/ethernet/mellanox/mlxsw/cmd.h b/drivers/net/ethernet/mellanox/mlxsw/cmd.h
index e827c78be114..e3271c845ee6 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/cmd.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/cmd.h
@@ -282,6 +282,12 @@ MLXSW_ITEM32(cmd_mbox, query_fw, fw_day, 0x14, 0, 8);
  */
 MLXSW_ITEM32(cmd_mbox, query_fw, lag_mode_support, 0x18, 1, 1);
 
+/* cmd_mbox_query_fw_cff_support
+ * 0: CONFIG_PROFILE.flood_mode = 5 (CFF) is not supported by FW
+ * 1: CONFIG_PROFILE.flood_mode = 5 (CFF) is supported by FW
+ */
+MLXSW_ITEM32(cmd_mbox, query_fw, cff_support, 0x18, 2, 1);
+
 /* cmd_mbox_query_fw_clr_int_base_offset
  * Clear Interrupt register's offset from clr_int_bar register
  * in PCI address space.
@@ -779,6 +785,11 @@ enum mlxsw_cmd_mbox_config_profile_flood_mode {
 	 * used.
 	 */
 	MLXSW_CMD_MBOX_CONFIG_PROFILE_FLOOD_MODE_CONTROLLED = 4,
+	/* CFF - Compressed FID Flood (CFF) mode.
+	 * Reserved when legacy bridge model is used.
+	 * Supported only by Spectrum-2+.
+	 */
+	MLXSW_CMD_MBOX_CONFIG_PROFILE_FLOOD_MODE_CFF = 5,
 };
 
 /* cmd_mbox_config_profile_flood_mode
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c
index f23421f038f3..e4d7739bd7c8 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/core.c
@@ -211,6 +211,13 @@ mlxsw_core_lag_mode(struct mlxsw_core *mlxsw_core)
 }
 EXPORT_SYMBOL(mlxsw_core_lag_mode);
 
+enum mlxsw_cmd_mbox_config_profile_flood_mode
+mlxsw_core_flood_mode(struct mlxsw_core *mlxsw_core)
+{
+	return mlxsw_core->bus->flood_mode(mlxsw_core->bus_priv);
+}
+EXPORT_SYMBOL(mlxsw_core_flood_mode);
+
 void *mlxsw_core_driver_priv(struct mlxsw_core *mlxsw_core)
 {
 	return mlxsw_core->driver_priv;
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.h b/drivers/net/ethernet/mellanox/mlxsw/core.h
index 764d14bd5bc0..6d11225594dd 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/core.h
@@ -38,6 +38,8 @@ unsigned int mlxsw_core_max_ports(const struct mlxsw_core *mlxsw_core);
 int mlxsw_core_max_lag(struct mlxsw_core *mlxsw_core, u16 *p_max_lag);
 enum mlxsw_cmd_mbox_config_profile_lag_mode
 mlxsw_core_lag_mode(struct mlxsw_core *mlxsw_core);
+enum mlxsw_cmd_mbox_config_profile_flood_mode
+mlxsw_core_flood_mode(struct mlxsw_core *mlxsw_core);
 
 void *mlxsw_core_driver_priv(struct mlxsw_core *mlxsw_core);
 
@@ -322,7 +324,12 @@ struct mlxsw_config_profile {
 	u16	max_regions;
 	u8	max_flood_tables;
 	u8	max_vid_flood_tables;
+
+	/* Flood mode to use if used_flood_mode. If flood_mode_prefer_cff,
+	 * the backup flood mode (if any) when CFF unsupported.
+	 */
 	u8	flood_mode;
+
 	u8	max_fid_offset_flood_tables;
 	u16	fid_offset_flood_table_size;
 	u8	max_fid_flood_tables;
@@ -338,6 +345,7 @@ struct mlxsw_config_profile {
 	u8	kvd_hash_double_parts;
 	u8	cqe_time_stamp_type;
 	bool	lag_mode_prefer_sw;
+	bool	flood_mode_prefer_cff;
 	struct mlxsw_swid_config swid_config[MLXSW_CONFIG_PROFILE_SWID_COUNT];
 };
 
@@ -489,6 +497,7 @@ struct mlxsw_bus {
 	u32 (*read_utc_sec)(void *bus_priv);
 	u32 (*read_utc_nsec)(void *bus_priv);
 	enum mlxsw_cmd_mbox_config_profile_lag_mode (*lag_mode)(void *bus_priv);
+	enum mlxsw_cmd_mbox_config_profile_flood_mode (*flood_mode)(void *priv);
 	u8 features;
 };
 
diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c
index e4b25e187467..0d58f13a7c7d 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/pci.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c
@@ -106,7 +106,9 @@ struct mlxsw_pci {
 	u64 utc_sec_offset;
 	u64 utc_nsec_offset;
 	bool lag_mode_support;
+	bool cff_support;
 	enum mlxsw_cmd_mbox_config_profile_lag_mode lag_mode;
+	enum mlxsw_cmd_mbox_config_profile_flood_mode flood_mode;
 	struct mlxsw_pci_queue_type_group queues[MLXSW_PCI_QUEUE_TYPE_COUNT];
 	u32 doorbell_offset;
 	struct mlxsw_core *core;
@@ -130,6 +132,7 @@ struct mlxsw_pci {
 	const struct pci_device_id *id;
 	enum mlxsw_pci_cqe_v max_cqe_ver; /* Maximal supported CQE version */
 	u8 num_sdq_cqs; /* Number of CQs used for SDQs */
+	bool skip_reset;
 };
 
 static void mlxsw_pci_queue_tasklet_schedule(struct mlxsw_pci_queue *q)
@@ -1245,11 +1248,22 @@ static int mlxsw_pci_config_profile(struct mlxsw_pci *mlxsw_pci, char *mbox,
 		mlxsw_cmd_mbox_config_profile_fid_flood_table_size_set(
 			mbox, profile->fid_flood_table_size);
 	}
-	if (profile->used_flood_mode) {
+	if (profile->flood_mode_prefer_cff && mlxsw_pci->cff_support) {
+		enum mlxsw_cmd_mbox_config_profile_flood_mode flood_mode =
+			MLXSW_CMD_MBOX_CONFIG_PROFILE_FLOOD_MODE_CFF;
+
+		mlxsw_cmd_mbox_config_profile_set_flood_mode_set(mbox, 1);
+		mlxsw_cmd_mbox_config_profile_flood_mode_set(mbox, flood_mode);
+		mlxsw_pci->flood_mode = flood_mode;
+	} else if (profile->used_flood_mode) {
 		mlxsw_cmd_mbox_config_profile_set_flood_mode_set(
 			mbox, 1);
 		mlxsw_cmd_mbox_config_profile_flood_mode_set(
 			mbox, profile->flood_mode);
+		mlxsw_pci->flood_mode = profile->flood_mode;
+	} else {
+		WARN_ON(1);
+		return -EINVAL;
 	}
 	if (profile->used_max_ib_mc) {
 		mlxsw_cmd_mbox_config_profile_set_max_ib_mc_set(
@@ -1476,11 +1490,47 @@ static int mlxsw_pci_sys_ready_wait(struct mlxsw_pci *mlxsw_pci,
 	return -EBUSY;
 }
 
-static int mlxsw_pci_sw_reset(struct mlxsw_pci *mlxsw_pci,
-			      const struct pci_device_id *id)
+static int mlxsw_pci_reset_at_pci_disable(struct mlxsw_pci *mlxsw_pci)
 {
 	struct pci_dev *pdev = mlxsw_pci->pdev;
 	char mrsr_pl[MLXSW_REG_MRSR_LEN];
+	int err;
+
+	mlxsw_reg_mrsr_pack(mrsr_pl,
+			    MLXSW_REG_MRSR_COMMAND_RESET_AT_PCI_DISABLE);
+	err = mlxsw_reg_write(mlxsw_pci->core, MLXSW_REG(mrsr), mrsr_pl);
+	if (err)
+		return err;
+
+	device_lock_assert(&pdev->dev);
+
+	pci_cfg_access_lock(pdev);
+	pci_save_state(pdev);
+
+	err = __pci_reset_function_locked(pdev);
+	if (err)
+		pci_err(pdev, "PCI function reset failed with %d\n", err);
+
+	pci_restore_state(pdev);
+	pci_cfg_access_unlock(pdev);
+
+	return err;
+}
+
+static int mlxsw_pci_reset_sw(struct mlxsw_pci *mlxsw_pci)
+{
+	char mrsr_pl[MLXSW_REG_MRSR_LEN];
+
+	mlxsw_reg_mrsr_pack(mrsr_pl, MLXSW_REG_MRSR_COMMAND_SOFTWARE_RESET);
+	return mlxsw_reg_write(mlxsw_pci->core, MLXSW_REG(mrsr), mrsr_pl);
+}
+
+static int
+mlxsw_pci_reset(struct mlxsw_pci *mlxsw_pci, const struct pci_device_id *id)
+{
+	struct pci_dev *pdev = mlxsw_pci->pdev;
+	char mcam_pl[MLXSW_REG_MCAM_LEN];
+	bool pci_reset_supported;
 	u32 sys_status;
 	int err;
 
@@ -1491,11 +1541,27 @@ static int mlxsw_pci_sw_reset(struct mlxsw_pci *mlxsw_pci,
 		return err;
 	}
 
-	mlxsw_reg_mrsr_pack(mrsr_pl);
-	err = mlxsw_reg_write(mlxsw_pci->core, MLXSW_REG(mrsr), mrsr_pl);
+	/* PCI core already issued a PCI reset, do not issue another reset. */
+	if (mlxsw_pci->skip_reset)
+		return 0;
+
+	mlxsw_reg_mcam_pack(mcam_pl,
+			    MLXSW_REG_MCAM_FEATURE_GROUP_ENHANCED_FEATURES);
+	err = mlxsw_reg_query(mlxsw_pci->core, MLXSW_REG(mcam), mcam_pl);
 	if (err)
 		return err;
 
+	mlxsw_reg_mcam_unpack(mcam_pl, MLXSW_REG_MCAM_PCI_RESET,
+			      &pci_reset_supported);
+
+	if (pci_reset_supported) {
+		pci_dbg(pdev, "Starting PCI reset flow\n");
+		err = mlxsw_pci_reset_at_pci_disable(mlxsw_pci);
+	} else {
+		pci_dbg(pdev, "Starting software reset flow\n");
+		err = mlxsw_pci_reset_sw(mlxsw_pci);
+	}
+
 	err = mlxsw_pci_sys_ready_wait(mlxsw_pci, id, &sys_status);
 	if (err) {
 		dev_err(&pdev->dev, "Failed to reach system ready status after reset. Status is 0x%x\n",
@@ -1537,9 +1603,9 @@ static int mlxsw_pci_init(void *bus_priv, struct mlxsw_core *mlxsw_core,
 	if (!mbox)
 		return -ENOMEM;
 
-	err = mlxsw_pci_sw_reset(mlxsw_pci, mlxsw_pci->id);
+	err = mlxsw_pci_reset(mlxsw_pci, mlxsw_pci->id);
 	if (err)
-		goto err_sw_reset;
+		goto err_reset;
 
 	err = mlxsw_pci_alloc_irq_vectors(mlxsw_pci);
 	if (err < 0) {
@@ -1601,6 +1667,9 @@ static int mlxsw_pci_init(void *bus_priv, struct mlxsw_core *mlxsw_core,
 
 	mlxsw_pci->lag_mode_support =
 		mlxsw_cmd_mbox_query_fw_lag_mode_support_get(mbox);
+	mlxsw_pci->cff_support =
+		mlxsw_cmd_mbox_query_fw_cff_support_get(mbox);
+
 	num_pages = mlxsw_cmd_mbox_query_fw_fw_pages_get(mbox);
 	err = mlxsw_pci_fw_area_init(mlxsw_pci, mbox, num_pages);
 	if (err)
@@ -1672,7 +1741,7 @@ static int mlxsw_pci_init(void *bus_priv, struct mlxsw_core *mlxsw_core,
 err_query_fw:
 	mlxsw_pci_free_irq_vectors(mlxsw_pci);
 err_alloc_irq:
-err_sw_reset:
+err_reset:
 mbox_put:
 	mlxsw_cmd_mbox_free(mbox);
 	return err;
@@ -1917,6 +1986,14 @@ mlxsw_pci_lag_mode(void *bus_priv)
 	return mlxsw_pci->lag_mode;
 }
 
+static enum mlxsw_cmd_mbox_config_profile_flood_mode
+mlxsw_pci_flood_mode(void *bus_priv)
+{
+	struct mlxsw_pci *mlxsw_pci = bus_priv;
+
+	return mlxsw_pci->flood_mode;
+}
+
 static const struct mlxsw_bus mlxsw_pci_bus = {
 	.kind			= "pci",
 	.init			= mlxsw_pci_init,
@@ -1929,6 +2006,7 @@ static const struct mlxsw_bus mlxsw_pci_bus = {
 	.read_utc_sec		= mlxsw_pci_read_utc_sec,
 	.read_utc_nsec		= mlxsw_pci_read_utc_nsec,
 	.lag_mode		= mlxsw_pci_lag_mode,
+	.flood_mode		= mlxsw_pci_flood_mode,
 	.features		= MLXSW_BUS_F_TXRX | MLXSW_BUS_F_RESET,
 };
 
@@ -2059,11 +2137,34 @@ static void mlxsw_pci_remove(struct pci_dev *pdev)
 	kfree(mlxsw_pci);
 }
 
+static void mlxsw_pci_reset_prepare(struct pci_dev *pdev)
+{
+	struct mlxsw_pci *mlxsw_pci = pci_get_drvdata(pdev);
+
+	mlxsw_core_bus_device_unregister(mlxsw_pci->core, false);
+}
+
+static void mlxsw_pci_reset_done(struct pci_dev *pdev)
+{
+	struct mlxsw_pci *mlxsw_pci = pci_get_drvdata(pdev);
+
+	mlxsw_pci->skip_reset = true;
+	mlxsw_core_bus_device_register(&mlxsw_pci->bus_info, &mlxsw_pci_bus,
+				       mlxsw_pci, false, NULL, NULL);
+	mlxsw_pci->skip_reset = false;
+}
+
+static const struct pci_error_handlers mlxsw_pci_err_handler = {
+	.reset_prepare = mlxsw_pci_reset_prepare,
+	.reset_done = mlxsw_pci_reset_done,
+};
+
 int mlxsw_pci_driver_register(struct pci_driver *pci_driver)
 {
 	pci_driver->probe = mlxsw_pci_probe;
 	pci_driver->remove = mlxsw_pci_remove;
 	pci_driver->shutdown = mlxsw_pci_remove;
+	pci_driver->err_handler = &mlxsw_pci_err_handler;
 	return pci_register_driver(pci_driver);
 }
 EXPORT_SYMBOL(mlxsw_pci_driver_register);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/reg.h b/drivers/net/ethernet/mellanox/mlxsw/reg.h
index 25b294fdeb3d..3aae4467e431 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/reg.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/reg.h
@@ -1024,6 +1024,8 @@ static inline void mlxsw_reg_spaft_pack(char *payload, u16 local_port,
  * ------------------------------------------
  * The following register controls the association of flooding tables and MIDs
  * to packet types used for flooding.
+ *
+ * Reserved when CONFIG_PROFILE.flood_mode = CFF.
  */
 #define MLXSW_REG_SFGC_ID 0x2011
 #define MLXSW_REG_SFGC_LEN 0x14
@@ -1862,6 +1864,7 @@ MLXSW_ITEM32(reg, sfmr, fid, 0x00, 0, 16);
  * Access: RW
  *
  * Note: Reserved when legacy bridge model is used.
+ * Reserved when CONFIG_PROFILE.flood_mode = CFF.
  */
 MLXSW_ITEM32(reg, sfmr, flood_rsp, 0x08, 31, 1);
 
@@ -1872,6 +1875,7 @@ MLXSW_ITEM32(reg, sfmr, flood_rsp, 0x08, 31, 1);
  * Access: RW
  *
  * Note: Reserved when legacy bridge model is used and when flood_rsp=1.
+ * Reserved when CONFIG_PROFILE.flood_mode = CFF
  */
 MLXSW_ITEM32(reg, sfmr, flood_bridge_type, 0x08, 28, 1);
 
@@ -1880,6 +1884,8 @@ MLXSW_ITEM32(reg, sfmr, flood_bridge_type, 0x08, 28, 1);
  * Used to point into the flooding table selected by SFGC register if
  * the table is of type FID-Offset. Otherwise, this field is reserved.
  * Access: RW
+ *
+ * Note: Reserved when CONFIG_PROFILE.flood_mode = CFF
  */
 MLXSW_ITEM32(reg, sfmr, fid_offset, 0x08, 0, 16);
 
@@ -1938,6 +1944,26 @@ MLXSW_ITEM32(reg, sfmr, irif_v, 0x14, 24, 1);
  */
 MLXSW_ITEM32(reg, sfmr, irif, 0x14, 0, 16);
 
+/* reg_sfmr_cff_mid_base
+ * Pointer to PGT table.
+ * Range: 0..(cap_max_pgt-1)
+ * Access: RW
+ *
+ * Note: Reserved when SwitchX/-2 and Spectrum-1.
+ * Supported when CONFIG_PROFILE.flood_mode = CFF.
+ */
+MLXSW_ITEM32(reg, sfmr, cff_mid_base, 0x20, 0, 16);
+
+/* reg_sfmr_cff_prf_id
+ * Compressed Fid Flooding profile_id
+ * Range 0..(max_cap_nve_flood_prf-1)
+ * Access: RW
+ *
+ * Note: Reserved when SwitchX/-2 and Spectrum-1
+ * Supported only when CONFIG_PROFLE.flood_mode = CFF.
+ */
+MLXSW_ITEM32(reg, sfmr, cff_prf_id, 0x24, 0, 2);
+
 /* reg_sfmr_smpe_valid
  * SMPE is valid.
  * Access: RW
@@ -1959,18 +1985,11 @@ MLXSW_ITEM32(reg, sfmr, smpe, 0x28, 0, 16);
 
 static inline void mlxsw_reg_sfmr_pack(char *payload,
 				       enum mlxsw_reg_sfmr_op op, u16 fid,
-				       u16 fid_offset, bool flood_rsp,
-				       enum mlxsw_reg_bridge_type bridge_type,
 				       bool smpe_valid, u16 smpe)
 {
 	MLXSW_REG_ZERO(sfmr, payload);
 	mlxsw_reg_sfmr_op_set(payload, op);
 	mlxsw_reg_sfmr_fid_set(payload, fid);
-	mlxsw_reg_sfmr_fid_offset_set(payload, fid_offset);
-	mlxsw_reg_sfmr_vtfp_set(payload, false);
-	mlxsw_reg_sfmr_vv_set(payload, false);
-	mlxsw_reg_sfmr_flood_rsp_set(payload, flood_rsp);
-	mlxsw_reg_sfmr_flood_bridge_type_set(payload, bridge_type);
 	mlxsw_reg_sfmr_smpe_valid_set(payload, smpe_valid);
 	mlxsw_reg_sfmr_smpe_set(payload, smpe);
 }
@@ -2168,6 +2187,50 @@ static inline void mlxsw_reg_spvc_pack(char *payload, u16 local_port, bool et1,
 	mlxsw_reg_spvc_et0_set(payload, et0);
 }
 
+/* SFFP - Switch FID Flooding Profiles Register
+ * --------------------------------------------
+ * The SFFP register populates the fid flooding profile tables used for the NVE
+ * flooding and Compressed-FID Flooding (CFF).
+ *
+ * Reserved on Spectrum-1.
+ */
+#define MLXSW_REG_SFFP_ID 0x2029
+#define MLXSW_REG_SFFP_LEN 0x0C
+
+MLXSW_REG_DEFINE(sffp, MLXSW_REG_SFFP_ID, MLXSW_REG_SFFP_LEN);
+
+/* reg_sffp_profile_id
+ * Profile ID a.k.a. SFMR.nve_flood_prf_id or SFMR.cff_prf_id
+ * Range 0..max_cap_nve_flood_prf-1
+ * Access: Index
+ */
+MLXSW_ITEM32(reg, sffp, profile_id, 0x00, 16, 2);
+
+/* reg_sffp_type
+ * The traffic type to reach the flooding table.
+ * Same as SFGC.type
+ * Access: Index
+ */
+MLXSW_ITEM32(reg, sffp, type, 0x00, 0, 4);
+
+/* reg_sffp_flood_offset
+ * Flood offset. Offset to add to SFMR.cff_mid_base to get the final PGT address
+ * for FID flood; or offset to add to SFMR.nve_tunnel_flood_ptr to get KVD
+ * pointer for NVE underlay.
+ * Access: RW
+ */
+MLXSW_ITEM32(reg, sffp, flood_offset, 0x04, 0, 3);
+
+static inline void mlxsw_reg_sffp_pack(char *payload, u8 profile_id,
+				       enum mlxsw_reg_sfgc_type type,
+				       u8 flood_offset)
+{
+	MLXSW_REG_ZERO(sffp, payload);
+	mlxsw_reg_sffp_profile_id_set(payload, profile_id);
+	mlxsw_reg_sffp_type_set(payload, type);
+	mlxsw_reg_sffp_flood_offset_set(payload, flood_offset);
+}
+
 /* SPEVET - Switch Port Egress VLAN EtherType
  * ------------------------------------------
  * The switch port egress VLAN EtherType configures which EtherType to push at
@@ -10122,6 +10185,15 @@ mlxsw_reg_mgir_unpack(char *payload, u32 *hw_rev, char *fw_info_psid,
 
 MLXSW_REG_DEFINE(mrsr, MLXSW_REG_MRSR_ID, MLXSW_REG_MRSR_LEN);
 
+enum mlxsw_reg_mrsr_command {
+	/* Switch soft reset, does not reset PCI firmware. */
+	MLXSW_REG_MRSR_COMMAND_SOFTWARE_RESET = 1,
+	/* Reset will be done when PCI link will be disabled.
+	 * This command will reset PCI firmware also.
+	 */
+	MLXSW_REG_MRSR_COMMAND_RESET_AT_PCI_DISABLE = 6,
+};
+
 /* reg_mrsr_command
  * Reset/shutdown command
  * 0 - do nothing
@@ -10130,10 +10202,11 @@ MLXSW_REG_DEFINE(mrsr, MLXSW_REG_MRSR_ID, MLXSW_REG_MRSR_LEN);
  */
 MLXSW_ITEM32(reg, mrsr, command, 0x00, 0, 4);
 
-static inline void mlxsw_reg_mrsr_pack(char *payload)
+static inline void mlxsw_reg_mrsr_pack(char *payload,
+				       enum mlxsw_reg_mrsr_command command)
 {
 	MLXSW_REG_ZERO(mrsr, payload);
-	mlxsw_reg_mrsr_command_set(payload, 1);
+	mlxsw_reg_mrsr_command_set(payload, command);
 }
 
 /* MLCR - Management LED Control Register
@@ -10584,6 +10657,8 @@ MLXSW_ITEM32(reg, mcam, feature_group, 0x00, 16, 8);
 enum mlxsw_reg_mcam_mng_feature_cap_mask_bits {
 	/* If set, MCIA supports 128 bytes payloads. Otherwise, 48 bytes. */
 	MLXSW_REG_MCAM_MCIA_128B = 34,
+	/* If set, MRSR.command=6 is supported. */
+	MLXSW_REG_MCAM_PCI_RESET = 48,
 };
 
 #define MLXSW_REG_BYTES_PER_DWORD 0x4
@@ -12934,6 +13009,7 @@ static const struct mlxsw_reg_info *mlxsw_reg_infos[] = {
 	MLXSW_REG(spvmlr),
 	MLXSW_REG(spfsr),
 	MLXSW_REG(spvc),
+	MLXSW_REG(sffp),
 	MLXSW_REG(spevet),
 	MLXSW_REG(smpe),
 	MLXSW_REG(smid2),
diff --git a/drivers/net/ethernet/mellanox/mlxsw/resources.h b/drivers/net/ethernet/mellanox/mlxsw/resources.h
index 89dd2777ec4d..9d7977ebe186 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/resources.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/resources.h
@@ -27,6 +27,7 @@ enum mlxsw_res_id {
 	MLXSW_RES_ID_FID,
 	MLXSW_RES_ID_MAX_LAG,
 	MLXSW_RES_ID_MAX_LAG_MEMBERS,
+	MLXSW_RES_ID_MAX_NVE_FLOOD_PRF,
 	MLXSW_RES_ID_GUARANTEED_SHARED_BUFFER,
 	MLXSW_RES_ID_CELL_SIZE,
 	MLXSW_RES_ID_MAX_HEADROOM_SIZE,
@@ -88,6 +89,7 @@ static u16 mlxsw_res_ids[] = {
 	[MLXSW_RES_ID_FID] = 0x2512,
 	[MLXSW_RES_ID_MAX_LAG] = 0x2520,
 	[MLXSW_RES_ID_MAX_LAG_MEMBERS] = 0x2521,
+	[MLXSW_RES_ID_MAX_NVE_FLOOD_PRF] = 0x2522,
 	[MLXSW_RES_ID_GUARANTEED_SHARED_BUFFER] = 0x2805,	/* Bytes */
 	[MLXSW_RES_ID_CELL_SIZE] = 0x2803,	/* Bytes */
 	[MLXSW_RES_ID_MAX_HEADROOM_SIZE] = 0x2811,	/* Bytes */
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
index c70333b460ea..800c461deefa 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
@@ -753,6 +753,8 @@ union mlxsw_sp_l3addr {
 };
 
 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif);
+int mlxsw_sp_rif_subport_port(const struct mlxsw_sp_rif *rif,
+			      u16 *port, bool *is_lag);
 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp,
 			 struct netlink_ext_ack *extack);
 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c
index e954b8cd2ee8..aad4bb17dfb1 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c
@@ -424,18 +424,35 @@ static enum mlxsw_reg_sfmr_op mlxsw_sp_sfmr_op(bool valid)
 		       MLXSW_REG_SFMR_OP_DESTROY_FID;
 }
 
-static int mlxsw_sp_fid_op(const struct mlxsw_sp_fid *fid, bool valid)
+static void mlxsw_sp_fid_pack(char *sfmr_pl,
+			      const struct mlxsw_sp_fid *fid,
+			      enum mlxsw_reg_sfmr_op op)
 {
-	struct mlxsw_sp *mlxsw_sp = fid->fid_family->mlxsw_sp;
-	char sfmr_pl[MLXSW_REG_SFMR_LEN];
 	u16 smpe;
 
 	smpe = fid->fid_family->smpe_index_valid ? fid->fid_index : 0;
 
-	mlxsw_reg_sfmr_pack(sfmr_pl, mlxsw_sp_sfmr_op(valid), fid->fid_index,
-			    fid->fid_offset, fid->fid_family->flood_rsp,
-			    fid->fid_family->bridge_type,
+	mlxsw_reg_sfmr_pack(sfmr_pl, op, fid->fid_index,
 			    fid->fid_family->smpe_index_valid, smpe);
+}
+
+static void mlxsw_sp_fid_pack_ctl(char *sfmr_pl,
+				  const struct mlxsw_sp_fid *fid,
+				  enum mlxsw_reg_sfmr_op op)
+{
+	mlxsw_sp_fid_pack(sfmr_pl, fid, op);
+	mlxsw_reg_sfmr_fid_offset_set(sfmr_pl, fid->fid_offset);
+	mlxsw_reg_sfmr_flood_rsp_set(sfmr_pl, fid->fid_family->flood_rsp);
+	mlxsw_reg_sfmr_flood_bridge_type_set(sfmr_pl,
+					     fid->fid_family->bridge_type);
+}
+
+static int mlxsw_sp_fid_op(const struct mlxsw_sp_fid *fid, bool valid)
+{
+	struct mlxsw_sp *mlxsw_sp = fid->fid_family->mlxsw_sp;
+	char sfmr_pl[MLXSW_REG_SFMR_LEN];
+
+	mlxsw_sp_fid_pack_ctl(sfmr_pl, fid, mlxsw_sp_sfmr_op(valid));
 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sfmr), sfmr_pl);
 }
 
@@ -444,15 +461,8 @@ static int mlxsw_sp_fid_edit_op(const struct mlxsw_sp_fid *fid,
 {
 	struct mlxsw_sp *mlxsw_sp = fid->fid_family->mlxsw_sp;
 	char sfmr_pl[MLXSW_REG_SFMR_LEN];
-	u16 smpe;
-
-	smpe = fid->fid_family->smpe_index_valid ? fid->fid_index : 0;
 
-	mlxsw_reg_sfmr_pack(sfmr_pl, MLXSW_REG_SFMR_OP_CREATE_FID,
-			    fid->fid_index, fid->fid_offset,
-			    fid->fid_family->flood_rsp,
-			    fid->fid_family->bridge_type,
-			    fid->fid_family->smpe_index_valid, smpe);
+	mlxsw_sp_fid_pack_ctl(sfmr_pl, fid, MLXSW_REG_SFMR_OP_CREATE_FID);
 	mlxsw_reg_sfmr_vv_set(sfmr_pl, fid->vni_valid);
 	mlxsw_reg_sfmr_vni_set(sfmr_pl, be32_to_cpu(fid->vni));
 	mlxsw_reg_sfmr_vtfp_set(sfmr_pl, fid->nve_flood_index_valid);
@@ -1687,9 +1697,6 @@ mlxsw_sp_fid_flood_tables_init(struct mlxsw_sp_fid_family *fid_family)
 	int err;
 	int i;
 
-	if (!fid_family->nr_flood_tables)
-		return 0;
-
 	pgt_size = mlxsw_sp_fid_family_pgt_size(fid_family);
 	err = mlxsw_sp_pgt_mid_alloc_range(mlxsw_sp, &fid_family->pgt_base,
 					   pgt_size);
@@ -1718,9 +1725,6 @@ mlxsw_sp_fid_flood_tables_fini(struct mlxsw_sp_fid_family *fid_family)
 	struct mlxsw_sp *mlxsw_sp = fid_family->mlxsw_sp;
 	u16 pgt_size;
 
-	if (!fid_family->nr_flood_tables)
-		return;
-
 	pgt_size = mlxsw_sp_fid_family_pgt_size(fid_family);
 	mlxsw_sp_pgt_mid_free_range(mlxsw_sp, fid_family->pgt_base, pgt_size);
 }
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
index 82a95125d9ca..2c255ed9b8a9 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
@@ -8419,6 +8419,9 @@ mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
 	rif->ops = ops;
 	rif->rif_entries = rif_entries;
 
+	if (ops->setup)
+		ops->setup(rif, params);
+
 	if (ops->fid_get) {
 		fid = ops->fid_get(rif, params, extack);
 		if (IS_ERR(fid)) {
@@ -8428,9 +8431,6 @@ mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
 		rif->fid = fid;
 	}
 
-	if (ops->setup)
-		ops->setup(rif, params);
-
 	err = ops->configure(rif, extack);
 	if (err)
 		goto err_configure;
@@ -8660,6 +8660,20 @@ mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
 	return container_of(rif, struct mlxsw_sp_rif_subport, common);
 }
 
+int mlxsw_sp_rif_subport_port(const struct mlxsw_sp_rif *rif,
+			      u16 *port, bool *is_lag)
+{
+	struct mlxsw_sp_rif_subport *rif_subport;
+
+	if (WARN_ON(rif->ops->type != MLXSW_SP_RIF_TYPE_SUBPORT))
+		return -EINVAL;
+
+	rif_subport = mlxsw_sp_rif_subport_rif(rif);
+	*is_lag = rif_subport->lag;
+	*port = *is_lag ? rif_subport->lag_id : rif_subport->system_port;
+	return 0;
+}
+
 static struct mlxsw_sp_rif *
 mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp,
 			 const struct mlxsw_sp_rif_params *params,
diff --git a/drivers/net/ethernet/micrel/ks8851_spi.c b/drivers/net/ethernet/micrel/ks8851_spi.c
index 70bc7253454f..7c41623dac90 100644
--- a/drivers/net/ethernet/micrel/ks8851_spi.c
+++ b/drivers/net/ethernet/micrel/ks8851_spi.c
@@ -156,7 +156,7 @@ static void ks8851_rdreg(struct ks8851_net *ks, unsigned int op,
 
 	txb[0] = cpu_to_le16(op | KS_SPIOP_RD);
 
-	if (kss->spidev->master->flags & SPI_MASTER_HALF_DUPLEX) {
+	if (kss->spidev->master->flags & SPI_CONTROLLER_HALF_DUPLEX) {
 		msg = &kss->spi_msg2;
 		xfer = kss->spi_xfer2;
 
@@ -180,7 +180,7 @@ static void ks8851_rdreg(struct ks8851_net *ks, unsigned int op,
 	ret = spi_sync(kss->spidev, msg);
 	if (ret < 0)
 		netdev_err(ks->netdev, "read: spi_sync() failed\n");
-	else if (kss->spidev->master->flags & SPI_MASTER_HALF_DUPLEX)
+	else if (kss->spidev->master->flags & SPI_CONTROLLER_HALF_DUPLEX)
 		memcpy(rxb, trx, rxl);
 	else
 		memcpy(rxb, trx + 2, rxl);
diff --git a/drivers/net/ethernet/microchip/lan743x_main.h b/drivers/net/ethernet/microchip/lan743x_main.h
index b648461787d2..be79cb0ae5af 100644
--- a/drivers/net/ethernet/microchip/lan743x_main.h
+++ b/drivers/net/ethernet/microchip/lan743x_main.h
@@ -1075,7 +1075,7 @@ struct lan743x_adapter {
 #define DMA_DESCRIPTOR_SPACING_32       (32)
 #define DMA_DESCRIPTOR_SPACING_64       (64)
 #define DMA_DESCRIPTOR_SPACING_128      (128)
-#define DEFAULT_DMA_DESCRIPTOR_SPACING  (L1_CACHE_BYTES)
+#define DEFAULT_DMA_DESCRIPTOR_SPACING  (DMA_DESCRIPTOR_SPACING_16)
 
 #define DMAC_CHANNEL_STATE_SET(start_bit, stop_bit) \
 	(((start_bit) ? 2 : 0) | ((stop_bit) ? 1 : 0))
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net.h b/drivers/net/ethernet/netronome/nfp/nfp_net.h
index 939cfce15830..bd0e26524417 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net.h
@@ -621,6 +621,9 @@ struct nfp_net_dp {
  * @mbox_amsg.lock:	Protect message list
  * @mbox_amsg.list:	List of message to process
  * @mbox_amsg.work:	Work to process message asynchronously
+ * @fs:			Flow steering
+ * @fs.count:		Flow count
+ * @fs.list:		List of flows
  * @app_priv:		APP private data for this vNIC
  */
 struct nfp_net {
@@ -728,9 +731,39 @@ struct nfp_net {
 		struct work_struct work;
 	} mbox_amsg;
 
+	struct {
+		u16 count;
+		struct list_head list;
+	} fs;
+
 	void *app_priv;
 };
 
+struct nfp_fs_entry {
+	struct list_head node;
+	u32 flow_type;
+	u32 loc;
+	struct {
+		union {
+			struct {
+				__be32 sip4;
+				__be32 dip4;
+			};
+			struct {
+				__be32 sip6[4];
+				__be32 dip6[4];
+			};
+		};
+		union {
+			__be16 l3_proto;
+			u8 l4_proto;
+		};
+		__be16 sport;
+		__be16 dport;
+	} key, msk;
+	u64 action;
+};
+
 struct nfp_mbox_amsg_entry {
 	struct list_head list;
 	int (*cfg)(struct nfp_net *nn, struct nfp_mbox_amsg_entry *entry);
@@ -987,6 +1020,9 @@ struct nfp_net_dp *nfp_net_clone_dp(struct nfp_net *nn);
 int nfp_net_ring_reconfig(struct nfp_net *nn, struct nfp_net_dp *new,
 			  struct netlink_ext_ack *extack);
 
+int nfp_net_fs_add_hw(struct nfp_net *nn, struct nfp_fs_entry *entry);
+int nfp_net_fs_del_hw(struct nfp_net *nn, struct nfp_fs_entry *entry);
+
 #ifdef CONFIG_NFP_DEBUG
 void nfp_net_debugfs_create(void);
 void nfp_net_debugfs_destroy(void);
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
index de0a5d5ded30..ac1f4514b1d0 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
@@ -1763,6 +1763,186 @@ nfp_net_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
 	return nfp_net_mbox_reconfig_and_unlock(nn, cmd);
 }
 
+static void
+nfp_net_fs_fill_v4(struct nfp_net *nn, struct nfp_fs_entry *entry, u32 op, u32 *addr)
+{
+	unsigned int i;
+
+	union {
+		struct {
+			__be16 loc;
+			u8 k_proto, m_proto;
+			__be32 k_sip, m_sip, k_dip, m_dip;
+			__be16 k_sport, m_sport, k_dport, m_dport;
+		};
+		__be32 val[7];
+	} v4_rule;
+
+	nn_writel(nn, *addr, op);
+	*addr += sizeof(u32);
+
+	v4_rule.loc     = cpu_to_be16(entry->loc);
+	v4_rule.k_proto = entry->key.l4_proto;
+	v4_rule.m_proto = entry->msk.l4_proto;
+	v4_rule.k_sip   = entry->key.sip4;
+	v4_rule.m_sip   = entry->msk.sip4;
+	v4_rule.k_dip   = entry->key.dip4;
+	v4_rule.m_dip   = entry->msk.dip4;
+	v4_rule.k_sport = entry->key.sport;
+	v4_rule.m_sport = entry->msk.sport;
+	v4_rule.k_dport = entry->key.dport;
+	v4_rule.m_dport = entry->msk.dport;
+
+	for (i = 0; i < ARRAY_SIZE(v4_rule.val); i++, *addr += sizeof(__be32))
+		nn_writel(nn, *addr, be32_to_cpu(v4_rule.val[i]));
+}
+
+static void
+nfp_net_fs_fill_v6(struct nfp_net *nn, struct nfp_fs_entry *entry, u32 op, u32 *addr)
+{
+	unsigned int i;
+
+	union {
+		struct {
+			__be16 loc;
+			u8 k_proto, m_proto;
+			__be32 k_sip[4], m_sip[4], k_dip[4], m_dip[4];
+			__be16 k_sport, m_sport, k_dport, m_dport;
+		};
+		__be32 val[19];
+	} v6_rule;
+
+	nn_writel(nn, *addr, op);
+	*addr += sizeof(u32);
+
+	v6_rule.loc     = cpu_to_be16(entry->loc);
+	v6_rule.k_proto = entry->key.l4_proto;
+	v6_rule.m_proto = entry->msk.l4_proto;
+	for (i = 0; i < 4; i++) {
+		v6_rule.k_sip[i] = entry->key.sip6[i];
+		v6_rule.m_sip[i] = entry->msk.sip6[i];
+		v6_rule.k_dip[i] = entry->key.dip6[i];
+		v6_rule.m_dip[i] = entry->msk.dip6[i];
+	}
+	v6_rule.k_sport = entry->key.sport;
+	v6_rule.m_sport = entry->msk.sport;
+	v6_rule.k_dport = entry->key.dport;
+	v6_rule.m_dport = entry->msk.dport;
+
+	for (i = 0; i < ARRAY_SIZE(v6_rule.val); i++, *addr += sizeof(__be32))
+		nn_writel(nn, *addr, be32_to_cpu(v6_rule.val[i]));
+}
+
+#define NFP_FS_QUEUE_ID	GENMASK(22, 16)
+#define NFP_FS_ACT	GENMASK(15, 0)
+#define NFP_FS_ACT_DROP	BIT(0)
+#define NFP_FS_ACT_Q	BIT(1)
+static void
+nfp_net_fs_fill_act(struct nfp_net *nn, struct nfp_fs_entry *entry, u32 addr)
+{
+	u32 action = 0; /* 0 means default passthrough */
+
+	if (entry->action == RX_CLS_FLOW_DISC)
+		action = NFP_FS_ACT_DROP;
+	else if (!(entry->flow_type & FLOW_RSS))
+		action = FIELD_PREP(NFP_FS_QUEUE_ID, entry->action) | NFP_FS_ACT_Q;
+
+	nn_writel(nn, addr, action);
+}
+
+int nfp_net_fs_add_hw(struct nfp_net *nn, struct nfp_fs_entry *entry)
+{
+	u32 addr = nn->tlv_caps.mbox_off + NFP_NET_CFG_MBOX_SIMPLE_VAL;
+	int err;
+
+	err = nfp_net_mbox_lock(nn, NFP_NET_CFG_FS_SZ);
+	if (err)
+		return err;
+
+	switch (entry->flow_type & ~FLOW_RSS) {
+	case TCP_V4_FLOW:
+	case UDP_V4_FLOW:
+	case SCTP_V4_FLOW:
+	case IPV4_USER_FLOW:
+		nfp_net_fs_fill_v4(nn, entry, NFP_NET_CFG_MBOX_CMD_FS_ADD_V4, &addr);
+		break;
+	case TCP_V6_FLOW:
+	case UDP_V6_FLOW:
+	case SCTP_V6_FLOW:
+	case IPV6_USER_FLOW:
+		nfp_net_fs_fill_v6(nn, entry, NFP_NET_CFG_MBOX_CMD_FS_ADD_V6, &addr);
+		break;
+	case ETHER_FLOW:
+		nn_writel(nn, addr, NFP_NET_CFG_MBOX_CMD_FS_ADD_ETHTYPE);
+		addr += sizeof(u32);
+		nn_writew(nn, addr, be16_to_cpu(entry->key.l3_proto));
+		addr += sizeof(u32);
+		break;
+	}
+
+	nfp_net_fs_fill_act(nn, entry, addr);
+
+	err = nfp_net_mbox_reconfig_and_unlock(nn, NFP_NET_CFG_MBOX_CMD_FLOW_STEER);
+	if (err) {
+		nn_err(nn, "Add new fs rule failed with %d\n", err);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+int nfp_net_fs_del_hw(struct nfp_net *nn, struct nfp_fs_entry *entry)
+{
+	u32 addr = nn->tlv_caps.mbox_off + NFP_NET_CFG_MBOX_SIMPLE_VAL;
+	int err;
+
+	err = nfp_net_mbox_lock(nn, NFP_NET_CFG_FS_SZ);
+	if (err)
+		return err;
+
+	switch (entry->flow_type & ~FLOW_RSS) {
+	case TCP_V4_FLOW:
+	case UDP_V4_FLOW:
+	case SCTP_V4_FLOW:
+	case IPV4_USER_FLOW:
+		nfp_net_fs_fill_v4(nn, entry, NFP_NET_CFG_MBOX_CMD_FS_DEL_V4, &addr);
+		break;
+	case TCP_V6_FLOW:
+	case UDP_V6_FLOW:
+	case SCTP_V6_FLOW:
+	case IPV6_USER_FLOW:
+		nfp_net_fs_fill_v6(nn, entry, NFP_NET_CFG_MBOX_CMD_FS_DEL_V6, &addr);
+		break;
+	case ETHER_FLOW:
+		nn_writel(nn, addr, NFP_NET_CFG_MBOX_CMD_FS_DEL_ETHTYPE);
+		addr += sizeof(u32);
+		nn_writew(nn, addr, be16_to_cpu(entry->key.l3_proto));
+		addr += sizeof(u32);
+		break;
+	}
+
+	nfp_net_fs_fill_act(nn, entry, addr);
+
+	err = nfp_net_mbox_reconfig_and_unlock(nn, NFP_NET_CFG_MBOX_CMD_FLOW_STEER);
+	if (err) {
+		nn_err(nn, "Delete fs rule failed with %d\n", err);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static void nfp_net_fs_clean(struct nfp_net *nn)
+{
+	struct nfp_fs_entry *entry, *tmp;
+
+	list_for_each_entry_safe(entry, tmp, &nn->fs.list, node) {
+		nfp_net_fs_del_hw(nn, entry);
+		list_del(&entry->node);
+		kfree(entry);
+	}
+}
+
 static void nfp_net_stat64(struct net_device *netdev,
 			   struct rtnl_link_stats64 *stats)
 {
@@ -2740,6 +2920,8 @@ int nfp_net_init(struct nfp_net *nn)
 	INIT_LIST_HEAD(&nn->mbox_amsg.list);
 	INIT_WORK(&nn->mbox_amsg.work, nfp_net_mbox_amsg_work);
 
+	INIT_LIST_HEAD(&nn->fs.list);
+
 	return register_netdev(nn->dp.netdev);
 
 err_clean_mbox:
@@ -2759,6 +2941,7 @@ void nfp_net_clean(struct nfp_net *nn)
 	unregister_netdev(nn->dp.netdev);
 	nfp_net_ipsec_clean(nn);
 	nfp_ccm_mbox_clean(nn);
+	nfp_net_fs_clean(nn);
 	flush_work(&nn->mbox_amsg.work);
 	nfp_net_reconfig_wait_posted(nn);
 }
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h b/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h
index 3e63f6d6a563..eaf4d3c499d1 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h
@@ -269,6 +269,7 @@
 #define   NFP_NET_CFG_CTRL_IPSEC	  (0x1 << 1) /* IPsec offload */
 #define   NFP_NET_CFG_CTRL_MCAST_FILTER	  (0x1 << 2) /* Multicast Filter */
 #define   NFP_NET_CFG_CTRL_FREELIST_EN	  (0x1 << 6) /* Freelist enable flag bit */
+#define   NFP_NET_CFG_CTRL_FLOW_STEER	  (0x1 << 8) /* Flow steering */
 
 #define NFP_NET_CFG_CAP_WORD1		0x00a4
 
@@ -418,6 +419,8 @@
 #define NFP_NET_CFG_MBOX_CMD_MULTICAST_ADD		8
 #define NFP_NET_CFG_MBOX_CMD_MULTICAST_DEL		9
 
+#define NFP_NET_CFG_MBOX_CMD_FLOW_STEER			10
+
 /* VLAN filtering using general use mailbox
  * %NFP_NET_CFG_VLAN_FILTER:		Base address of VLAN filter mailbox
  * %NFP_NET_CFG_VLAN_FILTER_VID:	VLAN ID to filter
@@ -440,6 +443,18 @@
 #define NFP_NET_CFG_MULTICAST_MAC_LO	(NFP_NET_CFG_MULTICAST + 6)
 #define NFP_NET_CFG_MULTICAST_SZ	0x0006
 
+/* Max size of FS rules in bytes */
+#define NFP_NET_CFG_FS_SZ		0x0054
+/* Sub commands for FS */
+enum {
+	NFP_NET_CFG_MBOX_CMD_FS_ADD_V4,
+	NFP_NET_CFG_MBOX_CMD_FS_DEL_V4,
+	NFP_NET_CFG_MBOX_CMD_FS_ADD_V6,
+	NFP_NET_CFG_MBOX_CMD_FS_DEL_V6,
+	NFP_NET_CFG_MBOX_CMD_FS_ADD_ETHTYPE,
+	NFP_NET_CFG_MBOX_CMD_FS_DEL_ETHTYPE,
+};
+
 /* TLV capabilities
  * %NFP_NET_CFG_TLV_TYPE:	Offset of type within the TLV
  * %NFP_NET_CFG_TLV_TYPE_REQUIRED: Driver must be able to parse the TLV
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c
index e75cbb287625..d7896391b8ba 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c
@@ -1317,6 +1317,116 @@ static int nfp_net_get_rss_hash_opts(struct nfp_net *nn,
 	return 0;
 }
 
+#define NFP_FS_MAX_ENTRY	1024
+
+static int nfp_net_fs_to_ethtool(struct nfp_fs_entry *entry, struct ethtool_rxnfc *cmd)
+{
+	struct ethtool_rx_flow_spec *fs = &cmd->fs;
+	unsigned int i;
+
+	switch (entry->flow_type & ~FLOW_RSS) {
+	case TCP_V4_FLOW:
+	case UDP_V4_FLOW:
+	case SCTP_V4_FLOW:
+		fs->h_u.tcp_ip4_spec.ip4src = entry->key.sip4;
+		fs->h_u.tcp_ip4_spec.ip4dst = entry->key.dip4;
+		fs->h_u.tcp_ip4_spec.psrc   = entry->key.sport;
+		fs->h_u.tcp_ip4_spec.pdst   = entry->key.dport;
+		fs->m_u.tcp_ip4_spec.ip4src = entry->msk.sip4;
+		fs->m_u.tcp_ip4_spec.ip4dst = entry->msk.dip4;
+		fs->m_u.tcp_ip4_spec.psrc   = entry->msk.sport;
+		fs->m_u.tcp_ip4_spec.pdst   = entry->msk.dport;
+		break;
+	case TCP_V6_FLOW:
+	case UDP_V6_FLOW:
+	case SCTP_V6_FLOW:
+		for (i = 0; i < 4; i++) {
+			fs->h_u.tcp_ip6_spec.ip6src[i] = entry->key.sip6[i];
+			fs->h_u.tcp_ip6_spec.ip6dst[i] = entry->key.dip6[i];
+			fs->m_u.tcp_ip6_spec.ip6src[i] = entry->msk.sip6[i];
+			fs->m_u.tcp_ip6_spec.ip6dst[i] = entry->msk.dip6[i];
+		}
+		fs->h_u.tcp_ip6_spec.psrc = entry->key.sport;
+		fs->h_u.tcp_ip6_spec.pdst = entry->key.dport;
+		fs->m_u.tcp_ip6_spec.psrc = entry->msk.sport;
+		fs->m_u.tcp_ip6_spec.pdst = entry->msk.dport;
+		break;
+	case IPV4_USER_FLOW:
+		fs->h_u.usr_ip4_spec.ip_ver = ETH_RX_NFC_IP4;
+		fs->h_u.usr_ip4_spec.ip4src = entry->key.sip4;
+		fs->h_u.usr_ip4_spec.ip4dst = entry->key.dip4;
+		fs->h_u.usr_ip4_spec.proto  = entry->key.l4_proto;
+		fs->m_u.usr_ip4_spec.ip4src = entry->msk.sip4;
+		fs->m_u.usr_ip4_spec.ip4dst = entry->msk.dip4;
+		fs->m_u.usr_ip4_spec.proto  = entry->msk.l4_proto;
+		break;
+	case IPV6_USER_FLOW:
+		for (i = 0; i < 4; i++) {
+			fs->h_u.usr_ip6_spec.ip6src[i] = entry->key.sip6[i];
+			fs->h_u.usr_ip6_spec.ip6dst[i] = entry->key.dip6[i];
+			fs->m_u.usr_ip6_spec.ip6src[i] = entry->msk.sip6[i];
+			fs->m_u.usr_ip6_spec.ip6dst[i] = entry->msk.dip6[i];
+		}
+		fs->h_u.usr_ip6_spec.l4_proto = entry->key.l4_proto;
+		fs->m_u.usr_ip6_spec.l4_proto = entry->msk.l4_proto;
+		break;
+	case ETHER_FLOW:
+		fs->h_u.ether_spec.h_proto = entry->key.l3_proto;
+		fs->m_u.ether_spec.h_proto = entry->msk.l3_proto;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	fs->flow_type   = entry->flow_type;
+	fs->ring_cookie = entry->action;
+
+	if (fs->flow_type & FLOW_RSS) {
+		/* Only rss_context of 0 is supported. */
+		cmd->rss_context = 0;
+		/* RSS is used, mask the ring. */
+		fs->ring_cookie |= ETHTOOL_RX_FLOW_SPEC_RING;
+	}
+
+	return 0;
+}
+
+static int nfp_net_get_fs_rule(struct nfp_net *nn, struct ethtool_rxnfc *cmd)
+{
+	struct nfp_fs_entry *entry;
+
+	if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_FLOW_STEER))
+		return -EOPNOTSUPP;
+
+	if (cmd->fs.location >= NFP_FS_MAX_ENTRY)
+		return -EINVAL;
+
+	list_for_each_entry(entry, &nn->fs.list, node) {
+		if (entry->loc == cmd->fs.location)
+			return nfp_net_fs_to_ethtool(entry, cmd);
+
+		if (entry->loc > cmd->fs.location)
+			/* no need to continue */
+			return -ENOENT;
+	}
+
+	return -ENOENT;
+}
+
+static int nfp_net_get_fs_loc(struct nfp_net *nn, u32 *rule_locs)
+{
+	struct nfp_fs_entry *entry;
+	u32 count = 0;
+
+	if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_FLOW_STEER))
+		return -EOPNOTSUPP;
+
+	list_for_each_entry(entry, &nn->fs.list, node)
+		rule_locs[count++] = entry->loc;
+
+	return 0;
+}
+
 static int nfp_net_get_rxnfc(struct net_device *netdev,
 			     struct ethtool_rxnfc *cmd, u32 *rule_locs)
 {
@@ -1326,6 +1436,14 @@ static int nfp_net_get_rxnfc(struct net_device *netdev,
 	case ETHTOOL_GRXRINGS:
 		cmd->data = nn->dp.num_rx_rings;
 		return 0;
+	case ETHTOOL_GRXCLSRLCNT:
+		cmd->rule_cnt = nn->fs.count;
+		return 0;
+	case ETHTOOL_GRXCLSRULE:
+		return nfp_net_get_fs_rule(nn, cmd);
+	case ETHTOOL_GRXCLSRLALL:
+		cmd->data = NFP_FS_MAX_ENTRY;
+		return nfp_net_get_fs_loc(nn, rule_locs);
 	case ETHTOOL_GRXFH:
 		return nfp_net_get_rss_hash_opts(nn, cmd);
 	default:
@@ -1385,6 +1503,253 @@ static int nfp_net_set_rss_hash_opt(struct nfp_net *nn,
 	return 0;
 }
 
+static int nfp_net_fs_from_ethtool(struct nfp_fs_entry *entry, struct ethtool_rx_flow_spec *fs)
+{
+	unsigned int i;
+
+	/* FLOW_EXT/FLOW_MAC_EXT is not supported. */
+	switch (fs->flow_type & ~FLOW_RSS) {
+	case TCP_V4_FLOW:
+	case UDP_V4_FLOW:
+	case SCTP_V4_FLOW:
+		entry->msk.sip4  = fs->m_u.tcp_ip4_spec.ip4src;
+		entry->msk.dip4  = fs->m_u.tcp_ip4_spec.ip4dst;
+		entry->msk.sport = fs->m_u.tcp_ip4_spec.psrc;
+		entry->msk.dport = fs->m_u.tcp_ip4_spec.pdst;
+		entry->key.sip4  = fs->h_u.tcp_ip4_spec.ip4src & entry->msk.sip4;
+		entry->key.dip4  = fs->h_u.tcp_ip4_spec.ip4dst & entry->msk.dip4;
+		entry->key.sport = fs->h_u.tcp_ip4_spec.psrc & entry->msk.sport;
+		entry->key.dport = fs->h_u.tcp_ip4_spec.pdst & entry->msk.dport;
+		break;
+	case TCP_V6_FLOW:
+	case UDP_V6_FLOW:
+	case SCTP_V6_FLOW:
+		for (i = 0; i < 4; i++) {
+			entry->msk.sip6[i] = fs->m_u.tcp_ip6_spec.ip6src[i];
+			entry->msk.dip6[i] = fs->m_u.tcp_ip6_spec.ip6dst[i];
+			entry->key.sip6[i] = fs->h_u.tcp_ip6_spec.ip6src[i] & entry->msk.sip6[i];
+			entry->key.dip6[i] = fs->h_u.tcp_ip6_spec.ip6dst[i] & entry->msk.dip6[i];
+		}
+		entry->msk.sport = fs->m_u.tcp_ip6_spec.psrc;
+		entry->msk.dport = fs->m_u.tcp_ip6_spec.pdst;
+		entry->key.sport = fs->h_u.tcp_ip6_spec.psrc & entry->msk.sport;
+		entry->key.dport = fs->h_u.tcp_ip6_spec.pdst & entry->msk.dport;
+		break;
+	case IPV4_USER_FLOW:
+		entry->msk.sip4     = fs->m_u.usr_ip4_spec.ip4src;
+		entry->msk.dip4     = fs->m_u.usr_ip4_spec.ip4dst;
+		entry->msk.l4_proto = fs->m_u.usr_ip4_spec.proto;
+		entry->key.sip4     = fs->h_u.usr_ip4_spec.ip4src & entry->msk.sip4;
+		entry->key.dip4     = fs->h_u.usr_ip4_spec.ip4dst & entry->msk.dip4;
+		entry->key.l4_proto = fs->h_u.usr_ip4_spec.proto & entry->msk.l4_proto;
+		break;
+	case IPV6_USER_FLOW:
+		for (i = 0; i < 4; i++) {
+			entry->msk.sip6[i] = fs->m_u.usr_ip6_spec.ip6src[i];
+			entry->msk.dip6[i] = fs->m_u.usr_ip6_spec.ip6dst[i];
+			entry->key.sip6[i] = fs->h_u.usr_ip6_spec.ip6src[i] & entry->msk.sip6[i];
+			entry->key.dip6[i] = fs->h_u.usr_ip6_spec.ip6dst[i] & entry->msk.dip6[i];
+		}
+		entry->msk.l4_proto = fs->m_u.usr_ip6_spec.l4_proto;
+		entry->key.l4_proto = fs->h_u.usr_ip6_spec.l4_proto & entry->msk.l4_proto;
+		break;
+	case ETHER_FLOW:
+		entry->msk.l3_proto = fs->m_u.ether_spec.h_proto;
+		entry->key.l3_proto = fs->h_u.ether_spec.h_proto & entry->msk.l3_proto;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (fs->flow_type & ~FLOW_RSS) {
+	case TCP_V4_FLOW:
+	case TCP_V6_FLOW:
+		entry->key.l4_proto = IPPROTO_TCP;
+		entry->msk.l4_proto = 0xff;
+		break;
+	case UDP_V4_FLOW:
+	case UDP_V6_FLOW:
+		entry->key.l4_proto = IPPROTO_UDP;
+		entry->msk.l4_proto = 0xff;
+		break;
+	case SCTP_V4_FLOW:
+	case SCTP_V6_FLOW:
+		entry->key.l4_proto = IPPROTO_SCTP;
+		entry->msk.l4_proto = 0xff;
+		break;
+	}
+
+	entry->flow_type = fs->flow_type;
+	entry->action    = fs->ring_cookie;
+	entry->loc       = fs->location;
+
+	return 0;
+}
+
+static int nfp_net_fs_check_existing(struct nfp_net *nn, struct nfp_fs_entry *new)
+{
+	struct nfp_fs_entry *entry;
+
+	list_for_each_entry(entry, &nn->fs.list, node) {
+		if (new->loc != entry->loc &&
+		    !((new->flow_type ^ entry->flow_type) & ~FLOW_RSS) &&
+		    !memcmp(&new->key, &entry->key, sizeof(new->key)) &&
+		    !memcmp(&new->msk, &entry->msk, sizeof(new->msk)))
+			return entry->loc;
+	}
+
+	/* -1 means no duplicates */
+	return -1;
+}
+
+static int nfp_net_fs_add(struct nfp_net *nn, struct ethtool_rxnfc *cmd)
+{
+	struct ethtool_rx_flow_spec *fs = &cmd->fs;
+	struct nfp_fs_entry *new, *entry;
+	bool unsupp_mask;
+	int err, id;
+
+	if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_FLOW_STEER))
+		return -EOPNOTSUPP;
+
+	/* Only default RSS context(0) is supported. */
+	if ((fs->flow_type & FLOW_RSS) && cmd->rss_context)
+		return -EOPNOTSUPP;
+
+	if (fs->location >= NFP_FS_MAX_ENTRY)
+		return -EINVAL;
+
+	if (fs->ring_cookie != RX_CLS_FLOW_DISC &&
+	    fs->ring_cookie >= nn->dp.num_rx_rings)
+		return -EINVAL;
+
+	/* FLOW_EXT/FLOW_MAC_EXT is not supported. */
+	switch (fs->flow_type & ~FLOW_RSS) {
+	case TCP_V4_FLOW:
+	case UDP_V4_FLOW:
+	case SCTP_V4_FLOW:
+		unsupp_mask = !!fs->m_u.tcp_ip4_spec.tos;
+		break;
+	case TCP_V6_FLOW:
+	case UDP_V6_FLOW:
+	case SCTP_V6_FLOW:
+		unsupp_mask = !!fs->m_u.tcp_ip6_spec.tclass;
+		break;
+	case IPV4_USER_FLOW:
+		unsupp_mask = !!fs->m_u.usr_ip4_spec.l4_4_bytes ||
+			      !!fs->m_u.usr_ip4_spec.tos ||
+			      !!fs->m_u.usr_ip4_spec.ip_ver;
+		/* ip_ver must be ETH_RX_NFC_IP4. */
+		unsupp_mask |= fs->h_u.usr_ip4_spec.ip_ver != ETH_RX_NFC_IP4;
+		break;
+	case IPV6_USER_FLOW:
+		unsupp_mask = !!fs->m_u.usr_ip6_spec.l4_4_bytes ||
+			      !!fs->m_u.usr_ip6_spec.tclass;
+		break;
+	case ETHER_FLOW:
+		if (fs->h_u.ether_spec.h_proto == htons(ETH_P_IP) ||
+		    fs->h_u.ether_spec.h_proto == htons(ETH_P_IPV6)) {
+			nn_err(nn, "Please use ip4/ip6 flow type instead.\n");
+			return -EOPNOTSUPP;
+		}
+		/* Only unmasked ethtype is supported. */
+		unsupp_mask = !is_zero_ether_addr(fs->m_u.ether_spec.h_dest) ||
+			      !is_zero_ether_addr(fs->m_u.ether_spec.h_source) ||
+			      (fs->m_u.ether_spec.h_proto != htons(0xffff));
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	if (unsupp_mask)
+		return -EOPNOTSUPP;
+
+	new = kzalloc(sizeof(*new), GFP_KERNEL);
+	if (!new)
+		return -ENOMEM;
+
+	nfp_net_fs_from_ethtool(new, fs);
+
+	id = nfp_net_fs_check_existing(nn, new);
+	if (id >= 0) {
+		nn_err(nn, "Identical rule is existing in %d.\n", id);
+		err = -EINVAL;
+		goto err;
+	}
+
+	/* Insert to list in ascending order of location. */
+	list_for_each_entry(entry, &nn->fs.list, node) {
+		if (entry->loc == fs->location) {
+			err = nfp_net_fs_del_hw(nn, entry);
+			if (err)
+				goto err;
+
+			nn->fs.count--;
+			err = nfp_net_fs_add_hw(nn, new);
+			if (err)
+				goto err;
+
+			nn->fs.count++;
+			list_replace(&entry->node, &new->node);
+			kfree(entry);
+
+			return 0;
+		}
+
+		if (entry->loc > fs->location)
+			break;
+	}
+
+	if (nn->fs.count == NFP_FS_MAX_ENTRY) {
+		err = -ENOSPC;
+		goto err;
+	}
+
+	err = nfp_net_fs_add_hw(nn, new);
+	if (err)
+		goto err;
+
+	list_add_tail(&new->node, &entry->node);
+	nn->fs.count++;
+
+	return 0;
+
+err:
+	kfree(new);
+	return err;
+}
+
+static int nfp_net_fs_del(struct nfp_net *nn, struct ethtool_rxnfc *cmd)
+{
+	struct nfp_fs_entry *entry;
+	int err;
+
+	if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_FLOW_STEER))
+		return -EOPNOTSUPP;
+
+	if (!nn->fs.count || cmd->fs.location >= NFP_FS_MAX_ENTRY)
+		return -EINVAL;
+
+	list_for_each_entry(entry, &nn->fs.list, node) {
+		if (entry->loc == cmd->fs.location) {
+			err = nfp_net_fs_del_hw(nn, entry);
+			if (err)
+				return err;
+
+			list_del(&entry->node);
+			kfree(entry);
+			nn->fs.count--;
+
+			return 0;
+		} else if (entry->loc > cmd->fs.location) {
+			/* no need to continue */
+			break;
+		}
+	}
+
+	return -ENOENT;
+}
+
 static int nfp_net_set_rxnfc(struct net_device *netdev,
 			     struct ethtool_rxnfc *cmd)
 {
@@ -1393,6 +1758,10 @@ static int nfp_net_set_rxnfc(struct net_device *netdev,
 	switch (cmd->cmd) {
 	case ETHTOOL_SRXFH:
 		return nfp_net_set_rss_hash_opt(nn, cmd);
+	case ETHTOOL_SRXCLSRLINS:
+		return nfp_net_fs_add(nn, cmd);
+	case ETHTOOL_SRXCLSRLDEL:
+		return nfp_net_fs_del(nn, cmd);
 	default:
 		return -EOPNOTSUPP;
 	}
diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c
index b9bb1d2f0237..a8b623145efc 100644
--- a/drivers/net/ethernet/realtek/r8169_main.c
+++ b/drivers/net/ethernet/realtek/r8169_main.c
@@ -3102,6 +3102,33 @@ static void rtl_hw_start_8168g_2(struct rtl8169_private *tp)
 	rtl_ephy_init(tp, e_info_8168g_2);
 }
 
+static void rtl8411b_fix_phy_down(struct rtl8169_private *tp)
+{
+	static const u16 fix_data[] = {
+/* 0xf800 */ 0xe008, 0xe00a, 0xe00c, 0xe00e, 0xe027, 0xe04f, 0xe05e, 0xe065,
+/* 0xf810 */ 0xc602, 0xbe00, 0x0000, 0xc502, 0xbd00, 0x074c, 0xc302, 0xbb00,
+/* 0xf820 */ 0x080a, 0x6420, 0x48c2, 0x8c20, 0xc516, 0x64a4, 0x49c0, 0xf009,
+/* 0xf830 */ 0x74a2, 0x8ca5, 0x74a0, 0xc50e, 0x9ca2, 0x1c11, 0x9ca0, 0xe006,
+/* 0xf840 */ 0x74f8, 0x48c4, 0x8cf8, 0xc404, 0xbc00, 0xc403, 0xbc00, 0x0bf2,
+/* 0xf850 */ 0x0c0a, 0xe434, 0xd3c0, 0x49d9, 0xf01f, 0xc526, 0x64a5, 0x1400,
+/* 0xf860 */ 0xf007, 0x0c01, 0x8ca5, 0x1c15, 0xc51b, 0x9ca0, 0xe013, 0xc519,
+/* 0xf870 */ 0x74a0, 0x48c4, 0x8ca0, 0xc516, 0x74a4, 0x48c8, 0x48ca, 0x9ca4,
+/* 0xf880 */ 0xc512, 0x1b00, 0x9ba0, 0x1b1c, 0x483f, 0x9ba2, 0x1b04, 0xc508,
+/* 0xf890 */ 0x9ba0, 0xc505, 0xbd00, 0xc502, 0xbd00, 0x0300, 0x051e, 0xe434,
+/* 0xf8a0 */ 0xe018, 0xe092, 0xde20, 0xd3c0, 0xc50f, 0x76a4, 0x49e3, 0xf007,
+/* 0xf8b0 */ 0x49c0, 0xf103, 0xc607, 0xbe00, 0xc606, 0xbe00, 0xc602, 0xbe00,
+/* 0xf8c0 */ 0x0c4c, 0x0c28, 0x0c2c, 0xdc00, 0xc707, 0x1d00, 0x8de2, 0x48c1,
+/* 0xf8d0 */ 0xc502, 0xbd00, 0x00aa, 0xe0c0, 0xc502, 0xbd00, 0x0132
+	};
+	unsigned long flags;
+	int i;
+
+	raw_spin_lock_irqsave(&tp->mac_ocp_lock, flags);
+	for (i = 0; i < ARRAY_SIZE(fix_data); i++)
+		__r8168_mac_ocp_write(tp, 0xf800 + 2 * i, fix_data[i]);
+	raw_spin_unlock_irqrestore(&tp->mac_ocp_lock, flags);
+}
+
 static void rtl_hw_start_8411_2(struct rtl8169_private *tp)
 {
 	static const struct ephy_info e_info_8411_2[] = {
@@ -3135,117 +3162,7 @@ static void rtl_hw_start_8411_2(struct rtl8169_private *tp)
 	mdelay(3);
 	r8168_mac_ocp_write(tp, 0xFC26, 0x0000);
 
-	r8168_mac_ocp_write(tp, 0xF800, 0xE008);
-	r8168_mac_ocp_write(tp, 0xF802, 0xE00A);
-	r8168_mac_ocp_write(tp, 0xF804, 0xE00C);
-	r8168_mac_ocp_write(tp, 0xF806, 0xE00E);
-	r8168_mac_ocp_write(tp, 0xF808, 0xE027);
-	r8168_mac_ocp_write(tp, 0xF80A, 0xE04F);
-	r8168_mac_ocp_write(tp, 0xF80C, 0xE05E);
-	r8168_mac_ocp_write(tp, 0xF80E, 0xE065);
-	r8168_mac_ocp_write(tp, 0xF810, 0xC602);
-	r8168_mac_ocp_write(tp, 0xF812, 0xBE00);
-	r8168_mac_ocp_write(tp, 0xF814, 0x0000);
-	r8168_mac_ocp_write(tp, 0xF816, 0xC502);
-	r8168_mac_ocp_write(tp, 0xF818, 0xBD00);
-	r8168_mac_ocp_write(tp, 0xF81A, 0x074C);
-	r8168_mac_ocp_write(tp, 0xF81C, 0xC302);
-	r8168_mac_ocp_write(tp, 0xF81E, 0xBB00);
-	r8168_mac_ocp_write(tp, 0xF820, 0x080A);
-	r8168_mac_ocp_write(tp, 0xF822, 0x6420);
-	r8168_mac_ocp_write(tp, 0xF824, 0x48C2);
-	r8168_mac_ocp_write(tp, 0xF826, 0x8C20);
-	r8168_mac_ocp_write(tp, 0xF828, 0xC516);
-	r8168_mac_ocp_write(tp, 0xF82A, 0x64A4);
-	r8168_mac_ocp_write(tp, 0xF82C, 0x49C0);
-	r8168_mac_ocp_write(tp, 0xF82E, 0xF009);
-	r8168_mac_ocp_write(tp, 0xF830, 0x74A2);
-	r8168_mac_ocp_write(tp, 0xF832, 0x8CA5);
-	r8168_mac_ocp_write(tp, 0xF834, 0x74A0);
-	r8168_mac_ocp_write(tp, 0xF836, 0xC50E);
-	r8168_mac_ocp_write(tp, 0xF838, 0x9CA2);
-	r8168_mac_ocp_write(tp, 0xF83A, 0x1C11);
-	r8168_mac_ocp_write(tp, 0xF83C, 0x9CA0);
-	r8168_mac_ocp_write(tp, 0xF83E, 0xE006);
-	r8168_mac_ocp_write(tp, 0xF840, 0x74F8);
-	r8168_mac_ocp_write(tp, 0xF842, 0x48C4);
-	r8168_mac_ocp_write(tp, 0xF844, 0x8CF8);
-	r8168_mac_ocp_write(tp, 0xF846, 0xC404);
-	r8168_mac_ocp_write(tp, 0xF848, 0xBC00);
-	r8168_mac_ocp_write(tp, 0xF84A, 0xC403);
-	r8168_mac_ocp_write(tp, 0xF84C, 0xBC00);
-	r8168_mac_ocp_write(tp, 0xF84E, 0x0BF2);
-	r8168_mac_ocp_write(tp, 0xF850, 0x0C0A);
-	r8168_mac_ocp_write(tp, 0xF852, 0xE434);
-	r8168_mac_ocp_write(tp, 0xF854, 0xD3C0);
-	r8168_mac_ocp_write(tp, 0xF856, 0x49D9);
-	r8168_mac_ocp_write(tp, 0xF858, 0xF01F);
-	r8168_mac_ocp_write(tp, 0xF85A, 0xC526);
-	r8168_mac_ocp_write(tp, 0xF85C, 0x64A5);
-	r8168_mac_ocp_write(tp, 0xF85E, 0x1400);
-	r8168_mac_ocp_write(tp, 0xF860, 0xF007);
-	r8168_mac_ocp_write(tp, 0xF862, 0x0C01);
-	r8168_mac_ocp_write(tp, 0xF864, 0x8CA5);
-	r8168_mac_ocp_write(tp, 0xF866, 0x1C15);
-	r8168_mac_ocp_write(tp, 0xF868, 0xC51B);
-	r8168_mac_ocp_write(tp, 0xF86A, 0x9CA0);
-	r8168_mac_ocp_write(tp, 0xF86C, 0xE013);
-	r8168_mac_ocp_write(tp, 0xF86E, 0xC519);
-	r8168_mac_ocp_write(tp, 0xF870, 0x74A0);
-	r8168_mac_ocp_write(tp, 0xF872, 0x48C4);
-	r8168_mac_ocp_write(tp, 0xF874, 0x8CA0);
-	r8168_mac_ocp_write(tp, 0xF876, 0xC516);
-	r8168_mac_ocp_write(tp, 0xF878, 0x74A4);
-	r8168_mac_ocp_write(tp, 0xF87A, 0x48C8);
-	r8168_mac_ocp_write(tp, 0xF87C, 0x48CA);
-	r8168_mac_ocp_write(tp, 0xF87E, 0x9CA4);
-	r8168_mac_ocp_write(tp, 0xF880, 0xC512);
-	r8168_mac_ocp_write(tp, 0xF882, 0x1B00);
-	r8168_mac_ocp_write(tp, 0xF884, 0x9BA0);
-	r8168_mac_ocp_write(tp, 0xF886, 0x1B1C);
-	r8168_mac_ocp_write(tp, 0xF888, 0x483F);
-	r8168_mac_ocp_write(tp, 0xF88A, 0x9BA2);
-	r8168_mac_ocp_write(tp, 0xF88C, 0x1B04);
-	r8168_mac_ocp_write(tp, 0xF88E, 0xC508);
-	r8168_mac_ocp_write(tp, 0xF890, 0x9BA0);
-	r8168_mac_ocp_write(tp, 0xF892, 0xC505);
-	r8168_mac_ocp_write(tp, 0xF894, 0xBD00);
-	r8168_mac_ocp_write(tp, 0xF896, 0xC502);
-	r8168_mac_ocp_write(tp, 0xF898, 0xBD00);
-	r8168_mac_ocp_write(tp, 0xF89A, 0x0300);
-	r8168_mac_ocp_write(tp, 0xF89C, 0x051E);
-	r8168_mac_ocp_write(tp, 0xF89E, 0xE434);
-	r8168_mac_ocp_write(tp, 0xF8A0, 0xE018);
-	r8168_mac_ocp_write(tp, 0xF8A2, 0xE092);
-	r8168_mac_ocp_write(tp, 0xF8A4, 0xDE20);
-	r8168_mac_ocp_write(tp, 0xF8A6, 0xD3C0);
-	r8168_mac_ocp_write(tp, 0xF8A8, 0xC50F);
-	r8168_mac_ocp_write(tp, 0xF8AA, 0x76A4);
-	r8168_mac_ocp_write(tp, 0xF8AC, 0x49E3);
-	r8168_mac_ocp_write(tp, 0xF8AE, 0xF007);
-	r8168_mac_ocp_write(tp, 0xF8B0, 0x49C0);
-	r8168_mac_ocp_write(tp, 0xF8B2, 0xF103);
-	r8168_mac_ocp_write(tp, 0xF8B4, 0xC607);
-	r8168_mac_ocp_write(tp, 0xF8B6, 0xBE00);
-	r8168_mac_ocp_write(tp, 0xF8B8, 0xC606);
-	r8168_mac_ocp_write(tp, 0xF8BA, 0xBE00);
-	r8168_mac_ocp_write(tp, 0xF8BC, 0xC602);
-	r8168_mac_ocp_write(tp, 0xF8BE, 0xBE00);
-	r8168_mac_ocp_write(tp, 0xF8C0, 0x0C4C);
-	r8168_mac_ocp_write(tp, 0xF8C2, 0x0C28);
-	r8168_mac_ocp_write(tp, 0xF8C4, 0x0C2C);
-	r8168_mac_ocp_write(tp, 0xF8C6, 0xDC00);
-	r8168_mac_ocp_write(tp, 0xF8C8, 0xC707);
-	r8168_mac_ocp_write(tp, 0xF8CA, 0x1D00);
-	r8168_mac_ocp_write(tp, 0xF8CC, 0x8DE2);
-	r8168_mac_ocp_write(tp, 0xF8CE, 0x48C1);
-	r8168_mac_ocp_write(tp, 0xF8D0, 0xC502);
-	r8168_mac_ocp_write(tp, 0xF8D2, 0xBD00);
-	r8168_mac_ocp_write(tp, 0xF8D4, 0x00AA);
-	r8168_mac_ocp_write(tp, 0xF8D6, 0xE0C0);
-	r8168_mac_ocp_write(tp, 0xF8D8, 0xC502);
-	r8168_mac_ocp_write(tp, 0xF8DA, 0xBD00);
-	r8168_mac_ocp_write(tp, 0xF8DC, 0x0132);
+	rtl8411b_fix_phy_down(tp);
 
 	r8168_mac_ocp_write(tp, 0xFC26, 0x8000);
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/Kconfig b/drivers/net/ethernet/stmicro/stmmac/Kconfig
index a2b9e289aa36..85dcda51df05 100644
--- a/drivers/net/ethernet/stmicro/stmmac/Kconfig
+++ b/drivers/net/ethernet/stmicro/stmmac/Kconfig
@@ -280,7 +280,7 @@ config DWMAC_INTEL
 config DWMAC_LOONGSON
 	tristate "Loongson PCI DWMAC support"
 	default MACH_LOONGSON64
-	depends on STMMAC_ETH && PCI
+	depends on (MACH_LOONGSON64 || COMPILE_TEST) && STMMAC_ETH && PCI
 	depends on COMMON_CLK
 	help
 	  This selects the LOONGSON PCI bus support for the stmmac driver,
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
index 2afb2bd25977..97598c1497ea 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
@@ -6180,30 +6180,23 @@ static struct dentry *stmmac_fs_dir;
 static void sysfs_display_ring(void *head, int size, int extend_desc,
 			       struct seq_file *seq, dma_addr_t dma_phy_addr)
 {
-	int i;
 	struct dma_extended_desc *ep = (struct dma_extended_desc *)head;
 	struct dma_desc *p = (struct dma_desc *)head;
+	unsigned int desc_size;
 	dma_addr_t dma_addr;
+	int i;
 
+	desc_size = extend_desc ? sizeof(*ep) : sizeof(*p);
 	for (i = 0; i < size; i++) {
-		if (extend_desc) {
-			dma_addr = dma_phy_addr + i * sizeof(*ep);
-			seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n",
-				   i, &dma_addr,
-				   le32_to_cpu(ep->basic.des0),
-				   le32_to_cpu(ep->basic.des1),
-				   le32_to_cpu(ep->basic.des2),
-				   le32_to_cpu(ep->basic.des3));
-			ep++;
-		} else {
-			dma_addr = dma_phy_addr + i * sizeof(*p);
-			seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n",
-				   i, &dma_addr,
-				   le32_to_cpu(p->des0), le32_to_cpu(p->des1),
-				   le32_to_cpu(p->des2), le32_to_cpu(p->des3));
+		dma_addr = dma_phy_addr + i * desc_size;
+		seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n",
+				i, &dma_addr,
+				le32_to_cpu(p->des0), le32_to_cpu(p->des1),
+				le32_to_cpu(p->des2), le32_to_cpu(p->des3));
+		if (extend_desc)
+			p = &(++ep)->basic;
+		else
 			p++;
-		}
-		seq_printf(seq, "\n");
 	}
 }
 
diff --git a/drivers/net/ethernet/ti/am65-cpsw-ethtool.c b/drivers/net/ethernet/ti/am65-cpsw-ethtool.c
index c51e2af91f69..b9e1d568604b 100644
--- a/drivers/net/ethernet/ti/am65-cpsw-ethtool.c
+++ b/drivers/net/ethernet/ti/am65-cpsw-ethtool.c
@@ -662,6 +662,31 @@ static void am65_cpsw_get_ethtool_stats(struct net_device *ndev,
 					hw_stats[i].offset);
 }
 
+static void am65_cpsw_get_eth_mac_stats(struct net_device *ndev,
+					struct ethtool_eth_mac_stats *s)
+{
+	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
+	struct am65_cpsw_stats_regs __iomem *stats;
+
+	stats = port->stat_base;
+
+	s->FramesTransmittedOK = readl_relaxed(&stats->tx_good_frames);
+	s->SingleCollisionFrames = readl_relaxed(&stats->tx_single_coll_frames);
+	s->MultipleCollisionFrames = readl_relaxed(&stats->tx_mult_coll_frames);
+	s->FramesReceivedOK = readl_relaxed(&stats->rx_good_frames);
+	s->FrameCheckSequenceErrors = readl_relaxed(&stats->rx_crc_errors);
+	s->AlignmentErrors = readl_relaxed(&stats->rx_align_code_errors);
+	s->OctetsTransmittedOK = readl_relaxed(&stats->tx_octets);
+	s->FramesWithDeferredXmissions = readl_relaxed(&stats->tx_deferred_frames);
+	s->LateCollisions = readl_relaxed(&stats->tx_late_collisions);
+	s->CarrierSenseErrors = readl_relaxed(&stats->tx_carrier_sense_errors);
+	s->OctetsReceivedOK = readl_relaxed(&stats->rx_octets);
+	s->MulticastFramesXmittedOK = readl_relaxed(&stats->tx_multicast_frames);
+	s->BroadcastFramesXmittedOK = readl_relaxed(&stats->tx_broadcast_frames);
+	s->MulticastFramesReceivedOK = readl_relaxed(&stats->rx_multicast_frames);
+	s->BroadcastFramesReceivedOK = readl_relaxed(&stats->rx_broadcast_frames);
+};
+
 static int am65_cpsw_get_ethtool_ts_info(struct net_device *ndev,
 					 struct ethtool_ts_info *info)
 {
@@ -729,6 +754,7 @@ const struct ethtool_ops am65_cpsw_ethtool_ops_slave = {
 	.get_sset_count		= am65_cpsw_get_sset_count,
 	.get_strings		= am65_cpsw_get_strings,
 	.get_ethtool_stats	= am65_cpsw_get_ethtool_stats,
+	.get_eth_mac_stats	= am65_cpsw_get_eth_mac_stats,
 	.get_ts_info		= am65_cpsw_get_ethtool_ts_info,
 	.get_priv_flags		= am65_cpsw_get_ethtool_priv_flags,
 	.set_priv_flags		= am65_cpsw_set_ethtool_priv_flags,
diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c
index ece9f8df98ae..7992a76ed4d8 100644
--- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c
+++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c
@@ -136,6 +136,8 @@
 			 NETIF_MSG_IFUP	| NETIF_MSG_PROBE | NETIF_MSG_IFDOWN | \
 			 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
 
+#define AM65_CPSW_DEFAULT_TX_CHNS	8
+
 static void am65_cpsw_port_set_sl_mac(struct am65_cpsw_port *slave,
 				      const u8 *dev_addr)
 {
@@ -367,10 +369,81 @@ static void am65_cpsw_init_host_port_emac(struct am65_cpsw_common *common);
 static void am65_cpsw_init_port_switch_ale(struct am65_cpsw_port *port);
 static void am65_cpsw_init_port_emac_ale(struct am65_cpsw_port *port);
 
+static void am65_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma)
+{
+	struct am65_cpsw_rx_chn *rx_chn = data;
+	struct cppi5_host_desc_t *desc_rx;
+	struct sk_buff *skb;
+	dma_addr_t buf_dma;
+	u32 buf_dma_len;
+	void **swdata;
+
+	desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);
+	swdata = cppi5_hdesc_get_swdata(desc_rx);
+	skb = *swdata;
+	cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
+	k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);
+
+	dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
+	k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
+
+	dev_kfree_skb_any(skb);
+}
+
+static void am65_cpsw_nuss_xmit_free(struct am65_cpsw_tx_chn *tx_chn,
+				     struct cppi5_host_desc_t *desc)
+{
+	struct cppi5_host_desc_t *first_desc, *next_desc;
+	dma_addr_t buf_dma, next_desc_dma;
+	u32 buf_dma_len;
+
+	first_desc = desc;
+	next_desc = first_desc;
+
+	cppi5_hdesc_get_obuf(first_desc, &buf_dma, &buf_dma_len);
+	k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);
+
+	dma_unmap_single(tx_chn->dma_dev, buf_dma, buf_dma_len, DMA_TO_DEVICE);
+
+	next_desc_dma = cppi5_hdesc_get_next_hbdesc(first_desc);
+	k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);
+	while (next_desc_dma) {
+		next_desc = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,
+						       next_desc_dma);
+		cppi5_hdesc_get_obuf(next_desc, &buf_dma, &buf_dma_len);
+		k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);
+
+		dma_unmap_page(tx_chn->dma_dev, buf_dma, buf_dma_len,
+			       DMA_TO_DEVICE);
+
+		next_desc_dma = cppi5_hdesc_get_next_hbdesc(next_desc);
+		k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);
+
+		k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);
+	}
+
+	k3_cppi_desc_pool_free(tx_chn->desc_pool, first_desc);
+}
+
+static void am65_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma)
+{
+	struct am65_cpsw_tx_chn *tx_chn = data;
+	struct cppi5_host_desc_t *desc_tx;
+	struct sk_buff *skb;
+	void **swdata;
+
+	desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);
+	swdata = cppi5_hdesc_get_swdata(desc_tx);
+	skb = *(swdata);
+	am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);
+
+	dev_kfree_skb_any(skb);
+}
+
 static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common)
 {
 	struct am65_cpsw_host *host_p = am65_common_get_host(common);
-	int port_idx, i, ret;
+	int port_idx, i, ret, tx;
 	struct sk_buff *skb;
 	u32 val, port_mask;
 
@@ -437,8 +510,12 @@ static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common)
 						  AM65_CPSW_MAX_PACKET_SIZE,
 						  GFP_KERNEL);
 		if (!skb) {
+			ret = -ENOMEM;
 			dev_err(common->dev, "cannot allocate skb\n");
-			return -ENOMEM;
+			if (i)
+				goto fail_rx;
+
+			return ret;
 		}
 
 		ret = am65_cpsw_nuss_rx_push(common, skb);
@@ -447,17 +524,28 @@ static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common)
 				"cannot submit skb to channel rx, error %d\n",
 				ret);
 			kfree_skb(skb);
+			if (i)
+				goto fail_rx;
+
 			return ret;
 		}
-		kmemleak_not_leak(skb);
 	}
-	k3_udma_glue_enable_rx_chn(common->rx_chns.rx_chn);
 
-	for (i = 0; i < common->tx_ch_num; i++) {
-		ret = k3_udma_glue_enable_tx_chn(common->tx_chns[i].tx_chn);
-		if (ret)
-			return ret;
-		napi_enable(&common->tx_chns[i].napi_tx);
+	ret = k3_udma_glue_enable_rx_chn(common->rx_chns.rx_chn);
+	if (ret) {
+		dev_err(common->dev, "couldn't enable rx chn: %d\n", ret);
+		goto fail_rx;
+	}
+
+	for (tx = 0; tx < common->tx_ch_num; tx++) {
+		ret = k3_udma_glue_enable_tx_chn(common->tx_chns[tx].tx_chn);
+		if (ret) {
+			dev_err(common->dev, "couldn't enable tx chn %d: %d\n",
+				tx, ret);
+			tx--;
+			goto fail_tx;
+		}
+		napi_enable(&common->tx_chns[tx].napi_tx);
 	}
 
 	napi_enable(&common->napi_rx);
@@ -468,10 +556,22 @@ static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common)
 
 	dev_dbg(common->dev, "cpsw_nuss started\n");
 	return 0;
-}
 
-static void am65_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma);
-static void am65_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma);
+fail_tx:
+	while (tx >= 0) {
+		napi_disable(&common->tx_chns[tx].napi_tx);
+		k3_udma_glue_disable_tx_chn(common->tx_chns[tx].tx_chn);
+		tx--;
+	}
+
+	k3_udma_glue_disable_rx_chn(common->rx_chns.rx_chn);
+
+fail_rx:
+	k3_udma_glue_reset_rx_chn(common->rx_chns.rx_chn, 0,
+				  &common->rx_chns,
+				  am65_cpsw_nuss_rx_cleanup, 0);
+	return ret;
+}
 
 static int am65_cpsw_nuss_common_stop(struct am65_cpsw_common *common)
 {
@@ -646,27 +746,6 @@ static int am65_cpsw_nuss_ndo_slave_open(struct net_device *ndev)
 	return ret;
 }
 
-static void am65_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma)
-{
-	struct am65_cpsw_rx_chn *rx_chn = data;
-	struct cppi5_host_desc_t *desc_rx;
-	struct sk_buff *skb;
-	dma_addr_t buf_dma;
-	u32 buf_dma_len;
-	void **swdata;
-
-	desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);
-	swdata = cppi5_hdesc_get_swdata(desc_rx);
-	skb = *swdata;
-	cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
-	k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);
-
-	dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
-	k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
-
-	dev_kfree_skb_any(skb);
-}
-
 static void am65_cpsw_nuss_rx_ts(struct sk_buff *skb, u32 *psdata)
 {
 	struct skb_shared_hwtstamps *ssh;
@@ -840,56 +919,6 @@ static int am65_cpsw_nuss_rx_poll(struct napi_struct *napi_rx, int budget)
 	return num_rx;
 }
 
-static void am65_cpsw_nuss_xmit_free(struct am65_cpsw_tx_chn *tx_chn,
-				     struct cppi5_host_desc_t *desc)
-{
-	struct cppi5_host_desc_t *first_desc, *next_desc;
-	dma_addr_t buf_dma, next_desc_dma;
-	u32 buf_dma_len;
-
-	first_desc = desc;
-	next_desc = first_desc;
-
-	cppi5_hdesc_get_obuf(first_desc, &buf_dma, &buf_dma_len);
-	k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);
-
-	dma_unmap_single(tx_chn->dma_dev, buf_dma, buf_dma_len, DMA_TO_DEVICE);
-
-	next_desc_dma = cppi5_hdesc_get_next_hbdesc(first_desc);
-	k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);
-	while (next_desc_dma) {
-		next_desc = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,
-						       next_desc_dma);
-		cppi5_hdesc_get_obuf(next_desc, &buf_dma, &buf_dma_len);
-		k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);
-
-		dma_unmap_page(tx_chn->dma_dev, buf_dma, buf_dma_len,
-			       DMA_TO_DEVICE);
-
-		next_desc_dma = cppi5_hdesc_get_next_hbdesc(next_desc);
-		k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);
-
-		k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);
-	}
-
-	k3_cppi_desc_pool_free(tx_chn->desc_pool, first_desc);
-}
-
-static void am65_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma)
-{
-	struct am65_cpsw_tx_chn *tx_chn = data;
-	struct cppi5_host_desc_t *desc_tx;
-	struct sk_buff *skb;
-	void **swdata;
-
-	desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);
-	swdata = cppi5_hdesc_get_swdata(desc_tx);
-	skb = *(swdata);
-	am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);
-
-	dev_kfree_skb_any(skb);
-}
-
 static struct sk_buff *
 am65_cpsw_nuss_tx_compl_packet(struct am65_cpsw_tx_chn *tx_chn,
 			       dma_addr_t desc_dma)
@@ -2897,7 +2926,7 @@ static int am65_cpsw_nuss_probe(struct platform_device *pdev)
 
 	common->rx_flow_id_base = -1;
 	init_completion(&common->tdown_complete);
-	common->tx_ch_num = 1;
+	common->tx_ch_num = AM65_CPSW_DEFAULT_TX_CHNS;
 	common->pf_p0_rx_ptype_rrobin = false;
 	common->default_vlan = 1;
 
diff --git a/drivers/net/ethernet/wangxun/libwx/wx_hw.c b/drivers/net/ethernet/wangxun/libwx/wx_hw.c
index a3c5de9d547a..533e912af089 100644
--- a/drivers/net/ethernet/wangxun/libwx/wx_hw.c
+++ b/drivers/net/ethernet/wangxun/libwx/wx_hw.c
@@ -1769,10 +1769,12 @@ int wx_sw_init(struct wx *wx)
 		wx->subsystem_device_id = pdev->subsystem_device;
 	} else {
 		err = wx_flash_read_dword(wx, 0xfffdc, &ssid);
-		if (!err)
-			wx->subsystem_device_id = swab16((u16)ssid);
+		if (err < 0) {
+			wx_err(wx, "read of internal subsystem device id failed\n");
+			return err;
+		}
 
-		return err;
+		wx->subsystem_device_id = swab16((u16)ssid);
 	}
 
 	wx->mac_table = kcalloc(wx->mac.num_rar_entries,
diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c
index 3d43f808c86b..8db804543e66 100644
--- a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c
+++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c
@@ -121,10 +121,8 @@ static int ngbe_sw_init(struct wx *wx)
 
 	/* PCI config space info */
 	err = wx_sw_init(wx);
-	if (err < 0) {
-		wx_err(wx, "read of internal subsystem device id failed\n");
+	if (err < 0)
 		return err;
-	}
 
 	/* mac type, phy type , oem type */
 	ngbe_init_type_code(wx);
diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c
index 70f0b5c01dac..526250102db2 100644
--- a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c
+++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c
@@ -364,10 +364,8 @@ static int txgbe_sw_init(struct wx *wx)
 
 	/* PCI config space info */
 	err = wx_sw_init(wx);
-	if (err < 0) {
-		wx_err(wx, "read of internal subsystem device id failed\n");
+	if (err < 0)
 		return err;
-	}
 
 	txgbe_init_type_code(wx);
 
diff --git a/drivers/net/ethernet/xilinx/Kconfig b/drivers/net/ethernet/xilinx/Kconfig
index 0014729b8865..35d96c633a33 100644
--- a/drivers/net/ethernet/xilinx/Kconfig
+++ b/drivers/net/ethernet/xilinx/Kconfig
@@ -26,6 +26,7 @@ config XILINX_EMACLITE
 config XILINX_AXI_EMAC
 	tristate "Xilinx 10/100/1000 AXI Ethernet support"
 	depends on HAS_IOMEM
+	depends on XILINX_DMA
 	select PHYLINK
 	help
 	  This driver supports the 10/100/1000 Ethernet from Xilinx for the
diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet.h b/drivers/net/ethernet/xilinx/xilinx_axienet.h
index 575ff9de8985..807ead678551 100644
--- a/drivers/net/ethernet/xilinx/xilinx_axienet.h
+++ b/drivers/net/ethernet/xilinx/xilinx_axienet.h
@@ -14,6 +14,7 @@
 #include <linux/interrupt.h>
 #include <linux/if_vlan.h>
 #include <linux/phylink.h>
+#include <linux/skbuff.h>
 
 /* Packet size info */
 #define XAE_HDR_SIZE			14 /* Size of Ethernet header */
@@ -378,6 +379,22 @@ struct axidma_bd {
 
 #define XAE_NUM_MISC_CLOCKS 3
 
+/**
+ * struct skbuf_dma_descriptor - skb for each dma descriptor
+ * @sgl: Pointer for sglist.
+ * @desc: Pointer to dma descriptor.
+ * @dma_address: dma address of sglist.
+ * @skb: Pointer to SKB transferred using DMA
+ * @sg_len: number of entries in the sglist.
+ */
+struct skbuf_dma_descriptor {
+	struct scatterlist sgl[MAX_SKB_FRAGS + 1];
+	struct dma_async_tx_descriptor *desc;
+	dma_addr_t dma_address;
+	struct sk_buff *skb;
+	int sg_len;
+};
+
 /**
  * struct axienet_local - axienet private per device data
  * @ndev:	Pointer for net_device to which it will be attached.
@@ -435,6 +452,15 @@ struct axidma_bd {
  * @coalesce_usec_rx:	IRQ coalesce delay for RX
  * @coalesce_count_tx:	Store the irq coalesce on TX side.
  * @coalesce_usec_tx:	IRQ coalesce delay for TX
+ * @use_dmaengine: flag to check dmaengine framework usage.
+ * @tx_chan:	TX DMA channel.
+ * @rx_chan:	RX DMA channel.
+ * @tx_skb_ring: Pointer to TX skb ring buffer array.
+ * @rx_skb_ring: Pointer to RX skb ring buffer array.
+ * @tx_ring_head: TX skb ring buffer head index.
+ * @tx_ring_tail: TX skb ring buffer tail index.
+ * @rx_ring_head: RX skb ring buffer head index.
+ * @rx_ring_tail: RX skb ring buffer tail index.
  */
 struct axienet_local {
 	struct net_device *ndev;
@@ -499,6 +525,15 @@ struct axienet_local {
 	u32 coalesce_usec_rx;
 	u32 coalesce_count_tx;
 	u32 coalesce_usec_tx;
+	u8  use_dmaengine;
+	struct dma_chan *tx_chan;
+	struct dma_chan *rx_chan;
+	struct skbuf_dma_descriptor **tx_skb_ring;
+	struct skbuf_dma_descriptor **rx_skb_ring;
+	int tx_ring_head;
+	int tx_ring_tail;
+	int rx_ring_head;
+	int rx_ring_tail;
 };
 
 /**
diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c
index 82d0d44b2b02..7d8e22929385 100644
--- a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c
+++ b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c
@@ -38,6 +38,11 @@
 #include <linux/phy.h>
 #include <linux/mii.h>
 #include <linux/ethtool.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include <linux/dma/xilinx_dma.h>
+#include <linux/circ_buf.h>
+#include <net/netdev_queues.h>
 
 #include "xilinx_axienet.h"
 
@@ -47,6 +52,9 @@
 #define TX_BD_NUM_MIN			(MAX_SKB_FRAGS + 1)
 #define TX_BD_NUM_MAX			4096
 #define RX_BD_NUM_MAX			4096
+#define DMA_NUM_APP_WORDS		5
+#define LEN_APP				4
+#define RX_BUF_NUM_DEFAULT		128
 
 /* Must be shorter than length of ethtool_drvinfo.driver field to fit */
 #define DRIVER_NAME		"xaxienet"
@@ -55,6 +63,8 @@
 
 #define AXIENET_REGS_N		40
 
+static void axienet_rx_submit_desc(struct net_device *ndev);
+
 /* Match table for of_platform binding */
 static const struct of_device_id axienet_of_match[] = {
 	{ .compatible = "xlnx,axi-ethernet-1.00.a", },
@@ -120,6 +130,16 @@ static struct axienet_option axienet_options[] = {
 	{}
 };
 
+static struct skbuf_dma_descriptor *axienet_get_rx_desc(struct axienet_local *lp, int i)
+{
+	return lp->rx_skb_ring[i & (RX_BUF_NUM_DEFAULT - 1)];
+}
+
+static struct skbuf_dma_descriptor *axienet_get_tx_desc(struct axienet_local *lp, int i)
+{
+	return lp->tx_skb_ring[i & (TX_BD_NUM_MAX - 1)];
+}
+
 /**
  * axienet_dma_in32 - Memory mapped Axi DMA register read
  * @lp:		Pointer to axienet local structure
@@ -589,10 +609,6 @@ static int axienet_device_reset(struct net_device *ndev)
 	struct axienet_local *lp = netdev_priv(ndev);
 	int ret;
 
-	ret = __axienet_device_reset(lp);
-	if (ret)
-		return ret;
-
 	lp->max_frm_size = XAE_MAX_VLAN_FRAME_SIZE;
 	lp->options |= XAE_OPTION_VLAN;
 	lp->options &= (~XAE_OPTION_JUMBO);
@@ -606,11 +622,17 @@ static int axienet_device_reset(struct net_device *ndev)
 			lp->options |= XAE_OPTION_JUMBO;
 	}
 
-	ret = axienet_dma_bd_init(ndev);
-	if (ret) {
-		netdev_err(ndev, "%s: descriptor allocation failed\n",
-			   __func__);
-		return ret;
+	if (!lp->use_dmaengine) {
+		ret = __axienet_device_reset(lp);
+		if (ret)
+			return ret;
+
+		ret = axienet_dma_bd_init(ndev);
+		if (ret) {
+			netdev_err(ndev, "%s: descriptor allocation failed\n",
+				   __func__);
+			return ret;
+		}
 	}
 
 	axienet_status = axienet_ior(lp, XAE_RCW1_OFFSET);
@@ -725,6 +747,128 @@ static inline int axienet_check_tx_bd_space(struct axienet_local *lp,
 	return 0;
 }
 
+/**
+ * axienet_dma_tx_cb - DMA engine callback for TX channel.
+ * @data:       Pointer to the axienet_local structure.
+ * @result:     error reporting through dmaengine_result.
+ * This function is called by dmaengine driver for TX channel to notify
+ * that the transmit is done.
+ */
+static void axienet_dma_tx_cb(void *data, const struct dmaengine_result *result)
+{
+	struct skbuf_dma_descriptor *skbuf_dma;
+	struct axienet_local *lp = data;
+	struct netdev_queue *txq;
+	int len;
+
+	skbuf_dma = axienet_get_tx_desc(lp, lp->tx_ring_tail++);
+	len = skbuf_dma->skb->len;
+	txq = skb_get_tx_queue(lp->ndev, skbuf_dma->skb);
+	u64_stats_update_begin(&lp->tx_stat_sync);
+	u64_stats_add(&lp->tx_bytes, len);
+	u64_stats_add(&lp->tx_packets, 1);
+	u64_stats_update_end(&lp->tx_stat_sync);
+	dma_unmap_sg(lp->dev, skbuf_dma->sgl, skbuf_dma->sg_len, DMA_TO_DEVICE);
+	dev_consume_skb_any(skbuf_dma->skb);
+	netif_txq_completed_wake(txq, 1, len,
+				 CIRC_SPACE(lp->tx_ring_head, lp->tx_ring_tail, TX_BD_NUM_MAX),
+				 2 * MAX_SKB_FRAGS);
+}
+
+/**
+ * axienet_start_xmit_dmaengine - Starts the transmission.
+ * @skb:        sk_buff pointer that contains data to be Txed.
+ * @ndev:       Pointer to net_device structure.
+ *
+ * Return: NETDEV_TX_OK on success or any non space errors.
+ *         NETDEV_TX_BUSY when free element in TX skb ring buffer
+ *         is not available.
+ *
+ * This function is invoked to initiate transmission. The
+ * function sets the skbs, register dma callback API and submit
+ * the dma transaction.
+ * Additionally if checksum offloading is supported,
+ * it populates AXI Stream Control fields with appropriate values.
+ */
+static netdev_tx_t
+axienet_start_xmit_dmaengine(struct sk_buff *skb, struct net_device *ndev)
+{
+	struct dma_async_tx_descriptor *dma_tx_desc = NULL;
+	struct axienet_local *lp = netdev_priv(ndev);
+	u32 app_metadata[DMA_NUM_APP_WORDS] = {0};
+	struct skbuf_dma_descriptor *skbuf_dma;
+	struct dma_device *dma_dev;
+	struct netdev_queue *txq;
+	u32 csum_start_off;
+	u32 csum_index_off;
+	int sg_len;
+	int ret;
+
+	dma_dev = lp->tx_chan->device;
+	sg_len = skb_shinfo(skb)->nr_frags + 1;
+	if (CIRC_SPACE(lp->tx_ring_head, lp->tx_ring_tail, TX_BD_NUM_MAX) <= sg_len) {
+		netif_stop_queue(ndev);
+		if (net_ratelimit())
+			netdev_warn(ndev, "TX ring unexpectedly full\n");
+		return NETDEV_TX_BUSY;
+	}
+
+	skbuf_dma = axienet_get_tx_desc(lp, lp->tx_ring_head);
+	if (!skbuf_dma)
+		goto xmit_error_drop_skb;
+
+	lp->tx_ring_head++;
+	sg_init_table(skbuf_dma->sgl, sg_len);
+	ret = skb_to_sgvec(skb, skbuf_dma->sgl, 0, skb->len);
+	if (ret < 0)
+		goto xmit_error_drop_skb;
+
+	ret = dma_map_sg(lp->dev, skbuf_dma->sgl, sg_len, DMA_TO_DEVICE);
+	if (!ret)
+		goto xmit_error_drop_skb;
+
+	/* Fill up app fields for checksum */
+	if (skb->ip_summed == CHECKSUM_PARTIAL) {
+		if (lp->features & XAE_FEATURE_FULL_TX_CSUM) {
+			/* Tx Full Checksum Offload Enabled */
+			app_metadata[0] |= 2;
+		} else if (lp->features & XAE_FEATURE_PARTIAL_TX_CSUM) {
+			csum_start_off = skb_transport_offset(skb);
+			csum_index_off = csum_start_off + skb->csum_offset;
+			/* Tx Partial Checksum Offload Enabled */
+			app_metadata[0] |= 1;
+			app_metadata[1] = (csum_start_off << 16) | csum_index_off;
+		}
+	} else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
+		app_metadata[0] |= 2; /* Tx Full Checksum Offload Enabled */
+	}
+
+	dma_tx_desc = dma_dev->device_prep_slave_sg(lp->tx_chan, skbuf_dma->sgl,
+			sg_len, DMA_MEM_TO_DEV,
+			DMA_PREP_INTERRUPT, (void *)app_metadata);
+	if (!dma_tx_desc)
+		goto xmit_error_unmap_sg;
+
+	skbuf_dma->skb = skb;
+	skbuf_dma->sg_len = sg_len;
+	dma_tx_desc->callback_param = lp;
+	dma_tx_desc->callback_result = axienet_dma_tx_cb;
+	dmaengine_submit(dma_tx_desc);
+	dma_async_issue_pending(lp->tx_chan);
+	txq = skb_get_tx_queue(lp->ndev, skb);
+	netdev_tx_sent_queue(txq, skb->len);
+	netif_txq_maybe_stop(txq, CIRC_SPACE(lp->tx_ring_head, lp->tx_ring_tail, TX_BD_NUM_MAX),
+			     MAX_SKB_FRAGS + 1, 2 * MAX_SKB_FRAGS);
+
+	return NETDEV_TX_OK;
+
+xmit_error_unmap_sg:
+	dma_unmap_sg(lp->dev, skbuf_dma->sgl, sg_len, DMA_TO_DEVICE);
+xmit_error_drop_skb:
+	dev_kfree_skb_any(skb);
+	return NETDEV_TX_OK;
+}
+
 /**
  * axienet_tx_poll - Invoked once a transmit is completed by the
  * Axi DMA Tx channel.
@@ -891,6 +1035,42 @@ axienet_start_xmit(struct sk_buff *skb, struct net_device *ndev)
 	return NETDEV_TX_OK;
 }
 
+/**
+ * axienet_dma_rx_cb - DMA engine callback for RX channel.
+ * @data:       Pointer to the skbuf_dma_descriptor structure.
+ * @result:     error reporting through dmaengine_result.
+ * This function is called by dmaengine driver for RX channel to notify
+ * that the packet is received.
+ */
+static void axienet_dma_rx_cb(void *data, const struct dmaengine_result *result)
+{
+	struct skbuf_dma_descriptor *skbuf_dma;
+	size_t meta_len, meta_max_len, rx_len;
+	struct axienet_local *lp = data;
+	struct sk_buff *skb;
+	u32 *app_metadata;
+
+	skbuf_dma = axienet_get_rx_desc(lp, lp->rx_ring_tail++);
+	skb = skbuf_dma->skb;
+	app_metadata = dmaengine_desc_get_metadata_ptr(skbuf_dma->desc, &meta_len,
+						       &meta_max_len);
+	dma_unmap_single(lp->dev, skbuf_dma->dma_address, lp->max_frm_size,
+			 DMA_FROM_DEVICE);
+	/* TODO: Derive app word index programmatically */
+	rx_len = (app_metadata[LEN_APP] & 0xFFFF);
+	skb_put(skb, rx_len);
+	skb->protocol = eth_type_trans(skb, lp->ndev);
+	skb->ip_summed = CHECKSUM_NONE;
+
+	__netif_rx(skb);
+	u64_stats_update_begin(&lp->rx_stat_sync);
+	u64_stats_add(&lp->rx_packets, 1);
+	u64_stats_add(&lp->rx_bytes, rx_len);
+	u64_stats_update_end(&lp->rx_stat_sync);
+	axienet_rx_submit_desc(lp->ndev);
+	dma_async_issue_pending(lp->rx_chan);
+}
+
 /**
  * axienet_rx_poll - Triggered by RX ISR to complete the BD processing.
  * @napi:	Pointer to NAPI structure.
@@ -1125,40 +1305,158 @@ static irqreturn_t axienet_eth_irq(int irq, void *_ndev)
 static void axienet_dma_err_handler(struct work_struct *work);
 
 /**
- * axienet_open - Driver open routine.
- * @ndev:	Pointer to net_device structure
+ * axienet_rx_submit_desc - Submit the rx descriptors to dmaengine.
+ * allocate skbuff, map the scatterlist and obtain a descriptor
+ * and then add the callback information and submit descriptor.
+ *
+ * @ndev:	net_device pointer
+ *
+ */
+static void axienet_rx_submit_desc(struct net_device *ndev)
+{
+	struct dma_async_tx_descriptor *dma_rx_desc = NULL;
+	struct axienet_local *lp = netdev_priv(ndev);
+	struct skbuf_dma_descriptor *skbuf_dma;
+	struct sk_buff *skb;
+	dma_addr_t addr;
+
+	skbuf_dma = axienet_get_rx_desc(lp, lp->rx_ring_head);
+	if (!skbuf_dma)
+		return;
+
+	lp->rx_ring_head++;
+	skb = netdev_alloc_skb(ndev, lp->max_frm_size);
+	if (!skb)
+		return;
+
+	sg_init_table(skbuf_dma->sgl, 1);
+	addr = dma_map_single(lp->dev, skb->data, lp->max_frm_size, DMA_FROM_DEVICE);
+	if (unlikely(dma_mapping_error(lp->dev, addr))) {
+		if (net_ratelimit())
+			netdev_err(ndev, "DMA mapping error\n");
+		goto rx_submit_err_free_skb;
+	}
+	sg_dma_address(skbuf_dma->sgl) = addr;
+	sg_dma_len(skbuf_dma->sgl) = lp->max_frm_size;
+	dma_rx_desc = dmaengine_prep_slave_sg(lp->rx_chan, skbuf_dma->sgl,
+					      1, DMA_DEV_TO_MEM,
+					      DMA_PREP_INTERRUPT);
+	if (!dma_rx_desc)
+		goto rx_submit_err_unmap_skb;
+
+	skbuf_dma->skb = skb;
+	skbuf_dma->dma_address = sg_dma_address(skbuf_dma->sgl);
+	skbuf_dma->desc = dma_rx_desc;
+	dma_rx_desc->callback_param = lp;
+	dma_rx_desc->callback_result = axienet_dma_rx_cb;
+	dmaengine_submit(dma_rx_desc);
+
+	return;
+
+rx_submit_err_unmap_skb:
+	dma_unmap_single(lp->dev, addr, lp->max_frm_size, DMA_FROM_DEVICE);
+rx_submit_err_free_skb:
+	dev_kfree_skb(skb);
+}
+
+/**
+ * axienet_init_dmaengine - init the dmaengine code.
+ * @ndev:       Pointer to net_device structure
  *
  * Return: 0, on success.
- *	    non-zero error value on failure
+ *          non-zero error value on failure
  *
- * This is the driver open routine. It calls phylink_start to start the
- * PHY device.
- * It also allocates interrupt service routines, enables the interrupt lines
- * and ISR handling. Axi Ethernet core is reset through Axi DMA core. Buffer
- * descriptors are initialized.
+ * This is the dmaengine initialization code.
  */
-static int axienet_open(struct net_device *ndev)
+static int axienet_init_dmaengine(struct net_device *ndev)
 {
-	int ret;
 	struct axienet_local *lp = netdev_priv(ndev);
+	struct skbuf_dma_descriptor *skbuf_dma;
+	int i, ret;
 
-	dev_dbg(&ndev->dev, "axienet_open()\n");
+	lp->tx_chan = dma_request_chan(lp->dev, "tx_chan0");
+	if (IS_ERR(lp->tx_chan)) {
+		dev_err(lp->dev, "No Ethernet DMA (TX) channel found\n");
+		return PTR_ERR(lp->tx_chan);
+	}
 
-	/* When we do an Axi Ethernet reset, it resets the complete core
-	 * including the MDIO. MDIO must be disabled before resetting.
-	 * Hold MDIO bus lock to avoid MDIO accesses during the reset.
-	 */
-	axienet_lock_mii(lp);
-	ret = axienet_device_reset(ndev);
-	axienet_unlock_mii(lp);
+	lp->rx_chan = dma_request_chan(lp->dev, "rx_chan0");
+	if (IS_ERR(lp->rx_chan)) {
+		ret = PTR_ERR(lp->rx_chan);
+		dev_err(lp->dev, "No Ethernet DMA (RX) channel found\n");
+		goto err_dma_release_tx;
+	}
 
-	ret = phylink_of_phy_connect(lp->phylink, lp->dev->of_node, 0);
-	if (ret) {
-		dev_err(lp->dev, "phylink_of_phy_connect() failed: %d\n", ret);
-		return ret;
+	lp->tx_ring_tail = 0;
+	lp->tx_ring_head = 0;
+	lp->rx_ring_tail = 0;
+	lp->rx_ring_head = 0;
+	lp->tx_skb_ring = kcalloc(TX_BD_NUM_MAX, sizeof(*lp->tx_skb_ring),
+				  GFP_KERNEL);
+	if (!lp->tx_skb_ring) {
+		ret = -ENOMEM;
+		goto err_dma_release_rx;
+	}
+	for (i = 0; i < TX_BD_NUM_MAX; i++) {
+		skbuf_dma = kzalloc(sizeof(*skbuf_dma), GFP_KERNEL);
+		if (!skbuf_dma) {
+			ret = -ENOMEM;
+			goto err_free_tx_skb_ring;
+		}
+		lp->tx_skb_ring[i] = skbuf_dma;
 	}
 
-	phylink_start(lp->phylink);
+	lp->rx_skb_ring = kcalloc(RX_BUF_NUM_DEFAULT, sizeof(*lp->rx_skb_ring),
+				  GFP_KERNEL);
+	if (!lp->rx_skb_ring) {
+		ret = -ENOMEM;
+		goto err_free_tx_skb_ring;
+	}
+	for (i = 0; i < RX_BUF_NUM_DEFAULT; i++) {
+		skbuf_dma = kzalloc(sizeof(*skbuf_dma), GFP_KERNEL);
+		if (!skbuf_dma) {
+			ret = -ENOMEM;
+			goto err_free_rx_skb_ring;
+		}
+		lp->rx_skb_ring[i] = skbuf_dma;
+	}
+	/* TODO: Instead of BD_NUM_DEFAULT use runtime support */
+	for (i = 0; i < RX_BUF_NUM_DEFAULT; i++)
+		axienet_rx_submit_desc(ndev);
+	dma_async_issue_pending(lp->rx_chan);
+
+	return 0;
+
+err_free_rx_skb_ring:
+	for (i = 0; i < RX_BUF_NUM_DEFAULT; i++)
+		kfree(lp->rx_skb_ring[i]);
+	kfree(lp->rx_skb_ring);
+err_free_tx_skb_ring:
+	for (i = 0; i < TX_BD_NUM_MAX; i++)
+		kfree(lp->tx_skb_ring[i]);
+	kfree(lp->tx_skb_ring);
+err_dma_release_rx:
+	dma_release_channel(lp->rx_chan);
+err_dma_release_tx:
+	dma_release_channel(lp->tx_chan);
+	return ret;
+}
+
+/**
+ * axienet_init_legacy_dma - init the dma legacy code.
+ * @ndev:       Pointer to net_device structure
+ *
+ * Return: 0, on success.
+ *          non-zero error value on failure
+ *
+ * This is the dma  initialization code. It also allocates interrupt
+ * service routines, enables the interrupt lines and ISR handling.
+ *
+ */
+static int axienet_init_legacy_dma(struct net_device *ndev)
+{
+	int ret;
+	struct axienet_local *lp = netdev_priv(ndev);
 
 	/* Enable worker thread for Axi DMA error handling */
 	INIT_WORK(&lp->dma_err_task, axienet_dma_err_handler);
@@ -1193,13 +1491,76 @@ static int axienet_open(struct net_device *ndev)
 err_tx_irq:
 	napi_disable(&lp->napi_tx);
 	napi_disable(&lp->napi_rx);
-	phylink_stop(lp->phylink);
-	phylink_disconnect_phy(lp->phylink);
 	cancel_work_sync(&lp->dma_err_task);
 	dev_err(lp->dev, "request_irq() failed\n");
 	return ret;
 }
 
+/**
+ * axienet_open - Driver open routine.
+ * @ndev:	Pointer to net_device structure
+ *
+ * Return: 0, on success.
+ *	    non-zero error value on failure
+ *
+ * This is the driver open routine. It calls phylink_start to start the
+ * PHY device.
+ * It also allocates interrupt service routines, enables the interrupt lines
+ * and ISR handling. Axi Ethernet core is reset through Axi DMA core. Buffer
+ * descriptors are initialized.
+ */
+static int axienet_open(struct net_device *ndev)
+{
+	int ret;
+	struct axienet_local *lp = netdev_priv(ndev);
+
+	dev_dbg(&ndev->dev, "%s\n", __func__);
+
+	/* When we do an Axi Ethernet reset, it resets the complete core
+	 * including the MDIO. MDIO must be disabled before resetting.
+	 * Hold MDIO bus lock to avoid MDIO accesses during the reset.
+	 */
+	axienet_lock_mii(lp);
+	ret = axienet_device_reset(ndev);
+	axienet_unlock_mii(lp);
+
+	ret = phylink_of_phy_connect(lp->phylink, lp->dev->of_node, 0);
+	if (ret) {
+		dev_err(lp->dev, "phylink_of_phy_connect() failed: %d\n", ret);
+		return ret;
+	}
+
+	phylink_start(lp->phylink);
+
+	if (lp->use_dmaengine) {
+		/* Enable interrupts for Axi Ethernet core (if defined) */
+		if (lp->eth_irq > 0) {
+			ret = request_irq(lp->eth_irq, axienet_eth_irq, IRQF_SHARED,
+					  ndev->name, ndev);
+			if (ret)
+				goto err_phy;
+		}
+
+		ret = axienet_init_dmaengine(ndev);
+		if (ret < 0)
+			goto err_free_eth_irq;
+	} else {
+		ret = axienet_init_legacy_dma(ndev);
+		if (ret)
+			goto err_phy;
+	}
+
+	return 0;
+
+err_free_eth_irq:
+	if (lp->eth_irq > 0)
+		free_irq(lp->eth_irq, ndev);
+err_phy:
+	phylink_stop(lp->phylink);
+	phylink_disconnect_phy(lp->phylink);
+	return ret;
+}
+
 /**
  * axienet_stop - Driver stop routine.
  * @ndev:	Pointer to net_device structure
@@ -1213,11 +1574,14 @@ static int axienet_open(struct net_device *ndev)
 static int axienet_stop(struct net_device *ndev)
 {
 	struct axienet_local *lp = netdev_priv(ndev);
+	int i;
 
 	dev_dbg(&ndev->dev, "axienet_close()\n");
 
-	napi_disable(&lp->napi_tx);
-	napi_disable(&lp->napi_rx);
+	if (!lp->use_dmaengine) {
+		napi_disable(&lp->napi_tx);
+		napi_disable(&lp->napi_rx);
+	}
 
 	phylink_stop(lp->phylink);
 	phylink_disconnect_phy(lp->phylink);
@@ -1225,18 +1589,33 @@ static int axienet_stop(struct net_device *ndev)
 	axienet_setoptions(ndev, lp->options &
 			   ~(XAE_OPTION_TXEN | XAE_OPTION_RXEN));
 
-	axienet_dma_stop(lp);
+	if (!lp->use_dmaengine) {
+		axienet_dma_stop(lp);
+		cancel_work_sync(&lp->dma_err_task);
+		free_irq(lp->tx_irq, ndev);
+		free_irq(lp->rx_irq, ndev);
+		axienet_dma_bd_release(ndev);
+	} else {
+		dmaengine_terminate_sync(lp->tx_chan);
+		dmaengine_synchronize(lp->tx_chan);
+		dmaengine_terminate_sync(lp->rx_chan);
+		dmaengine_synchronize(lp->rx_chan);
+
+		for (i = 0; i < TX_BD_NUM_MAX; i++)
+			kfree(lp->tx_skb_ring[i]);
+		kfree(lp->tx_skb_ring);
+		for (i = 0; i < RX_BUF_NUM_DEFAULT; i++)
+			kfree(lp->rx_skb_ring[i]);
+		kfree(lp->rx_skb_ring);
+
+		dma_release_channel(lp->rx_chan);
+		dma_release_channel(lp->tx_chan);
+	}
 
 	axienet_iow(lp, XAE_IE_OFFSET, 0);
 
-	cancel_work_sync(&lp->dma_err_task);
-
 	if (lp->eth_irq > 0)
 		free_irq(lp->eth_irq, ndev);
-	free_irq(lp->tx_irq, ndev);
-	free_irq(lp->rx_irq, ndev);
-
-	axienet_dma_bd_release(ndev);
 	return 0;
 }
 
@@ -1333,6 +1712,18 @@ static const struct net_device_ops axienet_netdev_ops = {
 #endif
 };
 
+static const struct net_device_ops axienet_netdev_dmaengine_ops = {
+	.ndo_open = axienet_open,
+	.ndo_stop = axienet_stop,
+	.ndo_start_xmit = axienet_start_xmit_dmaengine,
+	.ndo_get_stats64 = axienet_get_stats64,
+	.ndo_change_mtu	= axienet_change_mtu,
+	.ndo_set_mac_address = netdev_set_mac_address,
+	.ndo_validate_addr = eth_validate_addr,
+	.ndo_eth_ioctl = axienet_ioctl,
+	.ndo_set_rx_mode = axienet_set_multicast_list,
+};
+
 /**
  * axienet_ethtools_get_drvinfo - Get various Axi Ethernet driver information.
  * @ndev:	Pointer to net_device structure
@@ -1412,14 +1803,16 @@ static void axienet_ethtools_get_regs(struct net_device *ndev,
 	data[29] = axienet_ior(lp, XAE_FMI_OFFSET);
 	data[30] = axienet_ior(lp, XAE_AF0_OFFSET);
 	data[31] = axienet_ior(lp, XAE_AF1_OFFSET);
-	data[32] = axienet_dma_in32(lp, XAXIDMA_TX_CR_OFFSET);
-	data[33] = axienet_dma_in32(lp, XAXIDMA_TX_SR_OFFSET);
-	data[34] = axienet_dma_in32(lp, XAXIDMA_TX_CDESC_OFFSET);
-	data[35] = axienet_dma_in32(lp, XAXIDMA_TX_TDESC_OFFSET);
-	data[36] = axienet_dma_in32(lp, XAXIDMA_RX_CR_OFFSET);
-	data[37] = axienet_dma_in32(lp, XAXIDMA_RX_SR_OFFSET);
-	data[38] = axienet_dma_in32(lp, XAXIDMA_RX_CDESC_OFFSET);
-	data[39] = axienet_dma_in32(lp, XAXIDMA_RX_TDESC_OFFSET);
+	if (!lp->use_dmaengine) {
+		data[32] = axienet_dma_in32(lp, XAXIDMA_TX_CR_OFFSET);
+		data[33] = axienet_dma_in32(lp, XAXIDMA_TX_SR_OFFSET);
+		data[34] = axienet_dma_in32(lp, XAXIDMA_TX_CDESC_OFFSET);
+		data[35] = axienet_dma_in32(lp, XAXIDMA_TX_TDESC_OFFSET);
+		data[36] = axienet_dma_in32(lp, XAXIDMA_RX_CR_OFFSET);
+		data[37] = axienet_dma_in32(lp, XAXIDMA_RX_SR_OFFSET);
+		data[38] = axienet_dma_in32(lp, XAXIDMA_RX_CDESC_OFFSET);
+		data[39] = axienet_dma_in32(lp, XAXIDMA_RX_TDESC_OFFSET);
+	}
 }
 
 static void
@@ -1863,7 +2256,6 @@ static int axienet_probe(struct platform_device *pdev)
 	SET_NETDEV_DEV(ndev, &pdev->dev);
 	ndev->flags &= ~IFF_MULTICAST;  /* clear multicast */
 	ndev->features = NETIF_F_SG;
-	ndev->netdev_ops = &axienet_netdev_ops;
 	ndev->ethtool_ops = &axienet_ethtool_ops;
 
 	/* MTU range: 64 - 9000 */
@@ -1880,9 +2272,6 @@ static int axienet_probe(struct platform_device *pdev)
 	u64_stats_init(&lp->rx_stat_sync);
 	u64_stats_init(&lp->tx_stat_sync);
 
-	netif_napi_add(ndev, &lp->napi_rx, axienet_rx_poll);
-	netif_napi_add(ndev, &lp->napi_tx, axienet_tx_poll);
-
 	lp->axi_clk = devm_clk_get_optional(&pdev->dev, "s_axi_lite_clk");
 	if (!lp->axi_clk) {
 		/* For backward compatibility, if named AXI clock is not present,
@@ -2008,82 +2397,118 @@ static int axienet_probe(struct platform_device *pdev)
 		goto cleanup_clk;
 	}
 
-	/* Find the DMA node, map the DMA registers, and decode the DMA IRQs */
-	np = of_parse_phandle(pdev->dev.of_node, "axistream-connected", 0);
-	if (np) {
-		struct resource dmares;
+	if (!of_find_property(pdev->dev.of_node, "dmas", NULL)) {
+		/* Find the DMA node, map the DMA registers, and decode the DMA IRQs */
+		np = of_parse_phandle(pdev->dev.of_node, "axistream-connected", 0);
 
-		ret = of_address_to_resource(np, 0, &dmares);
-		if (ret) {
-			dev_err(&pdev->dev,
-				"unable to get DMA resource\n");
+		if (np) {
+			struct resource dmares;
+
+			ret = of_address_to_resource(np, 0, &dmares);
+			if (ret) {
+				dev_err(&pdev->dev,
+					"unable to get DMA resource\n");
+				of_node_put(np);
+				goto cleanup_clk;
+			}
+			lp->dma_regs = devm_ioremap_resource(&pdev->dev,
+							     &dmares);
+			lp->rx_irq = irq_of_parse_and_map(np, 1);
+			lp->tx_irq = irq_of_parse_and_map(np, 0);
 			of_node_put(np);
+			lp->eth_irq = platform_get_irq_optional(pdev, 0);
+		} else {
+			/* Check for these resources directly on the Ethernet node. */
+			lp->dma_regs = devm_platform_get_and_ioremap_resource(pdev, 1, NULL);
+			lp->rx_irq = platform_get_irq(pdev, 1);
+			lp->tx_irq = platform_get_irq(pdev, 0);
+			lp->eth_irq = platform_get_irq_optional(pdev, 2);
+		}
+		if (IS_ERR(lp->dma_regs)) {
+			dev_err(&pdev->dev, "could not map DMA regs\n");
+			ret = PTR_ERR(lp->dma_regs);
+			goto cleanup_clk;
+		}
+		if (lp->rx_irq <= 0 || lp->tx_irq <= 0) {
+			dev_err(&pdev->dev, "could not determine irqs\n");
+			ret = -ENOMEM;
 			goto cleanup_clk;
 		}
-		lp->dma_regs = devm_ioremap_resource(&pdev->dev,
-						     &dmares);
-		lp->rx_irq = irq_of_parse_and_map(np, 1);
-		lp->tx_irq = irq_of_parse_and_map(np, 0);
-		of_node_put(np);
-		lp->eth_irq = platform_get_irq_optional(pdev, 0);
-	} else {
-		/* Check for these resources directly on the Ethernet node. */
-		lp->dma_regs = devm_platform_get_and_ioremap_resource(pdev, 1, NULL);
-		lp->rx_irq = platform_get_irq(pdev, 1);
-		lp->tx_irq = platform_get_irq(pdev, 0);
-		lp->eth_irq = platform_get_irq_optional(pdev, 2);
-	}
-	if (IS_ERR(lp->dma_regs)) {
-		dev_err(&pdev->dev, "could not map DMA regs\n");
-		ret = PTR_ERR(lp->dma_regs);
-		goto cleanup_clk;
-	}
-	if ((lp->rx_irq <= 0) || (lp->tx_irq <= 0)) {
-		dev_err(&pdev->dev, "could not determine irqs\n");
-		ret = -ENOMEM;
-		goto cleanup_clk;
-	}
 
-	/* Reset core now that clocks are enabled, prior to accessing MDIO */
-	ret = __axienet_device_reset(lp);
-	if (ret)
-		goto cleanup_clk;
+		/* Reset core now that clocks are enabled, prior to accessing MDIO */
+		ret = __axienet_device_reset(lp);
+		if (ret)
+			goto cleanup_clk;
+
+		/* Autodetect the need for 64-bit DMA pointers.
+		 * When the IP is configured for a bus width bigger than 32 bits,
+		 * writing the MSB registers is mandatory, even if they are all 0.
+		 * We can detect this case by writing all 1's to one such register
+		 * and see if that sticks: when the IP is configured for 32 bits
+		 * only, those registers are RES0.
+		 * Those MSB registers were introduced in IP v7.1, which we check first.
+		 */
+		if ((axienet_ior(lp, XAE_ID_OFFSET) >> 24) >= 0x9) {
+			void __iomem *desc = lp->dma_regs + XAXIDMA_TX_CDESC_OFFSET + 4;
 
-	/* Autodetect the need for 64-bit DMA pointers.
-	 * When the IP is configured for a bus width bigger than 32 bits,
-	 * writing the MSB registers is mandatory, even if they are all 0.
-	 * We can detect this case by writing all 1's to one such register
-	 * and see if that sticks: when the IP is configured for 32 bits
-	 * only, those registers are RES0.
-	 * Those MSB registers were introduced in IP v7.1, which we check first.
-	 */
-	if ((axienet_ior(lp, XAE_ID_OFFSET) >> 24) >= 0x9) {
-		void __iomem *desc = lp->dma_regs + XAXIDMA_TX_CDESC_OFFSET + 4;
-
-		iowrite32(0x0, desc);
-		if (ioread32(desc) == 0) {	/* sanity check */
-			iowrite32(0xffffffff, desc);
-			if (ioread32(desc) > 0) {
-				lp->features |= XAE_FEATURE_DMA_64BIT;
-				addr_width = 64;
-				dev_info(&pdev->dev,
-					 "autodetected 64-bit DMA range\n");
-			}
 			iowrite32(0x0, desc);
+			if (ioread32(desc) == 0) {	/* sanity check */
+				iowrite32(0xffffffff, desc);
+				if (ioread32(desc) > 0) {
+					lp->features |= XAE_FEATURE_DMA_64BIT;
+					addr_width = 64;
+					dev_info(&pdev->dev,
+						 "autodetected 64-bit DMA range\n");
+				}
+				iowrite32(0x0, desc);
+			}
+		}
+		if (!IS_ENABLED(CONFIG_64BIT) && lp->features & XAE_FEATURE_DMA_64BIT) {
+			dev_err(&pdev->dev, "64-bit addressable DMA is not compatible with 32-bit archecture\n");
+			ret = -EINVAL;
+			goto cleanup_clk;
 		}
-	}
-	if (!IS_ENABLED(CONFIG_64BIT) && lp->features & XAE_FEATURE_DMA_64BIT) {
-		dev_err(&pdev->dev, "64-bit addressable DMA is not compatible with 32-bit archecture\n");
-		ret = -EINVAL;
-		goto cleanup_clk;
-	}
 
-	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(addr_width));
-	if (ret) {
-		dev_err(&pdev->dev, "No suitable DMA available\n");
-		goto cleanup_clk;
+		ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(addr_width));
+		if (ret) {
+			dev_err(&pdev->dev, "No suitable DMA available\n");
+			goto cleanup_clk;
+		}
+		netif_napi_add(ndev, &lp->napi_rx, axienet_rx_poll);
+		netif_napi_add(ndev, &lp->napi_tx, axienet_tx_poll);
+	} else {
+		struct xilinx_vdma_config cfg;
+		struct dma_chan *tx_chan;
+
+		lp->eth_irq = platform_get_irq_optional(pdev, 0);
+		if (lp->eth_irq < 0 && lp->eth_irq != -ENXIO) {
+			ret = lp->eth_irq;
+			goto cleanup_clk;
+		}
+		tx_chan = dma_request_chan(lp->dev, "tx_chan0");
+		if (IS_ERR(tx_chan)) {
+			ret = PTR_ERR(tx_chan);
+			dev_err_probe(lp->dev, ret, "No Ethernet DMA (TX) channel found\n");
+			goto cleanup_clk;
+		}
+
+		cfg.reset = 1;
+		/* As name says VDMA but it has support for DMA channel reset */
+		ret = xilinx_vdma_channel_set_config(tx_chan, &cfg);
+		if (ret < 0) {
+			dev_err(&pdev->dev, "Reset channel failed\n");
+			dma_release_channel(tx_chan);
+			goto cleanup_clk;
+		}
+
+		dma_release_channel(tx_chan);
+		lp->use_dmaengine = 1;
 	}
 
+	if (lp->use_dmaengine)
+		ndev->netdev_ops = &axienet_netdev_dmaengine_ops;
+	else
+		ndev->netdev_ops = &axienet_netdev_ops;
 	/* Check for Ethernet core IRQ (optional) */
 	if (lp->eth_irq <= 0)
 		dev_info(&pdev->dev, "Ethernet core IRQ not defined\n");
@@ -2099,8 +2524,8 @@ static int axienet_probe(struct platform_device *pdev)
 	}
 
 	lp->coalesce_count_rx = XAXIDMA_DFT_RX_THRESHOLD;
-	lp->coalesce_usec_rx = XAXIDMA_DFT_RX_USEC;
 	lp->coalesce_count_tx = XAXIDMA_DFT_TX_THRESHOLD;
+	lp->coalesce_usec_rx = XAXIDMA_DFT_RX_USEC;
 	lp->coalesce_usec_tx = XAXIDMA_DFT_TX_USEC;
 
 	ret = axienet_mdio_setup(lp);
diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c
index 3ba3c8fb28a5..706ea5263e87 100644
--- a/drivers/net/hyperv/netvsc_drv.c
+++ b/drivers/net/hyperv/netvsc_drv.c
@@ -2206,9 +2206,6 @@ static int netvsc_vf_join(struct net_device *vf_netdev,
 		goto upper_link_failed;
 	}
 
-	/* set slave flag before open to prevent IPv6 addrconf */
-	vf_netdev->flags |= IFF_SLAVE;
-
 	schedule_delayed_work(&ndev_ctx->vf_takeover, VF_TAKEOVER_INT);
 
 	call_netdevice_notifiers(NETDEV_JOIN, vf_netdev);
@@ -2315,16 +2312,18 @@ static struct net_device *get_netvsc_byslot(const struct net_device *vf_netdev)
 
 	}
 
-	/* Fallback path to check synthetic vf with
-	 * help of mac addr
+	/* Fallback path to check synthetic vf with help of mac addr.
+	 * Because this function can be called before vf_netdev is
+	 * initialized (NETDEV_POST_INIT) when its perm_addr has not been copied
+	 * from dev_addr, also try to match to its dev_addr.
+	 * Note: On Hyper-V and Azure, it's not possible to set a MAC address
+	 * on a VF that matches to the MAC of a unrelated NETVSC device.
 	 */
 	list_for_each_entry(ndev_ctx, &netvsc_dev_list, list) {
 		ndev = hv_get_drvdata(ndev_ctx->device_ctx);
-		if (ether_addr_equal(vf_netdev->perm_addr, ndev->perm_addr)) {
-			netdev_notice(vf_netdev,
-				      "falling back to mac addr based matching\n");
+		if (ether_addr_equal(vf_netdev->perm_addr, ndev->perm_addr) ||
+		    ether_addr_equal(vf_netdev->dev_addr, ndev->perm_addr))
 			return ndev;
-		}
 	}
 
 	netdev_notice(vf_netdev,
@@ -2332,6 +2331,19 @@ static struct net_device *get_netvsc_byslot(const struct net_device *vf_netdev)
 	return NULL;
 }
 
+static int netvsc_prepare_bonding(struct net_device *vf_netdev)
+{
+	struct net_device *ndev;
+
+	ndev = get_netvsc_byslot(vf_netdev);
+	if (!ndev)
+		return NOTIFY_DONE;
+
+	/* set slave flag before open to prevent IPv6 addrconf */
+	vf_netdev->flags |= IFF_SLAVE;
+	return NOTIFY_DONE;
+}
+
 static int netvsc_register_vf(struct net_device *vf_netdev)
 {
 	struct net_device_context *net_device_ctx;
@@ -2531,15 +2543,6 @@ static int netvsc_probe(struct hv_device *dev,
 		goto devinfo_failed;
 	}
 
-	nvdev = rndis_filter_device_add(dev, device_info);
-	if (IS_ERR(nvdev)) {
-		ret = PTR_ERR(nvdev);
-		netdev_err(net, "unable to add netvsc device (ret %d)\n", ret);
-		goto rndis_failed;
-	}
-
-	eth_hw_addr_set(net, device_info->mac_adr);
-
 	/* We must get rtnl lock before scheduling nvdev->subchan_work,
 	 * otherwise netvsc_subchan_work() can get rtnl lock first and wait
 	 * all subchannels to show up, but that may not happen because
@@ -2547,9 +2550,23 @@ static int netvsc_probe(struct hv_device *dev,
 	 * -> ... -> device_add() -> ... -> __device_attach() can't get
 	 * the device lock, so all the subchannels can't be processed --
 	 * finally netvsc_subchan_work() hangs forever.
+	 *
+	 * The rtnl lock also needs to be held before rndis_filter_device_add()
+	 * which advertises nvsp_2_vsc_capability / sriov bit, and triggers
+	 * VF NIC offering and registering. If VF NIC finished register_netdev()
+	 * earlier it may cause name based config failure.
 	 */
 	rtnl_lock();
 
+	nvdev = rndis_filter_device_add(dev, device_info);
+	if (IS_ERR(nvdev)) {
+		ret = PTR_ERR(nvdev);
+		netdev_err(net, "unable to add netvsc device (ret %d)\n", ret);
+		goto rndis_failed;
+	}
+
+	eth_hw_addr_set(net, device_info->mac_adr);
+
 	if (nvdev->num_chn > 1)
 		schedule_work(&nvdev->subchan_work);
 
@@ -2586,9 +2603,9 @@ static int netvsc_probe(struct hv_device *dev,
 	return 0;
 
 register_failed:
-	rtnl_unlock();
 	rndis_filter_device_remove(dev, nvdev);
 rndis_failed:
+	rtnl_unlock();
 	netvsc_devinfo_put(device_info);
 devinfo_failed:
 	free_percpu(net_device_ctx->vf_stats);
@@ -2753,6 +2770,8 @@ static int netvsc_netdev_event(struct notifier_block *this,
 		return NOTIFY_DONE;
 
 	switch (event) {
+	case NETDEV_POST_INIT:
+		return netvsc_prepare_bonding(event_dev);
 	case NETDEV_REGISTER:
 		return netvsc_register_vf(event_dev);
 	case NETDEV_UNREGISTER:
@@ -2788,12 +2807,17 @@ static int __init netvsc_drv_init(void)
 	}
 	netvsc_ring_bytes = ring_size * PAGE_SIZE;
 
+	register_netdevice_notifier(&netvsc_netdev_notifier);
+
 	ret = vmbus_driver_register(&netvsc_drv);
 	if (ret)
-		return ret;
+		goto err_vmbus_reg;
 
-	register_netdevice_notifier(&netvsc_netdev_notifier);
 	return 0;
+
+err_vmbus_reg:
+	unregister_netdevice_notifier(&netvsc_netdev_notifier);
+	return ret;
 }
 
 MODULE_LICENSE("GPL");
diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c
index c8da94af4161..88dd8a2245b0 100644
--- a/drivers/net/macvlan.c
+++ b/drivers/net/macvlan.c
@@ -1086,20 +1086,8 @@ static int macvlan_ethtool_get_ts_info(struct net_device *dev,
 				       struct ethtool_ts_info *info)
 {
 	struct net_device *real_dev = macvlan_dev_real_dev(dev);
-	const struct ethtool_ops *ops = real_dev->ethtool_ops;
-	struct phy_device *phydev = real_dev->phydev;
 
-	if (phy_has_tsinfo(phydev)) {
-		return phy_ts_info(phydev, info);
-	} else if (ops->get_ts_info) {
-		return ops->get_ts_info(real_dev, info);
-	} else {
-		info->so_timestamping = SOF_TIMESTAMPING_RX_SOFTWARE |
-			SOF_TIMESTAMPING_SOFTWARE;
-		info->phc_index = -1;
-	}
-
-	return 0;
+	return ethtool_get_ts_info_by_layer(real_dev, info);
 }
 
 static netdev_features_t macvlan_fix_features(struct net_device *dev,
diff --git a/drivers/net/netkit.c b/drivers/net/netkit.c
index 5a0f86f38f09..97bd6705c241 100644
--- a/drivers/net/netkit.c
+++ b/drivers/net/netkit.c
@@ -7,6 +7,7 @@
 #include <linux/filter.h>
 #include <linux/netfilter_netdev.h>
 #include <linux/bpf_mprog.h>
+#include <linux/indirect_call_wrapper.h>
 
 #include <net/netkit.h>
 #include <net/dst.h>
@@ -68,6 +69,7 @@ static netdev_tx_t netkit_xmit(struct sk_buff *skb, struct net_device *dev)
 	netdev_tx_t ret_dev = NET_XMIT_SUCCESS;
 	const struct bpf_mprog_entry *entry;
 	struct net_device *peer;
+	int len = skb->len;
 
 	rcu_read_lock();
 	peer = rcu_dereference(nk->peer);
@@ -85,15 +87,22 @@ static netdev_tx_t netkit_xmit(struct sk_buff *skb, struct net_device *dev)
 	case NETKIT_PASS:
 		skb->protocol = eth_type_trans(skb, skb->dev);
 		skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
-		__netif_rx(skb);
+		if (likely(__netif_rx(skb) == NET_RX_SUCCESS)) {
+			dev_sw_netstats_tx_add(dev, 1, len);
+			dev_sw_netstats_rx_add(peer, len);
+		} else {
+			goto drop_stats;
+		}
 		break;
 	case NETKIT_REDIRECT:
+		dev_sw_netstats_tx_add(dev, 1, len);
 		skb_do_redirect(skb);
 		break;
 	case NETKIT_DROP:
 	default:
 drop:
 		kfree_skb(skb);
+drop_stats:
 		dev_core_stats_tx_dropped_inc(dev);
 		ret_dev = NET_XMIT_DROP;
 		break;
@@ -169,11 +178,18 @@ static void netkit_set_headroom(struct net_device *dev, int headroom)
 	rcu_read_unlock();
 }
 
-static struct net_device *netkit_peer_dev(struct net_device *dev)
+INDIRECT_CALLABLE_SCOPE struct net_device *netkit_peer_dev(struct net_device *dev)
 {
 	return rcu_dereference(netkit_priv(dev)->peer);
 }
 
+static void netkit_get_stats(struct net_device *dev,
+			     struct rtnl_link_stats64 *stats)
+{
+	dev_fetch_sw_netstats(stats, dev->tstats);
+	stats->tx_dropped = DEV_STATS_READ(dev, tx_dropped);
+}
+
 static void netkit_uninit(struct net_device *dev);
 
 static const struct net_device_ops netkit_netdev_ops = {
@@ -184,6 +200,7 @@ static const struct net_device_ops netkit_netdev_ops = {
 	.ndo_set_rx_headroom	= netkit_set_headroom,
 	.ndo_get_iflink		= netkit_get_iflink,
 	.ndo_get_peer_dev	= netkit_peer_dev,
+	.ndo_get_stats64	= netkit_get_stats,
 	.ndo_uninit		= netkit_uninit,
 	.ndo_features_check	= passthru_features_check,
 };
@@ -218,6 +235,7 @@ static void netkit_setup(struct net_device *dev)
 
 	ether_setup(dev);
 	dev->max_mtu = ETH_MAX_MTU;
+	dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
 
 	dev->flags |= IFF_NOARP;
 	dev->priv_flags &= ~IFF_TX_SKB_SHARING;
diff --git a/drivers/net/phy/Kconfig b/drivers/net/phy/Kconfig
index 421d2b62918f..25cfc5ded1da 100644
--- a/drivers/net/phy/Kconfig
+++ b/drivers/net/phy/Kconfig
@@ -96,10 +96,7 @@ config ADIN1100_PHY
 	  Currently supports the:
 	  - ADIN1100 - Robust,Industrial, Low Power 10BASE-T1L Ethernet PHY
 
-config AQUANTIA_PHY
-	tristate "Aquantia PHYs"
-	help
-	  Currently supports the Aquantia AQ1202, AQ2104, AQR105, AQR405
+source "drivers/net/phy/aquantia/Kconfig"
 
 config AX88796B_PHY
 	tristate "Asix PHYs"
diff --git a/drivers/net/phy/Makefile b/drivers/net/phy/Makefile
index c945ed9bd14b..f65e85c91fc1 100644
--- a/drivers/net/phy/Makefile
+++ b/drivers/net/phy/Makefile
@@ -35,11 +35,7 @@ obj-y				+= $(sfp-obj-y) $(sfp-obj-m)
 obj-$(CONFIG_ADIN_PHY)		+= adin.o
 obj-$(CONFIG_ADIN1100_PHY)	+= adin1100.o
 obj-$(CONFIG_AMD_PHY)		+= amd.o
-aquantia-objs			+= aquantia_main.o
-ifdef CONFIG_HWMON
-aquantia-objs			+= aquantia_hwmon.o
-endif
-obj-$(CONFIG_AQUANTIA_PHY)	+= aquantia.o
+obj-$(CONFIG_AQUANTIA_PHY)	+= aquantia/
 obj-$(CONFIG_AT803X_PHY)	+= at803x.o
 obj-$(CONFIG_AX88796B_PHY)	+= ax88796b.o
 obj-$(CONFIG_BCM54140_PHY)	+= bcm54140.o
diff --git a/drivers/net/phy/aquantia.h b/drivers/net/phy/aquantia.h
deleted file mode 100644
index c684b65c642c..000000000000
--- a/drivers/net/phy/aquantia.h
+++ /dev/null
@@ -1,16 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/* HWMON driver for Aquantia PHY
- *
- * Author: Nikita Yushchenko <nikita.yoush@cogentembedded.com>
- * Author: Andrew Lunn <andrew@lunn.ch>
- * Author: Heiner Kallweit <hkallweit1@gmail.com>
- */
-
-#include <linux/device.h>
-#include <linux/phy.h>
-
-#if IS_REACHABLE(CONFIG_HWMON)
-int aqr_hwmon_probe(struct phy_device *phydev);
-#else
-static inline int aqr_hwmon_probe(struct phy_device *phydev) { return 0; }
-#endif
diff --git a/drivers/net/phy/aquantia/Kconfig b/drivers/net/phy/aquantia/Kconfig
new file mode 100644
index 000000000000..a35de4b9b554
--- /dev/null
+++ b/drivers/net/phy/aquantia/Kconfig
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config AQUANTIA_PHY
+	tristate "Aquantia PHYs"
+	select CRC_CCITT
+	help
+	  Currently supports the Aquantia AQ1202, AQ2104, AQR105, AQR405
diff --git a/drivers/net/phy/aquantia/Makefile b/drivers/net/phy/aquantia/Makefile
new file mode 100644
index 000000000000..aa77fb63c8ec
--- /dev/null
+++ b/drivers/net/phy/aquantia/Makefile
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0
+aquantia-objs			+= aquantia_main.o aquantia_firmware.o
+ifdef CONFIG_HWMON
+aquantia-objs			+= aquantia_hwmon.o
+endif
+obj-$(CONFIG_AQUANTIA_PHY)	+= aquantia.o
diff --git a/drivers/net/phy/aquantia/aquantia.h b/drivers/net/phy/aquantia/aquantia.h
new file mode 100644
index 000000000000..9ed38972abdb
--- /dev/null
+++ b/drivers/net/phy/aquantia/aquantia.h
@@ -0,0 +1,117 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* HWMON driver for Aquantia PHY
+ *
+ * Author: Nikita Yushchenko <nikita.yoush@cogentembedded.com>
+ * Author: Andrew Lunn <andrew@lunn.ch>
+ * Author: Heiner Kallweit <hkallweit1@gmail.com>
+ */
+
+#include <linux/device.h>
+#include <linux/phy.h>
+
+/* Vendor specific 1, MDIO_MMD_VEND1 */
+#define VEND1_GLOBAL_SC				0x0
+#define VEND1_GLOBAL_SC_SOFT_RESET		BIT(15)
+#define VEND1_GLOBAL_SC_LOW_POWER		BIT(11)
+
+#define VEND1_GLOBAL_FW_ID			0x0020
+#define VEND1_GLOBAL_FW_ID_MAJOR		GENMASK(15, 8)
+#define VEND1_GLOBAL_FW_ID_MINOR		GENMASK(7, 0)
+
+#define VEND1_GLOBAL_MAILBOX_INTERFACE1			0x0200
+#define VEND1_GLOBAL_MAILBOX_INTERFACE1_EXECUTE		BIT(15)
+#define VEND1_GLOBAL_MAILBOX_INTERFACE1_WRITE		BIT(14)
+#define VEND1_GLOBAL_MAILBOX_INTERFACE1_CRC_RESET	BIT(12)
+#define VEND1_GLOBAL_MAILBOX_INTERFACE1_BUSY		BIT(8)
+
+#define VEND1_GLOBAL_MAILBOX_INTERFACE2			0x0201
+#define VEND1_GLOBAL_MAILBOX_INTERFACE3			0x0202
+#define VEND1_GLOBAL_MAILBOX_INTERFACE3_MSW_ADDR_MASK	GENMASK(15, 0)
+#define VEND1_GLOBAL_MAILBOX_INTERFACE3_MSW_ADDR(x)	FIELD_PREP(VEND1_GLOBAL_MAILBOX_INTERFACE3_MSW_ADDR_MASK, (u16)((x) >> 16))
+#define VEND1_GLOBAL_MAILBOX_INTERFACE4			0x0203
+#define VEND1_GLOBAL_MAILBOX_INTERFACE4_LSW_ADDR_MASK	GENMASK(15, 2)
+#define VEND1_GLOBAL_MAILBOX_INTERFACE4_LSW_ADDR(x)	FIELD_PREP(VEND1_GLOBAL_MAILBOX_INTERFACE4_LSW_ADDR_MASK, (u16)(x))
+
+#define VEND1_GLOBAL_MAILBOX_INTERFACE5			0x0204
+#define VEND1_GLOBAL_MAILBOX_INTERFACE5_MSW_DATA_MASK	GENMASK(15, 0)
+#define VEND1_GLOBAL_MAILBOX_INTERFACE5_MSW_DATA(x)	FIELD_PREP(VEND1_GLOBAL_MAILBOX_INTERFACE5_MSW_DATA_MASK, (u16)((x) >> 16))
+#define VEND1_GLOBAL_MAILBOX_INTERFACE6			0x0205
+#define VEND1_GLOBAL_MAILBOX_INTERFACE6_LSW_DATA_MASK	GENMASK(15, 0)
+#define VEND1_GLOBAL_MAILBOX_INTERFACE6_LSW_DATA(x)	FIELD_PREP(VEND1_GLOBAL_MAILBOX_INTERFACE6_LSW_DATA_MASK, (u16)(x))
+
+/* The following registers all have similar layouts; first the registers... */
+#define VEND1_GLOBAL_CFG_10M			0x0310
+#define VEND1_GLOBAL_CFG_100M			0x031b
+#define VEND1_GLOBAL_CFG_1G			0x031c
+#define VEND1_GLOBAL_CFG_2_5G			0x031d
+#define VEND1_GLOBAL_CFG_5G			0x031e
+#define VEND1_GLOBAL_CFG_10G			0x031f
+/* ...and now the fields */
+#define VEND1_GLOBAL_CFG_RATE_ADAPT		GENMASK(8, 7)
+#define VEND1_GLOBAL_CFG_RATE_ADAPT_NONE	0
+#define VEND1_GLOBAL_CFG_RATE_ADAPT_USX		1
+#define VEND1_GLOBAL_CFG_RATE_ADAPT_PAUSE	2
+
+/* Vendor specific 1, MDIO_MMD_VEND2 */
+#define VEND1_GLOBAL_CONTROL2			0xc001
+#define VEND1_GLOBAL_CONTROL2_UP_RUN_STALL_RST	BIT(15)
+#define VEND1_GLOBAL_CONTROL2_UP_RUN_STALL_OVD	BIT(6)
+#define VEND1_GLOBAL_CONTROL2_UP_RUN_STALL	BIT(0)
+
+#define VEND1_THERMAL_PROV_HIGH_TEMP_FAIL	0xc421
+#define VEND1_THERMAL_PROV_LOW_TEMP_FAIL	0xc422
+#define VEND1_THERMAL_PROV_HIGH_TEMP_WARN	0xc423
+#define VEND1_THERMAL_PROV_LOW_TEMP_WARN	0xc424
+#define VEND1_THERMAL_STAT1			0xc820
+#define VEND1_THERMAL_STAT2			0xc821
+#define VEND1_THERMAL_STAT2_VALID		BIT(0)
+#define VEND1_GENERAL_STAT1			0xc830
+#define VEND1_GENERAL_STAT1_HIGH_TEMP_FAIL	BIT(14)
+#define VEND1_GENERAL_STAT1_LOW_TEMP_FAIL	BIT(13)
+#define VEND1_GENERAL_STAT1_HIGH_TEMP_WARN	BIT(12)
+#define VEND1_GENERAL_STAT1_LOW_TEMP_WARN	BIT(11)
+
+#define VEND1_GLOBAL_GEN_STAT2			0xc831
+#define VEND1_GLOBAL_GEN_STAT2_OP_IN_PROG	BIT(15)
+
+#define VEND1_GLOBAL_RSVD_STAT1			0xc885
+#define VEND1_GLOBAL_RSVD_STAT1_FW_BUILD_ID	GENMASK(7, 4)
+#define VEND1_GLOBAL_RSVD_STAT1_PROV_ID		GENMASK(3, 0)
+
+#define VEND1_GLOBAL_RSVD_STAT9			0xc88d
+#define VEND1_GLOBAL_RSVD_STAT9_MODE		GENMASK(7, 0)
+#define VEND1_GLOBAL_RSVD_STAT9_1000BT2		0x23
+
+#define VEND1_GLOBAL_INT_STD_STATUS		0xfc00
+#define VEND1_GLOBAL_INT_VEND_STATUS		0xfc01
+
+#define VEND1_GLOBAL_INT_STD_MASK		0xff00
+#define VEND1_GLOBAL_INT_STD_MASK_PMA1		BIT(15)
+#define VEND1_GLOBAL_INT_STD_MASK_PMA2		BIT(14)
+#define VEND1_GLOBAL_INT_STD_MASK_PCS1		BIT(13)
+#define VEND1_GLOBAL_INT_STD_MASK_PCS2		BIT(12)
+#define VEND1_GLOBAL_INT_STD_MASK_PCS3		BIT(11)
+#define VEND1_GLOBAL_INT_STD_MASK_PHY_XS1	BIT(10)
+#define VEND1_GLOBAL_INT_STD_MASK_PHY_XS2	BIT(9)
+#define VEND1_GLOBAL_INT_STD_MASK_AN1		BIT(8)
+#define VEND1_GLOBAL_INT_STD_MASK_AN2		BIT(7)
+#define VEND1_GLOBAL_INT_STD_MASK_GBE		BIT(6)
+#define VEND1_GLOBAL_INT_STD_MASK_ALL		BIT(0)
+
+#define VEND1_GLOBAL_INT_VEND_MASK		0xff01
+#define VEND1_GLOBAL_INT_VEND_MASK_PMA		BIT(15)
+#define VEND1_GLOBAL_INT_VEND_MASK_PCS		BIT(14)
+#define VEND1_GLOBAL_INT_VEND_MASK_PHY_XS	BIT(13)
+#define VEND1_GLOBAL_INT_VEND_MASK_AN		BIT(12)
+#define VEND1_GLOBAL_INT_VEND_MASK_GBE		BIT(11)
+#define VEND1_GLOBAL_INT_VEND_MASK_GLOBAL1	BIT(2)
+#define VEND1_GLOBAL_INT_VEND_MASK_GLOBAL2	BIT(1)
+#define VEND1_GLOBAL_INT_VEND_MASK_GLOBAL3	BIT(0)
+
+#if IS_REACHABLE(CONFIG_HWMON)
+int aqr_hwmon_probe(struct phy_device *phydev);
+#else
+static inline int aqr_hwmon_probe(struct phy_device *phydev) { return 0; }
+#endif
+
+int aqr_firmware_load(struct phy_device *phydev);
diff --git a/drivers/net/phy/aquantia/aquantia_firmware.c b/drivers/net/phy/aquantia/aquantia_firmware.c
new file mode 100644
index 000000000000..c5f292b1c4c8
--- /dev/null
+++ b/drivers/net/phy/aquantia/aquantia_firmware.c
@@ -0,0 +1,370 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/bitfield.h>
+#include <linux/of.h>
+#include <linux/firmware.h>
+#include <linux/crc-ccitt.h>
+#include <linux/nvmem-consumer.h>
+
+#include <asm/unaligned.h>
+
+#include "aquantia.h"
+
+#define UP_RESET_SLEEP		100
+
+/* addresses of memory segments in the phy */
+#define DRAM_BASE_ADDR		0x3FFE0000
+#define IRAM_BASE_ADDR		0x40000000
+
+/* firmware image format constants */
+#define VERSION_STRING_SIZE		0x40
+#define VERSION_STRING_OFFSET		0x0200
+/* primary offset is written at an offset from the start of the fw blob */
+#define PRIMARY_OFFSET_OFFSET		0x8
+/* primary offset needs to be then added to a base offset */
+#define PRIMARY_OFFSET_SHIFT		12
+#define PRIMARY_OFFSET(x)		((x) << PRIMARY_OFFSET_SHIFT)
+#define HEADER_OFFSET			0x300
+
+struct aqr_fw_header {
+	u32 padding;
+	u8 iram_offset[3];
+	u8 iram_size[3];
+	u8 dram_offset[3];
+	u8 dram_size[3];
+} __packed;
+
+enum aqr_fw_src {
+	AQR_FW_SRC_NVMEM = 0,
+	AQR_FW_SRC_FS,
+};
+
+static const char * const aqr_fw_src_string[] = {
+	[AQR_FW_SRC_NVMEM] = "NVMEM",
+	[AQR_FW_SRC_FS] = "FS",
+};
+
+/* AQR firmware doesn't have fixed offsets for iram and dram section
+ * but instead provide an header with the offset to use on reading
+ * and parsing the firmware.
+ *
+ * AQR firmware can't be trusted and each offset is validated to be
+ * not negative and be in the size of the firmware itself.
+ */
+static bool aqr_fw_validate_get(size_t size, size_t offset, size_t get_size)
+{
+	return offset + get_size <= size;
+}
+
+static int aqr_fw_get_be16(const u8 *data, size_t offset, size_t size, u16 *value)
+{
+	if (!aqr_fw_validate_get(size, offset, sizeof(u16)))
+		return -EINVAL;
+
+	*value = get_unaligned_be16(data + offset);
+
+	return 0;
+}
+
+static int aqr_fw_get_le16(const u8 *data, size_t offset, size_t size, u16 *value)
+{
+	if (!aqr_fw_validate_get(size, offset, sizeof(u16)))
+		return -EINVAL;
+
+	*value = get_unaligned_le16(data + offset);
+
+	return 0;
+}
+
+static int aqr_fw_get_le24(const u8 *data, size_t offset, size_t size, u32 *value)
+{
+	if (!aqr_fw_validate_get(size, offset, sizeof(u8) * 3))
+		return -EINVAL;
+
+	*value = get_unaligned_le24(data + offset);
+
+	return 0;
+}
+
+/* load data into the phy's memory */
+static int aqr_fw_load_memory(struct phy_device *phydev, u32 addr,
+			      const u8 *data, size_t len)
+{
+	u16 crc = 0, up_crc;
+	size_t pos;
+
+	/* PHY expect addr in LE */
+	addr = (__force u32)cpu_to_le32(addr);
+
+	phy_write_mmd(phydev, MDIO_MMD_VEND1,
+		      VEND1_GLOBAL_MAILBOX_INTERFACE1,
+		      VEND1_GLOBAL_MAILBOX_INTERFACE1_CRC_RESET);
+	phy_write_mmd(phydev, MDIO_MMD_VEND1,
+		      VEND1_GLOBAL_MAILBOX_INTERFACE3,
+		      VEND1_GLOBAL_MAILBOX_INTERFACE3_MSW_ADDR(addr));
+	phy_write_mmd(phydev, MDIO_MMD_VEND1,
+		      VEND1_GLOBAL_MAILBOX_INTERFACE4,
+		      VEND1_GLOBAL_MAILBOX_INTERFACE4_LSW_ADDR(addr));
+
+	/* We assume and enforce the size to be word aligned.
+	 * If a firmware that is not word aligned is found, please report upstream.
+	 */
+	for (pos = 0; pos < len; pos += sizeof(u32)) {
+		u32 word;
+
+		/* FW data is always stored in little-endian */
+		word = get_unaligned((const u32 *)(data + pos));
+
+		phy_write_mmd(phydev, MDIO_MMD_VEND1, VEND1_GLOBAL_MAILBOX_INTERFACE5,
+			      VEND1_GLOBAL_MAILBOX_INTERFACE5_MSW_DATA(word));
+		phy_write_mmd(phydev, MDIO_MMD_VEND1, VEND1_GLOBAL_MAILBOX_INTERFACE6,
+			      VEND1_GLOBAL_MAILBOX_INTERFACE6_LSW_DATA(word));
+
+		phy_write_mmd(phydev, MDIO_MMD_VEND1, VEND1_GLOBAL_MAILBOX_INTERFACE1,
+			      VEND1_GLOBAL_MAILBOX_INTERFACE1_EXECUTE |
+			      VEND1_GLOBAL_MAILBOX_INTERFACE1_WRITE);
+
+		/* calculate CRC as we load data to the mailbox.
+		 * We convert word to big-endian as PHY is BE and mailbox will
+		 * return a BE CRC.
+		 */
+		word = (__force u32)cpu_to_be32(word);
+		crc = crc_ccitt_false(crc, (u8 *)&word, sizeof(word));
+	}
+
+	up_crc = phy_read_mmd(phydev, MDIO_MMD_VEND1, VEND1_GLOBAL_MAILBOX_INTERFACE2);
+	if (crc != up_crc) {
+		phydev_err(phydev, "CRC mismatch: calculated 0x%04x PHY 0x%04x\n",
+			   crc, up_crc);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int aqr_fw_boot(struct phy_device *phydev, const u8 *data, size_t size,
+		       enum aqr_fw_src fw_src)
+{
+	u16 calculated_crc, read_crc, read_primary_offset;
+	u32 iram_offset = 0, iram_size = 0;
+	u32 dram_offset = 0, dram_size = 0;
+	char version[VERSION_STRING_SIZE];
+	u32 primary_offset = 0;
+	int ret;
+
+	/* extract saved CRC at the end of the fw
+	 * CRC is saved in big-endian as PHY is BE
+	 */
+	ret = aqr_fw_get_be16(data, size - sizeof(u16), size, &read_crc);
+	if (ret) {
+		phydev_err(phydev, "bad firmware CRC in firmware\n");
+		return ret;
+	}
+	calculated_crc = crc_ccitt_false(0, data, size - sizeof(u16));
+	if (read_crc != calculated_crc) {
+		phydev_err(phydev, "bad firmware CRC: file 0x%04x calculated 0x%04x\n",
+			   read_crc, calculated_crc);
+		return -EINVAL;
+	}
+
+	/* Get the primary offset to extract DRAM and IRAM sections. */
+	ret = aqr_fw_get_le16(data, PRIMARY_OFFSET_OFFSET, size, &read_primary_offset);
+	if (ret) {
+		phydev_err(phydev, "bad primary offset in firmware\n");
+		return ret;
+	}
+	primary_offset = PRIMARY_OFFSET(read_primary_offset);
+
+	/* Find the DRAM and IRAM sections within the firmware file.
+	 * Make sure the fw_header is correctly in the firmware.
+	 */
+	if (!aqr_fw_validate_get(size, primary_offset + HEADER_OFFSET,
+				 sizeof(struct aqr_fw_header))) {
+		phydev_err(phydev, "bad fw_header in firmware\n");
+		return -EINVAL;
+	}
+
+	/* offset are in LE and values needs to be converted to cpu endian */
+	ret = aqr_fw_get_le24(data, primary_offset + HEADER_OFFSET +
+			      offsetof(struct aqr_fw_header, iram_offset),
+			      size, &iram_offset);
+	if (ret) {
+		phydev_err(phydev, "bad iram offset in firmware\n");
+		return ret;
+	}
+	ret = aqr_fw_get_le24(data, primary_offset + HEADER_OFFSET +
+			      offsetof(struct aqr_fw_header, iram_size),
+			      size, &iram_size);
+	if (ret) {
+		phydev_err(phydev, "invalid iram size in firmware\n");
+		return ret;
+	}
+	ret = aqr_fw_get_le24(data, primary_offset + HEADER_OFFSET +
+			      offsetof(struct aqr_fw_header, dram_offset),
+			      size, &dram_offset);
+	if (ret) {
+		phydev_err(phydev, "bad dram offset in firmware\n");
+		return ret;
+	}
+	ret = aqr_fw_get_le24(data, primary_offset + HEADER_OFFSET +
+			      offsetof(struct aqr_fw_header, dram_size),
+			      size, &dram_size);
+	if (ret) {
+		phydev_err(phydev, "invalid dram size in firmware\n");
+		return ret;
+	}
+
+	/* Increment the offset with the primary offset.
+	 * Validate iram/dram offset and size.
+	 */
+	iram_offset += primary_offset;
+	if (iram_size % sizeof(u32)) {
+		phydev_err(phydev, "iram size if not aligned to word size. Please report this upstream!\n");
+		return -EINVAL;
+	}
+	if (!aqr_fw_validate_get(size, iram_offset, iram_size)) {
+		phydev_err(phydev, "invalid iram offset for iram size\n");
+		return -EINVAL;
+	}
+
+	dram_offset += primary_offset;
+	if (dram_size % sizeof(u32)) {
+		phydev_err(phydev, "dram size if not aligned to word size. Please report this upstream!\n");
+		return -EINVAL;
+	}
+	if (!aqr_fw_validate_get(size, dram_offset, dram_size)) {
+		phydev_err(phydev, "invalid iram offset for iram size\n");
+		return -EINVAL;
+	}
+
+	phydev_dbg(phydev, "primary %d IRAM offset=%d size=%d DRAM offset=%d size=%d\n",
+		   primary_offset, iram_offset, iram_size, dram_offset, dram_size);
+
+	if (!aqr_fw_validate_get(size, dram_offset + VERSION_STRING_OFFSET,
+				 VERSION_STRING_SIZE)) {
+		phydev_err(phydev, "invalid version in firmware\n");
+		return -EINVAL;
+	}
+	strscpy(version, (char *)data + dram_offset + VERSION_STRING_OFFSET,
+		VERSION_STRING_SIZE);
+	if (version[0] == '\0') {
+		phydev_err(phydev, "invalid version in firmware\n");
+		return -EINVAL;
+	}
+	phydev_info(phydev, "loading firmware version '%s' from '%s'\n", version,
+		    aqr_fw_src_string[fw_src]);
+
+	/* stall the microcprocessor */
+	phy_write_mmd(phydev, MDIO_MMD_VEND1, VEND1_GLOBAL_CONTROL2,
+		      VEND1_GLOBAL_CONTROL2_UP_RUN_STALL | VEND1_GLOBAL_CONTROL2_UP_RUN_STALL_OVD);
+
+	phydev_dbg(phydev, "loading DRAM 0x%08x from offset=%d size=%d\n",
+		   DRAM_BASE_ADDR, dram_offset, dram_size);
+	ret = aqr_fw_load_memory(phydev, DRAM_BASE_ADDR, data + dram_offset,
+				 dram_size);
+	if (ret)
+		return ret;
+
+	phydev_dbg(phydev, "loading IRAM 0x%08x from offset=%d size=%d\n",
+		   IRAM_BASE_ADDR, iram_offset, iram_size);
+	ret = aqr_fw_load_memory(phydev, IRAM_BASE_ADDR, data + iram_offset,
+				 iram_size);
+	if (ret)
+		return ret;
+
+	/* make sure soft reset and low power mode are clear */
+	phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, VEND1_GLOBAL_SC,
+			   VEND1_GLOBAL_SC_SOFT_RESET | VEND1_GLOBAL_SC_LOW_POWER);
+
+	/* Release the microprocessor. UP_RESET must be held for 100 usec. */
+	phy_write_mmd(phydev, MDIO_MMD_VEND1, VEND1_GLOBAL_CONTROL2,
+		      VEND1_GLOBAL_CONTROL2_UP_RUN_STALL |
+		      VEND1_GLOBAL_CONTROL2_UP_RUN_STALL_OVD |
+		      VEND1_GLOBAL_CONTROL2_UP_RUN_STALL_RST);
+	usleep_range(UP_RESET_SLEEP, UP_RESET_SLEEP * 2);
+
+	phy_write_mmd(phydev, MDIO_MMD_VEND1, VEND1_GLOBAL_CONTROL2,
+		      VEND1_GLOBAL_CONTROL2_UP_RUN_STALL_OVD);
+
+	return 0;
+}
+
+static int aqr_firmware_load_nvmem(struct phy_device *phydev)
+{
+	struct nvmem_cell *cell;
+	size_t size;
+	u8 *buf;
+	int ret;
+
+	cell = nvmem_cell_get(&phydev->mdio.dev, "firmware");
+	if (IS_ERR(cell))
+		return PTR_ERR(cell);
+
+	buf = nvmem_cell_read(cell, &size);
+	if (IS_ERR(buf)) {
+		ret = PTR_ERR(buf);
+		goto exit;
+	}
+
+	ret = aqr_fw_boot(phydev, buf, size, AQR_FW_SRC_NVMEM);
+	if (ret)
+		phydev_err(phydev, "firmware loading failed: %d\n", ret);
+
+	kfree(buf);
+exit:
+	nvmem_cell_put(cell);
+
+	return ret;
+}
+
+static int aqr_firmware_load_fs(struct phy_device *phydev)
+{
+	struct device *dev = &phydev->mdio.dev;
+	const struct firmware *fw;
+	const char *fw_name;
+	int ret;
+
+	ret = of_property_read_string(dev->of_node, "firmware-name",
+				      &fw_name);
+	if (ret)
+		return ret;
+
+	ret = request_firmware(&fw, fw_name, dev);
+	if (ret) {
+		phydev_err(phydev, "failed to find FW file %s (%d)\n",
+			   fw_name, ret);
+		return ret;
+	}
+
+	ret = aqr_fw_boot(phydev, fw->data, fw->size, AQR_FW_SRC_FS);
+	if (ret)
+		phydev_err(phydev, "firmware loading failed: %d\n", ret);
+
+	release_firmware(fw);
+
+	return ret;
+}
+
+int aqr_firmware_load(struct phy_device *phydev)
+{
+	int ret;
+
+	/* Check if the firmware is not already loaded by pooling
+	 * the current version returned by the PHY. If 0 is returned,
+	 * no firmware is loaded.
+	 */
+	ret = phy_read_mmd(phydev, MDIO_MMD_VEND1, VEND1_GLOBAL_FW_ID);
+	if (ret > 0)
+		goto exit;
+
+	ret = aqr_firmware_load_nvmem(phydev);
+	if (!ret)
+		goto exit;
+
+	ret = aqr_firmware_load_fs(phydev);
+	if (ret)
+		return ret;
+
+exit:
+	return 0;
+}
diff --git a/drivers/net/phy/aquantia/aquantia_hwmon.c b/drivers/net/phy/aquantia/aquantia_hwmon.c
new file mode 100644
index 000000000000..7b3c49c3bf49
--- /dev/null
+++ b/drivers/net/phy/aquantia/aquantia_hwmon.c
@@ -0,0 +1,236 @@
+// SPDX-License-Identifier: GPL-2.0
+/* HWMON driver for Aquantia PHY
+ *
+ * Author: Nikita Yushchenko <nikita.yoush@cogentembedded.com>
+ * Author: Andrew Lunn <andrew@lunn.ch>
+ * Author: Heiner Kallweit <hkallweit1@gmail.com>
+ */
+
+#include <linux/phy.h>
+#include <linux/device.h>
+#include <linux/ctype.h>
+#include <linux/hwmon.h>
+
+#include "aquantia.h"
+
+#if IS_REACHABLE(CONFIG_HWMON)
+
+static umode_t aqr_hwmon_is_visible(const void *data,
+				    enum hwmon_sensor_types type,
+				    u32 attr, int channel)
+{
+	if (type != hwmon_temp)
+		return 0;
+
+	switch (attr) {
+	case hwmon_temp_input:
+	case hwmon_temp_min_alarm:
+	case hwmon_temp_max_alarm:
+	case hwmon_temp_lcrit_alarm:
+	case hwmon_temp_crit_alarm:
+		return 0444;
+	case hwmon_temp_min:
+	case hwmon_temp_max:
+	case hwmon_temp_lcrit:
+	case hwmon_temp_crit:
+		return 0644;
+	default:
+		return 0;
+	}
+}
+
+static int aqr_hwmon_get(struct phy_device *phydev, int reg, long *value)
+{
+	int temp = phy_read_mmd(phydev, MDIO_MMD_VEND1, reg);
+
+	if (temp < 0)
+		return temp;
+
+	/* 16 bit value is 2's complement with LSB = 1/256th degree Celsius */
+	*value = (s16)temp * 1000 / 256;
+
+	return 0;
+}
+
+static int aqr_hwmon_set(struct phy_device *phydev, int reg, long value)
+{
+	int temp;
+
+	if (value >= 128000 || value < -128000)
+		return -ERANGE;
+
+	temp = value * 256 / 1000;
+
+	/* temp is in s16 range and we're interested in lower 16 bits only */
+	return phy_write_mmd(phydev, MDIO_MMD_VEND1, reg, (u16)temp);
+}
+
+static int aqr_hwmon_test_bit(struct phy_device *phydev, int reg, int bit)
+{
+	int val = phy_read_mmd(phydev, MDIO_MMD_VEND1, reg);
+
+	if (val < 0)
+		return val;
+
+	return !!(val & bit);
+}
+
+static int aqr_hwmon_status1(struct phy_device *phydev, int bit, long *value)
+{
+	int val = aqr_hwmon_test_bit(phydev, VEND1_GENERAL_STAT1, bit);
+
+	if (val < 0)
+		return val;
+
+	*value = val;
+
+	return 0;
+}
+
+static int aqr_hwmon_read(struct device *dev, enum hwmon_sensor_types type,
+			  u32 attr, int channel, long *value)
+{
+	struct phy_device *phydev = dev_get_drvdata(dev);
+	int reg;
+
+	if (type != hwmon_temp)
+		return -EOPNOTSUPP;
+
+	switch (attr) {
+	case hwmon_temp_input:
+		reg = aqr_hwmon_test_bit(phydev, VEND1_THERMAL_STAT2,
+					 VEND1_THERMAL_STAT2_VALID);
+		if (reg < 0)
+			return reg;
+		if (!reg)
+			return -EBUSY;
+
+		return aqr_hwmon_get(phydev, VEND1_THERMAL_STAT1, value);
+
+	case hwmon_temp_lcrit:
+		return aqr_hwmon_get(phydev, VEND1_THERMAL_PROV_LOW_TEMP_FAIL,
+				     value);
+	case hwmon_temp_min:
+		return aqr_hwmon_get(phydev, VEND1_THERMAL_PROV_LOW_TEMP_WARN,
+				     value);
+	case hwmon_temp_max:
+		return aqr_hwmon_get(phydev, VEND1_THERMAL_PROV_HIGH_TEMP_WARN,
+				     value);
+	case hwmon_temp_crit:
+		return aqr_hwmon_get(phydev, VEND1_THERMAL_PROV_HIGH_TEMP_FAIL,
+				     value);
+	case hwmon_temp_lcrit_alarm:
+		return aqr_hwmon_status1(phydev,
+					 VEND1_GENERAL_STAT1_LOW_TEMP_FAIL,
+					 value);
+	case hwmon_temp_min_alarm:
+		return aqr_hwmon_status1(phydev,
+					 VEND1_GENERAL_STAT1_LOW_TEMP_WARN,
+					 value);
+	case hwmon_temp_max_alarm:
+		return aqr_hwmon_status1(phydev,
+					 VEND1_GENERAL_STAT1_HIGH_TEMP_WARN,
+					 value);
+	case hwmon_temp_crit_alarm:
+		return aqr_hwmon_status1(phydev,
+					 VEND1_GENERAL_STAT1_HIGH_TEMP_FAIL,
+					 value);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int aqr_hwmon_write(struct device *dev, enum hwmon_sensor_types type,
+			   u32 attr, int channel, long value)
+{
+	struct phy_device *phydev = dev_get_drvdata(dev);
+
+	if (type != hwmon_temp)
+		return -EOPNOTSUPP;
+
+	switch (attr) {
+	case hwmon_temp_lcrit:
+		return aqr_hwmon_set(phydev, VEND1_THERMAL_PROV_LOW_TEMP_FAIL,
+				     value);
+	case hwmon_temp_min:
+		return aqr_hwmon_set(phydev, VEND1_THERMAL_PROV_LOW_TEMP_WARN,
+				     value);
+	case hwmon_temp_max:
+		return aqr_hwmon_set(phydev, VEND1_THERMAL_PROV_HIGH_TEMP_WARN,
+				     value);
+	case hwmon_temp_crit:
+		return aqr_hwmon_set(phydev, VEND1_THERMAL_PROV_HIGH_TEMP_FAIL,
+				     value);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static const struct hwmon_ops aqr_hwmon_ops = {
+	.is_visible = aqr_hwmon_is_visible,
+	.read = aqr_hwmon_read,
+	.write = aqr_hwmon_write,
+};
+
+static u32 aqr_hwmon_chip_config[] = {
+	HWMON_C_REGISTER_TZ,
+	0,
+};
+
+static const struct hwmon_channel_info aqr_hwmon_chip = {
+	.type = hwmon_chip,
+	.config = aqr_hwmon_chip_config,
+};
+
+static u32 aqr_hwmon_temp_config[] = {
+	HWMON_T_INPUT |
+	HWMON_T_MAX | HWMON_T_MIN |
+	HWMON_T_MAX_ALARM | HWMON_T_MIN_ALARM |
+	HWMON_T_CRIT | HWMON_T_LCRIT |
+	HWMON_T_CRIT_ALARM | HWMON_T_LCRIT_ALARM,
+	0,
+};
+
+static const struct hwmon_channel_info aqr_hwmon_temp = {
+	.type = hwmon_temp,
+	.config = aqr_hwmon_temp_config,
+};
+
+static const struct hwmon_channel_info * const aqr_hwmon_info[] = {
+	&aqr_hwmon_chip,
+	&aqr_hwmon_temp,
+	NULL,
+};
+
+static const struct hwmon_chip_info aqr_hwmon_chip_info = {
+	.ops = &aqr_hwmon_ops,
+	.info = aqr_hwmon_info,
+};
+
+int aqr_hwmon_probe(struct phy_device *phydev)
+{
+	struct device *dev = &phydev->mdio.dev;
+	struct device *hwmon_dev;
+	char *hwmon_name;
+	int i, j;
+
+	hwmon_name = devm_kstrdup(dev, dev_name(dev), GFP_KERNEL);
+	if (!hwmon_name)
+		return -ENOMEM;
+
+	for (i = j = 0; hwmon_name[i]; i++) {
+		if (isalnum(hwmon_name[i])) {
+			if (i != j)
+				hwmon_name[j] = hwmon_name[i];
+			j++;
+		}
+	}
+	hwmon_name[j] = '\0';
+
+	hwmon_dev = devm_hwmon_device_register_with_info(dev, hwmon_name,
+					phydev, &aqr_hwmon_chip_info, NULL);
+
+	return PTR_ERR_OR_ZERO(hwmon_dev);
+}
+
+#endif
diff --git a/drivers/net/phy/aquantia/aquantia_main.c b/drivers/net/phy/aquantia/aquantia_main.c
new file mode 100644
index 000000000000..cc4a97741c4a
--- /dev/null
+++ b/drivers/net/phy/aquantia/aquantia_main.c
@@ -0,0 +1,833 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for Aquantia PHY
+ *
+ * Author: Shaohui Xie <Shaohui.Xie@freescale.com>
+ *
+ * Copyright 2015 Freescale Semiconductor, Inc.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/delay.h>
+#include <linux/bitfield.h>
+#include <linux/phy.h>
+
+#include "aquantia.h"
+
+#define PHY_ID_AQ1202	0x03a1b445
+#define PHY_ID_AQ2104	0x03a1b460
+#define PHY_ID_AQR105	0x03a1b4a2
+#define PHY_ID_AQR106	0x03a1b4d0
+#define PHY_ID_AQR107	0x03a1b4e0
+#define PHY_ID_AQCS109	0x03a1b5c2
+#define PHY_ID_AQR405	0x03a1b4b0
+#define PHY_ID_AQR112	0x03a1b662
+#define PHY_ID_AQR412	0x03a1b712
+#define PHY_ID_AQR113C	0x31c31c12
+
+#define MDIO_PHYXS_VEND_IF_STATUS		0xe812
+#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_MASK	GENMASK(7, 3)
+#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_KR	0
+#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_KX	1
+#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_XFI	2
+#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_USXGMII	3
+#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_XAUI	4
+#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_SGMII	6
+#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_RXAUI	7
+#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_OCSGMII	10
+
+#define MDIO_AN_VEND_PROV			0xc400
+#define MDIO_AN_VEND_PROV_1000BASET_FULL	BIT(15)
+#define MDIO_AN_VEND_PROV_1000BASET_HALF	BIT(14)
+#define MDIO_AN_VEND_PROV_5000BASET_FULL	BIT(11)
+#define MDIO_AN_VEND_PROV_2500BASET_FULL	BIT(10)
+#define MDIO_AN_VEND_PROV_DOWNSHIFT_EN		BIT(4)
+#define MDIO_AN_VEND_PROV_DOWNSHIFT_MASK	GENMASK(3, 0)
+#define MDIO_AN_VEND_PROV_DOWNSHIFT_DFLT	4
+
+#define MDIO_AN_TX_VEND_STATUS1			0xc800
+#define MDIO_AN_TX_VEND_STATUS1_RATE_MASK	GENMASK(3, 1)
+#define MDIO_AN_TX_VEND_STATUS1_10BASET		0
+#define MDIO_AN_TX_VEND_STATUS1_100BASETX	1
+#define MDIO_AN_TX_VEND_STATUS1_1000BASET	2
+#define MDIO_AN_TX_VEND_STATUS1_10GBASET	3
+#define MDIO_AN_TX_VEND_STATUS1_2500BASET	4
+#define MDIO_AN_TX_VEND_STATUS1_5000BASET	5
+#define MDIO_AN_TX_VEND_STATUS1_FULL_DUPLEX	BIT(0)
+
+#define MDIO_AN_TX_VEND_INT_STATUS1		0xcc00
+#define MDIO_AN_TX_VEND_INT_STATUS1_DOWNSHIFT	BIT(1)
+
+#define MDIO_AN_TX_VEND_INT_STATUS2		0xcc01
+#define MDIO_AN_TX_VEND_INT_STATUS2_MASK	BIT(0)
+
+#define MDIO_AN_TX_VEND_INT_MASK2		0xd401
+#define MDIO_AN_TX_VEND_INT_MASK2_LINK		BIT(0)
+
+#define MDIO_AN_RX_LP_STAT1			0xe820
+#define MDIO_AN_RX_LP_STAT1_1000BASET_FULL	BIT(15)
+#define MDIO_AN_RX_LP_STAT1_1000BASET_HALF	BIT(14)
+#define MDIO_AN_RX_LP_STAT1_SHORT_REACH		BIT(13)
+#define MDIO_AN_RX_LP_STAT1_AQRATE_DOWNSHIFT	BIT(12)
+#define MDIO_AN_RX_LP_STAT1_AQ_PHY		BIT(2)
+
+#define MDIO_AN_RX_LP_STAT4			0xe823
+#define MDIO_AN_RX_LP_STAT4_FW_MAJOR		GENMASK(15, 8)
+#define MDIO_AN_RX_LP_STAT4_FW_MINOR		GENMASK(7, 0)
+
+#define MDIO_AN_RX_VEND_STAT3			0xe832
+#define MDIO_AN_RX_VEND_STAT3_AFR		BIT(0)
+
+/* MDIO_MMD_C22EXT */
+#define MDIO_C22EXT_STAT_SGMII_RX_GOOD_FRAMES		0xd292
+#define MDIO_C22EXT_STAT_SGMII_RX_BAD_FRAMES		0xd294
+#define MDIO_C22EXT_STAT_SGMII_RX_FALSE_CARRIER		0xd297
+#define MDIO_C22EXT_STAT_SGMII_TX_GOOD_FRAMES		0xd313
+#define MDIO_C22EXT_STAT_SGMII_TX_BAD_FRAMES		0xd315
+#define MDIO_C22EXT_STAT_SGMII_TX_FALSE_CARRIER		0xd317
+#define MDIO_C22EXT_STAT_SGMII_TX_COLLISIONS		0xd318
+#define MDIO_C22EXT_STAT_SGMII_TX_LINE_COLLISIONS	0xd319
+#define MDIO_C22EXT_STAT_SGMII_TX_FRAME_ALIGN_ERR	0xd31a
+#define MDIO_C22EXT_STAT_SGMII_TX_RUNT_FRAMES		0xd31b
+
+/* Sleep and timeout for checking if the Processor-Intensive
+ * MDIO operation is finished
+ */
+#define AQR107_OP_IN_PROG_SLEEP		1000
+#define AQR107_OP_IN_PROG_TIMEOUT	100000
+
+struct aqr107_hw_stat {
+	const char *name;
+	int reg;
+	int size;
+};
+
+#define SGMII_STAT(n, r, s) { n, MDIO_C22EXT_STAT_SGMII_ ## r, s }
+static const struct aqr107_hw_stat aqr107_hw_stats[] = {
+	SGMII_STAT("sgmii_rx_good_frames",	    RX_GOOD_FRAMES,	26),
+	SGMII_STAT("sgmii_rx_bad_frames",	    RX_BAD_FRAMES,	26),
+	SGMII_STAT("sgmii_rx_false_carrier_events", RX_FALSE_CARRIER,	 8),
+	SGMII_STAT("sgmii_tx_good_frames",	    TX_GOOD_FRAMES,	26),
+	SGMII_STAT("sgmii_tx_bad_frames",	    TX_BAD_FRAMES,	26),
+	SGMII_STAT("sgmii_tx_false_carrier_events", TX_FALSE_CARRIER,	 8),
+	SGMII_STAT("sgmii_tx_collisions",	    TX_COLLISIONS,	 8),
+	SGMII_STAT("sgmii_tx_line_collisions",	    TX_LINE_COLLISIONS,	 8),
+	SGMII_STAT("sgmii_tx_frame_alignment_err",  TX_FRAME_ALIGN_ERR,	16),
+	SGMII_STAT("sgmii_tx_runt_frames",	    TX_RUNT_FRAMES,	22),
+};
+#define AQR107_SGMII_STAT_SZ ARRAY_SIZE(aqr107_hw_stats)
+
+struct aqr107_priv {
+	u64 sgmii_stats[AQR107_SGMII_STAT_SZ];
+};
+
+static int aqr107_get_sset_count(struct phy_device *phydev)
+{
+	return AQR107_SGMII_STAT_SZ;
+}
+
+static void aqr107_get_strings(struct phy_device *phydev, u8 *data)
+{
+	int i;
+
+	for (i = 0; i < AQR107_SGMII_STAT_SZ; i++)
+		strscpy(data + i * ETH_GSTRING_LEN, aqr107_hw_stats[i].name,
+			ETH_GSTRING_LEN);
+}
+
+static u64 aqr107_get_stat(struct phy_device *phydev, int index)
+{
+	const struct aqr107_hw_stat *stat = aqr107_hw_stats + index;
+	int len_l = min(stat->size, 16);
+	int len_h = stat->size - len_l;
+	u64 ret;
+	int val;
+
+	val = phy_read_mmd(phydev, MDIO_MMD_C22EXT, stat->reg);
+	if (val < 0)
+		return U64_MAX;
+
+	ret = val & GENMASK(len_l - 1, 0);
+	if (len_h) {
+		val = phy_read_mmd(phydev, MDIO_MMD_C22EXT, stat->reg + 1);
+		if (val < 0)
+			return U64_MAX;
+
+		ret += (val & GENMASK(len_h - 1, 0)) << 16;
+	}
+
+	return ret;
+}
+
+static void aqr107_get_stats(struct phy_device *phydev,
+			     struct ethtool_stats *stats, u64 *data)
+{
+	struct aqr107_priv *priv = phydev->priv;
+	u64 val;
+	int i;
+
+	for (i = 0; i < AQR107_SGMII_STAT_SZ; i++) {
+		val = aqr107_get_stat(phydev, i);
+		if (val == U64_MAX)
+			phydev_err(phydev, "Reading HW Statistics failed for %s\n",
+				   aqr107_hw_stats[i].name);
+		else
+			priv->sgmii_stats[i] += val;
+
+		data[i] = priv->sgmii_stats[i];
+	}
+}
+
+static int aqr_config_aneg(struct phy_device *phydev)
+{
+	bool changed = false;
+	u16 reg;
+	int ret;
+
+	if (phydev->autoneg == AUTONEG_DISABLE)
+		return genphy_c45_pma_setup_forced(phydev);
+
+	ret = genphy_c45_an_config_aneg(phydev);
+	if (ret < 0)
+		return ret;
+	if (ret > 0)
+		changed = true;
+
+	/* Clause 45 has no standardized support for 1000BaseT, therefore
+	 * use vendor registers for this mode.
+	 */
+	reg = 0;
+	if (linkmode_test_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT,
+			      phydev->advertising))
+		reg |= MDIO_AN_VEND_PROV_1000BASET_FULL;
+
+	if (linkmode_test_bit(ETHTOOL_LINK_MODE_1000baseT_Half_BIT,
+			      phydev->advertising))
+		reg |= MDIO_AN_VEND_PROV_1000BASET_HALF;
+
+	/* Handle the case when the 2.5G and 5G speeds are not advertised */
+	if (linkmode_test_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT,
+			      phydev->advertising))
+		reg |= MDIO_AN_VEND_PROV_2500BASET_FULL;
+
+	if (linkmode_test_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT,
+			      phydev->advertising))
+		reg |= MDIO_AN_VEND_PROV_5000BASET_FULL;
+
+	ret = phy_modify_mmd_changed(phydev, MDIO_MMD_AN, MDIO_AN_VEND_PROV,
+				     MDIO_AN_VEND_PROV_1000BASET_HALF |
+				     MDIO_AN_VEND_PROV_1000BASET_FULL |
+				     MDIO_AN_VEND_PROV_2500BASET_FULL |
+				     MDIO_AN_VEND_PROV_5000BASET_FULL, reg);
+	if (ret < 0)
+		return ret;
+	if (ret > 0)
+		changed = true;
+
+	return genphy_c45_check_and_restart_aneg(phydev, changed);
+}
+
+static int aqr_config_intr(struct phy_device *phydev)
+{
+	bool en = phydev->interrupts == PHY_INTERRUPT_ENABLED;
+	int err;
+
+	if (en) {
+		/* Clear any pending interrupts before enabling them */
+		err = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_AN_TX_VEND_INT_STATUS2);
+		if (err < 0)
+			return err;
+	}
+
+	err = phy_write_mmd(phydev, MDIO_MMD_AN, MDIO_AN_TX_VEND_INT_MASK2,
+			    en ? MDIO_AN_TX_VEND_INT_MASK2_LINK : 0);
+	if (err < 0)
+		return err;
+
+	err = phy_write_mmd(phydev, MDIO_MMD_VEND1, VEND1_GLOBAL_INT_STD_MASK,
+			    en ? VEND1_GLOBAL_INT_STD_MASK_ALL : 0);
+	if (err < 0)
+		return err;
+
+	err = phy_write_mmd(phydev, MDIO_MMD_VEND1, VEND1_GLOBAL_INT_VEND_MASK,
+			    en ? VEND1_GLOBAL_INT_VEND_MASK_GLOBAL3 |
+			    VEND1_GLOBAL_INT_VEND_MASK_AN : 0);
+	if (err < 0)
+		return err;
+
+	if (!en) {
+		/* Clear any pending interrupts after we have disabled them */
+		err = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_AN_TX_VEND_INT_STATUS2);
+		if (err < 0)
+			return err;
+	}
+
+	return 0;
+}
+
+static irqreturn_t aqr_handle_interrupt(struct phy_device *phydev)
+{
+	int irq_status;
+
+	irq_status = phy_read_mmd(phydev, MDIO_MMD_AN,
+				  MDIO_AN_TX_VEND_INT_STATUS2);
+	if (irq_status < 0) {
+		phy_error(phydev);
+		return IRQ_NONE;
+	}
+
+	if (!(irq_status & MDIO_AN_TX_VEND_INT_STATUS2_MASK))
+		return IRQ_NONE;
+
+	phy_trigger_machine(phydev);
+
+	return IRQ_HANDLED;
+}
+
+static int aqr_read_status(struct phy_device *phydev)
+{
+	int val;
+
+	if (phydev->autoneg == AUTONEG_ENABLE) {
+		val = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_AN_RX_LP_STAT1);
+		if (val < 0)
+			return val;
+
+		linkmode_mod_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT,
+				 phydev->lp_advertising,
+				 val & MDIO_AN_RX_LP_STAT1_1000BASET_FULL);
+		linkmode_mod_bit(ETHTOOL_LINK_MODE_1000baseT_Half_BIT,
+				 phydev->lp_advertising,
+				 val & MDIO_AN_RX_LP_STAT1_1000BASET_HALF);
+	}
+
+	return genphy_c45_read_status(phydev);
+}
+
+static int aqr107_read_rate(struct phy_device *phydev)
+{
+	u32 config_reg;
+	int val;
+
+	val = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_AN_TX_VEND_STATUS1);
+	if (val < 0)
+		return val;
+
+	if (val & MDIO_AN_TX_VEND_STATUS1_FULL_DUPLEX)
+		phydev->duplex = DUPLEX_FULL;
+	else
+		phydev->duplex = DUPLEX_HALF;
+
+	switch (FIELD_GET(MDIO_AN_TX_VEND_STATUS1_RATE_MASK, val)) {
+	case MDIO_AN_TX_VEND_STATUS1_10BASET:
+		phydev->speed = SPEED_10;
+		config_reg = VEND1_GLOBAL_CFG_10M;
+		break;
+	case MDIO_AN_TX_VEND_STATUS1_100BASETX:
+		phydev->speed = SPEED_100;
+		config_reg = VEND1_GLOBAL_CFG_100M;
+		break;
+	case MDIO_AN_TX_VEND_STATUS1_1000BASET:
+		phydev->speed = SPEED_1000;
+		config_reg = VEND1_GLOBAL_CFG_1G;
+		break;
+	case MDIO_AN_TX_VEND_STATUS1_2500BASET:
+		phydev->speed = SPEED_2500;
+		config_reg = VEND1_GLOBAL_CFG_2_5G;
+		break;
+	case MDIO_AN_TX_VEND_STATUS1_5000BASET:
+		phydev->speed = SPEED_5000;
+		config_reg = VEND1_GLOBAL_CFG_5G;
+		break;
+	case MDIO_AN_TX_VEND_STATUS1_10GBASET:
+		phydev->speed = SPEED_10000;
+		config_reg = VEND1_GLOBAL_CFG_10G;
+		break;
+	default:
+		phydev->speed = SPEED_UNKNOWN;
+		return 0;
+	}
+
+	val = phy_read_mmd(phydev, MDIO_MMD_VEND1, config_reg);
+	if (val < 0)
+		return val;
+
+	if (FIELD_GET(VEND1_GLOBAL_CFG_RATE_ADAPT, val) ==
+	    VEND1_GLOBAL_CFG_RATE_ADAPT_PAUSE)
+		phydev->rate_matching = RATE_MATCH_PAUSE;
+	else
+		phydev->rate_matching = RATE_MATCH_NONE;
+
+	return 0;
+}
+
+static int aqr107_read_status(struct phy_device *phydev)
+{
+	int val, ret;
+
+	ret = aqr_read_status(phydev);
+	if (ret)
+		return ret;
+
+	if (!phydev->link || phydev->autoneg == AUTONEG_DISABLE)
+		return 0;
+
+	val = phy_read_mmd(phydev, MDIO_MMD_PHYXS, MDIO_PHYXS_VEND_IF_STATUS);
+	if (val < 0)
+		return val;
+
+	switch (FIELD_GET(MDIO_PHYXS_VEND_IF_STATUS_TYPE_MASK, val)) {
+	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_KR:
+		phydev->interface = PHY_INTERFACE_MODE_10GKR;
+		break;
+	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_KX:
+		phydev->interface = PHY_INTERFACE_MODE_1000BASEKX;
+		break;
+	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_XFI:
+		phydev->interface = PHY_INTERFACE_MODE_10GBASER;
+		break;
+	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_USXGMII:
+		phydev->interface = PHY_INTERFACE_MODE_USXGMII;
+		break;
+	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_XAUI:
+		phydev->interface = PHY_INTERFACE_MODE_XAUI;
+		break;
+	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_SGMII:
+		phydev->interface = PHY_INTERFACE_MODE_SGMII;
+		break;
+	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_RXAUI:
+		phydev->interface = PHY_INTERFACE_MODE_RXAUI;
+		break;
+	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_OCSGMII:
+		phydev->interface = PHY_INTERFACE_MODE_2500BASEX;
+		break;
+	default:
+		phydev->interface = PHY_INTERFACE_MODE_NA;
+		break;
+	}
+
+	/* Read possibly downshifted rate from vendor register */
+	return aqr107_read_rate(phydev);
+}
+
+static int aqr107_get_downshift(struct phy_device *phydev, u8 *data)
+{
+	int val, cnt, enable;
+
+	val = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_AN_VEND_PROV);
+	if (val < 0)
+		return val;
+
+	enable = FIELD_GET(MDIO_AN_VEND_PROV_DOWNSHIFT_EN, val);
+	cnt = FIELD_GET(MDIO_AN_VEND_PROV_DOWNSHIFT_MASK, val);
+
+	*data = enable && cnt ? cnt : DOWNSHIFT_DEV_DISABLE;
+
+	return 0;
+}
+
+static int aqr107_set_downshift(struct phy_device *phydev, u8 cnt)
+{
+	int val = 0;
+
+	if (!FIELD_FIT(MDIO_AN_VEND_PROV_DOWNSHIFT_MASK, cnt))
+		return -E2BIG;
+
+	if (cnt != DOWNSHIFT_DEV_DISABLE) {
+		val = MDIO_AN_VEND_PROV_DOWNSHIFT_EN;
+		val |= FIELD_PREP(MDIO_AN_VEND_PROV_DOWNSHIFT_MASK, cnt);
+	}
+
+	return phy_modify_mmd(phydev, MDIO_MMD_AN, MDIO_AN_VEND_PROV,
+			      MDIO_AN_VEND_PROV_DOWNSHIFT_EN |
+			      MDIO_AN_VEND_PROV_DOWNSHIFT_MASK, val);
+}
+
+static int aqr107_get_tunable(struct phy_device *phydev,
+			      struct ethtool_tunable *tuna, void *data)
+{
+	switch (tuna->id) {
+	case ETHTOOL_PHY_DOWNSHIFT:
+		return aqr107_get_downshift(phydev, data);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int aqr107_set_tunable(struct phy_device *phydev,
+			      struct ethtool_tunable *tuna, const void *data)
+{
+	switch (tuna->id) {
+	case ETHTOOL_PHY_DOWNSHIFT:
+		return aqr107_set_downshift(phydev, *(const u8 *)data);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+/* If we configure settings whilst firmware is still initializing the chip,
+ * then these settings may be overwritten. Therefore make sure chip
+ * initialization has completed. Use presence of the firmware ID as
+ * indicator for initialization having completed.
+ * The chip also provides a "reset completed" bit, but it's cleared after
+ * read. Therefore function would time out if called again.
+ */
+static int aqr107_wait_reset_complete(struct phy_device *phydev)
+{
+	int val;
+
+	return phy_read_mmd_poll_timeout(phydev, MDIO_MMD_VEND1,
+					 VEND1_GLOBAL_FW_ID, val, val != 0,
+					 20000, 2000000, false);
+}
+
+static void aqr107_chip_info(struct phy_device *phydev)
+{
+	u8 fw_major, fw_minor, build_id, prov_id;
+	int val;
+
+	val = phy_read_mmd(phydev, MDIO_MMD_VEND1, VEND1_GLOBAL_FW_ID);
+	if (val < 0)
+		return;
+
+	fw_major = FIELD_GET(VEND1_GLOBAL_FW_ID_MAJOR, val);
+	fw_minor = FIELD_GET(VEND1_GLOBAL_FW_ID_MINOR, val);
+
+	val = phy_read_mmd(phydev, MDIO_MMD_VEND1, VEND1_GLOBAL_RSVD_STAT1);
+	if (val < 0)
+		return;
+
+	build_id = FIELD_GET(VEND1_GLOBAL_RSVD_STAT1_FW_BUILD_ID, val);
+	prov_id = FIELD_GET(VEND1_GLOBAL_RSVD_STAT1_PROV_ID, val);
+
+	phydev_dbg(phydev, "FW %u.%u, Build %u, Provisioning %u\n",
+		   fw_major, fw_minor, build_id, prov_id);
+}
+
+static int aqr107_config_init(struct phy_device *phydev)
+{
+	int ret;
+
+	/* Check that the PHY interface type is compatible */
+	if (phydev->interface != PHY_INTERFACE_MODE_SGMII &&
+	    phydev->interface != PHY_INTERFACE_MODE_1000BASEKX &&
+	    phydev->interface != PHY_INTERFACE_MODE_2500BASEX &&
+	    phydev->interface != PHY_INTERFACE_MODE_XGMII &&
+	    phydev->interface != PHY_INTERFACE_MODE_USXGMII &&
+	    phydev->interface != PHY_INTERFACE_MODE_10GKR &&
+	    phydev->interface != PHY_INTERFACE_MODE_10GBASER &&
+	    phydev->interface != PHY_INTERFACE_MODE_XAUI &&
+	    phydev->interface != PHY_INTERFACE_MODE_RXAUI)
+		return -ENODEV;
+
+	WARN(phydev->interface == PHY_INTERFACE_MODE_XGMII,
+	     "Your devicetree is out of date, please update it. The AQR107 family doesn't support XGMII, maybe you mean USXGMII.\n");
+
+	ret = aqr107_wait_reset_complete(phydev);
+	if (!ret)
+		aqr107_chip_info(phydev);
+
+	return aqr107_set_downshift(phydev, MDIO_AN_VEND_PROV_DOWNSHIFT_DFLT);
+}
+
+static int aqcs109_config_init(struct phy_device *phydev)
+{
+	int ret;
+
+	/* Check that the PHY interface type is compatible */
+	if (phydev->interface != PHY_INTERFACE_MODE_SGMII &&
+	    phydev->interface != PHY_INTERFACE_MODE_2500BASEX)
+		return -ENODEV;
+
+	ret = aqr107_wait_reset_complete(phydev);
+	if (!ret)
+		aqr107_chip_info(phydev);
+
+	/* AQCS109 belongs to a chip family partially supporting 10G and 5G.
+	 * PMA speed ability bits are the same for all members of the family,
+	 * AQCS109 however supports speeds up to 2.5G only.
+	 */
+	phy_set_max_speed(phydev, SPEED_2500);
+
+	return aqr107_set_downshift(phydev, MDIO_AN_VEND_PROV_DOWNSHIFT_DFLT);
+}
+
+static void aqr107_link_change_notify(struct phy_device *phydev)
+{
+	u8 fw_major, fw_minor;
+	bool downshift, short_reach, afr;
+	int mode, val;
+
+	if (phydev->state != PHY_RUNNING || phydev->autoneg == AUTONEG_DISABLE)
+		return;
+
+	val = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_AN_RX_LP_STAT1);
+	/* call failed or link partner is no Aquantia PHY */
+	if (val < 0 || !(val & MDIO_AN_RX_LP_STAT1_AQ_PHY))
+		return;
+
+	short_reach = val & MDIO_AN_RX_LP_STAT1_SHORT_REACH;
+	downshift = val & MDIO_AN_RX_LP_STAT1_AQRATE_DOWNSHIFT;
+
+	val = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_AN_RX_LP_STAT4);
+	if (val < 0)
+		return;
+
+	fw_major = FIELD_GET(MDIO_AN_RX_LP_STAT4_FW_MAJOR, val);
+	fw_minor = FIELD_GET(MDIO_AN_RX_LP_STAT4_FW_MINOR, val);
+
+	val = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_AN_RX_VEND_STAT3);
+	if (val < 0)
+		return;
+
+	afr = val & MDIO_AN_RX_VEND_STAT3_AFR;
+
+	phydev_dbg(phydev, "Link partner is Aquantia PHY, FW %u.%u%s%s%s\n",
+		   fw_major, fw_minor,
+		   short_reach ? ", short reach mode" : "",
+		   downshift ? ", fast-retrain downshift advertised" : "",
+		   afr ? ", fast reframe advertised" : "");
+
+	val = phy_read_mmd(phydev, MDIO_MMD_VEND1, VEND1_GLOBAL_RSVD_STAT9);
+	if (val < 0)
+		return;
+
+	mode = FIELD_GET(VEND1_GLOBAL_RSVD_STAT9_MODE, val);
+	if (mode == VEND1_GLOBAL_RSVD_STAT9_1000BT2)
+		phydev_info(phydev, "Aquantia 1000Base-T2 mode active\n");
+}
+
+static int aqr107_wait_processor_intensive_op(struct phy_device *phydev)
+{
+	int val, err;
+
+	/* The datasheet notes to wait at least 1ms after issuing a
+	 * processor intensive operation before checking.
+	 * We cannot use the 'sleep_before_read' parameter of read_poll_timeout
+	 * because that just determines the maximum time slept, not the minimum.
+	 */
+	usleep_range(1000, 5000);
+
+	err = phy_read_mmd_poll_timeout(phydev, MDIO_MMD_VEND1,
+					VEND1_GLOBAL_GEN_STAT2, val,
+					!(val & VEND1_GLOBAL_GEN_STAT2_OP_IN_PROG),
+					AQR107_OP_IN_PROG_SLEEP,
+					AQR107_OP_IN_PROG_TIMEOUT, false);
+	if (err) {
+		phydev_err(phydev, "timeout: processor-intensive MDIO operation\n");
+		return err;
+	}
+
+	return 0;
+}
+
+static int aqr107_get_rate_matching(struct phy_device *phydev,
+				    phy_interface_t iface)
+{
+	if (iface == PHY_INTERFACE_MODE_10GBASER ||
+	    iface == PHY_INTERFACE_MODE_2500BASEX ||
+	    iface == PHY_INTERFACE_MODE_NA)
+		return RATE_MATCH_PAUSE;
+	return RATE_MATCH_NONE;
+}
+
+static int aqr107_suspend(struct phy_device *phydev)
+{
+	int err;
+
+	err = phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, MDIO_CTRL1,
+			       MDIO_CTRL1_LPOWER);
+	if (err)
+		return err;
+
+	return aqr107_wait_processor_intensive_op(phydev);
+}
+
+static int aqr107_resume(struct phy_device *phydev)
+{
+	int err;
+
+	err = phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, MDIO_CTRL1,
+				 MDIO_CTRL1_LPOWER);
+	if (err)
+		return err;
+
+	return aqr107_wait_processor_intensive_op(phydev);
+}
+
+static int aqr107_probe(struct phy_device *phydev)
+{
+	int ret;
+
+	phydev->priv = devm_kzalloc(&phydev->mdio.dev,
+				    sizeof(struct aqr107_priv), GFP_KERNEL);
+	if (!phydev->priv)
+		return -ENOMEM;
+
+	ret = aqr_firmware_load(phydev);
+	if (ret)
+		return ret;
+
+	return aqr_hwmon_probe(phydev);
+}
+
+static struct phy_driver aqr_driver[] = {
+{
+	PHY_ID_MATCH_MODEL(PHY_ID_AQ1202),
+	.name		= "Aquantia AQ1202",
+	.config_aneg    = aqr_config_aneg,
+	.config_intr	= aqr_config_intr,
+	.handle_interrupt = aqr_handle_interrupt,
+	.read_status	= aqr_read_status,
+},
+{
+	PHY_ID_MATCH_MODEL(PHY_ID_AQ2104),
+	.name		= "Aquantia AQ2104",
+	.config_aneg    = aqr_config_aneg,
+	.config_intr	= aqr_config_intr,
+	.handle_interrupt = aqr_handle_interrupt,
+	.read_status	= aqr_read_status,
+},
+{
+	PHY_ID_MATCH_MODEL(PHY_ID_AQR105),
+	.name		= "Aquantia AQR105",
+	.config_aneg    = aqr_config_aneg,
+	.config_intr	= aqr_config_intr,
+	.handle_interrupt = aqr_handle_interrupt,
+	.read_status	= aqr_read_status,
+	.suspend	= aqr107_suspend,
+	.resume		= aqr107_resume,
+},
+{
+	PHY_ID_MATCH_MODEL(PHY_ID_AQR106),
+	.name		= "Aquantia AQR106",
+	.config_aneg    = aqr_config_aneg,
+	.config_intr	= aqr_config_intr,
+	.handle_interrupt = aqr_handle_interrupt,
+	.read_status	= aqr_read_status,
+},
+{
+	PHY_ID_MATCH_MODEL(PHY_ID_AQR107),
+	.name		= "Aquantia AQR107",
+	.probe		= aqr107_probe,
+	.get_rate_matching = aqr107_get_rate_matching,
+	.config_init	= aqr107_config_init,
+	.config_aneg    = aqr_config_aneg,
+	.config_intr	= aqr_config_intr,
+	.handle_interrupt = aqr_handle_interrupt,
+	.read_status	= aqr107_read_status,
+	.get_tunable    = aqr107_get_tunable,
+	.set_tunable    = aqr107_set_tunable,
+	.suspend	= aqr107_suspend,
+	.resume		= aqr107_resume,
+	.get_sset_count	= aqr107_get_sset_count,
+	.get_strings	= aqr107_get_strings,
+	.get_stats	= aqr107_get_stats,
+	.link_change_notify = aqr107_link_change_notify,
+},
+{
+	PHY_ID_MATCH_MODEL(PHY_ID_AQCS109),
+	.name		= "Aquantia AQCS109",
+	.probe		= aqr107_probe,
+	.get_rate_matching = aqr107_get_rate_matching,
+	.config_init	= aqcs109_config_init,
+	.config_aneg    = aqr_config_aneg,
+	.config_intr	= aqr_config_intr,
+	.handle_interrupt = aqr_handle_interrupt,
+	.read_status	= aqr107_read_status,
+	.get_tunable    = aqr107_get_tunable,
+	.set_tunable    = aqr107_set_tunable,
+	.suspend	= aqr107_suspend,
+	.resume		= aqr107_resume,
+	.get_sset_count	= aqr107_get_sset_count,
+	.get_strings	= aqr107_get_strings,
+	.get_stats	= aqr107_get_stats,
+	.link_change_notify = aqr107_link_change_notify,
+},
+{
+	PHY_ID_MATCH_MODEL(PHY_ID_AQR405),
+	.name		= "Aquantia AQR405",
+	.config_aneg    = aqr_config_aneg,
+	.config_intr	= aqr_config_intr,
+	.handle_interrupt = aqr_handle_interrupt,
+	.read_status	= aqr_read_status,
+},
+{
+	PHY_ID_MATCH_MODEL(PHY_ID_AQR112),
+	.name		= "Aquantia AQR112",
+	.probe		= aqr107_probe,
+	.config_aneg    = aqr_config_aneg,
+	.config_intr	= aqr_config_intr,
+	.handle_interrupt = aqr_handle_interrupt,
+	.get_tunable    = aqr107_get_tunable,
+	.set_tunable    = aqr107_set_tunable,
+	.suspend	= aqr107_suspend,
+	.resume		= aqr107_resume,
+	.read_status	= aqr107_read_status,
+	.get_rate_matching = aqr107_get_rate_matching,
+	.get_sset_count = aqr107_get_sset_count,
+	.get_strings	= aqr107_get_strings,
+	.get_stats	= aqr107_get_stats,
+	.link_change_notify = aqr107_link_change_notify,
+},
+{
+	PHY_ID_MATCH_MODEL(PHY_ID_AQR412),
+	.name		= "Aquantia AQR412",
+	.probe		= aqr107_probe,
+	.config_aneg    = aqr_config_aneg,
+	.config_intr	= aqr_config_intr,
+	.handle_interrupt = aqr_handle_interrupt,
+	.get_tunable    = aqr107_get_tunable,
+	.set_tunable    = aqr107_set_tunable,
+	.suspend	= aqr107_suspend,
+	.resume		= aqr107_resume,
+	.read_status	= aqr107_read_status,
+	.get_rate_matching = aqr107_get_rate_matching,
+	.get_sset_count = aqr107_get_sset_count,
+	.get_strings	= aqr107_get_strings,
+	.get_stats	= aqr107_get_stats,
+	.link_change_notify = aqr107_link_change_notify,
+},
+{
+	PHY_ID_MATCH_MODEL(PHY_ID_AQR113C),
+	.name           = "Aquantia AQR113C",
+	.probe          = aqr107_probe,
+	.get_rate_matching = aqr107_get_rate_matching,
+	.config_init    = aqr107_config_init,
+	.config_aneg    = aqr_config_aneg,
+	.config_intr    = aqr_config_intr,
+	.handle_interrupt       = aqr_handle_interrupt,
+	.read_status    = aqr107_read_status,
+	.get_tunable    = aqr107_get_tunable,
+	.set_tunable    = aqr107_set_tunable,
+	.suspend        = aqr107_suspend,
+	.resume         = aqr107_resume,
+	.get_sset_count = aqr107_get_sset_count,
+	.get_strings    = aqr107_get_strings,
+	.get_stats      = aqr107_get_stats,
+	.link_change_notify = aqr107_link_change_notify,
+},
+};
+
+module_phy_driver(aqr_driver);
+
+static struct mdio_device_id __maybe_unused aqr_tbl[] = {
+	{ PHY_ID_MATCH_MODEL(PHY_ID_AQ1202) },
+	{ PHY_ID_MATCH_MODEL(PHY_ID_AQ2104) },
+	{ PHY_ID_MATCH_MODEL(PHY_ID_AQR105) },
+	{ PHY_ID_MATCH_MODEL(PHY_ID_AQR106) },
+	{ PHY_ID_MATCH_MODEL(PHY_ID_AQR107) },
+	{ PHY_ID_MATCH_MODEL(PHY_ID_AQCS109) },
+	{ PHY_ID_MATCH_MODEL(PHY_ID_AQR405) },
+	{ PHY_ID_MATCH_MODEL(PHY_ID_AQR112) },
+	{ PHY_ID_MATCH_MODEL(PHY_ID_AQR412) },
+	{ PHY_ID_MATCH_MODEL(PHY_ID_AQR113C) },
+	{ }
+};
+
+MODULE_DEVICE_TABLE(mdio, aqr_tbl);
+
+MODULE_DESCRIPTION("Aquantia PHY driver");
+MODULE_AUTHOR("Shaohui Xie <Shaohui.Xie@freescale.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/phy/aquantia_hwmon.c b/drivers/net/phy/aquantia_hwmon.c
deleted file mode 100644
index 0da451e46f69..000000000000
--- a/drivers/net/phy/aquantia_hwmon.c
+++ /dev/null
@@ -1,250 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/* HWMON driver for Aquantia PHY
- *
- * Author: Nikita Yushchenko <nikita.yoush@cogentembedded.com>
- * Author: Andrew Lunn <andrew@lunn.ch>
- * Author: Heiner Kallweit <hkallweit1@gmail.com>
- */
-
-#include <linux/phy.h>
-#include <linux/device.h>
-#include <linux/ctype.h>
-#include <linux/hwmon.h>
-
-#include "aquantia.h"
-
-/* Vendor specific 1, MDIO_MMD_VEND2 */
-#define VEND1_THERMAL_PROV_HIGH_TEMP_FAIL	0xc421
-#define VEND1_THERMAL_PROV_LOW_TEMP_FAIL	0xc422
-#define VEND1_THERMAL_PROV_HIGH_TEMP_WARN	0xc423
-#define VEND1_THERMAL_PROV_LOW_TEMP_WARN	0xc424
-#define VEND1_THERMAL_STAT1			0xc820
-#define VEND1_THERMAL_STAT2			0xc821
-#define VEND1_THERMAL_STAT2_VALID		BIT(0)
-#define VEND1_GENERAL_STAT1			0xc830
-#define VEND1_GENERAL_STAT1_HIGH_TEMP_FAIL	BIT(14)
-#define VEND1_GENERAL_STAT1_LOW_TEMP_FAIL	BIT(13)
-#define VEND1_GENERAL_STAT1_HIGH_TEMP_WARN	BIT(12)
-#define VEND1_GENERAL_STAT1_LOW_TEMP_WARN	BIT(11)
-
-#if IS_REACHABLE(CONFIG_HWMON)
-
-static umode_t aqr_hwmon_is_visible(const void *data,
-				    enum hwmon_sensor_types type,
-				    u32 attr, int channel)
-{
-	if (type != hwmon_temp)
-		return 0;
-
-	switch (attr) {
-	case hwmon_temp_input:
-	case hwmon_temp_min_alarm:
-	case hwmon_temp_max_alarm:
-	case hwmon_temp_lcrit_alarm:
-	case hwmon_temp_crit_alarm:
-		return 0444;
-	case hwmon_temp_min:
-	case hwmon_temp_max:
-	case hwmon_temp_lcrit:
-	case hwmon_temp_crit:
-		return 0644;
-	default:
-		return 0;
-	}
-}
-
-static int aqr_hwmon_get(struct phy_device *phydev, int reg, long *value)
-{
-	int temp = phy_read_mmd(phydev, MDIO_MMD_VEND1, reg);
-
-	if (temp < 0)
-		return temp;
-
-	/* 16 bit value is 2's complement with LSB = 1/256th degree Celsius */
-	*value = (s16)temp * 1000 / 256;
-
-	return 0;
-}
-
-static int aqr_hwmon_set(struct phy_device *phydev, int reg, long value)
-{
-	int temp;
-
-	if (value >= 128000 || value < -128000)
-		return -ERANGE;
-
-	temp = value * 256 / 1000;
-
-	/* temp is in s16 range and we're interested in lower 16 bits only */
-	return phy_write_mmd(phydev, MDIO_MMD_VEND1, reg, (u16)temp);
-}
-
-static int aqr_hwmon_test_bit(struct phy_device *phydev, int reg, int bit)
-{
-	int val = phy_read_mmd(phydev, MDIO_MMD_VEND1, reg);
-
-	if (val < 0)
-		return val;
-
-	return !!(val & bit);
-}
-
-static int aqr_hwmon_status1(struct phy_device *phydev, int bit, long *value)
-{
-	int val = aqr_hwmon_test_bit(phydev, VEND1_GENERAL_STAT1, bit);
-
-	if (val < 0)
-		return val;
-
-	*value = val;
-
-	return 0;
-}
-
-static int aqr_hwmon_read(struct device *dev, enum hwmon_sensor_types type,
-			  u32 attr, int channel, long *value)
-{
-	struct phy_device *phydev = dev_get_drvdata(dev);
-	int reg;
-
-	if (type != hwmon_temp)
-		return -EOPNOTSUPP;
-
-	switch (attr) {
-	case hwmon_temp_input:
-		reg = aqr_hwmon_test_bit(phydev, VEND1_THERMAL_STAT2,
-					 VEND1_THERMAL_STAT2_VALID);
-		if (reg < 0)
-			return reg;
-		if (!reg)
-			return -EBUSY;
-
-		return aqr_hwmon_get(phydev, VEND1_THERMAL_STAT1, value);
-
-	case hwmon_temp_lcrit:
-		return aqr_hwmon_get(phydev, VEND1_THERMAL_PROV_LOW_TEMP_FAIL,
-				     value);
-	case hwmon_temp_min:
-		return aqr_hwmon_get(phydev, VEND1_THERMAL_PROV_LOW_TEMP_WARN,
-				     value);
-	case hwmon_temp_max:
-		return aqr_hwmon_get(phydev, VEND1_THERMAL_PROV_HIGH_TEMP_WARN,
-				     value);
-	case hwmon_temp_crit:
-		return aqr_hwmon_get(phydev, VEND1_THERMAL_PROV_HIGH_TEMP_FAIL,
-				     value);
-	case hwmon_temp_lcrit_alarm:
-		return aqr_hwmon_status1(phydev,
-					 VEND1_GENERAL_STAT1_LOW_TEMP_FAIL,
-					 value);
-	case hwmon_temp_min_alarm:
-		return aqr_hwmon_status1(phydev,
-					 VEND1_GENERAL_STAT1_LOW_TEMP_WARN,
-					 value);
-	case hwmon_temp_max_alarm:
-		return aqr_hwmon_status1(phydev,
-					 VEND1_GENERAL_STAT1_HIGH_TEMP_WARN,
-					 value);
-	case hwmon_temp_crit_alarm:
-		return aqr_hwmon_status1(phydev,
-					 VEND1_GENERAL_STAT1_HIGH_TEMP_FAIL,
-					 value);
-	default:
-		return -EOPNOTSUPP;
-	}
-}
-
-static int aqr_hwmon_write(struct device *dev, enum hwmon_sensor_types type,
-			   u32 attr, int channel, long value)
-{
-	struct phy_device *phydev = dev_get_drvdata(dev);
-
-	if (type != hwmon_temp)
-		return -EOPNOTSUPP;
-
-	switch (attr) {
-	case hwmon_temp_lcrit:
-		return aqr_hwmon_set(phydev, VEND1_THERMAL_PROV_LOW_TEMP_FAIL,
-				     value);
-	case hwmon_temp_min:
-		return aqr_hwmon_set(phydev, VEND1_THERMAL_PROV_LOW_TEMP_WARN,
-				     value);
-	case hwmon_temp_max:
-		return aqr_hwmon_set(phydev, VEND1_THERMAL_PROV_HIGH_TEMP_WARN,
-				     value);
-	case hwmon_temp_crit:
-		return aqr_hwmon_set(phydev, VEND1_THERMAL_PROV_HIGH_TEMP_FAIL,
-				     value);
-	default:
-		return -EOPNOTSUPP;
-	}
-}
-
-static const struct hwmon_ops aqr_hwmon_ops = {
-	.is_visible = aqr_hwmon_is_visible,
-	.read = aqr_hwmon_read,
-	.write = aqr_hwmon_write,
-};
-
-static u32 aqr_hwmon_chip_config[] = {
-	HWMON_C_REGISTER_TZ,
-	0,
-};
-
-static const struct hwmon_channel_info aqr_hwmon_chip = {
-	.type = hwmon_chip,
-	.config = aqr_hwmon_chip_config,
-};
-
-static u32 aqr_hwmon_temp_config[] = {
-	HWMON_T_INPUT |
-	HWMON_T_MAX | HWMON_T_MIN |
-	HWMON_T_MAX_ALARM | HWMON_T_MIN_ALARM |
-	HWMON_T_CRIT | HWMON_T_LCRIT |
-	HWMON_T_CRIT_ALARM | HWMON_T_LCRIT_ALARM,
-	0,
-};
-
-static const struct hwmon_channel_info aqr_hwmon_temp = {
-	.type = hwmon_temp,
-	.config = aqr_hwmon_temp_config,
-};
-
-static const struct hwmon_channel_info * const aqr_hwmon_info[] = {
-	&aqr_hwmon_chip,
-	&aqr_hwmon_temp,
-	NULL,
-};
-
-static const struct hwmon_chip_info aqr_hwmon_chip_info = {
-	.ops = &aqr_hwmon_ops,
-	.info = aqr_hwmon_info,
-};
-
-int aqr_hwmon_probe(struct phy_device *phydev)
-{
-	struct device *dev = &phydev->mdio.dev;
-	struct device *hwmon_dev;
-	char *hwmon_name;
-	int i, j;
-
-	hwmon_name = devm_kstrdup(dev, dev_name(dev), GFP_KERNEL);
-	if (!hwmon_name)
-		return -ENOMEM;
-
-	for (i = j = 0; hwmon_name[i]; i++) {
-		if (isalnum(hwmon_name[i])) {
-			if (i != j)
-				hwmon_name[j] = hwmon_name[i];
-			j++;
-		}
-	}
-	hwmon_name[j] = '\0';
-
-	hwmon_dev = devm_hwmon_device_register_with_info(dev, hwmon_name,
-					phydev, &aqr_hwmon_chip_info, NULL);
-
-	return PTR_ERR_OR_ZERO(hwmon_dev);
-}
-
-#endif
diff --git a/drivers/net/phy/aquantia_main.c b/drivers/net/phy/aquantia_main.c
deleted file mode 100644
index 334a6904ca5a..000000000000
--- a/drivers/net/phy/aquantia_main.c
+++ /dev/null
@@ -1,882 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Driver for Aquantia PHY
- *
- * Author: Shaohui Xie <Shaohui.Xie@freescale.com>
- *
- * Copyright 2015 Freescale Semiconductor, Inc.
- */
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/delay.h>
-#include <linux/bitfield.h>
-#include <linux/phy.h>
-
-#include "aquantia.h"
-
-#define PHY_ID_AQ1202	0x03a1b445
-#define PHY_ID_AQ2104	0x03a1b460
-#define PHY_ID_AQR105	0x03a1b4a2
-#define PHY_ID_AQR106	0x03a1b4d0
-#define PHY_ID_AQR107	0x03a1b4e0
-#define PHY_ID_AQCS109	0x03a1b5c2
-#define PHY_ID_AQR405	0x03a1b4b0
-#define PHY_ID_AQR112	0x03a1b662
-#define PHY_ID_AQR412	0x03a1b712
-#define PHY_ID_AQR113C	0x31c31c12
-
-#define MDIO_PHYXS_VEND_IF_STATUS		0xe812
-#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_MASK	GENMASK(7, 3)
-#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_KR	0
-#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_KX	1
-#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_XFI	2
-#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_USXGMII	3
-#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_XAUI	4
-#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_SGMII	6
-#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_RXAUI	7
-#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_OCSGMII	10
-
-#define MDIO_AN_VEND_PROV			0xc400
-#define MDIO_AN_VEND_PROV_1000BASET_FULL	BIT(15)
-#define MDIO_AN_VEND_PROV_1000BASET_HALF	BIT(14)
-#define MDIO_AN_VEND_PROV_5000BASET_FULL	BIT(11)
-#define MDIO_AN_VEND_PROV_2500BASET_FULL	BIT(10)
-#define MDIO_AN_VEND_PROV_DOWNSHIFT_EN		BIT(4)
-#define MDIO_AN_VEND_PROV_DOWNSHIFT_MASK	GENMASK(3, 0)
-#define MDIO_AN_VEND_PROV_DOWNSHIFT_DFLT	4
-
-#define MDIO_AN_TX_VEND_STATUS1			0xc800
-#define MDIO_AN_TX_VEND_STATUS1_RATE_MASK	GENMASK(3, 1)
-#define MDIO_AN_TX_VEND_STATUS1_10BASET		0
-#define MDIO_AN_TX_VEND_STATUS1_100BASETX	1
-#define MDIO_AN_TX_VEND_STATUS1_1000BASET	2
-#define MDIO_AN_TX_VEND_STATUS1_10GBASET	3
-#define MDIO_AN_TX_VEND_STATUS1_2500BASET	4
-#define MDIO_AN_TX_VEND_STATUS1_5000BASET	5
-#define MDIO_AN_TX_VEND_STATUS1_FULL_DUPLEX	BIT(0)
-
-#define MDIO_AN_TX_VEND_INT_STATUS1		0xcc00
-#define MDIO_AN_TX_VEND_INT_STATUS1_DOWNSHIFT	BIT(1)
-
-#define MDIO_AN_TX_VEND_INT_STATUS2		0xcc01
-#define MDIO_AN_TX_VEND_INT_STATUS2_MASK	BIT(0)
-
-#define MDIO_AN_TX_VEND_INT_MASK2		0xd401
-#define MDIO_AN_TX_VEND_INT_MASK2_LINK		BIT(0)
-
-#define MDIO_AN_RX_LP_STAT1			0xe820
-#define MDIO_AN_RX_LP_STAT1_1000BASET_FULL	BIT(15)
-#define MDIO_AN_RX_LP_STAT1_1000BASET_HALF	BIT(14)
-#define MDIO_AN_RX_LP_STAT1_SHORT_REACH		BIT(13)
-#define MDIO_AN_RX_LP_STAT1_AQRATE_DOWNSHIFT	BIT(12)
-#define MDIO_AN_RX_LP_STAT1_AQ_PHY		BIT(2)
-
-#define MDIO_AN_RX_LP_STAT4			0xe823
-#define MDIO_AN_RX_LP_STAT4_FW_MAJOR		GENMASK(15, 8)
-#define MDIO_AN_RX_LP_STAT4_FW_MINOR		GENMASK(7, 0)
-
-#define MDIO_AN_RX_VEND_STAT3			0xe832
-#define MDIO_AN_RX_VEND_STAT3_AFR		BIT(0)
-
-/* MDIO_MMD_C22EXT */
-#define MDIO_C22EXT_STAT_SGMII_RX_GOOD_FRAMES		0xd292
-#define MDIO_C22EXT_STAT_SGMII_RX_BAD_FRAMES		0xd294
-#define MDIO_C22EXT_STAT_SGMII_RX_FALSE_CARRIER		0xd297
-#define MDIO_C22EXT_STAT_SGMII_TX_GOOD_FRAMES		0xd313
-#define MDIO_C22EXT_STAT_SGMII_TX_BAD_FRAMES		0xd315
-#define MDIO_C22EXT_STAT_SGMII_TX_FALSE_CARRIER		0xd317
-#define MDIO_C22EXT_STAT_SGMII_TX_COLLISIONS		0xd318
-#define MDIO_C22EXT_STAT_SGMII_TX_LINE_COLLISIONS	0xd319
-#define MDIO_C22EXT_STAT_SGMII_TX_FRAME_ALIGN_ERR	0xd31a
-#define MDIO_C22EXT_STAT_SGMII_TX_RUNT_FRAMES		0xd31b
-
-/* Vendor specific 1, MDIO_MMD_VEND1 */
-#define VEND1_GLOBAL_FW_ID			0x0020
-#define VEND1_GLOBAL_FW_ID_MAJOR		GENMASK(15, 8)
-#define VEND1_GLOBAL_FW_ID_MINOR		GENMASK(7, 0)
-
-#define VEND1_GLOBAL_GEN_STAT2			0xc831
-#define VEND1_GLOBAL_GEN_STAT2_OP_IN_PROG	BIT(15)
-
-/* The following registers all have similar layouts; first the registers... */
-#define VEND1_GLOBAL_CFG_10M			0x0310
-#define VEND1_GLOBAL_CFG_100M			0x031b
-#define VEND1_GLOBAL_CFG_1G			0x031c
-#define VEND1_GLOBAL_CFG_2_5G			0x031d
-#define VEND1_GLOBAL_CFG_5G			0x031e
-#define VEND1_GLOBAL_CFG_10G			0x031f
-/* ...and now the fields */
-#define VEND1_GLOBAL_CFG_RATE_ADAPT		GENMASK(8, 7)
-#define VEND1_GLOBAL_CFG_RATE_ADAPT_NONE	0
-#define VEND1_GLOBAL_CFG_RATE_ADAPT_USX		1
-#define VEND1_GLOBAL_CFG_RATE_ADAPT_PAUSE	2
-
-#define VEND1_GLOBAL_RSVD_STAT1			0xc885
-#define VEND1_GLOBAL_RSVD_STAT1_FW_BUILD_ID	GENMASK(7, 4)
-#define VEND1_GLOBAL_RSVD_STAT1_PROV_ID		GENMASK(3, 0)
-
-#define VEND1_GLOBAL_RSVD_STAT9			0xc88d
-#define VEND1_GLOBAL_RSVD_STAT9_MODE		GENMASK(7, 0)
-#define VEND1_GLOBAL_RSVD_STAT9_1000BT2		0x23
-
-#define VEND1_GLOBAL_INT_STD_STATUS		0xfc00
-#define VEND1_GLOBAL_INT_VEND_STATUS		0xfc01
-
-#define VEND1_GLOBAL_INT_STD_MASK		0xff00
-#define VEND1_GLOBAL_INT_STD_MASK_PMA1		BIT(15)
-#define VEND1_GLOBAL_INT_STD_MASK_PMA2		BIT(14)
-#define VEND1_GLOBAL_INT_STD_MASK_PCS1		BIT(13)
-#define VEND1_GLOBAL_INT_STD_MASK_PCS2		BIT(12)
-#define VEND1_GLOBAL_INT_STD_MASK_PCS3		BIT(11)
-#define VEND1_GLOBAL_INT_STD_MASK_PHY_XS1	BIT(10)
-#define VEND1_GLOBAL_INT_STD_MASK_PHY_XS2	BIT(9)
-#define VEND1_GLOBAL_INT_STD_MASK_AN1		BIT(8)
-#define VEND1_GLOBAL_INT_STD_MASK_AN2		BIT(7)
-#define VEND1_GLOBAL_INT_STD_MASK_GBE		BIT(6)
-#define VEND1_GLOBAL_INT_STD_MASK_ALL		BIT(0)
-
-#define VEND1_GLOBAL_INT_VEND_MASK		0xff01
-#define VEND1_GLOBAL_INT_VEND_MASK_PMA		BIT(15)
-#define VEND1_GLOBAL_INT_VEND_MASK_PCS		BIT(14)
-#define VEND1_GLOBAL_INT_VEND_MASK_PHY_XS	BIT(13)
-#define VEND1_GLOBAL_INT_VEND_MASK_AN		BIT(12)
-#define VEND1_GLOBAL_INT_VEND_MASK_GBE		BIT(11)
-#define VEND1_GLOBAL_INT_VEND_MASK_GLOBAL1	BIT(2)
-#define VEND1_GLOBAL_INT_VEND_MASK_GLOBAL2	BIT(1)
-#define VEND1_GLOBAL_INT_VEND_MASK_GLOBAL3	BIT(0)
-
-/* Sleep and timeout for checking if the Processor-Intensive
- * MDIO operation is finished
- */
-#define AQR107_OP_IN_PROG_SLEEP		1000
-#define AQR107_OP_IN_PROG_TIMEOUT	100000
-
-struct aqr107_hw_stat {
-	const char *name;
-	int reg;
-	int size;
-};
-
-#define SGMII_STAT(n, r, s) { n, MDIO_C22EXT_STAT_SGMII_ ## r, s }
-static const struct aqr107_hw_stat aqr107_hw_stats[] = {
-	SGMII_STAT("sgmii_rx_good_frames",	    RX_GOOD_FRAMES,	26),
-	SGMII_STAT("sgmii_rx_bad_frames",	    RX_BAD_FRAMES,	26),
-	SGMII_STAT("sgmii_rx_false_carrier_events", RX_FALSE_CARRIER,	 8),
-	SGMII_STAT("sgmii_tx_good_frames",	    TX_GOOD_FRAMES,	26),
-	SGMII_STAT("sgmii_tx_bad_frames",	    TX_BAD_FRAMES,	26),
-	SGMII_STAT("sgmii_tx_false_carrier_events", TX_FALSE_CARRIER,	 8),
-	SGMII_STAT("sgmii_tx_collisions",	    TX_COLLISIONS,	 8),
-	SGMII_STAT("sgmii_tx_line_collisions",	    TX_LINE_COLLISIONS,	 8),
-	SGMII_STAT("sgmii_tx_frame_alignment_err",  TX_FRAME_ALIGN_ERR,	16),
-	SGMII_STAT("sgmii_tx_runt_frames",	    TX_RUNT_FRAMES,	22),
-};
-#define AQR107_SGMII_STAT_SZ ARRAY_SIZE(aqr107_hw_stats)
-
-struct aqr107_priv {
-	u64 sgmii_stats[AQR107_SGMII_STAT_SZ];
-};
-
-static int aqr107_get_sset_count(struct phy_device *phydev)
-{
-	return AQR107_SGMII_STAT_SZ;
-}
-
-static void aqr107_get_strings(struct phy_device *phydev, u8 *data)
-{
-	int i;
-
-	for (i = 0; i < AQR107_SGMII_STAT_SZ; i++)
-		strscpy(data + i * ETH_GSTRING_LEN, aqr107_hw_stats[i].name,
-			ETH_GSTRING_LEN);
-}
-
-static u64 aqr107_get_stat(struct phy_device *phydev, int index)
-{
-	const struct aqr107_hw_stat *stat = aqr107_hw_stats + index;
-	int len_l = min(stat->size, 16);
-	int len_h = stat->size - len_l;
-	u64 ret;
-	int val;
-
-	val = phy_read_mmd(phydev, MDIO_MMD_C22EXT, stat->reg);
-	if (val < 0)
-		return U64_MAX;
-
-	ret = val & GENMASK(len_l - 1, 0);
-	if (len_h) {
-		val = phy_read_mmd(phydev, MDIO_MMD_C22EXT, stat->reg + 1);
-		if (val < 0)
-			return U64_MAX;
-
-		ret += (val & GENMASK(len_h - 1, 0)) << 16;
-	}
-
-	return ret;
-}
-
-static void aqr107_get_stats(struct phy_device *phydev,
-			     struct ethtool_stats *stats, u64 *data)
-{
-	struct aqr107_priv *priv = phydev->priv;
-	u64 val;
-	int i;
-
-	for (i = 0; i < AQR107_SGMII_STAT_SZ; i++) {
-		val = aqr107_get_stat(phydev, i);
-		if (val == U64_MAX)
-			phydev_err(phydev, "Reading HW Statistics failed for %s\n",
-				   aqr107_hw_stats[i].name);
-		else
-			priv->sgmii_stats[i] += val;
-
-		data[i] = priv->sgmii_stats[i];
-	}
-}
-
-static int aqr_config_aneg(struct phy_device *phydev)
-{
-	bool changed = false;
-	u16 reg;
-	int ret;
-
-	if (phydev->autoneg == AUTONEG_DISABLE)
-		return genphy_c45_pma_setup_forced(phydev);
-
-	ret = genphy_c45_an_config_aneg(phydev);
-	if (ret < 0)
-		return ret;
-	if (ret > 0)
-		changed = true;
-
-	/* Clause 45 has no standardized support for 1000BaseT, therefore
-	 * use vendor registers for this mode.
-	 */
-	reg = 0;
-	if (linkmode_test_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT,
-			      phydev->advertising))
-		reg |= MDIO_AN_VEND_PROV_1000BASET_FULL;
-
-	if (linkmode_test_bit(ETHTOOL_LINK_MODE_1000baseT_Half_BIT,
-			      phydev->advertising))
-		reg |= MDIO_AN_VEND_PROV_1000BASET_HALF;
-
-	/* Handle the case when the 2.5G and 5G speeds are not advertised */
-	if (linkmode_test_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT,
-			      phydev->advertising))
-		reg |= MDIO_AN_VEND_PROV_2500BASET_FULL;
-
-	if (linkmode_test_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT,
-			      phydev->advertising))
-		reg |= MDIO_AN_VEND_PROV_5000BASET_FULL;
-
-	ret = phy_modify_mmd_changed(phydev, MDIO_MMD_AN, MDIO_AN_VEND_PROV,
-				     MDIO_AN_VEND_PROV_1000BASET_HALF |
-				     MDIO_AN_VEND_PROV_1000BASET_FULL |
-				     MDIO_AN_VEND_PROV_2500BASET_FULL |
-				     MDIO_AN_VEND_PROV_5000BASET_FULL, reg);
-	if (ret < 0)
-		return ret;
-	if (ret > 0)
-		changed = true;
-
-	return genphy_c45_check_and_restart_aneg(phydev, changed);
-}
-
-static int aqr_config_intr(struct phy_device *phydev)
-{
-	bool en = phydev->interrupts == PHY_INTERRUPT_ENABLED;
-	int err;
-
-	if (en) {
-		/* Clear any pending interrupts before enabling them */
-		err = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_AN_TX_VEND_INT_STATUS2);
-		if (err < 0)
-			return err;
-	}
-
-	err = phy_write_mmd(phydev, MDIO_MMD_AN, MDIO_AN_TX_VEND_INT_MASK2,
-			    en ? MDIO_AN_TX_VEND_INT_MASK2_LINK : 0);
-	if (err < 0)
-		return err;
-
-	err = phy_write_mmd(phydev, MDIO_MMD_VEND1, VEND1_GLOBAL_INT_STD_MASK,
-			    en ? VEND1_GLOBAL_INT_STD_MASK_ALL : 0);
-	if (err < 0)
-		return err;
-
-	err = phy_write_mmd(phydev, MDIO_MMD_VEND1, VEND1_GLOBAL_INT_VEND_MASK,
-			    en ? VEND1_GLOBAL_INT_VEND_MASK_GLOBAL3 |
-			    VEND1_GLOBAL_INT_VEND_MASK_AN : 0);
-	if (err < 0)
-		return err;
-
-	if (!en) {
-		/* Clear any pending interrupts after we have disabled them */
-		err = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_AN_TX_VEND_INT_STATUS2);
-		if (err < 0)
-			return err;
-	}
-
-	return 0;
-}
-
-static irqreturn_t aqr_handle_interrupt(struct phy_device *phydev)
-{
-	int irq_status;
-
-	irq_status = phy_read_mmd(phydev, MDIO_MMD_AN,
-				  MDIO_AN_TX_VEND_INT_STATUS2);
-	if (irq_status < 0) {
-		phy_error(phydev);
-		return IRQ_NONE;
-	}
-
-	if (!(irq_status & MDIO_AN_TX_VEND_INT_STATUS2_MASK))
-		return IRQ_NONE;
-
-	phy_trigger_machine(phydev);
-
-	return IRQ_HANDLED;
-}
-
-static int aqr_read_status(struct phy_device *phydev)
-{
-	int val;
-
-	if (phydev->autoneg == AUTONEG_ENABLE) {
-		val = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_AN_RX_LP_STAT1);
-		if (val < 0)
-			return val;
-
-		linkmode_mod_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT,
-				 phydev->lp_advertising,
-				 val & MDIO_AN_RX_LP_STAT1_1000BASET_FULL);
-		linkmode_mod_bit(ETHTOOL_LINK_MODE_1000baseT_Half_BIT,
-				 phydev->lp_advertising,
-				 val & MDIO_AN_RX_LP_STAT1_1000BASET_HALF);
-	}
-
-	return genphy_c45_read_status(phydev);
-}
-
-static int aqr107_read_rate(struct phy_device *phydev)
-{
-	u32 config_reg;
-	int val;
-
-	val = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_AN_TX_VEND_STATUS1);
-	if (val < 0)
-		return val;
-
-	if (val & MDIO_AN_TX_VEND_STATUS1_FULL_DUPLEX)
-		phydev->duplex = DUPLEX_FULL;
-	else
-		phydev->duplex = DUPLEX_HALF;
-
-	switch (FIELD_GET(MDIO_AN_TX_VEND_STATUS1_RATE_MASK, val)) {
-	case MDIO_AN_TX_VEND_STATUS1_10BASET:
-		phydev->speed = SPEED_10;
-		config_reg = VEND1_GLOBAL_CFG_10M;
-		break;
-	case MDIO_AN_TX_VEND_STATUS1_100BASETX:
-		phydev->speed = SPEED_100;
-		config_reg = VEND1_GLOBAL_CFG_100M;
-		break;
-	case MDIO_AN_TX_VEND_STATUS1_1000BASET:
-		phydev->speed = SPEED_1000;
-		config_reg = VEND1_GLOBAL_CFG_1G;
-		break;
-	case MDIO_AN_TX_VEND_STATUS1_2500BASET:
-		phydev->speed = SPEED_2500;
-		config_reg = VEND1_GLOBAL_CFG_2_5G;
-		break;
-	case MDIO_AN_TX_VEND_STATUS1_5000BASET:
-		phydev->speed = SPEED_5000;
-		config_reg = VEND1_GLOBAL_CFG_5G;
-		break;
-	case MDIO_AN_TX_VEND_STATUS1_10GBASET:
-		phydev->speed = SPEED_10000;
-		config_reg = VEND1_GLOBAL_CFG_10G;
-		break;
-	default:
-		phydev->speed = SPEED_UNKNOWN;
-		return 0;
-	}
-
-	val = phy_read_mmd(phydev, MDIO_MMD_VEND1, config_reg);
-	if (val < 0)
-		return val;
-
-	if (FIELD_GET(VEND1_GLOBAL_CFG_RATE_ADAPT, val) ==
-	    VEND1_GLOBAL_CFG_RATE_ADAPT_PAUSE)
-		phydev->rate_matching = RATE_MATCH_PAUSE;
-	else
-		phydev->rate_matching = RATE_MATCH_NONE;
-
-	return 0;
-}
-
-static int aqr107_read_status(struct phy_device *phydev)
-{
-	int val, ret;
-
-	ret = aqr_read_status(phydev);
-	if (ret)
-		return ret;
-
-	if (!phydev->link || phydev->autoneg == AUTONEG_DISABLE)
-		return 0;
-
-	val = phy_read_mmd(phydev, MDIO_MMD_PHYXS, MDIO_PHYXS_VEND_IF_STATUS);
-	if (val < 0)
-		return val;
-
-	switch (FIELD_GET(MDIO_PHYXS_VEND_IF_STATUS_TYPE_MASK, val)) {
-	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_KR:
-		phydev->interface = PHY_INTERFACE_MODE_10GKR;
-		break;
-	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_KX:
-		phydev->interface = PHY_INTERFACE_MODE_1000BASEKX;
-		break;
-	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_XFI:
-		phydev->interface = PHY_INTERFACE_MODE_10GBASER;
-		break;
-	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_USXGMII:
-		phydev->interface = PHY_INTERFACE_MODE_USXGMII;
-		break;
-	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_XAUI:
-		phydev->interface = PHY_INTERFACE_MODE_XAUI;
-		break;
-	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_SGMII:
-		phydev->interface = PHY_INTERFACE_MODE_SGMII;
-		break;
-	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_RXAUI:
-		phydev->interface = PHY_INTERFACE_MODE_RXAUI;
-		break;
-	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_OCSGMII:
-		phydev->interface = PHY_INTERFACE_MODE_2500BASEX;
-		break;
-	default:
-		phydev->interface = PHY_INTERFACE_MODE_NA;
-		break;
-	}
-
-	/* Read possibly downshifted rate from vendor register */
-	return aqr107_read_rate(phydev);
-}
-
-static int aqr107_get_downshift(struct phy_device *phydev, u8 *data)
-{
-	int val, cnt, enable;
-
-	val = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_AN_VEND_PROV);
-	if (val < 0)
-		return val;
-
-	enable = FIELD_GET(MDIO_AN_VEND_PROV_DOWNSHIFT_EN, val);
-	cnt = FIELD_GET(MDIO_AN_VEND_PROV_DOWNSHIFT_MASK, val);
-
-	*data = enable && cnt ? cnt : DOWNSHIFT_DEV_DISABLE;
-
-	return 0;
-}
-
-static int aqr107_set_downshift(struct phy_device *phydev, u8 cnt)
-{
-	int val = 0;
-
-	if (!FIELD_FIT(MDIO_AN_VEND_PROV_DOWNSHIFT_MASK, cnt))
-		return -E2BIG;
-
-	if (cnt != DOWNSHIFT_DEV_DISABLE) {
-		val = MDIO_AN_VEND_PROV_DOWNSHIFT_EN;
-		val |= FIELD_PREP(MDIO_AN_VEND_PROV_DOWNSHIFT_MASK, cnt);
-	}
-
-	return phy_modify_mmd(phydev, MDIO_MMD_AN, MDIO_AN_VEND_PROV,
-			      MDIO_AN_VEND_PROV_DOWNSHIFT_EN |
-			      MDIO_AN_VEND_PROV_DOWNSHIFT_MASK, val);
-}
-
-static int aqr107_get_tunable(struct phy_device *phydev,
-			      struct ethtool_tunable *tuna, void *data)
-{
-	switch (tuna->id) {
-	case ETHTOOL_PHY_DOWNSHIFT:
-		return aqr107_get_downshift(phydev, data);
-	default:
-		return -EOPNOTSUPP;
-	}
-}
-
-static int aqr107_set_tunable(struct phy_device *phydev,
-			      struct ethtool_tunable *tuna, const void *data)
-{
-	switch (tuna->id) {
-	case ETHTOOL_PHY_DOWNSHIFT:
-		return aqr107_set_downshift(phydev, *(const u8 *)data);
-	default:
-		return -EOPNOTSUPP;
-	}
-}
-
-/* If we configure settings whilst firmware is still initializing the chip,
- * then these settings may be overwritten. Therefore make sure chip
- * initialization has completed. Use presence of the firmware ID as
- * indicator for initialization having completed.
- * The chip also provides a "reset completed" bit, but it's cleared after
- * read. Therefore function would time out if called again.
- */
-static int aqr107_wait_reset_complete(struct phy_device *phydev)
-{
-	int val;
-
-	return phy_read_mmd_poll_timeout(phydev, MDIO_MMD_VEND1,
-					 VEND1_GLOBAL_FW_ID, val, val != 0,
-					 20000, 2000000, false);
-}
-
-static void aqr107_chip_info(struct phy_device *phydev)
-{
-	u8 fw_major, fw_minor, build_id, prov_id;
-	int val;
-
-	val = phy_read_mmd(phydev, MDIO_MMD_VEND1, VEND1_GLOBAL_FW_ID);
-	if (val < 0)
-		return;
-
-	fw_major = FIELD_GET(VEND1_GLOBAL_FW_ID_MAJOR, val);
-	fw_minor = FIELD_GET(VEND1_GLOBAL_FW_ID_MINOR, val);
-
-	val = phy_read_mmd(phydev, MDIO_MMD_VEND1, VEND1_GLOBAL_RSVD_STAT1);
-	if (val < 0)
-		return;
-
-	build_id = FIELD_GET(VEND1_GLOBAL_RSVD_STAT1_FW_BUILD_ID, val);
-	prov_id = FIELD_GET(VEND1_GLOBAL_RSVD_STAT1_PROV_ID, val);
-
-	phydev_dbg(phydev, "FW %u.%u, Build %u, Provisioning %u\n",
-		   fw_major, fw_minor, build_id, prov_id);
-}
-
-static int aqr107_config_init(struct phy_device *phydev)
-{
-	int ret;
-
-	/* Check that the PHY interface type is compatible */
-	if (phydev->interface != PHY_INTERFACE_MODE_SGMII &&
-	    phydev->interface != PHY_INTERFACE_MODE_1000BASEKX &&
-	    phydev->interface != PHY_INTERFACE_MODE_2500BASEX &&
-	    phydev->interface != PHY_INTERFACE_MODE_XGMII &&
-	    phydev->interface != PHY_INTERFACE_MODE_USXGMII &&
-	    phydev->interface != PHY_INTERFACE_MODE_10GKR &&
-	    phydev->interface != PHY_INTERFACE_MODE_10GBASER &&
-	    phydev->interface != PHY_INTERFACE_MODE_XAUI &&
-	    phydev->interface != PHY_INTERFACE_MODE_RXAUI)
-		return -ENODEV;
-
-	WARN(phydev->interface == PHY_INTERFACE_MODE_XGMII,
-	     "Your devicetree is out of date, please update it. The AQR107 family doesn't support XGMII, maybe you mean USXGMII.\n");
-
-	ret = aqr107_wait_reset_complete(phydev);
-	if (!ret)
-		aqr107_chip_info(phydev);
-
-	return aqr107_set_downshift(phydev, MDIO_AN_VEND_PROV_DOWNSHIFT_DFLT);
-}
-
-static int aqcs109_config_init(struct phy_device *phydev)
-{
-	int ret;
-
-	/* Check that the PHY interface type is compatible */
-	if (phydev->interface != PHY_INTERFACE_MODE_SGMII &&
-	    phydev->interface != PHY_INTERFACE_MODE_2500BASEX)
-		return -ENODEV;
-
-	ret = aqr107_wait_reset_complete(phydev);
-	if (!ret)
-		aqr107_chip_info(phydev);
-
-	/* AQCS109 belongs to a chip family partially supporting 10G and 5G.
-	 * PMA speed ability bits are the same for all members of the family,
-	 * AQCS109 however supports speeds up to 2.5G only.
-	 */
-	phy_set_max_speed(phydev, SPEED_2500);
-
-	return aqr107_set_downshift(phydev, MDIO_AN_VEND_PROV_DOWNSHIFT_DFLT);
-}
-
-static void aqr107_link_change_notify(struct phy_device *phydev)
-{
-	u8 fw_major, fw_minor;
-	bool downshift, short_reach, afr;
-	int mode, val;
-
-	if (phydev->state != PHY_RUNNING || phydev->autoneg == AUTONEG_DISABLE)
-		return;
-
-	val = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_AN_RX_LP_STAT1);
-	/* call failed or link partner is no Aquantia PHY */
-	if (val < 0 || !(val & MDIO_AN_RX_LP_STAT1_AQ_PHY))
-		return;
-
-	short_reach = val & MDIO_AN_RX_LP_STAT1_SHORT_REACH;
-	downshift = val & MDIO_AN_RX_LP_STAT1_AQRATE_DOWNSHIFT;
-
-	val = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_AN_RX_LP_STAT4);
-	if (val < 0)
-		return;
-
-	fw_major = FIELD_GET(MDIO_AN_RX_LP_STAT4_FW_MAJOR, val);
-	fw_minor = FIELD_GET(MDIO_AN_RX_LP_STAT4_FW_MINOR, val);
-
-	val = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_AN_RX_VEND_STAT3);
-	if (val < 0)
-		return;
-
-	afr = val & MDIO_AN_RX_VEND_STAT3_AFR;
-
-	phydev_dbg(phydev, "Link partner is Aquantia PHY, FW %u.%u%s%s%s\n",
-		   fw_major, fw_minor,
-		   short_reach ? ", short reach mode" : "",
-		   downshift ? ", fast-retrain downshift advertised" : "",
-		   afr ? ", fast reframe advertised" : "");
-
-	val = phy_read_mmd(phydev, MDIO_MMD_VEND1, VEND1_GLOBAL_RSVD_STAT9);
-	if (val < 0)
-		return;
-
-	mode = FIELD_GET(VEND1_GLOBAL_RSVD_STAT9_MODE, val);
-	if (mode == VEND1_GLOBAL_RSVD_STAT9_1000BT2)
-		phydev_info(phydev, "Aquantia 1000Base-T2 mode active\n");
-}
-
-static int aqr107_wait_processor_intensive_op(struct phy_device *phydev)
-{
-	int val, err;
-
-	/* The datasheet notes to wait at least 1ms after issuing a
-	 * processor intensive operation before checking.
-	 * We cannot use the 'sleep_before_read' parameter of read_poll_timeout
-	 * because that just determines the maximum time slept, not the minimum.
-	 */
-	usleep_range(1000, 5000);
-
-	err = phy_read_mmd_poll_timeout(phydev, MDIO_MMD_VEND1,
-					VEND1_GLOBAL_GEN_STAT2, val,
-					!(val & VEND1_GLOBAL_GEN_STAT2_OP_IN_PROG),
-					AQR107_OP_IN_PROG_SLEEP,
-					AQR107_OP_IN_PROG_TIMEOUT, false);
-	if (err) {
-		phydev_err(phydev, "timeout: processor-intensive MDIO operation\n");
-		return err;
-	}
-
-	return 0;
-}
-
-static int aqr107_get_rate_matching(struct phy_device *phydev,
-				    phy_interface_t iface)
-{
-	if (iface == PHY_INTERFACE_MODE_10GBASER ||
-	    iface == PHY_INTERFACE_MODE_2500BASEX ||
-	    iface == PHY_INTERFACE_MODE_NA)
-		return RATE_MATCH_PAUSE;
-	return RATE_MATCH_NONE;
-}
-
-static int aqr107_suspend(struct phy_device *phydev)
-{
-	int err;
-
-	err = phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, MDIO_CTRL1,
-			       MDIO_CTRL1_LPOWER);
-	if (err)
-		return err;
-
-	return aqr107_wait_processor_intensive_op(phydev);
-}
-
-static int aqr107_resume(struct phy_device *phydev)
-{
-	int err;
-
-	err = phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, MDIO_CTRL1,
-				 MDIO_CTRL1_LPOWER);
-	if (err)
-		return err;
-
-	return aqr107_wait_processor_intensive_op(phydev);
-}
-
-static int aqr107_probe(struct phy_device *phydev)
-{
-	phydev->priv = devm_kzalloc(&phydev->mdio.dev,
-				    sizeof(struct aqr107_priv), GFP_KERNEL);
-	if (!phydev->priv)
-		return -ENOMEM;
-
-	return aqr_hwmon_probe(phydev);
-}
-
-static struct phy_driver aqr_driver[] = {
-{
-	PHY_ID_MATCH_MODEL(PHY_ID_AQ1202),
-	.name		= "Aquantia AQ1202",
-	.config_aneg    = aqr_config_aneg,
-	.config_intr	= aqr_config_intr,
-	.handle_interrupt = aqr_handle_interrupt,
-	.read_status	= aqr_read_status,
-},
-{
-	PHY_ID_MATCH_MODEL(PHY_ID_AQ2104),
-	.name		= "Aquantia AQ2104",
-	.config_aneg    = aqr_config_aneg,
-	.config_intr	= aqr_config_intr,
-	.handle_interrupt = aqr_handle_interrupt,
-	.read_status	= aqr_read_status,
-},
-{
-	PHY_ID_MATCH_MODEL(PHY_ID_AQR105),
-	.name		= "Aquantia AQR105",
-	.config_aneg    = aqr_config_aneg,
-	.config_intr	= aqr_config_intr,
-	.handle_interrupt = aqr_handle_interrupt,
-	.read_status	= aqr_read_status,
-	.suspend	= aqr107_suspend,
-	.resume		= aqr107_resume,
-},
-{
-	PHY_ID_MATCH_MODEL(PHY_ID_AQR106),
-	.name		= "Aquantia AQR106",
-	.config_aneg    = aqr_config_aneg,
-	.config_intr	= aqr_config_intr,
-	.handle_interrupt = aqr_handle_interrupt,
-	.read_status	= aqr_read_status,
-},
-{
-	PHY_ID_MATCH_MODEL(PHY_ID_AQR107),
-	.name		= "Aquantia AQR107",
-	.probe		= aqr107_probe,
-	.get_rate_matching = aqr107_get_rate_matching,
-	.config_init	= aqr107_config_init,
-	.config_aneg    = aqr_config_aneg,
-	.config_intr	= aqr_config_intr,
-	.handle_interrupt = aqr_handle_interrupt,
-	.read_status	= aqr107_read_status,
-	.get_tunable    = aqr107_get_tunable,
-	.set_tunable    = aqr107_set_tunable,
-	.suspend	= aqr107_suspend,
-	.resume		= aqr107_resume,
-	.get_sset_count	= aqr107_get_sset_count,
-	.get_strings	= aqr107_get_strings,
-	.get_stats	= aqr107_get_stats,
-	.link_change_notify = aqr107_link_change_notify,
-},
-{
-	PHY_ID_MATCH_MODEL(PHY_ID_AQCS109),
-	.name		= "Aquantia AQCS109",
-	.probe		= aqr107_probe,
-	.get_rate_matching = aqr107_get_rate_matching,
-	.config_init	= aqcs109_config_init,
-	.config_aneg    = aqr_config_aneg,
-	.config_intr	= aqr_config_intr,
-	.handle_interrupt = aqr_handle_interrupt,
-	.read_status	= aqr107_read_status,
-	.get_tunable    = aqr107_get_tunable,
-	.set_tunable    = aqr107_set_tunable,
-	.suspend	= aqr107_suspend,
-	.resume		= aqr107_resume,
-	.get_sset_count	= aqr107_get_sset_count,
-	.get_strings	= aqr107_get_strings,
-	.get_stats	= aqr107_get_stats,
-	.link_change_notify = aqr107_link_change_notify,
-},
-{
-	PHY_ID_MATCH_MODEL(PHY_ID_AQR405),
-	.name		= "Aquantia AQR405",
-	.config_aneg    = aqr_config_aneg,
-	.config_intr	= aqr_config_intr,
-	.handle_interrupt = aqr_handle_interrupt,
-	.read_status	= aqr_read_status,
-},
-{
-	PHY_ID_MATCH_MODEL(PHY_ID_AQR112),
-	.name		= "Aquantia AQR112",
-	.probe		= aqr107_probe,
-	.config_aneg    = aqr_config_aneg,
-	.config_intr	= aqr_config_intr,
-	.handle_interrupt = aqr_handle_interrupt,
-	.get_tunable    = aqr107_get_tunable,
-	.set_tunable    = aqr107_set_tunable,
-	.suspend	= aqr107_suspend,
-	.resume		= aqr107_resume,
-	.read_status	= aqr107_read_status,
-	.get_rate_matching = aqr107_get_rate_matching,
-	.get_sset_count = aqr107_get_sset_count,
-	.get_strings	= aqr107_get_strings,
-	.get_stats	= aqr107_get_stats,
-	.link_change_notify = aqr107_link_change_notify,
-},
-{
-	PHY_ID_MATCH_MODEL(PHY_ID_AQR412),
-	.name		= "Aquantia AQR412",
-	.probe		= aqr107_probe,
-	.config_aneg    = aqr_config_aneg,
-	.config_intr	= aqr_config_intr,
-	.handle_interrupt = aqr_handle_interrupt,
-	.get_tunable    = aqr107_get_tunable,
-	.set_tunable    = aqr107_set_tunable,
-	.suspend	= aqr107_suspend,
-	.resume		= aqr107_resume,
-	.read_status	= aqr107_read_status,
-	.get_rate_matching = aqr107_get_rate_matching,
-	.get_sset_count = aqr107_get_sset_count,
-	.get_strings	= aqr107_get_strings,
-	.get_stats	= aqr107_get_stats,
-	.link_change_notify = aqr107_link_change_notify,
-},
-{
-	PHY_ID_MATCH_MODEL(PHY_ID_AQR113C),
-	.name           = "Aquantia AQR113C",
-	.probe          = aqr107_probe,
-	.get_rate_matching = aqr107_get_rate_matching,
-	.config_init    = aqr107_config_init,
-	.config_aneg    = aqr_config_aneg,
-	.config_intr    = aqr_config_intr,
-	.handle_interrupt       = aqr_handle_interrupt,
-	.read_status    = aqr107_read_status,
-	.get_tunable    = aqr107_get_tunable,
-	.set_tunable    = aqr107_set_tunable,
-	.suspend        = aqr107_suspend,
-	.resume         = aqr107_resume,
-	.get_sset_count = aqr107_get_sset_count,
-	.get_strings    = aqr107_get_strings,
-	.get_stats      = aqr107_get_stats,
-	.link_change_notify = aqr107_link_change_notify,
-},
-};
-
-module_phy_driver(aqr_driver);
-
-static struct mdio_device_id __maybe_unused aqr_tbl[] = {
-	{ PHY_ID_MATCH_MODEL(PHY_ID_AQ1202) },
-	{ PHY_ID_MATCH_MODEL(PHY_ID_AQ2104) },
-	{ PHY_ID_MATCH_MODEL(PHY_ID_AQR105) },
-	{ PHY_ID_MATCH_MODEL(PHY_ID_AQR106) },
-	{ PHY_ID_MATCH_MODEL(PHY_ID_AQR107) },
-	{ PHY_ID_MATCH_MODEL(PHY_ID_AQCS109) },
-	{ PHY_ID_MATCH_MODEL(PHY_ID_AQR405) },
-	{ PHY_ID_MATCH_MODEL(PHY_ID_AQR112) },
-	{ PHY_ID_MATCH_MODEL(PHY_ID_AQR412) },
-	{ PHY_ID_MATCH_MODEL(PHY_ID_AQR113C) },
-	{ }
-};
-
-MODULE_DEVICE_TABLE(mdio, aqr_tbl);
-
-MODULE_DESCRIPTION("Aquantia PHY driver");
-MODULE_AUTHOR("Shaohui Xie <Shaohui.Xie@freescale.com>");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/phy/bcm-phy-ptp.c b/drivers/net/phy/bcm-phy-ptp.c
index cb4b91af5e17..617d384d4551 100644
--- a/drivers/net/phy/bcm-phy-ptp.c
+++ b/drivers/net/phy/bcm-phy-ptp.c
@@ -782,16 +782,13 @@ static void bcm_ptp_txtstamp(struct mii_timestamper *mii_ts,
 }
 
 static int bcm_ptp_hwtstamp(struct mii_timestamper *mii_ts,
-			    struct ifreq *ifr)
+			    struct kernel_hwtstamp_config *cfg,
+			    struct netlink_ext_ack *extack)
 {
 	struct bcm_ptp_private *priv = mii2priv(mii_ts);
-	struct hwtstamp_config cfg;
 	u16 mode, ctrl;
 
-	if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
-		return -EFAULT;
-
-	switch (cfg.rx_filter) {
+	switch (cfg->rx_filter) {
 	case HWTSTAMP_FILTER_NONE:
 		priv->hwts_rx = false;
 		break;
@@ -804,14 +801,14 @@ static int bcm_ptp_hwtstamp(struct mii_timestamper *mii_ts,
 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
-		cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
+		cfg->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
 		priv->hwts_rx = true;
 		break;
 	default:
 		return -ERANGE;
 	}
 
-	priv->tx_type = cfg.tx_type;
+	priv->tx_type = cfg->tx_type;
 
 	ctrl  = priv->hwts_rx ? SLICE_RX_EN : 0;
 	ctrl |= priv->tx_type != HWTSTAMP_TX_OFF ? SLICE_TX_EN : 0;
@@ -840,7 +837,7 @@ static int bcm_ptp_hwtstamp(struct mii_timestamper *mii_ts,
 	/* purge existing data */
 	skb_queue_purge(&priv->tx_queue);
 
-	return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
+	return 0;
 }
 
 static int bcm_ptp_ts_info(struct mii_timestamper *mii_ts,
diff --git a/drivers/net/phy/broadcom.c b/drivers/net/phy/broadcom.c
index 3a627105675a..312a8bb35d78 100644
--- a/drivers/net/phy/broadcom.c
+++ b/drivers/net/phy/broadcom.c
@@ -1135,6 +1135,8 @@ static struct phy_driver broadcom_drivers[] = {
 	.handle_interrupt = bcm_phy_handle_interrupt,
 	.link_change_notify	= bcm54xx_link_change_notify,
 	.led_brightness_set	= bcm_phy_led_brightness_set,
+	.suspend	= bcm54xx_suspend,
+	.resume		= bcm54xx_resume,
 }, {
 	.phy_id		= PHY_ID_BCM54616S,
 	.phy_id_mask	= 0xfffffff0,
diff --git a/drivers/net/phy/dp83640.c b/drivers/net/phy/dp83640.c
index 2657be7cc049..5c42c47dc564 100644
--- a/drivers/net/phy/dp83640.c
+++ b/drivers/net/phy/dp83640.c
@@ -1207,22 +1207,20 @@ static irqreturn_t dp83640_handle_interrupt(struct phy_device *phydev)
 	return IRQ_HANDLED;
 }
 
-static int dp83640_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr)
+static int dp83640_hwtstamp(struct mii_timestamper *mii_ts,
+			    struct kernel_hwtstamp_config *cfg,
+			    struct netlink_ext_ack *extack)
 {
 	struct dp83640_private *dp83640 =
 		container_of(mii_ts, struct dp83640_private, mii_ts);
-	struct hwtstamp_config cfg;
 	u16 txcfg0, rxcfg0;
 
-	if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
-		return -EFAULT;
-
-	if (cfg.tx_type < 0 || cfg.tx_type > HWTSTAMP_TX_ONESTEP_SYNC)
+	if (cfg->tx_type < 0 || cfg->tx_type > HWTSTAMP_TX_ONESTEP_SYNC)
 		return -ERANGE;
 
-	dp83640->hwts_tx_en = cfg.tx_type;
+	dp83640->hwts_tx_en = cfg->tx_type;
 
-	switch (cfg.rx_filter) {
+	switch (cfg->rx_filter) {
 	case HWTSTAMP_FILTER_NONE:
 		dp83640->hwts_rx_en = 0;
 		dp83640->layer = 0;
@@ -1234,7 +1232,7 @@ static int dp83640_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr)
 		dp83640->hwts_rx_en = 1;
 		dp83640->layer = PTP_CLASS_L4;
 		dp83640->version = PTP_CLASS_V1;
-		cfg.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
+		cfg->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 		break;
 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
 	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
@@ -1242,7 +1240,7 @@ static int dp83640_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr)
 		dp83640->hwts_rx_en = 1;
 		dp83640->layer = PTP_CLASS_L4;
 		dp83640->version = PTP_CLASS_V2;
-		cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
+		cfg->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
 		break;
 	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
 	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
@@ -1250,7 +1248,7 @@ static int dp83640_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr)
 		dp83640->hwts_rx_en = 1;
 		dp83640->layer = PTP_CLASS_L2;
 		dp83640->version = PTP_CLASS_V2;
-		cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT;
+		cfg->rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT;
 		break;
 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
@@ -1258,7 +1256,7 @@ static int dp83640_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr)
 		dp83640->hwts_rx_en = 1;
 		dp83640->layer = PTP_CLASS_L4 | PTP_CLASS_L2;
 		dp83640->version = PTP_CLASS_V2;
-		cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
+		cfg->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
 		break;
 	default:
 		return -ERANGE;
@@ -1292,7 +1290,7 @@ static int dp83640_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr)
 
 	mutex_unlock(&dp83640->clock->extreg_lock);
 
-	return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
+	return 0;
 }
 
 static void rx_timestamp_work(struct work_struct *work)
diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c
index 08e3915001c3..bd4cd082662f 100644
--- a/drivers/net/phy/micrel.c
+++ b/drivers/net/phy/micrel.c
@@ -2395,24 +2395,22 @@ static void lan8814_flush_fifo(struct phy_device *phydev, bool egress)
 	lanphy_read_page_reg(phydev, 5, PTP_TSU_INT_STS);
 }
 
-static int lan8814_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr)
+static int lan8814_hwtstamp(struct mii_timestamper *mii_ts,
+			    struct kernel_hwtstamp_config *config,
+			    struct netlink_ext_ack *extack)
 {
 	struct kszphy_ptp_priv *ptp_priv =
 			  container_of(mii_ts, struct kszphy_ptp_priv, mii_ts);
 	struct phy_device *phydev = ptp_priv->phydev;
 	struct lan8814_shared_priv *shared = phydev->shared->priv;
 	struct lan8814_ptp_rx_ts *rx_ts, *tmp;
-	struct hwtstamp_config config;
 	int txcfg = 0, rxcfg = 0;
 	int pkt_ts_enable;
 
-	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
-		return -EFAULT;
+	ptp_priv->hwts_tx_type = config->tx_type;
+	ptp_priv->rx_filter = config->rx_filter;
 
-	ptp_priv->hwts_tx_type = config.tx_type;
-	ptp_priv->rx_filter = config.rx_filter;
-
-	switch (config.rx_filter) {
+	switch (config->rx_filter) {
 	case HWTSTAMP_FILTER_NONE:
 		ptp_priv->layer = 0;
 		ptp_priv->version = 0;
@@ -2458,13 +2456,13 @@ static int lan8814_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr)
 		lanphy_write_page_reg(ptp_priv->phydev, 5, PTP_TX_MOD,
 				      PTP_TX_MOD_TX_PTP_SYNC_TS_INSERT_);
 
-	if (config.rx_filter != HWTSTAMP_FILTER_NONE)
+	if (config->rx_filter != HWTSTAMP_FILTER_NONE)
 		lan8814_config_ts_intr(ptp_priv->phydev, true);
 	else
 		lan8814_config_ts_intr(ptp_priv->phydev, false);
 
 	mutex_lock(&shared->shared_lock);
-	if (config.rx_filter != HWTSTAMP_FILTER_NONE)
+	if (config->rx_filter != HWTSTAMP_FILTER_NONE)
 		shared->ref++;
 	else
 		shared->ref--;
@@ -2488,7 +2486,7 @@ static int lan8814_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr)
 	lan8814_flush_fifo(ptp_priv->phydev, false);
 	lan8814_flush_fifo(ptp_priv->phydev, true);
 
-	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? -EFAULT : 0;
+	return 0;
 }
 
 static void lan8814_txtstamp(struct mii_timestamper *mii_ts,
@@ -3631,12 +3629,8 @@ static int lan8841_ts_info(struct mii_timestamper *mii_ts,
 
 	info->phc_index = ptp_priv->ptp_clock ?
 				ptp_clock_index(ptp_priv->ptp_clock) : -1;
-	if (info->phc_index == -1) {
-		info->so_timestamping |= SOF_TIMESTAMPING_TX_SOFTWARE |
-					 SOF_TIMESTAMPING_RX_SOFTWARE |
-					 SOF_TIMESTAMPING_SOFTWARE;
+	if (info->phc_index == -1)
 		return 0;
-	}
 
 	info->so_timestamping = SOF_TIMESTAMPING_TX_HARDWARE |
 				SOF_TIMESTAMPING_RX_HARDWARE |
@@ -3703,21 +3697,19 @@ static void lan8841_ptp_enable_processing(struct kszphy_ptp_priv *ptp_priv,
 #define LAN8841_PTP_TX_TIMESTAMP_EN		443
 #define LAN8841_PTP_TX_MOD			445
 
-static int lan8841_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr)
+static int lan8841_hwtstamp(struct mii_timestamper *mii_ts,
+			    struct kernel_hwtstamp_config *config,
+			    struct netlink_ext_ack *extack)
 {
 	struct kszphy_ptp_priv *ptp_priv = container_of(mii_ts, struct kszphy_ptp_priv, mii_ts);
 	struct phy_device *phydev = ptp_priv->phydev;
-	struct hwtstamp_config config;
 	int txcfg = 0, rxcfg = 0;
 	int pkt_ts_enable;
 
-	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
-		return -EFAULT;
+	ptp_priv->hwts_tx_type = config->tx_type;
+	ptp_priv->rx_filter = config->rx_filter;
 
-	ptp_priv->hwts_tx_type = config.tx_type;
-	ptp_priv->rx_filter = config.rx_filter;
-
-	switch (config.rx_filter) {
+	switch (config->rx_filter) {
 	case HWTSTAMP_FILTER_NONE:
 		ptp_priv->layer = 0;
 		ptp_priv->version = 0;
@@ -3771,13 +3763,13 @@ static int lan8841_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr)
 
 	/* Now enable/disable the timestamping */
 	lan8841_ptp_enable_processing(ptp_priv,
-				      config.rx_filter != HWTSTAMP_FILTER_NONE);
+				      config->rx_filter != HWTSTAMP_FILTER_NONE);
 
 	skb_queue_purge(&ptp_priv->tx_queue);
 
 	lan8841_ptp_flush_fifo(ptp_priv);
 
-	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? -EFAULT : 0;
+	return 0;
 }
 
 static bool lan8841_rxtstamp(struct mii_timestamper *mii_ts,
diff --git a/drivers/net/phy/mscc/mscc_ptp.c b/drivers/net/phy/mscc/mscc_ptp.c
index cf728bfd83e2..eb0b032cb613 100644
--- a/drivers/net/phy/mscc/mscc_ptp.c
+++ b/drivers/net/phy/mscc/mscc_ptp.c
@@ -1045,19 +1045,17 @@ static void vsc85xx_ts_reset_fifo(struct phy_device *phydev)
 			     val);
 }
 
-static int vsc85xx_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr)
+static int vsc85xx_hwtstamp(struct mii_timestamper *mii_ts,
+			    struct kernel_hwtstamp_config *cfg,
+			    struct netlink_ext_ack *extack)
 {
 	struct vsc8531_private *vsc8531 =
 		container_of(mii_ts, struct vsc8531_private, mii_ts);
 	struct phy_device *phydev = vsc8531->ptp->phydev;
-	struct hwtstamp_config cfg;
 	bool one_step = false;
 	u32 val;
 
-	if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
-		return -EFAULT;
-
-	switch (cfg.tx_type) {
+	switch (cfg->tx_type) {
 	case HWTSTAMP_TX_ONESTEP_SYNC:
 		one_step = true;
 		break;
@@ -1069,9 +1067,9 @@ static int vsc85xx_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr)
 		return -ERANGE;
 	}
 
-	vsc8531->ptp->tx_type = cfg.tx_type;
+	vsc8531->ptp->tx_type = cfg->tx_type;
 
-	switch (cfg.rx_filter) {
+	switch (cfg->rx_filter) {
 	case HWTSTAMP_FILTER_NONE:
 		break;
 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
@@ -1084,7 +1082,7 @@ static int vsc85xx_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr)
 		return -ERANGE;
 	}
 
-	vsc8531->ptp->rx_filter = cfg.rx_filter;
+	vsc8531->ptp->rx_filter = cfg->rx_filter;
 
 	mutex_lock(&vsc8531->ts_lock);
 
@@ -1132,7 +1130,7 @@ static int vsc85xx_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr)
 	vsc8531->ptp->configured = 1;
 	mutex_unlock(&vsc8531->ts_lock);
 
-	return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
+	return 0;
 }
 
 static int vsc85xx_ts_info(struct mii_timestamper *mii_ts,
diff --git a/drivers/net/phy/nxp-c45-tja11xx.c b/drivers/net/phy/nxp-c45-tja11xx.c
index 7ab080ff02df..780ad353cf55 100644
--- a/drivers/net/phy/nxp-c45-tja11xx.c
+++ b/drivers/net/phy/nxp-c45-tja11xx.c
@@ -1022,24 +1022,21 @@ static bool nxp_c45_rxtstamp(struct mii_timestamper *mii_ts,
 }
 
 static int nxp_c45_hwtstamp(struct mii_timestamper *mii_ts,
-			    struct ifreq *ifreq)
+			    struct kernel_hwtstamp_config *cfg,
+			    struct netlink_ext_ack *extack)
 {
 	struct nxp_c45_phy *priv = container_of(mii_ts, struct nxp_c45_phy,
 						mii_ts);
 	struct phy_device *phydev = priv->phydev;
 	const struct nxp_c45_phy_data *data;
-	struct hwtstamp_config cfg;
 
-	if (copy_from_user(&cfg, ifreq->ifr_data, sizeof(cfg)))
-		return -EFAULT;
-
-	if (cfg.tx_type < 0 || cfg.tx_type > HWTSTAMP_TX_ON)
+	if (cfg->tx_type < 0 || cfg->tx_type > HWTSTAMP_TX_ON)
 		return -ERANGE;
 
 	data = nxp_c45_get_data(phydev);
-	priv->hwts_tx = cfg.tx_type;
+	priv->hwts_tx = cfg->tx_type;
 
-	switch (cfg.rx_filter) {
+	switch (cfg->rx_filter) {
 	case HWTSTAMP_FILTER_NONE:
 		priv->hwts_rx = 0;
 		break;
@@ -1047,7 +1044,7 @@ static int nxp_c45_hwtstamp(struct mii_timestamper *mii_ts,
 	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
 	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
 		priv->hwts_rx = 1;
-		cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT;
+		cfg->rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT;
 		break;
 	default:
 		return -ERANGE;
@@ -1074,7 +1071,7 @@ static int nxp_c45_hwtstamp(struct mii_timestamper *mii_ts,
 		nxp_c45_clear_reg_field(phydev, &data->regmap->irq_egr_ts_en);
 
 nxp_c45_no_ptp_irq:
-	return copy_to_user(ifreq->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
+	return 0;
 }
 
 static int nxp_c45_ts_info(struct mii_timestamper *mii_ts,
diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c
index a5fa077650e8..3376e58e2b88 100644
--- a/drivers/net/phy/phy.c
+++ b/drivers/net/phy/phy.c
@@ -325,9 +325,13 @@ EXPORT_SYMBOL(phy_ethtool_ksettings_get);
 int phy_mii_ioctl(struct phy_device *phydev, struct ifreq *ifr, int cmd)
 {
 	struct mii_ioctl_data *mii_data = if_mii(ifr);
+	struct kernel_hwtstamp_config kernel_cfg;
+	struct netlink_ext_ack extack = {};
 	u16 val = mii_data->val_in;
 	bool change_autoneg = false;
+	struct hwtstamp_config cfg;
 	int prtad, devad;
+	int ret;
 
 	switch (cmd) {
 	case SIOCGMIIPHY:
@@ -411,8 +415,21 @@ int phy_mii_ioctl(struct phy_device *phydev, struct ifreq *ifr, int cmd)
 		return 0;
 
 	case SIOCSHWTSTAMP:
-		if (phydev->mii_ts && phydev->mii_ts->hwtstamp)
-			return phydev->mii_ts->hwtstamp(phydev->mii_ts, ifr);
+		if (phydev->mii_ts && phydev->mii_ts->hwtstamp) {
+			if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
+				return -EFAULT;
+
+			hwtstamp_config_to_kernel(&kernel_cfg, &cfg);
+			ret = phydev->mii_ts->hwtstamp(phydev->mii_ts, &kernel_cfg, &extack);
+			if (ret)
+				return ret;
+
+			hwtstamp_config_from_kernel(&cfg, &kernel_cfg);
+			if (copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)))
+				return -EFAULT;
+
+			return 0;
+		}
 		fallthrough;
 
 	default:
@@ -469,7 +486,7 @@ int __phy_hwtstamp_get(struct phy_device *phydev,
 	if (!phydev)
 		return -ENODEV;
 
-	return phy_mii_ioctl(phydev, config->ifr, SIOCGHWTSTAMP);
+	return -EOPNOTSUPP;
 }
 
 /**
@@ -486,7 +503,10 @@ int __phy_hwtstamp_set(struct phy_device *phydev,
 	if (!phydev)
 		return -ENODEV;
 
-	return phy_mii_ioctl(phydev, config->ifr, SIOCSHWTSTAMP);
+	if (phydev->mii_ts && phydev->mii_ts->hwtstamp)
+		return phydev->mii_ts->hwtstamp(phydev->mii_ts, config, extack);
+
+	return -EOPNOTSUPP;
 }
 
 /**
diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c
index 25c19496a336..c276f9482f78 100644
--- a/drivers/net/phy/phylink.c
+++ b/drivers/net/phy/phylink.c
@@ -697,18 +697,16 @@ static int phylink_validate_mask(struct phylink *pl, unsigned long *supported,
 	__ETHTOOL_DECLARE_LINK_MODE_MASK(all_s) = { 0, };
 	__ETHTOOL_DECLARE_LINK_MODE_MASK(s);
 	struct phylink_link_state t;
-	int intf;
+	int interface;
 
-	for (intf = 0; intf < PHY_INTERFACE_MODE_MAX; intf++) {
-		if (test_bit(intf, interfaces)) {
-			linkmode_copy(s, supported);
+	for_each_set_bit(interface, interfaces, PHY_INTERFACE_MODE_MAX) {
+		linkmode_copy(s, supported);
 
-			t = *state;
-			t.interface = intf;
-			if (!phylink_validate_mac_and_pcs(pl, s, &t)) {
-				linkmode_or(all_s, all_s, s);
-				linkmode_or(all_adv, all_adv, t.advertising);
-			}
+		t = *state;
+		t.interface = interface;
+		if (!phylink_validate_mac_and_pcs(pl, s, &t)) {
+			linkmode_or(all_s, all_s, s);
+			linkmode_or(all_adv, all_adv, t.advertising);
 		}
 	}
 
@@ -805,7 +803,7 @@ static int phylink_parse_fixedlink(struct phylink *pl,
 		phylink_warn(pl, "fixed link specifies half duplex for %dMbps link?\n",
 			     pl->link_config.speed);
 
-	bitmap_fill(pl->supported, __ETHTOOL_LINK_MODE_MASK_NBITS);
+	linkmode_fill(pl->supported);
 	linkmode_copy(pl->link_config.advertising, pl->supported);
 	phylink_validate(pl, pl->supported, &pl->link_config);
 
@@ -1640,7 +1638,7 @@ struct phylink *phylink_create(struct phylink_config *config,
 	__set_bit(PHYLINK_DISABLE_STOPPED, &pl->phylink_disable_state);
 	timer_setup(&pl->link_poll, phylink_fixed_poll, 0);
 
-	bitmap_fill(pl->supported, __ETHTOOL_LINK_MODE_MASK_NBITS);
+	linkmode_fill(pl->supported);
 	linkmode_copy(pl->link_config.advertising, pl->supported);
 	phylink_validate(pl, pl->supported, &pl->link_config);
 
diff --git a/drivers/net/phy/sfp-bus.c b/drivers/net/phy/sfp-bus.c
index 208a9393c2df..6fa679b36290 100644
--- a/drivers/net/phy/sfp-bus.c
+++ b/drivers/net/phy/sfp-bus.c
@@ -328,7 +328,7 @@ void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
 	 * modules use 2500Mbaud rather than 3100 or 3200Mbaud for
 	 * 2500BASE-X, so we allow some slack here.
 	 */
-	if (bitmap_empty(modes, __ETHTOOL_LINK_MODE_MASK_NBITS) && br_nom) {
+	if (linkmode_empty(modes) && br_nom) {
 		if (br_min <= 1300 && br_max >= 1200) {
 			phylink_set(modes, 1000baseX_Full);
 			__set_bit(PHY_INTERFACE_MODE_1000BASEX, interfaces);
diff --git a/drivers/net/usb/aqc111.c b/drivers/net/usb/aqc111.c
index a017e9de2119..7b8afa589a53 100644
--- a/drivers/net/usb/aqc111.c
+++ b/drivers/net/usb/aqc111.c
@@ -1079,17 +1079,17 @@ static int aqc111_rx_fixup(struct usbnet *dev, struct sk_buff *skb)
 	u16 pkt_count = 0;
 	u64 desc_hdr = 0;
 	u16 vlan_tag = 0;
-	u32 skb_len = 0;
+	u32 skb_len;
 
 	if (!skb)
 		goto err;
 
-	if (skb->len == 0)
+	skb_len = skb->len;
+	if (skb_len < sizeof(desc_hdr))
 		goto err;
 
-	skb_len = skb->len;
 	/* RX Descriptor Header */
-	skb_trim(skb, skb->len - sizeof(desc_hdr));
+	skb_trim(skb, skb_len - sizeof(desc_hdr));
 	desc_hdr = le64_to_cpup((u64 *)skb_tail_pointer(skb));
 
 	/* Check these packets */
diff --git a/drivers/net/usb/ax88179_178a.c b/drivers/net/usb/ax88179_178a.c
index aff39bf3161d..fb0903ad6de3 100644
--- a/drivers/net/usb/ax88179_178a.c
+++ b/drivers/net/usb/ax88179_178a.c
@@ -1298,8 +1298,6 @@ static int ax88179_bind(struct usbnet *dev, struct usb_interface *intf)
 
 	netif_set_tso_max_size(dev->net, 16384);
 
-	ax88179_reset(dev);
-
 	return 0;
 }
 
diff --git a/drivers/net/veth.c b/drivers/net/veth.c
index 9980517ed8b0..6cc352296c67 100644
--- a/drivers/net/veth.c
+++ b/drivers/net/veth.c
@@ -373,7 +373,7 @@ static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev)
 	skb_tx_timestamp(skb);
 	if (likely(veth_forward_skb(rcv, skb, rq, use_napi) == NET_RX_SUCCESS)) {
 		if (!use_napi)
-			dev_lstats_add(dev, length);
+			dev_sw_netstats_tx_add(dev, 1, length);
 		else
 			__veth_xdp_flush(rq);
 	} else {
@@ -387,14 +387,6 @@ static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev)
 	return ret;
 }
 
-static u64 veth_stats_tx(struct net_device *dev, u64 *packets, u64 *bytes)
-{
-	struct veth_priv *priv = netdev_priv(dev);
-
-	dev_lstats_read(dev, packets, bytes);
-	return atomic64_read(&priv->dropped);
-}
-
 static void veth_stats_rx(struct veth_stats *result, struct net_device *dev)
 {
 	struct veth_priv *priv = netdev_priv(dev);
@@ -432,24 +424,24 @@ static void veth_get_stats64(struct net_device *dev,
 	struct veth_priv *priv = netdev_priv(dev);
 	struct net_device *peer;
 	struct veth_stats rx;
-	u64 packets, bytes;
 
-	tot->tx_dropped = veth_stats_tx(dev, &packets, &bytes);
-	tot->tx_bytes = bytes;
-	tot->tx_packets = packets;
+	tot->tx_dropped = atomic64_read(&priv->dropped);
+	dev_fetch_sw_netstats(tot, dev->tstats);
 
 	veth_stats_rx(&rx, dev);
 	tot->tx_dropped += rx.xdp_tx_err;
 	tot->rx_dropped = rx.rx_drops + rx.peer_tq_xdp_xmit_err;
-	tot->rx_bytes = rx.xdp_bytes;
-	tot->rx_packets = rx.xdp_packets;
+	tot->rx_bytes += rx.xdp_bytes;
+	tot->rx_packets += rx.xdp_packets;
 
 	rcu_read_lock();
 	peer = rcu_dereference(priv->peer);
 	if (peer) {
-		veth_stats_tx(peer, &packets, &bytes);
-		tot->rx_bytes += bytes;
-		tot->rx_packets += packets;
+		struct rtnl_link_stats64 tot_peer = {};
+
+		dev_fetch_sw_netstats(&tot_peer, peer->tstats);
+		tot->rx_bytes += tot_peer.tx_bytes;
+		tot->rx_packets += tot_peer.tx_packets;
 
 		veth_stats_rx(&rx, peer);
 		tot->tx_dropped += rx.peer_tq_xdp_xmit_err;
@@ -1506,25 +1498,12 @@ static void veth_free_queues(struct net_device *dev)
 
 static int veth_dev_init(struct net_device *dev)
 {
-	int err;
-
-	dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats);
-	if (!dev->lstats)
-		return -ENOMEM;
-
-	err = veth_alloc_queues(dev);
-	if (err) {
-		free_percpu(dev->lstats);
-		return err;
-	}
-
-	return 0;
+	return veth_alloc_queues(dev);
 }
 
 static void veth_dev_free(struct net_device *dev)
 {
 	veth_free_queues(dev);
-	free_percpu(dev->lstats);
 }
 
 #ifdef CONFIG_NET_POLL_CONTROLLER
@@ -1796,6 +1775,7 @@ static void veth_setup(struct net_device *dev)
 			       NETIF_F_HW_VLAN_STAG_RX);
 	dev->needs_free_netdev = true;
 	dev->priv_destructor = veth_dev_free;
+	dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
 	dev->max_mtu = ETH_MAX_MTU;
 
 	dev->hw_features = VETH_FEATURES;
diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c
index db766941b78f..bb95ce43cd97 100644
--- a/drivers/net/vrf.c
+++ b/drivers/net/vrf.c
@@ -121,22 +121,12 @@ struct net_vrf {
 	int			ifindex;
 };
 
-struct pcpu_dstats {
-	u64			tx_pkts;
-	u64			tx_bytes;
-	u64			tx_drps;
-	u64			rx_pkts;
-	u64			rx_bytes;
-	u64			rx_drps;
-	struct u64_stats_sync	syncp;
-};
-
 static void vrf_rx_stats(struct net_device *dev, int len)
 {
 	struct pcpu_dstats *dstats = this_cpu_ptr(dev->dstats);
 
 	u64_stats_update_begin(&dstats->syncp);
-	dstats->rx_pkts++;
+	dstats->rx_packets++;
 	dstats->rx_bytes += len;
 	u64_stats_update_end(&dstats->syncp);
 }
@@ -161,10 +151,10 @@ static void vrf_get_stats64(struct net_device *dev,
 		do {
 			start = u64_stats_fetch_begin(&dstats->syncp);
 			tbytes = dstats->tx_bytes;
-			tpkts = dstats->tx_pkts;
-			tdrops = dstats->tx_drps;
+			tpkts = dstats->tx_packets;
+			tdrops = dstats->tx_drops;
 			rbytes = dstats->rx_bytes;
-			rpkts = dstats->rx_pkts;
+			rpkts = dstats->rx_packets;
 		} while (u64_stats_fetch_retry(&dstats->syncp, start));
 		stats->tx_bytes += tbytes;
 		stats->tx_packets += tpkts;
@@ -421,7 +411,7 @@ static int vrf_local_xmit(struct sk_buff *skb, struct net_device *dev,
 	if (likely(__netif_rx(skb) == NET_RX_SUCCESS))
 		vrf_rx_stats(dev, len);
 	else
-		this_cpu_inc(dev->dstats->rx_drps);
+		this_cpu_inc(dev->dstats->rx_drops);
 
 	return NETDEV_TX_OK;
 }
@@ -616,11 +606,11 @@ static netdev_tx_t vrf_xmit(struct sk_buff *skb, struct net_device *dev)
 		struct pcpu_dstats *dstats = this_cpu_ptr(dev->dstats);
 
 		u64_stats_update_begin(&dstats->syncp);
-		dstats->tx_pkts++;
+		dstats->tx_packets++;
 		dstats->tx_bytes += len;
 		u64_stats_update_end(&dstats->syncp);
 	} else {
-		this_cpu_inc(dev->dstats->tx_drps);
+		this_cpu_inc(dev->dstats->tx_drops);
 	}
 
 	return ret;
@@ -1174,22 +1164,15 @@ static void vrf_dev_uninit(struct net_device *dev)
 
 	vrf_rtable_release(dev, vrf);
 	vrf_rt6_release(dev, vrf);
-
-	free_percpu(dev->dstats);
-	dev->dstats = NULL;
 }
 
 static int vrf_dev_init(struct net_device *dev)
 {
 	struct net_vrf *vrf = netdev_priv(dev);
 
-	dev->dstats = netdev_alloc_pcpu_stats(struct pcpu_dstats);
-	if (!dev->dstats)
-		goto out_nomem;
-
 	/* create the default dst which points back to us */
 	if (vrf_rtable_create(dev) != 0)
-		goto out_stats;
+		goto out_nomem;
 
 	if (vrf_rt6_create(dev) != 0)
 		goto out_rth;
@@ -1203,9 +1186,6 @@ static int vrf_dev_init(struct net_device *dev)
 
 out_rth:
 	vrf_rtable_release(dev, vrf);
-out_stats:
-	free_percpu(dev->dstats);
-	dev->dstats = NULL;
 out_nomem:
 	return -ENOMEM;
 }
@@ -1704,6 +1684,8 @@ static void vrf_setup(struct net_device *dev)
 	dev->min_mtu = IPV6_MIN_MTU;
 	dev->max_mtu = IP6_MAX_MTU;
 	dev->mtu = dev->max_mtu;
+
+	dev->pcpu_stat_type = NETDEV_PCPU_STAT_DSTATS;
 }
 
 static int vrf_validate(struct nlattr *tb[], struct nlattr *data[],
diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c
index 412c3c0b6990..764ea02ff911 100644
--- a/drivers/net/vxlan/vxlan_core.c
+++ b/drivers/net/vxlan/vxlan_core.c
@@ -2379,7 +2379,17 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 		else
 			udp_sum = !(flags & VXLAN_F_UDP_ZERO_CSUM6_TX);
 #if IS_ENABLED(CONFIG_IPV6)
-		key.label = vxlan->cfg.label;
+		switch (vxlan->cfg.label_policy) {
+		case VXLAN_LABEL_FIXED:
+			key.label = vxlan->cfg.label;
+			break;
+		case VXLAN_LABEL_INHERIT:
+			key.label = ip_tunnel_get_flowlabel(old_iph, skb);
+			break;
+		default:
+			DEBUG_NET_WARN_ON_ONCE(1);
+			goto drop;
+		}
 #endif
 	} else {
 		if (!info) {
@@ -3366,6 +3376,7 @@ static const struct nla_policy vxlan_policy[IFLA_VXLAN_MAX + 1] = {
 	[IFLA_VXLAN_DF]		= { .type = NLA_U8 },
 	[IFLA_VXLAN_VNIFILTER]	= { .type = NLA_U8 },
 	[IFLA_VXLAN_LOCALBYPASS]	= NLA_POLICY_MAX(NLA_U8, 1),
+	[IFLA_VXLAN_LABEL_POLICY]       = NLA_POLICY_MAX(NLA_U32, VXLAN_LABEL_MAX),
 };
 
 static int vxlan_validate(struct nlattr *tb[], struct nlattr *data[],
@@ -3740,6 +3751,12 @@ static int vxlan_config_validate(struct net *src_net, struct vxlan_config *conf,
 		return -EINVAL;
 	}
 
+	if (conf->label_policy && !use_ipv6) {
+		NL_SET_ERR_MSG(extack,
+			       "Label policy only applies to IPv6 VXLAN devices");
+		return -EINVAL;
+	}
+
 	if (conf->remote_ifindex) {
 		struct net_device *lowerdev;
 
@@ -4082,6 +4099,8 @@ static int vxlan_nl2conf(struct nlattr *tb[], struct nlattr *data[],
 	if (data[IFLA_VXLAN_LABEL])
 		conf->label = nla_get_be32(data[IFLA_VXLAN_LABEL]) &
 			     IPV6_FLOWLABEL_MASK;
+	if (data[IFLA_VXLAN_LABEL_POLICY])
+		conf->label_policy = nla_get_u32(data[IFLA_VXLAN_LABEL_POLICY]);
 
 	if (data[IFLA_VXLAN_LEARNING]) {
 		err = vxlan_nl2flag(conf, data, IFLA_VXLAN_LEARNING,
@@ -4398,6 +4417,7 @@ static size_t vxlan_get_size(const struct net_device *dev)
 		nla_total_size(sizeof(__u8)) +	/* IFLA_VXLAN_TOS */
 		nla_total_size(sizeof(__u8)) +	/* IFLA_VXLAN_DF */
 		nla_total_size(sizeof(__be32)) + /* IFLA_VXLAN_LABEL */
+		nla_total_size(sizeof(__u32)) +  /* IFLA_VXLAN_LABEL_POLICY */
 		nla_total_size(sizeof(__u8)) +	/* IFLA_VXLAN_LEARNING */
 		nla_total_size(sizeof(__u8)) +	/* IFLA_VXLAN_PROXY */
 		nla_total_size(sizeof(__u8)) +	/* IFLA_VXLAN_RSC */
@@ -4471,6 +4491,7 @@ static int vxlan_fill_info(struct sk_buff *skb, const struct net_device *dev)
 	    nla_put_u8(skb, IFLA_VXLAN_TOS, vxlan->cfg.tos) ||
 	    nla_put_u8(skb, IFLA_VXLAN_DF, vxlan->cfg.df) ||
 	    nla_put_be32(skb, IFLA_VXLAN_LABEL, vxlan->cfg.label) ||
+	    nla_put_u32(skb, IFLA_VXLAN_LABEL_POLICY, vxlan->cfg.label_policy) ||
 	    nla_put_u8(skb, IFLA_VXLAN_LEARNING,
 		       !!(vxlan->cfg.flags & VXLAN_F_LEARN)) ||
 	    nla_put_u8(skb, IFLA_VXLAN_PROXY,
diff --git a/drivers/net/wireguard/device.c b/drivers/net/wireguard/device.c
index 258dcc103921..deb9636b0ecf 100644
--- a/drivers/net/wireguard/device.c
+++ b/drivers/net/wireguard/device.c
@@ -210,7 +210,7 @@ static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev)
 	 */
 	while (skb_queue_len(&peer->staged_packet_queue) > MAX_STAGED_PACKETS) {
 		dev_kfree_skb(__skb_dequeue(&peer->staged_packet_queue));
-		++dev->stats.tx_dropped;
+		DEV_STATS_INC(dev, tx_dropped);
 	}
 	skb_queue_splice_tail(&packets, &peer->staged_packet_queue);
 	spin_unlock_bh(&peer->staged_packet_queue.lock);
@@ -228,7 +228,7 @@ static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev)
 	else if (skb->protocol == htons(ETH_P_IPV6))
 		icmpv6_ndo_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0);
 err:
-	++dev->stats.tx_errors;
+	DEV_STATS_INC(dev, tx_errors);
 	kfree_skb(skb);
 	return ret;
 }
diff --git a/drivers/net/wireguard/receive.c b/drivers/net/wireguard/receive.c
index 0b3f0c843550..a176653c8861 100644
--- a/drivers/net/wireguard/receive.c
+++ b/drivers/net/wireguard/receive.c
@@ -416,20 +416,20 @@ static void wg_packet_consume_data_done(struct wg_peer *peer,
 	net_dbg_skb_ratelimited("%s: Packet has unallowed src IP (%pISc) from peer %llu (%pISpfsc)\n",
 				dev->name, skb, peer->internal_id,
 				&peer->endpoint.addr);
-	++dev->stats.rx_errors;
-	++dev->stats.rx_frame_errors;
+	DEV_STATS_INC(dev, rx_errors);
+	DEV_STATS_INC(dev, rx_frame_errors);
 	goto packet_processed;
 dishonest_packet_type:
 	net_dbg_ratelimited("%s: Packet is neither ipv4 nor ipv6 from peer %llu (%pISpfsc)\n",
 			    dev->name, peer->internal_id, &peer->endpoint.addr);
-	++dev->stats.rx_errors;
-	++dev->stats.rx_frame_errors;
+	DEV_STATS_INC(dev, rx_errors);
+	DEV_STATS_INC(dev, rx_frame_errors);
 	goto packet_processed;
 dishonest_packet_size:
 	net_dbg_ratelimited("%s: Packet has incorrect size from peer %llu (%pISpfsc)\n",
 			    dev->name, peer->internal_id, &peer->endpoint.addr);
-	++dev->stats.rx_errors;
-	++dev->stats.rx_length_errors;
+	DEV_STATS_INC(dev, rx_errors);
+	DEV_STATS_INC(dev, rx_length_errors);
 	goto packet_processed;
 packet_processed:
 	dev_kfree_skb(skb);
diff --git a/drivers/net/wireguard/send.c b/drivers/net/wireguard/send.c
index 95c853b59e1d..0d48e0f4a1ba 100644
--- a/drivers/net/wireguard/send.c
+++ b/drivers/net/wireguard/send.c
@@ -333,7 +333,8 @@ static void wg_packet_create_data(struct wg_peer *peer, struct sk_buff *first)
 void wg_packet_purge_staged_packets(struct wg_peer *peer)
 {
 	spin_lock_bh(&peer->staged_packet_queue.lock);
-	peer->device->dev->stats.tx_dropped += peer->staged_packet_queue.qlen;
+	DEV_STATS_ADD(peer->device->dev, tx_dropped,
+		      peer->staged_packet_queue.qlen);
 	__skb_queue_purge(&peer->staged_packet_queue);
 	spin_unlock_bh(&peer->staged_packet_queue.lock);
 }
diff --git a/drivers/net/wireless/Kconfig b/drivers/net/wireless/Kconfig
index 7555af5195ec..c6599594dc99 100644
--- a/drivers/net/wireless/Kconfig
+++ b/drivers/net/wireless/Kconfig
@@ -22,7 +22,6 @@ source "drivers/net/wireless/admtek/Kconfig"
 source "drivers/net/wireless/ath/Kconfig"
 source "drivers/net/wireless/atmel/Kconfig"
 source "drivers/net/wireless/broadcom/Kconfig"
-source "drivers/net/wireless/cisco/Kconfig"
 source "drivers/net/wireless/intel/Kconfig"
 source "drivers/net/wireless/intersil/Kconfig"
 source "drivers/net/wireless/marvell/Kconfig"
@@ -38,8 +37,6 @@ source "drivers/net/wireless/ti/Kconfig"
 source "drivers/net/wireless/zydas/Kconfig"
 source "drivers/net/wireless/quantenna/Kconfig"
 
-source "drivers/net/wireless/legacy/Kconfig"
-
 source "drivers/net/wireless/virtual/Kconfig"
 
 endif # WLAN
diff --git a/drivers/net/wireless/Makefile b/drivers/net/wireless/Makefile
index 4d7374d567d1..e1c4141c6004 100644
--- a/drivers/net/wireless/Makefile
+++ b/drivers/net/wireless/Makefile
@@ -7,7 +7,6 @@ obj-$(CONFIG_WLAN_VENDOR_ADMTEK) += admtek/
 obj-$(CONFIG_WLAN_VENDOR_ATH) += ath/
 obj-$(CONFIG_WLAN_VENDOR_ATMEL) += atmel/
 obj-$(CONFIG_WLAN_VENDOR_BROADCOM) += broadcom/
-obj-$(CONFIG_WLAN_VENDOR_CISCO) += cisco/
 obj-$(CONFIG_WLAN_VENDOR_INTEL) += intel/
 obj-$(CONFIG_WLAN_VENDOR_INTERSIL) += intersil/
 obj-$(CONFIG_WLAN_VENDOR_MARVELL) += marvell/
@@ -23,5 +22,4 @@ obj-$(CONFIG_WLAN_VENDOR_ST) += st/
 obj-$(CONFIG_WLAN_VENDOR_TI) += ti/
 obj-$(CONFIG_WLAN_VENDOR_ZYDAS) += zydas/
 
-obj-$(CONFIG_WLAN) += legacy/
 obj-$(CONFIG_WLAN) += virtual/
diff --git a/drivers/net/wireless/atmel/Kconfig b/drivers/net/wireless/atmel/Kconfig
index bafdd57b049a..7a2bb7a58ab7 100644
--- a/drivers/net/wireless/atmel/Kconfig
+++ b/drivers/net/wireless/atmel/Kconfig
@@ -12,41 +12,6 @@ config WLAN_VENDOR_ATMEL
 
 if WLAN_VENDOR_ATMEL
 
-config ATMEL
-	tristate "Atmel at76c50x chipset  802.11b support"
-	depends on CFG80211 && (PCI || PCMCIA) && HAS_IOPORT
-	select WIRELESS_EXT
-	select WEXT_PRIV
-	select FW_LOADER
-	select CRC32
-	help
-	  A driver 802.11b wireless cards based on the Atmel fast-vnet
-	  chips. This driver supports standard Linux wireless extensions.
-
-	  Many  cards based on this chipset do not have flash memory
-	  and need their firmware loaded at start-up. If yours is
-	  one of these, you will need to provide a firmware image
-	  to be loaded into the card by the driver. The Atmel
-	  firmware package can be downloaded from
-	  <http://www.thekelleys.org.uk/atmel>
-
-config PCI_ATMEL
-	tristate "Atmel at76c506 PCI cards"
-	depends on ATMEL && PCI
-	help
-	  Enable support for PCI and mini-PCI cards containing the
-	  Atmel at76c506 chip.
-
-config PCMCIA_ATMEL
-	tristate "Atmel at76c502/at76c504 PCMCIA cards"
-	depends on ATMEL && PCMCIA
-	select WIRELESS_EXT
-	select FW_LOADER
-	select CRC32
-	help
-	  Enable support for PCMCIA cards containing the
-	  Atmel at76c502 and at76c504 chips.
-
 config AT76C50X_USB
 	tristate "Atmel at76c503/at76c505/at76c505a USB cards"
 	depends on MAC80211 && USB
diff --git a/drivers/net/wireless/atmel/Makefile b/drivers/net/wireless/atmel/Makefile
index 17e62805677d..8338d7098ba6 100644
--- a/drivers/net/wireless/atmel/Makefile
+++ b/drivers/net/wireless/atmel/Makefile
@@ -1,6 +1,2 @@
 # SPDX-License-Identifier: GPL-2.0-only
-obj-$(CONFIG_ATMEL)             += atmel.o
-obj-$(CONFIG_PCI_ATMEL)         += atmel_pci.o 
-obj-$(CONFIG_PCMCIA_ATMEL)      += atmel_cs.o
-
 obj-$(CONFIG_AT76C50X_USB)      += at76c50x-usb.o
diff --git a/drivers/net/wireless/atmel/atmel.c b/drivers/net/wireless/atmel/atmel.c
deleted file mode 100644
index 461dce21de2b..000000000000
--- a/drivers/net/wireless/atmel/atmel.c
+++ /dev/null
@@ -1,4452 +0,0 @@
-/*** -*- linux-c -*- **********************************************************
-
-     Driver for Atmel at76c502 at76c504 and at76c506 wireless cards.
-
-	Copyright 2000-2001 ATMEL Corporation.
-	Copyright 2003-2004 Simon Kelley.
-
-    This code was developed from version 2.1.1 of the Atmel drivers,
-    released by Atmel corp. under the GPL in December 2002. It also
-    includes code from the Linux aironet drivers (C) Benjamin Reed,
-    and the Linux PCMCIA package, (C) David Hinds and the Linux wireless
-    extensions, (C) Jean Tourrilhes.
-
-    The firmware module for reading the MAC address of the card comes from
-    net.russotto.AtmelMACFW, written by Matthew T. Russotto and copyright
-    by him. net.russotto.AtmelMACFW is used under the GPL license version 2.
-    This file contains the module in binary form and, under the terms
-    of the GPL, in source form. The source is located at the end of the file.
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This software is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with Atmel wireless lan drivers; if not, see
-    <http://www.gnu.org/licenses/>.
-
-    For all queries about this code, please contact the current author,
-    Simon Kelley <simon@thekelleys.org.uk> and not Atmel Corporation.
-
-    Credit is due to HP UK and Cambridge Online Systems Ltd for supplying
-    hardware used during development of this driver.
-
-******************************************************************************/
-
-#include <linux/interrupt.h>
-
-#include <linux/kernel.h>
-#include <linux/ptrace.h>
-#include <linux/slab.h>
-#include <linux/string.h>
-#include <linux/timer.h>
-#include <asm/byteorder.h>
-#include <asm/io.h>
-#include <linux/uaccess.h>
-#include <linux/module.h>
-#include <linux/netdevice.h>
-#include <linux/etherdevice.h>
-#include <linux/skbuff.h>
-#include <linux/if_arp.h>
-#include <linux/ioport.h>
-#include <linux/fcntl.h>
-#include <linux/delay.h>
-#include <linux/wireless.h>
-#include <net/iw_handler.h>
-#include <linux/crc32.h>
-#include <linux/proc_fs.h>
-#include <linux/seq_file.h>
-#include <linux/device.h>
-#include <linux/moduleparam.h>
-#include <linux/firmware.h>
-#include <linux/jiffies.h>
-#include <net/cfg80211.h>
-#include "atmel.h"
-
-#define DRIVER_MAJOR 0
-#define DRIVER_MINOR 98
-
-MODULE_AUTHOR("Simon Kelley");
-MODULE_DESCRIPTION("Support for Atmel at76c50x 802.11 wireless ethernet cards.");
-MODULE_LICENSE("GPL");
-
-/* The name of the firmware file to be loaded
-   over-rides any automatic selection */
-static char *firmware = NULL;
-module_param(firmware, charp, 0);
-
-/* table of firmware file names */
-static struct {
-	AtmelFWType fw_type;
-	const char *fw_file;
-	const char *fw_file_ext;
-} fw_table[] = {
-	{ ATMEL_FW_TYPE_502,		"atmel_at76c502",	"bin" },
-	{ ATMEL_FW_TYPE_502D,		"atmel_at76c502d",	"bin" },
-	{ ATMEL_FW_TYPE_502E,		"atmel_at76c502e",	"bin" },
-	{ ATMEL_FW_TYPE_502_3COM,	"atmel_at76c502_3com",	"bin" },
-	{ ATMEL_FW_TYPE_504,		"atmel_at76c504",	"bin" },
-	{ ATMEL_FW_TYPE_504_2958,	"atmel_at76c504_2958",	"bin" },
-	{ ATMEL_FW_TYPE_504A_2958,	"atmel_at76c504a_2958",	"bin" },
-	{ ATMEL_FW_TYPE_506,		"atmel_at76c506",	"bin" },
-	{ ATMEL_FW_TYPE_NONE,		NULL,			NULL }
-};
-MODULE_FIRMWARE("atmel_at76c502-wpa.bin");
-MODULE_FIRMWARE("atmel_at76c502.bin");
-MODULE_FIRMWARE("atmel_at76c502d-wpa.bin");
-MODULE_FIRMWARE("atmel_at76c502d.bin");
-MODULE_FIRMWARE("atmel_at76c502e-wpa.bin");
-MODULE_FIRMWARE("atmel_at76c502e.bin");
-MODULE_FIRMWARE("atmel_at76c502_3com-wpa.bin");
-MODULE_FIRMWARE("atmel_at76c502_3com.bin");
-MODULE_FIRMWARE("atmel_at76c504-wpa.bin");
-MODULE_FIRMWARE("atmel_at76c504.bin");
-MODULE_FIRMWARE("atmel_at76c504_2958-wpa.bin");
-MODULE_FIRMWARE("atmel_at76c504_2958.bin");
-MODULE_FIRMWARE("atmel_at76c504a_2958-wpa.bin");
-MODULE_FIRMWARE("atmel_at76c504a_2958.bin");
-MODULE_FIRMWARE("atmel_at76c506-wpa.bin");
-MODULE_FIRMWARE("atmel_at76c506.bin");
-
-#define MAX_SSID_LENGTH 32
-#define MGMT_JIFFIES (256 * HZ / 100)
-
-#define MAX_BSS_ENTRIES	64
-
-/* registers */
-#define GCR  0x00    /* (SIR0)  General Configuration Register */
-#define BSR  0x02    /* (SIR1)  Bank Switching Select Register */
-#define AR   0x04
-#define DR   0x08
-#define MR1  0x12    /* Mirror Register 1 */
-#define MR2  0x14    /* Mirror Register 2 */
-#define MR3  0x16    /* Mirror Register 3 */
-#define MR4  0x18    /* Mirror Register 4 */
-
-#define GPR1                            0x0c
-#define GPR2                            0x0e
-#define GPR3                            0x10
-/*
- * Constants for the GCR register.
- */
-#define GCR_REMAP     0x0400          /* Remap internal SRAM to 0 */
-#define GCR_SWRES     0x0080          /* BIU reset (ARM and PAI are NOT reset) */
-#define GCR_CORES     0x0060          /* Core Reset (ARM and PAI are reset) */
-#define GCR_ENINT     0x0002          /* Enable Interrupts */
-#define GCR_ACKINT    0x0008          /* Acknowledge Interrupts */
-
-#define BSS_SRAM      0x0200          /* AMBA module selection --> SRAM */
-#define BSS_IRAM      0x0100          /* AMBA module selection --> IRAM */
-/*
- *Constants for the MR registers.
- */
-#define MAC_INIT_COMPLETE       0x0001        /* MAC init has been completed */
-#define MAC_BOOT_COMPLETE       0x0010        /* MAC boot has been completed */
-#define MAC_INIT_OK             0x0002        /* MAC boot has been completed */
-
-#define MIB_MAX_DATA_BYTES    212
-#define MIB_HEADER_SIZE       4    /* first four fields */
-
-struct get_set_mib {
-	u8 type;
-	u8 size;
-	u8 index;
-	u8 reserved;
-	u8 data[MIB_MAX_DATA_BYTES];
-};
-
-struct rx_desc {
-	u32          Next;
-	u16          MsduPos;
-	u16          MsduSize;
-
-	u8           State;
-	u8           Status;
-	u8           Rate;
-	u8           Rssi;
-	u8           LinkQuality;
-	u8           PreambleType;
-	u16          Duration;
-	u32          RxTime;
-};
-
-#define RX_DESC_FLAG_VALID       0x80
-#define RX_DESC_FLAG_CONSUMED    0x40
-#define RX_DESC_FLAG_IDLE        0x00
-
-#define RX_STATUS_SUCCESS        0x00
-
-#define RX_DESC_MSDU_POS_OFFSET      4
-#define RX_DESC_MSDU_SIZE_OFFSET     6
-#define RX_DESC_FLAGS_OFFSET         8
-#define RX_DESC_STATUS_OFFSET        9
-#define RX_DESC_RSSI_OFFSET          11
-#define RX_DESC_LINK_QUALITY_OFFSET  12
-#define RX_DESC_PREAMBLE_TYPE_OFFSET 13
-#define RX_DESC_DURATION_OFFSET      14
-#define RX_DESC_RX_TIME_OFFSET       16
-
-struct tx_desc {
-	u32       NextDescriptor;
-	u16       TxStartOfFrame;
-	u16       TxLength;
-
-	u8        TxState;
-	u8        TxStatus;
-	u8        RetryCount;
-
-	u8        TxRate;
-
-	u8        KeyIndex;
-	u8        ChiperType;
-	u8        ChipreLength;
-	u8        Reserved1;
-
-	u8        Reserved;
-	u8        PacketType;
-	u16       HostTxLength;
-};
-
-#define TX_DESC_NEXT_OFFSET          0
-#define TX_DESC_POS_OFFSET           4
-#define TX_DESC_SIZE_OFFSET          6
-#define TX_DESC_FLAGS_OFFSET         8
-#define TX_DESC_STATUS_OFFSET        9
-#define TX_DESC_RETRY_OFFSET         10
-#define TX_DESC_RATE_OFFSET          11
-#define TX_DESC_KEY_INDEX_OFFSET     12
-#define TX_DESC_CIPHER_TYPE_OFFSET   13
-#define TX_DESC_CIPHER_LENGTH_OFFSET 14
-#define TX_DESC_PACKET_TYPE_OFFSET   17
-#define TX_DESC_HOST_LENGTH_OFFSET   18
-
-/*
- * Host-MAC interface
- */
-
-#define TX_STATUS_SUCCESS       0x00
-
-#define TX_FIRM_OWN             0x80
-#define TX_DONE                 0x40
-
-#define TX_ERROR                0x01
-
-#define TX_PACKET_TYPE_DATA     0x01
-#define TX_PACKET_TYPE_MGMT     0x02
-
-#define ISR_EMPTY               0x00        /* no bits set in ISR */
-#define ISR_TxCOMPLETE          0x01        /* packet transmitted */
-#define ISR_RxCOMPLETE          0x02        /* packet received */
-#define ISR_RxFRAMELOST         0x04        /* Rx Frame lost */
-#define ISR_FATAL_ERROR         0x08        /* Fatal error */
-#define ISR_COMMAND_COMPLETE    0x10        /* command completed */
-#define ISR_OUT_OF_RANGE        0x20        /* command completed */
-#define ISR_IBSS_MERGE          0x40        /* (4.1.2.30): IBSS merge */
-#define ISR_GENERIC_IRQ         0x80
-
-#define Local_Mib_Type          0x01
-#define Mac_Address_Mib_Type    0x02
-#define Mac_Mib_Type            0x03
-#define Statistics_Mib_Type     0x04
-#define Mac_Mgmt_Mib_Type       0x05
-#define Mac_Wep_Mib_Type        0x06
-#define Phy_Mib_Type            0x07
-#define Multi_Domain_MIB        0x08
-
-#define MAC_MGMT_MIB_CUR_BSSID_POS            14
-#define MAC_MIB_FRAG_THRESHOLD_POS            8
-#define MAC_MIB_RTS_THRESHOLD_POS             10
-#define MAC_MIB_SHORT_RETRY_POS               16
-#define MAC_MIB_LONG_RETRY_POS                17
-#define MAC_MIB_SHORT_RETRY_LIMIT_POS         16
-#define MAC_MGMT_MIB_BEACON_PER_POS           0
-#define MAC_MGMT_MIB_STATION_ID_POS           6
-#define MAC_MGMT_MIB_CUR_PRIVACY_POS          11
-#define MAC_MGMT_MIB_CUR_BSSID_POS            14
-#define MAC_MGMT_MIB_PS_MODE_POS              53
-#define MAC_MGMT_MIB_LISTEN_INTERVAL_POS      54
-#define MAC_MGMT_MIB_MULTI_DOMAIN_IMPLEMENTED 56
-#define MAC_MGMT_MIB_MULTI_DOMAIN_ENABLED     57
-#define PHY_MIB_CHANNEL_POS                   14
-#define PHY_MIB_RATE_SET_POS                  20
-#define PHY_MIB_REG_DOMAIN_POS                26
-#define LOCAL_MIB_AUTO_TX_RATE_POS            3
-#define LOCAL_MIB_SSID_SIZE                   5
-#define LOCAL_MIB_TX_PROMISCUOUS_POS          6
-#define LOCAL_MIB_TX_MGMT_RATE_POS            7
-#define LOCAL_MIB_TX_CONTROL_RATE_POS         8
-#define LOCAL_MIB_PREAMBLE_TYPE               9
-#define MAC_ADDR_MIB_MAC_ADDR_POS             0
-
-#define         CMD_Set_MIB_Vars              0x01
-#define         CMD_Get_MIB_Vars              0x02
-#define         CMD_Scan                      0x03
-#define         CMD_Join                      0x04
-#define         CMD_Start                     0x05
-#define         CMD_EnableRadio               0x06
-#define         CMD_DisableRadio              0x07
-#define         CMD_SiteSurvey                0x0B
-
-#define         CMD_STATUS_IDLE                   0x00
-#define         CMD_STATUS_COMPLETE               0x01
-#define         CMD_STATUS_UNKNOWN                0x02
-#define         CMD_STATUS_INVALID_PARAMETER      0x03
-#define         CMD_STATUS_FUNCTION_NOT_SUPPORTED 0x04
-#define         CMD_STATUS_TIME_OUT               0x07
-#define         CMD_STATUS_IN_PROGRESS            0x08
-#define         CMD_STATUS_REJECTED_RADIO_OFF     0x09
-#define         CMD_STATUS_HOST_ERROR             0xFF
-#define         CMD_STATUS_BUSY                   0xFE
-
-#define CMD_BLOCK_COMMAND_OFFSET        0
-#define CMD_BLOCK_STATUS_OFFSET         1
-#define CMD_BLOCK_PARAMETERS_OFFSET     4
-
-#define SCAN_OPTIONS_SITE_SURVEY        0x80
-
-#define MGMT_FRAME_BODY_OFFSET		24
-#define MAX_AUTHENTICATION_RETRIES	3
-#define MAX_ASSOCIATION_RETRIES		3
-
-#define AUTHENTICATION_RESPONSE_TIME_OUT  1000
-
-#define MAX_WIRELESS_BODY  2316 /* mtu is 2312, CRC is 4 */
-#define LOOP_RETRY_LIMIT   500000
-
-#define ACTIVE_MODE	1
-#define PS_MODE		2
-
-#define MAX_ENCRYPTION_KEYS 4
-#define MAX_ENCRYPTION_KEY_SIZE 40
-
-/*
- * 802.11 related definitions
- */
-
-/*
- * Regulatory Domains
- */
-
-#define REG_DOMAIN_FCC		0x10	/* Channels	1-11	USA				*/
-#define REG_DOMAIN_DOC		0x20	/* Channel	1-11	Canada				*/
-#define REG_DOMAIN_ETSI		0x30	/* Channel	1-13	Europe (ex Spain/France)	*/
-#define REG_DOMAIN_SPAIN	0x31	/* Channel	10-11	Spain				*/
-#define REG_DOMAIN_FRANCE	0x32	/* Channel	10-13	France				*/
-#define REG_DOMAIN_MKK		0x40	/* Channel	14	Japan				*/
-#define REG_DOMAIN_MKK1		0x41	/* Channel	1-14	Japan(MKK1)			*/
-#define REG_DOMAIN_ISRAEL	0x50	/* Channel	3-9	ISRAEL				*/
-
-#define BSS_TYPE_AD_HOC		1
-#define BSS_TYPE_INFRASTRUCTURE 2
-
-#define SCAN_TYPE_ACTIVE	0
-#define SCAN_TYPE_PASSIVE	1
-
-#define LONG_PREAMBLE		0
-#define SHORT_PREAMBLE		1
-#define AUTO_PREAMBLE		2
-
-#define DATA_FRAME_WS_HEADER_SIZE   30
-
-/* promiscuous mode control */
-#define PROM_MODE_OFF			0x0
-#define PROM_MODE_UNKNOWN		0x1
-#define PROM_MODE_CRC_FAILED		0x2
-#define PROM_MODE_DUPLICATED		0x4
-#define PROM_MODE_MGMT			0x8
-#define PROM_MODE_CTRL			0x10
-#define PROM_MODE_BAD_PROTOCOL		0x20
-
-#define IFACE_INT_STATUS_OFFSET		0
-#define IFACE_INT_MASK_OFFSET		1
-#define IFACE_LOCKOUT_HOST_OFFSET	2
-#define IFACE_LOCKOUT_MAC_OFFSET	3
-#define IFACE_FUNC_CTRL_OFFSET		28
-#define IFACE_MAC_STAT_OFFSET		30
-#define IFACE_GENERIC_INT_TYPE_OFFSET	32
-
-#define CIPHER_SUITE_NONE     0
-#define CIPHER_SUITE_WEP_64   1
-#define CIPHER_SUITE_TKIP     2
-#define CIPHER_SUITE_AES      3
-#define CIPHER_SUITE_CCX      4
-#define CIPHER_SUITE_WEP_128  5
-
-/*
- * IFACE MACROS & definitions
- */
-
-/*
- * FuncCtrl field:
- */
-#define FUNC_CTRL_TxENABLE		0x10
-#define FUNC_CTRL_RxENABLE		0x20
-#define FUNC_CTRL_INIT_COMPLETE		0x01
-
-/* A stub firmware image which reads the MAC address from NVRAM on the card.
-   For copyright information and source see the end of this file. */
-static u8 mac_reader[] = {
-	0x06, 0x00, 0x00, 0xea, 0x04, 0x00, 0x00, 0xea, 0x03, 0x00, 0x00, 0xea, 0x02, 0x00, 0x00, 0xea,
-	0x01, 0x00, 0x00, 0xea, 0x00, 0x00, 0x00, 0xea, 0xff, 0xff, 0xff, 0xea, 0xfe, 0xff, 0xff, 0xea,
-	0xd3, 0x00, 0xa0, 0xe3, 0x00, 0xf0, 0x21, 0xe1, 0x0e, 0x04, 0xa0, 0xe3, 0x00, 0x10, 0xa0, 0xe3,
-	0x81, 0x11, 0xa0, 0xe1, 0x00, 0x10, 0x81, 0xe3, 0x00, 0x10, 0x80, 0xe5, 0x1c, 0x10, 0x90, 0xe5,
-	0x10, 0x10, 0xc1, 0xe3, 0x1c, 0x10, 0x80, 0xe5, 0x01, 0x10, 0xa0, 0xe3, 0x08, 0x10, 0x80, 0xe5,
-	0x02, 0x03, 0xa0, 0xe3, 0x00, 0x10, 0xa0, 0xe3, 0xb0, 0x10, 0xc0, 0xe1, 0xb4, 0x10, 0xc0, 0xe1,
-	0xb8, 0x10, 0xc0, 0xe1, 0xbc, 0x10, 0xc0, 0xe1, 0x56, 0xdc, 0xa0, 0xe3, 0x21, 0x00, 0x00, 0xeb,
-	0x0a, 0x00, 0xa0, 0xe3, 0x1a, 0x00, 0x00, 0xeb, 0x10, 0x00, 0x00, 0xeb, 0x07, 0x00, 0x00, 0xeb,
-	0x02, 0x03, 0xa0, 0xe3, 0x02, 0x14, 0xa0, 0xe3, 0xb4, 0x10, 0xc0, 0xe1, 0x4c, 0x10, 0x9f, 0xe5,
-	0xbc, 0x10, 0xc0, 0xe1, 0x10, 0x10, 0xa0, 0xe3, 0xb8, 0x10, 0xc0, 0xe1, 0xfe, 0xff, 0xff, 0xea,
-	0x00, 0x40, 0x2d, 0xe9, 0x00, 0x20, 0xa0, 0xe3, 0x02, 0x3c, 0xa0, 0xe3, 0x00, 0x10, 0xa0, 0xe3,
-	0x28, 0x00, 0x9f, 0xe5, 0x37, 0x00, 0x00, 0xeb, 0x00, 0x40, 0xbd, 0xe8, 0x1e, 0xff, 0x2f, 0xe1,
-	0x00, 0x40, 0x2d, 0xe9, 0x12, 0x2e, 0xa0, 0xe3, 0x06, 0x30, 0xa0, 0xe3, 0x00, 0x10, 0xa0, 0xe3,
-	0x02, 0x04, 0xa0, 0xe3, 0x2f, 0x00, 0x00, 0xeb, 0x00, 0x40, 0xbd, 0xe8, 0x1e, 0xff, 0x2f, 0xe1,
-	0x00, 0x02, 0x00, 0x02, 0x80, 0x01, 0x90, 0xe0, 0x01, 0x00, 0x00, 0x0a, 0x01, 0x00, 0x50, 0xe2,
-	0xfc, 0xff, 0xff, 0xea, 0x1e, 0xff, 0x2f, 0xe1, 0x80, 0x10, 0xa0, 0xe3, 0xf3, 0x06, 0xa0, 0xe3,
-	0x00, 0x10, 0x80, 0xe5, 0x00, 0x10, 0xa0, 0xe3, 0x00, 0x10, 0x80, 0xe5, 0x01, 0x10, 0xa0, 0xe3,
-	0x04, 0x10, 0x80, 0xe5, 0x00, 0x10, 0x80, 0xe5, 0x0e, 0x34, 0xa0, 0xe3, 0x1c, 0x10, 0x93, 0xe5,
-	0x02, 0x1a, 0x81, 0xe3, 0x1c, 0x10, 0x83, 0xe5, 0x58, 0x11, 0x9f, 0xe5, 0x30, 0x10, 0x80, 0xe5,
-	0x54, 0x11, 0x9f, 0xe5, 0x34, 0x10, 0x80, 0xe5, 0x38, 0x10, 0x80, 0xe5, 0x3c, 0x10, 0x80, 0xe5,
-	0x10, 0x10, 0x90, 0xe5, 0x08, 0x00, 0x90, 0xe5, 0x1e, 0xff, 0x2f, 0xe1, 0xf3, 0x16, 0xa0, 0xe3,
-	0x08, 0x00, 0x91, 0xe5, 0x05, 0x00, 0xa0, 0xe3, 0x0c, 0x00, 0x81, 0xe5, 0x10, 0x00, 0x91, 0xe5,
-	0x02, 0x00, 0x10, 0xe3, 0xfc, 0xff, 0xff, 0x0a, 0xff, 0x00, 0xa0, 0xe3, 0x0c, 0x00, 0x81, 0xe5,
-	0x10, 0x00, 0x91, 0xe5, 0x02, 0x00, 0x10, 0xe3, 0xfc, 0xff, 0xff, 0x0a, 0x08, 0x00, 0x91, 0xe5,
-	0x10, 0x00, 0x91, 0xe5, 0x01, 0x00, 0x10, 0xe3, 0xfc, 0xff, 0xff, 0x0a, 0x08, 0x00, 0x91, 0xe5,
-	0xff, 0x00, 0x00, 0xe2, 0x1e, 0xff, 0x2f, 0xe1, 0x30, 0x40, 0x2d, 0xe9, 0x00, 0x50, 0xa0, 0xe1,
-	0x03, 0x40, 0xa0, 0xe1, 0xa2, 0x02, 0xa0, 0xe1, 0x08, 0x00, 0x00, 0xe2, 0x03, 0x00, 0x80, 0xe2,
-	0xd8, 0x10, 0x9f, 0xe5, 0x00, 0x00, 0xc1, 0xe5, 0x01, 0x20, 0xc1, 0xe5, 0xe2, 0xff, 0xff, 0xeb,
-	0x01, 0x00, 0x10, 0xe3, 0xfc, 0xff, 0xff, 0x1a, 0x14, 0x00, 0xa0, 0xe3, 0xc4, 0xff, 0xff, 0xeb,
-	0x04, 0x20, 0xa0, 0xe1, 0x05, 0x10, 0xa0, 0xe1, 0x02, 0x00, 0xa0, 0xe3, 0x01, 0x00, 0x00, 0xeb,
-	0x30, 0x40, 0xbd, 0xe8, 0x1e, 0xff, 0x2f, 0xe1, 0x70, 0x40, 0x2d, 0xe9, 0xf3, 0x46, 0xa0, 0xe3,
-	0x00, 0x30, 0xa0, 0xe3, 0x00, 0x00, 0x50, 0xe3, 0x08, 0x00, 0x00, 0x9a, 0x8c, 0x50, 0x9f, 0xe5,
-	0x03, 0x60, 0xd5, 0xe7, 0x0c, 0x60, 0x84, 0xe5, 0x10, 0x60, 0x94, 0xe5, 0x02, 0x00, 0x16, 0xe3,
-	0xfc, 0xff, 0xff, 0x0a, 0x01, 0x30, 0x83, 0xe2, 0x00, 0x00, 0x53, 0xe1, 0xf7, 0xff, 0xff, 0x3a,
-	0xff, 0x30, 0xa0, 0xe3, 0x0c, 0x30, 0x84, 0xe5, 0x08, 0x00, 0x94, 0xe5, 0x10, 0x00, 0x94, 0xe5,
-	0x01, 0x00, 0x10, 0xe3, 0xfc, 0xff, 0xff, 0x0a, 0x08, 0x00, 0x94, 0xe5, 0x00, 0x00, 0xa0, 0xe3,
-	0x00, 0x00, 0x52, 0xe3, 0x0b, 0x00, 0x00, 0x9a, 0x10, 0x50, 0x94, 0xe5, 0x02, 0x00, 0x15, 0xe3,
-	0xfc, 0xff, 0xff, 0x0a, 0x0c, 0x30, 0x84, 0xe5, 0x10, 0x50, 0x94, 0xe5, 0x01, 0x00, 0x15, 0xe3,
-	0xfc, 0xff, 0xff, 0x0a, 0x08, 0x50, 0x94, 0xe5, 0x01, 0x50, 0xc1, 0xe4, 0x01, 0x00, 0x80, 0xe2,
-	0x02, 0x00, 0x50, 0xe1, 0xf3, 0xff, 0xff, 0x3a, 0xc8, 0x00, 0xa0, 0xe3, 0x98, 0xff, 0xff, 0xeb,
-	0x70, 0x40, 0xbd, 0xe8, 0x1e, 0xff, 0x2f, 0xe1, 0x01, 0x0c, 0x00, 0x02, 0x01, 0x02, 0x00, 0x02,
-	0x00, 0x01, 0x00, 0x02
-};
-
-struct atmel_private {
-	void *card; /* Bus dependent structure varies for PCcard */
-	int (*present_callback)(void *); /* And callback which uses it */
-	char firmware_id[32];
-	AtmelFWType firmware_type;
-	u8 *firmware;
-	int firmware_length;
-	struct timer_list management_timer;
-	struct net_device *dev;
-	struct device *sys_dev;
-	struct iw_statistics wstats;
-	spinlock_t irqlock, timerlock;	/* spinlocks */
-	enum { BUS_TYPE_PCCARD, BUS_TYPE_PCI } bus_type;
-	enum {
-		CARD_TYPE_PARALLEL_FLASH,
-		CARD_TYPE_SPI_FLASH,
-		CARD_TYPE_EEPROM
-	} card_type;
-	int do_rx_crc; /* If we need to CRC incoming packets */
-	int probe_crc; /* set if we don't yet know */
-	int crc_ok_cnt, crc_ko_cnt; /* counters for probing */
-	u16 rx_desc_head;
-	u16 tx_desc_free, tx_desc_head, tx_desc_tail, tx_desc_previous;
-	u16 tx_free_mem, tx_buff_head, tx_buff_tail;
-
-	u16 frag_seq, frag_len, frag_no;
-	u8 frag_source[6];
-
-	u8 wep_is_on, default_key, exclude_unencrypted, encryption_level;
-	u8 group_cipher_suite, pairwise_cipher_suite;
-	u8 wep_keys[MAX_ENCRYPTION_KEYS][MAX_ENCRYPTION_KEY_SIZE];
-	int wep_key_len[MAX_ENCRYPTION_KEYS];
-	int use_wpa, radio_on_broken; /* firmware dependent stuff. */
-
-	u16 host_info_base;
-	struct host_info_struct {
-		/* NB this is matched to the hardware, don't change. */
-		u8 volatile int_status;
-		u8 volatile int_mask;
-		u8 volatile lockout_host;
-		u8 volatile lockout_mac;
-
-		u16 tx_buff_pos;
-		u16 tx_buff_size;
-		u16 tx_desc_pos;
-		u16 tx_desc_count;
-
-		u16 rx_buff_pos;
-		u16 rx_buff_size;
-		u16 rx_desc_pos;
-		u16 rx_desc_count;
-
-		u16 build_version;
-		u16 command_pos;
-
-		u16 major_version;
-		u16 minor_version;
-
-		u16 func_ctrl;
-		u16 mac_status;
-		u16 generic_IRQ_type;
-		u8  reserved[2];
-	} host_info;
-
-	enum {
-		STATION_STATE_SCANNING,
-		STATION_STATE_JOINNING,
-		STATION_STATE_AUTHENTICATING,
-		STATION_STATE_ASSOCIATING,
-		STATION_STATE_READY,
-		STATION_STATE_REASSOCIATING,
-		STATION_STATE_DOWN,
-		STATION_STATE_MGMT_ERROR
-	} station_state;
-
-	int operating_mode, power_mode;
-	unsigned long last_qual;
-	int beacons_this_sec;
-	int channel;
-	int reg_domain, config_reg_domain;
-	int tx_rate;
-	int auto_tx_rate;
-	int rts_threshold;
-	int frag_threshold;
-	int long_retry, short_retry;
-	int preamble;
-	int default_beacon_period, beacon_period, listen_interval;
-	int CurrentAuthentTransactionSeqNum, ExpectedAuthentTransactionSeqNum;
-	int AuthenticationRequestRetryCnt, AssociationRequestRetryCnt, ReAssociationRequestRetryCnt;
-	enum {
-		SITE_SURVEY_IDLE,
-		SITE_SURVEY_IN_PROGRESS,
-		SITE_SURVEY_COMPLETED
-	} site_survey_state;
-	unsigned long last_survey;
-
-	int station_was_associated, station_is_associated;
-	int fast_scan;
-
-	struct bss_info {
-		int channel;
-		int SSIDsize;
-		int RSSI;
-		int UsingWEP;
-		int preamble;
-		int beacon_period;
-		int BSStype;
-		u8 BSSID[6];
-		u8 SSID[MAX_SSID_LENGTH];
-	} BSSinfo[MAX_BSS_ENTRIES];
-	int BSS_list_entries, current_BSS;
-	int connect_to_any_BSS;
-	int SSID_size, new_SSID_size;
-	u8 CurrentBSSID[6], BSSID[6];
-	u8 SSID[MAX_SSID_LENGTH], new_SSID[MAX_SSID_LENGTH];
-	u64 last_beacon_timestamp;
-	u8 rx_buf[MAX_WIRELESS_BODY];
-};
-
-static u8 atmel_basic_rates[4] = {0x82, 0x84, 0x0b, 0x16};
-
-static const struct {
-	int reg_domain;
-	int min, max;
-	char *name;
-} channel_table[] = { { REG_DOMAIN_FCC, 1, 11, "USA" },
-		      { REG_DOMAIN_DOC, 1, 11, "Canada" },
-		      { REG_DOMAIN_ETSI, 1, 13, "Europe" },
-		      { REG_DOMAIN_SPAIN, 10, 11, "Spain" },
-		      { REG_DOMAIN_FRANCE, 10, 13, "France" },
-		      { REG_DOMAIN_MKK, 14, 14, "MKK" },
-		      { REG_DOMAIN_MKK1, 1, 14, "MKK1" },
-		      { REG_DOMAIN_ISRAEL, 3, 9, "Israel"} };
-
-static void build_wpa_mib(struct atmel_private *priv);
-static void atmel_copy_to_card(struct net_device *dev, u16 dest,
-			       const unsigned char *src, u16 len);
-static void atmel_copy_to_host(struct net_device *dev, unsigned char *dest,
-			       u16 src, u16 len);
-static void atmel_set_gcr(struct net_device *dev, u16 mask);
-static void atmel_clear_gcr(struct net_device *dev, u16 mask);
-static int atmel_lock_mac(struct atmel_private *priv);
-static void atmel_wmem32(struct atmel_private *priv, u16 pos, u32 data);
-static void atmel_command_irq(struct atmel_private *priv);
-static int atmel_validate_channel(struct atmel_private *priv, int channel);
-static void atmel_management_frame(struct atmel_private *priv,
-				   struct ieee80211_hdr *header,
-				   u16 frame_len, u8 rssi);
-static void atmel_management_timer(struct timer_list *t);
-static void atmel_send_command(struct atmel_private *priv, int command,
-			       void *cmd, int cmd_size);
-static int atmel_send_command_wait(struct atmel_private *priv, int command,
-				   void *cmd, int cmd_size);
-static void atmel_transmit_management_frame(struct atmel_private *priv,
-					    struct ieee80211_hdr *header,
-					    u8 *body, int body_len);
-
-static u8 atmel_get_mib8(struct atmel_private *priv, u8 type, u8 index);
-static void atmel_set_mib8(struct atmel_private *priv, u8 type, u8 index,
-			   u8 data);
-static void atmel_set_mib16(struct atmel_private *priv, u8 type, u8 index,
-			    u16 data);
-static void atmel_set_mib(struct atmel_private *priv, u8 type, u8 index,
-			  const u8 *data, int data_len);
-static void atmel_get_mib(struct atmel_private *priv, u8 type, u8 index,
-			  u8 *data, int data_len);
-static void atmel_scan(struct atmel_private *priv, int specific_ssid);
-static void atmel_join_bss(struct atmel_private *priv, int bss_index);
-static void atmel_smooth_qual(struct atmel_private *priv);
-static void atmel_writeAR(struct net_device *dev, u16 data);
-static int probe_atmel_card(struct net_device *dev);
-static int reset_atmel_card(struct net_device *dev);
-static void atmel_enter_state(struct atmel_private *priv, int new_state);
-int atmel_open (struct net_device *dev);
-
-static inline u16 atmel_hi(struct atmel_private *priv, u16 offset)
-{
-	return priv->host_info_base + offset;
-}
-
-static inline u16 atmel_co(struct atmel_private *priv, u16 offset)
-{
-	return priv->host_info.command_pos + offset;
-}
-
-static inline u16 atmel_rx(struct atmel_private *priv, u16 offset, u16 desc)
-{
-	return priv->host_info.rx_desc_pos + (sizeof(struct rx_desc) * desc) + offset;
-}
-
-static inline u16 atmel_tx(struct atmel_private *priv, u16 offset, u16 desc)
-{
-	return priv->host_info.tx_desc_pos + (sizeof(struct tx_desc) * desc) + offset;
-}
-
-static inline u8 atmel_read8(struct net_device *dev, u16 offset)
-{
-	return inb(dev->base_addr + offset);
-}
-
-static inline void atmel_write8(struct net_device *dev, u16 offset, u8 data)
-{
-	outb(data, dev->base_addr + offset);
-}
-
-static inline u16 atmel_read16(struct net_device *dev, u16 offset)
-{
-	return inw(dev->base_addr + offset);
-}
-
-static inline void atmel_write16(struct net_device *dev, u16 offset, u16 data)
-{
-	outw(data, dev->base_addr + offset);
-}
-
-static inline u8 atmel_rmem8(struct atmel_private *priv, u16 pos)
-{
-	atmel_writeAR(priv->dev, pos);
-	return atmel_read8(priv->dev, DR);
-}
-
-static inline void atmel_wmem8(struct atmel_private *priv, u16 pos, u16 data)
-{
-	atmel_writeAR(priv->dev, pos);
-	atmel_write8(priv->dev, DR, data);
-}
-
-static inline u16 atmel_rmem16(struct atmel_private *priv, u16 pos)
-{
-	atmel_writeAR(priv->dev, pos);
-	return atmel_read16(priv->dev, DR);
-}
-
-static inline void atmel_wmem16(struct atmel_private *priv, u16 pos, u16 data)
-{
-	atmel_writeAR(priv->dev, pos);
-	atmel_write16(priv->dev, DR, data);
-}
-
-static const struct iw_handler_def atmel_handler_def;
-
-static void tx_done_irq(struct atmel_private *priv)
-{
-	int i;
-
-	for (i = 0;
-	     atmel_rmem8(priv, atmel_tx(priv, TX_DESC_FLAGS_OFFSET, priv->tx_desc_head)) == TX_DONE &&
-		     i < priv->host_info.tx_desc_count;
-	     i++) {
-		u8 status = atmel_rmem8(priv, atmel_tx(priv, TX_DESC_STATUS_OFFSET, priv->tx_desc_head));
-		u16 msdu_size = atmel_rmem16(priv, atmel_tx(priv, TX_DESC_SIZE_OFFSET, priv->tx_desc_head));
-		u8 type = atmel_rmem8(priv, atmel_tx(priv, TX_DESC_PACKET_TYPE_OFFSET, priv->tx_desc_head));
-
-		atmel_wmem8(priv, atmel_tx(priv, TX_DESC_FLAGS_OFFSET, priv->tx_desc_head), 0);
-
-		priv->tx_free_mem += msdu_size;
-		priv->tx_desc_free++;
-
-		if (priv->tx_buff_head + msdu_size > (priv->host_info.tx_buff_pos + priv->host_info.tx_buff_size))
-			priv->tx_buff_head = 0;
-		else
-			priv->tx_buff_head += msdu_size;
-
-		if (priv->tx_desc_head < (priv->host_info.tx_desc_count - 1))
-			priv->tx_desc_head++ ;
-		else
-			priv->tx_desc_head = 0;
-
-		if (type == TX_PACKET_TYPE_DATA) {
-			if (status == TX_STATUS_SUCCESS)
-				priv->dev->stats.tx_packets++;
-			else
-				priv->dev->stats.tx_errors++;
-			netif_wake_queue(priv->dev);
-		}
-	}
-}
-
-static u16 find_tx_buff(struct atmel_private *priv, u16 len)
-{
-	u16 bottom_free = priv->host_info.tx_buff_size - priv->tx_buff_tail;
-
-	if (priv->tx_desc_free == 3 || priv->tx_free_mem < len)
-		return 0;
-
-	if (bottom_free >= len)
-		return priv->host_info.tx_buff_pos + priv->tx_buff_tail;
-
-	if (priv->tx_free_mem - bottom_free >= len) {
-		priv->tx_buff_tail = 0;
-		return priv->host_info.tx_buff_pos;
-	}
-
-	return 0;
-}
-
-static void tx_update_descriptor(struct atmel_private *priv, int is_bcast,
-				 u16 len, u16 buff, u8 type)
-{
-	atmel_wmem16(priv, atmel_tx(priv, TX_DESC_POS_OFFSET, priv->tx_desc_tail), buff);
-	atmel_wmem16(priv, atmel_tx(priv, TX_DESC_SIZE_OFFSET, priv->tx_desc_tail), len);
-	if (!priv->use_wpa)
-		atmel_wmem16(priv, atmel_tx(priv, TX_DESC_HOST_LENGTH_OFFSET, priv->tx_desc_tail), len);
-	atmel_wmem8(priv, atmel_tx(priv, TX_DESC_PACKET_TYPE_OFFSET, priv->tx_desc_tail), type);
-	atmel_wmem8(priv, atmel_tx(priv, TX_DESC_RATE_OFFSET, priv->tx_desc_tail), priv->tx_rate);
-	atmel_wmem8(priv, atmel_tx(priv, TX_DESC_RETRY_OFFSET, priv->tx_desc_tail), 0);
-	if (priv->use_wpa) {
-		int cipher_type, cipher_length;
-		if (is_bcast) {
-			cipher_type = priv->group_cipher_suite;
-			if (cipher_type == CIPHER_SUITE_WEP_64 ||
-			    cipher_type == CIPHER_SUITE_WEP_128)
-				cipher_length = 8;
-			else if (cipher_type == CIPHER_SUITE_TKIP)
-				cipher_length = 12;
-			else if (priv->pairwise_cipher_suite == CIPHER_SUITE_WEP_64 ||
-				 priv->pairwise_cipher_suite == CIPHER_SUITE_WEP_128) {
-				cipher_type = priv->pairwise_cipher_suite;
-				cipher_length = 8;
-			} else {
-				cipher_type = CIPHER_SUITE_NONE;
-				cipher_length = 0;
-			}
-		} else {
-			cipher_type = priv->pairwise_cipher_suite;
-			if (cipher_type == CIPHER_SUITE_WEP_64 ||
-			    cipher_type == CIPHER_SUITE_WEP_128)
-				cipher_length = 8;
-			else if (cipher_type == CIPHER_SUITE_TKIP)
-				cipher_length = 12;
-			else if (priv->group_cipher_suite == CIPHER_SUITE_WEP_64 ||
-				 priv->group_cipher_suite == CIPHER_SUITE_WEP_128) {
-				cipher_type = priv->group_cipher_suite;
-				cipher_length = 8;
-			} else {
-				cipher_type = CIPHER_SUITE_NONE;
-				cipher_length = 0;
-			}
-		}
-
-		atmel_wmem8(priv, atmel_tx(priv, TX_DESC_CIPHER_TYPE_OFFSET, priv->tx_desc_tail),
-			    cipher_type);
-		atmel_wmem8(priv, atmel_tx(priv, TX_DESC_CIPHER_LENGTH_OFFSET, priv->tx_desc_tail),
-			    cipher_length);
-	}
-	atmel_wmem32(priv, atmel_tx(priv, TX_DESC_NEXT_OFFSET, priv->tx_desc_tail), 0x80000000L);
-	atmel_wmem8(priv, atmel_tx(priv, TX_DESC_FLAGS_OFFSET, priv->tx_desc_tail), TX_FIRM_OWN);
-	if (priv->tx_desc_previous != priv->tx_desc_tail)
-		atmel_wmem32(priv, atmel_tx(priv, TX_DESC_NEXT_OFFSET, priv->tx_desc_previous), 0);
-	priv->tx_desc_previous = priv->tx_desc_tail;
-	if (priv->tx_desc_tail < (priv->host_info.tx_desc_count - 1))
-		priv->tx_desc_tail++;
-	else
-		priv->tx_desc_tail = 0;
-	priv->tx_desc_free--;
-	priv->tx_free_mem -= len;
-}
-
-static netdev_tx_t start_tx(struct sk_buff *skb, struct net_device *dev)
-{
-	struct atmel_private *priv = netdev_priv(dev);
-	struct ieee80211_hdr header;
-	unsigned long flags;
-	u16 buff, frame_ctl, len = (ETH_ZLEN < skb->len) ? skb->len : ETH_ZLEN;
-
-	if (priv->card && priv->present_callback &&
-	    !(*priv->present_callback)(priv->card)) {
-		dev->stats.tx_errors++;
-		dev_kfree_skb(skb);
-		return NETDEV_TX_OK;
-	}
-
-	if (priv->station_state != STATION_STATE_READY) {
-		dev->stats.tx_errors++;
-		dev_kfree_skb(skb);
-		return NETDEV_TX_OK;
-	}
-
-	/* first ensure the timer func cannot run */
-	spin_lock_bh(&priv->timerlock);
-	/* then stop the hardware ISR */
-	spin_lock_irqsave(&priv->irqlock, flags);
-	/* nb doing the above in the opposite order will deadlock */
-
-	/* The Wireless Header is 30 bytes. In the Ethernet packet we "cut" the
-	   12 first bytes (containing DA/SA) and put them in the appropriate
-	   fields of the Wireless Header. Thus the packet length is then the
-	   initial + 18 (+30-12) */
-
-	if (!(buff = find_tx_buff(priv, len + 18))) {
-		dev->stats.tx_dropped++;
-		spin_unlock_irqrestore(&priv->irqlock, flags);
-		spin_unlock_bh(&priv->timerlock);
-		netif_stop_queue(dev);
-		return NETDEV_TX_BUSY;
-	}
-
-	frame_ctl = IEEE80211_FTYPE_DATA;
-	header.duration_id = 0;
-	header.seq_ctrl = 0;
-	if (priv->wep_is_on)
-		frame_ctl |= IEEE80211_FCTL_PROTECTED;
-	if (priv->operating_mode == IW_MODE_ADHOC) {
-		skb_copy_from_linear_data(skb, &header.addr1, ETH_ALEN);
-		memcpy(&header.addr2, dev->dev_addr, ETH_ALEN);
-		memcpy(&header.addr3, priv->BSSID, ETH_ALEN);
-	} else {
-		frame_ctl |= IEEE80211_FCTL_TODS;
-		memcpy(&header.addr1, priv->CurrentBSSID, ETH_ALEN);
-		memcpy(&header.addr2, dev->dev_addr, ETH_ALEN);
-		skb_copy_from_linear_data(skb, &header.addr3, ETH_ALEN);
-	}
-
-	if (priv->use_wpa)
-		memcpy(&header.addr4, rfc1042_header, ETH_ALEN);
-
-	header.frame_control = cpu_to_le16(frame_ctl);
-	/* Copy the wireless header into the card */
-	atmel_copy_to_card(dev, buff, (unsigned char *)&header, DATA_FRAME_WS_HEADER_SIZE);
-	/* Copy the packet sans its 802.3 header addresses which have been replaced */
-	atmel_copy_to_card(dev, buff + DATA_FRAME_WS_HEADER_SIZE, skb->data + 12, len - 12);
-	priv->tx_buff_tail += len - 12 + DATA_FRAME_WS_HEADER_SIZE;
-
-	/* low bit of first byte of destination tells us if broadcast */
-	tx_update_descriptor(priv, *(skb->data) & 0x01, len + 18, buff, TX_PACKET_TYPE_DATA);
-	dev->stats.tx_bytes += len;
-
-	spin_unlock_irqrestore(&priv->irqlock, flags);
-	spin_unlock_bh(&priv->timerlock);
-	dev_kfree_skb(skb);
-
-	return NETDEV_TX_OK;
-}
-
-static void atmel_transmit_management_frame(struct atmel_private *priv,
-					    struct ieee80211_hdr *header,
-					    u8 *body, int body_len)
-{
-	u16 buff;
-	int len = MGMT_FRAME_BODY_OFFSET + body_len;
-
-	if (!(buff = find_tx_buff(priv, len)))
-		return;
-
-	atmel_copy_to_card(priv->dev, buff, (u8 *)header, MGMT_FRAME_BODY_OFFSET);
-	atmel_copy_to_card(priv->dev, buff + MGMT_FRAME_BODY_OFFSET, body, body_len);
-	priv->tx_buff_tail += len;
-	tx_update_descriptor(priv, header->addr1[0] & 0x01, len, buff, TX_PACKET_TYPE_MGMT);
-}
-
-static void fast_rx_path(struct atmel_private *priv,
-			 struct ieee80211_hdr *header,
-			 u16 msdu_size, u16 rx_packet_loc, u32 crc)
-{
-	/* fast path: unfragmented packet copy directly into skbuf */
-	u8 mac4[6];
-	struct sk_buff	*skb;
-	unsigned char *skbp;
-
-	/* get the final, mac 4 header field, this tells us encapsulation */
-	atmel_copy_to_host(priv->dev, mac4, rx_packet_loc + 24, 6);
-	msdu_size -= 6;
-
-	if (priv->do_rx_crc) {
-		crc = crc32_le(crc, mac4, 6);
-		msdu_size -= 4;
-	}
-
-	if (!(skb = dev_alloc_skb(msdu_size + 14))) {
-		priv->dev->stats.rx_dropped++;
-		return;
-	}
-
-	skb_reserve(skb, 2);
-	skbp = skb_put(skb, msdu_size + 12);
-	atmel_copy_to_host(priv->dev, skbp + 12, rx_packet_loc + 30, msdu_size);
-
-	if (priv->do_rx_crc) {
-		u32 netcrc;
-		crc = crc32_le(crc, skbp + 12, msdu_size);
-		atmel_copy_to_host(priv->dev, (void *)&netcrc, rx_packet_loc + 30 + msdu_size, 4);
-		if ((crc ^ 0xffffffff) != netcrc) {
-			priv->dev->stats.rx_crc_errors++;
-			dev_kfree_skb(skb);
-			return;
-		}
-	}
-
-	memcpy(skbp, header->addr1, ETH_ALEN); /* destination address */
-	if (le16_to_cpu(header->frame_control) & IEEE80211_FCTL_FROMDS)
-		memcpy(&skbp[ETH_ALEN], header->addr3, ETH_ALEN);
-	else
-		memcpy(&skbp[ETH_ALEN], header->addr2, ETH_ALEN); /* source address */
-
-	skb->protocol = eth_type_trans(skb, priv->dev);
-	skb->ip_summed = CHECKSUM_NONE;
-	netif_rx(skb);
-	priv->dev->stats.rx_bytes += 12 + msdu_size;
-	priv->dev->stats.rx_packets++;
-}
-
-/* Test to see if the packet in card memory at packet_loc has a valid CRC
-   It doesn't matter that this is slow: it is only used to proble the first few
-   packets. */
-static int probe_crc(struct atmel_private *priv, u16 packet_loc, u16 msdu_size)
-{
-	int i = msdu_size - 4;
-	u32 netcrc, crc = 0xffffffff;
-
-	if (msdu_size < 4)
-		return 0;
-
-	atmel_copy_to_host(priv->dev, (void *)&netcrc, packet_loc + i, 4);
-
-	atmel_writeAR(priv->dev, packet_loc);
-	while (i--) {
-		u8 octet = atmel_read8(priv->dev, DR);
-		crc = crc32_le(crc, &octet, 1);
-	}
-
-	return (crc ^ 0xffffffff) == netcrc;
-}
-
-static void frag_rx_path(struct atmel_private *priv,
-			 struct ieee80211_hdr *header,
-			 u16 msdu_size, u16 rx_packet_loc, u32 crc, u16 seq_no,
-			 u8 frag_no, int more_frags)
-{
-	u8 mac4[ETH_ALEN];
-	u8 source[ETH_ALEN];
-	struct sk_buff *skb;
-
-	if (le16_to_cpu(header->frame_control) & IEEE80211_FCTL_FROMDS)
-		memcpy(source, header->addr3, ETH_ALEN);
-	else
-		memcpy(source, header->addr2, ETH_ALEN);
-
-	rx_packet_loc += 24; /* skip header */
-
-	if (priv->do_rx_crc)
-		msdu_size -= 4;
-
-	if (frag_no == 0) { /* first fragment */
-		atmel_copy_to_host(priv->dev, mac4, rx_packet_loc, ETH_ALEN);
-		msdu_size -= ETH_ALEN;
-		rx_packet_loc += ETH_ALEN;
-
-		if (priv->do_rx_crc)
-			crc = crc32_le(crc, mac4, 6);
-
-		priv->frag_seq = seq_no;
-		priv->frag_no = 1;
-		priv->frag_len = msdu_size;
-		memcpy(priv->frag_source, source, ETH_ALEN);
-		memcpy(&priv->rx_buf[ETH_ALEN], source, ETH_ALEN);
-		memcpy(priv->rx_buf, header->addr1, ETH_ALEN);
-
-		atmel_copy_to_host(priv->dev, &priv->rx_buf[12], rx_packet_loc, msdu_size);
-
-		if (priv->do_rx_crc) {
-			u32 netcrc;
-			crc = crc32_le(crc, &priv->rx_buf[12], msdu_size);
-			atmel_copy_to_host(priv->dev, (void *)&netcrc, rx_packet_loc + msdu_size, 4);
-			if ((crc ^ 0xffffffff) != netcrc) {
-				priv->dev->stats.rx_crc_errors++;
-				eth_broadcast_addr(priv->frag_source);
-			}
-		}
-
-	} else if (priv->frag_no == frag_no &&
-		   priv->frag_seq == seq_no &&
-		   memcmp(priv->frag_source, source, ETH_ALEN) == 0) {
-
-		atmel_copy_to_host(priv->dev, &priv->rx_buf[12 + priv->frag_len],
-				   rx_packet_loc, msdu_size);
-		if (priv->do_rx_crc) {
-			u32 netcrc;
-			crc = crc32_le(crc,
-				       &priv->rx_buf[12 + priv->frag_len],
-				       msdu_size);
-			atmel_copy_to_host(priv->dev, (void *)&netcrc, rx_packet_loc + msdu_size, 4);
-			if ((crc ^ 0xffffffff) != netcrc) {
-				priv->dev->stats.rx_crc_errors++;
-				eth_broadcast_addr(priv->frag_source);
-				more_frags = 1; /* don't send broken assembly */
-			}
-		}
-
-		priv->frag_len += msdu_size;
-		priv->frag_no++;
-
-		if (!more_frags) { /* last one */
-			eth_broadcast_addr(priv->frag_source);
-			if (!(skb = dev_alloc_skb(priv->frag_len + 14))) {
-				priv->dev->stats.rx_dropped++;
-			} else {
-				skb_reserve(skb, 2);
-				skb_put_data(skb, priv->rx_buf,
-				             priv->frag_len + 12);
-				skb->protocol = eth_type_trans(skb, priv->dev);
-				skb->ip_summed = CHECKSUM_NONE;
-				netif_rx(skb);
-				priv->dev->stats.rx_bytes += priv->frag_len + 12;
-				priv->dev->stats.rx_packets++;
-			}
-		}
-	} else
-		priv->wstats.discard.fragment++;
-}
-
-static void rx_done_irq(struct atmel_private *priv)
-{
-	int i;
-	struct ieee80211_hdr header;
-
-	for (i = 0;
-	     atmel_rmem8(priv, atmel_rx(priv, RX_DESC_FLAGS_OFFSET, priv->rx_desc_head)) == RX_DESC_FLAG_VALID &&
-		     i < priv->host_info.rx_desc_count;
-	     i++) {
-
-		u16 msdu_size, rx_packet_loc, frame_ctl, seq_control;
-		u8 status = atmel_rmem8(priv, atmel_rx(priv, RX_DESC_STATUS_OFFSET, priv->rx_desc_head));
-		u32 crc = 0xffffffff;
-
-		if (status != RX_STATUS_SUCCESS) {
-			if (status == 0xc1) /* determined by experiment */
-				priv->wstats.discard.nwid++;
-			else
-				priv->dev->stats.rx_errors++;
-			goto next;
-		}
-
-		msdu_size = atmel_rmem16(priv, atmel_rx(priv, RX_DESC_MSDU_SIZE_OFFSET, priv->rx_desc_head));
-		rx_packet_loc = atmel_rmem16(priv, atmel_rx(priv, RX_DESC_MSDU_POS_OFFSET, priv->rx_desc_head));
-
-		if (msdu_size < 30) {
-			priv->dev->stats.rx_errors++;
-			goto next;
-		}
-
-		/* Get header as far as end of seq_ctrl */
-		atmel_copy_to_host(priv->dev, (char *)&header, rx_packet_loc, 24);
-		frame_ctl = le16_to_cpu(header.frame_control);
-		seq_control = le16_to_cpu(header.seq_ctrl);
-
-		/* probe for CRC use here if needed  once five packets have
-		   arrived with the same crc status, we assume we know what's
-		   happening and stop probing */
-		if (priv->probe_crc) {
-			if (!priv->wep_is_on || !(frame_ctl & IEEE80211_FCTL_PROTECTED)) {
-				priv->do_rx_crc = probe_crc(priv, rx_packet_loc, msdu_size);
-			} else {
-				priv->do_rx_crc = probe_crc(priv, rx_packet_loc + 24, msdu_size - 24);
-			}
-			if (priv->do_rx_crc) {
-				if (priv->crc_ok_cnt++ > 5)
-					priv->probe_crc = 0;
-			} else {
-				if (priv->crc_ko_cnt++ > 5)
-					priv->probe_crc = 0;
-			}
-		}
-
-		/* don't CRC header when WEP in use */
-		if (priv->do_rx_crc && (!priv->wep_is_on || !(frame_ctl & IEEE80211_FCTL_PROTECTED))) {
-			crc = crc32_le(0xffffffff, (unsigned char *)&header, 24);
-		}
-		msdu_size -= 24; /* header */
-
-		if ((frame_ctl & IEEE80211_FCTL_FTYPE) == IEEE80211_FTYPE_DATA) {
-			int more_fragments = frame_ctl & IEEE80211_FCTL_MOREFRAGS;
-			u8 packet_fragment_no = seq_control & IEEE80211_SCTL_FRAG;
-			u16 packet_sequence_no = (seq_control & IEEE80211_SCTL_SEQ) >> 4;
-
-			if (!more_fragments && packet_fragment_no == 0) {
-				fast_rx_path(priv, &header, msdu_size, rx_packet_loc, crc);
-			} else {
-				frag_rx_path(priv, &header, msdu_size, rx_packet_loc, crc,
-					     packet_sequence_no, packet_fragment_no, more_fragments);
-			}
-		}
-
-		if ((frame_ctl & IEEE80211_FCTL_FTYPE) == IEEE80211_FTYPE_MGMT) {
-			/* copy rest of packet into buffer */
-			atmel_copy_to_host(priv->dev, (unsigned char *)&priv->rx_buf, rx_packet_loc + 24, msdu_size);
-
-			/* we use the same buffer for frag reassembly and control packets */
-			eth_broadcast_addr(priv->frag_source);
-
-			if (priv->do_rx_crc) {
-				/* last 4 octets is crc */
-				msdu_size -= 4;
-				crc = crc32_le(crc, (unsigned char *)&priv->rx_buf, msdu_size);
-				if ((crc ^ 0xffffffff) != (*((u32 *)&priv->rx_buf[msdu_size]))) {
-					priv->dev->stats.rx_crc_errors++;
-					goto next;
-				}
-			}
-
-			atmel_management_frame(priv, &header, msdu_size,
-					       atmel_rmem8(priv, atmel_rx(priv, RX_DESC_RSSI_OFFSET, priv->rx_desc_head)));
-		}
-
-next:
-		/* release descriptor */
-		atmel_wmem8(priv, atmel_rx(priv, RX_DESC_FLAGS_OFFSET, priv->rx_desc_head), RX_DESC_FLAG_CONSUMED);
-
-		if (priv->rx_desc_head < (priv->host_info.rx_desc_count - 1))
-			priv->rx_desc_head++;
-		else
-			priv->rx_desc_head = 0;
-	}
-}
-
-static irqreturn_t service_interrupt(int irq, void *dev_id)
-{
-	struct net_device *dev = (struct net_device *) dev_id;
-	struct atmel_private *priv = netdev_priv(dev);
-	u8 isr;
-	int i = -1;
-	static const u8 irq_order[] = {
-		ISR_OUT_OF_RANGE,
-		ISR_RxCOMPLETE,
-		ISR_TxCOMPLETE,
-		ISR_RxFRAMELOST,
-		ISR_FATAL_ERROR,
-		ISR_COMMAND_COMPLETE,
-		ISR_IBSS_MERGE,
-		ISR_GENERIC_IRQ
-	};
-
-	if (priv->card && priv->present_callback &&
-	    !(*priv->present_callback)(priv->card))
-		return IRQ_HANDLED;
-
-	/* In this state upper-level code assumes it can mess with
-	   the card unhampered by interrupts which may change register state.
-	   Note that even though the card shouldn't generate interrupts
-	   the inturrupt line may be shared. This allows card setup
-	   to go on without disabling interrupts for a long time. */
-	if (priv->station_state == STATION_STATE_DOWN)
-		return IRQ_NONE;
-
-	atmel_clear_gcr(dev, GCR_ENINT); /* disable interrupts */
-
-	while (1) {
-		if (!atmel_lock_mac(priv)) {
-			/* failed to contact card */
-			printk(KERN_ALERT "%s: failed to contact MAC.\n", dev->name);
-			return IRQ_HANDLED;
-		}
-
-		isr = atmel_rmem8(priv, atmel_hi(priv, IFACE_INT_STATUS_OFFSET));
-		atmel_wmem8(priv, atmel_hi(priv, IFACE_LOCKOUT_MAC_OFFSET), 0);
-
-		if (!isr) {
-			atmel_set_gcr(dev, GCR_ENINT); /* enable interrupts */
-			return i == -1 ? IRQ_NONE : IRQ_HANDLED;
-		}
-
-		atmel_set_gcr(dev, GCR_ACKINT); /* acknowledge interrupt */
-
-		for (i = 0; i < ARRAY_SIZE(irq_order); i++)
-			if (isr & irq_order[i])
-				break;
-
-		if (!atmel_lock_mac(priv)) {
-			/* failed to contact card */
-			printk(KERN_ALERT "%s: failed to contact MAC.\n", dev->name);
-			return IRQ_HANDLED;
-		}
-
-		isr = atmel_rmem8(priv, atmel_hi(priv, IFACE_INT_STATUS_OFFSET));
-		isr ^= irq_order[i];
-		atmel_wmem8(priv, atmel_hi(priv, IFACE_INT_STATUS_OFFSET), isr);
-		atmel_wmem8(priv, atmel_hi(priv, IFACE_LOCKOUT_MAC_OFFSET), 0);
-
-		switch (irq_order[i]) {
-
-		case ISR_OUT_OF_RANGE:
-			if (priv->operating_mode == IW_MODE_INFRA &&
-			    priv->station_state == STATION_STATE_READY) {
-				priv->station_is_associated = 0;
-				atmel_scan(priv, 1);
-			}
-			break;
-
-		case ISR_RxFRAMELOST:
-			priv->wstats.discard.misc++;
-			fallthrough;
-		case ISR_RxCOMPLETE:
-			rx_done_irq(priv);
-			break;
-
-		case ISR_TxCOMPLETE:
-			tx_done_irq(priv);
-			break;
-
-		case ISR_FATAL_ERROR:
-			printk(KERN_ALERT "%s: *** FATAL error interrupt ***\n", dev->name);
-			atmel_enter_state(priv, STATION_STATE_MGMT_ERROR);
-			break;
-
-		case ISR_COMMAND_COMPLETE:
-			atmel_command_irq(priv);
-			break;
-
-		case ISR_IBSS_MERGE:
-			atmel_get_mib(priv, Mac_Mgmt_Mib_Type, MAC_MGMT_MIB_CUR_BSSID_POS,
-				      priv->CurrentBSSID, 6);
-			/* The WPA stuff cares about the current AP address */
-			if (priv->use_wpa)
-				build_wpa_mib(priv);
-			break;
-		case ISR_GENERIC_IRQ:
-			printk(KERN_INFO "%s: Generic_irq received.\n", dev->name);
-			break;
-		}
-	}
-}
-
-static struct iw_statistics *atmel_get_wireless_stats(struct net_device *dev)
-{
-	struct atmel_private *priv = netdev_priv(dev);
-
-	/* update the link quality here in case we are seeing no beacons
-	   at all to drive the process */
-	atmel_smooth_qual(priv);
-
-	priv->wstats.status = priv->station_state;
-
-	if (priv->operating_mode == IW_MODE_INFRA) {
-		if (priv->station_state != STATION_STATE_READY) {
-			priv->wstats.qual.qual = 0;
-			priv->wstats.qual.level = 0;
-			priv->wstats.qual.updated = (IW_QUAL_QUAL_INVALID
-					| IW_QUAL_LEVEL_INVALID);
-		}
-		priv->wstats.qual.noise = 0;
-		priv->wstats.qual.updated |= IW_QUAL_NOISE_INVALID;
-	} else {
-		/* Quality levels cannot be determined in ad-hoc mode,
-		   because we can 'hear' more that one remote station. */
-		priv->wstats.qual.qual = 0;
-		priv->wstats.qual.level	= 0;
-		priv->wstats.qual.noise	= 0;
-		priv->wstats.qual.updated = IW_QUAL_QUAL_INVALID
-					| IW_QUAL_LEVEL_INVALID
-					| IW_QUAL_NOISE_INVALID;
-		priv->wstats.miss.beacon = 0;
-	}
-
-	return &priv->wstats;
-}
-
-static int atmel_set_mac_address(struct net_device *dev, void *p)
-{
-	struct sockaddr *addr = p;
-
-	eth_hw_addr_set(dev, addr->sa_data);
-	return atmel_open(dev);
-}
-
-EXPORT_SYMBOL(atmel_open);
-
-int atmel_open(struct net_device *dev)
-{
-	struct atmel_private *priv = netdev_priv(dev);
-	int i, channel, err;
-
-	/* any scheduled timer is no longer needed and might screw things up.. */
-	del_timer_sync(&priv->management_timer);
-
-	/* Interrupts will not touch the card once in this state... */
-	priv->station_state = STATION_STATE_DOWN;
-
-	if (priv->new_SSID_size) {
-		memcpy(priv->SSID, priv->new_SSID, priv->new_SSID_size);
-		priv->SSID_size = priv->new_SSID_size;
-		priv->new_SSID_size = 0;
-	}
-	priv->BSS_list_entries = 0;
-
-	priv->AuthenticationRequestRetryCnt = 0;
-	priv->AssociationRequestRetryCnt = 0;
-	priv->ReAssociationRequestRetryCnt = 0;
-	priv->CurrentAuthentTransactionSeqNum = 0x0001;
-	priv->ExpectedAuthentTransactionSeqNum = 0x0002;
-
-	priv->site_survey_state = SITE_SURVEY_IDLE;
-	priv->station_is_associated = 0;
-
-	err = reset_atmel_card(dev);
-	if (err)
-		return err;
-
-	if (priv->config_reg_domain) {
-		priv->reg_domain = priv->config_reg_domain;
-		atmel_set_mib8(priv, Phy_Mib_Type, PHY_MIB_REG_DOMAIN_POS, priv->reg_domain);
-	} else {
-		priv->reg_domain = atmel_get_mib8(priv, Phy_Mib_Type, PHY_MIB_REG_DOMAIN_POS);
-		for (i = 0; i < ARRAY_SIZE(channel_table); i++)
-			if (priv->reg_domain == channel_table[i].reg_domain)
-				break;
-		if (i == ARRAY_SIZE(channel_table)) {
-			priv->reg_domain = REG_DOMAIN_MKK1;
-			printk(KERN_ALERT "%s: failed to get regulatory domain: assuming MKK1.\n", dev->name);
-		}
-	}
-
-	if ((channel = atmel_validate_channel(priv, priv->channel)))
-		priv->channel = channel;
-
-	/* this moves station_state on.... */
-	atmel_scan(priv, 1);
-
-	atmel_set_gcr(priv->dev, GCR_ENINT); /* enable interrupts */
-	return 0;
-}
-
-static int atmel_close(struct net_device *dev)
-{
-	struct atmel_private *priv = netdev_priv(dev);
-
-	/* Send event to userspace that we are disassociating */
-	if (priv->station_state == STATION_STATE_READY) {
-		union iwreq_data wrqu;
-
-		wrqu.data.length = 0;
-		wrqu.data.flags = 0;
-		wrqu.ap_addr.sa_family = ARPHRD_ETHER;
-		eth_zero_addr(wrqu.ap_addr.sa_data);
-		wireless_send_event(priv->dev, SIOCGIWAP, &wrqu, NULL);
-	}
-
-	atmel_enter_state(priv, STATION_STATE_DOWN);
-
-	if (priv->bus_type == BUS_TYPE_PCCARD)
-		atmel_write16(dev, GCR, 0x0060);
-	atmel_write16(dev, GCR, 0x0040);
-	return 0;
-}
-
-static int atmel_validate_channel(struct atmel_private *priv, int channel)
-{
-	/* check that channel is OK, if so return zero,
-	   else return suitable default channel */
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(channel_table); i++)
-		if (priv->reg_domain == channel_table[i].reg_domain) {
-			if (channel >= channel_table[i].min &&
-			    channel <= channel_table[i].max)
-				return 0;
-			else
-				return channel_table[i].min;
-		}
-	return 0;
-}
-
-#ifdef CONFIG_PROC_FS
-static int atmel_proc_show(struct seq_file *m, void *v)
-{
-	struct atmel_private *priv = m->private;
-	int i;
-	char *s, *r, *c;
-
-	seq_printf(m, "Driver version:\t\t%d.%d\n", DRIVER_MAJOR, DRIVER_MINOR);
-
-	if (priv->station_state != STATION_STATE_DOWN) {
-		seq_printf(m,
-			   "Firmware version:\t%d.%d build %d\n"
-			   "Firmware location:\t",
-			   priv->host_info.major_version,
-			   priv->host_info.minor_version,
-			   priv->host_info.build_version);
-
-		if (priv->card_type != CARD_TYPE_EEPROM)
-			seq_puts(m, "on card\n");
-		else if (priv->firmware)
-			seq_printf(m, "%s loaded by host\n", priv->firmware_id);
-		else
-			seq_printf(m, "%s loaded by hotplug\n", priv->firmware_id);
-
-		switch (priv->card_type) {
-		case CARD_TYPE_PARALLEL_FLASH:
-			c = "Parallel flash";
-			break;
-		case CARD_TYPE_SPI_FLASH:
-			c = "SPI flash\n";
-			break;
-		case CARD_TYPE_EEPROM:
-			c = "EEPROM";
-			break;
-		default:
-			c = "<unknown>";
-		}
-
-		r = "<unknown>";
-		for (i = 0; i < ARRAY_SIZE(channel_table); i++)
-			if (priv->reg_domain == channel_table[i].reg_domain)
-				r = channel_table[i].name;
-
-		seq_printf(m, "MAC memory type:\t%s\n", c);
-		seq_printf(m, "Regulatory domain:\t%s\n", r);
-		seq_printf(m, "Host CRC checking:\t%s\n",
-			 priv->do_rx_crc ? "On" : "Off");
-		seq_printf(m, "WPA-capable firmware:\t%s\n",
-			 priv->use_wpa ? "Yes" : "No");
-	}
-
-	switch (priv->station_state) {
-	case STATION_STATE_SCANNING:
-		s = "Scanning";
-		break;
-	case STATION_STATE_JOINNING:
-		s = "Joining";
-		break;
-	case STATION_STATE_AUTHENTICATING:
-		s = "Authenticating";
-		break;
-	case STATION_STATE_ASSOCIATING:
-		s = "Associating";
-		break;
-	case STATION_STATE_READY:
-		s = "Ready";
-		break;
-	case STATION_STATE_REASSOCIATING:
-		s = "Reassociating";
-		break;
-	case STATION_STATE_MGMT_ERROR:
-		s = "Management error";
-		break;
-	case STATION_STATE_DOWN:
-		s = "Down";
-		break;
-	default:
-		s = "<unknown>";
-	}
-
-	seq_printf(m, "Current state:\t\t%s\n", s);
-	return 0;
-}
-#endif
-
-static const struct net_device_ops atmel_netdev_ops = {
-	.ndo_open 		= atmel_open,
-	.ndo_stop		= atmel_close,
-	.ndo_set_mac_address 	= atmel_set_mac_address,
-	.ndo_start_xmit 	= start_tx,
-	.ndo_validate_addr	= eth_validate_addr,
-};
-
-struct net_device *init_atmel_card(unsigned short irq, unsigned long port,
-				   const AtmelFWType fw_type,
-				   struct device *sys_dev,
-				   int (*card_present)(void *), void *card)
-{
-	struct net_device *dev;
-	struct atmel_private *priv;
-	int rc;
-
-	/* Create the network device object. */
-	dev = alloc_etherdev(sizeof(*priv));
-	if (!dev)
-		return NULL;
-
-	if (dev_alloc_name(dev, dev->name) < 0) {
-		printk(KERN_ERR "atmel: Couldn't get name!\n");
-		goto err_out_free;
-	}
-
-	priv = netdev_priv(dev);
-	priv->dev = dev;
-	priv->sys_dev = sys_dev;
-	priv->present_callback = card_present;
-	priv->card = card;
-	priv->firmware = NULL;
-	priv->firmware_type = fw_type;
-	if (firmware) /* module parameter */
-		strscpy(priv->firmware_id, firmware, sizeof(priv->firmware_id));
-	priv->bus_type = card_present ? BUS_TYPE_PCCARD : BUS_TYPE_PCI;
-	priv->station_state = STATION_STATE_DOWN;
-	priv->do_rx_crc = 0;
-	/* For PCMCIA cards, some chips need CRC, some don't
-	   so we have to probe. */
-	if (priv->bus_type == BUS_TYPE_PCCARD) {
-		priv->probe_crc = 1;
-		priv->crc_ok_cnt = priv->crc_ko_cnt = 0;
-	} else
-		priv->probe_crc = 0;
-	priv->last_qual = jiffies;
-	priv->last_beacon_timestamp = 0;
-	memset(priv->frag_source, 0xff, sizeof(priv->frag_source));
-	eth_zero_addr(priv->BSSID);
-	priv->CurrentBSSID[0] = 0xFF; /* Initialize to something invalid.... */
-	priv->station_was_associated = 0;
-
-	priv->last_survey = jiffies;
-	priv->preamble = LONG_PREAMBLE;
-	priv->operating_mode = IW_MODE_INFRA;
-	priv->connect_to_any_BSS = 0;
-	priv->config_reg_domain = 0;
-	priv->reg_domain = 0;
-	priv->tx_rate = 3;
-	priv->auto_tx_rate = 1;
-	priv->channel = 4;
-	priv->power_mode = 0;
-	priv->SSID[0] = '\0';
-	priv->SSID_size = 0;
-	priv->new_SSID_size = 0;
-	priv->frag_threshold = 2346;
-	priv->rts_threshold = 2347;
-	priv->short_retry = 7;
-	priv->long_retry = 4;
-
-	priv->wep_is_on = 0;
-	priv->default_key = 0;
-	priv->encryption_level = 0;
-	priv->exclude_unencrypted = 0;
-	priv->group_cipher_suite = priv->pairwise_cipher_suite = CIPHER_SUITE_NONE;
-	priv->use_wpa = 0;
-	memset(priv->wep_keys, 0, sizeof(priv->wep_keys));
-	memset(priv->wep_key_len, 0, sizeof(priv->wep_key_len));
-
-	priv->default_beacon_period = priv->beacon_period = 100;
-	priv->listen_interval = 1;
-
-	timer_setup(&priv->management_timer, atmel_management_timer, 0);
-	spin_lock_init(&priv->irqlock);
-	spin_lock_init(&priv->timerlock);
-
-	dev->netdev_ops = &atmel_netdev_ops;
-	dev->wireless_handlers = &atmel_handler_def;
-	dev->irq = irq;
-	dev->base_addr = port;
-
-	/* MTU range: 68 - 2312 */
-	dev->min_mtu = 68;
-	dev->max_mtu = MAX_WIRELESS_BODY - ETH_FCS_LEN;
-
-	SET_NETDEV_DEV(dev, sys_dev);
-
-	if ((rc = request_irq(dev->irq, service_interrupt, IRQF_SHARED, dev->name, dev))) {
-		printk(KERN_ERR "%s: register interrupt %d failed, rc %d\n", dev->name, irq, rc);
-		goto err_out_free;
-	}
-
-	if (!request_region(dev->base_addr, 32,
-			    priv->bus_type == BUS_TYPE_PCCARD ?  "atmel_cs" : "atmel_pci")) {
-		goto err_out_irq;
-	}
-
-	if (register_netdev(dev))
-		goto err_out_res;
-
-	if (!probe_atmel_card(dev)) {
-		unregister_netdev(dev);
-		goto err_out_res;
-	}
-
-	netif_carrier_off(dev);
-
-	if (!proc_create_single_data("driver/atmel", 0, NULL, atmel_proc_show,
-			priv))
-		printk(KERN_WARNING "atmel: unable to create /proc entry.\n");
-
-	printk(KERN_INFO "%s: Atmel at76c50x. Version %d.%d. MAC %pM\n",
-	       dev->name, DRIVER_MAJOR, DRIVER_MINOR, dev->dev_addr);
-
-	return dev;
-
-err_out_res:
-	release_region(dev->base_addr, 32);
-err_out_irq:
-	free_irq(dev->irq, dev);
-err_out_free:
-	free_netdev(dev);
-	return NULL;
-}
-
-EXPORT_SYMBOL(init_atmel_card);
-
-void stop_atmel_card(struct net_device *dev)
-{
-	struct atmel_private *priv = netdev_priv(dev);
-
-	/* put a brick on it... */
-	if (priv->bus_type == BUS_TYPE_PCCARD)
-		atmel_write16(dev, GCR, 0x0060);
-	atmel_write16(dev, GCR, 0x0040);
-
-	del_timer_sync(&priv->management_timer);
-	unregister_netdev(dev);
-	remove_proc_entry("driver/atmel", NULL);
-	free_irq(dev->irq, dev);
-	kfree(priv->firmware);
-	release_region(dev->base_addr, 32);
-	free_netdev(dev);
-}
-
-EXPORT_SYMBOL(stop_atmel_card);
-
-static int atmel_set_essid(struct net_device *dev,
-			   struct iw_request_info *info,
-			   union iwreq_data *wrqu,
-			   char *extra)
-{
-	struct iw_point *dwrq = &wrqu->essid;
-	struct atmel_private *priv = netdev_priv(dev);
-
-	/* Check if we asked for `any' */
-	if (dwrq->flags == 0) {
-		priv->connect_to_any_BSS = 1;
-	} else {
-		int index = (dwrq->flags & IW_ENCODE_INDEX) - 1;
-
-		priv->connect_to_any_BSS = 0;
-
-		/* Check the size of the string */
-		if (dwrq->length > MAX_SSID_LENGTH)
-			 return -E2BIG;
-		if (index != 0)
-			return -EINVAL;
-
-		memcpy(priv->new_SSID, extra, dwrq->length);
-		priv->new_SSID_size = dwrq->length;
-	}
-
-	return -EINPROGRESS;
-}
-
-static int atmel_get_essid(struct net_device *dev,
-			   struct iw_request_info *info,
-			   union iwreq_data *wrqu,
-			   char *extra)
-{
-	struct iw_point *dwrq = &wrqu->essid;
-	struct atmel_private *priv = netdev_priv(dev);
-
-	/* Get the current SSID */
-	if (priv->new_SSID_size != 0) {
-		memcpy(extra, priv->new_SSID, priv->new_SSID_size);
-		dwrq->length = priv->new_SSID_size;
-	} else {
-		memcpy(extra, priv->SSID, priv->SSID_size);
-		dwrq->length = priv->SSID_size;
-	}
-
-	dwrq->flags = !priv->connect_to_any_BSS; /* active */
-
-	return 0;
-}
-
-static int atmel_get_wap(struct net_device *dev,
-			 struct iw_request_info *info,
-			 union iwreq_data *wrqu,
-			 char *extra)
-{
-	struct sockaddr *awrq = &wrqu->ap_addr;
-	struct atmel_private *priv = netdev_priv(dev);
-	memcpy(awrq->sa_data, priv->CurrentBSSID, ETH_ALEN);
-	awrq->sa_family = ARPHRD_ETHER;
-
-	return 0;
-}
-
-static int atmel_set_encode(struct net_device *dev,
-			    struct iw_request_info *info,
-			    union iwreq_data *wrqu,
-			    char *extra)
-{
-	struct iw_point *dwrq = &wrqu->encoding;
-	struct atmel_private *priv = netdev_priv(dev);
-
-	/* Basic checking: do we have a key to set ?
-	 * Note : with the new API, it's impossible to get a NULL pointer.
-	 * Therefore, we need to check a key size == 0 instead.
-	 * New version of iwconfig properly set the IW_ENCODE_NOKEY flag
-	 * when no key is present (only change flags), but older versions
-	 * don't do it. - Jean II */
-	if (dwrq->length > 0) {
-		int index = (dwrq->flags & IW_ENCODE_INDEX) - 1;
-		int current_index = priv->default_key;
-		/* Check the size of the key */
-		if (dwrq->length > 13) {
-			return -EINVAL;
-		}
-		/* Check the index (none -> use current) */
-		if (index < 0 || index >= 4)
-			index = current_index;
-		else
-			priv->default_key = index;
-		/* Set the length */
-		if (dwrq->length > 5)
-			priv->wep_key_len[index] = 13;
-		else
-			if (dwrq->length > 0)
-				priv->wep_key_len[index] = 5;
-			else
-				/* Disable the key */
-				priv->wep_key_len[index] = 0;
-		/* Check if the key is not marked as invalid */
-		if (!(dwrq->flags & IW_ENCODE_NOKEY)) {
-			/* Cleanup */
-			memset(priv->wep_keys[index], 0, 13);
-			/* Copy the key in the driver */
-			memcpy(priv->wep_keys[index], extra, dwrq->length);
-		}
-		/* WE specify that if a valid key is set, encryption
-		 * should be enabled (user may turn it off later)
-		 * This is also how "iwconfig ethX key on" works */
-		if (index == current_index &&
-		    priv->wep_key_len[index] > 0) {
-			priv->wep_is_on = 1;
-			priv->exclude_unencrypted = 1;
-			if (priv->wep_key_len[index] > 5) {
-				priv->pairwise_cipher_suite = CIPHER_SUITE_WEP_128;
-				priv->encryption_level = 2;
-			} else {
-				priv->pairwise_cipher_suite = CIPHER_SUITE_WEP_64;
-				priv->encryption_level = 1;
-			}
-		}
-	} else {
-		/* Do we want to just set the transmit key index ? */
-		int index = (dwrq->flags & IW_ENCODE_INDEX) - 1;
-		if (index >= 0 && index < 4) {
-			priv->default_key = index;
-		} else
-			/* Don't complain if only change the mode */
-			if (!(dwrq->flags & IW_ENCODE_MODE))
-				return -EINVAL;
-	}
-	/* Read the flags */
-	if (dwrq->flags & IW_ENCODE_DISABLED) {
-		priv->wep_is_on = 0;
-		priv->encryption_level = 0;
-		priv->pairwise_cipher_suite = CIPHER_SUITE_NONE;
-	} else {
-		priv->wep_is_on = 1;
-		if (priv->wep_key_len[priv->default_key] > 5) {
-			priv->pairwise_cipher_suite = CIPHER_SUITE_WEP_128;
-			priv->encryption_level = 2;
-		} else {
-			priv->pairwise_cipher_suite = CIPHER_SUITE_WEP_64;
-			priv->encryption_level = 1;
-		}
-	}
-	if (dwrq->flags & IW_ENCODE_RESTRICTED)
-		priv->exclude_unencrypted = 1;
-	if (dwrq->flags & IW_ENCODE_OPEN)
-		priv->exclude_unencrypted = 0;
-
-	return -EINPROGRESS;		/* Call commit handler */
-}
-
-static int atmel_get_encode(struct net_device *dev,
-			    struct iw_request_info *info,
-			    union iwreq_data *wrqu,
-			    char *extra)
-{
-	struct iw_point *dwrq = &wrqu->encoding;
-	struct atmel_private *priv = netdev_priv(dev);
-	int index = (dwrq->flags & IW_ENCODE_INDEX) - 1;
-
-	if (!priv->wep_is_on)
-		dwrq->flags = IW_ENCODE_DISABLED;
-	else {
-		if (priv->exclude_unencrypted)
-			dwrq->flags = IW_ENCODE_RESTRICTED;
-		else
-			dwrq->flags = IW_ENCODE_OPEN;
-	}
-		/* Which key do we want ? -1 -> tx index */
-	if (index < 0 || index >= 4)
-		index = priv->default_key;
-	dwrq->flags |= index + 1;
-	/* Copy the key to the user buffer */
-	dwrq->length = priv->wep_key_len[index];
-	if (dwrq->length > 16) {
-		dwrq->length = 0;
-	} else {
-		memset(extra, 0, 16);
-		memcpy(extra, priv->wep_keys[index], dwrq->length);
-	}
-
-	return 0;
-}
-
-static int atmel_set_encodeext(struct net_device *dev,
-			    struct iw_request_info *info,
-			    union iwreq_data *wrqu,
-			    char *extra)
-{
-	struct atmel_private *priv = netdev_priv(dev);
-	struct iw_point *encoding = &wrqu->encoding;
-	struct iw_encode_ext *ext = (struct iw_encode_ext *)extra;
-	int idx, key_len, alg = ext->alg, set_key = 1;
-
-	/* Determine and validate the key index */
-	idx = encoding->flags & IW_ENCODE_INDEX;
-	if (idx) {
-		if (idx < 1 || idx > 4)
-			return -EINVAL;
-		idx--;
-	} else
-		idx = priv->default_key;
-
-	if (encoding->flags & IW_ENCODE_DISABLED)
-	    alg = IW_ENCODE_ALG_NONE;
-
-	if (ext->ext_flags & IW_ENCODE_EXT_SET_TX_KEY) {
-		priv->default_key = idx;
-		set_key = ext->key_len > 0 ? 1 : 0;
-	}
-
-	if (set_key) {
-		/* Set the requested key first */
-		switch (alg) {
-		case IW_ENCODE_ALG_NONE:
-			priv->wep_is_on = 0;
-			priv->encryption_level = 0;
-			priv->pairwise_cipher_suite = CIPHER_SUITE_NONE;
-			break;
-		case IW_ENCODE_ALG_WEP:
-			if (ext->key_len > 5) {
-				priv->wep_key_len[idx] = 13;
-				priv->pairwise_cipher_suite = CIPHER_SUITE_WEP_128;
-				priv->encryption_level = 2;
-			} else if (ext->key_len > 0) {
-				priv->wep_key_len[idx] = 5;
-				priv->pairwise_cipher_suite = CIPHER_SUITE_WEP_64;
-				priv->encryption_level = 1;
-			} else {
-				return -EINVAL;
-			}
-			priv->wep_is_on = 1;
-			memset(priv->wep_keys[idx], 0, 13);
-			key_len = min ((int)ext->key_len, priv->wep_key_len[idx]);
-			memcpy(priv->wep_keys[idx], ext->key, key_len);
-			break;
-		default:
-			return -EINVAL;
-		}
-	}
-
-	return -EINPROGRESS;
-}
-
-static int atmel_get_encodeext(struct net_device *dev,
-			    struct iw_request_info *info,
-			    union iwreq_data *wrqu,
-			    char *extra)
-{
-	struct atmel_private *priv = netdev_priv(dev);
-	struct iw_point *encoding = &wrqu->encoding;
-	struct iw_encode_ext *ext = (struct iw_encode_ext *)extra;
-	int idx, max_key_len;
-
-	max_key_len = encoding->length - sizeof(*ext);
-	if (max_key_len < 0)
-		return -EINVAL;
-
-	idx = encoding->flags & IW_ENCODE_INDEX;
-	if (idx) {
-		if (idx < 1 || idx > 4)
-			return -EINVAL;
-		idx--;
-	} else
-		idx = priv->default_key;
-
-	encoding->flags = idx + 1;
-	memset(ext, 0, sizeof(*ext));
-
-	if (!priv->wep_is_on) {
-		ext->alg = IW_ENCODE_ALG_NONE;
-		ext->key_len = 0;
-		encoding->flags |= IW_ENCODE_DISABLED;
-	} else {
-		if (priv->encryption_level > 0)
-			ext->alg = IW_ENCODE_ALG_WEP;
-		else
-			return -EINVAL;
-
-		ext->key_len = priv->wep_key_len[idx];
-		memcpy(ext->key, priv->wep_keys[idx], ext->key_len);
-		encoding->flags |= IW_ENCODE_ENABLED;
-	}
-
-	return 0;
-}
-
-static int atmel_set_auth(struct net_device *dev,
-			       struct iw_request_info *info,
-			       union iwreq_data *wrqu, char *extra)
-{
-	struct atmel_private *priv = netdev_priv(dev);
-	struct iw_param *param = &wrqu->param;
-
-	switch (param->flags & IW_AUTH_INDEX) {
-	case IW_AUTH_WPA_VERSION:
-	case IW_AUTH_CIPHER_PAIRWISE:
-	case IW_AUTH_CIPHER_GROUP:
-	case IW_AUTH_KEY_MGMT:
-	case IW_AUTH_RX_UNENCRYPTED_EAPOL:
-	case IW_AUTH_PRIVACY_INVOKED:
-		/*
-		 * atmel does not use these parameters
-		 */
-		break;
-
-	case IW_AUTH_DROP_UNENCRYPTED:
-		priv->exclude_unencrypted = param->value ? 1 : 0;
-		break;
-
-	case IW_AUTH_80211_AUTH_ALG: {
-			if (param->value & IW_AUTH_ALG_SHARED_KEY) {
-				priv->exclude_unencrypted = 1;
-			} else if (param->value & IW_AUTH_ALG_OPEN_SYSTEM) {
-				priv->exclude_unencrypted = 0;
-			} else
-				return -EINVAL;
-			break;
-		}
-
-	case IW_AUTH_WPA_ENABLED:
-		/* Silently accept disable of WPA */
-		if (param->value > 0)
-			return -EOPNOTSUPP;
-		break;
-
-	default:
-		return -EOPNOTSUPP;
-	}
-	return -EINPROGRESS;
-}
-
-static int atmel_get_auth(struct net_device *dev,
-			       struct iw_request_info *info,
-			       union iwreq_data *wrqu, char *extra)
-{
-	struct atmel_private *priv = netdev_priv(dev);
-	struct iw_param *param = &wrqu->param;
-
-	switch (param->flags & IW_AUTH_INDEX) {
-	case IW_AUTH_DROP_UNENCRYPTED:
-		param->value = priv->exclude_unencrypted;
-		break;
-
-	case IW_AUTH_80211_AUTH_ALG:
-		if (priv->exclude_unencrypted == 1)
-			param->value = IW_AUTH_ALG_SHARED_KEY;
-		else
-			param->value = IW_AUTH_ALG_OPEN_SYSTEM;
-		break;
-
-	case IW_AUTH_WPA_ENABLED:
-		param->value = 0;
-		break;
-
-	default:
-		return -EOPNOTSUPP;
-	}
-	return 0;
-}
-
-
-static int atmel_get_name(struct net_device *dev,
-			  struct iw_request_info *info,
-			  union iwreq_data *wrqu,
-			  char *extra)
-{
-	strcpy(wrqu->name, "IEEE 802.11-DS");
-	return 0;
-}
-
-static int atmel_set_rate(struct net_device *dev,
-			  struct iw_request_info *info,
-			  union iwreq_data *wrqu,
-			  char *extra)
-{
-	struct iw_param *vwrq = &wrqu->bitrate;
-	struct atmel_private *priv = netdev_priv(dev);
-
-	if (vwrq->fixed == 0) {
-		priv->tx_rate = 3;
-		priv->auto_tx_rate = 1;
-	} else {
-		priv->auto_tx_rate = 0;
-
-		/* Which type of value ? */
-		if ((vwrq->value < 4) && (vwrq->value >= 0)) {
-			/* Setting by rate index */
-			priv->tx_rate = vwrq->value;
-		} else {
-		/* Setting by frequency value */
-			switch (vwrq->value) {
-			case  1000000:
-				priv->tx_rate = 0;
-				break;
-			case  2000000:
-				priv->tx_rate = 1;
-				break;
-			case  5500000:
-				priv->tx_rate = 2;
-				break;
-			case 11000000:
-				priv->tx_rate = 3;
-				break;
-			default:
-				return -EINVAL;
-			}
-		}
-	}
-
-	return -EINPROGRESS;
-}
-
-static int atmel_set_mode(struct net_device *dev,
-			  struct iw_request_info *info,
-			  union iwreq_data *wrqu,
-			  char *extra)
-{
-	__u32 *uwrq = &wrqu->mode;
-	struct atmel_private *priv = netdev_priv(dev);
-
-	if (*uwrq != IW_MODE_ADHOC && *uwrq != IW_MODE_INFRA)
-		return -EINVAL;
-
-	priv->operating_mode = *uwrq;
-	return -EINPROGRESS;
-}
-
-static int atmel_get_mode(struct net_device *dev,
-			  struct iw_request_info *info,
-			  union iwreq_data *wrqu,
-			  char *extra)
-{
-	__u32 *uwrq = &wrqu->mode;
-	struct atmel_private *priv = netdev_priv(dev);
-
-	*uwrq = priv->operating_mode;
-	return 0;
-}
-
-static int atmel_get_rate(struct net_device *dev,
-			 struct iw_request_info *info,
-			 union iwreq_data *wrqu,
-			 char *extra)
-{
-	struct iw_param *vwrq = &wrqu->bitrate;
-	struct atmel_private *priv = netdev_priv(dev);
-
-	if (priv->auto_tx_rate) {
-		vwrq->fixed = 0;
-		vwrq->value = 11000000;
-	} else {
-		vwrq->fixed = 1;
-		switch (priv->tx_rate) {
-		case 0:
-			vwrq->value =  1000000;
-			break;
-		case 1:
-			vwrq->value =  2000000;
-			break;
-		case 2:
-			vwrq->value =  5500000;
-			break;
-		case 3:
-			vwrq->value = 11000000;
-			break;
-		}
-	}
-	return 0;
-}
-
-static int atmel_set_power(struct net_device *dev,
-			   struct iw_request_info *info,
-			   union iwreq_data *wrqu,
-			   char *extra)
-{
-	struct iw_param *vwrq = &wrqu->power;
-	struct atmel_private *priv = netdev_priv(dev);
-	priv->power_mode = vwrq->disabled ? 0 : 1;
-	return -EINPROGRESS;
-}
-
-static int atmel_get_power(struct net_device *dev,
-			   struct iw_request_info *info,
-			   union iwreq_data *wrqu,
-			   char *extra)
-{
-	struct iw_param *vwrq = &wrqu->power;
-	struct atmel_private *priv = netdev_priv(dev);
-	vwrq->disabled = priv->power_mode ? 0 : 1;
-	vwrq->flags = IW_POWER_ON;
-	return 0;
-}
-
-static int atmel_set_retry(struct net_device *dev,
-			   struct iw_request_info *info,
-			   union iwreq_data *wrqu,
-			   char *extra)
-{
-	struct iw_param *vwrq = &wrqu->retry;
-	struct atmel_private *priv = netdev_priv(dev);
-
-	if (!vwrq->disabled && (vwrq->flags & IW_RETRY_LIMIT)) {
-		if (vwrq->flags & IW_RETRY_LONG)
-			priv->long_retry = vwrq->value;
-		else if (vwrq->flags & IW_RETRY_SHORT)
-			priv->short_retry = vwrq->value;
-		else {
-			/* No modifier : set both */
-			priv->long_retry = vwrq->value;
-			priv->short_retry = vwrq->value;
-		}
-		return -EINPROGRESS;
-	}
-
-	return -EINVAL;
-}
-
-static int atmel_get_retry(struct net_device *dev,
-			   struct iw_request_info *info,
-			   union iwreq_data *wrqu,
-			   char *extra)
-{
-	struct iw_param *vwrq = &wrqu->retry;
-	struct atmel_private *priv = netdev_priv(dev);
-
-	vwrq->disabled = 0;      /* Can't be disabled */
-
-	/* Note : by default, display the short retry number */
-	if (vwrq->flags & IW_RETRY_LONG) {
-		vwrq->flags = IW_RETRY_LIMIT | IW_RETRY_LONG;
-		vwrq->value = priv->long_retry;
-	} else {
-		vwrq->flags = IW_RETRY_LIMIT;
-		vwrq->value = priv->short_retry;
-		if (priv->long_retry != priv->short_retry)
-			vwrq->flags |= IW_RETRY_SHORT;
-	}
-
-	return 0;
-}
-
-static int atmel_set_rts(struct net_device *dev,
-			 struct iw_request_info *info,
-			 union iwreq_data *wrqu,
-			 char *extra)
-{
-	struct iw_param *vwrq = &wrqu->rts;
-	struct atmel_private *priv = netdev_priv(dev);
-	int rthr = vwrq->value;
-
-	if (vwrq->disabled)
-		rthr = 2347;
-	if ((rthr < 0) || (rthr > 2347)) {
-		return -EINVAL;
-	}
-	priv->rts_threshold = rthr;
-
-	return -EINPROGRESS;		/* Call commit handler */
-}
-
-static int atmel_get_rts(struct net_device *dev,
-			 struct iw_request_info *info,
-			 union iwreq_data *wrqu,
-			 char *extra)
-{
-	struct iw_param *vwrq = &wrqu->rts;
-	struct atmel_private *priv = netdev_priv(dev);
-
-	vwrq->value = priv->rts_threshold;
-	vwrq->disabled = (vwrq->value >= 2347);
-	vwrq->fixed = 1;
-
-	return 0;
-}
-
-static int atmel_set_frag(struct net_device *dev,
-			  struct iw_request_info *info,
-			  union iwreq_data *wrqu,
-			  char *extra)
-{
-	struct iw_param *vwrq = &wrqu->frag;
-	struct atmel_private *priv = netdev_priv(dev);
-	int fthr = vwrq->value;
-
-	if (vwrq->disabled)
-		fthr = 2346;
-	if ((fthr < 256) || (fthr > 2346)) {
-		return -EINVAL;
-	}
-	fthr &= ~0x1;	/* Get an even value - is it really needed ??? */
-	priv->frag_threshold = fthr;
-
-	return -EINPROGRESS;		/* Call commit handler */
-}
-
-static int atmel_get_frag(struct net_device *dev,
-			  struct iw_request_info *info,
-			  union iwreq_data *wrqu,
-			  char *extra)
-{
-	struct iw_param *vwrq = &wrqu->frag;
-	struct atmel_private *priv = netdev_priv(dev);
-
-	vwrq->value = priv->frag_threshold;
-	vwrq->disabled = (vwrq->value >= 2346);
-	vwrq->fixed = 1;
-
-	return 0;
-}
-
-static int atmel_set_freq(struct net_device *dev,
-			  struct iw_request_info *info,
-			  union iwreq_data *wrqu,
-			  char *extra)
-{
-	struct iw_freq *fwrq = &wrqu->freq;
-	struct atmel_private *priv = netdev_priv(dev);
-	int rc = -EINPROGRESS;		/* Call commit handler */
-
-	/* If setting by frequency, convert to a channel */
-	if (fwrq->e == 1) {
-		int f = fwrq->m / 100000;
-
-		/* Hack to fall through... */
-		fwrq->e = 0;
-		fwrq->m = ieee80211_frequency_to_channel(f);
-	}
-	/* Setting by channel number */
-	if (fwrq->m < 0 || fwrq->m > 1000 || fwrq->e > 0)
-		rc = -EOPNOTSUPP;
-	else {
-		int channel = fwrq->m;
-		if (atmel_validate_channel(priv, channel) == 0) {
-			priv->channel = channel;
-		} else {
-			rc = -EINVAL;
-		}
-	}
-	return rc;
-}
-
-static int atmel_get_freq(struct net_device *dev,
-			  struct iw_request_info *info,
-			  union iwreq_data *wrqu,
-			  char *extra)
-{
-	struct iw_freq *fwrq = &wrqu->freq;
-	struct atmel_private *priv = netdev_priv(dev);
-
-	fwrq->m = priv->channel;
-	fwrq->e = 0;
-	return 0;
-}
-
-static int atmel_set_scan(struct net_device *dev,
-			  struct iw_request_info *info,
-			  union iwreq_data *dwrq,
-			  char *extra)
-{
-	struct atmel_private *priv = netdev_priv(dev);
-	unsigned long flags;
-
-	/* Note : you may have realised that, as this is a SET operation,
-	 * this is privileged and therefore a normal user can't
-	 * perform scanning.
-	 * This is not an error, while the device perform scanning,
-	 * traffic doesn't flow, so it's a perfect DoS...
-	 * Jean II */
-
-	if (priv->station_state == STATION_STATE_DOWN)
-		return -EAGAIN;
-
-	/* Timeout old surveys. */
-	if (time_after(jiffies, priv->last_survey + 20 * HZ))
-		priv->site_survey_state = SITE_SURVEY_IDLE;
-	priv->last_survey = jiffies;
-
-	/* Initiate a scan command */
-	if (priv->site_survey_state == SITE_SURVEY_IN_PROGRESS)
-		return -EBUSY;
-
-	del_timer_sync(&priv->management_timer);
-	spin_lock_irqsave(&priv->irqlock, flags);
-
-	priv->site_survey_state = SITE_SURVEY_IN_PROGRESS;
-	priv->fast_scan = 0;
-	atmel_scan(priv, 0);
-	spin_unlock_irqrestore(&priv->irqlock, flags);
-
-	return 0;
-}
-
-static int atmel_get_scan(struct net_device *dev,
-			  struct iw_request_info *info,
-			  union iwreq_data *wrqu,
-			  char *extra)
-{
-	struct iw_point *dwrq = &wrqu->data;
-	struct atmel_private *priv = netdev_priv(dev);
-	int i;
-	char *current_ev = extra;
-	struct iw_event	iwe;
-
-	if (priv->site_survey_state != SITE_SURVEY_COMPLETED)
-		return -EAGAIN;
-
-	for (i = 0; i < priv->BSS_list_entries; i++) {
-		iwe.cmd = SIOCGIWAP;
-		iwe.u.ap_addr.sa_family = ARPHRD_ETHER;
-		memcpy(iwe.u.ap_addr.sa_data, priv->BSSinfo[i].BSSID, ETH_ALEN);
-		current_ev = iwe_stream_add_event(info, current_ev,
-						  extra + IW_SCAN_MAX_DATA,
-						  &iwe, IW_EV_ADDR_LEN);
-
-		iwe.u.data.length =  priv->BSSinfo[i].SSIDsize;
-		if (iwe.u.data.length > 32)
-			iwe.u.data.length = 32;
-		iwe.cmd = SIOCGIWESSID;
-		iwe.u.data.flags = 1;
-		current_ev = iwe_stream_add_point(info, current_ev,
-						  extra + IW_SCAN_MAX_DATA,
-						  &iwe, priv->BSSinfo[i].SSID);
-
-		iwe.cmd = SIOCGIWMODE;
-		iwe.u.mode = priv->BSSinfo[i].BSStype;
-		current_ev = iwe_stream_add_event(info, current_ev,
-						  extra + IW_SCAN_MAX_DATA,
-						  &iwe, IW_EV_UINT_LEN);
-
-		iwe.cmd = SIOCGIWFREQ;
-		iwe.u.freq.m = priv->BSSinfo[i].channel;
-		iwe.u.freq.e = 0;
-		current_ev = iwe_stream_add_event(info, current_ev,
-						  extra + IW_SCAN_MAX_DATA,
-						  &iwe, IW_EV_FREQ_LEN);
-
-		/* Add quality statistics */
-		iwe.cmd = IWEVQUAL;
-		iwe.u.qual.level = priv->BSSinfo[i].RSSI;
-		iwe.u.qual.qual  = iwe.u.qual.level;
-		/* iwe.u.qual.noise  = SOMETHING */
-		current_ev = iwe_stream_add_event(info, current_ev,
-						  extra + IW_SCAN_MAX_DATA,
-						  &iwe, IW_EV_QUAL_LEN);
-
-
-		iwe.cmd = SIOCGIWENCODE;
-		if (priv->BSSinfo[i].UsingWEP)
-			iwe.u.data.flags = IW_ENCODE_ENABLED | IW_ENCODE_NOKEY;
-		else
-			iwe.u.data.flags = IW_ENCODE_DISABLED;
-		iwe.u.data.length = 0;
-		current_ev = iwe_stream_add_point(info, current_ev,
-						  extra + IW_SCAN_MAX_DATA,
-						  &iwe, NULL);
-	}
-
-	/* Length of data */
-	dwrq->length = (current_ev - extra);
-	dwrq->flags = 0;
-
-	return 0;
-}
-
-static int atmel_get_range(struct net_device *dev,
-			   struct iw_request_info *info,
-			   union iwreq_data *wrqu,
-			   char *extra)
-{
-	struct iw_point *dwrq = &wrqu->data;
-	struct atmel_private *priv = netdev_priv(dev);
-	struct iw_range *range = (struct iw_range *) extra;
-	int k, i, j;
-
-	dwrq->length = sizeof(struct iw_range);
-	memset(range, 0, sizeof(struct iw_range));
-	range->min_nwid = 0x0000;
-	range->max_nwid = 0x0000;
-	range->num_channels = 0;
-	for (j = 0; j < ARRAY_SIZE(channel_table); j++)
-		if (priv->reg_domain == channel_table[j].reg_domain) {
-			range->num_channels = channel_table[j].max - channel_table[j].min + 1;
-			break;
-		}
-	if (range->num_channels != 0) {
-		for (k = 0, i = channel_table[j].min; i <= channel_table[j].max; i++) {
-			range->freq[k].i = i; /* List index */
-
-			/* Values in MHz -> * 10^5 * 10 */
-			range->freq[k].m = 100000 *
-			 ieee80211_channel_to_frequency(i, NL80211_BAND_2GHZ);
-			range->freq[k++].e = 1;
-		}
-		range->num_frequency = k;
-	}
-
-	range->max_qual.qual = 100;
-	range->max_qual.level = 100;
-	range->max_qual.noise = 0;
-	range->max_qual.updated = IW_QUAL_NOISE_INVALID;
-
-	range->avg_qual.qual = 50;
-	range->avg_qual.level = 50;
-	range->avg_qual.noise = 0;
-	range->avg_qual.updated = IW_QUAL_NOISE_INVALID;
-
-	range->sensitivity = 0;
-
-	range->bitrate[0] =  1000000;
-	range->bitrate[1] =  2000000;
-	range->bitrate[2] =  5500000;
-	range->bitrate[3] = 11000000;
-	range->num_bitrates = 4;
-
-	range->min_rts = 0;
-	range->max_rts = 2347;
-	range->min_frag = 256;
-	range->max_frag = 2346;
-
-	range->encoding_size[0] = 5;
-	range->encoding_size[1] = 13;
-	range->num_encoding_sizes = 2;
-	range->max_encoding_tokens = 4;
-
-	range->pmp_flags = IW_POWER_ON;
-	range->pmt_flags = IW_POWER_ON;
-	range->pm_capa = 0;
-
-	range->we_version_source = WIRELESS_EXT;
-	range->we_version_compiled = WIRELESS_EXT;
-	range->retry_capa = IW_RETRY_LIMIT ;
-	range->retry_flags = IW_RETRY_LIMIT;
-	range->r_time_flags = 0;
-	range->min_retry = 1;
-	range->max_retry = 65535;
-
-	return 0;
-}
-
-static int atmel_set_wap(struct net_device *dev,
-			 struct iw_request_info *info,
-			 union iwreq_data *wrqu,
-			 char *extra)
-{
-	struct sockaddr *awrq = &wrqu->ap_addr;
-	struct atmel_private *priv = netdev_priv(dev);
-	int i;
-	static const u8 any[] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
-	static const u8 off[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
-	unsigned long flags;
-
-	if (awrq->sa_family != ARPHRD_ETHER)
-		return -EINVAL;
-
-	if (!memcmp(any, awrq->sa_data, 6) ||
-	    !memcmp(off, awrq->sa_data, 6)) {
-		del_timer_sync(&priv->management_timer);
-		spin_lock_irqsave(&priv->irqlock, flags);
-		atmel_scan(priv, 1);
-		spin_unlock_irqrestore(&priv->irqlock, flags);
-		return 0;
-	}
-
-	for (i = 0; i < priv->BSS_list_entries; i++) {
-		if (memcmp(priv->BSSinfo[i].BSSID, awrq->sa_data, 6) == 0) {
-			if (!priv->wep_is_on && priv->BSSinfo[i].UsingWEP) {
-				return -EINVAL;
-			} else if  (priv->wep_is_on && !priv->BSSinfo[i].UsingWEP) {
-				return -EINVAL;
-			} else {
-				del_timer_sync(&priv->management_timer);
-				spin_lock_irqsave(&priv->irqlock, flags);
-				atmel_join_bss(priv, i);
-				spin_unlock_irqrestore(&priv->irqlock, flags);
-				return 0;
-			}
-		}
-	}
-
-	return -EINVAL;
-}
-
-static int atmel_config_commit(struct net_device *dev,
-			       struct iw_request_info *info,	/* NULL */
-			       union iwreq_data *zwrq,		/* NULL */
-			       char *extra)			/* NULL */
-{
-	return atmel_open(dev);
-}
-
-static const iw_handler atmel_handler[] =
-{
-	IW_HANDLER(SIOCSIWCOMMIT,	atmel_config_commit),
-	IW_HANDLER(SIOCGIWNAME,		atmel_get_name),
-	IW_HANDLER(SIOCSIWFREQ,		atmel_set_freq),
-	IW_HANDLER(SIOCGIWFREQ,		atmel_get_freq),
-	IW_HANDLER(SIOCSIWMODE,		atmel_set_mode),
-	IW_HANDLER(SIOCGIWMODE,		atmel_get_mode),
-	IW_HANDLER(SIOCGIWRANGE,	atmel_get_range),
-	IW_HANDLER(SIOCSIWAP,		atmel_set_wap),
-	IW_HANDLER(SIOCGIWAP,		atmel_get_wap),
-	IW_HANDLER(SIOCSIWSCAN,		atmel_set_scan),
-	IW_HANDLER(SIOCGIWSCAN,		atmel_get_scan),
-	IW_HANDLER(SIOCSIWESSID,	atmel_set_essid),
-	IW_HANDLER(SIOCGIWESSID,	atmel_get_essid),
-	IW_HANDLER(SIOCSIWRATE,		atmel_set_rate),
-	IW_HANDLER(SIOCGIWRATE,		atmel_get_rate),
-	IW_HANDLER(SIOCSIWRTS,		atmel_set_rts),
-	IW_HANDLER(SIOCGIWRTS,		atmel_get_rts),
-	IW_HANDLER(SIOCSIWFRAG,		atmel_set_frag),
-	IW_HANDLER(SIOCGIWFRAG,		atmel_get_frag),
-	IW_HANDLER(SIOCSIWRETRY,	atmel_set_retry),
-	IW_HANDLER(SIOCGIWRETRY,	atmel_get_retry),
-	IW_HANDLER(SIOCSIWENCODE,	atmel_set_encode),
-	IW_HANDLER(SIOCGIWENCODE,	atmel_get_encode),
-	IW_HANDLER(SIOCSIWPOWER,	atmel_set_power),
-	IW_HANDLER(SIOCGIWPOWER,	atmel_get_power),
-	IW_HANDLER(SIOCSIWAUTH,		atmel_set_auth),
-	IW_HANDLER(SIOCGIWAUTH,		atmel_get_auth),
-	IW_HANDLER(SIOCSIWENCODEEXT,	atmel_set_encodeext),
-	IW_HANDLER(SIOCGIWENCODEEXT,	atmel_get_encodeext),
-};
-
-static const iw_handler atmel_private_handler[] =
-{
-	NULL,				/* SIOCIWFIRSTPRIV */
-};
-
-struct atmel_priv_ioctl {
-	char id[32];
-	unsigned char __user *data;
-	unsigned short len;
-};
-
-#define ATMELFWL	SIOCIWFIRSTPRIV
-#define ATMELIDIFC	ATMELFWL + 1
-#define ATMELRD		ATMELFWL + 2
-#define ATMELMAGIC 0x51807
-#define REGDOMAINSZ 20
-
-static const struct iw_priv_args atmel_private_args[] = {
-	{
-		.cmd = ATMELFWL,
-		.set_args = IW_PRIV_TYPE_BYTE
-				| IW_PRIV_SIZE_FIXED
-				| sizeof(struct atmel_priv_ioctl),
-		.get_args = IW_PRIV_TYPE_NONE,
-		.name = "atmelfwl"
-	}, {
-		.cmd = ATMELIDIFC,
-		.set_args = IW_PRIV_TYPE_NONE,
-		.get_args = IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1,
-		.name = "atmelidifc"
-	}, {
-		.cmd = ATMELRD,
-		.set_args = IW_PRIV_TYPE_CHAR | REGDOMAINSZ,
-		.get_args = IW_PRIV_TYPE_NONE,
-		.name = "regdomain"
-	},
-};
-
-static const struct iw_handler_def atmel_handler_def = {
-	.num_standard	= ARRAY_SIZE(atmel_handler),
-	.num_private	= ARRAY_SIZE(atmel_private_handler),
-	.num_private_args = ARRAY_SIZE(atmel_private_args),
-	.standard	= atmel_handler,
-	.private	= atmel_private_handler,
-	.private_args	= (struct iw_priv_args *) atmel_private_args,
-	.get_wireless_stats = atmel_get_wireless_stats
-};
-
-struct auth_body {
-	__le16 alg;
-	__le16 trans_seq;
-	__le16 status;
-	u8 el_id;
-	u8 chall_text_len;
-	u8 chall_text[253];
-};
-
-static void atmel_enter_state(struct atmel_private *priv, int new_state)
-{
-	int old_state = priv->station_state;
-
-	if (new_state == old_state)
-		return;
-
-	priv->station_state = new_state;
-
-	if (new_state == STATION_STATE_READY) {
-		netif_start_queue(priv->dev);
-		netif_carrier_on(priv->dev);
-	}
-
-	if (old_state == STATION_STATE_READY) {
-		netif_carrier_off(priv->dev);
-		if (netif_running(priv->dev))
-			netif_stop_queue(priv->dev);
-		priv->last_beacon_timestamp = 0;
-	}
-}
-
-static void atmel_scan(struct atmel_private *priv, int specific_ssid)
-{
-	struct {
-		u8 BSSID[ETH_ALEN];
-		u8 SSID[MAX_SSID_LENGTH];
-		u8 scan_type;
-		u8 channel;
-		__le16 BSS_type;
-		__le16 min_channel_time;
-		__le16 max_channel_time;
-		u8 options;
-		u8 SSID_size;
-	} cmd;
-
-	eth_broadcast_addr(cmd.BSSID);
-
-	if (priv->fast_scan) {
-		cmd.SSID_size = priv->SSID_size;
-		memcpy(cmd.SSID, priv->SSID, priv->SSID_size);
-		cmd.min_channel_time = cpu_to_le16(10);
-		cmd.max_channel_time = cpu_to_le16(50);
-	} else {
-		priv->BSS_list_entries = 0;
-		cmd.SSID_size = 0;
-		cmd.min_channel_time = cpu_to_le16(10);
-		cmd.max_channel_time = cpu_to_le16(120);
-	}
-
-	cmd.options = 0;
-
-	if (!specific_ssid)
-		cmd.options |= SCAN_OPTIONS_SITE_SURVEY;
-
-	cmd.channel = (priv->channel & 0x7f);
-	cmd.scan_type = SCAN_TYPE_ACTIVE;
-	cmd.BSS_type = cpu_to_le16(priv->operating_mode == IW_MODE_ADHOC ?
-		BSS_TYPE_AD_HOC : BSS_TYPE_INFRASTRUCTURE);
-
-	atmel_send_command(priv, CMD_Scan, &cmd, sizeof(cmd));
-
-	/* This must come after all hardware access to avoid being messed up
-	   by stuff happening in interrupt context after we leave STATE_DOWN */
-	atmel_enter_state(priv, STATION_STATE_SCANNING);
-}
-
-static void join(struct atmel_private *priv, int type)
-{
-	struct {
-		u8 BSSID[6];
-		u8 SSID[MAX_SSID_LENGTH];
-		u8 BSS_type; /* this is a short in a scan command - weird */
-		u8 channel;
-		__le16 timeout;
-		u8 SSID_size;
-		u8 reserved;
-	} cmd;
-
-	cmd.SSID_size = priv->SSID_size;
-	memcpy(cmd.SSID, priv->SSID, priv->SSID_size);
-	memcpy(cmd.BSSID, priv->CurrentBSSID, ETH_ALEN);
-	cmd.channel = (priv->channel & 0x7f);
-	cmd.BSS_type = type;
-	cmd.timeout = cpu_to_le16(2000);
-
-	atmel_send_command(priv, CMD_Join, &cmd, sizeof(cmd));
-}
-
-static void start(struct atmel_private *priv, int type)
-{
-	struct {
-		u8 BSSID[6];
-		u8 SSID[MAX_SSID_LENGTH];
-		u8 BSS_type;
-		u8 channel;
-		u8 SSID_size;
-		u8 reserved[3];
-	} cmd;
-
-	cmd.SSID_size = priv->SSID_size;
-	memcpy(cmd.SSID, priv->SSID, priv->SSID_size);
-	memcpy(cmd.BSSID, priv->BSSID, ETH_ALEN);
-	cmd.BSS_type = type;
-	cmd.channel = (priv->channel & 0x7f);
-
-	atmel_send_command(priv, CMD_Start, &cmd, sizeof(cmd));
-}
-
-static void handle_beacon_probe(struct atmel_private *priv, u16 capability,
-				u8 channel)
-{
-	int rejoin = 0;
-	int new = capability & WLAN_CAPABILITY_SHORT_PREAMBLE ?
-		SHORT_PREAMBLE : LONG_PREAMBLE;
-
-	if (priv->preamble != new) {
-		priv->preamble = new;
-		rejoin = 1;
-		atmel_set_mib8(priv, Local_Mib_Type, LOCAL_MIB_PREAMBLE_TYPE, new);
-	}
-
-	if (priv->channel != channel) {
-		priv->channel = channel;
-		rejoin = 1;
-		atmel_set_mib8(priv, Phy_Mib_Type, PHY_MIB_CHANNEL_POS, channel);
-	}
-
-	if (rejoin) {
-		priv->station_is_associated = 0;
-		atmel_enter_state(priv, STATION_STATE_JOINNING);
-
-		if (priv->operating_mode == IW_MODE_INFRA)
-			join(priv, BSS_TYPE_INFRASTRUCTURE);
-		else
-			join(priv, BSS_TYPE_AD_HOC);
-	}
-}
-
-static void send_authentication_request(struct atmel_private *priv, u16 system,
-					u8 *challenge, int challenge_len)
-{
-	struct ieee80211_hdr header;
-	struct auth_body auth;
-
-	header.frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_AUTH);
-	header.duration_id = cpu_to_le16(0x8000);
-	header.seq_ctrl = 0;
-	memcpy(header.addr1, priv->CurrentBSSID, ETH_ALEN);
-	memcpy(header.addr2, priv->dev->dev_addr, ETH_ALEN);
-	memcpy(header.addr3, priv->CurrentBSSID, ETH_ALEN);
-
-	if (priv->wep_is_on && priv->CurrentAuthentTransactionSeqNum != 1)
-		/* no WEP for authentication frames with TrSeqNo 1 */
-		header.frame_control |=  cpu_to_le16(IEEE80211_FCTL_PROTECTED);
-
-	auth.alg = cpu_to_le16(system);
-
-	auth.status = 0;
-	auth.trans_seq = cpu_to_le16(priv->CurrentAuthentTransactionSeqNum);
-	priv->ExpectedAuthentTransactionSeqNum = priv->CurrentAuthentTransactionSeqNum+1;
-	priv->CurrentAuthentTransactionSeqNum += 2;
-
-	if (challenge_len != 0)	{
-		auth.el_id = 16; /* challenge_text */
-		auth.chall_text_len = challenge_len;
-		memcpy(auth.chall_text, challenge, challenge_len);
-		atmel_transmit_management_frame(priv, &header, (u8 *)&auth, 8 + challenge_len);
-	} else {
-		atmel_transmit_management_frame(priv, &header, (u8 *)&auth, 6);
-	}
-}
-
-static void send_association_request(struct atmel_private *priv, int is_reassoc)
-{
-	u8 *ssid_el_p;
-	int bodysize;
-	struct ieee80211_hdr header;
-	struct ass_req_format {
-		__le16 capability;
-		__le16 listen_interval;
-		u8 ap[ETH_ALEN]; /* nothing after here directly accessible */
-		u8 ssid_el_id;
-		u8 ssid_len;
-		u8 ssid[MAX_SSID_LENGTH];
-		u8 sup_rates_el_id;
-		u8 sup_rates_len;
-		u8 rates[4];
-	} body;
-
-	header.frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT |
-		(is_reassoc ? IEEE80211_STYPE_REASSOC_REQ : IEEE80211_STYPE_ASSOC_REQ));
-	header.duration_id = cpu_to_le16(0x8000);
-	header.seq_ctrl = 0;
-
-	memcpy(header.addr1, priv->CurrentBSSID, ETH_ALEN);
-	memcpy(header.addr2, priv->dev->dev_addr, ETH_ALEN);
-	memcpy(header.addr3, priv->CurrentBSSID, ETH_ALEN);
-
-	body.capability = cpu_to_le16(WLAN_CAPABILITY_ESS);
-	if (priv->wep_is_on)
-		body.capability |= cpu_to_le16(WLAN_CAPABILITY_PRIVACY);
-	if (priv->preamble == SHORT_PREAMBLE)
-		body.capability |= cpu_to_le16(WLAN_CAPABILITY_SHORT_PREAMBLE);
-
-	body.listen_interval = cpu_to_le16(priv->listen_interval * priv->beacon_period);
-
-	/* current AP address - only in reassoc frame */
-	if (is_reassoc) {
-		memcpy(body.ap, priv->CurrentBSSID, ETH_ALEN);
-		ssid_el_p = &body.ssid_el_id;
-		bodysize = 18 + priv->SSID_size;
-	} else {
-		ssid_el_p = &body.ap[0];
-		bodysize = 12 + priv->SSID_size;
-	}
-
-	ssid_el_p[0] = WLAN_EID_SSID;
-	ssid_el_p[1] = priv->SSID_size;
-	memcpy(ssid_el_p + 2, priv->SSID, priv->SSID_size);
-	ssid_el_p[2 + priv->SSID_size] = WLAN_EID_SUPP_RATES;
-	ssid_el_p[3 + priv->SSID_size] = 4; /* len of supported rates */
-	memcpy(ssid_el_p + 4 + priv->SSID_size, atmel_basic_rates, 4);
-
-	atmel_transmit_management_frame(priv, &header, (void *)&body, bodysize);
-}
-
-static int is_frame_from_current_bss(struct atmel_private *priv,
-				     struct ieee80211_hdr *header)
-{
-	if (le16_to_cpu(header->frame_control) & IEEE80211_FCTL_FROMDS)
-		return memcmp(header->addr3, priv->CurrentBSSID, 6) == 0;
-	else
-		return memcmp(header->addr2, priv->CurrentBSSID, 6) == 0;
-}
-
-static int retrieve_bss(struct atmel_private *priv)
-{
-	int i;
-	int max_rssi = -128;
-	int max_index = -1;
-
-	if (priv->BSS_list_entries == 0)
-		return -1;
-
-	if (priv->connect_to_any_BSS) {
-		/* Select a BSS with the max-RSSI but of the same type and of
-		   the same WEP mode and that it is not marked as 'bad' (i.e.
-		   we had previously failed to connect to this BSS with the
-		   settings that we currently use) */
-		priv->current_BSS = 0;
-		for (i = 0; i < priv->BSS_list_entries; i++) {
-			if (priv->operating_mode == priv->BSSinfo[i].BSStype &&
-			    ((!priv->wep_is_on && !priv->BSSinfo[i].UsingWEP) ||
-			     (priv->wep_is_on && priv->BSSinfo[i].UsingWEP)) &&
-			    !(priv->BSSinfo[i].channel & 0x80)) {
-				max_rssi = priv->BSSinfo[i].RSSI;
-				priv->current_BSS = max_index = i;
-			}
-		}
-		return max_index;
-	}
-
-	for (i = 0; i < priv->BSS_list_entries; i++) {
-		if (priv->SSID_size == priv->BSSinfo[i].SSIDsize &&
-		    memcmp(priv->SSID, priv->BSSinfo[i].SSID, priv->SSID_size) == 0 &&
-		    priv->operating_mode == priv->BSSinfo[i].BSStype &&
-		    atmel_validate_channel(priv, priv->BSSinfo[i].channel) == 0) {
-			if (priv->BSSinfo[i].RSSI >= max_rssi) {
-				max_rssi = priv->BSSinfo[i].RSSI;
-				max_index = i;
-			}
-		}
-	}
-	return max_index;
-}
-
-static void store_bss_info(struct atmel_private *priv,
-			   struct ieee80211_hdr *header, u16 capability,
-			   u16 beacon_period, u8 channel, u8 rssi, u8 ssid_len,
-			   u8 *ssid, int is_beacon)
-{
-	u8 *bss = capability & WLAN_CAPABILITY_ESS ? header->addr2 : header->addr3;
-	int i, index;
-
-	for (index = -1, i = 0; i < priv->BSS_list_entries; i++)
-		if (memcmp(bss, priv->BSSinfo[i].BSSID, ETH_ALEN) == 0)
-			index = i;
-
-	/* If we process a probe and an entry from this BSS exists
-	   we will update the BSS entry with the info from this BSS.
-	   If we process a beacon we will only update RSSI */
-
-	if (index == -1) {
-		if (priv->BSS_list_entries == MAX_BSS_ENTRIES)
-			return;
-		index = priv->BSS_list_entries++;
-		memcpy(priv->BSSinfo[index].BSSID, bss, ETH_ALEN);
-		priv->BSSinfo[index].RSSI = rssi;
-	} else {
-		if (rssi > priv->BSSinfo[index].RSSI)
-			priv->BSSinfo[index].RSSI = rssi;
-		if (is_beacon)
-			return;
-	}
-
-	priv->BSSinfo[index].channel = channel;
-	priv->BSSinfo[index].beacon_period = beacon_period;
-	priv->BSSinfo[index].UsingWEP = capability & WLAN_CAPABILITY_PRIVACY;
-	memcpy(priv->BSSinfo[index].SSID, ssid, ssid_len);
-	priv->BSSinfo[index].SSIDsize = ssid_len;
-
-	if (capability & WLAN_CAPABILITY_IBSS)
-		priv->BSSinfo[index].BSStype = IW_MODE_ADHOC;
-	else if (capability & WLAN_CAPABILITY_ESS)
-		priv->BSSinfo[index].BSStype = IW_MODE_INFRA;
-
-	priv->BSSinfo[index].preamble = capability & WLAN_CAPABILITY_SHORT_PREAMBLE ?
-		SHORT_PREAMBLE : LONG_PREAMBLE;
-}
-
-static void authenticate(struct atmel_private *priv, u16 frame_len)
-{
-	struct auth_body *auth = (struct auth_body *)priv->rx_buf;
-	u16 status = le16_to_cpu(auth->status);
-	u16 trans_seq_no = le16_to_cpu(auth->trans_seq);
-	u16 system = le16_to_cpu(auth->alg);
-
-	if (status == WLAN_STATUS_SUCCESS && !priv->wep_is_on) {
-		/* no WEP */
-		if (priv->station_was_associated) {
-			atmel_enter_state(priv, STATION_STATE_REASSOCIATING);
-			send_association_request(priv, 1);
-			return;
-		} else {
-			atmel_enter_state(priv, STATION_STATE_ASSOCIATING);
-			send_association_request(priv, 0);
-			return;
-		}
-	}
-
-	if (status == WLAN_STATUS_SUCCESS && priv->wep_is_on) {
-		int should_associate = 0;
-		/* WEP */
-		if (trans_seq_no != priv->ExpectedAuthentTransactionSeqNum)
-			return;
-
-		if (system == WLAN_AUTH_OPEN) {
-			if (trans_seq_no == 0x0002) {
-				should_associate = 1;
-			}
-		} else if (system == WLAN_AUTH_SHARED_KEY) {
-			if (trans_seq_no == 0x0002 &&
-			    auth->el_id == WLAN_EID_CHALLENGE) {
-				send_authentication_request(priv, system, auth->chall_text, auth->chall_text_len);
-				return;
-			} else if (trans_seq_no == 0x0004) {
-				should_associate = 1;
-			}
-		}
-
-		if (should_associate) {
-			if (priv->station_was_associated) {
-				atmel_enter_state(priv, STATION_STATE_REASSOCIATING);
-				send_association_request(priv, 1);
-				return;
-			} else {
-				atmel_enter_state(priv, STATION_STATE_ASSOCIATING);
-				send_association_request(priv, 0);
-				return;
-			}
-		}
-	}
-
-	if (status == WLAN_STATUS_NOT_SUPPORTED_AUTH_ALG) {
-		/* Flip back and forth between WEP auth modes until the max
-		 * authentication tries has been exceeded.
-		 */
-		if (system == WLAN_AUTH_OPEN) {
-			priv->CurrentAuthentTransactionSeqNum = 0x001;
-			priv->exclude_unencrypted = 1;
-			send_authentication_request(priv, WLAN_AUTH_SHARED_KEY, NULL, 0);
-			return;
-		} else if (system == WLAN_AUTH_SHARED_KEY
-			   && priv->wep_is_on) {
-			priv->CurrentAuthentTransactionSeqNum = 0x001;
-			priv->exclude_unencrypted = 0;
-			send_authentication_request(priv, WLAN_AUTH_OPEN, NULL, 0);
-			return;
-		} else if (priv->connect_to_any_BSS) {
-			int bss_index;
-
-			priv->BSSinfo[(int)(priv->current_BSS)].channel |= 0x80;
-
-			if ((bss_index  = retrieve_bss(priv)) != -1) {
-				atmel_join_bss(priv, bss_index);
-				return;
-			}
-		}
-	}
-
-	priv->AuthenticationRequestRetryCnt = 0;
-	atmel_enter_state(priv,  STATION_STATE_MGMT_ERROR);
-	priv->station_is_associated = 0;
-}
-
-static void associate(struct atmel_private *priv, u16 frame_len, u16 subtype)
-{
-	struct ass_resp_format {
-		__le16 capability;
-		__le16 status;
-		__le16 ass_id;
-		u8 el_id;
-		u8 length;
-		u8 rates[4];
-	} *ass_resp = (struct ass_resp_format *)priv->rx_buf;
-
-	u16 status = le16_to_cpu(ass_resp->status);
-	u16 ass_id = le16_to_cpu(ass_resp->ass_id);
-	u16 rates_len = ass_resp->length > 4 ? 4 : ass_resp->length;
-
-	union iwreq_data wrqu;
-
-	if (frame_len < 8 + rates_len)
-		return;
-
-	if (status == WLAN_STATUS_SUCCESS) {
-		if (subtype == IEEE80211_STYPE_ASSOC_RESP)
-			priv->AssociationRequestRetryCnt = 0;
-		else
-			priv->ReAssociationRequestRetryCnt = 0;
-
-		atmel_set_mib16(priv, Mac_Mgmt_Mib_Type,
-				MAC_MGMT_MIB_STATION_ID_POS, ass_id & 0x3fff);
-		atmel_set_mib(priv, Phy_Mib_Type,
-			      PHY_MIB_RATE_SET_POS, ass_resp->rates, rates_len);
-		if (priv->power_mode == 0) {
-			priv->listen_interval = 1;
-			atmel_set_mib8(priv, Mac_Mgmt_Mib_Type,
-				       MAC_MGMT_MIB_PS_MODE_POS, ACTIVE_MODE);
-			atmel_set_mib16(priv, Mac_Mgmt_Mib_Type,
-					MAC_MGMT_MIB_LISTEN_INTERVAL_POS, 1);
-		} else {
-			priv->listen_interval = 2;
-			atmel_set_mib8(priv, Mac_Mgmt_Mib_Type,
-				       MAC_MGMT_MIB_PS_MODE_POS,  PS_MODE);
-			atmel_set_mib16(priv, Mac_Mgmt_Mib_Type,
-					MAC_MGMT_MIB_LISTEN_INTERVAL_POS, 2);
-		}
-
-		priv->station_is_associated = 1;
-		priv->station_was_associated = 1;
-		atmel_enter_state(priv, STATION_STATE_READY);
-
-		/* Send association event to userspace */
-		wrqu.data.length = 0;
-		wrqu.data.flags = 0;
-		memcpy(wrqu.ap_addr.sa_data, priv->CurrentBSSID, ETH_ALEN);
-		wrqu.ap_addr.sa_family = ARPHRD_ETHER;
-		wireless_send_event(priv->dev, SIOCGIWAP, &wrqu, NULL);
-
-		return;
-	}
-
-	if (subtype == IEEE80211_STYPE_ASSOC_RESP &&
-	    status != WLAN_STATUS_ASSOC_DENIED_RATES &&
-	    status != WLAN_STATUS_CAPS_UNSUPPORTED &&
-	    priv->AssociationRequestRetryCnt < MAX_ASSOCIATION_RETRIES) {
-		mod_timer(&priv->management_timer, jiffies + MGMT_JIFFIES);
-		priv->AssociationRequestRetryCnt++;
-		send_association_request(priv, 0);
-		return;
-	}
-
-	if (subtype == IEEE80211_STYPE_REASSOC_RESP &&
-	    status != WLAN_STATUS_ASSOC_DENIED_RATES &&
-	    status != WLAN_STATUS_CAPS_UNSUPPORTED &&
-	    priv->ReAssociationRequestRetryCnt < MAX_ASSOCIATION_RETRIES) {
-		mod_timer(&priv->management_timer, jiffies + MGMT_JIFFIES);
-		priv->ReAssociationRequestRetryCnt++;
-		send_association_request(priv, 1);
-		return;
-	}
-
-	atmel_enter_state(priv,  STATION_STATE_MGMT_ERROR);
-	priv->station_is_associated = 0;
-
-	if (priv->connect_to_any_BSS) {
-		int bss_index;
-		priv->BSSinfo[(int)(priv->current_BSS)].channel |= 0x80;
-
-		if ((bss_index = retrieve_bss(priv)) != -1)
-			atmel_join_bss(priv, bss_index);
-	}
-}
-
-static void atmel_join_bss(struct atmel_private *priv, int bss_index)
-{
-	struct bss_info *bss =  &priv->BSSinfo[bss_index];
-
-	memcpy(priv->CurrentBSSID, bss->BSSID, ETH_ALEN);
-	memcpy(priv->SSID, bss->SSID, priv->SSID_size = bss->SSIDsize);
-
-	/* The WPA stuff cares about the current AP address */
-	if (priv->use_wpa)
-		build_wpa_mib(priv);
-
-	/* When switching to AdHoc turn OFF Power Save if needed */
-
-	if (bss->BSStype == IW_MODE_ADHOC &&
-	    priv->operating_mode != IW_MODE_ADHOC &&
-	    priv->power_mode) {
-		priv->power_mode = 0;
-		priv->listen_interval = 1;
-		atmel_set_mib8(priv, Mac_Mgmt_Mib_Type,
-			       MAC_MGMT_MIB_PS_MODE_POS,  ACTIVE_MODE);
-		atmel_set_mib16(priv, Mac_Mgmt_Mib_Type,
-				MAC_MGMT_MIB_LISTEN_INTERVAL_POS, 1);
-	}
-
-	priv->operating_mode = bss->BSStype;
-	priv->channel = bss->channel & 0x7f;
-	priv->beacon_period = bss->beacon_period;
-
-	if (priv->preamble != bss->preamble) {
-		priv->preamble = bss->preamble;
-		atmel_set_mib8(priv, Local_Mib_Type,
-			       LOCAL_MIB_PREAMBLE_TYPE, bss->preamble);
-	}
-
-	if (!priv->wep_is_on && bss->UsingWEP) {
-		atmel_enter_state(priv, STATION_STATE_MGMT_ERROR);
-		priv->station_is_associated = 0;
-		return;
-	}
-
-	if (priv->wep_is_on && !bss->UsingWEP) {
-		atmel_enter_state(priv, STATION_STATE_MGMT_ERROR);
-		priv->station_is_associated = 0;
-		return;
-	}
-
-	atmel_enter_state(priv, STATION_STATE_JOINNING);
-
-	if (priv->operating_mode == IW_MODE_INFRA)
-		join(priv, BSS_TYPE_INFRASTRUCTURE);
-	else
-		join(priv, BSS_TYPE_AD_HOC);
-}
-
-static void restart_search(struct atmel_private *priv)
-{
-	int bss_index;
-
-	if (!priv->connect_to_any_BSS) {
-		atmel_scan(priv, 1);
-	} else {
-		priv->BSSinfo[(int)(priv->current_BSS)].channel |= 0x80;
-
-		if ((bss_index = retrieve_bss(priv)) != -1)
-			atmel_join_bss(priv, bss_index);
-		else
-			atmel_scan(priv, 0);
-	}
-}
-
-static void smooth_rssi(struct atmel_private *priv, u8 rssi)
-{
-	u8 old = priv->wstats.qual.level;
-	u8 max_rssi = 42; /* 502-rmfd-revd max by experiment, default for now */
-
-	switch (priv->firmware_type) {
-	case ATMEL_FW_TYPE_502E:
-		max_rssi = 63; /* 502-rmfd-reve max by experiment */
-		break;
-	default:
-		break;
-	}
-
-	rssi = rssi * 100 / max_rssi;
-	if ((rssi + old) % 2)
-		priv->wstats.qual.level = (rssi + old) / 2 + 1;
-	else
-		priv->wstats.qual.level = (rssi + old) / 2;
-	priv->wstats.qual.updated |= IW_QUAL_LEVEL_UPDATED;
-	priv->wstats.qual.updated &= ~IW_QUAL_LEVEL_INVALID;
-}
-
-static void atmel_smooth_qual(struct atmel_private *priv)
-{
-	unsigned long time_diff = (jiffies - priv->last_qual) / HZ;
-	while (time_diff--) {
-		priv->last_qual += HZ;
-		priv->wstats.qual.qual = priv->wstats.qual.qual / 2;
-		priv->wstats.qual.qual +=
-			priv->beacons_this_sec * priv->beacon_period * (priv->wstats.qual.level + 100) / 4000;
-		priv->beacons_this_sec = 0;
-	}
-	priv->wstats.qual.updated |= IW_QUAL_QUAL_UPDATED;
-	priv->wstats.qual.updated &= ~IW_QUAL_QUAL_INVALID;
-}
-
-/* deals with incoming management frames. */
-static void atmel_management_frame(struct atmel_private *priv,
-				   struct ieee80211_hdr *header,
-				   u16 frame_len, u8 rssi)
-{
-	u16 subtype;
-
-	subtype = le16_to_cpu(header->frame_control) & IEEE80211_FCTL_STYPE;
-	switch (subtype) {
-	case IEEE80211_STYPE_BEACON:
-	case IEEE80211_STYPE_PROBE_RESP:
-
-		/* beacon frame has multiple variable-length fields -
-		   never let an engineer loose with a data structure design. */
-		{
-			struct beacon_format {
-				__le64 timestamp;
-				__le16 interval;
-				__le16 capability;
-				u8 ssid_el_id;
-				u8 ssid_length;
-				/* ssid here */
-				u8 rates_el_id;
-				u8 rates_length;
-				/* rates here */
-				u8 ds_el_id;
-				u8 ds_length;
-				/* ds here */
-			} *beacon = (struct beacon_format *)priv->rx_buf;
-
-			u8 channel, rates_length, ssid_length;
-			u64 timestamp = le64_to_cpu(beacon->timestamp);
-			u16 beacon_interval = le16_to_cpu(beacon->interval);
-			u16 capability = le16_to_cpu(beacon->capability);
-			u8 *beaconp = priv->rx_buf;
-			ssid_length = beacon->ssid_length;
-			/* this blows chunks. */
-			if (frame_len < 14 || frame_len < ssid_length + 15)
-				return;
-			rates_length = beaconp[beacon->ssid_length + 15];
-			if (frame_len < ssid_length + rates_length + 18)
-				return;
-			if (ssid_length >  MAX_SSID_LENGTH)
-				return;
-			channel = beaconp[ssid_length + rates_length + 18];
-
-			if (priv->station_state == STATION_STATE_READY) {
-				smooth_rssi(priv, rssi);
-				if (is_frame_from_current_bss(priv, header)) {
-					priv->beacons_this_sec++;
-					atmel_smooth_qual(priv);
-					if (priv->last_beacon_timestamp) {
-						/* Note truncate this to 32 bits - kernel can't divide a long */
-						u32 beacon_delay = timestamp - priv->last_beacon_timestamp;
-						int beacons = beacon_delay / (beacon_interval * 1000);
-						if (beacons > 1)
-							priv->wstats.miss.beacon += beacons - 1;
-					}
-					priv->last_beacon_timestamp = timestamp;
-					handle_beacon_probe(priv, capability, channel);
-				}
-			}
-
-			if (priv->station_state == STATION_STATE_SCANNING)
-				store_bss_info(priv, header, capability,
-					       beacon_interval, channel, rssi,
-					       ssid_length,
-					       &beacon->rates_el_id,
-					       subtype == IEEE80211_STYPE_BEACON);
-		}
-		break;
-
-	case IEEE80211_STYPE_AUTH:
-
-		if (priv->station_state == STATION_STATE_AUTHENTICATING)
-			authenticate(priv, frame_len);
-
-		break;
-
-	case IEEE80211_STYPE_ASSOC_RESP:
-	case IEEE80211_STYPE_REASSOC_RESP:
-
-		if (priv->station_state == STATION_STATE_ASSOCIATING ||
-		    priv->station_state == STATION_STATE_REASSOCIATING)
-			associate(priv, frame_len, subtype);
-
-		break;
-
-	case IEEE80211_STYPE_DISASSOC:
-		if (priv->station_is_associated &&
-		    priv->operating_mode == IW_MODE_INFRA &&
-		    is_frame_from_current_bss(priv, header)) {
-			priv->station_was_associated = 0;
-			priv->station_is_associated = 0;
-
-			atmel_enter_state(priv, STATION_STATE_JOINNING);
-			join(priv, BSS_TYPE_INFRASTRUCTURE);
-		}
-
-		break;
-
-	case IEEE80211_STYPE_DEAUTH:
-		if (priv->operating_mode == IW_MODE_INFRA &&
-		    is_frame_from_current_bss(priv, header)) {
-			priv->station_was_associated = 0;
-
-			atmel_enter_state(priv, STATION_STATE_JOINNING);
-			join(priv, BSS_TYPE_INFRASTRUCTURE);
-		}
-
-		break;
-	}
-}
-
-/* run when timer expires */
-static void atmel_management_timer(struct timer_list *t)
-{
-	struct atmel_private *priv = from_timer(priv, t, management_timer);
-	unsigned long flags;
-
-	/* Check if the card has been yanked. */
-	if (priv->card && priv->present_callback &&
-		!(*priv->present_callback)(priv->card))
-		return;
-
-	spin_lock_irqsave(&priv->irqlock, flags);
-
-	switch (priv->station_state) {
-
-	case STATION_STATE_AUTHENTICATING:
-		if (priv->AuthenticationRequestRetryCnt >= MAX_AUTHENTICATION_RETRIES) {
-			atmel_enter_state(priv, STATION_STATE_MGMT_ERROR);
-			priv->station_is_associated = 0;
-			priv->AuthenticationRequestRetryCnt = 0;
-			restart_search(priv);
-		} else {
-			int auth = WLAN_AUTH_OPEN;
-			priv->AuthenticationRequestRetryCnt++;
-			priv->CurrentAuthentTransactionSeqNum = 0x0001;
-			mod_timer(&priv->management_timer, jiffies + MGMT_JIFFIES);
-			if (priv->wep_is_on && priv->exclude_unencrypted)
-				auth = WLAN_AUTH_SHARED_KEY;
-			send_authentication_request(priv, auth, NULL, 0);
-	  }
-	  break;
-
-	case STATION_STATE_ASSOCIATING:
-		if (priv->AssociationRequestRetryCnt == MAX_ASSOCIATION_RETRIES) {
-			atmel_enter_state(priv, STATION_STATE_MGMT_ERROR);
-			priv->station_is_associated = 0;
-			priv->AssociationRequestRetryCnt = 0;
-			restart_search(priv);
-		} else {
-			priv->AssociationRequestRetryCnt++;
-			mod_timer(&priv->management_timer, jiffies + MGMT_JIFFIES);
-			send_association_request(priv, 0);
-		}
-	  break;
-
-	case STATION_STATE_REASSOCIATING:
-		if (priv->ReAssociationRequestRetryCnt == MAX_ASSOCIATION_RETRIES) {
-			atmel_enter_state(priv, STATION_STATE_MGMT_ERROR);
-			priv->station_is_associated = 0;
-			priv->ReAssociationRequestRetryCnt = 0;
-			restart_search(priv);
-		} else {
-			priv->ReAssociationRequestRetryCnt++;
-			mod_timer(&priv->management_timer, jiffies + MGMT_JIFFIES);
-			send_association_request(priv, 1);
-		}
-		break;
-
-	default:
-		break;
-	}
-
-	spin_unlock_irqrestore(&priv->irqlock, flags);
-}
-
-static void atmel_command_irq(struct atmel_private *priv)
-{
-	u8 status = atmel_rmem8(priv, atmel_co(priv, CMD_BLOCK_STATUS_OFFSET));
-	u8 command = atmel_rmem8(priv, atmel_co(priv, CMD_BLOCK_COMMAND_OFFSET));
-	int fast_scan;
-	union iwreq_data wrqu;
-
-	if (status == CMD_STATUS_IDLE ||
-	    status == CMD_STATUS_IN_PROGRESS)
-		return;
-
-	switch (command) {
-	case CMD_Start:
-		if (status == CMD_STATUS_COMPLETE) {
-			priv->station_was_associated = priv->station_is_associated;
-			atmel_get_mib(priv, Mac_Mgmt_Mib_Type, MAC_MGMT_MIB_CUR_BSSID_POS,
-				      (u8 *)priv->CurrentBSSID, 6);
-			atmel_enter_state(priv, STATION_STATE_READY);
-		}
-		break;
-
-	case CMD_Scan:
-		fast_scan = priv->fast_scan;
-		priv->fast_scan = 0;
-
-		if (status != CMD_STATUS_COMPLETE) {
-			atmel_scan(priv, 1);
-		} else {
-			int bss_index = retrieve_bss(priv);
-			int notify_scan_complete = 1;
-			if (bss_index != -1) {
-				atmel_join_bss(priv, bss_index);
-			} else if (priv->operating_mode == IW_MODE_ADHOC &&
-				   priv->SSID_size != 0) {
-				start(priv, BSS_TYPE_AD_HOC);
-			} else {
-				priv->fast_scan = !fast_scan;
-				atmel_scan(priv, 1);
-				notify_scan_complete = 0;
-			}
-			priv->site_survey_state = SITE_SURVEY_COMPLETED;
-			if (notify_scan_complete) {
-				wrqu.data.length = 0;
-				wrqu.data.flags = 0;
-				wireless_send_event(priv->dev, SIOCGIWSCAN, &wrqu, NULL);
-			}
-		}
-		break;
-
-	case CMD_SiteSurvey:
-		priv->fast_scan = 0;
-
-		if (status != CMD_STATUS_COMPLETE)
-			return;
-
-		priv->site_survey_state = SITE_SURVEY_COMPLETED;
-		if (priv->station_is_associated) {
-			atmel_enter_state(priv, STATION_STATE_READY);
-			wrqu.data.length = 0;
-			wrqu.data.flags = 0;
-			wireless_send_event(priv->dev, SIOCGIWSCAN, &wrqu, NULL);
-		} else {
-			atmel_scan(priv, 1);
-		}
-		break;
-
-	case CMD_Join:
-		if (status == CMD_STATUS_COMPLETE) {
-			if (priv->operating_mode == IW_MODE_ADHOC) {
-				priv->station_was_associated = priv->station_is_associated;
-				atmel_enter_state(priv, STATION_STATE_READY);
-			} else {
-				int auth = WLAN_AUTH_OPEN;
-				priv->AuthenticationRequestRetryCnt = 0;
-				atmel_enter_state(priv, STATION_STATE_AUTHENTICATING);
-
-				mod_timer(&priv->management_timer, jiffies + MGMT_JIFFIES);
-				priv->CurrentAuthentTransactionSeqNum = 0x0001;
-				if (priv->wep_is_on && priv->exclude_unencrypted)
-					auth = WLAN_AUTH_SHARED_KEY;
-				send_authentication_request(priv, auth, NULL, 0);
-			}
-			return;
-		}
-
-		atmel_scan(priv, 1);
-	}
-}
-
-static int atmel_wakeup_firmware(struct atmel_private *priv)
-{
-	struct host_info_struct *iface = &priv->host_info;
-	u16 mr1, mr3;
-	int i;
-
-	if (priv->card_type == CARD_TYPE_SPI_FLASH)
-		atmel_set_gcr(priv->dev, GCR_REMAP);
-
-	/* wake up on-board processor */
-	atmel_clear_gcr(priv->dev, 0x0040);
-	atmel_write16(priv->dev, BSR, BSS_SRAM);
-
-	if (priv->card_type == CARD_TYPE_SPI_FLASH)
-		mdelay(100);
-
-	/* and wait for it */
-	for (i = LOOP_RETRY_LIMIT; i; i--) {
-		mr1 = atmel_read16(priv->dev, MR1);
-		mr3 = atmel_read16(priv->dev, MR3);
-
-		if (mr3 & MAC_BOOT_COMPLETE)
-			break;
-		if (mr1 & MAC_BOOT_COMPLETE &&
-		    priv->bus_type == BUS_TYPE_PCCARD)
-			break;
-	}
-
-	if (i == 0) {
-		printk(KERN_ALERT "%s: MAC failed to boot.\n", priv->dev->name);
-		return -EIO;
-	}
-
-	if ((priv->host_info_base = atmel_read16(priv->dev, MR2)) == 0xffff) {
-		printk(KERN_ALERT "%s: card missing.\n", priv->dev->name);
-		return -ENODEV;
-	}
-
-	/* now check for completion of MAC initialization through
-	   the FunCtrl field of the IFACE, poll MR1 to detect completion of
-	   MAC initialization, check completion status, set interrupt mask,
-	   enables interrupts and calls Tx and Rx initialization functions */
-
-	atmel_wmem8(priv, atmel_hi(priv, IFACE_FUNC_CTRL_OFFSET), FUNC_CTRL_INIT_COMPLETE);
-
-	for (i = LOOP_RETRY_LIMIT; i; i--) {
-		mr1 = atmel_read16(priv->dev, MR1);
-		mr3 = atmel_read16(priv->dev, MR3);
-
-		if (mr3 & MAC_INIT_COMPLETE)
-			break;
-		if (mr1 & MAC_INIT_COMPLETE &&
-		    priv->bus_type == BUS_TYPE_PCCARD)
-			break;
-	}
-
-	if (i == 0) {
-		printk(KERN_ALERT "%s: MAC failed to initialise.\n",
-				priv->dev->name);
-		return -EIO;
-	}
-
-	/* Check for MAC_INIT_OK only on the register that the MAC_INIT_OK was set */
-	if ((mr3 & MAC_INIT_COMPLETE) &&
-	    !(atmel_read16(priv->dev, MR3) & MAC_INIT_OK)) {
-		printk(KERN_ALERT "%s: MAC failed MR3 self-test.\n", priv->dev->name);
-		return -EIO;
-	}
-	if ((mr1 & MAC_INIT_COMPLETE) &&
-	    !(atmel_read16(priv->dev, MR1) & MAC_INIT_OK)) {
-		printk(KERN_ALERT "%s: MAC failed MR1 self-test.\n", priv->dev->name);
-		return -EIO;
-	}
-
-	atmel_copy_to_host(priv->dev, (unsigned char *)iface,
-			   priv->host_info_base, sizeof(*iface));
-
-	iface->tx_buff_pos = le16_to_cpu(iface->tx_buff_pos);
-	iface->tx_buff_size = le16_to_cpu(iface->tx_buff_size);
-	iface->tx_desc_pos = le16_to_cpu(iface->tx_desc_pos);
-	iface->tx_desc_count = le16_to_cpu(iface->tx_desc_count);
-	iface->rx_buff_pos = le16_to_cpu(iface->rx_buff_pos);
-	iface->rx_buff_size = le16_to_cpu(iface->rx_buff_size);
-	iface->rx_desc_pos = le16_to_cpu(iface->rx_desc_pos);
-	iface->rx_desc_count = le16_to_cpu(iface->rx_desc_count);
-	iface->build_version = le16_to_cpu(iface->build_version);
-	iface->command_pos = le16_to_cpu(iface->command_pos);
-	iface->major_version = le16_to_cpu(iface->major_version);
-	iface->minor_version = le16_to_cpu(iface->minor_version);
-	iface->func_ctrl = le16_to_cpu(iface->func_ctrl);
-	iface->mac_status = le16_to_cpu(iface->mac_status);
-
-	return 0;
-}
-
-/* determine type of memory and MAC address */
-static int probe_atmel_card(struct net_device *dev)
-{
-	int rc = 0;
-	struct atmel_private *priv = netdev_priv(dev);
-	u8 addr[ETH_ALEN] = {};
-
-	/* reset pccard */
-	if (priv->bus_type == BUS_TYPE_PCCARD)
-		atmel_write16(dev, GCR, 0x0060);
-
-	atmel_write16(dev, GCR, 0x0040);
-	msleep(500);
-
-	if (atmel_read16(dev, MR2) == 0) {
-		/* No stored firmware so load a small stub which just
-		   tells us the MAC address */
-		int i;
-		priv->card_type = CARD_TYPE_EEPROM;
-		atmel_write16(dev, BSR, BSS_IRAM);
-		atmel_copy_to_card(dev, 0, mac_reader, sizeof(mac_reader));
-		atmel_set_gcr(dev, GCR_REMAP);
-		atmel_clear_gcr(priv->dev, 0x0040);
-		atmel_write16(dev, BSR, BSS_SRAM);
-		for (i = LOOP_RETRY_LIMIT; i; i--)
-			if (atmel_read16(dev, MR3) & MAC_BOOT_COMPLETE)
-				break;
-		if (i == 0) {
-			printk(KERN_ALERT "%s: MAC failed to boot MAC address reader.\n", dev->name);
-		} else {
-
-			atmel_copy_to_host(dev, addr, atmel_read16(dev, MR2), 6);
-			eth_hw_addr_set(dev, addr);
-			/* got address, now squash it again until the network
-			   interface is opened */
-			if (priv->bus_type == BUS_TYPE_PCCARD)
-				atmel_write16(dev, GCR, 0x0060);
-			atmel_write16(dev, GCR, 0x0040);
-			rc = 1;
-		}
-	} else if (atmel_read16(dev, MR4) == 0) {
-		/* Mac address easy in this case. */
-		priv->card_type = CARD_TYPE_PARALLEL_FLASH;
-		atmel_write16(dev,  BSR, 1);
-		atmel_copy_to_host(dev, addr, 0xc000, 6);
-		eth_hw_addr_set(dev, addr);
-		atmel_write16(dev,  BSR, 0x200);
-		rc = 1;
-	} else {
-		/* Standard firmware in flash, boot it up and ask
-		   for the Mac Address */
-		priv->card_type = CARD_TYPE_SPI_FLASH;
-		if (atmel_wakeup_firmware(priv) == 0) {
-			atmel_get_mib(priv, Mac_Address_Mib_Type, 0, addr, 6);
-			eth_hw_addr_set(dev, addr);
-
-			/* got address, now squash it again until the network
-			   interface is opened */
-			if (priv->bus_type == BUS_TYPE_PCCARD)
-				atmel_write16(dev, GCR, 0x0060);
-			atmel_write16(dev, GCR, 0x0040);
-			rc = 1;
-		}
-	}
-
-	if (rc) {
-		if (dev->dev_addr[0] == 0xFF) {
-			static const u8 default_mac[] = {
-				0x00, 0x04, 0x25, 0x00, 0x00, 0x00
-			};
-			printk(KERN_ALERT "%s: *** Invalid MAC address. UPGRADE Firmware ****\n", dev->name);
-			eth_hw_addr_set(dev, default_mac);
-		}
-	}
-
-	return rc;
-}
-
-/* Move the encyption information on the MIB structure.
-   This routine is for the pre-WPA firmware: later firmware has
-   a different format MIB and a different routine. */
-static void build_wep_mib(struct atmel_private *priv)
-{
-	struct { /* NB this is matched to the hardware, don't change. */
-		u8 wep_is_on;
-		u8 default_key; /* 0..3 */
-		u8 reserved;
-		u8 exclude_unencrypted;
-
-		u32 WEPICV_error_count;
-		u32 WEP_excluded_count;
-
-		u8 wep_keys[MAX_ENCRYPTION_KEYS][13];
-		u8 encryption_level; /* 0, 1, 2 */
-		u8 reserved2[3];
-	} mib;
-	int i;
-
-	mib.wep_is_on = priv->wep_is_on;
-	if (priv->wep_is_on) {
-		if (priv->wep_key_len[priv->default_key] > 5)
-			mib.encryption_level = 2;
-		else
-			mib.encryption_level = 1;
-	} else {
-		mib.encryption_level = 0;
-	}
-
-	mib.default_key = priv->default_key;
-	mib.exclude_unencrypted = priv->exclude_unencrypted;
-
-	for (i = 0; i < MAX_ENCRYPTION_KEYS; i++)
-		memcpy(mib.wep_keys[i], priv->wep_keys[i], 13);
-
-	atmel_set_mib(priv, Mac_Wep_Mib_Type, 0, (u8 *)&mib, sizeof(mib));
-}
-
-static void build_wpa_mib(struct atmel_private *priv)
-{
-	/* This is for the later (WPA enabled) firmware. */
-
-	struct { /* NB this is matched to the hardware, don't change. */
-		u8 cipher_default_key_value[MAX_ENCRYPTION_KEYS][MAX_ENCRYPTION_KEY_SIZE];
-		u8 receiver_address[ETH_ALEN];
-		u8 wep_is_on;
-		u8 default_key; /* 0..3 */
-		u8 group_key;
-		u8 exclude_unencrypted;
-		u8 encryption_type;
-		u8 reserved;
-
-		u32 WEPICV_error_count;
-		u32 WEP_excluded_count;
-
-		u8 key_RSC[4][8];
-	} mib;
-
-	int i;
-
-	mib.wep_is_on = priv->wep_is_on;
-	mib.exclude_unencrypted = priv->exclude_unencrypted;
-	memcpy(mib.receiver_address, priv->CurrentBSSID, ETH_ALEN);
-
-	/* zero all the keys before adding in valid ones. */
-	memset(mib.cipher_default_key_value, 0, sizeof(mib.cipher_default_key_value));
-
-	if (priv->wep_is_on) {
-		/* There's a comment in the Atmel code to the effect that this
-		   is only valid when still using WEP, it may need to be set to
-		   something to use WPA */
-		memset(mib.key_RSC, 0, sizeof(mib.key_RSC));
-
-		mib.default_key = mib.group_key = 255;
-		for (i = 0; i < MAX_ENCRYPTION_KEYS; i++) {
-			if (priv->wep_key_len[i] > 0) {
-				memcpy(mib.cipher_default_key_value[i], priv->wep_keys[i], MAX_ENCRYPTION_KEY_SIZE);
-				if (i == priv->default_key) {
-					mib.default_key = i;
-					mib.cipher_default_key_value[i][MAX_ENCRYPTION_KEY_SIZE-1] = 7;
-					mib.cipher_default_key_value[i][MAX_ENCRYPTION_KEY_SIZE-2] = priv->pairwise_cipher_suite;
-				} else {
-					mib.group_key = i;
-					priv->group_cipher_suite = priv->pairwise_cipher_suite;
-					mib.cipher_default_key_value[i][MAX_ENCRYPTION_KEY_SIZE-1] = 1;
-					mib.cipher_default_key_value[i][MAX_ENCRYPTION_KEY_SIZE-2] = priv->group_cipher_suite;
-				}
-			}
-		}
-		if (mib.default_key == 255)
-			mib.default_key = mib.group_key != 255 ? mib.group_key : 0;
-		if (mib.group_key == 255)
-			mib.group_key = mib.default_key;
-
-	}
-
-	atmel_set_mib(priv, Mac_Wep_Mib_Type, 0, (u8 *)&mib, sizeof(mib));
-}
-
-static int reset_atmel_card(struct net_device *dev)
-{
-	/* do everything necessary to wake up the hardware, including
-	   waiting for the lightning strike and throwing the knife switch....
-
-	   set all the Mib values which matter in the card to match
-	   their settings in the atmel_private structure. Some of these
-	   can be altered on the fly, but many (WEP, infrastructure or ad-hoc)
-	   can only be changed by tearing down the world and coming back through
-	   here.
-
-	   This routine is also responsible for initialising some
-	   hardware-specific fields in the atmel_private structure,
-	   including a copy of the firmware's hostinfo structure
-	   which is the route into the rest of the firmware datastructures. */
-
-	struct atmel_private *priv = netdev_priv(dev);
-	u8 configuration;
-	int old_state = priv->station_state;
-	int err = 0;
-
-	/* data to add to the firmware names, in priority order
-	   this implemenents firmware versioning */
-
-	static char *firmware_modifier[] = {
-		"-wpa",
-		"",
-		NULL
-	};
-
-	/* reset pccard */
-	if (priv->bus_type == BUS_TYPE_PCCARD)
-		atmel_write16(priv->dev, GCR, 0x0060);
-
-	/* stop card , disable interrupts */
-	atmel_write16(priv->dev, GCR, 0x0040);
-
-	if (priv->card_type == CARD_TYPE_EEPROM) {
-		/* copy in firmware if needed */
-		const struct firmware *fw_entry = NULL;
-		const unsigned char *fw;
-		int len = priv->firmware_length;
-		if (!(fw = priv->firmware)) {
-			if (priv->firmware_type == ATMEL_FW_TYPE_NONE) {
-				if (strlen(priv->firmware_id) == 0) {
-					printk(KERN_INFO
-					       "%s: card type is unknown: assuming at76c502 firmware is OK.\n",
-					       dev->name);
-					printk(KERN_INFO
-					       "%s: if not, use the firmware= module parameter.\n",
-					       dev->name);
-					strcpy(priv->firmware_id, "atmel_at76c502.bin");
-				}
-				err = request_firmware(&fw_entry, priv->firmware_id, priv->sys_dev);
-				if (err != 0) {
-					printk(KERN_ALERT
-					       "%s: firmware %s is missing, cannot continue.\n",
-					       dev->name, priv->firmware_id);
-					return err;
-				}
-			} else {
-				int fw_index = 0;
-				int success = 0;
-
-				/* get firmware filename entry based on firmware type ID */
-				while (fw_table[fw_index].fw_type != priv->firmware_type
-						&& fw_table[fw_index].fw_type != ATMEL_FW_TYPE_NONE)
-					fw_index++;
-
-				/* construct the actual firmware file name */
-				if (fw_table[fw_index].fw_type != ATMEL_FW_TYPE_NONE) {
-					int i;
-					for (i = 0; firmware_modifier[i]; i++) {
-						snprintf(priv->firmware_id, 32, "%s%s.%s", fw_table[fw_index].fw_file,
-							firmware_modifier[i], fw_table[fw_index].fw_file_ext);
-						priv->firmware_id[31] = '\0';
-						if (request_firmware(&fw_entry, priv->firmware_id, priv->sys_dev) == 0) {
-							success = 1;
-							break;
-						}
-					}
-				}
-				if (!success) {
-					printk(KERN_ALERT
-					       "%s: firmware %s is missing, cannot start.\n",
-					       dev->name, priv->firmware_id);
-					priv->firmware_id[0] = '\0';
-					return -ENOENT;
-				}
-			}
-
-			fw = fw_entry->data;
-			len = fw_entry->size;
-		}
-
-		if (len <= 0x6000) {
-			atmel_write16(priv->dev, BSR, BSS_IRAM);
-			atmel_copy_to_card(priv->dev, 0, fw, len);
-			atmel_set_gcr(priv->dev, GCR_REMAP);
-		} else {
-			/* Remap */
-			atmel_set_gcr(priv->dev, GCR_REMAP);
-			atmel_write16(priv->dev, BSR, BSS_IRAM);
-			atmel_copy_to_card(priv->dev, 0, fw, 0x6000);
-			atmel_write16(priv->dev, BSR, 0x2ff);
-			atmel_copy_to_card(priv->dev, 0x8000, &fw[0x6000], len - 0x6000);
-		}
-
-		release_firmware(fw_entry);
-	}
-
-	err = atmel_wakeup_firmware(priv);
-	if (err != 0)
-		return err;
-
-	/* Check the version and set the correct flag for wpa stuff,
-	   old and new firmware is incompatible.
-	   The pre-wpa 3com firmware reports major version 5,
-	   the wpa 3com firmware is major version 4 and doesn't need
-	   the 3com broken-ness filter. */
-	priv->use_wpa = (priv->host_info.major_version == 4);
-	priv->radio_on_broken = (priv->host_info.major_version == 5);
-
-	/* unmask all irq sources */
-	atmel_wmem8(priv, atmel_hi(priv, IFACE_INT_MASK_OFFSET), 0xff);
-
-	/* int Tx system and enable Tx */
-	atmel_wmem8(priv, atmel_tx(priv, TX_DESC_FLAGS_OFFSET, 0), 0);
-	atmel_wmem32(priv, atmel_tx(priv, TX_DESC_NEXT_OFFSET, 0), 0x80000000L);
-	atmel_wmem16(priv, atmel_tx(priv, TX_DESC_POS_OFFSET, 0), 0);
-	atmel_wmem16(priv, atmel_tx(priv, TX_DESC_SIZE_OFFSET, 0), 0);
-
-	priv->tx_desc_free = priv->host_info.tx_desc_count;
-	priv->tx_desc_head = 0;
-	priv->tx_desc_tail = 0;
-	priv->tx_desc_previous = 0;
-	priv->tx_free_mem = priv->host_info.tx_buff_size;
-	priv->tx_buff_head = 0;
-	priv->tx_buff_tail = 0;
-
-	configuration = atmel_rmem8(priv, atmel_hi(priv, IFACE_FUNC_CTRL_OFFSET));
-	atmel_wmem8(priv, atmel_hi(priv, IFACE_FUNC_CTRL_OFFSET),
-				   configuration | FUNC_CTRL_TxENABLE);
-
-	/* init Rx system and enable */
-	priv->rx_desc_head = 0;
-
-	configuration = atmel_rmem8(priv, atmel_hi(priv, IFACE_FUNC_CTRL_OFFSET));
-	atmel_wmem8(priv, atmel_hi(priv, IFACE_FUNC_CTRL_OFFSET),
-				   configuration | FUNC_CTRL_RxENABLE);
-
-	if (!priv->radio_on_broken) {
-		if (atmel_send_command_wait(priv, CMD_EnableRadio, NULL, 0) ==
-		    CMD_STATUS_REJECTED_RADIO_OFF) {
-			printk(KERN_INFO "%s: cannot turn the radio on.\n",
-			       dev->name);
-			return -EIO;
-		}
-	}
-
-	/* set up enough MIB values to run. */
-	atmel_set_mib8(priv, Local_Mib_Type, LOCAL_MIB_AUTO_TX_RATE_POS, priv->auto_tx_rate);
-	atmel_set_mib8(priv, Local_Mib_Type,  LOCAL_MIB_TX_PROMISCUOUS_POS,  PROM_MODE_OFF);
-	atmel_set_mib16(priv, Mac_Mib_Type, MAC_MIB_RTS_THRESHOLD_POS, priv->rts_threshold);
-	atmel_set_mib16(priv, Mac_Mib_Type, MAC_MIB_FRAG_THRESHOLD_POS, priv->frag_threshold);
-	atmel_set_mib8(priv, Mac_Mib_Type, MAC_MIB_SHORT_RETRY_POS, priv->short_retry);
-	atmel_set_mib8(priv, Mac_Mib_Type, MAC_MIB_LONG_RETRY_POS, priv->long_retry);
-	atmel_set_mib8(priv, Local_Mib_Type, LOCAL_MIB_PREAMBLE_TYPE, priv->preamble);
-	atmel_set_mib(priv, Mac_Address_Mib_Type, MAC_ADDR_MIB_MAC_ADDR_POS,
-		      priv->dev->dev_addr, 6);
-	atmel_set_mib8(priv, Mac_Mgmt_Mib_Type, MAC_MGMT_MIB_PS_MODE_POS, ACTIVE_MODE);
-	atmel_set_mib16(priv, Mac_Mgmt_Mib_Type, MAC_MGMT_MIB_LISTEN_INTERVAL_POS, 1);
-	atmel_set_mib16(priv, Mac_Mgmt_Mib_Type, MAC_MGMT_MIB_BEACON_PER_POS, priv->default_beacon_period);
-	atmel_set_mib(priv, Phy_Mib_Type, PHY_MIB_RATE_SET_POS, atmel_basic_rates, 4);
-	atmel_set_mib8(priv, Mac_Mgmt_Mib_Type, MAC_MGMT_MIB_CUR_PRIVACY_POS, priv->wep_is_on);
-	if (priv->use_wpa)
-		build_wpa_mib(priv);
-	else
-		build_wep_mib(priv);
-
-	if (old_state == STATION_STATE_READY) {
-		union iwreq_data wrqu;
-
-		wrqu.data.length = 0;
-		wrqu.data.flags = 0;
-		wrqu.ap_addr.sa_family = ARPHRD_ETHER;
-		eth_zero_addr(wrqu.ap_addr.sa_data);
-		wireless_send_event(priv->dev, SIOCGIWAP, &wrqu, NULL);
-	}
-
-	return 0;
-}
-
-static void atmel_send_command(struct atmel_private *priv, int command,
-			       void *cmd, int cmd_size)
-{
-	if (cmd)
-		atmel_copy_to_card(priv->dev, atmel_co(priv, CMD_BLOCK_PARAMETERS_OFFSET),
-				   cmd, cmd_size);
-
-	atmel_wmem8(priv, atmel_co(priv, CMD_BLOCK_COMMAND_OFFSET), command);
-	atmel_wmem8(priv, atmel_co(priv, CMD_BLOCK_STATUS_OFFSET), 0);
-}
-
-static int atmel_send_command_wait(struct atmel_private *priv, int command,
-				   void *cmd, int cmd_size)
-{
-	int i, status;
-
-	atmel_send_command(priv, command, cmd, cmd_size);
-
-	for (i = 5000; i; i--) {
-		status = atmel_rmem8(priv, atmel_co(priv, CMD_BLOCK_STATUS_OFFSET));
-		if (status != CMD_STATUS_IDLE &&
-		    status != CMD_STATUS_IN_PROGRESS)
-			break;
-		udelay(20);
-	}
-
-	if (i == 0) {
-		printk(KERN_ALERT "%s: failed to contact MAC.\n", priv->dev->name);
-		status =  CMD_STATUS_HOST_ERROR;
-	} else {
-		if (command != CMD_EnableRadio)
-			status = CMD_STATUS_COMPLETE;
-	}
-
-	return status;
-}
-
-static u8 atmel_get_mib8(struct atmel_private *priv, u8 type, u8 index)
-{
-	struct get_set_mib m;
-	m.type = type;
-	m.size = 1;
-	m.index = index;
-
-	atmel_send_command_wait(priv, CMD_Get_MIB_Vars, &m, MIB_HEADER_SIZE + 1);
-	return atmel_rmem8(priv, atmel_co(priv, CMD_BLOCK_PARAMETERS_OFFSET + MIB_HEADER_SIZE));
-}
-
-static void atmel_set_mib8(struct atmel_private *priv, u8 type, u8 index, u8 data)
-{
-	struct get_set_mib m;
-	m.type = type;
-	m.size = 1;
-	m.index = index;
-	m.data[0] = data;
-
-	atmel_send_command_wait(priv, CMD_Set_MIB_Vars, &m, MIB_HEADER_SIZE + 1);
-}
-
-static void atmel_set_mib16(struct atmel_private *priv, u8 type, u8 index,
-			    u16 data)
-{
-	struct get_set_mib m;
-	m.type = type;
-	m.size = 2;
-	m.index = index;
-	m.data[0] = data;
-	m.data[1] = data >> 8;
-
-	atmel_send_command_wait(priv, CMD_Set_MIB_Vars, &m, MIB_HEADER_SIZE + 2);
-}
-
-static void atmel_set_mib(struct atmel_private *priv, u8 type, u8 index,
-			  const u8 *data, int data_len)
-{
-	struct get_set_mib m;
-	m.type = type;
-	m.size = data_len;
-	m.index = index;
-
-	if (data_len > MIB_MAX_DATA_BYTES)
-		printk(KERN_ALERT "%s: MIB buffer too small.\n", priv->dev->name);
-
-	memcpy(m.data, data, data_len);
-	atmel_send_command_wait(priv, CMD_Set_MIB_Vars, &m, MIB_HEADER_SIZE + data_len);
-}
-
-static void atmel_get_mib(struct atmel_private *priv, u8 type, u8 index,
-			  u8 *data, int data_len)
-{
-	struct get_set_mib m;
-	m.type = type;
-	m.size = data_len;
-	m.index = index;
-
-	if (data_len > MIB_MAX_DATA_BYTES)
-		printk(KERN_ALERT "%s: MIB buffer too small.\n", priv->dev->name);
-
-	atmel_send_command_wait(priv, CMD_Get_MIB_Vars, &m, MIB_HEADER_SIZE + data_len);
-	atmel_copy_to_host(priv->dev, data,
-			   atmel_co(priv, CMD_BLOCK_PARAMETERS_OFFSET + MIB_HEADER_SIZE), data_len);
-}
-
-static void atmel_writeAR(struct net_device *dev, u16 data)
-{
-	int i;
-	outw(data, dev->base_addr + AR);
-	/* Address register appears to need some convincing..... */
-	for (i = 0; data != inw(dev->base_addr + AR) && i < 10; i++)
-		outw(data, dev->base_addr + AR);
-}
-
-static void atmel_copy_to_card(struct net_device *dev, u16 dest,
-			       const unsigned char *src, u16 len)
-{
-	int i;
-	atmel_writeAR(dev, dest);
-	if (dest % 2) {
-		atmel_write8(dev, DR, *src);
-		src++; len--;
-	}
-	for (i = len; i > 1 ; i -= 2) {
-		u8 lb = *src++;
-		u8 hb = *src++;
-		atmel_write16(dev, DR, lb | (hb << 8));
-	}
-	if (i)
-		atmel_write8(dev, DR, *src);
-}
-
-static void atmel_copy_to_host(struct net_device *dev, unsigned char *dest,
-			       u16 src, u16 len)
-{
-	int i;
-	atmel_writeAR(dev, src);
-	if (src % 2) {
-		*dest = atmel_read8(dev, DR);
-		dest++; len--;
-	}
-	for (i = len; i > 1 ; i -= 2) {
-		u16 hw = atmel_read16(dev, DR);
-		*dest++ = hw;
-		*dest++ = hw >> 8;
-	}
-	if (i)
-		*dest = atmel_read8(dev, DR);
-}
-
-static void atmel_set_gcr(struct net_device *dev, u16 mask)
-{
-	outw(inw(dev->base_addr + GCR) | mask, dev->base_addr + GCR);
-}
-
-static void atmel_clear_gcr(struct net_device *dev, u16 mask)
-{
-	outw(inw(dev->base_addr + GCR) & ~mask, dev->base_addr + GCR);
-}
-
-static int atmel_lock_mac(struct atmel_private *priv)
-{
-	int i, j = 20;
- retry:
-	for (i = 5000; i; i--) {
-		if (!atmel_rmem8(priv, atmel_hi(priv, IFACE_LOCKOUT_HOST_OFFSET)))
-			break;
-		udelay(20);
-	}
-
-	if (!i)
-		return 0; /* timed out */
-
-	atmel_wmem8(priv, atmel_hi(priv, IFACE_LOCKOUT_MAC_OFFSET), 1);
-	if (atmel_rmem8(priv, atmel_hi(priv, IFACE_LOCKOUT_HOST_OFFSET))) {
-		atmel_wmem8(priv, atmel_hi(priv, IFACE_LOCKOUT_MAC_OFFSET), 0);
-		if (!j--)
-			return 0; /* timed out */
-		goto retry;
-	}
-
-	return 1;
-}
-
-static void atmel_wmem32(struct atmel_private *priv, u16 pos, u32 data)
-{
-	atmel_writeAR(priv->dev, pos);
-	atmel_write16(priv->dev, DR, data); /* card is little-endian */
-	atmel_write16(priv->dev, DR, data >> 16);
-}
-
-/***************************************************************************/
-/* There follows the source form of the MAC address reading firmware       */
-/***************************************************************************/
-#if 0
-
-/* Copyright 2003 Matthew T. Russotto                                      */
-/* But derived from the Atmel 76C502 firmware written by Atmel and         */
-/* included in "atmel wireless lan drivers" package                        */
-/*
-    This file is part of net.russotto.AtmelMACFW, hereto referred to
-    as AtmelMACFW
-
-    AtmelMACFW is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License version 2
-    as published by the Free Software Foundation.
-
-    AtmelMACFW is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with AtmelMACFW; if not, see <http://www.gnu.org/licenses/>.
-
-****************************************************************************/
-/* This firmware should work on the 76C502 RFMD, RFMD_D, and RFMD_E        */
-/* It will probably work on the 76C504 and 76C502 RFMD_3COM                */
-/* It only works on SPI EEPROM versions of the card.                       */
-
-/* This firmware initializes the SPI controller and clock, reads the MAC   */
-/* address from the EEPROM into SRAM, and puts the SRAM offset of the MAC  */
-/* address in MR2, and sets MR3 to 0x10 to indicate it is done             */
-/* It also puts a complete copy of the EEPROM in SRAM with the offset in   */
-/* MR4, for investigational purposes (maybe we can determine chip type     */
-/* from that?)                                                             */
-
-	.org 0
-    .set MRBASE, 0x8000000
-	.set CPSR_INITIAL, 0xD3 /* IRQ/FIQ disabled, ARM mode, Supervisor state */
-	.set CPSR_USER, 0xD1 /* IRQ/FIQ disabled, ARM mode, USER state */
-	.set SRAM_BASE,  0x02000000
-	.set SP_BASE,    0x0F300000
-	.set UNK_BASE,   0x0F000000 /* Some internal device, but which one? */
-	.set SPI_CGEN_BASE,  0x0E000000 /* Some internal device, but which one? */
-	.set UNK3_BASE,  0x02014000 /* Some internal device, but which one? */
-	.set STACK_BASE, 0x5600
-	.set SP_SR, 0x10
-	.set SP_TDRE, 2 /* status register bit -- TDR empty */
-	.set SP_RDRF, 1 /* status register bit -- RDR full */
-	.set SP_SWRST, 0x80
-	.set SP_SPIEN, 0x1
-	.set SP_CR, 0   /* control register */
-	.set SP_MR, 4   /* mode register */
-	.set SP_RDR, 0x08 /* Read Data Register */
-	.set SP_TDR, 0x0C /* Transmit Data Register */
-	.set SP_CSR0, 0x30 /* chip select registers */
-	.set SP_CSR1, 0x34
-	.set SP_CSR2, 0x38
-	.set SP_CSR3, 0x3C
-	.set NVRAM_CMD_RDSR, 5 /* read status register */
-	.set NVRAM_CMD_READ, 3 /* read data */
-	.set NVRAM_SR_RDY, 1 /* RDY bit.  This bit is inverted */
-	.set SPI_8CLOCKS, 0xFF /* Writing this to the TDR doesn't do anything to the
-				  serial output, since SO is normally high.  But it
-				  does cause 8 clock cycles and thus 8 bits to be
-				  clocked in to the chip.  See Atmel's SPI
-				  controller (e.g. AT91M55800) timing and 4K
-				  SPI EEPROM manuals */
-
-	.set NVRAM_SCRATCH, 0x02000100  /* arbitrary area for scratchpad memory */
-	.set NVRAM_IMAGE, 0x02000200
-	.set NVRAM_LENGTH, 0x0200
-	.set MAC_ADDRESS_MIB, SRAM_BASE
-	.set MAC_ADDRESS_LENGTH, 6
-	.set MAC_BOOT_FLAG, 0x10
-	.set MR1, 0
-	.set MR2, 4
-	.set MR3, 8
-	.set MR4, 0xC
-RESET_VECTOR:
-	b RESET_HANDLER
-UNDEF_VECTOR:
-	b HALT1
-SWI_VECTOR:
-	b HALT1
-IABORT_VECTOR:
-	b HALT1
-DABORT_VECTOR:
-RESERVED_VECTOR:
-	b HALT1
-IRQ_VECTOR:
-	b HALT1
-FIQ_VECTOR:
-	b HALT1
-HALT1:	b HALT1
-RESET_HANDLER:
-	mov     r0, #CPSR_INITIAL
-	msr	CPSR_c, r0	/* This is probably unnecessary */
-
-/* I'm guessing this is initializing clock generator electronics for SPI */
-	ldr	r0, =SPI_CGEN_BASE
-	mov	r1, #0
-	mov	r1, r1, lsl #3
-	orr	r1, r1, #0
-	str	r1, [r0]
-	ldr	r1, [r0, #28]
-	bic	r1, r1, #16
-	str	r1, [r0, #28]
-	mov	r1, #1
-	str	r1, [r0, #8]
-
-	ldr	r0, =MRBASE
-	mov	r1, #0
-	strh	r1, [r0, #MR1]
-	strh	r1, [r0, #MR2]
-	strh	r1, [r0, #MR3]
-	strh	r1, [r0, #MR4]
-
-	mov	sp, #STACK_BASE
-	bl	SP_INIT
-	mov	r0, #10
-	bl	DELAY9
-	bl	GET_MAC_ADDR
-	bl	GET_WHOLE_NVRAM
-	ldr	r0, =MRBASE
-	ldr	r1, =MAC_ADDRESS_MIB
-	strh	r1, [r0, #MR2]
-	ldr	r1, =NVRAM_IMAGE
-	strh	r1, [r0, #MR4]
-	mov	r1, #MAC_BOOT_FLAG
-	strh	r1, [r0, #MR3]
-HALT2:	b HALT2
-.func Get_Whole_NVRAM, GET_WHOLE_NVRAM
-GET_WHOLE_NVRAM:
-	stmdb	sp!, {lr}
-	mov	r2, #0 /* 0th bytes of NVRAM */
-	mov	r3, #NVRAM_LENGTH
-	mov	r1, #0		/* not used in routine */
-	ldr	r0, =NVRAM_IMAGE
-	bl	NVRAM_XFER
-	ldmia	sp!, {lr}
-	bx	lr
-.endfunc
-
-.func Get_MAC_Addr, GET_MAC_ADDR
-GET_MAC_ADDR:
-	stmdb	sp!, {lr}
-	mov	r2, #0x120	/* address of MAC Address within NVRAM */
-	mov	r3, #MAC_ADDRESS_LENGTH
-	mov	r1, #0		/* not used in routine */
-	ldr	r0, =MAC_ADDRESS_MIB
-	bl	NVRAM_XFER
-	ldmia	sp!, {lr}
-	bx	lr
-.endfunc
-.ltorg
-.func Delay9, DELAY9
-DELAY9:
-	adds	r0, r0, r0, LSL #3   /* r0 = r0 * 9 */
-DELAYLOOP:
-	beq	DELAY9_done
-	subs	r0, r0, #1
-	b	DELAYLOOP
-DELAY9_done:
-	bx	lr
-.endfunc
-
-.func SP_Init, SP_INIT
-SP_INIT:
-	mov	r1, #SP_SWRST
-	ldr	r0, =SP_BASE
-	str	r1, [r0, #SP_CR] /* reset the SPI */
-	mov	r1, #0
-	str	r1, [r0, #SP_CR] /* release SPI from reset state */
-	mov	r1, #SP_SPIEN
-	str	r1, [r0, #SP_MR] /* set the SPI to MASTER mode*/
-	str	r1, [r0, #SP_CR] /* enable the SPI */
-
-/*  My guess would be this turns on the SPI clock */
-	ldr	r3, =SPI_CGEN_BASE
-	ldr	r1, [r3, #28]
-	orr	r1, r1, #0x2000
-	str	r1, [r3, #28]
-
-	ldr	r1, =0x2000c01
-	str	r1, [r0, #SP_CSR0]
-	ldr	r1, =0x2000201
-	str	r1, [r0, #SP_CSR1]
-	str	r1, [r0, #SP_CSR2]
-	str	r1, [r0, #SP_CSR3]
-	ldr	r1, [r0, #SP_SR]
-	ldr	r0, [r0, #SP_RDR]
-	bx	lr
-.endfunc
-.func NVRAM_Init, NVRAM_INIT
-NVRAM_INIT:
-	ldr	r1, =SP_BASE
-	ldr	r0, [r1, #SP_RDR]
-	mov	r0, #NVRAM_CMD_RDSR
-	str	r0, [r1, #SP_TDR]
-SP_loop1:
-	ldr	r0, [r1, #SP_SR]
-	tst	r0, #SP_TDRE
-	beq	SP_loop1
-
-	mov	r0, #SPI_8CLOCKS
-	str	r0, [r1, #SP_TDR]
-SP_loop2:
-	ldr	r0, [r1, #SP_SR]
-	tst	r0, #SP_TDRE
-	beq	SP_loop2
-
-	ldr	r0, [r1, #SP_RDR]
-SP_loop3:
-	ldr	r0, [r1, #SP_SR]
-	tst	r0, #SP_RDRF
-	beq	SP_loop3
-
-	ldr	r0, [r1, #SP_RDR]
-	and	r0, r0, #255
-	bx	lr
-.endfunc
-
-.func NVRAM_Xfer, NVRAM_XFER
-	/* r0 = dest address */
-	/* r1 = not used */
-	/* r2 = src address within NVRAM */
-	/* r3 = length */
-NVRAM_XFER:
-	stmdb	sp!, {r4, r5, lr}
-	mov	r5, r0		/* save r0 (dest address) */
-	mov	r4, r3		/* save r3 (length) */
-	mov	r0, r2, LSR #5 /*  SPI memories put A8 in the command field */
-	and	r0, r0, #8
-	add	r0, r0, #NVRAM_CMD_READ
-	ldr	r1, =NVRAM_SCRATCH
-	strb	r0, [r1, #0]	/* save command in NVRAM_SCRATCH[0] */
-	strb	r2, [r1, #1]    /* save low byte of source address in NVRAM_SCRATCH[1] */
-_local1:
-	bl	NVRAM_INIT
-	tst	r0, #NVRAM_SR_RDY
-	bne	_local1
-	mov	r0, #20
-	bl	DELAY9
-	mov	r2, r4		/* length */
-	mov	r1, r5		/* dest address */
-	mov	r0, #2		/* bytes to transfer in command */
-	bl	NVRAM_XFER2
-	ldmia	sp!, {r4, r5, lr}
-	bx	lr
-.endfunc
-
-.func NVRAM_Xfer2, NVRAM_XFER2
-NVRAM_XFER2:
-	stmdb	sp!, {r4, r5, r6, lr}
-	ldr	r4, =SP_BASE
-	mov	r3, #0
-	cmp	r0, #0
-	bls	_local2
-	ldr	r5, =NVRAM_SCRATCH
-_local4:
-	ldrb	r6, [r5, r3]
-	str	r6, [r4, #SP_TDR]
-_local3:
-	ldr	r6, [r4, #SP_SR]
-	tst	r6, #SP_TDRE
-	beq	_local3
-	add	r3, r3, #1
-	cmp	r3, r0 /* r0 is # of bytes to send out (command+addr) */
-	blo	_local4
-_local2:
-	mov	r3, #SPI_8CLOCKS
-	str	r3, [r4, #SP_TDR]
-	ldr	r0, [r4, #SP_RDR]
-_local5:
-	ldr	r0, [r4, #SP_SR]
-	tst	r0, #SP_RDRF
-	beq	_local5
-	ldr	r0, [r4, #SP_RDR] /* what's this byte?  It's the byte read while writing the TDR -- nonsense, because the NVRAM doesn't read and write at the same time */
-	mov	r0, #0
-	cmp	r2, #0  /* r2 is # of bytes to copy in */
-	bls	_local6
-_local7:
-	ldr	r5, [r4, #SP_SR]
-	tst	r5, #SP_TDRE
-	beq	_local7
-	str	r3, [r4, #SP_TDR]  /* r3 has SPI_8CLOCKS */
-_local8:
-	ldr	r5, [r4, #SP_SR]
-	tst	r5, #SP_RDRF
-	beq	_local8
-	ldr	r5, [r4, #SP_RDR] /* but didn't we read this byte above? */
-	strb	r5, [r1], #1 /* postindexed */
-	add	r0, r0, #1
-	cmp	r0, r2
-	blo	_local7 /* since we don't send another address, the NVRAM must be capable of sequential reads */
-_local6:
-	mov	r0, #200
-	bl	DELAY9
-	ldmia	sp!, {r4, r5, r6, lr}
-	bx	lr
-#endif
diff --git a/drivers/net/wireless/atmel/atmel.h b/drivers/net/wireless/atmel/atmel.h
deleted file mode 100644
index d2aa52cf6e8b..000000000000
--- a/drivers/net/wireless/atmel/atmel.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*** -*- linux-c -*- **********************************************************
-
-     Driver for Atmel at76c502 at76c504 and at76c506 wireless cards.
-
-         Copyright 2005 Dan Williams and Red Hat, Inc.
-
-
-******************************************************************************/
-
-#ifndef _ATMEL_H
-#define _ATMEL_H
-
-typedef enum {
-	ATMEL_FW_TYPE_NONE = 0,
-	ATMEL_FW_TYPE_502,
-	ATMEL_FW_TYPE_502D,
-	ATMEL_FW_TYPE_502E,
-	ATMEL_FW_TYPE_502_3COM,
-	ATMEL_FW_TYPE_504,
-	ATMEL_FW_TYPE_504_2958,
-	ATMEL_FW_TYPE_504A_2958,
-	ATMEL_FW_TYPE_506
-} AtmelFWType;
-
-struct net_device *init_atmel_card(unsigned short, unsigned long, const AtmelFWType, struct device *, 
-				    int (*present_func)(void *), void * );
-void stop_atmel_card( struct net_device *);
-int atmel_open( struct net_device * );
-
-#endif
diff --git a/drivers/net/wireless/atmel/atmel_cs.c b/drivers/net/wireless/atmel/atmel_cs.c
deleted file mode 100644
index 58bba9875d36..000000000000
--- a/drivers/net/wireless/atmel/atmel_cs.c
+++ /dev/null
@@ -1,292 +0,0 @@
-/*** -*- linux-c -*- **********************************************************
-
-     Driver for Atmel at76c502 at76c504 and at76c506 wireless cards.
-
-        Copyright 2000-2001 ATMEL Corporation.
-        Copyright 2003 Simon Kelley.
-
-    This code was developed from version 2.1.1 of the Atmel drivers,
-    released by Atmel corp. under the GPL in December 2002. It also
-    includes code from the Linux aironet drivers (C) Benjamin Reed,
-    and the Linux PCMCIA package, (C) David Hinds.
-
-    For all queries about this code, please contact the current author,
-    Simon Kelley <simon@thekelleys.org.uk> and not Atmel Corporation.
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This software is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with Atmel wireless lan drivers; if not, see
-    <http://www.gnu.org/licenses/>.
-
-******************************************************************************/
-
-#ifdef __IN_PCMCIA_PACKAGE__
-#include <pcmcia/k_compat.h>
-#endif
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/ptrace.h>
-#include <linux/slab.h>
-#include <linux/string.h>
-#include <linux/netdevice.h>
-#include <linux/moduleparam.h>
-#include <linux/device.h>
-
-#include <pcmcia/cistpl.h>
-#include <pcmcia/cisreg.h>
-#include <pcmcia/ds.h>
-#include <pcmcia/ciscode.h>
-
-#include <asm/io.h>
-#include <linux/wireless.h>
-
-#include "atmel.h"
-
-
-/*====================================================================*/
-
-MODULE_AUTHOR("Simon Kelley");
-MODULE_DESCRIPTION("Support for Atmel at76c50x 802.11 wireless ethernet cards.");
-MODULE_LICENSE("GPL");
-
-/*====================================================================*/
-
-static int atmel_config(struct pcmcia_device *link);
-static void atmel_release(struct pcmcia_device *link);
-
-static void atmel_detach(struct pcmcia_device *p_dev);
-
-struct local_info {
-	struct net_device *eth_dev;
-};
-
-static int atmel_probe(struct pcmcia_device *p_dev)
-{
-	struct local_info *local;
-	int ret;
-
-	dev_dbg(&p_dev->dev, "atmel_attach()\n");
-
-	/* Allocate space for private device-specific data */
-	local = kzalloc(sizeof(*local), GFP_KERNEL);
-	if (!local)
-		return -ENOMEM;
-
-	p_dev->priv = local;
-
-	ret = atmel_config(p_dev);
-	if (ret)
-		goto err_free_priv;
-
-	return 0;
-
-err_free_priv:
-	kfree(p_dev->priv);
-	return ret;
-}
-
-static void atmel_detach(struct pcmcia_device *link)
-{
-	dev_dbg(&link->dev, "atmel_detach\n");
-
-	atmel_release(link);
-
-	kfree(link->priv);
-}
-
-/* Call-back function to interrogate PCMCIA-specific information
-   about the current existence of the card */
-static int card_present(void *arg)
-{
-	struct pcmcia_device *link = (struct pcmcia_device *)arg;
-
-	if (pcmcia_dev_present(link))
-		return 1;
-
-	return 0;
-}
-
-static int atmel_config_check(struct pcmcia_device *p_dev, void *priv_data)
-{
-	if (p_dev->config_index == 0)
-		return -EINVAL;
-
-	return pcmcia_request_io(p_dev);
-}
-
-static int atmel_config(struct pcmcia_device *link)
-{
-	int ret;
-	const struct pcmcia_device_id *did;
-
-	did = dev_get_drvdata(&link->dev);
-
-	dev_dbg(&link->dev, "atmel_config\n");
-
-	link->config_flags |= CONF_ENABLE_IRQ | CONF_AUTO_SET_VPP |
-		CONF_AUTO_AUDIO | CONF_AUTO_SET_IO;
-
-	if (pcmcia_loop_config(link, atmel_config_check, NULL))
-		goto failed;
-
-	if (!link->irq) {
-		dev_err(&link->dev, "atmel: cannot assign IRQ: check that CONFIG_ISA is set in kernel config.");
-		goto failed;
-	}
-
-	ret = pcmcia_enable_device(link);
-	if (ret)
-		goto failed;
-
-	((struct local_info *)link->priv)->eth_dev =
-		init_atmel_card(link->irq,
-				link->resource[0]->start,
-				did ? did->driver_info : ATMEL_FW_TYPE_NONE,
-				&link->dev,
-				card_present,
-				link);
-	if (!((struct local_info *)link->priv)->eth_dev)
-			goto failed;
-
-
-	return 0;
-
- failed:
-	atmel_release(link);
-	return -ENODEV;
-}
-
-static void atmel_release(struct pcmcia_device *link)
-{
-	struct net_device *dev = ((struct local_info *)link->priv)->eth_dev;
-
-	dev_dbg(&link->dev, "atmel_release\n");
-
-	if (dev)
-		stop_atmel_card(dev);
-	((struct local_info *)link->priv)->eth_dev = NULL;
-
-	pcmcia_disable_device(link);
-}
-
-static int atmel_suspend(struct pcmcia_device *link)
-{
-	struct local_info *local = link->priv;
-
-	netif_device_detach(local->eth_dev);
-
-	return 0;
-}
-
-static int atmel_resume(struct pcmcia_device *link)
-{
-	struct local_info *local = link->priv;
-
-	atmel_open(local->eth_dev);
-	netif_device_attach(local->eth_dev);
-
-	return 0;
-}
-
-/*====================================================================*/
-/* We use the driver_info field to store the correct firmware type for a card. */
-
-#define PCMCIA_DEVICE_MANF_CARD_INFO(manf, card, info) { \
-	.match_flags = PCMCIA_DEV_ID_MATCH_MANF_ID| \
-			PCMCIA_DEV_ID_MATCH_CARD_ID, \
-	.manf_id = (manf), \
-	.card_id = (card), \
-        .driver_info = (kernel_ulong_t)(info), }
-
-#define PCMCIA_DEVICE_PROD_ID12_INFO(v1, v2, vh1, vh2, info) { \
-	.match_flags = PCMCIA_DEV_ID_MATCH_PROD_ID1| \
-			PCMCIA_DEV_ID_MATCH_PROD_ID2, \
-	.prod_id = { (v1), (v2), NULL, NULL }, \
-	.prod_id_hash = { (vh1), (vh2), 0, 0 }, \
-        .driver_info = (kernel_ulong_t)(info), }
-
-static const struct pcmcia_device_id atmel_ids[] = {
-	PCMCIA_DEVICE_MANF_CARD_INFO(0x0101, 0x0620, ATMEL_FW_TYPE_502_3COM),
-	PCMCIA_DEVICE_MANF_CARD_INFO(0x0101, 0x0696, ATMEL_FW_TYPE_502_3COM),
-	PCMCIA_DEVICE_MANF_CARD_INFO(0x01bf, 0x3302, ATMEL_FW_TYPE_502E),
-	PCMCIA_DEVICE_MANF_CARD_INFO(0xd601, 0x0007, ATMEL_FW_TYPE_502),
-	PCMCIA_DEVICE_PROD_ID12_INFO("11WAVE", "11WP611AL-E", 0x9eb2da1f, 0xc9a0d3f9, ATMEL_FW_TYPE_502E),
-	PCMCIA_DEVICE_PROD_ID12_INFO("ATMEL", "AT76C502AR", 0xabda4164, 0x41b37e1f, ATMEL_FW_TYPE_502),
-	PCMCIA_DEVICE_PROD_ID12_INFO("ATMEL", "AT76C502AR_D", 0xabda4164, 0x3675d704, ATMEL_FW_TYPE_502D),
-	PCMCIA_DEVICE_PROD_ID12_INFO("ATMEL", "AT76C502AR_E", 0xabda4164, 0x4172e792, ATMEL_FW_TYPE_502E),
-	PCMCIA_DEVICE_PROD_ID12_INFO("ATMEL", "AT76C504_R", 0xabda4164, 0x917f3d72, ATMEL_FW_TYPE_504_2958),
-	PCMCIA_DEVICE_PROD_ID12_INFO("ATMEL", "AT76C504", 0xabda4164, 0x5040670a, ATMEL_FW_TYPE_504),
-	PCMCIA_DEVICE_PROD_ID12_INFO("ATMEL", "AT76C504A", 0xabda4164, 0xe15ed87f, ATMEL_FW_TYPE_504A_2958),
-	PCMCIA_DEVICE_PROD_ID12_INFO("BT", "Voyager 1020 Laptop Adapter", 0xae49b86a, 0x1e957cd5, ATMEL_FW_TYPE_502),
-	PCMCIA_DEVICE_PROD_ID12_INFO("CNet", "CNWLC 11Mbps Wireless PC Card V-5", 0xbc477dde, 0x502fae6b, ATMEL_FW_TYPE_502E),
-	PCMCIA_DEVICE_PROD_ID12_INFO("IEEE 802.11b", "Wireless LAN PC Card", 0x5b878724, 0x122f1df6, ATMEL_FW_TYPE_502),
-	PCMCIA_DEVICE_PROD_ID12_INFO("IEEE 802.11b", "Wireless LAN Card S", 0x5b878724, 0x5fba533a, ATMEL_FW_TYPE_504_2958),
-	PCMCIA_DEVICE_PROD_ID12_INFO("OEM", "11Mbps Wireless LAN PC Card V-3", 0xfea54c90, 0x1c5b0f68, ATMEL_FW_TYPE_502),
-	PCMCIA_DEVICE_PROD_ID12_INFO("SMC", "2632W", 0xc4f8b18b, 0x30f38774, ATMEL_FW_TYPE_502D),
-	PCMCIA_DEVICE_PROD_ID12_INFO("SMC", "2632W-V2", 0xc4f8b18b, 0x172d1377, ATMEL_FW_TYPE_502),
-	PCMCIA_DEVICE_PROD_ID12_INFO("Wireless", "PC_CARD", 0xa407ecdd, 0x119f6314, ATMEL_FW_TYPE_502D),
-	PCMCIA_DEVICE_PROD_ID12_INFO("WLAN", "802.11b PC CARD", 0x575c516c, 0xb1f6dbc4, ATMEL_FW_TYPE_502D),
-	PCMCIA_DEVICE_PROD_ID12_INFO("LG", "LW2100N", 0xb474d43a, 0x6b1fec94, ATMEL_FW_TYPE_502E),
-	PCMCIA_DEVICE_NULL
-};
-
-MODULE_DEVICE_TABLE(pcmcia, atmel_ids);
-
-static struct pcmcia_driver atmel_driver = {
-	.owner		= THIS_MODULE,
-	.name		= "atmel_cs",
-	.probe          = atmel_probe,
-	.remove		= atmel_detach,
-	.id_table	= atmel_ids,
-	.suspend	= atmel_suspend,
-	.resume		= atmel_resume,
-};
-module_pcmcia_driver(atmel_driver);
-
-/*
-    This program is free software; you can redistribute it and/or
-    modify it under the terms of the GNU General Public License
-    as published by the Free Software Foundation; either version 2
-    of the License, or (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    In addition:
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions
-    are met:
-
-    1. Redistributions of source code must retain the above copyright
-       notice, this list of conditions and the following disclaimer.
-    2. Redistributions in binary form must reproduce the above copyright
-       notice, this list of conditions and the following disclaimer in the
-       documentation and/or other materials provided with the distribution.
-    3. The name of the author may not be used to endorse or promote
-       products derived from this software without specific prior written
-       permission.
-
-    THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
-    IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-    ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
-    INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-    (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-    SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-    HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-    STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
-    IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-    POSSIBILITY OF SUCH DAMAGE.
-*/
diff --git a/drivers/net/wireless/atmel/atmel_pci.c b/drivers/net/wireless/atmel/atmel_pci.c
deleted file mode 100644
index f428dc79d916..000000000000
--- a/drivers/net/wireless/atmel/atmel_pci.c
+++ /dev/null
@@ -1,65 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*** -*- linux-c -*- **********************************************************
-
-     Driver for Atmel at76c502 at76c504 and at76c506 wireless cards.
-
-         Copyright 2004 Simon Kelley.
-
-
-******************************************************************************/
-#include <linux/pci.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/netdevice.h>
-#include "atmel.h"
-
-MODULE_AUTHOR("Simon Kelley");
-MODULE_DESCRIPTION("Support for Atmel at76c50x 802.11 wireless ethernet cards.");
-MODULE_LICENSE("GPL");
-
-static const struct pci_device_id card_ids[] = {
-	{ 0x1114, 0x0506, PCI_ANY_ID, PCI_ANY_ID },
-	{ 0, }
-};
-
-MODULE_DEVICE_TABLE(pci, card_ids);
-
-static int atmel_pci_probe(struct pci_dev *, const struct pci_device_id *);
-static void atmel_pci_remove(struct pci_dev *);
-
-static struct pci_driver atmel_driver = {
-	.name     = "atmel",
-	.id_table = card_ids,
-	.probe    = atmel_pci_probe,
-	.remove   = atmel_pci_remove,
-};
-
-
-static int atmel_pci_probe(struct pci_dev *pdev,
-				     const struct pci_device_id *pent)
-{
-	struct net_device *dev;
-
-	if (pci_enable_device(pdev))
-		return -ENODEV;
-
-	pci_set_master(pdev);
-
-	dev = init_atmel_card(pdev->irq, pdev->resource[1].start,
-			      ATMEL_FW_TYPE_506,
-			      &pdev->dev, NULL, NULL);
-	if (!dev) {
-		pci_disable_device(pdev);
-		return -ENODEV;
-	}
-
-	pci_set_drvdata(pdev, dev);
-	return 0;
-}
-
-static void atmel_pci_remove(struct pci_dev *pdev)
-{
-	stop_atmel_card(pci_get_drvdata(pdev));
-}
-
-module_pci_driver(atmel_driver);
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
index 667462369a32..133c5ea6429c 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
@@ -866,7 +866,7 @@ struct wireless_dev *brcmf_apsta_add_vif(struct wiphy *wiphy, const char *name,
 		goto fail;
 	}
 
-	strncpy(ifp->ndev->name, name, sizeof(ifp->ndev->name) - 1);
+	strscpy(ifp->ndev->name, name, sizeof(ifp->ndev->name));
 	err = brcmf_net_attach(ifp, true);
 	if (err) {
 		bphy_err(drvr, "Registering netdevice failed\n");
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c
index d4492d02e4ea..6e0c90f4718b 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c
@@ -2334,7 +2334,7 @@ struct wireless_dev *brcmf_p2p_add_vif(struct wiphy *wiphy, const char *name,
 		goto fail;
 	}
 
-	strncpy(ifp->ndev->name, name, sizeof(ifp->ndev->name) - 1);
+	strscpy(ifp->ndev->name, name, sizeof(ifp->ndev->name));
 	ifp->ndev->name_assign_type = name_assign_type;
 	err = brcmf_net_attach(ifp, true);
 	if (err) {
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/channel.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/channel.c
index 5a6d9c86552a..f6962e558d7c 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/channel.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/channel.c
@@ -341,7 +341,7 @@ struct brcms_cm_info *brcms_c_channel_mgr_attach(struct brcms_c_info *wlc)
 	/* store the country code for passing up as a regulatory hint */
 	wlc_cm->world_regd = brcms_world_regd(ccode, ccode_len);
 	if (brcms_c_country_valid(ccode))
-		strncpy(wlc->pub->srom_ccode, ccode, ccode_len);
+		memcpy(wlc->pub->srom_ccode, ccode, ccode_len);
 
 	/*
 	 * If no custom world domain is found in the SROM, use the
@@ -354,10 +354,10 @@ struct brcms_cm_info *brcms_c_channel_mgr_attach(struct brcms_c_info *wlc)
 	}
 
 	/* save default country for exiting 11d regulatory mode */
-	strncpy(wlc->country_default, ccode, ccode_len);
+	memcpy(wlc->country_default, ccode, ccode_len);
 
 	/* initialize autocountry_default to driver default */
-	strncpy(wlc->autocountry_default, ccode, ccode_len);
+	memcpy(wlc->autocountry_default, ccode, ccode_len);
 
 	brcms_c_set_country(wlc_cm, wlc_cm->world_regd);
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/dma.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/dma.c
index b7df576bb84d..3d5c1ef8f7f2 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/dma.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/dma.c
@@ -584,8 +584,7 @@ struct dma_pub *dma_attach(char *name, struct brcms_c_info *wlc,
 		      rxextheadroom, nrxpost, rxoffset, txregbase, rxregbase);
 
 	/* make a private copy of our callers name */
-	strncpy(di->name, name, MAXNAMEL);
-	di->name[MAXNAMEL - 1] = '\0';
+	strscpy(di->name, name, sizeof(di->name));
 
 	di->dmadev = core->dma_dev;
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/main.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/main.c
index b3663c5ef382..34460b5815d0 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/main.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/main.c
@@ -5551,8 +5551,8 @@ int brcms_c_module_register(struct brcms_pub *pub,
 	/* find an empty entry and just add, no duplication check! */
 	for (i = 0; i < BRCMS_MAXMODULES; i++) {
 		if (wlc->modulecb[i].name[0] == '\0') {
-			strncpy(wlc->modulecb[i].name, name,
-				sizeof(wlc->modulecb[i].name) - 1);
+			strscpy(wlc->modulecb[i].name, name,
+				sizeof(wlc->modulecb[i].name));
 			wlc->modulecb[i].hdl = hdl;
 			wlc->modulecb[i].down_fn = d_fn;
 			return 0;
diff --git a/drivers/net/wireless/cisco/Kconfig b/drivers/net/wireless/cisco/Kconfig
deleted file mode 100644
index b40ee25aca99..000000000000
--- a/drivers/net/wireless/cisco/Kconfig
+++ /dev/null
@@ -1,59 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-config WLAN_VENDOR_CISCO
-	bool "Cisco devices"
-	default y
-	help
-	  If you have a wireless card belonging to this class, say Y.
-
-	  Note that the answer to this question doesn't directly affect the
-	  kernel: saying N will just cause the configurator to skip all the
-	  questions about these cards. If you say Y, you will be asked for
-	  your specific card in the following questions.
-
-if WLAN_VENDOR_CISCO
-
-config AIRO
-	tristate "Cisco/Aironet 34X/35X/4500/4800 ISA and PCI cards"
-	depends on CFG80211 && (PCI || BROKEN)
-	select WIRELESS_EXT
-	select CRYPTO
-	select CRYPTO_SKCIPHER
-	select WEXT_SPY
-	select WEXT_PRIV
-	help
-	  This is the standard Linux driver to support Cisco/Aironet ISA and
-	  PCI 802.11 wireless cards.
-	  It supports the new 802.11b cards from Cisco (Cisco 34X, Cisco 35X
-	  - with or without encryption) as well as card before the Cisco
-	  acquisition (Aironet 4500, Aironet 4800, Aironet 4800B).
-
-	  This driver support both the standard Linux Wireless Extensions
-	  and Cisco proprietary API, so both the Linux Wireless Tools and the
-	  Cisco Linux utilities can be used to configure the card.
-
-	  The driver can be compiled as a module and will be named "airo".
-
-config AIRO_CS
-	tristate "Cisco/Aironet 34X/35X/4500/4800 PCMCIA cards"
-	depends on CFG80211 && PCMCIA
-	select WIRELESS_EXT
-	select WEXT_SPY
-	select WEXT_PRIV
-	select CRYPTO
-	select CRYPTO_AES
-	select CRYPTO_CTR
-	help
-	  This is the standard Linux driver to support Cisco/Aironet PCMCIA
-	  802.11 wireless cards.  This driver is the same as the Aironet
-	  driver part of the Linux Pcmcia package.
-	  It supports the new 802.11b cards from Cisco (Cisco 34X, Cisco 35X
-	  - with or without encryption) as well as card before the Cisco
-	  acquisition (Aironet 4500, Aironet 4800, Aironet 4800B). It also
-	  supports OEM of Cisco such as the DELL TrueMobile 4800 and Xircom
-	  802.11b cards.
-
-	  This driver support both the standard Linux Wireless Extensions
-	  and Cisco proprietary API, so both the Linux Wireless Tools and the
-	  Cisco Linux utilities can be used to configure the card.
-
-endif # WLAN_VENDOR_CISCO
diff --git a/drivers/net/wireless/cisco/Makefile b/drivers/net/wireless/cisco/Makefile
deleted file mode 100644
index 506a19ce375f..000000000000
--- a/drivers/net/wireless/cisco/Makefile
+++ /dev/null
@@ -1,3 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-obj-$(CONFIG_AIRO)		+= airo.o
-obj-$(CONFIG_AIRO_CS)		+= airo_cs.o airo.o
diff --git a/drivers/net/wireless/cisco/airo.c b/drivers/net/wireless/cisco/airo.c
deleted file mode 100644
index dbd13f7aa3e6..000000000000
--- a/drivers/net/wireless/cisco/airo.c
+++ /dev/null
@@ -1,8288 +0,0 @@
-/*======================================================================
-
-    Aironet driver for 4500 and 4800 series cards
-
-    This code is released under both the GPL version 2 and BSD licenses.
-    Either license may be used.  The respective licenses are found at
-    the end of this file.
-
-    This code was developed by Benjamin Reed <breed@users.sourceforge.net>
-    including portions of which come from the Aironet PC4500
-    Developer's Reference Manual and used with permission.  Copyright
-    (C) 1999 Benjamin Reed.  All Rights Reserved.  Permission to use
-    code in the Developer's manual was granted for this driver by
-    Aironet.  Major code contributions were received from Javier Achirica
-    <achirica@users.sourceforge.net> and Jean Tourrilhes <jt@hpl.hp.com>.
-    Code was also integrated from the Cisco Aironet driver for Linux.
-    Support for MPI350 cards was added by Fabrice Bellet
-    <fabrice@bellet.info>.
-
-======================================================================*/
-
-#include <linux/err.h>
-#include <linux/init.h>
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/proc_fs.h>
-
-#include <linux/sched.h>
-#include <linux/ptrace.h>
-#include <linux/slab.h>
-#include <linux/string.h>
-#include <linux/timer.h>
-#include <linux/interrupt.h>
-#include <linux/in.h>
-#include <linux/bitops.h>
-#include <linux/scatterlist.h>
-#include <linux/crypto.h>
-#include <linux/io.h>
-#include <asm/unaligned.h>
-
-#include <linux/netdevice.h>
-#include <linux/etherdevice.h>
-#include <linux/skbuff.h>
-#include <linux/if_arp.h>
-#include <linux/ioport.h>
-#include <linux/pci.h>
-#include <linux/uaccess.h>
-#include <linux/kthread.h>
-#include <linux/freezer.h>
-
-#include <crypto/aes.h>
-#include <crypto/skcipher.h>
-
-#include <net/cfg80211.h>
-#include <net/iw_handler.h>
-
-#include "airo.h"
-
-#define DRV_NAME "airo"
-
-#ifdef CONFIG_PCI
-static const struct pci_device_id card_ids[] = {
-	{ 0x14b9, 1, PCI_ANY_ID, PCI_ANY_ID, },
-	{ 0x14b9, 0x4500, PCI_ANY_ID, PCI_ANY_ID },
-	{ 0x14b9, 0x4800, PCI_ANY_ID, PCI_ANY_ID, },
-	{ 0x14b9, 0x0340, PCI_ANY_ID, PCI_ANY_ID, },
-	{ 0x14b9, 0x0350, PCI_ANY_ID, PCI_ANY_ID, },
-	{ 0x14b9, 0x5000, PCI_ANY_ID, PCI_ANY_ID, },
-	{ 0x14b9, 0xa504, PCI_ANY_ID, PCI_ANY_ID, },
-	{ 0, }
-};
-MODULE_DEVICE_TABLE(pci, card_ids);
-
-static int airo_pci_probe(struct pci_dev *, const struct pci_device_id *);
-static void airo_pci_remove(struct pci_dev *);
-static int __maybe_unused airo_pci_suspend(struct device *dev);
-static int __maybe_unused airo_pci_resume(struct device *dev);
-
-static SIMPLE_DEV_PM_OPS(airo_pci_pm_ops,
-			 airo_pci_suspend,
-			 airo_pci_resume);
-
-static struct pci_driver airo_driver = {
-	.name      = DRV_NAME,
-	.id_table  = card_ids,
-	.probe     = airo_pci_probe,
-	.remove    = airo_pci_remove,
-	.driver.pm = &airo_pci_pm_ops,
-};
-#endif /* CONFIG_PCI */
-
-/* Include Wireless Extension definition and check version - Jean II */
-#include <linux/wireless.h>
-#define WIRELESS_SPY		/* enable iwspy support */
-
-#define CISCO_EXT		/* enable Cisco extensions */
-#ifdef CISCO_EXT
-#include <linux/delay.h>
-#endif
-
-/* Hack to do some power saving */
-#define POWER_ON_DOWN
-
-/* As you can see this list is HUGH!
-   I really don't know what a lot of these counts are about, but they
-   are all here for completeness.  If the IGNLABEL macro is put in
-   infront of the label, that statistic will not be included in the list
-   of statistics in the /proc filesystem */
-
-#define IGNLABEL(comment) NULL
-static const char *statsLabels[] = {
-	"RxOverrun",
-	IGNLABEL("RxPlcpCrcErr"),
-	IGNLABEL("RxPlcpFormatErr"),
-	IGNLABEL("RxPlcpLengthErr"),
-	"RxMacCrcErr",
-	"RxMacCrcOk",
-	"RxWepErr",
-	"RxWepOk",
-	"RetryLong",
-	"RetryShort",
-	"MaxRetries",
-	"NoAck",
-	"NoCts",
-	"RxAck",
-	"RxCts",
-	"TxAck",
-	"TxRts",
-	"TxCts",
-	"TxMc",
-	"TxBc",
-	"TxUcFrags",
-	"TxUcPackets",
-	"TxBeacon",
-	"RxBeacon",
-	"TxSinColl",
-	"TxMulColl",
-	"DefersNo",
-	"DefersProt",
-	"DefersEngy",
-	"DupFram",
-	"RxFragDisc",
-	"TxAged",
-	"RxAged",
-	"LostSync-MaxRetry",
-	"LostSync-MissedBeacons",
-	"LostSync-ArlExceeded",
-	"LostSync-Deauth",
-	"LostSync-Disassoced",
-	"LostSync-TsfTiming",
-	"HostTxMc",
-	"HostTxBc",
-	"HostTxUc",
-	"HostTxFail",
-	"HostRxMc",
-	"HostRxBc",
-	"HostRxUc",
-	"HostRxDiscard",
-	IGNLABEL("HmacTxMc"),
-	IGNLABEL("HmacTxBc"),
-	IGNLABEL("HmacTxUc"),
-	IGNLABEL("HmacTxFail"),
-	IGNLABEL("HmacRxMc"),
-	IGNLABEL("HmacRxBc"),
-	IGNLABEL("HmacRxUc"),
-	IGNLABEL("HmacRxDiscard"),
-	IGNLABEL("HmacRxAccepted"),
-	"SsidMismatch",
-	"ApMismatch",
-	"RatesMismatch",
-	"AuthReject",
-	"AuthTimeout",
-	"AssocReject",
-	"AssocTimeout",
-	IGNLABEL("ReasonOutsideTable"),
-	IGNLABEL("ReasonStatus1"),
-	IGNLABEL("ReasonStatus2"),
-	IGNLABEL("ReasonStatus3"),
-	IGNLABEL("ReasonStatus4"),
-	IGNLABEL("ReasonStatus5"),
-	IGNLABEL("ReasonStatus6"),
-	IGNLABEL("ReasonStatus7"),
-	IGNLABEL("ReasonStatus8"),
-	IGNLABEL("ReasonStatus9"),
-	IGNLABEL("ReasonStatus10"),
-	IGNLABEL("ReasonStatus11"),
-	IGNLABEL("ReasonStatus12"),
-	IGNLABEL("ReasonStatus13"),
-	IGNLABEL("ReasonStatus14"),
-	IGNLABEL("ReasonStatus15"),
-	IGNLABEL("ReasonStatus16"),
-	IGNLABEL("ReasonStatus17"),
-	IGNLABEL("ReasonStatus18"),
-	IGNLABEL("ReasonStatus19"),
-	"RxMan",
-	"TxMan",
-	"RxRefresh",
-	"TxRefresh",
-	"RxPoll",
-	"TxPoll",
-	"HostRetries",
-	"LostSync-HostReq",
-	"HostTxBytes",
-	"HostRxBytes",
-	"ElapsedUsec",
-	"ElapsedSec",
-	"LostSyncBetterAP",
-	"PrivacyMismatch",
-	"Jammed",
-	"DiscRxNotWepped",
-	"PhyEleMismatch",
-	(char*)-1 };
-#ifndef RUN_AT
-#define RUN_AT(x) (jiffies+(x))
-#endif
-
-
-/* These variables are for insmod, since it seems that the rates
-   can only be set in setup_card.  Rates should be a comma separated
-   (no spaces) list of rates (up to 8). */
-
-static int rates[8];
-static char *ssids[3];
-
-static int io[4];
-static int irq[4];
-
-static
-int maxencrypt /* = 0 */; /* The highest rate that the card can encrypt at.
-		       0 means no limit.  For old cards this was 4 */
-
-static int auto_wep /* = 0 */; /* If set, it tries to figure out the wep mode */
-static int aux_bap /* = 0 */; /* Checks to see if the aux ports are needed to read
-		    the bap, needed on some older cards and buses. */
-static int adhoc;
-
-static int probe = 1;
-
-static kuid_t proc_kuid;
-static int proc_uid /* = 0 */;
-
-static kgid_t proc_kgid;
-static int proc_gid /* = 0 */;
-
-static int airo_perm = 0555;
-
-static int proc_perm = 0644;
-
-MODULE_AUTHOR("Benjamin Reed");
-MODULE_DESCRIPTION("Support for Cisco/Aironet 802.11 wireless ethernet cards.  "
-		   "Direct support for ISA/PCI/MPI cards and support for PCMCIA when used with airo_cs.");
-MODULE_LICENSE("Dual BSD/GPL");
-module_param_hw_array(io, int, ioport, NULL, 0);
-module_param_hw_array(irq, int, irq, NULL, 0);
-module_param_array(rates, int, NULL, 0);
-module_param_array(ssids, charp, NULL, 0);
-module_param(auto_wep, int, 0);
-MODULE_PARM_DESC(auto_wep,
-		 "If non-zero, the driver will keep looping through the authentication options until an association is made.  "
-		 "The value of auto_wep is number of the wep keys to check.  "
-		 "A value of 2 will try using the key at index 0 and index 1.");
-module_param(aux_bap, int, 0);
-MODULE_PARM_DESC(aux_bap,
-		 "If non-zero, the driver will switch into a mode that seems to work better for older cards with some older buses.  "
-		 "Before switching it checks that the switch is needed.");
-module_param(maxencrypt, int, 0);
-MODULE_PARM_DESC(maxencrypt,
-		 "The maximum speed that the card can do encryption.  "
-		 "Units are in 512kbs.  "
-		 "Zero (default) means there is no limit.  "
-		 "Older cards used to be limited to 2mbs (4).");
-module_param(adhoc, int, 0);
-MODULE_PARM_DESC(adhoc, "If non-zero, the card will start in adhoc mode.");
-module_param(probe, int, 0);
-MODULE_PARM_DESC(probe, "If zero, the driver won't start the card.");
-
-module_param(proc_uid, int, 0);
-MODULE_PARM_DESC(proc_uid, "The uid that the /proc files will belong to.");
-module_param(proc_gid, int, 0);
-MODULE_PARM_DESC(proc_gid, "The gid that the /proc files will belong to.");
-module_param(airo_perm, int, 0);
-MODULE_PARM_DESC(airo_perm, "The permission bits of /proc/[driver/]aironet.");
-module_param(proc_perm, int, 0);
-MODULE_PARM_DESC(proc_perm, "The permission bits of the files in /proc");
-
-/* This is a kind of sloppy hack to get this information to OUT4500 and
-   IN4500.  I would be extremely interested in the situation where this
-   doesn't work though!!! */
-static int do8bitIO /* = 0 */;
-
-/* Return codes */
-#define SUCCESS 0
-#define ERROR -1
-#define NO_PACKET -2
-
-/* Commands */
-#define NOP2		0x0000
-#define MAC_ENABLE	0x0001
-#define MAC_DISABLE	0x0002
-#define CMD_LOSE_SYNC	0x0003 /* Not sure what this does... */
-#define CMD_SOFTRESET	0x0004
-#define HOSTSLEEP	0x0005
-#define CMD_MAGIC_PKT	0x0006
-#define CMD_SETWAKEMASK	0x0007
-#define CMD_READCFG	0x0008
-#define CMD_SETMODE	0x0009
-#define CMD_ALLOCATETX	0x000a
-#define CMD_TRANSMIT	0x000b
-#define CMD_DEALLOCATETX 0x000c
-#define NOP		0x0010
-#define CMD_WORKAROUND	0x0011
-#define CMD_ALLOCATEAUX 0x0020
-#define CMD_ACCESS	0x0021
-#define CMD_PCIBAP	0x0022
-#define CMD_PCIAUX	0x0023
-#define CMD_ALLOCBUF	0x0028
-#define CMD_GETTLV	0x0029
-#define CMD_PUTTLV	0x002a
-#define CMD_DELTLV	0x002b
-#define CMD_FINDNEXTTLV	0x002c
-#define CMD_PSPNODES	0x0030
-#define CMD_SETCW	0x0031
-#define CMD_SETPCF	0x0032
-#define CMD_SETPHYREG	0x003e
-#define CMD_TXTEST	0x003f
-#define MAC_ENABLETX	0x0101
-#define CMD_LISTBSS	0x0103
-#define CMD_SAVECFG	0x0108
-#define CMD_ENABLEAUX	0x0111
-#define CMD_WRITERID	0x0121
-#define CMD_USEPSPNODES	0x0130
-#define MAC_ENABLERX	0x0201
-
-/* Command errors */
-#define ERROR_QUALIF 0x00
-#define ERROR_ILLCMD 0x01
-#define ERROR_ILLFMT 0x02
-#define ERROR_INVFID 0x03
-#define ERROR_INVRID 0x04
-#define ERROR_LARGE 0x05
-#define ERROR_NDISABL 0x06
-#define ERROR_ALLOCBSY 0x07
-#define ERROR_NORD 0x0B
-#define ERROR_NOWR 0x0C
-#define ERROR_INVFIDTX 0x0D
-#define ERROR_TESTACT 0x0E
-#define ERROR_TAGNFND 0x12
-#define ERROR_DECODE 0x20
-#define ERROR_DESCUNAV 0x21
-#define ERROR_BADLEN 0x22
-#define ERROR_MODE 0x80
-#define ERROR_HOP 0x81
-#define ERROR_BINTER 0x82
-#define ERROR_RXMODE 0x83
-#define ERROR_MACADDR 0x84
-#define ERROR_RATES 0x85
-#define ERROR_ORDER 0x86
-#define ERROR_SCAN 0x87
-#define ERROR_AUTH 0x88
-#define ERROR_PSMODE 0x89
-#define ERROR_RTYPE 0x8A
-#define ERROR_DIVER 0x8B
-#define ERROR_SSID 0x8C
-#define ERROR_APLIST 0x8D
-#define ERROR_AUTOWAKE 0x8E
-#define ERROR_LEAP 0x8F
-
-/* Registers */
-#define COMMAND 0x00
-#define PARAM0 0x02
-#define PARAM1 0x04
-#define PARAM2 0x06
-#define STATUS 0x08
-#define RESP0 0x0a
-#define RESP1 0x0c
-#define RESP2 0x0e
-#define LINKSTAT 0x10
-#define SELECT0 0x18
-#define OFFSET0 0x1c
-#define RXFID 0x20
-#define TXALLOCFID 0x22
-#define TXCOMPLFID 0x24
-#define DATA0 0x36
-#define EVSTAT 0x30
-#define EVINTEN 0x32
-#define EVACK 0x34
-#define SWS0 0x28
-#define SWS1 0x2a
-#define SWS2 0x2c
-#define SWS3 0x2e
-#define AUXPAGE 0x3A
-#define AUXOFF 0x3C
-#define AUXDATA 0x3E
-
-#define FID_TX 1
-#define FID_RX 2
-/* Offset into aux memory for descriptors */
-#define AUX_OFFSET 0x800
-/* Size of allocated packets */
-#define PKTSIZE 1840
-#define RIDSIZE 2048
-/* Size of the transmit queue */
-#define MAXTXQ 64
-
-/* BAP selectors */
-#define BAP0 0 /* Used for receiving packets */
-#define BAP1 2 /* Used for xmiting packets and working with RIDS */
-
-/* Flags */
-#define COMMAND_BUSY 0x8000
-
-#define BAP_BUSY 0x8000
-#define BAP_ERR 0x4000
-#define BAP_DONE 0x2000
-
-#define PROMISC 0xffff
-#define NOPROMISC 0x0000
-
-#define EV_CMD 0x10
-#define EV_CLEARCOMMANDBUSY 0x4000
-#define EV_RX 0x01
-#define EV_TX 0x02
-#define EV_TXEXC 0x04
-#define EV_ALLOC 0x08
-#define EV_LINK 0x80
-#define EV_AWAKE 0x100
-#define EV_TXCPY 0x400
-#define EV_UNKNOWN 0x800
-#define EV_MIC 0x1000 /* Message Integrity Check Interrupt */
-#define EV_AWAKEN 0x2000
-#define STATUS_INTS (EV_AWAKE|EV_LINK|EV_TXEXC|EV_TX|EV_TXCPY|EV_RX|EV_MIC)
-
-#ifdef CHECK_UNKNOWN_INTS
-#define IGNORE_INTS (EV_CMD | EV_UNKNOWN)
-#else
-#define IGNORE_INTS (~STATUS_INTS)
-#endif
-
-/* RID TYPES */
-#define RID_RW 0x20
-
-/* The RIDs */
-#define RID_CAPABILITIES 0xFF00
-#define RID_APINFO     0xFF01
-#define RID_RADIOINFO  0xFF02
-#define RID_UNKNOWN3   0xFF03
-#define RID_RSSI       0xFF04
-#define RID_CONFIG     0xFF10
-#define RID_SSID       0xFF11
-#define RID_APLIST     0xFF12
-#define RID_DRVNAME    0xFF13
-#define RID_ETHERENCAP 0xFF14
-#define RID_WEP_TEMP   0xFF15
-#define RID_WEP_PERM   0xFF16
-#define RID_MODULATION 0xFF17
-#define RID_OPTIONS    0xFF18
-#define RID_ACTUALCONFIG 0xFF20 /*readonly*/
-#define RID_FACTORYCONFIG 0xFF21
-#define RID_UNKNOWN22  0xFF22
-#define RID_LEAPUSERNAME 0xFF23
-#define RID_LEAPPASSWORD 0xFF24
-#define RID_STATUS     0xFF50
-#define RID_BEACON_HST 0xFF51
-#define RID_BUSY_HST   0xFF52
-#define RID_RETRIES_HST 0xFF53
-#define RID_UNKNOWN54  0xFF54
-#define RID_UNKNOWN55  0xFF55
-#define RID_UNKNOWN56  0xFF56
-#define RID_MIC        0xFF57
-#define RID_STATS16    0xFF60
-#define RID_STATS16DELTA 0xFF61
-#define RID_STATS16DELTACLEAR 0xFF62
-#define RID_STATS      0xFF68
-#define RID_STATSDELTA 0xFF69
-#define RID_STATSDELTACLEAR 0xFF6A
-#define RID_ECHOTEST_RID 0xFF70
-#define RID_ECHOTEST_RESULTS 0xFF71
-#define RID_BSSLISTFIRST 0xFF72
-#define RID_BSSLISTNEXT  0xFF73
-#define RID_WPA_BSSLISTFIRST 0xFF74
-#define RID_WPA_BSSLISTNEXT  0xFF75
-
-typedef struct {
-	u16 cmd;
-	u16 parm0;
-	u16 parm1;
-	u16 parm2;
-} Cmd;
-
-typedef struct {
-	u16 status;
-	u16 rsp0;
-	u16 rsp1;
-	u16 rsp2;
-} Resp;
-
-/*
- * Rids and endian-ness:  The Rids will always be in cpu endian, since
- * this all the patches from the big-endian guys end up doing that.
- * so all rid access should use the read/writeXXXRid routines.
- */
-
-/* This structure came from an email sent to me from an engineer at
-   aironet for inclusion into this driver */
-typedef struct WepKeyRid WepKeyRid;
-struct WepKeyRid {
-	__le16 len;
-	__le16 kindex;
-	u8 mac[ETH_ALEN];
-	__le16 klen;
-	u8 key[16];
-} __packed;
-
-/* These structures are from the Aironet's PC4500 Developers Manual */
-typedef struct Ssid Ssid;
-struct Ssid {
-	__le16 len;
-	u8 ssid[32];
-} __packed;
-
-typedef struct SsidRid SsidRid;
-struct SsidRid {
-	__le16 len;
-	Ssid ssids[3];
-} __packed;
-
-typedef struct ModulationRid ModulationRid;
-struct ModulationRid {
-        __le16 len;
-        __le16 modulation;
-#define MOD_DEFAULT cpu_to_le16(0)
-#define MOD_CCK cpu_to_le16(1)
-#define MOD_MOK cpu_to_le16(2)
-} __packed;
-
-typedef struct ConfigRid ConfigRid;
-struct ConfigRid {
-	__le16 len; /* sizeof(ConfigRid) */
-	__le16 opmode; /* operating mode */
-#define MODE_STA_IBSS cpu_to_le16(0)
-#define MODE_STA_ESS cpu_to_le16(1)
-#define MODE_AP cpu_to_le16(2)
-#define MODE_AP_RPTR cpu_to_le16(3)
-#define MODE_CFG_MASK cpu_to_le16(0xff)
-#define MODE_ETHERNET_HOST cpu_to_le16(0<<8) /* rx payloads converted */
-#define MODE_LLC_HOST cpu_to_le16(1<<8) /* rx payloads left as is */
-#define MODE_AIRONET_EXTEND cpu_to_le16(1<<9) /* enable Aironet extensions */
-#define MODE_AP_INTERFACE cpu_to_le16(1<<10) /* enable ap interface extensions */
-#define MODE_ANTENNA_ALIGN cpu_to_le16(1<<11) /* enable antenna alignment */
-#define MODE_ETHER_LLC cpu_to_le16(1<<12) /* enable ethernet LLC */
-#define MODE_LEAF_NODE cpu_to_le16(1<<13) /* enable leaf node bridge */
-#define MODE_CF_POLLABLE cpu_to_le16(1<<14) /* enable CF pollable */
-#define MODE_MIC cpu_to_le16(1<<15) /* enable MIC */
-	__le16 rmode; /* receive mode */
-#define RXMODE_BC_MC_ADDR cpu_to_le16(0)
-#define RXMODE_BC_ADDR cpu_to_le16(1) /* ignore multicasts */
-#define RXMODE_ADDR cpu_to_le16(2) /* ignore multicast and broadcast */
-#define RXMODE_RFMON cpu_to_le16(3) /* wireless monitor mode */
-#define RXMODE_RFMON_ANYBSS cpu_to_le16(4)
-#define RXMODE_LANMON cpu_to_le16(5) /* lan style monitor -- data packets only */
-#define RXMODE_MASK cpu_to_le16(255)
-#define RXMODE_DISABLE_802_3_HEADER cpu_to_le16(1<<8) /* disables 802.3 header on rx */
-#define RXMODE_FULL_MASK (RXMODE_MASK | RXMODE_DISABLE_802_3_HEADER)
-#define RXMODE_NORMALIZED_RSSI cpu_to_le16(1<<9) /* return normalized RSSI */
-	__le16 fragThresh;
-	__le16 rtsThres;
-	u8 macAddr[ETH_ALEN];
-	u8 rates[8];
-	__le16 shortRetryLimit;
-	__le16 longRetryLimit;
-	__le16 txLifetime; /* in kusec */
-	__le16 rxLifetime; /* in kusec */
-	__le16 stationary;
-	__le16 ordering;
-	__le16 u16deviceType; /* for overriding device type */
-	__le16 cfpRate;
-	__le16 cfpDuration;
-	__le16 _reserved1[3];
-	/*---------- Scanning/Associating ----------*/
-	__le16 scanMode;
-#define SCANMODE_ACTIVE cpu_to_le16(0)
-#define SCANMODE_PASSIVE cpu_to_le16(1)
-#define SCANMODE_AIROSCAN cpu_to_le16(2)
-	__le16 probeDelay; /* in kusec */
-	__le16 probeEnergyTimeout; /* in kusec */
-        __le16 probeResponseTimeout;
-	__le16 beaconListenTimeout;
-	__le16 joinNetTimeout;
-	__le16 authTimeout;
-	__le16 authType;
-#define AUTH_OPEN cpu_to_le16(0x1)
-#define AUTH_ENCRYPT cpu_to_le16(0x101)
-#define AUTH_SHAREDKEY cpu_to_le16(0x102)
-#define AUTH_ALLOW_UNENCRYPTED cpu_to_le16(0x200)
-	__le16 associationTimeout;
-	__le16 specifiedApTimeout;
-	__le16 offlineScanInterval;
-	__le16 offlineScanDuration;
-	__le16 linkLossDelay;
-	__le16 maxBeaconLostTime;
-	__le16 refreshInterval;
-#define DISABLE_REFRESH cpu_to_le16(0xFFFF)
-	__le16 _reserved1a[1];
-	/*---------- Power save operation ----------*/
-	__le16 powerSaveMode;
-#define POWERSAVE_CAM cpu_to_le16(0)
-#define POWERSAVE_PSP cpu_to_le16(1)
-#define POWERSAVE_PSPCAM cpu_to_le16(2)
-	__le16 sleepForDtims;
-	__le16 listenInterval;
-	__le16 fastListenInterval;
-	__le16 listenDecay;
-	__le16 fastListenDelay;
-	__le16 _reserved2[2];
-	/*---------- Ap/Ibss config items ----------*/
-	__le16 beaconPeriod;
-	__le16 atimDuration;
-	__le16 hopPeriod;
-	__le16 channelSet;
-	__le16 channel;
-	__le16 dtimPeriod;
-	__le16 bridgeDistance;
-	__le16 radioID;
-	/*---------- Radio configuration ----------*/
-	__le16 radioType;
-#define RADIOTYPE_DEFAULT cpu_to_le16(0)
-#define RADIOTYPE_802_11 cpu_to_le16(1)
-#define RADIOTYPE_LEGACY cpu_to_le16(2)
-	u8 rxDiversity;
-	u8 txDiversity;
-	__le16 txPower;
-#define TXPOWER_DEFAULT 0
-	__le16 rssiThreshold;
-#define RSSI_DEFAULT 0
-        __le16 modulation;
-#define PREAMBLE_AUTO cpu_to_le16(0)
-#define PREAMBLE_LONG cpu_to_le16(1)
-#define PREAMBLE_SHORT cpu_to_le16(2)
-	__le16 preamble;
-	__le16 homeProduct;
-	__le16 radioSpecific;
-	/*---------- Aironet Extensions ----------*/
-	u8 nodeName[16];
-	__le16 arlThreshold;
-	__le16 arlDecay;
-	__le16 arlDelay;
-	__le16 _reserved4[1];
-	/*---------- Aironet Extensions ----------*/
-	u8 magicAction;
-#define MAGIC_ACTION_STSCHG 1
-#define MAGIC_ACTION_RESUME 2
-#define MAGIC_IGNORE_MCAST (1<<8)
-#define MAGIC_IGNORE_BCAST (1<<9)
-#define MAGIC_SWITCH_TO_PSP (0<<10)
-#define MAGIC_STAY_IN_CAM (1<<10)
-	u8 magicControl;
-	__le16 autoWake;
-} __packed;
-
-typedef struct StatusRid StatusRid;
-struct StatusRid {
-	__le16 len;
-	u8 mac[ETH_ALEN];
-	__le16 mode;
-	__le16 errorCode;
-	__le16 sigQuality;
-	__le16 SSIDlen;
-	char SSID[32];
-	char apName[16];
-	u8 bssid[4][ETH_ALEN];
-	__le16 beaconPeriod;
-	__le16 dimPeriod;
-	__le16 atimDuration;
-	__le16 hopPeriod;
-	__le16 channelSet;
-	__le16 channel;
-	__le16 hopsToBackbone;
-	__le16 apTotalLoad;
-	__le16 generatedLoad;
-	__le16 accumulatedArl;
-	__le16 signalQuality;
-	__le16 currentXmitRate;
-	__le16 apDevExtensions;
-	__le16 normalizedSignalStrength;
-	__le16 shortPreamble;
-	u8 apIP[4];
-	u8 noisePercent; /* Noise percent in last second */
-	u8 noisedBm; /* Noise dBm in last second */
-	u8 noiseAvePercent; /* Noise percent in last minute */
-	u8 noiseAvedBm; /* Noise dBm in last minute */
-	u8 noiseMaxPercent; /* Highest noise percent in last minute */
-	u8 noiseMaxdBm; /* Highest noise dbm in last minute */
-	__le16 load;
-	u8 carrier[4];
-	__le16 assocStatus;
-#define STAT_NOPACKETS 0
-#define STAT_NOCARRIERSET 10
-#define STAT_GOTCARRIERSET 11
-#define STAT_WRONGSSID 20
-#define STAT_BADCHANNEL 25
-#define STAT_BADBITRATES 30
-#define STAT_BADPRIVACY 35
-#define STAT_APFOUND 40
-#define STAT_APREJECTED 50
-#define STAT_AUTHENTICATING 60
-#define STAT_DEAUTHENTICATED 61
-#define STAT_AUTHTIMEOUT 62
-#define STAT_ASSOCIATING 70
-#define STAT_DEASSOCIATED 71
-#define STAT_ASSOCTIMEOUT 72
-#define STAT_NOTAIROAP 73
-#define STAT_ASSOCIATED 80
-#define STAT_LEAPING 90
-#define STAT_LEAPFAILED 91
-#define STAT_LEAPTIMEDOUT 92
-#define STAT_LEAPCOMPLETE 93
-} __packed;
-
-typedef struct StatsRid StatsRid;
-struct StatsRid {
-	__le16 len;
-	__le16 spacer;
-	__le32 vals[100];
-} __packed;
-
-typedef struct APListRid APListRid;
-struct APListRid {
-	__le16 len;
-	u8 ap[4][ETH_ALEN];
-} __packed;
-
-typedef struct CapabilityRid CapabilityRid;
-struct CapabilityRid {
-	__le16 len;
-	char oui[3];
-	char zero;
-	__le16 prodNum;
-	char manName[32];
-	char prodName[16];
-	char prodVer[8];
-	char factoryAddr[ETH_ALEN];
-	char aironetAddr[ETH_ALEN];
-	__le16 radioType;
-	__le16 country;
-	char callid[ETH_ALEN];
-	char supportedRates[8];
-	char rxDiversity;
-	char txDiversity;
-	__le16 txPowerLevels[8];
-	__le16 hardVer;
-	__le16 hardCap;
-	__le16 tempRange;
-	__le16 softVer;
-	__le16 softSubVer;
-	__le16 interfaceVer;
-	__le16 softCap;
-	__le16 bootBlockVer;
-	__le16 requiredHard;
-	__le16 extSoftCap;
-} __packed;
-
-/* Only present on firmware >= 5.30.17 */
-typedef struct BSSListRidExtra BSSListRidExtra;
-struct BSSListRidExtra {
-  __le16 unknown[4];
-  u8 fixed[12]; /* WLAN management frame */
-  u8 iep[624];
-} __packed;
-
-typedef struct BSSListRid BSSListRid;
-struct BSSListRid {
-  __le16 len;
-  __le16 index; /* First is 0 and 0xffff means end of list */
-#define RADIO_FH 1 /* Frequency hopping radio type */
-#define RADIO_DS 2 /* Direct sequence radio type */
-#define RADIO_TMA 4 /* Proprietary radio used in old cards (2500) */
-  __le16 radioType;
-  u8 bssid[ETH_ALEN]; /* Mac address of the BSS */
-  u8 zero;
-  u8 ssidLen;
-  u8 ssid[32];
-  __le16 dBm;
-#define CAP_ESS cpu_to_le16(1<<0)
-#define CAP_IBSS cpu_to_le16(1<<1)
-#define CAP_PRIVACY cpu_to_le16(1<<4)
-#define CAP_SHORTHDR cpu_to_le16(1<<5)
-  __le16 cap;
-  __le16 beaconInterval;
-  u8 rates[8]; /* Same as rates for config rid */
-  struct { /* For frequency hopping only */
-    __le16 dwell;
-    u8 hopSet;
-    u8 hopPattern;
-    u8 hopIndex;
-    u8 fill;
-  } fh;
-  __le16 dsChannel;
-  __le16 atimWindow;
-
-  /* Only present on firmware >= 5.30.17 */
-  BSSListRidExtra extra;
-} __packed;
-
-typedef struct {
-  BSSListRid bss;
-  struct list_head list;
-} BSSListElement;
-
-typedef struct tdsRssiEntry tdsRssiEntry;
-struct tdsRssiEntry {
-  u8 rssipct;
-  u8 rssidBm;
-} __packed;
-
-typedef struct tdsRssiRid tdsRssiRid;
-struct tdsRssiRid {
-  u16 len;
-  tdsRssiEntry x[256];
-} __packed;
-
-typedef struct MICRid MICRid;
-struct MICRid {
-	__le16 len;
-	__le16 state;
-	__le16 multicastValid;
-	u8  multicast[16];
-	__le16 unicastValid;
-	u8  unicast[16];
-} __packed;
-
-typedef struct MICBuffer MICBuffer;
-struct MICBuffer {
-	__be16 typelen;
-
-	union {
-	    u8 snap[8];
-	    struct {
-		u8 dsap;
-		u8 ssap;
-		u8 control;
-		u8 orgcode[3];
-		u8 fieldtype[2];
-	    } llc;
-	} u;
-	__be32 mic;
-	__be32 seq;
-} __packed;
-
-typedef struct {
-	u8 da[ETH_ALEN];
-	u8 sa[ETH_ALEN];
-} etherHead;
-
-#define TXCTL_TXOK (1<<1) /* report if tx is ok */
-#define TXCTL_TXEX (1<<2) /* report if tx fails */
-#define TXCTL_802_3 (0<<3) /* 802.3 packet */
-#define TXCTL_802_11 (1<<3) /* 802.11 mac packet */
-#define TXCTL_ETHERNET (0<<4) /* payload has ethertype */
-#define TXCTL_LLC (1<<4) /* payload is llc */
-#define TXCTL_RELEASE (0<<5) /* release after completion */
-#define TXCTL_NORELEASE (1<<5) /* on completion returns to host */
-
-#define BUSY_FID 0x10000
-
-#ifdef CISCO_EXT
-#define AIROMAGIC	0xa55a
-/* Warning : SIOCDEVPRIVATE may disapear during 2.5.X - Jean II */
-#ifdef SIOCIWFIRSTPRIV
-#ifdef SIOCDEVPRIVATE
-#define AIROOLDIOCTL	SIOCDEVPRIVATE
-#define AIROOLDIDIFC 	AIROOLDIOCTL + 1
-#endif /* SIOCDEVPRIVATE */
-#else /* SIOCIWFIRSTPRIV */
-#define SIOCIWFIRSTPRIV SIOCDEVPRIVATE
-#endif /* SIOCIWFIRSTPRIV */
-/* This may be wrong. When using the new SIOCIWFIRSTPRIV range, we probably
- * should use only "GET" ioctls (last bit set to 1). "SET" ioctls are root
- * only and don't return the modified struct ifreq to the application which
- * is usually a problem. - Jean II */
-#define AIROIOCTL	SIOCIWFIRSTPRIV
-#define AIROIDIFC 	AIROIOCTL + 1
-
-/* Ioctl constants to be used in airo_ioctl.command */
-
-#define	AIROGCAP  		0	// Capability rid
-#define AIROGCFG		1       // USED A LOT
-#define AIROGSLIST		2	// System ID list
-#define AIROGVLIST		3       // List of specified AP's
-#define AIROGDRVNAM		4	//  NOTUSED
-#define AIROGEHTENC		5	// NOTUSED
-#define AIROGWEPKTMP		6
-#define AIROGWEPKNV		7
-#define AIROGSTAT		8
-#define AIROGSTATSC32		9
-#define AIROGSTATSD32		10
-#define AIROGMICRID		11
-#define AIROGMICSTATS		12
-#define AIROGFLAGS		13
-#define AIROGID			14
-#define AIRORRID		15
-#define AIRORSWVERSION		17
-
-/* Leave gap of 40 commands after AIROGSTATSD32 for future */
-
-#define AIROPCAP               	AIROGSTATSD32 + 40
-#define AIROPVLIST              AIROPCAP      + 1
-#define AIROPSLIST		AIROPVLIST    + 1
-#define AIROPCFG		AIROPSLIST    + 1
-#define AIROPSIDS		AIROPCFG      + 1
-#define AIROPAPLIST		AIROPSIDS     + 1
-#define AIROPMACON		AIROPAPLIST   + 1	/* Enable mac  */
-#define AIROPMACOFF		AIROPMACON    + 1 	/* Disable mac */
-#define AIROPSTCLR		AIROPMACOFF   + 1
-#define AIROPWEPKEY		AIROPSTCLR    + 1
-#define AIROPWEPKEYNV		AIROPWEPKEY   + 1
-#define AIROPLEAPPWD            AIROPWEPKEYNV + 1
-#define AIROPLEAPUSR            AIROPLEAPPWD  + 1
-
-/* Flash codes */
-
-#define AIROFLSHRST	       AIROPWEPKEYNV  + 40
-#define AIROFLSHGCHR           AIROFLSHRST    + 1
-#define AIROFLSHSTFL           AIROFLSHGCHR   + 1
-#define AIROFLSHPCHR           AIROFLSHSTFL   + 1
-#define AIROFLPUTBUF           AIROFLSHPCHR   + 1
-#define AIRORESTART            AIROFLPUTBUF   + 1
-
-#define FLASHSIZE	32768
-#define AUXMEMSIZE	(256 * 1024)
-
-typedef struct aironet_ioctl {
-	unsigned short command;		// What to do
-	unsigned short len;		// Len of data
-	unsigned short ridnum;		// rid number
-	unsigned char __user *data;	// d-data
-} aironet_ioctl;
-
-static const char swversion[] = "2.1";
-#endif /* CISCO_EXT */
-
-#define NUM_MODULES       2
-#define MIC_MSGLEN_MAX    2400
-#define EMMH32_MSGLEN_MAX MIC_MSGLEN_MAX
-#define AIRO_DEF_MTU      2312
-
-typedef struct {
-	u32   size;            // size
-	u8    enabled;         // MIC enabled or not
-	u32   rxSuccess;       // successful packets received
-	u32   rxIncorrectMIC;  // pkts dropped due to incorrect MIC comparison
-	u32   rxNotMICed;      // pkts dropped due to not being MIC'd
-	u32   rxMICPlummed;    // pkts dropped due to not having a MIC plummed
-	u32   rxWrongSequence; // pkts dropped due to sequence number violation
-	u32   reserve[32];
-} mic_statistics;
-
-typedef struct {
-	__be32 coeff[((EMMH32_MSGLEN_MAX)+3)>>2];
-	u64 accum;	// accumulated mic, reduced to u32 in final()
-	int position;	// current position (byte offset) in message
-	union {
-		u8  d8[4];
-		__be32 d32;
-	} part;	// saves partial message word across update() calls
-} emmh32_context;
-
-typedef struct {
-	emmh32_context seed;	    // Context - the seed
-	u32		 rx;	    // Received sequence number
-	u32		 tx;	    // Tx sequence number
-	u32		 window;    // Start of window
-	u8		 valid;	    // Flag to say if context is valid or not
-	u8		 key[16];
-} miccntx;
-
-typedef struct {
-	miccntx mCtx;		// Multicast context
-	miccntx uCtx;		// Unicast context
-} mic_module;
-
-typedef struct {
-	unsigned int  rid: 16;
-	unsigned int  len: 15;
-	unsigned int  valid: 1;
-	dma_addr_t host_addr;
-} Rid;
-
-typedef struct {
-	unsigned int  offset: 15;
-	unsigned int  eoc: 1;
-	unsigned int  len: 15;
-	unsigned int  valid: 1;
-	dma_addr_t host_addr;
-} TxFid;
-
-struct rx_hdr {
-	__le16 status, len;
-	u8 rssi[2];
-	u8 rate;
-	u8 freq;
-	__le16 tmp[4];
-} __packed;
-
-typedef struct {
-	unsigned int  ctl: 15;
-	unsigned int  rdy: 1;
-	unsigned int  len: 15;
-	unsigned int  valid: 1;
-	dma_addr_t host_addr;
-} RxFid;
-
-/*
- * Host receive descriptor
- */
-typedef struct {
-	unsigned char __iomem *card_ram_off; /* offset into card memory of the
-						desc */
-	RxFid         rx_desc;		     /* card receive descriptor */
-	char          *virtual_host_addr;    /* virtual address of host receive
-					        buffer */
-	int           pending;
-} HostRxDesc;
-
-/*
- * Host transmit descriptor
- */
-typedef struct {
-	unsigned char __iomem *card_ram_off;	     /* offset into card memory of the
-						desc */
-	TxFid         tx_desc;		     /* card transmit descriptor */
-	char          *virtual_host_addr;    /* virtual address of host receive
-					        buffer */
-	int           pending;
-} HostTxDesc;
-
-/*
- * Host RID descriptor
- */
-typedef struct {
-	unsigned char __iomem *card_ram_off;      /* offset into card memory of the
-					     descriptor */
-	Rid           rid_desc;		  /* card RID descriptor */
-	char          *virtual_host_addr; /* virtual address of host receive
-					     buffer */
-} HostRidDesc;
-
-typedef struct {
-	u16 sw0;
-	u16 sw1;
-	u16 status;
-	u16 len;
-#define HOST_SET (1 << 0)
-#define HOST_INT_TX (1 << 1) /* Interrupt on successful TX */
-#define HOST_INT_TXERR (1 << 2) /* Interrupt on unseccessful TX */
-#define HOST_LCC_PAYLOAD (1 << 4) /* LLC payload, 0 = Ethertype */
-#define HOST_DONT_RLSE (1 << 5) /* Don't release buffer when done */
-#define HOST_DONT_RETRY (1 << 6) /* Don't retry trasmit */
-#define HOST_CLR_AID (1 << 7) /* clear AID failure */
-#define HOST_RTS (1 << 9) /* Force RTS use */
-#define HOST_SHORT (1 << 10) /* Do short preamble */
-	u16 ctl;
-	u16 aid;
-	u16 retries;
-	u16 fill;
-} TxCtlHdr;
-
-typedef struct {
-        u16 ctl;
-        u16 duration;
-        char addr1[6];
-        char addr2[6];
-        char addr3[6];
-        u16 seq;
-        char addr4[6];
-} WifiHdr;
-
-
-typedef struct {
-	TxCtlHdr ctlhdr;
-	u16 fill1;
-	u16 fill2;
-	WifiHdr wifihdr;
-	u16 gaplen;
-	u16 status;
-} WifiCtlHdr;
-
-static WifiCtlHdr wifictlhdr8023 = {
-	.ctlhdr = {
-		.ctl	= HOST_DONT_RLSE,
-	}
-};
-
-// A few details needed for WEP (Wireless Equivalent Privacy)
-#define MAX_KEY_SIZE 13			// 128 (?) bits
-#define MIN_KEY_SIZE  5			// 40 bits RC4 - WEP
-typedef struct wep_key_t {
-	u16	len;
-	u8	key[16];	/* 40-bit and 104-bit keys */
-} wep_key_t;
-
-/* List of Wireless Handlers (new API) */
-static const struct iw_handler_def	airo_handler_def;
-
-static const char version[] = "airo.c 0.6 (Ben Reed & Javier Achirica)";
-
-struct airo_info;
-
-static int get_dec_u16(char *buffer, int *start, int limit);
-static void OUT4500(struct airo_info *, u16 reg, u16 value);
-static unsigned short IN4500(struct airo_info *, u16 reg);
-static u16 setup_card(struct airo_info*, struct net_device *dev, int lock);
-static int enable_MAC(struct airo_info *ai, int lock);
-static void disable_MAC(struct airo_info *ai, int lock);
-static void enable_interrupts(struct airo_info*);
-static void disable_interrupts(struct airo_info*);
-static u16 issuecommand(struct airo_info*, Cmd *pCmd, Resp *pRsp,
-			bool may_sleep);
-static int bap_setup(struct airo_info*, u16 rid, u16 offset, int whichbap);
-static int aux_bap_read(struct airo_info*, __le16 *pu16Dst, int bytelen,
-			int whichbap);
-static int fast_bap_read(struct airo_info*, __le16 *pu16Dst, int bytelen,
-			 int whichbap);
-static int bap_write(struct airo_info*, const __le16 *pu16Src, int bytelen,
-		     int whichbap);
-static int PC4500_accessrid(struct airo_info*, u16 rid, u16 accmd);
-static int PC4500_readrid(struct airo_info*, u16 rid, void *pBuf, int len, int lock);
-static int PC4500_writerid(struct airo_info*, u16 rid, const void
-			   *pBuf, int len, int lock);
-static int do_writerid(struct airo_info*, u16 rid, const void *rid_data,
-			int len, int dummy);
-static u16 transmit_allocate(struct airo_info*, int lenPayload, int raw);
-static int transmit_802_3_packet(struct airo_info*, int len, char *pPacket,
-				 bool may_sleep);
-static int transmit_802_11_packet(struct airo_info*, int len, char *pPacket,
-				  bool may_sleep);
-
-static int mpi_send_packet(struct net_device *dev);
-static void mpi_unmap_card(struct pci_dev *pci);
-static void mpi_receive_802_3(struct airo_info *ai);
-static void mpi_receive_802_11(struct airo_info *ai);
-static int waitbusy(struct airo_info *ai);
-
-static irqreturn_t airo_interrupt(int irq, void* dev_id);
-static int airo_thread(void *data);
-static void timer_func(struct net_device *dev);
-static int airo_siocdevprivate(struct net_device *dev, struct ifreq *rq, void __user *, int cmd);
-static struct iw_statistics *airo_get_wireless_stats(struct net_device *dev);
-#ifdef CISCO_EXT
-static int readrids(struct net_device *dev, aironet_ioctl *comp);
-static int writerids(struct net_device *dev, aironet_ioctl *comp);
-static int flashcard(struct net_device *dev, aironet_ioctl *comp);
-#endif /* CISCO_EXT */
-static void micinit(struct airo_info *ai);
-static int micsetup(struct airo_info *ai);
-static int encapsulate(struct airo_info *ai, etherHead *pPacket, MICBuffer *buffer, int len);
-static int decapsulate(struct airo_info *ai, MICBuffer *mic, etherHead *pPacket, u16 payLen);
-
-static u8 airo_rssi_to_dbm(tdsRssiEntry *rssi_rid, u8 rssi);
-static u8 airo_dbm_to_pct(tdsRssiEntry *rssi_rid, u8 dbm);
-
-static void airo_networks_free(struct airo_info *ai);
-
-struct airo_info {
-	struct net_device             *dev;
-	struct list_head              dev_list;
-	/* Note, we can have MAX_FIDS outstanding.  FIDs are 16-bits, so we
-	   use the high bit to mark whether it is in use. */
-#define MAX_FIDS 6
-#define MPI_MAX_FIDS 1
-	u32                           fids[MAX_FIDS];
-	ConfigRid config;
-	char keyindex; // Used with auto wep
-	char defindex; // Used with auto wep
-	struct proc_dir_entry *proc_entry;
-        spinlock_t aux_lock;
-#define FLAG_RADIO_OFF	0	/* User disabling of MAC */
-#define FLAG_RADIO_DOWN	1	/* ifup/ifdown disabling of MAC */
-#define FLAG_RADIO_MASK 0x03
-#define FLAG_ENABLED	2
-#define FLAG_ADHOC	3	/* Needed by MIC */
-#define FLAG_MIC_CAPABLE 4
-#define FLAG_UPDATE_MULTI 5
-#define FLAG_UPDATE_UNI 6
-#define FLAG_802_11	7
-#define FLAG_PROMISC	8	/* IFF_PROMISC 0x100 - include/linux/if.h */
-#define FLAG_PENDING_XMIT 9
-#define FLAG_PENDING_XMIT11 10
-#define FLAG_MPI	11
-#define FLAG_REGISTERED	12
-#define FLAG_COMMIT	13
-#define FLAG_RESET	14
-#define FLAG_FLASHING	15
-#define FLAG_WPA_CAPABLE	16
-	unsigned long flags;
-#define JOB_DIE	0
-#define JOB_XMIT	1
-#define JOB_XMIT11	2
-#define JOB_STATS	3
-#define JOB_PROMISC	4
-#define JOB_MIC	5
-#define JOB_EVENT	6
-#define JOB_AUTOWEP	7
-#define JOB_SCAN_RESULTS  9
-	unsigned long jobs;
-	int (*bap_read)(struct airo_info*, __le16 *pu16Dst, int bytelen,
-			int whichbap);
-	unsigned short *flash;
-	tdsRssiEntry *rssi;
-	struct task_struct *list_bss_task;
-	struct task_struct *airo_thread_task;
-	struct semaphore sem;
-	wait_queue_head_t thr_wait;
-	unsigned long expires;
-	struct {
-		struct sk_buff *skb;
-		int fid;
-	} xmit, xmit11;
-	struct net_device *wifidev;
-	struct iw_statistics	wstats;		// wireless stats
-	unsigned long		scan_timeout;	/* Time scan should be read */
-	struct iw_spy_data	spy_data;
-	struct iw_public_data	wireless_data;
-	/* MIC stuff */
-	struct crypto_sync_skcipher	*tfm;
-	mic_module		mod[2];
-	mic_statistics		micstats;
-	HostRxDesc rxfids[MPI_MAX_FIDS]; // rx/tx/config MPI350 descriptors
-	HostTxDesc txfids[MPI_MAX_FIDS];
-	HostRidDesc config_desc;
-	unsigned long ridbus; // phys addr of config_desc
-	struct sk_buff_head txq;// tx queue used by mpi350 code
-	struct pci_dev          *pci;
-	unsigned char		__iomem *pcimem;
-	unsigned char		__iomem *pciaux;
-	unsigned char		*shared;
-	dma_addr_t		shared_dma;
-	pm_message_t		power;
-	SsidRid			*SSID;
-	APListRid		APList;
-#define	PCI_SHARED_LEN		2*MPI_MAX_FIDS*PKTSIZE+RIDSIZE
-	char			proc_name[IFNAMSIZ];
-
-	int			wep_capable;
-	int			max_wep_idx;
-	int			last_auth;
-
-	/* WPA-related stuff */
-	unsigned int bssListFirst;
-	unsigned int bssListNext;
-	unsigned int bssListRidLen;
-
-	struct list_head network_list;
-	struct list_head network_free_list;
-	BSSListElement *networks;
-};
-
-static inline int bap_read(struct airo_info *ai, __le16 *pu16Dst, int bytelen,
-			   int whichbap)
-{
-	return ai->bap_read(ai, pu16Dst, bytelen, whichbap);
-}
-
-static int setup_proc_entry(struct net_device *dev,
-			     struct airo_info *apriv);
-static int takedown_proc_entry(struct net_device *dev,
-				struct airo_info *apriv);
-
-static int cmdreset(struct airo_info *ai);
-static int setflashmode(struct airo_info *ai);
-static int flashgchar(struct airo_info *ai, int matchbyte, int dwelltime);
-static int flashputbuf(struct airo_info *ai);
-static int flashrestart(struct airo_info *ai, struct net_device *dev);
-
-#define airo_print(type, name, fmt, args...) \
-	printk(type DRV_NAME "(%s): " fmt "\n", name, ##args)
-
-#define airo_print_info(name, fmt, args...) \
-	airo_print(KERN_INFO, name, fmt, ##args)
-
-#define airo_print_dbg(name, fmt, args...) \
-	airo_print(KERN_DEBUG, name, fmt, ##args)
-
-#define airo_print_warn(name, fmt, args...) \
-	airo_print(KERN_WARNING, name, fmt, ##args)
-
-#define airo_print_err(name, fmt, args...) \
-	airo_print(KERN_ERR, name, fmt, ##args)
-
-#define AIRO_FLASH(dev) (((struct airo_info *)dev->ml_priv)->flash)
-
-/***********************************************************************
- *                              MIC ROUTINES                           *
- ***********************************************************************
- */
-
-static int RxSeqValid(struct airo_info *ai, miccntx *context, int mcast, u32 micSeq);
-static void MoveWindow(miccntx *context, u32 micSeq);
-static void emmh32_setseed(emmh32_context *context, u8 *pkey, int keylen,
-			   struct crypto_sync_skcipher *tfm);
-static void emmh32_init(emmh32_context *context);
-static void emmh32_update(emmh32_context *context, u8 *pOctets, int len);
-static void emmh32_final(emmh32_context *context, u8 digest[4]);
-static int flashpchar(struct airo_info *ai, int byte, int dwelltime);
-
-static void age_mic_context(miccntx *cur, miccntx *old, u8 *key, int key_len,
-			    struct crypto_sync_skcipher *tfm)
-{
-	/* If the current MIC context is valid and its key is the same as
-	 * the MIC register, there's nothing to do.
-	 */
-	if (cur->valid && (memcmp(cur->key, key, key_len) == 0))
-		return;
-
-	/* Age current mic Context */
-	memcpy(old, cur, sizeof(*cur));
-
-	/* Initialize new context */
-	memcpy(cur->key, key, key_len);
-	cur->window  = 33; /* Window always points to the middle */
-	cur->rx      = 0;  /* Rx Sequence numbers */
-	cur->tx      = 0;  /* Tx sequence numbers */
-	cur->valid   = 1;  /* Key is now valid */
-
-	/* Give key to mic seed */
-	emmh32_setseed(&cur->seed, key, key_len, tfm);
-}
-
-/* micinit - Initialize mic seed */
-
-static void micinit(struct airo_info *ai)
-{
-	MICRid mic_rid;
-
-	clear_bit(JOB_MIC, &ai->jobs);
-	PC4500_readrid(ai, RID_MIC, &mic_rid, sizeof(mic_rid), 0);
-	up(&ai->sem);
-
-	ai->micstats.enabled = (le16_to_cpu(mic_rid.state) & 0x00FF) ? 1 : 0;
-	if (!ai->micstats.enabled) {
-		/* So next time we have a valid key and mic is enabled, we will
-		 * update the sequence number if the key is the same as before.
-		 */
-		ai->mod[0].uCtx.valid = 0;
-		ai->mod[0].mCtx.valid = 0;
-		return;
-	}
-
-	if (mic_rid.multicastValid) {
-		age_mic_context(&ai->mod[0].mCtx, &ai->mod[1].mCtx,
-		                mic_rid.multicast, sizeof(mic_rid.multicast),
-		                ai->tfm);
-	}
-
-	if (mic_rid.unicastValid) {
-		age_mic_context(&ai->mod[0].uCtx, &ai->mod[1].uCtx,
-				mic_rid.unicast, sizeof(mic_rid.unicast),
-				ai->tfm);
-	}
-}
-
-/* micsetup - Get ready for business */
-
-static int micsetup(struct airo_info *ai)
-{
-	int i;
-
-	if (ai->tfm == NULL)
-		ai->tfm = crypto_alloc_sync_skcipher("ctr(aes)", 0, 0);
-
-        if (IS_ERR(ai->tfm)) {
-                airo_print_err(ai->dev->name, "failed to load transform for AES");
-                ai->tfm = NULL;
-                return ERROR;
-        }
-
-	for (i = 0; i < NUM_MODULES; i++) {
-		memset(&ai->mod[i].mCtx, 0, sizeof(miccntx));
-		memset(&ai->mod[i].uCtx, 0, sizeof(miccntx));
-	}
-	return SUCCESS;
-}
-
-static const u8 micsnap[] = {0xAA, 0xAA, 0x03, 0x00, 0x40, 0x96, 0x00, 0x02};
-
-/*===========================================================================
- * Description: Mic a packet
- *
- *      Inputs: etherHead * pointer to an 802.3 frame
- *
- *     Returns: BOOLEAN if successful, otherwise false.
- *             PacketTxLen will be updated with the mic'd packets size.
- *
- *    Caveats: It is assumed that the frame buffer will already
- *             be big enough to hold the largets mic message possible.
- *            (No memory allocation is done here).
- *
- *    Author: sbraneky (10/15/01)
- *    Merciless hacks by rwilcher (1/14/02)
- */
-
-static int encapsulate(struct airo_info *ai, etherHead *frame, MICBuffer *mic, int payLen)
-{
-	miccntx   *context;
-
-	// Determine correct context
-	// If not adhoc, always use unicast key
-
-	if (test_bit(FLAG_ADHOC, &ai->flags) && (frame->da[0] & 0x1))
-		context = &ai->mod[0].mCtx;
-	else
-		context = &ai->mod[0].uCtx;
-
-	if (!context->valid)
-		return ERROR;
-
-	mic->typelen = htons(payLen + 16); //Length of Mic'd packet
-
-	memcpy(&mic->u.snap, micsnap, sizeof(micsnap)); // Add Snap
-
-	// Add Tx sequence
-	mic->seq = htonl(context->tx);
-	context->tx += 2;
-
-	emmh32_init(&context->seed); // Mic the packet
-	emmh32_update(&context->seed, frame->da, ETH_ALEN * 2); // DA, SA
-	emmh32_update(&context->seed, (u8*)&mic->typelen, 10); // Type/Length and Snap
-	emmh32_update(&context->seed, (u8*)&mic->seq, sizeof(mic->seq)); //SEQ
-	emmh32_update(&context->seed, (u8*)(frame + 1), payLen); //payload
-	emmh32_final(&context->seed, (u8*)&mic->mic);
-
-	/*    New Type/length ?????????? */
-	mic->typelen = 0; //Let NIC know it could be an oversized packet
-	return SUCCESS;
-}
-
-typedef enum {
-    NONE,
-    NOMIC,
-    NOMICPLUMMED,
-    SEQUENCE,
-    INCORRECTMIC,
-} mic_error;
-
-/*===========================================================================
- *  Description: Decapsulates a MIC'd packet and returns the 802.3 packet
- *               (removes the MIC stuff) if packet is a valid packet.
- *
- *       Inputs: etherHead  pointer to the 802.3 packet
- *
- *      Returns: BOOLEAN - TRUE if packet should be dropped otherwise FALSE
- *
- *      Author: sbraneky (10/15/01)
- *    Merciless hacks by rwilcher (1/14/02)
- *---------------------------------------------------------------------------
- */
-
-static int decapsulate(struct airo_info *ai, MICBuffer *mic, etherHead *eth, u16 payLen)
-{
-	int      i;
-	u32      micSEQ;
-	miccntx  *context;
-	u8       digest[4];
-	mic_error micError = NONE;
-
-	// Check if the packet is a Mic'd packet
-
-	if (!ai->micstats.enabled) {
-		//No Mic set or Mic OFF but we received a MIC'd packet.
-		if (memcmp ((u8*)eth + 14, micsnap, sizeof(micsnap)) == 0) {
-			ai->micstats.rxMICPlummed++;
-			return ERROR;
-		}
-		return SUCCESS;
-	}
-
-	if (ntohs(mic->typelen) == 0x888E)
-		return SUCCESS;
-
-	if (memcmp (mic->u.snap, micsnap, sizeof(micsnap)) != 0) {
-	    // Mic enabled but packet isn't Mic'd
-		ai->micstats.rxMICPlummed++;
-	    	return ERROR;
-	}
-
-	micSEQ = ntohl(mic->seq);            //store SEQ as CPU order
-
-	//At this point we a have a mic'd packet and mic is enabled
-	//Now do the mic error checking.
-
-	//Receive seq must be odd
-	if ((micSEQ & 1) == 0) {
-		ai->micstats.rxWrongSequence++;
-		return ERROR;
-	}
-
-	for (i = 0; i < NUM_MODULES; i++) {
-		int mcast = eth->da[0] & 1;
-		//Determine proper context
-		context = mcast ? &ai->mod[i].mCtx : &ai->mod[i].uCtx;
-
-		//Make sure context is valid
-		if (!context->valid) {
-			if (i == 0)
-				micError = NOMICPLUMMED;
-			continue;
-		}
-		//DeMic it
-
-		if (!mic->typelen)
-			mic->typelen = htons(payLen + sizeof(MICBuffer) - 2);
-
-		emmh32_init(&context->seed);
-		emmh32_update(&context->seed, eth->da, ETH_ALEN*2);
-		emmh32_update(&context->seed, (u8 *)&mic->typelen, sizeof(mic->typelen)+sizeof(mic->u.snap));
-		emmh32_update(&context->seed, (u8 *)&mic->seq, sizeof(mic->seq));
-		emmh32_update(&context->seed, (u8 *)(eth + 1), payLen);
-		//Calculate MIC
-		emmh32_final(&context->seed, digest);
-
-		if (memcmp(digest, &mic->mic, 4)) { //Make sure the mics match
-		  //Invalid Mic
-			if (i == 0)
-				micError = INCORRECTMIC;
-			continue;
-		}
-
-		//Check Sequence number if mics pass
-		if (RxSeqValid(ai, context, mcast, micSEQ) == SUCCESS) {
-			ai->micstats.rxSuccess++;
-			return SUCCESS;
-		}
-		if (i == 0)
-			micError = SEQUENCE;
-	}
-
-	// Update statistics
-	switch (micError) {
-		case NOMICPLUMMED: ai->micstats.rxMICPlummed++;   break;
-		case SEQUENCE:    ai->micstats.rxWrongSequence++; break;
-		case INCORRECTMIC: ai->micstats.rxIncorrectMIC++; break;
-		case NONE:  break;
-		case NOMIC: break;
-	}
-	return ERROR;
-}
-
-/*===========================================================================
- * Description:  Checks the Rx Seq number to make sure it is valid
- *               and hasn't already been received
- *
- *     Inputs: miccntx - mic context to check seq against
- *             micSeq  - the Mic seq number
- *
- *    Returns: TRUE if valid otherwise FALSE.
- *
- *    Author: sbraneky (10/15/01)
- *    Merciless hacks by rwilcher (1/14/02)
- *---------------------------------------------------------------------------
- */
-
-static int RxSeqValid(struct airo_info *ai, miccntx *context, int mcast, u32 micSeq)
-{
-	u32 seq, index;
-
-	//Allow for the ap being rebooted - if it is then use the next
-	//sequence number of the current sequence number - might go backwards
-
-	if (mcast) {
-		if (test_bit(FLAG_UPDATE_MULTI, &ai->flags)) {
-			clear_bit (FLAG_UPDATE_MULTI, &ai->flags);
-			context->window = (micSeq > 33) ? micSeq : 33;
-			context->rx     = 0;        // Reset rx
-		}
-	} else if (test_bit(FLAG_UPDATE_UNI, &ai->flags)) {
-		clear_bit (FLAG_UPDATE_UNI, &ai->flags);
-		context->window = (micSeq > 33) ? micSeq : 33; // Move window
-		context->rx     = 0;        // Reset rx
-	}
-
-	//Make sequence number relative to START of window
-	seq = micSeq - (context->window - 33);
-
-	//Too old of a SEQ number to check.
-	if ((s32)seq < 0)
-		return ERROR;
-
-	if (seq > 64) {
-		//Window is infinite forward
-		MoveWindow(context, micSeq);
-		return SUCCESS;
-	}
-
-	// We are in the window. Now check the context rx bit to see if it was already sent
-	seq >>= 1;         //divide by 2 because we only have odd numbers
-	index = 1 << seq;  //Get an index number
-
-	if (!(context->rx & index)) {
-		//micSEQ falls inside the window.
-		//Add seqence number to the list of received numbers.
-		context->rx |= index;
-
-		MoveWindow(context, micSeq);
-
-		return SUCCESS;
-	}
-	return ERROR;
-}
-
-static void MoveWindow(miccntx *context, u32 micSeq)
-{
-	u32 shift;
-
-	//Move window if seq greater than the middle of the window
-	if (micSeq > context->window) {
-		shift = (micSeq - context->window) >> 1;
-
-		    //Shift out old
-		if (shift < 32)
-			context->rx >>= shift;
-		else
-			context->rx = 0;
-
-		context->window = micSeq;      //Move window
-	}
-}
-
-/*==============================================*/
-/*========== EMMH ROUTINES  ====================*/
-/*==============================================*/
-
-/* mic accumulate */
-#define MIC_ACCUM(val)	\
-	context->accum += (u64)(val) * be32_to_cpu(context->coeff[coeff_position++]);
-
-/* expand the key to fill the MMH coefficient array */
-static void emmh32_setseed(emmh32_context *context, u8 *pkey, int keylen,
-			   struct crypto_sync_skcipher *tfm)
-{
-  /* take the keying material, expand if necessary, truncate at 16-bytes */
-  /* run through AES counter mode to generate context->coeff[] */
-
-	SYNC_SKCIPHER_REQUEST_ON_STACK(req, tfm);
-	struct scatterlist sg;
-	u8 iv[AES_BLOCK_SIZE] = {};
-	int ret;
-
-	crypto_sync_skcipher_setkey(tfm, pkey, 16);
-
-	memset(context->coeff, 0, sizeof(context->coeff));
-	sg_init_one(&sg, context->coeff, sizeof(context->coeff));
-
-	skcipher_request_set_sync_tfm(req, tfm);
-	skcipher_request_set_callback(req, 0, NULL, NULL);
-	skcipher_request_set_crypt(req, &sg, &sg, sizeof(context->coeff), iv);
-
-	ret = crypto_skcipher_encrypt(req);
-	WARN_ON_ONCE(ret);
-}
-
-/* prepare for calculation of a new mic */
-static void emmh32_init(emmh32_context *context)
-{
-	/* prepare for new mic calculation */
-	context->accum = 0;
-	context->position = 0;
-}
-
-/* add some bytes to the mic calculation */
-static void emmh32_update(emmh32_context *context, u8 *pOctets, int len)
-{
-	int	coeff_position, byte_position;
-
-	if (len == 0) return;
-
-	coeff_position = context->position >> 2;
-
-	/* deal with partial 32-bit word left over from last update */
-	byte_position = context->position & 3;
-	if (byte_position) {
-		/* have a partial word in part to deal with */
-		do {
-			if (len == 0) return;
-			context->part.d8[byte_position++] = *pOctets++;
-			context->position++;
-			len--;
-		} while (byte_position < 4);
-		MIC_ACCUM(ntohl(context->part.d32));
-	}
-
-	/* deal with full 32-bit words */
-	while (len >= 4) {
-		MIC_ACCUM(ntohl(*(__be32 *)pOctets));
-		context->position += 4;
-		pOctets += 4;
-		len -= 4;
-	}
-
-	/* deal with partial 32-bit word that will be left over from this update */
-	byte_position = 0;
-	while (len > 0) {
-		context->part.d8[byte_position++] = *pOctets++;
-		context->position++;
-		len--;
-	}
-}
-
-/* mask used to zero empty bytes for final partial word */
-static u32 mask32[4] = { 0x00000000L, 0xFF000000L, 0xFFFF0000L, 0xFFFFFF00L };
-
-/* calculate the mic */
-static void emmh32_final(emmh32_context *context, u8 digest[4])
-{
-	int	coeff_position, byte_position;
-	u32	val;
-
-	u64 sum, utmp;
-	s64 stmp;
-
-	coeff_position = context->position >> 2;
-
-	/* deal with partial 32-bit word left over from last update */
-	byte_position = context->position & 3;
-	if (byte_position) {
-		/* have a partial word in part to deal with */
-		val = ntohl(context->part.d32);
-		MIC_ACCUM(val & mask32[byte_position]);	/* zero empty bytes */
-	}
-
-	/* reduce the accumulated u64 to a 32-bit MIC */
-	sum = context->accum;
-	stmp = (sum  & 0xffffffffLL) - ((sum >> 32)  * 15);
-	utmp = (stmp & 0xffffffffLL) - ((stmp >> 32) * 15);
-	sum = utmp & 0xffffffffLL;
-	if (utmp > 0x10000000fLL)
-		sum -= 15;
-
-	val = (u32)sum;
-	digest[0] = (val>>24) & 0xFF;
-	digest[1] = (val>>16) & 0xFF;
-	digest[2] = (val>>8) & 0xFF;
-	digest[3] = val & 0xFF;
-}
-
-static int readBSSListRid(struct airo_info *ai, int first,
-		      BSSListRid *list)
-{
-	Cmd cmd;
-	Resp rsp;
-
-	if (first == 1) {
-		if (ai->flags & FLAG_RADIO_MASK) return -ENETDOWN;
-		memset(&cmd, 0, sizeof(cmd));
-		cmd.cmd = CMD_LISTBSS;
-		if (down_interruptible(&ai->sem))
-			return -ERESTARTSYS;
-		ai->list_bss_task = current;
-		issuecommand(ai, &cmd, &rsp, true);
-		up(&ai->sem);
-		/* Let the command take effect */
-		schedule_timeout_uninterruptible(3 * HZ);
-		ai->list_bss_task = NULL;
-	}
-	return PC4500_readrid(ai, first ? ai->bssListFirst : ai->bssListNext,
-			    list, ai->bssListRidLen, 1);
-}
-
-static int readWepKeyRid(struct airo_info *ai, WepKeyRid *wkr, int temp, int lock)
-{
-	return PC4500_readrid(ai, temp ? RID_WEP_TEMP : RID_WEP_PERM,
-				wkr, sizeof(*wkr), lock);
-}
-
-static int writeWepKeyRid(struct airo_info *ai, WepKeyRid *wkr, int perm, int lock)
-{
-	int rc;
-	rc = PC4500_writerid(ai, RID_WEP_TEMP, wkr, sizeof(*wkr), lock);
-	if (rc!=SUCCESS)
-		airo_print_err(ai->dev->name, "WEP_TEMP set %x", rc);
-	if (perm) {
-		rc = PC4500_writerid(ai, RID_WEP_PERM, wkr, sizeof(*wkr), lock);
-		if (rc!=SUCCESS)
-			airo_print_err(ai->dev->name, "WEP_PERM set %x", rc);
-	}
-	return rc;
-}
-
-static int readSsidRid(struct airo_info*ai, SsidRid *ssidr)
-{
-	return PC4500_readrid(ai, RID_SSID, ssidr, sizeof(*ssidr), 1);
-}
-
-static int writeSsidRid(struct airo_info*ai, SsidRid *pssidr, int lock)
-{
-	return PC4500_writerid(ai, RID_SSID, pssidr, sizeof(*pssidr), lock);
-}
-
-static int readConfigRid(struct airo_info *ai, int lock)
-{
-	int rc;
-	ConfigRid cfg;
-
-	if (ai->config.len)
-		return SUCCESS;
-
-	rc = PC4500_readrid(ai, RID_ACTUALCONFIG, &cfg, sizeof(cfg), lock);
-	if (rc != SUCCESS)
-		return rc;
-
-	ai->config = cfg;
-	return SUCCESS;
-}
-
-static inline void checkThrottle(struct airo_info *ai)
-{
-	int i;
-/* Old hardware had a limit on encryption speed */
-	if (ai->config.authType != AUTH_OPEN && maxencrypt) {
-		for (i = 0; i<8; i++) {
-			if (ai->config.rates[i] > maxencrypt) {
-				ai->config.rates[i] = 0;
-			}
-		}
-	}
-}
-
-static int writeConfigRid(struct airo_info *ai, int lock)
-{
-	ConfigRid cfgr;
-
-	if (!test_bit (FLAG_COMMIT, &ai->flags))
-		return SUCCESS;
-
-	clear_bit (FLAG_COMMIT, &ai->flags);
-	clear_bit (FLAG_RESET, &ai->flags);
-	checkThrottle(ai);
-	cfgr = ai->config;
-
-	if ((cfgr.opmode & MODE_CFG_MASK) == MODE_STA_IBSS)
-		set_bit(FLAG_ADHOC, &ai->flags);
-	else
-		clear_bit(FLAG_ADHOC, &ai->flags);
-
-	return PC4500_writerid(ai, RID_CONFIG, &cfgr, sizeof(cfgr), lock);
-}
-
-static int readStatusRid(struct airo_info *ai, StatusRid *statr, int lock)
-{
-	return PC4500_readrid(ai, RID_STATUS, statr, sizeof(*statr), lock);
-}
-
-static int writeAPListRid(struct airo_info *ai, APListRid *aplr, int lock)
-{
-	return PC4500_writerid(ai, RID_APLIST, aplr, sizeof(*aplr), lock);
-}
-
-static int readCapabilityRid(struct airo_info *ai, CapabilityRid *capr, int lock)
-{
-	return PC4500_readrid(ai, RID_CAPABILITIES, capr, sizeof(*capr), lock);
-}
-
-static int readStatsRid(struct airo_info*ai, StatsRid *sr, int rid, int lock)
-{
-	return PC4500_readrid(ai, rid, sr, sizeof(*sr), lock);
-}
-
-static void try_auto_wep(struct airo_info *ai)
-{
-	if (auto_wep && !test_bit(FLAG_RADIO_DOWN, &ai->flags)) {
-		ai->expires = RUN_AT(3*HZ);
-		wake_up_interruptible(&ai->thr_wait);
-	}
-}
-
-static int airo_open(struct net_device *dev)
-{
-	struct airo_info *ai = dev->ml_priv;
-	int rc = 0;
-
-	if (test_bit(FLAG_FLASHING, &ai->flags))
-		return -EIO;
-
-	/* Make sure the card is configured.
-	 * Wireless Extensions may postpone config changes until the card
-	 * is open (to pipeline changes and speed-up card setup). If
-	 * those changes are not yet committed, do it now - Jean II */
-	if (test_bit(FLAG_COMMIT, &ai->flags)) {
-		disable_MAC(ai, 1);
-		writeConfigRid(ai, 1);
-	}
-
-	if (ai->wifidev != dev) {
-		clear_bit(JOB_DIE, &ai->jobs);
-		ai->airo_thread_task = kthread_run(airo_thread, dev, "%s",
-						   dev->name);
-		if (IS_ERR(ai->airo_thread_task))
-			return (int)PTR_ERR(ai->airo_thread_task);
-
-		rc = request_irq(dev->irq, airo_interrupt, IRQF_SHARED,
-			dev->name, dev);
-		if (rc) {
-			airo_print_err(dev->name,
-				"register interrupt %d failed, rc %d",
-				dev->irq, rc);
-			set_bit(JOB_DIE, &ai->jobs);
-			kthread_stop(ai->airo_thread_task);
-			return rc;
-		}
-
-		/* Power on the MAC controller (which may have been disabled) */
-		clear_bit(FLAG_RADIO_DOWN, &ai->flags);
-		enable_interrupts(ai);
-
-		try_auto_wep(ai);
-	}
-	enable_MAC(ai, 1);
-
-	netif_start_queue(dev);
-	return 0;
-}
-
-static netdev_tx_t mpi_start_xmit(struct sk_buff *skb,
-					struct net_device *dev)
-{
-	int npacks, pending;
-	unsigned long flags;
-	struct airo_info *ai = dev->ml_priv;
-
-	if (!skb) {
-		airo_print_err(dev->name, "%s: skb == NULL!",__func__);
-		return NETDEV_TX_OK;
-	}
-	if (skb_padto(skb, ETH_ZLEN)) {
-		dev->stats.tx_dropped++;
-		return NETDEV_TX_OK;
-	}
-	npacks = skb_queue_len (&ai->txq);
-
-	if (npacks >= MAXTXQ - 1) {
-		netif_stop_queue (dev);
-		if (npacks > MAXTXQ) {
-			dev->stats.tx_fifo_errors++;
-			return NETDEV_TX_BUSY;
-		}
-		skb_queue_tail (&ai->txq, skb);
-		return NETDEV_TX_OK;
-	}
-
-	spin_lock_irqsave(&ai->aux_lock, flags);
-	skb_queue_tail (&ai->txq, skb);
-	pending = test_bit(FLAG_PENDING_XMIT, &ai->flags);
-	spin_unlock_irqrestore(&ai->aux_lock, flags);
-	netif_wake_queue (dev);
-
-	if (pending == 0) {
-		set_bit(FLAG_PENDING_XMIT, &ai->flags);
-		mpi_send_packet (dev);
-	}
-	return NETDEV_TX_OK;
-}
-
-/*
- * @mpi_send_packet
- *
- * Attempt to transmit a packet. Can be called from interrupt
- * or transmit . return number of packets we tried to send
- */
-
-static int mpi_send_packet (struct net_device *dev)
-{
-	struct sk_buff *skb;
-	unsigned char *buffer;
-	s16 len;
-	__le16 *payloadLen;
-	struct airo_info *ai = dev->ml_priv;
-	u8 *sendbuf;
-
-	/* get a packet to send */
-
-	if ((skb = skb_dequeue(&ai->txq)) == NULL) {
-		airo_print_err(dev->name,
-			"%s: Dequeue'd zero in send_packet()",
-			__func__);
-		return 0;
-	}
-
-	/* check min length*/
-	len = ETH_ZLEN < skb->len ? skb->len : ETH_ZLEN;
-	buffer = skb->data;
-
-	ai->txfids[0].tx_desc.offset = 0;
-	ai->txfids[0].tx_desc.valid = 1;
-	ai->txfids[0].tx_desc.eoc = 1;
-	ai->txfids[0].tx_desc.len =len+sizeof(WifiHdr);
-
-/*
- * Magic, the cards firmware needs a length count (2 bytes) in the host buffer
- * right after  TXFID_HDR.The TXFID_HDR contains the status short so payloadlen
- * is immediately after it. ------------------------------------------------
- *                         |TXFIDHDR+STATUS|PAYLOADLEN|802.3HDR|PACKETDATA|
- *                         ------------------------------------------------
- */
-
-	memcpy(ai->txfids[0].virtual_host_addr,
-		(char *)&wifictlhdr8023, sizeof(wifictlhdr8023));
-
-	payloadLen = (__le16 *)(ai->txfids[0].virtual_host_addr +
-		sizeof(wifictlhdr8023));
-	sendbuf = ai->txfids[0].virtual_host_addr +
-		sizeof(wifictlhdr8023) + 2 ;
-
-	/*
-	 * Firmware automatically puts 802 header on so
-	 * we don't need to account for it in the length
-	 */
-	if (test_bit(FLAG_MIC_CAPABLE, &ai->flags) && ai->micstats.enabled &&
-		(ntohs(((__be16 *)buffer)[6]) != 0x888E)) {
-		MICBuffer pMic;
-
-		if (encapsulate(ai, (etherHead *)buffer, &pMic, len - sizeof(etherHead)) != SUCCESS)
-			return ERROR;
-
-		*payloadLen = cpu_to_le16(len-sizeof(etherHead)+sizeof(pMic));
-		ai->txfids[0].tx_desc.len += sizeof(pMic);
-		/* copy data into airo dma buffer */
-		memcpy (sendbuf, buffer, sizeof(etherHead));
-		buffer += sizeof(etherHead);
-		sendbuf += sizeof(etherHead);
-		memcpy (sendbuf, &pMic, sizeof(pMic));
-		sendbuf += sizeof(pMic);
-		memcpy (sendbuf, buffer, len - sizeof(etherHead));
-	} else {
-		*payloadLen = cpu_to_le16(len - sizeof(etherHead));
-
-		netif_trans_update(dev);
-
-		/* copy data into airo dma buffer */
-		memcpy(sendbuf, buffer, len);
-	}
-
-	memcpy_toio(ai->txfids[0].card_ram_off,
-		&ai->txfids[0].tx_desc, sizeof(TxFid));
-
-	OUT4500(ai, EVACK, 8);
-
-	dev_kfree_skb_any(skb);
-	return 1;
-}
-
-static void get_tx_error(struct airo_info *ai, s32 fid)
-{
-	__le16 status;
-
-	if (fid < 0)
-		status = ((WifiCtlHdr *)ai->txfids[0].virtual_host_addr)->ctlhdr.status;
-	else {
-		if (bap_setup(ai, ai->fids[fid] & 0xffff, 4, BAP0) != SUCCESS)
-			return;
-		bap_read(ai, &status, 2, BAP0);
-	}
-	if (le16_to_cpu(status) & 2) /* Too many retries */
-		ai->dev->stats.tx_aborted_errors++;
-	if (le16_to_cpu(status) & 4) /* Transmit lifetime exceeded */
-		ai->dev->stats.tx_heartbeat_errors++;
-	if (le16_to_cpu(status) & 8) /* Aid fail */
-		{ }
-	if (le16_to_cpu(status) & 0x10) /* MAC disabled */
-		ai->dev->stats.tx_carrier_errors++;
-	if (le16_to_cpu(status) & 0x20) /* Association lost */
-		{ }
-	/* We produce a TXDROP event only for retry or lifetime
-	 * exceeded, because that's the only status that really mean
-	 * that this particular node went away.
-	 * Other errors means that *we* screwed up. - Jean II */
-	if ((le16_to_cpu(status) & 2) ||
-	     (le16_to_cpu(status) & 4)) {
-		union iwreq_data	wrqu;
-		char junk[0x18];
-
-		/* Faster to skip over useless data than to do
-		 * another bap_setup(). We are at offset 0x6 and
-		 * need to go to 0x18 and read 6 bytes - Jean II */
-		bap_read(ai, (__le16 *) junk, 0x18, BAP0);
-
-		/* Copy 802.11 dest address.
-		 * We use the 802.11 header because the frame may
-		 * not be 802.3 or may be mangled...
-		 * In Ad-Hoc mode, it will be the node address.
-		 * In managed mode, it will be most likely the AP addr
-		 * User space will figure out how to convert it to
-		 * whatever it needs (IP address or else).
-		 * - Jean II */
-		memcpy(wrqu.addr.sa_data, junk + 0x12, ETH_ALEN);
-		wrqu.addr.sa_family = ARPHRD_ETHER;
-
-		/* Send event to user space */
-		wireless_send_event(ai->dev, IWEVTXDROP, &wrqu, NULL);
-	}
-}
-
-static void airo_end_xmit(struct net_device *dev, bool may_sleep)
-{
-	u16 status;
-	int i;
-	struct airo_info *priv = dev->ml_priv;
-	struct sk_buff *skb = priv->xmit.skb;
-	int fid = priv->xmit.fid;
-	u32 *fids = priv->fids;
-
-	clear_bit(JOB_XMIT, &priv->jobs);
-	clear_bit(FLAG_PENDING_XMIT, &priv->flags);
-	status = transmit_802_3_packet(priv, fids[fid], skb->data, may_sleep);
-	up(&priv->sem);
-
-	i = 0;
-	if (status == SUCCESS) {
-		netif_trans_update(dev);
-		for (; i < MAX_FIDS / 2 && (priv->fids[i] & 0xffff0000); i++);
-	} else {
-		priv->fids[fid] &= 0xffff;
-		dev->stats.tx_window_errors++;
-	}
-	if (i < MAX_FIDS / 2)
-		netif_wake_queue(dev);
-	dev_kfree_skb(skb);
-}
-
-static netdev_tx_t airo_start_xmit(struct sk_buff *skb,
-					 struct net_device *dev)
-{
-	s16 len;
-	int i, j;
-	struct airo_info *priv = dev->ml_priv;
-	u32 *fids = priv->fids;
-
-	if (skb == NULL) {
-		airo_print_err(dev->name, "%s: skb == NULL!", __func__);
-		return NETDEV_TX_OK;
-	}
-	if (skb_padto(skb, ETH_ZLEN)) {
-		dev->stats.tx_dropped++;
-		return NETDEV_TX_OK;
-	}
-
-	/* Find a vacant FID */
-	for (i = 0; i < MAX_FIDS / 2 && (fids[i] & 0xffff0000); i++);
-	for (j = i + 1; j < MAX_FIDS / 2 && (fids[j] & 0xffff0000); j++);
-
-	if (j >= MAX_FIDS / 2) {
-		netif_stop_queue(dev);
-
-		if (i == MAX_FIDS / 2) {
-			dev->stats.tx_fifo_errors++;
-			return NETDEV_TX_BUSY;
-		}
-	}
-	/* check min length*/
-	len = ETH_ZLEN < skb->len ? skb->len : ETH_ZLEN;
-        /* Mark fid as used & save length for later */
-	fids[i] |= (len << 16);
-	priv->xmit.skb = skb;
-	priv->xmit.fid = i;
-	if (down_trylock(&priv->sem) != 0) {
-		set_bit(FLAG_PENDING_XMIT, &priv->flags);
-		netif_stop_queue(dev);
-		set_bit(JOB_XMIT, &priv->jobs);
-		wake_up_interruptible(&priv->thr_wait);
-	} else
-		airo_end_xmit(dev, false);
-	return NETDEV_TX_OK;
-}
-
-static void airo_end_xmit11(struct net_device *dev, bool may_sleep)
-{
-	u16 status;
-	int i;
-	struct airo_info *priv = dev->ml_priv;
-	struct sk_buff *skb = priv->xmit11.skb;
-	int fid = priv->xmit11.fid;
-	u32 *fids = priv->fids;
-
-	clear_bit(JOB_XMIT11, &priv->jobs);
-	clear_bit(FLAG_PENDING_XMIT11, &priv->flags);
-	status = transmit_802_11_packet(priv, fids[fid], skb->data, may_sleep);
-	up(&priv->sem);
-
-	i = MAX_FIDS / 2;
-	if (status == SUCCESS) {
-		netif_trans_update(dev);
-		for (; i < MAX_FIDS && (priv->fids[i] & 0xffff0000); i++);
-	} else {
-		priv->fids[fid] &= 0xffff;
-		dev->stats.tx_window_errors++;
-	}
-	if (i < MAX_FIDS)
-		netif_wake_queue(dev);
-	dev_kfree_skb(skb);
-}
-
-static netdev_tx_t airo_start_xmit11(struct sk_buff *skb,
-					   struct net_device *dev)
-{
-	s16 len;
-	int i, j;
-	struct airo_info *priv = dev->ml_priv;
-	u32 *fids = priv->fids;
-
-	if (test_bit(FLAG_MPI, &priv->flags)) {
-		/* Not implemented yet for MPI350 */
-		netif_stop_queue(dev);
-		dev_kfree_skb_any(skb);
-		return NETDEV_TX_OK;
-	}
-
-	if (skb == NULL) {
-		airo_print_err(dev->name, "%s: skb == NULL!", __func__);
-		return NETDEV_TX_OK;
-	}
-	if (skb_padto(skb, ETH_ZLEN)) {
-		dev->stats.tx_dropped++;
-		return NETDEV_TX_OK;
-	}
-
-	/* Find a vacant FID */
-	for (i = MAX_FIDS / 2; i < MAX_FIDS && (fids[i] & 0xffff0000); i++);
-	for (j = i + 1; j < MAX_FIDS && (fids[j] & 0xffff0000); j++);
-
-	if (j >= MAX_FIDS) {
-		netif_stop_queue(dev);
-
-		if (i == MAX_FIDS) {
-			dev->stats.tx_fifo_errors++;
-			return NETDEV_TX_BUSY;
-		}
-	}
-	/* check min length*/
-	len = ETH_ZLEN < skb->len ? skb->len : ETH_ZLEN;
-        /* Mark fid as used & save length for later */
-	fids[i] |= (len << 16);
-	priv->xmit11.skb = skb;
-	priv->xmit11.fid = i;
-	if (down_trylock(&priv->sem) != 0) {
-		set_bit(FLAG_PENDING_XMIT11, &priv->flags);
-		netif_stop_queue(dev);
-		set_bit(JOB_XMIT11, &priv->jobs);
-		wake_up_interruptible(&priv->thr_wait);
-	} else
-		airo_end_xmit11(dev, false);
-	return NETDEV_TX_OK;
-}
-
-static void airo_read_stats(struct net_device *dev)
-{
-	struct airo_info *ai = dev->ml_priv;
-	StatsRid stats_rid;
-	__le32 *vals = stats_rid.vals;
-
-	clear_bit(JOB_STATS, &ai->jobs);
-	if (ai->power.event) {
-		up(&ai->sem);
-		return;
-	}
-	readStatsRid(ai, &stats_rid, RID_STATS, 0);
-	up(&ai->sem);
-
-	dev->stats.rx_packets = le32_to_cpu(vals[43]) + le32_to_cpu(vals[44]) +
-			       le32_to_cpu(vals[45]);
-	dev->stats.tx_packets = le32_to_cpu(vals[39]) + le32_to_cpu(vals[40]) +
-			       le32_to_cpu(vals[41]);
-	dev->stats.rx_bytes = le32_to_cpu(vals[92]);
-	dev->stats.tx_bytes = le32_to_cpu(vals[91]);
-	dev->stats.rx_errors = le32_to_cpu(vals[0]) + le32_to_cpu(vals[2]) +
-			      le32_to_cpu(vals[3]) + le32_to_cpu(vals[4]);
-	dev->stats.tx_errors = le32_to_cpu(vals[42]) +
-			      dev->stats.tx_fifo_errors;
-	dev->stats.multicast = le32_to_cpu(vals[43]);
-	dev->stats.collisions = le32_to_cpu(vals[89]);
-
-	/* detailed rx_errors: */
-	dev->stats.rx_length_errors = le32_to_cpu(vals[3]);
-	dev->stats.rx_crc_errors = le32_to_cpu(vals[4]);
-	dev->stats.rx_frame_errors = le32_to_cpu(vals[2]);
-	dev->stats.rx_fifo_errors = le32_to_cpu(vals[0]);
-}
-
-static struct net_device_stats *airo_get_stats(struct net_device *dev)
-{
-	struct airo_info *local =  dev->ml_priv;
-
-	if (!test_bit(JOB_STATS, &local->jobs)) {
-		set_bit(JOB_STATS, &local->jobs);
-		wake_up_interruptible(&local->thr_wait);
-	}
-
-	return &dev->stats;
-}
-
-static void airo_set_promisc(struct airo_info *ai, bool may_sleep)
-{
-	Cmd cmd;
-	Resp rsp;
-
-	memset(&cmd, 0, sizeof(cmd));
-	cmd.cmd = CMD_SETMODE;
-	clear_bit(JOB_PROMISC, &ai->jobs);
-	cmd.parm0=(ai->flags&IFF_PROMISC) ? PROMISC : NOPROMISC;
-	issuecommand(ai, &cmd, &rsp, may_sleep);
-	up(&ai->sem);
-}
-
-static void airo_set_multicast_list(struct net_device *dev)
-{
-	struct airo_info *ai = dev->ml_priv;
-
-	if ((dev->flags ^ ai->flags) & IFF_PROMISC) {
-		change_bit(FLAG_PROMISC, &ai->flags);
-		if (down_trylock(&ai->sem) != 0) {
-			set_bit(JOB_PROMISC, &ai->jobs);
-			wake_up_interruptible(&ai->thr_wait);
-		} else
-			airo_set_promisc(ai, false);
-	}
-
-	if ((dev->flags&IFF_ALLMULTI) || !netdev_mc_empty(dev)) {
-		/* Turn on multicast.  (Should be already setup...) */
-	}
-}
-
-static int airo_set_mac_address(struct net_device *dev, void *p)
-{
-	struct airo_info *ai = dev->ml_priv;
-	struct sockaddr *addr = p;
-
-	readConfigRid(ai, 1);
-	memcpy (ai->config.macAddr, addr->sa_data, dev->addr_len);
-	set_bit (FLAG_COMMIT, &ai->flags);
-	disable_MAC(ai, 1);
-	writeConfigRid (ai, 1);
-	enable_MAC(ai, 1);
-	dev_addr_set(ai->dev, addr->sa_data);
-	if (ai->wifidev)
-		dev_addr_set(ai->wifidev, addr->sa_data);
-	return 0;
-}
-
-static LIST_HEAD(airo_devices);
-
-static void add_airo_dev(struct airo_info *ai)
-{
-	/* Upper layers already keep track of PCI devices,
-	 * so we only need to remember our non-PCI cards. */
-	if (!ai->pci)
-		list_add_tail(&ai->dev_list, &airo_devices);
-}
-
-static void del_airo_dev(struct airo_info *ai)
-{
-	if (!ai->pci)
-		list_del(&ai->dev_list);
-}
-
-static int airo_close(struct net_device *dev)
-{
-	struct airo_info *ai = dev->ml_priv;
-
-	netif_stop_queue(dev);
-
-	if (ai->wifidev != dev) {
-#ifdef POWER_ON_DOWN
-		/* Shut power to the card. The idea is that the user can save
-		 * power when he doesn't need the card with "ifconfig down".
-		 * That's the method that is most friendly towards the network
-		 * stack (i.e. the network stack won't try to broadcast
-		 * anything on the interface and routes are gone. Jean II */
-		set_bit(FLAG_RADIO_DOWN, &ai->flags);
-		disable_MAC(ai, 1);
-#endif
-		disable_interrupts(ai);
-
-		free_irq(dev->irq, dev);
-
-		set_bit(JOB_DIE, &ai->jobs);
-		kthread_stop(ai->airo_thread_task);
-	}
-	return 0;
-}
-
-void stop_airo_card(struct net_device *dev, int freeres)
-{
-	struct airo_info *ai = dev->ml_priv;
-
-	set_bit(FLAG_RADIO_DOWN, &ai->flags);
-	disable_MAC(ai, 1);
-	disable_interrupts(ai);
-	takedown_proc_entry(dev, ai);
-	if (test_bit(FLAG_REGISTERED, &ai->flags)) {
-		unregister_netdev(dev);
-		if (ai->wifidev) {
-			unregister_netdev(ai->wifidev);
-			free_netdev(ai->wifidev);
-			ai->wifidev = NULL;
-		}
-		clear_bit(FLAG_REGISTERED, &ai->flags);
-	}
-	/*
-	 * Clean out tx queue
-	 */
-	if (test_bit(FLAG_MPI, &ai->flags) && !skb_queue_empty(&ai->txq)) {
-		struct sk_buff *skb = NULL;
-		for (;(skb = skb_dequeue(&ai->txq));)
-			dev_kfree_skb(skb);
-	}
-
-	airo_networks_free (ai);
-
-	kfree(ai->flash);
-	kfree(ai->rssi);
-	kfree(ai->SSID);
-	if (freeres) {
-		/* PCMCIA frees this stuff, so only for PCI and ISA */
-		release_region(dev->base_addr, 64);
-		if (test_bit(FLAG_MPI, &ai->flags)) {
-			if (ai->pci)
-				mpi_unmap_card(ai->pci);
-			if (ai->pcimem)
-				iounmap(ai->pcimem);
-			if (ai->pciaux)
-				iounmap(ai->pciaux);
-			dma_free_coherent(&ai->pci->dev, PCI_SHARED_LEN,
-					  ai->shared, ai->shared_dma);
-		}
-        }
-	crypto_free_sync_skcipher(ai->tfm);
-	del_airo_dev(ai);
-	free_netdev(dev);
-}
-
-EXPORT_SYMBOL(stop_airo_card);
-
-static int wll_header_parse(const struct sk_buff *skb, unsigned char *haddr)
-{
-	memcpy(haddr, skb_mac_header(skb) + 10, ETH_ALEN);
-	return ETH_ALEN;
-}
-
-static void mpi_unmap_card(struct pci_dev *pci)
-{
-	unsigned long mem_start = pci_resource_start(pci, 1);
-	unsigned long mem_len = pci_resource_len(pci, 1);
-	unsigned long aux_start = pci_resource_start(pci, 2);
-	unsigned long aux_len = AUXMEMSIZE;
-
-	release_mem_region(aux_start, aux_len);
-	release_mem_region(mem_start, mem_len);
-}
-
-/*************************************************************
- *  This routine assumes that descriptors have been setup .
- *  Run at insmod time or after reset when the descriptors
- *  have been initialized . Returns 0 if all is well nz
- *  otherwise . Does not allocate memory but sets up card
- *  using previously allocated descriptors.
- */
-static int mpi_init_descriptors (struct airo_info *ai)
-{
-	Cmd cmd;
-	Resp rsp;
-	int i;
-	int rc = SUCCESS;
-
-	/* Alloc  card RX descriptors */
-	netif_stop_queue(ai->dev);
-
-	memset(&rsp, 0, sizeof(rsp));
-	memset(&cmd, 0, sizeof(cmd));
-
-	cmd.cmd = CMD_ALLOCATEAUX;
-	cmd.parm0 = FID_RX;
-	cmd.parm1 = (ai->rxfids[0].card_ram_off - ai->pciaux);
-	cmd.parm2 = MPI_MAX_FIDS;
-	rc = issuecommand(ai, &cmd, &rsp, true);
-	if (rc != SUCCESS) {
-		airo_print_err(ai->dev->name, "Couldn't allocate RX FID");
-		return rc;
-	}
-
-	for (i = 0; i<MPI_MAX_FIDS; i++) {
-		memcpy_toio(ai->rxfids[i].card_ram_off,
-			&ai->rxfids[i].rx_desc, sizeof(RxFid));
-	}
-
-	/* Alloc card TX descriptors */
-
-	memset(&rsp, 0, sizeof(rsp));
-	memset(&cmd, 0, sizeof(cmd));
-
-	cmd.cmd = CMD_ALLOCATEAUX;
-	cmd.parm0 = FID_TX;
-	cmd.parm1 = (ai->txfids[0].card_ram_off - ai->pciaux);
-	cmd.parm2 = MPI_MAX_FIDS;
-
-	for (i = 0; i<MPI_MAX_FIDS; i++) {
-		ai->txfids[i].tx_desc.valid = 1;
-		memcpy_toio(ai->txfids[i].card_ram_off,
-			&ai->txfids[i].tx_desc, sizeof(TxFid));
-	}
-	ai->txfids[i-1].tx_desc.eoc = 1; /* Last descriptor has EOC set */
-
-	rc = issuecommand(ai, &cmd, &rsp, true);
-	if (rc != SUCCESS) {
-		airo_print_err(ai->dev->name, "Couldn't allocate TX FID");
-		return rc;
-	}
-
-	/* Alloc card Rid descriptor */
-	memset(&rsp, 0, sizeof(rsp));
-	memset(&cmd, 0, sizeof(cmd));
-
-	cmd.cmd = CMD_ALLOCATEAUX;
-	cmd.parm0 = RID_RW;
-	cmd.parm1 = (ai->config_desc.card_ram_off - ai->pciaux);
-	cmd.parm2 = 1; /* Magic number... */
-	rc = issuecommand(ai, &cmd, &rsp, true);
-	if (rc != SUCCESS) {
-		airo_print_err(ai->dev->name, "Couldn't allocate RID");
-		return rc;
-	}
-
-	memcpy_toio(ai->config_desc.card_ram_off,
-		&ai->config_desc.rid_desc, sizeof(Rid));
-
-	return rc;
-}
-
-/*
- * We are setting up three things here:
- * 1) Map AUX memory for descriptors: Rid, TxFid, or RxFid.
- * 2) Map PCI memory for issuing commands.
- * 3) Allocate memory (shared) to send and receive ethernet frames.
- */
-static int mpi_map_card(struct airo_info *ai, struct pci_dev *pci)
-{
-	unsigned long mem_start, mem_len, aux_start, aux_len;
-	int rc = -1;
-	int i;
-	dma_addr_t busaddroff;
-	unsigned char *vpackoff;
-	unsigned char __iomem *pciaddroff;
-
-	mem_start = pci_resource_start(pci, 1);
-	mem_len = pci_resource_len(pci, 1);
-	aux_start = pci_resource_start(pci, 2);
-	aux_len = AUXMEMSIZE;
-
-	if (!request_mem_region(mem_start, mem_len, DRV_NAME)) {
-		airo_print_err("", "Couldn't get region %x[%x]",
-			(int)mem_start, (int)mem_len);
-		goto out;
-	}
-	if (!request_mem_region(aux_start, aux_len, DRV_NAME)) {
-		airo_print_err("", "Couldn't get region %x[%x]",
-			(int)aux_start, (int)aux_len);
-		goto free_region1;
-	}
-
-	ai->pcimem = ioremap(mem_start, mem_len);
-	if (!ai->pcimem) {
-		airo_print_err("", "Couldn't map region %x[%x]",
-			(int)mem_start, (int)mem_len);
-		goto free_region2;
-	}
-	ai->pciaux = ioremap(aux_start, aux_len);
-	if (!ai->pciaux) {
-		airo_print_err("", "Couldn't map region %x[%x]",
-			(int)aux_start, (int)aux_len);
-		goto free_memmap;
-	}
-
-	/* Reserve PKTSIZE for each fid and 2K for the Rids */
-	ai->shared = dma_alloc_coherent(&pci->dev, PCI_SHARED_LEN,
-					&ai->shared_dma, GFP_KERNEL);
-	if (!ai->shared) {
-		airo_print_err("", "Couldn't alloc_coherent %d",
-			PCI_SHARED_LEN);
-		goto free_auxmap;
-	}
-
-	/*
-	 * Setup descriptor RX, TX, CONFIG
-	 */
-	busaddroff = ai->shared_dma;
-	pciaddroff = ai->pciaux + AUX_OFFSET;
-	vpackoff   = ai->shared;
-
-	/* RX descriptor setup */
-	for (i = 0; i < MPI_MAX_FIDS; i++) {
-		ai->rxfids[i].pending = 0;
-		ai->rxfids[i].card_ram_off = pciaddroff;
-		ai->rxfids[i].virtual_host_addr = vpackoff;
-		ai->rxfids[i].rx_desc.host_addr = busaddroff;
-		ai->rxfids[i].rx_desc.valid = 1;
-		ai->rxfids[i].rx_desc.len = PKTSIZE;
-		ai->rxfids[i].rx_desc.rdy = 0;
-
-		pciaddroff += sizeof(RxFid);
-		busaddroff += PKTSIZE;
-		vpackoff   += PKTSIZE;
-	}
-
-	/* TX descriptor setup */
-	for (i = 0; i < MPI_MAX_FIDS; i++) {
-		ai->txfids[i].card_ram_off = pciaddroff;
-		ai->txfids[i].virtual_host_addr = vpackoff;
-		ai->txfids[i].tx_desc.valid = 1;
-		ai->txfids[i].tx_desc.host_addr = busaddroff;
-		memcpy(ai->txfids[i].virtual_host_addr,
-			&wifictlhdr8023, sizeof(wifictlhdr8023));
-
-		pciaddroff += sizeof(TxFid);
-		busaddroff += PKTSIZE;
-		vpackoff   += PKTSIZE;
-	}
-	ai->txfids[i-1].tx_desc.eoc = 1; /* Last descriptor has EOC set */
-
-	/* Rid descriptor setup */
-	ai->config_desc.card_ram_off = pciaddroff;
-	ai->config_desc.virtual_host_addr = vpackoff;
-	ai->config_desc.rid_desc.host_addr = busaddroff;
-	ai->ridbus = busaddroff;
-	ai->config_desc.rid_desc.rid = 0;
-	ai->config_desc.rid_desc.len = RIDSIZE;
-	ai->config_desc.rid_desc.valid = 1;
-	pciaddroff += sizeof(Rid);
-	busaddroff += RIDSIZE;
-	vpackoff   += RIDSIZE;
-
-	/* Tell card about descriptors */
-	if (mpi_init_descriptors (ai) != SUCCESS)
-		goto free_shared;
-
-	return 0;
- free_shared:
-	dma_free_coherent(&pci->dev, PCI_SHARED_LEN, ai->shared,
-			  ai->shared_dma);
- free_auxmap:
-	iounmap(ai->pciaux);
- free_memmap:
-	iounmap(ai->pcimem);
- free_region2:
-	release_mem_region(aux_start, aux_len);
- free_region1:
-	release_mem_region(mem_start, mem_len);
- out:
-	return rc;
-}
-
-static const struct header_ops airo_header_ops = {
-	.parse = wll_header_parse,
-};
-
-static const struct net_device_ops airo11_netdev_ops = {
-	.ndo_open 		= airo_open,
-	.ndo_stop 		= airo_close,
-	.ndo_start_xmit 	= airo_start_xmit11,
-	.ndo_get_stats 		= airo_get_stats,
-	.ndo_set_mac_address	= airo_set_mac_address,
-	.ndo_siocdevprivate	= airo_siocdevprivate,
-};
-
-static void wifi_setup(struct net_device *dev)
-{
-	dev->netdev_ops = &airo11_netdev_ops;
-	dev->header_ops = &airo_header_ops;
-	dev->wireless_handlers = &airo_handler_def;
-
-	dev->type               = ARPHRD_IEEE80211;
-	dev->hard_header_len    = ETH_HLEN;
-	dev->mtu                = AIRO_DEF_MTU;
-	dev->min_mtu            = 68;
-	dev->max_mtu            = MIC_MSGLEN_MAX;
-	dev->addr_len           = ETH_ALEN;
-	dev->tx_queue_len       = 100;
-
-	eth_broadcast_addr(dev->broadcast);
-
-	dev->flags              = IFF_BROADCAST|IFF_MULTICAST;
-}
-
-static struct net_device *init_wifidev(struct airo_info *ai,
-					struct net_device *ethdev)
-{
-	int err;
-	struct net_device *dev = alloc_netdev(0, "wifi%d", NET_NAME_UNKNOWN,
-					      wifi_setup);
-	if (!dev)
-		return NULL;
-	dev->ml_priv = ethdev->ml_priv;
-	dev->irq = ethdev->irq;
-	dev->base_addr = ethdev->base_addr;
-	dev->wireless_data = ethdev->wireless_data;
-	SET_NETDEV_DEV(dev, ethdev->dev.parent);
-	eth_hw_addr_inherit(dev, ethdev);
-	err = register_netdev(dev);
-	if (err<0) {
-		free_netdev(dev);
-		return NULL;
-	}
-	return dev;
-}
-
-static int reset_card(struct net_device *dev, int lock)
-{
-	struct airo_info *ai = dev->ml_priv;
-
-	if (lock && down_interruptible(&ai->sem))
-		return -1;
-	waitbusy (ai);
-	OUT4500(ai, COMMAND, CMD_SOFTRESET);
-	msleep(200);
-	waitbusy (ai);
-	msleep(200);
-	if (lock)
-		up(&ai->sem);
-	return 0;
-}
-
-#define AIRO_MAX_NETWORK_COUNT	64
-static int airo_networks_allocate(struct airo_info *ai)
-{
-	if (ai->networks)
-		return 0;
-
-	ai->networks = kcalloc(AIRO_MAX_NETWORK_COUNT, sizeof(BSSListElement),
-			       GFP_KERNEL);
-	if (!ai->networks) {
-		airo_print_warn("", "Out of memory allocating beacons");
-		return -ENOMEM;
-	}
-
-	return 0;
-}
-
-static void airo_networks_free(struct airo_info *ai)
-{
-	kfree(ai->networks);
-	ai->networks = NULL;
-}
-
-static void airo_networks_initialize(struct airo_info *ai)
-{
-	int i;
-
-	INIT_LIST_HEAD(&ai->network_free_list);
-	INIT_LIST_HEAD(&ai->network_list);
-	for (i = 0; i < AIRO_MAX_NETWORK_COUNT; i++)
-		list_add_tail(&ai->networks[i].list,
-			      &ai->network_free_list);
-}
-
-static const struct net_device_ops airo_netdev_ops = {
-	.ndo_open		= airo_open,
-	.ndo_stop		= airo_close,
-	.ndo_start_xmit		= airo_start_xmit,
-	.ndo_get_stats		= airo_get_stats,
-	.ndo_set_rx_mode	= airo_set_multicast_list,
-	.ndo_set_mac_address	= airo_set_mac_address,
-	.ndo_siocdevprivate	= airo_siocdevprivate,
-	.ndo_validate_addr	= eth_validate_addr,
-};
-
-static const struct net_device_ops mpi_netdev_ops = {
-	.ndo_open		= airo_open,
-	.ndo_stop		= airo_close,
-	.ndo_start_xmit		= mpi_start_xmit,
-	.ndo_get_stats		= airo_get_stats,
-	.ndo_set_rx_mode	= airo_set_multicast_list,
-	.ndo_set_mac_address	= airo_set_mac_address,
-	.ndo_siocdevprivate	= airo_siocdevprivate,
-	.ndo_validate_addr	= eth_validate_addr,
-};
-
-
-static struct net_device *_init_airo_card(unsigned short irq, int port,
-					   int is_pcmcia, struct pci_dev *pci,
-					   struct device *dmdev)
-{
-	struct net_device *dev;
-	struct airo_info *ai;
-	int i, rc;
-	CapabilityRid cap_rid;
-
-	/* Create the network device object. */
-	dev = alloc_netdev(sizeof(*ai), "", NET_NAME_UNKNOWN, ether_setup);
-	if (!dev) {
-		airo_print_err("", "Couldn't alloc_etherdev");
-		return NULL;
-	}
-
-	ai = dev->ml_priv = netdev_priv(dev);
-	ai->wifidev = NULL;
-	ai->flags = 1 << FLAG_RADIO_DOWN;
-	ai->jobs = 0;
-	ai->dev = dev;
-	if (pci && (pci->device == 0x5000 || pci->device == 0xa504)) {
-		airo_print_dbg("", "Found an MPI350 card");
-		set_bit(FLAG_MPI, &ai->flags);
-	}
-	spin_lock_init(&ai->aux_lock);
-	sema_init(&ai->sem, 1);
-	ai->config.len = 0;
-	ai->pci = pci;
-	init_waitqueue_head (&ai->thr_wait);
-	ai->tfm = NULL;
-	add_airo_dev(ai);
-	ai->APList.len = cpu_to_le16(sizeof(struct APListRid));
-
-	if (airo_networks_allocate (ai))
-		goto err_out_free;
-	airo_networks_initialize (ai);
-
-	skb_queue_head_init (&ai->txq);
-
-	/* The Airo-specific entries in the device structure. */
-	if (test_bit(FLAG_MPI,&ai->flags))
-		dev->netdev_ops = &mpi_netdev_ops;
-	else
-		dev->netdev_ops = &airo_netdev_ops;
-	dev->wireless_handlers = &airo_handler_def;
-	ai->wireless_data.spy_data = &ai->spy_data;
-	dev->wireless_data = &ai->wireless_data;
-	dev->irq = irq;
-	dev->base_addr = port;
-	dev->priv_flags &= ~IFF_TX_SKB_SHARING;
-	dev->max_mtu = MIC_MSGLEN_MAX;
-
-	SET_NETDEV_DEV(dev, dmdev);
-
-	reset_card (dev, 1);
-	msleep(400);
-
-	if (!is_pcmcia) {
-		if (!request_region(dev->base_addr, 64, DRV_NAME)) {
-			rc = -EBUSY;
-			airo_print_err(dev->name, "Couldn't request region");
-			goto err_out_nets;
-		}
-	}
-
-	if (test_bit(FLAG_MPI,&ai->flags)) {
-		if (mpi_map_card(ai, pci)) {
-			airo_print_err("", "Could not map memory");
-			goto err_out_res;
-		}
-	}
-
-	if (probe) {
-		if (setup_card(ai, dev, 1) != SUCCESS) {
-			airo_print_err(dev->name, "MAC could not be enabled");
-			rc = -EIO;
-			goto err_out_map;
-		}
-	} else if (!test_bit(FLAG_MPI,&ai->flags)) {
-		ai->bap_read = fast_bap_read;
-		set_bit(FLAG_FLASHING, &ai->flags);
-	}
-
-	strcpy(dev->name, "eth%d");
-	rc = register_netdev(dev);
-	if (rc) {
-		airo_print_err(dev->name, "Couldn't register_netdev");
-		goto err_out_map;
-	}
-	ai->wifidev = init_wifidev(ai, dev);
-	if (!ai->wifidev)
-		goto err_out_reg;
-
-	rc = readCapabilityRid(ai, &cap_rid, 1);
-	if (rc != SUCCESS) {
-		rc = -EIO;
-		goto err_out_wifi;
-	}
-	/* WEP capability discovery */
-	ai->wep_capable = (cap_rid.softCap & cpu_to_le16(0x02)) ? 1 : 0;
-	ai->max_wep_idx = (cap_rid.softCap & cpu_to_le16(0x80)) ? 3 : 0;
-
-	airo_print_info(dev->name, "Firmware version %x.%x.%02d",
-	                ((le16_to_cpu(cap_rid.softVer) >> 8) & 0xF),
-	                (le16_to_cpu(cap_rid.softVer) & 0xFF),
-	                le16_to_cpu(cap_rid.softSubVer));
-
-	/* Test for WPA support */
-	/* Only firmware versions 5.30.17 or better can do WPA */
-	if (le16_to_cpu(cap_rid.softVer) > 0x530
-	 || (le16_to_cpu(cap_rid.softVer) == 0x530
-	      && le16_to_cpu(cap_rid.softSubVer) >= 17)) {
-		airo_print_info(ai->dev->name, "WPA supported.");
-
-		set_bit(FLAG_WPA_CAPABLE, &ai->flags);
-		ai->bssListFirst = RID_WPA_BSSLISTFIRST;
-		ai->bssListNext = RID_WPA_BSSLISTNEXT;
-		ai->bssListRidLen = sizeof(BSSListRid);
-	} else {
-		airo_print_info(ai->dev->name, "WPA unsupported with firmware "
-			"versions older than 5.30.17.");
-
-		ai->bssListFirst = RID_BSSLISTFIRST;
-		ai->bssListNext = RID_BSSLISTNEXT;
-		ai->bssListRidLen = sizeof(BSSListRid) - sizeof(BSSListRidExtra);
-	}
-
-	set_bit(FLAG_REGISTERED,&ai->flags);
-	airo_print_info(dev->name, "MAC enabled %pM", dev->dev_addr);
-
-	/* Allocate the transmit buffers */
-	if (probe && !test_bit(FLAG_MPI,&ai->flags))
-		for (i = 0; i < MAX_FIDS; i++)
-			ai->fids[i] = transmit_allocate(ai, AIRO_DEF_MTU, i>=MAX_FIDS/2);
-
-	if (setup_proc_entry(dev, dev->ml_priv) < 0)
-		goto err_out_wifi;
-
-	return dev;
-
-err_out_wifi:
-	unregister_netdev(ai->wifidev);
-	free_netdev(ai->wifidev);
-err_out_reg:
-	unregister_netdev(dev);
-err_out_map:
-	if (test_bit(FLAG_MPI,&ai->flags) && pci) {
-		dma_free_coherent(&pci->dev, PCI_SHARED_LEN, ai->shared,
-				  ai->shared_dma);
-		iounmap(ai->pciaux);
-		iounmap(ai->pcimem);
-		mpi_unmap_card(ai->pci);
-	}
-err_out_res:
-	if (!is_pcmcia)
-		release_region(dev->base_addr, 64);
-err_out_nets:
-	airo_networks_free(ai);
-err_out_free:
-	del_airo_dev(ai);
-	free_netdev(dev);
-	return NULL;
-}
-
-struct net_device *init_airo_card(unsigned short irq, int port, int is_pcmcia,
-				  struct device *dmdev)
-{
-	return _init_airo_card (irq, port, is_pcmcia, NULL, dmdev);
-}
-
-EXPORT_SYMBOL(init_airo_card);
-
-static int waitbusy (struct airo_info *ai)
-{
-	int delay = 0;
-	while ((IN4500(ai, COMMAND) & COMMAND_BUSY) && (delay < 10000)) {
-		udelay (10);
-		if ((++delay % 20) == 0)
-			OUT4500(ai, EVACK, EV_CLEARCOMMANDBUSY);
-	}
-	return delay < 10000;
-}
-
-int reset_airo_card(struct net_device *dev)
-{
-	int i;
-	struct airo_info *ai = dev->ml_priv;
-
-	if (reset_card (dev, 1))
-		return -1;
-
-	if (setup_card(ai, dev, 1) != SUCCESS) {
-		airo_print_err(dev->name, "MAC could not be enabled");
-		return -1;
-	}
-	airo_print_info(dev->name, "MAC enabled %pM", dev->dev_addr);
-	/* Allocate the transmit buffers if needed */
-	if (!test_bit(FLAG_MPI,&ai->flags))
-		for (i = 0; i < MAX_FIDS; i++)
-			ai->fids[i] = transmit_allocate (ai, AIRO_DEF_MTU, i>=MAX_FIDS/2);
-
-	enable_interrupts(ai);
-	netif_wake_queue(dev);
-	return 0;
-}
-
-EXPORT_SYMBOL(reset_airo_card);
-
-static void airo_send_event(struct net_device *dev)
-{
-	struct airo_info *ai = dev->ml_priv;
-	union iwreq_data wrqu;
-	StatusRid status_rid;
-
-	clear_bit(JOB_EVENT, &ai->jobs);
-	PC4500_readrid(ai, RID_STATUS, &status_rid, sizeof(status_rid), 0);
-	up(&ai->sem);
-	wrqu.data.length = 0;
-	wrqu.data.flags = 0;
-	memcpy(wrqu.ap_addr.sa_data, status_rid.bssid[0], ETH_ALEN);
-	wrqu.ap_addr.sa_family = ARPHRD_ETHER;
-
-	/* Send event to user space */
-	wireless_send_event(dev, SIOCGIWAP, &wrqu, NULL);
-}
-
-static void airo_process_scan_results (struct airo_info *ai)
-{
-	union iwreq_data	wrqu;
-	BSSListRid bss;
-	int rc;
-	BSSListElement * loop_net;
-	BSSListElement * tmp_net;
-
-	/* Blow away current list of scan results */
-	list_for_each_entry_safe (loop_net, tmp_net, &ai->network_list, list) {
-		list_move_tail (&loop_net->list, &ai->network_free_list);
-		/* Don't blow away ->list, just BSS data */
-		memset (loop_net, 0, sizeof (loop_net->bss));
-	}
-
-	/* Try to read the first entry of the scan result */
-	rc = PC4500_readrid(ai, ai->bssListFirst, &bss, ai->bssListRidLen, 0);
-	if ((rc) || (bss.index == cpu_to_le16(0xffff))) {
-		/* No scan results */
-		goto out;
-	}
-
-	/* Read and parse all entries */
-	tmp_net = NULL;
-	while ((!rc) && (bss.index != cpu_to_le16(0xffff))) {
-		/* Grab a network off the free list */
-		if (!list_empty(&ai->network_free_list)) {
-			tmp_net = list_entry(ai->network_free_list.next,
-					    BSSListElement, list);
-			list_del(ai->network_free_list.next);
-		}
-
-		if (tmp_net != NULL) {
-			memcpy(tmp_net, &bss, sizeof(tmp_net->bss));
-			list_add_tail(&tmp_net->list, &ai->network_list);
-			tmp_net = NULL;
-		}
-
-		/* Read next entry */
-		rc = PC4500_readrid(ai, ai->bssListNext,
-				    &bss, ai->bssListRidLen, 0);
-	}
-
-out:
-	/* write APList back (we cleared it in airo_set_scan) */
-	disable_MAC(ai, 2);
-	writeAPListRid(ai, &ai->APList, 0);
-	enable_MAC(ai, 0);
-
-	ai->scan_timeout = 0;
-	clear_bit(JOB_SCAN_RESULTS, &ai->jobs);
-	up(&ai->sem);
-
-	/* Send an empty event to user space.
-	 * We don't send the received data on
-	 * the event because it would require
-	 * us to do complex transcoding, and
-	 * we want to minimise the work done in
-	 * the irq handler. Use a request to
-	 * extract the data - Jean II */
-	wrqu.data.length = 0;
-	wrqu.data.flags = 0;
-	wireless_send_event(ai->dev, SIOCGIWSCAN, &wrqu, NULL);
-}
-
-static int airo_thread(void *data)
-{
-	struct net_device *dev = data;
-	struct airo_info *ai = dev->ml_priv;
-	int locked;
-
-	set_freezable();
-	while (1) {
-		/* make swsusp happy with our thread */
-		try_to_freeze();
-
-		if (test_bit(JOB_DIE, &ai->jobs))
-			break;
-
-		if (ai->jobs) {
-			locked = down_interruptible(&ai->sem);
-		} else {
-			wait_queue_entry_t wait;
-
-			init_waitqueue_entry(&wait, current);
-			add_wait_queue(&ai->thr_wait, &wait);
-			for (;;) {
-				set_current_state(TASK_INTERRUPTIBLE);
-				if (ai->jobs)
-					break;
-				if (ai->expires || ai->scan_timeout) {
-					if (ai->scan_timeout &&
-							time_after_eq(jiffies, ai->scan_timeout)) {
-						set_bit(JOB_SCAN_RESULTS, &ai->jobs);
-						break;
-					} else if (ai->expires &&
-							time_after_eq(jiffies, ai->expires)) {
-						set_bit(JOB_AUTOWEP, &ai->jobs);
-						break;
-					}
-					if (!kthread_should_stop() &&
-					    !freezing(current)) {
-						unsigned long wake_at;
-						if (!ai->expires || !ai->scan_timeout) {
-							wake_at = max(ai->expires,
-								ai->scan_timeout);
-						} else {
-							wake_at = min(ai->expires,
-								ai->scan_timeout);
-						}
-						schedule_timeout(wake_at - jiffies);
-						continue;
-					}
-				} else if (!kthread_should_stop() &&
-					   !freezing(current)) {
-					schedule();
-					continue;
-				}
-				break;
-			}
-			__set_current_state(TASK_RUNNING);
-			remove_wait_queue(&ai->thr_wait, &wait);
-			locked = 1;
-		}
-
-		if (locked)
-			continue;
-
-		if (test_bit(JOB_DIE, &ai->jobs)) {
-			up(&ai->sem);
-			break;
-		}
-
-		if (ai->power.event || test_bit(FLAG_FLASHING, &ai->flags)) {
-			up(&ai->sem);
-			continue;
-		}
-
-		if (test_bit(JOB_XMIT, &ai->jobs))
-			airo_end_xmit(dev, true);
-		else if (test_bit(JOB_XMIT11, &ai->jobs))
-			airo_end_xmit11(dev, true);
-		else if (test_bit(JOB_STATS, &ai->jobs))
-			airo_read_stats(dev);
-		else if (test_bit(JOB_PROMISC, &ai->jobs))
-			airo_set_promisc(ai, true);
-		else if (test_bit(JOB_MIC, &ai->jobs))
-			micinit(ai);
-		else if (test_bit(JOB_EVENT, &ai->jobs))
-			airo_send_event(dev);
-		else if (test_bit(JOB_AUTOWEP, &ai->jobs))
-			timer_func(dev);
-		else if (test_bit(JOB_SCAN_RESULTS, &ai->jobs))
-			airo_process_scan_results(ai);
-		else  /* Shouldn't get here, but we make sure to unlock */
-			up(&ai->sem);
-	}
-
-	return 0;
-}
-
-static int header_len(__le16 ctl)
-{
-	u16 fc = le16_to_cpu(ctl);
-	switch (fc & 0xc) {
-	case 4:
-		if ((fc & 0xe0) == 0xc0)
-			return 10;	/* one-address control packet */
-		return 16;	/* two-address control packet */
-	case 8:
-		if ((fc & 0x300) == 0x300)
-			return 30;	/* WDS packet */
-	}
-	return 24;
-}
-
-static void airo_handle_cisco_mic(struct airo_info *ai)
-{
-	if (test_bit(FLAG_MIC_CAPABLE, &ai->flags)) {
-		set_bit(JOB_MIC, &ai->jobs);
-		wake_up_interruptible(&ai->thr_wait);
-	}
-}
-
-/* Airo Status codes */
-#define STAT_NOBEACON	0x8000 /* Loss of sync - missed beacons */
-#define STAT_MAXRETRIES	0x8001 /* Loss of sync - max retries */
-#define STAT_MAXARL	0x8002 /* Loss of sync - average retry level exceeded*/
-#define STAT_FORCELOSS	0x8003 /* Loss of sync - host request */
-#define STAT_TSFSYNC	0x8004 /* Loss of sync - TSF synchronization */
-#define STAT_DEAUTH	0x8100 /* low byte is 802.11 reason code */
-#define STAT_DISASSOC	0x8200 /* low byte is 802.11 reason code */
-#define STAT_ASSOC_FAIL	0x8400 /* low byte is 802.11 reason code */
-#define STAT_AUTH_FAIL	0x0300 /* low byte is 802.11 reason code */
-#define STAT_ASSOC	0x0400 /* Associated */
-#define STAT_REASSOC    0x0600 /* Reassociated?  Only on firmware >= 5.30.17 */
-
-static void airo_print_status(const char *devname, u16 status)
-{
-	u8 reason = status & 0xFF;
-
-	switch (status & 0xFF00) {
-	case STAT_NOBEACON:
-		switch (status) {
-		case STAT_NOBEACON:
-			airo_print_dbg(devname, "link lost (missed beacons)");
-			break;
-		case STAT_MAXRETRIES:
-		case STAT_MAXARL:
-			airo_print_dbg(devname, "link lost (max retries)");
-			break;
-		case STAT_FORCELOSS:
-			airo_print_dbg(devname, "link lost (local choice)");
-			break;
-		case STAT_TSFSYNC:
-			airo_print_dbg(devname, "link lost (TSF sync lost)");
-			break;
-		default:
-			airo_print_dbg(devname, "unknown status %x\n", status);
-			break;
-		}
-		break;
-	case STAT_DEAUTH:
-		airo_print_dbg(devname, "deauthenticated (reason: %d)", reason);
-		break;
-	case STAT_DISASSOC:
-		airo_print_dbg(devname, "disassociated (reason: %d)", reason);
-		break;
-	case STAT_ASSOC_FAIL:
-		airo_print_dbg(devname, "association failed (reason: %d)",
-			       reason);
-		break;
-	case STAT_AUTH_FAIL:
-		airo_print_dbg(devname, "authentication failed (reason: %d)",
-			       reason);
-		break;
-	case STAT_ASSOC:
-	case STAT_REASSOC:
-		break;
-	default:
-		airo_print_dbg(devname, "unknown status %x\n", status);
-		break;
-	}
-}
-
-static void airo_handle_link(struct airo_info *ai)
-{
-	union iwreq_data wrqu;
-	int scan_forceloss = 0;
-	u16 status;
-
-	/* Get new status and acknowledge the link change */
-	status = le16_to_cpu(IN4500(ai, LINKSTAT));
-	OUT4500(ai, EVACK, EV_LINK);
-
-	if ((status == STAT_FORCELOSS) && (ai->scan_timeout > 0))
-		scan_forceloss = 1;
-
-	airo_print_status(ai->dev->name, status);
-
-	if ((status == STAT_ASSOC) || (status == STAT_REASSOC)) {
-		if (auto_wep)
-			ai->expires = 0;
-		if (ai->list_bss_task)
-			wake_up_process(ai->list_bss_task);
-		set_bit(FLAG_UPDATE_UNI, &ai->flags);
-		set_bit(FLAG_UPDATE_MULTI, &ai->flags);
-
-		set_bit(JOB_EVENT, &ai->jobs);
-		wake_up_interruptible(&ai->thr_wait);
-
-		netif_carrier_on(ai->dev);
-	} else if (!scan_forceloss) {
-		if (auto_wep && !ai->expires) {
-			ai->expires = RUN_AT(3*HZ);
-			wake_up_interruptible(&ai->thr_wait);
-		}
-
-		/* Send event to user space */
-		eth_zero_addr(wrqu.ap_addr.sa_data);
-		wrqu.ap_addr.sa_family = ARPHRD_ETHER;
-		wireless_send_event(ai->dev, SIOCGIWAP, &wrqu, NULL);
-		netif_carrier_off(ai->dev);
-	} else {
-		netif_carrier_off(ai->dev);
-	}
-}
-
-static void airo_handle_rx(struct airo_info *ai)
-{
-	struct sk_buff *skb = NULL;
-	__le16 fc, v, *buffer, tmpbuf[4];
-	u16 len, hdrlen = 0, gap, fid;
-	struct rx_hdr hdr;
-	int success = 0;
-
-	if (test_bit(FLAG_MPI, &ai->flags)) {
-		if (test_bit(FLAG_802_11, &ai->flags))
-			mpi_receive_802_11(ai);
-		else
-			mpi_receive_802_3(ai);
-		OUT4500(ai, EVACK, EV_RX);
-		return;
-	}
-
-	fid = IN4500(ai, RXFID);
-
-	/* Get the packet length */
-	if (test_bit(FLAG_802_11, &ai->flags)) {
-		bap_setup (ai, fid, 4, BAP0);
-		bap_read (ai, (__le16*)&hdr, sizeof(hdr), BAP0);
-		/* Bad CRC. Ignore packet */
-		if (le16_to_cpu(hdr.status) & 2)
-			hdr.len = 0;
-		if (ai->wifidev == NULL)
-			hdr.len = 0;
-	} else {
-		bap_setup(ai, fid, 0x36, BAP0);
-		bap_read(ai, &hdr.len, 2, BAP0);
-	}
-	len = le16_to_cpu(hdr.len);
-
-	if (len > AIRO_DEF_MTU) {
-		airo_print_err(ai->dev->name, "Bad size %d", len);
-		goto done;
-	}
-	if (len == 0)
-		goto done;
-
-	if (test_bit(FLAG_802_11, &ai->flags)) {
-		bap_read(ai, &fc, sizeof (fc), BAP0);
-		hdrlen = header_len(fc);
-	} else
-		hdrlen = ETH_ALEN * 2;
-
-	skb = dev_alloc_skb(len + hdrlen + 2 + 2);
-	if (!skb) {
-		ai->dev->stats.rx_dropped++;
-		goto done;
-	}
-
-	skb_reserve(skb, 2); /* This way the IP header is aligned */
-	buffer = skb_put(skb, len + hdrlen);
-	if (test_bit(FLAG_802_11, &ai->flags)) {
-		buffer[0] = fc;
-		bap_read(ai, buffer + 1, hdrlen - 2, BAP0);
-		if (hdrlen == 24)
-			bap_read(ai, tmpbuf, 6, BAP0);
-
-		bap_read(ai, &v, sizeof(v), BAP0);
-		gap = le16_to_cpu(v);
-		if (gap) {
-			if (gap <= 8) {
-				bap_read(ai, tmpbuf, gap, BAP0);
-			} else {
-				airo_print_err(ai->dev->name, "gaplen too "
-					"big. Problems will follow...");
-			}
-		}
-		bap_read(ai, buffer + hdrlen/2, len, BAP0);
-	} else {
-		MICBuffer micbuf;
-
-		bap_read(ai, buffer, ETH_ALEN * 2, BAP0);
-		if (ai->micstats.enabled) {
-			bap_read(ai, (__le16 *) &micbuf, sizeof (micbuf), BAP0);
-			if (ntohs(micbuf.typelen) > 0x05DC)
-				bap_setup(ai, fid, 0x44, BAP0);
-			else {
-				if (len <= sizeof (micbuf)) {
-					dev_kfree_skb_irq(skb);
-					goto done;
-				}
-
-				len -= sizeof(micbuf);
-				skb_trim(skb, len + hdrlen);
-			}
-		}
-
-		bap_read(ai, buffer + ETH_ALEN, len, BAP0);
-		if (decapsulate(ai, &micbuf, (etherHead*) buffer, len))
-			dev_kfree_skb_irq (skb);
-		else
-			success = 1;
-	}
-
-#ifdef WIRELESS_SPY
-	if (success && (ai->spy_data.spy_number > 0)) {
-		char *sa;
-		struct iw_quality wstats;
-
-		/* Prepare spy data : addr + qual */
-		if (!test_bit(FLAG_802_11, &ai->flags)) {
-			sa = (char *) buffer + 6;
-			bap_setup(ai, fid, 8, BAP0);
-			bap_read(ai, (__le16 *) hdr.rssi, 2, BAP0);
-		} else
-			sa = (char *) buffer + 10;
-		wstats.qual = hdr.rssi[0];
-		if (ai->rssi)
-			wstats.level = 0x100 - ai->rssi[hdr.rssi[1]].rssidBm;
-		else
-			wstats.level = (hdr.rssi[1] + 321) / 2;
-		wstats.noise = ai->wstats.qual.noise;
-		wstats.updated =  IW_QUAL_LEVEL_UPDATED
-				| IW_QUAL_QUAL_UPDATED
-				| IW_QUAL_DBM;
-		/* Update spy records */
-		wireless_spy_update(ai->dev, sa, &wstats);
-	}
-#endif /* WIRELESS_SPY */
-
-done:
-	OUT4500(ai, EVACK, EV_RX);
-
-	if (success) {
-		if (test_bit(FLAG_802_11, &ai->flags)) {
-			skb_reset_mac_header(skb);
-			skb->pkt_type = PACKET_OTHERHOST;
-			skb->dev = ai->wifidev;
-			skb->protocol = htons(ETH_P_802_2);
-		} else
-			skb->protocol = eth_type_trans(skb, ai->dev);
-		skb->ip_summed = CHECKSUM_NONE;
-
-		netif_rx(skb);
-	}
-}
-
-static void airo_handle_tx(struct airo_info *ai, u16 status)
-{
-	int i, index = -1;
-	u16 fid;
-
-	if (test_bit(FLAG_MPI, &ai->flags)) {
-		unsigned long flags;
-
-		if (status & EV_TXEXC)
-			get_tx_error(ai, -1);
-
-		spin_lock_irqsave(&ai->aux_lock, flags);
-		if (!skb_queue_empty(&ai->txq)) {
-			spin_unlock_irqrestore(&ai->aux_lock, flags);
-			mpi_send_packet(ai->dev);
-		} else {
-			clear_bit(FLAG_PENDING_XMIT, &ai->flags);
-			spin_unlock_irqrestore(&ai->aux_lock, flags);
-			netif_wake_queue(ai->dev);
-		}
-		OUT4500(ai, EVACK, status & (EV_TX | EV_TXCPY | EV_TXEXC));
-		return;
-	}
-
-	fid = IN4500(ai, TXCOMPLFID);
-
-	for (i = 0; i < MAX_FIDS; i++) {
-		if ((ai->fids[i] & 0xffff) == fid)
-			index = i;
-	}
-
-	if (index != -1) {
-		if (status & EV_TXEXC)
-			get_tx_error(ai, index);
-
-		OUT4500(ai, EVACK, status & (EV_TX | EV_TXEXC));
-
-		/* Set up to be used again */
-		ai->fids[index] &= 0xffff;
-		if (index < MAX_FIDS / 2) {
-			if (!test_bit(FLAG_PENDING_XMIT, &ai->flags))
-				netif_wake_queue(ai->dev);
-		} else {
-			if (!test_bit(FLAG_PENDING_XMIT11, &ai->flags))
-				netif_wake_queue(ai->wifidev);
-		}
-	} else {
-		OUT4500(ai, EVACK, status & (EV_TX | EV_TXCPY | EV_TXEXC));
-		airo_print_err(ai->dev->name, "Unallocated FID was used to xmit");
-	}
-}
-
-static irqreturn_t airo_interrupt(int irq, void *dev_id)
-{
-	struct net_device *dev = dev_id;
-	u16 status, savedInterrupts = 0;
-	struct airo_info *ai = dev->ml_priv;
-	int handled = 0;
-
-	if (!netif_device_present(dev))
-		return IRQ_NONE;
-
-	for (;;) {
-		status = IN4500(ai, EVSTAT);
-		if (!(status & STATUS_INTS) || (status == 0xffff))
-			break;
-
-		handled = 1;
-
-		if (status & EV_AWAKE) {
-			OUT4500(ai, EVACK, EV_AWAKE);
-			OUT4500(ai, EVACK, EV_AWAKE);
-		}
-
-		if (!savedInterrupts) {
-			savedInterrupts = IN4500(ai, EVINTEN);
-			OUT4500(ai, EVINTEN, 0);
-		}
-
-		if (status & EV_MIC) {
-			OUT4500(ai, EVACK, EV_MIC);
-			airo_handle_cisco_mic(ai);
-		}
-
-		if (status & EV_LINK) {
-			/* Link status changed */
-			airo_handle_link(ai);
-		}
-
-		/* Check to see if there is something to receive */
-		if (status & EV_RX)
-			airo_handle_rx(ai);
-
-		/* Check to see if a packet has been transmitted */
-		if (status & (EV_TX | EV_TXCPY | EV_TXEXC))
-			airo_handle_tx(ai, status);
-
-		if (status & ~STATUS_INTS & ~IGNORE_INTS) {
-			airo_print_warn(ai->dev->name, "Got weird status %x",
-				status & ~STATUS_INTS & ~IGNORE_INTS);
-		}
-	}
-
-	if (savedInterrupts)
-		OUT4500(ai, EVINTEN, savedInterrupts);
-
-	return IRQ_RETVAL(handled);
-}
-
-/*
- *  Routines to talk to the card
- */
-
-/*
- *  This was originally written for the 4500, hence the name
- *  NOTE:  If use with 8bit mode and SMP bad things will happen!
- *         Why would some one do 8 bit IO in an SMP machine?!?
- */
-static void OUT4500(struct airo_info *ai, u16 reg, u16 val)
-{
-	if (test_bit(FLAG_MPI,&ai->flags))
-		reg <<= 1;
-	if (!do8bitIO)
-		outw(val, ai->dev->base_addr + reg);
-	else {
-		outb(val & 0xff, ai->dev->base_addr + reg);
-		outb(val >> 8, ai->dev->base_addr + reg + 1);
-	}
-}
-
-static u16 IN4500(struct airo_info *ai, u16 reg)
-{
-	unsigned short rc;
-
-	if (test_bit(FLAG_MPI,&ai->flags))
-		reg <<= 1;
-	if (!do8bitIO)
-		rc = inw(ai->dev->base_addr + reg);
-	else {
-		rc = inb(ai->dev->base_addr + reg);
-		rc += ((int)inb(ai->dev->base_addr + reg + 1)) << 8;
-	}
-	return rc;
-}
-
-static int enable_MAC(struct airo_info *ai, int lock)
-{
-	int rc;
-	Cmd cmd;
-	Resp rsp;
-
-	/* FLAG_RADIO_OFF : Radio disabled via /proc or Wireless Extensions
-	 * FLAG_RADIO_DOWN : Radio disabled via "ifconfig ethX down"
-	 * Note : we could try to use !netif_running(dev) in enable_MAC()
-	 * instead of this flag, but I don't trust it *within* the
-	 * open/close functions, and testing both flags together is
-	 * "cheaper" - Jean II */
-	if (ai->flags & FLAG_RADIO_MASK) return SUCCESS;
-
-	if (lock && down_interruptible(&ai->sem))
-		return -ERESTARTSYS;
-
-	if (!test_bit(FLAG_ENABLED, &ai->flags)) {
-		memset(&cmd, 0, sizeof(cmd));
-		cmd.cmd = MAC_ENABLE;
-		rc = issuecommand(ai, &cmd, &rsp, true);
-		if (rc == SUCCESS)
-			set_bit(FLAG_ENABLED, &ai->flags);
-	} else
-		rc = SUCCESS;
-
-	if (lock)
-	    up(&ai->sem);
-
-	if (rc)
-		airo_print_err(ai->dev->name, "Cannot enable MAC");
-	else if ((rsp.status & 0xFF00) != 0) {
-		airo_print_err(ai->dev->name, "Bad MAC enable reason=%x, "
-			"rid=%x, offset=%d", rsp.rsp0, rsp.rsp1, rsp.rsp2);
-		rc = ERROR;
-	}
-	return rc;
-}
-
-static void disable_MAC(struct airo_info *ai, int lock)
-{
-        Cmd cmd;
-	Resp rsp;
-
-	if (lock == 1 && down_interruptible(&ai->sem))
-		return;
-
-	if (test_bit(FLAG_ENABLED, &ai->flags)) {
-		if (lock != 2) /* lock == 2 means don't disable carrier */
-			netif_carrier_off(ai->dev);
-		memset(&cmd, 0, sizeof(cmd));
-		cmd.cmd = MAC_DISABLE; // disable in case already enabled
-		issuecommand(ai, &cmd, &rsp, true);
-		clear_bit(FLAG_ENABLED, &ai->flags);
-	}
-	if (lock == 1)
-		up(&ai->sem);
-}
-
-static void enable_interrupts(struct airo_info *ai)
-{
-	/* Enable the interrupts */
-	OUT4500(ai, EVINTEN, STATUS_INTS);
-}
-
-static void disable_interrupts(struct airo_info *ai)
-{
-	OUT4500(ai, EVINTEN, 0);
-}
-
-static void mpi_receive_802_3(struct airo_info *ai)
-{
-	RxFid rxd;
-	int len = 0;
-	struct sk_buff *skb;
-	char *buffer;
-	int off = 0;
-	MICBuffer micbuf;
-
-	memcpy_fromio(&rxd, ai->rxfids[0].card_ram_off, sizeof(rxd));
-	/* Make sure we got something */
-	if (rxd.rdy && rxd.valid == 0) {
-		len = rxd.len + 12;
-		if (len < 12 || len > 2048)
-			goto badrx;
-
-		skb = dev_alloc_skb(len);
-		if (!skb) {
-			ai->dev->stats.rx_dropped++;
-			goto badrx;
-		}
-		buffer = skb_put(skb, len);
-		memcpy(buffer, ai->rxfids[0].virtual_host_addr, ETH_ALEN * 2);
-		if (ai->micstats.enabled) {
-			memcpy(&micbuf,
-				ai->rxfids[0].virtual_host_addr + ETH_ALEN * 2,
-				sizeof(micbuf));
-			if (ntohs(micbuf.typelen) <= 0x05DC) {
-				if (len <= sizeof(micbuf) + ETH_ALEN * 2)
-					goto badmic;
-
-				off = sizeof(micbuf);
-				skb_trim (skb, len - off);
-			}
-		}
-		memcpy(buffer + ETH_ALEN * 2,
-			ai->rxfids[0].virtual_host_addr + ETH_ALEN * 2 + off,
-			len - ETH_ALEN * 2 - off);
-		if (decapsulate (ai, &micbuf, (etherHead*)buffer, len - off - ETH_ALEN * 2)) {
-badmic:
-			dev_kfree_skb_irq (skb);
-			goto badrx;
-		}
-#ifdef WIRELESS_SPY
-		if (ai->spy_data.spy_number > 0) {
-			char *sa;
-			struct iw_quality wstats;
-			/* Prepare spy data : addr + qual */
-			sa = buffer + ETH_ALEN;
-			wstats.qual = 0; /* XXX Where do I get that info from ??? */
-			wstats.level = 0;
-			wstats.updated = 0;
-			/* Update spy records */
-			wireless_spy_update(ai->dev, sa, &wstats);
-		}
-#endif /* WIRELESS_SPY */
-
-		skb->ip_summed = CHECKSUM_NONE;
-		skb->protocol = eth_type_trans(skb, ai->dev);
-		netif_rx(skb);
-	}
-badrx:
-	if (rxd.valid == 0) {
-		rxd.valid = 1;
-		rxd.rdy = 0;
-		rxd.len = PKTSIZE;
-		memcpy_toio(ai->rxfids[0].card_ram_off, &rxd, sizeof(rxd));
-	}
-}
-
-static void mpi_receive_802_11(struct airo_info *ai)
-{
-	RxFid rxd;
-	struct sk_buff *skb = NULL;
-	u16 len, hdrlen = 0;
-	__le16 fc;
-	struct rx_hdr hdr;
-	u16 gap;
-	u16 *buffer;
-	char *ptr = ai->rxfids[0].virtual_host_addr + 4;
-
-	memcpy_fromio(&rxd, ai->rxfids[0].card_ram_off, sizeof(rxd));
-	memcpy ((char *)&hdr, ptr, sizeof(hdr));
-	ptr += sizeof(hdr);
-	/* Bad CRC. Ignore packet */
-	if (le16_to_cpu(hdr.status) & 2)
-		hdr.len = 0;
-	if (ai->wifidev == NULL)
-		hdr.len = 0;
-	len = le16_to_cpu(hdr.len);
-	if (len > AIRO_DEF_MTU) {
-		airo_print_err(ai->dev->name, "Bad size %d", len);
-		goto badrx;
-	}
-	if (len == 0)
-		goto badrx;
-
-	fc = get_unaligned((__le16 *)ptr);
-	hdrlen = header_len(fc);
-
-	skb = dev_alloc_skb(len + hdrlen + 2);
-	if (!skb) {
-		ai->dev->stats.rx_dropped++;
-		goto badrx;
-	}
-	buffer = skb_put(skb, len + hdrlen);
-	memcpy ((char *)buffer, ptr, hdrlen);
-	ptr += hdrlen;
-	if (hdrlen == 24)
-		ptr += 6;
-	gap = get_unaligned_le16(ptr);
-	ptr += sizeof(__le16);
-	if (gap) {
-		if (gap <= 8)
-			ptr += gap;
-		else
-			airo_print_err(ai->dev->name,
-			    "gaplen too big. Problems will follow...");
-	}
-	memcpy ((char *)buffer + hdrlen, ptr, len);
-	ptr += len;
-#ifdef IW_WIRELESS_SPY	  /* defined in iw_handler.h */
-	if (ai->spy_data.spy_number > 0) {
-		char *sa;
-		struct iw_quality wstats;
-		/* Prepare spy data : addr + qual */
-		sa = (char*)buffer + 10;
-		wstats.qual = hdr.rssi[0];
-		if (ai->rssi)
-			wstats.level = 0x100 - ai->rssi[hdr.rssi[1]].rssidBm;
-		else
-			wstats.level = (hdr.rssi[1] + 321) / 2;
-		wstats.noise = ai->wstats.qual.noise;
-		wstats.updated = IW_QUAL_QUAL_UPDATED
-			| IW_QUAL_LEVEL_UPDATED
-			| IW_QUAL_DBM;
-		/* Update spy records */
-		wireless_spy_update(ai->dev, sa, &wstats);
-	}
-#endif /* IW_WIRELESS_SPY */
-	skb_reset_mac_header(skb);
-	skb->pkt_type = PACKET_OTHERHOST;
-	skb->dev = ai->wifidev;
-	skb->protocol = htons(ETH_P_802_2);
-	skb->ip_summed = CHECKSUM_NONE;
-	netif_rx(skb);
-
-badrx:
-	if (rxd.valid == 0) {
-		rxd.valid = 1;
-		rxd.rdy = 0;
-		rxd.len = PKTSIZE;
-		memcpy_toio(ai->rxfids[0].card_ram_off, &rxd, sizeof(rxd));
-	}
-}
-
-static inline void set_auth_type(struct airo_info *local, int auth_type)
-{
-	local->config.authType = auth_type;
-	/* Cache the last auth type used (of AUTH_OPEN and AUTH_ENCRYPT).
-	 * Used by airo_set_auth()
-	 */
-	if (auth_type == AUTH_OPEN || auth_type == AUTH_ENCRYPT)
-		local->last_auth = auth_type;
-}
-
-static int noinline_for_stack airo_readconfig(struct airo_info *ai,
-					      struct net_device *dev, int lock)
-{
-	int i, status;
-	/* large variables, so don't inline this function,
-	 * maybe change to kmalloc
-	 */
-	tdsRssiRid rssi_rid;
-	CapabilityRid cap_rid;
-
-	kfree(ai->SSID);
-	ai->SSID = NULL;
-	// general configuration (read/modify/write)
-	status = readConfigRid(ai, lock);
-	if (status != SUCCESS) return ERROR;
-
-	status = readCapabilityRid(ai, &cap_rid, lock);
-	if (status != SUCCESS) return ERROR;
-
-	status = PC4500_readrid(ai, RID_RSSI, &rssi_rid, sizeof(rssi_rid), lock);
-	if (status == SUCCESS) {
-		if (ai->rssi || (ai->rssi = kmalloc(512, GFP_KERNEL)) != NULL)
-			memcpy(ai->rssi, (u8*)&rssi_rid + 2, 512); /* Skip RID length member */
-	}
-	else {
-		kfree(ai->rssi);
-		ai->rssi = NULL;
-		if (cap_rid.softCap & cpu_to_le16(8))
-			ai->config.rmode |= RXMODE_NORMALIZED_RSSI;
-		else
-			airo_print_warn(ai->dev->name, "unknown received signal "
-					"level scale");
-	}
-	ai->config.opmode = adhoc ? MODE_STA_IBSS : MODE_STA_ESS;
-	set_auth_type(ai, AUTH_OPEN);
-	ai->config.modulation = MOD_CCK;
-
-	if (le16_to_cpu(cap_rid.len) >= sizeof(cap_rid) &&
-	    (cap_rid.extSoftCap & cpu_to_le16(1)) &&
-	    micsetup(ai) == SUCCESS) {
-		ai->config.opmode |= MODE_MIC;
-		set_bit(FLAG_MIC_CAPABLE, &ai->flags);
-	}
-
-	/* Save off the MAC */
-	eth_hw_addr_set(dev, ai->config.macAddr);
-
-	/* Check to see if there are any insmod configured
-	   rates to add */
-	if (rates[0]) {
-		memset(ai->config.rates, 0, sizeof(ai->config.rates));
-		for (i = 0; i < 8 && rates[i]; i++) {
-			ai->config.rates[i] = rates[i];
-		}
-	}
-	set_bit (FLAG_COMMIT, &ai->flags);
-
-	return SUCCESS;
-}
-
-
-static u16 setup_card(struct airo_info *ai, struct net_device *dev, int lock)
-{
-	Cmd cmd;
-	Resp rsp;
-	int status;
-	SsidRid mySsid;
-	__le16 lastindex;
-	WepKeyRid wkr;
-	int rc;
-
-	memset(&mySsid, 0, sizeof(mySsid));
-	kfree (ai->flash);
-	ai->flash = NULL;
-
-	/* The NOP is the first step in getting the card going */
-	cmd.cmd = NOP;
-	cmd.parm0 = cmd.parm1 = cmd.parm2 = 0;
-	if (lock && down_interruptible(&ai->sem))
-		return ERROR;
-	if (issuecommand(ai, &cmd, &rsp, true) != SUCCESS) {
-		if (lock)
-			up(&ai->sem);
-		return ERROR;
-	}
-	disable_MAC(ai, 0);
-
-	// Let's figure out if we need to use the AUX port
-	if (!test_bit(FLAG_MPI,&ai->flags)) {
-		cmd.cmd = CMD_ENABLEAUX;
-		if (issuecommand(ai, &cmd, &rsp, true) != SUCCESS) {
-			if (lock)
-				up(&ai->sem);
-			airo_print_err(ai->dev->name, "Error checking for AUX port");
-			return ERROR;
-		}
-		if (!aux_bap || rsp.status & 0xff00) {
-			ai->bap_read = fast_bap_read;
-			airo_print_dbg(ai->dev->name, "Doing fast bap_reads");
-		} else {
-			ai->bap_read = aux_bap_read;
-			airo_print_dbg(ai->dev->name, "Doing AUX bap_reads");
-		}
-	}
-	if (lock)
-		up(&ai->sem);
-	if (ai->config.len == 0) {
-		status = airo_readconfig(ai, dev, lock);
-		if (status != SUCCESS)
-			return ERROR;
-	}
-
-	/* Setup the SSIDs if present */
-	if (ssids[0]) {
-		int i;
-		for (i = 0; i < 3 && ssids[i]; i++) {
-			size_t len = strlen(ssids[i]);
-			if (len > 32)
-				len = 32;
-			mySsid.ssids[i].len = cpu_to_le16(len);
-			memcpy(mySsid.ssids[i].ssid, ssids[i], len);
-		}
-		mySsid.len = cpu_to_le16(sizeof(mySsid));
-	}
-
-	status = writeConfigRid(ai, lock);
-	if (status != SUCCESS) return ERROR;
-
-	/* Set up the SSID list */
-	if (ssids[0]) {
-		status = writeSsidRid(ai, &mySsid, lock);
-		if (status != SUCCESS) return ERROR;
-	}
-
-	status = enable_MAC(ai, lock);
-	if (status != SUCCESS)
-		return ERROR;
-
-	/* Grab the initial wep key, we gotta save it for auto_wep */
-	rc = readWepKeyRid(ai, &wkr, 1, lock);
-	if (rc == SUCCESS) do {
-		lastindex = wkr.kindex;
-		if (wkr.kindex == cpu_to_le16(0xffff)) {
-			ai->defindex = wkr.mac[0];
-		}
-		rc = readWepKeyRid(ai, &wkr, 0, lock);
-	} while (lastindex != wkr.kindex);
-
-	try_auto_wep(ai);
-
-	return SUCCESS;
-}
-
-static u16 issuecommand(struct airo_info *ai, Cmd *pCmd, Resp *pRsp,
-			bool may_sleep)
-{
-        // Im really paranoid about letting it run forever!
-	int max_tries = 600000;
-
-	if (IN4500(ai, EVSTAT) & EV_CMD)
-		OUT4500(ai, EVACK, EV_CMD);
-
-	OUT4500(ai, PARAM0, pCmd->parm0);
-	OUT4500(ai, PARAM1, pCmd->parm1);
-	OUT4500(ai, PARAM2, pCmd->parm2);
-	OUT4500(ai, COMMAND, pCmd->cmd);
-
-	while (max_tries-- && (IN4500(ai, EVSTAT) & EV_CMD) == 0) {
-		if ((IN4500(ai, COMMAND)) == pCmd->cmd)
-			// PC4500 didn't notice command, try again
-			OUT4500(ai, COMMAND, pCmd->cmd);
-		if (may_sleep && (max_tries & 255) == 0)
-			cond_resched();
-	}
-
-	if (max_tries == -1) {
-		airo_print_err(ai->dev->name,
-			"Max tries exceeded when issuing command");
-		if (IN4500(ai, COMMAND) & COMMAND_BUSY)
-			OUT4500(ai, EVACK, EV_CLEARCOMMANDBUSY);
-		return ERROR;
-	}
-
-	// command completed
-	pRsp->status = IN4500(ai, STATUS);
-	pRsp->rsp0 = IN4500(ai, RESP0);
-	pRsp->rsp1 = IN4500(ai, RESP1);
-	pRsp->rsp2 = IN4500(ai, RESP2);
-	if ((pRsp->status & 0xff00)!=0 && pCmd->cmd != CMD_SOFTRESET)
-		airo_print_err(ai->dev->name,
-			"cmd:%x status:%x rsp0:%x rsp1:%x rsp2:%x",
-			pCmd->cmd, pRsp->status, pRsp->rsp0, pRsp->rsp1,
-			pRsp->rsp2);
-
-	// clear stuck command busy if necessary
-	if (IN4500(ai, COMMAND) & COMMAND_BUSY) {
-		OUT4500(ai, EVACK, EV_CLEARCOMMANDBUSY);
-	}
-	// acknowledge processing the status/response
-	OUT4500(ai, EVACK, EV_CMD);
-
-	return SUCCESS;
-}
-
-/* Sets up the bap to start exchange data.  whichbap should
- * be one of the BAP0 or BAP1 defines.  Locks should be held before
- * calling! */
-static int bap_setup(struct airo_info *ai, u16 rid, u16 offset, int whichbap)
-{
-	int timeout = 50;
-	int max_tries = 3;
-
-	OUT4500(ai, SELECT0+whichbap, rid);
-	OUT4500(ai, OFFSET0+whichbap, offset);
-	while (1) {
-		int status = IN4500(ai, OFFSET0+whichbap);
-		if (status & BAP_BUSY) {
-                        /* This isn't really a timeout, but its kinda
-			   close */
-			if (timeout--) {
-				continue;
-			}
-		} else if (status & BAP_ERR) {
-			/* invalid rid or offset */
-			airo_print_err(ai->dev->name, "BAP error %x %d",
-				status, whichbap);
-			return ERROR;
-		} else if (status & BAP_DONE) { // success
-			return SUCCESS;
-		}
-		if (!(max_tries--)) {
-			airo_print_err(ai->dev->name,
-				"BAP setup error too many retries\n");
-			return ERROR;
-		}
-		// -- PC4500 missed it, try again
-		OUT4500(ai, SELECT0+whichbap, rid);
-		OUT4500(ai, OFFSET0+whichbap, offset);
-		timeout = 50;
-	}
-}
-
-/* should only be called by aux_bap_read.  This aux function and the
-   following use concepts not documented in the developers guide.  I
-   got them from a patch given to my by Aironet */
-static u16 aux_setup(struct airo_info *ai, u16 page,
-		     u16 offset, u16 *len)
-{
-	u16 next;
-
-	OUT4500(ai, AUXPAGE, page);
-	OUT4500(ai, AUXOFF, 0);
-	next = IN4500(ai, AUXDATA);
-	*len = IN4500(ai, AUXDATA)&0xff;
-	if (offset != 4) OUT4500(ai, AUXOFF, offset);
-	return next;
-}
-
-/* requires call to bap_setup() first */
-static int aux_bap_read(struct airo_info *ai, __le16 *pu16Dst,
-			int bytelen, int whichbap)
-{
-	u16 len;
-	u16 page;
-	u16 offset;
-	u16 next;
-	int words;
-	int i;
-	unsigned long flags;
-
-	spin_lock_irqsave(&ai->aux_lock, flags);
-	page = IN4500(ai, SWS0+whichbap);
-	offset = IN4500(ai, SWS2+whichbap);
-	next = aux_setup(ai, page, offset, &len);
-	words = (bytelen+1)>>1;
-
-	for (i = 0; i<words;) {
-		int count;
-		count = (len>>1) < (words-i) ? (len>>1) : (words-i);
-		if (!do8bitIO)
-			insw(ai->dev->base_addr+DATA0+whichbap,
-			      pu16Dst+i, count);
-		else
-			insb(ai->dev->base_addr+DATA0+whichbap,
-			      pu16Dst+i, count << 1);
-		i += count;
-		if (i<words) {
-			next = aux_setup(ai, next, 4, &len);
-		}
-	}
-	spin_unlock_irqrestore(&ai->aux_lock, flags);
-	return SUCCESS;
-}
-
-
-/* requires call to bap_setup() first */
-static int fast_bap_read(struct airo_info *ai, __le16 *pu16Dst,
-			 int bytelen, int whichbap)
-{
-	bytelen = (bytelen + 1) & (~1); // round up to even value
-	if (!do8bitIO)
-		insw(ai->dev->base_addr+DATA0+whichbap, pu16Dst, bytelen>>1);
-	else
-		insb(ai->dev->base_addr+DATA0+whichbap, pu16Dst, bytelen);
-	return SUCCESS;
-}
-
-/* requires call to bap_setup() first */
-static int bap_write(struct airo_info *ai, const __le16 *pu16Src,
-		     int bytelen, int whichbap)
-{
-	bytelen = (bytelen + 1) & (~1); // round up to even value
-	if (!do8bitIO)
-		outsw(ai->dev->base_addr+DATA0+whichbap,
-		       pu16Src, bytelen>>1);
-	else
-		outsb(ai->dev->base_addr+DATA0+whichbap, pu16Src, bytelen);
-	return SUCCESS;
-}
-
-static int PC4500_accessrid(struct airo_info *ai, u16 rid, u16 accmd)
-{
-	Cmd cmd; /* for issuing commands */
-	Resp rsp; /* response from commands */
-	u16 status;
-
-	memset(&cmd, 0, sizeof(cmd));
-	cmd.cmd = accmd;
-	cmd.parm0 = rid;
-	status = issuecommand(ai, &cmd, &rsp, true);
-	if (status != 0) return status;
-	if ((rsp.status & 0x7F00) != 0) {
-		return (accmd << 8) + (rsp.rsp0 & 0xFF);
-	}
-	return 0;
-}
-
-/*  Note, that we are using BAP1 which is also used by transmit, so
- *  we must get a lock. */
-static int PC4500_readrid(struct airo_info *ai, u16 rid, void *pBuf, int len, int lock)
-{
-	u16 status;
-        int rc = SUCCESS;
-
-	if (lock) {
-		if (down_interruptible(&ai->sem))
-			return ERROR;
-	}
-	if (test_bit(FLAG_MPI,&ai->flags)) {
-		Cmd cmd;
-		Resp rsp;
-
-		memset(&cmd, 0, sizeof(cmd));
-		memset(&rsp, 0, sizeof(rsp));
-		ai->config_desc.rid_desc.valid = 1;
-		ai->config_desc.rid_desc.len = RIDSIZE;
-		ai->config_desc.rid_desc.rid = 0;
-		ai->config_desc.rid_desc.host_addr = ai->ridbus;
-
-		cmd.cmd = CMD_ACCESS;
-		cmd.parm0 = rid;
-
-		memcpy_toio(ai->config_desc.card_ram_off,
-			&ai->config_desc.rid_desc, sizeof(Rid));
-
-		rc = issuecommand(ai, &cmd, &rsp, true);
-
-		if (rsp.status & 0x7f00)
-			rc = rsp.rsp0;
-		if (!rc)
-			memcpy(pBuf, ai->config_desc.virtual_host_addr, len);
-		goto done;
-	} else {
-		if ((status = PC4500_accessrid(ai, rid, CMD_ACCESS))!=SUCCESS) {
-	                rc = status;
-	                goto done;
-	        }
-		if (bap_setup(ai, rid, 0, BAP1) != SUCCESS) {
-			rc = ERROR;
-	                goto done;
-	        }
-		// read the rid length field
-		bap_read(ai, pBuf, 2, BAP1);
-		// length for remaining part of rid
-		len = min(len, (int)le16_to_cpu(*(__le16*)pBuf)) - 2;
-
-		if (len <= 2) {
-			airo_print_err(ai->dev->name,
-				"Rid %x has a length of %d which is too short",
-				(int)rid, (int)len);
-			rc = ERROR;
-	                goto done;
-		}
-		// read remainder of the rid
-		rc = bap_read(ai, ((__le16*)pBuf)+1, len, BAP1);
-	}
-done:
-	if (lock)
-		up(&ai->sem);
-	return rc;
-}
-
-/*  Note, that we are using BAP1 which is also used by transmit, so
- *  make sure this isn't called when a transmit is happening */
-static int PC4500_writerid(struct airo_info *ai, u16 rid,
-			   const void *pBuf, int len, int lock)
-{
-	u16 status;
-	int rc = SUCCESS;
-
-	*(__le16*)pBuf = cpu_to_le16((u16)len);
-
-	if (lock) {
-		if (down_interruptible(&ai->sem))
-			return ERROR;
-	}
-	if (test_bit(FLAG_MPI,&ai->flags)) {
-		Cmd cmd;
-		Resp rsp;
-
-		if (test_bit(FLAG_ENABLED, &ai->flags) && (RID_WEP_TEMP != rid))
-			airo_print_err(ai->dev->name,
-				"%s: MAC should be disabled (rid=%04x)",
-				__func__, rid);
-		memset(&cmd, 0, sizeof(cmd));
-		memset(&rsp, 0, sizeof(rsp));
-
-		ai->config_desc.rid_desc.valid = 1;
-		ai->config_desc.rid_desc.len = *((u16 *)pBuf);
-		ai->config_desc.rid_desc.rid = 0;
-
-		cmd.cmd = CMD_WRITERID;
-		cmd.parm0 = rid;
-
-		memcpy_toio(ai->config_desc.card_ram_off,
-			&ai->config_desc.rid_desc, sizeof(Rid));
-
-		if (len < 4 || len > 2047) {
-			airo_print_err(ai->dev->name, "%s: len=%d", __func__, len);
-			rc = -1;
-		} else {
-			memcpy(ai->config_desc.virtual_host_addr,
-				pBuf, len);
-
-			rc = issuecommand(ai, &cmd, &rsp, true);
-			if ((rc & 0xff00) != 0) {
-				airo_print_err(ai->dev->name, "%s: Write rid Error %d",
-						__func__, rc);
-				airo_print_err(ai->dev->name, "%s: Cmd=%04x",
-						__func__, cmd.cmd);
-			}
-
-			if ((rsp.status & 0x7f00))
-				rc = rsp.rsp0;
-		}
-	} else {
-		// --- first access so that we can write the rid data
-		if ((status = PC4500_accessrid(ai, rid, CMD_ACCESS)) != 0) {
-	                rc = status;
-	                goto done;
-	        }
-		// --- now write the rid data
-		if (bap_setup(ai, rid, 0, BAP1) != SUCCESS) {
-	                rc = ERROR;
-	                goto done;
-	        }
-		bap_write(ai, pBuf, len, BAP1);
-		// ---now commit the rid data
-		rc = PC4500_accessrid(ai, rid, 0x100|CMD_ACCESS);
-	}
-done:
-	if (lock)
-		up(&ai->sem);
-        return rc;
-}
-
-/* Allocates a FID to be used for transmitting packets.  We only use
-   one for now. */
-static u16 transmit_allocate(struct airo_info *ai, int lenPayload, int raw)
-{
-	unsigned int loop = 3000;
-	Cmd cmd;
-	Resp rsp;
-	u16 txFid;
-	__le16 txControl;
-
-	cmd.cmd = CMD_ALLOCATETX;
-	cmd.parm0 = lenPayload;
-	if (down_interruptible(&ai->sem))
-		return ERROR;
-	if (issuecommand(ai, &cmd, &rsp, true) != SUCCESS) {
-		txFid = ERROR;
-		goto done;
-	}
-	if ((rsp.status & 0xFF00) != 0) {
-		txFid = ERROR;
-		goto done;
-	}
-	/* wait for the allocate event/indication
-	 * It makes me kind of nervous that this can just sit here and spin,
-	 * but in practice it only loops like four times. */
-	while (((IN4500(ai, EVSTAT) & EV_ALLOC) == 0) && --loop);
-	if (!loop) {
-		txFid = ERROR;
-		goto done;
-	}
-
-	// get the allocated fid and acknowledge
-	txFid = IN4500(ai, TXALLOCFID);
-	OUT4500(ai, EVACK, EV_ALLOC);
-
-	/*  The CARD is pretty cool since it converts the ethernet packet
-	 *  into 802.11.  Also note that we don't release the FID since we
-	 *  will be using the same one over and over again. */
-	/*  We only have to setup the control once since we are not
-	 *  releasing the fid. */
-	if (raw)
-		txControl = cpu_to_le16(TXCTL_TXOK | TXCTL_TXEX | TXCTL_802_11
-			| TXCTL_ETHERNET | TXCTL_NORELEASE);
-	else
-		txControl = cpu_to_le16(TXCTL_TXOK | TXCTL_TXEX | TXCTL_802_3
-			| TXCTL_ETHERNET | TXCTL_NORELEASE);
-	if (bap_setup(ai, txFid, 0x0008, BAP1) != SUCCESS)
-		txFid = ERROR;
-	else
-		bap_write(ai, &txControl, sizeof(txControl), BAP1);
-
-done:
-	up(&ai->sem);
-
-	return txFid;
-}
-
-/* In general BAP1 is dedicated to transmiting packets.  However,
-   since we need a BAP when accessing RIDs, we also use BAP1 for that.
-   Make sure the BAP1 spinlock is held when this is called. */
-static int transmit_802_3_packet(struct airo_info *ai, int len, char *pPacket,
-				 bool may_sleep)
-{
-	__le16 payloadLen;
-	Cmd cmd;
-	Resp rsp;
-	int miclen = 0;
-	u16 txFid = len;
-	MICBuffer pMic;
-
-	len >>= 16;
-
-	if (len <= ETH_ALEN * 2) {
-		airo_print_warn(ai->dev->name, "Short packet %d", len);
-		return ERROR;
-	}
-	len -= ETH_ALEN * 2;
-
-	if (test_bit(FLAG_MIC_CAPABLE, &ai->flags) && ai->micstats.enabled &&
-	    (ntohs(((__be16 *)pPacket)[6]) != 0x888E)) {
-		if (encapsulate(ai, (etherHead *)pPacket,&pMic, len) != SUCCESS)
-			return ERROR;
-		miclen = sizeof(pMic);
-	}
-	// packet is destination[6], source[6], payload[len-12]
-	// write the payload length and dst/src/payload
-	if (bap_setup(ai, txFid, 0x0036, BAP1) != SUCCESS) return ERROR;
-	/* The hardware addresses aren't counted as part of the payload, so
-	 * we have to subtract the 12 bytes for the addresses off */
-	payloadLen = cpu_to_le16(len + miclen);
-	bap_write(ai, &payloadLen, sizeof(payloadLen), BAP1);
-	bap_write(ai, (__le16*)pPacket, sizeof(etherHead), BAP1);
-	if (miclen)
-		bap_write(ai, (__le16*)&pMic, miclen, BAP1);
-	bap_write(ai, (__le16*)(pPacket + sizeof(etherHead)), len, BAP1);
-	// issue the transmit command
-	memset(&cmd, 0, sizeof(cmd));
-	cmd.cmd = CMD_TRANSMIT;
-	cmd.parm0 = txFid;
-	if (issuecommand(ai, &cmd, &rsp, may_sleep) != SUCCESS)
-		return ERROR;
-	if ((rsp.status & 0xFF00) != 0) return ERROR;
-	return SUCCESS;
-}
-
-static int transmit_802_11_packet(struct airo_info *ai, int len, char *pPacket,
-				  bool may_sleep)
-{
-	__le16 fc, payloadLen;
-	Cmd cmd;
-	Resp rsp;
-	int hdrlen;
-	static u8 tail[(30-10) + 2 + 6] = {[30-10] = 6};
-	/* padding of header to full size + le16 gaplen (6) + gaplen bytes */
-	u16 txFid = len;
-	len >>= 16;
-
-	fc = *(__le16*)pPacket;
-	hdrlen = header_len(fc);
-
-	if (len < hdrlen) {
-		airo_print_warn(ai->dev->name, "Short packet %d", len);
-		return ERROR;
-	}
-
-	/* packet is 802.11 header +  payload
-	 * write the payload length and dst/src/payload */
-	if (bap_setup(ai, txFid, 6, BAP1) != SUCCESS) return ERROR;
-	/* The 802.11 header aren't counted as part of the payload, so
-	 * we have to subtract the header bytes off */
-	payloadLen = cpu_to_le16(len-hdrlen);
-	bap_write(ai, &payloadLen, sizeof(payloadLen), BAP1);
-	if (bap_setup(ai, txFid, 0x0014, BAP1) != SUCCESS) return ERROR;
-	bap_write(ai, (__le16 *)pPacket, hdrlen, BAP1);
-	bap_write(ai, (__le16 *)(tail + (hdrlen - 10)), 38 - hdrlen, BAP1);
-
-	bap_write(ai, (__le16 *)(pPacket + hdrlen), len - hdrlen, BAP1);
-	// issue the transmit command
-	memset(&cmd, 0, sizeof(cmd));
-	cmd.cmd = CMD_TRANSMIT;
-	cmd.parm0 = txFid;
-	if (issuecommand(ai, &cmd, &rsp, may_sleep) != SUCCESS)
-		return ERROR;
-	if ((rsp.status & 0xFF00) != 0) return ERROR;
-	return SUCCESS;
-}
-
-/*
- *  This is the proc_fs routines.  It is a bit messier than I would
- *  like!  Feel free to clean it up!
- */
-
-static ssize_t proc_read(struct file *file,
-			  char __user *buffer,
-			  size_t len,
-			  loff_t *offset);
-
-static ssize_t proc_write(struct file *file,
-			   const char __user *buffer,
-			   size_t len,
-			   loff_t *offset);
-static int proc_close(struct inode *inode, struct file *file);
-
-static int proc_stats_open(struct inode *inode, struct file *file);
-static int proc_statsdelta_open(struct inode *inode, struct file *file);
-static int proc_status_open(struct inode *inode, struct file *file);
-static int proc_SSID_open(struct inode *inode, struct file *file);
-static int proc_APList_open(struct inode *inode, struct file *file);
-static int proc_BSSList_open(struct inode *inode, struct file *file);
-static int proc_config_open(struct inode *inode, struct file *file);
-static int proc_wepkey_open(struct inode *inode, struct file *file);
-
-static const struct proc_ops proc_statsdelta_ops = {
-	.proc_read	= proc_read,
-	.proc_open	= proc_statsdelta_open,
-	.proc_release	= proc_close,
-	.proc_lseek	= default_llseek,
-};
-
-static const struct proc_ops proc_stats_ops = {
-	.proc_read	= proc_read,
-	.proc_open	= proc_stats_open,
-	.proc_release	= proc_close,
-	.proc_lseek	= default_llseek,
-};
-
-static const struct proc_ops proc_status_ops = {
-	.proc_read	= proc_read,
-	.proc_open	= proc_status_open,
-	.proc_release	= proc_close,
-	.proc_lseek	= default_llseek,
-};
-
-static const struct proc_ops proc_SSID_ops = {
-	.proc_read	= proc_read,
-	.proc_write	= proc_write,
-	.proc_open	= proc_SSID_open,
-	.proc_release	= proc_close,
-	.proc_lseek	= default_llseek,
-};
-
-static const struct proc_ops proc_BSSList_ops = {
-	.proc_read	= proc_read,
-	.proc_write	= proc_write,
-	.proc_open	= proc_BSSList_open,
-	.proc_release	= proc_close,
-	.proc_lseek	= default_llseek,
-};
-
-static const struct proc_ops proc_APList_ops = {
-	.proc_read	= proc_read,
-	.proc_write	= proc_write,
-	.proc_open	= proc_APList_open,
-	.proc_release	= proc_close,
-	.proc_lseek	= default_llseek,
-};
-
-static const struct proc_ops proc_config_ops = {
-	.proc_read	= proc_read,
-	.proc_write	= proc_write,
-	.proc_open	= proc_config_open,
-	.proc_release	= proc_close,
-	.proc_lseek	= default_llseek,
-};
-
-static const struct proc_ops proc_wepkey_ops = {
-	.proc_read	= proc_read,
-	.proc_write	= proc_write,
-	.proc_open	= proc_wepkey_open,
-	.proc_release	= proc_close,
-	.proc_lseek	= default_llseek,
-};
-
-static struct proc_dir_entry *airo_entry;
-
-struct proc_data {
-	int release_buffer;
-	int readlen;
-	char *rbuffer;
-	int writelen;
-	int maxwritelen;
-	char *wbuffer;
-	void (*on_close) (struct inode *, struct file *);
-};
-
-static int setup_proc_entry(struct net_device *dev,
-			     struct airo_info *apriv)
-{
-	struct proc_dir_entry *entry;
-
-	/* First setup the device directory */
-	strcpy(apriv->proc_name, dev->name);
-	apriv->proc_entry = proc_mkdir_mode(apriv->proc_name, airo_perm,
-					    airo_entry);
-	if (!apriv->proc_entry)
-		return -ENOMEM;
-	proc_set_user(apriv->proc_entry, proc_kuid, proc_kgid);
-
-	/* Setup the StatsDelta */
-	entry = proc_create_data("StatsDelta", 0444 & proc_perm,
-				 apriv->proc_entry, &proc_statsdelta_ops, dev);
-	if (!entry)
-		goto fail;
-	proc_set_user(entry, proc_kuid, proc_kgid);
-
-	/* Setup the Stats */
-	entry = proc_create_data("Stats", 0444 & proc_perm,
-				 apriv->proc_entry, &proc_stats_ops, dev);
-	if (!entry)
-		goto fail;
-	proc_set_user(entry, proc_kuid, proc_kgid);
-
-	/* Setup the Status */
-	entry = proc_create_data("Status", 0444 & proc_perm,
-				 apriv->proc_entry, &proc_status_ops, dev);
-	if (!entry)
-		goto fail;
-	proc_set_user(entry, proc_kuid, proc_kgid);
-
-	/* Setup the Config */
-	entry = proc_create_data("Config", proc_perm,
-				 apriv->proc_entry, &proc_config_ops, dev);
-	if (!entry)
-		goto fail;
-	proc_set_user(entry, proc_kuid, proc_kgid);
-
-	/* Setup the SSID */
-	entry = proc_create_data("SSID", proc_perm,
-				 apriv->proc_entry, &proc_SSID_ops, dev);
-	if (!entry)
-		goto fail;
-	proc_set_user(entry, proc_kuid, proc_kgid);
-
-	/* Setup the APList */
-	entry = proc_create_data("APList", proc_perm,
-				 apriv->proc_entry, &proc_APList_ops, dev);
-	if (!entry)
-		goto fail;
-	proc_set_user(entry, proc_kuid, proc_kgid);
-
-	/* Setup the BSSList */
-	entry = proc_create_data("BSSList", proc_perm,
-				 apriv->proc_entry, &proc_BSSList_ops, dev);
-	if (!entry)
-		goto fail;
-	proc_set_user(entry, proc_kuid, proc_kgid);
-
-	/* Setup the WepKey */
-	entry = proc_create_data("WepKey", proc_perm,
-				 apriv->proc_entry, &proc_wepkey_ops, dev);
-	if (!entry)
-		goto fail;
-	proc_set_user(entry, proc_kuid, proc_kgid);
-	return 0;
-
-fail:
-	remove_proc_subtree(apriv->proc_name, airo_entry);
-	return -ENOMEM;
-}
-
-static int takedown_proc_entry(struct net_device *dev,
-				struct airo_info *apriv)
-{
-	remove_proc_subtree(apriv->proc_name, airo_entry);
-	return 0;
-}
-
-/*
- *  What we want from the proc_fs is to be able to efficiently read
- *  and write the configuration.  To do this, we want to read the
- *  configuration when the file is opened and write it when the file is
- *  closed.  So basically we allocate a read buffer at open and fill it
- *  with data, and allocate a write buffer and read it at close.
- */
-
-/*
- *  The read routine is generic, it relies on the preallocated rbuffer
- *  to supply the data.
- */
-static ssize_t proc_read(struct file *file,
-			  char __user *buffer,
-			  size_t len,
-			  loff_t *offset)
-{
-	struct proc_data *priv = file->private_data;
-
-	if (!priv->rbuffer)
-		return -EINVAL;
-
-	return simple_read_from_buffer(buffer, len, offset, priv->rbuffer,
-					priv->readlen);
-}
-
-/*
- *  The write routine is generic, it fills in a preallocated rbuffer
- *  to supply the data.
- */
-static ssize_t proc_write(struct file *file,
-			   const char __user *buffer,
-			   size_t len,
-			   loff_t *offset)
-{
-	ssize_t ret;
-	struct proc_data *priv = file->private_data;
-
-	if (!priv->wbuffer)
-		return -EINVAL;
-
-	ret = simple_write_to_buffer(priv->wbuffer, priv->maxwritelen, offset,
-					buffer, len);
-	if (ret > 0)
-		priv->writelen = max_t(int, priv->writelen, *offset);
-
-	return ret;
-}
-
-static int proc_status_open(struct inode *inode, struct file *file)
-{
-	struct proc_data *data;
-	struct net_device *dev = pde_data(inode);
-	struct airo_info *apriv = dev->ml_priv;
-	CapabilityRid cap_rid;
-	StatusRid status_rid;
-	u16 mode;
-	int i;
-
-	if ((file->private_data = kzalloc(sizeof(struct proc_data), GFP_KERNEL)) == NULL)
-		return -ENOMEM;
-	data = file->private_data;
-	if ((data->rbuffer = kmalloc(2048, GFP_KERNEL)) == NULL) {
-		kfree (file->private_data);
-		return -ENOMEM;
-	}
-
-	readStatusRid(apriv, &status_rid, 1);
-	readCapabilityRid(apriv, &cap_rid, 1);
-
-	mode = le16_to_cpu(status_rid.mode);
-
-        i = sprintf(data->rbuffer, "Status: %s%s%s%s%s%s%s%s%s\n",
-                    mode & 1 ? "CFG ": "",
-                    mode & 2 ? "ACT ": "",
-                    mode & 0x10 ? "SYN ": "",
-                    mode & 0x20 ? "LNK ": "",
-                    mode & 0x40 ? "LEAP ": "",
-                    mode & 0x80 ? "PRIV ": "",
-                    mode & 0x100 ? "KEY ": "",
-                    mode & 0x200 ? "WEP ": "",
-                    mode & 0x8000 ? "ERR ": "");
-	sprintf(data->rbuffer+i, "Mode: %x\n"
-		 "Signal Strength: %d\n"
-		 "Signal Quality: %d\n"
-		 "SSID: %-.*s\n"
-		 "AP: %-.16s\n"
-		 "Freq: %d\n"
-		 "BitRate: %dmbs\n"
-		 "Driver Version: %s\n"
-		 "Device: %s\nManufacturer: %s\nFirmware Version: %s\n"
-		 "Radio type: %x\nCountry: %x\nHardware Version: %x\n"
-		 "Software Version: %x\nSoftware Subversion: %x\n"
-		 "Boot block version: %x\n",
-		 le16_to_cpu(status_rid.mode),
-		 le16_to_cpu(status_rid.normalizedSignalStrength),
-		 le16_to_cpu(status_rid.signalQuality),
-		 le16_to_cpu(status_rid.SSIDlen),
-		 status_rid.SSID,
-		 status_rid.apName,
-		 le16_to_cpu(status_rid.channel),
-		 le16_to_cpu(status_rid.currentXmitRate) / 2,
-		 version,
-		 cap_rid.prodName,
-		 cap_rid.manName,
-		 cap_rid.prodVer,
-		 le16_to_cpu(cap_rid.radioType),
-		 le16_to_cpu(cap_rid.country),
-		 le16_to_cpu(cap_rid.hardVer),
-		 le16_to_cpu(cap_rid.softVer),
-		 le16_to_cpu(cap_rid.softSubVer),
-		 le16_to_cpu(cap_rid.bootBlockVer));
-	data->readlen = strlen(data->rbuffer);
-	return 0;
-}
-
-static int proc_stats_rid_open(struct inode*, struct file*, u16);
-static int proc_statsdelta_open(struct inode *inode,
-				 struct file *file)
-{
-	if (file->f_mode&FMODE_WRITE) {
-		return proc_stats_rid_open(inode, file, RID_STATSDELTACLEAR);
-	}
-	return proc_stats_rid_open(inode, file, RID_STATSDELTA);
-}
-
-static int proc_stats_open(struct inode *inode, struct file *file)
-{
-	return proc_stats_rid_open(inode, file, RID_STATS);
-}
-
-static int proc_stats_rid_open(struct inode *inode,
-				struct file *file,
-				u16 rid)
-{
-	struct proc_data *data;
-	struct net_device *dev = pde_data(inode);
-	struct airo_info *apriv = dev->ml_priv;
-	StatsRid stats;
-	int i, j;
-	__le32 *vals = stats.vals;
-	int len;
-
-	if ((file->private_data = kzalloc(sizeof(struct proc_data), GFP_KERNEL)) == NULL)
-		return -ENOMEM;
-	data = file->private_data;
-	if ((data->rbuffer = kmalloc(4096, GFP_KERNEL)) == NULL) {
-		kfree (file->private_data);
-		return -ENOMEM;
-	}
-
-	readStatsRid(apriv, &stats, rid, 1);
-	len = le16_to_cpu(stats.len);
-
-        j = 0;
-	for (i = 0; statsLabels[i]!=(char *)-1 && i*4<len; i++) {
-		if (!statsLabels[i]) continue;
-		if (j+strlen(statsLabels[i])+16>4096) {
-			airo_print_warn(apriv->dev->name,
-			       "Potentially disastrous buffer overflow averted!");
-			break;
-		}
-		j+=sprintf(data->rbuffer+j, "%s: %u\n", statsLabels[i],
-				le32_to_cpu(vals[i]));
-	}
-	if (i*4 >= len) {
-		airo_print_warn(apriv->dev->name, "Got a short rid");
-	}
-	data->readlen = j;
-	return 0;
-}
-
-static int get_dec_u16(char *buffer, int *start, int limit)
-{
-	u16 value;
-	int valid = 0;
-	for (value = 0; *start < limit && buffer[*start] >= '0' &&
-			buffer[*start] <= '9'; (*start)++) {
-		valid = 1;
-		value *= 10;
-		value += buffer[*start] - '0';
-	}
-	if (!valid) return -1;
-	return value;
-}
-
-static int airo_config_commit(struct net_device *dev,
-			      struct iw_request_info *info,
-			      union iwreq_data *wrqu,
-			      char *extra);
-
-static inline int sniffing_mode(struct airo_info *ai)
-{
-	return (le16_to_cpu(ai->config.rmode) & le16_to_cpu(RXMODE_MASK)) >=
-		le16_to_cpu(RXMODE_RFMON);
-}
-
-static void proc_config_on_close(struct inode *inode, struct file *file)
-{
-	struct proc_data *data = file->private_data;
-	struct net_device *dev = pde_data(inode);
-	struct airo_info *ai = dev->ml_priv;
-	char *line;
-
-	if (!data->writelen) return;
-
-	readConfigRid(ai, 1);
-	set_bit (FLAG_COMMIT, &ai->flags);
-
-	line = data->wbuffer;
-	while (line[0]) {
-/*** Mode processing */
-		if (!strncmp(line, "Mode: ", 6)) {
-			line += 6;
-			if (sniffing_mode(ai))
-				set_bit (FLAG_RESET, &ai->flags);
-			ai->config.rmode &= ~RXMODE_FULL_MASK;
-			clear_bit (FLAG_802_11, &ai->flags);
-			ai->config.opmode &= ~MODE_CFG_MASK;
-			ai->config.scanMode = SCANMODE_ACTIVE;
-			if (line[0] == 'a') {
-				ai->config.opmode |= MODE_STA_IBSS;
-			} else {
-				ai->config.opmode |= MODE_STA_ESS;
-				if (line[0] == 'r') {
-					ai->config.rmode |= RXMODE_RFMON | RXMODE_DISABLE_802_3_HEADER;
-					ai->config.scanMode = SCANMODE_PASSIVE;
-					set_bit (FLAG_802_11, &ai->flags);
-				} else if (line[0] == 'y') {
-					ai->config.rmode |= RXMODE_RFMON_ANYBSS | RXMODE_DISABLE_802_3_HEADER;
-					ai->config.scanMode = SCANMODE_PASSIVE;
-					set_bit (FLAG_802_11, &ai->flags);
-				} else if (line[0] == 'l')
-					ai->config.rmode |= RXMODE_LANMON;
-			}
-			set_bit (FLAG_COMMIT, &ai->flags);
-		}
-
-/*** Radio status */
-		else if (!strncmp(line,"Radio: ", 7)) {
-			line += 7;
-			if (!strncmp(line,"off", 3)) {
-				set_bit (FLAG_RADIO_OFF, &ai->flags);
-			} else {
-				clear_bit (FLAG_RADIO_OFF, &ai->flags);
-			}
-		}
-/*** NodeName processing */
-		else if (!strncmp(line, "NodeName: ", 10)) {
-			int j;
-
-			line += 10;
-			memset(ai->config.nodeName, 0, 16);
-/* Do the name, assume a space between the mode and node name */
-			for (j = 0; j < 16 && line[j] != '\n'; j++) {
-				ai->config.nodeName[j] = line[j];
-			}
-			set_bit (FLAG_COMMIT, &ai->flags);
-		}
-
-/*** PowerMode processing */
-		else if (!strncmp(line, "PowerMode: ", 11)) {
-			line += 11;
-			if (!strncmp(line, "PSPCAM", 6)) {
-				ai->config.powerSaveMode = POWERSAVE_PSPCAM;
-				set_bit (FLAG_COMMIT, &ai->flags);
-			} else if (!strncmp(line, "PSP", 3)) {
-				ai->config.powerSaveMode = POWERSAVE_PSP;
-				set_bit (FLAG_COMMIT, &ai->flags);
-			} else {
-				ai->config.powerSaveMode = POWERSAVE_CAM;
-				set_bit (FLAG_COMMIT, &ai->flags);
-			}
-		} else if (!strncmp(line, "DataRates: ", 11)) {
-			int v, i = 0, k = 0; /* i is index into line,
-						k is index to rates */
-
-			line += 11;
-			while ((v = get_dec_u16(line, &i, 3))!=-1) {
-				ai->config.rates[k++] = (u8)v;
-				line += i + 1;
-				i = 0;
-			}
-			set_bit (FLAG_COMMIT, &ai->flags);
-		} else if (!strncmp(line, "Channel: ", 9)) {
-			int v, i = 0;
-			line += 9;
-			v = get_dec_u16(line, &i, i+3);
-			if (v != -1) {
-				ai->config.channelSet = cpu_to_le16(v);
-				set_bit (FLAG_COMMIT, &ai->flags);
-			}
-		} else if (!strncmp(line, "XmitPower: ", 11)) {
-			int v, i = 0;
-			line += 11;
-			v = get_dec_u16(line, &i, i+3);
-			if (v != -1) {
-				ai->config.txPower = cpu_to_le16(v);
-				set_bit (FLAG_COMMIT, &ai->flags);
-			}
-		} else if (!strncmp(line, "WEP: ", 5)) {
-			line += 5;
-			switch(line[0]) {
-			case 's':
-				set_auth_type(ai, AUTH_SHAREDKEY);
-				break;
-			case 'e':
-				set_auth_type(ai, AUTH_ENCRYPT);
-				break;
-			default:
-				set_auth_type(ai, AUTH_OPEN);
-				break;
-			}
-			set_bit (FLAG_COMMIT, &ai->flags);
-		} else if (!strncmp(line, "LongRetryLimit: ", 16)) {
-			int v, i = 0;
-
-			line += 16;
-			v = get_dec_u16(line, &i, 3);
-			v = (v<0) ? 0 : ((v>255) ? 255 : v);
-			ai->config.longRetryLimit = cpu_to_le16(v);
-			set_bit (FLAG_COMMIT, &ai->flags);
-		} else if (!strncmp(line, "ShortRetryLimit: ", 17)) {
-			int v, i = 0;
-
-			line += 17;
-			v = get_dec_u16(line, &i, 3);
-			v = (v<0) ? 0 : ((v>255) ? 255 : v);
-			ai->config.shortRetryLimit = cpu_to_le16(v);
-			set_bit (FLAG_COMMIT, &ai->flags);
-		} else if (!strncmp(line, "RTSThreshold: ", 14)) {
-			int v, i = 0;
-
-			line += 14;
-			v = get_dec_u16(line, &i, 4);
-			v = (v<0) ? 0 : ((v>AIRO_DEF_MTU) ? AIRO_DEF_MTU : v);
-			ai->config.rtsThres = cpu_to_le16(v);
-			set_bit (FLAG_COMMIT, &ai->flags);
-		} else if (!strncmp(line, "TXMSDULifetime: ", 16)) {
-			int v, i = 0;
-
-			line += 16;
-			v = get_dec_u16(line, &i, 5);
-			v = (v<0) ? 0 : v;
-			ai->config.txLifetime = cpu_to_le16(v);
-			set_bit (FLAG_COMMIT, &ai->flags);
-		} else if (!strncmp(line, "RXMSDULifetime: ", 16)) {
-			int v, i = 0;
-
-			line += 16;
-			v = get_dec_u16(line, &i, 5);
-			v = (v<0) ? 0 : v;
-			ai->config.rxLifetime = cpu_to_le16(v);
-			set_bit (FLAG_COMMIT, &ai->flags);
-		} else if (!strncmp(line, "TXDiversity: ", 13)) {
-			ai->config.txDiversity =
-				(line[13]=='l') ? 1 :
-				((line[13]=='r')? 2: 3);
-			set_bit (FLAG_COMMIT, &ai->flags);
-		} else if (!strncmp(line, "RXDiversity: ", 13)) {
-			ai->config.rxDiversity =
-				(line[13]=='l') ? 1 :
-				((line[13]=='r')? 2: 3);
-			set_bit (FLAG_COMMIT, &ai->flags);
-		} else if (!strncmp(line, "FragThreshold: ", 15)) {
-			int v, i = 0;
-
-			line += 15;
-			v = get_dec_u16(line, &i, 4);
-			v = (v<256) ? 256 : ((v>AIRO_DEF_MTU) ? AIRO_DEF_MTU : v);
-			v = v & 0xfffe; /* Make sure its even */
-			ai->config.fragThresh = cpu_to_le16(v);
-			set_bit (FLAG_COMMIT, &ai->flags);
-		} else if (!strncmp(line, "Modulation: ", 12)) {
-			line += 12;
-			switch(*line) {
-			case 'd':  ai->config.modulation = MOD_DEFAULT; set_bit(FLAG_COMMIT, &ai->flags); break;
-			case 'c':  ai->config.modulation = MOD_CCK; set_bit(FLAG_COMMIT, &ai->flags); break;
-			case 'm':  ai->config.modulation = MOD_MOK; set_bit(FLAG_COMMIT, &ai->flags); break;
-			default: airo_print_warn(ai->dev->name, "Unknown modulation");
-			}
-		} else if (!strncmp(line, "Preamble: ", 10)) {
-			line += 10;
-			switch(*line) {
-			case 'a': ai->config.preamble = PREAMBLE_AUTO; set_bit(FLAG_COMMIT, &ai->flags); break;
-			case 'l': ai->config.preamble = PREAMBLE_LONG; set_bit(FLAG_COMMIT, &ai->flags); break;
-			case 's': ai->config.preamble = PREAMBLE_SHORT; set_bit(FLAG_COMMIT, &ai->flags); break;
-			default: airo_print_warn(ai->dev->name, "Unknown preamble");
-			}
-		} else {
-			airo_print_warn(ai->dev->name, "Couldn't figure out %s", line);
-		}
-		while (line[0] && line[0] != '\n') line++;
-		if (line[0]) line++;
-	}
-	airo_config_commit(dev, NULL, NULL, NULL);
-}
-
-static const char *get_rmode(__le16 mode)
-{
-        switch(mode & RXMODE_MASK) {
-        case RXMODE_RFMON:  return "rfmon";
-        case RXMODE_RFMON_ANYBSS:  return "yna (any) bss rfmon";
-        case RXMODE_LANMON:  return "lanmon";
-        }
-        return "ESS";
-}
-
-static int proc_config_open(struct inode *inode, struct file *file)
-{
-	struct proc_data *data;
-	struct net_device *dev = pde_data(inode);
-	struct airo_info *ai = dev->ml_priv;
-	int i;
-	__le16 mode;
-
-	if ((file->private_data = kzalloc(sizeof(struct proc_data), GFP_KERNEL)) == NULL)
-		return -ENOMEM;
-	data = file->private_data;
-	if ((data->rbuffer = kmalloc(2048, GFP_KERNEL)) == NULL) {
-		kfree (file->private_data);
-		return -ENOMEM;
-	}
-	if ((data->wbuffer = kzalloc(2048, GFP_KERNEL)) == NULL) {
-		kfree (data->rbuffer);
-		kfree (file->private_data);
-		return -ENOMEM;
-	}
-	data->maxwritelen = 2048;
-	data->on_close = proc_config_on_close;
-
-	readConfigRid(ai, 1);
-
-	mode = ai->config.opmode & MODE_CFG_MASK;
-	i = sprintf(data->rbuffer,
-		     "Mode: %s\n"
-		     "Radio: %s\n"
-		     "NodeName: %-16s\n"
-		     "PowerMode: %s\n"
-		     "DataRates: %d %d %d %d %d %d %d %d\n"
-		     "Channel: %d\n"
-		     "XmitPower: %d\n",
-		     mode == MODE_STA_IBSS ? "adhoc" :
-		     mode == MODE_STA_ESS ? get_rmode(ai->config.rmode):
-		     mode == MODE_AP ? "AP" :
-		     mode == MODE_AP_RPTR ? "AP RPTR" : "Error",
-		     test_bit(FLAG_RADIO_OFF, &ai->flags) ? "off" : "on",
-		     ai->config.nodeName,
-		     ai->config.powerSaveMode == POWERSAVE_CAM ? "CAM" :
-		     ai->config.powerSaveMode == POWERSAVE_PSP ? "PSP" :
-		     ai->config.powerSaveMode == POWERSAVE_PSPCAM ? "PSPCAM" :
-		     "Error",
-		     (int)ai->config.rates[0],
-		     (int)ai->config.rates[1],
-		     (int)ai->config.rates[2],
-		     (int)ai->config.rates[3],
-		     (int)ai->config.rates[4],
-		     (int)ai->config.rates[5],
-		     (int)ai->config.rates[6],
-		     (int)ai->config.rates[7],
-		     le16_to_cpu(ai->config.channelSet),
-		     le16_to_cpu(ai->config.txPower)
-		);
-	sprintf(data->rbuffer + i,
-		 "LongRetryLimit: %d\n"
-		 "ShortRetryLimit: %d\n"
-		 "RTSThreshold: %d\n"
-		 "TXMSDULifetime: %d\n"
-		 "RXMSDULifetime: %d\n"
-		 "TXDiversity: %s\n"
-		 "RXDiversity: %s\n"
-		 "FragThreshold: %d\n"
-		 "WEP: %s\n"
-		 "Modulation: %s\n"
-		 "Preamble: %s\n",
-		 le16_to_cpu(ai->config.longRetryLimit),
-		 le16_to_cpu(ai->config.shortRetryLimit),
-		 le16_to_cpu(ai->config.rtsThres),
-		 le16_to_cpu(ai->config.txLifetime),
-		 le16_to_cpu(ai->config.rxLifetime),
-		 ai->config.txDiversity == 1 ? "left" :
-		 ai->config.txDiversity == 2 ? "right" : "both",
-		 ai->config.rxDiversity == 1 ? "left" :
-		 ai->config.rxDiversity == 2 ? "right" : "both",
-		 le16_to_cpu(ai->config.fragThresh),
-		 ai->config.authType == AUTH_ENCRYPT ? "encrypt" :
-		 ai->config.authType == AUTH_SHAREDKEY ? "shared" : "open",
-		 ai->config.modulation == MOD_DEFAULT ? "default" :
-		 ai->config.modulation == MOD_CCK ? "cck" :
-		 ai->config.modulation == MOD_MOK ? "mok" : "error",
-		 ai->config.preamble == PREAMBLE_AUTO ? "auto" :
-		 ai->config.preamble == PREAMBLE_LONG ? "long" :
-		 ai->config.preamble == PREAMBLE_SHORT ? "short" : "error"
-		);
-	data->readlen = strlen(data->rbuffer);
-	return 0;
-}
-
-static void proc_SSID_on_close(struct inode *inode, struct file *file)
-{
-	struct proc_data *data = file->private_data;
-	struct net_device *dev = pde_data(inode);
-	struct airo_info *ai = dev->ml_priv;
-	SsidRid SSID_rid;
-	int i;
-	char *p = data->wbuffer;
-	char *end = p + data->writelen;
-
-	if (!data->writelen)
-		return;
-
-	*end = '\n'; /* sentinel; we have space for it */
-
-	memset(&SSID_rid, 0, sizeof(SSID_rid));
-
-	for (i = 0; i < 3 && p < end; i++) {
-		int j = 0;
-		/* copy up to 32 characters from this line */
-		while (*p != '\n' && j < 32)
-			SSID_rid.ssids[i].ssid[j++] = *p++;
-		if (j == 0)
-			break;
-		SSID_rid.ssids[i].len = cpu_to_le16(j);
-		/* skip to the beginning of the next line */
-		while (*p++ != '\n')
-			;
-	}
-	if (i)
-		SSID_rid.len = cpu_to_le16(sizeof(SSID_rid));
-	disable_MAC(ai, 1);
-	writeSsidRid(ai, &SSID_rid, 1);
-	enable_MAC(ai, 1);
-}
-
-static void proc_APList_on_close(struct inode *inode, struct file *file)
-{
-	struct proc_data *data = file->private_data;
-	struct net_device *dev = pde_data(inode);
-	struct airo_info *ai = dev->ml_priv;
-	APListRid *APList_rid = &ai->APList;
-	int i;
-
-	if (!data->writelen) return;
-
-	memset(APList_rid, 0, sizeof(*APList_rid));
-	APList_rid->len = cpu_to_le16(sizeof(*APList_rid));
-
-	for (i = 0; i < 4 && data->writelen >= (i + 1) * 6 * 3; i++)
-		mac_pton(data->wbuffer + i * 6 * 3, APList_rid->ap[i]);
-
-	disable_MAC(ai, 1);
-	writeAPListRid(ai, APList_rid, 1);
-	enable_MAC(ai, 1);
-}
-
-/* This function wraps PC4500_writerid with a MAC disable */
-static int do_writerid(struct airo_info *ai, u16 rid, const void *rid_data,
-			int len, int dummy)
-{
-	int rc;
-
-	disable_MAC(ai, 1);
-	rc = PC4500_writerid(ai, rid, rid_data, len, 1);
-	enable_MAC(ai, 1);
-	return rc;
-}
-
-/* Returns the WEP key at the specified index, or -1 if that key does
- * not exist.  The buffer is assumed to be at least 16 bytes in length.
- */
-static int get_wep_key(struct airo_info *ai, u16 index, char *buf, u16 buflen)
-{
-	WepKeyRid wkr;
-	int rc;
-	__le16 lastindex;
-
-	rc = readWepKeyRid(ai, &wkr, 1, 1);
-	if (rc != SUCCESS)
-		return -1;
-	do {
-		lastindex = wkr.kindex;
-		if (le16_to_cpu(wkr.kindex) == index) {
-			int klen = min_t(int, buflen, le16_to_cpu(wkr.klen));
-			memcpy(buf, wkr.key, klen);
-			return klen;
-		}
-		rc = readWepKeyRid(ai, &wkr, 0, 1);
-		if (rc != SUCCESS)
-			return -1;
-	} while (lastindex != wkr.kindex);
-	return -1;
-}
-
-static int get_wep_tx_idx(struct airo_info *ai)
-{
-	WepKeyRid wkr;
-	int rc;
-	__le16 lastindex;
-
-	rc = readWepKeyRid(ai, &wkr, 1, 1);
-	if (rc != SUCCESS)
-		return -1;
-	do {
-		lastindex = wkr.kindex;
-		if (wkr.kindex == cpu_to_le16(0xffff))
-			return wkr.mac[0];
-		rc = readWepKeyRid(ai, &wkr, 0, 1);
-		if (rc != SUCCESS)
-			return -1;
-	} while (lastindex != wkr.kindex);
-	return -1;
-}
-
-static int set_wep_key(struct airo_info *ai, u16 index, const u8 *key,
-		       u16 keylen, int perm, int lock)
-{
-	static const unsigned char macaddr[ETH_ALEN] = { 0x01, 0, 0, 0, 0, 0 };
-	WepKeyRid wkr;
-	int rc;
-
-	if (WARN_ON(keylen == 0))
-		return -1;
-
-	memset(&wkr, 0, sizeof(wkr));
-	wkr.len = cpu_to_le16(sizeof(wkr));
-	wkr.kindex = cpu_to_le16(index);
-	wkr.klen = cpu_to_le16(keylen);
-	memcpy(wkr.key, key, keylen);
-	memcpy(wkr.mac, macaddr, ETH_ALEN);
-
-	if (perm) disable_MAC(ai, lock);
-	rc = writeWepKeyRid(ai, &wkr, perm, lock);
-	if (perm) enable_MAC(ai, lock);
-	return rc;
-}
-
-static int set_wep_tx_idx(struct airo_info *ai, u16 index, int perm, int lock)
-{
-	WepKeyRid wkr;
-	int rc;
-
-	memset(&wkr, 0, sizeof(wkr));
-	wkr.len = cpu_to_le16(sizeof(wkr));
-	wkr.kindex = cpu_to_le16(0xffff);
-	wkr.mac[0] = (char)index;
-
-	if (perm) {
-		ai->defindex = (char)index;
-		disable_MAC(ai, lock);
-	}
-
-	rc = writeWepKeyRid(ai, &wkr, perm, lock);
-
-	if (perm)
-		enable_MAC(ai, lock);
-	return rc;
-}
-
-static void proc_wepkey_on_close(struct inode *inode, struct file *file)
-{
-	struct proc_data *data;
-	struct net_device *dev = pde_data(inode);
-	struct airo_info *ai = dev->ml_priv;
-	int i, rc;
-	u8 key[16];
-	u16 index = 0;
-	int j = 0;
-
-	memset(key, 0, sizeof(key));
-
-	data = file->private_data;
-	if (!data->writelen) return;
-
-	if (data->wbuffer[0] >= '0' && data->wbuffer[0] <= '3' &&
-	    (data->wbuffer[1] == ' ' || data->wbuffer[1] == '\n')) {
-		index = data->wbuffer[0] - '0';
-		if (data->wbuffer[1] == '\n') {
-			rc = set_wep_tx_idx(ai, index, 1, 1);
-			if (rc < 0) {
-				airo_print_err(ai->dev->name, "failed to set "
-				               "WEP transmit index to %d: %d.",
-				               index, rc);
-			}
-			return;
-		}
-		j = 2;
-	} else {
-		airo_print_err(ai->dev->name, "WepKey passed invalid key index");
-		return;
-	}
-
-	for (i = 0; i < 16*3 && data->wbuffer[i+j]; i++) {
-		int val;
-
-		if (i % 3 == 2)
-			continue;
-
-		val = hex_to_bin(data->wbuffer[i+j]);
-		if (val < 0) {
-			airo_print_err(ai->dev->name, "WebKey passed invalid key hex");
-			return;
-		}
-		switch(i%3) {
-		case 0:
-			key[i/3] = (u8)val << 4;
-			break;
-		case 1:
-			key[i/3] |= (u8)val;
-			break;
-		}
-	}
-
-	rc = set_wep_key(ai, index, key, i/3, 1, 1);
-	if (rc < 0) {
-		airo_print_err(ai->dev->name, "failed to set WEP key at index "
-		               "%d: %d.", index, rc);
-	}
-}
-
-static int proc_wepkey_open(struct inode *inode, struct file *file)
-{
-	struct proc_data *data;
-	struct net_device *dev = pde_data(inode);
-	struct airo_info *ai = dev->ml_priv;
-	char *ptr;
-	WepKeyRid wkr;
-	__le16 lastindex;
-	int j = 0;
-	int rc;
-
-	if ((file->private_data = kzalloc(sizeof(struct proc_data), GFP_KERNEL)) == NULL)
-		return -ENOMEM;
-	memset(&wkr, 0, sizeof(wkr));
-	data = file->private_data;
-	if ((data->rbuffer = kzalloc(180, GFP_KERNEL)) == NULL) {
-		kfree (file->private_data);
-		return -ENOMEM;
-	}
-	data->writelen = 0;
-	data->maxwritelen = 80;
-	if ((data->wbuffer = kzalloc(80, GFP_KERNEL)) == NULL) {
-		kfree (data->rbuffer);
-		kfree (file->private_data);
-		return -ENOMEM;
-	}
-	data->on_close = proc_wepkey_on_close;
-
-	ptr = data->rbuffer;
-	strcpy(ptr, "No wep keys\n");
-	rc = readWepKeyRid(ai, &wkr, 1, 1);
-	if (rc == SUCCESS) do {
-		lastindex = wkr.kindex;
-		if (wkr.kindex == cpu_to_le16(0xffff)) {
-			j += sprintf(ptr+j, "Tx key = %d\n",
-				     (int)wkr.mac[0]);
-		} else {
-			j += sprintf(ptr+j, "Key %d set with length = %d\n",
-				     le16_to_cpu(wkr.kindex),
-				     le16_to_cpu(wkr.klen));
-		}
-		readWepKeyRid(ai, &wkr, 0, 1);
-	} while ((lastindex != wkr.kindex) && (j < 180-30));
-
-	data->readlen = strlen(data->rbuffer);
-	return 0;
-}
-
-static int proc_SSID_open(struct inode *inode, struct file *file)
-{
-	struct proc_data *data;
-	struct net_device *dev = pde_data(inode);
-	struct airo_info *ai = dev->ml_priv;
-	int i;
-	char *ptr;
-	SsidRid SSID_rid;
-
-	if ((file->private_data = kzalloc(sizeof(struct proc_data), GFP_KERNEL)) == NULL)
-		return -ENOMEM;
-	data = file->private_data;
-	if ((data->rbuffer = kmalloc(104, GFP_KERNEL)) == NULL) {
-		kfree (file->private_data);
-		return -ENOMEM;
-	}
-	data->writelen = 0;
-	data->maxwritelen = 33*3;
-	/* allocate maxwritelen + 1; we'll want a sentinel */
-	if ((data->wbuffer = kzalloc(33*3 + 1, GFP_KERNEL)) == NULL) {
-		kfree (data->rbuffer);
-		kfree (file->private_data);
-		return -ENOMEM;
-	}
-	data->on_close = proc_SSID_on_close;
-
-	readSsidRid(ai, &SSID_rid);
-	ptr = data->rbuffer;
-	for (i = 0; i < 3; i++) {
-		int j;
-		size_t len = le16_to_cpu(SSID_rid.ssids[i].len);
-		if (!len)
-			break;
-		if (len > 32)
-			len = 32;
-		for (j = 0; j < len && SSID_rid.ssids[i].ssid[j]; j++)
-			*ptr++ = SSID_rid.ssids[i].ssid[j];
-		*ptr++ = '\n';
-	}
-	*ptr = '\0';
-	data->readlen = strlen(data->rbuffer);
-	return 0;
-}
-
-static int proc_APList_open(struct inode *inode, struct file *file)
-{
-	struct proc_data *data;
-	struct net_device *dev = pde_data(inode);
-	struct airo_info *ai = dev->ml_priv;
-	int i;
-	char *ptr;
-	APListRid *APList_rid = &ai->APList;
-
-	if ((file->private_data = kzalloc(sizeof(struct proc_data), GFP_KERNEL)) == NULL)
-		return -ENOMEM;
-	data = file->private_data;
-	if ((data->rbuffer = kmalloc(104, GFP_KERNEL)) == NULL) {
-		kfree (file->private_data);
-		return -ENOMEM;
-	}
-	data->writelen = 0;
-	data->maxwritelen = 4*6*3;
-	if ((data->wbuffer = kzalloc(data->maxwritelen, GFP_KERNEL)) == NULL) {
-		kfree (data->rbuffer);
-		kfree (file->private_data);
-		return -ENOMEM;
-	}
-	data->on_close = proc_APList_on_close;
-
-	ptr = data->rbuffer;
-	for (i = 0; i < 4; i++) {
-// We end when we find a zero MAC
-		if (!*(int*)APList_rid->ap[i] &&
-		     !*(int*)&APList_rid->ap[i][2]) break;
-		ptr += sprintf(ptr, "%pM\n", APList_rid->ap[i]);
-	}
-	if (i==0) ptr += sprintf(ptr, "Not using specific APs\n");
-
-	*ptr = '\0';
-	data->readlen = strlen(data->rbuffer);
-	return 0;
-}
-
-static int proc_BSSList_open(struct inode *inode, struct file *file)
-{
-	struct proc_data *data;
-	struct net_device *dev = pde_data(inode);
-	struct airo_info *ai = dev->ml_priv;
-	char *ptr;
-	BSSListRid BSSList_rid;
-	int rc;
-	/* If doLoseSync is not 1, we won't do a Lose Sync */
-	int doLoseSync = -1;
-
-	if ((file->private_data = kzalloc(sizeof(struct proc_data), GFP_KERNEL)) == NULL)
-		return -ENOMEM;
-	data = file->private_data;
-	if ((data->rbuffer = kmalloc(1024, GFP_KERNEL)) == NULL) {
-		kfree (file->private_data);
-		return -ENOMEM;
-	}
-	data->writelen = 0;
-	data->maxwritelen = 0;
-	data->wbuffer = NULL;
-	data->on_close = NULL;
-
-	if (file->f_mode & FMODE_WRITE) {
-		if (!(file->f_mode & FMODE_READ)) {
-			Cmd cmd;
-			Resp rsp;
-
-			if (ai->flags & FLAG_RADIO_MASK) {
-				kfree(data->rbuffer);
-				kfree(file->private_data);
-				return -ENETDOWN;
-			}
-			memset(&cmd, 0, sizeof(cmd));
-			cmd.cmd = CMD_LISTBSS;
-			if (down_interruptible(&ai->sem)) {
-				kfree(data->rbuffer);
-				kfree(file->private_data);
-				return -ERESTARTSYS;
-			}
-			issuecommand(ai, &cmd, &rsp, true);
-			up(&ai->sem);
-			data->readlen = 0;
-			return 0;
-		}
-		doLoseSync = 1;
-	}
-	ptr = data->rbuffer;
-	/* There is a race condition here if there are concurrent opens.
-           Since it is a rare condition, we'll just live with it, otherwise
-           we have to add a spin lock... */
-	rc = readBSSListRid(ai, doLoseSync, &BSSList_rid);
-	while (rc == 0 && BSSList_rid.index != cpu_to_le16(0xffff)) {
-		ptr += sprintf(ptr, "%pM %.*s rssi = %d",
-			       BSSList_rid.bssid,
-				(int)BSSList_rid.ssidLen,
-				BSSList_rid.ssid,
-				le16_to_cpu(BSSList_rid.dBm));
-		ptr += sprintf(ptr, " channel = %d %s %s %s %s\n",
-				le16_to_cpu(BSSList_rid.dsChannel),
-				BSSList_rid.cap & CAP_ESS ? "ESS" : "",
-				BSSList_rid.cap & CAP_IBSS ? "adhoc" : "",
-				BSSList_rid.cap & CAP_PRIVACY ? "wep" : "",
-				BSSList_rid.cap & CAP_SHORTHDR ? "shorthdr" : "");
-		rc = readBSSListRid(ai, 0, &BSSList_rid);
-	}
-	*ptr = '\0';
-	data->readlen = strlen(data->rbuffer);
-	return 0;
-}
-
-static int proc_close(struct inode *inode, struct file *file)
-{
-	struct proc_data *data = file->private_data;
-
-	if (data->on_close != NULL)
-		data->on_close(inode, file);
-	kfree(data->rbuffer);
-	kfree(data->wbuffer);
-	kfree(data);
-	return 0;
-}
-
-/* Since the card doesn't automatically switch to the right WEP mode,
-   we will make it do it.  If the card isn't associated, every secs we
-   will switch WEP modes to see if that will help.  If the card is
-   associated we will check every minute to see if anything has
-   changed. */
-static void timer_func(struct net_device *dev)
-{
-	struct airo_info *apriv = dev->ml_priv;
-
-/* We don't have a link so try changing the authtype */
-	readConfigRid(apriv, 0);
-	disable_MAC(apriv, 0);
-	switch(apriv->config.authType) {
-		case AUTH_ENCRYPT:
-/* So drop to OPEN */
-			apriv->config.authType = AUTH_OPEN;
-			break;
-		case AUTH_SHAREDKEY:
-			if (apriv->keyindex < auto_wep) {
-				set_wep_tx_idx(apriv, apriv->keyindex, 0, 0);
-				apriv->config.authType = AUTH_SHAREDKEY;
-				apriv->keyindex++;
-			} else {
-			        /* Drop to ENCRYPT */
-				apriv->keyindex = 0;
-				set_wep_tx_idx(apriv, apriv->defindex, 0, 0);
-				apriv->config.authType = AUTH_ENCRYPT;
-			}
-			break;
-		default:  /* We'll escalate to SHAREDKEY */
-			apriv->config.authType = AUTH_SHAREDKEY;
-	}
-	set_bit (FLAG_COMMIT, &apriv->flags);
-	writeConfigRid(apriv, 0);
-	enable_MAC(apriv, 0);
-	up(&apriv->sem);
-
-/* Schedule check to see if the change worked */
-	clear_bit(JOB_AUTOWEP, &apriv->jobs);
-	apriv->expires = RUN_AT(HZ*3);
-}
-
-#ifdef CONFIG_PCI
-static int airo_pci_probe(struct pci_dev *pdev,
-				    const struct pci_device_id *pent)
-{
-	struct net_device *dev;
-
-	if (pci_enable_device(pdev))
-		return -ENODEV;
-	pci_set_master(pdev);
-
-	if (pdev->device == 0x5000 || pdev->device == 0xa504)
-			dev = _init_airo_card(pdev->irq, pdev->resource[0].start, 0, pdev, &pdev->dev);
-	else
-			dev = _init_airo_card(pdev->irq, pdev->resource[2].start, 0, pdev, &pdev->dev);
-	if (!dev) {
-		pci_disable_device(pdev);
-		return -ENODEV;
-	}
-
-	pci_set_drvdata(pdev, dev);
-	return 0;
-}
-
-static void airo_pci_remove(struct pci_dev *pdev)
-{
-	struct net_device *dev = pci_get_drvdata(pdev);
-
-	airo_print_info(dev->name, "Unregistering...");
-	stop_airo_card(dev, 1);
-	pci_disable_device(pdev);
-}
-
-static int __maybe_unused airo_pci_suspend(struct device *dev_d)
-{
-	struct net_device *dev = dev_get_drvdata(dev_d);
-	struct airo_info *ai = dev->ml_priv;
-	Cmd cmd;
-	Resp rsp;
-
-	if (!ai->SSID)
-		ai->SSID = kmalloc(sizeof(SsidRid), GFP_KERNEL);
-	if (!ai->SSID)
-		return -ENOMEM;
-	readSsidRid(ai, ai->SSID);
-	memset(&cmd, 0, sizeof(cmd));
-	/* the lock will be released at the end of the resume callback */
-	if (down_interruptible(&ai->sem))
-		return -EAGAIN;
-	disable_MAC(ai, 0);
-	netif_device_detach(dev);
-	ai->power = PMSG_SUSPEND;
-	cmd.cmd = HOSTSLEEP;
-	issuecommand(ai, &cmd, &rsp, true);
-
-	device_wakeup_enable(dev_d);
-	return 0;
-}
-
-static int __maybe_unused airo_pci_resume(struct device *dev_d)
-{
-	struct net_device *dev = dev_get_drvdata(dev_d);
-	struct airo_info *ai = dev->ml_priv;
-	pci_power_t prev_state = to_pci_dev(dev_d)->current_state;
-
-	device_wakeup_disable(dev_d);
-
-	if (prev_state != PCI_D1) {
-		reset_card(dev, 0);
-		mpi_init_descriptors(ai);
-		setup_card(ai, dev, 0);
-		clear_bit(FLAG_RADIO_OFF, &ai->flags);
-		clear_bit(FLAG_PENDING_XMIT, &ai->flags);
-	} else {
-		OUT4500(ai, EVACK, EV_AWAKEN);
-		OUT4500(ai, EVACK, EV_AWAKEN);
-		msleep(100);
-	}
-
-	set_bit(FLAG_COMMIT, &ai->flags);
-	disable_MAC(ai, 0);
-        msleep(200);
-	if (ai->SSID) {
-		writeSsidRid(ai, ai->SSID, 0);
-		kfree(ai->SSID);
-		ai->SSID = NULL;
-	}
-	writeAPListRid(ai, &ai->APList, 0);
-	writeConfigRid(ai, 0);
-	enable_MAC(ai, 0);
-	ai->power = PMSG_ON;
-	netif_device_attach(dev);
-	netif_wake_queue(dev);
-	enable_interrupts(ai);
-	up(&ai->sem);
-	return 0;
-}
-#endif
-
-static int __init airo_init_module(void)
-{
-	int i;
-
-	proc_kuid = make_kuid(&init_user_ns, proc_uid);
-	proc_kgid = make_kgid(&init_user_ns, proc_gid);
-	if (!uid_valid(proc_kuid) || !gid_valid(proc_kgid))
-		return -EINVAL;
-
-	airo_entry = proc_mkdir_mode("driver/aironet", airo_perm, NULL);
-
-	if (airo_entry)
-		proc_set_user(airo_entry, proc_kuid, proc_kgid);
-
-	for (i = 0; i < 4 && io[i] && irq[i]; i++) {
-		airo_print_info("", "Trying to configure ISA adapter at irq=%d "
-			"io = 0x%x", irq[i], io[i]);
-		if (init_airo_card(irq[i], io[i], 0, NULL)) {
-			/* do nothing */ ;
-		}
-	}
-
-#ifdef CONFIG_PCI
-	airo_print_info("", "Probing for PCI adapters");
-	i = pci_register_driver(&airo_driver);
-	airo_print_info("", "Finished probing for PCI adapters");
-
-	if (i) {
-		remove_proc_entry("driver/aironet", NULL);
-		return i;
-	}
-#endif
-
-	/* Always exit with success, as we are a library module
-	 * as well as a driver module
-	 */
-	return 0;
-}
-
-static void __exit airo_cleanup_module(void)
-{
-	struct airo_info *ai;
-	while (!list_empty(&airo_devices)) {
-		ai = list_entry(airo_devices.next, struct airo_info, dev_list);
-		airo_print_info(ai->dev->name, "Unregistering...");
-		stop_airo_card(ai->dev, 1);
-	}
-#ifdef CONFIG_PCI
-	pci_unregister_driver(&airo_driver);
-#endif
-	remove_proc_entry("driver/aironet", NULL);
-}
-
-/*
- * Initial Wireless Extension code for Aironet driver by :
- *	Jean Tourrilhes <jt@hpl.hp.com> - HPL - 17 November 00
- * Conversion to new driver API by :
- *	Jean Tourrilhes <jt@hpl.hp.com> - HPL - 26 March 02
- * Javier also did a good amount of work here, adding some new extensions
- * and fixing my code. Let's just say that without him this code just
- * would not work at all... - Jean II
- */
-
-static u8 airo_rssi_to_dbm (tdsRssiEntry *rssi_rid, u8 rssi)
-{
-	if (!rssi_rid)
-		return 0;
-
-	return (0x100 - rssi_rid[rssi].rssidBm);
-}
-
-static u8 airo_dbm_to_pct (tdsRssiEntry *rssi_rid, u8 dbm)
-{
-	int i;
-
-	if (!rssi_rid)
-		return 0;
-
-	for (i = 0; i < 256; i++)
-		if (rssi_rid[i].rssidBm == dbm)
-			return rssi_rid[i].rssipct;
-
-	return 0;
-}
-
-
-static int airo_get_quality (StatusRid *status_rid, CapabilityRid *cap_rid)
-{
-	int quality = 0;
-	u16 sq;
-
-	if ((status_rid->mode & cpu_to_le16(0x3f)) != cpu_to_le16(0x3f))
-		return 0;
-
-	if (!(cap_rid->hardCap & cpu_to_le16(8)))
-		return 0;
-
-	sq = le16_to_cpu(status_rid->signalQuality);
-	if (memcmp(cap_rid->prodName, "350", 3))
-		if (sq > 0x20)
-			quality = 0;
-		else
-			quality = 0x20 - sq;
-	else
-		if (sq > 0xb0)
-			quality = 0;
-		else if (sq < 0x10)
-			quality = 0xa0;
-		else
-			quality = 0xb0 - sq;
-	return quality;
-}
-
-#define airo_get_max_quality(cap_rid) (memcmp((cap_rid)->prodName, "350", 3) ? 0x20 : 0xa0)
-#define airo_get_avg_quality(cap_rid) (memcmp((cap_rid)->prodName, "350", 3) ? 0x10 : 0x50)
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get protocol name
- */
-static int airo_get_name(struct net_device *dev,
-			 struct iw_request_info *info,
-			 union iwreq_data *cwrq,
-			 char *extra)
-{
-	strcpy(cwrq->name, "IEEE 802.11-DS");
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : set frequency
- */
-static int airo_set_freq(struct net_device *dev,
-			 struct iw_request_info *info,
-			 union iwreq_data *wrqu,
-			 char *extra)
-{
-	struct iw_freq *fwrq = &wrqu->freq;
-	struct airo_info *local = dev->ml_priv;
-	int rc = -EINPROGRESS;		/* Call commit handler */
-
-	/* If setting by frequency, convert to a channel */
-	if (fwrq->e == 1) {
-		int f = fwrq->m / 100000;
-
-		/* Hack to fall through... */
-		fwrq->e = 0;
-		fwrq->m = ieee80211_frequency_to_channel(f);
-	}
-	/* Setting by channel number */
-	if (fwrq->m < 0 || fwrq->m > 1000 || fwrq->e > 0)
-		rc = -EOPNOTSUPP;
-	else {
-		int channel = fwrq->m;
-		/* We should do a better check than that,
-		 * based on the card capability !!! */
-		if ((channel < 1) || (channel > 14)) {
-			airo_print_dbg(dev->name, "New channel value of %d is invalid!",
-				fwrq->m);
-			rc = -EINVAL;
-		} else {
-			readConfigRid(local, 1);
-			/* Yes ! We can set it !!! */
-			local->config.channelSet = cpu_to_le16(channel);
-			set_bit (FLAG_COMMIT, &local->flags);
-		}
-	}
-	return rc;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get frequency
- */
-static int airo_get_freq(struct net_device *dev,
-			 struct iw_request_info *info,
-			 union iwreq_data *wrqu,
-			 char *extra)
-{
-	struct iw_freq *fwrq = &wrqu->freq;
-	struct airo_info *local = dev->ml_priv;
-	StatusRid status_rid;		/* Card status info */
-	int ch;
-
-	readConfigRid(local, 1);
-	if ((local->config.opmode & MODE_CFG_MASK) == MODE_STA_ESS)
-		status_rid.channel = local->config.channelSet;
-	else
-		readStatusRid(local, &status_rid, 1);
-
-	ch = le16_to_cpu(status_rid.channel);
-	if ((ch > 0) && (ch < 15)) {
-		fwrq->m = 100000 *
-			ieee80211_channel_to_frequency(ch, NL80211_BAND_2GHZ);
-		fwrq->e = 1;
-	} else {
-		fwrq->m = ch;
-		fwrq->e = 0;
-	}
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : set ESSID
- */
-static int airo_set_essid(struct net_device *dev,
-			  struct iw_request_info *info,
-			  union iwreq_data *wrqu,
-			  char *extra)
-{
-	struct iw_point *dwrq = &wrqu->essid;
-	struct airo_info *local = dev->ml_priv;
-	SsidRid SSID_rid;		/* SSIDs */
-
-	/* Reload the list of current SSID */
-	readSsidRid(local, &SSID_rid);
-
-	/* Check if we asked for `any' */
-	if (dwrq->flags == 0) {
-		/* Just send an empty SSID list */
-		memset(&SSID_rid, 0, sizeof(SSID_rid));
-	} else {
-		unsigned index = (dwrq->flags & IW_ENCODE_INDEX) - 1;
-
-		/* Check the size of the string */
-		if (dwrq->length > IW_ESSID_MAX_SIZE)
-			return -E2BIG ;
-
-		/* Check if index is valid */
-		if (index >= ARRAY_SIZE(SSID_rid.ssids))
-			return -EINVAL;
-
-		/* Set the SSID */
-		memset(SSID_rid.ssids[index].ssid, 0,
-		       sizeof(SSID_rid.ssids[index].ssid));
-		memcpy(SSID_rid.ssids[index].ssid, extra, dwrq->length);
-		SSID_rid.ssids[index].len = cpu_to_le16(dwrq->length);
-	}
-	SSID_rid.len = cpu_to_le16(sizeof(SSID_rid));
-	/* Write it to the card */
-	disable_MAC(local, 1);
-	writeSsidRid(local, &SSID_rid, 1);
-	enable_MAC(local, 1);
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get ESSID
- */
-static int airo_get_essid(struct net_device *dev,
-			  struct iw_request_info *info,
-			  union iwreq_data *wrqu,
-			  char *extra)
-{
-	struct iw_point *dwrq = &wrqu->essid;
-	struct airo_info *local = dev->ml_priv;
-	StatusRid status_rid;		/* Card status info */
-
-	readStatusRid(local, &status_rid, 1);
-
-	/* Note : if dwrq->flags != 0, we should
-	 * get the relevant SSID from the SSID list... */
-
-	/* Get the current SSID */
-	memcpy(extra, status_rid.SSID, le16_to_cpu(status_rid.SSIDlen));
-	/* If none, we may want to get the one that was set */
-
-	/* Push it out ! */
-	dwrq->length = le16_to_cpu(status_rid.SSIDlen);
-	dwrq->flags = 1; /* active */
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : set AP address
- */
-static int airo_set_wap(struct net_device *dev,
-			struct iw_request_info *info,
-			union iwreq_data *wrqu,
-			char *extra)
-{
-	struct sockaddr *awrq = &wrqu->ap_addr;
-	struct airo_info *local = dev->ml_priv;
-	Cmd cmd;
-	Resp rsp;
-	APListRid *APList_rid = &local->APList;
-
-	if (awrq->sa_family != ARPHRD_ETHER)
-		return -EINVAL;
-	else if (is_broadcast_ether_addr(awrq->sa_data) ||
-		 is_zero_ether_addr(awrq->sa_data)) {
-		memset(&cmd, 0, sizeof(cmd));
-		cmd.cmd = CMD_LOSE_SYNC;
-		if (down_interruptible(&local->sem))
-			return -ERESTARTSYS;
-		issuecommand(local, &cmd, &rsp, true);
-		up(&local->sem);
-	} else {
-		memset(APList_rid, 0, sizeof(*APList_rid));
-		APList_rid->len = cpu_to_le16(sizeof(*APList_rid));
-		memcpy(APList_rid->ap[0], awrq->sa_data, ETH_ALEN);
-		disable_MAC(local, 1);
-		writeAPListRid(local, APList_rid, 1);
-		enable_MAC(local, 1);
-	}
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get AP address
- */
-static int airo_get_wap(struct net_device *dev,
-			struct iw_request_info *info,
-			union iwreq_data *wrqu,
-			char *extra)
-{
-	struct sockaddr *awrq = &wrqu->ap_addr;
-	struct airo_info *local = dev->ml_priv;
-	StatusRid status_rid;		/* Card status info */
-
-	readStatusRid(local, &status_rid, 1);
-
-	/* Tentative. This seems to work, wow, I'm lucky !!! */
-	memcpy(awrq->sa_data, status_rid.bssid[0], ETH_ALEN);
-	awrq->sa_family = ARPHRD_ETHER;
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : set Nickname
- */
-static int airo_set_nick(struct net_device *dev,
-			 struct iw_request_info *info,
-			 union iwreq_data *wrqu,
-			 char *extra)
-{
-	struct iw_point *dwrq = &wrqu->data;
-	struct airo_info *local = dev->ml_priv;
-
-	/* Check the size of the string */
-	if (dwrq->length > 16) {
-		return -E2BIG;
-	}
-	readConfigRid(local, 1);
-	memset(local->config.nodeName, 0, sizeof(local->config.nodeName));
-	memcpy(local->config.nodeName, extra, dwrq->length);
-	set_bit (FLAG_COMMIT, &local->flags);
-
-	return -EINPROGRESS;		/* Call commit handler */
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get Nickname
- */
-static int airo_get_nick(struct net_device *dev,
-			 struct iw_request_info *info,
-			 union iwreq_data *wrqu,
-			 char *extra)
-{
-	struct iw_point *dwrq = &wrqu->data;
-	struct airo_info *local = dev->ml_priv;
-
-	readConfigRid(local, 1);
-	strncpy(extra, local->config.nodeName, 16);
-	extra[16] = '\0';
-	dwrq->length = strlen(extra);
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : set Bit-Rate
- */
-static int airo_set_rate(struct net_device *dev,
-			 struct iw_request_info *info,
-			 union iwreq_data *wrqu,
-			 char *extra)
-{
-	struct iw_param *vwrq = &wrqu->bitrate;
-	struct airo_info *local = dev->ml_priv;
-	CapabilityRid cap_rid;		/* Card capability info */
-	u8	brate = 0;
-	int	i;
-
-	/* First : get a valid bit rate value */
-	readCapabilityRid(local, &cap_rid, 1);
-
-	/* Which type of value ? */
-	if ((vwrq->value < 8) && (vwrq->value >= 0)) {
-		/* Setting by rate index */
-		/* Find value in the magic rate table */
-		brate = cap_rid.supportedRates[vwrq->value];
-	} else {
-		/* Setting by frequency value */
-		u8	normvalue = (u8) (vwrq->value/500000);
-
-		/* Check if rate is valid */
-		for (i = 0 ; i < 8 ; i++) {
-			if (normvalue == cap_rid.supportedRates[i]) {
-				brate = normvalue;
-				break;
-			}
-		}
-	}
-	/* -1 designed the max rate (mostly auto mode) */
-	if (vwrq->value == -1) {
-		/* Get the highest available rate */
-		for (i = 0 ; i < 8 ; i++) {
-			if (cap_rid.supportedRates[i] == 0)
-				break;
-		}
-		if (i != 0)
-			brate = cap_rid.supportedRates[i - 1];
-	}
-	/* Check that it is valid */
-	if (brate == 0) {
-		return -EINVAL;
-	}
-
-	readConfigRid(local, 1);
-	/* Now, check if we want a fixed or auto value */
-	if (vwrq->fixed == 0) {
-		/* Fill all the rates up to this max rate */
-		memset(local->config.rates, 0, 8);
-		for (i = 0 ; i < 8 ; i++) {
-			local->config.rates[i] = cap_rid.supportedRates[i];
-			if (local->config.rates[i] == brate)
-				break;
-		}
-	} else {
-		/* Fixed mode */
-		/* One rate, fixed */
-		memset(local->config.rates, 0, 8);
-		local->config.rates[0] = brate;
-	}
-	set_bit (FLAG_COMMIT, &local->flags);
-
-	return -EINPROGRESS;		/* Call commit handler */
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get Bit-Rate
- */
-static int airo_get_rate(struct net_device *dev,
-			 struct iw_request_info *info,
-			 union iwreq_data *wrqu,
-			 char *extra)
-{
-	struct iw_param *vwrq = &wrqu->bitrate;
-	struct airo_info *local = dev->ml_priv;
-	StatusRid status_rid;		/* Card status info */
-	int ret;
-
-	ret = readStatusRid(local, &status_rid, 1);
-	if (ret)
-		return -EBUSY;
-
-	vwrq->value = le16_to_cpu(status_rid.currentXmitRate) * 500000;
-	/* If more than one rate, set auto */
-	readConfigRid(local, 1);
-	vwrq->fixed = (local->config.rates[1] == 0);
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : set RTS threshold
- */
-static int airo_set_rts(struct net_device *dev,
-			struct iw_request_info *info,
-			union iwreq_data *wrqu,
-			char *extra)
-{
-	struct iw_param *vwrq = &wrqu->rts;
-	struct airo_info *local = dev->ml_priv;
-	int rthr = vwrq->value;
-
-	if (vwrq->disabled)
-		rthr = AIRO_DEF_MTU;
-	if ((rthr < 0) || (rthr > AIRO_DEF_MTU)) {
-		return -EINVAL;
-	}
-	readConfigRid(local, 1);
-	local->config.rtsThres = cpu_to_le16(rthr);
-	set_bit (FLAG_COMMIT, &local->flags);
-
-	return -EINPROGRESS;		/* Call commit handler */
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get RTS threshold
- */
-static int airo_get_rts(struct net_device *dev,
-			struct iw_request_info *info,
-			union iwreq_data *wrqu,
-			char *extra)
-{
-	struct iw_param *vwrq = &wrqu->rts;
-	struct airo_info *local = dev->ml_priv;
-
-	readConfigRid(local, 1);
-	vwrq->value = le16_to_cpu(local->config.rtsThres);
-	vwrq->disabled = (vwrq->value >= AIRO_DEF_MTU);
-	vwrq->fixed = 1;
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : set Fragmentation threshold
- */
-static int airo_set_frag(struct net_device *dev,
-			 struct iw_request_info *info,
-			 union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *vwrq = &wrqu->frag;
-	struct airo_info *local = dev->ml_priv;
-	int fthr = vwrq->value;
-
-	if (vwrq->disabled)
-		fthr = AIRO_DEF_MTU;
-	if ((fthr < 256) || (fthr > AIRO_DEF_MTU)) {
-		return -EINVAL;
-	}
-	fthr &= ~0x1;	/* Get an even value - is it really needed ??? */
-	readConfigRid(local, 1);
-	local->config.fragThresh = cpu_to_le16(fthr);
-	set_bit (FLAG_COMMIT, &local->flags);
-
-	return -EINPROGRESS;		/* Call commit handler */
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get Fragmentation threshold
- */
-static int airo_get_frag(struct net_device *dev,
-			 struct iw_request_info *info,
-			 union iwreq_data *wrqu,
-			 char *extra)
-{
-	struct iw_param *vwrq = &wrqu->frag;
-	struct airo_info *local = dev->ml_priv;
-
-	readConfigRid(local, 1);
-	vwrq->value = le16_to_cpu(local->config.fragThresh);
-	vwrq->disabled = (vwrq->value >= AIRO_DEF_MTU);
-	vwrq->fixed = 1;
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : set Mode of Operation
- */
-static int airo_set_mode(struct net_device *dev,
-			 struct iw_request_info *info,
-			 union iwreq_data *uwrq,
-			 char *extra)
-{
-	__u32 mode = uwrq->mode;
-	struct airo_info *local = dev->ml_priv;
-	int reset = 0;
-
-	readConfigRid(local, 1);
-	if (sniffing_mode(local))
-		reset = 1;
-
-	switch (mode) {
-		case IW_MODE_ADHOC:
-			local->config.opmode &= ~MODE_CFG_MASK;
-			local->config.opmode |= MODE_STA_IBSS;
-			local->config.rmode &= ~RXMODE_FULL_MASK;
-			local->config.scanMode = SCANMODE_ACTIVE;
-			clear_bit (FLAG_802_11, &local->flags);
-			break;
-		case IW_MODE_INFRA:
-			local->config.opmode &= ~MODE_CFG_MASK;
-			local->config.opmode |= MODE_STA_ESS;
-			local->config.rmode &= ~RXMODE_FULL_MASK;
-			local->config.scanMode = SCANMODE_ACTIVE;
-			clear_bit (FLAG_802_11, &local->flags);
-			break;
-		case IW_MODE_MASTER:
-			local->config.opmode &= ~MODE_CFG_MASK;
-			local->config.opmode |= MODE_AP;
-			local->config.rmode &= ~RXMODE_FULL_MASK;
-			local->config.scanMode = SCANMODE_ACTIVE;
-			clear_bit (FLAG_802_11, &local->flags);
-			break;
-		case IW_MODE_REPEAT:
-			local->config.opmode &= ~MODE_CFG_MASK;
-			local->config.opmode |= MODE_AP_RPTR;
-			local->config.rmode &= ~RXMODE_FULL_MASK;
-			local->config.scanMode = SCANMODE_ACTIVE;
-			clear_bit (FLAG_802_11, &local->flags);
-			break;
-		case IW_MODE_MONITOR:
-			local->config.opmode &= ~MODE_CFG_MASK;
-			local->config.opmode |= MODE_STA_ESS;
-			local->config.rmode &= ~RXMODE_FULL_MASK;
-			local->config.rmode |= RXMODE_RFMON | RXMODE_DISABLE_802_3_HEADER;
-			local->config.scanMode = SCANMODE_PASSIVE;
-			set_bit (FLAG_802_11, &local->flags);
-			break;
-		default:
-			return -EINVAL;
-	}
-	if (reset)
-		set_bit (FLAG_RESET, &local->flags);
-	set_bit (FLAG_COMMIT, &local->flags);
-
-	return -EINPROGRESS;		/* Call commit handler */
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get Mode of Operation
- */
-static int airo_get_mode(struct net_device *dev,
-			 struct iw_request_info *info,
-			 union iwreq_data *uwrq,
-			 char *extra)
-{
-	struct airo_info *local = dev->ml_priv;
-
-	readConfigRid(local, 1);
-	/* If not managed, assume it's ad-hoc */
-	switch (local->config.opmode & MODE_CFG_MASK) {
-		case MODE_STA_ESS:
-			uwrq->mode = IW_MODE_INFRA;
-			break;
-		case MODE_AP:
-			uwrq->mode = IW_MODE_MASTER;
-			break;
-		case MODE_AP_RPTR:
-			uwrq->mode = IW_MODE_REPEAT;
-			break;
-		default:
-			uwrq->mode = IW_MODE_ADHOC;
-	}
-
-	return 0;
-}
-
-static inline int valid_index(struct airo_info *ai, int index)
-{
-	return (index >= 0) && (index <= ai->max_wep_idx);
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : set Encryption Key
- */
-static int airo_set_encode(struct net_device *dev,
-			   struct iw_request_info *info,
-			   union iwreq_data *wrqu,
-			   char *extra)
-{
-	struct iw_point *dwrq = &wrqu->encoding;
-	struct airo_info *local = dev->ml_priv;
-	int perm = (dwrq->flags & IW_ENCODE_TEMP ? 0 : 1);
-	__le16 currentAuthType = local->config.authType;
-	int rc = 0;
-
-	if (!local->wep_capable)
-		return -EOPNOTSUPP;
-
-	readConfigRid(local, 1);
-
-	/* Basic checking: do we have a key to set ?
-	 * Note : with the new API, it's impossible to get a NULL pointer.
-	 * Therefore, we need to check a key size == 0 instead.
-	 * New version of iwconfig properly set the IW_ENCODE_NOKEY flag
-	 * when no key is present (only change flags), but older versions
-	 * don't do it. - Jean II */
-	if (dwrq->length > 0) {
-		wep_key_t key;
-		int index = (dwrq->flags & IW_ENCODE_INDEX) - 1;
-		int current_index;
-
-		/* Check the size of the key */
-		if (dwrq->length > MAX_KEY_SIZE) {
-			return -EINVAL;
-		}
-
-		current_index = get_wep_tx_idx(local);
-		if (current_index < 0)
-			current_index = 0;
-
-		/* Check the index (none -> use current) */
-		if (!valid_index(local, index))
-			index = current_index;
-
-		/* Set the length */
-		if (dwrq->length > MIN_KEY_SIZE)
-			key.len = MAX_KEY_SIZE;
-		else
-			key.len = MIN_KEY_SIZE;
-		/* Check if the key is not marked as invalid */
-		if (!(dwrq->flags & IW_ENCODE_NOKEY)) {
-			/* Cleanup */
-			memset(key.key, 0, MAX_KEY_SIZE);
-			/* Copy the key in the driver */
-			memcpy(key.key, extra, dwrq->length);
-			/* Send the key to the card */
-			rc = set_wep_key(local, index, key.key, key.len, perm, 1);
-			if (rc < 0) {
-				airo_print_err(local->dev->name, "failed to set"
-				               " WEP key at index %d: %d.",
-				               index, rc);
-				return rc;
-			}
-		}
-		/* WE specify that if a valid key is set, encryption
-		 * should be enabled (user may turn it off later)
-		 * This is also how "iwconfig ethX key on" works */
-		if ((index == current_index) && (key.len > 0) &&
-		   (local->config.authType == AUTH_OPEN))
-			set_auth_type(local, AUTH_ENCRYPT);
-	} else {
-		/* Do we want to just set the transmit key index ? */
-		int index = (dwrq->flags & IW_ENCODE_INDEX) - 1;
-		if (valid_index(local, index)) {
-			rc = set_wep_tx_idx(local, index, perm, 1);
-			if (rc < 0) {
-				airo_print_err(local->dev->name, "failed to set"
-				               " WEP transmit index to %d: %d.",
-				               index, rc);
-				return rc;
-			}
-		} else {
-			/* Don't complain if only change the mode */
-			if (!(dwrq->flags & IW_ENCODE_MODE))
-				return -EINVAL;
-		}
-	}
-	/* Read the flags */
-	if (dwrq->flags & IW_ENCODE_DISABLED)
-		set_auth_type(local, AUTH_OPEN);	/* disable encryption */
-	if (dwrq->flags & IW_ENCODE_RESTRICTED)
-		set_auth_type(local, AUTH_SHAREDKEY);	/* Only Both */
-	if (dwrq->flags & IW_ENCODE_OPEN)
-		set_auth_type(local, AUTH_ENCRYPT);	/* Only Wep */
-	/* Commit the changes to flags if needed */
-	if (local->config.authType != currentAuthType)
-		set_bit (FLAG_COMMIT, &local->flags);
-	return -EINPROGRESS;		/* Call commit handler */
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get Encryption Key
- */
-static int airo_get_encode(struct net_device *dev,
-			   struct iw_request_info *info,
-			   union iwreq_data *wrqu,
-			   char *extra)
-{
-	struct iw_point *dwrq = &wrqu->encoding;
-	struct airo_info *local = dev->ml_priv;
-	int index = (dwrq->flags & IW_ENCODE_INDEX) - 1;
-	int wep_key_len;
-	u8 buf[16];
-
-	if (!local->wep_capable)
-		return -EOPNOTSUPP;
-
-	readConfigRid(local, 1);
-
-	/* Check encryption mode */
-	switch(local->config.authType)	{
-		case AUTH_ENCRYPT:
-			dwrq->flags = IW_ENCODE_OPEN;
-			break;
-		case AUTH_SHAREDKEY:
-			dwrq->flags = IW_ENCODE_RESTRICTED;
-			break;
-		default:
-		case AUTH_OPEN:
-			dwrq->flags = IW_ENCODE_DISABLED;
-			break;
-	}
-	/* We can't return the key, so set the proper flag and return zero */
-	dwrq->flags |= IW_ENCODE_NOKEY;
-	memset(extra, 0, 16);
-
-	/* Which key do we want ? -1 -> tx index */
-	if (!valid_index(local, index)) {
-		index = get_wep_tx_idx(local);
-		if (index < 0)
-			index = 0;
-	}
-	dwrq->flags |= index + 1;
-
-	/* Copy the key to the user buffer */
-	wep_key_len = get_wep_key(local, index, &buf[0], sizeof(buf));
-	if (wep_key_len < 0) {
-		dwrq->length = 0;
-	} else {
-		dwrq->length = wep_key_len;
-		memcpy(extra, buf, dwrq->length);
-	}
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : set extended Encryption parameters
- */
-static int airo_set_encodeext(struct net_device *dev,
-			   struct iw_request_info *info,
-			    union iwreq_data *wrqu,
-			    char *extra)
-{
-	struct airo_info *local = dev->ml_priv;
-	struct iw_point *encoding = &wrqu->encoding;
-	struct iw_encode_ext *ext = (struct iw_encode_ext *)extra;
-	int perm = (encoding->flags & IW_ENCODE_TEMP ? 0 : 1);
-	__le16 currentAuthType = local->config.authType;
-	int idx, key_len, alg = ext->alg, set_key = 1, rc;
-	wep_key_t key;
-
-	if (!local->wep_capable)
-		return -EOPNOTSUPP;
-
-	readConfigRid(local, 1);
-
-	/* Determine and validate the key index */
-	idx = encoding->flags & IW_ENCODE_INDEX;
-	if (idx) {
-		if (!valid_index(local, idx - 1))
-			return -EINVAL;
-		idx--;
-	} else {
-		idx = get_wep_tx_idx(local);
-		if (idx < 0)
-			idx = 0;
-	}
-
-	if (encoding->flags & IW_ENCODE_DISABLED)
-		alg = IW_ENCODE_ALG_NONE;
-
-	if (ext->ext_flags & IW_ENCODE_EXT_SET_TX_KEY) {
-		/* Only set transmit key index here, actual
-		 * key is set below if needed.
-		 */
-		rc = set_wep_tx_idx(local, idx, perm, 1);
-		if (rc < 0) {
-			airo_print_err(local->dev->name, "failed to set "
-			               "WEP transmit index to %d: %d.",
-			               idx, rc);
-			return rc;
-		}
-		set_key = ext->key_len > 0 ? 1 : 0;
-	}
-
-	if (set_key) {
-		/* Set the requested key first */
-		memset(key.key, 0, MAX_KEY_SIZE);
-		switch (alg) {
-		case IW_ENCODE_ALG_NONE:
-			key.len = 0;
-			break;
-		case IW_ENCODE_ALG_WEP:
-			if (ext->key_len > MIN_KEY_SIZE) {
-				key.len = MAX_KEY_SIZE;
-			} else if (ext->key_len > 0) {
-				key.len = MIN_KEY_SIZE;
-			} else {
-				return -EINVAL;
-			}
-			key_len = min (ext->key_len, key.len);
-			memcpy(key.key, ext->key, key_len);
-			break;
-		default:
-			return -EINVAL;
-		}
-		if (key.len == 0) {
-			rc = set_wep_tx_idx(local, idx, perm, 1);
-			if (rc < 0) {
-				airo_print_err(local->dev->name,
-					       "failed to set WEP transmit index to %d: %d.",
-					       idx, rc);
-				return rc;
-			}
-		} else {
-			rc = set_wep_key(local, idx, key.key, key.len, perm, 1);
-			if (rc < 0) {
-				airo_print_err(local->dev->name,
-					       "failed to set WEP key at index %d: %d.",
-					       idx, rc);
-				return rc;
-			}
-		}
-	}
-
-	/* Read the flags */
-	if (encoding->flags & IW_ENCODE_DISABLED)
-		set_auth_type(local, AUTH_OPEN);	/* disable encryption */
-	if (encoding->flags & IW_ENCODE_RESTRICTED)
-		set_auth_type(local, AUTH_SHAREDKEY);	/* Only Both */
-	if (encoding->flags & IW_ENCODE_OPEN)
-		set_auth_type(local, AUTH_ENCRYPT);
-	/* Commit the changes to flags if needed */
-	if (local->config.authType != currentAuthType)
-		set_bit (FLAG_COMMIT, &local->flags);
-
-	return -EINPROGRESS;
-}
-
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get extended Encryption parameters
- */
-static int airo_get_encodeext(struct net_device *dev,
-			    struct iw_request_info *info,
-			    union iwreq_data *wrqu,
-			    char *extra)
-{
-	struct airo_info *local = dev->ml_priv;
-	struct iw_point *encoding = &wrqu->encoding;
-	struct iw_encode_ext *ext = (struct iw_encode_ext *)extra;
-	int idx, max_key_len, wep_key_len;
-	u8 buf[16];
-
-	if (!local->wep_capable)
-		return -EOPNOTSUPP;
-
-	readConfigRid(local, 1);
-
-	max_key_len = encoding->length - sizeof(*ext);
-	if (max_key_len < 0)
-		return -EINVAL;
-
-	idx = encoding->flags & IW_ENCODE_INDEX;
-	if (idx) {
-		if (!valid_index(local, idx - 1))
-			return -EINVAL;
-		idx--;
-	} else {
-		idx = get_wep_tx_idx(local);
-		if (idx < 0)
-			idx = 0;
-	}
-
-	encoding->flags = idx + 1;
-	memset(ext, 0, sizeof(*ext));
-
-	/* Check encryption mode */
-	switch(local->config.authType) {
-		case AUTH_ENCRYPT:
-			encoding->flags = IW_ENCODE_ALG_WEP | IW_ENCODE_ENABLED;
-			break;
-		case AUTH_SHAREDKEY:
-			encoding->flags = IW_ENCODE_ALG_WEP | IW_ENCODE_ENABLED;
-			break;
-		default:
-		case AUTH_OPEN:
-			encoding->flags = IW_ENCODE_ALG_NONE | IW_ENCODE_DISABLED;
-			break;
-	}
-	/* We can't return the key, so set the proper flag and return zero */
-	encoding->flags |= IW_ENCODE_NOKEY;
-	memset(extra, 0, 16);
-
-	/* Copy the key to the user buffer */
-	wep_key_len = get_wep_key(local, idx, &buf[0], sizeof(buf));
-	if (wep_key_len < 0) {
-		ext->key_len = 0;
-	} else {
-		ext->key_len = wep_key_len;
-		memcpy(extra, buf, ext->key_len);
-	}
-
-	return 0;
-}
-
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : set extended authentication parameters
- */
-static int airo_set_auth(struct net_device *dev,
-			       struct iw_request_info *info,
-			       union iwreq_data *wrqu, char *extra)
-{
-	struct airo_info *local = dev->ml_priv;
-	struct iw_param *param = &wrqu->param;
-	__le16 currentAuthType = local->config.authType;
-
-	switch (param->flags & IW_AUTH_INDEX) {
-	case IW_AUTH_WPA_VERSION:
-	case IW_AUTH_CIPHER_PAIRWISE:
-	case IW_AUTH_CIPHER_GROUP:
-	case IW_AUTH_KEY_MGMT:
-	case IW_AUTH_RX_UNENCRYPTED_EAPOL:
-	case IW_AUTH_PRIVACY_INVOKED:
-		/*
-		 * airo does not use these parameters
-		 */
-		break;
-
-	case IW_AUTH_DROP_UNENCRYPTED:
-		if (param->value) {
-			/* Only change auth type if unencrypted */
-			if (currentAuthType == AUTH_OPEN)
-				set_auth_type(local, AUTH_ENCRYPT);
-		} else {
-			set_auth_type(local, AUTH_OPEN);
-		}
-
-		/* Commit the changes to flags if needed */
-		if (local->config.authType != currentAuthType)
-			set_bit (FLAG_COMMIT, &local->flags);
-		break;
-
-	case IW_AUTH_80211_AUTH_ALG: {
-			if (param->value & IW_AUTH_ALG_SHARED_KEY) {
-				set_auth_type(local, AUTH_SHAREDKEY);
-			} else if (param->value & IW_AUTH_ALG_OPEN_SYSTEM) {
-				/* We don't know here if WEP open system or
-				 * unencrypted mode was requested - so use the
-				 * last mode (of these two) used last time
-				 */
-				set_auth_type(local, local->last_auth);
-			} else
-				return -EINVAL;
-
-			/* Commit the changes to flags if needed */
-			if (local->config.authType != currentAuthType)
-				set_bit (FLAG_COMMIT, &local->flags);
-			break;
-		}
-
-	case IW_AUTH_WPA_ENABLED:
-		/* Silently accept disable of WPA */
-		if (param->value > 0)
-			return -EOPNOTSUPP;
-		break;
-
-	default:
-		return -EOPNOTSUPP;
-	}
-	return -EINPROGRESS;
-}
-
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get extended authentication parameters
- */
-static int airo_get_auth(struct net_device *dev,
-			       struct iw_request_info *info,
-			       union iwreq_data *wrqu, char *extra)
-{
-	struct airo_info *local = dev->ml_priv;
-	struct iw_param *param = &wrqu->param;
-	__le16 currentAuthType = local->config.authType;
-
-	switch (param->flags & IW_AUTH_INDEX) {
-	case IW_AUTH_DROP_UNENCRYPTED:
-		switch (currentAuthType) {
-		case AUTH_SHAREDKEY:
-		case AUTH_ENCRYPT:
-			param->value = 1;
-			break;
-		default:
-			param->value = 0;
-			break;
-		}
-		break;
-
-	case IW_AUTH_80211_AUTH_ALG:
-		switch (currentAuthType) {
-		case AUTH_SHAREDKEY:
-			param->value = IW_AUTH_ALG_SHARED_KEY;
-			break;
-		case AUTH_ENCRYPT:
-		default:
-			param->value = IW_AUTH_ALG_OPEN_SYSTEM;
-			break;
-		}
-		break;
-
-	case IW_AUTH_WPA_ENABLED:
-		param->value = 0;
-		break;
-
-	default:
-		return -EOPNOTSUPP;
-	}
-	return 0;
-}
-
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : set Tx-Power
- */
-static int airo_set_txpow(struct net_device *dev,
-			  struct iw_request_info *info,
-			  union iwreq_data *wrqu,
-			  char *extra)
-{
-	struct iw_param *vwrq = &wrqu->txpower;
-	struct airo_info *local = dev->ml_priv;
-	CapabilityRid cap_rid;		/* Card capability info */
-	int i;
-	int rc = -EINVAL;
-	__le16 v = cpu_to_le16(vwrq->value);
-
-	readCapabilityRid(local, &cap_rid, 1);
-
-	if (vwrq->disabled) {
-		set_bit (FLAG_RADIO_OFF, &local->flags);
-		set_bit (FLAG_COMMIT, &local->flags);
-		return -EINPROGRESS;		/* Call commit handler */
-	}
-	if (vwrq->flags != IW_TXPOW_MWATT) {
-		return -EINVAL;
-	}
-	clear_bit (FLAG_RADIO_OFF, &local->flags);
-	for (i = 0; i < 8 && cap_rid.txPowerLevels[i]; i++)
-		if (v == cap_rid.txPowerLevels[i]) {
-			readConfigRid(local, 1);
-			local->config.txPower = v;
-			set_bit (FLAG_COMMIT, &local->flags);
-			rc = -EINPROGRESS;	/* Call commit handler */
-			break;
-		}
-	return rc;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get Tx-Power
- */
-static int airo_get_txpow(struct net_device *dev,
-			  struct iw_request_info *info,
-			  union iwreq_data *wrqu,
-			  char *extra)
-{
-	struct iw_param *vwrq = &wrqu->txpower;
-	struct airo_info *local = dev->ml_priv;
-
-	readConfigRid(local, 1);
-	vwrq->value = le16_to_cpu(local->config.txPower);
-	vwrq->fixed = 1;	/* No power control */
-	vwrq->disabled = test_bit(FLAG_RADIO_OFF, &local->flags);
-	vwrq->flags = IW_TXPOW_MWATT;
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : set Retry limits
- */
-static int airo_set_retry(struct net_device *dev,
-			  struct iw_request_info *info,
-			  union iwreq_data *wrqu,
-			  char *extra)
-{
-	struct iw_param *vwrq = &wrqu->retry;
-	struct airo_info *local = dev->ml_priv;
-	int rc = -EINVAL;
-
-	if (vwrq->disabled) {
-		return -EINVAL;
-	}
-	readConfigRid(local, 1);
-	if (vwrq->flags & IW_RETRY_LIMIT) {
-		__le16 v = cpu_to_le16(vwrq->value);
-		if (vwrq->flags & IW_RETRY_LONG)
-			local->config.longRetryLimit = v;
-		else if (vwrq->flags & IW_RETRY_SHORT)
-			local->config.shortRetryLimit = v;
-		else {
-			/* No modifier : set both */
-			local->config.longRetryLimit = v;
-			local->config.shortRetryLimit = v;
-		}
-		set_bit (FLAG_COMMIT, &local->flags);
-		rc = -EINPROGRESS;		/* Call commit handler */
-	}
-	if (vwrq->flags & IW_RETRY_LIFETIME) {
-		local->config.txLifetime = cpu_to_le16(vwrq->value / 1024);
-		set_bit (FLAG_COMMIT, &local->flags);
-		rc = -EINPROGRESS;		/* Call commit handler */
-	}
-	return rc;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get Retry limits
- */
-static int airo_get_retry(struct net_device *dev,
-			  struct iw_request_info *info,
-			  union iwreq_data *wrqu,
-			  char *extra)
-{
-	struct iw_param *vwrq = &wrqu->retry;
-	struct airo_info *local = dev->ml_priv;
-
-	vwrq->disabled = 0;      /* Can't be disabled */
-
-	readConfigRid(local, 1);
-	/* Note : by default, display the min retry number */
-	if ((vwrq->flags & IW_RETRY_TYPE) == IW_RETRY_LIFETIME) {
-		vwrq->flags = IW_RETRY_LIFETIME;
-		vwrq->value = le16_to_cpu(local->config.txLifetime) * 1024;
-	} else if ((vwrq->flags & IW_RETRY_LONG)) {
-		vwrq->flags = IW_RETRY_LIMIT | IW_RETRY_LONG;
-		vwrq->value = le16_to_cpu(local->config.longRetryLimit);
-	} else {
-		vwrq->flags = IW_RETRY_LIMIT;
-		vwrq->value = le16_to_cpu(local->config.shortRetryLimit);
-		if (local->config.shortRetryLimit != local->config.longRetryLimit)
-			vwrq->flags |= IW_RETRY_SHORT;
-	}
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get range info
- */
-static int airo_get_range(struct net_device *dev,
-			  struct iw_request_info *info,
-			  union iwreq_data *wrqu,
-			  char *extra)
-{
-	struct iw_point *dwrq = &wrqu->data;
-	struct airo_info *local = dev->ml_priv;
-	struct iw_range *range = (struct iw_range *) extra;
-	CapabilityRid cap_rid;		/* Card capability info */
-	int		i;
-	int		k;
-
-	readCapabilityRid(local, &cap_rid, 1);
-
-	dwrq->length = sizeof(struct iw_range);
-	memset(range, 0, sizeof(*range));
-	range->min_nwid = 0x0000;
-	range->max_nwid = 0x0000;
-	range->num_channels = 14;
-	/* Should be based on cap_rid.country to give only
-	 * what the current card support */
-	k = 0;
-	for (i = 0; i < 14; i++) {
-		range->freq[k].i = i + 1; /* List index */
-		range->freq[k].m = 100000 *
-		     ieee80211_channel_to_frequency(i + 1, NL80211_BAND_2GHZ);
-		range->freq[k++].e = 1;	/* Values in MHz -> * 10^5 * 10 */
-	}
-	range->num_frequency = k;
-
-	range->sensitivity = 65535;
-
-	/* Hum... Should put the right values there */
-	if (local->rssi)
-		range->max_qual.qual = 100;	/* % */
-	else
-		range->max_qual.qual = airo_get_max_quality(&cap_rid);
-	range->max_qual.level = 0x100 - 120;	/* -120 dBm */
-	range->max_qual.noise = 0x100 - 120;	/* -120 dBm */
-
-	/* Experimental measurements - boundary 11/5.5 Mb/s */
-	/* Note : with or without the (local->rssi), results
-	 * are somewhat different. - Jean II */
-	if (local->rssi) {
-		range->avg_qual.qual = 50;		/* % */
-		range->avg_qual.level = 0x100 - 70;	/* -70 dBm */
-	} else {
-		range->avg_qual.qual = airo_get_avg_quality(&cap_rid);
-		range->avg_qual.level = 0x100 - 80;	/* -80 dBm */
-	}
-	range->avg_qual.noise = 0x100 - 85;		/* -85 dBm */
-
-	for (i = 0 ; i < 8 ; i++) {
-		range->bitrate[i] = cap_rid.supportedRates[i] * 500000;
-		if (range->bitrate[i] == 0)
-			break;
-	}
-	range->num_bitrates = i;
-
-	/* Set an indication of the max TCP throughput
-	 * in bit/s that we can expect using this interface.
-	 * May be use for QoS stuff... Jean II */
-	if (i > 2)
-		range->throughput = 5000 * 1000;
-	else
-		range->throughput = 1500 * 1000;
-
-	range->min_rts = 0;
-	range->max_rts = AIRO_DEF_MTU;
-	range->min_frag = 256;
-	range->max_frag = AIRO_DEF_MTU;
-
-	if (cap_rid.softCap & cpu_to_le16(2)) {
-		// WEP: RC4 40 bits
-		range->encoding_size[0] = 5;
-		// RC4 ~128 bits
-		if (cap_rid.softCap & cpu_to_le16(0x100)) {
-			range->encoding_size[1] = 13;
-			range->num_encoding_sizes = 2;
-		} else
-			range->num_encoding_sizes = 1;
-		range->max_encoding_tokens =
-			cap_rid.softCap & cpu_to_le16(0x80) ? 4 : 1;
-	} else {
-		range->num_encoding_sizes = 0;
-		range->max_encoding_tokens = 0;
-	}
-	range->min_pmp = 0;
-	range->max_pmp = 5000000;	/* 5 secs */
-	range->min_pmt = 0;
-	range->max_pmt = 65535 * 1024;	/* ??? */
-	range->pmp_flags = IW_POWER_PERIOD;
-	range->pmt_flags = IW_POWER_TIMEOUT;
-	range->pm_capa = IW_POWER_PERIOD | IW_POWER_TIMEOUT | IW_POWER_ALL_R;
-
-	/* Transmit Power - values are in mW */
-	for (i = 0 ; i < 8 ; i++) {
-		range->txpower[i] = le16_to_cpu(cap_rid.txPowerLevels[i]);
-		if (range->txpower[i] == 0)
-			break;
-	}
-	range->num_txpower = i;
-	range->txpower_capa = IW_TXPOW_MWATT;
-	range->we_version_source = 19;
-	range->we_version_compiled = WIRELESS_EXT;
-	range->retry_capa = IW_RETRY_LIMIT | IW_RETRY_LIFETIME;
-	range->retry_flags = IW_RETRY_LIMIT;
-	range->r_time_flags = IW_RETRY_LIFETIME;
-	range->min_retry = 1;
-	range->max_retry = 65535;
-	range->min_r_time = 1024;
-	range->max_r_time = 65535 * 1024;
-
-	/* Event capability (kernel + driver) */
-	range->event_capa[0] = (IW_EVENT_CAPA_K_0 |
-				IW_EVENT_CAPA_MASK(SIOCGIWTHRSPY) |
-				IW_EVENT_CAPA_MASK(SIOCGIWAP) |
-				IW_EVENT_CAPA_MASK(SIOCGIWSCAN));
-	range->event_capa[1] = IW_EVENT_CAPA_K_1;
-	range->event_capa[4] = IW_EVENT_CAPA_MASK(IWEVTXDROP);
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : set Power Management
- */
-static int airo_set_power(struct net_device *dev,
-			  struct iw_request_info *info,
-			  union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *vwrq = &wrqu->power;
-	struct airo_info *local = dev->ml_priv;
-
-	readConfigRid(local, 1);
-	if (vwrq->disabled) {
-		if (sniffing_mode(local))
-			return -EINVAL;
-		local->config.powerSaveMode = POWERSAVE_CAM;
-		local->config.rmode &= ~RXMODE_MASK;
-		local->config.rmode |= RXMODE_BC_MC_ADDR;
-		set_bit (FLAG_COMMIT, &local->flags);
-		return -EINPROGRESS;		/* Call commit handler */
-	}
-	if ((vwrq->flags & IW_POWER_TYPE) == IW_POWER_TIMEOUT) {
-		local->config.fastListenDelay = cpu_to_le16((vwrq->value + 500) / 1024);
-		local->config.powerSaveMode = POWERSAVE_PSPCAM;
-		set_bit (FLAG_COMMIT, &local->flags);
-	} else if ((vwrq->flags & IW_POWER_TYPE) == IW_POWER_PERIOD) {
-		local->config.fastListenInterval =
-		local->config.listenInterval =
-			cpu_to_le16((vwrq->value + 500) / 1024);
-		local->config.powerSaveMode = POWERSAVE_PSPCAM;
-		set_bit (FLAG_COMMIT, &local->flags);
-	}
-	switch (vwrq->flags & IW_POWER_MODE) {
-		case IW_POWER_UNICAST_R:
-			if (sniffing_mode(local))
-				return -EINVAL;
-			local->config.rmode &= ~RXMODE_MASK;
-			local->config.rmode |= RXMODE_ADDR;
-			set_bit (FLAG_COMMIT, &local->flags);
-			break;
-		case IW_POWER_ALL_R:
-			if (sniffing_mode(local))
-				return -EINVAL;
-			local->config.rmode &= ~RXMODE_MASK;
-			local->config.rmode |= RXMODE_BC_MC_ADDR;
-			set_bit (FLAG_COMMIT, &local->flags);
-			break;
-		case IW_POWER_ON:
-			/* This is broken, fixme ;-) */
-			break;
-		default:
-			return -EINVAL;
-	}
-	// Note : we may want to factor local->need_commit here
-	// Note2 : may also want to factor RXMODE_RFMON test
-	return -EINPROGRESS;		/* Call commit handler */
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get Power Management
- */
-static int airo_get_power(struct net_device *dev,
-			  struct iw_request_info *info,
-			  union iwreq_data *wrqu,
-			  char *extra)
-{
-	struct iw_param *vwrq = &wrqu->power;
-	struct airo_info *local = dev->ml_priv;
-	__le16 mode;
-
-	readConfigRid(local, 1);
-	mode = local->config.powerSaveMode;
-	if ((vwrq->disabled = (mode == POWERSAVE_CAM)))
-		return 0;
-	if ((vwrq->flags & IW_POWER_TYPE) == IW_POWER_TIMEOUT) {
-		vwrq->value = le16_to_cpu(local->config.fastListenDelay) * 1024;
-		vwrq->flags = IW_POWER_TIMEOUT;
-	} else {
-		vwrq->value = le16_to_cpu(local->config.fastListenInterval) * 1024;
-		vwrq->flags = IW_POWER_PERIOD;
-	}
-	if ((local->config.rmode & RXMODE_MASK) == RXMODE_ADDR)
-		vwrq->flags |= IW_POWER_UNICAST_R;
-	else
-		vwrq->flags |= IW_POWER_ALL_R;
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : set Sensitivity
- */
-static int airo_set_sens(struct net_device *dev,
-			 struct iw_request_info *info,
-			 union iwreq_data *wrqu,
-			 char *extra)
-{
-	struct iw_param *vwrq = &wrqu->sens;
-	struct airo_info *local = dev->ml_priv;
-
-	readConfigRid(local, 1);
-	local->config.rssiThreshold =
-		cpu_to_le16(vwrq->disabled ? RSSI_DEFAULT : vwrq->value);
-	set_bit (FLAG_COMMIT, &local->flags);
-
-	return -EINPROGRESS;		/* Call commit handler */
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get Sensitivity
- */
-static int airo_get_sens(struct net_device *dev,
-			 struct iw_request_info *info,
-			 union iwreq_data *wrqu,
-			 char *extra)
-{
-	struct iw_param *vwrq = &wrqu->sens;
-	struct airo_info *local = dev->ml_priv;
-
-	readConfigRid(local, 1);
-	vwrq->value = le16_to_cpu(local->config.rssiThreshold);
-	vwrq->disabled = (vwrq->value == 0);
-	vwrq->fixed = 1;
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get AP List
- * Note : this is deprecated in favor of IWSCAN
- */
-static int airo_get_aplist(struct net_device *dev,
-			   struct iw_request_info *info,
-			   union iwreq_data *wrqu,
-			   char *extra)
-{
-	struct iw_point *dwrq = &wrqu->data;
-	struct airo_info *local = dev->ml_priv;
-	struct sockaddr *address = (struct sockaddr *) extra;
-	struct iw_quality *qual;
-	BSSListRid BSSList;
-	int i;
-	int loseSync = capable(CAP_NET_ADMIN) ? 1: -1;
-
-	qual = kmalloc_array(IW_MAX_AP, sizeof(*qual), GFP_KERNEL);
-	if (!qual)
-		return -ENOMEM;
-
-	for (i = 0; i < IW_MAX_AP; i++) {
-		u16 dBm;
-		if (readBSSListRid(local, loseSync, &BSSList))
-			break;
-		loseSync = 0;
-		memcpy(address[i].sa_data, BSSList.bssid, ETH_ALEN);
-		address[i].sa_family = ARPHRD_ETHER;
-		dBm = le16_to_cpu(BSSList.dBm);
-		if (local->rssi) {
-			qual[i].level = 0x100 - dBm;
-			qual[i].qual = airo_dbm_to_pct(local->rssi, dBm);
-			qual[i].updated = IW_QUAL_QUAL_UPDATED
-					| IW_QUAL_LEVEL_UPDATED
-					| IW_QUAL_DBM;
-		} else {
-			qual[i].level = (dBm + 321) / 2;
-			qual[i].qual = 0;
-			qual[i].updated = IW_QUAL_QUAL_INVALID
-					| IW_QUAL_LEVEL_UPDATED
-					| IW_QUAL_DBM;
-		}
-		qual[i].noise = local->wstats.qual.noise;
-		if (BSSList.index == cpu_to_le16(0xffff))
-			break;
-	}
-	if (!i) {
-		StatusRid status_rid;		/* Card status info */
-		readStatusRid(local, &status_rid, 1);
-		for (i = 0;
-		     i < min(IW_MAX_AP, 4) &&
-			     (status_rid.bssid[i][0]
-			      & status_rid.bssid[i][1]
-			      & status_rid.bssid[i][2]
-			      & status_rid.bssid[i][3]
-			      & status_rid.bssid[i][4]
-			      & status_rid.bssid[i][5])!=0xff &&
-			     (status_rid.bssid[i][0]
-			      | status_rid.bssid[i][1]
-			      | status_rid.bssid[i][2]
-			      | status_rid.bssid[i][3]
-			      | status_rid.bssid[i][4]
-			      | status_rid.bssid[i][5]);
-		     i++) {
-			memcpy(address[i].sa_data,
-			       status_rid.bssid[i], ETH_ALEN);
-			address[i].sa_family = ARPHRD_ETHER;
-		}
-	} else {
-		dwrq->flags = 1; /* Should be define'd */
-		memcpy(extra + sizeof(struct sockaddr) * i, qual,
-		       sizeof(struct iw_quality) * i);
-	}
-	dwrq->length = i;
-
-	kfree(qual);
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : Initiate Scan
- */
-static int airo_set_scan(struct net_device *dev,
-			 struct iw_request_info *info,
-			 union iwreq_data *wrqu,
-			 char *extra)
-{
-	struct airo_info *ai = dev->ml_priv;
-	Cmd cmd;
-	Resp rsp;
-	int wake = 0;
-	APListRid APList_rid_empty;
-
-	/* Note : you may have realised that, as this is a SET operation,
-	 * this is privileged and therefore a normal user can't
-	 * perform scanning.
-	 * This is not an error, while the device perform scanning,
-	 * traffic doesn't flow, so it's a perfect DoS...
-	 * Jean II */
-	if (ai->flags & FLAG_RADIO_MASK) return -ENETDOWN;
-
-	if (down_interruptible(&ai->sem))
-		return -ERESTARTSYS;
-
-	/* If there's already a scan in progress, don't
-	 * trigger another one. */
-	if (ai->scan_timeout > 0)
-		goto out;
-
-	/* Clear APList as it affects scan results */
-	memset(&APList_rid_empty, 0, sizeof(APList_rid_empty));
-	APList_rid_empty.len = cpu_to_le16(sizeof(APList_rid_empty));
-	disable_MAC(ai, 2);
-	writeAPListRid(ai, &APList_rid_empty, 0);
-	enable_MAC(ai, 0);
-
-	/* Initiate a scan command */
-	ai->scan_timeout = RUN_AT(3*HZ);
-	memset(&cmd, 0, sizeof(cmd));
-	cmd.cmd = CMD_LISTBSS;
-	issuecommand(ai, &cmd, &rsp, true);
-	wake = 1;
-
-out:
-	up(&ai->sem);
-	if (wake)
-		wake_up_interruptible(&ai->thr_wait);
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Translate scan data returned from the card to a card independent
- * format that the Wireless Tools will understand - Jean II
- */
-static inline char *airo_translate_scan(struct net_device *dev,
-					struct iw_request_info *info,
-					char *current_ev,
-					char *end_buf,
-					BSSListRid *bss)
-{
-	struct airo_info *ai = dev->ml_priv;
-	struct iw_event		iwe;		/* Temporary buffer */
-	__le16			capabilities;
-	char *			current_val;	/* For rates */
-	int			i;
-	char *		buf;
-	u16 dBm;
-
-	/* First entry *MUST* be the AP MAC address */
-	iwe.cmd = SIOCGIWAP;
-	iwe.u.ap_addr.sa_family = ARPHRD_ETHER;
-	memcpy(iwe.u.ap_addr.sa_data, bss->bssid, ETH_ALEN);
-	current_ev = iwe_stream_add_event(info, current_ev, end_buf,
-					  &iwe, IW_EV_ADDR_LEN);
-
-	/* Other entries will be displayed in the order we give them */
-
-	/* Add the ESSID */
-	iwe.u.data.length = bss->ssidLen;
-	if (iwe.u.data.length > 32)
-		iwe.u.data.length = 32;
-	iwe.cmd = SIOCGIWESSID;
-	iwe.u.data.flags = 1;
-	current_ev = iwe_stream_add_point(info, current_ev, end_buf,
-					  &iwe, bss->ssid);
-
-	/* Add mode */
-	iwe.cmd = SIOCGIWMODE;
-	capabilities = bss->cap;
-	if (capabilities & (CAP_ESS | CAP_IBSS)) {
-		if (capabilities & CAP_ESS)
-			iwe.u.mode = IW_MODE_MASTER;
-		else
-			iwe.u.mode = IW_MODE_ADHOC;
-		current_ev = iwe_stream_add_event(info, current_ev, end_buf,
-						  &iwe, IW_EV_UINT_LEN);
-	}
-
-	/* Add frequency */
-	iwe.cmd = SIOCGIWFREQ;
-	iwe.u.freq.m = le16_to_cpu(bss->dsChannel);
-	iwe.u.freq.m = 100000 *
-	      ieee80211_channel_to_frequency(iwe.u.freq.m, NL80211_BAND_2GHZ);
-	iwe.u.freq.e = 1;
-	current_ev = iwe_stream_add_event(info, current_ev, end_buf,
-					  &iwe, IW_EV_FREQ_LEN);
-
-	dBm = le16_to_cpu(bss->dBm);
-
-	/* Add quality statistics */
-	iwe.cmd = IWEVQUAL;
-	if (ai->rssi) {
-		iwe.u.qual.level = 0x100 - dBm;
-		iwe.u.qual.qual = airo_dbm_to_pct(ai->rssi, dBm);
-		iwe.u.qual.updated = IW_QUAL_QUAL_UPDATED
-				| IW_QUAL_LEVEL_UPDATED
-				| IW_QUAL_DBM;
-	} else {
-		iwe.u.qual.level = (dBm + 321) / 2;
-		iwe.u.qual.qual = 0;
-		iwe.u.qual.updated = IW_QUAL_QUAL_INVALID
-				| IW_QUAL_LEVEL_UPDATED
-				| IW_QUAL_DBM;
-	}
-	iwe.u.qual.noise = ai->wstats.qual.noise;
-	current_ev = iwe_stream_add_event(info, current_ev, end_buf,
-					  &iwe, IW_EV_QUAL_LEN);
-
-	/* Add encryption capability */
-	iwe.cmd = SIOCGIWENCODE;
-	if (capabilities & CAP_PRIVACY)
-		iwe.u.data.flags = IW_ENCODE_ENABLED | IW_ENCODE_NOKEY;
-	else
-		iwe.u.data.flags = IW_ENCODE_DISABLED;
-	iwe.u.data.length = 0;
-	current_ev = iwe_stream_add_point(info, current_ev, end_buf,
-					  &iwe, bss->ssid);
-
-	/* Rate : stuffing multiple values in a single event require a bit
-	 * more of magic - Jean II */
-	current_val = current_ev + iwe_stream_lcp_len(info);
-
-	iwe.cmd = SIOCGIWRATE;
-	/* Those two flags are ignored... */
-	iwe.u.bitrate.fixed = iwe.u.bitrate.disabled = 0;
-	/* Max 8 values */
-	for (i = 0 ; i < 8 ; i++) {
-		/* NULL terminated */
-		if (bss->rates[i] == 0)
-			break;
-		/* Bit rate given in 500 kb/s units (+ 0x80) */
-		iwe.u.bitrate.value = ((bss->rates[i] & 0x7f) * 500000);
-		/* Add new value to event */
-		current_val = iwe_stream_add_value(info, current_ev,
-						   current_val, end_buf,
-						   &iwe, IW_EV_PARAM_LEN);
-	}
-	/* Check if we added any event */
-	if ((current_val - current_ev) > iwe_stream_lcp_len(info))
-		current_ev = current_val;
-
-	/* Beacon interval */
-	buf = kmalloc(30, GFP_KERNEL);
-	if (buf) {
-		iwe.cmd = IWEVCUSTOM;
-		sprintf(buf, "bcn_int=%d", bss->beaconInterval);
-		iwe.u.data.length = strlen(buf);
-		current_ev = iwe_stream_add_point(info, current_ev, end_buf,
-						  &iwe, buf);
-		kfree(buf);
-	}
-
-	/* Put WPA/RSN Information Elements into the event stream */
-	if (test_bit(FLAG_WPA_CAPABLE, &ai->flags)) {
-		unsigned int num_null_ies = 0;
-		u16 length = sizeof (bss->extra.iep);
-		u8 *ie = (void *)&bss->extra.iep;
-
-		while ((length >= 2) && (num_null_ies < 2)) {
-			if (2 + ie[1] > length) {
-				/* Invalid element, don't continue parsing IE */
-				break;
-			}
-
-			switch (ie[0]) {
-			case WLAN_EID_SSID:
-				/* Two zero-length SSID elements
-				 * mean we're done parsing elements */
-				if (!ie[1])
-					num_null_ies++;
-				break;
-
-			case WLAN_EID_VENDOR_SPECIFIC:
-				if (ie[1] >= 4 &&
-				    ie[2] == 0x00 &&
-				    ie[3] == 0x50 &&
-				    ie[4] == 0xf2 &&
-				    ie[5] == 0x01) {
-					iwe.cmd = IWEVGENIE;
-					/* 64 is an arbitrary cut-off */
-					iwe.u.data.length = min(ie[1] + 2,
-								64);
-					current_ev = iwe_stream_add_point(
-							info, current_ev,
-							end_buf, &iwe, ie);
-				}
-				break;
-
-			case WLAN_EID_RSN:
-				iwe.cmd = IWEVGENIE;
-				/* 64 is an arbitrary cut-off */
-				iwe.u.data.length = min(ie[1] + 2, 64);
-				current_ev = iwe_stream_add_point(
-					info, current_ev, end_buf,
-					&iwe, ie);
-				break;
-
-			default:
-				break;
-			}
-
-			length -= 2 + ie[1];
-			ie += 2 + ie[1];
-		}
-	}
-	return current_ev;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : Read Scan Results
- */
-static int airo_get_scan(struct net_device *dev,
-			 struct iw_request_info *info,
-			 union iwreq_data *wrqu,
-			 char *extra)
-{
-	struct iw_point *dwrq = &wrqu->data;
-	struct airo_info *ai = dev->ml_priv;
-	BSSListElement *net;
-	int err = 0;
-	char *current_ev = extra;
-
-	/* If a scan is in-progress, return -EAGAIN */
-	if (ai->scan_timeout > 0)
-		return -EAGAIN;
-
-	if (down_interruptible(&ai->sem))
-		return -EAGAIN;
-
-	list_for_each_entry (net, &ai->network_list, list) {
-		/* Translate to WE format this entry */
-		current_ev = airo_translate_scan(dev, info, current_ev,
-						 extra + dwrq->length,
-						 &net->bss);
-
-		/* Check if there is space for one more entry */
-		if ((extra + dwrq->length - current_ev) <= IW_EV_ADDR_LEN) {
-			/* Ask user space to try again with a bigger buffer */
-			err = -E2BIG;
-			goto out;
-		}
-	}
-
-	/* Length of data */
-	dwrq->length = (current_ev - extra);
-	dwrq->flags = 0;	/* todo */
-
-out:
-	up(&ai->sem);
-	return err;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Commit handler : called after a bunch of SET operations
- */
-static int airo_config_commit(struct net_device *dev,
-			      struct iw_request_info *info,	/* NULL */
-			      union iwreq_data *wrqu,		/* NULL */
-			      char *extra)			/* NULL */
-{
-	struct airo_info *local = dev->ml_priv;
-
-	if (!test_bit (FLAG_COMMIT, &local->flags))
-		return 0;
-
-	/* Some of the "SET" function may have modified some of the
-	 * parameters. It's now time to commit them in the card */
-	disable_MAC(local, 1);
-	if (test_bit (FLAG_RESET, &local->flags)) {
-		SsidRid SSID_rid;
-
-		readSsidRid(local, &SSID_rid);
-		if (test_bit(FLAG_MPI,&local->flags))
-			setup_card(local, dev, 1);
-		else
-			reset_airo_card(dev);
-		disable_MAC(local, 1);
-		writeSsidRid(local, &SSID_rid, 1);
-		writeAPListRid(local, &local->APList, 1);
-	}
-	if (down_interruptible(&local->sem))
-		return -ERESTARTSYS;
-	writeConfigRid(local, 0);
-	enable_MAC(local, 0);
-	if (test_bit (FLAG_RESET, &local->flags))
-		airo_set_promisc(local, true);
-	else
-		up(&local->sem);
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Structures to export the Wireless Handlers
- */
-
-static const struct iw_priv_args airo_private_args[] = {
-/*{ cmd,         set_args,                            get_args, name } */
-  { AIROIOCTL, IW_PRIV_TYPE_BYTE | IW_PRIV_SIZE_FIXED | sizeof (aironet_ioctl),
-    IW_PRIV_TYPE_BYTE | 2047, "airoioctl" },
-  { AIROIDIFC, IW_PRIV_TYPE_BYTE | IW_PRIV_SIZE_FIXED | sizeof (aironet_ioctl),
-    IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "airoidifc" },
-};
-
-static const iw_handler		airo_handler[] =
-{
-	IW_HANDLER(SIOCSIWCOMMIT,	airo_config_commit),
-	IW_HANDLER(SIOCGIWNAME,		airo_get_name),
-	IW_HANDLER(SIOCSIWFREQ,		airo_set_freq),
-	IW_HANDLER(SIOCGIWFREQ,		airo_get_freq),
-	IW_HANDLER(SIOCSIWMODE,		airo_set_mode),
-	IW_HANDLER(SIOCGIWMODE,		airo_get_mode),
-	IW_HANDLER(SIOCSIWSENS,		airo_set_sens),
-	IW_HANDLER(SIOCGIWSENS,		airo_get_sens),
-	IW_HANDLER(SIOCGIWRANGE,	airo_get_range),
-	IW_HANDLER(SIOCSIWSPY,		iw_handler_set_spy),
-	IW_HANDLER(SIOCGIWSPY,		iw_handler_get_spy),
-	IW_HANDLER(SIOCSIWTHRSPY,	iw_handler_set_thrspy),
-	IW_HANDLER(SIOCGIWTHRSPY,	iw_handler_get_thrspy),
-	IW_HANDLER(SIOCSIWAP,		airo_set_wap),
-	IW_HANDLER(SIOCGIWAP,		airo_get_wap),
-	IW_HANDLER(SIOCGIWAPLIST,	airo_get_aplist),
-	IW_HANDLER(SIOCSIWSCAN,		airo_set_scan),
-	IW_HANDLER(SIOCGIWSCAN,		airo_get_scan),
-	IW_HANDLER(SIOCSIWESSID,	airo_set_essid),
-	IW_HANDLER(SIOCGIWESSID,	airo_get_essid),
-	IW_HANDLER(SIOCSIWNICKN,	airo_set_nick),
-	IW_HANDLER(SIOCGIWNICKN,	airo_get_nick),
-	IW_HANDLER(SIOCSIWRATE,		airo_set_rate),
-	IW_HANDLER(SIOCGIWRATE,		airo_get_rate),
-	IW_HANDLER(SIOCSIWRTS,		airo_set_rts),
-	IW_HANDLER(SIOCGIWRTS,		airo_get_rts),
-	IW_HANDLER(SIOCSIWFRAG,		airo_set_frag),
-	IW_HANDLER(SIOCGIWFRAG,		airo_get_frag),
-	IW_HANDLER(SIOCSIWTXPOW,	airo_set_txpow),
-	IW_HANDLER(SIOCGIWTXPOW,	airo_get_txpow),
-	IW_HANDLER(SIOCSIWRETRY,	airo_set_retry),
-	IW_HANDLER(SIOCGIWRETRY,	airo_get_retry),
-	IW_HANDLER(SIOCSIWENCODE,	airo_set_encode),
-	IW_HANDLER(SIOCGIWENCODE,	airo_get_encode),
-	IW_HANDLER(SIOCSIWPOWER,	airo_set_power),
-	IW_HANDLER(SIOCGIWPOWER,	airo_get_power),
-	IW_HANDLER(SIOCSIWAUTH,		airo_set_auth),
-	IW_HANDLER(SIOCGIWAUTH,		airo_get_auth),
-	IW_HANDLER(SIOCSIWENCODEEXT,	airo_set_encodeext),
-	IW_HANDLER(SIOCGIWENCODEEXT,	airo_get_encodeext),
-};
-
-/* Note : don't describe AIROIDIFC and AIROOLDIDIFC in here.
- * We want to force the use of the ioctl code, because those can't be
- * won't work the iw_handler code (because they simultaneously read
- * and write data and iw_handler can't do that).
- * Note that it's perfectly legal to read/write on a single ioctl command,
- * you just can't use iwpriv and need to force it via the ioctl handler.
- * Jean II */
-static const iw_handler		airo_private_handler[] =
-{
-	NULL,				/* SIOCIWFIRSTPRIV */
-};
-
-static const struct iw_handler_def	airo_handler_def =
-{
-	.num_standard	= ARRAY_SIZE(airo_handler),
-	.num_private	= ARRAY_SIZE(airo_private_handler),
-	.num_private_args = ARRAY_SIZE(airo_private_args),
-	.standard	= airo_handler,
-	.private	= airo_private_handler,
-	.private_args	= airo_private_args,
-	.get_wireless_stats = airo_get_wireless_stats,
-};
-
-/*
- * This defines the configuration part of the Wireless Extensions
- * Note : irq and spinlock protection will occur in the subroutines
- *
- * TODO :
- *	o Check input value more carefully and fill correct values in range
- *	o Test and shakeout the bugs (if any)
- *
- * Jean II
- *
- * Javier Achirica did a great job of merging code from the unnamed CISCO
- * developer that added support for flashing the card.
- */
-static int airo_siocdevprivate(struct net_device *dev, struct ifreq *rq,
-			       void __user *data, int cmd)
-{
-	int rc = 0;
-	struct airo_info *ai = dev->ml_priv;
-
-	if (ai->power.event)
-		return 0;
-
-	switch (cmd) {
-#ifdef CISCO_EXT
-	case AIROIDIFC:
-#ifdef AIROOLDIDIFC
-	case AIROOLDIDIFC:
-#endif
-	{
-		int val = AIROMAGIC;
-		aironet_ioctl com;
-		if (copy_from_user(&com, data, sizeof(com)))
-			rc = -EFAULT;
-		else if (copy_to_user(com.data, (char *)&val, sizeof(val)))
-			rc = -EFAULT;
-	}
-	break;
-
-	case AIROIOCTL:
-#ifdef AIROOLDIOCTL
-	case AIROOLDIOCTL:
-#endif
-		/* Get the command struct and hand it off for evaluation by
-		 * the proper subfunction
-		 */
-	{
-		aironet_ioctl com;
-		if (copy_from_user(&com, data, sizeof(com))) {
-			rc = -EFAULT;
-			break;
-		}
-
-		/* Separate R/W functions bracket legality here
-		 */
-		if (com.command == AIRORSWVERSION) {
-			if (copy_to_user(com.data, swversion, sizeof(swversion)))
-				rc = -EFAULT;
-			else
-				rc = 0;
-		}
-		else if (com.command <= AIRORRID)
-			rc = readrids(dev,&com);
-		else if (com.command >= AIROPCAP && com.command <= (AIROPLEAPUSR+2))
-			rc = writerids(dev,&com);
-		else if (com.command >= AIROFLSHRST && com.command <= AIRORESTART)
-			rc = flashcard(dev,&com);
-		else
-			rc = -EINVAL;      /* Bad command in ioctl */
-	}
-	break;
-#endif /* CISCO_EXT */
-
-	// All other calls are currently unsupported
-	default:
-		rc = -EOPNOTSUPP;
-	}
-	return rc;
-}
-
-/*
- * Get the Wireless stats out of the driver
- * Note : irq and spinlock protection will occur in the subroutines
- *
- * TODO :
- *	o Check if work in Ad-Hoc mode (otherwise, use SPY, as in wvlan_cs)
- *
- * Jean
- */
-static void airo_read_wireless_stats(struct airo_info *local)
-{
-	StatusRid status_rid;
-	StatsRid stats_rid;
-	CapabilityRid cap_rid;
-	__le32 *vals = stats_rid.vals;
-
-	/* Get stats out of the card */
-	if (local->power.event)
-		return;
-
-	readCapabilityRid(local, &cap_rid, 0);
-	readStatusRid(local, &status_rid, 0);
-	readStatsRid(local, &stats_rid, RID_STATS, 0);
-
-	/* The status */
-	local->wstats.status = le16_to_cpu(status_rid.mode);
-
-	/* Signal quality and co */
-	if (local->rssi) {
-		local->wstats.qual.level =
-			airo_rssi_to_dbm(local->rssi,
-					 le16_to_cpu(status_rid.sigQuality));
-		/* normalizedSignalStrength appears to be a percentage */
-		local->wstats.qual.qual =
-			le16_to_cpu(status_rid.normalizedSignalStrength);
-	} else {
-		local->wstats.qual.level =
-			(le16_to_cpu(status_rid.normalizedSignalStrength) + 321) / 2;
-		local->wstats.qual.qual = airo_get_quality(&status_rid, &cap_rid);
-	}
-	if (le16_to_cpu(status_rid.len) >= 124) {
-		local->wstats.qual.noise = 0x100 - status_rid.noisedBm;
-		local->wstats.qual.updated = IW_QUAL_ALL_UPDATED | IW_QUAL_DBM;
-	} else {
-		local->wstats.qual.noise = 0;
-		local->wstats.qual.updated = IW_QUAL_QUAL_UPDATED | IW_QUAL_LEVEL_UPDATED | IW_QUAL_NOISE_INVALID | IW_QUAL_DBM;
-	}
-
-	/* Packets discarded in the wireless adapter due to wireless
-	 * specific problems */
-	local->wstats.discard.nwid = le32_to_cpu(vals[56]) +
-				     le32_to_cpu(vals[57]) +
-				     le32_to_cpu(vals[58]); /* SSID Mismatch */
-	local->wstats.discard.code = le32_to_cpu(vals[6]);/* RxWepErr */
-	local->wstats.discard.fragment = le32_to_cpu(vals[30]);
-	local->wstats.discard.retries = le32_to_cpu(vals[10]);
-	local->wstats.discard.misc = le32_to_cpu(vals[1]) +
-				     le32_to_cpu(vals[32]);
-	local->wstats.miss.beacon = le32_to_cpu(vals[34]);
-}
-
-static struct iw_statistics *airo_get_wireless_stats(struct net_device *dev)
-{
-	struct airo_info *local =  dev->ml_priv;
-
-	if (!down_interruptible(&local->sem)) {
-		airo_read_wireless_stats(local);
-		up(&local->sem);
-	}
-	return &local->wstats;
-}
-
-#ifdef CISCO_EXT
-/*
- * This just translates from driver IOCTL codes to the command codes to
- * feed to the radio's host interface. Things can be added/deleted
- * as needed.  This represents the READ side of control I/O to
- * the card
- */
-static int readrids(struct net_device *dev, aironet_ioctl *comp)
-{
-	unsigned short ridcode;
-	unsigned char *iobuf;
-	int len;
-	struct airo_info *ai = dev->ml_priv;
-
-	if (test_bit(FLAG_FLASHING, &ai->flags))
-		return -EIO;
-
-	switch(comp->command)
-	{
-	case AIROGCAP:      ridcode = RID_CAPABILITIES; break;
-	case AIROGCFG:      ridcode = RID_CONFIG;
-		if (test_bit(FLAG_COMMIT, &ai->flags)) {
-			disable_MAC (ai, 1);
-			writeConfigRid (ai, 1);
-			enable_MAC(ai, 1);
-		}
-		break;
-	case AIROGSLIST:    ridcode = RID_SSID;         break;
-	case AIROGVLIST:    ridcode = RID_APLIST;       break;
-	case AIROGDRVNAM:   ridcode = RID_DRVNAME;      break;
-	case AIROGEHTENC:   ridcode = RID_ETHERENCAP;   break;
-	case AIROGWEPKTMP:  ridcode = RID_WEP_TEMP;	break;
-	case AIROGWEPKNV:   ridcode = RID_WEP_PERM;	break;
-	case AIROGSTAT:     ridcode = RID_STATUS;       break;
-	case AIROGSTATSD32: ridcode = RID_STATSDELTA;   break;
-	case AIROGSTATSC32: ridcode = RID_STATS;        break;
-	case AIROGMICSTATS:
-		if (copy_to_user(comp->data, &ai->micstats,
-				 min((int)comp->len, (int)sizeof(ai->micstats))))
-			return -EFAULT;
-		return 0;
-	case AIRORRID:      ridcode = comp->ridnum;     break;
-	default:
-		return -EINVAL;
-	}
-
-	if (ridcode == RID_WEP_TEMP || ridcode == RID_WEP_PERM) {
-		/* Only super-user can read WEP keys */
-		if (!capable(CAP_NET_ADMIN))
-			return -EPERM;
-	}
-
-	if ((iobuf = kzalloc(RIDSIZE, GFP_KERNEL)) == NULL)
-		return -ENOMEM;
-
-	PC4500_readrid(ai, ridcode, iobuf, RIDSIZE, 1);
-	/* get the count of bytes in the rid  docs say 1st 2 bytes is it.
-	 * then return it to the user
-	 * 9/22/2000 Honor user given length
-	 */
-	len = comp->len;
-
-	if (copy_to_user(comp->data, iobuf, min(len, (int)RIDSIZE))) {
-		kfree (iobuf);
-		return -EFAULT;
-	}
-	kfree (iobuf);
-	return 0;
-}
-
-/*
- * Danger Will Robinson write the rids here
- */
-
-static int writerids(struct net_device *dev, aironet_ioctl *comp)
-{
-	struct airo_info *ai = dev->ml_priv;
-	int  ridcode;
-        int  enabled;
-	int (*writer)(struct airo_info *, u16 rid, const void *, int, int);
-	unsigned char *iobuf;
-
-	/* Only super-user can write RIDs */
-	if (!capable(CAP_NET_ADMIN))
-		return -EPERM;
-
-	if (test_bit(FLAG_FLASHING, &ai->flags))
-		return -EIO;
-
-	ridcode = 0;
-	writer = do_writerid;
-
-	switch(comp->command)
-	{
-	case AIROPSIDS:     ridcode = RID_SSID;         break;
-	case AIROPCAP:      ridcode = RID_CAPABILITIES; break;
-	case AIROPAPLIST:   ridcode = RID_APLIST;       break;
-	case AIROPCFG: ai->config.len = 0;
-			    clear_bit(FLAG_COMMIT, &ai->flags);
-			    ridcode = RID_CONFIG;       break;
-	case AIROPWEPKEYNV: ridcode = RID_WEP_PERM;     break;
-	case AIROPLEAPUSR:  ridcode = RID_LEAPUSERNAME; break;
-	case AIROPLEAPPWD:  ridcode = RID_LEAPPASSWORD; break;
-	case AIROPWEPKEY:   ridcode = RID_WEP_TEMP; writer = PC4500_writerid;
-		break;
-	case AIROPLEAPUSR+1: ridcode = 0xFF2A;          break;
-	case AIROPLEAPUSR+2: ridcode = 0xFF2B;          break;
-
-		/* this is not really a rid but a command given to the card
-		 * same with MAC off
-		 */
-	case AIROPMACON:
-		if (enable_MAC(ai, 1) != 0)
-			return -EIO;
-		return 0;
-
-		/*
-		 * Evidently this code in the airo driver does not get a symbol
-		 * as disable_MAC. it's probably so short the compiler does not gen one.
-		 */
-	case AIROPMACOFF:
-		disable_MAC(ai, 1);
-		return 0;
-
-		/* This command merely clears the counts does not actually store any data
-		 * only reads rid. But as it changes the cards state, I put it in the
-		 * writerid routines.
-		 */
-	case AIROPSTCLR:
-		if ((iobuf = kmalloc(RIDSIZE, GFP_KERNEL)) == NULL)
-			return -ENOMEM;
-
-		PC4500_readrid(ai, RID_STATSDELTACLEAR, iobuf, RIDSIZE, 1);
-
-		enabled = ai->micstats.enabled;
-		memset(&ai->micstats, 0, sizeof(ai->micstats));
-		ai->micstats.enabled = enabled;
-
-		if (copy_to_user(comp->data, iobuf,
-				 min((int)comp->len, (int)RIDSIZE))) {
-			kfree (iobuf);
-			return -EFAULT;
-		}
-		kfree (iobuf);
-		return 0;
-
-	default:
-		return -EOPNOTSUPP;	/* Blarg! */
-	}
-	if (comp->len > RIDSIZE)
-		return -EINVAL;
-
-	if ((iobuf = kmalloc(RIDSIZE, GFP_KERNEL)) == NULL)
-		return -ENOMEM;
-
-	if (copy_from_user(iobuf, comp->data, comp->len)) {
-		kfree (iobuf);
-		return -EFAULT;
-	}
-
-	if (comp->command == AIROPCFG) {
-		ConfigRid *cfg = (ConfigRid *)iobuf;
-
-		if (test_bit(FLAG_MIC_CAPABLE, &ai->flags))
-			cfg->opmode |= MODE_MIC;
-
-		if ((cfg->opmode & MODE_CFG_MASK) == MODE_STA_IBSS)
-			set_bit (FLAG_ADHOC, &ai->flags);
-		else
-			clear_bit (FLAG_ADHOC, &ai->flags);
-	}
-
-	if ((*writer)(ai, ridcode, iobuf, comp->len, 1)) {
-		kfree (iobuf);
-		return -EIO;
-	}
-	kfree (iobuf);
-	return 0;
-}
-
-/*****************************************************************************
- * Ancillary flash / mod functions much black magic lurkes here              *
- *****************************************************************************
- */
-
-/*
- * Flash command switch table
- */
-
-static int flashcard(struct net_device *dev, aironet_ioctl *comp)
-{
-	int z;
-
-	/* Only super-user can modify flash */
-	if (!capable(CAP_NET_ADMIN))
-		return -EPERM;
-
-	switch(comp->command)
-	{
-	case AIROFLSHRST:
-		return cmdreset((struct airo_info *)dev->ml_priv);
-
-	case AIROFLSHSTFL:
-		if (!AIRO_FLASH(dev) &&
-		    (AIRO_FLASH(dev) = kmalloc(FLASHSIZE, GFP_KERNEL)) == NULL)
-			return -ENOMEM;
-		return setflashmode((struct airo_info *)dev->ml_priv);
-
-	case AIROFLSHGCHR: /* Get char from aux */
-		if (comp->len != sizeof(int))
-			return -EINVAL;
-		if (copy_from_user(&z, comp->data, comp->len))
-			return -EFAULT;
-		return flashgchar((struct airo_info *)dev->ml_priv, z, 8000);
-
-	case AIROFLSHPCHR: /* Send char to card. */
-		if (comp->len != sizeof(int))
-			return -EINVAL;
-		if (copy_from_user(&z, comp->data, comp->len))
-			return -EFAULT;
-		return flashpchar((struct airo_info *)dev->ml_priv, z, 8000);
-
-	case AIROFLPUTBUF: /* Send 32k to card */
-		if (!AIRO_FLASH(dev))
-			return -ENOMEM;
-		if (comp->len > FLASHSIZE)
-			return -EINVAL;
-		if (copy_from_user(AIRO_FLASH(dev), comp->data, comp->len))
-			return -EFAULT;
-
-		flashputbuf((struct airo_info *)dev->ml_priv);
-		return 0;
-
-	case AIRORESTART:
-		if (flashrestart((struct airo_info *)dev->ml_priv, dev))
-			return -EIO;
-		return 0;
-	}
-	return -EINVAL;
-}
-
-#define FLASH_COMMAND  0x7e7e
-
-/*
- * STEP 1)
- * Disable MAC and do soft reset on
- * card.
- */
-
-static int cmdreset(struct airo_info *ai)
-{
-	disable_MAC(ai, 1);
-
-	if (!waitbusy (ai)) {
-		airo_print_info(ai->dev->name, "Waitbusy hang before RESET");
-		return -EBUSY;
-	}
-
-	OUT4500(ai, COMMAND, CMD_SOFTRESET);
-
-	ssleep(1);			/* WAS 600 12/7/00 */
-
-	if (!waitbusy (ai)) {
-		airo_print_info(ai->dev->name, "Waitbusy hang AFTER RESET");
-		return -EBUSY;
-	}
-	return 0;
-}
-
-/* STEP 2)
- * Put the card in legendary flash
- * mode
- */
-
-static int setflashmode (struct airo_info *ai)
-{
-	set_bit (FLAG_FLASHING, &ai->flags);
-
-	OUT4500(ai, SWS0, FLASH_COMMAND);
-	OUT4500(ai, SWS1, FLASH_COMMAND);
-	if (probe) {
-		OUT4500(ai, SWS0, FLASH_COMMAND);
-		OUT4500(ai, COMMAND, 0x10);
-	} else {
-		OUT4500(ai, SWS2, FLASH_COMMAND);
-		OUT4500(ai, SWS3, FLASH_COMMAND);
-		OUT4500(ai, COMMAND, 0);
-	}
-	msleep(500);		/* 500ms delay */
-
-	if (!waitbusy(ai)) {
-		clear_bit (FLAG_FLASHING, &ai->flags);
-		airo_print_info(ai->dev->name, "Waitbusy hang after setflash mode");
-		return -EIO;
-	}
-	return 0;
-}
-
-/* Put character to SWS0 wait for dwelltime
- * x 50us for  echo .
- */
-
-static int flashpchar(struct airo_info *ai, int byte, int dwelltime)
-{
-	int echo;
-	int waittime;
-
-	byte |= 0x8000;
-
-	if (dwelltime == 0)
-		dwelltime = 200;
-
-	waittime = dwelltime;
-
-	/* Wait for busy bit d15 to go false indicating buffer empty */
-	while ((IN4500 (ai, SWS0) & 0x8000) && waittime > 0) {
-		udelay (50);
-		waittime -= 50;
-	}
-
-	/* timeout for busy clear wait */
-	if (waittime <= 0) {
-		airo_print_info(ai->dev->name, "flash putchar busywait timeout!");
-		return -EBUSY;
-	}
-
-	/* Port is clear now write byte and wait for it to echo back */
-	do {
-		OUT4500(ai, SWS0, byte);
-		udelay(50);
-		dwelltime -= 50;
-		echo = IN4500(ai, SWS1);
-	} while (dwelltime >= 0 && echo != byte);
-
-	OUT4500(ai, SWS1, 0);
-
-	return (echo == byte) ? 0 : -EIO;
-}
-
-/*
- * Get a character from the card matching matchbyte
- * Step 3)
- */
-static int flashgchar(struct airo_info *ai, int matchbyte, int dwelltime)
-{
-	int           rchar;
-	unsigned char rbyte = 0;
-
-	do {
-		rchar = IN4500(ai, SWS1);
-
-		if (dwelltime && !(0x8000 & rchar)) {
-			dwelltime -= 10;
-			mdelay(10);
-			continue;
-		}
-		rbyte = 0xff & rchar;
-
-		if ((rbyte == matchbyte) && (0x8000 & rchar)) {
-			OUT4500(ai, SWS1, 0);
-			return 0;
-		}
-		if (rbyte == 0x81 || rbyte == 0x82 || rbyte == 0x83 || rbyte == 0x1a || 0xffff == rchar)
-			break;
-		OUT4500(ai, SWS1, 0);
-
-	} while (dwelltime > 0);
-	return -EIO;
-}
-
-/*
- * Transfer 32k of firmware data from user buffer to our buffer and
- * send to the card
- */
-
-static int flashputbuf(struct airo_info *ai)
-{
-	int            nwords;
-
-	/* Write stuff */
-	if (test_bit(FLAG_MPI,&ai->flags))
-		memcpy_toio(ai->pciaux + 0x8000, ai->flash, FLASHSIZE);
-	else {
-		OUT4500(ai, AUXPAGE, 0x100);
-		OUT4500(ai, AUXOFF, 0);
-
-		for (nwords = 0; nwords != FLASHSIZE / 2; nwords++) {
-			OUT4500(ai, AUXDATA, ai->flash[nwords] & 0xffff);
-		}
-	}
-	OUT4500(ai, SWS0, 0x8000);
-
-	return 0;
-}
-
-/*
- *
- */
-static int flashrestart(struct airo_info *ai, struct net_device *dev)
-{
-	int    i, status;
-
-	ssleep(1);			/* Added 12/7/00 */
-	clear_bit (FLAG_FLASHING, &ai->flags);
-	if (test_bit(FLAG_MPI, &ai->flags)) {
-		status = mpi_init_descriptors(ai);
-		if (status != SUCCESS)
-			return status;
-	}
-	status = setup_card(ai, dev, 1);
-
-	if (!test_bit(FLAG_MPI,&ai->flags))
-		for (i = 0; i < MAX_FIDS; i++) {
-			ai->fids[i] = transmit_allocate
-				(ai, AIRO_DEF_MTU, i >= MAX_FIDS / 2);
-		}
-
-	ssleep(1);			/* Added 12/7/00 */
-	return status;
-}
-#endif /* CISCO_EXT */
-
-/*
-    This program is free software; you can redistribute it and/or
-    modify it under the terms of the GNU General Public License
-    as published by the Free Software Foundation; either version 2
-    of the License, or (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    In addition:
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions
-    are met:
-
-    1. Redistributions of source code must retain the above copyright
-       notice, this list of conditions and the following disclaimer.
-    2. Redistributions in binary form must reproduce the above copyright
-       notice, this list of conditions and the following disclaimer in the
-       documentation and/or other materials provided with the distribution.
-    3. The name of the author may not be used to endorse or promote
-       products derived from this software without specific prior written
-       permission.
-
-    THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
-    IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-    ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
-    INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-    (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-    SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-    HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-    STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
-    IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-    POSSIBILITY OF SUCH DAMAGE.
-*/
-
-module_init(airo_init_module);
-module_exit(airo_cleanup_module);
diff --git a/drivers/net/wireless/cisco/airo.h b/drivers/net/wireless/cisco/airo.h
deleted file mode 100644
index 8a02977a2e2b..000000000000
--- a/drivers/net/wireless/cisco/airo.h
+++ /dev/null
@@ -1,10 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _AIRO_H_
-#define _AIRO_H_
-
-struct net_device *init_airo_card(unsigned short irq, int port, int is_pcmcia,
-				  struct device *dmdev);
-int reset_airo_card(struct net_device *dev);
-void stop_airo_card(struct net_device *dev, int freeres);
-
-#endif  /*  _AIRO_H_  */
diff --git a/drivers/net/wireless/cisco/airo_cs.c b/drivers/net/wireless/cisco/airo_cs.c
deleted file mode 100644
index fcfe4c6d62f0..000000000000
--- a/drivers/net/wireless/cisco/airo_cs.c
+++ /dev/null
@@ -1,218 +0,0 @@
-/*======================================================================
-
-    Aironet driver for 4500 and 4800 series cards
-
-    This code is released under both the GPL version 2 and BSD licenses.
-    Either license may be used.  The respective licenses are found at
-    the end of this file.
-
-    This code was developed by Benjamin Reed <breed@users.sourceforge.net>
-    including portions of which come from the Aironet PC4500
-    Developer's Reference Manual and used with permission.  Copyright
-    (C) 1999 Benjamin Reed.  All Rights Reserved.  Permission to use
-    code in the Developer's manual was granted for this driver by
-    Aironet.
-
-    In addition this module was derived from dummy_cs.
-    The initial developer of dummy_cs is David A. Hinds
-    <dahinds@users.sourceforge.net>.  Portions created by David A. Hinds
-    are Copyright (C) 1999 David A. Hinds.  All Rights Reserved.
-
-======================================================================*/
-
-#ifdef __IN_PCMCIA_PACKAGE__
-#include <pcmcia/k_compat.h>
-#endif
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/ptrace.h>
-#include <linux/slab.h>
-#include <linux/string.h>
-#include <linux/timer.h>
-#include <linux/netdevice.h>
-
-#include <pcmcia/cistpl.h>
-#include <pcmcia/cisreg.h>
-#include <pcmcia/ds.h>
-
-#include <linux/io.h>
-
-#include "airo.h"
-
-
-/*====================================================================*/
-
-MODULE_AUTHOR("Benjamin Reed");
-MODULE_DESCRIPTION("Support for Cisco/Aironet 802.11 wireless ethernet "
-		   "cards.  This is the module that links the PCMCIA card "
-		   "with the airo module.");
-MODULE_LICENSE("Dual BSD/GPL");
-
-/*====================================================================*/
-
-static int airo_config(struct pcmcia_device *link);
-static void airo_release(struct pcmcia_device *link);
-
-static void airo_detach(struct pcmcia_device *p_dev);
-
-struct local_info {
-	struct net_device *eth_dev;
-};
-
-static int airo_probe(struct pcmcia_device *p_dev)
-{
-	struct local_info *local;
-
-	dev_dbg(&p_dev->dev, "airo_attach()\n");
-
-	/* Allocate space for private device-specific data */
-	local = kzalloc(sizeof(*local), GFP_KERNEL);
-	if (!local)
-		return -ENOMEM;
-
-	p_dev->priv = local;
-
-	return airo_config(p_dev);
-} /* airo_attach */
-
-static void airo_detach(struct pcmcia_device *link)
-{
-	dev_dbg(&link->dev, "airo_detach\n");
-
-	airo_release(link);
-
-	if (((struct local_info *)link->priv)->eth_dev) {
-		stop_airo_card(((struct local_info *)link->priv)->eth_dev,
-			       0);
-	}
-	((struct local_info *)link->priv)->eth_dev = NULL;
-
-	kfree(link->priv);
-} /* airo_detach */
-
-static int airo_cs_config_check(struct pcmcia_device *p_dev, void *priv_data)
-{
-	if (p_dev->config_index == 0)
-		return -EINVAL;
-
-	return pcmcia_request_io(p_dev);
-}
-
-
-static int airo_config(struct pcmcia_device *link)
-{
-	int ret;
-
-	dev_dbg(&link->dev, "airo_config\n");
-
-	link->config_flags |= CONF_ENABLE_IRQ | CONF_AUTO_SET_VPP |
-		CONF_AUTO_AUDIO | CONF_AUTO_SET_IO;
-
-	ret = pcmcia_loop_config(link, airo_cs_config_check, NULL);
-	if (ret)
-		goto failed;
-
-	if (!link->irq)
-		goto failed;
-
-	ret = pcmcia_enable_device(link);
-	if (ret)
-		goto failed;
-	((struct local_info *)link->priv)->eth_dev =
-		init_airo_card(link->irq,
-			       link->resource[0]->start, 1, &link->dev);
-	if (!((struct local_info *)link->priv)->eth_dev)
-		goto failed;
-
-	return 0;
-
- failed:
-	airo_release(link);
-	return -ENODEV;
-} /* airo_config */
-
-static void airo_release(struct pcmcia_device *link)
-{
-	dev_dbg(&link->dev, "airo_release\n");
-	pcmcia_disable_device(link);
-}
-
-static int airo_suspend(struct pcmcia_device *link)
-{
-	struct local_info *local = link->priv;
-
-	netif_device_detach(local->eth_dev);
-
-	return 0;
-}
-
-static int airo_resume(struct pcmcia_device *link)
-{
-	struct local_info *local = link->priv;
-
-	if (link->open) {
-		reset_airo_card(local->eth_dev);
-		netif_device_attach(local->eth_dev);
-	}
-
-	return 0;
-}
-
-static const struct pcmcia_device_id airo_ids[] = {
-	PCMCIA_DEVICE_MANF_CARD(0x015f, 0x000a),
-	PCMCIA_DEVICE_MANF_CARD(0x015f, 0x0005),
-	PCMCIA_DEVICE_MANF_CARD(0x015f, 0x0007),
-	PCMCIA_DEVICE_MANF_CARD(0x0105, 0x0007),
-	PCMCIA_DEVICE_NULL,
-};
-MODULE_DEVICE_TABLE(pcmcia, airo_ids);
-
-static struct pcmcia_driver airo_driver = {
-	.owner		= THIS_MODULE,
-	.name		= "airo_cs",
-	.probe		= airo_probe,
-	.remove		= airo_detach,
-	.id_table       = airo_ids,
-	.suspend	= airo_suspend,
-	.resume		= airo_resume,
-};
-module_pcmcia_driver(airo_driver);
-
-/*
-    This program is free software; you can redistribute it and/or
-    modify it under the terms of the GNU General Public License
-    as published by the Free Software Foundation; either version 2
-    of the License, or (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    In addition:
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions
-    are met:
-
-    1. Redistributions of source code must retain the above copyright
-       notice, this list of conditions and the following disclaimer.
-    2. Redistributions in binary form must reproduce the above copyright
-       notice, this list of conditions and the following disclaimer in the
-       documentation and/or other materials provided with the distribution.
-    3. The name of the author may not be used to endorse or promote
-       products derived from this software without specific prior written
-       permission.
-
-    THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
-    IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-    ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
-    INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-    (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-    SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-    HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-    STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
-    IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-    POSSIBILITY OF SUCH DAMAGE.
-*/
diff --git a/drivers/net/wireless/intel/iwlegacy/4965-mac.c b/drivers/net/wireless/intel/iwlegacy/4965-mac.c
index 69276266ce6f..70e420df1643 100644
--- a/drivers/net/wireless/intel/iwlegacy/4965-mac.c
+++ b/drivers/net/wireless/intel/iwlegacy/4965-mac.c
@@ -4231,8 +4231,6 @@ il4965_rx_handle(struct il_priv *il)
 		fill_rx = 1;
 
 	while (i != r) {
-		int len;
-
 		rxb = rxq->queue[i];
 
 		/* If an RXB doesn't have a Rx queue slot associated with it,
@@ -4246,10 +4244,6 @@ il4965_rx_handle(struct il_priv *il)
 			       PAGE_SIZE << il->hw_params.rx_page_order,
 			       DMA_FROM_DEVICE);
 		pkt = rxb_addr(rxb);
-
-		len = le32_to_cpu(pkt->len_n_flags) & IL_RX_FRAME_SIZE_MSK;
-		len += sizeof(u32);	/* account for status word */
-
 		reclaim = il_need_reclaim(il, pkt);
 
 		/* Based on type of command response or notification,
diff --git a/drivers/net/wireless/intersil/Kconfig b/drivers/net/wireless/intersil/Kconfig
index bd6bf70ece03..201b1534a9ca 100644
--- a/drivers/net/wireless/intersil/Kconfig
+++ b/drivers/net/wireless/intersil/Kconfig
@@ -12,8 +12,6 @@ config WLAN_VENDOR_INTERSIL
 
 if WLAN_VENDOR_INTERSIL
 
-source "drivers/net/wireless/intersil/hostap/Kconfig"
-source "drivers/net/wireless/intersil/orinoco/Kconfig"
 source "drivers/net/wireless/intersil/p54/Kconfig"
 
 endif # WLAN_VENDOR_INTERSIL
diff --git a/drivers/net/wireless/intersil/Makefile b/drivers/net/wireless/intersil/Makefile
index 65281d1b3d85..27e9b2869da1 100644
--- a/drivers/net/wireless/intersil/Makefile
+++ b/drivers/net/wireless/intersil/Makefile
@@ -1,4 +1,2 @@
 # SPDX-License-Identifier: GPL-2.0-only
-obj-$(CONFIG_HOSTAP)		+= hostap/
-obj-$(CONFIG_HERMES)		+= orinoco/
 obj-$(CONFIG_P54_COMMON)	+= p54/
diff --git a/drivers/net/wireless/intersil/hostap/Kconfig b/drivers/net/wireless/intersil/hostap/Kconfig
deleted file mode 100644
index 2edff8efbcbb..000000000000
--- a/drivers/net/wireless/intersil/hostap/Kconfig
+++ /dev/null
@@ -1,95 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-config HOSTAP
-	tristate "IEEE 802.11 for Host AP (Prism2/2.5/3 and WEP/TKIP/CCMP)"
-	select WIRELESS_EXT
-	select WEXT_SPY
-	select WEXT_PRIV
-	select CRYPTO
-	select CRYPTO_MICHAEL_MIC
-	select CRC32
-	select LIB80211
-	select LIB80211_CRYPT_WEP
-	select LIB80211_CRYPT_TKIP
-	select LIB80211_CRYPT_CCMP
-	help
-	Shared driver code for IEEE 802.11b wireless cards based on
-	Intersil Prism2/2.5/3 chipset. This driver supports so called
-	Host AP mode that allows the card to act as an IEEE 802.11
-	access point.
-
-	See <http://hostap.epitest.fi/> for more information about the
-	Host AP driver configuration and tools. This site includes
-	information and tools (hostapd and wpa_supplicant) for WPA/WPA2
-	support.
-
-	This option includes the base Host AP driver code that is shared by
-	different hardware models. You will also need to enable support for
-	PLX/PCI/CS version of the driver to actually use the driver.
-
-	The driver can be compiled as a module and it will be called
-	hostap.
-
-config HOSTAP_FIRMWARE
-	bool "Support downloading firmware images with Host AP driver"
-	depends on HOSTAP
-	help
-	Configure Host AP driver to include support for firmware image
-	download. This option by itself only enables downloading to the
-	volatile memory, i.e. the card RAM. This option is required to
-	support cards that don't have firmware in flash, such as D-Link
-	DWL-520 rev E and D-Link DWL-650 rev P.
-
-	Firmware image downloading needs a user space tool, prism2_srec.
-	It is available from http://hostap.epitest.fi/.
-
-config HOSTAP_FIRMWARE_NVRAM
-	bool "Support for non-volatile firmware download"
-	depends on HOSTAP_FIRMWARE
-	help
-	Allow Host AP driver to write firmware images to the non-volatile
-	card memory, i.e. flash memory that survives power cycling.
-	Enable this option if you want to be able to change card firmware
-	permanently.
-
-	Firmware image downloading needs a user space tool, prism2_srec.
-	It is available from http://hostap.epitest.fi/.
-
-config HOSTAP_PLX
-	tristate "Host AP driver for Prism2/2.5/3 in PLX9052 PCI adaptors"
-	depends on PCI && HOSTAP && HAS_IOPORT
-	help
-	Host AP driver's version for Prism2/2.5/3 PC Cards in PLX9052 based
-	PCI adaptors.
-
-	"Host AP support for Prism2/2.5/3 IEEE 802.11b" is required for this
-	driver and its help text includes more information about the Host AP
-	driver.
-
-	The driver can be compiled as a module and will be named
-	hostap_plx.
-
-config HOSTAP_PCI
-	tristate "Host AP driver for Prism2.5 PCI adaptors"
-	depends on PCI && HOSTAP
-	help
-	Host AP driver's version for Prism2.5 PCI adaptors.
-
-	"Host AP support for Prism2/2.5/3 IEEE 802.11b" is required for this
-	driver and its help text includes more information about the Host AP
-	driver.
-
-	The driver can be compiled as a module and will be named
-	hostap_pci.
-
-config HOSTAP_CS
-	tristate "Host AP driver for Prism2/2.5/3 PC Cards"
-	depends on PCMCIA && HOSTAP
-	help
-	Host AP driver's version for Prism2/2.5/3 PC Cards.
-
-	"Host AP support for Prism2/2.5/3 IEEE 802.11b" is required for this
-	driver and its help text includes more information about the Host AP
-	driver.
-
-	The driver can be compiled as a module and will be named
-	hostap_cs.
diff --git a/drivers/net/wireless/intersil/hostap/Makefile b/drivers/net/wireless/intersil/hostap/Makefile
deleted file mode 100644
index ae3bb73b2d99..000000000000
--- a/drivers/net/wireless/intersil/hostap/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-hostap-y := hostap_80211_rx.o hostap_80211_tx.o hostap_ap.o hostap_info.o \
-            hostap_ioctl.o hostap_main.o hostap_proc.o 
-obj-$(CONFIG_HOSTAP) += hostap.o
-
-obj-$(CONFIG_HOSTAP_CS) += hostap_cs.o
-obj-$(CONFIG_HOSTAP_PLX) += hostap_plx.o
-obj-$(CONFIG_HOSTAP_PCI) += hostap_pci.o
diff --git a/drivers/net/wireless/intersil/hostap/hostap.h b/drivers/net/wireless/intersil/hostap/hostap.h
deleted file mode 100644
index 552ae33d7875..000000000000
--- a/drivers/net/wireless/intersil/hostap/hostap.h
+++ /dev/null
@@ -1,98 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef HOSTAP_H
-#define HOSTAP_H
-
-#include <linux/ethtool.h>
-#include <linux/kernel.h>
-
-#include "hostap_wlan.h"
-#include "hostap_ap.h"
-
-static const long __maybe_unused freq_list[] = {
-	2412, 2417, 2422, 2427, 2432, 2437, 2442,
-	2447, 2452, 2457, 2462, 2467, 2472, 2484
-};
-#define FREQ_COUNT ARRAY_SIZE(freq_list)
-
-/* hostap.c */
-
-extern struct proc_dir_entry *hostap_proc;
-
-u16 hostap_tx_callback_register(local_info_t *local,
-				void (*func)(struct sk_buff *, int ok, void *),
-				void *data);
-int hostap_tx_callback_unregister(local_info_t *local, u16 idx);
-int hostap_set_word(struct net_device *dev, int rid, u16 val);
-int hostap_set_string(struct net_device *dev, int rid, const char *val);
-u16 hostap_get_porttype(local_info_t *local);
-int hostap_set_encryption(local_info_t *local);
-int hostap_set_antsel(local_info_t *local);
-int hostap_set_roaming(local_info_t *local);
-int hostap_set_auth_algs(local_info_t *local);
-void hostap_dump_rx_header(const char *name,
-			   const struct hfa384x_rx_frame *rx);
-void hostap_dump_tx_header(const char *name,
-			   const struct hfa384x_tx_frame *tx);
-extern const struct header_ops hostap_80211_ops;
-int hostap_80211_get_hdrlen(__le16 fc);
-struct net_device_stats *hostap_get_stats(struct net_device *dev);
-void hostap_setup_dev(struct net_device *dev, local_info_t *local,
-		      int type);
-void hostap_set_multicast_list_queue(struct work_struct *work);
-int hostap_set_hostapd(local_info_t *local, int val, int rtnl_locked);
-int hostap_set_hostapd_sta(local_info_t *local, int val, int rtnl_locked);
-void hostap_cleanup(local_info_t *local);
-void hostap_cleanup_handler(void *data);
-struct net_device * hostap_add_interface(struct local_info *local,
-					 int type, int rtnl_locked,
-					 const char *prefix, const char *name);
-void hostap_remove_interface(struct net_device *dev, int rtnl_locked,
-			     int remove_from_list);
-int prism2_update_comms_qual(struct net_device *dev);
-int prism2_sta_send_mgmt(local_info_t *local, u8 *dst, u16 stype,
-			 u8 *body, size_t bodylen);
-int prism2_sta_deauth(local_info_t *local, u16 reason);
-int prism2_wds_add(local_info_t *local, u8 *remote_addr,
-		   int rtnl_locked);
-int prism2_wds_del(local_info_t *local, u8 *remote_addr,
-		   int rtnl_locked, int do_not_remove);
-
-
-/* hostap_ap.c */
-
-int ap_control_add_mac(struct mac_restrictions *mac_restrictions, u8 *mac);
-int ap_control_del_mac(struct mac_restrictions *mac_restrictions, u8 *mac);
-void ap_control_flush_macs(struct mac_restrictions *mac_restrictions);
-int ap_control_kick_mac(struct ap_data *ap, struct net_device *dev, u8 *mac);
-void ap_control_kickall(struct ap_data *ap);
-void * ap_crypt_get_ptrs(struct ap_data *ap, u8 *addr, int permanent,
-			 struct lib80211_crypt_data ***crypt);
-int prism2_ap_get_sta_qual(local_info_t *local, struct sockaddr addr[],
-			   struct iw_quality qual[], int buf_size,
-			   int aplist);
-int prism2_ap_translate_scan(struct net_device *dev,
-			     struct iw_request_info *info, char *buffer);
-int prism2_hostapd(struct ap_data *ap, struct prism2_hostapd_param *param);
-
-
-/* hostap_proc.c */
-
-void hostap_init_proc(local_info_t *local);
-void hostap_remove_proc(local_info_t *local);
-
-
-/* hostap_info.c */
-
-void hostap_info_init(local_info_t *local);
-void hostap_info_process(local_info_t *local, struct sk_buff *skb);
-
-
-/* hostap_ioctl.c */
-
-extern const struct iw_handler_def hostap_iw_handler_def;
-extern const struct ethtool_ops prism2_ethtool_ops;
-
-int hostap_siocdevprivate(struct net_device *dev, struct ifreq *ifr,
-			  void __user *data, int cmd);
-
-#endif /* HOSTAP_H */
diff --git a/drivers/net/wireless/intersil/hostap/hostap_80211.h b/drivers/net/wireless/intersil/hostap/hostap_80211.h
deleted file mode 100644
index 1452cf6ecb07..000000000000
--- a/drivers/net/wireless/intersil/hostap/hostap_80211.h
+++ /dev/null
@@ -1,97 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef HOSTAP_80211_H
-#define HOSTAP_80211_H
-
-#include <linux/types.h>
-#include <linux/skbuff.h>
-#include <linux/netdevice.h>
-
-struct hostap_ieee80211_mgmt {
-	__le16 frame_control;
-	__le16 duration;
-	u8 da[6];
-	u8 sa[6];
-	u8 bssid[6];
-	__le16 seq_ctrl;
-	union {
-		struct {
-			__le16 auth_alg;
-			__le16 auth_transaction;
-			__le16 status_code;
-			/* possibly followed by Challenge text */
-			u8 variable[0];
-		} __packed auth;
-		struct {
-			__le16 reason_code;
-		} __packed deauth;
-		struct {
-			__le16 capab_info;
-			__le16 listen_interval;
-			/* followed by SSID and Supported rates */
-			u8 variable[0];
-		} __packed assoc_req;
-		struct {
-			__le16 capab_info;
-			__le16 status_code;
-			__le16 aid;
-			/* followed by Supported rates */
-			u8 variable[0];
-		} __packed assoc_resp, reassoc_resp;
-		struct {
-			__le16 capab_info;
-			__le16 listen_interval;
-			u8 current_ap[6];
-			/* followed by SSID and Supported rates */
-			u8 variable[0];
-		} __packed reassoc_req;
-		struct {
-			__le16 reason_code;
-		} __packed disassoc;
-		struct {
-		} __packed probe_req;
-		struct {
-			u8 timestamp[8];
-			__le16 beacon_int;
-			__le16 capab_info;
-			/* followed by some of SSID, Supported rates,
-			 * FH Params, DS Params, CF Params, IBSS Params, TIM */
-			u8 variable[0];
-		} __packed beacon, probe_resp;
-	} u;
-} __packed;
-
-
-#define IEEE80211_MGMT_HDR_LEN 24
-#define IEEE80211_DATA_HDR3_LEN 24
-#define IEEE80211_DATA_HDR4_LEN 30
-
-
-struct hostap_80211_rx_status {
-	u32 mac_time;
-	u8 signal;
-	u8 noise;
-	u16 rate; /* in 100 kbps */
-};
-
-/* prism2_rx_80211 'type' argument */
-enum {
-	PRISM2_RX_MONITOR, PRISM2_RX_MGMT, PRISM2_RX_NON_ASSOC,
-	PRISM2_RX_NULLFUNC_ACK
-};
-
-int prism2_rx_80211(struct net_device *dev, struct sk_buff *skb,
-		    struct hostap_80211_rx_status *rx_stats, int type);
-void hostap_80211_rx(struct net_device *dev, struct sk_buff *skb,
-		     struct hostap_80211_rx_status *rx_stats);
-void hostap_dump_rx_80211(const char *name, struct sk_buff *skb,
-			  struct hostap_80211_rx_status *rx_stats);
-
-void hostap_dump_tx_80211(const char *name, struct sk_buff *skb);
-netdev_tx_t hostap_data_start_xmit(struct sk_buff *skb,
-				   struct net_device *dev);
-netdev_tx_t hostap_mgmt_start_xmit(struct sk_buff *skb,
-				   struct net_device *dev);
-netdev_tx_t hostap_master_start_xmit(struct sk_buff *skb,
-				     struct net_device *dev);
-
-#endif /* HOSTAP_80211_H */
diff --git a/drivers/net/wireless/intersil/hostap/hostap_80211_rx.c b/drivers/net/wireless/intersil/hostap/hostap_80211_rx.c
deleted file mode 100644
index 61be822f90b5..000000000000
--- a/drivers/net/wireless/intersil/hostap/hostap_80211_rx.c
+++ /dev/null
@@ -1,1116 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-#include <linux/etherdevice.h>
-#include <linux/slab.h>
-#include <linux/export.h>
-#include <net/lib80211.h>
-#include <linux/if_arp.h>
-
-#include "hostap_80211.h"
-#include "hostap.h"
-#include "hostap_ap.h"
-
-/* See IEEE 802.1H for LLC/SNAP encapsulation/decapsulation */
-/* Ethernet-II snap header (RFC1042 for most EtherTypes) */
-static unsigned char rfc1042_header[] =
-{ 0xaa, 0xaa, 0x03, 0x00, 0x00, 0x00 };
-/* Bridge-Tunnel header (for EtherTypes ETH_P_AARP and ETH_P_IPX) */
-static unsigned char bridge_tunnel_header[] =
-{ 0xaa, 0xaa, 0x03, 0x00, 0x00, 0xf8 };
-/* No encapsulation header if EtherType < 0x600 (=length) */
-
-void hostap_dump_rx_80211(const char *name, struct sk_buff *skb,
-			  struct hostap_80211_rx_status *rx_stats)
-{
-	struct ieee80211_hdr *hdr;
-	u16 fc;
-
-	hdr = (struct ieee80211_hdr *) skb->data;
-
-	printk(KERN_DEBUG "%s: RX signal=%d noise=%d rate=%d len=%d "
-	       "jiffies=%ld\n",
-	       name, rx_stats->signal, rx_stats->noise, rx_stats->rate,
-	       skb->len, jiffies);
-
-	if (skb->len < 2)
-		return;
-
-	fc = le16_to_cpu(hdr->frame_control);
-	printk(KERN_DEBUG "   FC=0x%04x (type=%d:%d)%s%s",
-	       fc, (fc & IEEE80211_FCTL_FTYPE) >> 2,
-	       (fc & IEEE80211_FCTL_STYPE) >> 4,
-	       fc & IEEE80211_FCTL_TODS ? " [ToDS]" : "",
-	       fc & IEEE80211_FCTL_FROMDS ? " [FromDS]" : "");
-
-	if (skb->len < IEEE80211_DATA_HDR3_LEN) {
-		printk("\n");
-		return;
-	}
-
-	printk(" dur=0x%04x seq=0x%04x\n", le16_to_cpu(hdr->duration_id),
-	       le16_to_cpu(hdr->seq_ctrl));
-
-	printk(KERN_DEBUG "   A1=%pM", hdr->addr1);
-	printk(" A2=%pM", hdr->addr2);
-	printk(" A3=%pM", hdr->addr3);
-	if (skb->len >= 30)
-		printk(" A4=%pM", hdr->addr4);
-	printk("\n");
-}
-
-
-/* Send RX frame to netif with 802.11 (and possible prism) header.
- * Called from hardware or software IRQ context. */
-int prism2_rx_80211(struct net_device *dev, struct sk_buff *skb,
-		    struct hostap_80211_rx_status *rx_stats, int type)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int hdrlen, phdrlen, head_need, tail_need;
-	u16 fc;
-	int prism_header, ret;
-	struct ieee80211_hdr *fhdr;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (dev->type == ARPHRD_IEEE80211_PRISM) {
-		if (local->monitor_type == PRISM2_MONITOR_PRISM) {
-			prism_header = 1;
-			phdrlen = sizeof(struct linux_wlan_ng_prism_hdr);
-		} else { /* local->monitor_type == PRISM2_MONITOR_CAPHDR */
-			prism_header = 2;
-			phdrlen = sizeof(struct linux_wlan_ng_cap_hdr);
-		}
-	} else if (dev->type == ARPHRD_IEEE80211_RADIOTAP) {
-		prism_header = 3;
-		phdrlen = sizeof(struct hostap_radiotap_rx);
-	} else {
-		prism_header = 0;
-		phdrlen = 0;
-	}
-
-	fhdr = (struct ieee80211_hdr *) skb->data;
-	fc = le16_to_cpu(fhdr->frame_control);
-
-	if (type == PRISM2_RX_MGMT && (fc & IEEE80211_FCTL_VERS)) {
-		printk(KERN_DEBUG "%s: dropped management frame with header "
-		       "version %d\n", dev->name, fc & IEEE80211_FCTL_VERS);
-		dev_kfree_skb_any(skb);
-		return 0;
-	}
-
-	hdrlen = hostap_80211_get_hdrlen(fhdr->frame_control);
-
-	/* check if there is enough room for extra data; if not, expand skb
-	 * buffer to be large enough for the changes */
-	head_need = phdrlen;
-	tail_need = 0;
-#ifdef PRISM2_ADD_BOGUS_CRC
-	tail_need += 4;
-#endif /* PRISM2_ADD_BOGUS_CRC */
-
-	head_need -= skb_headroom(skb);
-	tail_need -= skb_tailroom(skb);
-
-	if (head_need > 0 || tail_need > 0) {
-		if (pskb_expand_head(skb, head_need > 0 ? head_need : 0,
-				     tail_need > 0 ? tail_need : 0,
-				     GFP_ATOMIC)) {
-			printk(KERN_DEBUG "%s: prism2_rx_80211 failed to "
-			       "reallocate skb buffer\n", dev->name);
-			dev_kfree_skb_any(skb);
-			return 0;
-		}
-	}
-
-	/* We now have an skb with enough head and tail room, so just insert
-	 * the extra data */
-
-#ifdef PRISM2_ADD_BOGUS_CRC
-	memset(skb_put(skb, 4), 0xff, 4); /* Prism2 strips CRC */
-#endif /* PRISM2_ADD_BOGUS_CRC */
-
-	if (prism_header == 1) {
-		struct linux_wlan_ng_prism_hdr *hdr;
-		hdr = skb_push(skb, phdrlen);
-		memset(hdr, 0, phdrlen);
-		hdr->msgcode = LWNG_CAP_DID_BASE;
-		hdr->msglen = sizeof(*hdr);
-		memcpy(hdr->devname, dev->name, sizeof(hdr->devname));
-#define LWNG_SETVAL(f,i,s,l,d) \
-hdr->f.did = LWNG_CAP_DID_BASE | (i << 12); \
-hdr->f.status = s; hdr->f.len = l; hdr->f.data = d
-		LWNG_SETVAL(hosttime, 1, 0, 4, jiffies);
-		LWNG_SETVAL(mactime, 2, 0, 4, rx_stats->mac_time);
-		LWNG_SETVAL(channel, 3, 1 /* no value */, 4, 0);
-		LWNG_SETVAL(rssi, 4, 1 /* no value */, 4, 0);
-		LWNG_SETVAL(sq, 5, 1 /* no value */, 4, 0);
-		LWNG_SETVAL(signal, 6, 0, 4, rx_stats->signal);
-		LWNG_SETVAL(noise, 7, 0, 4, rx_stats->noise);
-		LWNG_SETVAL(rate, 8, 0, 4, rx_stats->rate / 5);
-		LWNG_SETVAL(istx, 9, 0, 4, 0);
-		LWNG_SETVAL(frmlen, 10, 0, 4, skb->len - phdrlen);
-#undef LWNG_SETVAL
-	} else if (prism_header == 2) {
-		struct linux_wlan_ng_cap_hdr *hdr;
-		hdr = skb_push(skb, phdrlen);
-		memset(hdr, 0, phdrlen);
-		hdr->version    = htonl(LWNG_CAPHDR_VERSION);
-		hdr->length     = htonl(phdrlen);
-		hdr->mactime    = __cpu_to_be64(rx_stats->mac_time);
-		hdr->hosttime   = __cpu_to_be64(jiffies);
-		hdr->phytype    = htonl(4); /* dss_dot11_b */
-		hdr->channel    = htonl(local->channel);
-		hdr->datarate   = htonl(rx_stats->rate);
-		hdr->antenna    = htonl(0); /* unknown */
-		hdr->priority   = htonl(0); /* unknown */
-		hdr->ssi_type   = htonl(3); /* raw */
-		hdr->ssi_signal = htonl(rx_stats->signal);
-		hdr->ssi_noise  = htonl(rx_stats->noise);
-		hdr->preamble   = htonl(0); /* unknown */
-		hdr->encoding   = htonl(1); /* cck */
-	} else if (prism_header == 3) {
-		struct hostap_radiotap_rx *hdr;
-		hdr = skb_push(skb, phdrlen);
-		memset(hdr, 0, phdrlen);
-		hdr->hdr.it_len = cpu_to_le16(phdrlen);
-		hdr->hdr.it_present =
-			cpu_to_le32((1 << IEEE80211_RADIOTAP_TSFT) |
-				    (1 << IEEE80211_RADIOTAP_CHANNEL) |
-				    (1 << IEEE80211_RADIOTAP_RATE) |
-				    (1 << IEEE80211_RADIOTAP_DBM_ANTSIGNAL) |
-				    (1 << IEEE80211_RADIOTAP_DBM_ANTNOISE));
-		hdr->tsft = cpu_to_le64(rx_stats->mac_time);
-		hdr->chan_freq = cpu_to_le16(freq_list[local->channel - 1]);
-		hdr->chan_flags = cpu_to_le16(IEEE80211_CHAN_CCK |
-						 IEEE80211_CHAN_2GHZ);
-		hdr->rate = rx_stats->rate / 5;
-		hdr->dbm_antsignal = rx_stats->signal;
-		hdr->dbm_antnoise = rx_stats->noise;
-	}
-
-	ret = skb->len - phdrlen;
-	skb->dev = dev;
-	skb_reset_mac_header(skb);
-	skb_pull(skb, hdrlen);
-	if (prism_header)
-		skb_pull(skb, phdrlen);
-	skb->pkt_type = PACKET_OTHERHOST;
-	skb->protocol = cpu_to_be16(ETH_P_802_2);
-	memset(skb->cb, 0, sizeof(skb->cb));
-	netif_rx(skb);
-
-	return ret;
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static void monitor_rx(struct net_device *dev, struct sk_buff *skb,
-		       struct hostap_80211_rx_status *rx_stats)
-{
-	int len;
-
-	len = prism2_rx_80211(dev, skb, rx_stats, PRISM2_RX_MONITOR);
-	dev->stats.rx_packets++;
-	dev->stats.rx_bytes += len;
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static struct prism2_frag_entry *
-prism2_frag_cache_find(local_info_t *local, unsigned int seq,
-		       unsigned int frag, u8 *src, u8 *dst)
-{
-	struct prism2_frag_entry *entry;
-	int i;
-
-	for (i = 0; i < PRISM2_FRAG_CACHE_LEN; i++) {
-		entry = &local->frag_cache[i];
-		if (entry->skb != NULL &&
-		    time_after(jiffies, entry->first_frag_time + 2 * HZ)) {
-			printk(KERN_DEBUG "%s: expiring fragment cache entry "
-			       "seq=%u last_frag=%u\n",
-			       local->dev->name, entry->seq, entry->last_frag);
-			dev_kfree_skb(entry->skb);
-			entry->skb = NULL;
-		}
-
-		if (entry->skb != NULL && entry->seq == seq &&
-		    (entry->last_frag + 1 == frag || frag == -1) &&
-		    memcmp(entry->src_addr, src, ETH_ALEN) == 0 &&
-		    memcmp(entry->dst_addr, dst, ETH_ALEN) == 0)
-			return entry;
-	}
-
-	return NULL;
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static struct sk_buff *
-prism2_frag_cache_get(local_info_t *local, struct ieee80211_hdr *hdr)
-{
-	struct sk_buff *skb = NULL;
-	u16 sc;
-	unsigned int frag, seq;
-	struct prism2_frag_entry *entry;
-
-	sc = le16_to_cpu(hdr->seq_ctrl);
-	frag = sc & IEEE80211_SCTL_FRAG;
-	seq = (sc & IEEE80211_SCTL_SEQ) >> 4;
-
-	if (frag == 0) {
-		/* Reserve enough space to fit maximum frame length */
-		skb = dev_alloc_skb(local->dev->mtu +
-				    sizeof(struct ieee80211_hdr) +
-				    8 /* LLC */ +
-				    2 /* alignment */ +
-				    8 /* WEP */ + ETH_ALEN /* WDS */);
-		if (skb == NULL)
-			return NULL;
-
-		entry = &local->frag_cache[local->frag_next_idx];
-		local->frag_next_idx++;
-		if (local->frag_next_idx >= PRISM2_FRAG_CACHE_LEN)
-			local->frag_next_idx = 0;
-
-		if (entry->skb != NULL)
-			dev_kfree_skb(entry->skb);
-
-		entry->first_frag_time = jiffies;
-		entry->seq = seq;
-		entry->last_frag = frag;
-		entry->skb = skb;
-		memcpy(entry->src_addr, hdr->addr2, ETH_ALEN);
-		memcpy(entry->dst_addr, hdr->addr1, ETH_ALEN);
-	} else {
-		/* received a fragment of a frame for which the head fragment
-		 * should have already been received */
-		entry = prism2_frag_cache_find(local, seq, frag, hdr->addr2,
-					       hdr->addr1);
-		if (entry != NULL) {
-			entry->last_frag = frag;
-			skb = entry->skb;
-		}
-	}
-
-	return skb;
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static int prism2_frag_cache_invalidate(local_info_t *local,
-					struct ieee80211_hdr *hdr)
-{
-	u16 sc;
-	unsigned int seq;
-	struct prism2_frag_entry *entry;
-
-	sc = le16_to_cpu(hdr->seq_ctrl);
-	seq = (sc & IEEE80211_SCTL_SEQ) >> 4;
-
-	entry = prism2_frag_cache_find(local, seq, -1, hdr->addr2, hdr->addr1);
-
-	if (entry == NULL) {
-		printk(KERN_DEBUG "%s: could not invalidate fragment cache "
-		       "entry (seq=%u)\n",
-		       local->dev->name, seq);
-		return -1;
-	}
-
-	entry->skb = NULL;
-	return 0;
-}
-
-
-static struct hostap_bss_info *__hostap_get_bss(local_info_t *local, u8 *bssid,
-						u8 *ssid, size_t ssid_len)
-{
-	struct list_head *ptr;
-	struct hostap_bss_info *bss;
-
-	list_for_each(ptr, &local->bss_list) {
-		bss = list_entry(ptr, struct hostap_bss_info, list);
-		if (memcmp(bss->bssid, bssid, ETH_ALEN) == 0 &&
-		    (ssid == NULL ||
-		     (ssid_len == bss->ssid_len &&
-		      memcmp(ssid, bss->ssid, ssid_len) == 0))) {
-			list_move(&bss->list, &local->bss_list);
-			return bss;
-		}
-	}
-
-	return NULL;
-}
-
-
-static struct hostap_bss_info *__hostap_add_bss(local_info_t *local, u8 *bssid,
-						u8 *ssid, size_t ssid_len)
-{
-	struct hostap_bss_info *bss;
-
-	if (local->num_bss_info >= HOSTAP_MAX_BSS_COUNT) {
-		bss = list_entry(local->bss_list.prev,
-				 struct hostap_bss_info, list);
-		list_del(&bss->list);
-		local->num_bss_info--;
-	} else {
-		bss = kmalloc(sizeof(*bss), GFP_ATOMIC);
-		if (bss == NULL)
-			return NULL;
-	}
-
-	memset(bss, 0, sizeof(*bss));
-	memcpy(bss->bssid, bssid, ETH_ALEN);
-	memcpy(bss->ssid, ssid, ssid_len);
-	bss->ssid_len = ssid_len;
-	local->num_bss_info++;
-	list_add(&bss->list, &local->bss_list);
-	return bss;
-}
-
-
-static void __hostap_expire_bss(local_info_t *local)
-{
-	struct hostap_bss_info *bss;
-
-	while (local->num_bss_info > 0) {
-		bss = list_entry(local->bss_list.prev,
-				 struct hostap_bss_info, list);
-		if (!time_after(jiffies, bss->last_update + 60 * HZ))
-			break;
-
-		list_del(&bss->list);
-		local->num_bss_info--;
-		kfree(bss);
-	}
-}
-
-
-/* Both IEEE 802.11 Beacon and Probe Response frames have similar structure, so
- * the same routine can be used to parse both of them. */
-static void hostap_rx_sta_beacon(local_info_t *local, struct sk_buff *skb,
-				 int stype)
-{
-	struct hostap_ieee80211_mgmt *mgmt;
-	int left, chan = 0;
-	u8 *pos;
-	u8 *ssid = NULL, *wpa = NULL, *rsn = NULL;
-	size_t ssid_len = 0, wpa_len = 0, rsn_len = 0;
-	struct hostap_bss_info *bss;
-
-	if (skb->len < IEEE80211_MGMT_HDR_LEN + sizeof(mgmt->u.beacon))
-		return;
-
-	mgmt = (struct hostap_ieee80211_mgmt *) skb->data;
-	pos = mgmt->u.beacon.variable;
-	left = skb->len - (pos - skb->data);
-
-	while (left >= 2) {
-		if (2 + pos[1] > left)
-			return; /* parse failed */
-		switch (*pos) {
-		case WLAN_EID_SSID:
-			ssid = pos + 2;
-			ssid_len = pos[1];
-			break;
-		case WLAN_EID_VENDOR_SPECIFIC:
-			if (pos[1] >= 4 &&
-			    pos[2] == 0x00 && pos[3] == 0x50 &&
-			    pos[4] == 0xf2 && pos[5] == 1) {
-				wpa = pos;
-				wpa_len = pos[1] + 2;
-			}
-			break;
-		case WLAN_EID_RSN:
-			rsn = pos;
-			rsn_len = pos[1] + 2;
-			break;
-		case WLAN_EID_DS_PARAMS:
-			if (pos[1] >= 1)
-				chan = pos[2];
-			break;
-		}
-		left -= 2 + pos[1];
-		pos += 2 + pos[1];
-	}
-
-	if (wpa_len > MAX_WPA_IE_LEN)
-		wpa_len = MAX_WPA_IE_LEN;
-	if (rsn_len > MAX_WPA_IE_LEN)
-		rsn_len = MAX_WPA_IE_LEN;
-	if (ssid_len > sizeof(bss->ssid))
-		ssid_len = sizeof(bss->ssid);
-
-	spin_lock(&local->lock);
-	bss = __hostap_get_bss(local, mgmt->bssid, ssid, ssid_len);
-	if (bss == NULL)
-		bss = __hostap_add_bss(local, mgmt->bssid, ssid, ssid_len);
-	if (bss) {
-		bss->last_update = jiffies;
-		bss->count++;
-		bss->capab_info = le16_to_cpu(mgmt->u.beacon.capab_info);
-		if (wpa) {
-			memcpy(bss->wpa_ie, wpa, wpa_len);
-			bss->wpa_ie_len = wpa_len;
-		} else
-			bss->wpa_ie_len = 0;
-		if (rsn) {
-			memcpy(bss->rsn_ie, rsn, rsn_len);
-			bss->rsn_ie_len = rsn_len;
-		} else
-			bss->rsn_ie_len = 0;
-		bss->chan = chan;
-	}
-	__hostap_expire_bss(local);
-	spin_unlock(&local->lock);
-}
-
-
-static int
-hostap_rx_frame_mgmt(local_info_t *local, struct sk_buff *skb,
-		     struct hostap_80211_rx_status *rx_stats, u16 type,
-		     u16 stype)
-{
-	if (local->iw_mode == IW_MODE_MASTER)
-		hostap_update_sta_ps(local, (struct ieee80211_hdr *) skb->data);
-
-	if (local->hostapd && type == IEEE80211_FTYPE_MGMT) {
-		if (stype == IEEE80211_STYPE_BEACON &&
-		    local->iw_mode == IW_MODE_MASTER) {
-			struct sk_buff *skb2;
-			/* Process beacon frames also in kernel driver to
-			 * update STA(AP) table statistics */
-			skb2 = skb_clone(skb, GFP_ATOMIC);
-			if (skb2)
-				hostap_rx(skb2->dev, skb2, rx_stats);
-		}
-
-		/* send management frames to the user space daemon for
-		 * processing */
-		local->apdevstats.rx_packets++;
-		local->apdevstats.rx_bytes += skb->len;
-		if (local->apdev == NULL)
-			return -1;
-		prism2_rx_80211(local->apdev, skb, rx_stats, PRISM2_RX_MGMT);
-		return 0;
-	}
-
-	if (local->iw_mode == IW_MODE_MASTER) {
-		if (type != IEEE80211_FTYPE_MGMT &&
-		    type != IEEE80211_FTYPE_CTL) {
-			printk(KERN_DEBUG "%s: unknown management frame "
-			       "(type=0x%02x, stype=0x%02x) dropped\n",
-			       skb->dev->name, type >> 2, stype >> 4);
-			return -1;
-		}
-
-		hostap_rx(skb->dev, skb, rx_stats);
-		return 0;
-	} else if (type == IEEE80211_FTYPE_MGMT &&
-		   (stype == IEEE80211_STYPE_BEACON ||
-		    stype == IEEE80211_STYPE_PROBE_RESP)) {
-		hostap_rx_sta_beacon(local, skb, stype);
-		return -1;
-	} else if (type == IEEE80211_FTYPE_MGMT &&
-		   (stype == IEEE80211_STYPE_ASSOC_RESP ||
-		    stype == IEEE80211_STYPE_REASSOC_RESP)) {
-		/* Ignore (Re)AssocResp silently since these are not currently
-		 * needed but are still received when WPA/RSN mode is enabled.
-		 */
-		return -1;
-	} else {
-		printk(KERN_DEBUG "%s: hostap_rx_frame_mgmt: dropped unhandled"
-		       " management frame in non-Host AP mode (type=%d:%d)\n",
-		       skb->dev->name, type >> 2, stype >> 4);
-		return -1;
-	}
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static struct net_device *prism2_rx_get_wds(local_info_t *local,
-						   u8 *addr)
-{
-	struct hostap_interface *iface = NULL;
-	struct list_head *ptr;
-
-	read_lock_bh(&local->iface_lock);
-	list_for_each(ptr, &local->hostap_interfaces) {
-		iface = list_entry(ptr, struct hostap_interface, list);
-		if (iface->type == HOSTAP_INTERFACE_WDS &&
-		    memcmp(iface->u.wds.remote_addr, addr, ETH_ALEN) == 0)
-			break;
-		iface = NULL;
-	}
-	read_unlock_bh(&local->iface_lock);
-
-	return iface ? iface->dev : NULL;
-}
-
-
-static int
-hostap_rx_frame_wds(local_info_t *local, struct ieee80211_hdr *hdr, u16 fc,
-		    struct net_device **wds)
-{
-	/* FIX: is this really supposed to accept WDS frames only in Master
-	 * mode? What about Repeater or Managed with WDS frames? */
-	if ((fc & (IEEE80211_FCTL_TODS | IEEE80211_FCTL_FROMDS)) !=
-	    (IEEE80211_FCTL_TODS | IEEE80211_FCTL_FROMDS) &&
-	    (local->iw_mode != IW_MODE_MASTER || !(fc & IEEE80211_FCTL_TODS)))
-		return 0; /* not a WDS frame */
-
-	/* Possible WDS frame: either IEEE 802.11 compliant (if FromDS)
-	 * or own non-standard frame with 4th address after payload */
-	if (!ether_addr_equal(hdr->addr1, local->dev->dev_addr) &&
-	    (hdr->addr1[0] != 0xff || hdr->addr1[1] != 0xff ||
-	     hdr->addr1[2] != 0xff || hdr->addr1[3] != 0xff ||
-	     hdr->addr1[4] != 0xff || hdr->addr1[5] != 0xff)) {
-		/* RA (or BSSID) is not ours - drop */
-		PDEBUG(DEBUG_EXTRA2, "%s: received WDS frame with "
-		       "not own or broadcast %s=%pM\n",
-		       local->dev->name,
-		       fc & IEEE80211_FCTL_FROMDS ? "RA" : "BSSID",
-		       hdr->addr1);
-		return -1;
-	}
-
-	/* check if the frame came from a registered WDS connection */
-	*wds = prism2_rx_get_wds(local, hdr->addr2);
-	if (*wds == NULL && fc & IEEE80211_FCTL_FROMDS &&
-	    (local->iw_mode != IW_MODE_INFRA ||
-	     !(local->wds_type & HOSTAP_WDS_AP_CLIENT) ||
-	     memcmp(hdr->addr2, local->bssid, ETH_ALEN) != 0)) {
-		/* require that WDS link has been registered with TA or the
-		 * frame is from current AP when using 'AP client mode' */
-		PDEBUG(DEBUG_EXTRA, "%s: received WDS[4 addr] frame "
-		       "from unknown TA=%pM\n",
-		       local->dev->name, hdr->addr2);
-		if (local->ap && local->ap->autom_ap_wds)
-			hostap_wds_link_oper(local, hdr->addr2, WDS_ADD);
-		return -1;
-	}
-
-	if (*wds && !(fc & IEEE80211_FCTL_FROMDS) && local->ap &&
-	    hostap_is_sta_assoc(local->ap, hdr->addr2)) {
-		/* STA is actually associated with us even though it has a
-		 * registered WDS link. Assume it is in 'AP client' mode.
-		 * Since this is a 3-addr frame, assume it is not (bogus) WDS
-		 * frame and process it like any normal ToDS frame from
-		 * associated STA. */
-		*wds = NULL;
-	}
-
-	return 0;
-}
-
-
-static int hostap_is_eapol_frame(local_info_t *local, struct sk_buff *skb)
-{
-	struct net_device *dev = local->dev;
-	u16 fc, ethertype;
-	struct ieee80211_hdr *hdr;
-	u8 *pos;
-
-	if (skb->len < 24)
-		return 0;
-
-	hdr = (struct ieee80211_hdr *) skb->data;
-	fc = le16_to_cpu(hdr->frame_control);
-
-	/* check that the frame is unicast frame to us */
-	if ((fc & (IEEE80211_FCTL_TODS | IEEE80211_FCTL_FROMDS)) ==
-	    IEEE80211_FCTL_TODS &&
-	    ether_addr_equal(hdr->addr1, dev->dev_addr) &&
-	    ether_addr_equal(hdr->addr3, dev->dev_addr)) {
-		/* ToDS frame with own addr BSSID and DA */
-	} else if ((fc & (IEEE80211_FCTL_TODS | IEEE80211_FCTL_FROMDS)) ==
-		   IEEE80211_FCTL_FROMDS &&
-		   ether_addr_equal(hdr->addr1, dev->dev_addr)) {
-		/* FromDS frame with own addr as DA */
-	} else
-		return 0;
-
-	if (skb->len < 24 + 8)
-		return 0;
-
-	/* check for port access entity Ethernet type */
-	pos = skb->data + 24;
-	ethertype = (pos[6] << 8) | pos[7];
-	if (ethertype == ETH_P_PAE)
-		return 1;
-
-	return 0;
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static int
-hostap_rx_frame_decrypt(local_info_t *local, struct sk_buff *skb,
-			struct lib80211_crypt_data *crypt)
-{
-	struct ieee80211_hdr *hdr;
-	int res, hdrlen;
-
-	if (crypt == NULL || crypt->ops->decrypt_mpdu == NULL)
-		return 0;
-
-	hdr = (struct ieee80211_hdr *) skb->data;
-	hdrlen = hostap_80211_get_hdrlen(hdr->frame_control);
-
-	if (local->tkip_countermeasures &&
-	    strcmp(crypt->ops->name, "TKIP") == 0) {
-		if (net_ratelimit()) {
-			printk(KERN_DEBUG "%s: TKIP countermeasures: dropped "
-			       "received packet from %pM\n",
-			       local->dev->name, hdr->addr2);
-		}
-		return -1;
-	}
-
-	atomic_inc(&crypt->refcnt);
-	res = crypt->ops->decrypt_mpdu(skb, hdrlen, crypt->priv);
-	atomic_dec(&crypt->refcnt);
-	if (res < 0) {
-		printk(KERN_DEBUG "%s: decryption failed (SA=%pM) res=%d\n",
-		       local->dev->name, hdr->addr2, res);
-		local->comm_tallies.rx_discards_wep_undecryptable++;
-		return -1;
-	}
-
-	return res;
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static int
-hostap_rx_frame_decrypt_msdu(local_info_t *local, struct sk_buff *skb,
-			     int keyidx, struct lib80211_crypt_data *crypt)
-{
-	struct ieee80211_hdr *hdr;
-	int res, hdrlen;
-
-	if (crypt == NULL || crypt->ops->decrypt_msdu == NULL)
-		return 0;
-
-	hdr = (struct ieee80211_hdr *) skb->data;
-	hdrlen = hostap_80211_get_hdrlen(hdr->frame_control);
-
-	atomic_inc(&crypt->refcnt);
-	res = crypt->ops->decrypt_msdu(skb, keyidx, hdrlen, crypt->priv);
-	atomic_dec(&crypt->refcnt);
-	if (res < 0) {
-		printk(KERN_DEBUG "%s: MSDU decryption/MIC verification failed"
-		       " (SA=%pM keyidx=%d)\n",
-		       local->dev->name, hdr->addr2, keyidx);
-		return -1;
-	}
-
-	return 0;
-}
-
-
-/* All received frames are sent to this function. @skb contains the frame in
- * IEEE 802.11 format, i.e., in the format it was sent over air.
- * This function is called only as a tasklet (software IRQ). */
-void hostap_80211_rx(struct net_device *dev, struct sk_buff *skb,
-		     struct hostap_80211_rx_status *rx_stats)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	struct ieee80211_hdr *hdr;
-	size_t hdrlen;
-	u16 fc, type, stype, sc;
-	struct net_device *wds = NULL;
-	unsigned int frag;
-	u8 *payload;
-	struct sk_buff *skb2 = NULL;
-	u16 ethertype;
-	int frame_authorized = 0;
-	int from_assoc_ap = 0;
-	u8 dst[ETH_ALEN];
-	u8 src[ETH_ALEN];
-	struct lib80211_crypt_data *crypt = NULL;
-	void *sta = NULL;
-	int keyidx = 0;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-	iface->stats.rx_packets++;
-	iface->stats.rx_bytes += skb->len;
-
-	/* dev is the master radio device; change this to be the default
-	 * virtual interface (this may be changed to WDS device below) */
-	dev = local->ddev;
-	iface = netdev_priv(dev);
-
-	hdr = (struct ieee80211_hdr *) skb->data;
-
-	if (skb->len < 10)
-		goto rx_dropped;
-
-	fc = le16_to_cpu(hdr->frame_control);
-	type = fc & IEEE80211_FCTL_FTYPE;
-	stype = fc & IEEE80211_FCTL_STYPE;
-	sc = le16_to_cpu(hdr->seq_ctrl);
-	frag = sc & IEEE80211_SCTL_FRAG;
-	hdrlen = hostap_80211_get_hdrlen(hdr->frame_control);
-
-	/* Put this code here so that we avoid duplicating it in all
-	 * Rx paths. - Jean II */
-#ifdef IW_WIRELESS_SPY		/* defined in iw_handler.h */
-	/* If spy monitoring on */
-	if (iface->spy_data.spy_number > 0) {
-		struct iw_quality wstats;
-		wstats.level = rx_stats->signal;
-		wstats.noise = rx_stats->noise;
-		wstats.updated = IW_QUAL_LEVEL_UPDATED | IW_QUAL_NOISE_UPDATED
-			| IW_QUAL_QUAL_INVALID | IW_QUAL_DBM;
-		/* Update spy records */
-		wireless_spy_update(dev, hdr->addr2, &wstats);
-	}
-#endif /* IW_WIRELESS_SPY */
-	hostap_update_rx_stats(local->ap, hdr, rx_stats);
-
-	if (local->iw_mode == IW_MODE_MONITOR) {
-		monitor_rx(dev, skb, rx_stats);
-		return;
-	}
-
-	if (local->host_decrypt) {
-		int idx = 0;
-		if (skb->len >= hdrlen + 3)
-			idx = skb->data[hdrlen + 3] >> 6;
-		crypt = local->crypt_info.crypt[idx];
-		sta = NULL;
-
-		/* Use station specific key to override default keys if the
-		 * receiver address is a unicast address ("individual RA"). If
-		 * bcrx_sta_key parameter is set, station specific key is used
-		 * even with broad/multicast targets (this is against IEEE
-		 * 802.11, but makes it easier to use different keys with
-		 * stations that do not support WEP key mapping). */
-
-		if (!(hdr->addr1[0] & 0x01) || local->bcrx_sta_key)
-			(void) hostap_handle_sta_crypto(local, hdr, &crypt,
-							&sta);
-
-		/* allow NULL decrypt to indicate an station specific override
-		 * for default encryption */
-		if (crypt && (crypt->ops == NULL ||
-			      crypt->ops->decrypt_mpdu == NULL))
-			crypt = NULL;
-
-		if (!crypt && (fc & IEEE80211_FCTL_PROTECTED)) {
-#if 0
-			/* This seems to be triggered by some (multicast?)
-			 * frames from other than current BSS, so just drop the
-			 * frames silently instead of filling system log with
-			 * these reports. */
-			printk(KERN_DEBUG "%s: WEP decryption failed (not set)"
-			       " (SA=%pM)\n",
-			       local->dev->name, hdr->addr2);
-#endif
-			local->comm_tallies.rx_discards_wep_undecryptable++;
-			goto rx_dropped;
-		}
-	}
-
-	if (type != IEEE80211_FTYPE_DATA) {
-		if (type == IEEE80211_FTYPE_MGMT &&
-		    stype == IEEE80211_STYPE_AUTH &&
-		    fc & IEEE80211_FCTL_PROTECTED && local->host_decrypt &&
-		    (keyidx = hostap_rx_frame_decrypt(local, skb, crypt)) < 0)
-		{
-			printk(KERN_DEBUG "%s: failed to decrypt mgmt::auth "
-			       "from %pM\n", dev->name, hdr->addr2);
-			/* TODO: could inform hostapd about this so that it
-			 * could send auth failure report */
-			goto rx_dropped;
-		}
-
-		if (hostap_rx_frame_mgmt(local, skb, rx_stats, type, stype))
-			goto rx_dropped;
-		else
-			goto rx_exit;
-	}
-
-	/* Data frame - extract src/dst addresses */
-	if (skb->len < IEEE80211_DATA_HDR3_LEN)
-		goto rx_dropped;
-
-	switch (fc & (IEEE80211_FCTL_FROMDS | IEEE80211_FCTL_TODS)) {
-	case IEEE80211_FCTL_FROMDS:
-		memcpy(dst, hdr->addr1, ETH_ALEN);
-		memcpy(src, hdr->addr3, ETH_ALEN);
-		break;
-	case IEEE80211_FCTL_TODS:
-		memcpy(dst, hdr->addr3, ETH_ALEN);
-		memcpy(src, hdr->addr2, ETH_ALEN);
-		break;
-	case IEEE80211_FCTL_FROMDS | IEEE80211_FCTL_TODS:
-		if (skb->len < IEEE80211_DATA_HDR4_LEN)
-			goto rx_dropped;
-		memcpy(dst, hdr->addr3, ETH_ALEN);
-		memcpy(src, hdr->addr4, ETH_ALEN);
-		break;
-	default:
-		memcpy(dst, hdr->addr1, ETH_ALEN);
-		memcpy(src, hdr->addr2, ETH_ALEN);
-		break;
-	}
-
-	if (hostap_rx_frame_wds(local, hdr, fc, &wds))
-		goto rx_dropped;
-	if (wds)
-		skb->dev = dev = wds;
-
-	if (local->iw_mode == IW_MODE_MASTER && !wds &&
-	    (fc & (IEEE80211_FCTL_TODS | IEEE80211_FCTL_FROMDS)) ==
-	    IEEE80211_FCTL_FROMDS &&
-	    local->stadev &&
-	    memcmp(hdr->addr2, local->assoc_ap_addr, ETH_ALEN) == 0) {
-		/* Frame from BSSID of the AP for which we are a client */
-		skb->dev = dev = local->stadev;
-		from_assoc_ap = 1;
-	}
-
-	if ((local->iw_mode == IW_MODE_MASTER ||
-	     local->iw_mode == IW_MODE_REPEAT) &&
-	    !from_assoc_ap) {
-		switch (hostap_handle_sta_rx(local, dev, skb, rx_stats,
-					     wds != NULL)) {
-		case AP_RX_CONTINUE_NOT_AUTHORIZED:
-			frame_authorized = 0;
-			break;
-		case AP_RX_CONTINUE:
-			frame_authorized = 1;
-			break;
-		case AP_RX_DROP:
-			goto rx_dropped;
-		case AP_RX_EXIT:
-			goto rx_exit;
-		}
-	}
-
-	/* Nullfunc frames may have PS-bit set, so they must be passed to
-	 * hostap_handle_sta_rx() before being dropped here. */
-	if (stype != IEEE80211_STYPE_DATA &&
-	    stype != IEEE80211_STYPE_DATA_CFACK &&
-	    stype != IEEE80211_STYPE_DATA_CFPOLL &&
-	    stype != IEEE80211_STYPE_DATA_CFACKPOLL) {
-		if (stype != IEEE80211_STYPE_NULLFUNC)
-			printk(KERN_DEBUG "%s: RX: dropped data frame "
-			       "with no data (type=0x%02x, subtype=0x%02x)\n",
-			       dev->name, type >> 2, stype >> 4);
-		goto rx_dropped;
-	}
-
-	/* skb: hdr + (possibly fragmented, possibly encrypted) payload */
-
-	if (local->host_decrypt && (fc & IEEE80211_FCTL_PROTECTED) &&
-	    (keyidx = hostap_rx_frame_decrypt(local, skb, crypt)) < 0)
-		goto rx_dropped;
-	hdr = (struct ieee80211_hdr *) skb->data;
-
-	/* skb: hdr + (possibly fragmented) plaintext payload */
-
-	if (local->host_decrypt && (fc & IEEE80211_FCTL_PROTECTED) &&
-	    (frag != 0 || (fc & IEEE80211_FCTL_MOREFRAGS))) {
-		int flen;
-		struct sk_buff *frag_skb =
-			prism2_frag_cache_get(local, hdr);
-		if (!frag_skb) {
-			printk(KERN_DEBUG "%s: Rx cannot get skb from "
-			       "fragment cache (morefrag=%d seq=%u frag=%u)\n",
-			       dev->name, (fc & IEEE80211_FCTL_MOREFRAGS) != 0,
-			       (sc & IEEE80211_SCTL_SEQ) >> 4, frag);
-			goto rx_dropped;
-		}
-
-		flen = skb->len;
-		if (frag != 0)
-			flen -= hdrlen;
-
-		if (frag_skb->tail + flen > frag_skb->end) {
-			printk(KERN_WARNING "%s: host decrypted and "
-			       "reassembled frame did not fit skb\n",
-			       dev->name);
-			prism2_frag_cache_invalidate(local, hdr);
-			goto rx_dropped;
-		}
-
-		if (frag == 0) {
-			/* copy first fragment (including full headers) into
-			 * beginning of the fragment cache skb */
-			skb_copy_from_linear_data(skb, skb_put(frag_skb, flen),
-						  flen);
-		} else {
-			/* append frame payload to the end of the fragment
-			 * cache skb */
-			skb_copy_from_linear_data_offset(skb, hdrlen,
-							 skb_put(frag_skb,
-								 flen), flen);
-		}
-		dev_kfree_skb(skb);
-		skb = NULL;
-
-		if (fc & IEEE80211_FCTL_MOREFRAGS) {
-			/* more fragments expected - leave the skb in fragment
-			 * cache for now; it will be delivered to upper layers
-			 * after all fragments have been received */
-			goto rx_exit;
-		}
-
-		/* this was the last fragment and the frame will be
-		 * delivered, so remove skb from fragment cache */
-		skb = frag_skb;
-		hdr = (struct ieee80211_hdr *) skb->data;
-		prism2_frag_cache_invalidate(local, hdr);
-	}
-
-	/* skb: hdr + (possible reassembled) full MSDU payload; possibly still
-	 * encrypted/authenticated */
-
-	if (local->host_decrypt && (fc & IEEE80211_FCTL_PROTECTED) &&
-	    hostap_rx_frame_decrypt_msdu(local, skb, keyidx, crypt))
-		goto rx_dropped;
-
-	hdr = (struct ieee80211_hdr *) skb->data;
-	if (crypt && !(fc & IEEE80211_FCTL_PROTECTED) && !local->open_wep) {
-		if (local->ieee_802_1x &&
-		    hostap_is_eapol_frame(local, skb)) {
-			/* pass unencrypted EAPOL frames even if encryption is
-			 * configured */
-			PDEBUG(DEBUG_EXTRA2, "%s: RX: IEEE 802.1X - passing "
-			       "unencrypted EAPOL frame\n", local->dev->name);
-		} else {
-			printk(KERN_DEBUG "%s: encryption configured, but RX "
-			       "frame not encrypted (SA=%pM)\n",
-			       local->dev->name, hdr->addr2);
-			goto rx_dropped;
-		}
-	}
-
-	if (local->drop_unencrypted && !(fc & IEEE80211_FCTL_PROTECTED) &&
-	    !hostap_is_eapol_frame(local, skb)) {
-		if (net_ratelimit()) {
-			printk(KERN_DEBUG "%s: dropped unencrypted RX data "
-			       "frame from %pM (drop_unencrypted=1)\n",
-			       dev->name, hdr->addr2);
-		}
-		goto rx_dropped;
-	}
-
-	/* skb: hdr + (possible reassembled) full plaintext payload */
-
-	payload = skb->data + hdrlen;
-	ethertype = (payload[6] << 8) | payload[7];
-
-	/* If IEEE 802.1X is used, check whether the port is authorized to send
-	 * the received frame. */
-	if (local->ieee_802_1x && local->iw_mode == IW_MODE_MASTER) {
-		if (ethertype == ETH_P_PAE) {
-			PDEBUG(DEBUG_EXTRA2, "%s: RX: IEEE 802.1X frame\n",
-			       dev->name);
-			if (local->hostapd && local->apdev) {
-				/* Send IEEE 802.1X frames to the user
-				 * space daemon for processing */
-				prism2_rx_80211(local->apdev, skb, rx_stats,
-						PRISM2_RX_MGMT);
-				local->apdevstats.rx_packets++;
-				local->apdevstats.rx_bytes += skb->len;
-				goto rx_exit;
-			}
-		} else if (!frame_authorized) {
-			printk(KERN_DEBUG "%s: dropped frame from "
-			       "unauthorized port (IEEE 802.1X): "
-			       "ethertype=0x%04x\n",
-			       dev->name, ethertype);
-			goto rx_dropped;
-		}
-	}
-
-	/* convert hdr + possible LLC headers into Ethernet header */
-	if (skb->len - hdrlen >= 8 &&
-	    ((memcmp(payload, rfc1042_header, 6) == 0 &&
-	      ethertype != ETH_P_AARP && ethertype != ETH_P_IPX) ||
-	     memcmp(payload, bridge_tunnel_header, 6) == 0)) {
-		/* remove RFC1042 or Bridge-Tunnel encapsulation and
-		 * replace EtherType */
-		skb_pull(skb, hdrlen + 6);
-		memcpy(skb_push(skb, ETH_ALEN), src, ETH_ALEN);
-		memcpy(skb_push(skb, ETH_ALEN), dst, ETH_ALEN);
-	} else {
-		__be16 len;
-		/* Leave Ethernet header part of hdr and full payload */
-		skb_pull(skb, hdrlen);
-		len = htons(skb->len);
-		memcpy(skb_push(skb, 2), &len, 2);
-		memcpy(skb_push(skb, ETH_ALEN), src, ETH_ALEN);
-		memcpy(skb_push(skb, ETH_ALEN), dst, ETH_ALEN);
-	}
-
-	if (wds && ((fc & (IEEE80211_FCTL_TODS | IEEE80211_FCTL_FROMDS)) ==
-		    IEEE80211_FCTL_TODS) &&
-	    skb->len >= ETH_HLEN + ETH_ALEN) {
-		/* Non-standard frame: get addr4 from its bogus location after
-		 * the payload */
-		skb_copy_from_linear_data_offset(skb, skb->len - ETH_ALEN,
-						 skb->data + ETH_ALEN,
-						 ETH_ALEN);
-		skb_trim(skb, skb->len - ETH_ALEN);
-	}
-
-	dev->stats.rx_packets++;
-	dev->stats.rx_bytes += skb->len;
-
-	if (local->iw_mode == IW_MODE_MASTER && !wds &&
-	    local->ap->bridge_packets) {
-		if (dst[0] & 0x01) {
-			/* copy multicast frame both to the higher layers and
-			 * to the wireless media */
-			local->ap->bridged_multicast++;
-			skb2 = skb_clone(skb, GFP_ATOMIC);
-			if (skb2 == NULL)
-				printk(KERN_DEBUG "%s: skb_clone failed for "
-				       "multicast frame\n", dev->name);
-		} else if (hostap_is_sta_authorized(local->ap, dst)) {
-			/* send frame directly to the associated STA using
-			 * wireless media and not passing to higher layers */
-			local->ap->bridged_unicast++;
-			skb2 = skb;
-			skb = NULL;
-		}
-	}
-
-	if (skb2 != NULL) {
-		/* send to wireless media */
-		skb2->dev = dev;
-		skb2->protocol = cpu_to_be16(ETH_P_802_3);
-		skb_reset_mac_header(skb2);
-		skb_reset_network_header(skb2);
-		/* skb2->network_header += ETH_HLEN; */
-		dev_queue_xmit(skb2);
-	}
-
-	if (skb) {
-		skb->protocol = eth_type_trans(skb, dev);
-		memset(skb->cb, 0, sizeof(skb->cb));
-		netif_rx(skb);
-	}
-
- rx_exit:
-	if (sta)
-		hostap_handle_sta_release(sta);
-	return;
-
- rx_dropped:
-	dev_kfree_skb(skb);
-
-	dev->stats.rx_dropped++;
-	goto rx_exit;
-}
-
-
-EXPORT_SYMBOL(hostap_80211_rx);
diff --git a/drivers/net/wireless/intersil/hostap/hostap_80211_tx.c b/drivers/net/wireless/intersil/hostap/hostap_80211_tx.c
deleted file mode 100644
index c47da06945c2..000000000000
--- a/drivers/net/wireless/intersil/hostap/hostap_80211_tx.c
+++ /dev/null
@@ -1,554 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-#include <linux/slab.h>
-#include <linux/export.h>
-#include <linux/etherdevice.h>
-
-#include "hostap_80211.h"
-#include "hostap_common.h"
-#include "hostap_wlan.h"
-#include "hostap.h"
-#include "hostap_ap.h"
-
-/* See IEEE 802.1H for LLC/SNAP encapsulation/decapsulation */
-/* Ethernet-II snap header (RFC1042 for most EtherTypes) */
-static unsigned char rfc1042_header[] =
-{ 0xaa, 0xaa, 0x03, 0x00, 0x00, 0x00 };
-/* Bridge-Tunnel header (for EtherTypes ETH_P_AARP and ETH_P_IPX) */
-static unsigned char bridge_tunnel_header[] =
-{ 0xaa, 0xaa, 0x03, 0x00, 0x00, 0xf8 };
-/* No encapsulation header if EtherType < 0x600 (=length) */
-
-void hostap_dump_tx_80211(const char *name, struct sk_buff *skb)
-{
-	struct ieee80211_hdr *hdr;
-	u16 fc;
-
-	hdr = (struct ieee80211_hdr *) skb->data;
-
-	printk(KERN_DEBUG "%s: TX len=%d jiffies=%ld\n",
-	       name, skb->len, jiffies);
-
-	if (skb->len < 2)
-		return;
-
-	fc = le16_to_cpu(hdr->frame_control);
-	printk(KERN_DEBUG "   FC=0x%04x (type=%d:%d)%s%s",
-	       fc, (fc & IEEE80211_FCTL_FTYPE) >> 2,
-	       (fc & IEEE80211_FCTL_STYPE) >> 4,
-	       fc & IEEE80211_FCTL_TODS ? " [ToDS]" : "",
-	       fc & IEEE80211_FCTL_FROMDS ? " [FromDS]" : "");
-
-	if (skb->len < IEEE80211_DATA_HDR3_LEN) {
-		printk("\n");
-		return;
-	}
-
-	printk(" dur=0x%04x seq=0x%04x\n", le16_to_cpu(hdr->duration_id),
-	       le16_to_cpu(hdr->seq_ctrl));
-
-	printk(KERN_DEBUG "   A1=%pM", hdr->addr1);
-	printk(" A2=%pM", hdr->addr2);
-	printk(" A3=%pM", hdr->addr3);
-	if (skb->len >= 30)
-		printk(" A4=%pM", hdr->addr4);
-	printk("\n");
-}
-
-
-/* hard_start_xmit function for data interfaces (wlan#, wlan#wds#, wlan#sta)
- * Convert Ethernet header into a suitable IEEE 802.11 header depending on
- * device configuration. */
-netdev_tx_t hostap_data_start_xmit(struct sk_buff *skb,
-				   struct net_device *dev)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int need_headroom, need_tailroom = 0;
-	struct ieee80211_hdr hdr;
-	u16 fc, ethertype = 0;
-	enum {
-		WDS_NO = 0, WDS_OWN_FRAME, WDS_COMPLIANT_FRAME
-	} use_wds = WDS_NO;
-	u8 *encaps_data;
-	int hdr_len, encaps_len, skip_header_bytes;
-	int to_assoc_ap = 0;
-	struct hostap_skb_tx_data *meta;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (skb->len < ETH_HLEN) {
-		printk(KERN_DEBUG "%s: hostap_data_start_xmit: short skb "
-		       "(len=%d)\n", dev->name, skb->len);
-		kfree_skb(skb);
-		return NETDEV_TX_OK;
-	}
-
-	if (local->ddev != dev) {
-		use_wds = (local->iw_mode == IW_MODE_MASTER &&
-			   !(local->wds_type & HOSTAP_WDS_STANDARD_FRAME)) ?
-			WDS_OWN_FRAME : WDS_COMPLIANT_FRAME;
-		if (dev == local->stadev) {
-			to_assoc_ap = 1;
-			use_wds = WDS_NO;
-		} else if (dev == local->apdev) {
-			printk(KERN_DEBUG "%s: prism2_tx: trying to use "
-			       "AP device with Ethernet net dev\n", dev->name);
-			kfree_skb(skb);
-			return NETDEV_TX_OK;
-		}
-	} else {
-		if (local->iw_mode == IW_MODE_REPEAT) {
-			printk(KERN_DEBUG "%s: prism2_tx: trying to use "
-			       "non-WDS link in Repeater mode\n", dev->name);
-			kfree_skb(skb);
-			return NETDEV_TX_OK;
-		} else if (local->iw_mode == IW_MODE_INFRA &&
-			   (local->wds_type & HOSTAP_WDS_AP_CLIENT) &&
-			   !ether_addr_equal(skb->data + ETH_ALEN, dev->dev_addr)) {
-			/* AP client mode: send frames with foreign src addr
-			 * using 4-addr WDS frames */
-			use_wds = WDS_COMPLIANT_FRAME;
-		}
-	}
-
-	/* Incoming skb->data: dst_addr[6], src_addr[6], proto[2], payload
-	 * ==>
-	 * Prism2 TX frame with 802.11 header:
-	 * txdesc (address order depending on used mode; includes dst_addr and
-	 * src_addr), possible encapsulation (RFC1042/Bridge-Tunnel;
-	 * proto[2], payload {, possible addr4[6]} */
-
-	ethertype = (skb->data[12] << 8) | skb->data[13];
-
-	memset(&hdr, 0, sizeof(hdr));
-
-	/* Length of data after IEEE 802.11 header */
-	encaps_data = NULL;
-	encaps_len = 0;
-	skip_header_bytes = ETH_HLEN;
-	if (ethertype == ETH_P_AARP || ethertype == ETH_P_IPX) {
-		encaps_data = bridge_tunnel_header;
-		encaps_len = sizeof(bridge_tunnel_header);
-		skip_header_bytes -= 2;
-	} else if (ethertype >= 0x600) {
-		encaps_data = rfc1042_header;
-		encaps_len = sizeof(rfc1042_header);
-		skip_header_bytes -= 2;
-	}
-
-	fc = IEEE80211_FTYPE_DATA | IEEE80211_STYPE_DATA;
-	hdr_len = IEEE80211_DATA_HDR3_LEN;
-
-	if (use_wds != WDS_NO) {
-		/* Note! Prism2 station firmware has problems with sending real
-		 * 802.11 frames with four addresses; until these problems can
-		 * be fixed or worked around, 4-addr frames needed for WDS are
-		 * using incompatible format: FromDS flag is not set and the
-		 * fourth address is added after the frame payload; it is
-		 * assumed, that the receiving station knows how to handle this
-		 * frame format */
-
-		if (use_wds == WDS_COMPLIANT_FRAME) {
-			fc |= IEEE80211_FCTL_FROMDS | IEEE80211_FCTL_TODS;
-			/* From&To DS: Addr1 = RA, Addr2 = TA, Addr3 = DA,
-			 * Addr4 = SA */
-			skb_copy_from_linear_data_offset(skb, ETH_ALEN,
-							 &hdr.addr4, ETH_ALEN);
-			hdr_len += ETH_ALEN;
-		} else {
-			/* bogus 4-addr format to workaround Prism2 station
-			 * f/w bug */
-			fc |= IEEE80211_FCTL_TODS;
-			/* From DS: Addr1 = DA (used as RA),
-			 * Addr2 = BSSID (used as TA), Addr3 = SA (used as DA),
-			 */
-
-			/* SA from skb->data + ETH_ALEN will be added after
-			 * frame payload; use hdr.addr4 as a temporary buffer
-			 */
-			skb_copy_from_linear_data_offset(skb, ETH_ALEN,
-							 &hdr.addr4, ETH_ALEN);
-			need_tailroom += ETH_ALEN;
-		}
-
-		/* send broadcast and multicast frames to broadcast RA, if
-		 * configured; otherwise, use unicast RA of the WDS link */
-		if ((local->wds_type & HOSTAP_WDS_BROADCAST_RA) &&
-		    is_multicast_ether_addr(skb->data))
-			eth_broadcast_addr(hdr.addr1);
-		else if (iface->type == HOSTAP_INTERFACE_WDS)
-			memcpy(&hdr.addr1, iface->u.wds.remote_addr,
-			       ETH_ALEN);
-		else
-			memcpy(&hdr.addr1, local->bssid, ETH_ALEN);
-		memcpy(&hdr.addr2, dev->dev_addr, ETH_ALEN);
-		skb_copy_from_linear_data(skb, &hdr.addr3, ETH_ALEN);
-	} else if (local->iw_mode == IW_MODE_MASTER && !to_assoc_ap) {
-		fc |= IEEE80211_FCTL_FROMDS;
-		/* From DS: Addr1 = DA, Addr2 = BSSID, Addr3 = SA */
-		skb_copy_from_linear_data(skb, &hdr.addr1, ETH_ALEN);
-		memcpy(&hdr.addr2, dev->dev_addr, ETH_ALEN);
-		skb_copy_from_linear_data_offset(skb, ETH_ALEN, &hdr.addr3,
-						 ETH_ALEN);
-	} else if (local->iw_mode == IW_MODE_INFRA || to_assoc_ap) {
-		fc |= IEEE80211_FCTL_TODS;
-		/* To DS: Addr1 = BSSID, Addr2 = SA, Addr3 = DA */
-		memcpy(&hdr.addr1, to_assoc_ap ?
-		       local->assoc_ap_addr : local->bssid, ETH_ALEN);
-		skb_copy_from_linear_data_offset(skb, ETH_ALEN, &hdr.addr2,
-						 ETH_ALEN);
-		skb_copy_from_linear_data(skb, &hdr.addr3, ETH_ALEN);
-	} else if (local->iw_mode == IW_MODE_ADHOC) {
-		/* not From/To DS: Addr1 = DA, Addr2 = SA, Addr3 = BSSID */
-		skb_copy_from_linear_data(skb, &hdr.addr1, ETH_ALEN);
-		skb_copy_from_linear_data_offset(skb, ETH_ALEN, &hdr.addr2,
-						 ETH_ALEN);
-		memcpy(&hdr.addr3, local->bssid, ETH_ALEN);
-	}
-
-	hdr.frame_control = cpu_to_le16(fc);
-
-	skb_pull(skb, skip_header_bytes);
-	need_headroom = local->func->need_tx_headroom + hdr_len + encaps_len;
-	if (skb_tailroom(skb) < need_tailroom) {
-		skb = skb_unshare(skb, GFP_ATOMIC);
-		if (skb == NULL) {
-			iface->stats.tx_dropped++;
-			return NETDEV_TX_OK;
-		}
-		if (pskb_expand_head(skb, need_headroom, need_tailroom,
-				     GFP_ATOMIC)) {
-			kfree_skb(skb);
-			iface->stats.tx_dropped++;
-			return NETDEV_TX_OK;
-		}
-	} else if (skb_headroom(skb) < need_headroom) {
-		struct sk_buff *tmp = skb;
-		skb = skb_realloc_headroom(skb, need_headroom);
-		kfree_skb(tmp);
-		if (skb == NULL) {
-			iface->stats.tx_dropped++;
-			return NETDEV_TX_OK;
-		}
-	} else {
-		skb = skb_unshare(skb, GFP_ATOMIC);
-		if (skb == NULL) {
-			iface->stats.tx_dropped++;
-			return NETDEV_TX_OK;
-		}
-	}
-
-	if (encaps_data)
-		memcpy(skb_push(skb, encaps_len), encaps_data, encaps_len);
-	memcpy(skb_push(skb, hdr_len), &hdr, hdr_len);
-	if (use_wds == WDS_OWN_FRAME) {
-		skb_put_data(skb, &hdr.addr4, ETH_ALEN);
-	}
-
-	iface->stats.tx_packets++;
-	iface->stats.tx_bytes += skb->len;
-
-	skb_reset_mac_header(skb);
-	meta = (struct hostap_skb_tx_data *) skb->cb;
-	memset(meta, 0, sizeof(*meta));
-	meta->magic = HOSTAP_SKB_TX_DATA_MAGIC;
-	if (use_wds)
-		meta->flags |= HOSTAP_TX_FLAGS_WDS;
-	meta->ethertype = ethertype;
-	meta->iface = iface;
-
-	/* Send IEEE 802.11 encapsulated frame using the master radio device */
-	skb->dev = local->dev;
-	dev_queue_xmit(skb);
-	return NETDEV_TX_OK;
-}
-
-
-/* hard_start_xmit function for hostapd wlan#ap interfaces */
-netdev_tx_t hostap_mgmt_start_xmit(struct sk_buff *skb,
-				   struct net_device *dev)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	struct hostap_skb_tx_data *meta;
-	struct ieee80211_hdr *hdr;
-	u16 fc;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (skb->len < 10) {
-		printk(KERN_DEBUG "%s: hostap_mgmt_start_xmit: short skb "
-		       "(len=%d)\n", dev->name, skb->len);
-		kfree_skb(skb);
-		return NETDEV_TX_OK;
-	}
-
-	iface->stats.tx_packets++;
-	iface->stats.tx_bytes += skb->len;
-
-	meta = (struct hostap_skb_tx_data *) skb->cb;
-	memset(meta, 0, sizeof(*meta));
-	meta->magic = HOSTAP_SKB_TX_DATA_MAGIC;
-	meta->iface = iface;
-
-	if (skb->len >= IEEE80211_DATA_HDR3_LEN + sizeof(rfc1042_header) + 2) {
-		hdr = (struct ieee80211_hdr *) skb->data;
-		fc = le16_to_cpu(hdr->frame_control);
-		if (ieee80211_is_data(hdr->frame_control) &&
-		    (fc & IEEE80211_FCTL_STYPE) == IEEE80211_STYPE_DATA) {
-			u8 *pos = &skb->data[IEEE80211_DATA_HDR3_LEN +
-					     sizeof(rfc1042_header)];
-			meta->ethertype = (pos[0] << 8) | pos[1];
-		}
-	}
-
-	/* Send IEEE 802.11 encapsulated frame using the master radio device */
-	skb->dev = local->dev;
-	dev_queue_xmit(skb);
-	return NETDEV_TX_OK;
-}
-
-
-/* Called only from software IRQ */
-static struct sk_buff * hostap_tx_encrypt(struct sk_buff *skb,
-					  struct lib80211_crypt_data *crypt)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	struct ieee80211_hdr *hdr;
-	int prefix_len, postfix_len, hdr_len, res;
-
-	iface = netdev_priv(skb->dev);
-	local = iface->local;
-
-	if (skb->len < IEEE80211_DATA_HDR3_LEN) {
-		kfree_skb(skb);
-		return NULL;
-	}
-
-	if (local->tkip_countermeasures &&
-	    strcmp(crypt->ops->name, "TKIP") == 0) {
-		hdr = (struct ieee80211_hdr *) skb->data;
-		if (net_ratelimit()) {
-			printk(KERN_DEBUG "%s: TKIP countermeasures: dropped "
-			       "TX packet to %pM\n",
-			       local->dev->name, hdr->addr1);
-		}
-		kfree_skb(skb);
-		return NULL;
-	}
-
-	skb = skb_unshare(skb, GFP_ATOMIC);
-	if (skb == NULL)
-		return NULL;
-
-	prefix_len = crypt->ops->extra_mpdu_prefix_len +
-		crypt->ops->extra_msdu_prefix_len;
-	postfix_len = crypt->ops->extra_mpdu_postfix_len +
-		crypt->ops->extra_msdu_postfix_len;
-	if ((skb_headroom(skb) < prefix_len ||
-	     skb_tailroom(skb) < postfix_len) &&
-	    pskb_expand_head(skb, prefix_len, postfix_len, GFP_ATOMIC)) {
-		kfree_skb(skb);
-		return NULL;
-	}
-
-	hdr = (struct ieee80211_hdr *) skb->data;
-	hdr_len = hostap_80211_get_hdrlen(hdr->frame_control);
-
-	/* Host-based IEEE 802.11 fragmentation for TX is not yet supported, so
-	 * call both MSDU and MPDU encryption functions from here. */
-	atomic_inc(&crypt->refcnt);
-	res = 0;
-	if (crypt->ops->encrypt_msdu)
-		res = crypt->ops->encrypt_msdu(skb, hdr_len, crypt->priv);
-	if (res == 0 && crypt->ops->encrypt_mpdu)
-		res = crypt->ops->encrypt_mpdu(skb, hdr_len, crypt->priv);
-	atomic_dec(&crypt->refcnt);
-	if (res < 0) {
-		kfree_skb(skb);
-		return NULL;
-	}
-
-	return skb;
-}
-
-
-/* hard_start_xmit function for master radio interface wifi#.
- * AP processing (TX rate control, power save buffering, etc.).
- * Use hardware TX function to send the frame. */
-netdev_tx_t hostap_master_start_xmit(struct sk_buff *skb,
-				     struct net_device *dev)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	netdev_tx_t ret = NETDEV_TX_BUSY;
-	u16 fc;
-	struct hostap_tx_data tx;
-	ap_tx_ret tx_ret;
-	struct hostap_skb_tx_data *meta;
-	int no_encrypt = 0;
-	struct ieee80211_hdr *hdr;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	tx.skb = skb;
-	tx.sta_ptr = NULL;
-
-	meta = (struct hostap_skb_tx_data *) skb->cb;
-	if (meta->magic != HOSTAP_SKB_TX_DATA_MAGIC) {
-		printk(KERN_DEBUG "%s: invalid skb->cb magic (0x%08x, "
-		       "expected 0x%08x)\n",
-		       dev->name, meta->magic, HOSTAP_SKB_TX_DATA_MAGIC);
-		ret = NETDEV_TX_OK;
-		iface->stats.tx_dropped++;
-		goto fail;
-	}
-
-	if (local->host_encrypt) {
-		/* Set crypt to default algorithm and key; will be replaced in
-		 * AP code if STA has own alg/key */
-		tx.crypt = local->crypt_info.crypt[local->crypt_info.tx_keyidx];
-		tx.host_encrypt = 1;
-	} else {
-		tx.crypt = NULL;
-		tx.host_encrypt = 0;
-	}
-
-	if (skb->len < 24) {
-		printk(KERN_DEBUG "%s: hostap_master_start_xmit: short skb "
-		       "(len=%d)\n", dev->name, skb->len);
-		ret = NETDEV_TX_OK;
-		iface->stats.tx_dropped++;
-		goto fail;
-	}
-
-	/* FIX (?):
-	 * Wi-Fi 802.11b test plan suggests that AP should ignore power save
-	 * bit in authentication and (re)association frames and assume tha
-	 * STA remains awake for the response. */
-	tx_ret = hostap_handle_sta_tx(local, &tx);
-	skb = tx.skb;
-	meta = (struct hostap_skb_tx_data *) skb->cb;
-	hdr = (struct ieee80211_hdr *) skb->data;
-	fc = le16_to_cpu(hdr->frame_control);
-	switch (tx_ret) {
-	case AP_TX_CONTINUE:
-		break;
-	case AP_TX_CONTINUE_NOT_AUTHORIZED:
-		if (local->ieee_802_1x &&
-		    ieee80211_is_data(hdr->frame_control) &&
-		    meta->ethertype != ETH_P_PAE &&
-		    !(meta->flags & HOSTAP_TX_FLAGS_WDS)) {
-			printk(KERN_DEBUG "%s: dropped frame to unauthorized "
-			       "port (IEEE 802.1X): ethertype=0x%04x\n",
-			       dev->name, meta->ethertype);
-			hostap_dump_tx_80211(dev->name, skb);
-
-			ret = NETDEV_TX_OK; /* drop packet */
-			iface->stats.tx_dropped++;
-			goto fail;
-		}
-		break;
-	case AP_TX_DROP:
-		ret = NETDEV_TX_OK; /* drop packet */
-		iface->stats.tx_dropped++;
-		goto fail;
-	case AP_TX_RETRY:
-		goto fail;
-	case AP_TX_BUFFERED:
-		/* do not free skb here, it will be freed when the
-		 * buffered frame is sent/timed out */
-		ret = NETDEV_TX_OK;
-		goto tx_exit;
-	}
-
-	/* Request TX callback if protocol version is 2 in 802.11 header;
-	 * this version 2 is a special case used between hostapd and kernel
-	 * driver */
-	if (((fc & IEEE80211_FCTL_VERS) == BIT(1)) &&
-	    local->ap && local->ap->tx_callback_idx && meta->tx_cb_idx == 0) {
-		meta->tx_cb_idx = local->ap->tx_callback_idx;
-
-		/* remove special version from the frame header */
-		fc &= ~IEEE80211_FCTL_VERS;
-		hdr->frame_control = cpu_to_le16(fc);
-	}
-
-	if (!ieee80211_is_data(hdr->frame_control)) {
-		no_encrypt = 1;
-		tx.crypt = NULL;
-	}
-
-	if (local->ieee_802_1x && meta->ethertype == ETH_P_PAE && tx.crypt &&
-	    !(fc & IEEE80211_FCTL_PROTECTED)) {
-		no_encrypt = 1;
-		PDEBUG(DEBUG_EXTRA2, "%s: TX: IEEE 802.1X - passing "
-		       "unencrypted EAPOL frame\n", dev->name);
-		tx.crypt = NULL; /* no encryption for IEEE 802.1X frames */
-	}
-
-	if (tx.crypt && (!tx.crypt->ops || !tx.crypt->ops->encrypt_mpdu))
-		tx.crypt = NULL;
-	else if ((tx.crypt ||
-		 local->crypt_info.crypt[local->crypt_info.tx_keyidx]) &&
-		 !no_encrypt) {
-		/* Add ISWEP flag both for firmware and host based encryption
-		 */
-		fc |= IEEE80211_FCTL_PROTECTED;
-		hdr->frame_control = cpu_to_le16(fc);
-	} else if (local->drop_unencrypted &&
-		   ieee80211_is_data(hdr->frame_control) &&
-		   meta->ethertype != ETH_P_PAE) {
-		if (net_ratelimit()) {
-			printk(KERN_DEBUG "%s: dropped unencrypted TX data "
-			       "frame (drop_unencrypted=1)\n", dev->name);
-		}
-		iface->stats.tx_dropped++;
-		ret = NETDEV_TX_OK;
-		goto fail;
-	}
-
-	if (tx.crypt) {
-		skb = hostap_tx_encrypt(skb, tx.crypt);
-		if (skb == NULL) {
-			printk(KERN_DEBUG "%s: TX - encryption failed\n",
-			       dev->name);
-			ret = NETDEV_TX_OK;
-			goto fail;
-		}
-		meta = (struct hostap_skb_tx_data *) skb->cb;
-		if (meta->magic != HOSTAP_SKB_TX_DATA_MAGIC) {
-			printk(KERN_DEBUG "%s: invalid skb->cb magic (0x%08x, "
-			       "expected 0x%08x) after hostap_tx_encrypt\n",
-			       dev->name, meta->magic,
-			       HOSTAP_SKB_TX_DATA_MAGIC);
-			ret = NETDEV_TX_OK;
-			iface->stats.tx_dropped++;
-			goto fail;
-		}
-	}
-
-	if (local->func->tx == NULL || local->func->tx(skb, dev)) {
-		ret = NETDEV_TX_OK;
-		iface->stats.tx_dropped++;
-	} else {
-		ret = NETDEV_TX_OK;
-		iface->stats.tx_packets++;
-		iface->stats.tx_bytes += skb->len;
-	}
-
- fail:
-	if (ret == NETDEV_TX_OK && skb)
-		dev_kfree_skb(skb);
- tx_exit:
-	if (tx.sta_ptr)
-		hostap_handle_sta_release(tx.sta_ptr);
-	return ret;
-}
-
-
-EXPORT_SYMBOL(hostap_master_start_xmit);
diff --git a/drivers/net/wireless/intersil/hostap/hostap_ap.c b/drivers/net/wireless/intersil/hostap/hostap_ap.c
deleted file mode 100644
index 9b546a71e7a2..000000000000
--- a/drivers/net/wireless/intersil/hostap/hostap_ap.c
+++ /dev/null
@@ -1,3277 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Intersil Prism2 driver with Host AP (software access point) support
- * Copyright (c) 2001-2002, SSH Communications Security Corp and Jouni Malinen
- * <j@w1.fi>
- * Copyright (c) 2002-2005, Jouni Malinen <j@w1.fi>
- *
- * This file is to be included into hostap.c when S/W AP functionality is
- * compiled.
- *
- * AP:  FIX:
- * - if unicast Class 2 (assoc,reassoc,disassoc) frame received from
- *   unauthenticated STA, send deauth. frame (8802.11: 5.5)
- * - if unicast Class 3 (data with to/from DS,deauth,pspoll) frame received
- *   from authenticated, but unassoc STA, send disassoc frame (8802.11: 5.5)
- * - if unicast Class 3 received from unauthenticated STA, send deauth. frame
- *   (8802.11: 5.5)
- */
-
-#include <linux/proc_fs.h>
-#include <linux/seq_file.h>
-#include <linux/delay.h>
-#include <linux/random.h>
-#include <linux/if_arp.h>
-#include <linux/slab.h>
-#include <linux/export.h>
-#include <linux/moduleparam.h>
-#include <linux/etherdevice.h>
-
-#include "hostap_wlan.h"
-#include "hostap.h"
-#include "hostap_ap.h"
-
-static int other_ap_policy[MAX_PARM_DEVICES] = { AP_OTHER_AP_SKIP_ALL,
-						 DEF_INTS };
-module_param_array(other_ap_policy, int, NULL, 0444);
-MODULE_PARM_DESC(other_ap_policy, "Other AP beacon monitoring policy (0-3)");
-
-static int ap_max_inactivity[MAX_PARM_DEVICES] = { AP_MAX_INACTIVITY_SEC,
-						   DEF_INTS };
-module_param_array(ap_max_inactivity, int, NULL, 0444);
-MODULE_PARM_DESC(ap_max_inactivity, "AP timeout (in seconds) for station "
-		 "inactivity");
-
-static int ap_bridge_packets[MAX_PARM_DEVICES] = { 1, DEF_INTS };
-module_param_array(ap_bridge_packets, int, NULL, 0444);
-MODULE_PARM_DESC(ap_bridge_packets, "Bridge packets directly between "
-		 "stations");
-
-static int autom_ap_wds[MAX_PARM_DEVICES] = { 0, DEF_INTS };
-module_param_array(autom_ap_wds, int, NULL, 0444);
-MODULE_PARM_DESC(autom_ap_wds, "Add WDS connections to other APs "
-		 "automatically");
-
-
-static struct sta_info* ap_get_sta(struct ap_data *ap, u8 *sta);
-static void hostap_event_expired_sta(struct net_device *dev,
-				     struct sta_info *sta);
-static void handle_add_proc_queue(struct work_struct *work);
-
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-static void handle_wds_oper_queue(struct work_struct *work);
-static void prism2_send_mgmt(struct net_device *dev,
-			     u16 type_subtype, char *body,
-			     int body_len, u8 *addr, u16 tx_cb_idx);
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-
-
-#if !defined(PRISM2_NO_PROCFS_DEBUG) && defined(CONFIG_PROC_FS)
-static int ap_debug_proc_show(struct seq_file *m, void *v)
-{
-	struct ap_data *ap = pde_data(file_inode(m->file));
-
-	seq_printf(m, "BridgedUnicastFrames=%u\n", ap->bridged_unicast);
-	seq_printf(m, "BridgedMulticastFrames=%u\n", ap->bridged_multicast);
-	seq_printf(m, "max_inactivity=%u\n", ap->max_inactivity / HZ);
-	seq_printf(m, "bridge_packets=%u\n", ap->bridge_packets);
-	seq_printf(m, "nullfunc_ack=%u\n", ap->nullfunc_ack);
-	seq_printf(m, "autom_ap_wds=%u\n", ap->autom_ap_wds);
-	seq_printf(m, "auth_algs=%u\n", ap->local->auth_algs);
-	seq_printf(m, "tx_drop_nonassoc=%u\n", ap->tx_drop_nonassoc);
-	return 0;
-}
-#endif
-
-static void ap_sta_hash_add(struct ap_data *ap, struct sta_info *sta)
-{
-	sta->hnext = ap->sta_hash[STA_HASH(sta->addr)];
-	ap->sta_hash[STA_HASH(sta->addr)] = sta;
-}
-
-static void ap_sta_hash_del(struct ap_data *ap, struct sta_info *sta)
-{
-	struct sta_info *s;
-
-	s = ap->sta_hash[STA_HASH(sta->addr)];
-	if (s == NULL) return;
-	if (ether_addr_equal(s->addr, sta->addr)) {
-		ap->sta_hash[STA_HASH(sta->addr)] = s->hnext;
-		return;
-	}
-
-	while (s->hnext != NULL && !ether_addr_equal(s->hnext->addr, sta->addr))
-		s = s->hnext;
-	if (s->hnext != NULL)
-		s->hnext = s->hnext->hnext;
-	else
-		printk("AP: could not remove STA %pM from hash table\n",
-		       sta->addr);
-}
-
-static void ap_free_sta(struct ap_data *ap, struct sta_info *sta)
-{
-	if (sta->ap && sta->local)
-		hostap_event_expired_sta(sta->local->dev, sta);
-
-	if (ap->proc != NULL) {
-		char name[20];
-		sprintf(name, "%pM", sta->addr);
-		remove_proc_entry(name, ap->proc);
-	}
-
-	if (sta->crypt) {
-		sta->crypt->ops->deinit(sta->crypt->priv);
-		kfree(sta->crypt);
-		sta->crypt = NULL;
-	}
-
-	skb_queue_purge(&sta->tx_buf);
-
-	ap->num_sta--;
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-	if (sta->aid > 0)
-		ap->sta_aid[sta->aid - 1] = NULL;
-
-	if (!sta->ap)
-		kfree(sta->u.sta.challenge);
-	timer_shutdown_sync(&sta->timer);
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-
-	kfree(sta);
-}
-
-
-static void hostap_set_tim(local_info_t *local, int aid, int set)
-{
-	if (local->func->set_tim)
-		local->func->set_tim(local->dev, aid, set);
-}
-
-
-static void hostap_event_new_sta(struct net_device *dev, struct sta_info *sta)
-{
-	union iwreq_data wrqu;
-	memset(&wrqu, 0, sizeof(wrqu));
-	memcpy(wrqu.addr.sa_data, sta->addr, ETH_ALEN);
-	wrqu.addr.sa_family = ARPHRD_ETHER;
-	wireless_send_event(dev, IWEVREGISTERED, &wrqu, NULL);
-}
-
-
-static void hostap_event_expired_sta(struct net_device *dev,
-				     struct sta_info *sta)
-{
-	union iwreq_data wrqu;
-	memset(&wrqu, 0, sizeof(wrqu));
-	memcpy(wrqu.addr.sa_data, sta->addr, ETH_ALEN);
-	wrqu.addr.sa_family = ARPHRD_ETHER;
-	wireless_send_event(dev, IWEVEXPIRED, &wrqu, NULL);
-}
-
-
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-
-static void ap_handle_timer(struct timer_list *t)
-{
-	struct sta_info *sta = from_timer(sta, t, timer);
-	local_info_t *local;
-	struct ap_data *ap;
-	unsigned long next_time = 0;
-	int was_assoc;
-
-	if (sta == NULL || sta->local == NULL || sta->local->ap == NULL) {
-		PDEBUG(DEBUG_AP, "ap_handle_timer() called with NULL data\n");
-		return;
-	}
-
-	local = sta->local;
-	ap = local->ap;
-	was_assoc = sta->flags & WLAN_STA_ASSOC;
-
-	if (atomic_read(&sta->users) != 0)
-		next_time = jiffies + HZ;
-	else if ((sta->flags & WLAN_STA_PERM) && !(sta->flags & WLAN_STA_AUTH))
-		next_time = jiffies + ap->max_inactivity;
-
-	if (time_before(jiffies, sta->last_rx + ap->max_inactivity)) {
-		/* station activity detected; reset timeout state */
-		sta->timeout_next = STA_NULLFUNC;
-		next_time = sta->last_rx + ap->max_inactivity;
-	} else if (sta->timeout_next == STA_DISASSOC &&
-		   !(sta->flags & WLAN_STA_PENDING_POLL)) {
-		/* STA ACKed data nullfunc frame poll */
-		sta->timeout_next = STA_NULLFUNC;
-		next_time = jiffies + ap->max_inactivity;
-	}
-
-	if (next_time) {
-		sta->timer.expires = next_time;
-		add_timer(&sta->timer);
-		return;
-	}
-
-	if (sta->ap)
-		sta->timeout_next = STA_DEAUTH;
-
-	if (sta->timeout_next == STA_DEAUTH && !(sta->flags & WLAN_STA_PERM)) {
-		spin_lock(&ap->sta_table_lock);
-		ap_sta_hash_del(ap, sta);
-		list_del(&sta->list);
-		spin_unlock(&ap->sta_table_lock);
-		sta->flags &= ~(WLAN_STA_AUTH | WLAN_STA_ASSOC);
-	} else if (sta->timeout_next == STA_DISASSOC)
-		sta->flags &= ~WLAN_STA_ASSOC;
-
-	if (was_assoc && !(sta->flags & WLAN_STA_ASSOC) && !sta->ap)
-		hostap_event_expired_sta(local->dev, sta);
-
-	if (sta->timeout_next == STA_DEAUTH && sta->aid > 0 &&
-	    !skb_queue_empty(&sta->tx_buf)) {
-		hostap_set_tim(local, sta->aid, 0);
-		sta->flags &= ~WLAN_STA_TIM;
-	}
-
-	if (sta->ap) {
-		if (ap->autom_ap_wds) {
-			PDEBUG(DEBUG_AP, "%s: removing automatic WDS "
-			       "connection to AP %pM\n",
-			       local->dev->name, sta->addr);
-			hostap_wds_link_oper(local, sta->addr, WDS_DEL);
-		}
-	} else if (sta->timeout_next == STA_NULLFUNC) {
-		/* send data frame to poll STA and check whether this frame
-		 * is ACKed */
-		/* FIX: IEEE80211_STYPE_NULLFUNC would be more appropriate, but
-		 * it is apparently not retried so TX Exc events are not
-		 * received for it */
-		sta->flags |= WLAN_STA_PENDING_POLL;
-		prism2_send_mgmt(local->dev, IEEE80211_FTYPE_DATA |
-				 IEEE80211_STYPE_DATA, NULL, 0,
-				 sta->addr, ap->tx_callback_poll);
-	} else {
-		int deauth = sta->timeout_next == STA_DEAUTH;
-		__le16 resp;
-		PDEBUG(DEBUG_AP, "%s: sending %s info to STA %pM"
-		       "(last=%lu, jiffies=%lu)\n",
-		       local->dev->name,
-		       deauth ? "deauthentication" : "disassociation",
-		       sta->addr, sta->last_rx, jiffies);
-
-		resp = cpu_to_le16(deauth ? WLAN_REASON_PREV_AUTH_NOT_VALID :
-				   WLAN_REASON_DISASSOC_DUE_TO_INACTIVITY);
-		prism2_send_mgmt(local->dev, IEEE80211_FTYPE_MGMT |
-				 (deauth ? IEEE80211_STYPE_DEAUTH :
-				  IEEE80211_STYPE_DISASSOC),
-				 (char *) &resp, 2, sta->addr, 0);
-	}
-
-	if (sta->timeout_next == STA_DEAUTH) {
-		if (sta->flags & WLAN_STA_PERM) {
-			PDEBUG(DEBUG_AP, "%s: STA %pM"
-			       " would have been removed, "
-			       "but it has 'perm' flag\n",
-			       local->dev->name, sta->addr);
-		} else
-			ap_free_sta(ap, sta);
-		return;
-	}
-
-	if (sta->timeout_next == STA_NULLFUNC) {
-		sta->timeout_next = STA_DISASSOC;
-		sta->timer.expires = jiffies + AP_DISASSOC_DELAY;
-	} else {
-		sta->timeout_next = STA_DEAUTH;
-		sta->timer.expires = jiffies + AP_DEAUTH_DELAY;
-	}
-
-	add_timer(&sta->timer);
-}
-
-
-void hostap_deauth_all_stas(struct net_device *dev, struct ap_data *ap,
-			    int resend)
-{
-	u8 addr[ETH_ALEN];
-	__le16 resp;
-	int i;
-
-	PDEBUG(DEBUG_AP, "%s: Deauthenticate all stations\n", dev->name);
-	eth_broadcast_addr(addr);
-
-	resp = cpu_to_le16(WLAN_REASON_PREV_AUTH_NOT_VALID);
-
-	/* deauth message sent; try to resend it few times; the message is
-	 * broadcast, so it may be delayed until next DTIM; there is not much
-	 * else we can do at this point since the driver is going to be shut
-	 * down */
-	for (i = 0; i < 5; i++) {
-		prism2_send_mgmt(dev, IEEE80211_FTYPE_MGMT |
-				 IEEE80211_STYPE_DEAUTH,
-				 (char *) &resp, 2, addr, 0);
-
-		if (!resend || ap->num_sta <= 0)
-			return;
-
-		mdelay(50);
-	}
-}
-
-
-static int ap_control_proc_show(struct seq_file *m, void *v)
-{
-	struct ap_data *ap = pde_data(file_inode(m->file));
-	char *policy_txt;
-	struct mac_entry *entry;
-
-	if (v == SEQ_START_TOKEN) {
-		switch (ap->mac_restrictions.policy) {
-		case MAC_POLICY_OPEN:
-			policy_txt = "open";
-			break;
-		case MAC_POLICY_ALLOW:
-			policy_txt = "allow";
-			break;
-		case MAC_POLICY_DENY:
-			policy_txt = "deny";
-			break;
-		default:
-			policy_txt = "unknown";
-			break;
-		}
-		seq_printf(m, "MAC policy: %s\n", policy_txt);
-		seq_printf(m, "MAC entries: %u\n", ap->mac_restrictions.entries);
-		seq_puts(m, "MAC list:\n");
-		return 0;
-	}
-
-	entry = v;
-	seq_printf(m, "%pM\n", entry->addr);
-	return 0;
-}
-
-static void *ap_control_proc_start(struct seq_file *m, loff_t *_pos)
-{
-	struct ap_data *ap = pde_data(file_inode(m->file));
-	spin_lock_bh(&ap->mac_restrictions.lock);
-	return seq_list_start_head(&ap->mac_restrictions.mac_list, *_pos);
-}
-
-static void *ap_control_proc_next(struct seq_file *m, void *v, loff_t *_pos)
-{
-	struct ap_data *ap = pde_data(file_inode(m->file));
-	return seq_list_next(v, &ap->mac_restrictions.mac_list, _pos);
-}
-
-static void ap_control_proc_stop(struct seq_file *m, void *v)
-{
-	struct ap_data *ap = pde_data(file_inode(m->file));
-	spin_unlock_bh(&ap->mac_restrictions.lock);
-}
-
-static const struct seq_operations ap_control_proc_seqops = {
-	.start	= ap_control_proc_start,
-	.next	= ap_control_proc_next,
-	.stop	= ap_control_proc_stop,
-	.show	= ap_control_proc_show,
-};
-
-int ap_control_add_mac(struct mac_restrictions *mac_restrictions, u8 *mac)
-{
-	struct mac_entry *entry;
-
-	entry = kmalloc(sizeof(struct mac_entry), GFP_KERNEL);
-	if (entry == NULL)
-		return -ENOMEM;
-
-	memcpy(entry->addr, mac, ETH_ALEN);
-
-	spin_lock_bh(&mac_restrictions->lock);
-	list_add_tail(&entry->list, &mac_restrictions->mac_list);
-	mac_restrictions->entries++;
-	spin_unlock_bh(&mac_restrictions->lock);
-
-	return 0;
-}
-
-
-int ap_control_del_mac(struct mac_restrictions *mac_restrictions, u8 *mac)
-{
-	struct list_head *ptr;
-	struct mac_entry *entry;
-
-	spin_lock_bh(&mac_restrictions->lock);
-	for (ptr = mac_restrictions->mac_list.next;
-	     ptr != &mac_restrictions->mac_list; ptr = ptr->next) {
-		entry = list_entry(ptr, struct mac_entry, list);
-
-		if (ether_addr_equal(entry->addr, mac)) {
-			list_del(ptr);
-			kfree(entry);
-			mac_restrictions->entries--;
-			spin_unlock_bh(&mac_restrictions->lock);
-			return 0;
-		}
-	}
-	spin_unlock_bh(&mac_restrictions->lock);
-	return -1;
-}
-
-
-static int ap_control_mac_deny(struct mac_restrictions *mac_restrictions,
-			       u8 *mac)
-{
-	struct mac_entry *entry;
-	int found = 0;
-
-	if (mac_restrictions->policy == MAC_POLICY_OPEN)
-		return 0;
-
-	spin_lock_bh(&mac_restrictions->lock);
-	list_for_each_entry(entry, &mac_restrictions->mac_list, list) {
-		if (ether_addr_equal(entry->addr, mac)) {
-			found = 1;
-			break;
-		}
-	}
-	spin_unlock_bh(&mac_restrictions->lock);
-
-	if (mac_restrictions->policy == MAC_POLICY_ALLOW)
-		return !found;
-	else
-		return found;
-}
-
-
-void ap_control_flush_macs(struct mac_restrictions *mac_restrictions)
-{
-	struct list_head *ptr, *n;
-	struct mac_entry *entry;
-
-	if (mac_restrictions->entries == 0)
-		return;
-
-	spin_lock_bh(&mac_restrictions->lock);
-	for (ptr = mac_restrictions->mac_list.next, n = ptr->next;
-	     ptr != &mac_restrictions->mac_list;
-	     ptr = n, n = ptr->next) {
-		entry = list_entry(ptr, struct mac_entry, list);
-		list_del(ptr);
-		kfree(entry);
-	}
-	mac_restrictions->entries = 0;
-	spin_unlock_bh(&mac_restrictions->lock);
-}
-
-
-int ap_control_kick_mac(struct ap_data *ap, struct net_device *dev, u8 *mac)
-{
-	struct sta_info *sta;
-	__le16 resp;
-
-	spin_lock_bh(&ap->sta_table_lock);
-	sta = ap_get_sta(ap, mac);
-	if (sta) {
-		ap_sta_hash_del(ap, sta);
-		list_del(&sta->list);
-	}
-	spin_unlock_bh(&ap->sta_table_lock);
-
-	if (!sta)
-		return -EINVAL;
-
-	resp = cpu_to_le16(WLAN_REASON_PREV_AUTH_NOT_VALID);
-	prism2_send_mgmt(dev, IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_DEAUTH,
-			 (char *) &resp, 2, sta->addr, 0);
-
-	if ((sta->flags & WLAN_STA_ASSOC) && !sta->ap)
-		hostap_event_expired_sta(dev, sta);
-
-	ap_free_sta(ap, sta);
-
-	return 0;
-}
-
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-
-
-void ap_control_kickall(struct ap_data *ap)
-{
-	struct list_head *ptr, *n;
-	struct sta_info *sta;
-
-	spin_lock_bh(&ap->sta_table_lock);
-	for (ptr = ap->sta_list.next, n = ptr->next; ptr != &ap->sta_list;
-	     ptr = n, n = ptr->next) {
-		sta = list_entry(ptr, struct sta_info, list);
-		ap_sta_hash_del(ap, sta);
-		list_del(&sta->list);
-		if ((sta->flags & WLAN_STA_ASSOC) && !sta->ap && sta->local)
-			hostap_event_expired_sta(sta->local->dev, sta);
-		ap_free_sta(ap, sta);
-	}
-	spin_unlock_bh(&ap->sta_table_lock);
-}
-
-
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-
-static int prism2_ap_proc_show(struct seq_file *m, void *v)
-{
-	struct sta_info *sta = v;
-	int i;
-
-	if (v == SEQ_START_TOKEN) {
-		seq_printf(m, "# BSSID CHAN SIGNAL NOISE RATE SSID FLAGS\n");
-		return 0;
-	}
-
-	if (!sta->ap)
-		return 0;
-
-	seq_printf(m, "%pM %d %d %d %d '",
-		   sta->addr,
-		   sta->u.ap.channel, sta->last_rx_signal,
-		   sta->last_rx_silence, sta->last_rx_rate);
-
-	for (i = 0; i < sta->u.ap.ssid_len; i++) {
-		if (sta->u.ap.ssid[i] >= 32 && sta->u.ap.ssid[i] < 127)
-			seq_putc(m, sta->u.ap.ssid[i]);
-		else
-			seq_printf(m, "<%02x>", sta->u.ap.ssid[i]);
-	}
-
-	seq_putc(m, '\'');
-	if (sta->capability & WLAN_CAPABILITY_ESS)
-		seq_puts(m, " [ESS]");
-	if (sta->capability & WLAN_CAPABILITY_IBSS)
-		seq_puts(m, " [IBSS]");
-	if (sta->capability & WLAN_CAPABILITY_PRIVACY)
-		seq_puts(m, " [WEP]");
-	seq_putc(m, '\n');
-	return 0;
-}
-
-static void *prism2_ap_proc_start(struct seq_file *m, loff_t *_pos)
-{
-	struct ap_data *ap = pde_data(file_inode(m->file));
-	spin_lock_bh(&ap->sta_table_lock);
-	return seq_list_start_head(&ap->sta_list, *_pos);
-}
-
-static void *prism2_ap_proc_next(struct seq_file *m, void *v, loff_t *_pos)
-{
-	struct ap_data *ap = pde_data(file_inode(m->file));
-	return seq_list_next(v, &ap->sta_list, _pos);
-}
-
-static void prism2_ap_proc_stop(struct seq_file *m, void *v)
-{
-	struct ap_data *ap = pde_data(file_inode(m->file));
-	spin_unlock_bh(&ap->sta_table_lock);
-}
-
-static const struct seq_operations prism2_ap_proc_seqops = {
-	.start	= prism2_ap_proc_start,
-	.next	= prism2_ap_proc_next,
-	.stop	= prism2_ap_proc_stop,
-	.show	= prism2_ap_proc_show,
-};
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-
-
-void hostap_check_sta_fw_version(struct ap_data *ap, int sta_fw_ver)
-{
-	if (!ap)
-		return;
-
-	if (sta_fw_ver == PRISM2_FW_VER(0,8,0)) {
-		PDEBUG(DEBUG_AP, "Using data::nullfunc ACK workaround - "
-		       "firmware upgrade recommended\n");
-		ap->nullfunc_ack = 1;
-	} else
-		ap->nullfunc_ack = 0;
-
-	if (sta_fw_ver == PRISM2_FW_VER(1,4,2)) {
-		printk(KERN_WARNING "%s: Warning: secondary station firmware "
-		       "version 1.4.2 does not seem to work in Host AP mode\n",
-		       ap->local->dev->name);
-	}
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static void hostap_ap_tx_cb(struct sk_buff *skb, int ok, void *data)
-{
-	struct ap_data *ap = data;
-	struct ieee80211_hdr *hdr;
-
-	if (!ap->local->hostapd || !ap->local->apdev) {
-		dev_kfree_skb(skb);
-		return;
-	}
-
-	/* Pass the TX callback frame to the hostapd; use 802.11 header version
-	 * 1 to indicate failure (no ACK) and 2 success (frame ACKed) */
-
-	hdr = (struct ieee80211_hdr *) skb->data;
-	hdr->frame_control &= cpu_to_le16(~IEEE80211_FCTL_VERS);
-	hdr->frame_control |= cpu_to_le16(ok ? BIT(1) : BIT(0));
-
-	skb->dev = ap->local->apdev;
-	skb_pull(skb, hostap_80211_get_hdrlen(hdr->frame_control));
-	skb->pkt_type = PACKET_OTHERHOST;
-	skb->protocol = cpu_to_be16(ETH_P_802_2);
-	memset(skb->cb, 0, sizeof(skb->cb));
-	netif_rx(skb);
-}
-
-
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-/* Called only as a tasklet (software IRQ) */
-static void hostap_ap_tx_cb_auth(struct sk_buff *skb, int ok, void *data)
-{
-	struct ap_data *ap = data;
-	struct net_device *dev = ap->local->dev;
-	struct ieee80211_hdr *hdr;
-	u16 auth_alg, auth_transaction, status;
-	__le16 *pos;
-	struct sta_info *sta = NULL;
-	char *txt = NULL;
-
-	if (ap->local->hostapd) {
-		dev_kfree_skb(skb);
-		return;
-	}
-
-	hdr = (struct ieee80211_hdr *) skb->data;
-	if (!ieee80211_is_auth(hdr->frame_control) ||
-	    skb->len < IEEE80211_MGMT_HDR_LEN + 6) {
-		printk(KERN_DEBUG "%s: hostap_ap_tx_cb_auth received invalid "
-		       "frame\n", dev->name);
-		dev_kfree_skb(skb);
-		return;
-	}
-
-	pos = (__le16 *) (skb->data + IEEE80211_MGMT_HDR_LEN);
-	auth_alg = le16_to_cpu(*pos++);
-	auth_transaction = le16_to_cpu(*pos++);
-	status = le16_to_cpu(*pos++);
-
-	if (!ok) {
-		txt = "frame was not ACKed";
-		goto done;
-	}
-
-	spin_lock(&ap->sta_table_lock);
-	sta = ap_get_sta(ap, hdr->addr1);
-	if (sta)
-		atomic_inc(&sta->users);
-	spin_unlock(&ap->sta_table_lock);
-
-	if (!sta) {
-		txt = "STA not found";
-		goto done;
-	}
-
-	if (status == WLAN_STATUS_SUCCESS &&
-	    ((auth_alg == WLAN_AUTH_OPEN && auth_transaction == 2) ||
-	     (auth_alg == WLAN_AUTH_SHARED_KEY && auth_transaction == 4))) {
-		txt = "STA authenticated";
-		sta->flags |= WLAN_STA_AUTH;
-		sta->last_auth = jiffies;
-	} else if (status != WLAN_STATUS_SUCCESS)
-		txt = "authentication failed";
-
- done:
-	if (sta)
-		atomic_dec(&sta->users);
-	if (txt) {
-		PDEBUG(DEBUG_AP, "%s: %pM auth_cb - alg=%d "
-		       "trans#=%d status=%d - %s\n",
-		       dev->name, hdr->addr1,
-		       auth_alg, auth_transaction, status, txt);
-	}
-	dev_kfree_skb(skb);
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static void hostap_ap_tx_cb_assoc(struct sk_buff *skb, int ok, void *data)
-{
-	struct ap_data *ap = data;
-	struct net_device *dev = ap->local->dev;
-	struct ieee80211_hdr *hdr;
-	u16 status;
-	__le16 *pos;
-	struct sta_info *sta = NULL;
-	char *txt = NULL;
-
-	if (ap->local->hostapd) {
-		dev_kfree_skb(skb);
-		return;
-	}
-
-	hdr = (struct ieee80211_hdr *) skb->data;
-	if ((!ieee80211_is_assoc_resp(hdr->frame_control) &&
-	     !ieee80211_is_reassoc_resp(hdr->frame_control)) ||
-	    skb->len < IEEE80211_MGMT_HDR_LEN + 4) {
-		printk(KERN_DEBUG "%s: hostap_ap_tx_cb_assoc received invalid "
-		       "frame\n", dev->name);
-		dev_kfree_skb(skb);
-		return;
-	}
-
-	if (!ok) {
-		txt = "frame was not ACKed";
-		goto done;
-	}
-
-	spin_lock(&ap->sta_table_lock);
-	sta = ap_get_sta(ap, hdr->addr1);
-	if (sta)
-		atomic_inc(&sta->users);
-	spin_unlock(&ap->sta_table_lock);
-
-	if (!sta) {
-		txt = "STA not found";
-		goto done;
-	}
-
-	pos = (__le16 *) (skb->data + IEEE80211_MGMT_HDR_LEN);
-	pos++;
-	status = le16_to_cpu(*pos++);
-	if (status == WLAN_STATUS_SUCCESS) {
-		if (!(sta->flags & WLAN_STA_ASSOC))
-			hostap_event_new_sta(dev, sta);
-		txt = "STA associated";
-		sta->flags |= WLAN_STA_ASSOC;
-		sta->last_assoc = jiffies;
-	} else
-		txt = "association failed";
-
- done:
-	if (sta)
-		atomic_dec(&sta->users);
-	if (txt) {
-		PDEBUG(DEBUG_AP, "%s: %pM assoc_cb - %s\n",
-		       dev->name, hdr->addr1, txt);
-	}
-	dev_kfree_skb(skb);
-}
-
-/* Called only as a tasklet (software IRQ); TX callback for poll frames used
- * in verifying whether the STA is still present. */
-static void hostap_ap_tx_cb_poll(struct sk_buff *skb, int ok, void *data)
-{
-	struct ap_data *ap = data;
-	struct ieee80211_hdr *hdr;
-	struct sta_info *sta;
-
-	if (skb->len < 24)
-		goto fail;
-	hdr = (struct ieee80211_hdr *) skb->data;
-	if (ok) {
-		spin_lock(&ap->sta_table_lock);
-		sta = ap_get_sta(ap, hdr->addr1);
-		if (sta)
-			sta->flags &= ~WLAN_STA_PENDING_POLL;
-		spin_unlock(&ap->sta_table_lock);
-	} else {
-		PDEBUG(DEBUG_AP,
-		       "%s: STA %pM did not ACK activity poll frame\n",
-		       ap->local->dev->name, hdr->addr1);
-	}
-
- fail:
-	dev_kfree_skb(skb);
-}
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-
-
-void hostap_init_data(local_info_t *local)
-{
-	struct ap_data *ap = local->ap;
-
-	if (ap == NULL) {
-		printk(KERN_WARNING "hostap_init_data: ap == NULL\n");
-		return;
-	}
-	memset(ap, 0, sizeof(struct ap_data));
-	ap->local = local;
-
-	ap->ap_policy = GET_INT_PARM(other_ap_policy, local->card_idx);
-	ap->bridge_packets = GET_INT_PARM(ap_bridge_packets, local->card_idx);
-	ap->max_inactivity =
-		GET_INT_PARM(ap_max_inactivity, local->card_idx) * HZ;
-	ap->autom_ap_wds = GET_INT_PARM(autom_ap_wds, local->card_idx);
-
-	spin_lock_init(&ap->sta_table_lock);
-	INIT_LIST_HEAD(&ap->sta_list);
-
-	/* Initialize task queue structure for AP management */
-	INIT_WORK(&local->ap->add_sta_proc_queue, handle_add_proc_queue);
-
-	ap->tx_callback_idx =
-		hostap_tx_callback_register(local, hostap_ap_tx_cb, ap);
-	if (ap->tx_callback_idx == 0)
-		printk(KERN_WARNING "%s: failed to register TX callback for "
-		       "AP\n", local->dev->name);
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-	INIT_WORK(&local->ap->wds_oper_queue, handle_wds_oper_queue);
-
-	ap->tx_callback_auth =
-		hostap_tx_callback_register(local, hostap_ap_tx_cb_auth, ap);
-	ap->tx_callback_assoc =
-		hostap_tx_callback_register(local, hostap_ap_tx_cb_assoc, ap);
-	ap->tx_callback_poll =
-		hostap_tx_callback_register(local, hostap_ap_tx_cb_poll, ap);
-	if (ap->tx_callback_auth == 0 || ap->tx_callback_assoc == 0 ||
-		ap->tx_callback_poll == 0)
-		printk(KERN_WARNING "%s: failed to register TX callback for "
-		       "AP\n", local->dev->name);
-
-	spin_lock_init(&ap->mac_restrictions.lock);
-	INIT_LIST_HEAD(&ap->mac_restrictions.mac_list);
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-
-	ap->initialized = 1;
-}
-
-
-void hostap_init_ap_proc(local_info_t *local)
-{
-	struct ap_data *ap = local->ap;
-
-	ap->proc = local->proc;
-	if (ap->proc == NULL)
-		return;
-
-#ifndef PRISM2_NO_PROCFS_DEBUG
-	proc_create_single_data("ap_debug", 0, ap->proc, ap_debug_proc_show, ap);
-#endif /* PRISM2_NO_PROCFS_DEBUG */
-
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-	proc_create_seq_data("ap_control", 0, ap->proc, &ap_control_proc_seqops,
-			ap);
-	proc_create_seq_data("ap", 0, ap->proc, &prism2_ap_proc_seqops, ap);
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-
-}
-
-
-void hostap_free_data(struct ap_data *ap)
-{
-	struct sta_info *n, *sta;
-
-	if (ap == NULL || !ap->initialized) {
-		printk(KERN_DEBUG "hostap_free_data: ap has not yet been "
-		       "initialized - skip resource freeing\n");
-		return;
-	}
-
-	flush_work(&ap->add_sta_proc_queue);
-
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-	flush_work(&ap->wds_oper_queue);
-	if (ap->crypt)
-		ap->crypt->deinit(ap->crypt_priv);
-	ap->crypt = ap->crypt_priv = NULL;
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-
-	list_for_each_entry_safe(sta, n, &ap->sta_list, list) {
-		ap_sta_hash_del(ap, sta);
-		list_del(&sta->list);
-		if ((sta->flags & WLAN_STA_ASSOC) && !sta->ap && sta->local)
-			hostap_event_expired_sta(sta->local->dev, sta);
-		ap_free_sta(ap, sta);
-	}
-
-#ifndef PRISM2_NO_PROCFS_DEBUG
-	if (ap->proc != NULL) {
-		remove_proc_entry("ap_debug", ap->proc);
-	}
-#endif /* PRISM2_NO_PROCFS_DEBUG */
-
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-	if (ap->proc != NULL) {
-	  remove_proc_entry("ap", ap->proc);
-		remove_proc_entry("ap_control", ap->proc);
-	}
-	ap_control_flush_macs(&ap->mac_restrictions);
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-
-	ap->initialized = 0;
-}
-
-
-/* caller should have mutex for AP STA list handling */
-static struct sta_info* ap_get_sta(struct ap_data *ap, u8 *sta)
-{
-	struct sta_info *s;
-
-	s = ap->sta_hash[STA_HASH(sta)];
-	while (s != NULL && !ether_addr_equal(s->addr, sta))
-		s = s->hnext;
-	return s;
-}
-
-
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-
-/* Called from timer handler and from scheduled AP queue handlers */
-static void prism2_send_mgmt(struct net_device *dev,
-			     u16 type_subtype, char *body,
-			     int body_len, u8 *addr, u16 tx_cb_idx)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	struct ieee80211_hdr *hdr;
-	u16 fc;
-	struct sk_buff *skb;
-	struct hostap_skb_tx_data *meta;
-	int hdrlen;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-	dev = local->dev; /* always use master radio device */
-	iface = netdev_priv(dev);
-
-	if (!(dev->flags & IFF_UP)) {
-		PDEBUG(DEBUG_AP, "%s: prism2_send_mgmt - device is not UP - "
-		       "cannot send frame\n", dev->name);
-		return;
-	}
-
-	skb = dev_alloc_skb(sizeof(*hdr) + body_len);
-	if (skb == NULL) {
-		PDEBUG(DEBUG_AP, "%s: prism2_send_mgmt failed to allocate "
-		       "skb\n", dev->name);
-		return;
-	}
-
-	fc = type_subtype;
-	hdrlen = hostap_80211_get_hdrlen(cpu_to_le16(type_subtype));
-	hdr = skb_put_zero(skb, hdrlen);
-	if (body)
-		skb_put_data(skb, body, body_len);
-
-	/* FIX: ctrl::ack sending used special HFA384X_TX_CTRL_802_11
-	 * tx_control instead of using local->tx_control */
-
-
-	memcpy(hdr->addr1, addr, ETH_ALEN); /* DA / RA */
-	if (ieee80211_is_data(hdr->frame_control)) {
-		fc |= IEEE80211_FCTL_FROMDS;
-		memcpy(hdr->addr2, dev->dev_addr, ETH_ALEN); /* BSSID */
-		memcpy(hdr->addr3, dev->dev_addr, ETH_ALEN); /* SA */
-	} else if (ieee80211_is_ctl(hdr->frame_control)) {
-		/* control:ACK does not have addr2 or addr3 */
-		eth_zero_addr(hdr->addr2);
-		eth_zero_addr(hdr->addr3);
-	} else {
-		memcpy(hdr->addr2, dev->dev_addr, ETH_ALEN); /* SA */
-		memcpy(hdr->addr3, dev->dev_addr, ETH_ALEN); /* BSSID */
-	}
-
-	hdr->frame_control = cpu_to_le16(fc);
-
-	meta = (struct hostap_skb_tx_data *) skb->cb;
-	memset(meta, 0, sizeof(*meta));
-	meta->magic = HOSTAP_SKB_TX_DATA_MAGIC;
-	meta->iface = iface;
-	meta->tx_cb_idx = tx_cb_idx;
-
-	skb->dev = dev;
-	skb_reset_mac_header(skb);
-	skb_reset_network_header(skb);
-	dev_queue_xmit(skb);
-}
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-
-#ifdef CONFIG_PROC_FS
-static int prism2_sta_proc_show(struct seq_file *m, void *v)
-{
-	struct sta_info *sta = m->private;
-	int i;
-
-	/* FIX: possible race condition.. the STA data could have just expired,
-	 * but proc entry was still here so that the read could have started;
-	 * some locking should be done here.. */
-
-	seq_printf(m,
-		   "%s=%pM\nusers=%d\naid=%d\n"
-		   "flags=0x%04x%s%s%s%s%s%s%s\n"
-		   "capability=0x%02x\nlisten_interval=%d\nsupported_rates=",
-		   sta->ap ? "AP" : "STA",
-		   sta->addr, atomic_read(&sta->users), sta->aid,
-		   sta->flags,
-		   sta->flags & WLAN_STA_AUTH ? " AUTH" : "",
-		   sta->flags & WLAN_STA_ASSOC ? " ASSOC" : "",
-		   sta->flags & WLAN_STA_PS ? " PS" : "",
-		   sta->flags & WLAN_STA_TIM ? " TIM" : "",
-		   sta->flags & WLAN_STA_PERM ? " PERM" : "",
-		   sta->flags & WLAN_STA_AUTHORIZED ? " AUTHORIZED" : "",
-		   sta->flags & WLAN_STA_PENDING_POLL ? " POLL" : "",
-		   sta->capability, sta->listen_interval);
-	/* supported_rates: 500 kbit/s units with msb ignored */
-	for (i = 0; i < sizeof(sta->supported_rates); i++)
-		if (sta->supported_rates[i] != 0)
-			seq_printf(m, "%d%sMbps ",
-				   (sta->supported_rates[i] & 0x7f) / 2,
-				   sta->supported_rates[i] & 1 ? ".5" : "");
-	seq_printf(m,
-		   "\njiffies=%lu\nlast_auth=%lu\nlast_assoc=%lu\n"
-		   "last_rx=%lu\nlast_tx=%lu\nrx_packets=%lu\n"
-		   "tx_packets=%lu\n"
-		   "rx_bytes=%lu\ntx_bytes=%lu\nbuffer_count=%d\n"
-		   "last_rx: silence=%d dBm signal=%d dBm rate=%d%s Mbps\n"
-		   "tx_rate=%d\ntx[1M]=%d\ntx[2M]=%d\ntx[5.5M]=%d\n"
-		   "tx[11M]=%d\n"
-		   "rx[1M]=%d\nrx[2M]=%d\nrx[5.5M]=%d\nrx[11M]=%d\n",
-		   jiffies, sta->last_auth, sta->last_assoc, sta->last_rx,
-		   sta->last_tx,
-		   sta->rx_packets, sta->tx_packets, sta->rx_bytes,
-		   sta->tx_bytes, skb_queue_len(&sta->tx_buf),
-		   sta->last_rx_silence,
-		   sta->last_rx_signal, sta->last_rx_rate / 10,
-		   sta->last_rx_rate % 10 ? ".5" : "",
-		   sta->tx_rate, sta->tx_count[0], sta->tx_count[1],
-		   sta->tx_count[2], sta->tx_count[3],  sta->rx_count[0],
-		   sta->rx_count[1], sta->rx_count[2], sta->rx_count[3]);
-	if (sta->crypt && sta->crypt->ops && sta->crypt->ops->print_stats)
-		sta->crypt->ops->print_stats(m, sta->crypt->priv);
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-	if (sta->ap) {
-		if (sta->u.ap.channel >= 0)
-			seq_printf(m, "channel=%d\n", sta->u.ap.channel);
-		seq_puts(m, "ssid=");
-		for (i = 0; i < sta->u.ap.ssid_len; i++) {
-			if (sta->u.ap.ssid[i] >= 32 && sta->u.ap.ssid[i] < 127)
-				seq_putc(m, sta->u.ap.ssid[i]);
-			else
-				seq_printf(m, "<%02x>", sta->u.ap.ssid[i]);
-		}
-		seq_putc(m, '\n');
-	}
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-
-	return 0;
-}
-#endif
-
-static void handle_add_proc_queue(struct work_struct *work)
-{
-	struct ap_data *ap = container_of(work, struct ap_data,
-					  add_sta_proc_queue);
-	struct sta_info *sta;
-	char name[20];
-	struct add_sta_proc_data *entry, *prev;
-
-	entry = ap->add_sta_proc_entries;
-	ap->add_sta_proc_entries = NULL;
-
-	while (entry) {
-		spin_lock_bh(&ap->sta_table_lock);
-		sta = ap_get_sta(ap, entry->addr);
-		if (sta)
-			atomic_inc(&sta->users);
-		spin_unlock_bh(&ap->sta_table_lock);
-
-		if (sta) {
-			sprintf(name, "%pM", sta->addr);
-			sta->proc = proc_create_single_data(
-				name, 0, ap->proc,
-				prism2_sta_proc_show, sta);
-
-			atomic_dec(&sta->users);
-		}
-
-		prev = entry;
-		entry = entry->next;
-		kfree(prev);
-	}
-}
-
-
-static struct sta_info * ap_add_sta(struct ap_data *ap, u8 *addr)
-{
-	struct sta_info *sta;
-
-	sta = kzalloc(sizeof(struct sta_info), GFP_ATOMIC);
-	if (sta == NULL) {
-		PDEBUG(DEBUG_AP, "AP: kmalloc failed\n");
-		return NULL;
-	}
-
-	/* initialize STA info data */
-	sta->local = ap->local;
-	skb_queue_head_init(&sta->tx_buf);
-	memcpy(sta->addr, addr, ETH_ALEN);
-
-	atomic_inc(&sta->users);
-	spin_lock_bh(&ap->sta_table_lock);
-	list_add(&sta->list, &ap->sta_list);
-	ap->num_sta++;
-	ap_sta_hash_add(ap, sta);
-	spin_unlock_bh(&ap->sta_table_lock);
-
-	if (ap->proc) {
-		struct add_sta_proc_data *entry;
-		/* schedule a non-interrupt context process to add a procfs
-		 * entry for the STA since procfs code use GFP_KERNEL */
-		entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
-		if (entry) {
-			memcpy(entry->addr, sta->addr, ETH_ALEN);
-			entry->next = ap->add_sta_proc_entries;
-			ap->add_sta_proc_entries = entry;
-			schedule_work(&ap->add_sta_proc_queue);
-		} else
-			printk(KERN_DEBUG "Failed to add STA proc data\n");
-	}
-
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-	timer_setup(&sta->timer, ap_handle_timer, 0);
-	sta->timer.expires = jiffies + ap->max_inactivity;
-	if (!ap->local->hostapd)
-		add_timer(&sta->timer);
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-
-	return sta;
-}
-
-
-static int ap_tx_rate_ok(int rateidx, struct sta_info *sta,
-			 local_info_t *local)
-{
-	if (rateidx > sta->tx_max_rate ||
-	    !(sta->tx_supp_rates & (1 << rateidx)))
-		return 0;
-
-	if (local->tx_rate_control != 0 &&
-	    !(local->tx_rate_control & (1 << rateidx)))
-		return 0;
-
-	return 1;
-}
-
-
-static void prism2_check_tx_rates(struct sta_info *sta)
-{
-	int i;
-
-	sta->tx_supp_rates = 0;
-	for (i = 0; i < sizeof(sta->supported_rates); i++) {
-		if ((sta->supported_rates[i] & 0x7f) == 2)
-			sta->tx_supp_rates |= WLAN_RATE_1M;
-		if ((sta->supported_rates[i] & 0x7f) == 4)
-			sta->tx_supp_rates |= WLAN_RATE_2M;
-		if ((sta->supported_rates[i] & 0x7f) == 11)
-			sta->tx_supp_rates |= WLAN_RATE_5M5;
-		if ((sta->supported_rates[i] & 0x7f) == 22)
-			sta->tx_supp_rates |= WLAN_RATE_11M;
-	}
-	sta->tx_max_rate = sta->tx_rate = sta->tx_rate_idx = 0;
-	if (sta->tx_supp_rates & WLAN_RATE_1M) {
-		sta->tx_max_rate = 0;
-		if (ap_tx_rate_ok(0, sta, sta->local)) {
-			sta->tx_rate = 10;
-			sta->tx_rate_idx = 0;
-		}
-	}
-	if (sta->tx_supp_rates & WLAN_RATE_2M) {
-		sta->tx_max_rate = 1;
-		if (ap_tx_rate_ok(1, sta, sta->local)) {
-			sta->tx_rate = 20;
-			sta->tx_rate_idx = 1;
-		}
-	}
-	if (sta->tx_supp_rates & WLAN_RATE_5M5) {
-		sta->tx_max_rate = 2;
-		if (ap_tx_rate_ok(2, sta, sta->local)) {
-			sta->tx_rate = 55;
-			sta->tx_rate_idx = 2;
-		}
-	}
-	if (sta->tx_supp_rates & WLAN_RATE_11M) {
-		sta->tx_max_rate = 3;
-		if (ap_tx_rate_ok(3, sta, sta->local)) {
-			sta->tx_rate = 110;
-			sta->tx_rate_idx = 3;
-		}
-	}
-}
-
-
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-
-static void ap_crypt_init(struct ap_data *ap)
-{
-	ap->crypt = lib80211_get_crypto_ops("WEP");
-
-	if (ap->crypt) {
-		if (ap->crypt->init) {
-			ap->crypt_priv = ap->crypt->init(0);
-			if (ap->crypt_priv == NULL)
-				ap->crypt = NULL;
-			else {
-				u8 key[WEP_KEY_LEN];
-				get_random_bytes(key, WEP_KEY_LEN);
-				ap->crypt->set_key(key, WEP_KEY_LEN, NULL,
-						   ap->crypt_priv);
-			}
-		}
-	}
-
-	if (ap->crypt == NULL) {
-		printk(KERN_WARNING "AP could not initialize WEP: load module "
-		       "lib80211_crypt_wep.ko\n");
-	}
-}
-
-
-/* Generate challenge data for shared key authentication. IEEE 802.11 specifies
- * that WEP algorithm is used for generating challenge. This should be unique,
- * but otherwise there is not really need for randomness etc. Initialize WEP
- * with pseudo random key and then use increasing IV to get unique challenge
- * streams.
- *
- * Called only as a scheduled task for pending AP frames.
- */
-static char * ap_auth_make_challenge(struct ap_data *ap)
-{
-	char *tmpbuf;
-	struct sk_buff *skb;
-
-	if (ap->crypt == NULL) {
-		ap_crypt_init(ap);
-		if (ap->crypt == NULL)
-			return NULL;
-	}
-
-	tmpbuf = kmalloc(WLAN_AUTH_CHALLENGE_LEN, GFP_ATOMIC);
-	if (tmpbuf == NULL) {
-		PDEBUG(DEBUG_AP, "AP: kmalloc failed for challenge\n");
-		return NULL;
-	}
-
-	skb = dev_alloc_skb(WLAN_AUTH_CHALLENGE_LEN +
-			    ap->crypt->extra_mpdu_prefix_len +
-			    ap->crypt->extra_mpdu_postfix_len);
-	if (skb == NULL) {
-		kfree(tmpbuf);
-		return NULL;
-	}
-
-	skb_reserve(skb, ap->crypt->extra_mpdu_prefix_len);
-	skb_put_zero(skb, WLAN_AUTH_CHALLENGE_LEN);
-	if (ap->crypt->encrypt_mpdu(skb, 0, ap->crypt_priv)) {
-		dev_kfree_skb(skb);
-		kfree(tmpbuf);
-		return NULL;
-	}
-
-	skb_copy_from_linear_data_offset(skb, ap->crypt->extra_mpdu_prefix_len,
-					 tmpbuf, WLAN_AUTH_CHALLENGE_LEN);
-	dev_kfree_skb(skb);
-
-	return tmpbuf;
-}
-
-
-/* Called only as a scheduled task for pending AP frames. */
-static void handle_authen(local_info_t *local, struct sk_buff *skb,
-			  struct hostap_80211_rx_status *rx_stats)
-{
-	struct net_device *dev = local->dev;
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
-	size_t hdrlen;
-	struct ap_data *ap = local->ap;
-	char body[8 + WLAN_AUTH_CHALLENGE_LEN], *challenge = NULL;
-	int len, olen;
-	u16 auth_alg, auth_transaction, status_code;
-	__le16 *pos;
-	u16 resp = WLAN_STATUS_SUCCESS;
-	struct sta_info *sta = NULL;
-	struct lib80211_crypt_data *crypt;
-	char *txt = "";
-
-	len = skb->len - IEEE80211_MGMT_HDR_LEN;
-
-	hdrlen = hostap_80211_get_hdrlen(hdr->frame_control);
-
-	if (len < 6) {
-		PDEBUG(DEBUG_AP, "%s: handle_authen - too short payload "
-		       "(len=%d) from %pM\n", dev->name, len, hdr->addr2);
-		return;
-	}
-
-	spin_lock_bh(&local->ap->sta_table_lock);
-	sta = ap_get_sta(local->ap, hdr->addr2);
-	if (sta)
-		atomic_inc(&sta->users);
-	spin_unlock_bh(&local->ap->sta_table_lock);
-
-	if (sta && sta->crypt)
-		crypt = sta->crypt;
-	else {
-		int idx = 0;
-		if (skb->len >= hdrlen + 3)
-			idx = skb->data[hdrlen + 3] >> 6;
-		crypt = local->crypt_info.crypt[idx];
-	}
-
-	pos = (__le16 *) (skb->data + IEEE80211_MGMT_HDR_LEN);
-	auth_alg = __le16_to_cpu(*pos);
-	pos++;
-	auth_transaction = __le16_to_cpu(*pos);
-	pos++;
-	status_code = __le16_to_cpu(*pos);
-	pos++;
-
-	if (ether_addr_equal(dev->dev_addr, hdr->addr2) ||
-	    ap_control_mac_deny(&ap->mac_restrictions, hdr->addr2)) {
-		txt = "authentication denied";
-		resp = WLAN_STATUS_UNSPECIFIED_FAILURE;
-		goto fail;
-	}
-
-	if (((local->auth_algs & PRISM2_AUTH_OPEN) &&
-	     auth_alg == WLAN_AUTH_OPEN) ||
-	    ((local->auth_algs & PRISM2_AUTH_SHARED_KEY) &&
-	     crypt && auth_alg == WLAN_AUTH_SHARED_KEY)) {
-	} else {
-		txt = "unsupported algorithm";
-		resp = WLAN_STATUS_NOT_SUPPORTED_AUTH_ALG;
-		goto fail;
-	}
-
-	if (len >= 8) {
-		u8 *u = (u8 *) pos;
-		if (*u == WLAN_EID_CHALLENGE) {
-			if (*(u + 1) != WLAN_AUTH_CHALLENGE_LEN) {
-				txt = "invalid challenge len";
-				resp = WLAN_STATUS_CHALLENGE_FAIL;
-				goto fail;
-			}
-			if (len - 8 < WLAN_AUTH_CHALLENGE_LEN) {
-				txt = "challenge underflow";
-				resp = WLAN_STATUS_CHALLENGE_FAIL;
-				goto fail;
-			}
-			challenge = (char *) (u + 2);
-		}
-	}
-
-	if (sta && sta->ap) {
-		if (time_after(jiffies, sta->u.ap.last_beacon +
-			       (10 * sta->listen_interval * HZ) / 1024)) {
-			PDEBUG(DEBUG_AP, "%s: no beacons received for a while,"
-			       " assuming AP %pM is now STA\n",
-			       dev->name, sta->addr);
-			sta->ap = 0;
-			sta->flags = 0;
-			sta->u.sta.challenge = NULL;
-		} else {
-			txt = "AP trying to authenticate?";
-			resp = WLAN_STATUS_UNSPECIFIED_FAILURE;
-			goto fail;
-		}
-	}
-
-	if ((auth_alg == WLAN_AUTH_OPEN && auth_transaction == 1) ||
-	    (auth_alg == WLAN_AUTH_SHARED_KEY &&
-	     (auth_transaction == 1 ||
-	      (auth_transaction == 3 && sta != NULL &&
-	       sta->u.sta.challenge != NULL)))) {
-	} else {
-		txt = "unknown authentication transaction number";
-		resp = WLAN_STATUS_UNKNOWN_AUTH_TRANSACTION;
-		goto fail;
-	}
-
-	if (sta == NULL) {
-		txt = "new STA";
-
-		if (local->ap->num_sta >= MAX_STA_COUNT) {
-			/* FIX: might try to remove some old STAs first? */
-			txt = "no more room for new STAs";
-			resp = WLAN_STATUS_UNSPECIFIED_FAILURE;
-			goto fail;
-		}
-
-		sta = ap_add_sta(local->ap, hdr->addr2);
-		if (sta == NULL) {
-			txt = "ap_add_sta failed";
-			resp = WLAN_STATUS_UNSPECIFIED_FAILURE;
-			goto fail;
-		}
-	}
-
-	switch (auth_alg) {
-	case WLAN_AUTH_OPEN:
-		txt = "authOK";
-		/* IEEE 802.11 standard is not completely clear about
-		 * whether STA is considered authenticated after
-		 * authentication OK frame has been send or after it
-		 * has been ACKed. In order to reduce interoperability
-		 * issues, mark the STA authenticated before ACK. */
-		sta->flags |= WLAN_STA_AUTH;
-		break;
-
-	case WLAN_AUTH_SHARED_KEY:
-		if (auth_transaction == 1) {
-			if (sta->u.sta.challenge == NULL) {
-				sta->u.sta.challenge =
-					ap_auth_make_challenge(local->ap);
-				if (sta->u.sta.challenge == NULL) {
-					resp = WLAN_STATUS_UNSPECIFIED_FAILURE;
-					goto fail;
-				}
-			}
-		} else {
-			if (sta->u.sta.challenge == NULL ||
-			    challenge == NULL ||
-			    memcmp(sta->u.sta.challenge, challenge,
-				   WLAN_AUTH_CHALLENGE_LEN) != 0 ||
-			    !ieee80211_has_protected(hdr->frame_control)) {
-				txt = "challenge response incorrect";
-				resp = WLAN_STATUS_CHALLENGE_FAIL;
-				goto fail;
-			}
-
-			txt = "challenge OK - authOK";
-			/* IEEE 802.11 standard is not completely clear about
-			 * whether STA is considered authenticated after
-			 * authentication OK frame has been send or after it
-			 * has been ACKed. In order to reduce interoperability
-			 * issues, mark the STA authenticated before ACK. */
-			sta->flags |= WLAN_STA_AUTH;
-			kfree(sta->u.sta.challenge);
-			sta->u.sta.challenge = NULL;
-		}
-		break;
-	}
-
- fail:
-	pos = (__le16 *) body;
-	*pos = cpu_to_le16(auth_alg);
-	pos++;
-	*pos = cpu_to_le16(auth_transaction + 1);
-	pos++;
-	*pos = cpu_to_le16(resp); /* status_code */
-	pos++;
-	olen = 6;
-
-	if (resp == WLAN_STATUS_SUCCESS && sta != NULL &&
-	    sta->u.sta.challenge != NULL &&
-	    auth_alg == WLAN_AUTH_SHARED_KEY && auth_transaction == 1) {
-		u8 *tmp = (u8 *) pos;
-		*tmp++ = WLAN_EID_CHALLENGE;
-		*tmp++ = WLAN_AUTH_CHALLENGE_LEN;
-		pos++;
-		memcpy(pos, sta->u.sta.challenge, WLAN_AUTH_CHALLENGE_LEN);
-		olen += 2 + WLAN_AUTH_CHALLENGE_LEN;
-	}
-
-	prism2_send_mgmt(dev, IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_AUTH,
-			 body, olen, hdr->addr2, ap->tx_callback_auth);
-
-	if (sta) {
-		sta->last_rx = jiffies;
-		atomic_dec(&sta->users);
-	}
-
-	if (resp) {
-		PDEBUG(DEBUG_AP, "%s: %pM auth (alg=%d "
-		       "trans#=%d stat=%d len=%d fc=%04x) ==> %d (%s)\n",
-		       dev->name, hdr->addr2,
-		       auth_alg, auth_transaction, status_code, len,
-		       le16_to_cpu(hdr->frame_control), resp, txt);
-	}
-}
-
-
-/* Called only as a scheduled task for pending AP frames. */
-static void handle_assoc(local_info_t *local, struct sk_buff *skb,
-			 struct hostap_80211_rx_status *rx_stats, int reassoc)
-{
-	struct net_device *dev = local->dev;
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
-	char body[12], *p, *lpos;
-	int len, left;
-	__le16 *pos;
-	u16 resp = WLAN_STATUS_SUCCESS;
-	struct sta_info *sta = NULL;
-	int send_deauth = 0;
-	char __always_unused *txt = "";
-	u8 prev_ap[ETH_ALEN];
-
-	left = len = skb->len - IEEE80211_MGMT_HDR_LEN;
-
-	if (len < (reassoc ? 10 : 4)) {
-		PDEBUG(DEBUG_AP, "%s: handle_assoc - too short payload "
-		       "(len=%d, reassoc=%d) from %pM\n",
-		       dev->name, len, reassoc, hdr->addr2);
-		return;
-	}
-
-	spin_lock_bh(&local->ap->sta_table_lock);
-	sta = ap_get_sta(local->ap, hdr->addr2);
-	if (sta == NULL || (sta->flags & WLAN_STA_AUTH) == 0) {
-		spin_unlock_bh(&local->ap->sta_table_lock);
-		txt = "trying to associate before authentication";
-		send_deauth = 1;
-		resp = WLAN_STATUS_UNSPECIFIED_FAILURE;
-		sta = NULL; /* do not decrement sta->users */
-		goto fail;
-	}
-	atomic_inc(&sta->users);
-	spin_unlock_bh(&local->ap->sta_table_lock);
-
-	pos = (__le16 *) (skb->data + IEEE80211_MGMT_HDR_LEN);
-	sta->capability = __le16_to_cpu(*pos);
-	pos++; left -= 2;
-	sta->listen_interval = __le16_to_cpu(*pos);
-	pos++; left -= 2;
-
-	if (reassoc) {
-		memcpy(prev_ap, pos, ETH_ALEN);
-		pos++; pos++; pos++; left -= 6;
-	} else
-		eth_zero_addr(prev_ap);
-
-	if (left >= 2) {
-		unsigned int ileft;
-		unsigned char *u = (unsigned char *) pos;
-
-		if (*u == WLAN_EID_SSID) {
-			u++; left--;
-			ileft = *u;
-			u++; left--;
-
-			if (ileft > left || ileft > MAX_SSID_LEN) {
-				txt = "SSID overflow";
-				resp = WLAN_STATUS_UNSPECIFIED_FAILURE;
-				goto fail;
-			}
-
-			if (ileft != strlen(local->essid) ||
-			    memcmp(local->essid, u, ileft) != 0) {
-				txt = "not our SSID";
-				resp = WLAN_STATUS_ASSOC_DENIED_UNSPEC;
-				goto fail;
-			}
-
-			u += ileft;
-			left -= ileft;
-		}
-
-		if (left >= 2 && *u == WLAN_EID_SUPP_RATES) {
-			u++; left--;
-			ileft = *u;
-			u++; left--;
-
-			if (ileft > left || ileft == 0 ||
-			    ileft > WLAN_SUPP_RATES_MAX) {
-				txt = "SUPP_RATES len error";
-				resp = WLAN_STATUS_UNSPECIFIED_FAILURE;
-				goto fail;
-			}
-
-			memset(sta->supported_rates, 0,
-			       sizeof(sta->supported_rates));
-			memcpy(sta->supported_rates, u, ileft);
-			prism2_check_tx_rates(sta);
-
-			u += ileft;
-			left -= ileft;
-		}
-
-		if (left > 0) {
-			PDEBUG(DEBUG_AP, "%s: assoc from %pM"
-			       " with extra data (%d bytes) [",
-			       dev->name, hdr->addr2, left);
-			while (left > 0) {
-				PDEBUG2(DEBUG_AP, "<%02x>", *u);
-				u++; left--;
-			}
-			PDEBUG2(DEBUG_AP, "]\n");
-		}
-	} else {
-		txt = "frame underflow";
-		resp = WLAN_STATUS_UNSPECIFIED_FAILURE;
-		goto fail;
-	}
-
-	/* get a unique AID */
-	if (sta->aid > 0)
-		txt = "OK, old AID";
-	else {
-		spin_lock_bh(&local->ap->sta_table_lock);
-		for (sta->aid = 1; sta->aid <= MAX_AID_TABLE_SIZE; sta->aid++)
-			if (local->ap->sta_aid[sta->aid - 1] == NULL)
-				break;
-		if (sta->aid > MAX_AID_TABLE_SIZE) {
-			sta->aid = 0;
-			spin_unlock_bh(&local->ap->sta_table_lock);
-			resp = WLAN_STATUS_AP_UNABLE_TO_HANDLE_NEW_STA;
-			txt = "no room for more AIDs";
-		} else {
-			local->ap->sta_aid[sta->aid - 1] = sta;
-			spin_unlock_bh(&local->ap->sta_table_lock);
-			txt = "OK, new AID";
-		}
-	}
-
- fail:
-	pos = (__le16 *) body;
-
-	if (send_deauth) {
-		*pos = cpu_to_le16(WLAN_REASON_STA_REQ_ASSOC_WITHOUT_AUTH);
-		pos++;
-	} else {
-		/* FIX: CF-Pollable and CF-PollReq should be set to match the
-		 * values in beacons/probe responses */
-		/* FIX: how about privacy and WEP? */
-		/* capability */
-		*pos = cpu_to_le16(WLAN_CAPABILITY_ESS);
-		pos++;
-
-		/* status_code */
-		*pos = cpu_to_le16(resp);
-		pos++;
-
-		*pos = cpu_to_le16((sta && sta->aid > 0 ? sta->aid : 0) |
-				     BIT(14) | BIT(15)); /* AID */
-		pos++;
-
-		/* Supported rates (Information element) */
-		p = (char *) pos;
-		*p++ = WLAN_EID_SUPP_RATES;
-		lpos = p;
-		*p++ = 0; /* len */
-		if (local->tx_rate_control & WLAN_RATE_1M) {
-			*p++ = local->basic_rates & WLAN_RATE_1M ? 0x82 : 0x02;
-			(*lpos)++;
-		}
-		if (local->tx_rate_control & WLAN_RATE_2M) {
-			*p++ = local->basic_rates & WLAN_RATE_2M ? 0x84 : 0x04;
-			(*lpos)++;
-		}
-		if (local->tx_rate_control & WLAN_RATE_5M5) {
-			*p++ = local->basic_rates & WLAN_RATE_5M5 ?
-				0x8b : 0x0b;
-			(*lpos)++;
-		}
-		if (local->tx_rate_control & WLAN_RATE_11M) {
-			*p++ = local->basic_rates & WLAN_RATE_11M ?
-				0x96 : 0x16;
-			(*lpos)++;
-		}
-		pos = (__le16 *) p;
-	}
-
-	prism2_send_mgmt(dev, IEEE80211_FTYPE_MGMT |
-			 (send_deauth ? IEEE80211_STYPE_DEAUTH :
-			  (reassoc ? IEEE80211_STYPE_REASSOC_RESP :
-			   IEEE80211_STYPE_ASSOC_RESP)),
-			 body, (u8 *) pos - (u8 *) body,
-			 hdr->addr2,
-			 send_deauth ? 0 : local->ap->tx_callback_assoc);
-
-	if (sta) {
-		if (resp == WLAN_STATUS_SUCCESS) {
-			sta->last_rx = jiffies;
-			/* STA will be marked associated from TX callback, if
-			 * AssocResp is ACKed */
-		}
-		atomic_dec(&sta->users);
-	}
-
-#if 0
-	PDEBUG(DEBUG_AP, "%s: %pM %sassoc (len=%d "
-	       "prev_ap=%pM) => %d(%d) (%s)\n",
-	       dev->name,
-	       hdr->addr2,
-	       reassoc ? "re" : "", len,
-	       prev_ap,
-	       resp, send_deauth, txt);
-#endif
-}
-
-
-/* Called only as a scheduled task for pending AP frames. */
-static void handle_deauth(local_info_t *local, struct sk_buff *skb,
-			  struct hostap_80211_rx_status *rx_stats)
-{
-	struct net_device *dev = local->dev;
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
-	char *body = (char *) (skb->data + IEEE80211_MGMT_HDR_LEN);
-	int len;
-	u16 reason_code;
-	__le16 *pos;
-	struct sta_info *sta = NULL;
-
-	len = skb->len - IEEE80211_MGMT_HDR_LEN;
-
-	if (len < 2) {
-		printk("handle_deauth - too short payload (len=%d)\n", len);
-		return;
-	}
-
-	pos = (__le16 *) body;
-	reason_code = le16_to_cpu(*pos);
-
-	PDEBUG(DEBUG_AP, "%s: deauthentication: %pM len=%d, "
-	       "reason_code=%d\n", dev->name, hdr->addr2,
-	       len, reason_code);
-
-	spin_lock_bh(&local->ap->sta_table_lock);
-	sta = ap_get_sta(local->ap, hdr->addr2);
-	if (sta != NULL) {
-		if ((sta->flags & WLAN_STA_ASSOC) && !sta->ap)
-			hostap_event_expired_sta(local->dev, sta);
-		sta->flags &= ~(WLAN_STA_AUTH | WLAN_STA_ASSOC);
-	}
-	spin_unlock_bh(&local->ap->sta_table_lock);
-	if (sta == NULL) {
-		printk("%s: deauthentication from %pM, "
-	       "reason_code=%d, but STA not authenticated\n", dev->name,
-		       hdr->addr2, reason_code);
-	}
-}
-
-
-/* Called only as a scheduled task for pending AP frames. */
-static void handle_disassoc(local_info_t *local, struct sk_buff *skb,
-			    struct hostap_80211_rx_status *rx_stats)
-{
-	struct net_device *dev = local->dev;
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
-	char *body = skb->data + IEEE80211_MGMT_HDR_LEN;
-	int len;
-	u16 reason_code;
-	__le16 *pos;
-	struct sta_info *sta = NULL;
-
-	len = skb->len - IEEE80211_MGMT_HDR_LEN;
-
-	if (len < 2) {
-		printk("handle_disassoc - too short payload (len=%d)\n", len);
-		return;
-	}
-
-	pos = (__le16 *) body;
-	reason_code = le16_to_cpu(*pos);
-
-	PDEBUG(DEBUG_AP, "%s: disassociation: %pM len=%d, "
-	       "reason_code=%d\n", dev->name, hdr->addr2,
-	       len, reason_code);
-
-	spin_lock_bh(&local->ap->sta_table_lock);
-	sta = ap_get_sta(local->ap, hdr->addr2);
-	if (sta != NULL) {
-		if ((sta->flags & WLAN_STA_ASSOC) && !sta->ap)
-			hostap_event_expired_sta(local->dev, sta);
-		sta->flags &= ~WLAN_STA_ASSOC;
-	}
-	spin_unlock_bh(&local->ap->sta_table_lock);
-	if (sta == NULL) {
-		printk("%s: disassociation from %pM, "
-		       "reason_code=%d, but STA not authenticated\n",
-		       dev->name, hdr->addr2, reason_code);
-	}
-}
-
-
-/* Called only as a scheduled task for pending AP frames. */
-static void ap_handle_data_nullfunc(local_info_t *local,
-				    struct ieee80211_hdr *hdr)
-{
-	struct net_device *dev = local->dev;
-
-	/* some STA f/w's seem to require control::ACK frame for
-	 * data::nullfunc, but at least Prism2 station f/w version 0.8.0 does
-	 * not send this..
-	 * send control::ACK for the data::nullfunc */
-
-	printk(KERN_DEBUG "Sending control::ACK for data::nullfunc\n");
-	prism2_send_mgmt(dev, IEEE80211_FTYPE_CTL | IEEE80211_STYPE_ACK,
-			 NULL, 0, hdr->addr2, 0);
-}
-
-
-/* Called only as a scheduled task for pending AP frames. */
-static void ap_handle_dropped_data(local_info_t *local,
-				   struct ieee80211_hdr *hdr)
-{
-	struct net_device *dev = local->dev;
-	struct sta_info *sta;
-	__le16 reason;
-
-	spin_lock_bh(&local->ap->sta_table_lock);
-	sta = ap_get_sta(local->ap, hdr->addr2);
-	if (sta)
-		atomic_inc(&sta->users);
-	spin_unlock_bh(&local->ap->sta_table_lock);
-
-	if (sta != NULL && (sta->flags & WLAN_STA_ASSOC)) {
-		PDEBUG(DEBUG_AP, "ap_handle_dropped_data: STA is now okay?\n");
-		atomic_dec(&sta->users);
-		return;
-	}
-
-	reason = cpu_to_le16(WLAN_REASON_CLASS3_FRAME_FROM_NONASSOC_STA);
-	prism2_send_mgmt(dev, IEEE80211_FTYPE_MGMT |
-			 ((sta == NULL || !(sta->flags & WLAN_STA_ASSOC)) ?
-			  IEEE80211_STYPE_DEAUTH : IEEE80211_STYPE_DISASSOC),
-			 (char *) &reason, sizeof(reason), hdr->addr2, 0);
-
-	if (sta)
-		atomic_dec(&sta->users);
-}
-
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-
-
-/* Called only as a scheduled task for pending AP frames. */
-static void pspoll_send_buffered(local_info_t *local, struct sta_info *sta,
-				 struct sk_buff *skb)
-{
-	struct hostap_skb_tx_data *meta;
-
-	if (!(sta->flags & WLAN_STA_PS)) {
-		/* Station has moved to non-PS mode, so send all buffered
-		 * frames using normal device queue. */
-		dev_queue_xmit(skb);
-		return;
-	}
-
-	/* add a flag for hostap_handle_sta_tx() to know that this skb should
-	 * be passed through even though STA is using PS */
-	meta = (struct hostap_skb_tx_data *) skb->cb;
-	meta->flags |= HOSTAP_TX_FLAGS_BUFFERED_FRAME;
-	if (!skb_queue_empty(&sta->tx_buf)) {
-		/* indicate to STA that more frames follow */
-		meta->flags |= HOSTAP_TX_FLAGS_ADD_MOREDATA;
-	}
-	dev_queue_xmit(skb);
-}
-
-
-/* Called only as a scheduled task for pending AP frames. */
-static void handle_pspoll(local_info_t *local,
-			  struct ieee80211_hdr *hdr,
-			  struct hostap_80211_rx_status *rx_stats)
-{
-	struct net_device *dev = local->dev;
-	struct sta_info *sta;
-	u16 aid;
-	struct sk_buff *skb;
-
-	PDEBUG(DEBUG_PS2, "handle_pspoll: BSSID=%pM, TA=%pM PWRMGT=%d\n",
-	       hdr->addr1, hdr->addr2, !!ieee80211_has_pm(hdr->frame_control));
-
-	if (!ether_addr_equal(hdr->addr1, dev->dev_addr)) {
-		PDEBUG(DEBUG_AP,
-		       "handle_pspoll - addr1(BSSID)=%pM not own MAC\n",
-		       hdr->addr1);
-		return;
-	}
-
-	aid = le16_to_cpu(hdr->duration_id);
-	if ((aid & (BIT(15) | BIT(14))) != (BIT(15) | BIT(14))) {
-		PDEBUG(DEBUG_PS, "   PSPOLL and AID[15:14] not set\n");
-		return;
-	}
-	aid &= ~(BIT(15) | BIT(14));
-	if (aid == 0 || aid > MAX_AID_TABLE_SIZE) {
-		PDEBUG(DEBUG_PS, "   invalid aid=%d\n", aid);
-		return;
-	}
-	PDEBUG(DEBUG_PS2, "   aid=%d\n", aid);
-
-	spin_lock_bh(&local->ap->sta_table_lock);
-	sta = ap_get_sta(local->ap, hdr->addr2);
-	if (sta)
-		atomic_inc(&sta->users);
-	spin_unlock_bh(&local->ap->sta_table_lock);
-
-	if (sta == NULL) {
-		PDEBUG(DEBUG_PS, "   STA not found\n");
-		return;
-	}
-	if (sta->aid != aid) {
-		PDEBUG(DEBUG_PS, "   received aid=%i does not match with "
-		       "assoc.aid=%d\n", aid, sta->aid);
-		return;
-	}
-
-	/* FIX: todo:
-	 * - add timeout for buffering (clear aid in TIM vector if buffer timed
-	 *   out (expiry time must be longer than ListenInterval for
-	 *   the corresponding STA; "8802-11: 11.2.1.9 AP aging function"
-	 * - what to do, if buffered, pspolled, and sent frame is not ACKed by
-	 *   sta; store buffer for later use and leave TIM aid bit set? use
-	 *   TX event to check whether frame was ACKed?
-	 */
-
-	while ((skb = skb_dequeue(&sta->tx_buf)) != NULL) {
-		/* send buffered frame .. */
-		PDEBUG(DEBUG_PS2, "Sending buffered frame to STA after PS POLL"
-		       " (buffer_count=%d)\n", skb_queue_len(&sta->tx_buf));
-
-		pspoll_send_buffered(local, sta, skb);
-
-		if (sta->flags & WLAN_STA_PS) {
-			/* send only one buffered packet per PS Poll */
-			/* FIX: should ignore further PS Polls until the
-			 * buffered packet that was just sent is acknowledged
-			 * (Tx or TxExc event) */
-			break;
-		}
-	}
-
-	if (skb_queue_empty(&sta->tx_buf)) {
-		/* try to clear aid from TIM */
-		if (!(sta->flags & WLAN_STA_TIM))
-			PDEBUG(DEBUG_PS2,  "Re-unsetting TIM for aid %d\n",
-			       aid);
-		hostap_set_tim(local, aid, 0);
-		sta->flags &= ~WLAN_STA_TIM;
-	}
-
-	atomic_dec(&sta->users);
-}
-
-
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-
-static void handle_wds_oper_queue(struct work_struct *work)
-{
-	struct ap_data *ap = container_of(work, struct ap_data,
-					  wds_oper_queue);
-	local_info_t *local = ap->local;
-	struct wds_oper_data *entry, *prev;
-
-	spin_lock_bh(&local->lock);
-	entry = local->ap->wds_oper_entries;
-	local->ap->wds_oper_entries = NULL;
-	spin_unlock_bh(&local->lock);
-
-	while (entry) {
-		PDEBUG(DEBUG_AP, "%s: %s automatic WDS connection "
-		       "to AP %pM\n",
-		       local->dev->name,
-		       entry->type == WDS_ADD ? "adding" : "removing",
-		       entry->addr);
-		if (entry->type == WDS_ADD)
-			prism2_wds_add(local, entry->addr, 0);
-		else if (entry->type == WDS_DEL)
-			prism2_wds_del(local, entry->addr, 0, 1);
-
-		prev = entry;
-		entry = entry->next;
-		kfree(prev);
-	}
-}
-
-
-/* Called only as a scheduled task for pending AP frames. */
-static void handle_beacon(local_info_t *local, struct sk_buff *skb,
-			  struct hostap_80211_rx_status *rx_stats)
-{
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
-	char *body = skb->data + IEEE80211_MGMT_HDR_LEN;
-	int len, left;
-	u16 beacon_int, capability;
-	__le16 *pos;
-	char *ssid = NULL;
-	unsigned char *supp_rates = NULL;
-	int ssid_len = 0, supp_rates_len = 0;
-	struct sta_info *sta = NULL;
-	int new_sta = 0, channel = -1;
-
-	len = skb->len - IEEE80211_MGMT_HDR_LEN;
-
-	if (len < 8 + 2 + 2) {
-		printk(KERN_DEBUG "handle_beacon - too short payload "
-		       "(len=%d)\n", len);
-		return;
-	}
-
-	pos = (__le16 *) body;
-	left = len;
-
-	/* Timestamp (8 octets) */
-	pos += 4; left -= 8;
-	/* Beacon interval (2 octets) */
-	beacon_int = le16_to_cpu(*pos);
-	pos++; left -= 2;
-	/* Capability information (2 octets) */
-	capability = le16_to_cpu(*pos);
-	pos++; left -= 2;
-
-	if (local->ap->ap_policy != AP_OTHER_AP_EVEN_IBSS &&
-	    capability & WLAN_CAPABILITY_IBSS)
-		return;
-
-	if (left >= 2) {
-		unsigned int ileft;
-		unsigned char *u = (unsigned char *) pos;
-
-		if (*u == WLAN_EID_SSID) {
-			u++; left--;
-			ileft = *u;
-			u++; left--;
-
-			if (ileft > left || ileft > MAX_SSID_LEN) {
-				PDEBUG(DEBUG_AP, "SSID: overflow\n");
-				return;
-			}
-
-			if (local->ap->ap_policy == AP_OTHER_AP_SAME_SSID &&
-			    (ileft != strlen(local->essid) ||
-			     memcmp(local->essid, u, ileft) != 0)) {
-				/* not our SSID */
-				return;
-			}
-
-			ssid = u;
-			ssid_len = ileft;
-
-			u += ileft;
-			left -= ileft;
-		}
-
-		if (*u == WLAN_EID_SUPP_RATES) {
-			u++; left--;
-			ileft = *u;
-			u++; left--;
-
-			if (ileft > left || ileft == 0 || ileft > 8) {
-				PDEBUG(DEBUG_AP, " - SUPP_RATES len error\n");
-				return;
-			}
-
-			supp_rates = u;
-			supp_rates_len = ileft;
-
-			u += ileft;
-			left -= ileft;
-		}
-
-		if (*u == WLAN_EID_DS_PARAMS) {
-			u++; left--;
-			ileft = *u;
-			u++; left--;
-
-			if (ileft > left || ileft != 1) {
-				PDEBUG(DEBUG_AP, " - DS_PARAMS len error\n");
-				return;
-			}
-
-			channel = *u;
-
-			u += ileft;
-			left -= ileft;
-		}
-	}
-
-	spin_lock_bh(&local->ap->sta_table_lock);
-	sta = ap_get_sta(local->ap, hdr->addr2);
-	if (sta != NULL)
-		atomic_inc(&sta->users);
-	spin_unlock_bh(&local->ap->sta_table_lock);
-
-	if (sta == NULL) {
-		/* add new AP */
-		new_sta = 1;
-		sta = ap_add_sta(local->ap, hdr->addr2);
-		if (sta == NULL) {
-			printk(KERN_INFO "prism2: kmalloc failed for AP "
-			       "data structure\n");
-			return;
-		}
-		hostap_event_new_sta(local->dev, sta);
-
-		/* mark APs authentication and associated for pseudo ad-hoc
-		 * style communication */
-		sta->flags = WLAN_STA_AUTH | WLAN_STA_ASSOC;
-
-		if (local->ap->autom_ap_wds) {
-			hostap_wds_link_oper(local, sta->addr, WDS_ADD);
-		}
-	}
-
-	sta->ap = 1;
-	if (ssid) {
-		sta->u.ap.ssid_len = ssid_len;
-		memcpy(sta->u.ap.ssid, ssid, ssid_len);
-		sta->u.ap.ssid[ssid_len] = '\0';
-	} else {
-		sta->u.ap.ssid_len = 0;
-		sta->u.ap.ssid[0] = '\0';
-	}
-	sta->u.ap.channel = channel;
-	sta->rx_packets++;
-	sta->rx_bytes += len;
-	sta->u.ap.last_beacon = sta->last_rx = jiffies;
-	sta->capability = capability;
-	sta->listen_interval = beacon_int;
-
-	atomic_dec(&sta->users);
-
-	if (new_sta) {
-		memset(sta->supported_rates, 0, sizeof(sta->supported_rates));
-		memcpy(sta->supported_rates, supp_rates, supp_rates_len);
-		prism2_check_tx_rates(sta);
-	}
-}
-
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-
-
-/* Called only as a tasklet. */
-static void handle_ap_item(local_info_t *local, struct sk_buff *skb,
-			   struct hostap_80211_rx_status *rx_stats)
-{
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-	struct net_device *dev = local->dev;
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-	u16 fc, type, stype;
-	struct ieee80211_hdr *hdr;
-
-	/* FIX: should give skb->len to handler functions and check that the
-	 * buffer is long enough */
-	hdr = (struct ieee80211_hdr *) skb->data;
-	fc = le16_to_cpu(hdr->frame_control);
-	type = fc & IEEE80211_FCTL_FTYPE;
-	stype = fc & IEEE80211_FCTL_STYPE;
-
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-	if (!local->hostapd && type == IEEE80211_FTYPE_DATA) {
-		PDEBUG(DEBUG_AP, "handle_ap_item - data frame\n");
-
-		if (!(fc & IEEE80211_FCTL_TODS) ||
-		    (fc & IEEE80211_FCTL_FROMDS)) {
-			if (stype == IEEE80211_STYPE_NULLFUNC) {
-				/* no ToDS nullfunc seems to be used to check
-				 * AP association; so send reject message to
-				 * speed up re-association */
-				ap_handle_dropped_data(local, hdr);
-				goto done;
-			}
-			PDEBUG(DEBUG_AP, "   not ToDS frame (fc=0x%04x)\n",
-			       fc);
-			goto done;
-		}
-
-		if (!ether_addr_equal(hdr->addr1, dev->dev_addr)) {
-			PDEBUG(DEBUG_AP, "handle_ap_item - addr1(BSSID)=%pM"
-			       " not own MAC\n", hdr->addr1);
-			goto done;
-		}
-
-		if (local->ap->nullfunc_ack &&
-		    stype == IEEE80211_STYPE_NULLFUNC)
-			ap_handle_data_nullfunc(local, hdr);
-		else
-			ap_handle_dropped_data(local, hdr);
-		goto done;
-	}
-
-	if (type == IEEE80211_FTYPE_MGMT && stype == IEEE80211_STYPE_BEACON) {
-		handle_beacon(local, skb, rx_stats);
-		goto done;
-	}
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-
-	if (type == IEEE80211_FTYPE_CTL && stype == IEEE80211_STYPE_PSPOLL) {
-		handle_pspoll(local, hdr, rx_stats);
-		goto done;
-	}
-
-	if (local->hostapd) {
-		PDEBUG(DEBUG_AP, "Unknown frame in AP queue: type=0x%02x "
-		       "subtype=0x%02x\n", type, stype);
-		goto done;
-	}
-
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-	if (type != IEEE80211_FTYPE_MGMT) {
-		PDEBUG(DEBUG_AP, "handle_ap_item - not a management frame?\n");
-		goto done;
-	}
-
-	if (!ether_addr_equal(hdr->addr1, dev->dev_addr)) {
-		PDEBUG(DEBUG_AP, "handle_ap_item - addr1(DA)=%pM"
-		       " not own MAC\n", hdr->addr1);
-		goto done;
-	}
-
-	if (!ether_addr_equal(hdr->addr3, dev->dev_addr)) {
-		PDEBUG(DEBUG_AP, "handle_ap_item - addr3(BSSID)=%pM"
-		       " not own MAC\n", hdr->addr3);
-		goto done;
-	}
-
-	switch (stype) {
-	case IEEE80211_STYPE_ASSOC_REQ:
-		handle_assoc(local, skb, rx_stats, 0);
-		break;
-	case IEEE80211_STYPE_ASSOC_RESP:
-		PDEBUG(DEBUG_AP, "==> ASSOC RESP (ignored)\n");
-		break;
-	case IEEE80211_STYPE_REASSOC_REQ:
-		handle_assoc(local, skb, rx_stats, 1);
-		break;
-	case IEEE80211_STYPE_REASSOC_RESP:
-		PDEBUG(DEBUG_AP, "==> REASSOC RESP (ignored)\n");
-		break;
-	case IEEE80211_STYPE_ATIM:
-		PDEBUG(DEBUG_AP, "==> ATIM (ignored)\n");
-		break;
-	case IEEE80211_STYPE_DISASSOC:
-		handle_disassoc(local, skb, rx_stats);
-		break;
-	case IEEE80211_STYPE_AUTH:
-		handle_authen(local, skb, rx_stats);
-		break;
-	case IEEE80211_STYPE_DEAUTH:
-		handle_deauth(local, skb, rx_stats);
-		break;
-	default:
-		PDEBUG(DEBUG_AP, "Unknown mgmt frame subtype 0x%02x\n",
-		       stype >> 4);
-		break;
-	}
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-
- done:
-	dev_kfree_skb(skb);
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-void hostap_rx(struct net_device *dev, struct sk_buff *skb,
-	       struct hostap_80211_rx_status *rx_stats)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	struct ieee80211_hdr *hdr;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (skb->len < 16)
-		goto drop;
-
-	dev->stats.rx_packets++;
-
-	hdr = (struct ieee80211_hdr *) skb->data;
-
-	if (local->ap->ap_policy == AP_OTHER_AP_SKIP_ALL &&
-	    ieee80211_is_beacon(hdr->frame_control))
-		goto drop;
-
-	skb->protocol = cpu_to_be16(ETH_P_HOSTAP);
-	handle_ap_item(local, skb, rx_stats);
-	return;
-
- drop:
-	dev_kfree_skb(skb);
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static void schedule_packet_send(local_info_t *local, struct sta_info *sta)
-{
-	struct sk_buff *skb;
-	struct ieee80211_hdr *hdr;
-	struct hostap_80211_rx_status rx_stats;
-
-	if (skb_queue_empty(&sta->tx_buf))
-		return;
-
-	skb = dev_alloc_skb(16);
-	if (skb == NULL) {
-		printk(KERN_DEBUG "%s: schedule_packet_send: skb alloc "
-		       "failed\n", local->dev->name);
-		return;
-	}
-
-	hdr = skb_put(skb, 16);
-
-	/* Generate a fake pspoll frame to start packet delivery */
-	hdr->frame_control = cpu_to_le16(
-		IEEE80211_FTYPE_CTL | IEEE80211_STYPE_PSPOLL);
-	memcpy(hdr->addr1, local->dev->dev_addr, ETH_ALEN);
-	memcpy(hdr->addr2, sta->addr, ETH_ALEN);
-	hdr->duration_id = cpu_to_le16(sta->aid | BIT(15) | BIT(14));
-
-	PDEBUG(DEBUG_PS2,
-	       "%s: Scheduling buffered packet delivery for STA %pM\n",
-	       local->dev->name, sta->addr);
-
-	skb->dev = local->dev;
-
-	memset(&rx_stats, 0, sizeof(rx_stats));
-	hostap_rx(local->dev, skb, &rx_stats);
-}
-
-
-int prism2_ap_get_sta_qual(local_info_t *local, struct sockaddr addr[],
-			   struct iw_quality qual[], int buf_size,
-			   int aplist)
-{
-	struct ap_data *ap = local->ap;
-	struct list_head *ptr;
-	int count = 0;
-
-	spin_lock_bh(&ap->sta_table_lock);
-
-	for (ptr = ap->sta_list.next; ptr != NULL && ptr != &ap->sta_list;
-	     ptr = ptr->next) {
-		struct sta_info *sta = (struct sta_info *) ptr;
-
-		if (aplist && !sta->ap)
-			continue;
-		addr[count].sa_family = ARPHRD_ETHER;
-		memcpy(addr[count].sa_data, sta->addr, ETH_ALEN);
-		if (sta->last_rx_silence == 0)
-			qual[count].qual = sta->last_rx_signal < 27 ?
-				0 : (sta->last_rx_signal - 27) * 92 / 127;
-		else
-			qual[count].qual = sta->last_rx_signal -
-				sta->last_rx_silence - 35;
-		qual[count].level = HFA384X_LEVEL_TO_dBm(sta->last_rx_signal);
-		qual[count].noise = HFA384X_LEVEL_TO_dBm(sta->last_rx_silence);
-		qual[count].updated = sta->last_rx_updated;
-
-		sta->last_rx_updated = IW_QUAL_DBM;
-
-		count++;
-		if (count >= buf_size)
-			break;
-	}
-	spin_unlock_bh(&ap->sta_table_lock);
-
-	return count;
-}
-
-
-/* Translate our list of Access Points & Stations to a card independent
- * format that the Wireless Tools will understand - Jean II */
-int prism2_ap_translate_scan(struct net_device *dev,
-			     struct iw_request_info *info, char *buffer)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	struct ap_data *ap;
-	struct list_head *ptr;
-	struct iw_event iwe;
-	char *current_ev = buffer;
-	char *end_buf = buffer + IW_SCAN_MAX_DATA;
-#if !defined(PRISM2_NO_KERNEL_IEEE80211_MGMT)
-	char buf[64];
-#endif
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-	ap = local->ap;
-
-	spin_lock_bh(&ap->sta_table_lock);
-
-	for (ptr = ap->sta_list.next; ptr != NULL && ptr != &ap->sta_list;
-	     ptr = ptr->next) {
-		struct sta_info *sta = (struct sta_info *) ptr;
-
-		/* First entry *MUST* be the AP MAC address */
-		memset(&iwe, 0, sizeof(iwe));
-		iwe.cmd = SIOCGIWAP;
-		iwe.u.ap_addr.sa_family = ARPHRD_ETHER;
-		memcpy(iwe.u.ap_addr.sa_data, sta->addr, ETH_ALEN);
-		iwe.len = IW_EV_ADDR_LEN;
-		current_ev = iwe_stream_add_event(info, current_ev, end_buf,
-						  &iwe, IW_EV_ADDR_LEN);
-
-		/* Use the mode to indicate if it's a station or
-		 * an Access Point */
-		memset(&iwe, 0, sizeof(iwe));
-		iwe.cmd = SIOCGIWMODE;
-		if (sta->ap)
-			iwe.u.mode = IW_MODE_MASTER;
-		else
-			iwe.u.mode = IW_MODE_INFRA;
-		iwe.len = IW_EV_UINT_LEN;
-		current_ev = iwe_stream_add_event(info, current_ev, end_buf,
-						  &iwe, IW_EV_UINT_LEN);
-
-		/* Some quality */
-		memset(&iwe, 0, sizeof(iwe));
-		iwe.cmd = IWEVQUAL;
-		if (sta->last_rx_silence == 0)
-			iwe.u.qual.qual = sta->last_rx_signal < 27 ?
-				0 : (sta->last_rx_signal - 27) * 92 / 127;
-		else
-			iwe.u.qual.qual = sta->last_rx_signal -
-				sta->last_rx_silence - 35;
-		iwe.u.qual.level = HFA384X_LEVEL_TO_dBm(sta->last_rx_signal);
-		iwe.u.qual.noise = HFA384X_LEVEL_TO_dBm(sta->last_rx_silence);
-		iwe.u.qual.updated = sta->last_rx_updated;
-		iwe.len = IW_EV_QUAL_LEN;
-		current_ev = iwe_stream_add_event(info, current_ev, end_buf,
-						  &iwe, IW_EV_QUAL_LEN);
-
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-		if (sta->ap) {
-			memset(&iwe, 0, sizeof(iwe));
-			iwe.cmd = SIOCGIWESSID;
-			iwe.u.data.length = sta->u.ap.ssid_len;
-			iwe.u.data.flags = 1;
-			current_ev = iwe_stream_add_point(info, current_ev,
-							  end_buf, &iwe,
-							  sta->u.ap.ssid);
-
-			memset(&iwe, 0, sizeof(iwe));
-			iwe.cmd = SIOCGIWENCODE;
-			if (sta->capability & WLAN_CAPABILITY_PRIVACY)
-				iwe.u.data.flags =
-					IW_ENCODE_ENABLED | IW_ENCODE_NOKEY;
-			else
-				iwe.u.data.flags = IW_ENCODE_DISABLED;
-			current_ev = iwe_stream_add_point(info, current_ev,
-							  end_buf, &iwe,
-							  sta->u.ap.ssid);
-
-			if (sta->u.ap.channel > 0 &&
-			    sta->u.ap.channel <= FREQ_COUNT) {
-				memset(&iwe, 0, sizeof(iwe));
-				iwe.cmd = SIOCGIWFREQ;
-				iwe.u.freq.m = freq_list[sta->u.ap.channel - 1]
-					* 100000;
-				iwe.u.freq.e = 1;
-				current_ev = iwe_stream_add_event(
-					info, current_ev, end_buf, &iwe,
-					IW_EV_FREQ_LEN);
-			}
-
-			memset(&iwe, 0, sizeof(iwe));
-			iwe.cmd = IWEVCUSTOM;
-			sprintf(buf, "beacon_interval=%d",
-				sta->listen_interval);
-			iwe.u.data.length = strlen(buf);
-			current_ev = iwe_stream_add_point(info, current_ev,
-							  end_buf, &iwe, buf);
-		}
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-
-		sta->last_rx_updated = IW_QUAL_DBM;
-
-		/* To be continued, we should make good use of IWEVCUSTOM */
-	}
-
-	spin_unlock_bh(&ap->sta_table_lock);
-
-	return current_ev - buffer;
-}
-
-
-static int prism2_hostapd_add_sta(struct ap_data *ap,
-				  struct prism2_hostapd_param *param)
-{
-	struct sta_info *sta;
-
-	spin_lock_bh(&ap->sta_table_lock);
-	sta = ap_get_sta(ap, param->sta_addr);
-	if (sta)
-		atomic_inc(&sta->users);
-	spin_unlock_bh(&ap->sta_table_lock);
-
-	if (sta == NULL) {
-		sta = ap_add_sta(ap, param->sta_addr);
-		if (sta == NULL)
-			return -1;
-	}
-
-	if (!(sta->flags & WLAN_STA_ASSOC) && !sta->ap && sta->local)
-		hostap_event_new_sta(sta->local->dev, sta);
-
-	sta->flags |= WLAN_STA_AUTH | WLAN_STA_ASSOC;
-	sta->last_rx = jiffies;
-	sta->aid = param->u.add_sta.aid;
-	sta->capability = param->u.add_sta.capability;
-	sta->tx_supp_rates = param->u.add_sta.tx_supp_rates;
-	if (sta->tx_supp_rates & WLAN_RATE_1M)
-		sta->supported_rates[0] = 2;
-	if (sta->tx_supp_rates & WLAN_RATE_2M)
-		sta->supported_rates[1] = 4;
-	if (sta->tx_supp_rates & WLAN_RATE_5M5)
-		sta->supported_rates[2] = 11;
-	if (sta->tx_supp_rates & WLAN_RATE_11M)
-		sta->supported_rates[3] = 22;
-	prism2_check_tx_rates(sta);
-	atomic_dec(&sta->users);
-	return 0;
-}
-
-
-static int prism2_hostapd_remove_sta(struct ap_data *ap,
-				     struct prism2_hostapd_param *param)
-{
-	struct sta_info *sta;
-
-	spin_lock_bh(&ap->sta_table_lock);
-	sta = ap_get_sta(ap, param->sta_addr);
-	if (sta) {
-		ap_sta_hash_del(ap, sta);
-		list_del(&sta->list);
-	}
-	spin_unlock_bh(&ap->sta_table_lock);
-
-	if (!sta)
-		return -ENOENT;
-
-	if ((sta->flags & WLAN_STA_ASSOC) && !sta->ap && sta->local)
-		hostap_event_expired_sta(sta->local->dev, sta);
-	ap_free_sta(ap, sta);
-
-	return 0;
-}
-
-
-static int prism2_hostapd_get_info_sta(struct ap_data *ap,
-				       struct prism2_hostapd_param *param)
-{
-	struct sta_info *sta;
-
-	spin_lock_bh(&ap->sta_table_lock);
-	sta = ap_get_sta(ap, param->sta_addr);
-	if (sta)
-		atomic_inc(&sta->users);
-	spin_unlock_bh(&ap->sta_table_lock);
-
-	if (!sta)
-		return -ENOENT;
-
-	param->u.get_info_sta.inactive_sec = (jiffies - sta->last_rx) / HZ;
-
-	atomic_dec(&sta->users);
-
-	return 1;
-}
-
-
-static int prism2_hostapd_set_flags_sta(struct ap_data *ap,
-					struct prism2_hostapd_param *param)
-{
-	struct sta_info *sta;
-
-	spin_lock_bh(&ap->sta_table_lock);
-	sta = ap_get_sta(ap, param->sta_addr);
-	if (sta) {
-		sta->flags |= param->u.set_flags_sta.flags_or;
-		sta->flags &= param->u.set_flags_sta.flags_and;
-	}
-	spin_unlock_bh(&ap->sta_table_lock);
-
-	if (!sta)
-		return -ENOENT;
-
-	return 0;
-}
-
-
-static int prism2_hostapd_sta_clear_stats(struct ap_data *ap,
-					  struct prism2_hostapd_param *param)
-{
-	struct sta_info *sta;
-	int rate;
-
-	spin_lock_bh(&ap->sta_table_lock);
-	sta = ap_get_sta(ap, param->sta_addr);
-	if (sta) {
-		sta->rx_packets = sta->tx_packets = 0;
-		sta->rx_bytes = sta->tx_bytes = 0;
-		for (rate = 0; rate < WLAN_RATE_COUNT; rate++) {
-			sta->tx_count[rate] = 0;
-			sta->rx_count[rate] = 0;
-		}
-	}
-	spin_unlock_bh(&ap->sta_table_lock);
-
-	if (!sta)
-		return -ENOENT;
-
-	return 0;
-}
-
-
-int prism2_hostapd(struct ap_data *ap, struct prism2_hostapd_param *param)
-{
-	switch (param->cmd) {
-	case PRISM2_HOSTAPD_FLUSH:
-		ap_control_kickall(ap);
-		return 0;
-	case PRISM2_HOSTAPD_ADD_STA:
-		return prism2_hostapd_add_sta(ap, param);
-	case PRISM2_HOSTAPD_REMOVE_STA:
-		return prism2_hostapd_remove_sta(ap, param);
-	case PRISM2_HOSTAPD_GET_INFO_STA:
-		return prism2_hostapd_get_info_sta(ap, param);
-	case PRISM2_HOSTAPD_SET_FLAGS_STA:
-		return prism2_hostapd_set_flags_sta(ap, param);
-	case PRISM2_HOSTAPD_STA_CLEAR_STATS:
-		return prism2_hostapd_sta_clear_stats(ap, param);
-	default:
-		printk(KERN_WARNING "prism2_hostapd: unknown cmd=%d\n",
-		       param->cmd);
-		return -EOPNOTSUPP;
-	}
-}
-
-
-/* Update station info for host-based TX rate control and return current
- * TX rate */
-static int ap_update_sta_tx_rate(struct sta_info *sta, struct net_device *dev)
-{
-	int ret = sta->tx_rate;
-	struct hostap_interface *iface;
-	local_info_t *local;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	sta->tx_count[sta->tx_rate_idx]++;
-	sta->tx_since_last_failure++;
-	sta->tx_consecutive_exc = 0;
-	if (sta->tx_since_last_failure >= WLAN_RATE_UPDATE_COUNT &&
-	    sta->tx_rate_idx < sta->tx_max_rate) {
-		/* use next higher rate */
-		int old_rate, new_rate;
-		old_rate = new_rate = sta->tx_rate_idx;
-		while (new_rate < sta->tx_max_rate) {
-			new_rate++;
-			if (ap_tx_rate_ok(new_rate, sta, local)) {
-				sta->tx_rate_idx = new_rate;
-				break;
-			}
-		}
-		if (old_rate != sta->tx_rate_idx) {
-			switch (sta->tx_rate_idx) {
-			case 0: sta->tx_rate = 10; break;
-			case 1: sta->tx_rate = 20; break;
-			case 2: sta->tx_rate = 55; break;
-			case 3: sta->tx_rate = 110; break;
-			default: sta->tx_rate = 0; break;
-			}
-			PDEBUG(DEBUG_AP, "%s: STA %pM TX rate raised to %d\n",
-			       dev->name, sta->addr, sta->tx_rate);
-		}
-		sta->tx_since_last_failure = 0;
-	}
-
-	return ret;
-}
-
-
-/* Called only from software IRQ. Called for each TX frame prior possible
- * encryption and transmit. */
-ap_tx_ret hostap_handle_sta_tx(local_info_t *local, struct hostap_tx_data *tx)
-{
-	struct sta_info *sta = NULL;
-	struct sk_buff *skb = tx->skb;
-	int set_tim, ret;
-	struct ieee80211_hdr *hdr;
-	struct hostap_skb_tx_data *meta;
-
-	meta = (struct hostap_skb_tx_data *) skb->cb;
-	ret = AP_TX_CONTINUE;
-	if (local->ap == NULL || skb->len < 10 ||
-	    meta->iface->type == HOSTAP_INTERFACE_STA)
-		goto out;
-
-	hdr = (struct ieee80211_hdr *) skb->data;
-
-	if (hdr->addr1[0] & 0x01) {
-		/* broadcast/multicast frame - no AP related processing */
-		if (local->ap->num_sta <= 0)
-			ret = AP_TX_DROP;
-		goto out;
-	}
-
-	/* unicast packet - check whether destination STA is associated */
-	spin_lock(&local->ap->sta_table_lock);
-	sta = ap_get_sta(local->ap, hdr->addr1);
-	if (sta)
-		atomic_inc(&sta->users);
-	spin_unlock(&local->ap->sta_table_lock);
-
-	if (local->iw_mode == IW_MODE_MASTER && sta == NULL &&
-	    !(meta->flags & HOSTAP_TX_FLAGS_WDS) &&
-	    meta->iface->type != HOSTAP_INTERFACE_MASTER &&
-	    meta->iface->type != HOSTAP_INTERFACE_AP) {
-#if 0
-		/* This can happen, e.g., when wlan0 is added to a bridge and
-		 * bridging code does not know which port is the correct target
-		 * for a unicast frame. In this case, the packet is send to all
-		 * ports of the bridge. Since this is a valid scenario, do not
-		 * print out any errors here. */
-		if (net_ratelimit()) {
-			printk(KERN_DEBUG "AP: drop packet to non-associated "
-			       "STA %pM\n", hdr->addr1);
-		}
-#endif
-		local->ap->tx_drop_nonassoc++;
-		ret = AP_TX_DROP;
-		goto out;
-	}
-
-	if (sta == NULL)
-		goto out;
-
-	if (!(sta->flags & WLAN_STA_AUTHORIZED))
-		ret = AP_TX_CONTINUE_NOT_AUTHORIZED;
-
-	/* Set tx_rate if using host-based TX rate control */
-	if (!local->fw_tx_rate_control)
-		local->ap->last_tx_rate = meta->rate =
-			ap_update_sta_tx_rate(sta, local->dev);
-
-	if (local->iw_mode != IW_MODE_MASTER)
-		goto out;
-
-	if (!(sta->flags & WLAN_STA_PS))
-		goto out;
-
-	if (meta->flags & HOSTAP_TX_FLAGS_ADD_MOREDATA) {
-		/* indicate to STA that more frames follow */
-		hdr->frame_control |=
-			cpu_to_le16(IEEE80211_FCTL_MOREDATA);
-	}
-
-	if (meta->flags & HOSTAP_TX_FLAGS_BUFFERED_FRAME) {
-		/* packet was already buffered and now send due to
-		 * PS poll, so do not rebuffer it */
-		goto out;
-	}
-
-	if (skb_queue_len(&sta->tx_buf) >= STA_MAX_TX_BUFFER) {
-		PDEBUG(DEBUG_PS, "%s: No more space in STA (%pM)'s"
-		       "PS mode buffer\n",
-		       local->dev->name, sta->addr);
-		/* Make sure that TIM is set for the station (it might not be
-		 * after AP wlan hw reset). */
-		/* FIX: should fix hw reset to restore bits based on STA
-		 * buffer state.. */
-		hostap_set_tim(local, sta->aid, 1);
-		sta->flags |= WLAN_STA_TIM;
-		ret = AP_TX_DROP;
-		goto out;
-	}
-
-	/* STA in PS mode, buffer frame for later delivery */
-	set_tim = skb_queue_empty(&sta->tx_buf);
-	skb_queue_tail(&sta->tx_buf, skb);
-	/* FIX: could save RX time to skb and expire buffered frames after
-	 * some time if STA does not poll for them */
-
-	if (set_tim) {
-		if (sta->flags & WLAN_STA_TIM)
-			PDEBUG(DEBUG_PS2, "Re-setting TIM for aid %d\n",
-			       sta->aid);
-		hostap_set_tim(local, sta->aid, 1);
-		sta->flags |= WLAN_STA_TIM;
-	}
-
-	ret = AP_TX_BUFFERED;
-
- out:
-	if (sta != NULL) {
-		if (ret == AP_TX_CONTINUE ||
-		    ret == AP_TX_CONTINUE_NOT_AUTHORIZED) {
-			sta->tx_packets++;
-			sta->tx_bytes += skb->len;
-			sta->last_tx = jiffies;
-		}
-
-		if ((ret == AP_TX_CONTINUE ||
-		     ret == AP_TX_CONTINUE_NOT_AUTHORIZED) &&
-		    sta->crypt && tx->host_encrypt) {
-			tx->crypt = sta->crypt;
-			tx->sta_ptr = sta; /* hostap_handle_sta_release() will
-					    * be called to release sta info
-					    * later */
-		} else
-			atomic_dec(&sta->users);
-	}
-
-	return ret;
-}
-
-
-void hostap_handle_sta_release(void *ptr)
-{
-	struct sta_info *sta = ptr;
-	atomic_dec(&sta->users);
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-void hostap_handle_sta_tx_exc(local_info_t *local, struct sk_buff *skb)
-{
-	struct sta_info *sta;
-	struct ieee80211_hdr *hdr;
-	struct hostap_skb_tx_data *meta;
-
-	hdr = (struct ieee80211_hdr *) skb->data;
-	meta = (struct hostap_skb_tx_data *) skb->cb;
-
-	spin_lock(&local->ap->sta_table_lock);
-	sta = ap_get_sta(local->ap, hdr->addr1);
-	if (!sta) {
-		spin_unlock(&local->ap->sta_table_lock);
-		PDEBUG(DEBUG_AP, "%s: Could not find STA %pM"
-		       " for this TX error (@%lu)\n",
-		       local->dev->name, hdr->addr1, jiffies);
-		return;
-	}
-
-	sta->tx_since_last_failure = 0;
-	sta->tx_consecutive_exc++;
-
-	if (sta->tx_consecutive_exc >= WLAN_RATE_DECREASE_THRESHOLD &&
-	    sta->tx_rate_idx > 0 && meta->rate <= sta->tx_rate) {
-		/* use next lower rate */
-		int old, rate;
-		old = rate = sta->tx_rate_idx;
-		while (rate > 0) {
-			rate--;
-			if (ap_tx_rate_ok(rate, sta, local)) {
-				sta->tx_rate_idx = rate;
-				break;
-			}
-		}
-		if (old != sta->tx_rate_idx) {
-			switch (sta->tx_rate_idx) {
-			case 0: sta->tx_rate = 10; break;
-			case 1: sta->tx_rate = 20; break;
-			case 2: sta->tx_rate = 55; break;
-			case 3: sta->tx_rate = 110; break;
-			default: sta->tx_rate = 0; break;
-			}
-			PDEBUG(DEBUG_AP,
-			       "%s: STA %pM TX rate lowered to %d\n",
-			       local->dev->name, sta->addr, sta->tx_rate);
-		}
-		sta->tx_consecutive_exc = 0;
-	}
-	spin_unlock(&local->ap->sta_table_lock);
-}
-
-
-static void hostap_update_sta_ps2(local_info_t *local, struct sta_info *sta,
-				  int pwrmgt, int type, int stype)
-{
-	if (pwrmgt && !(sta->flags & WLAN_STA_PS)) {
-		sta->flags |= WLAN_STA_PS;
-		PDEBUG(DEBUG_PS2, "STA %pM changed to use PS "
-		       "mode (type=0x%02X, stype=0x%02X)\n",
-		       sta->addr, type >> 2, stype >> 4);
-	} else if (!pwrmgt && (sta->flags & WLAN_STA_PS)) {
-		sta->flags &= ~WLAN_STA_PS;
-		PDEBUG(DEBUG_PS2, "STA %pM changed to not use "
-		       "PS mode (type=0x%02X, stype=0x%02X)\n",
-		       sta->addr, type >> 2, stype >> 4);
-		if (type != IEEE80211_FTYPE_CTL ||
-		    stype != IEEE80211_STYPE_PSPOLL)
-			schedule_packet_send(local, sta);
-	}
-}
-
-
-/* Called only as a tasklet (software IRQ). Called for each RX frame to update
- * STA power saving state. pwrmgt is a flag from 802.11 frame_control field. */
-int hostap_update_sta_ps(local_info_t *local, struct ieee80211_hdr *hdr)
-{
-	struct sta_info *sta;
-	u16 fc;
-
-	spin_lock(&local->ap->sta_table_lock);
-	sta = ap_get_sta(local->ap, hdr->addr2);
-	if (sta)
-		atomic_inc(&sta->users);
-	spin_unlock(&local->ap->sta_table_lock);
-
-	if (!sta)
-		return -1;
-
-	fc = le16_to_cpu(hdr->frame_control);
-	hostap_update_sta_ps2(local, sta, fc & IEEE80211_FCTL_PM,
-			      fc & IEEE80211_FCTL_FTYPE,
-			      fc & IEEE80211_FCTL_STYPE);
-
-	atomic_dec(&sta->users);
-	return 0;
-}
-
-
-/* Called only as a tasklet (software IRQ). Called for each RX frame after
- * getting RX header and payload from hardware. */
-ap_rx_ret hostap_handle_sta_rx(local_info_t *local, struct net_device *dev,
-			       struct sk_buff *skb,
-			       struct hostap_80211_rx_status *rx_stats,
-			       int wds)
-{
-	int ret;
-	struct sta_info *sta;
-	u16 fc, type, stype;
-	struct ieee80211_hdr *hdr;
-
-	if (local->ap == NULL)
-		return AP_RX_CONTINUE;
-
-	hdr = (struct ieee80211_hdr *) skb->data;
-
-	fc = le16_to_cpu(hdr->frame_control);
-	type = fc & IEEE80211_FCTL_FTYPE;
-	stype = fc & IEEE80211_FCTL_STYPE;
-
-	spin_lock(&local->ap->sta_table_lock);
-	sta = ap_get_sta(local->ap, hdr->addr2);
-	if (sta)
-		atomic_inc(&sta->users);
-	spin_unlock(&local->ap->sta_table_lock);
-
-	if (sta && !(sta->flags & WLAN_STA_AUTHORIZED))
-		ret = AP_RX_CONTINUE_NOT_AUTHORIZED;
-	else
-		ret = AP_RX_CONTINUE;
-
-
-	if (fc & IEEE80211_FCTL_TODS) {
-		if (!wds && (sta == NULL || !(sta->flags & WLAN_STA_ASSOC))) {
-			if (local->hostapd) {
-				prism2_rx_80211(local->apdev, skb, rx_stats,
-						PRISM2_RX_NON_ASSOC);
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-			} else {
-				printk(KERN_DEBUG "%s: dropped received packet"
-				       " from non-associated STA %pM"
-				       " (type=0x%02x, subtype=0x%02x)\n",
-				       dev->name, hdr->addr2,
-				       type >> 2, stype >> 4);
-				hostap_rx(dev, skb, rx_stats);
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-			}
-			ret = AP_RX_EXIT;
-			goto out;
-		}
-	} else if (fc & IEEE80211_FCTL_FROMDS) {
-		if (!wds) {
-			/* FromDS frame - not for us; probably
-			 * broadcast/multicast in another BSS - drop */
-			if (ether_addr_equal(hdr->addr1, dev->dev_addr)) {
-				printk(KERN_DEBUG "Odd.. FromDS packet "
-				       "received with own BSSID\n");
-				hostap_dump_rx_80211(dev->name, skb, rx_stats);
-			}
-			ret = AP_RX_DROP;
-			goto out;
-		}
-	} else if (stype == IEEE80211_STYPE_NULLFUNC && sta == NULL &&
-		   ether_addr_equal(hdr->addr1, dev->dev_addr)) {
-
-		if (local->hostapd) {
-			prism2_rx_80211(local->apdev, skb, rx_stats,
-					PRISM2_RX_NON_ASSOC);
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-		} else {
-			/* At least Lucent f/w seems to send data::nullfunc
-			 * frames with no ToDS flag when the current AP returns
-			 * after being unavailable for some time. Speed up
-			 * re-association by informing the station about it not
-			 * being associated. */
-			printk(KERN_DEBUG "%s: rejected received nullfunc frame"
-			       " without ToDS from not associated STA %pM\n",
-			       dev->name, hdr->addr2);
-			hostap_rx(dev, skb, rx_stats);
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-		}
-		ret = AP_RX_EXIT;
-		goto out;
-	} else if (stype == IEEE80211_STYPE_NULLFUNC) {
-		/* At least Lucent cards seem to send periodic nullfunc
-		 * frames with ToDS. Let these through to update SQ
-		 * stats and PS state. Nullfunc frames do not contain
-		 * any data and they will be dropped below. */
-	} else {
-		/* If BSSID (Addr3) is foreign, this frame is a normal
-		 * broadcast frame from an IBSS network. Drop it silently.
-		 * If BSSID is own, report the dropping of this frame. */
-		if (ether_addr_equal(hdr->addr3, dev->dev_addr)) {
-			printk(KERN_DEBUG "%s: dropped received packet from %pM"
-			       " with no ToDS flag "
-			       "(type=0x%02x, subtype=0x%02x)\n", dev->name,
-			       hdr->addr2, type >> 2, stype >> 4);
-			hostap_dump_rx_80211(dev->name, skb, rx_stats);
-		}
-		ret = AP_RX_DROP;
-		goto out;
-	}
-
-	if (sta) {
-		hostap_update_sta_ps2(local, sta, fc & IEEE80211_FCTL_PM,
-				      type, stype);
-
-		sta->rx_packets++;
-		sta->rx_bytes += skb->len;
-		sta->last_rx = jiffies;
-	}
-
-	if (local->ap->nullfunc_ack && stype == IEEE80211_STYPE_NULLFUNC &&
-	    fc & IEEE80211_FCTL_TODS) {
-		if (local->hostapd) {
-			prism2_rx_80211(local->apdev, skb, rx_stats,
-					PRISM2_RX_NULLFUNC_ACK);
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-		} else {
-			/* some STA f/w's seem to require control::ACK frame
-			 * for data::nullfunc, but Prism2 f/w 0.8.0 (at least
-			 * from Compaq) does not send this.. Try to generate
-			 * ACK for these frames from the host driver to make
-			 * power saving work with, e.g., Lucent WaveLAN f/w */
-			hostap_rx(dev, skb, rx_stats);
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-		}
-		ret = AP_RX_EXIT;
-		goto out;
-	}
-
- out:
-	if (sta)
-		atomic_dec(&sta->users);
-
-	return ret;
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-int hostap_handle_sta_crypto(local_info_t *local,
-			     struct ieee80211_hdr *hdr,
-			     struct lib80211_crypt_data **crypt,
-			     void **sta_ptr)
-{
-	struct sta_info *sta;
-
-	spin_lock(&local->ap->sta_table_lock);
-	sta = ap_get_sta(local->ap, hdr->addr2);
-	if (sta)
-		atomic_inc(&sta->users);
-	spin_unlock(&local->ap->sta_table_lock);
-
-	if (!sta)
-		return -1;
-
-	if (sta->crypt) {
-		*crypt = sta->crypt;
-		*sta_ptr = sta;
-		/* hostap_handle_sta_release() will be called to release STA
-		 * info */
-	} else
-		atomic_dec(&sta->users);
-
-	return 0;
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-int hostap_is_sta_assoc(struct ap_data *ap, u8 *sta_addr)
-{
-	struct sta_info *sta;
-	int ret = 0;
-
-	spin_lock(&ap->sta_table_lock);
-	sta = ap_get_sta(ap, sta_addr);
-	if (sta != NULL && (sta->flags & WLAN_STA_ASSOC) && !sta->ap)
-		ret = 1;
-	spin_unlock(&ap->sta_table_lock);
-
-	return ret;
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-int hostap_is_sta_authorized(struct ap_data *ap, u8 *sta_addr)
-{
-	struct sta_info *sta;
-	int ret = 0;
-
-	spin_lock(&ap->sta_table_lock);
-	sta = ap_get_sta(ap, sta_addr);
-	if (sta != NULL && (sta->flags & WLAN_STA_ASSOC) && !sta->ap &&
-	    ((sta->flags & WLAN_STA_AUTHORIZED) ||
-	     ap->local->ieee_802_1x == 0))
-		ret = 1;
-	spin_unlock(&ap->sta_table_lock);
-
-	return ret;
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-int hostap_add_sta(struct ap_data *ap, u8 *sta_addr)
-{
-	struct sta_info *sta;
-	int ret = 1;
-
-	if (!ap)
-		return -1;
-
-	spin_lock(&ap->sta_table_lock);
-	sta = ap_get_sta(ap, sta_addr);
-	if (sta)
-		ret = 0;
-	spin_unlock(&ap->sta_table_lock);
-
-	if (ret == 1) {
-		sta = ap_add_sta(ap, sta_addr);
-		if (!sta)
-			return -1;
-		sta->flags = WLAN_STA_AUTH | WLAN_STA_ASSOC;
-		sta->ap = 1;
-		memset(sta->supported_rates, 0, sizeof(sta->supported_rates));
-		/* No way of knowing which rates are supported since we did not
-		 * get supported rates element from beacon/assoc req. Assume
-		 * that remote end supports all 802.11b rates. */
-		sta->supported_rates[0] = 0x82;
-		sta->supported_rates[1] = 0x84;
-		sta->supported_rates[2] = 0x0b;
-		sta->supported_rates[3] = 0x16;
-		sta->tx_supp_rates = WLAN_RATE_1M | WLAN_RATE_2M |
-			WLAN_RATE_5M5 | WLAN_RATE_11M;
-		sta->tx_rate = 110;
-		sta->tx_max_rate = sta->tx_rate_idx = 3;
-	}
-
-	return ret;
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-int hostap_update_rx_stats(struct ap_data *ap,
-			   struct ieee80211_hdr *hdr,
-			   struct hostap_80211_rx_status *rx_stats)
-{
-	struct sta_info *sta;
-
-	if (!ap)
-		return -1;
-
-	spin_lock(&ap->sta_table_lock);
-	sta = ap_get_sta(ap, hdr->addr2);
-	if (sta) {
-		sta->last_rx_silence = rx_stats->noise;
-		sta->last_rx_signal = rx_stats->signal;
-		sta->last_rx_rate = rx_stats->rate;
-		sta->last_rx_updated = IW_QUAL_ALL_UPDATED | IW_QUAL_DBM;
-		if (rx_stats->rate == 10)
-			sta->rx_count[0]++;
-		else if (rx_stats->rate == 20)
-			sta->rx_count[1]++;
-		else if (rx_stats->rate == 55)
-			sta->rx_count[2]++;
-		else if (rx_stats->rate == 110)
-			sta->rx_count[3]++;
-	}
-	spin_unlock(&ap->sta_table_lock);
-
-	return sta ? 0 : -1;
-}
-
-
-void hostap_update_rates(local_info_t *local)
-{
-	struct sta_info *sta;
-	struct ap_data *ap = local->ap;
-
-	if (!ap)
-		return;
-
-	spin_lock_bh(&ap->sta_table_lock);
-	list_for_each_entry(sta, &ap->sta_list, list) {
-		prism2_check_tx_rates(sta);
-	}
-	spin_unlock_bh(&ap->sta_table_lock);
-}
-
-
-void * ap_crypt_get_ptrs(struct ap_data *ap, u8 *addr, int permanent,
-			 struct lib80211_crypt_data ***crypt)
-{
-	struct sta_info *sta;
-
-	spin_lock_bh(&ap->sta_table_lock);
-	sta = ap_get_sta(ap, addr);
-	if (sta)
-		atomic_inc(&sta->users);
-	spin_unlock_bh(&ap->sta_table_lock);
-
-	if (!sta && permanent)
-		sta = ap_add_sta(ap, addr);
-
-	if (!sta)
-		return NULL;
-
-	if (permanent)
-		sta->flags |= WLAN_STA_PERM;
-
-	*crypt = &sta->crypt;
-
-	return sta;
-}
-
-
-void hostap_add_wds_links(local_info_t *local)
-{
-	struct ap_data *ap = local->ap;
-	struct sta_info *sta;
-
-	spin_lock_bh(&ap->sta_table_lock);
-	list_for_each_entry(sta, &ap->sta_list, list) {
-		if (sta->ap)
-			hostap_wds_link_oper(local, sta->addr, WDS_ADD);
-	}
-	spin_unlock_bh(&ap->sta_table_lock);
-
-	schedule_work(&local->ap->wds_oper_queue);
-}
-
-
-void hostap_wds_link_oper(local_info_t *local, u8 *addr, wds_oper_type type)
-{
-	struct wds_oper_data *entry;
-
-	entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
-	if (!entry)
-		return;
-	memcpy(entry->addr, addr, ETH_ALEN);
-	entry->type = type;
-	spin_lock_bh(&local->lock);
-	entry->next = local->ap->wds_oper_entries;
-	local->ap->wds_oper_entries = entry;
-	spin_unlock_bh(&local->lock);
-
-	schedule_work(&local->ap->wds_oper_queue);
-}
-
-
-EXPORT_SYMBOL(hostap_init_data);
-EXPORT_SYMBOL(hostap_init_ap_proc);
-EXPORT_SYMBOL(hostap_free_data);
-EXPORT_SYMBOL(hostap_check_sta_fw_version);
-EXPORT_SYMBOL(hostap_handle_sta_tx_exc);
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
diff --git a/drivers/net/wireless/intersil/hostap/hostap_ap.h b/drivers/net/wireless/intersil/hostap/hostap_ap.h
deleted file mode 100644
index b7ac9e2f1a39..000000000000
--- a/drivers/net/wireless/intersil/hostap/hostap_ap.h
+++ /dev/null
@@ -1,264 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef HOSTAP_AP_H
-#define HOSTAP_AP_H
-
-#include "hostap_80211.h"
-
-/* AP data structures for STAs */
-
-/* maximum number of frames to buffer per STA */
-#define STA_MAX_TX_BUFFER 32
-
-/* STA flags */
-#define WLAN_STA_AUTH BIT(0)
-#define WLAN_STA_ASSOC BIT(1)
-#define WLAN_STA_PS BIT(2)
-#define WLAN_STA_TIM BIT(3) /* TIM bit is on for PS stations */
-#define WLAN_STA_PERM BIT(4) /* permanent; do not remove entry on expiration */
-#define WLAN_STA_AUTHORIZED BIT(5) /* If 802.1X is used, this flag is
-				    * controlling whether STA is authorized to
-				    * send and receive non-IEEE 802.1X frames
-				    */
-#define WLAN_STA_PENDING_POLL BIT(6) /* pending activity poll not ACKed */
-
-#define WLAN_RATE_1M BIT(0)
-#define WLAN_RATE_2M BIT(1)
-#define WLAN_RATE_5M5 BIT(2)
-#define WLAN_RATE_11M BIT(3)
-#define WLAN_RATE_COUNT 4
-
-/* Maximum size of Supported Rates info element. IEEE 802.11 has a limit of 8,
- * but some pre-standard IEEE 802.11g products use longer elements. */
-#define WLAN_SUPP_RATES_MAX 32
-
-/* Try to increase TX rate after # successfully sent consecutive packets */
-#define WLAN_RATE_UPDATE_COUNT 50
-
-/* Decrease TX rate after # consecutive dropped packets */
-#define WLAN_RATE_DECREASE_THRESHOLD 2
-
-struct sta_info {
-	struct list_head list;
-	struct sta_info *hnext; /* next entry in hash table list */
-	atomic_t users; /* number of users (do not remove if > 0) */
-	struct proc_dir_entry *proc;
-
-	u8 addr[6];
-	u16 aid; /* STA's unique AID (1 .. 2007) or 0 if not yet assigned */
-	u32 flags;
-	u16 capability;
-	u16 listen_interval; /* or beacon_int for APs */
-	u8 supported_rates[WLAN_SUPP_RATES_MAX];
-
-	unsigned long last_auth;
-	unsigned long last_assoc;
-	unsigned long last_rx;
-	unsigned long last_tx;
-	unsigned long rx_packets, tx_packets;
-	unsigned long rx_bytes, tx_bytes;
-	struct sk_buff_head tx_buf;
-	/* FIX: timeout buffers with an expiry time somehow derived from
-	 * listen_interval */
-
-	s8 last_rx_silence; /* Noise in dBm */
-	s8 last_rx_signal; /* Signal strength in dBm */
-	u8 last_rx_rate; /* TX rate in 0.1 Mbps */
-	u8 last_rx_updated; /* IWSPY's struct iw_quality::updated */
-
-	u8 tx_supp_rates; /* bit field of supported TX rates */
-	u8 tx_rate; /* current TX rate (in 0.1 Mbps) */
-	u8 tx_rate_idx; /* current TX rate (WLAN_RATE_*) */
-	u8 tx_max_rate; /* max TX rate (WLAN_RATE_*) */
-	u32 tx_count[WLAN_RATE_COUNT]; /* number of frames sent (per rate) */
-	u32 rx_count[WLAN_RATE_COUNT]; /* number of frames received (per rate)
-					*/
-	u32 tx_since_last_failure;
-	u32 tx_consecutive_exc;
-
-	struct lib80211_crypt_data *crypt;
-
-	int ap; /* whether this station is an AP */
-
-	local_info_t *local;
-
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-	union {
-		struct {
-			char *challenge; /* shared key authentication
-					  * challenge */
-		} sta;
-		struct {
-			int ssid_len;
-			unsigned char ssid[MAX_SSID_LEN + 1]; /* AP's ssid */
-			int channel;
-			unsigned long last_beacon; /* last RX beacon time */
-		} ap;
-	} u;
-
-	struct timer_list timer;
-	enum { STA_NULLFUNC = 0, STA_DISASSOC, STA_DEAUTH } timeout_next;
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-};
-
-
-#define MAX_STA_COUNT 1024
-
-/* Maximum number of AIDs to use for STAs; must be 2007 or lower
- * (8802.11 limitation) */
-#define MAX_AID_TABLE_SIZE 128
-
-#define STA_HASH_SIZE 256
-#define STA_HASH(sta) (sta[5])
-
-
-/* Default value for maximum station inactivity. After AP_MAX_INACTIVITY_SEC
- * has passed since last received frame from the station, a nullfunc data
- * frame is sent to the station. If this frame is not acknowledged and no other
- * frames have been received, the station will be disassociated after
- * AP_DISASSOC_DELAY. Similarly, a the station will be deauthenticated after
- * AP_DEAUTH_DELAY. AP_TIMEOUT_RESOLUTION is the resolution that is used with
- * max inactivity timer. */
-#define AP_MAX_INACTIVITY_SEC (5 * 60)
-#define AP_DISASSOC_DELAY (HZ)
-#define AP_DEAUTH_DELAY (HZ)
-
-/* ap_policy: whether to accept frames to/from other APs/IBSS */
-typedef enum {
-	AP_OTHER_AP_SKIP_ALL = 0,
-	AP_OTHER_AP_SAME_SSID = 1,
-	AP_OTHER_AP_ALL = 2,
-	AP_OTHER_AP_EVEN_IBSS = 3
-} ap_policy_enum;
-
-#define PRISM2_AUTH_OPEN BIT(0)
-#define PRISM2_AUTH_SHARED_KEY BIT(1)
-
-
-/* MAC address-based restrictions */
-struct mac_entry {
-	struct list_head list;
-	u8 addr[6];
-};
-
-struct mac_restrictions {
-	enum { MAC_POLICY_OPEN = 0, MAC_POLICY_ALLOW, MAC_POLICY_DENY } policy;
-	unsigned int entries;
-	struct list_head mac_list;
-	spinlock_t lock;
-};
-
-
-struct add_sta_proc_data {
-	u8 addr[ETH_ALEN];
-	struct add_sta_proc_data *next;
-};
-
-
-typedef enum { WDS_ADD, WDS_DEL } wds_oper_type;
-struct wds_oper_data {
-	wds_oper_type type;
-	u8 addr[ETH_ALEN];
-	struct wds_oper_data *next;
-};
-
-
-struct ap_data {
-	int initialized; /* whether ap_data has been initialized */
-	local_info_t *local;
-	int bridge_packets; /* send packet to associated STAs directly to the
-			     * wireless media instead of higher layers in the
-			     * kernel */
-	unsigned int bridged_unicast; /* number of unicast frames bridged on
-				       * wireless media */
-	unsigned int bridged_multicast; /* number of non-unicast frames
-					 * bridged on wireless media */
-	unsigned int tx_drop_nonassoc; /* number of unicast TX packets dropped
-					* because they were to an address that
-					* was not associated */
-	int nullfunc_ack; /* use workaround for nullfunc frame ACKs */
-
-	spinlock_t sta_table_lock;
-	int num_sta; /* number of entries in sta_list */
-	struct list_head sta_list; /* STA info list head */
-	struct sta_info *sta_hash[STA_HASH_SIZE];
-
-	struct proc_dir_entry *proc;
-
-	ap_policy_enum ap_policy;
-	unsigned int max_inactivity;
-	int autom_ap_wds;
-
-	struct mac_restrictions mac_restrictions; /* MAC-based auth */
-	int last_tx_rate;
-
-	struct work_struct add_sta_proc_queue;
-	struct add_sta_proc_data *add_sta_proc_entries;
-
-	struct work_struct wds_oper_queue;
-	struct wds_oper_data *wds_oper_entries;
-
-	u16 tx_callback_idx;
-
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-	/* pointers to STA info; based on allocated AID or NULL if AID free
-	 * AID is in the range 1-2007, so sta_aid[0] corresponders to AID 1
-	 * and so on
-	 */
-	struct sta_info *sta_aid[MAX_AID_TABLE_SIZE];
-
-	u16 tx_callback_auth, tx_callback_assoc, tx_callback_poll;
-
-	/* WEP operations for generating challenges to be used with shared key
-	 * authentication */
-	struct lib80211_crypto_ops *crypt;
-	void *crypt_priv;
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-};
-
-
-void hostap_rx(struct net_device *dev, struct sk_buff *skb,
-	       struct hostap_80211_rx_status *rx_stats);
-void hostap_init_data(local_info_t *local);
-void hostap_init_ap_proc(local_info_t *local);
-void hostap_free_data(struct ap_data *ap);
-void hostap_check_sta_fw_version(struct ap_data *ap, int sta_fw_ver);
-
-typedef enum {
-	AP_TX_CONTINUE, AP_TX_DROP, AP_TX_RETRY, AP_TX_BUFFERED,
-	AP_TX_CONTINUE_NOT_AUTHORIZED
-} ap_tx_ret;
-struct hostap_tx_data {
-	struct sk_buff *skb;
-	int host_encrypt;
-	struct lib80211_crypt_data *crypt;
-	void *sta_ptr;
-};
-ap_tx_ret hostap_handle_sta_tx(local_info_t *local, struct hostap_tx_data *tx);
-void hostap_handle_sta_release(void *ptr);
-void hostap_handle_sta_tx_exc(local_info_t *local, struct sk_buff *skb);
-int hostap_update_sta_ps(local_info_t *local, struct ieee80211_hdr *hdr);
-typedef enum {
-	AP_RX_CONTINUE, AP_RX_DROP, AP_RX_EXIT, AP_RX_CONTINUE_NOT_AUTHORIZED
-} ap_rx_ret;
-ap_rx_ret hostap_handle_sta_rx(local_info_t *local, struct net_device *dev,
-			       struct sk_buff *skb,
-			       struct hostap_80211_rx_status *rx_stats,
-			       int wds);
-int hostap_handle_sta_crypto(local_info_t *local, struct ieee80211_hdr *hdr,
-			     struct lib80211_crypt_data **crypt,
-			     void **sta_ptr);
-int hostap_is_sta_assoc(struct ap_data *ap, u8 *sta_addr);
-int hostap_is_sta_authorized(struct ap_data *ap, u8 *sta_addr);
-int hostap_add_sta(struct ap_data *ap, u8 *sta_addr);
-int hostap_update_rx_stats(struct ap_data *ap, struct ieee80211_hdr *hdr,
-			   struct hostap_80211_rx_status *rx_stats);
-void hostap_update_rates(local_info_t *local);
-void hostap_add_wds_links(local_info_t *local);
-void hostap_wds_link_oper(local_info_t *local, u8 *addr, wds_oper_type type);
-
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-void hostap_deauth_all_stas(struct net_device *dev, struct ap_data *ap,
-			    int resend);
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-
-#endif /* HOSTAP_AP_H */
diff --git a/drivers/net/wireless/intersil/hostap/hostap_common.h b/drivers/net/wireless/intersil/hostap/hostap_common.h
deleted file mode 100644
index dd29a8e8d349..000000000000
--- a/drivers/net/wireless/intersil/hostap/hostap_common.h
+++ /dev/null
@@ -1,420 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef HOSTAP_COMMON_H
-#define HOSTAP_COMMON_H
-
-#include <linux/types.h>
-#include <linux/if_ether.h>
-
-/* IEEE 802.11 defines */
-
-/* HFA384X Configuration RIDs */
-#define HFA384X_RID_CNFPORTTYPE 0xFC00
-#define HFA384X_RID_CNFOWNMACADDR 0xFC01
-#define HFA384X_RID_CNFDESIREDSSID 0xFC02
-#define HFA384X_RID_CNFOWNCHANNEL 0xFC03
-#define HFA384X_RID_CNFOWNSSID 0xFC04
-#define HFA384X_RID_CNFOWNATIMWINDOW 0xFC05
-#define HFA384X_RID_CNFSYSTEMSCALE 0xFC06
-#define HFA384X_RID_CNFMAXDATALEN 0xFC07
-#define HFA384X_RID_CNFWDSADDRESS 0xFC08
-#define HFA384X_RID_CNFPMENABLED 0xFC09
-#define HFA384X_RID_CNFPMEPS 0xFC0A
-#define HFA384X_RID_CNFMULTICASTRECEIVE 0xFC0B
-#define HFA384X_RID_CNFMAXSLEEPDURATION 0xFC0C
-#define HFA384X_RID_CNFPMHOLDOVERDURATION 0xFC0D
-#define HFA384X_RID_CNFOWNNAME 0xFC0E
-#define HFA384X_RID_CNFOWNDTIMPERIOD 0xFC10
-#define HFA384X_RID_CNFWDSADDRESS1 0xFC11 /* AP f/w only */
-#define HFA384X_RID_CNFWDSADDRESS2 0xFC12 /* AP f/w only */
-#define HFA384X_RID_CNFWDSADDRESS3 0xFC13 /* AP f/w only */
-#define HFA384X_RID_CNFWDSADDRESS4 0xFC14 /* AP f/w only */
-#define HFA384X_RID_CNFWDSADDRESS5 0xFC15 /* AP f/w only */
-#define HFA384X_RID_CNFWDSADDRESS6 0xFC16 /* AP f/w only */
-#define HFA384X_RID_CNFMULTICASTPMBUFFERING 0xFC17 /* AP f/w only */
-#define HFA384X_RID_UNKNOWN1 0xFC20
-#define HFA384X_RID_UNKNOWN2 0xFC21
-#define HFA384X_RID_CNFWEPDEFAULTKEYID 0xFC23
-#define HFA384X_RID_CNFDEFAULTKEY0 0xFC24
-#define HFA384X_RID_CNFDEFAULTKEY1 0xFC25
-#define HFA384X_RID_CNFDEFAULTKEY2 0xFC26
-#define HFA384X_RID_CNFDEFAULTKEY3 0xFC27
-#define HFA384X_RID_CNFWEPFLAGS 0xFC28
-#define HFA384X_RID_CNFWEPKEYMAPPINGTABLE 0xFC29
-#define HFA384X_RID_CNFAUTHENTICATION 0xFC2A
-#define HFA384X_RID_CNFMAXASSOCSTA 0xFC2B /* AP f/w only */
-#define HFA384X_RID_CNFTXCONTROL 0xFC2C
-#define HFA384X_RID_CNFROAMINGMODE 0xFC2D
-#define HFA384X_RID_CNFHOSTAUTHENTICATION 0xFC2E /* AP f/w only */
-#define HFA384X_RID_CNFRCVCRCERROR 0xFC30
-#define HFA384X_RID_CNFMMLIFE 0xFC31
-#define HFA384X_RID_CNFALTRETRYCOUNT 0xFC32
-#define HFA384X_RID_CNFBEACONINT 0xFC33
-#define HFA384X_RID_CNFAPPCFINFO 0xFC34 /* AP f/w only */
-#define HFA384X_RID_CNFSTAPCFINFO 0xFC35
-#define HFA384X_RID_CNFPRIORITYQUSAGE 0xFC37
-#define HFA384X_RID_CNFTIMCTRL 0xFC40
-#define HFA384X_RID_UNKNOWN3 0xFC41 /* added in STA f/w 0.7.x */
-#define HFA384X_RID_CNFTHIRTY2TALLY 0xFC42 /* added in STA f/w 0.8.0 */
-#define HFA384X_RID_CNFENHSECURITY 0xFC43 /* AP f/w or STA f/w >= 1.6.3 */
-#define HFA384X_RID_CNFDBMADJUST 0xFC46 /* added in STA f/w 1.3.1 */
-#define HFA384X_RID_GENERICELEMENT 0xFC48 /* added in STA f/w 1.7.0;
-					   * write only */
-#define HFA384X_RID_PROPAGATIONDELAY 0xFC49 /* added in STA f/w 1.7.6 */
-#define HFA384X_RID_GROUPADDRESSES 0xFC80
-#define HFA384X_RID_CREATEIBSS 0xFC81
-#define HFA384X_RID_FRAGMENTATIONTHRESHOLD 0xFC82
-#define HFA384X_RID_RTSTHRESHOLD 0xFC83
-#define HFA384X_RID_TXRATECONTROL 0xFC84
-#define HFA384X_RID_PROMISCUOUSMODE 0xFC85
-#define HFA384X_RID_FRAGMENTATIONTHRESHOLD0 0xFC90 /* AP f/w only */
-#define HFA384X_RID_FRAGMENTATIONTHRESHOLD1 0xFC91 /* AP f/w only */
-#define HFA384X_RID_FRAGMENTATIONTHRESHOLD2 0xFC92 /* AP f/w only */
-#define HFA384X_RID_FRAGMENTATIONTHRESHOLD3 0xFC93 /* AP f/w only */
-#define HFA384X_RID_FRAGMENTATIONTHRESHOLD4 0xFC94 /* AP f/w only */
-#define HFA384X_RID_FRAGMENTATIONTHRESHOLD5 0xFC95 /* AP f/w only */
-#define HFA384X_RID_FRAGMENTATIONTHRESHOLD6 0xFC96 /* AP f/w only */
-#define HFA384X_RID_RTSTHRESHOLD0 0xFC97 /* AP f/w only */
-#define HFA384X_RID_RTSTHRESHOLD1 0xFC98 /* AP f/w only */
-#define HFA384X_RID_RTSTHRESHOLD2 0xFC99 /* AP f/w only */
-#define HFA384X_RID_RTSTHRESHOLD3 0xFC9A /* AP f/w only */
-#define HFA384X_RID_RTSTHRESHOLD4 0xFC9B /* AP f/w only */
-#define HFA384X_RID_RTSTHRESHOLD5 0xFC9C /* AP f/w only */
-#define HFA384X_RID_RTSTHRESHOLD6 0xFC9D /* AP f/w only */
-#define HFA384X_RID_TXRATECONTROL0 0xFC9E /* AP f/w only */
-#define HFA384X_RID_TXRATECONTROL1 0xFC9F /* AP f/w only */
-#define HFA384X_RID_TXRATECONTROL2 0xFCA0 /* AP f/w only */
-#define HFA384X_RID_TXRATECONTROL3 0xFCA1 /* AP f/w only */
-#define HFA384X_RID_TXRATECONTROL4 0xFCA2 /* AP f/w only */
-#define HFA384X_RID_TXRATECONTROL5 0xFCA3 /* AP f/w only */
-#define HFA384X_RID_TXRATECONTROL6 0xFCA4 /* AP f/w only */
-#define HFA384X_RID_CNFSHORTPREAMBLE 0xFCB0
-#define HFA384X_RID_CNFEXCLUDELONGPREAMBLE 0xFCB1
-#define HFA384X_RID_CNFAUTHENTICATIONRSPTO 0xFCB2
-#define HFA384X_RID_CNFBASICRATES 0xFCB3
-#define HFA384X_RID_CNFSUPPORTEDRATES 0xFCB4
-#define HFA384X_RID_CNFFALLBACKCTRL 0xFCB5 /* added in STA f/w 1.3.1 */
-#define HFA384X_RID_WEPKEYDISABLE 0xFCB6 /* added in STA f/w 1.3.1 */
-#define HFA384X_RID_WEPKEYMAPINDEX 0xFCB7 /* ? */
-#define HFA384X_RID_BROADCASTKEYID 0xFCB8 /* ? */
-#define HFA384X_RID_ENTSECFLAGEYID 0xFCB9 /* ? */
-#define HFA384X_RID_CNFPASSIVESCANCTRL 0xFCBA /* added in STA f/w 1.5.0 */
-#define HFA384X_RID_SSNHANDLINGMODE 0xFCBB /* added in STA f/w 1.7.0 */
-#define HFA384X_RID_MDCCONTROL 0xFCBC /* added in STA f/w 1.7.0 */
-#define HFA384X_RID_MDCCOUNTRY 0xFCBD /* added in STA f/w 1.7.0 */
-#define HFA384X_RID_TXPOWERMAX 0xFCBE /* added in STA f/w 1.7.0 */
-#define HFA384X_RID_CNFLFOENABLED 0xFCBF /* added in STA f/w 1.6.3 */
-#define HFA384X_RID_CAPINFO 0xFCC0 /* added in STA f/w 1.7.0 */
-#define HFA384X_RID_LISTENINTERVAL 0xFCC1 /* added in STA f/w 1.7.0 */
-#define HFA384X_RID_SW_ANT_DIV 0xFCC2 /* added in STA f/w 1.7.0; Prism3 */
-#define HFA384X_RID_LED_CTRL 0xFCC4 /* added in STA f/w 1.7.6 */
-#define HFA384X_RID_HFODELAY 0xFCC5 /* added in STA f/w 1.7.6 */
-#define HFA384X_RID_DISALLOWEDBSSID 0xFCC6 /* added in STA f/w 1.8.0 */
-#define HFA384X_RID_TICKTIME 0xFCE0
-#define HFA384X_RID_SCANREQUEST 0xFCE1
-#define HFA384X_RID_JOINREQUEST 0xFCE2
-#define HFA384X_RID_AUTHENTICATESTATION 0xFCE3 /* AP f/w only */
-#define HFA384X_RID_CHANNELINFOREQUEST 0xFCE4 /* AP f/w only */
-#define HFA384X_RID_HOSTSCAN 0xFCE5 /* added in STA f/w 1.3.1 */
-
-/* HFA384X Information RIDs */
-#define HFA384X_RID_MAXLOADTIME 0xFD00
-#define HFA384X_RID_DOWNLOADBUFFER 0xFD01
-#define HFA384X_RID_PRIID 0xFD02
-#define HFA384X_RID_PRISUPRANGE 0xFD03
-#define HFA384X_RID_CFIACTRANGES 0xFD04
-#define HFA384X_RID_NICSERNUM 0xFD0A
-#define HFA384X_RID_NICID 0xFD0B
-#define HFA384X_RID_MFISUPRANGE 0xFD0C
-#define HFA384X_RID_CFISUPRANGE 0xFD0D
-#define HFA384X_RID_CHANNELLIST 0xFD10
-#define HFA384X_RID_REGULATORYDOMAINS 0xFD11
-#define HFA384X_RID_TEMPTYPE 0xFD12
-#define HFA384X_RID_CIS 0xFD13
-#define HFA384X_RID_STAID 0xFD20
-#define HFA384X_RID_STASUPRANGE 0xFD21
-#define HFA384X_RID_MFIACTRANGES 0xFD22
-#define HFA384X_RID_CFIACTRANGES2 0xFD23
-#define HFA384X_RID_PRODUCTNAME 0xFD24 /* added in STA f/w 1.3.1;
-					* only Prism2.5(?) */
-#define HFA384X_RID_PORTSTATUS 0xFD40
-#define HFA384X_RID_CURRENTSSID 0xFD41
-#define HFA384X_RID_CURRENTBSSID 0xFD42
-#define HFA384X_RID_COMMSQUALITY 0xFD43
-#define HFA384X_RID_CURRENTTXRATE 0xFD44
-#define HFA384X_RID_CURRENTBEACONINTERVAL 0xFD45
-#define HFA384X_RID_CURRENTSCALETHRESHOLDS 0xFD46
-#define HFA384X_RID_PROTOCOLRSPTIME 0xFD47
-#define HFA384X_RID_SHORTRETRYLIMIT 0xFD48
-#define HFA384X_RID_LONGRETRYLIMIT 0xFD49
-#define HFA384X_RID_MAXTRANSMITLIFETIME 0xFD4A
-#define HFA384X_RID_MAXRECEIVELIFETIME 0xFD4B
-#define HFA384X_RID_CFPOLLABLE 0xFD4C
-#define HFA384X_RID_AUTHENTICATIONALGORITHMS 0xFD4D
-#define HFA384X_RID_PRIVACYOPTIONIMPLEMENTED 0xFD4F
-#define HFA384X_RID_DBMCOMMSQUALITY 0xFD51 /* added in STA f/w 1.3.1 */
-#define HFA384X_RID_CURRENTTXRATE1 0xFD80 /* AP f/w only */
-#define HFA384X_RID_CURRENTTXRATE2 0xFD81 /* AP f/w only */
-#define HFA384X_RID_CURRENTTXRATE3 0xFD82 /* AP f/w only */
-#define HFA384X_RID_CURRENTTXRATE4 0xFD83 /* AP f/w only */
-#define HFA384X_RID_CURRENTTXRATE5 0xFD84 /* AP f/w only */
-#define HFA384X_RID_CURRENTTXRATE6 0xFD85 /* AP f/w only */
-#define HFA384X_RID_OWNMACADDR 0xFD86 /* AP f/w only */
-#define HFA384X_RID_SCANRESULTSTABLE 0xFD88 /* added in STA f/w 0.8.3 */
-#define HFA384X_RID_HOSTSCANRESULTS 0xFD89 /* added in STA f/w 1.3.1 */
-#define HFA384X_RID_AUTHENTICATIONUSED 0xFD8A /* added in STA f/w 1.3.4 */
-#define HFA384X_RID_CNFFAASWITCHCTRL 0xFD8B /* added in STA f/w 1.6.3 */
-#define HFA384X_RID_ASSOCIATIONFAILURE 0xFD8D /* added in STA f/w 1.8.0 */
-#define HFA384X_RID_PHYTYPE 0xFDC0
-#define HFA384X_RID_CURRENTCHANNEL 0xFDC1
-#define HFA384X_RID_CURRENTPOWERSTATE 0xFDC2
-#define HFA384X_RID_CCAMODE 0xFDC3
-#define HFA384X_RID_SUPPORTEDDATARATES 0xFDC6
-#define HFA384X_RID_LFO_VOLT_REG_TEST_RES 0xFDC7 /* added in STA f/w 1.7.1 */
-#define HFA384X_RID_BUILDSEQ 0xFFFE
-#define HFA384X_RID_FWID 0xFFFF
-
-
-struct hfa384x_comp_ident
-{
-	__le16 id;
-	__le16 variant;
-	__le16 major;
-	__le16 minor;
-} __packed;
-
-#define HFA384X_COMP_ID_PRI 0x15
-#define HFA384X_COMP_ID_STA 0x1f
-#define HFA384X_COMP_ID_FW_AP 0x14b
-
-struct hfa384x_sup_range
-{
-	__le16 role;
-	__le16 id;
-	__le16 variant;
-	__le16 bottom;
-	__le16 top;
-} __packed;
-
-
-struct hfa384x_build_id
-{
-	__le16 pri_seq;
-	__le16 sec_seq;
-} __packed;
-
-/* FD01 - Download Buffer */
-struct hfa384x_rid_download_buffer
-{
-	__le16 page;
-	__le16 offset;
-	__le16 length;
-} __packed;
-
-/* BSS connection quality (RID FD43 range, RID FD51 dBm-normalized) */
-struct hfa384x_comms_quality {
-	__le16 comm_qual; /* 0 .. 92 */
-	__le16 signal_level; /* 27 .. 154 */
-	__le16 noise_level; /* 27 .. 154 */
-} __packed;
-
-
-/* netdevice private ioctls (used, e.g., with iwpriv from user space) */
-
-/* New wireless extensions API - SET/GET convention (even ioctl numbers are
- * root only)
- */
-#define PRISM2_IOCTL_PRISM2_PARAM (SIOCIWFIRSTPRIV + 0)
-#define PRISM2_IOCTL_GET_PRISM2_PARAM (SIOCIWFIRSTPRIV + 1)
-#define PRISM2_IOCTL_WRITEMIF (SIOCIWFIRSTPRIV + 2)
-#define PRISM2_IOCTL_READMIF (SIOCIWFIRSTPRIV + 3)
-#define PRISM2_IOCTL_MONITOR (SIOCIWFIRSTPRIV + 4)
-#define PRISM2_IOCTL_RESET (SIOCIWFIRSTPRIV + 6)
-#define PRISM2_IOCTL_INQUIRE (SIOCIWFIRSTPRIV + 8)
-#define PRISM2_IOCTL_WDS_ADD (SIOCIWFIRSTPRIV + 10)
-#define PRISM2_IOCTL_WDS_DEL (SIOCIWFIRSTPRIV + 12)
-#define PRISM2_IOCTL_SET_RID_WORD (SIOCIWFIRSTPRIV + 14)
-#define PRISM2_IOCTL_MACCMD (SIOCIWFIRSTPRIV + 16)
-#define PRISM2_IOCTL_ADDMAC (SIOCIWFIRSTPRIV + 18)
-#define PRISM2_IOCTL_DELMAC (SIOCIWFIRSTPRIV + 20)
-#define PRISM2_IOCTL_KICKMAC (SIOCIWFIRSTPRIV + 22)
-
-/* following are not in SIOCGIWPRIV list; check permission in the driver code
- */
-#define PRISM2_IOCTL_DOWNLOAD (SIOCDEVPRIVATE + 13)
-#define PRISM2_IOCTL_HOSTAPD (SIOCDEVPRIVATE + 14)
-
-
-/* PRISM2_IOCTL_PRISM2_PARAM ioctl() subtypes: */
-enum {
-	/* PRISM2_PARAM_PTYPE = 1, */ /* REMOVED 2003-10-22 */
-	PRISM2_PARAM_TXRATECTRL = 2,
-	PRISM2_PARAM_BEACON_INT = 3,
-	PRISM2_PARAM_PSEUDO_IBSS = 4,
-	PRISM2_PARAM_ALC = 5,
-	/* PRISM2_PARAM_TXPOWER = 6, */ /* REMOVED 2003-10-22 */
-	PRISM2_PARAM_DUMP = 7,
-	PRISM2_PARAM_OTHER_AP_POLICY = 8,
-	PRISM2_PARAM_AP_MAX_INACTIVITY = 9,
-	PRISM2_PARAM_AP_BRIDGE_PACKETS = 10,
-	PRISM2_PARAM_DTIM_PERIOD = 11,
-	PRISM2_PARAM_AP_NULLFUNC_ACK = 12,
-	PRISM2_PARAM_MAX_WDS = 13,
-	PRISM2_PARAM_AP_AUTOM_AP_WDS = 14,
-	PRISM2_PARAM_AP_AUTH_ALGS = 15,
-	PRISM2_PARAM_MONITOR_ALLOW_FCSERR = 16,
-	PRISM2_PARAM_HOST_ENCRYPT = 17,
-	PRISM2_PARAM_HOST_DECRYPT = 18,
-	/* PRISM2_PARAM_BUS_MASTER_THRESHOLD_RX = 19, REMOVED 2005-08-14 */
-	/* PRISM2_PARAM_BUS_MASTER_THRESHOLD_TX = 20, REMOVED 2005-08-14 */
-	PRISM2_PARAM_HOST_ROAMING = 21,
-	PRISM2_PARAM_BCRX_STA_KEY = 22,
-	PRISM2_PARAM_IEEE_802_1X = 23,
-	PRISM2_PARAM_ANTSEL_TX = 24,
-	PRISM2_PARAM_ANTSEL_RX = 25,
-	PRISM2_PARAM_MONITOR_TYPE = 26,
-	PRISM2_PARAM_WDS_TYPE = 27,
-	PRISM2_PARAM_HOSTSCAN = 28,
-	PRISM2_PARAM_AP_SCAN = 29,
-	PRISM2_PARAM_ENH_SEC = 30,
-	PRISM2_PARAM_IO_DEBUG = 31,
-	PRISM2_PARAM_BASIC_RATES = 32,
-	PRISM2_PARAM_OPER_RATES = 33,
-	PRISM2_PARAM_HOSTAPD = 34,
-	PRISM2_PARAM_HOSTAPD_STA = 35,
-	PRISM2_PARAM_WPA = 36,
-	PRISM2_PARAM_PRIVACY_INVOKED = 37,
-	PRISM2_PARAM_TKIP_COUNTERMEASURES = 38,
-	PRISM2_PARAM_DROP_UNENCRYPTED = 39,
-	PRISM2_PARAM_SCAN_CHANNEL_MASK = 40,
-};
-
-enum { HOSTAP_ANTSEL_DO_NOT_TOUCH = 0, HOSTAP_ANTSEL_DIVERSITY = 1,
-       HOSTAP_ANTSEL_LOW = 2, HOSTAP_ANTSEL_HIGH = 3 };
-
-
-/* PRISM2_IOCTL_MACCMD ioctl() subcommands: */
-enum { AP_MAC_CMD_POLICY_OPEN = 0, AP_MAC_CMD_POLICY_ALLOW = 1,
-       AP_MAC_CMD_POLICY_DENY = 2, AP_MAC_CMD_FLUSH = 3,
-       AP_MAC_CMD_KICKALL = 4 };
-
-
-/* PRISM2_IOCTL_DOWNLOAD ioctl() dl_cmd: */
-enum {
-	PRISM2_DOWNLOAD_VOLATILE = 1 /* RAM */,
-	/* Note! Old versions of prism2_srec have a fatal error in CRC-16
-	 * calculation, which will corrupt all non-volatile downloads.
-	 * PRISM2_DOWNLOAD_NON_VOLATILE used to be 2, but it is now 3 to
-	 * prevent use of old versions of prism2_srec for non-volatile
-	 * download. */
-	PRISM2_DOWNLOAD_NON_VOLATILE = 3 /* FLASH */,
-	PRISM2_DOWNLOAD_VOLATILE_GENESIS = 4 /* RAM in Genesis mode */,
-	/* Persistent versions of volatile download commands (keep firmware
-	 * data in memory and automatically re-download after hw_reset */
-	PRISM2_DOWNLOAD_VOLATILE_PERSISTENT = 5,
-	PRISM2_DOWNLOAD_VOLATILE_GENESIS_PERSISTENT = 6,
-};
-
-struct prism2_download_param {
-	u32 dl_cmd;
-	u32 start_addr;
-	u32 num_areas;
-	struct prism2_download_area {
-		u32 addr; /* wlan card address */
-		u32 len;
-		void __user *ptr; /* pointer to data in user space */
-	} data[];
-};
-
-#define PRISM2_MAX_DOWNLOAD_AREA_LEN 131072
-#define PRISM2_MAX_DOWNLOAD_LEN 262144
-
-
-/* PRISM2_IOCTL_HOSTAPD ioctl() cmd: */
-enum {
-	PRISM2_HOSTAPD_FLUSH = 1,
-	PRISM2_HOSTAPD_ADD_STA = 2,
-	PRISM2_HOSTAPD_REMOVE_STA = 3,
-	PRISM2_HOSTAPD_GET_INFO_STA = 4,
-	/* REMOVED: PRISM2_HOSTAPD_RESET_TXEXC_STA = 5, */
-	PRISM2_SET_ENCRYPTION = 6,
-	PRISM2_GET_ENCRYPTION = 7,
-	PRISM2_HOSTAPD_SET_FLAGS_STA = 8,
-	PRISM2_HOSTAPD_GET_RID = 9,
-	PRISM2_HOSTAPD_SET_RID = 10,
-	PRISM2_HOSTAPD_SET_ASSOC_AP_ADDR = 11,
-	PRISM2_HOSTAPD_SET_GENERIC_ELEMENT = 12,
-	PRISM2_HOSTAPD_MLME = 13,
-	PRISM2_HOSTAPD_SCAN_REQ = 14,
-	PRISM2_HOSTAPD_STA_CLEAR_STATS = 15,
-};
-
-#define PRISM2_HOSTAPD_MAX_BUF_SIZE 1024
-#define PRISM2_HOSTAPD_RID_HDR_LEN \
-offsetof(struct prism2_hostapd_param, u.rid.data)
-#define PRISM2_HOSTAPD_GENERIC_ELEMENT_HDR_LEN \
-offsetof(struct prism2_hostapd_param, u.generic_elem.data)
-
-/* Maximum length for algorithm names (-1 for nul termination) used in ioctl()
- */
-#define HOSTAP_CRYPT_ALG_NAME_LEN 16
-
-
-struct prism2_hostapd_param {
-	u32 cmd;
-	u8 sta_addr[ETH_ALEN];
-	union {
-		struct {
-			u16 aid;
-			u16 capability;
-			u8 tx_supp_rates;
-		} add_sta;
-		struct {
-			u32 inactive_sec;
-		} get_info_sta;
-		struct {
-			u8 alg[HOSTAP_CRYPT_ALG_NAME_LEN];
-			u32 flags;
-			u32 err;
-			u8 idx;
-			u8 seq[8]; /* sequence counter (set: RX, get: TX) */
-			u16 key_len;
-			u8 key[0];
-		} crypt;
-		struct {
-			u32 flags_and;
-			u32 flags_or;
-		} set_flags_sta;
-		struct {
-			u16 rid;
-			u16 len;
-			u8 data[0];
-		} rid;
-		struct {
-			u8 len;
-			u8 data[0];
-		} generic_elem;
-		struct {
-#define MLME_STA_DEAUTH 0
-#define MLME_STA_DISASSOC 1
-			u16 cmd;
-			u16 reason_code;
-		} mlme;
-		struct {
-			u8 ssid_len;
-			u8 ssid[32];
-		} scan_req;
-	} u;
-};
-
-#define HOSTAP_CRYPT_FLAG_SET_TX_KEY BIT(0)
-#define HOSTAP_CRYPT_FLAG_PERMANENT BIT(1)
-
-#define HOSTAP_CRYPT_ERR_UNKNOWN_ALG 2
-#define HOSTAP_CRYPT_ERR_UNKNOWN_ADDR 3
-#define HOSTAP_CRYPT_ERR_CRYPT_INIT_FAILED 4
-#define HOSTAP_CRYPT_ERR_KEY_SET_FAILED 5
-#define HOSTAP_CRYPT_ERR_TX_KEY_SET_FAILED 6
-#define HOSTAP_CRYPT_ERR_CARD_CONF_FAILED 7
-
-
-#endif /* HOSTAP_COMMON_H */
diff --git a/drivers/net/wireless/intersil/hostap/hostap_config.h b/drivers/net/wireless/intersil/hostap/hostap_config.h
deleted file mode 100644
index 3ebd55847fad..000000000000
--- a/drivers/net/wireless/intersil/hostap/hostap_config.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef HOSTAP_CONFIG_H
-#define HOSTAP_CONFIG_H
-
-/* In the previous versions of Host AP driver, support for user space version
- * of IEEE 802.11 management (hostapd) used to be disabled in the default
- * configuration. From now on, support for hostapd is always included and it is
- * possible to disable kernel driver version of IEEE 802.11 management with a
- * separate define, PRISM2_NO_KERNEL_IEEE80211_MGMT. */
-/* #define PRISM2_NO_KERNEL_IEEE80211_MGMT */
-
-/* Maximum number of events handler per one interrupt */
-#define PRISM2_MAX_INTERRUPT_EVENTS 20
-
-/* Include code for downloading firmware images into volatile RAM. */
-#define PRISM2_DOWNLOAD_SUPPORT
-
-/* Allow kernel configuration to enable download support. */
-#if !defined(PRISM2_DOWNLOAD_SUPPORT) && defined(CONFIG_HOSTAP_FIRMWARE)
-#define PRISM2_DOWNLOAD_SUPPORT
-#endif
-
-/* Allow kernel configuration to enable non-volatile download support. */
-#ifdef CONFIG_HOSTAP_FIRMWARE_NVRAM
-#define PRISM2_NON_VOLATILE_DOWNLOAD
-#endif
-
-/* Save low-level I/O for debugging. This should not be enabled in normal use.
- */
-/* #define PRISM2_IO_DEBUG */
-
-/* Following defines can be used to remove unneeded parts of the driver, e.g.,
- * to limit the size of the kernel module. Definitions can be added here in
- * hostap_config.h or they can be added to make command with ccflags-y,
- * e.g.,
- * 'make pccard ccflags-y="-DPRISM2_NO_DEBUG -DPRISM2_NO_PROCFS_DEBUG"'
- */
-
-/* Do not include debug messages into the driver */
-/* #define PRISM2_NO_DEBUG */
-
-/* Do not include /proc/net/prism2/wlan#/{registers,debug} */
-/* #define PRISM2_NO_PROCFS_DEBUG */
-
-/* Do not include station functionality (i.e., allow only Master (Host AP) mode
- */
-/* #define PRISM2_NO_STATION_MODES */
-
-#endif /* HOSTAP_CONFIG_H */
diff --git a/drivers/net/wireless/intersil/hostap/hostap_cs.c b/drivers/net/wireless/intersil/hostap/hostap_cs.c
deleted file mode 100644
index ec7db2badc40..000000000000
--- a/drivers/net/wireless/intersil/hostap/hostap_cs.c
+++ /dev/null
@@ -1,710 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-#define PRISM2_PCCARD
-
-#include <linux/module.h>
-#include <linux/if.h>
-#include <linux/slab.h>
-#include <linux/wait.h>
-#include <linux/timer.h>
-#include <linux/skbuff.h>
-#include <linux/netdevice.h>
-#include <linux/workqueue.h>
-#include <linux/wireless.h>
-#include <net/iw_handler.h>
-
-#include <pcmcia/cistpl.h>
-#include <pcmcia/cisreg.h>
-#include <pcmcia/ds.h>
-
-#include <asm/io.h>
-
-#include "hostap_wlan.h"
-
-
-static char *dev_info = "hostap_cs";
-
-MODULE_AUTHOR("Jouni Malinen");
-MODULE_DESCRIPTION("Support for Intersil Prism2-based 802.11 wireless LAN "
-		   "cards (PC Card).");
-MODULE_LICENSE("GPL");
-
-
-static int ignore_cis_vcc;
-module_param(ignore_cis_vcc, int, 0444);
-MODULE_PARM_DESC(ignore_cis_vcc, "Ignore broken CIS VCC entry");
-
-
-/* struct local_info::hw_priv */
-struct hostap_cs_priv {
-	struct pcmcia_device *link;
-	int sandisk_connectplus;
-};
-
-
-#ifdef PRISM2_IO_DEBUG
-
-static inline void hfa384x_outb_debug(struct net_device *dev, int a, u8 v)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	unsigned long flags;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-	spin_lock_irqsave(&local->lock, flags);
-	prism2_io_debug_add(dev, PRISM2_IO_DEBUG_CMD_OUTB, a, v);
-	outb(v, dev->base_addr + a);
-	spin_unlock_irqrestore(&local->lock, flags);
-}
-
-static inline u8 hfa384x_inb_debug(struct net_device *dev, int a)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	unsigned long flags;
-	u8 v;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-	spin_lock_irqsave(&local->lock, flags);
-	v = inb(dev->base_addr + a);
-	prism2_io_debug_add(dev, PRISM2_IO_DEBUG_CMD_INB, a, v);
-	spin_unlock_irqrestore(&local->lock, flags);
-	return v;
-}
-
-static inline void hfa384x_outw_debug(struct net_device *dev, int a, u16 v)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	unsigned long flags;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-	spin_lock_irqsave(&local->lock, flags);
-	prism2_io_debug_add(dev, PRISM2_IO_DEBUG_CMD_OUTW, a, v);
-	outw(v, dev->base_addr + a);
-	spin_unlock_irqrestore(&local->lock, flags);
-}
-
-static inline u16 hfa384x_inw_debug(struct net_device *dev, int a)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	unsigned long flags;
-	u16 v;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-	spin_lock_irqsave(&local->lock, flags);
-	v = inw(dev->base_addr + a);
-	prism2_io_debug_add(dev, PRISM2_IO_DEBUG_CMD_INW, a, v);
-	spin_unlock_irqrestore(&local->lock, flags);
-	return v;
-}
-
-static inline void hfa384x_outsw_debug(struct net_device *dev, int a,
-				       u8 *buf, int wc)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	unsigned long flags;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-	spin_lock_irqsave(&local->lock, flags);
-	prism2_io_debug_add(dev, PRISM2_IO_DEBUG_CMD_OUTSW, a, wc);
-	outsw(dev->base_addr + a, buf, wc);
-	spin_unlock_irqrestore(&local->lock, flags);
-}
-
-static inline void hfa384x_insw_debug(struct net_device *dev, int a,
-				      u8 *buf, int wc)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	unsigned long flags;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-	spin_lock_irqsave(&local->lock, flags);
-	prism2_io_debug_add(dev, PRISM2_IO_DEBUG_CMD_INSW, a, wc);
-	insw(dev->base_addr + a, buf, wc);
-	spin_unlock_irqrestore(&local->lock, flags);
-}
-
-#define HFA384X_OUTB(v,a) hfa384x_outb_debug(dev, (a), (v))
-#define HFA384X_INB(a) hfa384x_inb_debug(dev, (a))
-#define HFA384X_OUTW(v,a) hfa384x_outw_debug(dev, (a), (v))
-#define HFA384X_INW(a) hfa384x_inw_debug(dev, (a))
-#define HFA384X_OUTSW(a, buf, wc) hfa384x_outsw_debug(dev, (a), (buf), (wc))
-#define HFA384X_INSW(a, buf, wc) hfa384x_insw_debug(dev, (a), (buf), (wc))
-
-#else /* PRISM2_IO_DEBUG */
-
-#define HFA384X_OUTB(v,a) outb((v), dev->base_addr + (a))
-#define HFA384X_INB(a) inb(dev->base_addr + (a))
-#define HFA384X_OUTW(v,a) outw((v), dev->base_addr + (a))
-#define HFA384X_INW(a) inw(dev->base_addr + (a))
-#define HFA384X_INSW(a, buf, wc) insw(dev->base_addr + (a), buf, wc)
-#define HFA384X_OUTSW(a, buf, wc) outsw(dev->base_addr + (a), buf, wc)
-
-#endif /* PRISM2_IO_DEBUG */
-
-
-static int hfa384x_from_bap(struct net_device *dev, u16 bap, void *buf,
-			    int len)
-{
-	u16 d_off;
-	u16 *pos;
-
-	d_off = (bap == 1) ? HFA384X_DATA1_OFF : HFA384X_DATA0_OFF;
-	pos = (u16 *) buf;
-
-	if (len / 2)
-		HFA384X_INSW(d_off, buf, len / 2);
-	pos += len / 2;
-
-	if (len & 1)
-		*((char *) pos) = HFA384X_INB(d_off);
-
-	return 0;
-}
-
-
-static int hfa384x_to_bap(struct net_device *dev, u16 bap, void *buf, int len)
-{
-	u16 d_off;
-	u16 *pos;
-
-	d_off = (bap == 1) ? HFA384X_DATA1_OFF : HFA384X_DATA0_OFF;
-	pos = (u16 *) buf;
-
-	if (len / 2)
-		HFA384X_OUTSW(d_off, buf, len / 2);
-	pos += len / 2;
-
-	if (len & 1)
-		HFA384X_OUTB(*((char *) pos), d_off);
-
-	return 0;
-}
-
-
-/* FIX: This might change at some point.. */
-#include "hostap_hw.c"
-
-
-
-static void prism2_detach(struct pcmcia_device *p_dev);
-static void prism2_release(u_long arg);
-static int prism2_config(struct pcmcia_device *link);
-
-
-static int prism2_pccard_card_present(local_info_t *local)
-{
-	struct hostap_cs_priv *hw_priv = local->hw_priv;
-	if (hw_priv != NULL && hw_priv->link != NULL && pcmcia_dev_present(hw_priv->link))
-		return 1;
-	return 0;
-}
-
-
-/*
- * SanDisk CompactFlash WLAN Flashcard - Product Manual v1.0
- * Document No. 20-10-00058, January 2004
- * http://www.sandisk.com/pdf/industrial/ProdManualCFWLANv1.0.pdf
- */
-#define SANDISK_WLAN_ACTIVATION_OFF 0x40
-#define SANDISK_HCR_OFF 0x42
-
-
-static void sandisk_set_iobase(local_info_t *local)
-{
-	int res;
-	struct hostap_cs_priv *hw_priv = local->hw_priv;
-
-	res = pcmcia_write_config_byte(hw_priv->link, 0x10,
-				hw_priv->link->resource[0]->start & 0x00ff);
-	if (res != 0) {
-		printk(KERN_DEBUG "Prism3 SanDisk - failed to set I/O base 0 -"
-		       " res=%d\n", res);
-	}
-	udelay(10);
-
-	res = pcmcia_write_config_byte(hw_priv->link, 0x12,
-				(hw_priv->link->resource[0]->start >> 8) & 0x00ff);
-	if (res != 0) {
-		printk(KERN_DEBUG "Prism3 SanDisk - failed to set I/O base 1 -"
-		       " res=%d\n", res);
-	}
-}
-
-
-static void sandisk_write_hcr(local_info_t *local, int hcr)
-{
-	struct net_device *dev = local->dev;
-	int i;
-
-	HFA384X_OUTB(0x80, SANDISK_WLAN_ACTIVATION_OFF);
-	udelay(50);
-	for (i = 0; i < 10; i++) {
-		HFA384X_OUTB(hcr, SANDISK_HCR_OFF);
-	}
-	udelay(55);
-	HFA384X_OUTB(0x45, SANDISK_WLAN_ACTIVATION_OFF);
-}
-
-
-static int sandisk_enable_wireless(struct net_device *dev)
-{
-	int res, ret = 0;
-	struct hostap_interface *iface = netdev_priv(dev);
-	local_info_t *local = iface->local;
-	struct hostap_cs_priv *hw_priv = local->hw_priv;
-
-	if (resource_size(hw_priv->link->resource[0]) < 0x42) {
-		/* Not enough ports to be SanDisk multi-function card */
-		ret = -ENODEV;
-		goto done;
-	}
-
-	if (hw_priv->link->manf_id != 0xd601 || hw_priv->link->card_id != 0x0101) {
-		/* No SanDisk manfid found */
-		ret = -ENODEV;
-		goto done;
-	}
-
-	if (hw_priv->link->socket->functions < 2) {
-		/* No multi-function links found */
-		ret = -ENODEV;
-		goto done;
-	}
-
-	printk(KERN_DEBUG "%s: Multi-function SanDisk ConnectPlus detected"
-	       " - using vendor-specific initialization\n", dev->name);
-	hw_priv->sandisk_connectplus = 1;
-
-	res = pcmcia_write_config_byte(hw_priv->link, CISREG_COR,
-				COR_SOFT_RESET);
-	if (res != 0) {
-		printk(KERN_DEBUG "%s: SanDisk - COR sreset failed (%d)\n",
-		       dev->name, res);
-		goto done;
-	}
-	mdelay(5);
-
-	/*
-	 * Do not enable interrupts here to avoid some bogus events. Interrupts
-	 * will be enabled during the first cor_sreset call.
-	 */
-	res = pcmcia_write_config_byte(hw_priv->link, CISREG_COR,
-				(COR_LEVEL_REQ | 0x8 | COR_ADDR_DECODE |
-					COR_FUNC_ENA));
-	if (res != 0) {
-		printk(KERN_DEBUG "%s: SanDisk - COR sreset failed (%d)\n",
-		       dev->name, res);
-		goto done;
-	}
-	mdelay(5);
-
-	sandisk_set_iobase(local);
-
-	HFA384X_OUTB(0xc5, SANDISK_WLAN_ACTIVATION_OFF);
-	udelay(10);
-	HFA384X_OUTB(0x4b, SANDISK_WLAN_ACTIVATION_OFF);
-	udelay(10);
-
-done:
-	return ret;
-}
-
-
-static void prism2_pccard_cor_sreset(local_info_t *local)
-{
-	int res;
-	u8 val;
-	struct hostap_cs_priv *hw_priv = local->hw_priv;
-
-	if (!prism2_pccard_card_present(local))
-	       return;
-
-	res = pcmcia_read_config_byte(hw_priv->link, CISREG_COR, &val);
-	if (res != 0) {
-		printk(KERN_DEBUG "prism2_pccard_cor_sreset failed 1 (%d)\n",
-		       res);
-		return;
-	}
-	printk(KERN_DEBUG "prism2_pccard_cor_sreset: original COR %02x\n",
-		val);
-
-	val |= COR_SOFT_RESET;
-	res = pcmcia_write_config_byte(hw_priv->link, CISREG_COR, val);
-	if (res != 0) {
-		printk(KERN_DEBUG "prism2_pccard_cor_sreset failed 2 (%d)\n",
-		       res);
-		return;
-	}
-
-	mdelay(hw_priv->sandisk_connectplus ? 5 : 2);
-
-	val &= ~COR_SOFT_RESET;
-	if (hw_priv->sandisk_connectplus)
-		val |= COR_IREQ_ENA;
-	res = pcmcia_write_config_byte(hw_priv->link, CISREG_COR, val);
-	if (res != 0) {
-		printk(KERN_DEBUG "prism2_pccard_cor_sreset failed 3 (%d)\n",
-		       res);
-		return;
-	}
-
-	mdelay(hw_priv->sandisk_connectplus ? 5 : 2);
-
-	if (hw_priv->sandisk_connectplus)
-		sandisk_set_iobase(local);
-}
-
-
-static void prism2_pccard_genesis_reset(local_info_t *local, int hcr)
-{
-	int res;
-	u8 old_cor;
-	struct hostap_cs_priv *hw_priv = local->hw_priv;
-
-	if (!prism2_pccard_card_present(local))
-	       return;
-
-	if (hw_priv->sandisk_connectplus) {
-		sandisk_write_hcr(local, hcr);
-		return;
-	}
-
-	res = pcmcia_read_config_byte(hw_priv->link, CISREG_COR, &old_cor);
-	if (res != 0) {
-		printk(KERN_DEBUG "%s failed 1 (%d)\n", __func__, res);
-		return;
-	}
-	printk(KERN_DEBUG "%s: original COR %02x\n", __func__, old_cor);
-
-	res = pcmcia_write_config_byte(hw_priv->link, CISREG_COR,
-				old_cor | COR_SOFT_RESET);
-	if (res != 0) {
-		printk(KERN_DEBUG "%s failed 2 (%d)\n", __func__, res);
-		return;
-	}
-
-	mdelay(10);
-
-	/* Setup Genesis mode */
-	res = pcmcia_write_config_byte(hw_priv->link, CISREG_CCSR, hcr);
-	if (res != 0) {
-		printk(KERN_DEBUG "%s failed 3 (%d)\n", __func__, res);
-		return;
-	}
-	mdelay(10);
-
-	res = pcmcia_write_config_byte(hw_priv->link, CISREG_COR,
-				old_cor & ~COR_SOFT_RESET);
-	if (res != 0) {
-		printk(KERN_DEBUG "%s failed 4 (%d)\n", __func__, res);
-		return;
-	}
-
-	mdelay(10);
-}
-
-
-static struct prism2_helper_functions prism2_pccard_funcs =
-{
-	.card_present	= prism2_pccard_card_present,
-	.cor_sreset	= prism2_pccard_cor_sreset,
-	.genesis_reset	= prism2_pccard_genesis_reset,
-	.hw_type	= HOSTAP_HW_PCCARD,
-};
-
-
-/* allocate local data and register with CardServices
- * initialize dev_link structure, but do not configure the card yet */
-static int hostap_cs_probe(struct pcmcia_device *p_dev)
-{
-	int ret;
-
-	PDEBUG(DEBUG_HW, "%s: setting Vcc=33 (constant)\n", dev_info);
-
-	ret = prism2_config(p_dev);
-	if (ret) {
-		PDEBUG(DEBUG_EXTRA, "prism2_config() failed\n");
-	}
-
-	return ret;
-}
-
-
-static void prism2_detach(struct pcmcia_device *link)
-{
-	PDEBUG(DEBUG_FLOW, "prism2_detach\n");
-
-	prism2_release((u_long)link);
-
-	/* release net devices */
-	if (link->priv) {
-		struct hostap_cs_priv *hw_priv;
-		struct net_device *dev;
-		struct hostap_interface *iface;
-		dev = link->priv;
-		iface = netdev_priv(dev);
-		hw_priv = iface->local->hw_priv;
-		prism2_free_local_data(dev);
-		kfree(hw_priv);
-	}
-}
-
-
-static int prism2_config_check(struct pcmcia_device *p_dev, void *priv_data)
-{
-	if (p_dev->config_index == 0)
-		return -EINVAL;
-
-	return pcmcia_request_io(p_dev);
-}
-
-static int prism2_config(struct pcmcia_device *link)
-{
-	struct net_device *dev;
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int ret;
-	struct hostap_cs_priv *hw_priv;
-	unsigned long flags;
-
-	PDEBUG(DEBUG_FLOW, "prism2_config()\n");
-
-	hw_priv = kzalloc(sizeof(*hw_priv), GFP_KERNEL);
-	if (hw_priv == NULL) {
-		ret = -ENOMEM;
-		goto failed;
-	}
-
-	/* Look for an appropriate configuration table entry in the CIS */
-	link->config_flags |= CONF_AUTO_SET_VPP | CONF_AUTO_AUDIO |
-		CONF_AUTO_CHECK_VCC | CONF_AUTO_SET_IO | CONF_ENABLE_IRQ;
-	if (ignore_cis_vcc)
-		link->config_flags &= ~CONF_AUTO_CHECK_VCC;
-	ret = pcmcia_loop_config(link, prism2_config_check, NULL);
-	if (ret) {
-		if (!ignore_cis_vcc)
-			printk(KERN_ERR "GetNextTuple(): No matching "
-			       "CIS configuration.  Maybe you need the "
-			       "ignore_cis_vcc=1 parameter.\n");
-		goto failed;
-	}
-
-	/* Need to allocate net_device before requesting IRQ handler */
-	dev = prism2_init_local_data(&prism2_pccard_funcs, 0,
-				     &link->dev);
-	if (!dev) {
-		ret = -ENOMEM;
-		goto failed;
-	}
-	link->priv = dev;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-	local->hw_priv = hw_priv;
-	hw_priv->link = link;
-
-	/*
-	 * We enable IRQ here, but IRQ handler will not proceed
-	 * until dev->base_addr is set below. This protect us from
-	 * receive interrupts when driver is not initialized.
-	 */
-	ret = pcmcia_request_irq(link, prism2_interrupt);
-	if (ret)
-		goto failed;
-
-	ret = pcmcia_enable_device(link);
-	if (ret)
-		goto failed;
-
-	spin_lock_irqsave(&local->irq_init_lock, flags);
-	dev->irq = link->irq;
-	dev->base_addr = link->resource[0]->start;
-	spin_unlock_irqrestore(&local->irq_init_lock, flags);
-
-	local->shutdown = 0;
-
-	sandisk_enable_wireless(dev);
-
-	ret = prism2_hw_config(dev, 1);
-	if (!ret)
-		ret = hostap_hw_ready(dev);
-
-	return ret;
-
- failed:
-	kfree(hw_priv);
-	prism2_release((u_long)link);
-	return ret;
-}
-
-
-static void prism2_release(u_long arg)
-{
-	struct pcmcia_device *link = (struct pcmcia_device *)arg;
-
-	PDEBUG(DEBUG_FLOW, "prism2_release\n");
-
-	if (link->priv) {
-		struct net_device *dev = link->priv;
-		struct hostap_interface *iface;
-
-		iface = netdev_priv(dev);
-		prism2_hw_shutdown(dev, 0);
-		iface->local->shutdown = 1;
-	}
-
-	pcmcia_disable_device(link);
-	PDEBUG(DEBUG_FLOW, "release - done\n");
-}
-
-static int hostap_cs_suspend(struct pcmcia_device *link)
-{
-	struct net_device *dev = (struct net_device *) link->priv;
-	int dev_open = 0;
-	struct hostap_interface *iface = NULL;
-
-	if (!dev)
-		return -ENODEV;
-
-	iface = netdev_priv(dev);
-
-	PDEBUG(DEBUG_EXTRA, "%s: CS_EVENT_PM_SUSPEND\n", dev_info);
-	if (iface && iface->local)
-		dev_open = iface->local->num_dev_open > 0;
-	if (dev_open) {
-		netif_stop_queue(dev);
-		netif_device_detach(dev);
-	}
-	prism2_suspend(dev);
-
-	return 0;
-}
-
-static int hostap_cs_resume(struct pcmcia_device *link)
-{
-	struct net_device *dev = (struct net_device *) link->priv;
-	int dev_open = 0;
-	struct hostap_interface *iface = NULL;
-
-	if (!dev)
-		return -ENODEV;
-
-	iface = netdev_priv(dev);
-
-	PDEBUG(DEBUG_EXTRA, "%s: CS_EVENT_PM_RESUME\n", dev_info);
-
-	if (iface && iface->local)
-		dev_open = iface->local->num_dev_open > 0;
-
-	prism2_hw_shutdown(dev, 1);
-	prism2_hw_config(dev, dev_open ? 0 : 1);
-	if (dev_open) {
-		netif_device_attach(dev);
-		netif_start_queue(dev);
-	}
-
-	return 0;
-}
-
-static const struct pcmcia_device_id hostap_cs_ids[] = {
-	PCMCIA_DEVICE_MANF_CARD(0x000b, 0x7100),
-	PCMCIA_DEVICE_MANF_CARD(0x000b, 0x7300),
-	PCMCIA_DEVICE_MANF_CARD(0x0101, 0x0777),
-	PCMCIA_DEVICE_MANF_CARD(0x0126, 0x8000),
-	PCMCIA_DEVICE_MANF_CARD(0x0138, 0x0002),
-	PCMCIA_DEVICE_MANF_CARD(0x01bf, 0x3301),
-	PCMCIA_DEVICE_MANF_CARD(0x0250, 0x0002),
-	PCMCIA_DEVICE_MANF_CARD(0x026f, 0x030b),
-	PCMCIA_DEVICE_MANF_CARD(0x0274, 0x1612),
-	PCMCIA_DEVICE_MANF_CARD(0x0274, 0x1613),
-	PCMCIA_DEVICE_MANF_CARD(0x028a, 0x0002),
-	PCMCIA_DEVICE_MANF_CARD(0x02aa, 0x0002),
-	PCMCIA_DEVICE_MANF_CARD(0x02d2, 0x0001),
-	PCMCIA_DEVICE_MANF_CARD(0x50c2, 0x0001),
-	PCMCIA_DEVICE_MANF_CARD(0x50c2, 0x7300),
-/*	PCMCIA_DEVICE_MANF_CARD(0xc00f, 0x0000),    conflict with pcnet_cs */
-	PCMCIA_DEVICE_MANF_CARD(0xc250, 0x0002),
-	PCMCIA_DEVICE_MANF_CARD(0xd601, 0x0002),
-	PCMCIA_DEVICE_MANF_CARD(0xd601, 0x0005),
-	PCMCIA_DEVICE_MANF_CARD(0xd601, 0x0010),
-	PCMCIA_DEVICE_MANF_CARD(0x0126, 0x0002),
-	PCMCIA_DEVICE_MANF_CARD_PROD_ID1(0xd601, 0x0005, "ADLINK 345 CF",
-					 0x2d858104),
-	PCMCIA_DEVICE_MANF_CARD_PROD_ID1(0x0156, 0x0002, "INTERSIL",
-					 0x74c5e40d),
-	PCMCIA_DEVICE_MANF_CARD_PROD_ID1(0x0156, 0x0002, "Intersil",
-					 0x4b801a17),
-	PCMCIA_DEVICE_MANF_CARD_PROD_ID3(0x0156, 0x0002, "Version 01.02",
-					 0x4b74baa0),
-	PCMCIA_MFC_DEVICE_PROD_ID12(0, "SanDisk", "ConnectPlus",
-				    0x7a954bd9, 0x74be00c6),
-	PCMCIA_DEVICE_PROD_ID123(
-		"Addtron", "AWP-100 Wireless PCMCIA", "Version 01.02",
-		0xe6ec52ce, 0x08649af2, 0x4b74baa0),
-	PCMCIA_DEVICE_PROD_ID123(
-		"Canon", "Wireless LAN CF Card K30225", "Version 01.00",
-		0x96ef6fe2, 0x263fcbab, 0xa57adb8c),
-	PCMCIA_DEVICE_PROD_ID123(
-		"D", "Link DWL-650 11Mbps WLAN Card", "Version 01.02",
-		0x71b18589, 0xb6f1b0ab, 0x4b74baa0),
-	PCMCIA_DEVICE_PROD_ID123(
-		"Instant Wireless ", " Network PC CARD", "Version 01.02",
-		0x11d901af, 0x6e9bd926, 0x4b74baa0),
-	PCMCIA_DEVICE_PROD_ID123(
-		"SMC", "SMC2632W", "Version 01.02",
-		0xc4f8b18b, 0x474a1f2a, 0x4b74baa0),
-	PCMCIA_DEVICE_PROD_ID12("BUFFALO", "WLI-CF-S11G", 
-				0x2decece3, 0x82067c18),
-	PCMCIA_DEVICE_PROD_ID12("Compaq", "WL200_11Mbps_Wireless_PCI_Card",
-				0x54f7c49c, 0x15a75e5b),
-	PCMCIA_DEVICE_PROD_ID12("INTERSIL", "HFA384x/IEEE",
-				0x74c5e40d, 0xdb472a18),
-	PCMCIA_DEVICE_PROD_ID12("Linksys", "Wireless CompactFlash Card",
-				0x0733cc81, 0x0c52f395),
-	PCMCIA_DEVICE_PROD_ID12(
-		"ZoomAir 11Mbps High", "Rate wireless Networking",
-		0x273fe3db, 0x32a1eaee),
-	PCMCIA_DEVICE_PROD_ID12("NETGEAR MA401 Wireless PC", "Card",
-		0xa37434e9, 0x9762e8f1),
-	PCMCIA_DEVICE_PROD_ID123(
-		"Pretec", "CompactWLAN Card 802.11b", "2.5",
-		0x1cadd3e5, 0xe697636c, 0x7a5bfcf1),
-	PCMCIA_DEVICE_PROD_ID123(
-		"U.S. Robotics", "IEEE 802.11b PC-CARD", "Version 01.02",
-		0xc7b8df9d, 0x1700d087, 0x4b74baa0),
-	PCMCIA_DEVICE_PROD_ID123(
-		"Allied Telesyn", "AT-WCL452 Wireless PCMCIA Radio",
-		"Ver. 1.00",
-		0x5cd01705, 0x4271660f, 0x9d08ee12),
-	PCMCIA_DEVICE_PROD_ID123(
-		"Wireless LAN" , "11Mbps PC Card", "Version 01.02",
-		0x4b8870ff, 0x70e946d1, 0x4b74baa0),
-	PCMCIA_DEVICE_PROD_ID3("HFA3863", 0x355cb092),
-	PCMCIA_DEVICE_PROD_ID3("ISL37100P", 0x630d52b2),
-	PCMCIA_DEVICE_PROD_ID3("ISL37101P-10", 0xdd97a26b),
-	PCMCIA_DEVICE_PROD_ID3("ISL37300P", 0xc9049a39),
-	PCMCIA_DEVICE_NULL
-};
-MODULE_DEVICE_TABLE(pcmcia, hostap_cs_ids);
-
-
-static struct pcmcia_driver hostap_driver = {
-	.name		= "hostap_cs",
-	.probe		= hostap_cs_probe,
-	.remove		= prism2_detach,
-	.owner		= THIS_MODULE,
-	.id_table	= hostap_cs_ids,
-	.suspend	= hostap_cs_suspend,
-	.resume		= hostap_cs_resume,
-};
-module_pcmcia_driver(hostap_driver);
diff --git a/drivers/net/wireless/intersil/hostap/hostap_download.c b/drivers/net/wireless/intersil/hostap/hostap_download.c
deleted file mode 100644
index 5e5bada28b5b..000000000000
--- a/drivers/net/wireless/intersil/hostap/hostap_download.c
+++ /dev/null
@@ -1,810 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-static int prism2_enable_aux_port(struct net_device *dev, int enable)
-{
-	u16 val, reg;
-	int i, tries;
-	unsigned long flags;
-	struct hostap_interface *iface;
-	local_info_t *local;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (local->no_pri) {
-		if (enable) {
-			PDEBUG(DEBUG_EXTRA2, "%s: no PRI f/w - assuming Aux "
-			       "port is already enabled\n", dev->name);
-		}
-		return 0;
-	}
-
-	spin_lock_irqsave(&local->cmdlock, flags);
-
-	/* wait until busy bit is clear */
-	tries = HFA384X_CMD_BUSY_TIMEOUT;
-	while (HFA384X_INW(HFA384X_CMD_OFF) & HFA384X_CMD_BUSY && tries > 0) {
-		tries--;
-		udelay(1);
-	}
-	if (tries == 0) {
-		reg = HFA384X_INW(HFA384X_CMD_OFF);
-		spin_unlock_irqrestore(&local->cmdlock, flags);
-		printk("%s: prism2_enable_aux_port - timeout - reg=0x%04x\n",
-		       dev->name, reg);
-		return -ETIMEDOUT;
-	}
-
-	val = HFA384X_INW(HFA384X_CONTROL_OFF);
-
-	if (enable) {
-		HFA384X_OUTW(HFA384X_AUX_MAGIC0, HFA384X_PARAM0_OFF);
-		HFA384X_OUTW(HFA384X_AUX_MAGIC1, HFA384X_PARAM1_OFF);
-		HFA384X_OUTW(HFA384X_AUX_MAGIC2, HFA384X_PARAM2_OFF);
-
-		if ((val & HFA384X_AUX_PORT_MASK) != HFA384X_AUX_PORT_DISABLED)
-			printk("prism2_enable_aux_port: was not disabled!?\n");
-		val &= ~HFA384X_AUX_PORT_MASK;
-		val |= HFA384X_AUX_PORT_ENABLE;
-	} else {
-		HFA384X_OUTW(0, HFA384X_PARAM0_OFF);
-		HFA384X_OUTW(0, HFA384X_PARAM1_OFF);
-		HFA384X_OUTW(0, HFA384X_PARAM2_OFF);
-
-		if ((val & HFA384X_AUX_PORT_MASK) != HFA384X_AUX_PORT_ENABLED)
-			printk("prism2_enable_aux_port: was not enabled!?\n");
-		val &= ~HFA384X_AUX_PORT_MASK;
-		val |= HFA384X_AUX_PORT_DISABLE;
-	}
-	HFA384X_OUTW(val, HFA384X_CONTROL_OFF);
-
-	udelay(5);
-
-	i = 10000;
-	while (i > 0) {
-		val = HFA384X_INW(HFA384X_CONTROL_OFF);
-		val &= HFA384X_AUX_PORT_MASK;
-
-		if ((enable && val == HFA384X_AUX_PORT_ENABLED) ||
-		    (!enable && val == HFA384X_AUX_PORT_DISABLED))
-			break;
-
-		udelay(10);
-		i--;
-	}
-
-	spin_unlock_irqrestore(&local->cmdlock, flags);
-
-	if (i == 0) {
-		printk("prism2_enable_aux_port(%d) timed out\n",
-		       enable);
-		return -ETIMEDOUT;
-	}
-
-	return 0;
-}
-
-
-static int hfa384x_from_aux(struct net_device *dev, unsigned int addr, int len,
-			    void *buf)
-{
-	u16 page, offset;
-	if (addr & 1 || len & 1)
-		return -1;
-
-	page = addr >> 7;
-	offset = addr & 0x7f;
-
-	HFA384X_OUTW(page, HFA384X_AUXPAGE_OFF);
-	HFA384X_OUTW(offset, HFA384X_AUXOFFSET_OFF);
-
-	udelay(5);
-
-#ifdef PRISM2_PCI
-	{
-		__le16 *pos = (__le16 *) buf;
-		while (len > 0) {
-			*pos++ = HFA384X_INW_DATA(HFA384X_AUXDATA_OFF);
-			len -= 2;
-		}
-	}
-#else /* PRISM2_PCI */
-	HFA384X_INSW(HFA384X_AUXDATA_OFF, buf, len / 2);
-#endif /* PRISM2_PCI */
-
-	return 0;
-}
-
-
-static int hfa384x_to_aux(struct net_device *dev, unsigned int addr, int len,
-			  void *buf)
-{
-	u16 page, offset;
-	if (addr & 1 || len & 1)
-		return -1;
-
-	page = addr >> 7;
-	offset = addr & 0x7f;
-
-	HFA384X_OUTW(page, HFA384X_AUXPAGE_OFF);
-	HFA384X_OUTW(offset, HFA384X_AUXOFFSET_OFF);
-
-	udelay(5);
-
-#ifdef PRISM2_PCI
-	{
-		__le16 *pos = (__le16 *) buf;
-		while (len > 0) {
-			HFA384X_OUTW_DATA(*pos++, HFA384X_AUXDATA_OFF);
-			len -= 2;
-		}
-	}
-#else /* PRISM2_PCI */
-	HFA384X_OUTSW(HFA384X_AUXDATA_OFF, buf, len / 2);
-#endif /* PRISM2_PCI */
-
-	return 0;
-}
-
-
-static int prism2_pda_ok(u8 *buf)
-{
-	__le16 *pda = (__le16 *) buf;
-	int pos;
-	u16 len, pdr;
-
-	if (buf[0] == 0xff && buf[1] == 0x00 && buf[2] == 0xff &&
-	    buf[3] == 0x00)
-		return 0;
-
-	pos = 0;
-	while (pos + 1 < PRISM2_PDA_SIZE / 2) {
-		len = le16_to_cpu(pda[pos]);
-		pdr = le16_to_cpu(pda[pos + 1]);
-		if (len == 0 || pos + len > PRISM2_PDA_SIZE / 2)
-			return 0;
-
-		if (pdr == 0x0000 && len == 2) {
-			/* PDA end found */
-			return 1;
-		}
-
-		pos += len + 1;
-	}
-
-	return 0;
-}
-
-
-#define prism2_download_aux_dump_npages 65536
-
-struct prism2_download_aux_dump {
-	local_info_t *local;
-	u16 page[0x80];
-};
-
-static int prism2_download_aux_dump_proc_show(struct seq_file *m, void *v)
-{
-	struct prism2_download_aux_dump *ctx = m->private;
-
-	hfa384x_from_aux(ctx->local->dev, (unsigned long)v - 1, 0x80, ctx->page);
-	seq_write(m, ctx->page, 0x80);
-	return 0;
-}
-
-static void *prism2_download_aux_dump_proc_start(struct seq_file *m, loff_t *_pos)
-{
-	struct prism2_download_aux_dump *ctx = m->private;
-	prism2_enable_aux_port(ctx->local->dev, 1);
-	if (*_pos >= prism2_download_aux_dump_npages)
-		return NULL;
-	return (void *)((unsigned long)*_pos + 1);
-}
-
-static void *prism2_download_aux_dump_proc_next(struct seq_file *m, void *v, loff_t *_pos)
-{
-	++*_pos;
-	if (*_pos >= prism2_download_aux_dump_npages)
-		return NULL;
-	return (void *)((unsigned long)*_pos + 1);
-}
-
-static void prism2_download_aux_dump_proc_stop(struct seq_file *m, void *v)
-{
-	struct prism2_download_aux_dump *ctx = m->private;
-	prism2_enable_aux_port(ctx->local->dev, 0);
-}
-
-static const struct seq_operations prism2_download_aux_dump_proc_seqops = {
-	.start	= prism2_download_aux_dump_proc_start,
-	.next	= prism2_download_aux_dump_proc_next,
-	.stop	= prism2_download_aux_dump_proc_stop,
-	.show	= prism2_download_aux_dump_proc_show,
-};
-
-static int prism2_download_aux_dump_proc_open(struct inode *inode, struct file *file)
-{
-	int ret = seq_open_private(file, &prism2_download_aux_dump_proc_seqops,
-				   sizeof(struct prism2_download_aux_dump));
-	if (ret == 0) {
-		struct seq_file *m = file->private_data;
-		m->private = pde_data(inode);
-	}
-	return ret;
-}
-
-static const struct proc_ops prism2_download_aux_dump_proc_ops = {
-	.proc_open		= prism2_download_aux_dump_proc_open,
-	.proc_read		= seq_read,
-	.proc_lseek		= seq_lseek,
-	.proc_release		= seq_release_private,
-};
-
-
-static u8 * prism2_read_pda(struct net_device *dev)
-{
-	u8 *buf;
-	int res, i, found = 0;
-#define NUM_PDA_ADDRS 4
-	unsigned int pda_addr[NUM_PDA_ADDRS] = {
-		0x7f0000 /* others than HFA3841 */,
-		0x3f0000 /* HFA3841 */,
-		0x390000 /* apparently used in older cards */,
-		0x7f0002 /* Intel PRO/Wireless 2011B (PCI) */,
-	};
-
-	buf = kmalloc(PRISM2_PDA_SIZE, GFP_KERNEL);
-	if (buf == NULL)
-		return NULL;
-
-	/* Note: wlan card should be in initial state (just after init cmd)
-	 * and no other operations should be performed concurrently. */
-
-	prism2_enable_aux_port(dev, 1);
-
-	for (i = 0; i < NUM_PDA_ADDRS; i++) {
-		PDEBUG(DEBUG_EXTRA2, "%s: trying to read PDA from 0x%08x",
-		       dev->name, pda_addr[i]);
-		res = hfa384x_from_aux(dev, pda_addr[i], PRISM2_PDA_SIZE, buf);
-		if (res)
-			continue;
-		if (res == 0 && prism2_pda_ok(buf)) {
-			PDEBUG2(DEBUG_EXTRA2, ": OK\n");
-			found = 1;
-			break;
-		} else {
-			PDEBUG2(DEBUG_EXTRA2, ": failed\n");
-		}
-	}
-
-	prism2_enable_aux_port(dev, 0);
-
-	if (!found) {
-		printk(KERN_DEBUG "%s: valid PDA not found\n", dev->name);
-		kfree(buf);
-		buf = NULL;
-	}
-
-	return buf;
-}
-
-
-static int prism2_download_volatile(local_info_t *local,
-				    struct prism2_download_data *param)
-{
-	struct net_device *dev = local->dev;
-	int ret = 0, i;
-	u16 param0, param1;
-
-	if (local->hw_downloading) {
-		printk(KERN_WARNING "%s: Already downloading - aborting new "
-		       "request\n", dev->name);
-		return -1;
-	}
-
-	local->hw_downloading = 1;
-	if (local->pri_only) {
-		hfa384x_disable_interrupts(dev);
-	} else {
-		prism2_hw_shutdown(dev, 0);
-
-		if (prism2_hw_init(dev, 0)) {
-			printk(KERN_WARNING "%s: Could not initialize card for"
-			       " download\n", dev->name);
-			ret = -1;
-			goto out;
-		}
-	}
-
-	if (prism2_enable_aux_port(dev, 1)) {
-		printk(KERN_WARNING "%s: Could not enable AUX port\n",
-		       dev->name);
-		ret = -1;
-		goto out;
-	}
-
-	param0 = param->start_addr & 0xffff;
-	param1 = param->start_addr >> 16;
-
-	HFA384X_OUTW(0, HFA384X_PARAM2_OFF);
-	HFA384X_OUTW(param1, HFA384X_PARAM1_OFF);
-	if (hfa384x_cmd_wait(dev, HFA384X_CMDCODE_DOWNLOAD |
-			     (HFA384X_PROGMODE_ENABLE_VOLATILE << 8),
-			     param0)) {
-		printk(KERN_WARNING "%s: Download command execution failed\n",
-		       dev->name);
-		ret = -1;
-		goto out;
-	}
-
-	for (i = 0; i < param->num_areas; i++) {
-		PDEBUG(DEBUG_EXTRA2, "%s: Writing %d bytes at 0x%08x\n",
-		       dev->name, param->data[i].len, param->data[i].addr);
-		if (hfa384x_to_aux(dev, param->data[i].addr,
-				   param->data[i].len, param->data[i].data)) {
-			printk(KERN_WARNING "%s: RAM download at 0x%08x "
-			       "(len=%d) failed\n", dev->name,
-			       param->data[i].addr, param->data[i].len);
-			ret = -1;
-			goto out;
-		}
-	}
-
-	HFA384X_OUTW(param1, HFA384X_PARAM1_OFF);
-	HFA384X_OUTW(0, HFA384X_PARAM2_OFF);
-	if (hfa384x_cmd_no_wait(dev, HFA384X_CMDCODE_DOWNLOAD |
-				(HFA384X_PROGMODE_DISABLE << 8), param0)) {
-		printk(KERN_WARNING "%s: Download command execution failed\n",
-		       dev->name);
-		ret = -1;
-		goto out;
-	}
-	/* ProgMode disable causes the hardware to restart itself from the
-	 * given starting address. Give hw some time and ACK command just in
-	 * case restart did not happen. */
-	mdelay(5);
-	HFA384X_OUTW(HFA384X_EV_CMD, HFA384X_EVACK_OFF);
-
-	if (prism2_enable_aux_port(dev, 0)) {
-		printk(KERN_DEBUG "%s: Disabling AUX port failed\n",
-		       dev->name);
-		/* continue anyway.. restart should have taken care of this */
-	}
-
-	mdelay(5);
-	local->hw_downloading = 0;
-	if (prism2_hw_config(dev, 2)) {
-		printk(KERN_WARNING "%s: Card configuration after RAM "
-		       "download failed\n", dev->name);
-		ret = -1;
-		goto out;
-	}
-
- out:
-	local->hw_downloading = 0;
-	return ret;
-}
-
-
-static int prism2_enable_genesis(local_info_t *local, int hcr)
-{
-	struct net_device *dev = local->dev;
-	u8 initseq[4] = { 0x00, 0xe1, 0xa1, 0xff };
-	u8 readbuf[4];
-
-	printk(KERN_DEBUG "%s: test Genesis mode with HCR 0x%02x\n",
-	       dev->name, hcr);
-	local->func->cor_sreset(local);
-	hfa384x_to_aux(dev, 0x7e0038, sizeof(initseq), initseq);
-	local->func->genesis_reset(local, hcr);
-
-	/* Readback test */
-	hfa384x_from_aux(dev, 0x7e0038, sizeof(readbuf), readbuf);
-	hfa384x_to_aux(dev, 0x7e0038, sizeof(initseq), initseq);
-	hfa384x_from_aux(dev, 0x7e0038, sizeof(readbuf), readbuf);
-
-	if (memcmp(initseq, readbuf, sizeof(initseq)) == 0) {
-		printk(KERN_DEBUG "Readback test succeeded, HCR 0x%02x\n",
-		       hcr);
-		return 0;
-	} else {
-		printk(KERN_DEBUG "Readback test failed, HCR 0x%02x write %4ph read %4ph\n",
-		       hcr, initseq, readbuf);
-		return 1;
-	}
-}
-
-
-static int prism2_get_ram_size(local_info_t *local)
-{
-	int ret;
-
-	/* Try to enable genesis mode; 0x1F for x8 SRAM or 0x0F for x16 SRAM */
-	if (prism2_enable_genesis(local, 0x1f) == 0)
-		ret = 8;
-	else if (prism2_enable_genesis(local, 0x0f) == 0)
-		ret = 16;
-	else
-		ret = -1;
-
-	/* Disable genesis mode */
-	local->func->genesis_reset(local, ret == 16 ? 0x07 : 0x17);
-
-	return ret;
-}
-
-
-static int prism2_download_genesis(local_info_t *local,
-				   struct prism2_download_data *param)
-{
-	struct net_device *dev = local->dev;
-	int ram16 = 0, i;
-	int ret = 0;
-
-	if (local->hw_downloading) {
-		printk(KERN_WARNING "%s: Already downloading - aborting new "
-		       "request\n", dev->name);
-		return -EBUSY;
-	}
-
-	if (!local->func->genesis_reset || !local->func->cor_sreset) {
-		printk(KERN_INFO "%s: Genesis mode downloading not supported "
-		       "with this hwmodel\n", dev->name);
-		return -EOPNOTSUPP;
-	}
-
-	local->hw_downloading = 1;
-
-	if (prism2_enable_aux_port(dev, 1)) {
-		printk(KERN_DEBUG "%s: failed to enable AUX port\n",
-		       dev->name);
-		ret = -EIO;
-		goto out;
-	}
-
-	if (local->sram_type == -1) {
-		/* 0x1F for x8 SRAM or 0x0F for x16 SRAM */
-		if (prism2_enable_genesis(local, 0x1f) == 0) {
-			ram16 = 0;
-			PDEBUG(DEBUG_EXTRA2, "%s: Genesis mode OK using x8 "
-			       "SRAM\n", dev->name);
-		} else if (prism2_enable_genesis(local, 0x0f) == 0) {
-			ram16 = 1;
-			PDEBUG(DEBUG_EXTRA2, "%s: Genesis mode OK using x16 "
-			       "SRAM\n", dev->name);
-		} else {
-			printk(KERN_DEBUG "%s: Could not initiate genesis "
-			       "mode\n", dev->name);
-			ret = -EIO;
-			goto out;
-		}
-	} else {
-		if (prism2_enable_genesis(local, local->sram_type == 8 ?
-					  0x1f : 0x0f)) {
-			printk(KERN_DEBUG "%s: Failed to set Genesis "
-			       "mode (sram_type=%d)\n", dev->name,
-			       local->sram_type);
-			ret = -EIO;
-			goto out;
-		}
-		ram16 = local->sram_type != 8;
-	}
-
-	for (i = 0; i < param->num_areas; i++) {
-		PDEBUG(DEBUG_EXTRA2, "%s: Writing %d bytes at 0x%08x\n",
-		       dev->name, param->data[i].len, param->data[i].addr);
-		if (hfa384x_to_aux(dev, param->data[i].addr,
-				   param->data[i].len, param->data[i].data)) {
-			printk(KERN_WARNING "%s: RAM download at 0x%08x "
-			       "(len=%d) failed\n", dev->name,
-			       param->data[i].addr, param->data[i].len);
-			ret = -EIO;
-			goto out;
-		}
-	}
-
-	PDEBUG(DEBUG_EXTRA2, "Disable genesis mode\n");
-	local->func->genesis_reset(local, ram16 ? 0x07 : 0x17);
-	if (prism2_enable_aux_port(dev, 0)) {
-		printk(KERN_DEBUG "%s: Failed to disable AUX port\n",
-		       dev->name);
-	}
-
-	mdelay(5);
-	local->hw_downloading = 0;
-
-	PDEBUG(DEBUG_EXTRA2, "Trying to initialize card\n");
-	/*
-	 * Make sure the INIT command does not generate a command completion
-	 * event by disabling interrupts.
-	 */
-	hfa384x_disable_interrupts(dev);
-	if (prism2_hw_init(dev, 1)) {
-		printk(KERN_DEBUG "%s: Initialization after genesis mode "
-		       "download failed\n", dev->name);
-		ret = -EIO;
-		goto out;
-	}
-
-	PDEBUG(DEBUG_EXTRA2, "Card initialized - running PRI only\n");
-	if (prism2_hw_init2(dev, 1)) {
-		printk(KERN_DEBUG "%s: Initialization(2) after genesis mode "
-		       "download failed\n", dev->name);
-		ret = -EIO;
-		goto out;
-	}
-
- out:
-	local->hw_downloading = 0;
-	return ret;
-}
-
-
-#ifdef PRISM2_NON_VOLATILE_DOWNLOAD
-/* Note! Non-volatile downloading functionality has not yet been tested
- * thoroughly and it may corrupt flash image and effectively kill the card that
- * is being updated. You have been warned. */
-
-static inline int prism2_download_block(struct net_device *dev,
-					u32 addr, u8 *data,
-					u32 bufaddr, int rest_len)
-{
-	u16 param0, param1;
-	int block_len;
-
-	block_len = rest_len < 4096 ? rest_len : 4096;
-
-	param0 = addr & 0xffff;
-	param1 = addr >> 16;
-
-	HFA384X_OUTW(block_len, HFA384X_PARAM2_OFF);
-	HFA384X_OUTW(param1, HFA384X_PARAM1_OFF);
-
-	if (hfa384x_cmd_wait(dev, HFA384X_CMDCODE_DOWNLOAD |
-			     (HFA384X_PROGMODE_ENABLE_NON_VOLATILE << 8),
-			     param0)) {
-		printk(KERN_WARNING "%s: Flash download command execution "
-		       "failed\n", dev->name);
-		return -1;
-	}
-
-	if (hfa384x_to_aux(dev, bufaddr, block_len, data)) {
-		printk(KERN_WARNING "%s: flash download at 0x%08x "
-		       "(len=%d) failed\n", dev->name, addr, block_len);
-		return -1;
-	}
-
-	HFA384X_OUTW(0, HFA384X_PARAM2_OFF);
-	HFA384X_OUTW(0, HFA384X_PARAM1_OFF);
-	if (hfa384x_cmd_wait(dev, HFA384X_CMDCODE_DOWNLOAD |
-			     (HFA384X_PROGMODE_PROGRAM_NON_VOLATILE << 8),
-			     0)) {
-		printk(KERN_WARNING "%s: Flash write command execution "
-		       "failed\n", dev->name);
-		return -1;
-	}
-
-	return block_len;
-}
-
-
-static int prism2_download_nonvolatile(local_info_t *local,
-				       struct prism2_download_data *dl)
-{
-	struct net_device *dev = local->dev;
-	int ret = 0, i;
-	struct {
-		__le16 page;
-		__le16 offset;
-		__le16 len;
-	} dlbuffer;
-	u32 bufaddr;
-
-	if (local->hw_downloading) {
-		printk(KERN_WARNING "%s: Already downloading - aborting new "
-		       "request\n", dev->name);
-		return -1;
-	}
-
-	ret = local->func->get_rid(dev, HFA384X_RID_DOWNLOADBUFFER,
-				   &dlbuffer, 6, 0);
-
-	if (ret < 0) {
-		printk(KERN_WARNING "%s: Could not read download buffer "
-		       "parameters\n", dev->name);
-		goto out;
-	}
-
-	printk(KERN_DEBUG "Download buffer: %d bytes at 0x%04x:0x%04x\n",
-	       le16_to_cpu(dlbuffer.len),
-	       le16_to_cpu(dlbuffer.page),
-	       le16_to_cpu(dlbuffer.offset));
-
-	bufaddr = (le16_to_cpu(dlbuffer.page) << 7) + le16_to_cpu(dlbuffer.offset);
-
-	local->hw_downloading = 1;
-
-	if (!local->pri_only) {
-		prism2_hw_shutdown(dev, 0);
-
-		if (prism2_hw_init(dev, 0)) {
-			printk(KERN_WARNING "%s: Could not initialize card for"
-			       " download\n", dev->name);
-			ret = -1;
-			goto out;
-		}
-	}
-
-	hfa384x_disable_interrupts(dev);
-
-	if (prism2_enable_aux_port(dev, 1)) {
-		printk(KERN_WARNING "%s: Could not enable AUX port\n",
-		       dev->name);
-		ret = -1;
-		goto out;
-	}
-
-	printk(KERN_DEBUG "%s: starting flash download\n", dev->name);
-	for (i = 0; i < dl->num_areas; i++) {
-		int rest_len = dl->data[i].len;
-		int data_off = 0;
-
-		while (rest_len > 0) {
-			int block_len;
-
-			block_len = prism2_download_block(
-				dev, dl->data[i].addr + data_off,
-				dl->data[i].data + data_off, bufaddr,
-				rest_len);
-
-			if (block_len < 0) {
-				ret = -1;
-				goto out;
-			}
-
-			rest_len -= block_len;
-			data_off += block_len;
-		}
-	}
-
-	HFA384X_OUTW(0, HFA384X_PARAM1_OFF);
-	HFA384X_OUTW(0, HFA384X_PARAM2_OFF);
-	if (hfa384x_cmd_wait(dev, HFA384X_CMDCODE_DOWNLOAD |
-				(HFA384X_PROGMODE_DISABLE << 8), 0)) {
-		printk(KERN_WARNING "%s: Download command execution failed\n",
-		       dev->name);
-		ret = -1;
-		goto out;
-	}
-
-	if (prism2_enable_aux_port(dev, 0)) {
-		printk(KERN_DEBUG "%s: Disabling AUX port failed\n",
-		       dev->name);
-		/* continue anyway.. restart should have taken care of this */
-	}
-
-	mdelay(5);
-
-	local->func->hw_reset(dev);
-	local->hw_downloading = 0;
-	if (prism2_hw_config(dev, 2)) {
-		printk(KERN_WARNING "%s: Card configuration after flash "
-		       "download failed\n", dev->name);
-		ret = -1;
-	} else {
-		printk(KERN_INFO "%s: Card initialized successfully after "
-		       "flash download\n", dev->name);
-	}
-
- out:
-	local->hw_downloading = 0;
-	return ret;
-}
-#endif /* PRISM2_NON_VOLATILE_DOWNLOAD */
-
-
-static void prism2_download_free_data(struct prism2_download_data *dl)
-{
-	int i;
-
-	if (dl == NULL)
-		return;
-
-	for (i = 0; i < dl->num_areas; i++)
-		kfree(dl->data[i].data);
-	kfree(dl);
-}
-
-
-static int prism2_download(local_info_t *local,
-			   struct prism2_download_param *param)
-{
-	int ret = 0;
-	int i;
-	u32 total_len = 0;
-	struct prism2_download_data *dl = NULL;
-
-	printk(KERN_DEBUG "prism2_download: dl_cmd=%d start_addr=0x%08x "
-	       "num_areas=%d\n",
-	       param->dl_cmd, param->start_addr, param->num_areas);
-
-	if (param->num_areas > 100) {
-		ret = -EINVAL;
-		goto out;
-	}
-
-	dl = kzalloc(struct_size(dl, data, param->num_areas), GFP_KERNEL);
-	if (dl == NULL) {
-		ret = -ENOMEM;
-		goto out;
-	}
-	dl->dl_cmd = param->dl_cmd;
-	dl->start_addr = param->start_addr;
-	dl->num_areas = param->num_areas;
-	for (i = 0; i < param->num_areas; i++) {
-		PDEBUG(DEBUG_EXTRA2,
-		       "  area %d: addr=0x%08x len=%d ptr=0x%p\n",
-		       i, param->data[i].addr, param->data[i].len,
-		       param->data[i].ptr);
-
-		dl->data[i].addr = param->data[i].addr;
-		dl->data[i].len = param->data[i].len;
-
-		total_len += param->data[i].len;
-		if (param->data[i].len > PRISM2_MAX_DOWNLOAD_AREA_LEN ||
-		    total_len > PRISM2_MAX_DOWNLOAD_LEN) {
-			ret = -E2BIG;
-			goto out;
-		}
-
-		dl->data[i].data = kmalloc(dl->data[i].len, GFP_KERNEL);
-		if (dl->data[i].data == NULL) {
-			ret = -ENOMEM;
-			goto out;
-		}
-
-		if (copy_from_user(dl->data[i].data, param->data[i].ptr,
-				   param->data[i].len)) {
-			ret = -EFAULT;
-			goto out;
-		}
-	}
-
-	switch (param->dl_cmd) {
-	case PRISM2_DOWNLOAD_VOLATILE:
-	case PRISM2_DOWNLOAD_VOLATILE_PERSISTENT:
-		ret = prism2_download_volatile(local, dl);
-		break;
-	case PRISM2_DOWNLOAD_VOLATILE_GENESIS:
-	case PRISM2_DOWNLOAD_VOLATILE_GENESIS_PERSISTENT:
-		ret = prism2_download_genesis(local, dl);
-		break;
-	case PRISM2_DOWNLOAD_NON_VOLATILE:
-#ifdef PRISM2_NON_VOLATILE_DOWNLOAD
-		ret = prism2_download_nonvolatile(local, dl);
-#else /* PRISM2_NON_VOLATILE_DOWNLOAD */
-		printk(KERN_INFO "%s: non-volatile downloading not enabled\n",
-		       local->dev->name);
-		ret = -EOPNOTSUPP;
-#endif /* PRISM2_NON_VOLATILE_DOWNLOAD */
-		break;
-	default:
-		printk(KERN_DEBUG "%s: unsupported download command %d\n",
-		       local->dev->name, param->dl_cmd);
-		ret = -EINVAL;
-		break;
-	}
-
- out:
-	if (ret == 0 && dl &&
-	    param->dl_cmd == PRISM2_DOWNLOAD_VOLATILE_GENESIS_PERSISTENT) {
-		prism2_download_free_data(local->dl_pri);
-		local->dl_pri = dl;
-	} else if (ret == 0 && dl &&
-		   param->dl_cmd == PRISM2_DOWNLOAD_VOLATILE_PERSISTENT) {
-		prism2_download_free_data(local->dl_sec);
-		local->dl_sec = dl;
-	} else
-		prism2_download_free_data(dl);
-
-	return ret;
-}
diff --git a/drivers/net/wireless/intersil/hostap/hostap_hw.c b/drivers/net/wireless/intersil/hostap/hostap_hw.c
deleted file mode 100644
index b74f4cb5d6d3..000000000000
--- a/drivers/net/wireless/intersil/hostap/hostap_hw.c
+++ /dev/null
@@ -1,3387 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Host AP (software wireless LAN access point) driver for
- * Intersil Prism2/2.5/3.
- *
- * Copyright (c) 2001-2002, SSH Communications Security Corp and Jouni Malinen
- * <j@w1.fi>
- * Copyright (c) 2002-2005, Jouni Malinen <j@w1.fi>
- *
- * FIX:
- * - there is currently no way of associating TX packets to correct wds device
- *   when TX Exc/OK event occurs, so all tx_packets and some
- *   tx_errors/tx_dropped are added to the main netdevice; using sw_support
- *   field in txdesc might be used to fix this (using Alloc event to increment
- *   tx_packets would need some further info in txfid table)
- *
- * Buffer Access Path (BAP) usage:
- *   Prism2 cards have two separate BAPs for accessing the card memory. These
- *   should allow concurrent access to two different frames and the driver
- *   previously used BAP0 for sending data and BAP1 for receiving data.
- *   However, there seems to be number of issues with concurrent access and at
- *   least one know hardware bug in using BAP0 and BAP1 concurrently with PCI
- *   Prism2.5. Therefore, the driver now only uses BAP0 for moving data between
- *   host and card memories. BAP0 accesses are protected with local->baplock
- *   (spin_lock_bh) to prevent concurrent use.
- */
-
-
-
-#include <asm/delay.h>
-#include <linux/uaccess.h>
-
-#include <linux/slab.h>
-#include <linux/netdevice.h>
-#include <linux/etherdevice.h>
-#include <linux/proc_fs.h>
-#include <linux/seq_file.h>
-#include <linux/if_arp.h>
-#include <linux/delay.h>
-#include <linux/random.h>
-#include <linux/wait.h>
-#include <linux/sched/signal.h>
-#include <linux/rtnetlink.h>
-#include <linux/wireless.h>
-#include <net/iw_handler.h>
-#include <net/lib80211.h>
-#include <asm/irq.h>
-
-#include "hostap_80211.h"
-#include "hostap.h"
-#include "hostap_ap.h"
-
-
-/* #define final_version */
-
-static int mtu = 1500;
-module_param(mtu, int, 0444);
-MODULE_PARM_DESC(mtu, "Maximum transfer unit");
-
-static int channel[MAX_PARM_DEVICES] = { 3, DEF_INTS };
-module_param_array(channel, int, NULL, 0444);
-MODULE_PARM_DESC(channel, "Initial channel");
-
-static char essid[33] = "test";
-module_param_string(essid, essid, sizeof(essid), 0444);
-MODULE_PARM_DESC(essid, "Host AP's ESSID");
-
-static int iw_mode[MAX_PARM_DEVICES] = { IW_MODE_MASTER, DEF_INTS };
-module_param_array(iw_mode, int, NULL, 0444);
-MODULE_PARM_DESC(iw_mode, "Initial operation mode");
-
-static int beacon_int[MAX_PARM_DEVICES] = { 100, DEF_INTS };
-module_param_array(beacon_int, int, NULL, 0444);
-MODULE_PARM_DESC(beacon_int, "Beacon interval (1 = 1024 usec)");
-
-static int dtim_period[MAX_PARM_DEVICES] = { 1, DEF_INTS };
-module_param_array(dtim_period, int, NULL, 0444);
-MODULE_PARM_DESC(dtim_period, "DTIM period");
-
-static char dev_template[16] = "wlan%d";
-module_param_string(dev_template, dev_template, sizeof(dev_template), 0444);
-MODULE_PARM_DESC(dev_template, "Prefix for network device name (default: "
-		 "wlan%d)");
-
-#ifdef final_version
-#define EXTRA_EVENTS_WTERR 0
-#else
-/* check WTERR events (Wait Time-out) in development versions */
-#define EXTRA_EVENTS_WTERR HFA384X_EV_WTERR
-#endif
-
-/* Events that will be using BAP0 */
-#define HFA384X_BAP0_EVENTS \
-	(HFA384X_EV_TXEXC | HFA384X_EV_RX | HFA384X_EV_INFO | HFA384X_EV_TX)
-
-/* event mask, i.e., events that will result in an interrupt */
-#define HFA384X_EVENT_MASK \
-	(HFA384X_BAP0_EVENTS | HFA384X_EV_ALLOC | HFA384X_EV_INFDROP | \
-	HFA384X_EV_CMD | HFA384X_EV_TICK | \
-	EXTRA_EVENTS_WTERR)
-
-/* Default TX control flags: use 802.11 headers and request interrupt for
- * failed transmits. Frames that request ACK callback, will add
- * _TX_OK flag and _ALT_RTRY flag may be used to select different retry policy.
- */
-#define HFA384X_TX_CTRL_FLAGS \
-	(HFA384X_TX_CTRL_802_11 | HFA384X_TX_CTRL_TX_EX)
-
-
-/* ca. 1 usec */
-#define HFA384X_CMD_BUSY_TIMEOUT 5000
-#define HFA384X_BAP_BUSY_TIMEOUT 50000
-
-/* ca. 10 usec */
-#define HFA384X_CMD_COMPL_TIMEOUT 20000
-#define HFA384X_DL_COMPL_TIMEOUT 1000000
-
-/* Wait times for initialization; yield to other processes to avoid busy
- * waiting for long time. */
-#define HFA384X_INIT_TIMEOUT (HZ / 2) /* 500 ms */
-#define HFA384X_ALLOC_COMPL_TIMEOUT (HZ / 20) /* 50 ms */
-
-
-static void prism2_hw_reset(struct net_device *dev);
-static void prism2_check_sta_fw_version(local_info_t *local);
-
-#ifdef PRISM2_DOWNLOAD_SUPPORT
-/* hostap_download.c */
-static const struct proc_ops prism2_download_aux_dump_proc_ops;
-static u8 * prism2_read_pda(struct net_device *dev);
-static int prism2_download(local_info_t *local,
-			   struct prism2_download_param *param);
-static void prism2_download_free_data(struct prism2_download_data *dl);
-static int prism2_download_volatile(local_info_t *local,
-				    struct prism2_download_data *param);
-static int prism2_download_genesis(local_info_t *local,
-				   struct prism2_download_data *param);
-static int prism2_get_ram_size(local_info_t *local);
-#endif /* PRISM2_DOWNLOAD_SUPPORT */
-
-
-
-
-#ifndef final_version
-/* magic value written to SWSUPPORT0 reg. for detecting whether card is still
- * present */
-#define HFA384X_MAGIC 0x8A32
-#endif
-
-static void hfa384x_read_regs(struct net_device *dev,
-			      struct hfa384x_regs *regs)
-{
-	regs->cmd = HFA384X_INW(HFA384X_CMD_OFF);
-	regs->evstat = HFA384X_INW(HFA384X_EVSTAT_OFF);
-	regs->offset0 = HFA384X_INW(HFA384X_OFFSET0_OFF);
-	regs->offset1 = HFA384X_INW(HFA384X_OFFSET1_OFF);
-	regs->swsupport0 = HFA384X_INW(HFA384X_SWSUPPORT0_OFF);
-}
-
-
-/**
- * __hostap_cmd_queue_free - Free Prism2 command queue entry (private)
- * @local: pointer to private Host AP driver data
- * @entry: Prism2 command queue entry to be freed
- * @del_req: request the entry to be removed
- *
- * Internal helper function for freeing Prism2 command queue entries.
- * Caller must have acquired local->cmdlock before calling this function.
- */
-static inline void __hostap_cmd_queue_free(local_info_t *local,
-					   struct hostap_cmd_queue *entry,
-					   int del_req)
-{
-	if (del_req) {
-		entry->del_req = 1;
-		if (!list_empty(&entry->list)) {
-			list_del_init(&entry->list);
-			local->cmd_queue_len--;
-		}
-	}
-
-	if (refcount_dec_and_test(&entry->usecnt) && entry->del_req)
-		kfree(entry);
-}
-
-
-/**
- * hostap_cmd_queue_free - Free Prism2 command queue entry
- * @local: pointer to private Host AP driver data
- * @entry: Prism2 command queue entry to be freed
- * @del_req: request the entry to be removed
- *
- * Free a Prism2 command queue entry.
- */
-static inline void hostap_cmd_queue_free(local_info_t *local,
-					 struct hostap_cmd_queue *entry,
-					 int del_req)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&local->cmdlock, flags);
-	__hostap_cmd_queue_free(local, entry, del_req);
-	spin_unlock_irqrestore(&local->cmdlock, flags);
-}
-
-
-/**
- * prism2_clear_cmd_queue - Free all pending Prism2 command queue entries
- * @local: pointer to private Host AP driver data
- */
-static void prism2_clear_cmd_queue(local_info_t *local)
-{
-	struct list_head *ptr, *n;
-	unsigned long flags;
-	struct hostap_cmd_queue *entry;
-
-	spin_lock_irqsave(&local->cmdlock, flags);
-	list_for_each_safe(ptr, n, &local->cmd_queue) {
-		entry = list_entry(ptr, struct hostap_cmd_queue, list);
-		refcount_inc(&entry->usecnt);
-		printk(KERN_DEBUG "%s: removed pending cmd_queue entry "
-		       "(type=%d, cmd=0x%04x, param0=0x%04x)\n",
-		       local->dev->name, entry->type, entry->cmd,
-		       entry->param0);
-		__hostap_cmd_queue_free(local, entry, 1);
-	}
-	if (local->cmd_queue_len) {
-		/* This should not happen; print debug message and clear
-		 * queue length. */
-		printk(KERN_DEBUG "%s: cmd_queue_len (%d) not zero after "
-		       "flush\n", local->dev->name, local->cmd_queue_len);
-		local->cmd_queue_len = 0;
-	}
-	spin_unlock_irqrestore(&local->cmdlock, flags);
-}
-
-
-/**
- * hfa384x_cmd_issue - Issue a Prism2 command to the hardware
- * @dev: pointer to net_device
- * @entry: Prism2 command queue entry to be issued
- */
-static int hfa384x_cmd_issue(struct net_device *dev,
-				    struct hostap_cmd_queue *entry)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int tries;
-	u16 reg;
-	unsigned long flags;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (local->func->card_present && !local->func->card_present(local))
-		return -ENODEV;
-
-	if (entry->issued) {
-		printk(KERN_DEBUG "%s: driver bug - re-issuing command @%p\n",
-		       dev->name, entry);
-	}
-
-	/* wait until busy bit is clear; this should always be clear since the
-	 * commands are serialized */
-	tries = HFA384X_CMD_BUSY_TIMEOUT;
-	while (HFA384X_INW(HFA384X_CMD_OFF) & HFA384X_CMD_BUSY && tries > 0) {
-		tries--;
-		udelay(1);
-	}
-#ifndef final_version
-	if (tries != HFA384X_CMD_BUSY_TIMEOUT) {
-		prism2_io_debug_error(dev, 1);
-		printk(KERN_DEBUG "%s: hfa384x_cmd_issue: cmd reg was busy "
-		       "for %d usec\n", dev->name,
-		       HFA384X_CMD_BUSY_TIMEOUT - tries);
-	}
-#endif
-	if (tries == 0) {
-		reg = HFA384X_INW(HFA384X_CMD_OFF);
-		prism2_io_debug_error(dev, 2);
-		printk(KERN_DEBUG "%s: hfa384x_cmd_issue - timeout - "
-		       "reg=0x%04x\n", dev->name, reg);
-		return -ETIMEDOUT;
-	}
-
-	/* write command */
-	spin_lock_irqsave(&local->cmdlock, flags);
-	HFA384X_OUTW(entry->param0, HFA384X_PARAM0_OFF);
-	HFA384X_OUTW(entry->param1, HFA384X_PARAM1_OFF);
-	HFA384X_OUTW(entry->cmd, HFA384X_CMD_OFF);
-	entry->issued = 1;
-	spin_unlock_irqrestore(&local->cmdlock, flags);
-
-	return 0;
-}
-
-
-/**
- * hfa384x_cmd - Issue a Prism2 command and wait (sleep) for completion
- * @dev: pointer to net_device
- * @cmd: Prism2 command code (HFA384X_CMD_CODE_*)
- * @param0: value for Param0 register
- * @param1: value for Param1 register (pointer; %NULL if not used)
- * @resp0: pointer for Resp0 data or %NULL if Resp0 is not needed
- *
- * Issue given command (possibly after waiting in command queue) and sleep
- * until the command is completed (or timed out or interrupted). This can be
- * called only from user process context.
- */
-static int hfa384x_cmd(struct net_device *dev, u16 cmd, u16 param0,
-		       u16 *param1, u16 *resp0)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int err, res, issue, issued = 0;
-	unsigned long flags;
-	struct hostap_cmd_queue *entry;
-	DECLARE_WAITQUEUE(wait, current);
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (local->cmd_queue_len >= HOSTAP_CMD_QUEUE_MAX_LEN) {
-		printk(KERN_DEBUG "%s: hfa384x_cmd: cmd_queue full\n",
-		       dev->name);
-		return -1;
-	}
-
-	if (signal_pending(current))
-		return -EINTR;
-
-	entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
-	if (entry == NULL)
-		return -ENOMEM;
-
-	refcount_set(&entry->usecnt, 1);
-	entry->type = CMD_SLEEP;
-	entry->cmd = cmd;
-	entry->param0 = param0;
-	if (param1)
-		entry->param1 = *param1;
-	init_waitqueue_head(&entry->compl);
-
-	/* prepare to wait for command completion event, but do not sleep yet
-	 */
-	add_wait_queue(&entry->compl, &wait);
-	set_current_state(TASK_INTERRUPTIBLE);
-
-	spin_lock_irqsave(&local->cmdlock, flags);
-	issue = list_empty(&local->cmd_queue);
-	if (issue)
-		entry->issuing = 1;
-	list_add_tail(&entry->list, &local->cmd_queue);
-	local->cmd_queue_len++;
-	spin_unlock_irqrestore(&local->cmdlock, flags);
-
-	err = 0;
-	if (!issue)
-		goto wait_completion;
-
-	if (signal_pending(current))
-		err = -EINTR;
-
-	if (!err) {
-		if (hfa384x_cmd_issue(dev, entry))
-			err = -ETIMEDOUT;
-		else
-			issued = 1;
-	}
-
- wait_completion:
-	if (!err && entry->type != CMD_COMPLETED) {
-		/* sleep until command is completed or timed out */
-		res = schedule_timeout(2 * HZ);
-	} else
-		res = -1;
-
-	if (!err && signal_pending(current))
-		err = -EINTR;
-
-	if (err && issued) {
-		/* the command was issued, so a CmdCompl event should occur
-		 * soon; however, there's a pending signal and
-		 * schedule_timeout() would be interrupted; wait a short period
-		 * of time to avoid removing entry from the list before
-		 * CmdCompl event */
-		udelay(300);
-	}
-
-	set_current_state(TASK_RUNNING);
-	remove_wait_queue(&entry->compl, &wait);
-
-	/* If entry->list is still in the list, it must be removed
-	 * first and in this case prism2_cmd_ev() does not yet have
-	 * local reference to it, and the data can be kfree()'d
-	 * here. If the command completion event is still generated,
-	 * it will be assigned to next (possibly) pending command, but
-	 * the driver will reset the card anyway due to timeout
-	 *
-	 * If the entry is not in the list prism2_cmd_ev() has a local
-	 * reference to it, but keeps cmdlock as long as the data is
-	 * needed, so the data can be kfree()'d here. */
-
-	/* FIX: if the entry->list is in the list, it has not been completed
-	 * yet, so removing it here is somewhat wrong.. this could cause
-	 * references to freed memory and next list_del() causing NULL pointer
-	 * dereference.. it would probably be better to leave the entry in the
-	 * list and the list should be emptied during hw reset */
-
-	spin_lock_irqsave(&local->cmdlock, flags);
-	if (!list_empty(&entry->list)) {
-		printk(KERN_DEBUG "%s: hfa384x_cmd: entry still in list? "
-		       "(entry=%p, type=%d, res=%d)\n", dev->name, entry,
-		       entry->type, res);
-		list_del_init(&entry->list);
-		local->cmd_queue_len--;
-	}
-	spin_unlock_irqrestore(&local->cmdlock, flags);
-
-	if (err) {
-		printk(KERN_DEBUG "%s: hfa384x_cmd: interrupted; err=%d\n",
-		       dev->name, err);
-		res = err;
-		goto done;
-	}
-
-	if (entry->type != CMD_COMPLETED) {
-		u16 reg = HFA384X_INW(HFA384X_EVSTAT_OFF);
-		printk(KERN_DEBUG "%s: hfa384x_cmd: command was not "
-		       "completed (res=%d, entry=%p, type=%d, cmd=0x%04x, "
-		       "param0=0x%04x, EVSTAT=%04x INTEN=%04x)\n", dev->name,
-		       res, entry, entry->type, entry->cmd, entry->param0, reg,
-		       HFA384X_INW(HFA384X_INTEN_OFF));
-		if (reg & HFA384X_EV_CMD) {
-			/* Command completion event is pending, but the
-			 * interrupt was not delivered - probably an issue
-			 * with pcmcia-cs configuration. */
-			printk(KERN_WARNING "%s: interrupt delivery does not "
-			       "seem to work\n", dev->name);
-		}
-		prism2_io_debug_error(dev, 3);
-		res = -ETIMEDOUT;
-		goto done;
-	}
-
-	if (resp0 != NULL)
-		*resp0 = entry->resp0;
-#ifndef final_version
-	if (entry->res) {
-		printk(KERN_DEBUG "%s: CMD=0x%04x => res=0x%02x, "
-		       "resp0=0x%04x\n",
-		       dev->name, cmd, entry->res, entry->resp0);
-	}
-#endif /* final_version */
-
-	res = entry->res;
- done:
-	hostap_cmd_queue_free(local, entry, 1);
-	return res;
-}
-
-
-/**
- * hfa384x_cmd_callback - Issue a Prism2 command; callback when completed
- * @dev: pointer to net_device
- * @cmd: Prism2 command code (HFA384X_CMD_CODE_*)
- * @param0: value for Param0 register
- * @callback: command completion callback function (%NULL = no callback)
- * @context: context data to be given to the callback function
- *
- * Issue given command (possibly after waiting in command queue) and use
- * callback function to indicate command completion. This can be called both
- * from user and interrupt context. The callback function will be called in
- * hardware IRQ context. It can be %NULL, when no function is called when
- * command is completed.
- */
-static int hfa384x_cmd_callback(struct net_device *dev, u16 cmd, u16 param0,
-				void (*callback)(struct net_device *dev,
-						 long context, u16 resp0,
-						 u16 status),
-				long context)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int issue, ret;
-	unsigned long flags;
-	struct hostap_cmd_queue *entry;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (local->cmd_queue_len >= HOSTAP_CMD_QUEUE_MAX_LEN + 2) {
-		printk(KERN_DEBUG "%s: hfa384x_cmd: cmd_queue full\n",
-		       dev->name);
-		return -1;
-	}
-
-	entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
-	if (entry == NULL)
-		return -ENOMEM;
-
-	refcount_set(&entry->usecnt, 1);
-	entry->type = CMD_CALLBACK;
-	entry->cmd = cmd;
-	entry->param0 = param0;
-	entry->callback = callback;
-	entry->context = context;
-
-	spin_lock_irqsave(&local->cmdlock, flags);
-	issue = list_empty(&local->cmd_queue);
-	if (issue)
-		entry->issuing = 1;
-	list_add_tail(&entry->list, &local->cmd_queue);
-	local->cmd_queue_len++;
-	spin_unlock_irqrestore(&local->cmdlock, flags);
-
-	if (issue && hfa384x_cmd_issue(dev, entry))
-		ret = -ETIMEDOUT;
-	else
-		ret = 0;
-
-	hostap_cmd_queue_free(local, entry, ret);
-
-	return ret;
-}
-
-
-/**
- * __hfa384x_cmd_no_wait - Issue a Prism2 command (private)
- * @dev: pointer to net_device
- * @cmd: Prism2 command code (HFA384X_CMD_CODE_*)
- * @param0: value for Param0 register
- * @io_debug_num: I/O debug error number
- *
- * Shared helper function for hfa384x_cmd_wait() and hfa384x_cmd_no_wait().
- */
-static int __hfa384x_cmd_no_wait(struct net_device *dev, u16 cmd, u16 param0,
-				 int io_debug_num)
-{
-	int tries;
-	u16 reg;
-
-	/* wait until busy bit is clear; this should always be clear since the
-	 * commands are serialized */
-	tries = HFA384X_CMD_BUSY_TIMEOUT;
-	while (HFA384X_INW(HFA384X_CMD_OFF) & HFA384X_CMD_BUSY && tries > 0) {
-		tries--;
-		udelay(1);
-	}
-	if (tries == 0) {
-		reg = HFA384X_INW(HFA384X_CMD_OFF);
-		prism2_io_debug_error(dev, io_debug_num);
-		printk(KERN_DEBUG "%s: __hfa384x_cmd_no_wait(%d) - timeout - "
-		       "reg=0x%04x\n", dev->name, io_debug_num, reg);
-		return -ETIMEDOUT;
-	}
-
-	/* write command */
-	HFA384X_OUTW(param0, HFA384X_PARAM0_OFF);
-	HFA384X_OUTW(cmd, HFA384X_CMD_OFF);
-
-	return 0;
-}
-
-
-/**
- * hfa384x_cmd_wait - Issue a Prism2 command and busy wait for completion
- * @dev: pointer to net_device
- * @cmd: Prism2 command code (HFA384X_CMD_CODE_*)
- * @param0: value for Param0 register
- */
-static int hfa384x_cmd_wait(struct net_device *dev, u16 cmd, u16 param0)
-{
-	int res, tries;
-	u16 reg;
-
-	res = __hfa384x_cmd_no_wait(dev, cmd, param0, 4);
-	if (res)
-		return res;
-
-        /* wait for command completion */
-	if ((cmd & HFA384X_CMDCODE_MASK) == HFA384X_CMDCODE_DOWNLOAD)
-		tries = HFA384X_DL_COMPL_TIMEOUT;
-	else
-		tries = HFA384X_CMD_COMPL_TIMEOUT;
-
-        while (!(HFA384X_INW(HFA384X_EVSTAT_OFF) & HFA384X_EV_CMD) &&
-               tries > 0) {
-                tries--;
-                udelay(10);
-        }
-        if (tries == 0) {
-                reg = HFA384X_INW(HFA384X_EVSTAT_OFF);
-		prism2_io_debug_error(dev, 5);
-                printk(KERN_DEBUG "%s: hfa384x_cmd_wait - timeout2 - "
-		       "reg=0x%04x\n", dev->name, reg);
-                return -ETIMEDOUT;
-        }
-
-        res = (HFA384X_INW(HFA384X_STATUS_OFF) &
-               (BIT(14) | BIT(13) | BIT(12) | BIT(11) | BIT(10) | BIT(9) |
-                BIT(8))) >> 8;
-#ifndef final_version
-	if (res) {
-		printk(KERN_DEBUG "%s: CMD=0x%04x => res=0x%02x\n",
-		       dev->name, cmd, res);
-	}
-#endif
-
-	HFA384X_OUTW(HFA384X_EV_CMD, HFA384X_EVACK_OFF);
-
-	return res;
-}
-
-
-/**
- * hfa384x_cmd_no_wait - Issue a Prism2 command; do not wait for completion
- * @dev: pointer to net_device
- * @cmd: Prism2 command code (HFA384X_CMD_CODE_*)
- * @param0: value for Param0 register
- */
-static inline int hfa384x_cmd_no_wait(struct net_device *dev, u16 cmd,
-				      u16 param0)
-{
-	return __hfa384x_cmd_no_wait(dev, cmd, param0, 6);
-}
-
-
-/**
- * prism2_cmd_ev - Prism2 command completion event handler
- * @dev: pointer to net_device
- *
- * Interrupt handler for command completion events. Called by the main
- * interrupt handler in hardware IRQ context. Read Resp0 and status registers
- * from the hardware and ACK the event. Depending on the issued command type
- * either wake up the sleeping process that is waiting for command completion
- * or call the callback function. Issue the next command, if one is pending.
- */
-static void prism2_cmd_ev(struct net_device *dev)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	struct hostap_cmd_queue *entry = NULL;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	spin_lock(&local->cmdlock);
-	if (!list_empty(&local->cmd_queue)) {
-		entry = list_entry(local->cmd_queue.next,
-				   struct hostap_cmd_queue, list);
-		refcount_inc(&entry->usecnt);
-		list_del_init(&entry->list);
-		local->cmd_queue_len--;
-
-		if (!entry->issued) {
-			printk(KERN_DEBUG "%s: Command completion event, but "
-			       "cmd not issued\n", dev->name);
-			__hostap_cmd_queue_free(local, entry, 1);
-			entry = NULL;
-		}
-	}
-	spin_unlock(&local->cmdlock);
-
-	if (!entry) {
-		HFA384X_OUTW(HFA384X_EV_CMD, HFA384X_EVACK_OFF);
-		printk(KERN_DEBUG "%s: Command completion event, but no "
-		       "pending commands\n", dev->name);
-		return;
-	}
-
-	entry->resp0 = HFA384X_INW(HFA384X_RESP0_OFF);
-	entry->res = (HFA384X_INW(HFA384X_STATUS_OFF) &
-		      (BIT(14) | BIT(13) | BIT(12) | BIT(11) | BIT(10) |
-		       BIT(9) | BIT(8))) >> 8;
-	HFA384X_OUTW(HFA384X_EV_CMD, HFA384X_EVACK_OFF);
-
-	/* TODO: rest of the CmdEv handling could be moved to tasklet */
-	if (entry->type == CMD_SLEEP) {
-		entry->type = CMD_COMPLETED;
-		wake_up_interruptible(&entry->compl);
-	} else if (entry->type == CMD_CALLBACK) {
-		if (entry->callback)
-			entry->callback(dev, entry->context, entry->resp0,
-					entry->res);
-	} else {
-		printk(KERN_DEBUG "%s: Invalid command completion type %d\n",
-		       dev->name, entry->type);
-	}
-	hostap_cmd_queue_free(local, entry, 1);
-
-	/* issue next command, if pending */
-	entry = NULL;
-	spin_lock(&local->cmdlock);
-	if (!list_empty(&local->cmd_queue)) {
-		entry = list_entry(local->cmd_queue.next,
-				   struct hostap_cmd_queue, list);
-		if (entry->issuing) {
-			/* hfa384x_cmd() has already started issuing this
-			 * command, so do not start here */
-			entry = NULL;
-		}
-		if (entry)
-			refcount_inc(&entry->usecnt);
-	}
-	spin_unlock(&local->cmdlock);
-
-	if (entry) {
-		/* issue next command; if command issuing fails, remove the
-		 * entry from cmd_queue */
-		int res = hfa384x_cmd_issue(dev, entry);
-		spin_lock(&local->cmdlock);
-		__hostap_cmd_queue_free(local, entry, res);
-		spin_unlock(&local->cmdlock);
-	}
-}
-
-
-static int hfa384x_wait_offset(struct net_device *dev, u16 o_off)
-{
-	int tries = HFA384X_BAP_BUSY_TIMEOUT;
-	int res = HFA384X_INW(o_off) & HFA384X_OFFSET_BUSY;
-
-	while (res && tries > 0) {
-		tries--;
-		udelay(1);
-		res = HFA384X_INW(o_off) & HFA384X_OFFSET_BUSY;
-	}
-	return res;
-}
-
-
-/* Offset must be even */
-static int hfa384x_setup_bap(struct net_device *dev, u16 bap, u16 id,
-			     int offset)
-{
-	u16 o_off, s_off;
-	int ret = 0;
-
-	if (offset % 2 || bap > 1)
-		return -EINVAL;
-
-	if (bap == BAP1) {
-		o_off = HFA384X_OFFSET1_OFF;
-		s_off = HFA384X_SELECT1_OFF;
-	} else {
-		o_off = HFA384X_OFFSET0_OFF;
-		s_off = HFA384X_SELECT0_OFF;
-	}
-
-	if (hfa384x_wait_offset(dev, o_off)) {
-		prism2_io_debug_error(dev, 7);
-		printk(KERN_DEBUG "%s: hfa384x_setup_bap - timeout before\n",
-		       dev->name);
-		ret = -ETIMEDOUT;
-		goto out;
-	}
-
-	HFA384X_OUTW(id, s_off);
-	HFA384X_OUTW(offset, o_off);
-
-	if (hfa384x_wait_offset(dev, o_off)) {
-		prism2_io_debug_error(dev, 8);
-		printk(KERN_DEBUG "%s: hfa384x_setup_bap - timeout after\n",
-		       dev->name);
-		ret = -ETIMEDOUT;
-		goto out;
-	}
-#ifndef final_version
-	if (HFA384X_INW(o_off) & HFA384X_OFFSET_ERR) {
-		prism2_io_debug_error(dev, 9);
-		printk(KERN_DEBUG "%s: hfa384x_setup_bap - offset error "
-		       "(%d,0x04%x,%d); reg=0x%04x\n",
-		       dev->name, bap, id, offset, HFA384X_INW(o_off));
-		ret = -EINVAL;
-	}
-#endif
-
- out:
-	return ret;
-}
-
-
-static int hfa384x_get_rid(struct net_device *dev, u16 rid, void *buf, int len,
-			   int exact_len)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int res, rlen = 0;
-	struct hfa384x_rid_hdr rec;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (local->no_pri) {
-		printk(KERN_DEBUG "%s: cannot get RID %04x (len=%d) - no PRI "
-		       "f/w\n", dev->name, rid, len);
-		return -ENOTTY; /* Well.. not really correct, but return
-				 * something unique enough.. */
-	}
-
-	if ((local->func->card_present && !local->func->card_present(local)) ||
-	    local->hw_downloading)
-		return -ENODEV;
-
-	res = mutex_lock_interruptible(&local->rid_bap_mtx);
-	if (res)
-		return res;
-
-	res = hfa384x_cmd(dev, HFA384X_CMDCODE_ACCESS, rid, NULL, NULL);
-	if (res) {
-		printk(KERN_DEBUG "%s: hfa384x_get_rid: CMDCODE_ACCESS failed "
-		       "(res=%d, rid=%04x, len=%d)\n",
-		       dev->name, res, rid, len);
-		mutex_unlock(&local->rid_bap_mtx);
-		return res;
-	}
-
-	spin_lock_bh(&local->baplock);
-
-	res = hfa384x_setup_bap(dev, BAP0, rid, 0);
-	if (res)
-		goto unlock;
-
-	res = hfa384x_from_bap(dev, BAP0, &rec, sizeof(rec));
-	if (res)
-		goto unlock;
-
-	if (le16_to_cpu(rec.len) == 0) {
-		/* RID not available */
-		res = -ENODATA;
-		goto unlock;
-	}
-
-	rlen = (le16_to_cpu(rec.len) - 1) * 2;
-	if (exact_len && rlen != len) {
-		printk(KERN_DEBUG "%s: hfa384x_get_rid - RID len mismatch: "
-		       "rid=0x%04x, len=%d (expected %d)\n",
-		       dev->name, rid, rlen, len);
-		res = -ENODATA;
-	}
-
-	res = hfa384x_from_bap(dev, BAP0, buf, len);
-
-unlock:
-	spin_unlock_bh(&local->baplock);
-	mutex_unlock(&local->rid_bap_mtx);
-
-	if (res) {
-		if (res != -ENODATA)
-			printk(KERN_DEBUG "%s: hfa384x_get_rid (rid=%04x, "
-			       "len=%d) - failed - res=%d\n", dev->name, rid,
-			       len, res);
-		if (res == -ETIMEDOUT)
-			prism2_hw_reset(dev);
-		return res;
-	}
-
-	return rlen;
-}
-
-
-static int hfa384x_set_rid(struct net_device *dev, u16 rid, void *buf, int len)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	struct hfa384x_rid_hdr rec;
-	int res;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (local->no_pri) {
-		printk(KERN_DEBUG "%s: cannot set RID %04x (len=%d) - no PRI "
-		       "f/w\n", dev->name, rid, len);
-		return -ENOTTY; /* Well.. not really correct, but return
-				 * something unique enough.. */
-	}
-
-	if ((local->func->card_present && !local->func->card_present(local)) ||
-	    local->hw_downloading)
-		return -ENODEV;
-
-	rec.rid = cpu_to_le16(rid);
-	/* RID len in words and +1 for rec.rid */
-	rec.len = cpu_to_le16(len / 2 + len % 2 + 1);
-
-	res = mutex_lock_interruptible(&local->rid_bap_mtx);
-	if (res)
-		return res;
-
-	spin_lock_bh(&local->baplock);
-	res = hfa384x_setup_bap(dev, BAP0, rid, 0);
-	if (!res)
-		res = hfa384x_to_bap(dev, BAP0, &rec, sizeof(rec));
-	if (!res)
-		res = hfa384x_to_bap(dev, BAP0, buf, len);
-	spin_unlock_bh(&local->baplock);
-
-	if (res) {
-		printk(KERN_DEBUG "%s: hfa384x_set_rid (rid=%04x, len=%d) - "
-		       "failed - res=%d\n", dev->name, rid, len, res);
-		mutex_unlock(&local->rid_bap_mtx);
-		return res;
-	}
-
-	res = hfa384x_cmd(dev, HFA384X_CMDCODE_ACCESS_WRITE, rid, NULL, NULL);
-	mutex_unlock(&local->rid_bap_mtx);
-
-	if (res) {
-		printk(KERN_DEBUG "%s: hfa384x_set_rid: CMDCODE_ACCESS_WRITE "
-		       "failed (res=%d, rid=%04x, len=%d)\n",
-		       dev->name, res, rid, len);
-
-		if (res == -ETIMEDOUT)
-			prism2_hw_reset(dev);
-	}
-
-	return res;
-}
-
-
-static void hfa384x_disable_interrupts(struct net_device *dev)
-{
-	/* disable interrupts and clear event status */
-	HFA384X_OUTW(0, HFA384X_INTEN_OFF);
-	HFA384X_OUTW(0xffff, HFA384X_EVACK_OFF);
-}
-
-
-static void hfa384x_enable_interrupts(struct net_device *dev)
-{
-	/* ack pending events and enable interrupts from selected events */
-	HFA384X_OUTW(0xffff, HFA384X_EVACK_OFF);
-	HFA384X_OUTW(HFA384X_EVENT_MASK, HFA384X_INTEN_OFF);
-}
-
-
-static void hfa384x_events_no_bap0(struct net_device *dev)
-{
-	HFA384X_OUTW(HFA384X_EVENT_MASK & ~HFA384X_BAP0_EVENTS,
-		     HFA384X_INTEN_OFF);
-}
-
-
-static void hfa384x_events_all(struct net_device *dev)
-{
-	HFA384X_OUTW(HFA384X_EVENT_MASK, HFA384X_INTEN_OFF);
-}
-
-
-static void hfa384x_events_only_cmd(struct net_device *dev)
-{
-	HFA384X_OUTW(HFA384X_EV_CMD, HFA384X_INTEN_OFF);
-}
-
-
-static u16 hfa384x_allocate_fid(struct net_device *dev, int len)
-{
-	u16 fid;
-	unsigned long delay;
-
-	/* FIX: this could be replace with hfa384x_cmd() if the Alloc event
-	 * below would be handled like CmdCompl event (sleep here, wake up from
-	 * interrupt handler */
-	if (hfa384x_cmd_wait(dev, HFA384X_CMDCODE_ALLOC, len)) {
-		printk(KERN_DEBUG "%s: cannot allocate fid, len=%d\n",
-		       dev->name, len);
-		return 0xffff;
-	}
-
-	delay = jiffies + HFA384X_ALLOC_COMPL_TIMEOUT;
-	while (!(HFA384X_INW(HFA384X_EVSTAT_OFF) & HFA384X_EV_ALLOC) &&
-	       time_before(jiffies, delay))
-		yield();
-	if (!(HFA384X_INW(HFA384X_EVSTAT_OFF) & HFA384X_EV_ALLOC)) {
-		printk("%s: fid allocate, len=%d - timeout\n", dev->name, len);
-		return 0xffff;
-	}
-
-	fid = HFA384X_INW(HFA384X_ALLOCFID_OFF);
-	HFA384X_OUTW(HFA384X_EV_ALLOC, HFA384X_EVACK_OFF);
-
-	return fid;
-}
-
-
-static int prism2_reset_port(struct net_device *dev)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int res;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (!local->dev_enabled)
-		return 0;
-
-	res = hfa384x_cmd(dev, HFA384X_CMDCODE_DISABLE, 0,
-			  NULL, NULL);
-	if (res)
-		printk(KERN_DEBUG "%s: reset port failed to disable port\n",
-		       dev->name);
-	else {
-		res = hfa384x_cmd(dev, HFA384X_CMDCODE_ENABLE, 0,
-				  NULL, NULL);
-		if (res)
-			printk(KERN_DEBUG "%s: reset port failed to enable "
-			       "port\n", dev->name);
-	}
-
-	/* It looks like at least some STA firmware versions reset
-	 * fragmentation threshold back to 2346 after enable command. Restore
-	 * the configured value, if it differs from this default. */
-	if (local->fragm_threshold != 2346 &&
-	    hostap_set_word(dev, HFA384X_RID_FRAGMENTATIONTHRESHOLD,
-			    local->fragm_threshold)) {
-		printk(KERN_DEBUG "%s: failed to restore fragmentation "
-		       "threshold (%d) after Port0 enable\n",
-		       dev->name, local->fragm_threshold);
-	}
-
-	/* Some firmwares lose antenna selection settings on reset */
-	(void) hostap_set_antsel(local);
-
-	return res;
-}
-
-
-static int prism2_get_version_info(struct net_device *dev, u16 rid,
-				   const char *txt)
-{
-	struct hfa384x_comp_ident comp;
-	struct hostap_interface *iface;
-	local_info_t *local;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (local->no_pri) {
-		/* PRI f/w not yet available - cannot read RIDs */
-		return -1;
-	}
-	if (hfa384x_get_rid(dev, rid, &comp, sizeof(comp), 1) < 0) {
-		printk(KERN_DEBUG "Could not get RID for component %s\n", txt);
-		return -1;
-	}
-
-	printk(KERN_INFO "%s: %s: id=0x%02x v%d.%d.%d\n", dev->name, txt,
-	       __le16_to_cpu(comp.id), __le16_to_cpu(comp.major),
-	       __le16_to_cpu(comp.minor), __le16_to_cpu(comp.variant));
-	return 0;
-}
-
-
-static int prism2_setup_rids(struct net_device *dev)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	__le16 tmp;
-	int ret = 0;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	hostap_set_word(dev, HFA384X_RID_TICKTIME, 2000);
-
-	if (!local->fw_ap) {
-		u16 tmp1 = hostap_get_porttype(local);
-		ret = hostap_set_word(dev, HFA384X_RID_CNFPORTTYPE, tmp1);
-		if (ret) {
-			printk("%s: Port type setting to %d failed\n",
-			       dev->name, tmp1);
-			goto fail;
-		}
-	}
-
-	/* Setting SSID to empty string seems to kill the card in Host AP mode
-	 */
-	if (local->iw_mode != IW_MODE_MASTER || local->essid[0] != '\0') {
-		ret = hostap_set_string(dev, HFA384X_RID_CNFOWNSSID,
-					local->essid);
-		if (ret) {
-			printk("%s: AP own SSID setting failed\n", dev->name);
-			goto fail;
-		}
-	}
-
-	ret = hostap_set_word(dev, HFA384X_RID_CNFMAXDATALEN,
-			      PRISM2_DATA_MAXLEN);
-	if (ret) {
-		printk("%s: MAC data length setting to %d failed\n",
-		       dev->name, PRISM2_DATA_MAXLEN);
-		goto fail;
-	}
-
-	if (hfa384x_get_rid(dev, HFA384X_RID_CHANNELLIST, &tmp, 2, 1) < 0) {
-		printk("%s: Channel list read failed\n", dev->name);
-		ret = -EINVAL;
-		goto fail;
-	}
-	local->channel_mask = le16_to_cpu(tmp);
-
-	if (local->channel < 1 || local->channel > 14 ||
-	    !(local->channel_mask & (1 << (local->channel - 1)))) {
-		printk(KERN_WARNING "%s: Channel setting out of range "
-		       "(%d)!\n", dev->name, local->channel);
-		ret = -EBUSY;
-		goto fail;
-	}
-
-	ret = hostap_set_word(dev, HFA384X_RID_CNFOWNCHANNEL, local->channel);
-	if (ret) {
-		printk("%s: Channel setting to %d failed\n",
-		       dev->name, local->channel);
-		goto fail;
-	}
-
-	ret = hostap_set_word(dev, HFA384X_RID_CNFBEACONINT,
-			      local->beacon_int);
-	if (ret) {
-		printk("%s: Beacon interval setting to %d failed\n",
-		       dev->name, local->beacon_int);
-		/* this may fail with Symbol/Lucent firmware */
-		if (ret == -ETIMEDOUT)
-			goto fail;
-	}
-
-	ret = hostap_set_word(dev, HFA384X_RID_CNFOWNDTIMPERIOD,
-			      local->dtim_period);
-	if (ret) {
-		printk("%s: DTIM period setting to %d failed\n",
-		       dev->name, local->dtim_period);
-		/* this may fail with Symbol/Lucent firmware */
-		if (ret == -ETIMEDOUT)
-			goto fail;
-	}
-
-	ret = hostap_set_word(dev, HFA384X_RID_PROMISCUOUSMODE,
-			      local->is_promisc);
-	if (ret)
-		printk(KERN_INFO "%s: Setting promiscuous mode (%d) failed\n",
-		       dev->name, local->is_promisc);
-
-	if (!local->fw_ap) {
-		ret = hostap_set_string(dev, HFA384X_RID_CNFDESIREDSSID,
-					local->essid);
-		if (ret) {
-			printk("%s: Desired SSID setting failed\n", dev->name);
-			goto fail;
-		}
-	}
-
-	/* Setup TXRateControl, defaults to allow use of 1, 2, 5.5, and
-	 * 11 Mbps in automatic TX rate fallback and 1 and 2 Mbps as basic
-	 * rates */
-	if (local->tx_rate_control == 0) {
-		local->tx_rate_control =
-			HFA384X_RATES_1MBPS |
-			HFA384X_RATES_2MBPS |
-			HFA384X_RATES_5MBPS |
-			HFA384X_RATES_11MBPS;
-	}
-	if (local->basic_rates == 0)
-		local->basic_rates = HFA384X_RATES_1MBPS | HFA384X_RATES_2MBPS;
-
-	if (!local->fw_ap) {
-		ret = hostap_set_word(dev, HFA384X_RID_TXRATECONTROL,
-				      local->tx_rate_control);
-		if (ret) {
-			printk("%s: TXRateControl setting to %d failed\n",
-			       dev->name, local->tx_rate_control);
-			goto fail;
-		}
-
-		ret = hostap_set_word(dev, HFA384X_RID_CNFSUPPORTEDRATES,
-				      local->tx_rate_control);
-		if (ret) {
-			printk("%s: cnfSupportedRates setting to %d failed\n",
-			       dev->name, local->tx_rate_control);
-		}
-
-		ret = hostap_set_word(dev, HFA384X_RID_CNFBASICRATES,
-				      local->basic_rates);
-		if (ret) {
-			printk("%s: cnfBasicRates setting to %d failed\n",
-			       dev->name, local->basic_rates);
-		}
-
-		ret = hostap_set_word(dev, HFA384X_RID_CREATEIBSS, 1);
-		if (ret) {
-			printk("%s: Create IBSS setting to 1 failed\n",
-			       dev->name);
-		}
-	}
-
-	if (local->name_set)
-		(void) hostap_set_string(dev, HFA384X_RID_CNFOWNNAME,
-					 local->name);
-
-	if (hostap_set_encryption(local)) {
-		printk(KERN_INFO "%s: could not configure encryption\n",
-		       dev->name);
-	}
-
-	(void) hostap_set_antsel(local);
-
-	if (hostap_set_roaming(local)) {
-		printk(KERN_INFO "%s: could not set host roaming\n",
-		       dev->name);
-	}
-
-	if (local->sta_fw_ver >= PRISM2_FW_VER(1,6,3) &&
-	    hostap_set_word(dev, HFA384X_RID_CNFENHSECURITY, local->enh_sec))
-		printk(KERN_INFO "%s: cnfEnhSecurity setting to 0x%x failed\n",
-		       dev->name, local->enh_sec);
-
-	/* 32-bit tallies were added in STA f/w 0.8.0, but they were apparently
-	 * not working correctly (last seven counters report bogus values).
-	 * This has been fixed in 0.8.2, so enable 32-bit tallies only
-	 * beginning with that firmware version. Another bug fix for 32-bit
-	 * tallies in 1.4.0; should 16-bit tallies be used for some other
-	 * versions, too? */
-	if (local->sta_fw_ver >= PRISM2_FW_VER(0,8,2)) {
-		if (hostap_set_word(dev, HFA384X_RID_CNFTHIRTY2TALLY, 1)) {
-			printk(KERN_INFO "%s: cnfThirty2Tally setting "
-			       "failed\n", dev->name);
-			local->tallies32 = 0;
-		} else
-			local->tallies32 = 1;
-	} else
-		local->tallies32 = 0;
-
-	hostap_set_auth_algs(local);
-
-	if (hostap_set_word(dev, HFA384X_RID_FRAGMENTATIONTHRESHOLD,
-			    local->fragm_threshold)) {
-		printk(KERN_INFO "%s: setting FragmentationThreshold to %d "
-		       "failed\n", dev->name, local->fragm_threshold);
-	}
-
-	if (hostap_set_word(dev, HFA384X_RID_RTSTHRESHOLD,
-			    local->rts_threshold)) {
-		printk(KERN_INFO "%s: setting RTSThreshold to %d failed\n",
-		       dev->name, local->rts_threshold);
-	}
-
-	if (local->manual_retry_count >= 0 &&
-	    hostap_set_word(dev, HFA384X_RID_CNFALTRETRYCOUNT,
-			    local->manual_retry_count)) {
-		printk(KERN_INFO "%s: setting cnfAltRetryCount to %d failed\n",
-		       dev->name, local->manual_retry_count);
-	}
-
-	if (local->sta_fw_ver >= PRISM2_FW_VER(1,3,1) &&
-	    hfa384x_get_rid(dev, HFA384X_RID_CNFDBMADJUST, &tmp, 2, 1) == 2) {
-		local->rssi_to_dBm = le16_to_cpu(tmp);
-	}
-
-	if (local->sta_fw_ver >= PRISM2_FW_VER(1,7,0) && local->wpa &&
-	    hostap_set_word(dev, HFA384X_RID_SSNHANDLINGMODE, 1)) {
-		printk(KERN_INFO "%s: setting ssnHandlingMode to 1 failed\n",
-		       dev->name);
-	}
-
-	if (local->sta_fw_ver >= PRISM2_FW_VER(1,7,0) && local->generic_elem &&
-	    hfa384x_set_rid(dev, HFA384X_RID_GENERICELEMENT,
-			    local->generic_elem, local->generic_elem_len)) {
-		printk(KERN_INFO "%s: setting genericElement failed\n",
-		       dev->name);
-	}
-
- fail:
-	return ret;
-}
-
-
-static int prism2_hw_init(struct net_device *dev, int initial)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int ret, first = 1;
-	unsigned long start, delay;
-
-	PDEBUG(DEBUG_FLOW, "prism2_hw_init()\n");
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	clear_bit(HOSTAP_BITS_TRANSMIT, &local->bits);
-
- init:
-	/* initialize HFA 384x */
-	ret = hfa384x_cmd_no_wait(dev, HFA384X_CMDCODE_INIT, 0);
-	if (ret) {
-		printk(KERN_INFO "%s: first command failed - assuming card "
-		       "does not have primary firmware\n", dev_info);
-	}
-
-	if (first && (HFA384X_INW(HFA384X_EVSTAT_OFF) & HFA384X_EV_CMD)) {
-		/* EvStat has Cmd bit set in some cases, so retry once if no
-		 * wait was needed */
-		HFA384X_OUTW(HFA384X_EV_CMD, HFA384X_EVACK_OFF);
-		printk(KERN_DEBUG "%s: init command completed too quickly - "
-		       "retrying\n", dev->name);
-		first = 0;
-		goto init;
-	}
-
-	start = jiffies;
-	delay = jiffies + HFA384X_INIT_TIMEOUT;
-	while (!(HFA384X_INW(HFA384X_EVSTAT_OFF) & HFA384X_EV_CMD) &&
-	       time_before(jiffies, delay))
-		yield();
-	if (!(HFA384X_INW(HFA384X_EVSTAT_OFF) & HFA384X_EV_CMD)) {
-		printk(KERN_DEBUG "%s: assuming no Primary image in "
-		       "flash - card initialization not completed\n",
-		       dev_info);
-		local->no_pri = 1;
-#ifdef PRISM2_DOWNLOAD_SUPPORT
-			if (local->sram_type == -1)
-				local->sram_type = prism2_get_ram_size(local);
-#endif /* PRISM2_DOWNLOAD_SUPPORT */
-		return 1;
-	}
-	local->no_pri = 0;
-	printk(KERN_DEBUG "prism2_hw_init: initialized in %lu ms\n",
-	       (jiffies - start) * 1000 / HZ);
-	HFA384X_OUTW(HFA384X_EV_CMD, HFA384X_EVACK_OFF);
-	return 0;
-}
-
-
-static int prism2_hw_init2(struct net_device *dev, int initial)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int i;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-#ifdef PRISM2_DOWNLOAD_SUPPORT
-	kfree(local->pda);
-	if (local->no_pri)
-		local->pda = NULL;
-	else
-		local->pda = prism2_read_pda(dev);
-#endif /* PRISM2_DOWNLOAD_SUPPORT */
-
-	hfa384x_disable_interrupts(dev);
-
-#ifndef final_version
-	HFA384X_OUTW(HFA384X_MAGIC, HFA384X_SWSUPPORT0_OFF);
-	if (HFA384X_INW(HFA384X_SWSUPPORT0_OFF) != HFA384X_MAGIC) {
-		printk("SWSUPPORT0 write/read failed: %04X != %04X\n",
-		       HFA384X_INW(HFA384X_SWSUPPORT0_OFF), HFA384X_MAGIC);
-		goto failed;
-	}
-#endif
-
-	if (initial || local->pri_only) {
-		hfa384x_events_only_cmd(dev);
-		/* get card version information */
-		if (prism2_get_version_info(dev, HFA384X_RID_NICID, "NIC") ||
-		    prism2_get_version_info(dev, HFA384X_RID_PRIID, "PRI")) {
-			hfa384x_disable_interrupts(dev);
-			goto failed;
-		}
-
-		if (prism2_get_version_info(dev, HFA384X_RID_STAID, "STA")) {
-			printk(KERN_DEBUG "%s: Failed to read STA f/w version "
-			       "- only Primary f/w present\n", dev->name);
-			local->pri_only = 1;
-			return 0;
-		}
-		local->pri_only = 0;
-		hfa384x_disable_interrupts(dev);
-	}
-
-	/* FIX: could convert allocate_fid to use sleeping CmdCompl wait and
-	 * enable interrupts before this. This would also require some sort of
-	 * sleeping AllocEv waiting */
-
-	/* allocate TX FIDs */
-	local->txfid_len = PRISM2_TXFID_LEN;
-	for (i = 0; i < PRISM2_TXFID_COUNT; i++) {
-		local->txfid[i] = hfa384x_allocate_fid(dev, local->txfid_len);
-		if (local->txfid[i] == 0xffff && local->txfid_len > 1600) {
-			local->txfid[i] = hfa384x_allocate_fid(dev, 1600);
-			if (local->txfid[i] != 0xffff) {
-				printk(KERN_DEBUG "%s: Using shorter TX FID "
-				       "(1600 bytes)\n", dev->name);
-				local->txfid_len = 1600;
-			}
-		}
-		if (local->txfid[i] == 0xffff)
-			goto failed;
-		local->intransmitfid[i] = PRISM2_TXFID_EMPTY;
-	}
-
-	hfa384x_events_only_cmd(dev);
-
-	if (initial) {
-		u8 addr[ETH_ALEN] = {};
-		struct list_head *ptr;
-
-		prism2_check_sta_fw_version(local);
-
-		if (hfa384x_get_rid(dev, HFA384X_RID_CNFOWNMACADDR,
-				    addr, ETH_ALEN, 1) < 0) {
-			printk("%s: could not get own MAC address\n",
-			       dev->name);
-		}
-		eth_hw_addr_set(dev, addr);
-		list_for_each(ptr, &local->hostap_interfaces) {
-			iface = list_entry(ptr, struct hostap_interface, list);
-			eth_hw_addr_inherit(iface->dev, dev);
-		}
-	} else if (local->fw_ap)
-		prism2_check_sta_fw_version(local);
-
-	prism2_setup_rids(dev);
-
-	/* MAC is now configured, but port 0 is not yet enabled */
-	return 0;
-
- failed:
-	if (!local->no_pri)
-		printk(KERN_WARNING "%s: Initialization failed\n", dev_info);
-	return 1;
-}
-
-
-static int prism2_hw_enable(struct net_device *dev, int initial)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int was_resetting;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-	was_resetting = local->hw_resetting;
-
-	if (hfa384x_cmd(dev, HFA384X_CMDCODE_ENABLE, 0, NULL, NULL)) {
-		printk("%s: MAC port 0 enabling failed\n", dev->name);
-		return 1;
-	}
-
-	local->hw_ready = 1;
-	local->hw_reset_tries = 0;
-	local->hw_resetting = 0;
-	hfa384x_enable_interrupts(dev);
-
-	/* at least D-Link DWL-650 seems to require additional port reset
-	 * before it starts acting as an AP, so reset port automatically
-	 * here just in case */
-	if (initial && prism2_reset_port(dev)) {
-		printk("%s: MAC port 0 resetting failed\n", dev->name);
-		return 1;
-	}
-
-	if (was_resetting && netif_queue_stopped(dev)) {
-		/* If hw_reset() was called during pending transmit, netif
-		 * queue was stopped. Wake it up now since the wlan card has
-		 * been resetted. */
-		netif_wake_queue(dev);
-	}
-
-	return 0;
-}
-
-
-static int prism2_hw_config(struct net_device *dev, int initial)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (local->hw_downloading)
-		return 1;
-
-	if (prism2_hw_init(dev, initial)) {
-		return local->no_pri ? 0 : 1;
-	}
-
-	if (prism2_hw_init2(dev, initial))
-		return 1;
-
-	/* Enable firmware if secondary image is loaded and at least one of the
-	 * netdevices is up. */
-	if (!local->pri_only &&
-	    (initial == 0 || (initial == 2 && local->num_dev_open > 0))) {
-		if (!local->dev_enabled)
-			prism2_callback(local, PRISM2_CALLBACK_ENABLE);
-		local->dev_enabled = 1;
-		return prism2_hw_enable(dev, initial);
-	}
-
-	return 0;
-}
-
-
-static void prism2_hw_shutdown(struct net_device *dev, int no_disable)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	/* Allow only command completion events during disable */
-	hfa384x_events_only_cmd(dev);
-
-	local->hw_ready = 0;
-	if (local->dev_enabled)
-		prism2_callback(local, PRISM2_CALLBACK_DISABLE);
-	local->dev_enabled = 0;
-
-	if (local->func->card_present && !local->func->card_present(local)) {
-		printk(KERN_DEBUG "%s: card already removed or not configured "
-		       "during shutdown\n", dev->name);
-		return;
-	}
-
-	if ((no_disable & HOSTAP_HW_NO_DISABLE) == 0 &&
-	    hfa384x_cmd(dev, HFA384X_CMDCODE_DISABLE, 0, NULL, NULL))
-		printk(KERN_WARNING "%s: Shutdown failed\n", dev_info);
-
-	hfa384x_disable_interrupts(dev);
-
-	if (no_disable & HOSTAP_HW_ENABLE_CMDCOMPL)
-		hfa384x_events_only_cmd(dev);
-	else
-		prism2_clear_cmd_queue(local);
-}
-
-
-static void prism2_hw_reset(struct net_device *dev)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-
-#if 0
-	static long last_reset = 0;
-
-	/* do not reset card more than once per second to avoid ending up in a
-	 * busy loop resetting the card */
-	if (time_before_eq(jiffies, last_reset + HZ))
-		return;
-	last_reset = jiffies;
-#endif
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (local->hw_downloading)
-		return;
-
-	if (local->hw_resetting) {
-		printk(KERN_WARNING "%s: %s: already resetting card - "
-		       "ignoring reset request\n", dev_info, dev->name);
-		return;
-	}
-
-	local->hw_reset_tries++;
-	if (local->hw_reset_tries > 10) {
-		printk(KERN_WARNING "%s: too many reset tries, skipping\n",
-		       dev->name);
-		return;
-	}
-
-	printk(KERN_WARNING "%s: %s: resetting card\n", dev_info, dev->name);
-	hfa384x_disable_interrupts(dev);
-	local->hw_resetting = 1;
-	if (local->func->cor_sreset) {
-		/* Host system seems to hang in some cases with high traffic
-		 * load or shared interrupts during COR sreset. Disable shared
-		 * interrupts during reset to avoid these crashes. COS sreset
-		 * takes quite a long time, so it is unfortunate that this
-		 * seems to be needed. Anyway, I do not know of any better way
-		 * of avoiding the crash. */
-		disable_irq(dev->irq);
-		local->func->cor_sreset(local);
-		enable_irq(dev->irq);
-	}
-	prism2_hw_shutdown(dev, 1);
-	prism2_hw_config(dev, 0);
-	local->hw_resetting = 0;
-
-#ifdef PRISM2_DOWNLOAD_SUPPORT
-	if (local->dl_pri) {
-		printk(KERN_DEBUG "%s: persistent download of primary "
-		       "firmware\n", dev->name);
-		if (prism2_download_genesis(local, local->dl_pri) < 0)
-			printk(KERN_WARNING "%s: download (PRI) failed\n",
-			       dev->name);
-	}
-
-	if (local->dl_sec) {
-		printk(KERN_DEBUG "%s: persistent download of secondary "
-		       "firmware\n", dev->name);
-		if (prism2_download_volatile(local, local->dl_sec) < 0)
-			printk(KERN_WARNING "%s: download (SEC) failed\n",
-			       dev->name);
-	}
-#endif /* PRISM2_DOWNLOAD_SUPPORT */
-
-	/* TODO: restore beacon TIM bits for STAs that have buffered frames */
-}
-
-
-static void prism2_schedule_reset(local_info_t *local)
-{
-	schedule_work(&local->reset_queue);
-}
-
-
-/* Called only as scheduled task after noticing card timeout in interrupt
- * context */
-static void handle_reset_queue(struct work_struct *work)
-{
-	local_info_t *local = container_of(work, local_info_t, reset_queue);
-
-	printk(KERN_DEBUG "%s: scheduled card reset\n", local->dev->name);
-	prism2_hw_reset(local->dev);
-
-	if (netif_queue_stopped(local->dev)) {
-		int i;
-
-		for (i = 0; i < PRISM2_TXFID_COUNT; i++)
-			if (local->intransmitfid[i] == PRISM2_TXFID_EMPTY) {
-				PDEBUG(DEBUG_EXTRA, "prism2_tx_timeout: "
-				       "wake up queue\n");
-				netif_wake_queue(local->dev);
-				break;
-			}
-	}
-}
-
-
-static int prism2_get_txfid_idx(local_info_t *local)
-{
-	int idx, end;
-	unsigned long flags;
-
-	spin_lock_irqsave(&local->txfidlock, flags);
-	end = idx = local->next_txfid;
-	do {
-		if (local->intransmitfid[idx] == PRISM2_TXFID_EMPTY) {
-			local->intransmitfid[idx] = PRISM2_TXFID_RESERVED;
-			spin_unlock_irqrestore(&local->txfidlock, flags);
-			return idx;
-		}
-		idx++;
-		if (idx >= PRISM2_TXFID_COUNT)
-			idx = 0;
-	} while (idx != end);
-	spin_unlock_irqrestore(&local->txfidlock, flags);
-
-	PDEBUG(DEBUG_EXTRA2, "prism2_get_txfid_idx: no room in txfid buf: "
-	       "packet dropped\n");
-	local->dev->stats.tx_dropped++;
-
-	return -1;
-}
-
-
-/* Called only from hardware IRQ */
-static void prism2_transmit_cb(struct net_device *dev, long context,
-			       u16 resp0, u16 res)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int idx = (int) context;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (res) {
-		printk(KERN_DEBUG "%s: prism2_transmit_cb - res=0x%02x\n",
-		       dev->name, res);
-		return;
-	}
-
-	if (idx < 0 || idx >= PRISM2_TXFID_COUNT) {
-		printk(KERN_DEBUG "%s: prism2_transmit_cb called with invalid "
-		       "idx=%d\n", dev->name, idx);
-		return;
-	}
-
-	if (!test_and_clear_bit(HOSTAP_BITS_TRANSMIT, &local->bits)) {
-		printk(KERN_DEBUG "%s: driver bug: prism2_transmit_cb called "
-		       "with no pending transmit\n", dev->name);
-	}
-
-	if (netif_queue_stopped(dev)) {
-		/* ready for next TX, so wake up queue that was stopped in
-		 * prism2_transmit() */
-		netif_wake_queue(dev);
-	}
-
-	spin_lock(&local->txfidlock);
-
-	/* With reclaim, Resp0 contains new txfid for transmit; the old txfid
-	 * will be automatically allocated for the next TX frame */
-	local->intransmitfid[idx] = resp0;
-
-	PDEBUG(DEBUG_FID, "%s: prism2_transmit_cb: txfid[%d]=0x%04x, "
-	       "resp0=0x%04x, transmit_txfid=0x%04x\n",
-	       dev->name, idx, local->txfid[idx],
-	       resp0, local->intransmitfid[local->next_txfid]);
-
-	idx++;
-	if (idx >= PRISM2_TXFID_COUNT)
-		idx = 0;
-	local->next_txfid = idx;
-
-	/* check if all TX buffers are occupied */
-	do {
-		if (local->intransmitfid[idx] == PRISM2_TXFID_EMPTY) {
-			spin_unlock(&local->txfidlock);
-			return;
-		}
-		idx++;
-		if (idx >= PRISM2_TXFID_COUNT)
-			idx = 0;
-	} while (idx != local->next_txfid);
-	spin_unlock(&local->txfidlock);
-
-	/* no empty TX buffers, stop queue */
-	netif_stop_queue(dev);
-}
-
-
-/* Called only from software IRQ if PCI bus master is not used (with bus master
- * this can be called both from software and hardware IRQ) */
-static int prism2_transmit(struct net_device *dev, int idx)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int res;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	/* The driver tries to stop netif queue so that there would not be
-	 * more than one attempt to transmit frames going on; check that this
-	 * is really the case */
-
-	if (test_and_set_bit(HOSTAP_BITS_TRANSMIT, &local->bits)) {
-		printk(KERN_DEBUG "%s: driver bug - prism2_transmit() called "
-		       "when previous TX was pending\n", dev->name);
-		return -1;
-	}
-
-	/* stop the queue for the time that transmit is pending */
-	netif_stop_queue(dev);
-
-	/* transmit packet */
-	res = hfa384x_cmd_callback(
-		dev,
-		HFA384X_CMDCODE_TRANSMIT | HFA384X_CMD_TX_RECLAIM,
-		local->txfid[idx],
-		prism2_transmit_cb, (long) idx);
-
-	if (res) {
-		printk(KERN_DEBUG "%s: prism2_transmit: CMDCODE_TRANSMIT "
-		       "failed (res=%d)\n", dev->name, res);
-		dev->stats.tx_dropped++;
-		netif_wake_queue(dev);
-		return -1;
-	}
-	netif_trans_update(dev);
-
-	/* Since we did not wait for command completion, the card continues
-	 * to process on the background and we will finish handling when
-	 * command completion event is handled (prism2_cmd_ev() function) */
-
-	return 0;
-}
-
-
-/* Send IEEE 802.11 frame (convert the header into Prism2 TX descriptor and
- * send the payload with this descriptor) */
-/* Called only from software IRQ */
-static int prism2_tx_80211(struct sk_buff *skb, struct net_device *dev)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	struct hfa384x_tx_frame txdesc;
-	struct hostap_skb_tx_data *meta;
-	int hdr_len, data_len, idx, res, ret = -1;
-	u16 tx_control;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	meta = (struct hostap_skb_tx_data *) skb->cb;
-
-	prism2_callback(local, PRISM2_CALLBACK_TX_START);
-
-	if ((local->func->card_present && !local->func->card_present(local)) ||
-	    !local->hw_ready || local->hw_downloading || local->pri_only) {
-		if (net_ratelimit()) {
-			printk(KERN_DEBUG "%s: prism2_tx_80211: hw not ready -"
-			       " skipping\n", dev->name);
-		}
-		goto fail;
-	}
-
-	memset(&txdesc, 0, sizeof(txdesc));
-
-	/* skb->data starts with txdesc->frame_control */
-	hdr_len = sizeof(txdesc.header);
-	BUILD_BUG_ON(hdr_len != 24);
-	skb_copy_from_linear_data(skb, &txdesc.header, hdr_len);
-	if (ieee80211_is_data(txdesc.frame_control) &&
-	    ieee80211_has_a4(txdesc.frame_control) &&
-	    skb->len >= 30) {
-		/* Addr4 */
-		skb_copy_from_linear_data_offset(skb, hdr_len, txdesc.addr4,
-						 ETH_ALEN);
-		hdr_len += ETH_ALEN;
-	}
-
-	tx_control = local->tx_control;
-	if (meta->tx_cb_idx) {
-		tx_control |= HFA384X_TX_CTRL_TX_OK;
-		txdesc.sw_support = cpu_to_le32(meta->tx_cb_idx);
-	}
-	txdesc.tx_control = cpu_to_le16(tx_control);
-	txdesc.tx_rate = meta->rate;
-
-	data_len = skb->len - hdr_len;
-	txdesc.data_len = cpu_to_le16(data_len);
-	txdesc.len = cpu_to_be16(data_len);
-
-	idx = prism2_get_txfid_idx(local);
-	if (idx < 0)
-		goto fail;
-
-	if (local->frame_dump & PRISM2_DUMP_TX_HDR)
-		hostap_dump_tx_header(dev->name, &txdesc);
-
-	spin_lock(&local->baplock);
-	res = hfa384x_setup_bap(dev, BAP0, local->txfid[idx], 0);
-
-	if (!res)
-		res = hfa384x_to_bap(dev, BAP0, &txdesc, sizeof(txdesc));
-	if (!res)
-		res = hfa384x_to_bap(dev, BAP0, skb->data + hdr_len,
-				     skb->len - hdr_len);
-	spin_unlock(&local->baplock);
-
-	if (!res)
-		res = prism2_transmit(dev, idx);
-	if (res) {
-		printk(KERN_DEBUG "%s: prism2_tx_80211 - to BAP0 failed\n",
-		       dev->name);
-		local->intransmitfid[idx] = PRISM2_TXFID_EMPTY;
-		schedule_work(&local->reset_queue);
-		goto fail;
-	}
-
-	ret = 0;
-
-fail:
-	prism2_callback(local, PRISM2_CALLBACK_TX_END);
-	return ret;
-}
-
-
-/* Some SMP systems have reported number of odd errors with hostap_pci. fid
- * register has changed values between consecutive reads for an unknown reason.
- * This should really not happen, so more debugging is needed. This test
- * version is a bit slower, but it will detect most of such register changes
- * and will try to get the correct fid eventually. */
-#define EXTRA_FID_READ_TESTS
-
-static u16 prism2_read_fid_reg(struct net_device *dev, u16 reg)
-{
-#ifdef EXTRA_FID_READ_TESTS
-	u16 val, val2, val3;
-	int i;
-
-	for (i = 0; i < 10; i++) {
-		val = HFA384X_INW(reg);
-		val2 = HFA384X_INW(reg);
-		val3 = HFA384X_INW(reg);
-
-		if (val == val2 && val == val3)
-			return val;
-
-		printk(KERN_DEBUG "%s: detected fid change (try=%d, reg=%04x):"
-		       " %04x %04x %04x\n",
-		       dev->name, i, reg, val, val2, val3);
-		if ((val == val2 || val == val3) && val != 0)
-			return val;
-		if (val2 == val3 && val2 != 0)
-			return val2;
-	}
-	printk(KERN_WARNING "%s: Uhhuh.. could not read good fid from reg "
-	       "%04x (%04x %04x %04x)\n", dev->name, reg, val, val2, val3);
-	return val;
-#else /* EXTRA_FID_READ_TESTS */
-	return HFA384X_INW(reg);
-#endif /* EXTRA_FID_READ_TESTS */
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static void prism2_rx(local_info_t *local)
-{
-	struct net_device *dev = local->dev;
-	int res, rx_pending = 0;
-	u16 len, hdr_len, rxfid, status, macport;
-	struct hfa384x_rx_frame rxdesc;
-	struct sk_buff *skb = NULL;
-
-	prism2_callback(local, PRISM2_CALLBACK_RX_START);
-
-	rxfid = prism2_read_fid_reg(dev, HFA384X_RXFID_OFF);
-#ifndef final_version
-	if (rxfid == 0) {
-		rxfid = HFA384X_INW(HFA384X_RXFID_OFF);
-		printk(KERN_DEBUG "prism2_rx: rxfid=0 (next 0x%04x)\n",
-		       rxfid);
-		if (rxfid == 0) {
-			schedule_work(&local->reset_queue);
-			goto rx_dropped;
-		}
-		/* try to continue with the new rxfid value */
-	}
-#endif
-
-	spin_lock(&local->baplock);
-	res = hfa384x_setup_bap(dev, BAP0, rxfid, 0);
-	if (!res)
-		res = hfa384x_from_bap(dev, BAP0, &rxdesc, sizeof(rxdesc));
-
-	if (res) {
-		spin_unlock(&local->baplock);
-		printk(KERN_DEBUG "%s: copy from BAP0 failed %d\n", dev->name,
-		       res);
-		if (res == -ETIMEDOUT) {
-			schedule_work(&local->reset_queue);
-		}
-		goto rx_dropped;
-	}
-
-	len = le16_to_cpu(rxdesc.data_len);
-	hdr_len = sizeof(rxdesc);
-	status = le16_to_cpu(rxdesc.status);
-	macport = (status >> 8) & 0x07;
-
-	/* Drop frames with too large reported payload length. Monitor mode
-	 * seems to sometimes pass frames (e.g., ctrl::ack) with signed and
-	 * negative value, so allow also values 65522 .. 65534 (-14 .. -2) for
-	 * macport 7 */
-	if (len > PRISM2_DATA_MAXLEN + 8 /* WEP */) {
-		if (macport == 7 && local->iw_mode == IW_MODE_MONITOR) {
-			if (len >= (u16) -14) {
-				hdr_len -= 65535 - len;
-				hdr_len--;
-			}
-			len = 0;
-		} else {
-			spin_unlock(&local->baplock);
-			printk(KERN_DEBUG "%s: Received frame with invalid "
-			       "length 0x%04x\n", dev->name, len);
-			hostap_dump_rx_header(dev->name, &rxdesc);
-			goto rx_dropped;
-		}
-	}
-
-	skb = dev_alloc_skb(len + hdr_len);
-	if (!skb) {
-		spin_unlock(&local->baplock);
-		printk(KERN_DEBUG "%s: RX failed to allocate skb\n",
-		       dev->name);
-		goto rx_dropped;
-	}
-	skb->dev = dev;
-	skb_put_data(skb, &rxdesc, hdr_len);
-
-	if (len > 0)
-		res = hfa384x_from_bap(dev, BAP0, skb_put(skb, len), len);
-	spin_unlock(&local->baplock);
-	if (res) {
-		printk(KERN_DEBUG "%s: RX failed to read "
-		       "frame data\n", dev->name);
-		goto rx_dropped;
-	}
-
-	skb_queue_tail(&local->rx_list, skb);
-	tasklet_schedule(&local->rx_tasklet);
-
- rx_exit:
-	prism2_callback(local, PRISM2_CALLBACK_RX_END);
-	if (!rx_pending) {
-		HFA384X_OUTW(HFA384X_EV_RX, HFA384X_EVACK_OFF);
-	}
-
-	return;
-
- rx_dropped:
-	dev->stats.rx_dropped++;
-	if (skb)
-		dev_kfree_skb(skb);
-	goto rx_exit;
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static void hostap_rx_skb(local_info_t *local, struct sk_buff *skb)
-{
-	struct hfa384x_rx_frame *rxdesc;
-	struct net_device *dev = skb->dev;
-	struct hostap_80211_rx_status stats;
-	int hdrlen, rx_hdrlen;
-
-	rx_hdrlen = sizeof(*rxdesc);
-	if (skb->len < sizeof(*rxdesc)) {
-		/* Allow monitor mode to receive shorter frames */
-		if (local->iw_mode == IW_MODE_MONITOR &&
-		    skb->len >= sizeof(*rxdesc) - 30) {
-			rx_hdrlen = skb->len;
-		} else {
-			dev_kfree_skb(skb);
-			return;
-		}
-	}
-
-	rxdesc = (struct hfa384x_rx_frame *) skb->data;
-
-	if (local->frame_dump & PRISM2_DUMP_RX_HDR &&
-	    skb->len >= sizeof(*rxdesc))
-		hostap_dump_rx_header(dev->name, rxdesc);
-
-	if (le16_to_cpu(rxdesc->status) & HFA384X_RX_STATUS_FCSERR &&
-	    (!local->monitor_allow_fcserr ||
-	     local->iw_mode != IW_MODE_MONITOR))
-		goto drop;
-
-	if (skb->len > PRISM2_DATA_MAXLEN) {
-		printk(KERN_DEBUG "%s: RX: len(%d) > MAX(%d)\n",
-		       dev->name, skb->len, PRISM2_DATA_MAXLEN);
-		goto drop;
-	}
-
-	stats.mac_time = le32_to_cpu(rxdesc->time);
-	stats.signal = rxdesc->signal - local->rssi_to_dBm;
-	stats.noise = rxdesc->silence - local->rssi_to_dBm;
-	stats.rate = rxdesc->rate;
-
-	/* Convert Prism2 RX structure into IEEE 802.11 header */
-	hdrlen = hostap_80211_get_hdrlen(rxdesc->frame_control);
-	if (hdrlen > rx_hdrlen)
-		hdrlen = rx_hdrlen;
-
-	memmove(skb_pull(skb, rx_hdrlen - hdrlen),
-		&rxdesc->frame_control, hdrlen);
-
-	hostap_80211_rx(dev, skb, &stats);
-	return;
-
- drop:
-	dev_kfree_skb(skb);
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static void hostap_rx_tasklet(struct tasklet_struct *t)
-{
-	local_info_t *local = from_tasklet(local, t, rx_tasklet);
-	struct sk_buff *skb;
-
-	while ((skb = skb_dequeue(&local->rx_list)) != NULL)
-		hostap_rx_skb(local, skb);
-}
-
-
-/* Called only from hardware IRQ */
-static void prism2_alloc_ev(struct net_device *dev)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int idx;
-	u16 fid;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	fid = prism2_read_fid_reg(dev, HFA384X_ALLOCFID_OFF);
-
-	PDEBUG(DEBUG_FID, "FID: interrupt: ALLOC - fid=0x%04x\n", fid);
-
-	spin_lock(&local->txfidlock);
-	idx = local->next_alloc;
-
-	do {
-		if (local->txfid[idx] == fid) {
-			PDEBUG(DEBUG_FID, "FID: found matching txfid[%d]\n",
-			       idx);
-
-#ifndef final_version
-			if (local->intransmitfid[idx] == PRISM2_TXFID_EMPTY)
-				printk("Already released txfid found at idx "
-				       "%d\n", idx);
-			if (local->intransmitfid[idx] == PRISM2_TXFID_RESERVED)
-				printk("Already reserved txfid found at idx "
-				       "%d\n", idx);
-#endif
-			local->intransmitfid[idx] = PRISM2_TXFID_EMPTY;
-			idx++;
-			local->next_alloc = idx >= PRISM2_TXFID_COUNT ? 0 :
-				idx;
-
-			if (!test_bit(HOSTAP_BITS_TRANSMIT, &local->bits) &&
-			    netif_queue_stopped(dev))
-				netif_wake_queue(dev);
-
-			spin_unlock(&local->txfidlock);
-			return;
-		}
-
-		idx++;
-		if (idx >= PRISM2_TXFID_COUNT)
-			idx = 0;
-	} while (idx != local->next_alloc);
-
-	printk(KERN_WARNING "%s: could not find matching txfid (0x%04x, new "
-	       "read 0x%04x) for alloc event\n", dev->name, fid,
-	       HFA384X_INW(HFA384X_ALLOCFID_OFF));
-	printk(KERN_DEBUG "TXFIDs:");
-	for (idx = 0; idx < PRISM2_TXFID_COUNT; idx++)
-		printk(" %04x[%04x]", local->txfid[idx],
-		       local->intransmitfid[idx]);
-	printk("\n");
-	spin_unlock(&local->txfidlock);
-
-	/* FIX: should probably schedule reset; reference to one txfid was lost
-	 * completely.. Bad things will happen if we run out of txfids
-	 * Actually, this will cause netdev watchdog to notice TX timeout and
-	 * then card reset after all txfids have been leaked. */
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static void hostap_tx_callback(local_info_t *local,
-			       struct hfa384x_tx_frame *txdesc, int ok,
-			       char *payload)
-{
-	u16 sw_support, hdrlen, len;
-	struct sk_buff *skb;
-	struct hostap_tx_callback_info *cb;
-
-	/* Make sure that frame was from us. */
-	if (!ether_addr_equal(txdesc->addr2, local->dev->dev_addr)) {
-		printk(KERN_DEBUG "%s: TX callback - foreign frame\n",
-		       local->dev->name);
-		return;
-	}
-
-	sw_support = le32_to_cpu(txdesc->sw_support);
-
-	spin_lock(&local->lock);
-	cb = local->tx_callback;
-	while (cb != NULL && cb->idx != sw_support)
-		cb = cb->next;
-	spin_unlock(&local->lock);
-
-	if (cb == NULL) {
-		printk(KERN_DEBUG "%s: could not find TX callback (idx %d)\n",
-		       local->dev->name, sw_support);
-		return;
-	}
-
-	hdrlen = hostap_80211_get_hdrlen(txdesc->frame_control);
-	len = le16_to_cpu(txdesc->data_len);
-	skb = dev_alloc_skb(hdrlen + len);
-	if (skb == NULL) {
-		printk(KERN_DEBUG "%s: hostap_tx_callback failed to allocate "
-		       "skb\n", local->dev->name);
-		return;
-	}
-
-	skb_put_data(skb, (void *)&txdesc->frame_control, hdrlen);
-	if (payload)
-		skb_put_data(skb, payload, len);
-
-	skb->dev = local->dev;
-	skb_reset_mac_header(skb);
-
-	cb->func(skb, ok, cb->data);
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static int hostap_tx_compl_read(local_info_t *local, int error,
-				struct hfa384x_tx_frame *txdesc,
-				char **payload)
-{
-	u16 fid, len;
-	int res, ret = 0;
-	struct net_device *dev = local->dev;
-
-	fid = prism2_read_fid_reg(dev, HFA384X_TXCOMPLFID_OFF);
-
-	PDEBUG(DEBUG_FID, "interrupt: TX (err=%d) - fid=0x%04x\n", fid, error);
-
-	spin_lock(&local->baplock);
-	res = hfa384x_setup_bap(dev, BAP0, fid, 0);
-	if (!res)
-		res = hfa384x_from_bap(dev, BAP0, txdesc, sizeof(*txdesc));
-	if (res) {
-		PDEBUG(DEBUG_EXTRA, "%s: TX (err=%d) - fid=0x%04x - could not "
-		       "read txdesc\n", dev->name, error, fid);
-		if (res == -ETIMEDOUT) {
-			schedule_work(&local->reset_queue);
-		}
-		ret = -1;
-		goto fail;
-	}
-	if (txdesc->sw_support) {
-		len = le16_to_cpu(txdesc->data_len);
-		if (len < PRISM2_DATA_MAXLEN) {
-			*payload = kmalloc(len, GFP_ATOMIC);
-			if (*payload == NULL ||
-			    hfa384x_from_bap(dev, BAP0, *payload, len)) {
-				PDEBUG(DEBUG_EXTRA, "%s: could not read TX "
-				       "frame payload\n", dev->name);
-				kfree(*payload);
-				*payload = NULL;
-				ret = -1;
-				goto fail;
-			}
-		}
-	}
-
- fail:
-	spin_unlock(&local->baplock);
-
-	return ret;
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static void prism2_tx_ev(local_info_t *local)
-{
-	struct net_device *dev = local->dev;
-	char *payload = NULL;
-	struct hfa384x_tx_frame txdesc;
-
-	if (hostap_tx_compl_read(local, 0, &txdesc, &payload))
-		goto fail;
-
-	if (local->frame_dump & PRISM2_DUMP_TX_HDR) {
-		PDEBUG(DEBUG_EXTRA, "%s: TX - status=0x%04x "
-		       "retry_count=%d tx_rate=%d seq_ctrl=%d "
-		       "duration_id=%d\n",
-		       dev->name, le16_to_cpu(txdesc.status),
-		       txdesc.retry_count, txdesc.tx_rate,
-		       le16_to_cpu(txdesc.seq_ctrl),
-		       le16_to_cpu(txdesc.duration_id));
-	}
-
-	if (txdesc.sw_support)
-		hostap_tx_callback(local, &txdesc, 1, payload);
-	kfree(payload);
-
- fail:
-	HFA384X_OUTW(HFA384X_EV_TX, HFA384X_EVACK_OFF);
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static void hostap_sta_tx_exc_tasklet(struct tasklet_struct *t)
-{
-	local_info_t *local = from_tasklet(local, t, sta_tx_exc_tasklet);
-	struct sk_buff *skb;
-
-	while ((skb = skb_dequeue(&local->sta_tx_exc_list)) != NULL) {
-		struct hfa384x_tx_frame *txdesc =
-			(struct hfa384x_tx_frame *) skb->data;
-
-		if (skb->len >= sizeof(*txdesc)) {
-			/* Convert Prism2 RX structure into IEEE 802.11 header
-			 */
-			int hdrlen = hostap_80211_get_hdrlen(txdesc->frame_control);
-			memmove(skb_pull(skb, sizeof(*txdesc) - hdrlen),
-				&txdesc->frame_control, hdrlen);
-
-			hostap_handle_sta_tx_exc(local, skb);
-		}
-		dev_kfree_skb(skb);
-	}
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static void prism2_txexc(local_info_t *local)
-{
-	struct net_device *dev = local->dev;
-	u16 status, fc;
-	int show_dump, res;
-	char *payload = NULL;
-	struct hfa384x_tx_frame txdesc;
-
-	show_dump = local->frame_dump & PRISM2_DUMP_TXEXC_HDR;
-	dev->stats.tx_errors++;
-
-	res = hostap_tx_compl_read(local, 1, &txdesc, &payload);
-	HFA384X_OUTW(HFA384X_EV_TXEXC, HFA384X_EVACK_OFF);
-	if (res)
-		return;
-
-	status = le16_to_cpu(txdesc.status);
-
-	/* We produce a TXDROP event only for retry or lifetime
-	 * exceeded, because that's the only status that really mean
-	 * that this particular node went away.
-	 * Other errors means that *we* screwed up. - Jean II */
-	if (status & (HFA384X_TX_STATUS_RETRYERR | HFA384X_TX_STATUS_AGEDERR))
-	{
-		union iwreq_data wrqu;
-
-		/* Copy 802.11 dest address. */
-		memcpy(wrqu.addr.sa_data, txdesc.addr1, ETH_ALEN);
-		wrqu.addr.sa_family = ARPHRD_ETHER;
-		wireless_send_event(dev, IWEVTXDROP, &wrqu, NULL);
-	} else
-		show_dump = 1;
-
-	if (local->iw_mode == IW_MODE_MASTER ||
-	    local->iw_mode == IW_MODE_REPEAT ||
-	    local->wds_type & HOSTAP_WDS_AP_CLIENT) {
-		struct sk_buff *skb;
-		skb = dev_alloc_skb(sizeof(txdesc));
-		if (skb) {
-			skb_put_data(skb, &txdesc, sizeof(txdesc));
-			skb_queue_tail(&local->sta_tx_exc_list, skb);
-			tasklet_schedule(&local->sta_tx_exc_tasklet);
-		}
-	}
-
-	if (txdesc.sw_support)
-		hostap_tx_callback(local, &txdesc, 0, payload);
-	kfree(payload);
-
-	if (!show_dump)
-		return;
-
-	PDEBUG(DEBUG_EXTRA, "%s: TXEXC - status=0x%04x (%s%s%s%s)"
-	       " tx_control=%04x\n",
-	       dev->name, status,
-	       status & HFA384X_TX_STATUS_RETRYERR ? "[RetryErr]" : "",
-	       status & HFA384X_TX_STATUS_AGEDERR ? "[AgedErr]" : "",
-	       status & HFA384X_TX_STATUS_DISCON ? "[Discon]" : "",
-	       status & HFA384X_TX_STATUS_FORMERR ? "[FormErr]" : "",
-	       le16_to_cpu(txdesc.tx_control));
-
-	fc = le16_to_cpu(txdesc.frame_control);
-	PDEBUG(DEBUG_EXTRA, "   retry_count=%d tx_rate=%d fc=0x%04x "
-	       "(%s%s%s::%d%s%s)\n",
-	       txdesc.retry_count, txdesc.tx_rate, fc,
-	       ieee80211_is_mgmt(txdesc.frame_control) ? "Mgmt" : "",
-	       ieee80211_is_ctl(txdesc.frame_control) ? "Ctrl" : "",
-	       ieee80211_is_data(txdesc.frame_control) ? "Data" : "",
-	       (fc & IEEE80211_FCTL_STYPE) >> 4,
-	       ieee80211_has_tods(txdesc.frame_control) ? " ToDS" : "",
-	       ieee80211_has_fromds(txdesc.frame_control) ? " FromDS" : "");
-	PDEBUG(DEBUG_EXTRA, "   A1=%pM A2=%pM A3=%pM A4=%pM\n",
-	       txdesc.addr1, txdesc.addr2,
-	       txdesc.addr3, txdesc.addr4);
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static void hostap_info_tasklet(struct tasklet_struct *t)
-{
-	local_info_t *local = from_tasklet(local, t, info_tasklet);
-	struct sk_buff *skb;
-
-	while ((skb = skb_dequeue(&local->info_list)) != NULL) {
-		hostap_info_process(local, skb);
-		dev_kfree_skb(skb);
-	}
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static void prism2_info(local_info_t *local)
-{
-	struct net_device *dev = local->dev;
-	u16 fid;
-	int res, left;
-	struct hfa384x_info_frame info;
-	struct sk_buff *skb;
-
-	fid = HFA384X_INW(HFA384X_INFOFID_OFF);
-
-	spin_lock(&local->baplock);
-	res = hfa384x_setup_bap(dev, BAP0, fid, 0);
-	if (!res)
-		res = hfa384x_from_bap(dev, BAP0, &info, sizeof(info));
-	if (res) {
-		spin_unlock(&local->baplock);
-		printk(KERN_DEBUG "Could not get info frame (fid=0x%04x)\n",
-		       fid);
-		if (res == -ETIMEDOUT) {
-			schedule_work(&local->reset_queue);
-		}
-		goto out;
-	}
-
-	left = (le16_to_cpu(info.len) - 1) * 2;
-
-	if (info.len & cpu_to_le16(0x8000) || info.len == 0 || left > 2060) {
-		/* data register seems to give 0x8000 in some error cases even
-		 * though busy bit is not set in offset register;
-		 * in addition, length must be at least 1 due to type field */
-		spin_unlock(&local->baplock);
-		printk(KERN_DEBUG "%s: Received info frame with invalid "
-		       "length 0x%04x (type 0x%04x)\n", dev->name,
-		       le16_to_cpu(info.len), le16_to_cpu(info.type));
-		goto out;
-	}
-
-	skb = dev_alloc_skb(sizeof(info) + left);
-	if (skb == NULL) {
-		spin_unlock(&local->baplock);
-		printk(KERN_DEBUG "%s: Could not allocate skb for info "
-		       "frame\n", dev->name);
-		goto out;
-	}
-
-	skb_put_data(skb, &info, sizeof(info));
-	if (left > 0 && hfa384x_from_bap(dev, BAP0, skb_put(skb, left), left))
-	{
-		spin_unlock(&local->baplock);
-		printk(KERN_WARNING "%s: Info frame read failed (fid=0x%04x, "
-		       "len=0x%04x, type=0x%04x\n", dev->name, fid,
-		       le16_to_cpu(info.len), le16_to_cpu(info.type));
-		dev_kfree_skb(skb);
-		goto out;
-	}
-	spin_unlock(&local->baplock);
-
-	skb_queue_tail(&local->info_list, skb);
-	tasklet_schedule(&local->info_tasklet);
-
- out:
-	HFA384X_OUTW(HFA384X_EV_INFO, HFA384X_EVACK_OFF);
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static void hostap_bap_tasklet(struct tasklet_struct *t)
-{
-	local_info_t *local = from_tasklet(local, t, bap_tasklet);
-	struct net_device *dev = local->dev;
-	u16 ev;
-	int frames = 30;
-
-	if (local->func->card_present && !local->func->card_present(local))
-		return;
-
-	set_bit(HOSTAP_BITS_BAP_TASKLET, &local->bits);
-
-	/* Process all pending BAP events without generating new interrupts
-	 * for them */
-	while (frames-- > 0) {
-		ev = HFA384X_INW(HFA384X_EVSTAT_OFF);
-		if (ev == 0xffff || !(ev & HFA384X_BAP0_EVENTS))
-			break;
-		if (ev & HFA384X_EV_RX)
-			prism2_rx(local);
-		if (ev & HFA384X_EV_INFO)
-			prism2_info(local);
-		if (ev & HFA384X_EV_TX)
-			prism2_tx_ev(local);
-		if (ev & HFA384X_EV_TXEXC)
-			prism2_txexc(local);
-	}
-
-	set_bit(HOSTAP_BITS_BAP_TASKLET2, &local->bits);
-	clear_bit(HOSTAP_BITS_BAP_TASKLET, &local->bits);
-
-	/* Enable interrupts for new BAP events */
-	hfa384x_events_all(dev);
-	clear_bit(HOSTAP_BITS_BAP_TASKLET2, &local->bits);
-}
-
-
-/* Called only from hardware IRQ */
-static void prism2_infdrop(struct net_device *dev)
-{
-	static unsigned long last_inquire = 0;
-
-	PDEBUG(DEBUG_EXTRA, "%s: INFDROP event\n", dev->name);
-
-	/* some firmware versions seem to get stuck with
-	 * full CommTallies in high traffic load cases; every
-	 * packet will then cause INFDROP event and CommTallies
-	 * info frame will not be sent automatically. Try to
-	 * get out of this state by inquiring CommTallies. */
-	if (!last_inquire || time_after(jiffies, last_inquire + HZ)) {
-		hfa384x_cmd_callback(dev, HFA384X_CMDCODE_INQUIRE,
-				     HFA384X_INFO_COMMTALLIES, NULL, 0);
-		last_inquire = jiffies;
-	}
-}
-
-
-/* Called only from hardware IRQ */
-static void prism2_ev_tick(struct net_device *dev)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	u16 evstat, inten;
-	static int prev_stuck = 0;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (time_after(jiffies, local->last_tick_timer + 5 * HZ) &&
-	    local->last_tick_timer) {
-		evstat = HFA384X_INW(HFA384X_EVSTAT_OFF);
-		inten = HFA384X_INW(HFA384X_INTEN_OFF);
-		if (!prev_stuck) {
-			printk(KERN_INFO "%s: SW TICK stuck? "
-			       "bits=0x%lx EvStat=%04x IntEn=%04x\n",
-			       dev->name, local->bits, evstat, inten);
-		}
-		local->sw_tick_stuck++;
-		if ((evstat & HFA384X_BAP0_EVENTS) &&
-		    (inten & HFA384X_BAP0_EVENTS)) {
-			printk(KERN_INFO "%s: trying to recover from IRQ "
-			       "hang\n", dev->name);
-			hfa384x_events_no_bap0(dev);
-		}
-		prev_stuck = 1;
-	} else
-		prev_stuck = 0;
-}
-
-
-/* Called only from hardware IRQ */
-static void prism2_check_magic(local_info_t *local)
-{
-	/* at least PCI Prism2.5 with bus mastering seems to sometimes
-	 * return 0x0000 in SWSUPPORT0 for unknown reason, but re-reading the
-	 * register once or twice seems to get the correct value.. PCI cards
-	 * cannot anyway be removed during normal operation, so there is not
-	 * really any need for this verification with them. */
-
-#ifndef PRISM2_PCI
-#ifndef final_version
-	static unsigned long last_magic_err = 0;
-	struct net_device *dev = local->dev;
-
-	if (HFA384X_INW(HFA384X_SWSUPPORT0_OFF) != HFA384X_MAGIC) {
-		if (!local->hw_ready)
-			return;
-		HFA384X_OUTW(0xffff, HFA384X_EVACK_OFF);
-		if (time_after(jiffies, last_magic_err + 10 * HZ)) {
-			printk("%s: Interrupt, but SWSUPPORT0 does not match: "
-			       "%04X != %04X - card removed?\n", dev->name,
-			       HFA384X_INW(HFA384X_SWSUPPORT0_OFF),
-			       HFA384X_MAGIC);
-			last_magic_err = jiffies;
-		} else if (net_ratelimit()) {
-			printk(KERN_DEBUG "%s: interrupt - SWSUPPORT0=%04x "
-			       "MAGIC=%04x\n", dev->name,
-			       HFA384X_INW(HFA384X_SWSUPPORT0_OFF),
-			       HFA384X_MAGIC);
-		}
-		if (HFA384X_INW(HFA384X_SWSUPPORT0_OFF) != 0xffff)
-			schedule_work(&local->reset_queue);
-		return;
-	}
-#endif /* final_version */
-#endif /* !PRISM2_PCI */
-}
-
-
-/* Called only from hardware IRQ */
-static irqreturn_t prism2_interrupt(int irq, void *dev_id)
-{
-	struct net_device *dev = dev_id;
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int events = 0;
-	u16 ev;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	/* Detect early interrupt before driver is fully configured */
-	spin_lock(&local->irq_init_lock);
-	if (!dev->base_addr) {
-		if (net_ratelimit()) {
-			printk(KERN_DEBUG "%s: Interrupt, but dev not configured\n",
-			       dev->name);
-		}
-		spin_unlock(&local->irq_init_lock);
-		return IRQ_HANDLED;
-	}
-	spin_unlock(&local->irq_init_lock);
-
-	prism2_io_debug_add(dev, PRISM2_IO_DEBUG_CMD_INTERRUPT, 0, 0);
-
-	if (local->func->card_present && !local->func->card_present(local)) {
-		if (net_ratelimit()) {
-			printk(KERN_DEBUG "%s: Interrupt, but dev not OK\n",
-			       dev->name);
-		}
-		return IRQ_HANDLED;
-	}
-
-	prism2_check_magic(local);
-
-	for (;;) {
-		ev = HFA384X_INW(HFA384X_EVSTAT_OFF);
-		if (ev == 0xffff) {
-			if (local->shutdown)
-				return IRQ_HANDLED;
-			HFA384X_OUTW(0xffff, HFA384X_EVACK_OFF);
-			printk(KERN_DEBUG "%s: prism2_interrupt: ev=0xffff\n",
-			       dev->name);
-			return IRQ_HANDLED;
-		}
-
-		ev &= HFA384X_INW(HFA384X_INTEN_OFF);
-		if (ev == 0)
-			break;
-
-		if (ev & HFA384X_EV_CMD) {
-			prism2_cmd_ev(dev);
-		}
-
-		/* Above events are needed even before hw is ready, but other
-		 * events should be skipped during initialization. This may
-		 * change for AllocEv if allocate_fid is implemented without
-		 * busy waiting. */
-		if (!local->hw_ready || local->hw_resetting ||
-		    !local->dev_enabled) {
-			ev = HFA384X_INW(HFA384X_EVSTAT_OFF);
-			if (ev & HFA384X_EV_CMD)
-				goto next_event;
-			if ((ev & HFA384X_EVENT_MASK) == 0)
-				return IRQ_HANDLED;
-			if (local->dev_enabled && (ev & ~HFA384X_EV_TICK) &&
-			    net_ratelimit()) {
-				printk(KERN_DEBUG "%s: prism2_interrupt: hw "
-				       "not ready; skipping events 0x%04x "
-				       "(IntEn=0x%04x)%s%s%s\n",
-				       dev->name, ev,
-				       HFA384X_INW(HFA384X_INTEN_OFF),
-				       !local->hw_ready ? " (!hw_ready)" : "",
-				       local->hw_resetting ?
-				       " (hw_resetting)" : "",
-				       !local->dev_enabled ?
-				       " (!dev_enabled)" : "");
-			}
-			HFA384X_OUTW(ev, HFA384X_EVACK_OFF);
-			return IRQ_HANDLED;
-		}
-
-		if (ev & HFA384X_EV_TICK) {
-			prism2_ev_tick(dev);
-			HFA384X_OUTW(HFA384X_EV_TICK, HFA384X_EVACK_OFF);
-		}
-
-		if (ev & HFA384X_EV_ALLOC) {
-			prism2_alloc_ev(dev);
-			HFA384X_OUTW(HFA384X_EV_ALLOC, HFA384X_EVACK_OFF);
-		}
-
-		/* Reading data from the card is quite time consuming, so do it
-		 * in tasklets. TX, TXEXC, RX, and INFO events will be ACKed
-		 * and unmasked after needed data has been read completely. */
-		if (ev & HFA384X_BAP0_EVENTS) {
-			hfa384x_events_no_bap0(dev);
-			tasklet_schedule(&local->bap_tasklet);
-		}
-
-#ifndef final_version
-		if (ev & HFA384X_EV_WTERR) {
-			PDEBUG(DEBUG_EXTRA, "%s: WTERR event\n", dev->name);
-			HFA384X_OUTW(HFA384X_EV_WTERR, HFA384X_EVACK_OFF);
-		}
-#endif /* final_version */
-
-		if (ev & HFA384X_EV_INFDROP) {
-			prism2_infdrop(dev);
-			HFA384X_OUTW(HFA384X_EV_INFDROP, HFA384X_EVACK_OFF);
-		}
-
-	next_event:
-		events++;
-		if (events >= PRISM2_MAX_INTERRUPT_EVENTS) {
-			PDEBUG(DEBUG_EXTRA, "prism2_interrupt: >%d events "
-			       "(EvStat=0x%04x)\n",
-			       PRISM2_MAX_INTERRUPT_EVENTS,
-			       HFA384X_INW(HFA384X_EVSTAT_OFF));
-			break;
-		}
-	}
-	prism2_io_debug_add(dev, PRISM2_IO_DEBUG_CMD_INTERRUPT, 0, 1);
-	return IRQ_RETVAL(events);
-}
-
-
-static void prism2_check_sta_fw_version(local_info_t *local)
-{
-	struct hfa384x_comp_ident comp;
-	int id, variant, major, minor;
-
-	if (hfa384x_get_rid(local->dev, HFA384X_RID_STAID,
-			    &comp, sizeof(comp), 1) < 0)
-		return;
-
-	local->fw_ap = 0;
-	id = le16_to_cpu(comp.id);
-	if (id != HFA384X_COMP_ID_STA) {
-		if (id == HFA384X_COMP_ID_FW_AP)
-			local->fw_ap = 1;
-		return;
-	}
-
-	major = __le16_to_cpu(comp.major);
-	minor = __le16_to_cpu(comp.minor);
-	variant = __le16_to_cpu(comp.variant);
-	local->sta_fw_ver = PRISM2_FW_VER(major, minor, variant);
-
-	/* Station firmware versions before 1.4.x seem to have a bug in
-	 * firmware-based WEP encryption when using Host AP mode, so use
-	 * host_encrypt as a default for them. Firmware version 1.4.9 is the
-	 * first one that has been seen to produce correct encryption, but the
-	 * bug might be fixed before that (although, at least 1.4.2 is broken).
-	 */
-	local->fw_encrypt_ok = local->sta_fw_ver >= PRISM2_FW_VER(1,4,9);
-
-	if (local->iw_mode == IW_MODE_MASTER && !local->host_encrypt &&
-	    !local->fw_encrypt_ok) {
-		printk(KERN_DEBUG "%s: defaulting to host-based encryption as "
-		       "a workaround for firmware bug in Host AP mode WEP\n",
-		       local->dev->name);
-		local->host_encrypt = 1;
-	}
-
-	/* IEEE 802.11 standard compliant WDS frames (4 addresses) were broken
-	 * in station firmware versions before 1.5.x. With these versions, the
-	 * driver uses a workaround with bogus frame format (4th address after
-	 * the payload). This is not compatible with other AP devices. Since
-	 * the firmware bug is fixed in the latest station firmware versions,
-	 * automatically enable standard compliant mode for cards using station
-	 * firmware version 1.5.0 or newer. */
-	if (local->sta_fw_ver >= PRISM2_FW_VER(1,5,0))
-		local->wds_type |= HOSTAP_WDS_STANDARD_FRAME;
-	else {
-		printk(KERN_DEBUG "%s: defaulting to bogus WDS frame as a "
-		       "workaround for firmware bug in Host AP mode WDS\n",
-		       local->dev->name);
-	}
-
-	hostap_check_sta_fw_version(local->ap, local->sta_fw_ver);
-}
-
-
-static void hostap_passive_scan(struct timer_list *t)
-{
-	local_info_t *local = from_timer(local, t, passive_scan_timer);
-	struct net_device *dev = local->dev;
-	u16 chan;
-
-	if (local->passive_scan_interval <= 0)
-		return;
-
-	if (local->passive_scan_state == PASSIVE_SCAN_LISTEN) {
-		int max_tries = 16;
-
-		/* Even though host system does not really know when the WLAN
-		 * MAC is sending frames, try to avoid changing channels for
-		 * passive scanning when a host-generated frame is being
-		 * transmitted */
-		if (test_bit(HOSTAP_BITS_TRANSMIT, &local->bits)) {
-			printk(KERN_DEBUG "%s: passive scan detected pending "
-			       "TX - delaying\n", dev->name);
-			local->passive_scan_timer.expires = jiffies + HZ / 10;
-			add_timer(&local->passive_scan_timer);
-			return;
-		}
-
-		do {
-			local->passive_scan_channel++;
-			if (local->passive_scan_channel > 14)
-				local->passive_scan_channel = 1;
-			max_tries--;
-		} while (!(local->channel_mask &
-			   (1 << (local->passive_scan_channel - 1))) &&
-			 max_tries > 0);
-
-		if (max_tries == 0) {
-			printk(KERN_INFO "%s: no allowed passive scan channels"
-			       " found\n", dev->name);
-			return;
-		}
-
-		printk(KERN_DEBUG "%s: passive scan channel %d\n",
-		       dev->name, local->passive_scan_channel);
-		chan = local->passive_scan_channel;
-		local->passive_scan_state = PASSIVE_SCAN_WAIT;
-		local->passive_scan_timer.expires = jiffies + HZ / 10;
-	} else {
-		chan = local->channel;
-		local->passive_scan_state = PASSIVE_SCAN_LISTEN;
-		local->passive_scan_timer.expires = jiffies +
-			local->passive_scan_interval * HZ;
-	}
-
-	if (hfa384x_cmd_callback(dev, HFA384X_CMDCODE_TEST |
-				 (HFA384X_TEST_CHANGE_CHANNEL << 8),
-				 chan, NULL, 0))
-		printk(KERN_ERR "%s: passive scan channel set %d "
-		       "failed\n", dev->name, chan);
-
-	add_timer(&local->passive_scan_timer);
-}
-
-
-/* Called only as a scheduled task when communications quality values should
- * be updated. */
-static void handle_comms_qual_update(struct work_struct *work)
-{
-	local_info_t *local =
-		container_of(work, local_info_t, comms_qual_update);
-	prism2_update_comms_qual(local->dev);
-}
-
-
-/* Software watchdog - called as a timer. Hardware interrupt (Tick event) is
- * used to monitor that local->last_tick_timer is being updated. If not,
- * interrupt busy-loop is assumed and driver tries to recover by masking out
- * some events. */
-static void hostap_tick_timer(struct timer_list *t)
-{
-	static unsigned long last_inquire = 0;
-	local_info_t *local = from_timer(local, t, tick_timer);
-	local->last_tick_timer = jiffies;
-
-	/* Inquire CommTallies every 10 seconds to keep the statistics updated
-	 * more often during low load and when using 32-bit tallies. */
-	if ((!last_inquire || time_after(jiffies, last_inquire + 10 * HZ)) &&
-	    !local->hw_downloading && local->hw_ready &&
-	    !local->hw_resetting && local->dev_enabled) {
-		hfa384x_cmd_callback(local->dev, HFA384X_CMDCODE_INQUIRE,
-				     HFA384X_INFO_COMMTALLIES, NULL, 0);
-		last_inquire = jiffies;
-	}
-
-	if ((local->last_comms_qual_update == 0 ||
-	     time_after(jiffies, local->last_comms_qual_update + 10 * HZ)) &&
-	    (local->iw_mode == IW_MODE_INFRA ||
-	     local->iw_mode == IW_MODE_ADHOC)) {
-		schedule_work(&local->comms_qual_update);
-	}
-
-	local->tick_timer.expires = jiffies + 2 * HZ;
-	add_timer(&local->tick_timer);
-}
-
-
-#if !defined(PRISM2_NO_PROCFS_DEBUG) && defined(CONFIG_PROC_FS)
-static u16 hfa384x_read_reg(struct net_device *dev, u16 reg)
-{
-	return HFA384X_INW(reg);
-}
-
-static int prism2_registers_proc_show(struct seq_file *m, void *v)
-{
-	local_info_t *local = m->private;
-
-#define SHOW_REG(n) \
-  seq_printf(m, #n "=%04x\n", hfa384x_read_reg(local->dev, HFA384X_##n##_OFF))
-
-	SHOW_REG(CMD);
-	SHOW_REG(PARAM0);
-	SHOW_REG(PARAM1);
-	SHOW_REG(PARAM2);
-	SHOW_REG(STATUS);
-	SHOW_REG(RESP0);
-	SHOW_REG(RESP1);
-	SHOW_REG(RESP2);
-	SHOW_REG(INFOFID);
-	SHOW_REG(CONTROL);
-	SHOW_REG(SELECT0);
-	SHOW_REG(SELECT1);
-	SHOW_REG(OFFSET0);
-	SHOW_REG(OFFSET1);
-	SHOW_REG(RXFID);
-	SHOW_REG(ALLOCFID);
-	SHOW_REG(TXCOMPLFID);
-	SHOW_REG(SWSUPPORT0);
-	SHOW_REG(SWSUPPORT1);
-	SHOW_REG(SWSUPPORT2);
-	SHOW_REG(EVSTAT);
-	SHOW_REG(INTEN);
-	SHOW_REG(EVACK);
-	/* Do not read data registers, because they change the state of the
-	 * MAC (offset += 2) */
-	/* SHOW_REG(DATA0); */
-	/* SHOW_REG(DATA1); */
-	SHOW_REG(AUXPAGE);
-	SHOW_REG(AUXOFFSET);
-	/* SHOW_REG(AUXDATA); */
-#ifdef PRISM2_PCI
-	SHOW_REG(PCICOR);
-	SHOW_REG(PCIHCR);
-	SHOW_REG(PCI_M0_ADDRH);
-	SHOW_REG(PCI_M0_ADDRL);
-	SHOW_REG(PCI_M0_LEN);
-	SHOW_REG(PCI_M0_CTL);
-	SHOW_REG(PCI_STATUS);
-	SHOW_REG(PCI_M1_ADDRH);
-	SHOW_REG(PCI_M1_ADDRL);
-	SHOW_REG(PCI_M1_LEN);
-	SHOW_REG(PCI_M1_CTL);
-#endif /* PRISM2_PCI */
-
-	return 0;
-}
-#endif
-
-struct set_tim_data {
-	struct list_head list;
-	int aid;
-	int set;
-};
-
-static int prism2_set_tim(struct net_device *dev, int aid, int set)
-{
-	struct list_head *ptr;
-	struct set_tim_data *new_entry;
-	struct hostap_interface *iface;
-	local_info_t *local;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	new_entry = kzalloc(sizeof(*new_entry), GFP_ATOMIC);
-	if (new_entry == NULL)
-		return -ENOMEM;
-
-	new_entry->aid = aid;
-	new_entry->set = set;
-
-	spin_lock_bh(&local->set_tim_lock);
-	list_for_each(ptr, &local->set_tim_list) {
-		struct set_tim_data *entry =
-			list_entry(ptr, struct set_tim_data, list);
-		if (entry->aid == aid) {
-			PDEBUG(DEBUG_PS2, "%s: prism2_set_tim: aid=%d "
-			       "set=%d ==> %d\n",
-			       local->dev->name, aid, entry->set, set);
-			entry->set = set;
-			kfree(new_entry);
-			new_entry = NULL;
-			break;
-		}
-	}
-	if (new_entry)
-		list_add_tail(&new_entry->list, &local->set_tim_list);
-	spin_unlock_bh(&local->set_tim_lock);
-
-	schedule_work(&local->set_tim_queue);
-
-	return 0;
-}
-
-
-static void handle_set_tim_queue(struct work_struct *work)
-{
-	local_info_t *local = container_of(work, local_info_t, set_tim_queue);
-	struct set_tim_data *entry;
-	u16 val;
-
-	for (;;) {
-		entry = NULL;
-		spin_lock_bh(&local->set_tim_lock);
-		if (!list_empty(&local->set_tim_list)) {
-			entry = list_entry(local->set_tim_list.next,
-					   struct set_tim_data, list);
-			list_del(&entry->list);
-		}
-		spin_unlock_bh(&local->set_tim_lock);
-		if (!entry)
-			break;
-
-		PDEBUG(DEBUG_PS2, "%s: handle_set_tim_queue: aid=%d set=%d\n",
-		       local->dev->name, entry->aid, entry->set);
-
-		val = entry->aid;
-		if (entry->set)
-			val |= 0x8000;
-		if (hostap_set_word(local->dev, HFA384X_RID_CNFTIMCTRL, val)) {
-			printk(KERN_DEBUG "%s: set_tim failed (aid=%d "
-			       "set=%d)\n",
-			       local->dev->name, entry->aid, entry->set);
-		}
-
-		kfree(entry);
-	}
-}
-
-
-static void prism2_clear_set_tim_queue(local_info_t *local)
-{
-	struct list_head *ptr, *n;
-
-	list_for_each_safe(ptr, n, &local->set_tim_list) {
-		struct set_tim_data *entry;
-		entry = list_entry(ptr, struct set_tim_data, list);
-		list_del(&entry->list);
-		kfree(entry);
-	}
-}
-
-
-/*
- * HostAP uses two layers of net devices, where the inner
- * layer gets called all the time from the outer layer.
- * This is a natural nesting, which needs a split lock type.
- */
-static struct lock_class_key hostap_netdev_xmit_lock_key;
-static struct lock_class_key hostap_netdev_addr_lock_key;
-
-static void prism2_set_lockdep_class_one(struct net_device *dev,
-					 struct netdev_queue *txq,
-					 void *_unused)
-{
-	lockdep_set_class(&txq->_xmit_lock,
-			  &hostap_netdev_xmit_lock_key);
-}
-
-static void prism2_set_lockdep_class(struct net_device *dev)
-{
-	lockdep_set_class(&dev->addr_list_lock,
-			  &hostap_netdev_addr_lock_key);
-	netdev_for_each_tx_queue(dev, prism2_set_lockdep_class_one, NULL);
-}
-
-static struct net_device *
-prism2_init_local_data(struct prism2_helper_functions *funcs, int card_idx,
-		       struct device *sdev)
-{
-	struct net_device *dev;
-	struct hostap_interface *iface;
-	struct local_info *local;
-	int len, i, ret;
-
-	if (funcs == NULL)
-		return NULL;
-
-	len = strlen(dev_template);
-	if (len >= IFNAMSIZ || strstr(dev_template, "%d") == NULL) {
-		printk(KERN_WARNING "hostap: Invalid dev_template='%s'\n",
-		       dev_template);
-		return NULL;
-	}
-
-	len = sizeof(struct hostap_interface) +
-		3 + sizeof(struct local_info) +
-		3 + sizeof(struct ap_data);
-
-	dev = alloc_etherdev(len);
-	if (dev == NULL)
-		return NULL;
-
-	iface = netdev_priv(dev);
-	local = (struct local_info *) ((((long) (iface + 1)) + 3) & ~3);
-	local->ap = (struct ap_data *) ((((long) (local + 1)) + 3) & ~3);
-	local->dev = iface->dev = dev;
-	iface->local = local;
-	iface->type = HOSTAP_INTERFACE_MASTER;
-	INIT_LIST_HEAD(&local->hostap_interfaces);
-
-	local->hw_module = THIS_MODULE;
-
-#ifdef PRISM2_IO_DEBUG
-	local->io_debug_enabled = 1;
-#endif /* PRISM2_IO_DEBUG */
-
-	local->func = funcs;
-	local->func->cmd = hfa384x_cmd;
-	local->func->read_regs = hfa384x_read_regs;
-	local->func->get_rid = hfa384x_get_rid;
-	local->func->set_rid = hfa384x_set_rid;
-	local->func->hw_enable = prism2_hw_enable;
-	local->func->hw_config = prism2_hw_config;
-	local->func->hw_reset = prism2_hw_reset;
-	local->func->hw_shutdown = prism2_hw_shutdown;
-	local->func->reset_port = prism2_reset_port;
-	local->func->schedule_reset = prism2_schedule_reset;
-#ifdef PRISM2_DOWNLOAD_SUPPORT
-	local->func->read_aux_proc_ops = &prism2_download_aux_dump_proc_ops;
-	local->func->download = prism2_download;
-#endif /* PRISM2_DOWNLOAD_SUPPORT */
-	local->func->tx = prism2_tx_80211;
-	local->func->set_tim = prism2_set_tim;
-	local->func->need_tx_headroom = 0; /* no need to add txdesc in
-					    * skb->data (FIX: maybe for DMA bus
-					    * mastering? */
-
-	local->mtu = mtu;
-
-	rwlock_init(&local->iface_lock);
-	spin_lock_init(&local->txfidlock);
-	spin_lock_init(&local->cmdlock);
-	spin_lock_init(&local->baplock);
-	spin_lock_init(&local->lock);
-	spin_lock_init(&local->irq_init_lock);
-	mutex_init(&local->rid_bap_mtx);
-
-	if (card_idx < 0 || card_idx >= MAX_PARM_DEVICES)
-		card_idx = 0;
-	local->card_idx = card_idx;
-
-	len = strlen(essid);
-	memcpy(local->essid, essid,
-	       len > MAX_SSID_LEN ? MAX_SSID_LEN : len);
-	local->essid[MAX_SSID_LEN] = '\0';
-	i = GET_INT_PARM(iw_mode, card_idx);
-	if ((i >= IW_MODE_ADHOC && i <= IW_MODE_REPEAT) ||
-	    i == IW_MODE_MONITOR) {
-		local->iw_mode = i;
-	} else {
-		printk(KERN_WARNING "prism2: Unknown iw_mode %d; using "
-		       "IW_MODE_MASTER\n", i);
-		local->iw_mode = IW_MODE_MASTER;
-	}
-	local->channel = GET_INT_PARM(channel, card_idx);
-	local->beacon_int = GET_INT_PARM(beacon_int, card_idx);
-	local->dtim_period = GET_INT_PARM(dtim_period, card_idx);
-	local->wds_max_connections = 16;
-	local->tx_control = HFA384X_TX_CTRL_FLAGS;
-	local->manual_retry_count = -1;
-	local->rts_threshold = 2347;
-	local->fragm_threshold = 2346;
-	local->rssi_to_dBm = 100; /* default; to be overriden by
-				   * cnfDbmAdjust, if available */
-	local->auth_algs = PRISM2_AUTH_OPEN | PRISM2_AUTH_SHARED_KEY;
-	local->sram_type = -1;
-	local->scan_channel_mask = 0xffff;
-	local->monitor_type = PRISM2_MONITOR_RADIOTAP;
-
-	/* Initialize task queue structures */
-	INIT_WORK(&local->reset_queue, handle_reset_queue);
-	INIT_WORK(&local->set_multicast_list_queue,
-		  hostap_set_multicast_list_queue);
-
-	INIT_WORK(&local->set_tim_queue, handle_set_tim_queue);
-	INIT_LIST_HEAD(&local->set_tim_list);
-	spin_lock_init(&local->set_tim_lock);
-
-	INIT_WORK(&local->comms_qual_update, handle_comms_qual_update);
-
-	/* Initialize tasklets for handling hardware IRQ related operations
-	 * outside hw IRQ handler */
-	tasklet_setup(&local->bap_tasklet, hostap_bap_tasklet);
-	tasklet_setup(&local->info_tasklet, hostap_info_tasklet);
-	hostap_info_init(local);
-
-	tasklet_setup(&local->rx_tasklet, hostap_rx_tasklet);
-	skb_queue_head_init(&local->rx_list);
-
-	tasklet_setup(&local->sta_tx_exc_tasklet,
-			    hostap_sta_tx_exc_tasklet);
-	skb_queue_head_init(&local->sta_tx_exc_list);
-
-	INIT_LIST_HEAD(&local->cmd_queue);
-	init_waitqueue_head(&local->hostscan_wq);
-
-	lib80211_crypt_info_init(&local->crypt_info, dev->name, &local->lock);
-
-	timer_setup(&local->passive_scan_timer, hostap_passive_scan, 0);
-	timer_setup(&local->tick_timer, hostap_tick_timer, 0);
-	local->tick_timer.expires = jiffies + 2 * HZ;
-	add_timer(&local->tick_timer);
-
-	INIT_LIST_HEAD(&local->bss_list);
-
-	hostap_setup_dev(dev, local, HOSTAP_INTERFACE_MASTER);
-
-	dev->type = ARPHRD_IEEE80211;
-	dev->header_ops = &hostap_80211_ops;
-
-	rtnl_lock();
-	ret = dev_alloc_name(dev, "wifi%d");
-	SET_NETDEV_DEV(dev, sdev);
-	if (ret >= 0)
-		ret = register_netdevice(dev);
-
-	prism2_set_lockdep_class(dev);
-	rtnl_unlock();
-	if (ret < 0) {
-		printk(KERN_WARNING "%s: register netdevice failed!\n",
-		       dev_info);
-		goto fail;
-	}
-	printk(KERN_INFO "%s: Registered netdevice %s\n", dev_info, dev->name);
-
-	hostap_init_data(local);
-	return dev;
-
- fail:
-	free_netdev(dev);
-	return NULL;
-}
-
-
-static int hostap_hw_ready(struct net_device *dev)
-{
-	struct hostap_interface *iface;
-	struct local_info *local;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-	local->ddev = hostap_add_interface(local, HOSTAP_INTERFACE_MAIN, 0,
-					   "", dev_template);
-
-	if (local->ddev) {
-		if (local->iw_mode == IW_MODE_INFRA ||
-		    local->iw_mode == IW_MODE_ADHOC) {
-			netif_carrier_off(local->dev);
-			netif_carrier_off(local->ddev);
-		}
-		hostap_init_proc(local);
-#ifndef PRISM2_NO_PROCFS_DEBUG
-		proc_create_single_data("registers", 0, local->proc,
-				 prism2_registers_proc_show, local);
-#endif /* PRISM2_NO_PROCFS_DEBUG */
-		hostap_init_ap_proc(local);
-		return 0;
-	}
-
-	return -1;
-}
-
-
-static void prism2_free_local_data(struct net_device *dev)
-{
-	struct hostap_tx_callback_info *tx_cb, *tx_cb_prev;
-	int i;
-	struct hostap_interface *iface;
-	struct local_info *local;
-	struct list_head *ptr, *n;
-
-	if (dev == NULL)
-		return;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	/* Unregister all netdevs before freeing local data. */
-	list_for_each_safe(ptr, n, &local->hostap_interfaces) {
-		iface = list_entry(ptr, struct hostap_interface, list);
-		if (iface->type == HOSTAP_INTERFACE_MASTER) {
-			/* special handling for this interface below */
-			continue;
-		}
-		hostap_remove_interface(iface->dev, 0, 1);
-	}
-
-	unregister_netdev(local->dev);
-
-	flush_work(&local->reset_queue);
-	flush_work(&local->set_multicast_list_queue);
-	flush_work(&local->set_tim_queue);
-#ifndef PRISM2_NO_STATION_MODES
-	flush_work(&local->info_queue);
-#endif
-	flush_work(&local->comms_qual_update);
-
-	lib80211_crypt_info_free(&local->crypt_info);
-
-	if (timer_pending(&local->passive_scan_timer))
-		del_timer(&local->passive_scan_timer);
-
-	if (timer_pending(&local->tick_timer))
-		del_timer(&local->tick_timer);
-
-	prism2_clear_cmd_queue(local);
-
-	skb_queue_purge(&local->info_list);
-	skb_queue_purge(&local->rx_list);
-	skb_queue_purge(&local->sta_tx_exc_list);
-
-	if (local->dev_enabled)
-		prism2_callback(local, PRISM2_CALLBACK_DISABLE);
-
-	if (local->ap != NULL)
-		hostap_free_data(local->ap);
-
-#ifndef PRISM2_NO_PROCFS_DEBUG
-	if (local->proc != NULL)
-		remove_proc_entry("registers", local->proc);
-#endif /* PRISM2_NO_PROCFS_DEBUG */
-	hostap_remove_proc(local);
-
-	tx_cb = local->tx_callback;
-	while (tx_cb != NULL) {
-		tx_cb_prev = tx_cb;
-		tx_cb = tx_cb->next;
-		kfree(tx_cb_prev);
-	}
-
-	hostap_set_hostapd(local, 0, 0);
-	hostap_set_hostapd_sta(local, 0, 0);
-
-	for (i = 0; i < PRISM2_FRAG_CACHE_LEN; i++) {
-		if (local->frag_cache[i].skb != NULL)
-			dev_kfree_skb(local->frag_cache[i].skb);
-	}
-
-#ifdef PRISM2_DOWNLOAD_SUPPORT
-	prism2_download_free_data(local->dl_pri);
-	prism2_download_free_data(local->dl_sec);
-#endif /* PRISM2_DOWNLOAD_SUPPORT */
-
-	prism2_clear_set_tim_queue(local);
-
-	list_for_each_safe(ptr, n, &local->bss_list) {
-		struct hostap_bss_info *bss =
-			list_entry(ptr, struct hostap_bss_info, list);
-		kfree(bss);
-	}
-
-	kfree(local->pda);
-	kfree(local->last_scan_results);
-	kfree(local->generic_elem);
-
-	free_netdev(local->dev);
-}
-
-
-#if defined(PRISM2_PCI) || defined(PRISM2_PCCARD)
-static void __maybe_unused prism2_suspend(struct net_device *dev)
-{
-	struct hostap_interface *iface;
-	struct local_info *local;
-	union iwreq_data wrqu;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	/* Send disconnect event, e.g., to trigger reassociation after resume
-	 * if wpa_supplicant is used. */
-	memset(&wrqu, 0, sizeof(wrqu));
-	wrqu.ap_addr.sa_family = ARPHRD_ETHER;
-	wireless_send_event(local->dev, SIOCGIWAP, &wrqu, NULL);
-
-	/* Disable hardware and firmware */
-	prism2_hw_shutdown(dev, 0);
-}
-#endif /* PRISM2_PCI || PRISM2_PCCARD */
-
-
-/* These might at some point be compiled separately and used as separate
- * kernel modules or linked into one */
-#ifdef PRISM2_DOWNLOAD_SUPPORT
-#include "hostap_download.c"
-#endif /* PRISM2_DOWNLOAD_SUPPORT */
-
-#ifdef PRISM2_CALLBACK
-/* External hostap_callback.c file can be used to, e.g., blink activity led.
- * This can use platform specific code and must define prism2_callback()
- * function (if PRISM2_CALLBACK is not defined, these function calls are not
- * used. */
-#include "hostap_callback.c"
-#endif /* PRISM2_CALLBACK */
diff --git a/drivers/net/wireless/intersil/hostap/hostap_info.c b/drivers/net/wireless/intersil/hostap/hostap_info.c
deleted file mode 100644
index da8c30f10d92..000000000000
--- a/drivers/net/wireless/intersil/hostap/hostap_info.c
+++ /dev/null
@@ -1,509 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/* Host AP driver Info Frame processing (part of hostap.o module) */
-
-#include <linux/if_arp.h>
-#include <linux/sched.h>
-#include <linux/slab.h>
-#include <linux/export.h>
-#include <linux/etherdevice.h>
-#include "hostap_wlan.h"
-#include "hostap.h"
-#include "hostap_ap.h"
-
-/* Called only as a tasklet (software IRQ) */
-static void prism2_info_commtallies16(local_info_t *local, unsigned char *buf,
-				      int left)
-{
-	struct hfa384x_comm_tallies *tallies;
-
-	if (left < sizeof(struct hfa384x_comm_tallies)) {
-		printk(KERN_DEBUG "%s: too short (len=%d) commtallies "
-		       "info frame\n", local->dev->name, left);
-		return;
-	}
-
-	tallies = (struct hfa384x_comm_tallies *) buf;
-#define ADD_COMM_TALLIES(name) \
-local->comm_tallies.name += le16_to_cpu(tallies->name)
-	ADD_COMM_TALLIES(tx_unicast_frames);
-	ADD_COMM_TALLIES(tx_multicast_frames);
-	ADD_COMM_TALLIES(tx_fragments);
-	ADD_COMM_TALLIES(tx_unicast_octets);
-	ADD_COMM_TALLIES(tx_multicast_octets);
-	ADD_COMM_TALLIES(tx_deferred_transmissions);
-	ADD_COMM_TALLIES(tx_single_retry_frames);
-	ADD_COMM_TALLIES(tx_multiple_retry_frames);
-	ADD_COMM_TALLIES(tx_retry_limit_exceeded);
-	ADD_COMM_TALLIES(tx_discards);
-	ADD_COMM_TALLIES(rx_unicast_frames);
-	ADD_COMM_TALLIES(rx_multicast_frames);
-	ADD_COMM_TALLIES(rx_fragments);
-	ADD_COMM_TALLIES(rx_unicast_octets);
-	ADD_COMM_TALLIES(rx_multicast_octets);
-	ADD_COMM_TALLIES(rx_fcs_errors);
-	ADD_COMM_TALLIES(rx_discards_no_buffer);
-	ADD_COMM_TALLIES(tx_discards_wrong_sa);
-	ADD_COMM_TALLIES(rx_discards_wep_undecryptable);
-	ADD_COMM_TALLIES(rx_message_in_msg_fragments);
-	ADD_COMM_TALLIES(rx_message_in_bad_msg_fragments);
-#undef ADD_COMM_TALLIES
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static void prism2_info_commtallies32(local_info_t *local, unsigned char *buf,
-				      int left)
-{
-	struct hfa384x_comm_tallies32 *tallies;
-
-	if (left < sizeof(struct hfa384x_comm_tallies32)) {
-		printk(KERN_DEBUG "%s: too short (len=%d) commtallies32 "
-		       "info frame\n", local->dev->name, left);
-		return;
-	}
-
-	tallies = (struct hfa384x_comm_tallies32 *) buf;
-#define ADD_COMM_TALLIES(name) \
-local->comm_tallies.name += le32_to_cpu(tallies->name)
-	ADD_COMM_TALLIES(tx_unicast_frames);
-	ADD_COMM_TALLIES(tx_multicast_frames);
-	ADD_COMM_TALLIES(tx_fragments);
-	ADD_COMM_TALLIES(tx_unicast_octets);
-	ADD_COMM_TALLIES(tx_multicast_octets);
-	ADD_COMM_TALLIES(tx_deferred_transmissions);
-	ADD_COMM_TALLIES(tx_single_retry_frames);
-	ADD_COMM_TALLIES(tx_multiple_retry_frames);
-	ADD_COMM_TALLIES(tx_retry_limit_exceeded);
-	ADD_COMM_TALLIES(tx_discards);
-	ADD_COMM_TALLIES(rx_unicast_frames);
-	ADD_COMM_TALLIES(rx_multicast_frames);
-	ADD_COMM_TALLIES(rx_fragments);
-	ADD_COMM_TALLIES(rx_unicast_octets);
-	ADD_COMM_TALLIES(rx_multicast_octets);
-	ADD_COMM_TALLIES(rx_fcs_errors);
-	ADD_COMM_TALLIES(rx_discards_no_buffer);
-	ADD_COMM_TALLIES(tx_discards_wrong_sa);
-	ADD_COMM_TALLIES(rx_discards_wep_undecryptable);
-	ADD_COMM_TALLIES(rx_message_in_msg_fragments);
-	ADD_COMM_TALLIES(rx_message_in_bad_msg_fragments);
-#undef ADD_COMM_TALLIES
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static void prism2_info_commtallies(local_info_t *local, unsigned char *buf,
-				    int left)
-{
-	if (local->tallies32)
-		prism2_info_commtallies32(local, buf, left);
-	else
-		prism2_info_commtallies16(local, buf, left);
-}
-
-
-#ifndef PRISM2_NO_STATION_MODES
-#ifndef PRISM2_NO_DEBUG
-static const char* hfa384x_linkstatus_str(u16 linkstatus)
-{
-	switch (linkstatus) {
-	case HFA384X_LINKSTATUS_CONNECTED:
-		return "Connected";
-	case HFA384X_LINKSTATUS_DISCONNECTED:
-		return "Disconnected";
-	case HFA384X_LINKSTATUS_AP_CHANGE:
-		return "Access point change";
-	case HFA384X_LINKSTATUS_AP_OUT_OF_RANGE:
-		return "Access point out of range";
-	case HFA384X_LINKSTATUS_AP_IN_RANGE:
-		return "Access point in range";
-	case HFA384X_LINKSTATUS_ASSOC_FAILED:
-		return "Association failed";
-	default:
-		return "Unknown";
-	}
-}
-#endif /* PRISM2_NO_DEBUG */
-
-
-/* Called only as a tasklet (software IRQ) */
-static void prism2_info_linkstatus(local_info_t *local, unsigned char *buf,
-				    int left)
-{
-	u16 val;
-	int non_sta_mode;
-
-	/* Alloc new JoinRequests to occur since LinkStatus for the previous
-	 * has been received */
-	local->last_join_time = 0;
-
-	if (left != 2) {
-		printk(KERN_DEBUG "%s: invalid linkstatus info frame "
-		       "length %d\n", local->dev->name, left);
-		return;
-	}
-
-	non_sta_mode = local->iw_mode == IW_MODE_MASTER ||
-		local->iw_mode == IW_MODE_REPEAT ||
-		local->iw_mode == IW_MODE_MONITOR;
-
-	val = buf[0] | (buf[1] << 8);
-	if (!non_sta_mode || val != HFA384X_LINKSTATUS_DISCONNECTED) {
-		PDEBUG(DEBUG_EXTRA, "%s: LinkStatus=%d (%s)\n",
-		       local->dev->name, val, hfa384x_linkstatus_str(val));
-	}
-
-	if (non_sta_mode) {
-		netif_carrier_on(local->dev);
-		netif_carrier_on(local->ddev);
-		return;
-	}
-
-	/* Get current BSSID later in scheduled task */
-	set_bit(PRISM2_INFO_PENDING_LINKSTATUS, &local->pending_info);
-	local->prev_link_status = val;
-	schedule_work(&local->info_queue);
-}
-
-
-static void prism2_host_roaming(local_info_t *local)
-{
-	struct hfa384x_join_request req;
-	struct net_device *dev = local->dev;
-	struct hfa384x_hostscan_result *selected, *entry;
-	int i;
-	unsigned long flags;
-
-	if (local->last_join_time &&
-	    time_before(jiffies, local->last_join_time + 10 * HZ)) {
-		PDEBUG(DEBUG_EXTRA, "%s: last join request has not yet been "
-		       "completed - waiting for it before issuing new one\n",
-		       dev->name);
-		return;
-	}
-
-	/* ScanResults are sorted: first ESS results in decreasing signal
-	 * quality then IBSS results in similar order.
-	 * Trivial roaming policy: just select the first entry.
-	 * This could probably be improved by adding hysteresis to limit
-	 * number of handoffs, etc.
-	 *
-	 * Could do periodic RID_SCANREQUEST or Inquire F101 to get new
-	 * ScanResults */
-	spin_lock_irqsave(&local->lock, flags);
-	if (local->last_scan_results == NULL ||
-	    local->last_scan_results_count == 0) {
-		spin_unlock_irqrestore(&local->lock, flags);
-		PDEBUG(DEBUG_EXTRA, "%s: no scan results for host roaming\n",
-		       dev->name);
-		return;
-	}
-
-	selected = &local->last_scan_results[0];
-
-	if (local->preferred_ap[0] || local->preferred_ap[1] ||
-	    local->preferred_ap[2] || local->preferred_ap[3] ||
-	    local->preferred_ap[4] || local->preferred_ap[5]) {
-		/* Try to find preferred AP */
-		PDEBUG(DEBUG_EXTRA, "%s: Preferred AP BSSID %pM\n",
-		       dev->name, local->preferred_ap);
-		for (i = 0; i < local->last_scan_results_count; i++) {
-			entry = &local->last_scan_results[i];
-			if (memcmp(local->preferred_ap, entry->bssid, 6) == 0)
-			{
-				PDEBUG(DEBUG_EXTRA, "%s: using preferred AP "
-				       "selection\n", dev->name);
-				selected = entry;
-				break;
-			}
-		}
-	}
-
-	memcpy(req.bssid, selected->bssid, ETH_ALEN);
-	req.channel = selected->chid;
-	spin_unlock_irqrestore(&local->lock, flags);
-
-	PDEBUG(DEBUG_EXTRA, "%s: JoinRequest: BSSID=%pM"
-	       " channel=%d\n",
-	       dev->name, req.bssid, le16_to_cpu(req.channel));
-	if (local->func->set_rid(dev, HFA384X_RID_JOINREQUEST, &req,
-				 sizeof(req))) {
-		printk(KERN_DEBUG "%s: JoinRequest failed\n", dev->name);
-	}
-	local->last_join_time = jiffies;
-}
-
-
-static void hostap_report_scan_complete(local_info_t *local)
-{
-	union iwreq_data wrqu;
-
-	/* Inform user space about new scan results (just empty event,
-	 * SIOCGIWSCAN can be used to fetch data */
-	wrqu.data.length = 0;
-	wrqu.data.flags = 0;
-	wireless_send_event(local->dev, SIOCGIWSCAN, &wrqu, NULL);
-
-	/* Allow SIOCGIWSCAN handling to occur since we have received
-	 * scanning result */
-	local->scan_timestamp = 0;
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static void prism2_info_scanresults(local_info_t *local, unsigned char *buf,
-				    int left)
-{
-	u16 *pos;
-	int new_count, i;
-	unsigned long flags;
-	struct hfa384x_scan_result *res;
-	struct hfa384x_hostscan_result *results, *prev;
-
-	if (left < 4) {
-		printk(KERN_DEBUG "%s: invalid scanresult info frame "
-		       "length %d\n", local->dev->name, left);
-		return;
-	}
-
-	pos = (u16 *) buf;
-	pos++;
-	pos++;
-	left -= 4;
-
-	new_count = left / sizeof(struct hfa384x_scan_result);
-	results = kmalloc_array(new_count,
-				sizeof(struct hfa384x_hostscan_result),
-				GFP_ATOMIC);
-	if (results == NULL)
-		return;
-
-	/* Convert to hostscan result format. */
-	res = (struct hfa384x_scan_result *) pos;
-	for (i = 0; i < new_count; i++) {
-		memcpy(&results[i], &res[i],
-		       sizeof(struct hfa384x_scan_result));
-		results[i].atim = 0;
-	}
-
-	spin_lock_irqsave(&local->lock, flags);
-	local->last_scan_type = PRISM2_SCAN;
-	prev = local->last_scan_results;
-	local->last_scan_results = results;
-	local->last_scan_results_count = new_count;
-	spin_unlock_irqrestore(&local->lock, flags);
-	kfree(prev);
-
-	hostap_report_scan_complete(local);
-
-	/* Perform rest of ScanResults handling later in scheduled task */
-	set_bit(PRISM2_INFO_PENDING_SCANRESULTS, &local->pending_info);
-	schedule_work(&local->info_queue);
-}
-
-
-/* Called only as a tasklet (software IRQ) */
-static void prism2_info_hostscanresults(local_info_t *local,
-					unsigned char *buf, int left)
-{
-	int i, result_size, copy_len, new_count;
-	struct hfa384x_hostscan_result *results, *prev;
-	unsigned long flags;
-	__le16 *pos;
-	u8 *ptr;
-
-	wake_up_interruptible(&local->hostscan_wq);
-
-	if (left < 4) {
-		printk(KERN_DEBUG "%s: invalid hostscanresult info frame "
-		       "length %d\n", local->dev->name, left);
-		return;
-	}
-
-	pos = (__le16 *) buf;
-	copy_len = result_size = le16_to_cpu(*pos);
-	if (result_size == 0) {
-		printk(KERN_DEBUG "%s: invalid result_size (0) in "
-		       "hostscanresults\n", local->dev->name);
-		return;
-	}
-	if (copy_len > sizeof(struct hfa384x_hostscan_result))
-		copy_len = sizeof(struct hfa384x_hostscan_result);
-
-	pos++;
-	pos++;
-	left -= 4;
-	ptr = (u8 *) pos;
-
-	new_count = left / result_size;
-	results = kcalloc(new_count, sizeof(struct hfa384x_hostscan_result),
-			  GFP_ATOMIC);
-	if (results == NULL)
-		return;
-
-	for (i = 0; i < new_count; i++) {
-		memcpy(&results[i], ptr, copy_len);
-		ptr += result_size;
-		left -= result_size;
-	}
-
-	if (left) {
-		printk(KERN_DEBUG "%s: short HostScan result entry (%d/%d)\n",
-		       local->dev->name, left, result_size);
-	}
-
-	spin_lock_irqsave(&local->lock, flags);
-	local->last_scan_type = PRISM2_HOSTSCAN;
-	prev = local->last_scan_results;
-	local->last_scan_results = results;
-	local->last_scan_results_count = new_count;
-	spin_unlock_irqrestore(&local->lock, flags);
-	kfree(prev);
-
-	hostap_report_scan_complete(local);
-}
-#endif /* PRISM2_NO_STATION_MODES */
-
-
-/* Called only as a tasklet (software IRQ) */
-void hostap_info_process(local_info_t *local, struct sk_buff *skb)
-{
-	struct hfa384x_info_frame *info;
-	unsigned char *buf;
-	int left;
-#ifndef PRISM2_NO_DEBUG
-	int i;
-#endif /* PRISM2_NO_DEBUG */
-
-	info = (struct hfa384x_info_frame *) skb->data;
-	buf = skb->data + sizeof(*info);
-	left = skb->len - sizeof(*info);
-
-	switch (le16_to_cpu(info->type)) {
-	case HFA384X_INFO_COMMTALLIES:
-		prism2_info_commtallies(local, buf, left);
-		break;
-
-#ifndef PRISM2_NO_STATION_MODES
-	case HFA384X_INFO_LINKSTATUS:
-		prism2_info_linkstatus(local, buf, left);
-		break;
-
-	case HFA384X_INFO_SCANRESULTS:
-		prism2_info_scanresults(local, buf, left);
-		break;
-
-	case HFA384X_INFO_HOSTSCANRESULTS:
-		prism2_info_hostscanresults(local, buf, left);
-		break;
-#endif /* PRISM2_NO_STATION_MODES */
-
-#ifndef PRISM2_NO_DEBUG
-	default:
-		PDEBUG(DEBUG_EXTRA, "%s: INFO - len=%d type=0x%04x\n",
-		       local->dev->name, le16_to_cpu(info->len),
-		       le16_to_cpu(info->type));
-		PDEBUG(DEBUG_EXTRA, "Unknown info frame:");
-		for (i = 0; i < (left < 100 ? left : 100); i++)
-			PDEBUG2(DEBUG_EXTRA, " %02x", buf[i]);
-		PDEBUG2(DEBUG_EXTRA, "\n");
-		break;
-#endif /* PRISM2_NO_DEBUG */
-	}
-}
-
-
-#ifndef PRISM2_NO_STATION_MODES
-static void handle_info_queue_linkstatus(local_info_t *local)
-{
-	int val = local->prev_link_status;
-	int connected;
-	union iwreq_data wrqu;
-
-	connected =
-		val == HFA384X_LINKSTATUS_CONNECTED ||
-		val == HFA384X_LINKSTATUS_AP_CHANGE ||
-		val == HFA384X_LINKSTATUS_AP_IN_RANGE;
-
-	if (local->func->get_rid(local->dev, HFA384X_RID_CURRENTBSSID,
-				 local->bssid, ETH_ALEN, 1) < 0) {
-		printk(KERN_DEBUG "%s: could not read CURRENTBSSID after "
-		       "LinkStatus event\n", local->dev->name);
-	} else {
-		PDEBUG(DEBUG_EXTRA, "%s: LinkStatus: BSSID=%pM\n",
-		       local->dev->name,
-		       (unsigned char *) local->bssid);
-		if (local->wds_type & HOSTAP_WDS_AP_CLIENT)
-			hostap_add_sta(local->ap, local->bssid);
-	}
-
-	/* Get BSSID if we have a valid AP address */
-	if (connected) {
-		netif_carrier_on(local->dev);
-		netif_carrier_on(local->ddev);
-		memcpy(wrqu.ap_addr.sa_data, local->bssid, ETH_ALEN);
-	} else {
-		netif_carrier_off(local->dev);
-		netif_carrier_off(local->ddev);
-		eth_zero_addr(wrqu.ap_addr.sa_data);
-	}
-	wrqu.ap_addr.sa_family = ARPHRD_ETHER;
-
-	/*
-	 * Filter out sequential disconnect events in order not to cause a
-	 * flood of SIOCGIWAP events that have a race condition with EAPOL
-	 * frames and can confuse wpa_supplicant about the current association
-	 * status.
-	 */
-	if (connected || local->prev_linkstatus_connected)
-		wireless_send_event(local->dev, SIOCGIWAP, &wrqu, NULL);
-	local->prev_linkstatus_connected = connected;
-}
-
-
-static void handle_info_queue_scanresults(local_info_t *local)
-{
-	if (local->host_roaming == 1 && local->iw_mode == IW_MODE_INFRA)
-		prism2_host_roaming(local);
-
-	if (local->host_roaming == 2 && local->iw_mode == IW_MODE_INFRA &&
-	    !is_zero_ether_addr(local->preferred_ap)) {
-		/*
-		 * Firmware seems to be getting into odd state in host_roaming
-		 * mode 2 when hostscan is used without join command, so try
-		 * to fix this by re-joining the current AP. This does not
-		 * actually trigger a new association if the current AP is
-		 * still in the scan results.
-		 */
-		prism2_host_roaming(local);
-	}
-}
-
-
-/* Called only as scheduled task after receiving info frames (used to avoid
- * pending too much time in HW IRQ handler). */
-static void handle_info_queue(struct work_struct *work)
-{
-	local_info_t *local = container_of(work, local_info_t, info_queue);
-
-	if (test_and_clear_bit(PRISM2_INFO_PENDING_LINKSTATUS,
-			       &local->pending_info))
-		handle_info_queue_linkstatus(local);
-
-	if (test_and_clear_bit(PRISM2_INFO_PENDING_SCANRESULTS,
-			       &local->pending_info))
-		handle_info_queue_scanresults(local);
-}
-#endif /* PRISM2_NO_STATION_MODES */
-
-
-void hostap_info_init(local_info_t *local)
-{
-	skb_queue_head_init(&local->info_list);
-#ifndef PRISM2_NO_STATION_MODES
-	INIT_WORK(&local->info_queue, handle_info_queue);
-#endif /* PRISM2_NO_STATION_MODES */
-}
-
-
-EXPORT_SYMBOL(hostap_info_init);
-EXPORT_SYMBOL(hostap_info_process);
diff --git a/drivers/net/wireless/intersil/hostap/hostap_ioctl.c b/drivers/net/wireless/intersil/hostap/hostap_ioctl.c
deleted file mode 100644
index 26162f92e3c3..000000000000
--- a/drivers/net/wireless/intersil/hostap/hostap_ioctl.c
+++ /dev/null
@@ -1,3847 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/* ioctl() (mostly Linux Wireless Extensions) routines for Host AP driver */
-
-#include <linux/slab.h>
-#include <linux/types.h>
-#include <linux/sched/signal.h>
-#include <linux/ethtool.h>
-#include <linux/if_arp.h>
-#include <linux/module.h>
-#include <linux/etherdevice.h>
-#include <net/lib80211.h>
-
-#include "hostap_wlan.h"
-#include "hostap.h"
-#include "hostap_ap.h"
-
-static struct iw_statistics *hostap_get_wireless_stats(struct net_device *dev)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	struct iw_statistics *wstats;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	/* Why are we doing that ? Jean II */
-	if (iface->type != HOSTAP_INTERFACE_MAIN)
-		return NULL;
-
-	wstats = &local->wstats;
-
-	wstats->status = 0;
-	wstats->discard.code =
-		local->comm_tallies.rx_discards_wep_undecryptable;
-	wstats->discard.misc =
-		local->comm_tallies.rx_fcs_errors +
-		local->comm_tallies.rx_discards_no_buffer +
-		local->comm_tallies.tx_discards_wrong_sa;
-
-	wstats->discard.retries =
-		local->comm_tallies.tx_retry_limit_exceeded;
-	wstats->discard.fragment =
-		local->comm_tallies.rx_message_in_bad_msg_fragments;
-
-	if (local->iw_mode != IW_MODE_MASTER &&
-	    local->iw_mode != IW_MODE_REPEAT) {
-
-		if (prism2_update_comms_qual(dev) == 0)
-			wstats->qual.updated = IW_QUAL_ALL_UPDATED |
-				IW_QUAL_DBM;
-
-		wstats->qual.qual = local->comms_qual;
-		wstats->qual.level = local->avg_signal;
-		wstats->qual.noise = local->avg_noise;
-	} else {
-		wstats->qual.qual = 0;
-		wstats->qual.level = 0;
-		wstats->qual.noise = 0;
-		wstats->qual.updated = IW_QUAL_ALL_INVALID;
-	}
-
-	return wstats;
-}
-
-
-static int prism2_get_datarates(struct net_device *dev, u8 *rates)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	u8 buf[12];
-	int len;
-	u16 val;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	len = local->func->get_rid(dev, HFA384X_RID_SUPPORTEDDATARATES, buf,
-				   sizeof(buf), 0);
-	if (len < 2)
-		return 0;
-
-	val = le16_to_cpu(*(__le16 *) buf); /* string length */
-
-	if (len - 2 < val || val > 10)
-		return 0;
-
-	memcpy(rates, buf + 2, val);
-	return val;
-}
-
-
-static int prism2_get_name(struct net_device *dev,
-			   struct iw_request_info *info,
-			   union iwreq_data *wrqu, char *extra)
-{
-	u8 rates[10];
-	int len, i, over2 = 0;
-
-	len = prism2_get_datarates(dev, rates);
-
-	for (i = 0; i < len; i++) {
-		if (rates[i] == 0x0b || rates[i] == 0x16) {
-			over2 = 1;
-			break;
-		}
-	}
-
-	strcpy(wrqu->name, over2 ? "IEEE 802.11b" : "IEEE 802.11-DS");
-
-	return 0;
-}
-
-
-static int prism2_ioctl_siwencode(struct net_device *dev,
-				  struct iw_request_info *info,
-				  union iwreq_data *wrqu, char *keybuf)
-{
-	struct iw_point *erq = &wrqu->encoding;
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int i;
-	struct lib80211_crypt_data **crypt;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	i = erq->flags & IW_ENCODE_INDEX;
-	if (i < 1 || i > 4)
-		i = local->crypt_info.tx_keyidx;
-	else
-		i--;
-	if (i < 0 || i >= WEP_KEYS)
-		return -EINVAL;
-
-	crypt = &local->crypt_info.crypt[i];
-
-	if (erq->flags & IW_ENCODE_DISABLED) {
-		if (*crypt)
-			lib80211_crypt_delayed_deinit(&local->crypt_info, crypt);
-		goto done;
-	}
-
-	if (*crypt != NULL && (*crypt)->ops != NULL &&
-	    strcmp((*crypt)->ops->name, "WEP") != 0) {
-		/* changing to use WEP; deinit previously used algorithm */
-		lib80211_crypt_delayed_deinit(&local->crypt_info, crypt);
-	}
-
-	if (*crypt == NULL) {
-		struct lib80211_crypt_data *new_crypt;
-
-		/* take WEP into use */
-		new_crypt = kzalloc(sizeof(struct lib80211_crypt_data),
-				GFP_KERNEL);
-		if (new_crypt == NULL)
-			return -ENOMEM;
-		new_crypt->ops = lib80211_get_crypto_ops("WEP");
-		if (!new_crypt->ops) {
-			request_module("lib80211_crypt_wep");
-			new_crypt->ops = lib80211_get_crypto_ops("WEP");
-		}
-		if (new_crypt->ops && try_module_get(new_crypt->ops->owner))
-			new_crypt->priv = new_crypt->ops->init(i);
-		if (!new_crypt->ops || !new_crypt->priv) {
-			kfree(new_crypt);
-			new_crypt = NULL;
-
-			printk(KERN_WARNING "%s: could not initialize WEP: "
-			       "load module hostap_crypt_wep.o\n",
-			       dev->name);
-			return -EOPNOTSUPP;
-		}
-		*crypt = new_crypt;
-	}
-
-	if (erq->length > 0) {
-		int len = erq->length <= 5 ? 5 : 13;
-		int first = 1, j;
-		if (len > erq->length)
-			memset(keybuf + erq->length, 0, len - erq->length);
-		(*crypt)->ops->set_key(keybuf, len, NULL, (*crypt)->priv);
-		for (j = 0; j < WEP_KEYS; j++) {
-			if (j != i && local->crypt_info.crypt[j]) {
-				first = 0;
-				break;
-			}
-		}
-		if (first)
-			local->crypt_info.tx_keyidx = i;
-	} else {
-		/* No key data - just set the default TX key index */
-		local->crypt_info.tx_keyidx = i;
-	}
-
- done:
-	local->open_wep = erq->flags & IW_ENCODE_OPEN;
-
-	if (hostap_set_encryption(local)) {
-		printk(KERN_DEBUG "%s: set_encryption failed\n", dev->name);
-		return -EINVAL;
-	}
-
-	/* Do not reset port0 if card is in Managed mode since resetting will
-	 * generate new IEEE 802.11 authentication which may end up in looping
-	 * with IEEE 802.1X. Prism2 documentation seem to require port reset
-	 * after WEP configuration. However, keys are apparently changed at
-	 * least in Managed mode. */
-	if (local->iw_mode != IW_MODE_INFRA && local->func->reset_port(dev)) {
-		printk(KERN_DEBUG "%s: reset_port failed\n", dev->name);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-
-static int prism2_ioctl_giwencode(struct net_device *dev,
-				  struct iw_request_info *info,
-				  union iwreq_data *wrqu, char *key)
-{
-	struct iw_point *erq = &wrqu->encoding;
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int i, len;
-	u16 val;
-	struct lib80211_crypt_data *crypt;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	i = erq->flags & IW_ENCODE_INDEX;
-	if (i < 1 || i > 4)
-		i = local->crypt_info.tx_keyidx;
-	else
-		i--;
-	if (i < 0 || i >= WEP_KEYS)
-		return -EINVAL;
-
-	crypt = local->crypt_info.crypt[i];
-	erq->flags = i + 1;
-
-	if (crypt == NULL || crypt->ops == NULL) {
-		erq->length = 0;
-		erq->flags |= IW_ENCODE_DISABLED;
-		return 0;
-	}
-
-	if (strcmp(crypt->ops->name, "WEP") != 0) {
-		/* only WEP is supported with wireless extensions, so just
-		 * report that encryption is used */
-		erq->length = 0;
-		erq->flags |= IW_ENCODE_ENABLED;
-		return 0;
-	}
-
-	/* Reads from HFA384X_RID_CNFDEFAULTKEY* return bogus values, so show
-	 * the keys from driver buffer */
-	len = crypt->ops->get_key(key, WEP_KEY_LEN, NULL, crypt->priv);
-	erq->length = (len >= 0 ? len : 0);
-
-	if (local->func->get_rid(dev, HFA384X_RID_CNFWEPFLAGS, &val, 2, 1) < 0)
-	{
-		printk("CNFWEPFLAGS reading failed\n");
-		return -EOPNOTSUPP;
-	}
-	le16_to_cpus(&val);
-	if (val & HFA384X_WEPFLAGS_PRIVACYINVOKED)
-		erq->flags |= IW_ENCODE_ENABLED;
-	else
-		erq->flags |= IW_ENCODE_DISABLED;
-	if (val & HFA384X_WEPFLAGS_EXCLUDEUNENCRYPTED)
-		erq->flags |= IW_ENCODE_RESTRICTED;
-	else
-		erq->flags |= IW_ENCODE_OPEN;
-
-	return 0;
-}
-
-
-static int hostap_set_rate(struct net_device *dev)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int ret, basic_rates;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	basic_rates = local->basic_rates & local->tx_rate_control;
-	if (!basic_rates || basic_rates != local->basic_rates) {
-		printk(KERN_INFO "%s: updating basic rate set automatically "
-		       "to match with the new supported rate set\n",
-		       dev->name);
-		if (!basic_rates)
-			basic_rates = local->tx_rate_control;
-
-		local->basic_rates = basic_rates;
-		if (hostap_set_word(dev, HFA384X_RID_CNFBASICRATES,
-				    basic_rates))
-			printk(KERN_WARNING "%s: failed to set "
-			       "cnfBasicRates\n", dev->name);
-	}
-
-	ret = (hostap_set_word(dev, HFA384X_RID_TXRATECONTROL,
-			       local->tx_rate_control) ||
-	       hostap_set_word(dev, HFA384X_RID_CNFSUPPORTEDRATES,
-			       local->tx_rate_control) ||
-	       local->func->reset_port(dev));
-
-	if (ret) {
-		printk(KERN_WARNING "%s: TXRateControl/cnfSupportedRates "
-		       "setting to 0x%x failed\n",
-		       dev->name, local->tx_rate_control);
-	}
-
-	/* Update TX rate configuration for all STAs based on new operational
-	 * rate set. */
-	hostap_update_rates(local);
-
-	return ret;
-}
-
-
-static int prism2_ioctl_siwrate(struct net_device *dev,
-				struct iw_request_info *info,
-				union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *rrq = &wrqu->bitrate;
-	struct hostap_interface *iface;
-	local_info_t *local;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (rrq->fixed) {
-		switch (rrq->value) {
-		case 11000000:
-			local->tx_rate_control = HFA384X_RATES_11MBPS;
-			break;
-		case 5500000:
-			local->tx_rate_control = HFA384X_RATES_5MBPS;
-			break;
-		case 2000000:
-			local->tx_rate_control = HFA384X_RATES_2MBPS;
-			break;
-		case 1000000:
-			local->tx_rate_control = HFA384X_RATES_1MBPS;
-			break;
-		default:
-			local->tx_rate_control = HFA384X_RATES_1MBPS |
-				HFA384X_RATES_2MBPS | HFA384X_RATES_5MBPS |
-				HFA384X_RATES_11MBPS;
-			break;
-		}
-	} else {
-		switch (rrq->value) {
-		case 11000000:
-			local->tx_rate_control = HFA384X_RATES_1MBPS |
-				HFA384X_RATES_2MBPS | HFA384X_RATES_5MBPS |
-				HFA384X_RATES_11MBPS;
-			break;
-		case 5500000:
-			local->tx_rate_control = HFA384X_RATES_1MBPS |
-				HFA384X_RATES_2MBPS | HFA384X_RATES_5MBPS;
-			break;
-		case 2000000:
-			local->tx_rate_control = HFA384X_RATES_1MBPS |
-				HFA384X_RATES_2MBPS;
-			break;
-		case 1000000:
-			local->tx_rate_control = HFA384X_RATES_1MBPS;
-			break;
-		default:
-			local->tx_rate_control = HFA384X_RATES_1MBPS |
-				HFA384X_RATES_2MBPS | HFA384X_RATES_5MBPS |
-				HFA384X_RATES_11MBPS;
-			break;
-		}
-	}
-
-	return hostap_set_rate(dev);
-}
-
-
-static int prism2_ioctl_giwrate(struct net_device *dev,
-				struct iw_request_info *info,
-				union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *rrq = &wrqu->bitrate;
-	u16 val;
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int ret = 0;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (local->func->get_rid(dev, HFA384X_RID_TXRATECONTROL, &val, 2, 1) <
-	    0)
-		return -EINVAL;
-
-	if ((val & 0x1) && (val > 1))
-		rrq->fixed = 0;
-	else
-		rrq->fixed = 1;
-
-	if (local->iw_mode == IW_MODE_MASTER && local->ap != NULL &&
-	    !local->fw_tx_rate_control) {
-		/* HFA384X_RID_CURRENTTXRATE seems to always be 2 Mbps in
-		 * Host AP mode, so use the recorded TX rate of the last sent
-		 * frame */
-		rrq->value = local->ap->last_tx_rate > 0 ?
-			local->ap->last_tx_rate * 100000 : 11000000;
-		return 0;
-	}
-
-	if (local->func->get_rid(dev, HFA384X_RID_CURRENTTXRATE, &val, 2, 1) <
-	    0)
-		return -EINVAL;
-
-	switch (val) {
-	case HFA384X_RATES_1MBPS:
-		rrq->value = 1000000;
-		break;
-	case HFA384X_RATES_2MBPS:
-		rrq->value = 2000000;
-		break;
-	case HFA384X_RATES_5MBPS:
-		rrq->value = 5500000;
-		break;
-	case HFA384X_RATES_11MBPS:
-		rrq->value = 11000000;
-		break;
-	default:
-		/* should not happen */
-		rrq->value = 11000000;
-		ret = -EINVAL;
-		break;
-	}
-
-	return ret;
-}
-
-
-static int prism2_ioctl_siwsens(struct net_device *dev,
-				struct iw_request_info *info,
-				union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *sens = &wrqu->sens;
-	struct hostap_interface *iface;
-	local_info_t *local;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	/* Set the desired AP density */
-	if (sens->value < 1 || sens->value > 3)
-		return -EINVAL;
-
-	if (hostap_set_word(dev, HFA384X_RID_CNFSYSTEMSCALE, sens->value) ||
-	    local->func->reset_port(dev))
-		return -EINVAL;
-
-	return 0;
-}
-
-static int prism2_ioctl_giwsens(struct net_device *dev,
-				struct iw_request_info *info,
-				union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *sens = &wrqu->sens;
-	struct hostap_interface *iface;
-	local_info_t *local;
-	__le16 val;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	/* Get the current AP density */
-	if (local->func->get_rid(dev, HFA384X_RID_CNFSYSTEMSCALE, &val, 2, 1) <
-	    0)
-		return -EINVAL;
-
-	sens->value = le16_to_cpu(val);
-	sens->fixed = 1;
-
-	return 0;
-}
-
-
-/* Deprecated in new wireless extension API */
-static int prism2_ioctl_giwaplist(struct net_device *dev,
-				  struct iw_request_info *info,
-				  union iwreq_data *wrqu, char *extra)
-{
-	struct iw_point *data = &wrqu->data;
-	struct hostap_interface *iface;
-	local_info_t *local;
-	struct sockaddr *addr;
-	struct iw_quality *qual;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (local->iw_mode != IW_MODE_MASTER) {
-		printk(KERN_DEBUG "SIOCGIWAPLIST is currently only supported "
-		       "in Host AP mode\n");
-		data->length = 0;
-		return -EOPNOTSUPP;
-	}
-
-	addr = kmalloc_array(IW_MAX_AP, sizeof(struct sockaddr), GFP_KERNEL);
-	qual = kmalloc_array(IW_MAX_AP, sizeof(struct iw_quality), GFP_KERNEL);
-	if (addr == NULL || qual == NULL) {
-		kfree(addr);
-		kfree(qual);
-		data->length = 0;
-		return -ENOMEM;
-	}
-
-	data->length = prism2_ap_get_sta_qual(local, addr, qual, IW_MAX_AP, 1);
-
-	memcpy(extra, addr, sizeof(struct sockaddr) * data->length);
-	data->flags = 1; /* has quality information */
-	memcpy(extra + sizeof(struct sockaddr) * data->length, qual,
-	       sizeof(struct iw_quality) * data->length);
-
-	kfree(addr);
-	kfree(qual);
-	return 0;
-}
-
-
-static int prism2_ioctl_siwrts(struct net_device *dev,
-			       struct iw_request_info *info,
-			       union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *rts = &wrqu->rts;
-	struct hostap_interface *iface;
-	local_info_t *local;
-	__le16 val;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (rts->disabled)
-		val = cpu_to_le16(2347);
-	else if (rts->value < 0 || rts->value > 2347)
-		return -EINVAL;
-	else
-		val = cpu_to_le16(rts->value);
-
-	if (local->func->set_rid(dev, HFA384X_RID_RTSTHRESHOLD, &val, 2) ||
-	    local->func->reset_port(dev))
-		return -EINVAL;
-
-	local->rts_threshold = rts->value;
-
-	return 0;
-}
-
-static int prism2_ioctl_giwrts(struct net_device *dev,
-			       struct iw_request_info *info,
-			       union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *rts = &wrqu->rts;
-	struct hostap_interface *iface;
-	local_info_t *local;
-	__le16 val;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (local->func->get_rid(dev, HFA384X_RID_RTSTHRESHOLD, &val, 2, 1) <
-	    0)
-		return -EINVAL;
-
-	rts->value = le16_to_cpu(val);
-	rts->disabled = (rts->value == 2347);
-	rts->fixed = 1;
-
-	return 0;
-}
-
-
-static int prism2_ioctl_siwfrag(struct net_device *dev,
-				struct iw_request_info *info,
-				union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *rts = &wrqu->rts;
-	struct hostap_interface *iface;
-	local_info_t *local;
-	__le16 val;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (rts->disabled)
-		val = cpu_to_le16(2346);
-	else if (rts->value < 256 || rts->value > 2346)
-		return -EINVAL;
-	else
-		val = cpu_to_le16(rts->value & ~0x1); /* even numbers only */
-
-	local->fragm_threshold = rts->value & ~0x1;
-	if (local->func->set_rid(dev, HFA384X_RID_FRAGMENTATIONTHRESHOLD, &val,
-				 2)
-	    || local->func->reset_port(dev))
-		return -EINVAL;
-
-	return 0;
-}
-
-static int prism2_ioctl_giwfrag(struct net_device *dev,
-				struct iw_request_info *info,
-				union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *rts = &wrqu->rts;
-	struct hostap_interface *iface;
-	local_info_t *local;
-	__le16 val;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (local->func->get_rid(dev, HFA384X_RID_FRAGMENTATIONTHRESHOLD,
-				 &val, 2, 1) < 0)
-		return -EINVAL;
-
-	rts->value = le16_to_cpu(val);
-	rts->disabled = (rts->value == 2346);
-	rts->fixed = 1;
-
-	return 0;
-}
-
-
-#ifndef PRISM2_NO_STATION_MODES
-static int hostap_join_ap(struct net_device *dev)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	struct hfa384x_join_request req;
-	unsigned long flags;
-	int i;
-	struct hfa384x_hostscan_result *entry;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	memcpy(req.bssid, local->preferred_ap, ETH_ALEN);
-	req.channel = 0;
-
-	spin_lock_irqsave(&local->lock, flags);
-	for (i = 0; i < local->last_scan_results_count; i++) {
-		if (!local->last_scan_results)
-			break;
-		entry = &local->last_scan_results[i];
-		if (ether_addr_equal(local->preferred_ap, entry->bssid)) {
-			req.channel = entry->chid;
-			break;
-		}
-	}
-	spin_unlock_irqrestore(&local->lock, flags);
-
-	if (local->func->set_rid(dev, HFA384X_RID_JOINREQUEST, &req,
-				 sizeof(req))) {
-		printk(KERN_DEBUG "%s: JoinRequest %pM failed\n",
-		       dev->name, local->preferred_ap);
-		return -1;
-	}
-
-	printk(KERN_DEBUG "%s: Trying to join BSSID %pM\n",
-	       dev->name, local->preferred_ap);
-
-	return 0;
-}
-#endif /* PRISM2_NO_STATION_MODES */
-
-
-static int prism2_ioctl_siwap(struct net_device *dev,
-			      struct iw_request_info *info,
-			      union iwreq_data *wrqu, char *extra)
-{
-	struct sockaddr *ap_addr = &wrqu->ap_addr;
-#ifdef PRISM2_NO_STATION_MODES
-	return -EOPNOTSUPP;
-#else /* PRISM2_NO_STATION_MODES */
-	struct hostap_interface *iface;
-	local_info_t *local;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	memcpy(local->preferred_ap, &ap_addr->sa_data, ETH_ALEN);
-
-	if (local->host_roaming == 1 && local->iw_mode == IW_MODE_INFRA) {
-		struct hfa384x_scan_request scan_req;
-		memset(&scan_req, 0, sizeof(scan_req));
-		scan_req.channel_list = cpu_to_le16(0x3fff);
-		scan_req.txrate = cpu_to_le16(HFA384X_RATES_1MBPS);
-		if (local->func->set_rid(dev, HFA384X_RID_SCANREQUEST,
-					 &scan_req, sizeof(scan_req))) {
-			printk(KERN_DEBUG "%s: ScanResults request failed - "
-			       "preferred AP delayed to next unsolicited "
-			       "scan\n", dev->name);
-		}
-	} else if (local->host_roaming == 2 &&
-		   local->iw_mode == IW_MODE_INFRA) {
-		if (hostap_join_ap(dev))
-			return -EINVAL;
-	} else {
-		printk(KERN_DEBUG "%s: Preferred AP (SIOCSIWAP) is used only "
-		       "in Managed mode when host_roaming is enabled\n",
-		       dev->name);
-	}
-
-	return 0;
-#endif /* PRISM2_NO_STATION_MODES */
-}
-
-static int prism2_ioctl_giwap(struct net_device *dev,
-			      struct iw_request_info *info,
-			      union iwreq_data *wrqu, char *extra)
-{
-	struct sockaddr *ap_addr = &wrqu->ap_addr;
-	struct hostap_interface *iface;
-	local_info_t *local;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	ap_addr->sa_family = ARPHRD_ETHER;
-	switch (iface->type) {
-	case HOSTAP_INTERFACE_AP:
-		memcpy(&ap_addr->sa_data, dev->dev_addr, ETH_ALEN);
-		break;
-	case HOSTAP_INTERFACE_STA:
-		memcpy(&ap_addr->sa_data, local->assoc_ap_addr, ETH_ALEN);
-		break;
-	case HOSTAP_INTERFACE_WDS:
-		memcpy(&ap_addr->sa_data, iface->u.wds.remote_addr, ETH_ALEN);
-		break;
-	default:
-		if (local->func->get_rid(dev, HFA384X_RID_CURRENTBSSID,
-					 &ap_addr->sa_data, ETH_ALEN, 1) < 0)
-			return -EOPNOTSUPP;
-
-		/* local->bssid is also updated in LinkStatus handler when in
-		 * station mode */
-		memcpy(local->bssid, &ap_addr->sa_data, ETH_ALEN);
-		break;
-	}
-
-	return 0;
-}
-
-
-static int prism2_ioctl_siwnickn(struct net_device *dev,
-				 struct iw_request_info *info,
-				 union iwreq_data *wrqu, char *nickname)
-{
-	struct iw_point *data = &wrqu->data;
-	struct hostap_interface *iface;
-	local_info_t *local;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	memset(local->name, 0, sizeof(local->name));
-	memcpy(local->name, nickname, data->length);
-	local->name_set = 1;
-
-	if (hostap_set_string(dev, HFA384X_RID_CNFOWNNAME, local->name) ||
-	    local->func->reset_port(dev))
-		return -EINVAL;
-
-	return 0;
-}
-
-static int prism2_ioctl_giwnickn(struct net_device *dev,
-				 struct iw_request_info *info,
-				 union iwreq_data *wrqu, char *nickname)
-{
-	struct iw_point *data = &wrqu->data;
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int len;
-	char name[MAX_NAME_LEN + 3];
-	u16 val;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	len = local->func->get_rid(dev, HFA384X_RID_CNFOWNNAME,
-				   &name, MAX_NAME_LEN + 2, 0);
-	val = le16_to_cpu(*(__le16 *) name);
-	if (len > MAX_NAME_LEN + 2 || len < 0 || val > MAX_NAME_LEN)
-		return -EOPNOTSUPP;
-
-	name[val + 2] = '\0';
-	data->length = val + 1;
-	memcpy(nickname, name + 2, val + 1);
-
-	return 0;
-}
-
-
-static int prism2_ioctl_siwfreq(struct net_device *dev,
-				struct iw_request_info *info,
-				union iwreq_data *wrqu, char *extra)
-{
-	struct iw_freq *freq = &wrqu->freq;
-	struct hostap_interface *iface;
-	local_info_t *local;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	/* freq => chan. */
-	if (freq->e == 1 &&
-	    freq->m / 100000 >= freq_list[0] &&
-	    freq->m / 100000 <= freq_list[FREQ_COUNT - 1]) {
-		int ch;
-		int fr = freq->m / 100000;
-		for (ch = 0; ch < FREQ_COUNT; ch++) {
-			if (fr == freq_list[ch]) {
-				freq->e = 0;
-				freq->m = ch + 1;
-				break;
-			}
-		}
-	}
-
-	if (freq->e != 0 || freq->m < 1 || freq->m > FREQ_COUNT ||
-	    !(local->channel_mask & (1 << (freq->m - 1))))
-		return -EINVAL;
-
-	local->channel = freq->m; /* channel is used in prism2_setup_rids() */
-	if (hostap_set_word(dev, HFA384X_RID_CNFOWNCHANNEL, local->channel) ||
-	    local->func->reset_port(dev))
-		return -EINVAL;
-
-	return 0;
-}
-
-static int prism2_ioctl_giwfreq(struct net_device *dev,
-				struct iw_request_info *info,
-				union iwreq_data *wrqu, char *extra)
-{
-	struct iw_freq *freq = &wrqu->freq;
-	struct hostap_interface *iface;
-	local_info_t *local;
-	u16 val;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (local->func->get_rid(dev, HFA384X_RID_CURRENTCHANNEL, &val, 2, 1) <
-	    0)
-		return -EINVAL;
-
-	le16_to_cpus(&val);
-	if (val < 1 || val > FREQ_COUNT)
-		return -EINVAL;
-
-	freq->m = freq_list[val - 1] * 100000;
-	freq->e = 1;
-
-	return 0;
-}
-
-
-static void hostap_monitor_set_type(local_info_t *local)
-{
-	struct net_device *dev = local->ddev;
-
-	if (dev == NULL)
-		return;
-
-	if (local->monitor_type == PRISM2_MONITOR_PRISM ||
-	    local->monitor_type == PRISM2_MONITOR_CAPHDR) {
-		dev->type = ARPHRD_IEEE80211_PRISM;
-	} else if (local->monitor_type == PRISM2_MONITOR_RADIOTAP) {
-		dev->type = ARPHRD_IEEE80211_RADIOTAP;
-	} else {
-		dev->type = ARPHRD_IEEE80211;
-	}
-}
-
-
-static int prism2_ioctl_siwessid(struct net_device *dev,
-				 struct iw_request_info *info,
-				 union iwreq_data *wrqu, char *ssid)
-{
-	struct iw_point *data = &wrqu->data;
-	struct hostap_interface *iface;
-	local_info_t *local;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (iface->type == HOSTAP_INTERFACE_WDS)
-		return -EOPNOTSUPP;
-
-	if (data->flags == 0)
-		ssid[0] = '\0'; /* ANY */
-
-	if (local->iw_mode == IW_MODE_MASTER && ssid[0] == '\0') {
-		/* Setting SSID to empty string seems to kill the card in
-		 * Host AP mode */
-		printk(KERN_DEBUG "%s: Host AP mode does not support "
-		       "'Any' essid\n", dev->name);
-		return -EINVAL;
-	}
-
-	memcpy(local->essid, ssid, data->length);
-	local->essid[data->length] = '\0';
-
-	if ((!local->fw_ap &&
-	     hostap_set_string(dev, HFA384X_RID_CNFDESIREDSSID, local->essid))
-	    || hostap_set_string(dev, HFA384X_RID_CNFOWNSSID, local->essid) ||
-	    local->func->reset_port(dev))
-		return -EINVAL;
-
-	return 0;
-}
-
-static int prism2_ioctl_giwessid(struct net_device *dev,
-				 struct iw_request_info *info,
-				 union iwreq_data *wrqu, char *essid)
-{
-	struct iw_point *data = &wrqu->data;
-	struct hostap_interface *iface;
-	local_info_t *local;
-	u16 val;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (iface->type == HOSTAP_INTERFACE_WDS)
-		return -EOPNOTSUPP;
-
-	data->flags = 1; /* active */
-	if (local->iw_mode == IW_MODE_MASTER) {
-		data->length = strlen(local->essid);
-		memcpy(essid, local->essid, IW_ESSID_MAX_SIZE);
-	} else {
-		int len;
-		char ssid[MAX_SSID_LEN + 2];
-		memset(ssid, 0, sizeof(ssid));
-		len = local->func->get_rid(dev, HFA384X_RID_CURRENTSSID,
-					   &ssid, MAX_SSID_LEN + 2, 0);
-		val = le16_to_cpu(*(__le16 *) ssid);
-		if (len > MAX_SSID_LEN + 2 || len < 0 || val > MAX_SSID_LEN) {
-			return -EOPNOTSUPP;
-		}
-		data->length = val;
-		memcpy(essid, ssid + 2, IW_ESSID_MAX_SIZE);
-	}
-
-	return 0;
-}
-
-
-static int prism2_ioctl_giwrange(struct net_device *dev,
-				 struct iw_request_info *info,
-				 union iwreq_data *wrqu, char *extra)
-{
-	struct iw_point *data = &wrqu->data;
-	struct hostap_interface *iface;
-	local_info_t *local;
-	struct iw_range *range = (struct iw_range *) extra;
-	u8 rates[10];
-	u16 val;
-	int i, len, over2;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	data->length = sizeof(struct iw_range);
-	memset(range, 0, sizeof(struct iw_range));
-
-	/* TODO: could fill num_txpower and txpower array with
-	 * something; however, there are 128 different values.. */
-
-	range->txpower_capa = IW_TXPOW_DBM;
-
-	if (local->iw_mode == IW_MODE_INFRA || local->iw_mode == IW_MODE_ADHOC)
-	{
-		range->min_pmp = 1 * 1024;
-		range->max_pmp = 65535 * 1024;
-		range->min_pmt = 1 * 1024;
-		range->max_pmt = 1000 * 1024;
-		range->pmp_flags = IW_POWER_PERIOD;
-		range->pmt_flags = IW_POWER_TIMEOUT;
-		range->pm_capa = IW_POWER_PERIOD | IW_POWER_TIMEOUT |
-			IW_POWER_UNICAST_R | IW_POWER_ALL_R;
-	}
-
-	range->we_version_compiled = WIRELESS_EXT;
-	range->we_version_source = 18;
-
-	range->retry_capa = IW_RETRY_LIMIT;
-	range->retry_flags = IW_RETRY_LIMIT;
-	range->min_retry = 0;
-	range->max_retry = 255;
-
-	range->num_channels = FREQ_COUNT;
-
-	val = 0;
-	for (i = 0; i < FREQ_COUNT; i++) {
-		if (local->channel_mask & (1 << i)) {
-			range->freq[val].i = i + 1;
-			range->freq[val].m = freq_list[i] * 100000;
-			range->freq[val].e = 1;
-			val++;
-		}
-		if (val == IW_MAX_FREQUENCIES)
-			break;
-	}
-	range->num_frequency = val;
-
-	if (local->sta_fw_ver >= PRISM2_FW_VER(1,3,1)) {
-		range->max_qual.qual = 70; /* what is correct max? This was not
-					    * documented exactly. At least
-					    * 69 has been observed. */
-		range->max_qual.level = 0; /* dB */
-		range->max_qual.noise = 0; /* dB */
-
-		/* What would be suitable values for "average/typical" qual? */
-		range->avg_qual.qual = 20;
-		range->avg_qual.level = -60;
-		range->avg_qual.noise = -95;
-	} else {
-		range->max_qual.qual = 92; /* 0 .. 92 */
-		range->max_qual.level = 154; /* 27 .. 154 */
-		range->max_qual.noise = 154; /* 27 .. 154 */
-	}
-	range->sensitivity = 3;
-
-	range->max_encoding_tokens = WEP_KEYS;
-	range->num_encoding_sizes = 2;
-	range->encoding_size[0] = 5;
-	range->encoding_size[1] = 13;
-
-	over2 = 0;
-	len = prism2_get_datarates(dev, rates);
-	range->num_bitrates = 0;
-	for (i = 0; i < len; i++) {
-		if (range->num_bitrates < IW_MAX_BITRATES) {
-			range->bitrate[range->num_bitrates] =
-				rates[i] * 500000;
-			range->num_bitrates++;
-		}
-		if (rates[i] == 0x0b || rates[i] == 0x16)
-			over2 = 1;
-	}
-	/* estimated maximum TCP throughput values (bps) */
-	range->throughput = over2 ? 5500000 : 1500000;
-
-	range->min_rts = 0;
-	range->max_rts = 2347;
-	range->min_frag = 256;
-	range->max_frag = 2346;
-
-	/* Event capability (kernel + driver) */
-	range->event_capa[0] = (IW_EVENT_CAPA_K_0 |
-				IW_EVENT_CAPA_MASK(SIOCGIWTHRSPY) |
-				IW_EVENT_CAPA_MASK(SIOCGIWAP) |
-				IW_EVENT_CAPA_MASK(SIOCGIWSCAN));
-	range->event_capa[1] = IW_EVENT_CAPA_K_1;
-	range->event_capa[4] = (IW_EVENT_CAPA_MASK(IWEVTXDROP) |
-				IW_EVENT_CAPA_MASK(IWEVCUSTOM) |
-				IW_EVENT_CAPA_MASK(IWEVREGISTERED) |
-				IW_EVENT_CAPA_MASK(IWEVEXPIRED));
-
-	range->enc_capa = IW_ENC_CAPA_WPA | IW_ENC_CAPA_WPA2 |
-		IW_ENC_CAPA_CIPHER_TKIP | IW_ENC_CAPA_CIPHER_CCMP;
-
-	if (local->sta_fw_ver >= PRISM2_FW_VER(1,3,1))
-		range->scan_capa = IW_SCAN_CAPA_ESSID;
-
-	return 0;
-}
-
-
-static int hostap_monitor_mode_enable(local_info_t *local)
-{
-	struct net_device *dev = local->dev;
-
-	printk(KERN_DEBUG "Enabling monitor mode\n");
-	hostap_monitor_set_type(local);
-
-	if (hostap_set_word(dev, HFA384X_RID_CNFPORTTYPE,
-			    HFA384X_PORTTYPE_PSEUDO_IBSS)) {
-		printk(KERN_DEBUG "Port type setting for monitor mode "
-		       "failed\n");
-		return -EOPNOTSUPP;
-	}
-
-	/* Host decrypt is needed to get the IV and ICV fields;
-	 * however, monitor mode seems to remove WEP flag from frame
-	 * control field */
-	if (hostap_set_word(dev, HFA384X_RID_CNFWEPFLAGS,
-			    HFA384X_WEPFLAGS_HOSTENCRYPT |
-			    HFA384X_WEPFLAGS_HOSTDECRYPT)) {
-		printk(KERN_DEBUG "WEP flags setting failed\n");
-		return -EOPNOTSUPP;
-	}
-
-	if (local->func->reset_port(dev) ||
-	    local->func->cmd(dev, HFA384X_CMDCODE_TEST |
-			     (HFA384X_TEST_MONITOR << 8),
-			     0, NULL, NULL)) {
-		printk(KERN_DEBUG "Setting monitor mode failed\n");
-		return -EOPNOTSUPP;
-	}
-
-	return 0;
-}
-
-
-static int hostap_monitor_mode_disable(local_info_t *local)
-{
-	struct net_device *dev = local->ddev;
-
-	if (dev == NULL)
-		return -1;
-
-	printk(KERN_DEBUG "%s: Disabling monitor mode\n", dev->name);
-	dev->type = ARPHRD_ETHER;
-
-	if (local->func->cmd(dev, HFA384X_CMDCODE_TEST |
-			     (HFA384X_TEST_STOP << 8),
-			     0, NULL, NULL))
-		return -1;
-	return hostap_set_encryption(local);
-}
-
-
-static int prism2_ioctl_siwmode(struct net_device *dev,
-				struct iw_request_info *info,
-				union iwreq_data *wrqu, char *extra)
-{
-	__u32 *mode = &wrqu->mode;
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int double_reset = 0;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (*mode != IW_MODE_ADHOC && *mode != IW_MODE_INFRA &&
-	    *mode != IW_MODE_MASTER && *mode != IW_MODE_REPEAT &&
-	    *mode != IW_MODE_MONITOR)
-		return -EOPNOTSUPP;
-
-#ifdef PRISM2_NO_STATION_MODES
-	if (*mode == IW_MODE_ADHOC || *mode == IW_MODE_INFRA)
-		return -EOPNOTSUPP;
-#endif /* PRISM2_NO_STATION_MODES */
-
-	if (*mode == local->iw_mode)
-		return 0;
-
-	if (*mode == IW_MODE_MASTER && local->essid[0] == '\0') {
-		printk(KERN_WARNING "%s: empty SSID not allowed in Master "
-		       "mode\n", dev->name);
-		return -EINVAL;
-	}
-
-	if (local->iw_mode == IW_MODE_MONITOR)
-		hostap_monitor_mode_disable(local);
-
-	if ((local->iw_mode == IW_MODE_ADHOC ||
-	     local->iw_mode == IW_MODE_MONITOR) && *mode == IW_MODE_MASTER) {
-		/* There seems to be a firmware bug in at least STA f/w v1.5.6
-		 * that leaves beacon frames to use IBSS type when moving from
-		 * IBSS to Host AP mode. Doing double Port0 reset seems to be
-		 * enough to workaround this. */
-		double_reset = 1;
-	}
-
-	printk(KERN_DEBUG "prism2: %s: operating mode changed "
-	       "%d -> %d\n", dev->name, local->iw_mode, *mode);
-	local->iw_mode = *mode;
-
-	if (local->iw_mode == IW_MODE_MONITOR)
-		hostap_monitor_mode_enable(local);
-	else if (local->iw_mode == IW_MODE_MASTER && !local->host_encrypt &&
-		 !local->fw_encrypt_ok) {
-		printk(KERN_DEBUG "%s: defaulting to host-based encryption as "
-		       "a workaround for firmware bug in Host AP mode WEP\n",
-		       dev->name);
-		local->host_encrypt = 1;
-	}
-
-	if (hostap_set_word(dev, HFA384X_RID_CNFPORTTYPE,
-			    hostap_get_porttype(local)))
-		return -EOPNOTSUPP;
-
-	if (local->func->reset_port(dev))
-		return -EINVAL;
-	if (double_reset && local->func->reset_port(dev))
-		return -EINVAL;
-
-	if (local->iw_mode != IW_MODE_INFRA && local->iw_mode != IW_MODE_ADHOC)
-	{
-		/* netif_carrier is used only in client modes for now, so make
-		 * sure carrier is on when moving to non-client modes. */
-		netif_carrier_on(local->dev);
-		netif_carrier_on(local->ddev);
-	}
-	return 0;
-}
-
-
-static int prism2_ioctl_giwmode(struct net_device *dev,
-				struct iw_request_info *info,
-				union iwreq_data *wrqu, char *extra)
-{
-	__u32 *mode = &wrqu->mode;
-	struct hostap_interface *iface;
-	local_info_t *local;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	switch (iface->type) {
-	case HOSTAP_INTERFACE_STA:
-		*mode = IW_MODE_INFRA;
-		break;
-	case HOSTAP_INTERFACE_WDS:
-		*mode = IW_MODE_REPEAT;
-		break;
-	default:
-		*mode = local->iw_mode;
-		break;
-	}
-	return 0;
-}
-
-
-static int prism2_ioctl_siwpower(struct net_device *dev,
-				 struct iw_request_info *info,
-				 union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *wrq = &wrqu->power;
-#ifdef PRISM2_NO_STATION_MODES
-	return -EOPNOTSUPP;
-#else /* PRISM2_NO_STATION_MODES */
-	int ret = 0;
-
-	if (wrq->disabled)
-		return hostap_set_word(dev, HFA384X_RID_CNFPMENABLED, 0);
-
-	switch (wrq->flags & IW_POWER_MODE) {
-	case IW_POWER_UNICAST_R:
-		ret = hostap_set_word(dev, HFA384X_RID_CNFMULTICASTRECEIVE, 0);
-		if (ret)
-			return ret;
-		ret = hostap_set_word(dev, HFA384X_RID_CNFPMENABLED, 1);
-		if (ret)
-			return ret;
-		break;
-	case IW_POWER_ALL_R:
-		ret = hostap_set_word(dev, HFA384X_RID_CNFMULTICASTRECEIVE, 1);
-		if (ret)
-			return ret;
-		ret = hostap_set_word(dev, HFA384X_RID_CNFPMENABLED, 1);
-		if (ret)
-			return ret;
-		break;
-	case IW_POWER_ON:
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	if (wrq->flags & IW_POWER_TIMEOUT) {
-		ret = hostap_set_word(dev, HFA384X_RID_CNFPMENABLED, 1);
-		if (ret)
-			return ret;
-		ret = hostap_set_word(dev, HFA384X_RID_CNFPMHOLDOVERDURATION,
-				      wrq->value / 1024);
-		if (ret)
-			return ret;
-	}
-	if (wrq->flags & IW_POWER_PERIOD) {
-		ret = hostap_set_word(dev, HFA384X_RID_CNFPMENABLED, 1);
-		if (ret)
-			return ret;
-		ret = hostap_set_word(dev, HFA384X_RID_CNFMAXSLEEPDURATION,
-				      wrq->value / 1024);
-		if (ret)
-			return ret;
-	}
-
-	return ret;
-#endif /* PRISM2_NO_STATION_MODES */
-}
-
-
-static int prism2_ioctl_giwpower(struct net_device *dev,
-				 struct iw_request_info *info,
-				 union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *rrq = &wrqu->power;
-#ifdef PRISM2_NO_STATION_MODES
-	return -EOPNOTSUPP;
-#else /* PRISM2_NO_STATION_MODES */
-	struct hostap_interface *iface;
-	local_info_t *local;
-	__le16 enable, mcast;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (local->func->get_rid(dev, HFA384X_RID_CNFPMENABLED, &enable, 2, 1)
-	    < 0)
-		return -EINVAL;
-
-	if (!le16_to_cpu(enable)) {
-		rrq->disabled = 1;
-		return 0;
-	}
-
-	rrq->disabled = 0;
-
-	if ((rrq->flags & IW_POWER_TYPE) == IW_POWER_TIMEOUT) {
-		__le16 timeout;
-		if (local->func->get_rid(dev,
-					 HFA384X_RID_CNFPMHOLDOVERDURATION,
-					 &timeout, 2, 1) < 0)
-			return -EINVAL;
-
-		rrq->flags = IW_POWER_TIMEOUT;
-		rrq->value = le16_to_cpu(timeout) * 1024;
-	} else {
-		__le16 period;
-		if (local->func->get_rid(dev, HFA384X_RID_CNFMAXSLEEPDURATION,
-					 &period, 2, 1) < 0)
-			return -EINVAL;
-
-		rrq->flags = IW_POWER_PERIOD;
-		rrq->value = le16_to_cpu(period) * 1024;
-	}
-
-	if (local->func->get_rid(dev, HFA384X_RID_CNFMULTICASTRECEIVE, &mcast,
-				 2, 1) < 0)
-		return -EINVAL;
-
-	if (le16_to_cpu(mcast))
-		rrq->flags |= IW_POWER_ALL_R;
-	else
-		rrq->flags |= IW_POWER_UNICAST_R;
-
-	return 0;
-#endif /* PRISM2_NO_STATION_MODES */
-}
-
-
-static int prism2_ioctl_siwretry(struct net_device *dev,
-				 struct iw_request_info *info,
-				 union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *rrq = &wrqu->retry;
-	struct hostap_interface *iface;
-	local_info_t *local;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (rrq->disabled)
-		return -EINVAL;
-
-	/* setting retry limits is not supported with the current station
-	 * firmware code; simulate this with alternative retry count for now */
-	if (rrq->flags == IW_RETRY_LIMIT) {
-		if (rrq->value < 0) {
-			/* disable manual retry count setting and use firmware
-			 * defaults */
-			local->manual_retry_count = -1;
-			local->tx_control &= ~HFA384X_TX_CTRL_ALT_RTRY;
-		} else {
-			if (hostap_set_word(dev, HFA384X_RID_CNFALTRETRYCOUNT,
-					    rrq->value)) {
-				printk(KERN_DEBUG "%s: Alternate retry count "
-				       "setting to %d failed\n",
-				       dev->name, rrq->value);
-				return -EOPNOTSUPP;
-			}
-
-			local->manual_retry_count = rrq->value;
-			local->tx_control |= HFA384X_TX_CTRL_ALT_RTRY;
-		}
-		return 0;
-	}
-
-	return -EOPNOTSUPP;
-
-#if 0
-	/* what could be done, if firmware would support this.. */
-
-	if (rrq->flags & IW_RETRY_LIMIT) {
-		if (rrq->flags & IW_RETRY_LONG)
-			HFA384X_RID_LONGRETRYLIMIT = rrq->value;
-		else if (rrq->flags & IW_RETRY_SHORT)
-			HFA384X_RID_SHORTRETRYLIMIT = rrq->value;
-		else {
-			HFA384X_RID_LONGRETRYLIMIT = rrq->value;
-			HFA384X_RID_SHORTRETRYLIMIT = rrq->value;
-		}
-
-	}
-
-	if (rrq->flags & IW_RETRY_LIFETIME) {
-		HFA384X_RID_MAXTRANSMITLIFETIME = rrq->value / 1024;
-	}
-
-	return 0;
-#endif /* 0 */
-}
-
-static int prism2_ioctl_giwretry(struct net_device *dev,
-				 struct iw_request_info *info,
-				 union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *rrq = &wrqu->retry;
-	struct hostap_interface *iface;
-	local_info_t *local;
-	__le16 shortretry, longretry, lifetime, altretry;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (local->func->get_rid(dev, HFA384X_RID_SHORTRETRYLIMIT, &shortretry,
-				 2, 1) < 0 ||
-	    local->func->get_rid(dev, HFA384X_RID_LONGRETRYLIMIT, &longretry,
-				 2, 1) < 0 ||
-	    local->func->get_rid(dev, HFA384X_RID_MAXTRANSMITLIFETIME,
-				 &lifetime, 2, 1) < 0)
-		return -EINVAL;
-
-	rrq->disabled = 0;
-
-	if ((rrq->flags & IW_RETRY_TYPE) == IW_RETRY_LIFETIME) {
-		rrq->flags = IW_RETRY_LIFETIME;
-		rrq->value = le16_to_cpu(lifetime) * 1024;
-	} else {
-		if (local->manual_retry_count >= 0) {
-			rrq->flags = IW_RETRY_LIMIT;
-			if (local->func->get_rid(dev,
-						 HFA384X_RID_CNFALTRETRYCOUNT,
-						 &altretry, 2, 1) >= 0)
-				rrq->value = le16_to_cpu(altretry);
-			else
-				rrq->value = local->manual_retry_count;
-		} else if ((rrq->flags & IW_RETRY_LONG)) {
-			rrq->flags = IW_RETRY_LIMIT | IW_RETRY_LONG;
-			rrq->value = le16_to_cpu(longretry);
-		} else {
-			rrq->flags = IW_RETRY_LIMIT;
-			rrq->value = le16_to_cpu(shortretry);
-			if (shortretry != longretry)
-				rrq->flags |= IW_RETRY_SHORT;
-		}
-	}
-	return 0;
-}
-
-
-/* Note! This TX power controlling is experimental and should not be used in
- * production use. It just sets raw power register and does not use any kind of
- * feedback information from the measured TX power (CR58). This is now
- * commented out to make sure that it is not used by accident. TX power
- * configuration will be enabled again after proper algorithm using feedback
- * has been implemented. */
-
-#ifdef RAW_TXPOWER_SETTING
-/* Map HFA386x's CR31 to and from dBm with some sort of ad hoc mapping..
- * This version assumes following mapping:
- * CR31 is 7-bit value with -64 to +63 range.
- * -64 is mapped into +20dBm and +63 into -43dBm.
- * This is certainly not an exact mapping for every card, but at least
- * increasing dBm value should correspond to increasing TX power.
- */
-
-static int prism2_txpower_hfa386x_to_dBm(u16 val)
-{
-	signed char tmp;
-
-	if (val > 255)
-		val = 255;
-
-	tmp = val;
-	tmp >>= 2;
-
-	return -12 - tmp;
-}
-
-static u16 prism2_txpower_dBm_to_hfa386x(int val)
-{
-	signed char tmp;
-
-	if (val > 20)
-		return 128;
-	else if (val < -43)
-		return 127;
-
-	tmp = val;
-	tmp = -12 - tmp;
-	tmp <<= 2;
-
-	return (unsigned char) tmp;
-}
-#endif /* RAW_TXPOWER_SETTING */
-
-
-static int prism2_ioctl_siwtxpow(struct net_device *dev,
-				 struct iw_request_info *info,
-				 union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *rrq = &wrqu->txpower;
-	struct hostap_interface *iface;
-	local_info_t *local;
-#ifdef RAW_TXPOWER_SETTING
-	char *tmp;
-#endif
-	u16 val;
-	int ret = 0;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (rrq->disabled) {
-		if (local->txpower_type != PRISM2_TXPOWER_OFF) {
-			val = 0xff; /* use all standby and sleep modes */
-			ret = local->func->cmd(dev, HFA384X_CMDCODE_WRITEMIF,
-					       HFA386X_CR_A_D_TEST_MODES2,
-					       &val, NULL);
-			printk(KERN_DEBUG "%s: Turning radio off: %s\n",
-			       dev->name, ret ? "failed" : "OK");
-			local->txpower_type = PRISM2_TXPOWER_OFF;
-		}
-		return (ret ? -EOPNOTSUPP : 0);
-	}
-
-	if (local->txpower_type == PRISM2_TXPOWER_OFF) {
-		val = 0; /* disable all standby and sleep modes */
-		ret = local->func->cmd(dev, HFA384X_CMDCODE_WRITEMIF,
-				       HFA386X_CR_A_D_TEST_MODES2, &val, NULL);
-		printk(KERN_DEBUG "%s: Turning radio on: %s\n",
-		       dev->name, ret ? "failed" : "OK");
-		local->txpower_type = PRISM2_TXPOWER_UNKNOWN;
-	}
-
-#ifdef RAW_TXPOWER_SETTING
-	if (!rrq->fixed && local->txpower_type != PRISM2_TXPOWER_AUTO) {
-		printk(KERN_DEBUG "Setting ALC on\n");
-		val = HFA384X_TEST_CFG_BIT_ALC;
-		local->func->cmd(dev, HFA384X_CMDCODE_TEST |
-				 (HFA384X_TEST_CFG_BITS << 8), 1, &val, NULL);
-		local->txpower_type = PRISM2_TXPOWER_AUTO;
-		return 0;
-	}
-
-	if (local->txpower_type != PRISM2_TXPOWER_FIXED) {
-		printk(KERN_DEBUG "Setting ALC off\n");
-		val = HFA384X_TEST_CFG_BIT_ALC;
-		local->func->cmd(dev, HFA384X_CMDCODE_TEST |
-				 (HFA384X_TEST_CFG_BITS << 8), 0, &val, NULL);
-			local->txpower_type = PRISM2_TXPOWER_FIXED;
-	}
-
-	if (rrq->flags == IW_TXPOW_DBM)
-		tmp = "dBm";
-	else if (rrq->flags == IW_TXPOW_MWATT)
-		tmp = "mW";
-	else
-		tmp = "UNKNOWN";
-	printk(KERN_DEBUG "Setting TX power to %d %s\n", rrq->value, tmp);
-
-	if (rrq->flags != IW_TXPOW_DBM) {
-		printk("SIOCSIWTXPOW with mW is not supported; use dBm\n");
-		return -EOPNOTSUPP;
-	}
-
-	local->txpower = rrq->value;
-	val = prism2_txpower_dBm_to_hfa386x(local->txpower);
-	if (local->func->cmd(dev, HFA384X_CMDCODE_WRITEMIF,
-			     HFA386X_CR_MANUAL_TX_POWER, &val, NULL))
-		ret = -EOPNOTSUPP;
-#else /* RAW_TXPOWER_SETTING */
-	if (rrq->fixed)
-		ret = -EOPNOTSUPP;
-#endif /* RAW_TXPOWER_SETTING */
-
-	return ret;
-}
-
-static int prism2_ioctl_giwtxpow(struct net_device *dev,
-				 struct iw_request_info *info,
-				 union iwreq_data *wrqu, char *extra)
-{
-#ifdef RAW_TXPOWER_SETTING
-	struct iw_param *rrq = &wrqu->txpower;
-	struct hostap_interface *iface;
-	local_info_t *local;
-	u16 resp0;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	rrq->flags = IW_TXPOW_DBM;
-	rrq->disabled = 0;
-	rrq->fixed = 0;
-
-	if (local->txpower_type == PRISM2_TXPOWER_AUTO) {
-		if (local->func->cmd(dev, HFA384X_CMDCODE_READMIF,
-				     HFA386X_CR_MANUAL_TX_POWER,
-				     NULL, &resp0) == 0) {
-			rrq->value = prism2_txpower_hfa386x_to_dBm(resp0);
-		} else {
-			/* Could not get real txpower; guess 15 dBm */
-			rrq->value = 15;
-		}
-	} else if (local->txpower_type == PRISM2_TXPOWER_OFF) {
-		rrq->value = 0;
-		rrq->disabled = 1;
-	} else if (local->txpower_type == PRISM2_TXPOWER_FIXED) {
-		rrq->value = local->txpower;
-		rrq->fixed = 1;
-	} else {
-		printk("SIOCGIWTXPOW - unknown txpower_type=%d\n",
-		       local->txpower_type);
-	}
-	return 0;
-#else /* RAW_TXPOWER_SETTING */
-	return -EOPNOTSUPP;
-#endif /* RAW_TXPOWER_SETTING */
-}
-
-
-#ifndef PRISM2_NO_STATION_MODES
-
-/* HostScan request works with and without host_roaming mode. In addition, it
- * does not break current association. However, it requires newer station
- * firmware version (>= 1.3.1) than scan request. */
-static int prism2_request_hostscan(struct net_device *dev,
-				   u8 *ssid, u8 ssid_len)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	struct hfa384x_hostscan_request scan_req;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	memset(&scan_req, 0, sizeof(scan_req));
-	scan_req.channel_list = cpu_to_le16(local->channel_mask &
-					    local->scan_channel_mask);
-	scan_req.txrate = cpu_to_le16(HFA384X_RATES_1MBPS);
-	if (ssid) {
-		if (ssid_len > 32)
-			return -EINVAL;
-		scan_req.target_ssid_len = cpu_to_le16(ssid_len);
-		memcpy(scan_req.target_ssid, ssid, ssid_len);
-	}
-
-	if (local->func->set_rid(dev, HFA384X_RID_HOSTSCAN, &scan_req,
-				 sizeof(scan_req))) {
-		printk(KERN_DEBUG "%s: HOSTSCAN failed\n", dev->name);
-		return -EINVAL;
-	}
-	return 0;
-}
-
-
-static int prism2_request_scan(struct net_device *dev)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	struct hfa384x_scan_request scan_req;
-	int ret = 0;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	memset(&scan_req, 0, sizeof(scan_req));
-	scan_req.channel_list = cpu_to_le16(local->channel_mask &
-					    local->scan_channel_mask);
-	scan_req.txrate = cpu_to_le16(HFA384X_RATES_1MBPS);
-
-	/* FIX:
-	 * It seems to be enough to set roaming mode for a short moment to
-	 * host-based and then setup scanrequest data and return the mode to
-	 * firmware-based.
-	 *
-	 * Master mode would need to drop to Managed mode for a short while
-	 * to make scanning work.. Or sweep through the different channels and
-	 * use passive scan based on beacons. */
-
-	if (!local->host_roaming)
-		hostap_set_word(dev, HFA384X_RID_CNFROAMINGMODE,
-				HFA384X_ROAMING_HOST);
-
-	if (local->func->set_rid(dev, HFA384X_RID_SCANREQUEST, &scan_req,
-				 sizeof(scan_req))) {
-		printk(KERN_DEBUG "SCANREQUEST failed\n");
-		ret = -EINVAL;
-	}
-
-	if (!local->host_roaming)
-		hostap_set_word(dev, HFA384X_RID_CNFROAMINGMODE,
-				HFA384X_ROAMING_FIRMWARE);
-
-	return ret;
-}
-
-#else /* !PRISM2_NO_STATION_MODES */
-
-static inline int prism2_request_hostscan(struct net_device *dev,
-					  u8 *ssid, u8 ssid_len)
-{
-	return -EOPNOTSUPP;
-}
-
-
-static inline int prism2_request_scan(struct net_device *dev)
-{
-	return -EOPNOTSUPP;
-}
-
-#endif /* !PRISM2_NO_STATION_MODES */
-
-
-static int prism2_ioctl_siwscan(struct net_device *dev,
-				struct iw_request_info *info,
-				union iwreq_data *wrqu, char *extra)
-{
-	struct iw_point *data = &wrqu->data;
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int ret;
-	u8 *ssid = NULL, ssid_len = 0;
-	struct iw_scan_req *req = (struct iw_scan_req *) extra;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (data->length < sizeof(struct iw_scan_req))
-		req = NULL;
-
-	if (local->iw_mode == IW_MODE_MASTER) {
-		/* In master mode, we just return the results of our local
-		 * tables, so we don't need to start anything...
-		 * Jean II */
-		data->length = 0;
-		return 0;
-	}
-
-	if (!local->dev_enabled)
-		return -ENETDOWN;
-
-	if (req && data->flags & IW_SCAN_THIS_ESSID) {
-		ssid = req->essid;
-		ssid_len = req->essid_len;
-
-		if (ssid_len &&
-		    ((local->iw_mode != IW_MODE_INFRA &&
-		      local->iw_mode != IW_MODE_ADHOC) ||
-		     (local->sta_fw_ver < PRISM2_FW_VER(1,3,1))))
-			return -EOPNOTSUPP;
-	}
-
-	if (local->sta_fw_ver >= PRISM2_FW_VER(1,3,1))
-		ret = prism2_request_hostscan(dev, ssid, ssid_len);
-	else
-		ret = prism2_request_scan(dev);
-
-	if (ret == 0)
-		local->scan_timestamp = jiffies;
-
-	/* Could inquire F101, F103 or wait for SIOCGIWSCAN and read RID */
-
-	return ret;
-}
-
-
-#ifndef PRISM2_NO_STATION_MODES
-static char * __prism2_translate_scan(local_info_t *local,
-				      struct iw_request_info *info,
-				      struct hfa384x_hostscan_result *scan,
-				      struct hostap_bss_info *bss,
-				      char *current_ev, char *end_buf)
-{
-	int i, chan;
-	struct iw_event iwe;
-	char *current_val;
-	u16 capabilities;
-	u8 *pos;
-	u8 *ssid, *bssid;
-	size_t ssid_len;
-	char *buf;
-
-	if (bss) {
-		ssid = bss->ssid;
-		ssid_len = bss->ssid_len;
-		bssid = bss->bssid;
-	} else {
-		ssid = scan->ssid;
-		ssid_len = le16_to_cpu(scan->ssid_len);
-		bssid = scan->bssid;
-	}
-	if (ssid_len > 32)
-		ssid_len = 32;
-
-	/* First entry *MUST* be the AP MAC address */
-	memset(&iwe, 0, sizeof(iwe));
-	iwe.cmd = SIOCGIWAP;
-	iwe.u.ap_addr.sa_family = ARPHRD_ETHER;
-	memcpy(iwe.u.ap_addr.sa_data, bssid, ETH_ALEN);
-	current_ev = iwe_stream_add_event(info, current_ev, end_buf, &iwe,
-					  IW_EV_ADDR_LEN);
-
-	/* Other entries will be displayed in the order we give them */
-
-	memset(&iwe, 0, sizeof(iwe));
-	iwe.cmd = SIOCGIWESSID;
-	iwe.u.data.length = ssid_len;
-	iwe.u.data.flags = 1;
-	current_ev = iwe_stream_add_point(info, current_ev, end_buf,
-					  &iwe, ssid);
-
-	memset(&iwe, 0, sizeof(iwe));
-	iwe.cmd = SIOCGIWMODE;
-	if (bss) {
-		capabilities = bss->capab_info;
-	} else {
-		capabilities = le16_to_cpu(scan->capability);
-	}
-	if (capabilities & (WLAN_CAPABILITY_ESS |
-			    WLAN_CAPABILITY_IBSS)) {
-		if (capabilities & WLAN_CAPABILITY_ESS)
-			iwe.u.mode = IW_MODE_MASTER;
-		else
-			iwe.u.mode = IW_MODE_ADHOC;
-		current_ev = iwe_stream_add_event(info, current_ev, end_buf,
-						  &iwe, IW_EV_UINT_LEN);
-	}
-
-	memset(&iwe, 0, sizeof(iwe));
-	iwe.cmd = SIOCGIWFREQ;
-	if (scan) {
-		chan = le16_to_cpu(scan->chid);
-	} else if (bss) {
-		chan = bss->chan;
-	} else {
-		chan = 0;
-	}
-
-	if (chan > 0) {
-		iwe.u.freq.m = freq_list[chan - 1] * 100000;
-		iwe.u.freq.e = 1;
-		current_ev = iwe_stream_add_event(info, current_ev, end_buf,
-						  &iwe, IW_EV_FREQ_LEN);
-	}
-
-	if (scan) {
-		memset(&iwe, 0, sizeof(iwe));
-		iwe.cmd = IWEVQUAL;
-		if (local->last_scan_type == PRISM2_HOSTSCAN) {
-			iwe.u.qual.level = le16_to_cpu(scan->sl);
-			iwe.u.qual.noise = le16_to_cpu(scan->anl);
-		} else {
-			iwe.u.qual.level =
-				HFA384X_LEVEL_TO_dBm(le16_to_cpu(scan->sl));
-			iwe.u.qual.noise =
-				HFA384X_LEVEL_TO_dBm(le16_to_cpu(scan->anl));
-		}
-		iwe.u.qual.updated = IW_QUAL_LEVEL_UPDATED
-			| IW_QUAL_NOISE_UPDATED
-			| IW_QUAL_QUAL_INVALID
-			| IW_QUAL_DBM;
-		current_ev = iwe_stream_add_event(info, current_ev, end_buf,
-						  &iwe, IW_EV_QUAL_LEN);
-	}
-
-	memset(&iwe, 0, sizeof(iwe));
-	iwe.cmd = SIOCGIWENCODE;
-	if (capabilities & WLAN_CAPABILITY_PRIVACY)
-		iwe.u.data.flags = IW_ENCODE_ENABLED | IW_ENCODE_NOKEY;
-	else
-		iwe.u.data.flags = IW_ENCODE_DISABLED;
-	iwe.u.data.length = 0;
-	current_ev = iwe_stream_add_point(info, current_ev, end_buf, &iwe, "");
-
-	/* TODO: add SuppRates into BSS table */
-	if (scan) {
-		memset(&iwe, 0, sizeof(iwe));
-		iwe.cmd = SIOCGIWRATE;
-		current_val = current_ev + iwe_stream_lcp_len(info);
-		pos = scan->sup_rates;
-		for (i = 0; i < sizeof(scan->sup_rates); i++) {
-			if (pos[i] == 0)
-				break;
-			/* Bit rate given in 500 kb/s units (+ 0x80) */
-			iwe.u.bitrate.value = ((pos[i] & 0x7f) * 500000);
-			current_val = iwe_stream_add_value(
-				info, current_ev, current_val, end_buf, &iwe,
-				IW_EV_PARAM_LEN);
-		}
-		/* Check if we added any event */
-		if ((current_val - current_ev) > iwe_stream_lcp_len(info))
-			current_ev = current_val;
-	}
-
-	/* TODO: add BeaconInt,resp_rate,atim into BSS table */
-	buf = kmalloc(MAX_WPA_IE_LEN * 2 + 30, GFP_ATOMIC);
-	if (buf && scan) {
-		memset(&iwe, 0, sizeof(iwe));
-		iwe.cmd = IWEVCUSTOM;
-		sprintf(buf, "bcn_int=%d", le16_to_cpu(scan->beacon_interval));
-		iwe.u.data.length = strlen(buf);
-		current_ev = iwe_stream_add_point(info, current_ev, end_buf,
-						  &iwe, buf);
-
-		memset(&iwe, 0, sizeof(iwe));
-		iwe.cmd = IWEVCUSTOM;
-		sprintf(buf, "resp_rate=%d", le16_to_cpu(scan->rate));
-		iwe.u.data.length = strlen(buf);
-		current_ev = iwe_stream_add_point(info, current_ev, end_buf,
-						  &iwe, buf);
-
-		if (local->last_scan_type == PRISM2_HOSTSCAN &&
-		    (capabilities & WLAN_CAPABILITY_IBSS)) {
-			memset(&iwe, 0, sizeof(iwe));
-			iwe.cmd = IWEVCUSTOM;
-			sprintf(buf, "atim=%d", le16_to_cpu(scan->atim));
-			iwe.u.data.length = strlen(buf);
-			current_ev = iwe_stream_add_point(info, current_ev,
-							  end_buf, &iwe, buf);
-		}
-	}
-	kfree(buf);
-
-	if (bss && bss->wpa_ie_len > 0 && bss->wpa_ie_len <= MAX_WPA_IE_LEN) {
-		memset(&iwe, 0, sizeof(iwe));
-		iwe.cmd = IWEVGENIE;
-		iwe.u.data.length = bss->wpa_ie_len;
-		current_ev = iwe_stream_add_point(info, current_ev, end_buf,
-						  &iwe, bss->wpa_ie);
-	}
-
-	if (bss && bss->rsn_ie_len > 0 && bss->rsn_ie_len <= MAX_WPA_IE_LEN) {
-		memset(&iwe, 0, sizeof(iwe));
-		iwe.cmd = IWEVGENIE;
-		iwe.u.data.length = bss->rsn_ie_len;
-		current_ev = iwe_stream_add_point(info, current_ev, end_buf,
-						  &iwe, bss->rsn_ie);
-	}
-
-	return current_ev;
-}
-
-
-/* Translate scan data returned from the card to a card independent
- * format that the Wireless Tools will understand - Jean II */
-static inline int prism2_translate_scan(local_info_t *local,
-					struct iw_request_info *info,
-					char *buffer, int buflen)
-{
-	struct hfa384x_hostscan_result *scan;
-	int entry;
-	char *current_ev = buffer;
-	char *end_buf = buffer + buflen;
-	struct list_head *ptr;
-
-	spin_lock_bh(&local->lock);
-
-	list_for_each(ptr, &local->bss_list) {
-		struct hostap_bss_info *bss;
-		bss = list_entry(ptr, struct hostap_bss_info, list);
-		bss->included = 0;
-	}
-
-	for (entry = 0; entry < local->last_scan_results_count; entry++) {
-		int found = 0;
-		scan = &local->last_scan_results[entry];
-
-		/* Report every SSID if the AP is using multiple SSIDs. If no
-		 * BSS record is found (e.g., when WPA mode is disabled),
-		 * report the AP once. */
-		list_for_each(ptr, &local->bss_list) {
-			struct hostap_bss_info *bss;
-			bss = list_entry(ptr, struct hostap_bss_info, list);
-			if (ether_addr_equal(bss->bssid, scan->bssid)) {
-				bss->included = 1;
-				current_ev = __prism2_translate_scan(
-					local, info, scan, bss, current_ev,
-					end_buf);
-				found++;
-			}
-		}
-		if (!found) {
-			current_ev = __prism2_translate_scan(
-				local, info, scan, NULL, current_ev, end_buf);
-		}
-		/* Check if there is space for one more entry */
-		if ((end_buf - current_ev) <= IW_EV_ADDR_LEN) {
-			/* Ask user space to try again with a bigger buffer */
-			spin_unlock_bh(&local->lock);
-			return -E2BIG;
-		}
-	}
-
-	/* Prism2 firmware has limits (32 at least in some versions) for number
-	 * of BSSes in scan results. Extend this limit by using local BSS list.
-	 */
-	list_for_each(ptr, &local->bss_list) {
-		struct hostap_bss_info *bss;
-		bss = list_entry(ptr, struct hostap_bss_info, list);
-		if (bss->included)
-			continue;
-		current_ev = __prism2_translate_scan(local, info, NULL, bss,
-						     current_ev, end_buf);
-		/* Check if there is space for one more entry */
-		if ((end_buf - current_ev) <= IW_EV_ADDR_LEN) {
-			/* Ask user space to try again with a bigger buffer */
-			spin_unlock_bh(&local->lock);
-			return -E2BIG;
-		}
-	}
-
-	spin_unlock_bh(&local->lock);
-
-	return current_ev - buffer;
-}
-#endif /* PRISM2_NO_STATION_MODES */
-
-
-static inline int prism2_ioctl_giwscan_sta(struct net_device *dev,
-					   struct iw_request_info *info,
-					   struct iw_point *data, char *extra)
-{
-#ifdef PRISM2_NO_STATION_MODES
-	return -EOPNOTSUPP;
-#else /* PRISM2_NO_STATION_MODES */
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int res;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	/* Wait until the scan is finished. We can probably do better
-	 * than that - Jean II */
-	if (local->scan_timestamp &&
-	    time_before(jiffies, local->scan_timestamp + 3 * HZ)) {
-		/* Important note : we don't want to block the caller
-		 * until results are ready for various reasons.
-		 * First, managing wait queues is complex and racy
-		 * (there may be multiple simultaneous callers).
-		 * Second, we grab some rtnetlink lock before coming
-		 * here (in dev_ioctl()).
-		 * Third, the caller can wait on the Wireless Event
-		 * - Jean II */
-		return -EAGAIN;
-	}
-	local->scan_timestamp = 0;
-
-	res = prism2_translate_scan(local, info, extra, data->length);
-
-	if (res >= 0) {
-		data->length = res;
-		return 0;
-	} else {
-		data->length = 0;
-		return res;
-	}
-#endif /* PRISM2_NO_STATION_MODES */
-}
-
-
-static int prism2_ioctl_giwscan(struct net_device *dev,
-				struct iw_request_info *info,
-				union iwreq_data *wrqu, char *extra)
-{
-	struct iw_point *data = &wrqu->data;
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int res;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (local->iw_mode == IW_MODE_MASTER) {
-		/* In MASTER mode, it doesn't make sense to go around
-		 * scanning the frequencies and make the stations we serve
-		 * wait when what the user is really interested about is the
-		 * list of stations and access points we are talking to.
-		 * So, just extract results from our cache...
-		 * Jean II */
-
-		/* Translate to WE format */
-		res = prism2_ap_translate_scan(dev, info, extra);
-		if (res >= 0) {
-			printk(KERN_DEBUG "Scan result translation succeeded "
-			       "(length=%d)\n", res);
-			data->length = res;
-			return 0;
-		} else {
-			printk(KERN_DEBUG
-			       "Scan result translation failed (res=%d)\n",
-			       res);
-			data->length = 0;
-			return res;
-		}
-	} else {
-		/* Station mode */
-		return prism2_ioctl_giwscan_sta(dev, info, data, extra);
-	}
-}
-
-
-static const struct iw_priv_args prism2_priv[] = {
-	{ PRISM2_IOCTL_MONITOR,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "monitor" },
-	{ PRISM2_IOCTL_READMIF,
-	  IW_PRIV_TYPE_BYTE | IW_PRIV_SIZE_FIXED | 1,
-	  IW_PRIV_TYPE_BYTE | IW_PRIV_SIZE_FIXED | 1, "readmif" },
-	{ PRISM2_IOCTL_WRITEMIF,
-	  IW_PRIV_TYPE_BYTE | IW_PRIV_SIZE_FIXED | 2, 0, "writemif" },
-	{ PRISM2_IOCTL_RESET,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "reset" },
-	{ PRISM2_IOCTL_INQUIRE,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "inquire" },
-	{ PRISM2_IOCTL_SET_RID_WORD,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 2, 0, "set_rid_word" },
-	{ PRISM2_IOCTL_MACCMD,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "maccmd" },
-	{ PRISM2_IOCTL_WDS_ADD,
-	  IW_PRIV_TYPE_ADDR | IW_PRIV_SIZE_FIXED | 1, 0, "wds_add" },
-	{ PRISM2_IOCTL_WDS_DEL,
-	  IW_PRIV_TYPE_ADDR | IW_PRIV_SIZE_FIXED | 1, 0, "wds_del" },
-	{ PRISM2_IOCTL_ADDMAC,
-	  IW_PRIV_TYPE_ADDR | IW_PRIV_SIZE_FIXED | 1, 0, "addmac" },
-	{ PRISM2_IOCTL_DELMAC,
-	  IW_PRIV_TYPE_ADDR | IW_PRIV_SIZE_FIXED | 1, 0, "delmac" },
-	{ PRISM2_IOCTL_KICKMAC,
-	  IW_PRIV_TYPE_ADDR | IW_PRIV_SIZE_FIXED | 1, 0, "kickmac" },
-	/* --- raw access to sub-ioctls --- */
-	{ PRISM2_IOCTL_PRISM2_PARAM,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 2, 0, "prism2_param" },
-	{ PRISM2_IOCTL_GET_PRISM2_PARAM,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getprism2_param" },
-	/* --- sub-ioctls handlers --- */
-	{ PRISM2_IOCTL_PRISM2_PARAM,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "" },
-	{ PRISM2_IOCTL_GET_PRISM2_PARAM,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "" },
-	/* --- sub-ioctls definitions --- */
-	{ PRISM2_PARAM_TXRATECTRL,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "txratectrl" },
-	{ PRISM2_PARAM_TXRATECTRL,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "gettxratectrl" },
-	{ PRISM2_PARAM_BEACON_INT,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "beacon_int" },
-	{ PRISM2_PARAM_BEACON_INT,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getbeacon_int" },
-#ifndef PRISM2_NO_STATION_MODES
-	{ PRISM2_PARAM_PSEUDO_IBSS,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "pseudo_ibss" },
-	{ PRISM2_PARAM_PSEUDO_IBSS,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getpseudo_ibss" },
-#endif /* PRISM2_NO_STATION_MODES */
-	{ PRISM2_PARAM_ALC,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "alc" },
-	{ PRISM2_PARAM_ALC,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getalc" },
-	{ PRISM2_PARAM_DUMP,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "dump" },
-	{ PRISM2_PARAM_DUMP,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getdump" },
-	{ PRISM2_PARAM_OTHER_AP_POLICY,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "other_ap_policy" },
-	{ PRISM2_PARAM_OTHER_AP_POLICY,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getother_ap_pol" },
-	{ PRISM2_PARAM_AP_MAX_INACTIVITY,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "max_inactivity" },
-	{ PRISM2_PARAM_AP_MAX_INACTIVITY,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getmax_inactivi" },
-	{ PRISM2_PARAM_AP_BRIDGE_PACKETS,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "bridge_packets" },
-	{ PRISM2_PARAM_AP_BRIDGE_PACKETS,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getbridge_packe" },
-	{ PRISM2_PARAM_DTIM_PERIOD,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "dtim_period" },
-	{ PRISM2_PARAM_DTIM_PERIOD,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getdtim_period" },
-	{ PRISM2_PARAM_AP_NULLFUNC_ACK,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "nullfunc_ack" },
-	{ PRISM2_PARAM_AP_NULLFUNC_ACK,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getnullfunc_ack" },
-	{ PRISM2_PARAM_MAX_WDS,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "max_wds" },
-	{ PRISM2_PARAM_MAX_WDS,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getmax_wds" },
-	{ PRISM2_PARAM_AP_AUTOM_AP_WDS,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "autom_ap_wds" },
-	{ PRISM2_PARAM_AP_AUTOM_AP_WDS,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getautom_ap_wds" },
-	{ PRISM2_PARAM_AP_AUTH_ALGS,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "ap_auth_algs" },
-	{ PRISM2_PARAM_AP_AUTH_ALGS,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getap_auth_algs" },
-	{ PRISM2_PARAM_MONITOR_ALLOW_FCSERR,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "allow_fcserr" },
-	{ PRISM2_PARAM_MONITOR_ALLOW_FCSERR,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getallow_fcserr" },
-	{ PRISM2_PARAM_HOST_ENCRYPT,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "host_encrypt" },
-	{ PRISM2_PARAM_HOST_ENCRYPT,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "gethost_encrypt" },
-	{ PRISM2_PARAM_HOST_DECRYPT,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "host_decrypt" },
-	{ PRISM2_PARAM_HOST_DECRYPT,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "gethost_decrypt" },
-#ifndef PRISM2_NO_STATION_MODES
-	{ PRISM2_PARAM_HOST_ROAMING,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "host_roaming" },
-	{ PRISM2_PARAM_HOST_ROAMING,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "gethost_roaming" },
-#endif /* PRISM2_NO_STATION_MODES */
-	{ PRISM2_PARAM_BCRX_STA_KEY,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "bcrx_sta_key" },
-	{ PRISM2_PARAM_BCRX_STA_KEY,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getbcrx_sta_key" },
-	{ PRISM2_PARAM_IEEE_802_1X,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "ieee_802_1x" },
-	{ PRISM2_PARAM_IEEE_802_1X,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getieee_802_1x" },
-	{ PRISM2_PARAM_ANTSEL_TX,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "antsel_tx" },
-	{ PRISM2_PARAM_ANTSEL_TX,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getantsel_tx" },
-	{ PRISM2_PARAM_ANTSEL_RX,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "antsel_rx" },
-	{ PRISM2_PARAM_ANTSEL_RX,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getantsel_rx" },
-	{ PRISM2_PARAM_MONITOR_TYPE,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "monitor_type" },
-	{ PRISM2_PARAM_MONITOR_TYPE,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getmonitor_type" },
-	{ PRISM2_PARAM_WDS_TYPE,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "wds_type" },
-	{ PRISM2_PARAM_WDS_TYPE,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getwds_type" },
-	{ PRISM2_PARAM_HOSTSCAN,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "hostscan" },
-	{ PRISM2_PARAM_HOSTSCAN,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "gethostscan" },
-	{ PRISM2_PARAM_AP_SCAN,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "ap_scan" },
-	{ PRISM2_PARAM_AP_SCAN,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getap_scan" },
-	{ PRISM2_PARAM_ENH_SEC,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "enh_sec" },
-	{ PRISM2_PARAM_ENH_SEC,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getenh_sec" },
-#ifdef PRISM2_IO_DEBUG
-	{ PRISM2_PARAM_IO_DEBUG,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "io_debug" },
-	{ PRISM2_PARAM_IO_DEBUG,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getio_debug" },
-#endif /* PRISM2_IO_DEBUG */
-	{ PRISM2_PARAM_BASIC_RATES,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "basic_rates" },
-	{ PRISM2_PARAM_BASIC_RATES,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getbasic_rates" },
-	{ PRISM2_PARAM_OPER_RATES,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "oper_rates" },
-	{ PRISM2_PARAM_OPER_RATES,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getoper_rates" },
-	{ PRISM2_PARAM_HOSTAPD,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "hostapd" },
-	{ PRISM2_PARAM_HOSTAPD,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "gethostapd" },
-	{ PRISM2_PARAM_HOSTAPD_STA,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "hostapd_sta" },
-	{ PRISM2_PARAM_HOSTAPD_STA,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "gethostapd_sta" },
-	{ PRISM2_PARAM_WPA,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "wpa" },
-	{ PRISM2_PARAM_WPA,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getwpa" },
-	{ PRISM2_PARAM_PRIVACY_INVOKED,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "privacy_invoked" },
-	{ PRISM2_PARAM_PRIVACY_INVOKED,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getprivacy_invo" },
-	{ PRISM2_PARAM_TKIP_COUNTERMEASURES,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "tkip_countermea" },
-	{ PRISM2_PARAM_TKIP_COUNTERMEASURES,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "gettkip_counter" },
-	{ PRISM2_PARAM_DROP_UNENCRYPTED,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "drop_unencrypte" },
-	{ PRISM2_PARAM_DROP_UNENCRYPTED,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getdrop_unencry" },
-	{ PRISM2_PARAM_SCAN_CHANNEL_MASK,
-	  IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, 0, "scan_channels" },
-	{ PRISM2_PARAM_SCAN_CHANNEL_MASK,
-	  0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getscan_channel" },
-};
-
-
-static int prism2_ioctl_priv_prism2_param(struct net_device *dev,
-					  struct iw_request_info *info,
-					  union iwreq_data *uwrq, char *extra)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int *i = (int *) extra;
-	int param = *i;
-	int value = *(i + 1);
-	int ret = 0;
-	u16 val;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	switch (param) {
-	case PRISM2_PARAM_TXRATECTRL:
-		local->fw_tx_rate_control = value;
-		break;
-
-	case PRISM2_PARAM_BEACON_INT:
-		if (hostap_set_word(dev, HFA384X_RID_CNFBEACONINT, value) ||
-		    local->func->reset_port(dev))
-			ret = -EINVAL;
-		else
-			local->beacon_int = value;
-		break;
-
-#ifndef PRISM2_NO_STATION_MODES
-	case PRISM2_PARAM_PSEUDO_IBSS:
-		if (value == local->pseudo_adhoc)
-			break;
-
-		if (value != 0 && value != 1) {
-			ret = -EINVAL;
-			break;
-		}
-
-		printk(KERN_DEBUG "prism2: %s: pseudo IBSS change %d -> %d\n",
-		       dev->name, local->pseudo_adhoc, value);
-		local->pseudo_adhoc = value;
-		if (local->iw_mode != IW_MODE_ADHOC)
-			break;
-
-		if (hostap_set_word(dev, HFA384X_RID_CNFPORTTYPE,
-				    hostap_get_porttype(local))) {
-			ret = -EOPNOTSUPP;
-			break;
-		}
-
-		if (local->func->reset_port(dev))
-			ret = -EINVAL;
-		break;
-#endif /* PRISM2_NO_STATION_MODES */
-
-	case PRISM2_PARAM_ALC:
-		printk(KERN_DEBUG "%s: %s ALC\n", dev->name,
-		       value == 0 ? "Disabling" : "Enabling");
-		val = HFA384X_TEST_CFG_BIT_ALC;
-		local->func->cmd(dev, HFA384X_CMDCODE_TEST |
-				 (HFA384X_TEST_CFG_BITS << 8),
-				 value == 0 ? 0 : 1, &val, NULL);
-		break;
-
-	case PRISM2_PARAM_DUMP:
-		local->frame_dump = value;
-		break;
-
-	case PRISM2_PARAM_OTHER_AP_POLICY:
-		if (value < 0 || value > 3) {
-			ret = -EINVAL;
-			break;
-		}
-		if (local->ap != NULL)
-			local->ap->ap_policy = value;
-		break;
-
-	case PRISM2_PARAM_AP_MAX_INACTIVITY:
-		if (value < 0 || value > 7 * 24 * 60 * 60) {
-			ret = -EINVAL;
-			break;
-		}
-		if (local->ap != NULL)
-			local->ap->max_inactivity = value * HZ;
-		break;
-
-	case PRISM2_PARAM_AP_BRIDGE_PACKETS:
-		if (local->ap != NULL)
-			local->ap->bridge_packets = value;
-		break;
-
-	case PRISM2_PARAM_DTIM_PERIOD:
-		if (value < 0 || value > 65535) {
-			ret = -EINVAL;
-			break;
-		}
-		if (hostap_set_word(dev, HFA384X_RID_CNFOWNDTIMPERIOD, value)
-		    || local->func->reset_port(dev))
-			ret = -EINVAL;
-		else
-			local->dtim_period = value;
-		break;
-
-	case PRISM2_PARAM_AP_NULLFUNC_ACK:
-		if (local->ap != NULL)
-			local->ap->nullfunc_ack = value;
-		break;
-
-	case PRISM2_PARAM_MAX_WDS:
-		local->wds_max_connections = value;
-		break;
-
-	case PRISM2_PARAM_AP_AUTOM_AP_WDS:
-		if (local->ap != NULL) {
-			if (!local->ap->autom_ap_wds && value) {
-				/* add WDS link to all APs in STA table */
-				hostap_add_wds_links(local);
-			}
-			local->ap->autom_ap_wds = value;
-		}
-		break;
-
-	case PRISM2_PARAM_AP_AUTH_ALGS:
-		local->auth_algs = value;
-		if (hostap_set_auth_algs(local))
-			ret = -EINVAL;
-		break;
-
-	case PRISM2_PARAM_MONITOR_ALLOW_FCSERR:
-		local->monitor_allow_fcserr = value;
-		break;
-
-	case PRISM2_PARAM_HOST_ENCRYPT:
-		local->host_encrypt = value;
-		if (hostap_set_encryption(local) ||
-		    local->func->reset_port(dev))
-			ret = -EINVAL;
-		break;
-
-	case PRISM2_PARAM_HOST_DECRYPT:
-		local->host_decrypt = value;
-		if (hostap_set_encryption(local) ||
-		    local->func->reset_port(dev))
-			ret = -EINVAL;
-		break;
-
-#ifndef PRISM2_NO_STATION_MODES
-	case PRISM2_PARAM_HOST_ROAMING:
-		if (value < 0 || value > 2) {
-			ret = -EINVAL;
-			break;
-		}
-		local->host_roaming = value;
-		if (hostap_set_roaming(local) || local->func->reset_port(dev))
-			ret = -EINVAL;
-		break;
-#endif /* PRISM2_NO_STATION_MODES */
-
-	case PRISM2_PARAM_BCRX_STA_KEY:
-		local->bcrx_sta_key = value;
-		break;
-
-	case PRISM2_PARAM_IEEE_802_1X:
-		local->ieee_802_1x = value;
-		break;
-
-	case PRISM2_PARAM_ANTSEL_TX:
-		if (value < 0 || value > HOSTAP_ANTSEL_HIGH) {
-			ret = -EINVAL;
-			break;
-		}
-		local->antsel_tx = value;
-		hostap_set_antsel(local);
-		break;
-
-	case PRISM2_PARAM_ANTSEL_RX:
-		if (value < 0 || value > HOSTAP_ANTSEL_HIGH) {
-			ret = -EINVAL;
-			break;
-		}
-		local->antsel_rx = value;
-		hostap_set_antsel(local);
-		break;
-
-	case PRISM2_PARAM_MONITOR_TYPE:
-		if (value != PRISM2_MONITOR_80211 &&
-		    value != PRISM2_MONITOR_CAPHDR &&
-		    value != PRISM2_MONITOR_PRISM &&
-		    value != PRISM2_MONITOR_RADIOTAP) {
-			ret = -EINVAL;
-			break;
-		}
-		local->monitor_type = value;
-		if (local->iw_mode == IW_MODE_MONITOR)
-			hostap_monitor_set_type(local);
-		break;
-
-	case PRISM2_PARAM_WDS_TYPE:
-		local->wds_type = value;
-		break;
-
-	case PRISM2_PARAM_HOSTSCAN:
-	{
-		struct hfa384x_hostscan_request scan_req;
-		u16 rate;
-
-		memset(&scan_req, 0, sizeof(scan_req));
-		scan_req.channel_list = cpu_to_le16(0x3fff);
-		switch (value) {
-		case 1: rate = HFA384X_RATES_1MBPS; break;
-		case 2: rate = HFA384X_RATES_2MBPS; break;
-		case 3: rate = HFA384X_RATES_5MBPS; break;
-		case 4: rate = HFA384X_RATES_11MBPS; break;
-		default: rate = HFA384X_RATES_1MBPS; break;
-		}
-		scan_req.txrate = cpu_to_le16(rate);
-		/* leave SSID empty to accept all SSIDs */
-
-		if (local->iw_mode == IW_MODE_MASTER) {
-			if (hostap_set_word(dev, HFA384X_RID_CNFPORTTYPE,
-					    HFA384X_PORTTYPE_BSS) ||
-			    local->func->reset_port(dev))
-				printk(KERN_DEBUG "Leaving Host AP mode "
-				       "for HostScan failed\n");
-		}
-
-		if (local->func->set_rid(dev, HFA384X_RID_HOSTSCAN, &scan_req,
-					 sizeof(scan_req))) {
-			printk(KERN_DEBUG "HOSTSCAN failed\n");
-			ret = -EINVAL;
-		}
-		if (local->iw_mode == IW_MODE_MASTER) {
-			wait_queue_entry_t __wait;
-			init_waitqueue_entry(&__wait, current);
-			add_wait_queue(&local->hostscan_wq, &__wait);
-			set_current_state(TASK_INTERRUPTIBLE);
-			schedule_timeout(HZ);
-			if (signal_pending(current))
-				ret = -EINTR;
-			set_current_state(TASK_RUNNING);
-			remove_wait_queue(&local->hostscan_wq, &__wait);
-
-			if (hostap_set_word(dev, HFA384X_RID_CNFPORTTYPE,
-					    HFA384X_PORTTYPE_HOSTAP) ||
-			    local->func->reset_port(dev))
-				printk(KERN_DEBUG "Returning to Host AP mode "
-				       "after HostScan failed\n");
-		}
-		break;
-	}
-
-	case PRISM2_PARAM_AP_SCAN:
-		local->passive_scan_interval = value;
-		if (timer_pending(&local->passive_scan_timer))
-			del_timer(&local->passive_scan_timer);
-		if (value > 0 && value < INT_MAX / HZ) {
-			local->passive_scan_timer.expires = jiffies +
-				local->passive_scan_interval * HZ;
-			add_timer(&local->passive_scan_timer);
-		}
-		break;
-
-	case PRISM2_PARAM_ENH_SEC:
-		if (value < 0 || value > 3) {
-			ret = -EINVAL;
-			break;
-		}
-		local->enh_sec = value;
-		if (hostap_set_word(dev, HFA384X_RID_CNFENHSECURITY,
-				    local->enh_sec) ||
-		    local->func->reset_port(dev)) {
-			printk(KERN_INFO "%s: cnfEnhSecurity requires STA f/w "
-			       "1.6.3 or newer\n", dev->name);
-			ret = -EOPNOTSUPP;
-		}
-		break;
-
-#ifdef PRISM2_IO_DEBUG
-	case PRISM2_PARAM_IO_DEBUG:
-		local->io_debug_enabled = value;
-		break;
-#endif /* PRISM2_IO_DEBUG */
-
-	case PRISM2_PARAM_BASIC_RATES:
-		if ((value & local->tx_rate_control) != value || value == 0) {
-			printk(KERN_INFO "%s: invalid basic rate set - basic "
-			       "rates must be in supported rate set\n",
-			       dev->name);
-			ret = -EINVAL;
-			break;
-		}
-		local->basic_rates = value;
-		if (hostap_set_word(dev, HFA384X_RID_CNFBASICRATES,
-				    local->basic_rates) ||
-		    local->func->reset_port(dev))
-			ret = -EINVAL;
-		break;
-
-	case PRISM2_PARAM_OPER_RATES:
-		local->tx_rate_control = value;
-		if (hostap_set_rate(dev))
-			ret = -EINVAL;
-		break;
-
-	case PRISM2_PARAM_HOSTAPD:
-		ret = hostap_set_hostapd(local, value, 1);
-		break;
-
-	case PRISM2_PARAM_HOSTAPD_STA:
-		ret = hostap_set_hostapd_sta(local, value, 1);
-		break;
-
-	case PRISM2_PARAM_WPA:
-		local->wpa = value;
-		if (local->sta_fw_ver < PRISM2_FW_VER(1,7,0))
-			ret = -EOPNOTSUPP;
-		else if (hostap_set_word(dev, HFA384X_RID_SSNHANDLINGMODE,
-					 value ? 1 : 0))
-			ret = -EINVAL;
-		break;
-
-	case PRISM2_PARAM_PRIVACY_INVOKED:
-		local->privacy_invoked = value;
-		if (hostap_set_encryption(local) ||
-		    local->func->reset_port(dev))
-			ret = -EINVAL;
-		break;
-
-	case PRISM2_PARAM_TKIP_COUNTERMEASURES:
-		local->tkip_countermeasures = value;
-		break;
-
-	case PRISM2_PARAM_DROP_UNENCRYPTED:
-		local->drop_unencrypted = value;
-		break;
-
-	case PRISM2_PARAM_SCAN_CHANNEL_MASK:
-		local->scan_channel_mask = value;
-		break;
-
-	default:
-		printk(KERN_DEBUG "%s: prism2_param: unknown param %d\n",
-		       dev->name, param);
-		ret = -EOPNOTSUPP;
-		break;
-	}
-
-	return ret;
-}
-
-
-static int prism2_ioctl_priv_get_prism2_param(struct net_device *dev,
-					      struct iw_request_info *info,
-					      union iwreq_data *wrqu, char *extra)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int *param = (int *) extra;
-	int ret = 0;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	switch (*param) {
-	case PRISM2_PARAM_TXRATECTRL:
-		*param = local->fw_tx_rate_control;
-		break;
-
-	case PRISM2_PARAM_BEACON_INT:
-		*param = local->beacon_int;
-		break;
-
-	case PRISM2_PARAM_PSEUDO_IBSS:
-		*param = local->pseudo_adhoc;
-		break;
-
-	case PRISM2_PARAM_ALC:
-		ret = -EOPNOTSUPP; /* FIX */
-		break;
-
-	case PRISM2_PARAM_DUMP:
-		*param = local->frame_dump;
-		break;
-
-	case PRISM2_PARAM_OTHER_AP_POLICY:
-		if (local->ap != NULL)
-			*param = local->ap->ap_policy;
-		else
-			ret = -EOPNOTSUPP;
-		break;
-
-	case PRISM2_PARAM_AP_MAX_INACTIVITY:
-		if (local->ap != NULL)
-			*param = local->ap->max_inactivity / HZ;
-		else
-			ret = -EOPNOTSUPP;
-		break;
-
-	case PRISM2_PARAM_AP_BRIDGE_PACKETS:
-		if (local->ap != NULL)
-			*param = local->ap->bridge_packets;
-		else
-			ret = -EOPNOTSUPP;
-		break;
-
-	case PRISM2_PARAM_DTIM_PERIOD:
-		*param = local->dtim_period;
-		break;
-
-	case PRISM2_PARAM_AP_NULLFUNC_ACK:
-		if (local->ap != NULL)
-			*param = local->ap->nullfunc_ack;
-		else
-			ret = -EOPNOTSUPP;
-		break;
-
-	case PRISM2_PARAM_MAX_WDS:
-		*param = local->wds_max_connections;
-		break;
-
-	case PRISM2_PARAM_AP_AUTOM_AP_WDS:
-		if (local->ap != NULL)
-			*param = local->ap->autom_ap_wds;
-		else
-			ret = -EOPNOTSUPP;
-		break;
-
-	case PRISM2_PARAM_AP_AUTH_ALGS:
-		*param = local->auth_algs;
-		break;
-
-	case PRISM2_PARAM_MONITOR_ALLOW_FCSERR:
-		*param = local->monitor_allow_fcserr;
-		break;
-
-	case PRISM2_PARAM_HOST_ENCRYPT:
-		*param = local->host_encrypt;
-		break;
-
-	case PRISM2_PARAM_HOST_DECRYPT:
-		*param = local->host_decrypt;
-		break;
-
-	case PRISM2_PARAM_HOST_ROAMING:
-		*param = local->host_roaming;
-		break;
-
-	case PRISM2_PARAM_BCRX_STA_KEY:
-		*param = local->bcrx_sta_key;
-		break;
-
-	case PRISM2_PARAM_IEEE_802_1X:
-		*param = local->ieee_802_1x;
-		break;
-
-	case PRISM2_PARAM_ANTSEL_TX:
-		*param = local->antsel_tx;
-		break;
-
-	case PRISM2_PARAM_ANTSEL_RX:
-		*param = local->antsel_rx;
-		break;
-
-	case PRISM2_PARAM_MONITOR_TYPE:
-		*param = local->monitor_type;
-		break;
-
-	case PRISM2_PARAM_WDS_TYPE:
-		*param = local->wds_type;
-		break;
-
-	case PRISM2_PARAM_HOSTSCAN:
-		ret = -EOPNOTSUPP;
-		break;
-
-	case PRISM2_PARAM_AP_SCAN:
-		*param = local->passive_scan_interval;
-		break;
-
-	case PRISM2_PARAM_ENH_SEC:
-		*param = local->enh_sec;
-		break;
-
-#ifdef PRISM2_IO_DEBUG
-	case PRISM2_PARAM_IO_DEBUG:
-		*param = local->io_debug_enabled;
-		break;
-#endif /* PRISM2_IO_DEBUG */
-
-	case PRISM2_PARAM_BASIC_RATES:
-		*param = local->basic_rates;
-		break;
-
-	case PRISM2_PARAM_OPER_RATES:
-		*param = local->tx_rate_control;
-		break;
-
-	case PRISM2_PARAM_HOSTAPD:
-		*param = local->hostapd;
-		break;
-
-	case PRISM2_PARAM_HOSTAPD_STA:
-		*param = local->hostapd_sta;
-		break;
-
-	case PRISM2_PARAM_WPA:
-		if (local->sta_fw_ver < PRISM2_FW_VER(1,7,0))
-			ret = -EOPNOTSUPP;
-		*param = local->wpa;
-		break;
-
-	case PRISM2_PARAM_PRIVACY_INVOKED:
-		*param = local->privacy_invoked;
-		break;
-
-	case PRISM2_PARAM_TKIP_COUNTERMEASURES:
-		*param = local->tkip_countermeasures;
-		break;
-
-	case PRISM2_PARAM_DROP_UNENCRYPTED:
-		*param = local->drop_unencrypted;
-		break;
-
-	case PRISM2_PARAM_SCAN_CHANNEL_MASK:
-		*param = local->scan_channel_mask;
-		break;
-
-	default:
-		printk(KERN_DEBUG "%s: get_prism2_param: unknown param %d\n",
-		       dev->name, *param);
-		ret = -EOPNOTSUPP;
-		break;
-	}
-
-	return ret;
-}
-
-
-static int prism2_ioctl_priv_readmif(struct net_device *dev,
-				     struct iw_request_info *info,
-				     union iwreq_data *wrqu, char *extra)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	u16 resp0;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (local->func->cmd(dev, HFA384X_CMDCODE_READMIF, *extra, NULL,
-			     &resp0))
-		return -EOPNOTSUPP;
-	else
-		*extra = resp0;
-
-	return 0;
-}
-
-
-static int prism2_ioctl_priv_writemif(struct net_device *dev,
-				      struct iw_request_info *info,
-				      union iwreq_data *wrqu, char *extra)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	u16 cr, val;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	cr = *extra;
-	val = *(extra + 1);
-	if (local->func->cmd(dev, HFA384X_CMDCODE_WRITEMIF, cr, &val, NULL))
-		return -EOPNOTSUPP;
-
-	return 0;
-}
-
-
-#ifdef PRISM2_DOWNLOAD_SUPPORT
-static int prism2_ioctl_priv_download(local_info_t *local, struct iw_point *p)
-{
-	struct prism2_download_param *param;
-	int ret = 0;
-
-	if (p->length < sizeof(struct prism2_download_param) ||
-	    p->length > 1024 || !p->pointer)
-		return -EINVAL;
-
-	param = memdup_user(p->pointer, p->length);
-	if (IS_ERR(param)) {
-		return PTR_ERR(param);
-	}
-
-	if (p->length < sizeof(struct prism2_download_param) +
-	    param->num_areas * sizeof(struct prism2_download_area)) {
-		ret = -EINVAL;
-		goto out;
-	}
-
-	ret = local->func->download(local, param);
-
- out:
-	kfree(param);
-	return ret;
-}
-#endif /* PRISM2_DOWNLOAD_SUPPORT */
-
-
-static int prism2_set_genericelement(struct net_device *dev, u8 *elem,
-				     size_t len)
-{
-	struct hostap_interface *iface = netdev_priv(dev);
-	local_info_t *local = iface->local;
-	u8 *buf;
-
-	/*
-	 * Add 16-bit length in the beginning of the buffer because Prism2 RID
-	 * includes it.
-	 */
-	buf = kmalloc(len + 2, GFP_KERNEL);
-	if (buf == NULL)
-		return -ENOMEM;
-
-	*((__le16 *) buf) = cpu_to_le16(len);
-	memcpy(buf + 2, elem, len);
-
-	kfree(local->generic_elem);
-	local->generic_elem = buf;
-	local->generic_elem_len = len + 2;
-
-	return local->func->set_rid(local->dev, HFA384X_RID_GENERICELEMENT,
-				    buf, len + 2);
-}
-
-
-static int prism2_ioctl_siwauth(struct net_device *dev,
-				struct iw_request_info *info,
-				union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *data = &wrqu->param;
-	struct hostap_interface *iface = netdev_priv(dev);
-	local_info_t *local = iface->local;
-
-	switch (data->flags & IW_AUTH_INDEX) {
-	case IW_AUTH_WPA_VERSION:
-	case IW_AUTH_CIPHER_PAIRWISE:
-	case IW_AUTH_CIPHER_GROUP:
-	case IW_AUTH_KEY_MGMT:
-		/*
-		 * Host AP driver does not use these parameters and allows
-		 * wpa_supplicant to control them internally.
-		 */
-		break;
-	case IW_AUTH_TKIP_COUNTERMEASURES:
-		local->tkip_countermeasures = data->value;
-		break;
-	case IW_AUTH_DROP_UNENCRYPTED:
-		local->drop_unencrypted = data->value;
-		break;
-	case IW_AUTH_80211_AUTH_ALG:
-		local->auth_algs = data->value;
-		break;
-	case IW_AUTH_WPA_ENABLED:
-		if (data->value == 0) {
-			local->wpa = 0;
-			if (local->sta_fw_ver < PRISM2_FW_VER(1,7,0))
-				break;
-			prism2_set_genericelement(dev, "", 0);
-			local->host_roaming = 0;
-			local->privacy_invoked = 0;
-			if (hostap_set_word(dev, HFA384X_RID_SSNHANDLINGMODE,
-					    0) ||
-			    hostap_set_roaming(local) ||
-			    hostap_set_encryption(local) ||
-			    local->func->reset_port(dev))
-				return -EINVAL;
-			break;
-		}
-		if (local->sta_fw_ver < PRISM2_FW_VER(1,7,0))
-			return -EOPNOTSUPP;
-		local->host_roaming = 2;
-		local->privacy_invoked = 1;
-		local->wpa = 1;
-		if (hostap_set_word(dev, HFA384X_RID_SSNHANDLINGMODE, 1) ||
-		    hostap_set_roaming(local) ||
-		    hostap_set_encryption(local) ||
-		    local->func->reset_port(dev))
-			return -EINVAL;
-		break;
-	case IW_AUTH_RX_UNENCRYPTED_EAPOL:
-		local->ieee_802_1x = data->value;
-		break;
-	case IW_AUTH_PRIVACY_INVOKED:
-		local->privacy_invoked = data->value;
-		break;
-	default:
-		return -EOPNOTSUPP;
-	}
-	return 0;
-}
-
-
-static int prism2_ioctl_giwauth(struct net_device *dev,
-				struct iw_request_info *info,
-				union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *data = &wrqu->param;
-	struct hostap_interface *iface = netdev_priv(dev);
-	local_info_t *local = iface->local;
-
-	switch (data->flags & IW_AUTH_INDEX) {
-	case IW_AUTH_WPA_VERSION:
-	case IW_AUTH_CIPHER_PAIRWISE:
-	case IW_AUTH_CIPHER_GROUP:
-	case IW_AUTH_KEY_MGMT:
-		/*
-		 * Host AP driver does not use these parameters and allows
-		 * wpa_supplicant to control them internally.
-		 */
-		return -EOPNOTSUPP;
-	case IW_AUTH_TKIP_COUNTERMEASURES:
-		data->value = local->tkip_countermeasures;
-		break;
-	case IW_AUTH_DROP_UNENCRYPTED:
-		data->value = local->drop_unencrypted;
-		break;
-	case IW_AUTH_80211_AUTH_ALG:
-		data->value = local->auth_algs;
-		break;
-	case IW_AUTH_WPA_ENABLED:
-		data->value = local->wpa;
-		break;
-	case IW_AUTH_RX_UNENCRYPTED_EAPOL:
-		data->value = local->ieee_802_1x;
-		break;
-	default:
-		return -EOPNOTSUPP;
-	}
-	return 0;
-}
-
-
-static int prism2_ioctl_siwencodeext(struct net_device *dev,
-				     struct iw_request_info *info,
-				     union iwreq_data *wrqu, char *extra)
-{
-	struct iw_point *erq = &wrqu->encoding;
-	struct hostap_interface *iface = netdev_priv(dev);
-	local_info_t *local = iface->local;
-	struct iw_encode_ext *ext = (struct iw_encode_ext *) extra;
-	int i, ret = 0;
-	struct lib80211_crypto_ops *ops;
-	struct lib80211_crypt_data **crypt;
-	void *sta_ptr;
-	u8 *addr;
-	const char *alg, *module;
-
-	i = erq->flags & IW_ENCODE_INDEX;
-	if (i > WEP_KEYS)
-		return -EINVAL;
-	if (i < 1 || i > WEP_KEYS)
-		i = local->crypt_info.tx_keyidx;
-	else
-		i--;
-	if (i < 0 || i >= WEP_KEYS)
-		return -EINVAL;
-
-	addr = ext->addr.sa_data;
-	if (is_broadcast_ether_addr(addr)) {
-		sta_ptr = NULL;
-		crypt = &local->crypt_info.crypt[i];
-	} else {
-		if (i != 0)
-			return -EINVAL;
-		sta_ptr = ap_crypt_get_ptrs(local->ap, addr, 0, &crypt);
-		if (sta_ptr == NULL) {
-			if (local->iw_mode == IW_MODE_INFRA) {
-				/*
-				 * TODO: add STA entry for the current AP so
-				 * that unicast key can be used. For now, this
-				 * is emulated by using default key idx 0.
-				 */
-				i = 0;
-				crypt = &local->crypt_info.crypt[i];
-			} else
-				return -EINVAL;
-		}
-	}
-
-	if ((erq->flags & IW_ENCODE_DISABLED) ||
-	    ext->alg == IW_ENCODE_ALG_NONE) {
-		if (*crypt)
-			lib80211_crypt_delayed_deinit(&local->crypt_info, crypt);
-		goto done;
-	}
-
-	switch (ext->alg) {
-	case IW_ENCODE_ALG_WEP:
-		alg = "WEP";
-		module = "lib80211_crypt_wep";
-		break;
-	case IW_ENCODE_ALG_TKIP:
-		alg = "TKIP";
-		module = "lib80211_crypt_tkip";
-		break;
-	case IW_ENCODE_ALG_CCMP:
-		alg = "CCMP";
-		module = "lib80211_crypt_ccmp";
-		break;
-	default:
-		printk(KERN_DEBUG "%s: unsupported algorithm %d\n",
-		       local->dev->name, ext->alg);
-		ret = -EOPNOTSUPP;
-		goto done;
-	}
-
-	ops = lib80211_get_crypto_ops(alg);
-	if (ops == NULL) {
-		request_module(module);
-		ops = lib80211_get_crypto_ops(alg);
-	}
-	if (ops == NULL) {
-		printk(KERN_DEBUG "%s: unknown crypto alg '%s'\n",
-		       local->dev->name, alg);
-		ret = -EOPNOTSUPP;
-		goto done;
-	}
-
-	if (sta_ptr || ext->alg != IW_ENCODE_ALG_WEP) {
-		/*
-		 * Per station encryption and other than WEP algorithms
-		 * require host-based encryption, so force them on
-		 * automatically.
-		 */
-		local->host_decrypt = local->host_encrypt = 1;
-	}
-
-	if (*crypt == NULL || (*crypt)->ops != ops) {
-		struct lib80211_crypt_data *new_crypt;
-
-		lib80211_crypt_delayed_deinit(&local->crypt_info, crypt);
-
-		new_crypt = kzalloc(sizeof(struct lib80211_crypt_data),
-				GFP_KERNEL);
-		if (new_crypt == NULL) {
-			ret = -ENOMEM;
-			goto done;
-		}
-		new_crypt->ops = ops;
-		if (new_crypt->ops && try_module_get(new_crypt->ops->owner))
-			new_crypt->priv = new_crypt->ops->init(i);
-		if (new_crypt->priv == NULL) {
-			kfree(new_crypt);
-			ret = -EINVAL;
-			goto done;
-		}
-
-		*crypt = new_crypt;
-	}
-
-	/*
-	 * TODO: if ext_flags does not have IW_ENCODE_EXT_RX_SEQ_VALID, the
-	 * existing seq# should not be changed.
-	 * TODO: if ext_flags has IW_ENCODE_EXT_TX_SEQ_VALID, next TX seq#
-	 * should be changed to something else than zero.
-	 */
-	if ((!(ext->ext_flags & IW_ENCODE_EXT_SET_TX_KEY) || ext->key_len > 0)
-	    && (*crypt)->ops->set_key &&
-	    (*crypt)->ops->set_key(ext->key, ext->key_len, ext->rx_seq,
-				   (*crypt)->priv) < 0) {
-		printk(KERN_DEBUG "%s: key setting failed\n",
-		       local->dev->name);
-		ret = -EINVAL;
-		goto done;
-	}
-
-	if (ext->ext_flags & IW_ENCODE_EXT_SET_TX_KEY) {
-		if (!sta_ptr)
-			local->crypt_info.tx_keyidx = i;
-	}
-
-
-	if (sta_ptr == NULL && ext->key_len > 0) {
-		int first = 1, j;
-		for (j = 0; j < WEP_KEYS; j++) {
-			if (j != i && local->crypt_info.crypt[j]) {
-				first = 0;
-				break;
-			}
-		}
-		if (first)
-			local->crypt_info.tx_keyidx = i;
-	}
-
- done:
-	if (sta_ptr)
-		hostap_handle_sta_release(sta_ptr);
-
-	local->open_wep = erq->flags & IW_ENCODE_OPEN;
-
-	/*
-	 * Do not reset port0 if card is in Managed mode since resetting will
-	 * generate new IEEE 802.11 authentication which may end up in looping
-	 * with IEEE 802.1X. Prism2 documentation seem to require port reset
-	 * after WEP configuration. However, keys are apparently changed at
-	 * least in Managed mode.
-	 */
-	if (ret == 0 &&
-	    (hostap_set_encryption(local) ||
-	     (local->iw_mode != IW_MODE_INFRA &&
-	      local->func->reset_port(local->dev))))
-		ret = -EINVAL;
-
-	return ret;
-}
-
-
-static int prism2_ioctl_giwencodeext(struct net_device *dev,
-				     struct iw_request_info *info,
-				     union iwreq_data *wrqu, char *extra)
-{
-	struct iw_point *erq = &wrqu->encoding;
-	struct hostap_interface *iface = netdev_priv(dev);
-	local_info_t *local = iface->local;
-	struct lib80211_crypt_data **crypt;
-	void *sta_ptr;
-	int max_key_len, i;
-	struct iw_encode_ext *ext = (struct iw_encode_ext *) extra;
-	u8 *addr;
-
-	max_key_len = erq->length - sizeof(*ext);
-	if (max_key_len < 0)
-		return -EINVAL;
-
-	i = erq->flags & IW_ENCODE_INDEX;
-	if (i < 1 || i > WEP_KEYS)
-		i = local->crypt_info.tx_keyidx;
-	else
-		i--;
-
-	addr = ext->addr.sa_data;
-	if (is_broadcast_ether_addr(addr)) {
-		sta_ptr = NULL;
-		crypt = &local->crypt_info.crypt[i];
-	} else {
-		i = 0;
-		sta_ptr = ap_crypt_get_ptrs(local->ap, addr, 0, &crypt);
-		if (sta_ptr == NULL)
-			return -EINVAL;
-	}
-	erq->flags = i + 1;
-	memset(ext, 0, sizeof(*ext));
-
-	if (*crypt == NULL || (*crypt)->ops == NULL) {
-		ext->alg = IW_ENCODE_ALG_NONE;
-		ext->key_len = 0;
-		erq->flags |= IW_ENCODE_DISABLED;
-	} else {
-		if (strcmp((*crypt)->ops->name, "WEP") == 0)
-			ext->alg = IW_ENCODE_ALG_WEP;
-		else if (strcmp((*crypt)->ops->name, "TKIP") == 0)
-			ext->alg = IW_ENCODE_ALG_TKIP;
-		else if (strcmp((*crypt)->ops->name, "CCMP") == 0)
-			ext->alg = IW_ENCODE_ALG_CCMP;
-		else
-			return -EINVAL;
-
-		if ((*crypt)->ops->get_key) {
-			ext->key_len =
-				(*crypt)->ops->get_key(ext->key,
-						       max_key_len,
-						       ext->tx_seq,
-						       (*crypt)->priv);
-			if (ext->key_len &&
-			    (ext->alg == IW_ENCODE_ALG_TKIP ||
-			     ext->alg == IW_ENCODE_ALG_CCMP))
-				ext->ext_flags |= IW_ENCODE_EXT_TX_SEQ_VALID;
-		}
-	}
-
-	if (sta_ptr)
-		hostap_handle_sta_release(sta_ptr);
-
-	return 0;
-}
-
-
-static int prism2_ioctl_set_encryption(local_info_t *local,
-				       struct prism2_hostapd_param *param,
-				       int param_len)
-{
-	int ret = 0;
-	struct lib80211_crypto_ops *ops;
-	struct lib80211_crypt_data **crypt;
-	void *sta_ptr;
-
-	param->u.crypt.err = 0;
-	param->u.crypt.alg[HOSTAP_CRYPT_ALG_NAME_LEN - 1] = '\0';
-
-	if (param_len !=
-	    (int) ((char *) param->u.crypt.key - (char *) param) +
-	    param->u.crypt.key_len)
-		return -EINVAL;
-
-	if (is_broadcast_ether_addr(param->sta_addr)) {
-		if (param->u.crypt.idx >= WEP_KEYS)
-			return -EINVAL;
-		sta_ptr = NULL;
-		crypt = &local->crypt_info.crypt[param->u.crypt.idx];
-	} else {
-		if (param->u.crypt.idx)
-			return -EINVAL;
-		sta_ptr = ap_crypt_get_ptrs(
-			local->ap, param->sta_addr,
-			(param->u.crypt.flags & HOSTAP_CRYPT_FLAG_PERMANENT),
-			&crypt);
-
-		if (sta_ptr == NULL) {
-			param->u.crypt.err = HOSTAP_CRYPT_ERR_UNKNOWN_ADDR;
-			return -EINVAL;
-		}
-	}
-
-	if (strcmp(param->u.crypt.alg, "none") == 0) {
-		if (crypt)
-			lib80211_crypt_delayed_deinit(&local->crypt_info, crypt);
-		goto done;
-	}
-
-	ops = lib80211_get_crypto_ops(param->u.crypt.alg);
-	if (ops == NULL && strcmp(param->u.crypt.alg, "WEP") == 0) {
-		request_module("lib80211_crypt_wep");
-		ops = lib80211_get_crypto_ops(param->u.crypt.alg);
-	} else if (ops == NULL && strcmp(param->u.crypt.alg, "TKIP") == 0) {
-		request_module("lib80211_crypt_tkip");
-		ops = lib80211_get_crypto_ops(param->u.crypt.alg);
-	} else if (ops == NULL && strcmp(param->u.crypt.alg, "CCMP") == 0) {
-		request_module("lib80211_crypt_ccmp");
-		ops = lib80211_get_crypto_ops(param->u.crypt.alg);
-	}
-	if (ops == NULL) {
-		printk(KERN_DEBUG "%s: unknown crypto alg '%s'\n",
-		       local->dev->name, param->u.crypt.alg);
-		param->u.crypt.err = HOSTAP_CRYPT_ERR_UNKNOWN_ALG;
-		ret = -EINVAL;
-		goto done;
-	}
-
-	/* station based encryption and other than WEP algorithms require
-	 * host-based encryption, so force them on automatically */
-	local->host_decrypt = local->host_encrypt = 1;
-
-	if (*crypt == NULL || (*crypt)->ops != ops) {
-		struct lib80211_crypt_data *new_crypt;
-
-		lib80211_crypt_delayed_deinit(&local->crypt_info, crypt);
-
-		new_crypt = kzalloc(sizeof(struct lib80211_crypt_data),
-				GFP_KERNEL);
-		if (new_crypt == NULL) {
-			ret = -ENOMEM;
-			goto done;
-		}
-		new_crypt->ops = ops;
-		new_crypt->priv = new_crypt->ops->init(param->u.crypt.idx);
-		if (new_crypt->priv == NULL) {
-			kfree(new_crypt);
-			param->u.crypt.err =
-				HOSTAP_CRYPT_ERR_CRYPT_INIT_FAILED;
-			ret = -EINVAL;
-			goto done;
-		}
-
-		*crypt = new_crypt;
-	}
-
-	if ((!(param->u.crypt.flags & HOSTAP_CRYPT_FLAG_SET_TX_KEY) ||
-	     param->u.crypt.key_len > 0) && (*crypt)->ops->set_key &&
-	    (*crypt)->ops->set_key(param->u.crypt.key,
-				   param->u.crypt.key_len, param->u.crypt.seq,
-				   (*crypt)->priv) < 0) {
-		printk(KERN_DEBUG "%s: key setting failed\n",
-		       local->dev->name);
-		param->u.crypt.err = HOSTAP_CRYPT_ERR_KEY_SET_FAILED;
-		ret = -EINVAL;
-		goto done;
-	}
-
-	if (param->u.crypt.flags & HOSTAP_CRYPT_FLAG_SET_TX_KEY) {
-		if (!sta_ptr)
-			local->crypt_info.tx_keyidx = param->u.crypt.idx;
-		else if (param->u.crypt.idx) {
-			printk(KERN_DEBUG "%s: TX key idx setting failed\n",
-			       local->dev->name);
-			param->u.crypt.err =
-				HOSTAP_CRYPT_ERR_TX_KEY_SET_FAILED;
-			ret = -EINVAL;
-			goto done;
-		}
-	}
-
- done:
-	if (sta_ptr)
-		hostap_handle_sta_release(sta_ptr);
-
-	/* Do not reset port0 if card is in Managed mode since resetting will
-	 * generate new IEEE 802.11 authentication which may end up in looping
-	 * with IEEE 802.1X. Prism2 documentation seem to require port reset
-	 * after WEP configuration. However, keys are apparently changed at
-	 * least in Managed mode. */
-	if (ret == 0 &&
-	    (hostap_set_encryption(local) ||
-	     (local->iw_mode != IW_MODE_INFRA &&
-	      local->func->reset_port(local->dev)))) {
-		param->u.crypt.err = HOSTAP_CRYPT_ERR_CARD_CONF_FAILED;
-		return -EINVAL;
-	}
-
-	return ret;
-}
-
-
-static int prism2_ioctl_get_encryption(local_info_t *local,
-				       struct prism2_hostapd_param *param,
-				       int param_len)
-{
-	struct lib80211_crypt_data **crypt;
-	void *sta_ptr;
-	int max_key_len;
-
-	param->u.crypt.err = 0;
-
-	max_key_len = param_len -
-		(int) ((char *) param->u.crypt.key - (char *) param);
-	if (max_key_len < 0)
-		return -EINVAL;
-
-	if (is_broadcast_ether_addr(param->sta_addr)) {
-		sta_ptr = NULL;
-		if (param->u.crypt.idx >= WEP_KEYS)
-			param->u.crypt.idx = local->crypt_info.tx_keyidx;
-		crypt = &local->crypt_info.crypt[param->u.crypt.idx];
-	} else {
-		param->u.crypt.idx = 0;
-		sta_ptr = ap_crypt_get_ptrs(local->ap, param->sta_addr, 0,
-					    &crypt);
-
-		if (sta_ptr == NULL) {
-			param->u.crypt.err = HOSTAP_CRYPT_ERR_UNKNOWN_ADDR;
-			return -EINVAL;
-		}
-	}
-
-	if (*crypt == NULL || (*crypt)->ops == NULL) {
-		memcpy(param->u.crypt.alg, "none", 5);
-		param->u.crypt.key_len = 0;
-		param->u.crypt.idx = 0xff;
-	} else {
-		strscpy(param->u.crypt.alg, (*crypt)->ops->name,
-			HOSTAP_CRYPT_ALG_NAME_LEN);
-		param->u.crypt.key_len = 0;
-
-		memset(param->u.crypt.seq, 0, 8);
-		if ((*crypt)->ops->get_key) {
-			param->u.crypt.key_len =
-				(*crypt)->ops->get_key(param->u.crypt.key,
-						       max_key_len,
-						       param->u.crypt.seq,
-						       (*crypt)->priv);
-		}
-	}
-
-	if (sta_ptr)
-		hostap_handle_sta_release(sta_ptr);
-
-	return 0;
-}
-
-
-static int prism2_ioctl_get_rid(local_info_t *local,
-				struct prism2_hostapd_param *param,
-				int param_len)
-{
-	int max_len, res;
-
-	max_len = param_len - PRISM2_HOSTAPD_RID_HDR_LEN;
-	if (max_len < 0)
-		return -EINVAL;
-
-	res = local->func->get_rid(local->dev, param->u.rid.rid,
-				   param->u.rid.data, param->u.rid.len, 0);
-	if (res >= 0) {
-		param->u.rid.len = res;
-		return 0;
-	}
-
-	return res;
-}
-
-
-static int prism2_ioctl_set_rid(local_info_t *local,
-				struct prism2_hostapd_param *param,
-				int param_len)
-{
-	int max_len;
-
-	max_len = param_len - PRISM2_HOSTAPD_RID_HDR_LEN;
-	if (max_len < 0 || max_len < param->u.rid.len)
-		return -EINVAL;
-
-	return local->func->set_rid(local->dev, param->u.rid.rid,
-				    param->u.rid.data, param->u.rid.len);
-}
-
-
-static int prism2_ioctl_set_assoc_ap_addr(local_info_t *local,
-					  struct prism2_hostapd_param *param,
-					  int param_len)
-{
-	printk(KERN_DEBUG "%ssta: associated as client with AP %pM\n",
-	       local->dev->name, param->sta_addr);
-	memcpy(local->assoc_ap_addr, param->sta_addr, ETH_ALEN);
-	return 0;
-}
-
-
-static int prism2_ioctl_siwgenie(struct net_device *dev,
-				 struct iw_request_info *info,
-				 union iwreq_data *wrqu, char *extra)
-{
-	struct iw_point *data = &wrqu->data;
-	return prism2_set_genericelement(dev, extra, data->length);
-}
-
-
-static int prism2_ioctl_giwgenie(struct net_device *dev,
-				 struct iw_request_info *info,
-				 union iwreq_data *wrqu, char *extra)
-{
-	struct iw_point *data = &wrqu->data;
-	struct hostap_interface *iface = netdev_priv(dev);
-	local_info_t *local = iface->local;
-	int len = local->generic_elem_len - 2;
-
-	if (len <= 0 || local->generic_elem == NULL) {
-		data->length = 0;
-		return 0;
-	}
-
-	if (data->length < len)
-		return -E2BIG;
-
-	data->length = len;
-	memcpy(extra, local->generic_elem + 2, len);
-
-	return 0;
-}
-
-
-static int prism2_ioctl_set_generic_element(local_info_t *local,
-					    struct prism2_hostapd_param *param,
-					    int param_len)
-{
-	int max_len, len;
-
-	len = param->u.generic_elem.len;
-	max_len = param_len - PRISM2_HOSTAPD_GENERIC_ELEMENT_HDR_LEN;
-	if (max_len < 0 || max_len < len)
-		return -EINVAL;
-
-	return prism2_set_genericelement(local->dev,
-					 param->u.generic_elem.data, len);
-}
-
-
-static int prism2_ioctl_siwmlme(struct net_device *dev,
-				struct iw_request_info *info,
-				union iwreq_data *wrqu, char *extra)
-{
-	struct hostap_interface *iface = netdev_priv(dev);
-	local_info_t *local = iface->local;
-	struct iw_mlme *mlme = (struct iw_mlme *) extra;
-	__le16 reason;
-
-	reason = cpu_to_le16(mlme->reason_code);
-
-	switch (mlme->cmd) {
-	case IW_MLME_DEAUTH:
-		return prism2_sta_send_mgmt(local, mlme->addr.sa_data,
-					    IEEE80211_STYPE_DEAUTH,
-					    (u8 *) &reason, 2);
-	case IW_MLME_DISASSOC:
-		return prism2_sta_send_mgmt(local, mlme->addr.sa_data,
-					    IEEE80211_STYPE_DISASSOC,
-					    (u8 *) &reason, 2);
-	default:
-		return -EOPNOTSUPP;
-	}
-}
-
-
-static int prism2_ioctl_mlme(local_info_t *local,
-			     struct prism2_hostapd_param *param)
-{
-	__le16 reason;
-
-	reason = cpu_to_le16(param->u.mlme.reason_code);
-	switch (param->u.mlme.cmd) {
-	case MLME_STA_DEAUTH:
-		return prism2_sta_send_mgmt(local, param->sta_addr,
-					    IEEE80211_STYPE_DEAUTH,
-					    (u8 *) &reason, 2);
-	case MLME_STA_DISASSOC:
-		return prism2_sta_send_mgmt(local, param->sta_addr,
-					    IEEE80211_STYPE_DISASSOC,
-					    (u8 *) &reason, 2);
-	default:
-		return -EOPNOTSUPP;
-	}
-}
-
-
-static int prism2_ioctl_scan_req(local_info_t *local,
-				 struct prism2_hostapd_param *param)
-{
-#ifndef PRISM2_NO_STATION_MODES
-	if ((local->iw_mode != IW_MODE_INFRA &&
-	     local->iw_mode != IW_MODE_ADHOC) ||
-	    (local->sta_fw_ver < PRISM2_FW_VER(1,3,1)))
-		return -EOPNOTSUPP;
-
-	if (!local->dev_enabled)
-		return -ENETDOWN;
-
-	return prism2_request_hostscan(local->dev, param->u.scan_req.ssid,
-				       param->u.scan_req.ssid_len);
-#else /* PRISM2_NO_STATION_MODES */
-	return -EOPNOTSUPP;
-#endif /* PRISM2_NO_STATION_MODES */
-}
-
-
-static int prism2_ioctl_priv_hostapd(local_info_t *local, struct iw_point *p)
-{
-	struct prism2_hostapd_param *param;
-	int ret = 0;
-	int ap_ioctl = 0;
-
-	if (p->length < sizeof(struct prism2_hostapd_param) ||
-	    p->length > PRISM2_HOSTAPD_MAX_BUF_SIZE || !p->pointer)
-		return -EINVAL;
-
-	param = memdup_user(p->pointer, p->length);
-	if (IS_ERR(param)) {
-		return PTR_ERR(param);
-	}
-
-	switch (param->cmd) {
-	case PRISM2_SET_ENCRYPTION:
-		ret = prism2_ioctl_set_encryption(local, param, p->length);
-		break;
-	case PRISM2_GET_ENCRYPTION:
-		ret = prism2_ioctl_get_encryption(local, param, p->length);
-		break;
-	case PRISM2_HOSTAPD_GET_RID:
-		ret = prism2_ioctl_get_rid(local, param, p->length);
-		break;
-	case PRISM2_HOSTAPD_SET_RID:
-		ret = prism2_ioctl_set_rid(local, param, p->length);
-		break;
-	case PRISM2_HOSTAPD_SET_ASSOC_AP_ADDR:
-		ret = prism2_ioctl_set_assoc_ap_addr(local, param, p->length);
-		break;
-	case PRISM2_HOSTAPD_SET_GENERIC_ELEMENT:
-		ret = prism2_ioctl_set_generic_element(local, param,
-						       p->length);
-		break;
-	case PRISM2_HOSTAPD_MLME:
-		ret = prism2_ioctl_mlme(local, param);
-		break;
-	case PRISM2_HOSTAPD_SCAN_REQ:
-		ret = prism2_ioctl_scan_req(local, param);
-		break;
-	default:
-		ret = prism2_hostapd(local->ap, param);
-		ap_ioctl = 1;
-		break;
-	}
-
-	if (ret == 1 || !ap_ioctl) {
-		if (copy_to_user(p->pointer, param, p->length)) {
-			ret = -EFAULT;
-			goto out;
-		} else if (ap_ioctl)
-			ret = 0;
-	}
-
- out:
-	kfree(param);
-	return ret;
-}
-
-
-static void prism2_get_drvinfo(struct net_device *dev,
-			       struct ethtool_drvinfo *info)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	strscpy(info->driver, "hostap", sizeof(info->driver));
-	snprintf(info->fw_version, sizeof(info->fw_version),
-		 "%d.%d.%d", (local->sta_fw_ver >> 16) & 0xff,
-		 (local->sta_fw_ver >> 8) & 0xff,
-		 local->sta_fw_ver & 0xff);
-}
-
-const struct ethtool_ops prism2_ethtool_ops = {
-	.get_drvinfo = prism2_get_drvinfo
-};
-
-
-/* Structures to export the Wireless Handlers */
-
-static const iw_handler prism2_handler[] =
-{
-	IW_HANDLER(SIOCGIWNAME,		prism2_get_name),
-	IW_HANDLER(SIOCSIWFREQ,		prism2_ioctl_siwfreq),
-	IW_HANDLER(SIOCGIWFREQ,		prism2_ioctl_giwfreq),
-	IW_HANDLER(SIOCSIWMODE,		prism2_ioctl_siwmode),
-	IW_HANDLER(SIOCGIWMODE,		prism2_ioctl_giwmode),
-	IW_HANDLER(SIOCSIWSENS,		prism2_ioctl_siwsens),
-	IW_HANDLER(SIOCGIWSENS,		prism2_ioctl_giwsens),
-	IW_HANDLER(SIOCGIWRANGE,	prism2_ioctl_giwrange),
-	IW_HANDLER(SIOCSIWSPY,		iw_handler_set_spy),
-	IW_HANDLER(SIOCGIWSPY,		iw_handler_get_spy),
-	IW_HANDLER(SIOCSIWTHRSPY,	iw_handler_set_thrspy),
-	IW_HANDLER(SIOCGIWTHRSPY,	iw_handler_get_thrspy),
-	IW_HANDLER(SIOCSIWAP,		prism2_ioctl_siwap),
-	IW_HANDLER(SIOCGIWAP,		prism2_ioctl_giwap),
-	IW_HANDLER(SIOCSIWMLME,		prism2_ioctl_siwmlme),
-	IW_HANDLER(SIOCGIWAPLIST,       prism2_ioctl_giwaplist),
-	IW_HANDLER(SIOCSIWSCAN,		prism2_ioctl_siwscan),
-	IW_HANDLER(SIOCGIWSCAN,		prism2_ioctl_giwscan),
-	IW_HANDLER(SIOCSIWESSID,        prism2_ioctl_siwessid),
-	IW_HANDLER(SIOCGIWESSID,        prism2_ioctl_giwessid),
-	IW_HANDLER(SIOCSIWNICKN,        prism2_ioctl_siwnickn),
-	IW_HANDLER(SIOCGIWNICKN,        prism2_ioctl_giwnickn),
-	IW_HANDLER(SIOCSIWRATE,		prism2_ioctl_siwrate),
-	IW_HANDLER(SIOCGIWRATE,		prism2_ioctl_giwrate),
-	IW_HANDLER(SIOCSIWRTS,		prism2_ioctl_siwrts),
-	IW_HANDLER(SIOCGIWRTS,		prism2_ioctl_giwrts),
-	IW_HANDLER(SIOCSIWFRAG,		prism2_ioctl_siwfrag),
-	IW_HANDLER(SIOCGIWFRAG,		prism2_ioctl_giwfrag),
-	IW_HANDLER(SIOCSIWTXPOW,        prism2_ioctl_siwtxpow),
-	IW_HANDLER(SIOCGIWTXPOW,        prism2_ioctl_giwtxpow),
-	IW_HANDLER(SIOCSIWRETRY,        prism2_ioctl_siwretry),
-	IW_HANDLER(SIOCGIWRETRY,        prism2_ioctl_giwretry),
-	IW_HANDLER(SIOCSIWENCODE,       prism2_ioctl_siwencode),
-	IW_HANDLER(SIOCGIWENCODE,       prism2_ioctl_giwencode),
-	IW_HANDLER(SIOCSIWPOWER,        prism2_ioctl_siwpower),
-	IW_HANDLER(SIOCGIWPOWER,        prism2_ioctl_giwpower),
-	IW_HANDLER(SIOCSIWGENIE,        prism2_ioctl_siwgenie),
-	IW_HANDLER(SIOCGIWGENIE,        prism2_ioctl_giwgenie),
-	IW_HANDLER(SIOCSIWAUTH,		prism2_ioctl_siwauth),
-	IW_HANDLER(SIOCGIWAUTH,		prism2_ioctl_giwauth),
-	IW_HANDLER(SIOCSIWENCODEEXT,    prism2_ioctl_siwencodeext),
-	IW_HANDLER(SIOCGIWENCODEEXT,    prism2_ioctl_giwencodeext),
-};
-
-static const iw_handler prism2_private_handler[] =
-{						/* SIOCIWFIRSTPRIV + */
-	prism2_ioctl_priv_prism2_param,		/* 0 */
-	prism2_ioctl_priv_get_prism2_param,	/* 1 */
-	prism2_ioctl_priv_writemif,		/* 2 */
-	prism2_ioctl_priv_readmif,		/* 3 */
-};
-
-const struct iw_handler_def hostap_iw_handler_def =
-{
-	.num_standard	= ARRAY_SIZE(prism2_handler),
-	.num_private	= ARRAY_SIZE(prism2_private_handler),
-	.num_private_args = ARRAY_SIZE(prism2_priv),
-	.standard	= prism2_handler,
-	.private	= prism2_private_handler,
-	.private_args	= (struct iw_priv_args *) prism2_priv,
-	.get_wireless_stats = hostap_get_wireless_stats,
-};
-
-
-/* Private ioctls that are not used with iwpriv;
- * in SIOCDEVPRIVATE range */
-int hostap_siocdevprivate(struct net_device *dev, struct ifreq *ifr,
-			  void __user *data, int cmd)
-{
-	struct iwreq *wrq = (struct iwreq *)ifr;
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int ret = 0;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (in_compat_syscall()) /* not implemented yet */
-		return -EOPNOTSUPP;
-
-	switch (cmd) {
-#ifdef PRISM2_DOWNLOAD_SUPPORT
-	case PRISM2_IOCTL_DOWNLOAD:
-		if (!capable(CAP_NET_ADMIN)) ret = -EPERM;
-		else ret = prism2_ioctl_priv_download(local, &wrq->u.data);
-		break;
-#endif /* PRISM2_DOWNLOAD_SUPPORT */
-
-	case PRISM2_IOCTL_HOSTAPD:
-		if (!capable(CAP_NET_ADMIN)) ret = -EPERM;
-		else ret = prism2_ioctl_priv_hostapd(local, &wrq->u.data);
-		break;
-
-	default:
-		ret = -EOPNOTSUPP;
-		break;
-	}
-
-	return ret;
-}
diff --git a/drivers/net/wireless/intersil/hostap/hostap_main.c b/drivers/net/wireless/intersil/hostap/hostap_main.c
deleted file mode 100644
index bf86ac26c2ac..000000000000
--- a/drivers/net/wireless/intersil/hostap/hostap_main.c
+++ /dev/null
@@ -1,1123 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Host AP (software wireless LAN access point) driver for
- * Intersil Prism2/2.5/3 - hostap.o module, common routines
- *
- * Copyright (c) 2001-2002, SSH Communications Security Corp and Jouni Malinen
- * <j@w1.fi>
- * Copyright (c) 2002-2005, Jouni Malinen <j@w1.fi>
- */
-
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/slab.h>
-#include <linux/proc_fs.h>
-#include <linux/if_arp.h>
-#include <linux/delay.h>
-#include <linux/random.h>
-#include <linux/workqueue.h>
-#include <linux/kmod.h>
-#include <linux/rtnetlink.h>
-#include <linux/wireless.h>
-#include <linux/etherdevice.h>
-#include <net/net_namespace.h>
-#include <net/iw_handler.h>
-#include <net/lib80211.h>
-#include <linux/uaccess.h>
-
-#include "hostap_wlan.h"
-#include "hostap_80211.h"
-#include "hostap_ap.h"
-#include "hostap.h"
-
-MODULE_AUTHOR("Jouni Malinen");
-MODULE_DESCRIPTION("Host AP common routines");
-MODULE_LICENSE("GPL");
-
-#define TX_TIMEOUT (2 * HZ)
-
-#define PRISM2_MAX_FRAME_SIZE 2304
-#define PRISM2_MIN_MTU 256
-/* FIX: */
-#define PRISM2_MAX_MTU (PRISM2_MAX_FRAME_SIZE - (6 /* LLC */ + 8 /* WEP */))
-
-
-struct net_device * hostap_add_interface(struct local_info *local,
-					 int type, int rtnl_locked,
-					 const char *prefix,
-					 const char *name)
-{
-	struct net_device *dev, *mdev;
-	struct hostap_interface *iface;
-	int ret;
-
-	dev = alloc_etherdev(sizeof(struct hostap_interface));
-	if (dev == NULL)
-		return NULL;
-
-	iface = netdev_priv(dev);
-	iface->dev = dev;
-	iface->local = local;
-	iface->type = type;
-	list_add(&iface->list, &local->hostap_interfaces);
-
-	mdev = local->dev;
-	eth_hw_addr_inherit(dev, mdev);
-	dev->base_addr = mdev->base_addr;
-	dev->irq = mdev->irq;
-	dev->mem_start = mdev->mem_start;
-	dev->mem_end = mdev->mem_end;
-
-	hostap_setup_dev(dev, local, type);
-	dev->needs_free_netdev = true;
-
-	sprintf(dev->name, "%s%s", prefix, name);
-	if (!rtnl_locked)
-		rtnl_lock();
-
-	SET_NETDEV_DEV(dev, mdev->dev.parent);
-	ret = register_netdevice(dev);
-
-	if (!rtnl_locked)
-		rtnl_unlock();
-
-	if (ret < 0) {
-		printk(KERN_WARNING "%s: failed to add new netdevice!\n",
-		       dev->name);
-		free_netdev(dev);
-		return NULL;
-	}
-
-	printk(KERN_DEBUG "%s: registered netdevice %s\n",
-	       mdev->name, dev->name);
-
-	return dev;
-}
-
-
-void hostap_remove_interface(struct net_device *dev, int rtnl_locked,
-			     int remove_from_list)
-{
-	struct hostap_interface *iface;
-
-	if (!dev)
-		return;
-
-	iface = netdev_priv(dev);
-
-	if (remove_from_list) {
-		list_del(&iface->list);
-	}
-
-	if (dev == iface->local->ddev)
-		iface->local->ddev = NULL;
-	else if (dev == iface->local->apdev)
-		iface->local->apdev = NULL;
-	else if (dev == iface->local->stadev)
-		iface->local->stadev = NULL;
-
-	if (rtnl_locked)
-		unregister_netdevice(dev);
-	else
-		unregister_netdev(dev);
-
-	/* 'dev->needs_free_netdev = true' implies device data, including
-	 * private data, will be freed when the device is removed */
-}
-
-
-static inline int prism2_wds_special_addr(u8 *addr)
-{
-	if (addr[0] || addr[1] || addr[2] || addr[3] || addr[4] || addr[5])
-		return 0;
-
-	return 1;
-}
-
-
-int prism2_wds_add(local_info_t *local, u8 *remote_addr,
-		   int rtnl_locked)
-{
-	struct net_device *dev;
-	struct list_head *ptr;
-	struct hostap_interface *iface, *empty, *match;
-
-	empty = match = NULL;
-	read_lock_bh(&local->iface_lock);
-	list_for_each(ptr, &local->hostap_interfaces) {
-		iface = list_entry(ptr, struct hostap_interface, list);
-		if (iface->type != HOSTAP_INTERFACE_WDS)
-			continue;
-
-		if (prism2_wds_special_addr(iface->u.wds.remote_addr))
-			empty = iface;
-		else if (ether_addr_equal(iface->u.wds.remote_addr, remote_addr)) {
-			match = iface;
-			break;
-		}
-	}
-	if (!match && empty && !prism2_wds_special_addr(remote_addr)) {
-		/* take pre-allocated entry into use */
-		memcpy(empty->u.wds.remote_addr, remote_addr, ETH_ALEN);
-		read_unlock_bh(&local->iface_lock);
-		printk(KERN_DEBUG "%s: using pre-allocated WDS netdevice %s\n",
-		       local->dev->name, empty->dev->name);
-		return 0;
-	}
-	read_unlock_bh(&local->iface_lock);
-
-	if (!prism2_wds_special_addr(remote_addr)) {
-		if (match)
-			return -EEXIST;
-		hostap_add_sta(local->ap, remote_addr);
-	}
-
-	if (local->wds_connections >= local->wds_max_connections)
-		return -ENOBUFS;
-
-	/* verify that there is room for wds# postfix in the interface name */
-	if (strlen(local->dev->name) >= IFNAMSIZ - 5) {
-		printk(KERN_DEBUG "'%s' too long base device name\n",
-		       local->dev->name);
-		return -EINVAL;
-	}
-
-	dev = hostap_add_interface(local, HOSTAP_INTERFACE_WDS, rtnl_locked,
-				   local->ddev->name, "wds%d");
-	if (dev == NULL)
-		return -ENOMEM;
-
-	iface = netdev_priv(dev);
-	memcpy(iface->u.wds.remote_addr, remote_addr, ETH_ALEN);
-
-	local->wds_connections++;
-
-	return 0;
-}
-
-
-int prism2_wds_del(local_info_t *local, u8 *remote_addr,
-		   int rtnl_locked, int do_not_remove)
-{
-	unsigned long flags;
-	struct list_head *ptr;
-	struct hostap_interface *iface, *selected = NULL;
-
-	write_lock_irqsave(&local->iface_lock, flags);
-	list_for_each(ptr, &local->hostap_interfaces) {
-		iface = list_entry(ptr, struct hostap_interface, list);
-		if (iface->type != HOSTAP_INTERFACE_WDS)
-			continue;
-
-		if (ether_addr_equal(iface->u.wds.remote_addr, remote_addr)) {
-			selected = iface;
-			break;
-		}
-	}
-	if (selected && !do_not_remove)
-		list_del(&selected->list);
-	write_unlock_irqrestore(&local->iface_lock, flags);
-
-	if (selected) {
-		if (do_not_remove)
-			eth_zero_addr(selected->u.wds.remote_addr);
-		else {
-			hostap_remove_interface(selected->dev, rtnl_locked, 0);
-			local->wds_connections--;
-		}
-	}
-
-	return selected ? 0 : -ENODEV;
-}
-
-
-u16 hostap_tx_callback_register(local_info_t *local,
-				void (*func)(struct sk_buff *, int ok, void *),
-				void *data)
-{
-	unsigned long flags;
-	struct hostap_tx_callback_info *entry;
-
-	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
-	if (entry == NULL)
-		return 0;
-
-	entry->func = func;
-	entry->data = data;
-
-	spin_lock_irqsave(&local->lock, flags);
-	entry->idx = local->tx_callback ? local->tx_callback->idx + 1 : 1;
-	entry->next = local->tx_callback;
-	local->tx_callback = entry;
-	spin_unlock_irqrestore(&local->lock, flags);
-
-	return entry->idx;
-}
-
-
-int hostap_tx_callback_unregister(local_info_t *local, u16 idx)
-{
-	unsigned long flags;
-	struct hostap_tx_callback_info *cb, *prev = NULL;
-
-	spin_lock_irqsave(&local->lock, flags);
-	cb = local->tx_callback;
-	while (cb != NULL && cb->idx != idx) {
-		prev = cb;
-		cb = cb->next;
-	}
-	if (cb) {
-		if (prev == NULL)
-			local->tx_callback = cb->next;
-		else
-			prev->next = cb->next;
-		kfree(cb);
-	}
-	spin_unlock_irqrestore(&local->lock, flags);
-
-	return cb ? 0 : -1;
-}
-
-
-/* val is in host byte order */
-int hostap_set_word(struct net_device *dev, int rid, u16 val)
-{
-	struct hostap_interface *iface;
-	__le16 tmp = cpu_to_le16(val);
-	iface = netdev_priv(dev);
-	return iface->local->func->set_rid(dev, rid, &tmp, 2);
-}
-
-
-int hostap_set_string(struct net_device *dev, int rid, const char *val)
-{
-	struct hostap_interface *iface;
-	char buf[MAX_SSID_LEN + 2];
-	int len;
-
-	iface = netdev_priv(dev);
-	len = strlen(val);
-	if (len > MAX_SSID_LEN)
-		return -1;
-	memset(buf, 0, sizeof(buf));
-	buf[0] = len; /* little endian 16 bit word */
-	memcpy(buf + 2, val, len);
-
-	return iface->local->func->set_rid(dev, rid, &buf, MAX_SSID_LEN + 2);
-}
-
-
-u16 hostap_get_porttype(local_info_t *local)
-{
-	if (local->iw_mode == IW_MODE_ADHOC && local->pseudo_adhoc)
-		return HFA384X_PORTTYPE_PSEUDO_IBSS;
-	if (local->iw_mode == IW_MODE_ADHOC)
-		return HFA384X_PORTTYPE_IBSS;
-	if (local->iw_mode == IW_MODE_INFRA)
-		return HFA384X_PORTTYPE_BSS;
-	if (local->iw_mode == IW_MODE_REPEAT)
-		return HFA384X_PORTTYPE_WDS;
-	if (local->iw_mode == IW_MODE_MONITOR)
-		return HFA384X_PORTTYPE_PSEUDO_IBSS;
-	return HFA384X_PORTTYPE_HOSTAP;
-}
-
-
-int hostap_set_encryption(local_info_t *local)
-{
-	u16 val, old_val;
-	int i, keylen, len, idx;
-	char keybuf[WEP_KEY_LEN + 1];
-	enum { NONE, WEP, OTHER } encrypt_type;
-
-	idx = local->crypt_info.tx_keyidx;
-	if (local->crypt_info.crypt[idx] == NULL ||
-	    local->crypt_info.crypt[idx]->ops == NULL)
-		encrypt_type = NONE;
-	else if (strcmp(local->crypt_info.crypt[idx]->ops->name, "WEP") == 0)
-		encrypt_type = WEP;
-	else
-		encrypt_type = OTHER;
-
-	if (local->func->get_rid(local->dev, HFA384X_RID_CNFWEPFLAGS, &val, 2,
-				 1) < 0) {
-		printk(KERN_DEBUG "Could not read current WEP flags.\n");
-		goto fail;
-	}
-	le16_to_cpus(&val);
-	old_val = val;
-
-	if (encrypt_type != NONE || local->privacy_invoked)
-		val |= HFA384X_WEPFLAGS_PRIVACYINVOKED;
-	else
-		val &= ~HFA384X_WEPFLAGS_PRIVACYINVOKED;
-
-	if (local->open_wep || encrypt_type == NONE ||
-	    ((local->ieee_802_1x || local->wpa) && local->host_decrypt))
-		val &= ~HFA384X_WEPFLAGS_EXCLUDEUNENCRYPTED;
-	else
-		val |= HFA384X_WEPFLAGS_EXCLUDEUNENCRYPTED;
-
-	if ((encrypt_type != NONE || local->privacy_invoked) &&
-	    (encrypt_type == OTHER || local->host_encrypt))
-		val |= HFA384X_WEPFLAGS_HOSTENCRYPT;
-	else
-		val &= ~HFA384X_WEPFLAGS_HOSTENCRYPT;
-	if ((encrypt_type != NONE || local->privacy_invoked) &&
-	    (encrypt_type == OTHER || local->host_decrypt))
-		val |= HFA384X_WEPFLAGS_HOSTDECRYPT;
-	else
-		val &= ~HFA384X_WEPFLAGS_HOSTDECRYPT;
-
-	if (val != old_val &&
-	    hostap_set_word(local->dev, HFA384X_RID_CNFWEPFLAGS, val)) {
-		printk(KERN_DEBUG "Could not write new WEP flags (0x%x)\n",
-		       val);
-		goto fail;
-	}
-
-	if (encrypt_type != WEP)
-		return 0;
-
-	/* 104-bit support seems to require that all the keys are set to the
-	 * same keylen */
-	keylen = 6; /* first 5 octets */
-	len = local->crypt_info.crypt[idx]->ops->get_key(keybuf, sizeof(keybuf), NULL,
-							   local->crypt_info.crypt[idx]->priv);
-	if (idx >= 0 && idx < WEP_KEYS && len > 5)
-		keylen = WEP_KEY_LEN + 1; /* first 13 octets */
-
-	for (i = 0; i < WEP_KEYS; i++) {
-		memset(keybuf, 0, sizeof(keybuf));
-		if (local->crypt_info.crypt[i]) {
-			(void) local->crypt_info.crypt[i]->ops->get_key(
-				keybuf, sizeof(keybuf),
-				NULL, local->crypt_info.crypt[i]->priv);
-		}
-		if (local->func->set_rid(local->dev,
-					 HFA384X_RID_CNFDEFAULTKEY0 + i,
-					 keybuf, keylen)) {
-			printk(KERN_DEBUG "Could not set key %d (len=%d)\n",
-			       i, keylen);
-			goto fail;
-		}
-	}
-	if (hostap_set_word(local->dev, HFA384X_RID_CNFWEPDEFAULTKEYID, idx)) {
-		printk(KERN_DEBUG "Could not set default keyid %d\n", idx);
-		goto fail;
-	}
-
-	return 0;
-
- fail:
-	printk(KERN_DEBUG "%s: encryption setup failed\n", local->dev->name);
-	return -1;
-}
-
-
-int hostap_set_antsel(local_info_t *local)
-{
-	u16 val;
-	int ret = 0;
-
-	if (local->antsel_tx != HOSTAP_ANTSEL_DO_NOT_TOUCH &&
-	    local->func->cmd(local->dev, HFA384X_CMDCODE_READMIF,
-			     HFA386X_CR_TX_CONFIGURE,
-			     NULL, &val) == 0) {
-		val &= ~(BIT(2) | BIT(1));
-		switch (local->antsel_tx) {
-		case HOSTAP_ANTSEL_DIVERSITY:
-			val |= BIT(1);
-			break;
-		case HOSTAP_ANTSEL_LOW:
-			break;
-		case HOSTAP_ANTSEL_HIGH:
-			val |= BIT(2);
-			break;
-		}
-
-		if (local->func->cmd(local->dev, HFA384X_CMDCODE_WRITEMIF,
-				     HFA386X_CR_TX_CONFIGURE, &val, NULL)) {
-			printk(KERN_INFO "%s: setting TX AntSel failed\n",
-			       local->dev->name);
-			ret = -1;
-		}
-	}
-
-	if (local->antsel_rx != HOSTAP_ANTSEL_DO_NOT_TOUCH &&
-	    local->func->cmd(local->dev, HFA384X_CMDCODE_READMIF,
-			     HFA386X_CR_RX_CONFIGURE,
-			     NULL, &val) == 0) {
-		val &= ~(BIT(1) | BIT(0));
-		switch (local->antsel_rx) {
-		case HOSTAP_ANTSEL_DIVERSITY:
-			break;
-		case HOSTAP_ANTSEL_LOW:
-			val |= BIT(0);
-			break;
-		case HOSTAP_ANTSEL_HIGH:
-			val |= BIT(0) | BIT(1);
-			break;
-		}
-
-		if (local->func->cmd(local->dev, HFA384X_CMDCODE_WRITEMIF,
-				     HFA386X_CR_RX_CONFIGURE, &val, NULL)) {
-			printk(KERN_INFO "%s: setting RX AntSel failed\n",
-			       local->dev->name);
-			ret = -1;
-		}
-	}
-
-	return ret;
-}
-
-
-int hostap_set_roaming(local_info_t *local)
-{
-	u16 val;
-
-	switch (local->host_roaming) {
-	case 1:
-		val = HFA384X_ROAMING_HOST;
-		break;
-	case 2:
-		val = HFA384X_ROAMING_DISABLED;
-		break;
-	case 0:
-	default:
-		val = HFA384X_ROAMING_FIRMWARE;
-		break;
-	}
-
-	return hostap_set_word(local->dev, HFA384X_RID_CNFROAMINGMODE, val);
-}
-
-
-int hostap_set_auth_algs(local_info_t *local)
-{
-	int val = local->auth_algs;
-	/* At least STA f/w v0.6.2 seems to have issues with cnfAuthentication
-	 * set to include both Open and Shared Key flags. It tries to use
-	 * Shared Key authentication in that case even if WEP keys are not
-	 * configured.. STA f/w v0.7.6 is able to handle such configuration,
-	 * but it is unknown when this was fixed between 0.6.2 .. 0.7.6. */
-	if (local->sta_fw_ver < PRISM2_FW_VER(0,7,0) &&
-	    val != PRISM2_AUTH_OPEN && val != PRISM2_AUTH_SHARED_KEY)
-		val = PRISM2_AUTH_OPEN;
-
-	if (hostap_set_word(local->dev, HFA384X_RID_CNFAUTHENTICATION, val)) {
-		printk(KERN_INFO "%s: cnfAuthentication setting to 0x%x "
-		       "failed\n", local->dev->name, local->auth_algs);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-
-void hostap_dump_rx_header(const char *name, const struct hfa384x_rx_frame *rx)
-{
-	u16 status, fc;
-
-	status = __le16_to_cpu(rx->status);
-
-	printk(KERN_DEBUG "%s: RX status=0x%04x (port=%d, type=%d, "
-	       "fcserr=%d) silence=%d signal=%d rate=%d rxflow=%d; "
-	       "jiffies=%ld\n",
-	       name, status, (status >> 8) & 0x07, status >> 13, status & 1,
-	       rx->silence, rx->signal, rx->rate, rx->rxflow, jiffies);
-
-	fc = __le16_to_cpu(rx->frame_control);
-	printk(KERN_DEBUG "   FC=0x%04x (type=%d:%d) dur=0x%04x seq=0x%04x "
-	       "data_len=%d%s%s\n",
-	       fc, (fc & IEEE80211_FCTL_FTYPE) >> 2,
-	       (fc & IEEE80211_FCTL_STYPE) >> 4,
-	       __le16_to_cpu(rx->duration_id), __le16_to_cpu(rx->seq_ctrl),
-	       __le16_to_cpu(rx->data_len),
-	       fc & IEEE80211_FCTL_TODS ? " [ToDS]" : "",
-	       fc & IEEE80211_FCTL_FROMDS ? " [FromDS]" : "");
-
-	printk(KERN_DEBUG "   A1=%pM A2=%pM A3=%pM A4=%pM\n",
-	       rx->addr1, rx->addr2, rx->addr3, rx->addr4);
-
-	printk(KERN_DEBUG "   dst=%pM src=%pM len=%d\n",
-	       rx->dst_addr, rx->src_addr,
-	       __be16_to_cpu(rx->len));
-}
-
-
-void hostap_dump_tx_header(const char *name, const struct hfa384x_tx_frame *tx)
-{
-	u16 fc;
-
-	printk(KERN_DEBUG "%s: TX status=0x%04x retry_count=%d tx_rate=%d "
-	       "tx_control=0x%04x; jiffies=%ld\n",
-	       name, __le16_to_cpu(tx->status), tx->retry_count, tx->tx_rate,
-	       __le16_to_cpu(tx->tx_control), jiffies);
-
-	fc = __le16_to_cpu(tx->frame_control);
-	printk(KERN_DEBUG "   FC=0x%04x (type=%d:%d) dur=0x%04x seq=0x%04x "
-	       "data_len=%d%s%s\n",
-	       fc, (fc & IEEE80211_FCTL_FTYPE) >> 2,
-	       (fc & IEEE80211_FCTL_STYPE) >> 4,
-	       __le16_to_cpu(tx->duration_id), __le16_to_cpu(tx->seq_ctrl),
-	       __le16_to_cpu(tx->data_len),
-	       fc & IEEE80211_FCTL_TODS ? " [ToDS]" : "",
-	       fc & IEEE80211_FCTL_FROMDS ? " [FromDS]" : "");
-
-	printk(KERN_DEBUG "   A1=%pM A2=%pM A3=%pM A4=%pM\n",
-	       tx->addr1, tx->addr2, tx->addr3, tx->addr4);
-
-	printk(KERN_DEBUG "   dst=%pM src=%pM len=%d\n",
-	       tx->dst_addr, tx->src_addr,
-	       __be16_to_cpu(tx->len));
-}
-
-
-static int hostap_80211_header_parse(const struct sk_buff *skb,
-				     unsigned char *haddr)
-{
-	memcpy(haddr, skb_mac_header(skb) + 10, ETH_ALEN); /* addr2 */
-	return ETH_ALEN;
-}
-
-
-int hostap_80211_get_hdrlen(__le16 fc)
-{
-	if (ieee80211_is_data(fc) && ieee80211_has_a4 (fc))
-		return 30; /* Addr4 */
-	else if (ieee80211_is_cts(fc) || ieee80211_is_ack(fc))
-		return 10;
-	else if (ieee80211_is_ctl(fc))
-		return 16;
-
-	return 24;
-}
-
-
-static int prism2_close(struct net_device *dev)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-
-	PDEBUG(DEBUG_FLOW, "%s: prism2_close\n", dev->name);
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (dev == local->ddev) {
-		prism2_sta_deauth(local, WLAN_REASON_DEAUTH_LEAVING);
-	}
-#ifndef PRISM2_NO_KERNEL_IEEE80211_MGMT
-	if (!local->hostapd && dev == local->dev &&
-	    (!local->func->card_present || local->func->card_present(local)) &&
-	    local->hw_ready && local->ap && local->iw_mode == IW_MODE_MASTER)
-		hostap_deauth_all_stas(dev, local->ap, 1);
-#endif /* PRISM2_NO_KERNEL_IEEE80211_MGMT */
-
-	if (dev == local->dev) {
-		local->func->hw_shutdown(dev, HOSTAP_HW_ENABLE_CMDCOMPL);
-	}
-
-	if (netif_running(dev)) {
-		netif_stop_queue(dev);
-		netif_device_detach(dev);
-	}
-
-	cancel_work_sync(&local->reset_queue);
-	cancel_work_sync(&local->set_multicast_list_queue);
-	cancel_work_sync(&local->set_tim_queue);
-#ifndef PRISM2_NO_STATION_MODES
-	cancel_work_sync(&local->info_queue);
-#endif
-	cancel_work_sync(&local->comms_qual_update);
-
-	module_put(local->hw_module);
-
-	local->num_dev_open--;
-
-	if (dev != local->dev && local->dev->flags & IFF_UP &&
-	    local->master_dev_auto_open && local->num_dev_open == 1) {
-		/* Close master radio interface automatically if it was also
-		 * opened automatically and we are now closing the last
-		 * remaining non-master device. */
-		dev_close(local->dev);
-	}
-
-	return 0;
-}
-
-
-static int prism2_open(struct net_device *dev)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-
-	PDEBUG(DEBUG_FLOW, "%s: prism2_open\n", dev->name);
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (local->no_pri) {
-		printk(KERN_DEBUG "%s: could not set interface UP - no PRI "
-		       "f/w\n", dev->name);
-		return -ENODEV;
-	}
-
-	if ((local->func->card_present && !local->func->card_present(local)) ||
-	    local->hw_downloading)
-		return -ENODEV;
-
-	if (!try_module_get(local->hw_module))
-		return -ENODEV;
-	local->num_dev_open++;
-
-	if (!local->dev_enabled && local->func->hw_enable(dev, 1)) {
-		printk(KERN_WARNING "%s: could not enable MAC port\n",
-		       dev->name);
-		prism2_close(dev);
-		return -ENODEV;
-	}
-	if (!local->dev_enabled)
-		prism2_callback(local, PRISM2_CALLBACK_ENABLE);
-	local->dev_enabled = 1;
-
-	if (dev != local->dev && !(local->dev->flags & IFF_UP)) {
-		/* Master radio interface is needed for all operation, so open
-		 * it automatically when any virtual net_device is opened. */
-		local->master_dev_auto_open = 1;
-		dev_open(local->dev, NULL);
-	}
-
-	netif_device_attach(dev);
-	netif_start_queue(dev);
-
-	return 0;
-}
-
-
-static int prism2_set_mac_address(struct net_device *dev, void *p)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	struct list_head *ptr;
-	struct sockaddr *addr = p;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	if (local->func->set_rid(dev, HFA384X_RID_CNFOWNMACADDR, addr->sa_data,
-				 ETH_ALEN) < 0 || local->func->reset_port(dev))
-		return -EINVAL;
-
-	read_lock_bh(&local->iface_lock);
-	list_for_each(ptr, &local->hostap_interfaces) {
-		iface = list_entry(ptr, struct hostap_interface, list);
-		eth_hw_addr_set(iface->dev, addr->sa_data);
-	}
-	eth_hw_addr_set(local->dev, addr->sa_data);
-	read_unlock_bh(&local->iface_lock);
-
-	return 0;
-}
-
-
-/* TODO: to be further implemented as soon as Prism2 fully supports
- *       GroupAddresses and correct documentation is available */
-void hostap_set_multicast_list_queue(struct work_struct *work)
-{
-	local_info_t *local =
-		container_of(work, local_info_t, set_multicast_list_queue);
-	struct net_device *dev = local->dev;
-
-	if (hostap_set_word(dev, HFA384X_RID_PROMISCUOUSMODE,
-			    local->is_promisc)) {
-		printk(KERN_INFO "%s: %sabling promiscuous mode failed\n",
-		       dev->name, local->is_promisc ? "en" : "dis");
-	}
-}
-
-
-static void hostap_set_multicast_list(struct net_device *dev)
-{
-#if 0
-	/* FIX: promiscuous mode seems to be causing a lot of problems with
-	 * some station firmware versions (FCSErr frames, invalid MACPort, etc.
-	 * corrupted incoming frames). This code is now commented out while the
-	 * problems are investigated. */
-	struct hostap_interface *iface;
-	local_info_t *local;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-	if ((dev->flags & IFF_ALLMULTI) || (dev->flags & IFF_PROMISC)) {
-		local->is_promisc = 1;
-	} else {
-		local->is_promisc = 0;
-	}
-
-	schedule_work(&local->set_multicast_list_queue);
-#endif
-}
-
-
-static void prism2_tx_timeout(struct net_device *dev, unsigned int txqueue)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	struct hfa384x_regs regs;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	printk(KERN_WARNING "%s Tx timed out! Resetting card\n", dev->name);
-	netif_stop_queue(local->dev);
-
-	local->func->read_regs(dev, &regs);
-	printk(KERN_DEBUG "%s: CMD=%04x EVSTAT=%04x "
-	       "OFFSET0=%04x OFFSET1=%04x SWSUPPORT0=%04x\n",
-	       dev->name, regs.cmd, regs.evstat, regs.offset0, regs.offset1,
-	       regs.swsupport0);
-
-	local->func->schedule_reset(local);
-}
-
-const struct header_ops hostap_80211_ops = {
-	.create		= eth_header,
-	.cache		= eth_header_cache,
-	.cache_update	= eth_header_cache_update,
-	.parse		= hostap_80211_header_parse,
-};
-EXPORT_SYMBOL(hostap_80211_ops);
-
-
-static const struct net_device_ops hostap_netdev_ops = {
-	.ndo_start_xmit		= hostap_data_start_xmit,
-
-	.ndo_open		= prism2_open,
-	.ndo_stop		= prism2_close,
-	.ndo_siocdevprivate	= hostap_siocdevprivate,
-	.ndo_set_mac_address	= prism2_set_mac_address,
-	.ndo_set_rx_mode	= hostap_set_multicast_list,
-	.ndo_tx_timeout 	= prism2_tx_timeout,
-	.ndo_validate_addr	= eth_validate_addr,
-};
-
-static const struct net_device_ops hostap_mgmt_netdev_ops = {
-	.ndo_start_xmit		= hostap_mgmt_start_xmit,
-
-	.ndo_open		= prism2_open,
-	.ndo_stop		= prism2_close,
-	.ndo_siocdevprivate	= hostap_siocdevprivate,
-	.ndo_set_mac_address	= prism2_set_mac_address,
-	.ndo_set_rx_mode	= hostap_set_multicast_list,
-	.ndo_tx_timeout 	= prism2_tx_timeout,
-	.ndo_validate_addr	= eth_validate_addr,
-};
-
-static const struct net_device_ops hostap_master_ops = {
-	.ndo_start_xmit 	= hostap_master_start_xmit,
-
-	.ndo_open		= prism2_open,
-	.ndo_stop		= prism2_close,
-	.ndo_siocdevprivate	= hostap_siocdevprivate,
-	.ndo_set_mac_address	= prism2_set_mac_address,
-	.ndo_set_rx_mode	= hostap_set_multicast_list,
-	.ndo_tx_timeout 	= prism2_tx_timeout,
-	.ndo_validate_addr	= eth_validate_addr,
-};
-
-void hostap_setup_dev(struct net_device *dev, local_info_t *local,
-		      int type)
-{
-	struct hostap_interface *iface;
-
-	iface = netdev_priv(dev);
-	ether_setup(dev);
-	dev->min_mtu = PRISM2_MIN_MTU;
-	dev->max_mtu = PRISM2_MAX_MTU;
-	dev->priv_flags &= ~IFF_TX_SKB_SHARING;
-
-	/* kernel callbacks */
-	if (iface) {
-		/* Currently, we point to the proper spy_data only on
-		 * the main_dev. This could be fixed. Jean II */
-		iface->wireless_data.spy_data = &iface->spy_data;
-		dev->wireless_data = &iface->wireless_data;
-	}
-	dev->wireless_handlers = &hostap_iw_handler_def;
-	dev->watchdog_timeo = TX_TIMEOUT;
-
-	switch(type) {
-	case HOSTAP_INTERFACE_AP:
-		dev->priv_flags |= IFF_NO_QUEUE;	/* use main radio device queue */
-		dev->netdev_ops = &hostap_mgmt_netdev_ops;
-		dev->type = ARPHRD_IEEE80211;
-		dev->header_ops = &hostap_80211_ops;
-		break;
-	case HOSTAP_INTERFACE_MASTER:
-		dev->netdev_ops = &hostap_master_ops;
-		break;
-	default:
-		dev->priv_flags |= IFF_NO_QUEUE;	/* use main radio device queue */
-		dev->netdev_ops = &hostap_netdev_ops;
-	}
-
-	dev->mtu = local->mtu;
-
-
-	dev->ethtool_ops = &prism2_ethtool_ops;
-
-}
-
-static int hostap_enable_hostapd(local_info_t *local, int rtnl_locked)
-{
-	struct net_device *dev = local->dev;
-
-	if (local->apdev)
-		return -EEXIST;
-
-	printk(KERN_DEBUG "%s: enabling hostapd mode\n", dev->name);
-
-	local->apdev = hostap_add_interface(local, HOSTAP_INTERFACE_AP,
-					    rtnl_locked, local->ddev->name,
-					    "ap");
-	if (local->apdev == NULL)
-		return -ENOMEM;
-
-	return 0;
-}
-
-
-static int hostap_disable_hostapd(local_info_t *local, int rtnl_locked)
-{
-	struct net_device *dev = local->dev;
-
-	printk(KERN_DEBUG "%s: disabling hostapd mode\n", dev->name);
-
-	hostap_remove_interface(local->apdev, rtnl_locked, 1);
-	local->apdev = NULL;
-
-	return 0;
-}
-
-
-static int hostap_enable_hostapd_sta(local_info_t *local, int rtnl_locked)
-{
-	struct net_device *dev = local->dev;
-
-	if (local->stadev)
-		return -EEXIST;
-
-	printk(KERN_DEBUG "%s: enabling hostapd STA mode\n", dev->name);
-
-	local->stadev = hostap_add_interface(local, HOSTAP_INTERFACE_STA,
-					     rtnl_locked, local->ddev->name,
-					     "sta");
-	if (local->stadev == NULL)
-		return -ENOMEM;
-
-	return 0;
-}
-
-
-static int hostap_disable_hostapd_sta(local_info_t *local, int rtnl_locked)
-{
-	struct net_device *dev = local->dev;
-
-	printk(KERN_DEBUG "%s: disabling hostapd mode\n", dev->name);
-
-	hostap_remove_interface(local->stadev, rtnl_locked, 1);
-	local->stadev = NULL;
-
-	return 0;
-}
-
-
-int hostap_set_hostapd(local_info_t *local, int val, int rtnl_locked)
-{
-	int ret;
-
-	if (val < 0 || val > 1)
-		return -EINVAL;
-
-	if (local->hostapd == val)
-		return 0;
-
-	if (val) {
-		ret = hostap_enable_hostapd(local, rtnl_locked);
-		if (ret == 0)
-			local->hostapd = 1;
-	} else {
-		local->hostapd = 0;
-		ret = hostap_disable_hostapd(local, rtnl_locked);
-		if (ret != 0)
-			local->hostapd = 1;
-	}
-
-	return ret;
-}
-
-
-int hostap_set_hostapd_sta(local_info_t *local, int val, int rtnl_locked)
-{
-	int ret;
-
-	if (val < 0 || val > 1)
-		return -EINVAL;
-
-	if (local->hostapd_sta == val)
-		return 0;
-
-	if (val) {
-		ret = hostap_enable_hostapd_sta(local, rtnl_locked);
-		if (ret == 0)
-			local->hostapd_sta = 1;
-	} else {
-		local->hostapd_sta = 0;
-		ret = hostap_disable_hostapd_sta(local, rtnl_locked);
-		if (ret != 0)
-			local->hostapd_sta = 1;
-	}
-
-
-	return ret;
-}
-
-
-int prism2_update_comms_qual(struct net_device *dev)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	int ret = 0;
-	struct hfa384x_comms_quality sq;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-	if (!local->sta_fw_ver)
-		ret = -1;
-	else if (local->sta_fw_ver >= PRISM2_FW_VER(1,3,1)) {
-		if (local->func->get_rid(local->dev,
-					 HFA384X_RID_DBMCOMMSQUALITY,
-					 &sq, sizeof(sq), 1) >= 0) {
-			local->comms_qual = (s16) le16_to_cpu(sq.comm_qual);
-			local->avg_signal = (s16) le16_to_cpu(sq.signal_level);
-			local->avg_noise = (s16) le16_to_cpu(sq.noise_level);
-			local->last_comms_qual_update = jiffies;
-		} else
-			ret = -1;
-	} else {
-		if (local->func->get_rid(local->dev, HFA384X_RID_COMMSQUALITY,
-					 &sq, sizeof(sq), 1) >= 0) {
-			local->comms_qual = le16_to_cpu(sq.comm_qual);
-			local->avg_signal = HFA384X_LEVEL_TO_dBm(
-				le16_to_cpu(sq.signal_level));
-			local->avg_noise = HFA384X_LEVEL_TO_dBm(
-				le16_to_cpu(sq.noise_level));
-			local->last_comms_qual_update = jiffies;
-		} else
-			ret = -1;
-	}
-
-	return ret;
-}
-
-
-int prism2_sta_send_mgmt(local_info_t *local, u8 *dst, u16 stype,
-			 u8 *body, size_t bodylen)
-{
-	struct sk_buff *skb;
-	struct hostap_ieee80211_mgmt *mgmt;
-	struct hostap_skb_tx_data *meta;
-	struct net_device *dev = local->dev;
-
-	skb = dev_alloc_skb(IEEE80211_MGMT_HDR_LEN + bodylen);
-	if (skb == NULL)
-		return -ENOMEM;
-
-	mgmt = skb_put_zero(skb, IEEE80211_MGMT_HDR_LEN);
-	mgmt->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT | stype);
-	memcpy(mgmt->da, dst, ETH_ALEN);
-	memcpy(mgmt->sa, dev->dev_addr, ETH_ALEN);
-	memcpy(mgmt->bssid, dst, ETH_ALEN);
-	if (body)
-		skb_put_data(skb, body, bodylen);
-
-	meta = (struct hostap_skb_tx_data *) skb->cb;
-	memset(meta, 0, sizeof(*meta));
-	meta->magic = HOSTAP_SKB_TX_DATA_MAGIC;
-	meta->iface = netdev_priv(dev);
-
-	skb->dev = dev;
-	skb_reset_mac_header(skb);
-	skb_reset_network_header(skb);
-	dev_queue_xmit(skb);
-
-	return 0;
-}
-
-
-int prism2_sta_deauth(local_info_t *local, u16 reason)
-{
-	union iwreq_data wrqu;
-	int ret;
-	__le16 val = cpu_to_le16(reason);
-
-	if (local->iw_mode != IW_MODE_INFRA ||
-	    is_zero_ether_addr(local->bssid) ||
-	    ether_addr_equal(local->bssid, "\x44\x44\x44\x44\x44\x44"))
-		return 0;
-
-	ret = prism2_sta_send_mgmt(local, local->bssid, IEEE80211_STYPE_DEAUTH,
-				   (u8 *) &val, 2);
-	eth_zero_addr(wrqu.ap_addr.sa_data);
-	wireless_send_event(local->dev, SIOCGIWAP, &wrqu, NULL);
-	return ret;
-}
-
-
-struct proc_dir_entry *hostap_proc;
-
-static int __init hostap_init(void)
-{
-	if (init_net.proc_net != NULL) {
-		hostap_proc = proc_mkdir("hostap", init_net.proc_net);
-		if (!hostap_proc)
-			printk(KERN_WARNING "Failed to mkdir "
-			       "/proc/net/hostap\n");
-	} else
-		hostap_proc = NULL;
-
-	return 0;
-}
-
-
-static void __exit hostap_exit(void)
-{
-	if (hostap_proc != NULL) {
-		hostap_proc = NULL;
-		remove_proc_entry("hostap", init_net.proc_net);
-	}
-}
-
-
-EXPORT_SYMBOL(hostap_set_word);
-EXPORT_SYMBOL(hostap_set_string);
-EXPORT_SYMBOL(hostap_get_porttype);
-EXPORT_SYMBOL(hostap_set_encryption);
-EXPORT_SYMBOL(hostap_set_antsel);
-EXPORT_SYMBOL(hostap_set_roaming);
-EXPORT_SYMBOL(hostap_set_auth_algs);
-EXPORT_SYMBOL(hostap_dump_rx_header);
-EXPORT_SYMBOL(hostap_dump_tx_header);
-EXPORT_SYMBOL(hostap_80211_get_hdrlen);
-EXPORT_SYMBOL(hostap_setup_dev);
-EXPORT_SYMBOL(hostap_set_multicast_list_queue);
-EXPORT_SYMBOL(hostap_set_hostapd);
-EXPORT_SYMBOL(hostap_set_hostapd_sta);
-EXPORT_SYMBOL(hostap_add_interface);
-EXPORT_SYMBOL(hostap_remove_interface);
-EXPORT_SYMBOL(prism2_update_comms_qual);
-
-module_init(hostap_init);
-module_exit(hostap_exit);
diff --git a/drivers/net/wireless/intersil/hostap/hostap_pci.c b/drivers/net/wireless/intersil/hostap/hostap_pci.c
deleted file mode 100644
index 52d77506effd..000000000000
--- a/drivers/net/wireless/intersil/hostap/hostap_pci.c
+++ /dev/null
@@ -1,445 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-#define PRISM2_PCI
-
-/* Host AP driver's support for Intersil Prism2.5 PCI cards is based on
- * driver patches from Reyk Floeter <reyk@vantronix.net> and
- * Andy Warner <andyw@pobox.com> */
-
-#include <linux/module.h>
-#include <linux/if.h>
-#include <linux/skbuff.h>
-#include <linux/netdevice.h>
-#include <linux/slab.h>
-#include <linux/workqueue.h>
-#include <linux/wireless.h>
-#include <net/iw_handler.h>
-
-#include <linux/ioport.h>
-#include <linux/pci.h>
-#include <asm/io.h>
-
-#include "hostap_wlan.h"
-
-
-static char *dev_info = "hostap_pci";
-
-
-MODULE_AUTHOR("Jouni Malinen");
-MODULE_DESCRIPTION("Support for Intersil Prism2.5-based 802.11 wireless LAN "
-		   "PCI cards.");
-MODULE_LICENSE("GPL");
-
-
-/* struct local_info::hw_priv */
-struct hostap_pci_priv {
-	void __iomem *mem_start;
-};
-
-
-/* FIX: do we need mb/wmb/rmb with memory operations? */
-
-
-static const struct pci_device_id prism2_pci_id_table[] = {
-	/* Intersil Prism3 ISL3872 11Mb/s WLAN Controller */
-	{ 0x1260, 0x3872, PCI_ANY_ID, PCI_ANY_ID },
-	/* Intersil Prism2.5 ISL3874 11Mb/s WLAN Controller */
-	{ 0x1260, 0x3873, PCI_ANY_ID, PCI_ANY_ID },
-	/* Samsung MagicLAN SWL-2210P */
-	{ 0x167d, 0xa000, PCI_ANY_ID, PCI_ANY_ID },
-	{ 0 }
-};
-
-
-#ifdef PRISM2_IO_DEBUG
-
-static inline void hfa384x_outb_debug(struct net_device *dev, int a, u8 v)
-{
-	struct hostap_interface *iface;
-	struct hostap_pci_priv *hw_priv;
-	local_info_t *local;
-	unsigned long flags;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-	hw_priv = local->hw_priv;
-
-	spin_lock_irqsave(&local->lock, flags);
-	prism2_io_debug_add(dev, PRISM2_IO_DEBUG_CMD_OUTB, a, v);
-	writeb(v, hw_priv->mem_start + a);
-	spin_unlock_irqrestore(&local->lock, flags);
-}
-
-static inline u8 hfa384x_inb_debug(struct net_device *dev, int a)
-{
-	struct hostap_interface *iface;
-	struct hostap_pci_priv *hw_priv;
-	local_info_t *local;
-	unsigned long flags;
-	u8 v;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-	hw_priv = local->hw_priv;
-
-	spin_lock_irqsave(&local->lock, flags);
-	v = readb(hw_priv->mem_start + a);
-	prism2_io_debug_add(dev, PRISM2_IO_DEBUG_CMD_INB, a, v);
-	spin_unlock_irqrestore(&local->lock, flags);
-	return v;
-}
-
-static inline void hfa384x_outw_debug(struct net_device *dev, int a, u16 v)
-{
-	struct hostap_interface *iface;
-	struct hostap_pci_priv *hw_priv;
-	local_info_t *local;
-	unsigned long flags;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-	hw_priv = local->hw_priv;
-
-	spin_lock_irqsave(&local->lock, flags);
-	prism2_io_debug_add(dev, PRISM2_IO_DEBUG_CMD_OUTW, a, v);
-	writew(v, hw_priv->mem_start + a);
-	spin_unlock_irqrestore(&local->lock, flags);
-}
-
-static inline u16 hfa384x_inw_debug(struct net_device *dev, int a)
-{
-	struct hostap_interface *iface;
-	struct hostap_pci_priv *hw_priv;
-	local_info_t *local;
-	unsigned long flags;
-	u16 v;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-	hw_priv = local->hw_priv;
-
-	spin_lock_irqsave(&local->lock, flags);
-	v = readw(hw_priv->mem_start + a);
-	prism2_io_debug_add(dev, PRISM2_IO_DEBUG_CMD_INW, a, v);
-	spin_unlock_irqrestore(&local->lock, flags);
-	return v;
-}
-
-#define HFA384X_OUTB(v,a) hfa384x_outb_debug(dev, (a), (v))
-#define HFA384X_INB(a) hfa384x_inb_debug(dev, (a))
-#define HFA384X_OUTW(v,a) hfa384x_outw_debug(dev, (a), (v))
-#define HFA384X_INW(a) hfa384x_inw_debug(dev, (a))
-#define HFA384X_OUTW_DATA(v,a) hfa384x_outw_debug(dev, (a), le16_to_cpu((v)))
-#define HFA384X_INW_DATA(a) cpu_to_le16(hfa384x_inw_debug(dev, (a)))
-
-#else /* PRISM2_IO_DEBUG */
-
-static inline void hfa384x_outb(struct net_device *dev, int a, u8 v)
-{
-	struct hostap_interface *iface;
-	struct hostap_pci_priv *hw_priv;
-	iface = netdev_priv(dev);
-	hw_priv = iface->local->hw_priv;
-	writeb(v, hw_priv->mem_start + a);
-}
-
-static inline u8 hfa384x_inb(struct net_device *dev, int a)
-{
-	struct hostap_interface *iface;
-	struct hostap_pci_priv *hw_priv;
-	iface = netdev_priv(dev);
-	hw_priv = iface->local->hw_priv;
-	return readb(hw_priv->mem_start + a);
-}
-
-static inline void hfa384x_outw(struct net_device *dev, int a, u16 v)
-{
-	struct hostap_interface *iface;
-	struct hostap_pci_priv *hw_priv;
-	iface = netdev_priv(dev);
-	hw_priv = iface->local->hw_priv;
-	writew(v, hw_priv->mem_start + a);
-}
-
-static inline u16 hfa384x_inw(struct net_device *dev, int a)
-{
-	struct hostap_interface *iface;
-	struct hostap_pci_priv *hw_priv;
-	iface = netdev_priv(dev);
-	hw_priv = iface->local->hw_priv;
-	return readw(hw_priv->mem_start + a);
-}
-
-#define HFA384X_OUTB(v,a) hfa384x_outb(dev, (a), (v))
-#define HFA384X_INB(a) hfa384x_inb(dev, (a))
-#define HFA384X_OUTW(v,a) hfa384x_outw(dev, (a), (v))
-#define HFA384X_INW(a) hfa384x_inw(dev, (a))
-#define HFA384X_OUTW_DATA(v,a) hfa384x_outw(dev, (a), le16_to_cpu((v)))
-#define HFA384X_INW_DATA(a) cpu_to_le16(hfa384x_inw(dev, (a)))
-
-#endif /* PRISM2_IO_DEBUG */
-
-
-static int hfa384x_from_bap(struct net_device *dev, u16 bap, void *buf,
-			    int len)
-{
-	u16 d_off;
-	__le16 *pos;
-
-	d_off = (bap == 1) ? HFA384X_DATA1_OFF : HFA384X_DATA0_OFF;
-	pos = (__le16 *) buf;
-
-	for ( ; len > 1; len -= 2)
-		*pos++ = HFA384X_INW_DATA(d_off);
-
-	if (len & 1)
-		*((char *) pos) = HFA384X_INB(d_off);
-
-	return 0;
-}
-
-
-static int hfa384x_to_bap(struct net_device *dev, u16 bap, void *buf, int len)
-{
-	u16 d_off;
-	__le16 *pos;
-
-	d_off = (bap == 1) ? HFA384X_DATA1_OFF : HFA384X_DATA0_OFF;
-	pos = (__le16 *) buf;
-
-	for ( ; len > 1; len -= 2)
-		HFA384X_OUTW_DATA(*pos++, d_off);
-
-	if (len & 1)
-		HFA384X_OUTB(*((char *) pos), d_off);
-
-	return 0;
-}
-
-
-/* FIX: This might change at some point.. */
-#include "hostap_hw.c"
-
-static void prism2_pci_cor_sreset(local_info_t *local)
-{
-	struct net_device *dev = local->dev;
-	u16 reg;
-
-	reg = HFA384X_INB(HFA384X_PCICOR_OFF);
-	printk(KERN_DEBUG "%s: Original COR value: 0x%0x\n", dev->name, reg);
-
-	/* linux-wlan-ng uses extremely long hold and settle times for
-	 * COR sreset. A comment in the driver code mentions that the long
-	 * delays appear to be necessary. However, at least IBM 22P6901 seems
-	 * to work fine with shorter delays.
-	 *
-	 * Longer delays can be configured by uncommenting following line: */
-/* #define PRISM2_PCI_USE_LONG_DELAYS */
-
-#ifdef PRISM2_PCI_USE_LONG_DELAYS
-	int i;
-
-	HFA384X_OUTW(reg | 0x0080, HFA384X_PCICOR_OFF);
-	mdelay(250);
-
-	HFA384X_OUTW(reg & ~0x0080, HFA384X_PCICOR_OFF);
-	mdelay(500);
-
-	/* Wait for f/w to complete initialization (CMD:BUSY == 0) */
-	i = 2000000 / 10;
-	while ((HFA384X_INW(HFA384X_CMD_OFF) & HFA384X_CMD_BUSY) && --i)
-		udelay(10);
-
-#else /* PRISM2_PCI_USE_LONG_DELAYS */
-
-	HFA384X_OUTW(reg | 0x0080, HFA384X_PCICOR_OFF);
-	mdelay(2);
-	HFA384X_OUTW(reg & ~0x0080, HFA384X_PCICOR_OFF);
-	mdelay(2);
-
-#endif /* PRISM2_PCI_USE_LONG_DELAYS */
-
-	if (HFA384X_INW(HFA384X_CMD_OFF) & HFA384X_CMD_BUSY) {
-		printk(KERN_DEBUG "%s: COR sreset timeout\n", dev->name);
-	}
-}
-
-
-static void prism2_pci_genesis_reset(local_info_t *local, int hcr)
-{
-	struct net_device *dev = local->dev;
-
-	HFA384X_OUTW(0x00C5, HFA384X_PCICOR_OFF);
-	mdelay(10);
-	HFA384X_OUTW(hcr, HFA384X_PCIHCR_OFF);
-	mdelay(10);
-	HFA384X_OUTW(0x0045, HFA384X_PCICOR_OFF);
-	mdelay(10);
-}
-
-
-static struct prism2_helper_functions prism2_pci_funcs =
-{
-	.card_present	= NULL,
-	.cor_sreset	= prism2_pci_cor_sreset,
-	.genesis_reset	= prism2_pci_genesis_reset,
-	.hw_type	= HOSTAP_HW_PCI,
-};
-
-
-static int prism2_pci_probe(struct pci_dev *pdev,
-			    const struct pci_device_id *id)
-{
-	unsigned long phymem;
-	void __iomem *mem = NULL;
-	local_info_t *local = NULL;
-	struct net_device *dev = NULL;
-	static int cards_found /* = 0 */;
-	int irq_registered = 0;
-	struct hostap_interface *iface;
-	struct hostap_pci_priv *hw_priv;
-
-	hw_priv = kzalloc(sizeof(*hw_priv), GFP_KERNEL);
-	if (hw_priv == NULL)
-		return -ENOMEM;
-
-	if (pci_enable_device(pdev))
-		goto err_out_free;
-
-	phymem = pci_resource_start(pdev, 0);
-
-	if (!request_mem_region(phymem, pci_resource_len(pdev, 0), "Prism2")) {
-		printk(KERN_ERR "prism2: Cannot reserve PCI memory region\n");
-		goto err_out_disable;
-	}
-
-	mem = pci_ioremap_bar(pdev, 0);
-	if (mem == NULL) {
-		printk(KERN_ERR "prism2: Cannot remap PCI memory region\n") ;
-		goto fail;
-	}
-
-	dev = prism2_init_local_data(&prism2_pci_funcs, cards_found,
-				     &pdev->dev);
-	if (dev == NULL)
-		goto fail;
-	iface = netdev_priv(dev);
-	local = iface->local;
-	local->hw_priv = hw_priv;
-	cards_found++;
-
-        dev->irq = pdev->irq;
-        hw_priv->mem_start = mem;
-	dev->base_addr = (unsigned long) mem;
-
-	prism2_pci_cor_sreset(local);
-
-	pci_set_drvdata(pdev, dev);
-
-	if (request_irq(dev->irq, prism2_interrupt, IRQF_SHARED, dev->name,
-			dev)) {
-		printk(KERN_WARNING "%s: request_irq failed\n", dev->name);
-		goto fail;
-	} else
-		irq_registered = 1;
-
-	if (!local->pri_only && prism2_hw_config(dev, 1)) {
-		printk(KERN_DEBUG "%s: hardware initialization failed\n",
-		       dev_info);
-		goto fail;
-	}
-
-	printk(KERN_INFO "%s: Intersil Prism2.5 PCI: "
-	       "mem=0x%lx, irq=%d\n", dev->name, phymem, dev->irq);
-
-	return hostap_hw_ready(dev);
-
- fail:
-	if (irq_registered && dev)
-		free_irq(dev->irq, dev);
-
-	if (mem)
-		iounmap(mem);
-
-	release_mem_region(phymem, pci_resource_len(pdev, 0));
-
- err_out_disable:
-	pci_disable_device(pdev);
-	prism2_free_local_data(dev);
-
- err_out_free:
-	kfree(hw_priv);
-
-	return -ENODEV;
-}
-
-
-static void prism2_pci_remove(struct pci_dev *pdev)
-{
-	struct net_device *dev;
-	struct hostap_interface *iface;
-	void __iomem *mem_start;
-	struct hostap_pci_priv *hw_priv;
-
-	dev = pci_get_drvdata(pdev);
-	iface = netdev_priv(dev);
-	hw_priv = iface->local->hw_priv;
-
-	/* Reset the hardware, and ensure interrupts are disabled. */
-	prism2_pci_cor_sreset(iface->local);
-	hfa384x_disable_interrupts(dev);
-
-	if (dev->irq)
-		free_irq(dev->irq, dev);
-
-	mem_start = hw_priv->mem_start;
-	prism2_free_local_data(dev);
-	kfree(hw_priv);
-
-	iounmap(mem_start);
-
-	release_mem_region(pci_resource_start(pdev, 0),
-			   pci_resource_len(pdev, 0));
-	pci_disable_device(pdev);
-}
-
-static int __maybe_unused prism2_pci_suspend(struct device *dev_d)
-{
-	struct net_device *dev = dev_get_drvdata(dev_d);
-
-	if (netif_running(dev)) {
-		netif_stop_queue(dev);
-		netif_device_detach(dev);
-	}
-	prism2_suspend(dev);
-
-	return 0;
-}
-
-static int __maybe_unused prism2_pci_resume(struct device *dev_d)
-{
-	struct net_device *dev = dev_get_drvdata(dev_d);
-
-	prism2_hw_config(dev, 0);
-	if (netif_running(dev)) {
-		netif_device_attach(dev);
-		netif_start_queue(dev);
-	}
-
-	return 0;
-}
-
-MODULE_DEVICE_TABLE(pci, prism2_pci_id_table);
-
-static SIMPLE_DEV_PM_OPS(prism2_pci_pm_ops,
-			 prism2_pci_suspend,
-			 prism2_pci_resume);
-
-static struct pci_driver prism2_pci_driver = {
-	.name		= "hostap_pci",
-	.id_table	= prism2_pci_id_table,
-	.probe		= prism2_pci_probe,
-	.remove		= prism2_pci_remove,
-	.driver.pm	= &prism2_pci_pm_ops,
-};
-
-module_pci_driver(prism2_pci_driver);
diff --git a/drivers/net/wireless/intersil/hostap/hostap_plx.c b/drivers/net/wireless/intersil/hostap/hostap_plx.c
deleted file mode 100644
index 58247290fcbc..000000000000
--- a/drivers/net/wireless/intersil/hostap/hostap_plx.c
+++ /dev/null
@@ -1,617 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-#define PRISM2_PLX
-
-/* Host AP driver's support for PC Cards on PCI adapters using PLX9052 is
- * based on:
- * - Host AP driver patch from james@madingley.org
- * - linux-wlan-ng driver, Copyright (C) AbsoluteValue Systems, Inc.
- */
-
-
-#include <linux/module.h>
-#include <linux/if.h>
-#include <linux/skbuff.h>
-#include <linux/netdevice.h>
-#include <linux/slab.h>
-#include <linux/workqueue.h>
-#include <linux/wireless.h>
-#include <net/iw_handler.h>
-
-#include <linux/ioport.h>
-#include <linux/pci.h>
-#include <asm/io.h>
-
-#include "hostap_wlan.h"
-
-
-static char *dev_info = "hostap_plx";
-
-
-MODULE_AUTHOR("Jouni Malinen");
-MODULE_DESCRIPTION("Support for Intersil Prism2-based 802.11 wireless LAN "
-		   "cards (PLX).");
-MODULE_LICENSE("GPL");
-
-
-static int ignore_cis;
-module_param(ignore_cis, int, 0444);
-MODULE_PARM_DESC(ignore_cis, "Do not verify manfid information in CIS");
-
-
-/* struct local_info::hw_priv */
-struct hostap_plx_priv {
-	void __iomem *attr_mem;
-	unsigned int cor_offset;
-};
-
-
-#define PLX_MIN_ATTR_LEN 512	/* at least 2 x 256 is needed for CIS */
-#define COR_SRESET       0x80
-#define COR_LEVLREQ      0x40
-#define COR_ENABLE_FUNC  0x01
-/* PCI Configuration Registers */
-#define PLX_PCIIPR       0x3d   /* PCI Interrupt Pin */
-/* Local Configuration Registers */
-#define PLX_INTCSR       0x4c   /* Interrupt Control/Status Register */
-#define PLX_INTCSR_PCI_INTEN BIT(6) /* PCI Interrupt Enable */
-#define PLX_CNTRL        0x50
-#define PLX_CNTRL_SERIAL_EEPROM_PRESENT BIT(28)
-
-
-#define PLXDEV(vendor,dev,str) { vendor, dev, PCI_ANY_ID, PCI_ANY_ID }
-
-static const struct pci_device_id prism2_plx_id_table[] = {
-	PLXDEV(0x10b7, 0x7770, "3Com AirConnect PCI 777A"),
-	PLXDEV(0x111a, 0x1023, "Siemens SpeedStream SS1023"),
-	PLXDEV(0x126c, 0x8030, "Nortel emobility"),
-	PLXDEV(0x1562, 0x0001, "Symbol LA-4123"),
-	PLXDEV(0x1385, 0x4100, "Netgear MA301"),
-	PLXDEV(0x15e8, 0x0130, "National Datacomm NCP130 (PLX9052)"),
-	PLXDEV(0x15e8, 0x0131, "National Datacomm NCP130 (TMD7160)"),
-	PLXDEV(0x1638, 0x1100, "Eumitcom WL11000"),
-	PLXDEV(0x16ab, 0x1100, "Global Sun Tech GL24110P"),
-	PLXDEV(0x16ab, 0x1101, "Global Sun Tech GL24110P (?)"),
-	PLXDEV(0x16ab, 0x1102, "Linksys WPC11 with WDT11"),
-	PLXDEV(0x16ab, 0x1103, "Longshine 8031"),
-	PLXDEV(0x16ec, 0x3685, "US Robotics USR2415"),
-	PLXDEV(0xec80, 0xec00, "Belkin F5D6000"),
-	{ 0 }
-};
-
-
-/* Array of known Prism2/2.5 PC Card manufactured ids. If your card's manfid
- * is not listed here, you will need to add it here to get the driver
- * initialized. */
-static struct prism2_plx_manfid {
-	u16 manfid1, manfid2;
-} prism2_plx_known_manfids[] = {
-	{ 0x000b, 0x7110 } /* D-Link DWL-650 Rev. P1 */,
-	{ 0x000b, 0x7300 } /* Philips 802.11b WLAN PCMCIA */,
-	{ 0x0101, 0x0777 } /* 3Com AirConnect PCI 777A */,
-	{ 0x0126, 0x8000 } /* Proxim RangeLAN */,
-	{ 0x0138, 0x0002 } /* Compaq WL100 */,
-	{ 0x0156, 0x0002 } /* Intersil Prism II Ref. Design (and others) */,
-	{ 0x026f, 0x030b } /* Buffalo WLI-CF-S11G */,
-	{ 0x0274, 0x1612 } /* Linksys WPC11 Ver 2.5 */,
-	{ 0x0274, 0x1613 } /* Linksys WPC11 Ver 3 */,
-	{ 0x028a, 0x0002 } /* D-Link DRC-650 */,
-	{ 0x0250, 0x0002 } /* Samsung SWL2000-N */,
-	{ 0xc250, 0x0002 } /* EMTAC A2424i */,
-	{ 0xd601, 0x0002 } /* Z-Com XI300 */,
-	{ 0xd601, 0x0005 } /* Zcomax XI-325H 200mW */,
-	{ 0, 0}
-};
-
-
-#ifdef PRISM2_IO_DEBUG
-
-static inline void hfa384x_outb_debug(struct net_device *dev, int a, u8 v)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	unsigned long flags;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	spin_lock_irqsave(&local->lock, flags);
-	prism2_io_debug_add(dev, PRISM2_IO_DEBUG_CMD_OUTB, a, v);
-	outb(v, dev->base_addr + a);
-	spin_unlock_irqrestore(&local->lock, flags);
-}
-
-static inline u8 hfa384x_inb_debug(struct net_device *dev, int a)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	unsigned long flags;
-	u8 v;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	spin_lock_irqsave(&local->lock, flags);
-	v = inb(dev->base_addr + a);
-	prism2_io_debug_add(dev, PRISM2_IO_DEBUG_CMD_INB, a, v);
-	spin_unlock_irqrestore(&local->lock, flags);
-	return v;
-}
-
-static inline void hfa384x_outw_debug(struct net_device *dev, int a, u16 v)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	unsigned long flags;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	spin_lock_irqsave(&local->lock, flags);
-	prism2_io_debug_add(dev, PRISM2_IO_DEBUG_CMD_OUTW, a, v);
-	outw(v, dev->base_addr + a);
-	spin_unlock_irqrestore(&local->lock, flags);
-}
-
-static inline u16 hfa384x_inw_debug(struct net_device *dev, int a)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	unsigned long flags;
-	u16 v;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	spin_lock_irqsave(&local->lock, flags);
-	v = inw(dev->base_addr + a);
-	prism2_io_debug_add(dev, PRISM2_IO_DEBUG_CMD_INW, a, v);
-	spin_unlock_irqrestore(&local->lock, flags);
-	return v;
-}
-
-static inline void hfa384x_outsw_debug(struct net_device *dev, int a,
-				       u8 *buf, int wc)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	unsigned long flags;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	spin_lock_irqsave(&local->lock, flags);
-	prism2_io_debug_add(dev, PRISM2_IO_DEBUG_CMD_OUTSW, a, wc);
-	outsw(dev->base_addr + a, buf, wc);
-	spin_unlock_irqrestore(&local->lock, flags);
-}
-
-static inline void hfa384x_insw_debug(struct net_device *dev, int a,
-				      u8 *buf, int wc)
-{
-	struct hostap_interface *iface;
-	local_info_t *local;
-	unsigned long flags;
-
-	iface = netdev_priv(dev);
-	local = iface->local;
-
-	spin_lock_irqsave(&local->lock, flags);
-	prism2_io_debug_add(dev, PRISM2_IO_DEBUG_CMD_INSW, a, wc);
-	insw(dev->base_addr + a, buf, wc);
-	spin_unlock_irqrestore(&local->lock, flags);
-}
-
-#define HFA384X_OUTB(v,a) hfa384x_outb_debug(dev, (a), (v))
-#define HFA384X_INB(a) hfa384x_inb_debug(dev, (a))
-#define HFA384X_OUTW(v,a) hfa384x_outw_debug(dev, (a), (v))
-#define HFA384X_INW(a) hfa384x_inw_debug(dev, (a))
-#define HFA384X_OUTSW(a, buf, wc) hfa384x_outsw_debug(dev, (a), (buf), (wc))
-#define HFA384X_INSW(a, buf, wc) hfa384x_insw_debug(dev, (a), (buf), (wc))
-
-#else /* PRISM2_IO_DEBUG */
-
-#define HFA384X_OUTB(v,a) outb((v), dev->base_addr + (a))
-#define HFA384X_INB(a) inb(dev->base_addr + (a))
-#define HFA384X_OUTW(v,a) outw((v), dev->base_addr + (a))
-#define HFA384X_INW(a) inw(dev->base_addr + (a))
-#define HFA384X_INSW(a, buf, wc) insw(dev->base_addr + (a), buf, wc)
-#define HFA384X_OUTSW(a, buf, wc) outsw(dev->base_addr + (a), buf, wc)
-
-#endif /* PRISM2_IO_DEBUG */
-
-
-static int hfa384x_from_bap(struct net_device *dev, u16 bap, void *buf,
-			    int len)
-{
-	u16 d_off;
-	u16 *pos;
-
-	d_off = (bap == 1) ? HFA384X_DATA1_OFF : HFA384X_DATA0_OFF;
-	pos = (u16 *) buf;
-
-	if (len / 2)
-		HFA384X_INSW(d_off, buf, len / 2);
-	pos += len / 2;
-
-	if (len & 1)
-		*((char *) pos) = HFA384X_INB(d_off);
-
-	return 0;
-}
-
-
-static int hfa384x_to_bap(struct net_device *dev, u16 bap, void *buf, int len)
-{
-	u16 d_off;
-	u16 *pos;
-
-	d_off = (bap == 1) ? HFA384X_DATA1_OFF : HFA384X_DATA0_OFF;
-	pos = (u16 *) buf;
-
-	if (len / 2)
-		HFA384X_OUTSW(d_off, buf, len / 2);
-	pos += len / 2;
-
-	if (len & 1)
-		HFA384X_OUTB(*((char *) pos), d_off);
-
-	return 0;
-}
-
-
-/* FIX: This might change at some point.. */
-#include "hostap_hw.c"
-
-
-static void prism2_plx_cor_sreset(local_info_t *local)
-{
-	unsigned char corsave;
-	struct hostap_plx_priv *hw_priv = local->hw_priv;
-
-	printk(KERN_DEBUG "%s: Doing reset via direct COR access.\n",
-	       dev_info);
-
-	/* Set sreset bit of COR and clear it after hold time */
-
-	if (hw_priv->attr_mem == NULL) {
-		/* TMD7160 - COR at card's first I/O addr */
-		corsave = inb(hw_priv->cor_offset);
-		outb(corsave | COR_SRESET, hw_priv->cor_offset);
-		mdelay(2);
-		outb(corsave & ~COR_SRESET, hw_priv->cor_offset);
-		mdelay(2);
-	} else {
-		/* PLX9052 */
-		corsave = readb(hw_priv->attr_mem + hw_priv->cor_offset);
-		writeb(corsave | COR_SRESET,
-		       hw_priv->attr_mem + hw_priv->cor_offset);
-		mdelay(2);
-		writeb(corsave & ~COR_SRESET,
-		       hw_priv->attr_mem + hw_priv->cor_offset);
-		mdelay(2);
-	}
-}
-
-
-static void prism2_plx_genesis_reset(local_info_t *local, int hcr)
-{
-	unsigned char corsave;
-	struct hostap_plx_priv *hw_priv = local->hw_priv;
-
-	if (hw_priv->attr_mem == NULL) {
-		/* TMD7160 - COR at card's first I/O addr */
-		corsave = inb(hw_priv->cor_offset);
-		outb(corsave | COR_SRESET, hw_priv->cor_offset);
-		mdelay(10);
-		outb(hcr, hw_priv->cor_offset + 2);
-		mdelay(10);
-		outb(corsave & ~COR_SRESET, hw_priv->cor_offset);
-		mdelay(10);
-	} else {
-		/* PLX9052 */
-		corsave = readb(hw_priv->attr_mem + hw_priv->cor_offset);
-		writeb(corsave | COR_SRESET,
-		       hw_priv->attr_mem + hw_priv->cor_offset);
-		mdelay(10);
-		writeb(hcr, hw_priv->attr_mem + hw_priv->cor_offset + 2);
-		mdelay(10);
-		writeb(corsave & ~COR_SRESET,
-		       hw_priv->attr_mem + hw_priv->cor_offset);
-		mdelay(10);
-	}
-}
-
-
-static struct prism2_helper_functions prism2_plx_funcs =
-{
-	.card_present	= NULL,
-	.cor_sreset	= prism2_plx_cor_sreset,
-	.genesis_reset	= prism2_plx_genesis_reset,
-	.hw_type	= HOSTAP_HW_PLX,
-};
-
-
-static int prism2_plx_check_cis(void __iomem *attr_mem, int attr_len,
-				unsigned int *cor_offset,
-				unsigned int *cor_index)
-{
-#define CISTPL_CONFIG 0x1A
-#define CISTPL_MANFID 0x20
-#define CISTPL_END 0xFF
-#define CIS_MAX_LEN 256
-	u8 *cis;
-	int i, pos;
-	unsigned int rmsz, rasz, manfid1, manfid2;
-	struct prism2_plx_manfid *manfid;
-
-	cis = kmalloc(CIS_MAX_LEN, GFP_KERNEL);
-	if (cis == NULL)
-		return -ENOMEM;
-
-	/* read CIS; it is in even offsets in the beginning of attr_mem */
-	for (i = 0; i < CIS_MAX_LEN; i++)
-		cis[i] = readb(attr_mem + 2 * i);
-	printk(KERN_DEBUG "%s: CIS: %6ph ...\n", dev_info, cis);
-
-	/* set reasonable defaults for Prism2 cards just in case CIS parsing
-	 * fails */
-	*cor_offset = 0x3e0;
-	*cor_index = 0x01;
-	manfid1 = manfid2 = 0;
-
-	pos = 0;
-	while (pos < CIS_MAX_LEN - 1 && cis[pos] != CISTPL_END) {
-		if (pos + 2 + cis[pos + 1] > CIS_MAX_LEN)
-			goto cis_error;
-
-		switch (cis[pos]) {
-		case CISTPL_CONFIG:
-			if (cis[pos + 1] < 2)
-				goto cis_error;
-			rmsz = (cis[pos + 2] & 0x3c) >> 2;
-			rasz = cis[pos + 2] & 0x03;
-			if (4 + rasz + rmsz > cis[pos + 1])
-				goto cis_error;
-			*cor_index = cis[pos + 3] & 0x3F;
-			*cor_offset = 0;
-			for (i = 0; i <= rasz; i++)
-				*cor_offset += cis[pos + 4 + i] << (8 * i);
-			printk(KERN_DEBUG "%s: cor_index=0x%x "
-			       "cor_offset=0x%x\n", dev_info,
-			       *cor_index, *cor_offset);
-			if (*cor_offset > attr_len) {
-				printk(KERN_ERR "%s: COR offset not within "
-				       "attr_mem\n", dev_info);
-				kfree(cis);
-				return -1;
-			}
-			break;
-
-		case CISTPL_MANFID:
-			if (cis[pos + 1] < 4)
-				goto cis_error;
-			manfid1 = cis[pos + 2] + (cis[pos + 3] << 8);
-			manfid2 = cis[pos + 4] + (cis[pos + 5] << 8);
-			printk(KERN_DEBUG "%s: manfid=0x%04x, 0x%04x\n",
-			       dev_info, manfid1, manfid2);
-			break;
-		}
-
-		pos += cis[pos + 1] + 2;
-	}
-
-	if (pos >= CIS_MAX_LEN || cis[pos] != CISTPL_END)
-		goto cis_error;
-
-	for (manfid = prism2_plx_known_manfids; manfid->manfid1 != 0; manfid++)
-		if (manfid1 == manfid->manfid1 && manfid2 == manfid->manfid2) {
-			kfree(cis);
-			return 0;
-		}
-
-	printk(KERN_INFO "%s: unknown manfid 0x%04x, 0x%04x - assuming this is"
-	       " not supported card\n", dev_info, manfid1, manfid2);
-	goto fail;
-
- cis_error:
-	printk(KERN_WARNING "%s: invalid CIS data\n", dev_info);
-
- fail:
-	kfree(cis);
-	if (ignore_cis) {
-		printk(KERN_INFO "%s: ignore_cis parameter set - ignoring "
-		       "errors during CIS verification\n", dev_info);
-		return 0;
-	}
-	return -1;
-}
-
-
-static int prism2_plx_probe(struct pci_dev *pdev,
-			    const struct pci_device_id *id)
-{
-	unsigned int pccard_ioaddr, plx_ioaddr;
-	unsigned long pccard_attr_mem;
-	unsigned int pccard_attr_len;
-	void __iomem *attr_mem = NULL;
-	unsigned int cor_offset = 0, cor_index = 0;
-	u32 reg;
-	local_info_t *local = NULL;
-	struct net_device *dev = NULL;
-	struct hostap_interface *iface;
-	static int cards_found /* = 0 */;
-	int irq_registered = 0;
-	int tmd7160;
-	struct hostap_plx_priv *hw_priv;
-
-	hw_priv = kzalloc(sizeof(*hw_priv), GFP_KERNEL);
-	if (hw_priv == NULL)
-		return -ENOMEM;
-
-	if (pci_enable_device(pdev))
-		goto err_out_free;
-
-	/* National Datacomm NCP130 based on TMD7160, not PLX9052. */
-	tmd7160 = (pdev->vendor == 0x15e8) && (pdev->device == 0x0131);
-
-	plx_ioaddr = pci_resource_start(pdev, 1);
-	pccard_ioaddr = pci_resource_start(pdev, tmd7160 ? 2 : 3);
-
-	if (tmd7160) {
-		/* TMD7160 */
-		attr_mem = NULL; /* no access to PC Card attribute memory */
-
-		printk(KERN_INFO "TMD7160 PCI/PCMCIA adapter: io=0x%x, "
-		       "irq=%d, pccard_io=0x%x\n",
-		       plx_ioaddr, pdev->irq, pccard_ioaddr);
-
-		cor_offset = plx_ioaddr;
-		cor_index = 0x04;
-
-		outb(cor_index | COR_LEVLREQ | COR_ENABLE_FUNC, plx_ioaddr);
-		mdelay(1);
-		reg = inb(plx_ioaddr);
-		if (reg != (cor_index | COR_LEVLREQ | COR_ENABLE_FUNC)) {
-			printk(KERN_ERR "%s: Error setting COR (expected="
-			       "0x%02x, was=0x%02x)\n", dev_info,
-			       cor_index | COR_LEVLREQ | COR_ENABLE_FUNC, reg);
-			goto fail;
-		}
-	} else {
-		/* PLX9052 */
-		pccard_attr_mem = pci_resource_start(pdev, 2);
-		pccard_attr_len = pci_resource_len(pdev, 2);
-		if (pccard_attr_len < PLX_MIN_ATTR_LEN)
-			goto fail;
-
-
-		attr_mem = ioremap(pccard_attr_mem, pccard_attr_len);
-		if (attr_mem == NULL) {
-			printk(KERN_ERR "%s: cannot remap attr_mem\n",
-			       dev_info);
-			goto fail;
-		}
-
-		printk(KERN_INFO "PLX9052 PCI/PCMCIA adapter: "
-		       "mem=0x%lx, plx_io=0x%x, irq=%d, pccard_io=0x%x\n",
-		       pccard_attr_mem, plx_ioaddr, pdev->irq, pccard_ioaddr);
-
-		if (prism2_plx_check_cis(attr_mem, pccard_attr_len,
-					 &cor_offset, &cor_index)) {
-			printk(KERN_INFO "Unknown PC Card CIS - not a "
-			       "Prism2/2.5 card?\n");
-			goto fail;
-		}
-
-		printk(KERN_DEBUG "Prism2/2.5 PC Card detected in PLX9052 "
-		       "adapter\n");
-
-		/* Write COR to enable PC Card */
-		writeb(cor_index | COR_LEVLREQ | COR_ENABLE_FUNC,
-		       attr_mem + cor_offset);
-
-		/* Enable PCI interrupts if they are not already enabled */
-		reg = inl(plx_ioaddr + PLX_INTCSR);
-		printk(KERN_DEBUG "PLX_INTCSR=0x%x\n", reg);
-		if (!(reg & PLX_INTCSR_PCI_INTEN)) {
-			outl(reg | PLX_INTCSR_PCI_INTEN,
-			     plx_ioaddr + PLX_INTCSR);
-			if (!(inl(plx_ioaddr + PLX_INTCSR) &
-			      PLX_INTCSR_PCI_INTEN)) {
-				printk(KERN_WARNING "%s: Could not enable "
-				       "Local Interrupts\n", dev_info);
-				goto fail;
-			}
-		}
-
-		reg = inl(plx_ioaddr + PLX_CNTRL);
-		printk(KERN_DEBUG "PLX_CNTRL=0x%x (Serial EEPROM "
-		       "present=%d)\n",
-		       reg, (reg & PLX_CNTRL_SERIAL_EEPROM_PRESENT) != 0);
-		/* should set PLX_PCIIPR to 0x01 (INTA#) if Serial EEPROM is
-		 * not present; but are there really such cards in use(?) */
-	}
-
-	dev = prism2_init_local_data(&prism2_plx_funcs, cards_found,
-				     &pdev->dev);
-	if (dev == NULL)
-		goto fail;
-	iface = netdev_priv(dev);
-	local = iface->local;
-	local->hw_priv = hw_priv;
-	cards_found++;
-
-	dev->irq = pdev->irq;
-	dev->base_addr = pccard_ioaddr;
-	hw_priv->attr_mem = attr_mem;
-	hw_priv->cor_offset = cor_offset;
-
-	pci_set_drvdata(pdev, dev);
-
-	if (request_irq(dev->irq, prism2_interrupt, IRQF_SHARED, dev->name,
-			dev)) {
-		printk(KERN_WARNING "%s: request_irq failed\n", dev->name);
-		goto fail;
-	} else
-		irq_registered = 1;
-
-	if (prism2_hw_config(dev, 1)) {
-		printk(KERN_DEBUG "%s: hardware initialization failed\n",
-		       dev_info);
-		goto fail;
-	}
-
-	return hostap_hw_ready(dev);
-
- fail:
-	if (irq_registered && dev)
-		free_irq(dev->irq, dev);
-
-	if (attr_mem)
-		iounmap(attr_mem);
-
-	pci_disable_device(pdev);
-	prism2_free_local_data(dev);
-
- err_out_free:
-	kfree(hw_priv);
-
-	return -ENODEV;
-}
-
-
-static void prism2_plx_remove(struct pci_dev *pdev)
-{
-	struct net_device *dev;
-	struct hostap_interface *iface;
-	struct hostap_plx_priv *hw_priv;
-
-	dev = pci_get_drvdata(pdev);
-	iface = netdev_priv(dev);
-	hw_priv = iface->local->hw_priv;
-
-	/* Reset the hardware, and ensure interrupts are disabled. */
-	prism2_plx_cor_sreset(iface->local);
-	hfa384x_disable_interrupts(dev);
-
-	if (hw_priv->attr_mem)
-		iounmap(hw_priv->attr_mem);
-	if (dev->irq)
-		free_irq(dev->irq, dev);
-
-	prism2_free_local_data(dev);
-	kfree(hw_priv);
-	pci_disable_device(pdev);
-}
-
-
-MODULE_DEVICE_TABLE(pci, prism2_plx_id_table);
-
-static struct pci_driver prism2_plx_driver = {
-	.name		= "hostap_plx",
-	.id_table	= prism2_plx_id_table,
-	.probe		= prism2_plx_probe,
-	.remove		= prism2_plx_remove,
-};
-
-module_pci_driver(prism2_plx_driver);
diff --git a/drivers/net/wireless/intersil/hostap/hostap_proc.c b/drivers/net/wireless/intersil/hostap/hostap_proc.c
deleted file mode 100644
index 61f68786056f..000000000000
--- a/drivers/net/wireless/intersil/hostap/hostap_proc.c
+++ /dev/null
@@ -1,411 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/* /proc routines for Host AP driver */
-
-#include <linux/types.h>
-#include <linux/proc_fs.h>
-#include <linux/export.h>
-#include <net/lib80211.h>
-
-#include "hostap_wlan.h"
-#include "hostap.h"
-
-#define PROC_LIMIT (PAGE_SIZE - 80)
-
-#if !defined(PRISM2_NO_PROCFS_DEBUG) && defined(CONFIG_PROC_FS)
-static int prism2_debug_proc_show(struct seq_file *m, void *v)
-{
-	local_info_t *local = m->private;
-	int i;
-
-	seq_printf(m, "next_txfid=%d next_alloc=%d\n",
-		   local->next_txfid, local->next_alloc);
-	for (i = 0; i < PRISM2_TXFID_COUNT; i++)
-		seq_printf(m, "FID: tx=%04X intransmit=%04X\n",
-			   local->txfid[i], local->intransmitfid[i]);
-	seq_printf(m, "FW TX rate control: %d\n", local->fw_tx_rate_control);
-	seq_printf(m, "beacon_int=%d\n", local->beacon_int);
-	seq_printf(m, "dtim_period=%d\n", local->dtim_period);
-	seq_printf(m, "wds_max_connections=%d\n", local->wds_max_connections);
-	seq_printf(m, "dev_enabled=%d\n", local->dev_enabled);
-	seq_printf(m, "sw_tick_stuck=%d\n", local->sw_tick_stuck);
-	for (i = 0; i < WEP_KEYS; i++) {
-		if (local->crypt_info.crypt[i] &&
-		    local->crypt_info.crypt[i]->ops) {
-			seq_printf(m, "crypt[%d]=%s\n", i,
-				   local->crypt_info.crypt[i]->ops->name);
-		}
-	}
-	seq_printf(m, "pri_only=%d\n", local->pri_only);
-	seq_printf(m, "pci=%d\n", local->func->hw_type == HOSTAP_HW_PCI);
-	seq_printf(m, "sram_type=%d\n", local->sram_type);
-	seq_printf(m, "no_pri=%d\n", local->no_pri);
-
-	return 0;
-}
-#endif
-
-#ifdef CONFIG_PROC_FS
-static int prism2_stats_proc_show(struct seq_file *m, void *v)
-{
-	local_info_t *local = m->private;
-	struct comm_tallies_sums *sums = &local->comm_tallies;
-
-	seq_printf(m, "TxUnicastFrames=%u\n", sums->tx_unicast_frames);
-	seq_printf(m, "TxMulticastframes=%u\n", sums->tx_multicast_frames);
-	seq_printf(m, "TxFragments=%u\n", sums->tx_fragments);
-	seq_printf(m, "TxUnicastOctets=%u\n", sums->tx_unicast_octets);
-	seq_printf(m, "TxMulticastOctets=%u\n", sums->tx_multicast_octets);
-	seq_printf(m, "TxDeferredTransmissions=%u\n",
-		   sums->tx_deferred_transmissions);
-	seq_printf(m, "TxSingleRetryFrames=%u\n", sums->tx_single_retry_frames);
-	seq_printf(m, "TxMultipleRetryFrames=%u\n",
-		   sums->tx_multiple_retry_frames);
-	seq_printf(m, "TxRetryLimitExceeded=%u\n",
-		   sums->tx_retry_limit_exceeded);
-	seq_printf(m, "TxDiscards=%u\n", sums->tx_discards);
-	seq_printf(m, "RxUnicastFrames=%u\n", sums->rx_unicast_frames);
-	seq_printf(m, "RxMulticastFrames=%u\n", sums->rx_multicast_frames);
-	seq_printf(m, "RxFragments=%u\n", sums->rx_fragments);
-	seq_printf(m, "RxUnicastOctets=%u\n", sums->rx_unicast_octets);
-	seq_printf(m, "RxMulticastOctets=%u\n", sums->rx_multicast_octets);
-	seq_printf(m, "RxFCSErrors=%u\n", sums->rx_fcs_errors);
-	seq_printf(m, "RxDiscardsNoBuffer=%u\n", sums->rx_discards_no_buffer);
-	seq_printf(m, "TxDiscardsWrongSA=%u\n", sums->tx_discards_wrong_sa);
-	seq_printf(m, "RxDiscardsWEPUndecryptable=%u\n",
-		   sums->rx_discards_wep_undecryptable);
-	seq_printf(m, "RxMessageInMsgFragments=%u\n",
-		   sums->rx_message_in_msg_fragments);
-	seq_printf(m, "RxMessageInBadMsgFragments=%u\n",
-		   sums->rx_message_in_bad_msg_fragments);
-	/* FIX: this may grow too long for one page(?) */
-
-	return 0;
-}
-#endif
-
-static int prism2_wds_proc_show(struct seq_file *m, void *v)
-{
-	struct list_head *ptr = v;
-	struct hostap_interface *iface;
-
-	iface = list_entry(ptr, struct hostap_interface, list);
-	if (iface->type == HOSTAP_INTERFACE_WDS)
-		seq_printf(m, "%s\t%pM\n",
-			   iface->dev->name, iface->u.wds.remote_addr);
-	return 0;
-}
-
-static void *prism2_wds_proc_start(struct seq_file *m, loff_t *_pos)
-{
-	local_info_t *local = pde_data(file_inode(m->file));
-	read_lock_bh(&local->iface_lock);
-	return seq_list_start(&local->hostap_interfaces, *_pos);
-}
-
-static void *prism2_wds_proc_next(struct seq_file *m, void *v, loff_t *_pos)
-{
-	local_info_t *local = pde_data(file_inode(m->file));
-	return seq_list_next(v, &local->hostap_interfaces, _pos);
-}
-
-static void prism2_wds_proc_stop(struct seq_file *m, void *v)
-{
-	local_info_t *local = pde_data(file_inode(m->file));
-	read_unlock_bh(&local->iface_lock);
-}
-
-static const struct seq_operations prism2_wds_proc_seqops = {
-	.start	= prism2_wds_proc_start,
-	.next	= prism2_wds_proc_next,
-	.stop	= prism2_wds_proc_stop,
-	.show	= prism2_wds_proc_show,
-};
-
-static int prism2_bss_list_proc_show(struct seq_file *m, void *v)
-{
-	local_info_t *local = pde_data(file_inode(m->file));
-	struct list_head *ptr = v;
-	struct hostap_bss_info *bss;
-
-	if (ptr == &local->bss_list) {
-		seq_printf(m, "#BSSID\tlast_update\tcount\tcapab_info\tSSID(txt)\t"
-			   "SSID(hex)\tWPA IE\n");
-		return 0;
-	}
-
-	bss = list_entry(ptr, struct hostap_bss_info, list);
-	seq_printf(m, "%pM\t%lu\t%u\t0x%x\t",
-		   bss->bssid, bss->last_update,
-		   bss->count, bss->capab_info);
-
-	seq_printf(m, "%*pE", (int)bss->ssid_len, bss->ssid);
-
-	seq_putc(m, '\t');
-	seq_printf(m, "%*phN", (int)bss->ssid_len, bss->ssid);
-	seq_putc(m, '\t');
-	seq_printf(m, "%*phN", (int)bss->wpa_ie_len, bss->wpa_ie);
-	seq_putc(m, '\n');
-	return 0;
-}
-
-static void *prism2_bss_list_proc_start(struct seq_file *m, loff_t *_pos)
-	__acquires(&local->lock)
-{
-	local_info_t *local = pde_data(file_inode(m->file));
-	spin_lock_bh(&local->lock);
-	return seq_list_start_head(&local->bss_list, *_pos);
-}
-
-static void *prism2_bss_list_proc_next(struct seq_file *m, void *v, loff_t *_pos)
-{
-	local_info_t *local = pde_data(file_inode(m->file));
-	return seq_list_next(v, &local->bss_list, _pos);
-}
-
-static void prism2_bss_list_proc_stop(struct seq_file *m, void *v)
-	__releases(&local->lock)
-{
-	local_info_t *local = pde_data(file_inode(m->file));
-	spin_unlock_bh(&local->lock);
-}
-
-static const struct seq_operations prism2_bss_list_proc_seqops = {
-	.start	= prism2_bss_list_proc_start,
-	.next	= prism2_bss_list_proc_next,
-	.stop	= prism2_bss_list_proc_stop,
-	.show	= prism2_bss_list_proc_show,
-};
-
-#ifdef CONFIG_PROC_FS
-static int prism2_crypt_proc_show(struct seq_file *m, void *v)
-{
-	local_info_t *local = m->private;
-	int i;
-
-	seq_printf(m, "tx_keyidx=%d\n", local->crypt_info.tx_keyidx);
-	for (i = 0; i < WEP_KEYS; i++) {
-		if (local->crypt_info.crypt[i] &&
-		    local->crypt_info.crypt[i]->ops &&
-		    local->crypt_info.crypt[i]->ops->print_stats) {
-			local->crypt_info.crypt[i]->ops->print_stats(
-				m, local->crypt_info.crypt[i]->priv);
-		}
-	}
-	return 0;
-}
-#endif
-
-static ssize_t prism2_pda_proc_read(struct file *file, char __user *buf,
-				    size_t count, loff_t *_pos)
-{
-	local_info_t *local = pde_data(file_inode(file));
-	size_t off;
-
-	if (local->pda == NULL || *_pos >= PRISM2_PDA_SIZE)
-		return 0;
-
-	off = *_pos;
-	if (count > PRISM2_PDA_SIZE - off)
-		count = PRISM2_PDA_SIZE - off;
-	if (copy_to_user(buf, local->pda + off, count) != 0)
-		return -EFAULT;
-	*_pos += count;
-	return count;
-}
-
-static const struct proc_ops prism2_pda_proc_ops = {
-	.proc_read	= prism2_pda_proc_read,
-	.proc_lseek	= generic_file_llseek,
-};
-
-
-static ssize_t prism2_aux_dump_proc_no_read(struct file *file, char __user *buf,
-					    size_t bufsize, loff_t *_pos)
-{
-	return 0;
-}
-
-static const struct proc_ops prism2_aux_dump_proc_ops = {
-	.proc_read	= prism2_aux_dump_proc_no_read,
-	.proc_lseek	= default_llseek,
-};
-
-
-#ifdef PRISM2_IO_DEBUG
-static int prism2_io_debug_proc_read(char *page, char **start, off_t off,
-				     int count, int *eof, void *data)
-{
-	local_info_t *local = (local_info_t *) data;
-	int head = local->io_debug_head;
-	int start_bytes, left, copy;
-
-	if (off + count > PRISM2_IO_DEBUG_SIZE * 4) {
-		*eof = 1;
-		if (off >= PRISM2_IO_DEBUG_SIZE * 4)
-			return 0;
-		count = PRISM2_IO_DEBUG_SIZE * 4 - off;
-	}
-
-	start_bytes = (PRISM2_IO_DEBUG_SIZE - head) * 4;
-	left = count;
-
-	if (off < start_bytes) {
-		copy = start_bytes - off;
-		if (copy > count)
-			copy = count;
-		memcpy(page, ((u8 *) &local->io_debug[head]) + off, copy);
-		left -= copy;
-		if (left > 0)
-			memcpy(&page[copy], local->io_debug, left);
-	} else {
-		memcpy(page, ((u8 *) local->io_debug) + (off - start_bytes),
-		       left);
-	}
-
-	*start = page;
-
-	return count;
-}
-#endif /* PRISM2_IO_DEBUG */
-
-
-#ifndef PRISM2_NO_STATION_MODES
-static int prism2_scan_results_proc_show(struct seq_file *m, void *v)
-{
-	local_info_t *local = pde_data(file_inode(m->file));
-	unsigned long entry;
-	int i, len;
-	struct hfa384x_hostscan_result *scanres;
-	u8 *p;
-
-	if (v == SEQ_START_TOKEN) {
-		seq_printf(m,
-			   "CHID ANL SL BcnInt Capab Rate BSSID ATIM SupRates SSID\n");
-		return 0;
-	}
-
-	entry = (unsigned long)v - 2;
-	scanres = &local->last_scan_results[entry];
-
-	seq_printf(m, "%d %d %d %d 0x%02x %d %pM %d ",
-		   le16_to_cpu(scanres->chid),
-		   (s16) le16_to_cpu(scanres->anl),
-		   (s16) le16_to_cpu(scanres->sl),
-		   le16_to_cpu(scanres->beacon_interval),
-		   le16_to_cpu(scanres->capability),
-		   le16_to_cpu(scanres->rate),
-		   scanres->bssid,
-		   le16_to_cpu(scanres->atim));
-
-	p = scanres->sup_rates;
-	for (i = 0; i < sizeof(scanres->sup_rates); i++) {
-		if (p[i] == 0)
-			break;
-		seq_printf(m, "<%02x>", p[i]);
-	}
-	seq_putc(m, ' ');
-
-	p = scanres->ssid;
-	len = le16_to_cpu(scanres->ssid_len);
-	if (len > 32)
-		len = 32;
-	for (i = 0; i < len; i++) {
-		unsigned char c = p[i];
-		if (c >= 32 && c < 127)
-			seq_putc(m, c);
-		else
-			seq_printf(m, "<%02x>", c);
-	}
-	seq_putc(m, '\n');
-	return 0;
-}
-
-static void *prism2_scan_results_proc_start(struct seq_file *m, loff_t *_pos)
-{
-	local_info_t *local = pde_data(file_inode(m->file));
-	spin_lock_bh(&local->lock);
-
-	/* We have a header (pos 0) + N results to show (pos 1...N) */
-	if (*_pos > local->last_scan_results_count)
-		return NULL;
-	return (void *)(unsigned long)(*_pos + 1); /* 0 would be EOF */
-}
-
-static void *prism2_scan_results_proc_next(struct seq_file *m, void *v, loff_t *_pos)
-{
-	local_info_t *local = pde_data(file_inode(m->file));
-
-	++*_pos;
-	if (*_pos > local->last_scan_results_count)
-		return NULL;
-	return (void *)(unsigned long)(*_pos + 1); /* 0 would be EOF */
-}
-
-static void prism2_scan_results_proc_stop(struct seq_file *m, void *v)
-{
-	local_info_t *local = pde_data(file_inode(m->file));
-	spin_unlock_bh(&local->lock);
-}
-
-static const struct seq_operations prism2_scan_results_proc_seqops = {
-	.start	= prism2_scan_results_proc_start,
-	.next	= prism2_scan_results_proc_next,
-	.stop	= prism2_scan_results_proc_stop,
-	.show	= prism2_scan_results_proc_show,
-};
-#endif /* PRISM2_NO_STATION_MODES */
-
-
-void hostap_init_proc(local_info_t *local)
-{
-	local->proc = NULL;
-
-	if (hostap_proc == NULL) {
-		printk(KERN_WARNING "%s: hostap proc directory not created\n",
-		       local->dev->name);
-		return;
-	}
-
-	local->proc = proc_mkdir(local->ddev->name, hostap_proc);
-	if (local->proc == NULL) {
-		printk(KERN_INFO "/proc/net/hostap/%s creation failed\n",
-		       local->ddev->name);
-		return;
-	}
-
-#ifndef PRISM2_NO_PROCFS_DEBUG
-	proc_create_single_data("debug", 0, local->proc,
-			prism2_debug_proc_show, local);
-#endif /* PRISM2_NO_PROCFS_DEBUG */
-	proc_create_single_data("stats", 0, local->proc, prism2_stats_proc_show,
-			local);
-	proc_create_seq_data("wds", 0, local->proc,
-			&prism2_wds_proc_seqops, local);
-	proc_create_data("pda", 0, local->proc,
-			 &prism2_pda_proc_ops, local);
-	proc_create_data("aux_dump", 0, local->proc,
-			 local->func->read_aux_proc_ops ?: &prism2_aux_dump_proc_ops,
-			 local);
-	proc_create_seq_data("bss_list", 0, local->proc,
-			&prism2_bss_list_proc_seqops, local);
-	proc_create_single_data("crypt", 0, local->proc, prism2_crypt_proc_show,
-		local);
-#ifdef PRISM2_IO_DEBUG
-	proc_create_single_data("io_debug", 0, local->proc,
-			prism2_debug_proc_show, local);
-#endif /* PRISM2_IO_DEBUG */
-#ifndef PRISM2_NO_STATION_MODES
-	proc_create_seq_data("scan_results", 0, local->proc,
-			&prism2_scan_results_proc_seqops, local);
-#endif /* PRISM2_NO_STATION_MODES */
-}
-
-
-void hostap_remove_proc(local_info_t *local)
-{
-	proc_remove(local->proc);
-}
-
-
-EXPORT_SYMBOL(hostap_init_proc);
-EXPORT_SYMBOL(hostap_remove_proc);
diff --git a/drivers/net/wireless/intersil/hostap/hostap_wlan.h b/drivers/net/wireless/intersil/hostap/hostap_wlan.h
deleted file mode 100644
index f71c0545c0be..000000000000
--- a/drivers/net/wireless/intersil/hostap/hostap_wlan.h
+++ /dev/null
@@ -1,1051 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef HOSTAP_WLAN_H
-#define HOSTAP_WLAN_H
-
-#include <linux/interrupt.h>
-#include <linux/wireless.h>
-#include <linux/netdevice.h>
-#include <linux/etherdevice.h>
-#include <linux/mutex.h>
-#include <linux/refcount.h>
-#include <net/iw_handler.h>
-#include <net/ieee80211_radiotap.h>
-#include <net/lib80211.h>
-
-#include "hostap_config.h"
-#include "hostap_common.h"
-
-#define MAX_PARM_DEVICES 8
-#define PARM_MIN_MAX "1-" __MODULE_STRING(MAX_PARM_DEVICES)
-#define DEF_INTS -1, -1, -1, -1, -1, -1, -1
-#define GET_INT_PARM(var,idx) var[var[idx] < 0 ? 0 : idx]
-
-
-/* Specific skb->protocol value that indicates that the packet already contains
- * txdesc header.
- * FIX: This might need own value that would be allocated especially for Prism2
- * txdesc; ETH_P_CONTROL is commented as "Card specific control frames".
- * However, these skb's should have only minimal path in the kernel side since
- * prism2_send_mgmt() sends these with dev_queue_xmit() to prism2_tx(). */
-#define ETH_P_HOSTAP ETH_P_CONTROL
-
-/* ARPHRD_IEEE80211_PRISM uses a bloated version of Prism2 RX frame header
- * (from linux-wlan-ng) */
-struct linux_wlan_ng_val {
-	u32 did;
-	u16 status, len;
-	u32 data;
-} __packed;
-
-struct linux_wlan_ng_prism_hdr {
-	u32 msgcode, msglen;
-	char devname[16];
-	struct linux_wlan_ng_val hosttime, mactime, channel, rssi, sq, signal,
-		noise, rate, istx, frmlen;
-} __packed;
-
-struct linux_wlan_ng_cap_hdr {
-	__be32 version;
-	__be32 length;
-	__be64 mactime;
-	__be64 hosttime;
-	__be32 phytype;
-	__be32 channel;
-	__be32 datarate;
-	__be32 antenna;
-	__be32 priority;
-	__be32 ssi_type;
-	__be32 ssi_signal;
-	__be32 ssi_noise;
-	__be32 preamble;
-	__be32 encoding;
-} __packed;
-
-struct hostap_radiotap_rx {
-	struct ieee80211_radiotap_header hdr;
-	__le64 tsft;
-	u8 rate;
-	u8 padding;
-	__le16 chan_freq;
-	__le16 chan_flags;
-	s8 dbm_antsignal;
-	s8 dbm_antnoise;
-} __packed;
-
-#define LWNG_CAP_DID_BASE   (4 | (1 << 6)) /* section 4, group 1 */
-#define LWNG_CAPHDR_VERSION 0x80211001
-
-struct hfa384x_rx_frame {
-	/* HFA384X RX frame descriptor */
-	__le16 status; /* HFA384X_RX_STATUS_ flags */
-	__le32 time; /* timestamp, 1 microsecond resolution */
-	u8 silence; /* 27 .. 154; seems to be 0 */
-	u8 signal; /* 27 .. 154 */
-	u8 rate; /* 10, 20, 55, or 110 */
-	u8 rxflow;
-	__le32 reserved;
-
-	/* 802.11 */
-	__le16 frame_control;
-	__le16 duration_id;
-	u8 addr1[ETH_ALEN];
-	u8 addr2[ETH_ALEN];
-	u8 addr3[ETH_ALEN];
-	__le16 seq_ctrl;
-	u8 addr4[ETH_ALEN];
-	__le16 data_len;
-
-	/* 802.3 */
-	u8 dst_addr[ETH_ALEN];
-	u8 src_addr[ETH_ALEN];
-	__be16 len;
-
-	/* followed by frame data; max 2304 bytes */
-} __packed;
-
-
-struct hfa384x_tx_frame {
-	/* HFA384X TX frame descriptor */
-	__le16 status; /* HFA384X_TX_STATUS_ flags */
-	__le16 reserved1;
-	__le16 reserved2;
-	__le32 sw_support;
-	u8 retry_count; /* not yet implemented */
-	u8 tx_rate; /* Host AP only; 0 = firmware, or 10, 20, 55, 110 */
-	__le16 tx_control; /* HFA384X_TX_CTRL_ flags */
-
-	/* 802.11 */
-	struct_group(header,
-		__le16 frame_control; /* parts not used */
-		__le16 duration_id;
-		u8 addr1[ETH_ALEN];
-		u8 addr2[ETH_ALEN]; /* filled by firmware */
-		u8 addr3[ETH_ALEN];
-		__le16 seq_ctrl; /* filled by firmware */
-	);
-	u8 addr4[ETH_ALEN];
-	__le16 data_len;
-
-	/* 802.3 */
-	u8 dst_addr[ETH_ALEN];
-	u8 src_addr[ETH_ALEN];
-	__be16 len;
-
-	/* followed by frame data; max 2304 bytes */
-} __packed;
-
-
-struct hfa384x_rid_hdr
-{
-	__le16 len;
-	__le16 rid;
-} __packed;
-
-
-/* Macro for converting signal levels (range 27 .. 154) to wireless ext
- * dBm value with some accuracy */
-#define HFA384X_LEVEL_TO_dBm(v) 0x100 + (v) * 100 / 255 - 100
-
-#define HFA384X_LEVEL_TO_dBm_sign(v) (v) * 100 / 255 - 100
-
-struct hfa384x_scan_request {
-	__le16 channel_list;
-	__le16 txrate; /* HFA384X_RATES_* */
-} __packed;
-
-struct hfa384x_hostscan_request {
-	__le16 channel_list;
-	__le16 txrate;
-	__le16 target_ssid_len;
-	u8 target_ssid[32];
-} __packed;
-
-struct hfa384x_join_request {
-	u8 bssid[ETH_ALEN];
-	__le16 channel;
-} __packed;
-
-struct hfa384x_info_frame {
-	__le16 len;
-	__le16 type;
-} __packed;
-
-struct hfa384x_comm_tallies {
-	__le16 tx_unicast_frames;
-	__le16 tx_multicast_frames;
-	__le16 tx_fragments;
-	__le16 tx_unicast_octets;
-	__le16 tx_multicast_octets;
-	__le16 tx_deferred_transmissions;
-	__le16 tx_single_retry_frames;
-	__le16 tx_multiple_retry_frames;
-	__le16 tx_retry_limit_exceeded;
-	__le16 tx_discards;
-	__le16 rx_unicast_frames;
-	__le16 rx_multicast_frames;
-	__le16 rx_fragments;
-	__le16 rx_unicast_octets;
-	__le16 rx_multicast_octets;
-	__le16 rx_fcs_errors;
-	__le16 rx_discards_no_buffer;
-	__le16 tx_discards_wrong_sa;
-	__le16 rx_discards_wep_undecryptable;
-	__le16 rx_message_in_msg_fragments;
-	__le16 rx_message_in_bad_msg_fragments;
-} __packed;
-
-struct hfa384x_comm_tallies32 {
-	__le32 tx_unicast_frames;
-	__le32 tx_multicast_frames;
-	__le32 tx_fragments;
-	__le32 tx_unicast_octets;
-	__le32 tx_multicast_octets;
-	__le32 tx_deferred_transmissions;
-	__le32 tx_single_retry_frames;
-	__le32 tx_multiple_retry_frames;
-	__le32 tx_retry_limit_exceeded;
-	__le32 tx_discards;
-	__le32 rx_unicast_frames;
-	__le32 rx_multicast_frames;
-	__le32 rx_fragments;
-	__le32 rx_unicast_octets;
-	__le32 rx_multicast_octets;
-	__le32 rx_fcs_errors;
-	__le32 rx_discards_no_buffer;
-	__le32 tx_discards_wrong_sa;
-	__le32 rx_discards_wep_undecryptable;
-	__le32 rx_message_in_msg_fragments;
-	__le32 rx_message_in_bad_msg_fragments;
-} __packed;
-
-struct hfa384x_scan_result_hdr {
-	__le16 reserved;
-	__le16 scan_reason;
-#define HFA384X_SCAN_IN_PROGRESS 0 /* no results available yet */
-#define HFA384X_SCAN_HOST_INITIATED 1
-#define HFA384X_SCAN_FIRMWARE_INITIATED 2
-#define HFA384X_SCAN_INQUIRY_FROM_HOST 3
-} __packed;
-
-#define HFA384X_SCAN_MAX_RESULTS 32
-
-struct hfa384x_scan_result {
-	__le16 chid;
-	__le16 anl;
-	__le16 sl;
-	u8 bssid[ETH_ALEN];
-	__le16 beacon_interval;
-	__le16 capability;
-	__le16 ssid_len;
-	u8 ssid[32];
-	u8 sup_rates[10];
-	__le16 rate;
-} __packed;
-
-struct hfa384x_hostscan_result {
-	__le16 chid;
-	__le16 anl;
-	__le16 sl;
-	u8 bssid[ETH_ALEN];
-	__le16 beacon_interval;
-	__le16 capability;
-	__le16 ssid_len;
-	u8 ssid[32];
-	u8 sup_rates[10];
-	__le16 rate;
-	__le16 atim;
-} __packed;
-
-struct comm_tallies_sums {
-	unsigned int tx_unicast_frames;
-	unsigned int tx_multicast_frames;
-	unsigned int tx_fragments;
-	unsigned int tx_unicast_octets;
-	unsigned int tx_multicast_octets;
-	unsigned int tx_deferred_transmissions;
-	unsigned int tx_single_retry_frames;
-	unsigned int tx_multiple_retry_frames;
-	unsigned int tx_retry_limit_exceeded;
-	unsigned int tx_discards;
-	unsigned int rx_unicast_frames;
-	unsigned int rx_multicast_frames;
-	unsigned int rx_fragments;
-	unsigned int rx_unicast_octets;
-	unsigned int rx_multicast_octets;
-	unsigned int rx_fcs_errors;
-	unsigned int rx_discards_no_buffer;
-	unsigned int tx_discards_wrong_sa;
-	unsigned int rx_discards_wep_undecryptable;
-	unsigned int rx_message_in_msg_fragments;
-	unsigned int rx_message_in_bad_msg_fragments;
-};
-
-
-struct hfa384x_regs {
-	u16 cmd;
-	u16 evstat;
-	u16 offset0;
-	u16 offset1;
-	u16 swsupport0;
-};
-
-
-#if defined(PRISM2_PCCARD) || defined(PRISM2_PLX)
-/* I/O ports for HFA384X Controller access */
-#define HFA384X_CMD_OFF 0x00
-#define HFA384X_PARAM0_OFF 0x02
-#define HFA384X_PARAM1_OFF 0x04
-#define HFA384X_PARAM2_OFF 0x06
-#define HFA384X_STATUS_OFF 0x08
-#define HFA384X_RESP0_OFF 0x0A
-#define HFA384X_RESP1_OFF 0x0C
-#define HFA384X_RESP2_OFF 0x0E
-#define HFA384X_INFOFID_OFF 0x10
-#define HFA384X_CONTROL_OFF 0x14
-#define HFA384X_SELECT0_OFF 0x18
-#define HFA384X_SELECT1_OFF 0x1A
-#define HFA384X_OFFSET0_OFF 0x1C
-#define HFA384X_OFFSET1_OFF 0x1E
-#define HFA384X_RXFID_OFF 0x20
-#define HFA384X_ALLOCFID_OFF 0x22
-#define HFA384X_TXCOMPLFID_OFF 0x24
-#define HFA384X_SWSUPPORT0_OFF 0x28
-#define HFA384X_SWSUPPORT1_OFF 0x2A
-#define HFA384X_SWSUPPORT2_OFF 0x2C
-#define HFA384X_EVSTAT_OFF 0x30
-#define HFA384X_INTEN_OFF 0x32
-#define HFA384X_EVACK_OFF 0x34
-#define HFA384X_DATA0_OFF 0x36
-#define HFA384X_DATA1_OFF 0x38
-#define HFA384X_AUXPAGE_OFF 0x3A
-#define HFA384X_AUXOFFSET_OFF 0x3C
-#define HFA384X_AUXDATA_OFF 0x3E
-#endif /* PRISM2_PCCARD || PRISM2_PLX */
-
-#ifdef PRISM2_PCI
-/* Memory addresses for ISL3874 controller access */
-#define HFA384X_CMD_OFF 0x00
-#define HFA384X_PARAM0_OFF 0x04
-#define HFA384X_PARAM1_OFF 0x08
-#define HFA384X_PARAM2_OFF 0x0C
-#define HFA384X_STATUS_OFF 0x10
-#define HFA384X_RESP0_OFF 0x14
-#define HFA384X_RESP1_OFF 0x18
-#define HFA384X_RESP2_OFF 0x1C
-#define HFA384X_INFOFID_OFF 0x20
-#define HFA384X_CONTROL_OFF 0x28
-#define HFA384X_SELECT0_OFF 0x30
-#define HFA384X_SELECT1_OFF 0x34
-#define HFA384X_OFFSET0_OFF 0x38
-#define HFA384X_OFFSET1_OFF 0x3C
-#define HFA384X_RXFID_OFF 0x40
-#define HFA384X_ALLOCFID_OFF 0x44
-#define HFA384X_TXCOMPLFID_OFF 0x48
-#define HFA384X_PCICOR_OFF 0x4C
-#define HFA384X_SWSUPPORT0_OFF 0x50
-#define HFA384X_SWSUPPORT1_OFF 0x54
-#define HFA384X_SWSUPPORT2_OFF 0x58
-#define HFA384X_PCIHCR_OFF 0x5C
-#define HFA384X_EVSTAT_OFF 0x60
-#define HFA384X_INTEN_OFF 0x64
-#define HFA384X_EVACK_OFF 0x68
-#define HFA384X_DATA0_OFF 0x6C
-#define HFA384X_DATA1_OFF 0x70
-#define HFA384X_AUXPAGE_OFF 0x74
-#define HFA384X_AUXOFFSET_OFF 0x78
-#define HFA384X_AUXDATA_OFF 0x7C
-#define HFA384X_PCI_M0_ADDRH_OFF 0x80
-#define HFA384X_PCI_M0_ADDRL_OFF 0x84
-#define HFA384X_PCI_M0_LEN_OFF 0x88
-#define HFA384X_PCI_M0_CTL_OFF 0x8C
-#define HFA384X_PCI_STATUS_OFF 0x98
-#define HFA384X_PCI_M1_ADDRH_OFF 0xA0
-#define HFA384X_PCI_M1_ADDRL_OFF 0xA4
-#define HFA384X_PCI_M1_LEN_OFF 0xA8
-#define HFA384X_PCI_M1_CTL_OFF 0xAC
-
-/* PCI bus master control bits (these are undocumented; based on guessing and
- * experimenting..) */
-#define HFA384X_PCI_CTL_FROM_BAP (BIT(5) | BIT(1) | BIT(0))
-#define HFA384X_PCI_CTL_TO_BAP (BIT(5) | BIT(0))
-
-#endif /* PRISM2_PCI */
-
-
-/* Command codes for CMD reg. */
-#define HFA384X_CMDCODE_INIT 0x00
-#define HFA384X_CMDCODE_ENABLE 0x01
-#define HFA384X_CMDCODE_DISABLE 0x02
-#define HFA384X_CMDCODE_ALLOC 0x0A
-#define HFA384X_CMDCODE_TRANSMIT 0x0B
-#define HFA384X_CMDCODE_INQUIRE 0x11
-#define HFA384X_CMDCODE_ACCESS 0x21
-#define HFA384X_CMDCODE_ACCESS_WRITE (0x21 | BIT(8))
-#define HFA384X_CMDCODE_DOWNLOAD 0x22
-#define HFA384X_CMDCODE_READMIF 0x30
-#define HFA384X_CMDCODE_WRITEMIF 0x31
-#define HFA384X_CMDCODE_TEST 0x38
-
-#define HFA384X_CMDCODE_MASK 0x3F
-
-/* Test mode operations */
-#define HFA384X_TEST_CHANGE_CHANNEL 0x08
-#define HFA384X_TEST_MONITOR 0x0B
-#define HFA384X_TEST_STOP 0x0F
-#define HFA384X_TEST_CFG_BITS 0x15
-#define HFA384X_TEST_CFG_BIT_ALC BIT(3)
-
-#define HFA384X_CMD_BUSY BIT(15)
-
-#define HFA384X_CMD_TX_RECLAIM BIT(8)
-
-#define HFA384X_OFFSET_ERR BIT(14)
-#define HFA384X_OFFSET_BUSY BIT(15)
-
-
-/* ProgMode for download command */
-#define HFA384X_PROGMODE_DISABLE 0
-#define HFA384X_PROGMODE_ENABLE_VOLATILE 1
-#define HFA384X_PROGMODE_ENABLE_NON_VOLATILE 2
-#define HFA384X_PROGMODE_PROGRAM_NON_VOLATILE 3
-
-#define HFA384X_AUX_MAGIC0 0xfe01
-#define HFA384X_AUX_MAGIC1 0xdc23
-#define HFA384X_AUX_MAGIC2 0xba45
-
-#define HFA384X_AUX_PORT_DISABLED 0
-#define HFA384X_AUX_PORT_DISABLE BIT(14)
-#define HFA384X_AUX_PORT_ENABLE BIT(15)
-#define HFA384X_AUX_PORT_ENABLED (BIT(14) | BIT(15))
-#define HFA384X_AUX_PORT_MASK (BIT(14) | BIT(15))
-
-#define PRISM2_PDA_SIZE 1024
-
-
-/* Events; EvStat, Interrupt mask (IntEn), and acknowledge bits (EvAck) */
-#define HFA384X_EV_TICK BIT(15)
-#define HFA384X_EV_WTERR BIT(14)
-#define HFA384X_EV_INFDROP BIT(13)
-#ifdef PRISM2_PCI
-#define HFA384X_EV_PCI_M1 BIT(9)
-#define HFA384X_EV_PCI_M0 BIT(8)
-#endif /* PRISM2_PCI */
-#define HFA384X_EV_INFO BIT(7)
-#define HFA384X_EV_DTIM BIT(5)
-#define HFA384X_EV_CMD BIT(4)
-#define HFA384X_EV_ALLOC BIT(3)
-#define HFA384X_EV_TXEXC BIT(2)
-#define HFA384X_EV_TX BIT(1)
-#define HFA384X_EV_RX BIT(0)
-
-
-/* HFA384X Information frames */
-#define HFA384X_INFO_HANDOVERADDR 0xF000 /* AP f/w ? */
-#define HFA384X_INFO_HANDOVERDEAUTHADDR 0xF001 /* AP f/w 1.3.7 */
-#define HFA384X_INFO_COMMTALLIES 0xF100
-#define HFA384X_INFO_SCANRESULTS 0xF101
-#define HFA384X_INFO_CHANNELINFORESULTS 0xF102 /* AP f/w only */
-#define HFA384X_INFO_HOSTSCANRESULTS 0xF103
-#define HFA384X_INFO_LINKSTATUS 0xF200
-#define HFA384X_INFO_ASSOCSTATUS 0xF201 /* ? */
-#define HFA384X_INFO_AUTHREQ 0xF202 /* ? */
-#define HFA384X_INFO_PSUSERCNT 0xF203 /* ? */
-#define HFA384X_INFO_KEYIDCHANGED 0xF204 /* ? */
-
-enum { HFA384X_LINKSTATUS_CONNECTED = 1,
-       HFA384X_LINKSTATUS_DISCONNECTED = 2,
-       HFA384X_LINKSTATUS_AP_CHANGE = 3,
-       HFA384X_LINKSTATUS_AP_OUT_OF_RANGE = 4,
-       HFA384X_LINKSTATUS_AP_IN_RANGE = 5,
-       HFA384X_LINKSTATUS_ASSOC_FAILED = 6 };
-
-enum { HFA384X_PORTTYPE_BSS = 1, HFA384X_PORTTYPE_WDS = 2,
-       HFA384X_PORTTYPE_PSEUDO_IBSS = 3, HFA384X_PORTTYPE_IBSS = 0,
-       HFA384X_PORTTYPE_HOSTAP = 6 };
-
-#define HFA384X_RATES_1MBPS BIT(0)
-#define HFA384X_RATES_2MBPS BIT(1)
-#define HFA384X_RATES_5MBPS BIT(2)
-#define HFA384X_RATES_11MBPS BIT(3)
-
-#define HFA384X_ROAMING_FIRMWARE 1
-#define HFA384X_ROAMING_HOST 2
-#define HFA384X_ROAMING_DISABLED 3
-
-#define HFA384X_WEPFLAGS_PRIVACYINVOKED BIT(0)
-#define HFA384X_WEPFLAGS_EXCLUDEUNENCRYPTED BIT(1)
-#define HFA384X_WEPFLAGS_HOSTENCRYPT BIT(4)
-#define HFA384X_WEPFLAGS_HOSTDECRYPT BIT(7)
-
-#define HFA384X_RX_STATUS_MSGTYPE (BIT(15) | BIT(14) | BIT(13))
-#define HFA384X_RX_STATUS_PCF BIT(12)
-#define HFA384X_RX_STATUS_MACPORT (BIT(10) | BIT(9) | BIT(8))
-#define HFA384X_RX_STATUS_UNDECR BIT(1)
-#define HFA384X_RX_STATUS_FCSERR BIT(0)
-
-#define HFA384X_RX_STATUS_GET_MSGTYPE(s) \
-(((s) & HFA384X_RX_STATUS_MSGTYPE) >> 13)
-#define HFA384X_RX_STATUS_GET_MACPORT(s) \
-(((s) & HFA384X_RX_STATUS_MACPORT) >> 8)
-
-enum { HFA384X_RX_MSGTYPE_NORMAL = 0, HFA384X_RX_MSGTYPE_RFC1042 = 1,
-       HFA384X_RX_MSGTYPE_BRIDGETUNNEL = 2, HFA384X_RX_MSGTYPE_MGMT = 4 };
-
-
-#define HFA384X_TX_CTRL_ALT_RTRY BIT(5)
-#define HFA384X_TX_CTRL_802_11 BIT(3)
-#define HFA384X_TX_CTRL_802_3 0
-#define HFA384X_TX_CTRL_TX_EX BIT(2)
-#define HFA384X_TX_CTRL_TX_OK BIT(1)
-
-#define HFA384X_TX_STATUS_RETRYERR BIT(0)
-#define HFA384X_TX_STATUS_AGEDERR BIT(1)
-#define HFA384X_TX_STATUS_DISCON BIT(2)
-#define HFA384X_TX_STATUS_FORMERR BIT(3)
-
-/* HFA3861/3863 (BBP) Control Registers */
-#define HFA386X_CR_TX_CONFIGURE 0x12 /* CR9 */
-#define HFA386X_CR_RX_CONFIGURE 0x14 /* CR10 */
-#define HFA386X_CR_A_D_TEST_MODES2 0x1A /* CR13 */
-#define HFA386X_CR_MANUAL_TX_POWER 0x3E /* CR31 */
-#define HFA386X_CR_MEASURED_TX_POWER 0x74 /* CR58 */
-
-
-#ifdef __KERNEL__
-
-#define PRISM2_TXFID_COUNT 8
-#define PRISM2_DATA_MAXLEN 2304
-#define PRISM2_TXFID_LEN (PRISM2_DATA_MAXLEN + sizeof(struct hfa384x_tx_frame))
-#define PRISM2_TXFID_EMPTY 0xffff
-#define PRISM2_TXFID_RESERVED 0xfffe
-#define PRISM2_DUMMY_FID 0xffff
-#define MAX_SSID_LEN 32
-#define MAX_NAME_LEN 32 /* this is assumed to be equal to MAX_SSID_LEN */
-
-#define PRISM2_DUMP_RX_HDR BIT(0)
-#define PRISM2_DUMP_TX_HDR BIT(1)
-#define PRISM2_DUMP_TXEXC_HDR BIT(2)
-
-struct hostap_tx_callback_info {
-	u16 idx;
-	void (*func)(struct sk_buff *, int ok, void *);
-	void *data;
-	struct hostap_tx_callback_info *next;
-};
-
-
-/* IEEE 802.11 requires that STA supports concurrent reception of at least
- * three fragmented frames. This define can be increased to support more
- * concurrent frames, but it should be noted that each entry can consume about
- * 2 kB of RAM and increasing cache size will slow down frame reassembly. */
-#define PRISM2_FRAG_CACHE_LEN 4
-
-struct prism2_frag_entry {
-	unsigned long first_frag_time;
-	unsigned int seq;
-	unsigned int last_frag;
-	struct sk_buff *skb;
-	u8 src_addr[ETH_ALEN];
-	u8 dst_addr[ETH_ALEN];
-};
-
-
-struct hostap_cmd_queue {
-	struct list_head list;
-	wait_queue_head_t compl;
-	volatile enum { CMD_SLEEP, CMD_CALLBACK, CMD_COMPLETED } type;
-	void (*callback)(struct net_device *dev, long context, u16 resp0,
-			 u16 res);
-	long context;
-	u16 cmd, param0, param1;
-	u16 resp0, res;
-	volatile int issued, issuing;
-
-	refcount_t usecnt;
-	int del_req;
-};
-
-/* options for hw_shutdown */
-#define HOSTAP_HW_NO_DISABLE BIT(0)
-#define HOSTAP_HW_ENABLE_CMDCOMPL BIT(1)
-
-typedef struct local_info local_info_t;
-
-struct prism2_helper_functions {
-	/* these functions are defined in hardware model specific files
-	 * (hostap_{cs,plx,pci}.c */
-	int (*card_present)(local_info_t *local);
-	void (*cor_sreset)(local_info_t *local);
-	void (*genesis_reset)(local_info_t *local, int hcr);
-
-	/* the following functions are from hostap_hw.c, but they may have some
-	 * hardware model specific code */
-
-	/* FIX: low-level commands like cmd might disappear at some point to
-	 * make it easier to change them if needed (e.g., cmd would be replaced
-	 * with write_mif/read_mif/testcmd/inquire); at least get_rid and
-	 * set_rid might move to hostap_{cs,plx,pci}.c */
-	int (*cmd)(struct net_device *dev, u16 cmd, u16 param0, u16 *param1,
-		   u16 *resp0);
-	void (*read_regs)(struct net_device *dev, struct hfa384x_regs *regs);
-	int (*get_rid)(struct net_device *dev, u16 rid, void *buf, int len,
-		       int exact_len);
-	int (*set_rid)(struct net_device *dev, u16 rid, void *buf, int len);
-	int (*hw_enable)(struct net_device *dev, int initial);
-	int (*hw_config)(struct net_device *dev, int initial);
-	void (*hw_reset)(struct net_device *dev);
-	void (*hw_shutdown)(struct net_device *dev, int no_disable);
-	int (*reset_port)(struct net_device *dev);
-	void (*schedule_reset)(local_info_t *local);
-	int (*download)(local_info_t *local,
-			struct prism2_download_param *param);
-	int (*tx)(struct sk_buff *skb, struct net_device *dev);
-	int (*set_tim)(struct net_device *dev, int aid, int set);
-	const struct proc_ops *read_aux_proc_ops;
-
-	int need_tx_headroom; /* number of bytes of headroom needed before
-			       * IEEE 802.11 header */
-	enum { HOSTAP_HW_PCCARD, HOSTAP_HW_PLX, HOSTAP_HW_PCI } hw_type;
-};
-
-
-struct prism2_download_data {
-	u32 dl_cmd;
-	u32 start_addr;
-	u32 num_areas;
-	struct prism2_download_data_area {
-		u32 addr; /* wlan card address */
-		u32 len;
-		u8 *data; /* allocated data */
-	} data[] __counted_by(num_areas);
-};
-
-
-#define HOSTAP_MAX_BSS_COUNT 64
-#define MAX_WPA_IE_LEN 64
-
-struct hostap_bss_info {
-	struct list_head list;
-	unsigned long last_update;
-	unsigned int count;
-	u8 bssid[ETH_ALEN];
-	u16 capab_info;
-	u8 ssid[32];
-	size_t ssid_len;
-	u8 wpa_ie[MAX_WPA_IE_LEN];
-	size_t wpa_ie_len;
-	u8 rsn_ie[MAX_WPA_IE_LEN];
-	size_t rsn_ie_len;
-	int chan;
-	int included;
-};
-
-
-/* Per radio private Host AP data - shared by all net devices interfaces used
- * by each radio (wlan#, wlan#ap, wlan#sta, WDS).
- * ((struct hostap_interface *) netdev_priv(dev))->local points to this
- * structure. */
-struct local_info {
-	struct module *hw_module;
-	int card_idx;
-	int dev_enabled;
-	int master_dev_auto_open; /* was master device opened automatically */
-	int num_dev_open; /* number of open devices */
-	struct net_device *dev; /* master radio device */
-	struct net_device *ddev; /* main data device */
-	struct list_head hostap_interfaces; /* Host AP interface list (contains
-					     * struct hostap_interface entries)
-					     */
-	rwlock_t iface_lock; /* hostap_interfaces read lock; use write lock
-			      * when removing entries from the list.
-			      * TX and RX paths can use read lock. */
-	spinlock_t cmdlock, baplock, lock, irq_init_lock;
-	struct mutex rid_bap_mtx;
-	u16 infofid; /* MAC buffer id for info frame */
-	/* txfid, intransmitfid, next_txtid, and next_alloc are protected by
-	 * txfidlock */
-	spinlock_t txfidlock;
-	int txfid_len; /* length of allocated TX buffers */
-	u16 txfid[PRISM2_TXFID_COUNT]; /* buffer IDs for TX frames */
-	/* buffer IDs for intransmit frames or PRISM2_TXFID_EMPTY if
-	 * corresponding txfid is free for next TX frame */
-	u16 intransmitfid[PRISM2_TXFID_COUNT];
-	int next_txfid; /* index to the next txfid to be checked for
-			 * availability */
-	int next_alloc; /* index to the next intransmitfid to be checked for
-			 * allocation events */
-
-	/* bitfield for atomic bitops */
-#define HOSTAP_BITS_TRANSMIT 0
-#define HOSTAP_BITS_BAP_TASKLET 1
-#define HOSTAP_BITS_BAP_TASKLET2 2
-	unsigned long bits;
-
-	struct ap_data *ap;
-
-	char essid[MAX_SSID_LEN + 1];
-	char name[MAX_NAME_LEN + 1];
-	int name_set;
-	u16 channel_mask; /* mask of allowed channels */
-	u16 scan_channel_mask; /* mask of channels to be scanned */
-	struct comm_tallies_sums comm_tallies;
-	struct proc_dir_entry *proc;
-	int iw_mode; /* operating mode (IW_MODE_*) */
-	int pseudo_adhoc; /* 0: IW_MODE_ADHOC is real 802.11 compliant IBSS
-			   * 1: IW_MODE_ADHOC is "pseudo IBSS" */
-	char bssid[ETH_ALEN];
-	int channel;
-	int beacon_int;
-	int dtim_period;
-	int mtu;
-	int frame_dump; /* dump RX/TX frame headers, PRISM2_DUMP_ flags */
-	int fw_tx_rate_control;
-	u16 tx_rate_control;
-	u16 basic_rates;
-	int hw_resetting;
-	int hw_ready;
-	int hw_reset_tries; /* how many times reset has been tried */
-	int hw_downloading;
-	int shutdown;
-	int pri_only;
-	int no_pri; /* no PRI f/w present */
-	int sram_type; /* 8 = x8 SRAM, 16 = x16 SRAM, -1 = unknown */
-
-	enum {
-		PRISM2_TXPOWER_AUTO = 0, PRISM2_TXPOWER_OFF,
-		PRISM2_TXPOWER_FIXED, PRISM2_TXPOWER_UNKNOWN
-	} txpower_type;
-	int txpower; /* if txpower_type == PRISM2_TXPOWER_FIXED */
-
-	/* command queue for hfa384x_cmd(); protected with cmdlock */
-	struct list_head cmd_queue;
-	/* max_len for cmd_queue; in addition, cmd_callback can use two
-	 * additional entries to prevent sleeping commands from stopping
-	 * transmits */
-#define HOSTAP_CMD_QUEUE_MAX_LEN 16
-	int cmd_queue_len; /* number of entries in cmd_queue */
-
-	/* if card timeout is detected in interrupt context, reset_queue is
-	 * used to schedule card reseting to be done in user context */
-	struct work_struct reset_queue;
-
-	/* For scheduling a change of the promiscuous mode RID */
-	int is_promisc;
-	struct work_struct set_multicast_list_queue;
-
-	struct work_struct set_tim_queue;
-	struct list_head set_tim_list;
-	spinlock_t set_tim_lock;
-
-	int wds_max_connections;
-	int wds_connections;
-#define HOSTAP_WDS_BROADCAST_RA BIT(0)
-#define HOSTAP_WDS_AP_CLIENT BIT(1)
-#define HOSTAP_WDS_STANDARD_FRAME BIT(2)
-	u32 wds_type;
-	u16 tx_control; /* flags to be used in TX description */
-	int manual_retry_count; /* -1 = use f/w default; otherwise retry count
-				 * to be used with all frames */
-
-	struct iw_statistics wstats;
-	unsigned long scan_timestamp; /* Time started to scan */
-	enum {
-		PRISM2_MONITOR_80211 = 0, PRISM2_MONITOR_PRISM = 1,
-		PRISM2_MONITOR_CAPHDR = 2, PRISM2_MONITOR_RADIOTAP = 3
-	} monitor_type;
-	int monitor_allow_fcserr;
-
-	int hostapd; /* whether user space daemon, hostapd, is used for AP
-		      * management */
-	int hostapd_sta; /* whether hostapd is used with an extra STA interface
-			  */
-	struct net_device *apdev;
-	struct net_device_stats apdevstats;
-
-	char assoc_ap_addr[ETH_ALEN];
-	struct net_device *stadev;
-	struct net_device_stats stadevstats;
-
-#define WEP_KEYS 4
-#define WEP_KEY_LEN 13
-	struct lib80211_crypt_info crypt_info;
-
-	int open_wep; /* allow unencrypted frames */
-	int host_encrypt;
-	int host_decrypt;
-	int privacy_invoked; /* force privacy invoked flag even if no keys are
-			      * configured */
-	int fw_encrypt_ok; /* whether firmware-based WEP encrypt is working
-			    * in Host AP mode (STA f/w 1.4.9 or newer) */
-	int bcrx_sta_key; /* use individual keys to override default keys even
-			   * with RX of broad/multicast frames */
-
-	struct prism2_frag_entry frag_cache[PRISM2_FRAG_CACHE_LEN];
-	unsigned int frag_next_idx;
-
-	int ieee_802_1x; /* is IEEE 802.1X used */
-
-	int antsel_tx, antsel_rx;
-	int rts_threshold; /* dot11RTSThreshold */
-	int fragm_threshold; /* dot11FragmentationThreshold */
-	int auth_algs; /* PRISM2_AUTH_ flags */
-
-	int enh_sec; /* cnfEnhSecurity options (broadcast SSID hide/ignore) */
-	int tallies32; /* 32-bit tallies in use */
-
-	struct prism2_helper_functions *func;
-
-	u8 *pda;
-	int fw_ap;
-#define PRISM2_FW_VER(major, minor, variant) \
-(((major) << 16) | ((minor) << 8) | variant)
-	u32 sta_fw_ver;
-
-	/* Tasklets for handling hardware IRQ related operations outside hw IRQ
-	 * handler */
-	struct tasklet_struct bap_tasklet;
-
-	struct tasklet_struct info_tasklet;
-	struct sk_buff_head info_list; /* info frames as skb's for
-					* info_tasklet */
-
-	struct hostap_tx_callback_info *tx_callback; /* registered TX callbacks
-						      */
-
-	struct tasklet_struct rx_tasklet;
-	struct sk_buff_head rx_list;
-
-	struct tasklet_struct sta_tx_exc_tasklet;
-	struct sk_buff_head sta_tx_exc_list;
-
-	int host_roaming;
-	unsigned long last_join_time; /* time of last JoinRequest */
-	struct hfa384x_hostscan_result *last_scan_results;
-	int last_scan_results_count;
-	enum { PRISM2_SCAN, PRISM2_HOSTSCAN } last_scan_type;
-	struct work_struct info_queue;
-	unsigned long pending_info; /* bit field of pending info_queue items */
-#define PRISM2_INFO_PENDING_LINKSTATUS 0
-#define PRISM2_INFO_PENDING_SCANRESULTS 1
-	int prev_link_status; /* previous received LinkStatus info */
-	int prev_linkstatus_connected;
-	u8 preferred_ap[ETH_ALEN]; /* use this AP if possible */
-
-#ifdef PRISM2_CALLBACK
-	void *callback_data; /* Can be used in callbacks; e.g., allocate
-			      * on enable event and free on disable event.
-			      * Host AP driver code does not touch this. */
-#endif /* PRISM2_CALLBACK */
-
-	wait_queue_head_t hostscan_wq;
-
-	/* Passive scan in Host AP mode */
-	struct timer_list passive_scan_timer;
-	int passive_scan_interval; /* in seconds, 0 = disabled */
-	int passive_scan_channel;
-	enum { PASSIVE_SCAN_WAIT, PASSIVE_SCAN_LISTEN } passive_scan_state;
-
-	struct timer_list tick_timer;
-	unsigned long last_tick_timer;
-	unsigned int sw_tick_stuck;
-
-	/* commsQuality / dBmCommsQuality data from periodic polling; only
-	 * valid for Managed and Ad-hoc modes */
-	unsigned long last_comms_qual_update;
-	int comms_qual; /* in some odd unit.. */
-	int avg_signal; /* in dB (note: negative) */
-	int avg_noise; /* in dB (note: negative) */
-	struct work_struct comms_qual_update;
-
-	/* RSSI to dBm adjustment (for RX descriptor fields) */
-	int rssi_to_dBm; /* subtract from RSSI to get approximate dBm value */
-
-	/* BSS list / protected by local->lock */
-	struct list_head bss_list;
-	int num_bss_info;
-	int wpa; /* WPA support enabled */
-	int tkip_countermeasures;
-	int drop_unencrypted;
-	/* Generic IEEE 802.11 info element to be added to
-	 * ProbeResp/Beacon/(Re)AssocReq */
-	u8 *generic_elem;
-	size_t generic_elem_len;
-
-#ifdef PRISM2_DOWNLOAD_SUPPORT
-	/* Persistent volatile download data */
-	struct prism2_download_data *dl_pri;
-	struct prism2_download_data *dl_sec;
-#endif /* PRISM2_DOWNLOAD_SUPPORT */
-
-#ifdef PRISM2_IO_DEBUG
-#define PRISM2_IO_DEBUG_SIZE 10000
-	u32 io_debug[PRISM2_IO_DEBUG_SIZE];
-	int io_debug_head;
-	int io_debug_enabled;
-#endif /* PRISM2_IO_DEBUG */
-
-	/* Pointer to hardware model specific (cs,pci,plx) private data. */
-	void *hw_priv;
-};
-
-
-/* Per interface private Host AP data
- * Allocated for each net device that Host AP uses (wlan#, wlan#ap, wlan#sta,
- * WDS) and netdev_priv(dev) points to this structure. */
-struct hostap_interface {
-	struct list_head list; /* list entry in Host AP interface list */
-	struct net_device *dev; /* pointer to this device */
-	struct local_info *local; /* pointer to shared private data */
-	struct net_device_stats stats;
-	struct iw_spy_data spy_data; /* iwspy support */
-	struct iw_public_data wireless_data;
-
-	enum {
-		HOSTAP_INTERFACE_MASTER,
-		HOSTAP_INTERFACE_MAIN,
-		HOSTAP_INTERFACE_AP,
-		HOSTAP_INTERFACE_STA,
-		HOSTAP_INTERFACE_WDS,
-	} type;
-
-	union {
-		struct hostap_interface_wds {
-			u8 remote_addr[ETH_ALEN];
-		} wds;
-	} u;
-};
-
-
-#define HOSTAP_SKB_TX_DATA_MAGIC 0xf08a36a2
-
-/*
- * TX meta data - stored in skb->cb buffer, so this must not be increased over
- * the 48-byte limit.
- * THE PADDING THIS STARTS WITH IS A HORRIBLE HACK THAT SHOULD NOT LIVE
- * TO SEE THE DAY.
- */
-struct hostap_skb_tx_data {
-	unsigned int __padding_for_default_qdiscs;
-	u32 magic; /* HOSTAP_SKB_TX_DATA_MAGIC */
-	u8 rate; /* transmit rate */
-#define HOSTAP_TX_FLAGS_WDS BIT(0)
-#define HOSTAP_TX_FLAGS_BUFFERED_FRAME BIT(1)
-#define HOSTAP_TX_FLAGS_ADD_MOREDATA BIT(2)
-	u8 flags; /* HOSTAP_TX_FLAGS_* */
-	u16 tx_cb_idx;
-	struct hostap_interface *iface;
-	unsigned long jiffies; /* queueing timestamp */
-	unsigned short ethertype;
-};
-
-
-#ifndef PRISM2_NO_DEBUG
-
-#define DEBUG_FID BIT(0)
-#define DEBUG_PS BIT(1)
-#define DEBUG_FLOW BIT(2)
-#define DEBUG_AP BIT(3)
-#define DEBUG_HW BIT(4)
-#define DEBUG_EXTRA BIT(5)
-#define DEBUG_EXTRA2 BIT(6)
-#define DEBUG_PS2 BIT(7)
-#define DEBUG_MASK (DEBUG_PS | DEBUG_AP | DEBUG_HW | DEBUG_EXTRA)
-#define PDEBUG(n, args...) \
-do { if ((n) & DEBUG_MASK) printk(KERN_DEBUG args); } while (0)
-#define PDEBUG2(n, args...) \
-do { if ((n) & DEBUG_MASK) printk(args); } while (0)
-
-#else /* PRISM2_NO_DEBUG */
-
-#define PDEBUG(n, args...)
-#define PDEBUG2(n, args...)
-
-#endif /* PRISM2_NO_DEBUG */
-
-enum { BAP0 = 0, BAP1 = 1 };
-
-#define PRISM2_IO_DEBUG_CMD_INB 0
-#define PRISM2_IO_DEBUG_CMD_INW 1
-#define PRISM2_IO_DEBUG_CMD_INSW 2
-#define PRISM2_IO_DEBUG_CMD_OUTB 3
-#define PRISM2_IO_DEBUG_CMD_OUTW 4
-#define PRISM2_IO_DEBUG_CMD_OUTSW 5
-#define PRISM2_IO_DEBUG_CMD_ERROR 6
-#define PRISM2_IO_DEBUG_CMD_INTERRUPT 7
-
-#ifdef PRISM2_IO_DEBUG
-
-#define PRISM2_IO_DEBUG_ENTRY(cmd, reg, value) \
-(((cmd) << 24) | ((reg) << 16) | value)
-
-static inline void prism2_io_debug_add(struct net_device *dev, int cmd,
-				       int reg, int value)
-{
-	struct hostap_interface *iface = netdev_priv(dev);
-	local_info_t *local = iface->local;
-
-	if (!local->io_debug_enabled)
-		return;
-
-	local->io_debug[local->io_debug_head] =	jiffies & 0xffffffff;
-	if (++local->io_debug_head >= PRISM2_IO_DEBUG_SIZE)
-		local->io_debug_head = 0;
-	local->io_debug[local->io_debug_head] =
-		PRISM2_IO_DEBUG_ENTRY(cmd, reg, value);
-	if (++local->io_debug_head >= PRISM2_IO_DEBUG_SIZE)
-		local->io_debug_head = 0;
-}
-
-
-static inline void prism2_io_debug_error(struct net_device *dev, int err)
-{
-	struct hostap_interface *iface = netdev_priv(dev);
-	local_info_t *local = iface->local;
-	unsigned long flags;
-
-	if (!local->io_debug_enabled)
-		return;
-
-	spin_lock_irqsave(&local->lock, flags);
-	prism2_io_debug_add(dev, PRISM2_IO_DEBUG_CMD_ERROR, 0, err);
-	if (local->io_debug_enabled == 1) {
-		local->io_debug_enabled = 0;
-		printk(KERN_DEBUG "%s: I/O debug stopped\n", dev->name);
-	}
-	spin_unlock_irqrestore(&local->lock, flags);
-}
-
-#else /* PRISM2_IO_DEBUG */
-
-static inline void prism2_io_debug_add(struct net_device *dev, int cmd,
-				       int reg, int value)
-{
-}
-
-static inline void prism2_io_debug_error(struct net_device *dev, int err)
-{
-}
-
-#endif /* PRISM2_IO_DEBUG */
-
-
-#ifdef PRISM2_CALLBACK
-enum {
-	/* Called when card is enabled */
-	PRISM2_CALLBACK_ENABLE,
-
-	/* Called when card is disabled */
-	PRISM2_CALLBACK_DISABLE,
-
-	/* Called when RX/TX starts/ends */
-	PRISM2_CALLBACK_RX_START, PRISM2_CALLBACK_RX_END,
-	PRISM2_CALLBACK_TX_START, PRISM2_CALLBACK_TX_END
-};
-void prism2_callback(local_info_t *local, int event);
-#else /* PRISM2_CALLBACK */
-#define prism2_callback(d, e) do { } while (0)
-#endif /* PRISM2_CALLBACK */
-
-#endif /* __KERNEL__ */
-
-#endif /* HOSTAP_WLAN_H */
diff --git a/drivers/net/wireless/intersil/orinoco/Kconfig b/drivers/net/wireless/intersil/orinoco/Kconfig
deleted file mode 100644
index f62730aa7be3..000000000000
--- a/drivers/net/wireless/intersil/orinoco/Kconfig
+++ /dev/null
@@ -1,143 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-config HERMES
-	tristate "Hermes chipset 802.11b support (Orinoco/Prism2/Symbol)"
-	depends on (PPC_PMAC || PCI || PCMCIA)
-	depends on CFG80211
-	select CFG80211_WEXT_EXPORT
-	select WIRELESS_EXT
-	select WEXT_SPY
-	select WEXT_PRIV
-	select FW_LOADER
-	select CRYPTO
-	select CRYPTO_MICHAEL_MIC
-	help
-	  A driver for 802.11b wireless cards based on the "Hermes" or
-	  Intersil HFA384x (Prism 2) MAC controller.  This includes the vast
-	  majority of the PCMCIA 802.11b cards (which are nearly all rebadges)
-	  - except for the Cisco/Aironet cards.  Cards supported include the
-	  Apple Airport (not a PCMCIA card), WavelanIEEE/Orinoco,
-	  Cabletron/EnteraSys Roamabout, ELSA AirLancer, MELCO Buffalo, Avaya,
-	  IBM High Rate Wireless, Farralon Syyline, Samsung MagicLAN, Netgear
-	  MA401, LinkSys WPC-11, D-Link DWL-650, 3Com AirConnect, Intel
-	  IPW2011, and Symbol Spectrum24 High Rate amongst others.
-
-	  This option includes the guts of the driver, but in order to
-	  actually use a card you will also need to enable support for PCMCIA
-	  Hermes cards, PLX9052 based PCI adaptors or the Apple Airport below.
-
-	  You will also very likely also need the Wireless Tools in order to
-	  configure your card and that /etc/pcmcia/wireless.opts works :
-	  <https://www.hpl.hp.com/personal/Jean_Tourrilhes/Linux/Tools.html>
-
-config HERMES_PRISM
-	bool "Support Prism 2/2.5 chipset"
-	depends on HERMES
-	help
-
-	  Say Y to enable support for Prism 2 and 2.5 chipsets.  These
-	  chipsets are better handled by the hostap driver.  This driver
-	  would not support WPA or firmware download for Prism chipset.
-
-	  If you are not sure, say N.
-
-config HERMES_CACHE_FW_ON_INIT
-	bool "Cache Hermes firmware on driver initialisation"
-	depends on HERMES
-	default y
-	help
-	  Say Y to cache any firmware required by the Hermes drivers
-	  on startup.  The firmware will remain cached until the
-	  driver is unloaded.  The cache uses 64K of RAM.
-
-	  Otherwise load the firmware from userspace as required.  In
-	  this case the driver should be unloaded and restarted
-	  whenever the firmware is changed.
-
-	  If you are not sure, say Y.
-
-config APPLE_AIRPORT
-	tristate "Apple Airport support (built-in)"
-	depends on PPC_PMAC && HERMES
-	help
-	  Say Y here to support the Airport 802.11b wireless Ethernet hardware
-	  built into the Macintosh iBook and other recent PowerPC-based
-	  Macintosh machines. This is essentially a Lucent Orinoco card with
-	  a non-standard interface.
-
-	  This driver does not support the Airport Extreme (802.11b/g). Use
-	  the BCM43xx driver for Airport Extreme cards.
-
-config PLX_HERMES
-	tristate "Hermes in PLX9052 based PCI adaptor support (Netgear MA301 etc.)"
-	depends on PCI && HERMES
-	help
-	  Enable support for PCMCIA cards supported by the "Hermes" (aka
-	  orinoco) driver when used in PLX9052 based PCI adaptors.  These
-	  adaptors are not a full PCMCIA controller but act as a more limited
-	  PCI <-> PCMCIA bridge.  Several vendors sell such adaptors so that
-	  802.11b PCMCIA cards can be used in desktop machines.  The Netgear
-	  MA301 is such an adaptor.
-
-config TMD_HERMES
-	tristate "Hermes in TMD7160 based PCI adaptor support"
-	depends on PCI && HERMES
-	help
-	  Enable support for PCMCIA cards supported by the "Hermes" (aka
-	  orinoco) driver when used in TMD7160 based PCI adaptors.  These
-	  adaptors are not a full PCMCIA controller but act as a more limited
-	  PCI <-> PCMCIA bridge.  Several vendors sell such adaptors so that
-	  802.11b PCMCIA cards can be used in desktop machines.
-
-config NORTEL_HERMES
-	tristate "Nortel emobility PCI adaptor support"
-	depends on PCI && HERMES
-	help
-	  Enable support for PCMCIA cards supported by the "Hermes" (aka
-	  orinoco) driver when used in Nortel emobility PCI adaptors.  These
-	  adaptors are not full PCMCIA controllers, but act as a more limited
-	  PCI <-> PCMCIA bridge.
-
-config PCI_HERMES
-	tristate "Prism 2.5 PCI 802.11b adaptor support"
-	depends on PCI && HERMES && HERMES_PRISM
-	help
-	  Enable support for PCI and mini-PCI 802.11b wireless NICs based on
-	  the Prism 2.5 chipset.  These are true PCI cards, not the 802.11b
-	  PCMCIA cards bundled with PCI<->PCMCIA adaptors which are also
-	  common.  Some of the built-in wireless adaptors in laptops are of
-	  this variety.
-
-config PCMCIA_HERMES
-	tristate "Hermes PCMCIA card support"
-	depends on PCMCIA && HERMES && HAS_IOPORT_MAP
-	help
-	  A driver for "Hermes" chipset based PCMCIA wireless adaptors, such
-	  as the Lucent WavelanIEEE/Orinoco cards and their OEM (Cabletron/
-	  EnteraSys RoamAbout 802.11, ELSA Airlancer, Melco Buffalo and
-	  others).  It should also be usable on various Prism II based cards
-	  such as the Linksys, D-Link and Farallon Skyline.  It should also
-	  work on Symbol cards such as the 3Com AirConnect and Ericsson WLAN.
-
-	  You will very likely need the Wireless Tools in order to
-	  configure your card and that /etc/pcmcia/wireless.opts works:
-	  <https://www.hpl.hp.com/personal/Jean_Tourrilhes/Linux/Tools.html>.
-
-config PCMCIA_SPECTRUM
-	tristate "Symbol Spectrum24 Trilogy PCMCIA card support"
-	depends on PCMCIA && HERMES && HAS_IOPORT_MAP
-	help
-
-	  This is a driver for 802.11b cards using RAM-loadable Symbol
-	  firmware, such as Symbol Wireless Networker LA4100, CompactFlash
-	  cards by Socket Communications and Intel PRO/Wireless 2011B.
-
-	  This driver requires firmware download on startup.  Utilities
-	  for downloading Symbol firmware are available at
-	  <http://sourceforge.net/projects/orinoco/>
-
-config ORINOCO_USB
-	tristate "Agere Orinoco USB support"
-	depends on USB && HERMES
-	select FW_LOADER
-	help
-	  This driver is for USB versions of the Agere Orinoco card.
diff --git a/drivers/net/wireless/intersil/orinoco/Makefile b/drivers/net/wireless/intersil/orinoco/Makefile
deleted file mode 100644
index 0c29c56c88d6..000000000000
--- a/drivers/net/wireless/intersil/orinoco/Makefile
+++ /dev/null
@@ -1,15 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-#
-# Makefile for the orinoco wireless device drivers.
-#
-orinoco-objs := main.o fw.o hw.o mic.o scan.o wext.o hermes_dld.o hermes.o cfg.o
-
-obj-$(CONFIG_HERMES)		+= orinoco.o
-obj-$(CONFIG_PCMCIA_HERMES)	+= orinoco_cs.o
-obj-$(CONFIG_APPLE_AIRPORT)	+= airport.o
-obj-$(CONFIG_PLX_HERMES)	+= orinoco_plx.o
-obj-$(CONFIG_PCI_HERMES)	+= orinoco_pci.o
-obj-$(CONFIG_TMD_HERMES)	+= orinoco_tmd.o
-obj-$(CONFIG_NORTEL_HERMES)	+= orinoco_nortel.o
-obj-$(CONFIG_PCMCIA_SPECTRUM)	+= spectrum_cs.o
-obj-$(CONFIG_ORINOCO_USB)	+= orinoco_usb.o
diff --git a/drivers/net/wireless/intersil/orinoco/airport.c b/drivers/net/wireless/intersil/orinoco/airport.c
deleted file mode 100644
index 45ac00fdafa5..000000000000
--- a/drivers/net/wireless/intersil/orinoco/airport.c
+++ /dev/null
@@ -1,268 +0,0 @@
-/* airport.c
- *
- * A driver for "Hermes" chipset based Apple Airport wireless
- * card.
- *
- * Copyright notice & release notes in file main.c
- *
- * Note specific to airport stub:
- *
- *  0.05 : first version of the new split driver
- *  0.06 : fix possible hang on powerup, add sleep support
- */
-
-#define DRIVER_NAME "airport"
-#define PFX DRIVER_NAME ": "
-
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/delay.h>
-#include <linux/mod_devicetable.h>
-#include <asm/pmac_feature.h>
-
-#include "orinoco.h"
-
-#define AIRPORT_IO_LEN	(0x1000)	/* one page */
-
-struct airport {
-	struct macio_dev *mdev;
-	void __iomem *vaddr;
-	unsigned int irq;
-	int irq_requested;
-	int ndev_registered;
-};
-
-static int
-airport_suspend(struct macio_dev *mdev, pm_message_t state)
-{
-	struct orinoco_private *priv = dev_get_drvdata(&mdev->ofdev.dev);
-	struct net_device *dev = priv->ndev;
-	struct airport *card = priv->card;
-	unsigned long flags;
-	int err;
-
-	printk(KERN_DEBUG "%s: Airport entering sleep mode\n", dev->name);
-
-	err = orinoco_lock(priv, &flags);
-	if (err) {
-		printk(KERN_ERR "%s: hw_unavailable on PBOOK_SLEEP_NOW\n",
-		       dev->name);
-		return 0;
-	}
-
-	orinoco_down(priv);
-	orinoco_unlock(priv, &flags);
-
-	disable_irq(card->irq);
-	pmac_call_feature(PMAC_FTR_AIRPORT_ENABLE,
-			  macio_get_of_node(mdev), 0, 0);
-
-	return 0;
-}
-
-static int
-airport_resume(struct macio_dev *mdev)
-{
-	struct orinoco_private *priv = dev_get_drvdata(&mdev->ofdev.dev);
-	struct net_device *dev = priv->ndev;
-	struct airport *card = priv->card;
-	unsigned long flags;
-	int err;
-
-	printk(KERN_DEBUG "%s: Airport waking up\n", dev->name);
-
-	pmac_call_feature(PMAC_FTR_AIRPORT_ENABLE,
-			  macio_get_of_node(mdev), 0, 1);
-	msleep(200);
-
-	enable_irq(card->irq);
-
-	priv->hw.ops->lock_irqsave(&priv->lock, &flags);
-	err = orinoco_up(priv);
-	priv->hw.ops->unlock_irqrestore(&priv->lock, &flags);
-
-	return err;
-}
-
-static int
-airport_detach(struct macio_dev *mdev)
-{
-	struct orinoco_private *priv = dev_get_drvdata(&mdev->ofdev.dev);
-	struct airport *card = priv->card;
-
-	if (card->ndev_registered)
-		orinoco_if_del(priv);
-	card->ndev_registered = 0;
-
-	if (card->irq_requested)
-		free_irq(card->irq, priv);
-	card->irq_requested = 0;
-
-	if (card->vaddr)
-		iounmap(card->vaddr);
-	card->vaddr = NULL;
-
-	macio_release_resource(mdev, 0);
-
-	pmac_call_feature(PMAC_FTR_AIRPORT_ENABLE,
-			  macio_get_of_node(mdev), 0, 0);
-	ssleep(1);
-
-	macio_set_drvdata(mdev, NULL);
-	free_orinocodev(priv);
-
-	return 0;
-}
-
-static int airport_hard_reset(struct orinoco_private *priv)
-{
-	/* It would be nice to power cycle the Airport for a real hard
-	 * reset, but for some reason although it appears to
-	 * re-initialize properly, it falls in a screaming heap
-	 * shortly afterwards. */
-#if 0
-	struct airport *card = priv->card;
-
-	/* Vitally important.  If we don't do this it seems we get an
-	 * interrupt somewhere during the power cycle, since
-	 * hw_unavailable is already set it doesn't get ACKed, we get
-	 * into an interrupt loop and the PMU decides to turn us
-	 * off. */
-	disable_irq(card->irq);
-
-	pmac_call_feature(PMAC_FTR_AIRPORT_ENABLE,
-			  macio_get_of_node(card->mdev), 0, 0);
-	ssleep(1);
-	pmac_call_feature(PMAC_FTR_AIRPORT_ENABLE,
-			  macio_get_of_node(card->mdev), 0, 1);
-	ssleep(1);
-
-	enable_irq(card->irq);
-	ssleep(1);
-#endif
-
-	return 0;
-}
-
-static int
-airport_attach(struct macio_dev *mdev, const struct of_device_id *match)
-{
-	struct orinoco_private *priv;
-	struct airport *card;
-	unsigned long phys_addr;
-	struct hermes *hw;
-
-	if (macio_resource_count(mdev) < 1 || macio_irq_count(mdev) < 1) {
-		printk(KERN_ERR PFX "Wrong interrupt/addresses in OF tree\n");
-		return -ENODEV;
-	}
-
-	/* Allocate space for private device-specific data */
-	priv = alloc_orinocodev(sizeof(*card), &mdev->ofdev.dev,
-				airport_hard_reset, NULL);
-	if (!priv) {
-		printk(KERN_ERR PFX "Cannot allocate network device\n");
-		return -ENODEV;
-	}
-	card = priv->card;
-
-	hw = &priv->hw;
-	card->mdev = mdev;
-
-	if (macio_request_resource(mdev, 0, DRIVER_NAME)) {
-		printk(KERN_ERR PFX "can't request IO resource !\n");
-		free_orinocodev(priv);
-		return -EBUSY;
-	}
-
-	macio_set_drvdata(mdev, priv);
-
-	/* Setup interrupts & base address */
-	card->irq = macio_irq(mdev, 0);
-	phys_addr = macio_resource_start(mdev, 0);  /* Physical address */
-	printk(KERN_DEBUG PFX "Physical address %lx\n", phys_addr);
-	card->vaddr = ioremap(phys_addr, AIRPORT_IO_LEN);
-	if (!card->vaddr) {
-		printk(KERN_ERR PFX "ioremap() failed\n");
-		goto failed;
-	}
-
-	hermes_struct_init(hw, card->vaddr, HERMES_16BIT_REGSPACING);
-
-	/* Power up card */
-	pmac_call_feature(PMAC_FTR_AIRPORT_ENABLE,
-			  macio_get_of_node(mdev), 0, 1);
-	ssleep(1);
-
-	/* Reset it before we get the interrupt */
-	hw->ops->init(hw);
-
-	if (request_irq(card->irq, orinoco_interrupt, 0, DRIVER_NAME, priv)) {
-		printk(KERN_ERR PFX "Couldn't get IRQ %d\n", card->irq);
-		goto failed;
-	}
-	card->irq_requested = 1;
-
-	/* Initialise the main driver */
-	if (orinoco_init(priv) != 0) {
-		printk(KERN_ERR PFX "orinoco_init() failed\n");
-		goto failed;
-	}
-
-	/* Register an interface with the stack */
-	if (orinoco_if_add(priv, phys_addr, card->irq, NULL) != 0) {
-		printk(KERN_ERR PFX "orinoco_if_add() failed\n");
-		goto failed;
-	}
-	card->ndev_registered = 1;
-	return 0;
- failed:
-	airport_detach(mdev);
-	return -ENODEV;
-}				/* airport_attach */
-
-
-static char version[] __initdata = DRIVER_NAME " " DRIVER_VERSION
-	" (Benjamin Herrenschmidt <benh@kernel.crashing.org>)";
-MODULE_AUTHOR("Benjamin Herrenschmidt <benh@kernel.crashing.org>");
-MODULE_DESCRIPTION("Driver for the Apple Airport wireless card.");
-MODULE_LICENSE("Dual MPL/GPL");
-
-static const struct of_device_id airport_match[] = {
-	{
-	.name		= "radio",
-	},
-	{},
-};
-
-MODULE_DEVICE_TABLE(of, airport_match);
-
-static struct macio_driver airport_driver = {
-	.driver = {
-		.name		= DRIVER_NAME,
-		.owner		= THIS_MODULE,
-		.of_match_table	= airport_match,
-	},
-	.probe		= airport_attach,
-	.remove		= airport_detach,
-	.suspend	= airport_suspend,
-	.resume		= airport_resume,
-};
-
-static int __init
-init_airport(void)
-{
-	printk(KERN_DEBUG "%s\n", version);
-
-	return macio_register_driver(&airport_driver);
-}
-
-static void __exit
-exit_airport(void)
-{
-	macio_unregister_driver(&airport_driver);
-}
-
-module_init(init_airport);
-module_exit(exit_airport);
diff --git a/drivers/net/wireless/intersil/orinoco/cfg.c b/drivers/net/wireless/intersil/orinoco/cfg.c
deleted file mode 100644
index b2d5ec8634b5..000000000000
--- a/drivers/net/wireless/intersil/orinoco/cfg.c
+++ /dev/null
@@ -1,291 +0,0 @@
-/* cfg80211 support
- *
- * See copyright notice in main.c
- */
-#include <linux/ieee80211.h>
-#include <net/cfg80211.h>
-#include "hw.h"
-#include "main.h"
-#include "orinoco.h"
-
-#include "cfg.h"
-
-/* Supported bitrates. Must agree with hw.c */
-static struct ieee80211_rate orinoco_rates[] = {
-	{ .bitrate = 10 },
-	{ .bitrate = 20 },
-	{ .bitrate = 55 },
-	{ .bitrate = 110 },
-};
-
-static const void * const orinoco_wiphy_privid = &orinoco_wiphy_privid;
-
-/* Called after orinoco_private is allocated. */
-void orinoco_wiphy_init(struct wiphy *wiphy)
-{
-	struct orinoco_private *priv = wiphy_priv(wiphy);
-
-	wiphy->privid = orinoco_wiphy_privid;
-
-	set_wiphy_dev(wiphy, priv->dev);
-}
-
-/* Called after firmware is initialised */
-int orinoco_wiphy_register(struct wiphy *wiphy)
-{
-	struct orinoco_private *priv = wiphy_priv(wiphy);
-	int i, channels = 0;
-
-	if (priv->firmware_type == FIRMWARE_TYPE_AGERE)
-		wiphy->max_scan_ssids = 1;
-	else
-		wiphy->max_scan_ssids = 0;
-
-	wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION);
-
-	/* TODO: should we set if we only have demo ad-hoc?
-	 *       (priv->has_port3)
-	 */
-	if (priv->has_ibss)
-		wiphy->interface_modes |= BIT(NL80211_IFTYPE_ADHOC);
-
-	if (!priv->broken_monitor || force_monitor)
-		wiphy->interface_modes |= BIT(NL80211_IFTYPE_MONITOR);
-
-	priv->band.bitrates = orinoco_rates;
-	priv->band.n_bitrates = ARRAY_SIZE(orinoco_rates);
-
-	/* Only support channels allowed by the card EEPROM */
-	for (i = 0; i < NUM_CHANNELS; i++) {
-		if (priv->channel_mask & (1 << i)) {
-			priv->channels[i].center_freq =
-				ieee80211_channel_to_frequency(i + 1,
-							   NL80211_BAND_2GHZ);
-			channels++;
-		}
-	}
-	priv->band.channels = priv->channels;
-	priv->band.n_channels = channels;
-
-	wiphy->bands[NL80211_BAND_2GHZ] = &priv->band;
-	wiphy->signal_type = CFG80211_SIGNAL_TYPE_MBM;
-
-	i = 0;
-	if (priv->has_wep) {
-		priv->cipher_suites[i] = WLAN_CIPHER_SUITE_WEP40;
-		i++;
-
-		if (priv->has_big_wep) {
-			priv->cipher_suites[i] = WLAN_CIPHER_SUITE_WEP104;
-			i++;
-		}
-	}
-	if (priv->has_wpa) {
-		priv->cipher_suites[i] = WLAN_CIPHER_SUITE_TKIP;
-		i++;
-	}
-	wiphy->cipher_suites = priv->cipher_suites;
-	wiphy->n_cipher_suites = i;
-
-	wiphy->rts_threshold = priv->rts_thresh;
-	if (!priv->has_mwo)
-		wiphy->frag_threshold = priv->frag_thresh + 1;
-	wiphy->retry_short = priv->short_retry_limit;
-	wiphy->retry_long = priv->long_retry_limit;
-
-	return wiphy_register(wiphy);
-}
-
-static int orinoco_change_vif(struct wiphy *wiphy, struct net_device *dev,
-			      enum nl80211_iftype type,
-			      struct vif_params *params)
-{
-	struct orinoco_private *priv = wiphy_priv(wiphy);
-	int err = 0;
-	unsigned long lock;
-
-	if (orinoco_lock(priv, &lock) != 0)
-		return -EBUSY;
-
-	switch (type) {
-	case NL80211_IFTYPE_ADHOC:
-		if (!priv->has_ibss && !priv->has_port3)
-			err = -EINVAL;
-		break;
-
-	case NL80211_IFTYPE_STATION:
-		break;
-
-	case NL80211_IFTYPE_MONITOR:
-		if (priv->broken_monitor && !force_monitor) {
-			wiphy_warn(wiphy,
-				   "Monitor mode support is buggy in this firmware, not enabling\n");
-			err = -EINVAL;
-		}
-		break;
-
-	default:
-		err = -EINVAL;
-	}
-
-	if (!err) {
-		priv->iw_mode = type;
-		set_port_type(priv);
-		err = orinoco_commit(priv);
-	}
-
-	orinoco_unlock(priv, &lock);
-
-	return err;
-}
-
-static int orinoco_scan(struct wiphy *wiphy,
-			struct cfg80211_scan_request *request)
-{
-	struct orinoco_private *priv = wiphy_priv(wiphy);
-	int err;
-
-	if (!request)
-		return -EINVAL;
-
-	if (priv->scan_request && priv->scan_request != request)
-		return -EBUSY;
-
-	priv->scan_request = request;
-
-	err = orinoco_hw_trigger_scan(priv, request->ssids);
-	/* On error the we aren't processing the request */
-	if (err)
-		priv->scan_request = NULL;
-
-	return err;
-}
-
-static int orinoco_set_monitor_channel(struct wiphy *wiphy,
-				       struct cfg80211_chan_def *chandef)
-{
-	struct orinoco_private *priv = wiphy_priv(wiphy);
-	int err = 0;
-	unsigned long flags;
-	int channel;
-
-	if (!chandef->chan)
-		return -EINVAL;
-
-	if (cfg80211_get_chandef_type(chandef) != NL80211_CHAN_NO_HT)
-		return -EINVAL;
-
-	if (chandef->chan->band != NL80211_BAND_2GHZ)
-		return -EINVAL;
-
-	channel = ieee80211_frequency_to_channel(chandef->chan->center_freq);
-
-	if ((channel < 1) || (channel > NUM_CHANNELS) ||
-	     !(priv->channel_mask & (1 << (channel - 1))))
-		return -EINVAL;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	priv->channel = channel;
-	if (priv->iw_mode == NL80211_IFTYPE_MONITOR) {
-		/* Fast channel change - no commit if successful */
-		struct hermes *hw = &priv->hw;
-		err = hw->ops->cmd_wait(hw, HERMES_CMD_TEST |
-					    HERMES_TEST_SET_CHANNEL,
-					channel, NULL);
-	}
-	orinoco_unlock(priv, &flags);
-
-	return err;
-}
-
-static int orinoco_set_wiphy_params(struct wiphy *wiphy, u32 changed)
-{
-	struct orinoco_private *priv = wiphy_priv(wiphy);
-	int frag_value = -1;
-	int rts_value = -1;
-	int err = 0;
-
-	if (changed & WIPHY_PARAM_RETRY_SHORT) {
-		/* Setting short retry not supported */
-		err = -EINVAL;
-	}
-
-	if (changed & WIPHY_PARAM_RETRY_LONG) {
-		/* Setting long retry not supported */
-		err = -EINVAL;
-	}
-
-	if (changed & WIPHY_PARAM_FRAG_THRESHOLD) {
-		/* Set fragmentation */
-		if (priv->has_mwo) {
-			if (wiphy->frag_threshold == -1)
-				frag_value = 0;
-			else {
-				printk(KERN_WARNING "%s: Fixed fragmentation "
-				       "is not supported on this firmware. "
-				       "Using MWO robust instead.\n",
-				       priv->ndev->name);
-				frag_value = 1;
-			}
-		} else {
-			if (wiphy->frag_threshold == -1)
-				frag_value = 2346;
-			else if ((wiphy->frag_threshold < 257) ||
-				 (wiphy->frag_threshold > 2347))
-				err = -EINVAL;
-			else
-				/* cfg80211 value is 257-2347 (odd only)
-				 * orinoco rid has range 256-2346 (even only) */
-				frag_value = wiphy->frag_threshold & ~0x1;
-		}
-	}
-
-	if (changed & WIPHY_PARAM_RTS_THRESHOLD) {
-		/* Set RTS.
-		 *
-		 * Prism documentation suggests default of 2432,
-		 * and a range of 0-3000.
-		 *
-		 * Current implementation uses 2347 as the default and
-		 * the upper limit.
-		 */
-
-		if (wiphy->rts_threshold == -1)
-			rts_value = 2347;
-		else if (wiphy->rts_threshold > 2347)
-			err = -EINVAL;
-		else
-			rts_value = wiphy->rts_threshold;
-	}
-
-	if (!err) {
-		unsigned long flags;
-
-		if (orinoco_lock(priv, &flags) != 0)
-			return -EBUSY;
-
-		if (frag_value >= 0) {
-			if (priv->has_mwo)
-				priv->mwo_robust = frag_value;
-			else
-				priv->frag_thresh = frag_value;
-		}
-		if (rts_value >= 0)
-			priv->rts_thresh = rts_value;
-
-		err = orinoco_commit(priv);
-
-		orinoco_unlock(priv, &flags);
-	}
-
-	return err;
-}
-
-const struct cfg80211_ops orinoco_cfg_ops = {
-	.change_virtual_intf = orinoco_change_vif,
-	.set_monitor_channel = orinoco_set_monitor_channel,
-	.scan = orinoco_scan,
-	.set_wiphy_params = orinoco_set_wiphy_params,
-};
diff --git a/drivers/net/wireless/intersil/orinoco/cfg.h b/drivers/net/wireless/intersil/orinoco/cfg.h
deleted file mode 100644
index 3ddc96a06cd7..000000000000
--- a/drivers/net/wireless/intersil/orinoco/cfg.h
+++ /dev/null
@@ -1,15 +0,0 @@
-/* cfg80211 support.
- *
- * See copyright notice in main.c
- */
-#ifndef ORINOCO_CFG_H
-#define ORINOCO_CFG_H
-
-#include <net/cfg80211.h>
-
-extern const struct cfg80211_ops orinoco_cfg_ops;
-
-void orinoco_wiphy_init(struct wiphy *wiphy);
-int orinoco_wiphy_register(struct wiphy *wiphy);
-
-#endif /* ORINOCO_CFG_H */
diff --git a/drivers/net/wireless/intersil/orinoco/fw.c b/drivers/net/wireless/intersil/orinoco/fw.c
deleted file mode 100644
index 015af782881b..000000000000
--- a/drivers/net/wireless/intersil/orinoco/fw.c
+++ /dev/null
@@ -1,387 +0,0 @@
-/* Firmware file reading and download helpers
- *
- * See copyright notice in main.c
- */
-#include <linux/kernel.h>
-#include <linux/slab.h>
-#include <linux/firmware.h>
-#include <linux/device.h>
-#include <linux/module.h>
-
-#include "hermes.h"
-#include "hermes_dld.h"
-#include "orinoco.h"
-
-#include "fw.h"
-
-/* End markers (for Symbol firmware only) */
-#define TEXT_END	0x1A		/* End of text header */
-
-struct fw_info {
-	char *pri_fw;
-	char *sta_fw;
-	char *ap_fw;
-	u32 pda_addr;
-	u16 pda_size;
-};
-
-static const struct fw_info orinoco_fw[] = {
-	{ NULL, "agere_sta_fw.bin", "agere_ap_fw.bin", 0x00390000, 1000 },
-	{ NULL, "prism_sta_fw.bin", "prism_ap_fw.bin", 0, 1024 },
-	{ "symbol_sp24t_prim_fw", "symbol_sp24t_sec_fw", NULL, 0x00003100, 512 }
-};
-MODULE_FIRMWARE("agere_sta_fw.bin");
-MODULE_FIRMWARE("agere_ap_fw.bin");
-MODULE_FIRMWARE("prism_sta_fw.bin");
-MODULE_FIRMWARE("prism_ap_fw.bin");
-MODULE_FIRMWARE("symbol_sp24t_prim_fw");
-MODULE_FIRMWARE("symbol_sp24t_sec_fw");
-
-/* Structure used to access fields in FW
- * Make sure LE decoding macros are used
- */
-struct orinoco_fw_header {
-	char hdr_vers[6];       /* ASCII string for header version */
-	__le16 headersize;      /* Total length of header */
-	__le32 entry_point;     /* NIC entry point */
-	__le32 blocks;          /* Number of blocks to program */
-	__le32 block_offset;    /* Offset of block data from eof header */
-	__le32 pdr_offset;      /* Offset to PDR data from eof header */
-	__le32 pri_offset;      /* Offset to primary plug data */
-	__le32 compat_offset;   /* Offset to compatibility data*/
-	char signature[];      /* FW signature length headersize-20 */
-} __packed;
-
-/* Check the range of various header entries. Return a pointer to a
- * description of the problem, or NULL if everything checks out. */
-static const char *validate_fw(const struct orinoco_fw_header *hdr, size_t len)
-{
-	u16 hdrsize;
-
-	if (len < sizeof(*hdr))
-		return "image too small";
-	if (memcmp(hdr->hdr_vers, "HFW", 3) != 0)
-		return "format not recognised";
-
-	hdrsize = le16_to_cpu(hdr->headersize);
-	if (hdrsize > len)
-		return "bad headersize";
-	if ((hdrsize + le32_to_cpu(hdr->block_offset)) > len)
-		return "bad block offset";
-	if ((hdrsize + le32_to_cpu(hdr->pdr_offset)) > len)
-		return "bad PDR offset";
-	if ((hdrsize + le32_to_cpu(hdr->pri_offset)) > len)
-		return "bad PRI offset";
-	if ((hdrsize + le32_to_cpu(hdr->compat_offset)) > len)
-		return "bad compat offset";
-
-	/* TODO: consider adding a checksum or CRC to the firmware format */
-	return NULL;
-}
-
-#if defined(CONFIG_HERMES_CACHE_FW_ON_INIT) || defined(CONFIG_PM_SLEEP)
-static inline const struct firmware *
-orinoco_cached_fw_get(struct orinoco_private *priv, bool primary)
-{
-	if (primary)
-		return priv->cached_pri_fw;
-	else
-		return priv->cached_fw;
-}
-#else
-#define orinoco_cached_fw_get(priv, primary) (NULL)
-#endif
-
-/* Download either STA or AP firmware into the card. */
-static int
-orinoco_dl_firmware(struct orinoco_private *priv,
-		    const struct fw_info *fw,
-		    int ap)
-{
-	/* Plug Data Area (PDA) */
-	__le16 *pda;
-
-	struct hermes *hw = &priv->hw;
-	const struct firmware *fw_entry;
-	const struct orinoco_fw_header *hdr;
-	const unsigned char *first_block;
-	const void *end;
-	const char *firmware;
-	const char *fw_err;
-	struct device *dev = priv->dev;
-	int err = 0;
-
-	pda = kzalloc(fw->pda_size, GFP_KERNEL);
-	if (!pda)
-		return -ENOMEM;
-
-	if (ap)
-		firmware = fw->ap_fw;
-	else
-		firmware = fw->sta_fw;
-
-	dev_dbg(dev, "Attempting to download firmware %s\n", firmware);
-
-	/* Read current plug data */
-	err = hw->ops->read_pda(hw, pda, fw->pda_addr, fw->pda_size);
-	dev_dbg(dev, "Read PDA returned %d\n", err);
-	if (err)
-		goto free;
-
-	if (!orinoco_cached_fw_get(priv, false)) {
-		err = request_firmware(&fw_entry, firmware, priv->dev);
-
-		if (err) {
-			dev_err(dev, "Cannot find firmware %s\n", firmware);
-			err = -ENOENT;
-			goto free;
-		}
-	} else
-		fw_entry = orinoco_cached_fw_get(priv, false);
-
-	hdr = (const struct orinoco_fw_header *) fw_entry->data;
-
-	fw_err = validate_fw(hdr, fw_entry->size);
-	if (fw_err) {
-		dev_warn(dev, "Invalid firmware image detected (%s). "
-			 "Aborting download\n", fw_err);
-		err = -EINVAL;
-		goto abort;
-	}
-
-	/* Enable aux port to allow programming */
-	err = hw->ops->program_init(hw, le32_to_cpu(hdr->entry_point));
-	dev_dbg(dev, "Program init returned %d\n", err);
-	if (err != 0)
-		goto abort;
-
-	/* Program data */
-	first_block = (fw_entry->data +
-		       le16_to_cpu(hdr->headersize) +
-		       le32_to_cpu(hdr->block_offset));
-	end = fw_entry->data + fw_entry->size;
-
-	err = hermes_program(hw, first_block, end);
-	dev_dbg(dev, "Program returned %d\n", err);
-	if (err != 0)
-		goto abort;
-
-	/* Update production data */
-	first_block = (fw_entry->data +
-		       le16_to_cpu(hdr->headersize) +
-		       le32_to_cpu(hdr->pdr_offset));
-
-	err = hermes_apply_pda_with_defaults(hw, first_block, end, pda,
-					     &pda[fw->pda_size / sizeof(*pda)]);
-	dev_dbg(dev, "Apply PDA returned %d\n", err);
-	if (err)
-		goto abort;
-
-	/* Tell card we've finished */
-	err = hw->ops->program_end(hw);
-	dev_dbg(dev, "Program end returned %d\n", err);
-	if (err != 0)
-		goto abort;
-
-	/* Check if we're running */
-	dev_dbg(dev, "hermes_present returned %d\n", hermes_present(hw));
-
-abort:
-	/* If we requested the firmware, release it. */
-	if (!orinoco_cached_fw_get(priv, false))
-		release_firmware(fw_entry);
-
-free:
-	kfree(pda);
-	return err;
-}
-
-/*
- * Process a firmware image - stop the card, load the firmware, reset
- * the card and make sure it responds.  For the secondary firmware take
- * care of the PDA - read it and then write it on top of the firmware.
- */
-static int
-symbol_dl_image(struct orinoco_private *priv, const struct fw_info *fw,
-		const unsigned char *image, const void *end,
-		int secondary)
-{
-	struct hermes *hw = &priv->hw;
-	int ret = 0;
-	const unsigned char *ptr;
-	const unsigned char *first_block;
-
-	/* Plug Data Area (PDA) */
-	__le16 *pda = NULL;
-
-	/* Binary block begins after the 0x1A marker */
-	ptr = image;
-	while (*ptr++ != TEXT_END);
-	first_block = ptr;
-
-	/* Read the PDA from EEPROM */
-	if (secondary) {
-		pda = kzalloc(fw->pda_size, GFP_KERNEL);
-		if (!pda)
-			return -ENOMEM;
-
-		ret = hw->ops->read_pda(hw, pda, fw->pda_addr, fw->pda_size);
-		if (ret)
-			goto free;
-	}
-
-	/* Stop the firmware, so that it can be safely rewritten */
-	if (priv->stop_fw) {
-		ret = priv->stop_fw(priv, 1);
-		if (ret)
-			goto free;
-	}
-
-	/* Program the adapter with new firmware */
-	ret = hermes_program(hw, first_block, end);
-	if (ret)
-		goto free;
-
-	/* Write the PDA to the adapter */
-	if (secondary) {
-		size_t len = hermes_blocks_length(first_block, end);
-		ptr = first_block + len;
-		ret = hermes_apply_pda(hw, ptr, end, pda,
-				       &pda[fw->pda_size / sizeof(*pda)]);
-		kfree(pda);
-		if (ret)
-			return ret;
-	}
-
-	/* Run the firmware */
-	if (priv->stop_fw) {
-		ret = priv->stop_fw(priv, 0);
-		if (ret)
-			return ret;
-	}
-
-	/* Reset hermes chip and make sure it responds */
-	ret = hw->ops->init(hw);
-
-	/* hermes_reset() should return 0 with the secondary firmware */
-	if (secondary && ret != 0)
-		return -ENODEV;
-
-	/* And this should work with any firmware */
-	if (!hermes_present(hw))
-		return -ENODEV;
-
-	return 0;
-
-free:
-	kfree(pda);
-	return ret;
-}
-
-
-/*
- * Download the firmware into the card, this also does a PCMCIA soft
- * reset on the card, to make sure it's in a sane state.
- */
-static int
-symbol_dl_firmware(struct orinoco_private *priv,
-		   const struct fw_info *fw)
-{
-	struct device *dev = priv->dev;
-	int ret;
-	const struct firmware *fw_entry;
-
-	if (!orinoco_cached_fw_get(priv, true)) {
-		if (request_firmware(&fw_entry, fw->pri_fw, priv->dev) != 0) {
-			dev_err(dev, "Cannot find firmware: %s\n", fw->pri_fw);
-			return -ENOENT;
-		}
-	} else
-		fw_entry = orinoco_cached_fw_get(priv, true);
-
-	/* Load primary firmware */
-	ret = symbol_dl_image(priv, fw, fw_entry->data,
-			      fw_entry->data + fw_entry->size, 0);
-
-	if (!orinoco_cached_fw_get(priv, true))
-		release_firmware(fw_entry);
-	if (ret) {
-		dev_err(dev, "Primary firmware download failed\n");
-		return ret;
-	}
-
-	if (!orinoco_cached_fw_get(priv, false)) {
-		if (request_firmware(&fw_entry, fw->sta_fw, priv->dev) != 0) {
-			dev_err(dev, "Cannot find firmware: %s\n", fw->sta_fw);
-			return -ENOENT;
-		}
-	} else
-		fw_entry = orinoco_cached_fw_get(priv, false);
-
-	/* Load secondary firmware */
-	ret = symbol_dl_image(priv, fw, fw_entry->data,
-			      fw_entry->data + fw_entry->size, 1);
-	if (!orinoco_cached_fw_get(priv, false))
-		release_firmware(fw_entry);
-	if (ret)
-		dev_err(dev, "Secondary firmware download failed\n");
-
-	return ret;
-}
-
-int orinoco_download(struct orinoco_private *priv)
-{
-	int err = 0;
-	/* Reload firmware */
-	switch (priv->firmware_type) {
-	case FIRMWARE_TYPE_AGERE:
-		/* case FIRMWARE_TYPE_INTERSIL: */
-		err = orinoco_dl_firmware(priv,
-					  &orinoco_fw[priv->firmware_type], 0);
-		break;
-
-	case FIRMWARE_TYPE_SYMBOL:
-		err = symbol_dl_firmware(priv,
-					 &orinoco_fw[priv->firmware_type]);
-		break;
-	case FIRMWARE_TYPE_INTERSIL:
-		break;
-	}
-	/* TODO: if we fail we probably need to reinitialise
-	 * the driver */
-
-	return err;
-}
-
-#if defined(CONFIG_HERMES_CACHE_FW_ON_INIT) || defined(CONFIG_PM_SLEEP)
-void orinoco_cache_fw(struct orinoco_private *priv, int ap)
-{
-	const struct firmware *fw_entry = NULL;
-	const char *pri_fw;
-	const char *fw;
-
-	pri_fw = orinoco_fw[priv->firmware_type].pri_fw;
-	if (ap)
-		fw = orinoco_fw[priv->firmware_type].ap_fw;
-	else
-		fw = orinoco_fw[priv->firmware_type].sta_fw;
-
-	if (pri_fw) {
-		if (request_firmware(&fw_entry, pri_fw, priv->dev) == 0)
-			priv->cached_pri_fw = fw_entry;
-	}
-
-	if (fw) {
-		if (request_firmware(&fw_entry, fw, priv->dev) == 0)
-			priv->cached_fw = fw_entry;
-	}
-}
-
-void orinoco_uncache_fw(struct orinoco_private *priv)
-{
-	release_firmware(priv->cached_pri_fw);
-	release_firmware(priv->cached_fw);
-	priv->cached_pri_fw = NULL;
-	priv->cached_fw = NULL;
-}
-#endif
diff --git a/drivers/net/wireless/intersil/orinoco/fw.h b/drivers/net/wireless/intersil/orinoco/fw.h
deleted file mode 100644
index aca63e3c4b5b..000000000000
--- a/drivers/net/wireless/intersil/orinoco/fw.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/* Firmware file reading and download helpers
- *
- * See copyright notice in main.c
- */
-#ifndef _ORINOCO_FW_H_
-#define _ORINOCO_FW_H_
-
-/* Forward declations */
-struct orinoco_private;
-
-int orinoco_download(struct orinoco_private *priv);
-
-#if defined(CONFIG_HERMES_CACHE_FW_ON_INIT) || defined(CONFIG_PM_SLEEP)
-void orinoco_cache_fw(struct orinoco_private *priv, int ap);
-void orinoco_uncache_fw(struct orinoco_private *priv);
-#else
-#define orinoco_cache_fw(priv, ap) do { } while (0)
-#define orinoco_uncache_fw(priv) do { } while (0)
-#endif
-
-#endif /* _ORINOCO_FW_H_ */
diff --git a/drivers/net/wireless/intersil/orinoco/hermes.c b/drivers/net/wireless/intersil/orinoco/hermes.c
deleted file mode 100644
index 4888286727ff..000000000000
--- a/drivers/net/wireless/intersil/orinoco/hermes.c
+++ /dev/null
@@ -1,778 +0,0 @@
-/* hermes.c
- *
- * Driver core for the "Hermes" wireless MAC controller, as used in
- * the Lucent Orinoco and Cabletron RoamAbout cards. It should also
- * work on the hfa3841 and hfa3842 MAC controller chips used in the
- * Prism II chipsets.
- *
- * This is not a complete driver, just low-level access routines for
- * the MAC controller itself.
- *
- * Based on the prism2 driver from Absolute Value Systems' linux-wlan
- * project, the Linux wvlan_cs driver, Lucent's HCF-Light
- * (wvlan_hcf.c) library, and the NetBSD wireless driver (in no
- * particular order).
- *
- * Copyright (C) 2000, David Gibson, Linuxcare Australia.
- * (C) Copyright David Gibson, IBM Corp. 2001-2003.
- *
- * The contents of this file are subject to the Mozilla Public License
- * Version 1.1 (the "License"); you may not use this file except in
- * compliance with the License. You may obtain a copy of the License
- * at http://www.mozilla.org/MPL/
- *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and
- * limitations under the License.
- *
- * Alternatively, the contents of this file may be used under the
- * terms of the GNU General Public License version 2 (the "GPL"), in
- * which case the provisions of the GPL are applicable instead of the
- * above.  If you wish to allow the use of your version of this file
- * only under the terms of the GPL and not to allow others to use your
- * version of this file under the MPL, indicate your decision by
- * deleting the provisions above and replace them with the notice and
- * other provisions required by the GPL.  If you do not delete the
- * provisions above, a recipient may use your version of this file
- * under either the MPL or the GPL.
- */
-
-#include <linux/net.h>
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/delay.h>
-
-#include "hermes.h"
-
-/* These are maximum timeouts. Most often, card wil react much faster */
-#define CMD_BUSY_TIMEOUT (100) /* In iterations of ~1us */
-#define CMD_INIT_TIMEOUT (50000) /* in iterations of ~10us */
-#define CMD_COMPL_TIMEOUT (20000) /* in iterations of ~10us */
-#define ALLOC_COMPL_TIMEOUT (1000) /* in iterations of ~10us */
-
-/*
- * AUX port access.  To unlock the AUX port write the access keys to the
- * PARAM0-2 registers, then write HERMES_AUX_ENABLE to the HERMES_CONTROL
- * register.  Then read it and make sure it's HERMES_AUX_ENABLED.
- */
-#define HERMES_AUX_ENABLE	0x8000	/* Enable auxiliary port access */
-#define HERMES_AUX_DISABLE	0x4000	/* Disable to auxiliary port access */
-#define HERMES_AUX_ENABLED	0xC000	/* Auxiliary port is open */
-#define HERMES_AUX_DISABLED	0x0000	/* Auxiliary port is closed */
-
-#define HERMES_AUX_PW0	0xFE01
-#define HERMES_AUX_PW1	0xDC23
-#define HERMES_AUX_PW2	0xBA45
-
-/* HERMES_CMD_DOWNLD */
-#define HERMES_PROGRAM_DISABLE             (0x0000 | HERMES_CMD_DOWNLD)
-#define HERMES_PROGRAM_ENABLE_VOLATILE     (0x0100 | HERMES_CMD_DOWNLD)
-#define HERMES_PROGRAM_ENABLE_NON_VOLATILE (0x0200 | HERMES_CMD_DOWNLD)
-#define HERMES_PROGRAM_NON_VOLATILE        (0x0300 | HERMES_CMD_DOWNLD)
-
-/*
- * Debugging helpers
- */
-
-#define DMSG(stuff...) do {printk(KERN_DEBUG "hermes @ %p: " , hw->iobase); \
-			printk(stuff); } while (0)
-
-#undef HERMES_DEBUG
-#ifdef HERMES_DEBUG
-
-#define DEBUG(lvl, stuff...) if ((lvl) <= HERMES_DEBUG) DMSG(stuff)
-
-#else /* ! HERMES_DEBUG */
-
-#define DEBUG(lvl, stuff...) do { } while (0)
-
-#endif /* ! HERMES_DEBUG */
-
-static const struct hermes_ops hermes_ops_local;
-
-/*
- * Internal functions
- */
-
-/* Issue a command to the chip. Waiting for it to complete is the caller's
-   problem.
-
-   Returns -EBUSY if the command register is busy, 0 on success.
-
-   Callable from any context.
-*/
-static int hermes_issue_cmd(struct hermes *hw, u16 cmd, u16 param0,
-			    u16 param1, u16 param2)
-{
-	int k = CMD_BUSY_TIMEOUT;
-	u16 reg;
-
-	/* First wait for the command register to unbusy */
-	reg = hermes_read_regn(hw, CMD);
-	while ((reg & HERMES_CMD_BUSY) && k) {
-		k--;
-		udelay(1);
-		reg = hermes_read_regn(hw, CMD);
-	}
-	if (reg & HERMES_CMD_BUSY)
-		return -EBUSY;
-
-	hermes_write_regn(hw, PARAM2, param2);
-	hermes_write_regn(hw, PARAM1, param1);
-	hermes_write_regn(hw, PARAM0, param0);
-	hermes_write_regn(hw, CMD, cmd);
-
-	return 0;
-}
-
-/*
- * Function definitions
- */
-
-/* For doing cmds that wipe the magic constant in SWSUPPORT0 */
-static int hermes_doicmd_wait(struct hermes *hw, u16 cmd,
-			      u16 parm0, u16 parm1, u16 parm2,
-			      struct hermes_response *resp)
-{
-	int err = 0;
-	int k;
-	u16 status, reg;
-
-	err = hermes_issue_cmd(hw, cmd, parm0, parm1, parm2);
-	if (err)
-		return err;
-
-	reg = hermes_read_regn(hw, EVSTAT);
-	k = CMD_INIT_TIMEOUT;
-	while ((!(reg & HERMES_EV_CMD)) && k) {
-		k--;
-		udelay(10);
-		reg = hermes_read_regn(hw, EVSTAT);
-	}
-
-	hermes_write_regn(hw, SWSUPPORT0, HERMES_MAGIC);
-
-	if (!hermes_present(hw)) {
-		DEBUG(0, "hermes @ 0x%x: Card removed during reset.\n",
-		       hw->iobase);
-		err = -ENODEV;
-		goto out;
-	}
-
-	if (!(reg & HERMES_EV_CMD)) {
-		printk(KERN_ERR "hermes @ %p: "
-		       "Timeout waiting for card to reset (reg=0x%04x)!\n",
-		       hw->iobase, reg);
-		err = -ETIMEDOUT;
-		goto out;
-	}
-
-	status = hermes_read_regn(hw, STATUS);
-	if (resp) {
-		resp->status = status;
-		resp->resp0 = hermes_read_regn(hw, RESP0);
-		resp->resp1 = hermes_read_regn(hw, RESP1);
-		resp->resp2 = hermes_read_regn(hw, RESP2);
-	}
-
-	hermes_write_regn(hw, EVACK, HERMES_EV_CMD);
-
-	if (status & HERMES_STATUS_RESULT)
-		err = -EIO;
-out:
-	return err;
-}
-
-void hermes_struct_init(struct hermes *hw, void __iomem *address,
-			int reg_spacing)
-{
-	hw->iobase = address;
-	hw->reg_spacing = reg_spacing;
-	hw->inten = 0x0;
-	hw->eeprom_pda = false;
-	hw->ops = &hermes_ops_local;
-}
-EXPORT_SYMBOL(hermes_struct_init);
-
-static int hermes_init(struct hermes *hw)
-{
-	u16 reg;
-	int err = 0;
-	int k;
-
-	/* We don't want to be interrupted while resetting the chipset */
-	hw->inten = 0x0;
-	hermes_write_regn(hw, INTEN, 0);
-	hermes_write_regn(hw, EVACK, 0xffff);
-
-	/* Normally it's a "can't happen" for the command register to
-	   be busy when we go to issue a command because we are
-	   serializing all commands.  However we want to have some
-	   chance of resetting the card even if it gets into a stupid
-	   state, so we actually wait to see if the command register
-	   will unbusy itself here. */
-	k = CMD_BUSY_TIMEOUT;
-	reg = hermes_read_regn(hw, CMD);
-	while (k && (reg & HERMES_CMD_BUSY)) {
-		if (reg == 0xffff) /* Special case - the card has probably been
-				      removed, so don't wait for the timeout */
-			return -ENODEV;
-
-		k--;
-		udelay(1);
-		reg = hermes_read_regn(hw, CMD);
-	}
-
-	/* No need to explicitly handle the timeout - if we've timed
-	   out hermes_issue_cmd() will probably return -EBUSY below */
-
-	/* According to the documentation, EVSTAT may contain
-	   obsolete event occurrence information.  We have to acknowledge
-	   it by writing EVACK. */
-	reg = hermes_read_regn(hw, EVSTAT);
-	hermes_write_regn(hw, EVACK, reg);
-
-	/* We don't use hermes_docmd_wait here, because the reset wipes
-	   the magic constant in SWSUPPORT0 away, and it gets confused */
-	err = hermes_doicmd_wait(hw, HERMES_CMD_INIT, 0, 0, 0, NULL);
-
-	return err;
-}
-
-/* Issue a command to the chip, and (busy!) wait for it to
- * complete.
- *
- * Returns:
- *     < 0 on internal error
- *       0 on success
- *     > 0 on error returned by the firmware
- *
- * Callable from any context, but locking is your problem. */
-static int hermes_docmd_wait(struct hermes *hw, u16 cmd, u16 parm0,
-			     struct hermes_response *resp)
-{
-	int err;
-	int k;
-	u16 reg;
-	u16 status;
-
-	err = hermes_issue_cmd(hw, cmd, parm0, 0, 0);
-	if (err) {
-		if (!hermes_present(hw)) {
-			if (net_ratelimit())
-				printk(KERN_WARNING "hermes @ %p: "
-				       "Card removed while issuing command "
-				       "0x%04x.\n", hw->iobase, cmd);
-			err = -ENODEV;
-		} else
-			if (net_ratelimit())
-				printk(KERN_ERR "hermes @ %p: "
-				       "Error %d issuing command 0x%04x.\n",
-				       hw->iobase, err, cmd);
-		goto out;
-	}
-
-	reg = hermes_read_regn(hw, EVSTAT);
-	k = CMD_COMPL_TIMEOUT;
-	while ((!(reg & HERMES_EV_CMD)) && k) {
-		k--;
-		udelay(10);
-		reg = hermes_read_regn(hw, EVSTAT);
-	}
-
-	if (!hermes_present(hw)) {
-		printk(KERN_WARNING "hermes @ %p: Card removed "
-		       "while waiting for command 0x%04x completion.\n",
-		       hw->iobase, cmd);
-		err = -ENODEV;
-		goto out;
-	}
-
-	if (!(reg & HERMES_EV_CMD)) {
-		printk(KERN_ERR "hermes @ %p: Timeout waiting for "
-		       "command 0x%04x completion.\n", hw->iobase, cmd);
-		err = -ETIMEDOUT;
-		goto out;
-	}
-
-	status = hermes_read_regn(hw, STATUS);
-	if (resp) {
-		resp->status = status;
-		resp->resp0 = hermes_read_regn(hw, RESP0);
-		resp->resp1 = hermes_read_regn(hw, RESP1);
-		resp->resp2 = hermes_read_regn(hw, RESP2);
-	}
-
-	hermes_write_regn(hw, EVACK, HERMES_EV_CMD);
-
-	if (status & HERMES_STATUS_RESULT)
-		err = -EIO;
-
- out:
-	return err;
-}
-
-static int hermes_allocate(struct hermes *hw, u16 size, u16 *fid)
-{
-	int err = 0;
-	int k;
-	u16 reg;
-
-	if ((size < HERMES_ALLOC_LEN_MIN) || (size > HERMES_ALLOC_LEN_MAX))
-		return -EINVAL;
-
-	err = hermes_docmd_wait(hw, HERMES_CMD_ALLOC, size, NULL);
-	if (err)
-		return err;
-
-	reg = hermes_read_regn(hw, EVSTAT);
-	k = ALLOC_COMPL_TIMEOUT;
-	while ((!(reg & HERMES_EV_ALLOC)) && k) {
-		k--;
-		udelay(10);
-		reg = hermes_read_regn(hw, EVSTAT);
-	}
-
-	if (!hermes_present(hw)) {
-		printk(KERN_WARNING "hermes @ %p: "
-		       "Card removed waiting for frame allocation.\n",
-		       hw->iobase);
-		return -ENODEV;
-	}
-
-	if (!(reg & HERMES_EV_ALLOC)) {
-		printk(KERN_ERR "hermes @ %p: "
-		       "Timeout waiting for frame allocation\n",
-		       hw->iobase);
-		return -ETIMEDOUT;
-	}
-
-	*fid = hermes_read_regn(hw, ALLOCFID);
-	hermes_write_regn(hw, EVACK, HERMES_EV_ALLOC);
-
-	return 0;
-}
-
-/* Set up a BAP to read a particular chunk of data from card's internal buffer.
- *
- * Returns:
- *     < 0 on internal failure (errno)
- *       0 on success
- *     > 0 on error
- * from firmware
- *
- * Callable from any context */
-static int hermes_bap_seek(struct hermes *hw, int bap, u16 id, u16 offset)
-{
-	int sreg = bap ? HERMES_SELECT1 : HERMES_SELECT0;
-	int oreg = bap ? HERMES_OFFSET1 : HERMES_OFFSET0;
-	int k;
-	u16 reg;
-
-	/* Paranoia.. */
-	if ((offset > HERMES_BAP_OFFSET_MAX) || (offset % 2))
-		return -EINVAL;
-
-	k = HERMES_BAP_BUSY_TIMEOUT;
-	reg = hermes_read_reg(hw, oreg);
-	while ((reg & HERMES_OFFSET_BUSY) && k) {
-		k--;
-		udelay(1);
-		reg = hermes_read_reg(hw, oreg);
-	}
-
-	if (reg & HERMES_OFFSET_BUSY)
-		return -ETIMEDOUT;
-
-	/* Now we actually set up the transfer */
-	hermes_write_reg(hw, sreg, id);
-	hermes_write_reg(hw, oreg, offset);
-
-	/* Wait for the BAP to be ready */
-	k = HERMES_BAP_BUSY_TIMEOUT;
-	reg = hermes_read_reg(hw, oreg);
-	while ((reg & (HERMES_OFFSET_BUSY | HERMES_OFFSET_ERR)) && k) {
-		k--;
-		udelay(1);
-		reg = hermes_read_reg(hw, oreg);
-	}
-
-	if (reg != offset) {
-		printk(KERN_ERR "hermes @ %p: BAP%d offset %s: "
-		       "reg=0x%x id=0x%x offset=0x%x\n", hw->iobase, bap,
-		       (reg & HERMES_OFFSET_BUSY) ? "timeout" : "error",
-		       reg, id, offset);
-
-		if (reg & HERMES_OFFSET_BUSY)
-			return -ETIMEDOUT;
-
-		return -EIO;		/* error or wrong offset */
-	}
-
-	return 0;
-}
-
-/* Read a block of data from the chip's buffer, via the
- * BAP. Synchronization/serialization is the caller's problem.  len
- * must be even.
- *
- * Returns:
- *     < 0 on internal failure (errno)
- *       0 on success
- *     > 0 on error from firmware
- */
-static int hermes_bap_pread(struct hermes *hw, int bap, void *buf, int len,
-			    u16 id, u16 offset)
-{
-	int dreg = bap ? HERMES_DATA1 : HERMES_DATA0;
-	int err = 0;
-
-	if ((len < 0) || (len % 2))
-		return -EINVAL;
-
-	err = hermes_bap_seek(hw, bap, id, offset);
-	if (err)
-		goto out;
-
-	/* Actually do the transfer */
-	hermes_read_words(hw, dreg, buf, len / 2);
-
- out:
-	return err;
-}
-
-/* Write a block of data to the chip's buffer, via the
- * BAP. Synchronization/serialization is the caller's problem.
- *
- * Returns:
- *     < 0 on internal failure (errno)
- *       0 on success
- *     > 0 on error from firmware
- */
-static int hermes_bap_pwrite(struct hermes *hw, int bap, const void *buf,
-			     int len, u16 id, u16 offset)
-{
-	int dreg = bap ? HERMES_DATA1 : HERMES_DATA0;
-	int err = 0;
-
-	if (len < 0)
-		return -EINVAL;
-
-	err = hermes_bap_seek(hw, bap, id, offset);
-	if (err)
-		goto out;
-
-	/* Actually do the transfer */
-	hermes_write_bytes(hw, dreg, buf, len);
-
- out:
-	return err;
-}
-
-/* Read a Length-Type-Value record from the card.
- *
- * If length is NULL, we ignore the length read from the card, and
- * read the entire buffer regardless. This is useful because some of
- * the configuration records appear to have incorrect lengths in
- * practice.
- *
- * Callable from user or bh context.  */
-static int hermes_read_ltv(struct hermes *hw, int bap, u16 rid,
-			   unsigned bufsize, u16 *length, void *buf)
-{
-	int err = 0;
-	int dreg = bap ? HERMES_DATA1 : HERMES_DATA0;
-	u16 rlength, rtype;
-	unsigned nwords;
-
-	if (bufsize % 2)
-		return -EINVAL;
-
-	err = hermes_docmd_wait(hw, HERMES_CMD_ACCESS, rid, NULL);
-	if (err)
-		return err;
-
-	err = hermes_bap_seek(hw, bap, rid, 0);
-	if (err)
-		return err;
-
-	rlength = hermes_read_reg(hw, dreg);
-
-	if (!rlength)
-		return -ENODATA;
-
-	rtype = hermes_read_reg(hw, dreg);
-
-	if (length)
-		*length = rlength;
-
-	if (rtype != rid)
-		printk(KERN_WARNING "hermes @ %p: %s(): "
-		       "rid (0x%04x) does not match type (0x%04x)\n",
-		       hw->iobase, __func__, rid, rtype);
-	if (HERMES_RECLEN_TO_BYTES(rlength) > bufsize)
-		printk(KERN_WARNING "hermes @ %p: "
-		       "Truncating LTV record from %d to %d bytes. "
-		       "(rid=0x%04x, len=0x%04x)\n", hw->iobase,
-		       HERMES_RECLEN_TO_BYTES(rlength), bufsize, rid, rlength);
-
-	nwords = min((unsigned)rlength - 1, bufsize / 2);
-	hermes_read_words(hw, dreg, buf, nwords);
-
-	return 0;
-}
-
-static int hermes_write_ltv(struct hermes *hw, int bap, u16 rid,
-			    u16 length, const void *value)
-{
-	int dreg = bap ? HERMES_DATA1 : HERMES_DATA0;
-	int err = 0;
-	unsigned count;
-
-	if (length == 0)
-		return -EINVAL;
-
-	err = hermes_bap_seek(hw, bap, rid, 0);
-	if (err)
-		return err;
-
-	hermes_write_reg(hw, dreg, length);
-	hermes_write_reg(hw, dreg, rid);
-
-	count = length - 1;
-
-	hermes_write_bytes(hw, dreg, value, count << 1);
-
-	err = hermes_docmd_wait(hw, HERMES_CMD_ACCESS | HERMES_CMD_WRITE,
-				rid, NULL);
-
-	return err;
-}
-
-/*** Hermes AUX control ***/
-
-static inline void
-hermes_aux_setaddr(struct hermes *hw, u32 addr)
-{
-	hermes_write_reg(hw, HERMES_AUXPAGE, (u16) (addr >> 7));
-	hermes_write_reg(hw, HERMES_AUXOFFSET, (u16) (addr & 0x7F));
-}
-
-static inline int
-hermes_aux_control(struct hermes *hw, int enabled)
-{
-	int desired_state = enabled ? HERMES_AUX_ENABLED : HERMES_AUX_DISABLED;
-	int action = enabled ? HERMES_AUX_ENABLE : HERMES_AUX_DISABLE;
-	int i;
-
-	/* Already open? */
-	if (hermes_read_reg(hw, HERMES_CONTROL) == desired_state)
-		return 0;
-
-	hermes_write_reg(hw, HERMES_PARAM0, HERMES_AUX_PW0);
-	hermes_write_reg(hw, HERMES_PARAM1, HERMES_AUX_PW1);
-	hermes_write_reg(hw, HERMES_PARAM2, HERMES_AUX_PW2);
-	hermes_write_reg(hw, HERMES_CONTROL, action);
-
-	for (i = 0; i < 20; i++) {
-		udelay(10);
-		if (hermes_read_reg(hw, HERMES_CONTROL) ==
-		    desired_state)
-			return 0;
-	}
-
-	return -EBUSY;
-}
-
-/*** Hermes programming ***/
-
-/* About to start programming data (Hermes I)
- * offset is the entry point
- *
- * Spectrum_cs' Symbol fw does not require this
- * wl_lkm Agere fw does
- * Don't know about intersil
- */
-static int hermesi_program_init(struct hermes *hw, u32 offset)
-{
-	int err;
-
-	/* Disable interrupts?*/
-	/*hw->inten = 0x0;*/
-	/*hermes_write_regn(hw, INTEN, 0);*/
-	/*hermes_set_irqmask(hw, 0);*/
-
-	/* Acknowledge any outstanding command */
-	hermes_write_regn(hw, EVACK, 0xFFFF);
-
-	/* Using init_cmd_wait rather than cmd_wait */
-	err = hw->ops->init_cmd_wait(hw,
-				     0x0100 | HERMES_CMD_INIT,
-				     0, 0, 0, NULL);
-	if (err)
-		return err;
-
-	err = hw->ops->init_cmd_wait(hw,
-				     0x0000 | HERMES_CMD_INIT,
-				     0, 0, 0, NULL);
-	if (err)
-		return err;
-
-	err = hermes_aux_control(hw, 1);
-	pr_debug("AUX enable returned %d\n", err);
-
-	if (err)
-		return err;
-
-	pr_debug("Enabling volatile, EP 0x%08x\n", offset);
-	err = hw->ops->init_cmd_wait(hw,
-				     HERMES_PROGRAM_ENABLE_VOLATILE,
-				     offset & 0xFFFFu,
-				     offset >> 16,
-				     0,
-				     NULL);
-	pr_debug("PROGRAM_ENABLE returned %d\n", err);
-
-	return err;
-}
-
-/* Done programming data (Hermes I)
- *
- * Spectrum_cs' Symbol fw does not require this
- * wl_lkm Agere fw does
- * Don't know about intersil
- */
-static int hermesi_program_end(struct hermes *hw)
-{
-	struct hermes_response resp;
-	int rc = 0;
-	int err;
-
-	rc = hw->ops->cmd_wait(hw, HERMES_PROGRAM_DISABLE, 0, &resp);
-
-	pr_debug("PROGRAM_DISABLE returned %d, "
-		 "r0 0x%04x, r1 0x%04x, r2 0x%04x\n",
-		 rc, resp.resp0, resp.resp1, resp.resp2);
-
-	if ((rc == 0) &&
-	    ((resp.status & HERMES_STATUS_CMDCODE) != HERMES_CMD_DOWNLD))
-		rc = -EIO;
-
-	err = hermes_aux_control(hw, 0);
-	pr_debug("AUX disable returned %d\n", err);
-
-	/* Acknowledge any outstanding command */
-	hermes_write_regn(hw, EVACK, 0xFFFF);
-
-	/* Reinitialise, ignoring return */
-	(void) hw->ops->init_cmd_wait(hw, 0x0000 | HERMES_CMD_INIT,
-				      0, 0, 0, NULL);
-
-	return rc ? rc : err;
-}
-
-static int hermes_program_bytes(struct hermes *hw, const char *data,
-				u32 addr, u32 len)
-{
-	/* wl lkm splits the programming into chunks of 2000 bytes.
-	 * This restriction appears to come from USB. The PCMCIA
-	 * adapters can program the whole lot in one go */
-	hermes_aux_setaddr(hw, addr);
-	hermes_write_bytes(hw, HERMES_AUXDATA, data, len);
-	return 0;
-}
-
-/* Read PDA from the adapter */
-static int hermes_read_pda(struct hermes *hw, __le16 *pda, u32 pda_addr,
-			   u16 pda_len)
-{
-	int ret;
-	u16 pda_size;
-	u16 data_len = pda_len;
-	__le16 *data = pda;
-
-	if (hw->eeprom_pda) {
-		/* PDA of spectrum symbol is in eeprom */
-
-		/* Issue command to read EEPROM */
-		ret = hw->ops->cmd_wait(hw, HERMES_CMD_READMIF, 0, NULL);
-		if (ret)
-			return ret;
-	} else {
-		/* wl_lkm does not include PDA size in the PDA area.
-		 * We will pad the information into pda, so other routines
-		 * don't have to be modified */
-		pda[0] = cpu_to_le16(pda_len - 2);
-			/* Includes CFG_PROD_DATA but not itself */
-		pda[1] = cpu_to_le16(0x0800); /* CFG_PROD_DATA */
-		data_len = pda_len - 4;
-		data = pda + 2;
-	}
-
-	/* Open auxiliary port */
-	ret = hermes_aux_control(hw, 1);
-	pr_debug("AUX enable returned %d\n", ret);
-	if (ret)
-		return ret;
-
-	/* Read PDA */
-	hermes_aux_setaddr(hw, pda_addr);
-	hermes_read_words(hw, HERMES_AUXDATA, data, data_len / 2);
-
-	/* Close aux port */
-	ret = hermes_aux_control(hw, 0);
-	pr_debug("AUX disable returned %d\n", ret);
-
-	/* Check PDA length */
-	pda_size = le16_to_cpu(pda[0]);
-	pr_debug("Actual PDA length %d, Max allowed %d\n",
-		 pda_size, pda_len);
-	if (pda_size > pda_len)
-		return -EINVAL;
-
-	return 0;
-}
-
-static void hermes_lock_irqsave(spinlock_t *lock,
-				unsigned long *flags) __acquires(lock)
-{
-	spin_lock_irqsave(lock, *flags);
-}
-
-static void hermes_unlock_irqrestore(spinlock_t *lock,
-				     unsigned long *flags) __releases(lock)
-{
-	spin_unlock_irqrestore(lock, *flags);
-}
-
-static void hermes_lock_irq(spinlock_t *lock) __acquires(lock)
-{
-	spin_lock_irq(lock);
-}
-
-static void hermes_unlock_irq(spinlock_t *lock) __releases(lock)
-{
-	spin_unlock_irq(lock);
-}
-
-/* Hermes operations for local buses */
-static const struct hermes_ops hermes_ops_local = {
-	.init = hermes_init,
-	.cmd_wait = hermes_docmd_wait,
-	.init_cmd_wait = hermes_doicmd_wait,
-	.allocate = hermes_allocate,
-	.read_ltv = hermes_read_ltv,
-	.read_ltv_pr = hermes_read_ltv,
-	.write_ltv = hermes_write_ltv,
-	.bap_pread = hermes_bap_pread,
-	.bap_pwrite = hermes_bap_pwrite,
-	.read_pda = hermes_read_pda,
-	.program_init = hermesi_program_init,
-	.program_end = hermesi_program_end,
-	.program = hermes_program_bytes,
-	.lock_irqsave = hermes_lock_irqsave,
-	.unlock_irqrestore = hermes_unlock_irqrestore,
-	.lock_irq = hermes_lock_irq,
-	.unlock_irq = hermes_unlock_irq,
-};
diff --git a/drivers/net/wireless/intersil/orinoco/hermes.h b/drivers/net/wireless/intersil/orinoco/hermes.h
deleted file mode 100644
index 3dc561a5cb7a..000000000000
--- a/drivers/net/wireless/intersil/orinoco/hermes.h
+++ /dev/null
@@ -1,534 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/* hermes.h
- *
- * Driver core for the "Hermes" wireless MAC controller, as used in
- * the Lucent Orinoco and Cabletron RoamAbout cards. It should also
- * work on the hfa3841 and hfa3842 MAC controller chips used in the
- * Prism I & II chipsets.
- *
- * This is not a complete driver, just low-level access routines for
- * the MAC controller itself.
- *
- * Based on the prism2 driver from Absolute Value Systems' linux-wlan
- * project, the Linux wvlan_cs driver, Lucent's HCF-Light
- * (wvlan_hcf.c) library, and the NetBSD wireless driver.
- *
- * Copyright (C) 2000, David Gibson, Linuxcare Australia.
- * (C) Copyright David Gibson, IBM Corp. 2001-2003.
- *
- * Portions taken from hfa384x.h.
- * Copyright (C) 1999 AbsoluteValue Systems, Inc. All Rights Reserved.
- */
-
-#ifndef _HERMES_H
-#define _HERMES_H
-
-/* Notes on locking:
- *
- * As a module of low level hardware access routines, there is no
- * locking. Users of this module should ensure that they serialize
- * access to the hermes structure, and to the hardware
-*/
-
-#include <linux/if_ether.h>
-#include <linux/io.h>
-
-/*
- * Limits and constants
- */
-#define		HERMES_ALLOC_LEN_MIN		(4)
-#define		HERMES_ALLOC_LEN_MAX		(2400)
-#define		HERMES_LTV_LEN_MAX		(34)
-#define		HERMES_BAP_DATALEN_MAX		(4096)
-#define		HERMES_BAP_OFFSET_MAX		(4096)
-#define		HERMES_PORTID_MAX		(7)
-#define		HERMES_NUMPORTS_MAX		(HERMES_PORTID_MAX + 1)
-#define		HERMES_PDR_LEN_MAX		(260)	/* in bytes, from EK */
-#define		HERMES_PDA_RECS_MAX		(200)	/* a guess */
-#define		HERMES_PDA_LEN_MAX		(1024)	/* in bytes, from EK */
-#define		HERMES_SCANRESULT_MAX		(35)
-#define		HERMES_CHINFORESULT_MAX		(8)
-#define		HERMES_MAX_MULTICAST		(16)
-#define		HERMES_MAGIC			(0x7d1f)
-
-/*
- * Hermes register offsets
- */
-#define		HERMES_CMD			(0x00)
-#define		HERMES_PARAM0			(0x02)
-#define		HERMES_PARAM1			(0x04)
-#define		HERMES_PARAM2			(0x06)
-#define		HERMES_STATUS			(0x08)
-#define		HERMES_RESP0			(0x0A)
-#define		HERMES_RESP1			(0x0C)
-#define		HERMES_RESP2			(0x0E)
-#define		HERMES_INFOFID			(0x10)
-#define		HERMES_RXFID			(0x20)
-#define		HERMES_ALLOCFID			(0x22)
-#define		HERMES_TXCOMPLFID		(0x24)
-#define		HERMES_SELECT0			(0x18)
-#define		HERMES_OFFSET0			(0x1C)
-#define		HERMES_DATA0			(0x36)
-#define		HERMES_SELECT1			(0x1A)
-#define		HERMES_OFFSET1			(0x1E)
-#define		HERMES_DATA1			(0x38)
-#define		HERMES_EVSTAT			(0x30)
-#define		HERMES_INTEN			(0x32)
-#define		HERMES_EVACK			(0x34)
-#define		HERMES_CONTROL			(0x14)
-#define		HERMES_SWSUPPORT0		(0x28)
-#define		HERMES_SWSUPPORT1		(0x2A)
-#define		HERMES_SWSUPPORT2		(0x2C)
-#define		HERMES_AUXPAGE			(0x3A)
-#define		HERMES_AUXOFFSET		(0x3C)
-#define		HERMES_AUXDATA			(0x3E)
-
-/*
- * CMD register bitmasks
- */
-#define		HERMES_CMD_BUSY			(0x8000)
-#define		HERMES_CMD_AINFO		(0x7f00)
-#define		HERMES_CMD_MACPORT		(0x0700)
-#define		HERMES_CMD_RECL			(0x0100)
-#define		HERMES_CMD_WRITE		(0x0100)
-#define		HERMES_CMD_PROGMODE		(0x0300)
-#define		HERMES_CMD_CMDCODE		(0x003f)
-
-/*
- * STATUS register bitmasks
- */
-#define		HERMES_STATUS_RESULT		(0x7f00)
-#define		HERMES_STATUS_CMDCODE		(0x003f)
-
-/*
- * OFFSET register bitmasks
- */
-#define		HERMES_OFFSET_BUSY		(0x8000)
-#define		HERMES_OFFSET_ERR		(0x4000)
-#define		HERMES_OFFSET_DATAOFF		(0x0ffe)
-
-/*
- * Event register bitmasks (INTEN, EVSTAT, EVACK)
- */
-#define		HERMES_EV_TICK			(0x8000)
-#define		HERMES_EV_WTERR			(0x4000)
-#define		HERMES_EV_INFDROP		(0x2000)
-#define		HERMES_EV_INFO			(0x0080)
-#define		HERMES_EV_DTIM			(0x0020)
-#define		HERMES_EV_CMD			(0x0010)
-#define		HERMES_EV_ALLOC			(0x0008)
-#define		HERMES_EV_TXEXC			(0x0004)
-#define		HERMES_EV_TX			(0x0002)
-#define		HERMES_EV_RX			(0x0001)
-
-/*
- * Command codes
- */
-/*--- Controller Commands ----------------------------*/
-#define		HERMES_CMD_INIT			(0x0000)
-#define		HERMES_CMD_ENABLE		(0x0001)
-#define		HERMES_CMD_DISABLE		(0x0002)
-#define		HERMES_CMD_DIAG			(0x0003)
-
-/*--- Buffer Mgmt Commands ---------------------------*/
-#define		HERMES_CMD_ALLOC		(0x000A)
-#define		HERMES_CMD_TX			(0x000B)
-
-/*--- Regulate Commands ------------------------------*/
-#define		HERMES_CMD_NOTIFY		(0x0010)
-#define		HERMES_CMD_INQUIRE		(0x0011)
-
-/*--- Configure Commands -----------------------------*/
-#define		HERMES_CMD_ACCESS		(0x0021)
-#define		HERMES_CMD_DOWNLD		(0x0022)
-
-/*--- Serial I/O Commands ----------------------------*/
-#define		HERMES_CMD_READMIF		(0x0030)
-#define		HERMES_CMD_WRITEMIF		(0x0031)
-
-/*--- Debugging Commands -----------------------------*/
-#define		HERMES_CMD_TEST			(0x0038)
-
-
-/* Test command arguments */
-#define		HERMES_TEST_SET_CHANNEL		0x0800
-#define		HERMES_TEST_MONITOR		0x0b00
-#define		HERMES_TEST_STOP		0x0f00
-
-/* Authentication algorithms */
-#define		HERMES_AUTH_OPEN		1
-#define		HERMES_AUTH_SHARED_KEY		2
-
-/* WEP settings */
-#define		HERMES_WEP_PRIVACY_INVOKED	0x0001
-#define		HERMES_WEP_EXCL_UNENCRYPTED	0x0002
-#define		HERMES_WEP_HOST_ENCRYPT		0x0010
-#define		HERMES_WEP_HOST_DECRYPT		0x0080
-
-/* Symbol hostscan options */
-#define		HERMES_HOSTSCAN_SYMBOL_5SEC	0x0001
-#define		HERMES_HOSTSCAN_SYMBOL_ONCE	0x0002
-#define		HERMES_HOSTSCAN_SYMBOL_PASSIVE	0x0040
-#define		HERMES_HOSTSCAN_SYMBOL_BCAST	0x0080
-
-/*
- * Frame structures and constants
- */
-
-#define HERMES_DESCRIPTOR_OFFSET	0
-#define HERMES_802_11_OFFSET		(14)
-#define HERMES_802_3_OFFSET		(14 + 32)
-#define HERMES_802_2_OFFSET		(14 + 32 + 14)
-#define HERMES_TXCNTL2_OFFSET		(HERMES_802_3_OFFSET - 2)
-
-#define HERMES_RXSTAT_ERR		(0x0003)
-#define	HERMES_RXSTAT_BADCRC		(0x0001)
-#define	HERMES_RXSTAT_UNDECRYPTABLE	(0x0002)
-#define	HERMES_RXSTAT_MIC		(0x0010)	/* Frame contains MIC */
-#define	HERMES_RXSTAT_MACPORT		(0x0700)
-#define HERMES_RXSTAT_PCF		(0x1000)	/* Frame was received in CF period */
-#define	HERMES_RXSTAT_MIC_KEY_ID	(0x1800)	/* MIC key used */
-#define	HERMES_RXSTAT_MSGTYPE		(0xE000)
-#define	HERMES_RXSTAT_1042		(0x2000)	/* RFC-1042 frame */
-#define	HERMES_RXSTAT_TUNNEL		(0x4000)	/* bridge-tunnel encoded frame */
-#define	HERMES_RXSTAT_WMP		(0x6000)	/* Wavelan-II Management Protocol frame */
-
-/* Shift amount for key ID in RXSTAT and TXCTRL */
-#define	HERMES_MIC_KEY_ID_SHIFT		11
-
-struct hermes_tx_descriptor {
-	__le16 status;
-	__le16 reserved1;
-	__le16 reserved2;
-	__le32 sw_support;
-	u8 retry_count;
-	u8 tx_rate;
-	__le16 tx_control;
-} __packed;
-
-#define HERMES_TXSTAT_RETRYERR		(0x0001)
-#define HERMES_TXSTAT_AGEDERR		(0x0002)
-#define HERMES_TXSTAT_DISCON		(0x0004)
-#define HERMES_TXSTAT_FORMERR		(0x0008)
-
-#define HERMES_TXCTRL_TX_OK		(0x0002)	/* ?? interrupt on Tx complete */
-#define HERMES_TXCTRL_TX_EX		(0x0004)	/* ?? interrupt on Tx exception */
-#define HERMES_TXCTRL_802_11		(0x0008)	/* We supply 802.11 header */
-#define HERMES_TXCTRL_MIC		(0x0010)	/* 802.3 + TKIP */
-#define HERMES_TXCTRL_MIC_KEY_ID	(0x1800)	/* MIC Key ID mask */
-#define HERMES_TXCTRL_ALT_RTRY		(0x0020)
-
-/* Inquiry constants and data types */
-
-#define HERMES_INQ_TALLIES		(0xF100)
-#define HERMES_INQ_SCAN			(0xF101)
-#define HERMES_INQ_CHANNELINFO		(0xF102)
-#define HERMES_INQ_HOSTSCAN		(0xF103)
-#define HERMES_INQ_HOSTSCAN_SYMBOL	(0xF104)
-#define HERMES_INQ_LINKSTATUS		(0xF200)
-#define HERMES_INQ_SEC_STAT_AGERE	(0xF202)
-
-struct hermes_tallies_frame {
-	__le16 TxUnicastFrames;
-	__le16 TxMulticastFrames;
-	__le16 TxFragments;
-	__le16 TxUnicastOctets;
-	__le16 TxMulticastOctets;
-	__le16 TxDeferredTransmissions;
-	__le16 TxSingleRetryFrames;
-	__le16 TxMultipleRetryFrames;
-	__le16 TxRetryLimitExceeded;
-	__le16 TxDiscards;
-	__le16 RxUnicastFrames;
-	__le16 RxMulticastFrames;
-	__le16 RxFragments;
-	__le16 RxUnicastOctets;
-	__le16 RxMulticastOctets;
-	__le16 RxFCSErrors;
-	__le16 RxDiscards_NoBuffer;
-	__le16 TxDiscardsWrongSA;
-	__le16 RxWEPUndecryptable;
-	__le16 RxMsgInMsgFragments;
-	__le16 RxMsgInBadMsgFragments;
-	/* Those last are probably not available in very old firmwares */
-	__le16 RxDiscards_WEPICVError;
-	__le16 RxDiscards_WEPExcluded;
-} __packed;
-
-/* Grabbed from wlan-ng - Thanks Mark... - Jean II
- * This is the result of a scan inquiry command */
-/* Structure describing info about an Access Point */
-struct prism2_scan_apinfo {
-	__le16 channel;		/* Channel where the AP sits */
-	__le16 noise;		/* Noise level */
-	__le16 level;		/* Signal level */
-	u8 bssid[ETH_ALEN];	/* MAC address of the Access Point */
-	__le16 beacon_interv;	/* Beacon interval */
-	__le16 capabilities;	/* Capabilities */
-	__le16 essid_len;	/* ESSID length */
-	u8 essid[32];		/* ESSID of the network */
-	u8 rates[10];		/* Bit rate supported */
-	__le16 proberesp_rate;	/* Data rate of the response frame */
-	__le16 atim;		/* ATIM window time, Kus (hostscan only) */
-} __packed;
-
-/* Same stuff for the Lucent/Agere card.
- * Thanks to h1kari <h1kari AT dachb0den.com> - Jean II */
-struct agere_scan_apinfo {
-	__le16 channel;		/* Channel where the AP sits */
-	__le16 noise;		/* Noise level */
-	__le16 level;		/* Signal level */
-	u8 bssid[ETH_ALEN];	/* MAC address of the Access Point */
-	__le16 beacon_interv;	/* Beacon interval */
-	__le16 capabilities;	/* Capabilities */
-	/* bits: 0-ess, 1-ibss, 4-privacy [wep] */
-	__le16 essid_len;	/* ESSID length */
-	u8 essid[32];		/* ESSID of the network */
-} __packed;
-
-/* Moustafa: Scan structure for Symbol cards */
-struct symbol_scan_apinfo {
-	u8 channel;		/* Channel where the AP sits */
-	u8 unknown1;		/* 8 in 2.9x and 3.9x f/w, 0 otherwise */
-	__le16 noise;		/* Noise level */
-	__le16 level;		/* Signal level */
-	u8 bssid[ETH_ALEN];	/* MAC address of the Access Point */
-	__le16 beacon_interv;	/* Beacon interval */
-	__le16 capabilities;	/* Capabilities */
-	/* bits: 0-ess, 1-ibss, 4-privacy [wep] */
-	__le16 essid_len;	/* ESSID length */
-	u8 essid[32];		/* ESSID of the network */
-	__le16 rates[5];	/* Bit rate supported */
-	__le16 basic_rates;	/* Basic rates bitmask */
-	u8 unknown2[6];		/* Always FF:FF:FF:FF:00:00 */
-	u8 unknown3[8];		/* Always 0, appeared in f/w 3.91-68 */
-} __packed;
-
-union hermes_scan_info {
-	struct agere_scan_apinfo	a;
-	struct prism2_scan_apinfo	p;
-	struct symbol_scan_apinfo	s;
-};
-
-/* Extended scan struct for HERMES_INQ_CHANNELINFO.
- * wl_lkm calls this an ACS scan (Automatic Channel Select).
- * Keep out of union hermes_scan_info because it is much bigger than
- * the older scan structures. */
-struct agere_ext_scan_info {
-	__le16	reserved0;
-
-	u8	noise;
-	u8	level;
-	u8	rx_flow;
-	u8	rate;
-	__le16	reserved1[2];
-
-	__le16	frame_control;
-	__le16	dur_id;
-	u8	addr1[ETH_ALEN];
-	u8	addr2[ETH_ALEN];
-	u8	bssid[ETH_ALEN];
-	__le16	sequence;
-	u8	addr4[ETH_ALEN];
-
-	__le16	data_length;
-
-	/* Next 3 fields do not get filled in. */
-	u8	daddr[ETH_ALEN];
-	u8	saddr[ETH_ALEN];
-	__le16	len_type;
-
-	__le64	timestamp;
-	__le16	beacon_interval;
-	__le16	capabilities;
-	u8	data[];
-} __packed;
-
-#define HERMES_LINKSTATUS_NOT_CONNECTED   (0x0000)
-#define HERMES_LINKSTATUS_CONNECTED       (0x0001)
-#define HERMES_LINKSTATUS_DISCONNECTED    (0x0002)
-#define HERMES_LINKSTATUS_AP_CHANGE       (0x0003)
-#define HERMES_LINKSTATUS_AP_OUT_OF_RANGE (0x0004)
-#define HERMES_LINKSTATUS_AP_IN_RANGE     (0x0005)
-#define HERMES_LINKSTATUS_ASSOC_FAILED    (0x0006)
-
-struct hermes_linkstatus {
-	__le16 linkstatus;         /* Link status */
-} __packed;
-
-struct hermes_response {
-	u16 status, resp0, resp1, resp2;
-};
-
-/* "ID" structure - used for ESSID and station nickname */
-struct hermes_idstring {
-	__le16 len;
-	__le16 val[16];
-} __packed;
-
-struct hermes_multicast {
-	u8 addr[HERMES_MAX_MULTICAST][ETH_ALEN];
-} __packed;
-
-/* Timeouts */
-#define HERMES_BAP_BUSY_TIMEOUT (10000) /* In iterations of ~1us */
-
-struct hermes;
-
-/* Functions to access hardware */
-struct hermes_ops {
-	int (*init)(struct hermes *hw);
-	int (*cmd_wait)(struct hermes *hw, u16 cmd, u16 parm0,
-			struct hermes_response *resp);
-	int (*init_cmd_wait)(struct hermes *hw, u16 cmd,
-			     u16 parm0, u16 parm1, u16 parm2,
-			     struct hermes_response *resp);
-	int (*allocate)(struct hermes *hw, u16 size, u16 *fid);
-	int (*read_ltv)(struct hermes *hw, int bap, u16 rid, unsigned buflen,
-			u16 *length, void *buf);
-	int (*read_ltv_pr)(struct hermes *hw, int bap, u16 rid,
-			      unsigned buflen, u16 *length, void *buf);
-	int (*write_ltv)(struct hermes *hw, int bap, u16 rid,
-			 u16 length, const void *value);
-	int (*bap_pread)(struct hermes *hw, int bap, void *buf, int len,
-			 u16 id, u16 offset);
-	int (*bap_pwrite)(struct hermes *hw, int bap, const void *buf,
-			  int len, u16 id, u16 offset);
-	int (*read_pda)(struct hermes *hw, __le16 *pda,
-			u32 pda_addr, u16 pda_len);
-	int (*program_init)(struct hermes *hw, u32 entry_point);
-	int (*program_end)(struct hermes *hw);
-	int (*program)(struct hermes *hw, const char *buf,
-		       u32 addr, u32 len);
-	void (*lock_irqsave)(spinlock_t *lock, unsigned long *flags);
-	void (*unlock_irqrestore)(spinlock_t *lock, unsigned long *flags);
-	void (*lock_irq)(spinlock_t *lock);
-	void (*unlock_irq)(spinlock_t *lock);
-};
-
-/* Basic control structure */
-struct hermes {
-	void __iomem *iobase;
-	int reg_spacing;
-#define HERMES_16BIT_REGSPACING	0
-#define HERMES_32BIT_REGSPACING	1
-	u16 inten; /* Which interrupts should be enabled? */
-	bool eeprom_pda;
-	const struct hermes_ops *ops;
-	void *priv;
-};
-
-/* Register access convenience macros */
-#define hermes_read_reg(hw, off) \
-	(ioread16((hw)->iobase + ((off) << (hw)->reg_spacing)))
-#define hermes_write_reg(hw, off, val) \
-	(iowrite16((val), (hw)->iobase + ((off) << (hw)->reg_spacing)))
-#define hermes_read_regn(hw, name) hermes_read_reg((hw), HERMES_##name)
-#define hermes_write_regn(hw, name, val) \
-	hermes_write_reg((hw), HERMES_##name, (val))
-
-/* Function prototypes */
-void hermes_struct_init(struct hermes *hw, void __iomem *address,
-			int reg_spacing);
-
-/* Inline functions */
-
-static inline int hermes_present(struct hermes *hw)
-{
-	return hermes_read_regn(hw, SWSUPPORT0) == HERMES_MAGIC;
-}
-
-static inline void hermes_set_irqmask(struct hermes *hw, u16 events)
-{
-	hw->inten = events;
-	hermes_write_regn(hw, INTEN, events);
-}
-
-static inline int hermes_enable_port(struct hermes *hw, int port)
-{
-	return hw->ops->cmd_wait(hw, HERMES_CMD_ENABLE | (port << 8),
-				 0, NULL);
-}
-
-static inline int hermes_disable_port(struct hermes *hw, int port)
-{
-	return hw->ops->cmd_wait(hw, HERMES_CMD_DISABLE | (port << 8),
-				 0, NULL);
-}
-
-/* Initiate an INQUIRE command (tallies or scan).  The result will come as an
- * information frame in __orinoco_ev_info() */
-static inline int hermes_inquire(struct hermes *hw, u16 rid)
-{
-	return hw->ops->cmd_wait(hw, HERMES_CMD_INQUIRE, rid, NULL);
-}
-
-#define HERMES_BYTES_TO_RECLEN(n) ((((n) + 1) / 2) + 1)
-#define HERMES_RECLEN_TO_BYTES(n) (((n) - 1) * 2)
-
-/* Note that for the next two, the count is in 16-bit words, not bytes */
-static inline void hermes_read_words(struct hermes *hw, int off,
-				     void *buf, unsigned count)
-{
-	off = off << hw->reg_spacing;
-	ioread16_rep(hw->iobase + off, buf, count);
-}
-
-static inline void hermes_write_bytes(struct hermes *hw, int off,
-				      const char *buf, unsigned count)
-{
-	off = off << hw->reg_spacing;
-	iowrite16_rep(hw->iobase + off, buf, count >> 1);
-	if (unlikely(count & 1))
-		iowrite8(buf[count - 1], hw->iobase + off);
-}
-
-static inline void hermes_clear_words(struct hermes *hw, int off,
-				      unsigned count)
-{
-	unsigned i;
-
-	off = off << hw->reg_spacing;
-
-	for (i = 0; i < count; i++)
-		iowrite16(0, hw->iobase + off);
-}
-
-#define HERMES_READ_RECORD(hw, bap, rid, buf) \
-	(hw->ops->read_ltv((hw), (bap), (rid), sizeof(*buf), NULL, (buf)))
-#define HERMES_READ_RECORD_PR(hw, bap, rid, buf) \
-	(hw->ops->read_ltv_pr((hw), (bap), (rid), sizeof(*buf), NULL, (buf)))
-#define HERMES_WRITE_RECORD(hw, bap, rid, buf) \
-	(hw->ops->write_ltv((hw), (bap), (rid), \
-			    HERMES_BYTES_TO_RECLEN(sizeof(*buf)), (buf)))
-
-static inline int hermes_read_wordrec(struct hermes *hw, int bap, u16 rid,
-				      u16 *word)
-{
-	__le16 rec;
-	int err;
-
-	err = HERMES_READ_RECORD(hw, bap, rid, &rec);
-	*word = le16_to_cpu(rec);
-	return err;
-}
-
-static inline int hermes_read_wordrec_pr(struct hermes *hw, int bap, u16 rid,
-					 u16 *word)
-{
-	__le16 rec;
-	int err;
-
-	err = HERMES_READ_RECORD_PR(hw, bap, rid, &rec);
-	*word = le16_to_cpu(rec);
-	return err;
-}
-
-static inline int hermes_write_wordrec(struct hermes *hw, int bap, u16 rid,
-				       u16 word)
-{
-	__le16 rec = cpu_to_le16(word);
-	return HERMES_WRITE_RECORD(hw, bap, rid, &rec);
-}
-
-#endif  /* _HERMES_H */
diff --git a/drivers/net/wireless/intersil/orinoco/hermes_dld.c b/drivers/net/wireless/intersil/orinoco/hermes_dld.c
deleted file mode 100644
index dbeadfcfefe2..000000000000
--- a/drivers/net/wireless/intersil/orinoco/hermes_dld.c
+++ /dev/null
@@ -1,477 +0,0 @@
-/*
- * Hermes download helper.
- *
- * This helper:
- *  - is capable of writing to the volatile area of the hermes device
- *  - is currently not capable of writing to non-volatile areas
- *  - provide helpers to identify and update plugin data
- *  - is not capable of interpreting a fw image directly. That is up to
- *    the main card driver.
- *  - deals with Hermes I devices. It can probably be modified to deal
- *    with Hermes II devices
- *
- * Copyright (C) 2007, David Kilroy
- *
- * Plug data code slightly modified from spectrum_cs driver
- *    Copyright (C) 2002-2005 Pavel Roskin <proski@gnu.org>
- * Portions based on information in wl_lkm_718 Agere driver
- *    COPYRIGHT (C) 2001-2004 by Agere Systems Inc. All Rights Reserved
- *
- * The contents of this file are subject to the Mozilla Public License
- * Version 1.1 (the "License"); you may not use this file except in
- * compliance with the License. You may obtain a copy of the License
- * at http://www.mozilla.org/MPL/
- *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and
- * limitations under the License.
- *
- * Alternatively, the contents of this file may be used under the
- * terms of the GNU General Public License version 2 (the "GPL"), in
- * which case the provisions of the GPL are applicable instead of the
- * above.  If you wish to allow the use of your version of this file
- * only under the terms of the GPL and not to allow others to use your
- * version of this file under the MPL, indicate your decision by
- * deleting the provisions above and replace them with the notice and
- * other provisions required by the GPL.  If you do not delete the
- * provisions above, a recipient may use your version of this file
- * under either the MPL or the GPL.
- */
-
-#include <linux/module.h>
-#include <linux/delay.h>
-#include "hermes.h"
-#include "hermes_dld.h"
-
-#define PFX "hermes_dld: "
-
-/* End markers used in dblocks */
-#define PDI_END		0x00000000	/* End of PDA */
-#define BLOCK_END	0xFFFFFFFF	/* Last image block */
-#define TEXT_END	0x1A		/* End of text header */
-
-/*
- * The following structures have little-endian fields denoted by
- * the leading underscore.  Don't access them directly - use inline
- * functions defined below.
- */
-
-/*
- * The binary image to be downloaded consists of series of data blocks.
- * Each block has the following structure.
- */
-struct dblock {
-	__le32 addr;		/* adapter address where to write the block */
-	__le16 len;		/* length of the data only, in bytes */
-	char data[];		/* data to be written */
-} __packed;
-
-/*
- * Plug Data References are located in the image after the last data
- * block.  They refer to areas in the adapter memory where the plug data
- * items with matching ID should be written.
- */
-struct pdr {
-	__le32 id;		/* record ID */
-	__le32 addr;		/* adapter address where to write the data */
-	__le32 len;		/* expected length of the data, in bytes */
-	char next[];		/* next PDR starts here */
-} __packed;
-
-/*
- * Plug Data Items are located in the EEPROM read from the adapter by
- * primary firmware.  They refer to the device-specific data that should
- * be plugged into the secondary firmware.
- */
-struct pdi {
-	__le16 len;		/* length of ID and data, in words */
-	__le16 id;		/* record ID */
-	char data[];		/* plug data */
-} __packed;
-
-/*** FW data block access functions ***/
-
-static inline u32
-dblock_addr(const struct dblock *blk)
-{
-	return le32_to_cpu(blk->addr);
-}
-
-static inline u32
-dblock_len(const struct dblock *blk)
-{
-	return le16_to_cpu(blk->len);
-}
-
-/*** PDR Access functions ***/
-
-static inline u32
-pdr_id(const struct pdr *pdr)
-{
-	return le32_to_cpu(pdr->id);
-}
-
-static inline u32
-pdr_addr(const struct pdr *pdr)
-{
-	return le32_to_cpu(pdr->addr);
-}
-
-static inline u32
-pdr_len(const struct pdr *pdr)
-{
-	return le32_to_cpu(pdr->len);
-}
-
-/*** PDI Access functions ***/
-
-static inline u32
-pdi_id(const struct pdi *pdi)
-{
-	return le16_to_cpu(pdi->id);
-}
-
-/* Return length of the data only, in bytes */
-static inline u32
-pdi_len(const struct pdi *pdi)
-{
-	return 2 * (le16_to_cpu(pdi->len) - 1);
-}
-
-/*** Plug Data Functions ***/
-
-/*
- * Scan PDR for the record with the specified RECORD_ID.
- * If it's not found, return NULL.
- */
-static const struct pdr *
-hermes_find_pdr(const struct pdr *first_pdr, u32 record_id, const void *end)
-{
-	const struct pdr *pdr = first_pdr;
-
-	end -= sizeof(struct pdr);
-
-	while (((void *) pdr <= end) &&
-	       (pdr_id(pdr) != PDI_END)) {
-		/*
-		 * PDR area is currently not terminated by PDI_END.
-		 * It's followed by CRC records, which have the type
-		 * field where PDR has length.  The type can be 0 or 1.
-		 */
-		if (pdr_len(pdr) < 2)
-			return NULL;
-
-		/* If the record ID matches, we are done */
-		if (pdr_id(pdr) == record_id)
-			return pdr;
-
-		pdr = (struct pdr *) pdr->next;
-	}
-	return NULL;
-}
-
-/* Scan production data items for a particular entry */
-static const struct pdi *
-hermes_find_pdi(const struct pdi *first_pdi, u32 record_id, const void *end)
-{
-	const struct pdi *pdi = first_pdi;
-
-	end -= sizeof(struct pdi);
-
-	while (((void *) pdi <= end) &&
-	       (pdi_id(pdi) != PDI_END)) {
-
-		/* If the record ID matches, we are done */
-		if (pdi_id(pdi) == record_id)
-			return pdi;
-
-		pdi = (struct pdi *) &pdi->data[pdi_len(pdi)];
-	}
-	return NULL;
-}
-
-/* Process one Plug Data Item - find corresponding PDR and plug it */
-static int
-hermes_plug_pdi(struct hermes *hw, const struct pdr *first_pdr,
-		const struct pdi *pdi, const void *pdr_end)
-{
-	const struct pdr *pdr;
-
-	/* Find the PDR corresponding to this PDI */
-	pdr = hermes_find_pdr(first_pdr, pdi_id(pdi), pdr_end);
-
-	/* No match is found, safe to ignore */
-	if (!pdr)
-		return 0;
-
-	/* Lengths of the data in PDI and PDR must match */
-	if (pdi_len(pdi) != pdr_len(pdr))
-		return -EINVAL;
-
-	/* do the actual plugging */
-	hw->ops->program(hw, pdi->data, pdr_addr(pdr), pdi_len(pdi));
-
-	return 0;
-}
-
-/* Parse PDA and write the records into the adapter
- *
- * Attempt to write every records that is in the specified pda
- * which also has a valid production data record for the firmware.
- */
-int hermes_apply_pda(struct hermes *hw,
-		     const char *first_pdr,
-		     const void *pdr_end,
-		     const __le16 *pda,
-		     const void *pda_end)
-{
-	int ret;
-	const struct pdi *pdi;
-	const struct pdr *pdr;
-
-	pdr = (const struct pdr *) first_pdr;
-	pda_end -= sizeof(struct pdi);
-
-	/* Go through every PDI and plug them into the adapter */
-	pdi = (const struct pdi *) (pda + 2);
-	while (((void *) pdi <= pda_end) &&
-	       (pdi_id(pdi) != PDI_END)) {
-		ret = hermes_plug_pdi(hw, pdr, pdi, pdr_end);
-		if (ret)
-			return ret;
-
-		/* Increment to the next PDI */
-		pdi = (const struct pdi *) &pdi->data[pdi_len(pdi)];
-	}
-	return 0;
-}
-
-/* Identify the total number of bytes in all blocks
- * including the header data.
- */
-size_t
-hermes_blocks_length(const char *first_block, const void *end)
-{
-	const struct dblock *blk = (const struct dblock *) first_block;
-	int total_len = 0;
-	int len;
-
-	end -= sizeof(*blk);
-
-	/* Skip all blocks to locate Plug Data References
-	 * (Spectrum CS) */
-	while (((void *) blk <= end) &&
-	       (dblock_addr(blk) != BLOCK_END)) {
-		len = dblock_len(blk);
-		total_len += sizeof(*blk) + len;
-		blk = (struct dblock *) &blk->data[len];
-	}
-
-	return total_len;
-}
-
-/*** Hermes programming ***/
-
-/* Program the data blocks */
-int hermes_program(struct hermes *hw, const char *first_block, const void *end)
-{
-	const struct dblock *blk;
-	u32 blkaddr;
-	u32 blklen;
-	int err = 0;
-
-	blk = (const struct dblock *) first_block;
-
-	if ((void *) blk > (end - sizeof(*blk)))
-		return -EIO;
-
-	blkaddr = dblock_addr(blk);
-	blklen = dblock_len(blk);
-
-	while ((blkaddr != BLOCK_END) &&
-	       (((void *) blk + blklen) <= end)) {
-		pr_debug(PFX "Programming block of length %d "
-			 "to address 0x%08x\n", blklen, blkaddr);
-
-		err = hw->ops->program(hw, blk->data, blkaddr, blklen);
-		if (err)
-			break;
-
-		blk = (const struct dblock *) &blk->data[blklen];
-
-		if ((void *) blk > (end - sizeof(*blk)))
-			return -EIO;
-
-		blkaddr = dblock_addr(blk);
-		blklen = dblock_len(blk);
-	}
-	return err;
-}
-
-/*** Default plugging data for Hermes I ***/
-/* Values from wl_lkm_718/hcf/dhf.c */
-
-#define DEFINE_DEFAULT_PDR(pid, length, data)				\
-static const struct {							\
-	__le16 len;							\
-	__le16 id;							\
-	u8 val[length];							\
-} __packed default_pdr_data_##pid = {			\
-	cpu_to_le16((sizeof(default_pdr_data_##pid)/			\
-				sizeof(__le16)) - 1),			\
-	cpu_to_le16(pid),						\
-	data								\
-}
-
-#define DEFAULT_PDR(pid) default_pdr_data_##pid
-
-/*  HWIF Compatibility */
-DEFINE_DEFAULT_PDR(0x0005, 10, "\x00\x00\x06\x00\x01\x00\x01\x00\x01\x00");
-
-/* PPPPSign */
-DEFINE_DEFAULT_PDR(0x0108, 4, "\x00\x00\x00\x00");
-
-/* PPPPProf */
-DEFINE_DEFAULT_PDR(0x0109, 10, "\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00");
-
-/* Antenna diversity */
-DEFINE_DEFAULT_PDR(0x0150, 2, "\x00\x3F");
-
-/* Modem VCO band Set-up */
-DEFINE_DEFAULT_PDR(0x0160, 28,
-		   "\x00\x00\x00\x00\x00\x00\x00\x00"
-		   "\x00\x00\x00\x00\x00\x00\x00\x00"
-		   "\x00\x00\x00\x00\x00\x00\x00\x00"
-		   "\x00\x00\x00\x00");
-
-/* Modem Rx Gain Table Values */
-DEFINE_DEFAULT_PDR(0x0161, 256,
-		   "\x3F\x01\x3F\01\x3F\x01\x3F\x01"
-		   "\x3F\x01\x3F\01\x3F\x01\x3F\x01"
-		   "\x3F\x01\x3F\01\x3F\x01\x3F\x01"
-		   "\x3F\x01\x3F\01\x3F\x01\x3F\x01"
-		   "\x3F\x01\x3E\01\x3E\x01\x3D\x01"
-		   "\x3D\x01\x3C\01\x3C\x01\x3B\x01"
-		   "\x3B\x01\x3A\01\x3A\x01\x39\x01"
-		   "\x39\x01\x38\01\x38\x01\x37\x01"
-		   "\x37\x01\x36\01\x36\x01\x35\x01"
-		   "\x35\x01\x34\01\x34\x01\x33\x01"
-		   "\x33\x01\x32\x01\x32\x01\x31\x01"
-		   "\x31\x01\x30\x01\x30\x01\x7B\x01"
-		   "\x7B\x01\x7A\x01\x7A\x01\x79\x01"
-		   "\x79\x01\x78\x01\x78\x01\x77\x01"
-		   "\x77\x01\x76\x01\x76\x01\x75\x01"
-		   "\x75\x01\x74\x01\x74\x01\x73\x01"
-		   "\x73\x01\x72\x01\x72\x01\x71\x01"
-		   "\x71\x01\x70\x01\x70\x01\x68\x01"
-		   "\x68\x01\x67\x01\x67\x01\x66\x01"
-		   "\x66\x01\x65\x01\x65\x01\x57\x01"
-		   "\x57\x01\x56\x01\x56\x01\x55\x01"
-		   "\x55\x01\x54\x01\x54\x01\x53\x01"
-		   "\x53\x01\x52\x01\x52\x01\x51\x01"
-		   "\x51\x01\x50\x01\x50\x01\x48\x01"
-		   "\x48\x01\x47\x01\x47\x01\x46\x01"
-		   "\x46\x01\x45\x01\x45\x01\x44\x01"
-		   "\x44\x01\x43\x01\x43\x01\x42\x01"
-		   "\x42\x01\x41\x01\x41\x01\x40\x01"
-		   "\x40\x01\x40\x01\x40\x01\x40\x01"
-		   "\x40\x01\x40\x01\x40\x01\x40\x01"
-		   "\x40\x01\x40\x01\x40\x01\x40\x01"
-		   "\x40\x01\x40\x01\x40\x01\x40\x01");
-
-/* Write PDA according to certain rules.
- *
- * For every production data record, look for a previous setting in
- * the pda, and use that.
- *
- * For certain records, use defaults if they are not found in pda.
- */
-int hermes_apply_pda_with_defaults(struct hermes *hw,
-				   const char *first_pdr,
-				   const void *pdr_end,
-				   const __le16 *pda,
-				   const void *pda_end)
-{
-	const struct pdr *pdr = (const struct pdr *) first_pdr;
-	const struct pdi *first_pdi = (const struct pdi *) &pda[2];
-	const struct pdi *pdi;
-	const struct pdi *default_pdi = NULL;
-	const struct pdi *outdoor_pdi;
-	int record_id;
-
-	pdr_end -= sizeof(struct pdr);
-
-	while (((void *) pdr <= pdr_end) &&
-	       (pdr_id(pdr) != PDI_END)) {
-		/*
-		 * For spectrum_cs firmwares,
-		 * PDR area is currently not terminated by PDI_END.
-		 * It's followed by CRC records, which have the type
-		 * field where PDR has length.  The type can be 0 or 1.
-		 */
-		if (pdr_len(pdr) < 2)
-			break;
-		record_id = pdr_id(pdr);
-
-		pdi = hermes_find_pdi(first_pdi, record_id, pda_end);
-		if (pdi)
-			pr_debug(PFX "Found record 0x%04x at %p\n",
-				 record_id, pdi);
-
-		switch (record_id) {
-		case 0x110: /* Modem REFDAC values */
-		case 0x120: /* Modem VGDAC values */
-			outdoor_pdi = hermes_find_pdi(first_pdi, record_id + 1,
-						      pda_end);
-			default_pdi = NULL;
-			if (outdoor_pdi) {
-				pdi = outdoor_pdi;
-				pr_debug(PFX
-					 "Using outdoor record 0x%04x at %p\n",
-					 record_id + 1, pdi);
-			}
-			break;
-		case 0x5: /*  HWIF Compatibility */
-			default_pdi = (struct pdi *) &DEFAULT_PDR(0x0005);
-			break;
-		case 0x108: /* PPPPSign */
-			default_pdi = (struct pdi *) &DEFAULT_PDR(0x0108);
-			break;
-		case 0x109: /* PPPPProf */
-			default_pdi = (struct pdi *) &DEFAULT_PDR(0x0109);
-			break;
-		case 0x150: /* Antenna diversity */
-			default_pdi = (struct pdi *) &DEFAULT_PDR(0x0150);
-			break;
-		case 0x160: /* Modem VCO band Set-up */
-			default_pdi = (struct pdi *) &DEFAULT_PDR(0x0160);
-			break;
-		case 0x161: /* Modem Rx Gain Table Values */
-			default_pdi = (struct pdi *) &DEFAULT_PDR(0x0161);
-			break;
-		default:
-			default_pdi = NULL;
-			break;
-		}
-		if (!pdi && default_pdi) {
-			/* Use default */
-			pdi = default_pdi;
-			pr_debug(PFX "Using default record 0x%04x at %p\n",
-				 record_id, pdi);
-		}
-
-		if (pdi) {
-			/* Lengths of the data in PDI and PDR must match */
-			if ((pdi_len(pdi) == pdr_len(pdr)) &&
-			    ((void *) pdi->data + pdi_len(pdi) < pda_end)) {
-				/* do the actual plugging */
-				hw->ops->program(hw, pdi->data, pdr_addr(pdr),
-						 pdi_len(pdi));
-			}
-		}
-
-		pdr++;
-	}
-	return 0;
-}
diff --git a/drivers/net/wireless/intersil/orinoco/hermes_dld.h b/drivers/net/wireless/intersil/orinoco/hermes_dld.h
deleted file mode 100644
index b5377e232c63..000000000000
--- a/drivers/net/wireless/intersil/orinoco/hermes_dld.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright (C) 2007, David Kilroy
- *
- * The contents of this file are subject to the Mozilla Public License
- * Version 1.1 (the "License"); you may not use this file except in
- * compliance with the License. You may obtain a copy of the License
- * at http://www.mozilla.org/MPL/
- *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and
- * limitations under the License.
- *
- * Alternatively, the contents of this file may be used under the
- * terms of the GNU General Public License version 2 (the "GPL"), in
- * which case the provisions of the GPL are applicable instead of the
- * above.  If you wish to allow the use of your version of this file
- * only under the terms of the GPL and not to allow others to use your
- * version of this file under the MPL, indicate your decision by
- * deleting the provisions above and replace them with the notice and
- * other provisions required by the GPL.  If you do not delete the
- * provisions above, a recipient may use your version of this file
- * under either the MPL or the GPL.
- */
-#ifndef _HERMES_DLD_H
-#define _HERMES_DLD_H
-
-#include "hermes.h"
-
-int hermesi_program_init(struct hermes *hw, u32 offset);
-int hermesi_program_end(struct hermes *hw);
-int hermes_program(struct hermes *hw, const char *first_block, const void *end);
-
-int hermes_read_pda(struct hermes *hw,
-		    __le16 *pda,
-		    u32 pda_addr,
-		    u16 pda_len,
-		    int use_eeprom);
-int hermes_apply_pda(struct hermes *hw,
-		     const char *first_pdr,
-		     const void *pdr_end,
-		     const __le16 *pda,
-		     const void *pda_end);
-int hermes_apply_pda_with_defaults(struct hermes *hw,
-				   const char *first_pdr,
-				   const void *pdr_end,
-				   const __le16 *pda,
-				   const void *pda_end);
-
-size_t hermes_blocks_length(const char *first_block, const void *end);
-
-#endif /* _HERMES_DLD_H */
diff --git a/drivers/net/wireless/intersil/orinoco/hermes_rid.h b/drivers/net/wireless/intersil/orinoco/hermes_rid.h
deleted file mode 100644
index 42eb67dea1df..000000000000
--- a/drivers/net/wireless/intersil/orinoco/hermes_rid.h
+++ /dev/null
@@ -1,165 +0,0 @@
-#ifndef _HERMES_RID_H
-#define _HERMES_RID_H
-
-/*
- * Configuration RIDs
- */
-#define HERMES_RID_CNFPORTTYPE			0xFC00
-#define HERMES_RID_CNFOWNMACADDR		0xFC01
-#define HERMES_RID_CNFDESIREDSSID		0xFC02
-#define HERMES_RID_CNFOWNCHANNEL		0xFC03
-#define HERMES_RID_CNFOWNSSID			0xFC04
-#define HERMES_RID_CNFOWNATIMWINDOW		0xFC05
-#define HERMES_RID_CNFSYSTEMSCALE		0xFC06
-#define HERMES_RID_CNFMAXDATALEN		0xFC07
-#define HERMES_RID_CNFWDSADDRESS		0xFC08
-#define HERMES_RID_CNFPMENABLED			0xFC09
-#define HERMES_RID_CNFPMEPS			0xFC0A
-#define HERMES_RID_CNFMULTICASTRECEIVE		0xFC0B
-#define HERMES_RID_CNFMAXSLEEPDURATION		0xFC0C
-#define HERMES_RID_CNFPMHOLDOVERDURATION	0xFC0D
-#define HERMES_RID_CNFOWNNAME			0xFC0E
-#define HERMES_RID_CNFOWNDTIMPERIOD		0xFC10
-#define HERMES_RID_CNFWDSADDRESS1		0xFC11
-#define HERMES_RID_CNFWDSADDRESS2		0xFC12
-#define HERMES_RID_CNFWDSADDRESS3		0xFC13
-#define HERMES_RID_CNFWDSADDRESS4		0xFC14
-#define HERMES_RID_CNFWDSADDRESS5		0xFC15
-#define HERMES_RID_CNFWDSADDRESS6		0xFC16
-#define HERMES_RID_CNFMULTICASTPMBUFFERING	0xFC17
-#define HERMES_RID_CNFWEPENABLED_AGERE		0xFC20
-#define HERMES_RID_CNFAUTHENTICATION_AGERE	0xFC21
-#define HERMES_RID_CNFMANDATORYBSSID_SYMBOL	0xFC21
-#define HERMES_RID_CNFDROPUNENCRYPTED		0xFC22
-#define HERMES_RID_CNFWEPDEFAULTKEYID		0xFC23
-#define HERMES_RID_CNFDEFAULTKEY0		0xFC24
-#define HERMES_RID_CNFDEFAULTKEY1		0xFC25
-#define HERMES_RID_CNFMWOROBUST_AGERE		0xFC25
-#define HERMES_RID_CNFDEFAULTKEY2		0xFC26
-#define HERMES_RID_CNFDEFAULTKEY3		0xFC27
-#define HERMES_RID_CNFWEPFLAGS_INTERSIL		0xFC28
-#define HERMES_RID_CNFWEPKEYMAPPINGTABLE	0xFC29
-#define HERMES_RID_CNFAUTHENTICATION		0xFC2A
-#define HERMES_RID_CNFMAXASSOCSTA		0xFC2B
-#define	HERMES_RID_CNFKEYLENGTH_SYMBOL		0xFC2B
-#define HERMES_RID_CNFTXCONTROL			0xFC2C
-#define HERMES_RID_CNFROAMINGMODE		0xFC2D
-#define HERMES_RID_CNFHOSTAUTHENTICATION	0xFC2E
-#define HERMES_RID_CNFRCVCRCERROR		0xFC30
-#define HERMES_RID_CNFMMLIFE			0xFC31
-#define HERMES_RID_CNFALTRETRYCOUNT		0xFC32
-#define HERMES_RID_CNFBEACONINT			0xFC33
-#define HERMES_RID_CNFAPPCFINFO			0xFC34
-#define HERMES_RID_CNFSTAPCFINFO		0xFC35
-#define HERMES_RID_CNFPRIORITYQUSAGE		0xFC37
-#define HERMES_RID_CNFTIMCTRL			0xFC40
-#define HERMES_RID_CNFTHIRTY2TALLY		0xFC42
-#define HERMES_RID_CNFENHSECURITY		0xFC43
-#define HERMES_RID_CNFGROUPADDRESSES		0xFC80
-#define HERMES_RID_CNFCREATEIBSS		0xFC81
-#define HERMES_RID_CNFFRAGMENTATIONTHRESHOLD	0xFC82
-#define HERMES_RID_CNFRTSTHRESHOLD		0xFC83
-#define HERMES_RID_CNFTXRATECONTROL		0xFC84
-#define HERMES_RID_CNFPROMISCUOUSMODE		0xFC85
-#define HERMES_RID_CNFBASICRATES_SYMBOL		0xFC8A
-#define HERMES_RID_CNFPREAMBLE_SYMBOL		0xFC8C
-#define HERMES_RID_CNFFRAGMENTATIONTHRESHOLD0	0xFC90
-#define HERMES_RID_CNFFRAGMENTATIONTHRESHOLD1	0xFC91
-#define HERMES_RID_CNFFRAGMENTATIONTHRESHOLD2	0xFC92
-#define HERMES_RID_CNFFRAGMENTATIONTHRESHOLD3	0xFC93
-#define HERMES_RID_CNFFRAGMENTATIONTHRESHOLD4	0xFC94
-#define HERMES_RID_CNFFRAGMENTATIONTHRESHOLD5	0xFC95
-#define HERMES_RID_CNFFRAGMENTATIONTHRESHOLD6	0xFC96
-#define HERMES_RID_CNFRTSTHRESHOLD0		0xFC97
-#define HERMES_RID_CNFRTSTHRESHOLD1		0xFC98
-#define HERMES_RID_CNFRTSTHRESHOLD2		0xFC99
-#define HERMES_RID_CNFRTSTHRESHOLD3		0xFC9A
-#define HERMES_RID_CNFRTSTHRESHOLD4		0xFC9B
-#define HERMES_RID_CNFRTSTHRESHOLD5		0xFC9C
-#define HERMES_RID_CNFRTSTHRESHOLD6		0xFC9D
-#define HERMES_RID_CNFHOSTSCAN_SYMBOL		0xFCAB
-#define HERMES_RID_CNFSHORTPREAMBLE		0xFCB0
-#define HERMES_RID_CNFWEPKEYS_AGERE		0xFCB0
-#define HERMES_RID_CNFEXCLUDELONGPREAMBLE	0xFCB1
-#define HERMES_RID_CNFTXKEY_AGERE		0xFCB1
-#define HERMES_RID_CNFAUTHENTICATIONRSPTO	0xFCB2
-#define HERMES_RID_CNFSCANSSID_AGERE		0xFCB2
-#define HERMES_RID_CNFBASICRATES		0xFCB3
-#define HERMES_RID_CNFSUPPORTEDRATES		0xFCB4
-#define HERMES_RID_CNFADDDEFAULTTKIPKEY_AGERE	0xFCB4
-#define HERMES_RID_CNFSETWPAAUTHMGMTSUITE_AGERE	0xFCB5
-#define HERMES_RID_CNFREMDEFAULTTKIPKEY_AGERE	0xFCB6
-#define HERMES_RID_CNFADDMAPPEDTKIPKEY_AGERE	0xFCB7
-#define HERMES_RID_CNFREMMAPPEDTKIPKEY_AGERE	0xFCB8
-#define HERMES_RID_CNFSETWPACAPABILITIES_AGERE	0xFCB9
-#define HERMES_RID_CNFCACHEDPMKADDRESS		0xFCBA
-#define HERMES_RID_CNFREMOVEPMKADDRESS		0xFCBB
-#define HERMES_RID_CNFSCANCHANNELS2GHZ		0xFCC2
-#define HERMES_RID_CNFDISASSOCIATE		0xFCC8
-#define HERMES_RID_CNFTICKTIME			0xFCE0
-#define HERMES_RID_CNFSCANREQUEST		0xFCE1
-#define HERMES_RID_CNFJOINREQUEST		0xFCE2
-#define HERMES_RID_CNFAUTHENTICATESTATION	0xFCE3
-#define HERMES_RID_CNFCHANNELINFOREQUEST	0xFCE4
-#define HERMES_RID_CNFHOSTSCAN			0xFCE5
-
-/*
- * Information RIDs
- */
-#define HERMES_RID_MAXLOADTIME			0xFD00
-#define HERMES_RID_DOWNLOADBUFFER		0xFD01
-#define HERMES_RID_PRIID			0xFD02
-#define HERMES_RID_PRISUPRANGE			0xFD03
-#define HERMES_RID_CFIACTRANGES			0xFD04
-#define HERMES_RID_NICSERNUM			0xFD0A
-#define HERMES_RID_NICID			0xFD0B
-#define HERMES_RID_MFISUPRANGE			0xFD0C
-#define HERMES_RID_CFISUPRANGE			0xFD0D
-#define HERMES_RID_CHANNELLIST			0xFD10
-#define HERMES_RID_REGULATORYDOMAINS		0xFD11
-#define HERMES_RID_TEMPTYPE			0xFD12
-#define HERMES_RID_CIS				0xFD13
-#define HERMES_RID_STAID			0xFD20
-#define HERMES_RID_STASUPRANGE			0xFD21
-#define HERMES_RID_MFIACTRANGES			0xFD22
-#define HERMES_RID_CFIACTRANGES2		0xFD23
-#define HERMES_RID_SECONDARYVERSION_SYMBOL	0xFD24
-#define HERMES_RID_PORTSTATUS			0xFD40
-#define HERMES_RID_CURRENTSSID			0xFD41
-#define HERMES_RID_CURRENTBSSID			0xFD42
-#define HERMES_RID_COMMSQUALITY			0xFD43
-#define HERMES_RID_CURRENTTXRATE		0xFD44
-#define HERMES_RID_CURRENTBEACONINTERVAL	0xFD45
-#define HERMES_RID_CURRENTSCALETHRESHOLDS	0xFD46
-#define HERMES_RID_PROTOCOLRSPTIME		0xFD47
-#define HERMES_RID_SHORTRETRYLIMIT		0xFD48
-#define HERMES_RID_LONGRETRYLIMIT		0xFD49
-#define HERMES_RID_MAXTRANSMITLIFETIME		0xFD4A
-#define HERMES_RID_MAXRECEIVELIFETIME		0xFD4B
-#define HERMES_RID_CFPOLLABLE			0xFD4C
-#define HERMES_RID_AUTHENTICATIONALGORITHMS	0xFD4D
-#define HERMES_RID_PRIVACYOPTIONIMPLEMENTED	0xFD4F
-#define HERMES_RID_DBMCOMMSQUALITY_INTERSIL	0xFD51
-#define HERMES_RID_CURRENTTXRATE1		0xFD80
-#define HERMES_RID_CURRENTTXRATE2		0xFD81
-#define HERMES_RID_CURRENTTXRATE3		0xFD82
-#define HERMES_RID_CURRENTTXRATE4		0xFD83
-#define HERMES_RID_CURRENTTXRATE5		0xFD84
-#define HERMES_RID_CURRENTTXRATE6		0xFD85
-#define HERMES_RID_OWNMACADDR			0xFD86
-#define HERMES_RID_SCANRESULTSTABLE		0xFD88
-#define HERMES_RID_CURRENT_COUNTRY_INFO		0xFD89
-#define HERMES_RID_CURRENT_WPA_IE		0xFD8A
-#define HERMES_RID_CURRENT_TKIP_IV		0xFD8B
-#define HERMES_RID_CURRENT_ASSOC_REQ_INFO	0xFD8C
-#define HERMES_RID_CURRENT_ASSOC_RESP_INFO	0xFD8D
-#define HERMES_RID_TXQUEUEEMPTY			0xFD91
-#define HERMES_RID_PHYTYPE			0xFDC0
-#define HERMES_RID_CURRENTCHANNEL		0xFDC1
-#define HERMES_RID_CURRENTPOWERSTATE		0xFDC2
-#define HERMES_RID_CCAMODE			0xFDC3
-#define HERMES_RID_SUPPORTEDDATARATES		0xFDC6
-#define HERMES_RID_BUILDSEQ			0xFFFE
-#define HERMES_RID_FWID				0xFFFF
-
-#endif
diff --git a/drivers/net/wireless/intersil/orinoco/hw.c b/drivers/net/wireless/intersil/orinoco/hw.c
deleted file mode 100644
index 4fcca08e50de..000000000000
--- a/drivers/net/wireless/intersil/orinoco/hw.c
+++ /dev/null
@@ -1,1362 +0,0 @@
-/* Encapsulate basic setting changes and retrieval on Hermes hardware
- *
- * See copyright notice in main.c
- */
-#include <linux/kernel.h>
-#include <linux/device.h>
-#include <linux/if_arp.h>
-#include <linux/ieee80211.h>
-#include <linux/wireless.h>
-#include <net/cfg80211.h>
-#include "hermes.h"
-#include "hermes_rid.h"
-#include "orinoco.h"
-
-#include "hw.h"
-
-#define SYMBOL_MAX_VER_LEN	(14)
-
-/* Symbol firmware has a bug allocating buffers larger than this */
-#define TX_NICBUF_SIZE_BUG	1585
-
-/********************************************************************/
-/* Data tables                                                      */
-/********************************************************************/
-
-/* This tables gives the actual meanings of the bitrate IDs returned
- * by the firmware. */
-static const struct {
-	int bitrate; /* in 100s of kilobits */
-	int automatic;
-	u16 agere_txratectrl;
-	u16 intersil_txratectrl;
-} bitrate_table[] = {
-	{110, 1,  3, 15}, /* Entry 0 is the default */
-	{10,  0,  1,  1},
-	{10,  1,  1,  1},
-	{20,  0,  2,  2},
-	{20,  1,  6,  3},
-	{55,  0,  4,  4},
-	{55,  1,  7,  7},
-	{110, 0,  5,  8},
-};
-#define BITRATE_TABLE_SIZE ARRAY_SIZE(bitrate_table)
-
-/* Firmware version encoding */
-struct comp_id {
-	u16 id, variant, major, minor;
-} __packed;
-
-static inline enum fwtype determine_firmware_type(struct comp_id *nic_id)
-{
-	if (nic_id->id < 0x8000)
-		return FIRMWARE_TYPE_AGERE;
-	else if (nic_id->id == 0x8000 && nic_id->major == 0)
-		return FIRMWARE_TYPE_SYMBOL;
-	else
-		return FIRMWARE_TYPE_INTERSIL;
-}
-
-/* Set priv->firmware type, determine firmware properties
- * This function can be called before we have registerred with netdev,
- * so all errors go out with dev_* rather than printk
- *
- * If non-NULL stores a firmware description in fw_name.
- * If non-NULL stores a HW version in hw_ver
- *
- * These are output via generic cfg80211 ethtool support.
- */
-int determine_fw_capabilities(struct orinoco_private *priv,
-			      char *fw_name, size_t fw_name_len,
-			      u32 *hw_ver)
-{
-	struct device *dev = priv->dev;
-	struct hermes *hw = &priv->hw;
-	int err;
-	struct comp_id nic_id, sta_id;
-	unsigned int firmver;
-	char tmp[SYMBOL_MAX_VER_LEN + 1] __attribute__((aligned(2)));
-
-	/* Get the hardware version */
-	err = HERMES_READ_RECORD_PR(hw, USER_BAP, HERMES_RID_NICID, &nic_id);
-	if (err) {
-		dev_err(dev, "Cannot read hardware identity: error %d\n",
-			err);
-		return err;
-	}
-
-	le16_to_cpus(&nic_id.id);
-	le16_to_cpus(&nic_id.variant);
-	le16_to_cpus(&nic_id.major);
-	le16_to_cpus(&nic_id.minor);
-	dev_info(dev, "Hardware identity %04x:%04x:%04x:%04x\n",
-		 nic_id.id, nic_id.variant, nic_id.major, nic_id.minor);
-
-	if (hw_ver)
-		*hw_ver = (((nic_id.id & 0xff) << 24) |
-			   ((nic_id.variant & 0xff) << 16) |
-			   ((nic_id.major & 0xff) << 8) |
-			   (nic_id.minor & 0xff));
-
-	priv->firmware_type = determine_firmware_type(&nic_id);
-
-	/* Get the firmware version */
-	err = HERMES_READ_RECORD_PR(hw, USER_BAP, HERMES_RID_STAID, &sta_id);
-	if (err) {
-		dev_err(dev, "Cannot read station identity: error %d\n",
-			err);
-		return err;
-	}
-
-	le16_to_cpus(&sta_id.id);
-	le16_to_cpus(&sta_id.variant);
-	le16_to_cpus(&sta_id.major);
-	le16_to_cpus(&sta_id.minor);
-	dev_info(dev, "Station identity  %04x:%04x:%04x:%04x\n",
-		 sta_id.id, sta_id.variant, sta_id.major, sta_id.minor);
-
-	switch (sta_id.id) {
-	case 0x15:
-		dev_err(dev, "Primary firmware is active\n");
-		return -ENODEV;
-	case 0x14b:
-		dev_err(dev, "Tertiary firmware is active\n");
-		return -ENODEV;
-	case 0x1f:	/* Intersil, Agere, Symbol Spectrum24 */
-	case 0x21:	/* Symbol Spectrum24 Trilogy */
-		break;
-	default:
-		dev_notice(dev, "Unknown station ID, please report\n");
-		break;
-	}
-
-	/* Default capabilities */
-	priv->has_sensitivity = 1;
-	priv->has_mwo = 0;
-	priv->has_preamble = 0;
-	priv->has_port3 = 1;
-	priv->has_ibss = 1;
-	priv->has_wep = 0;
-	priv->has_big_wep = 0;
-	priv->has_alt_txcntl = 0;
-	priv->has_ext_scan = 0;
-	priv->has_wpa = 0;
-	priv->do_fw_download = 0;
-
-	/* Determine capabilities from the firmware version */
-	switch (priv->firmware_type) {
-	case FIRMWARE_TYPE_AGERE:
-		/* Lucent Wavelan IEEE, Lucent Orinoco, Cabletron RoamAbout,
-		   ELSA, Melco, HP, IBM, Dell 1150, Compaq 110/210 */
-		if (fw_name)
-			snprintf(fw_name, fw_name_len, "Lucent/Agere %d.%02d",
-				 sta_id.major, sta_id.minor);
-
-		firmver = ((unsigned long)sta_id.major << 16) | sta_id.minor;
-
-		priv->has_ibss = (firmver >= 0x60006);
-		priv->has_wep = (firmver >= 0x40020);
-		priv->has_big_wep = 1; /* FIXME: this is wrong - how do we tell
-					  Gold cards from the others? */
-		priv->has_mwo = (firmver >= 0x60000);
-		priv->has_pm = (firmver >= 0x40020); /* Don't work in 7.52 ? */
-		priv->ibss_port = 1;
-		priv->has_hostscan = (firmver >= 0x8000a);
-		priv->do_fw_download = 1;
-		priv->broken_monitor = (firmver >= 0x80000);
-		priv->has_alt_txcntl = (firmver >= 0x90000); /* All 9.x ? */
-		priv->has_ext_scan = (firmver >= 0x90000); /* All 9.x ? */
-		priv->has_wpa = (firmver >= 0x9002a);
-		/* Tested with Agere firmware :
-		 *	1.16 ; 4.08 ; 4.52 ; 6.04 ; 6.16 ; 7.28 => Jean II
-		 * Tested CableTron firmware : 4.32 => Anton */
-		break;
-	case FIRMWARE_TYPE_SYMBOL:
-		/* Symbol , 3Com AirConnect, Intel, Ericsson WLAN */
-		/* Intel MAC : 00:02:B3:* */
-		/* 3Com MAC : 00:50:DA:* */
-		memset(tmp, 0, sizeof(tmp));
-		/* Get the Symbol firmware version */
-		err = hw->ops->read_ltv_pr(hw, USER_BAP,
-					HERMES_RID_SECONDARYVERSION_SYMBOL,
-					SYMBOL_MAX_VER_LEN, NULL, &tmp);
-		if (err) {
-			dev_warn(dev, "Error %d reading Symbol firmware info. "
-				 "Wildly guessing capabilities...\n", err);
-			firmver = 0;
-			tmp[0] = '\0';
-		} else {
-			/* The firmware revision is a string, the format is
-			 * something like : "V2.20-01".
-			 * Quick and dirty parsing... - Jean II
-			 */
-			firmver = ((tmp[1] - '0') << 16)
-				| ((tmp[3] - '0') << 12)
-				| ((tmp[4] - '0') << 8)
-				| ((tmp[6] - '0') << 4)
-				| (tmp[7] - '0');
-
-			tmp[SYMBOL_MAX_VER_LEN] = '\0';
-		}
-
-		if (fw_name)
-			snprintf(fw_name, fw_name_len, "Symbol %s", tmp);
-
-		priv->has_ibss = (firmver >= 0x20000);
-		priv->has_wep = (firmver >= 0x15012);
-		priv->has_big_wep = (firmver >= 0x20000);
-		priv->has_pm = (firmver >= 0x20000 && firmver < 0x22000) ||
-			       (firmver >= 0x29000 && firmver < 0x30000) ||
-			       firmver >= 0x31000;
-		priv->has_preamble = (firmver >= 0x20000);
-		priv->ibss_port = 4;
-
-		/* Symbol firmware is found on various cards, but
-		 * there has been no attempt to check firmware
-		 * download on non-spectrum_cs based cards.
-		 *
-		 * Given that the Agere firmware download works
-		 * differently, we should avoid doing a firmware
-		 * download with the Symbol algorithm on non-spectrum
-		 * cards.
-		 *
-		 * For now we can identify a spectrum_cs based card
-		 * because it has a firmware reset function.
-		 */
-		priv->do_fw_download = (priv->stop_fw != NULL);
-
-		priv->broken_disableport = (firmver == 0x25013) ||
-				(firmver >= 0x30000 && firmver <= 0x31000);
-		priv->has_hostscan = (firmver >= 0x31001) ||
-				     (firmver >= 0x29057 && firmver < 0x30000);
-		/* Tested with Intel firmware : 0x20015 => Jean II */
-		/* Tested with 3Com firmware : 0x15012 & 0x22001 => Jean II */
-		break;
-	case FIRMWARE_TYPE_INTERSIL:
-		/* D-Link, Linksys, Adtron, ZoomAir, and many others...
-		 * Samsung, Compaq 100/200 and Proxim are slightly
-		 * different and less well tested */
-		/* D-Link MAC : 00:40:05:* */
-		/* Addtron MAC : 00:90:D1:* */
-		if (fw_name)
-			snprintf(fw_name, fw_name_len, "Intersil %d.%d.%d",
-				 sta_id.major, sta_id.minor, sta_id.variant);
-
-		firmver = ((unsigned long)sta_id.major << 16) |
-			((unsigned long)sta_id.minor << 8) | sta_id.variant;
-
-		priv->has_ibss = (firmver >= 0x000700); /* FIXME */
-		priv->has_big_wep = priv->has_wep = (firmver >= 0x000800);
-		priv->has_pm = (firmver >= 0x000700);
-		priv->has_hostscan = (firmver >= 0x010301);
-
-		if (firmver >= 0x000800)
-			priv->ibss_port = 0;
-		else {
-			dev_notice(dev, "Intersil firmware earlier than v0.8.x"
-				   " - several features not supported\n");
-			priv->ibss_port = 1;
-		}
-		break;
-	}
-	if (fw_name)
-		dev_info(dev, "Firmware determined as %s\n", fw_name);
-
-#ifndef CONFIG_HERMES_PRISM
-	if (priv->firmware_type == FIRMWARE_TYPE_INTERSIL) {
-		dev_err(dev, "Support for Prism chipset is not enabled\n");
-		return -ENODEV;
-	}
-#endif
-
-	return 0;
-}
-
-/* Read settings from EEPROM into our private structure.
- * MAC address gets dropped into callers buffer
- * Can be called before netdev registration.
- */
-int orinoco_hw_read_card_settings(struct orinoco_private *priv, u8 *dev_addr)
-{
-	struct device *dev = priv->dev;
-	struct hermes_idstring nickbuf;
-	struct hermes *hw = &priv->hw;
-	int len;
-	int err;
-	u16 reclen;
-
-	/* Get the MAC address */
-	err = hw->ops->read_ltv_pr(hw, USER_BAP, HERMES_RID_CNFOWNMACADDR,
-				ETH_ALEN, NULL, dev_addr);
-	if (err) {
-		dev_warn(dev, "Failed to read MAC address!\n");
-		goto out;
-	}
-
-	dev_dbg(dev, "MAC address %pM\n", dev_addr);
-
-	/* Get the station name */
-	err = hw->ops->read_ltv_pr(hw, USER_BAP, HERMES_RID_CNFOWNNAME,
-				sizeof(nickbuf), &reclen, &nickbuf);
-	if (err) {
-		dev_err(dev, "failed to read station name\n");
-		goto out;
-	}
-	if (nickbuf.len)
-		len = min(IW_ESSID_MAX_SIZE, (int)le16_to_cpu(nickbuf.len));
-	else
-		len = min(IW_ESSID_MAX_SIZE, 2 * reclen);
-	memcpy(priv->nick, &nickbuf.val, len);
-	priv->nick[len] = '\0';
-
-	dev_dbg(dev, "Station name \"%s\"\n", priv->nick);
-
-	/* Get allowed channels */
-	err = hermes_read_wordrec_pr(hw, USER_BAP, HERMES_RID_CHANNELLIST,
-				  &priv->channel_mask);
-	if (err) {
-		dev_err(dev, "Failed to read channel list!\n");
-		goto out;
-	}
-
-	/* Get initial AP density */
-	err = hermes_read_wordrec_pr(hw, USER_BAP, HERMES_RID_CNFSYSTEMSCALE,
-				  &priv->ap_density);
-	if (err || priv->ap_density < 1 || priv->ap_density > 3)
-		priv->has_sensitivity = 0;
-
-	/* Get initial RTS threshold */
-	err = hermes_read_wordrec_pr(hw, USER_BAP, HERMES_RID_CNFRTSTHRESHOLD,
-				  &priv->rts_thresh);
-	if (err) {
-		dev_err(dev, "Failed to read RTS threshold!\n");
-		goto out;
-	}
-
-	/* Get initial fragmentation settings */
-	if (priv->has_mwo)
-		err = hermes_read_wordrec_pr(hw, USER_BAP,
-					  HERMES_RID_CNFMWOROBUST_AGERE,
-					  &priv->mwo_robust);
-	else
-		err = hermes_read_wordrec_pr(hw, USER_BAP,
-					  HERMES_RID_CNFFRAGMENTATIONTHRESHOLD,
-					  &priv->frag_thresh);
-	if (err) {
-		dev_err(dev, "Failed to read fragmentation settings!\n");
-		goto out;
-	}
-
-	/* Power management setup */
-	if (priv->has_pm) {
-		priv->pm_on = 0;
-		priv->pm_mcast = 1;
-		err = hermes_read_wordrec_pr(hw, USER_BAP,
-					  HERMES_RID_CNFMAXSLEEPDURATION,
-					  &priv->pm_period);
-		if (err) {
-			dev_err(dev, "Failed to read power management "
-				"period!\n");
-			goto out;
-		}
-		err = hermes_read_wordrec_pr(hw, USER_BAP,
-					  HERMES_RID_CNFPMHOLDOVERDURATION,
-					  &priv->pm_timeout);
-		if (err) {
-			dev_err(dev, "Failed to read power management "
-				"timeout!\n");
-			goto out;
-		}
-	}
-
-	/* Preamble setup */
-	if (priv->has_preamble) {
-		err = hermes_read_wordrec_pr(hw, USER_BAP,
-					  HERMES_RID_CNFPREAMBLE_SYMBOL,
-					  &priv->preamble);
-		if (err) {
-			dev_err(dev, "Failed to read preamble setup\n");
-			goto out;
-		}
-	}
-
-	/* Retry settings */
-	err = hermes_read_wordrec_pr(hw, USER_BAP, HERMES_RID_SHORTRETRYLIMIT,
-				  &priv->short_retry_limit);
-	if (err) {
-		dev_err(dev, "Failed to read short retry limit\n");
-		goto out;
-	}
-
-	err = hermes_read_wordrec_pr(hw, USER_BAP, HERMES_RID_LONGRETRYLIMIT,
-				  &priv->long_retry_limit);
-	if (err) {
-		dev_err(dev, "Failed to read long retry limit\n");
-		goto out;
-	}
-
-	err = hermes_read_wordrec_pr(hw, USER_BAP, HERMES_RID_MAXTRANSMITLIFETIME,
-				  &priv->retry_lifetime);
-	if (err) {
-		dev_err(dev, "Failed to read max retry lifetime\n");
-		goto out;
-	}
-
-out:
-	return err;
-}
-
-/* Can be called before netdev registration */
-int orinoco_hw_allocate_fid(struct orinoco_private *priv)
-{
-	struct device *dev = priv->dev;
-	struct hermes *hw = &priv->hw;
-	int err;
-
-	err = hw->ops->allocate(hw, priv->nicbuf_size, &priv->txfid);
-	if (err == -EIO && priv->nicbuf_size > TX_NICBUF_SIZE_BUG) {
-		/* Try workaround for old Symbol firmware bug */
-		priv->nicbuf_size = TX_NICBUF_SIZE_BUG;
-		err = hw->ops->allocate(hw, priv->nicbuf_size, &priv->txfid);
-
-		dev_warn(dev, "Firmware ALLOC bug detected "
-			 "(old Symbol firmware?). Work around %s\n",
-			 err ? "failed!" : "ok.");
-	}
-
-	return err;
-}
-
-int orinoco_get_bitratemode(int bitrate, int automatic)
-{
-	int ratemode = -1;
-	int i;
-
-	if ((bitrate != 10) && (bitrate != 20) &&
-	    (bitrate != 55) && (bitrate != 110))
-		return ratemode;
-
-	for (i = 0; i < BITRATE_TABLE_SIZE; i++) {
-		if ((bitrate_table[i].bitrate == bitrate) &&
-		    (bitrate_table[i].automatic == automatic)) {
-			ratemode = i;
-			break;
-		}
-	}
-	return ratemode;
-}
-
-void orinoco_get_ratemode_cfg(int ratemode, int *bitrate, int *automatic)
-{
-	BUG_ON((ratemode < 0) || (ratemode >= BITRATE_TABLE_SIZE));
-
-	*bitrate = bitrate_table[ratemode].bitrate * 100000;
-	*automatic = bitrate_table[ratemode].automatic;
-}
-
-int orinoco_hw_program_rids(struct orinoco_private *priv)
-{
-	struct net_device *dev = priv->ndev;
-	struct wireless_dev *wdev = netdev_priv(dev);
-	struct hermes *hw = &priv->hw;
-	int err;
-	struct hermes_idstring idbuf;
-
-	/* Set the MAC address */
-	err = hw->ops->write_ltv(hw, USER_BAP, HERMES_RID_CNFOWNMACADDR,
-				 HERMES_BYTES_TO_RECLEN(ETH_ALEN),
-				 dev->dev_addr);
-	if (err) {
-		printk(KERN_ERR "%s: Error %d setting MAC address\n",
-		       dev->name, err);
-		return err;
-	}
-
-	/* Set up the link mode */
-	err = hermes_write_wordrec(hw, USER_BAP, HERMES_RID_CNFPORTTYPE,
-				   priv->port_type);
-	if (err) {
-		printk(KERN_ERR "%s: Error %d setting port type\n",
-		       dev->name, err);
-		return err;
-	}
-	/* Set the channel/frequency */
-	if (priv->channel != 0 && priv->iw_mode != NL80211_IFTYPE_STATION) {
-		err = hermes_write_wordrec(hw, USER_BAP,
-					   HERMES_RID_CNFOWNCHANNEL,
-					   priv->channel);
-		if (err) {
-			printk(KERN_ERR "%s: Error %d setting channel %d\n",
-			       dev->name, err, priv->channel);
-			return err;
-		}
-	}
-
-	if (priv->has_ibss) {
-		u16 createibss;
-
-		if ((strlen(priv->desired_essid) == 0) && (priv->createibss)) {
-			printk(KERN_WARNING "%s: This firmware requires an "
-			       "ESSID in IBSS-Ad-Hoc mode.\n", dev->name);
-			/* With wvlan_cs, in this case, we would crash.
-			 * hopefully, this driver will behave better...
-			 * Jean II */
-			createibss = 0;
-		} else {
-			createibss = priv->createibss;
-		}
-
-		err = hermes_write_wordrec(hw, USER_BAP,
-					   HERMES_RID_CNFCREATEIBSS,
-					   createibss);
-		if (err) {
-			printk(KERN_ERR "%s: Error %d setting CREATEIBSS\n",
-			       dev->name, err);
-			return err;
-		}
-	}
-
-	/* Set the desired BSSID */
-	err = __orinoco_hw_set_wap(priv);
-	if (err) {
-		printk(KERN_ERR "%s: Error %d setting AP address\n",
-		       dev->name, err);
-		return err;
-	}
-
-	/* Set the desired ESSID */
-	idbuf.len = cpu_to_le16(strlen(priv->desired_essid));
-	memcpy(&idbuf.val, priv->desired_essid, sizeof(idbuf.val));
-	/* WinXP wants partner to configure OWNSSID even in IBSS mode. (jimc) */
-	err = hw->ops->write_ltv(hw, USER_BAP, HERMES_RID_CNFOWNSSID,
-			HERMES_BYTES_TO_RECLEN(strlen(priv->desired_essid) + 2),
-			&idbuf);
-	if (err) {
-		printk(KERN_ERR "%s: Error %d setting OWNSSID\n",
-		       dev->name, err);
-		return err;
-	}
-	err = hw->ops->write_ltv(hw, USER_BAP, HERMES_RID_CNFDESIREDSSID,
-			HERMES_BYTES_TO_RECLEN(strlen(priv->desired_essid) + 2),
-			&idbuf);
-	if (err) {
-		printk(KERN_ERR "%s: Error %d setting DESIREDSSID\n",
-		       dev->name, err);
-		return err;
-	}
-
-	/* Set the station name */
-	idbuf.len = cpu_to_le16(strlen(priv->nick));
-	memcpy(&idbuf.val, priv->nick, sizeof(idbuf.val));
-	err = hw->ops->write_ltv(hw, USER_BAP, HERMES_RID_CNFOWNNAME,
-				 HERMES_BYTES_TO_RECLEN(strlen(priv->nick) + 2),
-				 &idbuf);
-	if (err) {
-		printk(KERN_ERR "%s: Error %d setting nickname\n",
-		       dev->name, err);
-		return err;
-	}
-
-	/* Set AP density */
-	if (priv->has_sensitivity) {
-		err = hermes_write_wordrec(hw, USER_BAP,
-					   HERMES_RID_CNFSYSTEMSCALE,
-					   priv->ap_density);
-		if (err) {
-			printk(KERN_WARNING "%s: Error %d setting SYSTEMSCALE. "
-			       "Disabling sensitivity control\n",
-			       dev->name, err);
-
-			priv->has_sensitivity = 0;
-		}
-	}
-
-	/* Set RTS threshold */
-	err = hermes_write_wordrec(hw, USER_BAP, HERMES_RID_CNFRTSTHRESHOLD,
-				   priv->rts_thresh);
-	if (err) {
-		printk(KERN_ERR "%s: Error %d setting RTS threshold\n",
-		       dev->name, err);
-		return err;
-	}
-
-	/* Set fragmentation threshold or MWO robustness */
-	if (priv->has_mwo)
-		err = hermes_write_wordrec(hw, USER_BAP,
-					   HERMES_RID_CNFMWOROBUST_AGERE,
-					   priv->mwo_robust);
-	else
-		err = hermes_write_wordrec(hw, USER_BAP,
-					   HERMES_RID_CNFFRAGMENTATIONTHRESHOLD,
-					   priv->frag_thresh);
-	if (err) {
-		printk(KERN_ERR "%s: Error %d setting fragmentation\n",
-		       dev->name, err);
-		return err;
-	}
-
-	/* Set bitrate */
-	err = __orinoco_hw_set_bitrate(priv);
-	if (err) {
-		printk(KERN_ERR "%s: Error %d setting bitrate\n",
-		       dev->name, err);
-		return err;
-	}
-
-	/* Set power management */
-	if (priv->has_pm) {
-		err = hermes_write_wordrec(hw, USER_BAP,
-					   HERMES_RID_CNFPMENABLED,
-					   priv->pm_on);
-		if (err) {
-			printk(KERN_ERR "%s: Error %d setting up PM\n",
-			       dev->name, err);
-			return err;
-		}
-
-		err = hermes_write_wordrec(hw, USER_BAP,
-					   HERMES_RID_CNFMULTICASTRECEIVE,
-					   priv->pm_mcast);
-		if (err) {
-			printk(KERN_ERR "%s: Error %d setting up PM\n",
-			       dev->name, err);
-			return err;
-		}
-		err = hermes_write_wordrec(hw, USER_BAP,
-					   HERMES_RID_CNFMAXSLEEPDURATION,
-					   priv->pm_period);
-		if (err) {
-			printk(KERN_ERR "%s: Error %d setting up PM\n",
-			       dev->name, err);
-			return err;
-		}
-		err = hermes_write_wordrec(hw, USER_BAP,
-					   HERMES_RID_CNFPMHOLDOVERDURATION,
-					   priv->pm_timeout);
-		if (err) {
-			printk(KERN_ERR "%s: Error %d setting up PM\n",
-			       dev->name, err);
-			return err;
-		}
-	}
-
-	/* Set preamble - only for Symbol so far... */
-	if (priv->has_preamble) {
-		err = hermes_write_wordrec(hw, USER_BAP,
-					   HERMES_RID_CNFPREAMBLE_SYMBOL,
-					   priv->preamble);
-		if (err) {
-			printk(KERN_ERR "%s: Error %d setting preamble\n",
-			       dev->name, err);
-			return err;
-		}
-	}
-
-	/* Set up encryption */
-	if (priv->has_wep || priv->has_wpa) {
-		err = __orinoco_hw_setup_enc(priv);
-		if (err) {
-			printk(KERN_ERR "%s: Error %d activating encryption\n",
-			       dev->name, err);
-			return err;
-		}
-	}
-
-	if (priv->iw_mode == NL80211_IFTYPE_MONITOR) {
-		/* Enable monitor mode */
-		dev->type = ARPHRD_IEEE80211;
-		err = hw->ops->cmd_wait(hw, HERMES_CMD_TEST |
-					    HERMES_TEST_MONITOR, 0, NULL);
-	} else {
-		/* Disable monitor mode */
-		dev->type = ARPHRD_ETHER;
-		err = hw->ops->cmd_wait(hw, HERMES_CMD_TEST |
-					    HERMES_TEST_STOP, 0, NULL);
-	}
-	if (err)
-		return err;
-
-	/* Reset promiscuity / multicast*/
-	priv->promiscuous = 0;
-	priv->mc_count = 0;
-
-	/* Record mode change */
-	wdev->iftype = priv->iw_mode;
-
-	return 0;
-}
-
-/* Get tsc from the firmware */
-int orinoco_hw_get_tkip_iv(struct orinoco_private *priv, int key, u8 *tsc)
-{
-	struct hermes *hw = &priv->hw;
-	int err = 0;
-	u8 tsc_arr[4][ORINOCO_SEQ_LEN];
-
-	if ((key < 0) || (key >= 4))
-		return -EINVAL;
-
-	err = hw->ops->read_ltv(hw, USER_BAP, HERMES_RID_CURRENT_TKIP_IV,
-				sizeof(tsc_arr), NULL, &tsc_arr);
-	if (!err)
-		memcpy(tsc, &tsc_arr[key][0], sizeof(tsc_arr[0]));
-
-	return err;
-}
-
-int __orinoco_hw_set_bitrate(struct orinoco_private *priv)
-{
-	struct hermes *hw = &priv->hw;
-	int ratemode = priv->bitratemode;
-	int err = 0;
-
-	if (ratemode >= BITRATE_TABLE_SIZE) {
-		printk(KERN_ERR "%s: BUG: Invalid bitrate mode %d\n",
-		       priv->ndev->name, ratemode);
-		return -EINVAL;
-	}
-
-	switch (priv->firmware_type) {
-	case FIRMWARE_TYPE_AGERE:
-		err = hermes_write_wordrec(hw, USER_BAP,
-				HERMES_RID_CNFTXRATECONTROL,
-				bitrate_table[ratemode].agere_txratectrl);
-		break;
-	case FIRMWARE_TYPE_INTERSIL:
-	case FIRMWARE_TYPE_SYMBOL:
-		err = hermes_write_wordrec(hw, USER_BAP,
-				HERMES_RID_CNFTXRATECONTROL,
-				bitrate_table[ratemode].intersil_txratectrl);
-		break;
-	default:
-		BUG();
-	}
-
-	return err;
-}
-
-int orinoco_hw_get_act_bitrate(struct orinoco_private *priv, int *bitrate)
-{
-	struct hermes *hw = &priv->hw;
-	int i;
-	int err = 0;
-	u16 val;
-
-	err = hermes_read_wordrec(hw, USER_BAP,
-				  HERMES_RID_CURRENTTXRATE, &val);
-	if (err)
-		return err;
-
-	switch (priv->firmware_type) {
-	case FIRMWARE_TYPE_AGERE: /* Lucent style rate */
-		/* Note : in Lucent firmware, the return value of
-		 * HERMES_RID_CURRENTTXRATE is the bitrate in Mb/s,
-		 * and therefore is totally different from the
-		 * encoding of HERMES_RID_CNFTXRATECONTROL.
-		 * Don't forget that 6Mb/s is really 5.5Mb/s */
-		if (val == 6)
-			*bitrate = 5500000;
-		else
-			*bitrate = val * 1000000;
-		break;
-	case FIRMWARE_TYPE_INTERSIL: /* Intersil style rate */
-	case FIRMWARE_TYPE_SYMBOL: /* Symbol style rate */
-		for (i = 0; i < BITRATE_TABLE_SIZE; i++)
-			if (bitrate_table[i].intersil_txratectrl == val) {
-				*bitrate = bitrate_table[i].bitrate * 100000;
-				break;
-			}
-
-		if (i >= BITRATE_TABLE_SIZE) {
-			printk(KERN_INFO "%s: Unable to determine current bitrate (0x%04hx)\n",
-			       priv->ndev->name, val);
-			err = -EIO;
-		}
-
-		break;
-	default:
-		BUG();
-	}
-
-	return err;
-}
-
-/* Set fixed AP address */
-int __orinoco_hw_set_wap(struct orinoco_private *priv)
-{
-	int roaming_flag;
-	int err = 0;
-	struct hermes *hw = &priv->hw;
-
-	switch (priv->firmware_type) {
-	case FIRMWARE_TYPE_AGERE:
-		/* not supported */
-		break;
-	case FIRMWARE_TYPE_INTERSIL:
-		if (priv->bssid_fixed)
-			roaming_flag = 2;
-		else
-			roaming_flag = 1;
-
-		err = hermes_write_wordrec(hw, USER_BAP,
-					   HERMES_RID_CNFROAMINGMODE,
-					   roaming_flag);
-		break;
-	case FIRMWARE_TYPE_SYMBOL:
-		err = HERMES_WRITE_RECORD(hw, USER_BAP,
-					  HERMES_RID_CNFMANDATORYBSSID_SYMBOL,
-					  &priv->desired_bssid);
-		break;
-	}
-	return err;
-}
-
-/* Change the WEP keys and/or the current keys.  Can be called
- * either from __orinoco_hw_setup_enc() or directly from
- * orinoco_ioctl_setiwencode().  In the later case the association
- * with the AP is not broken (if the firmware can handle it),
- * which is needed for 802.1x implementations. */
-int __orinoco_hw_setup_wepkeys(struct orinoco_private *priv)
-{
-	struct hermes *hw = &priv->hw;
-	int err = 0;
-	int i;
-
-	switch (priv->firmware_type) {
-	case FIRMWARE_TYPE_AGERE:
-	{
-		struct orinoco_key keys[ORINOCO_MAX_KEYS];
-
-		memset(&keys, 0, sizeof(keys));
-		for (i = 0; i < ORINOCO_MAX_KEYS; i++) {
-			int len = min(priv->keys[i].key_len,
-				      ORINOCO_MAX_KEY_SIZE);
-			memcpy(&keys[i].data, priv->keys[i].key, len);
-			if (len > SMALL_KEY_SIZE)
-				keys[i].len = cpu_to_le16(LARGE_KEY_SIZE);
-			else if (len > 0)
-				keys[i].len = cpu_to_le16(SMALL_KEY_SIZE);
-			else
-				keys[i].len = cpu_to_le16(0);
-		}
-
-		err = HERMES_WRITE_RECORD(hw, USER_BAP,
-					  HERMES_RID_CNFWEPKEYS_AGERE,
-					  &keys);
-		if (err)
-			return err;
-		err = hermes_write_wordrec(hw, USER_BAP,
-					   HERMES_RID_CNFTXKEY_AGERE,
-					   priv->tx_key);
-		if (err)
-			return err;
-		break;
-	}
-	case FIRMWARE_TYPE_INTERSIL:
-	case FIRMWARE_TYPE_SYMBOL:
-		{
-			int keylen;
-
-			/* Force uniform key length to work around
-			 * firmware bugs */
-			keylen = priv->keys[priv->tx_key].key_len;
-
-			if (keylen > LARGE_KEY_SIZE) {
-				printk(KERN_ERR "%s: BUG: Key %d has oversize length %d.\n",
-				       priv->ndev->name, priv->tx_key, keylen);
-				return -E2BIG;
-			} else if (keylen > SMALL_KEY_SIZE)
-				keylen = LARGE_KEY_SIZE;
-			else if (keylen > 0)
-				keylen = SMALL_KEY_SIZE;
-			else
-				keylen = 0;
-
-			/* Write all 4 keys */
-			for (i = 0; i < ORINOCO_MAX_KEYS; i++) {
-				u8 key[LARGE_KEY_SIZE] = { 0 };
-
-				memcpy(key, priv->keys[i].key,
-				       priv->keys[i].key_len);
-
-				err = hw->ops->write_ltv(hw, USER_BAP,
-						HERMES_RID_CNFDEFAULTKEY0 + i,
-						HERMES_BYTES_TO_RECLEN(keylen),
-						key);
-				if (err)
-					return err;
-			}
-
-			/* Write the index of the key used in transmission */
-			err = hermes_write_wordrec(hw, USER_BAP,
-						HERMES_RID_CNFWEPDEFAULTKEYID,
-						priv->tx_key);
-			if (err)
-				return err;
-		}
-		break;
-	}
-
-	return 0;
-}
-
-int __orinoco_hw_setup_enc(struct orinoco_private *priv)
-{
-	struct hermes *hw = &priv->hw;
-	int err = 0;
-	int master_wep_flag;
-	int auth_flag;
-	int enc_flag;
-
-	/* Setup WEP keys */
-	if (priv->encode_alg == ORINOCO_ALG_WEP)
-		__orinoco_hw_setup_wepkeys(priv);
-
-	if (priv->wep_restrict)
-		auth_flag = HERMES_AUTH_SHARED_KEY;
-	else
-		auth_flag = HERMES_AUTH_OPEN;
-
-	if (priv->wpa_enabled)
-		enc_flag = 2;
-	else if (priv->encode_alg == ORINOCO_ALG_WEP)
-		enc_flag = 1;
-	else
-		enc_flag = 0;
-
-	switch (priv->firmware_type) {
-	case FIRMWARE_TYPE_AGERE: /* Agere style WEP */
-		if (priv->encode_alg == ORINOCO_ALG_WEP) {
-			/* Enable the shared-key authentication. */
-			err = hermes_write_wordrec(hw, USER_BAP,
-					HERMES_RID_CNFAUTHENTICATION_AGERE,
-					auth_flag);
-			if (err)
-				return err;
-		}
-		err = hermes_write_wordrec(hw, USER_BAP,
-					   HERMES_RID_CNFWEPENABLED_AGERE,
-					   enc_flag);
-		if (err)
-			return err;
-
-		if (priv->has_wpa) {
-			/* Set WPA key management */
-			err = hermes_write_wordrec(hw, USER_BAP,
-				  HERMES_RID_CNFSETWPAAUTHMGMTSUITE_AGERE,
-				  priv->key_mgmt);
-			if (err)
-				return err;
-		}
-
-		break;
-
-	case FIRMWARE_TYPE_INTERSIL: /* Intersil style WEP */
-	case FIRMWARE_TYPE_SYMBOL: /* Symbol style WEP */
-		if (priv->encode_alg == ORINOCO_ALG_WEP) {
-			if (priv->wep_restrict ||
-			    (priv->firmware_type == FIRMWARE_TYPE_SYMBOL))
-				master_wep_flag = HERMES_WEP_PRIVACY_INVOKED |
-						  HERMES_WEP_EXCL_UNENCRYPTED;
-			else
-				master_wep_flag = HERMES_WEP_PRIVACY_INVOKED;
-
-			err = hermes_write_wordrec(hw, USER_BAP,
-						   HERMES_RID_CNFAUTHENTICATION,
-						   auth_flag);
-			if (err)
-				return err;
-		} else
-			master_wep_flag = 0;
-
-		if (priv->iw_mode == NL80211_IFTYPE_MONITOR)
-			master_wep_flag |= HERMES_WEP_HOST_DECRYPT;
-
-		/* Master WEP setting : on/off */
-		err = hermes_write_wordrec(hw, USER_BAP,
-					   HERMES_RID_CNFWEPFLAGS_INTERSIL,
-					   master_wep_flag);
-		if (err)
-			return err;
-
-		break;
-	}
-
-	return 0;
-}
-
-/* key must be 32 bytes, including the tx and rx MIC keys.
- * rsc must be NULL or up to 8 bytes
- * tsc must be NULL or up to 8 bytes
- */
-int __orinoco_hw_set_tkip_key(struct orinoco_private *priv, int key_idx,
-			      int set_tx, const u8 *key, size_t key_len,
-			      const u8 *rsc, size_t rsc_len,
-			      const u8 *tsc, size_t tsc_len)
-{
-	struct {
-		__le16 idx;
-		u8 rsc[ORINOCO_SEQ_LEN];
-		struct {
-			u8 key[TKIP_KEYLEN];
-			u8 tx_mic[MIC_KEYLEN];
-			u8 rx_mic[MIC_KEYLEN];
-		} tkip;
-		u8 tsc[ORINOCO_SEQ_LEN];
-	} __packed buf;
-	struct hermes *hw = &priv->hw;
-	int ret;
-	int err;
-	int k;
-	u16 xmitting;
-
-	key_idx &= 0x3;
-
-	if (set_tx)
-		key_idx |= 0x8000;
-
-	buf.idx = cpu_to_le16(key_idx);
-	if (key_len != sizeof(buf.tkip))
-		return -EINVAL;
-	memcpy(&buf.tkip, key, sizeof(buf.tkip));
-
-	if (rsc_len > sizeof(buf.rsc))
-		rsc_len = sizeof(buf.rsc);
-
-	if (tsc_len > sizeof(buf.tsc))
-		tsc_len = sizeof(buf.tsc);
-
-	memset(buf.rsc, 0, sizeof(buf.rsc));
-	memset(buf.tsc, 0, sizeof(buf.tsc));
-
-	if (rsc != NULL)
-		memcpy(buf.rsc, rsc, rsc_len);
-
-	if (tsc != NULL)
-		memcpy(buf.tsc, tsc, tsc_len);
-	else
-		buf.tsc[4] = 0x10;
-
-	/* Wait up to 100ms for tx queue to empty */
-	for (k = 100; k > 0; k--) {
-		udelay(1000);
-		ret = hermes_read_wordrec(hw, USER_BAP, HERMES_RID_TXQUEUEEMPTY,
-					  &xmitting);
-		if (ret || !xmitting)
-			break;
-	}
-
-	if (k == 0)
-		ret = -ETIMEDOUT;
-
-	err = HERMES_WRITE_RECORD(hw, USER_BAP,
-				  HERMES_RID_CNFADDDEFAULTTKIPKEY_AGERE,
-				  &buf);
-
-	return ret ? ret : err;
-}
-
-int orinoco_clear_tkip_key(struct orinoco_private *priv, int key_idx)
-{
-	struct hermes *hw = &priv->hw;
-	int err;
-
-	err = hermes_write_wordrec(hw, USER_BAP,
-				   HERMES_RID_CNFREMDEFAULTTKIPKEY_AGERE,
-				   key_idx);
-	if (err)
-		printk(KERN_WARNING "%s: Error %d clearing TKIP key %d\n",
-		       priv->ndev->name, err, key_idx);
-	return err;
-}
-
-int __orinoco_hw_set_multicast_list(struct orinoco_private *priv,
-				    struct net_device *dev,
-				    int mc_count, int promisc)
-{
-	struct hermes *hw = &priv->hw;
-	int err = 0;
-
-	if (promisc != priv->promiscuous) {
-		err = hermes_write_wordrec(hw, USER_BAP,
-					   HERMES_RID_CNFPROMISCUOUSMODE,
-					   promisc);
-		if (err) {
-			printk(KERN_ERR "%s: Error %d setting PROMISCUOUSMODE to 1.\n",
-			       priv->ndev->name, err);
-		} else
-			priv->promiscuous = promisc;
-	}
-
-	/* If we're not in promiscuous mode, then we need to set the
-	 * group address if either we want to multicast, or if we were
-	 * multicasting and want to stop */
-	if (!promisc && (mc_count || priv->mc_count)) {
-		struct netdev_hw_addr *ha;
-		struct hermes_multicast mclist;
-		int i = 0;
-
-		netdev_for_each_mc_addr(ha, dev) {
-			if (i == mc_count)
-				break;
-			memcpy(mclist.addr[i++], ha->addr, ETH_ALEN);
-		}
-
-		err = hw->ops->write_ltv(hw, USER_BAP,
-				   HERMES_RID_CNFGROUPADDRESSES,
-				   HERMES_BYTES_TO_RECLEN(mc_count * ETH_ALEN),
-				   &mclist);
-		if (err)
-			printk(KERN_ERR "%s: Error %d setting multicast list.\n",
-			       priv->ndev->name, err);
-		else
-			priv->mc_count = mc_count;
-	}
-	return err;
-}
-
-/* Return : < 0 -> error code ; >= 0 -> length */
-int orinoco_hw_get_essid(struct orinoco_private *priv, int *active,
-			 char buf[IW_ESSID_MAX_SIZE + 1])
-{
-	struct hermes *hw = &priv->hw;
-	int err = 0;
-	struct hermes_idstring essidbuf;
-	char *p = (char *)(&essidbuf.val);
-	int len;
-	unsigned long flags;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	if (strlen(priv->desired_essid) > 0) {
-		/* We read the desired SSID from the hardware rather
-		   than from priv->desired_essid, just in case the
-		   firmware is allowed to change it on us. I'm not
-		   sure about this */
-		/* My guess is that the OWNSSID should always be whatever
-		 * we set to the card, whereas CURRENT_SSID is the one that
-		 * may change... - Jean II */
-		u16 rid;
-
-		*active = 1;
-
-		rid = (priv->port_type == 3) ? HERMES_RID_CNFOWNSSID :
-			HERMES_RID_CNFDESIREDSSID;
-
-		err = hw->ops->read_ltv(hw, USER_BAP, rid, sizeof(essidbuf),
-					NULL, &essidbuf);
-		if (err)
-			goto fail_unlock;
-	} else {
-		*active = 0;
-
-		err = hw->ops->read_ltv(hw, USER_BAP, HERMES_RID_CURRENTSSID,
-					sizeof(essidbuf), NULL, &essidbuf);
-		if (err)
-			goto fail_unlock;
-	}
-
-	len = le16_to_cpu(essidbuf.len);
-	BUG_ON(len > IW_ESSID_MAX_SIZE);
-
-	memset(buf, 0, IW_ESSID_MAX_SIZE);
-	memcpy(buf, p, len);
-	err = len;
-
- fail_unlock:
-	orinoco_unlock(priv, &flags);
-
-	return err;
-}
-
-int orinoco_hw_get_freq(struct orinoco_private *priv)
-{
-	struct hermes *hw = &priv->hw;
-	int err = 0;
-	u16 channel;
-	int freq = 0;
-	unsigned long flags;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	err = hermes_read_wordrec(hw, USER_BAP, HERMES_RID_CURRENTCHANNEL,
-				  &channel);
-	if (err)
-		goto out;
-
-	/* Intersil firmware 1.3.5 returns 0 when the interface is down */
-	if (channel == 0) {
-		err = -EBUSY;
-		goto out;
-	}
-
-	if ((channel < 1) || (channel > NUM_CHANNELS)) {
-		printk(KERN_WARNING "%s: Channel out of range (%d)!\n",
-		       priv->ndev->name, channel);
-		err = -EBUSY;
-		goto out;
-
-	}
-	freq = ieee80211_channel_to_frequency(channel, NL80211_BAND_2GHZ);
-
- out:
-	orinoco_unlock(priv, &flags);
-
-	if (err > 0)
-		err = -EBUSY;
-	return err ? err : freq;
-}
-
-int orinoco_hw_get_bitratelist(struct orinoco_private *priv,
-			       int *numrates, s32 *rates, int max)
-{
-	struct hermes *hw = &priv->hw;
-	struct hermes_idstring list;
-	unsigned char *p = (unsigned char *)&list.val;
-	int err = 0;
-	int num;
-	int i;
-	unsigned long flags;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	err = hw->ops->read_ltv(hw, USER_BAP, HERMES_RID_SUPPORTEDDATARATES,
-				sizeof(list), NULL, &list);
-	orinoco_unlock(priv, &flags);
-
-	if (err)
-		return err;
-
-	num = le16_to_cpu(list.len);
-	*numrates = num;
-	num = min(num, max);
-
-	for (i = 0; i < num; i++)
-		rates[i] = (p[i] & 0x7f) * 500000; /* convert to bps */
-
-	return 0;
-}
-
-int orinoco_hw_trigger_scan(struct orinoco_private *priv,
-			    const struct cfg80211_ssid *ssid)
-{
-	struct net_device *dev = priv->ndev;
-	struct hermes *hw = &priv->hw;
-	unsigned long flags;
-	int err = 0;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	/* Scanning with port 0 disabled would fail */
-	if (!netif_running(dev)) {
-		err = -ENETDOWN;
-		goto out;
-	}
-
-	/* In monitor mode, the scan results are always empty.
-	 * Probe responses are passed to the driver as received
-	 * frames and could be processed in software. */
-	if (priv->iw_mode == NL80211_IFTYPE_MONITOR) {
-		err = -EOPNOTSUPP;
-		goto out;
-	}
-
-	if (priv->has_hostscan) {
-		switch (priv->firmware_type) {
-		case FIRMWARE_TYPE_SYMBOL:
-			err = hermes_write_wordrec(hw, USER_BAP,
-						HERMES_RID_CNFHOSTSCAN_SYMBOL,
-						HERMES_HOSTSCAN_SYMBOL_ONCE |
-						HERMES_HOSTSCAN_SYMBOL_BCAST);
-			break;
-		case FIRMWARE_TYPE_INTERSIL: {
-			__le16 req[3];
-
-			req[0] = cpu_to_le16(0x3fff);	/* All channels */
-			req[1] = cpu_to_le16(0x0001);	/* rate 1 Mbps */
-			req[2] = 0;			/* Any ESSID */
-			err = HERMES_WRITE_RECORD(hw, USER_BAP,
-						  HERMES_RID_CNFHOSTSCAN, &req);
-			break;
-		}
-		case FIRMWARE_TYPE_AGERE:
-			if (ssid->ssid_len > 0) {
-				struct hermes_idstring idbuf;
-				size_t len = ssid->ssid_len;
-
-				idbuf.len = cpu_to_le16(len);
-				memcpy(idbuf.val, ssid->ssid, len);
-
-				err = hw->ops->write_ltv(hw, USER_BAP,
-					       HERMES_RID_CNFSCANSSID_AGERE,
-					       HERMES_BYTES_TO_RECLEN(len + 2),
-					       &idbuf);
-			} else
-				err = hermes_write_wordrec(hw, USER_BAP,
-						   HERMES_RID_CNFSCANSSID_AGERE,
-						   0);	/* Any ESSID */
-			if (err)
-				break;
-
-			if (priv->has_ext_scan) {
-				err = hermes_write_wordrec(hw, USER_BAP,
-						HERMES_RID_CNFSCANCHANNELS2GHZ,
-						0x7FFF);
-				if (err)
-					goto out;
-
-				err = hermes_inquire(hw,
-						     HERMES_INQ_CHANNELINFO);
-			} else
-				err = hermes_inquire(hw, HERMES_INQ_SCAN);
-
-			break;
-		}
-	} else
-		err = hermes_inquire(hw, HERMES_INQ_SCAN);
-
- out:
-	orinoco_unlock(priv, &flags);
-
-	return err;
-}
-
-/* Disassociate from node with BSSID addr */
-int orinoco_hw_disassociate(struct orinoco_private *priv,
-			    u8 *addr, u16 reason_code)
-{
-	struct hermes *hw = &priv->hw;
-	int err;
-
-	struct {
-		u8 addr[ETH_ALEN];
-		__le16 reason_code;
-	} __packed buf;
-
-	/* Currently only supported by WPA enabled Agere fw */
-	if (!priv->has_wpa)
-		return -EOPNOTSUPP;
-
-	memcpy(buf.addr, addr, ETH_ALEN);
-	buf.reason_code = cpu_to_le16(reason_code);
-	err = HERMES_WRITE_RECORD(hw, USER_BAP,
-				  HERMES_RID_CNFDISASSOCIATE,
-				  &buf);
-	return err;
-}
-
-int orinoco_hw_get_current_bssid(struct orinoco_private *priv,
-				 u8 *addr)
-{
-	struct hermes *hw = &priv->hw;
-	int err;
-
-	err = hw->ops->read_ltv(hw, USER_BAP, HERMES_RID_CURRENTBSSID,
-				ETH_ALEN, NULL, addr);
-
-	return err;
-}
diff --git a/drivers/net/wireless/intersil/orinoco/hw.h b/drivers/net/wireless/intersil/orinoco/hw.h
deleted file mode 100644
index da5804dbdf34..000000000000
--- a/drivers/net/wireless/intersil/orinoco/hw.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/* Encapsulate basic setting changes on Hermes hardware
- *
- * See copyright notice in main.c
- */
-#ifndef _ORINOCO_HW_H_
-#define _ORINOCO_HW_H_
-
-#include <linux/types.h>
-#include <linux/wireless.h>
-#include <net/cfg80211.h>
-
-/* Hardware BAPs */
-#define USER_BAP 0
-#define IRQ_BAP  1
-
-/* WEP key sizes */
-#define SMALL_KEY_SIZE 5
-#define LARGE_KEY_SIZE 13
-
-/* Number of supported channels */
-#define NUM_CHANNELS 14
-
-/* Forward declarations */
-struct orinoco_private;
-
-int determine_fw_capabilities(struct orinoco_private *priv, char *fw_name,
-			      size_t fw_name_len, u32 *hw_ver);
-int orinoco_hw_read_card_settings(struct orinoco_private *priv, u8 *dev_addr);
-int orinoco_hw_allocate_fid(struct orinoco_private *priv);
-int orinoco_get_bitratemode(int bitrate, int automatic);
-void orinoco_get_ratemode_cfg(int ratemode, int *bitrate, int *automatic);
-
-int orinoco_hw_program_rids(struct orinoco_private *priv);
-int orinoco_hw_get_tkip_iv(struct orinoco_private *priv, int key, u8 *tsc);
-int __orinoco_hw_set_bitrate(struct orinoco_private *priv);
-int orinoco_hw_get_act_bitrate(struct orinoco_private *priv, int *bitrate);
-int __orinoco_hw_set_wap(struct orinoco_private *priv);
-int __orinoco_hw_setup_wepkeys(struct orinoco_private *priv);
-int __orinoco_hw_setup_enc(struct orinoco_private *priv);
-int __orinoco_hw_set_tkip_key(struct orinoco_private *priv, int key_idx,
-			      int set_tx, const u8 *key, size_t key_len,
-			      const u8 *rsc, size_t rsc_len,
-			      const u8 *tsc, size_t tsc_len);
-int orinoco_clear_tkip_key(struct orinoco_private *priv, int key_idx);
-int __orinoco_hw_set_multicast_list(struct orinoco_private *priv,
-				    struct net_device *dev,
-				    int mc_count, int promisc);
-int orinoco_hw_get_essid(struct orinoco_private *priv, int *active,
-			 char buf[IW_ESSID_MAX_SIZE + 1]);
-int orinoco_hw_get_freq(struct orinoco_private *priv);
-int orinoco_hw_get_bitratelist(struct orinoco_private *priv,
-			       int *numrates, s32 *rates, int max);
-int orinoco_hw_trigger_scan(struct orinoco_private *priv,
-			    const struct cfg80211_ssid *ssid);
-int orinoco_hw_disassociate(struct orinoco_private *priv,
-			    u8 *addr, u16 reason_code);
-int orinoco_hw_get_current_bssid(struct orinoco_private *priv,
-				 u8 *addr);
-
-#endif /* _ORINOCO_HW_H_ */
diff --git a/drivers/net/wireless/intersil/orinoco/main.c b/drivers/net/wireless/intersil/orinoco/main.c
deleted file mode 100644
index 7df88d20ff3d..000000000000
--- a/drivers/net/wireless/intersil/orinoco/main.c
+++ /dev/null
@@ -1,2414 +0,0 @@
-/* main.c - (formerly known as dldwd_cs.c, orinoco_cs.c and orinoco.c)
- *
- * A driver for Hermes or Prism 2 chipset based PCMCIA wireless
- * adaptors, with Lucent/Agere, Intersil or Symbol firmware.
- *
- * Current maintainers (as of 29 September 2003) are:
- *	Pavel Roskin <proski AT gnu.org>
- * and	David Gibson <hermes AT gibson.dropbear.id.au>
- *
- * (C) Copyright David Gibson, IBM Corporation 2001-2003.
- * Copyright (C) 2000 David Gibson, Linuxcare Australia.
- *	With some help from :
- * Copyright (C) 2001 Jean Tourrilhes, HP Labs
- * Copyright (C) 2001 Benjamin Herrenschmidt
- *
- * Based on dummy_cs.c 1.27 2000/06/12 21:27:25
- *
- * Portions based on wvlan_cs.c 1.0.6, Copyright Andreas Neuhaus <andy
- * AT fasta.fh-dortmund.de>
- *      http://www.stud.fh-dortmund.de/~andy/wvlan/
- *
- * The contents of this file are subject to the Mozilla Public License
- * Version 1.1 (the "License"); you may not use this file except in
- * compliance with the License. You may obtain a copy of the License
- * at http://www.mozilla.org/MPL/
- *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and
- * limitations under the License.
- *
- * The initial developer of the original code is David A. Hinds
- * <dahinds AT users.sourceforge.net>.  Portions created by David
- * A. Hinds are Copyright (C) 1999 David A. Hinds.  All Rights
- * Reserved.
- *
- * Alternatively, the contents of this file may be used under the
- * terms of the GNU General Public License version 2 (the "GPL"), in
- * which case the provisions of the GPL are applicable instead of the
- * above.  If you wish to allow the use of your version of this file
- * only under the terms of the GPL and not to allow others to use your
- * version of this file under the MPL, indicate your decision by
- * deleting the provisions above and replace them with the notice and
- * other provisions required by the GPL.  If you do not delete the
- * provisions above, a recipient may use your version of this file
- * under either the MPL or the GPL.  */
-
-/*
- * TODO
- *	o Handle de-encapsulation within network layer, provide 802.11
- *	  headers (patch from Thomas 'Dent' Mirlacher)
- *	o Fix possible races in SPY handling.
- *	o Disconnect wireless extensions from fundamental configuration.
- *	o (maybe) Software WEP support (patch from Stano Meduna).
- *	o (maybe) Use multiple Tx buffers - driver handling queue
- *	  rather than firmware.
- */
-
-/* Locking and synchronization:
- *
- * The basic principle is that everything is serialized through a
- * single spinlock, priv->lock.  The lock is used in user, bh and irq
- * context, so when taken outside hardirq context it should always be
- * taken with interrupts disabled.  The lock protects both the
- * hardware and the struct orinoco_private.
- *
- * Another flag, priv->hw_unavailable indicates that the hardware is
- * unavailable for an extended period of time (e.g. suspended, or in
- * the middle of a hard reset).  This flag is protected by the
- * spinlock.  All code which touches the hardware should check the
- * flag after taking the lock, and if it is set, give up on whatever
- * they are doing and drop the lock again.  The orinoco_lock()
- * function handles this (it unlocks and returns -EBUSY if
- * hw_unavailable is non-zero).
- */
-
-#define DRIVER_NAME "orinoco"
-
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/slab.h>
-#include <linux/init.h>
-#include <linux/delay.h>
-#include <linux/device.h>
-#include <linux/netdevice.h>
-#include <linux/etherdevice.h>
-#include <linux/suspend.h>
-#include <linux/if_arp.h>
-#include <linux/wireless.h>
-#include <linux/ieee80211.h>
-#include <net/iw_handler.h>
-#include <net/cfg80211.h>
-
-#include "hermes_rid.h"
-#include "hermes_dld.h"
-#include "hw.h"
-#include "scan.h"
-#include "mic.h"
-#include "fw.h"
-#include "wext.h"
-#include "cfg.h"
-#include "main.h"
-
-#include "orinoco.h"
-
-/********************************************************************/
-/* Module information                                               */
-/********************************************************************/
-
-MODULE_AUTHOR("Pavel Roskin <proski@gnu.org> & "
-	      "David Gibson <hermes@gibson.dropbear.id.au>");
-MODULE_DESCRIPTION("Driver for Lucent Orinoco, Prism II based "
-		   "and similar wireless cards");
-MODULE_LICENSE("Dual MPL/GPL");
-
-/* Level of debugging. Used in the macros in orinoco.h */
-#ifdef ORINOCO_DEBUG
-int orinoco_debug = ORINOCO_DEBUG;
-EXPORT_SYMBOL(orinoco_debug);
-module_param(orinoco_debug, int, 0644);
-MODULE_PARM_DESC(orinoco_debug, "Debug level");
-#endif
-
-static bool suppress_linkstatus; /* = 0 */
-module_param(suppress_linkstatus, bool, 0644);
-MODULE_PARM_DESC(suppress_linkstatus, "Don't log link status changes");
-
-static int ignore_disconnect; /* = 0 */
-module_param(ignore_disconnect, int, 0644);
-MODULE_PARM_DESC(ignore_disconnect,
-		 "Don't report lost link to the network layer");
-
-int force_monitor; /* = 0 */
-module_param(force_monitor, int, 0644);
-MODULE_PARM_DESC(force_monitor, "Allow monitor mode for all firmware versions");
-
-/********************************************************************/
-/* Internal constants                                               */
-/********************************************************************/
-
-/* 802.2 LLC/SNAP header used for Ethernet encapsulation over 802.11 */
-static const u8 encaps_hdr[] = {0xaa, 0xaa, 0x03, 0x00, 0x00, 0x00};
-#define ENCAPS_OVERHEAD		(sizeof(encaps_hdr) + 2)
-
-#define ORINOCO_MIN_MTU		256
-#define ORINOCO_MAX_MTU		(IEEE80211_MAX_DATA_LEN - ENCAPS_OVERHEAD)
-
-#define MAX_IRQLOOPS_PER_IRQ	10
-#define MAX_IRQLOOPS_PER_JIFFY	(20000 / HZ)	/* Based on a guestimate of
-						 * how many events the
-						 * device could
-						 * legitimately generate */
-
-#define DUMMY_FID		0xFFFF
-
-/*#define MAX_MULTICAST(priv)	(priv->firmware_type == FIRMWARE_TYPE_AGERE ? \
-  HERMES_MAX_MULTICAST : 0)*/
-#define MAX_MULTICAST(priv)	(HERMES_MAX_MULTICAST)
-
-#define ORINOCO_INTEN		(HERMES_EV_RX | HERMES_EV_ALLOC \
-				 | HERMES_EV_TX | HERMES_EV_TXEXC \
-				 | HERMES_EV_WTERR | HERMES_EV_INFO \
-				 | HERMES_EV_INFDROP)
-
-/********************************************************************/
-/* Data types                                                       */
-/********************************************************************/
-
-/* Beginning of the Tx descriptor, used in TxExc handling */
-struct hermes_txexc_data {
-	struct hermes_tx_descriptor desc;
-	__le16 frame_ctl;
-	__le16 duration_id;
-	u8 addr1[ETH_ALEN];
-} __packed;
-
-/* Rx frame header except compatibility 802.3 header */
-struct hermes_rx_descriptor {
-	/* Control */
-	__le16 status;
-	__le32 time;
-	u8 silence;
-	u8 signal;
-	u8 rate;
-	u8 rxflow;
-	__le32 reserved;
-
-	/* 802.11 header */
-	__le16 frame_ctl;
-	__le16 duration_id;
-	u8 addr1[ETH_ALEN];
-	u8 addr2[ETH_ALEN];
-	u8 addr3[ETH_ALEN];
-	__le16 seq_ctl;
-	u8 addr4[ETH_ALEN];
-
-	/* Data length */
-	__le16 data_len;
-} __packed;
-
-struct orinoco_rx_data {
-	struct hermes_rx_descriptor *desc;
-	struct sk_buff *skb;
-	struct list_head list;
-};
-
-struct orinoco_scan_data {
-	void *buf;
-	size_t len;
-	int type;
-	struct list_head list;
-};
-
-/********************************************************************/
-/* Function prototypes                                              */
-/********************************************************************/
-
-static int __orinoco_set_multicast_list(struct net_device *dev);
-static int __orinoco_up(struct orinoco_private *priv);
-static int __orinoco_down(struct orinoco_private *priv);
-static int __orinoco_commit(struct orinoco_private *priv);
-
-/********************************************************************/
-/* Internal helper functions                                        */
-/********************************************************************/
-
-void set_port_type(struct orinoco_private *priv)
-{
-	switch (priv->iw_mode) {
-	case NL80211_IFTYPE_STATION:
-		priv->port_type = 1;
-		priv->createibss = 0;
-		break;
-	case NL80211_IFTYPE_ADHOC:
-		if (priv->prefer_port3) {
-			priv->port_type = 3;
-			priv->createibss = 0;
-		} else {
-			priv->port_type = priv->ibss_port;
-			priv->createibss = 1;
-		}
-		break;
-	case NL80211_IFTYPE_MONITOR:
-		priv->port_type = 3;
-		priv->createibss = 0;
-		break;
-	default:
-		printk(KERN_ERR "%s: Invalid priv->iw_mode in set_port_type()\n",
-		       priv->ndev->name);
-	}
-}
-
-/********************************************************************/
-/* Device methods                                                   */
-/********************************************************************/
-
-int orinoco_open(struct net_device *dev)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	unsigned long flags;
-	int err;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	err = __orinoco_up(priv);
-
-	if (!err)
-		priv->open = 1;
-
-	orinoco_unlock(priv, &flags);
-
-	return err;
-}
-EXPORT_SYMBOL(orinoco_open);
-
-int orinoco_stop(struct net_device *dev)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	int err = 0;
-
-	/* We mustn't use orinoco_lock() here, because we need to be
-	   able to close the interface even if hw_unavailable is set
-	   (e.g. as we're released after a PC Card removal) */
-	orinoco_lock_irq(priv);
-
-	priv->open = 0;
-
-	err = __orinoco_down(priv);
-
-	orinoco_unlock_irq(priv);
-
-	return err;
-}
-EXPORT_SYMBOL(orinoco_stop);
-
-void orinoco_set_multicast_list(struct net_device *dev)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	unsigned long flags;
-
-	if (orinoco_lock(priv, &flags) != 0) {
-		printk(KERN_DEBUG "%s: orinoco_set_multicast_list() "
-		       "called when hw_unavailable\n", dev->name);
-		return;
-	}
-
-	__orinoco_set_multicast_list(dev);
-	orinoco_unlock(priv, &flags);
-}
-EXPORT_SYMBOL(orinoco_set_multicast_list);
-
-int orinoco_change_mtu(struct net_device *dev, int new_mtu)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-
-	/* MTU + encapsulation + header length */
-	if ((new_mtu + ENCAPS_OVERHEAD + sizeof(struct ieee80211_hdr)) >
-	     (priv->nicbuf_size - ETH_HLEN))
-		return -EINVAL;
-
-	dev->mtu = new_mtu;
-
-	return 0;
-}
-EXPORT_SYMBOL(orinoco_change_mtu);
-
-/********************************************************************/
-/* Tx path                                                          */
-/********************************************************************/
-
-/* Add encapsulation and MIC to the existing SKB.
- * The main xmit routine will then send the whole lot to the card.
- * Need 8 bytes headroom
- * Need 8 bytes tailroom
- *
- *                          With encapsulated ethernet II frame
- *                          --------
- *                          803.3 header (14 bytes)
- *                           dst[6]
- * --------                  src[6]
- * 803.3 header (14 bytes)   len[2]
- *  dst[6]                  803.2 header (8 bytes)
- *  src[6]                   encaps[6]
- *  len[2] <- leave alone -> len[2]
- * --------                 -------- <-- 0
- * Payload                  Payload
- * ...                      ...
- *
- * --------                 --------
- *                          MIC (8 bytes)
- *                          --------
- *
- * returns 0 on success, -ENOMEM on error.
- */
-int orinoco_process_xmit_skb(struct sk_buff *skb,
-			     struct net_device *dev,
-			     struct orinoco_private *priv,
-			     int *tx_control,
-			     u8 *mic_buf)
-{
-	struct orinoco_tkip_key *key;
-	struct ethhdr *eh;
-	int do_mic;
-
-	key = (struct orinoco_tkip_key *) priv->keys[priv->tx_key].key;
-
-	do_mic = ((priv->encode_alg == ORINOCO_ALG_TKIP) &&
-		  (key != NULL));
-
-	if (do_mic)
-		*tx_control |= (priv->tx_key << HERMES_MIC_KEY_ID_SHIFT) |
-			HERMES_TXCTRL_MIC;
-
-	eh = (struct ethhdr *)skb->data;
-
-	/* Encapsulate Ethernet-II frames */
-	if (ntohs(eh->h_proto) > ETH_DATA_LEN) { /* Ethernet-II frame */
-		struct header_struct {
-			struct ethhdr eth;	/* 802.3 header */
-			u8 encap[6];		/* 802.2 header */
-		} __packed hdr;
-		int len = skb->len + sizeof(encaps_hdr) - (2 * ETH_ALEN);
-
-		if (skb_headroom(skb) < ENCAPS_OVERHEAD) {
-			if (net_ratelimit())
-				printk(KERN_ERR
-				       "%s: Not enough headroom for 802.2 headers %d\n",
-				       dev->name, skb_headroom(skb));
-			return -ENOMEM;
-		}
-
-		/* Fill in new header */
-		memcpy(&hdr.eth, eh, 2 * ETH_ALEN);
-		hdr.eth.h_proto = htons(len);
-		memcpy(hdr.encap, encaps_hdr, sizeof(encaps_hdr));
-
-		/* Make room for the new header, and copy it in */
-		eh = skb_push(skb, ENCAPS_OVERHEAD);
-		memcpy(eh, &hdr, sizeof(hdr));
-	}
-
-	/* Calculate Michael MIC */
-	if (do_mic) {
-		size_t len = skb->len - ETH_HLEN;
-		u8 *mic = &mic_buf[0];
-
-		/* Have to write to an even address, so copy the spare
-		 * byte across */
-		if (skb->len % 2) {
-			*mic = skb->data[skb->len - 1];
-			mic++;
-		}
-
-		orinoco_mic(priv->tx_tfm_mic, key->tx_mic,
-			    eh->h_dest, eh->h_source, 0 /* priority */,
-			    skb->data + ETH_HLEN,
-			    len, mic);
-	}
-
-	return 0;
-}
-EXPORT_SYMBOL(orinoco_process_xmit_skb);
-
-static netdev_tx_t orinoco_xmit(struct sk_buff *skb, struct net_device *dev)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	struct net_device_stats *stats = &dev->stats;
-	struct hermes *hw = &priv->hw;
-	int err = 0;
-	u16 txfid = priv->txfid;
-	int tx_control;
-	unsigned long flags;
-	u8 mic_buf[MICHAEL_MIC_LEN + 1];
-
-	if (!netif_running(dev)) {
-		printk(KERN_ERR "%s: Tx on stopped device!\n",
-		       dev->name);
-		return NETDEV_TX_BUSY;
-	}
-
-	if (netif_queue_stopped(dev)) {
-		printk(KERN_DEBUG "%s: Tx while transmitter busy!\n",
-		       dev->name);
-		return NETDEV_TX_BUSY;
-	}
-
-	if (orinoco_lock(priv, &flags) != 0) {
-		printk(KERN_ERR "%s: orinoco_xmit() called while hw_unavailable\n",
-		       dev->name);
-		return NETDEV_TX_BUSY;
-	}
-
-	if (!netif_carrier_ok(dev) ||
-	    (priv->iw_mode == NL80211_IFTYPE_MONITOR)) {
-		/* Oops, the firmware hasn't established a connection,
-		   silently drop the packet (this seems to be the
-		   safest approach). */
-		goto drop;
-	}
-
-	/* Check packet length */
-	if (skb->len < ETH_HLEN)
-		goto drop;
-
-	tx_control = HERMES_TXCTRL_TX_OK | HERMES_TXCTRL_TX_EX;
-
-	err = orinoco_process_xmit_skb(skb, dev, priv, &tx_control,
-				       &mic_buf[0]);
-	if (err)
-		goto drop;
-
-	if (priv->has_alt_txcntl) {
-		/* WPA enabled firmwares have tx_cntl at the end of
-		 * the 802.11 header.  So write zeroed descriptor and
-		 * 802.11 header at the same time
-		 */
-		char desc[HERMES_802_3_OFFSET];
-		__le16 *txcntl = (__le16 *) &desc[HERMES_TXCNTL2_OFFSET];
-
-		memset(&desc, 0, sizeof(desc));
-
-		*txcntl = cpu_to_le16(tx_control);
-		err = hw->ops->bap_pwrite(hw, USER_BAP, &desc, sizeof(desc),
-					  txfid, 0);
-		if (err) {
-			if (net_ratelimit())
-				printk(KERN_ERR "%s: Error %d writing Tx "
-				       "descriptor to BAP\n", dev->name, err);
-			goto busy;
-		}
-	} else {
-		struct hermes_tx_descriptor desc;
-
-		memset(&desc, 0, sizeof(desc));
-
-		desc.tx_control = cpu_to_le16(tx_control);
-		err = hw->ops->bap_pwrite(hw, USER_BAP, &desc, sizeof(desc),
-					  txfid, 0);
-		if (err) {
-			if (net_ratelimit())
-				printk(KERN_ERR "%s: Error %d writing Tx "
-				       "descriptor to BAP\n", dev->name, err);
-			goto busy;
-		}
-
-		/* Clear the 802.11 header and data length fields - some
-		 * firmwares (e.g. Lucent/Agere 8.xx) appear to get confused
-		 * if this isn't done. */
-		hermes_clear_words(hw, HERMES_DATA0,
-				   HERMES_802_3_OFFSET - HERMES_802_11_OFFSET);
-	}
-
-	err = hw->ops->bap_pwrite(hw, USER_BAP, skb->data, skb->len,
-				  txfid, HERMES_802_3_OFFSET);
-	if (err) {
-		printk(KERN_ERR "%s: Error %d writing packet to BAP\n",
-		       dev->name, err);
-		goto busy;
-	}
-
-	if (tx_control & HERMES_TXCTRL_MIC) {
-		size_t offset = HERMES_802_3_OFFSET + skb->len;
-		size_t len = MICHAEL_MIC_LEN;
-
-		if (offset % 2) {
-			offset--;
-			len++;
-		}
-		err = hw->ops->bap_pwrite(hw, USER_BAP, &mic_buf[0], len,
-					  txfid, offset);
-		if (err) {
-			printk(KERN_ERR "%s: Error %d writing MIC to BAP\n",
-			       dev->name, err);
-			goto busy;
-		}
-	}
-
-	/* Finally, we actually initiate the send */
-	netif_stop_queue(dev);
-
-	err = hw->ops->cmd_wait(hw, HERMES_CMD_TX | HERMES_CMD_RECL,
-				txfid, NULL);
-	if (err) {
-		netif_start_queue(dev);
-		if (net_ratelimit())
-			printk(KERN_ERR "%s: Error %d transmitting packet\n",
-				dev->name, err);
-		goto busy;
-	}
-
-	stats->tx_bytes += HERMES_802_3_OFFSET + skb->len;
-	goto ok;
-
- drop:
-	stats->tx_errors++;
-	stats->tx_dropped++;
-
- ok:
-	orinoco_unlock(priv, &flags);
-	dev_kfree_skb(skb);
-	return NETDEV_TX_OK;
-
- busy:
-	if (err == -EIO)
-		schedule_work(&priv->reset_work);
-	orinoco_unlock(priv, &flags);
-	return NETDEV_TX_BUSY;
-}
-
-static void __orinoco_ev_alloc(struct net_device *dev, struct hermes *hw)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	u16 fid = hermes_read_regn(hw, ALLOCFID);
-
-	if (fid != priv->txfid) {
-		if (fid != DUMMY_FID)
-			printk(KERN_WARNING "%s: Allocate event on unexpected fid (%04X)\n",
-			       dev->name, fid);
-		return;
-	}
-
-	hermes_write_regn(hw, ALLOCFID, DUMMY_FID);
-}
-
-static void __orinoco_ev_tx(struct net_device *dev, struct hermes *hw)
-{
-	dev->stats.tx_packets++;
-
-	netif_wake_queue(dev);
-
-	hermes_write_regn(hw, TXCOMPLFID, DUMMY_FID);
-}
-
-static void __orinoco_ev_txexc(struct net_device *dev, struct hermes *hw)
-{
-	struct net_device_stats *stats = &dev->stats;
-	u16 fid = hermes_read_regn(hw, TXCOMPLFID);
-	u16 status;
-	struct hermes_txexc_data hdr;
-	int err = 0;
-
-	if (fid == DUMMY_FID)
-		return; /* Nothing's really happened */
-
-	/* Read part of the frame header - we need status and addr1 */
-	err = hw->ops->bap_pread(hw, IRQ_BAP, &hdr,
-				 sizeof(struct hermes_txexc_data),
-				 fid, 0);
-
-	hermes_write_regn(hw, TXCOMPLFID, DUMMY_FID);
-	stats->tx_errors++;
-
-	if (err) {
-		printk(KERN_WARNING "%s: Unable to read descriptor on Tx error "
-		       "(FID=%04X error %d)\n",
-		       dev->name, fid, err);
-		return;
-	}
-
-	DEBUG(1, "%s: Tx error, err %d (FID=%04X)\n", dev->name,
-	      err, fid);
-
-	/* We produce a TXDROP event only for retry or lifetime
-	 * exceeded, because that's the only status that really mean
-	 * that this particular node went away.
-	 * Other errors means that *we* screwed up. - Jean II */
-	status = le16_to_cpu(hdr.desc.status);
-	if (status & (HERMES_TXSTAT_RETRYERR | HERMES_TXSTAT_AGEDERR)) {
-		union iwreq_data	wrqu;
-
-		/* Copy 802.11 dest address.
-		 * We use the 802.11 header because the frame may
-		 * not be 802.3 or may be mangled...
-		 * In Ad-Hoc mode, it will be the node address.
-		 * In managed mode, it will be most likely the AP addr
-		 * User space will figure out how to convert it to
-		 * whatever it needs (IP address or else).
-		 * - Jean II */
-		memcpy(wrqu.addr.sa_data, hdr.addr1, ETH_ALEN);
-		wrqu.addr.sa_family = ARPHRD_ETHER;
-
-		/* Send event to user space */
-		wireless_send_event(dev, IWEVTXDROP, &wrqu, NULL);
-	}
-
-	netif_wake_queue(dev);
-}
-
-void orinoco_tx_timeout(struct net_device *dev, unsigned int txqueue)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	struct net_device_stats *stats = &dev->stats;
-	struct hermes *hw = &priv->hw;
-
-	printk(KERN_WARNING "%s: Tx timeout! "
-	       "ALLOCFID=%04x, TXCOMPLFID=%04x, EVSTAT=%04x\n",
-	       dev->name, hermes_read_regn(hw, ALLOCFID),
-	       hermes_read_regn(hw, TXCOMPLFID), hermes_read_regn(hw, EVSTAT));
-
-	stats->tx_errors++;
-
-	schedule_work(&priv->reset_work);
-}
-EXPORT_SYMBOL(orinoco_tx_timeout);
-
-/********************************************************************/
-/* Rx path (data frames)                                            */
-/********************************************************************/
-
-/* Does the frame have a SNAP header indicating it should be
- * de-encapsulated to Ethernet-II? */
-static inline int is_ethersnap(void *_hdr)
-{
-	u8 *hdr = _hdr;
-
-	/* We de-encapsulate all packets which, a) have SNAP headers
-	 * (i.e. SSAP=DSAP=0xaa and CTRL=0x3 in the 802.2 LLC header
-	 * and where b) the OUI of the SNAP header is 00:00:00 or
-	 * 00:00:f8 - we need both because different APs appear to use
-	 * different OUIs for some reason */
-	return (memcmp(hdr, &encaps_hdr, 5) == 0)
-		&& ((hdr[5] == 0x00) || (hdr[5] == 0xf8));
-}
-
-static inline void orinoco_spy_gather(struct net_device *dev, u_char *mac,
-				      int level, int noise)
-{
-	struct iw_quality wstats;
-	wstats.level = level - 0x95;
-	wstats.noise = noise - 0x95;
-	wstats.qual = (level > noise) ? (level - noise) : 0;
-	wstats.updated = IW_QUAL_ALL_UPDATED | IW_QUAL_DBM;
-	/* Update spy records */
-	wireless_spy_update(dev, mac, &wstats);
-}
-
-static void orinoco_stat_gather(struct net_device *dev,
-				struct sk_buff *skb,
-				struct hermes_rx_descriptor *desc)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-
-	/* Using spy support with lots of Rx packets, like in an
-	 * infrastructure (AP), will really slow down everything, because
-	 * the MAC address must be compared to each entry of the spy list.
-	 * If the user really asks for it (set some address in the
-	 * spy list), we do it, but he will pay the price.
-	 * Note that to get here, you need both WIRELESS_SPY
-	 * compiled in AND some addresses in the list !!!
-	 */
-	/* Note : gcc will optimise the whole section away if
-	 * WIRELESS_SPY is not defined... - Jean II */
-	if (SPY_NUMBER(priv)) {
-		orinoco_spy_gather(dev, skb_mac_header(skb) + ETH_ALEN,
-				   desc->signal, desc->silence);
-	}
-}
-
-/*
- * orinoco_rx_monitor - handle received monitor frames.
- *
- * Arguments:
- *	dev		network device
- *	rxfid		received FID
- *	desc		rx descriptor of the frame
- *
- * Call context: interrupt
- */
-static void orinoco_rx_monitor(struct net_device *dev, u16 rxfid,
-			       struct hermes_rx_descriptor *desc)
-{
-	u32 hdrlen = 30;	/* return full header by default */
-	u32 datalen = 0;
-	u16 fc;
-	int err;
-	int len;
-	struct sk_buff *skb;
-	struct orinoco_private *priv = ndev_priv(dev);
-	struct net_device_stats *stats = &dev->stats;
-	struct hermes *hw = &priv->hw;
-
-	len = le16_to_cpu(desc->data_len);
-
-	/* Determine the size of the header and the data */
-	fc = le16_to_cpu(desc->frame_ctl);
-	switch (fc & IEEE80211_FCTL_FTYPE) {
-	case IEEE80211_FTYPE_DATA:
-		if ((fc & IEEE80211_FCTL_TODS)
-		    && (fc & IEEE80211_FCTL_FROMDS))
-			hdrlen = 30;
-		else
-			hdrlen = 24;
-		datalen = len;
-		break;
-	case IEEE80211_FTYPE_MGMT:
-		hdrlen = 24;
-		datalen = len;
-		break;
-	case IEEE80211_FTYPE_CTL:
-		switch (fc & IEEE80211_FCTL_STYPE) {
-		case IEEE80211_STYPE_PSPOLL:
-		case IEEE80211_STYPE_RTS:
-		case IEEE80211_STYPE_CFEND:
-		case IEEE80211_STYPE_CFENDACK:
-			hdrlen = 16;
-			break;
-		case IEEE80211_STYPE_CTS:
-		case IEEE80211_STYPE_ACK:
-			hdrlen = 10;
-			break;
-		}
-		break;
-	default:
-		/* Unknown frame type */
-		break;
-	}
-
-	/* sanity check the length */
-	if (datalen > IEEE80211_MAX_DATA_LEN + 12) {
-		printk(KERN_DEBUG "%s: oversized monitor frame, "
-		       "data length = %d\n", dev->name, datalen);
-		stats->rx_length_errors++;
-		goto update_stats;
-	}
-
-	skb = dev_alloc_skb(hdrlen + datalen);
-	if (!skb) {
-		printk(KERN_WARNING "%s: Cannot allocate skb for monitor frame\n",
-		       dev->name);
-		goto update_stats;
-	}
-
-	/* Copy the 802.11 header to the skb */
-	skb_put_data(skb, &(desc->frame_ctl), hdrlen);
-	skb_reset_mac_header(skb);
-
-	/* If any, copy the data from the card to the skb */
-	if (datalen > 0) {
-		err = hw->ops->bap_pread(hw, IRQ_BAP, skb_put(skb, datalen),
-					 ALIGN(datalen, 2), rxfid,
-					 HERMES_802_2_OFFSET);
-		if (err) {
-			printk(KERN_ERR "%s: error %d reading monitor frame\n",
-			       dev->name, err);
-			goto drop;
-		}
-	}
-
-	skb->dev = dev;
-	skb->ip_summed = CHECKSUM_NONE;
-	skb->pkt_type = PACKET_OTHERHOST;
-	skb->protocol = cpu_to_be16(ETH_P_802_2);
-
-	stats->rx_packets++;
-	stats->rx_bytes += skb->len;
-
-	netif_rx(skb);
-	return;
-
- drop:
-	dev_kfree_skb_irq(skb);
- update_stats:
-	stats->rx_errors++;
-	stats->rx_dropped++;
-}
-
-void __orinoco_ev_rx(struct net_device *dev, struct hermes *hw)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	struct net_device_stats *stats = &dev->stats;
-	struct iw_statistics *wstats = &priv->wstats;
-	struct sk_buff *skb = NULL;
-	u16 rxfid, status;
-	int length;
-	struct hermes_rx_descriptor *desc;
-	struct orinoco_rx_data *rx_data;
-	int err;
-
-	desc = kmalloc(sizeof(*desc), GFP_ATOMIC);
-	if (!desc)
-		goto update_stats;
-
-	rxfid = hermes_read_regn(hw, RXFID);
-
-	err = hw->ops->bap_pread(hw, IRQ_BAP, desc, sizeof(*desc),
-				 rxfid, 0);
-	if (err) {
-		printk(KERN_ERR "%s: error %d reading Rx descriptor. "
-		       "Frame dropped.\n", dev->name, err);
-		goto update_stats;
-	}
-
-	status = le16_to_cpu(desc->status);
-
-	if (status & HERMES_RXSTAT_BADCRC) {
-		DEBUG(1, "%s: Bad CRC on Rx. Frame dropped.\n",
-		      dev->name);
-		stats->rx_crc_errors++;
-		goto update_stats;
-	}
-
-	/* Handle frames in monitor mode */
-	if (priv->iw_mode == NL80211_IFTYPE_MONITOR) {
-		orinoco_rx_monitor(dev, rxfid, desc);
-		goto out;
-	}
-
-	if (status & HERMES_RXSTAT_UNDECRYPTABLE) {
-		DEBUG(1, "%s: Undecryptable frame on Rx. Frame dropped.\n",
-		      dev->name);
-		wstats->discard.code++;
-		goto update_stats;
-	}
-
-	length = le16_to_cpu(desc->data_len);
-
-	/* Sanity checks */
-	if (length < 3) { /* No for even an 802.2 LLC header */
-		/* At least on Symbol firmware with PCF we get quite a
-		   lot of these legitimately - Poll frames with no
-		   data. */
-		goto out;
-	}
-	if (length > IEEE80211_MAX_DATA_LEN) {
-		printk(KERN_WARNING "%s: Oversized frame received (%d bytes)\n",
-		       dev->name, length);
-		stats->rx_length_errors++;
-		goto update_stats;
-	}
-
-	/* Payload size does not include Michael MIC. Increase payload
-	 * size to read it together with the data. */
-	if (status & HERMES_RXSTAT_MIC)
-		length += MICHAEL_MIC_LEN;
-
-	/* We need space for the packet data itself, plus an ethernet
-	   header, plus 2 bytes so we can align the IP header on a
-	   32bit boundary, plus 1 byte so we can read in odd length
-	   packets from the card, which has an IO granularity of 16
-	   bits */
-	skb = dev_alloc_skb(length + ETH_HLEN + 2 + 1);
-	if (!skb) {
-		printk(KERN_WARNING "%s: Can't allocate skb for Rx\n",
-		       dev->name);
-		goto update_stats;
-	}
-
-	/* We'll prepend the header, so reserve space for it.  The worst
-	   case is no decapsulation, when 802.3 header is prepended and
-	   nothing is removed.  2 is for aligning the IP header.  */
-	skb_reserve(skb, ETH_HLEN + 2);
-
-	err = hw->ops->bap_pread(hw, IRQ_BAP, skb_put(skb, length),
-				 ALIGN(length, 2), rxfid,
-				 HERMES_802_2_OFFSET);
-	if (err) {
-		printk(KERN_ERR "%s: error %d reading frame. "
-		       "Frame dropped.\n", dev->name, err);
-		goto drop;
-	}
-
-	/* Add desc and skb to rx queue */
-	rx_data = kzalloc(sizeof(*rx_data), GFP_ATOMIC);
-	if (!rx_data)
-		goto drop;
-
-	rx_data->desc = desc;
-	rx_data->skb = skb;
-	list_add_tail(&rx_data->list, &priv->rx_list);
-	tasklet_schedule(&priv->rx_tasklet);
-
-	return;
-
-drop:
-	dev_kfree_skb_irq(skb);
-update_stats:
-	stats->rx_errors++;
-	stats->rx_dropped++;
-out:
-	kfree(desc);
-}
-EXPORT_SYMBOL(__orinoco_ev_rx);
-
-static void orinoco_rx(struct net_device *dev,
-		       struct hermes_rx_descriptor *desc,
-		       struct sk_buff *skb)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	struct net_device_stats *stats = &dev->stats;
-	u16 status, fc;
-	int length;
-	struct ethhdr *hdr;
-
-	status = le16_to_cpu(desc->status);
-	length = le16_to_cpu(desc->data_len);
-	fc = le16_to_cpu(desc->frame_ctl);
-
-	/* Calculate and check MIC */
-	if (status & HERMES_RXSTAT_MIC) {
-		struct orinoco_tkip_key *key;
-		int key_id = ((status & HERMES_RXSTAT_MIC_KEY_ID) >>
-			      HERMES_MIC_KEY_ID_SHIFT);
-		u8 mic[MICHAEL_MIC_LEN];
-		u8 *rxmic;
-		u8 *src = (fc & IEEE80211_FCTL_FROMDS) ?
-			desc->addr3 : desc->addr2;
-
-		/* Extract Michael MIC from payload */
-		rxmic = skb->data + skb->len - MICHAEL_MIC_LEN;
-
-		skb_trim(skb, skb->len - MICHAEL_MIC_LEN);
-		length -= MICHAEL_MIC_LEN;
-
-		key = (struct orinoco_tkip_key *) priv->keys[key_id].key;
-
-		if (!key) {
-			printk(KERN_WARNING "%s: Received encrypted frame from "
-			       "%pM using key %i, but key is not installed\n",
-			       dev->name, src, key_id);
-			goto drop;
-		}
-
-		orinoco_mic(priv->rx_tfm_mic, key->rx_mic, desc->addr1, src,
-			    0, /* priority or QoS? */
-			    skb->data, skb->len, &mic[0]);
-
-		if (memcmp(mic, rxmic,
-			   MICHAEL_MIC_LEN)) {
-			union iwreq_data wrqu;
-			struct iw_michaelmicfailure wxmic;
-
-			printk(KERN_WARNING "%s: "
-			       "Invalid Michael MIC in data frame from %pM, "
-			       "using key %i\n",
-			       dev->name, src, key_id);
-
-			/* TODO: update stats */
-
-			/* Notify userspace */
-			memset(&wxmic, 0, sizeof(wxmic));
-			wxmic.flags = key_id & IW_MICFAILURE_KEY_ID;
-			wxmic.flags |= (desc->addr1[0] & 1) ?
-				IW_MICFAILURE_GROUP : IW_MICFAILURE_PAIRWISE;
-			wxmic.src_addr.sa_family = ARPHRD_ETHER;
-			memcpy(wxmic.src_addr.sa_data, src, ETH_ALEN);
-
-			(void) orinoco_hw_get_tkip_iv(priv, key_id,
-						      &wxmic.tsc[0]);
-
-			memset(&wrqu, 0, sizeof(wrqu));
-			wrqu.data.length = sizeof(wxmic);
-			wireless_send_event(dev, IWEVMICHAELMICFAILURE, &wrqu,
-					    (char *) &wxmic);
-
-			goto drop;
-		}
-	}
-
-	/* Handle decapsulation
-	 * In most cases, the firmware tell us about SNAP frames.
-	 * For some reason, the SNAP frames sent by LinkSys APs
-	 * are not properly recognised by most firmwares.
-	 * So, check ourselves */
-	if (length >= ENCAPS_OVERHEAD &&
-	    (((status & HERMES_RXSTAT_MSGTYPE) == HERMES_RXSTAT_1042) ||
-	     ((status & HERMES_RXSTAT_MSGTYPE) == HERMES_RXSTAT_TUNNEL) ||
-	     is_ethersnap(skb->data))) {
-		/* These indicate a SNAP within 802.2 LLC within
-		   802.11 frame which we'll need to de-encapsulate to
-		   the original EthernetII frame. */
-		hdr = skb_push(skb, ETH_HLEN - ENCAPS_OVERHEAD);
-	} else {
-		/* 802.3 frame - prepend 802.3 header as is */
-		hdr = skb_push(skb, ETH_HLEN);
-		hdr->h_proto = htons(length);
-	}
-	memcpy(hdr->h_dest, desc->addr1, ETH_ALEN);
-	if (fc & IEEE80211_FCTL_FROMDS)
-		memcpy(hdr->h_source, desc->addr3, ETH_ALEN);
-	else
-		memcpy(hdr->h_source, desc->addr2, ETH_ALEN);
-
-	skb->protocol = eth_type_trans(skb, dev);
-	skb->ip_summed = CHECKSUM_NONE;
-	if (fc & IEEE80211_FCTL_TODS)
-		skb->pkt_type = PACKET_OTHERHOST;
-
-	/* Process the wireless stats if needed */
-	orinoco_stat_gather(dev, skb, desc);
-
-	/* Pass the packet to the networking stack */
-	netif_rx(skb);
-	stats->rx_packets++;
-	stats->rx_bytes += length;
-
-	return;
-
- drop:
-	dev_kfree_skb(skb);
-	stats->rx_errors++;
-	stats->rx_dropped++;
-}
-
-static void orinoco_rx_isr_tasklet(struct tasklet_struct *t)
-{
-	struct orinoco_private *priv = from_tasklet(priv, t, rx_tasklet);
-	struct net_device *dev = priv->ndev;
-	struct orinoco_rx_data *rx_data, *temp;
-	struct hermes_rx_descriptor *desc;
-	struct sk_buff *skb;
-	unsigned long flags;
-
-	/* orinoco_rx requires the driver lock, and we also need to
-	 * protect priv->rx_list, so just hold the lock over the
-	 * lot.
-	 *
-	 * If orinoco_lock fails, we've unplugged the card. In this
-	 * case just abort. */
-	if (orinoco_lock(priv, &flags) != 0)
-		return;
-
-	/* extract desc and skb from queue */
-	list_for_each_entry_safe(rx_data, temp, &priv->rx_list, list) {
-		desc = rx_data->desc;
-		skb = rx_data->skb;
-		list_del(&rx_data->list);
-		kfree(rx_data);
-
-		orinoco_rx(dev, desc, skb);
-
-		kfree(desc);
-	}
-
-	orinoco_unlock(priv, &flags);
-}
-
-/********************************************************************/
-/* Rx path (info frames)                                            */
-/********************************************************************/
-
-static void print_linkstatus(struct net_device *dev, u16 status)
-{
-	char *s;
-
-	if (suppress_linkstatus)
-		return;
-
-	switch (status) {
-	case HERMES_LINKSTATUS_NOT_CONNECTED:
-		s = "Not Connected";
-		break;
-	case HERMES_LINKSTATUS_CONNECTED:
-		s = "Connected";
-		break;
-	case HERMES_LINKSTATUS_DISCONNECTED:
-		s = "Disconnected";
-		break;
-	case HERMES_LINKSTATUS_AP_CHANGE:
-		s = "AP Changed";
-		break;
-	case HERMES_LINKSTATUS_AP_OUT_OF_RANGE:
-		s = "AP Out of Range";
-		break;
-	case HERMES_LINKSTATUS_AP_IN_RANGE:
-		s = "AP In Range";
-		break;
-	case HERMES_LINKSTATUS_ASSOC_FAILED:
-		s = "Association Failed";
-		break;
-	default:
-		s = "UNKNOWN";
-	}
-
-	printk(KERN_DEBUG "%s: New link status: %s (%04x)\n",
-	       dev->name, s, status);
-}
-
-/* Search scan results for requested BSSID, join it if found */
-static void orinoco_join_ap(struct work_struct *work)
-{
-	struct orinoco_private *priv =
-		container_of(work, struct orinoco_private, join_work);
-	struct net_device *dev = priv->ndev;
-	struct hermes *hw = &priv->hw;
-	int err;
-	unsigned long flags;
-	struct join_req {
-		u8 bssid[ETH_ALEN];
-		__le16 channel;
-	} __packed req;
-	const int atom_len = offsetof(struct prism2_scan_apinfo, atim);
-	struct prism2_scan_apinfo *atom = NULL;
-	int offset = 4;
-	int found = 0;
-	u8 *buf;
-	u16 len;
-
-	/* Allocate buffer for scan results */
-	buf = kmalloc(MAX_SCAN_LEN, GFP_KERNEL);
-	if (!buf)
-		return;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		goto fail_lock;
-
-	/* Sanity checks in case user changed something in the meantime */
-	if (!priv->bssid_fixed)
-		goto out;
-
-	if (strlen(priv->desired_essid) == 0)
-		goto out;
-
-	/* Read scan results from the firmware */
-	err = hw->ops->read_ltv(hw, USER_BAP,
-				HERMES_RID_SCANRESULTSTABLE,
-				MAX_SCAN_LEN, &len, buf);
-	if (err) {
-		printk(KERN_ERR "%s: Cannot read scan results\n",
-		       dev->name);
-		goto out;
-	}
-
-	len = HERMES_RECLEN_TO_BYTES(len);
-
-	/* Go through the scan results looking for the channel of the AP
-	 * we were requested to join */
-	for (; offset + atom_len <= len; offset += atom_len) {
-		atom = (struct prism2_scan_apinfo *) (buf + offset);
-		if (memcmp(&atom->bssid, priv->desired_bssid, ETH_ALEN) == 0) {
-			found = 1;
-			break;
-		}
-	}
-
-	if (!found) {
-		DEBUG(1, "%s: Requested AP not found in scan results\n",
-		      dev->name);
-		goto out;
-	}
-
-	memcpy(req.bssid, priv->desired_bssid, ETH_ALEN);
-	req.channel = atom->channel;	/* both are little-endian */
-	err = HERMES_WRITE_RECORD(hw, USER_BAP, HERMES_RID_CNFJOINREQUEST,
-				  &req);
-	if (err)
-		printk(KERN_ERR "%s: Error issuing join request\n", dev->name);
-
- out:
-	orinoco_unlock(priv, &flags);
-
- fail_lock:
-	kfree(buf);
-}
-
-/* Send new BSSID to userspace */
-static void orinoco_send_bssid_wevent(struct orinoco_private *priv)
-{
-	struct net_device *dev = priv->ndev;
-	struct hermes *hw = &priv->hw;
-	union iwreq_data wrqu;
-	int err;
-
-	err = hw->ops->read_ltv(hw, USER_BAP, HERMES_RID_CURRENTBSSID,
-				ETH_ALEN, NULL, wrqu.ap_addr.sa_data);
-	if (err != 0)
-		return;
-
-	wrqu.ap_addr.sa_family = ARPHRD_ETHER;
-
-	/* Send event to user space */
-	wireless_send_event(dev, SIOCGIWAP, &wrqu, NULL);
-}
-
-static void orinoco_send_assocreqie_wevent(struct orinoco_private *priv)
-{
-	struct net_device *dev = priv->ndev;
-	struct hermes *hw = &priv->hw;
-	union iwreq_data wrqu;
-	int err;
-	u8 buf[88];
-	u8 *ie;
-
-	if (!priv->has_wpa)
-		return;
-
-	err = hw->ops->read_ltv(hw, USER_BAP, HERMES_RID_CURRENT_ASSOC_REQ_INFO,
-				sizeof(buf), NULL, &buf);
-	if (err != 0)
-		return;
-
-	ie = orinoco_get_wpa_ie(buf, sizeof(buf));
-	if (ie) {
-		int rem = sizeof(buf) - (ie - &buf[0]);
-		wrqu.data.length = ie[1] + 2;
-		if (wrqu.data.length > rem)
-			wrqu.data.length = rem;
-
-		if (wrqu.data.length)
-			/* Send event to user space */
-			wireless_send_event(dev, IWEVASSOCREQIE, &wrqu, ie);
-	}
-}
-
-static void orinoco_send_assocrespie_wevent(struct orinoco_private *priv)
-{
-	struct net_device *dev = priv->ndev;
-	struct hermes *hw = &priv->hw;
-	union iwreq_data wrqu;
-	int err;
-	u8 buf[88]; /* TODO: verify max size or IW_GENERIC_IE_MAX */
-	u8 *ie;
-
-	if (!priv->has_wpa)
-		return;
-
-	err = hw->ops->read_ltv(hw, USER_BAP,
-				HERMES_RID_CURRENT_ASSOC_RESP_INFO,
-				sizeof(buf), NULL, &buf);
-	if (err != 0)
-		return;
-
-	ie = orinoco_get_wpa_ie(buf, sizeof(buf));
-	if (ie) {
-		int rem = sizeof(buf) - (ie - &buf[0]);
-		wrqu.data.length = ie[1] + 2;
-		if (wrqu.data.length > rem)
-			wrqu.data.length = rem;
-
-		if (wrqu.data.length)
-			/* Send event to user space */
-			wireless_send_event(dev, IWEVASSOCRESPIE, &wrqu, ie);
-	}
-}
-
-static void orinoco_send_wevents(struct work_struct *work)
-{
-	struct orinoco_private *priv =
-		container_of(work, struct orinoco_private, wevent_work);
-	unsigned long flags;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return;
-
-	orinoco_send_assocreqie_wevent(priv);
-	orinoco_send_assocrespie_wevent(priv);
-	orinoco_send_bssid_wevent(priv);
-
-	orinoco_unlock(priv, &flags);
-}
-
-static void qbuf_scan(struct orinoco_private *priv, void *buf,
-		      int len, int type)
-{
-	struct orinoco_scan_data *sd;
-	unsigned long flags;
-
-	sd = kmalloc(sizeof(*sd), GFP_ATOMIC);
-	if (!sd)
-		return;
-
-	sd->buf = buf;
-	sd->len = len;
-	sd->type = type;
-
-	spin_lock_irqsave(&priv->scan_lock, flags);
-	list_add_tail(&sd->list, &priv->scan_list);
-	spin_unlock_irqrestore(&priv->scan_lock, flags);
-
-	schedule_work(&priv->process_scan);
-}
-
-static void qabort_scan(struct orinoco_private *priv)
-{
-	struct orinoco_scan_data *sd;
-	unsigned long flags;
-
-	sd = kmalloc(sizeof(*sd), GFP_ATOMIC);
-	if (!sd)
-		return;
-
-	sd->len = -1; /* Abort */
-
-	spin_lock_irqsave(&priv->scan_lock, flags);
-	list_add_tail(&sd->list, &priv->scan_list);
-	spin_unlock_irqrestore(&priv->scan_lock, flags);
-
-	schedule_work(&priv->process_scan);
-}
-
-static void orinoco_process_scan_results(struct work_struct *work)
-{
-	struct orinoco_private *priv =
-		container_of(work, struct orinoco_private, process_scan);
-	struct orinoco_scan_data *sd, *temp;
-	unsigned long flags;
-	void *buf;
-	int len;
-	int type;
-
-	spin_lock_irqsave(&priv->scan_lock, flags);
-	list_for_each_entry_safe(sd, temp, &priv->scan_list, list) {
-
-		buf = sd->buf;
-		len = sd->len;
-		type = sd->type;
-
-		list_del(&sd->list);
-		spin_unlock_irqrestore(&priv->scan_lock, flags);
-		kfree(sd);
-
-		if (len > 0) {
-			if (type == HERMES_INQ_CHANNELINFO)
-				orinoco_add_extscan_result(priv, buf, len);
-			else
-				orinoco_add_hostscan_results(priv, buf, len);
-
-			kfree(buf);
-		} else {
-			/* Either abort or complete the scan */
-			orinoco_scan_done(priv, (len < 0));
-		}
-
-		spin_lock_irqsave(&priv->scan_lock, flags);
-	}
-	spin_unlock_irqrestore(&priv->scan_lock, flags);
-}
-
-void __orinoco_ev_info(struct net_device *dev, struct hermes *hw)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	u16 infofid;
-	struct {
-		__le16 len;
-		__le16 type;
-	} __packed info;
-	int len, type;
-	int err;
-
-	/* This is an answer to an INQUIRE command that we did earlier,
-	 * or an information "event" generated by the card
-	 * The controller return to us a pseudo frame containing
-	 * the information in question - Jean II */
-	infofid = hermes_read_regn(hw, INFOFID);
-
-	/* Read the info frame header - don't try too hard */
-	err = hw->ops->bap_pread(hw, IRQ_BAP, &info, sizeof(info),
-				 infofid, 0);
-	if (err) {
-		printk(KERN_ERR "%s: error %d reading info frame. "
-		       "Frame dropped.\n", dev->name, err);
-		return;
-	}
-
-	len = HERMES_RECLEN_TO_BYTES(le16_to_cpu(info.len));
-	type = le16_to_cpu(info.type);
-
-	switch (type) {
-	case HERMES_INQ_TALLIES: {
-		struct hermes_tallies_frame tallies;
-		struct iw_statistics *wstats = &priv->wstats;
-
-		if (len > sizeof(tallies)) {
-			printk(KERN_WARNING "%s: Tallies frame too long (%d bytes)\n",
-			       dev->name, len);
-			len = sizeof(tallies);
-		}
-
-		err = hw->ops->bap_pread(hw, IRQ_BAP, &tallies, len,
-					 infofid, sizeof(info));
-		if (err)
-			break;
-
-		/* Increment our various counters */
-		/* wstats->discard.nwid - no wrong BSSID stuff */
-		wstats->discard.code +=
-			le16_to_cpu(tallies.RxWEPUndecryptable);
-		if (len == sizeof(tallies))
-			wstats->discard.code +=
-				le16_to_cpu(tallies.RxDiscards_WEPICVError) +
-				le16_to_cpu(tallies.RxDiscards_WEPExcluded);
-		wstats->discard.misc +=
-			le16_to_cpu(tallies.TxDiscardsWrongSA);
-		wstats->discard.fragment +=
-			le16_to_cpu(tallies.RxMsgInBadMsgFragments);
-		wstats->discard.retries +=
-			le16_to_cpu(tallies.TxRetryLimitExceeded);
-		/* wstats->miss.beacon - no match */
-	}
-	break;
-	case HERMES_INQ_LINKSTATUS: {
-		struct hermes_linkstatus linkstatus;
-		u16 newstatus;
-		int connected;
-
-		if (priv->iw_mode == NL80211_IFTYPE_MONITOR)
-			break;
-
-		if (len != sizeof(linkstatus)) {
-			printk(KERN_WARNING "%s: Unexpected size for linkstatus frame (%d bytes)\n",
-			       dev->name, len);
-			break;
-		}
-
-		err = hw->ops->bap_pread(hw, IRQ_BAP, &linkstatus, len,
-					 infofid, sizeof(info));
-		if (err)
-			break;
-		newstatus = le16_to_cpu(linkstatus.linkstatus);
-
-		/* Symbol firmware uses "out of range" to signal that
-		 * the hostscan frame can be requested.  */
-		if (newstatus == HERMES_LINKSTATUS_AP_OUT_OF_RANGE &&
-		    priv->firmware_type == FIRMWARE_TYPE_SYMBOL &&
-		    priv->has_hostscan && priv->scan_request) {
-			hermes_inquire(hw, HERMES_INQ_HOSTSCAN_SYMBOL);
-			break;
-		}
-
-		connected = (newstatus == HERMES_LINKSTATUS_CONNECTED)
-			|| (newstatus == HERMES_LINKSTATUS_AP_CHANGE)
-			|| (newstatus == HERMES_LINKSTATUS_AP_IN_RANGE);
-
-		if (connected)
-			netif_carrier_on(dev);
-		else if (!ignore_disconnect)
-			netif_carrier_off(dev);
-
-		if (newstatus != priv->last_linkstatus) {
-			priv->last_linkstatus = newstatus;
-			print_linkstatus(dev, newstatus);
-			/* The info frame contains only one word which is the
-			 * status (see hermes.h). The status is pretty boring
-			 * in itself, that's why we export the new BSSID...
-			 * Jean II */
-			schedule_work(&priv->wevent_work);
-		}
-	}
-	break;
-	case HERMES_INQ_SCAN:
-		if (!priv->scan_request && priv->bssid_fixed &&
-		    priv->firmware_type == FIRMWARE_TYPE_INTERSIL) {
-			schedule_work(&priv->join_work);
-			break;
-		}
-		fallthrough;
-	case HERMES_INQ_HOSTSCAN:
-	case HERMES_INQ_HOSTSCAN_SYMBOL: {
-		/* Result of a scanning. Contains information about
-		 * cells in the vicinity - Jean II */
-		unsigned char *buf;
-
-		/* Sanity check */
-		if (len > 4096) {
-			printk(KERN_WARNING "%s: Scan results too large (%d bytes)\n",
-			       dev->name, len);
-			qabort_scan(priv);
-			break;
-		}
-
-		/* Allocate buffer for results */
-		buf = kmalloc(len, GFP_ATOMIC);
-		if (buf == NULL) {
-			/* No memory, so can't printk()... */
-			qabort_scan(priv);
-			break;
-		}
-
-		/* Read scan data */
-		err = hw->ops->bap_pread(hw, IRQ_BAP, (void *) buf, len,
-					 infofid, sizeof(info));
-		if (err) {
-			kfree(buf);
-			qabort_scan(priv);
-			break;
-		}
-
-#ifdef ORINOCO_DEBUG
-		{
-			int	i;
-			printk(KERN_DEBUG "Scan result [%02X", buf[0]);
-			for (i = 1; i < (len * 2); i++)
-				printk(":%02X", buf[i]);
-			printk("]\n");
-		}
-#endif	/* ORINOCO_DEBUG */
-
-		qbuf_scan(priv, buf, len, type);
-	}
-	break;
-	case HERMES_INQ_CHANNELINFO:
-	{
-		struct agere_ext_scan_info *bss;
-
-		if (!priv->scan_request) {
-			printk(KERN_DEBUG "%s: Got chaninfo without scan, "
-			       "len=%d\n", dev->name, len);
-			break;
-		}
-
-		/* An empty result indicates that the scan is complete */
-		if (len == 0) {
-			qbuf_scan(priv, NULL, len, type);
-			break;
-		}
-
-		/* Sanity check */
-		else if (len < (offsetof(struct agere_ext_scan_info,
-					   data) + 2)) {
-			/* Drop this result now so we don't have to
-			 * keep checking later */
-			printk(KERN_WARNING
-			       "%s: Ext scan results too short (%d bytes)\n",
-			       dev->name, len);
-			break;
-		}
-
-		bss = kmalloc(len, GFP_ATOMIC);
-		if (bss == NULL)
-			break;
-
-		/* Read scan data */
-		err = hw->ops->bap_pread(hw, IRQ_BAP, (void *) bss, len,
-					 infofid, sizeof(info));
-		if (err)
-			kfree(bss);
-		else
-			qbuf_scan(priv, bss, len, type);
-
-		break;
-	}
-	case HERMES_INQ_SEC_STAT_AGERE:
-		/* Security status (Agere specific) */
-		/* Ignore this frame for now */
-		if (priv->firmware_type == FIRMWARE_TYPE_AGERE)
-			break;
-		fallthrough;
-	default:
-		printk(KERN_DEBUG "%s: Unknown information frame received: "
-		       "type 0x%04x, length %d\n", dev->name, type, len);
-		/* We don't actually do anything about it */
-		break;
-	}
-}
-EXPORT_SYMBOL(__orinoco_ev_info);
-
-static void __orinoco_ev_infdrop(struct net_device *dev, struct hermes *hw)
-{
-	if (net_ratelimit())
-		printk(KERN_DEBUG "%s: Information frame lost.\n", dev->name);
-}
-
-/********************************************************************/
-/* Internal hardware control routines                               */
-/********************************************************************/
-
-static int __orinoco_up(struct orinoco_private *priv)
-{
-	struct net_device *dev = priv->ndev;
-	struct hermes *hw = &priv->hw;
-	int err;
-
-	netif_carrier_off(dev); /* just to make sure */
-
-	err = __orinoco_commit(priv);
-	if (err) {
-		printk(KERN_ERR "%s: Error %d configuring card\n",
-		       dev->name, err);
-		return err;
-	}
-
-	/* Fire things up again */
-	hermes_set_irqmask(hw, ORINOCO_INTEN);
-	err = hermes_enable_port(hw, 0);
-	if (err) {
-		printk(KERN_ERR "%s: Error %d enabling MAC port\n",
-		       dev->name, err);
-		return err;
-	}
-
-	netif_start_queue(dev);
-
-	return 0;
-}
-
-static int __orinoco_down(struct orinoco_private *priv)
-{
-	struct net_device *dev = priv->ndev;
-	struct hermes *hw = &priv->hw;
-	int err;
-
-	netif_stop_queue(dev);
-
-	if (!priv->hw_unavailable) {
-		if (!priv->broken_disableport) {
-			err = hermes_disable_port(hw, 0);
-			if (err) {
-				/* Some firmwares (e.g. Intersil 1.3.x) seem
-				 * to have problems disabling the port, oh
-				 * well, too bad. */
-				printk(KERN_WARNING "%s: Error %d disabling MAC port\n",
-				       dev->name, err);
-				priv->broken_disableport = 1;
-			}
-		}
-		hermes_set_irqmask(hw, 0);
-		hermes_write_regn(hw, EVACK, 0xffff);
-	}
-
-	orinoco_scan_done(priv, true);
-
-	/* firmware will have to reassociate */
-	netif_carrier_off(dev);
-	priv->last_linkstatus = 0xffff;
-
-	return 0;
-}
-
-static int orinoco_reinit_firmware(struct orinoco_private *priv)
-{
-	struct hermes *hw = &priv->hw;
-	int err;
-
-	err = hw->ops->init(hw);
-	if (priv->do_fw_download && !err) {
-		err = orinoco_download(priv);
-		if (err)
-			priv->do_fw_download = 0;
-	}
-	if (!err)
-		err = orinoco_hw_allocate_fid(priv);
-
-	return err;
-}
-
-static int
-__orinoco_set_multicast_list(struct net_device *dev)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	int err = 0;
-	int promisc, mc_count;
-
-	/* The Hermes doesn't seem to have an allmulti mode, so we go
-	 * into promiscuous mode and let the upper levels deal. */
-	if ((dev->flags & IFF_PROMISC) || (dev->flags & IFF_ALLMULTI) ||
-	    (netdev_mc_count(dev) > MAX_MULTICAST(priv))) {
-		promisc = 1;
-		mc_count = 0;
-	} else {
-		promisc = 0;
-		mc_count = netdev_mc_count(dev);
-	}
-
-	err = __orinoco_hw_set_multicast_list(priv, dev, mc_count, promisc);
-
-	return err;
-}
-
-/* This must be called from user context, without locks held - use
- * schedule_work() */
-void orinoco_reset(struct work_struct *work)
-{
-	struct orinoco_private *priv =
-		container_of(work, struct orinoco_private, reset_work);
-	struct net_device *dev = priv->ndev;
-	struct hermes *hw = &priv->hw;
-	int err;
-	unsigned long flags;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		/* When the hardware becomes available again, whatever
-		 * detects that is responsible for re-initializing
-		 * it. So no need for anything further */
-		return;
-
-	netif_stop_queue(dev);
-
-	/* Shut off interrupts.  Depending on what state the hardware
-	 * is in, this might not work, but we'll try anyway */
-	hermes_set_irqmask(hw, 0);
-	hermes_write_regn(hw, EVACK, 0xffff);
-
-	priv->hw_unavailable++;
-	priv->last_linkstatus = 0xffff; /* firmware will have to reassociate */
-	netif_carrier_off(dev);
-
-	orinoco_unlock(priv, &flags);
-
-	/* Scanning support: Notify scan cancellation */
-	orinoco_scan_done(priv, true);
-
-	if (priv->hard_reset) {
-		err = (*priv->hard_reset)(priv);
-		if (err) {
-			printk(KERN_ERR "%s: orinoco_reset: Error %d "
-			       "performing hard reset\n", dev->name, err);
-			goto disable;
-		}
-	}
-
-	err = orinoco_reinit_firmware(priv);
-	if (err) {
-		printk(KERN_ERR "%s: orinoco_reset: Error %d re-initializing firmware\n",
-		       dev->name, err);
-		goto disable;
-	}
-
-	/* This has to be called from user context */
-	orinoco_lock_irq(priv);
-
-	priv->hw_unavailable--;
-
-	/* priv->open or priv->hw_unavailable might have changed while
-	 * we dropped the lock */
-	if (priv->open && (!priv->hw_unavailable)) {
-		err = __orinoco_up(priv);
-		if (err) {
-			printk(KERN_ERR "%s: orinoco_reset: Error %d reenabling card\n",
-			       dev->name, err);
-		} else
-			netif_trans_update(dev);
-	}
-
-	orinoco_unlock_irq(priv);
-
-	return;
- disable:
-	hermes_set_irqmask(hw, 0);
-	netif_device_detach(dev);
-	printk(KERN_ERR "%s: Device has been disabled!\n", dev->name);
-}
-
-static int __orinoco_commit(struct orinoco_private *priv)
-{
-	struct net_device *dev = priv->ndev;
-	int err = 0;
-
-	/* If we've called commit, we are reconfiguring or bringing the
-	 * interface up. Maintaining countermeasures across this would
-	 * be confusing, so note that we've disabled them. The port will
-	 * be enabled later in orinoco_commit or __orinoco_up. */
-	priv->tkip_cm_active = 0;
-
-	err = orinoco_hw_program_rids(priv);
-
-	/* FIXME: what about netif_tx_lock */
-	(void) __orinoco_set_multicast_list(dev);
-
-	return err;
-}
-
-/* Ensures configuration changes are applied. May result in a reset.
- * The caller should hold priv->lock
- */
-int orinoco_commit(struct orinoco_private *priv)
-{
-	struct net_device *dev = priv->ndev;
-	struct hermes *hw = &priv->hw;
-	int err;
-
-	if (priv->broken_disableport) {
-		schedule_work(&priv->reset_work);
-		return 0;
-	}
-
-	err = hermes_disable_port(hw, 0);
-	if (err) {
-		printk(KERN_WARNING "%s: Unable to disable port "
-		       "while reconfiguring card\n", dev->name);
-		priv->broken_disableport = 1;
-		goto out;
-	}
-
-	err = __orinoco_commit(priv);
-	if (err) {
-		printk(KERN_WARNING "%s: Unable to reconfigure card\n",
-		       dev->name);
-		goto out;
-	}
-
-	err = hermes_enable_port(hw, 0);
-	if (err) {
-		printk(KERN_WARNING "%s: Unable to enable port while reconfiguring card\n",
-		       dev->name);
-		goto out;
-	}
-
- out:
-	if (err) {
-		printk(KERN_WARNING "%s: Resetting instead...\n", dev->name);
-		schedule_work(&priv->reset_work);
-		err = 0;
-	}
-	return err;
-}
-
-/********************************************************************/
-/* Interrupt handler                                                */
-/********************************************************************/
-
-static void __orinoco_ev_tick(struct net_device *dev, struct hermes *hw)
-{
-	printk(KERN_DEBUG "%s: TICK\n", dev->name);
-}
-
-static void __orinoco_ev_wterr(struct net_device *dev, struct hermes *hw)
-{
-	/* This seems to happen a fair bit under load, but ignoring it
-	   seems to work fine...*/
-	printk(KERN_DEBUG "%s: MAC controller error (WTERR). Ignoring.\n",
-	       dev->name);
-}
-
-irqreturn_t orinoco_interrupt(int irq, void *dev_id)
-{
-	struct orinoco_private *priv = dev_id;
-	struct net_device *dev = priv->ndev;
-	struct hermes *hw = &priv->hw;
-	int count = MAX_IRQLOOPS_PER_IRQ;
-	u16 evstat, events;
-	/* These are used to detect a runaway interrupt situation.
-	 *
-	 * If we get more than MAX_IRQLOOPS_PER_JIFFY iterations in a jiffy,
-	 * we panic and shut down the hardware
-	 */
-	/* jiffies value the last time we were called */
-	static int last_irq_jiffy; /* = 0 */
-	static int loops_this_jiffy; /* = 0 */
-	unsigned long flags;
-
-	if (orinoco_lock(priv, &flags) != 0) {
-		/* If hw is unavailable - we don't know if the irq was
-		 * for us or not */
-		return IRQ_HANDLED;
-	}
-
-	evstat = hermes_read_regn(hw, EVSTAT);
-	events = evstat & hw->inten;
-	if (!events) {
-		orinoco_unlock(priv, &flags);
-		return IRQ_NONE;
-	}
-
-	if (jiffies != last_irq_jiffy)
-		loops_this_jiffy = 0;
-	last_irq_jiffy = jiffies;
-
-	while (events && count--) {
-		if (++loops_this_jiffy > MAX_IRQLOOPS_PER_JIFFY) {
-			printk(KERN_WARNING "%s: IRQ handler is looping too "
-			       "much! Resetting.\n", dev->name);
-			/* Disable interrupts for now */
-			hermes_set_irqmask(hw, 0);
-			schedule_work(&priv->reset_work);
-			break;
-		}
-
-		/* Check the card hasn't been removed */
-		if (!hermes_present(hw)) {
-			DEBUG(0, "orinoco_interrupt(): card removed\n");
-			break;
-		}
-
-		if (events & HERMES_EV_TICK)
-			__orinoco_ev_tick(dev, hw);
-		if (events & HERMES_EV_WTERR)
-			__orinoco_ev_wterr(dev, hw);
-		if (events & HERMES_EV_INFDROP)
-			__orinoco_ev_infdrop(dev, hw);
-		if (events & HERMES_EV_INFO)
-			__orinoco_ev_info(dev, hw);
-		if (events & HERMES_EV_RX)
-			__orinoco_ev_rx(dev, hw);
-		if (events & HERMES_EV_TXEXC)
-			__orinoco_ev_txexc(dev, hw);
-		if (events & HERMES_EV_TX)
-			__orinoco_ev_tx(dev, hw);
-		if (events & HERMES_EV_ALLOC)
-			__orinoco_ev_alloc(dev, hw);
-
-		hermes_write_regn(hw, EVACK, evstat);
-
-		evstat = hermes_read_regn(hw, EVSTAT);
-		events = evstat & hw->inten;
-	}
-
-	orinoco_unlock(priv, &flags);
-	return IRQ_HANDLED;
-}
-EXPORT_SYMBOL(orinoco_interrupt);
-
-/********************************************************************/
-/* Power management                                                 */
-/********************************************************************/
-#if defined(CONFIG_PM_SLEEP) && !defined(CONFIG_HERMES_CACHE_FW_ON_INIT)
-static int orinoco_pm_notifier(struct notifier_block *notifier,
-			       unsigned long pm_event,
-			       void *unused)
-{
-	struct orinoco_private *priv = container_of(notifier,
-						    struct orinoco_private,
-						    pm_notifier);
-
-	/* All we need to do is cache the firmware before suspend, and
-	 * release it when we come out.
-	 *
-	 * Only need to do this if we're downloading firmware. */
-	if (!priv->do_fw_download)
-		return NOTIFY_DONE;
-
-	switch (pm_event) {
-	case PM_HIBERNATION_PREPARE:
-	case PM_SUSPEND_PREPARE:
-		orinoco_cache_fw(priv, 0);
-		break;
-
-	case PM_POST_RESTORE:
-		/* Restore from hibernation failed. We need to clean
-		 * up in exactly the same way, so fall through. */
-	case PM_POST_HIBERNATION:
-	case PM_POST_SUSPEND:
-		orinoco_uncache_fw(priv);
-		break;
-
-	case PM_RESTORE_PREPARE:
-	default:
-		break;
-	}
-
-	return NOTIFY_DONE;
-}
-
-static void orinoco_register_pm_notifier(struct orinoco_private *priv)
-{
-	priv->pm_notifier.notifier_call = orinoco_pm_notifier;
-	register_pm_notifier(&priv->pm_notifier);
-}
-
-static void orinoco_unregister_pm_notifier(struct orinoco_private *priv)
-{
-	unregister_pm_notifier(&priv->pm_notifier);
-}
-#else /* !PM_SLEEP || HERMES_CACHE_FW_ON_INIT */
-#define orinoco_register_pm_notifier(priv) do { } while (0)
-#define orinoco_unregister_pm_notifier(priv) do { } while (0)
-#endif
-
-/********************************************************************/
-/* Initialization                                                   */
-/********************************************************************/
-
-int orinoco_init(struct orinoco_private *priv)
-{
-	struct device *dev = priv->dev;
-	struct wiphy *wiphy = priv_to_wiphy(priv);
-	struct hermes *hw = &priv->hw;
-	int err = 0;
-
-	/* No need to lock, the hw_unavailable flag is already set in
-	 * alloc_orinocodev() */
-	priv->nicbuf_size = IEEE80211_MAX_FRAME_LEN + ETH_HLEN;
-
-	/* Initialize the firmware */
-	err = hw->ops->init(hw);
-	if (err != 0) {
-		dev_err(dev, "Failed to initialize firmware (err = %d)\n",
-			err);
-		goto out;
-	}
-
-	err = determine_fw_capabilities(priv, wiphy->fw_version,
-					sizeof(wiphy->fw_version),
-					&wiphy->hw_version);
-	if (err != 0) {
-		dev_err(dev, "Incompatible firmware, aborting\n");
-		goto out;
-	}
-
-	if (priv->do_fw_download) {
-#ifdef CONFIG_HERMES_CACHE_FW_ON_INIT
-		orinoco_cache_fw(priv, 0);
-#endif
-
-		err = orinoco_download(priv);
-		if (err)
-			priv->do_fw_download = 0;
-
-		/* Check firmware version again */
-		err = determine_fw_capabilities(priv, wiphy->fw_version,
-						sizeof(wiphy->fw_version),
-						&wiphy->hw_version);
-		if (err != 0) {
-			dev_err(dev, "Incompatible firmware, aborting\n");
-			goto out;
-		}
-	}
-
-	if (priv->has_port3)
-		dev_info(dev, "Ad-hoc demo mode supported\n");
-	if (priv->has_ibss)
-		dev_info(dev, "IEEE standard IBSS ad-hoc mode supported\n");
-	if (priv->has_wep)
-		dev_info(dev, "WEP supported, %s-bit key\n",
-			 priv->has_big_wep ? "104" : "40");
-	if (priv->has_wpa) {
-		dev_info(dev, "WPA-PSK supported\n");
-		if (orinoco_mic_init(priv)) {
-			dev_err(dev, "Failed to setup MIC crypto algorithm. "
-				"Disabling WPA support\n");
-			priv->has_wpa = 0;
-		}
-	}
-
-	err = orinoco_hw_read_card_settings(priv, wiphy->perm_addr);
-	if (err)
-		goto out;
-
-	err = orinoco_hw_allocate_fid(priv);
-	if (err) {
-		dev_err(dev, "Failed to allocate NIC buffer!\n");
-		goto out;
-	}
-
-	/* Set up the default configuration */
-	priv->iw_mode = NL80211_IFTYPE_STATION;
-	/* By default use IEEE/IBSS ad-hoc mode if we have it */
-	priv->prefer_port3 = priv->has_port3 && (!priv->has_ibss);
-	set_port_type(priv);
-	priv->channel = 0; /* use firmware default */
-
-	priv->promiscuous = 0;
-	priv->encode_alg = ORINOCO_ALG_NONE;
-	priv->tx_key = 0;
-	priv->wpa_enabled = 0;
-	priv->tkip_cm_active = 0;
-	priv->key_mgmt = 0;
-	priv->wpa_ie_len = 0;
-	priv->wpa_ie = NULL;
-
-	if (orinoco_wiphy_register(wiphy)) {
-		err = -ENODEV;
-		goto out;
-	}
-
-	/* Make the hardware available, as long as it hasn't been
-	 * removed elsewhere (e.g. by PCMCIA hot unplug) */
-	orinoco_lock_irq(priv);
-	priv->hw_unavailable--;
-	orinoco_unlock_irq(priv);
-
-	dev_dbg(dev, "Ready\n");
-
- out:
-	return err;
-}
-EXPORT_SYMBOL(orinoco_init);
-
-static const struct net_device_ops orinoco_netdev_ops = {
-	.ndo_open		= orinoco_open,
-	.ndo_stop		= orinoco_stop,
-	.ndo_start_xmit		= orinoco_xmit,
-	.ndo_set_rx_mode	= orinoco_set_multicast_list,
-	.ndo_change_mtu		= orinoco_change_mtu,
-	.ndo_set_mac_address	= eth_mac_addr,
-	.ndo_validate_addr	= eth_validate_addr,
-	.ndo_tx_timeout		= orinoco_tx_timeout,
-};
-
-/* Allocate private data.
- *
- * This driver has a number of structures associated with it
- *  netdev - Net device structure for each network interface
- *  wiphy - structure associated with wireless phy
- *  wireless_dev (wdev) - structure for each wireless interface
- *  hw - structure for hermes chip info
- *  card - card specific structure for use by the card driver
- *         (airport, orinoco_cs)
- *  priv - orinoco private data
- *  device - generic linux device structure
- *
- *  +---------+    +---------+
- *  |  wiphy  |    | netdev  |
- *  | +-------+    | +-------+
- *  | | priv  |    | | wdev  |
- *  | | +-----+    +-+-------+
- *  | | | hw  |
- *  | +-+-----+
- *  | | card  |
- *  +-+-------+
- *
- * priv has a link to netdev and device
- * wdev has a link to wiphy
- */
-struct orinoco_private
-*alloc_orinocodev(int sizeof_card,
-		  struct device *device,
-		  int (*hard_reset)(struct orinoco_private *),
-		  int (*stop_fw)(struct orinoco_private *, int))
-{
-	struct orinoco_private *priv;
-	struct wiphy *wiphy;
-
-	/* allocate wiphy
-	 * NOTE: We only support a single virtual interface
-	 *       but this may change when monitor mode is added
-	 */
-	wiphy = wiphy_new(&orinoco_cfg_ops,
-			  sizeof(struct orinoco_private) + sizeof_card);
-	if (!wiphy)
-		return NULL;
-
-	priv = wiphy_priv(wiphy);
-	priv->dev = device;
-
-	if (sizeof_card)
-		priv->card = (void *)((unsigned long)priv
-				      + sizeof(struct orinoco_private));
-	else
-		priv->card = NULL;
-
-	orinoco_wiphy_init(wiphy);
-
-#ifdef WIRELESS_SPY
-	priv->wireless_data.spy_data = &priv->spy_data;
-#endif
-
-	/* Set up default callbacks */
-	priv->hard_reset = hard_reset;
-	priv->stop_fw = stop_fw;
-
-	spin_lock_init(&priv->lock);
-	priv->open = 0;
-	priv->hw_unavailable = 1; /* orinoco_init() must clear this
-				   * before anything else touches the
-				   * hardware */
-	INIT_WORK(&priv->reset_work, orinoco_reset);
-	INIT_WORK(&priv->join_work, orinoco_join_ap);
-	INIT_WORK(&priv->wevent_work, orinoco_send_wevents);
-
-	INIT_LIST_HEAD(&priv->rx_list);
-	tasklet_setup(&priv->rx_tasklet, orinoco_rx_isr_tasklet);
-
-	spin_lock_init(&priv->scan_lock);
-	INIT_LIST_HEAD(&priv->scan_list);
-	INIT_WORK(&priv->process_scan, orinoco_process_scan_results);
-
-	priv->last_linkstatus = 0xffff;
-
-#if defined(CONFIG_HERMES_CACHE_FW_ON_INIT) || defined(CONFIG_PM_SLEEP)
-	priv->cached_pri_fw = NULL;
-	priv->cached_fw = NULL;
-#endif
-
-	/* Register PM notifiers */
-	orinoco_register_pm_notifier(priv);
-
-	return priv;
-}
-EXPORT_SYMBOL(alloc_orinocodev);
-
-/* We can only support a single interface. We provide a separate
- * function to set it up to distinguish between hardware
- * initialisation and interface setup.
- *
- * The base_addr and irq parameters are passed on to netdev for use
- * with SIOCGIFMAP.
- */
-int orinoco_if_add(struct orinoco_private *priv,
-		   unsigned long base_addr,
-		   unsigned int irq,
-		   const struct net_device_ops *ops)
-{
-	struct wiphy *wiphy = priv_to_wiphy(priv);
-	struct wireless_dev *wdev;
-	struct net_device *dev;
-	int ret;
-
-	dev = alloc_etherdev(sizeof(struct wireless_dev));
-
-	if (!dev)
-		return -ENOMEM;
-
-	/* Initialise wireless_dev */
-	wdev = netdev_priv(dev);
-	wdev->wiphy = wiphy;
-	wdev->iftype = NL80211_IFTYPE_STATION;
-
-	/* Setup / override net_device fields */
-	dev->ieee80211_ptr = wdev;
-	dev->watchdog_timeo = HZ; /* 1 second timeout */
-	dev->wireless_handlers = &orinoco_handler_def;
-#ifdef WIRELESS_SPY
-	dev->wireless_data = &priv->wireless_data;
-#endif
-	/* Default to standard ops if not set */
-	if (ops)
-		dev->netdev_ops = ops;
-	else
-		dev->netdev_ops = &orinoco_netdev_ops;
-
-	/* we use the default eth_mac_addr for setting the MAC addr */
-
-	/* Reserve space in skb for the SNAP header */
-	dev->needed_headroom = ENCAPS_OVERHEAD;
-
-	netif_carrier_off(dev);
-
-	eth_hw_addr_set(dev, wiphy->perm_addr);
-
-	dev->base_addr = base_addr;
-	dev->irq = irq;
-
-	dev->min_mtu = ORINOCO_MIN_MTU;
-	dev->max_mtu = ORINOCO_MAX_MTU;
-
-	SET_NETDEV_DEV(dev, priv->dev);
-	ret = register_netdev(dev);
-	if (ret)
-		goto fail;
-
-	priv->ndev = dev;
-
-	/* Report what we've done */
-	dev_dbg(priv->dev, "Registered interface %s.\n", dev->name);
-
-	return 0;
-
- fail:
-	free_netdev(dev);
-	return ret;
-}
-EXPORT_SYMBOL(orinoco_if_add);
-
-void orinoco_if_del(struct orinoco_private *priv)
-{
-	struct net_device *dev = priv->ndev;
-
-	unregister_netdev(dev);
-	free_netdev(dev);
-}
-EXPORT_SYMBOL(orinoco_if_del);
-
-void free_orinocodev(struct orinoco_private *priv)
-{
-	struct wiphy *wiphy = priv_to_wiphy(priv);
-	struct orinoco_rx_data *rx_data, *temp;
-	struct orinoco_scan_data *sd, *sdtemp;
-
-	/* If the tasklet is scheduled when we call tasklet_kill it
-	 * will run one final time. However the tasklet will only
-	 * drain priv->rx_list if the hw is still available. */
-	tasklet_kill(&priv->rx_tasklet);
-
-	/* Explicitly drain priv->rx_list */
-	list_for_each_entry_safe(rx_data, temp, &priv->rx_list, list) {
-		list_del(&rx_data->list);
-
-		dev_kfree_skb(rx_data->skb);
-		kfree(rx_data->desc);
-		kfree(rx_data);
-	}
-
-	cancel_work_sync(&priv->process_scan);
-	/* Explicitly drain priv->scan_list */
-	list_for_each_entry_safe(sd, sdtemp, &priv->scan_list, list) {
-		list_del(&sd->list);
-
-		if (sd->len > 0)
-			kfree(sd->buf);
-		kfree(sd);
-	}
-
-	orinoco_unregister_pm_notifier(priv);
-	orinoco_uncache_fw(priv);
-
-	priv->wpa_ie_len = 0;
-	kfree(priv->wpa_ie);
-	orinoco_mic_free(priv);
-	wiphy_free(wiphy);
-}
-EXPORT_SYMBOL(free_orinocodev);
-
-int orinoco_up(struct orinoco_private *priv)
-{
-	struct net_device *dev = priv->ndev;
-	unsigned long flags;
-	int err;
-
-	priv->hw.ops->lock_irqsave(&priv->lock, &flags);
-
-	err = orinoco_reinit_firmware(priv);
-	if (err) {
-		printk(KERN_ERR "%s: Error %d re-initializing firmware\n",
-		       dev->name, err);
-		goto exit;
-	}
-
-	netif_device_attach(dev);
-	priv->hw_unavailable--;
-
-	if (priv->open && !priv->hw_unavailable) {
-		err = __orinoco_up(priv);
-		if (err)
-			printk(KERN_ERR "%s: Error %d restarting card\n",
-			       dev->name, err);
-	}
-
-exit:
-	priv->hw.ops->unlock_irqrestore(&priv->lock, &flags);
-
-	return 0;
-}
-EXPORT_SYMBOL(orinoco_up);
-
-void orinoco_down(struct orinoco_private *priv)
-{
-	struct net_device *dev = priv->ndev;
-	unsigned long flags;
-	int err;
-
-	priv->hw.ops->lock_irqsave(&priv->lock, &flags);
-	err = __orinoco_down(priv);
-	if (err)
-		printk(KERN_WARNING "%s: Error %d downing interface\n",
-		       dev->name, err);
-
-	netif_device_detach(dev);
-	priv->hw_unavailable++;
-	priv->hw.ops->unlock_irqrestore(&priv->lock, &flags);
-}
-EXPORT_SYMBOL(orinoco_down);
-
-/********************************************************************/
-/* Module initialization                                            */
-/********************************************************************/
-
-/* Can't be declared "const" or the whole __initdata section will
- * become const */
-static char version[] __initdata = DRIVER_NAME " " DRIVER_VERSION
-	" (David Gibson <hermes@gibson.dropbear.id.au>, "
-	"Pavel Roskin <proski@gnu.org>, et al)";
-
-static int __init init_orinoco(void)
-{
-	printk(KERN_DEBUG "%s\n", version);
-	return 0;
-}
-
-static void __exit exit_orinoco(void)
-{
-}
-
-module_init(init_orinoco);
-module_exit(exit_orinoco);
diff --git a/drivers/net/wireless/intersil/orinoco/main.h b/drivers/net/wireless/intersil/orinoco/main.h
deleted file mode 100644
index 5a8fec26136e..000000000000
--- a/drivers/net/wireless/intersil/orinoco/main.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/* Exports from main to helper modules
- *
- * See copyright notice in main.c
- */
-#ifndef _ORINOCO_MAIN_H_
-#define _ORINOCO_MAIN_H_
-
-#include <linux/ieee80211.h>
-#include "orinoco.h"
-
-/********************************************************************/
-/* Compile time configuration and compatibility stuff               */
-/********************************************************************/
-
-/* We do this this way to avoid ifdefs in the actual code */
-#ifdef WIRELESS_SPY
-#define SPY_NUMBER(priv)	(priv->spy_data.spy_number)
-#else
-#define SPY_NUMBER(priv)	0
-#endif /* WIRELESS_SPY */
-
-/********************************************************************/
-
-/* Export module parameter */
-extern int force_monitor;
-
-/* Forward declarations */
-struct net_device;
-struct work_struct;
-
-void set_port_type(struct orinoco_private *priv);
-int orinoco_commit(struct orinoco_private *priv);
-void orinoco_reset(struct work_struct *work);
-
-/* Information element helpers - find a home for these... */
-#define WPA_OUI_TYPE	"\x00\x50\xF2\x01"
-#define WPA_SELECTOR_LEN 4
-static inline u8 *orinoco_get_wpa_ie(u8 *data, size_t len)
-{
-	u8 *p = data;
-	while ((p + 2 + WPA_SELECTOR_LEN) < (data + len)) {
-		if ((p[0] == WLAN_EID_VENDOR_SPECIFIC) &&
-		    (memcmp(&p[2], WPA_OUI_TYPE, WPA_SELECTOR_LEN) == 0))
-			return p;
-		p += p[1] + 2;
-	}
-	return NULL;
-}
-
-#endif /* _ORINOCO_MAIN_H_ */
diff --git a/drivers/net/wireless/intersil/orinoco/mic.c b/drivers/net/wireless/intersil/orinoco/mic.c
deleted file mode 100644
index a324bc4b7938..000000000000
--- a/drivers/net/wireless/intersil/orinoco/mic.c
+++ /dev/null
@@ -1,89 +0,0 @@
-/* Orinoco MIC helpers
- *
- * See copyright notice in main.c
- */
-#include <linux/kernel.h>
-#include <linux/string.h>
-#include <linux/if_ether.h>
-#include <linux/scatterlist.h>
-#include <crypto/hash.h>
-
-#include "orinoco.h"
-#include "mic.h"
-
-/********************************************************************/
-/* Michael MIC crypto setup                                         */
-/********************************************************************/
-int orinoco_mic_init(struct orinoco_private *priv)
-{
-	priv->tx_tfm_mic = crypto_alloc_shash("michael_mic", 0, 0);
-	if (IS_ERR(priv->tx_tfm_mic)) {
-		printk(KERN_DEBUG "%s: could not allocate "
-		       "crypto API michael_mic\n", __func__);
-		priv->tx_tfm_mic = NULL;
-		return -ENOMEM;
-	}
-
-	priv->rx_tfm_mic = crypto_alloc_shash("michael_mic", 0, 0);
-	if (IS_ERR(priv->rx_tfm_mic)) {
-		printk(KERN_DEBUG "%s: could not allocate "
-		       "crypto API michael_mic\n", __func__);
-		priv->rx_tfm_mic = NULL;
-		return -ENOMEM;
-	}
-
-	return 0;
-}
-
-void orinoco_mic_free(struct orinoco_private *priv)
-{
-	if (priv->tx_tfm_mic)
-		crypto_free_shash(priv->tx_tfm_mic);
-	if (priv->rx_tfm_mic)
-		crypto_free_shash(priv->rx_tfm_mic);
-}
-
-int orinoco_mic(struct crypto_shash *tfm_michael, u8 *key,
-		u8 *da, u8 *sa, u8 priority,
-		u8 *data, size_t data_len, u8 *mic)
-{
-	SHASH_DESC_ON_STACK(desc, tfm_michael);
-	u8 hdr[ETH_HLEN + 2]; /* size of header + padding */
-	int err;
-
-	if (tfm_michael == NULL) {
-		printk(KERN_WARNING "%s: tfm_michael == NULL\n", __func__);
-		return -1;
-	}
-
-	/* Copy header into buffer. We need the padding on the end zeroed */
-	memcpy(&hdr[0], da, ETH_ALEN);
-	memcpy(&hdr[ETH_ALEN], sa, ETH_ALEN);
-	hdr[ETH_ALEN * 2] = priority;
-	hdr[ETH_ALEN * 2 + 1] = 0;
-	hdr[ETH_ALEN * 2 + 2] = 0;
-	hdr[ETH_ALEN * 2 + 3] = 0;
-
-	desc->tfm = tfm_michael;
-
-	err = crypto_shash_setkey(tfm_michael, key, MIC_KEYLEN);
-	if (err)
-		return err;
-
-	err = crypto_shash_init(desc);
-	if (err)
-		return err;
-
-	err = crypto_shash_update(desc, hdr, sizeof(hdr));
-	if (err)
-		return err;
-
-	err = crypto_shash_update(desc, data, data_len);
-	if (err)
-		return err;
-
-	err = crypto_shash_final(desc, mic);
-	shash_desc_zero(desc);
-
-	return err;
-}
diff --git a/drivers/net/wireless/intersil/orinoco/mic.h b/drivers/net/wireless/intersil/orinoco/mic.h
deleted file mode 100644
index e8724e889219..000000000000
--- a/drivers/net/wireless/intersil/orinoco/mic.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/* Orinoco MIC helpers
- *
- * See copyright notice in main.c
- */
-#ifndef _ORINOCO_MIC_H_
-#define _ORINOCO_MIC_H_
-
-#include <linux/types.h>
-#include <crypto/hash.h>
-
-#define MICHAEL_MIC_LEN 8
-
-/* Forward declarations */
-struct orinoco_private;
-struct crypto_ahash;
-
-int orinoco_mic_init(struct orinoco_private *priv);
-void orinoco_mic_free(struct orinoco_private *priv);
-int orinoco_mic(struct crypto_shash *tfm_michael, u8 *key,
-		u8 *da, u8 *sa, u8 priority,
-		u8 *data, size_t data_len, u8 *mic);
-
-#endif /* ORINOCO_MIC_H */
diff --git a/drivers/net/wireless/intersil/orinoco/orinoco.h b/drivers/net/wireless/intersil/orinoco/orinoco.h
deleted file mode 100644
index cdd026af100b..000000000000
--- a/drivers/net/wireless/intersil/orinoco/orinoco.h
+++ /dev/null
@@ -1,251 +0,0 @@
-/* orinoco.h
- *
- * Common definitions to all pieces of the various orinoco
- * drivers
- */
-
-#ifndef _ORINOCO_H
-#define _ORINOCO_H
-
-#define DRIVER_VERSION "0.15"
-
-#include <linux/interrupt.h>
-#include <linux/suspend.h>
-#include <linux/netdevice.h>
-#include <linux/wireless.h>
-#include <net/iw_handler.h>
-#include <net/cfg80211.h>
-
-#include "hermes.h"
-
-/* To enable debug messages */
-/*#define ORINOCO_DEBUG		3*/
-
-#define WIRELESS_SPY		/* enable iwspy support */
-
-#define MAX_SCAN_LEN		4096
-
-#define ORINOCO_SEQ_LEN		8
-#define ORINOCO_MAX_KEY_SIZE	14
-#define ORINOCO_MAX_KEYS	4
-
-struct orinoco_key {
-	__le16 len;	/* always stored as little-endian */
-	char data[ORINOCO_MAX_KEY_SIZE];
-} __packed;
-
-#define TKIP_KEYLEN	16
-#define MIC_KEYLEN	8
-
-struct orinoco_tkip_key {
-	u8 tkip[TKIP_KEYLEN];
-	u8 tx_mic[MIC_KEYLEN];
-	u8 rx_mic[MIC_KEYLEN];
-};
-
-enum orinoco_alg {
-	ORINOCO_ALG_NONE,
-	ORINOCO_ALG_WEP,
-	ORINOCO_ALG_TKIP
-};
-
-enum fwtype {
-	FIRMWARE_TYPE_AGERE,
-	FIRMWARE_TYPE_INTERSIL,
-	FIRMWARE_TYPE_SYMBOL
-};
-
-struct firmware;
-
-struct orinoco_private {
-	void *card;	/* Pointer to card dependent structure */
-	struct device *dev;
-	int (*hard_reset)(struct orinoco_private *);
-	int (*stop_fw)(struct orinoco_private *, int);
-
-	struct ieee80211_supported_band band;
-	struct ieee80211_channel channels[14];
-	u32 cipher_suites[3];
-
-	/* Synchronisation stuff */
-	spinlock_t lock;
-	int hw_unavailable;
-	struct work_struct reset_work;
-
-	/* Interrupt tasklets */
-	struct tasklet_struct rx_tasklet;
-	struct list_head rx_list;
-
-	/* driver state */
-	int open;
-	u16 last_linkstatus;
-	struct work_struct join_work;
-	struct work_struct wevent_work;
-
-	/* Net device stuff */
-	struct net_device *ndev;
-	struct iw_statistics wstats;
-
-	/* Hardware control variables */
-	struct hermes hw;
-	u16 txfid;
-
-	/* Capabilities of the hardware/firmware */
-	enum fwtype firmware_type;
-	int ibss_port;
-	int nicbuf_size;
-	u16 channel_mask;
-
-	/* Boolean capabilities */
-	unsigned int has_ibss:1;
-	unsigned int has_port3:1;
-	unsigned int has_wep:1;
-	unsigned int has_big_wep:1;
-	unsigned int has_mwo:1;
-	unsigned int has_pm:1;
-	unsigned int has_preamble:1;
-	unsigned int has_sensitivity:1;
-	unsigned int has_hostscan:1;
-	unsigned int has_alt_txcntl:1;
-	unsigned int has_ext_scan:1;
-	unsigned int has_wpa:1;
-	unsigned int do_fw_download:1;
-	unsigned int broken_disableport:1;
-	unsigned int broken_monitor:1;
-	unsigned int prefer_port3:1;
-
-	/* Configuration paramaters */
-	enum nl80211_iftype iw_mode;
-	enum orinoco_alg encode_alg;
-	u16 wep_restrict, tx_key;
-	struct key_params keys[ORINOCO_MAX_KEYS];
-
-	int bitratemode;
-	char nick[IW_ESSID_MAX_SIZE + 1];
-	char desired_essid[IW_ESSID_MAX_SIZE + 1];
-	char desired_bssid[ETH_ALEN];
-	int bssid_fixed;
-	u16 frag_thresh, mwo_robust;
-	u16 channel;
-	u16 ap_density, rts_thresh;
-	u16 pm_on, pm_mcast, pm_period, pm_timeout;
-	u16 preamble;
-	u16 short_retry_limit, long_retry_limit;
-	u16 retry_lifetime;
-#ifdef WIRELESS_SPY
-	struct iw_spy_data spy_data; /* iwspy support */
-	struct iw_public_data	wireless_data;
-#endif
-
-	/* Configuration dependent variables */
-	int port_type, createibss;
-	int promiscuous, mc_count;
-
-	/* Scanning support */
-	struct cfg80211_scan_request *scan_request;
-	struct work_struct process_scan;
-	struct list_head scan_list;
-	spinlock_t scan_lock; /* protects the scan list */
-
-	/* WPA support */
-	u8 *wpa_ie;
-	int wpa_ie_len;
-
-	struct crypto_shash *rx_tfm_mic;
-	struct crypto_shash *tx_tfm_mic;
-
-	unsigned int wpa_enabled:1;
-	unsigned int tkip_cm_active:1;
-	unsigned int key_mgmt:3;
-
-#if defined(CONFIG_HERMES_CACHE_FW_ON_INIT) || defined(CONFIG_PM_SLEEP)
-	/* Cached in memory firmware to use during ->resume. */
-	const struct firmware *cached_pri_fw;
-	const struct firmware *cached_fw;
-#endif
-
-	struct notifier_block pm_notifier;
-};
-
-#ifdef ORINOCO_DEBUG
-extern int orinoco_debug;
-#define DEBUG(n, args...) do { \
-	if (orinoco_debug > (n)) \
-		printk(KERN_DEBUG args); \
-} while (0)
-#else
-#define DEBUG(n, args...) do { } while (0)
-#endif	/* ORINOCO_DEBUG */
-
-/********************************************************************/
-/* Exported prototypes                                              */
-/********************************************************************/
-
-struct orinoco_private *alloc_orinocodev(int sizeof_card, struct device *device,
-					 int (*hard_reset)(struct orinoco_private *),
-					 int (*stop_fw)(struct orinoco_private *, int));
-void free_orinocodev(struct orinoco_private *priv);
-int orinoco_init(struct orinoco_private *priv);
-int orinoco_if_add(struct orinoco_private *priv, unsigned long base_addr,
-		   unsigned int irq, const struct net_device_ops *ops);
-void orinoco_if_del(struct orinoco_private *priv);
-int orinoco_up(struct orinoco_private *priv);
-void orinoco_down(struct orinoco_private *priv);
-irqreturn_t orinoco_interrupt(int irq, void *dev_id);
-
-void __orinoco_ev_info(struct net_device *dev, struct hermes *hw);
-void __orinoco_ev_rx(struct net_device *dev, struct hermes *hw);
-
-int orinoco_process_xmit_skb(struct sk_buff *skb,
-			     struct net_device *dev,
-			     struct orinoco_private *priv,
-			     int *tx_control,
-			     u8 *mic);
-
-/* Common ndo functions exported for reuse by orinoco_usb */
-int orinoco_open(struct net_device *dev);
-int orinoco_stop(struct net_device *dev);
-void orinoco_set_multicast_list(struct net_device *dev);
-int orinoco_change_mtu(struct net_device *dev, int new_mtu);
-void orinoco_tx_timeout(struct net_device *dev, unsigned int txqueue);
-
-/********************************************************************/
-/* Locking and synchronization functions                            */
-/********************************************************************/
-
-static inline int orinoco_lock(struct orinoco_private *priv,
-			       unsigned long *flags)
-{
-	priv->hw.ops->lock_irqsave(&priv->lock, flags);
-	if (priv->hw_unavailable) {
-		DEBUG(1, "orinoco_lock() called with hw_unavailable (dev=%p)\n",
-		       priv->ndev);
-		priv->hw.ops->unlock_irqrestore(&priv->lock, flags);
-		return -EBUSY;
-	}
-	return 0;
-}
-
-static inline void orinoco_unlock(struct orinoco_private *priv,
-				  unsigned long *flags)
-{
-	priv->hw.ops->unlock_irqrestore(&priv->lock, flags);
-}
-
-static inline void orinoco_lock_irq(struct orinoco_private *priv)
-{
-	priv->hw.ops->lock_irq(&priv->lock);
-}
-
-static inline void orinoco_unlock_irq(struct orinoco_private *priv)
-{
-	priv->hw.ops->unlock_irq(&priv->lock);
-}
-
-/*** Navigate from net_device to orinoco_private ***/
-static inline struct orinoco_private *ndev_priv(struct net_device *dev)
-{
-	struct wireless_dev *wdev = netdev_priv(dev);
-	return wdev_priv(wdev);
-}
-#endif /* _ORINOCO_H */
diff --git a/drivers/net/wireless/intersil/orinoco/orinoco_cs.c b/drivers/net/wireless/intersil/orinoco/orinoco_cs.c
deleted file mode 100644
index 03bfd2482656..000000000000
--- a/drivers/net/wireless/intersil/orinoco/orinoco_cs.c
+++ /dev/null
@@ -1,350 +0,0 @@
-/* orinoco_cs.c (formerly known as dldwd_cs.c)
- *
- * A driver for "Hermes" chipset based PCMCIA wireless adaptors, such
- * as the Lucent WavelanIEEE/Orinoco cards and their OEM (Cabletron/
- * EnteraSys RoamAbout 802.11, ELSA Airlancer, Melco Buffalo and others).
- * It should also be usable on various Prism II based cards such as the
- * Linksys, D-Link and Farallon Skyline. It should also work on Symbol
- * cards such as the 3Com AirConnect and Ericsson WLAN.
- *
- * Copyright notice & release notes in file main.c
- */
-
-#define DRIVER_NAME "orinoco_cs"
-#define PFX DRIVER_NAME ": "
-
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/delay.h>
-#include <pcmcia/cistpl.h>
-#include <pcmcia/cisreg.h>
-#include <pcmcia/ds.h>
-
-#include "orinoco.h"
-
-/********************************************************************/
-/* Module stuff							    */
-/********************************************************************/
-
-MODULE_AUTHOR("David Gibson <hermes@gibson.dropbear.id.au>");
-MODULE_DESCRIPTION("Driver for PCMCIA Lucent Orinoco,"
-		   " Prism II based and similar wireless cards");
-MODULE_LICENSE("Dual MPL/GPL");
-
-/* Module parameters */
-
-/* Some D-Link cards have buggy CIS. They do work at 5v properly, but
- * don't have any CIS entry for it. This workaround it... */
-static int ignore_cis_vcc; /* = 0 */
-module_param(ignore_cis_vcc, int, 0);
-MODULE_PARM_DESC(ignore_cis_vcc, "Allow voltage mismatch between card and socket");
-
-/********************************************************************/
-/* Data structures						    */
-/********************************************************************/
-
-/* PCMCIA specific device information (goes in the card field of
- * struct orinoco_private */
-struct orinoco_pccard {
-	struct pcmcia_device	*p_dev;
-
-	/* Used to handle hard reset */
-	/* yuck, we need this hack to work around the insanity of the
-	 * PCMCIA layer */
-	unsigned long hard_reset_in_progress;
-};
-
-
-/********************************************************************/
-/* Function prototypes						    */
-/********************************************************************/
-
-static int orinoco_cs_config(struct pcmcia_device *link);
-static void orinoco_cs_release(struct pcmcia_device *link);
-static void orinoco_cs_detach(struct pcmcia_device *p_dev);
-
-/********************************************************************/
-/* Device methods						    */
-/********************************************************************/
-
-static int
-orinoco_cs_hard_reset(struct orinoco_private *priv)
-{
-	struct orinoco_pccard *card = priv->card;
-	struct pcmcia_device *link = card->p_dev;
-	int err;
-
-	/* We need atomic ops here, because we're not holding the lock */
-	set_bit(0, &card->hard_reset_in_progress);
-
-	err = pcmcia_reset_card(link->socket);
-	if (err)
-		return err;
-
-	msleep(100);
-	clear_bit(0, &card->hard_reset_in_progress);
-
-	return 0;
-}
-
-/********************************************************************/
-/* PCMCIA stuff							    */
-/********************************************************************/
-
-static int
-orinoco_cs_probe(struct pcmcia_device *link)
-{
-	struct orinoco_private *priv;
-	struct orinoco_pccard *card;
-	int ret;
-
-	priv = alloc_orinocodev(sizeof(*card), &link->dev,
-				orinoco_cs_hard_reset, NULL);
-	if (!priv)
-		return -ENOMEM;
-	card = priv->card;
-
-	/* Link both structures together */
-	card->p_dev = link;
-	link->priv = priv;
-
-	ret = orinoco_cs_config(link);
-	if (ret)
-		goto err_free_orinocodev;
-
-	return 0;
-
-err_free_orinocodev:
-	free_orinocodev(priv);
-	return ret;
-}
-
-static void orinoco_cs_detach(struct pcmcia_device *link)
-{
-	struct orinoco_private *priv = link->priv;
-
-	orinoco_if_del(priv);
-
-	orinoco_cs_release(link);
-
-	wiphy_unregister(priv_to_wiphy(priv));
-	free_orinocodev(priv);
-}				/* orinoco_cs_detach */
-
-static int orinoco_cs_config_check(struct pcmcia_device *p_dev, void *priv_data)
-{
-	if (p_dev->config_index == 0)
-		return -EINVAL;
-
-	return pcmcia_request_io(p_dev);
-};
-
-static int
-orinoco_cs_config(struct pcmcia_device *link)
-{
-	struct orinoco_private *priv = link->priv;
-	struct hermes *hw = &priv->hw;
-	int ret;
-	void __iomem *mem;
-
-	link->config_flags |= CONF_AUTO_SET_VPP | CONF_AUTO_CHECK_VCC |
-		CONF_AUTO_SET_IO | CONF_ENABLE_IRQ;
-	if (ignore_cis_vcc)
-		link->config_flags &= ~CONF_AUTO_CHECK_VCC;
-	ret = pcmcia_loop_config(link, orinoco_cs_config_check, NULL);
-	if (ret) {
-		if (!ignore_cis_vcc)
-			printk(KERN_ERR PFX "GetNextTuple(): No matching "
-			       "CIS configuration.  Maybe you need the "
-			       "ignore_cis_vcc=1 parameter.\n");
-		goto failed;
-	}
-
-	mem = ioport_map(link->resource[0]->start,
-			resource_size(link->resource[0]));
-	if (!mem)
-		goto failed;
-
-	/* We initialize the hermes structure before completing PCMCIA
-	 * configuration just in case the interrupt handler gets
-	 * called. */
-	hermes_struct_init(hw, mem, HERMES_16BIT_REGSPACING);
-
-	ret = pcmcia_request_irq(link, orinoco_interrupt);
-	if (ret)
-		goto failed;
-
-	ret = pcmcia_enable_device(link);
-	if (ret)
-		goto failed;
-
-	/* Initialise the main driver */
-	if (orinoco_init(priv) != 0) {
-		printk(KERN_ERR PFX "orinoco_init() failed\n");
-		goto failed;
-	}
-
-	/* Register an interface with the stack */
-	if (orinoco_if_add(priv, link->resource[0]->start,
-			   link->irq, NULL) != 0) {
-		printk(KERN_ERR PFX "orinoco_if_add() failed\n");
-		goto failed;
-	}
-
-	return 0;
-
- failed:
-	orinoco_cs_release(link);
-	return -ENODEV;
-}				/* orinoco_cs_config */
-
-static void
-orinoco_cs_release(struct pcmcia_device *link)
-{
-	struct orinoco_private *priv = link->priv;
-	unsigned long flags;
-
-	/* We're committed to taking the device away now, so mark the
-	 * hardware as unavailable */
-	priv->hw.ops->lock_irqsave(&priv->lock, &flags);
-	priv->hw_unavailable++;
-	priv->hw.ops->unlock_irqrestore(&priv->lock, &flags);
-
-	pcmcia_disable_device(link);
-	if (priv->hw.iobase)
-		ioport_unmap(priv->hw.iobase);
-}				/* orinoco_cs_release */
-
-static int orinoco_cs_suspend(struct pcmcia_device *link)
-{
-	struct orinoco_private *priv = link->priv;
-	struct orinoco_pccard *card = priv->card;
-
-	/* This is probably racy, but I can't think of
-	   a better way, short of rewriting the PCMCIA
-	   layer to not suck :-( */
-	if (!test_bit(0, &card->hard_reset_in_progress))
-		orinoco_down(priv);
-
-	return 0;
-}
-
-static int orinoco_cs_resume(struct pcmcia_device *link)
-{
-	struct orinoco_private *priv = link->priv;
-	struct orinoco_pccard *card = priv->card;
-	int err = 0;
-
-	if (!test_bit(0, &card->hard_reset_in_progress))
-		err = orinoco_up(priv);
-
-	return err;
-}
-
-
-/********************************************************************/
-/* Module initialization					    */
-/********************************************************************/
-
-static const struct pcmcia_device_id orinoco_cs_ids[] = {
-	PCMCIA_DEVICE_MANF_CARD(0x0101, 0x0777), /* 3Com AirConnect PCI 777A */
-	PCMCIA_DEVICE_MANF_CARD(0x016b, 0x0001), /* Ericsson WLAN Card C11 */
-	PCMCIA_DEVICE_MANF_CARD(0x01eb, 0x080a), /* Nortel Networks eMobility 802.11 Wireless Adapter */
-	PCMCIA_DEVICE_MANF_CARD(0x0261, 0x0002), /* AirWay 802.11 Adapter (PCMCIA) */
-	PCMCIA_DEVICE_MANF_CARD(0x0268, 0x0001), /* ARtem Onair */
-	PCMCIA_DEVICE_MANF_CARD(0x0268, 0x0003), /* ARtem Onair Comcard 11 */
-	PCMCIA_DEVICE_MANF_CARD(0x026f, 0x0305), /* Buffalo WLI-PCM-S11 */
-	PCMCIA_DEVICE_MANF_CARD(0x02aa, 0x0002), /* ASUS SpaceLink WL-100 */
-	PCMCIA_DEVICE_MANF_CARD(0x02ac, 0x0002), /* SpeedStream SS1021 Wireless Adapter */
-	PCMCIA_DEVICE_MANF_CARD(0x02ac, 0x3021), /* SpeedStream Wireless Adapter */
-	PCMCIA_DEVICE_MANF_CARD(0x14ea, 0xb001), /* PLANEX RoadLannerWave GW-NS11H */
-	PCMCIA_DEVICE_PROD_ID12("3Com", "3CRWE737A AirConnect Wireless LAN PC Card", 0x41240e5b, 0x56010af3),
-	PCMCIA_DEVICE_PROD_ID12("Allied Telesyn", "AT-WCL452 Wireless PCMCIA Radio", 0x5cd01705, 0x4271660f),
-	PCMCIA_DEVICE_PROD_ID12("ASUS", "802_11B_CF_CARD_25", 0x78fc06ee, 0x45a50c1e),
-	PCMCIA_DEVICE_PROD_ID12("ASUS", "802_11b_PC_CARD_25", 0x78fc06ee, 0xdb9aa842),
-	PCMCIA_DEVICE_PROD_ID12("Avaya Communication", "Avaya Wireless PC Card", 0xd8a43b78, 0x0d341169),
-	PCMCIA_DEVICE_PROD_ID12("BENQ", "AWL100 PCMCIA ADAPTER", 0x35dadc74, 0x01f7fedb),
-	PCMCIA_DEVICE_PROD_ID12("Cabletron", "RoamAbout 802.11 DS", 0x32d445f5, 0xedeffd90),
-	PCMCIA_DEVICE_PROD_ID12("D-Link Corporation", "D-Link DWL-650H 11Mbps WLAN Adapter", 0xef544d24, 0xcd8ea916),
-	PCMCIA_DEVICE_PROD_ID12("ELSA", "AirLancer MC-11", 0x4507a33a, 0xef54f0e3),
-	PCMCIA_DEVICE_PROD_ID12("HyperLink", "Wireless PC Card 11Mbps", 0x56cc3f1a, 0x0bcf220c),
-	PCMCIA_DEVICE_PROD_ID12("Intel", "PRO/Wireless 2011 LAN PC Card", 0x816cc815, 0x07f58077),
-	PCMCIA_DEVICE_PROD_ID12("LeArtery", "SYNCBYAIR 11Mbps Wireless LAN PC Card", 0x7e3b326a, 0x49893e92),
-	PCMCIA_DEVICE_PROD_ID12("Lucent Technologies", "WaveLAN/IEEE", 0x23eb9949, 0xc562e72a),
-	PCMCIA_DEVICE_PROD_ID12("MELCO", "WLI-PCM-L11", 0x481e0094, 0x7360e410),
-	PCMCIA_DEVICE_PROD_ID12("MELCO", "WLI-PCM-L11G", 0x481e0094, 0xf57ca4b3),
-	PCMCIA_DEVICE_PROD_ID12("NCR", "WaveLAN/IEEE", 0x24358cd4, 0xc562e72a),
-	PCMCIA_DEVICE_PROD_ID12("Nortel Networks", "emobility 802.11 Wireless LAN PC Card", 0x2d617ea0, 0x88cd5767),
-	PCMCIA_DEVICE_PROD_ID12("OTC", "Wireless AirEZY 2411-PCC WLAN Card", 0x4ac44287, 0x235a6bed),
-	PCMCIA_DEVICE_PROD_ID12("PROXIM", "LAN PC CARD HARMONY 80211B", 0xc6536a5e, 0x090c3cd9),
-	PCMCIA_DEVICE_PROD_ID12("PROXIM", "LAN PCI CARD HARMONY 80211B", 0xc6536a5e, 0x9f494e26),
-	PCMCIA_DEVICE_PROD_ID12("SAMSUNG", "11Mbps WLAN Card", 0x43d74cb4, 0x579bd91b),
-	PCMCIA_DEVICE_PROD_ID12("Symbol Technologies", "LA4111 Spectrum24 Wireless LAN PC Card", 0x3f02b4d6, 0x3663cb0e),
-	PCMCIA_DEVICE_MANF_CARD_PROD_ID3(0x0156, 0x0002, "Version 01.01", 0xd27deb1a), /* Lucent Orinoco */
-#ifdef CONFIG_HERMES_PRISM
-	/* Only entries that certainly identify Prism chipset */
-	PCMCIA_DEVICE_MANF_CARD(0x000b, 0x7100), /* SonicWALL Long Range Wireless Card */
-	PCMCIA_DEVICE_MANF_CARD(0x000b, 0x7300), /* Sohoware NCP110, Philips 802.11b */
-	PCMCIA_DEVICE_MANF_CARD(0x0089, 0x0002), /* AnyPoint(TM) Wireless II PC Card */
-	PCMCIA_DEVICE_MANF_CARD(0x0126, 0x8000), /* PROXIM RangeLAN-DS/LAN PC CARD */
-	PCMCIA_DEVICE_MANF_CARD(0x0138, 0x0002), /* Compaq WL100 11 Mbps Wireless Adapter */
-	PCMCIA_DEVICE_MANF_CARD(0x01ff, 0x0008), /* Intermec MobileLAN 11Mbps 802.11b WLAN Card */
-	PCMCIA_DEVICE_MANF_CARD(0x0250, 0x0002), /* Samsung SWL2000-N 11Mb/s WLAN Card */
-	PCMCIA_DEVICE_MANF_CARD(0x0274, 0x1612), /* Linksys WPC11 Version 2.5 */
-	PCMCIA_DEVICE_MANF_CARD(0x0274, 0x1613), /* Linksys WPC11 Version 3 */
-	PCMCIA_DEVICE_MANF_CARD(0x028a, 0x0002), /* Compaq HNW-100 11 Mbps Wireless Adapter */
-	PCMCIA_DEVICE_MANF_CARD(0x028a, 0x0673), /* Linksys WCF12 Wireless CompactFlash Card */
-	PCMCIA_DEVICE_MANF_CARD(0x50c2, 0x7300), /* Airvast WN-100 */
-	PCMCIA_DEVICE_MANF_CARD(0x9005, 0x0021), /* Adaptec Ultra Wireless ANW-8030 */
-	PCMCIA_DEVICE_MANF_CARD(0xc001, 0x0008), /* CONTEC FLEXSCAN/FX-DDS110-PCC */
-	PCMCIA_DEVICE_MANF_CARD(0xc250, 0x0002), /* Conceptronic CON11Cpro, EMTAC A2424i */
-	PCMCIA_DEVICE_MANF_CARD(0xd601, 0x0002), /* Safeway 802.11b, ZCOMAX AirRunner/XI-300 */
-	PCMCIA_DEVICE_MANF_CARD(0xd601, 0x0005), /* D-Link DCF660, Sandisk Connect SDWCFB-000 */
-	PCMCIA_DEVICE_PROD_ID123("Instant Wireless ", " Network PC CARD", "Version 01.02", 0x11d901af, 0x6e9bd926, 0x4b74baa0),
-	PCMCIA_DEVICE_PROD_ID12("ACTIONTEC", "PRISM Wireless LAN PC Card", 0x393089da, 0xa71e69d5),
-	PCMCIA_DEVICE_PROD_ID12("Addtron", "AWP-100 Wireless PCMCIA", 0xe6ec52ce, 0x08649af2),
-	PCMCIA_DEVICE_PROD_ID12("BUFFALO", "WLI-CF-S11G", 0x2decece3, 0x82067c18),
-	PCMCIA_DEVICE_PROD_ID12("BUFFALO", "WLI-PCM-L11G", 0x2decece3, 0xf57ca4b3),
-	PCMCIA_DEVICE_PROD_ID12("Compaq", "WL200_11Mbps_Wireless_PCI_Card", 0x54f7c49c, 0x15a75e5b),
-	PCMCIA_DEVICE_PROD_ID12("corega K.K.", "Wireless LAN PCC-11", 0x5261440f, 0xa6405584),
-	PCMCIA_DEVICE_PROD_ID12("corega K.K.", "Wireless LAN PCCA-11", 0x5261440f, 0xdf6115f9),
-	PCMCIA_DEVICE_PROD_ID12("corega_K.K.", "Wireless_LAN_PCCB-11", 0x29e33311, 0xee7a27ae),
-	PCMCIA_DEVICE_PROD_ID12("Digital Data Communications", "WPC-0100", 0xfdd73470, 0xe0b6f146),
-	PCMCIA_DEVICE_PROD_ID12("D", "Link DRC-650 11Mbps WLAN Card", 0x71b18589, 0xf144e3ac),
-	PCMCIA_DEVICE_PROD_ID12("D", "Link DWL-650 11Mbps WLAN Card", 0x71b18589, 0xb6f1b0ab),
-	PCMCIA_DEVICE_PROD_ID12(" ", "IEEE 802.11 Wireless LAN/PC Card", 0x3b6e20c8, 0xefccafe9),
-	PCMCIA_DEVICE_PROD_ID12("INTERSIL", "HFA384x/IEEE", 0x74c5e40d, 0xdb472a18),
-	PCMCIA_DEVICE_PROD_ID12("INTERSIL", "I-GATE 11M PC Card / PC Card plus", 0x74c5e40d, 0x8304ff77),
-	PCMCIA_DEVICE_PROD_ID12("Intersil", "PRISM 2_5 PCMCIA ADAPTER", 0x4b801a17, 0x6345a0bf),
-	PCMCIA_DEVICE_PROD_ID12("Linksys", "Wireless CompactFlash Card", 0x0733cc81, 0x0c52f395),
-	PCMCIA_DEVICE_PROD_ID12("Microsoft", "Wireless Notebook Adapter MN-520", 0x5961bf85, 0x6eec8c01),
-	PCMCIA_DEVICE_PROD_ID12("NETGEAR MA401RA Wireless PC", "Card", 0x0306467f, 0x9762e8f1),
-	PCMCIA_DEVICE_PROD_ID12("NETGEAR MA401 Wireless PC", "Card", 0xa37434e9, 0x9762e8f1),
-	PCMCIA_DEVICE_PROD_ID12("OEM", "PRISM2 IEEE 802.11 PC-Card", 0xfea54c90, 0x48f2bdd6),
-	PCMCIA_DEVICE_PROD_ID12("PLANEX", "GeoWave/GW-CF110", 0x209f40ab, 0xd9715264),
-	PCMCIA_DEVICE_PROD_ID12("PLANEX", "GeoWave/GW-NS110", 0x209f40ab, 0x46263178),
-	PCMCIA_DEVICE_PROD_ID12("SMC", "SMC2532W-B EliteConnect Wireless Adapter", 0xc4f8b18b, 0x196bd757),
-	PCMCIA_DEVICE_PROD_ID12("SMC", "SMC2632W", 0xc4f8b18b, 0x474a1f2a),
-	PCMCIA_DEVICE_PROD_ID12("ZoomAir 11Mbps High", "Rate wireless Networking", 0x273fe3db, 0x32a1eaee),
-	PCMCIA_DEVICE_PROD_ID3("HFA3863", 0x355cb092),
-	PCMCIA_DEVICE_PROD_ID3("ISL37100P", 0x630d52b2),
-	PCMCIA_DEVICE_PROD_ID3("ISL37101P-10", 0xdd97a26b),
-	PCMCIA_DEVICE_PROD_ID3("ISL37300P", 0xc9049a39),
-
-	/* This may be Agere or Intersil Firmware */
-	PCMCIA_DEVICE_MANF_CARD(0x0156, 0x0002),
-#endif
-	PCMCIA_DEVICE_NULL,
-};
-MODULE_DEVICE_TABLE(pcmcia, orinoco_cs_ids);
-
-static struct pcmcia_driver orinoco_driver = {
-	.owner		= THIS_MODULE,
-	.name		= DRIVER_NAME,
-	.probe		= orinoco_cs_probe,
-	.remove		= orinoco_cs_detach,
-	.id_table       = orinoco_cs_ids,
-	.suspend	= orinoco_cs_suspend,
-	.resume		= orinoco_cs_resume,
-};
-module_pcmcia_driver(orinoco_driver);
diff --git a/drivers/net/wireless/intersil/orinoco/orinoco_nortel.c b/drivers/net/wireless/intersil/orinoco/orinoco_nortel.c
deleted file mode 100644
index 18bd0d9876c2..000000000000
--- a/drivers/net/wireless/intersil/orinoco/orinoco_nortel.c
+++ /dev/null
@@ -1,314 +0,0 @@
-/* orinoco_nortel.c
- *
- * Driver for Prism II devices which would usually be driven by orinoco_cs,
- * but are connected to the PCI bus by a PCI-to-PCMCIA adapter used in
- * Nortel emobility, Symbol LA-4113 and Symbol LA-4123.
- *
- * Copyright (C) 2002 Tobias Hoffmann
- *           (C) 2003 Christoph Jungegger <disdos@traum404.de>
- *
- * Some of this code is borrowed from orinoco_plx.c
- *	Copyright (C) 2001 Daniel Barlow
- * Some of this code is borrowed from orinoco_pci.c
- *  Copyright (C) 2001 Jean Tourrilhes
- * Some of this code is "inspired" by linux-wlan-ng-0.1.10, but nothing
- * has been copied from it. linux-wlan-ng-0.1.10 is originally :
- *	Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
- *
- * The contents of this file are subject to the Mozilla Public License
- * Version 1.1 (the "License"); you may not use this file except in
- * compliance with the License. You may obtain a copy of the License
- * at http://www.mozilla.org/MPL/
- *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and
- * limitations under the License.
- *
- * Alternatively, the contents of this file may be used under the
- * terms of the GNU General Public License version 2 (the "GPL"), in
- * which case the provisions of the GPL are applicable instead of the
- * above.  If you wish to allow the use of your version of this file
- * only under the terms of the GPL and not to allow others to use your
- * version of this file under the MPL, indicate your decision by
- * deleting the provisions above and replace them with the notice and
- * other provisions required by the GPL.  If you do not delete the
- * provisions above, a recipient may use your version of this file
- * under either the MPL or the GPL.
- */
-
-#define DRIVER_NAME "orinoco_nortel"
-#define PFX DRIVER_NAME ": "
-
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/delay.h>
-#include <linux/pci.h>
-#include <pcmcia/cisreg.h>
-
-#include "orinoco.h"
-#include "orinoco_pci.h"
-
-#define COR_OFFSET    (0xe0)	/* COR attribute offset of Prism2 PC card */
-#define COR_VALUE     (COR_LEVEL_REQ | COR_FUNC_ENA)	/* Enable PC card with interrupt in level trigger */
-
-
-/*
- * Do a soft reset of the card using the Configuration Option Register
- * We need this to get going...
- * This is the part of the code that is strongly inspired from wlan-ng
- *
- * Note bis : Don't try to access HERMES_CMD during the reset phase.
- * It just won't work !
- */
-static int orinoco_nortel_cor_reset(struct orinoco_private *priv)
-{
-	struct orinoco_pci_card *card = priv->card;
-
-	/* Assert the reset until the card notices */
-	iowrite16(8, card->bridge_io + 2);
-	ioread16(card->attr_io + COR_OFFSET);
-	iowrite16(0x80, card->attr_io + COR_OFFSET);
-	mdelay(1);
-
-	/* Give time for the card to recover from this hard effort */
-	iowrite16(0, card->attr_io + COR_OFFSET);
-	iowrite16(0, card->attr_io + COR_OFFSET);
-	mdelay(1);
-
-	/* Set COR as usual */
-	iowrite16(COR_VALUE, card->attr_io + COR_OFFSET);
-	iowrite16(COR_VALUE, card->attr_io + COR_OFFSET);
-	mdelay(1);
-
-	iowrite16(0x228, card->bridge_io + 2);
-
-	return 0;
-}
-
-static int orinoco_nortel_hw_init(struct orinoco_pci_card *card)
-{
-	int i;
-	u32 reg;
-
-	/* Setup bridge */
-	if (ioread16(card->bridge_io) & 1) {
-		printk(KERN_ERR PFX "brg1 answer1 wrong\n");
-		return -EBUSY;
-	}
-	iowrite16(0x118, card->bridge_io + 2);
-	iowrite16(0x108, card->bridge_io + 2);
-	mdelay(30);
-	iowrite16(0x8, card->bridge_io + 2);
-	for (i = 0; i < 30; i++) {
-		mdelay(30);
-		if (ioread16(card->bridge_io) & 0x10)
-			break;
-	}
-	if (i == 30) {
-		printk(KERN_ERR PFX "brg1 timed out\n");
-		return -EBUSY;
-	}
-	if (ioread16(card->attr_io + COR_OFFSET) & 1) {
-		printk(KERN_ERR PFX "brg2 answer1 wrong\n");
-		return -EBUSY;
-	}
-	if (ioread16(card->attr_io + COR_OFFSET + 2) & 1) {
-		printk(KERN_ERR PFX "brg2 answer2 wrong\n");
-		return -EBUSY;
-	}
-	if (ioread16(card->attr_io + COR_OFFSET + 4) & 1) {
-		printk(KERN_ERR PFX "brg2 answer3 wrong\n");
-		return -EBUSY;
-	}
-
-	/* Set the PCMCIA COR register */
-	iowrite16(COR_VALUE, card->attr_io + COR_OFFSET);
-	mdelay(1);
-	reg = ioread16(card->attr_io + COR_OFFSET);
-	if (reg != COR_VALUE) {
-		printk(KERN_ERR PFX "Error setting COR value (reg=%x)\n",
-		       reg);
-		return -EBUSY;
-	}
-
-	/* Set LEDs */
-	iowrite16(1, card->bridge_io + 10);
-	return 0;
-}
-
-static int orinoco_nortel_init_one(struct pci_dev *pdev,
-				   const struct pci_device_id *ent)
-{
-	int err;
-	struct orinoco_private *priv;
-	struct orinoco_pci_card *card;
-	void __iomem *hermes_io, *bridge_io, *attr_io;
-
-	err = pci_enable_device(pdev);
-	if (err) {
-		printk(KERN_ERR PFX "Cannot enable PCI device\n");
-		return err;
-	}
-
-	err = pci_request_regions(pdev, DRIVER_NAME);
-	if (err) {
-		printk(KERN_ERR PFX "Cannot obtain PCI resources\n");
-		goto fail_resources;
-	}
-
-	bridge_io = pci_iomap(pdev, 0, 0);
-	if (!bridge_io) {
-		printk(KERN_ERR PFX "Cannot map bridge registers\n");
-		err = -EIO;
-		goto fail_map_bridge;
-	}
-
-	attr_io = pci_iomap(pdev, 1, 0);
-	if (!attr_io) {
-		printk(KERN_ERR PFX "Cannot map PCMCIA attributes\n");
-		err = -EIO;
-		goto fail_map_attr;
-	}
-
-	hermes_io = pci_iomap(pdev, 2, 0);
-	if (!hermes_io) {
-		printk(KERN_ERR PFX "Cannot map chipset registers\n");
-		err = -EIO;
-		goto fail_map_hermes;
-	}
-
-	/* Allocate network device */
-	priv = alloc_orinocodev(sizeof(*card), &pdev->dev,
-				orinoco_nortel_cor_reset, NULL);
-	if (!priv) {
-		printk(KERN_ERR PFX "Cannot allocate network device\n");
-		err = -ENOMEM;
-		goto fail_alloc;
-	}
-
-	card = priv->card;
-	card->bridge_io = bridge_io;
-	card->attr_io = attr_io;
-
-	hermes_struct_init(&priv->hw, hermes_io, HERMES_16BIT_REGSPACING);
-
-	err = request_irq(pdev->irq, orinoco_interrupt, IRQF_SHARED,
-			  DRIVER_NAME, priv);
-	if (err) {
-		printk(KERN_ERR PFX "Cannot allocate IRQ %d\n", pdev->irq);
-		err = -EBUSY;
-		goto fail_irq;
-	}
-
-	err = orinoco_nortel_hw_init(card);
-	if (err) {
-		printk(KERN_ERR PFX "Hardware initialization failed\n");
-		goto fail;
-	}
-
-	err = orinoco_nortel_cor_reset(priv);
-	if (err) {
-		printk(KERN_ERR PFX "Initial reset failed\n");
-		goto fail;
-	}
-
-	err = orinoco_init(priv);
-	if (err) {
-		printk(KERN_ERR PFX "orinoco_init() failed\n");
-		goto fail;
-	}
-
-	err = orinoco_if_add(priv, 0, 0, NULL);
-	if (err) {
-		printk(KERN_ERR PFX "orinoco_if_add() failed\n");
-		goto fail_wiphy;
-	}
-
-	pci_set_drvdata(pdev, priv);
-
-	return 0;
-
- fail_wiphy:
-	wiphy_unregister(priv_to_wiphy(priv));
- fail:
-	free_irq(pdev->irq, priv);
-
- fail_irq:
-	free_orinocodev(priv);
-
- fail_alloc:
-	pci_iounmap(pdev, hermes_io);
-
- fail_map_hermes:
-	pci_iounmap(pdev, attr_io);
-
- fail_map_attr:
-	pci_iounmap(pdev, bridge_io);
-
- fail_map_bridge:
-	pci_release_regions(pdev);
-
- fail_resources:
-	pci_disable_device(pdev);
-
-	return err;
-}
-
-static void orinoco_nortel_remove_one(struct pci_dev *pdev)
-{
-	struct orinoco_private *priv = pci_get_drvdata(pdev);
-	struct orinoco_pci_card *card = priv->card;
-
-	/* Clear LEDs */
-	iowrite16(0, card->bridge_io + 10);
-
-	orinoco_if_del(priv);
-	wiphy_unregister(priv_to_wiphy(priv));
-	free_irq(pdev->irq, priv);
-	free_orinocodev(priv);
-	pci_iounmap(pdev, priv->hw.iobase);
-	pci_iounmap(pdev, card->attr_io);
-	pci_iounmap(pdev, card->bridge_io);
-	pci_release_regions(pdev);
-	pci_disable_device(pdev);
-}
-
-static const struct pci_device_id orinoco_nortel_id_table[] = {
-	/* Nortel emobility PCI */
-	{0x126c, 0x8030, PCI_ANY_ID, PCI_ANY_ID,},
-	/* Symbol LA-4123 PCI */
-	{0x1562, 0x0001, PCI_ANY_ID, PCI_ANY_ID,},
-	{0,},
-};
-
-MODULE_DEVICE_TABLE(pci, orinoco_nortel_id_table);
-
-static struct pci_driver orinoco_nortel_driver = {
-	.name		= DRIVER_NAME,
-	.id_table	= orinoco_nortel_id_table,
-	.probe		= orinoco_nortel_init_one,
-	.remove		= orinoco_nortel_remove_one,
-	.driver.pm	= &orinoco_pci_pm_ops,
-};
-
-static char version[] __initdata = DRIVER_NAME " " DRIVER_VERSION
-	" (Tobias Hoffmann & Christoph Jungegger <disdos@traum404.de>)";
-MODULE_AUTHOR("Christoph Jungegger <disdos@traum404.de>");
-MODULE_DESCRIPTION("Driver for wireless LAN cards using the Nortel PCI bridge");
-MODULE_LICENSE("Dual MPL/GPL");
-
-static int __init orinoco_nortel_init(void)
-{
-	printk(KERN_DEBUG "%s\n", version);
-	return pci_register_driver(&orinoco_nortel_driver);
-}
-
-static void __exit orinoco_nortel_exit(void)
-{
-	pci_unregister_driver(&orinoco_nortel_driver);
-}
-
-module_init(orinoco_nortel_init);
-module_exit(orinoco_nortel_exit);
diff --git a/drivers/net/wireless/intersil/orinoco/orinoco_pci.c b/drivers/net/wireless/intersil/orinoco/orinoco_pci.c
deleted file mode 100644
index 7e3a6dd60c15..000000000000
--- a/drivers/net/wireless/intersil/orinoco/orinoco_pci.c
+++ /dev/null
@@ -1,257 +0,0 @@
-/* orinoco_pci.c
- *
- * Driver for Prism 2.5/3 devices that have a direct PCI interface
- * (i.e. these are not PCMCIA cards in a PCMCIA-to-PCI bridge).
- * The card contains only one PCI region, which contains all the usual
- * hermes registers, as well as the COR register.
- *
- * Current maintainers are:
- *	Pavel Roskin <proski AT gnu.org>
- * and	David Gibson <hermes AT gibson.dropbear.id.au>
- *
- * Some of this code is borrowed from orinoco_plx.c
- *	Copyright (C) 2001 Daniel Barlow <dan AT telent.net>
- * Some of this code is "inspired" by linux-wlan-ng-0.1.10, but nothing
- * has been copied from it. linux-wlan-ng-0.1.10 is originally :
- *	Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
- * This file originally written by:
- *	Copyright (C) 2001 Jean Tourrilhes <jt AT hpl.hp.com>
- * And is now maintained by:
- *	(C) Copyright David Gibson, IBM Corp. 2002-2003.
- *
- * The contents of this file are subject to the Mozilla Public License
- * Version 1.1 (the "License"); you may not use this file except in
- * compliance with the License. You may obtain a copy of the License
- * at http://www.mozilla.org/MPL/
- *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and
- * limitations under the License.
- *
- * Alternatively, the contents of this file may be used under the
- * terms of the GNU General Public License version 2 (the "GPL"), in
- * which case the provisions of the GPL are applicable instead of the
- * above.  If you wish to allow the use of your version of this file
- * only under the terms of the GPL and not to allow others to use your
- * version of this file under the MPL, indicate your decision by
- * deleting the provisions above and replace them with the notice and
- * other provisions required by the GPL.  If you do not delete the
- * provisions above, a recipient may use your version of this file
- * under either the MPL or the GPL.
- */
-
-#define DRIVER_NAME "orinoco_pci"
-#define PFX DRIVER_NAME ": "
-
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/delay.h>
-#include <linux/pci.h>
-
-#include "orinoco.h"
-#include "orinoco_pci.h"
-
-/* Offset of the COR register of the PCI card */
-#define HERMES_PCI_COR		(0x26)
-
-/* Bitmask to reset the card */
-#define HERMES_PCI_COR_MASK	(0x0080)
-
-/* Magic timeouts for doing the reset.
- * Those times are straight from wlan-ng, and it is claimed that they
- * are necessary. Alan will kill me. Take your time and grab a coffee. */
-#define HERMES_PCI_COR_ONT	(250)		/* ms */
-#define HERMES_PCI_COR_OFFT	(500)		/* ms */
-#define HERMES_PCI_COR_BUSYT	(500)		/* ms */
-
-/*
- * Do a soft reset of the card using the Configuration Option Register
- * We need this to get going...
- * This is the part of the code that is strongly inspired from wlan-ng
- *
- * Note : This code is done with irq enabled. This mean that many
- * interrupts will occur while we are there. This is why we use the
- * jiffies to regulate time instead of a straight mdelay(). Usually we
- * need only around 245 iteration of the loop to do 250 ms delay.
- *
- * Note bis : Don't try to access HERMES_CMD during the reset phase.
- * It just won't work !
- */
-static int orinoco_pci_cor_reset(struct orinoco_private *priv)
-{
-	struct hermes *hw = &priv->hw;
-	unsigned long timeout;
-	u16 reg;
-
-	/* Assert the reset until the card notices */
-	hermes_write_regn(hw, PCI_COR, HERMES_PCI_COR_MASK);
-	mdelay(HERMES_PCI_COR_ONT);
-
-	/* Give time for the card to recover from this hard effort */
-	hermes_write_regn(hw, PCI_COR, 0x0000);
-	mdelay(HERMES_PCI_COR_OFFT);
-
-	/* The card is ready when it's no longer busy */
-	timeout = jiffies + msecs_to_jiffies(HERMES_PCI_COR_BUSYT);
-	reg = hermes_read_regn(hw, CMD);
-	while (time_before(jiffies, timeout) && (reg & HERMES_CMD_BUSY)) {
-		mdelay(1);
-		reg = hermes_read_regn(hw, CMD);
-	}
-
-	/* Still busy? */
-	if (reg & HERMES_CMD_BUSY) {
-		printk(KERN_ERR PFX "Busy timeout\n");
-		return -ETIMEDOUT;
-	}
-
-	return 0;
-}
-
-static int orinoco_pci_init_one(struct pci_dev *pdev,
-				const struct pci_device_id *ent)
-{
-	int err;
-	struct orinoco_private *priv;
-	struct orinoco_pci_card *card;
-	void __iomem *hermes_io;
-
-	err = pci_enable_device(pdev);
-	if (err) {
-		printk(KERN_ERR PFX "Cannot enable PCI device\n");
-		return err;
-	}
-
-	err = pci_request_regions(pdev, DRIVER_NAME);
-	if (err) {
-		printk(KERN_ERR PFX "Cannot obtain PCI resources\n");
-		goto fail_resources;
-	}
-
-	hermes_io = pci_iomap(pdev, 0, 0);
-	if (!hermes_io) {
-		printk(KERN_ERR PFX "Cannot remap chipset registers\n");
-		err = -EIO;
-		goto fail_map_hermes;
-	}
-
-	/* Allocate network device */
-	priv = alloc_orinocodev(sizeof(*card), &pdev->dev,
-				orinoco_pci_cor_reset, NULL);
-	if (!priv) {
-		printk(KERN_ERR PFX "Cannot allocate network device\n");
-		err = -ENOMEM;
-		goto fail_alloc;
-	}
-
-	card = priv->card;
-
-	hermes_struct_init(&priv->hw, hermes_io, HERMES_32BIT_REGSPACING);
-
-	err = request_irq(pdev->irq, orinoco_interrupt, IRQF_SHARED,
-			  DRIVER_NAME, priv);
-	if (err) {
-		printk(KERN_ERR PFX "Cannot allocate IRQ %d\n", pdev->irq);
-		err = -EBUSY;
-		goto fail_irq;
-	}
-
-	err = orinoco_pci_cor_reset(priv);
-	if (err) {
-		printk(KERN_ERR PFX "Initial reset failed\n");
-		goto fail;
-	}
-
-	err = orinoco_init(priv);
-	if (err) {
-		printk(KERN_ERR PFX "orinoco_init() failed\n");
-		goto fail;
-	}
-
-	err = orinoco_if_add(priv, 0, 0, NULL);
-	if (err) {
-		printk(KERN_ERR PFX "orinoco_if_add() failed\n");
-		goto fail_wiphy;
-	}
-
-	pci_set_drvdata(pdev, priv);
-
-	return 0;
-
- fail_wiphy:
-	wiphy_unregister(priv_to_wiphy(priv));
- fail:
-	free_irq(pdev->irq, priv);
-
- fail_irq:
-	free_orinocodev(priv);
-
- fail_alloc:
-	pci_iounmap(pdev, hermes_io);
-
- fail_map_hermes:
-	pci_release_regions(pdev);
-
- fail_resources:
-	pci_disable_device(pdev);
-
-	return err;
-}
-
-static void orinoco_pci_remove_one(struct pci_dev *pdev)
-{
-	struct orinoco_private *priv = pci_get_drvdata(pdev);
-
-	orinoco_if_del(priv);
-	wiphy_unregister(priv_to_wiphy(priv));
-	free_irq(pdev->irq, priv);
-	free_orinocodev(priv);
-	pci_iounmap(pdev, priv->hw.iobase);
-	pci_release_regions(pdev);
-	pci_disable_device(pdev);
-}
-
-static const struct pci_device_id orinoco_pci_id_table[] = {
-	/* Intersil Prism 3 */
-	{0x1260, 0x3872, PCI_ANY_ID, PCI_ANY_ID,},
-	/* Intersil Prism 2.5 */
-	{0x1260, 0x3873, PCI_ANY_ID, PCI_ANY_ID,},
-	/* Samsung MagicLAN SWL-2210P */
-	{0x167d, 0xa000, PCI_ANY_ID, PCI_ANY_ID,},
-	{0,},
-};
-
-MODULE_DEVICE_TABLE(pci, orinoco_pci_id_table);
-
-static struct pci_driver orinoco_pci_driver = {
-	.name		= DRIVER_NAME,
-	.id_table	= orinoco_pci_id_table,
-	.probe		= orinoco_pci_init_one,
-	.remove		= orinoco_pci_remove_one,
-	.driver.pm	= &orinoco_pci_pm_ops,
-};
-
-static char version[] __initdata = DRIVER_NAME " " DRIVER_VERSION
-	" (Pavel Roskin <proski@gnu.org>,"
-	" David Gibson <hermes@gibson.dropbear.id.au> &"
-	" Jean Tourrilhes <jt@hpl.hp.com>)";
-MODULE_AUTHOR("Pavel Roskin <proski@gnu.org> &"
-	      " David Gibson <hermes@gibson.dropbear.id.au>");
-MODULE_DESCRIPTION("Driver for wireless LAN cards using direct PCI interface");
-MODULE_LICENSE("Dual MPL/GPL");
-
-static int __init orinoco_pci_init(void)
-{
-	printk(KERN_DEBUG "%s\n", version);
-	return pci_register_driver(&orinoco_pci_driver);
-}
-
-static void __exit orinoco_pci_exit(void)
-{
-	pci_unregister_driver(&orinoco_pci_driver);
-}
-
-module_init(orinoco_pci_init);
-module_exit(orinoco_pci_exit);
diff --git a/drivers/net/wireless/intersil/orinoco/orinoco_pci.h b/drivers/net/wireless/intersil/orinoco/orinoco_pci.h
deleted file mode 100644
index d49d940864b4..000000000000
--- a/drivers/net/wireless/intersil/orinoco/orinoco_pci.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/* orinoco_pci.h
- *
- * Common code for all Orinoco drivers for PCI devices, including
- * both native PCI and PCMCIA-to-PCI bridges.
- *
- * Copyright (C) 2005, Pavel Roskin.
- * See main.c for license.
- */
-
-#ifndef _ORINOCO_PCI_H
-#define _ORINOCO_PCI_H
-
-#include <linux/netdevice.h>
-
-/* Driver specific data */
-struct orinoco_pci_card {
-	void __iomem *bridge_io;
-	void __iomem *attr_io;
-};
-
-static int __maybe_unused orinoco_pci_suspend(struct device *dev_d)
-{
-	struct pci_dev *pdev = to_pci_dev(dev_d);
-	struct orinoco_private *priv = pci_get_drvdata(pdev);
-
-	orinoco_down(priv);
-	free_irq(pdev->irq, priv);
-
-	return 0;
-}
-
-static int __maybe_unused orinoco_pci_resume(struct device *dev_d)
-{
-	struct pci_dev *pdev = to_pci_dev(dev_d);
-	struct orinoco_private *priv = pci_get_drvdata(pdev);
-	struct net_device *dev = priv->ndev;
-	int err;
-
-	err = request_irq(pdev->irq, orinoco_interrupt, IRQF_SHARED,
-			  dev->name, priv);
-	if (err) {
-		printk(KERN_ERR "%s: cannot re-allocate IRQ on resume\n",
-		       dev->name);
-		return -EBUSY;
-	}
-
-	return orinoco_up(priv);
-}
-
-static SIMPLE_DEV_PM_OPS(orinoco_pci_pm_ops,
-			 orinoco_pci_suspend,
-			 orinoco_pci_resume);
-
-#endif /* _ORINOCO_PCI_H */
diff --git a/drivers/net/wireless/intersil/orinoco/orinoco_plx.c b/drivers/net/wireless/intersil/orinoco/orinoco_plx.c
deleted file mode 100644
index 73e6ae124013..000000000000
--- a/drivers/net/wireless/intersil/orinoco/orinoco_plx.c
+++ /dev/null
@@ -1,362 +0,0 @@
-/* orinoco_plx.c
- *
- * Driver for Prism II devices which would usually be driven by orinoco_cs,
- * but are connected to the PCI bus by a PLX9052.
- *
- * Current maintainers are:
- *	Pavel Roskin <proski AT gnu.org>
- * and	David Gibson <hermes AT gibson.dropbear.id.au>
- *
- * (C) Copyright David Gibson, IBM Corp. 2001-2003.
- * Copyright (C) 2001 Daniel Barlow
- *
- * The contents of this file are subject to the Mozilla Public License
- * Version 1.1 (the "License"); you may not use this file except in
- * compliance with the License. You may obtain a copy of the License
- * at http://www.mozilla.org/MPL/
- *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and
- * limitations under the License.
- *
- * Alternatively, the contents of this file may be used under the
- * terms of the GNU General Public License version 2 (the "GPL"), in
- * which case the provisions of the GPL are applicable instead of the
- * above.  If you wish to allow the use of your version of this file
- * only under the terms of the GPL and not to allow others to use your
- * version of this file under the MPL, indicate your decision by
- * deleting the provisions above and replace them with the notice and
- * other provisions required by the GPL.  If you do not delete the
- * provisions above, a recipient may use your version of this file
- * under either the MPL or the GPL.
- *
- * Here's the general details on how the PLX9052 adapter works:
- *
- * - Two PCI I/O address spaces, one 0x80 long which contains the
- * PLX9052 registers, and one that's 0x40 long mapped to the PCMCIA
- * slot I/O address space.
- *
- * - One PCI memory address space, mapped to the PCMCIA attribute space
- * (containing the CIS).
- *
- * Using the later, you can read through the CIS data to make sure the
- * card is compatible with the driver. Keep in mind that the PCMCIA
- * spec specifies the CIS as the lower 8 bits of each word read from
- * the CIS, so to read the bytes of the CIS, read every other byte
- * (0,2,4,...). Passing that test, you need to enable the I/O address
- * space on the PCMCIA card via the PCMCIA COR register. This is the
- * first byte following the CIS. In my case (which may not have any
- * relation to what's on the PRISM2 cards), COR was at offset 0x800
- * within the PCI memory space. Write 0x41 to the COR register to
- * enable I/O mode and to select level triggered interrupts. To
- * confirm you actually succeeded, read the COR register back and make
- * sure it actually got set to 0x41, in case you have an unexpected
- * card inserted.
- *
- * Following that, you can treat the second PCI I/O address space (the
- * one that's not 0x80 in length) as the PCMCIA I/O space.
- *
- * Note that in the Eumitcom's source for their drivers, they register
- * the interrupt as edge triggered when registering it with the
- * Windows kernel. I don't recall how to register edge triggered on
- * Linux (if it can be done at all). But in some experimentation, I
- * don't see much operational difference between using either
- * interrupt mode. Don't mess with the interrupt mode in the COR
- * register though, as the PLX9052 wants level triggers with the way
- * the serial EEPROM configures it on the WL11000.
- *
- * There's some other little quirks related to timing that I bumped
- * into, but I don't recall right now. Also, there's two variants of
- * the WL11000 I've seen, revision A1 and T2. These seem to differ
- * slightly in the timings configured in the wait-state generator in
- * the PLX9052. There have also been some comments from Eumitcom that
- * cards shouldn't be hot swapped, apparently due to risk of cooking
- * the PLX9052. I'm unsure why they believe this, as I can't see
- * anything in the design that would really cause a problem, except
- * for crashing drivers not written to expect it. And having developed
- * drivers for the WL11000, I'd say it's quite tricky to write code
- * that will successfully deal with a hot unplug. Very odd things
- * happen on the I/O side of things. But anyway, be warned. Despite
- * that, I've hot-swapped a number of times during debugging and
- * driver development for various reasons (stuck WAIT# line after the
- * radio card's firmware locks up).
- */
-
-#define DRIVER_NAME "orinoco_plx"
-#define PFX DRIVER_NAME ": "
-
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/delay.h>
-#include <linux/pci.h>
-#include <pcmcia/cisreg.h>
-
-#include "orinoco.h"
-#include "orinoco_pci.h"
-
-#define COR_OFFSET	(0x3e0)	/* COR attribute offset of Prism2 PC card */
-#define COR_VALUE	(COR_LEVEL_REQ | COR_FUNC_ENA) /* Enable PC card with interrupt in level trigger */
-#define COR_RESET     (0x80)	/* reset bit in the COR register */
-#define PLX_RESET_TIME	(500)	/* milliseconds */
-
-#define PLX_INTCSR		0x4c /* Interrupt Control & Status Register */
-#define PLX_INTCSR_INTEN	(1 << 6) /* Interrupt Enable bit */
-
-/*
- * Do a soft reset of the card using the Configuration Option Register
- */
-static int orinoco_plx_cor_reset(struct orinoco_private *priv)
-{
-	struct hermes *hw = &priv->hw;
-	struct orinoco_pci_card *card = priv->card;
-	unsigned long timeout;
-	u16 reg;
-
-	iowrite8(COR_VALUE | COR_RESET, card->attr_io + COR_OFFSET);
-	mdelay(1);
-
-	iowrite8(COR_VALUE, card->attr_io + COR_OFFSET);
-	mdelay(1);
-
-	/* Just in case, wait more until the card is no longer busy */
-	timeout = jiffies + msecs_to_jiffies(PLX_RESET_TIME);
-	reg = hermes_read_regn(hw, CMD);
-	while (time_before(jiffies, timeout) && (reg & HERMES_CMD_BUSY)) {
-		mdelay(1);
-		reg = hermes_read_regn(hw, CMD);
-	}
-
-	/* Still busy? */
-	if (reg & HERMES_CMD_BUSY) {
-		printk(KERN_ERR PFX "Busy timeout\n");
-		return -ETIMEDOUT;
-	}
-
-	return 0;
-}
-
-static int orinoco_plx_hw_init(struct orinoco_pci_card *card)
-{
-	int i;
-	u32 csr_reg;
-	static const u8 cis_magic[] = {
-		0x01, 0x03, 0x00, 0x00, 0xff, 0x17, 0x04, 0x67
-	};
-
-	printk(KERN_DEBUG PFX "CIS: ");
-	for (i = 0; i < 16; i++)
-		printk("%02X:", ioread8(card->attr_io + (i << 1)));
-	printk("\n");
-
-	/* Verify whether a supported PC card is present */
-	/* FIXME: we probably need to be smarted about this */
-	for (i = 0; i < sizeof(cis_magic); i++) {
-		if (cis_magic[i] != ioread8(card->attr_io + (i << 1))) {
-			printk(KERN_ERR PFX "The CIS value of Prism2 PC "
-			       "card is unexpected\n");
-			return -ENODEV;
-		}
-	}
-
-	/* bjoern: We need to tell the card to enable interrupts, in
-	   case the serial eprom didn't do this already.  See the
-	   PLX9052 data book, p8-1 and 8-24 for reference. */
-	csr_reg = ioread32(card->bridge_io + PLX_INTCSR);
-	if (!(csr_reg & PLX_INTCSR_INTEN)) {
-		csr_reg |= PLX_INTCSR_INTEN;
-		iowrite32(csr_reg, card->bridge_io + PLX_INTCSR);
-		csr_reg = ioread32(card->bridge_io + PLX_INTCSR);
-		if (!(csr_reg & PLX_INTCSR_INTEN)) {
-			printk(KERN_ERR PFX "Cannot enable interrupts\n");
-			return -EIO;
-		}
-	}
-
-	return 0;
-}
-
-static int orinoco_plx_init_one(struct pci_dev *pdev,
-				const struct pci_device_id *ent)
-{
-	int err;
-	struct orinoco_private *priv;
-	struct orinoco_pci_card *card;
-	void __iomem *hermes_io, *attr_io, *bridge_io;
-
-	err = pci_enable_device(pdev);
-	if (err) {
-		printk(KERN_ERR PFX "Cannot enable PCI device\n");
-		return err;
-	}
-
-	err = pci_request_regions(pdev, DRIVER_NAME);
-	if (err) {
-		printk(KERN_ERR PFX "Cannot obtain PCI resources\n");
-		goto fail_resources;
-	}
-
-	bridge_io = pci_iomap(pdev, 1, 0);
-	if (!bridge_io) {
-		printk(KERN_ERR PFX "Cannot map bridge registers\n");
-		err = -EIO;
-		goto fail_map_bridge;
-	}
-
-	attr_io = pci_iomap(pdev, 2, 0);
-	if (!attr_io) {
-		printk(KERN_ERR PFX "Cannot map PCMCIA attributes\n");
-		err = -EIO;
-		goto fail_map_attr;
-	}
-
-	hermes_io = pci_iomap(pdev, 3, 0);
-	if (!hermes_io) {
-		printk(KERN_ERR PFX "Cannot map chipset registers\n");
-		err = -EIO;
-		goto fail_map_hermes;
-	}
-
-	/* Allocate network device */
-	priv = alloc_orinocodev(sizeof(*card), &pdev->dev,
-				orinoco_plx_cor_reset, NULL);
-	if (!priv) {
-		printk(KERN_ERR PFX "Cannot allocate network device\n");
-		err = -ENOMEM;
-		goto fail_alloc;
-	}
-
-	card = priv->card;
-	card->bridge_io = bridge_io;
-	card->attr_io = attr_io;
-
-	hermes_struct_init(&priv->hw, hermes_io, HERMES_16BIT_REGSPACING);
-
-	err = request_irq(pdev->irq, orinoco_interrupt, IRQF_SHARED,
-			  DRIVER_NAME, priv);
-	if (err) {
-		printk(KERN_ERR PFX "Cannot allocate IRQ %d\n", pdev->irq);
-		err = -EBUSY;
-		goto fail_irq;
-	}
-
-	err = orinoco_plx_hw_init(card);
-	if (err) {
-		printk(KERN_ERR PFX "Hardware initialization failed\n");
-		goto fail;
-	}
-
-	err = orinoco_plx_cor_reset(priv);
-	if (err) {
-		printk(KERN_ERR PFX "Initial reset failed\n");
-		goto fail;
-	}
-
-	err = orinoco_init(priv);
-	if (err) {
-		printk(KERN_ERR PFX "orinoco_init() failed\n");
-		goto fail;
-	}
-
-	err = orinoco_if_add(priv, 0, 0, NULL);
-	if (err) {
-		printk(KERN_ERR PFX "orinoco_if_add() failed\n");
-		goto fail_wiphy;
-	}
-
-	pci_set_drvdata(pdev, priv);
-
-	return 0;
-
- fail_wiphy:
-	wiphy_unregister(priv_to_wiphy(priv));
- fail:
-	free_irq(pdev->irq, priv);
-
- fail_irq:
-	free_orinocodev(priv);
-
- fail_alloc:
-	pci_iounmap(pdev, hermes_io);
-
- fail_map_hermes:
-	pci_iounmap(pdev, attr_io);
-
- fail_map_attr:
-	pci_iounmap(pdev, bridge_io);
-
- fail_map_bridge:
-	pci_release_regions(pdev);
-
- fail_resources:
-	pci_disable_device(pdev);
-
-	return err;
-}
-
-static void orinoco_plx_remove_one(struct pci_dev *pdev)
-{
-	struct orinoco_private *priv = pci_get_drvdata(pdev);
-	struct orinoco_pci_card *card = priv->card;
-
-	orinoco_if_del(priv);
-	wiphy_unregister(priv_to_wiphy(priv));
-	free_irq(pdev->irq, priv);
-	free_orinocodev(priv);
-	pci_iounmap(pdev, priv->hw.iobase);
-	pci_iounmap(pdev, card->attr_io);
-	pci_iounmap(pdev, card->bridge_io);
-	pci_release_regions(pdev);
-	pci_disable_device(pdev);
-}
-
-static const struct pci_device_id orinoco_plx_id_table[] = {
-	{0x111a, 0x1023, PCI_ANY_ID, PCI_ANY_ID,},	/* Siemens SpeedStream SS1023 */
-	{0x1385, 0x4100, PCI_ANY_ID, PCI_ANY_ID,},	/* Netgear MA301 */
-	{0x15e8, 0x0130, PCI_ANY_ID, PCI_ANY_ID,},	/* Correga  - does this work? */
-	{0x1638, 0x1100, PCI_ANY_ID, PCI_ANY_ID,},	/* SMC EZConnect SMC2602W,
-							   Eumitcom PCI WL11000,
-							   Addtron AWA-100 */
-	{0x16ab, 0x1100, PCI_ANY_ID, PCI_ANY_ID,},	/* Global Sun Tech GL24110P */
-	{0x16ab, 0x1101, PCI_ANY_ID, PCI_ANY_ID,},	/* Reported working, but unknown */
-	{0x16ab, 0x1102, PCI_ANY_ID, PCI_ANY_ID,},	/* Linksys WDT11 */
-	{0x16ec, 0x3685, PCI_ANY_ID, PCI_ANY_ID,},	/* USR 2415 */
-	{0xec80, 0xec00, PCI_ANY_ID, PCI_ANY_ID,},	/* Belkin F5D6000 tested by
-							   Brendan W. McAdams <rit AT jacked-in.org> */
-	{0x10b7, 0x7770, PCI_ANY_ID, PCI_ANY_ID,},	/* 3Com AirConnect PCI tested by
-							   Damien Persohn <damien AT persohn.net> */
-	{0,},
-};
-
-MODULE_DEVICE_TABLE(pci, orinoco_plx_id_table);
-
-static struct pci_driver orinoco_plx_driver = {
-	.name		= DRIVER_NAME,
-	.id_table	= orinoco_plx_id_table,
-	.probe		= orinoco_plx_init_one,
-	.remove		= orinoco_plx_remove_one,
-	.driver.pm	= &orinoco_pci_pm_ops,
-};
-
-static char version[] __initdata = DRIVER_NAME " " DRIVER_VERSION
-	" (Pavel Roskin <proski@gnu.org>,"
-	" David Gibson <hermes@gibson.dropbear.id.au>,"
-	" Daniel Barlow <dan@telent.net>)";
-MODULE_AUTHOR("Daniel Barlow <dan@telent.net>");
-MODULE_DESCRIPTION("Driver for wireless LAN cards using the PLX9052 PCI bridge");
-MODULE_LICENSE("Dual MPL/GPL");
-
-static int __init orinoco_plx_init(void)
-{
-	printk(KERN_DEBUG "%s\n", version);
-	return pci_register_driver(&orinoco_plx_driver);
-}
-
-static void __exit orinoco_plx_exit(void)
-{
-	pci_unregister_driver(&orinoco_plx_driver);
-}
-
-module_init(orinoco_plx_init);
-module_exit(orinoco_plx_exit);
diff --git a/drivers/net/wireless/intersil/orinoco/orinoco_tmd.c b/drivers/net/wireless/intersil/orinoco/orinoco_tmd.c
deleted file mode 100644
index 939d5a1dce97..000000000000
--- a/drivers/net/wireless/intersil/orinoco/orinoco_tmd.c
+++ /dev/null
@@ -1,237 +0,0 @@
-/* orinoco_tmd.c
- *
- * Driver for Prism II devices which would usually be driven by orinoco_cs,
- * but are connected to the PCI bus by a TMD7160.
- *
- * Copyright (C) 2003 Joerg Dorchain <joerg AT dorchain.net>
- * based heavily upon orinoco_plx.c Copyright (C) 2001 Daniel Barlow
- *
- * The contents of this file are subject to the Mozilla Public License
- * Version 1.1 (the "License"); you may not use this file except in
- * compliance with the License. You may obtain a copy of the License
- * at http://www.mozilla.org/MPL/
- *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and
- * limitations under the License.
- *
- * Alternatively, the contents of this file may be used under the
- * terms of the GNU General Public License version 2 (the "GPL"), in
- * which case the provisions of the GPL are applicable instead of the
- * above.  If you wish to allow the use of your version of this file
- * only under the terms of the GPL and not to allow others to use your
- * version of this file under the MPL, indicate your decision by
- * deleting the provisions above and replace them with the notice and
- * other provisions required by the GPL.  If you do not delete the
- * provisions above, a recipient may use your version of this file
- * under either the MPL or the GPL.
- *
- * The actual driving is done by main.c, this is just resource
- * allocation stuff.
- *
- * This driver is modeled after the orinoco_plx driver. The main
- * difference is that the TMD chip has only IO port ranges and doesn't
- * provide access to the PCMCIA attribute space.
- *
- * Pheecom sells cards with the TMD chip as "ASIC version"
- */
-
-#define DRIVER_NAME "orinoco_tmd"
-#define PFX DRIVER_NAME ": "
-
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/delay.h>
-#include <linux/pci.h>
-#include <pcmcia/cisreg.h>
-
-#include "orinoco.h"
-#include "orinoco_pci.h"
-
-#define COR_VALUE	(COR_LEVEL_REQ | COR_FUNC_ENA) /* Enable PC card with interrupt in level trigger */
-#define COR_RESET     (0x80)	/* reset bit in the COR register */
-#define TMD_RESET_TIME	(500)	/* milliseconds */
-
-/*
- * Do a soft reset of the card using the Configuration Option Register
- */
-static int orinoco_tmd_cor_reset(struct orinoco_private *priv)
-{
-	struct hermes *hw = &priv->hw;
-	struct orinoco_pci_card *card = priv->card;
-	unsigned long timeout;
-	u16 reg;
-
-	iowrite8(COR_VALUE | COR_RESET, card->bridge_io);
-	mdelay(1);
-
-	iowrite8(COR_VALUE, card->bridge_io);
-	mdelay(1);
-
-	/* Just in case, wait more until the card is no longer busy */
-	timeout = jiffies + msecs_to_jiffies(TMD_RESET_TIME);
-	reg = hermes_read_regn(hw, CMD);
-	while (time_before(jiffies, timeout) && (reg & HERMES_CMD_BUSY)) {
-		mdelay(1);
-		reg = hermes_read_regn(hw, CMD);
-	}
-
-	/* Still busy? */
-	if (reg & HERMES_CMD_BUSY) {
-		printk(KERN_ERR PFX "Busy timeout\n");
-		return -ETIMEDOUT;
-	}
-
-	return 0;
-}
-
-
-static int orinoco_tmd_init_one(struct pci_dev *pdev,
-				const struct pci_device_id *ent)
-{
-	int err;
-	struct orinoco_private *priv;
-	struct orinoco_pci_card *card;
-	void __iomem *hermes_io, *bridge_io;
-
-	err = pci_enable_device(pdev);
-	if (err) {
-		printk(KERN_ERR PFX "Cannot enable PCI device\n");
-		return err;
-	}
-
-	err = pci_request_regions(pdev, DRIVER_NAME);
-	if (err) {
-		printk(KERN_ERR PFX "Cannot obtain PCI resources\n");
-		goto fail_resources;
-	}
-
-	bridge_io = pci_iomap(pdev, 1, 0);
-	if (!bridge_io) {
-		printk(KERN_ERR PFX "Cannot map bridge registers\n");
-		err = -EIO;
-		goto fail_map_bridge;
-	}
-
-	hermes_io = pci_iomap(pdev, 2, 0);
-	if (!hermes_io) {
-		printk(KERN_ERR PFX "Cannot map chipset registers\n");
-		err = -EIO;
-		goto fail_map_hermes;
-	}
-
-	/* Allocate network device */
-	priv = alloc_orinocodev(sizeof(*card), &pdev->dev,
-				orinoco_tmd_cor_reset, NULL);
-	if (!priv) {
-		printk(KERN_ERR PFX "Cannot allocate network device\n");
-		err = -ENOMEM;
-		goto fail_alloc;
-	}
-
-	card = priv->card;
-	card->bridge_io = bridge_io;
-
-	hermes_struct_init(&priv->hw, hermes_io, HERMES_16BIT_REGSPACING);
-
-	err = request_irq(pdev->irq, orinoco_interrupt, IRQF_SHARED,
-			  DRIVER_NAME, priv);
-	if (err) {
-		printk(KERN_ERR PFX "Cannot allocate IRQ %d\n", pdev->irq);
-		err = -EBUSY;
-		goto fail_irq;
-	}
-
-	err = orinoco_tmd_cor_reset(priv);
-	if (err) {
-		printk(KERN_ERR PFX "Initial reset failed\n");
-		goto fail;
-	}
-
-	err = orinoco_init(priv);
-	if (err) {
-		printk(KERN_ERR PFX "orinoco_init() failed\n");
-		goto fail;
-	}
-
-	err = orinoco_if_add(priv, 0, 0, NULL);
-	if (err) {
-		printk(KERN_ERR PFX "orinoco_if_add() failed\n");
-		goto fail;
-	}
-
-	pci_set_drvdata(pdev, priv);
-
-	return 0;
-
- fail:
-	free_irq(pdev->irq, priv);
-
- fail_irq:
-	free_orinocodev(priv);
-
- fail_alloc:
-	pci_iounmap(pdev, hermes_io);
-
- fail_map_hermes:
-	pci_iounmap(pdev, bridge_io);
-
- fail_map_bridge:
-	pci_release_regions(pdev);
-
- fail_resources:
-	pci_disable_device(pdev);
-
-	return err;
-}
-
-static void orinoco_tmd_remove_one(struct pci_dev *pdev)
-{
-	struct orinoco_private *priv = pci_get_drvdata(pdev);
-	struct orinoco_pci_card *card = priv->card;
-
-	orinoco_if_del(priv);
-	free_irq(pdev->irq, priv);
-	free_orinocodev(priv);
-	pci_iounmap(pdev, priv->hw.iobase);
-	pci_iounmap(pdev, card->bridge_io);
-	pci_release_regions(pdev);
-	pci_disable_device(pdev);
-}
-
-static const struct pci_device_id orinoco_tmd_id_table[] = {
-	{0x15e8, 0x0131, PCI_ANY_ID, PCI_ANY_ID,},      /* NDC and OEMs, e.g. pheecom */
-	{0,},
-};
-
-MODULE_DEVICE_TABLE(pci, orinoco_tmd_id_table);
-
-static struct pci_driver orinoco_tmd_driver = {
-	.name		= DRIVER_NAME,
-	.id_table	= orinoco_tmd_id_table,
-	.probe		= orinoco_tmd_init_one,
-	.remove		= orinoco_tmd_remove_one,
-	.driver.pm	= &orinoco_pci_pm_ops,
-};
-
-static char version[] __initdata = DRIVER_NAME " " DRIVER_VERSION
-	" (Joerg Dorchain <joerg@dorchain.net>)";
-MODULE_AUTHOR("Joerg Dorchain <joerg@dorchain.net>");
-MODULE_DESCRIPTION("Driver for wireless LAN cards using the TMD7160 PCI bridge");
-MODULE_LICENSE("Dual MPL/GPL");
-
-static int __init orinoco_tmd_init(void)
-{
-	printk(KERN_DEBUG "%s\n", version);
-	return pci_register_driver(&orinoco_tmd_driver);
-}
-
-static void __exit orinoco_tmd_exit(void)
-{
-	pci_unregister_driver(&orinoco_tmd_driver);
-}
-
-module_init(orinoco_tmd_init);
-module_exit(orinoco_tmd_exit);
diff --git a/drivers/net/wireless/intersil/orinoco/orinoco_usb.c b/drivers/net/wireless/intersil/orinoco/orinoco_usb.c
deleted file mode 100644
index 866e0230df25..000000000000
--- a/drivers/net/wireless/intersil/orinoco/orinoco_usb.c
+++ /dev/null
@@ -1,1787 +0,0 @@
-/*
- * USB Orinoco driver
- *
- * Copyright (c) 2003 Manuel Estrada Sainz
- *
- * The contents of this file are subject to the Mozilla Public License
- * Version 1.1 (the "License"); you may not use this file except in
- * compliance with the License. You may obtain a copy of the License
- * at http://www.mozilla.org/MPL/
- *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and
- * limitations under the License.
- *
- * Alternatively, the contents of this file may be used under the
- * terms of the GNU General Public License version 2 (the "GPL"), in
- * which case the provisions of the GPL are applicable instead of the
- * above.  If you wish to allow the use of your version of this file
- * only under the terms of the GPL and not to allow others to use your
- * version of this file under the MPL, indicate your decision by
- * deleting the provisions above and replace them with the notice and
- * other provisions required by the GPL.  If you do not delete the
- * provisions above, a recipient may use your version of this file
- * under either the MPL or the GPL.
- *
- * Queueing code based on linux-wlan-ng 0.2.1-pre5
- *
- * Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
- *
- *	The license is the same as above.
- *
- * Initialy based on USB Skeleton driver - 0.7
- *
- * Copyright (c) 2001 Greg Kroah-Hartman (greg@kroah.com)
- *
- *	This program is free software; you can redistribute it and/or
- *	modify it under the terms of the GNU General Public License as
- *	published by the Free Software Foundation; either version 2 of
- *	the License, or (at your option) any later version.
- *
- * NOTE: The original USB Skeleton driver is GPL, but all that code is
- * gone so MPL/GPL applies.
- */
-
-#define DRIVER_NAME "orinoco_usb"
-#define PFX DRIVER_NAME ": "
-
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/sched.h>
-#include <linux/signal.h>
-#include <linux/errno.h>
-#include <linux/poll.h>
-#include <linux/slab.h>
-#include <linux/fcntl.h>
-#include <linux/spinlock.h>
-#include <linux/list.h>
-#include <linux/usb.h>
-#include <linux/timer.h>
-
-#include <linux/netdevice.h>
-#include <linux/if_arp.h>
-#include <linux/etherdevice.h>
-#include <linux/wireless.h>
-#include <linux/firmware.h>
-#include <linux/refcount.h>
-
-#include "mic.h"
-#include "orinoco.h"
-
-#ifndef URB_ASYNC_UNLINK
-#define URB_ASYNC_UNLINK 0
-#endif
-
-struct header_struct {
-	/* 802.3 */
-	u8 dest[ETH_ALEN];
-	u8 src[ETH_ALEN];
-	__be16 len;
-	/* 802.2 */
-	u8 dsap;
-	u8 ssap;
-	u8 ctrl;
-	/* SNAP */
-	u8 oui[3];
-	__be16 ethertype;
-} __packed;
-
-struct ez_usb_fw {
-	u16 size;
-	const u8 *code;
-};
-
-static struct ez_usb_fw firmware = {
-	.size = 0,
-	.code = NULL,
-};
-
-/* Debugging macros */
-#undef err
-#define err(format, arg...) \
-	do { printk(KERN_ERR PFX format "\n", ## arg); } while (0)
-
-MODULE_FIRMWARE("orinoco_ezusb_fw");
-
-/*
- * Under some conditions, the card gets stuck and stops paying attention
- * to the world (i.e. data communication stalls) until we do something to
- * it.  Sending an INQ_TALLIES command seems to be enough and should be
- * harmless otherwise.  This behaviour has been observed when using the
- * driver on a systemimager client during installation.  In the past a
- * timer was used to send INQ_TALLIES commands when there was no other
- * activity, but it was troublesome and was removed.
- */
-
-#define USB_COMPAQ_VENDOR_ID     0x049f /* Compaq Computer Corp. */
-#define USB_COMPAQ_WL215_ID      0x001f /* Compaq WL215 USB Adapter */
-#define USB_COMPAQ_W200_ID       0x0076 /* Compaq W200 USB Adapter */
-#define USB_HP_WL215_ID          0x0082 /* Compaq WL215 USB Adapter */
-
-#define USB_MELCO_VENDOR_ID      0x0411
-#define USB_BUFFALO_L11_ID       0x0006 /* BUFFALO WLI-USB-L11 */
-#define USB_BUFFALO_L11G_WR_ID   0x000B /* BUFFALO WLI-USB-L11G-WR */
-#define USB_BUFFALO_L11G_ID      0x000D /* BUFFALO WLI-USB-L11G */
-
-#define USB_LUCENT_VENDOR_ID     0x047E /* Lucent Technologies */
-#define USB_LUCENT_ORINOCO_ID    0x0300 /* Lucent/Agere Orinoco USB Client */
-
-#define USB_AVAYA8_VENDOR_ID     0x0D98
-#define USB_AVAYAE_VENDOR_ID     0x0D9E
-#define USB_AVAYA_WIRELESS_ID    0x0300 /* Avaya USB Wireless Card */
-
-#define USB_AGERE_VENDOR_ID      0x0D4E /* Agere Systems */
-#define USB_AGERE_MODEL0801_ID   0x1000 /* USB Wireless Card Model 0801 */
-#define USB_AGERE_MODEL0802_ID   0x1001 /* USB Wireless Card Model 0802 */
-#define USB_AGERE_REBRANDED_ID   0x047A /* USB WLAN Card */
-
-#define USB_ELSA_VENDOR_ID       0x05CC
-#define USB_ELSA_AIRLANCER_ID    0x3100 /* ELSA AirLancer USB-11 */
-
-#define USB_LEGEND_VENDOR_ID     0x0E7C
-#define USB_LEGEND_JOYNET_ID     0x0300 /* Joynet USB WLAN Card */
-
-#define USB_SAMSUNG_VENDOR_ID    0x04E8
-#define USB_SAMSUNG_SEW2001U1_ID 0x5002 /* Samsung SEW-2001u Card */
-#define USB_SAMSUNG_SEW2001U2_ID 0x5B11 /* Samsung SEW-2001u Card */
-#define USB_SAMSUNG_SEW2003U_ID  0x7011 /* Samsung SEW-2003U Card */
-
-#define USB_IGATE_VENDOR_ID      0x0681
-#define USB_IGATE_IGATE_11M_ID   0x0012 /* I-GATE 11M USB Card */
-
-#define USB_FUJITSU_VENDOR_ID    0x0BF8
-#define USB_FUJITSU_E1100_ID     0x1002 /* connect2AIR WLAN E-1100 USB */
-
-#define USB_2WIRE_VENDOR_ID      0x1630
-#define USB_2WIRE_WIRELESS_ID    0xff81 /* 2Wire USB Wireless adapter */
-
-
-#define EZUSB_REQUEST_FW_TRANS		0xA0
-#define EZUSB_REQUEST_TRIGGER		0xAA
-#define EZUSB_REQUEST_TRIG_AC		0xAC
-#define EZUSB_CPUCS_REG			0x7F92
-
-#define EZUSB_RID_TX			0x0700
-#define EZUSB_RID_RX			0x0701
-#define EZUSB_RID_INIT1			0x0702
-#define EZUSB_RID_ACK			0x0710
-#define EZUSB_RID_READ_PDA		0x0800
-#define EZUSB_RID_PROG_INIT		0x0852
-#define EZUSB_RID_PROG_SET_ADDR		0x0853
-#define EZUSB_RID_PROG_BYTES		0x0854
-#define EZUSB_RID_PROG_END		0x0855
-#define EZUSB_RID_DOCMD			0x0860
-
-/* Recognize info frames */
-#define EZUSB_IS_INFO(id)		((id >= 0xF000) && (id <= 0xF2FF))
-
-#define EZUSB_MAGIC			0x0210
-
-#define EZUSB_FRAME_DATA		1
-#define EZUSB_FRAME_CONTROL		2
-
-#define DEF_TIMEOUT			(3 * HZ)
-
-#define BULK_BUF_SIZE			2048
-
-#define MAX_DL_SIZE (BULK_BUF_SIZE - sizeof(struct ezusb_packet))
-
-#define FW_BUF_SIZE			64
-#define FW_VAR_OFFSET_PTR		0x359
-#define FW_VAR_VALUE			0
-#define FW_HOLE_START			0x100
-#define FW_HOLE_END			0x300
-
-struct ezusb_packet {
-	__le16 magic;		/* 0x0210 */
-	u8 req_reply_count;
-	u8 ans_reply_count;
-	__le16 frame_type;	/* 0x01 for data frames, 0x02 otherwise */
-	__le16 size;		/* transport size */
-	__le16 crc;		/* CRC up to here */
-	__le16 hermes_len;
-	__le16 hermes_rid;
-	u8 data[];
-} __packed;
-
-/* Table of devices that work or may work with this driver */
-static const struct usb_device_id ezusb_table[] = {
-	{USB_DEVICE(USB_COMPAQ_VENDOR_ID, USB_COMPAQ_WL215_ID)},
-	{USB_DEVICE(USB_COMPAQ_VENDOR_ID, USB_HP_WL215_ID)},
-	{USB_DEVICE(USB_COMPAQ_VENDOR_ID, USB_COMPAQ_W200_ID)},
-	{USB_DEVICE(USB_MELCO_VENDOR_ID, USB_BUFFALO_L11_ID)},
-	{USB_DEVICE(USB_MELCO_VENDOR_ID, USB_BUFFALO_L11G_WR_ID)},
-	{USB_DEVICE(USB_MELCO_VENDOR_ID, USB_BUFFALO_L11G_ID)},
-	{USB_DEVICE(USB_LUCENT_VENDOR_ID, USB_LUCENT_ORINOCO_ID)},
-	{USB_DEVICE(USB_AVAYA8_VENDOR_ID, USB_AVAYA_WIRELESS_ID)},
-	{USB_DEVICE(USB_AVAYAE_VENDOR_ID, USB_AVAYA_WIRELESS_ID)},
-	{USB_DEVICE(USB_AGERE_VENDOR_ID, USB_AGERE_MODEL0801_ID)},
-	{USB_DEVICE(USB_AGERE_VENDOR_ID, USB_AGERE_MODEL0802_ID)},
-	{USB_DEVICE(USB_ELSA_VENDOR_ID, USB_ELSA_AIRLANCER_ID)},
-	{USB_DEVICE(USB_LEGEND_VENDOR_ID, USB_LEGEND_JOYNET_ID)},
-	{USB_DEVICE_VER(USB_SAMSUNG_VENDOR_ID, USB_SAMSUNG_SEW2001U1_ID,
-			0, 0)},
-	{USB_DEVICE(USB_SAMSUNG_VENDOR_ID, USB_SAMSUNG_SEW2001U2_ID)},
-	{USB_DEVICE(USB_SAMSUNG_VENDOR_ID, USB_SAMSUNG_SEW2003U_ID)},
-	{USB_DEVICE(USB_IGATE_VENDOR_ID, USB_IGATE_IGATE_11M_ID)},
-	{USB_DEVICE(USB_FUJITSU_VENDOR_ID, USB_FUJITSU_E1100_ID)},
-	{USB_DEVICE(USB_2WIRE_VENDOR_ID, USB_2WIRE_WIRELESS_ID)},
-	{USB_DEVICE(USB_AGERE_VENDOR_ID, USB_AGERE_REBRANDED_ID)},
-	{}			/* Terminating entry */
-};
-
-MODULE_DEVICE_TABLE(usb, ezusb_table);
-
-/* Structure to hold all of our device specific stuff */
-struct ezusb_priv {
-	struct usb_device *udev;
-	struct net_device *dev;
-	struct mutex mtx;
-	spinlock_t req_lock;
-	struct list_head req_pending;
-	struct list_head req_active;
-	spinlock_t reply_count_lock;
-	u16 hermes_reg_fake[0x40];
-	u8 *bap_buf;
-	struct urb *read_urb;
-	int read_pipe;
-	int write_pipe;
-	u8 reply_count;
-};
-
-enum ezusb_state {
-	EZUSB_CTX_START,
-	EZUSB_CTX_QUEUED,
-	EZUSB_CTX_REQ_SUBMITTED,
-	EZUSB_CTX_REQ_COMPLETE,
-	EZUSB_CTX_RESP_RECEIVED,
-	EZUSB_CTX_REQ_TIMEOUT,
-	EZUSB_CTX_REQ_FAILED,
-	EZUSB_CTX_RESP_TIMEOUT,
-	EZUSB_CTX_REQSUBMIT_FAIL,
-	EZUSB_CTX_COMPLETE,
-};
-
-struct request_context {
-	struct list_head list;
-	refcount_t refcount;
-	struct completion done;	/* Signals that CTX is dead */
-	int killed;
-	struct urb *outurb;	/* OUT for req pkt */
-	struct ezusb_priv *upriv;
-	struct ezusb_packet *buf;
-	int buf_length;
-	struct timer_list timer;	/* Timeout handling */
-	enum ezusb_state state;	/* Current state */
-	/* the RID that we will wait for */
-	u16 out_rid;
-	u16 in_rid;
-};
-
-
-/* Forward declarations */
-static void ezusb_ctx_complete(struct request_context *ctx);
-static void ezusb_req_queue_run(struct ezusb_priv *upriv);
-static void ezusb_bulk_in_callback(struct urb *urb);
-
-static inline u8 ezusb_reply_inc(u8 count)
-{
-	if (count < 0x7F)
-		return count + 1;
-	else
-		return 1;
-}
-
-static void ezusb_request_context_put(struct request_context *ctx)
-{
-	if (!refcount_dec_and_test(&ctx->refcount))
-		return;
-
-	WARN_ON(!ctx->done.done);
-	BUG_ON(ctx->outurb->status == -EINPROGRESS);
-	BUG_ON(timer_pending(&ctx->timer));
-	usb_free_urb(ctx->outurb);
-	kfree(ctx->buf);
-	kfree(ctx);
-}
-
-static inline void ezusb_mod_timer(struct ezusb_priv *upriv,
-				   struct timer_list *timer,
-				   unsigned long expire)
-{
-	if (!upriv->udev)
-		return;
-	mod_timer(timer, expire);
-}
-
-static void ezusb_request_timerfn(struct timer_list *t)
-{
-	struct request_context *ctx = from_timer(ctx, t, timer);
-
-	ctx->outurb->transfer_flags |= URB_ASYNC_UNLINK;
-	if (usb_unlink_urb(ctx->outurb) == -EINPROGRESS) {
-		ctx->state = EZUSB_CTX_REQ_TIMEOUT;
-	} else {
-		ctx->state = EZUSB_CTX_RESP_TIMEOUT;
-		dev_dbg(&ctx->outurb->dev->dev, "couldn't unlink\n");
-		refcount_inc(&ctx->refcount);
-		ctx->killed = 1;
-		ezusb_ctx_complete(ctx);
-		ezusb_request_context_put(ctx);
-	}
-};
-
-static struct request_context *ezusb_alloc_ctx(struct ezusb_priv *upriv,
-					       u16 out_rid, u16 in_rid)
-{
-	struct request_context *ctx;
-
-	ctx = kzalloc(sizeof(*ctx), GFP_ATOMIC);
-	if (!ctx)
-		return NULL;
-
-	ctx->buf = kmalloc(BULK_BUF_SIZE, GFP_ATOMIC);
-	if (!ctx->buf) {
-		kfree(ctx);
-		return NULL;
-	}
-	ctx->outurb = usb_alloc_urb(0, GFP_ATOMIC);
-	if (!ctx->outurb) {
-		kfree(ctx->buf);
-		kfree(ctx);
-		return NULL;
-	}
-
-	ctx->upriv = upriv;
-	ctx->state = EZUSB_CTX_START;
-	ctx->out_rid = out_rid;
-	ctx->in_rid = in_rid;
-
-	refcount_set(&ctx->refcount, 1);
-	init_completion(&ctx->done);
-
-	timer_setup(&ctx->timer, ezusb_request_timerfn, 0);
-	return ctx;
-}
-
-static void ezusb_ctx_complete(struct request_context *ctx)
-{
-	struct ezusb_priv *upriv = ctx->upriv;
-	unsigned long flags;
-
-	spin_lock_irqsave(&upriv->req_lock, flags);
-
-	list_del_init(&ctx->list);
-	if (upriv->udev) {
-		spin_unlock_irqrestore(&upriv->req_lock, flags);
-		ezusb_req_queue_run(upriv);
-		spin_lock_irqsave(&upriv->req_lock, flags);
-	}
-
-	switch (ctx->state) {
-	case EZUSB_CTX_COMPLETE:
-	case EZUSB_CTX_REQSUBMIT_FAIL:
-	case EZUSB_CTX_REQ_FAILED:
-	case EZUSB_CTX_REQ_TIMEOUT:
-	case EZUSB_CTX_RESP_TIMEOUT:
-		spin_unlock_irqrestore(&upriv->req_lock, flags);
-
-		if ((ctx->out_rid == EZUSB_RID_TX) && upriv->dev) {
-			struct net_device *dev = upriv->dev;
-			struct net_device_stats *stats = &dev->stats;
-
-			if (ctx->state != EZUSB_CTX_COMPLETE)
-				stats->tx_errors++;
-			else
-				stats->tx_packets++;
-
-			netif_wake_queue(dev);
-		}
-		complete_all(&ctx->done);
-		ezusb_request_context_put(ctx);
-		break;
-
-	default:
-		spin_unlock_irqrestore(&upriv->req_lock, flags);
-		if (!upriv->udev) {
-			/* This is normal, as all request contexts get flushed
-			 * when the device is disconnected */
-			err("Called, CTX not terminating, but device gone");
-			complete_all(&ctx->done);
-			ezusb_request_context_put(ctx);
-			break;
-		}
-
-		err("Called, CTX not in terminating state.");
-		/* Things are really bad if this happens. Just leak
-		 * the CTX because it may still be linked to the
-		 * queue or the OUT urb may still be active.
-		 * Just leaking at least prevents an Oops or Panic.
-		 */
-		break;
-	}
-}
-
-/*
- * ezusb_req_queue_run:
- * Description:
- *	Note: Only one active CTX at any one time, because there's no
- *	other (reliable) way to match the response URB to the correct
- *	CTX.
- */
-static void ezusb_req_queue_run(struct ezusb_priv *upriv)
-{
-	unsigned long flags;
-	struct request_context *ctx;
-	int result;
-
-	spin_lock_irqsave(&upriv->req_lock, flags);
-
-	if (!list_empty(&upriv->req_active))
-		goto unlock;
-
-	if (list_empty(&upriv->req_pending))
-		goto unlock;
-
-	ctx =
-	    list_entry(upriv->req_pending.next, struct request_context,
-		       list);
-
-	if (!ctx->upriv->udev)
-		goto unlock;
-
-	/* We need to split this off to avoid a race condition */
-	list_move_tail(&ctx->list, &upriv->req_active);
-
-	if (ctx->state == EZUSB_CTX_QUEUED) {
-		refcount_inc(&ctx->refcount);
-		result = usb_submit_urb(ctx->outurb, GFP_ATOMIC);
-		if (result) {
-			ctx->state = EZUSB_CTX_REQSUBMIT_FAIL;
-
-			spin_unlock_irqrestore(&upriv->req_lock, flags);
-
-			err("Fatal, failed to submit command urb."
-			    " error=%d\n", result);
-
-			ezusb_ctx_complete(ctx);
-			ezusb_request_context_put(ctx);
-			goto done;
-		}
-
-		ctx->state = EZUSB_CTX_REQ_SUBMITTED;
-		ezusb_mod_timer(ctx->upriv, &ctx->timer,
-				jiffies + DEF_TIMEOUT);
-	}
-
- unlock:
-	spin_unlock_irqrestore(&upriv->req_lock, flags);
-
- done:
-	return;
-}
-
-static void ezusb_req_enqueue_run(struct ezusb_priv *upriv,
-				  struct request_context *ctx)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&upriv->req_lock, flags);
-
-	if (!ctx->upriv->udev) {
-		spin_unlock_irqrestore(&upriv->req_lock, flags);
-		goto done;
-	}
-	refcount_inc(&ctx->refcount);
-	list_add_tail(&ctx->list, &upriv->req_pending);
-	spin_unlock_irqrestore(&upriv->req_lock, flags);
-
-	ctx->state = EZUSB_CTX_QUEUED;
-	ezusb_req_queue_run(upriv);
-
- done:
-	return;
-}
-
-static void ezusb_request_out_callback(struct urb *urb)
-{
-	unsigned long flags;
-	enum ezusb_state state;
-	struct request_context *ctx = urb->context;
-	struct ezusb_priv *upriv = ctx->upriv;
-
-	spin_lock_irqsave(&upriv->req_lock, flags);
-
-	del_timer(&ctx->timer);
-
-	if (ctx->killed) {
-		spin_unlock_irqrestore(&upriv->req_lock, flags);
-		pr_warn("interrupt called with dead ctx\n");
-		goto out;
-	}
-
-	state = ctx->state;
-
-	if (urb->status == 0) {
-		switch (state) {
-		case EZUSB_CTX_REQ_SUBMITTED:
-			if (ctx->in_rid) {
-				ctx->state = EZUSB_CTX_REQ_COMPLETE;
-				/* reply URB still pending */
-				ezusb_mod_timer(upriv, &ctx->timer,
-						jiffies + DEF_TIMEOUT);
-				spin_unlock_irqrestore(&upriv->req_lock,
-						       flags);
-				break;
-			}
-			fallthrough;
-		case EZUSB_CTX_RESP_RECEIVED:
-			/* IN already received before this OUT-ACK */
-			ctx->state = EZUSB_CTX_COMPLETE;
-			spin_unlock_irqrestore(&upriv->req_lock, flags);
-			ezusb_ctx_complete(ctx);
-			break;
-
-		default:
-			spin_unlock_irqrestore(&upriv->req_lock, flags);
-			err("Unexpected state(0x%x, %d) in OUT URB",
-			    state, urb->status);
-			break;
-		}
-	} else {
-		/* If someone cancels the OUT URB then its status
-		 * should be either -ECONNRESET or -ENOENT.
-		 */
-		switch (state) {
-		case EZUSB_CTX_REQ_SUBMITTED:
-		case EZUSB_CTX_RESP_RECEIVED:
-			ctx->state = EZUSB_CTX_REQ_FAILED;
-			fallthrough;
-
-		case EZUSB_CTX_REQ_FAILED:
-		case EZUSB_CTX_REQ_TIMEOUT:
-			spin_unlock_irqrestore(&upriv->req_lock, flags);
-
-			ezusb_ctx_complete(ctx);
-			break;
-
-		default:
-			spin_unlock_irqrestore(&upriv->req_lock, flags);
-
-			err("Unexpected state(0x%x, %d) in OUT URB",
-			    state, urb->status);
-			break;
-		}
-	}
- out:
-	ezusb_request_context_put(ctx);
-}
-
-static void ezusb_request_in_callback(struct ezusb_priv *upriv,
-				      struct urb *urb)
-{
-	struct ezusb_packet *ans = urb->transfer_buffer;
-	struct request_context *ctx = NULL;
-	enum ezusb_state state;
-	unsigned long flags;
-
-	/* Find the CTX on the active queue that requested this URB */
-	spin_lock_irqsave(&upriv->req_lock, flags);
-	if (upriv->udev) {
-		struct list_head *item;
-
-		list_for_each(item, &upriv->req_active) {
-			struct request_context *c;
-			int reply_count;
-
-			c = list_entry(item, struct request_context, list);
-			reply_count =
-			    ezusb_reply_inc(c->buf->req_reply_count);
-			if ((ans->ans_reply_count == reply_count)
-			    && (le16_to_cpu(ans->hermes_rid) == c->in_rid)) {
-				ctx = c;
-				break;
-			}
-			netdev_dbg(upriv->dev, "Skipped (0x%x/0x%x) (%d/%d)\n",
-				   le16_to_cpu(ans->hermes_rid), c->in_rid,
-				   ans->ans_reply_count, reply_count);
-		}
-	}
-
-	if (ctx == NULL) {
-		spin_unlock_irqrestore(&upriv->req_lock, flags);
-		err("%s: got unexpected RID: 0x%04X", __func__,
-		    le16_to_cpu(ans->hermes_rid));
-		ezusb_req_queue_run(upriv);
-		return;
-	}
-
-	/* The data we want is in the in buffer, exchange */
-	urb->transfer_buffer = ctx->buf;
-	ctx->buf = (void *) ans;
-	ctx->buf_length = urb->actual_length;
-
-	state = ctx->state;
-	switch (state) {
-	case EZUSB_CTX_REQ_SUBMITTED:
-		/* We have received our response URB before
-		 * our request has been acknowledged. Do NOT
-		 * destroy our CTX yet, because our OUT URB
-		 * is still alive ...
-		 */
-		ctx->state = EZUSB_CTX_RESP_RECEIVED;
-		spin_unlock_irqrestore(&upriv->req_lock, flags);
-
-		/* Let the machine continue running. */
-		break;
-
-	case EZUSB_CTX_REQ_COMPLETE:
-		/* This is the usual path: our request
-		 * has already been acknowledged, and
-		 * we have now received the reply.
-		 */
-		ctx->state = EZUSB_CTX_COMPLETE;
-
-		/* Stop the intimer */
-		del_timer(&ctx->timer);
-		spin_unlock_irqrestore(&upriv->req_lock, flags);
-
-		/* Call the completion handler */
-		ezusb_ctx_complete(ctx);
-		break;
-
-	default:
-		spin_unlock_irqrestore(&upriv->req_lock, flags);
-
-		pr_warn("Matched IN URB, unexpected context state(0x%x)\n",
-			state);
-		/* Throw this CTX away and try submitting another */
-		del_timer(&ctx->timer);
-		ctx->outurb->transfer_flags |= URB_ASYNC_UNLINK;
-		usb_unlink_urb(ctx->outurb);
-		ezusb_req_queue_run(upriv);
-		break;
-	}			/* switch */
-}
-
-typedef void (*ezusb_ctx_wait)(struct ezusb_priv *, struct request_context *);
-
-static void ezusb_req_ctx_wait_compl(struct ezusb_priv *upriv,
-				     struct request_context *ctx)
-{
-	switch (ctx->state) {
-	case EZUSB_CTX_QUEUED:
-	case EZUSB_CTX_REQ_SUBMITTED:
-	case EZUSB_CTX_REQ_COMPLETE:
-	case EZUSB_CTX_RESP_RECEIVED:
-		wait_for_completion(&ctx->done);
-		break;
-	default:
-		/* Done or failed - nothing to wait for */
-		break;
-	}
-}
-
-static void ezusb_req_ctx_wait_poll(struct ezusb_priv *upriv,
-				    struct request_context *ctx)
-{
-	int msecs;
-
-	switch (ctx->state) {
-	case EZUSB_CTX_QUEUED:
-	case EZUSB_CTX_REQ_SUBMITTED:
-	case EZUSB_CTX_REQ_COMPLETE:
-	case EZUSB_CTX_RESP_RECEIVED:
-		/* If we get called from a timer or with our lock acquired, then
-		 * we can't wait for the completion and have to poll. This won't
-		 * happen if the USB controller completes the URB requests in
-		 * BH.
-		 */
-		msecs = DEF_TIMEOUT * (1000 / HZ);
-
-		while (!try_wait_for_completion(&ctx->done) && msecs--)
-			udelay(1000);
-		break;
-	default:
-		/* Done or failed - nothing to wait for */
-		break;
-	}
-}
-
-static void ezusb_req_ctx_wait_skip(struct ezusb_priv *upriv,
-				    struct request_context *ctx)
-{
-	WARN(1, "Shouldn't be invoked for in_rid\n");
-}
-
-static inline u16 build_crc(struct ezusb_packet *data)
-{
-	u16 crc = 0;
-	u8 *bytes = (u8 *)data;
-	int i;
-
-	for (i = 0; i < 8; i++)
-		crc = (crc << 1) + bytes[i];
-
-	return crc;
-}
-
-/*
- * ezusb_fill_req:
- *
- * if data == NULL and length > 0 the data is assumed to be already in
- * the target buffer and only the header is filled.
- *
- */
-static int ezusb_fill_req(struct ezusb_packet *req, u16 length, u16 rid,
-			  const void *data, u16 frame_type, u8 reply_count)
-{
-	int total_size = sizeof(*req) + length;
-
-	BUG_ON(total_size > BULK_BUF_SIZE);
-
-	req->magic = cpu_to_le16(EZUSB_MAGIC);
-	req->req_reply_count = reply_count;
-	req->ans_reply_count = 0;
-	req->frame_type = cpu_to_le16(frame_type);
-	req->size = cpu_to_le16(length + 4);
-	req->crc = cpu_to_le16(build_crc(req));
-	req->hermes_len = cpu_to_le16(HERMES_BYTES_TO_RECLEN(length));
-	req->hermes_rid = cpu_to_le16(rid);
-	if (data)
-		memcpy(req->data, data, length);
-	return total_size;
-}
-
-static int ezusb_submit_in_urb(struct ezusb_priv *upriv)
-{
-	int retval = 0;
-	void *cur_buf = upriv->read_urb->transfer_buffer;
-
-	if (upriv->read_urb->status == -EINPROGRESS) {
-		netdev_dbg(upriv->dev, "urb busy, not resubmiting\n");
-		retval = -EBUSY;
-		goto exit;
-	}
-	usb_fill_bulk_urb(upriv->read_urb, upriv->udev, upriv->read_pipe,
-			  cur_buf, BULK_BUF_SIZE,
-			  ezusb_bulk_in_callback, upriv);
-	upriv->read_urb->transfer_flags = 0;
-	retval = usb_submit_urb(upriv->read_urb, GFP_ATOMIC);
-	if (retval)
-		err("%s submit failed %d", __func__, retval);
-
- exit:
-	return retval;
-}
-
-static inline int ezusb_8051_cpucs(struct ezusb_priv *upriv, int reset)
-{
-	int ret;
-	u8 *res_val = NULL;
-
-	if (!upriv->udev) {
-		err("%s: !upriv->udev", __func__);
-		return -EFAULT;
-	}
-
-	res_val = kmalloc(sizeof(*res_val), GFP_KERNEL);
-
-	if (!res_val)
-		return -ENOMEM;
-
-	*res_val = reset;	/* avoid argument promotion */
-
-	ret =  usb_control_msg(upriv->udev,
-			       usb_sndctrlpipe(upriv->udev, 0),
-			       EZUSB_REQUEST_FW_TRANS,
-			       USB_TYPE_VENDOR | USB_RECIP_DEVICE |
-			       USB_DIR_OUT, EZUSB_CPUCS_REG, 0, res_val,
-			       sizeof(*res_val), DEF_TIMEOUT);
-
-	kfree(res_val);
-
-	return ret;
-}
-
-static int ezusb_firmware_download(struct ezusb_priv *upriv,
-				   struct ez_usb_fw *fw)
-{
-	u8 *fw_buffer;
-	int retval, addr;
-	int variant_offset;
-
-	fw_buffer = kmalloc(FW_BUF_SIZE, GFP_KERNEL);
-	if (!fw_buffer) {
-		printk(KERN_ERR PFX "Out of memory for firmware buffer.\n");
-		return -ENOMEM;
-	}
-	/*
-	 * This byte is 1 and should be replaced with 0.  The offset is
-	 * 0x10AD in version 0.0.6.  The byte in question should follow
-	 * the end of the code pointed to by the jump in the beginning
-	 * of the firmware.  Also, it is read by code located at 0x358.
-	 */
-	variant_offset = be16_to_cpup((__be16 *) &fw->code[FW_VAR_OFFSET_PTR]);
-	if (variant_offset >= fw->size) {
-		printk(KERN_ERR PFX "Invalid firmware variant offset: "
-		       "0x%04x\n", variant_offset);
-		retval = -EINVAL;
-		goto fail;
-	}
-
-	retval = ezusb_8051_cpucs(upriv, 1);
-	if (retval < 0)
-		goto fail;
-	for (addr = 0; addr < fw->size; addr += FW_BUF_SIZE) {
-		/* 0x100-0x300 should be left alone, it contains card
-		 * specific data, like USB enumeration information */
-		if ((addr >= FW_HOLE_START) && (addr < FW_HOLE_END))
-			continue;
-
-		memcpy(fw_buffer, &fw->code[addr], FW_BUF_SIZE);
-		if (variant_offset >= addr &&
-		    variant_offset < addr + FW_BUF_SIZE) {
-			netdev_dbg(upriv->dev,
-				   "Patching card_variant byte at 0x%04X\n",
-				   variant_offset);
-			fw_buffer[variant_offset - addr] = FW_VAR_VALUE;
-		}
-		retval = usb_control_msg(upriv->udev,
-					 usb_sndctrlpipe(upriv->udev, 0),
-					 EZUSB_REQUEST_FW_TRANS,
-					 USB_TYPE_VENDOR | USB_RECIP_DEVICE
-					 | USB_DIR_OUT,
-					 addr, 0x0,
-					 fw_buffer, FW_BUF_SIZE,
-					 DEF_TIMEOUT);
-
-		if (retval < 0)
-			goto fail;
-	}
-	retval = ezusb_8051_cpucs(upriv, 0);
-	if (retval < 0)
-		goto fail;
-
-	goto exit;
- fail:
-	printk(KERN_ERR PFX "Firmware download failed, error %d\n",
-	       retval);
- exit:
-	kfree(fw_buffer);
-	return retval;
-}
-
-static int ezusb_access_ltv(struct ezusb_priv *upriv,
-			    struct request_context *ctx,
-			    u16 length, const void *data, u16 frame_type,
-			    void *ans_buff, unsigned ans_size, u16 *ans_length,
-			    ezusb_ctx_wait ezusb_ctx_wait_func)
-{
-	int req_size;
-	int retval = 0;
-	enum ezusb_state state;
-
-	if (!upriv->udev) {
-		retval = -ENODEV;
-		goto exit;
-	}
-
-	if (upriv->read_urb->status != -EINPROGRESS)
-		err("%s: in urb not pending", __func__);
-
-	/* protect upriv->reply_count, guarantee sequential numbers */
-	spin_lock_bh(&upriv->reply_count_lock);
-	req_size = ezusb_fill_req(ctx->buf, length, ctx->out_rid, data,
-				  frame_type, upriv->reply_count);
-	usb_fill_bulk_urb(ctx->outurb, upriv->udev, upriv->write_pipe,
-			  ctx->buf, req_size,
-			  ezusb_request_out_callback, ctx);
-
-	if (ctx->in_rid)
-		upriv->reply_count = ezusb_reply_inc(upriv->reply_count);
-
-	ezusb_req_enqueue_run(upriv, ctx);
-
-	spin_unlock_bh(&upriv->reply_count_lock);
-
-	if (ctx->in_rid)
-		ezusb_ctx_wait_func(upriv, ctx);
-
-	state = ctx->state;
-	switch (state) {
-	case EZUSB_CTX_COMPLETE:
-		retval = ctx->outurb->status;
-		break;
-
-	case EZUSB_CTX_QUEUED:
-	case EZUSB_CTX_REQ_SUBMITTED:
-		if (!ctx->in_rid)
-			break;
-		fallthrough;
-	default:
-		err("%s: Unexpected context state %d", __func__,
-		    state);
-		fallthrough;
-	case EZUSB_CTX_REQ_TIMEOUT:
-	case EZUSB_CTX_REQ_FAILED:
-	case EZUSB_CTX_RESP_TIMEOUT:
-	case EZUSB_CTX_REQSUBMIT_FAIL:
-		printk(KERN_ERR PFX "Access failed, resetting (state %d,"
-		       " reply_count %d)\n", state, upriv->reply_count);
-		upriv->reply_count = 0;
-		if (state == EZUSB_CTX_REQ_TIMEOUT
-		    || state == EZUSB_CTX_RESP_TIMEOUT) {
-			printk(KERN_ERR PFX "ctx timed out\n");
-			retval = -ETIMEDOUT;
-		} else {
-			printk(KERN_ERR PFX "ctx failed\n");
-			retval = -EFAULT;
-		}
-		goto exit;
-	}
-	if (ctx->in_rid) {
-		struct ezusb_packet *ans = ctx->buf;
-		unsigned exp_len;
-
-		if (ans->hermes_len != 0)
-			exp_len = le16_to_cpu(ans->hermes_len) * 2 + 12;
-		else
-			exp_len = 14;
-
-		if (exp_len != ctx->buf_length) {
-			err("%s: length mismatch for RID 0x%04x: "
-			    "expected %d, got %d", __func__,
-			    ctx->in_rid, exp_len, ctx->buf_length);
-			retval = -EIO;
-			goto exit;
-		}
-
-		if (ans_buff)
-			memcpy(ans_buff, ans->data, min(exp_len, ans_size));
-		if (ans_length)
-			*ans_length = le16_to_cpu(ans->hermes_len);
-	}
- exit:
-	ezusb_request_context_put(ctx);
-	return retval;
-}
-
-static int __ezusb_write_ltv(struct hermes *hw, int bap, u16 rid,
-			   u16 length, const void *data,
-			   ezusb_ctx_wait ezusb_ctx_wait_func)
-{
-	struct ezusb_priv *upriv = hw->priv;
-	u16 frame_type;
-	struct request_context *ctx;
-
-	if (length == 0)
-		return -EINVAL;
-
-	length = HERMES_RECLEN_TO_BYTES(length);
-
-	/* On memory mapped devices HERMES_RID_CNFGROUPADDRESSES can be
-	 * set to be empty, but the USB bridge doesn't like it */
-	if (length == 0)
-		return 0;
-
-	ctx = ezusb_alloc_ctx(upriv, rid, EZUSB_RID_ACK);
-	if (!ctx)
-		return -ENOMEM;
-
-	if (rid == EZUSB_RID_TX)
-		frame_type = EZUSB_FRAME_DATA;
-	else
-		frame_type = EZUSB_FRAME_CONTROL;
-
-	return ezusb_access_ltv(upriv, ctx, length, data, frame_type,
-				NULL, 0, NULL, ezusb_ctx_wait_func);
-}
-
-static int ezusb_write_ltv(struct hermes *hw, int bap, u16 rid,
-			   u16 length, const void *data)
-{
-	return __ezusb_write_ltv(hw, bap, rid, length, data,
-				 ezusb_req_ctx_wait_poll);
-}
-
-static int __ezusb_read_ltv(struct hermes *hw, int bap, u16 rid,
-			    unsigned bufsize, u16 *length, void *buf,
-			    ezusb_ctx_wait ezusb_ctx_wait_func)
-
-{
-	struct ezusb_priv *upriv = hw->priv;
-	struct request_context *ctx;
-
-	if (bufsize % 2)
-		return -EINVAL;
-
-	ctx = ezusb_alloc_ctx(upriv, rid, rid);
-	if (!ctx)
-		return -ENOMEM;
-
-	return ezusb_access_ltv(upriv, ctx, 0, NULL, EZUSB_FRAME_CONTROL,
-				buf, bufsize, length, ezusb_req_ctx_wait_poll);
-}
-
-static int ezusb_read_ltv(struct hermes *hw, int bap, u16 rid,
-			    unsigned bufsize, u16 *length, void *buf)
-{
-	return __ezusb_read_ltv(hw, bap, rid, bufsize, length, buf,
-				ezusb_req_ctx_wait_poll);
-}
-
-static int ezusb_read_ltv_preempt(struct hermes *hw, int bap, u16 rid,
-				  unsigned bufsize, u16 *length, void *buf)
-{
-	return __ezusb_read_ltv(hw, bap, rid, bufsize, length, buf,
-				ezusb_req_ctx_wait_compl);
-}
-
-static int ezusb_doicmd_wait(struct hermes *hw, u16 cmd, u16 parm0, u16 parm1,
-			     u16 parm2, struct hermes_response *resp)
-{
-	WARN_ON_ONCE(1);
-	return -EINVAL;
-}
-
-static int __ezusb_docmd_wait(struct hermes *hw, u16 cmd, u16 parm0,
-			    struct hermes_response *resp,
-			    ezusb_ctx_wait ezusb_ctx_wait_func)
-{
-	struct ezusb_priv *upriv = hw->priv;
-	struct request_context *ctx;
-
-	__le16 data[4] = {
-		cpu_to_le16(cmd),
-		cpu_to_le16(parm0),
-		0,
-		0,
-	};
-	netdev_dbg(upriv->dev, "0x%04X, parm0 0x%04X\n", cmd, parm0);
-	ctx = ezusb_alloc_ctx(upriv, EZUSB_RID_DOCMD, EZUSB_RID_ACK);
-	if (!ctx)
-		return -ENOMEM;
-
-	return ezusb_access_ltv(upriv, ctx, sizeof(data), &data,
-				EZUSB_FRAME_CONTROL, NULL, 0, NULL,
-				ezusb_ctx_wait_func);
-}
-
-static int ezusb_docmd_wait(struct hermes *hw, u16 cmd, u16 parm0,
-			    struct hermes_response *resp)
-{
-	return __ezusb_docmd_wait(hw, cmd, parm0, resp, ezusb_req_ctx_wait_poll);
-}
-
-static int ezusb_bap_pread(struct hermes *hw, int bap,
-			   void *buf, int len, u16 id, u16 offset)
-{
-	struct ezusb_priv *upriv = hw->priv;
-	struct ezusb_packet *ans = (void *) upriv->read_urb->transfer_buffer;
-	int actual_length = upriv->read_urb->actual_length;
-
-	if (id == EZUSB_RID_RX) {
-		if ((sizeof(*ans) + offset + len) > actual_length) {
-			printk(KERN_ERR PFX "BAP read beyond buffer end "
-			       "in rx frame\n");
-			return -EINVAL;
-		}
-		memcpy(buf, ans->data + offset, len);
-		return 0;
-	}
-
-	if (EZUSB_IS_INFO(id)) {
-		/* Include 4 bytes for length/type */
-		if ((sizeof(*ans) + offset + len - 4) > actual_length) {
-			printk(KERN_ERR PFX "BAP read beyond buffer end "
-			       "in info frame\n");
-			return -EFAULT;
-		}
-		memcpy(buf, ans->data + offset - 4, len);
-	} else {
-		printk(KERN_ERR PFX "Unexpected fid 0x%04x\n", id);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int ezusb_read_pda(struct hermes *hw, __le16 *pda,
-			  u32 pda_addr, u16 pda_len)
-{
-	struct ezusb_priv *upriv = hw->priv;
-	struct request_context *ctx;
-	__le16 data[] = {
-		cpu_to_le16(pda_addr & 0xffff),
-		cpu_to_le16(pda_len - 4)
-	};
-	ctx = ezusb_alloc_ctx(upriv, EZUSB_RID_READ_PDA, EZUSB_RID_READ_PDA);
-	if (!ctx)
-		return -ENOMEM;
-
-	/* wl_lkm does not include PDA size in the PDA area.
-	 * We will pad the information into pda, so other routines
-	 * don't have to be modified */
-	pda[0] = cpu_to_le16(pda_len - 2);
-	/* Includes CFG_PROD_DATA but not itself */
-	pda[1] = cpu_to_le16(0x0800); /* CFG_PROD_DATA */
-
-	return ezusb_access_ltv(upriv, ctx, sizeof(data), &data,
-				EZUSB_FRAME_CONTROL, &pda[2], pda_len - 4,
-				NULL, ezusb_req_ctx_wait_compl);
-}
-
-static int ezusb_program_init(struct hermes *hw, u32 entry_point)
-{
-	struct ezusb_priv *upriv = hw->priv;
-	struct request_context *ctx;
-	__le32 data = cpu_to_le32(entry_point);
-
-	ctx = ezusb_alloc_ctx(upriv, EZUSB_RID_PROG_INIT, EZUSB_RID_ACK);
-	if (!ctx)
-		return -ENOMEM;
-
-	return ezusb_access_ltv(upriv, ctx, sizeof(data), &data,
-				EZUSB_FRAME_CONTROL, NULL, 0, NULL,
-				ezusb_req_ctx_wait_compl);
-}
-
-static int ezusb_program_end(struct hermes *hw)
-{
-	struct ezusb_priv *upriv = hw->priv;
-	struct request_context *ctx;
-
-	ctx = ezusb_alloc_ctx(upriv, EZUSB_RID_PROG_END, EZUSB_RID_ACK);
-	if (!ctx)
-		return -ENOMEM;
-
-	return ezusb_access_ltv(upriv, ctx, 0, NULL,
-				EZUSB_FRAME_CONTROL, NULL, 0, NULL,
-				ezusb_req_ctx_wait_compl);
-}
-
-static int ezusb_program_bytes(struct hermes *hw, const char *buf,
-			       u32 addr, u32 len)
-{
-	struct ezusb_priv *upriv = hw->priv;
-	struct request_context *ctx;
-	__le32 data = cpu_to_le32(addr);
-	int err;
-
-	ctx = ezusb_alloc_ctx(upriv, EZUSB_RID_PROG_SET_ADDR, EZUSB_RID_ACK);
-	if (!ctx)
-		return -ENOMEM;
-
-	err = ezusb_access_ltv(upriv, ctx, sizeof(data), &data,
-			       EZUSB_FRAME_CONTROL, NULL, 0, NULL,
-			       ezusb_req_ctx_wait_compl);
-	if (err)
-		return err;
-
-	ctx = ezusb_alloc_ctx(upriv, EZUSB_RID_PROG_BYTES, EZUSB_RID_ACK);
-	if (!ctx)
-		return -ENOMEM;
-
-	return ezusb_access_ltv(upriv, ctx, len, buf,
-				EZUSB_FRAME_CONTROL, NULL, 0, NULL,
-				ezusb_req_ctx_wait_compl);
-}
-
-static int ezusb_program(struct hermes *hw, const char *buf,
-			 u32 addr, u32 len)
-{
-	u32 ch_addr;
-	u32 ch_len;
-	int err = 0;
-
-	/* We can only send 2048 bytes out of the bulk xmit at a time,
-	 * so we have to split any programming into chunks of <2048
-	 * bytes. */
-
-	ch_len = (len < MAX_DL_SIZE) ? len : MAX_DL_SIZE;
-	ch_addr = addr;
-
-	while (ch_addr < (addr + len)) {
-		pr_debug("Programming subblock of length %d "
-			 "to address 0x%08x. Data @ %p\n",
-			 ch_len, ch_addr, &buf[ch_addr - addr]);
-
-		err = ezusb_program_bytes(hw, &buf[ch_addr - addr],
-					  ch_addr, ch_len);
-		if (err)
-			break;
-
-		ch_addr += ch_len;
-		ch_len = ((addr + len - ch_addr) < MAX_DL_SIZE) ?
-			(addr + len - ch_addr) : MAX_DL_SIZE;
-	}
-
-	return err;
-}
-
-static netdev_tx_t ezusb_xmit(struct sk_buff *skb, struct net_device *dev)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	struct net_device_stats *stats = &dev->stats;
-	struct ezusb_priv *upriv = priv->card;
-	u8 mic[MICHAEL_MIC_LEN + 1];
-	int err = 0;
-	int tx_control;
-	unsigned long flags;
-	struct request_context *ctx;
-	u8 *buf;
-	int tx_size;
-
-	if (!netif_running(dev)) {
-		printk(KERN_ERR "%s: Tx on stopped device!\n",
-		       dev->name);
-		return NETDEV_TX_BUSY;
-	}
-
-	if (netif_queue_stopped(dev)) {
-		printk(KERN_DEBUG "%s: Tx while transmitter busy!\n",
-		       dev->name);
-		return NETDEV_TX_BUSY;
-	}
-
-	if (orinoco_lock(priv, &flags) != 0) {
-		printk(KERN_ERR
-		       "%s: ezusb_xmit() called while hw_unavailable\n",
-		       dev->name);
-		return NETDEV_TX_BUSY;
-	}
-
-	if (!netif_carrier_ok(dev) ||
-	    (priv->iw_mode == NL80211_IFTYPE_MONITOR)) {
-		/* Oops, the firmware hasn't established a connection,
-		   silently drop the packet (this seems to be the
-		   safest approach). */
-		goto drop;
-	}
-
-	/* Check packet length */
-	if (skb->len < ETH_HLEN)
-		goto drop;
-
-	tx_control = 0;
-
-	err = orinoco_process_xmit_skb(skb, dev, priv, &tx_control,
-				       &mic[0]);
-	if (err)
-		goto drop;
-
-	ctx = ezusb_alloc_ctx(upriv, EZUSB_RID_TX, 0);
-	if (!ctx)
-		goto drop;
-
-	memset(ctx->buf, 0, BULK_BUF_SIZE);
-	buf = ctx->buf->data;
-
-	{
-		__le16 *tx_cntl = (__le16 *)buf;
-		*tx_cntl = cpu_to_le16(tx_control);
-		buf += sizeof(*tx_cntl);
-	}
-
-	memcpy(buf, skb->data, skb->len);
-	buf += skb->len;
-
-	if (tx_control & HERMES_TXCTRL_MIC) {
-		u8 *m = mic;
-		/* Mic has been offset so it can be copied to an even
-		 * address. We're copying eveything anyway, so we
-		 * don't need to copy that first byte. */
-		if (skb->len % 2)
-			m++;
-		memcpy(buf, m, MICHAEL_MIC_LEN);
-		buf += MICHAEL_MIC_LEN;
-	}
-
-	/* Finally, we actually initiate the send */
-	netif_stop_queue(dev);
-
-	/* The card may behave better if we send evenly sized usb transfers */
-	tx_size = ALIGN(buf - ctx->buf->data, 2);
-
-	err = ezusb_access_ltv(upriv, ctx, tx_size, NULL,
-			       EZUSB_FRAME_DATA, NULL, 0, NULL,
-			       ezusb_req_ctx_wait_skip);
-
-	if (err) {
-		netif_start_queue(dev);
-		if (net_ratelimit())
-			printk(KERN_ERR "%s: Error %d transmitting packet\n",
-				dev->name, err);
-		goto busy;
-	}
-
-	netif_trans_update(dev);
-	stats->tx_bytes += skb->len;
-	goto ok;
-
- drop:
-	stats->tx_errors++;
-	stats->tx_dropped++;
-
- ok:
-	orinoco_unlock(priv, &flags);
-	dev_kfree_skb(skb);
-	return NETDEV_TX_OK;
-
- busy:
-	orinoco_unlock(priv, &flags);
-	return NETDEV_TX_BUSY;
-}
-
-static int ezusb_allocate(struct hermes *hw, u16 size, u16 *fid)
-{
-	*fid = EZUSB_RID_TX;
-	return 0;
-}
-
-
-static int ezusb_hard_reset(struct orinoco_private *priv)
-{
-	struct ezusb_priv *upriv = priv->card;
-	int retval = ezusb_8051_cpucs(upriv, 1);
-
-	if (retval < 0) {
-		err("Failed to reset");
-		return retval;
-	}
-
-	retval = ezusb_8051_cpucs(upriv, 0);
-	if (retval < 0) {
-		err("Failed to unreset");
-		return retval;
-	}
-
-	netdev_dbg(upriv->dev, "sending control message\n");
-	retval = usb_control_msg(upriv->udev,
-				 usb_sndctrlpipe(upriv->udev, 0),
-				 EZUSB_REQUEST_TRIGGER,
-				 USB_TYPE_VENDOR | USB_RECIP_DEVICE |
-				 USB_DIR_OUT, 0x0, 0x0, NULL, 0,
-				 DEF_TIMEOUT);
-	if (retval < 0) {
-		err("EZUSB_REQUEST_TRIGGER failed retval %d", retval);
-		return retval;
-	}
-#if 0
-	dbg("Sending EZUSB_REQUEST_TRIG_AC");
-	retval = usb_control_msg(upriv->udev,
-				 usb_sndctrlpipe(upriv->udev, 0),
-				 EZUSB_REQUEST_TRIG_AC,
-				 USB_TYPE_VENDOR | USB_RECIP_DEVICE |
-				 USB_DIR_OUT, 0x00FA, 0x0, NULL, 0,
-				 DEF_TIMEOUT);
-	if (retval < 0) {
-		err("EZUSB_REQUEST_TRIG_AC failed retval %d", retval);
-		return retval;
-	}
-#endif
-
-	return 0;
-}
-
-
-static int ezusb_init(struct hermes *hw)
-{
-	struct ezusb_priv *upriv = hw->priv;
-	int retval;
-
-	if (!upriv)
-		return -EINVAL;
-
-	upriv->reply_count = 0;
-	/* Write the MAGIC number on the simulated registers to keep
-	 * orinoco.c happy */
-	hermes_write_regn(hw, SWSUPPORT0, HERMES_MAGIC);
-	hermes_write_regn(hw, RXFID, EZUSB_RID_RX);
-
-	usb_kill_urb(upriv->read_urb);
-	ezusb_submit_in_urb(upriv);
-
-	retval = __ezusb_write_ltv(hw, 0, EZUSB_RID_INIT1,
-				 HERMES_BYTES_TO_RECLEN(2), "\x10\x00",
-				 ezusb_req_ctx_wait_compl);
-	if (retval < 0) {
-		printk(KERN_ERR PFX "EZUSB_RID_INIT1 error %d\n", retval);
-		return retval;
-	}
-
-	retval = __ezusb_docmd_wait(hw, HERMES_CMD_INIT, 0, NULL,
-				    ezusb_req_ctx_wait_compl);
-	if (retval < 0) {
-		printk(KERN_ERR PFX "HERMES_CMD_INIT error %d\n", retval);
-		return retval;
-	}
-
-	return 0;
-}
-
-static void ezusb_bulk_in_callback(struct urb *urb)
-{
-	struct ezusb_priv *upriv = (struct ezusb_priv *) urb->context;
-	struct ezusb_packet *ans = urb->transfer_buffer;
-	u16 crc;
-	u16 hermes_rid;
-
-	if (upriv->udev == NULL)
-		return;
-
-	if (urb->status == -ETIMEDOUT) {
-		/* When a device gets unplugged we get this every time
-		 * we resubmit, flooding the logs.  Since we don't use
-		 * USB timeouts, it shouldn't happen any other time*/
-		pr_warn("%s: urb timed out, not resubmitting\n", __func__);
-		return;
-	}
-	if (urb->status == -ECONNABORTED) {
-		pr_warn("%s: connection abort, resubmitting urb\n",
-			__func__);
-		goto resubmit;
-	}
-	if ((urb->status == -EILSEQ)
-	    || (urb->status == -ENOENT)
-	    || (urb->status == -ECONNRESET)) {
-		netdev_dbg(upriv->dev, "status %d, not resubmiting\n",
-			   urb->status);
-		return;
-	}
-	if (urb->status)
-		netdev_dbg(upriv->dev, "status: %d length: %d\n",
-			   urb->status, urb->actual_length);
-	if (urb->actual_length < sizeof(*ans)) {
-		err("%s: short read, ignoring", __func__);
-		goto resubmit;
-	}
-	crc = build_crc(ans);
-	if (le16_to_cpu(ans->crc) != crc) {
-		err("CRC error, ignoring packet");
-		goto resubmit;
-	}
-
-	hermes_rid = le16_to_cpu(ans->hermes_rid);
-	if ((hermes_rid != EZUSB_RID_RX) && !EZUSB_IS_INFO(hermes_rid)) {
-		ezusb_request_in_callback(upriv, urb);
-	} else if (upriv->dev) {
-		struct net_device *dev = upriv->dev;
-		struct orinoco_private *priv = ndev_priv(dev);
-		struct hermes *hw = &priv->hw;
-
-		if (hermes_rid == EZUSB_RID_RX) {
-			__orinoco_ev_rx(dev, hw);
-		} else {
-			hermes_write_regn(hw, INFOFID,
-					  le16_to_cpu(ans->hermes_rid));
-			__orinoco_ev_info(dev, hw);
-		}
-	}
-
- resubmit:
-	if (upriv->udev)
-		ezusb_submit_in_urb(upriv);
-}
-
-static inline void ezusb_delete(struct ezusb_priv *upriv)
-{
-	struct list_head *item;
-	struct list_head *tmp_item;
-	unsigned long flags;
-
-	BUG_ON(!upriv);
-
-	mutex_lock(&upriv->mtx);
-
-	upriv->udev = NULL;	/* No timer will be rearmed from here */
-
-	usb_kill_urb(upriv->read_urb);
-
-	spin_lock_irqsave(&upriv->req_lock, flags);
-	list_for_each_safe(item, tmp_item, &upriv->req_active) {
-		struct request_context *ctx;
-		int err;
-
-		ctx = list_entry(item, struct request_context, list);
-		refcount_inc(&ctx->refcount);
-
-		ctx->outurb->transfer_flags |= URB_ASYNC_UNLINK;
-		err = usb_unlink_urb(ctx->outurb);
-
-		spin_unlock_irqrestore(&upriv->req_lock, flags);
-		if (err == -EINPROGRESS)
-			wait_for_completion(&ctx->done);
-
-		del_timer_sync(&ctx->timer);
-		/* FIXME: there is an slight chance for the irq handler to
-		 * be running */
-		if (!list_empty(&ctx->list))
-			ezusb_ctx_complete(ctx);
-
-		ezusb_request_context_put(ctx);
-		spin_lock_irqsave(&upriv->req_lock, flags);
-	}
-	spin_unlock_irqrestore(&upriv->req_lock, flags);
-
-	list_for_each_safe(item, tmp_item, &upriv->req_pending)
-	    ezusb_ctx_complete(list_entry(item,
-					  struct request_context, list));
-
-	if (upriv->read_urb && upriv->read_urb->status == -EINPROGRESS)
-		printk(KERN_ERR PFX "Some URB in progress\n");
-
-	mutex_unlock(&upriv->mtx);
-
-	if (upriv->read_urb) {
-		kfree(upriv->read_urb->transfer_buffer);
-		usb_free_urb(upriv->read_urb);
-	}
-	kfree(upriv->bap_buf);
-	if (upriv->dev) {
-		struct orinoco_private *priv = ndev_priv(upriv->dev);
-		orinoco_if_del(priv);
-		wiphy_unregister(priv_to_wiphy(upriv));
-		free_orinocodev(priv);
-	}
-}
-
-static void ezusb_lock_irqsave(spinlock_t *lock,
-			       unsigned long *flags) __acquires(lock)
-{
-	spin_lock_bh(lock);
-}
-
-static void ezusb_unlock_irqrestore(spinlock_t *lock,
-				    unsigned long *flags) __releases(lock)
-{
-	spin_unlock_bh(lock);
-}
-
-static void ezusb_lock_irq(spinlock_t *lock) __acquires(lock)
-{
-	spin_lock_bh(lock);
-}
-
-static void ezusb_unlock_irq(spinlock_t *lock) __releases(lock)
-{
-	spin_unlock_bh(lock);
-}
-
-static const struct hermes_ops ezusb_ops = {
-	.init = ezusb_init,
-	.cmd_wait = ezusb_docmd_wait,
-	.init_cmd_wait = ezusb_doicmd_wait,
-	.allocate = ezusb_allocate,
-	.read_ltv = ezusb_read_ltv,
-	.read_ltv_pr = ezusb_read_ltv_preempt,
-	.write_ltv = ezusb_write_ltv,
-	.bap_pread = ezusb_bap_pread,
-	.read_pda = ezusb_read_pda,
-	.program_init = ezusb_program_init,
-	.program_end = ezusb_program_end,
-	.program = ezusb_program,
-	.lock_irqsave = ezusb_lock_irqsave,
-	.unlock_irqrestore = ezusb_unlock_irqrestore,
-	.lock_irq = ezusb_lock_irq,
-	.unlock_irq = ezusb_unlock_irq,
-};
-
-static const struct net_device_ops ezusb_netdev_ops = {
-	.ndo_open		= orinoco_open,
-	.ndo_stop		= orinoco_stop,
-	.ndo_start_xmit		= ezusb_xmit,
-	.ndo_set_rx_mode	= orinoco_set_multicast_list,
-	.ndo_change_mtu		= orinoco_change_mtu,
-	.ndo_set_mac_address	= eth_mac_addr,
-	.ndo_validate_addr	= eth_validate_addr,
-	.ndo_tx_timeout		= orinoco_tx_timeout,
-};
-
-static int ezusb_probe(struct usb_interface *interface,
-		       const struct usb_device_id *id)
-{
-	struct usb_device *udev = interface_to_usbdev(interface);
-	struct orinoco_private *priv;
-	struct hermes *hw;
-	struct ezusb_priv *upriv = NULL;
-	struct usb_interface_descriptor *iface_desc;
-	struct usb_endpoint_descriptor *ep;
-	const struct firmware *fw_entry = NULL;
-	int retval = 0;
-	int i;
-
-	priv = alloc_orinocodev(sizeof(*upriv), &udev->dev,
-				ezusb_hard_reset, NULL);
-	if (!priv) {
-		err("Couldn't allocate orinocodev");
-		retval = -ENOMEM;
-		goto exit;
-	}
-
-	hw = &priv->hw;
-
-	upriv = priv->card;
-
-	mutex_init(&upriv->mtx);
-	spin_lock_init(&upriv->reply_count_lock);
-
-	spin_lock_init(&upriv->req_lock);
-	INIT_LIST_HEAD(&upriv->req_pending);
-	INIT_LIST_HEAD(&upriv->req_active);
-
-	upriv->udev = udev;
-
-	hw->iobase = (void __force __iomem *) &upriv->hermes_reg_fake;
-	hw->reg_spacing = HERMES_16BIT_REGSPACING;
-	hw->priv = upriv;
-	hw->ops = &ezusb_ops;
-
-	/* set up the endpoint information */
-	/* check out the endpoints */
-
-	iface_desc = &interface->cur_altsetting->desc;
-	for (i = 0; i < iface_desc->bNumEndpoints; ++i) {
-		ep = &interface->cur_altsetting->endpoint[i].desc;
-
-		if (usb_endpoint_is_bulk_in(ep)) {
-			/* we found a bulk in endpoint */
-			if (upriv->read_urb != NULL) {
-				pr_warn("Found a second bulk in ep, ignored\n");
-				continue;
-			}
-
-			upriv->read_urb = usb_alloc_urb(0, GFP_KERNEL);
-			if (!upriv->read_urb)
-				goto error;
-			if (le16_to_cpu(ep->wMaxPacketSize) != 64)
-				pr_warn("bulk in: wMaxPacketSize!= 64\n");
-			if (ep->bEndpointAddress != (2 | USB_DIR_IN))
-				pr_warn("bulk in: bEndpointAddress: %d\n",
-					ep->bEndpointAddress);
-			upriv->read_pipe = usb_rcvbulkpipe(udev,
-							 ep->
-							 bEndpointAddress);
-			upriv->read_urb->transfer_buffer =
-			    kmalloc(BULK_BUF_SIZE, GFP_KERNEL);
-			if (!upriv->read_urb->transfer_buffer) {
-				err("Couldn't allocate IN buffer");
-				goto error;
-			}
-		}
-
-		if (usb_endpoint_is_bulk_out(ep)) {
-			/* we found a bulk out endpoint */
-			if (upriv->bap_buf != NULL) {
-				pr_warn("Found a second bulk out ep, ignored\n");
-				continue;
-			}
-
-			if (le16_to_cpu(ep->wMaxPacketSize) != 64)
-				pr_warn("bulk out: wMaxPacketSize != 64\n");
-			if (ep->bEndpointAddress != 2)
-				pr_warn("bulk out: bEndpointAddress: %d\n",
-					ep->bEndpointAddress);
-			upriv->write_pipe = usb_sndbulkpipe(udev,
-							  ep->
-							  bEndpointAddress);
-			upriv->bap_buf = kmalloc(BULK_BUF_SIZE, GFP_KERNEL);
-			if (!upriv->bap_buf) {
-				err("Couldn't allocate bulk_out_buffer");
-				goto error;
-			}
-		}
-	}
-	if (!upriv->bap_buf || !upriv->read_urb) {
-		err("Didn't find the required bulk endpoints");
-		goto error;
-	}
-
-	if (request_firmware(&fw_entry, "orinoco_ezusb_fw",
-			     &interface->dev) == 0) {
-		firmware.size = fw_entry->size;
-		firmware.code = fw_entry->data;
-	}
-	if (firmware.size && firmware.code) {
-		if (ezusb_firmware_download(upriv, &firmware) < 0)
-			goto error;
-	} else {
-		err("No firmware to download");
-		goto error;
-	}
-
-	if (ezusb_hard_reset(priv) < 0) {
-		err("Cannot reset the device");
-		goto error;
-	}
-
-	/* If the firmware is already downloaded orinoco.c will call
-	 * ezusb_init but if the firmware is not already there, that will make
-	 * the kernel very unstable, so we try initializing here and quit in
-	 * case of error */
-	if (ezusb_init(hw) < 0) {
-		err("Couldn't initialize the device");
-		err("Firmware may not be downloaded or may be wrong.");
-		goto error;
-	}
-
-	/* Initialise the main driver */
-	if (orinoco_init(priv) != 0) {
-		err("orinoco_init() failed\n");
-		goto error;
-	}
-
-	if (orinoco_if_add(priv, 0, 0, &ezusb_netdev_ops) != 0) {
-		upriv->dev = NULL;
-		err("%s: orinoco_if_add() failed", __func__);
-		wiphy_unregister(priv_to_wiphy(priv));
-		goto error;
-	}
-	upriv->dev = priv->ndev;
-
-	goto exit;
-
- error:
-	ezusb_delete(upriv);
-	if (upriv->dev) {
-		/* upriv->dev was 0, so ezusb_delete() didn't free it */
-		free_orinocodev(priv);
-	}
-	upriv = NULL;
-	retval = -EFAULT;
- exit:
-	if (fw_entry) {
-		firmware.code = NULL;
-		firmware.size = 0;
-		release_firmware(fw_entry);
-	}
-	usb_set_intfdata(interface, upriv);
-	return retval;
-}
-
-
-static void ezusb_disconnect(struct usb_interface *intf)
-{
-	struct ezusb_priv *upriv = usb_get_intfdata(intf);
-	usb_set_intfdata(intf, NULL);
-	ezusb_delete(upriv);
-	printk(KERN_INFO PFX "Disconnected\n");
-}
-
-
-/* usb specific object needed to register this driver with the usb subsystem */
-static struct usb_driver orinoco_driver = {
-	.name = DRIVER_NAME,
-	.probe = ezusb_probe,
-	.disconnect = ezusb_disconnect,
-	.id_table = ezusb_table,
-	.disable_hub_initiated_lpm = 1,
-};
-
-module_usb_driver(orinoco_driver);
-
-MODULE_AUTHOR("Manuel Estrada Sainz");
-MODULE_DESCRIPTION("Driver for Orinoco wireless LAN cards using EZUSB bridge");
-MODULE_LICENSE("Dual MPL/GPL");
diff --git a/drivers/net/wireless/intersil/orinoco/scan.c b/drivers/net/wireless/intersil/orinoco/scan.c
deleted file mode 100644
index 6d1d084854fb..000000000000
--- a/drivers/net/wireless/intersil/orinoco/scan.c
+++ /dev/null
@@ -1,259 +0,0 @@
-/* Helpers for managing scan queues
- *
- * See copyright notice in main.c
- */
-
-#include <linux/gfp.h>
-#include <linux/kernel.h>
-#include <linux/string.h>
-#include <linux/ieee80211.h>
-#include <net/cfg80211.h>
-
-#include "hermes.h"
-#include "orinoco.h"
-#include "main.h"
-
-#include "scan.h"
-
-#define ZERO_DBM_OFFSET 0x95
-#define MAX_SIGNAL_LEVEL 0x8A
-#define MIN_SIGNAL_LEVEL 0x2F
-
-#define SIGNAL_TO_DBM(x)					\
-	(clamp_t(s32, (x), MIN_SIGNAL_LEVEL, MAX_SIGNAL_LEVEL)	\
-	 - ZERO_DBM_OFFSET)
-#define SIGNAL_TO_MBM(x) (SIGNAL_TO_DBM(x) * 100)
-
-static int symbol_build_supp_rates(u8 *buf, const __le16 *rates)
-{
-	int i;
-	u8 rate;
-
-	buf[0] = WLAN_EID_SUPP_RATES;
-	for (i = 0; i < 5; i++) {
-		rate = le16_to_cpu(rates[i]);
-		/* NULL terminated */
-		if (rate == 0x0)
-			break;
-		buf[i + 2] = rate;
-	}
-	buf[1] = i;
-
-	return i + 2;
-}
-
-static int prism_build_supp_rates(u8 *buf, const u8 *rates)
-{
-	int i;
-
-	buf[0] = WLAN_EID_SUPP_RATES;
-	for (i = 0; i < 8; i++) {
-		/* NULL terminated */
-		if (rates[i] == 0x0)
-			break;
-		buf[i + 2] = rates[i];
-	}
-	buf[1] = i;
-
-	/* We might still have another 2 rates, which need to go in
-	 * extended supported rates */
-	if (i == 8 && rates[i] > 0) {
-		buf[10] = WLAN_EID_EXT_SUPP_RATES;
-		for (; i < 10; i++) {
-			/* NULL terminated */
-			if (rates[i] == 0x0)
-				break;
-			buf[i + 2] = rates[i];
-		}
-		buf[11] = i - 8;
-	}
-
-	return (i < 8) ? i + 2 : i + 4;
-}
-
-static void orinoco_add_hostscan_result(struct orinoco_private *priv,
-					const union hermes_scan_info *bss)
-{
-	struct wiphy *wiphy = priv_to_wiphy(priv);
-	struct ieee80211_channel *channel;
-	struct cfg80211_bss *cbss;
-	u8 *ie;
-	u8 ie_buf[46];
-	u64 timestamp;
-	s32 signal;
-	u16 capability;
-	u16 beacon_interval;
-	int ie_len;
-	int freq;
-	int len;
-
-	len = le16_to_cpu(bss->a.essid_len);
-
-	/* Reconstruct SSID and bitrate IEs to pass up */
-	ie_buf[0] = WLAN_EID_SSID;
-	ie_buf[1] = len;
-	memcpy(&ie_buf[2], bss->a.essid, len);
-
-	ie = ie_buf + len + 2;
-	ie_len = ie_buf[1] + 2;
-	switch (priv->firmware_type) {
-	case FIRMWARE_TYPE_SYMBOL:
-		ie_len += symbol_build_supp_rates(ie, bss->s.rates);
-		break;
-
-	case FIRMWARE_TYPE_INTERSIL:
-		ie_len += prism_build_supp_rates(ie, bss->p.rates);
-		break;
-
-	case FIRMWARE_TYPE_AGERE:
-	default:
-		break;
-	}
-
-	freq = ieee80211_channel_to_frequency(
-		le16_to_cpu(bss->a.channel), NL80211_BAND_2GHZ);
-	channel = ieee80211_get_channel(wiphy, freq);
-	if (!channel) {
-		printk(KERN_DEBUG "Invalid channel designation %04X(%04X)",
-			bss->a.channel, freq);
-		return;	/* Then ignore it for now */
-	}
-	timestamp = 0;
-	capability = le16_to_cpu(bss->a.capabilities);
-	beacon_interval = le16_to_cpu(bss->a.beacon_interv);
-	signal = SIGNAL_TO_MBM(le16_to_cpu(bss->a.level));
-
-	cbss = cfg80211_inform_bss(wiphy, channel, CFG80211_BSS_FTYPE_UNKNOWN,
-				   bss->a.bssid, timestamp, capability,
-				   beacon_interval, ie_buf, ie_len, signal,
-				   GFP_KERNEL);
-	cfg80211_put_bss(wiphy, cbss);
-}
-
-void orinoco_add_extscan_result(struct orinoco_private *priv,
-				struct agere_ext_scan_info *bss,
-				size_t len)
-{
-	struct wiphy *wiphy = priv_to_wiphy(priv);
-	struct ieee80211_channel *channel;
-	struct cfg80211_bss *cbss;
-	const u8 *ie;
-	u64 timestamp;
-	s32 signal;
-	u16 capability;
-	u16 beacon_interval;
-	size_t ie_len;
-	int chan, freq;
-
-	ie_len = len - sizeof(*bss);
-	ie = cfg80211_find_ie(WLAN_EID_DS_PARAMS, bss->data, ie_len);
-	chan = ie ? ie[2] : 0;
-	freq = ieee80211_channel_to_frequency(chan, NL80211_BAND_2GHZ);
-	channel = ieee80211_get_channel(wiphy, freq);
-
-	timestamp = le64_to_cpu(bss->timestamp);
-	capability = le16_to_cpu(bss->capabilities);
-	beacon_interval = le16_to_cpu(bss->beacon_interval);
-	ie = bss->data;
-	signal = SIGNAL_TO_MBM(bss->level);
-
-	cbss = cfg80211_inform_bss(wiphy, channel, CFG80211_BSS_FTYPE_UNKNOWN,
-				   bss->bssid, timestamp, capability,
-				   beacon_interval, ie, ie_len, signal,
-				   GFP_KERNEL);
-	cfg80211_put_bss(wiphy, cbss);
-}
-
-void orinoco_add_hostscan_results(struct orinoco_private *priv,
-				  unsigned char *buf,
-				  size_t len)
-{
-	int offset;		/* In the scan data */
-	size_t atom_len;
-	bool abort = false;
-
-	switch (priv->firmware_type) {
-	case FIRMWARE_TYPE_AGERE:
-		atom_len = sizeof(struct agere_scan_apinfo);
-		offset = 0;
-		break;
-
-	case FIRMWARE_TYPE_SYMBOL:
-		/* Lack of documentation necessitates this hack.
-		 * Different firmwares have 68 or 76 byte long atoms.
-		 * We try modulo first.  If the length divides by both,
-		 * we check what would be the channel in the second
-		 * frame for a 68-byte atom.  76-byte atoms have 0 there.
-		 * Valid channel cannot be 0.  */
-		if (len % 76)
-			atom_len = 68;
-		else if (len % 68)
-			atom_len = 76;
-		else if (len >= 1292 && buf[68] == 0)
-			atom_len = 76;
-		else
-			atom_len = 68;
-		offset = 0;
-		break;
-
-	case FIRMWARE_TYPE_INTERSIL:
-		offset = 4;
-		if (priv->has_hostscan) {
-			atom_len = le16_to_cpup((__le16 *)buf);
-			/* Sanity check for atom_len */
-			if (atom_len < sizeof(struct prism2_scan_apinfo)) {
-				printk(KERN_ERR "%s: Invalid atom_len in scan "
-				       "data: %zu\n", priv->ndev->name,
-				       atom_len);
-				abort = true;
-				goto scan_abort;
-			}
-		} else
-			atom_len = offsetof(struct prism2_scan_apinfo, atim);
-		break;
-
-	default:
-		abort = true;
-		goto scan_abort;
-	}
-
-	/* Check that we got an whole number of atoms */
-	if ((len - offset) % atom_len) {
-		printk(KERN_ERR "%s: Unexpected scan data length %zu, "
-		       "atom_len %zu, offset %d\n", priv->ndev->name, len,
-		       atom_len, offset);
-		abort = true;
-		goto scan_abort;
-	}
-
-	/* Process the entries one by one */
-	for (; offset + atom_len <= len; offset += atom_len) {
-		union hermes_scan_info *atom;
-
-		atom = (union hermes_scan_info *) (buf + offset);
-
-		orinoco_add_hostscan_result(priv, atom);
-	}
-
- scan_abort:
-	if (priv->scan_request) {
-		struct cfg80211_scan_info info = {
-			.aborted = abort,
-		};
-
-		cfg80211_scan_done(priv->scan_request, &info);
-		priv->scan_request = NULL;
-	}
-}
-
-void orinoco_scan_done(struct orinoco_private *priv, bool abort)
-{
-	if (priv->scan_request) {
-		struct cfg80211_scan_info info = {
-			.aborted = abort,
-		};
-
-		cfg80211_scan_done(priv->scan_request, &info);
-		priv->scan_request = NULL;
-	}
-}
diff --git a/drivers/net/wireless/intersil/orinoco/scan.h b/drivers/net/wireless/intersil/orinoco/scan.h
deleted file mode 100644
index 27281fb0a6dc..000000000000
--- a/drivers/net/wireless/intersil/orinoco/scan.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/* Helpers for managing scan queues
- *
- * See copyright notice in main.c
- */
-#ifndef _ORINOCO_SCAN_H_
-#define _ORINOCO_SCAN_H_
-
-/* Forward declarations */
-struct orinoco_private;
-struct agere_ext_scan_info;
-
-/* Add scan results */
-void orinoco_add_extscan_result(struct orinoco_private *priv,
-				struct agere_ext_scan_info *atom,
-				size_t len);
-void orinoco_add_hostscan_results(struct orinoco_private *dev,
-				  unsigned char *buf,
-				  size_t len);
-void orinoco_scan_done(struct orinoco_private *priv, bool abort);
-
-#endif /* _ORINOCO_SCAN_H_ */
diff --git a/drivers/net/wireless/intersil/orinoco/spectrum_cs.c b/drivers/net/wireless/intersil/orinoco/spectrum_cs.c
deleted file mode 100644
index 841d623c621a..000000000000
--- a/drivers/net/wireless/intersil/orinoco/spectrum_cs.c
+++ /dev/null
@@ -1,328 +0,0 @@
-/*
- * Driver for 802.11b cards using RAM-loadable Symbol firmware, such as
- * Symbol Wireless Networker LA4137, CompactFlash cards by Socket
- * Communications and Intel PRO/Wireless 2011B.
- *
- * The driver implements Symbol firmware download.  The rest is handled
- * in hermes.c and main.c.
- *
- * Utilities for downloading the Symbol firmware are available at
- * http://sourceforge.net/projects/orinoco/
- *
- * Copyright (C) 2002-2005 Pavel Roskin <proski@gnu.org>
- * Portions based on orinoco_cs.c:
- *	Copyright (C) David Gibson, Linuxcare Australia
- * Portions based on Spectrum24tDnld.c from original spectrum24 driver:
- *	Copyright (C) Symbol Technologies.
- *
- * See copyright notice in file main.c.
- */
-
-#define DRIVER_NAME "spectrum_cs"
-#define PFX DRIVER_NAME ": "
-
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/delay.h>
-#include <pcmcia/cistpl.h>
-#include <pcmcia/cisreg.h>
-#include <pcmcia/ds.h>
-
-#include "orinoco.h"
-
-/********************************************************************/
-/* Module stuff							    */
-/********************************************************************/
-
-MODULE_AUTHOR("Pavel Roskin <proski@gnu.org>");
-MODULE_DESCRIPTION("Driver for Symbol Spectrum24 Trilogy cards with firmware downloader");
-MODULE_LICENSE("Dual MPL/GPL");
-
-/* Module parameters */
-
-/* Some D-Link cards have buggy CIS. They do work at 5v properly, but
- * don't have any CIS entry for it. This workaround it... */
-static int ignore_cis_vcc; /* = 0 */
-module_param(ignore_cis_vcc, int, 0);
-MODULE_PARM_DESC(ignore_cis_vcc, "Allow voltage mismatch between card and socket");
-
-/********************************************************************/
-/* Data structures						    */
-/********************************************************************/
-
-/* PCMCIA specific device information (goes in the card field of
- * struct orinoco_private */
-struct orinoco_pccard {
-	struct pcmcia_device	*p_dev;
-};
-
-/********************************************************************/
-/* Function prototypes						    */
-/********************************************************************/
-
-static int spectrum_cs_config(struct pcmcia_device *link);
-static void spectrum_cs_release(struct pcmcia_device *link);
-
-/* Constants for the CISREG_CCSR register */
-#define HCR_RUN		0x07	/* run firmware after reset */
-#define HCR_IDLE	0x0E	/* don't run firmware after reset */
-#define HCR_MEM16	0x10	/* memory width bit, should be preserved */
-
-
-/*
- * Reset the card using configuration registers COR and CCSR.
- * If IDLE is 1, stop the firmware, so that it can be safely rewritten.
- */
-static int
-spectrum_reset(struct pcmcia_device *link, int idle)
-{
-	int ret;
-	u8 save_cor;
-	u8 ccsr;
-
-	/* Doing it if hardware is gone is guaranteed crash */
-	if (!pcmcia_dev_present(link))
-		return -ENODEV;
-
-	/* Save original COR value */
-	ret = pcmcia_read_config_byte(link, CISREG_COR, &save_cor);
-	if (ret)
-		goto failed;
-
-	/* Soft-Reset card */
-	ret = pcmcia_write_config_byte(link, CISREG_COR,
-				(save_cor | COR_SOFT_RESET));
-	if (ret)
-		goto failed;
-	udelay(1000);
-
-	/* Read CCSR */
-	ret = pcmcia_read_config_byte(link, CISREG_CCSR, &ccsr);
-	if (ret)
-		goto failed;
-
-	/*
-	 * Start or stop the firmware.  Memory width bit should be
-	 * preserved from the value we've just read.
-	 */
-	ccsr = (idle ? HCR_IDLE : HCR_RUN) | (ccsr & HCR_MEM16);
-	ret = pcmcia_write_config_byte(link, CISREG_CCSR, ccsr);
-	if (ret)
-		goto failed;
-	udelay(1000);
-
-	/* Restore original COR configuration index */
-	ret = pcmcia_write_config_byte(link, CISREG_COR,
-				(save_cor & ~COR_SOFT_RESET));
-	if (ret)
-		goto failed;
-	udelay(1000);
-	return 0;
-
-failed:
-	return -ENODEV;
-}
-
-/********************************************************************/
-/* Device methods						    */
-/********************************************************************/
-
-static int
-spectrum_cs_hard_reset(struct orinoco_private *priv)
-{
-	struct orinoco_pccard *card = priv->card;
-	struct pcmcia_device *link = card->p_dev;
-
-	/* Soft reset using COR and HCR */
-	spectrum_reset(link, 0);
-
-	return 0;
-}
-
-static int
-spectrum_cs_stop_firmware(struct orinoco_private *priv, int idle)
-{
-	struct orinoco_pccard *card = priv->card;
-	struct pcmcia_device *link = card->p_dev;
-
-	return spectrum_reset(link, idle);
-}
-
-/********************************************************************/
-/* PCMCIA stuff							    */
-/********************************************************************/
-
-static int
-spectrum_cs_probe(struct pcmcia_device *link)
-{
-	struct orinoco_private *priv;
-	struct orinoco_pccard *card;
-	int ret;
-
-	priv = alloc_orinocodev(sizeof(*card), &link->dev,
-				spectrum_cs_hard_reset,
-				spectrum_cs_stop_firmware);
-	if (!priv)
-		return -ENOMEM;
-	card = priv->card;
-
-	/* Link both structures together */
-	card->p_dev = link;
-	link->priv = priv;
-
-	ret = spectrum_cs_config(link);
-	if (ret)
-		goto err_free_orinocodev;
-
-	return 0;
-
-err_free_orinocodev:
-	free_orinocodev(priv);
-	return ret;
-}
-
-static void spectrum_cs_detach(struct pcmcia_device *link)
-{
-	struct orinoco_private *priv = link->priv;
-
-	orinoco_if_del(priv);
-
-	spectrum_cs_release(link);
-
-	free_orinocodev(priv);
-}				/* spectrum_cs_detach */
-
-static int spectrum_cs_config_check(struct pcmcia_device *p_dev,
-				    void *priv_data)
-{
-	if (p_dev->config_index == 0)
-		return -EINVAL;
-
-	return pcmcia_request_io(p_dev);
-};
-
-static int
-spectrum_cs_config(struct pcmcia_device *link)
-{
-	struct orinoco_private *priv = link->priv;
-	struct hermes *hw = &priv->hw;
-	int ret;
-	void __iomem *mem;
-
-	link->config_flags |= CONF_AUTO_SET_VPP | CONF_AUTO_CHECK_VCC |
-		CONF_AUTO_SET_IO | CONF_ENABLE_IRQ;
-	if (ignore_cis_vcc)
-		link->config_flags &= ~CONF_AUTO_CHECK_VCC;
-	ret = pcmcia_loop_config(link, spectrum_cs_config_check, NULL);
-	if (ret) {
-		if (!ignore_cis_vcc)
-			printk(KERN_ERR PFX "GetNextTuple(): No matching "
-			       "CIS configuration.  Maybe you need the "
-			       "ignore_cis_vcc=1 parameter.\n");
-		goto failed;
-	}
-
-	mem = ioport_map(link->resource[0]->start,
-			resource_size(link->resource[0]));
-	if (!mem)
-		goto failed;
-
-	/* We initialize the hermes structure before completing PCMCIA
-	 * configuration just in case the interrupt handler gets
-	 * called. */
-	hermes_struct_init(hw, mem, HERMES_16BIT_REGSPACING);
-	hw->eeprom_pda = true;
-
-	ret = pcmcia_request_irq(link, orinoco_interrupt);
-	if (ret)
-		goto failed;
-
-	ret = pcmcia_enable_device(link);
-	if (ret)
-		goto failed;
-
-	/* Reset card */
-	if (spectrum_cs_hard_reset(priv) != 0)
-		goto failed;
-
-	/* Initialise the main driver */
-	if (orinoco_init(priv) != 0) {
-		printk(KERN_ERR PFX "orinoco_init() failed\n");
-		goto failed;
-	}
-
-	/* Register an interface with the stack */
-	if (orinoco_if_add(priv, link->resource[0]->start,
-			   link->irq, NULL) != 0) {
-		printk(KERN_ERR PFX "orinoco_if_add() failed\n");
-		goto failed;
-	}
-
-	return 0;
-
- failed:
-	spectrum_cs_release(link);
-	return -ENODEV;
-}				/* spectrum_cs_config */
-
-static void
-spectrum_cs_release(struct pcmcia_device *link)
-{
-	struct orinoco_private *priv = link->priv;
-	unsigned long flags;
-
-	/* We're committed to taking the device away now, so mark the
-	 * hardware as unavailable */
-	priv->hw.ops->lock_irqsave(&priv->lock, &flags);
-	priv->hw_unavailable++;
-	priv->hw.ops->unlock_irqrestore(&priv->lock, &flags);
-
-	pcmcia_disable_device(link);
-	if (priv->hw.iobase)
-		ioport_unmap(priv->hw.iobase);
-}				/* spectrum_cs_release */
-
-
-static int
-spectrum_cs_suspend(struct pcmcia_device *link)
-{
-	struct orinoco_private *priv = link->priv;
-
-	/* Mark the device as stopped, to block IO until later */
-	orinoco_down(priv);
-
-	return 0;
-}
-
-static int
-spectrum_cs_resume(struct pcmcia_device *link)
-{
-	struct orinoco_private *priv = link->priv;
-	int err = orinoco_up(priv);
-
-	return err;
-}
-
-
-/********************************************************************/
-/* Module initialization					    */
-/********************************************************************/
-
-static const struct pcmcia_device_id spectrum_cs_ids[] = {
-	PCMCIA_DEVICE_MANF_CARD(0x026c, 0x0001), /* Symbol Spectrum24 LA4137 */
-	PCMCIA_DEVICE_MANF_CARD(0x0104, 0x0001), /* Socket Communications CF */
-	PCMCIA_DEVICE_PROD_ID12("Intel", "PRO/Wireless LAN PC Card", 0x816cc815, 0x6fbf459a), /* 2011B, not 2011 */
-	PCMCIA_DEVICE_NULL,
-};
-MODULE_DEVICE_TABLE(pcmcia, spectrum_cs_ids);
-
-static struct pcmcia_driver orinoco_driver = {
-	.owner		= THIS_MODULE,
-	.name		= DRIVER_NAME,
-	.probe		= spectrum_cs_probe,
-	.remove		= spectrum_cs_detach,
-	.suspend	= spectrum_cs_suspend,
-	.resume		= spectrum_cs_resume,
-	.id_table       = spectrum_cs_ids,
-};
-module_pcmcia_driver(orinoco_driver);
diff --git a/drivers/net/wireless/intersil/orinoco/wext.c b/drivers/net/wireless/intersil/orinoco/wext.c
deleted file mode 100644
index dea1ff044342..000000000000
--- a/drivers/net/wireless/intersil/orinoco/wext.c
+++ /dev/null
@@ -1,1428 +0,0 @@
-/* Wireless extensions support.
- *
- * See copyright notice in main.c
- */
-#include <linux/slab.h>
-#include <linux/kernel.h>
-#include <linux/if_arp.h>
-#include <linux/wireless.h>
-#include <linux/ieee80211.h>
-#include <linux/etherdevice.h>
-#include <net/iw_handler.h>
-#include <net/cfg80211.h>
-#include <net/cfg80211-wext.h>
-
-#include "hermes.h"
-#include "hermes_rid.h"
-#include "orinoco.h"
-
-#include "hw.h"
-#include "mic.h"
-#include "scan.h"
-#include "main.h"
-
-#include "wext.h"
-
-#define MAX_RID_LEN 1024
-
-/* Helper routine to record keys
- * It is called under orinoco_lock so it may not sleep */
-static int orinoco_set_key(struct orinoco_private *priv, int index,
-			   enum orinoco_alg alg, const u8 *key, int key_len,
-			   const u8 *seq, int seq_len)
-{
-	kfree_sensitive(priv->keys[index].key);
-	kfree_sensitive(priv->keys[index].seq);
-
-	if (key_len) {
-		priv->keys[index].key = kzalloc(key_len, GFP_ATOMIC);
-		if (!priv->keys[index].key)
-			goto nomem;
-	} else
-		priv->keys[index].key = NULL;
-
-	if (seq_len) {
-		priv->keys[index].seq = kzalloc(seq_len, GFP_ATOMIC);
-		if (!priv->keys[index].seq)
-			goto free_key;
-	} else
-		priv->keys[index].seq = NULL;
-
-	priv->keys[index].key_len = key_len;
-	priv->keys[index].seq_len = seq_len;
-
-	if (key_len)
-		memcpy((void *)priv->keys[index].key, key, key_len);
-	if (seq_len)
-		memcpy((void *)priv->keys[index].seq, seq, seq_len);
-
-	switch (alg) {
-	case ORINOCO_ALG_TKIP:
-		priv->keys[index].cipher = WLAN_CIPHER_SUITE_TKIP;
-		break;
-
-	case ORINOCO_ALG_WEP:
-		priv->keys[index].cipher = (key_len > SMALL_KEY_SIZE) ?
-			WLAN_CIPHER_SUITE_WEP104 : WLAN_CIPHER_SUITE_WEP40;
-		break;
-
-	case ORINOCO_ALG_NONE:
-	default:
-		priv->keys[index].cipher = 0;
-		break;
-	}
-
-	return 0;
-
-free_key:
-	kfree(priv->keys[index].key);
-	priv->keys[index].key = NULL;
-
-nomem:
-	priv->keys[index].key_len = 0;
-	priv->keys[index].seq_len = 0;
-	priv->keys[index].cipher = 0;
-
-	return -ENOMEM;
-}
-
-static struct iw_statistics *orinoco_get_wireless_stats(struct net_device *dev)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	struct hermes *hw = &priv->hw;
-	struct iw_statistics *wstats = &priv->wstats;
-	int err;
-	unsigned long flags;
-
-	if (!netif_device_present(dev)) {
-		printk(KERN_WARNING "%s: get_wireless_stats() called while device not present\n",
-		       dev->name);
-		return NULL; /* FIXME: Can we do better than this? */
-	}
-
-	/* If busy, return the old stats.  Returning NULL may cause
-	 * the interface to disappear from /proc/net/wireless */
-	if (orinoco_lock(priv, &flags) != 0)
-		return wstats;
-
-	/* We can't really wait for the tallies inquiry command to
-	 * complete, so we just use the previous results and trigger
-	 * a new tallies inquiry command for next time - Jean II */
-	/* FIXME: Really we should wait for the inquiry to come back -
-	 * as it is the stats we give don't make a whole lot of sense.
-	 * Unfortunately, it's not clear how to do that within the
-	 * wireless extensions framework: I think we're in user
-	 * context, but a lock seems to be held by the time we get in
-	 * here so we're not safe to sleep here. */
-	hermes_inquire(hw, HERMES_INQ_TALLIES);
-
-	if (priv->iw_mode == NL80211_IFTYPE_ADHOC) {
-		memset(&wstats->qual, 0, sizeof(wstats->qual));
-		/* If a spy address is defined, we report stats of the
-		 * first spy address - Jean II */
-		if (SPY_NUMBER(priv)) {
-			wstats->qual.qual = priv->spy_data.spy_stat[0].qual;
-			wstats->qual.level = priv->spy_data.spy_stat[0].level;
-			wstats->qual.noise = priv->spy_data.spy_stat[0].noise;
-			wstats->qual.updated =
-				priv->spy_data.spy_stat[0].updated;
-		}
-	} else {
-		struct {
-			__le16 qual, signal, noise, unused;
-		} __packed cq;
-
-		err = HERMES_READ_RECORD(hw, USER_BAP,
-					 HERMES_RID_COMMSQUALITY, &cq);
-
-		if (!err) {
-			wstats->qual.qual = (int)le16_to_cpu(cq.qual);
-			wstats->qual.level = (int)le16_to_cpu(cq.signal) - 0x95;
-			wstats->qual.noise = (int)le16_to_cpu(cq.noise) - 0x95;
-			wstats->qual.updated =
-				IW_QUAL_ALL_UPDATED | IW_QUAL_DBM;
-		}
-	}
-
-	orinoco_unlock(priv, &flags);
-	return wstats;
-}
-
-/********************************************************************/
-/* Wireless extensions                                              */
-/********************************************************************/
-
-static int orinoco_ioctl_setwap(struct net_device *dev,
-				struct iw_request_info *info,
-				union iwreq_data *wrqu,
-				char *extra)
-{
-	struct sockaddr *ap_addr = &wrqu->ap_addr;
-	struct orinoco_private *priv = ndev_priv(dev);
-	int err = -EINPROGRESS;		/* Call commit handler */
-	unsigned long flags;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	/* Enable automatic roaming - no sanity checks are needed */
-	if (is_zero_ether_addr(ap_addr->sa_data) ||
-	    is_broadcast_ether_addr(ap_addr->sa_data)) {
-		priv->bssid_fixed = 0;
-		eth_zero_addr(priv->desired_bssid);
-
-		/* "off" means keep existing connection */
-		if (ap_addr->sa_data[0] == 0) {
-			__orinoco_hw_set_wap(priv);
-			err = 0;
-		}
-		goto out;
-	}
-
-	if (priv->firmware_type == FIRMWARE_TYPE_AGERE) {
-		printk(KERN_WARNING "%s: Lucent/Agere firmware doesn't "
-		       "support manual roaming\n",
-		       dev->name);
-		err = -EOPNOTSUPP;
-		goto out;
-	}
-
-	if (priv->iw_mode != NL80211_IFTYPE_STATION) {
-		printk(KERN_WARNING "%s: Manual roaming supported only in "
-		       "managed mode\n", dev->name);
-		err = -EOPNOTSUPP;
-		goto out;
-	}
-
-	/* Intersil firmware hangs without Desired ESSID */
-	if (priv->firmware_type == FIRMWARE_TYPE_INTERSIL &&
-	    strlen(priv->desired_essid) == 0) {
-		printk(KERN_WARNING "%s: Desired ESSID must be set for "
-		       "manual roaming\n", dev->name);
-		err = -EOPNOTSUPP;
-		goto out;
-	}
-
-	/* Finally, enable manual roaming */
-	priv->bssid_fixed = 1;
-	memcpy(priv->desired_bssid, &ap_addr->sa_data, ETH_ALEN);
-
- out:
-	orinoco_unlock(priv, &flags);
-	return err;
-}
-
-static int orinoco_ioctl_getwap(struct net_device *dev,
-				struct iw_request_info *info,
-				union iwreq_data *wrqu,
-				char *extra)
-{
-	struct sockaddr *ap_addr = &wrqu->ap_addr;
-	struct orinoco_private *priv = ndev_priv(dev);
-
-	int err = 0;
-	unsigned long flags;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	ap_addr->sa_family = ARPHRD_ETHER;
-	err = orinoco_hw_get_current_bssid(priv, ap_addr->sa_data);
-
-	orinoco_unlock(priv, &flags);
-
-	return err;
-}
-
-static int orinoco_ioctl_setiwencode(struct net_device *dev,
-				     struct iw_request_info *info,
-				     union iwreq_data *wrqu,
-				     char *keybuf)
-{
-	struct iw_point *erq = &wrqu->encoding;
-	struct orinoco_private *priv = ndev_priv(dev);
-	int index = (erq->flags & IW_ENCODE_INDEX) - 1;
-	int setindex = priv->tx_key;
-	enum orinoco_alg encode_alg = priv->encode_alg;
-	int restricted = priv->wep_restrict;
-	int err = -EINPROGRESS;		/* Call commit handler */
-	unsigned long flags;
-
-	if (!priv->has_wep)
-		return -EOPNOTSUPP;
-
-	if (erq->pointer) {
-		/* We actually have a key to set - check its length */
-		if (erq->length > LARGE_KEY_SIZE)
-			return -E2BIG;
-
-		if ((erq->length > SMALL_KEY_SIZE) && !priv->has_big_wep)
-			return -E2BIG;
-	}
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	/* Clear any TKIP key we have */
-	if ((priv->has_wpa) && (priv->encode_alg == ORINOCO_ALG_TKIP))
-		(void) orinoco_clear_tkip_key(priv, setindex);
-
-	if (erq->length > 0) {
-		if ((index < 0) || (index >= ORINOCO_MAX_KEYS))
-			index = priv->tx_key;
-
-		/* Switch on WEP if off */
-		if (encode_alg != ORINOCO_ALG_WEP) {
-			setindex = index;
-			encode_alg = ORINOCO_ALG_WEP;
-		}
-	} else {
-		/* Important note : if the user do "iwconfig eth0 enc off",
-		 * we will arrive there with an index of -1. This is valid
-		 * but need to be taken care off... Jean II */
-		if ((index < 0) || (index >= ORINOCO_MAX_KEYS)) {
-			if ((index != -1) || (erq->flags == 0)) {
-				err = -EINVAL;
-				goto out;
-			}
-		} else {
-			/* Set the index : Check that the key is valid */
-			if (priv->keys[index].key_len == 0) {
-				err = -EINVAL;
-				goto out;
-			}
-			setindex = index;
-		}
-	}
-
-	if (erq->flags & IW_ENCODE_DISABLED)
-		encode_alg = ORINOCO_ALG_NONE;
-	if (erq->flags & IW_ENCODE_OPEN)
-		restricted = 0;
-	if (erq->flags & IW_ENCODE_RESTRICTED)
-		restricted = 1;
-
-	if (erq->pointer && erq->length > 0) {
-		err = orinoco_set_key(priv, index, ORINOCO_ALG_WEP, keybuf,
-				      erq->length, NULL, 0);
-	}
-	priv->tx_key = setindex;
-
-	/* Try fast key change if connected and only keys are changed */
-	if ((priv->encode_alg == encode_alg) &&
-	    (priv->wep_restrict == restricted) &&
-	    netif_carrier_ok(dev)) {
-		err = __orinoco_hw_setup_wepkeys(priv);
-		/* No need to commit if successful */
-		goto out;
-	}
-
-	priv->encode_alg = encode_alg;
-	priv->wep_restrict = restricted;
-
- out:
-	orinoco_unlock(priv, &flags);
-
-	return err;
-}
-
-static int orinoco_ioctl_getiwencode(struct net_device *dev,
-				     struct iw_request_info *info,
-				     union iwreq_data *wrqu,
-				     char *keybuf)
-{
-	struct iw_point *erq = &wrqu->encoding;
-	struct orinoco_private *priv = ndev_priv(dev);
-	int index = (erq->flags & IW_ENCODE_INDEX) - 1;
-	unsigned long flags;
-
-	if (!priv->has_wep)
-		return -EOPNOTSUPP;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	if ((index < 0) || (index >= ORINOCO_MAX_KEYS))
-		index = priv->tx_key;
-
-	erq->flags = 0;
-	if (!priv->encode_alg)
-		erq->flags |= IW_ENCODE_DISABLED;
-	erq->flags |= index + 1;
-
-	if (priv->wep_restrict)
-		erq->flags |= IW_ENCODE_RESTRICTED;
-	else
-		erq->flags |= IW_ENCODE_OPEN;
-
-	erq->length = priv->keys[index].key_len;
-
-	memcpy(keybuf, priv->keys[index].key, erq->length);
-
-	orinoco_unlock(priv, &flags);
-	return 0;
-}
-
-static int orinoco_ioctl_setessid(struct net_device *dev,
-				  struct iw_request_info *info,
-				  union iwreq_data *wrqu,
-				  char *essidbuf)
-{
-	struct iw_point *erq = &wrqu->essid;
-	struct orinoco_private *priv = ndev_priv(dev);
-	unsigned long flags;
-
-	/* Note : ESSID is ignored in Ad-Hoc demo mode, but we can set it
-	 * anyway... - Jean II */
-
-	/* Hum... Should not use Wireless Extension constant (may change),
-	 * should use our own... - Jean II */
-	if (erq->length > IW_ESSID_MAX_SIZE)
-		return -E2BIG;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	/* NULL the string (for NULL termination & ESSID = ANY) - Jean II */
-	memset(priv->desired_essid, 0, sizeof(priv->desired_essid));
-
-	/* If not ANY, get the new ESSID */
-	if (erq->flags)
-		memcpy(priv->desired_essid, essidbuf, erq->length);
-
-	orinoco_unlock(priv, &flags);
-
-	return -EINPROGRESS;		/* Call commit handler */
-}
-
-static int orinoco_ioctl_getessid(struct net_device *dev,
-				  struct iw_request_info *info,
-				  union iwreq_data *wrqu,
-				  char *essidbuf)
-{
-	struct iw_point *erq = &wrqu->essid;
-	struct orinoco_private *priv = ndev_priv(dev);
-	int active;
-	int err = 0;
-	unsigned long flags;
-
-	if (netif_running(dev)) {
-		err = orinoco_hw_get_essid(priv, &active, essidbuf);
-		if (err < 0)
-			return err;
-		erq->length = err;
-	} else {
-		if (orinoco_lock(priv, &flags) != 0)
-			return -EBUSY;
-		memcpy(essidbuf, priv->desired_essid, IW_ESSID_MAX_SIZE);
-		erq->length = strlen(priv->desired_essid);
-		orinoco_unlock(priv, &flags);
-	}
-
-	erq->flags = 1;
-
-	return 0;
-}
-
-static int orinoco_ioctl_setfreq(struct net_device *dev,
-				 struct iw_request_info *info,
-				 union iwreq_data *wrqu,
-				 char *extra)
-{
-	struct iw_freq *frq = &wrqu->freq;
-	struct orinoco_private *priv = ndev_priv(dev);
-	int chan = -1;
-	unsigned long flags;
-	int err = -EINPROGRESS;		/* Call commit handler */
-
-	/* In infrastructure mode the AP sets the channel */
-	if (priv->iw_mode == NL80211_IFTYPE_STATION)
-		return -EBUSY;
-
-	if ((frq->e == 0) && (frq->m <= 1000)) {
-		/* Setting by channel number */
-		chan = frq->m;
-	} else {
-		/* Setting by frequency */
-		int denom = 1;
-		int i;
-
-		/* Calculate denominator to rescale to MHz */
-		for (i = 0; i < (6 - frq->e); i++)
-			denom *= 10;
-
-		chan = ieee80211_frequency_to_channel(frq->m / denom);
-	}
-
-	if ((chan < 1) || (chan > NUM_CHANNELS) ||
-	     !(priv->channel_mask & (1 << (chan - 1))))
-		return -EINVAL;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	priv->channel = chan;
-	if (priv->iw_mode == NL80211_IFTYPE_MONITOR) {
-		/* Fast channel change - no commit if successful */
-		struct hermes *hw = &priv->hw;
-		err = hw->ops->cmd_wait(hw, HERMES_CMD_TEST |
-					    HERMES_TEST_SET_CHANNEL,
-					chan, NULL);
-	}
-	orinoco_unlock(priv, &flags);
-
-	return err;
-}
-
-static int orinoco_ioctl_getfreq(struct net_device *dev,
-				 struct iw_request_info *info,
-				 union iwreq_data *wrqu,
-				 char *extra)
-{
-	struct iw_freq *frq = &wrqu->freq;
-	struct orinoco_private *priv = ndev_priv(dev);
-	int tmp;
-
-	/* Locking done in there */
-	tmp = orinoco_hw_get_freq(priv);
-	if (tmp < 0)
-		return tmp;
-
-	frq->m = tmp * 100000;
-	frq->e = 1;
-
-	return 0;
-}
-
-static int orinoco_ioctl_getsens(struct net_device *dev,
-				 struct iw_request_info *info,
-				 union iwreq_data *wrqu,
-				 char *extra)
-{
-	struct iw_param *srq = &wrqu->sens;
-	struct orinoco_private *priv = ndev_priv(dev);
-	struct hermes *hw = &priv->hw;
-	u16 val;
-	int err;
-	unsigned long flags;
-
-	if (!priv->has_sensitivity)
-		return -EOPNOTSUPP;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-	err = hermes_read_wordrec(hw, USER_BAP,
-				  HERMES_RID_CNFSYSTEMSCALE, &val);
-	orinoco_unlock(priv, &flags);
-
-	if (err)
-		return err;
-
-	srq->value = val;
-	srq->fixed = 0; /* auto */
-
-	return 0;
-}
-
-static int orinoco_ioctl_setsens(struct net_device *dev,
-				 struct iw_request_info *info,
-				 union iwreq_data *wrqu,
-				 char *extra)
-{
-	struct iw_param *srq = &wrqu->sens;
-	struct orinoco_private *priv = ndev_priv(dev);
-	int val = srq->value;
-	unsigned long flags;
-
-	if (!priv->has_sensitivity)
-		return -EOPNOTSUPP;
-
-	if ((val < 1) || (val > 3))
-		return -EINVAL;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-	priv->ap_density = val;
-	orinoco_unlock(priv, &flags);
-
-	return -EINPROGRESS;		/* Call commit handler */
-}
-
-static int orinoco_ioctl_setrate(struct net_device *dev,
-				 struct iw_request_info *info,
-				 union iwreq_data *wrqu,
-				 char *extra)
-{
-	struct iw_param *rrq = &wrqu->bitrate;
-	struct orinoco_private *priv = ndev_priv(dev);
-	int ratemode;
-	int bitrate; /* 100s of kilobits */
-	unsigned long flags;
-
-	/* As the user space doesn't know our highest rate, it uses -1
-	 * to ask us to set the highest rate.  Test it using "iwconfig
-	 * ethX rate auto" - Jean II */
-	if (rrq->value == -1)
-		bitrate = 110;
-	else {
-		if (rrq->value % 100000)
-			return -EINVAL;
-		bitrate = rrq->value / 100000;
-	}
-
-	ratemode = orinoco_get_bitratemode(bitrate, !rrq->fixed);
-
-	if (ratemode == -1)
-		return -EINVAL;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-	priv->bitratemode = ratemode;
-	orinoco_unlock(priv, &flags);
-
-	return -EINPROGRESS;
-}
-
-static int orinoco_ioctl_getrate(struct net_device *dev,
-				 struct iw_request_info *info,
-				 union iwreq_data *wrqu,
-				 char *extra)
-{
-	struct iw_param *rrq = &wrqu->bitrate;
-	struct orinoco_private *priv = ndev_priv(dev);
-	int err = 0;
-	int bitrate, automatic;
-	unsigned long flags;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	orinoco_get_ratemode_cfg(priv->bitratemode, &bitrate, &automatic);
-
-	/* If the interface is running we try to find more about the
-	   current mode */
-	if (netif_running(dev)) {
-		int act_bitrate;
-		int lerr;
-
-		/* Ignore errors if we can't get the actual bitrate */
-		lerr = orinoco_hw_get_act_bitrate(priv, &act_bitrate);
-		if (!lerr)
-			bitrate = act_bitrate;
-	}
-
-	orinoco_unlock(priv, &flags);
-
-	rrq->value = bitrate;
-	rrq->fixed = !automatic;
-	rrq->disabled = 0;
-
-	return err;
-}
-
-static int orinoco_ioctl_setpower(struct net_device *dev,
-				  struct iw_request_info *info,
-				  union iwreq_data *wrqu,
-				  char *extra)
-{
-	struct iw_param *prq = &wrqu->power;
-	struct orinoco_private *priv = ndev_priv(dev);
-	int err = -EINPROGRESS;		/* Call commit handler */
-	unsigned long flags;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	if (prq->disabled) {
-		priv->pm_on = 0;
-	} else {
-		switch (prq->flags & IW_POWER_MODE) {
-		case IW_POWER_UNICAST_R:
-			priv->pm_mcast = 0;
-			priv->pm_on = 1;
-			break;
-		case IW_POWER_ALL_R:
-			priv->pm_mcast = 1;
-			priv->pm_on = 1;
-			break;
-		case IW_POWER_ON:
-			/* No flags : but we may have a value - Jean II */
-			break;
-		default:
-			err = -EINVAL;
-			goto out;
-		}
-
-		if (prq->flags & IW_POWER_TIMEOUT) {
-			priv->pm_on = 1;
-			priv->pm_timeout = prq->value / 1000;
-		}
-		if (prq->flags & IW_POWER_PERIOD) {
-			priv->pm_on = 1;
-			priv->pm_period = prq->value / 1000;
-		}
-		/* It's valid to not have a value if we are just toggling
-		 * the flags... Jean II */
-		if (!priv->pm_on) {
-			err = -EINVAL;
-			goto out;
-		}
-	}
-
- out:
-	orinoco_unlock(priv, &flags);
-
-	return err;
-}
-
-static int orinoco_ioctl_getpower(struct net_device *dev,
-				  struct iw_request_info *info,
-				  union iwreq_data *wrqu,
-				  char *extra)
-{
-	struct iw_param *prq = &wrqu->power;
-	struct orinoco_private *priv = ndev_priv(dev);
-	struct hermes *hw = &priv->hw;
-	int err = 0;
-	u16 enable, period, timeout, mcast;
-	unsigned long flags;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	err = hermes_read_wordrec(hw, USER_BAP,
-				  HERMES_RID_CNFPMENABLED, &enable);
-	if (err)
-		goto out;
-
-	err = hermes_read_wordrec(hw, USER_BAP,
-				  HERMES_RID_CNFMAXSLEEPDURATION, &period);
-	if (err)
-		goto out;
-
-	err = hermes_read_wordrec(hw, USER_BAP,
-				  HERMES_RID_CNFPMHOLDOVERDURATION, &timeout);
-	if (err)
-		goto out;
-
-	err = hermes_read_wordrec(hw, USER_BAP,
-				  HERMES_RID_CNFMULTICASTRECEIVE, &mcast);
-	if (err)
-		goto out;
-
-	prq->disabled = !enable;
-	/* Note : by default, display the period */
-	if ((prq->flags & IW_POWER_TYPE) == IW_POWER_TIMEOUT) {
-		prq->flags = IW_POWER_TIMEOUT;
-		prq->value = timeout * 1000;
-	} else {
-		prq->flags = IW_POWER_PERIOD;
-		prq->value = period * 1000;
-	}
-	if (mcast)
-		prq->flags |= IW_POWER_ALL_R;
-	else
-		prq->flags |= IW_POWER_UNICAST_R;
-
- out:
-	orinoco_unlock(priv, &flags);
-
-	return err;
-}
-
-static int orinoco_ioctl_set_encodeext(struct net_device *dev,
-				       struct iw_request_info *info,
-				       union iwreq_data *wrqu,
-				       char *extra)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	struct iw_point *encoding = &wrqu->encoding;
-	struct iw_encode_ext *ext = (struct iw_encode_ext *)extra;
-	int idx, alg = ext->alg, set_key = 1;
-	unsigned long flags;
-	int err = -EINVAL;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	/* Determine and validate the key index */
-	idx = encoding->flags & IW_ENCODE_INDEX;
-	if (idx) {
-		if ((idx < 1) || (idx > 4))
-			goto out;
-		idx--;
-	} else
-		idx = priv->tx_key;
-
-	if (encoding->flags & IW_ENCODE_DISABLED)
-		alg = IW_ENCODE_ALG_NONE;
-
-	if (priv->has_wpa && (alg != IW_ENCODE_ALG_TKIP)) {
-		/* Clear any TKIP TX key we had */
-		(void) orinoco_clear_tkip_key(priv, priv->tx_key);
-	}
-
-	if (ext->ext_flags & IW_ENCODE_EXT_SET_TX_KEY) {
-		priv->tx_key = idx;
-		set_key = ((alg == IW_ENCODE_ALG_TKIP) ||
-			   (ext->key_len > 0)) ? 1 : 0;
-	}
-
-	if (set_key) {
-		/* Set the requested key first */
-		switch (alg) {
-		case IW_ENCODE_ALG_NONE:
-			priv->encode_alg = ORINOCO_ALG_NONE;
-			err = orinoco_set_key(priv, idx, ORINOCO_ALG_NONE,
-					      NULL, 0, NULL, 0);
-			break;
-
-		case IW_ENCODE_ALG_WEP:
-			if (ext->key_len <= 0)
-				goto out;
-
-			priv->encode_alg = ORINOCO_ALG_WEP;
-			err = orinoco_set_key(priv, idx, ORINOCO_ALG_WEP,
-					      ext->key, ext->key_len, NULL, 0);
-			break;
-
-		case IW_ENCODE_ALG_TKIP:
-		{
-			u8 *tkip_iv = NULL;
-
-			if (!priv->has_wpa ||
-			    (ext->key_len > sizeof(struct orinoco_tkip_key)))
-				goto out;
-
-			priv->encode_alg = ORINOCO_ALG_TKIP;
-
-			if (ext->ext_flags & IW_ENCODE_EXT_RX_SEQ_VALID)
-				tkip_iv = &ext->rx_seq[0];
-
-			err = orinoco_set_key(priv, idx, ORINOCO_ALG_TKIP,
-					      ext->key, ext->key_len, tkip_iv,
-					      ORINOCO_SEQ_LEN);
-
-			err = __orinoco_hw_set_tkip_key(priv, idx,
-				 ext->ext_flags & IW_ENCODE_EXT_SET_TX_KEY,
-				 priv->keys[idx].key, priv->keys[idx].key_len,
-				 tkip_iv, ORINOCO_SEQ_LEN, NULL, 0);
-			if (err)
-				printk(KERN_ERR "%s: Error %d setting TKIP key"
-				       "\n", dev->name, err);
-
-			goto out;
-		}
-		default:
-			goto out;
-		}
-	}
-	err = -EINPROGRESS;
- out:
-	orinoco_unlock(priv, &flags);
-
-	return err;
-}
-
-static int orinoco_ioctl_get_encodeext(struct net_device *dev,
-				       struct iw_request_info *info,
-				       union iwreq_data *wrqu,
-				       char *extra)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	struct iw_point *encoding = &wrqu->encoding;
-	struct iw_encode_ext *ext = (struct iw_encode_ext *)extra;
-	int idx, max_key_len;
-	unsigned long flags;
-	int err;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	err = -EINVAL;
-	max_key_len = encoding->length - sizeof(*ext);
-	if (max_key_len < 0)
-		goto out;
-
-	idx = encoding->flags & IW_ENCODE_INDEX;
-	if (idx) {
-		if ((idx < 1) || (idx > 4))
-			goto out;
-		idx--;
-	} else
-		idx = priv->tx_key;
-
-	encoding->flags = idx + 1;
-	memset(ext, 0, sizeof(*ext));
-
-	switch (priv->encode_alg) {
-	case ORINOCO_ALG_NONE:
-		ext->alg = IW_ENCODE_ALG_NONE;
-		ext->key_len = 0;
-		encoding->flags |= IW_ENCODE_DISABLED;
-		break;
-	case ORINOCO_ALG_WEP:
-		ext->alg = IW_ENCODE_ALG_WEP;
-		ext->key_len = min(priv->keys[idx].key_len, max_key_len);
-		memcpy(ext->key, priv->keys[idx].key, ext->key_len);
-		encoding->flags |= IW_ENCODE_ENABLED;
-		break;
-	case ORINOCO_ALG_TKIP:
-		ext->alg = IW_ENCODE_ALG_TKIP;
-		ext->key_len = min(priv->keys[idx].key_len, max_key_len);
-		memcpy(ext->key, priv->keys[idx].key, ext->key_len);
-		encoding->flags |= IW_ENCODE_ENABLED;
-		break;
-	}
-
-	err = 0;
- out:
-	orinoco_unlock(priv, &flags);
-
-	return err;
-}
-
-static int orinoco_ioctl_set_auth(struct net_device *dev,
-				  struct iw_request_info *info,
-				  union iwreq_data *wrqu, char *extra)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	struct hermes *hw = &priv->hw;
-	struct iw_param *param = &wrqu->param;
-	unsigned long flags;
-	int ret = -EINPROGRESS;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	switch (param->flags & IW_AUTH_INDEX) {
-	case IW_AUTH_WPA_VERSION:
-	case IW_AUTH_CIPHER_PAIRWISE:
-	case IW_AUTH_CIPHER_GROUP:
-	case IW_AUTH_RX_UNENCRYPTED_EAPOL:
-	case IW_AUTH_PRIVACY_INVOKED:
-	case IW_AUTH_DROP_UNENCRYPTED:
-		/*
-		 * orinoco does not use these parameters
-		 */
-		break;
-
-	case IW_AUTH_MFP:
-		/* Management Frame Protection not supported.
-		 * Only fail if set to required.
-		 */
-		if (param->value == IW_AUTH_MFP_REQUIRED)
-			ret = -EINVAL;
-		break;
-
-	case IW_AUTH_KEY_MGMT:
-		/* wl_lkm implies value 2 == PSK for Hermes I
-		 * which ties in with WEXT
-		 * no other hints tho :(
-		 */
-		priv->key_mgmt = param->value;
-		break;
-
-	case IW_AUTH_TKIP_COUNTERMEASURES:
-		/* When countermeasures are enabled, shut down the
-		 * card; when disabled, re-enable the card. This must
-		 * take effect immediately.
-		 *
-		 * TODO: Make sure that the EAPOL message is getting
-		 *       out before card disabled
-		 */
-		if (param->value) {
-			priv->tkip_cm_active = 1;
-			ret = hermes_disable_port(hw, 0);
-		} else {
-			priv->tkip_cm_active = 0;
-			ret = hermes_enable_port(hw, 0);
-		}
-		break;
-
-	case IW_AUTH_80211_AUTH_ALG:
-		if (param->value & IW_AUTH_ALG_SHARED_KEY)
-			priv->wep_restrict = 1;
-		else if (param->value & IW_AUTH_ALG_OPEN_SYSTEM)
-			priv->wep_restrict = 0;
-		else
-			ret = -EINVAL;
-		break;
-
-	case IW_AUTH_WPA_ENABLED:
-		if (priv->has_wpa) {
-			priv->wpa_enabled = param->value ? 1 : 0;
-		} else {
-			if (param->value)
-				ret = -EOPNOTSUPP;
-			/* else silently accept disable of WPA */
-			priv->wpa_enabled = 0;
-		}
-		break;
-
-	default:
-		ret = -EOPNOTSUPP;
-	}
-
-	orinoco_unlock(priv, &flags);
-	return ret;
-}
-
-static int orinoco_ioctl_get_auth(struct net_device *dev,
-				  struct iw_request_info *info,
-				  union iwreq_data *wrqu, char *extra)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	struct iw_param *param = &wrqu->param;
-	unsigned long flags;
-	int ret = 0;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	switch (param->flags & IW_AUTH_INDEX) {
-	case IW_AUTH_KEY_MGMT:
-		param->value = priv->key_mgmt;
-		break;
-
-	case IW_AUTH_TKIP_COUNTERMEASURES:
-		param->value = priv->tkip_cm_active;
-		break;
-
-	case IW_AUTH_80211_AUTH_ALG:
-		if (priv->wep_restrict)
-			param->value = IW_AUTH_ALG_SHARED_KEY;
-		else
-			param->value = IW_AUTH_ALG_OPEN_SYSTEM;
-		break;
-
-	case IW_AUTH_WPA_ENABLED:
-		param->value = priv->wpa_enabled;
-		break;
-
-	default:
-		ret = -EOPNOTSUPP;
-	}
-
-	orinoco_unlock(priv, &flags);
-	return ret;
-}
-
-static int orinoco_ioctl_set_genie(struct net_device *dev,
-				   struct iw_request_info *info,
-				   union iwreq_data *wrqu, char *extra)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	u8 *buf;
-	unsigned long flags;
-
-	/* cut off at IEEE80211_MAX_DATA_LEN */
-	if ((wrqu->data.length > IEEE80211_MAX_DATA_LEN) ||
-	    (wrqu->data.length && (extra == NULL)))
-		return -EINVAL;
-
-	if (wrqu->data.length) {
-		buf = kmemdup(extra, wrqu->data.length, GFP_KERNEL);
-		if (buf == NULL)
-			return -ENOMEM;
-	} else
-		buf = NULL;
-
-	if (orinoco_lock(priv, &flags) != 0) {
-		kfree(buf);
-		return -EBUSY;
-	}
-
-	kfree(priv->wpa_ie);
-	priv->wpa_ie = buf;
-	priv->wpa_ie_len = wrqu->data.length;
-
-	if (priv->wpa_ie) {
-		/* Looks like wl_lkm wants to check the auth alg, and
-		 * somehow pass it to the firmware.
-		 * Instead it just calls the key mgmt rid
-		 *   - we do this in set auth.
-		 */
-	}
-
-	orinoco_unlock(priv, &flags);
-	return 0;
-}
-
-static int orinoco_ioctl_get_genie(struct net_device *dev,
-				   struct iw_request_info *info,
-				   union iwreq_data *wrqu, char *extra)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	unsigned long flags;
-	int err = 0;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	if ((priv->wpa_ie_len == 0) || (priv->wpa_ie == NULL)) {
-		wrqu->data.length = 0;
-		goto out;
-	}
-
-	if (wrqu->data.length < priv->wpa_ie_len) {
-		err = -E2BIG;
-		goto out;
-	}
-
-	wrqu->data.length = priv->wpa_ie_len;
-	memcpy(extra, priv->wpa_ie, priv->wpa_ie_len);
-
-out:
-	orinoco_unlock(priv, &flags);
-	return err;
-}
-
-static int orinoco_ioctl_set_mlme(struct net_device *dev,
-				  struct iw_request_info *info,
-				  union iwreq_data *wrqu, char *extra)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	struct iw_mlme *mlme = (struct iw_mlme *)extra;
-	unsigned long flags;
-	int ret = 0;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	switch (mlme->cmd) {
-	case IW_MLME_DEAUTH:
-		/* silently ignore */
-		break;
-
-	case IW_MLME_DISASSOC:
-
-		ret = orinoco_hw_disassociate(priv, mlme->addr.sa_data,
-					      mlme->reason_code);
-		break;
-
-	default:
-		ret = -EOPNOTSUPP;
-	}
-
-	orinoco_unlock(priv, &flags);
-	return ret;
-}
-
-static int orinoco_ioctl_reset(struct net_device *dev,
-			       struct iw_request_info *info,
-			       union iwreq_data *wrqu,
-			       char *extra)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-
-	if (!capable(CAP_NET_ADMIN))
-		return -EPERM;
-
-	if (info->cmd == (SIOCIWFIRSTPRIV + 0x1)) {
-		printk(KERN_DEBUG "%s: Forcing reset!\n", dev->name);
-
-		/* Firmware reset */
-		orinoco_reset(&priv->reset_work);
-	} else {
-		printk(KERN_DEBUG "%s: Force scheduling reset!\n", dev->name);
-
-		schedule_work(&priv->reset_work);
-	}
-
-	return 0;
-}
-
-static int orinoco_ioctl_setibssport(struct net_device *dev,
-				     struct iw_request_info *info,
-				     union iwreq_data *wrqu,
-				     char *extra)
-
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	int val = *((int *) extra);
-	unsigned long flags;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	priv->ibss_port = val;
-
-	/* Actually update the mode we are using */
-	set_port_type(priv);
-
-	orinoco_unlock(priv, &flags);
-	return -EINPROGRESS;		/* Call commit handler */
-}
-
-static int orinoco_ioctl_getibssport(struct net_device *dev,
-				     struct iw_request_info *info,
-				     union iwreq_data *wrqu,
-				     char *extra)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	int *val = (int *) extra;
-
-	*val = priv->ibss_port;
-	return 0;
-}
-
-static int orinoco_ioctl_setport3(struct net_device *dev,
-				  struct iw_request_info *info,
-				  union iwreq_data *wrqu,
-				  char *extra)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	int val = *((int *) extra);
-	int err = 0;
-	unsigned long flags;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	switch (val) {
-	case 0: /* Try to do IEEE ad-hoc mode */
-		if (!priv->has_ibss) {
-			err = -EINVAL;
-			break;
-		}
-		priv->prefer_port3 = 0;
-
-		break;
-
-	case 1: /* Try to do Lucent proprietary ad-hoc mode */
-		if (!priv->has_port3) {
-			err = -EINVAL;
-			break;
-		}
-		priv->prefer_port3 = 1;
-		break;
-
-	default:
-		err = -EINVAL;
-	}
-
-	if (!err) {
-		/* Actually update the mode we are using */
-		set_port_type(priv);
-		err = -EINPROGRESS;
-	}
-
-	orinoco_unlock(priv, &flags);
-
-	return err;
-}
-
-static int orinoco_ioctl_getport3(struct net_device *dev,
-				  struct iw_request_info *info,
-				  union iwreq_data *wrqu,
-				  char *extra)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	int *val = (int *) extra;
-
-	*val = priv->prefer_port3;
-	return 0;
-}
-
-static int orinoco_ioctl_setpreamble(struct net_device *dev,
-				     struct iw_request_info *info,
-				     union iwreq_data *wrqu,
-				     char *extra)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	unsigned long flags;
-	int val;
-
-	if (!priv->has_preamble)
-		return -EOPNOTSUPP;
-
-	/* 802.11b has recently defined some short preamble.
-	 * Basically, the Phy header has been reduced in size.
-	 * This increase performance, especially at high rates
-	 * (the preamble is transmitted at 1Mb/s), unfortunately
-	 * this give compatibility troubles... - Jean II */
-	val = *((int *) extra);
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	if (val)
-		priv->preamble = 1;
-	else
-		priv->preamble = 0;
-
-	orinoco_unlock(priv, &flags);
-
-	return -EINPROGRESS;		/* Call commit handler */
-}
-
-static int orinoco_ioctl_getpreamble(struct net_device *dev,
-				     struct iw_request_info *info,
-				     union iwreq_data *wrqu,
-				     char *extra)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	int *val = (int *) extra;
-
-	if (!priv->has_preamble)
-		return -EOPNOTSUPP;
-
-	*val = priv->preamble;
-	return 0;
-}
-
-/* ioctl interface to hermes_read_ltv()
- * To use with iwpriv, pass the RID as the token argument, e.g.
- * iwpriv get_rid [0xfc00]
- * At least Wireless Tools 25 is required to use iwpriv.
- * For Wireless Tools 25 and 26 append "dummy" are the end. */
-static int orinoco_ioctl_getrid(struct net_device *dev,
-				struct iw_request_info *info,
-				union iwreq_data *wrqu,
-				char *extra)
-{
-	struct iw_point *data = &wrqu->data;
-	struct orinoco_private *priv = ndev_priv(dev);
-	struct hermes *hw = &priv->hw;
-	int rid = data->flags;
-	u16 length;
-	int err;
-	unsigned long flags;
-
-	/* It's a "get" function, but we don't want users to access the
-	 * WEP key and other raw firmware data */
-	if (!capable(CAP_NET_ADMIN))
-		return -EPERM;
-
-	if (rid < 0xfc00 || rid > 0xffff)
-		return -EINVAL;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return -EBUSY;
-
-	err = hw->ops->read_ltv(hw, USER_BAP, rid, MAX_RID_LEN, &length,
-				extra);
-	if (err)
-		goto out;
-
-	data->length = min_t(u16, HERMES_RECLEN_TO_BYTES(length),
-			     MAX_RID_LEN);
-
- out:
-	orinoco_unlock(priv, &flags);
-	return err;
-}
-
-
-/* Commit handler, called after set operations */
-static int orinoco_ioctl_commit(struct net_device *dev,
-				struct iw_request_info *info,
-				union iwreq_data *wrqu,
-				char *extra)
-{
-	struct orinoco_private *priv = ndev_priv(dev);
-	unsigned long flags;
-	int err = 0;
-
-	if (!priv->open)
-		return 0;
-
-	if (orinoco_lock(priv, &flags) != 0)
-		return err;
-
-	err = orinoco_commit(priv);
-
-	orinoco_unlock(priv, &flags);
-	return err;
-}
-
-static const struct iw_priv_args orinoco_privtab[] = {
-	{ SIOCIWFIRSTPRIV + 0x0, 0, 0, "force_reset" },
-	{ SIOCIWFIRSTPRIV + 0x1, 0, 0, "card_reset" },
-	{ SIOCIWFIRSTPRIV + 0x2, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1,
-	  0, "set_port3" },
-	{ SIOCIWFIRSTPRIV + 0x3, 0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1,
-	  "get_port3" },
-	{ SIOCIWFIRSTPRIV + 0x4, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1,
-	  0, "set_preamble" },
-	{ SIOCIWFIRSTPRIV + 0x5, 0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1,
-	  "get_preamble" },
-	{ SIOCIWFIRSTPRIV + 0x6, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1,
-	  0, "set_ibssport" },
-	{ SIOCIWFIRSTPRIV + 0x7, 0, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1,
-	  "get_ibssport" },
-	{ SIOCIWFIRSTPRIV + 0x9, 0, IW_PRIV_TYPE_BYTE | MAX_RID_LEN,
-	  "get_rid" },
-};
-
-
-/*
- * Structures to export the Wireless Handlers
- */
-
-static const iw_handler	orinoco_handler[] = {
-	IW_HANDLER(SIOCSIWCOMMIT,	orinoco_ioctl_commit),
-	IW_HANDLER(SIOCGIWNAME,		cfg80211_wext_giwname),
-	IW_HANDLER(SIOCSIWFREQ,		orinoco_ioctl_setfreq),
-	IW_HANDLER(SIOCGIWFREQ,		orinoco_ioctl_getfreq),
-	IW_HANDLER(SIOCSIWMODE,		cfg80211_wext_siwmode),
-	IW_HANDLER(SIOCGIWMODE,		cfg80211_wext_giwmode),
-	IW_HANDLER(SIOCSIWSENS,		orinoco_ioctl_setsens),
-	IW_HANDLER(SIOCGIWSENS,		orinoco_ioctl_getsens),
-	IW_HANDLER(SIOCGIWRANGE,	cfg80211_wext_giwrange),
-	IW_HANDLER(SIOCSIWSPY,		iw_handler_set_spy),
-	IW_HANDLER(SIOCGIWSPY,		iw_handler_get_spy),
-	IW_HANDLER(SIOCSIWTHRSPY,	iw_handler_set_thrspy),
-	IW_HANDLER(SIOCGIWTHRSPY,	iw_handler_get_thrspy),
-	IW_HANDLER(SIOCSIWAP,		orinoco_ioctl_setwap),
-	IW_HANDLER(SIOCGIWAP,		orinoco_ioctl_getwap),
-	IW_HANDLER(SIOCSIWSCAN,		cfg80211_wext_siwscan),
-	IW_HANDLER(SIOCGIWSCAN,		cfg80211_wext_giwscan),
-	IW_HANDLER(SIOCSIWESSID,	orinoco_ioctl_setessid),
-	IW_HANDLER(SIOCGIWESSID,	orinoco_ioctl_getessid),
-	IW_HANDLER(SIOCSIWRATE,		orinoco_ioctl_setrate),
-	IW_HANDLER(SIOCGIWRATE,		orinoco_ioctl_getrate),
-	IW_HANDLER(SIOCSIWRTS,		cfg80211_wext_siwrts),
-	IW_HANDLER(SIOCGIWRTS,		cfg80211_wext_giwrts),
-	IW_HANDLER(SIOCSIWFRAG,		cfg80211_wext_siwfrag),
-	IW_HANDLER(SIOCGIWFRAG,		cfg80211_wext_giwfrag),
-	IW_HANDLER(SIOCGIWRETRY,	cfg80211_wext_giwretry),
-	IW_HANDLER(SIOCSIWENCODE,	orinoco_ioctl_setiwencode),
-	IW_HANDLER(SIOCGIWENCODE,	orinoco_ioctl_getiwencode),
-	IW_HANDLER(SIOCSIWPOWER,	orinoco_ioctl_setpower),
-	IW_HANDLER(SIOCGIWPOWER,	orinoco_ioctl_getpower),
-	IW_HANDLER(SIOCSIWGENIE,	orinoco_ioctl_set_genie),
-	IW_HANDLER(SIOCGIWGENIE,	orinoco_ioctl_get_genie),
-	IW_HANDLER(SIOCSIWMLME,		orinoco_ioctl_set_mlme),
-	IW_HANDLER(SIOCSIWAUTH,		orinoco_ioctl_set_auth),
-	IW_HANDLER(SIOCGIWAUTH,		orinoco_ioctl_get_auth),
-	IW_HANDLER(SIOCSIWENCODEEXT,	orinoco_ioctl_set_encodeext),
-	IW_HANDLER(SIOCGIWENCODEEXT,	orinoco_ioctl_get_encodeext),
-};
-
-
-/*
-  Added typecasting since we no longer use iwreq_data -- Moustafa
- */
-static const iw_handler	orinoco_private_handler[] = {
-	[0] = orinoco_ioctl_reset,
-	[1] = orinoco_ioctl_reset,
-	[2] = orinoco_ioctl_setport3,
-	[3] = orinoco_ioctl_getport3,
-	[4] = orinoco_ioctl_setpreamble,
-	[5] = orinoco_ioctl_getpreamble,
-	[6] = orinoco_ioctl_setibssport,
-	[7] = orinoco_ioctl_getibssport,
-	[9] = orinoco_ioctl_getrid,
-};
-
-const struct iw_handler_def orinoco_handler_def = {
-	.num_standard = ARRAY_SIZE(orinoco_handler),
-	.num_private = ARRAY_SIZE(orinoco_private_handler),
-	.num_private_args = ARRAY_SIZE(orinoco_privtab),
-	.standard = orinoco_handler,
-	.private = orinoco_private_handler,
-	.private_args = orinoco_privtab,
-	.get_wireless_stats = orinoco_get_wireless_stats,
-};
diff --git a/drivers/net/wireless/intersil/orinoco/wext.h b/drivers/net/wireless/intersil/orinoco/wext.h
deleted file mode 100644
index 1479f4e26dde..000000000000
--- a/drivers/net/wireless/intersil/orinoco/wext.h
+++ /dev/null
@@ -1,13 +0,0 @@
-/* Wireless extensions support.
- *
- * See copyright notice in main.c
- */
-#ifndef _ORINOCO_WEXT_H_
-#define _ORINOCO_WEXT_H_
-
-#include <net/iw_handler.h>
-
-/* Structure defining all our WEXT handlers */
-extern const struct iw_handler_def orinoco_handler_def;
-
-#endif /* _ORINOCO_WEXT_H_ */
diff --git a/drivers/net/wireless/legacy/Kconfig b/drivers/net/wireless/legacy/Kconfig
deleted file mode 100644
index 3a5275941212..000000000000
--- a/drivers/net/wireless/legacy/Kconfig
+++ /dev/null
@@ -1,55 +0,0 @@
-config PCMCIA_RAYCS
-	tristate "Aviator/Raytheon 2.4GHz wireless support"
-	depends on PCMCIA
-	select WIRELESS_EXT
-	select WEXT_SPY
-	select WEXT_PRIV
-	help
-	  Say Y here if you intend to attach an Aviator/Raytheon PCMCIA
-	  (PC-card) wireless Ethernet networking card to your computer.
-	  Please read the file
-	  <file:Documentation/networking/device_drivers/wifi/ray_cs.rst> for
-	  details.
-
-	  To compile this driver as a module, choose M here: the module will be
-	  called ray_cs.  If unsure, say N.
-
-config PCMCIA_WL3501
-	tristate "Planet WL3501 PCMCIA cards"
-	depends on CFG80211 && PCMCIA
-	select WIRELESS_EXT
-	select WEXT_SPY
-	help
-	  A driver for WL3501 PCMCIA 802.11 wireless cards made by Planet.
-	  It has basic support for Linux wireless extensions and initial
-	  micro support for ethtool.
-
-config USB_NET_RNDIS_WLAN
-	tristate "Wireless RNDIS USB support"
-	depends on USB
-	depends on CFG80211
-	select USB_NET_DRIVERS
-	select USB_USBNET
-	select USB_NET_CDCETHER
-	select USB_NET_RNDIS_HOST
-	help
-	  This is a driver for wireless RNDIS devices.
-	  These are USB based adapters found in devices such as:
-
-	  Buffalo WLI-U2-KG125S
-	  U.S. Robotics USR5421
-	  Belkin F5D7051
-	  Linksys WUSB54GSv2
-	  Linksys WUSB54GSC
-	  Asus WL169gE
-	  Eminent EM4045
-	  BT Voyager 1055
-	  Linksys WUSB54GSv1
-	  U.S. Robotics USR5420
-	  BUFFALO WLI-USB-G54
-
-	  All of these devices are based on Broadcom 4320 chip which is the
-	  only wireless RNDIS chip known to date.
-
-	  If you choose to build a module, it'll be called rndis_wlan.
-
diff --git a/drivers/net/wireless/legacy/Makefile b/drivers/net/wireless/legacy/Makefile
deleted file mode 100644
index 36878f080bfc..000000000000
--- a/drivers/net/wireless/legacy/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-# 16-bit wireless PCMCIA client drivers
-obj-$(CONFIG_PCMCIA_RAYCS)	+= ray_cs.o
-obj-$(CONFIG_PCMCIA_WL3501)	+= wl3501_cs.o
-
-obj-$(CONFIG_USB_NET_RNDIS_WLAN)	+= rndis_wlan.o
-
diff --git a/drivers/net/wireless/legacy/ray_cs.c b/drivers/net/wireless/legacy/ray_cs.c
deleted file mode 100644
index c95a79e01cd0..000000000000
--- a/drivers/net/wireless/legacy/ray_cs.c
+++ /dev/null
@@ -1,2824 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*=============================================================================
- *
- * A  PCMCIA client driver for the Raylink wireless LAN card.
- * The starting point for this module was the skeleton.c in the
- * PCMCIA 2.9.12 package written by David Hinds, dahinds@users.sourceforge.net
- *
- * Copyright (c) 1998  Corey Thomas (corey@world.std.com)
- *
- * Changes:
- * Arnaldo Carvalho de Melo <acme@conectiva.com.br> - 08/08/2000
- * - reorganize kmallocs in ray_attach, checking all for failure
- *   and releasing the previous allocations if one fails
- *
- * Daniele Bellucci <bellucda@tiscali.it> - 07/10/2003
- * - Audit copy_to_user in ioctl(SIOCGIWESSID)
- *
-=============================================================================*/
-
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/proc_fs.h>
-#include <linux/ptrace.h>
-#include <linux/seq_file.h>
-#include <linux/string.h>
-#include <linux/timer.h>
-#include <linux/init.h>
-#include <linux/netdevice.h>
-#include <linux/etherdevice.h>
-#include <linux/if_arp.h>
-#include <linux/ioport.h>
-#include <linux/skbuff.h>
-#include <linux/ieee80211.h>
-
-#include <pcmcia/cistpl.h>
-#include <pcmcia/cisreg.h>
-#include <pcmcia/ds.h>
-
-#include <linux/wireless.h>
-#include <net/iw_handler.h>
-
-#include <asm/io.h>
-#include <asm/byteorder.h>
-#include <linux/uaccess.h>
-
-/* Warning : these stuff will slow down the driver... */
-#define WIRELESS_SPY		/* Enable spying addresses */
-/* Definitions we need for spy */
-typedef struct iw_statistics iw_stats;
-typedef u_char mac_addr[ETH_ALEN];	/* Hardware address */
-
-#include "rayctl.h"
-#include "ray_cs.h"
-
-
-/** Prototypes based on PCMCIA skeleton driver *******************************/
-static int ray_config(struct pcmcia_device *link);
-static void ray_release(struct pcmcia_device *link);
-static void ray_detach(struct pcmcia_device *p_dev);
-
-/***** Prototypes indicated by device structure ******************************/
-static int ray_dev_close(struct net_device *dev);
-static int ray_dev_config(struct net_device *dev, struct ifmap *map);
-static struct net_device_stats *ray_get_stats(struct net_device *dev);
-static int ray_dev_init(struct net_device *dev);
-
-static int ray_open(struct net_device *dev);
-static netdev_tx_t ray_dev_start_xmit(struct sk_buff *skb,
-					    struct net_device *dev);
-static void set_multicast_list(struct net_device *dev);
-static void ray_update_multi_list(struct net_device *dev, int all);
-static int translate_frame(ray_dev_t *local, struct tx_msg __iomem *ptx,
-			   unsigned char *data, int len);
-static void ray_build_header(ray_dev_t *local, struct tx_msg __iomem *ptx,
-			     UCHAR msg_type, unsigned char *data);
-static void untranslate(ray_dev_t *local, struct sk_buff *skb, int len);
-static iw_stats *ray_get_wireless_stats(struct net_device *dev);
-static const struct iw_handler_def ray_handler_def;
-
-/***** Prototypes for raylink functions **************************************/
-static void authenticate(ray_dev_t *local);
-static int build_auth_frame(ray_dev_t *local, UCHAR *dest, int auth_type);
-static void authenticate_timeout(struct timer_list *t);
-static int get_free_ccs(ray_dev_t *local);
-static int get_free_tx_ccs(ray_dev_t *local);
-static void init_startup_params(ray_dev_t *local);
-static int parse_addr(char *in_str, UCHAR *out);
-static int ray_hw_xmit(unsigned char *data, int len, struct net_device *dev, UCHAR type);
-static int ray_init(struct net_device *dev);
-static int interrupt_ecf(ray_dev_t *local, int ccs);
-static void ray_reset(struct net_device *dev);
-static void ray_update_parm(struct net_device *dev, UCHAR objid, UCHAR *value, int len);
-static void verify_dl_startup(struct timer_list *t);
-
-/* Prototypes for interrpt time functions **********************************/
-static irqreturn_t ray_interrupt(int reg, void *dev_id);
-static void clear_interrupt(ray_dev_t *local);
-static void rx_deauthenticate(ray_dev_t *local, struct rcs __iomem *prcs,
-			      unsigned int pkt_addr, int rx_len);
-static int copy_from_rx_buff(ray_dev_t *local, UCHAR *dest, int pkt_addr, int len);
-static void ray_rx(struct net_device *dev, ray_dev_t *local, struct rcs __iomem *prcs);
-static void release_frag_chain(ray_dev_t *local, struct rcs __iomem *prcs);
-static void rx_authenticate(ray_dev_t *local, struct rcs __iomem *prcs,
-			    unsigned int pkt_addr, int rx_len);
-static void rx_data(struct net_device *dev, struct rcs __iomem *prcs,
-		    unsigned int pkt_addr, int rx_len);
-static void associate(ray_dev_t *local);
-
-/* Card command functions */
-static int dl_startup_params(struct net_device *dev);
-static void join_net(struct timer_list *t);
-static void start_net(struct timer_list *t);
-
-/*===========================================================================*/
-/* Parameters that can be set with 'insmod' */
-
-/* ADHOC=0, Infrastructure=1 */
-static int net_type = ADHOC;
-
-/* Hop dwell time in Kus (1024 us units defined by 802.11) */
-static int hop_dwell = 128;
-
-/* Beacon period in Kus */
-static int beacon_period = 256;
-
-/* power save mode (0 = off, 1 = save power) */
-static int psm;
-
-/* String for network's Extended Service Set ID. 32 Characters max */
-static char *essid;
-
-/* Default to encapsulation unless translation requested */
-static bool translate = true;
-
-static int country = USA;
-
-static int sniffer;
-
-static int bc;
-
-/* 48 bit physical card address if overriding card's real physical
- * address is required.  Since IEEE 802.11 addresses are 48 bits
- * like ethernet, an int can't be used, so a string is used. To
- * allow use of addresses starting with a decimal digit, the first
- * character must be a letter and will be ignored. This letter is
- * followed by up to 12 hex digits which are the address.  If less
- * than 12 digits are used, the address will be left filled with 0's.
- * Note that bit 0 of the first byte is the broadcast bit, and evil
- * things will happen if it is not 0 in a card address.
- */
-static char *phy_addr = NULL;
-
-static unsigned int ray_mem_speed = 500;
-
-/* WARNING: THIS DRIVER IS NOT CAPABLE OF HANDLING MULTIPLE DEVICES! */
-static struct pcmcia_device *this_device = NULL;
-
-MODULE_AUTHOR("Corey Thomas <corey@world.std.com>");
-MODULE_DESCRIPTION("Raylink/WebGear wireless LAN driver");
-MODULE_LICENSE("GPL");
-
-module_param(net_type, int, 0);
-module_param(hop_dwell, int, 0);
-module_param(beacon_period, int, 0);
-module_param(psm, int, 0);
-module_param(essid, charp, 0);
-module_param(translate, bool, 0);
-module_param(country, int, 0);
-module_param(sniffer, int, 0);
-module_param(bc, int, 0);
-module_param(phy_addr, charp, 0);
-module_param(ray_mem_speed, int, 0);
-
-static const UCHAR b5_default_startup_parms[] = {
-	0, 0,			/* Adhoc station */
-	'L', 'I', 'N', 'U', 'X', 0, 0, 0,	/* 32 char ESSID */
-	0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 0, 0, 0, 0,
-	1, 0,			/* Active scan, CA Mode */
-	0, 0, 0, 0, 0, 0,	/* No default MAC addr  */
-	0x7f, 0xff,		/* Frag threshold */
-	0x00, 0x80,		/* Hop time 128 Kus */
-	0x01, 0x00,		/* Beacon period 256 Kus */
-	0x01, 0x07, 0xa3,	/* DTIM, retries, ack timeout */
-	0x1d, 0x82, 0x4e,	/* SIFS, DIFS, PIFS */
-	0x7f, 0xff,		/* RTS threshold */
-	0x04, 0xe2, 0x38, 0xA4,	/* scan_dwell, max_scan_dwell */
-	0x05,			/* assoc resp timeout thresh */
-	0x08, 0x02, 0x08,	/* adhoc, infra, super cycle max */
-	0,			/* Promiscuous mode */
-	0x0c, 0x0bd,		/* Unique word */
-	0x32,			/* Slot time */
-	0xff, 0xff,		/* roam-low snr, low snr count */
-	0x05, 0xff,		/* Infra, adhoc missed bcn thresh */
-	0x01, 0x0b, 0x4f,	/* USA, hop pattern, hop pat length */
-/* b4 - b5 differences start here */
-	0x00, 0x3f,		/* CW max */
-	0x00, 0x0f,		/* CW min */
-	0x04, 0x08,		/* Noise gain, limit offset */
-	0x28, 0x28,		/* det rssi, med busy offsets */
-	7,			/* det sync thresh */
-	0, 2, 2,		/* test mode, min, max */
-	0,			/* allow broadcast SSID probe resp */
-	0, 0,			/* privacy must start, can join */
-	2, 0, 0, 0, 0, 0, 0, 0	/* basic rate set */
-};
-
-static const UCHAR b4_default_startup_parms[] = {
-	0, 0,			/* Adhoc station */
-	'L', 'I', 'N', 'U', 'X', 0, 0, 0,	/* 32 char ESSID */
-	0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 0, 0, 0, 0,
-	0, 0, 0, 0, 0, 0, 0, 0,
-	1, 0,			/* Active scan, CA Mode */
-	0, 0, 0, 0, 0, 0,	/* No default MAC addr  */
-	0x7f, 0xff,		/* Frag threshold */
-	0x02, 0x00,		/* Hop time */
-	0x00, 0x01,		/* Beacon period */
-	0x01, 0x07, 0xa3,	/* DTIM, retries, ack timeout */
-	0x1d, 0x82, 0xce,	/* SIFS, DIFS, PIFS */
-	0x7f, 0xff,		/* RTS threshold */
-	0xfb, 0x1e, 0xc7, 0x5c,	/* scan_dwell, max_scan_dwell */
-	0x05,			/* assoc resp timeout thresh */
-	0x04, 0x02, 0x4,	/* adhoc, infra, super cycle max */
-	0,			/* Promiscuous mode */
-	0x0c, 0x0bd,		/* Unique word */
-	0x4e,			/* Slot time (TBD seems wrong) */
-	0xff, 0xff,		/* roam-low snr, low snr count */
-	0x05, 0xff,		/* Infra, adhoc missed bcn thresh */
-	0x01, 0x0b, 0x4e,	/* USA, hop pattern, hop pat length */
-/* b4 - b5 differences start here */
-	0x3f, 0x0f,		/* CW max, min */
-	0x04, 0x08,		/* Noise gain, limit offset */
-	0x28, 0x28,		/* det rssi, med busy offsets */
-	7,			/* det sync thresh */
-	0, 2, 2,		/* test mode, min, max */
-	0,			/* rx/tx delay */
-	0, 0, 0, 0, 0, 0,	/* current BSS id */
-	0			/* hop set */
-};
-
-/*===========================================================================*/
-static const u8 eth2_llc[] = { 0xaa, 0xaa, 3, 0, 0, 0 };
-
-static const char hop_pattern_length[] = { 1,
-	USA_HOP_MOD, EUROPE_HOP_MOD,
-	JAPAN_HOP_MOD, KOREA_HOP_MOD,
-	SPAIN_HOP_MOD, FRANCE_HOP_MOD,
-	ISRAEL_HOP_MOD, AUSTRALIA_HOP_MOD,
-	JAPAN_TEST_HOP_MOD
-};
-
-static const char rcsid[] =
-    "Raylink/WebGear wireless LAN - Corey <Thomas corey@world.std.com>";
-
-static const struct net_device_ops ray_netdev_ops = {
-	.ndo_init 		= ray_dev_init,
-	.ndo_open 		= ray_open,
-	.ndo_stop 		= ray_dev_close,
-	.ndo_start_xmit		= ray_dev_start_xmit,
-	.ndo_set_config		= ray_dev_config,
-	.ndo_get_stats		= ray_get_stats,
-	.ndo_set_rx_mode	= set_multicast_list,
-	.ndo_set_mac_address 	= eth_mac_addr,
-	.ndo_validate_addr	= eth_validate_addr,
-};
-
-static int ray_probe(struct pcmcia_device *p_dev)
-{
-	ray_dev_t *local;
-	struct net_device *dev;
-	int ret;
-
-	dev_dbg(&p_dev->dev, "ray_attach()\n");
-
-	/* Allocate space for private device-specific data */
-	dev = alloc_etherdev(sizeof(ray_dev_t));
-	if (!dev)
-		return -ENOMEM;
-
-	local = netdev_priv(dev);
-	local->finder = p_dev;
-
-	/* The io structure describes IO port mapping. None used here */
-	p_dev->resource[0]->end = 0;
-	p_dev->resource[0]->flags |= IO_DATA_PATH_WIDTH_8;
-
-	/* General socket configuration */
-	p_dev->config_flags |= CONF_ENABLE_IRQ;
-	p_dev->config_index = 1;
-
-	p_dev->priv = dev;
-
-	local->finder = p_dev;
-	local->card_status = CARD_INSERTED;
-	local->authentication_state = UNAUTHENTICATED;
-	local->num_multi = 0;
-	dev_dbg(&p_dev->dev, "ray_attach p_dev = %p,  dev = %p,  local = %p, intr = %p\n",
-	      p_dev, dev, local, &ray_interrupt);
-
-	/* Raylink entries in the device structure */
-	dev->netdev_ops = &ray_netdev_ops;
-	dev->wireless_handlers = &ray_handler_def;
-#ifdef WIRELESS_SPY
-	local->wireless_data.spy_data = &local->spy_data;
-	dev->wireless_data = &local->wireless_data;
-#endif /* WIRELESS_SPY */
-
-
-	dev_dbg(&p_dev->dev, "ray_cs ray_attach calling ether_setup.)\n");
-	netif_stop_queue(dev);
-
-	timer_setup(&local->timer, NULL, 0);
-
-	this_device = p_dev;
-	ret = ray_config(p_dev);
-	if (ret)
-		goto err_free_dev;
-
-	return 0;
-
-err_free_dev:
-	free_netdev(dev);
-	return ret;
-}
-
-static void ray_detach(struct pcmcia_device *link)
-{
-	struct net_device *dev;
-
-	dev_dbg(&link->dev, "ray_detach\n");
-
-	this_device = NULL;
-	dev = link->priv;
-
-	ray_release(link);
-
-	if (link->priv) {
-		unregister_netdev(dev);
-		free_netdev(dev);
-	}
-	dev_dbg(&link->dev, "ray_cs ray_detach ending\n");
-} /* ray_detach */
-
-#define MAX_TUPLE_SIZE 128
-static int ray_config(struct pcmcia_device *link)
-{
-	int ret = 0;
-	int i;
-	struct net_device *dev = link->priv;
-	ray_dev_t *local = netdev_priv(dev);
-
-	dev_dbg(&link->dev, "ray_config\n");
-
-	/* Determine card type and firmware version */
-	printk(KERN_INFO "ray_cs Detected: %s%s%s%s\n",
-	       link->prod_id[0] ? link->prod_id[0] : " ",
-	       link->prod_id[1] ? link->prod_id[1] : " ",
-	       link->prod_id[2] ? link->prod_id[2] : " ",
-	       link->prod_id[3] ? link->prod_id[3] : " ");
-
-	/* Now allocate an interrupt line.  Note that this does not
-	   actually assign a handler to the interrupt.
-	 */
-	ret = pcmcia_request_irq(link, ray_interrupt);
-	if (ret)
-		goto failed;
-	dev->irq = link->irq;
-
-	ret = pcmcia_enable_device(link);
-	if (ret)
-		goto failed;
-
-/*** Set up 32k window for shared memory (transmit and control) ************/
-	link->resource[2]->flags |= WIN_DATA_WIDTH_8 | WIN_MEMORY_TYPE_CM | WIN_ENABLE | WIN_USE_WAIT;
-	link->resource[2]->start = 0;
-	link->resource[2]->end = 0x8000;
-	ret = pcmcia_request_window(link, link->resource[2], ray_mem_speed);
-	if (ret)
-		goto failed;
-	ret = pcmcia_map_mem_page(link, link->resource[2], 0);
-	if (ret)
-		goto failed;
-	local->sram = ioremap(link->resource[2]->start,
-			resource_size(link->resource[2]));
-	if (!local->sram)
-		goto failed;
-
-/*** Set up 16k window for shared memory (receive buffer) ***************/
-	link->resource[3]->flags |=
-	    WIN_DATA_WIDTH_8 | WIN_MEMORY_TYPE_CM | WIN_ENABLE | WIN_USE_WAIT;
-	link->resource[3]->start = 0;
-	link->resource[3]->end = 0x4000;
-	ret = pcmcia_request_window(link, link->resource[3], ray_mem_speed);
-	if (ret)
-		goto failed;
-	ret = pcmcia_map_mem_page(link, link->resource[3], 0x8000);
-	if (ret)
-		goto failed;
-	local->rmem = ioremap(link->resource[3]->start,
-			resource_size(link->resource[3]));
-	if (!local->rmem)
-		goto failed;
-
-/*** Set up window for attribute memory ***********************************/
-	link->resource[4]->flags |=
-	    WIN_DATA_WIDTH_8 | WIN_MEMORY_TYPE_AM | WIN_ENABLE | WIN_USE_WAIT;
-	link->resource[4]->start = 0;
-	link->resource[4]->end = 0x1000;
-	ret = pcmcia_request_window(link, link->resource[4], ray_mem_speed);
-	if (ret)
-		goto failed;
-	ret = pcmcia_map_mem_page(link, link->resource[4], 0);
-	if (ret)
-		goto failed;
-	local->amem = ioremap(link->resource[4]->start,
-			resource_size(link->resource[4]));
-	if (!local->amem)
-		goto failed;
-
-	dev_dbg(&link->dev, "ray_config sram=%p\n", local->sram);
-	dev_dbg(&link->dev, "ray_config rmem=%p\n", local->rmem);
-	dev_dbg(&link->dev, "ray_config amem=%p\n", local->amem);
-	if (ray_init(dev) < 0) {
-		ray_release(link);
-		return -ENODEV;
-	}
-
-	SET_NETDEV_DEV(dev, &link->dev);
-	i = register_netdev(dev);
-	if (i != 0) {
-		printk("ray_config register_netdev() failed\n");
-		ray_release(link);
-		return i;
-	}
-
-	printk(KERN_INFO "%s: RayLink, irq %d, hw_addr %pM\n",
-	       dev->name, dev->irq, dev->dev_addr);
-
-	return 0;
-
-failed:
-	ray_release(link);
-	return -ENODEV;
-} /* ray_config */
-
-static inline struct ccs __iomem *ccs_base(ray_dev_t *dev)
-{
-	return dev->sram + CCS_BASE;
-}
-
-static inline struct rcs __iomem *rcs_base(ray_dev_t *dev)
-{
-	/*
-	 * This looks nonsensical, since there is a separate
-	 * RCS_BASE. But the difference between a "struct rcs"
-	 * and a "struct ccs" ends up being in the _index_ off
-	 * the base, so the base pointer is the same for both
-	 * ccs/rcs.
-	 */
-	return dev->sram + CCS_BASE;
-}
-
-/*===========================================================================*/
-static int ray_init(struct net_device *dev)
-{
-	int i;
-	struct ccs __iomem *pccs;
-	ray_dev_t *local = netdev_priv(dev);
-	struct pcmcia_device *link = local->finder;
-	dev_dbg(&link->dev, "ray_init(0x%p)\n", dev);
-	if (!(pcmcia_dev_present(link))) {
-		dev_dbg(&link->dev, "ray_init - device not present\n");
-		return -1;
-	}
-
-	local->net_type = net_type;
-	local->sta_type = TYPE_STA;
-
-	/* Copy the startup results to local memory */
-	memcpy_fromio(&local->startup_res, local->sram + ECF_TO_HOST_BASE,
-		      sizeof(struct startup_res_6));
-
-	/* Check Power up test status and get mac address from card */
-	if (local->startup_res.startup_word != 0x80) {
-		printk(KERN_INFO "ray_init ERROR card status = %2x\n",
-		       local->startup_res.startup_word);
-		local->card_status = CARD_INIT_ERROR;
-		return -1;
-	}
-
-	local->fw_ver = local->startup_res.firmware_version[0];
-	local->fw_bld = local->startup_res.firmware_version[1];
-	local->fw_var = local->startup_res.firmware_version[2];
-	dev_dbg(&link->dev, "ray_init firmware version %d.%d\n", local->fw_ver,
-	      local->fw_bld);
-
-	local->tib_length = 0x20;
-	if ((local->fw_ver == 5) && (local->fw_bld >= 30))
-		local->tib_length = local->startup_res.tib_length;
-	dev_dbg(&link->dev, "ray_init tib_length = 0x%02x\n", local->tib_length);
-	/* Initialize CCS's to buffer free state */
-	pccs = ccs_base(local);
-	for (i = 0; i < NUMBER_OF_CCS; i++) {
-		writeb(CCS_BUFFER_FREE, &(pccs++)->buffer_status);
-	}
-	init_startup_params(local);
-
-	/* copy mac address to startup parameters */
-	if (!parse_addr(phy_addr, local->sparm.b4.a_mac_addr)) {
-		memcpy(&local->sparm.b4.a_mac_addr,
-		       &local->startup_res.station_addr, ADDRLEN);
-	}
-
-	clear_interrupt(local);	/* Clear any interrupt from the card */
-	local->card_status = CARD_AWAITING_PARAM;
-	dev_dbg(&link->dev, "ray_init ending\n");
-	return 0;
-} /* ray_init */
-
-/*===========================================================================*/
-/* Download startup parameters to the card and command it to read them       */
-static int dl_startup_params(struct net_device *dev)
-{
-	int ccsindex;
-	ray_dev_t *local = netdev_priv(dev);
-	struct ccs __iomem *pccs;
-	struct pcmcia_device *link = local->finder;
-
-	dev_dbg(&link->dev, "dl_startup_params entered\n");
-	if (!(pcmcia_dev_present(link))) {
-		dev_dbg(&link->dev, "ray_cs dl_startup_params - device not present\n");
-		return -1;
-	}
-
-	/* Copy parameters to host to ECF area */
-	if (local->fw_ver == 0x55)
-		memcpy_toio(local->sram + HOST_TO_ECF_BASE, &local->sparm.b4,
-			    sizeof(struct b4_startup_params));
-	else
-		memcpy_toio(local->sram + HOST_TO_ECF_BASE, &local->sparm.b5,
-			    sizeof(struct b5_startup_params));
-
-	/* Fill in the CCS fields for the ECF */
-	if ((ccsindex = get_free_ccs(local)) < 0)
-		return -1;
-	local->dl_param_ccs = ccsindex;
-	pccs = ccs_base(local) + ccsindex;
-	writeb(CCS_DOWNLOAD_STARTUP_PARAMS, &pccs->cmd);
-	dev_dbg(&link->dev, "dl_startup_params start ccsindex = %d\n",
-	      local->dl_param_ccs);
-	/* Interrupt the firmware to process the command */
-	if (interrupt_ecf(local, ccsindex)) {
-		printk(KERN_INFO "ray dl_startup_params failed - "
-		       "ECF not ready for intr\n");
-		local->card_status = CARD_DL_PARAM_ERROR;
-		writeb(CCS_BUFFER_FREE, &(pccs++)->buffer_status);
-		return -2;
-	}
-	local->card_status = CARD_DL_PARAM;
-	/* Start kernel timer to wait for dl startup to complete. */
-	local->timer.expires = jiffies + HZ / 2;
-	local->timer.function = verify_dl_startup;
-	add_timer(&local->timer);
-	dev_dbg(&link->dev,
-	      "ray_cs dl_startup_params started timer for verify_dl_startup\n");
-	return 0;
-} /* dl_startup_params */
-
-/*===========================================================================*/
-static void init_startup_params(ray_dev_t *local)
-{
-	int i;
-
-	if (country > JAPAN_TEST)
-		country = USA;
-	else if (country < USA)
-		country = USA;
-	/* structure for hop time and beacon period is defined here using
-	 * New 802.11D6.1 format.  Card firmware is still using old format
-	 * until version 6.
-	 *    Before                    After
-	 *    a_hop_time ms byte        a_hop_time ms byte
-	 *    a_hop_time 2s byte        a_hop_time ls byte
-	 *    a_hop_time ls byte        a_beacon_period ms byte
-	 *    a_beacon_period           a_beacon_period ls byte
-	 *
-	 *    a_hop_time = uS           a_hop_time = KuS
-	 *    a_beacon_period = hops    a_beacon_period = KuS
-	 *//* 64ms = 010000 */
-	if (local->fw_ver == 0x55) {
-		memcpy(&local->sparm.b4, b4_default_startup_parms,
-		       sizeof(struct b4_startup_params));
-		/* Translate sane kus input values to old build 4/5 format */
-		/* i = hop time in uS truncated to 3 bytes */
-		i = (hop_dwell * 1024) & 0xffffff;
-		local->sparm.b4.a_hop_time[0] = (i >> 16) & 0xff;
-		local->sparm.b4.a_hop_time[1] = (i >> 8) & 0xff;
-		local->sparm.b4.a_beacon_period[0] = 0;
-		local->sparm.b4.a_beacon_period[1] =
-		    ((beacon_period / hop_dwell) - 1) & 0xff;
-		local->sparm.b4.a_curr_country_code = country;
-		local->sparm.b4.a_hop_pattern_length =
-		    hop_pattern_length[(int)country] - 1;
-		if (bc) {
-			local->sparm.b4.a_ack_timeout = 0x50;
-			local->sparm.b4.a_sifs = 0x3f;
-		}
-	} else { /* Version 5 uses real kus values */
-		memcpy((UCHAR *) &local->sparm.b5, b5_default_startup_parms,
-		       sizeof(struct b5_startup_params));
-
-		local->sparm.b5.a_hop_time[0] = (hop_dwell >> 8) & 0xff;
-		local->sparm.b5.a_hop_time[1] = hop_dwell & 0xff;
-		local->sparm.b5.a_beacon_period[0] =
-		    (beacon_period >> 8) & 0xff;
-		local->sparm.b5.a_beacon_period[1] = beacon_period & 0xff;
-		if (psm)
-			local->sparm.b5.a_power_mgt_state = 1;
-		local->sparm.b5.a_curr_country_code = country;
-		local->sparm.b5.a_hop_pattern_length =
-		    hop_pattern_length[(int)country];
-	}
-
-	local->sparm.b4.a_network_type = net_type & 0x01;
-	local->sparm.b4.a_acting_as_ap_status = TYPE_STA;
-
-	if (essid != NULL)
-		strscpy(local->sparm.b4.a_current_ess_id, essid, ESSID_SIZE);
-} /* init_startup_params */
-
-/*===========================================================================*/
-static void verify_dl_startup(struct timer_list *t)
-{
-	ray_dev_t *local = from_timer(local, t, timer);
-	struct ccs __iomem *pccs = ccs_base(local) + local->dl_param_ccs;
-	UCHAR status;
-	struct pcmcia_device *link = local->finder;
-
-	if (!(pcmcia_dev_present(link))) {
-		dev_dbg(&link->dev, "ray_cs verify_dl_startup - device not present\n");
-		return;
-	}
-#if 0
-	{
-		int i;
-		printk(KERN_DEBUG
-		       "verify_dl_startup parameters sent via ccs %d:\n",
-		       local->dl_param_ccs);
-		for (i = 0; i < sizeof(struct b5_startup_params); i++) {
-			printk(" %2x",
-			       (unsigned int)readb(local->sram +
-						   HOST_TO_ECF_BASE + i));
-		}
-		printk("\n");
-	}
-#endif
-
-	status = readb(&pccs->buffer_status);
-	if (status != CCS_BUFFER_FREE) {
-		printk(KERN_INFO
-		       "Download startup params failed.  Status = %d\n",
-		       status);
-		local->card_status = CARD_DL_PARAM_ERROR;
-		return;
-	}
-	if (local->sparm.b4.a_network_type == ADHOC)
-		start_net(&local->timer);
-	else
-		join_net(&local->timer);
-} /* end verify_dl_startup */
-
-/*===========================================================================*/
-/* Command card to start a network */
-static void start_net(struct timer_list *t)
-{
-	ray_dev_t *local = from_timer(local, t, timer);
-	struct ccs __iomem *pccs;
-	int ccsindex;
-	struct pcmcia_device *link = local->finder;
-	if (!(pcmcia_dev_present(link))) {
-		dev_dbg(&link->dev, "ray_cs start_net - device not present\n");
-		return;
-	}
-	/* Fill in the CCS fields for the ECF */
-	if ((ccsindex = get_free_ccs(local)) < 0)
-		return;
-	pccs = ccs_base(local) + ccsindex;
-	writeb(CCS_START_NETWORK, &pccs->cmd);
-	writeb(0, &pccs->var.start_network.update_param);
-	/* Interrupt the firmware to process the command */
-	if (interrupt_ecf(local, ccsindex)) {
-		dev_dbg(&link->dev, "ray start net failed - card not ready for intr\n");
-		writeb(CCS_BUFFER_FREE, &(pccs++)->buffer_status);
-		return;
-	}
-	local->card_status = CARD_DOING_ACQ;
-} /* end start_net */
-
-/*===========================================================================*/
-/* Command card to join a network */
-static void join_net(struct timer_list *t)
-{
-	ray_dev_t *local = from_timer(local, t, timer);
-
-	struct ccs __iomem *pccs;
-	int ccsindex;
-	struct pcmcia_device *link = local->finder;
-
-	if (!(pcmcia_dev_present(link))) {
-		dev_dbg(&link->dev, "ray_cs join_net - device not present\n");
-		return;
-	}
-	/* Fill in the CCS fields for the ECF */
-	if ((ccsindex = get_free_ccs(local)) < 0)
-		return;
-	pccs = ccs_base(local) + ccsindex;
-	writeb(CCS_JOIN_NETWORK, &pccs->cmd);
-	writeb(0, &pccs->var.join_network.update_param);
-	writeb(0, &pccs->var.join_network.net_initiated);
-	/* Interrupt the firmware to process the command */
-	if (interrupt_ecf(local, ccsindex)) {
-		dev_dbg(&link->dev, "ray join net failed - card not ready for intr\n");
-		writeb(CCS_BUFFER_FREE, &(pccs++)->buffer_status);
-		return;
-	}
-	local->card_status = CARD_DOING_ACQ;
-}
-
-
-static void ray_release(struct pcmcia_device *link)
-{
-	struct net_device *dev = link->priv;
-	ray_dev_t *local = netdev_priv(dev);
-
-	dev_dbg(&link->dev, "ray_release\n");
-
-	del_timer_sync(&local->timer);
-
-	if (local->sram)
-		iounmap(local->sram);
-	if (local->rmem)
-		iounmap(local->rmem);
-	if (local->amem)
-		iounmap(local->amem);
-	pcmcia_disable_device(link);
-
-	dev_dbg(&link->dev, "ray_release ending\n");
-}
-
-static int ray_suspend(struct pcmcia_device *link)
-{
-	struct net_device *dev = link->priv;
-
-	if (link->open)
-		netif_device_detach(dev);
-
-	return 0;
-}
-
-static int ray_resume(struct pcmcia_device *link)
-{
-	struct net_device *dev = link->priv;
-
-	if (link->open) {
-		ray_reset(dev);
-		netif_device_attach(dev);
-	}
-
-	return 0;
-}
-
-/*===========================================================================*/
-static int ray_dev_init(struct net_device *dev)
-{
-#ifdef RAY_IMMEDIATE_INIT
-	int i;
-#endif /* RAY_IMMEDIATE_INIT */
-	ray_dev_t *local = netdev_priv(dev);
-	struct pcmcia_device *link = local->finder;
-
-	dev_dbg(&link->dev, "ray_dev_init(dev=%p)\n", dev);
-	if (!(pcmcia_dev_present(link))) {
-		dev_dbg(&link->dev, "ray_dev_init - device not present\n");
-		return -1;
-	}
-#ifdef RAY_IMMEDIATE_INIT
-	/* Download startup parameters */
-	if ((i = dl_startup_params(dev)) < 0) {
-		printk(KERN_INFO "ray_dev_init dl_startup_params failed - "
-		       "returns 0x%x\n", i);
-		return -1;
-	}
-#else /* RAY_IMMEDIATE_INIT */
-	/* Postpone the card init so that we can still configure the card,
-	 * for example using the Wireless Extensions. The init will happen
-	 * in ray_open() - Jean II */
-	dev_dbg(&link->dev,
-	      "ray_dev_init: postponing card init to ray_open() ; Status = %d\n",
-	      local->card_status);
-#endif /* RAY_IMMEDIATE_INIT */
-
-	/* copy mac and broadcast addresses to linux device */
-	eth_hw_addr_set(dev, local->sparm.b4.a_mac_addr);
-	eth_broadcast_addr(dev->broadcast);
-
-	dev_dbg(&link->dev, "ray_dev_init ending\n");
-	return 0;
-}
-
-/*===========================================================================*/
-static int ray_dev_config(struct net_device *dev, struct ifmap *map)
-{
-	ray_dev_t *local = netdev_priv(dev);
-	struct pcmcia_device *link = local->finder;
-	/* Dummy routine to satisfy device structure */
-	dev_dbg(&link->dev, "ray_dev_config(dev=%p,ifmap=%p)\n", dev, map);
-	if (!(pcmcia_dev_present(link))) {
-		dev_dbg(&link->dev, "ray_dev_config - device not present\n");
-		return -1;
-	}
-
-	return 0;
-}
-
-/*===========================================================================*/
-static netdev_tx_t ray_dev_start_xmit(struct sk_buff *skb,
-					    struct net_device *dev)
-{
-	ray_dev_t *local = netdev_priv(dev);
-	struct pcmcia_device *link = local->finder;
-	short length = skb->len;
-
-	if (!pcmcia_dev_present(link)) {
-		dev_dbg(&link->dev, "ray_dev_start_xmit - device not present\n");
-		dev_kfree_skb(skb);
-		return NETDEV_TX_OK;
-	}
-
-	dev_dbg(&link->dev, "ray_dev_start_xmit(skb=%p, dev=%p)\n", skb, dev);
-	if (local->authentication_state == NEED_TO_AUTH) {
-		dev_dbg(&link->dev, "ray_cs Sending authentication request.\n");
-		if (!build_auth_frame(local, local->auth_id, OPEN_AUTH_REQUEST)) {
-			local->authentication_state = AUTHENTICATED;
-			netif_stop_queue(dev);
-			return NETDEV_TX_BUSY;
-		}
-	}
-
-	if (length < ETH_ZLEN) {
-		if (skb_padto(skb, ETH_ZLEN))
-			return NETDEV_TX_OK;
-		length = ETH_ZLEN;
-	}
-	switch (ray_hw_xmit(skb->data, length, dev, DATA_TYPE)) {
-	case XMIT_NO_CCS:
-	case XMIT_NEED_AUTH:
-		netif_stop_queue(dev);
-		return NETDEV_TX_BUSY;
-	case XMIT_NO_INTR:
-	case XMIT_MSG_BAD:
-	case XMIT_OK:
-	default:
-		dev_kfree_skb(skb);
-	}
-
-	return NETDEV_TX_OK;
-} /* ray_dev_start_xmit */
-
-/*===========================================================================*/
-static int ray_hw_xmit(unsigned char *data, int len, struct net_device *dev,
-		       UCHAR msg_type)
-{
-	ray_dev_t *local = netdev_priv(dev);
-	struct ccs __iomem *pccs;
-	int ccsindex;
-	int offset;
-	struct tx_msg __iomem *ptx;	/* Address of xmit buffer in PC space */
-	short int addr;		/* Address of xmit buffer in card space */
-
-	pr_debug("ray_hw_xmit(data=%p, len=%d, dev=%p)\n", data, len, dev);
-	if (len + TX_HEADER_LENGTH > TX_BUF_SIZE) {
-		printk(KERN_INFO "ray_hw_xmit packet too large: %d bytes\n",
-		       len);
-		return XMIT_MSG_BAD;
-	}
-	switch (ccsindex = get_free_tx_ccs(local)) {
-	case ECCSBUSY:
-		pr_debug("ray_hw_xmit tx_ccs table busy\n");
-		fallthrough;
-	case ECCSFULL:
-		pr_debug("ray_hw_xmit No free tx ccs\n");
-		fallthrough;
-	case ECARDGONE:
-		netif_stop_queue(dev);
-		return XMIT_NO_CCS;
-	default:
-		break;
-	}
-	addr = TX_BUF_BASE + (ccsindex << 11);
-
-	if (msg_type == DATA_TYPE) {
-		local->stats.tx_bytes += len;
-		local->stats.tx_packets++;
-	}
-
-	ptx = local->sram + addr;
-
-	ray_build_header(local, ptx, msg_type, data);
-	if (translate) {
-		offset = translate_frame(local, ptx, data, len);
-	} else { /* Encapsulate frame */
-		/* TBD TIB length will move address of ptx->var */
-		memcpy_toio(&ptx->var, data, len);
-		offset = 0;
-	}
-
-	/* fill in the CCS */
-	pccs = ccs_base(local) + ccsindex;
-	len += TX_HEADER_LENGTH + offset;
-	writeb(CCS_TX_REQUEST, &pccs->cmd);
-	writeb(addr >> 8, &pccs->var.tx_request.tx_data_ptr[0]);
-	writeb(local->tib_length, &pccs->var.tx_request.tx_data_ptr[1]);
-	writeb(len >> 8, &pccs->var.tx_request.tx_data_length[0]);
-	writeb(len & 0xff, &pccs->var.tx_request.tx_data_length[1]);
-/* TBD still need psm_cam? */
-	writeb(PSM_CAM, &pccs->var.tx_request.pow_sav_mode);
-	writeb(local->net_default_tx_rate, &pccs->var.tx_request.tx_rate);
-	writeb(0, &pccs->var.tx_request.antenna);
-	pr_debug("ray_hw_xmit default_tx_rate = 0x%x\n",
-	      local->net_default_tx_rate);
-
-	/* Interrupt the firmware to process the command */
-	if (interrupt_ecf(local, ccsindex)) {
-		pr_debug("ray_hw_xmit failed - ECF not ready for intr\n");
-/* TBD very inefficient to copy packet to buffer, and then not
-   send it, but the alternative is to queue the messages and that
-   won't be done for a while.  Maybe set tbusy until a CCS is free?
-*/
-		writeb(CCS_BUFFER_FREE, &pccs->buffer_status);
-		return XMIT_NO_INTR;
-	}
-	return XMIT_OK;
-} /* end ray_hw_xmit */
-
-/*===========================================================================*/
-static int translate_frame(ray_dev_t *local, struct tx_msg __iomem *ptx,
-			   unsigned char *data, int len)
-{
-	__be16 proto = ((struct ethhdr *)data)->h_proto;
-	if (ntohs(proto) >= ETH_P_802_3_MIN) { /* DIX II ethernet frame */
-		pr_debug("ray_cs translate_frame DIX II\n");
-		/* Copy LLC header to card buffer */
-		memcpy_toio(&ptx->var, eth2_llc, sizeof(eth2_llc));
-		memcpy_toio(((void __iomem *)&ptx->var) + sizeof(eth2_llc),
-			    (UCHAR *) &proto, 2);
-		if (proto == htons(ETH_P_AARP) || proto == htons(ETH_P_IPX)) {
-			/* This is the selective translation table, only 2 entries */
-			writeb(0xf8,
-			       &((struct snaphdr_t __iomem *)ptx->var)->org[2]);
-		}
-		/* Copy body of ethernet packet without ethernet header */
-		memcpy_toio((void __iomem *)&ptx->var +
-			    sizeof(struct snaphdr_t), data + ETH_HLEN,
-			    len - ETH_HLEN);
-		return (int)sizeof(struct snaphdr_t) - ETH_HLEN;
-	} else { /* already  802 type, and proto is length */
-		pr_debug("ray_cs translate_frame 802\n");
-		if (proto == htons(0xffff)) { /* evil netware IPX 802.3 without LLC */
-			pr_debug("ray_cs translate_frame evil IPX\n");
-			memcpy_toio(&ptx->var, data + ETH_HLEN, len - ETH_HLEN);
-			return 0 - ETH_HLEN;
-		}
-		memcpy_toio(&ptx->var, data + ETH_HLEN, len - ETH_HLEN);
-		return 0 - ETH_HLEN;
-	}
-	/* TBD do other frame types */
-} /* end translate_frame */
-
-/*===========================================================================*/
-static void ray_build_header(ray_dev_t *local, struct tx_msg __iomem *ptx,
-			     UCHAR msg_type, unsigned char *data)
-{
-	writeb(PROTOCOL_VER | msg_type, &ptx->mac.frame_ctl_1);
-/*** IEEE 802.11 Address field assignments *************
-		TODS	FROMDS	addr_1		addr_2		addr_3	addr_4
-Adhoc		0	0	dest		src (terminal)	BSSID	N/A
-AP to Terminal	0	1	dest		AP(BSSID)	source	N/A
-Terminal to AP	1	0	AP(BSSID)	src (terminal)	dest	N/A
-AP to AP	1	1	dest AP		src AP		dest	source
-*******************************************************/
-	if (local->net_type == ADHOC) {
-		writeb(0, &ptx->mac.frame_ctl_2);
-		memcpy_toio(ptx->mac.addr_1, ((struct ethhdr *)data)->h_dest,
-			    ADDRLEN);
-		memcpy_toio(ptx->mac.addr_2, ((struct ethhdr *)data)->h_source,
-			    ADDRLEN);
-		memcpy_toio(ptx->mac.addr_3, local->bss_id, ADDRLEN);
-	} else { /* infrastructure */
-
-		if (local->sparm.b4.a_acting_as_ap_status) {
-			writeb(FC2_FROM_DS, &ptx->mac.frame_ctl_2);
-			memcpy_toio(ptx->mac.addr_1,
-				    ((struct ethhdr *)data)->h_dest, ADDRLEN);
-			memcpy_toio(ptx->mac.addr_2, local->bss_id, 6);
-			memcpy_toio(ptx->mac.addr_3,
-				    ((struct ethhdr *)data)->h_source, ADDRLEN);
-		} else { /* Terminal */
-
-			writeb(FC2_TO_DS, &ptx->mac.frame_ctl_2);
-			memcpy_toio(ptx->mac.addr_1, local->bss_id, ADDRLEN);
-			memcpy_toio(ptx->mac.addr_2,
-				    ((struct ethhdr *)data)->h_source, ADDRLEN);
-			memcpy_toio(ptx->mac.addr_3,
-				    ((struct ethhdr *)data)->h_dest, ADDRLEN);
-		}
-	}
-} /* end encapsulate_frame */
-
-/*====================================================================*/
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get protocol name
- */
-static int ray_get_name(struct net_device *dev, struct iw_request_info *info,
-			union iwreq_data *wrqu, char *extra)
-{
-	strcpy(wrqu->name, "IEEE 802.11-FH");
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : set frequency
- */
-static int ray_set_freq(struct net_device *dev, struct iw_request_info *info,
-			union iwreq_data *wrqu, char *extra)
-{
-	ray_dev_t *local = netdev_priv(dev);
-	int err = -EINPROGRESS;	/* Call commit handler */
-
-	/* Reject if card is already initialised */
-	if (local->card_status != CARD_AWAITING_PARAM)
-		return -EBUSY;
-
-	/* Setting by channel number */
-	if ((wrqu->freq.m > USA_HOP_MOD) || (wrqu->freq.e > 0))
-		err = -EOPNOTSUPP;
-	else
-		local->sparm.b5.a_hop_pattern = wrqu->freq.m;
-
-	return err;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get frequency
- */
-static int ray_get_freq(struct net_device *dev, struct iw_request_info *info,
-			union iwreq_data *wrqu, char *extra)
-{
-	ray_dev_t *local = netdev_priv(dev);
-
-	wrqu->freq.m = local->sparm.b5.a_hop_pattern;
-	wrqu->freq.e = 0;
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : set ESSID
- */
-static int ray_set_essid(struct net_device *dev, struct iw_request_info *info,
-			 union iwreq_data *wrqu, char *extra)
-{
-	ray_dev_t *local = netdev_priv(dev);
-
-	/* Reject if card is already initialised */
-	if (local->card_status != CARD_AWAITING_PARAM)
-		return -EBUSY;
-
-	/* Check if we asked for `any' */
-	if (wrqu->essid.flags == 0)
-		/* Corey : can you do that ? */
-		return -EOPNOTSUPP;
-
-	/* Check the size of the string */
-	if (wrqu->essid.length > IW_ESSID_MAX_SIZE)
-		return -E2BIG;
-
-	/* Set the ESSID in the card */
-	memset(local->sparm.b5.a_current_ess_id, 0, IW_ESSID_MAX_SIZE);
-	memcpy(local->sparm.b5.a_current_ess_id, extra, wrqu->essid.length);
-
-	return -EINPROGRESS;	/* Call commit handler */
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get ESSID
- */
-static int ray_get_essid(struct net_device *dev, struct iw_request_info *info,
-			 union iwreq_data *wrqu, char *extra)
-{
-	ray_dev_t *local = netdev_priv(dev);
-	UCHAR tmp[IW_ESSID_MAX_SIZE + 1];
-
-	/* Get the essid that was set */
-	memcpy(extra, local->sparm.b5.a_current_ess_id, IW_ESSID_MAX_SIZE);
-	memcpy(tmp, local->sparm.b5.a_current_ess_id, IW_ESSID_MAX_SIZE);
-	tmp[IW_ESSID_MAX_SIZE] = '\0';
-
-	/* Push it out ! */
-	wrqu->essid.length = strlen(tmp);
-	wrqu->essid.flags = 1;	/* active */
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get AP address
- */
-static int ray_get_wap(struct net_device *dev, struct iw_request_info *info,
-		       union iwreq_data *wrqu, char *extra)
-{
-	ray_dev_t *local = netdev_priv(dev);
-
-	memcpy(wrqu->ap_addr.sa_data, local->bss_id, ETH_ALEN);
-	wrqu->ap_addr.sa_family = ARPHRD_ETHER;
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : set Bit-Rate
- */
-static int ray_set_rate(struct net_device *dev, struct iw_request_info *info,
-			union iwreq_data *wrqu, char *extra)
-{
-	ray_dev_t *local = netdev_priv(dev);
-
-	/* Reject if card is already initialised */
-	if (local->card_status != CARD_AWAITING_PARAM)
-		return -EBUSY;
-
-	/* Check if rate is in range */
-	if ((wrqu->bitrate.value != 1000000) && (wrqu->bitrate.value != 2000000))
-		return -EINVAL;
-
-	/* Hack for 1.5 Mb/s instead of 2 Mb/s */
-	if ((local->fw_ver == 0x55) &&	/* Please check */
-	    (wrqu->bitrate.value == 2000000))
-		local->net_default_tx_rate = 3;
-	else
-		local->net_default_tx_rate = wrqu->bitrate.value / 500000;
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get Bit-Rate
- */
-static int ray_get_rate(struct net_device *dev, struct iw_request_info *info,
-			union iwreq_data *wrqu, char *extra)
-{
-	ray_dev_t *local = netdev_priv(dev);
-
-	if (local->net_default_tx_rate == 3)
-		wrqu->bitrate.value = 2000000;	/* Hum... */
-	else
-		wrqu->bitrate.value = local->net_default_tx_rate * 500000;
-	wrqu->bitrate.fixed = 0;	/* We are in auto mode */
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : set RTS threshold
- */
-static int ray_set_rts(struct net_device *dev, struct iw_request_info *info,
-		       union iwreq_data *wrqu, char *extra)
-{
-	ray_dev_t *local = netdev_priv(dev);
-	int rthr = wrqu->rts.value;
-
-	/* Reject if card is already initialised */
-	if (local->card_status != CARD_AWAITING_PARAM)
-		return -EBUSY;
-
-	/* if(wrq->u.rts.fixed == 0) we should complain */
-	if (wrqu->rts.disabled)
-		rthr = 32767;
-	else {
-		if ((rthr < 0) || (rthr > 2347))   /* What's the max packet size ??? */
-			return -EINVAL;
-	}
-	local->sparm.b5.a_rts_threshold[0] = (rthr >> 8) & 0xFF;
-	local->sparm.b5.a_rts_threshold[1] = rthr & 0xFF;
-
-	return -EINPROGRESS;	/* Call commit handler */
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get RTS threshold
- */
-static int ray_get_rts(struct net_device *dev, struct iw_request_info *info,
-		       union iwreq_data *wrqu, char *extra)
-{
-	ray_dev_t *local = netdev_priv(dev);
-
-	wrqu->rts.value = (local->sparm.b5.a_rts_threshold[0] << 8)
-	    + local->sparm.b5.a_rts_threshold[1];
-	wrqu->rts.disabled = (wrqu->rts.value == 32767);
-	wrqu->rts.fixed = 1;
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : set Fragmentation threshold
- */
-static int ray_set_frag(struct net_device *dev, struct iw_request_info *info,
-			union iwreq_data *wrqu, char *extra)
-{
-	ray_dev_t *local = netdev_priv(dev);
-	int fthr = wrqu->frag.value;
-
-	/* Reject if card is already initialised */
-	if (local->card_status != CARD_AWAITING_PARAM)
-		return -EBUSY;
-
-	/* if(wrq->u.frag.fixed == 0) should complain */
-	if (wrqu->frag.disabled)
-		fthr = 32767;
-	else {
-		if ((fthr < 256) || (fthr > 2347))	/* To check out ! */
-			return -EINVAL;
-	}
-	local->sparm.b5.a_frag_threshold[0] = (fthr >> 8) & 0xFF;
-	local->sparm.b5.a_frag_threshold[1] = fthr & 0xFF;
-
-	return -EINPROGRESS;	/* Call commit handler */
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get Fragmentation threshold
- */
-static int ray_get_frag(struct net_device *dev, struct iw_request_info *info,
-			union iwreq_data *wrqu, char *extra)
-{
-	ray_dev_t *local = netdev_priv(dev);
-
-	wrqu->frag.value = (local->sparm.b5.a_frag_threshold[0] << 8)
-	    + local->sparm.b5.a_frag_threshold[1];
-	wrqu->frag.disabled = (wrqu->frag.value == 32767);
-	wrqu->frag.fixed = 1;
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : set Mode of Operation
- */
-static int ray_set_mode(struct net_device *dev, struct iw_request_info *info,
-			union iwreq_data *wrqu, char *extra)
-{
-	ray_dev_t *local = netdev_priv(dev);
-	int err = -EINPROGRESS;	/* Call commit handler */
-	char card_mode = 1;
-
-	/* Reject if card is already initialised */
-	if (local->card_status != CARD_AWAITING_PARAM)
-		return -EBUSY;
-
-	switch (wrqu->mode) {
-	case IW_MODE_ADHOC:
-		card_mode = 0;
-		fallthrough;
-	case IW_MODE_INFRA:
-		local->sparm.b5.a_network_type = card_mode;
-		break;
-	default:
-		err = -EINVAL;
-	}
-
-	return err;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get Mode of Operation
- */
-static int ray_get_mode(struct net_device *dev, struct iw_request_info *info,
-			union iwreq_data *wrqu, char *extra)
-{
-	ray_dev_t *local = netdev_priv(dev);
-
-	if (local->sparm.b5.a_network_type)
-		wrqu->mode = IW_MODE_INFRA;
-	else
-		wrqu->mode = IW_MODE_ADHOC;
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Handler : get range info
- */
-static int ray_get_range(struct net_device *dev, struct iw_request_info *info,
-			 union iwreq_data *wrqu, char *extra)
-{
-	struct iw_range *range = (struct iw_range *)extra;
-
-	memset(range, 0, sizeof(struct iw_range));
-
-	/* Set the length (very important for backward compatibility) */
-	wrqu->data.length = sizeof(struct iw_range);
-
-	/* Set the Wireless Extension versions */
-	range->we_version_compiled = WIRELESS_EXT;
-	range->we_version_source = 9;
-
-	/* Set information in the range struct */
-	range->throughput = 1.1 * 1000 * 1000;	/* Put the right number here */
-	range->num_channels = hop_pattern_length[(int)country];
-	range->num_frequency = 0;
-	range->max_qual.qual = 0;
-	range->max_qual.level = 255;	/* What's the correct value ? */
-	range->max_qual.noise = 255;	/* Idem */
-	range->num_bitrates = 2;
-	range->bitrate[0] = 1000000;	/* 1 Mb/s */
-	range->bitrate[1] = 2000000;	/* 2 Mb/s */
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Private Handler : set framing mode
- */
-static int ray_set_framing(struct net_device *dev, struct iw_request_info *info,
-			   union iwreq_data *wrqu, char *extra)
-{
-	translate = !!*(extra);	/* Set framing mode */
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Private Handler : get framing mode
- */
-static int ray_get_framing(struct net_device *dev, struct iw_request_info *info,
-			   union iwreq_data *wrqu, char *extra)
-{
-	*(extra) = translate;
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Wireless Private Handler : get country
- */
-static int ray_get_country(struct net_device *dev, struct iw_request_info *info,
-			   union iwreq_data *wrqu, char *extra)
-{
-	*(extra) = country;
-
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Commit handler : called after a bunch of SET operations
- */
-static int ray_commit(struct net_device *dev, struct iw_request_info *info,
-		      union iwreq_data *wrqu, char *extra)
-{
-	return 0;
-}
-
-/*------------------------------------------------------------------*/
-/*
- * Stats handler : return Wireless Stats
- */
-static iw_stats *ray_get_wireless_stats(struct net_device *dev)
-{
-	ray_dev_t *local = netdev_priv(dev);
-	struct pcmcia_device *link = local->finder;
-	struct status __iomem *p = local->sram + STATUS_BASE;
-
-	local->wstats.status = local->card_status;
-#ifdef WIRELESS_SPY
-	if ((local->spy_data.spy_number > 0)
-	    && (local->sparm.b5.a_network_type == 0)) {
-		/* Get it from the first node in spy list */
-		local->wstats.qual.qual = local->spy_data.spy_stat[0].qual;
-		local->wstats.qual.level = local->spy_data.spy_stat[0].level;
-		local->wstats.qual.noise = local->spy_data.spy_stat[0].noise;
-		local->wstats.qual.updated =
-		    local->spy_data.spy_stat[0].updated;
-	}
-#endif /* WIRELESS_SPY */
-
-	if (pcmcia_dev_present(link)) {
-		local->wstats.qual.noise = readb(&p->rxnoise);
-		local->wstats.qual.updated |= 4;
-	}
-
-	return &local->wstats;
-} /* end ray_get_wireless_stats */
-
-/*------------------------------------------------------------------*/
-/*
- * Structures to export the Wireless Handlers
- */
-
-static const iw_handler ray_handler[] = {
-	IW_HANDLER(SIOCSIWCOMMIT, ray_commit),
-	IW_HANDLER(SIOCGIWNAME, ray_get_name),
-	IW_HANDLER(SIOCSIWFREQ, ray_set_freq),
-	IW_HANDLER(SIOCGIWFREQ, ray_get_freq),
-	IW_HANDLER(SIOCSIWMODE, ray_set_mode),
-	IW_HANDLER(SIOCGIWMODE, ray_get_mode),
-	IW_HANDLER(SIOCGIWRANGE, ray_get_range),
-#ifdef WIRELESS_SPY
-	IW_HANDLER(SIOCSIWSPY, iw_handler_set_spy),
-	IW_HANDLER(SIOCGIWSPY, iw_handler_get_spy),
-	IW_HANDLER(SIOCSIWTHRSPY, iw_handler_set_thrspy),
-	IW_HANDLER(SIOCGIWTHRSPY, iw_handler_get_thrspy),
-#endif /* WIRELESS_SPY */
-	IW_HANDLER(SIOCGIWAP, ray_get_wap),
-	IW_HANDLER(SIOCSIWESSID, ray_set_essid),
-	IW_HANDLER(SIOCGIWESSID, ray_get_essid),
-	IW_HANDLER(SIOCSIWRATE, ray_set_rate),
-	IW_HANDLER(SIOCGIWRATE, ray_get_rate),
-	IW_HANDLER(SIOCSIWRTS, ray_set_rts),
-	IW_HANDLER(SIOCGIWRTS, ray_get_rts),
-	IW_HANDLER(SIOCSIWFRAG, ray_set_frag),
-	IW_HANDLER(SIOCGIWFRAG, ray_get_frag),
-};
-
-#define SIOCSIPFRAMING	SIOCIWFIRSTPRIV	/* Set framing mode */
-#define SIOCGIPFRAMING	SIOCIWFIRSTPRIV + 1	/* Get framing mode */
-#define SIOCGIPCOUNTRY	SIOCIWFIRSTPRIV + 3	/* Get country code */
-
-static const iw_handler ray_private_handler[] = {
-	[0] = ray_set_framing,
-	[1] = ray_get_framing,
-	[3] = ray_get_country,
-};
-
-static const struct iw_priv_args ray_private_args[] = {
-/* cmd,		set_args,	get_args,	name */
-	{SIOCSIPFRAMING, IW_PRIV_TYPE_BYTE | IW_PRIV_SIZE_FIXED | 1, 0,
-	 "set_framing"},
-	{SIOCGIPFRAMING, 0, IW_PRIV_TYPE_BYTE | IW_PRIV_SIZE_FIXED | 1,
-	 "get_framing"},
-	{SIOCGIPCOUNTRY, 0, IW_PRIV_TYPE_BYTE | IW_PRIV_SIZE_FIXED | 1,
-	 "get_country"},
-};
-
-static const struct iw_handler_def ray_handler_def = {
-	.num_standard = ARRAY_SIZE(ray_handler),
-	.num_private = ARRAY_SIZE(ray_private_handler),
-	.num_private_args = ARRAY_SIZE(ray_private_args),
-	.standard = ray_handler,
-	.private = ray_private_handler,
-	.private_args = ray_private_args,
-	.get_wireless_stats = ray_get_wireless_stats,
-};
-
-/*===========================================================================*/
-static int ray_open(struct net_device *dev)
-{
-	ray_dev_t *local = netdev_priv(dev);
-	struct pcmcia_device *link;
-	link = local->finder;
-
-	dev_dbg(&link->dev, "ray_open('%s')\n", dev->name);
-
-	if (link->open == 0)
-		local->num_multi = 0;
-	link->open++;
-
-	/* If the card is not started, time to start it ! - Jean II */
-	if (local->card_status == CARD_AWAITING_PARAM) {
-		int i;
-
-		dev_dbg(&link->dev, "ray_open: doing init now !\n");
-
-		/* Download startup parameters */
-		if ((i = dl_startup_params(dev)) < 0) {
-			printk(KERN_INFO
-			       "ray_dev_init dl_startup_params failed - "
-			       "returns 0x%x\n", i);
-			return -1;
-		}
-	}
-
-	if (sniffer)
-		netif_stop_queue(dev);
-	else
-		netif_start_queue(dev);
-
-	dev_dbg(&link->dev, "ray_open ending\n");
-	return 0;
-} /* end ray_open */
-
-/*===========================================================================*/
-static int ray_dev_close(struct net_device *dev)
-{
-	ray_dev_t *local = netdev_priv(dev);
-	struct pcmcia_device *link;
-	link = local->finder;
-
-	dev_dbg(&link->dev, "ray_dev_close('%s')\n", dev->name);
-
-	link->open--;
-	netif_stop_queue(dev);
-
-	/* In here, we should stop the hardware (stop card from beeing active)
-	 * and set local->card_status to CARD_AWAITING_PARAM, so that while the
-	 * card is closed we can chage its configuration.
-	 * Probably also need a COR reset to get sane state - Jean II */
-
-	return 0;
-} /* end ray_dev_close */
-
-/*===========================================================================*/
-static void ray_reset(struct net_device *dev)
-{
-	pr_debug("ray_reset entered\n");
-}
-
-/*===========================================================================*/
-/* Cause a firmware interrupt if it is ready for one                         */
-/* Return nonzero if not ready                                               */
-static int interrupt_ecf(ray_dev_t *local, int ccs)
-{
-	int i = 50;
-	struct pcmcia_device *link = local->finder;
-
-	if (!(pcmcia_dev_present(link))) {
-		dev_dbg(&link->dev, "ray_cs interrupt_ecf - device not present\n");
-		return -1;
-	}
-	dev_dbg(&link->dev, "interrupt_ecf(local=%p, ccs = 0x%x\n", local, ccs);
-
-	while (i &&
-	       (readb(local->amem + CIS_OFFSET + ECF_INTR_OFFSET) &
-		ECF_INTR_SET))
-		i--;
-	if (i == 0) {
-		dev_dbg(&link->dev, "ray_cs interrupt_ecf card not ready for interrupt\n");
-		return -1;
-	}
-	/* Fill the mailbox, then kick the card */
-	writeb(ccs, local->sram + SCB_BASE);
-	writeb(ECF_INTR_SET, local->amem + CIS_OFFSET + ECF_INTR_OFFSET);
-	return 0;
-} /* interrupt_ecf */
-
-/*===========================================================================*/
-/* Get next free transmit CCS                                                */
-/* Return - index of current tx ccs                                          */
-static int get_free_tx_ccs(ray_dev_t *local)
-{
-	int i;
-	struct ccs __iomem *pccs = ccs_base(local);
-	struct pcmcia_device *link = local->finder;
-
-	if (!(pcmcia_dev_present(link))) {
-		dev_dbg(&link->dev, "ray_cs get_free_tx_ccs - device not present\n");
-		return ECARDGONE;
-	}
-
-	if (test_and_set_bit(0, &local->tx_ccs_lock)) {
-		dev_dbg(&link->dev, "ray_cs tx_ccs_lock busy\n");
-		return ECCSBUSY;
-	}
-
-	for (i = 0; i < NUMBER_OF_TX_CCS; i++) {
-		if (readb(&(pccs + i)->buffer_status) == CCS_BUFFER_FREE) {
-			writeb(CCS_BUFFER_BUSY, &(pccs + i)->buffer_status);
-			writeb(CCS_END_LIST, &(pccs + i)->link);
-			local->tx_ccs_lock = 0;
-			return i;
-		}
-	}
-	local->tx_ccs_lock = 0;
-	dev_dbg(&link->dev, "ray_cs ERROR no free tx CCS for raylink card\n");
-	return ECCSFULL;
-} /* get_free_tx_ccs */
-
-/*===========================================================================*/
-/* Get next free CCS                                                         */
-/* Return - index of current ccs                                             */
-static int get_free_ccs(ray_dev_t *local)
-{
-	int i;
-	struct ccs __iomem *pccs = ccs_base(local);
-	struct pcmcia_device *link = local->finder;
-
-	if (!(pcmcia_dev_present(link))) {
-		dev_dbg(&link->dev, "ray_cs get_free_ccs - device not present\n");
-		return ECARDGONE;
-	}
-	if (test_and_set_bit(0, &local->ccs_lock)) {
-		dev_dbg(&link->dev, "ray_cs ccs_lock busy\n");
-		return ECCSBUSY;
-	}
-
-	for (i = NUMBER_OF_TX_CCS; i < NUMBER_OF_CCS; i++) {
-		if (readb(&(pccs + i)->buffer_status) == CCS_BUFFER_FREE) {
-			writeb(CCS_BUFFER_BUSY, &(pccs + i)->buffer_status);
-			writeb(CCS_END_LIST, &(pccs + i)->link);
-			local->ccs_lock = 0;
-			return i;
-		}
-	}
-	local->ccs_lock = 0;
-	dev_dbg(&link->dev, "ray_cs ERROR no free CCS for raylink card\n");
-	return ECCSFULL;
-} /* get_free_ccs */
-
-/*===========================================================================*/
-static void authenticate_timeout(struct timer_list *t)
-{
-	ray_dev_t *local = from_timer(local, t, timer);
-	del_timer(&local->timer);
-	printk(KERN_INFO "ray_cs Authentication with access point failed"
-	       " - timeout\n");
-	join_net(&local->timer);
-}
-
-/*===========================================================================*/
-static int parse_addr(char *in_str, UCHAR *out)
-{
-	int i, k;
-	int len;
-
-	if (in_str == NULL)
-		return 0;
-	len = strnlen(in_str, ADDRLEN * 2 + 1) - 1;
-	if (len < 1)
-		return 0;
-	memset(out, 0, ADDRLEN);
-
-	i = 5;
-
-	while (len > 0) {
-		if ((k = hex_to_bin(in_str[len--])) != -1)
-			out[i] = k;
-		else
-			return 0;
-
-		if (len == 0)
-			break;
-		if ((k = hex_to_bin(in_str[len--])) != -1)
-			out[i] += k << 4;
-		else
-			return 0;
-		if (!i--)
-			break;
-	}
-	return 1;
-}
-
-/*===========================================================================*/
-static struct net_device_stats *ray_get_stats(struct net_device *dev)
-{
-	ray_dev_t *local = netdev_priv(dev);
-	struct pcmcia_device *link = local->finder;
-	struct status __iomem *p = local->sram + STATUS_BASE;
-	if (!(pcmcia_dev_present(link))) {
-		dev_dbg(&link->dev, "ray_cs net_device_stats - device not present\n");
-		return &local->stats;
-	}
-	if (readb(&p->mrx_overflow_for_host)) {
-		local->stats.rx_over_errors += swab16(readw(&p->mrx_overflow));
-		writeb(0, &p->mrx_overflow);
-		writeb(0, &p->mrx_overflow_for_host);
-	}
-	if (readb(&p->mrx_checksum_error_for_host)) {
-		local->stats.rx_crc_errors +=
-		    swab16(readw(&p->mrx_checksum_error));
-		writeb(0, &p->mrx_checksum_error);
-		writeb(0, &p->mrx_checksum_error_for_host);
-	}
-	if (readb(&p->rx_hec_error_for_host)) {
-		local->stats.rx_frame_errors += swab16(readw(&p->rx_hec_error));
-		writeb(0, &p->rx_hec_error);
-		writeb(0, &p->rx_hec_error_for_host);
-	}
-	return &local->stats;
-}
-
-/*===========================================================================*/
-static void ray_update_parm(struct net_device *dev, UCHAR objid, UCHAR *value,
-			    int len)
-{
-	ray_dev_t *local = netdev_priv(dev);
-	struct pcmcia_device *link = local->finder;
-	int ccsindex;
-	int i;
-	struct ccs __iomem *pccs;
-
-	if (!(pcmcia_dev_present(link))) {
-		dev_dbg(&link->dev, "ray_update_parm - device not present\n");
-		return;
-	}
-
-	if ((ccsindex = get_free_ccs(local)) < 0) {
-		dev_dbg(&link->dev, "ray_update_parm - No free ccs\n");
-		return;
-	}
-	pccs = ccs_base(local) + ccsindex;
-	writeb(CCS_UPDATE_PARAMS, &pccs->cmd);
-	writeb(objid, &pccs->var.update_param.object_id);
-	writeb(1, &pccs->var.update_param.number_objects);
-	writeb(0, &pccs->var.update_param.failure_cause);
-	for (i = 0; i < len; i++) {
-		writeb(value[i], local->sram + HOST_TO_ECF_BASE);
-	}
-	/* Interrupt the firmware to process the command */
-	if (interrupt_ecf(local, ccsindex)) {
-		dev_dbg(&link->dev, "ray_cs associate failed - ECF not ready for intr\n");
-		writeb(CCS_BUFFER_FREE, &(pccs++)->buffer_status);
-	}
-}
-
-/*===========================================================================*/
-static void ray_update_multi_list(struct net_device *dev, int all)
-{
-	int ccsindex;
-	struct ccs __iomem *pccs;
-	ray_dev_t *local = netdev_priv(dev);
-	struct pcmcia_device *link = local->finder;
-	void __iomem *p = local->sram + HOST_TO_ECF_BASE;
-
-	if (!(pcmcia_dev_present(link))) {
-		dev_dbg(&link->dev, "ray_update_multi_list - device not present\n");
-		return;
-	} else
-		dev_dbg(&link->dev, "ray_update_multi_list(%p)\n", dev);
-	if ((ccsindex = get_free_ccs(local)) < 0) {
-		dev_dbg(&link->dev, "ray_update_multi - No free ccs\n");
-		return;
-	}
-	pccs = ccs_base(local) + ccsindex;
-	writeb(CCS_UPDATE_MULTICAST_LIST, &pccs->cmd);
-
-	if (all) {
-		writeb(0xff, &pccs->var);
-		local->num_multi = 0xff;
-	} else {
-		struct netdev_hw_addr *ha;
-		int i = 0;
-
-		/* Copy the kernel's list of MC addresses to card */
-		netdev_for_each_mc_addr(ha, dev) {
-			memcpy_toio(p, ha->addr, ETH_ALEN);
-			dev_dbg(&link->dev, "ray_update_multi add addr %pm\n",
-				ha->addr);
-			p += ETH_ALEN;
-			i++;
-		}
-		if (i > 256 / ADDRLEN)
-			i = 256 / ADDRLEN;
-		writeb((UCHAR) i, &pccs->var);
-		dev_dbg(&link->dev, "ray_cs update_multi %d addresses in list\n", i);
-		/* Interrupt the firmware to process the command */
-		local->num_multi = i;
-	}
-	if (interrupt_ecf(local, ccsindex)) {
-		dev_dbg(&link->dev,
-		      "ray_cs update_multi failed - ECF not ready for intr\n");
-		writeb(CCS_BUFFER_FREE, &(pccs++)->buffer_status);
-	}
-} /* end ray_update_multi_list */
-
-/*===========================================================================*/
-static void set_multicast_list(struct net_device *dev)
-{
-	ray_dev_t *local = netdev_priv(dev);
-	UCHAR promisc;
-
-	pr_debug("ray_cs set_multicast_list(%p)\n", dev);
-
-	if (dev->flags & IFF_PROMISC) {
-		if (local->sparm.b5.a_promiscuous_mode == 0) {
-			pr_debug("ray_cs set_multicast_list promisc on\n");
-			local->sparm.b5.a_promiscuous_mode = 1;
-			promisc = 1;
-			ray_update_parm(dev, OBJID_promiscuous_mode,
-					&promisc, sizeof(promisc));
-		}
-	} else {
-		if (local->sparm.b5.a_promiscuous_mode == 1) {
-			pr_debug("ray_cs set_multicast_list promisc off\n");
-			local->sparm.b5.a_promiscuous_mode = 0;
-			promisc = 0;
-			ray_update_parm(dev, OBJID_promiscuous_mode,
-					&promisc, sizeof(promisc));
-		}
-	}
-
-	if (dev->flags & IFF_ALLMULTI)
-		ray_update_multi_list(dev, 1);
-	else {
-		if (local->num_multi != netdev_mc_count(dev))
-			ray_update_multi_list(dev, 0);
-	}
-} /* end set_multicast_list */
-
-/*=============================================================================
- * All routines below here are run at interrupt time.
-=============================================================================*/
-static irqreturn_t ray_interrupt(int irq, void *dev_id)
-{
-	struct net_device *dev = dev_id;
-	struct pcmcia_device *link;
-	ray_dev_t *local;
-	struct ccs __iomem *pccs;
-	struct rcs __iomem *prcs;
-	UCHAR rcsindex;
-	UCHAR tmp;
-	UCHAR cmd;
-	UCHAR status;
-	UCHAR memtmp[ESSID_SIZE + 1];
-
-
-	if (dev == NULL)	/* Note that we want interrupts with dev->start == 0 */
-		return IRQ_NONE;
-
-	pr_debug("ray_cs: interrupt for *dev=%p\n", dev);
-
-	local = netdev_priv(dev);
-	link = local->finder;
-	if (!pcmcia_dev_present(link)) {
-		pr_debug(
-			"ray_cs interrupt from device not present or suspended.\n");
-		return IRQ_NONE;
-	}
-	rcsindex = readb(&((struct scb __iomem *)(local->sram))->rcs_index);
-
-	if (rcsindex >= (NUMBER_OF_CCS + NUMBER_OF_RCS)) {
-		dev_dbg(&link->dev, "ray_cs interrupt bad rcsindex = 0x%x\n", rcsindex);
-		clear_interrupt(local);
-		return IRQ_HANDLED;
-	}
-	if (rcsindex < NUMBER_OF_CCS) { /* If it's a returned CCS */
-		pccs = ccs_base(local) + rcsindex;
-		cmd = readb(&pccs->cmd);
-		status = readb(&pccs->buffer_status);
-		switch (cmd) {
-		case CCS_DOWNLOAD_STARTUP_PARAMS:	/* Happens in firmware someday */
-			del_timer(&local->timer);
-			if (status == CCS_COMMAND_COMPLETE) {
-				dev_dbg(&link->dev,
-				      "ray_cs interrupt download_startup_parameters OK\n");
-			} else {
-				dev_dbg(&link->dev,
-				      "ray_cs interrupt download_startup_parameters fail\n");
-			}
-			break;
-		case CCS_UPDATE_PARAMS:
-			dev_dbg(&link->dev, "ray_cs interrupt update params done\n");
-			if (status != CCS_COMMAND_COMPLETE) {
-				tmp =
-				    readb(&pccs->var.update_param.
-					  failure_cause);
-				dev_dbg(&link->dev,
-				      "ray_cs interrupt update params failed - reason %d\n",
-				      tmp);
-			}
-			break;
-		case CCS_REPORT_PARAMS:
-			dev_dbg(&link->dev, "ray_cs interrupt report params done\n");
-			break;
-		case CCS_UPDATE_MULTICAST_LIST:	/* Note that this CCS isn't returned */
-			dev_dbg(&link->dev,
-			      "ray_cs interrupt CCS Update Multicast List done\n");
-			break;
-		case CCS_UPDATE_POWER_SAVINGS_MODE:
-			dev_dbg(&link->dev,
-			      "ray_cs interrupt update power save mode done\n");
-			break;
-		case CCS_START_NETWORK:
-		case CCS_JOIN_NETWORK:
-			memcpy(memtmp, local->sparm.b4.a_current_ess_id,
-								ESSID_SIZE);
-			memtmp[ESSID_SIZE] = '\0';
-
-			if (status == CCS_COMMAND_COMPLETE) {
-				if (readb
-				    (&pccs->var.start_network.net_initiated) ==
-				    1) {
-					dev_dbg(&link->dev,
-					      "ray_cs interrupt network \"%s\" started\n",
-					      memtmp);
-				} else {
-					dev_dbg(&link->dev,
-					      "ray_cs interrupt network \"%s\" joined\n",
-					      memtmp);
-				}
-				memcpy_fromio(&local->bss_id,
-					      pccs->var.start_network.bssid,
-					      ADDRLEN);
-
-				if (local->fw_ver == 0x55)
-					local->net_default_tx_rate = 3;
-				else
-					local->net_default_tx_rate =
-					    readb(&pccs->var.start_network.
-						  net_default_tx_rate);
-				local->encryption =
-				    readb(&pccs->var.start_network.encryption);
-				if (!sniffer && (local->net_type == INFRA)
-				    && !(local->sparm.b4.a_acting_as_ap_status)) {
-					authenticate(local);
-				}
-				local->card_status = CARD_ACQ_COMPLETE;
-			} else {
-				local->card_status = CARD_ACQ_FAILED;
-
-				del_timer(&local->timer);
-				local->timer.expires = jiffies + HZ * 5;
-				if (status == CCS_START_NETWORK) {
-					dev_dbg(&link->dev,
-					      "ray_cs interrupt network \"%s\" start failed\n",
-					      memtmp);
-					local->timer.function = start_net;
-				} else {
-					dev_dbg(&link->dev,
-					      "ray_cs interrupt network \"%s\" join failed\n",
-					      memtmp);
-					local->timer.function = join_net;
-				}
-				add_timer(&local->timer);
-			}
-			break;
-		case CCS_START_ASSOCIATION:
-			if (status == CCS_COMMAND_COMPLETE) {
-				local->card_status = CARD_ASSOC_COMPLETE;
-				dev_dbg(&link->dev, "ray_cs association successful\n");
-			} else {
-				dev_dbg(&link->dev, "ray_cs association failed,\n");
-				local->card_status = CARD_ASSOC_FAILED;
-				join_net(&local->timer);
-			}
-			break;
-		case CCS_TX_REQUEST:
-			if (status == CCS_COMMAND_COMPLETE) {
-				dev_dbg(&link->dev,
-				      "ray_cs interrupt tx request complete\n");
-			} else {
-				dev_dbg(&link->dev,
-				      "ray_cs interrupt tx request failed\n");
-			}
-			if (!sniffer)
-				netif_start_queue(dev);
-			netif_wake_queue(dev);
-			break;
-		case CCS_TEST_MEMORY:
-			dev_dbg(&link->dev, "ray_cs interrupt mem test done\n");
-			break;
-		case CCS_SHUTDOWN:
-			dev_dbg(&link->dev,
-			      "ray_cs interrupt Unexpected CCS returned - Shutdown\n");
-			break;
-		case CCS_DUMP_MEMORY:
-			dev_dbg(&link->dev, "ray_cs interrupt dump memory done\n");
-			break;
-		case CCS_START_TIMER:
-			dev_dbg(&link->dev,
-			      "ray_cs interrupt DING - raylink timer expired\n");
-			break;
-		default:
-			dev_dbg(&link->dev,
-			      "ray_cs interrupt Unexpected CCS 0x%x returned 0x%x\n",
-			      rcsindex, cmd);
-		}
-		writeb(CCS_BUFFER_FREE, &pccs->buffer_status);
-	} else { /* It's an RCS */
-
-		prcs = rcs_base(local) + rcsindex;
-
-		switch (readb(&prcs->interrupt_id)) {
-		case PROCESS_RX_PACKET:
-			ray_rx(dev, local, prcs);
-			break;
-		case REJOIN_NET_COMPLETE:
-			dev_dbg(&link->dev, "ray_cs interrupt rejoin net complete\n");
-			local->card_status = CARD_ACQ_COMPLETE;
-			/* do we need to clear tx buffers CCS's? */
-			if (local->sparm.b4.a_network_type == ADHOC) {
-				if (!sniffer)
-					netif_start_queue(dev);
-			} else {
-				memcpy_fromio(&local->bss_id,
-					      prcs->var.rejoin_net_complete.
-					      bssid, ADDRLEN);
-				dev_dbg(&link->dev, "ray_cs new BSSID = %pm\n",
-					local->bss_id);
-				if (!sniffer)
-					authenticate(local);
-			}
-			break;
-		case ROAMING_INITIATED:
-			dev_dbg(&link->dev, "ray_cs interrupt roaming initiated\n");
-			netif_stop_queue(dev);
-			local->card_status = CARD_DOING_ACQ;
-			break;
-		case JAPAN_CALL_SIGN_RXD:
-			dev_dbg(&link->dev, "ray_cs interrupt japan call sign rx\n");
-			break;
-		default:
-			dev_dbg(&link->dev,
-			      "ray_cs Unexpected interrupt for RCS 0x%x cmd = 0x%x\n",
-			      rcsindex,
-			      (unsigned int)readb(&prcs->interrupt_id));
-			break;
-		}
-		writeb(CCS_BUFFER_FREE, &prcs->buffer_status);
-	}
-	clear_interrupt(local);
-	return IRQ_HANDLED;
-} /* ray_interrupt */
-
-/*===========================================================================*/
-static void ray_rx(struct net_device *dev, ray_dev_t *local,
-		   struct rcs __iomem *prcs)
-{
-	int rx_len;
-	unsigned int pkt_addr;
-	void __iomem *pmsg;
-	pr_debug("ray_rx process rx packet\n");
-
-	/* Calculate address of packet within Rx buffer */
-	pkt_addr = ((readb(&prcs->var.rx_packet.rx_data_ptr[0]) << 8)
-		    + readb(&prcs->var.rx_packet.rx_data_ptr[1])) & RX_BUFF_END;
-	/* Length of first packet fragment */
-	rx_len = (readb(&prcs->var.rx_packet.rx_data_length[0]) << 8)
-	    + readb(&prcs->var.rx_packet.rx_data_length[1]);
-
-	local->last_rsl = readb(&prcs->var.rx_packet.rx_sig_lev);
-	pmsg = local->rmem + pkt_addr;
-	switch (readb(pmsg)) {
-	case DATA_TYPE:
-		pr_debug("ray_rx data type\n");
-		rx_data(dev, prcs, pkt_addr, rx_len);
-		break;
-	case AUTHENTIC_TYPE:
-		pr_debug("ray_rx authentic type\n");
-		if (sniffer)
-			rx_data(dev, prcs, pkt_addr, rx_len);
-		else
-			rx_authenticate(local, prcs, pkt_addr, rx_len);
-		break;
-	case DEAUTHENTIC_TYPE:
-		pr_debug("ray_rx deauth type\n");
-		if (sniffer)
-			rx_data(dev, prcs, pkt_addr, rx_len);
-		else
-			rx_deauthenticate(local, prcs, pkt_addr, rx_len);
-		break;
-	case NULL_MSG_TYPE:
-		pr_debug("ray_cs rx NULL msg\n");
-		break;
-	case BEACON_TYPE:
-		pr_debug("ray_rx beacon type\n");
-		if (sniffer)
-			rx_data(dev, prcs, pkt_addr, rx_len);
-
-		copy_from_rx_buff(local, (UCHAR *) &local->last_bcn, pkt_addr,
-				  rx_len < sizeof(struct beacon_rx) ?
-				  rx_len : sizeof(struct beacon_rx));
-
-		local->beacon_rxed = 1;
-		/* Get the statistics so the card counters never overflow */
-		ray_get_stats(dev);
-		break;
-	default:
-		pr_debug("ray_cs unknown pkt type %2x\n",
-		      (unsigned int)readb(pmsg));
-		break;
-	}
-
-} /* end ray_rx */
-
-/*===========================================================================*/
-static void rx_data(struct net_device *dev, struct rcs __iomem *prcs,
-		    unsigned int pkt_addr, int rx_len)
-{
-	struct sk_buff *skb = NULL;
-	struct rcs __iomem *prcslink = prcs;
-	ray_dev_t *local = netdev_priv(dev);
-	UCHAR *rx_ptr;
-	int total_len;
-	int tmp;
-#ifdef WIRELESS_SPY
-	int siglev = local->last_rsl;
-	u_char linksrcaddr[ETH_ALEN];	/* Other end of the wireless link */
-#endif
-
-	if (!sniffer) {
-		if (translate) {
-/* TBD length needs fixing for translated header */
-			if (rx_len < (ETH_HLEN + RX_MAC_HEADER_LENGTH) ||
-			    rx_len >
-			    (dev->mtu + RX_MAC_HEADER_LENGTH + ETH_HLEN +
-			     FCS_LEN)) {
-				pr_debug(
-				      "ray_cs invalid packet length %d received\n",
-				      rx_len);
-				return;
-			}
-		} else { /* encapsulated ethernet */
-
-			if (rx_len < (ETH_HLEN + RX_MAC_HEADER_LENGTH) ||
-			    rx_len >
-			    (dev->mtu + RX_MAC_HEADER_LENGTH + ETH_HLEN +
-			     FCS_LEN)) {
-				pr_debug(
-				      "ray_cs invalid packet length %d received\n",
-				      rx_len);
-				return;
-			}
-		}
-	}
-	pr_debug("ray_cs rx_data packet\n");
-	/* If fragmented packet, verify sizes of fragments add up */
-	if (readb(&prcs->var.rx_packet.next_frag_rcs_index) != 0xFF) {
-		pr_debug("ray_cs rx'ed fragment\n");
-		tmp = (readb(&prcs->var.rx_packet.totalpacketlength[0]) << 8)
-		    + readb(&prcs->var.rx_packet.totalpacketlength[1]);
-		total_len = tmp;
-		prcslink = prcs;
-		do {
-			tmp -=
-			    (readb(&prcslink->var.rx_packet.rx_data_length[0])
-			     << 8)
-			    + readb(&prcslink->var.rx_packet.rx_data_length[1]);
-			if (readb(&prcslink->var.rx_packet.next_frag_rcs_index)
-			    == 0xFF || tmp < 0)
-				break;
-			prcslink = rcs_base(local)
-			    + readb(&prcslink->link_field);
-		} while (1);
-
-		if (tmp < 0) {
-			pr_debug(
-			      "ray_cs rx_data fragment lengths don't add up\n");
-			local->stats.rx_dropped++;
-			release_frag_chain(local, prcs);
-			return;
-		}
-	} else { /* Single unfragmented packet */
-		total_len = rx_len;
-	}
-
-	skb = dev_alloc_skb(total_len + 5);
-	if (skb == NULL) {
-		pr_debug("ray_cs rx_data could not allocate skb\n");
-		local->stats.rx_dropped++;
-		if (readb(&prcs->var.rx_packet.next_frag_rcs_index) != 0xFF)
-			release_frag_chain(local, prcs);
-		return;
-	}
-	skb_reserve(skb, 2);	/* Align IP on 16 byte (TBD check this) */
-
-	pr_debug("ray_cs rx_data total_len = %x, rx_len = %x\n", total_len,
-	      rx_len);
-
-/************************/
-	/* Reserve enough room for the whole damn packet. */
-	rx_ptr = skb_put(skb, total_len);
-	/* Copy the whole packet to sk_buff */
-	rx_ptr +=
-	    copy_from_rx_buff(local, rx_ptr, pkt_addr & RX_BUFF_END, rx_len);
-	/* Get source address */
-#ifdef WIRELESS_SPY
-	skb_copy_from_linear_data_offset(skb,
-					 offsetof(struct mac_header, addr_2),
-					 linksrcaddr, ETH_ALEN);
-#endif
-	/* Now, deal with encapsulation/translation/sniffer */
-	if (!sniffer) {
-		if (!translate) {
-			/* Encapsulated ethernet, so just lop off 802.11 MAC header */
-/* TBD reserve            skb_reserve( skb, RX_MAC_HEADER_LENGTH); */
-			skb_pull(skb, RX_MAC_HEADER_LENGTH);
-		} else {
-			/* Do translation */
-			untranslate(local, skb, total_len);
-		}
-	} else { /* sniffer mode, so just pass whole packet */
-	}
-
-/************************/
-	/* Now pick up the rest of the fragments if any */
-	tmp = 17;
-	if (readb(&prcs->var.rx_packet.next_frag_rcs_index) != 0xFF) {
-		prcslink = prcs;
-		pr_debug("ray_cs rx_data in fragment loop\n");
-		do {
-			prcslink = rcs_base(local)
-			    +
-			    readb(&prcslink->var.rx_packet.next_frag_rcs_index);
-			rx_len =
-			    ((readb(&prcslink->var.rx_packet.rx_data_length[0])
-			      << 8)
-			     +
-			     readb(&prcslink->var.rx_packet.rx_data_length[1]))
-			    & RX_BUFF_END;
-			pkt_addr =
-			    ((readb(&prcslink->var.rx_packet.rx_data_ptr[0]) <<
-			      8)
-			     + readb(&prcslink->var.rx_packet.rx_data_ptr[1]))
-			    & RX_BUFF_END;
-
-			rx_ptr +=
-			    copy_from_rx_buff(local, rx_ptr, pkt_addr, rx_len);
-
-		} while (tmp-- &&
-			 readb(&prcslink->var.rx_packet.next_frag_rcs_index) !=
-			 0xFF);
-		release_frag_chain(local, prcs);
-	}
-
-	skb->protocol = eth_type_trans(skb, dev);
-	netif_rx(skb);
-	local->stats.rx_packets++;
-	local->stats.rx_bytes += total_len;
-
-	/* Gather signal strength per address */
-#ifdef WIRELESS_SPY
-	/* For the Access Point or the node having started the ad-hoc net
-	 * note : ad-hoc work only in some specific configurations, but we
-	 * kludge in ray_get_wireless_stats... */
-	if (!memcmp(linksrcaddr, local->bss_id, ETH_ALEN)) {
-		/* Update statistics */
-		/*local->wstats.qual.qual = none ? */
-		local->wstats.qual.level = siglev;
-		/*local->wstats.qual.noise = none ? */
-		local->wstats.qual.updated = 0x2;
-	}
-	/* Now, update the spy stuff */
-	{
-		struct iw_quality wstats;
-		wstats.level = siglev;
-		/* wstats.noise = none ? */
-		/* wstats.qual = none ? */
-		wstats.updated = 0x2;
-		/* Update spy records */
-		wireless_spy_update(dev, linksrcaddr, &wstats);
-	}
-#endif /* WIRELESS_SPY */
-} /* end rx_data */
-
-/*===========================================================================*/
-static void untranslate(ray_dev_t *local, struct sk_buff *skb, int len)
-{
-	snaphdr_t *psnap = (snaphdr_t *) (skb->data + RX_MAC_HEADER_LENGTH);
-	struct ieee80211_hdr *pmac = (struct ieee80211_hdr *)skb->data;
-	__be16 type = *(__be16 *) psnap->ethertype;
-	int delta;
-	struct ethhdr *peth;
-	UCHAR srcaddr[ADDRLEN];
-	UCHAR destaddr[ADDRLEN];
-	static const UCHAR org_bridge[3] = { 0, 0, 0xf8 };
-	static const UCHAR org_1042[3] = { 0, 0, 0 };
-
-	memcpy(destaddr, ieee80211_get_DA(pmac), ADDRLEN);
-	memcpy(srcaddr, ieee80211_get_SA(pmac), ADDRLEN);
-
-#if 0
-	if {
-		print_hex_dump(KERN_DEBUG, "skb->data before untranslate: ",
-			       DUMP_PREFIX_NONE, 16, 1,
-			       skb->data, 64, true);
-		printk(KERN_DEBUG
-		       "type = %08x, xsap = %02x%02x%02x, org = %02x02x02x\n",
-		       ntohs(type), psnap->dsap, psnap->ssap, psnap->ctrl,
-		       psnap->org[0], psnap->org[1], psnap->org[2]);
-		printk(KERN_DEBUG "untranslate skb->data = %p\n", skb->data);
-	}
-#endif
-
-	if (psnap->dsap != 0xaa || psnap->ssap != 0xaa || psnap->ctrl != 3) {
-		/* not a snap type so leave it alone */
-		pr_debug("ray_cs untranslate NOT SNAP %02x %02x %02x\n",
-		      psnap->dsap, psnap->ssap, psnap->ctrl);
-
-		delta = RX_MAC_HEADER_LENGTH - ETH_HLEN;
-		peth = (struct ethhdr *)(skb->data + delta);
-		peth->h_proto = htons(len - RX_MAC_HEADER_LENGTH);
-	} else { /* Its a SNAP */
-		if (memcmp(psnap->org, org_bridge, 3) == 0) {
-		/* EtherII and nuke the LLC */
-			pr_debug("ray_cs untranslate Bridge encap\n");
-			delta = RX_MAC_HEADER_LENGTH
-			    + sizeof(struct snaphdr_t) - ETH_HLEN;
-			peth = (struct ethhdr *)(skb->data + delta);
-			peth->h_proto = type;
-		} else if (memcmp(psnap->org, org_1042, 3) == 0) {
-			switch (ntohs(type)) {
-			case ETH_P_IPX:
-			case ETH_P_AARP:
-				pr_debug("ray_cs untranslate RFC IPX/AARP\n");
-				delta = RX_MAC_HEADER_LENGTH - ETH_HLEN;
-				peth = (struct ethhdr *)(skb->data + delta);
-				peth->h_proto =
-				    htons(len - RX_MAC_HEADER_LENGTH);
-				break;
-			default:
-				pr_debug("ray_cs untranslate RFC default\n");
-				delta = RX_MAC_HEADER_LENGTH +
-				    sizeof(struct snaphdr_t) - ETH_HLEN;
-				peth = (struct ethhdr *)(skb->data + delta);
-				peth->h_proto = type;
-				break;
-			}
-		} else {
-			printk("ray_cs untranslate very confused by packet\n");
-			delta = RX_MAC_HEADER_LENGTH - ETH_HLEN;
-			peth = (struct ethhdr *)(skb->data + delta);
-			peth->h_proto = type;
-		}
-	}
-/* TBD reserve  skb_reserve(skb, delta); */
-	skb_pull(skb, delta);
-	pr_debug("untranslate after skb_pull(%d), skb->data = %p\n", delta,
-	      skb->data);
-	memcpy(peth->h_dest, destaddr, ADDRLEN);
-	memcpy(peth->h_source, srcaddr, ADDRLEN);
-#if 0
-	{
-		int i;
-		printk(KERN_DEBUG "skb->data after untranslate:");
-		for (i = 0; i < 64; i++)
-			printk("%02x ", skb->data[i]);
-		printk("\n");
-	}
-#endif
-} /* end untranslate */
-
-/*===========================================================================*/
-/* Copy data from circular receive buffer to PC memory.
- * dest     = destination address in PC memory
- * pkt_addr = source address in receive buffer
- * len      = length of packet to copy
- */
-static int copy_from_rx_buff(ray_dev_t *local, UCHAR *dest, int pkt_addr,
-			     int length)
-{
-	int wrap_bytes = (pkt_addr + length) - (RX_BUFF_END + 1);
-	if (wrap_bytes <= 0) {
-		memcpy_fromio(dest, local->rmem + pkt_addr, length);
-	} else { /* Packet wrapped in circular buffer */
-
-		memcpy_fromio(dest, local->rmem + pkt_addr,
-			      length - wrap_bytes);
-		memcpy_fromio(dest + length - wrap_bytes, local->rmem,
-			      wrap_bytes);
-	}
-	return length;
-}
-
-/*===========================================================================*/
-static void release_frag_chain(ray_dev_t *local, struct rcs __iomem *prcs)
-{
-	struct rcs __iomem *prcslink = prcs;
-	int tmp = 17;
-	unsigned rcsindex = readb(&prcs->var.rx_packet.next_frag_rcs_index);
-
-	while (tmp--) {
-		writeb(CCS_BUFFER_FREE, &prcslink->buffer_status);
-		if (rcsindex >= (NUMBER_OF_CCS + NUMBER_OF_RCS)) {
-			pr_debug("ray_cs interrupt bad rcsindex = 0x%x\n",
-			      rcsindex);
-			break;
-		}
-		prcslink = rcs_base(local) + rcsindex;
-		rcsindex = readb(&prcslink->var.rx_packet.next_frag_rcs_index);
-	}
-	writeb(CCS_BUFFER_FREE, &prcslink->buffer_status);
-}
-
-/*===========================================================================*/
-static void authenticate(ray_dev_t *local)
-{
-	struct pcmcia_device *link = local->finder;
-	dev_dbg(&link->dev, "ray_cs Starting authentication.\n");
-	if (!(pcmcia_dev_present(link))) {
-		dev_dbg(&link->dev, "ray_cs authenticate - device not present\n");
-		return;
-	}
-
-	del_timer(&local->timer);
-	if (build_auth_frame(local, local->bss_id, OPEN_AUTH_REQUEST)) {
-		local->timer.function = join_net;
-	} else {
-		local->timer.function = authenticate_timeout;
-	}
-	local->timer.expires = jiffies + HZ * 2;
-	add_timer(&local->timer);
-	local->authentication_state = AWAITING_RESPONSE;
-} /* end authenticate */
-
-/*===========================================================================*/
-static void rx_authenticate(ray_dev_t *local, struct rcs __iomem *prcs,
-			    unsigned int pkt_addr, int rx_len)
-{
-	UCHAR buff[256];
-	struct ray_rx_msg *msg = (struct ray_rx_msg *) buff;
-
-	del_timer(&local->timer);
-
-	copy_from_rx_buff(local, buff, pkt_addr, rx_len & 0xff);
-	/* if we are trying to get authenticated */
-	if (local->sparm.b4.a_network_type == ADHOC) {
-		pr_debug("ray_cs rx_auth var= %6ph\n", msg->var);
-		if (msg->var[2] == 1) {
-			pr_debug("ray_cs Sending authentication response.\n");
-			if (!build_auth_frame
-			    (local, msg->mac.addr_2, OPEN_AUTH_RESPONSE)) {
-				local->authentication_state = NEED_TO_AUTH;
-				memcpy(local->auth_id, msg->mac.addr_2,
-				       ADDRLEN);
-			}
-		}
-	} else { /* Infrastructure network */
-
-		if (local->authentication_state == AWAITING_RESPONSE) {
-			/* Verify authentication sequence #2 and success */
-			if (msg->var[2] == 2) {
-				if ((msg->var[3] | msg->var[4]) == 0) {
-					pr_debug("Authentication successful\n");
-					local->card_status = CARD_AUTH_COMPLETE;
-					associate(local);
-					local->authentication_state =
-					    AUTHENTICATED;
-				} else {
-					pr_debug("Authentication refused\n");
-					local->card_status = CARD_AUTH_REFUSED;
-					join_net(&local->timer);
-					local->authentication_state =
-					    UNAUTHENTICATED;
-				}
-			}
-		}
-	}
-
-} /* end rx_authenticate */
-
-/*===========================================================================*/
-static void associate(ray_dev_t *local)
-{
-	struct ccs __iomem *pccs;
-	struct pcmcia_device *link = local->finder;
-	struct net_device *dev = link->priv;
-	int ccsindex;
-	if (!(pcmcia_dev_present(link))) {
-		dev_dbg(&link->dev, "ray_cs associate - device not present\n");
-		return;
-	}
-	/* If no tx buffers available, return */
-	if ((ccsindex = get_free_ccs(local)) < 0) {
-/* TBD should never be here but... what if we are? */
-		dev_dbg(&link->dev, "ray_cs associate - No free ccs\n");
-		return;
-	}
-	dev_dbg(&link->dev, "ray_cs Starting association with access point\n");
-	pccs = ccs_base(local) + ccsindex;
-	/* fill in the CCS */
-	writeb(CCS_START_ASSOCIATION, &pccs->cmd);
-	/* Interrupt the firmware to process the command */
-	if (interrupt_ecf(local, ccsindex)) {
-		dev_dbg(&link->dev, "ray_cs associate failed - ECF not ready for intr\n");
-		writeb(CCS_BUFFER_FREE, &(pccs++)->buffer_status);
-
-		del_timer(&local->timer);
-		local->timer.expires = jiffies + HZ * 2;
-		local->timer.function = join_net;
-		add_timer(&local->timer);
-		local->card_status = CARD_ASSOC_FAILED;
-		return;
-	}
-	if (!sniffer)
-		netif_start_queue(dev);
-
-} /* end associate */
-
-/*===========================================================================*/
-static void rx_deauthenticate(ray_dev_t *local, struct rcs __iomem *prcs,
-			      unsigned int pkt_addr, int rx_len)
-{
-/*  UCHAR buff[256];
-    struct ray_rx_msg *msg = (struct ray_rx_msg *) buff;
-*/
-	pr_debug("Deauthentication frame received\n");
-	local->authentication_state = UNAUTHENTICATED;
-	/* Need to reauthenticate or rejoin depending on reason code */
-/*  copy_from_rx_buff(local, buff, pkt_addr, rx_len & 0xff);
- */
-}
-
-/*===========================================================================*/
-static void clear_interrupt(ray_dev_t *local)
-{
-	writeb(0, local->amem + CIS_OFFSET + HCS_INTR_OFFSET);
-}
-
-/*===========================================================================*/
-#ifdef CONFIG_PROC_FS
-#define MAXDATA (PAGE_SIZE - 80)
-
-static const char *card_status[] = {
-	"Card inserted - uninitialized",	/* 0 */
-	"Card not downloaded",			/* 1 */
-	"Waiting for download parameters",	/* 2 */
-	"Card doing acquisition",		/* 3 */
-	"Acquisition complete",			/* 4 */
-	"Authentication complete",		/* 5 */
-	"Association complete",			/* 6 */
-	"???", "???", "???", "???",		/* 7 8 9 10 undefined */
-	"Card init error",			/* 11 */
-	"Download parameters error",		/* 12 */
-	"???",					/* 13 */
-	"Acquisition failed",			/* 14 */
-	"Authentication refused",		/* 15 */
-	"Association failed"			/* 16 */
-};
-
-static const char *nettype[] = { "Adhoc", "Infra " };
-static const char *framing[] = { "Encapsulation", "Translation" }
-
-;
-/*===========================================================================*/
-static int ray_cs_proc_show(struct seq_file *m, void *v)
-{
-/* Print current values which are not available via other means
- * eg ifconfig
- */
-	int i;
-	struct pcmcia_device *link;
-	struct net_device *dev;
-	ray_dev_t *local;
-	UCHAR *p;
-	struct freq_hop_element *pfh;
-	UCHAR c[33];
-
-	link = this_device;
-	if (!link)
-		return 0;
-	dev = link->priv;
-	if (!dev)
-		return 0;
-	local = netdev_priv(dev);
-	if (!local)
-		return 0;
-
-	seq_puts(m, "Raylink Wireless LAN driver status\n");
-	seq_printf(m, "%s\n", rcsid);
-	/* build 4 does not report version, and field is 0x55 after memtest */
-	seq_puts(m, "Firmware version     = ");
-	if (local->fw_ver == 0x55)
-		seq_puts(m, "4 - Use dump_cis for more details\n");
-	else
-		seq_printf(m, "%2d.%02d.%02d\n",
-			   local->fw_ver, local->fw_bld, local->fw_var);
-
-	for (i = 0; i < 32; i++)
-		c[i] = local->sparm.b5.a_current_ess_id[i];
-	c[32] = 0;
-	seq_printf(m, "%s network ESSID = \"%s\"\n",
-		   nettype[local->sparm.b5.a_network_type], c);
-
-	p = local->bss_id;
-	seq_printf(m, "BSSID                = %pM\n", p);
-
-	seq_printf(m, "Country code         = %d\n",
-		   local->sparm.b5.a_curr_country_code);
-
-	i = local->card_status;
-	if (i < 0)
-		i = 10;
-	if (i > 16)
-		i = 10;
-	seq_printf(m, "Card status          = %s\n", card_status[i]);
-
-	seq_printf(m, "Framing mode         = %s\n", framing[translate]);
-
-	seq_printf(m, "Last pkt signal lvl  = %d\n", local->last_rsl);
-
-	if (local->beacon_rxed) {
-		/* Pull some fields out of last beacon received */
-		seq_printf(m, "Beacon Interval      = %d Kus\n",
-			   local->last_bcn.beacon_intvl[0]
-			   + 256 * local->last_bcn.beacon_intvl[1]);
-
-		p = local->last_bcn.elements;
-		if (p[0] == C_ESSID_ELEMENT_ID)
-			p += p[1] + 2;
-		else {
-			seq_printf(m,
-				   "Parse beacon failed at essid element id = %d\n",
-				   p[0]);
-			return 0;
-		}
-
-		if (p[0] == C_SUPPORTED_RATES_ELEMENT_ID) {
-			seq_puts(m, "Supported rate codes = ");
-			for (i = 2; i < p[1] + 2; i++)
-				seq_printf(m, "0x%02x ", p[i]);
-			seq_putc(m, '\n');
-			p += p[1] + 2;
-		} else {
-			seq_puts(m, "Parse beacon failed at rates element\n");
-			return 0;
-		}
-
-		if (p[0] == C_FH_PARAM_SET_ELEMENT_ID) {
-			pfh = (struct freq_hop_element *)p;
-			seq_printf(m, "Hop dwell            = %d Kus\n",
-				   pfh->dwell_time[0] +
-				   256 * pfh->dwell_time[1]);
-			seq_printf(m, "Hop set              = %d\n",
-				   pfh->hop_set);
-			seq_printf(m, "Hop pattern          = %d\n",
-				   pfh->hop_pattern);
-			seq_printf(m, "Hop index            = %d\n",
-				   pfh->hop_index);
-			p += p[1] + 2;
-		} else {
-			seq_puts(m,
-				 "Parse beacon failed at FH param element\n");
-			return 0;
-		}
-	} else {
-		seq_puts(m, "No beacons received\n");
-	}
-	return 0;
-}
-#endif
-/*===========================================================================*/
-static int build_auth_frame(ray_dev_t *local, UCHAR *dest, int auth_type)
-{
-	int addr;
-	struct ccs __iomem *pccs;
-	struct tx_msg __iomem *ptx;
-	int ccsindex;
-
-	/* If no tx buffers available, return */
-	if ((ccsindex = get_free_tx_ccs(local)) < 0) {
-		pr_debug("ray_cs send authenticate - No free tx ccs\n");
-		return -1;
-	}
-
-	pccs = ccs_base(local) + ccsindex;
-
-	/* Address in card space */
-	addr = TX_BUF_BASE + (ccsindex << 11);
-	/* fill in the CCS */
-	writeb(CCS_TX_REQUEST, &pccs->cmd);
-	writeb(addr >> 8, pccs->var.tx_request.tx_data_ptr);
-	writeb(0x20, pccs->var.tx_request.tx_data_ptr + 1);
-	writeb(TX_AUTHENTICATE_LENGTH_MSB, pccs->var.tx_request.tx_data_length);
-	writeb(TX_AUTHENTICATE_LENGTH_LSB,
-	       pccs->var.tx_request.tx_data_length + 1);
-	writeb(0, &pccs->var.tx_request.pow_sav_mode);
-
-	ptx = local->sram + addr;
-	/* fill in the mac header */
-	writeb(PROTOCOL_VER | AUTHENTIC_TYPE, &ptx->mac.frame_ctl_1);
-	writeb(0, &ptx->mac.frame_ctl_2);
-
-	memcpy_toio(ptx->mac.addr_1, dest, ADDRLEN);
-	memcpy_toio(ptx->mac.addr_2, local->sparm.b4.a_mac_addr, ADDRLEN);
-	memcpy_toio(ptx->mac.addr_3, local->bss_id, ADDRLEN);
-
-	/* Fill in msg body with protocol 00 00, sequence 01 00 ,status 00 00 */
-	memset_io(ptx->var, 0, 6);
-	writeb(auth_type & 0xff, ptx->var + 2);
-
-	/* Interrupt the firmware to process the command */
-	if (interrupt_ecf(local, ccsindex)) {
-		pr_debug(
-		      "ray_cs send authentication request failed - ECF not ready for intr\n");
-		writeb(CCS_BUFFER_FREE, &(pccs++)->buffer_status);
-		return -1;
-	}
-	return 0;
-} /* End build_auth_frame */
-
-/*===========================================================================*/
-#ifdef CONFIG_PROC_FS
-static ssize_t ray_cs_essid_proc_write(struct file *file,
-		const char __user *buffer, size_t count, loff_t *pos)
-{
-	static char proc_essid[33];
-	unsigned int len = count;
-
-	if (len > 32)
-		len = 32;
-	memset(proc_essid, 0, 33);
-	if (copy_from_user(proc_essid, buffer, len))
-		return -EFAULT;
-	essid = proc_essid;
-	return count;
-}
-
-static const struct proc_ops ray_cs_essid_proc_ops = {
-	.proc_write	= ray_cs_essid_proc_write,
-	.proc_lseek	= noop_llseek,
-};
-
-static ssize_t int_proc_write(struct file *file, const char __user *buffer,
-			      size_t count, loff_t *pos)
-{
-	static char proc_number[10];
-	char *p;
-	int nr, len;
-
-	if (!count)
-		return 0;
-
-	if (count > 9)
-		return -EINVAL;
-	if (copy_from_user(proc_number, buffer, count))
-		return -EFAULT;
-	p = proc_number;
-	nr = 0;
-	len = count;
-	do {
-		unsigned int c = *p - '0';
-		if (c > 9)
-			return -EINVAL;
-		nr = nr * 10 + c;
-		p++;
-	} while (--len);
-	*(int *)pde_data(file_inode(file)) = nr;
-	return count;
-}
-
-static const struct proc_ops int_proc_ops = {
-	.proc_write	= int_proc_write,
-	.proc_lseek	= noop_llseek,
-};
-#endif
-
-static const struct pcmcia_device_id ray_ids[] = {
-	PCMCIA_DEVICE_MANF_CARD(0x01a6, 0x0000),
-	PCMCIA_DEVICE_NULL,
-};
-
-MODULE_DEVICE_TABLE(pcmcia, ray_ids);
-
-static struct pcmcia_driver ray_driver = {
-	.owner = THIS_MODULE,
-	.name = "ray_cs",
-	.probe = ray_probe,
-	.remove = ray_detach,
-	.id_table = ray_ids,
-	.suspend = ray_suspend,
-	.resume = ray_resume,
-};
-
-static int __init init_ray_cs(void)
-{
-	int rc;
-
-	pr_debug("%s\n", rcsid);
-	rc = pcmcia_register_driver(&ray_driver);
-	pr_debug("raylink init_module register_pcmcia_driver returns 0x%x\n",
-	      rc);
-	if (rc)
-		return rc;
-
-#ifdef CONFIG_PROC_FS
-	proc_mkdir("driver/ray_cs", NULL);
-
-	proc_create_single("driver/ray_cs/ray_cs", 0, NULL, ray_cs_proc_show);
-	proc_create("driver/ray_cs/essid", 0200, NULL, &ray_cs_essid_proc_ops);
-	proc_create_data("driver/ray_cs/net_type", 0200, NULL, &int_proc_ops,
-			 &net_type);
-	proc_create_data("driver/ray_cs/translate", 0200, NULL, &int_proc_ops,
-			 &translate);
-#endif
-	translate = !!translate;
-	return 0;
-} /* init_ray_cs */
-
-/*===========================================================================*/
-
-static void __exit exit_ray_cs(void)
-{
-	pr_debug("ray_cs: cleanup_module\n");
-
-#ifdef CONFIG_PROC_FS
-	remove_proc_subtree("driver/ray_cs", NULL);
-#endif
-
-	pcmcia_unregister_driver(&ray_driver);
-} /* exit_ray_cs */
-
-module_init(init_ray_cs);
-module_exit(exit_ray_cs);
-
-/*===========================================================================*/
diff --git a/drivers/net/wireless/legacy/ray_cs.h b/drivers/net/wireless/legacy/ray_cs.h
deleted file mode 100644
index 0609d8625019..000000000000
--- a/drivers/net/wireless/legacy/ray_cs.h
+++ /dev/null
@@ -1,74 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/* Raytheon wireless LAN PCMCIA card driver for Linux 
-   A  PCMCIA client driver for the Raylink wireless network card
-   Written by Corey Thomas
-*/
-
-#ifndef _RAY_CS_H_
-#define _RAY_CS_H_
-
-struct beacon_rx {
-    struct mac_header mac;
-    UCHAR timestamp[8];
-    UCHAR beacon_intvl[2];
-    UCHAR capability[2];
-    UCHAR elements[sizeof(struct essid_element) 
-                  + sizeof(struct rates_element)
-                  + sizeof(struct freq_hop_element) 
-                  + sizeof(struct japan_call_sign_element)
-                  + sizeof(struct tim_element)];
-};
-
-/* Return values for get_free{,_tx}_ccs */
-#define ECCSFULL  (-1)
-#define ECCSBUSY  (-2)
-#define ECARDGONE (-3)
-
-typedef struct ray_dev_t {
-    int card_status;
-    int authentication_state;
-    void __iomem *sram;            /* pointer to beginning of shared RAM     */
-    void __iomem *amem;            /* pointer to attribute mem window        */
-    void __iomem *rmem;            /* pointer to receive buffer window       */
-    struct pcmcia_device *finder;            /* pointer back to struct pcmcia_device for card    */
-    struct timer_list timer;
-    unsigned long tx_ccs_lock;
-    unsigned long ccs_lock;
-    int   dl_param_ccs;
-    union {
-        struct b4_startup_params b4;
-        struct b5_startup_params b5;
-    } sparm;
-    int timeout_flag;
-    UCHAR supported_rates[8];
-    UCHAR japan_call_sign[12];
-    struct startup_res_6 startup_res;
-    int num_multi;
-    /* Network parameters from start/join */
-    UCHAR bss_id[6];
-    UCHAR auth_id[6];
-    UCHAR net_default_tx_rate;
-    UCHAR encryption;
-    struct net_device_stats stats;
-
-    UCHAR net_type;
-    UCHAR sta_type;
-    UCHAR fw_ver;
-    UCHAR fw_bld;
-    UCHAR fw_var;
-    UCHAR ASIC_version;
-    UCHAR assoc_id[2];
-    UCHAR tib_length;
-    UCHAR last_rsl;
-    int beacon_rxed;
-    struct beacon_rx last_bcn;
-    iw_stats	wstats;		/* Wireless specific stats */
-#ifdef WIRELESS_SPY
-    struct iw_spy_data		spy_data;
-    struct iw_public_data	wireless_data;
-#endif	/* WIRELESS_SPY */
-
-} ray_dev_t;
-/*****************************************************************************/
-
-#endif /* _RAY_CS_H_ */
diff --git a/drivers/net/wireless/legacy/rayctl.h b/drivers/net/wireless/legacy/rayctl.h
deleted file mode 100644
index 1f3bde8ac73d..000000000000
--- a/drivers/net/wireless/legacy/rayctl.h
+++ /dev/null
@@ -1,734 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _RAYCTL_H_
-#define _RAYCTL_H_
-
-typedef unsigned char UCHAR;
-
-/****** IEEE 802.11 constants ************************************************/
-#define ADDRLEN           6
-/* Frame control 1 bit fields */
-#define PROTOCOL_VER      0x00
-#define DATA_TYPE         0x08
-#define ASSOC_REQ_TYPE    0x00
-#define ASSOC_RESP_TYPE   0x10
-#define REASSOC_REQ_TYPE  0x20
-#define REASSOC_RESP_TYPE 0x30
-#define NULL_MSG_TYPE     0x48
-#define BEACON_TYPE       0x80
-#define DISASSOC_TYPE     0xA0
-#define PSPOLL_TYPE       0xA4
-#define AUTHENTIC_TYPE    0xB0
-#define DEAUTHENTIC_TYPE  0xC0
-/* Frame control 2 bit fields */
-#define FC2_TO_DS         0x01
-#define FC2_FROM_DS       0x02
-#define FC2_MORE_FRAG     0x04
-#define FC2_RETRY         0x08
-#define FC2_PSM           0x10
-#define FC2_MORE_DATA     0x20
-#define FC2_WEP           0x40
-#define FC2_ORDER         0x80
-/*****************************************************************************/
-/* 802.11 element ID's and lengths */
-#define C_BP_CAPABILITY_ESS             0x01
-#define C_BP_CAPABILITY_IBSS            0x02
-#define C_BP_CAPABILITY_CF_POLLABLE     0x04
-#define C_BP_CAPABILITY_CF_POLL_REQUEST 0x08
-#define C_BP_CAPABILITY_PRIVACY         0x10
-
-#define C_ESSID_ELEMENT_ID               0
-#define C_ESSID_ELEMENT_MAX_LENGTH       32
-
-#define C_SUPPORTED_RATES_ELEMENT_ID     1
-#define C_SUPPORTED_RATES_ELEMENT_LENGTH 2
-
-#define C_FH_PARAM_SET_ELEMENT_ID        2
-#define C_FH_PARAM_SET_ELEMENT_LNGTH     5
-
-#define C_CF_PARAM_SET_ELEMENT_ID        4
-#define C_CF_PARAM_SET_ELEMENT_LNGTH     6
-
-#define C_TIM_ELEMENT_ID                 5
-#define C_TIM_BITMAP_LENGTH            251
-#define C_TIM_BMCAST_BIT              0x01
-
-#define C_IBSS_ELEMENT_ID                6
-#define C_IBSS_ELEMENT_LENGTH            2
-
-#define C_JAPAN_CALL_SIGN_ELEMENT_ID    51
-#define C_JAPAN_CALL_SIGN_ELEMENT_LNGTH 12
-
-#define C_DISASSOC_REASON_CODE_LEN       2
-#define C_DISASSOC_REASON_CODE_DEFAULT   8
-
-#define C_CRC_LEN                        4
-#define C_NUM_SUPPORTED_RATES            8 
-/****** IEEE 802.11 mac header for type data packets *************************/
-struct mac_header {
-  UCHAR frame_ctl_1;                          
-  UCHAR frame_ctl_2;
-  UCHAR duration_lsb;
-  UCHAR duration_msb;
-  UCHAR addr_1[ADDRLEN];
-  UCHAR addr_2[ADDRLEN];
-  UCHAR addr_3[ADDRLEN];
-  UCHAR seq_frag_num[2];
-/*  UCHAR addr_4[ADDRLEN]; *//* only present for AP to AP (TO DS and FROM DS */
-};
-/****** IEEE 802.11 frame element structures *********************************/
-struct essid_element
-{
-  UCHAR id;
-  UCHAR length;
-  UCHAR text[C_ESSID_ELEMENT_MAX_LENGTH];
-};
-struct rates_element
-{
-  UCHAR id;
-  UCHAR length;
-  UCHAR value[8];
-};
-struct freq_hop_element
-{
-  UCHAR id;
-  UCHAR length;
-  UCHAR dwell_time[2];
-  UCHAR hop_set;
-  UCHAR hop_pattern;
-  UCHAR hop_index;
-};
-struct tim_element
-{
-  UCHAR id;
-  UCHAR length;
-  UCHAR dtim_count;
-  UCHAR dtim_period;    
-  UCHAR bitmap_control;
-  UCHAR tim[C_TIM_BITMAP_LENGTH];
-};
-struct ibss_element
-{
-  UCHAR id;
-  UCHAR length;
-  UCHAR atim_window[2];
-};
-struct japan_call_sign_element
-{
-  UCHAR id;
-  UCHAR length;
-  UCHAR call_sign[12];
-};
-/****** Beacon message structures ********************************************/
-/* .elements is a large lump of max size because elements are variable size  */
-struct infra_beacon
-{
-    UCHAR timestamp[8];
-    UCHAR beacon_intvl[2];
-    UCHAR capability[2];
-    UCHAR elements[sizeof(struct essid_element) 
-                  + sizeof(struct rates_element)
-                  + sizeof(struct freq_hop_element) 
-                  + sizeof(struct japan_call_sign_element)
-                  + sizeof(struct tim_element)];
-};
-struct adhoc_beacon
-{
-    UCHAR timestamp[8];
-    UCHAR beacon_intvl[2];
-    UCHAR capability[2];
-    UCHAR elements[sizeof(struct essid_element) 
-                  + sizeof(struct rates_element)
-                  + sizeof(struct freq_hop_element) 
-                  + sizeof(struct japan_call_sign_element)
-                  + sizeof(struct ibss_element)];
-};
-/*****************************************************************************/
-/*****************************************************************************/
-/* #define C_MAC_HDR_2_WEP 0x40 */
-/* TX/RX CCS constants */
-#define TX_HEADER_LENGTH 0x1C
-#define RX_MAC_HEADER_LENGTH 0x18
-#define TX_AUTHENTICATE_LENGTH (TX_HEADER_LENGTH + 6)
-#define TX_AUTHENTICATE_LENGTH_MSB (TX_AUTHENTICATE_LENGTH >> 8)
-#define TX_AUTHENTICATE_LENGTH_LSB (TX_AUTHENTICATE_LENGTH & 0xff)
-#define TX_DEAUTHENTICATE_LENGTH (TX_HEADER_LENGTH + 2)
-#define TX_DEAUTHENTICATE_LENGTH_MSB (TX_AUTHENTICATE_LENGTH >> 8)
-#define TX_DEAUTHENTICATE_LENGTH_LSB (TX_AUTHENTICATE_LENGTH & 0xff)
-#define FCS_LEN           4
-
-#define ADHOC                 0
-#define INFRA                 1
-
-#define TYPE_STA              0
-#define TYPE_AP               1
-
-#define PASSIVE_SCAN          1
-#define ACTIVE_SCAN           1
-
-#define PSM_CAM               0
-
-/* Country codes */
-#define USA                   1
-#define EUROPE                2
-#define JAPAN                 3
-#define KOREA                 4
-#define SPAIN                 5
-#define FRANCE                6
-#define ISRAEL                7
-#define AUSTRALIA             8
-#define JAPAN_TEST            9
-
-/* Hop pattern lengths */
-#define USA_HOP_MOD          79 
-#define EUROPE_HOP_MOD       79 
-#define JAPAN_HOP_MOD        23
-#define KOREA_HOP_MOD        23
-#define SPAIN_HOP_MOD        27
-#define FRANCE_HOP_MOD       35
-#define ISRAEL_HOP_MOD       35
-#define AUSTRALIA_HOP_MOD    47
-#define JAPAN_TEST_HOP_MOD   23
-
-#define ESSID_SIZE           32
-/**********************************************************************/
-/* CIS Register Constants */
-#define CIS_OFFSET             0x0f00
-/* Configuration Option Register (0x0F00) */
-#define COR_OFFSET             0x00
-#define COR_SOFT_RESET         0x80
-#define COR_LEVEL_IRQ          0x40
-#define COR_CONFIG_NUM         0x01
-#define COR_DEFAULT            (COR_LEVEL_IRQ | COR_CONFIG_NUM)
-
-/* Card Configuration and Status Register (0x0F01) */
-#define CCSR_OFFSET            0x01
-#define CCSR_HOST_INTR_PENDING 0x01
-#define CCSR_POWER_DOWN        0x04
-
-/* HCS Interrupt Register (0x0F05) */
-#define HCS_INTR_OFFSET        0x05
-/* #define HCS_INTR_OFFSET        0x0A */
-#define HCS_INTR_CLEAR         0x00
-
-/* ECF Interrupt Register (0x0F06) */
-#define ECF_INTR_OFFSET        0x06
-/* #define ECF_INTR_OFFSET        0x0C */
-#define ECF_INTR_SET           0x01
-
-/* Authorization Register 0 (0x0F08) */
-#define AUTH_0_ON              0x57
-
-/* Authorization Register 1 (0x0F09) */
-#define AUTH_1_ON              0x82
-
-/* Program Mode Register (0x0F0A) */
-#define PC2PM                  0x02
-#define PC2CAL                 0x10
-#define PC2MLSE                0x20
-
-/* PC Test Mode Register (0x0F0B) */
-#define PC_TEST_MODE           0x08
-
-/* Frequency Control Word (0x0F10) */
-/* Range 0x02 - 0xA6 */
-
-/* Test Mode Control 1-4 (0x0F14 - 0x0F17) */
-
-/**********************************************************************/
-
-/* Shared RAM Area */
-#define SCB_BASE               0x0000
-#define STATUS_BASE            0x0100
-#define HOST_TO_ECF_BASE       0x0200
-#define ECF_TO_HOST_BASE       0x0300
-#define CCS_BASE               0x0400
-#define RCS_BASE               0x0800
-#define INFRA_TIM_BASE         0x0C00
-#define SSID_LIST_BASE         0x0D00
-#define TX_BUF_BASE            0x1000
-#define RX_BUF_BASE            0x8000
-
-#define NUMBER_OF_CCS    64
-#define NUMBER_OF_RCS    64
-/*#define NUMBER_OF_TX_CCS 14 */
-#define NUMBER_OF_TX_CCS 14
-
-#define TX_BUF_SIZE      (2048 - sizeof(struct tx_msg))
-#define RX_BUFF_END      0x3FFF
-/* Values for buffer_status */
-#define CCS_BUFFER_FREE       0
-#define CCS_BUFFER_BUSY       1
-#define CCS_COMMAND_COMPLETE  2
-#define CCS_COMMAND_FAILED    3
-
-/* Values for cmd */
-#define CCS_DOWNLOAD_STARTUP_PARAMS    1
-#define CCS_UPDATE_PARAMS              2
-#define CCS_REPORT_PARAMS              3
-#define CCS_UPDATE_MULTICAST_LIST      4
-#define CCS_UPDATE_POWER_SAVINGS_MODE  5
-#define CCS_START_NETWORK              6
-#define CCS_JOIN_NETWORK               7
-#define CCS_START_ASSOCIATION          8
-#define CCS_TX_REQUEST                 9
-#define CCS_TEST_MEMORY              0xa
-#define CCS_SHUTDOWN                 0xb
-#define CCS_DUMP_MEMORY              0xc
-#define CCS_START_TIMER              0xe
-#define CCS_LAST_CMD                 CCS_START_TIMER
-
-/* Values for link field */
-#define CCS_END_LIST                 0xff
-
-/* values for buffer_status field */
-#define RCS_BUFFER_FREE       0
-#define RCS_BUFFER_BUSY       1
-#define RCS_COMPLETE          2
-#define RCS_FAILED            3
-#define RCS_BUFFER_RELEASE    0xFF
-
-/* values for interrupt_id field */
-#define PROCESS_RX_PACKET           0x80 /* */
-#define REJOIN_NET_COMPLETE         0x81 /* RCS ID: Rejoin Net Complete */
-#define ROAMING_INITIATED           0x82 /* RCS ID: Roaming Initiated   */
-#define JAPAN_CALL_SIGN_RXD         0x83 /* RCS ID: New Japan Call Sign */
-
-/*****************************************************************************/
-/* Memory types for dump memory command */
-#define C_MEM_PROG  0
-#define C_MEM_XDATA 1
-#define C_MEM_SFR   2
-#define C_MEM_IDATA 3
-
-/*** Return values for hw_xmit **********/
-#define XMIT_OK        (0)
-#define XMIT_MSG_BAD   (-1)
-#define XMIT_NO_CCS    (-2)
-#define XMIT_NO_INTR   (-3)
-#define XMIT_NEED_AUTH (-4)
-
-/*** Values for card status */
-#define CARD_INSERTED       (0)
-
-#define CARD_AWAITING_PARAM (1)
-#define CARD_INIT_ERROR     (11)
-
-#define CARD_DL_PARAM       (2)
-#define CARD_DL_PARAM_ERROR (12)
-
-#define CARD_DOING_ACQ      (3)
-
-#define CARD_ACQ_COMPLETE   (4)
-#define CARD_ACQ_FAILED     (14)
-
-#define CARD_AUTH_COMPLETE  (5)
-#define CARD_AUTH_REFUSED   (15)
-
-#define CARD_ASSOC_COMPLETE (6)
-#define CARD_ASSOC_FAILED   (16)
-
-/*** Values for authentication_state ***********************************/
-#define UNAUTHENTICATED     (0)
-#define AWAITING_RESPONSE   (1)
-#define AUTHENTICATED       (2)
-#define NEED_TO_AUTH        (3)
-
-/*** Values for authentication type ************************************/
-#define OPEN_AUTH_REQUEST   (1)
-#define OPEN_AUTH_RESPONSE  (2)
-#define BROADCAST_DEAUTH    (0xc0)
-/*** Values for timer functions ****************************************/
-#define TODO_NOTHING              (0)
-#define TODO_VERIFY_DL_START      (-1)
-#define TODO_START_NET            (-2)
-#define TODO_JOIN_NET             (-3)
-#define TODO_AUTHENTICATE_TIMEOUT (-4)
-#define TODO_SEND_CCS             (-5)
-/***********************************************************************/
-/* Parameter passing structure for update/report parameter CCS's */
-struct object_id {
-    void          *object_addr;
-    unsigned char object_length;
-};
-
-#define OBJID_network_type            0
-#define OBJID_acting_as_ap_status     1
-#define OBJID_current_ess_id          2
-#define OBJID_scanning_mode           3
-#define OBJID_power_mgt_state         4
-#define OBJID_mac_address             5
-#define OBJID_frag_threshold          6
-#define OBJID_hop_time                7
-#define OBJID_beacon_period           8
-#define OBJID_dtim_period             9
-#define OBJID_retry_max              10
-#define OBJID_ack_timeout            11
-#define OBJID_sifs                   12
-#define OBJID_difs                   13
-#define OBJID_pifs                   14
-#define OBJID_rts_threshold          15
-#define OBJID_scan_dwell_time        16
-#define OBJID_max_scan_dwell_time    17
-#define OBJID_assoc_resp_timeout     18
-#define OBJID_adhoc_scan_cycle_max   19
-#define OBJID_infra_scan_cycle_max   20
-#define OBJID_infra_super_cycle_max  21
-#define OBJID_promiscuous_mode       22
-#define OBJID_unique_word            23
-#define OBJID_slot_time              24
-#define OBJID_roaming_low_snr        25
-#define OBJID_low_snr_count_thresh   26
-#define OBJID_infra_missed_bcn       27
-#define OBJID_adhoc_missed_bcn       28
-#define OBJID_curr_country_code      29
-#define OBJID_hop_pattern            30
-#define OBJID_reserved               31
-#define OBJID_cw_max_msb             32
-#define OBJID_cw_min_msb             33
-#define OBJID_noise_filter_gain      34
-#define OBJID_noise_limit_offset     35
-#define OBJID_det_rssi_thresh_offset 36
-#define OBJID_med_busy_thresh_offset 37
-#define OBJID_det_sync_thresh        38
-#define OBJID_test_mode              39
-#define OBJID_test_min_chan_num      40
-#define OBJID_test_max_chan_num      41
-#define OBJID_allow_bcast_ID_prbrsp  42
-#define OBJID_privacy_must_start     43
-#define OBJID_privacy_can_join       44
-#define OBJID_basic_rate_set         45
-
-/**** Configuration/Status/Control Area ***************************/
-/*    System Control Block (SCB) Area
- *    Located at Shared RAM offset 0
- */
-struct scb {
-    UCHAR ccs_index;
-    UCHAR rcs_index;
-};
-
-/****** Status area at Shared RAM offset 0x0100 ******************************/
-struct status {
-    UCHAR mrx_overflow_for_host;         /* 0=ECF may write, 1=host may write*/
-    UCHAR mrx_checksum_error_for_host;   /* 0=ECF may write, 1=host may write*/
-    UCHAR rx_hec_error_for_host;         /* 0=ECF may write, 1=host may write*/
-    UCHAR reserved1;
-    short mrx_overflow;                  /* ECF increments on rx overflow    */
-    short mrx_checksum_error;            /* ECF increments on rx CRC error   */
-    short rx_hec_error;                  /* ECF incs on mac header CRC error */
-    UCHAR rxnoise;                       /* Average RSL measurement          */
-};
-
-/****** Host-to-ECF Data Area at Shared RAM offset 0x200 *********************/
-struct host_to_ecf_area {
-    
-};
-
-/****** ECF-to-Host Data Area at Shared RAM offset 0x0300 ********************/
-struct startup_res_518 {
-    UCHAR startup_word;
-    UCHAR station_addr[ADDRLEN];
-    UCHAR calc_prog_chksum;
-    UCHAR calc_cis_chksum;
-    UCHAR ecf_spare[7];
-    UCHAR japan_call_sign[12];
-};
-
-struct startup_res_6 {
-    UCHAR startup_word;
-    UCHAR station_addr[ADDRLEN];
-    UCHAR reserved;
-    UCHAR supp_rates[8];
-    UCHAR japan_call_sign[12];
-    UCHAR calc_prog_chksum;
-    UCHAR calc_cis_chksum;
-    UCHAR firmware_version[3];
-    UCHAR asic_version;
-    UCHAR tib_length;
-};
-
-struct start_join_net_params {
-    UCHAR net_type;
-    UCHAR ssid[ESSID_SIZE];
-    UCHAR reserved;
-    UCHAR privacy_can_join;
-};
-
-/****** Command Control Structure area at Shared ram offset 0x0400 ***********/
-/* Structures for command specific parameters (ccs.var) */
-struct update_param_cmd {
-    UCHAR object_id;
-    UCHAR number_objects;
-    UCHAR failure_cause;
-};
-struct report_param_cmd {
-    UCHAR object_id;
-    UCHAR number_objects;
-    UCHAR failure_cause;
-    UCHAR length;
-};
-struct start_network_cmd {
-    UCHAR update_param;
-    UCHAR bssid[ADDRLEN];
-    UCHAR net_initiated;
-    UCHAR net_default_tx_rate;
-    UCHAR encryption;
-};
-struct join_network_cmd {
-    UCHAR update_param;
-    UCHAR bssid[ADDRLEN];
-    UCHAR net_initiated;
-    UCHAR net_default_tx_rate;
-    UCHAR encryption;
-};
-struct tx_requested_cmd {
- 
-    UCHAR tx_data_ptr[2];
-    UCHAR tx_data_length[2];
-    UCHAR host_reserved[2];
-    UCHAR reserved[3];
-    UCHAR tx_rate;
-    UCHAR pow_sav_mode;
-    UCHAR retries;
-    UCHAR antenna;
-};
-struct tx_requested_cmd_4 {
- 
-    UCHAR tx_data_ptr[2];
-    UCHAR tx_data_length[2];
-    UCHAR dest_addr[ADDRLEN];
-    UCHAR pow_sav_mode;
-    UCHAR retries;
-    UCHAR station_id;
-};
-struct memory_dump_cmd {
-    UCHAR memory_type;
-    UCHAR memory_ptr[2];
-    UCHAR length;
-};
-struct update_association_cmd {
-    UCHAR status;
-    UCHAR aid[2];
-};
-struct start_timer_cmd {
-    UCHAR duration[2];
-};
-
-struct ccs {
-    UCHAR buffer_status;                 /* 0 = buffer free, 1 = buffer busy */
-                                         /* 2 = command complete, 3 = failed */
-    UCHAR cmd;                           /* command to ECF                   */
-    UCHAR link;                          /* link to next CCS, FF=end of list */
-    /* command specific parameters      */
-    union {
-        char reserved[13];
-        struct update_param_cmd update_param;
-        struct report_param_cmd report_param;
-        UCHAR nummulticast;
-        UCHAR mode;
-        struct start_network_cmd start_network;
-        struct join_network_cmd join_network;
-        struct tx_requested_cmd tx_request;
-        struct memory_dump_cmd memory_dump;
-        struct update_association_cmd update_assoc;
-        struct start_timer_cmd start_timer;
-    } var;
-};
-
-/*****************************************************************************/
-/* Transmit buffer structures */
-struct tib_structure {
-    UCHAR ccs_index;
-    UCHAR psm;
-    UCHAR pass_fail;
-    UCHAR retry_count;
-    UCHAR max_retries;
-    UCHAR frags_remaining;
-    UCHAR no_rb;
-    UCHAR rts_reqd;
-    UCHAR csma_tx_cntrl_2;
-    UCHAR sifs_tx_cntrl_2;
-    UCHAR tx_dma_addr_1[2];
-    UCHAR tx_dma_addr_2[2];
-    UCHAR var_dur_2mhz[2];
-    UCHAR var_dur_1mhz[2];
-    UCHAR max_dur_2mhz[2];
-    UCHAR max_dur_1mhz[2];
-    UCHAR hdr_len;
-    UCHAR max_frag_len[2];
-    UCHAR var_len[2];
-    UCHAR phy_hdr_4;
-    UCHAR mac_hdr_1;
-    UCHAR mac_hdr_2;
-    UCHAR sid[2];
-};
-
-struct phy_header {
-    UCHAR sfd[2];
-    UCHAR hdr_3;
-    UCHAR hdr_4;
-};
-struct ray_rx_msg {
-    struct mac_header mac;
-	UCHAR	var[];
-};
-
-struct tx_msg {
-    struct tib_structure tib;
-    struct phy_header phy;
-    struct mac_header mac;
-    UCHAR  var[];
-};
-
-/****** ECF Receive Control Structure (RCS) Area at Shared RAM offset 0x0800  */
-/* Structures for command specific parameters (rcs.var) */
-struct rx_packet_cmd {
-    UCHAR rx_data_ptr[2];
-    UCHAR rx_data_length[2];
-    UCHAR rx_sig_lev;
-    UCHAR next_frag_rcs_index;
-    UCHAR totalpacketlength[2];
-};
-struct rejoin_net_cmplt_cmd {
-    UCHAR reserved;
-    UCHAR bssid[ADDRLEN];
-};
-struct japan_call_sign_rxd {
-    UCHAR rxd_call_sign[8];
-    UCHAR reserved[5];
-};
-
-struct rcs {
-    UCHAR buffer_status;
-    UCHAR interrupt_id;
-    UCHAR link_field;
-    /* command specific parameters      */
-    union {
-        UCHAR reserved[13]; 
-        struct rx_packet_cmd rx_packet;
-        struct rejoin_net_cmplt_cmd rejoin_net_complete;
-        struct japan_call_sign_rxd japan_call_sign;
-    } var;
-};
-
-/****** Startup parameter structures for both versions of firmware ***********/
-struct b4_startup_params {
-    UCHAR a_network_type;                /* C_ADHOC, C_INFRA                 */
-    UCHAR a_acting_as_ap_status;         /* C_TYPE_STA, C_TYPE_AP            */
-    UCHAR a_current_ess_id[ESSID_SIZE];  /* Null terminated unless 32 long   */
-    UCHAR a_scanning_mode;               /* passive 0, active 1              */
-    UCHAR a_power_mgt_state;             /* CAM 0,                           */
-    UCHAR a_mac_addr[ADDRLEN];           /*                                  */
-    UCHAR a_frag_threshold[2];           /* 512                              */
-    UCHAR a_hop_time[2];                 /* 16k * 2**n, n=0-4 in Kus         */
-    UCHAR a_beacon_period[2];            /* n * a_hop_time  in Kus           */
-    UCHAR a_dtim_period;                 /* in beacons                       */
-    UCHAR a_retry_max;                   /*                                  */
-    UCHAR a_ack_timeout;                 /*                                  */
-    UCHAR a_sifs;                        /*                                  */
-    UCHAR a_difs;                        /*                                  */
-    UCHAR a_pifs;                        /*                                  */
-    UCHAR a_rts_threshold[2];            /*                                  */
-    UCHAR a_scan_dwell_time[2];          /*                                  */
-    UCHAR a_max_scan_dwell_time[2];      /*                                  */
-    UCHAR a_assoc_resp_timeout_thresh;   /*                                  */
-    UCHAR a_adhoc_scan_cycle_max;        /*                                  */
-    UCHAR a_infra_scan_cycle_max;        /*                                  */
-    UCHAR a_infra_super_scan_cycle_max;  /*                                  */
-    UCHAR a_promiscuous_mode;            /*                                  */
-    UCHAR a_unique_word[2];              /*                                  */
-    UCHAR a_slot_time;                   /*                                  */
-    UCHAR a_roaming_low_snr_thresh;      /*                                  */
-    UCHAR a_low_snr_count_thresh;        /*                                  */
-    UCHAR a_infra_missed_bcn_thresh;     /*                                  */
-    UCHAR a_adhoc_missed_bcn_thresh;     /*                                  */
-    UCHAR a_curr_country_code;           /* C_USA                            */
-    UCHAR a_hop_pattern;                 /*                                  */
-    UCHAR a_hop_pattern_length;          /*                                  */
-/* b4 - b5 differences start here */
-    UCHAR a_cw_max;                      /*                                  */
-    UCHAR a_cw_min;                      /*                                  */
-    UCHAR a_noise_filter_gain;           /*                                  */
-    UCHAR a_noise_limit_offset;          /*                                  */
-    UCHAR a_det_rssi_thresh_offset;      /*                                  */
-    UCHAR a_med_busy_thresh_offset;      /*                                  */
-    UCHAR a_det_sync_thresh;             /*                                  */
-    UCHAR a_test_mode;                   /*                                  */
-    UCHAR a_test_min_chan_num;           /*                                  */
-    UCHAR a_test_max_chan_num;           /*                                  */
-    UCHAR a_rx_tx_delay;                 /*                                  */
-    UCHAR a_current_bss_id[ADDRLEN];     /*                                  */
-    UCHAR a_hop_set;                     /*                                  */
-};
-struct b5_startup_params {
-    UCHAR a_network_type;                /* C_ADHOC, C_INFRA                 */
-    UCHAR a_acting_as_ap_status;         /* C_TYPE_STA, C_TYPE_AP            */
-    UCHAR a_current_ess_id[ESSID_SIZE];  /* Null terminated unless 32 long   */
-    UCHAR a_scanning_mode;               /* passive 0, active 1              */
-    UCHAR a_power_mgt_state;             /* CAM 0,                           */
-    UCHAR a_mac_addr[ADDRLEN];           /*                                  */
-    UCHAR a_frag_threshold[2];           /* 512                              */
-    UCHAR a_hop_time[2];                 /* 16k * 2**n, n=0-4 in Kus         */
-    UCHAR a_beacon_period[2];            /* n * a_hop_time  in Kus           */
-    UCHAR a_dtim_period;                 /* in beacons                       */
-    UCHAR a_retry_max;                   /* 4                                */
-    UCHAR a_ack_timeout;                 /*                                  */
-    UCHAR a_sifs;                        /*                                  */
-    UCHAR a_difs;                        /*                                  */
-    UCHAR a_pifs;                        /*                                  */
-    UCHAR a_rts_threshold[2];            /*                                  */
-    UCHAR a_scan_dwell_time[2];          /*                                  */
-    UCHAR a_max_scan_dwell_time[2];      /*                                  */
-    UCHAR a_assoc_resp_timeout_thresh;   /*                                  */
-    UCHAR a_adhoc_scan_cycle_max;        /*                                  */
-    UCHAR a_infra_scan_cycle_max;        /*                                  */
-    UCHAR a_infra_super_scan_cycle_max;  /*                                  */
-    UCHAR a_promiscuous_mode;            /*                                  */
-    UCHAR a_unique_word[2];              /*                                  */
-    UCHAR a_slot_time;                   /*                                  */
-    UCHAR a_roaming_low_snr_thresh;      /*                                  */
-    UCHAR a_low_snr_count_thresh;        /*                                  */
-    UCHAR a_infra_missed_bcn_thresh;     /*                                  */
-    UCHAR a_adhoc_missed_bcn_thresh;     /*                                  */
-    UCHAR a_curr_country_code;           /* C_USA                            */
-    UCHAR a_hop_pattern;                 /*                                  */
-    UCHAR a_hop_pattern_length;          /*                                  */
-/* b4 - b5 differences start here */
-    UCHAR a_cw_max[2];                   /*                                  */
-    UCHAR a_cw_min[2];                   /*                                  */
-    UCHAR a_noise_filter_gain;           /*                                  */
-    UCHAR a_noise_limit_offset;          /*                                  */
-    UCHAR a_det_rssi_thresh_offset;      /*                                  */
-    UCHAR a_med_busy_thresh_offset;      /*                                  */
-    UCHAR a_det_sync_thresh;             /*                                  */
-    UCHAR a_test_mode;                   /*                                  */
-    UCHAR a_test_min_chan_num;           /*                                  */
-    UCHAR a_test_max_chan_num;           /*                                  */
-    UCHAR a_allow_bcast_SSID_probe_rsp;
-    UCHAR a_privacy_must_start;
-    UCHAR a_privacy_can_join;
-    UCHAR a_basic_rate_set[8];
-};
-
-/*****************************************************************************/
-#define RAY_IOCG_PARMS (SIOCDEVPRIVATE)
-#define RAY_IOCS_PARMS (SIOCDEVPRIVATE + 1)
-#define RAY_DO_CMD     (SIOCDEVPRIVATE + 2)
-
-/****** ethernet <-> 802.11 translation **************************************/
-typedef struct snaphdr_t
-{
-  UCHAR   dsap;
-  UCHAR   ssap;
-  UCHAR   ctrl;
-  UCHAR   org[3];
-  UCHAR   ethertype[2];
-} snaphdr_t;
-
-#define BRIDGE_ENCAP  0xf80000
-#define RFC1042_ENCAP 0
-#define SNAP_ID       0x0003aaaa
-#define RAY_IPX_TYPE  0x8137
-#define APPLEARP_TYPE 0x80f3
-/*****************************************************************************/
-#endif /* _RAYCTL_H_ */
diff --git a/drivers/net/wireless/legacy/rndis_wlan.c b/drivers/net/wireless/legacy/rndis_wlan.c
deleted file mode 100644
index e7fea7ded6d5..000000000000
--- a/drivers/net/wireless/legacy/rndis_wlan.c
+++ /dev/null
@@ -1,3760 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Driver for RNDIS based USB wireless devices.
- *
- * Copyright (C) 2007 by Bjorge Dijkstra <bjd@jooz.net>
- * Copyright (C) 2008-2009 by Jussi Kivilinna <jussi.kivilinna@iki.fi>
- *
- *  Portions of this file are based on NDISwrapper project,
- *  Copyright (C) 2003-2005 Pontus Fuchs, Giridhar Pemmasani
- *  http://ndiswrapper.sourceforge.net/
- */
-
-// #define	DEBUG			// error path messages, extra info
-// #define	VERBOSE			// more; success messages
-
-#include <linux/module.h>
-#include <linux/netdevice.h>
-#include <linux/etherdevice.h>
-#include <linux/ethtool.h>
-#include <linux/workqueue.h>
-#include <linux/mutex.h>
-#include <linux/mii.h>
-#include <linux/usb.h>
-#include <linux/usb/cdc.h>
-#include <linux/ieee80211.h>
-#include <linux/if_arp.h>
-#include <linux/ctype.h>
-#include <linux/spinlock.h>
-#include <linux/slab.h>
-#include <net/cfg80211.h>
-#include <linux/usb/usbnet.h>
-#include <linux/usb/rndis_host.h>
-
-
-/* NOTE: All these are settings for Broadcom chipset */
-static char modparam_country[4] = "EU";
-module_param_string(country, modparam_country, 4, 0444);
-MODULE_PARM_DESC(country, "Country code (ISO 3166-1 alpha-2), default: EU");
-
-static int modparam_frameburst = 1;
-module_param_named(frameburst, modparam_frameburst, int, 0444);
-MODULE_PARM_DESC(frameburst, "enable frame bursting (default: on)");
-
-static int modparam_afterburner = 0;
-module_param_named(afterburner, modparam_afterburner, int, 0444);
-MODULE_PARM_DESC(afterburner,
-	"enable afterburner aka '125 High Speed Mode' (default: off)");
-
-static int modparam_power_save = 0;
-module_param_named(power_save, modparam_power_save, int, 0444);
-MODULE_PARM_DESC(power_save,
-	"set power save mode: 0=off, 1=on, 2=fast (default: off)");
-
-static int modparam_power_output = 3;
-module_param_named(power_output, modparam_power_output, int, 0444);
-MODULE_PARM_DESC(power_output,
-	"set power output: 0=25%, 1=50%, 2=75%, 3=100% (default: 100%)");
-
-static int modparam_roamtrigger = -70;
-module_param_named(roamtrigger, modparam_roamtrigger, int, 0444);
-MODULE_PARM_DESC(roamtrigger,
-	"set roaming dBm trigger: -80=optimize for distance, "
-				"-60=bandwidth (default: -70)");
-
-static int modparam_roamdelta = 1;
-module_param_named(roamdelta, modparam_roamdelta, int, 0444);
-MODULE_PARM_DESC(roamdelta,
-	"set roaming tendency: 0=aggressive, 1=moderate, "
-				"2=conservative (default: moderate)");
-
-static int modparam_workaround_interval;
-module_param_named(workaround_interval, modparam_workaround_interval,
-							int, 0444);
-MODULE_PARM_DESC(workaround_interval,
-	"set stall workaround interval in msecs (0=disabled) (default: 0)");
-
-/* Typical noise/maximum signal level values taken from ndiswrapper iw_ndis.h */
-#define	WL_NOISE	-96	/* typical noise level in dBm */
-#define	WL_SIGMAX	-32	/* typical maximum signal level in dBm */
-
-
-/* Assume that Broadcom 4320 (only chipset at time of writing known to be
- * based on wireless rndis) has default txpower of 13dBm.
- * This value is from Linksys WUSB54GSC User Guide, Appendix F: Specifications.
- *  100% : 20 mW ~ 13dBm
- *   75% : 15 mW ~ 12dBm
- *   50% : 10 mW ~ 10dBm
- *   25% :  5 mW ~  7dBm
- */
-#define BCM4320_DEFAULT_TXPOWER_DBM_100 13
-#define BCM4320_DEFAULT_TXPOWER_DBM_75  12
-#define BCM4320_DEFAULT_TXPOWER_DBM_50  10
-#define BCM4320_DEFAULT_TXPOWER_DBM_25  7
-
-/* Known device types */
-#define RNDIS_UNKNOWN	0
-#define RNDIS_BCM4320A	1
-#define RNDIS_BCM4320B	2
-
-
-/* NDIS data structures. Taken from wpa_supplicant driver_ndis.c
- * slightly modified for datatype endianess, etc
- */
-#define NDIS_802_11_LENGTH_SSID 32
-#define NDIS_802_11_LENGTH_RATES 8
-#define NDIS_802_11_LENGTH_RATES_EX 16
-
-enum ndis_80211_net_type {
-	NDIS_80211_TYPE_FREQ_HOP,
-	NDIS_80211_TYPE_DIRECT_SEQ,
-	NDIS_80211_TYPE_OFDM_A,
-	NDIS_80211_TYPE_OFDM_G
-};
-
-enum ndis_80211_net_infra {
-	NDIS_80211_INFRA_ADHOC,
-	NDIS_80211_INFRA_INFRA,
-	NDIS_80211_INFRA_AUTO_UNKNOWN
-};
-
-enum ndis_80211_auth_mode {
-	NDIS_80211_AUTH_OPEN,
-	NDIS_80211_AUTH_SHARED,
-	NDIS_80211_AUTH_AUTO_SWITCH,
-	NDIS_80211_AUTH_WPA,
-	NDIS_80211_AUTH_WPA_PSK,
-	NDIS_80211_AUTH_WPA_NONE,
-	NDIS_80211_AUTH_WPA2,
-	NDIS_80211_AUTH_WPA2_PSK
-};
-
-enum ndis_80211_encr_status {
-	NDIS_80211_ENCR_WEP_ENABLED,
-	NDIS_80211_ENCR_DISABLED,
-	NDIS_80211_ENCR_WEP_KEY_ABSENT,
-	NDIS_80211_ENCR_NOT_SUPPORTED,
-	NDIS_80211_ENCR_TKIP_ENABLED,
-	NDIS_80211_ENCR_TKIP_KEY_ABSENT,
-	NDIS_80211_ENCR_CCMP_ENABLED,
-	NDIS_80211_ENCR_CCMP_KEY_ABSENT
-};
-
-enum ndis_80211_priv_filter {
-	NDIS_80211_PRIV_ACCEPT_ALL,
-	NDIS_80211_PRIV_8021X_WEP
-};
-
-enum ndis_80211_status_type {
-	NDIS_80211_STATUSTYPE_AUTHENTICATION,
-	NDIS_80211_STATUSTYPE_MEDIASTREAMMODE,
-	NDIS_80211_STATUSTYPE_PMKID_CANDIDATELIST,
-	NDIS_80211_STATUSTYPE_RADIOSTATE,
-};
-
-enum ndis_80211_media_stream_mode {
-	NDIS_80211_MEDIA_STREAM_OFF,
-	NDIS_80211_MEDIA_STREAM_ON
-};
-
-enum ndis_80211_radio_status {
-	NDIS_80211_RADIO_STATUS_ON,
-	NDIS_80211_RADIO_STATUS_HARDWARE_OFF,
-	NDIS_80211_RADIO_STATUS_SOFTWARE_OFF,
-};
-
-enum ndis_80211_addkey_bits {
-	NDIS_80211_ADDKEY_8021X_AUTH = cpu_to_le32(1 << 28),
-	NDIS_80211_ADDKEY_SET_INIT_RECV_SEQ = cpu_to_le32(1 << 29),
-	NDIS_80211_ADDKEY_PAIRWISE_KEY = cpu_to_le32(1 << 30),
-	NDIS_80211_ADDKEY_TRANSMIT_KEY = cpu_to_le32(1 << 31)
-};
-
-enum ndis_80211_addwep_bits {
-	NDIS_80211_ADDWEP_PERCLIENT_KEY = cpu_to_le32(1 << 30),
-	NDIS_80211_ADDWEP_TRANSMIT_KEY = cpu_to_le32(1 << 31)
-};
-
-enum ndis_80211_power_mode {
-	NDIS_80211_POWER_MODE_CAM,
-	NDIS_80211_POWER_MODE_MAX_PSP,
-	NDIS_80211_POWER_MODE_FAST_PSP,
-};
-
-enum ndis_80211_pmkid_cand_list_flag_bits {
-	NDIS_80211_PMKID_CAND_PREAUTH = cpu_to_le32(1 << 0)
-};
-
-struct ndis_80211_auth_request {
-	__le32 length;
-	u8 bssid[ETH_ALEN];
-	u8 padding[2];
-	__le32 flags;
-} __packed;
-
-struct ndis_80211_pmkid_candidate {
-	u8 bssid[ETH_ALEN];
-	u8 padding[2];
-	__le32 flags;
-} __packed;
-
-struct ndis_80211_pmkid_cand_list {
-	__le32 version;
-	__le32 num_candidates;
-	struct ndis_80211_pmkid_candidate candidate_list[];
-} __packed;
-
-struct ndis_80211_status_indication {
-	__le32 status_type;
-	union {
-		__le32					media_stream_mode;
-		__le32					radio_status;
-		DECLARE_FLEX_ARRAY(struct ndis_80211_auth_request, auth_request);
-		struct ndis_80211_pmkid_cand_list	cand_list;
-	} u;
-} __packed;
-
-struct ndis_80211_ssid {
-	__le32 length;
-	u8 essid[NDIS_802_11_LENGTH_SSID];
-} __packed;
-
-struct ndis_80211_conf_freq_hop {
-	__le32 length;
-	__le32 hop_pattern;
-	__le32 hop_set;
-	__le32 dwell_time;
-} __packed;
-
-struct ndis_80211_conf {
-	__le32 length;
-	__le32 beacon_period;
-	__le32 atim_window;
-	__le32 ds_config;
-	struct ndis_80211_conf_freq_hop fh_config;
-} __packed;
-
-struct ndis_80211_bssid_ex {
-	__le32 length;
-	u8 mac[ETH_ALEN];
-	u8 padding[2];
-	struct ndis_80211_ssid ssid;
-	__le32 privacy;
-	__le32 rssi;
-	__le32 net_type;
-	struct ndis_80211_conf config;
-	__le32 net_infra;
-	u8 rates[NDIS_802_11_LENGTH_RATES_EX];
-	__le32 ie_length;
-	u8 ies[];
-} __packed;
-
-struct ndis_80211_bssid_list_ex {
-	__le32 num_items;
-	u8 bssid_data[];
-} __packed;
-
-struct ndis_80211_fixed_ies {
-	u8 timestamp[8];
-	__le16 beacon_interval;
-	__le16 capabilities;
-} __packed;
-
-struct ndis_80211_wep_key {
-	__le32 size;
-	__le32 index;
-	__le32 length;
-	u8 material[32];
-} __packed;
-
-struct ndis_80211_key {
-	__le32 size;
-	__le32 index;
-	__le32 length;
-	u8 bssid[ETH_ALEN];
-	u8 padding[6];
-	u8 rsc[8];
-	u8 material[32];
-} __packed;
-
-struct ndis_80211_remove_key {
-	__le32 size;
-	__le32 index;
-	u8 bssid[ETH_ALEN];
-	u8 padding[2];
-} __packed;
-
-struct ndis_config_param {
-	__le32 name_offs;
-	__le32 name_length;
-	__le32 type;
-	__le32 value_offs;
-	__le32 value_length;
-} __packed;
-
-struct ndis_80211_assoc_info {
-	__le32 length;
-	__le16 req_ies;
-	struct req_ie {
-		__le16 capa;
-		__le16 listen_interval;
-		u8 cur_ap_address[ETH_ALEN];
-	} req_ie;
-	__le32 req_ie_length;
-	__le32 offset_req_ies;
-	__le16 resp_ies;
-	struct resp_ie {
-		__le16 capa;
-		__le16 status_code;
-		__le16 assoc_id;
-	} resp_ie;
-	__le32 resp_ie_length;
-	__le32 offset_resp_ies;
-} __packed;
-
-struct ndis_80211_capability {
-	__le32 length;
-	__le32 version;
-	__le32 num_pmkids;
-	__le32 num_auth_encr_pair;
-} __packed;
-
-struct ndis_80211_bssid_info {
-	u8 bssid[ETH_ALEN];
-	u8 pmkid[16];
-} __packed;
-
-struct ndis_80211_pmkid {
-	__le32 length;
-	__le32 bssid_info_count;
-	struct ndis_80211_bssid_info bssid_info[];
-} __packed;
-
-/*
- *  private data
- */
-#define CAP_MODE_80211A		1
-#define CAP_MODE_80211B		2
-#define CAP_MODE_80211G		4
-#define CAP_MODE_MASK		7
-
-#define WORK_LINK_UP		0
-#define WORK_LINK_DOWN		1
-#define WORK_SET_MULTICAST_LIST	2
-
-#define RNDIS_WLAN_ALG_NONE	0
-#define RNDIS_WLAN_ALG_WEP	(1<<0)
-#define RNDIS_WLAN_ALG_TKIP	(1<<1)
-#define RNDIS_WLAN_ALG_CCMP	(1<<2)
-
-#define RNDIS_WLAN_NUM_KEYS		4
-#define RNDIS_WLAN_KEY_MGMT_NONE	0
-#define RNDIS_WLAN_KEY_MGMT_802_1X	(1<<0)
-#define RNDIS_WLAN_KEY_MGMT_PSK		(1<<1)
-
-#define COMMAND_BUFFER_SIZE	(CONTROL_BUFFER_SIZE + sizeof(struct rndis_set))
-
-static const struct ieee80211_channel rndis_channels[] = {
-	{ .center_freq = 2412 },
-	{ .center_freq = 2417 },
-	{ .center_freq = 2422 },
-	{ .center_freq = 2427 },
-	{ .center_freq = 2432 },
-	{ .center_freq = 2437 },
-	{ .center_freq = 2442 },
-	{ .center_freq = 2447 },
-	{ .center_freq = 2452 },
-	{ .center_freq = 2457 },
-	{ .center_freq = 2462 },
-	{ .center_freq = 2467 },
-	{ .center_freq = 2472 },
-	{ .center_freq = 2484 },
-};
-
-static const struct ieee80211_rate rndis_rates[] = {
-	{ .bitrate = 10 },
-	{ .bitrate = 20, .flags = IEEE80211_RATE_SHORT_PREAMBLE },
-	{ .bitrate = 55, .flags = IEEE80211_RATE_SHORT_PREAMBLE },
-	{ .bitrate = 110, .flags = IEEE80211_RATE_SHORT_PREAMBLE },
-	{ .bitrate = 60 },
-	{ .bitrate = 90 },
-	{ .bitrate = 120 },
-	{ .bitrate = 180 },
-	{ .bitrate = 240 },
-	{ .bitrate = 360 },
-	{ .bitrate = 480 },
-	{ .bitrate = 540 }
-};
-
-static const u32 rndis_cipher_suites[] = {
-	WLAN_CIPHER_SUITE_WEP40,
-	WLAN_CIPHER_SUITE_WEP104,
-	WLAN_CIPHER_SUITE_TKIP,
-	WLAN_CIPHER_SUITE_CCMP,
-};
-
-struct rndis_wlan_encr_key {
-	int len;
-	u32 cipher;
-	u8 material[32];
-	u8 bssid[ETH_ALEN];
-	bool pairwise;
-	bool tx_key;
-};
-
-/* RNDIS device private data */
-struct rndis_wlan_private {
-	struct usbnet *usbdev;
-
-	struct wireless_dev wdev;
-
-	struct cfg80211_scan_request *scan_request;
-
-	struct workqueue_struct *workqueue;
-	struct delayed_work dev_poller_work;
-	struct delayed_work scan_work;
-	struct work_struct work;
-	struct mutex command_lock;
-	unsigned long work_pending;
-	int last_qual;
-	s32 cqm_rssi_thold;
-	u32 cqm_rssi_hyst;
-	int last_cqm_event_rssi;
-
-	struct ieee80211_supported_band band;
-	struct ieee80211_channel channels[ARRAY_SIZE(rndis_channels)];
-	struct ieee80211_rate rates[ARRAY_SIZE(rndis_rates)];
-	u32 cipher_suites[ARRAY_SIZE(rndis_cipher_suites)];
-
-	int device_type;
-	int caps;
-	int multicast_size;
-
-	/* module parameters */
-	char param_country[4];
-	int  param_frameburst;
-	int  param_afterburner;
-	int  param_power_save;
-	int  param_power_output;
-	int  param_roamtrigger;
-	int  param_roamdelta;
-	u32  param_workaround_interval;
-
-	/* hardware state */
-	bool radio_on;
-	int power_mode;
-	int infra_mode;
-	bool connected;
-	u8 bssid[ETH_ALEN];
-	u32 current_command_oid;
-
-	/* encryption stuff */
-	u8 encr_tx_key_index;
-	struct rndis_wlan_encr_key encr_keys[RNDIS_WLAN_NUM_KEYS];
-	int  wpa_version;
-
-	u8 command_buffer[COMMAND_BUFFER_SIZE];
-};
-
-/*
- * cfg80211 ops
- */
-static int rndis_change_virtual_intf(struct wiphy *wiphy,
-					struct net_device *dev,
-					enum nl80211_iftype type,
-					struct vif_params *params);
-
-static int rndis_scan(struct wiphy *wiphy,
-			struct cfg80211_scan_request *request);
-
-static int rndis_set_wiphy_params(struct wiphy *wiphy, u32 changed);
-
-static int rndis_set_tx_power(struct wiphy *wiphy,
-			      struct wireless_dev *wdev,
-			      enum nl80211_tx_power_setting type,
-			      int mbm);
-static int rndis_get_tx_power(struct wiphy *wiphy,
-			      struct wireless_dev *wdev,
-			      int *dbm);
-
-static int rndis_connect(struct wiphy *wiphy, struct net_device *dev,
-				struct cfg80211_connect_params *sme);
-
-static int rndis_disconnect(struct wiphy *wiphy, struct net_device *dev,
-				u16 reason_code);
-
-static int rndis_join_ibss(struct wiphy *wiphy, struct net_device *dev,
-					struct cfg80211_ibss_params *params);
-
-static int rndis_leave_ibss(struct wiphy *wiphy, struct net_device *dev);
-
-static int rndis_add_key(struct wiphy *wiphy, struct net_device *netdev,
-			 int link_id,  u8 key_index, bool pairwise,
-			 const u8 *mac_addr, struct key_params *params);
-
-static int rndis_del_key(struct wiphy *wiphy, struct net_device *netdev,
-			 int link_id, u8 key_index, bool pairwise,
-			 const u8 *mac_addr);
-
-static int rndis_set_default_key(struct wiphy *wiphy, struct net_device *netdev,
-				 int link_id, u8 key_index, bool unicast,
-				 bool multicast);
-
-static int rndis_get_station(struct wiphy *wiphy, struct net_device *dev,
-			     const u8 *mac, struct station_info *sinfo);
-
-static int rndis_dump_station(struct wiphy *wiphy, struct net_device *dev,
-			       int idx, u8 *mac, struct station_info *sinfo);
-
-static int rndis_set_pmksa(struct wiphy *wiphy, struct net_device *netdev,
-				struct cfg80211_pmksa *pmksa);
-
-static int rndis_del_pmksa(struct wiphy *wiphy, struct net_device *netdev,
-				struct cfg80211_pmksa *pmksa);
-
-static int rndis_flush_pmksa(struct wiphy *wiphy, struct net_device *netdev);
-
-static int rndis_set_power_mgmt(struct wiphy *wiphy, struct net_device *dev,
-				bool enabled, int timeout);
-
-static int rndis_set_cqm_rssi_config(struct wiphy *wiphy,
-					struct net_device *dev,
-					s32 rssi_thold, u32 rssi_hyst);
-
-static const struct cfg80211_ops rndis_config_ops = {
-	.change_virtual_intf = rndis_change_virtual_intf,
-	.scan = rndis_scan,
-	.set_wiphy_params = rndis_set_wiphy_params,
-	.set_tx_power = rndis_set_tx_power,
-	.get_tx_power = rndis_get_tx_power,
-	.connect = rndis_connect,
-	.disconnect = rndis_disconnect,
-	.join_ibss = rndis_join_ibss,
-	.leave_ibss = rndis_leave_ibss,
-	.add_key = rndis_add_key,
-	.del_key = rndis_del_key,
-	.set_default_key = rndis_set_default_key,
-	.get_station = rndis_get_station,
-	.dump_station = rndis_dump_station,
-	.set_pmksa = rndis_set_pmksa,
-	.del_pmksa = rndis_del_pmksa,
-	.flush_pmksa = rndis_flush_pmksa,
-	.set_power_mgmt = rndis_set_power_mgmt,
-	.set_cqm_rssi_config = rndis_set_cqm_rssi_config,
-};
-
-static void *rndis_wiphy_privid = &rndis_wiphy_privid;
-
-
-static struct rndis_wlan_private *get_rndis_wlan_priv(struct usbnet *dev)
-{
-	return (struct rndis_wlan_private *)dev->driver_priv;
-}
-
-static u32 get_bcm4320_power_dbm(struct rndis_wlan_private *priv)
-{
-	switch (priv->param_power_output) {
-	default:
-	case 3:
-		return BCM4320_DEFAULT_TXPOWER_DBM_100;
-	case 2:
-		return BCM4320_DEFAULT_TXPOWER_DBM_75;
-	case 1:
-		return BCM4320_DEFAULT_TXPOWER_DBM_50;
-	case 0:
-		return BCM4320_DEFAULT_TXPOWER_DBM_25;
-	}
-}
-
-static bool is_wpa_key(struct rndis_wlan_private *priv, u8 idx)
-{
-	int cipher = priv->encr_keys[idx].cipher;
-
-	return (cipher == WLAN_CIPHER_SUITE_CCMP ||
-		cipher == WLAN_CIPHER_SUITE_TKIP);
-}
-
-static int rndis_cipher_to_alg(u32 cipher)
-{
-	switch (cipher) {
-	default:
-		return RNDIS_WLAN_ALG_NONE;
-	case WLAN_CIPHER_SUITE_WEP40:
-	case WLAN_CIPHER_SUITE_WEP104:
-		return RNDIS_WLAN_ALG_WEP;
-	case WLAN_CIPHER_SUITE_TKIP:
-		return RNDIS_WLAN_ALG_TKIP;
-	case WLAN_CIPHER_SUITE_CCMP:
-		return RNDIS_WLAN_ALG_CCMP;
-	}
-}
-
-static int rndis_akm_suite_to_key_mgmt(u32 akm_suite)
-{
-	switch (akm_suite) {
-	default:
-		return RNDIS_WLAN_KEY_MGMT_NONE;
-	case WLAN_AKM_SUITE_8021X:
-		return RNDIS_WLAN_KEY_MGMT_802_1X;
-	case WLAN_AKM_SUITE_PSK:
-		return RNDIS_WLAN_KEY_MGMT_PSK;
-	}
-}
-
-#ifdef DEBUG
-static const char *oid_to_string(u32 oid)
-{
-	switch (oid) {
-#define OID_STR(oid) case oid: return(#oid)
-		/* from rndis_host.h */
-		OID_STR(RNDIS_OID_802_3_PERMANENT_ADDRESS);
-		OID_STR(RNDIS_OID_GEN_MAXIMUM_FRAME_SIZE);
-		OID_STR(RNDIS_OID_GEN_CURRENT_PACKET_FILTER);
-		OID_STR(RNDIS_OID_GEN_PHYSICAL_MEDIUM);
-
-		/* from rndis_wlan.c */
-		OID_STR(RNDIS_OID_GEN_LINK_SPEED);
-		OID_STR(RNDIS_OID_GEN_RNDIS_CONFIG_PARAMETER);
-
-		OID_STR(RNDIS_OID_GEN_XMIT_OK);
-		OID_STR(RNDIS_OID_GEN_RCV_OK);
-		OID_STR(RNDIS_OID_GEN_XMIT_ERROR);
-		OID_STR(RNDIS_OID_GEN_RCV_ERROR);
-		OID_STR(RNDIS_OID_GEN_RCV_NO_BUFFER);
-
-		OID_STR(RNDIS_OID_802_3_CURRENT_ADDRESS);
-		OID_STR(RNDIS_OID_802_3_MULTICAST_LIST);
-		OID_STR(RNDIS_OID_802_3_MAXIMUM_LIST_SIZE);
-
-		OID_STR(RNDIS_OID_802_11_BSSID);
-		OID_STR(RNDIS_OID_802_11_SSID);
-		OID_STR(RNDIS_OID_802_11_INFRASTRUCTURE_MODE);
-		OID_STR(RNDIS_OID_802_11_ADD_WEP);
-		OID_STR(RNDIS_OID_802_11_REMOVE_WEP);
-		OID_STR(RNDIS_OID_802_11_DISASSOCIATE);
-		OID_STR(RNDIS_OID_802_11_AUTHENTICATION_MODE);
-		OID_STR(RNDIS_OID_802_11_PRIVACY_FILTER);
-		OID_STR(RNDIS_OID_802_11_BSSID_LIST_SCAN);
-		OID_STR(RNDIS_OID_802_11_ENCRYPTION_STATUS);
-		OID_STR(RNDIS_OID_802_11_ADD_KEY);
-		OID_STR(RNDIS_OID_802_11_REMOVE_KEY);
-		OID_STR(RNDIS_OID_802_11_ASSOCIATION_INFORMATION);
-		OID_STR(RNDIS_OID_802_11_CAPABILITY);
-		OID_STR(RNDIS_OID_802_11_PMKID);
-		OID_STR(RNDIS_OID_802_11_NETWORK_TYPES_SUPPORTED);
-		OID_STR(RNDIS_OID_802_11_NETWORK_TYPE_IN_USE);
-		OID_STR(RNDIS_OID_802_11_TX_POWER_LEVEL);
-		OID_STR(RNDIS_OID_802_11_RSSI);
-		OID_STR(RNDIS_OID_802_11_RSSI_TRIGGER);
-		OID_STR(RNDIS_OID_802_11_FRAGMENTATION_THRESHOLD);
-		OID_STR(RNDIS_OID_802_11_RTS_THRESHOLD);
-		OID_STR(RNDIS_OID_802_11_SUPPORTED_RATES);
-		OID_STR(RNDIS_OID_802_11_CONFIGURATION);
-		OID_STR(RNDIS_OID_802_11_POWER_MODE);
-		OID_STR(RNDIS_OID_802_11_BSSID_LIST);
-#undef OID_STR
-	}
-
-	return "?";
-}
-#else
-static const char *oid_to_string(u32 oid)
-{
-	return "?";
-}
-#endif
-
-/* translate error code */
-static int rndis_error_status(__le32 rndis_status)
-{
-	int ret = -EINVAL;
-	switch (le32_to_cpu(rndis_status)) {
-	case RNDIS_STATUS_SUCCESS:
-		ret = 0;
-		break;
-	case RNDIS_STATUS_FAILURE:
-	case RNDIS_STATUS_INVALID_DATA:
-		ret = -EINVAL;
-		break;
-	case RNDIS_STATUS_NOT_SUPPORTED:
-		ret = -EOPNOTSUPP;
-		break;
-	case RNDIS_STATUS_ADAPTER_NOT_READY:
-	case RNDIS_STATUS_ADAPTER_NOT_OPEN:
-		ret = -EBUSY;
-		break;
-	}
-	return ret;
-}
-
-static int rndis_query_oid(struct usbnet *dev, u32 oid, void *data, int *len)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(dev);
-	union {
-		void			*buf;
-		struct rndis_msg_hdr	*header;
-		struct rndis_query	*get;
-		struct rndis_query_c	*get_c;
-	} u;
-	int ret;
-	size_t buflen, resplen, respoffs, copylen;
-
-	buflen = *len + sizeof(*u.get);
-	if (buflen < CONTROL_BUFFER_SIZE)
-		buflen = CONTROL_BUFFER_SIZE;
-
-	if (buflen > COMMAND_BUFFER_SIZE) {
-		u.buf = kmalloc(buflen, GFP_KERNEL);
-		if (!u.buf)
-			return -ENOMEM;
-	} else {
-		u.buf = priv->command_buffer;
-	}
-
-	mutex_lock(&priv->command_lock);
-
-	memset(u.get, 0, sizeof *u.get);
-	u.get->msg_type = cpu_to_le32(RNDIS_MSG_QUERY);
-	u.get->msg_len = cpu_to_le32(sizeof *u.get);
-	u.get->oid = cpu_to_le32(oid);
-
-	priv->current_command_oid = oid;
-	ret = rndis_command(dev, u.header, buflen);
-	priv->current_command_oid = 0;
-	if (ret < 0)
-		netdev_dbg(dev->net, "%s(%s): rndis_command() failed, %d (%08x)\n",
-			   __func__, oid_to_string(oid), ret,
-			   le32_to_cpu(u.get_c->status));
-
-	if (ret == 0) {
-		resplen = le32_to_cpu(u.get_c->len);
-		respoffs = le32_to_cpu(u.get_c->offset) + 8;
-
-		if (respoffs > buflen) {
-			/* Device returned data offset outside buffer, error. */
-			netdev_dbg(dev->net,
-				   "%s(%s): received invalid data offset: %zu > %zu\n",
-				   __func__, oid_to_string(oid), respoffs, buflen);
-
-			ret = -EINVAL;
-			goto exit_unlock;
-		}
-
-		copylen = min(resplen, buflen - respoffs);
-
-		if (copylen > *len)
-			copylen = *len;
-
-		memcpy(data, u.buf + respoffs, copylen);
-
-		*len = resplen;
-
-		ret = rndis_error_status(u.get_c->status);
-		if (ret < 0)
-			netdev_dbg(dev->net, "%s(%s): device returned error,  0x%08x (%d)\n",
-				   __func__, oid_to_string(oid),
-				   le32_to_cpu(u.get_c->status), ret);
-	}
-
-exit_unlock:
-	mutex_unlock(&priv->command_lock);
-
-	if (u.buf != priv->command_buffer)
-		kfree(u.buf);
-	return ret;
-}
-
-static int rndis_set_oid(struct usbnet *dev, u32 oid, const void *data,
-			 int len)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(dev);
-	union {
-		void			*buf;
-		struct rndis_msg_hdr	*header;
-		struct rndis_set	*set;
-		struct rndis_set_c	*set_c;
-	} u;
-	int ret, buflen;
-
-	buflen = len + sizeof(*u.set);
-	if (buflen < CONTROL_BUFFER_SIZE)
-		buflen = CONTROL_BUFFER_SIZE;
-
-	if (buflen > COMMAND_BUFFER_SIZE) {
-		u.buf = kmalloc(buflen, GFP_KERNEL);
-		if (!u.buf)
-			return -ENOMEM;
-	} else {
-		u.buf = priv->command_buffer;
-	}
-
-	mutex_lock(&priv->command_lock);
-
-	memset(u.set, 0, sizeof *u.set);
-	u.set->msg_type = cpu_to_le32(RNDIS_MSG_SET);
-	u.set->msg_len = cpu_to_le32(sizeof(*u.set) + len);
-	u.set->oid = cpu_to_le32(oid);
-	u.set->len = cpu_to_le32(len);
-	u.set->offset = cpu_to_le32(sizeof(*u.set) - 8);
-	u.set->handle = cpu_to_le32(0);
-	memcpy(u.buf + sizeof(*u.set), data, len);
-
-	priv->current_command_oid = oid;
-	ret = rndis_command(dev, u.header, buflen);
-	priv->current_command_oid = 0;
-	if (ret < 0)
-		netdev_dbg(dev->net, "%s(%s): rndis_command() failed, %d (%08x)\n",
-			   __func__, oid_to_string(oid), ret,
-			   le32_to_cpu(u.set_c->status));
-
-	if (ret == 0) {
-		ret = rndis_error_status(u.set_c->status);
-
-		if (ret < 0)
-			netdev_dbg(dev->net, "%s(%s): device returned error, 0x%08x (%d)\n",
-				   __func__, oid_to_string(oid),
-				   le32_to_cpu(u.set_c->status), ret);
-	}
-
-	mutex_unlock(&priv->command_lock);
-
-	if (u.buf != priv->command_buffer)
-		kfree(u.buf);
-	return ret;
-}
-
-static int rndis_reset(struct usbnet *usbdev)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	struct rndis_reset *reset;
-	int ret;
-
-	mutex_lock(&priv->command_lock);
-
-	reset = (void *)priv->command_buffer;
-	memset(reset, 0, sizeof(*reset));
-	reset->msg_type = cpu_to_le32(RNDIS_MSG_RESET);
-	reset->msg_len = cpu_to_le32(sizeof(*reset));
-	priv->current_command_oid = 0;
-	ret = rndis_command(usbdev, (void *)reset, CONTROL_BUFFER_SIZE);
-
-	mutex_unlock(&priv->command_lock);
-
-	if (ret < 0)
-		return ret;
-	return 0;
-}
-
-/*
- * Specs say that we can only set config parameters only soon after device
- * initialization.
- *   value_type: 0 = u32, 2 = unicode string
- */
-static int rndis_set_config_parameter(struct usbnet *dev, char *param,
-						int value_type, void *value)
-{
-	struct ndis_config_param *infobuf;
-	int value_len, info_len, param_len, ret, i;
-	__le16 *unibuf;
-	__le32 *dst_value;
-
-	if (value_type == 0)
-		value_len = sizeof(__le32);
-	else if (value_type == 2)
-		value_len = strlen(value) * sizeof(__le16);
-	else
-		return -EINVAL;
-
-	param_len = strlen(param) * sizeof(__le16);
-	info_len = sizeof(*infobuf) + param_len + value_len;
-
-#ifdef DEBUG
-	info_len += 12;
-#endif
-	infobuf = kmalloc(info_len, GFP_KERNEL);
-	if (!infobuf)
-		return -ENOMEM;
-
-#ifdef DEBUG
-	info_len -= 12;
-	/* extra 12 bytes are for padding (debug output) */
-	memset(infobuf, 0xCC, info_len + 12);
-#endif
-
-	if (value_type == 2)
-		netdev_dbg(dev->net, "setting config parameter: %s, value: %s\n",
-			   param, (u8 *)value);
-	else
-		netdev_dbg(dev->net, "setting config parameter: %s, value: %d\n",
-			   param, *(u32 *)value);
-
-	infobuf->name_offs = cpu_to_le32(sizeof(*infobuf));
-	infobuf->name_length = cpu_to_le32(param_len);
-	infobuf->type = cpu_to_le32(value_type);
-	infobuf->value_offs = cpu_to_le32(sizeof(*infobuf) + param_len);
-	infobuf->value_length = cpu_to_le32(value_len);
-
-	/* simple string to unicode string conversion */
-	unibuf = (void *)infobuf + sizeof(*infobuf);
-	for (i = 0; i < param_len / sizeof(__le16); i++)
-		unibuf[i] = cpu_to_le16(param[i]);
-
-	if (value_type == 2) {
-		unibuf = (void *)infobuf + sizeof(*infobuf) + param_len;
-		for (i = 0; i < value_len / sizeof(__le16); i++)
-			unibuf[i] = cpu_to_le16(((u8 *)value)[i]);
-	} else {
-		dst_value = (void *)infobuf + sizeof(*infobuf) + param_len;
-		*dst_value = cpu_to_le32(*(u32 *)value);
-	}
-
-#ifdef DEBUG
-	netdev_dbg(dev->net, "info buffer (len: %d)\n", info_len);
-	for (i = 0; i < info_len; i += 12) {
-		u32 *tmp = (u32 *)((u8 *)infobuf + i);
-		netdev_dbg(dev->net, "%08X:%08X:%08X\n",
-			   cpu_to_be32(tmp[0]),
-			   cpu_to_be32(tmp[1]),
-			   cpu_to_be32(tmp[2]));
-	}
-#endif
-
-	ret = rndis_set_oid(dev, RNDIS_OID_GEN_RNDIS_CONFIG_PARAMETER,
-							infobuf, info_len);
-	if (ret != 0)
-		netdev_dbg(dev->net, "setting rndis config parameter failed, %d\n",
-			   ret);
-
-	kfree(infobuf);
-	return ret;
-}
-
-static int rndis_set_config_parameter_str(struct usbnet *dev,
-						char *param, char *value)
-{
-	return rndis_set_config_parameter(dev, param, 2, value);
-}
-
-/*
- * data conversion functions
- */
-static int level_to_qual(int level)
-{
-	int qual = 100 * (level - WL_NOISE) / (WL_SIGMAX - WL_NOISE);
-	return qual >= 0 ? (qual <= 100 ? qual : 100) : 0;
-}
-
-/*
- * common functions
- */
-static int set_infra_mode(struct usbnet *usbdev, int mode);
-static void restore_keys(struct usbnet *usbdev);
-static int rndis_check_bssid_list(struct usbnet *usbdev, u8 *match_bssid,
-					bool *matched);
-
-static int rndis_start_bssid_list_scan(struct usbnet *usbdev)
-{
-	__le32 tmp;
-
-	/* Note: RNDIS_OID_802_11_BSSID_LIST_SCAN clears internal BSS list. */
-	tmp = cpu_to_le32(1);
-	return rndis_set_oid(usbdev, RNDIS_OID_802_11_BSSID_LIST_SCAN, &tmp,
-							sizeof(tmp));
-}
-
-static int set_essid(struct usbnet *usbdev, struct ndis_80211_ssid *ssid)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	int ret;
-
-	ret = rndis_set_oid(usbdev, RNDIS_OID_802_11_SSID,
-			    ssid, sizeof(*ssid));
-	if (ret < 0) {
-		netdev_warn(usbdev->net, "setting SSID failed (%08X)\n", ret);
-		return ret;
-	}
-	if (ret == 0) {
-		priv->radio_on = true;
-		netdev_dbg(usbdev->net, "%s(): radio_on = true\n", __func__);
-	}
-
-	return ret;
-}
-
-static int set_bssid(struct usbnet *usbdev, const u8 *bssid)
-{
-	int ret;
-
-	ret = rndis_set_oid(usbdev, RNDIS_OID_802_11_BSSID,
-			    bssid, ETH_ALEN);
-	if (ret < 0) {
-		netdev_warn(usbdev->net, "setting BSSID[%pM] failed (%08X)\n",
-			    bssid, ret);
-		return ret;
-	}
-
-	return ret;
-}
-
-static int clear_bssid(struct usbnet *usbdev)
-{
-	static const u8 broadcast_mac[ETH_ALEN] = {
-		0xff, 0xff, 0xff, 0xff, 0xff, 0xff
-	};
-
-	return set_bssid(usbdev, broadcast_mac);
-}
-
-static int get_bssid(struct usbnet *usbdev, u8 bssid[ETH_ALEN])
-{
-	int ret, len;
-
-	len = ETH_ALEN;
-	ret = rndis_query_oid(usbdev, RNDIS_OID_802_11_BSSID,
-			      bssid, &len);
-
-	if (ret != 0)
-		eth_zero_addr(bssid);
-
-	return ret;
-}
-
-static int get_association_info(struct usbnet *usbdev,
-			struct ndis_80211_assoc_info *info, int len)
-{
-	return rndis_query_oid(usbdev,
-			RNDIS_OID_802_11_ASSOCIATION_INFORMATION,
-			info, &len);
-}
-
-static bool is_associated(struct usbnet *usbdev)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	u8 bssid[ETH_ALEN];
-
-	if (!priv->radio_on)
-		return false;
-
-	return (get_bssid(usbdev, bssid) == 0 && !is_zero_ether_addr(bssid));
-}
-
-static int disassociate(struct usbnet *usbdev, bool reset_ssid)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	struct ndis_80211_ssid ssid;
-	int i, ret = 0;
-
-	if (priv->radio_on) {
-		ret = rndis_set_oid(usbdev,
-				RNDIS_OID_802_11_DISASSOCIATE,
-				NULL, 0);
-		if (ret == 0) {
-			priv->radio_on = false;
-			netdev_dbg(usbdev->net, "%s(): radio_on = false\n",
-				   __func__);
-
-			if (reset_ssid)
-				msleep(100);
-		}
-	}
-
-	/* disassociate causes radio to be turned off; if reset_ssid
-	 * is given, set random ssid to enable radio */
-	if (reset_ssid) {
-		/* Set device to infrastructure mode so we don't get ad-hoc
-		 * 'media connect' indications with the random ssid.
-		 */
-		set_infra_mode(usbdev, NDIS_80211_INFRA_INFRA);
-
-		ssid.length = cpu_to_le32(sizeof(ssid.essid));
-		get_random_bytes(&ssid.essid[2], sizeof(ssid.essid)-2);
-		ssid.essid[0] = 0x1;
-		ssid.essid[1] = 0xff;
-		for (i = 2; i < sizeof(ssid.essid); i++)
-			ssid.essid[i] = 0x1 + (ssid.essid[i] * 0xfe / 0xff);
-		ret = set_essid(usbdev, &ssid);
-	}
-	return ret;
-}
-
-static int set_auth_mode(struct usbnet *usbdev, u32 wpa_version,
-				enum nl80211_auth_type auth_type, int keymgmt)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	__le32 tmp;
-	int auth_mode, ret;
-
-	netdev_dbg(usbdev->net, "%s(): wpa_version=0x%x authalg=0x%x keymgmt=0x%x\n",
-		   __func__, wpa_version, auth_type, keymgmt);
-
-	if (wpa_version & NL80211_WPA_VERSION_2) {
-		if (keymgmt & RNDIS_WLAN_KEY_MGMT_802_1X)
-			auth_mode = NDIS_80211_AUTH_WPA2;
-		else
-			auth_mode = NDIS_80211_AUTH_WPA2_PSK;
-	} else if (wpa_version & NL80211_WPA_VERSION_1) {
-		if (keymgmt & RNDIS_WLAN_KEY_MGMT_802_1X)
-			auth_mode = NDIS_80211_AUTH_WPA;
-		else if (keymgmt & RNDIS_WLAN_KEY_MGMT_PSK)
-			auth_mode = NDIS_80211_AUTH_WPA_PSK;
-		else
-			auth_mode = NDIS_80211_AUTH_WPA_NONE;
-	} else if (auth_type == NL80211_AUTHTYPE_SHARED_KEY)
-		auth_mode = NDIS_80211_AUTH_SHARED;
-	else if (auth_type == NL80211_AUTHTYPE_OPEN_SYSTEM)
-		auth_mode = NDIS_80211_AUTH_OPEN;
-	else if (auth_type == NL80211_AUTHTYPE_AUTOMATIC)
-		auth_mode = NDIS_80211_AUTH_AUTO_SWITCH;
-	else
-		return -ENOTSUPP;
-
-	tmp = cpu_to_le32(auth_mode);
-	ret = rndis_set_oid(usbdev,
-			    RNDIS_OID_802_11_AUTHENTICATION_MODE,
-			    &tmp, sizeof(tmp));
-	if (ret != 0) {
-		netdev_warn(usbdev->net, "setting auth mode failed (%08X)\n",
-			    ret);
-		return ret;
-	}
-
-	priv->wpa_version = wpa_version;
-
-	return 0;
-}
-
-static int set_priv_filter(struct usbnet *usbdev)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	__le32 tmp;
-
-	netdev_dbg(usbdev->net, "%s(): wpa_version=0x%x\n",
-		   __func__, priv->wpa_version);
-
-	if (priv->wpa_version & NL80211_WPA_VERSION_2 ||
-	    priv->wpa_version & NL80211_WPA_VERSION_1)
-		tmp = cpu_to_le32(NDIS_80211_PRIV_8021X_WEP);
-	else
-		tmp = cpu_to_le32(NDIS_80211_PRIV_ACCEPT_ALL);
-
-	return rndis_set_oid(usbdev,
-			     RNDIS_OID_802_11_PRIVACY_FILTER, &tmp,
-			     sizeof(tmp));
-}
-
-static int set_encr_mode(struct usbnet *usbdev, int pairwise, int groupwise)
-{
-	__le32 tmp;
-	int encr_mode, ret;
-
-	netdev_dbg(usbdev->net, "%s(): cipher_pair=0x%x cipher_group=0x%x\n",
-		   __func__, pairwise, groupwise);
-
-	if (pairwise & RNDIS_WLAN_ALG_CCMP)
-		encr_mode = NDIS_80211_ENCR_CCMP_ENABLED;
-	else if (pairwise & RNDIS_WLAN_ALG_TKIP)
-		encr_mode = NDIS_80211_ENCR_TKIP_ENABLED;
-	else if (pairwise & RNDIS_WLAN_ALG_WEP)
-		encr_mode = NDIS_80211_ENCR_WEP_ENABLED;
-	else if (groupwise & RNDIS_WLAN_ALG_CCMP)
-		encr_mode = NDIS_80211_ENCR_CCMP_ENABLED;
-	else if (groupwise & RNDIS_WLAN_ALG_TKIP)
-		encr_mode = NDIS_80211_ENCR_TKIP_ENABLED;
-	else
-		encr_mode = NDIS_80211_ENCR_DISABLED;
-
-	tmp = cpu_to_le32(encr_mode);
-	ret = rndis_set_oid(usbdev,
-			RNDIS_OID_802_11_ENCRYPTION_STATUS, &tmp,
-			sizeof(tmp));
-	if (ret != 0) {
-		netdev_warn(usbdev->net, "setting encr mode failed (%08X)\n",
-			    ret);
-		return ret;
-	}
-
-	return 0;
-}
-
-static int set_infra_mode(struct usbnet *usbdev, int mode)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	__le32 tmp;
-	int ret;
-
-	netdev_dbg(usbdev->net, "%s(): infra_mode=0x%x\n",
-		   __func__, priv->infra_mode);
-
-	tmp = cpu_to_le32(mode);
-	ret = rndis_set_oid(usbdev,
-			    RNDIS_OID_802_11_INFRASTRUCTURE_MODE,
-			    &tmp, sizeof(tmp));
-	if (ret != 0) {
-		netdev_warn(usbdev->net, "setting infra mode failed (%08X)\n",
-			    ret);
-		return ret;
-	}
-
-	/* NDIS drivers clear keys when infrastructure mode is
-	 * changed. But Linux tools assume otherwise. So set the
-	 * keys */
-	restore_keys(usbdev);
-
-	priv->infra_mode = mode;
-	return 0;
-}
-
-static int set_rts_threshold(struct usbnet *usbdev, u32 rts_threshold)
-{
-	__le32 tmp;
-
-	netdev_dbg(usbdev->net, "%s(): %i\n", __func__, rts_threshold);
-
-	if (rts_threshold == -1 || rts_threshold > 2347)
-		rts_threshold = 2347;
-
-	tmp = cpu_to_le32(rts_threshold);
-	return rndis_set_oid(usbdev,
-			     RNDIS_OID_802_11_RTS_THRESHOLD,
-			     &tmp, sizeof(tmp));
-}
-
-static int set_frag_threshold(struct usbnet *usbdev, u32 frag_threshold)
-{
-	__le32 tmp;
-
-	netdev_dbg(usbdev->net, "%s(): %i\n", __func__, frag_threshold);
-
-	if (frag_threshold < 256 || frag_threshold > 2346)
-		frag_threshold = 2346;
-
-	tmp = cpu_to_le32(frag_threshold);
-	return rndis_set_oid(usbdev,
-			RNDIS_OID_802_11_FRAGMENTATION_THRESHOLD,
-			&tmp, sizeof(tmp));
-}
-
-static void set_default_iw_params(struct usbnet *usbdev)
-{
-	set_infra_mode(usbdev, NDIS_80211_INFRA_INFRA);
-	set_auth_mode(usbdev, 0, NL80211_AUTHTYPE_OPEN_SYSTEM,
-						RNDIS_WLAN_KEY_MGMT_NONE);
-	set_priv_filter(usbdev);
-	set_encr_mode(usbdev, RNDIS_WLAN_ALG_NONE, RNDIS_WLAN_ALG_NONE);
-}
-
-static int deauthenticate(struct usbnet *usbdev)
-{
-	int ret;
-
-	ret = disassociate(usbdev, true);
-	set_default_iw_params(usbdev);
-	return ret;
-}
-
-static int set_channel(struct usbnet *usbdev, int channel)
-{
-	struct ndis_80211_conf config;
-	unsigned int dsconfig;
-	int len, ret;
-
-	netdev_dbg(usbdev->net, "%s(%d)\n", __func__, channel);
-
-	/* this OID is valid only when not associated */
-	if (is_associated(usbdev))
-		return 0;
-
-	dsconfig = 1000 *
-		ieee80211_channel_to_frequency(channel, NL80211_BAND_2GHZ);
-
-	len = sizeof(config);
-	ret = rndis_query_oid(usbdev,
-			RNDIS_OID_802_11_CONFIGURATION,
-			&config, &len);
-	if (ret < 0) {
-		netdev_dbg(usbdev->net, "%s(): querying configuration failed\n",
-			   __func__);
-		return ret;
-	}
-
-	config.ds_config = cpu_to_le32(dsconfig);
-	ret = rndis_set_oid(usbdev,
-			RNDIS_OID_802_11_CONFIGURATION,
-			&config, sizeof(config));
-
-	netdev_dbg(usbdev->net, "%s(): %d -> %d\n", __func__, channel, ret);
-
-	return ret;
-}
-
-static struct ieee80211_channel *get_current_channel(struct usbnet *usbdev,
-						     u32 *beacon_period)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	struct ieee80211_channel *channel;
-	struct ndis_80211_conf config;
-	int len, ret;
-
-	/* Get channel and beacon interval */
-	len = sizeof(config);
-	ret = rndis_query_oid(usbdev,
-			RNDIS_OID_802_11_CONFIGURATION,
-			&config, &len);
-	netdev_dbg(usbdev->net, "%s(): RNDIS_OID_802_11_CONFIGURATION -> %d\n",
-				__func__, ret);
-	if (ret < 0)
-		return NULL;
-
-	channel = ieee80211_get_channel(priv->wdev.wiphy,
-				KHZ_TO_MHZ(le32_to_cpu(config.ds_config)));
-	if (!channel)
-		return NULL;
-
-	if (beacon_period)
-		*beacon_period = le32_to_cpu(config.beacon_period);
-	return channel;
-}
-
-/* index must be 0 - N, as per NDIS  */
-static int add_wep_key(struct usbnet *usbdev, const u8 *key, int key_len,
-								u8 index)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	struct ndis_80211_wep_key ndis_key;
-	u32 cipher;
-	int ret;
-
-	netdev_dbg(usbdev->net, "%s(idx: %d, len: %d)\n",
-		   __func__, index, key_len);
-
-	if (index >= RNDIS_WLAN_NUM_KEYS)
-		return -EINVAL;
-
-	if (key_len == 5)
-		cipher = WLAN_CIPHER_SUITE_WEP40;
-	else if (key_len == 13)
-		cipher = WLAN_CIPHER_SUITE_WEP104;
-	else
-		return -EINVAL;
-
-	memset(&ndis_key, 0, sizeof(ndis_key));
-
-	ndis_key.size = cpu_to_le32(sizeof(ndis_key));
-	ndis_key.length = cpu_to_le32(key_len);
-	ndis_key.index = cpu_to_le32(index);
-	memcpy(&ndis_key.material, key, key_len);
-
-	if (index == priv->encr_tx_key_index) {
-		ndis_key.index |= NDIS_80211_ADDWEP_TRANSMIT_KEY;
-		ret = set_encr_mode(usbdev, RNDIS_WLAN_ALG_WEP,
-							RNDIS_WLAN_ALG_NONE);
-		if (ret)
-			netdev_warn(usbdev->net, "encryption couldn't be enabled (%08X)\n",
-				    ret);
-	}
-
-	ret = rndis_set_oid(usbdev,
-			RNDIS_OID_802_11_ADD_WEP, &ndis_key,
-			sizeof(ndis_key));
-	if (ret != 0) {
-		netdev_warn(usbdev->net, "adding encryption key %d failed (%08X)\n",
-			    index + 1, ret);
-		return ret;
-	}
-
-	priv->encr_keys[index].len = key_len;
-	priv->encr_keys[index].cipher = cipher;
-	memcpy(&priv->encr_keys[index].material, key, key_len);
-	eth_broadcast_addr(priv->encr_keys[index].bssid);
-
-	return 0;
-}
-
-static int add_wpa_key(struct usbnet *usbdev, const u8 *key, int key_len,
-			u8 index, const u8 *addr, const u8 *rx_seq,
-			int seq_len, u32 cipher, __le32 flags)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	struct ndis_80211_key ndis_key;
-	bool is_addr_ok;
-	int ret;
-
-	if (index >= RNDIS_WLAN_NUM_KEYS) {
-		netdev_dbg(usbdev->net, "%s(): index out of range (%i)\n",
-			   __func__, index);
-		return -EINVAL;
-	}
-	if (key_len > sizeof(ndis_key.material) || key_len < 0) {
-		netdev_dbg(usbdev->net, "%s(): key length out of range (%i)\n",
-			   __func__, key_len);
-		return -EINVAL;
-	}
-	if (flags & NDIS_80211_ADDKEY_SET_INIT_RECV_SEQ) {
-		if (!rx_seq || seq_len <= 0) {
-			netdev_dbg(usbdev->net, "%s(): recv seq flag without buffer\n",
-				   __func__);
-			return -EINVAL;
-		}
-		if (rx_seq && seq_len > sizeof(ndis_key.rsc)) {
-			netdev_dbg(usbdev->net, "%s(): too big recv seq buffer\n", __func__);
-			return -EINVAL;
-		}
-	}
-
-	is_addr_ok = addr && !is_zero_ether_addr(addr) &&
-					!is_broadcast_ether_addr(addr);
-	if ((flags & NDIS_80211_ADDKEY_PAIRWISE_KEY) && !is_addr_ok) {
-		netdev_dbg(usbdev->net, "%s(): pairwise but bssid invalid (%pM)\n",
-			   __func__, addr);
-		return -EINVAL;
-	}
-
-	netdev_dbg(usbdev->net, "%s(%i): flags:%i%i%i\n",
-		   __func__, index,
-		   !!(flags & NDIS_80211_ADDKEY_TRANSMIT_KEY),
-		   !!(flags & NDIS_80211_ADDKEY_PAIRWISE_KEY),
-		   !!(flags & NDIS_80211_ADDKEY_SET_INIT_RECV_SEQ));
-
-	memset(&ndis_key, 0, sizeof(ndis_key));
-
-	ndis_key.size = cpu_to_le32(sizeof(ndis_key) -
-				sizeof(ndis_key.material) + key_len);
-	ndis_key.length = cpu_to_le32(key_len);
-	ndis_key.index = cpu_to_le32(index) | flags;
-
-	if (cipher == WLAN_CIPHER_SUITE_TKIP && key_len == 32) {
-		/* wpa_supplicant gives us the Michael MIC RX/TX keys in
-		 * different order than NDIS spec, so swap the order here. */
-		memcpy(ndis_key.material, key, 16);
-		memcpy(ndis_key.material + 16, key + 24, 8);
-		memcpy(ndis_key.material + 24, key + 16, 8);
-	} else
-		memcpy(ndis_key.material, key, key_len);
-
-	if (flags & NDIS_80211_ADDKEY_SET_INIT_RECV_SEQ)
-		memcpy(ndis_key.rsc, rx_seq, seq_len);
-
-	if (flags & NDIS_80211_ADDKEY_PAIRWISE_KEY) {
-		/* pairwise key */
-		memcpy(ndis_key.bssid, addr, ETH_ALEN);
-	} else {
-		/* group key */
-		if (priv->infra_mode == NDIS_80211_INFRA_ADHOC)
-			eth_broadcast_addr(ndis_key.bssid);
-		else
-			get_bssid(usbdev, ndis_key.bssid);
-	}
-
-	ret = rndis_set_oid(usbdev,
-			RNDIS_OID_802_11_ADD_KEY, &ndis_key,
-			le32_to_cpu(ndis_key.size));
-	netdev_dbg(usbdev->net, "%s(): RNDIS_OID_802_11_ADD_KEY -> %08X\n",
-		   __func__, ret);
-	if (ret != 0)
-		return ret;
-
-	memset(&priv->encr_keys[index], 0, sizeof(priv->encr_keys[index]));
-	priv->encr_keys[index].len = key_len;
-	priv->encr_keys[index].cipher = cipher;
-	memcpy(&priv->encr_keys[index].material, key, key_len);
-	if (flags & NDIS_80211_ADDKEY_PAIRWISE_KEY)
-		memcpy(&priv->encr_keys[index].bssid, ndis_key.bssid, ETH_ALEN);
-	else
-		eth_broadcast_addr(priv->encr_keys[index].bssid);
-
-	if (flags & NDIS_80211_ADDKEY_TRANSMIT_KEY)
-		priv->encr_tx_key_index = index;
-
-	return 0;
-}
-
-static int restore_key(struct usbnet *usbdev, u8 key_idx)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	struct rndis_wlan_encr_key key;
-
-	if (is_wpa_key(priv, key_idx))
-		return 0;
-
-	key = priv->encr_keys[key_idx];
-
-	netdev_dbg(usbdev->net, "%s(): %i:%i\n", __func__, key_idx, key.len);
-
-	if (key.len == 0)
-		return 0;
-
-	return add_wep_key(usbdev, key.material, key.len, key_idx);
-}
-
-static void restore_keys(struct usbnet *usbdev)
-{
-	int i;
-
-	for (i = 0; i < 4; i++)
-		restore_key(usbdev, i);
-}
-
-static void clear_key(struct rndis_wlan_private *priv, u8 idx)
-{
-	memset(&priv->encr_keys[idx], 0, sizeof(priv->encr_keys[idx]));
-}
-
-/* remove_key is for both wep and wpa */
-static int remove_key(struct usbnet *usbdev, u8 index, const u8 *bssid)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	struct ndis_80211_remove_key remove_key;
-	__le32 keyindex;
-	bool is_wpa;
-	int ret;
-
-	if (index >= RNDIS_WLAN_NUM_KEYS)
-		return -ENOENT;
-
-	if (priv->encr_keys[index].len == 0)
-		return 0;
-
-	is_wpa = is_wpa_key(priv, index);
-
-	netdev_dbg(usbdev->net, "%s(): %i:%s:%i\n",
-		   __func__, index, is_wpa ? "wpa" : "wep",
-		   priv->encr_keys[index].len);
-
-	clear_key(priv, index);
-
-	if (is_wpa) {
-		remove_key.size = cpu_to_le32(sizeof(remove_key));
-		remove_key.index = cpu_to_le32(index);
-		if (bssid) {
-			/* pairwise key */
-			if (!is_broadcast_ether_addr(bssid))
-				remove_key.index |=
-					NDIS_80211_ADDKEY_PAIRWISE_KEY;
-			memcpy(remove_key.bssid, bssid,
-					sizeof(remove_key.bssid));
-		} else
-			memset(remove_key.bssid, 0xff,
-						sizeof(remove_key.bssid));
-
-		ret = rndis_set_oid(usbdev,
-				RNDIS_OID_802_11_REMOVE_KEY,
-				&remove_key, sizeof(remove_key));
-		if (ret != 0)
-			return ret;
-	} else {
-		keyindex = cpu_to_le32(index);
-		ret = rndis_set_oid(usbdev,
-				RNDIS_OID_802_11_REMOVE_WEP,
-				&keyindex, sizeof(keyindex));
-		if (ret != 0) {
-			netdev_warn(usbdev->net,
-				    "removing encryption key %d failed (%08X)\n",
-				    index, ret);
-			return ret;
-		}
-	}
-
-	/* if it is transmit key, disable encryption */
-	if (index == priv->encr_tx_key_index)
-		set_encr_mode(usbdev, RNDIS_WLAN_ALG_NONE, RNDIS_WLAN_ALG_NONE);
-
-	return 0;
-}
-
-static void set_multicast_list(struct usbnet *usbdev)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	struct netdev_hw_addr *ha;
-	__le32 filter, basefilter;
-	int ret;
-	char *mc_addrs = NULL;
-	int mc_count;
-
-	basefilter = filter = cpu_to_le32(RNDIS_PACKET_TYPE_DIRECTED |
-					  RNDIS_PACKET_TYPE_BROADCAST);
-
-	if (usbdev->net->flags & IFF_PROMISC) {
-		filter |= cpu_to_le32(RNDIS_PACKET_TYPE_PROMISCUOUS |
-				      RNDIS_PACKET_TYPE_ALL_LOCAL);
-	} else if (usbdev->net->flags & IFF_ALLMULTI) {
-		filter |= cpu_to_le32(RNDIS_PACKET_TYPE_ALL_MULTICAST);
-	}
-
-	if (filter != basefilter)
-		goto set_filter;
-
-	/*
-	 * mc_list should be accessed holding the lock, so copy addresses to
-	 * local buffer first.
-	 */
-	netif_addr_lock_bh(usbdev->net);
-	mc_count = netdev_mc_count(usbdev->net);
-	if (mc_count > priv->multicast_size) {
-		filter |= cpu_to_le32(RNDIS_PACKET_TYPE_ALL_MULTICAST);
-	} else if (mc_count) {
-		int i = 0;
-
-		mc_addrs = kmalloc_array(mc_count, ETH_ALEN, GFP_ATOMIC);
-		if (!mc_addrs) {
-			netif_addr_unlock_bh(usbdev->net);
-			return;
-		}
-
-		netdev_for_each_mc_addr(ha, usbdev->net)
-			memcpy(mc_addrs + i++ * ETH_ALEN,
-			       ha->addr, ETH_ALEN);
-	}
-	netif_addr_unlock_bh(usbdev->net);
-
-	if (filter != basefilter)
-		goto set_filter;
-
-	if (mc_count) {
-		ret = rndis_set_oid(usbdev,
-				RNDIS_OID_802_3_MULTICAST_LIST,
-				mc_addrs, mc_count * ETH_ALEN);
-		kfree(mc_addrs);
-		if (ret == 0)
-			filter |= cpu_to_le32(RNDIS_PACKET_TYPE_MULTICAST);
-		else
-			filter |= cpu_to_le32(RNDIS_PACKET_TYPE_ALL_MULTICAST);
-
-		netdev_dbg(usbdev->net, "RNDIS_OID_802_3_MULTICAST_LIST(%d, max: %d) -> %d\n",
-			   mc_count, priv->multicast_size, ret);
-	}
-
-set_filter:
-	ret = rndis_set_oid(usbdev, RNDIS_OID_GEN_CURRENT_PACKET_FILTER, &filter,
-							sizeof(filter));
-	if (ret < 0) {
-		netdev_warn(usbdev->net, "couldn't set packet filter: %08x\n",
-			    le32_to_cpu(filter));
-	}
-
-	netdev_dbg(usbdev->net, "RNDIS_OID_GEN_CURRENT_PACKET_FILTER(%08x) -> %d\n",
-		   le32_to_cpu(filter), ret);
-}
-
-#ifdef DEBUG
-static void debug_print_pmkids(struct usbnet *usbdev,
-				struct ndis_80211_pmkid *pmkids,
-				const char *func_str)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	int i, len, count, max_pmkids, entry_len;
-
-	max_pmkids = priv->wdev.wiphy->max_num_pmkids;
-	len = le32_to_cpu(pmkids->length);
-	count = le32_to_cpu(pmkids->bssid_info_count);
-
-	entry_len = (count > 0) ? (len - sizeof(*pmkids)) / count : -1;
-
-	netdev_dbg(usbdev->net, "%s(): %d PMKIDs (data len: %d, entry len: "
-				"%d)\n", func_str, count, len, entry_len);
-
-	if (count > max_pmkids)
-		count = max_pmkids;
-
-	for (i = 0; i < count; i++) {
-		u32 *tmp = (u32 *)pmkids->bssid_info[i].pmkid;
-
-		netdev_dbg(usbdev->net, "%s():  bssid: %pM, "
-				"pmkid: %08X:%08X:%08X:%08X\n",
-				func_str, pmkids->bssid_info[i].bssid,
-				cpu_to_be32(tmp[0]), cpu_to_be32(tmp[1]),
-				cpu_to_be32(tmp[2]), cpu_to_be32(tmp[3]));
-	}
-}
-#else
-static void debug_print_pmkids(struct usbnet *usbdev,
-				struct ndis_80211_pmkid *pmkids,
-				const char *func_str)
-{
-	return;
-}
-#endif
-
-static struct ndis_80211_pmkid *get_device_pmkids(struct usbnet *usbdev)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	struct ndis_80211_pmkid *pmkids;
-	int len, ret, max_pmkids;
-
-	max_pmkids = priv->wdev.wiphy->max_num_pmkids;
-	len = struct_size(pmkids, bssid_info, max_pmkids);
-
-	pmkids = kzalloc(len, GFP_KERNEL);
-	if (!pmkids)
-		return ERR_PTR(-ENOMEM);
-
-	pmkids->length = cpu_to_le32(len);
-	pmkids->bssid_info_count = cpu_to_le32(max_pmkids);
-
-	ret = rndis_query_oid(usbdev, RNDIS_OID_802_11_PMKID,
-			pmkids, &len);
-	if (ret < 0) {
-		netdev_dbg(usbdev->net, "%s(): RNDIS_OID_802_11_PMKID(%d, %d)"
-				" -> %d\n", __func__, len, max_pmkids, ret);
-
-		kfree(pmkids);
-		return ERR_PTR(ret);
-	}
-
-	if (le32_to_cpu(pmkids->bssid_info_count) > max_pmkids)
-		pmkids->bssid_info_count = cpu_to_le32(max_pmkids);
-
-	debug_print_pmkids(usbdev, pmkids, __func__);
-
-	return pmkids;
-}
-
-static int set_device_pmkids(struct usbnet *usbdev,
-				struct ndis_80211_pmkid *pmkids)
-{
-	int ret, len, num_pmkids;
-
-	num_pmkids = le32_to_cpu(pmkids->bssid_info_count);
-	len = struct_size(pmkids, bssid_info, num_pmkids);
-	pmkids->length = cpu_to_le32(len);
-
-	debug_print_pmkids(usbdev, pmkids, __func__);
-
-	ret = rndis_set_oid(usbdev, RNDIS_OID_802_11_PMKID, pmkids,
-			    le32_to_cpu(pmkids->length));
-	if (ret < 0) {
-		netdev_dbg(usbdev->net, "%s(): RNDIS_OID_802_11_PMKID(%d, %d) -> %d"
-				"\n", __func__, len, num_pmkids, ret);
-	}
-
-	kfree(pmkids);
-	return ret;
-}
-
-static struct ndis_80211_pmkid *remove_pmkid(struct usbnet *usbdev,
-						struct ndis_80211_pmkid *pmkids,
-						struct cfg80211_pmksa *pmksa,
-						int max_pmkids)
-{
-	int i, err;
-	unsigned int count;
-
-	count = le32_to_cpu(pmkids->bssid_info_count);
-
-	if (count > max_pmkids)
-		count = max_pmkids;
-
-	for (i = 0; i < count; i++)
-		if (ether_addr_equal(pmkids->bssid_info[i].bssid,
-				     pmksa->bssid))
-			break;
-
-	/* pmkid not found */
-	if (i == count) {
-		netdev_dbg(usbdev->net, "%s(): bssid not found (%pM)\n",
-					__func__, pmksa->bssid);
-		err = -ENOENT;
-		goto error;
-	}
-
-	for (; i + 1 < count; i++)
-		pmkids->bssid_info[i] = pmkids->bssid_info[i + 1];
-
-	count--;
-	pmkids->length = cpu_to_le32(struct_size(pmkids, bssid_info, count));
-	pmkids->bssid_info_count = cpu_to_le32(count);
-
-	return pmkids;
-error:
-	kfree(pmkids);
-	return ERR_PTR(err);
-}
-
-static struct ndis_80211_pmkid *update_pmkid(struct usbnet *usbdev,
-						struct ndis_80211_pmkid *pmkids,
-						struct cfg80211_pmksa *pmksa,
-						int max_pmkids)
-{
-	struct ndis_80211_pmkid *new_pmkids;
-	int i, err, newlen;
-	unsigned int count;
-
-	count = le32_to_cpu(pmkids->bssid_info_count);
-
-	if (count > max_pmkids)
-		count = max_pmkids;
-
-	/* update with new pmkid */
-	for (i = 0; i < count; i++) {
-		if (!ether_addr_equal(pmkids->bssid_info[i].bssid,
-				      pmksa->bssid))
-			continue;
-
-		memcpy(pmkids->bssid_info[i].pmkid, pmksa->pmkid,
-								WLAN_PMKID_LEN);
-
-		return pmkids;
-	}
-
-	/* out of space, return error */
-	if (i == max_pmkids) {
-		netdev_dbg(usbdev->net, "%s(): out of space\n", __func__);
-		err = -ENOSPC;
-		goto error;
-	}
-
-	/* add new pmkid */
-	newlen = struct_size(pmkids, bssid_info, count + 1);
-
-	new_pmkids = krealloc(pmkids, newlen, GFP_KERNEL);
-	if (!new_pmkids) {
-		err = -ENOMEM;
-		goto error;
-	}
-	pmkids = new_pmkids;
-
-	pmkids->length = cpu_to_le32(newlen);
-	pmkids->bssid_info_count = cpu_to_le32(count + 1);
-
-	memcpy(pmkids->bssid_info[count].bssid, pmksa->bssid, ETH_ALEN);
-	memcpy(pmkids->bssid_info[count].pmkid, pmksa->pmkid, WLAN_PMKID_LEN);
-
-	return pmkids;
-error:
-	kfree(pmkids);
-	return ERR_PTR(err);
-}
-
-/*
- * cfg80211 ops
- */
-static int rndis_change_virtual_intf(struct wiphy *wiphy,
-					struct net_device *dev,
-					enum nl80211_iftype type,
-					struct vif_params *params)
-{
-	struct rndis_wlan_private *priv = wiphy_priv(wiphy);
-	struct usbnet *usbdev = priv->usbdev;
-	int mode;
-
-	switch (type) {
-	case NL80211_IFTYPE_ADHOC:
-		mode = NDIS_80211_INFRA_ADHOC;
-		break;
-	case NL80211_IFTYPE_STATION:
-		mode = NDIS_80211_INFRA_INFRA;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	priv->wdev.iftype = type;
-
-	return set_infra_mode(usbdev, mode);
-}
-
-static int rndis_set_wiphy_params(struct wiphy *wiphy, u32 changed)
-{
-	struct rndis_wlan_private *priv = wiphy_priv(wiphy);
-	struct usbnet *usbdev = priv->usbdev;
-	int err;
-
-	if (changed & WIPHY_PARAM_FRAG_THRESHOLD) {
-		err = set_frag_threshold(usbdev, wiphy->frag_threshold);
-		if (err < 0)
-			return err;
-	}
-
-	if (changed & WIPHY_PARAM_RTS_THRESHOLD) {
-		err = set_rts_threshold(usbdev, wiphy->rts_threshold);
-		if (err < 0)
-			return err;
-	}
-
-	return 0;
-}
-
-static int rndis_set_tx_power(struct wiphy *wiphy,
-			      struct wireless_dev *wdev,
-			      enum nl80211_tx_power_setting type,
-			      int mbm)
-{
-	struct rndis_wlan_private *priv = wiphy_priv(wiphy);
-	struct usbnet *usbdev = priv->usbdev;
-
-	netdev_dbg(usbdev->net, "%s(): type:0x%x mbm:%i\n",
-		   __func__, type, mbm);
-
-	if (mbm < 0 || (mbm % 100))
-		return -ENOTSUPP;
-
-	/* Device doesn't support changing txpower after initialization, only
-	 * turn off/on radio. Support 'auto' mode and setting same dBm that is
-	 * currently used.
-	 */
-	if (type == NL80211_TX_POWER_AUTOMATIC ||
-	    MBM_TO_DBM(mbm) == get_bcm4320_power_dbm(priv)) {
-		if (!priv->radio_on)
-			disassociate(usbdev, true); /* turn on radio */
-
-		return 0;
-	}
-
-	return -ENOTSUPP;
-}
-
-static int rndis_get_tx_power(struct wiphy *wiphy,
-			      struct wireless_dev *wdev,
-			      int *dbm)
-{
-	struct rndis_wlan_private *priv = wiphy_priv(wiphy);
-	struct usbnet *usbdev = priv->usbdev;
-
-	*dbm = get_bcm4320_power_dbm(priv);
-
-	netdev_dbg(usbdev->net, "%s(): dbm:%i\n", __func__, *dbm);
-
-	return 0;
-}
-
-#define SCAN_DELAY_JIFFIES (6 * HZ)
-static int rndis_scan(struct wiphy *wiphy,
-			struct cfg80211_scan_request *request)
-{
-	struct net_device *dev = request->wdev->netdev;
-	struct usbnet *usbdev = netdev_priv(dev);
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	int ret;
-	int delay = SCAN_DELAY_JIFFIES;
-
-	netdev_dbg(usbdev->net, "cfg80211.scan\n");
-
-	/* Get current bssid list from device before new scan, as new scan
-	 * clears internal bssid list.
-	 */
-	rndis_check_bssid_list(usbdev, NULL, NULL);
-
-	if (priv->scan_request && priv->scan_request != request)
-		return -EBUSY;
-
-	priv->scan_request = request;
-
-	ret = rndis_start_bssid_list_scan(usbdev);
-	if (ret == 0) {
-		if (priv->device_type == RNDIS_BCM4320A)
-			delay = HZ;
-
-		/* Wait before retrieving scan results from device */
-		queue_delayed_work(priv->workqueue, &priv->scan_work, delay);
-	}
-
-	return ret;
-}
-
-static bool rndis_bss_info_update(struct usbnet *usbdev,
-				  struct ndis_80211_bssid_ex *bssid)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	struct ieee80211_channel *channel;
-	struct cfg80211_bss *bss;
-	s32 signal;
-	u64 timestamp;
-	u16 capability;
-	u16 beacon_interval;
-	struct ndis_80211_fixed_ies *fixed;
-	int ie_len, bssid_len;
-	u8 *ie;
-
-	netdev_dbg(usbdev->net, " found bssid: '%.32s' [%pM], len: %d\n",
-		   bssid->ssid.essid, bssid->mac, le32_to_cpu(bssid->length));
-
-	/* parse bssid structure */
-	bssid_len = le32_to_cpu(bssid->length);
-
-	if (bssid_len < sizeof(struct ndis_80211_bssid_ex) +
-			sizeof(struct ndis_80211_fixed_ies))
-		return false;
-
-	fixed = (struct ndis_80211_fixed_ies *)bssid->ies;
-
-	ie = (void *)(bssid->ies + sizeof(struct ndis_80211_fixed_ies));
-	ie_len = min(bssid_len - (int)sizeof(*bssid),
-					(int)le32_to_cpu(bssid->ie_length));
-	ie_len -= sizeof(struct ndis_80211_fixed_ies);
-	if (ie_len < 0)
-		return false;
-
-	/* extract data for cfg80211_inform_bss */
-	channel = ieee80211_get_channel(priv->wdev.wiphy,
-			KHZ_TO_MHZ(le32_to_cpu(bssid->config.ds_config)));
-	if (!channel)
-		return false;
-
-	signal = level_to_qual(le32_to_cpu(bssid->rssi));
-	timestamp = le64_to_cpu(*(__le64 *)fixed->timestamp);
-	capability = le16_to_cpu(fixed->capabilities);
-	beacon_interval = le16_to_cpu(fixed->beacon_interval);
-
-	bss = cfg80211_inform_bss(priv->wdev.wiphy, channel,
-				  CFG80211_BSS_FTYPE_UNKNOWN, bssid->mac,
-				  timestamp, capability, beacon_interval,
-				  ie, ie_len, signal, GFP_KERNEL);
-	cfg80211_put_bss(priv->wdev.wiphy, bss);
-
-	return (bss != NULL);
-}
-
-static struct ndis_80211_bssid_ex *next_bssid_list_item(
-					struct ndis_80211_bssid_ex *bssid,
-					int *bssid_len, void *buf, int len)
-{
-	void *buf_end, *bssid_end;
-
-	buf_end = (char *)buf + len;
-	bssid_end = (char *)bssid + *bssid_len;
-
-	if ((int)(buf_end - bssid_end) < sizeof(bssid->length)) {
-		*bssid_len = 0;
-		return NULL;
-	} else {
-		bssid = (void *)((char *)bssid + *bssid_len);
-		*bssid_len = le32_to_cpu(bssid->length);
-		return bssid;
-	}
-}
-
-static bool check_bssid_list_item(struct ndis_80211_bssid_ex *bssid,
-				  int bssid_len, void *buf, int len)
-{
-	void *buf_end, *bssid_end;
-
-	if (!bssid || bssid_len <= 0 || bssid_len > len)
-		return false;
-
-	buf_end = (char *)buf + len;
-	bssid_end = (char *)bssid + bssid_len;
-
-	return (int)(buf_end - bssid_end) >= 0 && (int)(bssid_end - buf) >= 0;
-}
-
-static int rndis_check_bssid_list(struct usbnet *usbdev, u8 *match_bssid,
-					bool *matched)
-{
-	void *buf = NULL;
-	struct ndis_80211_bssid_list_ex *bssid_list;
-	struct ndis_80211_bssid_ex *bssid;
-	int ret = -EINVAL, len, count, bssid_len, real_count, new_len;
-
-	netdev_dbg(usbdev->net, "%s()\n", __func__);
-
-	len = CONTROL_BUFFER_SIZE;
-resize_buf:
-	buf = kzalloc(len, GFP_KERNEL);
-	if (!buf) {
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	/* BSSID-list might have got bigger last time we checked, keep
-	 * resizing until it won't get any bigger.
-	 */
-	new_len = len;
-	ret = rndis_query_oid(usbdev, RNDIS_OID_802_11_BSSID_LIST,
-			      buf, &new_len);
-	if (ret != 0 || new_len < sizeof(struct ndis_80211_bssid_list_ex))
-		goto out;
-
-	if (new_len > len) {
-		len = new_len;
-		kfree(buf);
-		goto resize_buf;
-	}
-
-	len = new_len;
-
-	bssid_list = buf;
-	count = le32_to_cpu(bssid_list->num_items);
-	real_count = 0;
-	netdev_dbg(usbdev->net, "%s(): buflen: %d\n", __func__, len);
-
-	bssid_len = 0;
-	bssid = next_bssid_list_item((void *)bssid_list->bssid_data,
-				     &bssid_len, buf, len);
-
-	/* Device returns incorrect 'num_items'. Workaround by ignoring the
-	 * received 'num_items' and walking through full bssid buffer instead.
-	 */
-	while (check_bssid_list_item(bssid, bssid_len, buf, len)) {
-		if (rndis_bss_info_update(usbdev, bssid) && match_bssid &&
-		    matched) {
-			if (ether_addr_equal(bssid->mac, match_bssid))
-				*matched = true;
-		}
-
-		real_count++;
-		bssid = next_bssid_list_item(bssid, &bssid_len, buf, len);
-	}
-
-	netdev_dbg(usbdev->net, "%s(): num_items from device: %d, really found:"
-				" %d\n", __func__, count, real_count);
-
-out:
-	kfree(buf);
-	return ret;
-}
-
-static void rndis_get_scan_results(struct work_struct *work)
-{
-	struct rndis_wlan_private *priv =
-		container_of(work, struct rndis_wlan_private, scan_work.work);
-	struct usbnet *usbdev = priv->usbdev;
-	struct cfg80211_scan_info info = {};
-	int ret;
-
-	netdev_dbg(usbdev->net, "get_scan_results\n");
-
-	if (!priv->scan_request)
-		return;
-
-	ret = rndis_check_bssid_list(usbdev, NULL, NULL);
-
-	info.aborted = ret < 0;
-	cfg80211_scan_done(priv->scan_request, &info);
-
-	priv->scan_request = NULL;
-}
-
-static int rndis_connect(struct wiphy *wiphy, struct net_device *dev,
-					struct cfg80211_connect_params *sme)
-{
-	struct rndis_wlan_private *priv = wiphy_priv(wiphy);
-	struct usbnet *usbdev = priv->usbdev;
-	struct ieee80211_channel *channel = sme->channel;
-	struct ndis_80211_ssid ssid;
-	int pairwise = RNDIS_WLAN_ALG_NONE;
-	int groupwise = RNDIS_WLAN_ALG_NONE;
-	int keymgmt = RNDIS_WLAN_KEY_MGMT_NONE;
-	int length, i, ret, chan = -1;
-
-	if (channel)
-		chan = ieee80211_frequency_to_channel(channel->center_freq);
-
-	groupwise = rndis_cipher_to_alg(sme->crypto.cipher_group);
-	for (i = 0; i < sme->crypto.n_ciphers_pairwise; i++)
-		pairwise |=
-			rndis_cipher_to_alg(sme->crypto.ciphers_pairwise[i]);
-
-	if (sme->crypto.n_ciphers_pairwise > 0 &&
-			pairwise == RNDIS_WLAN_ALG_NONE) {
-		netdev_err(usbdev->net, "Unsupported pairwise cipher\n");
-		return -ENOTSUPP;
-	}
-
-	for (i = 0; i < sme->crypto.n_akm_suites; i++)
-		keymgmt |=
-			rndis_akm_suite_to_key_mgmt(sme->crypto.akm_suites[i]);
-
-	if (sme->crypto.n_akm_suites > 0 &&
-			keymgmt == RNDIS_WLAN_KEY_MGMT_NONE) {
-		netdev_err(usbdev->net, "Invalid keymgmt\n");
-		return -ENOTSUPP;
-	}
-
-	netdev_dbg(usbdev->net, "cfg80211.connect('%.32s':[%pM]:%d:[%d,0x%x:0x%x]:[0x%x:0x%x]:0x%x)\n",
-		   sme->ssid, sme->bssid, chan,
-		   sme->privacy, sme->crypto.wpa_versions, sme->auth_type,
-		   groupwise, pairwise, keymgmt);
-
-	if (is_associated(usbdev))
-		disassociate(usbdev, false);
-
-	ret = set_infra_mode(usbdev, NDIS_80211_INFRA_INFRA);
-	if (ret < 0) {
-		netdev_dbg(usbdev->net, "connect: set_infra_mode failed, %d\n",
-			   ret);
-		goto err_turn_radio_on;
-	}
-
-	ret = set_auth_mode(usbdev, sme->crypto.wpa_versions, sme->auth_type,
-								keymgmt);
-	if (ret < 0) {
-		netdev_dbg(usbdev->net, "connect: set_auth_mode failed, %d\n",
-			   ret);
-		goto err_turn_radio_on;
-	}
-
-	set_priv_filter(usbdev);
-
-	ret = set_encr_mode(usbdev, pairwise, groupwise);
-	if (ret < 0) {
-		netdev_dbg(usbdev->net, "connect: set_encr_mode failed, %d\n",
-			   ret);
-		goto err_turn_radio_on;
-	}
-
-	if (channel) {
-		ret = set_channel(usbdev, chan);
-		if (ret < 0) {
-			netdev_dbg(usbdev->net, "connect: set_channel failed, %d\n",
-				   ret);
-			goto err_turn_radio_on;
-		}
-	}
-
-	if (sme->key && ((groupwise | pairwise) & RNDIS_WLAN_ALG_WEP)) {
-		priv->encr_tx_key_index = sme->key_idx;
-		ret = add_wep_key(usbdev, sme->key, sme->key_len, sme->key_idx);
-		if (ret < 0) {
-			netdev_dbg(usbdev->net, "connect: add_wep_key failed, %d (%d, %d)\n",
-				   ret, sme->key_len, sme->key_idx);
-			goto err_turn_radio_on;
-		}
-	}
-
-	if (sme->bssid && !is_zero_ether_addr(sme->bssid) &&
-				!is_broadcast_ether_addr(sme->bssid)) {
-		ret = set_bssid(usbdev, sme->bssid);
-		if (ret < 0) {
-			netdev_dbg(usbdev->net, "connect: set_bssid failed, %d\n",
-				   ret);
-			goto err_turn_radio_on;
-		}
-	} else
-		clear_bssid(usbdev);
-
-	length = sme->ssid_len;
-	if (length > NDIS_802_11_LENGTH_SSID)
-		length = NDIS_802_11_LENGTH_SSID;
-
-	memset(&ssid, 0, sizeof(ssid));
-	ssid.length = cpu_to_le32(length);
-	memcpy(ssid.essid, sme->ssid, length);
-
-	/* Pause and purge rx queue, so we don't pass packets before
-	 * 'media connect'-indication.
-	 */
-	usbnet_pause_rx(usbdev);
-	usbnet_purge_paused_rxq(usbdev);
-
-	ret = set_essid(usbdev, &ssid);
-	if (ret < 0)
-		netdev_dbg(usbdev->net, "connect: set_essid failed, %d\n", ret);
-	return ret;
-
-err_turn_radio_on:
-	disassociate(usbdev, true);
-
-	return ret;
-}
-
-static int rndis_disconnect(struct wiphy *wiphy, struct net_device *dev,
-								u16 reason_code)
-{
-	struct rndis_wlan_private *priv = wiphy_priv(wiphy);
-	struct usbnet *usbdev = priv->usbdev;
-
-	netdev_dbg(usbdev->net, "cfg80211.disconnect(%d)\n", reason_code);
-
-	priv->connected = false;
-	eth_zero_addr(priv->bssid);
-
-	return deauthenticate(usbdev);
-}
-
-static int rndis_join_ibss(struct wiphy *wiphy, struct net_device *dev,
-					struct cfg80211_ibss_params *params)
-{
-	struct rndis_wlan_private *priv = wiphy_priv(wiphy);
-	struct usbnet *usbdev = priv->usbdev;
-	struct ieee80211_channel *channel = params->chandef.chan;
-	struct ndis_80211_ssid ssid;
-	enum nl80211_auth_type auth_type;
-	int ret, alg, length, chan = -1;
-
-	if (channel)
-		chan = ieee80211_frequency_to_channel(channel->center_freq);
-
-	/* TODO: How to handle ad-hoc encryption?
-	 * connect() has *key, join_ibss() doesn't. RNDIS requires key to be
-	 * pre-shared for encryption (open/shared/wpa), is key set before
-	 * join_ibss? Which auth_type to use (not in params)? What about WPA?
-	 */
-	if (params->privacy) {
-		auth_type = NL80211_AUTHTYPE_SHARED_KEY;
-		alg = RNDIS_WLAN_ALG_WEP;
-	} else {
-		auth_type = NL80211_AUTHTYPE_OPEN_SYSTEM;
-		alg = RNDIS_WLAN_ALG_NONE;
-	}
-
-	netdev_dbg(usbdev->net, "cfg80211.join_ibss('%.32s':[%pM]:%d:%d)\n",
-		   params->ssid, params->bssid, chan, params->privacy);
-
-	if (is_associated(usbdev))
-		disassociate(usbdev, false);
-
-	ret = set_infra_mode(usbdev, NDIS_80211_INFRA_ADHOC);
-	if (ret < 0) {
-		netdev_dbg(usbdev->net, "join_ibss: set_infra_mode failed, %d\n",
-			   ret);
-		goto err_turn_radio_on;
-	}
-
-	ret = set_auth_mode(usbdev, 0, auth_type, RNDIS_WLAN_KEY_MGMT_NONE);
-	if (ret < 0) {
-		netdev_dbg(usbdev->net, "join_ibss: set_auth_mode failed, %d\n",
-			   ret);
-		goto err_turn_radio_on;
-	}
-
-	set_priv_filter(usbdev);
-
-	ret = set_encr_mode(usbdev, alg, RNDIS_WLAN_ALG_NONE);
-	if (ret < 0) {
-		netdev_dbg(usbdev->net, "join_ibss: set_encr_mode failed, %d\n",
-			   ret);
-		goto err_turn_radio_on;
-	}
-
-	if (channel) {
-		ret = set_channel(usbdev, chan);
-		if (ret < 0) {
-			netdev_dbg(usbdev->net, "join_ibss: set_channel failed, %d\n",
-				   ret);
-			goto err_turn_radio_on;
-		}
-	}
-
-	if (params->bssid && !is_zero_ether_addr(params->bssid) &&
-				!is_broadcast_ether_addr(params->bssid)) {
-		ret = set_bssid(usbdev, params->bssid);
-		if (ret < 0) {
-			netdev_dbg(usbdev->net, "join_ibss: set_bssid failed, %d\n",
-				   ret);
-			goto err_turn_radio_on;
-		}
-	} else
-		clear_bssid(usbdev);
-
-	length = params->ssid_len;
-	if (length > NDIS_802_11_LENGTH_SSID)
-		length = NDIS_802_11_LENGTH_SSID;
-
-	memset(&ssid, 0, sizeof(ssid));
-	ssid.length = cpu_to_le32(length);
-	memcpy(ssid.essid, params->ssid, length);
-
-	/* Don't need to pause rx queue for ad-hoc. */
-	usbnet_purge_paused_rxq(usbdev);
-	usbnet_resume_rx(usbdev);
-
-	ret = set_essid(usbdev, &ssid);
-	if (ret < 0)
-		netdev_dbg(usbdev->net, "join_ibss: set_essid failed, %d\n",
-			   ret);
-	return ret;
-
-err_turn_radio_on:
-	disassociate(usbdev, true);
-
-	return ret;
-}
-
-static int rndis_leave_ibss(struct wiphy *wiphy, struct net_device *dev)
-{
-	struct rndis_wlan_private *priv = wiphy_priv(wiphy);
-	struct usbnet *usbdev = priv->usbdev;
-
-	netdev_dbg(usbdev->net, "cfg80211.leave_ibss()\n");
-
-	priv->connected = false;
-	eth_zero_addr(priv->bssid);
-
-	return deauthenticate(usbdev);
-}
-
-static int rndis_add_key(struct wiphy *wiphy, struct net_device *netdev,
-			 int link_id,  u8 key_index, bool pairwise,
-			 const u8 *mac_addr, struct key_params *params)
-{
-	struct rndis_wlan_private *priv = wiphy_priv(wiphy);
-	struct usbnet *usbdev = priv->usbdev;
-	__le32 flags;
-
-	netdev_dbg(usbdev->net, "%s(%i, %pM, %08x)\n",
-		   __func__, key_index, mac_addr, params->cipher);
-
-	switch (params->cipher) {
-	case WLAN_CIPHER_SUITE_WEP40:
-	case WLAN_CIPHER_SUITE_WEP104:
-		return add_wep_key(usbdev, params->key, params->key_len,
-								key_index);
-	case WLAN_CIPHER_SUITE_TKIP:
-	case WLAN_CIPHER_SUITE_CCMP:
-		flags = 0;
-
-		if (params->seq && params->seq_len > 0)
-			flags |= NDIS_80211_ADDKEY_SET_INIT_RECV_SEQ;
-		if (mac_addr)
-			flags |= NDIS_80211_ADDKEY_PAIRWISE_KEY |
-					NDIS_80211_ADDKEY_TRANSMIT_KEY;
-
-		return add_wpa_key(usbdev, params->key, params->key_len,
-				key_index, mac_addr, params->seq,
-				params->seq_len, params->cipher, flags);
-	default:
-		netdev_dbg(usbdev->net, "%s(): unsupported cipher %08x\n",
-			   __func__, params->cipher);
-		return -ENOTSUPP;
-	}
-}
-
-static int rndis_del_key(struct wiphy *wiphy, struct net_device *netdev,
-			 int link_id, u8 key_index, bool pairwise,
-			 const u8 *mac_addr)
-{
-	struct rndis_wlan_private *priv = wiphy_priv(wiphy);
-	struct usbnet *usbdev = priv->usbdev;
-
-	netdev_dbg(usbdev->net, "%s(%i, %pM)\n", __func__, key_index, mac_addr);
-
-	return remove_key(usbdev, key_index, mac_addr);
-}
-
-static int rndis_set_default_key(struct wiphy *wiphy, struct net_device *netdev,
-				 int link_id, u8 key_index, bool unicast,
-				 bool multicast)
-{
-	struct rndis_wlan_private *priv = wiphy_priv(wiphy);
-	struct usbnet *usbdev = priv->usbdev;
-	struct rndis_wlan_encr_key key;
-
-	netdev_dbg(usbdev->net, "%s(%i)\n", __func__, key_index);
-
-	if (key_index >= RNDIS_WLAN_NUM_KEYS)
-		return -ENOENT;
-
-	priv->encr_tx_key_index = key_index;
-
-	if (is_wpa_key(priv, key_index))
-		return 0;
-
-	key = priv->encr_keys[key_index];
-
-	return add_wep_key(usbdev, key.material, key.len, key_index);
-}
-
-static void rndis_fill_station_info(struct usbnet *usbdev,
-						struct station_info *sinfo)
-{
-	__le32 linkspeed, rssi;
-	int ret, len;
-
-	memset(sinfo, 0, sizeof(*sinfo));
-
-	len = sizeof(linkspeed);
-	ret = rndis_query_oid(usbdev, RNDIS_OID_GEN_LINK_SPEED, &linkspeed, &len);
-	if (ret == 0) {
-		sinfo->txrate.legacy = le32_to_cpu(linkspeed) / 1000;
-		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE);
-	}
-
-	len = sizeof(rssi);
-	ret = rndis_query_oid(usbdev, RNDIS_OID_802_11_RSSI,
-			      &rssi, &len);
-	if (ret == 0) {
-		sinfo->signal = level_to_qual(le32_to_cpu(rssi));
-		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL);
-	}
-}
-
-static int rndis_get_station(struct wiphy *wiphy, struct net_device *dev,
-			     const u8 *mac, struct station_info *sinfo)
-{
-	struct rndis_wlan_private *priv = wiphy_priv(wiphy);
-	struct usbnet *usbdev = priv->usbdev;
-
-	if (!ether_addr_equal(priv->bssid, mac))
-		return -ENOENT;
-
-	rndis_fill_station_info(usbdev, sinfo);
-
-	return 0;
-}
-
-static int rndis_dump_station(struct wiphy *wiphy, struct net_device *dev,
-			       int idx, u8 *mac, struct station_info *sinfo)
-{
-	struct rndis_wlan_private *priv = wiphy_priv(wiphy);
-	struct usbnet *usbdev = priv->usbdev;
-
-	if (idx != 0)
-		return -ENOENT;
-
-	memcpy(mac, priv->bssid, ETH_ALEN);
-
-	rndis_fill_station_info(usbdev, sinfo);
-
-	return 0;
-}
-
-static int rndis_set_pmksa(struct wiphy *wiphy, struct net_device *netdev,
-				struct cfg80211_pmksa *pmksa)
-{
-	struct rndis_wlan_private *priv = wiphy_priv(wiphy);
-	struct usbnet *usbdev = priv->usbdev;
-	struct ndis_80211_pmkid *pmkids;
-	u32 *tmp = (u32 *)pmksa->pmkid;
-
-	netdev_dbg(usbdev->net, "%s(%pM, %08X:%08X:%08X:%08X)\n", __func__,
-			pmksa->bssid,
-			cpu_to_be32(tmp[0]), cpu_to_be32(tmp[1]),
-			cpu_to_be32(tmp[2]), cpu_to_be32(tmp[3]));
-
-	pmkids = get_device_pmkids(usbdev);
-	if (IS_ERR(pmkids)) {
-		/* couldn't read PMKID cache from device */
-		return PTR_ERR(pmkids);
-	}
-
-	pmkids = update_pmkid(usbdev, pmkids, pmksa, wiphy->max_num_pmkids);
-	if (IS_ERR(pmkids)) {
-		/* not found, list full, etc */
-		return PTR_ERR(pmkids);
-	}
-
-	return set_device_pmkids(usbdev, pmkids);
-}
-
-static int rndis_del_pmksa(struct wiphy *wiphy, struct net_device *netdev,
-				struct cfg80211_pmksa *pmksa)
-{
-	struct rndis_wlan_private *priv = wiphy_priv(wiphy);
-	struct usbnet *usbdev = priv->usbdev;
-	struct ndis_80211_pmkid *pmkids;
-	u32 *tmp = (u32 *)pmksa->pmkid;
-
-	netdev_dbg(usbdev->net, "%s(%pM, %08X:%08X:%08X:%08X)\n", __func__,
-			pmksa->bssid,
-			cpu_to_be32(tmp[0]), cpu_to_be32(tmp[1]),
-			cpu_to_be32(tmp[2]), cpu_to_be32(tmp[3]));
-
-	pmkids = get_device_pmkids(usbdev);
-	if (IS_ERR(pmkids)) {
-		/* Couldn't read PMKID cache from device */
-		return PTR_ERR(pmkids);
-	}
-
-	pmkids = remove_pmkid(usbdev, pmkids, pmksa, wiphy->max_num_pmkids);
-	if (IS_ERR(pmkids)) {
-		/* not found, etc */
-		return PTR_ERR(pmkids);
-	}
-
-	return set_device_pmkids(usbdev, pmkids);
-}
-
-static int rndis_flush_pmksa(struct wiphy *wiphy, struct net_device *netdev)
-{
-	struct rndis_wlan_private *priv = wiphy_priv(wiphy);
-	struct usbnet *usbdev = priv->usbdev;
-	struct ndis_80211_pmkid pmkid;
-
-	netdev_dbg(usbdev->net, "%s()\n", __func__);
-
-	memset(&pmkid, 0, sizeof(pmkid));
-
-	pmkid.length = cpu_to_le32(sizeof(pmkid));
-	pmkid.bssid_info_count = cpu_to_le32(0);
-
-	return rndis_set_oid(usbdev, RNDIS_OID_802_11_PMKID,
-			     &pmkid, sizeof(pmkid));
-}
-
-static int rndis_set_power_mgmt(struct wiphy *wiphy, struct net_device *dev,
-				bool enabled, int timeout)
-{
-	struct rndis_wlan_private *priv = wiphy_priv(wiphy);
-	struct usbnet *usbdev = priv->usbdev;
-	int power_mode;
-	__le32 mode;
-	int ret;
-
-	if (priv->device_type != RNDIS_BCM4320B)
-		return -ENOTSUPP;
-
-	netdev_dbg(usbdev->net, "%s(): %s, %d\n", __func__,
-				enabled ? "enabled" : "disabled",
-				timeout);
-
-	if (enabled)
-		power_mode = NDIS_80211_POWER_MODE_FAST_PSP;
-	else
-		power_mode = NDIS_80211_POWER_MODE_CAM;
-
-	if (power_mode == priv->power_mode)
-		return 0;
-
-	priv->power_mode = power_mode;
-
-	mode = cpu_to_le32(power_mode);
-	ret = rndis_set_oid(usbdev, RNDIS_OID_802_11_POWER_MODE,
-			    &mode, sizeof(mode));
-
-	netdev_dbg(usbdev->net, "%s(): RNDIS_OID_802_11_POWER_MODE -> %d\n",
-				__func__, ret);
-
-	return ret;
-}
-
-static int rndis_set_cqm_rssi_config(struct wiphy *wiphy,
-					struct net_device *dev,
-					s32 rssi_thold, u32 rssi_hyst)
-{
-	struct rndis_wlan_private *priv = wiphy_priv(wiphy);
-
-	priv->cqm_rssi_thold = rssi_thold;
-	priv->cqm_rssi_hyst = rssi_hyst;
-	priv->last_cqm_event_rssi = 0;
-
-	return 0;
-}
-
-static void rndis_wlan_craft_connected_bss(struct usbnet *usbdev, u8 *bssid,
-					   struct ndis_80211_assoc_info *info)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	struct ieee80211_channel *channel;
-	struct ndis_80211_ssid ssid;
-	struct cfg80211_bss *bss;
-	s32 signal;
-	u64 timestamp;
-	u16 capability;
-	u32 beacon_period = 0;
-	__le32 rssi;
-	u8 ie_buf[34];
-	int len, ret, ie_len;
-
-	/* Get signal quality, in case of error use rssi=0 and ignore error. */
-	len = sizeof(rssi);
-	rssi = 0;
-	ret = rndis_query_oid(usbdev, RNDIS_OID_802_11_RSSI,
-			      &rssi, &len);
-	signal = level_to_qual(le32_to_cpu(rssi));
-
-	netdev_dbg(usbdev->net, "%s(): RNDIS_OID_802_11_RSSI -> %d, "
-		   "rssi:%d, qual: %d\n", __func__, ret, le32_to_cpu(rssi),
-		   level_to_qual(le32_to_cpu(rssi)));
-
-	/* Get AP capabilities */
-	if (info) {
-		capability = le16_to_cpu(info->resp_ie.capa);
-	} else {
-		/* Set atleast ESS/IBSS capability */
-		capability = (priv->infra_mode == NDIS_80211_INFRA_INFRA) ?
-				WLAN_CAPABILITY_ESS : WLAN_CAPABILITY_IBSS;
-	}
-
-	/* Get channel and beacon interval */
-	channel = get_current_channel(usbdev, &beacon_period);
-	if (!channel) {
-		netdev_warn(usbdev->net, "%s(): could not get channel.\n",
-					__func__);
-		return;
-	}
-
-	/* Get SSID, in case of error, use zero length SSID and ignore error. */
-	len = sizeof(ssid);
-	memset(&ssid, 0, sizeof(ssid));
-	ret = rndis_query_oid(usbdev, RNDIS_OID_802_11_SSID,
-			      &ssid, &len);
-	netdev_dbg(usbdev->net, "%s(): RNDIS_OID_802_11_SSID -> %d, len: %d, ssid: "
-				"'%.32s'\n", __func__, ret,
-				le32_to_cpu(ssid.length), ssid.essid);
-
-	if (le32_to_cpu(ssid.length) > 32)
-		ssid.length = cpu_to_le32(32);
-
-	ie_buf[0] = WLAN_EID_SSID;
-	ie_buf[1] = le32_to_cpu(ssid.length);
-	memcpy(&ie_buf[2], ssid.essid, le32_to_cpu(ssid.length));
-
-	ie_len = le32_to_cpu(ssid.length) + 2;
-
-	/* no tsf */
-	timestamp = 0;
-
-	netdev_dbg(usbdev->net, "%s(): channel:%d(freq), bssid:[%pM], tsf:%d, "
-		"capa:%x, beacon int:%d, resp_ie(len:%d, essid:'%.32s'), "
-		"signal:%d\n", __func__, (channel ? channel->center_freq : -1),
-		bssid, (u32)timestamp, capability, beacon_period, ie_len,
-		ssid.essid, signal);
-
-	bss = cfg80211_inform_bss(priv->wdev.wiphy, channel,
-				  CFG80211_BSS_FTYPE_UNKNOWN, bssid,
-				  timestamp, capability, beacon_period,
-				  ie_buf, ie_len, signal, GFP_KERNEL);
-	cfg80211_put_bss(priv->wdev.wiphy, bss);
-}
-
-/*
- * workers, indication handlers, device poller
- */
-static void rndis_wlan_do_link_up_work(struct usbnet *usbdev)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	struct ndis_80211_assoc_info *info = NULL;
-	u8 bssid[ETH_ALEN];
-	unsigned int resp_ie_len, req_ie_len;
-	unsigned int offset;
-	u8 *req_ie, *resp_ie;
-	int ret;
-	bool roamed = false;
-	bool match_bss;
-
-	if (priv->infra_mode == NDIS_80211_INFRA_INFRA && priv->connected) {
-		/* received media connect indication while connected, either
-		 * device reassociated with same AP or roamed to new. */
-		roamed = true;
-	}
-
-	req_ie_len = 0;
-	resp_ie_len = 0;
-	req_ie = NULL;
-	resp_ie = NULL;
-
-	if (priv->infra_mode == NDIS_80211_INFRA_INFRA) {
-		info = kzalloc(CONTROL_BUFFER_SIZE, GFP_KERNEL);
-		if (!info) {
-			/* No memory? Try resume work later */
-			set_bit(WORK_LINK_UP, &priv->work_pending);
-			queue_work(priv->workqueue, &priv->work);
-			return;
-		}
-
-		/* Get association info IEs from device. */
-		ret = get_association_info(usbdev, info, CONTROL_BUFFER_SIZE);
-		if (!ret) {
-			req_ie_len = le32_to_cpu(info->req_ie_length);
-			if (req_ie_len > CONTROL_BUFFER_SIZE)
-				req_ie_len = CONTROL_BUFFER_SIZE;
-			if (req_ie_len != 0) {
-				offset = le32_to_cpu(info->offset_req_ies);
-
-				if (offset > CONTROL_BUFFER_SIZE)
-					offset = CONTROL_BUFFER_SIZE;
-
-				req_ie = (u8 *)info + offset;
-
-				if (offset + req_ie_len > CONTROL_BUFFER_SIZE)
-					req_ie_len =
-						CONTROL_BUFFER_SIZE - offset;
-			}
-
-			resp_ie_len = le32_to_cpu(info->resp_ie_length);
-			if (resp_ie_len > CONTROL_BUFFER_SIZE)
-				resp_ie_len = CONTROL_BUFFER_SIZE;
-			if (resp_ie_len != 0) {
-				offset = le32_to_cpu(info->offset_resp_ies);
-
-				if (offset > CONTROL_BUFFER_SIZE)
-					offset = CONTROL_BUFFER_SIZE;
-
-				resp_ie = (u8 *)info + offset;
-
-				if (offset + resp_ie_len > CONTROL_BUFFER_SIZE)
-					resp_ie_len =
-						CONTROL_BUFFER_SIZE - offset;
-			}
-		} else {
-			/* Since rndis_wlan_craft_connected_bss() might use info
-			 * later and expects info to contain valid data if
-			 * non-null, free info and set NULL here.
-			 */
-			kfree(info);
-			info = NULL;
-		}
-	} else if (WARN_ON(priv->infra_mode != NDIS_80211_INFRA_ADHOC))
-		return;
-
-	ret = get_bssid(usbdev, bssid);
-	if (ret < 0)
-		memset(bssid, 0, sizeof(bssid));
-
-	netdev_dbg(usbdev->net, "link up work: [%pM]%s\n",
-		   bssid, roamed ? " roamed" : "");
-
-	/* Internal bss list in device should contain at least the currently
-	 * connected bss and we can get it to cfg80211 with
-	 * rndis_check_bssid_list().
-	 *
-	 * NDIS spec says: "If the device is associated, but the associated
-	 *  BSSID is not in its BSSID scan list, then the driver must add an
-	 *  entry for the BSSID at the end of the data that it returns in
-	 *  response to query of RNDIS_OID_802_11_BSSID_LIST."
-	 *
-	 * NOTE: Seems to be true for BCM4320b variant, but not BCM4320a.
-	 */
-	match_bss = false;
-	rndis_check_bssid_list(usbdev, bssid, &match_bss);
-
-	if (!is_zero_ether_addr(bssid) && !match_bss) {
-		/* Couldn't get bss from device, we need to manually craft bss
-		 * for cfg80211.
-		 */
-		rndis_wlan_craft_connected_bss(usbdev, bssid, info);
-	}
-
-	if (priv->infra_mode == NDIS_80211_INFRA_INFRA) {
-		if (!roamed) {
-			cfg80211_connect_result(usbdev->net, bssid, req_ie,
-						req_ie_len, resp_ie,
-						resp_ie_len, 0, GFP_KERNEL);
-		} else {
-			struct cfg80211_roam_info roam_info = {
-				.links[0].channel =
-					get_current_channel(usbdev, NULL),
-				.links[0].bssid = bssid,
-				.req_ie = req_ie,
-				.req_ie_len = req_ie_len,
-				.resp_ie = resp_ie,
-				.resp_ie_len = resp_ie_len,
-			};
-
-			cfg80211_roamed(usbdev->net, &roam_info, GFP_KERNEL);
-		}
-	} else if (priv->infra_mode == NDIS_80211_INFRA_ADHOC)
-		cfg80211_ibss_joined(usbdev->net, bssid,
-				     get_current_channel(usbdev, NULL),
-				     GFP_KERNEL);
-
-	kfree(info);
-
-	priv->connected = true;
-	memcpy(priv->bssid, bssid, ETH_ALEN);
-
-	usbnet_resume_rx(usbdev);
-	netif_carrier_on(usbdev->net);
-}
-
-static void rndis_wlan_do_link_down_work(struct usbnet *usbdev)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-
-	if (priv->connected) {
-		priv->connected = false;
-		eth_zero_addr(priv->bssid);
-
-		deauthenticate(usbdev);
-
-		cfg80211_disconnected(usbdev->net, 0, NULL, 0, true, GFP_KERNEL);
-	}
-
-	netif_carrier_off(usbdev->net);
-}
-
-static void rndis_wlan_worker(struct work_struct *work)
-{
-	struct rndis_wlan_private *priv =
-		container_of(work, struct rndis_wlan_private, work);
-	struct usbnet *usbdev = priv->usbdev;
-
-	if (test_and_clear_bit(WORK_LINK_UP, &priv->work_pending))
-		rndis_wlan_do_link_up_work(usbdev);
-
-	if (test_and_clear_bit(WORK_LINK_DOWN, &priv->work_pending))
-		rndis_wlan_do_link_down_work(usbdev);
-
-	if (test_and_clear_bit(WORK_SET_MULTICAST_LIST, &priv->work_pending))
-		set_multicast_list(usbdev);
-}
-
-static void rndis_wlan_set_multicast_list(struct net_device *dev)
-{
-	struct usbnet *usbdev = netdev_priv(dev);
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-
-	if (test_bit(WORK_SET_MULTICAST_LIST, &priv->work_pending))
-		return;
-
-	set_bit(WORK_SET_MULTICAST_LIST, &priv->work_pending);
-	queue_work(priv->workqueue, &priv->work);
-}
-
-static void rndis_wlan_auth_indication(struct usbnet *usbdev,
-				struct ndis_80211_status_indication *indication,
-				int len)
-{
-	u8 *buf;
-	const char *type;
-	int flags, buflen, key_id;
-	bool pairwise_error, group_error;
-	struct ndis_80211_auth_request *auth_req;
-	enum nl80211_key_type key_type;
-
-	/* must have at least one array entry */
-	if (len < offsetof(struct ndis_80211_status_indication, u) +
-				sizeof(struct ndis_80211_auth_request)) {
-		netdev_info(usbdev->net, "authentication indication: too short message (%i)\n",
-			    len);
-		return;
-	}
-
-	buf = (void *)&indication->u.auth_request[0];
-	buflen = len - offsetof(struct ndis_80211_status_indication, u);
-
-	while (buflen >= sizeof(*auth_req)) {
-		auth_req = (void *)buf;
-		if (buflen < le32_to_cpu(auth_req->length))
-			return;
-		type = "unknown";
-		flags = le32_to_cpu(auth_req->flags);
-		pairwise_error = false;
-		group_error = false;
-
-		if (flags & 0x1)
-			type = "reauth request";
-		if (flags & 0x2)
-			type = "key update request";
-		if (flags & 0x6) {
-			pairwise_error = true;
-			type = "pairwise_error";
-		}
-		if (flags & 0xe) {
-			group_error = true;
-			type = "group_error";
-		}
-
-		netdev_info(usbdev->net, "authentication indication: %s (0x%08x)\n",
-			    type, le32_to_cpu(auth_req->flags));
-
-		if (pairwise_error) {
-			key_type = NL80211_KEYTYPE_PAIRWISE;
-			key_id = -1;
-
-			cfg80211_michael_mic_failure(usbdev->net,
-							auth_req->bssid,
-							key_type, key_id, NULL,
-							GFP_KERNEL);
-		}
-
-		if (group_error) {
-			key_type = NL80211_KEYTYPE_GROUP;
-			key_id = -1;
-
-			cfg80211_michael_mic_failure(usbdev->net,
-							auth_req->bssid,
-							key_type, key_id, NULL,
-							GFP_KERNEL);
-		}
-
-		buflen -= le32_to_cpu(auth_req->length);
-		buf += le32_to_cpu(auth_req->length);
-	}
-}
-
-static void rndis_wlan_pmkid_cand_list_indication(struct usbnet *usbdev,
-				struct ndis_80211_status_indication *indication,
-				int len)
-{
-	struct ndis_80211_pmkid_cand_list *cand_list;
-	int list_len, expected_len, i;
-
-	if (len < offsetof(struct ndis_80211_status_indication, u) +
-				sizeof(struct ndis_80211_pmkid_cand_list)) {
-		netdev_info(usbdev->net, "pmkid candidate list indication: too short message (%i)\n",
-			    len);
-		return;
-	}
-
-	list_len = le32_to_cpu(indication->u.cand_list.num_candidates) *
-			sizeof(struct ndis_80211_pmkid_candidate);
-	expected_len = sizeof(struct ndis_80211_pmkid_cand_list) + list_len +
-			offsetof(struct ndis_80211_status_indication, u);
-
-	if (len < expected_len) {
-		netdev_info(usbdev->net, "pmkid candidate list indication: list larger than buffer (%i < %i)\n",
-			    len, expected_len);
-		return;
-	}
-
-	cand_list = &indication->u.cand_list;
-
-	netdev_info(usbdev->net, "pmkid candidate list indication: version %i, candidates %i\n",
-		    le32_to_cpu(cand_list->version),
-		    le32_to_cpu(cand_list->num_candidates));
-
-	if (le32_to_cpu(cand_list->version) != 1)
-		return;
-
-	for (i = 0; i < le32_to_cpu(cand_list->num_candidates); i++) {
-		struct ndis_80211_pmkid_candidate *cand =
-						&cand_list->candidate_list[i];
-		bool preauth = !!(cand->flags & NDIS_80211_PMKID_CAND_PREAUTH);
-
-		netdev_dbg(usbdev->net, "cand[%i]: flags: 0x%08x, preauth: %d, bssid: %pM\n",
-			   i, le32_to_cpu(cand->flags), preauth, cand->bssid);
-
-		cfg80211_pmksa_candidate_notify(usbdev->net, i, cand->bssid,
-						preauth, GFP_ATOMIC);
-	}
-}
-
-static void rndis_wlan_media_specific_indication(struct usbnet *usbdev,
-			struct rndis_indicate *msg, int buflen)
-{
-	struct ndis_80211_status_indication *indication;
-	unsigned int len, offset;
-
-	offset = offsetof(struct rndis_indicate, status) +
-			le32_to_cpu(msg->offset);
-	len = le32_to_cpu(msg->length);
-
-	if (len < 8) {
-		netdev_info(usbdev->net, "media specific indication, ignore too short message (%i < 8)\n",
-			    len);
-		return;
-	}
-
-	if (len > buflen || offset > buflen || offset + len > buflen) {
-		netdev_info(usbdev->net, "media specific indication, too large to fit to buffer (%i > %i)\n",
-			    offset + len, buflen);
-		return;
-	}
-
-	indication = (void *)((u8 *)msg + offset);
-
-	switch (le32_to_cpu(indication->status_type)) {
-	case NDIS_80211_STATUSTYPE_RADIOSTATE:
-		netdev_info(usbdev->net, "radio state indication: %i\n",
-			    le32_to_cpu(indication->u.radio_status));
-		return;
-
-	case NDIS_80211_STATUSTYPE_MEDIASTREAMMODE:
-		netdev_info(usbdev->net, "media stream mode indication: %i\n",
-			    le32_to_cpu(indication->u.media_stream_mode));
-		return;
-
-	case NDIS_80211_STATUSTYPE_AUTHENTICATION:
-		rndis_wlan_auth_indication(usbdev, indication, len);
-		return;
-
-	case NDIS_80211_STATUSTYPE_PMKID_CANDIDATELIST:
-		rndis_wlan_pmkid_cand_list_indication(usbdev, indication, len);
-		return;
-
-	default:
-		netdev_info(usbdev->net, "media specific indication: unknown status type 0x%08x\n",
-			    le32_to_cpu(indication->status_type));
-	}
-}
-
-static void rndis_wlan_indication(struct usbnet *usbdev, void *ind, int buflen)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	struct rndis_indicate *msg = ind;
-
-	switch (le32_to_cpu(msg->status)) {
-	case RNDIS_STATUS_MEDIA_CONNECT:
-		if (priv->current_command_oid == RNDIS_OID_802_11_ADD_KEY) {
-			/* RNDIS_OID_802_11_ADD_KEY causes sometimes extra
-			 * "media connect" indications which confuses driver
-			 * and userspace to think that device is
-			 * roaming/reassociating when it isn't.
-			 */
-			netdev_dbg(usbdev->net, "ignored RNDIS_OID_802_11_ADD_KEY triggered 'media connect'\n");
-			return;
-		}
-
-		usbnet_pause_rx(usbdev);
-
-		netdev_info(usbdev->net, "media connect\n");
-
-		/* queue work to avoid recursive calls into rndis_command */
-		set_bit(WORK_LINK_UP, &priv->work_pending);
-		queue_work(priv->workqueue, &priv->work);
-		break;
-
-	case RNDIS_STATUS_MEDIA_DISCONNECT:
-		netdev_info(usbdev->net, "media disconnect\n");
-
-		/* queue work to avoid recursive calls into rndis_command */
-		set_bit(WORK_LINK_DOWN, &priv->work_pending);
-		queue_work(priv->workqueue, &priv->work);
-		break;
-
-	case RNDIS_STATUS_MEDIA_SPECIFIC_INDICATION:
-		rndis_wlan_media_specific_indication(usbdev, msg, buflen);
-		break;
-
-	default:
-		netdev_info(usbdev->net, "indication: 0x%08x\n",
-			    le32_to_cpu(msg->status));
-		break;
-	}
-}
-
-static int rndis_wlan_get_caps(struct usbnet *usbdev, struct wiphy *wiphy)
-{
-	struct {
-		__le32	num_items;
-		__le32	items[8];
-	} networks_supported;
-	struct ndis_80211_capability caps;
-	int len, retval, i, n;
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-
-	/* determine supported modes */
-	len = sizeof(networks_supported);
-	retval = rndis_query_oid(usbdev,
-				 RNDIS_OID_802_11_NETWORK_TYPES_SUPPORTED,
-				 &networks_supported, &len);
-	if (!retval) {
-		n = le32_to_cpu(networks_supported.num_items);
-		if (n > 8)
-			n = 8;
-		for (i = 0; i < n; i++) {
-			switch (le32_to_cpu(networks_supported.items[i])) {
-			case NDIS_80211_TYPE_FREQ_HOP:
-			case NDIS_80211_TYPE_DIRECT_SEQ:
-				priv->caps |= CAP_MODE_80211B;
-				break;
-			case NDIS_80211_TYPE_OFDM_A:
-				priv->caps |= CAP_MODE_80211A;
-				break;
-			case NDIS_80211_TYPE_OFDM_G:
-				priv->caps |= CAP_MODE_80211G;
-				break;
-			}
-		}
-	}
-
-	/* get device 802.11 capabilities, number of PMKIDs */
-	len = sizeof(caps);
-	retval = rndis_query_oid(usbdev,
-				 RNDIS_OID_802_11_CAPABILITY,
-				 &caps, &len);
-	if (!retval) {
-		netdev_dbg(usbdev->net, "RNDIS_OID_802_11_CAPABILITY -> len %d, "
-				"ver %d, pmkids %d, auth-encr-pairs %d\n",
-				le32_to_cpu(caps.length),
-				le32_to_cpu(caps.version),
-				le32_to_cpu(caps.num_pmkids),
-				le32_to_cpu(caps.num_auth_encr_pair));
-		wiphy->max_num_pmkids = le32_to_cpu(caps.num_pmkids);
-	} else
-		wiphy->max_num_pmkids = 0;
-
-	return retval;
-}
-
-static void rndis_do_cqm(struct usbnet *usbdev, s32 rssi)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	enum nl80211_cqm_rssi_threshold_event event;
-	int thold, hyst, last_event;
-
-	if (priv->cqm_rssi_thold >= 0 || rssi >= 0)
-		return;
-	if (priv->infra_mode != NDIS_80211_INFRA_INFRA)
-		return;
-
-	last_event = priv->last_cqm_event_rssi;
-	thold = priv->cqm_rssi_thold;
-	hyst = priv->cqm_rssi_hyst;
-
-	if (rssi < thold && (last_event == 0 || rssi < last_event - hyst))
-		event = NL80211_CQM_RSSI_THRESHOLD_EVENT_LOW;
-	else if (rssi > thold && (last_event == 0 || rssi > last_event + hyst))
-		event = NL80211_CQM_RSSI_THRESHOLD_EVENT_HIGH;
-	else
-		return;
-
-	priv->last_cqm_event_rssi = rssi;
-	cfg80211_cqm_rssi_notify(usbdev->net, event, rssi, GFP_KERNEL);
-}
-
-#define DEVICE_POLLER_JIFFIES (HZ)
-static void rndis_device_poller(struct work_struct *work)
-{
-	struct rndis_wlan_private *priv =
-		container_of(work, struct rndis_wlan_private,
-							dev_poller_work.work);
-	struct usbnet *usbdev = priv->usbdev;
-	__le32 rssi, tmp;
-	int len, ret, j;
-	int update_jiffies = DEVICE_POLLER_JIFFIES;
-	void *buf;
-
-	/* Only check/do workaround when connected. Calling is_associated()
-	 * also polls device with rndis_command() and catches for media link
-	 * indications.
-	 */
-	if (!is_associated(usbdev)) {
-		/* Workaround bad scanning in BCM4320a devices with active
-		 * background scanning when not associated.
-		 */
-		if (priv->device_type == RNDIS_BCM4320A && priv->radio_on &&
-		    !priv->scan_request) {
-			/* Get previous scan results */
-			rndis_check_bssid_list(usbdev, NULL, NULL);
-
-			/* Initiate new scan */
-			rndis_start_bssid_list_scan(usbdev);
-		}
-
-		goto end;
-	}
-
-	len = sizeof(rssi);
-	ret = rndis_query_oid(usbdev, RNDIS_OID_802_11_RSSI,
-			      &rssi, &len);
-	if (ret == 0) {
-		priv->last_qual = level_to_qual(le32_to_cpu(rssi));
-		rndis_do_cqm(usbdev, le32_to_cpu(rssi));
-	}
-
-	netdev_dbg(usbdev->net, "dev-poller: RNDIS_OID_802_11_RSSI -> %d, rssi:%d, qual: %d\n",
-		   ret, le32_to_cpu(rssi), level_to_qual(le32_to_cpu(rssi)));
-
-	/* Workaround transfer stalls on poor quality links.
-	 * TODO: find right way to fix these stalls (as stalls do not happen
-	 * with ndiswrapper/windows driver). */
-	if (priv->param_workaround_interval > 0 && priv->last_qual <= 25) {
-		/* Decrease stats worker interval to catch stalls.
-		 * faster. Faster than 400-500ms causes packet loss,
-		 * Slower doesn't catch stalls fast enough.
-		 */
-		j = msecs_to_jiffies(priv->param_workaround_interval);
-		if (j > DEVICE_POLLER_JIFFIES)
-			j = DEVICE_POLLER_JIFFIES;
-		else if (j <= 0)
-			j = 1;
-		update_jiffies = j;
-
-		/* Send scan OID. Use of both OIDs is required to get device
-		 * working.
-		 */
-		tmp = cpu_to_le32(1);
-		rndis_set_oid(usbdev,
-			      RNDIS_OID_802_11_BSSID_LIST_SCAN,
-			      &tmp, sizeof(tmp));
-
-		len = CONTROL_BUFFER_SIZE;
-		buf = kmalloc(len, GFP_KERNEL);
-		if (!buf)
-			goto end;
-
-		rndis_query_oid(usbdev,
-				RNDIS_OID_802_11_BSSID_LIST,
-				buf, &len);
-		kfree(buf);
-	}
-
-end:
-	if (update_jiffies >= HZ)
-		update_jiffies = round_jiffies_relative(update_jiffies);
-	else {
-		j = round_jiffies_relative(update_jiffies);
-		if (abs(j - update_jiffies) <= 10)
-			update_jiffies = j;
-	}
-
-	queue_delayed_work(priv->workqueue, &priv->dev_poller_work,
-								update_jiffies);
-}
-
-/*
- * driver/device initialization
- */
-static void rndis_copy_module_params(struct usbnet *usbdev, int device_type)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-
-	priv->device_type = device_type;
-
-	priv->param_country[0] = modparam_country[0];
-	priv->param_country[1] = modparam_country[1];
-	priv->param_country[2] = 0;
-	priv->param_frameburst   = modparam_frameburst;
-	priv->param_afterburner  = modparam_afterburner;
-	priv->param_power_save   = modparam_power_save;
-	priv->param_power_output = modparam_power_output;
-	priv->param_roamtrigger  = modparam_roamtrigger;
-	priv->param_roamdelta    = modparam_roamdelta;
-
-	priv->param_country[0] = toupper(priv->param_country[0]);
-	priv->param_country[1] = toupper(priv->param_country[1]);
-	/* doesn't support EU as country code, use FI instead */
-	if (!strcmp(priv->param_country, "EU"))
-		strcpy(priv->param_country, "FI");
-
-	if (priv->param_power_save < 0)
-		priv->param_power_save = 0;
-	else if (priv->param_power_save > 2)
-		priv->param_power_save = 2;
-
-	if (priv->param_power_output < 0)
-		priv->param_power_output = 0;
-	else if (priv->param_power_output > 3)
-		priv->param_power_output = 3;
-
-	if (priv->param_roamtrigger < -80)
-		priv->param_roamtrigger = -80;
-	else if (priv->param_roamtrigger > -60)
-		priv->param_roamtrigger = -60;
-
-	if (priv->param_roamdelta < 0)
-		priv->param_roamdelta = 0;
-	else if (priv->param_roamdelta > 2)
-		priv->param_roamdelta = 2;
-
-	if (modparam_workaround_interval < 0)
-		priv->param_workaround_interval = 500;
-	else
-		priv->param_workaround_interval = modparam_workaround_interval;
-}
-
-static int unknown_early_init(struct usbnet *usbdev)
-{
-	/* copy module parameters for unknown so that iwconfig reports txpower
-	 * and workaround parameter is copied to private structure correctly.
-	 */
-	rndis_copy_module_params(usbdev, RNDIS_UNKNOWN);
-
-	/* This is unknown device, so do not try set configuration parameters.
-	 */
-
-	return 0;
-}
-
-static int bcm4320a_early_init(struct usbnet *usbdev)
-{
-	/* copy module parameters for bcm4320a so that iwconfig reports txpower
-	 * and workaround parameter is copied to private structure correctly.
-	 */
-	rndis_copy_module_params(usbdev, RNDIS_BCM4320A);
-
-	/* bcm4320a doesn't handle configuration parameters well. Try
-	 * set any and you get partially zeroed mac and broken device.
-	 */
-
-	return 0;
-}
-
-static int bcm4320b_early_init(struct usbnet *usbdev)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	char buf[8];
-
-	rndis_copy_module_params(usbdev, RNDIS_BCM4320B);
-
-	/* Early initialization settings, setting these won't have effect
-	 * if called after generic_rndis_bind().
-	 */
-
-	rndis_set_config_parameter_str(usbdev, "Country", priv->param_country);
-	rndis_set_config_parameter_str(usbdev, "FrameBursting",
-					priv->param_frameburst ? "1" : "0");
-	rndis_set_config_parameter_str(usbdev, "Afterburner",
-					priv->param_afterburner ? "1" : "0");
-	sprintf(buf, "%d", priv->param_power_save);
-	rndis_set_config_parameter_str(usbdev, "PowerSaveMode", buf);
-	sprintf(buf, "%d", priv->param_power_output);
-	rndis_set_config_parameter_str(usbdev, "PwrOut", buf);
-	sprintf(buf, "%d", priv->param_roamtrigger);
-	rndis_set_config_parameter_str(usbdev, "RoamTrigger", buf);
-	sprintf(buf, "%d", priv->param_roamdelta);
-	rndis_set_config_parameter_str(usbdev, "RoamDelta", buf);
-
-	return 0;
-}
-
-/* same as rndis_netdev_ops but with local multicast handler */
-static const struct net_device_ops rndis_wlan_netdev_ops = {
-	.ndo_open		= usbnet_open,
-	.ndo_stop		= usbnet_stop,
-	.ndo_start_xmit		= usbnet_start_xmit,
-	.ndo_tx_timeout		= usbnet_tx_timeout,
-	.ndo_get_stats64	= dev_get_tstats64,
-	.ndo_set_mac_address 	= eth_mac_addr,
-	.ndo_validate_addr	= eth_validate_addr,
-	.ndo_set_rx_mode	= rndis_wlan_set_multicast_list,
-};
-
-static int rndis_wlan_bind(struct usbnet *usbdev, struct usb_interface *intf)
-{
-	struct wiphy *wiphy;
-	struct rndis_wlan_private *priv;
-	int retval, len;
-	__le32 tmp;
-
-	/* allocate wiphy and rndis private data
-	 * NOTE: We only support a single virtual interface, so wiphy
-	 * and wireless_dev are somewhat synonymous for this device.
-	 */
-	wiphy = wiphy_new(&rndis_config_ops, sizeof(struct rndis_wlan_private));
-	if (!wiphy)
-		return -ENOMEM;
-
-	priv = wiphy_priv(wiphy);
-	usbdev->net->ieee80211_ptr = &priv->wdev;
-	priv->wdev.wiphy = wiphy;
-	priv->wdev.iftype = NL80211_IFTYPE_STATION;
-
-	/* These have to be initialized before calling generic_rndis_bind().
-	 * Otherwise we'll be in big trouble in rndis_wlan_early_init().
-	 */
-	usbdev->driver_priv = priv;
-	priv->usbdev = usbdev;
-
-	mutex_init(&priv->command_lock);
-
-	/* because rndis_command() sleeps we need to use workqueue */
-	priv->workqueue = create_singlethread_workqueue("rndis_wlan");
-	if (!priv->workqueue) {
-		wiphy_free(wiphy);
-		return -ENOMEM;
-	}
-	INIT_WORK(&priv->work, rndis_wlan_worker);
-	INIT_DELAYED_WORK(&priv->dev_poller_work, rndis_device_poller);
-	INIT_DELAYED_WORK(&priv->scan_work, rndis_get_scan_results);
-
-	/* try bind rndis_host */
-	retval = generic_rndis_bind(usbdev, intf, FLAG_RNDIS_PHYM_WIRELESS);
-	if (retval < 0)
-		goto fail;
-
-	/* generic_rndis_bind set packet filter to multicast_all+
-	 * promisc mode which doesn't work well for our devices (device
-	 * picks up rssi to closest station instead of to access point).
-	 *
-	 * rndis_host wants to avoid all OID as much as possible
-	 * so do promisc/multicast handling in rndis_wlan.
-	 */
-	usbdev->net->netdev_ops = &rndis_wlan_netdev_ops;
-
-	tmp = cpu_to_le32(RNDIS_PACKET_TYPE_DIRECTED | RNDIS_PACKET_TYPE_BROADCAST);
-	retval = rndis_set_oid(usbdev,
-			       RNDIS_OID_GEN_CURRENT_PACKET_FILTER,
-			       &tmp, sizeof(tmp));
-
-	len = sizeof(tmp);
-	retval = rndis_query_oid(usbdev,
-				 RNDIS_OID_802_3_MAXIMUM_LIST_SIZE,
-				 &tmp, &len);
-	priv->multicast_size = le32_to_cpu(tmp);
-	if (retval < 0 || priv->multicast_size < 0)
-		priv->multicast_size = 0;
-	if (priv->multicast_size > 0)
-		usbdev->net->flags |= IFF_MULTICAST;
-	else
-		usbdev->net->flags &= ~IFF_MULTICAST;
-
-	/* fill-out wiphy structure and register w/ cfg80211 */
-	memcpy(wiphy->perm_addr, usbdev->net->dev_addr, ETH_ALEN);
-	wiphy->privid = rndis_wiphy_privid;
-	wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION)
-					| BIT(NL80211_IFTYPE_ADHOC);
-	wiphy->max_scan_ssids = 1;
-
-	/* TODO: fill-out band/encr information based on priv->caps */
-	rndis_wlan_get_caps(usbdev, wiphy);
-
-	memcpy(priv->channels, rndis_channels, sizeof(rndis_channels));
-	memcpy(priv->rates, rndis_rates, sizeof(rndis_rates));
-	priv->band.channels = priv->channels;
-	priv->band.n_channels = ARRAY_SIZE(rndis_channels);
-	priv->band.bitrates = priv->rates;
-	priv->band.n_bitrates = ARRAY_SIZE(rndis_rates);
-	wiphy->bands[NL80211_BAND_2GHZ] = &priv->band;
-	wiphy->signal_type = CFG80211_SIGNAL_TYPE_UNSPEC;
-
-	memcpy(priv->cipher_suites, rndis_cipher_suites,
-						sizeof(rndis_cipher_suites));
-	wiphy->cipher_suites = priv->cipher_suites;
-	wiphy->n_cipher_suites = ARRAY_SIZE(rndis_cipher_suites);
-
-	set_wiphy_dev(wiphy, &usbdev->udev->dev);
-
-	if (wiphy_register(wiphy)) {
-		retval = -ENODEV;
-		goto fail;
-	}
-
-	set_default_iw_params(usbdev);
-
-	priv->power_mode = -1;
-
-	/* set default rts/frag */
-	rndis_set_wiphy_params(wiphy,
-			WIPHY_PARAM_FRAG_THRESHOLD | WIPHY_PARAM_RTS_THRESHOLD);
-
-	/* turn radio off on init */
-	priv->radio_on = false;
-	disassociate(usbdev, false);
-	netif_carrier_off(usbdev->net);
-
-	return 0;
-
-fail:
-	cancel_delayed_work_sync(&priv->dev_poller_work);
-	cancel_delayed_work_sync(&priv->scan_work);
-	cancel_work_sync(&priv->work);
-	destroy_workqueue(priv->workqueue);
-
-	wiphy_free(wiphy);
-	return retval;
-}
-
-static void rndis_wlan_unbind(struct usbnet *usbdev, struct usb_interface *intf)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-
-	/* turn radio off */
-	disassociate(usbdev, false);
-
-	cancel_delayed_work_sync(&priv->dev_poller_work);
-	cancel_delayed_work_sync(&priv->scan_work);
-	cancel_work_sync(&priv->work);
-	destroy_workqueue(priv->workqueue);
-
-	rndis_unbind(usbdev, intf);
-
-	wiphy_unregister(priv->wdev.wiphy);
-	wiphy_free(priv->wdev.wiphy);
-}
-
-static int rndis_wlan_reset(struct usbnet *usbdev)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	int retval;
-
-	netdev_dbg(usbdev->net, "%s()\n", __func__);
-
-	retval = rndis_reset(usbdev);
-	if (retval)
-		netdev_warn(usbdev->net, "rndis_reset failed: %d\n", retval);
-
-	/* rndis_reset cleared multicast list, so restore here.
-	   (set_multicast_list() also turns on current packet filter) */
-	set_multicast_list(usbdev);
-
-	queue_delayed_work(priv->workqueue, &priv->dev_poller_work,
-		round_jiffies_relative(DEVICE_POLLER_JIFFIES));
-
-	return deauthenticate(usbdev);
-}
-
-static int rndis_wlan_stop(struct usbnet *usbdev)
-{
-	struct rndis_wlan_private *priv = get_rndis_wlan_priv(usbdev);
-	int retval;
-	__le32 filter;
-
-	netdev_dbg(usbdev->net, "%s()\n", __func__);
-
-	retval = disassociate(usbdev, false);
-
-	priv->work_pending = 0;
-	cancel_delayed_work_sync(&priv->dev_poller_work);
-	cancel_delayed_work_sync(&priv->scan_work);
-	cancel_work_sync(&priv->work);
-	flush_workqueue(priv->workqueue);
-
-	if (priv->scan_request) {
-		struct cfg80211_scan_info info = {
-			.aborted = true,
-		};
-
-		cfg80211_scan_done(priv->scan_request, &info);
-		priv->scan_request = NULL;
-	}
-
-	/* Set current packet filter zero to block receiving data packets from
-	   device. */
-	filter = 0;
-	rndis_set_oid(usbdev, RNDIS_OID_GEN_CURRENT_PACKET_FILTER, &filter,
-								sizeof(filter));
-
-	return retval;
-}
-
-static const struct driver_info	bcm4320b_info = {
-	.description =	"Wireless RNDIS device, BCM4320b based",
-	.flags =	FLAG_WLAN | FLAG_FRAMING_RN | FLAG_NO_SETINT |
-				FLAG_AVOID_UNLINK_URBS,
-	.bind =		rndis_wlan_bind,
-	.unbind =	rndis_wlan_unbind,
-	.status =	rndis_status,
-	.rx_fixup =	rndis_rx_fixup,
-	.tx_fixup =	rndis_tx_fixup,
-	.reset =	rndis_wlan_reset,
-	.stop =		rndis_wlan_stop,
-	.early_init =	bcm4320b_early_init,
-	.indication =	rndis_wlan_indication,
-};
-
-static const struct driver_info	bcm4320a_info = {
-	.description =	"Wireless RNDIS device, BCM4320a based",
-	.flags =	FLAG_WLAN | FLAG_FRAMING_RN | FLAG_NO_SETINT |
-				FLAG_AVOID_UNLINK_URBS,
-	.bind =		rndis_wlan_bind,
-	.unbind =	rndis_wlan_unbind,
-	.status =	rndis_status,
-	.rx_fixup =	rndis_rx_fixup,
-	.tx_fixup =	rndis_tx_fixup,
-	.reset =	rndis_wlan_reset,
-	.stop =		rndis_wlan_stop,
-	.early_init =	bcm4320a_early_init,
-	.indication =	rndis_wlan_indication,
-};
-
-static const struct driver_info rndis_wlan_info = {
-	.description =	"Wireless RNDIS device",
-	.flags =	FLAG_WLAN | FLAG_FRAMING_RN | FLAG_NO_SETINT |
-				FLAG_AVOID_UNLINK_URBS,
-	.bind =		rndis_wlan_bind,
-	.unbind =	rndis_wlan_unbind,
-	.status =	rndis_status,
-	.rx_fixup =	rndis_rx_fixup,
-	.tx_fixup =	rndis_tx_fixup,
-	.reset =	rndis_wlan_reset,
-	.stop =		rndis_wlan_stop,
-	.early_init =	unknown_early_init,
-	.indication =	rndis_wlan_indication,
-};
-
-/*-------------------------------------------------------------------------*/
-
-static const struct usb_device_id products [] = {
-#define	RNDIS_MASTER_INTERFACE \
-	.bInterfaceClass	= USB_CLASS_COMM, \
-	.bInterfaceSubClass	= 2 /* ACM */, \
-	.bInterfaceProtocol	= 0x0ff
-
-/* INF driver for these devices have DriverVer >= 4.xx.xx.xx and many custom
- * parameters available. Chipset marked as 'BCM4320SKFBG' in NDISwrapper-wiki.
- */
-{
-	.match_flags	=   USB_DEVICE_ID_MATCH_INT_INFO
-			  | USB_DEVICE_ID_MATCH_DEVICE,
-	.idVendor		= 0x0411,
-	.idProduct		= 0x00bc,	/* Buffalo WLI-U2-KG125S */
-	RNDIS_MASTER_INTERFACE,
-	.driver_info		= (unsigned long) &bcm4320b_info,
-}, {
-	.match_flags	=   USB_DEVICE_ID_MATCH_INT_INFO
-			  | USB_DEVICE_ID_MATCH_DEVICE,
-	.idVendor		= 0x0baf,
-	.idProduct		= 0x011b,	/* U.S. Robotics USR5421 */
-	RNDIS_MASTER_INTERFACE,
-	.driver_info		= (unsigned long) &bcm4320b_info,
-}, {
-	.match_flags	=   USB_DEVICE_ID_MATCH_INT_INFO
-			  | USB_DEVICE_ID_MATCH_DEVICE,
-	.idVendor		= 0x050d,
-	.idProduct		= 0x011b,	/* Belkin F5D7051 */
-	RNDIS_MASTER_INTERFACE,
-	.driver_info		= (unsigned long) &bcm4320b_info,
-}, {
-	.match_flags	=   USB_DEVICE_ID_MATCH_INT_INFO
-			  | USB_DEVICE_ID_MATCH_DEVICE,
-	.idVendor		= 0x1799,	/* Belkin has two vendor ids */
-	.idProduct		= 0x011b,	/* Belkin F5D7051 */
-	RNDIS_MASTER_INTERFACE,
-	.driver_info		= (unsigned long) &bcm4320b_info,
-}, {
-	.match_flags	=   USB_DEVICE_ID_MATCH_INT_INFO
-			  | USB_DEVICE_ID_MATCH_DEVICE,
-	.idVendor		= 0x13b1,
-	.idProduct		= 0x0014,	/* Linksys WUSB54GSv2 */
-	RNDIS_MASTER_INTERFACE,
-	.driver_info		= (unsigned long) &bcm4320b_info,
-}, {
-	.match_flags	=   USB_DEVICE_ID_MATCH_INT_INFO
-			  | USB_DEVICE_ID_MATCH_DEVICE,
-	.idVendor		= 0x13b1,
-	.idProduct		= 0x0026,	/* Linksys WUSB54GSC */
-	RNDIS_MASTER_INTERFACE,
-	.driver_info		= (unsigned long) &bcm4320b_info,
-}, {
-	.match_flags	=   USB_DEVICE_ID_MATCH_INT_INFO
-			  | USB_DEVICE_ID_MATCH_DEVICE,
-	.idVendor		= 0x0b05,
-	.idProduct		= 0x1717,	/* Asus WL169gE */
-	RNDIS_MASTER_INTERFACE,
-	.driver_info		= (unsigned long) &bcm4320b_info,
-}, {
-	.match_flags	=   USB_DEVICE_ID_MATCH_INT_INFO
-			  | USB_DEVICE_ID_MATCH_DEVICE,
-	.idVendor		= 0x0a5c,
-	.idProduct		= 0xd11b,	/* Eminent EM4045 */
-	RNDIS_MASTER_INTERFACE,
-	.driver_info		= (unsigned long) &bcm4320b_info,
-}, {
-	.match_flags	=   USB_DEVICE_ID_MATCH_INT_INFO
-			  | USB_DEVICE_ID_MATCH_DEVICE,
-	.idVendor		= 0x1690,
-	.idProduct		= 0x0715,	/* BT Voyager 1055 */
-	RNDIS_MASTER_INTERFACE,
-	.driver_info		= (unsigned long) &bcm4320b_info,
-},
-/* These devices have DriverVer < 4.xx.xx.xx and do not have any custom
- * parameters available, hardware probably contain older firmware version with
- * no way of updating. Chipset marked as 'BCM4320????' in NDISwrapper-wiki.
- */
-{
-	.match_flags	=   USB_DEVICE_ID_MATCH_INT_INFO
-			  | USB_DEVICE_ID_MATCH_DEVICE,
-	.idVendor		= 0x13b1,
-	.idProduct		= 0x000e,	/* Linksys WUSB54GSv1 */
-	RNDIS_MASTER_INTERFACE,
-	.driver_info		= (unsigned long) &bcm4320a_info,
-}, {
-	.match_flags	=   USB_DEVICE_ID_MATCH_INT_INFO
-			  | USB_DEVICE_ID_MATCH_DEVICE,
-	.idVendor		= 0x0baf,
-	.idProduct		= 0x0111,	/* U.S. Robotics USR5420 */
-	RNDIS_MASTER_INTERFACE,
-	.driver_info		= (unsigned long) &bcm4320a_info,
-}, {
-	.match_flags	=   USB_DEVICE_ID_MATCH_INT_INFO
-			  | USB_DEVICE_ID_MATCH_DEVICE,
-	.idVendor		= 0x0411,
-	.idProduct		= 0x004b,	/* BUFFALO WLI-USB-G54 */
-	RNDIS_MASTER_INTERFACE,
-	.driver_info		= (unsigned long) &bcm4320a_info,
-},
-/* Generic Wireless RNDIS devices that we don't have exact
- * idVendor/idProduct/chip yet.
- */
-{
-	/* RNDIS is MSFT's un-official variant of CDC ACM */
-	USB_INTERFACE_INFO(USB_CLASS_COMM, 2 /* ACM */, 0x0ff),
-	.driver_info = (unsigned long) &rndis_wlan_info,
-}, {
-	/* "ActiveSync" is an undocumented variant of RNDIS, used in WM5 */
-	USB_INTERFACE_INFO(USB_CLASS_MISC, 1, 1),
-	.driver_info = (unsigned long) &rndis_wlan_info,
-},
-	{ },		// END
-};
-MODULE_DEVICE_TABLE(usb, products);
-
-static struct usb_driver rndis_wlan_driver = {
-	.name =		"rndis_wlan",
-	.id_table =	products,
-	.probe =	usbnet_probe,
-	.disconnect =	usbnet_disconnect,
-	.suspend =	usbnet_suspend,
-	.resume =	usbnet_resume,
-	.disable_hub_initiated_lpm = 1,
-};
-
-module_usb_driver(rndis_wlan_driver);
-
-MODULE_AUTHOR("Bjorge Dijkstra");
-MODULE_AUTHOR("Jussi Kivilinna");
-MODULE_DESCRIPTION("Driver for RNDIS based USB Wireless adapters");
-MODULE_LICENSE("GPL");
-
diff --git a/drivers/net/wireless/legacy/wl3501.h b/drivers/net/wireless/legacy/wl3501.h
deleted file mode 100644
index 91f276dd22a1..000000000000
--- a/drivers/net/wireless/legacy/wl3501.h
+++ /dev/null
@@ -1,615 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __WL3501_H__
-#define __WL3501_H__
-
-#include <linux/spinlock.h>
-#include <linux/ieee80211.h>
-
-/* define for WLA 2.0 */
-#define WL3501_BLKSZ 256
-/*
- * ID for input Signals of DRIVER block
- * bit[7-5] is block ID: 000
- * bit[4-0] is signal ID
-*/
-enum wl3501_signals {
-	WL3501_SIG_ALARM,
-	WL3501_SIG_MD_CONFIRM,
-	WL3501_SIG_MD_IND,
-	WL3501_SIG_ASSOC_CONFIRM,
-	WL3501_SIG_ASSOC_IND,
-	WL3501_SIG_AUTH_CONFIRM,
-	WL3501_SIG_AUTH_IND,
-	WL3501_SIG_DEAUTH_CONFIRM,
-	WL3501_SIG_DEAUTH_IND,
-	WL3501_SIG_DISASSOC_CONFIRM,
-	WL3501_SIG_DISASSOC_IND,
-	WL3501_SIG_GET_CONFIRM,
-	WL3501_SIG_JOIN_CONFIRM,
-	WL3501_SIG_PWR_MGMT_CONFIRM,
-	WL3501_SIG_REASSOC_CONFIRM,
-	WL3501_SIG_REASSOC_IND,
-	WL3501_SIG_SCAN_CONFIRM,
-	WL3501_SIG_SET_CONFIRM,
-	WL3501_SIG_START_CONFIRM,
-	WL3501_SIG_RESYNC_CONFIRM,
-	WL3501_SIG_SITE_CONFIRM,
-	WL3501_SIG_SAVE_CONFIRM,
-	WL3501_SIG_RFTEST_CONFIRM,
-/*
- * ID for input Signals of MLME block
- * bit[7-5] is block ID: 010
- * bit[4-0] is signal ID
- */
-	WL3501_SIG_ASSOC_REQ = 0x20,
-	WL3501_SIG_AUTH_REQ,
-	WL3501_SIG_DEAUTH_REQ,
-	WL3501_SIG_DISASSOC_REQ,
-	WL3501_SIG_GET_REQ,
-	WL3501_SIG_JOIN_REQ,
-	WL3501_SIG_PWR_MGMT_REQ,
-	WL3501_SIG_REASSOC_REQ,
-	WL3501_SIG_SCAN_REQ,
-	WL3501_SIG_SET_REQ,
-	WL3501_SIG_START_REQ,
-	WL3501_SIG_MD_REQ,
-	WL3501_SIG_RESYNC_REQ,
-	WL3501_SIG_SITE_REQ,
-	WL3501_SIG_SAVE_REQ,
-	WL3501_SIG_RF_TEST_REQ,
-	WL3501_SIG_MM_CONFIRM = 0x60,
-	WL3501_SIG_MM_IND,
-};
-
-enum wl3501_mib_attribs {
-	WL3501_MIB_ATTR_STATION_ID,
-	WL3501_MIB_ATTR_AUTH_ALGORITHMS,
-	WL3501_MIB_ATTR_AUTH_TYPE,
-	WL3501_MIB_ATTR_MEDIUM_OCCUPANCY_LIMIT,
-	WL3501_MIB_ATTR_CF_POLLABLE,
-	WL3501_MIB_ATTR_CFP_PERIOD,
-	WL3501_MIB_ATTR_CFPMAX_DURATION,
-	WL3501_MIB_ATTR_AUTH_RESP_TMOUT,
-	WL3501_MIB_ATTR_RX_DTIMS,
-	WL3501_MIB_ATTR_PRIV_OPT_IMPLEMENTED,
-	WL3501_MIB_ATTR_PRIV_INVOKED,
-	WL3501_MIB_ATTR_WEP_DEFAULT_KEYS,
-	WL3501_MIB_ATTR_WEP_DEFAULT_KEY_ID,
-	WL3501_MIB_ATTR_WEP_KEY_MAPPINGS,
-	WL3501_MIB_ATTR_WEP_KEY_MAPPINGS_LEN,
-	WL3501_MIB_ATTR_EXCLUDE_UNENCRYPTED,
-	WL3501_MIB_ATTR_WEP_ICV_ERROR_COUNT,
-	WL3501_MIB_ATTR_WEP_UNDECRYPTABLE_COUNT,
-	WL3501_MIB_ATTR_WEP_EXCLUDED_COUNT,
-	WL3501_MIB_ATTR_MAC_ADDR,
-	WL3501_MIB_ATTR_GROUP_ADDRS,
-	WL3501_MIB_ATTR_RTS_THRESHOLD,
-	WL3501_MIB_ATTR_SHORT_RETRY_LIMIT,
-	WL3501_MIB_ATTR_LONG_RETRY_LIMIT,
-	WL3501_MIB_ATTR_FRAG_THRESHOLD,
-	WL3501_MIB_ATTR_MAX_TX_MSDU_LIFETIME,
-	WL3501_MIB_ATTR_MAX_RX_LIFETIME,
-	WL3501_MIB_ATTR_MANUFACTURER_ID,
-	WL3501_MIB_ATTR_PRODUCT_ID,
-	WL3501_MIB_ATTR_TX_FRAG_COUNT,
-	WL3501_MIB_ATTR_MULTICAST_TX_FRAME_COUNT,
-	WL3501_MIB_ATTR_FAILED_COUNT,
-	WL3501_MIB_ATTR_RX_FRAG_COUNT,
-	WL3501_MIB_ATTR_MULTICAST_RX_COUNT,
-	WL3501_MIB_ATTR_FCS_ERROR_COUNT,
-	WL3501_MIB_ATTR_RETRY_COUNT,
-	WL3501_MIB_ATTR_MULTIPLE_RETRY_COUNT,
-	WL3501_MIB_ATTR_RTS_SUCCESS_COUNT,
-	WL3501_MIB_ATTR_RTS_FAILURE_COUNT,
-	WL3501_MIB_ATTR_ACK_FAILURE_COUNT,
-	WL3501_MIB_ATTR_FRAME_DUPLICATE_COUNT,
-	WL3501_MIB_ATTR_PHY_TYPE,
-	WL3501_MIB_ATTR_REG_DOMAINS_SUPPORT,
-	WL3501_MIB_ATTR_CURRENT_REG_DOMAIN,
-	WL3501_MIB_ATTR_SLOT_TIME,
-	WL3501_MIB_ATTR_CCA_TIME,
-	WL3501_MIB_ATTR_RX_TX_TURNAROUND_TIME,
-	WL3501_MIB_ATTR_TX_PLCP_DELAY,
-	WL3501_MIB_ATTR_RX_TX_SWITCH_TIME,
-	WL3501_MIB_ATTR_TX_RAMP_ON_TIME,
-	WL3501_MIB_ATTR_TX_RF_DELAY,
-	WL3501_MIB_ATTR_SIFS_TIME,
-	WL3501_MIB_ATTR_RX_RF_DELAY,
-	WL3501_MIB_ATTR_RX_PLCP_DELAY,
-	WL3501_MIB_ATTR_MAC_PROCESSING_DELAY,
-	WL3501_MIB_ATTR_TX_RAMP_OFF_TIME,
-	WL3501_MIB_ATTR_PREAMBLE_LEN,
-	WL3501_MIB_ATTR_PLCP_HEADER_LEN,
-	WL3501_MIB_ATTR_MPDU_DURATION_FACTOR,
-	WL3501_MIB_ATTR_AIR_PROPAGATION_TIME,
-	WL3501_MIB_ATTR_TEMP_TYPE,
-	WL3501_MIB_ATTR_CW_MIN,
-	WL3501_MIB_ATTR_CW_MAX,
-	WL3501_MIB_ATTR_SUPPORT_DATA_RATES_TX,
-	WL3501_MIB_ATTR_SUPPORT_DATA_RATES_RX,
-	WL3501_MIB_ATTR_MPDU_MAX_LEN,
-	WL3501_MIB_ATTR_SUPPORT_TX_ANTENNAS,
-	WL3501_MIB_ATTR_CURRENT_TX_ANTENNA,
-	WL3501_MIB_ATTR_SUPPORT_RX_ANTENNAS,
-	WL3501_MIB_ATTR_DIVERSITY_SUPPORT,
-	WL3501_MIB_ATTR_DIVERSITY_SELECTION_RS,
-	WL3501_MIB_ATTR_NR_SUPPORTED_PWR_LEVELS,
-	WL3501_MIB_ATTR_TX_PWR_LEVEL1,
-	WL3501_MIB_ATTR_TX_PWR_LEVEL2,
-	WL3501_MIB_ATTR_TX_PWR_LEVEL3,
-	WL3501_MIB_ATTR_TX_PWR_LEVEL4,
-	WL3501_MIB_ATTR_TX_PWR_LEVEL5,
-	WL3501_MIB_ATTR_TX_PWR_LEVEL6,
-	WL3501_MIB_ATTR_TX_PWR_LEVEL7,
-	WL3501_MIB_ATTR_TX_PWR_LEVEL8,
-	WL3501_MIB_ATTR_CURRENT_TX_PWR_LEVEL,
-	WL3501_MIB_ATTR_CURRENT_CHAN,
-	WL3501_MIB_ATTR_CCA_MODE_SUPPORTED,
-	WL3501_MIB_ATTR_CURRENT_CCA_MODE,
-	WL3501_MIB_ATTR_ED_THRESHOLD,
-	WL3501_MIB_ATTR_SINTHESIZER_LOCKED,
-	WL3501_MIB_ATTR_CURRENT_PWR_STATE,
-	WL3501_MIB_ATTR_DOZE_TURNON_TIME,
-	WL3501_MIB_ATTR_RCR33,
-	WL3501_MIB_ATTR_DEFAULT_CHAN,
-	WL3501_MIB_ATTR_SSID,
-	WL3501_MIB_ATTR_PWR_MGMT_ENABLE,
-	WL3501_MIB_ATTR_NET_CAPABILITY,
-	WL3501_MIB_ATTR_ROUTING,
-};
-
-enum wl3501_net_type {
-	WL3501_NET_TYPE_INFRA,
-	WL3501_NET_TYPE_ADHOC,
-	WL3501_NET_TYPE_ANY_BSS,
-};
-
-enum wl3501_scan_type {
-	WL3501_SCAN_TYPE_ACTIVE,
-	WL3501_SCAN_TYPE_PASSIVE,
-};
-
-enum wl3501_tx_result {
-	WL3501_TX_RESULT_SUCCESS,
-	WL3501_TX_RESULT_NO_BSS,
-	WL3501_TX_RESULT_RETRY_LIMIT,
-};
-
-enum wl3501_sys_type {
-	WL3501_SYS_TYPE_OPEN,
-	WL3501_SYS_TYPE_SHARE_KEY,
-};
-
-enum wl3501_status {
-	WL3501_STATUS_SUCCESS,
-	WL3501_STATUS_INVALID,
-	WL3501_STATUS_TIMEOUT,
-	WL3501_STATUS_REFUSED,
-	WL3501_STATUS_MANY_REQ,
-	WL3501_STATUS_ALREADY_BSS,
-};
-
-#define WL3501_MGMT_CAPABILITY_ESS		0x0001  /* see 802.11 p.58 */
-#define WL3501_MGMT_CAPABILITY_IBSS		0x0002  /*      - " -	   */
-#define WL3501_MGMT_CAPABILITY_CF_POLLABLE	0x0004  /*      - " -	   */
-#define WL3501_MGMT_CAPABILITY_CF_POLL_REQUEST	0x0008  /*      - " -	   */
-#define WL3501_MGMT_CAPABILITY_PRIVACY		0x0010  /*      - " -	   */
-
-#define IW_REG_DOMAIN_FCC	0x10	/* Channel 1 to 11	USA    */
-#define IW_REG_DOMAIN_DOC	0x20	/* Channel 1 to 11	Canada */
-#define IW_REG_DOMAIN_ETSI	0x30	/* Channel 1 to 13	Europe */
-#define IW_REG_DOMAIN_SPAIN	0x31	/* Channel 10 to 11	Spain  */
-#define IW_REG_DOMAIN_FRANCE	0x32	/* Channel 10 to 13	France */
-#define IW_REG_DOMAIN_MKK	0x40	/* Channel 14		Japan  */
-#define IW_REG_DOMAIN_MKK1	0x41	/* Channel 1-14		Japan  */
-#define IW_REG_DOMAIN_ISRAEL	0x50	/* Channel 3 - 9	Israel */
-
-#define IW_MGMT_RATE_LABEL_MANDATORY 128 /* MSB */
-
-enum iw_mgmt_rate_labels {
-	IW_MGMT_RATE_LABEL_1MBIT   = 2,
-	IW_MGMT_RATE_LABEL_2MBIT   = 4,
-	IW_MGMT_RATE_LABEL_5_5MBIT = 11,
-	IW_MGMT_RATE_LABEL_11MBIT  = 22,
-};
-
-enum iw_mgmt_info_element_ids {
-	IW_MGMT_INFO_ELEMENT_SSID,		  /* Service Set Identity */
-	IW_MGMT_INFO_ELEMENT_SUPPORTED_RATES,
-	IW_MGMT_INFO_ELEMENT_FH_PARAMETER_SET,
-	IW_MGMT_INFO_ELEMENT_DS_PARAMETER_SET,
-	IW_MGMT_INFO_ELEMENT_CS_PARAMETER_SET,
-	IW_MGMT_INFO_ELEMENT_CS_TIM,		  /* Traffic Information Map */
-	IW_MGMT_INFO_ELEMENT_IBSS_PARAMETER_SET,
-	/* 7-15: Reserved, unused */
-	IW_MGMT_INFO_ELEMENT_CHALLENGE_TEXT = 16,
-	/* 17-31 Reserved for challenge text extension */
-	/* 32-255 Reserved, unused */
-};
-
-struct iw_mgmt_info_element {
-	u8 id; /* one of enum iw_mgmt_info_element_ids,
-		  but sizeof(enum) > sizeof(u8) :-( */
-	u8 len;
-	u8 data[];
-} __packed;
-
-struct iw_mgmt_essid_pset {
-	struct iw_mgmt_info_element el;
-	u8 			    essid[IW_ESSID_MAX_SIZE];
-} __packed;
-
-/*
- * According to 802.11 Wireless Networks, the definitive guide - O'Reilly
- * Pg 75
- */ 
-#define IW_DATA_RATE_MAX_LABELS 8
-
-struct iw_mgmt_data_rset {
-	struct iw_mgmt_info_element el;
-	u8 			    data_rate_labels[IW_DATA_RATE_MAX_LABELS];
-} __packed;
-
-struct iw_mgmt_ds_pset {
-	struct iw_mgmt_info_element el;
-	u8 			    chan;
-} __packed;
-
-struct iw_mgmt_cf_pset {
-	struct iw_mgmt_info_element el;
-	u8 			    cfp_count;
-	u8 			    cfp_period;
-	u16 			    cfp_max_duration;
-	u16 			    cfp_dur_remaining;
-} __packed;
-
-struct iw_mgmt_ibss_pset {
-	struct iw_mgmt_info_element el;
-	u16 			    atim_window;
-} __packed;
-
-struct wl3501_tx_hdr {
-	u16	tx_cnt;
-	u8	sync[16];
-	u16	sfd;
-	u8	signal;
-	u8	service;
-	u16	len;
-	u16	crc16;
-	u16	frame_ctrl;
-	u16	duration_id;
-	u8	addr1[ETH_ALEN];
-	u8	addr2[ETH_ALEN];
-	u8	addr3[ETH_ALEN];
-	u16	seq_ctrl;
-	u8	addr4[ETH_ALEN];
-};
-
-struct wl3501_rx_hdr {
-	u16	rx_next_blk;
-	u16	rc_next_frame_blk;
-	u8	rx_blk_ctrl;
-	u8	rx_next_frame;
-	u8	rx_next_frame1;
-	u8	rssi;
-	char	time[8];
-	u8	signal;
-	u8	service;
-	u16	len;
-	u16	crc16;
-	u16	frame_ctrl;
-	u16	duration;
-	u8	addr1[ETH_ALEN];
-	u8	addr2[ETH_ALEN];
-	u8	addr3[ETH_ALEN];
-	u16	seq;
-	u8	addr4[ETH_ALEN];
-};
-
-struct wl3501_start_req {
-	u16			    next_blk;
-	u8			    sig_id;
-	u8			    bss_type;
-	u16			    beacon_period;
-	u16			    dtim_period;
-	u16			    probe_delay;
-	u16			    cap_info;
-	struct iw_mgmt_essid_pset   ssid;
-	struct iw_mgmt_data_rset    bss_basic_rset;
-	struct iw_mgmt_data_rset    operational_rset;
-	struct iw_mgmt_cf_pset	    cf_pset;
-	struct iw_mgmt_ds_pset	    ds_pset;
-	struct iw_mgmt_ibss_pset    ibss_pset;
-};
-
-struct wl3501_assoc_req {
-	u16	next_blk;
-	u8	sig_id;
-	u8	reserved;
-	u16	timeout;
-	u16	cap_info;
-	u16	listen_interval;
-	u8	mac_addr[ETH_ALEN];
-};
-
-struct wl3501_assoc_confirm {
-	u16	next_blk;
-	u8	sig_id;
-	u8	reserved;
-	u16	status;
-};
-
-struct wl3501_assoc_ind {
-	u16	next_blk;
-	u8	sig_id;
-	u8	mac_addr[ETH_ALEN];
-};
-
-struct wl3501_auth_req {
-	u16	next_blk;
-	u8	sig_id;
-	u8	reserved;
-	u16	type;
-	u16	timeout;
-	u8	mac_addr[ETH_ALEN];
-};
-
-struct wl3501_auth_confirm {
-	u16	next_blk;
-	u8	sig_id;
-	u8	reserved;
-	u16	type;
-	u16	status;
-	u8	mac_addr[ETH_ALEN];
-};
-
-struct wl3501_get_req {
-	u16	next_blk;
-	u8	sig_id;
-	u8	reserved;
-	u16	mib_attrib;
-};
-
-struct wl3501_get_confirm {
-	u16	next_blk;
-	u8	sig_id;
-	u8	reserved;
-	u16	mib_status;
-	u16	mib_attrib;
-	u8	mib_value[100];
-};
-
-struct wl3501_req {
-	u16			    beacon_period;
-	u16			    dtim_period;
-	u16			    cap_info;
-	u8			    bss_type;
-	u8			    bssid[ETH_ALEN];
-	struct iw_mgmt_essid_pset   ssid;
-	struct iw_mgmt_ds_pset	    ds_pset;
-	struct iw_mgmt_cf_pset	    cf_pset;
-	struct iw_mgmt_ibss_pset    ibss_pset;
-	struct iw_mgmt_data_rset    bss_basic_rset;
-};
-
-struct wl3501_join_req {
-	u16			    next_blk;
-	u8			    sig_id;
-	u8			    reserved;
-	struct iw_mgmt_data_rset    operational_rset;
-	u16			    reserved2;
-	u16			    timeout;
-	u16			    probe_delay;
-	u8			    timestamp[8];
-	u8			    local_time[8];
-	struct wl3501_req	    req;
-};
-
-struct wl3501_join_confirm {
-	u16	next_blk;
-	u8	sig_id;
-	u8	reserved;
-	u16	status;
-};
-
-struct wl3501_pwr_mgmt_req {
-	u16	next_blk;
-	u8	sig_id;
-	u8	pwr_save;
-	u8	wake_up;
-	u8	receive_dtims;
-};
-
-struct wl3501_pwr_mgmt_confirm {
-	u16	next_blk;
-	u8	sig_id;
-	u8	reserved;
-	u16	status;
-};
-
-struct wl3501_scan_req {
-	u16			    next_blk;
-	u8			    sig_id;
-	u8			    bss_type;
-	u16			    probe_delay;
-	u16			    min_chan_time;
-	u16			    max_chan_time;
-	u8			    chan_list[14];
-	u8			    bssid[ETH_ALEN];
-	struct iw_mgmt_essid_pset   ssid;
-	enum wl3501_scan_type	    scan_type;
-};
-
-struct wl3501_scan_confirm {
-	u16			    next_blk;
-	u8			    sig_id;
-	u8			    reserved;
-	u16			    status;
-	char			    timestamp[8];
-	char			    localtime[8];
-	struct wl3501_req	    req;
-	u8			    rssi;
-};
-
-struct wl3501_start_confirm {
-	u16	next_blk;
-	u8	sig_id;
-	u8	reserved;
-	u16	status;
-};
-
-struct wl3501_md_req {
-	u16	next_blk;
-	u8	sig_id;
-	u8	routing;
-	u16	data;
-	u16	size;
-	u8	pri;
-	u8	service_class;
-	struct {
-		u8	daddr[ETH_ALEN];
-		u8	saddr[ETH_ALEN];
-	} addr;
-};
-
-struct wl3501_md_ind {
-	u16	next_blk;
-	u8	sig_id;
-	u8	routing;
-	u16	data;
-	u16	size;
-	u8	reception;
-	u8	pri;
-	u8	service_class;
-	struct {
-		u8	daddr[ETH_ALEN];
-		u8	saddr[ETH_ALEN];
-	} addr;
-};
-
-struct wl3501_md_confirm {
-	u16	next_blk;
-	u8	sig_id;
-	u8	reserved;
-	u16	data;
-	u8	status;
-	u8	pri;
-	u8	service_class;
-};
-
-struct wl3501_resync_req {
-	u16	next_blk;
-	u8	sig_id;
-};
-
-/* Definitions for supporting clone adapters. */
-/* System Interface Registers (SIR space) */
-#define WL3501_NIC_GCR ((u8)0x00)	/* SIR0 - General Conf Register */
-#define WL3501_NIC_BSS ((u8)0x01)	/* SIR1 - Bank Switching Select Reg */
-#define WL3501_NIC_LMAL ((u8)0x02)	/* SIR2 - Local Mem addr Reg [7:0] */
-#define WL3501_NIC_LMAH ((u8)0x03)	/* SIR3 - Local Mem addr Reg [14:8] */
-#define WL3501_NIC_IODPA ((u8)0x04)	/* SIR4 - I/O Data Port A */
-#define WL3501_NIC_IODPB ((u8)0x05)	/* SIR5 - I/O Data Port B */
-#define WL3501_NIC_IODPC ((u8)0x06)	/* SIR6 - I/O Data Port C */
-#define WL3501_NIC_IODPD ((u8)0x07)	/* SIR7 - I/O Data Port D */
-
-/* Bits in GCR */
-#define WL3501_GCR_SWRESET ((u8)0x80)
-#define WL3501_GCR_CORESET ((u8)0x40)
-#define WL3501_GCR_DISPWDN ((u8)0x20)
-#define WL3501_GCR_ECWAIT  ((u8)0x10)
-#define WL3501_GCR_ECINT   ((u8)0x08)
-#define WL3501_GCR_INT2EC  ((u8)0x04)
-#define WL3501_GCR_ENECINT ((u8)0x02)
-#define WL3501_GCR_DAM     ((u8)0x01)
-
-/* Bits in BSS (Bank Switching Select Register) */
-#define WL3501_BSS_FPAGE0 ((u8)0x20)	/* Flash memory page0 */
-#define WL3501_BSS_FPAGE1 ((u8)0x28)
-#define WL3501_BSS_FPAGE2 ((u8)0x30)
-#define WL3501_BSS_FPAGE3 ((u8)0x38)
-#define WL3501_BSS_SPAGE0 ((u8)0x00)	/* SRAM page0 */
-#define WL3501_BSS_SPAGE1 ((u8)0x08)
-#define WL3501_BSS_SPAGE2 ((u8)0x10)
-#define WL3501_BSS_SPAGE3 ((u8)0x18)
-
-/* Define Driver Interface */
-/* Refer IEEE 802.11 */
-/* Tx packet header, include PLCP and MPDU */
-/* Tx PLCP Header */
-struct wl3501_80211_tx_plcp_hdr {
-	u8	sync[16];
-	u16	sfd;
-	u8	signal;
-	u8	service;
-	u16	len;
-	u16	crc16;
-} __packed;
-
-struct wl3501_80211_tx_hdr {
-	struct wl3501_80211_tx_plcp_hdr	pclp_hdr;
-	struct ieee80211_hdr		mac_hdr;
-} __packed __aligned(2);
-
-/*
-   Reserve the beginning Tx space for descriptor use.
-
-   TxBlockOffset -->	*----*----*----*----* \
-	(TxFreeDesc)	|  0 |  1 |  2 |  3 |  \
-			|  4 |  5 |  6 |  7 |   |
-			|  8 |  9 | 10 | 11 |   TX_DESC * 20
-			| 12 | 13 | 14 | 15 |   |
-			| 16 | 17 | 18 | 19 |  /
-   TxBufferBegin -->	*----*----*----*----* /
-   (TxBufferHead)	| 		    |
-   (TxBufferTail)	| 		    |
-			|    Send Buffer    |
-			| 		    |
-			|		    |
-			*-------------------*
-   TxBufferEnd    -------------------------/
-
-*/
-
-struct wl3501_card {
-	int				base_addr;
-	u8				mac_addr[ETH_ALEN];
-	spinlock_t			lock;
-	wait_queue_head_t		wait;
-	struct wl3501_get_confirm	sig_get_confirm;
-	struct wl3501_pwr_mgmt_confirm	sig_pwr_mgmt_confirm;
-	u16				tx_buffer_size;
-	u16				tx_buffer_head;
-	u16				tx_buffer_tail;
-	u16				tx_buffer_cnt;
-	u16				esbq_req_start;
-	u16				esbq_req_end;
-	u16				esbq_req_head;
-	u16				esbq_req_tail;
-	u16				esbq_confirm_start;
-	u16				esbq_confirm_end;
-	u16				esbq_confirm;
-	struct iw_mgmt_essid_pset  	essid;
-	struct iw_mgmt_essid_pset  	keep_essid;
-	u8				bssid[ETH_ALEN];
-	int				net_type;
-	char				nick[32];
-	char				card_name[32];
-	char				firmware_date[32];
-	u8				chan;
-	u8				cap_info;
-	u16				start_seg;
-	u16				bss_cnt;
-	u16				join_sta_bss;
-	u8				rssi;
-	u8				adhoc_times;
-	u8				reg_domain;
-	u8				version[2];
-	struct wl3501_scan_confirm	bss_set[20];
-
-	struct iw_statistics		wstats;
-	struct iw_spy_data		spy_data;
-	struct iw_public_data		wireless_data;
-	struct pcmcia_device		*p_dev;
-};
-#endif
diff --git a/drivers/net/wireless/legacy/wl3501_cs.c b/drivers/net/wireless/legacy/wl3501_cs.c
deleted file mode 100644
index c45c4b7cbbaf..000000000000
--- a/drivers/net/wireless/legacy/wl3501_cs.c
+++ /dev/null
@@ -1,2036 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * WL3501 Wireless LAN PCMCIA Card Driver for Linux
- * Written originally for Linux 2.0.30 by Fox Chen, mhchen@golf.ccl.itri.org.tw
- * Ported to 2.2, 2.4 & 2.5 by Arnaldo Carvalho de Melo <acme@conectiva.com.br>
- * Wireless extensions in 2.4 by Gustavo Niemeyer <niemeyer@conectiva.com>
- *
- * References used by Fox Chen while writing the original driver for 2.0.30:
- *
- *   1. WL24xx packet drivers (tooasm.asm)
- *   2. Access Point Firmware Interface Specification for IEEE 802.11 SUTRO
- *   3. IEEE 802.11
- *   4. Linux network driver (/usr/src/linux/drivers/net)
- *   5. ISA card driver - wl24.c
- *   6. Linux PCMCIA skeleton driver - skeleton.c
- *   7. Linux PCMCIA 3c589 network driver - 3c589_cs.c
- *
- * Tested with WL2400 firmware 1.2, Linux 2.0.30, and pcmcia-cs-2.9.12
- *   1. Performance: about 165 Kbytes/sec in TCP/IP with Ad-Hoc mode.
- *      rsh 192.168.1.3 "dd if=/dev/zero bs=1k count=1000" > /dev/null
- *      (Specification 2M bits/sec. is about 250 Kbytes/sec., but we must deduct
- *       ETHER/IP/UDP/TCP header, and acknowledgement overhead)
- *
- * Tested with Planet AP in 2.4.17, 184 Kbytes/s in UDP in Infrastructure mode,
- * 173 Kbytes/s in TCP.
- *
- * Tested with Planet AP in 2.5.73-bk, 216 Kbytes/s in Infrastructure mode
- * with a SMP machine (dual pentium 100), using pktgen, 432 pps (pkt_size = 60)
- */
-
-#include <linux/delay.h>
-#include <linux/types.h>
-#include <linux/interrupt.h>
-#include <linux/in.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/fcntl.h>
-#include <linux/if_arp.h>
-#include <linux/ioport.h>
-#include <linux/netdevice.h>
-#include <linux/etherdevice.h>
-#include <linux/skbuff.h>
-#include <linux/slab.h>
-#include <linux/string.h>
-#include <linux/wireless.h>
-#include <net/cfg80211.h>
-
-#include <net/iw_handler.h>
-
-#include <pcmcia/cistpl.h>
-#include <pcmcia/cisreg.h>
-#include <pcmcia/ds.h>
-
-#include <asm/io.h>
-#include <linux/uaccess.h>
-
-#include "wl3501.h"
-
-#ifndef __i386__
-#define slow_down_io()
-#endif
-
-/* For rough constant delay */
-#define WL3501_NOPLOOP(n) { int x = 0; while (x++ < n) slow_down_io(); }
-
-
-
-#define wl3501_outb(a, b) { outb(a, b); slow_down_io(); }
-#define wl3501_outb_p(a, b) { outb_p(a, b); slow_down_io(); }
-#define wl3501_outsb(a, b, c) { outsb(a, b, c); slow_down_io(); }
-
-#define WL3501_RELEASE_TIMEOUT (25 * HZ)
-#define WL3501_MAX_ADHOC_TRIES 16
-
-#define WL3501_RESUME	0
-#define WL3501_SUSPEND	1
-
-static int wl3501_config(struct pcmcia_device *link);
-static void wl3501_release(struct pcmcia_device *link);
-
-static const struct {
-	int reg_domain;
-	int min, max, deflt;
-} iw_channel_table[] = {
-	{
-		.reg_domain = IW_REG_DOMAIN_FCC,
-		.min	    = 1,
-		.max	    = 11,
-		.deflt	    = 1,
-	},
-	{
-		.reg_domain = IW_REG_DOMAIN_DOC,
-		.min	    = 1,
-		.max	    = 11,
-		.deflt	    = 1,
-	},
-	{
-		.reg_domain = IW_REG_DOMAIN_ETSI,
-		.min	    = 1,
-		.max	    = 13,
-		.deflt	    = 1,
-	},
-	{
-		.reg_domain = IW_REG_DOMAIN_SPAIN,
-		.min	    = 10,
-		.max	    = 11,
-		.deflt	    = 10,
-	},
-	{
-		.reg_domain = IW_REG_DOMAIN_FRANCE,
-		.min	    = 10,
-		.max	    = 13,
-		.deflt	    = 10,
-	},
-	{
-		.reg_domain = IW_REG_DOMAIN_MKK,
-		.min	    = 14,
-		.max	    = 14,
-		.deflt	    = 14,
-	},
-	{
-		.reg_domain = IW_REG_DOMAIN_MKK1,
-		.min	    = 1,
-		.max	    = 14,
-		.deflt	    = 1,
-	},
-	{
-		.reg_domain = IW_REG_DOMAIN_ISRAEL,
-		.min	    = 3,
-		.max	    = 9,
-		.deflt	    = 9,
-	},
-};
-
-/**
- * iw_valid_channel - validate channel in regulatory domain
- * @reg_domain: regulatory domain
- * @channel: channel to validate
- *
- * Returns 0 if invalid in the specified regulatory domain, non-zero if valid.
- */
-static int iw_valid_channel(int reg_domain, int channel)
-{
-	int i, rc = 0;
-
-	for (i = 0; i < ARRAY_SIZE(iw_channel_table); i++)
-		if (reg_domain == iw_channel_table[i].reg_domain) {
-			rc = channel >= iw_channel_table[i].min &&
-			     channel <= iw_channel_table[i].max;
-			break;
-		}
-	return rc;
-}
-
-/**
- * iw_default_channel - get default channel for a regulatory domain
- * @reg_domain: regulatory domain
- *
- * Returns the default channel for a regulatory domain
- */
-static int iw_default_channel(int reg_domain)
-{
-	int i, rc = 1;
-
-	for (i = 0; i < ARRAY_SIZE(iw_channel_table); i++)
-		if (reg_domain == iw_channel_table[i].reg_domain) {
-			rc = iw_channel_table[i].deflt;
-			break;
-		}
-	return rc;
-}
-
-static void iw_set_mgmt_info_element(enum iw_mgmt_info_element_ids id,
-				     struct iw_mgmt_info_element *el,
-				     void *value, int len)
-{
-	el->id  = id;
-	el->len = len;
-	memcpy(el->data, value, len);
-}
-
-static void iw_copy_mgmt_info_element(struct iw_mgmt_info_element *to,
-				      struct iw_mgmt_info_element *from)
-{
-	iw_set_mgmt_info_element(from->id, to, from->data, from->len);
-}
-
-static inline void wl3501_switch_page(struct wl3501_card *this, u8 page)
-{
-	wl3501_outb(page, this->base_addr + WL3501_NIC_BSS);
-}
-
-/*
- * Get Ethernet MAC address.
- *
- * WARNING: We switch to FPAGE0 and switc back again.
- *          Making sure there is no other WL function beening called by ISR.
- */
-static int wl3501_get_flash_mac_addr(struct wl3501_card *this)
-{
-	int base_addr = this->base_addr;
-
-	/* get MAC addr */
-	wl3501_outb(WL3501_BSS_FPAGE3, base_addr + WL3501_NIC_BSS); /* BSS */
-	wl3501_outb(0x00, base_addr + WL3501_NIC_LMAL);	/* LMAL */
-	wl3501_outb(0x40, base_addr + WL3501_NIC_LMAH);	/* LMAH */
-
-	/* wait for reading EEPROM */
-	WL3501_NOPLOOP(100);
-	this->mac_addr[0] = inb(base_addr + WL3501_NIC_IODPA);
-	WL3501_NOPLOOP(100);
-	this->mac_addr[1] = inb(base_addr + WL3501_NIC_IODPA);
-	WL3501_NOPLOOP(100);
-	this->mac_addr[2] = inb(base_addr + WL3501_NIC_IODPA);
-	WL3501_NOPLOOP(100);
-	this->mac_addr[3] = inb(base_addr + WL3501_NIC_IODPA);
-	WL3501_NOPLOOP(100);
-	this->mac_addr[4] = inb(base_addr + WL3501_NIC_IODPA);
-	WL3501_NOPLOOP(100);
-	this->mac_addr[5] = inb(base_addr + WL3501_NIC_IODPA);
-	WL3501_NOPLOOP(100);
-	this->reg_domain = inb(base_addr + WL3501_NIC_IODPA);
-	WL3501_NOPLOOP(100);
-	wl3501_outb(WL3501_BSS_FPAGE0, base_addr + WL3501_NIC_BSS);
-	wl3501_outb(0x04, base_addr + WL3501_NIC_LMAL);
-	wl3501_outb(0x40, base_addr + WL3501_NIC_LMAH);
-	WL3501_NOPLOOP(100);
-	this->version[0] = inb(base_addr + WL3501_NIC_IODPA);
-	WL3501_NOPLOOP(100);
-	this->version[1] = inb(base_addr + WL3501_NIC_IODPA);
-	/* switch to SRAM Page 0 (for safety) */
-	wl3501_switch_page(this, WL3501_BSS_SPAGE0);
-
-	/* The MAC addr should be 00:60:... */
-	return this->mac_addr[0] == 0x00 && this->mac_addr[1] == 0x60;
-}
-
-/**
- * wl3501_set_to_wla - Move 'size' bytes from PC to card
- * @this: Card
- * @dest: Card addressing space
- * @src: PC addressing space
- * @size: Bytes to move
- *
- * Move 'size' bytes from PC to card. (Shouldn't be interrupted)
- */
-static void wl3501_set_to_wla(struct wl3501_card *this, u16 dest, void *src,
-			      int size)
-{
-	/* switch to SRAM Page 0 */
-	wl3501_switch_page(this, (dest & 0x8000) ? WL3501_BSS_SPAGE1 :
-						   WL3501_BSS_SPAGE0);
-	/* set LMAL and LMAH */
-	wl3501_outb(dest & 0xff, this->base_addr + WL3501_NIC_LMAL);
-	wl3501_outb(((dest >> 8) & 0x7f), this->base_addr + WL3501_NIC_LMAH);
-
-	/* rep out to Port A */
-	wl3501_outsb(this->base_addr + WL3501_NIC_IODPA, src, size);
-}
-
-/**
- * wl3501_get_from_wla - Move 'size' bytes from card to PC
- * @this: Card
- * @src: Card addressing space
- * @dest: PC addressing space
- * @size: Bytes to move
- *
- * Move 'size' bytes from card to PC. (Shouldn't be interrupted)
- */
-static void wl3501_get_from_wla(struct wl3501_card *this, u16 src, void *dest,
-				int size)
-{
-	/* switch to SRAM Page 0 */
-	wl3501_switch_page(this, (src & 0x8000) ? WL3501_BSS_SPAGE1 :
-						  WL3501_BSS_SPAGE0);
-	/* set LMAL and LMAH */
-	wl3501_outb(src & 0xff, this->base_addr + WL3501_NIC_LMAL);
-	wl3501_outb((src >> 8) & 0x7f, this->base_addr + WL3501_NIC_LMAH);
-
-	/* rep get from Port A */
-	insb(this->base_addr + WL3501_NIC_IODPA, dest, size);
-}
-
-/*
- * Get/Allocate a free Tx Data Buffer
- *
- *  *--------------*-----------------*----------------------------------*
- *  |    PLCP      |    MAC Header   |  DST  SRC         Data ...       |
- *  |  (24 bytes)  |    (30 bytes)   |  (6)  (6)  (Ethernet Row Data)   |
- *  *--------------*-----------------*----------------------------------*
- *  \               \- IEEE 802.11 -/ \-------------- len --------------/
- *   \-struct wl3501_80211_tx_hdr--/   \-------- Ethernet Frame -------/
- *
- * Return = Position in Card
- */
-static u16 wl3501_get_tx_buffer(struct wl3501_card *this, u16 len)
-{
-	u16 next, blk_cnt = 0, zero = 0;
-	u16 full_len = sizeof(struct wl3501_80211_tx_hdr) + len;
-	u16 ret = 0;
-
-	if (full_len > this->tx_buffer_cnt * 254)
-		goto out;
-	ret = this->tx_buffer_head;
-	while (full_len) {
-		if (full_len < 254)
-			full_len = 0;
-		else
-			full_len -= 254;
-		wl3501_get_from_wla(this, this->tx_buffer_head, &next,
-				    sizeof(next));
-		if (!full_len)
-			wl3501_set_to_wla(this, this->tx_buffer_head, &zero,
-					  sizeof(zero));
-		this->tx_buffer_head = next;
-		blk_cnt++;
-		/* if buffer is not enough */
-		if (!next && full_len) {
-			this->tx_buffer_head = ret;
-			ret = 0;
-			goto out;
-		}
-	}
-	this->tx_buffer_cnt -= blk_cnt;
-out:
-	return ret;
-}
-
-/*
- * Free an allocated Tx Buffer. ptr must be correct position.
- */
-static void wl3501_free_tx_buffer(struct wl3501_card *this, u16 ptr)
-{
-	/* check if all space is not free */
-	if (!this->tx_buffer_head)
-		this->tx_buffer_head = ptr;
-	else
-		wl3501_set_to_wla(this, this->tx_buffer_tail,
-				  &ptr, sizeof(ptr));
-	while (ptr) {
-		u16 next;
-
-		this->tx_buffer_cnt++;
-		wl3501_get_from_wla(this, ptr, &next, sizeof(next));
-		this->tx_buffer_tail = ptr;
-		ptr = next;
-	}
-}
-
-static int wl3501_esbq_req_test(struct wl3501_card *this)
-{
-	u8 tmp = 0;
-
-	wl3501_get_from_wla(this, this->esbq_req_head + 3, &tmp, sizeof(tmp));
-	return tmp & 0x80;
-}
-
-static void wl3501_esbq_req(struct wl3501_card *this, u16 *ptr)
-{
-	u16 tmp = 0;
-
-	wl3501_set_to_wla(this, this->esbq_req_head, ptr, 2);
-	wl3501_set_to_wla(this, this->esbq_req_head + 2, &tmp, sizeof(tmp));
-	this->esbq_req_head += 4;
-	if (this->esbq_req_head >= this->esbq_req_end)
-		this->esbq_req_head = this->esbq_req_start;
-}
-
-static int wl3501_esbq_exec(struct wl3501_card *this, void *sig, int sig_size)
-{
-	int rc = -EIO;
-
-	if (wl3501_esbq_req_test(this)) {
-		u16 ptr = wl3501_get_tx_buffer(this, sig_size);
-		if (ptr) {
-			wl3501_set_to_wla(this, ptr, sig, sig_size);
-			wl3501_esbq_req(this, &ptr);
-			rc = 0;
-		}
-	}
-	return rc;
-}
-
-static int wl3501_request_mib(struct wl3501_card *this, u8 index, void *bf)
-{
-	struct wl3501_get_req sig = {
-		.sig_id	    = WL3501_SIG_GET_REQ,
-		.mib_attrib = index,
-	};
-	unsigned long flags;
-	int rc = -EIO;
-
-	spin_lock_irqsave(&this->lock, flags);
-	if (wl3501_esbq_req_test(this)) {
-		u16 ptr = wl3501_get_tx_buffer(this, sizeof(sig));
-		if (ptr) {
-			wl3501_set_to_wla(this, ptr, &sig, sizeof(sig));
-			wl3501_esbq_req(this, &ptr);
-			this->sig_get_confirm.mib_status = 255;
-			rc = 0;
-		}
-	}
-	spin_unlock_irqrestore(&this->lock, flags);
-
-	return rc;
-}
-
-static int wl3501_get_mib_value(struct wl3501_card *this, u8 index,
-				void *bf, int size)
-{
-	int rc;
-
-	rc = wl3501_request_mib(this, index, bf);
-	if (rc)
-		return rc;
-
-	rc = wait_event_interruptible(this->wait,
-		this->sig_get_confirm.mib_status != 255);
-	if (rc)
-		return rc;
-
-	memcpy(bf, this->sig_get_confirm.mib_value, size);
-	return 0;
-}
-
-static int wl3501_pwr_mgmt(struct wl3501_card *this, int suspend)
-{
-	struct wl3501_pwr_mgmt_req sig = {
-		.sig_id		= WL3501_SIG_PWR_MGMT_REQ,
-		.pwr_save	= suspend,
-		.wake_up	= !suspend,
-		.receive_dtims	= 10,
-	};
-	unsigned long flags;
-	int rc = -EIO;
-
-	spin_lock_irqsave(&this->lock, flags);
-	if (wl3501_esbq_req_test(this)) {
-		u16 ptr = wl3501_get_tx_buffer(this, sizeof(sig));
-		if (ptr) {
-			wl3501_set_to_wla(this, ptr, &sig, sizeof(sig));
-			wl3501_esbq_req(this, &ptr);
-			this->sig_pwr_mgmt_confirm.status = 255;
-			spin_unlock_irqrestore(&this->lock, flags);
-			rc = wait_event_interruptible(this->wait,
-				this->sig_pwr_mgmt_confirm.status != 255);
-			printk(KERN_INFO "%s: %s status=%d\n", __func__,
-			       suspend ? "suspend" : "resume",
-			       this->sig_pwr_mgmt_confirm.status);
-			goto out;
-		}
-	}
-	spin_unlock_irqrestore(&this->lock, flags);
-out:
-	return rc;
-}
-
-/**
- * wl3501_send_pkt - Send a packet.
- * @this: Card
- * @data: Ethernet raw frame.  (e.g. data[0] - data[5] is Dest MAC Addr,
- *                                   data[6] - data[11] is Src MAC Addr)
- * @len: Packet length
- * Ref: IEEE 802.11
- */
-static int wl3501_send_pkt(struct wl3501_card *this, u8 *data, u16 len)
-{
-	u16 bf, sig_bf, next, tmplen, pktlen;
-	struct wl3501_md_req sig = {
-		.sig_id = WL3501_SIG_MD_REQ,
-	};
-	size_t sig_addr_len = sizeof(sig.addr);
-	u8 *pdata = (char *)data;
-	int rc = -EIO;
-
-	if (wl3501_esbq_req_test(this)) {
-		sig_bf = wl3501_get_tx_buffer(this, sizeof(sig));
-		rc = -ENOMEM;
-		if (!sig_bf)	/* No free buffer available */
-			goto out;
-		bf = wl3501_get_tx_buffer(this, len + 26 + 24);
-		if (!bf) {
-			/* No free buffer available */
-			wl3501_free_tx_buffer(this, sig_bf);
-			goto out;
-		}
-		rc = 0;
-		memcpy(&sig.addr, pdata, sig_addr_len);
-		pktlen = len - sig_addr_len;
-		pdata += sig_addr_len;
-		sig.data = bf;
-		if (((*pdata) * 256 + (*(pdata + 1))) > 1500) {
-			u8 addr4[ETH_ALEN] = {
-				[0] = 0xAA, [1] = 0xAA, [2] = 0x03, [4] = 0x00,
-			};
-
-			wl3501_set_to_wla(this, bf + 2 +
-					  offsetof(struct wl3501_tx_hdr, addr4),
-					  addr4, sizeof(addr4));
-			sig.size = pktlen + 24 + 4 + 6;
-			if (pktlen > (254 - sizeof(struct wl3501_tx_hdr))) {
-				tmplen = 254 - sizeof(struct wl3501_tx_hdr);
-				pktlen -= tmplen;
-			} else {
-				tmplen = pktlen;
-				pktlen = 0;
-			}
-			wl3501_set_to_wla(this,
-					  bf + 2 + sizeof(struct wl3501_tx_hdr),
-					  pdata, tmplen);
-			pdata += tmplen;
-			wl3501_get_from_wla(this, bf, &next, sizeof(next));
-			bf = next;
-		} else {
-			sig.size = pktlen + 24 + 4 - 2;
-			pdata += 2;
-			pktlen -= 2;
-			if (pktlen > (254 - sizeof(struct wl3501_tx_hdr) + 6)) {
-				tmplen = 254 - sizeof(struct wl3501_tx_hdr) + 6;
-				pktlen -= tmplen;
-			} else {
-				tmplen = pktlen;
-				pktlen = 0;
-			}
-			wl3501_set_to_wla(this, bf + 2 +
-					  offsetof(struct wl3501_tx_hdr, addr4),
-					  pdata, tmplen);
-			pdata += tmplen;
-			wl3501_get_from_wla(this, bf, &next, sizeof(next));
-			bf = next;
-		}
-		while (pktlen > 0) {
-			if (pktlen > 254) {
-				tmplen = 254;
-				pktlen -= 254;
-			} else {
-				tmplen = pktlen;
-				pktlen = 0;
-			}
-			wl3501_set_to_wla(this, bf + 2, pdata, tmplen);
-			pdata += tmplen;
-			wl3501_get_from_wla(this, bf, &next, sizeof(next));
-			bf = next;
-		}
-		wl3501_set_to_wla(this, sig_bf, &sig, sizeof(sig));
-		wl3501_esbq_req(this, &sig_bf);
-	}
-out:
-	return rc;
-}
-
-static int wl3501_mgmt_resync(struct wl3501_card *this)
-{
-	struct wl3501_resync_req sig = {
-		.sig_id = WL3501_SIG_RESYNC_REQ,
-	};
-
-	return wl3501_esbq_exec(this, &sig, sizeof(sig));
-}
-
-static inline int wl3501_fw_bss_type(struct wl3501_card *this)
-{
-	return this->net_type == IW_MODE_INFRA ? WL3501_NET_TYPE_INFRA :
-						 WL3501_NET_TYPE_ADHOC;
-}
-
-static inline int wl3501_fw_cap_info(struct wl3501_card *this)
-{
-	return this->net_type == IW_MODE_INFRA ? WL3501_MGMT_CAPABILITY_ESS :
-						 WL3501_MGMT_CAPABILITY_IBSS;
-}
-
-static int wl3501_mgmt_scan(struct wl3501_card *this, u16 chan_time)
-{
-	struct wl3501_scan_req sig = {
-		.sig_id		= WL3501_SIG_SCAN_REQ,
-		.scan_type	= WL3501_SCAN_TYPE_ACTIVE,
-		.probe_delay	= 0x10,
-		.min_chan_time	= chan_time,
-		.max_chan_time	= chan_time,
-		.bss_type	= wl3501_fw_bss_type(this),
-	};
-
-	this->bss_cnt = this->join_sta_bss = 0;
-	return wl3501_esbq_exec(this, &sig, sizeof(sig));
-}
-
-static int wl3501_mgmt_join(struct wl3501_card *this, u16 stas)
-{
-	struct wl3501_join_req sig = {
-		.sig_id		  = WL3501_SIG_JOIN_REQ,
-		.timeout	  = 10,
-		.req.ds_pset = {
-			.el = {
-				.id  = IW_MGMT_INFO_ELEMENT_DS_PARAMETER_SET,
-				.len = 1,
-			},
-			.chan	= this->chan,
-		},
-	};
-
-	memcpy(&sig.req, &this->bss_set[stas].req, sizeof(sig.req));
-	return wl3501_esbq_exec(this, &sig, sizeof(sig));
-}
-
-static int wl3501_mgmt_start(struct wl3501_card *this)
-{
-	struct wl3501_start_req sig = {
-		.sig_id			= WL3501_SIG_START_REQ,
-		.beacon_period		= 400,
-		.dtim_period		= 1,
-		.ds_pset = {
-			.el = {
-				.id  = IW_MGMT_INFO_ELEMENT_DS_PARAMETER_SET,
-				.len = 1,
-			},
-			.chan	= this->chan,
-		},
-		.bss_basic_rset	= {
-			.el = {
-				.id	= IW_MGMT_INFO_ELEMENT_SUPPORTED_RATES,
-				.len = 2,
-			},
-			.data_rate_labels = {
-				[0] = IW_MGMT_RATE_LABEL_MANDATORY |
-				      IW_MGMT_RATE_LABEL_1MBIT,
-				[1] = IW_MGMT_RATE_LABEL_MANDATORY |
-				      IW_MGMT_RATE_LABEL_2MBIT,
-			},
-		},
-		.operational_rset	= {
-			.el = {
-				.id	= IW_MGMT_INFO_ELEMENT_SUPPORTED_RATES,
-				.len = 2,
-			},
-			.data_rate_labels = {
-				[0] = IW_MGMT_RATE_LABEL_MANDATORY |
-				      IW_MGMT_RATE_LABEL_1MBIT,
-				[1] = IW_MGMT_RATE_LABEL_MANDATORY |
-				      IW_MGMT_RATE_LABEL_2MBIT,
-			},
-		},
-		.ibss_pset		= {
-			.el = {
-				.id	 = IW_MGMT_INFO_ELEMENT_IBSS_PARAMETER_SET,
-				.len     = 2,
-			},
-			.atim_window = 10,
-		},
-		.bss_type		= wl3501_fw_bss_type(this),
-		.cap_info		= wl3501_fw_cap_info(this),
-	};
-
-	iw_copy_mgmt_info_element(&sig.ssid.el, &this->essid.el);
-	iw_copy_mgmt_info_element(&this->keep_essid.el, &this->essid.el);
-	return wl3501_esbq_exec(this, &sig, sizeof(sig));
-}
-
-static void wl3501_mgmt_scan_confirm(struct wl3501_card *this, u16 addr)
-{
-	u16 i = 0;
-	int matchflag = 0;
-	struct wl3501_scan_confirm sig;
-
-	pr_debug("entry");
-	wl3501_get_from_wla(this, addr, &sig, sizeof(sig));
-	if (sig.status == WL3501_STATUS_SUCCESS) {
-		pr_debug("success");
-		if ((this->net_type == IW_MODE_INFRA &&
-		     (sig.req.cap_info & WL3501_MGMT_CAPABILITY_ESS)) ||
-		    (this->net_type == IW_MODE_ADHOC &&
-		     (sig.req.cap_info & WL3501_MGMT_CAPABILITY_IBSS)) ||
-		    this->net_type == IW_MODE_AUTO) {
-			if (!this->essid.el.len)
-				matchflag = 1;
-			else if (this->essid.el.len == 3 &&
-				 !memcmp(this->essid.essid, "ANY", 3))
-				matchflag = 1;
-			else if (this->essid.el.len != sig.req.ssid.el.len)
-				matchflag = 0;
-			else if (memcmp(this->essid.essid, sig.req.ssid.essid,
-					this->essid.el.len))
-				matchflag = 0;
-			else
-				matchflag = 1;
-			if (matchflag) {
-				for (i = 0; i < this->bss_cnt; i++) {
-					if (ether_addr_equal_unaligned(this->bss_set[i].req.bssid,
-								       sig.req.bssid)) {
-						matchflag = 0;
-						break;
-					}
-				}
-			}
-			if (matchflag && (i < 20)) {
-				memcpy(&this->bss_set[i].req,
-				       &sig.req, sizeof(sig.req));
-				this->bss_cnt++;
-				this->rssi = sig.rssi;
-				this->bss_set[i].rssi = sig.rssi;
-			}
-		}
-	} else if (sig.status == WL3501_STATUS_TIMEOUT) {
-		pr_debug("timeout");
-		this->join_sta_bss = 0;
-		for (i = this->join_sta_bss; i < this->bss_cnt; i++)
-			if (!wl3501_mgmt_join(this, i))
-				break;
-		this->join_sta_bss = i;
-		if (this->join_sta_bss == this->bss_cnt) {
-			if (this->net_type == IW_MODE_INFRA)
-				wl3501_mgmt_scan(this, 100);
-			else {
-				this->adhoc_times++;
-				if (this->adhoc_times > WL3501_MAX_ADHOC_TRIES)
-					wl3501_mgmt_start(this);
-				else
-					wl3501_mgmt_scan(this, 100);
-			}
-		}
-	}
-}
-
-/**
- * wl3501_block_interrupt - Mask interrupt from SUTRO
- * @this: Card
- *
- * Mask interrupt from SUTRO. (i.e. SUTRO cannot interrupt the HOST)
- * Return: 1 if interrupt is originally enabled
- */
-static int wl3501_block_interrupt(struct wl3501_card *this)
-{
-	u8 old = inb(this->base_addr + WL3501_NIC_GCR);
-	u8 new = old & (~(WL3501_GCR_ECINT | WL3501_GCR_INT2EC |
-			WL3501_GCR_ENECINT));
-
-	wl3501_outb(new, this->base_addr + WL3501_NIC_GCR);
-	return old & WL3501_GCR_ENECINT;
-}
-
-/**
- * wl3501_unblock_interrupt - Enable interrupt from SUTRO
- * @this: Card
- *
- * Enable interrupt from SUTRO. (i.e. SUTRO can interrupt the HOST)
- * Return: 1 if interrupt is originally enabled
- */
-static int wl3501_unblock_interrupt(struct wl3501_card *this)
-{
-	u8 old = inb(this->base_addr + WL3501_NIC_GCR);
-	u8 new = (old & ~(WL3501_GCR_ECINT | WL3501_GCR_INT2EC)) |
-		  WL3501_GCR_ENECINT;
-
-	wl3501_outb(new, this->base_addr + WL3501_NIC_GCR);
-	return old & WL3501_GCR_ENECINT;
-}
-
-/**
- * wl3501_receive - Receive data from Receive Queue.
- *
- * Receive data from Receive Queue.
- *
- * @this: card
- * @bf: address of host
- * @size: size of buffer.
- */
-static u16 wl3501_receive(struct wl3501_card *this, u8 *bf, u16 size)
-{
-	u16 next_addr, next_addr1;
-	u8 *data = bf + 12;
-
-	size -= 12;
-	wl3501_get_from_wla(this, this->start_seg + 2,
-			    &next_addr, sizeof(next_addr));
-	if (size > WL3501_BLKSZ - sizeof(struct wl3501_rx_hdr)) {
-		wl3501_get_from_wla(this,
-				    this->start_seg +
-					sizeof(struct wl3501_rx_hdr), data,
-				    WL3501_BLKSZ -
-					sizeof(struct wl3501_rx_hdr));
-		size -= WL3501_BLKSZ - sizeof(struct wl3501_rx_hdr);
-		data += WL3501_BLKSZ - sizeof(struct wl3501_rx_hdr);
-	} else {
-		wl3501_get_from_wla(this,
-				    this->start_seg +
-					sizeof(struct wl3501_rx_hdr),
-				    data, size);
-		size = 0;
-	}
-	while (size > 0) {
-		if (size > WL3501_BLKSZ - 5) {
-			wl3501_get_from_wla(this, next_addr + 5, data,
-					    WL3501_BLKSZ - 5);
-			size -= WL3501_BLKSZ - 5;
-			data += WL3501_BLKSZ - 5;
-			wl3501_get_from_wla(this, next_addr + 2, &next_addr1,
-					    sizeof(next_addr1));
-			next_addr = next_addr1;
-		} else {
-			wl3501_get_from_wla(this, next_addr + 5, data, size);
-			size = 0;
-		}
-	}
-	return 0;
-}
-
-static void wl3501_esbq_req_free(struct wl3501_card *this)
-{
-	u8 tmp;
-	u16 addr;
-
-	if (this->esbq_req_head == this->esbq_req_tail)
-		goto out;
-	wl3501_get_from_wla(this, this->esbq_req_tail + 3, &tmp, sizeof(tmp));
-	if (!(tmp & 0x80))
-		goto out;
-	wl3501_get_from_wla(this, this->esbq_req_tail, &addr, sizeof(addr));
-	wl3501_free_tx_buffer(this, addr);
-	this->esbq_req_tail += 4;
-	if (this->esbq_req_tail >= this->esbq_req_end)
-		this->esbq_req_tail = this->esbq_req_start;
-out:
-	return;
-}
-
-static int wl3501_esbq_confirm(struct wl3501_card *this)
-{
-	u8 tmp;
-
-	wl3501_get_from_wla(this, this->esbq_confirm + 3, &tmp, sizeof(tmp));
-	return tmp & 0x80;
-}
-
-static void wl3501_online(struct net_device *dev)
-{
-	struct wl3501_card *this = netdev_priv(dev);
-
-	printk(KERN_INFO "%s: Wireless LAN online. BSSID: %pM\n",
-	       dev->name, this->bssid);
-	netif_wake_queue(dev);
-}
-
-static void wl3501_esbq_confirm_done(struct wl3501_card *this)
-{
-	u8 tmp = 0;
-
-	wl3501_set_to_wla(this, this->esbq_confirm + 3, &tmp, sizeof(tmp));
-	this->esbq_confirm += 4;
-	if (this->esbq_confirm >= this->esbq_confirm_end)
-		this->esbq_confirm = this->esbq_confirm_start;
-}
-
-static int wl3501_mgmt_auth(struct wl3501_card *this)
-{
-	struct wl3501_auth_req sig = {
-		.sig_id	 = WL3501_SIG_AUTH_REQ,
-		.type	 = WL3501_SYS_TYPE_OPEN,
-		.timeout = 1000,
-	};
-
-	pr_debug("entry");
-	memcpy(sig.mac_addr, this->bssid, ETH_ALEN);
-	return wl3501_esbq_exec(this, &sig, sizeof(sig));
-}
-
-static int wl3501_mgmt_association(struct wl3501_card *this)
-{
-	struct wl3501_assoc_req sig = {
-		.sig_id		 = WL3501_SIG_ASSOC_REQ,
-		.timeout	 = 1000,
-		.listen_interval = 5,
-		.cap_info	 = this->cap_info,
-	};
-
-	pr_debug("entry");
-	memcpy(sig.mac_addr, this->bssid, ETH_ALEN);
-	return wl3501_esbq_exec(this, &sig, sizeof(sig));
-}
-
-static void wl3501_mgmt_join_confirm(struct net_device *dev, u16 addr)
-{
-	struct wl3501_card *this = netdev_priv(dev);
-	struct wl3501_join_confirm sig;
-
-	pr_debug("entry");
-	wl3501_get_from_wla(this, addr, &sig, sizeof(sig));
-	if (sig.status == WL3501_STATUS_SUCCESS) {
-		if (this->net_type == IW_MODE_INFRA) {
-			if (this->join_sta_bss < this->bss_cnt) {
-				const int i = this->join_sta_bss;
-				memcpy(this->bssid,
-				       this->bss_set[i].req.bssid, ETH_ALEN);
-				this->chan = this->bss_set[i].req.ds_pset.chan;
-				iw_copy_mgmt_info_element(&this->keep_essid.el,
-						     &this->bss_set[i].req.ssid.el);
-				wl3501_mgmt_auth(this);
-			}
-		} else {
-			const int i = this->join_sta_bss;
-
-			memcpy(&this->bssid, &this->bss_set[i].req.bssid, ETH_ALEN);
-			this->chan = this->bss_set[i].req.ds_pset.chan;
-			iw_copy_mgmt_info_element(&this->keep_essid.el,
-						  &this->bss_set[i].req.ssid.el);
-			wl3501_online(dev);
-		}
-	} else {
-		int i;
-		this->join_sta_bss++;
-		for (i = this->join_sta_bss; i < this->bss_cnt; i++)
-			if (!wl3501_mgmt_join(this, i))
-				break;
-		this->join_sta_bss = i;
-		if (this->join_sta_bss == this->bss_cnt) {
-			if (this->net_type == IW_MODE_INFRA)
-				wl3501_mgmt_scan(this, 100);
-			else {
-				this->adhoc_times++;
-				if (this->adhoc_times > WL3501_MAX_ADHOC_TRIES)
-					wl3501_mgmt_start(this);
-				else
-					wl3501_mgmt_scan(this, 100);
-			}
-		}
-	}
-}
-
-static inline void wl3501_alarm_interrupt(struct net_device *dev,
-					  struct wl3501_card *this)
-{
-	if (this->net_type == IW_MODE_INFRA) {
-		printk(KERN_INFO "Wireless LAN offline\n");
-		netif_stop_queue(dev);
-		wl3501_mgmt_resync(this);
-	}
-}
-
-static inline void wl3501_md_confirm_interrupt(struct net_device *dev,
-					       struct wl3501_card *this,
-					       u16 addr)
-{
-	struct wl3501_md_confirm sig;
-
-	pr_debug("entry");
-	wl3501_get_from_wla(this, addr, &sig, sizeof(sig));
-	wl3501_free_tx_buffer(this, sig.data);
-	if (netif_queue_stopped(dev))
-		netif_wake_queue(dev);
-}
-
-static inline void wl3501_md_ind_interrupt(struct net_device *dev,
-					   struct wl3501_card *this, u16 addr)
-{
-	struct wl3501_md_ind sig;
-	struct sk_buff *skb;
-	u8 rssi, addr4[ETH_ALEN];
-	u16 pkt_len;
-
-	wl3501_get_from_wla(this, addr, &sig, sizeof(sig));
-	this->start_seg = sig.data;
-	wl3501_get_from_wla(this,
-			    sig.data + offsetof(struct wl3501_rx_hdr, rssi),
-			    &rssi, sizeof(rssi));
-	this->rssi = rssi <= 63 ? (rssi * 100) / 64 : 255;
-
-	wl3501_get_from_wla(this,
-			    sig.data +
-				offsetof(struct wl3501_rx_hdr, addr4),
-			    &addr4, sizeof(addr4));
-	if (!(addr4[0] == 0xAA && addr4[1] == 0xAA &&
-	      addr4[2] == 0x03 && addr4[4] == 0x00)) {
-		printk(KERN_INFO "Unsupported packet type!\n");
-		return;
-	}
-	pkt_len = sig.size + 12 - 24 - 4 - 6;
-
-	skb = dev_alloc_skb(pkt_len + 5);
-
-	if (!skb) {
-		printk(KERN_WARNING "%s: Can't alloc a sk_buff of size %d.\n",
-		       dev->name, pkt_len);
-		dev->stats.rx_dropped++;
-	} else {
-		skb->dev = dev;
-		skb_reserve(skb, 2); /* IP headers on 16 bytes boundaries */
-		skb_copy_to_linear_data(skb, (unsigned char *)&sig.addr,
-					sizeof(sig.addr));
-		wl3501_receive(this, skb->data, pkt_len);
-		skb_put(skb, pkt_len);
-		skb->protocol	= eth_type_trans(skb, dev);
-		dev->stats.rx_packets++;
-		dev->stats.rx_bytes += skb->len;
-		netif_rx(skb);
-	}
-}
-
-static inline void wl3501_get_confirm_interrupt(struct wl3501_card *this,
-						u16 addr, void *sig, int size)
-{
-	pr_debug("entry");
-	wl3501_get_from_wla(this, addr, &this->sig_get_confirm,
-			    sizeof(this->sig_get_confirm));
-	wake_up(&this->wait);
-}
-
-static inline void wl3501_start_confirm_interrupt(struct net_device *dev,
-						  struct wl3501_card *this,
-						  u16 addr)
-{
-	struct wl3501_start_confirm sig;
-
-	pr_debug("entry");
-	wl3501_get_from_wla(this, addr, &sig, sizeof(sig));
-	if (sig.status == WL3501_STATUS_SUCCESS)
-		netif_wake_queue(dev);
-}
-
-static inline void wl3501_assoc_confirm_interrupt(struct net_device *dev,
-						  u16 addr)
-{
-	struct wl3501_card *this = netdev_priv(dev);
-	struct wl3501_assoc_confirm sig;
-
-	pr_debug("entry");
-	wl3501_get_from_wla(this, addr, &sig, sizeof(sig));
-
-	if (sig.status == WL3501_STATUS_SUCCESS)
-		wl3501_online(dev);
-}
-
-static inline void wl3501_auth_confirm_interrupt(struct wl3501_card *this,
-						 u16 addr)
-{
-	struct wl3501_auth_confirm sig;
-
-	pr_debug("entry");
-	wl3501_get_from_wla(this, addr, &sig, sizeof(sig));
-
-	if (sig.status == WL3501_STATUS_SUCCESS)
-		wl3501_mgmt_association(this);
-	else
-		wl3501_mgmt_resync(this);
-}
-
-static inline void wl3501_rx_interrupt(struct net_device *dev)
-{
-	int morepkts;
-	u16 addr;
-	u8 sig_id;
-	struct wl3501_card *this = netdev_priv(dev);
-
-	pr_debug("entry");
-loop:
-	morepkts = 0;
-	if (!wl3501_esbq_confirm(this))
-		goto free;
-	wl3501_get_from_wla(this, this->esbq_confirm, &addr, sizeof(addr));
-	wl3501_get_from_wla(this, addr + 2, &sig_id, sizeof(sig_id));
-
-	switch (sig_id) {
-	case WL3501_SIG_DEAUTH_IND:
-	case WL3501_SIG_DISASSOC_IND:
-	case WL3501_SIG_ALARM:
-		wl3501_alarm_interrupt(dev, this);
-		break;
-	case WL3501_SIG_MD_CONFIRM:
-		wl3501_md_confirm_interrupt(dev, this, addr);
-		break;
-	case WL3501_SIG_MD_IND:
-		wl3501_md_ind_interrupt(dev, this, addr);
-		break;
-	case WL3501_SIG_GET_CONFIRM:
-		wl3501_get_confirm_interrupt(this, addr,
-					     &this->sig_get_confirm,
-					     sizeof(this->sig_get_confirm));
-		break;
-	case WL3501_SIG_PWR_MGMT_CONFIRM:
-		wl3501_get_confirm_interrupt(this, addr,
-					     &this->sig_pwr_mgmt_confirm,
-					    sizeof(this->sig_pwr_mgmt_confirm));
-		break;
-	case WL3501_SIG_START_CONFIRM:
-		wl3501_start_confirm_interrupt(dev, this, addr);
-		break;
-	case WL3501_SIG_SCAN_CONFIRM:
-		wl3501_mgmt_scan_confirm(this, addr);
-		break;
-	case WL3501_SIG_JOIN_CONFIRM:
-		wl3501_mgmt_join_confirm(dev, addr);
-		break;
-	case WL3501_SIG_ASSOC_CONFIRM:
-		wl3501_assoc_confirm_interrupt(dev, addr);
-		break;
-	case WL3501_SIG_AUTH_CONFIRM:
-		wl3501_auth_confirm_interrupt(this, addr);
-		break;
-	case WL3501_SIG_RESYNC_CONFIRM:
-		wl3501_mgmt_resync(this); /* FIXME: should be resync_confirm */
-		break;
-	}
-	wl3501_esbq_confirm_done(this);
-	morepkts = 1;
-	/* free request if necessary */
-free:
-	wl3501_esbq_req_free(this);
-	if (morepkts)
-		goto loop;
-}
-
-static inline void wl3501_ack_interrupt(struct wl3501_card *this)
-{
-	wl3501_outb(WL3501_GCR_ECINT, this->base_addr + WL3501_NIC_GCR);
-}
-
-/**
- * wl3501_interrupt - Hardware interrupt from card.
- * @irq: Interrupt number
- * @dev_id: net_device
- *
- * We must acknowledge the interrupt as soon as possible, and block the
- * interrupt from the same card immediately to prevent re-entry.
- *
- * Before accessing the Control_Status_Block, we must lock SUTRO first.
- * On the other hand, to prevent SUTRO from malfunctioning, we must
- * unlock the SUTRO as soon as possible.
- */
-static irqreturn_t wl3501_interrupt(int irq, void *dev_id)
-{
-	struct net_device *dev = dev_id;
-	struct wl3501_card *this;
-
-	this = netdev_priv(dev);
-	spin_lock(&this->lock);
-	wl3501_ack_interrupt(this);
-	wl3501_block_interrupt(this);
-	wl3501_rx_interrupt(dev);
-	wl3501_unblock_interrupt(this);
-	spin_unlock(&this->lock);
-
-	return IRQ_HANDLED;
-}
-
-static int wl3501_reset_board(struct wl3501_card *this)
-{
-	u8 tmp = 0;
-	int i, rc = 0;
-
-	/* Coreset */
-	wl3501_outb_p(WL3501_GCR_CORESET, this->base_addr + WL3501_NIC_GCR);
-	wl3501_outb_p(0, this->base_addr + WL3501_NIC_GCR);
-	wl3501_outb_p(WL3501_GCR_CORESET, this->base_addr + WL3501_NIC_GCR);
-
-	/* Reset SRAM 0x480 to zero */
-	wl3501_set_to_wla(this, 0x480, &tmp, sizeof(tmp));
-
-	/* Start up */
-	wl3501_outb_p(0, this->base_addr + WL3501_NIC_GCR);
-
-	WL3501_NOPLOOP(1024 * 50);
-
-	wl3501_unblock_interrupt(this);	/* acme: was commented */
-
-	/* Polling Self_Test_Status */
-	for (i = 0; i < 10000; i++) {
-		wl3501_get_from_wla(this, 0x480, &tmp, sizeof(tmp));
-
-		if (tmp == 'W') {
-			/* firmware complete all test successfully */
-			tmp = 'A';
-			wl3501_set_to_wla(this, 0x480, &tmp, sizeof(tmp));
-			goto out;
-		}
-		WL3501_NOPLOOP(10);
-	}
-	printk(KERN_WARNING "%s: failed to reset the board!\n", __func__);
-	rc = -ENODEV;
-out:
-	return rc;
-}
-
-static int wl3501_init_firmware(struct wl3501_card *this)
-{
-	u16 ptr, next;
-	int rc = wl3501_reset_board(this);
-
-	if (rc)
-		goto fail;
-	this->card_name[0] = '\0';
-	wl3501_get_from_wla(this, 0x1a00,
-			    this->card_name, sizeof(this->card_name));
-	this->card_name[sizeof(this->card_name) - 1] = '\0';
-	this->firmware_date[0] = '\0';
-	wl3501_get_from_wla(this, 0x1a40,
-			    this->firmware_date, sizeof(this->firmware_date));
-	this->firmware_date[sizeof(this->firmware_date) - 1] = '\0';
-	/* Switch to SRAM Page 0 */
-	wl3501_switch_page(this, WL3501_BSS_SPAGE0);
-	/* Read parameter from card */
-	wl3501_get_from_wla(this, 0x482, &this->esbq_req_start, 2);
-	wl3501_get_from_wla(this, 0x486, &this->esbq_req_end, 2);
-	wl3501_get_from_wla(this, 0x488, &this->esbq_confirm_start, 2);
-	wl3501_get_from_wla(this, 0x48c, &this->esbq_confirm_end, 2);
-	wl3501_get_from_wla(this, 0x48e, &this->tx_buffer_head, 2);
-	wl3501_get_from_wla(this, 0x492, &this->tx_buffer_size, 2);
-	this->esbq_req_tail	= this->esbq_req_head = this->esbq_req_start;
-	this->esbq_req_end     += this->esbq_req_start;
-	this->esbq_confirm	= this->esbq_confirm_start;
-	this->esbq_confirm_end += this->esbq_confirm_start;
-	/* Initial Tx Buffer */
-	this->tx_buffer_cnt = 1;
-	ptr = this->tx_buffer_head;
-	next = ptr + WL3501_BLKSZ;
-	while ((next - this->tx_buffer_head) < this->tx_buffer_size) {
-		this->tx_buffer_cnt++;
-		wl3501_set_to_wla(this, ptr, &next, sizeof(next));
-		ptr = next;
-		next = ptr + WL3501_BLKSZ;
-	}
-	rc = 0;
-	next = 0;
-	wl3501_set_to_wla(this, ptr, &next, sizeof(next));
-	this->tx_buffer_tail = ptr;
-out:
-	return rc;
-fail:
-	printk(KERN_WARNING "%s: failed!\n", __func__);
-	goto out;
-}
-
-static int wl3501_close(struct net_device *dev)
-{
-	struct wl3501_card *this = netdev_priv(dev);
-	unsigned long flags;
-	struct pcmcia_device *link;
-	link = this->p_dev;
-
-	spin_lock_irqsave(&this->lock, flags);
-	link->open--;
-
-	/* Stop wl3501_hard_start_xmit() from now on */
-	netif_stop_queue(dev);
-	wl3501_ack_interrupt(this);
-
-	/* Mask interrupts from the SUTRO */
-	wl3501_block_interrupt(this);
-
-	printk(KERN_INFO "%s: WL3501 closed\n", dev->name);
-	spin_unlock_irqrestore(&this->lock, flags);
-	return 0;
-}
-
-/**
- * wl3501_reset - Reset the SUTRO.
- * @dev: network device
- *
- * It is almost the same as wl3501_open(). In fact, we may just wl3501_close()
- * and wl3501_open() again, but I wouldn't like to free_irq() when the driver
- * is running. It seems to be dangerous.
- */
-static int wl3501_reset(struct net_device *dev)
-{
-	struct wl3501_card *this = netdev_priv(dev);
-	int rc = -ENODEV;
-	unsigned long flags;
-
-	spin_lock_irqsave(&this->lock, flags);
-	wl3501_block_interrupt(this);
-
-	if (wl3501_init_firmware(this)) {
-		printk(KERN_WARNING "%s: Can't initialize Firmware!\n",
-		       dev->name);
-		/* Free IRQ, and mark IRQ as unused */
-		free_irq(dev->irq, dev);
-		goto out;
-	}
-
-	/*
-	 * Queue has to be started only when the Card is Started
-	 */
-	netif_stop_queue(dev);
-	this->adhoc_times = 0;
-	wl3501_ack_interrupt(this);
-	wl3501_unblock_interrupt(this);
-	wl3501_mgmt_scan(this, 100);
-	pr_debug("%s: device reset", dev->name);
-	rc = 0;
-out:
-	spin_unlock_irqrestore(&this->lock, flags);
-	return rc;
-}
-
-static void wl3501_tx_timeout(struct net_device *dev, unsigned int txqueue)
-{
-	struct net_device_stats *stats = &dev->stats;
-	int rc;
-
-	stats->tx_errors++;
-	rc = wl3501_reset(dev);
-	if (rc)
-		printk(KERN_ERR "%s: Error %d resetting card on Tx timeout!\n",
-		       dev->name, rc);
-	else {
-		netif_trans_update(dev); /* prevent tx timeout */
-		netif_wake_queue(dev);
-	}
-}
-
-/*
- * Return : 0 - OK
- *	    1 - Could not transmit (dev_queue_xmit will queue it)
- *		and try to sent it later
- */
-static netdev_tx_t wl3501_hard_start_xmit(struct sk_buff *skb,
-						struct net_device *dev)
-{
-	int enabled, rc;
-	struct wl3501_card *this = netdev_priv(dev);
-	unsigned long flags;
-
-	spin_lock_irqsave(&this->lock, flags);
-	enabled = wl3501_block_interrupt(this);
-	rc = wl3501_send_pkt(this, skb->data, skb->len);
-	if (enabled)
-		wl3501_unblock_interrupt(this);
-	if (rc) {
-		++dev->stats.tx_dropped;
-		netif_stop_queue(dev);
-	} else {
-		++dev->stats.tx_packets;
-		dev->stats.tx_bytes += skb->len;
-		dev_kfree_skb_irq(skb);
-
-		if (this->tx_buffer_cnt < 2)
-			netif_stop_queue(dev);
-	}
-	spin_unlock_irqrestore(&this->lock, flags);
-	return NETDEV_TX_OK;
-}
-
-static int wl3501_open(struct net_device *dev)
-{
-	int rc = -ENODEV;
-	struct wl3501_card *this = netdev_priv(dev);
-	unsigned long flags;
-	struct pcmcia_device *link;
-	link = this->p_dev;
-
-	spin_lock_irqsave(&this->lock, flags);
-	if (!pcmcia_dev_present(link))
-		goto out;
-	netif_device_attach(dev);
-	link->open++;
-
-	/* Initial WL3501 firmware */
-	pr_debug("%s: Initialize WL3501 firmware...", dev->name);
-	if (wl3501_init_firmware(this))
-		goto fail;
-	/* Initial device variables */
-	this->adhoc_times = 0;
-	/* Acknowledge Interrupt, for cleaning last state */
-	wl3501_ack_interrupt(this);
-
-	/* Enable interrupt from card after all */
-	wl3501_unblock_interrupt(this);
-	wl3501_mgmt_scan(this, 100);
-	rc = 0;
-	pr_debug("%s: WL3501 opened", dev->name);
-	printk(KERN_INFO "%s: Card Name: %s\n"
-			 "%s: Firmware Date: %s\n",
-			 dev->name, this->card_name,
-			 dev->name, this->firmware_date);
-out:
-	spin_unlock_irqrestore(&this->lock, flags);
-	return rc;
-fail:
-	printk(KERN_WARNING "%s: Can't initialize firmware!\n", dev->name);
-	goto out;
-}
-
-static struct iw_statistics *wl3501_get_wireless_stats(struct net_device *dev)
-{
-	struct wl3501_card *this = netdev_priv(dev);
-	struct iw_statistics *wstats = &this->wstats;
-	u32 value; /* size checked: it is u32 */
-
-	memset(wstats, 0, sizeof(*wstats));
-	wstats->status = netif_running(dev);
-	if (!wl3501_get_mib_value(this, WL3501_MIB_ATTR_WEP_ICV_ERROR_COUNT,
-				  &value, sizeof(value)))
-		wstats->discard.code += value;
-	if (!wl3501_get_mib_value(this, WL3501_MIB_ATTR_WEP_UNDECRYPTABLE_COUNT,
-				  &value, sizeof(value)))
-		wstats->discard.code += value;
-	if (!wl3501_get_mib_value(this, WL3501_MIB_ATTR_WEP_EXCLUDED_COUNT,
-				  &value, sizeof(value)))
-		wstats->discard.code += value;
-	if (!wl3501_get_mib_value(this, WL3501_MIB_ATTR_RETRY_COUNT,
-				  &value, sizeof(value)))
-		wstats->discard.retries	= value;
-	if (!wl3501_get_mib_value(this, WL3501_MIB_ATTR_FAILED_COUNT,
-				  &value, sizeof(value)))
-		wstats->discard.misc += value;
-	if (!wl3501_get_mib_value(this, WL3501_MIB_ATTR_RTS_FAILURE_COUNT,
-				  &value, sizeof(value)))
-		wstats->discard.misc += value;
-	if (!wl3501_get_mib_value(this, WL3501_MIB_ATTR_ACK_FAILURE_COUNT,
-				  &value, sizeof(value)))
-		wstats->discard.misc += value;
-	if (!wl3501_get_mib_value(this, WL3501_MIB_ATTR_FRAME_DUPLICATE_COUNT,
-				  &value, sizeof(value)))
-		wstats->discard.misc += value;
-	return wstats;
-}
-
-/**
- * wl3501_detach - deletes a driver "instance"
- * @link: FILL_IN
- *
- * This deletes a driver "instance". The device is de-registered with Card
- * Services. If it has been released, all local data structures are freed.
- * Otherwise, the structures will be freed when the device is released.
- */
-static void wl3501_detach(struct pcmcia_device *link)
-{
-	struct net_device *dev = link->priv;
-
-	/* If the device is currently configured and active, we won't actually
-	 * delete it yet.  Instead, it is marked so that when the release()
-	 * function is called, that will trigger a proper detach(). */
-
-	while (link->open > 0)
-		wl3501_close(dev);
-
-	netif_device_detach(dev);
-	wl3501_release(link);
-
-	unregister_netdev(dev);
-	free_netdev(dev);
-}
-
-static int wl3501_get_name(struct net_device *dev, struct iw_request_info *info,
-			   union iwreq_data *wrqu, char *extra)
-{
-	strscpy(wrqu->name, "IEEE 802.11-DS", sizeof(wrqu->name));
-	return 0;
-}
-
-static int wl3501_set_freq(struct net_device *dev, struct iw_request_info *info,
-			   union iwreq_data *wrqu, char *extra)
-{
-	struct wl3501_card *this = netdev_priv(dev);
-	int channel = wrqu->freq.m;
-	int rc = -EINVAL;
-
-	if (iw_valid_channel(this->reg_domain, channel)) {
-		this->chan = channel;
-		rc = wl3501_reset(dev);
-	}
-	return rc;
-}
-
-static int wl3501_get_freq(struct net_device *dev, struct iw_request_info *info,
-			   union iwreq_data *wrqu, char *extra)
-{
-	struct wl3501_card *this = netdev_priv(dev);
-
-	wrqu->freq.m = 100000 *
-		ieee80211_channel_to_frequency(this->chan, NL80211_BAND_2GHZ);
-	wrqu->freq.e = 1;
-	return 0;
-}
-
-static int wl3501_set_mode(struct net_device *dev, struct iw_request_info *info,
-			   union iwreq_data *wrqu, char *extra)
-{
-	int rc = -EINVAL;
-
-	if (wrqu->mode == IW_MODE_INFRA ||
-	    wrqu->mode == IW_MODE_ADHOC ||
-	    wrqu->mode == IW_MODE_AUTO) {
-		struct wl3501_card *this = netdev_priv(dev);
-
-		this->net_type = wrqu->mode;
-		rc = wl3501_reset(dev);
-	}
-	return rc;
-}
-
-static int wl3501_get_mode(struct net_device *dev, struct iw_request_info *info,
-			   union iwreq_data *wrqu, char *extra)
-{
-	struct wl3501_card *this = netdev_priv(dev);
-
-	wrqu->mode = this->net_type;
-	return 0;
-}
-
-static int wl3501_get_sens(struct net_device *dev, struct iw_request_info *info,
-			   union iwreq_data *wrqu, char *extra)
-{
-	struct wl3501_card *this = netdev_priv(dev);
-
-	wrqu->sens.value = this->rssi;
-	wrqu->sens.disabled = !wrqu->sens.value;
-	wrqu->sens.fixed = 1;
-	return 0;
-}
-
-static int wl3501_get_range(struct net_device *dev,
-			    struct iw_request_info *info,
-			    union iwreq_data *wrqu, char *extra)
-{
-	struct iw_range *range = (struct iw_range *)extra;
-
-	/* Set the length (very important for backward compatibility) */
-	wrqu->data.length = sizeof(*range);
-
-	/* Set all the info we don't care or don't know about to zero */
-	memset(range, 0, sizeof(*range));
-
-	/* Set the Wireless Extension versions */
-	range->we_version_compiled	= WIRELESS_EXT;
-	range->we_version_source	= 1;
-	range->throughput		= 2 * 1000 * 1000;     /* ~2 Mb/s */
-	/* FIXME: study the code to fill in more fields... */
-	return 0;
-}
-
-static int wl3501_set_wap(struct net_device *dev, struct iw_request_info *info,
-			  union iwreq_data *wrqu, char *extra)
-{
-	struct wl3501_card *this = netdev_priv(dev);
-	int rc = -EINVAL;
-
-	/* FIXME: we support other ARPHRDs...*/
-	if (wrqu->ap_addr.sa_family != ARPHRD_ETHER)
-		goto out;
-	if (is_broadcast_ether_addr(wrqu->ap_addr.sa_data)) {
-		/* FIXME: rescan? */
-	} else
-		memcpy(this->bssid, wrqu->ap_addr.sa_data, ETH_ALEN);
-		/* FIXME: rescan? deassoc & scan? */
-	rc = 0;
-out:
-	return rc;
-}
-
-static int wl3501_get_wap(struct net_device *dev, struct iw_request_info *info,
-			  union iwreq_data *wrqu, char *extra)
-{
-	struct wl3501_card *this = netdev_priv(dev);
-
-	wrqu->ap_addr.sa_family = ARPHRD_ETHER;
-	memcpy(wrqu->ap_addr.sa_data, this->bssid, ETH_ALEN);
-	return 0;
-}
-
-static int wl3501_set_scan(struct net_device *dev, struct iw_request_info *info,
-			   union iwreq_data *wrqu, char *extra)
-{
-	/*
-	 * FIXME: trigger scanning with a reset, yes, I'm lazy
-	 */
-	return wl3501_reset(dev);
-}
-
-static int wl3501_get_scan(struct net_device *dev, struct iw_request_info *info,
-			   union iwreq_data *wrqu, char *extra)
-{
-	struct wl3501_card *this = netdev_priv(dev);
-	int i;
-	char *current_ev = extra;
-	struct iw_event iwe;
-
-	for (i = 0; i < this->bss_cnt; ++i) {
-		iwe.cmd			= SIOCGIWAP;
-		iwe.u.ap_addr.sa_family = ARPHRD_ETHER;
-		memcpy(iwe.u.ap_addr.sa_data, this->bss_set[i].req.bssid, ETH_ALEN);
-		current_ev = iwe_stream_add_event(info, current_ev,
-						  extra + IW_SCAN_MAX_DATA,
-						  &iwe, IW_EV_ADDR_LEN);
-		iwe.cmd		  = SIOCGIWESSID;
-		iwe.u.data.flags  = 1;
-		iwe.u.data.length = this->bss_set[i].req.ssid.el.len;
-		current_ev = iwe_stream_add_point(info, current_ev,
-						  extra + IW_SCAN_MAX_DATA,
-						  &iwe,
-						  this->bss_set[i].req.ssid.essid);
-		iwe.cmd	   = SIOCGIWMODE;
-		iwe.u.mode = this->bss_set[i].req.bss_type;
-		current_ev = iwe_stream_add_event(info, current_ev,
-						  extra + IW_SCAN_MAX_DATA,
-						  &iwe, IW_EV_UINT_LEN);
-		iwe.cmd = SIOCGIWFREQ;
-		iwe.u.freq.m = this->bss_set[i].req.ds_pset.chan;
-		iwe.u.freq.e = 0;
-		current_ev = iwe_stream_add_event(info, current_ev,
-						  extra + IW_SCAN_MAX_DATA,
-						  &iwe, IW_EV_FREQ_LEN);
-		iwe.cmd = SIOCGIWENCODE;
-		if (this->bss_set[i].req.cap_info & WL3501_MGMT_CAPABILITY_PRIVACY)
-			iwe.u.data.flags = IW_ENCODE_ENABLED | IW_ENCODE_NOKEY;
-		else
-			iwe.u.data.flags = IW_ENCODE_DISABLED;
-		iwe.u.data.length = 0;
-		current_ev = iwe_stream_add_point(info, current_ev,
-						  extra + IW_SCAN_MAX_DATA,
-						  &iwe, NULL);
-	}
-	/* Length of data */
-	wrqu->data.length = (current_ev - extra);
-	wrqu->data.flags = 0; /* FIXME: set properly these flags */
-	return 0;
-}
-
-static int wl3501_set_essid(struct net_device *dev,
-			    struct iw_request_info *info,
-			    union iwreq_data *wrqu, char *extra)
-{
-	struct wl3501_card *this = netdev_priv(dev);
-
-	if (wrqu->data.flags) {
-		iw_set_mgmt_info_element(IW_MGMT_INFO_ELEMENT_SSID,
-					 &this->essid.el,
-					 extra, wrqu->data.length);
-	} else { /* We accept any ESSID */
-		iw_set_mgmt_info_element(IW_MGMT_INFO_ELEMENT_SSID,
-					 &this->essid.el, "ANY", 3);
-	}
-	return wl3501_reset(dev);
-}
-
-static int wl3501_get_essid(struct net_device *dev,
-			    struct iw_request_info *info,
-			    union iwreq_data *wrqu, char *extra)
-{
-	struct wl3501_card *this = netdev_priv(dev);
-	unsigned long flags;
-
-	spin_lock_irqsave(&this->lock, flags);
-	wrqu->essid.flags  = 1;
-	wrqu->essid.length = this->essid.el.len;
-	memcpy(extra, this->essid.essid, this->essid.el.len);
-	spin_unlock_irqrestore(&this->lock, flags);
-	return 0;
-}
-
-static int wl3501_set_nick(struct net_device *dev, struct iw_request_info *info,
-			   union iwreq_data *wrqu, char *extra)
-{
-	struct wl3501_card *this = netdev_priv(dev);
-
-	if (wrqu->data.length > sizeof(this->nick))
-		return -E2BIG;
-	strscpy(this->nick, extra, wrqu->data.length);
-	return 0;
-}
-
-static int wl3501_get_nick(struct net_device *dev, struct iw_request_info *info,
-			   union iwreq_data *wrqu, char *extra)
-{
-	struct wl3501_card *this = netdev_priv(dev);
-
-	strscpy(extra, this->nick, 32);
-	wrqu->data.length = strlen(extra);
-	return 0;
-}
-
-static int wl3501_get_rate(struct net_device *dev, struct iw_request_info *info,
-			   union iwreq_data *wrqu, char *extra)
-{
-	/*
-	 * FIXME: have to see from where to get this info, perhaps this card
-	 * works at 1 Mbit/s too... for now leave at 2 Mbit/s that is the most
-	 * common with the Planet Access Points. -acme
-	 */
-	wrqu->bitrate.value = 2000000;
-	wrqu->bitrate.fixed = 1;
-	return 0;
-}
-
-static int wl3501_get_rts_threshold(struct net_device *dev,
-				    struct iw_request_info *info,
-				    union iwreq_data *wrqu, char *extra)
-{
-	u16 threshold; /* size checked: it is u16 */
-	struct wl3501_card *this = netdev_priv(dev);
-	int rc = wl3501_get_mib_value(this, WL3501_MIB_ATTR_RTS_THRESHOLD,
-				      &threshold, sizeof(threshold));
-	if (!rc) {
-		wrqu->rts.value = threshold;
-		wrqu->rts.disabled = threshold >= 2347;
-		wrqu->rts.fixed = 1;
-	}
-	return rc;
-}
-
-static int wl3501_get_frag_threshold(struct net_device *dev,
-				     struct iw_request_info *info,
-				     union iwreq_data *wrqu, char *extra)
-{
-	u16 threshold; /* size checked: it is u16 */
-	struct wl3501_card *this = netdev_priv(dev);
-	int rc = wl3501_get_mib_value(this, WL3501_MIB_ATTR_FRAG_THRESHOLD,
-				      &threshold, sizeof(threshold));
-	if (!rc) {
-		wrqu->frag.value = threshold;
-		wrqu->frag.disabled = threshold >= 2346;
-		wrqu->frag.fixed = 1;
-	}
-	return rc;
-}
-
-static int wl3501_get_txpow(struct net_device *dev,
-			    struct iw_request_info *info,
-			    union iwreq_data *wrqu, char *extra)
-{
-	u16 txpow;
-	struct wl3501_card *this = netdev_priv(dev);
-	int rc = wl3501_get_mib_value(this,
-				      WL3501_MIB_ATTR_CURRENT_TX_PWR_LEVEL,
-				      &txpow, sizeof(txpow));
-	if (!rc) {
-		wrqu->txpower.value = txpow;
-		wrqu->txpower.disabled = 0;
-		/*
-		 * From the MIB values I think this can be configurable,
-		 * as it lists several tx power levels -acme
-		 */
-		wrqu->txpower.fixed = 0;
-		wrqu->txpower.flags = IW_TXPOW_MWATT;
-	}
-	return rc;
-}
-
-static int wl3501_get_retry(struct net_device *dev,
-			    struct iw_request_info *info,
-			    union iwreq_data *wrqu, char *extra)
-{
-	u8 retry; /* size checked: it is u8 */
-	struct wl3501_card *this = netdev_priv(dev);
-	int rc = wl3501_get_mib_value(this,
-				      WL3501_MIB_ATTR_LONG_RETRY_LIMIT,
-				      &retry, sizeof(retry));
-	if (rc)
-		goto out;
-	if (wrqu->retry.flags & IW_RETRY_LONG) {
-		wrqu->retry.flags = IW_RETRY_LIMIT | IW_RETRY_LONG;
-		goto set_value;
-	}
-	rc = wl3501_get_mib_value(this, WL3501_MIB_ATTR_SHORT_RETRY_LIMIT,
-				  &retry, sizeof(retry));
-	if (rc)
-		goto out;
-	wrqu->retry.flags = IW_RETRY_LIMIT | IW_RETRY_SHORT;
-set_value:
-	wrqu->retry.value = retry;
-	wrqu->retry.disabled = 0;
-out:
-	return rc;
-}
-
-static int wl3501_get_encode(struct net_device *dev,
-			     struct iw_request_info *info,
-			     union iwreq_data *wrqu, char *extra)
-{
-	u8 implemented, restricted, keys[100], len_keys, tocopy;
-	struct wl3501_card *this = netdev_priv(dev);
-	int rc = wl3501_get_mib_value(this,
-				      WL3501_MIB_ATTR_PRIV_OPT_IMPLEMENTED,
-				      &implemented, sizeof(implemented));
-	if (rc)
-		goto out;
-	if (!implemented) {
-		wrqu->encoding.flags = IW_ENCODE_DISABLED;
-		goto out;
-	}
-	rc = wl3501_get_mib_value(this, WL3501_MIB_ATTR_EXCLUDE_UNENCRYPTED,
-				  &restricted, sizeof(restricted));
-	if (rc)
-		goto out;
-	wrqu->encoding.flags = restricted ? IW_ENCODE_RESTRICTED :
-					    IW_ENCODE_OPEN;
-	rc = wl3501_get_mib_value(this, WL3501_MIB_ATTR_WEP_KEY_MAPPINGS_LEN,
-				  &len_keys, sizeof(len_keys));
-	if (rc)
-		goto out;
-	rc = wl3501_get_mib_value(this, WL3501_MIB_ATTR_WEP_KEY_MAPPINGS,
-				  keys, len_keys);
-	if (rc)
-		goto out;
-	tocopy = min_t(u16, len_keys, wrqu->encoding.length);
-	tocopy = min_t(u8, tocopy, 100);
-	wrqu->encoding.length = tocopy;
-	memcpy(extra, keys, tocopy);
-out:
-	return rc;
-}
-
-static int wl3501_get_power(struct net_device *dev,
-			    struct iw_request_info *info,
-			    union iwreq_data *wrqu, char *extra)
-{
-	u8 pwr_state;
-	struct wl3501_card *this = netdev_priv(dev);
-	int rc = wl3501_get_mib_value(this,
-				      WL3501_MIB_ATTR_CURRENT_PWR_STATE,
-				      &pwr_state, sizeof(pwr_state));
-	if (rc)
-		goto out;
-	wrqu->power.disabled = !pwr_state;
-	wrqu->power.flags = IW_POWER_ON;
-out:
-	return rc;
-}
-
-static const iw_handler	wl3501_handler[] = {
-	IW_HANDLER(SIOCGIWNAME, wl3501_get_name),
-	IW_HANDLER(SIOCSIWFREQ, wl3501_set_freq),
-	IW_HANDLER(SIOCGIWFREQ, wl3501_get_freq),
-	IW_HANDLER(SIOCSIWMODE, wl3501_set_mode),
-	IW_HANDLER(SIOCGIWMODE, wl3501_get_mode),
-	IW_HANDLER(SIOCGIWSENS, wl3501_get_sens),
-	IW_HANDLER(SIOCGIWRANGE, wl3501_get_range),
-	IW_HANDLER(SIOCSIWSPY, iw_handler_set_spy),
-	IW_HANDLER(SIOCGIWSPY, iw_handler_get_spy),
-	IW_HANDLER(SIOCSIWTHRSPY, iw_handler_set_thrspy),
-	IW_HANDLER(SIOCGIWTHRSPY, iw_handler_get_thrspy),
-	IW_HANDLER(SIOCSIWAP, wl3501_set_wap),
-	IW_HANDLER(SIOCGIWAP, wl3501_get_wap),
-	IW_HANDLER(SIOCSIWSCAN, wl3501_set_scan),
-	IW_HANDLER(SIOCGIWSCAN, wl3501_get_scan),
-	IW_HANDLER(SIOCSIWESSID, wl3501_set_essid),
-	IW_HANDLER(SIOCGIWESSID, wl3501_get_essid),
-	IW_HANDLER(SIOCSIWNICKN, wl3501_set_nick),
-	IW_HANDLER(SIOCGIWNICKN, wl3501_get_nick),
-	IW_HANDLER(SIOCGIWRATE, wl3501_get_rate),
-	IW_HANDLER(SIOCGIWRTS, wl3501_get_rts_threshold),
-	IW_HANDLER(SIOCGIWFRAG, wl3501_get_frag_threshold),
-	IW_HANDLER(SIOCGIWTXPOW, wl3501_get_txpow),
-	IW_HANDLER(SIOCGIWRETRY, wl3501_get_retry),
-	IW_HANDLER(SIOCGIWENCODE, wl3501_get_encode),
-	IW_HANDLER(SIOCGIWPOWER, wl3501_get_power),
-};
-
-static const struct iw_handler_def wl3501_handler_def = {
-	.num_standard	= ARRAY_SIZE(wl3501_handler),
-	.standard	= (iw_handler *)wl3501_handler,
-	.get_wireless_stats = wl3501_get_wireless_stats,
-};
-
-static const struct net_device_ops wl3501_netdev_ops = {
-	.ndo_open		= wl3501_open,
-	.ndo_stop		= wl3501_close,
-	.ndo_start_xmit		= wl3501_hard_start_xmit,
-	.ndo_tx_timeout		= wl3501_tx_timeout,
-	.ndo_set_mac_address 	= eth_mac_addr,
-	.ndo_validate_addr	= eth_validate_addr,
-};
-
-static int wl3501_probe(struct pcmcia_device *p_dev)
-{
-	struct net_device *dev;
-	struct wl3501_card *this;
-	int ret;
-
-	/* The io structure describes IO port mapping */
-	p_dev->resource[0]->end	= 16;
-	p_dev->resource[0]->flags	= IO_DATA_PATH_WIDTH_8;
-
-	/* General socket configuration */
-	p_dev->config_flags	= CONF_ENABLE_IRQ;
-	p_dev->config_index	= 1;
-
-	dev = alloc_etherdev(sizeof(struct wl3501_card));
-	if (!dev)
-		return -ENOMEM;
-
-	dev->netdev_ops		= &wl3501_netdev_ops;
-	dev->watchdog_timeo	= 5 * HZ;
-
-	this = netdev_priv(dev);
-	this->wireless_data.spy_data = &this->spy_data;
-	this->p_dev = p_dev;
-	dev->wireless_data	= &this->wireless_data;
-	dev->wireless_handlers	= &wl3501_handler_def;
-	netif_stop_queue(dev);
-	p_dev->priv = dev;
-
-	ret = wl3501_config(p_dev);
-	if (ret)
-		goto out_free_etherdev;
-
-	return 0;
-
-out_free_etherdev:
-	free_netdev(dev);
-	return ret;
-}
-
-static int wl3501_config(struct pcmcia_device *link)
-{
-	struct net_device *dev = link->priv;
-	int i = 0, j, ret;
-	struct wl3501_card *this;
-
-	/* Try allocating IO ports.  This tries a few fixed addresses.  If you
-	 * want, you can also read the card's config table to pick addresses --
-	 * see the serial driver for an example. */
-	link->io_lines = 5;
-
-	for (j = 0x280; j < 0x400; j += 0x20) {
-		/* The '^0x300' is so that we probe 0x300-0x3ff first, then
-		 * 0x200-0x2ff, and so on, because this seems safer */
-		link->resource[0]->start = j;
-		link->resource[1]->start = link->resource[0]->start + 0x10;
-		i = pcmcia_request_io(link);
-		if (i == 0)
-			break;
-	}
-	if (i != 0)
-		goto failed;
-
-	/* Now allocate an interrupt line. Note that this does not actually
-	 * assign a handler to the interrupt. */
-
-	ret = pcmcia_request_irq(link, wl3501_interrupt);
-	if (ret)
-		goto failed;
-
-	ret = pcmcia_enable_device(link);
-	if (ret)
-		goto failed;
-
-	dev->irq = link->irq;
-	dev->base_addr = link->resource[0]->start;
-	SET_NETDEV_DEV(dev, &link->dev);
-	if (register_netdev(dev)) {
-		printk(KERN_NOTICE "wl3501_cs: register_netdev() failed\n");
-		goto failed;
-	}
-
-	this = netdev_priv(dev);
-
-	this->base_addr = dev->base_addr;
-
-	if (!wl3501_get_flash_mac_addr(this)) {
-		printk(KERN_WARNING "%s: Can't read MAC addr in flash ROM?\n",
-		       dev->name);
-		unregister_netdev(dev);
-		goto failed;
-	}
-
-	eth_hw_addr_set(dev, this->mac_addr);
-
-	/* print probe information */
-	printk(KERN_INFO "%s: wl3501 @ 0x%3.3x, IRQ %d, "
-	       "MAC addr in flash ROM:%pM\n",
-	       dev->name, this->base_addr, (int)dev->irq,
-	       dev->dev_addr);
-	/*
-	 * Initialize card parameters - added by jss
-	 */
-	this->net_type		= IW_MODE_INFRA;
-	this->bss_cnt		= 0;
-	this->join_sta_bss	= 0;
-	this->adhoc_times	= 0;
-	iw_set_mgmt_info_element(IW_MGMT_INFO_ELEMENT_SSID, &this->essid.el,
-				 "ANY", 3);
-	this->card_name[0]	= '\0';
-	this->firmware_date[0]	= '\0';
-	this->rssi		= 255;
-	this->chan		= iw_default_channel(this->reg_domain);
-	strscpy(this->nick, "Planet WL3501", sizeof(this->nick));
-	spin_lock_init(&this->lock);
-	init_waitqueue_head(&this->wait);
-	netif_start_queue(dev);
-	return 0;
-
-failed:
-	wl3501_release(link);
-	return -ENODEV;
-}
-
-static void wl3501_release(struct pcmcia_device *link)
-{
-	pcmcia_disable_device(link);
-}
-
-static int wl3501_suspend(struct pcmcia_device *link)
-{
-	struct net_device *dev = link->priv;
-
-	wl3501_pwr_mgmt(netdev_priv(dev), WL3501_SUSPEND);
-	if (link->open)
-		netif_device_detach(dev);
-
-	return 0;
-}
-
-static int wl3501_resume(struct pcmcia_device *link)
-{
-	struct net_device *dev = link->priv;
-
-	wl3501_pwr_mgmt(netdev_priv(dev), WL3501_RESUME);
-	if (link->open) {
-		wl3501_reset(dev);
-		netif_device_attach(dev);
-	}
-
-	return 0;
-}
-
-
-static const struct pcmcia_device_id wl3501_ids[] = {
-	PCMCIA_DEVICE_MANF_CARD(0xd601, 0x0001),
-	PCMCIA_DEVICE_NULL
-};
-MODULE_DEVICE_TABLE(pcmcia, wl3501_ids);
-
-static struct pcmcia_driver wl3501_driver = {
-	.owner		= THIS_MODULE,
-	.name		= "wl3501_cs",
-	.probe		= wl3501_probe,
-	.remove		= wl3501_detach,
-	.id_table	= wl3501_ids,
-	.suspend	= wl3501_suspend,
-	.resume		= wl3501_resume,
-};
-module_pcmcia_driver(wl3501_driver);
-
-MODULE_AUTHOR("Fox Chen <mhchen@golf.ccl.itri.org.tw>, "
-	      "Arnaldo Carvalho de Melo <acme@conectiva.com.br>,"
-	      "Gustavo Niemeyer <niemeyer@conectiva.com>");
-MODULE_DESCRIPTION("Planet wl3501 wireless driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/net/wireless/marvell/libertas/Kconfig b/drivers/net/wireless/marvell/libertas/Kconfig
index 6d62ab49aa8d..56156a021be3 100644
--- a/drivers/net/wireless/marvell/libertas/Kconfig
+++ b/drivers/net/wireless/marvell/libertas/Kconfig
@@ -1,9 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0-only
 config LIBERTAS
 	tristate "Marvell 8xxx Libertas WLAN driver support"
+	depends on USB || SDIO || SPI
 	depends on CFG80211
-	select WIRELESS_EXT
-	select WEXT_SPY
 	select LIB80211
 	select FW_LOADER
 	help
@@ -15,12 +14,6 @@ config LIBERTAS_USB
 	help
 	  A driver for Marvell Libertas 8388 USB devices.
 
-config LIBERTAS_CS
-	tristate "Marvell Libertas 8385 CompactFlash 802.11b/g cards"
-	depends on LIBERTAS && PCMCIA && HAS_IOPORT_MAP
-	help
-	  A driver for Marvell Libertas 8385 CompactFlash devices.
-
 config LIBERTAS_SDIO
 	tristate "Marvell Libertas 8385/8686/8688 SDIO 802.11b/g cards"
 	depends on LIBERTAS && MMC
diff --git a/drivers/net/wireless/marvell/libertas/Makefile b/drivers/net/wireless/marvell/libertas/Makefile
index 41b9b440a542..2ac04f4d61a5 100644
--- a/drivers/net/wireless/marvell/libertas/Makefile
+++ b/drivers/net/wireless/marvell/libertas/Makefile
@@ -17,6 +17,5 @@ libertas_spi-objs += if_spi.o
 
 obj-$(CONFIG_LIBERTAS)     += libertas.o
 obj-$(CONFIG_LIBERTAS_USB) += usb8xxx.o
-obj-$(CONFIG_LIBERTAS_CS)  += libertas_cs.o
 obj-$(CONFIG_LIBERTAS_SDIO) += libertas_sdio.o
 obj-$(CONFIG_LIBERTAS_SPI) += libertas_spi.o
diff --git a/drivers/net/wireless/marvell/libertas/if_cs.c b/drivers/net/wireless/marvell/libertas/if_cs.c
deleted file mode 100644
index 4103f15bca6b..000000000000
--- a/drivers/net/wireless/marvell/libertas/if_cs.c
+++ /dev/null
@@ -1,957 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
-
-  Driver for the Marvell 8385 based compact flash WLAN cards.
-
-  (C) 2007 by Holger Schurig <hs4233@mail.mn-solutions.de>
-
-
-*/
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/delay.h>
-#include <linux/moduleparam.h>
-#include <linux/firmware.h>
-#include <linux/netdevice.h>
-
-#include <pcmcia/cistpl.h>
-#include <pcmcia/ds.h>
-
-#include <linux/io.h>
-
-#define DRV_NAME "libertas_cs"
-
-#include "decl.h"
-#include "defs.h"
-#include "dev.h"
-
-
-/********************************************************************/
-/* Module stuff                                                     */
-/********************************************************************/
-
-MODULE_AUTHOR("Holger Schurig <hs4233@mail.mn-solutions.de>");
-MODULE_DESCRIPTION("Driver for Marvell 83xx compact flash WLAN cards");
-MODULE_LICENSE("GPL");
-
-
-
-/********************************************************************/
-/* Data structures                                                  */
-/********************************************************************/
-
-struct if_cs_card {
-	struct pcmcia_device *p_dev;
-	struct lbs_private *priv;
-	void __iomem *iobase;
-	bool align_regs;
-	u32 model;
-};
-
-
-enum {
-	MODEL_UNKNOWN = 0x00,
-	MODEL_8305 = 0x01,
-	MODEL_8381 = 0x02,
-	MODEL_8385 = 0x03
-};
-
-static const struct lbs_fw_table fw_table[] = {
-	{ MODEL_8305, "libertas/cf8305.bin", NULL },
-	{ MODEL_8305, "libertas_cs_helper.fw", NULL },
-	{ MODEL_8381, "libertas/cf8381_helper.bin", "libertas/cf8381.bin" },
-	{ MODEL_8381, "libertas_cs_helper.fw", "libertas_cs.fw" },
-	{ MODEL_8385, "libertas/cf8385_helper.bin", "libertas/cf8385.bin" },
-	{ MODEL_8385, "libertas_cs_helper.fw", "libertas_cs.fw" },
-	{ 0, NULL, NULL }
-};
-MODULE_FIRMWARE("libertas/cf8305.bin");
-MODULE_FIRMWARE("libertas/cf8381_helper.bin");
-MODULE_FIRMWARE("libertas/cf8381.bin");
-MODULE_FIRMWARE("libertas/cf8385_helper.bin");
-MODULE_FIRMWARE("libertas/cf8385.bin");
-MODULE_FIRMWARE("libertas_cs_helper.fw");
-MODULE_FIRMWARE("libertas_cs.fw");
-
-
-/********************************************************************/
-/* Hardware access                                                  */
-/********************************************************************/
-
-/* This define enables wrapper functions which allow you
-   to dump all register accesses. You normally won't this,
-   except for development */
-/* #define DEBUG_IO */
-
-#ifdef DEBUG_IO
-static int debug_output = 0;
-#else
-/* This way the compiler optimizes the printk's away */
-#define debug_output 0
-#endif
-
-static inline unsigned int if_cs_read8(struct if_cs_card *card, uint reg)
-{
-	unsigned int val = ioread8(card->iobase + reg);
-	if (debug_output)
-		printk(KERN_INFO "inb %08x<%02x\n", reg, val);
-	return val;
-}
-static inline unsigned int if_cs_read16(struct if_cs_card *card, uint reg)
-{
-	unsigned int val = ioread16(card->iobase + reg);
-	if (debug_output)
-		printk(KERN_INFO "inw %08x<%04x\n", reg, val);
-	return val;
-}
-static inline void if_cs_read16_rep(
-	struct if_cs_card *card,
-	uint reg,
-	void *buf,
-	unsigned long count)
-{
-	if (debug_output)
-		printk(KERN_INFO "insw %08x<(0x%lx words)\n",
-			reg, count);
-	ioread16_rep(card->iobase + reg, buf, count);
-}
-
-static inline void if_cs_write8(struct if_cs_card *card, uint reg, u8 val)
-{
-	if (debug_output)
-		printk(KERN_INFO "outb %08x>%02x\n", reg, val);
-	iowrite8(val, card->iobase + reg);
-}
-
-static inline void if_cs_write16(struct if_cs_card *card, uint reg, u16 val)
-{
-	if (debug_output)
-		printk(KERN_INFO "outw %08x>%04x\n", reg, val);
-	iowrite16(val, card->iobase + reg);
-}
-
-static inline void if_cs_write16_rep(
-	struct if_cs_card *card,
-	uint reg,
-	const void *buf,
-	unsigned long count)
-{
-	if (debug_output)
-		printk(KERN_INFO "outsw %08x>(0x%lx words)\n",
-			reg, count);
-	iowrite16_rep(card->iobase + reg, buf, count);
-}
-
-
-/*
- * I know that polling/delaying is frowned upon. However, this procedure
- * with polling is needed while downloading the firmware. At this stage,
- * the hardware does unfortunately not create any interrupts.
- *
- * Fortunately, this function is never used once the firmware is in
- * the card. :-)
- *
- * As a reference, see the "Firmware Specification v5.1", page 18
- * and 19. I did not follow their suggested timing to the word,
- * but this works nice & fast anyway.
- */
-static int if_cs_poll_while_fw_download(struct if_cs_card *card, uint addr, u8 reg)
-{
-	int i;
-
-	for (i = 0; i < 100000; i++) {
-		u8 val = if_cs_read8(card, addr);
-		if (val == reg)
-			return 0;
-		udelay(5);
-	}
-	return -ETIME;
-}
-
-
-
-/*
- * First the bitmasks for the host/card interrupt/status registers:
- */
-#define IF_CS_BIT_TX			0x0001
-#define IF_CS_BIT_RX			0x0002
-#define IF_CS_BIT_COMMAND		0x0004
-#define IF_CS_BIT_RESP			0x0008
-#define IF_CS_BIT_EVENT			0x0010
-#define	IF_CS_BIT_MASK			0x001f
-
-
-
-/*
- * It's not really clear to me what the host status register is for. It
- * needs to be set almost in union with "host int cause". The following
- * bits from above are used:
- *
- *   IF_CS_BIT_TX         driver downloaded a data packet
- *   IF_CS_BIT_RX         driver got a data packet
- *   IF_CS_BIT_COMMAND    driver downloaded a command
- *   IF_CS_BIT_RESP       not used (has some meaning with powerdown)
- *   IF_CS_BIT_EVENT      driver read a host event
- */
-#define IF_CS_HOST_STATUS		0x00000000
-
-/*
- * With the host int cause register can the host (that is, Linux) cause
- * an interrupt in the firmware, to tell the firmware about those events:
- *
- *   IF_CS_BIT_TX         a data packet has been downloaded
- *   IF_CS_BIT_RX         a received data packet has retrieved
- *   IF_CS_BIT_COMMAND    a firmware block or a command has been downloaded
- *   IF_CS_BIT_RESP       not used (has some meaning with powerdown)
- *   IF_CS_BIT_EVENT      a host event (link lost etc) has been retrieved
- */
-#define IF_CS_HOST_INT_CAUSE		0x00000002
-
-/*
- * The host int mask register is used to enable/disable interrupt.  However,
- * I have the suspicion that disabled interrupts are lost.
- */
-#define IF_CS_HOST_INT_MASK		0x00000004
-
-/*
- * Used to send or receive data packets:
- */
-#define IF_CS_WRITE			0x00000016
-#define IF_CS_WRITE_LEN			0x00000014
-#define IF_CS_READ			0x00000010
-#define IF_CS_READ_LEN			0x00000024
-
-/*
- * Used to send commands (and to send firmware block) and to
- * receive command responses:
- */
-#define IF_CS_CMD			0x0000001A
-#define IF_CS_CMD_LEN			0x00000018
-#define IF_CS_RESP			0x00000012
-#define IF_CS_RESP_LEN			0x00000030
-
-/*
- * The card status registers shows what the card/firmware actually
- * accepts:
- *
- *   IF_CS_BIT_TX        you may send a data packet
- *   IF_CS_BIT_RX        you may retrieve a data packet
- *   IF_CS_BIT_COMMAND   you may send a command
- *   IF_CS_BIT_RESP      you may retrieve a command response
- *   IF_CS_BIT_EVENT     the card has a event for use (link lost, snr low etc)
- *
- * When reading this register several times, you will get back the same
- * results --- with one exception: the IF_CS_BIT_EVENT clear itself
- * automatically.
- *
- * Not that we don't rely on BIT_RX,_BIT_RESP or BIT_EVENT because
- * we handle this via the card int cause register.
- */
-#define IF_CS_CARD_STATUS		0x00000020
-#define IF_CS_CARD_STATUS_MASK		0x7f00
-
-/*
- * The card int cause register is used by the card/firmware to notify us
- * about the following events:
- *
- *   IF_CS_BIT_TX        a data packet has successfully been sentx
- *   IF_CS_BIT_RX        a data packet has been received and can be retrieved
- *   IF_CS_BIT_COMMAND   not used
- *   IF_CS_BIT_RESP      the firmware has a command response for us
- *   IF_CS_BIT_EVENT     the card has a event for use (link lost, snr low etc)
- */
-#define IF_CS_CARD_INT_CAUSE		0x00000022
-
-/*
- * This is used to for handshaking with the card's bootloader/helper image
- * to synchronize downloading of firmware blocks.
- */
-#define IF_CS_SQ_READ_LOW		0x00000028
-#define IF_CS_SQ_HELPER_OK		0x10
-
-/*
- * The scratch register tells us ...
- *
- * IF_CS_SCRATCH_BOOT_OK     the bootloader runs
- * IF_CS_SCRATCH_HELPER_OK   the helper firmware already runs
- */
-#define IF_CS_SCRATCH			0x0000003F
-#define IF_CS_SCRATCH_BOOT_OK		0x00
-#define IF_CS_SCRATCH_HELPER_OK		0x5a
-
-/*
- * Used to detect ancient chips:
- */
-#define IF_CS_PRODUCT_ID		0x0000001C
-#define IF_CS_CF8385_B1_REV		0x12
-#define IF_CS_CF8381_B3_REV		0x04
-#define IF_CS_CF8305_B1_REV		0x03
-
-/*
- * Used to detect other cards than CF8385 since their revisions of silicon
- * doesn't match those from CF8385, eg. CF8381 B3 works with this driver.
- */
-#define CF8305_MANFID		0x02db
-#define CF8305_CARDID		0x8103
-#define CF8381_MANFID		0x02db
-#define CF8381_CARDID		0x6064
-#define CF8385_MANFID		0x02df
-#define CF8385_CARDID		0x8103
-
-/*
- * FIXME: just use the 'driver_info' field of 'struct pcmcia_device_id' when
- * that gets fixed.  Currently there's no way to access it from the probe hook.
- */
-static inline u32 get_model(u16 manf_id, u16 card_id)
-{
-	/* NOTE: keep in sync with if_cs_ids */
-	if (manf_id == CF8305_MANFID && card_id == CF8305_CARDID)
-		return MODEL_8305;
-	else if (manf_id == CF8381_MANFID && card_id == CF8381_CARDID)
-		return MODEL_8381;
-	else if (manf_id == CF8385_MANFID && card_id == CF8385_CARDID)
-		return MODEL_8385;
-	return MODEL_UNKNOWN;
-}
-
-/********************************************************************/
-/* I/O and interrupt handling                                       */
-/********************************************************************/
-
-static inline void if_cs_enable_ints(struct if_cs_card *card)
-{
-	if_cs_write16(card, IF_CS_HOST_INT_MASK, 0);
-}
-
-static inline void if_cs_disable_ints(struct if_cs_card *card)
-{
-	if_cs_write16(card, IF_CS_HOST_INT_MASK, IF_CS_BIT_MASK);
-}
-
-/*
- * Called from if_cs_host_to_card to send a command to the hardware
- */
-static int if_cs_send_cmd(struct lbs_private *priv, u8 *buf, u16 nb)
-{
-	struct if_cs_card *card = (struct if_cs_card *)priv->card;
-	int ret = -1;
-	int loops = 0;
-
-	if_cs_disable_ints(card);
-
-	/* Is hardware ready? */
-	while (1) {
-		u16 status = if_cs_read16(card, IF_CS_CARD_STATUS);
-		if (status & IF_CS_BIT_COMMAND)
-			break;
-		if (++loops > 100) {
-			netdev_err(priv->dev, "card not ready for commands\n");
-			goto done;
-		}
-		mdelay(1);
-	}
-
-	if_cs_write16(card, IF_CS_CMD_LEN, nb);
-
-	if_cs_write16_rep(card, IF_CS_CMD, buf, nb / 2);
-	/* Are we supposed to transfer an odd amount of bytes? */
-	if (nb & 1)
-		if_cs_write8(card, IF_CS_CMD, buf[nb-1]);
-
-	/* "Assert the download over interrupt command in the Host
-	 * status register" */
-	if_cs_write16(card, IF_CS_HOST_STATUS, IF_CS_BIT_COMMAND);
-
-	/* "Assert the download over interrupt command in the Card
-	 * interrupt case register" */
-	if_cs_write16(card, IF_CS_HOST_INT_CAUSE, IF_CS_BIT_COMMAND);
-	ret = 0;
-
-done:
-	if_cs_enable_ints(card);
-	return ret;
-}
-
-/*
- * Called from if_cs_host_to_card to send a data to the hardware
- */
-static void if_cs_send_data(struct lbs_private *priv, u8 *buf, u16 nb)
-{
-	struct if_cs_card *card = (struct if_cs_card *)priv->card;
-	u16 status;
-
-	if_cs_disable_ints(card);
-
-	status = if_cs_read16(card, IF_CS_CARD_STATUS);
-	BUG_ON((status & IF_CS_BIT_TX) == 0);
-
-	if_cs_write16(card, IF_CS_WRITE_LEN, nb);
-
-	/* write even number of bytes, then odd byte if necessary */
-	if_cs_write16_rep(card, IF_CS_WRITE, buf, nb / 2);
-	if (nb & 1)
-		if_cs_write8(card, IF_CS_WRITE, buf[nb-1]);
-
-	if_cs_write16(card, IF_CS_HOST_STATUS, IF_CS_BIT_TX);
-	if_cs_write16(card, IF_CS_HOST_INT_CAUSE, IF_CS_BIT_TX);
-	if_cs_enable_ints(card);
-}
-
-/*
- * Get the command result out of the card.
- */
-static int if_cs_receive_cmdres(struct lbs_private *priv, u8 *data, u32 *len)
-{
-	unsigned long flags;
-	int ret = -1;
-	u16 status;
-
-	/* is hardware ready? */
-	status = if_cs_read16(priv->card, IF_CS_CARD_STATUS);
-	if ((status & IF_CS_BIT_RESP) == 0) {
-		netdev_err(priv->dev, "no cmd response in card\n");
-		*len = 0;
-		goto out;
-	}
-
-	*len = if_cs_read16(priv->card, IF_CS_RESP_LEN);
-	if ((*len == 0) || (*len > LBS_CMD_BUFFER_SIZE)) {
-		netdev_err(priv->dev,
-			   "card cmd buffer has invalid # of bytes (%d)\n",
-			   *len);
-		goto out;
-	}
-
-	/* read even number of bytes, then odd byte if necessary */
-	if_cs_read16_rep(priv->card, IF_CS_RESP, data, *len/sizeof(u16));
-	if (*len & 1)
-		data[*len-1] = if_cs_read8(priv->card, IF_CS_RESP);
-
-	/* This is a workaround for a firmware that reports too much
-	 * bytes */
-	*len -= 8;
-	ret = 0;
-
-	/* Clear this flag again */
-	spin_lock_irqsave(&priv->driver_lock, flags);
-	priv->dnld_sent = DNLD_RES_RECEIVED;
-	spin_unlock_irqrestore(&priv->driver_lock, flags);
-
-out:
-	return ret;
-}
-
-static struct sk_buff *if_cs_receive_data(struct lbs_private *priv)
-{
-	struct sk_buff *skb = NULL;
-	u16 len;
-	u8 *data;
-
-	len = if_cs_read16(priv->card, IF_CS_READ_LEN);
-	if (len == 0 || len > MRVDRV_ETH_RX_PACKET_BUFFER_SIZE) {
-		netdev_err(priv->dev,
-			   "card data buffer has invalid # of bytes (%d)\n",
-			   len);
-		priv->dev->stats.rx_dropped++;
-		goto dat_err;
-	}
-
-	skb = dev_alloc_skb(MRVDRV_ETH_RX_PACKET_BUFFER_SIZE + 2);
-	if (!skb)
-		goto out;
-	skb_put(skb, len);
-	skb_reserve(skb, 2);/* 16 byte align */
-	data = skb->data;
-
-	/* read even number of bytes, then odd byte if necessary */
-	if_cs_read16_rep(priv->card, IF_CS_READ, data, len/sizeof(u16));
-	if (len & 1)
-		data[len-1] = if_cs_read8(priv->card, IF_CS_READ);
-
-dat_err:
-	if_cs_write16(priv->card, IF_CS_HOST_STATUS, IF_CS_BIT_RX);
-	if_cs_write16(priv->card, IF_CS_HOST_INT_CAUSE, IF_CS_BIT_RX);
-
-out:
-	return skb;
-}
-
-static irqreturn_t if_cs_interrupt(int irq, void *data)
-{
-	struct if_cs_card *card = data;
-	struct lbs_private *priv = card->priv;
-	u16 cause;
-
-	/* Ask card interrupt cause register if there is something for us */
-	cause = if_cs_read16(card, IF_CS_CARD_INT_CAUSE);
-	lbs_deb_cs("cause 0x%04x\n", cause);
-
-	if (cause == 0) {
-		/* Not for us */
-		return IRQ_NONE;
-	}
-
-	if (cause == 0xffff) {
-		/* Read in junk, the card has probably been removed */
-		card->priv->surpriseremoved = 1;
-		return IRQ_HANDLED;
-	}
-
-	if (cause & IF_CS_BIT_RX) {
-		struct sk_buff *skb;
-		lbs_deb_cs("rx packet\n");
-		skb = if_cs_receive_data(priv);
-		if (skb)
-			lbs_process_rxed_packet(priv, skb);
-	}
-
-	if (cause & IF_CS_BIT_TX) {
-		lbs_deb_cs("tx done\n");
-		lbs_host_to_card_done(priv);
-	}
-
-	if (cause & IF_CS_BIT_RESP) {
-		unsigned long flags;
-		u8 i;
-
-		lbs_deb_cs("cmd resp\n");
-		spin_lock_irqsave(&priv->driver_lock, flags);
-		i = (priv->resp_idx == 0) ? 1 : 0;
-		spin_unlock_irqrestore(&priv->driver_lock, flags);
-
-		BUG_ON(priv->resp_len[i]);
-		if_cs_receive_cmdres(priv, priv->resp_buf[i],
-			&priv->resp_len[i]);
-
-		spin_lock_irqsave(&priv->driver_lock, flags);
-		lbs_notify_command_response(priv, i);
-		spin_unlock_irqrestore(&priv->driver_lock, flags);
-	}
-
-	if (cause & IF_CS_BIT_EVENT) {
-		u16 status = if_cs_read16(priv->card, IF_CS_CARD_STATUS);
-		if_cs_write16(priv->card, IF_CS_HOST_INT_CAUSE,
-			IF_CS_BIT_EVENT);
-		lbs_queue_event(priv, (status & IF_CS_CARD_STATUS_MASK) >> 8);
-	}
-
-	/* Clear interrupt cause */
-	if_cs_write16(card, IF_CS_CARD_INT_CAUSE, cause & IF_CS_BIT_MASK);
-
-	return IRQ_HANDLED;
-}
-
-
-
-
-/********************************************************************/
-/* Firmware                                                         */
-/********************************************************************/
-
-/*
- * Tries to program the helper firmware.
- *
- * Return 0 on success
- */
-static int if_cs_prog_helper(struct if_cs_card *card, const struct firmware *fw)
-{
-	int ret = 0;
-	int sent = 0;
-	u8  scratch;
-
-	/*
-	 * This is the only place where an unaligned register access happens on
-	 * the CF8305 card, therefore for the sake of speed of the driver, we do
-	 * the alignment correction here.
-	 */
-	if (card->align_regs)
-		scratch = if_cs_read16(card, IF_CS_SCRATCH) >> 8;
-	else
-		scratch = if_cs_read8(card, IF_CS_SCRATCH);
-
-	/* "If the value is 0x5a, the firmware is already
-	 * downloaded successfully"
-	 */
-	if (scratch == IF_CS_SCRATCH_HELPER_OK)
-		goto done;
-
-	/* "If the value is != 00, it is invalid value of register */
-	if (scratch != IF_CS_SCRATCH_BOOT_OK) {
-		ret = -ENODEV;
-		goto done;
-	}
-
-	lbs_deb_cs("helper size %td\n", fw->size);
-
-	/* "Set the 5 bytes of the helper image to 0" */
-	/* Not needed, this contains an ARM branch instruction */
-
-	for (;;) {
-		/* "the number of bytes to send is 256" */
-		int count = 256;
-		int remain = fw->size - sent;
-
-		if (remain < count)
-			count = remain;
-
-		/*
-		 * "write the number of bytes to be sent to the I/O Command
-		 * write length register"
-		 */
-		if_cs_write16(card, IF_CS_CMD_LEN, count);
-
-		/* "write this to I/O Command port register as 16 bit writes */
-		if (count)
-			if_cs_write16_rep(card, IF_CS_CMD,
-				&fw->data[sent],
-				count >> 1);
-
-		/*
-		 * "Assert the download over interrupt command in the Host
-		 * status register"
-		 */
-		if_cs_write8(card, IF_CS_HOST_STATUS, IF_CS_BIT_COMMAND);
-
-		/*
-		 * "Assert the download over interrupt command in the Card
-		 * interrupt case register"
-		 */
-		if_cs_write16(card, IF_CS_HOST_INT_CAUSE, IF_CS_BIT_COMMAND);
-
-		/*
-		 * "The host polls the Card Status register ... for 50 ms before
-		 * declaring a failure"
-		 */
-		ret = if_cs_poll_while_fw_download(card, IF_CS_CARD_STATUS,
-			IF_CS_BIT_COMMAND);
-		if (ret < 0) {
-			pr_err("can't download helper at 0x%x, ret %d\n",
-			       sent, ret);
-			goto done;
-		}
-
-		if (count == 0)
-			break;
-
-		sent += count;
-	}
-
-done:
-	return ret;
-}
-
-
-static int if_cs_prog_real(struct if_cs_card *card, const struct firmware *fw)
-{
-	int ret = 0;
-	int retry = 0;
-	int len = 0;
-	int sent;
-
-	lbs_deb_cs("fw size %td\n", fw->size);
-
-	ret = if_cs_poll_while_fw_download(card, IF_CS_SQ_READ_LOW,
-		IF_CS_SQ_HELPER_OK);
-	if (ret < 0) {
-		pr_err("helper firmware doesn't answer\n");
-		goto done;
-	}
-
-	for (sent = 0; sent < fw->size; sent += len) {
-		len = if_cs_read16(card, IF_CS_SQ_READ_LOW);
-		if (len & 1) {
-			retry++;
-			pr_info("odd, need to retry this firmware block\n");
-		} else {
-			retry = 0;
-		}
-
-		if (retry > 20) {
-			pr_err("could not download firmware\n");
-			ret = -ENODEV;
-			goto done;
-		}
-		if (retry) {
-			sent -= len;
-		}
-
-
-		if_cs_write16(card, IF_CS_CMD_LEN, len);
-
-		if_cs_write16_rep(card, IF_CS_CMD,
-			&fw->data[sent],
-			(len+1) >> 1);
-		if_cs_write8(card, IF_CS_HOST_STATUS, IF_CS_BIT_COMMAND);
-		if_cs_write16(card, IF_CS_HOST_INT_CAUSE, IF_CS_BIT_COMMAND);
-
-		ret = if_cs_poll_while_fw_download(card, IF_CS_CARD_STATUS,
-			IF_CS_BIT_COMMAND);
-		if (ret < 0) {
-			pr_err("can't download firmware at 0x%x\n", sent);
-			goto done;
-		}
-	}
-
-	ret = if_cs_poll_while_fw_download(card, IF_CS_SCRATCH, 0x5a);
-	if (ret < 0)
-		pr_err("firmware download failed\n");
-
-done:
-	return ret;
-}
-
-static void if_cs_prog_firmware(struct lbs_private *priv, int ret,
-				 const struct firmware *helper,
-				 const struct firmware *mainfw)
-{
-	struct if_cs_card *card = priv->card;
-
-	if (ret) {
-		pr_err("failed to find firmware (%d)\n", ret);
-		return;
-	}
-
-	/* Load the firmware */
-	ret = if_cs_prog_helper(card, helper);
-	if (ret == 0 && (card->model != MODEL_8305))
-		ret = if_cs_prog_real(card, mainfw);
-	if (ret)
-		return;
-
-	/* Now actually get the IRQ */
-	ret = request_irq(card->p_dev->irq, if_cs_interrupt,
-		IRQF_SHARED, DRV_NAME, card);
-	if (ret) {
-		pr_err("error in request_irq\n");
-		return;
-	}
-
-	/*
-	 * Clear any interrupt cause that happened while sending
-	 * firmware/initializing card
-	 */
-	if_cs_write16(card, IF_CS_CARD_INT_CAUSE, IF_CS_BIT_MASK);
-	if_cs_enable_ints(card);
-
-	/* And finally bring the card up */
-	priv->fw_ready = 1;
-	if (lbs_start_card(priv) != 0) {
-		pr_err("could not activate card\n");
-		free_irq(card->p_dev->irq, card);
-	}
-}
-
-
-/********************************************************************/
-/* Callback functions for libertas.ko                               */
-/********************************************************************/
-
-/* Send commands or data packets to the card */
-static int if_cs_host_to_card(struct lbs_private *priv,
-	u8 type,
-	u8 *buf,
-	u16 nb)
-{
-	int ret = -1;
-
-	switch (type) {
-	case MVMS_DAT:
-		priv->dnld_sent = DNLD_DATA_SENT;
-		if_cs_send_data(priv, buf, nb);
-		ret = 0;
-		break;
-	case MVMS_CMD:
-		priv->dnld_sent = DNLD_CMD_SENT;
-		ret = if_cs_send_cmd(priv, buf, nb);
-		break;
-	default:
-		netdev_err(priv->dev, "%s: unsupported type %d\n",
-			   __func__, type);
-	}
-
-	return ret;
-}
-
-
-static void if_cs_release(struct pcmcia_device *p_dev)
-{
-	struct if_cs_card *card = p_dev->priv;
-
-	free_irq(p_dev->irq, card);
-	pcmcia_disable_device(p_dev);
-	if (card->iobase)
-		ioport_unmap(card->iobase);
-}
-
-
-static int if_cs_ioprobe(struct pcmcia_device *p_dev, void *priv_data)
-{
-	p_dev->resource[0]->flags &= ~IO_DATA_PATH_WIDTH;
-	p_dev->resource[0]->flags |= IO_DATA_PATH_WIDTH_AUTO;
-
-	if (p_dev->resource[1]->end) {
-		pr_err("wrong CIS (check number of IO windows)\n");
-		return -ENODEV;
-	}
-
-	/* This reserves IO space but doesn't actually enable it */
-	return pcmcia_request_io(p_dev);
-}
-
-static int if_cs_probe(struct pcmcia_device *p_dev)
-{
-	int ret = -ENOMEM;
-	unsigned int prod_id;
-	struct lbs_private *priv;
-	struct if_cs_card *card;
-
-	card = kzalloc(sizeof(struct if_cs_card), GFP_KERNEL);
-	if (!card)
-		goto out;
-
-	card->p_dev = p_dev;
-	p_dev->priv = card;
-
-	p_dev->config_flags |= CONF_ENABLE_IRQ | CONF_AUTO_SET_IO;
-
-	if (pcmcia_loop_config(p_dev, if_cs_ioprobe, NULL)) {
-		pr_err("error in pcmcia_loop_config\n");
-		goto out1;
-	}
-
-	/*
-	 * Allocate an interrupt line.  Note that this does not assign
-	 * a handler to the interrupt, unless the 'Handler' member of
-	 * the irq structure is initialized.
-	 */
-	if (!p_dev->irq)
-		goto out1;
-
-	/* Initialize io access */
-	card->iobase = ioport_map(p_dev->resource[0]->start,
-				resource_size(p_dev->resource[0]));
-	if (!card->iobase) {
-		pr_err("error in ioport_map\n");
-		ret = -EIO;
-		goto out1;
-	}
-
-	ret = pcmcia_enable_device(p_dev);
-	if (ret) {
-		pr_err("error in pcmcia_enable_device\n");
-		goto out2;
-	}
-
-	/* Finally, report what we've done */
-	lbs_deb_cs("irq %d, io %pR", p_dev->irq, p_dev->resource[0]);
-
-	/*
-	 * Most of the libertas cards can do unaligned register access, but some
-	 * weird ones cannot. That's especially true for the CF8305 card.
-	 */
-	card->align_regs = false;
-
-	card->model = get_model(p_dev->manf_id, p_dev->card_id);
-	if (card->model == MODEL_UNKNOWN) {
-		pr_err("unsupported manf_id 0x%04x / card_id 0x%04x\n",
-		       p_dev->manf_id, p_dev->card_id);
-		ret = -ENODEV;
-		goto out2;
-	}
-
-	/* Check if we have a current silicon */
-	prod_id = if_cs_read8(card, IF_CS_PRODUCT_ID);
-	if (card->model == MODEL_8305) {
-		card->align_regs = true;
-		if (prod_id < IF_CS_CF8305_B1_REV) {
-			pr_err("8305 rev B0 and older are not supported\n");
-			ret = -ENODEV;
-			goto out2;
-		}
-	}
-
-	if ((card->model == MODEL_8381) && prod_id < IF_CS_CF8381_B3_REV) {
-		pr_err("8381 rev B2 and older are not supported\n");
-		ret = -ENODEV;
-		goto out2;
-	}
-
-	if ((card->model == MODEL_8385) && prod_id < IF_CS_CF8385_B1_REV) {
-		pr_err("8385 rev B0 and older are not supported\n");
-		ret = -ENODEV;
-		goto out2;
-	}
-
-	/* Make this card known to the libertas driver */
-	priv = lbs_add_card(card, &p_dev->dev);
-	if (IS_ERR(priv)) {
-		ret = PTR_ERR(priv);
-		goto out2;
-	}
-
-	/* Set up fields in lbs_private */
-	card->priv = priv;
-	priv->card = card;
-	priv->hw_host_to_card = if_cs_host_to_card;
-	priv->enter_deep_sleep = NULL;
-	priv->exit_deep_sleep = NULL;
-	priv->reset_deep_sleep_wakeup = NULL;
-
-	/* Get firmware */
-	ret = lbs_get_firmware_async(priv, &p_dev->dev, card->model, fw_table,
-				     if_cs_prog_firmware);
-	if (ret) {
-		pr_err("failed to find firmware (%d)\n", ret);
-		goto out3;
-	}
-
-	goto out;
-
-out3:
-	lbs_remove_card(priv);
-out2:
-	ioport_unmap(card->iobase);
-out1:
-	pcmcia_disable_device(p_dev);
-out:
-	return ret;
-}
-
-
-static void if_cs_detach(struct pcmcia_device *p_dev)
-{
-	struct if_cs_card *card = p_dev->priv;
-
-	lbs_stop_card(card->priv);
-	lbs_remove_card(card->priv);
-	if_cs_disable_ints(card);
-	if_cs_release(p_dev);
-	kfree(card);
-}
-
-
-
-/********************************************************************/
-/* Module initialization                                            */
-/********************************************************************/
-
-static const struct pcmcia_device_id if_cs_ids[] = {
-	PCMCIA_DEVICE_MANF_CARD(CF8305_MANFID, CF8305_CARDID),
-	PCMCIA_DEVICE_MANF_CARD(CF8381_MANFID, CF8381_CARDID),
-	PCMCIA_DEVICE_MANF_CARD(CF8385_MANFID, CF8385_CARDID),
-	/* NOTE: keep in sync with get_model() */
-	PCMCIA_DEVICE_NULL,
-};
-MODULE_DEVICE_TABLE(pcmcia, if_cs_ids);
-
-static struct pcmcia_driver lbs_driver = {
-	.owner		= THIS_MODULE,
-	.name		= DRV_NAME,
-	.probe		= if_cs_probe,
-	.remove		= if_cs_detach,
-	.id_table       = if_cs_ids,
-};
-module_pcmcia_driver(lbs_driver);
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c
index 63f3d4a5c9aa..2cc2d2788f83 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c
@@ -375,6 +375,7 @@ static int mt7921_load_clc(struct mt792x_dev *dev, const char *fw_name)
 	int ret, i, len, offset = 0;
 	u8 *clc_base = NULL, hw_encap = 0;
 
+	dev->phy.clc_chan_conf = 0xff;
 	if (mt7921_disable_clc ||
 	    mt76_is_usb(&dev->mt76))
 		return 0;
diff --git a/drivers/net/wireless/microchip/wilc1000/cfg80211.c b/drivers/net/wireless/microchip/wilc1000/cfg80211.c
index da52f91693b5..ad2509d8c99a 100644
--- a/drivers/net/wireless/microchip/wilc1000/cfg80211.c
+++ b/drivers/net/wireless/microchip/wilc1000/cfg80211.c
@@ -105,10 +105,9 @@ struct wilc_ch_list_elem {
 } __packed;
 
 static void cfg_scan_result(enum scan_event scan_event,
-			    struct wilc_rcvd_net_info *info, void *user_void)
+			    struct wilc_rcvd_net_info *info,
+			    struct wilc_priv *priv)
 {
-	struct wilc_priv *priv = user_void;
-
 	if (!priv->cfg_scanning)
 		return;
 
@@ -162,9 +161,8 @@ static void cfg_scan_result(enum scan_event scan_event,
 }
 
 static void cfg_connect_result(enum conn_event conn_disconn_evt, u8 mac_status,
-			       void *priv_data)
+			       struct wilc_priv *priv)
 {
-	struct wilc_priv *priv = priv_data;
 	struct net_device *dev = priv->dev;
 	struct wilc_vif *vif = netdev_priv(dev);
 	struct wilc *wl = vif->wilc;
@@ -286,9 +284,8 @@ static int scan(struct wiphy *wiphy, struct cfg80211_scan_request *request)
 	else
 		scan_type = WILC_FW_PASSIVE_SCAN;
 
-	ret = wilc_scan(vif, WILC_FW_USER_SCAN, scan_type, scan_ch_list,
-			request->n_channels, cfg_scan_result, (void *)priv,
-			request);
+	ret = wilc_scan(vif, WILC_FW_USER_SCAN, scan_type,
+			scan_ch_list, cfg_scan_result, request);
 
 	if (ret) {
 		priv->scan_req = NULL;
@@ -412,9 +409,8 @@ static int connect(struct wiphy *wiphy, struct net_device *dev,
 
 	wfi_drv->conn_info.security = security;
 	wfi_drv->conn_info.auth_type = auth_type;
-	wfi_drv->conn_info.ch = ch;
 	wfi_drv->conn_info.conn_result = cfg_connect_result;
-	wfi_drv->conn_info.arg = priv;
+	wfi_drv->conn_info.priv = priv;
 	wfi_drv->conn_info.param = join_params;
 
 	if (sme->mfp == NL80211_MFP_OPTIONAL)
@@ -1094,9 +1090,8 @@ static void wilc_wfi_mgmt_tx_complete(void *priv, int status)
 	kfree(pv_data);
 }
 
-static void wilc_wfi_remain_on_channel_expired(void *data, u64 cookie)
+static void wilc_wfi_remain_on_channel_expired(struct wilc_vif *vif, u64 cookie)
 {
-	struct wilc_vif *vif = data;
 	struct wilc_priv *priv = &vif->priv;
 	struct wilc_wfi_p2p_listen_params *params = &priv->remain_on_ch_params;
 
@@ -1128,9 +1123,8 @@ static int remain_on_channel(struct wiphy *wiphy,
 	if (id == 0)
 		id = ++priv->inc_roc_cookie;
 
-	ret = wilc_remain_on_channel(vif, id, duration, chan->hw_value,
-				     wilc_wfi_remain_on_channel_expired,
-				     (void *)vif);
+	ret = wilc_remain_on_channel(vif, id, chan->hw_value,
+				     wilc_wfi_remain_on_channel_expired);
 	if (ret)
 		return ret;
 
diff --git a/drivers/net/wireless/microchip/wilc1000/hif.c b/drivers/net/wireless/microchip/wilc1000/hif.c
index a28da5938481..839f142663e8 100644
--- a/drivers/net/wireless/microchip/wilc1000/hif.c
+++ b/drivers/net/wireless/microchip/wilc1000/hif.c
@@ -144,18 +144,19 @@ static int handle_scan_done(struct wilc_vif *vif, enum scan_event evt)
 
 	scan_req = &hif_drv->usr_scan_req;
 	if (scan_req->scan_result) {
-		scan_req->scan_result(evt, NULL, scan_req->arg);
+		scan_req->scan_result(evt, NULL, scan_req->priv);
 		scan_req->scan_result = NULL;
 	}
 
 	return result;
 }
 
-int wilc_scan(struct wilc_vif *vif, u8 scan_source, u8 scan_type,
-	      u8 *ch_freq_list, u8 ch_list_len,
+int wilc_scan(struct wilc_vif *vif, u8 scan_source,
+	      u8 scan_type, u8 *ch_freq_list,
 	      void (*scan_result_fn)(enum scan_event,
-				     struct wilc_rcvd_net_info *, void *),
-	      void *user_arg, struct cfg80211_scan_request *request)
+				     struct wilc_rcvd_net_info *,
+				     struct wilc_priv *),
+	      struct cfg80211_scan_request *request)
 {
 	int result = 0;
 	struct wid wid_list[WILC_SCAN_WID_LIST_SIZE];
@@ -164,6 +165,7 @@ int wilc_scan(struct wilc_vif *vif, u8 scan_source, u8 scan_type,
 	u8 *buffer;
 	u8 valuesize = 0;
 	u8 *search_ssid_vals = NULL;
+	const u8 ch_list_len = request->n_channels;
 	struct host_if_drv *hif_drv = vif->hif_drv;
 
 	if (hif_drv->hif_state >= HOST_IF_SCANNING &&
@@ -249,7 +251,7 @@ int wilc_scan(struct wilc_vif *vif, u8 scan_source, u8 scan_type,
 	index++;
 
 	hif_drv->usr_scan_req.scan_result = scan_result_fn;
-	hif_drv->usr_scan_req.arg = user_arg;
+	hif_drv->usr_scan_req.priv = &vif->priv;
 
 	result = wilc_send_config_pkt(vif, WILC_SET_CFG, wid_list, index);
 	if (result) {
@@ -348,7 +350,7 @@ static void handle_connect_timeout(struct work_struct *work)
 	if (hif_drv->conn_info.conn_result) {
 		hif_drv->conn_info.conn_result(CONN_DISCONN_EVENT_CONN_RESP,
 					       WILC_MAC_STATUS_DISCONNECTED,
-					       hif_drv->conn_info.arg);
+					       hif_drv->conn_info.priv);
 
 	} else {
 		netdev_err(vif->ndev, "%s: conn_result is NULL\n", __func__);
@@ -371,8 +373,9 @@ static void handle_connect_timeout(struct work_struct *work)
 	kfree(msg);
 }
 
-void *wilc_parse_join_bss_param(struct cfg80211_bss *bss,
-				struct cfg80211_crypto_settings *crypto)
+struct wilc_join_bss_param *
+wilc_parse_join_bss_param(struct cfg80211_bss *bss,
+			  struct cfg80211_crypto_settings *crypto)
 {
 	struct wilc_join_bss_param *param;
 	struct ieee80211_p2p_noa_attr noa_attr;
@@ -545,7 +548,7 @@ static void handle_rcvd_ntwrk_info(struct work_struct *work)
 
 	if (scan_req->scan_result)
 		scan_req->scan_result(SCAN_EVENT_NETWORK_FOUND, rcvd_info,
-				      scan_req->arg);
+				      scan_req->priv);
 
 done:
 	kfree(rcvd_info->mgmt);
@@ -627,7 +630,7 @@ static inline void host_int_parse_assoc_resp_info(struct wilc_vif *vif,
 
 	del_timer(&hif_drv->connect_timer);
 	conn_info->conn_result(CONN_DISCONN_EVENT_CONN_RESP, mac_status,
-			       hif_drv->conn_info.arg);
+			       hif_drv->conn_info.priv);
 
 	if (mac_status == WILC_MAC_STATUS_CONNECTED &&
 	    conn_info->status == WLAN_STATUS_SUCCESS) {
@@ -657,7 +660,7 @@ void wilc_handle_disconnect(struct wilc_vif *vif)
 
 	if (hif_drv->conn_info.conn_result)
 		hif_drv->conn_info.conn_result(CONN_DISCONN_EVENT_DISCONN_NOTIF,
-					       0, hif_drv->conn_info.arg);
+					       0, hif_drv->conn_info.priv);
 
 	eth_zero_addr(hif_drv->assoc_bssid);
 
@@ -729,7 +732,7 @@ int wilc_disconnect(struct wilc_vif *vif)
 
 	if (scan_req->scan_result) {
 		del_timer(&hif_drv->scan_timer);
-		scan_req->scan_result(SCAN_EVENT_ABORTED, NULL, scan_req->arg);
+		scan_req->scan_result(SCAN_EVENT_ABORTED, NULL, scan_req->priv);
 		scan_req->scan_result = NULL;
 	}
 
@@ -739,7 +742,7 @@ int wilc_disconnect(struct wilc_vif *vif)
 			del_timer(&hif_drv->connect_timer);
 
 		conn_info->conn_result(CONN_DISCONN_EVENT_DISCONN_NOTIF, 0,
-				       conn_info->arg);
+				       conn_info->priv);
 	} else {
 		netdev_err(vif->ndev, "%s: conn_result is NULL\n", __func__);
 	}
@@ -878,7 +881,7 @@ static int handle_remain_on_chan(struct wilc_vif *vif,
 	if (result)
 		return -EBUSY;
 
-	hif_drv->remain_on_ch.arg = hif_remain_ch->arg;
+	hif_drv->remain_on_ch.vif = hif_remain_ch->vif;
 	hif_drv->remain_on_ch.expired = hif_remain_ch->expired;
 	hif_drv->remain_on_ch.ch = hif_remain_ch->ch;
 	hif_drv->remain_on_ch.cookie = hif_remain_ch->cookie;
@@ -915,7 +918,7 @@ static int wilc_handle_roc_expired(struct wilc_vif *vif, u64 cookie)
 		}
 
 		if (hif_drv->remain_on_ch.expired) {
-			hif_drv->remain_on_ch.expired(hif_drv->remain_on_ch.arg,
+			hif_drv->remain_on_ch.expired(hif_drv->remain_on_ch.vif,
 						      cookie);
 		}
 	} else {
@@ -1538,7 +1541,7 @@ int wilc_deinit(struct wilc_vif *vif)
 
 	if (hif_drv->usr_scan_req.scan_result) {
 		hif_drv->usr_scan_req.scan_result(SCAN_EVENT_ABORTED, NULL,
-						  hif_drv->usr_scan_req.arg);
+						  hif_drv->usr_scan_req.priv);
 		hif_drv->usr_scan_req.scan_result = NULL;
 	}
 
@@ -1669,18 +1672,15 @@ void wilc_scan_complete_received(struct wilc *wilc, u8 *buffer, u32 length)
 	}
 }
 
-int wilc_remain_on_channel(struct wilc_vif *vif, u64 cookie,
-			   u32 duration, u16 chan,
-			   void (*expired)(void *, u64),
-			   void *user_arg)
+int wilc_remain_on_channel(struct wilc_vif *vif, u64 cookie, u16 chan,
+			   void (*expired)(struct wilc_vif *, u64))
 {
 	struct wilc_remain_ch roc;
 	int result;
 
 	roc.ch = chan;
 	roc.expired = expired;
-	roc.arg = user_arg;
-	roc.duration = duration;
+	roc.vif = vif;
 	roc.cookie = cookie;
 	result = handle_remain_on_chan(vif, &roc);
 	if (result)
diff --git a/drivers/net/wireless/microchip/wilc1000/hif.h b/drivers/net/wireless/microchip/wilc1000/hif.h
index 8e386db72e45..0d380586b1d9 100644
--- a/drivers/net/wireless/microchip/wilc1000/hif.h
+++ b/drivers/net/wireless/microchip/wilc1000/hif.h
@@ -95,34 +95,37 @@ struct wilc_rcvd_net_info {
 	struct ieee80211_mgmt *mgmt;
 };
 
+struct wilc_priv;
 struct wilc_user_scan_req {
 	void (*scan_result)(enum scan_event evt,
-			    struct wilc_rcvd_net_info *info, void *priv);
-	void *arg;
+			    struct wilc_rcvd_net_info *info,
+			    struct wilc_priv *priv);
+	struct wilc_priv *priv;
 	u32 ch_cnt;
 };
 
+struct wilc_join_bss_param;
 struct wilc_conn_info {
 	u8 bssid[ETH_ALEN];
 	u8 security;
 	enum authtype auth_type;
 	enum mfptype mfp_type;
-	u8 ch;
 	u8 *req_ies;
 	size_t req_ies_len;
 	u8 *resp_ies;
 	u16 resp_ies_len;
 	u16 status;
-	void (*conn_result)(enum conn_event evt, u8 status, void *priv_data);
-	void *arg;
-	void *param;
+	void (*conn_result)(enum conn_event evt, u8 status,
+			    struct wilc_priv *priv);
+	struct wilc_priv *priv;
+	struct wilc_join_bss_param *param;
 };
 
+struct wilc_vif;
 struct wilc_remain_ch {
 	u16 ch;
-	u32 duration;
-	void (*expired)(void *priv, u64 cookie);
-	void *arg;
+	void (*expired)(struct wilc_vif *vif, u64 cookie);
+	struct wilc_vif *vif;
 	u64 cookie;
 };
 
@@ -150,7 +153,6 @@ struct host_if_drv {
 	u8 assoc_resp[WILC_MAX_ASSOC_RESP_FRAME_SIZE];
 };
 
-struct wilc_vif;
 int wilc_add_ptk(struct wilc_vif *vif, const u8 *ptk, u8 ptk_key_len,
 		 const u8 *mac_addr, const u8 *rx_mic, const u8 *tx_mic,
 		 u8 mode, u8 cipher_mode, u8 index);
@@ -171,11 +173,12 @@ int wilc_set_join_req(struct wilc_vif *vif, u8 *bssid, const u8 *ies,
 int wilc_disconnect(struct wilc_vif *vif);
 int wilc_set_mac_chnl_num(struct wilc_vif *vif, u8 channel);
 int wilc_get_rssi(struct wilc_vif *vif, s8 *rssi_level);
-int wilc_scan(struct wilc_vif *vif, u8 scan_source, u8 scan_type,
-	      u8 *ch_freq_list, u8 ch_list_len,
+int wilc_scan(struct wilc_vif *vif, u8 scan_source,
+	      u8 scan_type, u8 *ch_freq_list,
 	      void (*scan_result_fn)(enum scan_event,
-				     struct wilc_rcvd_net_info *, void *),
-	      void *user_arg, struct cfg80211_scan_request *request);
+				     struct wilc_rcvd_net_info *,
+				     struct wilc_priv *),
+	      struct cfg80211_scan_request *request);
 int wilc_hif_set_cfg(struct wilc_vif *vif,
 		     struct cfg_param_attr *cfg_param);
 int wilc_init(struct net_device *dev, struct host_if_drv **hif_drv_handler);
@@ -192,10 +195,8 @@ int wilc_edit_station(struct wilc_vif *vif, const u8 *mac,
 int wilc_set_power_mgmt(struct wilc_vif *vif, bool enabled, u32 timeout);
 int wilc_setup_multicast_filter(struct wilc_vif *vif, u32 enabled, u32 count,
 				u8 *mc_list);
-int wilc_remain_on_channel(struct wilc_vif *vif, u64 cookie,
-			   u32 duration, u16 chan,
-			   void (*expired)(void *, u64),
-			   void *user_arg);
+int wilc_remain_on_channel(struct wilc_vif *vif, u64 cookie, u16 chan,
+			   void (*expired)(struct wilc_vif *, u64));
 int wilc_listen_state_expired(struct wilc_vif *vif, u64 cookie);
 void wilc_frame_register(struct wilc_vif *vif, u16 frame_type, bool reg);
 int wilc_set_operation_mode(struct wilc_vif *vif, int index, u8 mode,
@@ -210,8 +211,9 @@ int wilc_set_external_auth_param(struct wilc_vif *vif,
 void wilc_scan_complete_received(struct wilc *wilc, u8 *buffer, u32 length);
 void wilc_network_info_received(struct wilc *wilc, u8 *buffer, u32 length);
 void wilc_gnrl_async_info_received(struct wilc *wilc, u8 *buffer, u32 length);
-void *wilc_parse_join_bss_param(struct cfg80211_bss *bss,
-				struct cfg80211_crypto_settings *crypto);
+struct wilc_join_bss_param *
+wilc_parse_join_bss_param(struct cfg80211_bss *bss,
+			  struct cfg80211_crypto_settings *crypto);
 int wilc_set_default_mgmt_key_index(struct wilc_vif *vif, u8 index);
 void wilc_handle_disconnect(struct wilc_vif *vif);
 
diff --git a/drivers/net/wireless/microchip/wilc1000/sdio.c b/drivers/net/wireless/microchip/wilc1000/sdio.c
index 87948ba69a22..0d13e3e46e98 100644
--- a/drivers/net/wireless/microchip/wilc1000/sdio.c
+++ b/drivers/net/wireless/microchip/wilc1000/sdio.c
@@ -106,9 +106,10 @@ static int wilc_sdio_cmd53(struct wilc *wilc, struct sdio_cmd53 *cmd)
 		size = cmd->count;
 
 	if (cmd->use_global_buf) {
-		if (size > sizeof(u32))
-			return -EINVAL;
-
+		if (size > sizeof(u32)) {
+			ret = -EINVAL;
+			goto out;
+		}
 		buf = sdio_priv->cmd53_buf;
 	}
 
@@ -123,7 +124,7 @@ static int wilc_sdio_cmd53(struct wilc *wilc, struct sdio_cmd53 *cmd)
 		if (cmd->use_global_buf)
 			memcpy(cmd->buffer, buf, size);
 	}
-
+out:
 	sdio_release_host(func);
 
 	if (ret)
diff --git a/drivers/net/wireless/purelifi/plfxlc/usb.c b/drivers/net/wireless/purelifi/plfxlc/usb.c
index 76d0a778636a..311676c1ece0 100644
--- a/drivers/net/wireless/purelifi/plfxlc/usb.c
+++ b/drivers/net/wireless/purelifi/plfxlc/usb.c
@@ -493,9 +493,12 @@ int plfxlc_usb_wreq_async(struct plfxlc_usb *usb, const u8 *buffer,
 			  void *context)
 {
 	struct usb_device *udev = interface_to_usbdev(usb->ez_usb);
-	struct urb *urb = usb_alloc_urb(0, GFP_ATOMIC);
+	struct urb *urb;
 	int r;
 
+	urb = usb_alloc_urb(0, GFP_ATOMIC);
+	if (!urb)
+		return -ENOMEM;
 	usb_fill_bulk_urb(urb, udev, usb_sndbulkpipe(udev, EP_DATA_OUT),
 			  (void *)buffer, buffer_len, complete_fn, context);
 
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800.h b/drivers/net/wireless/ralink/rt2x00/rt2800.h
index 48521e45577d..8930589b4967 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2800.h
+++ b/drivers/net/wireless/ralink/rt2x00/rt2800.h
@@ -3194,4 +3194,8 @@ enum rt2800_eeprom_word {
  */
 #define BCN_TBTT_OFFSET 64
 
+/* Watchdog type mask */
+#define RT2800_WATCHDOG_HANG		BIT(0)
+#define RT2800_WATCHDOG_DMA_BUSY	BIT(1)
+
 #endif /* RT2800_H */
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800lib.c b/drivers/net/wireless/ralink/rt2x00/rt2800lib.c
index ee880f749b3c..84b218adbaaf 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2800lib.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2800lib.c
@@ -30,9 +30,10 @@
 #include "rt2800lib.h"
 #include "rt2800.h"
 
-static bool modparam_watchdog;
-module_param_named(watchdog, modparam_watchdog, bool, S_IRUGO);
-MODULE_PARM_DESC(watchdog, "Enable watchdog to detect tx/rx hangs and reset hardware if detected");
+static unsigned int modparam_watchdog = RT2800_WATCHDOG_DMA_BUSY;
+module_param_named(watchdog, modparam_watchdog, uint, 0444);
+MODULE_PARM_DESC(watchdog, "Enable watchdog to recover tx/rx hangs.\n"
+		 "\t\t(0=disabled, 1=hang watchdog, 2=DMA watchdog(default), 3=both)");
 
 /*
  * Register access.
@@ -1261,15 +1262,12 @@ static void rt2800_update_survey(struct rt2x00_dev *rt2x00dev)
 	chan_survey->time_ext_busy += rt2800_register_read(rt2x00dev, CH_BUSY_STA_SEC);
 }
 
-void rt2800_watchdog(struct rt2x00_dev *rt2x00dev)
+static bool rt2800_watchdog_hung(struct rt2x00_dev *rt2x00dev)
 {
 	struct data_queue *queue;
 	bool hung_tx = false;
 	bool hung_rx = false;
 
-	if (test_bit(DEVICE_STATE_SCANNING, &rt2x00dev->flags))
-		return;
-
 	rt2800_update_survey(rt2x00dev);
 
 	queue_for_each(rt2x00dev, queue) {
@@ -1297,18 +1295,72 @@ void rt2800_watchdog(struct rt2x00_dev *rt2x00dev)
 		}
 	}
 
+	if (!hung_tx && !hung_rx)
+		return false;
+
 	if (hung_tx)
 		rt2x00_warn(rt2x00dev, "Watchdog TX hung detected\n");
 
 	if (hung_rx)
 		rt2x00_warn(rt2x00dev, "Watchdog RX hung detected\n");
 
-	if (hung_tx || hung_rx) {
-		queue_for_each(rt2x00dev, queue)
-			queue->wd_count = 0;
+	queue_for_each(rt2x00dev, queue)
+		queue->wd_count = 0;
+
+	return true;
+}
+
+static bool rt2800_watchdog_dma_busy(struct rt2x00_dev *rt2x00dev)
+{
+	bool busy_rx, busy_tx;
+	u32 reg_cfg = rt2800_register_read(rt2x00dev, WPDMA_GLO_CFG);
+	u32 reg_int = rt2800_register_read(rt2x00dev, INT_SOURCE_CSR);
+
+	if (rt2x00_get_field32(reg_cfg, WPDMA_GLO_CFG_RX_DMA_BUSY) &&
+	    rt2x00_get_field32(reg_int, INT_SOURCE_CSR_RX_COHERENT))
+		rt2x00dev->rxdma_busy++;
+	else
+		rt2x00dev->rxdma_busy = 0;
+
+	if (rt2x00_get_field32(reg_cfg, WPDMA_GLO_CFG_TX_DMA_BUSY) &&
+	    rt2x00_get_field32(reg_int, INT_SOURCE_CSR_TX_COHERENT))
+		rt2x00dev->txdma_busy++;
+	else
+		rt2x00dev->txdma_busy = 0;
+
+	busy_rx = rt2x00dev->rxdma_busy > 30 ? true : false;
+	busy_tx = rt2x00dev->txdma_busy > 30 ? true : false;
+
+	if (!busy_rx && !busy_tx)
+		return false;
+
+	if (busy_rx)
+		rt2x00_warn(rt2x00dev, "Watchdog RX DMA busy detected\n");
+
+	if (busy_tx)
+		rt2x00_warn(rt2x00dev, "Watchdog TX DMA busy detected\n");
+
+	rt2x00dev->rxdma_busy = 0;
+	rt2x00dev->txdma_busy = 0;
+
+	return true;
+}
+
+void rt2800_watchdog(struct rt2x00_dev *rt2x00dev)
+{
+	bool reset = false;
+
+	if (test_bit(DEVICE_STATE_SCANNING, &rt2x00dev->flags))
+		return;
+
+	if (modparam_watchdog & RT2800_WATCHDOG_DMA_BUSY)
+		reset = rt2800_watchdog_dma_busy(rt2x00dev);
 
+	if (modparam_watchdog & RT2800_WATCHDOG_HANG)
+		reset = rt2800_watchdog_hung(rt2x00dev) || reset;
+
+	if (reset)
 		ieee80211_restart_hw(rt2x00dev->hw);
-	}
 }
 EXPORT_SYMBOL_GPL(rt2800_watchdog);
 
@@ -6048,7 +6100,7 @@ static int rt2800_init_registers(struct rt2x00_dev *rt2x00dev)
 	rt2x00_set_field32(&reg, CCK_PROT_CFG_TX_OP_ALLOW_MM40, 0);
 	rt2x00_set_field32(&reg, CCK_PROT_CFG_TX_OP_ALLOW_GF20, 1);
 	rt2x00_set_field32(&reg, CCK_PROT_CFG_TX_OP_ALLOW_GF40, 0);
-	rt2x00_set_field32(&reg, CCK_PROT_CFG_RTS_TH_EN, 1);
+	rt2x00_set_field32(&reg, CCK_PROT_CFG_RTS_TH_EN, 0);
 	rt2800_register_write(rt2x00dev, CCK_PROT_CFG, reg);
 
 	reg = rt2800_register_read(rt2x00dev, OFDM_PROT_CFG);
@@ -6061,7 +6113,7 @@ static int rt2800_init_registers(struct rt2x00_dev *rt2x00dev)
 	rt2x00_set_field32(&reg, OFDM_PROT_CFG_TX_OP_ALLOW_MM40, 0);
 	rt2x00_set_field32(&reg, OFDM_PROT_CFG_TX_OP_ALLOW_GF20, 1);
 	rt2x00_set_field32(&reg, OFDM_PROT_CFG_TX_OP_ALLOW_GF40, 0);
-	rt2x00_set_field32(&reg, OFDM_PROT_CFG_RTS_TH_EN, 1);
+	rt2x00_set_field32(&reg, OFDM_PROT_CFG_RTS_TH_EN, 0);
 	rt2800_register_write(rt2x00dev, OFDM_PROT_CFG, reg);
 
 	reg = rt2800_register_read(rt2x00dev, MM20_PROT_CFG);
@@ -8659,7 +8711,7 @@ static void rt2800_rxdcoc_calibration(struct rt2x00_dev *rt2x00dev)
 	rt2800_rfcsr_write_bank(rt2x00dev, 5, 4, saverfb5r4);
 	rt2800_rfcsr_write_bank(rt2x00dev, 7, 4, saverfb7r4);
 
-	rt2800_bbp_write(rt2x00dev, 158, 141);
+	rt2800_bbp_write(rt2x00dev, 158, 140);
 	bbpreg = rt2800_bbp_read(rt2x00dev, 159);
 	bbpreg = bbpreg & (~0x40);
 	rt2800_bbp_write(rt2x00dev, 159, bbpreg);
@@ -12013,6 +12065,9 @@ int rt2800_probe_hw(struct rt2x00_dev *rt2x00dev)
 		__set_bit(REQUIRE_TASKLET_CONTEXT, &rt2x00dev->cap_flags);
 	}
 
+	/* USB NICs don't support DMA watchdog as INT_SOURCE_CSR is invalid */
+	if (rt2x00_is_usb(rt2x00dev))
+		modparam_watchdog &= ~RT2800_WATCHDOG_DMA_BUSY;
 	if (modparam_watchdog) {
 		__set_bit(CAPABILITY_RESTART_HW, &rt2x00dev->cap_flags);
 		rt2x00dev->link.watchdog_interval = msecs_to_jiffies(100);
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00.h b/drivers/net/wireless/ralink/rt2x00/rt2x00.h
index aaaf99331967..62fed38f41c0 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2x00.h
+++ b/drivers/net/wireless/ralink/rt2x00/rt2x00.h
@@ -926,6 +926,9 @@ struct rt2x00_dev {
 	 */
 	u16 beacon_int;
 
+	/* Rx/Tx DMA busy watchdog counter */
+	u16 rxdma_busy, txdma_busy;
+
 	/**
 	 * Timestamp of last received beacon
 	 */
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c b/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c
index c88ce446e117..9e7d9dbe954c 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c
@@ -101,6 +101,7 @@ void rt2x00lib_disable_radio(struct rt2x00_dev *rt2x00dev)
 	rt2x00link_stop_tuner(rt2x00dev);
 	rt2x00queue_stop_queues(rt2x00dev);
 	rt2x00queue_flush_queues(rt2x00dev, true);
+	rt2x00queue_stop_queue(rt2x00dev->bcn);
 
 	/*
 	 * Disable radio.
@@ -1286,6 +1287,7 @@ int rt2x00lib_start(struct rt2x00_dev *rt2x00dev)
 	rt2x00dev->intf_ap_count = 0;
 	rt2x00dev->intf_sta_count = 0;
 	rt2x00dev->intf_associated = 0;
+	rt2x00dev->intf_beaconing = 0;
 
 	/* Enable the radio */
 	retval = rt2x00lib_enable_radio(rt2x00dev);
@@ -1312,6 +1314,7 @@ void rt2x00lib_stop(struct rt2x00_dev *rt2x00dev)
 	rt2x00dev->intf_ap_count = 0;
 	rt2x00dev->intf_sta_count = 0;
 	rt2x00dev->intf_associated = 0;
+	rt2x00dev->intf_beaconing = 0;
 }
 
 static inline void rt2x00lib_set_if_combinations(struct rt2x00_dev *rt2x00dev)
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00mac.c b/drivers/net/wireless/ralink/rt2x00/rt2x00mac.c
index 4202c6517783..75fda72c14ca 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2x00mac.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2x00mac.c
@@ -598,6 +598,17 @@ void rt2x00mac_bss_info_changed(struct ieee80211_hw *hw,
 	 */
 	if (changes & BSS_CHANGED_BEACON_ENABLED) {
 		mutex_lock(&intf->beacon_skb_mutex);
+
+		/*
+		 * Clear the 'enable_beacon' flag and clear beacon because
+		 * the beacon queue has been stopped after hardware reset.
+		 */
+		if (test_bit(DEVICE_STATE_RESET, &rt2x00dev->flags) &&
+		    intf->enable_beacon) {
+			intf->enable_beacon = false;
+			rt2x00queue_clear_beacon(rt2x00dev, vif);
+		}
+
 		if (!bss_conf->enable_beacon && intf->enable_beacon) {
 			rt2x00dev->intf_beaconing--;
 			intf->enable_beacon = false;
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192c/phy_common.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192c/phy_common.c
index 3d29c8dbb255..d448efe2c229 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192c/phy_common.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192c/phy_common.c
@@ -1487,12 +1487,8 @@ EXPORT_SYMBOL(rtl92c_phy_lc_calibrate);
 
 void rtl92c_phy_ap_calibrate(struct ieee80211_hw *hw, s8 delta)
 {
-	struct rtl_priv *rtlpriv = rtl_priv(hw);
-	struct rtl_phy *rtlphy = &(rtlpriv->phy);
 	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
 
-	if (rtlphy->apk_done)
-		return;
 	if (IS_92C_SERIAL(rtlhal->version))
 		_rtl92c_phy_ap_calibrate(hw, delta, true);
 	else
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/dm.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/dm.c
index 5a828a934fe9..17486e3f322c 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/dm.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/dm.c
@@ -432,10 +432,8 @@ static void rtl92ee_dm_check_rssi_monitor(struct ieee80211_hw *hw)
 static void rtl92ee_dm_init_primary_cca_check(struct ieee80211_hw *hw)
 {
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
-	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
 	struct dynamic_primary_cca *primarycca = &rtlpriv->primarycca;
 
-	rtlhal->rts_en = 0;
 	primarycca->dup_rts_flag = 0;
 	primarycca->intf_flag = 0;
 	primarycca->intf_type = 0;
@@ -562,7 +560,7 @@ static void rtl92ee_dm_write_dynamic_cca(struct ieee80211_hw *hw,
 	primarycca->mf_state = cur_mf_state;
 }
 
-static void rtl92ee_dm_dynamic_primary_cca_ckeck(struct ieee80211_hw *hw)
+static void rtl92ee_dm_dynamic_primary_cca_check(struct ieee80211_hw *hw)
 {
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct false_alarm_statistics *falsealm_cnt = &rtlpriv->falsealm_cnt;
@@ -615,13 +613,11 @@ static void rtl92ee_dm_dynamic_primary_cca_ckeck(struct ieee80211_hw *hw)
 				rtl92ee_dm_write_dynamic_cca(hw, cur_mf_state);
 				primarycca->pricca_flag = 1;
 				primarycca->dup_rts_flag = 1;
-				rtlpriv->rtlhal.rts_en = 1;
 			} else {
 				primarycca->intf_type = 0;
 				primarycca->intf_flag = 0;
 				cur_mf_state = MF_USC_LSC;
 				rtl92ee_dm_write_dynamic_cca(hw, cur_mf_state);
-				rtlpriv->rtlhal.rts_en = 0;
 				primarycca->dup_rts_flag = 0;
 			}
 		} else if (sec_ch_offset == 1) {
@@ -642,13 +638,11 @@ static void rtl92ee_dm_dynamic_primary_cca_ckeck(struct ieee80211_hw *hw)
 				rtl92ee_dm_write_dynamic_cca(hw, cur_mf_state);
 				primarycca->pricca_flag = 1;
 				primarycca->dup_rts_flag = 1;
-				rtlpriv->rtlhal.rts_en = 1;
 			} else {
 				primarycca->intf_type = 0;
 				primarycca->intf_flag = 0;
 				cur_mf_state = MF_USC_LSC;
 				rtl92ee_dm_write_dynamic_cca(hw, cur_mf_state);
-				rtlpriv->rtlhal.rts_en = 0;
 				primarycca->dup_rts_flag = 0;
 			}
 		}
@@ -660,7 +654,6 @@ static void rtl92ee_dm_dynamic_primary_cca_ckeck(struct ieee80211_hw *hw)
 			cur_mf_state = MF_USC_LSC;
 			/* default */
 			rtl92ee_dm_write_dynamic_cca(hw, cur_mf_state);
-			rtlpriv->rtlhal.rts_en = 0;
 			primarycca->dup_rts_flag = 0;
 			primarycca->intf_type = 0;
 			primarycca->intf_flag = 0;
@@ -1090,7 +1083,7 @@ void rtl92ee_dm_watchdog(struct ieee80211_hw *hw)
 		rtl92ee_dm_refresh_rate_adaptive_mask(hw);
 		rtl92ee_dm_check_edca_turbo(hw);
 		rtl92ee_dm_dynamic_atc_switch(hw);
-		rtl92ee_dm_dynamic_primary_cca_ckeck(hw);
+		rtl92ee_dm_dynamic_primary_cca_check(hw);
 	}
 	spin_unlock(&rtlpriv->locks.rf_ps_lock);
 }
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c
index ebb7abd0c9ad..d4da5cdc8414 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c
@@ -1320,7 +1320,6 @@ int rtl92ee_hw_init(struct ieee80211_hw *hw)
 		err = 1;
 		return err;
 	}
-	rtlhal->rx_tag = 0;
 	rtl_write_word(rtlpriv, REG_PCIE_CTRL_REG, 0x8000);
 	err = rtl92ee_download_fw(hw, false);
 	if (err) {
diff --git a/drivers/net/wireless/realtek/rtlwifi/wifi.h b/drivers/net/wireless/realtek/rtlwifi/wifi.h
index 31a481f43a07..171a461cd812 100644
--- a/drivers/net/wireless/realtek/rtlwifi/wifi.h
+++ b/drivers/net/wireless/realtek/rtlwifi/wifi.h
@@ -1316,7 +1316,6 @@ struct rtl_phy {
 	u8 sw_chnl_stage;
 	u8 sw_chnl_step;
 	u8 current_channel;
-	u8 h2c_box_num;
 	u8 set_io_inprogress;
 	u8 lck_inprogress;
 
@@ -1329,9 +1328,6 @@ struct rtl_phy {
 	s32 reg_ebc;
 	s32 reg_ec4;
 	s32 reg_ecc;
-	u8 rfpienable;
-	u8 reserve_0;
-	u16 reserve_1;
 	u32 reg_c04, reg_c08, reg_874;
 	u32 adda_backup[16];
 	u32 iqk_mac_backup[IQK_MAC_REG_NUM];
@@ -1345,7 +1341,6 @@ struct rtl_phy {
 	struct iqk_matrix_regs iqk_matrix[IQK_MATRIX_SETTINGS_NUM];
 
 	bool rfpi_enable;
-	bool iqk_in_progress;
 
 	u8 pwrgroup_cnt;
 	u8 cck_high_power;
@@ -1383,7 +1378,6 @@ struct rtl_phy {
 			 [MAX_RF_PATH_NUM];
 
 	u32 rfreg_chnlval[2];
-	bool apk_done;
 	u32 reg_rf3c[2];	/* pathA / pathB  */
 
 	u32 backup_rf_0x1a;/*92ee*/
@@ -1395,7 +1389,6 @@ struct rtl_phy {
 	struct phy_parameters hwparam_tables[MAX_TAB];
 	u16 rf_pathmap;
 
-	u8 hw_rof_enable; /*Enable GPIO[9] as WL RF HW PDn source*/
 	enum rt_polarity_ctl polarity_ctl;
 };
 
@@ -1610,7 +1603,6 @@ struct rtl_hal {
 	bool up_first_time;
 	bool first_init;
 	bool being_init_adapter;
-	bool bbrf_ready;
 	bool mac_func_enable;
 	bool pre_edcca_enable;
 	struct bt_coexist_8723 hal_coex_8723;
@@ -1623,9 +1615,7 @@ struct rtl_hal {
 	u8 state;		/*stop 0, start 1 */
 	u8 board_type;
 	u8 package_type;
-	u8 external_pa;
 
-	u8 pa_mode;
 	u8 pa_type_2g;
 	u8 pa_type_5g;
 	u8 lna_type_2g;
@@ -1691,14 +1681,9 @@ struct rtl_hal {
 	bool master_of_dmsp;
 	bool slave_of_dmsp;
 
-	u16 rx_tag;/*for 92ee*/
-	u8 rts_en;
-
 	/*for wowlan*/
-	bool wow_enable;
 	bool enter_pnp_sleep;
 	bool wake_from_pnp_sleep;
-	bool wow_enabled;
 	time64_t last_suspend_sec;
 	u32 wowlan_fwsize;
 	u8 *wowlan_firmware;
diff --git a/drivers/net/wireless/realtek/rtw88/mac80211.c b/drivers/net/wireless/realtek/rtw88/mac80211.c
index a99b53d44267..d8d68f16014e 100644
--- a/drivers/net/wireless/realtek/rtw88/mac80211.c
+++ b/drivers/net/wireless/realtek/rtw88/mac80211.c
@@ -280,9 +280,9 @@ static void rtw_ops_configure_filter(struct ieee80211_hw *hw,
 
 	if (changed_flags & FIF_ALLMULTI) {
 		if (*new_flags & FIF_ALLMULTI)
-			rtwdev->hal.rcr |= BIT_AM | BIT_AB;
+			rtwdev->hal.rcr |= BIT_AM;
 		else
-			rtwdev->hal.rcr &= ~(BIT_AM | BIT_AB);
+			rtwdev->hal.rcr &= ~(BIT_AM);
 	}
 	if (changed_flags & FIF_FCSFAIL) {
 		if (*new_flags & FIF_FCSFAIL)
diff --git a/drivers/net/wireless/realtek/rtw88/tx.c b/drivers/net/wireless/realtek/rtw88/tx.c
index f63900b6621d..c02ac673be32 100644
--- a/drivers/net/wireless/realtek/rtw88/tx.c
+++ b/drivers/net/wireless/realtek/rtw88/tx.c
@@ -656,9 +656,8 @@ void __rtw_tx_work(struct rtw_dev *rtwdev)
 	list_for_each_entry_safe(rtwtxq, tmp, &rtwdev->txqs, list) {
 		struct ieee80211_txq *txq = rtwtxq_to_txq(rtwtxq);
 		unsigned long frame_cnt;
-		unsigned long byte_cnt;
 
-		ieee80211_txq_get_depth(txq, &frame_cnt, &byte_cnt);
+		ieee80211_txq_get_depth(txq, &frame_cnt, NULL);
 		rtw_txq_push(rtwdev, rtwtxq, frame_cnt);
 
 		list_del_init(&rtwtxq->list);
diff --git a/drivers/net/wireless/realtek/rtw89/cam.c b/drivers/net/wireless/realtek/rtw89/cam.c
index f5301c2bbf13..914c94988b2f 100644
--- a/drivers/net/wireless/realtek/rtw89/cam.c
+++ b/drivers/net/wireless/realtek/rtw89/cam.c
@@ -488,6 +488,20 @@ static int rtw89_cam_get_avail_addr_cam(struct rtw89_dev *rtwdev,
 	return 0;
 }
 
+static u8 rtw89_get_addr_cam_entry_size(struct rtw89_dev *rtwdev)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+
+	switch (chip->chip_id) {
+	case RTL8852A:
+	case RTL8852B:
+	case RTL8851B:
+		return ADDR_CAM_ENT_SIZE;
+	default:
+		return ADDR_CAM_ENT_SHORT_SIZE;
+	}
+}
+
 int rtw89_cam_init_addr_cam(struct rtw89_dev *rtwdev,
 			    struct rtw89_addr_cam_entry *addr_cam,
 			    const struct rtw89_bssid_cam_entry *bssid_cam)
@@ -509,7 +523,7 @@ int rtw89_cam_init_addr_cam(struct rtw89_dev *rtwdev,
 	}
 
 	addr_cam->addr_cam_idx = addr_cam_idx;
-	addr_cam->len = ADDR_CAM_ENT_SIZE;
+	addr_cam->len = rtw89_get_addr_cam_entry_size(rtwdev);
 	addr_cam->offset = 0;
 	addr_cam->valid = true;
 	addr_cam->addr_mask = 0;
diff --git a/drivers/net/wireless/realtek/rtw89/coex.c b/drivers/net/wireless/realtek/rtw89/coex.c
index bdcc172639e4..b842cd9a86f8 100644
--- a/drivers/net/wireless/realtek/rtw89/coex.c
+++ b/drivers/net/wireless/realtek/rtw89/coex.c
@@ -1576,13 +1576,13 @@ static void _append_tdma(struct rtw89_dev *rtwdev)
 	if (ver->fcxtdma == 1) {
 		v = (struct rtw89_btc_fbtc_tdma *)&tlv->val[0];
 		tlv->len = sizeof(*v);
-		memcpy(v, &dm->tdma, sizeof(*v));
+		*v = dm->tdma;
 		btc->policy_len += BTC_TLV_HDR_LEN + sizeof(*v);
 	} else {
 		tlv->len = sizeof(*v3);
 		v3 = (struct rtw89_btc_fbtc_tdma_v3 *)&tlv->val[0];
 		v3->fver = ver->fcxtdma;
-		memcpy(&v3->tdma, &dm->tdma, sizeof(v3->tdma));
+		v3->tdma = dm->tdma;
 		btc->policy_len += BTC_TLV_HDR_LEN + sizeof(*v3);
 	}
 
diff --git a/drivers/net/wireless/realtek/rtw89/core.c b/drivers/net/wireless/realtek/rtw89/core.c
index 3d75165e48be..b18f54b16f9a 100644
--- a/drivers/net/wireless/realtek/rtw89/core.c
+++ b/drivers/net/wireless/realtek/rtw89/core.c
@@ -1407,29 +1407,65 @@ static int rtw89_core_rx_process_mac_ppdu(struct rtw89_dev *rtwdev,
 					  struct sk_buff *skb,
 					  struct rtw89_rx_phy_ppdu *phy_ppdu)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
 	const struct rtw89_rxinfo *rxinfo = (const struct rtw89_rxinfo *)skb->data;
+	const struct rtw89_rxinfo_user *user;
+	enum rtw89_chip_gen chip_gen = rtwdev->chip->chip_gen;
+	int rx_cnt_size = RTW89_PPDU_MAC_RX_CNT_SIZE;
 	bool rx_cnt_valid = false;
+	bool invalid = false;
 	u8 plcp_size = 0;
-	u8 usr_num = 0;
 	u8 *phy_sts;
+	u8 usr_num;
+	int i;
+
+	if (chip_gen == RTW89_CHIP_BE) {
+		invalid = le32_get_bits(rxinfo->w0, RTW89_RXINFO_W0_INVALID_V1);
+		rx_cnt_size = RTW89_PPDU_MAC_RX_CNT_SIZE_V1;
+	}
+
+	if (invalid)
+		return -EINVAL;
 
 	rx_cnt_valid = le32_get_bits(rxinfo->w0, RTW89_RXINFO_W0_RX_CNT_VLD);
-	plcp_size = le32_get_bits(rxinfo->w1, RTW89_RXINFO_W1_PLCP_LEN) << 3;
-	usr_num = le32_get_bits(rxinfo->w0, RTW89_RXINFO_W0_USR_NUM);
-	if (usr_num > RTW89_PPDU_MAX_USR) {
-		rtw89_warn(rtwdev, "Invalid user number in mac info\n");
+	if (chip_gen == RTW89_CHIP_BE) {
+		plcp_size = le32_get_bits(rxinfo->w0, RTW89_RXINFO_W0_PLCP_LEN_V1) << 3;
+		usr_num = le32_get_bits(rxinfo->w0, RTW89_RXINFO_W0_USR_NUM_V1);
+	} else {
+		plcp_size = le32_get_bits(rxinfo->w1, RTW89_RXINFO_W1_PLCP_LEN) << 3;
+		usr_num = le32_get_bits(rxinfo->w0, RTW89_RXINFO_W0_USR_NUM);
+	}
+	if (usr_num > chip->ppdu_max_usr) {
+		rtw89_warn(rtwdev, "Invalid user number (%d) in mac info\n",
+			   usr_num);
 		return -EINVAL;
 	}
 
+	/* For WiFi 7 chips, RXWD.mac_id of PPDU status is not set by hardware,
+	 * so update mac_id by rxinfo_user[].mac_id.
+	 */
+	for (i = 0; i < usr_num && chip_gen == RTW89_CHIP_BE; i++) {
+		user = &rxinfo->user[i];
+		if (!le32_get_bits(user->w0, RTW89_RXINFO_USER_MAC_ID_VALID))
+			continue;
+
+		phy_ppdu->mac_id =
+			le32_get_bits(user->w0, RTW89_RXINFO_USER_MACID);
+		break;
+	}
+
 	phy_sts = skb->data + RTW89_PPDU_MAC_INFO_SIZE;
 	phy_sts += usr_num * RTW89_PPDU_MAC_INFO_USR_SIZE;
 	/* 8-byte alignment */
 	if (usr_num & BIT(0))
 		phy_sts += RTW89_PPDU_MAC_INFO_USR_SIZE;
 	if (rx_cnt_valid)
-		phy_sts += RTW89_PPDU_MAC_RX_CNT_SIZE;
+		phy_sts += rx_cnt_size;
 	phy_sts += plcp_size;
 
+	if (phy_sts > skb->data + skb->len)
+		return -EINVAL;
+
 	phy_ppdu->buf = phy_sts;
 	phy_ppdu->len = skb->data + skb->len - phy_sts;
 
@@ -1477,14 +1513,24 @@ static void rtw89_core_rx_process_phy_ppdu_iter(void *data,
 static u16 rtw89_core_get_phy_status_ie_len(struct rtw89_dev *rtwdev,
 					    const struct rtw89_phy_sts_iehdr *iehdr)
 {
-	static const u8 physts_ie_len_tab[32] = {
-		16, 32, 24, 24, 8, 8, 8, 8, VAR_LEN, 8, VAR_LEN, 176, VAR_LEN,
-		VAR_LEN, VAR_LEN, VAR_LEN, VAR_LEN, VAR_LEN, 16, 24, VAR_LEN,
-		VAR_LEN, VAR_LEN, 0, 24, 24, 24, 24, 32, 32, 32, 32
+	static const u8 physts_ie_len_tabs[RTW89_CHIP_GEN_NUM][32] = {
+		[RTW89_CHIP_AX] = {
+			16, 32, 24, 24, 8, 8, 8, 8, VAR_LEN, 8, VAR_LEN, 176, VAR_LEN,
+			VAR_LEN, VAR_LEN, VAR_LEN, VAR_LEN, VAR_LEN, 16, 24, VAR_LEN,
+			VAR_LEN, VAR_LEN, 0, 24, 24, 24, 24, 32, 32, 32, 32
+		},
+		[RTW89_CHIP_BE] = {
+			32, 40, 24, 24, 8, 8, 8, 8, VAR_LEN, 8, VAR_LEN, 176, VAR_LEN,
+			VAR_LEN, VAR_LEN, VAR_LEN, VAR_LEN, VAR_LEN, 16, 24, VAR_LEN,
+			VAR_LEN, VAR_LEN, 0, 24, 24, 24, 24, 32, 32, 32, 32
+		},
 	};
+	const u8 *physts_ie_len_tab;
 	u16 ie_len;
 	u8 ie;
 
+	physts_ie_len_tab = physts_ie_len_tabs[rtwdev->chip->chip_gen];
+
 	ie = le32_get_bits(iehdr->w0, RTW89_PHY_STS_IEHDR_TYPE);
 	if (physts_ie_len_tab[ie] != VAR_LEN)
 		ie_len = physts_ie_len_tab[ie];
@@ -1563,9 +1609,17 @@ static int rtw89_core_rx_process_phy_ppdu(struct rtw89_dev *rtwdev,
 {
 	const struct rtw89_phy_sts_hdr *hdr = phy_ppdu->buf;
 	u32 len_from_header;
+	bool physts_valid;
+
+	physts_valid = le32_get_bits(hdr->w0, RTW89_PHY_STS_HDR_W0_VALID);
+	if (!physts_valid)
+		return -EINVAL;
 
 	len_from_header = le32_get_bits(hdr->w0, RTW89_PHY_STS_HDR_W0_LEN) << 3;
 
+	if (rtwdev->chip->chip_gen == RTW89_CHIP_BE)
+		len_from_header += PHY_STS_HDR_LEN;
+
 	if (len_from_header != phy_ppdu->len) {
 		rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "phy ppdu len mismatch\n");
 		return -EINVAL;
@@ -2052,13 +2106,19 @@ static void rtw89_core_rx_process_ppdu_sts(struct rtw89_dev *rtwdev,
 					     .mac_id = desc_info->mac_id};
 	int ret;
 
-	if (desc_info->mac_info_valid)
-		rtw89_core_rx_process_mac_ppdu(rtwdev, skb, &phy_ppdu);
+	if (desc_info->mac_info_valid) {
+		ret = rtw89_core_rx_process_mac_ppdu(rtwdev, skb, &phy_ppdu);
+		if (ret)
+			goto out;
+	}
+
 	ret = rtw89_core_rx_process_phy_ppdu(rtwdev, &phy_ppdu);
 	if (ret)
-		rtw89_debug(rtwdev, RTW89_DBG_TXRX, "process ppdu failed\n");
+		goto out;
 
 	rtw89_core_rx_process_phy_sts(rtwdev, &phy_ppdu);
+
+out:
 	rtw89_core_rx_pending_skb(rtwdev, &phy_ppdu, desc_info, skb);
 	dev_kfree_skb_any(skb);
 }
@@ -2886,7 +2946,7 @@ void rtw89_roc_end(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
 
 	if (hw->conf.flags & IEEE80211_CONF_IDLE)
 		ieee80211_queue_delayed_work(hw, &roc->roc_work,
-					     RTW89_ROC_IDLE_TIMEOUT);
+					     msecs_to_jiffies(RTW89_ROC_IDLE_TIMEOUT));
 }
 
 void rtw89_roc_work(struct work_struct *work)
@@ -4176,6 +4236,8 @@ static int rtw89_chip_efuse_info_setup(struct rtw89_dev *rtwdev)
 
 	rtw89_core_setup_phycap(rtwdev);
 
+	rtw89_hci_mac_pre_deinit(rtwdev);
+
 	rtw89_mac_pwr_off(rtwdev);
 
 	return 0;
diff --git a/drivers/net/wireless/realtek/rtw89/core.h b/drivers/net/wireless/realtek/rtw89/core.h
index 91e4d4e79eea..07b529a76396 100644
--- a/drivers/net/wireless/realtek/rtw89/core.h
+++ b/drivers/net/wireless/realtek/rtw89/core.h
@@ -2815,10 +2815,10 @@ struct rtw89_ra_info {
 	u8 csi_bw:3;
 };
 
-#define RTW89_PPDU_MAX_USR 4
 #define RTW89_PPDU_MAC_INFO_USR_SIZE 4
 #define RTW89_PPDU_MAC_INFO_SIZE 8
 #define RTW89_PPDU_MAC_RX_CNT_SIZE 96
+#define RTW89_PPDU_MAC_RX_CNT_SIZE_V1 128
 
 #define RTW89_MAX_RX_AGG_NUM 64
 #define RTW89_MAX_TX_AGG_NUM 128
@@ -3064,6 +3064,7 @@ struct rtw89_hci_ops {
 	void (*write32)(struct rtw89_dev *rtwdev, u32 addr, u32 data);
 
 	int (*mac_pre_init)(struct rtw89_dev *rtwdev);
+	int (*mac_pre_deinit)(struct rtw89_dev *rtwdev);
 	int (*mac_post_init)(struct rtw89_dev *rtwdev);
 	int (*deinit)(struct rtw89_dev *rtwdev);
 
@@ -3644,6 +3645,7 @@ struct rtw89_chip_info {
 	u8 bacam_num;
 	u8 bacam_dynamic_num;
 	enum rtw89_bacam_ver bacam_ver;
+	u8 ppdu_max_usr;
 
 	u8 sec_ctrl_efuse_size;
 	u32 physical_efuse_size;
@@ -4802,6 +4804,11 @@ static inline void rtw89_hci_tx_kick_off(struct rtw89_dev *rtwdev, u8 txch)
 	return rtwdev->hci.ops->tx_kick_off(rtwdev, txch);
 }
 
+static inline int rtw89_hci_mac_pre_deinit(struct rtw89_dev *rtwdev)
+{
+	return rtwdev->hci.ops->mac_pre_deinit(rtwdev);
+}
+
 static inline void rtw89_hci_flush_queues(struct rtw89_dev *rtwdev, u32 queues,
 					  bool drop)
 {
diff --git a/drivers/net/wireless/realtek/rtw89/mac.h b/drivers/net/wireless/realtek/rtw89/mac.h
index c11c904f87fe..cd2e9b850c72 100644
--- a/drivers/net/wireless/realtek/rtw89/mac.h
+++ b/drivers/net/wireless/realtek/rtw89/mac.h
@@ -10,6 +10,7 @@
 
 #define MAC_MEM_DUMP_PAGE_SIZE 0x40000
 #define ADDR_CAM_ENT_SIZE  0x40
+#define ADDR_CAM_ENT_SHORT_SIZE 0x20
 #define BSSID_CAM_ENT_SIZE 0x08
 #define HFC_PAGE_UNIT 64
 #define RPWM_TRY_CNT 3
diff --git a/drivers/net/wireless/realtek/rtw89/pci.c b/drivers/net/wireless/realtek/rtw89/pci.c
index 14ddb0d39e63..0ca07ae63594 100644
--- a/drivers/net/wireless/realtek/rtw89/pci.c
+++ b/drivers/net/wireless/realtek/rtw89/pci.c
@@ -19,28 +19,25 @@ MODULE_PARM_DESC(disable_clkreq, "Set Y to disable PCI clkreq support");
 MODULE_PARM_DESC(disable_aspm_l1, "Set Y to disable PCI ASPM L1 support");
 MODULE_PARM_DESC(disable_aspm_l1ss, "Set Y to disable PCI L1SS support");
 
-static int rtw89_pci_rst_bdram_pcie(struct rtw89_dev *rtwdev)
+static int rtw89_pci_rst_bdram_ax(struct rtw89_dev *rtwdev)
 {
 	u32 val;
 	int ret;
 
-	rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1,
-		      rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) | B_AX_RST_BDRAM);
+	rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RST_BDRAM);
 
 	ret = read_poll_timeout_atomic(rtw89_read32, val, !(val & B_AX_RST_BDRAM),
 				       1, RTW89_PCI_POLL_BDRAM_RST_CNT, false,
 				       rtwdev, R_AX_PCIE_INIT_CFG1);
 
-	if (ret)
-		return -EBUSY;
-
-	return 0;
+	return ret;
 }
 
 static u32 rtw89_pci_dma_recalc(struct rtw89_dev *rtwdev,
 				struct rtw89_pci_dma_ring *bd_ring,
 				u32 cur_idx, bool tx)
 {
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
 	u32 cnt, cur_rp, wp, rp, len;
 
 	rp = bd_ring->rp;
@@ -48,10 +45,14 @@ static u32 rtw89_pci_dma_recalc(struct rtw89_dev *rtwdev,
 	len = bd_ring->len;
 
 	cur_rp = FIELD_GET(TXBD_HW_IDX_MASK, cur_idx);
-	if (tx)
+	if (tx) {
 		cnt = cur_rp >= rp ? cur_rp - rp : len - (rp - cur_rp);
-	else
+	} else {
+		if (info->rx_ring_eq_is_full)
+			wp += 1;
+
 		cnt = cur_rp >= wp ? cur_rp - wp : len - (wp - cur_rp);
+	}
 
 	bd_ring->rp = cur_rp;
 
@@ -226,6 +227,21 @@ rtw89_skb_put_rx_data(struct rtw89_dev *rtwdev, bool fs, bool ls,
 	return true;
 }
 
+static u32 rtw89_pci_get_rx_skb_idx(struct rtw89_dev *rtwdev,
+				    struct rtw89_pci_dma_ring *bd_ring)
+{
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
+	u32 wp = bd_ring->wp;
+
+	if (!info->rx_ring_eq_is_full)
+		return wp;
+
+	if (++wp >= bd_ring->len)
+		wp = 0;
+
+	return wp;
+}
+
 static u32 rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev *rtwdev,
 				       struct rtw89_pci_rx_ring *rx_ring)
 {
@@ -235,12 +251,14 @@ static u32 rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev *rtwdev,
 	struct sk_buff *new = rx_ring->diliver_skb;
 	struct sk_buff *skb;
 	u32 rxinfo_size = sizeof(struct rtw89_pci_rxbd_info);
+	u32 skb_idx;
 	u32 offset;
 	u32 cnt = 1;
 	bool fs, ls;
 	int ret;
 
-	skb = rx_ring->buf[bd_ring->wp];
+	skb_idx = rtw89_pci_get_rx_skb_idx(rtwdev, bd_ring);
+	skb = rx_ring->buf[skb_idx];
 	rtw89_pci_sync_skb_for_cpu(rtwdev, skb);
 
 	ret = rtw89_pci_rxbd_info_update(rtwdev, skb);
@@ -525,10 +543,12 @@ static u32 rtw89_pci_release_tx_skbs(struct rtw89_dev *rtwdev,
 	u32 cnt = 0;
 	u32 rpp_size = sizeof(struct rtw89_pci_rpp_fmt);
 	u32 rxinfo_size = sizeof(struct rtw89_pci_rxbd_info);
+	u32 skb_idx;
 	u32 offset;
 	int ret;
 
-	skb = rx_ring->buf[bd_ring->wp];
+	skb_idx = rtw89_pci_get_rx_skb_idx(rtwdev, bd_ring);
+	skb = rx_ring->buf[skb_idx];
 	rtw89_pci_sync_skb_for_cpu(rtwdev, skb);
 
 	ret = rtw89_pci_rxbd_info_update(rtwdev, skb);
@@ -676,11 +696,26 @@ void rtw89_pci_recognize_intrs_v1(struct rtw89_dev *rtwdev,
 }
 EXPORT_SYMBOL(rtw89_pci_recognize_intrs_v1);
 
-static void rtw89_pci_clear_isr0(struct rtw89_dev *rtwdev, u32 isr00)
+void rtw89_pci_recognize_intrs_v2(struct rtw89_dev *rtwdev,
+				  struct rtw89_pci *rtwpci,
+				  struct rtw89_pci_isrs *isrs)
 {
-	/* write 1 clear */
-	rtw89_write32(rtwdev, R_AX_PCIE_HISR00, isr00);
+	isrs->ind_isrs = rtw89_read32(rtwdev, R_BE_PCIE_HISR) & rtwpci->ind_intrs;
+	isrs->halt_c2h_isrs = isrs->ind_isrs & B_BE_HS0ISR_IND_INT ?
+			      rtw89_read32(rtwdev, R_BE_HISR0) & rtwpci->halt_c2h_intrs : 0;
+	isrs->isrs[0] = isrs->ind_isrs & B_BE_HCI_AXIDMA_INT ?
+			rtw89_read32(rtwdev, R_BE_HAXI_HISR00) & rtwpci->intrs[0] : 0;
+	isrs->isrs[1] = rtw89_read32(rtwdev, R_BE_PCIE_DMA_ISR);
+
+	if (isrs->halt_c2h_isrs)
+		rtw89_write32(rtwdev, R_BE_HISR0, isrs->halt_c2h_isrs);
+	if (isrs->isrs[0])
+		rtw89_write32(rtwdev, R_BE_HAXI_HISR00, isrs->isrs[0]);
+	if (isrs->isrs[1])
+		rtw89_write32(rtwdev, R_BE_PCIE_DMA_ISR, isrs->isrs[1]);
+	rtw89_write32(rtwdev, R_BE_PCIE_HISR, isrs->ind_isrs);
 }
+EXPORT_SYMBOL(rtw89_pci_recognize_intrs_v2);
 
 void rtw89_pci_enable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
 {
@@ -713,6 +748,22 @@ void rtw89_pci_disable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpc
 }
 EXPORT_SYMBOL(rtw89_pci_disable_intr_v1);
 
+void rtw89_pci_enable_intr_v2(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
+{
+	rtw89_write32(rtwdev, R_BE_HIMR0, rtwpci->halt_c2h_intrs);
+	rtw89_write32(rtwdev, R_BE_HAXI_HIMR00, rtwpci->intrs[0]);
+	rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, rtwpci->intrs[1]);
+	rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, rtwpci->ind_intrs);
+}
+EXPORT_SYMBOL(rtw89_pci_enable_intr_v2);
+
+void rtw89_pci_disable_intr_v2(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
+{
+	rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, 0);
+	rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, 0);
+}
+EXPORT_SYMBOL(rtw89_pci_disable_intr_v2);
+
 static void rtw89_pci_ops_recovery_start(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
@@ -753,6 +804,8 @@ static irqreturn_t rtw89_pci_interrupt_threadfn(int irq, void *dev)
 {
 	struct rtw89_dev *rtwdev = dev;
 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
+	const struct rtw89_pci_gen_def *gen_def = info->gen_def;
 	struct rtw89_pci_isrs isrs;
 	unsigned long flags;
 
@@ -760,13 +813,13 @@ static irqreturn_t rtw89_pci_interrupt_threadfn(int irq, void *dev)
 	rtw89_chip_recognize_intrs(rtwdev, rtwpci, &isrs);
 	spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
 
-	if (unlikely(isrs.isrs[0] & B_AX_RDU_INT))
+	if (unlikely(isrs.isrs[0] & gen_def->isr_rdu))
 		rtw89_pci_isr_rxd_unavail(rtwdev, rtwpci);
 
-	if (unlikely(isrs.halt_c2h_isrs & B_AX_HALT_C2H_INT_EN))
+	if (unlikely(isrs.halt_c2h_isrs & gen_def->isr_halt_c2h))
 		rtw89_ser_notify(rtwdev, rtw89_mac_get_err_status(rtwdev));
 
-	if (unlikely(isrs.halt_c2h_isrs & B_AX_WDT_TIMEOUT_INT_EN))
+	if (unlikely(isrs.halt_c2h_isrs & gen_def->isr_wdt_timeout))
 		rtw89_ser_notify(rtwdev, MAC_AX_ERR_L2_ERR_WDT_TIMEOUT_INT);
 
 	if (unlikely(rtwpci->under_recovery))
@@ -817,6 +870,15 @@ static irqreturn_t rtw89_pci_interrupt_handler(int irq, void *dev)
 	return irqret;
 }
 
+#define DEF_TXCHADDRS_TYPE2(gen, ch_idx, txch, v...) \
+	[RTW89_TXCH_##ch_idx] = { \
+		.num = R_##gen##_##txch##_TXBD_NUM ##v, \
+		.idx = R_##gen##_##txch##_TXBD_IDX ##v, \
+		.bdram = 0, \
+		.desa_l = R_##gen##_##txch##_TXBD_DESA_L ##v, \
+		.desa_h = R_##gen##_##txch##_TXBD_DESA_H ##v, \
+	}
+
 #define DEF_TXCHADDRS_TYPE1(info, txch, v...) \
 	[RTW89_TXCH_##txch] = { \
 		.num = R_AX_##txch##_TXBD_NUM ##v, \
@@ -835,12 +897,12 @@ static irqreturn_t rtw89_pci_interrupt_handler(int irq, void *dev)
 		.desa_h = R_AX_##txch##_TXBD_DESA_H ##v, \
 	}
 
-#define DEF_RXCHADDRS(info, rxch, v...) \
-	[RTW89_RXCH_##rxch] = { \
-		.num = R_AX_##rxch##_RXBD_NUM ##v, \
-		.idx = R_AX_##rxch##_RXBD_IDX ##v, \
-		.desa_l = R_AX_##rxch##_RXBD_DESA_L ##v, \
-		.desa_h = R_AX_##rxch##_RXBD_DESA_H ##v, \
+#define DEF_RXCHADDRS(gen, ch_idx, rxch, v...) \
+	[RTW89_RXCH_##ch_idx] = { \
+		.num = R_##gen##_##rxch##_RXBD_NUM ##v, \
+		.idx = R_##gen##_##rxch##_RXBD_IDX ##v, \
+		.desa_l = R_##gen##_##rxch##_RXBD_DESA_L ##v, \
+		.desa_h = R_##gen##_##rxch##_RXBD_DESA_H ##v, \
 	}
 
 const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set = {
@@ -860,8 +922,8 @@ const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set = {
 		DEF_TXCHADDRS(info, CH12),
 	},
 	.rx = {
-		DEF_RXCHADDRS(info, RXQ),
-		DEF_RXCHADDRS(info, RPQ),
+		DEF_RXCHADDRS(AX, RXQ, RXQ),
+		DEF_RXCHADDRS(AX, RPQ, RPQ),
 	},
 };
 EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set);
@@ -883,12 +945,35 @@ const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_v1 = {
 		DEF_TXCHADDRS(info, CH12, _V1),
 	},
 	.rx = {
-		DEF_RXCHADDRS(info, RXQ, _V1),
-		DEF_RXCHADDRS(info, RPQ, _V1),
+		DEF_RXCHADDRS(AX, RXQ, RXQ, _V1),
+		DEF_RXCHADDRS(AX, RPQ, RPQ, _V1),
 	},
 };
 EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set_v1);
 
+const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_be = {
+	.tx = {
+		DEF_TXCHADDRS_TYPE2(BE, ACH0, CH0, _V1),
+		DEF_TXCHADDRS_TYPE2(BE, ACH1, CH1, _V1),
+		DEF_TXCHADDRS_TYPE2(BE, ACH2, CH2, _V1),
+		DEF_TXCHADDRS_TYPE2(BE, ACH3, CH3, _V1),
+		DEF_TXCHADDRS_TYPE2(BE, ACH4, CH4, _V1),
+		DEF_TXCHADDRS_TYPE2(BE, ACH5, CH5, _V1),
+		DEF_TXCHADDRS_TYPE2(BE, ACH6, CH6, _V1),
+		DEF_TXCHADDRS_TYPE2(BE, ACH7, CH7, _V1),
+		DEF_TXCHADDRS_TYPE2(BE, CH8, CH8, _V1),
+		DEF_TXCHADDRS_TYPE2(BE, CH9, CH9, _V1),
+		DEF_TXCHADDRS_TYPE2(BE, CH10, CH10, _V1),
+		DEF_TXCHADDRS_TYPE2(BE, CH11, CH11, _V1),
+		DEF_TXCHADDRS_TYPE2(BE, CH12, CH12, _V1),
+	},
+	.rx = {
+		DEF_RXCHADDRS(BE, RXQ, RXQ0, _V1),
+		DEF_RXCHADDRS(BE, RPQ, RPQ0, _V1),
+	},
+};
+EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set_be);
+
 #undef DEF_TXCHADDRS_TYPE1
 #undef DEF_TXCHADDRS
 #undef DEF_RXCHADDRS
@@ -1422,6 +1507,7 @@ static void rtw89_pci_reset_trx_rings(struct rtw89_dev *rtwdev)
 	struct rtw89_pci_dma_ring *bd_ring;
 	const struct rtw89_pci_bd_ram *bd_ram;
 	u32 addr_num;
+	u32 addr_idx;
 	u32 addr_bdram;
 	u32 addr_desa_l;
 	u32 val32;
@@ -1433,19 +1519,21 @@ static void rtw89_pci_reset_trx_rings(struct rtw89_dev *rtwdev)
 
 		tx_ring = &rtwpci->tx_rings[i];
 		bd_ring = &tx_ring->bd_ring;
-		bd_ram = &bd_ram_table[i];
+		bd_ram = bd_ram_table ? &bd_ram_table[i] : NULL;
 		addr_num = bd_ring->addr.num;
 		addr_bdram = bd_ring->addr.bdram;
 		addr_desa_l = bd_ring->addr.desa_l;
 		bd_ring->wp = 0;
 		bd_ring->rp = 0;
 
-		val32 = FIELD_PREP(BDRAM_SIDX_MASK, bd_ram->start_idx) |
-			FIELD_PREP(BDRAM_MAX_MASK, bd_ram->max_num) |
-			FIELD_PREP(BDRAM_MIN_MASK, bd_ram->min_num);
-
 		rtw89_write16(rtwdev, addr_num, bd_ring->len);
-		rtw89_write32(rtwdev, addr_bdram, val32);
+		if (addr_bdram && bd_ram) {
+			val32 = FIELD_PREP(BDRAM_SIDX_MASK, bd_ram->start_idx) |
+				FIELD_PREP(BDRAM_MAX_MASK, bd_ram->max_num) |
+				FIELD_PREP(BDRAM_MIN_MASK, bd_ram->min_num);
+
+			rtw89_write32(rtwdev, addr_bdram, val32);
+		}
 		rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma);
 	}
 
@@ -1453,14 +1541,21 @@ static void rtw89_pci_reset_trx_rings(struct rtw89_dev *rtwdev)
 		rx_ring = &rtwpci->rx_rings[i];
 		bd_ring = &rx_ring->bd_ring;
 		addr_num = bd_ring->addr.num;
+		addr_idx = bd_ring->addr.idx;
 		addr_desa_l = bd_ring->addr.desa_l;
-		bd_ring->wp = 0;
+		if (info->rx_ring_eq_is_full)
+			bd_ring->wp = bd_ring->len - 1;
+		else
+			bd_ring->wp = 0;
 		bd_ring->rp = 0;
 		rx_ring->diliver_skb = NULL;
 		rx_ring->diliver_desc.ready = false;
 
 		rtw89_write16(rtwdev, addr_num, bd_ring->len);
 		rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma);
+
+		if (info->rx_ring_eq_is_full)
+			rtw89_write16(rtwdev, addr_idx, bd_ring->wp);
 	}
 }
 
@@ -1471,7 +1566,7 @@ static void rtw89_pci_release_tx_ring(struct rtw89_dev *rtwdev,
 	rtw89_pci_release_pending_txwd_skb(rtwdev, tx_ring);
 }
 
-static void rtw89_pci_ops_reset(struct rtw89_dev *rtwdev)
+void rtw89_pci_ops_reset(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
 	const struct rtw89_pci_info *info = rtwdev->pci_info;
@@ -1684,24 +1779,16 @@ static void rtw89_pci_ctrl_dma_trx(struct rtw89_dev *rtwdev, bool enable)
 
 static void rtw89_pci_ctrl_dma_io(struct rtw89_dev *rtwdev, bool enable)
 {
-	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
-	u32 reg, mask;
-
-	if (chip_id == RTL8852C) {
-		reg = R_AX_HAXI_INIT_CFG1;
-		mask = B_AX_STOP_AXI_MST;
-	} else {
-		reg = R_AX_PCIE_DMA_STOP1;
-		mask = B_AX_STOP_PCIEIO;
-	}
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
+	const struct rtw89_reg_def *reg = &info->dma_io_stop;
 
 	if (enable)
-		rtw89_write32_clr(rtwdev, reg, mask);
+		rtw89_write32_clr(rtwdev, reg->addr, reg->mask);
 	else
-		rtw89_write32_set(rtwdev, reg, mask);
+		rtw89_write32_set(rtwdev, reg->addr, reg->mask);
 }
 
-static void rtw89_pci_ctrl_dma_all(struct rtw89_dev *rtwdev, bool enable)
+void rtw89_pci_ctrl_dma_all(struct rtw89_dev *rtwdev, bool enable)
 {
 	rtw89_pci_ctrl_dma_io(rtwdev, enable);
 	rtw89_pci_ctrl_dma_trx(rtwdev, enable);
@@ -2303,7 +2390,7 @@ static void rtw89_pci_set_keep_reg(struct rtw89_dev *rtwdev)
 			  B_AX_PCIE_TXRST_KEEP_REG | B_AX_PCIE_RXRST_KEEP_REG);
 }
 
-static void rtw89_pci_clr_idx_all(struct rtw89_dev *rtwdev)
+static void rtw89_pci_clr_idx_all_ax(struct rtw89_dev *rtwdev)
 {
 	const struct rtw89_pci_info *info = rtwdev->pci_info;
 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
@@ -2491,7 +2578,7 @@ static int rtw89_pci_ops_deinit(struct rtw89_dev *rtwdev)
 	return 0;
 }
 
-static int rtw89_pci_ops_mac_pre_init(struct rtw89_dev *rtwdev)
+static int rtw89_pci_ops_mac_pre_init_ax(struct rtw89_dev *rtwdev)
 {
 	const struct rtw89_pci_info *info = rtwdev->pci_info;
 	int ret;
@@ -2550,7 +2637,7 @@ static int rtw89_pci_ops_mac_pre_init(struct rtw89_dev *rtwdev)
 	/* fill TRX BD indexes */
 	rtw89_pci_ops_reset(rtwdev);
 
-	ret = rtw89_pci_rst_bdram_pcie(rtwdev);
+	ret = rtw89_pci_rst_bdram_ax(rtwdev);
 	if (ret) {
 		rtw89_warn(rtwdev, "reset bdram busy\n");
 		return ret;
@@ -2648,7 +2735,7 @@ int rtw89_pci_ltr_set_v1(struct rtw89_dev *rtwdev, bool en)
 }
 EXPORT_SYMBOL(rtw89_pci_ltr_set_v1);
 
-static int rtw89_pci_ops_mac_post_init(struct rtw89_dev *rtwdev)
+static int rtw89_pci_ops_mac_post_init_ax(struct rtw89_dev *rtwdev)
 {
 	const struct rtw89_pci_info *info = rtwdev->pci_info;
 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
@@ -3026,6 +3113,7 @@ static int rtw89_pci_alloc_rx_ring(struct rtw89_dev *rtwdev,
 				   struct rtw89_pci_rx_ring *rx_ring,
 				   u32 desc_size, u32 len, u32 rxch)
 {
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
 	const struct rtw89_pci_ch_dma_addr *rxch_addr;
 	struct sk_buff *skb;
 	u8 *head;
@@ -3052,7 +3140,10 @@ static int rtw89_pci_alloc_rx_ring(struct rtw89_dev *rtwdev,
 	rx_ring->bd_ring.len = len;
 	rx_ring->bd_ring.desc_size = desc_size;
 	rx_ring->bd_ring.addr = *rxch_addr;
-	rx_ring->bd_ring.wp = 0;
+	if (info->rx_ring_eq_is_full)
+		rx_ring->bd_ring.wp = len - 1;
+	else
+		rx_ring->bd_ring.wp = 0;
 	rx_ring->bd_ring.rp = 0;
 	rx_ring->buf_sz = buf_sz;
 	rx_ring->diliver_skb = NULL;
@@ -3289,6 +3380,55 @@ void rtw89_pci_config_intr_mask_v1(struct rtw89_dev *rtwdev)
 }
 EXPORT_SYMBOL(rtw89_pci_config_intr_mask_v1);
 
+static void rtw89_pci_recovery_intr_mask_v2(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
+
+	rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0;
+	rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN;
+	rtwpci->intrs[0] = 0;
+	rtwpci->intrs[1] = B_BE_PCIE_RX_RX0P2_IMR0_V1 |
+			   B_BE_PCIE_RX_RPQ0_IMR0_V1;
+}
+
+static void rtw89_pci_default_intr_mask_v2(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
+
+	rtwpci->ind_intrs = B_BE_HCI_AXIDMA_INT_EN0 |
+			    B_BE_HS0_IND_INT_EN0;
+	rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN;
+	rtwpci->intrs[0] = B_BE_RDU_CH1_INT_IMR_V1 |
+			   B_BE_RDU_CH0_INT_IMR_V1;
+	rtwpci->intrs[1] = B_BE_PCIE_RX_RX0P2_IMR0_V1 |
+			   B_BE_PCIE_RX_RPQ0_IMR0_V1;
+}
+
+static void rtw89_pci_low_power_intr_mask_v2(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
+
+	rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0 |
+			    B_BE_HS1_IND_INT_EN0;
+	rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN;
+	rtwpci->intrs[0] = 0;
+	rtwpci->intrs[1] = B_BE_PCIE_RX_RX0P2_IMR0_V1 |
+			   B_BE_PCIE_RX_RPQ0_IMR0_V1;
+}
+
+void rtw89_pci_config_intr_mask_v2(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
+
+	if (rtwpci->under_recovery)
+		rtw89_pci_recovery_intr_mask_v2(rtwdev);
+	else if (rtwpci->low_power)
+		rtw89_pci_low_power_intr_mask_v2(rtwdev);
+	else
+		rtw89_pci_default_intr_mask_v2(rtwdev);
+}
+EXPORT_SYMBOL(rtw89_pci_config_intr_mask_v2);
+
 static int rtw89_pci_request_irq(struct rtw89_dev *rtwdev,
 				 struct pci_dev *pdev)
 {
@@ -3480,19 +3620,27 @@ static void rtw89_pci_aspm_set(struct rtw89_dev *rtwdev, bool enable)
 
 static void rtw89_pci_recalc_int_mit(struct rtw89_dev *rtwdev)
 {
+	enum rtw89_chip_gen chip_gen = rtwdev->chip->chip_gen;
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
 	struct rtw89_traffic_stats *stats = &rtwdev->stats;
 	enum rtw89_tfc_lv tx_tfc_lv = stats->tx_tfc_lv;
 	enum rtw89_tfc_lv rx_tfc_lv = stats->rx_tfc_lv;
 	u32 val = 0;
 
-	if (!rtwdev->scanning &&
-	    (tx_tfc_lv >= RTW89_TFC_HIGH || rx_tfc_lv >= RTW89_TFC_HIGH))
+	if (rtwdev->scanning ||
+	    (tx_tfc_lv < RTW89_TFC_HIGH && rx_tfc_lv < RTW89_TFC_HIGH))
+		goto out;
+
+	if (chip_gen == RTW89_CHIP_BE)
+		val = B_BE_PCIE_MIT_RX0P2_EN | B_BE_PCIE_MIT_RX0P1_EN;
+	else
 		val = B_AX_RXMIT_RXP2_SEL | B_AX_RXMIT_RXP1_SEL |
 		      FIELD_PREP(B_AX_RXCOUNTER_MATCH_MASK, RTW89_PCI_RXBD_NUM_MAX / 2) |
 		      FIELD_PREP(B_AX_RXTIMER_UNIT_MASK, AX_RXTIMER_UNIT_64US) |
 		      FIELD_PREP(B_AX_RXTIMER_MATCH_MASK, 2048 / 64);
 
-	rtw89_write32(rtwdev, R_AX_INT_MIT_RX, val);
+out:
+	rtw89_write32(rtwdev, info->mit_addr, val);
 }
 
 static void rtw89_pci_link_cfg(struct rtw89_dev *rtwdev)
@@ -3582,7 +3730,7 @@ static void rtw89_pci_l1ss_cfg(struct rtw89_dev *rtwdev)
 		rtw89_pci_l1ss_set(rtwdev, true);
 }
 
-static int rtw89_pci_poll_io_idle(struct rtw89_dev *rtwdev)
+static int rtw89_pci_poll_io_idle_ax(struct rtw89_dev *rtwdev)
 {
 	int ret = 0;
 	u32 sts;
@@ -3599,7 +3747,7 @@ static int rtw89_pci_poll_io_idle(struct rtw89_dev *rtwdev)
 	return ret;
 }
 
-static int rtw89_pci_lv1rst_stop_dma(struct rtw89_dev *rtwdev)
+static int rtw89_pci_lv1rst_stop_dma_ax(struct rtw89_dev *rtwdev)
 {
 	u32 val;
 	int ret;
@@ -3608,7 +3756,7 @@ static int rtw89_pci_lv1rst_stop_dma(struct rtw89_dev *rtwdev)
 		return 0;
 
 	rtw89_pci_ctrl_dma_all(rtwdev, false);
-	ret = rtw89_pci_poll_io_idle(rtwdev);
+	ret = rtw89_pci_poll_io_idle_ax(rtwdev);
 	if (ret) {
 		val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
 		rtw89_debug(rtwdev, RTW89_DBG_HCI,
@@ -3619,7 +3767,7 @@ static int rtw89_pci_lv1rst_stop_dma(struct rtw89_dev *rtwdev)
 		if (val & B_AX_RX_STUCK)
 			rtw89_mac_ctrl_hci_dma_rx(rtwdev, false);
 		rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
-		ret = rtw89_pci_poll_io_idle(rtwdev);
+		ret = rtw89_pci_poll_io_idle_ax(rtwdev);
 		val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
 		rtw89_debug(rtwdev, RTW89_DBG_HCI,
 			    "[PCIe] poll_io_idle fail, after 0x%08x: 0x%08x\n",
@@ -3629,23 +3777,7 @@ static int rtw89_pci_lv1rst_stop_dma(struct rtw89_dev *rtwdev)
 	return ret;
 }
 
-
-
-static int rtw89_pci_rst_bdram(struct rtw89_dev *rtwdev)
-{
-	int ret = 0;
-	u32 val32, sts;
-
-	val32 = B_AX_RST_BDRAM;
-	rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
-
-	ret = read_poll_timeout_atomic(rtw89_read32, sts,
-				       (sts & B_AX_RST_BDRAM) == 0x0, 1, 100,
-				       true, rtwdev, R_AX_PCIE_INIT_CFG1);
-	return ret;
-}
-
-static int rtw89_pci_lv1rst_start_dma(struct rtw89_dev *rtwdev)
+static int rtw89_pci_lv1rst_start_dma_ax(struct rtw89_dev *rtwdev)
 {
 	u32 ret;
 
@@ -3656,7 +3788,7 @@ static int rtw89_pci_lv1rst_start_dma(struct rtw89_dev *rtwdev)
 	rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
 	rtw89_pci_clr_idx_all(rtwdev);
 
-	ret = rtw89_pci_rst_bdram(rtwdev);
+	ret = rtw89_pci_rst_bdram_ax(rtwdev);
 	if (ret)
 		return ret;
 
@@ -3667,18 +3799,20 @@ static int rtw89_pci_lv1rst_start_dma(struct rtw89_dev *rtwdev)
 static int rtw89_pci_ops_mac_lv1_recovery(struct rtw89_dev *rtwdev,
 					  enum rtw89_lv1_rcvy_step step)
 {
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
+	const struct rtw89_pci_gen_def *gen_def = info->gen_def;
 	int ret;
 
 	switch (step) {
 	case RTW89_LV1_RCVY_STEP_1:
-		ret = rtw89_pci_lv1rst_stop_dma(rtwdev);
+		ret = gen_def->lv1rst_stop_dma(rtwdev);
 		if (ret)
 			rtw89_err(rtwdev, "lv1 rcvy pci stop dma fail\n");
 
 		break;
 
 	case RTW89_LV1_RCVY_STEP_2:
-		ret = rtw89_pci_lv1rst_start_dma(rtwdev);
+		ret = gen_def->lv1rst_start_dma(rtwdev);
 		if (ret)
 			rtw89_err(rtwdev, "lv1 rcvy pci start dma fail\n");
 		break;
@@ -3704,17 +3838,19 @@ static int rtw89_pci_napi_poll(struct napi_struct *napi, int budget)
 {
 	struct rtw89_dev *rtwdev = container_of(napi, struct rtw89_dev, napi);
 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
+	const struct rtw89_pci_gen_def *gen_def = info->gen_def;
 	unsigned long flags;
 	int work_done;
 
 	rtwdev->napi_budget_countdown = budget;
 
-	rtw89_pci_clear_isr0(rtwdev, B_AX_RPQDMA_INT | B_AX_RPQBD_FULL_INT);
+	rtw89_write32(rtwdev, gen_def->isr_clear_rpq.addr, gen_def->isr_clear_rpq.data);
 	work_done = rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown);
 	if (work_done == budget)
 		return budget;
 
-	rtw89_pci_clear_isr0(rtwdev, B_AX_RXP1DMA_INT | B_AX_RXDMA_INT | B_AX_RDU_INT);
+	rtw89_write32(rtwdev, gen_def->isr_clear_rxq.addr, gen_def->isr_clear_rxq.data);
 	work_done += rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown);
 	if (work_done < budget && napi_complete_done(napi, work_done)) {
 		spin_lock_irqsave(&rtwpci->irq_lock, flags);
@@ -3791,6 +3927,26 @@ static int __maybe_unused rtw89_pci_resume(struct device *dev)
 SIMPLE_DEV_PM_OPS(rtw89_pm_ops, rtw89_pci_suspend, rtw89_pci_resume);
 EXPORT_SYMBOL(rtw89_pm_ops);
 
+const struct rtw89_pci_gen_def rtw89_pci_gen_ax = {
+	.isr_rdu = B_AX_RDU_INT,
+	.isr_halt_c2h = B_AX_HALT_C2H_INT_EN,
+	.isr_wdt_timeout = B_AX_WDT_TIMEOUT_INT_EN,
+	.isr_clear_rpq = {R_AX_PCIE_HISR00, B_AX_RPQDMA_INT | B_AX_RPQBD_FULL_INT},
+	.isr_clear_rxq = {R_AX_PCIE_HISR00, B_AX_RXP1DMA_INT | B_AX_RXDMA_INT |
+					    B_AX_RDU_INT},
+
+	.mac_pre_init = rtw89_pci_ops_mac_pre_init_ax,
+	.mac_pre_deinit = NULL,
+	.mac_post_init = rtw89_pci_ops_mac_post_init_ax,
+
+	.clr_idx_all = rtw89_pci_clr_idx_all_ax,
+	.rst_bdram = rtw89_pci_rst_bdram_ax,
+
+	.lv1rst_stop_dma = rtw89_pci_lv1rst_stop_dma_ax,
+	.lv1rst_start_dma = rtw89_pci_lv1rst_start_dma_ax,
+};
+EXPORT_SYMBOL(rtw89_pci_gen_ax);
+
 static const struct rtw89_hci_ops rtw89_pci_ops = {
 	.tx_write	= rtw89_pci_ops_tx_write,
 	.tx_kick_off	= rtw89_pci_ops_tx_kick_off,
@@ -3810,6 +3966,7 @@ static const struct rtw89_hci_ops rtw89_pci_ops = {
 	.write32	= rtw89_pci_ops_write32,
 
 	.mac_pre_init	= rtw89_pci_ops_mac_pre_init,
+	.mac_pre_deinit	= rtw89_pci_ops_mac_pre_deinit,
 	.mac_post_init	= rtw89_pci_ops_mac_post_init,
 	.deinit		= rtw89_pci_ops_deinit,
 
@@ -3829,7 +3986,7 @@ static const struct rtw89_hci_ops rtw89_pci_ops = {
 	.clear		= rtw89_pci_clear_resource,
 	.disable_intr	= rtw89_pci_disable_intr_lock,
 	.enable_intr	= rtw89_pci_enable_intr_lock,
-	.rst_bdram	= rtw89_pci_rst_bdram_pcie,
+	.rst_bdram	= rtw89_pci_reset_bdram,
 };
 
 int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
diff --git a/drivers/net/wireless/realtek/rtw89/pci.h b/drivers/net/wireless/realtek/rtw89/pci.h
index 2f3d1ad3b0f7..e2d8eef52b20 100644
--- a/drivers/net/wireless/realtek/rtw89/pci.h
+++ b/drivers/net/wireless/realtek/rtw89/pci.h
@@ -245,6 +245,203 @@
 #define B_AX_HS1ISR_IND_INT BIT(25)
 #define B_AX_PCIE_DBG_STE_INT BIT(13)
 
+#define R_BE_PCIE_FRZ_CLK 0x3004
+#define B_BE_PCIE_FRZ_MAC_HW_RST BIT(31)
+#define B_BE_PCIE_FRZ_CFG_SPC_RST BIT(30)
+#define B_BE_PCIE_FRZ_ELBI_RST BIT(29)
+#define B_BE_PCIE_MAC_IS_ACTIVE BIT(28)
+#define B_BE_PCIE_FRZ_RTK_HW_RST BIT(27)
+#define B_BE_PCIE_FRZ_REG_RST BIT(26)
+#define B_BE_PCIE_FRZ_ANA_RST BIT(25)
+#define B_BE_PCIE_FRZ_WLAN_RST BIT(24)
+#define B_BE_PCIE_FRZ_FLR_RST BIT(23)
+#define B_BE_PCIE_FRZ_RET_NON_STKY_RST BIT(22)
+#define B_BE_PCIE_FRZ_RET_STKY_RST BIT(21)
+#define B_BE_PCIE_FRZ_NON_STKY_RST BIT(20)
+#define B_BE_PCIE_FRZ_STKY_RST BIT(19)
+#define B_BE_PCIE_FRZ_RET_CORE_RST BIT(18)
+#define B_BE_PCIE_FRZ_PWR_RST BIT(17)
+#define B_BE_PCIE_FRZ_PERST_RST BIT(16)
+#define B_BE_PCIE_FRZ_PHY_ALOAD BIT(15)
+#define B_BE_PCIE_FRZ_PHY_HW_RST BIT(14)
+#define B_BE_PCIE_DBG_CLK BIT(4)
+#define B_BE_PCIE_EN_CLK BIT(3)
+#define B_BE_PCIE_DBI_ACLK_ACT BIT(2)
+#define B_BE_PCIE_S1_ACLK_ACT BIT(1)
+#define B_BE_PCIE_EN_AUX_CLK BIT(0)
+
+#define R_BE_PCIE_PS_CTRL 0x3008
+#define B_BE_RSM_L0S_EN BIT(8)
+#define B_BE_CMAC_EXIT_L1_EN BIT(7)
+#define B_BE_DMAC0_EXIT_L1_EN BIT(6)
+#define B_BE_FORCE_L0 BIT(5)
+#define B_BE_DBI_RO_WR_DISABLE BIT(4)
+#define B_BE_SEL_XFER_PENDING BIT(3)
+#define B_BE_SEL_REQ_ENTR_L1 BIT(2)
+#define B_BE_PCIE_EN_SWENT_L23 BIT(1)
+#define B_BE_SEL_REQ_EXIT_L1 BIT(0)
+
+#define R_BE_PCIE_LAT_CTRL 0x3044
+#define B_BE_ELBI_PHY_REMAP_MASK GENMASK(29, 24)
+#define B_BE_SYS_SUS_L12_EN BIT(17)
+#define B_BE_MDIO_S_EN BIT(16)
+#define B_BE_SYM_AUX_CLK_SEL BIT(15)
+#define B_BE_RTK_LDO_POWER_LATENCY_MASK GENMASK(11, 10)
+#define B_BE_RTK_LDO_BIAS_LATENCY_MASK GENMASK(9, 8)
+#define B_BE_CLK_REQ_LAT_MASK GENMASK(7, 4)
+
+#define R_BE_PCIE_HIMR0 0x30B0
+#define B_BE_PCIE_HB1_IND_INTA_IMR BIT(31)
+#define B_BE_PCIE_HB0_IND_INTA_IMR BIT(30)
+#define B_BE_HCI_AXIDMA_INTA_IMR BIT(29)
+#define B_BE_HC0_IND_INTA_IMR BIT(28)
+#define B_BE_HD1_IND_INTA_IMR BIT(27)
+#define B_BE_HD0_IND_INTA_IMR BIT(26)
+#define B_BE_HS1_IND_INTA_IMR BIT(25)
+#define B_BE_HS0_IND_INTA_IMR BIT(24)
+#define B_BE_PCIE_HOTRST_INT_EN BIT(16)
+#define B_BE_PCIE_FLR_INT_EN BIT(15)
+#define B_BE_PCIE_PERST_INT_EN BIT(14)
+#define B_BE_PCIE_DBG_STE_INT_EN BIT(13)
+#define B_BE_HB1_IND_INT_EN0 BIT(9)
+#define B_BE_HB0_IND_INT_EN0 BIT(8)
+#define B_BE_HC1_IND_INT_EN0 BIT(7)
+#define B_BE_HCI_AXIDMA_INT_EN0 BIT(5)
+#define B_BE_HC0_IND_INT_EN0 BIT(4)
+#define B_BE_HD1_IND_INT_EN0 BIT(3)
+#define B_BE_HD0_IND_INT_EN0 BIT(2)
+#define B_BE_HS1_IND_INT_EN0 BIT(1)
+#define B_BE_HS0_IND_INT_EN0 BIT(0)
+
+#define R_BE_PCIE_HISR 0x30B4
+#define B_BE_PCIE_HOTRST_INT BIT(16)
+#define B_BE_PCIE_FLR_INT BIT(15)
+#define B_BE_PCIE_PERST_INT BIT(14)
+#define B_BE_PCIE_DBG_STE_INT BIT(13)
+#define B_BE_HB1IMR_IND BIT(9)
+#define B_BE_HB0IMR_IND BIT(8)
+#define B_BE_HC1ISR_IND_INT BIT(7)
+#define B_BE_HCI_AXIDMA_INT BIT(5)
+#define B_BE_HC0ISR_IND_INT BIT(4)
+#define B_BE_HD1ISR_IND_INT BIT(3)
+#define B_BE_HD0ISR_IND_INT BIT(2)
+#define B_BE_HS1ISR_IND_INT BIT(1)
+#define B_BE_HS0ISR_IND_INT BIT(0)
+
+#define R_BE_PCIE_DMA_IMR_0_V1 0x30B8
+#define B_BE_PCIE_RX_RX1P1_IMR0_V1 BIT(23)
+#define B_BE_PCIE_RX_RX0P1_IMR0_V1 BIT(22)
+#define B_BE_PCIE_RX_ROQ1_IMR0_V1 BIT(21)
+#define B_BE_PCIE_RX_RPQ1_IMR0_V1 BIT(20)
+#define B_BE_PCIE_RX_RX1P2_IMR0_V1 BIT(19)
+#define B_BE_PCIE_RX_ROQ0_IMR0_V1 BIT(18)
+#define B_BE_PCIE_RX_RPQ0_IMR0_V1 BIT(17)
+#define B_BE_PCIE_RX_RX0P2_IMR0_V1 BIT(16)
+#define B_BE_PCIE_TX_CH14_IMR0 BIT(14)
+#define B_BE_PCIE_TX_CH13_IMR0 BIT(13)
+#define B_BE_PCIE_TX_CH12_IMR0 BIT(12)
+#define B_BE_PCIE_TX_CH11_IMR0 BIT(11)
+#define B_BE_PCIE_TX_CH10_IMR0 BIT(10)
+#define B_BE_PCIE_TX_CH9_IMR0 BIT(9)
+#define B_BE_PCIE_TX_CH8_IMR0 BIT(8)
+#define B_BE_PCIE_TX_CH7_IMR0 BIT(7)
+#define B_BE_PCIE_TX_CH6_IMR0 BIT(6)
+#define B_BE_PCIE_TX_CH5_IMR0 BIT(5)
+#define B_BE_PCIE_TX_CH4_IMR0 BIT(4)
+#define B_BE_PCIE_TX_CH3_IMR0 BIT(3)
+#define B_BE_PCIE_TX_CH2_IMR0 BIT(2)
+#define B_BE_PCIE_TX_CH1_IMR0 BIT(1)
+#define B_BE_PCIE_TX_CH0_IMR0 BIT(0)
+
+#define R_BE_PCIE_DMA_ISR 0x30BC
+#define B_BE_PCIE_RX_RX1P1_ISR_V1 BIT(23)
+#define B_BE_PCIE_RX_RX0P1_ISR_V1 BIT(22)
+#define B_BE_PCIE_RX_ROQ1_ISR_V1 BIT(21)
+#define B_BE_PCIE_RX_RPQ1_ISR_V1 BIT(20)
+#define B_BE_PCIE_RX_RX1P2_ISR_V1 BIT(19)
+#define B_BE_PCIE_RX_ROQ0_ISR_V1 BIT(18)
+#define B_BE_PCIE_RX_RPQ0_ISR_V1 BIT(17)
+#define B_BE_PCIE_RX_RX0P2_ISR_V1 BIT(16)
+#define B_BE_PCIE_TX_CH14_ISR BIT(14)
+#define B_BE_PCIE_TX_CH13_ISR BIT(13)
+#define B_BE_PCIE_TX_CH12_ISR BIT(12)
+#define B_BE_PCIE_TX_CH11_ISR BIT(11)
+#define B_BE_PCIE_TX_CH10_ISR BIT(10)
+#define B_BE_PCIE_TX_CH9_ISR BIT(9)
+#define B_BE_PCIE_TX_CH8_ISR BIT(8)
+#define B_BE_PCIE_TX_CH7_ISR BIT(7)
+#define B_BE_PCIE_TX_CH6_ISR BIT(6)
+#define B_BE_PCIE_TX_CH5_ISR BIT(5)
+#define B_BE_PCIE_TX_CH4_ISR BIT(4)
+#define B_BE_PCIE_TX_CH3_ISR BIT(3)
+#define B_BE_PCIE_TX_CH2_ISR BIT(2)
+#define B_BE_PCIE_TX_CH1_ISR BIT(1)
+#define B_BE_PCIE_TX_CH0_ISR BIT(0)
+
+#define R_BE_HAXI_HIMR00 0xB0B0
+#define B_BE_RDU_CH5_INT_IMR_V1 BIT(30)
+#define B_BE_RDU_CH4_INT_IMR_V1 BIT(29)
+#define B_BE_RDU_CH3_INT_IMR_V1 BIT(28)
+#define B_BE_RDU_CH2_INT_IMR_V1 BIT(27)
+#define B_BE_RDU_CH1_INT_IMR_V1 BIT(26)
+#define B_BE_RDU_CH0_INT_IMR_V1 BIT(25)
+#define B_BE_RXDMA_STUCK_INT_EN_V1 BIT(24)
+#define B_BE_TXDMA_STUCK_INT_EN_V1 BIT(23)
+#define B_BE_TXDMA_CH14_INT_EN_V1 BIT(22)
+#define B_BE_TXDMA_CH13_INT_EN_V1 BIT(21)
+#define B_BE_TXDMA_CH12_INT_EN_V1 BIT(20)
+#define B_BE_TXDMA_CH11_INT_EN_V1 BIT(19)
+#define B_BE_TXDMA_CH10_INT_EN_V1 BIT(18)
+#define B_BE_TXDMA_CH9_INT_EN_V1 BIT(17)
+#define B_BE_TXDMA_CH8_INT_EN_V1 BIT(16)
+#define B_BE_TXDMA_CH7_INT_EN_V1 BIT(15)
+#define B_BE_TXDMA_CH6_INT_EN_V1 BIT(14)
+#define B_BE_TXDMA_CH5_INT_EN_V1 BIT(13)
+#define B_BE_TXDMA_CH4_INT_EN_V1 BIT(12)
+#define B_BE_TXDMA_CH3_INT_EN_V1 BIT(11)
+#define B_BE_TXDMA_CH2_INT_EN_V1 BIT(10)
+#define B_BE_TXDMA_CH1_INT_EN_V1 BIT(9)
+#define B_BE_TXDMA_CH0_INT_EN_V1 BIT(8)
+#define B_BE_RX1P1DMA_INT_EN_V1 BIT(7)
+#define B_BE_RX0P1DMA_INT_EN_V1 BIT(6)
+#define B_BE_RO1DMA_INT_EN BIT(5)
+#define B_BE_RP1DMA_INT_EN BIT(4)
+#define B_BE_RX1DMA_INT_EN BIT(3)
+#define B_BE_RO0DMA_INT_EN BIT(2)
+#define B_BE_RP0DMA_INT_EN BIT(1)
+#define B_BE_RX0DMA_INT_EN BIT(0)
+
+#define R_BE_HAXI_HISR00 0xB0B4
+#define B_BE_RDU_CH6_INT BIT(28)
+#define B_BE_RDU_CH5_INT BIT(27)
+#define B_BE_RDU_CH4_INT BIT(26)
+#define B_BE_RDU_CH2_INT BIT(25)
+#define B_BE_RDU_CH1_INT BIT(24)
+#define B_BE_RDU_CH0_INT BIT(23)
+#define B_BE_RXDMA_STUCK_INT BIT(22)
+#define B_BE_TXDMA_STUCK_INT BIT(21)
+#define B_BE_TXDMA_CH14_INT BIT(20)
+#define B_BE_TXDMA_CH13_INT BIT(19)
+#define B_BE_TXDMA_CH12_INT BIT(18)
+#define B_BE_TXDMA_CH11_INT BIT(17)
+#define B_BE_TXDMA_CH10_INT BIT(16)
+#define B_BE_TXDMA_CH9_INT BIT(15)
+#define B_BE_TXDMA_CH8_INT BIT(14)
+#define B_BE_TXDMA_CH7_INT BIT(13)
+#define B_BE_TXDMA_CH6_INT BIT(12)
+#define B_BE_TXDMA_CH5_INT BIT(11)
+#define B_BE_TXDMA_CH4_INT BIT(10)
+#define B_BE_TXDMA_CH3_INT BIT(9)
+#define B_BE_TXDMA_CH2_INT BIT(8)
+#define B_BE_TXDMA_CH1_INT BIT(7)
+#define B_BE_TXDMA_CH0_INT BIT(6)
+#define B_BE_RPQ1DMA_INT BIT(5)
+#define B_BE_RX1P1DMA_INT BIT(4)
+#define B_BE_RX1DMA_INT BIT(3)
+#define B_BE_RPQ0DMA_INT BIT(2)
+#define B_BE_RX0P1DMA_INT BIT(1)
+#define B_BE_RX0DMA_INT BIT(0)
+
 /* TX/RX */
 #define R_AX_DRV_FW_HSK_0	0x01B0
 #define R_AX_DRV_FW_HSK_1	0x01B4
@@ -496,6 +693,105 @@
 #define B_AX_CH11_BUSY			BIT(1)
 #define B_AX_CH10_BUSY			BIT(0)
 
+#define R_BE_HAXI_DMA_STOP1 0xB010
+#define B_BE_STOP_WPDMA BIT(31)
+#define B_BE_STOP_CH14 BIT(14)
+#define B_BE_STOP_CH13 BIT(13)
+#define B_BE_STOP_CH12 BIT(12)
+#define B_BE_STOP_CH11 BIT(11)
+#define B_BE_STOP_CH10 BIT(10)
+#define B_BE_STOP_CH9 BIT(9)
+#define B_BE_STOP_CH8 BIT(8)
+#define B_BE_STOP_CH7 BIT(7)
+#define B_BE_STOP_CH6 BIT(6)
+#define B_BE_STOP_CH5 BIT(5)
+#define B_BE_STOP_CH4 BIT(4)
+#define B_BE_STOP_CH3 BIT(3)
+#define B_BE_STOP_CH2 BIT(2)
+#define B_BE_STOP_CH1 BIT(1)
+#define B_BE_STOP_CH0 BIT(0)
+#define B_BE_TX_STOP1_MASK (B_BE_STOP_CH0 | B_BE_STOP_CH1 | \
+			    B_BE_STOP_CH2 | B_BE_STOP_CH3 | \
+			    B_BE_STOP_CH4 | B_BE_STOP_CH5 | \
+			    B_BE_STOP_CH6 | B_BE_STOP_CH7 | \
+			    B_BE_STOP_CH8 | B_BE_STOP_CH9 | \
+			    B_BE_STOP_CH10 | B_BE_STOP_CH11 | \
+			    B_BE_STOP_CH12)
+
+#define R_BE_CH0_TXBD_NUM_V1 0xB030
+#define R_BE_CH1_TXBD_NUM_V1 0xB032
+#define R_BE_CH2_TXBD_NUM_V1 0xB034
+#define R_BE_CH3_TXBD_NUM_V1 0xB036
+#define R_BE_CH4_TXBD_NUM_V1 0xB038
+#define R_BE_CH5_TXBD_NUM_V1 0xB03A
+#define R_BE_CH6_TXBD_NUM_V1 0xB03C
+#define R_BE_CH7_TXBD_NUM_V1 0xB03E
+#define R_BE_CH8_TXBD_NUM_V1 0xB040
+#define R_BE_CH9_TXBD_NUM_V1 0xB042
+#define R_BE_CH10_TXBD_NUM_V1 0xB044
+#define R_BE_CH11_TXBD_NUM_V1 0xB046
+#define R_BE_CH12_TXBD_NUM_V1 0xB048
+#define R_BE_CH13_TXBD_NUM_V1 0xB04C
+#define R_BE_CH14_TXBD_NUM_V1 0xB04E
+
+#define R_BE_RXQ0_RXBD_NUM_V1 0xB050
+#define R_BE_RPQ0_RXBD_NUM_V1 0xB052
+
+#define R_BE_CH0_TXBD_IDX_V1 0xB100
+#define R_BE_CH1_TXBD_IDX_V1 0xB104
+#define R_BE_CH2_TXBD_IDX_V1 0xB108
+#define R_BE_CH3_TXBD_IDX_V1 0xB10C
+#define R_BE_CH4_TXBD_IDX_V1 0xB110
+#define R_BE_CH5_TXBD_IDX_V1 0xB114
+#define R_BE_CH6_TXBD_IDX_V1 0xB118
+#define R_BE_CH7_TXBD_IDX_V1 0xB11C
+#define R_BE_CH8_TXBD_IDX_V1 0xB120
+#define R_BE_CH9_TXBD_IDX_V1 0xB124
+#define R_BE_CH10_TXBD_IDX_V1 0xB128
+#define R_BE_CH11_TXBD_IDX_V1 0xB12C
+#define R_BE_CH12_TXBD_IDX_V1 0xB130
+#define R_BE_CH13_TXBD_IDX_V1 0xB134
+#define R_BE_CH14_TXBD_IDX_V1 0xB138
+
+#define R_BE_RXQ0_RXBD_IDX_V1 0xB160
+#define R_BE_RPQ0_RXBD_IDX_V1 0xB164
+
+#define R_BE_CH0_TXBD_DESA_L_V1 0xB200
+#define R_BE_CH0_TXBD_DESA_H_V1 0xB204
+#define R_BE_CH1_TXBD_DESA_L_V1 0xB208
+#define R_BE_CH1_TXBD_DESA_H_V1 0xB20C
+#define R_BE_CH2_TXBD_DESA_L_V1 0xB210
+#define R_BE_CH2_TXBD_DESA_H_V1 0xB214
+#define R_BE_CH3_TXBD_DESA_L_V1 0xB218
+#define R_BE_CH3_TXBD_DESA_H_V1 0xB21C
+#define R_BE_CH4_TXBD_DESA_L_V1 0xB220
+#define R_BE_CH4_TXBD_DESA_H_V1 0xB224
+#define R_BE_CH5_TXBD_DESA_L_V1 0xB228
+#define R_BE_CH5_TXBD_DESA_H_V1 0xB22C
+#define R_BE_CH6_TXBD_DESA_L_V1 0xB230
+#define R_BE_CH6_TXBD_DESA_H_V1 0xB234
+#define R_BE_CH7_TXBD_DESA_L_V1 0xB238
+#define R_BE_CH7_TXBD_DESA_H_V1 0xB23C
+#define R_BE_CH8_TXBD_DESA_L_V1 0xB240
+#define R_BE_CH8_TXBD_DESA_H_V1 0xB244
+#define R_BE_CH9_TXBD_DESA_L_V1 0xB248
+#define R_BE_CH9_TXBD_DESA_H_V1 0xB24C
+#define R_BE_CH10_TXBD_DESA_L_V1 0xB250
+#define R_BE_CH10_TXBD_DESA_H_V1 0xB254
+#define R_BE_CH11_TXBD_DESA_L_V1 0xB258
+#define R_BE_CH11_TXBD_DESA_H_V1 0xB25C
+#define R_BE_CH12_TXBD_DESA_L_V1 0xB260
+#define R_BE_CH12_TXBD_DESA_H_V1 0xB264
+#define R_BE_CH13_TXBD_DESA_L_V1 0xB268
+#define R_BE_CH13_TXBD_DESA_H_V1 0xB26C
+#define R_BE_CH14_TXBD_DESA_L_V1 0xB270
+#define R_BE_CH14_TXBD_DESA_H_V1 0xB274
+
+#define R_BE_RXQ0_RXBD_DESA_L_V1 0xB300
+#define R_BE_RXQ0_RXBD_DESA_H_V1 0xB304
+#define R_BE_RPQ0_RXBD_DESA_L_V1 0xB308
+#define R_BE_RPQ0_RXBD_DESA_H_V1 0xB30C
+
 /* Configure */
 #define R_AX_PCIE_INIT_CFG2		0x1004
 #define B_AX_WD_ITVL_IDLE		GENMASK(27, 24)
@@ -516,6 +812,13 @@
 #define B_AX_RXCOUNTER_MATCH_MASK	GENMASK(15, 8)
 #define B_AX_RXTIMER_MATCH_MASK		GENMASK(7, 0)
 
+#define R_AX_INT_MIT_RX_V1 0x1184
+#define B_AX_RXMIT_RXP2_SEL_V1 BIT(19)
+#define B_AX_RXMIT_RXP1_SEL_V1 BIT(18)
+#define B_AX_MIT_RXTIMER_UNIT_MASK GENMASK(17, 16)
+#define B_AX_MIT_RXCOUNTER_MATCH_MASK GENMASK(15, 8)
+#define B_AX_MIT_RXTIMER_MATCH_MASK GENMASK(7, 0)
+
 #define R_AX_DBG_ERR_FLAG		0x11C4
 #define B_AX_PCIE_RPQ_FULL		BIT(29)
 #define B_AX_PCIE_RXQ_FULL		BIT(28)
@@ -554,6 +857,136 @@
 #define R_AX_PCIE_HRPWM_V1		0x30C0
 #define R_AX_PCIE_CRPWM			0x30C4
 
+#define R_BE_PCIE_HRPWM 0x30C0
+#define R_BE_PCIE_CRPWM 0x30C4
+
+#define R_BE_L1_2_CTRL_HCILDO 0x3110
+#define B_BE_PCIE_DIS_L1_2_CTRL_HCILDO BIT(0)
+
+#define R_BE_PL1_DBG_INFO 0x3120
+#define B_BE_END_PL1_CNT_MASK GENMASK(23, 16)
+#define B_BE_START_PL1_CNT_MASK GENMASK(7, 0)
+
+#define R_BE_PCIE_MIT0_TMR 0x3330
+#define B_BE_PCIE_MIT0_RX_TMR_MASK GENMASK(5, 4)
+#define BE_MIT0_TMR_UNIT_1MS 0
+#define BE_MIT0_TMR_UNIT_2MS 1
+#define BE_MIT0_TMR_UNIT_4MS 2
+#define BE_MIT0_TMR_UNIT_8MS 3
+#define B_BE_PCIE_MIT0_TX_TMR_MASK GENMASK(1, 0)
+
+#define R_BE_PCIE_MIT0_CNT 0x3334
+#define B_BE_PCIE_RX_MIT0_CNT_MASK GENMASK(31, 24)
+#define B_BE_PCIE_TX_MIT0_CNT_MASK GENMASK(23, 16)
+#define B_BE_PCIE_RX_MIT0_TMR_CNT_MASK GENMASK(15, 8)
+#define B_BE_PCIE_TX_MIT0_TMR_CNT_MASK GENMASK(7, 0)
+
+#define R_BE_PCIE_MIT_CH_EN 0x3338
+#define B_BE_PCIE_MIT_RX1P1_EN BIT(23)
+#define B_BE_PCIE_MIT_RX0P1_EN BIT(22)
+#define B_BE_PCIE_MIT_ROQ1_EN BIT(21)
+#define B_BE_PCIE_MIT_RPQ1_EN BIT(20)
+#define B_BE_PCIE_MIT_RX1P2_EN BIT(19)
+#define B_BE_PCIE_MIT_ROQ0_EN BIT(18)
+#define B_BE_PCIE_MIT_RPQ0_EN BIT(17)
+#define B_BE_PCIE_MIT_RX0P2_EN BIT(16)
+#define B_BE_PCIE_MIT_TXCH14_EN BIT(14)
+#define B_BE_PCIE_MIT_TXCH13_EN BIT(13)
+#define B_BE_PCIE_MIT_TXCH12_EN BIT(12)
+#define B_BE_PCIE_MIT_TXCH11_EN BIT(11)
+#define B_BE_PCIE_MIT_TXCH10_EN BIT(10)
+#define B_BE_PCIE_MIT_TXCH9_EN BIT(9)
+#define B_BE_PCIE_MIT_TXCH8_EN BIT(8)
+#define B_BE_PCIE_MIT_TXCH7_EN BIT(7)
+#define B_BE_PCIE_MIT_TXCH6_EN BIT(6)
+#define B_BE_PCIE_MIT_TXCH5_EN BIT(5)
+#define B_BE_PCIE_MIT_TXCH4_EN BIT(4)
+#define B_BE_PCIE_MIT_TXCH3_EN BIT(3)
+#define B_BE_PCIE_MIT_TXCH2_EN BIT(2)
+#define B_BE_PCIE_MIT_TXCH1_EN BIT(1)
+#define B_BE_PCIE_MIT_TXCH0_EN BIT(0)
+
+#define R_BE_SER_PL1_CTRL 0x34A8
+#define B_BE_PL1_SER_PL1_EN BIT(31)
+#define B_BE_PL1_IGNORE_HOT_RST BIT(30)
+#define B_BE_PL1_TIMER_UNIT_MASK GENMASK(19, 17)
+#define B_BE_PL1_TIMER_CLEAR BIT(0)
+
+#define R_BE_REG_PL1_MASK 0x34B0
+#define B_BE_SER_PCLKREQ_ACK_MASK BIT(5)
+#define B_BE_SER_PM_CLK_MASK BIT(4)
+#define B_BE_SER_LTSSM_IMR BIT(3)
+#define B_BE_SER_PM_MASTER_IMR BIT(2)
+#define B_BE_SER_L1SUB_IMR BIT(1)
+#define B_BE_SER_PMU_IMR BIT(0)
+
+#define R_BE_RX_APPEND_MODE 0x8920
+#define B_BE_APPEND_OFFSET_MASK GENMASK(23, 16)
+#define B_BE_APPEND_LEN_MASK GENMASK(15, 0)
+
+#define R_BE_TXBD_RWPTR_CLR1 0xB014
+#define B_BE_CLR_CH14_IDX BIT(14)
+#define B_BE_CLR_CH13_IDX BIT(13)
+#define B_BE_CLR_CH12_IDX BIT(12)
+#define B_BE_CLR_CH11_IDX BIT(11)
+#define B_BE_CLR_CH10_IDX BIT(10)
+#define B_BE_CLR_CH9_IDX BIT(9)
+#define B_BE_CLR_CH8_IDX BIT(8)
+#define B_BE_CLR_CH7_IDX BIT(7)
+#define B_BE_CLR_CH6_IDX BIT(6)
+#define B_BE_CLR_CH5_IDX BIT(5)
+#define B_BE_CLR_CH4_IDX BIT(4)
+#define B_BE_CLR_CH3_IDX BIT(3)
+#define B_BE_CLR_CH2_IDX BIT(2)
+#define B_BE_CLR_CH1_IDX BIT(1)
+#define B_BE_CLR_CH0_IDX BIT(0)
+
+#define R_BE_RXBD_RWPTR_CLR1_V1 0xB018
+#define B_BE_CLR_ROQ1_IDX_V1 BIT(5)
+#define B_BE_CLR_RPQ1_IDX_V1 BIT(4)
+#define B_BE_CLR_RXQ1_IDX_V1 BIT(3)
+#define B_BE_CLR_ROQ0_IDX BIT(2)
+#define B_BE_CLR_RPQ0_IDX BIT(1)
+#define B_BE_CLR_RXQ0_IDX BIT(0)
+
+#define R_BE_HAXI_DMA_BUSY1 0xB01C
+#define B_BE_HAXI_MST_BUSY BIT(31)
+#define B_BE_HAXI_RX_IDLE BIT(25)
+#define B_BE_HAXI_TX_IDLE BIT(24)
+#define B_BE_ROQ1_BUSY_V1 BIT(21)
+#define B_BE_RPQ1_BUSY_V1 BIT(20)
+#define B_BE_RXQ1_BUSY_V1 BIT(19)
+#define B_BE_ROQ0_BUSY_V1 BIT(18)
+#define B_BE_RPQ0_BUSY_V1 BIT(17)
+#define B_BE_RXQ0_BUSY_V1 BIT(16)
+#define B_BE_WPDMA_BUSY BIT(15)
+#define B_BE_CH14_BUSY BIT(14)
+#define B_BE_CH13_BUSY BIT(13)
+#define B_BE_CH12_BUSY BIT(12)
+#define B_BE_CH11_BUSY BIT(11)
+#define B_BE_CH10_BUSY BIT(10)
+#define B_BE_CH9_BUSY BIT(9)
+#define B_BE_CH8_BUSY BIT(8)
+#define B_BE_CH7_BUSY BIT(7)
+#define B_BE_CH6_BUSY BIT(6)
+#define B_BE_CH5_BUSY BIT(5)
+#define B_BE_CH4_BUSY BIT(4)
+#define B_BE_CH3_BUSY BIT(3)
+#define B_BE_CH2_BUSY BIT(2)
+#define B_BE_CH1_BUSY BIT(1)
+#define B_BE_CH0_BUSY BIT(0)
+#define DMA_BUSY1_CHECK_BE (B_BE_CH0_BUSY | B_BE_CH1_BUSY | B_BE_CH2_BUSY | \
+			    B_BE_CH3_BUSY | B_BE_CH4_BUSY | B_BE_CH5_BUSY | \
+			    B_BE_CH6_BUSY | B_BE_CH7_BUSY | B_BE_CH8_BUSY | \
+			    B_BE_CH9_BUSY | B_BE_CH10_BUSY | B_BE_CH11_BUSY | \
+			    B_BE_CH12_BUSY | B_BE_CH13_BUSY | B_BE_CH14_BUSY)
+
+#define R_BE_HAXI_EXP_CTRL_V1 0xB020
+#define B_BE_R_NO_SEC_ACCESS BIT(31)
+#define B_BE_FORCE_EN_DMA_RX_GCLK BIT(5)
+#define B_BE_FORCE_EN_DMA_TX_GCLK BIT(4)
+#define B_BE_MAX_TAG_NUM_MASK GENMASK(3, 0)
+
 #define RTW89_PCI_TXBD_NUM_MAX		256
 #define RTW89_PCI_RXBD_NUM_MAX		256
 #define RTW89_PCI_TXWD_NUM_MAX		512
@@ -565,12 +998,15 @@
 #define RTW89_PCI_MULTITAG		8
 
 /* PCIE CFG register */
+#define RTW89_PCIE_CAPABILITY_SPEED	0x7C
+#define RTW89_PCIE_SUPPORT_GEN_MASK	GENMASK(3, 0)
 #define RTW89_PCIE_L1_STS_V1		0x80
 #define RTW89_BCFG_LINK_SPEED_MASK	GENMASK(19, 16)
 #define RTW89_PCIE_GEN1_SPEED		0x01
 #define RTW89_PCIE_GEN2_SPEED		0x02
 #define RTW89_PCIE_PHY_RATE		0x82
 #define RTW89_PCIE_PHY_RATE_MASK	GENMASK(1, 0)
+#define RTW89_PCIE_LINK_CHANGE_SPEED	0xA0
 #define RTW89_PCIE_L1SS_STS_V1		0x0168
 #define RTW89_PCIE_BIT_ASPM_L11		BIT(3)
 #define RTW89_PCIE_BIT_ASPM_L12		BIT(2)
@@ -585,6 +1021,8 @@
 #define RTW89_PCIE_BIT_CLK		BIT(4)
 #define RTW89_PCIE_BIT_L1		BIT(3)
 #define RTW89_PCIE_CLK_CTRL		0x0725
+#define RTW89_PCIE_FTS			0x080C
+#define RTW89_PCIE_POLLING_BIT		BIT(17)
 #define RTW89_PCIE_RST_MSTATE		0x0B48
 #define RTW89_PCIE_BIT_CFG_RST_MSTATE	BIT(0)
 
@@ -757,7 +1195,26 @@ struct rtw89_pci_bd_ram {
 	u8 min_num;
 };
 
+struct rtw89_pci_gen_def {
+	u32 isr_rdu;
+	u32 isr_halt_c2h;
+	u32 isr_wdt_timeout;
+	struct rtw89_reg2_def isr_clear_rpq;
+	struct rtw89_reg2_def isr_clear_rxq;
+
+	int (*mac_pre_init)(struct rtw89_dev *rtwdev);
+	int (*mac_pre_deinit)(struct rtw89_dev *rtwdev);
+	int (*mac_post_init)(struct rtw89_dev *rtwdev);
+
+	void (*clr_idx_all)(struct rtw89_dev *rtwdev);
+	int (*rst_bdram)(struct rtw89_dev *rtwdev);
+
+	int (*lv1rst_stop_dma)(struct rtw89_dev *rtwdev);
+	int (*lv1rst_start_dma)(struct rtw89_dev *rtwdev);
+};
+
 struct rtw89_pci_info {
+	const struct rtw89_pci_gen_def *gen_def;
 	enum mac_ax_bd_trunc_mode txbd_trunc_mode;
 	enum mac_ax_bd_trunc_mode rxbd_trunc_mode;
 	enum mac_ax_rxbd_mode rxbd_mode;
@@ -772,6 +1229,7 @@ struct rtw89_pci_info {
 	enum mac_ax_pcie_func_ctrl autok_en;
 	enum mac_ax_pcie_func_ctrl io_rcy_en;
 	enum mac_ax_io_rcy_tmr io_rcy_tmr;
+	bool rx_ring_eq_is_full;
 
 	u32 init_cfg_reg;
 	u32 txhci_en_bit;
@@ -781,6 +1239,7 @@ struct rtw89_pci_info {
 	u32 max_tag_num_mask;
 	u32 rxbd_rwptr_clr_reg;
 	u32 txbd_rwptr_clr2_reg;
+	struct rtw89_reg_def dma_io_stop;
 	struct rtw89_reg_def dma_stop1;
 	struct rtw89_reg_def dma_stop2;
 	struct rtw89_reg_def dma_busy1;
@@ -789,6 +1248,7 @@ struct rtw89_pci_info {
 
 	u32 rpwm_addr;
 	u32 cpwm_addr;
+	u32 mit_addr;
 	u32 tx_dma_ch_mask;
 	const struct rtw89_pci_bd_idx_addr *bd_idx_addr_low_power;
 	const struct rtw89_pci_ch_dma_addr_set *dma_addr_set;
@@ -1059,33 +1519,45 @@ static inline bool rtw89_pci_ltr_is_err_reg_val(u32 val)
 extern const struct dev_pm_ops rtw89_pm_ops;
 extern const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set;
 extern const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_v1;
+extern const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_be;
 extern const struct rtw89_pci_bd_ram rtw89_bd_ram_table_dual[RTW89_TXCH_NUM];
 extern const struct rtw89_pci_bd_ram rtw89_bd_ram_table_single[RTW89_TXCH_NUM];
+extern const struct rtw89_pci_gen_def rtw89_pci_gen_ax;
+extern const struct rtw89_pci_gen_def rtw89_pci_gen_be;
 
 struct pci_device_id;
 
 int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id);
 void rtw89_pci_remove(struct pci_dev *pdev);
+void rtw89_pci_ops_reset(struct rtw89_dev *rtwdev);
 int rtw89_pci_ltr_set(struct rtw89_dev *rtwdev, bool en);
 int rtw89_pci_ltr_set_v1(struct rtw89_dev *rtwdev, bool en);
+int rtw89_pci_ltr_set_v2(struct rtw89_dev *rtwdev, bool en);
 u32 rtw89_pci_fill_txaddr_info(struct rtw89_dev *rtwdev,
 			       void *txaddr_info_addr, u32 total_len,
 			       dma_addr_t dma, u8 *add_info_nr);
 u32 rtw89_pci_fill_txaddr_info_v1(struct rtw89_dev *rtwdev,
 				  void *txaddr_info_addr, u32 total_len,
 				  dma_addr_t dma, u8 *add_info_nr);
+void rtw89_pci_ctrl_dma_all(struct rtw89_dev *rtwdev, bool enable);
 void rtw89_pci_config_intr_mask(struct rtw89_dev *rtwdev);
 void rtw89_pci_config_intr_mask_v1(struct rtw89_dev *rtwdev);
+void rtw89_pci_config_intr_mask_v2(struct rtw89_dev *rtwdev);
 void rtw89_pci_enable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci);
 void rtw89_pci_disable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci);
 void rtw89_pci_enable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci);
 void rtw89_pci_disable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci);
+void rtw89_pci_enable_intr_v2(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci);
+void rtw89_pci_disable_intr_v2(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci);
 void rtw89_pci_recognize_intrs(struct rtw89_dev *rtwdev,
 			       struct rtw89_pci *rtwpci,
 			       struct rtw89_pci_isrs *isrs);
 void rtw89_pci_recognize_intrs_v1(struct rtw89_dev *rtwdev,
 				  struct rtw89_pci *rtwpci,
 				  struct rtw89_pci_isrs *isrs);
+void rtw89_pci_recognize_intrs_v2(struct rtw89_dev *rtwdev,
+				  struct rtw89_pci *rtwpci,
+				  struct rtw89_pci_isrs *isrs);
 
 static inline
 u32 rtw89_chip_fill_txaddr_info(struct rtw89_dev *rtwdev,
@@ -1157,4 +1629,47 @@ void rtw89_chip_recognize_intrs(struct rtw89_dev *rtwdev,
 	info->recognize_intrs(rtwdev, rtwpci, isrs);
 }
 
+static inline int rtw89_pci_ops_mac_pre_init(struct rtw89_dev *rtwdev)
+{
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
+	const struct rtw89_pci_gen_def *gen_def = info->gen_def;
+
+	return gen_def->mac_pre_init(rtwdev);
+}
+
+static inline int rtw89_pci_ops_mac_pre_deinit(struct rtw89_dev *rtwdev)
+{
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
+	const struct rtw89_pci_gen_def *gen_def = info->gen_def;
+
+	if (!gen_def->mac_pre_deinit)
+		return 0;
+
+	return gen_def->mac_pre_deinit(rtwdev);
+}
+
+static inline int rtw89_pci_ops_mac_post_init(struct rtw89_dev *rtwdev)
+{
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
+	const struct rtw89_pci_gen_def *gen_def = info->gen_def;
+
+	return gen_def->mac_post_init(rtwdev);
+}
+
+static inline void rtw89_pci_clr_idx_all(struct rtw89_dev *rtwdev)
+{
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
+	const struct rtw89_pci_gen_def *gen_def = info->gen_def;
+
+	gen_def->clr_idx_all(rtwdev);
+}
+
+static inline int rtw89_pci_reset_bdram(struct rtw89_dev *rtwdev)
+{
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
+	const struct rtw89_pci_gen_def *gen_def = info->gen_def;
+
+	return gen_def->rst_bdram(rtwdev);
+}
+
 #endif
diff --git a/drivers/net/wireless/realtek/rtw89/pci_be.c b/drivers/net/wireless/realtek/rtw89/pci_be.c
new file mode 100644
index 000000000000..629ffa4bee91
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtw89/pci_be.c
@@ -0,0 +1,509 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* Copyright(c) 2023  Realtek Corporation
+ */
+
+#include <linux/pci.h>
+
+#include "mac.h"
+#include "pci.h"
+#include "reg.h"
+
+enum pcie_rxbd_mode {
+	PCIE_RXBD_NORM = 0,
+	PCIE_RXBD_SEP,
+	PCIE_RXBD_EXT,
+};
+
+#define PL0_TMR_SCALE_ASIC 1
+#define PL0_TMR_ANA_172US 0x800
+#define PL0_TMR_MAC_1MS 0x27100
+#define PL0_TMR_AUX_1MS 0x1E848
+
+static void _patch_pcie_power_wake_be(struct rtw89_dev *rtwdev, bool power_up)
+{
+	if (power_up)
+		rtw89_write32_set(rtwdev, R_BE_HCI_OPT_CTRL, BIT_WAKE_CTRL_V1);
+	else
+		rtw89_write32_clr(rtwdev, R_BE_HCI_OPT_CTRL, BIT_WAKE_CTRL_V1);
+}
+
+static void rtw89_pci_set_io_rcy_be(struct rtw89_dev *rtwdev)
+{
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
+	u32 scale = PL0_TMR_SCALE_ASIC;
+	u32 val32;
+
+	if (info->io_rcy_en == MAC_AX_PCIE_ENABLE) {
+		val32 = info->io_rcy_tmr == MAC_AX_IO_RCY_ANA_TMR_DEF ?
+			PL0_TMR_ANA_172US : info->io_rcy_tmr;
+		val32 /= scale;
+
+		rtw89_write32(rtwdev, R_BE_AON_WDT_TMR, val32);
+		rtw89_write32(rtwdev, R_BE_MDIO_WDT_TMR, val32);
+		rtw89_write32(rtwdev, R_BE_LA_MODE_WDT_TMR, val32);
+		rtw89_write32(rtwdev, R_BE_WDT_AR_TMR, val32);
+		rtw89_write32(rtwdev, R_BE_WDT_AW_TMR, val32);
+		rtw89_write32(rtwdev, R_BE_WDT_W_TMR, val32);
+		rtw89_write32(rtwdev, R_BE_WDT_B_TMR, val32);
+		rtw89_write32(rtwdev, R_BE_WDT_R_TMR, val32);
+
+		val32 = info->io_rcy_tmr == MAC_AX_IO_RCY_ANA_TMR_DEF ?
+			PL0_TMR_MAC_1MS : info->io_rcy_tmr;
+		val32 /= scale;
+		rtw89_write32(rtwdev, R_BE_WLAN_WDT_TMR, val32);
+		rtw89_write32(rtwdev, R_BE_AXIDMA_WDT_TMR, val32);
+
+		val32 = info->io_rcy_tmr == MAC_AX_IO_RCY_ANA_TMR_DEF ?
+			PL0_TMR_AUX_1MS : info->io_rcy_tmr;
+		val32 /= scale;
+		rtw89_write32(rtwdev, R_BE_LOCAL_WDT_TMR, val32);
+	} else {
+		rtw89_write32_clr(rtwdev, R_BE_WLAN_WDT, B_BE_WLAN_WDT_ENABLE);
+		rtw89_write32_clr(rtwdev, R_BE_AXIDMA_WDT, B_BE_AXIDMA_WDT_ENABLE);
+		rtw89_write32_clr(rtwdev, R_BE_AON_WDT, B_BE_AON_WDT_ENABLE);
+		rtw89_write32_clr(rtwdev, R_BE_LOCAL_WDT, B_BE_LOCAL_WDT_ENABLE);
+		rtw89_write32_clr(rtwdev, R_BE_MDIO_WDT, B_BE_MDIO_WDT_ENABLE);
+		rtw89_write32_clr(rtwdev, R_BE_LA_MODE_WDT, B_BE_LA_MODE_WDT_ENABLE);
+		rtw89_write32_clr(rtwdev, R_BE_WDT_AR, B_BE_WDT_AR_ENABLE);
+		rtw89_write32_clr(rtwdev, R_BE_WDT_AW, B_BE_WDT_AW_ENABLE);
+		rtw89_write32_clr(rtwdev, R_BE_WDT_W, B_BE_WDT_W_ENABLE);
+		rtw89_write32_clr(rtwdev, R_BE_WDT_B, B_BE_WDT_B_ENABLE);
+		rtw89_write32_clr(rtwdev, R_BE_WDT_R, B_BE_WDT_R_ENABLE);
+	}
+}
+
+static void rtw89_pci_ctrl_wpdma_pcie_be(struct rtw89_dev *rtwdev, bool en)
+{
+	if (en)
+		rtw89_write32_clr(rtwdev, R_BE_HAXI_DMA_STOP1, B_BE_STOP_WPDMA);
+	else
+		rtw89_write32_set(rtwdev, R_BE_HAXI_DMA_STOP1, B_BE_STOP_WPDMA);
+}
+
+static void rtw89_pci_ctrl_trxdma_pcie_be(struct rtw89_dev *rtwdev,
+					  enum mac_ax_pcie_func_ctrl tx_en,
+					  enum mac_ax_pcie_func_ctrl rx_en,
+					  enum mac_ax_pcie_func_ctrl io_en)
+{
+	u32 val;
+
+	val = rtw89_read32(rtwdev, R_BE_HAXI_INIT_CFG1);
+
+	if (tx_en == MAC_AX_PCIE_ENABLE)
+		val |= B_BE_TXDMA_EN;
+	else if (tx_en == MAC_AX_PCIE_DISABLE)
+		val &= ~B_BE_TXDMA_EN;
+
+	if (rx_en == MAC_AX_PCIE_ENABLE)
+		val |= B_BE_RXDMA_EN;
+	else if (rx_en == MAC_AX_PCIE_DISABLE)
+		val &= ~B_BE_RXDMA_EN;
+
+	if (io_en == MAC_AX_PCIE_ENABLE)
+		val &= ~B_BE_STOP_AXI_MST;
+	else if (io_en == MAC_AX_PCIE_DISABLE)
+		val |= B_BE_STOP_AXI_MST;
+
+	rtw89_write32(rtwdev, R_BE_HAXI_INIT_CFG1, val);
+}
+
+static void rtw89_pci_clr_idx_all_be(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
+	struct rtw89_pci_rx_ring *rx_ring;
+	u32 val;
+
+	val = B_BE_CLR_CH0_IDX | B_BE_CLR_CH1_IDX | B_BE_CLR_CH2_IDX |
+	      B_BE_CLR_CH3_IDX | B_BE_CLR_CH4_IDX | B_BE_CLR_CH5_IDX |
+	      B_BE_CLR_CH6_IDX | B_BE_CLR_CH7_IDX | B_BE_CLR_CH8_IDX |
+	      B_BE_CLR_CH9_IDX | B_BE_CLR_CH10_IDX | B_BE_CLR_CH11_IDX |
+	      B_BE_CLR_CH12_IDX | B_BE_CLR_CH13_IDX | B_BE_CLR_CH14_IDX;
+	rtw89_write32(rtwdev, R_BE_TXBD_RWPTR_CLR1, val);
+
+	rtw89_write32(rtwdev, R_BE_RXBD_RWPTR_CLR1_V1,
+		      B_BE_CLR_RXQ0_IDX | B_BE_CLR_RPQ0_IDX);
+
+	rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RXQ];
+	rtw89_write16(rtwdev, R_BE_RXQ0_RXBD_IDX_V1, rx_ring->bd_ring.len - 1);
+
+	rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RPQ];
+	rtw89_write16(rtwdev, R_BE_RPQ0_RXBD_IDX_V1, rx_ring->bd_ring.len - 1);
+}
+
+static int rtw89_pci_poll_txdma_ch_idle_be(struct rtw89_dev *rtwdev)
+{
+	u32 val;
+
+	return read_poll_timeout(rtw89_read32, val, (val & DMA_BUSY1_CHECK_BE) == 0,
+				 10, 1000, false, rtwdev, R_BE_HAXI_DMA_BUSY1);
+}
+
+static int rtw89_pci_poll_rxdma_ch_idle_be(struct rtw89_dev *rtwdev)
+{
+	u32 check;
+	u32 val;
+
+	check = B_BE_RXQ0_BUSY_V1 | B_BE_RPQ0_BUSY_V1;
+
+	return read_poll_timeout(rtw89_read32, val, (val & check) == 0,
+				 10, 1000, false, rtwdev, R_BE_HAXI_DMA_BUSY1);
+}
+
+static int rtw89_pci_poll_dma_all_idle_be(struct rtw89_dev *rtwdev)
+{
+	int ret;
+
+	ret = rtw89_pci_poll_txdma_ch_idle_be(rtwdev);
+	if (ret) {
+		rtw89_err(rtwdev, "txdma ch busy\n");
+		return ret;
+	}
+
+	ret = rtw89_pci_poll_rxdma_ch_idle_be(rtwdev);
+	if (ret) {
+		rtw89_err(rtwdev, "rxdma ch busy\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static void rtw89_pci_mode_op_be(struct rtw89_dev *rtwdev)
+{
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
+	u32 val32_init1, val32_rxapp, val32_exp;
+
+	val32_init1 = rtw89_read32(rtwdev, R_BE_HAXI_INIT_CFG1);
+	val32_rxapp = rtw89_read32(rtwdev, R_BE_RX_APPEND_MODE);
+	val32_exp = rtw89_read32(rtwdev, R_BE_HAXI_EXP_CTRL_V1);
+
+	if (info->rxbd_mode == MAC_AX_RXBD_PKT) {
+		val32_init1 = u32_replace_bits(val32_init1, PCIE_RXBD_NORM,
+					       B_BE_RXQ_RXBD_MODE_MASK);
+	} else if (info->rxbd_mode == MAC_AX_RXBD_SEP) {
+		val32_init1 = u32_replace_bits(val32_init1, PCIE_RXBD_SEP,
+					       B_BE_RXQ_RXBD_MODE_MASK);
+		val32_rxapp = u32_replace_bits(val32_rxapp, 0,
+					       B_BE_APPEND_LEN_MASK);
+	}
+
+	val32_init1 = u32_replace_bits(val32_init1, info->tx_burst,
+				       B_BE_MAX_TXDMA_MASK);
+	val32_init1 = u32_replace_bits(val32_init1, info->rx_burst,
+				       B_BE_MAX_RXDMA_MASK);
+	val32_exp = u32_replace_bits(val32_exp, info->multi_tag_num,
+				     B_BE_MAX_TAG_NUM_MASK);
+	val32_init1 = u32_replace_bits(val32_init1, info->wd_dma_idle_intvl,
+				       B_BE_CFG_WD_PERIOD_IDLE_MASK);
+	val32_init1 = u32_replace_bits(val32_init1, info->wd_dma_act_intvl,
+				       B_BE_CFG_WD_PERIOD_ACTIVE_MASK);
+
+	rtw89_write32(rtwdev, R_BE_HAXI_INIT_CFG1, val32_init1);
+	rtw89_write32(rtwdev, R_BE_RX_APPEND_MODE, val32_rxapp);
+	rtw89_write32(rtwdev, R_BE_HAXI_EXP_CTRL_V1, val32_exp);
+}
+
+static int rtw89_pci_rst_bdram_be(struct rtw89_dev *rtwdev)
+{
+	u32 val;
+
+	rtw89_write32_set(rtwdev, R_BE_HAXI_INIT_CFG1, B_BE_SET_BDRAM_BOUND);
+
+	return read_poll_timeout(rtw89_read32, val, !(val & B_BE_SET_BDRAM_BOUND),
+				 50, 500000, false, rtwdev, R_BE_HAXI_INIT_CFG1);
+}
+
+static void rtw89_pci_debounce_be(struct rtw89_dev *rtwdev)
+{
+	u32 val32;
+
+	val32 = rtw89_read32(rtwdev, R_BE_SYS_PAGE_CLK_GATED);
+	val32 = u32_replace_bits(val32, 0, B_BE_PCIE_PRST_DEBUNC_PERIOD_MASK);
+	val32 |= B_BE_SYM_PRST_DEBUNC_SEL;
+	rtw89_write32(rtwdev, R_BE_SYS_PAGE_CLK_GATED, val32);
+}
+
+static void rtw89_pci_ldo_low_pwr_be(struct rtw89_dev *rtwdev)
+{
+	rtw89_write32_set(rtwdev, R_BE_SYS_PW_CTRL, B_BE_PSUS_OFF_CAPC_EN);
+	rtw89_write32_set(rtwdev, R_BE_SYS_PAGE_CLK_GATED,
+			  B_BE_SOP_OFFPOOBS_PC | B_BE_CPHY_AUXCLK_OP |
+			  B_BE_CPHY_POWER_READY_CHK);
+	rtw89_write32_clr(rtwdev, R_BE_SYS_SDIO_CTRL, B_BE_PCIE_FORCE_IBX_EN |
+						      B_BE_PCIE_DIS_L2_RTK_PERST |
+						      B_BE_PCIE_DIS_L2__CTRL_LDO_HCI);
+	rtw89_write32_clr(rtwdev, R_BE_L1_2_CTRL_HCILDO, B_BE_PCIE_DIS_L1_2_CTRL_HCILDO);
+}
+
+static void rtw89_pci_pcie_setting_be(struct rtw89_dev *rtwdev)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+	struct rtw89_hal *hal = &rtwdev->hal;
+
+	rtw89_write32_set(rtwdev, R_BE_PCIE_FRZ_CLK, B_BE_PCIE_EN_AUX_CLK);
+	rtw89_write32_clr(rtwdev, R_BE_PCIE_PS_CTRL, B_BE_CMAC_EXIT_L1_EN);
+
+	if (chip->chip_id == RTL8922A && hal->cv == CHIP_CAV)
+		return;
+
+	rtw89_write32_set(rtwdev, R_BE_EFUSE_CTRL_2_V1, B_BE_R_SYM_AUTOLOAD_WITH_PMC_SEL);
+	rtw89_write32_set(rtwdev, R_BE_PCIE_LAT_CTRL, B_BE_SYM_AUX_CLK_SEL);
+}
+
+static void rtw89_pci_ser_setting_be(struct rtw89_dev *rtwdev)
+{
+	u32 val32;
+
+	rtw89_write32(rtwdev, R_BE_PL1_DBG_INFO, 0x0);
+	rtw89_write32_set(rtwdev, R_BE_FWS1IMR, B_BE_PCIE_SER_TIMEOUT_INDIC_EN);
+	rtw89_write32_set(rtwdev, R_BE_SER_PL1_CTRL, B_BE_PL1_SER_PL1_EN);
+
+	val32 = rtw89_read32(rtwdev, R_BE_REG_PL1_MASK);
+	val32 |= B_BE_SER_PMU_IMR | B_BE_SER_L1SUB_IMR | B_BE_SER_PM_MASTER_IMR |
+		 B_BE_SER_LTSSM_IMR | B_BE_SER_PM_CLK_MASK | B_BE_SER_PCLKREQ_ACK_MASK;
+	rtw89_write32(rtwdev, R_BE_REG_PL1_MASK, val32);
+}
+
+static void rtw89_pci_ctrl_txdma_ch_be(struct rtw89_dev *rtwdev, bool all_en,
+				       bool h2c_en)
+{
+	u32 mask_all;
+	u32 val;
+
+	mask_all = B_BE_STOP_CH0 | B_BE_STOP_CH1 | B_BE_STOP_CH2 |
+		   B_BE_STOP_CH3 | B_BE_STOP_CH4 | B_BE_STOP_CH5 |
+		   B_BE_STOP_CH6 | B_BE_STOP_CH7 | B_BE_STOP_CH8 |
+		   B_BE_STOP_CH9 | B_BE_STOP_CH10 | B_BE_STOP_CH11;
+
+	val = rtw89_read32(rtwdev, R_BE_HAXI_DMA_STOP1);
+	val |= B_BE_STOP_CH13 | B_BE_STOP_CH14;
+
+	if (all_en)
+		val &= ~mask_all;
+	else
+		val |= mask_all;
+
+	if (h2c_en)
+		val &= ~B_BE_STOP_CH12;
+	else
+		val |= B_BE_STOP_CH12;
+
+	rtw89_write32(rtwdev, R_BE_HAXI_DMA_STOP1, val);
+}
+
+static int rtw89_pci_ops_mac_pre_init_be(struct rtw89_dev *rtwdev)
+{
+	int ret;
+
+	rtw89_pci_set_io_rcy_be(rtwdev);
+	_patch_pcie_power_wake_be(rtwdev, true);
+	rtw89_pci_ctrl_wpdma_pcie_be(rtwdev, false);
+	rtw89_pci_ctrl_trxdma_pcie_be(rtwdev, MAC_AX_PCIE_DISABLE,
+				      MAC_AX_PCIE_DISABLE, MAC_AX_PCIE_DISABLE);
+	rtw89_pci_clr_idx_all_be(rtwdev);
+
+	ret = rtw89_pci_poll_dma_all_idle_be(rtwdev);
+	if (ret) {
+		rtw89_err(rtwdev, "[ERR] poll pcie dma all idle\n");
+		return ret;
+	}
+
+	rtw89_pci_mode_op_be(rtwdev);
+	rtw89_pci_ops_reset(rtwdev);
+
+	ret = rtw89_pci_rst_bdram_be(rtwdev);
+	if (ret) {
+		rtw89_err(rtwdev, "[ERR]pcie rst bdram\n");
+		return ret;
+	}
+
+	rtw89_pci_debounce_be(rtwdev);
+	rtw89_pci_ldo_low_pwr_be(rtwdev);
+	rtw89_pci_pcie_setting_be(rtwdev);
+	rtw89_pci_ser_setting_be(rtwdev);
+
+	rtw89_pci_ctrl_txdma_ch_be(rtwdev, false, true);
+	rtw89_pci_ctrl_trxdma_pcie_be(rtwdev, MAC_AX_PCIE_ENABLE,
+				      MAC_AX_PCIE_ENABLE, MAC_AX_PCIE_ENABLE);
+
+	return 0;
+}
+
+static int rtw89_pci_ops_mac_pre_deinit_be(struct rtw89_dev *rtwdev)
+{
+	u32 val;
+
+	_patch_pcie_power_wake_be(rtwdev, false);
+
+	val = rtw89_read32_mask(rtwdev, R_BE_IC_PWR_STATE, B_BE_WLMAC_PWR_STE_MASK);
+	if (val == 0)
+		return 0;
+
+	rtw89_pci_ctrl_trxdma_pcie_be(rtwdev, MAC_AX_PCIE_DISABLE,
+				      MAC_AX_PCIE_DISABLE, MAC_AX_PCIE_DISABLE);
+	rtw89_pci_clr_idx_all_be(rtwdev);
+
+	return 0;
+}
+
+int rtw89_pci_ltr_set_v2(struct rtw89_dev *rtwdev, bool en)
+{
+	u32 ctrl0, cfg0, cfg1, dec_ctrl, idle_ltcy, act_ltcy, dis_ltcy;
+
+	ctrl0 = rtw89_read32(rtwdev, R_BE_LTR_CTRL_0);
+	if (rtw89_pci_ltr_is_err_reg_val(ctrl0))
+		return -EINVAL;
+	cfg0 = rtw89_read32(rtwdev, R_BE_LTR_CFG_0);
+	if (rtw89_pci_ltr_is_err_reg_val(cfg0))
+		return -EINVAL;
+	cfg1 = rtw89_read32(rtwdev, R_BE_LTR_CFG_1);
+	if (rtw89_pci_ltr_is_err_reg_val(cfg1))
+		return -EINVAL;
+	dec_ctrl = rtw89_read32(rtwdev, R_BE_LTR_DECISION_CTRL_V1);
+	if (rtw89_pci_ltr_is_err_reg_val(dec_ctrl))
+		return -EINVAL;
+	idle_ltcy = rtw89_read32(rtwdev, R_BE_LTR_LATENCY_IDX3_V1);
+	if (rtw89_pci_ltr_is_err_reg_val(idle_ltcy))
+		return -EINVAL;
+	act_ltcy = rtw89_read32(rtwdev, R_BE_LTR_LATENCY_IDX1_V1);
+	if (rtw89_pci_ltr_is_err_reg_val(act_ltcy))
+		return -EINVAL;
+	dis_ltcy = rtw89_read32(rtwdev, R_BE_LTR_LATENCY_IDX0_V1);
+	if (rtw89_pci_ltr_is_err_reg_val(dis_ltcy))
+		return -EINVAL;
+
+	if (en) {
+		dec_ctrl |= B_BE_ENABLE_LTR_CTL_DECISION | B_BE_LTR_HW_DEC_EN_V1;
+		ctrl0 |= B_BE_LTR_HW_EN;
+	} else {
+		dec_ctrl &= ~(B_BE_ENABLE_LTR_CTL_DECISION | B_BE_LTR_HW_DEC_EN_V1 |
+			      B_BE_LTR_EN_PORT_V1_MASK);
+		ctrl0 &= ~B_BE_LTR_HW_EN;
+	}
+
+	dec_ctrl = u32_replace_bits(dec_ctrl, PCI_LTR_SPC_500US,
+				    B_BE_LTR_SPACE_IDX_MASK);
+	cfg0 = u32_replace_bits(cfg0, PCI_LTR_IDLE_TIMER_3_2MS,
+				B_BE_LTR_IDLE_TIMER_IDX_MASK);
+	cfg1 = u32_replace_bits(cfg1, 0xC0, B_BE_LTR_CMAC0_RX_USE_PG_TH_MASK);
+	cfg1 = u32_replace_bits(cfg1, 0xC0, B_BE_LTR_CMAC1_RX_USE_PG_TH_MASK);
+	cfg0 = u32_replace_bits(cfg0, 1, B_BE_LTR_IDX_ACTIVE_MASK);
+	cfg0 = u32_replace_bits(cfg0, 3, B_BE_LTR_IDX_IDLE_MASK);
+	dec_ctrl = u32_replace_bits(dec_ctrl, 0, B_BE_LTR_IDX_DISABLE_V1_MASK);
+
+	rtw89_write32(rtwdev, R_BE_LTR_LATENCY_IDX3_V1, 0x90039003);
+	rtw89_write32(rtwdev, R_BE_LTR_LATENCY_IDX1_V1, 0x880b880b);
+	rtw89_write32(rtwdev, R_BE_LTR_LATENCY_IDX0_V1, 0);
+	rtw89_write32(rtwdev, R_BE_LTR_DECISION_CTRL_V1, dec_ctrl);
+	rtw89_write32(rtwdev, R_BE_LTR_CFG_0, cfg0);
+	rtw89_write32(rtwdev, R_BE_LTR_CFG_1, cfg1);
+	rtw89_write32(rtwdev, R_BE_LTR_CTRL_0, ctrl0);
+
+	return 0;
+}
+EXPORT_SYMBOL(rtw89_pci_ltr_set_v2);
+
+static void rtw89_pci_configure_mit_be(struct rtw89_dev *rtwdev)
+{
+	u32 cnt;
+	u32 val;
+
+	rtw89_write32_mask(rtwdev, R_BE_PCIE_MIT0_TMR,
+			   B_BE_PCIE_MIT0_RX_TMR_MASK, BE_MIT0_TMR_UNIT_1MS);
+
+	val = rtw89_read32(rtwdev, R_BE_PCIE_MIT0_CNT);
+	cnt = min_t(u32, U8_MAX, RTW89_PCI_RXBD_NUM_MAX / 2);
+	val = u32_replace_bits(val, cnt, B_BE_PCIE_RX_MIT0_CNT_MASK);
+	val = u32_replace_bits(val, 2, B_BE_PCIE_RX_MIT0_TMR_CNT_MASK);
+	rtw89_write32(rtwdev, R_BE_PCIE_MIT0_CNT, val);
+}
+
+static int rtw89_pci_ops_mac_post_init_be(struct rtw89_dev *rtwdev)
+{
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
+	int ret;
+
+	ret = info->ltr_set(rtwdev, true);
+	if (ret) {
+		rtw89_err(rtwdev, "pci ltr set fail\n");
+		return ret;
+	}
+
+	rtw89_pci_ctrl_trxdma_pcie_be(rtwdev, MAC_AX_PCIE_IGNORE,
+				      MAC_AX_PCIE_IGNORE, MAC_AX_PCIE_ENABLE);
+	rtw89_pci_ctrl_wpdma_pcie_be(rtwdev, true);
+	rtw89_pci_ctrl_txdma_ch_be(rtwdev, true, true);
+	rtw89_pci_configure_mit_be(rtwdev);
+
+	return 0;
+}
+
+static int rtw89_pci_poll_io_idle_be(struct rtw89_dev *rtwdev)
+{
+	u32 sts;
+	int ret;
+
+	ret = read_poll_timeout_atomic(rtw89_read32, sts,
+				       !(sts & B_BE_HAXI_MST_BUSY),
+				       10, 1000, false, rtwdev,
+				       R_BE_HAXI_DMA_BUSY1);
+	if (ret) {
+		rtw89_err(rtwdev, "pci dmach busy1 0x%X\n", sts);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int rtw89_pci_lv1rst_stop_dma_be(struct rtw89_dev *rtwdev)
+{
+	int ret;
+
+	rtw89_pci_ctrl_dma_all(rtwdev, false);
+	ret = rtw89_pci_poll_io_idle_be(rtwdev);
+	if (!ret)
+		return 0;
+
+	rtw89_debug(rtwdev, RTW89_DBG_HCI,
+		    "[PCIe] poll_io_idle fail; reset hci dma trx\n");
+
+	rtw89_mac_ctrl_hci_dma_trx(rtwdev, false);
+	rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
+
+	return rtw89_pci_poll_io_idle_be(rtwdev);
+}
+
+static int rtw89_pci_lv1rst_start_dma_be(struct rtw89_dev *rtwdev)
+{
+	int ret;
+
+	rtw89_mac_ctrl_hci_dma_trx(rtwdev, false);
+	rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
+	rtw89_pci_clr_idx_all(rtwdev);
+
+	ret = rtw89_pci_rst_bdram_be(rtwdev);
+	if (ret)
+		return ret;
+
+	rtw89_pci_ctrl_dma_all(rtwdev, true);
+	return 0;
+}
+
+const struct rtw89_pci_gen_def rtw89_pci_gen_be = {
+	.isr_rdu = B_BE_RDU_CH1_INT | B_BE_RDU_CH0_INT,
+	.isr_halt_c2h = B_BE_HALT_C2H_INT,
+	.isr_wdt_timeout = B_BE_WDT_TIMEOUT_INT,
+	.isr_clear_rpq = {R_BE_PCIE_DMA_ISR, B_BE_PCIE_RX_RPQ0_ISR_V1},
+	.isr_clear_rxq = {R_BE_PCIE_DMA_ISR, B_BE_PCIE_RX_RX0P2_ISR_V1},
+
+	.mac_pre_init = rtw89_pci_ops_mac_pre_init_be,
+	.mac_pre_deinit = rtw89_pci_ops_mac_pre_deinit_be,
+	.mac_post_init = rtw89_pci_ops_mac_post_init_be,
+
+	.clr_idx_all = rtw89_pci_clr_idx_all_be,
+	.rst_bdram = rtw89_pci_rst_bdram_be,
+
+	.lv1rst_stop_dma = rtw89_pci_lv1rst_stop_dma_be,
+	.lv1rst_start_dma = rtw89_pci_lv1rst_start_dma_be,
+};
+EXPORT_SYMBOL(rtw89_pci_gen_be);
diff --git a/drivers/net/wireless/realtek/rtw89/reg.h b/drivers/net/wireless/realtek/rtw89/reg.h
index ccd5481e8a3d..080db70b87d2 100644
--- a/drivers/net/wireless/realtek/rtw89/reg.h
+++ b/drivers/net/wireless/realtek/rtw89/reg.h
@@ -116,6 +116,7 @@
 #define B_AX_LTE_MUX_CTRL_PATH BIT(26)
 
 #define R_AX_HCI_OPT_CTRL 0x0074
+#define BIT_WAKE_CTRL_V1 BIT(23)
 #define BIT_WAKE_CTRL BIT(5)
 
 #define R_AX_HCI_BG_CTRL 0x0078
@@ -3629,6 +3630,33 @@
 #define B_AX_GNT_BT_TX_SW_VAL BIT(1)
 #define B_AX_GNT_BT_TX_SW_CTRL BIT(0)
 
+#define R_BE_SYS_PW_CTRL 0x0004
+#define B_BE_SOP_ASWRM BIT(31)
+#define B_BE_SOP_EASWR BIT(30)
+#define B_BE_SOP_PWMM_DSWR BIT(29)
+#define B_BE_SOP_EDSWR BIT(28)
+#define B_BE_SOP_ACKF BIT(27)
+#define B_BE_SOP_ERCK BIT(26)
+#define B_BE_SOP_ANA_CLK_DIVISION_2 BIT(25)
+#define B_BE_SOP_EXTL BIT(24)
+#define B_BE_SOP_OFF_CAPC_EN BIT(23)
+#define B_BE_XTAL_OFF_A_DIE BIT(22)
+#define B_BE_ROP_SWPR BIT(21)
+#define B_BE_DIS_HW_LPLDM BIT(20)
+#define B_BE_DIS_HW_LPURLDO BIT(19)
+#define B_BE_DIS_WLBT_PDNSUSEN_SOPC BIT(18)
+#define B_BE_RDY_SYSPWR BIT(17)
+#define B_BE_EN_WLON BIT(16)
+#define B_BE_APDM_HPDN BIT(15)
+#define B_BE_PSUS_OFF_CAPC_EN BIT(14)
+#define B_BE_AFSM_PCIE_SUS_EN BIT(12)
+#define B_BE_AFSM_WLSUS_EN BIT(11)
+#define B_BE_APFM_SWLPS BIT(10)
+#define B_BE_APFM_OFFMAC BIT(9)
+#define B_BE_APFN_ONMAC BIT(8)
+#define B_BE_CHIP_PDN_EN BIT(7)
+#define B_BE_RDY_MACDIS BIT(6)
+
 #define R_BE_SYS_CLK_CTRL 0x0008
 #define B_BE_CPU_CLK_EN BIT(14)
 #define B_BE_SYMR_BE_CLK_EN BIT(13)
@@ -3639,6 +3667,90 @@
 #define B_BE_ANA_CLK_DIVISION_2 BIT(1)
 #define B_BE_CNTD16V_EN BIT(0)
 
+#define R_BE_SYS_PAGE_CLK_GATED 0x000C
+#define B_BE_USB_APHY_PC_DLP_OP BIT(27)
+#define B_BE_PCIE_APHY_PC_DLP_OP BIT(26)
+#define B_BE_UPHY_POWER_READY_CHK BIT(25)
+#define B_BE_CPHY_POWER_READY_CHK BIT(24)
+#define B_BE_PCIE_PRST_DEBUNC_PERIOD_MASK GENMASK(23, 22)
+#define B_BE_SYM_PRST_DEBUNC_SEL BIT(21)
+#define B_BE_CPHY_AUXCLK_OP BIT(20)
+#define B_BE_SOP_OFFUA_PC BIT(19)
+#define B_BE_SOP_OFFPOOBS_PC BIT(18)
+#define B_BE_PCIE_LAN1_MASK BIT(17)
+#define B_BE_PCIE_LAN0_MASK BIT(16)
+#define B_BE_DIS_CLK_REGF_GATE BIT(15)
+#define B_BE_DIS_CLK_REGE_GATE BIT(14)
+#define B_BE_DIS_CLK_REGD_GATE BIT(13)
+#define B_BE_DIS_CLK_REGC_GATE BIT(12)
+#define B_BE_DIS_CLK_REGB_GATE BIT(11)
+#define B_BE_DIS_CLK_REGA_GATE BIT(10)
+#define B_BE_DIS_CLK_REG9_GATE BIT(9)
+#define B_BE_DIS_CLK_REG8_GATE BIT(8)
+#define B_BE_DIS_CLK_REG7_GATE BIT(7)
+#define B_BE_DIS_CLK_REG6_GATE BIT(6)
+#define B_BE_DIS_CLK_REG5_GATE BIT(5)
+#define B_BE_DIS_CLK_REG4_GATE BIT(4)
+#define B_BE_DIS_CLK_REG3_GATE BIT(3)
+#define B_BE_DIS_CLK_REG2_GATE BIT(2)
+#define B_BE_DIS_CLK_REG1_GATE BIT(1)
+#define B_BE_DIS_CLK_REG0_GATE BIT(0)
+
+#define R_BE_SYS_SDIO_CTRL 0x0070
+#define B_BE_MCM_FLASH_EN BIT(28)
+#define B_BE_PCIE_SEC_LOAD BIT(26)
+#define B_BE_PCIE_SER_RSTB BIT(25)
+#define B_BE_PCIE_SEC_LOAD_CLR BIT(24)
+#define B_BE_SDIO_CMD_SW_RST BIT(20)
+#define B_BE_SDIO_INT_POLARITY BIT(19)
+#define B_BE_SDIO_OFF_EN BIT(17)
+#define B_BE_SDIO_ON_EN BIT(16)
+#define B_BE_PCIE_DIS_L2__CTRL_LDO_HCI BIT(15)
+#define B_BE_PCIE_DIS_L2_RTK_PERST BIT(14)
+#define B_BE_PCIE_FORCE_PWR_NGAT BIT(13)
+#define B_BE_PCIE_FORCE_IBX_EN BIT(12)
+#define B_BE_PCIE_AUXCLK_GATE BIT(11)
+#define B_BE_PCIE_WAIT_TIMEOUT_EVENT BIT(10)
+#define B_BE_PCIE_WAIT_TIME BIT(9)
+#define B_BE_L1OFF_TO_L0_RESUME_EVT BIT(8)
+#define B_BE_USBA_FORCE_PWR_NGAT BIT(7)
+#define B_BE_USBD_FORCE_PWR_NGAT BIT(6)
+#define B_BE_BT_CTRL_USB_PWR BIT(5)
+#define B_BE_USB_D_STATE_HOLD BIT(4)
+#define B_BE_R_BE_FORCE_DP BIT(3)
+#define B_BE_R_BE_DP_MODE BIT(2)
+#define B_BE_RES_USB_MASS_STORAGE_DESC BIT(1)
+#define B_BE_USB_WAIT_TIME BIT(0)
+
+#define R_BE_HCI_OPT_CTRL 0x0074
+#define B_BE_HCI_WLAN_IO_ST BIT(31)
+#define B_BE_HCI_WLAN_IO_EN BIT(28)
+#define B_BE_HAXIDMA_IO_ST BIT(27)
+#define B_BE_HAXIDMA_BACKUP_RESTORE_ST BIT(26)
+#define B_BE_HAXIDMA_IO_EN BIT(24)
+#define B_BE_EN_PCIE_WAKE BIT(23)
+#define B_BE_SDIO_PAD_H3L1 BIT(22)
+#define B_BE_USBMAC_ANACLK_SW BIT(21)
+#define B_BE_PCIE_CPHY_CCK_XTAL_SEL BIT(20)
+#define B_BE_SDIO_DATA_PAD_SMT BIT(19)
+#define B_BE_SDIO_PAD_E5 BIT(18)
+#define B_BE_FORCE_PCIE_AUXCLK BIT(17)
+#define B_BE_HCI_LA_ADDR_MAP BIT(16)
+#define B_BE_HCI_LA_GLO_RST BIT(15)
+#define B_BE_USB3_SUS_DIS BIT(14)
+#define B_BE_NOPWR_CTRL_SEL BIT(13)
+#define B_BE_USB_HOST_PWR_OFF_EN BIT(12)
+#define B_BE_SYM_LPS_BLOCK_EN BIT(11)
+#define B_BE_USB_LPM_ACT_EN BIT(10)
+#define B_BE_USB_LPM_NY BIT(9)
+#define B_BE_USB2_SUS_DIS BIT(8)
+#define B_BE_SDIO_PAD_E_MASK GENMASK(7, 5)
+#define B_BE_USB_LPPLL_EN BIT(4)
+#define B_BE_USB1_1_USB2_0_DECISION BIT(3)
+#define B_BE_ROP_SW15 BIT(2)
+#define B_BE_PCI_CKRDY_OPT BIT(1)
+#define B_BE_PCI_VAUX_EN BIT(0)
+
 #define R_BE_PLATFORM_ENABLE 0x0088
 #define B_BE_HOLD_AFTER_RESET BIT(11)
 #define B_BE_SYM_WLPLT_MEM_MUX_EN BIT(10)
@@ -3652,6 +3764,22 @@
 #define B_BE_WCPU_EN BIT(1)
 #define B_BE_PLATFORM_EN BIT(0)
 
+#define R_BE_EFUSE_CTRL_2_V1 0x00A4
+#define B_BE_EF_ENT BIT(31)
+#define B_BE_EF_TCOLUMN_EN BIT(29)
+#define B_BE_BT_OTP_PWC_DIS BIT(28)
+#define B_BE_EF_RDT BIT(27)
+#define B_BE_R_SYM_AUTOLOAD_WITH_PMC_SEL BIT(24)
+#define B_BE_EF_PGTS_MASK GENMASK(23, 20)
+#define B_BE_EF_BURST BIT(19)
+#define B_BE_EF_TEST_SEL_MASK GENMASK(18, 16)
+#define B_BE_EF_TROW_EN BIT(15)
+#define B_BE_EF_ERR_FLAG BIT(14)
+#define B_BE_EF_FBURST_DIS BIT(13)
+#define B_BE_EF_HT_SEL BIT(12)
+#define B_BE_EF_DSB_EN BIT(11)
+#define B_BE_EF_DLY_SEL_MASK GENMASK(3, 0)
+
 #define R_BE_HALT_H2C_CTRL 0x0160
 #define B_BE_HALT_H2C_TRIGGER BIT(0)
 
@@ -3676,6 +3804,82 @@
 
 #define R_BE_SECURE_BOOT_MALLOC_INFO 0x0184
 
+#define R_BE_FWS1IMR 0x0198
+#define B_BE_FS_RPWM_INT_EN_V1 BIT(24)
+#define B_BE_PCIE_HOTRST_EN BIT(22)
+#define B_BE_PCIE_SER_TIMEOUT_INDIC_EN BIT(21)
+#define B_BE_PCIE_RXI300_SLVTOUT_INDIC_EN BIT(20)
+#define B_BE_AON_PCIE_FLR_INT_EN BIT(19)
+#define B_BE_PCIE_ERR_INDIC_INT_EN BIT(18)
+#define B_BE_SDIO_ERR_INDIC_INT_EN BIT(17)
+#define B_BE_USB_ERR_INDIC_INT_EN BIT(16)
+#define B_BE_FS_GPIO27_INT_EN BIT(11)
+#define B_BE_FS_GPIO26_INT_EN BIT(10)
+#define B_BE_FS_GPIO25_INT_EN BIT(9)
+#define B_BE_FS_GPIO24_INT_EN BIT(8)
+#define B_BE_FS_GPIO23_INT_EN BIT(7)
+#define B_BE_FS_GPIO22_INT_EN BIT(6)
+#define B_BE_FS_GPIO21_INT_EN BIT(5)
+#define B_BE_FS_GPIO20_INT_EN BIT(4)
+#define B_BE_FS_GPIO19_INT_EN BIT(3)
+#define B_BE_FS_GPIO18_INT_EN BIT(2)
+#define B_BE_FS_GPIO17_INT_EN BIT(1)
+#define B_BE_FS_GPIO16_INT_EN BIT(0)
+
+#define R_BE_HIMR0 0x01A0
+#define B_BE_WDT_DATACPU_TIMEOUT_INT_EN BIT(25)
+#define B_BE_HALT_D2H_INT_EN BIT(24)
+#define B_BE_WDT_TIMEOUT_INT_EN BIT(22)
+#define B_BE_HALT_C2H_INT_EN BIT(21)
+#define B_BE_RON_INT_EN BIT(20)
+#define B_BE_PDNINT_EN BIT(19)
+#define B_BE_SPSANA_OCP_INT_EN BIT(18)
+#define B_BE_SPS_OCP_INT_EN BIT(17)
+#define B_BE_BTON_STS_UPDATE_INT_EN BIT(16)
+#define B_BE_GPIOF_INT_EN BIT(15)
+#define B_BE_GPIOE_INT_EN BIT(14)
+#define B_BE_GPIOD_INT_EN BIT(13)
+#define B_BE_GPIOC_INT_EN BIT(12)
+#define B_BE_GPIOB_INT_EN BIT(11)
+#define B_BE_GPIOA_INT_EN BIT(10)
+#define B_BE_GPIO9_INT_EN BIT(9)
+#define B_BE_GPIO8_INT_EN BIT(8)
+#define B_BE_GPIO7_INT_EN BIT(7)
+#define B_BE_GPIO6_INT_EN BIT(6)
+#define B_BE_GPIO5_INT_EN BIT(5)
+#define B_BE_GPIO4_INT_EN BIT(4)
+#define B_BE_GPIO3_INT_EN BIT(3)
+#define B_BE_GPIO2_INT_EN BIT(2)
+#define B_BE_GPIO1_INT_EN BIT(1)
+#define B_BE_GPIO0_INT_EN BIT(0)
+
+#define R_BE_HISR0 0x01A4
+#define B_BE_WDT_DATACPU_TIMEOUT_INT BIT(25)
+#define B_BE_HALT_D2H_INT BIT(24)
+#define B_BE_WDT_TIMEOUT_INT BIT(22)
+#define B_BE_HALT_C2H_INT BIT(21)
+#define B_BE_RON_INT BIT(20)
+#define B_BE_PDNINT BIT(19)
+#define B_BE_SPSANA_OCP_INT BIT(18)
+#define B_BE_SPS_OCP_INT BIT(17)
+#define B_BE_BTON_STS_UPDATE_INT BIT(16)
+#define B_BE_GPIOF_INT BIT(15)
+#define B_BE_GPIOE_INT BIT(14)
+#define B_BE_GPIOD_INT BIT(13)
+#define B_BE_GPIOC_INT BIT(12)
+#define B_BE_GPIOB_INT BIT(11)
+#define B_BE_GPIOA_INT BIT(10)
+#define B_BE_GPIO9_INT BIT(9)
+#define B_BE_GPIO8_INT BIT(8)
+#define B_BE_GPIO7_INT BIT(7)
+#define B_BE_GPIO6_INT BIT(6)
+#define B_BE_GPIO5_INT BIT(5)
+#define B_BE_GPIO4_INT BIT(4)
+#define B_BE_GPIO3_INT BIT(3)
+#define B_BE_GPIO2_INT BIT(2)
+#define B_BE_GPIO1_INT BIT(1)
+#define B_BE_GPIO0_INT BIT(0)
+
 #define R_BE_WCPU_FW_CTRL 0x01E0
 #define B_BE_RUN_ENV_MASK GENMASK(31, 30)
 #define B_BE_WCPU_FWDL_STATUS_MASK GENMASK(29, 26)
@@ -3721,6 +3925,15 @@
 #define R_BE_UDM2 0x01F8
 #define B_BE_UDM2_EPC_RA_MASK GENMASK(31, 0)
 
+#define R_BE_IC_PWR_STATE 0x03F0
+#define B_BE_WHOLE_SYS_PWR_STE_MASK GENMASK(25, 16)
+#define MAC_AX_SYS_ACT 0x220
+#define B_BE_WLMAC_PWR_STE_MASK GENMASK(9, 8)
+#define B_BE_UART_HCISYS_PWR_STE_MASK GENMASK(7, 6)
+#define B_BE_SDIO_HCISYS_PWR_STE_MASK GENMASK(5, 4)
+#define B_BE_USB_HCISYS_PWR_STE_MASK GENMASK(3, 2)
+#define B_BE_PCIE_HCISYS_PWR_STE_MASK GENMASK(1, 0)
+
 #define R_BE_DCPU_PLATFORM_ENABLE 0x0888
 #define B_BE_DCPU_SYM_DPLT_MEM_MUX_EN BIT(10)
 #define B_BE_DCPU_WARM_EN BIT(9)
@@ -3732,6 +3945,127 @@
 
 #define R_BE_FILTER_MODEL_ADDR 0x0C04
 
+#define R_BE_WLAN_WDT 0x3050
+#define B_BE_WLAN_WDT_TIMEOUT BIT(31)
+#define B_BE_WLAN_WDT_TIMER_CLEAR BIT(4)
+#define B_BE_WLAN_WDT_BYPASS BIT(1)
+#define B_BE_WLAN_WDT_ENABLE BIT(0)
+
+#define R_BE_AXIDMA_WDT 0x305C
+#define B_BE_AXIDMA_WDT_TIMEOUT BIT(31)
+#define B_BE_AXIDMA_WDT_TIMER_CLEAR BIT(4)
+#define B_BE_AXIDMA_WDT_BYPASS BIT(1)
+#define B_BE_AXIDMA_WDT_ENABLE BIT(0)
+
+#define R_BE_AON_WDT 0x3068
+#define B_BE_AON_WDT_TIMEOUT BIT(31)
+#define B_BE_AON_WDT_TIMER_CLEAR BIT(4)
+#define B_BE_AON_WDT_BYPASS BIT(1)
+#define B_BE_AON_WDT_ENABLE BIT(0)
+
+#define R_BE_AON_WDT_TMR 0x306C
+#define R_BE_MDIO_WDT_TMR 0x3090
+#define R_BE_LA_MODE_WDT_TMR 0x309C
+#define R_BE_WDT_AR_TMR 0x3144
+#define R_BE_WDT_AW_TMR 0x3150
+#define R_BE_WLAN_WDT_TMR 0x3054
+#define R_BE_WDT_W_TMR 0x315C
+#define R_BE_AXIDMA_WDT_TMR 0x3060
+#define R_BE_WDT_B_TMR 0x3164
+#define R_BE_WDT_R_TMR 0x316C
+#define R_BE_LOCAL_WDT_TMR 0x3084
+
+#define R_BE_LOCAL_WDT 0x3080
+#define B_BE_LOCAL_WDT_TIMEOUT BIT(31)
+#define B_BE_LOCAL_WDT_TIMER_CLEAR BIT(4)
+#define B_BE_LOCAL_WDT_BYPASS BIT(1)
+#define B_BE_LOCAL_WDT_ENABLE BIT(0)
+
+#define R_BE_MDIO_WDT 0x308C
+#define B_BE_MDIO_WDT_TIMEOUT BIT(31)
+#define B_BE_MDIO_WDT_TIMER_CLEAR BIT(4)
+#define B_BE_MDIO_WDT_BYPASS BIT(1)
+#define B_BE_MDIO_WDT_ENABLE BIT(0)
+
+#define R_BE_LA_MODE_WDT 0x3098
+#define B_BE_LA_MODE_WDT_TIMEOUT BIT(31)
+#define B_BE_LA_MODE_WDT_TIMER_CLEAR BIT(4)
+#define B_BE_LA_MODE_WDT_BYPASS BIT(1)
+#define B_BE_LA_MODE_WDT_ENABLE BIT(0)
+
+#define R_BE_WDT_AR 0x3140
+#define B_BE_WDT_AR_TIMEOUT BIT(31)
+#define B_BE_WDT_AR_TIMER_CLEAR BIT(4)
+#define B_BE_WDT_AR_BYPASS BIT(1)
+#define B_BE_WDT_AR_ENABLE BIT(0)
+
+#define R_BE_WDT_AW 0x314C
+#define B_BE_WDT_AW_TIMEOUT BIT(31)
+#define B_BE_WDT_AW_TIMER_CLEAR BIT(4)
+#define B_BE_WDT_AW_BYPASS BIT(1)
+#define B_BE_WDT_AW_ENABLE BIT(0)
+
+#define R_BE_WDT_W 0x3158
+#define B_BE_WDT_W_TIMEOUT BIT(31)
+#define B_BE_WDT_W_TIMER_CLEAR BIT(4)
+#define B_BE_WDT_W_BYPASS BIT(1)
+#define B_BE_WDT_W_ENABLE BIT(0)
+
+#define R_BE_WDT_B 0x3160
+#define B_BE_WDT_B_TIMEOUT BIT(31)
+#define B_BE_WDT_B_TIMER_CLEAR BIT(4)
+#define B_BE_WDT_B_BYPASS BIT(1)
+#define B_BE_WDT_B_ENABLE BIT(0)
+
+#define R_BE_WDT_R 0x3168
+#define B_BE_WDT_R_TIMEOUT BIT(31)
+#define B_BE_WDT_R_TIMER_CLEAR BIT(4)
+#define B_BE_WDT_R_BYPASS BIT(1)
+#define B_BE_WDT_R_ENABLE BIT(0)
+
+#define R_BE_LTR_DECISION_CTRL_V1 0x3610
+#define B_BE_ENABLE_LTR_CTL_DECISION BIT(31)
+#define B_BE_LAT_LTR_IDX_DRV_VLD_V1 BIT(24)
+#define B_BE_LAT_LTR_IDX_DRV_V1_MASK GENMASK(23, 22)
+#define B_BE_LAT_LTR_IDX_FW_VLD_V1 BIT(21)
+#define B_BE_LAT_LTR_IDX_FW_V1_MASK GENMASK(20, 19)
+#define B_BE_LAT_LTR_IDX_HW_VLD_V1 BIT(18)
+#define B_BE_LAT_LTR_IDX_HW_V1_MASK GENMASK(17, 16)
+#define B_BE_LTR_IDX_DRV_V1_MASK GENMASK(15, 14)
+#define B_BE_LTR_REQ_DRV_V1 BIT(13)
+#define B_BE_LTR_IDX_DISABLE_V1_MASK GENMASK(9, 8)
+#define B_BE_LTR_EN_PORT_V1_MASK GENMASK(6, 4)
+#define B_BE_LTR_DRV_DEC_EN_V1 BIT(6)
+#define B_BE_LTR_FW_DEC_EN_V1 BIT(5)
+#define B_BE_LTR_HW_DEC_EN_V1 BIT(4)
+#define B_BE_LTR_SPACE_IDX_MASK GENMASK(1, 0)
+
+#define R_BE_LTR_LATENCY_IDX0_V1 0x3614
+#define R_BE_LTR_LATENCY_IDX1_V1 0x3618
+#define R_BE_LTR_LATENCY_IDX2_V1 0x361C
+#define R_BE_LTR_LATENCY_IDX3_V1 0x3620
+
+#define R_BE_LTR_CTRL_0 0x8410
+#define B_BE_LTR_REQ_FW BIT(18)
+#define B_BE_LTR_IDX_FW_MASK GENMASK(17, 16)
+#define B_BE_LTR_IDLE_TIMER_IDX_MASK GENMASK(10, 8)
+#define B_BE_LTR_WD_NOEMP_CHK BIT(1)
+#define B_BE_LTR_HW_EN BIT(0)
+
+#define R_BE_LTR_CFG_0 0x8414
+#define B_BE_LTR_IDX_DISABLE_MASK GENMASK(17, 16)
+#define B_BE_LTR_IDX_IDLE_MASK GENMASK(15, 14)
+#define B_BE_LTR_IDX_ACTIVE_MASK GENMASK(13, 12)
+#define B_BE_LTR_IDLE_TIMER_IDX_MASK GENMASK(10, 8)
+#define B_BE_EN_LTR_CMAC_RX_USE_PG_CHK BIT(3)
+#define B_BE_EN_LTR_WD_NON_EMPTY_CHK BIT(2)
+#define B_BE_EN_LTR_HAXIDMA_TX_IDLE_CHK BIT(1)
+#define B_BE_EN_LTR_HAXIDMA_RX_IDLE_CHK BIT(0)
+
+#define R_BE_LTR_CFG_1 0x8418
+#define B_BE_LTR_CMAC1_RX_USE_PG_TH_MASK GENMASK(27, 16)
+#define B_BE_LTR_CMAC0_RX_USE_PG_TH_MASK GENMASK(11, 0)
+
 #define R_BE_PLE_DBG_FUN_INTF_CTL 0x9110
 #define B_BE_PLE_DFI_ACTIVE BIT(31)
 #define B_BE_PLE_DFI_TRGSEL_MASK GENMASK(19, 16)
@@ -3740,6 +4074,28 @@
 #define R_BE_PLE_DBG_FUN_INTF_DATA 0x9114
 #define B_BE_PLE_DFI_DATA_MASK GENMASK(31, 0)
 
+#define R_BE_HAXI_INIT_CFG1 0xB000
+#define B_BE_CFG_WD_PERIOD_IDLE_MASK GENMASK(31, 28)
+#define B_BE_CFG_WD_PERIOD_ACTIVE_MASK GENMASK(27, 24)
+#define B_BE_EN_RO_IDX_UPD_BY_IO BIT(19)
+#define B_BE_RST_KEEP_REG BIT(18)
+#define B_BE_FLUSH_HAXI_MST BIT(17)
+#define B_BE_SET_BDRAM_BOUND BIT(16)
+#define B_BE_ADDRINFO_ALIGN4B_EN BIT(15)
+#define B_BE_RXBD_DONE_MODE_MASK GENMASK(14, 13)
+#define B_BE_RXQ_RXBD_MODE_MASK GENMASK(12, 11)
+#define B_BE_DMA_MODE_MASK GENMASK(10, 8)
+#define S_BE_DMA_MOD_PCIE_NO_DATA_CPU 0x0
+#define S_BE_DMA_MOD_PCIE_DATA_CPU 0x1
+#define S_BE_DMA_MOD_USB 0x4
+#define S_BE_DMA_MOD_SDIO 0x6
+#define B_BE_STOP_AXI_MST BIT(7)
+#define B_BE_RXDMA_ALIGN64B_EN BIT(6)
+#define B_BE_RXDMA_EN BIT(5)
+#define B_BE_TXDMA_EN BIT(4)
+#define B_BE_MAX_RXDMA_MASK GENMASK(3, 2)
+#define B_BE_MAX_TXDMA_MASK GENMASK(1, 0)
+
 #define R_BE_CMAC_FUNC_EN 0x10000
 #define R_BE_CMAC_FUNC_EN_C1 0x14000
 #define B_BE_CMAC_CRPRT BIT(31)
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8851b.c b/drivers/net/wireless/realtek/rtw89/rtw8851b.c
index 50522ff85003..ffc464b2ac10 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8851b.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8851b.c
@@ -2393,6 +2393,7 @@ const struct rtw89_chip_info rtw8851b_chip_info = {
 	.bacam_num		= 2,
 	.bacam_dynamic_num	= 4,
 	.bacam_ver		= RTW89_BACAM_V0,
+	.ppdu_max_usr		= 4,
 	.sec_ctrl_efuse_size	= 4,
 	.physical_efuse_size	= 1216,
 	.logical_efuse_size	= 2048,
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8851be.c b/drivers/net/wireless/realtek/rtw89/rtw8851be.c
index 0f7711c50bd1..ade69bd30fc8 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8851be.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8851be.c
@@ -10,6 +10,7 @@
 #include "rtw8851b.h"
 
 static const struct rtw89_pci_info rtw8851b_pci_info = {
+	.gen_def		= &rtw89_pci_gen_ax,
 	.txbd_trunc_mode	= MAC_AX_BD_TRUNC,
 	.rxbd_trunc_mode	= MAC_AX_BD_TRUNC,
 	.rxbd_mode		= MAC_AX_RXBD_PKT,
@@ -33,6 +34,7 @@ static const struct rtw89_pci_info rtw8851b_pci_info = {
 	.max_tag_num_mask	= B_AX_MAX_TAG_NUM,
 	.rxbd_rwptr_clr_reg	= R_AX_RXBD_RWPTR_CLR,
 	.txbd_rwptr_clr2_reg	= 0,
+	.dma_io_stop		= {R_AX_PCIE_DMA_STOP1, B_AX_STOP_PCIEIO},
 	.dma_stop1		= {R_AX_PCIE_DMA_STOP1, B_AX_TX_STOP1_MASK_V1},
 	.dma_stop2		= {0},
 	.dma_busy1		= {R_AX_PCIE_DMA_BUSY1, DMA_BUSY1_CHECK_V1},
@@ -41,6 +43,7 @@ static const struct rtw89_pci_info rtw8851b_pci_info = {
 
 	.rpwm_addr		= R_AX_PCIE_HRPWM,
 	.cpwm_addr		= R_AX_CPWM,
+	.mit_addr		= R_AX_INT_MIT_RX,
 	.tx_dma_ch_mask		= BIT(RTW89_TXCH_ACH4) | BIT(RTW89_TXCH_ACH5) |
 				  BIT(RTW89_TXCH_ACH6) | BIT(RTW89_TXCH_ACH7) |
 				  BIT(RTW89_TXCH_CH10) | BIT(RTW89_TXCH_CH11),
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852a.c b/drivers/net/wireless/realtek/rtw89/rtw8852a.c
index 0c36e6180e25..0d6f87b900d5 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852a.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852a.c
@@ -2129,6 +2129,7 @@ const struct rtw89_chip_info rtw8852a_chip_info = {
 	.bacam_num		= 2,
 	.bacam_dynamic_num	= 4,
 	.bacam_ver		= RTW89_BACAM_V0,
+	.ppdu_max_usr		= 4,
 	.sec_ctrl_efuse_size	= 4,
 	.physical_efuse_size	= 1216,
 	.logical_efuse_size	= 1536,
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852ae.c b/drivers/net/wireless/realtek/rtw89/rtw8852ae.c
index d835a44a1d0d..f1e890bde049 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852ae.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852ae.c
@@ -10,6 +10,7 @@
 #include "rtw8852a.h"
 
 static const struct rtw89_pci_info rtw8852a_pci_info = {
+	.gen_def		= &rtw89_pci_gen_ax,
 	.txbd_trunc_mode	= MAC_AX_BD_TRUNC,
 	.rxbd_trunc_mode	= MAC_AX_BD_TRUNC,
 	.rxbd_mode		= MAC_AX_RXBD_PKT,
@@ -24,6 +25,7 @@ static const struct rtw89_pci_info rtw8852a_pci_info = {
 	.autok_en		= MAC_AX_PCIE_DISABLE,
 	.io_rcy_en		= MAC_AX_PCIE_DISABLE,
 	.io_rcy_tmr		= MAC_AX_IO_RCY_ANA_TMR_6MS,
+	.rx_ring_eq_is_full	= false,
 
 	.init_cfg_reg		= R_AX_PCIE_INIT_CFG1,
 	.txhci_en_bit		= B_AX_TXHCI_EN,
@@ -33,6 +35,7 @@ static const struct rtw89_pci_info rtw8852a_pci_info = {
 	.max_tag_num_mask	= B_AX_MAX_TAG_NUM,
 	.rxbd_rwptr_clr_reg	= R_AX_RXBD_RWPTR_CLR,
 	.txbd_rwptr_clr2_reg	= R_AX_TXBD_RWPTR_CLR2,
+	.dma_io_stop		= {R_AX_PCIE_DMA_STOP1, B_AX_STOP_PCIEIO},
 	.dma_stop1		= {R_AX_PCIE_DMA_STOP1, B_AX_TX_STOP1_MASK},
 	.dma_stop2		= {R_AX_PCIE_DMA_STOP2, B_AX_TX_STOP2_ALL},
 	.dma_busy1		= {R_AX_PCIE_DMA_BUSY1, DMA_BUSY1_CHECK},
@@ -41,6 +44,7 @@ static const struct rtw89_pci_info rtw8852a_pci_info = {
 
 	.rpwm_addr		= R_AX_PCIE_HRPWM,
 	.cpwm_addr		= R_AX_CPWM,
+	.mit_addr		= R_AX_INT_MIT_RX,
 	.tx_dma_ch_mask		= 0,
 	.bd_idx_addr_low_power	= NULL,
 	.dma_addr_set		= &rtw89_pci_ch_dma_addr_set,
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852b.c b/drivers/net/wireless/realtek/rtw89/rtw8852b.c
index 9d4e6f08218d..62c2feab569c 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852b.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852b.c
@@ -2563,6 +2563,7 @@ const struct rtw89_chip_info rtw8852b_chip_info = {
 	.bacam_num		= 2,
 	.bacam_dynamic_num	= 4,
 	.bacam_ver		= RTW89_BACAM_V0,
+	.ppdu_max_usr		= 4,
 	.sec_ctrl_efuse_size	= 4,
 	.physical_efuse_size	= 1216,
 	.logical_efuse_size	= 2048,
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852be.c b/drivers/net/wireless/realtek/rtw89/rtw8852be.c
index ecf39d2d9f81..920b20bbcfb7 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852be.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852be.c
@@ -10,6 +10,7 @@
 #include "rtw8852b.h"
 
 static const struct rtw89_pci_info rtw8852b_pci_info = {
+	.gen_def		= &rtw89_pci_gen_ax,
 	.txbd_trunc_mode	= MAC_AX_BD_TRUNC,
 	.rxbd_trunc_mode	= MAC_AX_BD_TRUNC,
 	.rxbd_mode		= MAC_AX_RXBD_PKT,
@@ -24,6 +25,7 @@ static const struct rtw89_pci_info rtw8852b_pci_info = {
 	.autok_en		= MAC_AX_PCIE_DISABLE,
 	.io_rcy_en		= MAC_AX_PCIE_DISABLE,
 	.io_rcy_tmr		= MAC_AX_IO_RCY_ANA_TMR_6MS,
+	.rx_ring_eq_is_full	= false,
 
 	.init_cfg_reg		= R_AX_PCIE_INIT_CFG1,
 	.txhci_en_bit		= B_AX_TXHCI_EN,
@@ -33,6 +35,7 @@ static const struct rtw89_pci_info rtw8852b_pci_info = {
 	.max_tag_num_mask	= B_AX_MAX_TAG_NUM,
 	.rxbd_rwptr_clr_reg	= R_AX_RXBD_RWPTR_CLR,
 	.txbd_rwptr_clr2_reg	= 0,
+	.dma_io_stop		= {R_AX_PCIE_DMA_STOP1, B_AX_STOP_PCIEIO},
 	.dma_stop1		= {R_AX_PCIE_DMA_STOP1, B_AX_TX_STOP1_MASK_V1},
 	.dma_stop2		= {0},
 	.dma_busy1		= {R_AX_PCIE_DMA_BUSY1, DMA_BUSY1_CHECK_V1},
@@ -41,6 +44,7 @@ static const struct rtw89_pci_info rtw8852b_pci_info = {
 
 	.rpwm_addr		= R_AX_PCIE_HRPWM,
 	.cpwm_addr		= R_AX_CPWM,
+	.mit_addr		= R_AX_INT_MIT_RX,
 	.tx_dma_ch_mask		= BIT(RTW89_TXCH_ACH4) | BIT(RTW89_TXCH_ACH5) |
 				  BIT(RTW89_TXCH_ACH6) | BIT(RTW89_TXCH_ACH7) |
 				  BIT(RTW89_TXCH_CH10) | BIT(RTW89_TXCH_CH11),
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852c.c b/drivers/net/wireless/realtek/rtw89/rtw8852c.c
index 3b7d8ab39bab..7bb5d359a06a 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852c.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852c.c
@@ -2877,6 +2877,7 @@ const struct rtw89_chip_info rtw8852c_chip_info = {
 	.bacam_num		= 8,
 	.bacam_dynamic_num	= 8,
 	.bacam_ver		= RTW89_BACAM_V0_EXT,
+	.ppdu_max_usr		= 8,
 	.sec_ctrl_efuse_size	= 4,
 	.physical_efuse_size	= 1216,
 	.logical_efuse_size	= 2048,
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852ce.c b/drivers/net/wireless/realtek/rtw89/rtw8852ce.c
index 80490a5437df..4592de3dbd94 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852ce.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852ce.c
@@ -19,6 +19,7 @@ static const struct rtw89_pci_bd_idx_addr rtw8852c_bd_idx_addr_low_power = {
 };
 
 static const struct rtw89_pci_info rtw8852c_pci_info = {
+	.gen_def		= &rtw89_pci_gen_ax,
 	.txbd_trunc_mode	= MAC_AX_BD_TRUNC,
 	.rxbd_trunc_mode	= MAC_AX_BD_TRUNC,
 	.rxbd_mode		= MAC_AX_RXBD_PKT,
@@ -33,6 +34,7 @@ static const struct rtw89_pci_info rtw8852c_pci_info = {
 	.autok_en		= MAC_AX_PCIE_DISABLE,
 	.io_rcy_en		= MAC_AX_PCIE_ENABLE,
 	.io_rcy_tmr		= MAC_AX_IO_RCY_ANA_TMR_6MS,
+	.rx_ring_eq_is_full	= false,
 
 	.init_cfg_reg		= R_AX_HAXI_INIT_CFG1,
 	.txhci_en_bit		= B_AX_TXHCI_EN_V1,
@@ -42,6 +44,7 @@ static const struct rtw89_pci_info rtw8852c_pci_info = {
 	.max_tag_num_mask	= B_AX_MAX_TAG_NUM_V1_MASK,
 	.rxbd_rwptr_clr_reg	= R_AX_RXBD_RWPTR_CLR_V1,
 	.txbd_rwptr_clr2_reg	= R_AX_TXBD_RWPTR_CLR2_V1,
+	.dma_io_stop		= {R_AX_HAXI_INIT_CFG1, B_AX_STOP_AXI_MST},
 	.dma_stop1		= {R_AX_HAXI_DMA_STOP1, B_AX_TX_STOP1_MASK},
 	.dma_stop2		= {R_AX_HAXI_DMA_STOP2, B_AX_TX_STOP2_ALL},
 	.dma_busy1		= {R_AX_HAXI_DMA_BUSY1, DMA_BUSY1_CHECK},
@@ -50,6 +53,7 @@ static const struct rtw89_pci_info rtw8852c_pci_info = {
 
 	.rpwm_addr		= R_AX_PCIE_HRPWM_V1,
 	.cpwm_addr		= R_AX_PCIE_CRPWM,
+	.mit_addr		= R_AX_INT_MIT_RX_V1,
 	.tx_dma_ch_mask		= 0,
 	.bd_idx_addr_low_power	= &rtw8852c_bd_idx_addr_low_power,
 	.dma_addr_set		= &rtw89_pci_ch_dma_addr_set_v1,
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8922ae.c b/drivers/net/wireless/realtek/rtw89/rtw8922ae.c
new file mode 100644
index 000000000000..414635d301aa
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtw89/rtw8922ae.c
@@ -0,0 +1,86 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* Copyright(c) 2023  Realtek Corporation
+ */
+
+#include <linux/module.h>
+#include <linux/pci.h>
+
+#include "pci.h"
+#include "reg.h"
+
+static const struct rtw89_pci_info rtw8922a_pci_info = {
+	.gen_def		= &rtw89_pci_gen_be,
+	.txbd_trunc_mode	= MAC_AX_BD_TRUNC,
+	.rxbd_trunc_mode	= MAC_AX_BD_TRUNC,
+	.rxbd_mode		= MAC_AX_RXBD_PKT,
+	.tag_mode		= MAC_AX_TAG_MULTI,
+	.tx_burst		= MAC_AX_TX_BURST_V1_256B,
+	.rx_burst		= MAC_AX_RX_BURST_V1_128B,
+	.wd_dma_idle_intvl	= MAC_AX_WD_DMA_INTVL_256NS,
+	.wd_dma_act_intvl	= MAC_AX_WD_DMA_INTVL_256NS,
+	.multi_tag_num		= MAC_AX_TAG_NUM_8,
+	.lbc_en			= MAC_AX_PCIE_ENABLE,
+	.lbc_tmr		= MAC_AX_LBC_TMR_2MS,
+	.autok_en		= MAC_AX_PCIE_DISABLE,
+	.io_rcy_en		= MAC_AX_PCIE_ENABLE,
+	.io_rcy_tmr		= MAC_AX_IO_RCY_ANA_TMR_DEF,
+	.rx_ring_eq_is_full	= true,
+
+	.init_cfg_reg		= R_BE_HAXI_INIT_CFG1,
+	.txhci_en_bit		= B_BE_TXDMA_EN,
+	.rxhci_en_bit		= B_BE_RXDMA_EN,
+	.rxbd_mode_bit		= B_BE_RXQ_RXBD_MODE_MASK,
+	.exp_ctrl_reg		= R_BE_HAXI_EXP_CTRL_V1,
+	.max_tag_num_mask	= B_BE_MAX_TAG_NUM_MASK,
+	.rxbd_rwptr_clr_reg	= R_BE_RXBD_RWPTR_CLR1_V1,
+	.txbd_rwptr_clr2_reg	= R_BE_TXBD_RWPTR_CLR1,
+	.dma_io_stop		= {R_BE_HAXI_INIT_CFG1, B_BE_STOP_AXI_MST},
+	.dma_stop1		= {R_BE_HAXI_DMA_STOP1, B_BE_TX_STOP1_MASK},
+	.dma_stop2		= {0},
+	.dma_busy1		= {R_BE_HAXI_DMA_BUSY1, DMA_BUSY1_CHECK_BE},
+	.dma_busy2_reg		= 0,
+	.dma_busy3_reg		= R_BE_HAXI_DMA_BUSY1,
+
+	.rpwm_addr		= R_BE_PCIE_HRPWM,
+	.cpwm_addr		= R_BE_PCIE_CRPWM,
+	.mit_addr		= R_BE_PCIE_MIT_CH_EN,
+	.tx_dma_ch_mask		= 0,
+	.bd_idx_addr_low_power	= NULL,
+	.dma_addr_set		= &rtw89_pci_ch_dma_addr_set_be,
+	.bd_ram_table		= NULL,
+
+	.ltr_set		= rtw89_pci_ltr_set_v2,
+	.fill_txaddr_info	= rtw89_pci_fill_txaddr_info_v1,
+	.config_intr_mask	= rtw89_pci_config_intr_mask_v2,
+	.enable_intr		= rtw89_pci_enable_intr_v2,
+	.disable_intr		= rtw89_pci_disable_intr_v2,
+	.recognize_intrs	= rtw89_pci_recognize_intrs_v2,
+};
+
+static const struct rtw89_driver_info rtw89_8922ae_info = {
+	.bus = {
+		.pci = &rtw8922a_pci_info,
+	},
+};
+
+static const struct pci_device_id rtw89_8922ae_id_table[] = {
+	{
+		PCI_DEVICE(PCI_VENDOR_ID_REALTEK, 0x8922),
+		.driver_data = (kernel_ulong_t)&rtw89_8922ae_info,
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(pci, rtw89_8922ae_id_table);
+
+static struct pci_driver rtw89_8922ae_driver = {
+	.name		= "rtw89_8922ae",
+	.id_table	= rtw89_8922ae_id_table,
+	.probe		= rtw89_pci_probe,
+	.remove		= rtw89_pci_remove,
+	.driver.pm	= &rtw89_pm_ops,
+};
+module_pci_driver(rtw89_8922ae_driver);
+
+MODULE_AUTHOR("Realtek Corporation");
+MODULE_DESCRIPTION("Realtek 802.11be wireless 8922AE driver");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/wireless/realtek/rtw89/ser.c b/drivers/net/wireless/realtek/rtw89/ser.c
index c1644353053f..1e4a79a3b814 100644
--- a/drivers/net/wireless/realtek/rtw89/ser.c
+++ b/drivers/net/wireless/realtek/rtw89/ser.c
@@ -361,6 +361,9 @@ static int hal_enable_dma(struct rtw89_ser *ser)
 	ret = rtwdev->hci.ops->mac_lv1_rcvy(rtwdev, RTW89_LV1_RCVY_STEP_2);
 	if (!ret)
 		clear_bit(RTW89_SER_HAL_STOP_DMA, ser->flags);
+	else
+		rtw89_debug(rtwdev, RTW89_DBG_SER,
+			    "lv1 rcvy fail to start dma: %d\n", ret);
 
 	return ret;
 }
@@ -376,6 +379,9 @@ static int hal_stop_dma(struct rtw89_ser *ser)
 	ret = rtwdev->hci.ops->mac_lv1_rcvy(rtwdev, RTW89_LV1_RCVY_STEP_1);
 	if (!ret)
 		set_bit(RTW89_SER_HAL_STOP_DMA, ser->flags);
+	else
+		rtw89_debug(rtwdev, RTW89_DBG_SER,
+			    "lv1 rcvy fail to stop dma: %d\n", ret);
 
 	return ret;
 }
diff --git a/drivers/net/wireless/realtek/rtw89/txrx.h b/drivers/net/wireless/realtek/rtw89/txrx.h
index 7142cce167de..c467a80ffa88 100644
--- a/drivers/net/wireless/realtek/rtw89/txrx.h
+++ b/drivers/net/wireless/realtek/rtw89/txrx.h
@@ -416,8 +416,11 @@ struct rtw89_rxinfo {
 } __packed;
 
 #define RTW89_RXINFO_W0_USR_NUM GENMASK(3, 0)
+#define RTW89_RXINFO_W0_USR_NUM_V1 GENMASK(4, 0)
 #define RTW89_RXINFO_W0_FW_DEFINE GENMASK(15, 8)
+#define RTW89_RXINFO_W0_PLCP_LEN_V1 GENMASK(23, 16)
 #define RTW89_RXINFO_W0_LSIG_LEN GENMASK(27, 16)
+#define RTW89_RXINFO_W0_INVALID_V1 BIT(27)
 #define RTW89_RXINFO_W0_IS_TO_SELF BIT(28)
 #define RTW89_RXINFO_W0_RX_CNT_VLD BIT(29)
 #define RTW89_RXINFO_W0_LONG_RXD GENMASK(31, 30)
@@ -430,6 +433,7 @@ struct rtw89_phy_sts_hdr {
 } __packed;
 
 #define RTW89_PHY_STS_HDR_W0_IE_MAP GENMASK(4, 0)
+#define RTW89_PHY_STS_HDR_W0_VALID BIT(7)
 #define RTW89_PHY_STS_HDR_W0_LEN GENMASK(15, 8)
 #define RTW89_PHY_STS_HDR_W0_RSSI_AVG GENMASK(31, 24)
 #define RTW89_PHY_STS_HDR_W1_RSSI_A GENMASK(7, 0)
diff --git a/drivers/net/wireless/zydas/Kconfig b/drivers/net/wireless/zydas/Kconfig
index 08574433df66..839e1217e855 100644
--- a/drivers/net/wireless/zydas/Kconfig
+++ b/drivers/net/wireless/zydas/Kconfig
@@ -12,25 +12,6 @@ config WLAN_VENDOR_ZYDAS
 
 if WLAN_VENDOR_ZYDAS
 
-config USB_ZD1201
-	tristate "USB ZD1201 based Wireless device support"
-	depends on CFG80211 && USB
-	select WIRELESS_EXT
-	select WEXT_PRIV
-	select FW_LOADER
-	help
-	  Say Y if you want to use wireless LAN adapters based on the ZyDAS
-	  ZD1201 chip.
-
-	  This driver makes the adapter appear as a normal Ethernet interface,
-	  typically on wlan0.
-
-	  The zd1201 device requires external firmware to be loaded.
-	  This can be found at http://linux-lc100020.sourceforge.net/
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called zd1201.
-
 source "drivers/net/wireless/zydas/zd1211rw/Kconfig"
 
 endif # WLAN_VENDOR_ZYDAS
diff --git a/drivers/net/wireless/zydas/Makefile b/drivers/net/wireless/zydas/Makefile
index c70003d30a8f..3e0a51db9874 100644
--- a/drivers/net/wireless/zydas/Makefile
+++ b/drivers/net/wireless/zydas/Makefile
@@ -1,4 +1,2 @@
 # SPDX-License-Identifier: GPL-2.0-only
 obj-$(CONFIG_ZD1211RW)		+= zd1211rw/
-
-obj-$(CONFIG_USB_ZD1201)	+= zd1201.o
diff --git a/drivers/net/wireless/zydas/zd1201.c b/drivers/net/wireless/zydas/zd1201.c
deleted file mode 100644
index 2814df1ecc78..000000000000
--- a/drivers/net/wireless/zydas/zd1201.c
+++ /dev/null
@@ -1,1909 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- *	Driver for ZyDAS zd1201 based USB wireless devices.
- *
- *	Copyright (c) 2004, 2005 Jeroen Vreeken (pe1rxq@amsat.org)
- *
- *	Parts of this driver have been derived from a wlan-ng version
- *	modified by ZyDAS. They also made documentation available, thanks!
- *	Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
- */
-
-#include <linux/module.h>
-#include <linux/usb.h>
-#include <linux/slab.h>
-#include <linux/netdevice.h>
-#include <linux/etherdevice.h>
-#include <linux/wireless.h>
-#include <net/cfg80211.h>
-#include <net/iw_handler.h>
-#include <linux/string.h>
-#include <linux/if_arp.h>
-#include <linux/firmware.h>
-#include "zd1201.h"
-
-static const struct usb_device_id zd1201_table[] = {
-	{USB_DEVICE(0x0586, 0x3400)}, /* Peabird USB Wireless Adapter */
-	{USB_DEVICE(0x0ace, 0x1201)}, /* ZyDAS ZD1201 USB Wireless Adapter */
-	{USB_DEVICE(0x050d, 0x6051)}, /* Belkin F5D6051 usb  adapter */
-	{USB_DEVICE(0x0db0, 0x6823)}, /* MSI UB11B usb  adapter */
-	{USB_DEVICE(0x1044, 0x8004)}, /* Gigabyte GN-WLBZ101 */
-	{USB_DEVICE(0x1044, 0x8005)}, /* GIGABYTE GN-WLBZ201 usb adapter */
-	{}
-};
-
-static int ap;	/* Are we an AP or a normal station? */
-
-#define ZD1201_VERSION	"0.15"
-
-MODULE_AUTHOR("Jeroen Vreeken <pe1rxq@amsat.org>");
-MODULE_DESCRIPTION("Driver for ZyDAS ZD1201 based USB Wireless adapters");
-MODULE_VERSION(ZD1201_VERSION);
-MODULE_LICENSE("GPL");
-module_param(ap, int, 0);
-MODULE_PARM_DESC(ap, "If non-zero Access Point firmware will be loaded");
-MODULE_DEVICE_TABLE(usb, zd1201_table);
-
-
-static int zd1201_fw_upload(struct usb_device *dev, int apfw)
-{
-	const struct firmware *fw_entry;
-	const char *data;
-	unsigned long len;
-	int err;
-	unsigned char ret;
-	char *buf;
-	char *fwfile;
-
-	if (apfw)
-		fwfile = "zd1201-ap.fw";
-	else
-		fwfile = "zd1201.fw";
-
-	err = request_firmware(&fw_entry, fwfile, &dev->dev);
-	if (err) {
-		dev_err(&dev->dev, "Failed to load %s firmware file!\n", fwfile);
-		dev_err(&dev->dev, "Make sure the hotplug firmware loader is installed.\n");
-		dev_err(&dev->dev, "Goto http://linux-lc100020.sourceforge.net for more info.\n");
-		return err;
-	}
-
-	data = fw_entry->data;
-        len = fw_entry->size;
-
-	buf = kmalloc(1024, GFP_ATOMIC);
-	if (!buf) {
-		err = -ENOMEM;
-		goto exit;
-	}
-	
-	while (len > 0) {
-		int translen = (len > 1024) ? 1024 : len;
-		memcpy(buf, data, translen);
-
-		err = usb_control_msg(dev, usb_sndctrlpipe(dev, 0), 0,
-		    USB_DIR_OUT | 0x40, 0, 0, buf, translen,
-		    ZD1201_FW_TIMEOUT);
-		if (err < 0)
-			goto exit;
-
-		len -= translen;
-		data += translen;
-	}
-                                        
-	err = usb_control_msg(dev, usb_sndctrlpipe(dev, 0), 0x2,
-	    USB_DIR_OUT | 0x40, 0, 0, NULL, 0, ZD1201_FW_TIMEOUT);
-	if (err < 0)
-		goto exit;
-
-	err = usb_control_msg(dev, usb_rcvctrlpipe(dev, 0), 0x4,
-	    USB_DIR_IN | 0x40, 0, 0, buf, sizeof(ret), ZD1201_FW_TIMEOUT);
-	if (err < 0)
-		goto exit;
-
-	memcpy(&ret, buf, sizeof(ret));
-
-	if (ret & 0x80) {
-		err = -EIO;
-		goto exit;
-	}
-
-	err = 0;
-exit:
-	kfree(buf);
-	release_firmware(fw_entry);
-	return err;
-}
-
-MODULE_FIRMWARE("zd1201-ap.fw");
-MODULE_FIRMWARE("zd1201.fw");
-
-static void zd1201_usbfree(struct urb *urb)
-{
-	struct zd1201 *zd = urb->context;
-
-	switch(urb->status) {
-		case -EILSEQ:
-		case -ENODEV:
-		case -ETIME:
-		case -ENOENT:
-		case -EPIPE:
-		case -EOVERFLOW:
-		case -ESHUTDOWN:
-			dev_warn(&zd->usb->dev, "%s: urb failed: %d\n", 
-			    zd->dev->name, urb->status);
-	}
-
-	kfree(urb->transfer_buffer);
-	usb_free_urb(urb);
-}
-
-/* cmdreq message: 
-	u32 type
-	u16 cmd
-	u16 parm0
-	u16 parm1
-	u16 parm2
-	u8  pad[4]
-
-	total: 4 + 2 + 2 + 2 + 2 + 4 = 16
-*/
-static int zd1201_docmd(struct zd1201 *zd, int cmd, int parm0,
-			int parm1, int parm2)
-{
-	unsigned char *command;
-	int ret;
-	struct urb *urb;
-
-	command = kmalloc(16, GFP_ATOMIC);
-	if (!command)
-		return -ENOMEM;
-
-	*((__le32*)command) = cpu_to_le32(ZD1201_USB_CMDREQ);
-	*((__le16*)&command[4]) = cpu_to_le16(cmd);
-	*((__le16*)&command[6]) = cpu_to_le16(parm0);
-	*((__le16*)&command[8]) = cpu_to_le16(parm1);
-	*((__le16*)&command[10])= cpu_to_le16(parm2);
-
-	urb = usb_alloc_urb(0, GFP_ATOMIC);
-	if (!urb) {
-		kfree(command);
-		return -ENOMEM;
-	}
-	usb_fill_bulk_urb(urb, zd->usb, usb_sndbulkpipe(zd->usb, zd->endp_out2),
-			  command, 16, zd1201_usbfree, zd);
-	ret = usb_submit_urb(urb, GFP_ATOMIC);
-	if (ret) {
-		kfree(command);
-		usb_free_urb(urb);
-	}
-
-	return ret;
-}
-
-/* Callback after sending out a packet */
-static void zd1201_usbtx(struct urb *urb)
-{
-	struct zd1201 *zd = urb->context;
-	netif_wake_queue(zd->dev);
-}
-
-/* Incoming data */
-static void zd1201_usbrx(struct urb *urb)
-{
-	struct zd1201 *zd = urb->context;
-	int free = 0;
-	unsigned char *data = urb->transfer_buffer;
-	struct sk_buff *skb;
-	unsigned char type;
-
-	if (!zd)
-		return;
-
-	switch(urb->status) {
-		case -EILSEQ:
-		case -ENODEV:
-		case -ETIME:
-		case -ENOENT:
-		case -EPIPE:
-		case -EOVERFLOW:
-		case -ESHUTDOWN:
-			dev_warn(&zd->usb->dev, "%s: rx urb failed: %d\n",
-			    zd->dev->name, urb->status);
-			free = 1;
-			goto exit;
-	}
-	
-	if (urb->status != 0 || urb->actual_length == 0)
-		goto resubmit;
-
-	type = data[0];
-	if (type == ZD1201_PACKET_EVENTSTAT || type == ZD1201_PACKET_RESOURCE) {
-		memcpy(zd->rxdata, data, urb->actual_length);
-		zd->rxlen = urb->actual_length;
-		zd->rxdatas = 1;
-		wake_up(&zd->rxdataq);
-	}
-	/* Info frame */
-	if (type == ZD1201_PACKET_INQUIRE) {
-		int i = 0;
-		unsigned short infotype, copylen;
-		infotype = le16_to_cpu(*(__le16*)&data[6]);
-
-		if (infotype == ZD1201_INF_LINKSTATUS) {
-			short linkstatus;
-
-			linkstatus = le16_to_cpu(*(__le16*)&data[8]);
-			switch(linkstatus) {
-				case 1:
-					netif_carrier_on(zd->dev);
-					break;
-				case 2:
-					netif_carrier_off(zd->dev);
-					break;
-				case 3:
-					netif_carrier_off(zd->dev);
-					break;
-				case 4:
-					netif_carrier_on(zd->dev);
-					break;
-				default:
-					netif_carrier_off(zd->dev);
-			}
-			goto resubmit;
-		}
-		if (infotype == ZD1201_INF_ASSOCSTATUS) {
-			short status = le16_to_cpu(*(__le16*)(data+8));
-			int event;
-			union iwreq_data wrqu;
-
-			switch (status) {
-				case ZD1201_ASSOCSTATUS_STAASSOC:
-				case ZD1201_ASSOCSTATUS_REASSOC:
-					event = IWEVREGISTERED;
-					break;
-				case ZD1201_ASSOCSTATUS_DISASSOC:
-				case ZD1201_ASSOCSTATUS_ASSOCFAIL:
-				case ZD1201_ASSOCSTATUS_AUTHFAIL:
-				default:
-					event = IWEVEXPIRED;
-			}
-			memcpy(wrqu.addr.sa_data, data+10, ETH_ALEN);
-			wrqu.addr.sa_family = ARPHRD_ETHER;
-
-			/* Send event to user space */
-			wireless_send_event(zd->dev, event, &wrqu, NULL);
-
-			goto resubmit;
-		}
-		if (infotype == ZD1201_INF_AUTHREQ) {
-			union iwreq_data wrqu;
-
-			memcpy(wrqu.addr.sa_data, data+8, ETH_ALEN);
-			wrqu.addr.sa_family = ARPHRD_ETHER;
-			/* There isn't a event that trully fits this request.
-			   We assume that userspace will be smart enough to
-			   see a new station being expired and sends back a
-			   authstation ioctl to authorize it. */
-			wireless_send_event(zd->dev, IWEVEXPIRED, &wrqu, NULL);
-			goto resubmit;
-		}
-		/* Other infotypes are handled outside this handler */
-		zd->rxlen = 0;
-		while (i < urb->actual_length) {
-			copylen = le16_to_cpu(*(__le16*)&data[i+2]);
-			/* Sanity check, sometimes we get junk */
-			if (copylen+zd->rxlen > sizeof(zd->rxdata))
-				break;
-			memcpy(zd->rxdata+zd->rxlen, data+i+4, copylen);
-			zd->rxlen += copylen;
-			i += 64;
-		}
-		if (i >= urb->actual_length) {
-			zd->rxdatas = 1;
-			wake_up(&zd->rxdataq);
-		}
-		goto  resubmit;
-	}
-	/* Actual data */
-	if (data[urb->actual_length-1] == ZD1201_PACKET_RXDATA) {
-		int datalen = urb->actual_length-1;
-		unsigned short len, fc, seq;
-
-		len = ntohs(*(__be16 *)&data[datalen-2]);
-		if (len>datalen)
-			len=datalen;
-		fc = le16_to_cpu(*(__le16 *)&data[datalen-16]);
-		seq = le16_to_cpu(*(__le16 *)&data[datalen-24]);
-
-		if (zd->monitor) {
-			if (datalen < 24)
-				goto resubmit;
-			if (!(skb = dev_alloc_skb(datalen+24)))
-				goto resubmit;
-			
-			skb_put_data(skb, &data[datalen - 16], 2);
-			skb_put_data(skb, &data[datalen - 2], 2);
-			skb_put_data(skb, &data[datalen - 14], 6);
-			skb_put_data(skb, &data[datalen - 22], 6);
-			skb_put_data(skb, &data[datalen - 8], 6);
-			skb_put_data(skb, &data[datalen - 24], 2);
-			skb_put_data(skb, data, len);
-			skb->protocol = eth_type_trans(skb, zd->dev);
-			zd->dev->stats.rx_packets++;
-			zd->dev->stats.rx_bytes += skb->len;
-			netif_rx(skb);
-			goto resubmit;
-		}
-			
-		if ((seq & IEEE80211_SCTL_FRAG) ||
-		    (fc & IEEE80211_FCTL_MOREFRAGS)) {
-			struct zd1201_frag *frag = NULL;
-			char *ptr;
-
-			if (datalen<14)
-				goto resubmit;
-			if ((seq & IEEE80211_SCTL_FRAG) == 0) {
-				frag = kmalloc(sizeof(*frag), GFP_ATOMIC);
-				if (!frag)
-					goto resubmit;
-				skb = dev_alloc_skb(IEEE80211_MAX_DATA_LEN +14+2);
-				if (!skb) {
-					kfree(frag);
-					goto resubmit;
-				}
-				frag->skb = skb;
-				frag->seq = seq & IEEE80211_SCTL_SEQ;
-				skb_reserve(skb, 2);
-				skb_put_data(skb, &data[datalen - 14], 12);
-				skb_put_data(skb, &data[6], 2);
-				skb_put_data(skb, data + 8, len);
-				hlist_add_head(&frag->fnode, &zd->fraglist);
-				goto resubmit;
-			}
-			hlist_for_each_entry(frag, &zd->fraglist, fnode)
-				if (frag->seq == (seq&IEEE80211_SCTL_SEQ))
-					break;
-			if (!frag)
-				goto resubmit;
-			skb = frag->skb;
-			ptr = skb_put(skb, len);
-			if (ptr)
-				memcpy(ptr, data+8, len);
-			if (fc & IEEE80211_FCTL_MOREFRAGS)
-				goto resubmit;
-			hlist_del_init(&frag->fnode);
-			kfree(frag);
-		} else {
-			if (datalen<14)
-				goto resubmit;
-			skb = dev_alloc_skb(len + 14 + 2);
-			if (!skb)
-				goto resubmit;
-			skb_reserve(skb, 2);
-			skb_put_data(skb, &data[datalen - 14], 12);
-			skb_put_data(skb, &data[6], 2);
-			skb_put_data(skb, data + 8, len);
-		}
-		skb->protocol = eth_type_trans(skb, zd->dev);
-		zd->dev->stats.rx_packets++;
-		zd->dev->stats.rx_bytes += skb->len;
-		netif_rx(skb);
-	}
-resubmit:
-	memset(data, 0, ZD1201_RXSIZE);
-
-	urb->status = 0;
-	urb->dev = zd->usb;
-	if(usb_submit_urb(urb, GFP_ATOMIC))
-		free = 1;
-
-exit:
-	if (free) {
-		zd->rxlen = 0;
-		zd->rxdatas = 1;
-		wake_up(&zd->rxdataq);
-		kfree(urb->transfer_buffer);
-	}
-}
-
-static int zd1201_getconfig(struct zd1201 *zd, int rid, void *riddata,
-	unsigned int riddatalen)
-{
-	int err;
-	int i = 0;
-	int code;
-	int rid_fid;
-	int length;
-	unsigned char *pdata;
-
-	zd->rxdatas = 0;
-	err = zd1201_docmd(zd, ZD1201_CMDCODE_ACCESS, rid, 0, 0);
-	if (err)
-		return err;
-
-	wait_event_interruptible(zd->rxdataq, zd->rxdatas);
-	if (!zd->rxlen)
-		return -EIO;
-
-	code = le16_to_cpu(*(__le16*)(&zd->rxdata[4]));
-	rid_fid = le16_to_cpu(*(__le16*)(&zd->rxdata[6]));
-	length = le16_to_cpu(*(__le16*)(&zd->rxdata[8]));
-	if (length > zd->rxlen)
-		length = zd->rxlen-6;
-
-	/* If access bit is not on, then error */
-	if ((code & ZD1201_ACCESSBIT) != ZD1201_ACCESSBIT || rid_fid != rid )
-		return -EINVAL;
-
-	/* Not enough buffer for allocating data */
-	if (riddatalen != (length - 4)) {
-		dev_dbg(&zd->usb->dev, "riddatalen mismatches, expected=%u, (packet=%u) length=%u, rid=0x%04X, rid_fid=0x%04X\n",
-		    riddatalen, zd->rxlen, length, rid, rid_fid);
-		return -ENODATA;
-	}
-
-	zd->rxdatas = 0;
-	/* Issue SetRxRid commnd */			
-	err = zd1201_docmd(zd, ZD1201_CMDCODE_SETRXRID, rid, 0, length);
-	if (err)
-		return err;
-
-	/* Receive RID record from resource packets */
-	wait_event_interruptible(zd->rxdataq, zd->rxdatas);
-	if (!zd->rxlen)
-		return -EIO;
-
-	if (zd->rxdata[zd->rxlen - 1] != ZD1201_PACKET_RESOURCE) {
-		dev_dbg(&zd->usb->dev, "Packet type mismatch: 0x%x not 0x3\n",
-		    zd->rxdata[zd->rxlen-1]);
-		return -EINVAL;
-	}
-
-	/* Set the data pointer and received data length */
-	pdata = zd->rxdata;
-	length = zd->rxlen;
-
-	do {
-		int actual_length;
-
-		actual_length = (length > 64) ? 64 : length;
-
-		if (pdata[0] != 0x3) {
-			dev_dbg(&zd->usb->dev, "Rx Resource packet type error: %02X\n",
-			    pdata[0]);
-			return -EINVAL;
-		}
-
-		if (actual_length != 64) {
-			/* Trim the last packet type byte */
-			actual_length--;
-		}
-
-		/* Skip the 4 bytes header (RID length and RID) */
-		if (i == 0) {
-			pdata += 8;
-			actual_length -= 8;
-		} else {
-			pdata += 4;
-			actual_length -= 4;
-		}
-		
-		memcpy(riddata, pdata, actual_length);
-		riddata += actual_length;
-		pdata += actual_length;
-		length -= 64;
-		i++;
-	} while (length > 0);
-
-	return 0;
-}
-
-/*
- *	resreq:
- *		byte	type
- *		byte	sequence
- *		u16	reserved
- *		byte	data[12]
- *	total: 16
- */
-static int zd1201_setconfig(struct zd1201 *zd, int rid, const void *buf, int len, int wait)
-{
-	int err;
-	unsigned char *request;
-	int reqlen;
-	char seq=0;
-	struct urb *urb;
-	gfp_t gfp_mask = wait ? GFP_NOIO : GFP_ATOMIC;
-
-	len += 4;			/* first 4 are for header */
-
-	zd->rxdatas = 0;
-	zd->rxlen = 0;
-	for (seq=0; len > 0; seq++) {
-		request = kzalloc(16, gfp_mask);
-		if (!request)
-			return -ENOMEM;
-		urb = usb_alloc_urb(0, gfp_mask);
-		if (!urb) {
-			kfree(request);
-			return -ENOMEM;
-		}
-		reqlen = len>12 ? 12 : len;
-		request[0] = ZD1201_USB_RESREQ;
-		request[1] = seq;
-		request[2] = 0;
-		request[3] = 0;
-		if (request[1] == 0) {
-			/* add header */
-			*(__le16*)&request[4] = cpu_to_le16((len-2+1)/2);
-			*(__le16*)&request[6] = cpu_to_le16(rid);
-			memcpy(request+8, buf, reqlen-4);
-			buf += reqlen-4;
-		} else {
-			memcpy(request+4, buf, reqlen);
-			buf += reqlen;
-		}
-
-		len -= reqlen;
-
-		usb_fill_bulk_urb(urb, zd->usb, usb_sndbulkpipe(zd->usb,
-		    zd->endp_out2), request, 16, zd1201_usbfree, zd);
-		err = usb_submit_urb(urb, gfp_mask);
-		if (err)
-			goto err;
-	}
-
-	request = kmalloc(16, gfp_mask);
-	if (!request)
-		return -ENOMEM;
-	urb = usb_alloc_urb(0, gfp_mask);
-	if (!urb) {
-		kfree(request);
-		return -ENOMEM;
-	}
-	*((__le32*)request) = cpu_to_le32(ZD1201_USB_CMDREQ);
-	*((__le16*)&request[4]) = 
-	    cpu_to_le16(ZD1201_CMDCODE_ACCESS|ZD1201_ACCESSBIT);
-	*((__le16*)&request[6]) = cpu_to_le16(rid);
-	*((__le16*)&request[8]) = cpu_to_le16(0);
-	*((__le16*)&request[10]) = cpu_to_le16(0);
-	usb_fill_bulk_urb(urb, zd->usb, usb_sndbulkpipe(zd->usb, zd->endp_out2),
-	     request, 16, zd1201_usbfree, zd);
-	err = usb_submit_urb(urb, gfp_mask);
-	if (err)
-		goto err;
-	
-	if (wait) {
-		wait_event_interruptible(zd->rxdataq, zd->rxdatas);
-		if (!zd->rxlen || le16_to_cpu(*(__le16*)&zd->rxdata[6]) != rid) {
-			dev_dbg(&zd->usb->dev, "wrong or no RID received\n");
-		}
-	}
-
-	return 0;
-err:
-	kfree(request);
-	usb_free_urb(urb);
-	return err;
-}
-
-static inline int zd1201_getconfig16(struct zd1201 *zd, int rid, short *val)
-{
-	int err;
-	__le16 zdval;
-
-	err = zd1201_getconfig(zd, rid, &zdval, sizeof(__le16));
-	if (err)
-		return err;
-	*val = le16_to_cpu(zdval);
-	return 0;
-}
-
-static inline int zd1201_setconfig16(struct zd1201 *zd, int rid, short val)
-{
-	__le16 zdval = cpu_to_le16(val);
-	return (zd1201_setconfig(zd, rid, &zdval, sizeof(__le16), 1));
-}
-
-static int zd1201_drvr_start(struct zd1201 *zd)
-{
-	int err, i;
-	short max;
-	__le16 zdmax;
-	unsigned char *buffer;
-
-	buffer = kzalloc(ZD1201_RXSIZE, GFP_KERNEL);
-	if (!buffer)
-		return -ENOMEM;
-
-	usb_fill_bulk_urb(zd->rx_urb, zd->usb, 
-	    usb_rcvbulkpipe(zd->usb, zd->endp_in), buffer, ZD1201_RXSIZE,
-	    zd1201_usbrx, zd);
-
-	err = usb_submit_urb(zd->rx_urb, GFP_KERNEL);
-	if (err)
-		goto err_buffer;
-
-	err = zd1201_docmd(zd, ZD1201_CMDCODE_INIT, 0, 0, 0);
-	if (err)
-		goto err_urb;
-
-	err = zd1201_getconfig(zd, ZD1201_RID_CNFMAXTXBUFFERNUMBER, &zdmax,
-	    sizeof(__le16));
-	if (err)
-		goto err_urb;
-
-	max = le16_to_cpu(zdmax);
-	for (i=0; i<max; i++) {
-		err = zd1201_docmd(zd, ZD1201_CMDCODE_ALLOC, 1514, 0, 0);
-		if (err)
-			goto err_urb;
-	}
-
-	return 0;
-
-err_urb:
-	usb_kill_urb(zd->rx_urb);
-	return err;
-err_buffer:
-	kfree(buffer);
-	return err;
-}
-
-/*	Magic alert: The firmware doesn't seem to like the MAC state being
- *	toggled in promisc (aka monitor) mode.
- *	(It works a number of times, but will halt eventually)
- *	So we turn it of before disabling and on after enabling if needed.
- */
-static int zd1201_enable(struct zd1201 *zd)
-{
-	int err;
-
-	if (zd->mac_enabled)
-		return 0;
-
-	err = zd1201_docmd(zd, ZD1201_CMDCODE_ENABLE, 0, 0, 0);
-	if (!err)
-		zd->mac_enabled = 1;
-
-	if (zd->monitor)
-		err = zd1201_setconfig16(zd, ZD1201_RID_PROMISCUOUSMODE, 1);
-
-	return err;
-}
-
-static int zd1201_disable(struct zd1201 *zd)
-{
-	int err;
-
-	if (!zd->mac_enabled)
-		return 0;
-	if (zd->monitor) {
-		err = zd1201_setconfig16(zd, ZD1201_RID_PROMISCUOUSMODE, 0);
-		if (err)
-			return err;
-	}
-
-	err = zd1201_docmd(zd, ZD1201_CMDCODE_DISABLE, 0, 0, 0);
-	if (!err)
-		zd->mac_enabled = 0;
-	return err;
-}
-
-static int zd1201_mac_reset(struct zd1201 *zd)
-{
-	if (!zd->mac_enabled)
-		return 0;
-	zd1201_disable(zd);
-	return zd1201_enable(zd);
-}
-
-static int zd1201_join(struct zd1201 *zd, char *essid, int essidlen)
-{
-	int err, val;
-	char buf[IW_ESSID_MAX_SIZE+2];
-
-	err = zd1201_disable(zd);
-	if (err)
-		return err;
-
-	val = ZD1201_CNFAUTHENTICATION_OPENSYSTEM;
-	val |= ZD1201_CNFAUTHENTICATION_SHAREDKEY;
-	err = zd1201_setconfig16(zd, ZD1201_RID_CNFAUTHENTICATION, val);
-	if (err)
-		return err;
-
-	*(__le16 *)buf = cpu_to_le16(essidlen);
-	memcpy(buf+2, essid, essidlen);
-	if (!zd->ap) {	/* Normal station */
-		err = zd1201_setconfig(zd, ZD1201_RID_CNFDESIREDSSID, buf,
-		    IW_ESSID_MAX_SIZE+2, 1);
-		if (err)
-			return err;
-	} else {	/* AP */
-		err = zd1201_setconfig(zd, ZD1201_RID_CNFOWNSSID, buf,
-		    IW_ESSID_MAX_SIZE+2, 1);
-		if (err)
-			return err;
-	}
-
-	err = zd1201_setconfig(zd, ZD1201_RID_CNFOWNMACADDR, 
-	    zd->dev->dev_addr, zd->dev->addr_len, 1);
-	if (err)
-		return err;
-
-	err = zd1201_enable(zd);
-	if (err)
-		return err;
-
-	msleep(100);
-	return 0;
-}
-
-static int zd1201_net_open(struct net_device *dev)
-{
-	struct zd1201 *zd = netdev_priv(dev);
-
-	/* Start MAC with wildcard if no essid set */
-	if (!zd->mac_enabled)
-		zd1201_join(zd, zd->essid, zd->essidlen);
-	netif_start_queue(dev);
-
-	return 0;
-}
-
-static int zd1201_net_stop(struct net_device *dev)
-{
-	netif_stop_queue(dev);
-	return 0;
-}
-
-/*
-	RFC 1042 encapsulates Ethernet frames in 802.11 frames
-	by prefixing them with 0xaa, 0xaa, 0x03) followed by a SNAP OID of 0
-	(0x00, 0x00, 0x00). Zd requires an additional padding, copy
-	of ethernet addresses, length of the standard RFC 1042 packet
-	and a command byte (which is nul for tx).
-	
-	tx frame (from Wlan NG):
-	RFC 1042:
-		llc		0xAA 0xAA 0x03 (802.2 LLC)
-		snap		0x00 0x00 0x00 (Ethernet encapsulated)
-		type		2 bytes, Ethernet type field
-		payload		(minus eth header)
-	Zydas specific:
-		padding		1B if (skb->len+8+1)%64==0
-		Eth MAC addr	12 bytes, Ethernet MAC addresses
-		length		2 bytes, RFC 1042 packet length 
-				(llc+snap+type+payload)
-		zd		1 null byte, zd1201 packet type
- */
-static netdev_tx_t zd1201_hard_start_xmit(struct sk_buff *skb,
-						struct net_device *dev)
-{
-	struct zd1201 *zd = netdev_priv(dev);
-	unsigned char *txbuf = zd->txdata;
-	int txbuflen, pad = 0, err;
-	struct urb *urb = zd->tx_urb;
-
-	if (!zd->mac_enabled || zd->monitor) {
-		dev->stats.tx_dropped++;
-		kfree_skb(skb);
-		return NETDEV_TX_OK;
-	}
-	netif_stop_queue(dev);
-
-	txbuflen = skb->len + 8 + 1;
-	if (txbuflen%64 == 0) {
-		pad = 1;
-		txbuflen++;
-	}
-	txbuf[0] = 0xAA;
-	txbuf[1] = 0xAA;
-	txbuf[2] = 0x03;
-	txbuf[3] = 0x00;	/* rfc1042 */
-	txbuf[4] = 0x00;
-	txbuf[5] = 0x00;
-
-	skb_copy_from_linear_data_offset(skb, 12, txbuf + 6, skb->len - 12);
-	if (pad)
-		txbuf[skb->len-12+6]=0;
-	skb_copy_from_linear_data(skb, txbuf + skb->len - 12 + 6 + pad, 12);
-	*(__be16*)&txbuf[skb->len+6+pad] = htons(skb->len-12+6);
-	txbuf[txbuflen-1] = 0;
-
-	usb_fill_bulk_urb(urb, zd->usb, usb_sndbulkpipe(zd->usb, zd->endp_out),
-	    txbuf, txbuflen, zd1201_usbtx, zd);
-
-	err = usb_submit_urb(zd->tx_urb, GFP_ATOMIC);
-	if (err) {
-		dev->stats.tx_errors++;
-		netif_start_queue(dev);
-	} else {
-		dev->stats.tx_packets++;
-		dev->stats.tx_bytes += skb->len;
-	}
-	kfree_skb(skb);
-
-	return NETDEV_TX_OK;
-}
-
-static void zd1201_tx_timeout(struct net_device *dev, unsigned int txqueue)
-{
-	struct zd1201 *zd = netdev_priv(dev);
-
-	if (!zd)
-		return;
-	dev_warn(&zd->usb->dev, "%s: TX timeout, shooting down urb\n",
-	    dev->name);
-	usb_unlink_urb(zd->tx_urb);
-	dev->stats.tx_errors++;
-	/* Restart the timeout to quiet the watchdog: */
-	netif_trans_update(dev); /* prevent tx timeout */
-}
-
-static int zd1201_set_mac_address(struct net_device *dev, void *p)
-{
-	struct sockaddr *addr = p;
-	struct zd1201 *zd = netdev_priv(dev);
-	int err;
-
-	if (!zd)
-		return -ENODEV;
-
-	err = zd1201_setconfig(zd, ZD1201_RID_CNFOWNMACADDR, 
-	    addr->sa_data, dev->addr_len, 1);
-	if (err)
-		return err;
-	eth_hw_addr_set(dev, addr->sa_data);
-
-	return zd1201_mac_reset(zd);
-}
-
-static struct iw_statistics *zd1201_get_wireless_stats(struct net_device *dev)
-{
-	struct zd1201 *zd = netdev_priv(dev);
-
-	return &zd->iwstats;
-}
-
-static void zd1201_set_multicast(struct net_device *dev)
-{
-	struct zd1201 *zd = netdev_priv(dev);
-	struct netdev_hw_addr *ha;
-	unsigned char reqbuf[ETH_ALEN*ZD1201_MAXMULTI];
-	int i;
-
-	if (netdev_mc_count(dev) > ZD1201_MAXMULTI)
-		return;
-
-	i = 0;
-	netdev_for_each_mc_addr(ha, dev)
-		memcpy(reqbuf + i++ * ETH_ALEN, ha->addr, ETH_ALEN);
-	zd1201_setconfig(zd, ZD1201_RID_CNFGROUPADDRESS, reqbuf,
-			 netdev_mc_count(dev) * ETH_ALEN, 0);
-}
-
-static int zd1201_config_commit(struct net_device *dev, 
-	struct iw_request_info *info, union iwreq_data *wrqu, char *essid)
-{
-	struct zd1201 *zd = netdev_priv(dev);
-
-	return zd1201_mac_reset(zd);
-}
-
-static int zd1201_get_name(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
-{
-	strcpy(wrqu->name, "IEEE 802.11b");
-	return 0;
-}
-
-static int zd1201_set_freq(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
-{
-	struct iw_freq *freq = &wrqu->freq;
-	struct zd1201 *zd = netdev_priv(dev);
-	short channel = 0;
-	int err;
-
-	if (freq->e == 0)
-		channel = freq->m;
-	else
-		channel = ieee80211_frequency_to_channel(freq->m);
-
-	err = zd1201_setconfig16(zd, ZD1201_RID_CNFOWNCHANNEL, channel);
-	if (err)
-		return err;
-
-	zd1201_mac_reset(zd);
-
-	return 0;
-}
-
-static int zd1201_get_freq(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
-{
-	struct iw_freq *freq = &wrqu->freq;
-	struct zd1201 *zd = netdev_priv(dev);
-	short channel;
-	int err;
-
-	err = zd1201_getconfig16(zd, ZD1201_RID_CNFOWNCHANNEL, &channel);
-	if (err)
-		return err;
-	freq->e = 0;
-	freq->m = channel;
-
-	return 0;
-}
-
-static int zd1201_set_mode(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
-{
-	__u32 *mode = &wrqu->mode;
-	struct zd1201 *zd = netdev_priv(dev);
-	short porttype, monitor = 0;
-	unsigned char buffer[IW_ESSID_MAX_SIZE+2];
-	int err;
-
-	if (zd->ap) {
-		if (*mode != IW_MODE_MASTER)
-			return -EINVAL;
-		return 0;
-	}
-
-	err = zd1201_setconfig16(zd, ZD1201_RID_PROMISCUOUSMODE, 0);
-	if (err)
-		return err;
-	zd->dev->type = ARPHRD_ETHER;
-	switch(*mode) {
-		case IW_MODE_MONITOR:
-			monitor = 1;
-			zd->dev->type = ARPHRD_IEEE80211;
-			/* Make sure we are no longer associated with by
-			   setting an 'impossible' essid.
-			   (otherwise we mess up firmware)
-			 */
-			zd1201_join(zd, "\0-*#\0", 5);
-			/* Put port in pIBSS */
-			fallthrough;
-		case 8: /* No pseudo-IBSS in wireless extensions (yet) */
-			porttype = ZD1201_PORTTYPE_PSEUDOIBSS;
-			break;
-		case IW_MODE_ADHOC:
-			porttype = ZD1201_PORTTYPE_IBSS;
-			break;
-		case IW_MODE_INFRA:
-			porttype = ZD1201_PORTTYPE_BSS;
-			break;
-		default:
-			return -EINVAL;
-	}
-
-	err = zd1201_setconfig16(zd, ZD1201_RID_CNFPORTTYPE, porttype);
-	if (err)
-		return err;
-	if (zd->monitor && !monitor) {
-			zd1201_disable(zd);
-			*(__le16 *)buffer = cpu_to_le16(zd->essidlen);
-			memcpy(buffer+2, zd->essid, zd->essidlen);
-			err = zd1201_setconfig(zd, ZD1201_RID_CNFDESIREDSSID,
-			    buffer, IW_ESSID_MAX_SIZE+2, 1);
-			if (err)
-				return err;
-	}
-	zd->monitor = monitor;
-	/* If monitor mode is set we don't actually turn it on here since it
-	 * is done during mac reset anyway (see zd1201_mac_enable).
-	 */
-	zd1201_mac_reset(zd);
-
-	return 0;
-}
-
-static int zd1201_get_mode(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
-{
-	__u32 *mode = &wrqu->mode;
-	struct zd1201 *zd = netdev_priv(dev);
-	short porttype;
-	int err;
-
-	err = zd1201_getconfig16(zd, ZD1201_RID_CNFPORTTYPE, &porttype);
-	if (err)
-		return err;
-	switch(porttype) {
-		case ZD1201_PORTTYPE_IBSS:
-			*mode = IW_MODE_ADHOC;
-			break;
-		case ZD1201_PORTTYPE_BSS:
-			*mode = IW_MODE_INFRA;
-			break;
-		case ZD1201_PORTTYPE_WDS:
-			*mode = IW_MODE_REPEAT;
-			break;
-		case ZD1201_PORTTYPE_PSEUDOIBSS:
-			*mode = 8;/* No Pseudo-IBSS... */
-			break;
-		case ZD1201_PORTTYPE_AP:
-			*mode = IW_MODE_MASTER;
-			break;
-		default:
-			dev_dbg(&zd->usb->dev, "Unknown porttype: %d\n",
-			    porttype);
-			*mode = IW_MODE_AUTO;
-	}
-	if (zd->monitor)
-		*mode = IW_MODE_MONITOR;
-
-	return 0;
-}
-
-static int zd1201_get_range(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
-{
-	struct iw_point *wrq = &wrqu->data;
-	struct iw_range *range = (struct iw_range *)extra;
-
-	wrq->length = sizeof(struct iw_range);
-	memset(range, 0, sizeof(struct iw_range));
-	range->we_version_compiled = WIRELESS_EXT;
-	range->we_version_source = WIRELESS_EXT;
-
-	range->max_qual.qual = 128;
-	range->max_qual.level = 128;
-	range->max_qual.noise = 128;
-	range->max_qual.updated = 7;
-
-	range->encoding_size[0] = 5;
-	range->encoding_size[1] = 13;
-	range->num_encoding_sizes = 2;
-	range->max_encoding_tokens = ZD1201_NUMKEYS;
-
-	range->num_bitrates = 4;
-	range->bitrate[0] = 1000000;
-	range->bitrate[1] = 2000000;
-	range->bitrate[2] = 5500000;
-	range->bitrate[3] = 11000000;
-
-	range->min_rts = 0;
-	range->min_frag = ZD1201_FRAGMIN;
-	range->max_rts = ZD1201_RTSMAX;
-	range->min_frag = ZD1201_FRAGMAX;
-
-	return 0;
-}
-
-/*	Little bit of magic here: we only get the quality if we poll
- *	for it, and we never get an actual request to trigger such
- *	a poll. Therefore we 'assume' that the user will soon ask for
- *	the stats after asking the bssid.
- */
-static int zd1201_get_wap(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
-{
-	struct sockaddr *ap_addr = &wrqu->ap_addr;
-	struct zd1201 *zd = netdev_priv(dev);
-	unsigned char buffer[6];
-
-	if (!zd1201_getconfig(zd, ZD1201_RID_COMMSQUALITY, buffer, 6)) {
-		/* Unfortunately the quality and noise reported is useless.
-		   they seem to be accumulators that increase until you
-		   read them, unless we poll on a fixed interval we can't
-		   use them
-		 */
-		/*zd->iwstats.qual.qual = le16_to_cpu(((__le16 *)buffer)[0]);*/
-		zd->iwstats.qual.level = le16_to_cpu(((__le16 *)buffer)[1]);
-		/*zd->iwstats.qual.noise = le16_to_cpu(((__le16 *)buffer)[2]);*/
-		zd->iwstats.qual.updated = 2;
-	}
-
-	return zd1201_getconfig(zd, ZD1201_RID_CURRENTBSSID, ap_addr->sa_data, 6);
-}
-
-static int zd1201_set_scan(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
-{
-	/* We do everything in get_scan */
-	return 0;
-}
-
-static int zd1201_get_scan(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
-{
-	struct iw_point *srq = &wrqu->data;
-	struct zd1201 *zd = netdev_priv(dev);
-	int err, i, j, enabled_save;
-	struct iw_event iwe;
-	char *cev = extra;
-	char *end_buf = extra + IW_SCAN_MAX_DATA;
-
-	/* No scanning in AP mode */
-	if (zd->ap)
-		return -EOPNOTSUPP;
-
-	/* Scan doesn't seem to work if disabled */
-	enabled_save = zd->mac_enabled;
-	zd1201_enable(zd);
-
-	zd->rxdatas = 0;
-	err = zd1201_docmd(zd, ZD1201_CMDCODE_INQUIRE, 
-	     ZD1201_INQ_SCANRESULTS, 0, 0);
-	if (err)
-		return err;
-
-	wait_event_interruptible(zd->rxdataq, zd->rxdatas);
-	if (!zd->rxlen)
-		return -EIO;
-
-	if (le16_to_cpu(*(__le16*)&zd->rxdata[2]) != ZD1201_INQ_SCANRESULTS)
-		return -EIO;
-
-	for(i=8; i<zd->rxlen; i+=62) {
-		iwe.cmd = SIOCGIWAP;
-		iwe.u.ap_addr.sa_family = ARPHRD_ETHER;
-		memcpy(iwe.u.ap_addr.sa_data, zd->rxdata+i+6, 6);
-		cev = iwe_stream_add_event(info, cev, end_buf,
-					   &iwe, IW_EV_ADDR_LEN);
-
-		iwe.cmd = SIOCGIWESSID;
-		iwe.u.data.length = zd->rxdata[i+16];
-		iwe.u.data.flags = 1;
-		cev = iwe_stream_add_point(info, cev, end_buf,
-					   &iwe, zd->rxdata+i+18);
-
-		iwe.cmd = SIOCGIWMODE;
-		if (zd->rxdata[i+14]&0x01)
-			iwe.u.mode = IW_MODE_MASTER;
-		else
-			iwe.u.mode = IW_MODE_ADHOC;
-		cev = iwe_stream_add_event(info, cev, end_buf,
-					   &iwe, IW_EV_UINT_LEN);
-		
-		iwe.cmd = SIOCGIWFREQ;
-		iwe.u.freq.m = zd->rxdata[i+0];
-		iwe.u.freq.e = 0;
-		cev = iwe_stream_add_event(info, cev, end_buf,
-					   &iwe, IW_EV_FREQ_LEN);
-		
-		iwe.cmd = SIOCGIWRATE;
-		iwe.u.bitrate.fixed = 0;
-		iwe.u.bitrate.disabled = 0;
-		for (j=0; j<10; j++) if (zd->rxdata[i+50+j]) {
-			iwe.u.bitrate.value = (zd->rxdata[i+50+j]&0x7f)*500000;
-			cev = iwe_stream_add_event(info, cev, end_buf,
-						   &iwe, IW_EV_PARAM_LEN);
-		}
-		
-		iwe.cmd = SIOCGIWENCODE;
-		iwe.u.data.length = 0;
-		if (zd->rxdata[i+14]&0x10)
-			iwe.u.data.flags = IW_ENCODE_ENABLED;
-		else
-			iwe.u.data.flags = IW_ENCODE_DISABLED;
-		cev = iwe_stream_add_point(info, cev, end_buf, &iwe, NULL);
-		
-		iwe.cmd = IWEVQUAL;
-		iwe.u.qual.qual = zd->rxdata[i+4];
-		iwe.u.qual.noise= zd->rxdata[i+2]/10-100;
-		iwe.u.qual.level = (256+zd->rxdata[i+4]*100)/255-100;
-		iwe.u.qual.updated = 7;
-		cev = iwe_stream_add_event(info, cev, end_buf,
-					   &iwe, IW_EV_QUAL_LEN);
-	}
-
-	if (!enabled_save)
-		zd1201_disable(zd);
-
-	srq->length = cev - extra;
-	srq->flags = 0;
-
-	return 0;
-}
-
-static int zd1201_set_essid(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *essid)
-{
-	struct iw_point *data = &wrqu->data;
-	struct zd1201 *zd = netdev_priv(dev);
-
-	if (data->length > IW_ESSID_MAX_SIZE)
-		return -EINVAL;
-	if (data->length < 1)
-		data->length = 1;
-	zd->essidlen = data->length;
-	memset(zd->essid, 0, IW_ESSID_MAX_SIZE+1);
-	memcpy(zd->essid, essid, data->length);
-	return zd1201_join(zd, zd->essid, zd->essidlen);
-}
-
-static int zd1201_get_essid(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *essid)
-{
-	struct iw_point *data = &wrqu->data;
-	struct zd1201 *zd = netdev_priv(dev);
-
-	memcpy(essid, zd->essid, zd->essidlen);
-	data->flags = 1;
-	data->length = zd->essidlen;
-
-	return 0;
-}
-
-static int zd1201_get_nick(struct net_device *dev, struct iw_request_info *info,
-	union iwreq_data *wrqu, char *nick)
-{
-	struct iw_point *data = &wrqu->data;
-	strcpy(nick, "zd1201");
-	data->flags = 1;
-	data->length = strlen(nick);
-	return 0;
-}
-
-static int zd1201_set_rate(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *rrq = &wrqu->bitrate;
-	struct zd1201 *zd = netdev_priv(dev);
-	short rate;
-	int err;
-
-	switch (rrq->value) {
-		case 1000000:
-			rate = ZD1201_RATEB1;
-			break;
-		case 2000000:
-			rate = ZD1201_RATEB2;
-			break;
-		case 5500000:
-			rate = ZD1201_RATEB5;
-			break;
-		case 11000000:
-		default:
-			rate = ZD1201_RATEB11;
-			break;
-	}
-	if (!rrq->fixed) { /* Also enable all lower bitrates */
-		rate |= rate-1;
-	}
-
-	err = zd1201_setconfig16(zd, ZD1201_RID_TXRATECNTL, rate);
-	if (err)
-		return err;
-
-	return zd1201_mac_reset(zd);
-}
-
-static int zd1201_get_rate(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *rrq = &wrqu->bitrate;
-	struct zd1201 *zd = netdev_priv(dev);
-	short rate;
-	int err;
-
-	err = zd1201_getconfig16(zd, ZD1201_RID_CURRENTTXRATE, &rate);
-	if (err)
-		return err;
-
-	switch(rate) {
-		case 1:
-			rrq->value = 1000000;
-			break;
-		case 2:
-			rrq->value = 2000000;
-			break;
-		case 5:
-			rrq->value = 5500000;
-			break;
-		case 11:
-			rrq->value = 11000000;
-			break;
-		default:
-			rrq->value = 0;
-	}
-	rrq->fixed = 0;
-	rrq->disabled = 0;
-
-	return 0;
-}
-
-static int zd1201_set_rts(struct net_device *dev, struct iw_request_info *info,
-	union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *rts = &wrqu->rts;
-	struct zd1201 *zd = netdev_priv(dev);
-	int err;
-	short val = rts->value;
-
-	if (rts->disabled || !rts->fixed)
-		val = ZD1201_RTSMAX;
-	if (val > ZD1201_RTSMAX)
-		return -EINVAL;
-	if (val < 0)
-		return -EINVAL;
-
-	err = zd1201_setconfig16(zd, ZD1201_RID_CNFRTSTHRESHOLD, val);
-	if (err)
-		return err;
-	return zd1201_mac_reset(zd);
-}
-
-static int zd1201_get_rts(struct net_device *dev, struct iw_request_info *info,
-	union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *rts = &wrqu->rts;
-	struct zd1201 *zd = netdev_priv(dev);
-	short rtst;
-	int err;
-
-	err = zd1201_getconfig16(zd, ZD1201_RID_CNFRTSTHRESHOLD, &rtst);
-	if (err)
-		return err;
-	rts->value = rtst;
-	rts->disabled = (rts->value == ZD1201_RTSMAX);
-	rts->fixed = 1;
-
-	return 0;
-}
-
-static int zd1201_set_frag(struct net_device *dev, struct iw_request_info *info,
-	union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *frag = &wrqu->frag;
-	struct zd1201 *zd = netdev_priv(dev);
-	int err;
-	short val = frag->value;
-
-	if (frag->disabled || !frag->fixed)
-		val = ZD1201_FRAGMAX;
-	if (val > ZD1201_FRAGMAX)
-		return -EINVAL;
-	if (val < ZD1201_FRAGMIN)
-		return -EINVAL;
-	if (val & 1)
-		return -EINVAL;
-	err = zd1201_setconfig16(zd, ZD1201_RID_CNFFRAGTHRESHOLD, val);
-	if (err)
-		return err;
-	return zd1201_mac_reset(zd);
-}
-
-static int zd1201_get_frag(struct net_device *dev, struct iw_request_info *info,
-	union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *frag = &wrqu->frag;
-	struct zd1201 *zd = netdev_priv(dev);
-	short fragt;
-	int err;
-
-	err = zd1201_getconfig16(zd, ZD1201_RID_CNFFRAGTHRESHOLD, &fragt);
-	if (err)
-		return err;
-	frag->value = fragt;
-	frag->disabled = (frag->value == ZD1201_FRAGMAX);
-	frag->fixed = 1;
-
-	return 0;
-}
-
-static int zd1201_set_retry(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
-{
-	return 0;
-}
-
-static int zd1201_get_retry(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
-{
-	return 0;
-}
-
-static int zd1201_set_encode(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *key)
-{
-	struct iw_point *erq = &wrqu->encoding;
-	struct zd1201 *zd = netdev_priv(dev);
-	short i;
-	int err, rid;
-
-	if (erq->length > ZD1201_MAXKEYLEN)
-		return -EINVAL;
-
-	i = (erq->flags & IW_ENCODE_INDEX)-1;
-	if (i == -1) {
-		err = zd1201_getconfig16(zd,ZD1201_RID_CNFDEFAULTKEYID,&i);
-		if (err)
-			return err;
-	} else {
-		err = zd1201_setconfig16(zd, ZD1201_RID_CNFDEFAULTKEYID, i);
-		if (err)
-			return err;
-	}
-
-	if (i < 0 || i >= ZD1201_NUMKEYS)
-		return -EINVAL;
-
-	rid = ZD1201_RID_CNFDEFAULTKEY0 + i;
-	err = zd1201_setconfig(zd, rid, key, erq->length, 1);
-	if (err)
-		return err;
-	zd->encode_keylen[i] = erq->length;
-	memcpy(zd->encode_keys[i], key, erq->length);
-
-	i=0;
-	if (!(erq->flags & IW_ENCODE_DISABLED & IW_ENCODE_MODE)) {
-		i |= 0x01;
-		zd->encode_enabled = 1;
-	} else
-		zd->encode_enabled = 0;
-	if (erq->flags & IW_ENCODE_RESTRICTED & IW_ENCODE_MODE) {
-		i |= 0x02;
-		zd->encode_restricted = 1;
-	} else
-		zd->encode_restricted = 0;
-	err = zd1201_setconfig16(zd, ZD1201_RID_CNFWEBFLAGS, i);
-	if (err)
-		return err;
-
-	if (zd->encode_enabled)
-		i = ZD1201_CNFAUTHENTICATION_SHAREDKEY;
-	else
-		i = ZD1201_CNFAUTHENTICATION_OPENSYSTEM;
-	err = zd1201_setconfig16(zd, ZD1201_RID_CNFAUTHENTICATION, i);
-	if (err)
-		return err;
-
-	return zd1201_mac_reset(zd);
-}
-
-static int zd1201_get_encode(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *key)
-{
-	struct iw_point *erq = &wrqu->encoding;
-	struct zd1201 *zd = netdev_priv(dev);
-	short i;
-	int err;
-
-	if (zd->encode_enabled)
-		erq->flags = IW_ENCODE_ENABLED;
-	else
-		erq->flags = IW_ENCODE_DISABLED;
-	if (zd->encode_restricted)
-		erq->flags |= IW_ENCODE_RESTRICTED;
-	else
-		erq->flags |= IW_ENCODE_OPEN;
-
-	i = (erq->flags & IW_ENCODE_INDEX) -1;
-	if (i == -1) {
-		err = zd1201_getconfig16(zd, ZD1201_RID_CNFDEFAULTKEYID, &i);
-		if (err)
-			return err;
-	}
-	if (i<0 || i>= ZD1201_NUMKEYS)
-		return -EINVAL;
-
-	erq->flags |= i+1;
-
-	erq->length = zd->encode_keylen[i];
-	memcpy(key, zd->encode_keys[i], erq->length);
-
-	return 0;
-}
-
-static int zd1201_set_power(struct net_device *dev, 
-	struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *vwrq = &wrqu->power;
-	struct zd1201 *zd = netdev_priv(dev);
-	short enabled, duration, level;
-	int err;
-
-	enabled = vwrq->disabled ? 0 : 1;
-	if (enabled) {
-		if (vwrq->flags & IW_POWER_PERIOD) {
-			duration = vwrq->value;
-			err = zd1201_setconfig16(zd, 
-			    ZD1201_RID_CNFMAXSLEEPDURATION, duration);
-			if (err)
-				return err;
-			goto out;
-		}
-		if (vwrq->flags & IW_POWER_TIMEOUT) {
-			err = zd1201_getconfig16(zd, 
-			    ZD1201_RID_CNFMAXSLEEPDURATION, &duration);
-			if (err)
-				return err;
-			level = vwrq->value * 4 / duration;
-			if (level > 4)
-				level = 4;
-			if (level < 0)
-				level = 0;
-			err = zd1201_setconfig16(zd, ZD1201_RID_CNFPMEPS,
-			    level);
-			if (err)
-				return err;
-			goto out;
-		}
-		return -EINVAL;
-	}
-out:
-	return zd1201_setconfig16(zd, ZD1201_RID_CNFPMENABLED, enabled);
-}
-
-static int zd1201_get_power(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *vwrq = &wrqu->power;
-	struct zd1201 *zd = netdev_priv(dev);
-	short enabled, level, duration;
-	int err;
-
-	err = zd1201_getconfig16(zd, ZD1201_RID_CNFPMENABLED, &enabled);
-	if (err)
-		return err;
-	err = zd1201_getconfig16(zd, ZD1201_RID_CNFPMEPS, &level);
-	if (err)
-		return err;
-	err = zd1201_getconfig16(zd, ZD1201_RID_CNFMAXSLEEPDURATION, &duration);
-	if (err)
-		return err;
-	vwrq->disabled = enabled ? 0 : 1;
-	if (vwrq->flags & IW_POWER_TYPE) {
-		if (vwrq->flags & IW_POWER_PERIOD) {
-			vwrq->value = duration;
-			vwrq->flags = IW_POWER_PERIOD;
-		} else {
-			vwrq->value = duration * level / 4;
-			vwrq->flags = IW_POWER_TIMEOUT;
-		}
-	}
-	if (vwrq->flags & IW_POWER_MODE) {
-		if (enabled && level)
-			vwrq->flags = IW_POWER_UNICAST_R;
-		else
-			vwrq->flags = IW_POWER_ALL_R;
-	}
-
-	return 0;
-}
-
-
-static const iw_handler zd1201_iw_handler[] =
-{
-	IW_HANDLER(SIOCSIWCOMMIT,	zd1201_config_commit),
-	IW_HANDLER(SIOCGIWNAME,		zd1201_get_name),
-	IW_HANDLER(SIOCSIWFREQ,		zd1201_set_freq),
-	IW_HANDLER(SIOCGIWFREQ,		zd1201_get_freq),
-	IW_HANDLER(SIOCSIWMODE,		zd1201_set_mode),
-	IW_HANDLER(SIOCGIWMODE,		zd1201_get_mode),
-	IW_HANDLER(SIOCGIWRANGE,	zd1201_get_range),
-	IW_HANDLER(SIOCGIWAP,		zd1201_get_wap),
-	IW_HANDLER(SIOCSIWSCAN,		zd1201_set_scan),
-	IW_HANDLER(SIOCGIWSCAN,		zd1201_get_scan),
-	IW_HANDLER(SIOCSIWESSID,	zd1201_set_essid),
-	IW_HANDLER(SIOCGIWESSID,	zd1201_get_essid),
-	IW_HANDLER(SIOCGIWNICKN,	zd1201_get_nick),
-	IW_HANDLER(SIOCSIWRATE,		zd1201_set_rate),
-	IW_HANDLER(SIOCGIWRATE,		zd1201_get_rate),
-	IW_HANDLER(SIOCSIWRTS,		zd1201_set_rts),
-	IW_HANDLER(SIOCGIWRTS,		zd1201_get_rts),
-	IW_HANDLER(SIOCSIWFRAG,		zd1201_set_frag),
-	IW_HANDLER(SIOCGIWFRAG,		zd1201_get_frag),
-	IW_HANDLER(SIOCSIWRETRY,	zd1201_set_retry),
-	IW_HANDLER(SIOCGIWRETRY,	zd1201_get_retry),
-	IW_HANDLER(SIOCSIWENCODE,	zd1201_set_encode),
-	IW_HANDLER(SIOCGIWENCODE,	zd1201_get_encode),
-	IW_HANDLER(SIOCSIWPOWER,	zd1201_set_power),
-	IW_HANDLER(SIOCGIWPOWER,	zd1201_get_power),
-};
-
-static int zd1201_set_hostauth(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *rrq = &wrqu->param;
-	struct zd1201 *zd = netdev_priv(dev);
-
-	if (!zd->ap)
-		return -EOPNOTSUPP;
-
-	return zd1201_setconfig16(zd, ZD1201_RID_CNFHOSTAUTH, rrq->value);
-}
-
-static int zd1201_get_hostauth(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *rrq = &wrqu->param;
-	struct zd1201 *zd = netdev_priv(dev);
-	short hostauth;
-	int err;
-
-	if (!zd->ap)
-		return -EOPNOTSUPP;
-
-	err = zd1201_getconfig16(zd, ZD1201_RID_CNFHOSTAUTH, &hostauth);
-	if (err)
-		return err;
-	rrq->value = hostauth;
-	rrq->fixed = 1;
-
-	return 0;
-}
-
-static int zd1201_auth_sta(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
-{
-	struct sockaddr *sta = &wrqu->ap_addr;
-	struct zd1201 *zd = netdev_priv(dev);
-	unsigned char buffer[10];
-
-	if (!zd->ap)
-		return -EOPNOTSUPP;
-
-	memcpy(buffer, sta->sa_data, ETH_ALEN);
-	*(short*)(buffer+6) = 0;	/* 0==success, 1==failure */
-	*(short*)(buffer+8) = 0;
-
-	return zd1201_setconfig(zd, ZD1201_RID_AUTHENTICATESTA, buffer, 10, 1);
-}
-
-static int zd1201_set_maxassoc(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *rrq = &wrqu->param;
-	struct zd1201 *zd = netdev_priv(dev);
-
-	if (!zd->ap)
-		return -EOPNOTSUPP;
-
-	return zd1201_setconfig16(zd, ZD1201_RID_CNFMAXASSOCSTATIONS, rrq->value);
-}
-
-static int zd1201_get_maxassoc(struct net_device *dev,
-	struct iw_request_info *info, union iwreq_data *wrqu, char *extra)
-{
-	struct iw_param *rrq = &wrqu->param;
-	struct zd1201 *zd = netdev_priv(dev);
-	short maxassoc;
-	int err;
-
-	if (!zd->ap)
-		return -EOPNOTSUPP;
-
-	err = zd1201_getconfig16(zd, ZD1201_RID_CNFMAXASSOCSTATIONS, &maxassoc);
-	if (err)
-		return err;
-	rrq->value = maxassoc;
-	rrq->fixed = 1;
-
-	return 0;
-}
-
-static const iw_handler zd1201_private_handler[] = {
-	zd1201_set_hostauth,	/* ZD1201SIWHOSTAUTH */
-	zd1201_get_hostauth,	/* ZD1201GIWHOSTAUTH */
-	zd1201_auth_sta,	/* ZD1201SIWAUTHSTA */
-	NULL,			/* nothing to get */
-	zd1201_set_maxassoc,	/* ZD1201SIMAXASSOC */
-	zd1201_get_maxassoc,	/* ZD1201GIMAXASSOC */
-};
-
-static const struct iw_priv_args zd1201_private_args[] = {
-	{ ZD1201SIWHOSTAUTH, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1,
-	    IW_PRIV_TYPE_NONE, "sethostauth" },
-	{ ZD1201GIWHOSTAUTH, IW_PRIV_TYPE_NONE,
-	    IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "gethostauth" },
-	{ ZD1201SIWAUTHSTA, IW_PRIV_TYPE_ADDR | IW_PRIV_SIZE_FIXED | 1,
-	    IW_PRIV_TYPE_NONE, "authstation" },
-	{ ZD1201SIWMAXASSOC, IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1,
-	    IW_PRIV_TYPE_NONE, "setmaxassoc" },
-	{ ZD1201GIWMAXASSOC, IW_PRIV_TYPE_NONE,
-	    IW_PRIV_TYPE_INT | IW_PRIV_SIZE_FIXED | 1, "getmaxassoc" },
-};
-
-static const struct iw_handler_def zd1201_iw_handlers = {
-	.num_standard 		= ARRAY_SIZE(zd1201_iw_handler),
-	.num_private 		= ARRAY_SIZE(zd1201_private_handler),
-	.num_private_args 	= ARRAY_SIZE(zd1201_private_args),
-	.standard		= zd1201_iw_handler,
-	.private		= zd1201_private_handler,
-	.private_args 		= (struct iw_priv_args *) zd1201_private_args,
-	.get_wireless_stats	= zd1201_get_wireless_stats,
-};
-
-static const struct net_device_ops zd1201_netdev_ops = {
-	.ndo_open		= zd1201_net_open,
-	.ndo_stop		= zd1201_net_stop,
-	.ndo_start_xmit		= zd1201_hard_start_xmit,
-	.ndo_tx_timeout		= zd1201_tx_timeout,
-	.ndo_set_rx_mode	= zd1201_set_multicast,
-	.ndo_set_mac_address	= zd1201_set_mac_address,
-	.ndo_validate_addr	= eth_validate_addr,
-};
-
-static int zd1201_probe(struct usb_interface *interface,
-			const struct usb_device_id *id)
-{
-	struct zd1201 *zd;
-	struct net_device *dev;
-	struct usb_device *usb;
-	int err;
-	short porttype;
-	char buf[IW_ESSID_MAX_SIZE+2];
-	u8 addr[ETH_ALEN];
-
-	usb = interface_to_usbdev(interface);
-
-	dev = alloc_etherdev(sizeof(*zd));
-	if (!dev)
-		return -ENOMEM;
-	zd = netdev_priv(dev);
-	zd->dev = dev;
-
-	zd->ap = ap;
-	zd->usb = usb;
-	zd->removed = 0;
-	init_waitqueue_head(&zd->rxdataq);
-	INIT_HLIST_HEAD(&zd->fraglist);
-	
-	err = zd1201_fw_upload(usb, zd->ap);
-	if (err) {
-		dev_err(&usb->dev, "zd1201 firmware upload failed: %d\n", err);
-		goto err_zd;
-	}
-	
-	zd->endp_in = 1;
-	zd->endp_out = 1;
-	zd->endp_out2 = 2;
-	zd->rx_urb = usb_alloc_urb(0, GFP_KERNEL);
-	zd->tx_urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!zd->rx_urb || !zd->tx_urb) {
-		err = -ENOMEM;
-		goto err_zd;
-	}
-
-	mdelay(100);
-	err = zd1201_drvr_start(zd);
-	if (err)
-		goto err_zd;
-
-	err = zd1201_setconfig16(zd, ZD1201_RID_CNFMAXDATALEN, 2312);
-	if (err)
-		goto err_start;
-
-	err = zd1201_setconfig16(zd, ZD1201_RID_TXRATECNTL,
-	    ZD1201_RATEB1 | ZD1201_RATEB2 | ZD1201_RATEB5 | ZD1201_RATEB11);
-	if (err)
-		goto err_start;
-
-	dev->netdev_ops = &zd1201_netdev_ops;
-	dev->wireless_handlers = &zd1201_iw_handlers;
-	dev->watchdog_timeo = ZD1201_TX_TIMEOUT;
-	strcpy(dev->name, "wlan%d");
-
-	err = zd1201_getconfig(zd, ZD1201_RID_CNFOWNMACADDR, addr, ETH_ALEN);
-	if (err)
-		goto err_start;
-	eth_hw_addr_set(dev, addr);
-
-	/* Set wildcard essid to match zd->essid */
-	*(__le16 *)buf = cpu_to_le16(0);
-	err = zd1201_setconfig(zd, ZD1201_RID_CNFDESIREDSSID, buf,
-	    IW_ESSID_MAX_SIZE+2, 1);
-	if (err)
-		goto err_start;
-
-	if (zd->ap)
-		porttype = ZD1201_PORTTYPE_AP;
-	else
-		porttype = ZD1201_PORTTYPE_BSS;
-	err = zd1201_setconfig16(zd, ZD1201_RID_CNFPORTTYPE, porttype);
-	if (err)
-		goto err_start;
-
-	SET_NETDEV_DEV(dev, &usb->dev);
-
-	err = register_netdev(dev);
-	if (err)
-		goto err_start;
-	dev_info(&usb->dev, "%s: ZD1201 USB Wireless interface\n",
-	    dev->name);
-
-	usb_set_intfdata(interface, zd);
-	zd1201_enable(zd);	/* zd1201 likes to startup enabled, */
-	zd1201_disable(zd);	/* interfering with all the wifis in range */
-	return 0;
-
-err_start:
-	/* Leave the device in reset state */
-	zd1201_docmd(zd, ZD1201_CMDCODE_INIT, 0, 0, 0);
-err_zd:
-	usb_free_urb(zd->tx_urb);
-	usb_free_urb(zd->rx_urb);
-	free_netdev(dev);
-	return err;
-}
-
-static void zd1201_disconnect(struct usb_interface *interface)
-{
-	struct zd1201 *zd = usb_get_intfdata(interface);
-	struct hlist_node *node2;
-	struct zd1201_frag *frag;
-
-	if (!zd)
-		return;
-	usb_set_intfdata(interface, NULL);
-
-	hlist_for_each_entry_safe(frag, node2, &zd->fraglist, fnode) {
-		hlist_del_init(&frag->fnode);
-		kfree_skb(frag->skb);
-		kfree(frag);
-	}
-
-	if (zd->tx_urb) {
-		usb_kill_urb(zd->tx_urb);
-		usb_free_urb(zd->tx_urb);
-	}
-	if (zd->rx_urb) {
-		usb_kill_urb(zd->rx_urb);
-		usb_free_urb(zd->rx_urb);
-	}
-
-	if (zd->dev) {
-		unregister_netdev(zd->dev);
-		free_netdev(zd->dev);
-	}
-}
-
-#ifdef CONFIG_PM
-
-static int zd1201_suspend(struct usb_interface *interface,
-			   pm_message_t message)
-{
-	struct zd1201 *zd = usb_get_intfdata(interface);
-
-	netif_device_detach(zd->dev);
-
-	zd->was_enabled = zd->mac_enabled;
-
-	if (zd->was_enabled)
-		return zd1201_disable(zd);
-	else
-		return 0;
-}
-
-static int zd1201_resume(struct usb_interface *interface)
-{
-	struct zd1201 *zd = usb_get_intfdata(interface);
-
-	if (!zd || !zd->dev)
-		return -ENODEV;
-
-	netif_device_attach(zd->dev);
-
-	if (zd->was_enabled)
-		return zd1201_enable(zd);
-	else
-		return 0;
-}
-
-#else
-
-#define zd1201_suspend NULL
-#define zd1201_resume  NULL
-
-#endif
-
-static struct usb_driver zd1201_usb = {
-	.name = "zd1201",
-	.probe = zd1201_probe,
-	.disconnect = zd1201_disconnect,
-	.id_table = zd1201_table,
-	.suspend = zd1201_suspend,
-	.resume = zd1201_resume,
-	.disable_hub_initiated_lpm = 1,
-};
-
-module_usb_driver(zd1201_usb);
diff --git a/drivers/net/wireless/zydas/zd1201.h b/drivers/net/wireless/zydas/zd1201.h
deleted file mode 100644
index c46ac87550d1..000000000000
--- a/drivers/net/wireless/zydas/zd1201.h
+++ /dev/null
@@ -1,144 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- *	Copyright (c) 2004, 2005 Jeroen Vreeken (pe1rxq@amsat.org)
- *
- *	Parts of this driver have been derived from a wlan-ng version
- *	modified by ZyDAS.
- *	Copyright (C) 1999 AbsoluteValue Systems, Inc.  All Rights Reserved.
- */
-
-#ifndef _INCLUDE_ZD1201_H_
-#define _INCLUDE_ZD1201_H_
-
-#define ZD1201_NUMKEYS		4
-#define ZD1201_MAXKEYLEN	13
-#define ZD1201_MAXMULTI		16
-#define ZD1201_FRAGMAX		2500
-#define ZD1201_FRAGMIN		256
-#define ZD1201_RTSMAX		2500
-
-#define ZD1201_RXSIZE		3000
-
-struct zd1201 {
-	struct usb_device	*usb;
-	int			removed;
-	struct net_device	*dev;
-	struct iw_statistics	iwstats;
-
-	int			endp_in;
-	int			endp_out;
-	int			endp_out2;
-	struct urb		*rx_urb;
-	struct urb		*tx_urb;
-
-	unsigned char 		rxdata[ZD1201_RXSIZE];
-	int			rxlen;
-	wait_queue_head_t	rxdataq;
-	int			rxdatas;
-	struct hlist_head	fraglist;
-	unsigned char		txdata[ZD1201_RXSIZE];
-
-	int			ap;
-	char			essid[IW_ESSID_MAX_SIZE+1];
-	int			essidlen;
-	int			mac_enabled;
-	int			was_enabled;
-	int			monitor;
-	int			encode_enabled;
-	int			encode_restricted;
-	unsigned char		encode_keys[ZD1201_NUMKEYS][ZD1201_MAXKEYLEN];
-	int			encode_keylen[ZD1201_NUMKEYS];
-};
-
-struct zd1201_frag {
-	struct hlist_node	fnode;
-	int			seq;
-	struct sk_buff		*skb;
-};
-
-#define ZD1201SIWHOSTAUTH SIOCIWFIRSTPRIV
-#define ZD1201GIWHOSTAUTH ZD1201SIWHOSTAUTH+1
-#define ZD1201SIWAUTHSTA SIOCIWFIRSTPRIV+2
-#define ZD1201SIWMAXASSOC SIOCIWFIRSTPRIV+4
-#define ZD1201GIWMAXASSOC ZD1201SIWMAXASSOC+1
-
-#define ZD1201_FW_TIMEOUT	(1000)
-
-#define ZD1201_TX_TIMEOUT	(2000)
-
-#define ZD1201_USB_CMDREQ	0
-#define ZD1201_USB_RESREQ	1
-
-#define	ZD1201_CMDCODE_INIT	0x00
-#define ZD1201_CMDCODE_ENABLE	0x01
-#define ZD1201_CMDCODE_DISABLE	0x02
-#define ZD1201_CMDCODE_ALLOC	0x0a
-#define ZD1201_CMDCODE_INQUIRE	0x11
-#define ZD1201_CMDCODE_SETRXRID	0x17
-#define ZD1201_CMDCODE_ACCESS	0x21
-
-#define ZD1201_PACKET_EVENTSTAT	0x0
-#define ZD1201_PACKET_RXDATA	0x1
-#define ZD1201_PACKET_INQUIRE	0x2
-#define ZD1201_PACKET_RESOURCE	0x3
-
-#define ZD1201_ACCESSBIT	0x0100
-
-#define ZD1201_RID_CNFPORTTYPE		0xfc00
-#define ZD1201_RID_CNFOWNMACADDR	0xfc01
-#define ZD1201_RID_CNFDESIREDSSID	0xfc02
-#define ZD1201_RID_CNFOWNCHANNEL	0xfc03
-#define ZD1201_RID_CNFOWNSSID		0xfc04
-#define ZD1201_RID_CNFMAXDATALEN	0xfc07
-#define ZD1201_RID_CNFPMENABLED		0xfc09
-#define ZD1201_RID_CNFPMEPS		0xfc0a
-#define ZD1201_RID_CNFMAXSLEEPDURATION	0xfc0c
-#define ZD1201_RID_CNFDEFAULTKEYID	0xfc23
-#define ZD1201_RID_CNFDEFAULTKEY0	0xfc24
-#define ZD1201_RID_CNFDEFAULTKEY1	0xfc25
-#define ZD1201_RID_CNFDEFAULTKEY2	0xfc26
-#define ZD1201_RID_CNFDEFAULTKEY3	0xfc27
-#define ZD1201_RID_CNFWEBFLAGS		0xfc28
-#define ZD1201_RID_CNFAUTHENTICATION	0xfc2a
-#define ZD1201_RID_CNFMAXASSOCSTATIONS	0xfc2b
-#define ZD1201_RID_CNFHOSTAUTH		0xfc2e
-#define ZD1201_RID_CNFGROUPADDRESS	0xfc80
-#define ZD1201_RID_CNFFRAGTHRESHOLD	0xfc82
-#define ZD1201_RID_CNFRTSTHRESHOLD	0xfc83
-#define ZD1201_RID_TXRATECNTL		0xfc84
-#define ZD1201_RID_PROMISCUOUSMODE	0xfc85
-#define ZD1201_RID_CNFBASICRATES	0xfcb3
-#define ZD1201_RID_AUTHENTICATESTA	0xfce3
-#define ZD1201_RID_CURRENTBSSID		0xfd42
-#define ZD1201_RID_COMMSQUALITY		0xfd43
-#define ZD1201_RID_CURRENTTXRATE	0xfd44
-#define ZD1201_RID_CNFMAXTXBUFFERNUMBER	0xfda0
-#define ZD1201_RID_CURRENTCHANNEL	0xfdc1
-
-#define ZD1201_INQ_SCANRESULTS		0xf101
-
-#define ZD1201_INF_LINKSTATUS		0xf200
-#define ZD1201_INF_ASSOCSTATUS		0xf201
-#define ZD1201_INF_AUTHREQ		0xf202
-
-#define ZD1201_ASSOCSTATUS_STAASSOC	0x1
-#define ZD1201_ASSOCSTATUS_REASSOC	0x2
-#define ZD1201_ASSOCSTATUS_DISASSOC	0x3
-#define ZD1201_ASSOCSTATUS_ASSOCFAIL	0x4
-#define ZD1201_ASSOCSTATUS_AUTHFAIL	0x5
-
-#define ZD1201_PORTTYPE_IBSS		0
-#define ZD1201_PORTTYPE_BSS		1
-#define ZD1201_PORTTYPE_WDS		2
-#define ZD1201_PORTTYPE_PSEUDOIBSS	3
-#define ZD1201_PORTTYPE_AP		6
-
-#define ZD1201_RATEB1	1
-#define ZD1201_RATEB2	2
-#define ZD1201_RATEB5	4	/* 5.5 really, but 5 is shorter :) */
-#define ZD1201_RATEB11	8
-
-#define ZD1201_CNFAUTHENTICATION_OPENSYSTEM	0x0001
-#define ZD1201_CNFAUTHENTICATION_SHAREDKEY	0x0002
-
-#endif /* _INCLUDE_ZD1201_H_ */
diff --git a/drivers/ntb/hw/intel/ntb_hw_gen1.c b/drivers/ntb/hw/intel/ntb_hw_gen1.c
index 9ab836d0d4f1..079b8cd79785 100644
--- a/drivers/ntb/hw/intel/ntb_hw_gen1.c
+++ b/drivers/ntb/hw/intel/ntb_hw_gen1.c
@@ -778,7 +778,7 @@ static void ndev_init_debugfs(struct intel_ntb_dev *ndev)
 		ndev->debugfs_dir =
 			debugfs_create_dir(pci_name(ndev->ntb.pdev),
 					   debugfs_dir);
-		if (!ndev->debugfs_dir)
+		if (IS_ERR(ndev->debugfs_dir))
 			ndev->debugfs_info = NULL;
 		else
 			ndev->debugfs_info =
diff --git a/drivers/ntb/test/ntb_perf.c b/drivers/ntb/test/ntb_perf.c
index 553f1f46bc66..72bc1d017a46 100644
--- a/drivers/ntb/test/ntb_perf.c
+++ b/drivers/ntb/test/ntb_perf.c
@@ -1227,7 +1227,7 @@ static ssize_t perf_dbgfs_read_info(struct file *filep, char __user *ubuf,
 			"\tOut buffer addr 0x%pK\n", peer->outbuf);
 
 		pos += scnprintf(buf + pos, buf_size - pos,
-			"\tOut buff phys addr %pa[p]\n", &peer->out_phys_addr);
+			"\tOut buff phys addr %pap\n", &peer->out_phys_addr);
 
 		pos += scnprintf(buf + pos, buf_size - pos,
 			"\tOut buffer size %pa\n", &peer->outbuf_size);
diff --git a/drivers/opp/core.c b/drivers/opp/core.c
index 84f345c69ea5..f2e2aa07b431 100644
--- a/drivers/opp/core.c
+++ b/drivers/opp/core.c
@@ -201,7 +201,7 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_get_freq_indexed);
  * @opp:	opp for which level value has to be returned for
  *
  * Return: level read from device tree corresponding to the opp, else
- * return 0.
+ * return U32_MAX.
  */
 unsigned int dev_pm_opp_get_level(struct dev_pm_opp *opp)
 {
@@ -221,7 +221,7 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_get_level);
  * @index:	index of the required opp
  *
  * Return: performance state read from device tree corresponding to the
- * required opp, else return 0.
+ * required opp, else return U32_MAX.
  */
 unsigned int dev_pm_opp_get_required_pstate(struct dev_pm_opp *opp,
 					    unsigned int index)
@@ -808,6 +808,14 @@ struct dev_pm_opp *dev_pm_opp_find_level_ceil(struct device *dev,
 	struct dev_pm_opp *opp;
 
 	opp = _find_key_ceil(dev, &temp, 0, true, _read_level, NULL);
+
+	/* False match */
+	if (temp == OPP_LEVEL_UNSET) {
+		dev_err(dev, "%s: OPP levels aren't available\n", __func__);
+		dev_pm_opp_put(opp);
+		return ERR_PTR(-ENODEV);
+	}
+
 	*level = temp;
 	return opp;
 }
@@ -1049,12 +1057,18 @@ static int _set_opp_bw(const struct opp_table *opp_table,
 static int _set_performance_state(struct device *dev, struct device *pd_dev,
 				  struct dev_pm_opp *opp, int i)
 {
-	unsigned int pstate = likely(opp) ? opp->required_opps[i]->level: 0;
+	unsigned int pstate = 0;
 	int ret;
 
 	if (!pd_dev)
 		return 0;
 
+	if (likely(opp)) {
+		pstate = opp->required_opps[i]->level;
+		if (pstate == OPP_LEVEL_UNSET)
+			return 0;
+	}
+
 	ret = dev_pm_domain_set_performance_state(pd_dev, pstate);
 	if (ret) {
 		dev_err(dev, "Failed to set performance state of %s: %d (%d)\n",
@@ -1135,7 +1149,7 @@ static int _set_opp_level(struct device *dev, struct opp_table *opp_table,
 	int ret = 0;
 
 	if (opp) {
-		if (!opp->level)
+		if (opp->level == OPP_LEVEL_UNSET)
 			return 0;
 
 		level = opp->level;
@@ -1867,6 +1881,8 @@ struct dev_pm_opp *_opp_allocate(struct opp_table *opp_table)
 
 	INIT_LIST_HEAD(&opp->node);
 
+	opp->level = OPP_LEVEL_UNSET;
+
 	return opp;
 }
 
diff --git a/drivers/opp/of.c b/drivers/opp/of.c
index 81fa27599d58..85fad7ca0007 100644
--- a/drivers/opp/of.c
+++ b/drivers/opp/of.c
@@ -1393,8 +1393,14 @@ int of_get_required_opp_performance_state(struct device_node *np, int index)
 
 	opp = _find_opp_of_np(opp_table, required_np);
 	if (opp) {
-		pstate = opp->level;
+		if (opp->level == OPP_LEVEL_UNSET) {
+			pr_err("%s: OPP levels aren't available for %pOF\n",
+			       __func__, np);
+		} else {
+			pstate = opp->level;
+		}
 		dev_pm_opp_put(opp);
+
 	}
 
 	dev_pm_opp_put_opp_table(opp_table);
diff --git a/drivers/opp/ti-opp-supply.c b/drivers/opp/ti-opp-supply.c
index 8f3f13fbbb25..e3b97cd1fbbf 100644
--- a/drivers/opp/ti-opp-supply.c
+++ b/drivers/opp/ti-opp-supply.c
@@ -18,6 +18,7 @@
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/pm_opp.h>
+#include <linux/property.h>
 #include <linux/regulator/consumer.h>
 #include <linux/slab.h>
 
@@ -373,23 +374,15 @@ static int ti_opp_supply_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct device *cpu_dev = get_cpu_device(0);
-	const struct of_device_id *match;
 	const struct ti_opp_supply_of_data *of_data;
 	int ret = 0;
 
-	match = of_match_device(ti_opp_supply_of_match, dev);
-	if (!match) {
-		/* We do not expect this to happen */
-		dev_err(dev, "%s: Unable to match device\n", __func__);
-		return -ENODEV;
-	}
-	if (!match->data) {
+	of_data = device_get_match_data(dev);
+	if (!of_data) {
 		/* Again, unlikely.. but mistakes do happen */
 		dev_err(dev, "%s: Bad data in match\n", __func__);
 		return -EINVAL;
 	}
-	of_data = match->data;
-
 	dev_set_drvdata(dev, (void *)of_data);
 
 	/* If we need optimized voltage */
diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c
index 30c7dfeccb16..1eaffff40b8d 100644
--- a/drivers/pci/controller/pci-hyperv.c
+++ b/drivers/pci/controller/pci-hyperv.c
@@ -650,13 +650,6 @@ static void hv_arch_irq_unmask(struct irq_data *data)
 			   PCI_FUNC(pdev->devfn);
 	params->int_target.vector = hv_msi_get_int_vector(data);
 
-	/*
-	 * Honoring apic->delivery_mode set to APIC_DELIVERY_MODE_FIXED by
-	 * setting the HV_DEVICE_INTERRUPT_TARGET_MULTICAST flag results in a
-	 * spurious interrupt storm. Not doing so does not seem to have a
-	 * negative effect (yet?).
-	 */
-
 	if (hbus->protocol_version >= PCI_PROTOCOL_VERSION_1_2) {
 		/*
 		 * PCI_PROTOCOL_VERSION_1_2 supports the VP_SET version of the
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index 55bc3576a985..69d20d585f88 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -1219,6 +1219,9 @@ static int pci_dev_wait(struct pci_dev *dev, char *reset_type, int timeout)
 	if (delay > PCI_RESET_WAIT)
 		pci_info(dev, "ready %dms after %s\n", delay - 1,
 			 reset_type);
+	else
+		pci_dbg(dev, "ready %dms after %s\n", delay - 1,
+			reset_type);
 
 	return 0;
 }
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
index ea476252280a..d208047d1b8f 100644
--- a/drivers/pci/quirks.c
+++ b/drivers/pci/quirks.c
@@ -3786,6 +3786,19 @@ static void quirk_no_pm_reset(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_CLASS_HEADER(PCI_VENDOR_ID_ATI, PCI_ANY_ID,
 			       PCI_CLASS_DISPLAY_VGA, 8, quirk_no_pm_reset);
 
+/*
+ * Spectrum-{1,2,3,4} devices report that a D3hot->D0 transition causes a reset
+ * (i.e., they advertise NoSoftRst-). However, this transition does not have
+ * any effect on the device: It continues to be operational and network ports
+ * remain up. Advertising this support makes it seem as if a PM reset is viable
+ * for these devices. Mark it as unavailable to skip it when testing reset
+ * methods.
+ */
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_MELLANOX, 0xcb84, quirk_no_pm_reset);
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_MELLANOX, 0xcf6c, quirk_no_pm_reset);
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_MELLANOX, 0xcf70, quirk_no_pm_reset);
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_MELLANOX, 0xcf80, quirk_no_pm_reset);
+
 /*
  * Thunderbolt controllers with broken MSI hotplug signaling:
  * Entire 1st generation (Light Ridge, Eagle Ridge, Light Peak) and part
diff --git a/drivers/phy/Kconfig b/drivers/phy/Kconfig
index 787354b849c7..4cef568231bf 100644
--- a/drivers/phy/Kconfig
+++ b/drivers/phy/Kconfig
@@ -87,7 +87,6 @@ source "drivers/phy/motorola/Kconfig"
 source "drivers/phy/mscc/Kconfig"
 source "drivers/phy/qualcomm/Kconfig"
 source "drivers/phy/ralink/Kconfig"
-source "drivers/phy/realtek/Kconfig"
 source "drivers/phy/renesas/Kconfig"
 source "drivers/phy/rockchip/Kconfig"
 source "drivers/phy/samsung/Kconfig"
diff --git a/drivers/phy/Makefile b/drivers/phy/Makefile
index 868a220ed0f6..fb3dc9de6111 100644
--- a/drivers/phy/Makefile
+++ b/drivers/phy/Makefile
@@ -26,7 +26,6 @@ obj-y					+= allwinner/	\
 					   mscc/	\
 					   qualcomm/	\
 					   ralink/	\
-					   realtek/	\
 					   renesas/	\
 					   rockchip/	\
 					   samsung/	\
diff --git a/drivers/phy/phy-core.c b/drivers/phy/phy-core.c
index 96a0b1e111f3..d9be6a4d5383 100644
--- a/drivers/phy/phy-core.c
+++ b/drivers/phy/phy-core.c
@@ -959,7 +959,7 @@ struct phy *phy_create(struct device *dev, struct device_node *node,
 	if (!phy)
 		return ERR_PTR(-ENOMEM);
 
-	id = ida_simple_get(&phy_ida, 0, 0, GFP_KERNEL);
+	id = ida_alloc(&phy_ida, GFP_KERNEL);
 	if (id < 0) {
 		dev_err(dev, "unable to get id\n");
 		ret = id;
@@ -1232,7 +1232,7 @@ static void phy_release(struct device *dev)
 	dev_vdbg(dev, "releasing '%s'\n", dev_name(dev));
 	debugfs_remove_recursive(phy->debugfs);
 	regulator_put(phy->pwr);
-	ida_simple_remove(&phy_ida, phy->id);
+	ida_free(&phy_ida, phy->id);
 	kfree(phy);
 }
 
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-combo.c b/drivers/phy/qualcomm/phy-qcom-qmp-combo.c
index 9c87845c78ec..0417856b8e7b 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp-combo.c
+++ b/drivers/phy/qualcomm/phy-qcom-qmp-combo.c
@@ -3558,6 +3558,10 @@ static const struct of_device_id qmp_combo_of_match_table[] = {
 		.compatible = "qcom,sm8550-qmp-usb3-dp-phy",
 		.data = &sm8550_usb3dpphy_cfg,
 	},
+	{
+		.compatible = "qcom,sm8650-qmp-usb3-dp-phy",
+		.data = &sm8550_usb3dpphy_cfg,
+	},
 	{ }
 };
 MODULE_DEVICE_TABLE(of, qmp_combo_of_match_table);
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c b/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c
index b64598ac59f4..2af7115ef968 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c
+++ b/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c
@@ -1909,6 +1909,35 @@ static const struct qmp_phy_init_tbl sm8550_qmp_gen4x2_pcie_pcs_misc_tbl[] = {
 	QMP_PHY_INIT_CFG(QPHY_PCIE_V6_20_PCS_G4_FOM_EQ_CONFIG5, 0xf2),
 };
 
+static const struct qmp_phy_init_tbl sm8650_qmp_gen4x2_pcie_rx_tbl[] = {
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_UCDR_FO_GAIN_RATE_2, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_UCDR_FO_GAIN_RATE_3, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_UCDR_PI_CONTROLS, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_UCDR_SO_ACC_DEFAULT_VAL_RATE3, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_IVCM_CAL_CTRL2, 0x82),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_DFE_3, 0x05),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_VGA_CAL_MAN_VAL, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_GM_CAL, 0x0d),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_EQU_ADAPTOR_CNTRL4, 0x0b),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_SIGDET_ENABLES, 0x1c),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_PHPRE_CTRL, 0x20),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_DFE_CTLE_POST_CAL_OFFSET, 0x38),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_MODE_RATE2_B0, 0xd3),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_MODE_RATE2_B1, 0xd3),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_MODE_RATE2_B2, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_MODE_RATE2_B3, 0x9a),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_MODE_RATE2_B4, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_MODE_RATE2_B5, 0xb6),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_MODE_RATE2_B6, 0xee),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_MODE_RATE3_B0, 0x23),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_MODE_RATE3_B1, 0x9b),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_MODE_RATE3_B2, 0x60),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_MODE_RATE3_B3, 0xdf),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_MODE_RATE3_B4, 0x43),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_MODE_RATE3_B5, 0x76),
+	QMP_PHY_INIT_CFG(QSERDES_V6_20_RX_MODE_RATE3_B6, 0xff),
+};
+
 static const struct qmp_phy_init_tbl sa8775p_qmp_gen4x2_pcie_serdes_alt_tbl[] = {
 	QMP_PHY_INIT_CFG(QSERDES_V5_COM_BIAS_EN_CLKBUFLR_EN, 0x14),
 	QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_IVCO, 0x0f),
@@ -3047,6 +3076,36 @@ static const struct qmp_phy_cfg sm8550_qmp_gen4x2_pciephy_cfg = {
 	.has_nocsr_reset	= true,
 };
 
+static const struct qmp_phy_cfg sm8650_qmp_gen4x2_pciephy_cfg = {
+	.lanes = 2,
+
+	.offsets		= &qmp_pcie_offsets_v6_20,
+
+	.tbls = {
+		.serdes			= sm8550_qmp_gen4x2_pcie_serdes_tbl,
+		.serdes_num		= ARRAY_SIZE(sm8550_qmp_gen4x2_pcie_serdes_tbl),
+		.tx			= sm8550_qmp_gen4x2_pcie_tx_tbl,
+		.tx_num			= ARRAY_SIZE(sm8550_qmp_gen4x2_pcie_tx_tbl),
+		.rx			= sm8650_qmp_gen4x2_pcie_rx_tbl,
+		.rx_num			= ARRAY_SIZE(sm8650_qmp_gen4x2_pcie_rx_tbl),
+		.pcs			= sm8550_qmp_gen4x2_pcie_pcs_tbl,
+		.pcs_num		= ARRAY_SIZE(sm8550_qmp_gen4x2_pcie_pcs_tbl),
+		.pcs_misc		= sm8550_qmp_gen4x2_pcie_pcs_misc_tbl,
+		.pcs_misc_num		= ARRAY_SIZE(sm8550_qmp_gen4x2_pcie_pcs_misc_tbl),
+		.ln_shrd		= sm8550_qmp_gen4x2_pcie_ln_shrd_tbl,
+		.ln_shrd_num		= ARRAY_SIZE(sm8550_qmp_gen4x2_pcie_ln_shrd_tbl),
+	},
+	.reset_list		= sdm845_pciephy_reset_l,
+	.num_resets		= ARRAY_SIZE(sdm845_pciephy_reset_l),
+	.vreg_list		= sm8550_qmp_phy_vreg_l,
+	.num_vregs		= ARRAY_SIZE(sm8550_qmp_phy_vreg_l),
+	.regs			= pciephy_v5_regs_layout,
+
+	.pwrdn_ctrl		= SW_PWRDN | REFCLK_DRV_DSBL,
+	.phy_status		= PHYSTATUS_4_20,
+	.has_nocsr_reset	= true,
+};
+
 static const struct qmp_phy_cfg sa8775p_qmp_gen4x2_pciephy_cfg = {
 	.lanes			= 2,
 	.offsets		= &qmp_pcie_offsets_v5_20,
@@ -3820,6 +3879,12 @@ static const struct of_device_id qmp_pcie_of_match_table[] = {
 	}, {
 		.compatible = "qcom,sm8550-qmp-gen4x2-pcie-phy",
 		.data = &sm8550_qmp_gen4x2_pciephy_cfg,
+	}, {
+		.compatible = "qcom,sm8650-qmp-gen3x2-pcie-phy",
+		.data = &sm8550_qmp_gen3x2_pciephy_cfg,
+	}, {
+		.compatible = "qcom,sm8650-qmp-gen4x2-pcie-phy",
+		.data = &sm8650_qmp_gen4x2_pciephy_cfg,
 	},
 	{ },
 };
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-pcs-ufs-v6.h b/drivers/phy/qualcomm/phy-qcom-qmp-pcs-ufs-v6.h
index c23d5e41e25b..fe6c450f6123 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp-pcs-ufs-v6.h
+++ b/drivers/phy/qualcomm/phy-qcom-qmp-pcs-ufs-v6.h
@@ -12,6 +12,7 @@
 #define QPHY_V6_PCS_UFS_SW_RESET			0x008
 #define QPHY_V6_PCS_UFS_TIMER_20US_CORECLK_STEPS_MSB	0x00c
 #define QPHY_V6_PCS_UFS_TIMER_20US_CORECLK_STEPS_LSB	0x010
+#define QPHY_V6_PCS_UFS_PCS_CTRL1			0x020
 #define QPHY_V6_PCS_UFS_PLL_CNTL			0x02c
 #define QPHY_V6_PCS_UFS_TX_LARGE_AMP_DRV_LVL		0x030
 #define QPHY_V6_PCS_UFS_TX_SMALL_AMP_DRV_LVL		0x038
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx-ufs-v6.h b/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx-ufs-v6.h
index 15bcb4ba9139..ae220fd04d10 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx-ufs-v6.h
+++ b/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx-ufs-v6.h
@@ -10,10 +10,17 @@
 #define QSERDES_UFS_V6_TX_RES_CODE_LANE_RX			0x2c
 #define QSERDES_UFS_V6_TX_RES_CODE_LANE_OFFSET_TX		0x30
 #define QSERDES_UFS_V6_TX_RES_CODE_LANE_OFFSET_RX		0x34
+#define QSERDES_UFS_V6_TX_LANE_MODE_1				0x7c
 
 #define QSERDES_UFS_V6_RX_UCDR_FASTLOCK_FO_GAIN_RATE2		0x08
 #define QSERDES_UFS_V6_RX_UCDR_FASTLOCK_FO_GAIN_RATE4		0x10
+#define QSERDES_UFS_V6_RX_UCDR_SO_SATURATION			0x28
+#define QSERDES_UFS_V6_RX_UCDR_PI_CTRL1				0x58
+#define QSERDES_UFS_V6_RX_RX_TERM_BW_CTRL0			0xc4
+#define QSERDES_UFS_V6_RX_UCDR_FO_GAIN_RATE2			0xd4
+#define QSERDES_UFS_V6_RX_UCDR_FO_GAIN_RATE4			0xdc
 #define QSERDES_UFS_V6_RX_VGA_CAL_MAN_VAL			0x178
+#define QSERDES_UFS_V6_RX_INTERFACE_MODE			0x1e0
 #define QSERDES_UFS_V6_RX_MODE_RATE_0_1_B0			0x208
 #define QSERDES_UFS_V6_RX_MODE_RATE_0_1_B1			0x20c
 #define QSERDES_UFS_V6_RX_MODE_RATE_0_1_B3			0x214
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c b/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c
index 514fa14df634..5f79d188b435 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c
+++ b/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c
@@ -803,6 +803,67 @@ static const struct qmp_phy_init_tbl sm8550_ufsphy_pcs[] = {
 	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_MULTI_LANE_CTRL1, 0x02),
 };
 
+static const struct qmp_phy_init_tbl sm8650_ufsphy_serdes[] = {
+	QMP_PHY_INIT_CFG(QSERDES_V6_COM_SYSCLK_EN_SEL, 0xd9),
+	QMP_PHY_INIT_CFG(QSERDES_V6_COM_CMN_CONFIG_1, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V6_COM_HSCLK_SEL_1, 0x11),
+	QMP_PHY_INIT_CFG(QSERDES_V6_COM_HSCLK_HS_SWITCH_SEL_1, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V6_COM_LOCK_CMP_EN, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V6_COM_PLL_IVCO, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V6_COM_VCO_TUNE_MAP, 0x44),
+	QMP_PHY_INIT_CFG(QSERDES_V6_COM_VCO_TUNE_INITVAL2, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V6_COM_DEC_START_MODE0, 0x41),
+	QMP_PHY_INIT_CFG(QSERDES_V6_COM_CP_CTRL_MODE0, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V6_COM_PLL_RCTRL_MODE0, 0x18),
+	QMP_PHY_INIT_CFG(QSERDES_V6_COM_PLL_CCTRL_MODE0, 0x14),
+	QMP_PHY_INIT_CFG(QSERDES_V6_COM_LOCK_CMP1_MODE0, 0x7f),
+	QMP_PHY_INIT_CFG(QSERDES_V6_COM_LOCK_CMP2_MODE0, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V6_COM_DEC_START_MODE1, 0x4c),
+	QMP_PHY_INIT_CFG(QSERDES_V6_COM_CP_CTRL_MODE1, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V6_COM_PLL_RCTRL_MODE1, 0x18),
+	QMP_PHY_INIT_CFG(QSERDES_V6_COM_PLL_CCTRL_MODE1, 0x14),
+	QMP_PHY_INIT_CFG(QSERDES_V6_COM_LOCK_CMP1_MODE1, 0x99),
+	QMP_PHY_INIT_CFG(QSERDES_V6_COM_LOCK_CMP2_MODE1, 0x07),
+};
+
+static const struct qmp_phy_init_tbl sm8650_ufsphy_tx[] = {
+	QMP_PHY_INIT_CFG(QSERDES_UFS_V6_TX_LANE_MODE_1, 0x05),
+	QMP_PHY_INIT_CFG(QSERDES_UFS_V6_TX_RES_CODE_LANE_OFFSET_TX, 0x07),
+};
+
+static const struct qmp_phy_init_tbl sm8650_ufsphy_rx[] = {
+	QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_UCDR_FO_GAIN_RATE2, 0x0c),
+	QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_UCDR_FO_GAIN_RATE4, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_VGA_CAL_MAN_VAL, 0x0e),
+	QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE_0_1_B0, 0xc2),
+	QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE_0_1_B1, 0xc2),
+	QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE_0_1_B3, 0x1a),
+	QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE_0_1_B6, 0x60),
+	QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE2_B3, 0x9e),
+	QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE2_B6, 0x60),
+	QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE3_B3, 0x9e),
+	QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE3_B4, 0x0e),
+	QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE3_B5, 0x36),
+	QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE3_B8, 0x02),
+	QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE4_B3, 0xb9),
+	QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_MODE_RATE4_B6, 0xff),
+	QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_UCDR_SO_SATURATION, 0x1f),
+	QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_UCDR_PI_CTRL1, 0x94),
+	QMP_PHY_INIT_CFG(QSERDES_UFS_V6_RX_RX_TERM_BW_CTRL0, 0xfa),
+};
+
+static const struct qmp_phy_init_tbl sm8650_ufsphy_pcs[] = {
+	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_MULTI_LANE_CTRL1, 0x00),
+	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_MID_TERM_CTRL1, 0x43),
+	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PCS_CTRL1, 0xc1),
+	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PLL_CNTL, 0x33),
+	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_HSGEAR_CAPABILITY, 0x04),
+	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_HSGEAR_CAPABILITY, 0x04),
+	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_LARGE_AMP_DRV_LVL, 0x0f),
+	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_SIGDET_CTRL2, 0x69),
+	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_MULTI_LANE_CTRL1, 0x02),
+};
+
 struct qmp_ufs_offsets {
 	u16 serdes;
 	u16 pcs;
@@ -1303,6 +1364,28 @@ static const struct qmp_phy_cfg sm8550_ufsphy_cfg = {
 	.regs			= ufsphy_v6_regs_layout,
 };
 
+static const struct qmp_phy_cfg sm8650_ufsphy_cfg = {
+	.lanes			= 2,
+
+	.offsets		= &qmp_ufs_offsets_v6,
+
+	.tbls = {
+		.serdes		= sm8650_ufsphy_serdes,
+		.serdes_num	= ARRAY_SIZE(sm8650_ufsphy_serdes),
+		.tx		= sm8650_ufsphy_tx,
+		.tx_num		= ARRAY_SIZE(sm8650_ufsphy_tx),
+		.rx		= sm8650_ufsphy_rx,
+		.rx_num		= ARRAY_SIZE(sm8650_ufsphy_rx),
+		.pcs		= sm8650_ufsphy_pcs,
+		.pcs_num	= ARRAY_SIZE(sm8650_ufsphy_pcs),
+	},
+	.clk_list		= sdm845_ufs_phy_clk_l,
+	.num_clks		= ARRAY_SIZE(sdm845_ufs_phy_clk_l),
+	.vreg_list		= qmp_phy_vreg_l,
+	.num_vregs		= ARRAY_SIZE(qmp_phy_vreg_l),
+	.regs			= ufsphy_v6_regs_layout,
+};
+
 static void qmp_ufs_configure_lane(void __iomem *base,
 					const struct qmp_phy_init_tbl tbl[],
 					int num,
@@ -1826,6 +1909,9 @@ static const struct of_device_id qmp_ufs_of_match_table[] = {
 	}, {
 		.compatible = "qcom,sm8550-qmp-ufs-phy",
 		.data = &sm8550_ufsphy_cfg,
+	}, {
+		.compatible = "qcom,sm8650-qmp-ufs-phy",
+		.data = &sm8650_ufsphy_cfg,
 	},
 	{ },
 };
diff --git a/drivers/phy/realtek/Kconfig b/drivers/phy/realtek/Kconfig
deleted file mode 100644
index 75ac7e7c31ae..000000000000
--- a/drivers/phy/realtek/Kconfig
+++ /dev/null
@@ -1,32 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-#
-# Phy drivers for Realtek platforms
-#
-
-if ARCH_REALTEK || COMPILE_TEST
-
-config PHY_RTK_RTD_USB2PHY
-	tristate "Realtek RTD USB2 PHY Transceiver Driver"
-	depends on USB_SUPPORT
-	select GENERIC_PHY
-	select USB_PHY
-	select USB_COMMON
-	help
-	  Enable this to support Realtek SoC USB2 phy transceiver.
-	  The DHC (digital home center) RTD series SoCs used the Synopsys
-	  DWC3 USB IP. This driver will do the PHY initialization
-	  of the parameters.
-
-config PHY_RTK_RTD_USB3PHY
-	tristate "Realtek RTD USB3 PHY Transceiver Driver"
-	depends on USB_SUPPORT
-	select GENERIC_PHY
-	select USB_PHY
-	select USB_COMMON
-	help
-	  Enable this to support Realtek SoC USB3 phy transceiver.
-	  The DHC (digital home center) RTD series SoCs used the Synopsys
-	  DWC3 USB IP. This driver will do the PHY initialization
-	  of the parameters.
-
-endif # ARCH_REALTEK || COMPILE_TEST
diff --git a/drivers/phy/realtek/Makefile b/drivers/phy/realtek/Makefile
deleted file mode 100644
index ed7b47ff8a26..000000000000
--- a/drivers/phy/realtek/Makefile
+++ /dev/null
@@ -1,3 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_PHY_RTK_RTD_USB2PHY)	+= phy-rtk-usb2.o
-obj-$(CONFIG_PHY_RTK_RTD_USB3PHY)	+= phy-rtk-usb3.o
diff --git a/drivers/phy/realtek/phy-rtk-usb2.c b/drivers/phy/realtek/phy-rtk-usb2.c
deleted file mode 100644
index 0a6426285c67..000000000000
--- a/drivers/phy/realtek/phy-rtk-usb2.c
+++ /dev/null
@@ -1,1325 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- *  phy-rtk-usb2.c RTK usb2.0 PHY driver
- *
- * Copyright (C) 2023 Realtek Semiconductor Corporation
- *
- */
-
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/of_address.h>
-#include <linux/platform_device.h>
-#include <linux/uaccess.h>
-#include <linux/debugfs.h>
-#include <linux/nvmem-consumer.h>
-#include <linux/regmap.h>
-#include <linux/sys_soc.h>
-#include <linux/mfd/syscon.h>
-#include <linux/phy/phy.h>
-#include <linux/usb.h>
-#include <linux/usb/phy.h>
-#include <linux/usb/hcd.h>
-
-/* GUSB2PHYACCn register */
-#define PHY_NEW_REG_REQ BIT(25)
-#define PHY_VSTS_BUSY   BIT(23)
-#define PHY_VCTRL_SHIFT 8
-#define PHY_REG_DATA_MASK 0xff
-
-#define GET_LOW_NIBBLE(addr) ((addr) & 0x0f)
-#define GET_HIGH_NIBBLE(addr) (((addr) & 0xf0) >> 4)
-
-#define EFUS_USB_DC_CAL_RATE 2
-#define EFUS_USB_DC_CAL_MAX 7
-
-#define EFUS_USB_DC_DIS_RATE 1
-#define EFUS_USB_DC_DIS_MAX 7
-
-#define MAX_PHY_DATA_SIZE 20
-#define OFFEST_PHY_READ 0x20
-
-#define MAX_USB_PHY_NUM 4
-#define MAX_USB_PHY_PAGE0_DATA_SIZE 16
-#define MAX_USB_PHY_PAGE1_DATA_SIZE 16
-#define MAX_USB_PHY_PAGE2_DATA_SIZE 8
-
-#define SET_PAGE_OFFSET 0xf4
-#define SET_PAGE_0 0x9b
-#define SET_PAGE_1 0xbb
-#define SET_PAGE_2 0xdb
-
-#define PAGE_START 0xe0
-#define PAGE0_0XE4 0xe4
-#define PAGE0_0XE6 0xe6
-#define PAGE0_0XE7 0xe7
-#define PAGE1_0XE0 0xe0
-#define PAGE1_0XE2 0xe2
-
-#define SENSITIVITY_CTRL (BIT(4) | BIT(5) | BIT(6))
-#define ENABLE_AUTO_SENSITIVITY_CALIBRATION BIT(2)
-#define DEFAULT_DC_DRIVING_VALUE (0x8)
-#define DEFAULT_DC_DISCONNECTION_VALUE (0x6)
-#define HS_CLK_SELECT BIT(6)
-
-struct phy_reg {
-	void __iomem *reg_wrap_vstatus;
-	void __iomem *reg_gusb2phyacc0;
-	int vstatus_index;
-};
-
-struct phy_data {
-	u8 addr;
-	u8 data;
-};
-
-struct phy_cfg {
-	int page0_size;
-	struct phy_data page0[MAX_USB_PHY_PAGE0_DATA_SIZE];
-	int page1_size;
-	struct phy_data page1[MAX_USB_PHY_PAGE1_DATA_SIZE];
-	int page2_size;
-	struct phy_data page2[MAX_USB_PHY_PAGE2_DATA_SIZE];
-
-	int num_phy;
-
-	bool check_efuse;
-	int check_efuse_version;
-#define CHECK_EFUSE_V1 1
-#define CHECK_EFUSE_V2 2
-	int efuse_dc_driving_rate;
-	int efuse_dc_disconnect_rate;
-	int dc_driving_mask;
-	int dc_disconnect_mask;
-	bool usb_dc_disconnect_at_page0;
-	int driving_updated_for_dev_dis;
-
-	bool do_toggle;
-	bool do_toggle_driving;
-	bool use_default_parameter;
-	bool is_double_sensitivity_mode;
-};
-
-struct phy_parameter {
-	struct phy_reg phy_reg;
-
-	/* Get from efuse */
-	s8 efuse_usb_dc_cal;
-	s8 efuse_usb_dc_dis;
-
-	/* Get from dts */
-	bool inverse_hstx_sync_clock;
-	u32 driving_level;
-	s32 driving_level_compensate;
-	s32 disconnection_compensate;
-};
-
-struct rtk_phy {
-	struct usb_phy phy;
-	struct device *dev;
-
-	struct phy_cfg *phy_cfg;
-	int num_phy;
-	struct phy_parameter *phy_parameter;
-
-	struct dentry *debug_dir;
-};
-
-/* mapping 0xE0 to 0 ... 0xE7 to 7, 0xF0 to 8 ,,, 0xF7 to 15 */
-static inline int page_addr_to_array_index(u8 addr)
-{
-	return (int)((((addr) - PAGE_START) & 0x7) +
-		((((addr) - PAGE_START) & 0x10) >> 1));
-}
-
-static inline u8 array_index_to_page_addr(int index)
-{
-	return ((((index) + PAGE_START) & 0x7) +
-		((((index) & 0x8) << 1) + PAGE_START));
-}
-
-#define PHY_IO_TIMEOUT_USEC		(50000)
-#define PHY_IO_DELAY_US			(100)
-
-static inline int utmi_wait_register(void __iomem *reg, u32 mask, u32 result)
-{
-	int ret;
-	unsigned int val;
-
-	ret = read_poll_timeout(readl, val, ((val & mask) == result),
-				PHY_IO_DELAY_US, PHY_IO_TIMEOUT_USEC, false, reg);
-	if (ret) {
-		pr_err("%s can't program USB phy\n", __func__);
-		return -ETIMEDOUT;
-	}
-
-	return 0;
-}
-
-static char rtk_phy_read(struct phy_reg *phy_reg, char addr)
-{
-	void __iomem *reg_gusb2phyacc0 = phy_reg->reg_gusb2phyacc0;
-	unsigned int val;
-	int ret = 0;
-
-	addr -= OFFEST_PHY_READ;
-
-	/* polling until VBusy == 0 */
-	ret = utmi_wait_register(reg_gusb2phyacc0, PHY_VSTS_BUSY, 0);
-	if (ret)
-		return (char)ret;
-
-	/* VCtrl = low nibble of addr, and set PHY_NEW_REG_REQ */
-	val = PHY_NEW_REG_REQ | (GET_LOW_NIBBLE(addr) << PHY_VCTRL_SHIFT);
-	writel(val, reg_gusb2phyacc0);
-	ret = utmi_wait_register(reg_gusb2phyacc0, PHY_VSTS_BUSY, 0);
-	if (ret)
-		return (char)ret;
-
-	/* VCtrl = high nibble of addr, and set PHY_NEW_REG_REQ */
-	val = PHY_NEW_REG_REQ | (GET_HIGH_NIBBLE(addr) << PHY_VCTRL_SHIFT);
-	writel(val, reg_gusb2phyacc0);
-	ret = utmi_wait_register(reg_gusb2phyacc0, PHY_VSTS_BUSY, 0);
-	if (ret)
-		return (char)ret;
-
-	val = readl(reg_gusb2phyacc0);
-
-	return (char)(val & PHY_REG_DATA_MASK);
-}
-
-static int rtk_phy_write(struct phy_reg *phy_reg, char addr, char data)
-{
-	unsigned int val;
-	void __iomem *reg_wrap_vstatus = phy_reg->reg_wrap_vstatus;
-	void __iomem *reg_gusb2phyacc0 = phy_reg->reg_gusb2phyacc0;
-	int shift_bits = phy_reg->vstatus_index * 8;
-	int ret = 0;
-
-	/* write data to VStatusOut2 (data output to phy) */
-	writel((u32)data << shift_bits, reg_wrap_vstatus);
-
-	ret = utmi_wait_register(reg_gusb2phyacc0, PHY_VSTS_BUSY, 0);
-	if (ret)
-		return ret;
-
-	/* VCtrl = low nibble of addr, set PHY_NEW_REG_REQ */
-	val = PHY_NEW_REG_REQ | (GET_LOW_NIBBLE(addr) << PHY_VCTRL_SHIFT);
-
-	writel(val, reg_gusb2phyacc0);
-	ret = utmi_wait_register(reg_gusb2phyacc0, PHY_VSTS_BUSY, 0);
-	if (ret)
-		return ret;
-
-	/* VCtrl = high nibble of addr, set PHY_NEW_REG_REQ */
-	val = PHY_NEW_REG_REQ | (GET_HIGH_NIBBLE(addr) << PHY_VCTRL_SHIFT);
-
-	writel(val, reg_gusb2phyacc0);
-	ret = utmi_wait_register(reg_gusb2phyacc0, PHY_VSTS_BUSY, 0);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static int rtk_phy_set_page(struct phy_reg *phy_reg, int page)
-{
-	switch (page) {
-	case 0:
-		return rtk_phy_write(phy_reg, SET_PAGE_OFFSET, SET_PAGE_0);
-	case 1:
-		return rtk_phy_write(phy_reg, SET_PAGE_OFFSET, SET_PAGE_1);
-	case 2:
-		return rtk_phy_write(phy_reg, SET_PAGE_OFFSET, SET_PAGE_2);
-	default:
-		pr_err("%s error page=%d\n", __func__, page);
-	}
-
-	return -EINVAL;
-}
-
-static u8 __updated_dc_disconnect_level_page0_0xe4(struct phy_cfg *phy_cfg,
-						   struct phy_parameter *phy_parameter, u8 data)
-{
-	u8 ret;
-	s32 val;
-	s32 dc_disconnect_mask = phy_cfg->dc_disconnect_mask;
-	int offset = 4;
-
-	val = (s32)((data >> offset) & dc_disconnect_mask)
-		     + phy_parameter->efuse_usb_dc_dis
-		     + phy_parameter->disconnection_compensate;
-
-	if (val > dc_disconnect_mask)
-		val = dc_disconnect_mask;
-	else if (val < 0)
-		val = 0;
-
-	ret = (data & (~(dc_disconnect_mask << offset))) |
-		    (val & dc_disconnect_mask) << offset;
-
-	return ret;
-}
-
-/* updated disconnect level at page0 */
-static void update_dc_disconnect_level_at_page0(struct rtk_phy *rtk_phy,
-						struct phy_parameter *phy_parameter, bool update)
-{
-	struct phy_cfg *phy_cfg;
-	struct phy_reg *phy_reg;
-	struct phy_data *phy_data_page;
-	struct phy_data *phy_data;
-	u8 addr, data;
-	int offset = 4;
-	s32 dc_disconnect_mask;
-	int i;
-
-	phy_cfg = rtk_phy->phy_cfg;
-	phy_reg = &phy_parameter->phy_reg;
-
-	/* Set page 0 */
-	phy_data_page = phy_cfg->page0;
-	rtk_phy_set_page(phy_reg, 0);
-
-	i = page_addr_to_array_index(PAGE0_0XE4);
-	phy_data = phy_data_page + i;
-	if (!phy_data->addr) {
-		phy_data->addr = PAGE0_0XE4;
-		phy_data->data = rtk_phy_read(phy_reg, PAGE0_0XE4);
-	}
-
-	addr = phy_data->addr;
-	data = phy_data->data;
-	dc_disconnect_mask = phy_cfg->dc_disconnect_mask;
-
-	if (update)
-		data = __updated_dc_disconnect_level_page0_0xe4(phy_cfg, phy_parameter, data);
-	else
-		data = (data & ~(dc_disconnect_mask << offset)) |
-			(DEFAULT_DC_DISCONNECTION_VALUE << offset);
-
-	if (rtk_phy_write(phy_reg, addr, data))
-		dev_err(rtk_phy->dev,
-			"%s: Error to set page1 parameter addr=0x%x value=0x%x\n",
-			__func__, addr, data);
-}
-
-static u8 __updated_dc_disconnect_level_page1_0xe2(struct phy_cfg *phy_cfg,
-						   struct phy_parameter *phy_parameter, u8 data)
-{
-	u8 ret;
-	s32 val;
-	s32 dc_disconnect_mask = phy_cfg->dc_disconnect_mask;
-
-	if (phy_cfg->check_efuse_version == CHECK_EFUSE_V1) {
-		val = (s32)(data & dc_disconnect_mask)
-			    + phy_parameter->efuse_usb_dc_dis
-			    + phy_parameter->disconnection_compensate;
-	} else { /* for CHECK_EFUSE_V2 or no efuse */
-		if (phy_parameter->efuse_usb_dc_dis)
-			val = (s32)(phy_parameter->efuse_usb_dc_dis +
-				    phy_parameter->disconnection_compensate);
-		else
-			val = (s32)((data & dc_disconnect_mask) +
-				    phy_parameter->disconnection_compensate);
-	}
-
-	if (val > dc_disconnect_mask)
-		val = dc_disconnect_mask;
-	else if (val < 0)
-		val = 0;
-
-	ret = (data & (~dc_disconnect_mask)) | (val & dc_disconnect_mask);
-
-	return ret;
-}
-
-/* updated disconnect level at page1 */
-static void update_dc_disconnect_level_at_page1(struct rtk_phy *rtk_phy,
-						struct phy_parameter *phy_parameter, bool update)
-{
-	struct phy_cfg *phy_cfg;
-	struct phy_data *phy_data_page;
-	struct phy_data *phy_data;
-	struct phy_reg *phy_reg;
-	u8 addr, data;
-	s32 dc_disconnect_mask;
-	int i;
-
-	phy_cfg = rtk_phy->phy_cfg;
-	phy_reg = &phy_parameter->phy_reg;
-
-	/* Set page 1 */
-	phy_data_page = phy_cfg->page1;
-	rtk_phy_set_page(phy_reg, 1);
-
-	i = page_addr_to_array_index(PAGE1_0XE2);
-	phy_data = phy_data_page + i;
-	if (!phy_data->addr) {
-		phy_data->addr = PAGE1_0XE2;
-		phy_data->data = rtk_phy_read(phy_reg, PAGE1_0XE2);
-	}
-
-	addr = phy_data->addr;
-	data = phy_data->data;
-	dc_disconnect_mask = phy_cfg->dc_disconnect_mask;
-
-	if (update)
-		data = __updated_dc_disconnect_level_page1_0xe2(phy_cfg, phy_parameter, data);
-	else
-		data = (data & ~dc_disconnect_mask) | DEFAULT_DC_DISCONNECTION_VALUE;
-
-	if (rtk_phy_write(phy_reg, addr, data))
-		dev_err(rtk_phy->dev,
-			"%s: Error to set page1 parameter addr=0x%x value=0x%x\n",
-			__func__, addr, data);
-}
-
-static void update_dc_disconnect_level(struct rtk_phy *rtk_phy,
-				       struct phy_parameter *phy_parameter, bool update)
-{
-	struct phy_cfg *phy_cfg = rtk_phy->phy_cfg;
-
-	if (phy_cfg->usb_dc_disconnect_at_page0)
-		update_dc_disconnect_level_at_page0(rtk_phy, phy_parameter, update);
-	else
-		update_dc_disconnect_level_at_page1(rtk_phy, phy_parameter, update);
-}
-
-static u8 __update_dc_driving_page0_0xe4(struct phy_cfg *phy_cfg,
-					 struct phy_parameter *phy_parameter, u8 data)
-{
-	s32 driving_level_compensate = phy_parameter->driving_level_compensate;
-	s32 dc_driving_mask = phy_cfg->dc_driving_mask;
-	s32 val;
-	u8 ret;
-
-	if (phy_cfg->check_efuse_version == CHECK_EFUSE_V1) {
-		val = (s32)(data & dc_driving_mask) + driving_level_compensate
-			    + phy_parameter->efuse_usb_dc_cal;
-	} else { /* for CHECK_EFUSE_V2 or no efuse */
-		if (phy_parameter->efuse_usb_dc_cal)
-			val = (s32)((phy_parameter->efuse_usb_dc_cal & dc_driving_mask)
-				    + driving_level_compensate);
-		else
-			val = (s32)(data & dc_driving_mask);
-	}
-
-	if (val > dc_driving_mask)
-		val = dc_driving_mask;
-	else if (val < 0)
-		val = 0;
-
-	ret = (data & (~dc_driving_mask)) | (val & dc_driving_mask);
-
-	return ret;
-}
-
-static void update_dc_driving_level(struct rtk_phy *rtk_phy,
-				    struct phy_parameter *phy_parameter)
-{
-	struct phy_cfg *phy_cfg;
-	struct phy_reg *phy_reg;
-
-	phy_reg = &phy_parameter->phy_reg;
-	phy_cfg = rtk_phy->phy_cfg;
-	if (!phy_cfg->page0[4].addr) {
-		rtk_phy_set_page(phy_reg, 0);
-		phy_cfg->page0[4].addr = PAGE0_0XE4;
-		phy_cfg->page0[4].data = rtk_phy_read(phy_reg, PAGE0_0XE4);
-	}
-
-	if (phy_parameter->driving_level != DEFAULT_DC_DRIVING_VALUE) {
-		u32 dc_driving_mask;
-		u8 driving_level;
-		u8 data;
-
-		data = phy_cfg->page0[4].data;
-		dc_driving_mask = phy_cfg->dc_driving_mask;
-		driving_level = data & dc_driving_mask;
-
-		dev_dbg(rtk_phy->dev, "%s driving_level=%d => dts driving_level=%d\n",
-			__func__, driving_level, phy_parameter->driving_level);
-
-		phy_cfg->page0[4].data = (data & (~dc_driving_mask)) |
-			    (phy_parameter->driving_level & dc_driving_mask);
-	}
-
-	phy_cfg->page0[4].data = __update_dc_driving_page0_0xe4(phy_cfg,
-								phy_parameter,
-								phy_cfg->page0[4].data);
-}
-
-static void update_hs_clk_select(struct rtk_phy *rtk_phy,
-				 struct phy_parameter *phy_parameter)
-{
-	struct phy_cfg *phy_cfg;
-	struct phy_reg *phy_reg;
-
-	phy_cfg = rtk_phy->phy_cfg;
-	phy_reg = &phy_parameter->phy_reg;
-
-	if (phy_parameter->inverse_hstx_sync_clock) {
-		if (!phy_cfg->page0[6].addr) {
-			rtk_phy_set_page(phy_reg, 0);
-			phy_cfg->page0[6].addr = PAGE0_0XE6;
-			phy_cfg->page0[6].data = rtk_phy_read(phy_reg, PAGE0_0XE6);
-		}
-
-		phy_cfg->page0[6].data = phy_cfg->page0[6].data | HS_CLK_SELECT;
-	}
-}
-
-static void do_rtk_phy_toggle(struct rtk_phy *rtk_phy,
-			      int index, bool connect)
-{
-	struct phy_parameter *phy_parameter;
-	struct phy_cfg *phy_cfg;
-	struct phy_reg *phy_reg;
-	struct phy_data *phy_data_page;
-	u8 addr, data;
-	int i;
-
-	phy_cfg = rtk_phy->phy_cfg;
-	phy_parameter = &((struct phy_parameter *)rtk_phy->phy_parameter)[index];
-	phy_reg = &phy_parameter->phy_reg;
-
-	if (!phy_cfg->do_toggle)
-		goto out;
-
-	if (phy_cfg->is_double_sensitivity_mode)
-		goto do_toggle_driving;
-
-	/* Set page 0 */
-	rtk_phy_set_page(phy_reg, 0);
-
-	addr = PAGE0_0XE7;
-	data = rtk_phy_read(phy_reg, addr);
-
-	if (connect)
-		rtk_phy_write(phy_reg, addr, data & (~SENSITIVITY_CTRL));
-	else
-		rtk_phy_write(phy_reg, addr, data | (SENSITIVITY_CTRL));
-
-do_toggle_driving:
-
-	if (!phy_cfg->do_toggle_driving)
-		goto do_toggle;
-
-	/* Page 0 addr 0xE4 driving capability */
-
-	/* Set page 0 */
-	phy_data_page = phy_cfg->page0;
-	rtk_phy_set_page(phy_reg, 0);
-
-	i = page_addr_to_array_index(PAGE0_0XE4);
-	addr = phy_data_page[i].addr;
-	data = phy_data_page[i].data;
-
-	if (connect) {
-		rtk_phy_write(phy_reg, addr, data);
-	} else {
-		u8 value;
-		s32 tmp;
-		s32 driving_updated =
-			    phy_cfg->driving_updated_for_dev_dis;
-		s32 dc_driving_mask = phy_cfg->dc_driving_mask;
-
-		tmp = (s32)(data & dc_driving_mask) + driving_updated;
-
-		if (tmp > dc_driving_mask)
-			tmp = dc_driving_mask;
-		else if (tmp < 0)
-			tmp = 0;
-
-		value = (data & (~dc_driving_mask)) | (tmp & dc_driving_mask);
-
-		rtk_phy_write(phy_reg, addr, value);
-	}
-
-do_toggle:
-	/* restore dc disconnect level before toggle */
-	update_dc_disconnect_level(rtk_phy, phy_parameter, false);
-
-	/* Set page 1 */
-	rtk_phy_set_page(phy_reg, 1);
-
-	addr = PAGE1_0XE0;
-	data = rtk_phy_read(phy_reg, addr);
-
-	rtk_phy_write(phy_reg, addr, data &
-		      (~ENABLE_AUTO_SENSITIVITY_CALIBRATION));
-	mdelay(1);
-	rtk_phy_write(phy_reg, addr, data |
-		      (ENABLE_AUTO_SENSITIVITY_CALIBRATION));
-
-	/* update dc disconnect level after toggle */
-	update_dc_disconnect_level(rtk_phy, phy_parameter, true);
-
-out:
-	return;
-}
-
-static int do_rtk_phy_init(struct rtk_phy *rtk_phy, int index)
-{
-	struct phy_parameter *phy_parameter;
-	struct phy_cfg *phy_cfg;
-	struct phy_data *phy_data_page;
-	struct phy_reg *phy_reg;
-	int i;
-
-	phy_cfg = rtk_phy->phy_cfg;
-	phy_parameter = &((struct phy_parameter *)rtk_phy->phy_parameter)[index];
-	phy_reg = &phy_parameter->phy_reg;
-
-	if (phy_cfg->use_default_parameter) {
-		dev_dbg(rtk_phy->dev, "%s phy#%d use default parameter\n",
-			__func__, index);
-		goto do_toggle;
-	}
-
-	/* Set page 0 */
-	phy_data_page = phy_cfg->page0;
-	rtk_phy_set_page(phy_reg, 0);
-
-	for (i = 0; i < phy_cfg->page0_size; i++) {
-		struct phy_data *phy_data = phy_data_page + i;
-		u8 addr = phy_data->addr;
-		u8 data = phy_data->data;
-
-		if (!addr)
-			continue;
-
-		if (rtk_phy_write(phy_reg, addr, data)) {
-			dev_err(rtk_phy->dev,
-				"%s: Error to set page0 parameter addr=0x%x value=0x%x\n",
-				__func__, addr, data);
-			return -EINVAL;
-		}
-	}
-
-	/* Set page 1 */
-	phy_data_page = phy_cfg->page1;
-	rtk_phy_set_page(phy_reg, 1);
-
-	for (i = 0; i < phy_cfg->page1_size; i++) {
-		struct phy_data *phy_data = phy_data_page + i;
-		u8 addr = phy_data->addr;
-		u8 data = phy_data->data;
-
-		if (!addr)
-			continue;
-
-		if (rtk_phy_write(phy_reg, addr, data)) {
-			dev_err(rtk_phy->dev,
-				"%s: Error to set page1 parameter addr=0x%x value=0x%x\n",
-				__func__, addr, data);
-			return -EINVAL;
-		}
-	}
-
-	if (phy_cfg->page2_size == 0)
-		goto do_toggle;
-
-	/* Set page 2 */
-	phy_data_page = phy_cfg->page2;
-	rtk_phy_set_page(phy_reg, 2);
-
-	for (i = 0; i < phy_cfg->page2_size; i++) {
-		struct phy_data *phy_data = phy_data_page + i;
-		u8 addr = phy_data->addr;
-		u8 data = phy_data->data;
-
-		if (!addr)
-			continue;
-
-		if (rtk_phy_write(phy_reg, addr, data)) {
-			dev_err(rtk_phy->dev,
-				"%s: Error to set page2 parameter addr=0x%x value=0x%x\n",
-				__func__, addr, data);
-			return -EINVAL;
-		}
-	}
-
-do_toggle:
-	do_rtk_phy_toggle(rtk_phy, index, false);
-
-	return 0;
-}
-
-static int rtk_phy_init(struct phy *phy)
-{
-	struct rtk_phy *rtk_phy = phy_get_drvdata(phy);
-	unsigned long phy_init_time = jiffies;
-	int i, ret = 0;
-
-	if (!rtk_phy)
-		return -EINVAL;
-
-	for (i = 0; i < rtk_phy->num_phy; i++)
-		ret = do_rtk_phy_init(rtk_phy, i);
-
-	dev_dbg(rtk_phy->dev, "Initialized RTK USB 2.0 PHY (take %dms)\n",
-		jiffies_to_msecs(jiffies - phy_init_time));
-	return ret;
-}
-
-static int rtk_phy_exit(struct phy *phy)
-{
-	return 0;
-}
-
-static const struct phy_ops ops = {
-	.init		= rtk_phy_init,
-	.exit		= rtk_phy_exit,
-	.owner		= THIS_MODULE,
-};
-
-static void rtk_phy_toggle(struct usb_phy *usb2_phy, bool connect, int port)
-{
-	int index = port;
-	struct rtk_phy *rtk_phy = NULL;
-
-	rtk_phy = dev_get_drvdata(usb2_phy->dev);
-
-	if (index > rtk_phy->num_phy) {
-		dev_err(rtk_phy->dev, "%s: The port=%d is not in usb phy (num_phy=%d)\n",
-			__func__, index, rtk_phy->num_phy);
-		return;
-	}
-
-	do_rtk_phy_toggle(rtk_phy, index, connect);
-}
-
-static int rtk_phy_notify_port_status(struct usb_phy *x, int port,
-				      u16 portstatus, u16 portchange)
-{
-	bool connect = false;
-
-	pr_debug("%s port=%d portstatus=0x%x portchange=0x%x\n",
-		 __func__, port, (int)portstatus, (int)portchange);
-	if (portstatus & USB_PORT_STAT_CONNECTION)
-		connect = true;
-
-	if (portchange & USB_PORT_STAT_C_CONNECTION)
-		rtk_phy_toggle(x, connect, port);
-
-	return 0;
-}
-
-#ifdef CONFIG_DEBUG_FS
-static struct dentry *create_phy_debug_root(void)
-{
-	struct dentry *phy_debug_root;
-
-	phy_debug_root = debugfs_lookup("phy", usb_debug_root);
-	if (!phy_debug_root)
-		phy_debug_root = debugfs_create_dir("phy", usb_debug_root);
-
-	return phy_debug_root;
-}
-
-static int rtk_usb2_parameter_show(struct seq_file *s, void *unused)
-{
-	struct rtk_phy *rtk_phy = s->private;
-	struct phy_cfg *phy_cfg;
-	int i, index;
-
-	phy_cfg = rtk_phy->phy_cfg;
-
-	seq_puts(s, "Property:\n");
-	seq_printf(s, "  check_efuse: %s\n",
-		   phy_cfg->check_efuse ? "Enable" : "Disable");
-	seq_printf(s, "  check_efuse_version: %d\n",
-		   phy_cfg->check_efuse_version);
-	seq_printf(s, "  efuse_dc_driving_rate: %d\n",
-		   phy_cfg->efuse_dc_driving_rate);
-	seq_printf(s, "  dc_driving_mask: 0x%x\n",
-		   phy_cfg->dc_driving_mask);
-	seq_printf(s, "  efuse_dc_disconnect_rate: %d\n",
-		   phy_cfg->efuse_dc_disconnect_rate);
-	seq_printf(s, "  dc_disconnect_mask: 0x%x\n",
-		   phy_cfg->dc_disconnect_mask);
-	seq_printf(s, "  usb_dc_disconnect_at_page0: %s\n",
-		   phy_cfg->usb_dc_disconnect_at_page0 ? "true" : "false");
-	seq_printf(s, "  do_toggle: %s\n",
-		   phy_cfg->do_toggle ? "Enable" : "Disable");
-	seq_printf(s, "  do_toggle_driving: %s\n",
-		   phy_cfg->do_toggle_driving ? "Enable" : "Disable");
-	seq_printf(s, "  driving_updated_for_dev_dis: 0x%x\n",
-		   phy_cfg->driving_updated_for_dev_dis);
-	seq_printf(s, "  use_default_parameter: %s\n",
-		   phy_cfg->use_default_parameter ? "Enable" : "Disable");
-	seq_printf(s, "  is_double_sensitivity_mode: %s\n",
-		   phy_cfg->is_double_sensitivity_mode ? "Enable" : "Disable");
-
-	for (index = 0; index < rtk_phy->num_phy; index++) {
-		struct phy_parameter *phy_parameter;
-		struct phy_reg *phy_reg;
-		struct phy_data *phy_data_page;
-
-		phy_parameter =  &((struct phy_parameter *)rtk_phy->phy_parameter)[index];
-		phy_reg = &phy_parameter->phy_reg;
-
-		seq_printf(s, "PHY %d:\n", index);
-
-		seq_puts(s, "Page 0:\n");
-		/* Set page 0 */
-		phy_data_page = phy_cfg->page0;
-		rtk_phy_set_page(phy_reg, 0);
-
-		for (i = 0; i < phy_cfg->page0_size; i++) {
-			struct phy_data *phy_data = phy_data_page + i;
-			u8 addr = array_index_to_page_addr(i);
-			u8 data = phy_data->data;
-			u8 value = rtk_phy_read(phy_reg, addr);
-
-			if (phy_data->addr)
-				seq_printf(s, "  Page 0: addr=0x%x data=0x%02x ==> read value=0x%02x\n",
-					   addr, data, value);
-			else
-				seq_printf(s, "  Page 0: addr=0x%x data=none ==> read value=0x%02x\n",
-					   addr, value);
-		}
-
-		seq_puts(s, "Page 1:\n");
-		/* Set page 1 */
-		phy_data_page = phy_cfg->page1;
-		rtk_phy_set_page(phy_reg, 1);
-
-		for (i = 0; i < phy_cfg->page1_size; i++) {
-			struct phy_data *phy_data = phy_data_page + i;
-			u8 addr = array_index_to_page_addr(i);
-			u8 data = phy_data->data;
-			u8 value = rtk_phy_read(phy_reg, addr);
-
-			if (phy_data->addr)
-				seq_printf(s, "  Page 1: addr=0x%x data=0x%02x ==> read value=0x%02x\n",
-					   addr, data, value);
-			else
-				seq_printf(s, "  Page 1: addr=0x%x data=none ==> read value=0x%02x\n",
-					   addr, value);
-		}
-
-		if (phy_cfg->page2_size == 0)
-			goto out;
-
-		seq_puts(s, "Page 2:\n");
-		/* Set page 2 */
-		phy_data_page = phy_cfg->page2;
-		rtk_phy_set_page(phy_reg, 2);
-
-		for (i = 0; i < phy_cfg->page2_size; i++) {
-			struct phy_data *phy_data = phy_data_page + i;
-			u8 addr = array_index_to_page_addr(i);
-			u8 data = phy_data->data;
-			u8 value = rtk_phy_read(phy_reg, addr);
-
-			if (phy_data->addr)
-				seq_printf(s, "  Page 2: addr=0x%x data=0x%02x ==> read value=0x%02x\n",
-					   addr, data, value);
-			else
-				seq_printf(s, "  Page 2: addr=0x%x data=none ==> read value=0x%02x\n",
-					   addr, value);
-		}
-
-out:
-		seq_puts(s, "PHY Property:\n");
-		seq_printf(s, "  efuse_usb_dc_cal: %d\n",
-			   (int)phy_parameter->efuse_usb_dc_cal);
-		seq_printf(s, "  efuse_usb_dc_dis: %d\n",
-			   (int)phy_parameter->efuse_usb_dc_dis);
-		seq_printf(s, "  inverse_hstx_sync_clock: %s\n",
-			   phy_parameter->inverse_hstx_sync_clock ? "Enable" : "Disable");
-		seq_printf(s, "  driving_level: %d\n",
-			   phy_parameter->driving_level);
-		seq_printf(s, "  driving_level_compensate: %d\n",
-			   phy_parameter->driving_level_compensate);
-		seq_printf(s, "  disconnection_compensate: %d\n",
-			   phy_parameter->disconnection_compensate);
-	}
-
-	return 0;
-}
-DEFINE_SHOW_ATTRIBUTE(rtk_usb2_parameter);
-
-static inline void create_debug_files(struct rtk_phy *rtk_phy)
-{
-	struct dentry *phy_debug_root = NULL;
-
-	phy_debug_root = create_phy_debug_root();
-	if (!phy_debug_root)
-		return;
-
-	rtk_phy->debug_dir = debugfs_create_dir(dev_name(rtk_phy->dev),
-						phy_debug_root);
-
-	debugfs_create_file("parameter", 0444, rtk_phy->debug_dir, rtk_phy,
-			    &rtk_usb2_parameter_fops);
-
-	return;
-}
-
-static inline void remove_debug_files(struct rtk_phy *rtk_phy)
-{
-	debugfs_remove_recursive(rtk_phy->debug_dir);
-}
-#else
-static inline void create_debug_files(struct rtk_phy *rtk_phy) { }
-static inline void remove_debug_files(struct rtk_phy *rtk_phy) { }
-#endif /* CONFIG_DEBUG_FS */
-
-static int get_phy_data_by_efuse(struct rtk_phy *rtk_phy,
-				 struct phy_parameter *phy_parameter, int index)
-{
-	struct phy_cfg *phy_cfg = rtk_phy->phy_cfg;
-	u8 value = 0;
-	struct nvmem_cell *cell;
-	struct soc_device_attribute rtk_soc_groot[] = {
-		    { .family = "Realtek Groot",},
-		    { /* empty */ } };
-
-	if (!phy_cfg->check_efuse)
-		goto out;
-
-	/* Read efuse for usb dc cal */
-	cell = nvmem_cell_get(rtk_phy->dev, "usb-dc-cal");
-	if (IS_ERR(cell)) {
-		dev_dbg(rtk_phy->dev, "%s no usb-dc-cal: %ld\n",
-			__func__, PTR_ERR(cell));
-	} else {
-		unsigned char *buf;
-		size_t buf_size;
-
-		buf = nvmem_cell_read(cell, &buf_size);
-		if (!IS_ERR(buf)) {
-			value = buf[0] & phy_cfg->dc_driving_mask;
-			kfree(buf);
-		}
-		nvmem_cell_put(cell);
-	}
-
-	if (phy_cfg->check_efuse_version == CHECK_EFUSE_V1) {
-		int rate = phy_cfg->efuse_dc_driving_rate;
-
-		if (value <= EFUS_USB_DC_CAL_MAX)
-			phy_parameter->efuse_usb_dc_cal = (int8_t)(value * rate);
-		else
-			phy_parameter->efuse_usb_dc_cal = -(int8_t)
-				    ((EFUS_USB_DC_CAL_MAX & value) * rate);
-
-		if (soc_device_match(rtk_soc_groot)) {
-			dev_dbg(rtk_phy->dev, "For groot IC we need a workaround to adjust efuse_usb_dc_cal\n");
-
-			/* We don't multiple dc_cal_rate=2 for positive dc cal compensate */
-			if (value <= EFUS_USB_DC_CAL_MAX)
-				phy_parameter->efuse_usb_dc_cal = (int8_t)(value);
-
-			/* We set max dc cal compensate is 0x8 if otp is 0x7 */
-			if (value == 0x7)
-				phy_parameter->efuse_usb_dc_cal = (int8_t)(value + 1);
-		}
-	} else { /* for CHECK_EFUSE_V2 */
-		phy_parameter->efuse_usb_dc_cal = value & phy_cfg->dc_driving_mask;
-	}
-
-	/* Read efuse for usb dc disconnect level */
-	value = 0;
-	cell = nvmem_cell_get(rtk_phy->dev, "usb-dc-dis");
-	if (IS_ERR(cell)) {
-		dev_dbg(rtk_phy->dev, "%s no usb-dc-dis: %ld\n",
-			__func__, PTR_ERR(cell));
-	} else {
-		unsigned char *buf;
-		size_t buf_size;
-
-		buf = nvmem_cell_read(cell, &buf_size);
-		if (!IS_ERR(buf)) {
-			value = buf[0] & phy_cfg->dc_disconnect_mask;
-			kfree(buf);
-		}
-		nvmem_cell_put(cell);
-	}
-
-	if (phy_cfg->check_efuse_version == CHECK_EFUSE_V1) {
-		int rate = phy_cfg->efuse_dc_disconnect_rate;
-
-		if (value <= EFUS_USB_DC_DIS_MAX)
-			phy_parameter->efuse_usb_dc_dis = (int8_t)(value * rate);
-		else
-			phy_parameter->efuse_usb_dc_dis = -(int8_t)
-				    ((EFUS_USB_DC_DIS_MAX & value) * rate);
-	} else { /* for CHECK_EFUSE_V2 */
-		phy_parameter->efuse_usb_dc_dis = value & phy_cfg->dc_disconnect_mask;
-	}
-
-out:
-	return 0;
-}
-
-static int parse_phy_data(struct rtk_phy *rtk_phy)
-{
-	struct device *dev = rtk_phy->dev;
-	struct device_node *np = dev->of_node;
-	struct phy_parameter *phy_parameter;
-	int ret = 0;
-	int index;
-
-	rtk_phy->phy_parameter = devm_kzalloc(dev, sizeof(struct phy_parameter) *
-						rtk_phy->num_phy, GFP_KERNEL);
-	if (!rtk_phy->phy_parameter)
-		return -ENOMEM;
-
-	for (index = 0; index < rtk_phy->num_phy; index++) {
-		phy_parameter = &((struct phy_parameter *)rtk_phy->phy_parameter)[index];
-
-		phy_parameter->phy_reg.reg_wrap_vstatus = of_iomap(np, 0);
-		phy_parameter->phy_reg.reg_gusb2phyacc0 = of_iomap(np, 1) + index;
-		phy_parameter->phy_reg.vstatus_index = index;
-
-		if (of_property_read_bool(np, "realtek,inverse-hstx-sync-clock"))
-			phy_parameter->inverse_hstx_sync_clock = true;
-		else
-			phy_parameter->inverse_hstx_sync_clock = false;
-
-		if (of_property_read_u32_index(np, "realtek,driving-level",
-					       index, &phy_parameter->driving_level))
-			phy_parameter->driving_level = DEFAULT_DC_DRIVING_VALUE;
-
-		if (of_property_read_u32_index(np, "realtek,driving-level-compensate",
-					       index, &phy_parameter->driving_level_compensate))
-			phy_parameter->driving_level_compensate = 0;
-
-		if (of_property_read_u32_index(np, "realtek,disconnection-compensate",
-					       index, &phy_parameter->disconnection_compensate))
-			phy_parameter->disconnection_compensate = 0;
-
-		get_phy_data_by_efuse(rtk_phy, phy_parameter, index);
-
-		update_dc_driving_level(rtk_phy, phy_parameter);
-
-		update_hs_clk_select(rtk_phy, phy_parameter);
-	}
-
-	return ret;
-}
-
-static int rtk_usb2phy_probe(struct platform_device *pdev)
-{
-	struct rtk_phy *rtk_phy;
-	struct device *dev = &pdev->dev;
-	struct phy *generic_phy;
-	struct phy_provider *phy_provider;
-	const struct phy_cfg *phy_cfg;
-	int ret = 0;
-
-	phy_cfg = of_device_get_match_data(dev);
-	if (!phy_cfg) {
-		dev_err(dev, "phy config are not assigned!\n");
-		return -EINVAL;
-	}
-
-	rtk_phy = devm_kzalloc(dev, sizeof(*rtk_phy), GFP_KERNEL);
-	if (!rtk_phy)
-		return -ENOMEM;
-
-	rtk_phy->dev			= &pdev->dev;
-	rtk_phy->phy.dev		= rtk_phy->dev;
-	rtk_phy->phy.label		= "rtk-usb2phy";
-	rtk_phy->phy.notify_port_status = rtk_phy_notify_port_status;
-
-	rtk_phy->phy_cfg = devm_kzalloc(dev, sizeof(*phy_cfg), GFP_KERNEL);
-
-	memcpy(rtk_phy->phy_cfg, phy_cfg, sizeof(*phy_cfg));
-
-	rtk_phy->num_phy = phy_cfg->num_phy;
-
-	ret = parse_phy_data(rtk_phy);
-	if (ret)
-		goto err;
-
-	platform_set_drvdata(pdev, rtk_phy);
-
-	generic_phy = devm_phy_create(rtk_phy->dev, NULL, &ops);
-	if (IS_ERR(generic_phy))
-		return PTR_ERR(generic_phy);
-
-	phy_set_drvdata(generic_phy, rtk_phy);
-
-	phy_provider = devm_of_phy_provider_register(rtk_phy->dev,
-						     of_phy_simple_xlate);
-	if (IS_ERR(phy_provider))
-		return PTR_ERR(phy_provider);
-
-	ret = usb_add_phy_dev(&rtk_phy->phy);
-	if (ret)
-		goto err;
-
-	create_debug_files(rtk_phy);
-
-err:
-	return ret;
-}
-
-static void rtk_usb2phy_remove(struct platform_device *pdev)
-{
-	struct rtk_phy *rtk_phy = platform_get_drvdata(pdev);
-
-	remove_debug_files(rtk_phy);
-
-	usb_remove_phy(&rtk_phy->phy);
-}
-
-static const struct phy_cfg rtd1295_phy_cfg = {
-	.page0_size = MAX_USB_PHY_PAGE0_DATA_SIZE,
-	.page0 = { [0] = {0xe0, 0x90},
-		   [3] = {0xe3, 0x3a},
-		   [4] = {0xe4, 0x68},
-		   [6] = {0xe6, 0x91},
-		  [13] = {0xf5, 0x81},
-		  [15] = {0xf7, 0x02}, },
-	.page1_size = 8,
-	.page1 = { /* default parameter */ },
-	.page2_size = 0,
-	.page2 = { /* no parameter */ },
-	.num_phy = 1,
-	.check_efuse = false,
-	.check_efuse_version = CHECK_EFUSE_V1,
-	.efuse_dc_driving_rate = 1,
-	.dc_driving_mask = 0xf,
-	.efuse_dc_disconnect_rate = EFUS_USB_DC_DIS_RATE,
-	.dc_disconnect_mask = 0xf,
-	.usb_dc_disconnect_at_page0 = true,
-	.do_toggle = true,
-	.do_toggle_driving = false,
-	.driving_updated_for_dev_dis = 0xf,
-	.use_default_parameter = false,
-	.is_double_sensitivity_mode = false,
-};
-
-static const struct phy_cfg rtd1395_phy_cfg = {
-	.page0_size = MAX_USB_PHY_PAGE0_DATA_SIZE,
-	.page0 = { [4] = {0xe4, 0xac},
-		  [13] = {0xf5, 0x00},
-		  [15] = {0xf7, 0x02}, },
-	.page1_size = 8,
-	.page1 = { /* default parameter */ },
-	.page2_size = 0,
-	.page2 = { /* no parameter */ },
-	.num_phy = 1,
-	.check_efuse = false,
-	.check_efuse_version = CHECK_EFUSE_V1,
-	.efuse_dc_driving_rate = 1,
-	.dc_driving_mask = 0xf,
-	.efuse_dc_disconnect_rate = EFUS_USB_DC_DIS_RATE,
-	.dc_disconnect_mask = 0xf,
-	.usb_dc_disconnect_at_page0 = true,
-	.do_toggle = true,
-	.do_toggle_driving = false,
-	.driving_updated_for_dev_dis = 0xf,
-	.use_default_parameter = false,
-	.is_double_sensitivity_mode = false,
-};
-
-static const struct phy_cfg rtd1395_phy_cfg_2port = {
-	.page0_size = MAX_USB_PHY_PAGE0_DATA_SIZE,
-	.page0 = { [4] = {0xe4, 0xac},
-		  [13] = {0xf5, 0x00},
-		  [15] = {0xf7, 0x02}, },
-	.page1_size = 8,
-	.page1 = { /* default parameter */ },
-	.page2_size = 0,
-	.page2 = { /* no parameter */ },
-	.num_phy = 2,
-	.check_efuse = false,
-	.check_efuse_version = CHECK_EFUSE_V1,
-	.efuse_dc_driving_rate = 1,
-	.dc_driving_mask = 0xf,
-	.efuse_dc_disconnect_rate = EFUS_USB_DC_DIS_RATE,
-	.dc_disconnect_mask = 0xf,
-	.usb_dc_disconnect_at_page0 = true,
-	.do_toggle = true,
-	.do_toggle_driving = false,
-	.driving_updated_for_dev_dis = 0xf,
-	.use_default_parameter = false,
-	.is_double_sensitivity_mode = false,
-};
-
-static const struct phy_cfg rtd1619_phy_cfg = {
-	.page0_size = MAX_USB_PHY_PAGE0_DATA_SIZE,
-	.page0 = { [4] = {0xe4, 0x68}, },
-	.page1_size = 8,
-	.page1 = { /* default parameter */ },
-	.page2_size = 0,
-	.page2 = { /* no parameter */ },
-	.num_phy = 1,
-	.check_efuse = true,
-	.check_efuse_version = CHECK_EFUSE_V1,
-	.efuse_dc_driving_rate = 1,
-	.dc_driving_mask = 0xf,
-	.efuse_dc_disconnect_rate = EFUS_USB_DC_DIS_RATE,
-	.dc_disconnect_mask = 0xf,
-	.usb_dc_disconnect_at_page0 = true,
-	.do_toggle = true,
-	.do_toggle_driving = false,
-	.driving_updated_for_dev_dis = 0xf,
-	.use_default_parameter = false,
-	.is_double_sensitivity_mode = false,
-};
-
-static const struct phy_cfg rtd1319_phy_cfg = {
-	.page0_size = MAX_USB_PHY_PAGE0_DATA_SIZE,
-	.page0 = { [0] = {0xe0, 0x18},
-		   [4] = {0xe4, 0x6a},
-		   [7] = {0xe7, 0x71},
-		  [13] = {0xf5, 0x15},
-		  [15] = {0xf7, 0x32}, },
-	.page1_size = 8,
-	.page1 = { [3] = {0xe3, 0x44}, },
-	.page2_size = MAX_USB_PHY_PAGE2_DATA_SIZE,
-	.page2 = { [0] = {0xe0, 0x01}, },
-	.num_phy = 1,
-	.check_efuse = true,
-	.check_efuse_version = CHECK_EFUSE_V1,
-	.efuse_dc_driving_rate = 1,
-	.dc_driving_mask = 0xf,
-	.efuse_dc_disconnect_rate = EFUS_USB_DC_DIS_RATE,
-	.dc_disconnect_mask = 0xf,
-	.usb_dc_disconnect_at_page0 = true,
-	.do_toggle = true,
-	.do_toggle_driving = true,
-	.driving_updated_for_dev_dis = 0xf,
-	.use_default_parameter = false,
-	.is_double_sensitivity_mode = true,
-};
-
-static const struct phy_cfg rtd1312c_phy_cfg = {
-	.page0_size = MAX_USB_PHY_PAGE0_DATA_SIZE,
-	.page0 = { [0] = {0xe0, 0x14},
-		   [4] = {0xe4, 0x67},
-		   [5] = {0xe5, 0x55}, },
-	.page1_size = 8,
-	.page1 = { [3] = {0xe3, 0x23},
-		   [6] = {0xe6, 0x58}, },
-	.page2_size = MAX_USB_PHY_PAGE2_DATA_SIZE,
-	.page2 = { /* default parameter */ },
-	.num_phy = 1,
-	.check_efuse = true,
-	.check_efuse_version = CHECK_EFUSE_V1,
-	.efuse_dc_driving_rate = 1,
-	.dc_driving_mask = 0xf,
-	.efuse_dc_disconnect_rate = EFUS_USB_DC_DIS_RATE,
-	.dc_disconnect_mask = 0xf,
-	.usb_dc_disconnect_at_page0 = true,
-	.do_toggle = true,
-	.do_toggle_driving = true,
-	.driving_updated_for_dev_dis = 0xf,
-	.use_default_parameter = false,
-	.is_double_sensitivity_mode = true,
-};
-
-static const struct phy_cfg rtd1619b_phy_cfg = {
-	.page0_size = MAX_USB_PHY_PAGE0_DATA_SIZE,
-	.page0 = { [0] = {0xe0, 0xa3},
-		   [4] = {0xe4, 0x88},
-		   [5] = {0xe5, 0x4f},
-		   [6] = {0xe6, 0x02}, },
-	.page1_size = 8,
-	.page1 = { [3] = {0xe3, 0x64}, },
-	.page2_size = MAX_USB_PHY_PAGE2_DATA_SIZE,
-	.page2 = { [7] = {0xe7, 0x45}, },
-	.num_phy = 1,
-	.check_efuse = true,
-	.check_efuse_version = CHECK_EFUSE_V1,
-	.efuse_dc_driving_rate = EFUS_USB_DC_CAL_RATE,
-	.dc_driving_mask = 0x1f,
-	.efuse_dc_disconnect_rate = EFUS_USB_DC_DIS_RATE,
-	.dc_disconnect_mask = 0xf,
-	.usb_dc_disconnect_at_page0 = false,
-	.do_toggle = true,
-	.do_toggle_driving = true,
-	.driving_updated_for_dev_dis = 0x8,
-	.use_default_parameter = false,
-	.is_double_sensitivity_mode = true,
-};
-
-static const struct phy_cfg rtd1319d_phy_cfg = {
-	.page0_size = MAX_USB_PHY_PAGE0_DATA_SIZE,
-	.page0 = { [0] = {0xe0, 0xa3},
-		   [4] = {0xe4, 0x8e},
-		   [5] = {0xe5, 0x4f},
-		   [6] = {0xe6, 0x02}, },
-	.page1_size = MAX_USB_PHY_PAGE1_DATA_SIZE,
-	.page1 = { [14] = {0xf5, 0x1}, },
-	.page2_size = MAX_USB_PHY_PAGE2_DATA_SIZE,
-	.page2 = { [7] = {0xe7, 0x44}, },
-	.check_efuse = true,
-	.num_phy = 1,
-	.check_efuse_version = CHECK_EFUSE_V1,
-	.efuse_dc_driving_rate = EFUS_USB_DC_CAL_RATE,
-	.dc_driving_mask = 0x1f,
-	.efuse_dc_disconnect_rate = EFUS_USB_DC_DIS_RATE,
-	.dc_disconnect_mask = 0xf,
-	.usb_dc_disconnect_at_page0 = false,
-	.do_toggle = true,
-	.do_toggle_driving = false,
-	.driving_updated_for_dev_dis = 0x8,
-	.use_default_parameter = false,
-	.is_double_sensitivity_mode = true,
-};
-
-static const struct phy_cfg rtd1315e_phy_cfg = {
-	.page0_size = MAX_USB_PHY_PAGE0_DATA_SIZE,
-	.page0 = { [0] = {0xe0, 0xa3},
-		   [4] = {0xe4, 0x8c},
-		   [5] = {0xe5, 0x4f},
-		   [6] = {0xe6, 0x02}, },
-	.page1_size = MAX_USB_PHY_PAGE1_DATA_SIZE,
-	.page1 = { [3] = {0xe3, 0x7f},
-		  [14] = {0xf5, 0x01}, },
-	.page2_size = MAX_USB_PHY_PAGE2_DATA_SIZE,
-	.page2 = { [7] = {0xe7, 0x44}, },
-	.num_phy = 1,
-	.check_efuse = true,
-	.check_efuse_version = CHECK_EFUSE_V2,
-	.efuse_dc_driving_rate = EFUS_USB_DC_CAL_RATE,
-	.dc_driving_mask = 0x1f,
-	.efuse_dc_disconnect_rate = EFUS_USB_DC_DIS_RATE,
-	.dc_disconnect_mask = 0xf,
-	.usb_dc_disconnect_at_page0 = false,
-	.do_toggle = true,
-	.do_toggle_driving = false,
-	.driving_updated_for_dev_dis = 0x8,
-	.use_default_parameter = false,
-	.is_double_sensitivity_mode = true,
-};
-
-static const struct of_device_id usbphy_rtk_dt_match[] = {
-	{ .compatible = "realtek,rtd1295-usb2phy", .data = &rtd1295_phy_cfg },
-	{ .compatible = "realtek,rtd1312c-usb2phy", .data = &rtd1312c_phy_cfg },
-	{ .compatible = "realtek,rtd1315e-usb2phy", .data = &rtd1315e_phy_cfg },
-	{ .compatible = "realtek,rtd1319-usb2phy", .data = &rtd1319_phy_cfg },
-	{ .compatible = "realtek,rtd1319d-usb2phy", .data = &rtd1319d_phy_cfg },
-	{ .compatible = "realtek,rtd1395-usb2phy", .data = &rtd1395_phy_cfg },
-	{ .compatible = "realtek,rtd1395-usb2phy-2port", .data = &rtd1395_phy_cfg_2port },
-	{ .compatible = "realtek,rtd1619-usb2phy", .data = &rtd1619_phy_cfg },
-	{ .compatible = "realtek,rtd1619b-usb2phy", .data = &rtd1619b_phy_cfg },
-	{},
-};
-MODULE_DEVICE_TABLE(of, usbphy_rtk_dt_match);
-
-static struct platform_driver rtk_usb2phy_driver = {
-	.probe		= rtk_usb2phy_probe,
-	.remove_new	= rtk_usb2phy_remove,
-	.driver		= {
-		.name	= "rtk-usb2phy",
-		.of_match_table = usbphy_rtk_dt_match,
-	},
-};
-
-module_platform_driver(rtk_usb2phy_driver);
-
-MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform: rtk-usb2phy");
-MODULE_AUTHOR("Stanley Chang <stanley_chang@realtek.com>");
-MODULE_DESCRIPTION("Realtek usb 2.0 phy driver");
diff --git a/drivers/phy/realtek/phy-rtk-usb3.c b/drivers/phy/realtek/phy-rtk-usb3.c
deleted file mode 100644
index 67446a85e968..000000000000
--- a/drivers/phy/realtek/phy-rtk-usb3.c
+++ /dev/null
@@ -1,761 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- *  phy-rtk-usb3.c RTK usb3.0 phy driver
- *
- * copyright (c) 2023 realtek semiconductor corporation
- *
- */
-
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/of_address.h>
-#include <linux/platform_device.h>
-#include <linux/uaccess.h>
-#include <linux/debugfs.h>
-#include <linux/nvmem-consumer.h>
-#include <linux/regmap.h>
-#include <linux/sys_soc.h>
-#include <linux/mfd/syscon.h>
-#include <linux/phy/phy.h>
-#include <linux/usb.h>
-#include <linux/usb/hcd.h>
-#include <linux/usb/phy.h>
-
-#define USB_MDIO_CTRL_PHY_BUSY BIT(7)
-#define USB_MDIO_CTRL_PHY_WRITE BIT(0)
-#define USB_MDIO_CTRL_PHY_ADDR_SHIFT 8
-#define USB_MDIO_CTRL_PHY_DATA_SHIFT 16
-
-#define MAX_USB_PHY_DATA_SIZE 0x30
-#define PHY_ADDR_0X09 0x09
-#define PHY_ADDR_0X0B 0x0b
-#define PHY_ADDR_0X0D 0x0d
-#define PHY_ADDR_0X10 0x10
-#define PHY_ADDR_0X1F 0x1f
-#define PHY_ADDR_0X20 0x20
-#define PHY_ADDR_0X21 0x21
-#define PHY_ADDR_0X30 0x30
-
-#define REG_0X09_FORCE_CALIBRATION BIT(9)
-#define REG_0X0B_RX_OFFSET_RANGE_MASK 0xc
-#define REG_0X0D_RX_DEBUG_TEST_EN BIT(6)
-#define REG_0X10_DEBUG_MODE_SETTING 0x3c0
-#define REG_0X10_DEBUG_MODE_SETTING_MASK 0x3f8
-#define REG_0X1F_RX_OFFSET_CODE_MASK 0x1e
-
-#define USB_U3_TX_LFPS_SWING_TRIM_SHIFT 4
-#define USB_U3_TX_LFPS_SWING_TRIM_MASK 0xf
-#define AMPLITUDE_CONTROL_COARSE_MASK 0xff
-#define AMPLITUDE_CONTROL_FINE_MASK 0xffff
-#define AMPLITUDE_CONTROL_COARSE_DEFAULT 0xff
-#define AMPLITUDE_CONTROL_FINE_DEFAULT 0xffff
-
-#define PHY_ADDR_MAP_ARRAY_INDEX(addr) (addr)
-#define ARRAY_INDEX_MAP_PHY_ADDR(index) (index)
-
-struct phy_reg {
-	void __iomem *reg_mdio_ctl;
-};
-
-struct phy_data {
-	u8 addr;
-	u16 data;
-};
-
-struct phy_cfg {
-	int param_size;
-	struct phy_data param[MAX_USB_PHY_DATA_SIZE];
-
-	bool check_efuse;
-	bool do_toggle;
-	bool do_toggle_once;
-	bool use_default_parameter;
-	bool check_rx_front_end_offset;
-};
-
-struct phy_parameter {
-	struct phy_reg phy_reg;
-
-	/* Get from efuse */
-	u8 efuse_usb_u3_tx_lfps_swing_trim;
-
-	/* Get from dts */
-	u32 amplitude_control_coarse;
-	u32 amplitude_control_fine;
-};
-
-struct rtk_phy {
-	struct usb_phy phy;
-	struct device *dev;
-
-	struct phy_cfg *phy_cfg;
-	int num_phy;
-	struct phy_parameter *phy_parameter;
-
-	struct dentry *debug_dir;
-};
-
-#define PHY_IO_TIMEOUT_USEC		(50000)
-#define PHY_IO_DELAY_US			(100)
-
-static inline int utmi_wait_register(void __iomem *reg, u32 mask, u32 result)
-{
-	int ret;
-	unsigned int val;
-
-	ret = read_poll_timeout(readl, val, ((val & mask) == result),
-				PHY_IO_DELAY_US, PHY_IO_TIMEOUT_USEC, false, reg);
-	if (ret) {
-		pr_err("%s can't program USB phy\n", __func__);
-		return -ETIMEDOUT;
-	}
-
-	return 0;
-}
-
-static int rtk_phy3_wait_vbusy(struct phy_reg *phy_reg)
-{
-	return utmi_wait_register(phy_reg->reg_mdio_ctl, USB_MDIO_CTRL_PHY_BUSY, 0);
-}
-
-static u16 rtk_phy_read(struct phy_reg *phy_reg, char addr)
-{
-	unsigned int tmp;
-	u32 value;
-
-	tmp = (addr << USB_MDIO_CTRL_PHY_ADDR_SHIFT);
-
-	writel(tmp, phy_reg->reg_mdio_ctl);
-
-	rtk_phy3_wait_vbusy(phy_reg);
-
-	value = readl(phy_reg->reg_mdio_ctl);
-	value = value >> USB_MDIO_CTRL_PHY_DATA_SHIFT;
-
-	return (u16)value;
-}
-
-static int rtk_phy_write(struct phy_reg *phy_reg, char addr, u16 data)
-{
-	unsigned int val;
-
-	val = USB_MDIO_CTRL_PHY_WRITE |
-		    (addr << USB_MDIO_CTRL_PHY_ADDR_SHIFT) |
-		    (data << USB_MDIO_CTRL_PHY_DATA_SHIFT);
-
-	writel(val, phy_reg->reg_mdio_ctl);
-
-	rtk_phy3_wait_vbusy(phy_reg);
-
-	return 0;
-}
-
-static void do_rtk_usb3_phy_toggle(struct rtk_phy *rtk_phy, int index, bool connect)
-{
-	struct phy_cfg *phy_cfg = rtk_phy->phy_cfg;
-	struct phy_reg *phy_reg;
-	struct phy_parameter *phy_parameter;
-	struct phy_data *phy_data;
-	u8 addr;
-	u16 data;
-	int i;
-
-	phy_parameter = &((struct phy_parameter *)rtk_phy->phy_parameter)[index];
-	phy_reg = &phy_parameter->phy_reg;
-
-	if (!phy_cfg->do_toggle)
-		return;
-
-	i = PHY_ADDR_MAP_ARRAY_INDEX(PHY_ADDR_0X09);
-	phy_data = phy_cfg->param + i;
-	addr = phy_data->addr;
-	data = phy_data->data;
-
-	if (!addr && !data) {
-		addr = PHY_ADDR_0X09;
-		data = rtk_phy_read(phy_reg, addr);
-		phy_data->addr = addr;
-		phy_data->data = data;
-	}
-
-	rtk_phy_write(phy_reg, addr, data & (~REG_0X09_FORCE_CALIBRATION));
-	mdelay(1);
-	rtk_phy_write(phy_reg, addr, data | REG_0X09_FORCE_CALIBRATION);
-}
-
-static int do_rtk_phy_init(struct rtk_phy *rtk_phy, int index)
-{
-	struct phy_cfg *phy_cfg;
-	struct phy_reg *phy_reg;
-	struct phy_parameter *phy_parameter;
-	int i = 0;
-
-	phy_cfg = rtk_phy->phy_cfg;
-	phy_parameter = &((struct phy_parameter *)rtk_phy->phy_parameter)[index];
-	phy_reg = &phy_parameter->phy_reg;
-
-	if (phy_cfg->use_default_parameter)
-		goto do_toggle;
-
-	for (i = 0; i < phy_cfg->param_size; i++) {
-		struct phy_data *phy_data = phy_cfg->param + i;
-		u8 addr = phy_data->addr;
-		u16 data = phy_data->data;
-
-		if (!addr && !data)
-			continue;
-
-		rtk_phy_write(phy_reg, addr, data);
-	}
-
-do_toggle:
-	if (phy_cfg->do_toggle_once)
-		phy_cfg->do_toggle = true;
-
-	do_rtk_usb3_phy_toggle(rtk_phy, index, false);
-
-	if (phy_cfg->do_toggle_once) {
-		u16 check_value = 0;
-		int count = 10;
-		u16 value_0x0d, value_0x10;
-
-		/* Enable Debug mode by set 0x0D and 0x10 */
-		value_0x0d = rtk_phy_read(phy_reg, PHY_ADDR_0X0D);
-		value_0x10 = rtk_phy_read(phy_reg, PHY_ADDR_0X10);
-
-		rtk_phy_write(phy_reg, PHY_ADDR_0X0D,
-			      value_0x0d | REG_0X0D_RX_DEBUG_TEST_EN);
-		rtk_phy_write(phy_reg, PHY_ADDR_0X10,
-			      (value_0x10 & ~REG_0X10_DEBUG_MODE_SETTING_MASK) |
-			      REG_0X10_DEBUG_MODE_SETTING);
-
-		check_value = rtk_phy_read(phy_reg, PHY_ADDR_0X30);
-
-		while (!(check_value & BIT(15))) {
-			check_value = rtk_phy_read(phy_reg, PHY_ADDR_0X30);
-			mdelay(1);
-			if (count-- < 0)
-				break;
-		}
-
-		if (!(check_value & BIT(15)))
-			dev_info(rtk_phy->dev, "toggle fail addr=0x%02x, data=0x%04x\n",
-				 PHY_ADDR_0X30, check_value);
-
-		/* Disable Debug mode by set 0x0D and 0x10 to default*/
-		rtk_phy_write(phy_reg, PHY_ADDR_0X0D, value_0x0d);
-		rtk_phy_write(phy_reg, PHY_ADDR_0X10, value_0x10);
-
-		phy_cfg->do_toggle = false;
-	}
-
-	if (phy_cfg->check_rx_front_end_offset) {
-		u16 rx_offset_code, rx_offset_range;
-		u16 code_mask = REG_0X1F_RX_OFFSET_CODE_MASK;
-		u16 range_mask = REG_0X0B_RX_OFFSET_RANGE_MASK;
-		bool do_update = false;
-
-		rx_offset_code = rtk_phy_read(phy_reg, PHY_ADDR_0X1F);
-		if (((rx_offset_code & code_mask) == 0x0) ||
-		    ((rx_offset_code & code_mask) == code_mask))
-			do_update = true;
-
-		rx_offset_range = rtk_phy_read(phy_reg, PHY_ADDR_0X0B);
-		if (((rx_offset_range & range_mask) == range_mask) && do_update) {
-			dev_warn(rtk_phy->dev, "Don't update rx_offset_range (rx_offset_code=0x%x, rx_offset_range=0x%x)\n",
-				 rx_offset_code, rx_offset_range);
-			do_update = false;
-		}
-
-		if (do_update) {
-			u16 tmp1, tmp2;
-
-			tmp1 = rx_offset_range & (~range_mask);
-			tmp2 = rx_offset_range & range_mask;
-			tmp2 += (1 << 2);
-			rx_offset_range = tmp1 | (tmp2 & range_mask);
-			rtk_phy_write(phy_reg, PHY_ADDR_0X0B, rx_offset_range);
-			goto do_toggle;
-		}
-	}
-
-	return 0;
-}
-
-static int rtk_phy_init(struct phy *phy)
-{
-	struct rtk_phy *rtk_phy = phy_get_drvdata(phy);
-	int ret = 0;
-	int i;
-	unsigned long phy_init_time = jiffies;
-
-	for (i = 0; i < rtk_phy->num_phy; i++)
-		ret = do_rtk_phy_init(rtk_phy, i);
-
-	dev_dbg(rtk_phy->dev, "Initialized RTK USB 3.0 PHY (take %dms)\n",
-		jiffies_to_msecs(jiffies - phy_init_time));
-
-	return ret;
-}
-
-static int rtk_phy_exit(struct phy *phy)
-{
-	return 0;
-}
-
-static const struct phy_ops ops = {
-	.init		= rtk_phy_init,
-	.exit		= rtk_phy_exit,
-	.owner		= THIS_MODULE,
-};
-
-static void rtk_phy_toggle(struct usb_phy *usb3_phy, bool connect, int port)
-{
-	int index = port;
-	struct rtk_phy *rtk_phy = NULL;
-
-	rtk_phy = dev_get_drvdata(usb3_phy->dev);
-
-	if (index > rtk_phy->num_phy) {
-		dev_err(rtk_phy->dev, "%s: The port=%d is not in usb phy (num_phy=%d)\n",
-			__func__, index, rtk_phy->num_phy);
-		return;
-	}
-
-	do_rtk_usb3_phy_toggle(rtk_phy, index, connect);
-}
-
-static int rtk_phy_notify_port_status(struct usb_phy *x, int port,
-				      u16 portstatus, u16 portchange)
-{
-	bool connect = false;
-
-	pr_debug("%s port=%d portstatus=0x%x portchange=0x%x\n",
-		 __func__, port, (int)portstatus, (int)portchange);
-	if (portstatus & USB_PORT_STAT_CONNECTION)
-		connect = true;
-
-	if (portchange & USB_PORT_STAT_C_CONNECTION)
-		rtk_phy_toggle(x, connect, port);
-
-	return 0;
-}
-
-#ifdef CONFIG_DEBUG_FS
-static struct dentry *create_phy_debug_root(void)
-{
-	struct dentry *phy_debug_root;
-
-	phy_debug_root = debugfs_lookup("phy", usb_debug_root);
-	if (!phy_debug_root)
-		phy_debug_root = debugfs_create_dir("phy", usb_debug_root);
-
-	return phy_debug_root;
-}
-
-static int rtk_usb3_parameter_show(struct seq_file *s, void *unused)
-{
-	struct rtk_phy *rtk_phy = s->private;
-	struct phy_cfg *phy_cfg;
-	int i, index;
-
-	phy_cfg = rtk_phy->phy_cfg;
-
-	seq_puts(s, "Property:\n");
-	seq_printf(s, "  check_efuse: %s\n",
-		   phy_cfg->check_efuse ? "Enable" : "Disable");
-	seq_printf(s, "  do_toggle: %s\n",
-		   phy_cfg->do_toggle ? "Enable" : "Disable");
-	seq_printf(s, "  do_toggle_once: %s\n",
-		   phy_cfg->do_toggle_once ? "Enable" : "Disable");
-	seq_printf(s, "  use_default_parameter: %s\n",
-		   phy_cfg->use_default_parameter ? "Enable" : "Disable");
-
-	for (index = 0; index < rtk_phy->num_phy; index++) {
-		struct phy_reg *phy_reg;
-		struct phy_parameter *phy_parameter;
-
-		phy_parameter = &((struct phy_parameter *)rtk_phy->phy_parameter)[index];
-		phy_reg = &phy_parameter->phy_reg;
-
-		seq_printf(s, "PHY %d:\n", index);
-
-		for (i = 0; i < phy_cfg->param_size; i++) {
-			struct phy_data *phy_data = phy_cfg->param + i;
-			u8 addr = ARRAY_INDEX_MAP_PHY_ADDR(i);
-			u16 data = phy_data->data;
-
-			if (!phy_data->addr && !data)
-				seq_printf(s, "  addr = 0x%02x, data = none   ==> read value = 0x%04x\n",
-					   addr, rtk_phy_read(phy_reg, addr));
-			else
-				seq_printf(s, "  addr = 0x%02x, data = 0x%04x ==> read value = 0x%04x\n",
-					   addr, data, rtk_phy_read(phy_reg, addr));
-		}
-
-		seq_puts(s, "PHY Property:\n");
-		seq_printf(s, "  efuse_usb_u3_tx_lfps_swing_trim: 0x%x\n",
-			   (int)phy_parameter->efuse_usb_u3_tx_lfps_swing_trim);
-		seq_printf(s, "  amplitude_control_coarse: 0x%x\n",
-			   (int)phy_parameter->amplitude_control_coarse);
-		seq_printf(s, "  amplitude_control_fine: 0x%x\n",
-			   (int)phy_parameter->amplitude_control_fine);
-	}
-
-	return 0;
-}
-DEFINE_SHOW_ATTRIBUTE(rtk_usb3_parameter);
-
-static inline void create_debug_files(struct rtk_phy *rtk_phy)
-{
-	struct dentry *phy_debug_root = NULL;
-
-	phy_debug_root = create_phy_debug_root();
-
-	if (!phy_debug_root)
-		return;
-
-	rtk_phy->debug_dir = debugfs_create_dir(dev_name(rtk_phy->dev), phy_debug_root);
-
-	debugfs_create_file("parameter", 0444, rtk_phy->debug_dir, rtk_phy,
-			    &rtk_usb3_parameter_fops);
-
-	return;
-}
-
-static inline void remove_debug_files(struct rtk_phy *rtk_phy)
-{
-	debugfs_remove_recursive(rtk_phy->debug_dir);
-}
-#else
-static inline void create_debug_files(struct rtk_phy *rtk_phy) { }
-static inline void remove_debug_files(struct rtk_phy *rtk_phy) { }
-#endif /* CONFIG_DEBUG_FS */
-
-static int get_phy_data_by_efuse(struct rtk_phy *rtk_phy,
-				 struct phy_parameter *phy_parameter, int index)
-{
-	struct phy_cfg *phy_cfg = rtk_phy->phy_cfg;
-	u8 value = 0;
-	struct nvmem_cell *cell;
-
-	if (!phy_cfg->check_efuse)
-		goto out;
-
-	cell = nvmem_cell_get(rtk_phy->dev, "usb_u3_tx_lfps_swing_trim");
-	if (IS_ERR(cell)) {
-		dev_dbg(rtk_phy->dev, "%s no usb_u3_tx_lfps_swing_trim: %ld\n",
-			__func__, PTR_ERR(cell));
-	} else {
-		unsigned char *buf;
-		size_t buf_size;
-
-		buf = nvmem_cell_read(cell, &buf_size);
-		if (!IS_ERR(buf)) {
-			value = buf[0] & USB_U3_TX_LFPS_SWING_TRIM_MASK;
-			kfree(buf);
-		}
-		nvmem_cell_put(cell);
-	}
-
-	if (value > 0 && value < 0x8)
-		phy_parameter->efuse_usb_u3_tx_lfps_swing_trim = 0x8;
-	else
-		phy_parameter->efuse_usb_u3_tx_lfps_swing_trim = (u8)value;
-
-out:
-	return 0;
-}
-
-static void update_amplitude_control_value(struct rtk_phy *rtk_phy,
-					   struct phy_parameter *phy_parameter)
-{
-	struct phy_cfg *phy_cfg;
-	struct phy_reg *phy_reg;
-
-	phy_reg = &phy_parameter->phy_reg;
-	phy_cfg = rtk_phy->phy_cfg;
-
-	if (phy_parameter->amplitude_control_coarse != AMPLITUDE_CONTROL_COARSE_DEFAULT) {
-		u16 val_mask = AMPLITUDE_CONTROL_COARSE_MASK;
-		u16 data;
-
-		if (!phy_cfg->param[PHY_ADDR_0X20].addr && !phy_cfg->param[PHY_ADDR_0X20].data) {
-			phy_cfg->param[PHY_ADDR_0X20].addr = PHY_ADDR_0X20;
-			data = rtk_phy_read(phy_reg, PHY_ADDR_0X20);
-		} else {
-			data = phy_cfg->param[PHY_ADDR_0X20].data;
-		}
-
-		data &= (~val_mask);
-		data |= (phy_parameter->amplitude_control_coarse & val_mask);
-
-		phy_cfg->param[PHY_ADDR_0X20].data = data;
-	}
-
-	if (phy_parameter->efuse_usb_u3_tx_lfps_swing_trim) {
-		u8 efuse_val = phy_parameter->efuse_usb_u3_tx_lfps_swing_trim;
-		u16 val_mask = USB_U3_TX_LFPS_SWING_TRIM_MASK;
-		int val_shift = USB_U3_TX_LFPS_SWING_TRIM_SHIFT;
-		u16 data;
-
-		if (!phy_cfg->param[PHY_ADDR_0X20].addr && !phy_cfg->param[PHY_ADDR_0X20].data) {
-			phy_cfg->param[PHY_ADDR_0X20].addr = PHY_ADDR_0X20;
-			data = rtk_phy_read(phy_reg, PHY_ADDR_0X20);
-		} else {
-			data = phy_cfg->param[PHY_ADDR_0X20].data;
-		}
-
-		data &= ~(val_mask << val_shift);
-		data |= ((efuse_val & val_mask) << val_shift);
-
-		phy_cfg->param[PHY_ADDR_0X20].data = data;
-	}
-
-	if (phy_parameter->amplitude_control_fine != AMPLITUDE_CONTROL_FINE_DEFAULT) {
-		u16 val_mask = AMPLITUDE_CONTROL_FINE_MASK;
-
-		if (!phy_cfg->param[PHY_ADDR_0X21].addr && !phy_cfg->param[PHY_ADDR_0X21].data)
-			phy_cfg->param[PHY_ADDR_0X21].addr = PHY_ADDR_0X21;
-
-		phy_cfg->param[PHY_ADDR_0X21].data =
-			    phy_parameter->amplitude_control_fine & val_mask;
-	}
-}
-
-static int parse_phy_data(struct rtk_phy *rtk_phy)
-{
-	struct device *dev = rtk_phy->dev;
-	struct phy_parameter *phy_parameter;
-	int ret = 0;
-	int index;
-
-	rtk_phy->phy_parameter = devm_kzalloc(dev, sizeof(struct phy_parameter) *
-					      rtk_phy->num_phy, GFP_KERNEL);
-	if (!rtk_phy->phy_parameter)
-		return -ENOMEM;
-
-	for (index = 0; index < rtk_phy->num_phy; index++) {
-		phy_parameter = &((struct phy_parameter *)rtk_phy->phy_parameter)[index];
-
-		phy_parameter->phy_reg.reg_mdio_ctl = of_iomap(dev->of_node, 0) + index;
-
-		/* Amplitude control address 0x20 bit 0 to bit 7 */
-		if (of_property_read_u32(dev->of_node, "realtek,amplitude-control-coarse-tuning",
-					 &phy_parameter->amplitude_control_coarse))
-			phy_parameter->amplitude_control_coarse = AMPLITUDE_CONTROL_COARSE_DEFAULT;
-
-		/* Amplitude control address 0x21 bit 0 to bit 16 */
-		if (of_property_read_u32(dev->of_node, "realtek,amplitude-control-fine-tuning",
-					 &phy_parameter->amplitude_control_fine))
-			phy_parameter->amplitude_control_fine = AMPLITUDE_CONTROL_FINE_DEFAULT;
-
-		get_phy_data_by_efuse(rtk_phy, phy_parameter, index);
-
-		update_amplitude_control_value(rtk_phy, phy_parameter);
-	}
-
-	return ret;
-}
-
-static int rtk_usb3phy_probe(struct platform_device *pdev)
-{
-	struct rtk_phy *rtk_phy;
-	struct device *dev = &pdev->dev;
-	struct phy *generic_phy;
-	struct phy_provider *phy_provider;
-	const struct phy_cfg *phy_cfg;
-	int ret;
-
-	phy_cfg = of_device_get_match_data(dev);
-	if (!phy_cfg) {
-		dev_err(dev, "phy config are not assigned!\n");
-		return -EINVAL;
-	}
-
-	rtk_phy = devm_kzalloc(dev, sizeof(*rtk_phy), GFP_KERNEL);
-	if (!rtk_phy)
-		return -ENOMEM;
-
-	rtk_phy->dev			= &pdev->dev;
-	rtk_phy->phy.dev		= rtk_phy->dev;
-	rtk_phy->phy.label		= "rtk-usb3phy";
-	rtk_phy->phy.notify_port_status = rtk_phy_notify_port_status;
-
-	rtk_phy->phy_cfg = devm_kzalloc(dev, sizeof(*phy_cfg), GFP_KERNEL);
-
-	memcpy(rtk_phy->phy_cfg, phy_cfg, sizeof(*phy_cfg));
-
-	rtk_phy->num_phy = 1;
-
-	ret = parse_phy_data(rtk_phy);
-	if (ret)
-		goto err;
-
-	platform_set_drvdata(pdev, rtk_phy);
-
-	generic_phy = devm_phy_create(rtk_phy->dev, NULL, &ops);
-	if (IS_ERR(generic_phy))
-		return PTR_ERR(generic_phy);
-
-	phy_set_drvdata(generic_phy, rtk_phy);
-
-	phy_provider = devm_of_phy_provider_register(rtk_phy->dev, of_phy_simple_xlate);
-	if (IS_ERR(phy_provider))
-		return PTR_ERR(phy_provider);
-
-	ret = usb_add_phy_dev(&rtk_phy->phy);
-	if (ret)
-		goto err;
-
-	create_debug_files(rtk_phy);
-
-err:
-	return ret;
-}
-
-static void rtk_usb3phy_remove(struct platform_device *pdev)
-{
-	struct rtk_phy *rtk_phy = platform_get_drvdata(pdev);
-
-	remove_debug_files(rtk_phy);
-
-	usb_remove_phy(&rtk_phy->phy);
-}
-
-static const struct phy_cfg rtd1295_phy_cfg = {
-	.param_size = MAX_USB_PHY_DATA_SIZE,
-	.param = {  [0] = {0x01, 0x4008},  [1] = {0x01, 0xe046},
-		    [2] = {0x02, 0x6046},  [3] = {0x03, 0x2779},
-		    [4] = {0x04, 0x72f5},  [5] = {0x05, 0x2ad3},
-		    [6] = {0x06, 0x000e},  [7] = {0x07, 0x2e00},
-		    [8] = {0x08, 0x3591},  [9] = {0x09, 0x525c},
-		   [10] = {0x0a, 0xa600}, [11] = {0x0b, 0xa904},
-		   [12] = {0x0c, 0xc000}, [13] = {0x0d, 0xef1c},
-		   [14] = {0x0e, 0x2000}, [15] = {0x0f, 0x0000},
-		   [16] = {0x10, 0x000c}, [17] = {0x11, 0x4c00},
-		   [18] = {0x12, 0xfc00}, [19] = {0x13, 0x0c81},
-		   [20] = {0x14, 0xde01}, [21] = {0x15, 0x0000},
-		   [22] = {0x16, 0x0000}, [23] = {0x17, 0x0000},
-		   [24] = {0x18, 0x0000}, [25] = {0x19, 0x4004},
-		   [26] = {0x1a, 0x1260}, [27] = {0x1b, 0xff00},
-		   [28] = {0x1c, 0xcb00}, [29] = {0x1d, 0xa03f},
-		   [30] = {0x1e, 0xc2e0}, [31] = {0x1f, 0x2807},
-		   [32] = {0x20, 0x947a}, [33] = {0x21, 0x88aa},
-		   [34] = {0x22, 0x0057}, [35] = {0x23, 0xab66},
-		   [36] = {0x24, 0x0800}, [37] = {0x25, 0x0000},
-		   [38] = {0x26, 0x040a}, [39] = {0x27, 0x01d6},
-		   [40] = {0x28, 0xf8c2}, [41] = {0x29, 0x3080},
-		   [42] = {0x2a, 0x3082}, [43] = {0x2b, 0x2078},
-		   [44] = {0x2c, 0xffff}, [45] = {0x2d, 0xffff},
-		   [46] = {0x2e, 0x0000}, [47] = {0x2f, 0x0040}, },
-	.check_efuse = false,
-	.do_toggle = true,
-	.do_toggle_once = false,
-	.use_default_parameter = false,
-	.check_rx_front_end_offset = false,
-};
-
-static const struct phy_cfg rtd1619_phy_cfg = {
-	.param_size = MAX_USB_PHY_DATA_SIZE,
-	.param = {  [8] = {0x08, 0x3591},
-		   [38] = {0x26, 0x840b},
-		   [40] = {0x28, 0xf842}, },
-	.check_efuse = false,
-	.do_toggle = true,
-	.do_toggle_once = false,
-	.use_default_parameter = false,
-	.check_rx_front_end_offset = false,
-};
-
-static const struct phy_cfg rtd1319_phy_cfg = {
-	.param_size = MAX_USB_PHY_DATA_SIZE,
-	.param = {  [1] = {0x01, 0xac86},
-		    [6] = {0x06, 0x0003},
-		    [9] = {0x09, 0x924c},
-		   [10] = {0x0a, 0xa608},
-		   [11] = {0x0b, 0xb905},
-		   [14] = {0x0e, 0x2010},
-		   [32] = {0x20, 0x705a},
-		   [33] = {0x21, 0xf645},
-		   [34] = {0x22, 0x0013},
-		   [35] = {0x23, 0xcb66},
-		   [41] = {0x29, 0xff00}, },
-	.check_efuse = true,
-	.do_toggle = true,
-	.do_toggle_once = false,
-	.use_default_parameter = false,
-	.check_rx_front_end_offset = false,
-};
-
-static const struct phy_cfg rtd1619b_phy_cfg = {
-	.param_size = MAX_USB_PHY_DATA_SIZE,
-	.param = {  [1] = {0x01, 0xac8c},
-		    [6] = {0x06, 0x0017},
-		    [9] = {0x09, 0x724c},
-		   [10] = {0x0a, 0xb610},
-		   [11] = {0x0b, 0xb90d},
-		   [13] = {0x0d, 0xef2a},
-		   [15] = {0x0f, 0x9050},
-		   [16] = {0x10, 0x000c},
-		   [32] = {0x20, 0x70ff},
-		   [34] = {0x22, 0x0013},
-		   [35] = {0x23, 0xdb66},
-		   [38] = {0x26, 0x8609},
-		   [41] = {0x29, 0xff13},
-		   [42] = {0x2a, 0x3070}, },
-	.check_efuse = true,
-	.do_toggle = false,
-	.do_toggle_once = true,
-	.use_default_parameter = false,
-	.check_rx_front_end_offset = false,
-};
-
-static const  struct phy_cfg rtd1319d_phy_cfg = {
-	.param_size = MAX_USB_PHY_DATA_SIZE,
-	.param = {  [1] = {0x01, 0xac89},
-		    [4] = {0x04, 0xf2f5},
-		    [6] = {0x06, 0x0017},
-		    [9] = {0x09, 0x424c},
-		   [10] = {0x0a, 0x9610},
-		   [11] = {0x0b, 0x9901},
-		   [12] = {0x0c, 0xf000},
-		   [13] = {0x0d, 0xef2a},
-		   [14] = {0x0e, 0x1000},
-		   [15] = {0x0f, 0x9050},
-		   [32] = {0x20, 0x7077},
-		   [35] = {0x23, 0x0b62},
-		   [37] = {0x25, 0x10ec},
-		   [42] = {0x2a, 0x3070}, },
-	.check_efuse = true,
-	.do_toggle = false,
-	.do_toggle_once = true,
-	.use_default_parameter = false,
-	.check_rx_front_end_offset = true,
-};
-
-static const struct of_device_id usbphy_rtk_dt_match[] = {
-	{ .compatible = "realtek,rtd1295-usb3phy", .data = &rtd1295_phy_cfg },
-	{ .compatible = "realtek,rtd1319-usb3phy", .data = &rtd1319_phy_cfg },
-	{ .compatible = "realtek,rtd1319d-usb3phy", .data = &rtd1319d_phy_cfg },
-	{ .compatible = "realtek,rtd1619-usb3phy", .data = &rtd1619_phy_cfg },
-	{ .compatible = "realtek,rtd1619b-usb3phy", .data = &rtd1619b_phy_cfg },
-	{},
-};
-MODULE_DEVICE_TABLE(of, usbphy_rtk_dt_match);
-
-static struct platform_driver rtk_usb3phy_driver = {
-	.probe		= rtk_usb3phy_probe,
-	.remove_new	= rtk_usb3phy_remove,
-	.driver		= {
-		.name	= "rtk-usb3phy",
-		.of_match_table = usbphy_rtk_dt_match,
-	},
-};
-
-module_platform_driver(rtk_usb3phy_driver);
-
-MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform: rtk-usb3phy");
-MODULE_AUTHOR("Stanley Chang <stanley_chang@realtek.com>");
-MODULE_DESCRIPTION("Realtek usb 3.0 phy driver");
diff --git a/drivers/phy/ti/phy-gmii-sel.c b/drivers/phy/ti/phy-gmii-sel.c
index 555b323f45da..bc847d3879f7 100644
--- a/drivers/phy/ti/phy-gmii-sel.c
+++ b/drivers/phy/ti/phy-gmii-sel.c
@@ -64,6 +64,7 @@ struct phy_gmii_sel_priv {
 	u32 num_ports;
 	u32 reg_offset;
 	u32 qsgmii_main_ports;
+	bool no_offset;
 };
 
 static int phy_gmii_sel_mode(struct phy *phy, enum phy_mode mode, int submode)
@@ -402,7 +403,8 @@ static int phy_gmii_sel_init_ports(struct phy_gmii_sel_priv *priv)
 		priv->num_ports = size / sizeof(u32);
 		if (!priv->num_ports)
 			return -EINVAL;
-		priv->reg_offset = __be32_to_cpu(*offset);
+		if (!priv->no_offset)
+			priv->reg_offset = __be32_to_cpu(*offset);
 	}
 
 	if_phys = devm_kcalloc(dev, priv->num_ports,
@@ -471,6 +473,7 @@ static int phy_gmii_sel_probe(struct platform_device *pdev)
 			dev_err(dev, "Failed to get syscon %d\n", ret);
 			return ret;
 		}
+		priv->no_offset = true;
 	}
 
 	ret = phy_gmii_sel_init_ports(priv);
diff --git a/drivers/pinctrl/Kconfig b/drivers/pinctrl/Kconfig
index 1de4e1edede0..02133fd9d924 100644
--- a/drivers/pinctrl/Kconfig
+++ b/drivers/pinctrl/Kconfig
@@ -469,6 +469,22 @@ config PINCTRL_TB10X
 	depends on OF && ARC_PLAT_TB10X
 	select GPIOLIB
 
+config PINCTRL_TPS6594
+	tristate "Pinctrl and GPIO driver for TI TPS6594 PMIC"
+	depends on OF && MFD_TPS6594
+	default MFD_TPS6594
+	select PINMUX
+	select GPIOLIB
+	select REGMAP
+	select GPIO_REGMAP
+	select GENERIC_PINCONF
+	help
+	  Say Y to select the pinmuxing and GPIOs driver for the TPS6594
+	  PMICs chip family.
+
+	  This driver can also be built as a module
+	  called tps6594-pinctrl.
+
 config PINCTRL_ZYNQ
 	bool "Pinctrl driver for Xilinx Zynq"
 	depends on ARCH_ZYNQ
diff --git a/drivers/pinctrl/Makefile b/drivers/pinctrl/Makefile
index 37575deb7a69..6e8f3416995c 100644
--- a/drivers/pinctrl/Makefile
+++ b/drivers/pinctrl/Makefile
@@ -48,6 +48,7 @@ obj-$(CONFIG_PINCTRL_ST) 	+= pinctrl-st.o
 obj-$(CONFIG_PINCTRL_STMFX) 	+= pinctrl-stmfx.o
 obj-$(CONFIG_PINCTRL_SX150X)	+= pinctrl-sx150x.o
 obj-$(CONFIG_PINCTRL_TB10X)	+= pinctrl-tb10x.o
+obj-$(CONFIG_PINCTRL_TPS6594)	+= pinctrl-tps6594.o
 obj-$(CONFIG_PINCTRL_ZYNQMP)	+= pinctrl-zynqmp.o
 obj-$(CONFIG_PINCTRL_ZYNQ)	+= pinctrl-zynq.o
 
diff --git a/drivers/pinctrl/cirrus/Kconfig b/drivers/pinctrl/cirrus/Kconfig
index d6318cb57aff..e7e827a8877a 100644
--- a/drivers/pinctrl/cirrus/Kconfig
+++ b/drivers/pinctrl/cirrus/Kconfig
@@ -12,7 +12,8 @@ config PINCTRL_CS42L43
 
 config PINCTRL_LOCHNAGAR
 	tristate "Cirrus Logic Lochnagar pinctrl driver"
-	depends on MFD_LOCHNAGAR
+	# Avoid clash caused by MIPS defining RST, which is used in the driver
+	depends on MFD_LOCHNAGAR && !MIPS
 	select GPIOLIB
 	select PINMUX
 	select PINCONF
diff --git a/drivers/pinctrl/core.c b/drivers/pinctrl/core.c
index 1fa89be29b8f..f2977eb65522 100644
--- a/drivers/pinctrl/core.c
+++ b/drivers/pinctrl/core.c
@@ -1262,17 +1262,17 @@ static void pinctrl_link_add(struct pinctrl_dev *pctldev,
 static int pinctrl_commit_state(struct pinctrl *p, struct pinctrl_state *state)
 {
 	struct pinctrl_setting *setting, *setting2;
-	struct pinctrl_state *old_state = p->state;
+	struct pinctrl_state *old_state = READ_ONCE(p->state);
 	int ret;
 
-	if (p->state) {
+	if (old_state) {
 		/*
 		 * For each pinmux setting in the old state, forget SW's record
 		 * of mux owner for that pingroup. Any pingroups which are
 		 * still owned by the new state will be re-acquired by the call
 		 * to pinmux_enable_setting() in the loop below.
 		 */
-		list_for_each_entry(setting, &p->state->settings, node) {
+		list_for_each_entry(setting, &old_state->settings, node) {
 			if (setting->type != PIN_MAP_TYPE_MUX_GROUP)
 				continue;
 			pinmux_disable_setting(setting);
diff --git a/drivers/pinctrl/intel/Kconfig b/drivers/pinctrl/intel/Kconfig
index d66f4f6932d8..d9bdd0e0e8a2 100644
--- a/drivers/pinctrl/intel/Kconfig
+++ b/drivers/pinctrl/intel/Kconfig
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 # Intel pin control drivers
 menu "Intel pinctrl drivers"
-	depends on ACPI && (X86 || COMPILE_TEST)
+	depends on (ACPI && X86) || COMPILE_TEST
 
 config PINCTRL_BAYTRAIL
 	bool "Intel Baytrail GPIO pin control"
@@ -37,6 +37,16 @@ config PINCTRL_INTEL
 	select GPIOLIB
 	select GPIOLIB_IRQCHIP
 
+config PINCTRL_INTEL_PLATFORM
+	tristate "Intel pinctrl and GPIO platform driver"
+	depends on ACPI
+	select PINCTRL_INTEL
+	help
+	  This pinctrl driver provides an interface that allows configuring
+	  of Intel PCH pins and using them as GPIOs. Currently the following
+	  Intel SoCs / platforms require this to be functional:
+	  - Lunar Lake
+
 config PINCTRL_ALDERLAKE
 	tristate "Intel Alder Lake pinctrl and GPIO driver"
 	select PINCTRL_INTEL
diff --git a/drivers/pinctrl/intel/Makefile b/drivers/pinctrl/intel/Makefile
index f6d30f2d973a..96c93ed4bd58 100644
--- a/drivers/pinctrl/intel/Makefile
+++ b/drivers/pinctrl/intel/Makefile
@@ -8,6 +8,7 @@ obj-$(CONFIG_PINCTRL_TANGIER)		+= pinctrl-tangier.o
 obj-$(CONFIG_PINCTRL_MERRIFIELD)	+= pinctrl-merrifield.o
 obj-$(CONFIG_PINCTRL_MOOREFIELD)	+= pinctrl-moorefield.o
 obj-$(CONFIG_PINCTRL_INTEL)		+= pinctrl-intel.o
+obj-$(CONFIG_PINCTRL_INTEL_PLATFORM)	+= pinctrl-intel-platform.o
 obj-$(CONFIG_PINCTRL_ALDERLAKE)		+= pinctrl-alderlake.o
 obj-$(CONFIG_PINCTRL_BROXTON)		+= pinctrl-broxton.o
 obj-$(CONFIG_PINCTRL_CANNONLAKE)	+= pinctrl-cannonlake.o
diff --git a/drivers/pinctrl/intel/pinctrl-alderlake.c b/drivers/pinctrl/intel/pinctrl-alderlake.c
index 4a37dc273d63..7d9948e5f422 100644
--- a/drivers/pinctrl/intel/pinctrl-alderlake.c
+++ b/drivers/pinctrl/intel/pinctrl-alderlake.c
@@ -9,6 +9,7 @@
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/pm.h>
 
 #include <linux/pinctrl/pinctrl.h>
 
@@ -733,14 +734,12 @@ static const struct acpi_device_id adl_pinctrl_acpi_match[] = {
 };
 MODULE_DEVICE_TABLE(acpi, adl_pinctrl_acpi_match);
 
-static INTEL_PINCTRL_PM_OPS(adl_pinctrl_pm_ops);
-
 static struct platform_driver adl_pinctrl_driver = {
 	.probe = intel_pinctrl_probe_by_hid,
 	.driver = {
 		.name = "alderlake-pinctrl",
 		.acpi_match_table = adl_pinctrl_acpi_match,
-		.pm = &adl_pinctrl_pm_ops,
+		.pm = pm_sleep_ptr(&intel_pinctrl_pm_ops),
 	},
 };
 module_platform_driver(adl_pinctrl_driver);
diff --git a/drivers/pinctrl/intel/pinctrl-broxton.c b/drivers/pinctrl/intel/pinctrl-broxton.c
index 3118c7c8842f..d99541676630 100644
--- a/drivers/pinctrl/intel/pinctrl-broxton.c
+++ b/drivers/pinctrl/intel/pinctrl-broxton.c
@@ -9,6 +9,7 @@
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/pm.h>
 
 #include <linux/pinctrl/pinctrl.h>
 
@@ -1000,14 +1001,12 @@ static const struct platform_device_id bxt_pinctrl_platform_ids[] = {
 };
 MODULE_DEVICE_TABLE(platform, bxt_pinctrl_platform_ids);
 
-static INTEL_PINCTRL_PM_OPS(bxt_pinctrl_pm_ops);
-
 static struct platform_driver bxt_pinctrl_driver = {
 	.probe = intel_pinctrl_probe_by_uid,
 	.driver = {
 		.name = "broxton-pinctrl",
 		.acpi_match_table = bxt_pinctrl_acpi_match,
-		.pm = &bxt_pinctrl_pm_ops,
+		.pm = pm_sleep_ptr(&intel_pinctrl_pm_ops),
 	},
 	.id_table = bxt_pinctrl_platform_ids,
 };
diff --git a/drivers/pinctrl/intel/pinctrl-cannonlake.c b/drivers/pinctrl/intel/pinctrl-cannonlake.c
index 95976abfb785..1aa09f950be1 100644
--- a/drivers/pinctrl/intel/pinctrl-cannonlake.c
+++ b/drivers/pinctrl/intel/pinctrl-cannonlake.c
@@ -10,6 +10,7 @@
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/pm.h>
 
 #include <linux/pinctrl/pinctrl.h>
 
@@ -824,14 +825,12 @@ static const struct acpi_device_id cnl_pinctrl_acpi_match[] = {
 };
 MODULE_DEVICE_TABLE(acpi, cnl_pinctrl_acpi_match);
 
-static INTEL_PINCTRL_PM_OPS(cnl_pinctrl_pm_ops);
-
 static struct platform_driver cnl_pinctrl_driver = {
 	.probe = intel_pinctrl_probe_by_hid,
 	.driver = {
 		.name = "cannonlake-pinctrl",
 		.acpi_match_table = cnl_pinctrl_acpi_match,
-		.pm = &cnl_pinctrl_pm_ops,
+		.pm = pm_sleep_ptr(&intel_pinctrl_pm_ops),
 	},
 };
 module_platform_driver(cnl_pinctrl_driver);
diff --git a/drivers/pinctrl/intel/pinctrl-cedarfork.c b/drivers/pinctrl/intel/pinctrl-cedarfork.c
index a20465eb2dc6..48af8930dd1f 100644
--- a/drivers/pinctrl/intel/pinctrl-cedarfork.c
+++ b/drivers/pinctrl/intel/pinctrl-cedarfork.c
@@ -9,6 +9,7 @@
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/pm.h>
 
 #include <linux/pinctrl/pinctrl.h>
 
@@ -319,8 +320,6 @@ static const struct intel_pinctrl_soc_data cdf_soc_data = {
 	.ncommunities = ARRAY_SIZE(cdf_communities),
 };
 
-static INTEL_PINCTRL_PM_OPS(cdf_pinctrl_pm_ops);
-
 static const struct acpi_device_id cdf_pinctrl_acpi_match[] = {
 	{ "INTC3001", (kernel_ulong_t)&cdf_soc_data },
 	{ }
@@ -332,7 +331,7 @@ static struct platform_driver cdf_pinctrl_driver = {
 	.driver = {
 		.name = "cedarfork-pinctrl",
 		.acpi_match_table = cdf_pinctrl_acpi_match,
-		.pm = &cdf_pinctrl_pm_ops,
+		.pm = pm_sleep_ptr(&intel_pinctrl_pm_ops),
 	},
 };
 
diff --git a/drivers/pinctrl/intel/pinctrl-denverton.c b/drivers/pinctrl/intel/pinctrl-denverton.c
index 562a4f9188e4..666507f54f27 100644
--- a/drivers/pinctrl/intel/pinctrl-denverton.c
+++ b/drivers/pinctrl/intel/pinctrl-denverton.c
@@ -9,6 +9,7 @@
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/pm.h>
 
 #include <linux/pinctrl/pinctrl.h>
 
@@ -249,8 +250,6 @@ static const struct intel_pinctrl_soc_data dnv_soc_data = {
 	.ncommunities = ARRAY_SIZE(dnv_communities),
 };
 
-static INTEL_PINCTRL_PM_OPS(dnv_pinctrl_pm_ops);
-
 static const struct acpi_device_id dnv_pinctrl_acpi_match[] = {
 	{ "INTC3000", (kernel_ulong_t)&dnv_soc_data },
 	{ }
@@ -268,7 +267,7 @@ static struct platform_driver dnv_pinctrl_driver = {
 	.driver = {
 		.name = "denverton-pinctrl",
 		.acpi_match_table = dnv_pinctrl_acpi_match,
-		.pm = &dnv_pinctrl_pm_ops,
+		.pm = pm_sleep_ptr(&intel_pinctrl_pm_ops),
 	},
 	.id_table = dnv_pinctrl_platform_ids,
 };
diff --git a/drivers/pinctrl/intel/pinctrl-elkhartlake.c b/drivers/pinctrl/intel/pinctrl-elkhartlake.c
index 81581ab85316..1678634ebc06 100644
--- a/drivers/pinctrl/intel/pinctrl-elkhartlake.c
+++ b/drivers/pinctrl/intel/pinctrl-elkhartlake.c
@@ -9,6 +9,7 @@
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/pm.h>
 
 #include <linux/pinctrl/pinctrl.h>
 
@@ -485,14 +486,12 @@ static const struct acpi_device_id ehl_pinctrl_acpi_match[] = {
 };
 MODULE_DEVICE_TABLE(acpi, ehl_pinctrl_acpi_match);
 
-static INTEL_PINCTRL_PM_OPS(ehl_pinctrl_pm_ops);
-
 static struct platform_driver ehl_pinctrl_driver = {
 	.probe = intel_pinctrl_probe_by_uid,
 	.driver = {
 		.name = "elkhartlake-pinctrl",
 		.acpi_match_table = ehl_pinctrl_acpi_match,
-		.pm = &ehl_pinctrl_pm_ops,
+		.pm = pm_sleep_ptr(&intel_pinctrl_pm_ops),
 	},
 };
 module_platform_driver(ehl_pinctrl_driver);
diff --git a/drivers/pinctrl/intel/pinctrl-emmitsburg.c b/drivers/pinctrl/intel/pinctrl-emmitsburg.c
index 099ec8351d5f..e4798d32492c 100644
--- a/drivers/pinctrl/intel/pinctrl-emmitsburg.c
+++ b/drivers/pinctrl/intel/pinctrl-emmitsburg.c
@@ -9,6 +9,7 @@
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/pm.h>
 
 #include <linux/pinctrl/pinctrl.h>
 
@@ -358,14 +359,12 @@ static const struct acpi_device_id ebg_pinctrl_acpi_match[] = {
 };
 MODULE_DEVICE_TABLE(acpi, ebg_pinctrl_acpi_match);
 
-static INTEL_PINCTRL_PM_OPS(ebg_pinctrl_pm_ops);
-
 static struct platform_driver ebg_pinctrl_driver = {
 	.probe = intel_pinctrl_probe_by_hid,
 	.driver = {
 		.name = "emmitsburg-pinctrl",
 		.acpi_match_table = ebg_pinctrl_acpi_match,
-		.pm = &ebg_pinctrl_pm_ops,
+		.pm = pm_sleep_ptr(&intel_pinctrl_pm_ops),
 	},
 };
 module_platform_driver(ebg_pinctrl_driver);
diff --git a/drivers/pinctrl/intel/pinctrl-geminilake.c b/drivers/pinctrl/intel/pinctrl-geminilake.c
index 9effa06b61e9..6dcf0ac2059f 100644
--- a/drivers/pinctrl/intel/pinctrl-geminilake.c
+++ b/drivers/pinctrl/intel/pinctrl-geminilake.c
@@ -9,6 +9,7 @@
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/pm.h>
 
 #include <linux/pinctrl/pinctrl.h>
 
@@ -447,14 +448,12 @@ static const struct acpi_device_id glk_pinctrl_acpi_match[] = {
 };
 MODULE_DEVICE_TABLE(acpi, glk_pinctrl_acpi_match);
 
-static INTEL_PINCTRL_PM_OPS(glk_pinctrl_pm_ops);
-
 static struct platform_driver glk_pinctrl_driver = {
 	.probe = intel_pinctrl_probe_by_uid,
 	.driver = {
 		.name = "geminilake-pinctrl",
 		.acpi_match_table = glk_pinctrl_acpi_match,
-		.pm = &glk_pinctrl_pm_ops,
+		.pm = pm_sleep_ptr(&intel_pinctrl_pm_ops),
 	},
 };
 
diff --git a/drivers/pinctrl/intel/pinctrl-icelake.c b/drivers/pinctrl/intel/pinctrl-icelake.c
index 300e1538c8d0..fe3042de891a 100644
--- a/drivers/pinctrl/intel/pinctrl-icelake.c
+++ b/drivers/pinctrl/intel/pinctrl-icelake.c
@@ -10,6 +10,7 @@
 #include <linux/acpi.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/pm.h>
 
 #include <linux/pinctrl/pinctrl.h>
 
@@ -668,8 +669,6 @@ static const struct intel_pinctrl_soc_data icln_soc_data = {
 	.ncommunities = ARRAY_SIZE(icln_communities),
 };
 
-static INTEL_PINCTRL_PM_OPS(icl_pinctrl_pm_ops);
-
 static const struct acpi_device_id icl_pinctrl_acpi_match[] = {
 	{ "INT3455", (kernel_ulong_t)&icllp_soc_data },
 	{ "INT34C3", (kernel_ulong_t)&icln_soc_data },
@@ -682,7 +681,7 @@ static struct platform_driver icl_pinctrl_driver = {
 	.driver = {
 		.name = "icelake-pinctrl",
 		.acpi_match_table = icl_pinctrl_acpi_match,
-		.pm = &icl_pinctrl_pm_ops,
+		.pm = pm_sleep_ptr(&intel_pinctrl_pm_ops),
 	},
 };
 module_platform_driver(icl_pinctrl_driver);
diff --git a/drivers/pinctrl/intel/pinctrl-intel-platform.c b/drivers/pinctrl/intel/pinctrl-intel-platform.c
new file mode 100644
index 000000000000..4a19ab3b4ba7
--- /dev/null
+++ b/drivers/pinctrl/intel/pinctrl-intel-platform.c
@@ -0,0 +1,225 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Intel PCH pinctrl/GPIO driver
+ *
+ * Copyright (C) 2021-2023, Intel Corporation
+ * Author: Andy Shevchenko <andriy.shevchenko@linux.intel.com>
+ */
+
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/property.h>
+#include <linux/string_helpers.h>
+
+#include <linux/pinctrl/pinctrl.h>
+
+#include "pinctrl-intel.h"
+
+struct intel_platform_pins {
+	struct pinctrl_pin_desc *pins;
+	size_t npins;
+};
+
+static int intel_platform_pinctrl_prepare_pins(struct device *dev, size_t base,
+					       const char *name, u32 size,
+					       struct intel_platform_pins *pins)
+{
+	struct pinctrl_pin_desc *descs;
+	char **pin_names;
+	unsigned int i;
+
+	pin_names = devm_kasprintf_strarray(dev, name, size);
+	if (IS_ERR(pin_names))
+		return PTR_ERR(pin_names);
+
+	descs = devm_krealloc_array(dev, pins->pins, base + size, sizeof(*descs), GFP_KERNEL);
+	if (!descs)
+		return -ENOMEM;
+
+	for (i = 0; i < size; i++) {
+		unsigned int pin_number = base + i;
+		char *pin_name = pin_names[i];
+		struct pinctrl_pin_desc *desc;
+
+		/* Unify delimiter for pin name */
+		strreplace(pin_name, '-', '_');
+
+		desc = &descs[pin_number];
+		desc->number = pin_number;
+		desc->name = pin_name;
+	}
+
+	pins->pins = descs;
+	pins->npins = base + size;
+
+	return 0;
+}
+
+static int intel_platform_pinctrl_prepare_group(struct device *dev,
+						struct fwnode_handle *child,
+						struct intel_padgroup *gpp,
+						struct intel_platform_pins *pins)
+{
+	size_t base = pins->npins;
+	const char *name;
+	u32 size;
+	int ret;
+
+	ret = fwnode_property_read_string(child, "intc-gpio-group-name", &name);
+	if (ret)
+		return ret;
+
+	ret = fwnode_property_read_u32(child, "intc-gpio-pad-count", &size);
+	if (ret)
+		return ret;
+
+	ret = intel_platform_pinctrl_prepare_pins(dev, base, name, size, pins);
+	if (ret)
+		return ret;
+
+	gpp->base = base;
+	gpp->size = size;
+	gpp->gpio_base = INTEL_GPIO_BASE_MATCH;
+
+	return 0;
+}
+
+static int intel_platform_pinctrl_prepare_community(struct device *dev,
+						    struct intel_community *community,
+						    struct intel_platform_pins *pins)
+{
+	struct fwnode_handle *child;
+	struct intel_padgroup *gpps;
+	unsigned int group;
+	size_t ngpps;
+	u32 offset;
+	int ret;
+
+	ret = device_property_read_u32(dev, "intc-gpio-pad-ownership-offset", &offset);
+	if (ret)
+		return ret;
+	community->padown_offset = offset;
+
+	ret = device_property_read_u32(dev, "intc-gpio-pad-configuration-lock-offset", &offset);
+	if (ret)
+		return ret;
+	community->padcfglock_offset = offset;
+
+	ret = device_property_read_u32(dev, "intc-gpio-host-software-pad-ownership-offset", &offset);
+	if (ret)
+		return ret;
+	community->hostown_offset = offset;
+
+	ret = device_property_read_u32(dev, "intc-gpio-gpi-interrupt-status-offset", &offset);
+	if (ret)
+		return ret;
+	community->is_offset = offset;
+
+	ret = device_property_read_u32(dev, "intc-gpio-gpi-interrupt-enable-offset", &offset);
+	if (ret)
+		return ret;
+	community->ie_offset = offset;
+
+	ngpps = device_get_child_node_count(dev);
+	if (!ngpps)
+		return -ENODEV;
+
+	gpps = devm_kcalloc(dev, ngpps, sizeof(*gpps), GFP_KERNEL);
+	if (!gpps)
+		return -ENOMEM;
+
+	group = 0;
+	device_for_each_child_node(dev, child) {
+		struct intel_padgroup *gpp = &gpps[group];
+
+		gpp->reg_num = group;
+
+		ret = intel_platform_pinctrl_prepare_group(dev, child, gpp, pins);
+		if (ret)
+			return ret;
+
+		group++;
+	}
+
+	community->ngpps = ngpps;
+	community->gpps = gpps;
+
+	return 0;
+}
+
+static int intel_platform_pinctrl_prepare_soc_data(struct device *dev,
+						   struct intel_pinctrl_soc_data *data)
+{
+	struct intel_platform_pins pins = {};
+	struct intel_community *communities;
+	size_t ncommunities;
+	unsigned int i;
+	int ret;
+
+	/* Version 1.0 of the specification assumes only a single community per device node */
+	ncommunities = 1,
+	communities = devm_kcalloc(dev, ncommunities, sizeof(*communities), GFP_KERNEL);
+	if (!communities)
+		return -ENOMEM;
+
+	for (i = 0; i < ncommunities; i++) {
+		struct intel_community *community = &communities[i];
+
+		community->barno = i;
+		community->pin_base = pins.npins;
+
+		ret = intel_platform_pinctrl_prepare_community(dev, community, &pins);
+		if (ret)
+			return ret;
+
+		community->npins = pins.npins - community->pin_base;
+	}
+
+	data->ncommunities = ncommunities;
+	data->communities = communities;
+
+	data->npins = pins.npins;
+	data->pins = pins.pins;
+
+	return 0;
+}
+
+static int intel_platform_pinctrl_probe(struct platform_device *pdev)
+{
+	struct intel_pinctrl_soc_data *data;
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	ret = intel_platform_pinctrl_prepare_soc_data(dev, data);
+	if (ret)
+		return ret;
+
+	return intel_pinctrl_probe(pdev, data);
+}
+
+static const struct acpi_device_id intel_platform_pinctrl_acpi_match[] = {
+	{ "INTC105F" },
+	{ }
+};
+MODULE_DEVICE_TABLE(acpi, intel_platform_pinctrl_acpi_match);
+
+static struct platform_driver intel_platform_pinctrl_driver = {
+	.probe = intel_platform_pinctrl_probe,
+	.driver = {
+		.name = "intel-pinctrl",
+		.acpi_match_table = intel_platform_pinctrl_acpi_match,
+		.pm = pm_sleep_ptr(&intel_pinctrl_pm_ops),
+	},
+};
+module_platform_driver(intel_platform_pinctrl_driver);
+
+MODULE_AUTHOR("Andy Shevchenko <andriy.shevchenko@linux.intel.com>");
+MODULE_DESCRIPTION("Intel PCH pinctrl/GPIO driver");
+MODULE_LICENSE("GPL v2");
+MODULE_IMPORT_NS(PINCTRL_INTEL);
diff --git a/drivers/pinctrl/intel/pinctrl-intel.c b/drivers/pinctrl/intel/pinctrl-intel.c
index 652ba451f885..2367c2747a83 100644
--- a/drivers/pinctrl/intel/pinctrl-intel.c
+++ b/drivers/pinctrl/intel/pinctrl-intel.c
@@ -677,10 +677,6 @@ static int intel_config_set_pull(struct intel_pinctrl *pctrl, unsigned int pin,
 	u32 term = 0, up = 0, value;
 	void __iomem *padcfg1;
 
-	/* Set default strength value in case none is given */
-	if (arg == 1)
-		arg = 5000;
-
 	switch (param) {
 	case PIN_CONFIG_BIAS_DISABLE:
 		break;
@@ -690,6 +686,7 @@ static int intel_config_set_pull(struct intel_pinctrl *pctrl, unsigned int pin,
 		case 20000:
 			term = PADCFG1_TERM_20K;
 			break;
+		case 1: /* Set default strength value in case none is given */
 		case 5000:
 			term = PADCFG1_TERM_5K;
 			break;
@@ -716,6 +713,7 @@ static int intel_config_set_pull(struct intel_pinctrl *pctrl, unsigned int pin,
 		case 20000:
 			term = PADCFG1_TERM_20K;
 			break;
+		case 1: /* Set default strength value in case none is given */
 		case 5000:
 			term = PADCFG1_TERM_5K;
 			break;
@@ -899,7 +897,7 @@ static int intel_gpio_to_pin(struct intel_pinctrl *pctrl, unsigned int offset,
  *
  * Return: a GPIO offset, or negative error code if translation can't be done.
  */
-static __maybe_unused int intel_pin_to_gpio(struct intel_pinctrl *pctrl, int pin)
+static int intel_pin_to_gpio(struct intel_pinctrl *pctrl, int pin)
 {
 	const struct intel_community *community;
 	const struct intel_padgroup *padgrp;
@@ -1506,8 +1504,8 @@ static int intel_pinctrl_probe_pwm(struct intel_pinctrl *pctrl,
 	return PTR_ERR_OR_ZERO(pwm);
 }
 
-static int intel_pinctrl_probe(struct platform_device *pdev,
-			       const struct intel_pinctrl_soc_data *soc_data)
+int intel_pinctrl_probe(struct platform_device *pdev,
+			const struct intel_pinctrl_soc_data *soc_data)
 {
 	struct device *dev = &pdev->dev;
 	struct intel_pinctrl *pctrl;
@@ -1625,6 +1623,7 @@ static int intel_pinctrl_probe(struct platform_device *pdev,
 
 	return 0;
 }
+EXPORT_SYMBOL_NS_GPL(intel_pinctrl_probe, PINCTRL_INTEL);
 
 int intel_pinctrl_probe_by_hid(struct platform_device *pdev)
 {
@@ -1653,7 +1652,7 @@ EXPORT_SYMBOL_NS_GPL(intel_pinctrl_probe_by_uid, PINCTRL_INTEL);
 const struct intel_pinctrl_soc_data *intel_pinctrl_get_soc_data(struct platform_device *pdev)
 {
 	const struct intel_pinctrl_soc_data * const *table;
-	const struct intel_pinctrl_soc_data *data = NULL;
+	const struct intel_pinctrl_soc_data *data;
 	struct device *dev = &pdev->dev;
 
 	table = device_get_match_data(dev);
@@ -1662,11 +1661,10 @@ const struct intel_pinctrl_soc_data *intel_pinctrl_get_soc_data(struct platform_
 		unsigned int i;
 
 		for (i = 0; table[i]; i++) {
-			if (acpi_dev_uid_match(adev, table[i]->uid)) {
-				data = table[i];
+			if (acpi_dev_uid_match(adev, table[i]->uid))
 				break;
-			}
 		}
+		data = table[i];
 	} else {
 		const struct platform_device_id *id;
 
@@ -1682,7 +1680,6 @@ const struct intel_pinctrl_soc_data *intel_pinctrl_get_soc_data(struct platform_
 }
 EXPORT_SYMBOL_NS_GPL(intel_pinctrl_get_soc_data, PINCTRL_INTEL);
 
-#ifdef CONFIG_PM_SLEEP
 static bool __intel_gpio_is_direct_irq(u32 value)
 {
 	return (value & PADCFG0_GPIROUTIOXAPIC) && (value & PADCFG0_GPIOTXDIS) &&
@@ -1728,7 +1725,7 @@ static bool intel_pinctrl_should_save(struct intel_pinctrl *pctrl, unsigned int
 	return false;
 }
 
-int intel_pinctrl_suspend_noirq(struct device *dev)
+static int intel_pinctrl_suspend_noirq(struct device *dev)
 {
 	struct intel_pinctrl *pctrl = dev_get_drvdata(dev);
 	struct intel_community_context *communities;
@@ -1771,7 +1768,6 @@ int intel_pinctrl_suspend_noirq(struct device *dev)
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(intel_pinctrl_suspend_noirq);
 
 static bool intel_gpio_update_reg(void __iomem *reg, u32 mask, u32 value)
 {
@@ -1838,7 +1834,7 @@ static void intel_restore_padcfg(struct intel_pinctrl *pctrl, unsigned int pin,
 	dev_dbg(dev, "restored pin %u padcfg%u %#08x\n", pin, n, readl(padcfg));
 }
 
-int intel_pinctrl_resume_noirq(struct device *dev)
+static int intel_pinctrl_resume_noirq(struct device *dev)
 {
 	struct intel_pinctrl *pctrl = dev_get_drvdata(dev);
 	const struct intel_community_context *communities;
@@ -1882,8 +1878,10 @@ int intel_pinctrl_resume_noirq(struct device *dev)
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(intel_pinctrl_resume_noirq);
-#endif
+
+EXPORT_NS_GPL_DEV_PM_OPS(intel_pinctrl_pm_ops, PINCTRL_INTEL) = {
+	NOIRQ_SYSTEM_SLEEP_PM_OPS(intel_pinctrl_suspend_noirq, intel_pinctrl_resume_noirq)
+};
 
 MODULE_AUTHOR("Mathias Nyman <mathias.nyman@linux.intel.com>");
 MODULE_AUTHOR("Mika Westerberg <mika.westerberg@linux.intel.com>");
diff --git a/drivers/pinctrl/intel/pinctrl-intel.h b/drivers/pinctrl/intel/pinctrl-intel.h
index 2bb553598e8b..fde65e18cd14 100644
--- a/drivers/pinctrl/intel/pinctrl-intel.h
+++ b/drivers/pinctrl/intel/pinctrl-intel.h
@@ -252,19 +252,13 @@ struct intel_pinctrl {
 	int irq;
 };
 
+int intel_pinctrl_probe(struct platform_device *pdev,
+			const struct intel_pinctrl_soc_data *soc_data);
+
 int intel_pinctrl_probe_by_hid(struct platform_device *pdev);
 int intel_pinctrl_probe_by_uid(struct platform_device *pdev);
 
-#ifdef CONFIG_PM_SLEEP
-int intel_pinctrl_suspend_noirq(struct device *dev);
-int intel_pinctrl_resume_noirq(struct device *dev);
-#endif
-
-#define INTEL_PINCTRL_PM_OPS(_name)					\
-const struct dev_pm_ops _name = {					\
-	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(intel_pinctrl_suspend_noirq,	\
-				      intel_pinctrl_resume_noirq)	\
-}
+extern const struct dev_pm_ops intel_pinctrl_pm_ops;
 
 struct intel_community *intel_get_community(struct intel_pinctrl *pctrl, unsigned int pin);
 
diff --git a/drivers/pinctrl/intel/pinctrl-jasperlake.c b/drivers/pinctrl/intel/pinctrl-jasperlake.c
index 50f137deed9c..3525480428ea 100644
--- a/drivers/pinctrl/intel/pinctrl-jasperlake.c
+++ b/drivers/pinctrl/intel/pinctrl-jasperlake.c
@@ -9,6 +9,7 @@
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/pm.h>
 
 #include <linux/pinctrl/pinctrl.h>
 
@@ -326,14 +327,12 @@ static const struct acpi_device_id jsl_pinctrl_acpi_match[] = {
 };
 MODULE_DEVICE_TABLE(acpi, jsl_pinctrl_acpi_match);
 
-static INTEL_PINCTRL_PM_OPS(jsl_pinctrl_pm_ops);
-
 static struct platform_driver jsl_pinctrl_driver = {
 	.probe = intel_pinctrl_probe_by_hid,
 	.driver = {
 		.name = "jasperlake-pinctrl",
 		.acpi_match_table = jsl_pinctrl_acpi_match,
-		.pm = &jsl_pinctrl_pm_ops,
+		.pm = pm_sleep_ptr(&intel_pinctrl_pm_ops),
 	},
 };
 module_platform_driver(jsl_pinctrl_driver);
diff --git a/drivers/pinctrl/intel/pinctrl-lakefield.c b/drivers/pinctrl/intel/pinctrl-lakefield.c
index 0b94e11b78ac..adef85db82ca 100644
--- a/drivers/pinctrl/intel/pinctrl-lakefield.c
+++ b/drivers/pinctrl/intel/pinctrl-lakefield.c
@@ -9,6 +9,7 @@
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/pm.h>
 
 #include <linux/pinctrl/pinctrl.h>
 
@@ -347,14 +348,12 @@ static const struct acpi_device_id lkf_pinctrl_acpi_match[] = {
 };
 MODULE_DEVICE_TABLE(acpi, lkf_pinctrl_acpi_match);
 
-static INTEL_PINCTRL_PM_OPS(lkf_pinctrl_pm_ops);
-
 static struct platform_driver lkf_pinctrl_driver = {
 	.probe = intel_pinctrl_probe_by_hid,
 	.driver = {
 		.name = "lakefield-pinctrl",
 		.acpi_match_table = lkf_pinctrl_acpi_match,
-		.pm = &lkf_pinctrl_pm_ops,
+		.pm = pm_sleep_ptr(&intel_pinctrl_pm_ops),
 	},
 };
 module_platform_driver(lkf_pinctrl_driver);
diff --git a/drivers/pinctrl/intel/pinctrl-lewisburg.c b/drivers/pinctrl/intel/pinctrl-lewisburg.c
index aa725a5d62b9..a304d30ea9ed 100644
--- a/drivers/pinctrl/intel/pinctrl-lewisburg.c
+++ b/drivers/pinctrl/intel/pinctrl-lewisburg.c
@@ -9,6 +9,7 @@
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/pm.h>
 
 #include <linux/pinctrl/pinctrl.h>
 
@@ -301,8 +302,6 @@ static const struct intel_pinctrl_soc_data lbg_soc_data = {
 	.ncommunities = ARRAY_SIZE(lbg_communities),
 };
 
-static INTEL_PINCTRL_PM_OPS(lbg_pinctrl_pm_ops);
-
 static const struct acpi_device_id lbg_pinctrl_acpi_match[] = {
 	{ "INT3536", (kernel_ulong_t)&lbg_soc_data },
 	{ }
@@ -314,7 +313,7 @@ static struct platform_driver lbg_pinctrl_driver = {
 	.driver = {
 		.name = "lewisburg-pinctrl",
 		.acpi_match_table = lbg_pinctrl_acpi_match,
-		.pm = &lbg_pinctrl_pm_ops,
+		.pm = pm_sleep_ptr(&intel_pinctrl_pm_ops),
 	},
 };
 module_platform_driver(lbg_pinctrl_driver);
diff --git a/drivers/pinctrl/intel/pinctrl-meteorlake.c b/drivers/pinctrl/intel/pinctrl-meteorlake.c
index 7ced2b402dce..cc44890c6699 100644
--- a/drivers/pinctrl/intel/pinctrl-meteorlake.c
+++ b/drivers/pinctrl/intel/pinctrl-meteorlake.c
@@ -9,6 +9,7 @@
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/pm.h>
 
 #include <linux/pinctrl/pinctrl.h>
 
@@ -589,14 +590,12 @@ static const struct acpi_device_id mtl_pinctrl_acpi_match[] = {
 };
 MODULE_DEVICE_TABLE(acpi, mtl_pinctrl_acpi_match);
 
-static INTEL_PINCTRL_PM_OPS(mtl_pinctrl_pm_ops);
-
 static struct platform_driver mtl_pinctrl_driver = {
 	.probe = intel_pinctrl_probe_by_hid,
 	.driver = {
 		.name = "meteorlake-pinctrl",
 		.acpi_match_table = mtl_pinctrl_acpi_match,
-		.pm = &mtl_pinctrl_pm_ops,
+		.pm = pm_sleep_ptr(&intel_pinctrl_pm_ops),
 	},
 };
 module_platform_driver(mtl_pinctrl_driver);
diff --git a/drivers/pinctrl/intel/pinctrl-sunrisepoint.c b/drivers/pinctrl/intel/pinctrl-sunrisepoint.c
index b7a40ab0bca8..55df9d2cfb1b 100644
--- a/drivers/pinctrl/intel/pinctrl-sunrisepoint.c
+++ b/drivers/pinctrl/intel/pinctrl-sunrisepoint.c
@@ -10,6 +10,7 @@
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/pm.h>
 
 #include <linux/pinctrl/pinctrl.h>
 
@@ -579,14 +580,12 @@ static const struct acpi_device_id spt_pinctrl_acpi_match[] = {
 };
 MODULE_DEVICE_TABLE(acpi, spt_pinctrl_acpi_match);
 
-static INTEL_PINCTRL_PM_OPS(spt_pinctrl_pm_ops);
-
 static struct platform_driver spt_pinctrl_driver = {
 	.probe = intel_pinctrl_probe_by_hid,
 	.driver = {
 		.name = "sunrisepoint-pinctrl",
 		.acpi_match_table = spt_pinctrl_acpi_match,
-		.pm = &spt_pinctrl_pm_ops,
+		.pm = pm_sleep_ptr(&intel_pinctrl_pm_ops),
 	},
 };
 
diff --git a/drivers/pinctrl/intel/pinctrl-tangier.c b/drivers/pinctrl/intel/pinctrl-tangier.c
index 40dd60c9e526..26e34ec0a972 100644
--- a/drivers/pinctrl/intel/pinctrl-tangier.c
+++ b/drivers/pinctrl/intel/pinctrl-tangier.c
@@ -368,20 +368,20 @@ static int tng_config_set_pin(struct tng_pinctrl *tp, unsigned int pin,
 		break;
 
 	case PIN_CONFIG_BIAS_PULL_UP:
-		/* Set default strength value in case none is given */
-		if (arg == 1)
-			arg = 20000;
-
 		switch (arg) {
 		case 50000:
 			term = BUFCFG_PUPD_VAL_50K;
 			break;
+		case 1: /* Set default strength value in case none is given */
 		case 20000:
 			term = BUFCFG_PUPD_VAL_20K;
 			break;
 		case 2000:
 			term = BUFCFG_PUPD_VAL_2K;
 			break;
+		case 910:
+			term = BUFCFG_PUPD_VAL_910;
+			break;
 		default:
 			return -EINVAL;
 		}
@@ -391,20 +391,20 @@ static int tng_config_set_pin(struct tng_pinctrl *tp, unsigned int pin,
 		break;
 
 	case PIN_CONFIG_BIAS_PULL_DOWN:
-		/* Set default strength value in case none is given */
-		if (arg == 1)
-			arg = 20000;
-
 		switch (arg) {
 		case 50000:
 			term = BUFCFG_PUPD_VAL_50K;
 			break;
+		case 1: /* Set default strength value in case none is given */
 		case 20000:
 			term = BUFCFG_PUPD_VAL_20K;
 			break;
 		case 2000:
 			term = BUFCFG_PUPD_VAL_2K;
 			break;
+		case 910:
+			term = BUFCFG_PUPD_VAL_910;
+			break;
 		default:
 			return -EINVAL;
 		}
diff --git a/drivers/pinctrl/intel/pinctrl-tigerlake.c b/drivers/pinctrl/intel/pinctrl-tigerlake.c
index 4768a69a9258..80cd7a06fe5a 100644
--- a/drivers/pinctrl/intel/pinctrl-tigerlake.c
+++ b/drivers/pinctrl/intel/pinctrl-tigerlake.c
@@ -10,6 +10,7 @@
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/pm.h>
 
 #include <linux/pinctrl/pinctrl.h>
 
@@ -743,14 +744,12 @@ static const struct acpi_device_id tgl_pinctrl_acpi_match[] = {
 };
 MODULE_DEVICE_TABLE(acpi, tgl_pinctrl_acpi_match);
 
-static INTEL_PINCTRL_PM_OPS(tgl_pinctrl_pm_ops);
-
 static struct platform_driver tgl_pinctrl_driver = {
 	.probe = intel_pinctrl_probe_by_hid,
 	.driver = {
 		.name = "tigerlake-pinctrl",
 		.acpi_match_table = tgl_pinctrl_acpi_match,
-		.pm = &tgl_pinctrl_pm_ops,
+		.pm = pm_sleep_ptr(&intel_pinctrl_pm_ops),
 	},
 };
 module_platform_driver(tgl_pinctrl_driver);
diff --git a/drivers/pinctrl/nxp/pinctrl-s32cc.c b/drivers/pinctrl/nxp/pinctrl-s32cc.c
index 7daff9f186cd..f0cad2c501f7 100644
--- a/drivers/pinctrl/nxp/pinctrl-s32cc.c
+++ b/drivers/pinctrl/nxp/pinctrl-s32cc.c
@@ -843,8 +843,8 @@ static int s32_pinctrl_probe_dt(struct platform_device *pdev,
 	if (!np)
 		return -ENODEV;
 
-	if (mem_regions == 0) {
-		dev_err(&pdev->dev, "mem_regions is 0\n");
+	if (mem_regions == 0 || mem_regions >= 10000) {
+		dev_err(&pdev->dev, "mem_regions is invalid: %u\n", mem_regions);
 		return -EINVAL;
 	}
 
diff --git a/drivers/pinctrl/pinctrl-cy8c95x0.c b/drivers/pinctrl/pinctrl-cy8c95x0.c
index 04285c930e94..4ccfa99ed93a 100644
--- a/drivers/pinctrl/pinctrl-cy8c95x0.c
+++ b/drivers/pinctrl/pinctrl-cy8c95x0.c
@@ -143,6 +143,7 @@ static const struct dmi_system_id cy8c95x0_dmi_acpi_irq_info[] = {
  * @pinctrl_desc:   pin controller description
  * @name:           Chip controller name
  * @tpin:           Total number of pins
+ * @gpio_reset:     GPIO line handler that can reset the IC
  */
 struct cy8c95x0_pinctrl {
 	struct regmap *regmap;
diff --git a/drivers/pinctrl/pinctrl-tps6594.c b/drivers/pinctrl/pinctrl-tps6594.c
new file mode 100644
index 000000000000..66985e54b74a
--- /dev/null
+++ b/drivers/pinctrl/pinctrl-tps6594.c
@@ -0,0 +1,373 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Pinmux and GPIO driver for tps6594 PMIC
+ *
+ * Copyright (C) 2023 BayLibre Incorporated - https://www.baylibre.com/
+ */
+
+#include <linux/gpio/driver.h>
+#include <linux/gpio/regmap.h>
+#include <linux/module.h>
+#include <linux/pinctrl/pinmux.h>
+#include <linux/platform_device.h>
+#include <linux/mod_devicetable.h>
+
+#include <linux/mfd/tps6594.h>
+
+#define TPS6594_PINCTRL_PINS_NB 11
+
+#define TPS6594_PINCTRL_GPIO_FUNCTION 0
+#define TPS6594_PINCTRL_SCL_I2C2_CS_SPI_FUNCTION 1
+#define TPS6594_PINCTRL_TRIG_WDOG_FUNCTION 1
+#define TPS6594_PINCTRL_CLK32KOUT_FUNCTION 1
+#define TPS6594_PINCTRL_SCLK_SPMI_FUNCTION 1
+#define TPS6594_PINCTRL_SDATA_SPMI_FUNCTION 1
+#define TPS6594_PINCTRL_NERR_MCU_FUNCTION 1
+#define TPS6594_PINCTRL_PDOG_FUNCTION 1
+#define TPS6594_PINCTRL_SYNCCLKIN_FUNCTION 1
+#define TPS6594_PINCTRL_NRSTOUT_SOC_FUNCTION 2
+#define TPS6594_PINCTRL_SYNCCLKOUT_FUNCTION 2
+#define TPS6594_PINCTRL_SDA_I2C2_SDO_SPI_FUNCTION 2
+#define TPS6594_PINCTRL_NERR_SOC_FUNCTION 2
+#define TPS6594_PINCTRL_DISABLE_WDOG_FUNCTION 3
+#define TPS6594_PINCTRL_NSLEEP1_FUNCTION 4
+#define TPS6594_PINCTRL_NSLEEP2_FUNCTION 5
+#define TPS6594_PINCTRL_WKUP1_FUNCTION 6
+#define TPS6594_PINCTRL_WKUP2_FUNCTION 7
+
+/* Special muxval for recalcitrant pins */
+#define TPS6594_PINCTRL_DISABLE_WDOG_FUNCTION_GPIO8 2
+#define TPS6594_PINCTRL_SYNCCLKOUT_FUNCTION_GPIO8 3
+#define TPS6594_PINCTRL_CLK32KOUT_FUNCTION_GPIO9 3
+
+#define TPS6594_OFFSET_GPIO_SEL 5
+
+#define FUNCTION(fname, v)									\
+{											\
+	.pinfunction = PINCTRL_PINFUNCTION(#fname,					\
+					tps6594_##fname##_func_group_names,		\
+					ARRAY_SIZE(tps6594_##fname##_func_group_names)),\
+	.muxval = v,									\
+}
+
+static const struct pinctrl_pin_desc tps6594_pins[TPS6594_PINCTRL_PINS_NB] = {
+	PINCTRL_PIN(0, "GPIO0"),   PINCTRL_PIN(1, "GPIO1"),
+	PINCTRL_PIN(2, "GPIO2"),   PINCTRL_PIN(3, "GPIO3"),
+	PINCTRL_PIN(4, "GPIO4"),   PINCTRL_PIN(5, "GPIO5"),
+	PINCTRL_PIN(6, "GPIO6"),   PINCTRL_PIN(7, "GPIO7"),
+	PINCTRL_PIN(8, "GPIO8"),   PINCTRL_PIN(9, "GPIO9"),
+	PINCTRL_PIN(10, "GPIO10"),
+};
+
+static const char *const tps6594_gpio_func_group_names[] = {
+	"GPIO0", "GPIO1", "GPIO2", "GPIO3", "GPIO4", "GPIO5",
+	"GPIO6", "GPIO7", "GPIO8", "GPIO9", "GPIO10",
+};
+
+static const char *const tps6594_nsleep1_func_group_names[] = {
+	"GPIO0", "GPIO1", "GPIO2", "GPIO3", "GPIO4", "GPIO5",
+	"GPIO6", "GPIO7", "GPIO8", "GPIO9", "GPIO10",
+};
+
+static const char *const tps6594_nsleep2_func_group_names[] = {
+	"GPIO0", "GPIO1", "GPIO2", "GPIO3", "GPIO4", "GPIO5",
+	"GPIO6", "GPIO7", "GPIO8", "GPIO9", "GPIO10",
+};
+
+static const char *const tps6594_wkup1_func_group_names[] = {
+	"GPIO0", "GPIO1", "GPIO2", "GPIO3", "GPIO4", "GPIO5",
+	"GPIO6", "GPIO7", "GPIO8", "GPIO9", "GPIO10",
+};
+
+static const char *const tps6594_wkup2_func_group_names[] = {
+	"GPIO0", "GPIO1", "GPIO2", "GPIO3", "GPIO4", "GPIO5",
+	"GPIO6", "GPIO7", "GPIO8", "GPIO9", "GPIO10",
+};
+
+static const char *const tps6594_scl_i2c2_cs_spi_func_group_names[] = {
+	"GPIO0",
+	"GPIO1",
+};
+
+static const char *const tps6594_nrstout_soc_func_group_names[] = {
+	"GPIO0",
+	"GPIO10",
+};
+
+static const char *const tps6594_trig_wdog_func_group_names[] = {
+	"GPIO1",
+	"GPIO10",
+};
+
+static const char *const tps6594_sda_i2c2_sdo_spi_func_group_names[] = {
+	"GPIO1",
+};
+
+static const char *const tps6594_clk32kout_func_group_names[] = {
+	"GPIO2",
+	"GPIO3",
+	"GPIO7",
+};
+
+static const char *const tps6594_nerr_soc_func_group_names[] = {
+	"GPIO2",
+};
+
+static const char *const tps6594_sclk_spmi_func_group_names[] = {
+	"GPIO4",
+};
+
+static const char *const tps6594_sdata_spmi_func_group_names[] = {
+	"GPIO5",
+};
+
+static const char *const tps6594_nerr_mcu_func_group_names[] = {
+	"GPIO6",
+};
+
+static const char *const tps6594_syncclkout_func_group_names[] = {
+	"GPIO7",
+	"GPIO9",
+};
+
+static const char *const tps6594_disable_wdog_func_group_names[] = {
+	"GPIO7",
+	"GPIO8",
+};
+
+static const char *const tps6594_pdog_func_group_names[] = {
+	"GPIO8",
+};
+
+static const char *const tps6594_syncclkin_func_group_names[] = {
+	"GPIO9",
+};
+
+struct tps6594_pinctrl_function {
+	struct pinfunction pinfunction;
+	u8 muxval;
+};
+
+static const struct tps6594_pinctrl_function pinctrl_functions[] = {
+	FUNCTION(gpio, TPS6594_PINCTRL_GPIO_FUNCTION),
+	FUNCTION(nsleep1, TPS6594_PINCTRL_NSLEEP1_FUNCTION),
+	FUNCTION(nsleep2, TPS6594_PINCTRL_NSLEEP2_FUNCTION),
+	FUNCTION(wkup1, TPS6594_PINCTRL_WKUP1_FUNCTION),
+	FUNCTION(wkup2, TPS6594_PINCTRL_WKUP2_FUNCTION),
+	FUNCTION(scl_i2c2_cs_spi, TPS6594_PINCTRL_SCL_I2C2_CS_SPI_FUNCTION),
+	FUNCTION(nrstout_soc, TPS6594_PINCTRL_NRSTOUT_SOC_FUNCTION),
+	FUNCTION(trig_wdog, TPS6594_PINCTRL_TRIG_WDOG_FUNCTION),
+	FUNCTION(sda_i2c2_sdo_spi, TPS6594_PINCTRL_SDA_I2C2_SDO_SPI_FUNCTION),
+	FUNCTION(clk32kout, TPS6594_PINCTRL_CLK32KOUT_FUNCTION),
+	FUNCTION(nerr_soc, TPS6594_PINCTRL_NERR_SOC_FUNCTION),
+	FUNCTION(sclk_spmi, TPS6594_PINCTRL_SCLK_SPMI_FUNCTION),
+	FUNCTION(sdata_spmi, TPS6594_PINCTRL_SDATA_SPMI_FUNCTION),
+	FUNCTION(nerr_mcu, TPS6594_PINCTRL_NERR_MCU_FUNCTION),
+	FUNCTION(syncclkout, TPS6594_PINCTRL_SYNCCLKOUT_FUNCTION),
+	FUNCTION(disable_wdog, TPS6594_PINCTRL_DISABLE_WDOG_FUNCTION),
+	FUNCTION(pdog, TPS6594_PINCTRL_PDOG_FUNCTION),
+	FUNCTION(syncclkin, TPS6594_PINCTRL_SYNCCLKIN_FUNCTION),
+};
+
+struct tps6594_pinctrl {
+	struct tps6594 *tps;
+	struct gpio_regmap *gpio_regmap;
+	struct pinctrl_dev *pctl_dev;
+	const struct tps6594_pinctrl_function *funcs;
+	const struct pinctrl_pin_desc *pins;
+};
+
+static int tps6594_gpio_regmap_xlate(struct gpio_regmap *gpio,
+				     unsigned int base, unsigned int offset,
+				     unsigned int *reg, unsigned int *mask)
+{
+	unsigned int line = offset % 8;
+	unsigned int stride = offset / 8;
+
+	switch (base) {
+	case TPS6594_REG_GPIOX_CONF(0):
+		*reg = TPS6594_REG_GPIOX_CONF(offset);
+		*mask = TPS6594_BIT_GPIO_DIR;
+		return 0;
+	case TPS6594_REG_GPIO_IN_1:
+	case TPS6594_REG_GPIO_OUT_1:
+		*reg = base + stride;
+		*mask = BIT(line);
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int tps6594_pmx_func_cnt(struct pinctrl_dev *pctldev)
+{
+	return ARRAY_SIZE(pinctrl_functions);
+}
+
+static const char *tps6594_pmx_func_name(struct pinctrl_dev *pctldev,
+					 unsigned int selector)
+{
+	struct tps6594_pinctrl *pinctrl = pinctrl_dev_get_drvdata(pctldev);
+
+	return pinctrl->funcs[selector].pinfunction.name;
+}
+
+static int tps6594_pmx_func_groups(struct pinctrl_dev *pctldev,
+				   unsigned int selector,
+				   const char *const **groups,
+				   unsigned int *num_groups)
+{
+	struct tps6594_pinctrl *pinctrl = pinctrl_dev_get_drvdata(pctldev);
+
+	*groups = pinctrl->funcs[selector].pinfunction.groups;
+	*num_groups = pinctrl->funcs[selector].pinfunction.ngroups;
+
+	return 0;
+}
+
+static int tps6594_pmx_set(struct tps6594_pinctrl *pinctrl, unsigned int pin,
+			   u8 muxval)
+{
+	u8 mux_sel_val = muxval << TPS6594_OFFSET_GPIO_SEL;
+
+	return regmap_update_bits(pinctrl->tps->regmap,
+				  TPS6594_REG_GPIOX_CONF(pin),
+				  TPS6594_MASK_GPIO_SEL, mux_sel_val);
+}
+
+static int tps6594_pmx_set_mux(struct pinctrl_dev *pctldev,
+			       unsigned int function, unsigned int group)
+{
+	struct tps6594_pinctrl *pinctrl = pinctrl_dev_get_drvdata(pctldev);
+	u8 muxval = pinctrl->funcs[function].muxval;
+
+	/* Some pins don't have the same muxval for the same function... */
+	if (group == 8) {
+		if (muxval == TPS6594_PINCTRL_DISABLE_WDOG_FUNCTION)
+			muxval = TPS6594_PINCTRL_DISABLE_WDOG_FUNCTION_GPIO8;
+		else if (muxval == TPS6594_PINCTRL_SYNCCLKOUT_FUNCTION)
+			muxval = TPS6594_PINCTRL_SYNCCLKOUT_FUNCTION_GPIO8;
+	} else if (group == 9) {
+		if (muxval == TPS6594_PINCTRL_CLK32KOUT_FUNCTION)
+			muxval = TPS6594_PINCTRL_CLK32KOUT_FUNCTION_GPIO9;
+	}
+
+	return tps6594_pmx_set(pinctrl, group, muxval);
+}
+
+static int tps6594_pmx_gpio_set_direction(struct pinctrl_dev *pctldev,
+					  struct pinctrl_gpio_range *range,
+					  unsigned int offset, bool input)
+{
+	struct tps6594_pinctrl *pinctrl = pinctrl_dev_get_drvdata(pctldev);
+	u8 muxval = pinctrl->funcs[TPS6594_PINCTRL_GPIO_FUNCTION].muxval;
+
+	return tps6594_pmx_set(pinctrl, offset, muxval);
+}
+
+static const struct pinmux_ops tps6594_pmx_ops = {
+	.get_functions_count = tps6594_pmx_func_cnt,
+	.get_function_name = tps6594_pmx_func_name,
+	.get_function_groups = tps6594_pmx_func_groups,
+	.set_mux = tps6594_pmx_set_mux,
+	.gpio_set_direction = tps6594_pmx_gpio_set_direction,
+	.strict = true,
+};
+
+static int tps6594_groups_cnt(struct pinctrl_dev *pctldev)
+{
+	return ARRAY_SIZE(tps6594_pins);
+}
+
+static int tps6594_group_pins(struct pinctrl_dev *pctldev,
+			      unsigned int selector, const unsigned int **pins,
+			      unsigned int *num_pins)
+{
+	struct tps6594_pinctrl *pinctrl = pinctrl_dev_get_drvdata(pctldev);
+
+	*pins = &pinctrl->pins[selector].number;
+	*num_pins = 1;
+
+	return 0;
+}
+
+static const char *tps6594_group_name(struct pinctrl_dev *pctldev,
+				      unsigned int selector)
+{
+	struct tps6594_pinctrl *pinctrl = pinctrl_dev_get_drvdata(pctldev);
+
+	return pinctrl->pins[selector].name;
+}
+
+static const struct pinctrl_ops tps6594_pctrl_ops = {
+	.dt_node_to_map = pinconf_generic_dt_node_to_map_group,
+	.dt_free_map = pinconf_generic_dt_free_map,
+	.get_groups_count = tps6594_groups_cnt,
+	.get_group_name = tps6594_group_name,
+	.get_group_pins = tps6594_group_pins,
+};
+
+static int tps6594_pinctrl_probe(struct platform_device *pdev)
+{
+	struct tps6594 *tps = dev_get_drvdata(pdev->dev.parent);
+	struct device *dev = &pdev->dev;
+	struct tps6594_pinctrl *pinctrl;
+	struct pinctrl_desc *pctrl_desc;
+	struct gpio_regmap_config config = {};
+
+	pctrl_desc = devm_kzalloc(dev, sizeof(*pctrl_desc), GFP_KERNEL);
+	if (!pctrl_desc)
+		return -ENOMEM;
+	pctrl_desc->name = dev_name(dev);
+	pctrl_desc->owner = THIS_MODULE;
+	pctrl_desc->pins = tps6594_pins;
+	pctrl_desc->npins = ARRAY_SIZE(tps6594_pins);
+	pctrl_desc->pctlops = &tps6594_pctrl_ops;
+	pctrl_desc->pmxops = &tps6594_pmx_ops;
+
+	pinctrl = devm_kzalloc(dev, sizeof(*pinctrl), GFP_KERNEL);
+	if (!pinctrl)
+		return -ENOMEM;
+	pinctrl->tps = dev_get_drvdata(dev->parent);
+	pinctrl->funcs = pinctrl_functions;
+	pinctrl->pins = tps6594_pins;
+	pinctrl->pctl_dev = devm_pinctrl_register(dev, pctrl_desc, pinctrl);
+	if (IS_ERR(pinctrl->pctl_dev))
+		return dev_err_probe(dev, PTR_ERR(pinctrl->pctl_dev),
+				     "Couldn't register pinctrl driver\n");
+
+	config.parent = tps->dev;
+	config.regmap = tps->regmap;
+	config.ngpio = TPS6594_PINCTRL_PINS_NB;
+	config.ngpio_per_reg = 8;
+	config.reg_dat_base = TPS6594_REG_GPIO_IN_1;
+	config.reg_set_base = TPS6594_REG_GPIO_OUT_1;
+	config.reg_dir_out_base = TPS6594_REG_GPIOX_CONF(0);
+	config.reg_mask_xlate = tps6594_gpio_regmap_xlate;
+
+	pinctrl->gpio_regmap = devm_gpio_regmap_register(dev, &config);
+	if (IS_ERR(pinctrl->gpio_regmap))
+		return dev_err_probe(dev, PTR_ERR(pinctrl->gpio_regmap),
+				     "Couldn't register gpio_regmap driver\n");
+
+	return 0;
+}
+
+static const struct platform_device_id tps6594_pinctrl_id_table[] = {
+	{ "tps6594-pinctrl", },
+	{}
+};
+MODULE_DEVICE_TABLE(platform, tps6594_pinctrl_id_table);
+
+static struct platform_driver tps6594_pinctrl_driver = {
+	.probe = tps6594_pinctrl_probe,
+	.driver = {
+		.name = "tps6594-pinctrl",
+	},
+	.id_table = tps6594_pinctrl_id_table,
+};
+module_platform_driver(tps6594_pinctrl_driver);
+
+MODULE_AUTHOR("Esteban Blanc <eblanc@baylibre.com>");
+MODULE_DESCRIPTION("TPS6594 pinctrl and GPIO driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/qcom/Kconfig b/drivers/pinctrl/qcom/Kconfig
index f84c0d3b7951..e0f2829c15d6 100644
--- a/drivers/pinctrl/qcom/Kconfig
+++ b/drivers/pinctrl/qcom/Kconfig
@@ -124,4 +124,14 @@ config PINCTRL_SM8550_LPASS_LPI
 	  (Low Power Island) found on the Qualcomm Technologies Inc SM8550
 	  platform.
 
+config PINCTRL_SM8650_LPASS_LPI
+	tristate "Qualcomm Technologies Inc SM8550 LPASS LPI pin controller driver"
+	depends on ARM64 || COMPILE_TEST
+	depends on PINCTRL_LPASS_LPI
+	help
+	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
+	  Qualcomm Technologies Inc LPASS (Low Power Audio SubSystem) LPI
+	  (Low Power Island) found on the Qualcomm Technologies Inc SM8650
+	  platform.
+
 endif
diff --git a/drivers/pinctrl/qcom/Kconfig.msm b/drivers/pinctrl/qcom/Kconfig.msm
index 01dd7b134354..341491de406b 100644
--- a/drivers/pinctrl/qcom/Kconfig.msm
+++ b/drivers/pinctrl/qcom/Kconfig.msm
@@ -366,4 +366,12 @@ config PINCTRL_SM8550
 	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	  Technologies Inc SM8550 platform.
 
+config PINCTRL_SM8650
+	tristate "Qualcomm Technologies Inc SM8650 pin controller driver"
+	depends on ARM64 || COMPILE_TEST
+	help
+	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
+	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
+	  Technologies Inc SM8650 platform.
+
 endif
diff --git a/drivers/pinctrl/qcom/Makefile b/drivers/pinctrl/qcom/Makefile
index 5910e08c84ce..9bcc1b63e857 100644
--- a/drivers/pinctrl/qcom/Makefile
+++ b/drivers/pinctrl/qcom/Makefile
@@ -58,5 +58,7 @@ obj-$(CONFIG_PINCTRL_SM8450) += pinctrl-sm8450.o
 obj-$(CONFIG_PINCTRL_SM8450_LPASS_LPI) += pinctrl-sm8450-lpass-lpi.o
 obj-$(CONFIG_PINCTRL_SM8550) += pinctrl-sm8550.o
 obj-$(CONFIG_PINCTRL_SM8550_LPASS_LPI) += pinctrl-sm8550-lpass-lpi.o
+obj-$(CONFIG_PINCTRL_SM8650) += pinctrl-sm8650.o
+obj-$(CONFIG_PINCTRL_SM8650_LPASS_LPI) += pinctrl-sm8650-lpass-lpi.o
 obj-$(CONFIG_PINCTRL_SC8280XP_LPASS_LPI) += pinctrl-sc8280xp-lpass-lpi.o
 obj-$(CONFIG_PINCTRL_LPASS_LPI) += pinctrl-lpass-lpi.o
diff --git a/drivers/pinctrl/qcom/pinctrl-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-lpass-lpi.c
index 9651aed048cf..9e410a281bfa 100644
--- a/drivers/pinctrl/qcom/pinctrl-lpass-lpi.c
+++ b/drivers/pinctrl/qcom/pinctrl-lpass-lpi.c
@@ -186,6 +186,41 @@ static int lpi_config_get(struct pinctrl_dev *pctldev,
 	return 0;
 }
 
+static int lpi_config_set_slew_rate(struct lpi_pinctrl *pctrl,
+				    const struct lpi_pingroup *g,
+				    unsigned int group, unsigned int slew)
+{
+	unsigned long sval;
+	void __iomem *reg;
+	int slew_offset;
+
+	if (slew > LPI_SLEW_RATE_MAX) {
+		dev_err(pctrl->dev, "invalid slew rate %u for pin: %d\n",
+			slew, group);
+		return -EINVAL;
+	}
+
+	slew_offset = g->slew_offset;
+	if (slew_offset == LPI_NO_SLEW)
+		return 0;
+
+	if (pctrl->data->flags & LPI_FLAG_SLEW_RATE_SAME_REG)
+		reg = pctrl->tlmm_base + LPI_TLMM_REG_OFFSET * group + LPI_GPIO_CFG_REG;
+	else
+		reg = pctrl->slew_base + LPI_SLEW_RATE_CTL_REG;
+
+	mutex_lock(&pctrl->lock);
+
+	sval = ioread32(reg);
+	sval &= ~(LPI_SLEW_RATE_MASK << slew_offset);
+	sval |= slew << slew_offset;
+	iowrite32(sval, reg);
+
+	mutex_unlock(&pctrl->lock);
+
+	return 0;
+}
+
 static int lpi_config_set(struct pinctrl_dev *pctldev, unsigned int group,
 			  unsigned long *configs, unsigned int nconfs)
 {
@@ -193,8 +228,7 @@ static int lpi_config_set(struct pinctrl_dev *pctldev, unsigned int group,
 	unsigned int param, arg, pullup = LPI_GPIO_BIAS_DISABLE, strength = 2;
 	bool value, output_enabled = false;
 	const struct lpi_pingroup *g;
-	unsigned long sval;
-	int i, slew_offset;
+	int i, ret;
 	u32 val;
 
 	g = &pctrl->data->groups[group];
@@ -226,24 +260,9 @@ static int lpi_config_set(struct pinctrl_dev *pctldev, unsigned int group,
 			strength = arg;
 			break;
 		case PIN_CONFIG_SLEW_RATE:
-			if (arg > LPI_SLEW_RATE_MAX) {
-				dev_err(pctldev->dev, "invalid slew rate %u for pin: %d\n",
-					arg, group);
-				return -EINVAL;
-			}
-
-			slew_offset = g->slew_offset;
-			if (slew_offset == LPI_NO_SLEW)
-				break;
-
-			mutex_lock(&pctrl->lock);
-
-			sval = ioread32(pctrl->slew_base + LPI_SLEW_RATE_CTL_REG);
-			sval &= ~(LPI_SLEW_RATE_MASK << slew_offset);
-			sval |= arg << slew_offset;
-			iowrite32(sval, pctrl->slew_base + LPI_SLEW_RATE_CTL_REG);
-
-			mutex_unlock(&pctrl->lock);
+			ret = lpi_config_set_slew_rate(pctrl, g, group, arg);
+			if (ret)
+				return ret;
 			break;
 		default:
 			return -EINVAL;
@@ -439,10 +458,12 @@ int lpi_pinctrl_probe(struct platform_device *pdev)
 		return dev_err_probe(dev, PTR_ERR(pctrl->tlmm_base),
 				     "TLMM resource not provided\n");
 
-	pctrl->slew_base = devm_platform_ioremap_resource(pdev, 1);
-	if (IS_ERR(pctrl->slew_base))
-		return dev_err_probe(dev, PTR_ERR(pctrl->slew_base),
-				     "Slew resource not provided\n");
+	if (!(data->flags & LPI_FLAG_SLEW_RATE_SAME_REG)) {
+		pctrl->slew_base = devm_platform_ioremap_resource(pdev, 1);
+		if (IS_ERR(pctrl->slew_base))
+			return dev_err_probe(dev, PTR_ERR(pctrl->slew_base),
+					     "Slew resource not provided\n");
+	}
 
 	ret = devm_clk_bulk_get_optional(dev, MAX_LPI_NUM_CLKS, pctrl->clks);
 	if (ret)
diff --git a/drivers/pinctrl/qcom/pinctrl-lpass-lpi.h b/drivers/pinctrl/qcom/pinctrl-lpass-lpi.h
index 387d83ee95b5..206b2c0ca828 100644
--- a/drivers/pinctrl/qcom/pinctrl-lpass-lpi.h
+++ b/drivers/pinctrl/qcom/pinctrl-lpass-lpi.h
@@ -60,6 +60,12 @@ struct pinctrl_pin_desc;
 		.nfuncs = 5,				\
 	}
 
+/*
+ * Slew rate control is done in the same register as rest of the
+ * pin configuration.
+ */
+#define LPI_FLAG_SLEW_RATE_SAME_REG			BIT(0)
+
 struct lpi_pingroup {
 	struct group_desc group;
 	unsigned int pin;
@@ -82,6 +88,7 @@ struct lpi_pinctrl_variant_data {
 	int ngroups;
 	const struct lpi_function *functions;
 	int nfunctions;
+	unsigned int flags;
 };
 
 int lpi_pinctrl_probe(struct platform_device *pdev);
diff --git a/drivers/pinctrl/qcom/pinctrl-msm.c b/drivers/pinctrl/qcom/pinctrl-msm.c
index 395040346d0f..207b41018580 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm.c
+++ b/drivers/pinctrl/qcom/pinctrl-msm.c
@@ -1196,6 +1196,8 @@ static int msm_gpio_irq_reqres(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
 	struct msm_pinctrl *pctrl = gpiochip_get_data(gc);
+	const struct msm_pingroup *g = &pctrl->soc->groups[d->hwirq];
+	unsigned long flags;
 	int ret;
 
 	if (!try_module_get(gc->owner))
@@ -1221,6 +1223,28 @@ static int msm_gpio_irq_reqres(struct irq_data *d)
 	 */
 	irq_set_status_flags(d->irq, IRQ_DISABLE_UNLAZY);
 
+	/*
+	 * If the wakeup_enable bit is present and marked as available for the
+	 * requested GPIO, it should be enabled when the GPIO is marked as
+	 * wake irq in order to allow the interrupt event to be transfered to
+	 * the PDC HW.
+	 * While the name implies only the wakeup event, it's also required for
+	 * the interrupt event.
+	 */
+	if (test_bit(d->hwirq, pctrl->skip_wake_irqs) && g->intr_wakeup_present_bit) {
+		u32 intr_cfg;
+
+		raw_spin_lock_irqsave(&pctrl->lock, flags);
+
+		intr_cfg = msm_readl_intr_cfg(pctrl, g);
+		if (intr_cfg & BIT(g->intr_wakeup_present_bit)) {
+			intr_cfg |= BIT(g->intr_wakeup_enable_bit);
+			msm_writel_intr_cfg(intr_cfg, pctrl, g);
+		}
+
+		raw_spin_unlock_irqrestore(&pctrl->lock, flags);
+	}
+
 	return 0;
 out:
 	module_put(gc->owner);
@@ -1230,6 +1254,24 @@ static int msm_gpio_irq_reqres(struct irq_data *d)
 static void msm_gpio_irq_relres(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
+	struct msm_pinctrl *pctrl = gpiochip_get_data(gc);
+	const struct msm_pingroup *g = &pctrl->soc->groups[d->hwirq];
+	unsigned long flags;
+
+	/* Disable the wakeup_enable bit if it has been set in msm_gpio_irq_reqres() */
+	if (test_bit(d->hwirq, pctrl->skip_wake_irqs) && g->intr_wakeup_present_bit) {
+		u32 intr_cfg;
+
+		raw_spin_lock_irqsave(&pctrl->lock, flags);
+
+		intr_cfg = msm_readl_intr_cfg(pctrl, g);
+		if (intr_cfg & BIT(g->intr_wakeup_present_bit)) {
+			intr_cfg &= ~BIT(g->intr_wakeup_enable_bit);
+			msm_writel_intr_cfg(intr_cfg, pctrl, g);
+		}
+
+		raw_spin_unlock_irqrestore(&pctrl->lock, flags);
+	}
 
 	gpiochip_unlock_as_irq(gc, d->hwirq);
 	module_put(gc->owner);
diff --git a/drivers/pinctrl/qcom/pinctrl-msm.h b/drivers/pinctrl/qcom/pinctrl-msm.h
index 4968d08a384d..63852ed70295 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm.h
+++ b/drivers/pinctrl/qcom/pinctrl-msm.h
@@ -58,6 +58,9 @@ struct pinctrl_pin_desc;
  * @intr_enable_bit:      Offset in @intr_cfg_reg for enabling the interrupt for this group.
  * @intr_status_bit:      Offset in @intr_status_reg for reading and acking the interrupt
  *                        status.
+ * @intr_wakeup_present_bit: Offset in @intr_target_reg specifying the GPIO can generate
+ *			  wakeup events.
+ * @intr_wakeup_enable_bit: Offset in @intr_target_reg to enable wakeup events for the GPIO.
  * @intr_target_bit:      Offset in @intr_target_reg for configuring the interrupt routing.
  * @intr_target_width:    Number of bits used for specifying interrupt routing target.
  * @intr_target_kpss_val: Value in @intr_target_bit for specifying that the interrupt from
@@ -100,6 +103,8 @@ struct msm_pingroup {
 	unsigned intr_status_bit:5;
 	unsigned intr_ack_high:1;
 
+	unsigned intr_wakeup_present_bit:5;
+	unsigned intr_wakeup_enable_bit:5;
 	unsigned intr_target_bit:5;
 	unsigned intr_target_width:5;
 	unsigned intr_target_kpss_val:5;
diff --git a/drivers/pinctrl/qcom/pinctrl-sm8650-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-sm8650-lpass-lpi.c
new file mode 100644
index 000000000000..6e4be91ff085
--- /dev/null
+++ b/drivers/pinctrl/qcom/pinctrl-sm8650-lpass-lpi.c
@@ -0,0 +1,255 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2022-2023 Linaro Ltd.
+ */
+
+#include <linux/gpio/driver.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#include "pinctrl-lpass-lpi.h"
+
+enum lpass_lpi_functions {
+	LPI_MUX_dmic1_clk,
+	LPI_MUX_dmic1_data,
+	LPI_MUX_dmic2_clk,
+	LPI_MUX_dmic2_data,
+	LPI_MUX_dmic3_clk,
+	LPI_MUX_dmic3_data,
+	LPI_MUX_dmic4_clk,
+	LPI_MUX_dmic4_data,
+	LPI_MUX_i2s0_clk,
+	LPI_MUX_i2s0_data,
+	LPI_MUX_i2s0_ws,
+	LPI_MUX_i2s1_clk,
+	LPI_MUX_i2s1_data,
+	LPI_MUX_i2s1_ws,
+	LPI_MUX_i2s2_clk,
+	LPI_MUX_i2s2_data,
+	LPI_MUX_i2s2_ws,
+	LPI_MUX_i2s3_clk,
+	LPI_MUX_i2s3_data,
+	LPI_MUX_i2s3_ws,
+	LPI_MUX_i2s4_clk,
+	LPI_MUX_i2s4_data,
+	LPI_MUX_i2s4_ws,
+	LPI_MUX_qca_swr_clk,
+	LPI_MUX_qca_swr_data,
+	LPI_MUX_slimbus_clk,
+	LPI_MUX_slimbus_data,
+	LPI_MUX_swr_rx_clk,
+	LPI_MUX_swr_rx_data,
+	LPI_MUX_swr_tx_clk,
+	LPI_MUX_swr_tx_data,
+	LPI_MUX_wsa_swr_clk,
+	LPI_MUX_wsa_swr_data,
+	LPI_MUX_wsa2_swr_clk,
+	LPI_MUX_wsa2_swr_data,
+	LPI_MUX_ext_mclk1_a,
+	LPI_MUX_ext_mclk1_b,
+	LPI_MUX_ext_mclk1_c,
+	LPI_MUX_ext_mclk1_d,
+	LPI_MUX_ext_mclk1_e,
+	LPI_MUX_gpio,
+	LPI_MUX__,
+};
+
+static int gpio0_pins[] = { 0 };
+static int gpio1_pins[] = { 1 };
+static int gpio2_pins[] = { 2 };
+static int gpio3_pins[] = { 3 };
+static int gpio4_pins[] = { 4 };
+static int gpio5_pins[] = { 5 };
+static int gpio6_pins[] = { 6 };
+static int gpio7_pins[] = { 7 };
+static int gpio8_pins[] = { 8 };
+static int gpio9_pins[] = { 9 };
+static int gpio10_pins[] = { 10 };
+static int gpio11_pins[] = { 11 };
+static int gpio12_pins[] = { 12 };
+static int gpio13_pins[] = { 13 };
+static int gpio14_pins[] = { 14 };
+static int gpio15_pins[] = { 15 };
+static int gpio16_pins[] = { 16 };
+static int gpio17_pins[] = { 17 };
+static int gpio18_pins[] = { 18 };
+static int gpio19_pins[] = { 19 };
+static int gpio20_pins[] = { 20 };
+static int gpio21_pins[] = { 21 };
+static int gpio22_pins[] = { 22 };
+
+static const struct pinctrl_pin_desc sm8650_lpi_pins[] = {
+	PINCTRL_PIN(0, "gpio0"),
+	PINCTRL_PIN(1, "gpio1"),
+	PINCTRL_PIN(2, "gpio2"),
+	PINCTRL_PIN(3, "gpio3"),
+	PINCTRL_PIN(4, "gpio4"),
+	PINCTRL_PIN(5, "gpio5"),
+	PINCTRL_PIN(6, "gpio6"),
+	PINCTRL_PIN(7, "gpio7"),
+	PINCTRL_PIN(8, "gpio8"),
+	PINCTRL_PIN(9, "gpio9"),
+	PINCTRL_PIN(10, "gpio10"),
+	PINCTRL_PIN(11, "gpio11"),
+	PINCTRL_PIN(12, "gpio12"),
+	PINCTRL_PIN(13, "gpio13"),
+	PINCTRL_PIN(14, "gpio14"),
+	PINCTRL_PIN(15, "gpio15"),
+	PINCTRL_PIN(16, "gpio16"),
+	PINCTRL_PIN(17, "gpio17"),
+	PINCTRL_PIN(18, "gpio18"),
+	PINCTRL_PIN(19, "gpio19"),
+	PINCTRL_PIN(20, "gpio20"),
+	PINCTRL_PIN(21, "gpio21"),
+	PINCTRL_PIN(22, "gpio22"),
+};
+
+static const char * const gpio_groups[] = {
+	"gpio0", "gpio1", "gpio2", "gpio3", "gpio4", "gpio5", "gpio6", "gpio7",
+	"gpio8", "gpio9", "gpio10", "gpio11", "gpio12", "gpio13", "gpio14",
+	"gpio15", "gpio16", "gpio17", "gpio18", "gpio19", "gpio20", "gpio21",
+	"gpio22",
+};
+
+static const char * const dmic1_clk_groups[] = { "gpio6" };
+static const char * const dmic1_data_groups[] = { "gpio7" };
+static const char * const dmic2_clk_groups[] = { "gpio8" };
+static const char * const dmic2_data_groups[] = { "gpio9" };
+static const char * const dmic3_clk_groups[] = { "gpio12" };
+static const char * const dmic3_data_groups[] = { "gpio13" };
+static const char * const dmic4_clk_groups[] = { "gpio17" };
+static const char * const dmic4_data_groups[] = { "gpio18" };
+static const char * const i2s0_clk_groups[] = { "gpio0" };
+static const char * const i2s0_ws_groups[] = { "gpio1" };
+static const char * const i2s0_data_groups[] = { "gpio2", "gpio3", "gpio4", "gpio5" };
+static const char * const i2s1_clk_groups[] = { "gpio6" };
+static const char * const i2s1_ws_groups[] = { "gpio7" };
+static const char * const i2s1_data_groups[] = { "gpio8", "gpio9" };
+static const char * const i2s2_clk_groups[] = { "gpio10" };
+static const char * const i2s2_ws_groups[] = { "gpio11" };
+static const char * const i2s2_data_groups[] = { "gpio15", "gpio16" };
+static const char * const i2s3_clk_groups[] = { "gpio12" };
+static const char * const i2s3_ws_groups[] = { "gpio13" };
+static const char * const i2s3_data_groups[] = { "gpio17", "gpio18" };
+static const char * const i2s4_clk_groups[] = { "gpio19"};
+static const char * const i2s4_ws_groups[] = { "gpio20"};
+static const char * const i2s4_data_groups[] = { "gpio21", "gpio22"};
+static const char * const qca_swr_clk_groups[] = { "gpio19" };
+static const char * const qca_swr_data_groups[] = { "gpio20" };
+static const char * const slimbus_clk_groups[] = { "gpio19"};
+static const char * const slimbus_data_groups[] = { "gpio20"};
+static const char * const swr_tx_clk_groups[] = { "gpio0" };
+static const char * const swr_tx_data_groups[] = { "gpio1", "gpio2", "gpio14" };
+static const char * const swr_rx_clk_groups[] = { "gpio3" };
+static const char * const swr_rx_data_groups[] = { "gpio4", "gpio5", "gpio15" };
+static const char * const wsa_swr_clk_groups[] = { "gpio10" };
+static const char * const wsa_swr_data_groups[] = { "gpio11" };
+static const char * const wsa2_swr_clk_groups[] = { "gpio15" };
+static const char * const wsa2_swr_data_groups[] = { "gpio16" };
+static const char * const ext_mclk1_c_groups[] = { "gpio5" };
+static const char * const ext_mclk1_b_groups[] = { "gpio9" };
+static const char * const ext_mclk1_a_groups[] = { "gpio13" };
+static const char * const ext_mclk1_d_groups[] = { "gpio14" };
+static const char * const ext_mclk1_e_groups[] = { "gpio22" };
+
+static const struct lpi_pingroup sm8650_groups[] = {
+	LPI_PINGROUP(0, 11, swr_tx_clk, i2s0_clk, _, _),
+	LPI_PINGROUP(1, 11, swr_tx_data, i2s0_ws, _, _),
+	LPI_PINGROUP(2, 11, swr_tx_data, i2s0_data, _, _),
+	LPI_PINGROUP(3, 11, swr_rx_clk, i2s0_data, _, _),
+	LPI_PINGROUP(4, 11, swr_rx_data, i2s0_data, _, _),
+	LPI_PINGROUP(5, 11, swr_rx_data, ext_mclk1_c, i2s0_data, _),
+	LPI_PINGROUP(6, LPI_NO_SLEW, dmic1_clk, i2s1_clk, _,  _),
+	LPI_PINGROUP(7, LPI_NO_SLEW, dmic1_data, i2s1_ws, _, _),
+	LPI_PINGROUP(8, LPI_NO_SLEW, dmic2_clk, i2s1_data, _, _),
+	LPI_PINGROUP(9, LPI_NO_SLEW, dmic2_data, i2s1_data, ext_mclk1_b, _),
+	LPI_PINGROUP(10, 11, i2s2_clk, wsa_swr_clk, _, _),
+	LPI_PINGROUP(11, 11, i2s2_ws, wsa_swr_data, _, _),
+	LPI_PINGROUP(12, LPI_NO_SLEW, dmic3_clk, i2s3_clk, _, _),
+	LPI_PINGROUP(13, LPI_NO_SLEW, dmic3_data, i2s3_ws, ext_mclk1_a, _),
+	LPI_PINGROUP(14, 11, swr_tx_data, ext_mclk1_d, _, _),
+	LPI_PINGROUP(15, 11, i2s2_data, wsa2_swr_clk, _, _),
+	LPI_PINGROUP(16, 11, i2s2_data, wsa2_swr_data, _, _),
+	LPI_PINGROUP(17, LPI_NO_SLEW, dmic4_clk, i2s3_data, _, _),
+	LPI_PINGROUP(18, LPI_NO_SLEW, dmic4_data, i2s3_data, _, _),
+	LPI_PINGROUP(19, 11, i2s4_clk, slimbus_clk, qca_swr_clk, _),
+	LPI_PINGROUP(20, 11, i2s4_ws, slimbus_data, qca_swr_data, _),
+	LPI_PINGROUP(21, LPI_NO_SLEW, i2s4_data, _, _, _),
+	LPI_PINGROUP(22, LPI_NO_SLEW, i2s4_data, ext_mclk1_e, _, _),
+};
+
+static const struct lpi_function sm8650_functions[] = {
+	LPI_FUNCTION(gpio),
+	LPI_FUNCTION(dmic1_clk),
+	LPI_FUNCTION(dmic1_data),
+	LPI_FUNCTION(dmic2_clk),
+	LPI_FUNCTION(dmic2_data),
+	LPI_FUNCTION(dmic3_clk),
+	LPI_FUNCTION(dmic3_data),
+	LPI_FUNCTION(dmic4_clk),
+	LPI_FUNCTION(dmic4_data),
+	LPI_FUNCTION(i2s0_clk),
+	LPI_FUNCTION(i2s0_data),
+	LPI_FUNCTION(i2s0_ws),
+	LPI_FUNCTION(i2s1_clk),
+	LPI_FUNCTION(i2s1_data),
+	LPI_FUNCTION(i2s1_ws),
+	LPI_FUNCTION(i2s2_clk),
+	LPI_FUNCTION(i2s2_data),
+	LPI_FUNCTION(i2s2_ws),
+	LPI_FUNCTION(i2s3_clk),
+	LPI_FUNCTION(i2s3_data),
+	LPI_FUNCTION(i2s3_ws),
+	LPI_FUNCTION(i2s4_clk),
+	LPI_FUNCTION(i2s4_data),
+	LPI_FUNCTION(i2s4_ws),
+	LPI_FUNCTION(qca_swr_clk),
+	LPI_FUNCTION(qca_swr_data),
+	LPI_FUNCTION(slimbus_clk),
+	LPI_FUNCTION(slimbus_data),
+	LPI_FUNCTION(swr_rx_clk),
+	LPI_FUNCTION(swr_rx_data),
+	LPI_FUNCTION(swr_tx_clk),
+	LPI_FUNCTION(swr_tx_data),
+	LPI_FUNCTION(wsa_swr_clk),
+	LPI_FUNCTION(wsa_swr_data),
+	LPI_FUNCTION(wsa2_swr_clk),
+	LPI_FUNCTION(wsa2_swr_data),
+	LPI_FUNCTION(ext_mclk1_a),
+	LPI_FUNCTION(ext_mclk1_b),
+	LPI_FUNCTION(ext_mclk1_c),
+	LPI_FUNCTION(ext_mclk1_d),
+	LPI_FUNCTION(ext_mclk1_e),
+};
+
+static const struct lpi_pinctrl_variant_data sm8650_lpi_data = {
+	.pins = sm8650_lpi_pins,
+	.npins = ARRAY_SIZE(sm8650_lpi_pins),
+	.groups = sm8650_groups,
+	.ngroups = ARRAY_SIZE(sm8650_groups),
+	.functions = sm8650_functions,
+	.nfunctions = ARRAY_SIZE(sm8650_functions),
+	.flags = LPI_FLAG_SLEW_RATE_SAME_REG,
+};
+
+static const struct of_device_id lpi_pinctrl_of_match[] = {
+	{
+	       .compatible = "qcom,sm8650-lpass-lpi-pinctrl",
+	       .data = &sm8650_lpi_data,
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, lpi_pinctrl_of_match);
+
+static struct platform_driver lpi_pinctrl_driver = {
+	.driver = {
+		   .name = "qcom-sm8650-lpass-lpi-pinctrl",
+		   .of_match_table = lpi_pinctrl_of_match,
+	},
+	.probe = lpi_pinctrl_probe,
+	.remove_new = lpi_pinctrl_remove,
+};
+
+module_platform_driver(lpi_pinctrl_driver);
+MODULE_DESCRIPTION("Qualcomm SM8650 LPI GPIO pin control driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/qcom/pinctrl-sm8650.c b/drivers/pinctrl/qcom/pinctrl-sm8650.c
new file mode 100644
index 000000000000..adaddd728662
--- /dev/null
+++ b/drivers/pinctrl/qcom/pinctrl-sm8650.c
@@ -0,0 +1,1762 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2021, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2023, Linaro Limited
+ */
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+
+#include "pinctrl-msm.h"
+
+#define REG_SIZE 0x1000
+
+#define PINGROUP(id, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10)	\
+	{					        \
+		.grp = PINCTRL_PINGROUP("gpio" #id,	\
+			gpio##id##_pins,		\
+			ARRAY_SIZE(gpio##id##_pins)),	\
+		.funcs = (int[]){			\
+			msm_mux_gpio, /* gpio mode */	\
+			msm_mux_##f1,			\
+			msm_mux_##f2,			\
+			msm_mux_##f3,			\
+			msm_mux_##f4,			\
+			msm_mux_##f5,			\
+			msm_mux_##f6,			\
+			msm_mux_##f7,			\
+			msm_mux_##f8,			\
+			msm_mux_##f9,			\
+			msm_mux_##f10			\
+		},				        \
+		.nfuncs = 11,				\
+		.ctl_reg = REG_SIZE * id,			\
+		.io_reg = 0x4 + REG_SIZE * id,		\
+		.intr_cfg_reg = 0x8 + REG_SIZE * id,		\
+		.intr_status_reg = 0xc + REG_SIZE * id,	\
+		.intr_target_reg = 0x8 + REG_SIZE * id,	\
+		.mux_bit = 2,			\
+		.pull_bit = 0,			\
+		.drv_bit = 6,			\
+		.i2c_pull_bit = 13,		\
+		.egpio_enable = 12,		\
+		.egpio_present = 11,		\
+		.oe_bit = 9,			\
+		.in_bit = 0,			\
+		.out_bit = 1,			\
+		.intr_enable_bit = 0,		\
+		.intr_status_bit = 0,		\
+		.intr_wakeup_present_bit = 6,   \
+		.intr_wakeup_enable_bit = 7,    \
+		.intr_target_bit = 8,		\
+		.intr_target_kpss_val = 3,	\
+		.intr_raw_status_bit = 4,	\
+		.intr_polarity_bit = 1,		\
+		.intr_detection_bit = 2,	\
+		.intr_detection_width = 2,	\
+	}
+
+#define SDC_QDSD_PINGROUP(pg_name, ctl, pull, drv)	\
+	{					        \
+		.grp = PINCTRL_PINGROUP(#pg_name,	\
+			pg_name##_pins,			\
+			ARRAY_SIZE(pg_name##_pins)),	\
+		.ctl_reg = ctl,				\
+		.io_reg = 0,				\
+		.intr_cfg_reg = 0,			\
+		.intr_status_reg = 0,			\
+		.intr_target_reg = 0,			\
+		.mux_bit = -1,				\
+		.pull_bit = pull,			\
+		.drv_bit = drv,				\
+		.oe_bit = -1,				\
+		.in_bit = -1,				\
+		.out_bit = -1,				\
+		.intr_enable_bit = -1,			\
+		.intr_status_bit = -1,			\
+		.intr_target_bit = -1,			\
+		.intr_raw_status_bit = -1,		\
+		.intr_polarity_bit = -1,		\
+		.intr_detection_bit = -1,		\
+		.intr_detection_width = -1,		\
+	}
+
+#define UFS_RESET(pg_name, ctl, io)			\
+	{					        \
+		.grp = PINCTRL_PINGROUP(#pg_name,	\
+			pg_name##_pins,			\
+			ARRAY_SIZE(pg_name##_pins)),	\
+		.ctl_reg = ctl,				\
+		.io_reg = io,				\
+		.intr_cfg_reg = 0,			\
+		.intr_status_reg = 0,			\
+		.intr_target_reg = 0,			\
+		.mux_bit = -1,				\
+		.pull_bit = 3,				\
+		.drv_bit = 0,				\
+		.oe_bit = -1,				\
+		.in_bit = -1,				\
+		.out_bit = 0,				\
+		.intr_enable_bit = -1,			\
+		.intr_status_bit = -1,			\
+		.intr_target_bit = -1,			\
+		.intr_raw_status_bit = -1,		\
+		.intr_polarity_bit = -1,		\
+		.intr_detection_bit = -1,		\
+		.intr_detection_width = -1,		\
+	}
+
+static const struct pinctrl_pin_desc sm8650_pins[] = {
+	PINCTRL_PIN(0, "GPIO_0"),
+	PINCTRL_PIN(1, "GPIO_1"),
+	PINCTRL_PIN(2, "GPIO_2"),
+	PINCTRL_PIN(3, "GPIO_3"),
+	PINCTRL_PIN(4, "GPIO_4"),
+	PINCTRL_PIN(5, "GPIO_5"),
+	PINCTRL_PIN(6, "GPIO_6"),
+	PINCTRL_PIN(7, "GPIO_7"),
+	PINCTRL_PIN(8, "GPIO_8"),
+	PINCTRL_PIN(9, "GPIO_9"),
+	PINCTRL_PIN(10, "GPIO_10"),
+	PINCTRL_PIN(11, "GPIO_11"),
+	PINCTRL_PIN(12, "GPIO_12"),
+	PINCTRL_PIN(13, "GPIO_13"),
+	PINCTRL_PIN(14, "GPIO_14"),
+	PINCTRL_PIN(15, "GPIO_15"),
+	PINCTRL_PIN(16, "GPIO_16"),
+	PINCTRL_PIN(17, "GPIO_17"),
+	PINCTRL_PIN(18, "GPIO_18"),
+	PINCTRL_PIN(19, "GPIO_19"),
+	PINCTRL_PIN(20, "GPIO_20"),
+	PINCTRL_PIN(21, "GPIO_21"),
+	PINCTRL_PIN(22, "GPIO_22"),
+	PINCTRL_PIN(23, "GPIO_23"),
+	PINCTRL_PIN(24, "GPIO_24"),
+	PINCTRL_PIN(25, "GPIO_25"),
+	PINCTRL_PIN(26, "GPIO_26"),
+	PINCTRL_PIN(27, "GPIO_27"),
+	PINCTRL_PIN(28, "GPIO_28"),
+	PINCTRL_PIN(29, "GPIO_29"),
+	PINCTRL_PIN(30, "GPIO_30"),
+	PINCTRL_PIN(31, "GPIO_31"),
+	PINCTRL_PIN(32, "GPIO_32"),
+	PINCTRL_PIN(33, "GPIO_33"),
+	PINCTRL_PIN(34, "GPIO_34"),
+	PINCTRL_PIN(35, "GPIO_35"),
+	PINCTRL_PIN(36, "GPIO_36"),
+	PINCTRL_PIN(37, "GPIO_37"),
+	PINCTRL_PIN(38, "GPIO_38"),
+	PINCTRL_PIN(39, "GPIO_39"),
+	PINCTRL_PIN(40, "GPIO_40"),
+	PINCTRL_PIN(41, "GPIO_41"),
+	PINCTRL_PIN(42, "GPIO_42"),
+	PINCTRL_PIN(43, "GPIO_43"),
+	PINCTRL_PIN(44, "GPIO_44"),
+	PINCTRL_PIN(45, "GPIO_45"),
+	PINCTRL_PIN(46, "GPIO_46"),
+	PINCTRL_PIN(47, "GPIO_47"),
+	PINCTRL_PIN(48, "GPIO_48"),
+	PINCTRL_PIN(49, "GPIO_49"),
+	PINCTRL_PIN(50, "GPIO_50"),
+	PINCTRL_PIN(51, "GPIO_51"),
+	PINCTRL_PIN(52, "GPIO_52"),
+	PINCTRL_PIN(53, "GPIO_53"),
+	PINCTRL_PIN(54, "GPIO_54"),
+	PINCTRL_PIN(55, "GPIO_55"),
+	PINCTRL_PIN(56, "GPIO_56"),
+	PINCTRL_PIN(57, "GPIO_57"),
+	PINCTRL_PIN(58, "GPIO_58"),
+	PINCTRL_PIN(59, "GPIO_59"),
+	PINCTRL_PIN(60, "GPIO_60"),
+	PINCTRL_PIN(61, "GPIO_61"),
+	PINCTRL_PIN(62, "GPIO_62"),
+	PINCTRL_PIN(63, "GPIO_63"),
+	PINCTRL_PIN(64, "GPIO_64"),
+	PINCTRL_PIN(65, "GPIO_65"),
+	PINCTRL_PIN(66, "GPIO_66"),
+	PINCTRL_PIN(67, "GPIO_67"),
+	PINCTRL_PIN(68, "GPIO_68"),
+	PINCTRL_PIN(69, "GPIO_69"),
+	PINCTRL_PIN(70, "GPIO_70"),
+	PINCTRL_PIN(71, "GPIO_71"),
+	PINCTRL_PIN(72, "GPIO_72"),
+	PINCTRL_PIN(73, "GPIO_73"),
+	PINCTRL_PIN(74, "GPIO_74"),
+	PINCTRL_PIN(75, "GPIO_75"),
+	PINCTRL_PIN(76, "GPIO_76"),
+	PINCTRL_PIN(77, "GPIO_77"),
+	PINCTRL_PIN(78, "GPIO_78"),
+	PINCTRL_PIN(79, "GPIO_79"),
+	PINCTRL_PIN(80, "GPIO_80"),
+	PINCTRL_PIN(81, "GPIO_81"),
+	PINCTRL_PIN(82, "GPIO_82"),
+	PINCTRL_PIN(83, "GPIO_83"),
+	PINCTRL_PIN(84, "GPIO_84"),
+	PINCTRL_PIN(85, "GPIO_85"),
+	PINCTRL_PIN(86, "GPIO_86"),
+	PINCTRL_PIN(87, "GPIO_87"),
+	PINCTRL_PIN(88, "GPIO_88"),
+	PINCTRL_PIN(89, "GPIO_89"),
+	PINCTRL_PIN(90, "GPIO_90"),
+	PINCTRL_PIN(91, "GPIO_91"),
+	PINCTRL_PIN(92, "GPIO_92"),
+	PINCTRL_PIN(93, "GPIO_93"),
+	PINCTRL_PIN(94, "GPIO_94"),
+	PINCTRL_PIN(95, "GPIO_95"),
+	PINCTRL_PIN(96, "GPIO_96"),
+	PINCTRL_PIN(97, "GPIO_97"),
+	PINCTRL_PIN(98, "GPIO_98"),
+	PINCTRL_PIN(99, "GPIO_99"),
+	PINCTRL_PIN(100, "GPIO_100"),
+	PINCTRL_PIN(101, "GPIO_101"),
+	PINCTRL_PIN(102, "GPIO_102"),
+	PINCTRL_PIN(103, "GPIO_103"),
+	PINCTRL_PIN(104, "GPIO_104"),
+	PINCTRL_PIN(105, "GPIO_105"),
+	PINCTRL_PIN(106, "GPIO_106"),
+	PINCTRL_PIN(107, "GPIO_107"),
+	PINCTRL_PIN(108, "GPIO_108"),
+	PINCTRL_PIN(109, "GPIO_109"),
+	PINCTRL_PIN(110, "GPIO_110"),
+	PINCTRL_PIN(111, "GPIO_111"),
+	PINCTRL_PIN(112, "GPIO_112"),
+	PINCTRL_PIN(113, "GPIO_113"),
+	PINCTRL_PIN(114, "GPIO_114"),
+	PINCTRL_PIN(115, "GPIO_115"),
+	PINCTRL_PIN(116, "GPIO_116"),
+	PINCTRL_PIN(117, "GPIO_117"),
+	PINCTRL_PIN(118, "GPIO_118"),
+	PINCTRL_PIN(119, "GPIO_119"),
+	PINCTRL_PIN(120, "GPIO_120"),
+	PINCTRL_PIN(121, "GPIO_121"),
+	PINCTRL_PIN(122, "GPIO_122"),
+	PINCTRL_PIN(123, "GPIO_123"),
+	PINCTRL_PIN(124, "GPIO_124"),
+	PINCTRL_PIN(125, "GPIO_125"),
+	PINCTRL_PIN(126, "GPIO_126"),
+	PINCTRL_PIN(127, "GPIO_127"),
+	PINCTRL_PIN(128, "GPIO_128"),
+	PINCTRL_PIN(129, "GPIO_129"),
+	PINCTRL_PIN(130, "GPIO_130"),
+	PINCTRL_PIN(131, "GPIO_131"),
+	PINCTRL_PIN(132, "GPIO_132"),
+	PINCTRL_PIN(133, "GPIO_133"),
+	PINCTRL_PIN(134, "GPIO_134"),
+	PINCTRL_PIN(135, "GPIO_135"),
+	PINCTRL_PIN(136, "GPIO_136"),
+	PINCTRL_PIN(137, "GPIO_137"),
+	PINCTRL_PIN(138, "GPIO_138"),
+	PINCTRL_PIN(139, "GPIO_139"),
+	PINCTRL_PIN(140, "GPIO_140"),
+	PINCTRL_PIN(141, "GPIO_141"),
+	PINCTRL_PIN(142, "GPIO_142"),
+	PINCTRL_PIN(143, "GPIO_143"),
+	PINCTRL_PIN(144, "GPIO_144"),
+	PINCTRL_PIN(145, "GPIO_145"),
+	PINCTRL_PIN(146, "GPIO_146"),
+	PINCTRL_PIN(147, "GPIO_147"),
+	PINCTRL_PIN(148, "GPIO_148"),
+	PINCTRL_PIN(149, "GPIO_149"),
+	PINCTRL_PIN(150, "GPIO_150"),
+	PINCTRL_PIN(151, "GPIO_151"),
+	PINCTRL_PIN(152, "GPIO_152"),
+	PINCTRL_PIN(153, "GPIO_153"),
+	PINCTRL_PIN(154, "GPIO_154"),
+	PINCTRL_PIN(155, "GPIO_155"),
+	PINCTRL_PIN(156, "GPIO_156"),
+	PINCTRL_PIN(157, "GPIO_157"),
+	PINCTRL_PIN(158, "GPIO_158"),
+	PINCTRL_PIN(159, "GPIO_159"),
+	PINCTRL_PIN(160, "GPIO_160"),
+	PINCTRL_PIN(161, "GPIO_161"),
+	PINCTRL_PIN(162, "GPIO_162"),
+	PINCTRL_PIN(163, "GPIO_163"),
+	PINCTRL_PIN(164, "GPIO_164"),
+	PINCTRL_PIN(165, "GPIO_165"),
+	PINCTRL_PIN(166, "GPIO_166"),
+	PINCTRL_PIN(167, "GPIO_167"),
+	PINCTRL_PIN(168, "GPIO_168"),
+	PINCTRL_PIN(169, "GPIO_169"),
+	PINCTRL_PIN(170, "GPIO_170"),
+	PINCTRL_PIN(171, "GPIO_171"),
+	PINCTRL_PIN(172, "GPIO_172"),
+	PINCTRL_PIN(173, "GPIO_173"),
+	PINCTRL_PIN(174, "GPIO_174"),
+	PINCTRL_PIN(175, "GPIO_175"),
+	PINCTRL_PIN(176, "GPIO_176"),
+	PINCTRL_PIN(177, "GPIO_177"),
+	PINCTRL_PIN(178, "GPIO_178"),
+	PINCTRL_PIN(179, "GPIO_179"),
+	PINCTRL_PIN(180, "GPIO_180"),
+	PINCTRL_PIN(181, "GPIO_181"),
+	PINCTRL_PIN(182, "GPIO_182"),
+	PINCTRL_PIN(183, "GPIO_183"),
+	PINCTRL_PIN(184, "GPIO_184"),
+	PINCTRL_PIN(185, "GPIO_185"),
+	PINCTRL_PIN(186, "GPIO_186"),
+	PINCTRL_PIN(187, "GPIO_187"),
+	PINCTRL_PIN(188, "GPIO_188"),
+	PINCTRL_PIN(189, "GPIO_189"),
+	PINCTRL_PIN(190, "GPIO_190"),
+	PINCTRL_PIN(191, "GPIO_191"),
+	PINCTRL_PIN(192, "GPIO_192"),
+	PINCTRL_PIN(193, "GPIO_193"),
+	PINCTRL_PIN(194, "GPIO_194"),
+	PINCTRL_PIN(195, "GPIO_195"),
+	PINCTRL_PIN(196, "GPIO_196"),
+	PINCTRL_PIN(197, "GPIO_197"),
+	PINCTRL_PIN(198, "GPIO_198"),
+	PINCTRL_PIN(199, "GPIO_199"),
+	PINCTRL_PIN(200, "GPIO_200"),
+	PINCTRL_PIN(201, "GPIO_201"),
+	PINCTRL_PIN(202, "GPIO_202"),
+	PINCTRL_PIN(203, "GPIO_203"),
+	PINCTRL_PIN(204, "GPIO_204"),
+	PINCTRL_PIN(205, "GPIO_205"),
+	PINCTRL_PIN(206, "GPIO_206"),
+	PINCTRL_PIN(207, "GPIO_207"),
+	PINCTRL_PIN(208, "GPIO_208"),
+	PINCTRL_PIN(209, "GPIO_209"),
+	PINCTRL_PIN(210, "UFS_RESET"),
+	PINCTRL_PIN(211, "SDC2_CLK"),
+	PINCTRL_PIN(212, "SDC2_CMD"),
+	PINCTRL_PIN(213, "SDC2_DATA"),
+};
+
+#define DECLARE_MSM_GPIO_PINS(pin) \
+	static const unsigned int gpio##pin##_pins[] = { pin }
+DECLARE_MSM_GPIO_PINS(0);
+DECLARE_MSM_GPIO_PINS(1);
+DECLARE_MSM_GPIO_PINS(2);
+DECLARE_MSM_GPIO_PINS(3);
+DECLARE_MSM_GPIO_PINS(4);
+DECLARE_MSM_GPIO_PINS(5);
+DECLARE_MSM_GPIO_PINS(6);
+DECLARE_MSM_GPIO_PINS(7);
+DECLARE_MSM_GPIO_PINS(8);
+DECLARE_MSM_GPIO_PINS(9);
+DECLARE_MSM_GPIO_PINS(10);
+DECLARE_MSM_GPIO_PINS(11);
+DECLARE_MSM_GPIO_PINS(12);
+DECLARE_MSM_GPIO_PINS(13);
+DECLARE_MSM_GPIO_PINS(14);
+DECLARE_MSM_GPIO_PINS(15);
+DECLARE_MSM_GPIO_PINS(16);
+DECLARE_MSM_GPIO_PINS(17);
+DECLARE_MSM_GPIO_PINS(18);
+DECLARE_MSM_GPIO_PINS(19);
+DECLARE_MSM_GPIO_PINS(20);
+DECLARE_MSM_GPIO_PINS(21);
+DECLARE_MSM_GPIO_PINS(22);
+DECLARE_MSM_GPIO_PINS(23);
+DECLARE_MSM_GPIO_PINS(24);
+DECLARE_MSM_GPIO_PINS(25);
+DECLARE_MSM_GPIO_PINS(26);
+DECLARE_MSM_GPIO_PINS(27);
+DECLARE_MSM_GPIO_PINS(28);
+DECLARE_MSM_GPIO_PINS(29);
+DECLARE_MSM_GPIO_PINS(30);
+DECLARE_MSM_GPIO_PINS(31);
+DECLARE_MSM_GPIO_PINS(32);
+DECLARE_MSM_GPIO_PINS(33);
+DECLARE_MSM_GPIO_PINS(34);
+DECLARE_MSM_GPIO_PINS(35);
+DECLARE_MSM_GPIO_PINS(36);
+DECLARE_MSM_GPIO_PINS(37);
+DECLARE_MSM_GPIO_PINS(38);
+DECLARE_MSM_GPIO_PINS(39);
+DECLARE_MSM_GPIO_PINS(40);
+DECLARE_MSM_GPIO_PINS(41);
+DECLARE_MSM_GPIO_PINS(42);
+DECLARE_MSM_GPIO_PINS(43);
+DECLARE_MSM_GPIO_PINS(44);
+DECLARE_MSM_GPIO_PINS(45);
+DECLARE_MSM_GPIO_PINS(46);
+DECLARE_MSM_GPIO_PINS(47);
+DECLARE_MSM_GPIO_PINS(48);
+DECLARE_MSM_GPIO_PINS(49);
+DECLARE_MSM_GPIO_PINS(50);
+DECLARE_MSM_GPIO_PINS(51);
+DECLARE_MSM_GPIO_PINS(52);
+DECLARE_MSM_GPIO_PINS(53);
+DECLARE_MSM_GPIO_PINS(54);
+DECLARE_MSM_GPIO_PINS(55);
+DECLARE_MSM_GPIO_PINS(56);
+DECLARE_MSM_GPIO_PINS(57);
+DECLARE_MSM_GPIO_PINS(58);
+DECLARE_MSM_GPIO_PINS(59);
+DECLARE_MSM_GPIO_PINS(60);
+DECLARE_MSM_GPIO_PINS(61);
+DECLARE_MSM_GPIO_PINS(62);
+DECLARE_MSM_GPIO_PINS(63);
+DECLARE_MSM_GPIO_PINS(64);
+DECLARE_MSM_GPIO_PINS(65);
+DECLARE_MSM_GPIO_PINS(66);
+DECLARE_MSM_GPIO_PINS(67);
+DECLARE_MSM_GPIO_PINS(68);
+DECLARE_MSM_GPIO_PINS(69);
+DECLARE_MSM_GPIO_PINS(70);
+DECLARE_MSM_GPIO_PINS(71);
+DECLARE_MSM_GPIO_PINS(72);
+DECLARE_MSM_GPIO_PINS(73);
+DECLARE_MSM_GPIO_PINS(74);
+DECLARE_MSM_GPIO_PINS(75);
+DECLARE_MSM_GPIO_PINS(76);
+DECLARE_MSM_GPIO_PINS(77);
+DECLARE_MSM_GPIO_PINS(78);
+DECLARE_MSM_GPIO_PINS(79);
+DECLARE_MSM_GPIO_PINS(80);
+DECLARE_MSM_GPIO_PINS(81);
+DECLARE_MSM_GPIO_PINS(82);
+DECLARE_MSM_GPIO_PINS(83);
+DECLARE_MSM_GPIO_PINS(84);
+DECLARE_MSM_GPIO_PINS(85);
+DECLARE_MSM_GPIO_PINS(86);
+DECLARE_MSM_GPIO_PINS(87);
+DECLARE_MSM_GPIO_PINS(88);
+DECLARE_MSM_GPIO_PINS(89);
+DECLARE_MSM_GPIO_PINS(90);
+DECLARE_MSM_GPIO_PINS(91);
+DECLARE_MSM_GPIO_PINS(92);
+DECLARE_MSM_GPIO_PINS(93);
+DECLARE_MSM_GPIO_PINS(94);
+DECLARE_MSM_GPIO_PINS(95);
+DECLARE_MSM_GPIO_PINS(96);
+DECLARE_MSM_GPIO_PINS(97);
+DECLARE_MSM_GPIO_PINS(98);
+DECLARE_MSM_GPIO_PINS(99);
+DECLARE_MSM_GPIO_PINS(100);
+DECLARE_MSM_GPIO_PINS(101);
+DECLARE_MSM_GPIO_PINS(102);
+DECLARE_MSM_GPIO_PINS(103);
+DECLARE_MSM_GPIO_PINS(104);
+DECLARE_MSM_GPIO_PINS(105);
+DECLARE_MSM_GPIO_PINS(106);
+DECLARE_MSM_GPIO_PINS(107);
+DECLARE_MSM_GPIO_PINS(108);
+DECLARE_MSM_GPIO_PINS(109);
+DECLARE_MSM_GPIO_PINS(110);
+DECLARE_MSM_GPIO_PINS(111);
+DECLARE_MSM_GPIO_PINS(112);
+DECLARE_MSM_GPIO_PINS(113);
+DECLARE_MSM_GPIO_PINS(114);
+DECLARE_MSM_GPIO_PINS(115);
+DECLARE_MSM_GPIO_PINS(116);
+DECLARE_MSM_GPIO_PINS(117);
+DECLARE_MSM_GPIO_PINS(118);
+DECLARE_MSM_GPIO_PINS(119);
+DECLARE_MSM_GPIO_PINS(120);
+DECLARE_MSM_GPIO_PINS(121);
+DECLARE_MSM_GPIO_PINS(122);
+DECLARE_MSM_GPIO_PINS(123);
+DECLARE_MSM_GPIO_PINS(124);
+DECLARE_MSM_GPIO_PINS(125);
+DECLARE_MSM_GPIO_PINS(126);
+DECLARE_MSM_GPIO_PINS(127);
+DECLARE_MSM_GPIO_PINS(128);
+DECLARE_MSM_GPIO_PINS(129);
+DECLARE_MSM_GPIO_PINS(130);
+DECLARE_MSM_GPIO_PINS(131);
+DECLARE_MSM_GPIO_PINS(132);
+DECLARE_MSM_GPIO_PINS(133);
+DECLARE_MSM_GPIO_PINS(134);
+DECLARE_MSM_GPIO_PINS(135);
+DECLARE_MSM_GPIO_PINS(136);
+DECLARE_MSM_GPIO_PINS(137);
+DECLARE_MSM_GPIO_PINS(138);
+DECLARE_MSM_GPIO_PINS(139);
+DECLARE_MSM_GPIO_PINS(140);
+DECLARE_MSM_GPIO_PINS(141);
+DECLARE_MSM_GPIO_PINS(142);
+DECLARE_MSM_GPIO_PINS(143);
+DECLARE_MSM_GPIO_PINS(144);
+DECLARE_MSM_GPIO_PINS(145);
+DECLARE_MSM_GPIO_PINS(146);
+DECLARE_MSM_GPIO_PINS(147);
+DECLARE_MSM_GPIO_PINS(148);
+DECLARE_MSM_GPIO_PINS(149);
+DECLARE_MSM_GPIO_PINS(150);
+DECLARE_MSM_GPIO_PINS(151);
+DECLARE_MSM_GPIO_PINS(152);
+DECLARE_MSM_GPIO_PINS(153);
+DECLARE_MSM_GPIO_PINS(154);
+DECLARE_MSM_GPIO_PINS(155);
+DECLARE_MSM_GPIO_PINS(156);
+DECLARE_MSM_GPIO_PINS(157);
+DECLARE_MSM_GPIO_PINS(158);
+DECLARE_MSM_GPIO_PINS(159);
+DECLARE_MSM_GPIO_PINS(160);
+DECLARE_MSM_GPIO_PINS(161);
+DECLARE_MSM_GPIO_PINS(162);
+DECLARE_MSM_GPIO_PINS(163);
+DECLARE_MSM_GPIO_PINS(164);
+DECLARE_MSM_GPIO_PINS(165);
+DECLARE_MSM_GPIO_PINS(166);
+DECLARE_MSM_GPIO_PINS(167);
+DECLARE_MSM_GPIO_PINS(168);
+DECLARE_MSM_GPIO_PINS(169);
+DECLARE_MSM_GPIO_PINS(170);
+DECLARE_MSM_GPIO_PINS(171);
+DECLARE_MSM_GPIO_PINS(172);
+DECLARE_MSM_GPIO_PINS(173);
+DECLARE_MSM_GPIO_PINS(174);
+DECLARE_MSM_GPIO_PINS(175);
+DECLARE_MSM_GPIO_PINS(176);
+DECLARE_MSM_GPIO_PINS(177);
+DECLARE_MSM_GPIO_PINS(178);
+DECLARE_MSM_GPIO_PINS(179);
+DECLARE_MSM_GPIO_PINS(180);
+DECLARE_MSM_GPIO_PINS(181);
+DECLARE_MSM_GPIO_PINS(182);
+DECLARE_MSM_GPIO_PINS(183);
+DECLARE_MSM_GPIO_PINS(184);
+DECLARE_MSM_GPIO_PINS(185);
+DECLARE_MSM_GPIO_PINS(186);
+DECLARE_MSM_GPIO_PINS(187);
+DECLARE_MSM_GPIO_PINS(188);
+DECLARE_MSM_GPIO_PINS(189);
+DECLARE_MSM_GPIO_PINS(190);
+DECLARE_MSM_GPIO_PINS(191);
+DECLARE_MSM_GPIO_PINS(192);
+DECLARE_MSM_GPIO_PINS(193);
+DECLARE_MSM_GPIO_PINS(194);
+DECLARE_MSM_GPIO_PINS(195);
+DECLARE_MSM_GPIO_PINS(196);
+DECLARE_MSM_GPIO_PINS(197);
+DECLARE_MSM_GPIO_PINS(198);
+DECLARE_MSM_GPIO_PINS(199);
+DECLARE_MSM_GPIO_PINS(200);
+DECLARE_MSM_GPIO_PINS(201);
+DECLARE_MSM_GPIO_PINS(202);
+DECLARE_MSM_GPIO_PINS(203);
+DECLARE_MSM_GPIO_PINS(204);
+DECLARE_MSM_GPIO_PINS(205);
+DECLARE_MSM_GPIO_PINS(206);
+DECLARE_MSM_GPIO_PINS(207);
+DECLARE_MSM_GPIO_PINS(208);
+DECLARE_MSM_GPIO_PINS(209);
+
+static const unsigned int ufs_reset_pins[] = { 210 };
+static const unsigned int sdc2_clk_pins[] = { 211 };
+static const unsigned int sdc2_cmd_pins[] = { 212 };
+static const unsigned int sdc2_data_pins[] = { 213 };
+
+enum sm8650_functions {
+	msm_mux_gpio,
+	msm_mux_aoss_cti,
+	msm_mux_atest_char,
+	msm_mux_atest_usb,
+	msm_mux_audio_ext_mclk0,
+	msm_mux_audio_ext_mclk1,
+	msm_mux_audio_ref_clk,
+	msm_mux_cam_aon_mclk2,
+	msm_mux_cam_aon_mclk4,
+	msm_mux_cam_mclk,
+	msm_mux_cci_async_in,
+	msm_mux_cci_i2c_scl,
+	msm_mux_cci_i2c_sda,
+	msm_mux_cci_timer,
+	msm_mux_cmu_rng,
+	msm_mux_coex_uart1_rx,
+	msm_mux_coex_uart1_tx,
+	msm_mux_coex_uart2_rx,
+	msm_mux_coex_uart2_tx,
+	msm_mux_cri_trng,
+	msm_mux_dbg_out_clk,
+	msm_mux_ddr_bist_complete,
+	msm_mux_ddr_bist_fail,
+	msm_mux_ddr_bist_start,
+	msm_mux_ddr_bist_stop,
+	msm_mux_ddr_pxi0,
+	msm_mux_ddr_pxi1,
+	msm_mux_ddr_pxi2,
+	msm_mux_ddr_pxi3,
+	msm_mux_do_not,
+	msm_mux_dp_hot,
+	msm_mux_egpio,
+	msm_mux_gcc_gp1,
+	msm_mux_gcc_gp2,
+	msm_mux_gcc_gp3,
+	msm_mux_gnss_adc0,
+	msm_mux_gnss_adc1,
+	msm_mux_i2chub0_se0,
+	msm_mux_i2chub0_se1,
+	msm_mux_i2chub0_se2,
+	msm_mux_i2chub0_se3,
+	msm_mux_i2chub0_se4,
+	msm_mux_i2chub0_se5,
+	msm_mux_i2chub0_se6,
+	msm_mux_i2chub0_se7,
+	msm_mux_i2chub0_se8,
+	msm_mux_i2chub0_se9,
+	msm_mux_i2s0_data0,
+	msm_mux_i2s0_data1,
+	msm_mux_i2s0_sck,
+	msm_mux_i2s0_ws,
+	msm_mux_i2s1_data0,
+	msm_mux_i2s1_data1,
+	msm_mux_i2s1_sck,
+	msm_mux_i2s1_ws,
+	msm_mux_ibi_i3c,
+	msm_mux_jitter_bist,
+	msm_mux_mdp_vsync,
+	msm_mux_mdp_vsync0_out,
+	msm_mux_mdp_vsync1_out,
+	msm_mux_mdp_vsync2_out,
+	msm_mux_mdp_vsync3_out,
+	msm_mux_mdp_vsync_e,
+	msm_mux_nav_gpio0,
+	msm_mux_nav_gpio1,
+	msm_mux_nav_gpio2,
+	msm_mux_nav_gpio3,
+	msm_mux_pcie0_clk_req_n,
+	msm_mux_pcie1_clk_req_n,
+	msm_mux_phase_flag,
+	msm_mux_pll_bist_sync,
+	msm_mux_pll_clk_aux,
+	msm_mux_prng_rosc0,
+	msm_mux_prng_rosc1,
+	msm_mux_prng_rosc2,
+	msm_mux_prng_rosc3,
+	msm_mux_qdss_cti,
+	msm_mux_qdss_gpio,
+	msm_mux_qlink_big_enable,
+	msm_mux_qlink_big_request,
+	msm_mux_qlink_little_enable,
+	msm_mux_qlink_little_request,
+	msm_mux_qlink_wmss,
+	msm_mux_qspi0,
+	msm_mux_qspi1,
+	msm_mux_qspi2,
+	msm_mux_qspi3,
+	msm_mux_qspi_clk,
+	msm_mux_qspi_cs,
+	msm_mux_qup1_se0,
+	msm_mux_qup1_se1,
+	msm_mux_qup1_se2,
+	msm_mux_qup1_se3,
+	msm_mux_qup1_se4,
+	msm_mux_qup1_se5,
+	msm_mux_qup1_se6,
+	msm_mux_qup1_se7,
+	msm_mux_qup2_se0,
+	msm_mux_qup2_se1,
+	msm_mux_qup2_se2,
+	msm_mux_qup2_se3,
+	msm_mux_qup2_se4,
+	msm_mux_qup2_se5,
+	msm_mux_qup2_se6,
+	msm_mux_qup2_se7,
+	msm_mux_sd_write_protect,
+	msm_mux_sdc40,
+	msm_mux_sdc41,
+	msm_mux_sdc42,
+	msm_mux_sdc43,
+	msm_mux_sdc4_clk,
+	msm_mux_sdc4_cmd,
+	msm_mux_tb_trig_sdc2,
+	msm_mux_tb_trig_sdc4,
+	msm_mux_tgu_ch0_trigout,
+	msm_mux_tgu_ch1_trigout,
+	msm_mux_tgu_ch2_trigout,
+	msm_mux_tgu_ch3_trigout,
+	msm_mux_tmess_prng0,
+	msm_mux_tmess_prng1,
+	msm_mux_tmess_prng2,
+	msm_mux_tmess_prng3,
+	msm_mux_tsense_pwm1,
+	msm_mux_tsense_pwm2,
+	msm_mux_tsense_pwm3,
+	msm_mux_uim0_clk,
+	msm_mux_uim0_data,
+	msm_mux_uim0_present,
+	msm_mux_uim0_reset,
+	msm_mux_uim1_clk,
+	msm_mux_uim1_data,
+	msm_mux_uim1_present,
+	msm_mux_uim1_reset,
+	msm_mux_usb1_hs,
+	msm_mux_usb_phy,
+	msm_mux_vfr_0,
+	msm_mux_vfr_1,
+	msm_mux_vsense_trigger_mirnat,
+	msm_mux__,
+};
+
+static const char *const gpio_groups[] = {
+	"gpio0",   "gpio1",   "gpio2",   "gpio3",
+	"gpio4",   "gpio5",   "gpio6",   "gpio7",
+	"gpio8",   "gpio9",   "gpio10",  "gpio11",
+	"gpio12",  "gpio13",  "gpio14",  "gpio15",
+	"gpio16",  "gpio17",  "gpio18",  "gpio19",
+	"gpio20",  "gpio21",  "gpio22",  "gpio23",
+	"gpio24",  "gpio25",  "gpio26",  "gpio27",
+	"gpio28",  "gpio29",  "gpio30",  "gpio31",
+	"gpio32",  "gpio33",  "gpio34",  "gpio35",
+	"gpio36",  "gpio37",  "gpio38",  "gpio39",
+	"gpio40",  "gpio41",  "gpio42",  "gpio43",
+	"gpio44",  "gpio45",  "gpio46",  "gpio47",
+	"gpio48",  "gpio49",  "gpio50",  "gpio51",
+	"gpio52",  "gpio53",  "gpio54",  "gpio55",
+	"gpio56",  "gpio57",  "gpio58",  "gpio59",
+	"gpio60",  "gpio61",  "gpio62",  "gpio63",
+	"gpio64",  "gpio65",  "gpio66",  "gpio67",
+	"gpio68",  "gpio69",  "gpio70",  "gpio71",
+	"gpio72",  "gpio73",  "gpio74",  "gpio75",
+	"gpio76",  "gpio77",  "gpio78",  "gpio79",
+	"gpio80",  "gpio81",  "gpio82",  "gpio83",
+	"gpio84",  "gpio85",  "gpio86",  "gpio87",
+	"gpio88",  "gpio89",  "gpio90",  "gpio91",
+	"gpio92",  "gpio93",  "gpio94",  "gpio95",
+	"gpio96",  "gpio97",  "gpio98",  "gpio99",
+	"gpio100", "gpio101", "gpio102", "gpio103",
+	"gpio104", "gpio105", "gpio106", "gpio107",
+	"gpio108", "gpio109", "gpio110", "gpio111",
+	"gpio112", "gpio113", "gpio114", "gpio115",
+	"gpio116", "gpio117", "gpio118", "gpio119",
+	"gpio120", "gpio121", "gpio122", "gpio123",
+	"gpio124", "gpio125", "gpio126", "gpio127",
+	"gpio128", "gpio129", "gpio130", "gpio131",
+	"gpio132", "gpio133", "gpio134", "gpio135",
+	"gpio136", "gpio137", "gpio138", "gpio139",
+	"gpio140", "gpio141", "gpio142", "gpio143",
+	"gpio144", "gpio145", "gpio146", "gpio147",
+	"gpio148", "gpio149", "gpio150", "gpio151",
+	"gpio152", "gpio153", "gpio154", "gpio155",
+	"gpio156", "gpio157", "gpio158", "gpio159",
+	"gpio160", "gpio161", "gpio162", "gpio163",
+	"gpio164", "gpio165", "gpio166", "gpio167",
+	"gpio168", "gpio169", "gpio170", "gpio171",
+	"gpio172", "gpio173", "gpio174", "gpio175",
+	"gpio176", "gpio177", "gpio178", "gpio179",
+	"gpio180", "gpio181", "gpio182", "gpio183",
+	"gpio184", "gpio185", "gpio186", "gpio187",
+	"gpio188", "gpio189", "gpio190", "gpio191",
+	"gpio192", "gpio193", "gpio194", "gpio195",
+	"gpio196", "gpio197", "gpio198", "gpio199",
+	"gpio200", "gpio201", "gpio202", "gpio203",
+	"gpio204", "gpio205", "gpio206", "gpio207",
+	"gpio208", "gpio209",
+};
+
+static const char * const egpio_groups[] = {
+	"gpio0",   "gpio1",   "gpio2",   "gpio3",   "gpio4",   "gpio5",
+	"gpio6",   "gpio7",   "gpio165", "gpio166", "gpio167", "gpio168",
+	"gpio169", "gpio170", "gpio171", "gpio172", "gpio173", "gpio174",
+	"gpio175", "gpio176", "gpio177", "gpio178", "gpio179", "gpio180",
+	"gpio181", "gpio182", "gpio183", "gpio184", "gpio185", "gpio186",
+	"gpio187", "gpio188", "gpio189", "gpio190", "gpio191", "gpio192",
+	"gpio193", "gpio194", "gpio195", "gpio196", "gpio197", "gpio198",
+	"gpio199", "gpio200", "gpio201", "gpio202", "gpio203", "gpio204",
+	"gpio205", "gpio206", "gpio207", "gpio208", "gpio209",
+};
+
+static const char * const aoss_cti_groups[] = {
+	"gpio50", "gpio51", "gpio60", "gpio61",
+};
+
+static const char *const atest_char_groups[] = {
+	"gpio130", "gpio131", "gpio132", "gpio133",
+	"gpio137",
+};
+
+static const char *const atest_usb_groups[] = {
+	"gpio71", "gpio72", "gpio74", "gpio130",
+	"gpio131",
+};
+
+static const char *const audio_ext_mclk0_groups[] = {
+	"gpio125",
+};
+
+static const char *const audio_ext_mclk1_groups[] = {
+	"gpio124",
+};
+
+static const char *const audio_ref_clk_groups[] = {
+	"gpio124",
+};
+
+static const char *const cam_aon_mclk2_groups[] = {
+	"gpio102",
+};
+
+static const char *const cam_aon_mclk4_groups[] = {
+	"gpio104",
+};
+
+static const char *const cam_mclk_groups[] = {
+	"gpio100", "gpio101", "gpio103", "gpio105",
+	"gpio106", "gpio108",
+};
+
+static const char *const cci_async_in_groups[] = {
+	"gpio15", "gpio163", "gpio164",
+};
+
+static const char *const cci_i2c_scl_groups[] = {
+	"gpio13",  "gpio114", "gpio116", "gpio118",
+	"gpio120", "gpio153",
+};
+
+static const char *const cci_i2c_sda_groups[] = {
+	"gpio12",  "gpio112", "gpio113", "gpio115",
+	"gpio117", "gpio119",
+};
+
+static const char *const cci_timer_groups[] = {
+	"gpio10", "gpio11", "gpio109", "gpio110",
+	"gpio111",
+};
+
+static const char *const cmu_rng_groups[] = {
+	"gpio95",  "gpio96", "gpio112", "gpio127",
+	"gpio122", "gpio128",
+};
+
+static const char *const coex_uart1_rx_groups[] = {
+	"gpio148",
+};
+
+static const char *const coex_uart1_tx_groups[] = {
+	"gpio149",
+};
+
+static const char *const coex_uart2_rx_groups[] = {
+	"gpio150",
+};
+
+static const char *const coex_uart2_tx_groups[] = {
+	"gpio151",
+};
+
+static const char *const cri_trng_groups[] = {
+	"gpio187",
+};
+
+static const char *const dbg_out_clk_groups[] = {
+	"gpio92",
+};
+
+static const char *const ddr_bist_complete_groups[] = {
+	"gpio44",
+};
+
+static const char *const ddr_bist_fail_groups[] = {
+	"gpio40",
+};
+
+static const char *const ddr_bist_start_groups[] = {
+	"gpio41",
+};
+
+static const char *const ddr_bist_stop_groups[] = {
+	"gpio45",
+};
+
+static const char *const ddr_pxi0_groups[] = {
+	"gpio75", "gpio76",
+};
+
+static const char *const ddr_pxi1_groups[] = {
+	"gpio44", "gpio45",
+};
+
+static const char *const ddr_pxi2_groups[] = {
+	"gpio51", "gpio62",
+};
+
+static const char *const ddr_pxi3_groups[] = {
+	"gpio46", "gpio47",
+};
+
+static const char *const do_not_groups[] = {
+	"gpio36", "gpio37", "gpio38", "gpio39",
+	"gpio134", "gpio135", "gpio136",
+};
+
+static const char *const dp_hot_groups[] = {
+	"gpio47",
+};
+
+static const char *const gcc_gp1_groups[] = {
+	"gpio86", "gpio134",
+};
+
+static const char *const gcc_gp2_groups[] = {
+	"gpio87", "gpio135",
+};
+
+static const char *const gcc_gp3_groups[] = {
+	"gpio88", "gpio136",
+};
+
+static const char *const gnss_adc0_groups[] = {
+	"gpio89", "gpio91",
+};
+
+static const char *const gnss_adc1_groups[] = {
+	"gpio90", "gpio92",
+};
+
+static const char *const i2chub0_se0_groups[] = {
+	"gpio64", "gpio65",
+};
+
+static const char *const i2chub0_se1_groups[] = {
+	"gpio66", "gpio67",
+};
+
+static const char *const i2chub0_se2_groups[] = {
+	"gpio68", "gpio69",
+};
+
+static const char *const i2chub0_se3_groups[] = {
+	"gpio70", "gpio71",
+};
+
+static const char *const i2chub0_se4_groups[] = {
+	"gpio72", "gpio73",
+};
+
+static const char *const i2chub0_se5_groups[] = {
+	"gpio74", "gpio75",
+};
+
+static const char *const i2chub0_se6_groups[] = {
+	"gpio76", "gpio77",
+};
+
+static const char *const i2chub0_se7_groups[] = {
+	"gpio78", "gpio79",
+};
+
+static const char *const i2chub0_se8_groups[] = {
+	"gpio206", "gpio207",
+};
+
+static const char *const i2chub0_se9_groups[] = {
+	"gpio80", "gpio81",
+};
+
+static const char *const i2s0_data0_groups[] = {
+	"gpio127",
+};
+
+static const char *const i2s0_data1_groups[] = {
+	"gpio128",
+};
+
+static const char *const i2s0_sck_groups[] = {
+	"gpio126",
+};
+
+static const char *const i2s0_ws_groups[] = {
+	"gpio129",
+};
+
+static const char *const i2s1_data0_groups[] = {
+	"gpio122",
+};
+
+static const char *const i2s1_data1_groups[] = {
+	"gpio124",
+};
+
+static const char *const i2s1_sck_groups[] = {
+	"gpio121",
+};
+
+static const char *const i2s1_ws_groups[] = {
+	"gpio123",
+};
+
+static const char *const ibi_i3c_groups[] = {
+	"gpio0",  "gpio1",  "gpio4",  "gpio5",
+	"gpio8",  "gpio9",  "gpio12", "gpio13",
+	"gpio32", "gpio33", "gpio36", "gpio37",
+	"gpio48", "gpio49", "gpio56", "gpio57",
+};
+
+static const char *const jitter_bist_groups[] = {
+	"gpio73",
+};
+
+static const char *const mdp_vsync_groups[] = {
+	"gpio86", "gpio87", "gpio133", "gpio137",
+};
+
+static const char *const mdp_vsync0_out_groups[] = {
+	"gpio86",
+};
+
+static const char *const mdp_vsync1_out_groups[] = {
+	"gpio86",
+};
+
+static const char *const mdp_vsync2_out_groups[] = {
+	"gpio87",
+};
+
+static const char *const mdp_vsync3_out_groups[] = {
+	"gpio87",
+};
+
+static const char *const mdp_vsync_e_groups[] = {
+	"gpio88",
+};
+
+static const char *const nav_gpio0_groups[] = {
+	"gpio154",
+};
+
+static const char *const nav_gpio1_groups[] = {
+	"gpio155",
+};
+
+static const char *const nav_gpio2_groups[] = {
+	"gpio152",
+};
+
+static const char *const nav_gpio3_groups[] = {
+	"gpio154",
+};
+
+static const char *const pcie0_clk_req_n_groups[] = {
+	"gpio95",
+};
+
+static const char *const pcie1_clk_req_n_groups[] = {
+	"gpio98",
+};
+
+static const char *const phase_flag_groups[] = {
+	"gpio0",   "gpio1",   "gpio3",   "gpio4",
+	"gpio5",   "gpio7",   "gpio8",   "gpio9",
+	"gpio11",  "gpio12",  "gpio13",  "gpio15",
+	"gpio16",  "gpio17",  "gpio19",  "gpio94",
+	"gpio95",  "gpio96",  "gpio109", "gpio111",
+	"gpio112", "gpio113", "gpio114", "gpio115",
+	"gpio116", "gpio117", "gpio118", "gpio119",
+	"gpio120", "gpio153", "gpio163", "gpio164",
+};
+
+static const char *const pll_bist_sync_groups[] = {
+	"gpio68",
+};
+
+static const char *const pll_clk_aux_groups[] = {
+	"gpio106",
+};
+
+static const char *const prng_rosc0_groups[] = {
+	"gpio186",
+};
+
+static const char *const prng_rosc1_groups[] = {
+	"gpio183",
+};
+
+static const char *const prng_rosc2_groups[] = {
+	"gpio182",
+};
+
+static const char *const prng_rosc3_groups[] = {
+	"gpio181",
+};
+
+static const char *const qdss_cti_groups[] = {
+	"gpio27", "gpio31", "gpio78",  "gpio79",
+	"gpio82", "gpio83", "gpio159", "gpio162",
+};
+
+static const char *const qdss_gpio_groups[] = {
+	"gpio3",   "gpio7",   "gpio8",   "gpio13",
+	"gpio15",  "gpio100", "gpio101", "gpio102",
+	"gpio103", "gpio104", "gpio105", "gpio113",
+	"gpio114", "gpio115", "gpio116", "gpio117",
+	"gpio118", "gpio140", "gpio141", "gpio142",
+	"gpio143", "gpio144", "gpio145", "gpio146",
+	"gpio147", "gpio148", "gpio149", "gpio150",
+	"gpio151", "gpio152", "gpio153", "gpio154",
+	"gpio155", "gpio156", "gpio157", "gpio158",
+};
+
+static const char *const qlink_big_enable_groups[] = {
+	"gpio160",
+};
+
+static const char *const qlink_big_request_groups[] = {
+	"gpio159",
+};
+
+static const char *const qlink_little_enable_groups[] = {
+	"gpio157",
+};
+
+static const char *const qlink_little_request_groups[] = {
+	"gpio156",
+};
+
+static const char *const qlink_wmss_groups[] = {
+	"gpio158",
+};
+
+static const char *const qspi0_groups[] = {
+	"gpio134",
+};
+
+static const char *const qspi1_groups[] = {
+	"gpio136",
+};
+
+static const char *const qspi2_groups[] = {
+	"gpio56",
+};
+
+static const char *const qspi3_groups[] = {
+	"gpio57",
+};
+
+static const char *const qspi_clk_groups[] = {
+	"gpio135",
+};
+
+static const char *const qspi_cs_groups[] = {
+	"gpio58", "gpio59",
+};
+
+static const char *const qup1_se0_groups[] = {
+	"gpio32", "gpio33", "gpio34", "gpio35",
+};
+
+static const char *const qup1_se1_groups[] = {
+	"gpio36", "gpio37", "gpio38", "gpio39",
+};
+
+static const char *const qup1_se2_groups[] = {
+	"gpio40", "gpio41", "gpio42", "gpio43",
+	"gpio44", "gpio45", "gpio46",
+};
+
+static const char *const qup1_se3_groups[] = {
+	"gpio44", "gpio45", "gpio46", "gpio47",
+};
+
+static const char *const qup1_se4_groups[] = {
+	"gpio48", "gpio49", "gpio50", "gpio51",
+};
+
+static const char *const qup1_se5_groups[] = {
+	"gpio52", "gpio53", "gpio54", "gpio55",
+};
+
+static const char *const qup1_se6_groups[] = {
+	"gpio56", "gpio57", "gpio58", "gpio59",
+};
+
+static const char *const qup1_se7_groups[] = {
+	"gpio60", "gpio61", "gpio62", "gpio63",
+};
+
+static const char *const qup2_se0_groups[] = {
+	"gpio0", "gpio1", "gpio2", "gpio3",
+};
+
+static const char *const qup2_se1_groups[] = {
+	"gpio4", "gpio5", "gpio6", "gpio7",
+};
+
+static const char *const qup2_se2_groups[] = {
+	"gpio8",  "gpio9",  "gpio10", "gpio11",
+	"gpio13", "gpio15", "gpio12",
+};
+
+static const char *const qup2_se3_groups[] = {
+	"gpio12", "gpio13", "gpio14", "gpio15",
+};
+
+static const char *const qup2_se4_groups[] = {
+	"gpio16", "gpio17", "gpio18", "gpio19",
+};
+
+static const char *const qup2_se5_groups[] = {
+	"gpio20", "gpio21", "gpio22", "gpio23",
+	"gpio23",
+};
+
+static const char *const qup2_se6_groups[] = {
+	"gpio24", "gpio25", "gpio26", "gpio27",
+};
+
+static const char *const qup2_se7_groups[] = {
+	"gpio28", "gpio29", "gpio30", "gpio31",
+};
+
+static const char *const sd_write_protect_groups[] = {
+	"gpio93",
+};
+
+static const char *const sdc40_groups[] = {
+	"gpio134",
+};
+
+static const char *const sdc41_groups[] = {
+	"gpio136",
+};
+
+static const char *const sdc42_groups[] = {
+	"gpio56",
+};
+
+static const char *const sdc43_groups[] = {
+	"gpio57",
+};
+
+static const char *const sdc4_clk_groups[] = {
+	"gpio135",
+};
+
+static const char *const sdc4_cmd_groups[] = {
+	"gpio59",
+};
+
+static const char *const tb_trig_sdc2_groups[] = {
+	"gpio8",
+};
+
+static const char *const tb_trig_sdc4_groups[] = {
+	"gpio58",
+};
+
+static const char *const tgu_ch0_trigout_groups[] = {
+	"gpio8",
+};
+
+static const char *const tgu_ch1_trigout_groups[] = {
+	"gpio9",
+};
+
+static const char *const tgu_ch2_trigout_groups[] = {
+	"gpio10",
+};
+
+static const char *const tgu_ch3_trigout_groups[] = {
+	"gpio11",
+};
+
+static const char *const tmess_prng0_groups[] = {
+	"gpio94",
+};
+
+static const char *const tmess_prng1_groups[] = {
+	"gpio95",
+};
+
+static const char *const tmess_prng2_groups[] = {
+	"gpio96",
+};
+
+static const char *const tmess_prng3_groups[] = {
+	"gpio109",
+};
+
+static const char *const tsense_pwm1_groups[] = {
+	"gpio58",
+};
+
+static const char *const tsense_pwm2_groups[] = {
+	"gpio58",
+};
+
+static const char *const tsense_pwm3_groups[] = {
+	"gpio58",
+};
+
+static const char *const uim0_clk_groups[] = {
+	"gpio131",
+};
+
+static const char *const uim0_data_groups[] = {
+	"gpio130",
+};
+
+static const char *const uim0_present_groups[] = {
+	"gpio47",
+};
+
+static const char *const uim0_reset_groups[] = {
+	"gpio132",
+};
+
+static const char *const uim1_clk_groups[] = {
+	"gpio135",
+};
+
+static const char *const uim1_data_groups[] = {
+	"gpio134",
+};
+
+static const char *const uim1_present_groups[] = {
+	"gpio76",
+};
+
+static const char *const uim1_reset_groups[] = {
+	"gpio136",
+};
+
+static const char *const usb1_hs_groups[] = {
+	"gpio89",
+};
+
+static const char *const usb_phy_groups[] = {
+	"gpio29", "gpio54",
+};
+
+static const char *const vfr_0_groups[] = {
+	"gpio150",
+};
+
+static const char *const vfr_1_groups[] = {
+	"gpio155",
+};
+
+static const char *const vsense_trigger_mirnat_groups[] = {
+	"gpio60",
+};
+
+static const struct pinfunction sm8650_functions[] = {
+	MSM_PIN_FUNCTION(gpio),
+	MSM_PIN_FUNCTION(aoss_cti),
+	MSM_PIN_FUNCTION(atest_char),
+	MSM_PIN_FUNCTION(atest_usb),
+	MSM_PIN_FUNCTION(audio_ext_mclk0),
+	MSM_PIN_FUNCTION(audio_ext_mclk1),
+	MSM_PIN_FUNCTION(audio_ref_clk),
+	MSM_PIN_FUNCTION(cam_aon_mclk2),
+	MSM_PIN_FUNCTION(cam_aon_mclk4),
+	MSM_PIN_FUNCTION(cam_mclk),
+	MSM_PIN_FUNCTION(cci_async_in),
+	MSM_PIN_FUNCTION(cci_i2c_scl),
+	MSM_PIN_FUNCTION(cci_i2c_sda),
+	MSM_PIN_FUNCTION(cci_timer),
+	MSM_PIN_FUNCTION(cmu_rng),
+	MSM_PIN_FUNCTION(coex_uart1_rx),
+	MSM_PIN_FUNCTION(coex_uart1_tx),
+	MSM_PIN_FUNCTION(coex_uart2_rx),
+	MSM_PIN_FUNCTION(coex_uart2_tx),
+	MSM_PIN_FUNCTION(cri_trng),
+	MSM_PIN_FUNCTION(dbg_out_clk),
+	MSM_PIN_FUNCTION(ddr_bist_complete),
+	MSM_PIN_FUNCTION(ddr_bist_fail),
+	MSM_PIN_FUNCTION(ddr_bist_start),
+	MSM_PIN_FUNCTION(ddr_bist_stop),
+	MSM_PIN_FUNCTION(ddr_pxi0),
+	MSM_PIN_FUNCTION(ddr_pxi1),
+	MSM_PIN_FUNCTION(ddr_pxi2),
+	MSM_PIN_FUNCTION(ddr_pxi3),
+	MSM_PIN_FUNCTION(do_not),
+	MSM_PIN_FUNCTION(dp_hot),
+	MSM_PIN_FUNCTION(egpio),
+	MSM_PIN_FUNCTION(gcc_gp1),
+	MSM_PIN_FUNCTION(gcc_gp2),
+	MSM_PIN_FUNCTION(gcc_gp3),
+	MSM_PIN_FUNCTION(gnss_adc0),
+	MSM_PIN_FUNCTION(gnss_adc1),
+	MSM_PIN_FUNCTION(i2chub0_se0),
+	MSM_PIN_FUNCTION(i2chub0_se1),
+	MSM_PIN_FUNCTION(i2chub0_se2),
+	MSM_PIN_FUNCTION(i2chub0_se3),
+	MSM_PIN_FUNCTION(i2chub0_se4),
+	MSM_PIN_FUNCTION(i2chub0_se5),
+	MSM_PIN_FUNCTION(i2chub0_se6),
+	MSM_PIN_FUNCTION(i2chub0_se7),
+	MSM_PIN_FUNCTION(i2chub0_se8),
+	MSM_PIN_FUNCTION(i2chub0_se9),
+	MSM_PIN_FUNCTION(i2s0_data0),
+	MSM_PIN_FUNCTION(i2s0_data1),
+	MSM_PIN_FUNCTION(i2s0_sck),
+	MSM_PIN_FUNCTION(i2s0_ws),
+	MSM_PIN_FUNCTION(i2s1_data0),
+	MSM_PIN_FUNCTION(i2s1_data1),
+	MSM_PIN_FUNCTION(i2s1_sck),
+	MSM_PIN_FUNCTION(i2s1_ws),
+	MSM_PIN_FUNCTION(ibi_i3c),
+	MSM_PIN_FUNCTION(jitter_bist),
+	MSM_PIN_FUNCTION(mdp_vsync),
+	MSM_PIN_FUNCTION(mdp_vsync0_out),
+	MSM_PIN_FUNCTION(mdp_vsync1_out),
+	MSM_PIN_FUNCTION(mdp_vsync2_out),
+	MSM_PIN_FUNCTION(mdp_vsync3_out),
+	MSM_PIN_FUNCTION(mdp_vsync_e),
+	MSM_PIN_FUNCTION(nav_gpio0),
+	MSM_PIN_FUNCTION(nav_gpio1),
+	MSM_PIN_FUNCTION(nav_gpio2),
+	MSM_PIN_FUNCTION(nav_gpio3),
+	MSM_PIN_FUNCTION(pcie0_clk_req_n),
+	MSM_PIN_FUNCTION(pcie1_clk_req_n),
+	MSM_PIN_FUNCTION(phase_flag),
+	MSM_PIN_FUNCTION(pll_bist_sync),
+	MSM_PIN_FUNCTION(pll_clk_aux),
+	MSM_PIN_FUNCTION(prng_rosc0),
+	MSM_PIN_FUNCTION(prng_rosc1),
+	MSM_PIN_FUNCTION(prng_rosc2),
+	MSM_PIN_FUNCTION(prng_rosc3),
+	MSM_PIN_FUNCTION(qdss_cti),
+	MSM_PIN_FUNCTION(qdss_gpio),
+	MSM_PIN_FUNCTION(qlink_big_enable),
+	MSM_PIN_FUNCTION(qlink_big_request),
+	MSM_PIN_FUNCTION(qlink_little_enable),
+	MSM_PIN_FUNCTION(qlink_little_request),
+	MSM_PIN_FUNCTION(qlink_wmss),
+	MSM_PIN_FUNCTION(qspi0),
+	MSM_PIN_FUNCTION(qspi1),
+	MSM_PIN_FUNCTION(qspi2),
+	MSM_PIN_FUNCTION(qspi3),
+	MSM_PIN_FUNCTION(qspi_clk),
+	MSM_PIN_FUNCTION(qspi_cs),
+	MSM_PIN_FUNCTION(qup1_se0),
+	MSM_PIN_FUNCTION(qup1_se1),
+	MSM_PIN_FUNCTION(qup1_se2),
+	MSM_PIN_FUNCTION(qup1_se3),
+	MSM_PIN_FUNCTION(qup1_se4),
+	MSM_PIN_FUNCTION(qup1_se5),
+	MSM_PIN_FUNCTION(qup1_se6),
+	MSM_PIN_FUNCTION(qup1_se7),
+	MSM_PIN_FUNCTION(qup2_se0),
+	MSM_PIN_FUNCTION(qup2_se1),
+	MSM_PIN_FUNCTION(qup2_se2),
+	MSM_PIN_FUNCTION(qup2_se3),
+	MSM_PIN_FUNCTION(qup2_se4),
+	MSM_PIN_FUNCTION(qup2_se5),
+	MSM_PIN_FUNCTION(qup2_se6),
+	MSM_PIN_FUNCTION(qup2_se7),
+	MSM_PIN_FUNCTION(sd_write_protect),
+	MSM_PIN_FUNCTION(sdc40),
+	MSM_PIN_FUNCTION(sdc41),
+	MSM_PIN_FUNCTION(sdc42),
+	MSM_PIN_FUNCTION(sdc43),
+	MSM_PIN_FUNCTION(sdc4_clk),
+	MSM_PIN_FUNCTION(sdc4_cmd),
+	MSM_PIN_FUNCTION(tb_trig_sdc2),
+	MSM_PIN_FUNCTION(tb_trig_sdc4),
+	MSM_PIN_FUNCTION(tgu_ch0_trigout),
+	MSM_PIN_FUNCTION(tgu_ch1_trigout),
+	MSM_PIN_FUNCTION(tgu_ch2_trigout),
+	MSM_PIN_FUNCTION(tgu_ch3_trigout),
+	MSM_PIN_FUNCTION(tmess_prng0),
+	MSM_PIN_FUNCTION(tmess_prng1),
+	MSM_PIN_FUNCTION(tmess_prng2),
+	MSM_PIN_FUNCTION(tmess_prng3),
+	MSM_PIN_FUNCTION(tsense_pwm1),
+	MSM_PIN_FUNCTION(tsense_pwm2),
+	MSM_PIN_FUNCTION(tsense_pwm3),
+	MSM_PIN_FUNCTION(uim0_clk),
+	MSM_PIN_FUNCTION(uim0_data),
+	MSM_PIN_FUNCTION(uim0_present),
+	MSM_PIN_FUNCTION(uim0_reset),
+	MSM_PIN_FUNCTION(uim1_clk),
+	MSM_PIN_FUNCTION(uim1_data),
+	MSM_PIN_FUNCTION(uim1_present),
+	MSM_PIN_FUNCTION(uim1_reset),
+	MSM_PIN_FUNCTION(usb1_hs),
+	MSM_PIN_FUNCTION(usb_phy),
+	MSM_PIN_FUNCTION(vfr_0),
+	MSM_PIN_FUNCTION(vfr_1),
+	MSM_PIN_FUNCTION(vsense_trigger_mirnat),
+};
+
+/*
+ * Every pin is maintained as a single group, and missing or non-existing pin
+ * would be maintained as dummy group to synchronize pin group index with
+ * pin descriptor registered with pinctrl core.
+ * Clients would not be able to request these dummy pin groups.
+ */
+static const struct msm_pingroup sm8650_groups[] = {
+	[0] = PINGROUP(0, qup2_se0, ibi_i3c, phase_flag, _, _, _, _, _, _, egpio),
+	[1] = PINGROUP(1, qup2_se0, ibi_i3c, phase_flag, _, _, _, _, _, _, egpio),
+	[2] = PINGROUP(2, qup2_se0, _, _, _, _, _, _, _, _, egpio),
+	[3] = PINGROUP(3, qup2_se0, phase_flag, _, qdss_gpio, _, _, _, _, _, egpio),
+	[4] = PINGROUP(4, qup2_se1, ibi_i3c, phase_flag, _, _, _, _, _, _, egpio),
+	[5] = PINGROUP(5, qup2_se1, ibi_i3c, phase_flag, _, _, _, _, _, _, egpio),
+	[6] = PINGROUP(6, qup2_se1, _, _, _, _, _, _, _, _, egpio),
+	[7] = PINGROUP(7, qup2_se1, phase_flag, _, qdss_gpio, _, _, _, _, _, egpio),
+	[8] = PINGROUP(8, qup2_se2, ibi_i3c, tb_trig_sdc2, phase_flag, tgu_ch0_trigout, _, qdss_gpio, _, _, _),
+	[9] = PINGROUP(9, qup2_se2, ibi_i3c, phase_flag, tgu_ch1_trigout, _, _, _, _, _, _),
+	[10] = PINGROUP(10, qup2_se2, cci_timer, tgu_ch2_trigout, _, _, _, _, _, _, _),
+	[11] = PINGROUP(11, qup2_se2, cci_timer, phase_flag, tgu_ch3_trigout, _, _, _, _, _, _),
+	[12] = PINGROUP(12, qup2_se3, cci_i2c_sda, ibi_i3c, qup2_se2, phase_flag, _, _, _, _, _),
+	[13] = PINGROUP(13, qup2_se3, cci_i2c_scl, ibi_i3c, qup2_se2, phase_flag, _, qdss_gpio, _, _, _),
+	[14] = PINGROUP(14, qup2_se3, _, _, _, _, _, _, _, _, _),
+	[15] = PINGROUP(15, qup2_se3, cci_async_in, qup2_se2, phase_flag, _, qdss_gpio, _, _, _, _),
+	[16] = PINGROUP(16, qup2_se4, phase_flag, _, _, _, _, _, _, _, _),
+	[17] = PINGROUP(17, qup2_se4, phase_flag, _, _, _, _, _, _, _, _),
+	[18] = PINGROUP(18, qup2_se4, _, _, _, _, _, _, _, _, _),
+	[19] = PINGROUP(19, qup2_se4, phase_flag, _, _, _, _, _, _, _, _),
+	[20] = PINGROUP(20, qup2_se5, _, _, _, _, _, _, _, _, _),
+	[21] = PINGROUP(21, qup2_se5, _, _, _, _, _, _, _, _, _),
+	[22] = PINGROUP(22, qup2_se5, _, _, _, _, _, _, _, _, _),
+	[23] = PINGROUP(23, qup2_se5, qup2_se5, _, _, _, _, _, _, _, _),
+	[24] = PINGROUP(24, qup2_se6, _, _, _, _, _, _, _, _, _),
+	[25] = PINGROUP(25, qup2_se6, _, _, _, _, _, _, _, _, _),
+	[26] = PINGROUP(26, qup2_se6, _, _, _, _, _, _, _, _, _),
+	[27] = PINGROUP(27, qup2_se6, qdss_cti, _, _, _, _, _, _, _, _),
+	[28] = PINGROUP(28, qup2_se7, _, _, _, _, _, _, _, _, _),
+	[29] = PINGROUP(29, qup2_se7, usb_phy, _, _, _, _, _, _, _, _),
+	[30] = PINGROUP(30, qup2_se7, _, _, _, _, _, _, _, _, _),
+	[31] = PINGROUP(31, qup2_se7, qdss_cti, _, _, _, _, _, _, _, _),
+	[32] = PINGROUP(32, qup1_se0, ibi_i3c, _, _, _, _, _, _, _, _),
+	[33] = PINGROUP(33, qup1_se0, ibi_i3c, _, _, _, _, _, _, _, _),
+	[34] = PINGROUP(34, qup1_se0, _, _, _, _, _, _, _, _, _),
+	[35] = PINGROUP(35, qup1_se0, _, _, _, _, _, _, _, _, _),
+	[36] = PINGROUP(36, qup1_se1, do_not, ibi_i3c, _, _, _, _, _, _, _),
+	[37] = PINGROUP(37, qup1_se1, do_not, ibi_i3c, _, _, _, _, _, _, _),
+	[38] = PINGROUP(38, qup1_se1, do_not, _, _, _, _, _, _, _, _),
+	[39] = PINGROUP(39, qup1_se1, do_not, _, _, _, _, _, _, _, _),
+	[40] = PINGROUP(40, qup1_se2, ddr_bist_fail, _, _, _, _, _, _, _, _),
+	[41] = PINGROUP(41, qup1_se2, ddr_bist_start, _, _, _, _, _, _, _, _),
+	[42] = PINGROUP(42, qup1_se2, _, _, _, _, _, _, _, _, _),
+	[43] = PINGROUP(43, qup1_se2, _, _, _, _, _, _, _, _, _),
+	[44] = PINGROUP(44, qup1_se3, qup1_se2, ddr_bist_complete, ddr_pxi1, _, _, _, _, _, _),
+	[45] = PINGROUP(45, qup1_se3, qup1_se2, ddr_bist_stop, ddr_pxi1, _, _, _, _, _, _),
+	[46] = PINGROUP(46, qup1_se3, qup1_se2, ddr_pxi3, _, _, _, _, _, _, _),
+	[47] = PINGROUP(47, qup1_se3, uim0_present, dp_hot, ddr_pxi3, _, _, _, _, _, _),
+	[48] = PINGROUP(48, qup1_se4, ibi_i3c, _, _, _, _, _, _, _, _),
+	[49] = PINGROUP(49, qup1_se4, ibi_i3c, _, _, _, _, _, _, _, _),
+	[50] = PINGROUP(50, qup1_se4, aoss_cti, _, _, _, _, _, _, _, _),
+	[51] = PINGROUP(51, qup1_se4, aoss_cti, ddr_pxi2, _, _, _, _, _, _, _),
+	[52] = PINGROUP(52, qup1_se5, _, _, _, _, _, _, _, _, _),
+	[53] = PINGROUP(53, qup1_se5, _, _, _, _, _, _, _, _, _),
+	[54] = PINGROUP(54, qup1_se5, usb_phy, _, _, _, _, _, _, _, _),
+	[55] = PINGROUP(55, qup1_se5, _, _, _, _, _, _, _, _, _),
+	[56] = PINGROUP(56, qup1_se6, ibi_i3c, qspi2, sdc42, _, _, _, _, _, _),
+	[57] = PINGROUP(57, qup1_se6, ibi_i3c, qspi3, sdc43, _, _, _, _, _, _),
+	[58] = PINGROUP(58, qup1_se6, qspi_cs, tb_trig_sdc4, tsense_pwm1, tsense_pwm2, tsense_pwm3, _, _, _, _),
+	[59] = PINGROUP(59, qup1_se6, _, qspi_cs, sdc4_cmd, _, _, _, _, _, _),
+	[60] = PINGROUP(60, qup1_se7, aoss_cti, vsense_trigger_mirnat, _, _, _, _, _, _, _),
+	[61] = PINGROUP(61, qup1_se7, aoss_cti, _, _, _, _, _, _, _, _),
+	[62] = PINGROUP(62, qup1_se7, ddr_pxi2, _, _, _, _, _, _, _, _),
+	[63] = PINGROUP(63, qup1_se7, _, _, _, _, _, _, _, _, _),
+	[64] = PINGROUP(64, i2chub0_se0, _, _, _, _, _, _, _, _, _),
+	[65] = PINGROUP(65, i2chub0_se0, _, _, _, _, _, _, _, _, _),
+	[66] = PINGROUP(66, i2chub0_se1, _, _, _, _, _, _, _, _, _),
+	[67] = PINGROUP(67, i2chub0_se1, _, _, _, _, _, _, _, _, _),
+	[68] = PINGROUP(68, i2chub0_se2, pll_bist_sync, _, _, _, _, _, _, _, _),
+	[69] = PINGROUP(69, i2chub0_se2, _, _, _, _, _, _, _, _, _),
+	[70] = PINGROUP(70, i2chub0_se3, _, _, _, _, _, _, _, _, _),
+	[71] = PINGROUP(71, i2chub0_se3, _, atest_usb, _, _, _, _, _, _, _),
+	[72] = PINGROUP(72, i2chub0_se4, _, atest_usb, _, _, _, _, _, _, _),
+	[73] = PINGROUP(73, i2chub0_se4, jitter_bist, _, _, _, _, _, _, _, _),
+	[74] = PINGROUP(74, i2chub0_se5, atest_usb, _, _, _, _, _, _, _, _),
+	[75] = PINGROUP(75, i2chub0_se5, ddr_pxi0, _, _, _, _, _, _, _, _),
+	[76] = PINGROUP(76, i2chub0_se6, ddr_pxi0, uim1_present, _, _, _, _, _, _, _),
+	[77] = PINGROUP(77, i2chub0_se6, _, _, _, _, _, _, _, _, _),
+	[78] = PINGROUP(78, i2chub0_se7, qdss_cti, _, _, _, _, _, _, _, _),
+	[79] = PINGROUP(79, i2chub0_se7, qdss_cti, _, _, _, _, _, _, _, _),
+	[80] = PINGROUP(80, i2chub0_se9, _, _, _, _, _, _, _, _, _),
+	[81] = PINGROUP(81, i2chub0_se9, _, _, _, _, _, _, _, _, _),
+	[82] = PINGROUP(82, qdss_cti, _, _, _, _, _, _, _, _, _),
+	[83] = PINGROUP(83, qdss_cti, _, _, _, _, _, _, _, _, _),
+	[84] = PINGROUP(84, _, _, _, _, _, _, _, _, _, _),
+	[85] = PINGROUP(85, _, _, _, _, _, _, _, _, _, _),
+	[86] = PINGROUP(86, mdp_vsync, mdp_vsync0_out, mdp_vsync1_out, gcc_gp1, _, _, _, _, _, _),
+	[87] = PINGROUP(87, mdp_vsync, mdp_vsync2_out, mdp_vsync3_out, gcc_gp2, _, _, _, _, _, _),
+	[88] = PINGROUP(88, mdp_vsync_e, gcc_gp3, _, _, _, _, _, _, _, _),
+	[89] = PINGROUP(89, usb1_hs, gnss_adc0, _, _, _, _, _, _, _, _),
+	[90] = PINGROUP(90, gnss_adc1, _, _, _, _, _, _, _, _, _),
+	[91] = PINGROUP(91, _, gnss_adc0, _, _, _, _, _, _, _, _),
+	[92] = PINGROUP(92, dbg_out_clk, gnss_adc1, _, _, _, _, _, _, _, _),
+	[93] = PINGROUP(93, sd_write_protect, _, _, _, _, _, _, _, _, _),
+	[94] = PINGROUP(94, cmu_rng, phase_flag, tmess_prng0, _, _, _, _, _, _, _),
+	[95] = PINGROUP(95, pcie0_clk_req_n, cmu_rng, phase_flag, tmess_prng1, _, _, _, _, _, _),
+	[96] = PINGROUP(96, cmu_rng, phase_flag, tmess_prng2, _, _, _, _, _, _, _),
+	[97] = PINGROUP(97, _, _, _, _, _, _, _, _, _, _),
+	[98] = PINGROUP(98, pcie1_clk_req_n, _, _, _, _, _, _, _, _, _),
+	[99] = PINGROUP(99, _, _, _, _, _, _, _, _, _, _),
+	[100] = PINGROUP(100, cam_mclk, qdss_gpio, _, _, _, _, _, _, _, _),
+	[101] = PINGROUP(101, cam_mclk, qdss_gpio, _, _, _, _, _, _, _, _),
+	[102] = PINGROUP(102, cam_aon_mclk2, qdss_gpio, _, _, _, _, _, _, _, _),
+	[103] = PINGROUP(103, cam_mclk, qdss_gpio, _, _, _, _, _, _, _, _),
+	[104] = PINGROUP(104, cam_aon_mclk4, qdss_gpio, _, _, _, _, _, _, _, _),
+	[105] = PINGROUP(105, cam_mclk, qdss_gpio, _, _, _, _, _, _, _, _),
+	[106] = PINGROUP(106, cam_mclk, pll_clk_aux, _, _, _, _, _, _, _, _),
+	[107] = PINGROUP(107, _, _, _, _, _, _, _, _, _, _),
+	[108] = PINGROUP(108, cam_mclk, _, _, _, _, _, _, _, _, _),
+	[109] = PINGROUP(109, cci_timer, phase_flag, tmess_prng3, _, _, _, _, _, _, _),
+	[110] = PINGROUP(110, cci_timer, _, _, _, _, _, _, _, _, _),
+	[111] = PINGROUP(111, cci_timer, phase_flag, _, _, _, _, _, _, _, _),
+	[112] = PINGROUP(112, cci_i2c_sda, cmu_rng, phase_flag, _, _, _, _, _, _, _),
+	[113] = PINGROUP(113, cci_i2c_sda, phase_flag, _, qdss_gpio, _, _, _, _, _, _),
+	[114] = PINGROUP(114, cci_i2c_scl, phase_flag, _, qdss_gpio, _, _, _, _, _, _),
+	[115] = PINGROUP(115, cci_i2c_sda, phase_flag, _, qdss_gpio, _, _, _, _, _, _),
+	[116] = PINGROUP(116, cci_i2c_scl, phase_flag, _, qdss_gpio, _, _, _, _, _, _),
+	[117] = PINGROUP(117, cci_i2c_sda, phase_flag, _, qdss_gpio, _, _, _, _, _, _),
+	[118] = PINGROUP(118, cci_i2c_scl, phase_flag, _, qdss_gpio, _, _, _, _, _, _),
+	[119] = PINGROUP(119, cci_i2c_sda, phase_flag, _, _, _, _, _, _, _, _),
+	[120] = PINGROUP(120, cci_i2c_scl, phase_flag, _, _, _, _, _, _, _, _),
+	[121] = PINGROUP(121, i2s1_sck, _, _, _, _, _, _, _, _, _),
+	[122] = PINGROUP(122, i2s1_data0, cmu_rng, _, _, _, _, _, _, _, _),
+	[123] = PINGROUP(123, i2s1_ws, _, _, _, _, _, _, _, _, _),
+	[124] = PINGROUP(124, i2s1_data1, audio_ext_mclk1, audio_ref_clk, _, _, _, _, _, _, _),
+	[125] = PINGROUP(125, audio_ext_mclk0, _, _, _, _, _, _, _, _, _),
+	[126] = PINGROUP(126, i2s0_sck, _, _, _, _, _, _, _, _, _),
+	[127] = PINGROUP(127, i2s0_data0, cmu_rng, _, _, _, _, _, _, _, _),
+	[128] = PINGROUP(128, i2s0_data1, cmu_rng, _, _, _, _, _, _, _, _),
+	[129] = PINGROUP(129, i2s0_ws, cmu_rng, _, _, _, _, _, _, _, _),
+	[130] = PINGROUP(130, uim0_data, atest_usb, atest_char, _, _, _, _, _, _, _),
+	[131] = PINGROUP(131, uim0_clk, atest_usb, atest_char, _, _, _, _, _, _, _),
+	[132] = PINGROUP(132, uim0_reset, atest_char, _, _, _, _, _, _, _, _),
+	[133] = PINGROUP(133, mdp_vsync, atest_char, _, _, _, _, _, _, _, _),
+	[134] = PINGROUP(134, uim1_data, do_not, qspi0, sdc40, gcc_gp1, _, _, _, _, _),
+	[135] = PINGROUP(135, uim1_clk, do_not, qspi_clk, sdc4_clk, gcc_gp2, _, _, _, _, _),
+	[136] = PINGROUP(136, uim1_reset, do_not, qspi1, sdc41, gcc_gp3, _, _, _, _, _),
+	[137] = PINGROUP(137, mdp_vsync, atest_char, _, _, _, _, _, _, _, _),
+	[138] = PINGROUP(138, _, _, _, _, _, _, _, _, _, _),
+	[139] = PINGROUP(139, _, _, _, _, _, _, _, _, _, _),
+	[140] = PINGROUP(140, _, _, qdss_gpio, _, _, _, _, _, _, _),
+	[141] = PINGROUP(141, _, _, qdss_gpio, _, _, _, _, _, _, _),
+	[142] = PINGROUP(142, _, _, qdss_gpio, _, _, _, _, _, _, _),
+	[143] = PINGROUP(143, _, _, qdss_gpio, _, _, _, _, _, _, _),
+	[144] = PINGROUP(144, _, qdss_gpio, _, _, _, _, _, _, _, _),
+	[145] = PINGROUP(145, _, qdss_gpio, _, _, _, _, _, _, _, _),
+	[146] = PINGROUP(146, _, qdss_gpio, _, _, _, _, _, _, _, _),
+	[147] = PINGROUP(147, _, qdss_gpio, _, _, _, _, _, _, _, _),
+	[148] = PINGROUP(148, coex_uart1_rx, qdss_gpio, _, _, _, _, _, _, _, _),
+	[149] = PINGROUP(149, coex_uart1_tx, qdss_gpio, _, _, _, _, _, _, _, _),
+	[150] = PINGROUP(150, _, vfr_0, coex_uart2_rx, qdss_gpio, _, _, _, _, _, _),
+	[151] = PINGROUP(151, _, coex_uart2_tx, qdss_gpio, _, _, _, _, _, _, _),
+	[152] = PINGROUP(152, nav_gpio2, _, qdss_gpio, _, _, _, _, _, _, _),
+	[153] = PINGROUP(153, cci_i2c_scl, phase_flag, _, qdss_gpio, _, _, _, _, _, _),
+	[154] = PINGROUP(154, nav_gpio0, nav_gpio3, qdss_gpio, _, _, _, _, _, _, _),
+	[155] = PINGROUP(155, nav_gpio1, vfr_1, qdss_gpio, _, _, _, _, _, _, _),
+	[156] = PINGROUP(156, qlink_little_request, qdss_gpio, _, _, _, _, _, _, _, _),
+	[157] = PINGROUP(157, qlink_little_enable, qdss_gpio, _, _, _, _, _, _, _, _),
+	[158] = PINGROUP(158, qlink_wmss, qdss_gpio, _, _, _, _, _, _, _, _),
+	[159] = PINGROUP(159, qlink_big_request, qdss_cti, _, _, _, _, _, _, _, _),
+	[160] = PINGROUP(160, qlink_big_enable, _, _, _, _, _, _, _, _, _),
+	[161] = PINGROUP(161, _, _, _, _, _, _, _, _, _, _),
+	[162] = PINGROUP(162, qdss_cti, _, _, _, _, _, _, _, _, _),
+	[163] = PINGROUP(163, cci_async_in, phase_flag, _, _, _, _, _, _, _, _),
+	[164] = PINGROUP(164, cci_async_in, phase_flag, _, _, _, _, _, _, _, _),
+	[165] = PINGROUP(165, _, _, _, _, _, _, _, _, _, egpio),
+	[166] = PINGROUP(166, _, _, _, _, _, _, _, _, _, egpio),
+	[167] = PINGROUP(167, _, _, _, _, _, _, _, _, _, egpio),
+	[168] = PINGROUP(168, _, _, _, _, _, _, _, _, _, egpio),
+	[169] = PINGROUP(169, _, _, _, _, _, _, _, _, _, egpio),
+	[170] = PINGROUP(170, _, _, _, _, _, _, _, _, _, egpio),
+	[171] = PINGROUP(171, _, _, _, _, _, _, _, _, _, egpio),
+	[172] = PINGROUP(172, _, _, _, _, _, _, _, _, _, egpio),
+	[173] = PINGROUP(173, _, _, _, _, _, _, _, _, _, egpio),
+	[174] = PINGROUP(174, _, _, _, _, _, _, _, _, _, egpio),
+	[175] = PINGROUP(175, _, _, _, _, _, _, _, _, _, egpio),
+	[176] = PINGROUP(176, _, _, _, _, _, _, _, _, _, egpio),
+	[177] = PINGROUP(177, _, _, _, _, _, _, _, _, _, egpio),
+	[178] = PINGROUP(178, _, _, _, _, _, _, _, _, _, egpio),
+	[179] = PINGROUP(179, _, _, _, _, _, _, _, _, _, egpio),
+	[180] = PINGROUP(180, _, _, _, _, _, _, _, _, _, egpio),
+	[181] = PINGROUP(181, prng_rosc3, _, _, _, _, _, _, _, _, egpio),
+	[182] = PINGROUP(182, prng_rosc2, _, _, _, _, _, _, _, _, egpio),
+	[183] = PINGROUP(183, prng_rosc1, _, _, _, _, _, _, _, _, egpio),
+	[184] = PINGROUP(184, _, _, _, _, _, _, _, _, _, egpio),
+	[185] = PINGROUP(185, _, _, _, _, _, _, _, _, _, egpio),
+	[186] = PINGROUP(186, prng_rosc0, _, _, _, _, _, _, _, _, egpio),
+	[187] = PINGROUP(187, cri_trng, _, _, _, _, _, _, _, _, egpio),
+	[188] = PINGROUP(188, _, _, _, _, _, _, _, _, _, egpio),
+	[189] = PINGROUP(189, _, _, _, _, _, _, _, _, _, egpio),
+	[190] = PINGROUP(190, _, _, _, _, _, _, _, _, _, egpio),
+	[191] = PINGROUP(191, _, _, _, _, _, _, _, _, _, egpio),
+	[192] = PINGROUP(192, _, _, _, _, _, _, _, _, _, egpio),
+	[193] = PINGROUP(193, _, _, _, _, _, _, _, _, _, egpio),
+	[194] = PINGROUP(194, _, _, _, _, _, _, _, _, _, egpio),
+	[195] = PINGROUP(195, _, _, _, _, _, _, _, _, _, egpio),
+	[196] = PINGROUP(196, _, _, _, _, _, _, _, _, _, egpio),
+	[197] = PINGROUP(197, _, _, _, _, _, _, _, _, _, egpio),
+	[198] = PINGROUP(198, _, _, _, _, _, _, _, _, _, egpio),
+	[199] = PINGROUP(199, _, _, _, _, _, _, _, _, _, egpio),
+	[200] = PINGROUP(200, _, _, _, _, _, _, _, _, _, egpio),
+	[201] = PINGROUP(201, _, _, _, _, _, _, _, _, _, egpio),
+	[202] = PINGROUP(202, _, _, _, _, _, _, _, _, _, egpio),
+	[203] = PINGROUP(203, _, _, _, _, _, _, _, _, _, egpio),
+	[204] = PINGROUP(204, _, _, _, _, _, _, _, _, _, egpio),
+	[205] = PINGROUP(205, _, _, _, _, _, _, _, _, _, egpio),
+	[206] = PINGROUP(206, i2chub0_se8, _, _, _, _, _, _, _, _, egpio),
+	[207] = PINGROUP(207, i2chub0_se8, _, _, _, _, _, _, _, _, egpio),
+	[208] = PINGROUP(208, _, _, _, _, _, _, _, _, _, egpio),
+	[209] = PINGROUP(209, _, _, _, _, _, _, _, _, _, egpio),
+	[210] = UFS_RESET(ufs_reset, 0xde004, 0xdf000),
+	[211] = SDC_QDSD_PINGROUP(sdc2_clk, 0xd6000, 14, 6),
+	[212] = SDC_QDSD_PINGROUP(sdc2_cmd, 0xd6000, 11, 3),
+	[213] = SDC_QDSD_PINGROUP(sdc2_data, 0xd6000, 9, 0),
+};
+
+static const struct msm_gpio_wakeirq_map sm8650_pdc_map[] = {
+	{ 0, 94 },    { 3, 105 },   { 4, 78 },    { 7, 67 },    { 8, 64 },
+	{ 11, 121 },  { 12, 71 },   { 15, 82 },   { 18, 75 },   { 19, 63 },
+	{ 20, 114 },  { 23, 84 },   { 27, 61 },   { 29, 112 },  { 31, 113 },
+	{ 32, 66 },   { 35, 52 },   { 36, 123 },  { 39, 56 },   { 43, 59 },
+	{ 46, 79 },   { 47, 124 },  { 48, 125 },  { 51, 93 },   { 54, 60 },
+	{ 55, 104 },  { 56, 72 },   { 57, 77 },   { 59, 51 },   { 63, 85 },
+	{ 64, 107 },  { 65, 108 },  { 66, 109 },  { 67, 83 },   { 68, 110 },
+	{ 69, 111 },  { 75, 96 },   { 76, 97 },   { 77, 98 },   { 80, 89 },
+	{ 81, 90 },   { 84, 106 },  { 85, 100 },  { 86, 87 },   { 87, 88 },
+	{ 88, 65 },   { 90, 92 },   { 92, 99 },   { 95, 118 },  { 96, 119 },
+	{ 98, 101 },  { 99, 62 },   { 112, 120 }, { 133, 80 },  { 136, 69 },
+	{ 137, 81 },  { 148, 57 },  { 150, 58 },  { 152, 127 }, { 153, 74 },
+	{ 154, 126 }, { 155, 73 },  { 156, 128 }, { 159, 129 }, { 162, 86 },
+	{ 163, 122 }, { 166, 139 }, { 169, 140 }, { 171, 141 }, { 172, 142 },
+	{ 174, 102 }, { 176, 143 }, { 177, 55 },  { 181, 144 }, { 182, 145 },
+	{ 185, 146 }, { 187, 95 },  { 188, 130 }, { 190, 131 }, { 191, 132 },
+	{ 192, 133 }, { 193, 134 }, { 195, 68 },  { 196, 135 }, { 197, 136 },
+	{ 198, 54 },  { 199, 103 }, { 200, 53 },  { 201, 137 }, { 202, 70 },
+	{ 203, 138 }, { 204, 76 },  { 205, 91 },
+};
+
+static const struct msm_pinctrl_soc_data sm8650_tlmm = {
+	.pins = sm8650_pins,
+	.npins = ARRAY_SIZE(sm8650_pins),
+	.functions = sm8650_functions,
+	.nfunctions = ARRAY_SIZE(sm8650_functions),
+	.groups = sm8650_groups,
+	.ngroups = ARRAY_SIZE(sm8650_groups),
+	.ngpios = 211,
+	.wakeirq_map = sm8650_pdc_map,
+	.nwakeirq_map = ARRAY_SIZE(sm8650_pdc_map),
+	.egpio_func = 10,
+};
+
+static int sm8650_tlmm_probe(struct platform_device *pdev)
+{
+	return msm_pinctrl_probe(pdev, &sm8650_tlmm);
+}
+
+static const struct of_device_id sm8650_tlmm_of_match[] = {
+	{ .compatible = "qcom,sm8650-tlmm", },
+	{},
+};
+
+static struct platform_driver sm8650_tlmm_driver = {
+	.driver = {
+		.name = "sm8650-tlmm",
+		.of_match_table = sm8650_tlmm_of_match,
+	},
+	.probe = sm8650_tlmm_probe,
+	.remove_new = msm_pinctrl_remove,
+};
+
+static int __init sm8650_tlmm_init(void)
+{
+	return platform_driver_register(&sm8650_tlmm_driver);
+}
+arch_initcall(sm8650_tlmm_init);
+
+static void __exit sm8650_tlmm_exit(void)
+{
+	platform_driver_unregister(&sm8650_tlmm_driver);
+}
+module_exit(sm8650_tlmm_exit);
+
+MODULE_DESCRIPTION("QTI SM8650 TLMM driver");
+MODULE_LICENSE("GPL");
+MODULE_DEVICE_TABLE(of, sm8650_tlmm_of_match);
diff --git a/drivers/pinctrl/stm32/pinctrl-stm32.c b/drivers/pinctrl/stm32/pinctrl-stm32.c
index 64e8201c7eac..603f900e88c1 100644
--- a/drivers/pinctrl/stm32/pinctrl-stm32.c
+++ b/drivers/pinctrl/stm32/pinctrl-stm32.c
@@ -1273,9 +1273,11 @@ static struct stm32_desc_pin *stm32_pctrl_get_desc_pin_from_gpio(struct stm32_pi
 	int i;
 
 	/* With few exceptions (e.g. bank 'Z'), pin number matches with pin index in array */
-	pin_desc = pctl->pins + stm32_pin_nb;
-	if (pin_desc->pin.number == stm32_pin_nb)
-		return pin_desc;
+	if (stm32_pin_nb < pctl->npins) {
+		pin_desc = pctl->pins + stm32_pin_nb;
+		if (pin_desc->pin.number == stm32_pin_nb)
+			return pin_desc;
+	}
 
 	/* Otherwise, loop all array to find the pin with the right number */
 	for (i = 0; i < pctl->npins; i++) {
@@ -1368,6 +1370,11 @@ static int stm32_gpiolib_register_bank(struct stm32_pinctrl *pctl, struct fwnode
 	}
 
 	names = devm_kcalloc(dev, npins, sizeof(char *), GFP_KERNEL);
+	if (!names) {
+		err = -ENOMEM;
+		goto err_clk;
+	}
+
 	for (i = 0; i < npins; i++) {
 		stm32_pin = stm32_pctrl_get_desc_pin_from_gpio(pctl, bank, i);
 		if (stm32_pin && stm32_pin->pin.name)
diff --git a/drivers/pinctrl/tegra/pinctrl-tegra.c b/drivers/pinctrl/tegra/pinctrl-tegra.c
index 6bf8db424bec..ccfa3870a67d 100644
--- a/drivers/pinctrl/tegra/pinctrl-tegra.c
+++ b/drivers/pinctrl/tegra/pinctrl-tegra.c
@@ -636,6 +636,14 @@ static void tegra_pinconf_group_dbg_show(struct pinctrl_dev *pctldev,
 		seq_printf(s, "\n\t%s=%u",
 			   strip_prefix(cfg_params[i].property), val);
 	}
+
+	if (g->mux_reg >= 0) {
+		/* read pinmux function and dump to seq_file */
+		val = pmx_readl(pmx, g->mux_bank, g->mux_reg);
+		val = g->funcs[(val >> g->mux_bit) & 0x3];
+
+		seq_printf(s, "\n\tfunction=%s", pmx->functions[val].name);
+	}
 }
 
 static void tegra_pinconf_config_dbg_show(struct pinctrl_dev *pctldev,
diff --git a/drivers/platform/chrome/cros_ec_sensorhub_ring.c b/drivers/platform/chrome/cros_ec_sensorhub_ring.c
index 71948dade0e2..1205219515d6 100644
--- a/drivers/platform/chrome/cros_ec_sensorhub_ring.c
+++ b/drivers/platform/chrome/cros_ec_sensorhub_ring.c
@@ -103,7 +103,7 @@ EXPORT_SYMBOL_GPL(cros_ec_sensorhub_unregister_push_data);
  * @sensorhub: Sensor Hub object
  * @on: true when events are requested.
  *
- * To be called before sleeping or when noone is listening.
+ * To be called before sleeping or when no one is listening.
  * Return: 0 on success, or an error when we can not communicate with the EC.
  *
  */
@@ -133,33 +133,61 @@ int cros_ec_sensorhub_ring_fifo_enable(struct cros_ec_sensorhub *sensorhub,
 	return ret;
 }
 
-static int cros_ec_sensor_ring_median_cmp(const void *pv1, const void *pv2)
+static void cros_ec_sensor_ring_median_swap(s64 *a, s64 *b)
 {
-	s64 v1 = *(s64 *)pv1;
-	s64 v2 = *(s64 *)pv2;
-
-	if (v1 > v2)
-		return 1;
-	else if (v1 < v2)
-		return -1;
-	else
-		return 0;
+	s64 tmp = *a;
+	*a = *b;
+	*b = tmp;
 }
 
 /*
  * cros_ec_sensor_ring_median: Gets median of an array of numbers
  *
- * For now it's implemented using an inefficient > O(n) sort then return
- * the middle element. A more optimal method would be something like
- * quickselect, but given that n = 64 we can probably live with it in the
- * name of clarity.
+ * It's implemented using the quickselect algorithm, which achieves an
+ * average time complexity of O(n) the middle element. In the worst case,
+ * the runtime of quickselect could regress to O(n^2). To mitigate this,
+ * algorithms like median-of-medians exist, which can guarantee O(n) even
+ * in the worst case. However, these algorithms come with a higher
+ * overhead and are more complex to implement, making quickselect a
+ * pragmatic choice for our use case.
  *
- * Warning: the input array gets modified (sorted)!
+ * Warning: the input array gets modified!
  */
 static s64 cros_ec_sensor_ring_median(s64 *array, size_t length)
 {
-	sort(array, length, sizeof(s64), cros_ec_sensor_ring_median_cmp, NULL);
-	return array[length / 2];
+	int lo = 0;
+	int hi = length - 1;
+
+	while (lo <= hi) {
+		int mid = lo + (hi - lo) / 2;
+		int pivot, i;
+
+		if (array[lo] > array[mid])
+			cros_ec_sensor_ring_median_swap(&array[lo], &array[mid]);
+		if (array[lo] > array[hi])
+			cros_ec_sensor_ring_median_swap(&array[lo], &array[hi]);
+		if (array[mid] < array[hi])
+			cros_ec_sensor_ring_median_swap(&array[mid], &array[hi]);
+
+		pivot = array[hi];
+		i = lo - 1;
+
+		for (int j = lo; j < hi; j++)
+			if (array[j] < pivot)
+				cros_ec_sensor_ring_median_swap(&array[++i], &array[j]);
+
+		/* The pivot's index corresponds to i+1. */
+		cros_ec_sensor_ring_median_swap(&array[i + 1], &array[hi]);
+		if (i + 1 == length / 2)
+			return array[i + 1];
+		if (i + 1 > length / 2)
+			hi = i;
+		else
+			lo = i + 2;
+	}
+
+	/* Should never reach here. */
+	return -1;
 }
 
 /*
@@ -175,8 +203,8 @@ static s64 cros_ec_sensor_ring_median(s64 *array, size_t length)
  *
  * While a and b are recorded at accurate times (due to the EC real time
  * nature); c is pretty untrustworthy, even though it's recorded the
- * first thing in ec_irq_handler(). There is a very good change we'll get
- * added lantency due to:
+ * first thing in ec_irq_handler(). There is a very good chance we'll get
+ * added latency due to:
  *   other irqs
  *   ddrfreq
  *   cpuidle
@@ -511,7 +539,7 @@ cros_ec_sensor_ring_process_event(struct cros_ec_sensorhub *sensorhub,
  *                                 ringbuffer.
  *
  * This is the new spreading code, assumes every sample's timestamp
- * preceeds the sample. Run if tight_timestamps == true.
+ * precedes the sample. Run if tight_timestamps == true.
  *
  * Sometimes the EC receives only one interrupt (hence timestamp) for
  * a batch of samples. Only the first sample will have the correct
@@ -595,7 +623,7 @@ cros_ec_sensor_ring_spread_add(struct cros_ec_sensorhub *sensorhub,
 			} else {
 				/*
 				 * Push first sample in the batch to the,
-				 * kifo, it's guaranteed to be correct, the
+				 * kfifo, it's guaranteed to be correct, the
 				 * rest will follow later on.
 				 */
 				sample_idx = 1;
@@ -701,7 +729,7 @@ cros_ec_sensor_ring_spread_add(struct cros_ec_sensorhub *sensorhub,
  *           last_out -->
  *
  *
- * We spread time for the samples using perod p = (current - TS1)/4.
+ * We spread time for the samples using period p = (current - TS1)/4.
  * between TS1 and TS2: [TS1+p/4, TS1+2p/4, TS1+3p/4, current_timestamp].
  *
  */
diff --git a/drivers/platform/x86/amd/pmc/pmc.c b/drivers/platform/x86/amd/pmc/pmc.c
index cd6ac04c1468..c3104714b480 100644
--- a/drivers/platform/x86/amd/pmc/pmc.c
+++ b/drivers/platform/x86/amd/pmc/pmc.c
@@ -964,33 +964,6 @@ static const struct pci_device_id pmc_pci_ids[] = {
 	{ }
 };
 
-static int amd_pmc_get_dram_size(struct amd_pmc_dev *dev)
-{
-	int ret;
-
-	switch (dev->cpu_id) {
-	case AMD_CPU_ID_YC:
-		if (!(dev->major > 90 || (dev->major == 90 && dev->minor > 39))) {
-			ret = -EINVAL;
-			goto err_dram_size;
-		}
-		break;
-	default:
-		ret = -EINVAL;
-		goto err_dram_size;
-	}
-
-	ret = amd_pmc_send_cmd(dev, S2D_DRAM_SIZE, &dev->dram_size, dev->s2d_msg_id, true);
-	if (ret || !dev->dram_size)
-		goto err_dram_size;
-
-	return 0;
-
-err_dram_size:
-	dev_err(dev->dev, "DRAM size command not supported for this platform\n");
-	return ret;
-}
-
 static int amd_pmc_s2d_init(struct amd_pmc_dev *dev)
 {
 	u32 phys_addr_low, phys_addr_hi;
@@ -1009,8 +982,8 @@ static int amd_pmc_s2d_init(struct amd_pmc_dev *dev)
 		return -EIO;
 
 	/* Get DRAM size */
-	ret = amd_pmc_get_dram_size(dev);
-	if (ret)
+	ret = amd_pmc_send_cmd(dev, S2D_DRAM_SIZE, &dev->dram_size, dev->s2d_msg_id, true);
+	if (ret || !dev->dram_size)
 		dev->dram_size = S2D_TELEMETRY_DRAMBYTES_MAX;
 
 	/* Get STB DRAM address */
diff --git a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c
index 5798b49ddaba..8c9f4f3227fc 100644
--- a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c
+++ b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c
@@ -588,17 +588,14 @@ static void release_attributes_data(void)
 static int hp_add_other_attributes(int attr_type)
 {
 	struct kobject *attr_name_kobj;
-	union acpi_object *obj = NULL;
 	int ret;
 	char *attr_name;
 
-	mutex_lock(&bioscfg_drv.mutex);
-
 	attr_name_kobj = kzalloc(sizeof(*attr_name_kobj), GFP_KERNEL);
-	if (!attr_name_kobj) {
-		ret = -ENOMEM;
-		goto err_other_attr_init;
-	}
+	if (!attr_name_kobj)
+		return -ENOMEM;
+
+	mutex_lock(&bioscfg_drv.mutex);
 
 	/* Check if attribute type is supported */
 	switch (attr_type) {
@@ -615,14 +612,14 @@ static int hp_add_other_attributes(int attr_type)
 	default:
 		pr_err("Error: Unknown attr_type: %d\n", attr_type);
 		ret = -EINVAL;
-		goto err_other_attr_init;
+		kfree(attr_name_kobj);
+		goto unlock_drv_mutex;
 	}
 
 	ret = kobject_init_and_add(attr_name_kobj, &attr_name_ktype,
 				   NULL, "%s", attr_name);
 	if (ret) {
 		pr_err("Error encountered [%d]\n", ret);
-		kobject_put(attr_name_kobj);
 		goto err_other_attr_init;
 	}
 
@@ -630,27 +627,26 @@ static int hp_add_other_attributes(int attr_type)
 	switch (attr_type) {
 	case HPWMI_SECURE_PLATFORM_TYPE:
 		ret = hp_populate_secure_platform_data(attr_name_kobj);
-		if (ret)
-			goto err_other_attr_init;
 		break;
 
 	case HPWMI_SURE_START_TYPE:
 		ret = hp_populate_sure_start_data(attr_name_kobj);
-		if (ret)
-			goto err_other_attr_init;
 		break;
 
 	default:
 		ret = -EINVAL;
-		goto err_other_attr_init;
 	}
 
+	if (ret)
+		goto err_other_attr_init;
+
 	mutex_unlock(&bioscfg_drv.mutex);
 	return 0;
 
 err_other_attr_init:
+	kobject_put(attr_name_kobj);
+unlock_drv_mutex:
 	mutex_unlock(&bioscfg_drv.mutex);
-	kfree(obj);
 	return ret;
 }
 
diff --git a/drivers/platform/x86/ideapad-laptop.c b/drivers/platform/x86/ideapad-laptop.c
index ac037540acfc..88eefccb6ed2 100644
--- a/drivers/platform/x86/ideapad-laptop.c
+++ b/drivers/platform/x86/ideapad-laptop.c
@@ -1425,18 +1425,17 @@ static int ideapad_kbd_bl_init(struct ideapad_private *priv)
 	if (WARN_ON(priv->kbd_bl.initialized))
 		return -EEXIST;
 
-	brightness = ideapad_kbd_bl_brightness_get(priv);
-	if (brightness < 0)
-		return brightness;
-
-	priv->kbd_bl.last_brightness = brightness;
-
 	if (ideapad_kbd_bl_check_tristate(priv->kbd_bl.type)) {
 		priv->kbd_bl.led.max_brightness = 2;
 	} else {
 		priv->kbd_bl.led.max_brightness = 1;
 	}
 
+	brightness = ideapad_kbd_bl_brightness_get(priv);
+	if (brightness < 0)
+		return brightness;
+
+	priv->kbd_bl.last_brightness = brightness;
 	priv->kbd_bl.led.name                    = "platform::" LED_FUNCTION_KBD_BACKLIGHT;
 	priv->kbd_bl.led.brightness_get          = ideapad_kbd_bl_led_cdev_brightness_get;
 	priv->kbd_bl.led.brightness_set_blocking = ideapad_kbd_bl_led_cdev_brightness_set;
diff --git a/drivers/platform/x86/intel/telemetry/core.c b/drivers/platform/x86/intel/telemetry/core.c
index fdf55b5d6948..e4be40f73eeb 100644
--- a/drivers/platform/x86/intel/telemetry/core.c
+++ b/drivers/platform/x86/intel/telemetry/core.c
@@ -102,7 +102,7 @@ static const struct telemetry_core_ops telm_defpltops = {
 /**
  * telemetry_update_events() - Update telemetry Configuration
  * @pss_evtconfig: PSS related config. No change if num_evts = 0.
- * @pss_evtconfig: IOSS related config. No change if num_evts = 0.
+ * @ioss_evtconfig: IOSS related config. No change if num_evts = 0.
  *
  * This API updates the IOSS & PSS Telemetry configuration. Old config
  * is overwritten. Call telemetry_reset_events when logging is over
@@ -176,7 +176,7 @@ EXPORT_SYMBOL_GPL(telemetry_reset_events);
 /**
  * telemetry_get_eventconfig() - Returns the pss and ioss events enabled
  * @pss_evtconfig: Pointer to PSS related configuration.
- * @pss_evtconfig: Pointer to IOSS related configuration.
+ * @ioss_evtconfig: Pointer to IOSS related configuration.
  * @pss_len:	   Number of u32 elements allocated for pss_evtconfig array
  * @ioss_len:	   Number of u32 elements allocated for ioss_evtconfig array
  *
diff --git a/drivers/power/reset/as3722-poweroff.c b/drivers/power/reset/as3722-poweroff.c
index 829e0dba2fda..ab3350ce2d62 100644
--- a/drivers/power/reset/as3722-poweroff.c
+++ b/drivers/power/reset/as3722-poweroff.c
@@ -61,13 +61,11 @@ static int as3722_poweroff_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int as3722_poweroff_remove(struct platform_device *pdev)
+static void as3722_poweroff_remove(struct platform_device *pdev)
 {
 	if (pm_power_off == as3722_pm_power_off)
 		pm_power_off = NULL;
 	as3722_pm_poweroff = NULL;
-
-	return 0;
 }
 
 static struct platform_driver as3722_poweroff_driver = {
@@ -75,7 +73,7 @@ static struct platform_driver as3722_poweroff_driver = {
 		.name = "as3722-power-off",
 	},
 	.probe = as3722_poweroff_probe,
-	.remove = as3722_poweroff_remove,
+	.remove_new = as3722_poweroff_remove,
 };
 
 module_platform_driver(as3722_poweroff_driver);
diff --git a/drivers/power/reset/at91-poweroff.c b/drivers/power/reset/at91-poweroff.c
index dd5399785b69..126e774e210c 100644
--- a/drivers/power/reset/at91-poweroff.c
+++ b/drivers/power/reset/at91-poweroff.c
@@ -149,7 +149,7 @@ static void at91_poweroff_dt_set_wakeup_mode(struct platform_device *pdev)
 	writel(wakeup_mode | mode, at91_shdwc.shdwc_base + AT91_SHDW_MR);
 }
 
-static int __init at91_poweroff_probe(struct platform_device *pdev)
+static int at91_poweroff_probe(struct platform_device *pdev)
 {
 	struct device_node *np;
 	u32 ddr_type;
@@ -202,7 +202,7 @@ static int __init at91_poweroff_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int __exit at91_poweroff_remove(struct platform_device *pdev)
+static void at91_poweroff_remove(struct platform_device *pdev)
 {
 	if (pm_power_off == at91_poweroff)
 		pm_power_off = NULL;
@@ -211,8 +211,6 @@ static int __exit at91_poweroff_remove(struct platform_device *pdev)
 		iounmap(at91_shdwc.mpddrc_base);
 
 	clk_disable_unprepare(at91_shdwc.sclk);
-
-	return 0;
 }
 
 static const struct of_device_id at91_poweroff_of_match[] = {
@@ -224,13 +222,14 @@ static const struct of_device_id at91_poweroff_of_match[] = {
 MODULE_DEVICE_TABLE(of, at91_poweroff_of_match);
 
 static struct platform_driver at91_poweroff_driver = {
-	.remove = __exit_p(at91_poweroff_remove),
+	.probe = at91_poweroff_probe,
+	.remove_new = at91_poweroff_remove,
 	.driver = {
 		.name = "at91-poweroff",
 		.of_match_table = at91_poweroff_of_match,
 	},
 };
-module_platform_driver_probe(at91_poweroff_driver, at91_poweroff_probe);
+module_platform_driver(at91_poweroff_driver);
 
 MODULE_AUTHOR("Atmel Corporation");
 MODULE_DESCRIPTION("Shutdown driver for Atmel SoCs");
diff --git a/drivers/power/reset/at91-reset.c b/drivers/power/reset/at91-reset.c
index aa9b012d3d00..16512654295f 100644
--- a/drivers/power/reset/at91-reset.c
+++ b/drivers/power/reset/at91-reset.c
@@ -337,7 +337,7 @@ static int at91_rcdev_init(struct at91_reset *reset,
 	return devm_reset_controller_register(&pdev->dev, &reset->rcdev);
 }
 
-static int __init at91_reset_probe(struct platform_device *pdev)
+static int at91_reset_probe(struct platform_device *pdev)
 {
 	const struct of_device_id *match;
 	struct at91_reset *reset;
@@ -417,24 +417,23 @@ static int __init at91_reset_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int __exit at91_reset_remove(struct platform_device *pdev)
+static void at91_reset_remove(struct platform_device *pdev)
 {
 	struct at91_reset *reset = platform_get_drvdata(pdev);
 
 	unregister_restart_handler(&reset->nb);
 	clk_disable_unprepare(reset->sclk);
-
-	return 0;
 }
 
 static struct platform_driver at91_reset_driver = {
-	.remove = __exit_p(at91_reset_remove),
+	.probe = at91_reset_probe,
+	.remove_new = at91_reset_remove,
 	.driver = {
 		.name = "at91-reset",
 		.of_match_table = at91_reset_of_match,
 	},
 };
-module_platform_driver_probe(at91_reset_driver, at91_reset_probe);
+module_platform_driver(at91_reset_driver);
 
 MODULE_AUTHOR("Atmel Corporation");
 MODULE_DESCRIPTION("Reset driver for Atmel SoCs");
diff --git a/drivers/power/reset/at91-sama5d2_shdwc.c b/drivers/power/reset/at91-sama5d2_shdwc.c
index e76b102b57b1..af95c7b39cb3 100644
--- a/drivers/power/reset/at91-sama5d2_shdwc.c
+++ b/drivers/power/reset/at91-sama5d2_shdwc.c
@@ -329,7 +329,7 @@ static const struct of_device_id at91_pmc_ids[] = {
 	{ /* Sentinel. */ }
 };
 
-static int __init at91_shdwc_probe(struct platform_device *pdev)
+static int at91_shdwc_probe(struct platform_device *pdev)
 {
 	const struct of_device_id *match;
 	struct device_node *np;
@@ -421,7 +421,7 @@ static int __init at91_shdwc_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int __exit at91_shdwc_remove(struct platform_device *pdev)
+static void at91_shdwc_remove(struct platform_device *pdev)
 {
 	struct shdwc *shdw = platform_get_drvdata(pdev);
 
@@ -437,18 +437,17 @@ static int __exit at91_shdwc_remove(struct platform_device *pdev)
 	iounmap(shdw->pmc_base);
 
 	clk_disable_unprepare(shdw->sclk);
-
-	return 0;
 }
 
 static struct platform_driver at91_shdwc_driver = {
-	.remove = __exit_p(at91_shdwc_remove),
+	.probe = at91_shdwc_probe,
+	.remove_new = at91_shdwc_remove,
 	.driver = {
 		.name = "at91-shdwc",
 		.of_match_table = at91_shdwc_of_match,
 	},
 };
-module_platform_driver_probe(at91_shdwc_driver, at91_shdwc_probe);
+module_platform_driver(at91_shdwc_driver);
 
 MODULE_AUTHOR("Nicolas Ferre <nicolas.ferre@atmel.com>");
 MODULE_DESCRIPTION("Atmel shutdown controller driver");
diff --git a/drivers/power/reset/atc260x-poweroff.c b/drivers/power/reset/atc260x-poweroff.c
index 98f20251a6d1..b4aa50e9685e 100644
--- a/drivers/power/reset/atc260x-poweroff.c
+++ b/drivers/power/reset/atc260x-poweroff.c
@@ -233,7 +233,7 @@ static int atc260x_pwrc_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int atc260x_pwrc_remove(struct platform_device *pdev)
+static void atc260x_pwrc_remove(struct platform_device *pdev)
 {
 	struct atc260x_pwrc *priv = platform_get_drvdata(pdev);
 
@@ -243,13 +243,11 @@ static int atc260x_pwrc_remove(struct platform_device *pdev)
 	}
 
 	unregister_restart_handler(&priv->restart_nb);
-
-	return 0;
 }
 
 static struct platform_driver atc260x_pwrc_driver = {
 	.probe = atc260x_pwrc_probe,
-	.remove = atc260x_pwrc_remove,
+	.remove_new = atc260x_pwrc_remove,
 	.driver = {
 		.name = "atc260x-pwrc",
 	},
diff --git a/drivers/power/reset/ltc2952-poweroff.c b/drivers/power/reset/ltc2952-poweroff.c
index eea05921a054..fa25fbd53934 100644
--- a/drivers/power/reset/ltc2952-poweroff.c
+++ b/drivers/power/reset/ltc2952-poweroff.c
@@ -286,7 +286,7 @@ static int ltc2952_poweroff_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int ltc2952_poweroff_remove(struct platform_device *pdev)
+static void ltc2952_poweroff_remove(struct platform_device *pdev)
 {
 	struct ltc2952_poweroff *data = platform_get_drvdata(pdev);
 
@@ -295,7 +295,6 @@ static int ltc2952_poweroff_remove(struct platform_device *pdev)
 	hrtimer_cancel(&data->timer_wde);
 	atomic_notifier_chain_unregister(&panic_notifier_list,
 					 &data->panic_notifier);
-	return 0;
 }
 
 static const struct of_device_id of_ltc2952_poweroff_match[] = {
@@ -306,7 +305,7 @@ MODULE_DEVICE_TABLE(of, of_ltc2952_poweroff_match);
 
 static struct platform_driver ltc2952_poweroff_driver = {
 	.probe = ltc2952_poweroff_probe,
-	.remove = ltc2952_poweroff_remove,
+	.remove_new = ltc2952_poweroff_remove,
 	.driver = {
 		.name = "ltc2952-poweroff",
 		.of_match_table = of_ltc2952_poweroff_match,
diff --git a/drivers/power/reset/mt6323-poweroff.c b/drivers/power/reset/mt6323-poweroff.c
index 108167f7738b..57a63c0ab7fb 100644
--- a/drivers/power/reset/mt6323-poweroff.c
+++ b/drivers/power/reset/mt6323-poweroff.c
@@ -70,12 +70,10 @@ static int mt6323_pwrc_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int mt6323_pwrc_remove(struct platform_device *pdev)
+static void mt6323_pwrc_remove(struct platform_device *pdev)
 {
 	if (pm_power_off == &mt6323_do_pwroff)
 		pm_power_off = NULL;
-
-	return 0;
 }
 
 static const struct of_device_id mt6323_pwrc_dt_match[] = {
@@ -86,7 +84,7 @@ MODULE_DEVICE_TABLE(of, mt6323_pwrc_dt_match);
 
 static struct platform_driver mt6323_pwrc_driver = {
 	.probe          = mt6323_pwrc_probe,
-	.remove         = mt6323_pwrc_remove,
+	.remove_new     = mt6323_pwrc_remove,
 	.driver         = {
 		.name   = "mt6323-pwrc",
 		.of_match_table = mt6323_pwrc_dt_match,
diff --git a/drivers/power/reset/pwr-mlxbf.c b/drivers/power/reset/pwr-mlxbf.c
index de35d24bb7ef..1775b318d0ef 100644
--- a/drivers/power/reset/pwr-mlxbf.c
+++ b/drivers/power/reset/pwr-mlxbf.c
@@ -17,11 +17,17 @@
 #include <linux/types.h>
 
 struct pwr_mlxbf {
-	struct work_struct send_work;
+	struct work_struct reboot_work;
+	struct work_struct shutdown_work;
 	const char *hid;
 };
 
-static void pwr_mlxbf_send_work(struct work_struct *work)
+static void pwr_mlxbf_reboot_work(struct work_struct *work)
+{
+	acpi_bus_generate_netlink_event("button/reboot.*", "Reboot Button", 0x80, 1);
+}
+
+static void pwr_mlxbf_shutdown_work(struct work_struct *work)
 {
 	acpi_bus_generate_netlink_event("button/power.*", "Power Button", 0x80, 1);
 }
@@ -33,10 +39,10 @@ static irqreturn_t pwr_mlxbf_irq(int irq, void *ptr)
 	struct pwr_mlxbf *priv = ptr;
 
 	if (!strncmp(priv->hid, rst_pwr_hid, 8))
-		emergency_restart();
+		schedule_work(&priv->reboot_work);
 
 	if (!strncmp(priv->hid, low_pwr_hid, 8))
-		schedule_work(&priv->send_work);
+		schedule_work(&priv->shutdown_work);
 
 	return IRQ_HANDLED;
 }
@@ -64,7 +70,11 @@ static int pwr_mlxbf_probe(struct platform_device *pdev)
 	if (irq < 0)
 		return dev_err_probe(dev, irq, "Error getting %s irq.\n", priv->hid);
 
-	err = devm_work_autocancel(dev, &priv->send_work, pwr_mlxbf_send_work);
+	err = devm_work_autocancel(dev, &priv->shutdown_work, pwr_mlxbf_shutdown_work);
+	if (err)
+		return err;
+
+	err = devm_work_autocancel(dev, &priv->reboot_work, pwr_mlxbf_reboot_work);
 	if (err)
 		return err;
 
diff --git a/drivers/power/reset/qnap-poweroff.c b/drivers/power/reset/qnap-poweroff.c
index 0ddf7f25f7b8..e0f2ff6b147c 100644
--- a/drivers/power/reset/qnap-poweroff.c
+++ b/drivers/power/reset/qnap-poweroff.c
@@ -111,15 +111,14 @@ static int qnap_power_off_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int qnap_power_off_remove(struct platform_device *pdev)
+static void qnap_power_off_remove(struct platform_device *pdev)
 {
 	pm_power_off = NULL;
-	return 0;
 }
 
 static struct platform_driver qnap_power_off_driver = {
 	.probe	= qnap_power_off_probe,
-	.remove	= qnap_power_off_remove,
+	.remove_new = qnap_power_off_remove,
 	.driver	= {
 		.name	= "qnap_power_off",
 		.of_match_table = of_match_ptr(qnap_power_off_of_match_table),
diff --git a/drivers/power/reset/regulator-poweroff.c b/drivers/power/reset/regulator-poweroff.c
index 7f87fbb8b051..15160809c423 100644
--- a/drivers/power/reset/regulator-poweroff.c
+++ b/drivers/power/reset/regulator-poweroff.c
@@ -52,12 +52,10 @@ static int regulator_poweroff_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int regulator_poweroff_remove(__maybe_unused struct platform_device *pdev)
+static void regulator_poweroff_remove(struct platform_device *pdev)
 {
 	if (pm_power_off == &regulator_poweroff_do_poweroff)
 		pm_power_off = NULL;
-
-	return 0;
 }
 
 static const struct of_device_id of_regulator_poweroff_match[] = {
@@ -68,7 +66,7 @@ MODULE_DEVICE_TABLE(of, of_regulator_poweroff_match);
 
 static struct platform_driver regulator_poweroff_driver = {
 	.probe = regulator_poweroff_probe,
-	.remove = regulator_poweroff_remove,
+	.remove_new = regulator_poweroff_remove,
 	.driver = {
 		.name = "poweroff-regulator",
 		.of_match_table = of_regulator_poweroff_match,
diff --git a/drivers/power/reset/restart-poweroff.c b/drivers/power/reset/restart-poweroff.c
index 28f1822db162..f4d6004793d3 100644
--- a/drivers/power/reset/restart-poweroff.c
+++ b/drivers/power/reset/restart-poweroff.c
@@ -33,12 +33,10 @@ static int restart_poweroff_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int restart_poweroff_remove(struct platform_device *pdev)
+static void restart_poweroff_remove(struct platform_device *pdev)
 {
 	if (pm_power_off == &restart_poweroff_do_poweroff)
 		pm_power_off = NULL;
-
-	return 0;
 }
 
 static const struct of_device_id of_restart_poweroff_match[] = {
@@ -49,7 +47,7 @@ MODULE_DEVICE_TABLE(of, of_restart_poweroff_match);
 
 static struct platform_driver restart_poweroff_driver = {
 	.probe = restart_poweroff_probe,
-	.remove = restart_poweroff_remove,
+	.remove_new = restart_poweroff_remove,
 	.driver = {
 		.name = "poweroff-restart",
 		.of_match_table = of_restart_poweroff_match,
diff --git a/drivers/power/reset/rmobile-reset.c b/drivers/power/reset/rmobile-reset.c
index bd3b396558e0..5df9b41c68c7 100644
--- a/drivers/power/reset/rmobile-reset.c
+++ b/drivers/power/reset/rmobile-reset.c
@@ -59,11 +59,10 @@ static int rmobile_reset_probe(struct platform_device *pdev)
 	return error;
 }
 
-static int rmobile_reset_remove(struct platform_device *pdev)
+static void rmobile_reset_remove(struct platform_device *pdev)
 {
 	unregister_restart_handler(&rmobile_reset_nb);
 	iounmap(sysc_base2);
-	return 0;
 }
 
 static const struct of_device_id rmobile_reset_of_match[] = {
@@ -74,7 +73,7 @@ MODULE_DEVICE_TABLE(of, rmobile_reset_of_match);
 
 static struct platform_driver rmobile_reset_driver = {
 	.probe = rmobile_reset_probe,
-	.remove = rmobile_reset_remove,
+	.remove_new = rmobile_reset_remove,
 	.driver = {
 		.name = "rmobile_reset",
 		.of_match_table = rmobile_reset_of_match,
diff --git a/drivers/power/reset/syscon-poweroff.c b/drivers/power/reset/syscon-poweroff.c
index c3aab7f59345..1b2ce7734260 100644
--- a/drivers/power/reset/syscon-poweroff.c
+++ b/drivers/power/reset/syscon-poweroff.c
@@ -76,12 +76,10 @@ static int syscon_poweroff_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int syscon_poweroff_remove(struct platform_device *pdev)
+static void syscon_poweroff_remove(struct platform_device *pdev)
 {
 	if (pm_power_off == syscon_poweroff)
 		pm_power_off = NULL;
-
-	return 0;
 }
 
 static const struct of_device_id syscon_poweroff_of_match[] = {
@@ -91,7 +89,7 @@ static const struct of_device_id syscon_poweroff_of_match[] = {
 
 static struct platform_driver syscon_poweroff_driver = {
 	.probe = syscon_poweroff_probe,
-	.remove = syscon_poweroff_remove,
+	.remove_new = syscon_poweroff_remove,
 	.driver = {
 		.name = "syscon-poweroff",
 		.of_match_table = syscon_poweroff_of_match,
diff --git a/drivers/power/reset/tps65086-restart.c b/drivers/power/reset/tps65086-restart.c
index 5ec819eac7da..ee8e9f4b837e 100644
--- a/drivers/power/reset/tps65086-restart.c
+++ b/drivers/power/reset/tps65086-restart.c
@@ -62,19 +62,21 @@ static int tps65086_restart_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int tps65086_restart_remove(struct platform_device *pdev)
+static void tps65086_restart_remove(struct platform_device *pdev)
 {
 	struct tps65086_restart *tps65086_restart = platform_get_drvdata(pdev);
 	int ret;
 
 	ret = unregister_restart_handler(&tps65086_restart->handler);
 	if (ret) {
+		/*
+		 * tps65086_restart_probe() registered the restart handler. So
+		 * unregistering should work fine. Checking the error code
+		 * shouldn't be needed, still doing it for completeness.
+		 */
 		dev_err(&pdev->dev, "%s: cannot unregister restart handler: %d\n",
 			__func__, ret);
-		return -ENODEV;
 	}
-
-	return 0;
 }
 
 static const struct platform_device_id tps65086_restart_id_table[] = {
@@ -88,7 +90,7 @@ static struct platform_driver tps65086_restart_driver = {
 		.name = "tps65086-restart",
 	},
 	.probe = tps65086_restart_probe,
-	.remove = tps65086_restart_remove,
+	.remove_new = tps65086_restart_remove,
 	.id_table = tps65086_restart_id_table,
 };
 module_platform_driver(tps65086_restart_driver);
diff --git a/drivers/power/supply/bq256xx_charger.c b/drivers/power/supply/bq256xx_charger.c
index 789a31bd70c3..1a935bc88510 100644
--- a/drivers/power/supply/bq256xx_charger.c
+++ b/drivers/power/supply/bq256xx_charger.c
@@ -1574,13 +1574,16 @@ static int bq256xx_hw_init(struct bq256xx_device *bq)
 			wd_reg_val = i;
 			break;
 		}
-		if (bq->watchdog_timer > bq256xx_watchdog_time[i] &&
+		if (i + 1 < BQ256XX_NUM_WD_VAL &&
+		    bq->watchdog_timer > bq256xx_watchdog_time[i] &&
 		    bq->watchdog_timer < bq256xx_watchdog_time[i + 1])
 			wd_reg_val = i;
 	}
 	ret = regmap_update_bits(bq->regmap, BQ256XX_CHARGER_CONTROL_1,
 				 BQ256XX_WATCHDOG_MASK, wd_reg_val <<
 						BQ256XX_WDT_BIT_SHIFT);
+	if (ret)
+		return ret;
 
 	ret = power_supply_get_battery_info(bq->charger, &bat_info);
 	if (ret == -ENOMEM)
diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c
index 4296600e8912..1c4a9d137744 100644
--- a/drivers/power/supply/bq27xxx_battery.c
+++ b/drivers/power/supply/bq27xxx_battery.c
@@ -2162,6 +2162,28 @@ void bq27xxx_battery_teardown(struct bq27xxx_device_info *di)
 }
 EXPORT_SYMBOL_GPL(bq27xxx_battery_teardown);
 
+#ifdef CONFIG_PM_SLEEP
+static int bq27xxx_battery_suspend(struct device *dev)
+{
+	struct bq27xxx_device_info *di = dev_get_drvdata(dev);
+
+	cancel_delayed_work(&di->work);
+	return 0;
+}
+
+static int bq27xxx_battery_resume(struct device *dev)
+{
+	struct bq27xxx_device_info *di = dev_get_drvdata(dev);
+
+	schedule_delayed_work(&di->work, 0);
+	return 0;
+}
+#endif /* CONFIG_PM_SLEEP */
+
+SIMPLE_DEV_PM_OPS(bq27xxx_battery_battery_pm_ops,
+		  bq27xxx_battery_suspend, bq27xxx_battery_resume);
+EXPORT_SYMBOL_GPL(bq27xxx_battery_battery_pm_ops);
+
 MODULE_AUTHOR("Rodolfo Giometti <giometti@linux.it>");
 MODULE_DESCRIPTION("BQ27xxx battery monitor driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/power/supply/bq27xxx_battery_i2c.c b/drivers/power/supply/bq27xxx_battery_i2c.c
index 9b5475590518..3a1798b0c1a7 100644
--- a/drivers/power/supply/bq27xxx_battery_i2c.c
+++ b/drivers/power/supply/bq27xxx_battery_i2c.c
@@ -295,6 +295,7 @@ static struct i2c_driver bq27xxx_battery_i2c_driver = {
 	.driver = {
 		.name = "bq27xxx-battery",
 		.of_match_table = of_match_ptr(bq27xxx_battery_i2c_of_match_table),
+		.pm = &bq27xxx_battery_battery_pm_ops,
 	},
 	.probe = bq27xxx_battery_i2c_probe,
 	.remove = bq27xxx_battery_i2c_remove,
diff --git a/drivers/power/supply/cw2015_battery.c b/drivers/power/supply/cw2015_battery.c
index bb29e9ebd24a..99f3ccdc30a6 100644
--- a/drivers/power/supply/cw2015_battery.c
+++ b/drivers/power/supply/cw2015_battery.c
@@ -491,7 +491,7 @@ static int cw_battery_get_property(struct power_supply *psy,
 
 	case POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW:
 		if (cw_battery_valid_time_to_empty(cw_bat))
-			val->intval = cw_bat->time_to_empty;
+			val->intval = cw_bat->time_to_empty * 60;
 		else
 			val->intval = 0;
 		break;
diff --git a/drivers/ptp/ptp_ines.c b/drivers/ptp/ptp_ines.c
index ed215b458183..1d2940a78455 100644
--- a/drivers/ptp/ptp_ines.c
+++ b/drivers/ptp/ptp_ines.c
@@ -328,17 +328,15 @@ static u64 ines_find_txts(struct ines_port *port, struct sk_buff *skb)
 	return ns;
 }
 
-static int ines_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr)
+static int ines_hwtstamp(struct mii_timestamper *mii_ts,
+			 struct kernel_hwtstamp_config *cfg,
+			 struct netlink_ext_ack *extack)
 {
 	struct ines_port *port = container_of(mii_ts, struct ines_port, mii_ts);
 	u32 cm_one_step = 0, port_conf, ts_stat_rx, ts_stat_tx;
-	struct hwtstamp_config cfg;
 	unsigned long flags;
 
-	if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
-		return -EFAULT;
-
-	switch (cfg.tx_type) {
+	switch (cfg->tx_type) {
 	case HWTSTAMP_TX_OFF:
 		ts_stat_tx = 0;
 		break;
@@ -353,7 +351,7 @@ static int ines_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr)
 		return -ERANGE;
 	}
 
-	switch (cfg.rx_filter) {
+	switch (cfg->rx_filter) {
 	case HWTSTAMP_FILTER_NONE:
 		ts_stat_rx = 0;
 		break;
@@ -372,7 +370,7 @@ static int ines_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr)
 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 		ts_stat_rx = TS_ENABLE;
-		cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
+		cfg->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
 		break;
 	default:
 		return -ERANGE;
@@ -393,7 +391,7 @@ static int ines_hwtstamp(struct mii_timestamper *mii_ts, struct ifreq *ifr)
 
 	spin_unlock_irqrestore(&port->lock, flags);
 
-	return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
+	return 0;
 }
 
 static void ines_link_state(struct mii_timestamper *mii_ts,
diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c
index 3137e40fcd3e..4aa9ec8c22f3 100644
--- a/drivers/regulator/core.c
+++ b/drivers/regulator/core.c
@@ -19,6 +19,7 @@
 #include <linux/delay.h>
 #include <linux/gpio/consumer.h>
 #include <linux/of.h>
+#include <linux/reboot.h>
 #include <linux/regmap.h>
 #include <linux/regulator/of_regulator.h>
 #include <linux/regulator/consumer.h>
@@ -2918,7 +2919,8 @@ static int _regulator_enable(struct regulator *regulator)
 		/* Fallthrough on positive return values - already enabled */
 	}
 
-	rdev->use_count++;
+	if (regulator->enable_count == 1)
+		rdev->use_count++;
 
 	return 0;
 
@@ -2993,37 +2995,40 @@ static int _regulator_disable(struct regulator *regulator)
 
 	lockdep_assert_held_once(&rdev->mutex.base);
 
-	if (WARN(rdev->use_count <= 0,
+	if (WARN(regulator->enable_count == 0,
 		 "unbalanced disables for %s\n", rdev_get_name(rdev)))
 		return -EIO;
 
-	/* are we the last user and permitted to disable ? */
-	if (rdev->use_count == 1 &&
-	    (rdev->constraints && !rdev->constraints->always_on)) {
-
-		/* we are last user */
-		if (regulator_ops_is_valid(rdev, REGULATOR_CHANGE_STATUS)) {
-			ret = _notifier_call_chain(rdev,
-						   REGULATOR_EVENT_PRE_DISABLE,
-						   NULL);
-			if (ret & NOTIFY_STOP_MASK)
-				return -EINVAL;
-
-			ret = _regulator_do_disable(rdev);
-			if (ret < 0) {
-				rdev_err(rdev, "failed to disable: %pe\n", ERR_PTR(ret));
-				_notifier_call_chain(rdev,
-						REGULATOR_EVENT_ABORT_DISABLE,
+	if (regulator->enable_count == 1) {
+	/* disabling last enable_count from this regulator */
+		/* are we the last user and permitted to disable ? */
+		if (rdev->use_count == 1 &&
+		    (rdev->constraints && !rdev->constraints->always_on)) {
+
+			/* we are last user */
+			if (regulator_ops_is_valid(rdev, REGULATOR_CHANGE_STATUS)) {
+				ret = _notifier_call_chain(rdev,
+							   REGULATOR_EVENT_PRE_DISABLE,
+							   NULL);
+				if (ret & NOTIFY_STOP_MASK)
+					return -EINVAL;
+
+				ret = _regulator_do_disable(rdev);
+				if (ret < 0) {
+					rdev_err(rdev, "failed to disable: %pe\n", ERR_PTR(ret));
+					_notifier_call_chain(rdev,
+							REGULATOR_EVENT_ABORT_DISABLE,
+							NULL);
+					return ret;
+				}
+				_notifier_call_chain(rdev, REGULATOR_EVENT_DISABLE,
 						NULL);
-				return ret;
 			}
-			_notifier_call_chain(rdev, REGULATOR_EVENT_DISABLE,
-					NULL);
-		}
 
-		rdev->use_count = 0;
-	} else if (rdev->use_count > 1) {
-		rdev->use_count--;
+			rdev->use_count = 0;
+		} else if (rdev->use_count > 1) {
+			rdev->use_count--;
+		}
 	}
 
 	if (ret == 0)
@@ -5061,6 +5066,41 @@ void regulator_bulk_free(int num_consumers,
 }
 EXPORT_SYMBOL_GPL(regulator_bulk_free);
 
+/**
+ * regulator_handle_critical - Handle events for system-critical regulators.
+ * @rdev: The regulator device.
+ * @event: The event being handled.
+ *
+ * This function handles critical events such as under-voltage, over-current,
+ * and unknown errors for regulators deemed system-critical. On detecting such
+ * events, it triggers a hardware protection shutdown with a defined timeout.
+ */
+static void regulator_handle_critical(struct regulator_dev *rdev,
+				      unsigned long event)
+{
+	const char *reason = NULL;
+
+	if (!rdev->constraints->system_critical)
+		return;
+
+	switch (event) {
+	case REGULATOR_EVENT_UNDER_VOLTAGE:
+		reason = "System critical regulator: voltage drop detected";
+		break;
+	case REGULATOR_EVENT_OVER_CURRENT:
+		reason = "System critical regulator: over-current detected";
+		break;
+	case REGULATOR_EVENT_FAIL:
+		reason = "System critical regulator: unknown error";
+	}
+
+	if (!reason)
+		return;
+
+	hw_protection_shutdown(reason,
+			       rdev->constraints->uv_less_critical_window_ms);
+}
+
 /**
  * regulator_notifier_call_chain - call regulator event notifier
  * @rdev: regulator source
@@ -5073,6 +5113,8 @@ EXPORT_SYMBOL_GPL(regulator_bulk_free);
 int regulator_notifier_call_chain(struct regulator_dev *rdev,
 				  unsigned long event, void *data)
 {
+	regulator_handle_critical(rdev, event);
+
 	_notifier_call_chain(rdev, event, data);
 	return NOTIFY_DONE;
 
@@ -6234,6 +6276,14 @@ static int regulator_late_cleanup(struct device *dev, void *data)
 	return 0;
 }
 
+static bool regulator_ignore_unused;
+static int __init regulator_ignore_unused_setup(char *__unused)
+{
+	regulator_ignore_unused = true;
+	return 1;
+}
+__setup("regulator_ignore_unused", regulator_ignore_unused_setup);
+
 static void regulator_init_complete_work_function(struct work_struct *work)
 {
 	/*
@@ -6246,6 +6296,15 @@ static void regulator_init_complete_work_function(struct work_struct *work)
 	class_for_each_device(&regulator_class, NULL, NULL,
 			      regulator_register_resolve_supply);
 
+	/*
+	 * For debugging purposes, it may be useful to prevent unused
+	 * regulators from being disabled.
+	 */
+	if (regulator_ignore_unused) {
+		pr_warn("regulator: Not disabling unused regulators\n");
+		return;
+	}
+
 	/* If we have a full configuration then disable any regulators
 	 * we have permission to change the status for and which are
 	 * not in use or always_on.  This is effectively the default
diff --git a/drivers/regulator/of_regulator.c b/drivers/regulator/of_regulator.c
index 1b65e5e4e40f..03afc160fc72 100644
--- a/drivers/regulator/of_regulator.c
+++ b/drivers/regulator/of_regulator.c
@@ -131,6 +131,8 @@ static int of_get_regulation_constraints(struct device *dev,
 		constraints->valid_ops_mask |= REGULATOR_CHANGE_STATUS;
 
 	constraints->pull_down = of_property_read_bool(np, "regulator-pull-down");
+	constraints->system_critical = of_property_read_bool(np,
+						"system-critical-regulator");
 
 	if (of_property_read_bool(np, "regulator-allow-bypass"))
 		constraints->valid_ops_mask |= REGULATOR_CHANGE_BYPASS;
@@ -173,6 +175,13 @@ static int of_get_regulation_constraints(struct device *dev,
 	if (!ret)
 		constraints->enable_time = pval;
 
+	ret = of_property_read_u32(np, "regulator-uv-survival-time-ms", &pval);
+	if (!ret)
+		constraints->uv_less_critical_window_ms = pval;
+	else
+		constraints->uv_less_critical_window_ms =
+				REGULATOR_DEF_UV_LESS_CRITICAL_WINDOW_MS;
+
 	constraints->soft_start = of_property_read_bool(np,
 					"regulator-soft-start");
 	ret = of_property_read_u32(np, "regulator-active-discharge", &pval);
diff --git a/drivers/regulator/palmas-regulator.c b/drivers/regulator/palmas-regulator.c
index e0dc033aae0f..60656a815b9e 100644
--- a/drivers/regulator/palmas-regulator.c
+++ b/drivers/regulator/palmas-regulator.c
@@ -1594,7 +1594,7 @@ static const struct of_device_id of_palmas_match_tbl[] = {
 static int palmas_regulators_probe(struct platform_device *pdev)
 {
 	struct palmas *palmas = dev_get_drvdata(pdev->dev.parent);
-	struct palmas_pmic_platform_data *pdata = dev_get_platdata(&pdev->dev);
+	struct palmas_pmic_platform_data *pdata;
 	struct device_node *node = pdev->dev.of_node;
 	struct palmas_pmic_driver_data *driver_data;
 	struct regulator_config config = { };
diff --git a/drivers/regulator/qcom_smd-regulator.c b/drivers/regulator/qcom_smd-regulator.c
index f53ada076252..09c471a0ba2e 100644
--- a/drivers/regulator/qcom_smd-regulator.c
+++ b/drivers/regulator/qcom_smd-regulator.c
@@ -1012,6 +1012,39 @@ static const struct rpm_regulator_data rpm_pm8916_regulators[] = {
 	{}
 };
 
+static const struct rpm_regulator_data rpm_pm8937_regulators[] = {
+	{ "s1", QCOM_SMD_RPM_SMPA, 1, &pm8994_hfsmps, "vdd_s1" },
+	{ "s2", QCOM_SMD_RPM_SMPA, 2, &pm8994_hfsmps, "vdd_s2" },
+	{ "s3", QCOM_SMD_RPM_SMPA, 3, &pm8994_hfsmps, "vdd_s3" },
+	{ "s4", QCOM_SMD_RPM_SMPA, 4, &pm8994_hfsmps, "vdd_s4" },
+	/* S5 - S6 are managed by SPMI */
+
+	{ "l1", QCOM_SMD_RPM_LDOA, 1, &pm8953_ult_nldo, "vdd_l1_l19" },
+	{ "l2", QCOM_SMD_RPM_LDOA, 2, &pm8953_ult_nldo, "vdd_l2_l23" },
+	{ "l3", QCOM_SMD_RPM_LDOA, 3, &pm8953_ult_nldo, "vdd_l3" },
+	{ "l4", QCOM_SMD_RPM_LDOA, 4, &pm8950_ult_pldo, "vdd_l4_l5_l6_l7_l16" },
+	{ "l5", QCOM_SMD_RPM_LDOA, 5, &pm8950_ult_pldo, "vdd_l4_l5_l6_l7_l16" },
+	{ "l6", QCOM_SMD_RPM_LDOA, 6, &pm8950_ult_pldo, "vdd_l4_l5_l6_l7_l16" },
+	{ "l7", QCOM_SMD_RPM_LDOA, 7, &pm8950_ult_pldo, "vdd_l4_l5_l6_l7_l16" },
+	{ "l8", QCOM_SMD_RPM_LDOA, 8, &pm8950_ult_pldo, "vdd_l8_l11_l12_l17_l22" },
+	{ "l9", QCOM_SMD_RPM_LDOA, 9, &pm8950_ult_pldo, "vdd_l9_l10_l13_l14_l15_l18" },
+	{ "l10", QCOM_SMD_RPM_LDOA, 10, &pm8950_ult_pldo, "vdd_l9_l10_l13_l14_l15_l18"},
+	{ "l11", QCOM_SMD_RPM_LDOA, 11, &pm8950_ult_pldo, "vdd_l8_l11_l12_l17_l22" },
+	{ "l12", QCOM_SMD_RPM_LDOA, 12, &pm8950_ult_pldo, "vdd_l8_l11_l12_l17_l22" },
+	{ "l13", QCOM_SMD_RPM_LDOA, 13, &pm8950_ult_pldo, "vdd_l9_l10_l13_l14_l15_l18" },
+	{ "l14", QCOM_SMD_RPM_LDOA, 14, &pm8950_ult_pldo, "vdd_l9_l10_l13_l14_l15_l18" },
+	{ "l15", QCOM_SMD_RPM_LDOA, 15, &pm8950_ult_pldo, "vdd_l9_l10_l13_l14_l15_l18" },
+	{ "l16", QCOM_SMD_RPM_LDOA, 16, &pm8950_ult_pldo, "vdd_l4_l5_l6_l7_l16" },
+	{ "l17", QCOM_SMD_RPM_LDOA, 17, &pm8950_ult_pldo, "vdd_l8_l11_l12_l17_l22" },
+	{ "l18", QCOM_SMD_RPM_LDOA, 18, &pm8950_ult_pldo, "vdd_l9_l10_l13_l14_l15_l18" },
+	{ "l19", QCOM_SMD_RPM_LDOA, 19, &pm8953_ult_nldo, "vdd_l1_l19" },
+	{ "l20", QCOM_SMD_RPM_LDOA, 20, &pm8953_lnldo, "vdd_l20_l21" },
+	{ "l21", QCOM_SMD_RPM_LDOA, 21, &pm8953_lnldo, "vdd_l20_l21" },
+	{ "l22", QCOM_SMD_RPM_LDOA, 22, &pm8950_ult_pldo, "vdd_l8_l11_l12_l17_l22" },
+	{ "l23", QCOM_SMD_RPM_LDOA, 23, &pm8994_nldo, "vdd_l2_l23" },
+	{}
+};
+
 static const struct rpm_regulator_data rpm_pm8941_regulators[] = {
 	{ "s1", QCOM_SMD_RPM_SMPA, 1, &pm8x41_hfsmps, "vdd_s1" },
 	{ "s2", QCOM_SMD_RPM_SMPA, 2, &pm8x41_hfsmps, "vdd_s2" },
@@ -1329,6 +1362,7 @@ static const struct of_device_id rpm_of_match[] = {
 	{ .compatible = "qcom,rpm-pm8841-regulators", .data = &rpm_pm8841_regulators },
 	{ .compatible = "qcom,rpm-pm8909-regulators", .data = &rpm_pm8909_regulators },
 	{ .compatible = "qcom,rpm-pm8916-regulators", .data = &rpm_pm8916_regulators },
+	{ .compatible = "qcom,rpm-pm8937-regulators", .data = &rpm_pm8937_regulators },
 	{ .compatible = "qcom,rpm-pm8941-regulators", .data = &rpm_pm8941_regulators },
 	{ .compatible = "qcom,rpm-pm8950-regulators", .data = &rpm_pm8950_regulators },
 	{ .compatible = "qcom,rpm-pm8953-regulators", .data = &rpm_pm8953_regulators },
diff --git a/drivers/regulator/qcom_spmi-regulator.c b/drivers/regulator/qcom_spmi-regulator.c
index 94f9092b29ef..9a9fa20dcd95 100644
--- a/drivers/regulator/qcom_spmi-regulator.c
+++ b/drivers/regulator/qcom_spmi-regulator.c
@@ -2239,6 +2239,39 @@ static const struct spmi_regulator_data pm8916_regulators[] = {
 	{ }
 };
 
+static const struct spmi_regulator_data pm8937_regulators[] = {
+	{ "s1", 0x1400, "vdd_s1", },
+	{ "s2", 0x1700, "vdd_s2", },
+	{ "s3", 0x1a00, "vdd_s3", },
+	{ "s4", 0x1d00, "vdd_s4", },
+	{ "s5", 0x2000, "vdd_s5", },
+	{ "s6", 0x2300, "vdd_s6", },
+	{ "l1", 0x4000, "vdd_l1_l19", },
+	{ "l2", 0x4100, "vdd_l2_l23", },
+	{ "l3", 0x4200, "vdd_l3", },
+	{ "l4", 0x4300, "vdd_l4_l5_l6_l7_l16", },
+	{ "l5", 0x4400, "vdd_l4_l5_l6_l7_l16", },
+	{ "l6", 0x4500, "vdd_l4_l5_l6_l7_l16", },
+	{ "l7", 0x4600, "vdd_l4_l5_l6_l7_l16", },
+	{ "l8", 0x4700, "vdd_l8_l11_l12_l17_l22", },
+	{ "l9", 0x4800, "vdd_l9_l10_l13_l14_l15_l18", },
+	{ "l10", 0x4900, "vdd_l9_l10_l13_l14_l15_l18", },
+	{ "l11", 0x4a00, "vdd_l8_l11_l12_l17_l22", },
+	{ "l12", 0x4b00, "vdd_l8_l11_l12_l17_l22", },
+	{ "l13", 0x4c00, "vdd_l9_l10_l13_l14_l15_l18", },
+	{ "l14", 0x4d00, "vdd_l9_l10_l13_l14_l15_l18", },
+	{ "l15", 0x4e00, "vdd_l9_l10_l13_l14_l15_l18", },
+	{ "l16", 0x4f00, "vdd_l4_l5_l6_l7_l16", },
+	{ "l17", 0x5000, "vdd_l8_l11_l12_l17_l22", },
+	{ "l18", 0x5100, "vdd_l9_l10_l13_l14_l15_l18", },
+	{ "l19", 0x5200, "vdd_l1_l19", },
+	{ "l20", 0x5300, "vdd_l20_l21", },
+	{ "l21", 0x5400, "vdd_l21_l21", },
+	{ "l22", 0x5500, "vdd_l8_l11_l12_l17_l22", },
+	{ "l23", 0x5600, "vdd_l2_l23", },
+	{ }
+};
+
 static const struct spmi_regulator_data pm8941_regulators[] = {
 	{ "s1", 0x1400, "vdd_s1", },
 	{ "s2", 0x1700, "vdd_s2", },
@@ -2453,6 +2486,7 @@ static const struct of_device_id qcom_spmi_regulator_match[] = {
 	{ .compatible = "qcom,pm8841-regulators", .data = &pm8841_regulators },
 	{ .compatible = "qcom,pm8909-regulators", .data = &pm8909_regulators },
 	{ .compatible = "qcom,pm8916-regulators", .data = &pm8916_regulators },
+	{ .compatible = "qcom,pm8937-regulators", .data = &pm8937_regulators },
 	{ .compatible = "qcom,pm8941-regulators", .data = &pm8941_regulators },
 	{ .compatible = "qcom,pm8950-regulators", .data = &pm8950_regulators },
 	{ .compatible = "qcom,pm8994-regulators", .data = &pm8994_regulators },
diff --git a/drivers/regulator/stpmic1_regulator.c b/drivers/regulator/stpmic1_regulator.c
index 79d1a3eb18d4..a498df7cb016 100644
--- a/drivers/regulator/stpmic1_regulator.c
+++ b/drivers/regulator/stpmic1_regulator.c
@@ -15,7 +15,7 @@
 #include <dt-bindings/mfd/st,stpmic1.h>
 
 /**
- * struct stpmic1 regulator description: this structure is used as driver data
+ * struct stpmic1_regulator_cfg - this structure is used as driver data
  * @desc: regulator framework description
  * @mask_reset_reg: mask reset register address
  * @mask_reset_mask: mask rank and mask reset register mask
diff --git a/drivers/remoteproc/imx_dsp_rproc.c b/drivers/remoteproc/imx_dsp_rproc.c
index 8fcda9b74545..a1c62d15f16c 100644
--- a/drivers/remoteproc/imx_dsp_rproc.c
+++ b/drivers/remoteproc/imx_dsp_rproc.c
@@ -940,6 +940,7 @@ static const struct rproc_ops imx_dsp_rproc_ops = {
 	.kick		= imx_dsp_rproc_kick,
 	.load		= imx_dsp_rproc_elf_load_segments,
 	.parse_fw	= imx_dsp_rproc_parse_fw,
+	.find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table,
 	.sanity_check	= rproc_elf_sanity_check,
 	.get_boot_addr	= rproc_elf_get_boot_addr,
 };
diff --git a/drivers/rtc/rtc-cmos.c b/drivers/rtc/rtc-cmos.c
index 228fb2d11c70..696cfa7025de 100644
--- a/drivers/rtc/rtc-cmos.c
+++ b/drivers/rtc/rtc-cmos.c
@@ -818,18 +818,24 @@ static void rtc_wake_off(struct device *dev)
 }
 
 #ifdef CONFIG_X86
-/* Enable use_acpi_alarm mode for Intel platforms no earlier than 2015 */
 static void use_acpi_alarm_quirks(void)
 {
-	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
+	switch (boot_cpu_data.x86_vendor) {
+	case X86_VENDOR_INTEL:
+		if (dmi_get_bios_year() < 2015)
+			return;
+		break;
+	case X86_VENDOR_AMD:
+	case X86_VENDOR_HYGON:
+		if (dmi_get_bios_year() < 2021)
+			return;
+		break;
+	default:
 		return;
-
+	}
 	if (!is_hpet_enabled())
 		return;
 
-	if (dmi_get_bios_year() < 2015)
-		return;
-
 	use_acpi_alarm = true;
 }
 #else
diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c
index d440319a7945..833cfab7d877 100644
--- a/drivers/s390/block/dasd.c
+++ b/drivers/s390/block/dasd.c
@@ -676,18 +676,20 @@ static void dasd_profile_start(struct dasd_block *block,
 	 * we count each request only once.
 	 */
 	device = cqr->startdev;
-	if (device->profile.data) {
-		counter = 1; /* request is not yet queued on the start device */
-		list_for_each(l, &device->ccw_queue)
-			if (++counter >= 31)
-				break;
-	}
+	if (!device->profile.data)
+		return;
+
+	spin_lock(get_ccwdev_lock(device->cdev));
+	counter = 1; /* request is not yet queued on the start device */
+	list_for_each(l, &device->ccw_queue)
+		if (++counter >= 31)
+			break;
+	spin_unlock(get_ccwdev_lock(device->cdev));
+
 	spin_lock(&device->profile.lock);
-	if (device->profile.data) {
-		device->profile.data->dasd_io_nr_req[counter]++;
-		if (rq_data_dir(req) == READ)
-			device->profile.data->dasd_read_nr_req[counter]++;
-	}
+	device->profile.data->dasd_io_nr_req[counter]++;
+	if (rq_data_dir(req) == READ)
+		device->profile.data->dasd_read_nr_req[counter]++;
 	spin_unlock(&device->profile.lock);
 }
 
diff --git a/drivers/s390/block/dasd_int.h b/drivers/s390/block/dasd_int.h
index 2e663131adaf..1b1b8a41c4d4 100644
--- a/drivers/s390/block/dasd_int.h
+++ b/drivers/s390/block/dasd_int.h
@@ -283,7 +283,7 @@ struct dasd_pprc_dev_info {
 	__u8 secondary;		/* 7       Secondary device address */
 	__u16 pprc_id;		/* 8-9     Peer-to-Peer Remote Copy ID */
 	__u8 reserved2[12];	/* 10-21   reserved */
-	__u16 prim_cu_ssid;	/* 22-23   Pimary Control Unit SSID */
+	__u16 prim_cu_ssid;	/* 22-23   Primary Control Unit SSID */
 	__u8 reserved3[12];	/* 24-35   reserved */
 	__u16 sec_cu_ssid;	/* 36-37   Secondary Control Unit SSID */
 	__u8 reserved4[90];	/* 38-127  reserved */
diff --git a/drivers/s390/net/Kconfig b/drivers/s390/net/Kconfig
index 4902d45e929c..c61e6427384c 100644
--- a/drivers/s390/net/Kconfig
+++ b/drivers/s390/net/Kconfig
@@ -103,10 +103,11 @@ config CCWGROUP
 config ISM
 	tristate "Support for ISM vPCI Adapter"
 	depends on PCI
+	imply SMC
 	default n
 	help
 	  Select this option if you want to use the Internal Shared Memory
-	  vPCI Adapter.
+	  vPCI Adapter. The adapter can be used with the SMC network protocol.
 
 	  To compile as a module choose M. The module name is ism.
 	  If unsure, choose N.
diff --git a/drivers/s390/net/ism_drv.c b/drivers/s390/net/ism_drv.c
index 6df7f377d2f9..81aabbfbbe2c 100644
--- a/drivers/s390/net/ism_drv.c
+++ b/drivers/s390/net/ism_drv.c
@@ -30,7 +30,6 @@ static const struct pci_device_id ism_device_table[] = {
 MODULE_DEVICE_TABLE(pci, ism_device_table);
 
 static debug_info_t *ism_debug_info;
-static const struct smcd_ops ism_ops;
 
 #define NO_CLIENT		0xff		/* must be >= MAX_CLIENTS */
 static struct ism_client *clients[MAX_CLIENTS];	/* use an array rather than */
@@ -289,22 +288,6 @@ static int ism_read_local_gid(struct ism_dev *ism)
 	return ret;
 }
 
-static int ism_query_rgid(struct ism_dev *ism, u64 rgid, u32 vid_valid,
-			  u32 vid)
-{
-	union ism_query_rgid cmd;
-
-	memset(&cmd, 0, sizeof(cmd));
-	cmd.request.hdr.cmd = ISM_QUERY_RGID;
-	cmd.request.hdr.len = sizeof(cmd.request);
-
-	cmd.request.rgid = rgid;
-	cmd.request.vlan_valid = vid_valid;
-	cmd.request.vlan_id = vid;
-
-	return ism_cmd(ism, &cmd);
-}
-
 static void ism_free_dmb(struct ism_dev *ism, struct ism_dmb *dmb)
 {
 	clear_bit(dmb->sba_idx, ism->sba_bitmap);
@@ -429,23 +412,6 @@ static int ism_del_vlan_id(struct ism_dev *ism, u64 vlan_id)
 	return ism_cmd(ism, &cmd);
 }
 
-static int ism_signal_ieq(struct ism_dev *ism, u64 rgid, u32 trigger_irq,
-			  u32 event_code, u64 info)
-{
-	union ism_sig_ieq cmd;
-
-	memset(&cmd, 0, sizeof(cmd));
-	cmd.request.hdr.cmd = ISM_SIGNAL_IEQ;
-	cmd.request.hdr.len = sizeof(cmd.request);
-
-	cmd.request.rgid = rgid;
-	cmd.request.trigger_irq = trigger_irq;
-	cmd.request.event_code = event_code;
-	cmd.request.info = info;
-
-	return ism_cmd(ism, &cmd);
-}
-
 static unsigned int max_bytes(unsigned int start, unsigned int len,
 			      unsigned int boundary)
 {
@@ -503,14 +469,6 @@ u8 *ism_get_seid(void)
 }
 EXPORT_SYMBOL_GPL(ism_get_seid);
 
-static u16 ism_get_chid(struct ism_dev *ism)
-{
-	if (!ism || !ism->pdev)
-		return 0;
-
-	return to_zpci(ism->pdev)->pchid;
-}
-
 static void ism_handle_event(struct ism_dev *ism)
 {
 	struct ism_event *entry;
@@ -569,11 +527,6 @@ static irqreturn_t ism_handle_irq(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
-static u64 ism_get_local_gid(struct ism_dev *ism)
-{
-	return ism->local_gid;
-}
-
 static int ism_dev_init(struct ism_dev *ism)
 {
 	struct pci_dev *pdev = ism->pdev;
@@ -774,6 +727,22 @@ module_exit(ism_exit);
 /*************************** SMC-D Implementation *****************************/
 
 #if IS_ENABLED(CONFIG_SMC)
+static int ism_query_rgid(struct ism_dev *ism, u64 rgid, u32 vid_valid,
+			  u32 vid)
+{
+	union ism_query_rgid cmd;
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.request.hdr.cmd = ISM_QUERY_RGID;
+	cmd.request.hdr.len = sizeof(cmd.request);
+
+	cmd.request.rgid = rgid;
+	cmd.request.vlan_valid = vid_valid;
+	cmd.request.vlan_id = vid;
+
+	return ism_cmd(ism, &cmd);
+}
+
 static int smcd_query_rgid(struct smcd_dev *smcd, u64 rgid, u32 vid_valid,
 			   u32 vid)
 {
@@ -811,6 +780,23 @@ static int smcd_reset_vlan_required(struct smcd_dev *smcd)
 	return ism_cmd_simple(smcd->priv, ISM_RESET_VLAN);
 }
 
+static int ism_signal_ieq(struct ism_dev *ism, u64 rgid, u32 trigger_irq,
+			  u32 event_code, u64 info)
+{
+	union ism_sig_ieq cmd;
+
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.request.hdr.cmd = ISM_SIGNAL_IEQ;
+	cmd.request.hdr.len = sizeof(cmd.request);
+
+	cmd.request.rgid = rgid;
+	cmd.request.trigger_irq = trigger_irq;
+	cmd.request.event_code = event_code;
+	cmd.request.info = info;
+
+	return ism_cmd(ism, &cmd);
+}
+
 static int smcd_signal_ieq(struct smcd_dev *smcd, u64 rgid, u32 trigger_irq,
 			   u32 event_code, u64 info)
 {
@@ -830,11 +816,24 @@ static int smcd_supports_v2(void)
 		SYSTEM_EID.type[0] != '0';
 }
 
+static u64 ism_get_local_gid(struct ism_dev *ism)
+{
+	return ism->local_gid;
+}
+
 static u64 smcd_get_local_gid(struct smcd_dev *smcd)
 {
 	return ism_get_local_gid(smcd->priv);
 }
 
+static u16 ism_get_chid(struct ism_dev *ism)
+{
+	if (!ism || !ism->pdev)
+		return 0;
+
+	return to_zpci(ism->pdev)->pchid;
+}
+
 static u16 smcd_get_chid(struct smcd_dev *smcd)
 {
 	return ism_get_chid(smcd->priv);
diff --git a/drivers/scsi/3w-sas.c b/drivers/scsi/3w-sas.c
index 55989eaa2d9f..9bdb75dfdcd7 100644
--- a/drivers/scsi/3w-sas.c
+++ b/drivers/scsi/3w-sas.c
@@ -1326,7 +1326,8 @@ static int twl_reset_sequence(TW_Device_Extension *tw_dev, int soft_reset)
 		}
 
 		/* Load rest of compatibility struct */
-		strncpy(tw_dev->tw_compat_info.driver_version, TW_DRIVER_VERSION, strlen(TW_DRIVER_VERSION));
+		strscpy(tw_dev->tw_compat_info.driver_version, TW_DRIVER_VERSION,
+			sizeof(tw_dev->tw_compat_info.driver_version));
 		tw_dev->tw_compat_info.driver_srl_high = TW_CURRENT_DRIVER_SRL;
 		tw_dev->tw_compat_info.driver_branch_high = TW_CURRENT_DRIVER_BRANCH;
 		tw_dev->tw_compat_info.driver_build_high = TW_CURRENT_DRIVER_BUILD;
diff --git a/drivers/scsi/bnx2fc/bnx2fc_fcoe.c b/drivers/scsi/bnx2fc/bnx2fc_fcoe.c
index 05ddbb9bb7d8..3ebfb09329ad 100644
--- a/drivers/scsi/bnx2fc/bnx2fc_fcoe.c
+++ b/drivers/scsi/bnx2fc/bnx2fc_fcoe.c
@@ -1737,32 +1737,32 @@ static int bnx2fc_bind_pcidev(struct bnx2fc_hba *hba)
 
 	switch (pdev->device) {
 	case PCI_DEVICE_ID_NX2_57710:
-		strncpy(hba->chip_num, "BCM57710", BCM_CHIP_LEN);
+		strscpy(hba->chip_num, "BCM57710", sizeof(hba->chip_num));
 		break;
 	case PCI_DEVICE_ID_NX2_57711:
-		strncpy(hba->chip_num, "BCM57711", BCM_CHIP_LEN);
+		strscpy(hba->chip_num, "BCM57711", sizeof(hba->chip_num));
 		break;
 	case PCI_DEVICE_ID_NX2_57712:
 	case PCI_DEVICE_ID_NX2_57712_MF:
 	case PCI_DEVICE_ID_NX2_57712_VF:
-		strncpy(hba->chip_num, "BCM57712", BCM_CHIP_LEN);
+		strscpy(hba->chip_num, "BCM57712", sizeof(hba->chip_num));
 		break;
 	case PCI_DEVICE_ID_NX2_57800:
 	case PCI_DEVICE_ID_NX2_57800_MF:
 	case PCI_DEVICE_ID_NX2_57800_VF:
-		strncpy(hba->chip_num, "BCM57800", BCM_CHIP_LEN);
+		strscpy(hba->chip_num, "BCM57800", sizeof(hba->chip_num));
 		break;
 	case PCI_DEVICE_ID_NX2_57810:
 	case PCI_DEVICE_ID_NX2_57810_MF:
 	case PCI_DEVICE_ID_NX2_57810_VF:
-		strncpy(hba->chip_num, "BCM57810", BCM_CHIP_LEN);
+		strscpy(hba->chip_num, "BCM57810", sizeof(hba->chip_num));
 		break;
 	case PCI_DEVICE_ID_NX2_57840:
 	case PCI_DEVICE_ID_NX2_57840_MF:
 	case PCI_DEVICE_ID_NX2_57840_VF:
 	case PCI_DEVICE_ID_NX2_57840_2_20:
 	case PCI_DEVICE_ID_NX2_57840_4_10:
-		strncpy(hba->chip_num, "BCM57840", BCM_CHIP_LEN);
+		strscpy(hba->chip_num, "BCM57840", sizeof(hba->chip_num));
 		break;
 	default:
 		pr_err(PFX "Unknown device id 0x%x\n", pdev->device);
@@ -1800,7 +1800,7 @@ static int bnx2fc_ulp_get_stats(void *handle)
 	if (!stats_addr)
 		return -EINVAL;
 
-	strncpy(stats_addr->version, BNX2FC_VERSION,
+	strscpy(stats_addr->version, BNX2FC_VERSION,
 		sizeof(stats_addr->version));
 	stats_addr->txq_size = BNX2FC_SQ_WQES_MAX;
 	stats_addr->rxq_size = BNX2FC_CQ_WQES_MAX;
diff --git a/drivers/scsi/ch.c b/drivers/scsi/ch.c
index cb0a399be1cc..2b864061e073 100644
--- a/drivers/scsi/ch.c
+++ b/drivers/scsi/ch.c
@@ -659,19 +659,23 @@ static long ch_ioctl(struct file *file,
 		memset(&vparams,0,sizeof(vparams));
 		if (ch->counts[CHET_V1]) {
 			vparams.cvp_n1  = ch->counts[CHET_V1];
-			strncpy(vparams.cvp_label1,vendor_labels[0],16);
+			strscpy(vparams.cvp_label1, vendor_labels[0],
+				sizeof(vparams.cvp_label1));
 		}
 		if (ch->counts[CHET_V2]) {
 			vparams.cvp_n2  = ch->counts[CHET_V2];
-			strncpy(vparams.cvp_label2,vendor_labels[1],16);
+			strscpy(vparams.cvp_label2, vendor_labels[1],
+				sizeof(vparams.cvp_label2));
 		}
 		if (ch->counts[CHET_V3]) {
 			vparams.cvp_n3  = ch->counts[CHET_V3];
-			strncpy(vparams.cvp_label3,vendor_labels[2],16);
+			strscpy(vparams.cvp_label3, vendor_labels[2],
+				sizeof(vparams.cvp_label3));
 		}
 		if (ch->counts[CHET_V4]) {
 			vparams.cvp_n4  = ch->counts[CHET_V4];
-			strncpy(vparams.cvp_label4,vendor_labels[3],16);
+			strscpy(vparams.cvp_label4, vendor_labels[3],
+				sizeof(vparams.cvp_label4));
 		}
 		if (copy_to_user(argp, &vparams, sizeof(vparams)))
 			return -EFAULT;
diff --git a/drivers/scsi/csiostor/csio_init.c b/drivers/scsi/csiostor/csio_init.c
index 0c32faefad7c..d649b7a2a879 100644
--- a/drivers/scsi/csiostor/csio_init.c
+++ b/drivers/scsi/csiostor/csio_init.c
@@ -521,7 +521,8 @@ static struct csio_hw *csio_hw_alloc(struct pci_dev *pdev)
 		goto err;
 
 	hw->pdev = pdev;
-	strncpy(hw->drv_version, CSIO_DRV_VERSION, 32);
+	strscpy(hw->drv_version, CSIO_DRV_VERSION,
+		sizeof(hw->drv_version));
 
 	/* memory pool/DMA pool allocation */
 	if (csio_resource_alloc(hw))
diff --git a/drivers/scsi/elx/libefc/efc_node.h b/drivers/scsi/elx/libefc/efc_node.h
index e9c600ac45d5..e57579988ba4 100644
--- a/drivers/scsi/elx/libefc/efc_node.h
+++ b/drivers/scsi/elx/libefc/efc_node.h
@@ -26,13 +26,13 @@ efc_node_evt_set(struct efc_sm_ctx *ctx, enum efc_sm_event evt,
 	struct efc_node *node = ctx->app;
 
 	if (evt == EFC_EVT_ENTER) {
-		strncpy(node->current_state_name, handler,
-			sizeof(node->current_state_name));
+		strscpy_pad(node->current_state_name, handler,
+			    sizeof(node->current_state_name));
 	} else if (evt == EFC_EVT_EXIT) {
-		strncpy(node->prev_state_name, node->current_state_name,
-			sizeof(node->prev_state_name));
-		strncpy(node->current_state_name, "invalid",
-			sizeof(node->current_state_name));
+		memcpy(node->prev_state_name, node->current_state_name,
+		       sizeof(node->prev_state_name));
+		strscpy_pad(node->current_state_name, "invalid",
+			    sizeof(node->current_state_name));
 	}
 	node->prev_evt = node->current_evt;
 	node->current_evt = evt;
diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h
index af15f7a22d25..04d608ea9106 100644
--- a/drivers/scsi/lpfc/lpfc.h
+++ b/drivers/scsi/lpfc/lpfc.h
@@ -33,6 +33,7 @@
 struct lpfc_sli2_slim;
 
 #define ELX_MODEL_NAME_SIZE	80
+#define ELX_FW_NAME_SIZE	84
 
 #define LPFC_PCI_DEV_LP		0x1
 #define LPFC_PCI_DEV_OC		0x2
diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c
index b1c9107d3408..48c727a51193 100644
--- a/drivers/scsi/lpfc/lpfc_attr.c
+++ b/drivers/scsi/lpfc/lpfc_attr.c
@@ -5905,11 +5905,11 @@ LPFC_ATTR_RW(ras_fwlog_level, 0, 0, 4, "Firmware Logging Level");
 /*
  * lpfc_ras_fwlog_func: Firmware logging enabled on function number
  * Default function which has RAS support : 0
- * Value Range is [0..7].
+ * Value Range is [0..3].
  * FW logging is a global action and enablement is via a specific
  * port.
  */
-LPFC_ATTR_RW(ras_fwlog_func, 0, 0, 7, "Firmware Logging Enabled on Function");
+LPFC_ATTR_RW(ras_fwlog_func, 0, 0, 3, "Firmware Logging Enabled on Function");
 
 /*
  * lpfc_enable_bbcr: Enable BB Credit Recovery
diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c
index f9627eddab08..f04326db8c19 100644
--- a/drivers/scsi/lpfc/lpfc_els.c
+++ b/drivers/scsi/lpfc/lpfc_els.c
@@ -2062,8 +2062,9 @@ lpfc_cmpl_els_plogi(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 	/* PLOGI completes to NPort <nlp_DID> */
 	lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
 			 "0102 PLOGI completes to NPort x%06x "
-			 "Data: x%x x%x x%x x%x x%x\n",
-			 ndlp->nlp_DID, ndlp->nlp_fc4_type,
+			 "IoTag x%x Data: x%x x%x x%x x%x x%x\n",
+			 ndlp->nlp_DID, iotag,
+			 ndlp->nlp_fc4_type,
 			 ulp_status, ulp_word4,
 			 disc, vport->num_disc_nodes);
 
@@ -2128,8 +2129,8 @@ lpfc_cmpl_els_plogi(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 						NLP_EVT_DEVICE_RM);
 	} else {
 		/* Good status, call state machine */
-		prsp = list_entry(cmdiocb->cmd_dmabuf->list.next,
-				  struct lpfc_dmabuf, list);
+		prsp = list_get_first(&cmdiocb->cmd_dmabuf->list,
+				      struct lpfc_dmabuf, list);
 		if (!prsp)
 			goto out;
 		if (!lpfc_is_els_acc_rsp(prsp))
@@ -2362,9 +2363,10 @@ lpfc_cmpl_els_prli(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 	/* PRLI completes to NPort <nlp_DID> */
 	lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
 			 "0103 PRLI completes to NPort x%06x "
-			 "Data: x%x x%x x%x x%x\n",
+			 "Data: x%x x%x x%x x%x x%x\n",
 			 ndlp->nlp_DID, ulp_status, ulp_word4,
-			 vport->num_disc_nodes, ndlp->fc4_prli_sent);
+			 vport->num_disc_nodes, ndlp->fc4_prli_sent,
+			 ndlp->fc4_xpt_flags);
 
 	/* Check to see if link went down during discovery */
 	if (lpfc_els_chk_latt(vport))
@@ -2805,7 +2807,7 @@ lpfc_cmpl_els_adisc(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 	IOCB_t *irsp;
 	struct lpfc_nodelist *ndlp;
 	int  disc;
-	u32 ulp_status, ulp_word4, tmo;
+	u32 ulp_status, ulp_word4, tmo, iotag;
 	bool release_node = false;
 
 	/* we pass cmdiocb to state machine which needs rspiocb as well */
@@ -2818,9 +2820,11 @@ lpfc_cmpl_els_adisc(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 
 	if (phba->sli_rev == LPFC_SLI_REV4) {
 		tmo = get_wqe_tmo(cmdiocb);
+		iotag = get_wqe_reqtag(cmdiocb);
 	} else {
 		irsp = &rspiocb->iocb;
 		tmo = irsp->ulpTimeout;
+		iotag = irsp->ulpIoTag;
 	}
 
 	lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_ELS_CMD,
@@ -2838,9 +2842,11 @@ lpfc_cmpl_els_adisc(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 	/* ADISC completes to NPort <nlp_DID> */
 	lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
 			 "0104 ADISC completes to NPort x%x "
-			 "Data: x%x x%x x%x x%x x%x\n",
-			 ndlp->nlp_DID, ulp_status, ulp_word4,
+			 "IoTag x%x Data: x%x x%x x%x x%x x%x\n",
+			 ndlp->nlp_DID, iotag,
+			 ulp_status, ulp_word4,
 			 tmo, disc, vport->num_disc_nodes);
+
 	/* Check to see if link went down during discovery */
 	if (lpfc_els_chk_latt(vport)) {
 		spin_lock_irq(&ndlp->lock);
@@ -3001,7 +3007,7 @@ lpfc_cmpl_els_logo(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 	int wake_up_waiter = 0;
 	u32 ulp_status;
 	u32 ulp_word4;
-	u32 tmo;
+	u32 tmo, iotag;
 
 	/* we pass cmdiocb to state machine which needs rspiocb as well */
 	cmdiocb->rsp_iocb = rspiocb;
@@ -3011,9 +3017,11 @@ lpfc_cmpl_els_logo(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 
 	if (phba->sli_rev == LPFC_SLI_REV4) {
 		tmo = get_wqe_tmo(cmdiocb);
+		iotag = get_wqe_reqtag(cmdiocb);
 	} else {
 		irsp = &rspiocb->iocb;
 		tmo = irsp->ulpTimeout;
+		iotag = irsp->ulpIoTag;
 	}
 
 	spin_lock_irq(&ndlp->lock);
@@ -3032,9 +3040,11 @@ lpfc_cmpl_els_logo(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 	/* LOGO completes to NPort <nlp_DID> */
 	lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
 			 "0105 LOGO completes to NPort x%x "
-			 "refcnt %d nflags x%x Data: x%x x%x x%x x%x\n",
-			 ndlp->nlp_DID, kref_read(&ndlp->kref), ndlp->nlp_flag,
-			 ulp_status, ulp_word4,
+			 "IoTag x%x refcnt %d nflags x%x xflags x%x "
+			 "Data: x%x x%x x%x x%x\n",
+			 ndlp->nlp_DID, iotag,
+			 kref_read(&ndlp->kref), ndlp->nlp_flag,
+			 ndlp->fc4_xpt_flags, ulp_status, ulp_word4,
 			 tmo, vport->num_disc_nodes);
 
 	if (lpfc_els_chk_latt(vport)) {
@@ -5075,16 +5085,19 @@ lpfc_els_retry(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 	if (logerr) {
 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
 			 "0137 No retry ELS command x%x to remote "
-			 "NPORT x%x: Out of Resources: Error:x%x/%x\n",
-			 cmd, did, ulp_status,
-			 ulp_word4);
+			 "NPORT x%x: Out of Resources: Error:x%x/%x "
+			 "IoTag x%x\n",
+			 cmd, did, ulp_status, ulp_word4,
+			 cmdiocb->iotag);
 	}
 	else {
 		lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
-			 "0108 No retry ELS command x%x to remote "
-			 "NPORT x%x Retried:%d Error:x%x/%x\n",
-			 cmd, did, cmdiocb->retry, ulp_status,
-			 ulp_word4);
+				 "0108 No retry ELS command x%x to remote "
+				 "NPORT x%x Retried:%d Error:x%x/%x "
+				 "IoTag x%x nflags x%x\n",
+				 cmd, did, cmdiocb->retry, ulp_status,
+				 ulp_word4, cmdiocb->iotag,
+				 (ndlp ? ndlp->nlp_flag : 0));
 	}
 	return 0;
 }
diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c
index 7ef9841f0728..f80bbc315f4c 100644
--- a/drivers/scsi/lpfc/lpfc_hbadisc.c
+++ b/drivers/scsi/lpfc/lpfc_hbadisc.c
@@ -411,7 +411,7 @@ lpfc_check_nlp_post_devloss(struct lpfc_vport *vport,
 				 "port_state = x%x\n",
 				 ndlp->nlp_DID, kref_read(&ndlp->kref), ndlp,
 				 ndlp->nlp_flag, vport->port_state);
-		spin_lock_irqsave(&ndlp->lock, iflags);
+		return;
 	}
 	spin_unlock_irqrestore(&ndlp->lock, iflags);
 }
diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c
index e7c47ee185a4..70bcee64bc8c 100644
--- a/drivers/scsi/lpfc/lpfc_init.c
+++ b/drivers/scsi/lpfc/lpfc_init.c
@@ -14721,7 +14721,7 @@ lpfc_write_firmware(const struct firmware *fw, void *context)
 int
 lpfc_sli4_request_firmware_update(struct lpfc_hba *phba, uint8_t fw_upgrade)
 {
-	uint8_t file_name[ELX_MODEL_NAME_SIZE];
+	char file_name[ELX_FW_NAME_SIZE] = {0};
 	int ret;
 	const struct firmware *fw;
 
@@ -14730,7 +14730,7 @@ lpfc_sli4_request_firmware_update(struct lpfc_hba *phba, uint8_t fw_upgrade)
 	    LPFC_SLI_INTF_IF_TYPE_2)
 		return -EPERM;
 
-	snprintf(file_name, ELX_MODEL_NAME_SIZE, "%s.grp", phba->ModelName);
+	scnprintf(file_name, sizeof(file_name), "%s.grp", phba->ModelName);
 
 	if (fw_upgrade == INT_FW_UPGRADE) {
 		ret = request_firmware_nowait(THIS_MODULE, FW_ACTION_UEVENT,
diff --git a/drivers/scsi/lpfc/lpfc_mbox.c b/drivers/scsi/lpfc/lpfc_mbox.c
index 0dfdc0c4c08c..cadcd16494e1 100644
--- a/drivers/scsi/lpfc/lpfc_mbox.c
+++ b/drivers/scsi/lpfc/lpfc_mbox.c
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2022 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2023 Broadcom. All Rights Reserved. The term *
  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.     *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -1814,7 +1814,9 @@ lpfc_sli4_mbox_cmd_free(struct lpfc_hba *phba, struct lpfcMboxq *mbox)
 		dma_free_coherent(&phba->pcidev->dev, SLI4_PAGE_SIZE,
 				  mbox->sge_array->addr[sgentry], phyaddr);
 	}
-	/* Free the sge address array memory */
+	/* Reinitialize the context pointers to avoid stale usage. */
+	mbox->ctx_buf = NULL;
+	mbox->context3 = NULL;
 	kfree(mbox->sge_array);
 	/* Finally, free the mailbox command itself */
 	mempool_free(mbox, phba->mbox_mem_pool);
diff --git a/drivers/scsi/lpfc/lpfc_mem.c b/drivers/scsi/lpfc/lpfc_mem.c
index 89cbeba06aea..2697da3248b3 100644
--- a/drivers/scsi/lpfc/lpfc_mem.c
+++ b/drivers/scsi/lpfc/lpfc_mem.c
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2022 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2023 Broadcom. All Rights Reserved. The term *
  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.     *
  * Copyright (C) 2004-2014 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -48,6 +48,29 @@
 #define LPFC_RRQ_POOL_SIZE	256	/* max elements in non-DMA  pool */
 #define LPFC_MBX_POOL_SIZE	256	/* max elements in MBX non-DMA pool */
 
+/* lpfc_mbox_free_sli_mbox
+ *
+ * @phba: HBA to free memory for
+ * @mbox: mailbox command to free
+ *
+ * This routine detects the mbox type and calls the correct
+ * free routine to fully release all associated memory.
+ */
+static void
+lpfc_mem_free_sli_mbox(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox)
+{
+	/* Detect if the caller's mbox is an SLI4_CONFIG type.  If so, this
+	 * mailbox type requires a different cleanup routine.  Otherwise, the
+	 * mailbox is just an mbuf and mem_pool release.
+	 */
+	if (phba->sli_rev == LPFC_SLI_REV4 &&
+	    bf_get(lpfc_mqe_command, &mbox->u.mqe) == MBX_SLI4_CONFIG) {
+		lpfc_sli4_mbox_cmd_free(phba, mbox);
+	} else {
+		lpfc_mbox_rsrc_cleanup(phba, mbox, MBOX_THD_UNLOCKED);
+	}
+}
+
 int
 lpfc_mem_alloc_active_rrq_pool_s4(struct lpfc_hba *phba) {
 	size_t bytes;
@@ -288,27 +311,16 @@ lpfc_mem_free_all(struct lpfc_hba *phba)
 {
 	struct lpfc_sli *psli = &phba->sli;
 	LPFC_MBOXQ_t *mbox, *next_mbox;
-	struct lpfc_dmabuf   *mp;
 
 	/* Free memory used in mailbox queue back to mailbox memory pool */
 	list_for_each_entry_safe(mbox, next_mbox, &psli->mboxq, list) {
-		mp = (struct lpfc_dmabuf *)(mbox->ctx_buf);
-		if (mp) {
-			lpfc_mbuf_free(phba, mp->virt, mp->phys);
-			kfree(mp);
-		}
 		list_del(&mbox->list);
-		mempool_free(mbox, phba->mbox_mem_pool);
+		lpfc_mem_free_sli_mbox(phba, mbox);
 	}
 	/* Free memory used in mailbox cmpl list back to mailbox memory pool */
 	list_for_each_entry_safe(mbox, next_mbox, &psli->mboxq_cmpl, list) {
-		mp = (struct lpfc_dmabuf *)(mbox->ctx_buf);
-		if (mp) {
-			lpfc_mbuf_free(phba, mp->virt, mp->phys);
-			kfree(mp);
-		}
 		list_del(&mbox->list);
-		mempool_free(mbox, phba->mbox_mem_pool);
+		lpfc_mem_free_sli_mbox(phba, mbox);
 	}
 	/* Free the active mailbox command back to the mailbox memory pool */
 	spin_lock_irq(&phba->hbalock);
@@ -316,12 +328,7 @@ lpfc_mem_free_all(struct lpfc_hba *phba)
 	spin_unlock_irq(&phba->hbalock);
 	if (psli->mbox_active) {
 		mbox = psli->mbox_active;
-		mp = (struct lpfc_dmabuf *)(mbox->ctx_buf);
-		if (mp) {
-			lpfc_mbuf_free(phba, mp->virt, mp->phys);
-			kfree(mp);
-		}
-		mempool_free(mbox, phba->mbox_mem_pool);
+		lpfc_mem_free_sli_mbox(phba, mbox);
 		psli->mbox_active = NULL;
 	}
 
diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c
index 9386e7b44750..46e6f807d1ca 100644
--- a/drivers/scsi/lpfc/lpfc_sli.c
+++ b/drivers/scsi/lpfc/lpfc_sli.c
@@ -2995,8 +2995,9 @@ lpfc_sli4_unreg_rpi_cmpl_clr(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
 		     LPFC_SLI_INTF_IF_TYPE_2)) {
 			if (ndlp) {
 				lpfc_printf_vlog(
-					 vport, KERN_INFO, LOG_MBOX | LOG_SLI,
-					 "0010 UNREG_LOGIN vpi:%x "
+					 vport, KERN_INFO,
+					 LOG_MBOX | LOG_SLI | LOG_NODE,
+					 "0010 UNREG_LOGIN vpi:x%x "
 					 "rpi:%x DID:%x defer x%x flg x%x "
 					 "x%px\n",
 					 vport->vpi, ndlp->nlp_rpi,
@@ -3012,7 +3013,8 @@ lpfc_sli4_unreg_rpi_cmpl_clr(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
 				    (ndlp->nlp_defer_did !=
 				    NLP_EVT_NOTHING_PENDING)) {
 					lpfc_printf_vlog(
-						vport, KERN_INFO, LOG_DISCOVERY,
+						vport, KERN_INFO,
+						LOG_MBOX | LOG_SLI | LOG_NODE,
 						"4111 UNREG cmpl deferred "
 						"clr x%x on "
 						"NPort x%x Data: x%x x%px\n",
@@ -3938,6 +3940,9 @@ void lpfc_poll_eratt(struct timer_list *t)
 	if (!(phba->hba_flag & HBA_SETUP))
 		return;
 
+	if (phba->pport->load_flag & FC_UNLOADING)
+		return;
+
 	/* Here we will also keep track of interrupts per sec of the hba */
 	sli_intr = phba->sli.slistat.sli_intr;
 
@@ -10141,11 +10146,12 @@ lpfc_sli_issue_mbox_s4(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq,
 	spin_unlock_irqrestore(&phba->hbalock, iflags);
 	lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
 			"(%d):0354 Mbox cmd issue - Enqueue Data: "
-			"x%x (x%x/x%x) x%x x%x x%x\n",
+			"x%x (x%x/x%x) x%x x%x x%x x%x\n",
 			mboxq->vport ? mboxq->vport->vpi : 0xffffff,
 			bf_get(lpfc_mqe_command, &mboxq->u.mqe),
 			lpfc_sli_config_mbox_subsys_get(phba, mboxq),
 			lpfc_sli_config_mbox_opcode_get(phba, mboxq),
+			mboxq->u.mb.un.varUnregLogin.rpi,
 			phba->pport->port_state,
 			psli->sli_flag, MBX_NOWAIT);
 	/* Wake up worker thread to transport mailbox command from head */
@@ -22167,6 +22173,12 @@ struct lpfc_io_buf *lpfc_get_io_buf(struct lpfc_hba *phba,
  * The data will be truncated if datasz is not large enough.
  * Version 1 is not supported with Embedded mbox cmd, so we must use version 0.
  * Returns the actual bytes read from the object.
+ *
+ * This routine is hard coded to use a poll completion.  Unlike other
+ * sli4_config mailboxes, it uses lpfc_mbuf memory which is not
+ * cleaned up in lpfc_sli4_cmd_mbox_free.  If this routine is modified
+ * to use interrupt-based completions, code is needed to fully cleanup
+ * the memory.
  */
 int
 lpfc_read_object(struct lpfc_hba *phba, char *rdobject, uint32_t *datap,
diff --git a/drivers/scsi/lpfc/lpfc_sli.h b/drivers/scsi/lpfc/lpfc_sli.h
index cd33dfec758c..c911a39cb46b 100644
--- a/drivers/scsi/lpfc/lpfc_sli.h
+++ b/drivers/scsi/lpfc/lpfc_sli.h
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2022 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2023 Broadcom. All Rights Reserved. The term *
  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.     *
  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -182,9 +182,11 @@ typedef struct lpfcMboxq {
 		struct lpfc_mqe mqe;
 	} u;
 	struct lpfc_vport *vport; /* virtual port pointer */
-	void *ctx_ndlp;		  /* caller ndlp information */
-	void *ctx_buf;		  /* caller buffer information */
-	void *context3;
+	void *ctx_ndlp;		  /* an lpfc_nodelist pointer */
+	void *ctx_buf;		  /* an lpfc_dmabuf pointer */
+	void *context3;           /* a generic pointer.  Code must
+				   * accommodate the actual datatype.
+				   */
 
 	void (*mbox_cmpl) (struct lpfc_hba *, struct lpfcMboxq *);
 	uint8_t mbox_flag;
diff --git a/drivers/scsi/lpfc/lpfc_version.h b/drivers/scsi/lpfc/lpfc_version.h
index 88068834cab9..b7d39e2f19fc 100644
--- a/drivers/scsi/lpfc/lpfc_version.h
+++ b/drivers/scsi/lpfc/lpfc_version.h
@@ -20,7 +20,7 @@
  * included with this package.                                     *
  *******************************************************************/
 
-#define LPFC_DRIVER_VERSION "14.2.0.15"
+#define LPFC_DRIVER_VERSION "14.2.0.16"
 #define LPFC_DRIVER_NAME		"lpfc"
 
 /* Used for SLI 2/3 */
diff --git a/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h b/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h
index 4d0be5ab98c1..587f7d248219 100644
--- a/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h
+++ b/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h
@@ -643,18 +643,14 @@ typedef struct _MPI2_CHIP_REVISION_ID {
 /*Manufacturing Page 2 */
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check Header.PageLength at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check Header.PageLength at
+ *runtime before using HwSettings[].
  */
-#ifndef MPI2_MAN_PAGE_2_HW_SETTINGS_WORDS
-#define MPI2_MAN_PAGE_2_HW_SETTINGS_WORDS   (1)
-#endif
 
 typedef struct _MPI2_CONFIG_PAGE_MAN_2 {
 	MPI2_CONFIG_PAGE_HEADER Header;                     /*0x00 */
 	MPI2_CHIP_REVISION_ID   ChipId;                     /*0x04 */
-	U32
-		HwSettings[MPI2_MAN_PAGE_2_HW_SETTINGS_WORDS];/*0x08 */
+	U32                     HwSettings[];               /*0x08 */
 } MPI2_CONFIG_PAGE_MAN_2,
 	*PTR_MPI2_CONFIG_PAGE_MAN_2,
 	Mpi2ManufacturingPage2_t,
@@ -666,18 +662,14 @@ typedef struct _MPI2_CONFIG_PAGE_MAN_2 {
 /*Manufacturing Page 3 */
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check Header.PageLength at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check Header.PageLength at
+ *runtime before using Info[].
  */
-#ifndef MPI2_MAN_PAGE_3_INFO_WORDS
-#define MPI2_MAN_PAGE_3_INFO_WORDS          (1)
-#endif
 
 typedef struct _MPI2_CONFIG_PAGE_MAN_3 {
 	MPI2_CONFIG_PAGE_HEADER             Header;         /*0x00 */
 	MPI2_CHIP_REVISION_ID               ChipId;         /*0x04 */
-	U32
-		Info[MPI2_MAN_PAGE_3_INFO_WORDS];/*0x08 */
+	U32                                 Info[];         /*0x08 */
 } MPI2_CONFIG_PAGE_MAN_3,
 	*PTR_MPI2_CONFIG_PAGE_MAN_3,
 	Mpi2ManufacturingPage3_t,
@@ -765,12 +757,9 @@ typedef struct _MPI2_CONFIG_PAGE_MAN_4 {
 /*Manufacturing Page 5 */
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for NumPhys at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check the value returned
+ *for NumPhys at runtime before using Phy[].
  */
-#ifndef MPI2_MAN_PAGE_5_PHY_ENTRIES
-#define MPI2_MAN_PAGE_5_PHY_ENTRIES         (1)
-#endif
 
 typedef struct _MPI2_MANUFACTURING5_ENTRY {
 	U64                                 WWID;           /*0x00 */
@@ -787,8 +776,7 @@ typedef struct _MPI2_CONFIG_PAGE_MAN_5 {
 	U16                                 Reserved2;      /*0x06 */
 	U32                                 Reserved3;      /*0x08 */
 	U32                                 Reserved4;      /*0x0C */
-	MPI2_MANUFACTURING5_ENTRY
-		Phy[MPI2_MAN_PAGE_5_PHY_ENTRIES];/*0x08 */
+	MPI2_MANUFACTURING5_ENTRY           Phy[];          /*0x10 */
 } MPI2_CONFIG_PAGE_MAN_5,
 	*PTR_MPI2_CONFIG_PAGE_MAN_5,
 	Mpi2ManufacturingPage5_t,
@@ -864,12 +852,9 @@ typedef struct _MPI2_MANPAGE7_CONNECTOR_INFO {
 #define MPI2_MANPAGE7_SLOT_UNKNOWN                      (0xFFFF)
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for NumPhys at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check the value returned
+ *for NumPhys at runtime before using ConnectorInfo[].
  */
-#ifndef MPI2_MANPAGE7_CONNECTOR_INFO_MAX
-#define MPI2_MANPAGE7_CONNECTOR_INFO_MAX  (1)
-#endif
 
 typedef struct _MPI2_CONFIG_PAGE_MAN_7 {
 	MPI2_CONFIG_PAGE_HEADER         Header;             /*0x00 */
@@ -880,8 +865,7 @@ typedef struct _MPI2_CONFIG_PAGE_MAN_7 {
 	U8                              NumPhys;            /*0x20 */
 	U8                              Reserved3;          /*0x21 */
 	U16                             Reserved4;          /*0x22 */
-	MPI2_MANPAGE7_CONNECTOR_INFO
-	ConnectorInfo[MPI2_MANPAGE7_CONNECTOR_INFO_MAX]; /*0x24 */
+	MPI2_MANPAGE7_CONNECTOR_INFO    ConnectorInfo[];    /*0x24 */
 } MPI2_CONFIG_PAGE_MAN_7,
 	*PTR_MPI2_CONFIG_PAGE_MAN_7,
 	Mpi2ManufacturingPage7_t,
@@ -990,7 +974,7 @@ typedef struct _MPI2_CONFIG_PAGE_IO_UNIT_1 {
 
 /*
  *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for GPIOCount at runtime.
+ *36 and check the value returned for GPIOCount at runtime.
  */
 #ifndef MPI2_IO_UNIT_PAGE_3_GPIO_VAL_MAX
 #define MPI2_IO_UNIT_PAGE_3_GPIO_VAL_MAX    (36)
@@ -1019,12 +1003,9 @@ typedef struct _MPI2_CONFIG_PAGE_IO_UNIT_3 {
 /*IO Unit Page 5 */
 
 /*
- *Upper layer code (drivers, utilities, etc.) should leave this define set to
- *one and check the value returned for NumDmaEngines at runtime.
+ *Upper layer code (drivers, utilities, etc.) should check the value returned
+ *for NumDmaEngines at runtime before using DmaEngineCapabilities[].
  */
-#ifndef MPI2_IOUNITPAGE5_DMAENGINE_ENTRIES
-#define MPI2_IOUNITPAGE5_DMAENGINE_ENTRIES      (1)
-#endif
 
 typedef struct _MPI2_CONFIG_PAGE_IO_UNIT_5 {
 	MPI2_CONFIG_PAGE_HEADER Header;                     /*0x00 */
@@ -1042,7 +1023,7 @@ typedef struct _MPI2_CONFIG_PAGE_IO_UNIT_5 {
 	U32                     Reserved2;                  /*0x24 */
 	U32                     Reserved3;                  /*0x28 */
 	U32
-	DmaEngineCapabilities[MPI2_IOUNITPAGE5_DMAENGINE_ENTRIES]; /*0x2C */
+		DmaEngineCapabilities[];                    /*0x2C */
 } MPI2_CONFIG_PAGE_IO_UNIT_5,
 	*PTR_MPI2_CONFIG_PAGE_IO_UNIT_5,
 	Mpi2IOUnitPage5_t, *pMpi2IOUnitPage5_t;
@@ -1219,12 +1200,9 @@ typedef struct _MPI2_IOUNIT8_SENSOR {
 #define MPI2_IOUNIT8_SENSOR_FLAGS_T0_ENABLE         (0x0001)
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for NumSensors at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check the value returned
+ *for NumSensors at runtime before using Sensor[].
  */
-#ifndef MPI2_IOUNITPAGE8_SENSOR_ENTRIES
-#define MPI2_IOUNITPAGE8_SENSOR_ENTRIES     (1)
-#endif
 
 typedef struct _MPI2_CONFIG_PAGE_IO_UNIT_8 {
 	MPI2_CONFIG_PAGE_HEADER Header;                 /*0x00 */
@@ -1233,8 +1211,7 @@ typedef struct _MPI2_CONFIG_PAGE_IO_UNIT_8 {
 	U8                      NumSensors;             /*0x0C */
 	U8                      PollingInterval;        /*0x0D */
 	U16                     Reserved3;              /*0x0E */
-	MPI2_IOUNIT8_SENSOR
-		Sensor[MPI2_IOUNITPAGE8_SENSOR_ENTRIES];/*0x10 */
+	MPI2_IOUNIT8_SENSOR     Sensor[];               /*0x10 */
 } MPI2_CONFIG_PAGE_IO_UNIT_8,
 	*PTR_MPI2_CONFIG_PAGE_IO_UNIT_8,
 	Mpi2IOUnitPage8_t, *pMpi2IOUnitPage8_t;
@@ -1259,12 +1236,9 @@ typedef struct _MPI2_IOUNIT9_SENSOR {
 #define MPI2_IOUNIT9_SENSOR_FLAGS_TEMP_VALID        (0x01)
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for NumSensors at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check the value returned
+ *for NumSensors at runtime before using Sensor[].
  */
-#ifndef MPI2_IOUNITPAGE9_SENSOR_ENTRIES
-#define MPI2_IOUNITPAGE9_SENSOR_ENTRIES     (1)
-#endif
 
 typedef struct _MPI2_CONFIG_PAGE_IO_UNIT_9 {
 	MPI2_CONFIG_PAGE_HEADER Header;                 /*0x00 */
@@ -1273,8 +1247,7 @@ typedef struct _MPI2_CONFIG_PAGE_IO_UNIT_9 {
 	U8                      NumSensors;             /*0x0C */
 	U8                      Reserved4;              /*0x0D */
 	U16                     Reserved3;              /*0x0E */
-	MPI2_IOUNIT9_SENSOR
-		Sensor[MPI2_IOUNITPAGE9_SENSOR_ENTRIES];/*0x10 */
+	MPI2_IOUNIT9_SENSOR     Sensor[];               /*0x10 */
 } MPI2_CONFIG_PAGE_IO_UNIT_9,
 	*PTR_MPI2_CONFIG_PAGE_IO_UNIT_9,
 	Mpi2IOUnitPage9_t, *pMpi2IOUnitPage9_t;
@@ -1294,12 +1267,9 @@ typedef struct _MPI2_IOUNIT10_FUNCTION {
 	*pMpi2IOUnit10Function_t;
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for NumFunctions at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check the value returned
+ *for NumFunctions at runtime before using Function[].
  */
-#ifndef MPI2_IOUNITPAGE10_FUNCTION_ENTRIES
-#define MPI2_IOUNITPAGE10_FUNCTION_ENTRIES      (1)
-#endif
 
 typedef struct _MPI2_CONFIG_PAGE_IO_UNIT_10 {
 	MPI2_CONFIG_PAGE_HEADER Header;                      /*0x00 */
@@ -1308,8 +1278,7 @@ typedef struct _MPI2_CONFIG_PAGE_IO_UNIT_10 {
 	U16                     Reserved2;                   /*0x06 */
 	U32                     Reserved3;                   /*0x08 */
 	U32                     Reserved4;                   /*0x0C */
-	MPI2_IOUNIT10_FUNCTION
-		Function[MPI2_IOUNITPAGE10_FUNCTION_ENTRIES];/*0x10 */
+	MPI2_IOUNIT10_FUNCTION  Function[];                  /*0x10 */
 } MPI2_CONFIG_PAGE_IO_UNIT_10,
 	*PTR_MPI2_CONFIG_PAGE_IO_UNIT_10,
 	Mpi2IOUnitPage10_t, *pMpi2IOUnitPage10_t;
@@ -1764,12 +1733,9 @@ typedef struct _MPI2_CONFIG_PAGE_BIOS_3 {
 /*BIOS Page 4 */
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for NumPhys at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check the value returned
+ *for NumPhys at runtime before using Phy[].
  */
-#ifndef MPI2_BIOS_PAGE_4_PHY_ENTRIES
-#define MPI2_BIOS_PAGE_4_PHY_ENTRIES        (1)
-#endif
 
 typedef struct _MPI2_BIOS4_ENTRY {
 	U64                     ReassignmentWWID;       /*0x00 */
@@ -1782,8 +1748,7 @@ typedef struct _MPI2_CONFIG_PAGE_BIOS_4 {
 	U8                      NumPhys;            /*0x04 */
 	U8                      Reserved1;          /*0x05 */
 	U16                     Reserved2;          /*0x06 */
-	MPI2_BIOS4_ENTRY
-		Phy[MPI2_BIOS_PAGE_4_PHY_ENTRIES];  /*0x08 */
+	MPI2_BIOS4_ENTRY        Phy[];              /*0x08 */
 } MPI2_CONFIG_PAGE_BIOS_4, *PTR_MPI2_CONFIG_PAGE_BIOS_4,
 	Mpi2BiosPage4_t, *pMpi2BiosPage4_t;
 
@@ -1836,12 +1801,9 @@ typedef struct _MPI2_RAIDVOL0_SETTINGS {
 #define MPI2_RAIDVOL0_SETTING_ENABLE_WRITE_CACHING      (0x0002)
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for NumPhysDisks at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check the value returned
+ *for NumPhysDisks at runtime before using PhysDisk[].
  */
-#ifndef MPI2_RAID_VOL_PAGE_0_PHYSDISK_MAX
-#define MPI2_RAID_VOL_PAGE_0_PHYSDISK_MAX       (1)
-#endif
 
 typedef struct _MPI2_CONFIG_PAGE_RAID_VOL_0 {
 	MPI2_CONFIG_PAGE_HEADER Header;            /*0x00 */
@@ -1861,8 +1823,7 @@ typedef struct _MPI2_CONFIG_PAGE_RAID_VOL_0 {
 	U8                      Reserved2;         /*0x25 */
 	U8                      Reserved3;         /*0x26 */
 	U8                      InactiveStatus;    /*0x27 */
-	MPI2_RAIDVOL0_PHYS_DISK
-	PhysDisk[MPI2_RAID_VOL_PAGE_0_PHYSDISK_MAX]; /*0x28 */
+	MPI2_RAIDVOL0_PHYS_DISK PhysDisk[];        /*0x28 */
 } MPI2_CONFIG_PAGE_RAID_VOL_0,
 	*PTR_MPI2_CONFIG_PAGE_RAID_VOL_0,
 	Mpi2RaidVolPage0_t, *pMpi2RaidVolPage0_t;
@@ -2045,12 +2006,9 @@ typedef struct _MPI2_CONFIG_PAGE_RD_PDISK_0 {
 /*RAID Physical Disk Page 1 */
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for NumPhysDiskPaths at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check the value returned
+ *for NumPhysDiskPaths at runtime before using PhysicalDiskPath[].
  */
-#ifndef MPI2_RAID_PHYS_DISK1_PATH_MAX
-#define MPI2_RAID_PHYS_DISK1_PATH_MAX   (1)
-#endif
 
 typedef struct _MPI2_RAIDPHYSDISK1_PATH {
 	U16             DevHandle;          /*0x00 */
@@ -2075,8 +2033,7 @@ typedef struct _MPI2_CONFIG_PAGE_RD_PDISK_1 {
 	U8                              PhysDiskNum;        /*0x05 */
 	U16                             Reserved1;          /*0x06 */
 	U32                             Reserved2;          /*0x08 */
-	MPI2_RAIDPHYSDISK1_PATH
-		PhysicalDiskPath[MPI2_RAID_PHYS_DISK1_PATH_MAX];/*0x0C */
+	MPI2_RAIDPHYSDISK1_PATH         PhysicalDiskPath[]; /*0x0C */
 } MPI2_CONFIG_PAGE_RD_PDISK_1,
 	*PTR_MPI2_CONFIG_PAGE_RD_PDISK_1,
 	Mpi2RaidPhysDiskPage1_t,
@@ -2221,12 +2178,9 @@ typedef struct _MPI2_SAS_IO_UNIT0_PHY_DATA {
 	*pMpi2SasIOUnit0PhyData_t;
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for NumPhys at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check the value returned
+ *for NumPhys at runtime before using PhyData[].
  */
-#ifndef MPI2_SAS_IOUNIT0_PHY_MAX
-#define MPI2_SAS_IOUNIT0_PHY_MAX        (1)
-#endif
 
 typedef struct _MPI2_CONFIG_PAGE_SASIOUNIT_0 {
 	MPI2_CONFIG_EXTENDED_PAGE_HEADER    Header;   /*0x00 */
@@ -2234,8 +2188,7 @@ typedef struct _MPI2_CONFIG_PAGE_SASIOUNIT_0 {
 	U8                                  NumPhys;  /*0x0C */
 	U8                                  Reserved2;/*0x0D */
 	U16                                 Reserved3;/*0x0E */
-	MPI2_SAS_IO_UNIT0_PHY_DATA
-		PhyData[MPI2_SAS_IOUNIT0_PHY_MAX];    /*0x10 */
+	MPI2_SAS_IO_UNIT0_PHY_DATA          PhyData[];/*0x10 */
 } MPI2_CONFIG_PAGE_SASIOUNIT_0,
 	*PTR_MPI2_CONFIG_PAGE_SASIOUNIT_0,
 	Mpi2SasIOUnitPage0_t, *pMpi2SasIOUnitPage0_t;
@@ -2296,12 +2249,9 @@ typedef struct _MPI2_SAS_IO_UNIT1_PHY_DATA {
 	*pMpi2SasIOUnit1PhyData_t;
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for NumPhys at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check the value returned
+ *for NumPhys at runtime before using PhyData[].
  */
-#ifndef MPI2_SAS_IOUNIT1_PHY_MAX
-#define MPI2_SAS_IOUNIT1_PHY_MAX        (1)
-#endif
 
 typedef struct _MPI2_CONFIG_PAGE_SASIOUNIT_1 {
 	MPI2_CONFIG_EXTENDED_PAGE_HEADER    Header; /*0x00 */
@@ -2322,7 +2272,7 @@ typedef struct _MPI2_CONFIG_PAGE_SASIOUNIT_1 {
 	U8
 		IODeviceMissingDelay;               /*0x13 */
 	MPI2_SAS_IO_UNIT1_PHY_DATA
-		PhyData[MPI2_SAS_IOUNIT1_PHY_MAX];  /*0x14 */
+		PhyData[];                          /*0x14 */
 } MPI2_CONFIG_PAGE_SASIOUNIT_1,
 	*PTR_MPI2_CONFIG_PAGE_SASIOUNIT_1,
 	Mpi2SasIOUnitPage1_t, *pMpi2SasIOUnitPage1_t;
@@ -2502,12 +2452,9 @@ typedef struct _MPI2_SAS_IO_UNIT5_PHY_PM_SETTINGS {
 #define MPI2_SASIOUNIT5_ITE_ONE_MICROSECOND             (0)
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for NumPhys at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check the value returned
+ *for NumPhys at runtime before using SASPhyPowerManagementSettings[].
  */
-#ifndef MPI2_SAS_IOUNIT5_PHY_MAX
-#define MPI2_SAS_IOUNIT5_PHY_MAX        (1)
-#endif
 
 typedef struct _MPI2_CONFIG_PAGE_SASIOUNIT_5 {
 	MPI2_CONFIG_EXTENDED_PAGE_HEADER    Header;   /*0x00 */
@@ -2516,7 +2463,7 @@ typedef struct _MPI2_CONFIG_PAGE_SASIOUNIT_5 {
 	U16                                 Reserved2;/*0x0A */
 	U32                                 Reserved3;/*0x0C */
 	MPI2_SAS_IO_UNIT5_PHY_PM_SETTINGS
-	SASPhyPowerManagementSettings[MPI2_SAS_IOUNIT5_PHY_MAX];/*0x10 */
+		SASPhyPowerManagementSettings[];      /*0x10 */
 } MPI2_CONFIG_PAGE_SASIOUNIT_5,
 	*PTR_MPI2_CONFIG_PAGE_SASIOUNIT_5,
 	Mpi2SasIOUnitPage5_t, *pMpi2SasIOUnitPage5_t;
@@ -2554,12 +2501,9 @@ typedef struct _MPI2_SAS_IO_UNIT6_PORT_WIDTH_MOD_GROUP_STATUS {
 #define MPI2_SASIOUNIT6_MODULATION_100_PERCENT                  (0x03)
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for NumGroups at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check the value returned
+ *for NumGroups at runtime before using PortWidthModulationGroupStatus[].
  */
-#ifndef MPI2_SAS_IOUNIT6_GROUP_MAX
-#define MPI2_SAS_IOUNIT6_GROUP_MAX      (1)
-#endif
 
 typedef struct _MPI2_CONFIG_PAGE_SASIOUNIT_6 {
 	MPI2_CONFIG_EXTENDED_PAGE_HEADER    Header;                 /*0x00 */
@@ -2569,7 +2513,7 @@ typedef struct _MPI2_CONFIG_PAGE_SASIOUNIT_6 {
 	U8                                  Reserved3;              /*0x11 */
 	U16                                 Reserved4;              /*0x12 */
 	MPI2_SAS_IO_UNIT6_PORT_WIDTH_MOD_GROUP_STATUS
-	PortWidthModulationGroupStatus[MPI2_SAS_IOUNIT6_GROUP_MAX]; /*0x14 */
+		PortWidthModulationGroupStatus[];                   /*0x14 */
 } MPI2_CONFIG_PAGE_SASIOUNIT_6,
 	*PTR_MPI2_CONFIG_PAGE_SASIOUNIT_6,
 	Mpi2SasIOUnitPage6_t, *pMpi2SasIOUnitPage6_t;
@@ -2597,12 +2541,9 @@ typedef struct _MPI2_SAS_IO_UNIT7_PORT_WIDTH_MOD_GROUP_SETTINGS {
 
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for NumGroups at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check the value returned
+ *for NumGroups at runtime before using PortWidthModulationGroupSettings[].
  */
-#ifndef MPI2_SAS_IOUNIT7_GROUP_MAX
-#define MPI2_SAS_IOUNIT7_GROUP_MAX      (1)
-#endif
 
 typedef struct _MPI2_CONFIG_PAGE_SASIOUNIT_7 {
 	MPI2_CONFIG_EXTENDED_PAGE_HEADER Header;             /*0x00 */
@@ -2615,7 +2556,7 @@ typedef struct _MPI2_CONFIG_PAGE_SASIOUNIT_7 {
 	U8                               Reserved4;          /*0x15 */
 	U16                              Reserved5;          /*0x16 */
 	MPI2_SAS_IO_UNIT7_PORT_WIDTH_MOD_GROUP_SETTINGS
-	PortWidthModulationGroupSettings[MPI2_SAS_IOUNIT7_GROUP_MAX];/*0x18 */
+		PortWidthModulationGroupSettings[];          /*0x18 */
 } MPI2_CONFIG_PAGE_SASIOUNIT_7,
 	*PTR_MPI2_CONFIG_PAGE_SASIOUNIT_7,
 	Mpi2SasIOUnitPage7_t, *pMpi2SasIOUnitPage7_t;
@@ -3086,12 +3027,9 @@ typedef struct _MPI2_SASPHY2_PHY_EVENT {
 
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for NumPhyEvents at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check the value returned
+ *for NumPhyEvents at runtime before using PhyEvent[].
  */
-#ifndef MPI2_SASPHY2_PHY_EVENT_MAX
-#define MPI2_SASPHY2_PHY_EVENT_MAX      (1)
-#endif
 
 typedef struct _MPI2_CONFIG_PAGE_SAS_PHY_2 {
 	MPI2_CONFIG_EXTENDED_PAGE_HEADER
@@ -3105,7 +3043,7 @@ typedef struct _MPI2_CONFIG_PAGE_SAS_PHY_2 {
 	U16
 		Reserved3;                  /*0x0E */
 	MPI2_SASPHY2_PHY_EVENT
-		PhyEvent[MPI2_SASPHY2_PHY_EVENT_MAX]; /*0x10 */
+		PhyEvent[];                 /*0x10 */
 } MPI2_CONFIG_PAGE_SAS_PHY_2,
 	*PTR_MPI2_CONFIG_PAGE_SAS_PHY_2,
 	Mpi2SasPhyPage2_t,
@@ -3200,12 +3138,9 @@ typedef struct _MPI2_SASPHY3_PHY_EVENT_CONFIG {
 #define MPI2_SASPHY3_TFLAGS_EVENT_NOTIFY                    (0x0001)
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for NumPhyEvents at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check the value returned
+ *for NumPhyEvents at runtime before using PhyEventConfig[].
  */
-#ifndef MPI2_SASPHY3_PHY_EVENT_MAX
-#define MPI2_SASPHY3_PHY_EVENT_MAX      (1)
-#endif
 
 typedef struct _MPI2_CONFIG_PAGE_SAS_PHY_3 {
 	MPI2_CONFIG_EXTENDED_PAGE_HEADER
@@ -3219,7 +3154,7 @@ typedef struct _MPI2_CONFIG_PAGE_SAS_PHY_3 {
 	U16
 		Reserved3;                  /*0x0E */
 	MPI2_SASPHY3_PHY_EVENT_CONFIG
-		PhyEventConfig[MPI2_SASPHY3_PHY_EVENT_MAX]; /*0x10 */
+		PhyEventConfig[];           /*0x10 */
 } MPI2_CONFIG_PAGE_SAS_PHY_3,
 	*PTR_MPI2_CONFIG_PAGE_SAS_PHY_3,
 	Mpi2SasPhyPage3_t, *pMpi2SasPhyPage3_t;
@@ -3358,12 +3293,9 @@ typedef struct _MPI2_CONFIG_PAGE_SAS_ENCLOSURE_0 {
 /*Log Page 0 */
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for NumLogEntries at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check the value returned
+ *for NumLogEntries at runtime before using LogEntry[].
  */
-#ifndef MPI2_LOG_0_NUM_LOG_ENTRIES
-#define MPI2_LOG_0_NUM_LOG_ENTRIES          (1)
-#endif
 
 #define MPI2_LOG_0_LOG_DATA_LENGTH          (0x1C)
 
@@ -3393,8 +3325,7 @@ typedef struct _MPI2_CONFIG_PAGE_LOG_0 {
 	U32                                 Reserved2;    /*0x0C */
 	U16                                 NumLogEntries;/*0x10 */
 	U16                                 Reserved3;    /*0x12 */
-	MPI2_LOG_0_ENTRY
-		LogEntry[MPI2_LOG_0_NUM_LOG_ENTRIES]; /*0x14 */
+	MPI2_LOG_0_ENTRY                    LogEntry[];   /*0x14 */
 } MPI2_CONFIG_PAGE_LOG_0, *PTR_MPI2_CONFIG_PAGE_LOG_0,
 	Mpi2LogPage0_t, *pMpi2LogPage0_t;
 
@@ -3408,12 +3339,9 @@ typedef struct _MPI2_CONFIG_PAGE_LOG_0 {
 /*RAID Page 0 */
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for NumElements at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check the value returned
+ *for NumElements at runtime before using ConfigElement[].
  */
-#ifndef MPI2_RAIDCONFIG0_MAX_ELEMENTS
-#define MPI2_RAIDCONFIG0_MAX_ELEMENTS       (1)
-#endif
 
 typedef struct _MPI2_RAIDCONFIG0_CONFIG_ELEMENT {
 	U16                     ElementFlags;             /*0x00 */
@@ -3446,8 +3374,7 @@ typedef struct _MPI2_CONFIG_PAGE_RAID_CONFIGURATION_0 {
 	U8                                  NumElements;    /*0x2C */
 	U8                                  Reserved2;      /*0x2D */
 	U16                                 Reserved3;      /*0x2E */
-	MPI2_RAIDCONFIG0_CONFIG_ELEMENT
-		ConfigElement[MPI2_RAIDCONFIG0_MAX_ELEMENTS]; /*0x30 */
+	MPI2_RAIDCONFIG0_CONFIG_ELEMENT     ConfigElement[];/*0x30 */
 } MPI2_CONFIG_PAGE_RAID_CONFIGURATION_0,
 	*PTR_MPI2_CONFIG_PAGE_RAID_CONFIGURATION_0,
 	Mpi2RaidConfigurationPage0_t,
@@ -3687,12 +3614,9 @@ typedef struct _MPI26_PCIE_IO_UNIT0_PHY_DATA {
 	Mpi26PCIeIOUnit0PhyData_t, *pMpi26PCIeIOUnit0PhyData_t;
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for NumPhys at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check the value returned
+ *for NumPhys at runtime before using PhyData[].
  */
-#ifndef MPI26_PCIE_IOUNIT0_PHY_MAX
-#define MPI26_PCIE_IOUNIT0_PHY_MAX      (1)
-#endif
 
 typedef struct _MPI26_CONFIG_PAGE_PIOUNIT_0 {
 	MPI2_CONFIG_EXTENDED_PAGE_HEADER	Header; /*0x00 */
@@ -3701,7 +3625,7 @@ typedef struct _MPI26_CONFIG_PAGE_PIOUNIT_0 {
 	U8	InitStatus;                             /*0x0D */
 	U16	Reserved3;                              /*0x0E */
 	MPI26_PCIE_IO_UNIT0_PHY_DATA
-		PhyData[MPI26_PCIE_IOUNIT0_PHY_MAX];    /*0x10 */
+		PhyData[];                              /*0x10 */
 } MPI26_CONFIG_PAGE_PIOUNIT_0,
 	*PTR_MPI26_CONFIG_PAGE_PIOUNIT_0,
 	Mpi26PCIeIOUnitPage0_t, *pMpi26PCIeIOUnitPage0_t;
@@ -3744,12 +3668,9 @@ typedef struct _MPI26_PCIE_IO_UNIT1_PHY_DATA {
 #define MPI26_PCIEIOUNIT1_LINKFLAGS_SRNS_EN                 (0x02)
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for NumPhys at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check the value returned
+ *for NumPhys at runtime before using PhyData[].
  */
-#ifndef MPI26_PCIE_IOUNIT1_PHY_MAX
-#define MPI26_PCIE_IOUNIT1_PHY_MAX      (1)
-#endif
 
 typedef struct _MPI26_CONFIG_PAGE_PIOUNIT_1 {
 	MPI2_CONFIG_EXTENDED_PAGE_HEADER	Header;	/*0x00 */
@@ -3761,7 +3682,7 @@ typedef struct _MPI26_CONFIG_PAGE_PIOUNIT_1 {
 	U8	DMDReportPCIe;                      /*0x11 */
 	U16	Reserved2;                          /*0x12 */
 	MPI26_PCIE_IO_UNIT1_PHY_DATA
-		PhyData[MPI26_PCIE_IOUNIT1_PHY_MAX];/*0x14 */
+		PhyData[];                          /*0x14 */
 } MPI26_CONFIG_PAGE_PIOUNIT_1,
 	*PTR_MPI26_CONFIG_PAGE_PIOUNIT_1,
 	Mpi26PCIeIOUnitPage1_t, *pMpi26PCIeIOUnitPage1_t;
@@ -3993,12 +3914,9 @@ typedef struct _MPI26_PCIELINK2_LINK_EVENT {
 
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for NumLinkEvents at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check the value returned
+ *for NumLinkEvents at runtime before using LinkEvent[].
  */
-#ifndef MPI26_PCIELINK2_LINK_EVENT_MAX
-#define MPI26_PCIELINK2_LINK_EVENT_MAX      (1)
-#endif
 
 typedef struct _MPI26_CONFIG_PAGE_PCIELINK_2 {
 	MPI2_CONFIG_EXTENDED_PAGE_HEADER	Header;	/*0x00 */
@@ -4009,7 +3927,7 @@ typedef struct _MPI26_CONFIG_PAGE_PCIELINK_2 {
 	U8	Reserved3;                  /*0x0D */
 	U16	Reserved4;                  /*0x0E */
 	MPI26_PCIELINK2_LINK_EVENT
-		LinkEvent[MPI26_PCIELINK2_LINK_EVENT_MAX];	/*0x10 */
+		LinkEvent[];                /*0x10 */
 } MPI26_CONFIG_PAGE_PCIELINK_2, *PTR_MPI26_CONFIG_PAGE_PCIELINK_2,
 	Mpi26PcieLinkPage2_t, *pMpi26PcieLinkPage2_t;
 
@@ -4067,12 +3985,9 @@ typedef struct _MPI26_PCIELINK3_LINK_EVENT_CONFIG {
 #define MPI26_PCIELINK3_TFLAGS_EVENT_NOTIFY                 (0x0001)
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check the value returned for NumLinkEvents at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check the value returned
+ *for NumLinkEvents at runtime before using LinkEventConfig[].
  */
-#ifndef MPI26_PCIELINK3_LINK_EVENT_MAX
-#define MPI26_PCIELINK3_LINK_EVENT_MAX      (1)
-#endif
 
 typedef struct _MPI26_CONFIG_PAGE_PCIELINK_3 {
 	MPI2_CONFIG_EXTENDED_PAGE_HEADER	Header;	/*0x00 */
@@ -4083,7 +3998,7 @@ typedef struct _MPI26_CONFIG_PAGE_PCIELINK_3 {
 	U8	Reserved3;                  /*0x0D */
 	U16	Reserved4;                  /*0x0E */
 	MPI26_PCIELINK3_LINK_EVENT_CONFIG
-		LinkEventConfig[MPI26_PCIELINK3_LINK_EVENT_MAX]; /*0x10 */
+		LinkEventConfig[];          /*0x10 */
 } MPI26_CONFIG_PAGE_PCIELINK_3, *PTR_MPI26_CONFIG_PAGE_PCIELINK_3,
 	Mpi26PcieLinkPage3_t, *pMpi26PcieLinkPage3_t;
 
diff --git a/drivers/scsi/mpt3sas/mpi/mpi2_image.h b/drivers/scsi/mpt3sas/mpi/mpi2_image.h
index 33b9c3a6fd40..798ab6e33eb9 100644
--- a/drivers/scsi/mpt3sas/mpi/mpi2_image.h
+++ b/drivers/scsi/mpt3sas/mpi/mpi2_image.h
@@ -295,20 +295,9 @@ typedef struct _MPI2_EXT_IMAGE_HEADER {
 /*FLASH Layout Extended Image Data */
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check RegionsPerLayout at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check NumberOfLayouts and
+ *RegionsPerLayout at runtime before using Layout[] and Region[].
  */
-#ifndef MPI2_FLASH_NUMBER_OF_REGIONS
-#define MPI2_FLASH_NUMBER_OF_REGIONS        (1)
-#endif
-
-/*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check NumberOfLayouts at runtime.
- */
-#ifndef MPI2_FLASH_NUMBER_OF_LAYOUTS
-#define MPI2_FLASH_NUMBER_OF_LAYOUTS        (1)
-#endif
 
 typedef struct _MPI2_FLASH_REGION {
 	U8 RegionType;		/*0x00 */
@@ -325,7 +314,7 @@ typedef struct _MPI2_FLASH_LAYOUT {
 	U32 Reserved1;		/*0x04 */
 	U32 Reserved2;		/*0x08 */
 	U32 Reserved3;		/*0x0C */
-	MPI2_FLASH_REGION Region[MPI2_FLASH_NUMBER_OF_REGIONS];	/*0x10 */
+	MPI2_FLASH_REGION Region[];	/*0x10 */
 } MPI2_FLASH_LAYOUT, *PTR_MPI2_FLASH_LAYOUT,
 	Mpi2FlashLayout_t, *pMpi2FlashLayout_t;
 
@@ -339,7 +328,7 @@ typedef struct _MPI2_FLASH_LAYOUT_DATA {
 	U16 MinimumSectorAlignment;	/*0x08 */
 	U16 Reserved3;		/*0x0A */
 	U32 Reserved4;		/*0x0C */
-	MPI2_FLASH_LAYOUT Layout[MPI2_FLASH_NUMBER_OF_LAYOUTS];	/*0x10 */
+	MPI2_FLASH_LAYOUT Layout[];	/*0x10 */
 } MPI2_FLASH_LAYOUT_DATA, *PTR_MPI2_FLASH_LAYOUT_DATA,
 	Mpi2FlashLayoutData_t, *pMpi2FlashLayoutData_t;
 
@@ -373,12 +362,9 @@ typedef struct _MPI2_FLASH_LAYOUT_DATA {
 /*Supported Devices Extended Image Data */
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check NumberOfDevices at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check NumberOfDevices at
+ *runtime before using SupportedDevice[].
  */
-#ifndef MPI2_SUPPORTED_DEVICES_IMAGE_NUM_DEVICES
-#define MPI2_SUPPORTED_DEVICES_IMAGE_NUM_DEVICES    (1)
-#endif
 
 typedef struct _MPI2_SUPPORTED_DEVICE {
 	U16 DeviceID;		/*0x00 */
@@ -399,7 +385,7 @@ typedef struct _MPI2_SUPPORTED_DEVICES_DATA {
 	U8 Reserved2;		/*0x03 */
 	U32 Reserved3;		/*0x04 */
 	MPI2_SUPPORTED_DEVICE
-	SupportedDevice[MPI2_SUPPORTED_DEVICES_IMAGE_NUM_DEVICES];/*0x08 */
+	SupportedDevice[];	/*0x08 */
 } MPI2_SUPPORTED_DEVICES_DATA, *PTR_MPI2_SUPPORTED_DEVICES_DATA,
 	Mpi2SupportedDevicesData_t, *pMpi2SupportedDevicesData_t;
 
@@ -464,7 +450,7 @@ typedef struct _MPI25_ENCRYPTED_HASH_ENTRY {
 	U8		EncryptionAlgorithm;	/*0x02 */
 	U8		Reserved1;		/*0x03 */
 	U32		Reserved2;		/*0x04 */
-	U32		EncryptedHash[1];	/*0x08 */ /* variable length */
+	U32		EncryptedHash[];	/*0x08 */
 } MPI25_ENCRYPTED_HASH_ENTRY, *PTR_MPI25_ENCRYPTED_HASH_ENTRY,
 Mpi25EncryptedHashEntry_t, *pMpi25EncryptedHashEntry_t;
 
@@ -508,7 +494,7 @@ typedef struct _MPI25_ENCRYPTED_HASH_DATA {
 	U8				NumHash;		/*0x01 */
 	U16				Reserved1;		/*0x02 */
 	U32				Reserved2;		/*0x04 */
-	MPI25_ENCRYPTED_HASH_ENTRY	EncryptedHashEntry[1];  /*0x08 */
+	MPI25_ENCRYPTED_HASH_ENTRY	EncryptedHashEntry[];	/*0x08 */
 } MPI25_ENCRYPTED_HASH_DATA, *PTR_MPI25_ENCRYPTED_HASH_DATA,
 Mpi25EncryptedHashData_t, *pMpi25EncryptedHashData_t;
 
diff --git a/drivers/scsi/mpt3sas/mpi/mpi2_ioc.h b/drivers/scsi/mpt3sas/mpi/mpi2_ioc.h
index 2c57115172cf..d92852591134 100644
--- a/drivers/scsi/mpt3sas/mpi/mpi2_ioc.h
+++ b/drivers/scsi/mpt3sas/mpi/mpi2_ioc.h
@@ -808,12 +808,9 @@ typedef struct _MPI2_EVENT_DATA_IR_PHYSICAL_DISK {
 /*Integrated RAID Configuration Change List Event data */
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check NumElements at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check NumElements at
+ *runtime before using ConfigElement[].
  */
-#ifndef MPI2_EVENT_IR_CONFIG_ELEMENT_COUNT
-#define MPI2_EVENT_IR_CONFIG_ELEMENT_COUNT          (1)
-#endif
 
 typedef struct _MPI2_EVENT_IR_CONFIG_ELEMENT {
 	U16 ElementFlags;	/*0x00 */
@@ -848,7 +845,7 @@ typedef struct _MPI2_EVENT_DATA_IR_CONFIG_CHANGE_LIST {
 	U8 ConfigNum;		/*0x03 */
 	U32 Flags;		/*0x04 */
 	MPI2_EVENT_IR_CONFIG_ELEMENT
-		ConfigElement[MPI2_EVENT_IR_CONFIG_ELEMENT_COUNT];/*0x08 */
+		ConfigElement[];/*0x08 */
 } MPI2_EVENT_DATA_IR_CONFIG_CHANGE_LIST,
 	*PTR_MPI2_EVENT_DATA_IR_CONFIG_CHANGE_LIST,
 	Mpi2EventDataIrConfigChangeList_t,
@@ -969,12 +966,9 @@ typedef struct _MPI2_EVENT_DATA_SAS_INIT_TABLE_OVERFLOW {
 /*SAS Topology Change List Event data */
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check NumEntries at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check NumEntries at
+ *runtime before using PHY[].
  */
-#ifndef MPI2_EVENT_SAS_TOPO_PHY_COUNT
-#define MPI2_EVENT_SAS_TOPO_PHY_COUNT           (1)
-#endif
 
 typedef struct _MPI2_EVENT_SAS_TOPO_PHY_ENTRY {
 	U16 AttachedDevHandle;	/*0x00 */
@@ -994,7 +988,7 @@ typedef struct _MPI2_EVENT_DATA_SAS_TOPOLOGY_CHANGE_LIST {
 	U8 ExpStatus;		/*0x0A */
 	U8 PhysicalPort;	/*0x0B */
 	MPI2_EVENT_SAS_TOPO_PHY_ENTRY
-	PHY[MPI2_EVENT_SAS_TOPO_PHY_COUNT];	/*0x0C */
+	PHY[];			/*0x0C */
 } MPI2_EVENT_DATA_SAS_TOPOLOGY_CHANGE_LIST,
 	*PTR_MPI2_EVENT_DATA_SAS_TOPOLOGY_CHANGE_LIST,
 	Mpi2EventDataSasTopologyChangeList_t,
@@ -1229,12 +1223,9 @@ typedef struct _MPI26_EVENT_DATA_PCIE_ENUMERATION {
 /*PCIe Topology Change List Event data (MPI v2.6 and later) */
 
 /*
- *Host code (drivers, BIOS, utilities, etc.) should leave this define set to
- *one and check NumEntries at runtime.
+ *Host code (drivers, BIOS, utilities, etc.) should check NumEntries at
+ *runtime before using PortEntry[].
  */
-#ifndef MPI26_EVENT_PCIE_TOPO_PORT_COUNT
-#define MPI26_EVENT_PCIE_TOPO_PORT_COUNT        (1)
-#endif
 
 typedef struct _MPI26_EVENT_PCIE_TOPO_PORT_ENTRY {
 	U16	AttachedDevHandle;      /*0x00 */
@@ -1286,7 +1277,7 @@ typedef struct _MPI26_EVENT_DATA_PCIE_TOPOLOGY_CHANGE_LIST {
 	U8	SwitchStatus;           /*0x0A */
 	U8	PhysicalPort;           /*0x0B */
 	MPI26_EVENT_PCIE_TOPO_PORT_ENTRY
-		PortEntry[MPI26_EVENT_PCIE_TOPO_PORT_COUNT]; /*0x0C */
+		PortEntry[];            /*0x0C */
 } MPI26_EVENT_DATA_PCIE_TOPOLOGY_CHANGE_LIST,
 	*PTR_MPI26_EVENT_DATA_PCIE_TOPOLOGY_CHANGE_LIST,
 	Mpi26EventDataPCIeTopologyChangeList_t,
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c
index a75f670bf551..8761bc58d965 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.c
@@ -4893,8 +4893,7 @@ mpt3sas_base_update_missing_delay(struct MPT3SAS_ADAPTER *ioc,
 	if (!num_phys)
 		return;
 
-	sz = offsetof(Mpi2SasIOUnitPage1_t, PhyData) + (num_phys *
-	    sizeof(Mpi2SasIOUnit1PhyData_t));
+	sz = struct_size(sas_iounit_pg1, PhyData, num_phys);
 	sas_iounit_pg1 = kzalloc(sz, GFP_KERNEL);
 	if (!sas_iounit_pg1) {
 		ioc_err(ioc, "failure at %s:%d/%s()!\n",
@@ -5044,7 +5043,7 @@ _base_get_event_diag_triggers(struct MPT3SAS_ADAPTER *ioc)
 {
 	Mpi26DriverTriggerPage2_t trigger_pg2;
 	struct SL_WH_EVENT_TRIGGER_T *event_tg;
-	MPI26_DRIVER_MPI_EVENT_TIGGER_ENTRY *mpi_event_tg;
+	MPI26_DRIVER_MPI_EVENT_TRIGGER_ENTRY *mpi_event_tg;
 	Mpi2ConfigReply_t mpi_reply;
 	int r = 0, i = 0;
 	u16 count = 0;
@@ -5096,7 +5095,7 @@ _base_get_scsi_diag_triggers(struct MPT3SAS_ADAPTER *ioc)
 {
 	Mpi26DriverTriggerPage3_t trigger_pg3;
 	struct SL_WH_SCSI_TRIGGER_T *scsi_tg;
-	MPI26_DRIVER_SCSI_SENSE_TIGGER_ENTRY *mpi_scsi_tg;
+	MPI26_DRIVER_SCSI_SENSE_TRIGGER_ENTRY *mpi_scsi_tg;
 	Mpi2ConfigReply_t mpi_reply;
 	int r = 0, i = 0;
 	u16 count = 0;
@@ -5148,7 +5147,7 @@ _base_get_mpi_diag_triggers(struct MPT3SAS_ADAPTER *ioc)
 {
 	Mpi26DriverTriggerPage4_t trigger_pg4;
 	struct SL_WH_MPI_TRIGGER_T *status_tg;
-	MPI26_DRIVER_IOCSTATUS_LOGINFO_TIGGER_ENTRY *mpi_status_tg;
+	MPI26_DRIVER_IOCSTATUS_LOGINFO_TRIGGER_ENTRY *mpi_status_tg;
 	Mpi2ConfigReply_t mpi_reply;
 	int r = 0, i = 0;
 	u16 count = 0;
@@ -5379,10 +5378,9 @@ _base_update_diag_trigger_pages(struct MPT3SAS_ADAPTER *ioc)
 static int _base_assign_fw_reported_qd(struct MPT3SAS_ADAPTER *ioc)
 {
 	Mpi2ConfigReply_t mpi_reply;
-	Mpi2SasIOUnitPage1_t *sas_iounit_pg1 = NULL;
+	Mpi2SasIOUnitPage1_t sas_iounit_pg1;
 	Mpi26PCIeIOUnitPage1_t pcie_iounit_pg1;
 	u16 depth;
-	int sz;
 	int rc = 0;
 
 	ioc->max_wideport_qd = MPT3SAS_SAS_QUEUE_DEPTH;
@@ -5392,28 +5390,21 @@ static int _base_assign_fw_reported_qd(struct MPT3SAS_ADAPTER *ioc)
 	if (!ioc->is_gen35_ioc)
 		goto out;
 	/* sas iounit page 1 */
-	sz = offsetof(Mpi2SasIOUnitPage1_t, PhyData);
-	sas_iounit_pg1 = kzalloc(sizeof(Mpi2SasIOUnitPage1_t), GFP_KERNEL);
-	if (!sas_iounit_pg1) {
-		pr_err("%s: failure at %s:%d/%s()!\n",
-		    ioc->name, __FILE__, __LINE__, __func__);
-		return rc;
-	}
 	rc = mpt3sas_config_get_sas_iounit_pg1(ioc, &mpi_reply,
-	    sas_iounit_pg1, sz);
+	    &sas_iounit_pg1, sizeof(Mpi2SasIOUnitPage1_t));
 	if (rc) {
 		pr_err("%s: failure at %s:%d/%s()!\n",
 		    ioc->name, __FILE__, __LINE__, __func__);
 		goto out;
 	}
 
-	depth = le16_to_cpu(sas_iounit_pg1->SASWideMaxQueueDepth);
+	depth = le16_to_cpu(sas_iounit_pg1.SASWideMaxQueueDepth);
 	ioc->max_wideport_qd = (depth ? depth : MPT3SAS_SAS_QUEUE_DEPTH);
 
-	depth = le16_to_cpu(sas_iounit_pg1->SASNarrowMaxQueueDepth);
+	depth = le16_to_cpu(sas_iounit_pg1.SASNarrowMaxQueueDepth);
 	ioc->max_narrowport_qd = (depth ? depth : MPT3SAS_SAS_QUEUE_DEPTH);
 
-	depth = sas_iounit_pg1->SATAMaxQDepth;
+	depth = sas_iounit_pg1.SATAMaxQDepth;
 	ioc->max_sata_qd = (depth ? depth : MPT3SAS_SATA_QUEUE_DEPTH);
 
 	/* pcie iounit page 1 */
@@ -5432,7 +5423,6 @@ static int _base_assign_fw_reported_qd(struct MPT3SAS_ADAPTER *ioc)
 	    "MaxWidePortQD: 0x%x MaxNarrowPortQD: 0x%x MaxSataQD: 0x%x MaxNvmeQD: 0x%x\n",
 	    ioc->max_wideport_qd, ioc->max_narrowport_qd,
 	    ioc->max_sata_qd, ioc->max_nvme_qd));
-	kfree(sas_iounit_pg1);
 	return rc;
 }
 
@@ -5588,6 +5578,7 @@ mpt3sas_atto_init(struct MPT3SAS_ADAPTER *ioc)
 static int
 _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc)
 {
+	Mpi2IOUnitPage8_t iounit_pg8;
 	Mpi2ConfigReply_t mpi_reply;
 	u32 iounit_pg1_flags;
 	int tg_flags = 0;
@@ -5684,7 +5675,7 @@ _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc)
 	rc = mpt3sas_config_get_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1);
 	if (rc)
 		return rc;
-	rc = mpt3sas_config_get_iounit_pg8(ioc, &mpi_reply, &ioc->iounit_pg8);
+	rc = mpt3sas_config_get_iounit_pg8(ioc, &mpi_reply, &iounit_pg8);
 	if (rc)
 		return rc;
 	_base_display_ioc_capabilities(ioc);
@@ -5706,8 +5697,8 @@ _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc)
 	if (rc)
 		return rc;
 
-	if (ioc->iounit_pg8.NumSensors)
-		ioc->temp_sensors_count = ioc->iounit_pg8.NumSensors;
+	if (iounit_pg8.NumSensors)
+		ioc->temp_sensors_count = iounit_pg8.NumSensors;
 	if (ioc->is_aero_ioc) {
 		rc = _base_update_ioc_page1_inlinewith_perf_mode(ioc);
 		if (rc)
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.h b/drivers/scsi/mpt3sas/mpt3sas_base.h
index 1be0850ca17a..6d0bc8c66700 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.h
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.h
@@ -1237,7 +1237,6 @@ typedef void (*MPT3SAS_FLUSH_RUNNING_CMDS)(struct MPT3SAS_ADAPTER *ioc);
  * @ioc_pg8: static ioc page 8
  * @iounit_pg0: static iounit page 0
  * @iounit_pg1: static iounit page 1
- * @iounit_pg8: static iounit page 8
  * @sas_hba: sas host object
  * @sas_expander_list: expander object list
  * @enclosure_list: enclosure object list
@@ -1465,7 +1464,6 @@ struct MPT3SAS_ADAPTER {
 	Mpi2IOCPage8_t ioc_pg8;
 	Mpi2IOUnitPage0_t iounit_pg0;
 	Mpi2IOUnitPage1_t iounit_pg1;
-	Mpi2IOUnitPage8_t iounit_pg8;
 	Mpi2IOCPage1_t	ioc_pg1_copy;
 
 	struct _boot_device req_boot_device;
@@ -1983,6 +1981,7 @@ extern const struct attribute_group *mpt3sas_host_groups[];
 extern const struct attribute_group *mpt3sas_dev_groups[];
 void mpt3sas_ctl_init(ushort hbas_to_enumerate);
 void mpt3sas_ctl_exit(ushort hbas_to_enumerate);
+void mpt3sas_ctl_release(struct MPT3SAS_ADAPTER *ioc);
 u8 mpt3sas_ctl_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
 	u32 reply);
 void mpt3sas_ctl_pre_reset_handler(struct MPT3SAS_ADAPTER *ioc);
diff --git a/drivers/scsi/mpt3sas/mpt3sas_config.c b/drivers/scsi/mpt3sas/mpt3sas_config.c
index d114ef381c44..2e88f456fc34 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_config.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_config.c
@@ -2334,7 +2334,7 @@ mpt3sas_config_update_driver_trigger_pg2(struct MPT3SAS_ADAPTER *ioc,
 		tg_pg2.NumMPIEventTrigger = 0;
 		memset(&tg_pg2.MPIEventTriggers[0], 0,
 		    NUM_VALID_ENTRIES * sizeof(
-		    MPI26_DRIVER_MPI_EVENT_TIGGER_ENTRY));
+		    MPI26_DRIVER_MPI_EVENT_TRIGGER_ENTRY));
 	}
 
 	rc = _config_set_driver_trigger_pg2(ioc, &mpi_reply, &tg_pg2);
@@ -2493,7 +2493,7 @@ mpt3sas_config_update_driver_trigger_pg3(struct MPT3SAS_ADAPTER *ioc,
 		tg_pg3.NumSCSISenseTrigger = 0;
 		memset(&tg_pg3.SCSISenseTriggers[0], 0,
 		    NUM_VALID_ENTRIES * sizeof(
-		    MPI26_DRIVER_SCSI_SENSE_TIGGER_ENTRY));
+		    MPI26_DRIVER_SCSI_SENSE_TRIGGER_ENTRY));
 	}
 
 	rc = _config_set_driver_trigger_pg3(ioc, &mpi_reply, &tg_pg3);
@@ -2649,7 +2649,7 @@ mpt3sas_config_update_driver_trigger_pg4(struct MPT3SAS_ADAPTER *ioc,
 		tg_pg4.NumIOCStatusLogInfoTrigger = 0;
 		memset(&tg_pg4.IOCStatusLoginfoTriggers[0], 0,
 		    NUM_VALID_ENTRIES * sizeof(
-		    MPI26_DRIVER_IOCSTATUS_LOGINFO_TIGGER_ENTRY));
+		    MPI26_DRIVER_IOCSTATUS_LOGINFO_TRIGGER_ENTRY));
 	}
 
 	rc = _config_set_driver_trigger_pg4(ioc, &mpi_reply, &tg_pg4);
diff --git a/drivers/scsi/mpt3sas/mpt3sas_ctl.c b/drivers/scsi/mpt3sas/mpt3sas_ctl.c
index efdb8178db32..147cb7088d55 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_ctl.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_ctl.c
@@ -4157,31 +4157,37 @@ mpt3sas_ctl_init(ushort hbas_to_enumerate)
 }
 
 /**
- * mpt3sas_ctl_exit - exit point for ctl
- * @hbas_to_enumerate: ?
+ * mpt3sas_ctl_release - release dma for ctl
+ * @ioc: per adapter object
  */
 void
-mpt3sas_ctl_exit(ushort hbas_to_enumerate)
+mpt3sas_ctl_release(struct MPT3SAS_ADAPTER *ioc)
 {
-	struct MPT3SAS_ADAPTER *ioc;
 	int i;
 
-	list_for_each_entry(ioc, &mpt3sas_ioc_list, list) {
+	/* free memory associated to diag buffers */
+	for (i = 0; i < MPI2_DIAG_BUF_TYPE_COUNT; i++) {
+		if (!ioc->diag_buffer[i])
+			continue;
+		dma_free_coherent(&ioc->pdev->dev,
+				  ioc->diag_buffer_sz[i],
+				  ioc->diag_buffer[i],
+				  ioc->diag_buffer_dma[i]);
+		ioc->diag_buffer[i] = NULL;
+		ioc->diag_buffer_status[i] = 0;
+	}
 
-		/* free memory associated to diag buffers */
-		for (i = 0; i < MPI2_DIAG_BUF_TYPE_COUNT; i++) {
-			if (!ioc->diag_buffer[i])
-				continue;
-			dma_free_coherent(&ioc->pdev->dev,
-					  ioc->diag_buffer_sz[i],
-					  ioc->diag_buffer[i],
-					  ioc->diag_buffer_dma[i]);
-			ioc->diag_buffer[i] = NULL;
-			ioc->diag_buffer_status[i] = 0;
-		}
+	kfree(ioc->event_log);
+}
+
+/**
+ * mpt3sas_ctl_exit - exit point for ctl
+ * @hbas_to_enumerate: ?
+ */
+void
+mpt3sas_ctl_exit(ushort hbas_to_enumerate)
+{
 
-		kfree(ioc->event_log);
-	}
 	if (hbas_to_enumerate != 1)
 		misc_deregister(&ctl_dev);
 	if (hbas_to_enumerate != 2)
diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
index 605013d3ee83..51b5788da040 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
@@ -2431,8 +2431,7 @@ _scsih_get_volume_capabilities(struct MPT3SAS_ADAPTER *ioc,
 	}
 
 	raid_device->num_pds = num_pds;
-	sz = offsetof(Mpi2RaidVolPage0_t, PhysDisk) + (num_pds *
-	    sizeof(Mpi2RaidVol0PhysDisk_t));
+	sz = struct_size(vol_pg0, PhysDisk, num_pds);
 	vol_pg0 = kzalloc(sz, GFP_KERNEL);
 	if (!vol_pg0) {
 		dfailprintk(ioc,
@@ -5966,8 +5965,7 @@ _scsih_update_vphys_after_reset(struct MPT3SAS_ADAPTER *ioc)
 	/*
 	 * Read SASIOUnitPage0 to get each HBA Phy's data.
 	 */
-	sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) +
-	    (ioc->sas_hba.num_phys * sizeof(Mpi2SasIOUnit0PhyData_t));
+	sz = struct_size(sas_iounit_pg0, PhyData, ioc->sas_hba.num_phys);
 	sas_iounit_pg0 = kzalloc(sz, GFP_KERNEL);
 	if (!sas_iounit_pg0) {
 		ioc_err(ioc, "failure at %s:%d/%s()!\n",
@@ -6145,8 +6143,7 @@ _scsih_get_port_table_after_reset(struct MPT3SAS_ADAPTER *ioc,
 	u64 attached_sas_addr;
 	u8 found = 0, port_count = 0, port_id;
 
-	sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) + (ioc->sas_hba.num_phys
-	    * sizeof(Mpi2SasIOUnit0PhyData_t));
+	sz = struct_size(sas_iounit_pg0, PhyData, ioc->sas_hba.num_phys);
 	sas_iounit_pg0 = kzalloc(sz, GFP_KERNEL);
 	if (!sas_iounit_pg0) {
 		ioc_err(ioc, "failure at %s:%d/%s()!\n",
@@ -6579,8 +6576,7 @@ _scsih_sas_host_refresh(struct MPT3SAS_ADAPTER *ioc)
 		  ioc_info(ioc, "updating handles for sas_host(0x%016llx)\n",
 			   (u64)ioc->sas_hba.sas_address));
 
-	sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) + (ioc->sas_hba.num_phys
-	    * sizeof(Mpi2SasIOUnit0PhyData_t));
+	sz = struct_size(sas_iounit_pg0, PhyData, ioc->sas_hba.num_phys);
 	sas_iounit_pg0 = kzalloc(sz, GFP_KERNEL);
 	if (!sas_iounit_pg0) {
 		ioc_err(ioc, "failure at %s:%d/%s()!\n",
@@ -6731,8 +6727,7 @@ _scsih_sas_host_add(struct MPT3SAS_ADAPTER *ioc)
 	ioc->sas_hba.num_phys = num_phys;
 
 	/* sas_iounit page 0 */
-	sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) + (ioc->sas_hba.num_phys *
-	    sizeof(Mpi2SasIOUnit0PhyData_t));
+	sz = struct_size(sas_iounit_pg0, PhyData, ioc->sas_hba.num_phys);
 	sas_iounit_pg0 = kzalloc(sz, GFP_KERNEL);
 	if (!sas_iounit_pg0) {
 		ioc_err(ioc, "failure at %s:%d/%s()!\n",
@@ -6754,8 +6749,7 @@ _scsih_sas_host_add(struct MPT3SAS_ADAPTER *ioc)
 	}
 
 	/* sas_iounit page 1 */
-	sz = offsetof(Mpi2SasIOUnitPage1_t, PhyData) + (ioc->sas_hba.num_phys *
-	    sizeof(Mpi2SasIOUnit1PhyData_t));
+	sz = struct_size(sas_iounit_pg1, PhyData, ioc->sas_hba.num_phys);
 	sas_iounit_pg1 = kzalloc(sz, GFP_KERNEL);
 	if (!sas_iounit_pg1) {
 		ioc_err(ioc, "failure at %s:%d/%s()!\n",
@@ -10376,8 +10370,8 @@ _scsih_scan_for_devices_after_reset(struct MPT3SAS_ADAPTER *ioc)
 	Mpi2ExpanderPage0_t expander_pg0;
 	Mpi2SasDevicePage0_t sas_device_pg0;
 	Mpi26PCIeDevicePage0_t pcie_device_pg0;
-	Mpi2RaidVolPage1_t *volume_pg1;
-	Mpi2RaidVolPage0_t *volume_pg0;
+	Mpi2RaidVolPage1_t volume_pg1;
+	Mpi2RaidVolPage0_t volume_pg0;
 	Mpi2RaidPhysDiskPage0_t pd_pg0;
 	Mpi2EventIrConfigElement_t element;
 	Mpi2ConfigReply_t mpi_reply;
@@ -10392,16 +10386,6 @@ _scsih_scan_for_devices_after_reset(struct MPT3SAS_ADAPTER *ioc)
 	u8 retry_count;
 	unsigned long flags;
 
-	volume_pg0 = kzalloc(sizeof(*volume_pg0), GFP_KERNEL);
-	if (!volume_pg0)
-		return;
-
-	volume_pg1 = kzalloc(sizeof(*volume_pg1), GFP_KERNEL);
-	if (!volume_pg1) {
-		kfree(volume_pg0);
-		return;
-	}
-
 	ioc_info(ioc, "scan devices: start\n");
 
 	_scsih_sas_host_refresh(ioc);
@@ -10511,7 +10495,7 @@ _scsih_scan_for_devices_after_reset(struct MPT3SAS_ADAPTER *ioc)
 	/* volumes */
 	handle = 0xFFFF;
 	while (!(mpt3sas_config_get_raid_volume_pg1(ioc, &mpi_reply,
-	    volume_pg1, MPI2_RAID_VOLUME_PGAD_FORM_GET_NEXT_HANDLE, handle))) {
+	    &volume_pg1, MPI2_RAID_VOLUME_PGAD_FORM_GET_NEXT_HANDLE, handle))) {
 		ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
 		    MPI2_IOCSTATUS_MASK;
 		if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
@@ -10519,15 +10503,15 @@ _scsih_scan_for_devices_after_reset(struct MPT3SAS_ADAPTER *ioc)
 				 ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
 			break;
 		}
-		handle = le16_to_cpu(volume_pg1->DevHandle);
+		handle = le16_to_cpu(volume_pg1.DevHandle);
 		spin_lock_irqsave(&ioc->raid_device_lock, flags);
 		raid_device = _scsih_raid_device_find_by_wwid(ioc,
-		    le64_to_cpu(volume_pg1->WWID));
+		    le64_to_cpu(volume_pg1.WWID));
 		spin_unlock_irqrestore(&ioc->raid_device_lock, flags);
 		if (raid_device)
 			continue;
 		if (mpt3sas_config_get_raid_volume_pg0(ioc, &mpi_reply,
-		    volume_pg0, MPI2_RAID_VOLUME_PGAD_FORM_HANDLE, handle,
+		    &volume_pg0, MPI2_RAID_VOLUME_PGAD_FORM_HANDLE, handle,
 		     sizeof(Mpi2RaidVolPage0_t)))
 			continue;
 		ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
@@ -10537,17 +10521,17 @@ _scsih_scan_for_devices_after_reset(struct MPT3SAS_ADAPTER *ioc)
 				 ioc_status, le32_to_cpu(mpi_reply.IOCLogInfo));
 			break;
 		}
-		if (volume_pg0->VolumeState == MPI2_RAID_VOL_STATE_OPTIMAL ||
-		    volume_pg0->VolumeState == MPI2_RAID_VOL_STATE_ONLINE ||
-		    volume_pg0->VolumeState == MPI2_RAID_VOL_STATE_DEGRADED) {
+		if (volume_pg0.VolumeState == MPI2_RAID_VOL_STATE_OPTIMAL ||
+		    volume_pg0.VolumeState == MPI2_RAID_VOL_STATE_ONLINE ||
+		    volume_pg0.VolumeState == MPI2_RAID_VOL_STATE_DEGRADED) {
 			memset(&element, 0, sizeof(Mpi2EventIrConfigElement_t));
 			element.ReasonCode = MPI2_EVENT_IR_CHANGE_RC_ADDED;
-			element.VolDevHandle = volume_pg1->DevHandle;
+			element.VolDevHandle = volume_pg1.DevHandle;
 			ioc_info(ioc, "\tBEFORE adding volume: handle (0x%04x)\n",
-				 volume_pg1->DevHandle);
+				 volume_pg1.DevHandle);
 			_scsih_sas_volume_add(ioc, &element);
 			ioc_info(ioc, "\tAFTER adding volume: handle (0x%04x)\n",
-				 volume_pg1->DevHandle);
+				 volume_pg1.DevHandle);
 		}
 	}
 
@@ -10636,9 +10620,6 @@ _scsih_scan_for_devices_after_reset(struct MPT3SAS_ADAPTER *ioc)
 			 handle, (u64)le64_to_cpu(pcie_device_pg0.WWID));
 	}
 
-	kfree(volume_pg0);
-	kfree(volume_pg1);
-
 	ioc_info(ioc, "\tpcie devices: pcie end devices complete\n");
 	ioc_info(ioc, "scan devices: complete\n");
 }
@@ -11350,6 +11331,7 @@ static void scsih_remove(struct pci_dev *pdev)
 	}
 
 	mpt3sas_base_detach(ioc);
+	mpt3sas_ctl_release(ioc);
 	spin_lock(&gioc_lock);
 	list_del(&ioc->list);
 	spin_unlock(&gioc_lock);
diff --git a/drivers/scsi/mpt3sas/mpt3sas_transport.c b/drivers/scsi/mpt3sas/mpt3sas_transport.c
index e8a4750f6ec4..421ea511b664 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_transport.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_transport.c
@@ -1792,8 +1792,7 @@ _transport_phy_enable(struct sas_phy *phy, int enable)
 	/* handle hba phys */
 
 	/* read sas_iounit page 0 */
-	sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) + (ioc->sas_hba.num_phys *
-	    sizeof(Mpi2SasIOUnit0PhyData_t));
+	sz = struct_size(sas_iounit_pg0, PhyData, ioc->sas_hba.num_phys);
 	sas_iounit_pg0 = kzalloc(sz, GFP_KERNEL);
 	if (!sas_iounit_pg0) {
 		ioc_err(ioc, "failure at %s:%d/%s()!\n",
@@ -1833,8 +1832,7 @@ _transport_phy_enable(struct sas_phy *phy, int enable)
 	}
 
 	/* read sas_iounit page 1 */
-	sz = offsetof(Mpi2SasIOUnitPage1_t, PhyData) + (ioc->sas_hba.num_phys *
-	    sizeof(Mpi2SasIOUnit1PhyData_t));
+	sz = struct_size(sas_iounit_pg1, PhyData, ioc->sas_hba.num_phys);
 	sas_iounit_pg1 = kzalloc(sz, GFP_KERNEL);
 	if (!sas_iounit_pg1) {
 		ioc_err(ioc, "failure at %s:%d/%s()!\n",
@@ -1944,8 +1942,7 @@ _transport_phy_speed(struct sas_phy *phy, struct sas_phy_linkrates *rates)
 	/* handle hba phys */
 
 	/* sas_iounit page 1 */
-	sz = offsetof(Mpi2SasIOUnitPage1_t, PhyData) + (ioc->sas_hba.num_phys *
-	    sizeof(Mpi2SasIOUnit1PhyData_t));
+	sz = struct_size(sas_iounit_pg1, PhyData, ioc->sas_hba.num_phys);
 	sas_iounit_pg1 = kzalloc(sz, GFP_KERNEL);
 	if (!sas_iounit_pg1) {
 		ioc_err(ioc, "failure at %s:%d/%s()!\n",
diff --git a/drivers/scsi/mpt3sas/mpt3sas_trigger_pages.h b/drivers/scsi/mpt3sas/mpt3sas_trigger_pages.h
index 5f3328f011a2..edb8fe709089 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_trigger_pages.h
+++ b/drivers/scsi/mpt3sas/mpt3sas_trigger_pages.h
@@ -20,12 +20,12 @@
 
 #define MPI2_CONFIG_EXTPAGETYPE_DRIVER_PERSISTENT_TRIGGER    (0xE0)
 #define MPI26_DRIVER_TRIGGER_PAGE0_PAGEVERSION               (0x01)
-typedef struct _MPI26_CONFIG_PAGE_DRIVER_TIGGER_0 {
+typedef struct _MPI26_CONFIG_PAGE_DRIVER_TRIGGER_0 {
 	MPI2_CONFIG_EXTENDED_PAGE_HEADER	Header;	/* 0x00  */
 	U16	TriggerFlags;		/* 0x08  */
 	U16	Reserved0xA;		/* 0x0A */
 	U32	Reserved0xC[61];	/* 0x0C */
-} _MPI26_CONFIG_PAGE_DRIVER_TIGGER_0, Mpi26DriverTriggerPage0_t;
+} _MPI26_CONFIG_PAGE_DRIVER_TRIGGER_0, Mpi26DriverTriggerPage0_t;
 
 /* Trigger Flags */
 #define  MPI26_DRIVER_TRIGGER0_FLAG_MASTER_TRIGGER_VALID       (0x0001)
@@ -34,61 +34,61 @@ typedef struct _MPI26_CONFIG_PAGE_DRIVER_TIGGER_0 {
 #define  MPI26_DRIVER_TRIGGER0_FLAG_LOGINFO_TRIGGER_VALID      (0x0008)
 
 #define MPI26_DRIVER_TRIGGER_PAGE1_PAGEVERSION               (0x01)
-typedef struct _MPI26_DRIVER_MASTER_TIGGER_ENTRY {
+typedef struct _MPI26_DRIVER_MASTER_TRIGGER_ENTRY {
 	U32	MasterTriggerFlags;
-} MPI26_DRIVER_MASTER_TIGGER_ENTRY;
+} MPI26_DRIVER_MASTER_TRIGGER_ENTRY;
 
 #define MPI26_MAX_MASTER_TRIGGERS                                   (1)
-typedef struct _MPI26_CONFIG_PAGE_DRIVER_TIGGER_1 {
+typedef struct _MPI26_CONFIG_PAGE_DRIVER_TRIGGER_1 {
 	MPI2_CONFIG_EXTENDED_PAGE_HEADER	Header;	/* 0x00 */
 	U16	NumMasterTrigger;	/* 0x08 */
 	U16	Reserved0xA;		/* 0x0A */
-	MPI26_DRIVER_MASTER_TIGGER_ENTRY MasterTriggers[MPI26_MAX_MASTER_TRIGGERS];	/* 0x0C */
-} MPI26_CONFIG_PAGE_DRIVER_TIGGER_1, Mpi26DriverTriggerPage1_t;
+	MPI26_DRIVER_MASTER_TRIGGER_ENTRY MasterTriggers[MPI26_MAX_MASTER_TRIGGERS];	/* 0x0C */
+} MPI26_CONFIG_PAGE_DRIVER_TRIGGER_1, Mpi26DriverTriggerPage1_t;
 
 #define MPI26_DRIVER_TRIGGER_PAGE2_PAGEVERSION               (0x01)
-typedef struct _MPI26_DRIVER_MPI_EVENT_TIGGER_ENTRY {
+typedef struct _MPI26_DRIVER_MPI_EVENT_TRIGGER_ENTRY {
 	U16	MPIEventCode;		/* 0x00 */
 	U16	MPIEventCodeSpecific;	/* 0x02 */
-} MPI26_DRIVER_MPI_EVENT_TIGGER_ENTRY;
+} MPI26_DRIVER_MPI_EVENT_TRIGGER_ENTRY;
 
 #define MPI26_MAX_MPI_EVENT_TRIGGERS                            (20)
-typedef struct _MPI26_CONFIG_PAGE_DRIVER_TIGGER_2 {
+typedef struct _MPI26_CONFIG_PAGE_DRIVER_TRIGGER_2 {
 	MPI2_CONFIG_EXTENDED_PAGE_HEADER        Header;	/* 0x00  */
 	U16	NumMPIEventTrigger;     /* 0x08  */
 	U16	Reserved0xA;		/* 0x0A */
-	MPI26_DRIVER_MPI_EVENT_TIGGER_ENTRY MPIEventTriggers[MPI26_MAX_MPI_EVENT_TRIGGERS]; /* 0x0C */
-} MPI26_CONFIG_PAGE_DRIVER_TIGGER_2, Mpi26DriverTriggerPage2_t;
+	MPI26_DRIVER_MPI_EVENT_TRIGGER_ENTRY MPIEventTriggers[MPI26_MAX_MPI_EVENT_TRIGGERS]; /* 0x0C */
+} MPI26_CONFIG_PAGE_DRIVER_TRIGGER_2, Mpi26DriverTriggerPage2_t;
 
 #define MPI26_DRIVER_TRIGGER_PAGE3_PAGEVERSION               (0x01)
-typedef struct _MPI26_DRIVER_SCSI_SENSE_TIGGER_ENTRY {
+typedef struct _MPI26_DRIVER_SCSI_SENSE_TRIGGER_ENTRY {
 	U8     ASCQ;		/* 0x00 */
 	U8     ASC;		/* 0x01 */
 	U8     SenseKey;	/* 0x02 */
 	U8     Reserved;	/* 0x03 */
-} MPI26_DRIVER_SCSI_SENSE_TIGGER_ENTRY;
+} MPI26_DRIVER_SCSI_SENSE_TRIGGER_ENTRY;
 
 #define MPI26_MAX_SCSI_SENSE_TRIGGERS                            (20)
-typedef struct _MPI26_CONFIG_PAGE_DRIVER_TIGGER_3 {
+typedef struct _MPI26_CONFIG_PAGE_DRIVER_TRIGGER_3 {
 	MPI2_CONFIG_EXTENDED_PAGE_HEADER	Header;	/* 0x00  */
 	U16	NumSCSISenseTrigger;			/* 0x08  */
 	U16	Reserved0xA;				/* 0x0A */
-	MPI26_DRIVER_SCSI_SENSE_TIGGER_ENTRY SCSISenseTriggers[MPI26_MAX_SCSI_SENSE_TRIGGERS];	/* 0x0C */
-} MPI26_CONFIG_PAGE_DRIVER_TIGGER_3, Mpi26DriverTriggerPage3_t;
+	MPI26_DRIVER_SCSI_SENSE_TRIGGER_ENTRY SCSISenseTriggers[MPI26_MAX_SCSI_SENSE_TRIGGERS];	/* 0x0C */
+} MPI26_CONFIG_PAGE_DRIVER_TRIGGER_3, Mpi26DriverTriggerPage3_t;
 
 #define MPI26_DRIVER_TRIGGER_PAGE4_PAGEVERSION               (0x01)
-typedef struct _MPI26_DRIVER_IOCSTATUS_LOGINFO_TIGGER_ENTRY {
+typedef struct _MPI26_DRIVER_IOCSTATUS_LOGINFO_TRIGGER_ENTRY {
 	U16        IOCStatus;      /* 0x00 */
 	U16        Reserved;       /* 0x02 */
 	U32        LogInfo;        /* 0x04 */
-} MPI26_DRIVER_IOCSTATUS_LOGINFO_TIGGER_ENTRY;
+} MPI26_DRIVER_IOCSTATUS_LOGINFO_TRIGGER_ENTRY;
 
 #define MPI26_MAX_LOGINFO_TRIGGERS                            (20)
-typedef struct _MPI26_CONFIG_PAGE_DRIVER_TIGGER_4 {
+typedef struct _MPI26_CONFIG_PAGE_DRIVER_TRIGGER_4 {
 	MPI2_CONFIG_EXTENDED_PAGE_HEADER	Header;	/* 0x00  */
 	U16	NumIOCStatusLogInfoTrigger;		/* 0x08  */
 	U16	Reserved0xA;				/* 0x0A */
-	MPI26_DRIVER_IOCSTATUS_LOGINFO_TIGGER_ENTRY IOCStatusLoginfoTriggers[MPI26_MAX_LOGINFO_TRIGGERS];	/* 0x0C */
-} MPI26_CONFIG_PAGE_DRIVER_TIGGER_4, Mpi26DriverTriggerPage4_t;
+	MPI26_DRIVER_IOCSTATUS_LOGINFO_TRIGGER_ENTRY IOCStatusLoginfoTriggers[MPI26_MAX_LOGINFO_TRIGGERS];	/* 0x0C */
+} MPI26_CONFIG_PAGE_DRIVER_TRIGGER_4, Mpi26DriverTriggerPage4_t;
 
 #endif
diff --git a/drivers/scsi/mpt3sas/mpt3sas_warpdrive.c b/drivers/scsi/mpt3sas/mpt3sas_warpdrive.c
index cc07ba41f507..1d64e5056a8a 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_warpdrive.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_warpdrive.c
@@ -141,8 +141,7 @@ mpt3sas_init_warpdrive_properties(struct MPT3SAS_ADAPTER *ioc,
 		return;
 	}
 
-	sz = offsetof(Mpi2RaidVolPage0_t, PhysDisk) + (num_pds *
-	    sizeof(Mpi2RaidVol0PhysDisk_t));
+	sz = struct_size(vol_pg0, PhysDisk, num_pds);
 	vol_pg0 = kzalloc(sz, GFP_KERNEL);
 	if (!vol_pg0) {
 		ioc_info(ioc, "WarpDrive : Direct IO is disabled Memory allocation failure for RVPG0\n");
diff --git a/drivers/soc/samsung/exynos-chipid.c b/drivers/soc/samsung/exynos-chipid.c
index 3fd0f2b84dd3..b1118d37779e 100644
--- a/drivers/soc/samsung/exynos-chipid.c
+++ b/drivers/soc/samsung/exynos-chipid.c
@@ -59,6 +59,7 @@ static const struct exynos_soc_id {
 	{ "EXYNOS7885", 0xE7885000 },
 	{ "EXYNOS850", 0xE3830000 },
 	{ "EXYNOSAUTOV9", 0xAAA80000 },
+	{ "EXYNOSAUTOV920", 0x0A920000 },
 };
 
 static const char *product_id_to_soc_id(unsigned int product_id)
diff --git a/drivers/soc/xilinx/xlnx_event_manager.c b/drivers/soc/xilinx/xlnx_event_manager.c
index 86a048a10a13..042553abe1bf 100644
--- a/drivers/soc/xilinx/xlnx_event_manager.c
+++ b/drivers/soc/xilinx/xlnx_event_manager.c
@@ -477,7 +477,7 @@ static void xlnx_call_notify_cb_handler(const u32 *payload)
 		}
 	}
 	if (!is_callback_found)
-		pr_warn("Didn't find any registered callback for 0x%x 0x%x\n",
+		pr_warn("Unhandled SGI node 0x%x event 0x%x. Expected with Xen hypervisor\n",
 			payload[1], payload[2]);
 }
 
@@ -555,7 +555,7 @@ static void xlnx_disable_percpu_irq(void *data)
 static int xlnx_event_init_sgi(struct platform_device *pdev)
 {
 	int ret = 0;
-	int cpu = smp_processor_id();
+	int cpu;
 	/*
 	 * IRQ related structures are used for the following:
 	 * for each SGI interrupt ensure its mapped by GIC IRQ domain
@@ -592,9 +592,12 @@ static int xlnx_event_init_sgi(struct platform_device *pdev)
 	sgi_fwspec.param[0] = sgi_num;
 	virq_sgi = irq_create_fwspec_mapping(&sgi_fwspec);
 
+	cpu = get_cpu();
 	per_cpu(cpu_number1, cpu) = cpu;
 	ret = request_percpu_irq(virq_sgi, xlnx_event_handler, "xlnx_event_mgmt",
 				 &cpu_number1);
+	put_cpu();
+
 	WARN_ON(ret);
 	if (ret) {
 		irq_dispose_mapping(virq_sgi);
diff --git a/drivers/soc/xilinx/zynqmp_power.c b/drivers/soc/xilinx/zynqmp_power.c
index c2c819701eec..abcf7e883279 100644
--- a/drivers/soc/xilinx/zynqmp_power.c
+++ b/drivers/soc/xilinx/zynqmp_power.c
@@ -83,8 +83,8 @@ static irqreturn_t zynqmp_pm_isr(int irq, void *data)
 			pm_suspend(PM_SUSPEND_MEM);
 			break;
 		default:
-			pr_err("%s Unsupported InitSuspendCb reason "
-				"code %d\n", __func__, payload[1]);
+			pr_err("%s Unsupported InitSuspendCb reason code %d\n",
+			       __func__, payload[1]);
 		}
 	}
 
@@ -252,8 +252,8 @@ static int zynqmp_pm_probe(struct platform_device *pdev)
 						dev_name(&pdev->dev),
 						&pdev->dev);
 		if (ret) {
-			dev_err(&pdev->dev, "devm_request_threaded_irq '%d' "
-					    "failed with %d\n", irq, ret);
+			dev_err(&pdev->dev, "devm_request_threaded_irq '%d' failed with %d\n",
+				irq, ret);
 			return ret;
 		}
 	} else {
@@ -275,7 +275,7 @@ static int zynqmp_pm_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int zynqmp_pm_remove(struct platform_device *pdev)
+static void zynqmp_pm_remove(struct platform_device *pdev)
 {
 	sysfs_remove_file(&pdev->dev.kobj, &dev_attr_suspend_mode.attr);
 	if (event_registered)
@@ -283,8 +283,6 @@ static int zynqmp_pm_remove(struct platform_device *pdev)
 
 	if (!rx_chan)
 		mbox_free_channel(rx_chan);
-
-	return 0;
 }
 
 static const struct of_device_id pm_of_match[] = {
@@ -295,7 +293,7 @@ MODULE_DEVICE_TABLE(of, pm_of_match);
 
 static struct platform_driver zynqmp_pm_platform_driver = {
 	.probe = zynqmp_pm_probe,
-	.remove = zynqmp_pm_remove,
+	.remove_new = zynqmp_pm_remove,
 	.driver = {
 		.name = "zynqmp_power",
 		.of_match_table = pm_of_match,
diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig
index 70c9dd6b6a31..ddae0fde798e 100644
--- a/drivers/spi/Kconfig
+++ b/drivers/spi/Kconfig
@@ -1177,9 +1177,10 @@ config SPI_ZYNQ_QSPI
 
 config SPI_ZYNQMP_GQSPI
 	tristate "Xilinx ZynqMP GQSPI controller"
-	depends on (SPI_MASTER && HAS_DMA) || COMPILE_TEST
+	depends on (SPI_MEM && HAS_DMA) || COMPILE_TEST
 	help
 	  Enables Xilinx GQSPI controller driver for Zynq UltraScale+ MPSoC.
+	  This controller only supports SPI memory interface.
 
 config SPI_AMD
 	tristate "AMD SPI controller"
diff --git a/drivers/spi/spi-axi-spi-engine.c b/drivers/spi/spi-axi-spi-engine.c
index b96e55f59d1a..cbca783830ea 100644
--- a/drivers/spi/spi-axi-spi-engine.c
+++ b/drivers/spi/spi-axi-spi-engine.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/clk.h>
+#include <linux/idr.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/of.h>
@@ -52,6 +53,7 @@
 
 #define SPI_ENGINE_CMD_REG_CLK_DIV		0x0
 #define SPI_ENGINE_CMD_REG_CONFIG		0x1
+#define SPI_ENGINE_CMD_REG_XFER_BITS		0x2
 
 #define SPI_ENGINE_MISC_SYNC			0x0
 #define SPI_ENGINE_MISC_SLEEP			0x1
@@ -78,29 +80,40 @@ struct spi_engine_program {
 	uint16_t instructions[];
 };
 
-struct spi_engine {
-	struct clk *clk;
-	struct clk *ref_clk;
-
-	spinlock_t lock;
-
-	void __iomem *base;
-
-	struct spi_message *msg;
+/**
+ * struct spi_engine_message_state - SPI engine per-message state
+ */
+struct spi_engine_message_state {
+	/** Instructions for executing this message. */
 	struct spi_engine_program *p;
+	/** Number of elements in cmd_buf array. */
 	unsigned cmd_length;
+	/** Array of commands not yet written to CMD FIFO. */
 	const uint16_t *cmd_buf;
-
+	/** Next xfer with tx_buf not yet fully written to TX FIFO. */
 	struct spi_transfer *tx_xfer;
+	/** Size of tx_buf in bytes. */
 	unsigned int tx_length;
+	/** Bytes not yet written to TX FIFO. */
 	const uint8_t *tx_buf;
-
+	/** Next xfer with rx_buf not yet fully written to RX FIFO. */
 	struct spi_transfer *rx_xfer;
+	/** Size of tx_buf in bytes. */
 	unsigned int rx_length;
+	/** Bytes not yet written to the RX FIFO. */
 	uint8_t *rx_buf;
+	/** ID to correlate SYNC interrupts with this message. */
+	u8 sync_id;
+};
 
-	unsigned int sync_id;
-	unsigned int completed_id;
+struct spi_engine {
+	struct clk *clk;
+	struct clk *ref_clk;
+
+	spinlock_t lock;
+
+	void __iomem *base;
+	struct ida sync_ida;
 
 	unsigned int int_enable;
 };
@@ -145,7 +158,14 @@ static unsigned int spi_engine_get_clk_div(struct spi_engine *spi_engine,
 static void spi_engine_gen_xfer(struct spi_engine_program *p, bool dry,
 	struct spi_transfer *xfer)
 {
-	unsigned int len = xfer->len;
+	unsigned int len;
+
+	if (xfer->bits_per_word <= 8)
+		len = xfer->len;
+	else if (xfer->bits_per_word <= 16)
+		len = xfer->len / 2;
+	else
+		len = xfer->len / 4;
 
 	while (len) {
 		unsigned int n = min(len, 256U);
@@ -204,7 +224,8 @@ static int spi_engine_compile_message(struct spi_engine *spi_engine,
 	struct spi_device *spi = msg->spi;
 	struct spi_transfer *xfer;
 	int clk_div, new_clk_div;
-	bool cs_change = true;
+	bool keep_cs = false;
+	u8 bits_per_word = 0;
 
 	clk_div = -1;
 
@@ -212,6 +233,9 @@ static int spi_engine_compile_message(struct spi_engine *spi_engine,
 		SPI_ENGINE_CMD_WRITE(SPI_ENGINE_CMD_REG_CONFIG,
 			spi_engine_get_config(spi)));
 
+	xfer = list_first_entry(&msg->transfers, struct spi_transfer, transfer_list);
+	spi_engine_gen_cs(p, dry, spi, !xfer->cs_off);
+
 	list_for_each_entry(xfer, &msg->transfers, transfer_list) {
 		new_clk_div = spi_engine_get_clk_div(spi_engine, spi, xfer);
 		if (new_clk_div != clk_div) {
@@ -221,27 +245,41 @@ static int spi_engine_compile_message(struct spi_engine *spi_engine,
 					clk_div));
 		}
 
-		if (cs_change)
-			spi_engine_gen_cs(p, dry, spi, true);
+		if (bits_per_word != xfer->bits_per_word) {
+			bits_per_word = xfer->bits_per_word;
+			spi_engine_program_add_cmd(p, dry,
+				SPI_ENGINE_CMD_WRITE(SPI_ENGINE_CMD_REG_XFER_BITS,
+					bits_per_word));
+		}
 
 		spi_engine_gen_xfer(p, dry, xfer);
 		spi_engine_gen_sleep(p, dry, spi_engine, clk_div, xfer);
 
-		cs_change = xfer->cs_change;
-		if (list_is_last(&xfer->transfer_list, &msg->transfers))
-			cs_change = !cs_change;
-
-		if (cs_change)
-			spi_engine_gen_cs(p, dry, spi, false);
+		if (xfer->cs_change) {
+			if (list_is_last(&xfer->transfer_list, &msg->transfers)) {
+				keep_cs = true;
+			} else {
+				if (!xfer->cs_off)
+					spi_engine_gen_cs(p, dry, spi, false);
+
+				if (!list_next_entry(xfer, transfer_list)->cs_off)
+					spi_engine_gen_cs(p, dry, spi, true);
+			}
+		} else if (!list_is_last(&xfer->transfer_list, &msg->transfers) &&
+			   xfer->cs_off != list_next_entry(xfer, transfer_list)->cs_off) {
+			spi_engine_gen_cs(p, dry, spi, xfer->cs_off);
+		}
 	}
 
+	if (!keep_cs)
+		spi_engine_gen_cs(p, dry, spi, false);
+
 	return 0;
 }
 
-static void spi_engine_xfer_next(struct spi_engine *spi_engine,
+static void spi_engine_xfer_next(struct spi_message *msg,
 	struct spi_transfer **_xfer)
 {
-	struct spi_message *msg = spi_engine->msg;
 	struct spi_transfer *xfer = *_xfer;
 
 	if (!xfer) {
@@ -256,146 +294,189 @@ static void spi_engine_xfer_next(struct spi_engine *spi_engine,
 	*_xfer = xfer;
 }
 
-static void spi_engine_tx_next(struct spi_engine *spi_engine)
+static void spi_engine_tx_next(struct spi_message *msg)
 {
-	struct spi_transfer *xfer = spi_engine->tx_xfer;
+	struct spi_engine_message_state *st = msg->state;
+	struct spi_transfer *xfer = st->tx_xfer;
 
 	do {
-		spi_engine_xfer_next(spi_engine, &xfer);
+		spi_engine_xfer_next(msg, &xfer);
 	} while (xfer && !xfer->tx_buf);
 
-	spi_engine->tx_xfer = xfer;
+	st->tx_xfer = xfer;
 	if (xfer) {
-		spi_engine->tx_length = xfer->len;
-		spi_engine->tx_buf = xfer->tx_buf;
+		st->tx_length = xfer->len;
+		st->tx_buf = xfer->tx_buf;
 	} else {
-		spi_engine->tx_buf = NULL;
+		st->tx_buf = NULL;
 	}
 }
 
-static void spi_engine_rx_next(struct spi_engine *spi_engine)
+static void spi_engine_rx_next(struct spi_message *msg)
 {
-	struct spi_transfer *xfer = spi_engine->rx_xfer;
+	struct spi_engine_message_state *st = msg->state;
+	struct spi_transfer *xfer = st->rx_xfer;
 
 	do {
-		spi_engine_xfer_next(spi_engine, &xfer);
+		spi_engine_xfer_next(msg, &xfer);
 	} while (xfer && !xfer->rx_buf);
 
-	spi_engine->rx_xfer = xfer;
+	st->rx_xfer = xfer;
 	if (xfer) {
-		spi_engine->rx_length = xfer->len;
-		spi_engine->rx_buf = xfer->rx_buf;
+		st->rx_length = xfer->len;
+		st->rx_buf = xfer->rx_buf;
 	} else {
-		spi_engine->rx_buf = NULL;
+		st->rx_buf = NULL;
 	}
 }
 
-static bool spi_engine_write_cmd_fifo(struct spi_engine *spi_engine)
+static bool spi_engine_write_cmd_fifo(struct spi_engine *spi_engine,
+				      struct spi_message *msg)
 {
 	void __iomem *addr = spi_engine->base + SPI_ENGINE_REG_CMD_FIFO;
+	struct spi_engine_message_state *st = msg->state;
 	unsigned int n, m, i;
 	const uint16_t *buf;
 
 	n = readl_relaxed(spi_engine->base + SPI_ENGINE_REG_CMD_FIFO_ROOM);
-	while (n && spi_engine->cmd_length) {
-		m = min(n, spi_engine->cmd_length);
-		buf = spi_engine->cmd_buf;
+	while (n && st->cmd_length) {
+		m = min(n, st->cmd_length);
+		buf = st->cmd_buf;
 		for (i = 0; i < m; i++)
 			writel_relaxed(buf[i], addr);
-		spi_engine->cmd_buf += m;
-		spi_engine->cmd_length -= m;
+		st->cmd_buf += m;
+		st->cmd_length -= m;
 		n -= m;
 	}
 
-	return spi_engine->cmd_length != 0;
+	return st->cmd_length != 0;
 }
 
-static bool spi_engine_write_tx_fifo(struct spi_engine *spi_engine)
+static bool spi_engine_write_tx_fifo(struct spi_engine *spi_engine,
+				     struct spi_message *msg)
 {
 	void __iomem *addr = spi_engine->base + SPI_ENGINE_REG_SDO_DATA_FIFO;
+	struct spi_engine_message_state *st = msg->state;
 	unsigned int n, m, i;
-	const uint8_t *buf;
 
 	n = readl_relaxed(spi_engine->base + SPI_ENGINE_REG_SDO_FIFO_ROOM);
-	while (n && spi_engine->tx_length) {
-		m = min(n, spi_engine->tx_length);
-		buf = spi_engine->tx_buf;
-		for (i = 0; i < m; i++)
-			writel_relaxed(buf[i], addr);
-		spi_engine->tx_buf += m;
-		spi_engine->tx_length -= m;
+	while (n && st->tx_length) {
+		if (st->tx_xfer->bits_per_word <= 8) {
+			const u8 *buf = st->tx_buf;
+
+			m = min(n, st->tx_length);
+			for (i = 0; i < m; i++)
+				writel_relaxed(buf[i], addr);
+			st->tx_buf += m;
+			st->tx_length -= m;
+		} else if (st->tx_xfer->bits_per_word <= 16) {
+			const u16 *buf = (const u16 *)st->tx_buf;
+
+			m = min(n, st->tx_length / 2);
+			for (i = 0; i < m; i++)
+				writel_relaxed(buf[i], addr);
+			st->tx_buf += m * 2;
+			st->tx_length -= m * 2;
+		} else {
+			const u32 *buf = (const u32 *)st->tx_buf;
+
+			m = min(n, st->tx_length / 4);
+			for (i = 0; i < m; i++)
+				writel_relaxed(buf[i], addr);
+			st->tx_buf += m * 4;
+			st->tx_length -= m * 4;
+		}
 		n -= m;
-		if (spi_engine->tx_length == 0)
-			spi_engine_tx_next(spi_engine);
+		if (st->tx_length == 0)
+			spi_engine_tx_next(msg);
 	}
 
-	return spi_engine->tx_length != 0;
+	return st->tx_length != 0;
 }
 
-static bool spi_engine_read_rx_fifo(struct spi_engine *spi_engine)
+static bool spi_engine_read_rx_fifo(struct spi_engine *spi_engine,
+				    struct spi_message *msg)
 {
 	void __iomem *addr = spi_engine->base + SPI_ENGINE_REG_SDI_DATA_FIFO;
+	struct spi_engine_message_state *st = msg->state;
 	unsigned int n, m, i;
-	uint8_t *buf;
 
 	n = readl_relaxed(spi_engine->base + SPI_ENGINE_REG_SDI_FIFO_LEVEL);
-	while (n && spi_engine->rx_length) {
-		m = min(n, spi_engine->rx_length);
-		buf = spi_engine->rx_buf;
-		for (i = 0; i < m; i++)
-			buf[i] = readl_relaxed(addr);
-		spi_engine->rx_buf += m;
-		spi_engine->rx_length -= m;
+	while (n && st->rx_length) {
+		if (st->rx_xfer->bits_per_word <= 8) {
+			u8 *buf = st->rx_buf;
+
+			m = min(n, st->rx_length);
+			for (i = 0; i < m; i++)
+				buf[i] = readl_relaxed(addr);
+			st->rx_buf += m;
+			st->rx_length -= m;
+		} else if (st->rx_xfer->bits_per_word <= 16) {
+			u16 *buf = (u16 *)st->rx_buf;
+
+			m = min(n, st->rx_length / 2);
+			for (i = 0; i < m; i++)
+				buf[i] = readl_relaxed(addr);
+			st->rx_buf += m * 2;
+			st->rx_length -= m * 2;
+		} else {
+			u32 *buf = (u32 *)st->rx_buf;
+
+			m = min(n, st->rx_length / 4);
+			for (i = 0; i < m; i++)
+				buf[i] = readl_relaxed(addr);
+			st->rx_buf += m * 4;
+			st->rx_length -= m * 4;
+		}
 		n -= m;
-		if (spi_engine->rx_length == 0)
-			spi_engine_rx_next(spi_engine);
+		if (st->rx_length == 0)
+			spi_engine_rx_next(msg);
 	}
 
-	return spi_engine->rx_length != 0;
+	return st->rx_length != 0;
 }
 
 static irqreturn_t spi_engine_irq(int irq, void *devid)
 {
 	struct spi_controller *host = devid;
+	struct spi_message *msg = host->cur_msg;
 	struct spi_engine *spi_engine = spi_controller_get_devdata(host);
 	unsigned int disable_int = 0;
 	unsigned int pending;
+	int completed_id = -1;
 
 	pending = readl_relaxed(spi_engine->base + SPI_ENGINE_REG_INT_PENDING);
 
 	if (pending & SPI_ENGINE_INT_SYNC) {
 		writel_relaxed(SPI_ENGINE_INT_SYNC,
 			spi_engine->base + SPI_ENGINE_REG_INT_PENDING);
-		spi_engine->completed_id = readl_relaxed(
+		completed_id = readl_relaxed(
 			spi_engine->base + SPI_ENGINE_REG_SYNC_ID);
 	}
 
 	spin_lock(&spi_engine->lock);
 
 	if (pending & SPI_ENGINE_INT_CMD_ALMOST_EMPTY) {
-		if (!spi_engine_write_cmd_fifo(spi_engine))
+		if (!spi_engine_write_cmd_fifo(spi_engine, msg))
 			disable_int |= SPI_ENGINE_INT_CMD_ALMOST_EMPTY;
 	}
 
 	if (pending & SPI_ENGINE_INT_SDO_ALMOST_EMPTY) {
-		if (!spi_engine_write_tx_fifo(spi_engine))
+		if (!spi_engine_write_tx_fifo(spi_engine, msg))
 			disable_int |= SPI_ENGINE_INT_SDO_ALMOST_EMPTY;
 	}
 
 	if (pending & (SPI_ENGINE_INT_SDI_ALMOST_FULL | SPI_ENGINE_INT_SYNC)) {
-		if (!spi_engine_read_rx_fifo(spi_engine))
+		if (!spi_engine_read_rx_fifo(spi_engine, msg))
 			disable_int |= SPI_ENGINE_INT_SDI_ALMOST_FULL;
 	}
 
-	if (pending & SPI_ENGINE_INT_SYNC) {
-		if (spi_engine->msg &&
-		    spi_engine->completed_id == spi_engine->sync_id) {
-			struct spi_message *msg = spi_engine->msg;
+	if (pending & SPI_ENGINE_INT_SYNC && msg) {
+		struct spi_engine_message_state *st = msg->state;
 
-			kfree(spi_engine->p);
+		if (completed_id == st->sync_id) {
 			msg->status = 0;
 			msg->actual_length = msg->frame_length;
-			spi_engine->msg = NULL;
 			spi_finalize_current_message(host);
 			disable_int |= SPI_ENGINE_INT_SYNC;
 		}
@@ -412,43 +493,82 @@ static irqreturn_t spi_engine_irq(int irq, void *devid)
 	return IRQ_HANDLED;
 }
 
-static int spi_engine_transfer_one_message(struct spi_controller *host,
-	struct spi_message *msg)
+static int spi_engine_prepare_message(struct spi_controller *host,
+				      struct spi_message *msg)
 {
 	struct spi_engine_program p_dry, *p;
 	struct spi_engine *spi_engine = spi_controller_get_devdata(host);
-	unsigned int int_enable = 0;
-	unsigned long flags;
+	struct spi_engine_message_state *st;
 	size_t size;
+	int ret;
+
+	st = kzalloc(sizeof(*st), GFP_KERNEL);
+	if (!st)
+		return -ENOMEM;
 
 	p_dry.length = 0;
 	spi_engine_compile_message(spi_engine, msg, true, &p_dry);
 
 	size = sizeof(*p->instructions) * (p_dry.length + 1);
 	p = kzalloc(sizeof(*p) + size, GFP_KERNEL);
-	if (!p)
+	if (!p) {
+		kfree(st);
 		return -ENOMEM;
+	}
+
+	ret = ida_alloc_range(&spi_engine->sync_ida, 0, U8_MAX, GFP_KERNEL);
+	if (ret < 0) {
+		kfree(p);
+		kfree(st);
+		return ret;
+	}
+
+	st->sync_id = ret;
+
 	spi_engine_compile_message(spi_engine, msg, false, p);
 
-	spin_lock_irqsave(&spi_engine->lock, flags);
-	spi_engine->sync_id = (spi_engine->sync_id + 1) & 0xff;
-	spi_engine_program_add_cmd(p, false,
-		SPI_ENGINE_CMD_SYNC(spi_engine->sync_id));
+	spi_engine_program_add_cmd(p, false, SPI_ENGINE_CMD_SYNC(st->sync_id));
 
-	spi_engine->msg = msg;
-	spi_engine->p = p;
+	st->p = p;
+	st->cmd_buf = p->instructions;
+	st->cmd_length = p->length;
+	msg->state = st;
 
-	spi_engine->cmd_buf = p->instructions;
-	spi_engine->cmd_length = p->length;
-	if (spi_engine_write_cmd_fifo(spi_engine))
+	return 0;
+}
+
+static int spi_engine_unprepare_message(struct spi_controller *host,
+					struct spi_message *msg)
+{
+	struct spi_engine *spi_engine = spi_controller_get_devdata(host);
+	struct spi_engine_message_state *st = msg->state;
+
+	ida_free(&spi_engine->sync_ida, st->sync_id);
+	kfree(st->p);
+	kfree(st);
+
+	return 0;
+}
+
+static int spi_engine_transfer_one_message(struct spi_controller *host,
+	struct spi_message *msg)
+{
+	struct spi_engine *spi_engine = spi_controller_get_devdata(host);
+	struct spi_engine_message_state *st = msg->state;
+	unsigned int int_enable = 0;
+	unsigned long flags;
+
+	spin_lock_irqsave(&spi_engine->lock, flags);
+
+	if (spi_engine_write_cmd_fifo(spi_engine, msg))
 		int_enable |= SPI_ENGINE_INT_CMD_ALMOST_EMPTY;
 
-	spi_engine_tx_next(spi_engine);
-	if (spi_engine_write_tx_fifo(spi_engine))
+	spi_engine_tx_next(msg);
+	if (spi_engine_write_tx_fifo(spi_engine, msg))
 		int_enable |= SPI_ENGINE_INT_SDO_ALMOST_EMPTY;
 
-	spi_engine_rx_next(spi_engine);
-	if (spi_engine->rx_length != 0)
+	spi_engine_rx_next(msg);
+	if (st->rx_length != 0)
 		int_enable |= SPI_ENGINE_INT_SDI_ALMOST_FULL;
 
 	int_enable |= SPI_ENGINE_INT_SYNC;
@@ -461,6 +581,15 @@ static int spi_engine_transfer_one_message(struct spi_controller *host,
 	return 0;
 }
 
+static void spi_engine_release_hw(void *p)
+{
+	struct spi_engine *spi_engine = p;
+
+	writel_relaxed(0xff, spi_engine->base + SPI_ENGINE_REG_INT_PENDING);
+	writel_relaxed(0x00, spi_engine->base + SPI_ENGINE_REG_INT_ENABLE);
+	writel_relaxed(0x01, spi_engine->base + SPI_ENGINE_REG_RESET);
+}
+
 static int spi_engine_probe(struct platform_device *pdev)
 {
 	struct spi_engine *spi_engine;
@@ -473,35 +602,26 @@ static int spi_engine_probe(struct platform_device *pdev)
 	if (irq < 0)
 		return irq;
 
-	spi_engine = devm_kzalloc(&pdev->dev, sizeof(*spi_engine), GFP_KERNEL);
-	if (!spi_engine)
-		return -ENOMEM;
-
-	host = spi_alloc_host(&pdev->dev, 0);
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*spi_engine));
 	if (!host)
 		return -ENOMEM;
 
-	spi_controller_set_devdata(host, spi_engine);
+	spi_engine = spi_controller_get_devdata(host);
 
 	spin_lock_init(&spi_engine->lock);
+	ida_init(&spi_engine->sync_ida);
 
 	spi_engine->clk = devm_clk_get_enabled(&pdev->dev, "s_axi_aclk");
-	if (IS_ERR(spi_engine->clk)) {
-		ret = PTR_ERR(spi_engine->clk);
-		goto err_put_host;
-	}
+	if (IS_ERR(spi_engine->clk))
+		return PTR_ERR(spi_engine->clk);
 
 	spi_engine->ref_clk = devm_clk_get_enabled(&pdev->dev, "spi_clk");
-	if (IS_ERR(spi_engine->ref_clk)) {
-		ret = PTR_ERR(spi_engine->ref_clk);
-		goto err_put_host;
-	}
+	if (IS_ERR(spi_engine->ref_clk))
+		return PTR_ERR(spi_engine->ref_clk);
 
 	spi_engine->base = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(spi_engine->base)) {
-		ret = PTR_ERR(spi_engine->base);
-		goto err_put_host;
-	}
+	if (IS_ERR(spi_engine->base))
+		return PTR_ERR(spi_engine->base);
 
 	version = readl(spi_engine->base + SPI_ENGINE_REG_VERSION);
 	if (SPI_ENGINE_VERSION_MAJOR(version) != 1) {
@@ -509,54 +629,42 @@ static int spi_engine_probe(struct platform_device *pdev)
 			SPI_ENGINE_VERSION_MAJOR(version),
 			SPI_ENGINE_VERSION_MINOR(version),
 			SPI_ENGINE_VERSION_PATCH(version));
-		ret = -ENODEV;
-		goto err_put_host;
+		return -ENODEV;
 	}
 
 	writel_relaxed(0x00, spi_engine->base + SPI_ENGINE_REG_RESET);
 	writel_relaxed(0xff, spi_engine->base + SPI_ENGINE_REG_INT_PENDING);
 	writel_relaxed(0x00, spi_engine->base + SPI_ENGINE_REG_INT_ENABLE);
 
-	ret = request_irq(irq, spi_engine_irq, 0, pdev->name, host);
+	ret = devm_add_action_or_reset(&pdev->dev, spi_engine_release_hw,
+				       spi_engine);
 	if (ret)
-		goto err_put_host;
+		return ret;
+
+	ret = devm_request_irq(&pdev->dev, irq, spi_engine_irq, 0, pdev->name,
+			       host);
+	if (ret)
+		return ret;
 
 	host->dev.of_node = pdev->dev.of_node;
 	host->mode_bits = SPI_CPOL | SPI_CPHA | SPI_3WIRE;
-	host->bits_per_word_mask = SPI_BPW_MASK(8);
+	host->bits_per_word_mask = SPI_BPW_RANGE_MASK(1, 32);
 	host->max_speed_hz = clk_get_rate(spi_engine->ref_clk) / 2;
 	host->transfer_one_message = spi_engine_transfer_one_message;
+	host->prepare_message = spi_engine_prepare_message;
+	host->unprepare_message = spi_engine_unprepare_message;
 	host->num_chipselect = 8;
 
-	ret = spi_register_controller(host);
+	if (host->max_speed_hz == 0)
+		return dev_err_probe(&pdev->dev, -EINVAL, "spi_clk rate is 0");
+
+	ret = devm_spi_register_controller(&pdev->dev, host);
 	if (ret)
-		goto err_free_irq;
+		return ret;
 
 	platform_set_drvdata(pdev, host);
 
 	return 0;
-err_free_irq:
-	free_irq(irq, host);
-err_put_host:
-	spi_controller_put(host);
-	return ret;
-}
-
-static void spi_engine_remove(struct platform_device *pdev)
-{
-	struct spi_controller *host = spi_controller_get(platform_get_drvdata(pdev));
-	struct spi_engine *spi_engine = spi_controller_get_devdata(host);
-	int irq = platform_get_irq(pdev, 0);
-
-	spi_unregister_controller(host);
-
-	free_irq(irq, host);
-
-	spi_controller_put(host);
-
-	writel_relaxed(0xff, spi_engine->base + SPI_ENGINE_REG_INT_PENDING);
-	writel_relaxed(0x00, spi_engine->base + SPI_ENGINE_REG_INT_ENABLE);
-	writel_relaxed(0x01, spi_engine->base + SPI_ENGINE_REG_RESET);
 }
 
 static const struct of_device_id spi_engine_match_table[] = {
@@ -567,7 +675,6 @@ MODULE_DEVICE_TABLE(of, spi_engine_match_table);
 
 static struct platform_driver spi_engine_driver = {
 	.probe = spi_engine_probe,
-	.remove_new = spi_engine_remove,
 	.driver = {
 		.name = "spi-engine",
 		.of_match_table = spi_engine_match_table,
diff --git a/drivers/spi/spi-cadence-xspi.c b/drivers/spi/spi-cadence-xspi.c
index b7e04b03be58..8648b8eb080d 100644
--- a/drivers/spi/spi-cadence-xspi.c
+++ b/drivers/spi/spi-cadence-xspi.c
@@ -619,7 +619,6 @@ MODULE_DEVICE_TABLE(of, cdns_xspi_of_match);
 
 static struct platform_driver cdns_xspi_platform_driver = {
 	.probe          = cdns_xspi_probe,
-	.remove         = NULL,
 	.driver = {
 		.name = CDNS_XSPI_NAME,
 		.of_match_table = cdns_xspi_of_match,
diff --git a/drivers/spi/spi-ingenic.c b/drivers/spi/spi-ingenic.c
index cc366936d72b..003a6d21c4c3 100644
--- a/drivers/spi/spi-ingenic.c
+++ b/drivers/spi/spi-ingenic.c
@@ -346,14 +346,17 @@ static bool spi_ingenic_can_dma(struct spi_controller *ctlr,
 static int spi_ingenic_request_dma(struct spi_controller *ctlr,
 				   struct device *dev)
 {
-	ctlr->dma_tx = dma_request_slave_channel(dev, "tx");
-	if (!ctlr->dma_tx)
-		return -ENODEV;
+	struct dma_chan *chan;
 
-	ctlr->dma_rx = dma_request_slave_channel(dev, "rx");
+	chan = dma_request_chan(dev, "tx");
+	if (IS_ERR(chan))
+		return PTR_ERR(chan);
+	ctlr->dma_tx = chan;
 
-	if (!ctlr->dma_rx)
-		return -ENODEV;
+	chan = dma_request_chan(dev, "rx");
+	if (IS_ERR(chan))
+		return PTR_ERR(chan);
+	ctlr->dma_rx = chan;
 
 	ctlr->can_dma = spi_ingenic_can_dma;
 
diff --git a/drivers/spi/spi-intel.c b/drivers/spi/spi-intel.c
index 98ec4dc22b81..3654ae35d2db 100644
--- a/drivers/spi/spi-intel.c
+++ b/drivers/spi/spi-intel.c
@@ -711,8 +711,7 @@ static bool intel_spi_cmp_mem_op(const struct intel_spi_mem_op *iop,
 {
 	if (iop->mem_op.cmd.nbytes != op->cmd.nbytes ||
 	    iop->mem_op.cmd.buswidth != op->cmd.buswidth ||
-	    iop->mem_op.cmd.dtr != op->cmd.dtr ||
-	    iop->mem_op.cmd.opcode != op->cmd.opcode)
+	    iop->mem_op.cmd.dtr != op->cmd.dtr)
 		return false;
 
 	if (iop->mem_op.addr.nbytes != op->addr.nbytes ||
@@ -737,11 +736,12 @@ intel_spi_match_mem_op(struct intel_spi *ispi, const struct spi_mem_op *op)
 	const struct intel_spi_mem_op *iop;
 
 	for (iop = ispi->mem_ops; iop->mem_op.cmd.opcode; iop++) {
-		if (intel_spi_cmp_mem_op(iop, op))
-			break;
+		if (iop->mem_op.cmd.opcode == op->cmd.opcode &&
+		    intel_spi_cmp_mem_op(iop, op))
+			return iop;
 	}
 
-	return iop->mem_op.cmd.opcode ? iop : NULL;
+	return NULL;
 }
 
 static bool intel_spi_supports_mem_op(struct spi_mem *mem,
diff --git a/drivers/spi/spi-stm32.c b/drivers/spi/spi-stm32.c
index e6e3e4ea29f9..94df3836834c 100644
--- a/drivers/spi/spi-stm32.c
+++ b/drivers/spi/spi-stm32.c
@@ -22,58 +22,65 @@
 
 #define DRIVER_NAME "spi_stm32"
 
-/* STM32F4 SPI registers */
-#define STM32F4_SPI_CR1			0x00
-#define STM32F4_SPI_CR2			0x04
-#define STM32F4_SPI_SR			0x08
-#define STM32F4_SPI_DR			0x0C
-#define STM32F4_SPI_I2SCFGR		0x1C
-
-/* STM32F4_SPI_CR1 bit fields */
-#define STM32F4_SPI_CR1_CPHA		BIT(0)
-#define STM32F4_SPI_CR1_CPOL		BIT(1)
-#define STM32F4_SPI_CR1_MSTR		BIT(2)
-#define STM32F4_SPI_CR1_BR_SHIFT	3
-#define STM32F4_SPI_CR1_BR		GENMASK(5, 3)
-#define STM32F4_SPI_CR1_SPE		BIT(6)
-#define STM32F4_SPI_CR1_LSBFRST		BIT(7)
-#define STM32F4_SPI_CR1_SSI		BIT(8)
-#define STM32F4_SPI_CR1_SSM		BIT(9)
-#define STM32F4_SPI_CR1_RXONLY		BIT(10)
+/* STM32F4/7 SPI registers */
+#define STM32FX_SPI_CR1			0x00
+#define STM32FX_SPI_CR2			0x04
+#define STM32FX_SPI_SR			0x08
+#define STM32FX_SPI_DR			0x0C
+#define STM32FX_SPI_I2SCFGR		0x1C
+
+/* STM32FX_SPI_CR1 bit fields */
+#define STM32FX_SPI_CR1_CPHA		BIT(0)
+#define STM32FX_SPI_CR1_CPOL		BIT(1)
+#define STM32FX_SPI_CR1_MSTR		BIT(2)
+#define STM32FX_SPI_CR1_BR_SHIFT	3
+#define STM32FX_SPI_CR1_BR		GENMASK(5, 3)
+#define STM32FX_SPI_CR1_SPE		BIT(6)
+#define STM32FX_SPI_CR1_LSBFRST		BIT(7)
+#define STM32FX_SPI_CR1_SSI		BIT(8)
+#define STM32FX_SPI_CR1_SSM		BIT(9)
+#define STM32FX_SPI_CR1_RXONLY		BIT(10)
 #define STM32F4_SPI_CR1_DFF		BIT(11)
-#define STM32F4_SPI_CR1_CRCNEXT		BIT(12)
-#define STM32F4_SPI_CR1_CRCEN		BIT(13)
-#define STM32F4_SPI_CR1_BIDIOE		BIT(14)
-#define STM32F4_SPI_CR1_BIDIMODE	BIT(15)
-#define STM32F4_SPI_CR1_BR_MIN		0
-#define STM32F4_SPI_CR1_BR_MAX		(GENMASK(5, 3) >> 3)
-
-/* STM32F4_SPI_CR2 bit fields */
-#define STM32F4_SPI_CR2_RXDMAEN		BIT(0)
-#define STM32F4_SPI_CR2_TXDMAEN		BIT(1)
-#define STM32F4_SPI_CR2_SSOE		BIT(2)
-#define STM32F4_SPI_CR2_FRF		BIT(4)
-#define STM32F4_SPI_CR2_ERRIE		BIT(5)
-#define STM32F4_SPI_CR2_RXNEIE		BIT(6)
-#define STM32F4_SPI_CR2_TXEIE		BIT(7)
-
-/* STM32F4_SPI_SR bit fields */
-#define STM32F4_SPI_SR_RXNE		BIT(0)
-#define STM32F4_SPI_SR_TXE		BIT(1)
-#define STM32F4_SPI_SR_CHSIDE		BIT(2)
-#define STM32F4_SPI_SR_UDR		BIT(3)
-#define STM32F4_SPI_SR_CRCERR		BIT(4)
-#define STM32F4_SPI_SR_MODF		BIT(5)
-#define STM32F4_SPI_SR_OVR		BIT(6)
-#define STM32F4_SPI_SR_BSY		BIT(7)
-#define STM32F4_SPI_SR_FRE		BIT(8)
-
-/* STM32F4_SPI_I2SCFGR bit fields */
-#define STM32F4_SPI_I2SCFGR_I2SMOD	BIT(11)
+#define STM32F7_SPI_CR1_CRCL		BIT(11)
+#define STM32FX_SPI_CR1_CRCNEXT		BIT(12)
+#define STM32FX_SPI_CR1_CRCEN		BIT(13)
+#define STM32FX_SPI_CR1_BIDIOE		BIT(14)
+#define STM32FX_SPI_CR1_BIDIMODE	BIT(15)
+#define STM32FX_SPI_CR1_BR_MIN		0
+#define STM32FX_SPI_CR1_BR_MAX		(GENMASK(5, 3) >> 3)
+
+/* STM32FX_SPI_CR2 bit fields */
+#define STM32FX_SPI_CR2_RXDMAEN		BIT(0)
+#define STM32FX_SPI_CR2_TXDMAEN		BIT(1)
+#define STM32FX_SPI_CR2_SSOE		BIT(2)
+#define STM32FX_SPI_CR2_FRF		BIT(4)
+#define STM32FX_SPI_CR2_ERRIE		BIT(5)
+#define STM32FX_SPI_CR2_RXNEIE		BIT(6)
+#define STM32FX_SPI_CR2_TXEIE		BIT(7)
+#define STM32F7_SPI_CR2_DS		GENMASK(11, 8)
+#define STM32F7_SPI_CR2_FRXTH		BIT(12)
+#define STM32F7_SPI_CR2_LDMA_RX		BIT(13)
+#define STM32F7_SPI_CR2_LDMA_TX		BIT(14)
+
+/* STM32FX_SPI_SR bit fields */
+#define STM32FX_SPI_SR_RXNE		BIT(0)
+#define STM32FX_SPI_SR_TXE		BIT(1)
+#define STM32FX_SPI_SR_CHSIDE		BIT(2)
+#define STM32FX_SPI_SR_UDR		BIT(3)
+#define STM32FX_SPI_SR_CRCERR		BIT(4)
+#define STM32FX_SPI_SR_MODF		BIT(5)
+#define STM32FX_SPI_SR_OVR		BIT(6)
+#define STM32FX_SPI_SR_BSY		BIT(7)
+#define STM32FX_SPI_SR_FRE		BIT(8)
+#define STM32F7_SPI_SR_FRLVL		GENMASK(10, 9)
+#define STM32F7_SPI_SR_FTLVL		GENMASK(12, 11)
+
+/* STM32FX_SPI_I2SCFGR bit fields */
+#define STM32FX_SPI_I2SCFGR_I2SMOD	BIT(11)
 
 /* STM32F4 SPI Baud Rate min/max divisor */
-#define STM32F4_SPI_BR_DIV_MIN		(2 << STM32F4_SPI_CR1_BR_MIN)
-#define STM32F4_SPI_BR_DIV_MAX		(2 << STM32F4_SPI_CR1_BR_MAX)
+#define STM32FX_SPI_BR_DIV_MIN		(2 << STM32FX_SPI_CR1_BR_MIN)
+#define STM32FX_SPI_BR_DIV_MAX		(2 << STM32FX_SPI_CR1_BR_MAX)
 
 /* STM32H7 SPI registers */
 #define STM32H7_SPI_CR1			0x00
@@ -229,6 +236,8 @@ struct stm32_spi;
  * time between frames (if driver has this functionality)
  * @set_number_of_data: optional routine to configure registers to desired
  * number of data (if driver has this functionality)
+ * @write_tx: routine to write to transmit register/FIFO
+ * @read_rx: routine to read from receive register/FIFO
  * @transfer_one_dma_start: routine to start transfer a single spi_transfer
  * using DMA
  * @dma_rx_cb: routine to call after DMA RX channel operation is complete
@@ -252,6 +261,8 @@ struct stm32_spi_cfg {
 	int (*set_mode)(struct stm32_spi *spi, unsigned int comm_type);
 	void (*set_data_idleness)(struct stm32_spi *spi, u32 length);
 	int (*set_number_of_data)(struct stm32_spi *spi, u32 length);
+	void (*write_tx)(struct stm32_spi *spi);
+	void (*read_rx)(struct stm32_spi *spi);
 	void (*transfer_one_dma_start)(struct stm32_spi *spi);
 	void (*dma_rx_cb)(void *data);
 	void (*dma_tx_cb)(void *data);
@@ -324,20 +335,20 @@ struct stm32_spi {
 	bool device_mode;
 };
 
-static const struct stm32_spi_regspec stm32f4_spi_regspec = {
-	.en = { STM32F4_SPI_CR1, STM32F4_SPI_CR1_SPE },
+static const struct stm32_spi_regspec stm32fx_spi_regspec = {
+	.en = { STM32FX_SPI_CR1, STM32FX_SPI_CR1_SPE },
 
-	.dma_rx_en = { STM32F4_SPI_CR2, STM32F4_SPI_CR2_RXDMAEN },
-	.dma_tx_en = { STM32F4_SPI_CR2, STM32F4_SPI_CR2_TXDMAEN },
+	.dma_rx_en = { STM32FX_SPI_CR2, STM32FX_SPI_CR2_RXDMAEN },
+	.dma_tx_en = { STM32FX_SPI_CR2, STM32FX_SPI_CR2_TXDMAEN },
 
-	.cpol = { STM32F4_SPI_CR1, STM32F4_SPI_CR1_CPOL },
-	.cpha = { STM32F4_SPI_CR1, STM32F4_SPI_CR1_CPHA },
-	.lsb_first = { STM32F4_SPI_CR1, STM32F4_SPI_CR1_LSBFRST },
+	.cpol = { STM32FX_SPI_CR1, STM32FX_SPI_CR1_CPOL },
+	.cpha = { STM32FX_SPI_CR1, STM32FX_SPI_CR1_CPHA },
+	.lsb_first = { STM32FX_SPI_CR1, STM32FX_SPI_CR1_LSBFRST },
 	.cs_high = {},
-	.br = { STM32F4_SPI_CR1, STM32F4_SPI_CR1_BR, STM32F4_SPI_CR1_BR_SHIFT },
+	.br = { STM32FX_SPI_CR1, STM32FX_SPI_CR1_BR, STM32FX_SPI_CR1_BR_SHIFT },
 
-	.rx = { STM32F4_SPI_DR },
-	.tx = { STM32F4_SPI_DR },
+	.rx = { STM32FX_SPI_DR },
+	.tx = { STM32FX_SPI_DR },
 };
 
 static const struct stm32_spi_regspec stm32h7_spi_regspec = {
@@ -409,6 +420,16 @@ static int stm32f4_spi_get_bpw_mask(struct stm32_spi *spi)
 	return SPI_BPW_MASK(8) | SPI_BPW_MASK(16);
 }
 
+/**
+ * stm32f7_spi_get_bpw_mask - Return bits per word mask
+ * @spi: pointer to the spi controller data structure
+ */
+static int stm32f7_spi_get_bpw_mask(struct stm32_spi *spi)
+{
+	dev_dbg(spi->dev, "16-bit maximum data frame\n");
+	return SPI_BPW_RANGE_MASK(4, 16);
+}
+
 /**
  * stm32h7_spi_get_bpw_mask - Return bits per word mask
  * @spi: pointer to the spi controller data structure
@@ -502,19 +523,48 @@ static u32 stm32h7_spi_prepare_fthlv(struct stm32_spi *spi, u32 xfer_len)
  */
 static void stm32f4_spi_write_tx(struct stm32_spi *spi)
 {
-	if ((spi->tx_len > 0) && (readl_relaxed(spi->base + STM32F4_SPI_SR) &
-				  STM32F4_SPI_SR_TXE)) {
+	if ((spi->tx_len > 0) && (readl_relaxed(spi->base + STM32FX_SPI_SR) &
+				  STM32FX_SPI_SR_TXE)) {
 		u32 offs = spi->cur_xferlen - spi->tx_len;
 
 		if (spi->cur_bpw == 16) {
 			const u16 *tx_buf16 = (const u16 *)(spi->tx_buf + offs);
 
-			writew_relaxed(*tx_buf16, spi->base + STM32F4_SPI_DR);
+			writew_relaxed(*tx_buf16, spi->base + STM32FX_SPI_DR);
+			spi->tx_len -= sizeof(u16);
+		} else {
+			const u8 *tx_buf8 = (const u8 *)(spi->tx_buf + offs);
+
+			writeb_relaxed(*tx_buf8, spi->base + STM32FX_SPI_DR);
+			spi->tx_len -= sizeof(u8);
+		}
+	}
+
+	dev_dbg(spi->dev, "%s: %d bytes left\n", __func__, spi->tx_len);
+}
+
+/**
+ * stm32f7_spi_write_tx - Write bytes to Transmit Data Register
+ * @spi: pointer to the spi controller data structure
+ *
+ * Read from tx_buf depends on remaining bytes to avoid to read beyond
+ * tx_buf end.
+ */
+static void stm32f7_spi_write_tx(struct stm32_spi *spi)
+{
+	if ((spi->tx_len > 0) && (readl_relaxed(spi->base + STM32FX_SPI_SR) &
+				  STM32FX_SPI_SR_TXE)) {
+		u32 offs = spi->cur_xferlen - spi->tx_len;
+
+		if (spi->tx_len >= sizeof(u16)) {
+			const u16 *tx_buf16 = (const u16 *)(spi->tx_buf + offs);
+
+			writew_relaxed(*tx_buf16, spi->base + STM32FX_SPI_DR);
 			spi->tx_len -= sizeof(u16);
 		} else {
 			const u8 *tx_buf8 = (const u8 *)(spi->tx_buf + offs);
 
-			writeb_relaxed(*tx_buf8, spi->base + STM32F4_SPI_DR);
+			writeb_relaxed(*tx_buf8, spi->base + STM32FX_SPI_DR);
 			spi->tx_len -= sizeof(u8);
 		}
 	}
@@ -566,19 +616,19 @@ static void stm32h7_spi_write_txfifo(struct stm32_spi *spi)
  */
 static void stm32f4_spi_read_rx(struct stm32_spi *spi)
 {
-	if ((spi->rx_len > 0) && (readl_relaxed(spi->base + STM32F4_SPI_SR) &
-				  STM32F4_SPI_SR_RXNE)) {
+	if ((spi->rx_len > 0) && (readl_relaxed(spi->base + STM32FX_SPI_SR) &
+				  STM32FX_SPI_SR_RXNE)) {
 		u32 offs = spi->cur_xferlen - spi->rx_len;
 
 		if (spi->cur_bpw == 16) {
 			u16 *rx_buf16 = (u16 *)(spi->rx_buf + offs);
 
-			*rx_buf16 = readw_relaxed(spi->base + STM32F4_SPI_DR);
+			*rx_buf16 = readw_relaxed(spi->base + STM32FX_SPI_DR);
 			spi->rx_len -= sizeof(u16);
 		} else {
 			u8 *rx_buf8 = (u8 *)(spi->rx_buf + offs);
 
-			*rx_buf8 = readb_relaxed(spi->base + STM32F4_SPI_DR);
+			*rx_buf8 = readb_relaxed(spi->base + STM32FX_SPI_DR);
 			spi->rx_len -= sizeof(u8);
 		}
 	}
@@ -586,6 +636,46 @@ static void stm32f4_spi_read_rx(struct stm32_spi *spi)
 	dev_dbg(spi->dev, "%s: %d bytes left\n", __func__, spi->rx_len);
 }
 
+/**
+ * stm32f7_spi_read_rx - Read bytes from Receive Data Register
+ * @spi: pointer to the spi controller data structure
+ *
+ * Write in rx_buf depends on remaining bytes to avoid to write beyond
+ * rx_buf end.
+ */
+static void stm32f7_spi_read_rx(struct stm32_spi *spi)
+{
+	u32 sr = readl_relaxed(spi->base + STM32FX_SPI_SR);
+	u32 frlvl = FIELD_GET(STM32F7_SPI_SR_FRLVL, sr);
+
+	while ((spi->rx_len > 0) && (frlvl > 0)) {
+		u32 offs = spi->cur_xferlen - spi->rx_len;
+
+		if ((spi->rx_len >= sizeof(u16)) && (frlvl >= 2)) {
+			u16 *rx_buf16 = (u16 *)(spi->rx_buf + offs);
+
+			*rx_buf16 = readw_relaxed(spi->base + STM32FX_SPI_DR);
+			spi->rx_len -= sizeof(u16);
+		} else {
+			u8 *rx_buf8 = (u8 *)(spi->rx_buf + offs);
+
+			*rx_buf8 = readb_relaxed(spi->base + STM32FX_SPI_DR);
+			spi->rx_len -= sizeof(u8);
+		}
+
+		sr = readl_relaxed(spi->base + STM32FX_SPI_SR);
+		frlvl = FIELD_GET(STM32F7_SPI_SR_FRLVL, sr);
+	}
+
+	if (spi->rx_len >= sizeof(u16))
+		stm32_spi_clr_bits(spi, STM32FX_SPI_CR2, STM32F7_SPI_CR2_FRXTH);
+	else
+		stm32_spi_set_bits(spi, STM32FX_SPI_CR2, STM32F7_SPI_CR2_FRXTH);
+
+	dev_dbg(spi->dev, "%s: %d bytes left (sr=%08x)\n",
+		__func__, spi->rx_len, sr);
+}
+
 /**
  * stm32h7_spi_read_rxfifo - Read bytes in Receive Data Register
  * @spi: pointer to the spi controller data structure
@@ -645,10 +735,10 @@ static void stm32_spi_enable(struct stm32_spi *spi)
 }
 
 /**
- * stm32f4_spi_disable - Disable SPI controller
+ * stm32fx_spi_disable - Disable SPI controller
  * @spi: pointer to the spi controller data structure
  */
-static void stm32f4_spi_disable(struct stm32_spi *spi)
+static void stm32fx_spi_disable(struct stm32_spi *spi)
 {
 	unsigned long flags;
 	u32 sr;
@@ -657,20 +747,20 @@ static void stm32f4_spi_disable(struct stm32_spi *spi)
 
 	spin_lock_irqsave(&spi->lock, flags);
 
-	if (!(readl_relaxed(spi->base + STM32F4_SPI_CR1) &
-	      STM32F4_SPI_CR1_SPE)) {
+	if (!(readl_relaxed(spi->base + STM32FX_SPI_CR1) &
+	      STM32FX_SPI_CR1_SPE)) {
 		spin_unlock_irqrestore(&spi->lock, flags);
 		return;
 	}
 
 	/* Disable interrupts */
-	stm32_spi_clr_bits(spi, STM32F4_SPI_CR2, STM32F4_SPI_CR2_TXEIE |
-						 STM32F4_SPI_CR2_RXNEIE |
-						 STM32F4_SPI_CR2_ERRIE);
+	stm32_spi_clr_bits(spi, STM32FX_SPI_CR2, STM32FX_SPI_CR2_TXEIE |
+						 STM32FX_SPI_CR2_RXNEIE |
+						 STM32FX_SPI_CR2_ERRIE);
 
 	/* Wait until BSY = 0 */
-	if (readl_relaxed_poll_timeout_atomic(spi->base + STM32F4_SPI_SR,
-					      sr, !(sr & STM32F4_SPI_SR_BSY),
+	if (readl_relaxed_poll_timeout_atomic(spi->base + STM32FX_SPI_SR,
+					      sr, !(sr & STM32FX_SPI_SR_BSY),
 					      10, 100000) < 0) {
 		dev_warn(spi->dev, "disabling condition timeout\n");
 	}
@@ -680,14 +770,14 @@ static void stm32f4_spi_disable(struct stm32_spi *spi)
 	if (spi->cur_usedma && spi->dma_rx)
 		dmaengine_terminate_async(spi->dma_rx);
 
-	stm32_spi_clr_bits(spi, STM32F4_SPI_CR1, STM32F4_SPI_CR1_SPE);
+	stm32_spi_clr_bits(spi, STM32FX_SPI_CR1, STM32FX_SPI_CR1_SPE);
 
-	stm32_spi_clr_bits(spi, STM32F4_SPI_CR2, STM32F4_SPI_CR2_TXDMAEN |
-						 STM32F4_SPI_CR2_RXDMAEN);
+	stm32_spi_clr_bits(spi, STM32FX_SPI_CR2, STM32FX_SPI_CR2_TXDMAEN |
+						 STM32FX_SPI_CR2_RXDMAEN);
 
 	/* Sequence to clear OVR flag */
-	readl_relaxed(spi->base + STM32F4_SPI_DR);
-	readl_relaxed(spi->base + STM32F4_SPI_SR);
+	readl_relaxed(spi->base + STM32FX_SPI_DR);
+	readl_relaxed(spi->base + STM32FX_SPI_SR);
 
 	spin_unlock_irqrestore(&spi->lock, flags);
 }
@@ -763,11 +853,11 @@ static bool stm32_spi_can_dma(struct spi_controller *ctrl,
 }
 
 /**
- * stm32f4_spi_irq_event - Interrupt handler for SPI controller events
+ * stm32fx_spi_irq_event - Interrupt handler for SPI controller events
  * @irq: interrupt line
  * @dev_id: SPI controller ctrl interface
  */
-static irqreturn_t stm32f4_spi_irq_event(int irq, void *dev_id)
+static irqreturn_t stm32fx_spi_irq_event(int irq, void *dev_id)
 {
 	struct spi_controller *ctrl = dev_id;
 	struct stm32_spi *spi = spi_controller_get_devdata(ctrl);
@@ -776,26 +866,26 @@ static irqreturn_t stm32f4_spi_irq_event(int irq, void *dev_id)
 
 	spin_lock(&spi->lock);
 
-	sr = readl_relaxed(spi->base + STM32F4_SPI_SR);
+	sr = readl_relaxed(spi->base + STM32FX_SPI_SR);
 	/*
 	 * BSY flag is not handled in interrupt but it is normal behavior when
 	 * this flag is set.
 	 */
-	sr &= ~STM32F4_SPI_SR_BSY;
+	sr &= ~STM32FX_SPI_SR_BSY;
 
 	if (!spi->cur_usedma && (spi->cur_comm == SPI_SIMPLEX_TX ||
 				 spi->cur_comm == SPI_3WIRE_TX)) {
 		/* OVR flag shouldn't be handled for TX only mode */
-		sr &= ~(STM32F4_SPI_SR_OVR | STM32F4_SPI_SR_RXNE);
-		mask |= STM32F4_SPI_SR_TXE;
+		sr &= ~(STM32FX_SPI_SR_OVR | STM32FX_SPI_SR_RXNE);
+		mask |= STM32FX_SPI_SR_TXE;
 	}
 
 	if (!spi->cur_usedma && (spi->cur_comm == SPI_FULL_DUPLEX ||
 				spi->cur_comm == SPI_SIMPLEX_RX ||
 				spi->cur_comm == SPI_3WIRE_RX)) {
 		/* TXE flag is set and is handled when RXNE flag occurs */
-		sr &= ~STM32F4_SPI_SR_TXE;
-		mask |= STM32F4_SPI_SR_RXNE | STM32F4_SPI_SR_OVR;
+		sr &= ~STM32FX_SPI_SR_TXE;
+		mask |= STM32FX_SPI_SR_RXNE | STM32FX_SPI_SR_OVR;
 	}
 
 	if (!(sr & mask)) {
@@ -804,12 +894,12 @@ static irqreturn_t stm32f4_spi_irq_event(int irq, void *dev_id)
 		return IRQ_NONE;
 	}
 
-	if (sr & STM32F4_SPI_SR_OVR) {
+	if (sr & STM32FX_SPI_SR_OVR) {
 		dev_warn(spi->dev, "Overrun: received value discarded\n");
 
 		/* Sequence to clear OVR flag */
-		readl_relaxed(spi->base + STM32F4_SPI_DR);
-		readl_relaxed(spi->base + STM32F4_SPI_SR);
+		readl_relaxed(spi->base + STM32FX_SPI_DR);
+		readl_relaxed(spi->base + STM32FX_SPI_SR);
 
 		/*
 		 * If overrun is detected, it means that something went wrong,
@@ -820,28 +910,28 @@ static irqreturn_t stm32f4_spi_irq_event(int irq, void *dev_id)
 		goto end_irq;
 	}
 
-	if (sr & STM32F4_SPI_SR_TXE) {
+	if (sr & STM32FX_SPI_SR_TXE) {
 		if (spi->tx_buf)
-			stm32f4_spi_write_tx(spi);
+			spi->cfg->write_tx(spi);
 		if (spi->tx_len == 0)
 			end = true;
 	}
 
-	if (sr & STM32F4_SPI_SR_RXNE) {
-		stm32f4_spi_read_rx(spi);
+	if (sr & STM32FX_SPI_SR_RXNE) {
+		spi->cfg->read_rx(spi);
 		if (spi->rx_len == 0)
 			end = true;
 		else if (spi->tx_buf)/* Load data for discontinuous mode */
-			stm32f4_spi_write_tx(spi);
+			spi->cfg->write_tx(spi);
 	}
 
 end_irq:
 	if (end) {
 		/* Immediately disable interrupts to do not generate new one */
-		stm32_spi_clr_bits(spi, STM32F4_SPI_CR2,
-					STM32F4_SPI_CR2_TXEIE |
-					STM32F4_SPI_CR2_RXNEIE |
-					STM32F4_SPI_CR2_ERRIE);
+		stm32_spi_clr_bits(spi, STM32FX_SPI_CR2,
+					STM32FX_SPI_CR2_TXEIE |
+					STM32FX_SPI_CR2_RXNEIE |
+					STM32FX_SPI_CR2_ERRIE);
 		spin_unlock(&spi->lock);
 		return IRQ_WAKE_THREAD;
 	}
@@ -851,17 +941,17 @@ static irqreturn_t stm32f4_spi_irq_event(int irq, void *dev_id)
 }
 
 /**
- * stm32f4_spi_irq_thread - Thread of interrupt handler for SPI controller
+ * stm32fx_spi_irq_thread - Thread of interrupt handler for SPI controller
  * @irq: interrupt line
  * @dev_id: SPI controller interface
  */
-static irqreturn_t stm32f4_spi_irq_thread(int irq, void *dev_id)
+static irqreturn_t stm32fx_spi_irq_thread(int irq, void *dev_id)
 {
 	struct spi_controller *ctrl = dev_id;
 	struct stm32_spi *spi = spi_controller_get_devdata(ctrl);
 
 	spi_finalize_current_transfer(ctrl);
-	stm32f4_spi_disable(spi);
+	stm32fx_spi_disable(spi);
 
 	return IRQ_HANDLED;
 }
@@ -1034,18 +1124,18 @@ static int stm32_spi_prepare_msg(struct spi_controller *ctrl,
 }
 
 /**
- * stm32f4_spi_dma_tx_cb - dma callback
+ * stm32fx_spi_dma_tx_cb - dma callback
  * @data: pointer to the spi controller data structure
  *
  * DMA callback is called when the transfer is complete for DMA TX channel.
  */
-static void stm32f4_spi_dma_tx_cb(void *data)
+static void stm32fx_spi_dma_tx_cb(void *data)
 {
 	struct stm32_spi *spi = data;
 
 	if (spi->cur_comm == SPI_SIMPLEX_TX || spi->cur_comm == SPI_3WIRE_TX) {
 		spi_finalize_current_transfer(spi->ctrl);
-		stm32f4_spi_disable(spi);
+		stm32fx_spi_disable(spi);
 	}
 }
 
@@ -1114,21 +1204,21 @@ static void stm32_spi_dma_config(struct stm32_spi *spi,
 }
 
 /**
- * stm32f4_spi_transfer_one_irq - transfer a single spi_transfer using
+ * stm32fx_spi_transfer_one_irq - transfer a single spi_transfer using
  *				  interrupts
  * @spi: pointer to the spi controller data structure
  *
  * It must returns 0 if the transfer is finished or 1 if the transfer is still
  * in progress.
  */
-static int stm32f4_spi_transfer_one_irq(struct stm32_spi *spi)
+static int stm32fx_spi_transfer_one_irq(struct stm32_spi *spi)
 {
 	unsigned long flags;
 	u32 cr2 = 0;
 
 	/* Enable the interrupts relative to the current communication mode */
 	if (spi->cur_comm == SPI_SIMPLEX_TX || spi->cur_comm == SPI_3WIRE_TX) {
-		cr2 |= STM32F4_SPI_CR2_TXEIE;
+		cr2 |= STM32FX_SPI_CR2_TXEIE;
 	} else if (spi->cur_comm == SPI_FULL_DUPLEX ||
 				spi->cur_comm == SPI_SIMPLEX_RX ||
 				spi->cur_comm == SPI_3WIRE_RX) {
@@ -1136,20 +1226,20 @@ static int stm32f4_spi_transfer_one_irq(struct stm32_spi *spi)
 		 * since the received data are never read. Therefore set OVR
 		 * interrupt only when rx buffer is available.
 		 */
-		cr2 |= STM32F4_SPI_CR2_RXNEIE | STM32F4_SPI_CR2_ERRIE;
+		cr2 |= STM32FX_SPI_CR2_RXNEIE | STM32FX_SPI_CR2_ERRIE;
 	} else {
 		return -EINVAL;
 	}
 
 	spin_lock_irqsave(&spi->lock, flags);
 
-	stm32_spi_set_bits(spi, STM32F4_SPI_CR2, cr2);
+	stm32_spi_set_bits(spi, STM32FX_SPI_CR2, cr2);
 
 	stm32_spi_enable(spi);
 
 	/* starting data transfer when buffer is loaded */
 	if (spi->tx_buf)
-		stm32f4_spi_write_tx(spi);
+		spi->cfg->write_tx(spi);
 
 	spin_unlock_irqrestore(&spi->lock, flags);
 
@@ -1200,11 +1290,11 @@ static int stm32h7_spi_transfer_one_irq(struct stm32_spi *spi)
 }
 
 /**
- * stm32f4_spi_transfer_one_dma_start - Set SPI driver registers to start
+ * stm32fx_spi_transfer_one_dma_start - Set SPI driver registers to start
  *					transfer using DMA
  * @spi: pointer to the spi controller data structure
  */
-static void stm32f4_spi_transfer_one_dma_start(struct stm32_spi *spi)
+static void stm32fx_spi_transfer_one_dma_start(struct stm32_spi *spi)
 {
 	/* In DMA mode end of transfer is handled by DMA TX or RX callback. */
 	if (spi->cur_comm == SPI_SIMPLEX_RX || spi->cur_comm == SPI_3WIRE_RX ||
@@ -1214,12 +1304,28 @@ static void stm32f4_spi_transfer_one_dma_start(struct stm32_spi *spi)
 		 * since the received data are never read. Therefore set OVR
 		 * interrupt only when rx buffer is available.
 		 */
-		stm32_spi_set_bits(spi, STM32F4_SPI_CR2, STM32F4_SPI_CR2_ERRIE);
+		stm32_spi_set_bits(spi, STM32FX_SPI_CR2, STM32FX_SPI_CR2_ERRIE);
 	}
 
 	stm32_spi_enable(spi);
 }
 
+/**
+ * stm32f7_spi_transfer_one_dma_start - Set SPI driver registers to start
+ *					transfer using DMA
+ * @spi: pointer to the spi controller data structure
+ */
+static void stm32f7_spi_transfer_one_dma_start(struct stm32_spi *spi)
+{
+	/* Configure DMA request trigger threshold according to DMA width */
+	if (spi->cur_bpw <= 8)
+		stm32_spi_set_bits(spi, STM32FX_SPI_CR2, STM32F7_SPI_CR2_FRXTH);
+	else
+		stm32_spi_clr_bits(spi, STM32FX_SPI_CR2, STM32F7_SPI_CR2_FRXTH);
+
+	stm32fx_spi_transfer_one_dma_start(spi);
+}
+
 /**
  * stm32h7_spi_transfer_one_dma_start - Set SPI driver registers to start
  *					transfer using DMA
@@ -1353,9 +1459,34 @@ static int stm32_spi_transfer_one_dma(struct stm32_spi *spi,
 static void stm32f4_spi_set_bpw(struct stm32_spi *spi)
 {
 	if (spi->cur_bpw == 16)
-		stm32_spi_set_bits(spi, STM32F4_SPI_CR1, STM32F4_SPI_CR1_DFF);
+		stm32_spi_set_bits(spi, STM32FX_SPI_CR1, STM32F4_SPI_CR1_DFF);
+	else
+		stm32_spi_clr_bits(spi, STM32FX_SPI_CR1, STM32F4_SPI_CR1_DFF);
+}
+
+/**
+ * stm32f7_spi_set_bpw - Configure bits per word
+ * @spi: pointer to the spi controller data structure
+ */
+static void stm32f7_spi_set_bpw(struct stm32_spi *spi)
+{
+	u32 bpw;
+	u32 cr2_clrb = 0, cr2_setb = 0;
+
+	bpw = spi->cur_bpw - 1;
+
+	cr2_clrb |= STM32F7_SPI_CR2_DS;
+	cr2_setb |= FIELD_PREP(STM32F7_SPI_CR2_DS, bpw);
+
+	if (spi->rx_len >= sizeof(u16))
+		cr2_clrb |= STM32F7_SPI_CR2_FRXTH;
 	else
-		stm32_spi_clr_bits(spi, STM32F4_SPI_CR1, STM32F4_SPI_CR1_DFF);
+		cr2_setb |= STM32F7_SPI_CR2_FRXTH;
+
+	writel_relaxed(
+		(readl_relaxed(spi->base + STM32FX_SPI_CR2) &
+		 ~cr2_clrb) | cr2_setb,
+		spi->base + STM32FX_SPI_CR2);
 }
 
 /**
@@ -1433,26 +1564,26 @@ static unsigned int stm32_spi_communication_type(struct spi_device *spi_dev,
 }
 
 /**
- * stm32f4_spi_set_mode - configure communication mode
+ * stm32fx_spi_set_mode - configure communication mode
  * @spi: pointer to the spi controller data structure
  * @comm_type: type of communication to configure
  */
-static int stm32f4_spi_set_mode(struct stm32_spi *spi, unsigned int comm_type)
+static int stm32fx_spi_set_mode(struct stm32_spi *spi, unsigned int comm_type)
 {
 	if (comm_type == SPI_3WIRE_TX || comm_type == SPI_SIMPLEX_TX) {
-		stm32_spi_set_bits(spi, STM32F4_SPI_CR1,
-					STM32F4_SPI_CR1_BIDIMODE |
-					STM32F4_SPI_CR1_BIDIOE);
+		stm32_spi_set_bits(spi, STM32FX_SPI_CR1,
+					STM32FX_SPI_CR1_BIDIMODE |
+					STM32FX_SPI_CR1_BIDIOE);
 	} else if (comm_type == SPI_FULL_DUPLEX ||
 				comm_type == SPI_SIMPLEX_RX) {
-		stm32_spi_clr_bits(spi, STM32F4_SPI_CR1,
-					STM32F4_SPI_CR1_BIDIMODE |
-					STM32F4_SPI_CR1_BIDIOE);
+		stm32_spi_clr_bits(spi, STM32FX_SPI_CR1,
+					STM32FX_SPI_CR1_BIDIMODE |
+					STM32FX_SPI_CR1_BIDIOE);
 	} else if (comm_type == SPI_3WIRE_RX) {
-		stm32_spi_set_bits(spi, STM32F4_SPI_CR1,
-					STM32F4_SPI_CR1_BIDIMODE);
-		stm32_spi_clr_bits(spi, STM32F4_SPI_CR1,
-					STM32F4_SPI_CR1_BIDIOE);
+		stm32_spi_set_bits(spi, STM32FX_SPI_CR1,
+					STM32FX_SPI_CR1_BIDIMODE);
+		stm32_spi_clr_bits(spi, STM32FX_SPI_CR1,
+					STM32FX_SPI_CR1_BIDIOE);
 	} else {
 		return -EINVAL;
 	}
@@ -1672,18 +1803,18 @@ static int stm32_spi_unprepare_msg(struct spi_controller *ctrl,
 }
 
 /**
- * stm32f4_spi_config - Configure SPI controller as SPI master
+ * stm32fx_spi_config - Configure SPI controller as SPI master
  * @spi: pointer to the spi controller data structure
  */
-static int stm32f4_spi_config(struct stm32_spi *spi)
+static int stm32fx_spi_config(struct stm32_spi *spi)
 {
 	unsigned long flags;
 
 	spin_lock_irqsave(&spi->lock, flags);
 
 	/* Ensure I2SMOD bit is kept cleared */
-	stm32_spi_clr_bits(spi, STM32F4_SPI_I2SCFGR,
-			   STM32F4_SPI_I2SCFGR_I2SMOD);
+	stm32_spi_clr_bits(spi, STM32FX_SPI_I2SCFGR,
+			   STM32FX_SPI_I2SCFGR_I2SMOD);
 
 	/*
 	 * - SS input value high
@@ -1692,10 +1823,10 @@ static int stm32f4_spi_config(struct stm32_spi *spi)
 	 * - Consider 1 master/n slaves configuration and
 	 *   SS input value is determined by the SSI bit
 	 */
-	stm32_spi_set_bits(spi, STM32F4_SPI_CR1, STM32F4_SPI_CR1_SSI |
-						 STM32F4_SPI_CR1_BIDIOE |
-						 STM32F4_SPI_CR1_MSTR |
-						 STM32F4_SPI_CR1_SSM);
+	stm32_spi_set_bits(spi, STM32FX_SPI_CR1, STM32FX_SPI_CR1_SSI |
+						 STM32FX_SPI_CR1_BIDIOE |
+						 STM32FX_SPI_CR1_MSTR |
+						 STM32FX_SPI_CR1_SSM);
 
 	spin_unlock_irqrestore(&spi->lock, flags);
 
@@ -1746,25 +1877,48 @@ static int stm32h7_spi_config(struct stm32_spi *spi)
 }
 
 static const struct stm32_spi_cfg stm32f4_spi_cfg = {
-	.regs = &stm32f4_spi_regspec,
+	.regs = &stm32fx_spi_regspec,
 	.get_bpw_mask = stm32f4_spi_get_bpw_mask,
-	.disable = stm32f4_spi_disable,
-	.config = stm32f4_spi_config,
+	.disable = stm32fx_spi_disable,
+	.config = stm32fx_spi_config,
 	.set_bpw = stm32f4_spi_set_bpw,
-	.set_mode = stm32f4_spi_set_mode,
-	.transfer_one_dma_start = stm32f4_spi_transfer_one_dma_start,
-	.dma_tx_cb = stm32f4_spi_dma_tx_cb,
+	.set_mode = stm32fx_spi_set_mode,
+	.write_tx = stm32f4_spi_write_tx,
+	.read_rx = stm32f4_spi_read_rx,
+	.transfer_one_dma_start = stm32fx_spi_transfer_one_dma_start,
+	.dma_tx_cb = stm32fx_spi_dma_tx_cb,
 	.dma_rx_cb = stm32_spi_dma_rx_cb,
-	.transfer_one_irq = stm32f4_spi_transfer_one_irq,
-	.irq_handler_event = stm32f4_spi_irq_event,
-	.irq_handler_thread = stm32f4_spi_irq_thread,
-	.baud_rate_div_min = STM32F4_SPI_BR_DIV_MIN,
-	.baud_rate_div_max = STM32F4_SPI_BR_DIV_MAX,
+	.transfer_one_irq = stm32fx_spi_transfer_one_irq,
+	.irq_handler_event = stm32fx_spi_irq_event,
+	.irq_handler_thread = stm32fx_spi_irq_thread,
+	.baud_rate_div_min = STM32FX_SPI_BR_DIV_MIN,
+	.baud_rate_div_max = STM32FX_SPI_BR_DIV_MAX,
 	.has_fifo = false,
 	.has_device_mode = false,
 	.flags = SPI_CONTROLLER_MUST_TX,
 };
 
+static const struct stm32_spi_cfg stm32f7_spi_cfg = {
+	.regs = &stm32fx_spi_regspec,
+	.get_bpw_mask = stm32f7_spi_get_bpw_mask,
+	.disable = stm32fx_spi_disable,
+	.config = stm32fx_spi_config,
+	.set_bpw = stm32f7_spi_set_bpw,
+	.set_mode = stm32fx_spi_set_mode,
+	.write_tx = stm32f7_spi_write_tx,
+	.read_rx = stm32f7_spi_read_rx,
+	.transfer_one_dma_start = stm32f7_spi_transfer_one_dma_start,
+	.dma_tx_cb = stm32fx_spi_dma_tx_cb,
+	.dma_rx_cb = stm32_spi_dma_rx_cb,
+	.transfer_one_irq = stm32fx_spi_transfer_one_irq,
+	.irq_handler_event = stm32fx_spi_irq_event,
+	.irq_handler_thread = stm32fx_spi_irq_thread,
+	.baud_rate_div_min = STM32FX_SPI_BR_DIV_MIN,
+	.baud_rate_div_max = STM32FX_SPI_BR_DIV_MAX,
+	.has_fifo = false,
+	.flags = SPI_CONTROLLER_MUST_TX,
+};
+
 static const struct stm32_spi_cfg stm32h7_spi_cfg = {
 	.regs = &stm32h7_spi_regspec,
 	.get_fifo_size = stm32h7_spi_get_fifo_size,
@@ -1775,6 +1929,8 @@ static const struct stm32_spi_cfg stm32h7_spi_cfg = {
 	.set_mode = stm32h7_spi_set_mode,
 	.set_data_idleness = stm32h7_spi_data_idleness,
 	.set_number_of_data = stm32h7_spi_number_of_data,
+	.write_tx = stm32h7_spi_write_txfifo,
+	.read_rx = stm32h7_spi_read_rxfifo,
 	.transfer_one_dma_start = stm32h7_spi_transfer_one_dma_start,
 	.dma_rx_cb = stm32_spi_dma_rx_cb,
 	/*
@@ -1792,6 +1948,7 @@ static const struct stm32_spi_cfg stm32h7_spi_cfg = {
 static const struct of_device_id stm32_spi_of_match[] = {
 	{ .compatible = "st,stm32h7-spi", .data = (void *)&stm32h7_spi_cfg },
 	{ .compatible = "st,stm32f4-spi", .data = (void *)&stm32f4_spi_cfg },
+	{ .compatible = "st,stm32f7-spi", .data = (void *)&stm32f7_spi_cfg },
 	{},
 };
 MODULE_DEVICE_TABLE(of, stm32_spi_of_match);
diff --git a/drivers/spi/spi-ti-qspi.c b/drivers/spi/spi-ti-qspi.c
index 4c81516b67db..fdc092a05284 100644
--- a/drivers/spi/spi-ti-qspi.c
+++ b/drivers/spi/spi-ti-qspi.c
@@ -907,21 +907,22 @@ static int ti_qspi_probe(struct platform_device *pdev)
 	return ret;
 }
 
-static int ti_qspi_remove(struct platform_device *pdev)
+static void ti_qspi_remove(struct platform_device *pdev)
 {
 	struct ti_qspi *qspi = platform_get_drvdata(pdev);
 	int rc;
 
 	rc = spi_master_suspend(qspi->master);
-	if (rc)
-		return rc;
+	if (rc) {
+		dev_alert(&pdev->dev, "spi_master_suspend() failed (%pe)\n",
+			  ERR_PTR(rc));
+		return;
+	}
 
 	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 
 	ti_qspi_dma_cleanup(qspi);
-
-	return 0;
 }
 
 static const struct dev_pm_ops ti_qspi_pm_ops = {
@@ -930,7 +931,7 @@ static const struct dev_pm_ops ti_qspi_pm_ops = {
 
 static struct platform_driver ti_qspi_driver = {
 	.probe	= ti_qspi_probe,
-	.remove = ti_qspi_remove,
+	.remove_new = ti_qspi_remove,
 	.driver = {
 		.name	= "ti-qspi",
 		.pm =   &ti_qspi_pm_ops,
diff --git a/drivers/tee/optee/call.c b/drivers/tee/optee/call.c
index df5fb5410b72..a91e50be11be 100644
--- a/drivers/tee/optee/call.c
+++ b/drivers/tee/optee/call.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (c) 2015-2021, Linaro Limited
+ * Copyright (c) 2015-2021, 2023 Linaro Limited
  */
 #include <linux/device.h>
 #include <linux/err.h>
@@ -39,9 +39,29 @@ struct optee_shm_arg_entry {
 	DECLARE_BITMAP(map, MAX_ARG_COUNT_PER_ENTRY);
 };
 
+void optee_cq_init(struct optee_call_queue *cq, int thread_count)
+{
+	mutex_init(&cq->mutex);
+	INIT_LIST_HEAD(&cq->waiters);
+
+	/*
+	 * If cq->total_thread_count is 0 then we're not trying to keep
+	 * track of how many free threads we have, instead we're relying on
+	 * the secure world to tell us when we're out of thread and have to
+	 * wait for another thread to become available.
+	 */
+	cq->total_thread_count = thread_count;
+	cq->free_thread_count = thread_count;
+}
+
 void optee_cq_wait_init(struct optee_call_queue *cq,
-			struct optee_call_waiter *w)
+			struct optee_call_waiter *w, bool sys_thread)
 {
+	unsigned int free_thread_threshold;
+	bool need_wait = false;
+
+	memset(w, 0, sizeof(*w));
+
 	/*
 	 * We're preparing to make a call to secure world. In case we can't
 	 * allocate a thread in secure world we'll end up waiting in
@@ -60,8 +80,38 @@ void optee_cq_wait_init(struct optee_call_queue *cq,
 	 */
 	init_completion(&w->c);
 	list_add_tail(&w->list_node, &cq->waiters);
+	w->sys_thread = sys_thread;
+
+	if (cq->total_thread_count) {
+		if (sys_thread || !cq->sys_thread_req_count)
+			free_thread_threshold = 0;
+		else
+			free_thread_threshold = 1;
+
+		if (cq->free_thread_count > free_thread_threshold)
+			cq->free_thread_count--;
+		else
+			need_wait = true;
+	}
 
 	mutex_unlock(&cq->mutex);
+
+	while (need_wait) {
+		optee_cq_wait_for_completion(cq, w);
+		mutex_lock(&cq->mutex);
+
+		if (sys_thread || !cq->sys_thread_req_count)
+			free_thread_threshold = 0;
+		else
+			free_thread_threshold = 1;
+
+		if (cq->free_thread_count > free_thread_threshold) {
+			cq->free_thread_count--;
+			need_wait = false;
+		}
+
+		mutex_unlock(&cq->mutex);
+	}
 }
 
 void optee_cq_wait_for_completion(struct optee_call_queue *cq,
@@ -83,6 +133,14 @@ static void optee_cq_complete_one(struct optee_call_queue *cq)
 {
 	struct optee_call_waiter *w;
 
+	/* Wake a waiting system session if any, prior to a normal session */
+	list_for_each_entry(w, &cq->waiters, list_node) {
+		if (w->sys_thread && !completion_done(&w->c)) {
+			complete(&w->c);
+			return;
+		}
+	}
+
 	list_for_each_entry(w, &cq->waiters, list_node) {
 		if (!completion_done(&w->c)) {
 			complete(&w->c);
@@ -104,6 +162,8 @@ void optee_cq_wait_final(struct optee_call_queue *cq,
 	/* Get out of the list */
 	list_del(&w->list_node);
 
+	cq->free_thread_count++;
+
 	/* Wake up one eventual waiting task */
 	optee_cq_complete_one(cq);
 
@@ -119,6 +179,28 @@ void optee_cq_wait_final(struct optee_call_queue *cq,
 	mutex_unlock(&cq->mutex);
 }
 
+/* Count registered system sessions to reserved a system thread or not */
+static bool optee_cq_incr_sys_thread_count(struct optee_call_queue *cq)
+{
+	if (cq->total_thread_count <= 1)
+		return false;
+
+	mutex_lock(&cq->mutex);
+	cq->sys_thread_req_count++;
+	mutex_unlock(&cq->mutex);
+
+	return true;
+}
+
+static void optee_cq_decr_sys_thread_count(struct optee_call_queue *cq)
+{
+	mutex_lock(&cq->mutex);
+	cq->sys_thread_req_count--;
+	/* If there's someone waiting, let it resume */
+	optee_cq_complete_one(cq);
+	mutex_unlock(&cq->mutex);
+}
+
 /* Requires the filpstate mutex to be held */
 static struct optee_session *find_session(struct optee_context_data *ctxdata,
 					  u32 session_id)
@@ -328,7 +410,8 @@ int optee_open_session(struct tee_context *ctx,
 		goto out;
 	}
 
-	if (optee->ops->do_call_with_arg(ctx, shm, offs)) {
+	if (optee->ops->do_call_with_arg(ctx, shm, offs,
+					 sess->use_sys_thread)) {
 		msg_arg->ret = TEEC_ERROR_COMMUNICATION;
 		msg_arg->ret_origin = TEEC_ORIGIN_COMMS;
 	}
@@ -360,7 +443,29 @@ int optee_open_session(struct tee_context *ctx,
 	return rc;
 }
 
-int optee_close_session_helper(struct tee_context *ctx, u32 session)
+int optee_system_session(struct tee_context *ctx, u32 session)
+{
+	struct optee *optee = tee_get_drvdata(ctx->teedev);
+	struct optee_context_data *ctxdata = ctx->data;
+	struct optee_session *sess;
+	int rc = -EINVAL;
+
+	mutex_lock(&ctxdata->mutex);
+
+	sess = find_session(ctxdata, session);
+	if (sess && (sess->use_sys_thread ||
+		     optee_cq_incr_sys_thread_count(&optee->call_queue))) {
+		sess->use_sys_thread = true;
+		rc = 0;
+	}
+
+	mutex_unlock(&ctxdata->mutex);
+
+	return rc;
+}
+
+int optee_close_session_helper(struct tee_context *ctx, u32 session,
+			       bool system_thread)
 {
 	struct optee *optee = tee_get_drvdata(ctx->teedev);
 	struct optee_shm_arg_entry *entry;
@@ -374,10 +479,13 @@ int optee_close_session_helper(struct tee_context *ctx, u32 session)
 
 	msg_arg->cmd = OPTEE_MSG_CMD_CLOSE_SESSION;
 	msg_arg->session = session;
-	optee->ops->do_call_with_arg(ctx, shm, offs);
+	optee->ops->do_call_with_arg(ctx, shm, offs, system_thread);
 
 	optee_free_msg_arg(ctx, entry, offs);
 
+	if (system_thread)
+		optee_cq_decr_sys_thread_count(&optee->call_queue);
+
 	return 0;
 }
 
@@ -385,6 +493,7 @@ int optee_close_session(struct tee_context *ctx, u32 session)
 {
 	struct optee_context_data *ctxdata = ctx->data;
 	struct optee_session *sess;
+	bool system_thread;
 
 	/* Check that the session is valid and remove it from the list */
 	mutex_lock(&ctxdata->mutex);
@@ -394,9 +503,10 @@ int optee_close_session(struct tee_context *ctx, u32 session)
 	mutex_unlock(&ctxdata->mutex);
 	if (!sess)
 		return -EINVAL;
+	system_thread = sess->use_sys_thread;
 	kfree(sess);
 
-	return optee_close_session_helper(ctx, session);
+	return optee_close_session_helper(ctx, session, system_thread);
 }
 
 int optee_invoke_func(struct tee_context *ctx, struct tee_ioctl_invoke_arg *arg,
@@ -408,12 +518,15 @@ int optee_invoke_func(struct tee_context *ctx, struct tee_ioctl_invoke_arg *arg,
 	struct optee_msg_arg *msg_arg;
 	struct optee_session *sess;
 	struct tee_shm *shm;
+	bool system_thread;
 	u_int offs;
 	int rc;
 
 	/* Check that the session is valid */
 	mutex_lock(&ctxdata->mutex);
 	sess = find_session(ctxdata, arg->session);
+	if (sess)
+		system_thread = sess->use_sys_thread;
 	mutex_unlock(&ctxdata->mutex);
 	if (!sess)
 		return -EINVAL;
@@ -432,7 +545,7 @@ int optee_invoke_func(struct tee_context *ctx, struct tee_ioctl_invoke_arg *arg,
 	if (rc)
 		goto out;
 
-	if (optee->ops->do_call_with_arg(ctx, shm, offs)) {
+	if (optee->ops->do_call_with_arg(ctx, shm, offs, system_thread)) {
 		msg_arg->ret = TEEC_ERROR_COMMUNICATION;
 		msg_arg->ret_origin = TEEC_ORIGIN_COMMS;
 	}
@@ -457,12 +570,15 @@ int optee_cancel_req(struct tee_context *ctx, u32 cancel_id, u32 session)
 	struct optee_shm_arg_entry *entry;
 	struct optee_msg_arg *msg_arg;
 	struct optee_session *sess;
+	bool system_thread;
 	struct tee_shm *shm;
 	u_int offs;
 
 	/* Check that the session is valid */
 	mutex_lock(&ctxdata->mutex);
 	sess = find_session(ctxdata, session);
+	if (sess)
+		system_thread = sess->use_sys_thread;
 	mutex_unlock(&ctxdata->mutex);
 	if (!sess)
 		return -EINVAL;
@@ -474,7 +590,7 @@ int optee_cancel_req(struct tee_context *ctx, u32 cancel_id, u32 session)
 	msg_arg->cmd = OPTEE_MSG_CMD_CANCEL;
 	msg_arg->session = session;
 	msg_arg->cancel_id = cancel_id;
-	optee->ops->do_call_with_arg(ctx, shm, offs);
+	optee->ops->do_call_with_arg(ctx, shm, offs, system_thread);
 
 	optee_free_msg_arg(ctx, entry, offs);
 	return 0;
@@ -524,3 +640,32 @@ int optee_check_mem_type(unsigned long start, size_t num_pages)
 
 	return rc;
 }
+
+static int simple_call_with_arg(struct tee_context *ctx, u32 cmd)
+{
+	struct optee *optee = tee_get_drvdata(ctx->teedev);
+	struct optee_shm_arg_entry *entry;
+	struct optee_msg_arg *msg_arg;
+	struct tee_shm *shm;
+	u_int offs;
+
+	msg_arg = optee_get_msg_arg(ctx, 0, &entry, &shm, &offs);
+	if (IS_ERR(msg_arg))
+		return PTR_ERR(msg_arg);
+
+	msg_arg->cmd = cmd;
+	optee->ops->do_call_with_arg(ctx, shm, offs, false);
+
+	optee_free_msg_arg(ctx, entry, offs);
+	return 0;
+}
+
+int optee_do_bottom_half(struct tee_context *ctx)
+{
+	return simple_call_with_arg(ctx, OPTEE_MSG_CMD_DO_BOTTOM_HALF);
+}
+
+int optee_stop_async_notif(struct tee_context *ctx)
+{
+	return simple_call_with_arg(ctx, OPTEE_MSG_CMD_STOP_ASYNC_NOTIF);
+}
diff --git a/drivers/tee/optee/core.c b/drivers/tee/optee/core.c
index 2a258bd3b6b5..8ee3c71bd989 100644
--- a/drivers/tee/optee/core.c
+++ b/drivers/tee/optee/core.c
@@ -15,7 +15,6 @@
 #include <linux/string.h>
 #include <linux/tee_drv.h>
 #include <linux/types.h>
-#include <linux/workqueue.h>
 #include "optee_private.h"
 
 int optee_pool_op_alloc_helper(struct tee_shm_pool *pool, struct tee_shm *shm,
@@ -110,12 +109,7 @@ int optee_open(struct tee_context *ctx, bool cap_memref_null)
 
 		if (!optee->scan_bus_done) {
 			INIT_WORK(&optee->scan_bus_work, optee_bus_scan);
-			optee->scan_bus_wq = create_workqueue("optee_bus_scan");
-			if (!optee->scan_bus_wq) {
-				kfree(ctxdata);
-				return -ECHILD;
-			}
-			queue_work(optee->scan_bus_wq, &optee->scan_bus_work);
+			schedule_work(&optee->scan_bus_work);
 			optee->scan_bus_done = true;
 		}
 	}
@@ -129,7 +123,8 @@ int optee_open(struct tee_context *ctx, bool cap_memref_null)
 
 static void optee_release_helper(struct tee_context *ctx,
 				 int (*close_session)(struct tee_context *ctx,
-						      u32 session))
+						      u32 session,
+						      bool system_thread))
 {
 	struct optee_context_data *ctxdata = ctx->data;
 	struct optee_session *sess;
@@ -141,7 +136,7 @@ static void optee_release_helper(struct tee_context *ctx,
 	list_for_each_entry_safe(sess, sess_tmp, &ctxdata->sess_list,
 				 list_node) {
 		list_del(&sess->list_node);
-		close_session(ctx, sess->session_id);
+		close_session(ctx, sess->session_id, sess->use_sys_thread);
 		kfree(sess);
 	}
 	kfree(ctxdata);
@@ -158,10 +153,7 @@ void optee_release_supp(struct tee_context *ctx)
 	struct optee *optee = tee_get_drvdata(ctx->teedev);
 
 	optee_release_helper(ctx, optee_close_session_helper);
-	if (optee->scan_bus_wq) {
-		destroy_workqueue(optee->scan_bus_wq);
-		optee->scan_bus_wq = NULL;
-	}
+
 	optee_supp_release(&optee->supp);
 }
 
diff --git a/drivers/tee/optee/device.c b/drivers/tee/optee/device.c
index 64f0e047c23d..4b1092127694 100644
--- a/drivers/tee/optee/device.c
+++ b/drivers/tee/optee/device.c
@@ -60,7 +60,16 @@ static void optee_release_device(struct device *dev)
 	kfree(optee_device);
 }
 
-static int optee_register_device(const uuid_t *device_uuid)
+static ssize_t need_supplicant_show(struct device *dev,
+				    struct device_attribute *attr,
+				    char *buf)
+{
+	return 0;
+}
+
+static DEVICE_ATTR_RO(need_supplicant);
+
+static int optee_register_device(const uuid_t *device_uuid, u32 func)
 {
 	struct tee_client_device *optee_device = NULL;
 	int rc;
@@ -83,6 +92,10 @@ static int optee_register_device(const uuid_t *device_uuid)
 		put_device(&optee_device->dev);
 	}
 
+	if (func == PTA_CMD_GET_DEVICES_SUPP)
+		device_create_file(&optee_device->dev,
+				   &dev_attr_need_supplicant);
+
 	return rc;
 }
 
@@ -142,7 +155,7 @@ static int __optee_enumerate_devices(u32 func)
 	num_devices = shm_size / sizeof(uuid_t);
 
 	for (idx = 0; idx < num_devices; idx++) {
-		rc = optee_register_device(&device_uuid[idx]);
+		rc = optee_register_device(&device_uuid[idx], func);
 		if (rc)
 			goto out_shm;
 	}
diff --git a/drivers/tee/optee/ffa_abi.c b/drivers/tee/optee/ffa_abi.c
index 0828240f27e6..ecb5eb079408 100644
--- a/drivers/tee/optee/ffa_abi.c
+++ b/drivers/tee/optee/ffa_abi.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (c) 2021, Linaro Limited
+ * Copyright (c) 2021, 2023 Linaro Limited
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
@@ -528,7 +528,8 @@ static void optee_handle_ffa_rpc(struct tee_context *ctx, struct optee *optee,
 
 static int optee_ffa_yielding_call(struct tee_context *ctx,
 				   struct ffa_send_direct_data *data,
-				   struct optee_msg_arg *rpc_arg)
+				   struct optee_msg_arg *rpc_arg,
+				   bool system_thread)
 {
 	struct optee *optee = tee_get_drvdata(ctx->teedev);
 	struct ffa_device *ffa_dev = optee->ffa.ffa_dev;
@@ -541,7 +542,7 @@ static int optee_ffa_yielding_call(struct tee_context *ctx,
 	int rc;
 
 	/* Initialize waiter */
-	optee_cq_wait_init(&optee->call_queue, &w);
+	optee_cq_wait_init(&optee->call_queue, &w, system_thread);
 	while (true) {
 		rc = msg_ops->sync_send_receive(ffa_dev, data);
 		if (rc)
@@ -604,6 +605,7 @@ static int optee_ffa_yielding_call(struct tee_context *ctx,
  * @ctx:	calling context
  * @shm:	shared memory holding the message to pass to secure world
  * @offs:	offset of the message in @shm
+ * @system_thread: true if caller requests TEE system thread support
  *
  * Does a FF-A call to OP-TEE in secure world and handles eventual resulting
  * Remote Procedure Calls (RPC) from OP-TEE.
@@ -612,7 +614,8 @@ static int optee_ffa_yielding_call(struct tee_context *ctx,
  */
 
 static int optee_ffa_do_call_with_arg(struct tee_context *ctx,
-				      struct tee_shm *shm, u_int offs)
+				      struct tee_shm *shm, u_int offs,
+				      bool system_thread)
 {
 	struct ffa_send_direct_data data = {
 		.data0 = OPTEE_FFA_YIELDING_CALL_WITH_ARG,
@@ -642,7 +645,7 @@ static int optee_ffa_do_call_with_arg(struct tee_context *ctx,
 	if (IS_ERR(rpc_arg))
 		return PTR_ERR(rpc_arg);
 
-	return optee_ffa_yielding_call(ctx, &data, rpc_arg);
+	return optee_ffa_yielding_call(ctx, &data, rpc_arg, system_thread);
 }
 
 /*
@@ -692,7 +695,8 @@ static bool optee_ffa_api_is_compatbile(struct ffa_device *ffa_dev,
 static bool optee_ffa_exchange_caps(struct ffa_device *ffa_dev,
 				    const struct ffa_ops *ops,
 				    u32 *sec_caps,
-				    unsigned int *rpc_param_count)
+				    unsigned int *rpc_param_count,
+				    unsigned int *max_notif_value)
 {
 	struct ffa_send_direct_data data = { OPTEE_FFA_EXCHANGE_CAPABILITIES };
 	int rc;
@@ -709,10 +713,39 @@ static bool optee_ffa_exchange_caps(struct ffa_device *ffa_dev,
 
 	*rpc_param_count = (u8)data.data1;
 	*sec_caps = data.data2;
+	if (data.data3)
+		*max_notif_value = data.data3;
+	else
+		*max_notif_value = OPTEE_DEFAULT_MAX_NOTIF_VALUE;
 
 	return true;
 }
 
+static void notif_callback(int notify_id, void *cb_data)
+{
+	struct optee *optee = cb_data;
+
+	if (notify_id == optee->ffa.bottom_half_value)
+		optee_do_bottom_half(optee->ctx);
+	else
+		optee_notif_send(optee, notify_id);
+}
+
+static int enable_async_notif(struct optee *optee)
+{
+	struct ffa_device *ffa_dev = optee->ffa.ffa_dev;
+	struct ffa_send_direct_data data = {
+		.data0 = OPTEE_FFA_ENABLE_ASYNC_NOTIF,
+		.data1 = optee->ffa.bottom_half_value,
+	};
+	int rc;
+
+	rc = ffa_dev->ops->msg_ops->sync_send_receive(ffa_dev, &data);
+	if (rc)
+		return rc;
+	return data.data0;
+}
+
 static void optee_ffa_get_version(struct tee_device *teedev,
 				  struct tee_ioctl_version_data *vers)
 {
@@ -775,7 +808,11 @@ static const struct optee_ops optee_ffa_ops = {
 static void optee_ffa_remove(struct ffa_device *ffa_dev)
 {
 	struct optee *optee = ffa_dev_get_drvdata(ffa_dev);
+	u32 bottom_half_id = optee->ffa.bottom_half_value;
 
+	if (bottom_half_id != U32_MAX)
+		ffa_dev->ops->notifier_ops->notify_relinquish(ffa_dev,
+							      bottom_half_id);
 	optee_remove_common(optee);
 
 	mutex_destroy(&optee->ffa.mutex);
@@ -784,9 +821,51 @@ static void optee_ffa_remove(struct ffa_device *ffa_dev)
 	kfree(optee);
 }
 
+static int optee_ffa_async_notif_init(struct ffa_device *ffa_dev,
+				      struct optee *optee)
+{
+	bool is_per_vcpu = false;
+	u32 notif_id = 0;
+	int rc;
+
+	while (true) {
+		rc = ffa_dev->ops->notifier_ops->notify_request(ffa_dev,
+								is_per_vcpu,
+								notif_callback,
+								optee,
+								notif_id);
+		if (!rc)
+			break;
+		/*
+		 * -EACCES means that the notification ID was
+		 * already bound, try the next one as long as we
+		 * haven't reached the max. Any other error is a
+		 * permanent error, so skip asynchronous
+		 * notifications in that case.
+		 */
+		if (rc != -EACCES)
+			return rc;
+		notif_id++;
+		if (notif_id >= OPTEE_FFA_MAX_ASYNC_NOTIF_VALUE)
+			return rc;
+	}
+	optee->ffa.bottom_half_value = notif_id;
+
+	rc = enable_async_notif(optee);
+	if (rc < 0) {
+		ffa_dev->ops->notifier_ops->notify_relinquish(ffa_dev,
+							      notif_id);
+		optee->ffa.bottom_half_value = U32_MAX;
+	}
+
+	return rc;
+}
+
 static int optee_ffa_probe(struct ffa_device *ffa_dev)
 {
+	const struct ffa_notifier_ops *notif_ops;
 	const struct ffa_ops *ffa_ops;
+	unsigned int max_notif_value;
 	unsigned int rpc_param_count;
 	struct tee_shm_pool *pool;
 	struct tee_device *teedev;
@@ -797,12 +876,13 @@ static int optee_ffa_probe(struct ffa_device *ffa_dev)
 	int rc;
 
 	ffa_ops = ffa_dev->ops;
+	notif_ops = ffa_ops->notifier_ops;
 
 	if (!optee_ffa_api_is_compatbile(ffa_dev, ffa_ops))
 		return -EINVAL;
 
 	if (!optee_ffa_exchange_caps(ffa_dev, ffa_ops, &sec_caps,
-				     &rpc_param_count))
+				     &rpc_param_count, &max_notif_value))
 		return -EINVAL;
 	if (sec_caps & OPTEE_FFA_SEC_CAP_ARG_OFFSET)
 		arg_cache_flags |= OPTEE_SHM_ARG_SHARED;
@@ -820,6 +900,7 @@ static int optee_ffa_probe(struct ffa_device *ffa_dev)
 
 	optee->ops = &optee_ffa_ops;
 	optee->ffa.ffa_dev = ffa_dev;
+	optee->ffa.bottom_half_value = U32_MAX;
 	optee->rpc_param_count = rpc_param_count;
 
 	teedev = tee_device_alloc(&optee_ffa_clnt_desc, NULL, optee->pool,
@@ -850,8 +931,7 @@ static int optee_ffa_probe(struct ffa_device *ffa_dev)
 	if (rc)
 		goto err_unreg_supp_teedev;
 	mutex_init(&optee->ffa.mutex);
-	mutex_init(&optee->call_queue.mutex);
-	INIT_LIST_HEAD(&optee->call_queue.waiters);
+	optee_cq_init(&optee->call_queue, 0);
 	optee_supp_init(&optee->supp);
 	optee_shm_arg_cache_init(optee, arg_cache_flags);
 	ffa_dev_set_drvdata(ffa_dev, optee);
@@ -864,6 +944,12 @@ static int optee_ffa_probe(struct ffa_device *ffa_dev)
 	rc = optee_notif_init(optee, OPTEE_DEFAULT_MAX_NOTIF_VALUE);
 	if (rc)
 		goto err_close_ctx;
+	if (sec_caps & OPTEE_FFA_SEC_CAP_ASYNC_NOTIF) {
+		rc = optee_ffa_async_notif_init(ffa_dev, optee);
+		if (rc < 0)
+			pr_err("Failed to initialize async notifications: %d",
+			       rc);
+	}
 
 	rc = optee_enumerate_devices(PTA_CMD_GET_DEVICES);
 	if (rc)
@@ -874,6 +960,9 @@ static int optee_ffa_probe(struct ffa_device *ffa_dev)
 
 err_unregister_devices:
 	optee_unregister_devices();
+	if (optee->ffa.bottom_half_value != U32_MAX)
+		notif_ops->notify_relinquish(ffa_dev,
+					     optee->ffa.bottom_half_value);
 	optee_notif_uninit(optee);
 err_close_ctx:
 	teedev_close_context(ctx);
diff --git a/drivers/tee/optee/optee_ffa.h b/drivers/tee/optee/optee_ffa.h
index 97266243deaa..5db779dc00de 100644
--- a/drivers/tee/optee/optee_ffa.h
+++ b/drivers/tee/optee/optee_ffa.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: BSD-2-Clause */
 /*
- * Copyright (c) 2019-2021, Linaro Limited
+ * Copyright (c) 2019-2021, 2023 Linaro Limited
  */
 
 /*
@@ -73,7 +73,7 @@
  *
  * Call register usage:
  * w3:    Service ID, OPTEE_FFA_EXCHANGE_CAPABILITIES
- * w4-w7: Note used (MBZ)
+ * w4-w7: Not used (MBZ)
  *
  * Return register usage:
  * w3:    Error code, 0 on success
@@ -82,14 +82,16 @@
  *                   OPTEE_FFA_YIELDING_CALL_WITH_ARG.
  *        Bit[31:8]: Reserved (MBZ)
  * w5:	  Bitfield of secure world capabilities OPTEE_FFA_SEC_CAP_* below,
- *	  unused bits MBZ.
- * w6-w7: Not used (MBZ)
+ * w6:	  The maximum secure world notification number
+ * w7:	  Not used (MBZ)
  */
 /*
  * Secure world supports giving an offset into the argument shared memory
  * object, see also OPTEE_FFA_YIELDING_CALL_WITH_ARG
  */
 #define OPTEE_FFA_SEC_CAP_ARG_OFFSET	BIT(0)
+/* OP-TEE supports asynchronous notification via FF-A */
+#define OPTEE_FFA_SEC_CAP_ASYNC_NOTIF	BIT(1)
 
 #define OPTEE_FFA_EXCHANGE_CAPABILITIES OPTEE_FFA_BLOCKING_CALL(2)
 
@@ -108,6 +110,24 @@
  */
 #define OPTEE_FFA_UNREGISTER_SHM	OPTEE_FFA_BLOCKING_CALL(3)
 
+/*
+ * Inform OP-TEE that the normal world is able to receive asynchronous
+ * notifications.
+ *
+ * Call register usage:
+ * w3:    Service ID, OPTEE_FFA_ENABLE_ASYNC_NOTIF
+ * w4:	  Notification value to request bottom half processing, should be
+ *	  less than OPTEE_FFA_MAX_ASYNC_NOTIF_VALUE.
+ * w5-w7: Not used (MBZ)
+ *
+ * Return register usage:
+ * w3:    Error code, 0 on success
+ * w4-w7: Note used (MBZ)
+ */
+#define OPTEE_FFA_ENABLE_ASYNC_NOTIF	OPTEE_FFA_BLOCKING_CALL(5)
+
+#define OPTEE_FFA_MAX_ASYNC_NOTIF_VALUE 64
+
 /*
  * Call with struct optee_msg_arg as argument in the supplied shared memory
  * with a zero internal offset and normal cached memory attributes.
diff --git a/drivers/tee/optee/optee_private.h b/drivers/tee/optee/optee_private.h
index 6bb5cae09688..7a5243c78b55 100644
--- a/drivers/tee/optee/optee_private.h
+++ b/drivers/tee/optee/optee_private.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * Copyright (c) 2015-2021, Linaro Limited
+ * Copyright (c) 2015-2021, 2023 Linaro Limited
  */
 
 #ifndef OPTEE_PRIVATE_H
@@ -40,15 +40,33 @@ typedef void (optee_invoke_fn)(unsigned long, unsigned long, unsigned long,
 				unsigned long, unsigned long,
 				struct arm_smccc_res *);
 
+/*
+ * struct optee_call_waiter - TEE entry may need to wait for a free TEE thread
+ * @list_node		Reference in waiters list
+ * @c			Waiting completion reference
+ * @sys_thread		True if waiter belongs to a system thread
+ */
 struct optee_call_waiter {
 	struct list_head list_node;
 	struct completion c;
+	bool sys_thread;
 };
 
+/*
+ * struct optee_call_queue - OP-TEE call queue management
+ * @mutex			Serializes access to this struct
+ * @waiters			List of threads waiting to enter OP-TEE
+ * @total_thread_count		Overall number of thread context in OP-TEE or 0
+ * @free_thread_count		Number of threads context free in OP-TEE
+ * @sys_thread_req_count	Number of registered system thread sessions
+ */
 struct optee_call_queue {
 	/* Serializes access to this struct */
 	struct mutex mutex;
 	struct list_head waiters;
+	int total_thread_count;
+	int free_thread_count;
+	int sys_thread_req_count;
 };
 
 struct optee_notif {
@@ -129,12 +147,14 @@ struct optee_smc {
  * struct optee_ffa_data -  FFA communication struct
  * @ffa_dev		FFA device, contains the destination id, the id of
  *			OP-TEE in secure world
- * @ffa_ops		FFA operations
+ * @bottom_half_value	Notification ID used for bottom half signalling or
+ *			U32_MAX if unused
  * @mutex		Serializes access to @global_ids
  * @global_ids		FF-A shared memory global handle translation
  */
 struct optee_ffa {
 	struct ffa_device *ffa_dev;
+	u32 bottom_half_value;
 	/* Serializes access to @global_ids */
 	struct mutex mutex;
 	struct rhashtable global_ids;
@@ -154,7 +174,8 @@ struct optee;
  */
 struct optee_ops {
 	int (*do_call_with_arg)(struct tee_context *ctx,
-				struct tee_shm *shm_arg, u_int offs);
+				struct tee_shm *shm_arg, u_int offs,
+				bool system_thread);
 	int (*to_msg_param)(struct optee *optee,
 			    struct optee_msg_param *msg_params,
 			    size_t num_params, const struct tee_param *params);
@@ -178,7 +199,6 @@ struct optee_ops {
  * @pool:		shared memory pool
  * @rpc_param_count:	If > 0 number of RPC parameters to make room for
  * @scan_bus_done	flag if device registation was already done.
- * @scan_bus_wq		workqueue to scan optee bus and register optee drivers
  * @scan_bus_work	workq to scan optee bus and register optee drivers
  */
 struct optee {
@@ -197,13 +217,13 @@ struct optee {
 	struct tee_shm_pool *pool;
 	unsigned int rpc_param_count;
 	bool   scan_bus_done;
-	struct workqueue_struct *scan_bus_wq;
 	struct work_struct scan_bus_work;
 };
 
 struct optee_session {
 	struct list_head list_node;
 	u32 session_id;
+	bool use_sys_thread;
 };
 
 struct optee_context_data {
@@ -250,7 +270,9 @@ int optee_supp_send(struct tee_context *ctx, u32 ret, u32 num_params,
 int optee_open_session(struct tee_context *ctx,
 		       struct tee_ioctl_open_session_arg *arg,
 		       struct tee_param *param);
-int optee_close_session_helper(struct tee_context *ctx, u32 session);
+int optee_system_session(struct tee_context *ctx, u32 session);
+int optee_close_session_helper(struct tee_context *ctx, u32 session,
+			       bool system_thread);
 int optee_close_session(struct tee_context *ctx, u32 session);
 int optee_invoke_func(struct tee_context *ctx, struct tee_ioctl_invoke_arg *arg,
 		      struct tee_param *param);
@@ -298,8 +320,9 @@ static inline void optee_to_msg_param_value(struct optee_msg_param *mp,
 	mp->u.value.c = p->u.value.c;
 }
 
+void optee_cq_init(struct optee_call_queue *cq, int thread_count);
 void optee_cq_wait_init(struct optee_call_queue *cq,
-			struct optee_call_waiter *w);
+			struct optee_call_waiter *w, bool sys_thread);
 void optee_cq_wait_for_completion(struct optee_call_queue *cq,
 				  struct optee_call_waiter *w);
 void optee_cq_wait_final(struct optee_call_queue *cq,
@@ -323,6 +346,9 @@ void optee_rpc_cmd_free_suppl(struct tee_context *ctx, struct tee_shm *shm);
 void optee_rpc_cmd(struct tee_context *ctx, struct optee *optee,
 		   struct optee_msg_arg *arg);
 
+int optee_do_bottom_half(struct tee_context *ctx);
+int optee_stop_async_notif(struct tee_context *ctx);
+
 /*
  * Small helpers
  */
diff --git a/drivers/tee/optee/smc_abi.c b/drivers/tee/optee/smc_abi.c
index d5b28fd35d66..7c17679b9fa7 100644
--- a/drivers/tee/optee/smc_abi.c
+++ b/drivers/tee/optee/smc_abi.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (c) 2015-2021, Linaro Limited
+ * Copyright (c) 2015-2021, 2023 Linaro Limited
  * Copyright (c) 2016, EPAM Systems
  */
 
@@ -283,7 +283,7 @@ static void optee_enable_shm_cache(struct optee *optee)
 	struct optee_call_waiter w;
 
 	/* We need to retry until secure world isn't busy. */
-	optee_cq_wait_init(&optee->call_queue, &w);
+	optee_cq_wait_init(&optee->call_queue, &w, false);
 	while (true) {
 		struct arm_smccc_res res;
 
@@ -308,7 +308,7 @@ static void __optee_disable_shm_cache(struct optee *optee, bool is_mapped)
 	struct optee_call_waiter w;
 
 	/* We need to retry until secure world isn't busy. */
-	optee_cq_wait_init(&optee->call_queue, &w);
+	optee_cq_wait_init(&optee->call_queue, &w, false);
 	while (true) {
 		union {
 			struct arm_smccc_res smccc;
@@ -507,7 +507,7 @@ static int optee_shm_register(struct tee_context *ctx, struct tee_shm *shm,
 	msg_arg->params->u.tmem.buf_ptr = virt_to_phys(pages_list) |
 	  (tee_shm_get_page_offset(shm) & (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1));
 
-	if (optee->ops->do_call_with_arg(ctx, shm_arg, 0) ||
+	if (optee->ops->do_call_with_arg(ctx, shm_arg, 0, false) ||
 	    msg_arg->ret != TEEC_SUCCESS)
 		rc = -EINVAL;
 
@@ -550,7 +550,7 @@ static int optee_shm_unregister(struct tee_context *ctx, struct tee_shm *shm)
 	msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;
 	msg_arg->params[0].u.rmem.shm_ref = (unsigned long)shm;
 
-	if (optee->ops->do_call_with_arg(ctx, shm_arg, 0) ||
+	if (optee->ops->do_call_with_arg(ctx, shm_arg, 0, false) ||
 	    msg_arg->ret != TEEC_SUCCESS)
 		rc = -EINVAL;
 out:
@@ -806,6 +806,7 @@ static void handle_rpc_func_cmd(struct tee_context *ctx, struct optee *optee,
 /**
  * optee_handle_rpc() - handle RPC from secure world
  * @ctx:	context doing the RPC
+ * @rpc_arg:	pointer to RPC arguments if any, or NULL if none
  * @param:	value of registers for the RPC
  * @call_ctx:	call context. Preserved during one OP-TEE invocation
  *
@@ -878,6 +879,7 @@ static void optee_handle_rpc(struct tee_context *ctx,
  * @ctx:	calling context
  * @shm:	shared memory holding the message to pass to secure world
  * @offs:	offset of the message in @shm
+ * @system_thread: true if caller requests TEE system thread support
  *
  * Does and SMC to OP-TEE in secure world and handles eventual resulting
  * Remote Procedure Calls (RPC) from OP-TEE.
@@ -885,7 +887,8 @@ static void optee_handle_rpc(struct tee_context *ctx,
  * Returns return code from secure world, 0 is OK
  */
 static int optee_smc_do_call_with_arg(struct tee_context *ctx,
-				      struct tee_shm *shm, u_int offs)
+				      struct tee_shm *shm, u_int offs,
+				      bool system_thread)
 {
 	struct optee *optee = tee_get_drvdata(ctx->teedev);
 	struct optee_call_waiter w;
@@ -926,7 +929,7 @@ static int optee_smc_do_call_with_arg(struct tee_context *ctx,
 		reg_pair_from_64(&param.a1, &param.a2, parg);
 	}
 	/* Initialize waiter */
-	optee_cq_wait_init(&optee->call_queue, &w);
+	optee_cq_wait_init(&optee->call_queue, &w, system_thread);
 	while (true) {
 		struct arm_smccc_res res;
 
@@ -965,34 +968,6 @@ static int optee_smc_do_call_with_arg(struct tee_context *ctx,
 	return rc;
 }
 
-static int simple_call_with_arg(struct tee_context *ctx, u32 cmd)
-{
-	struct optee_shm_arg_entry *entry;
-	struct optee_msg_arg *msg_arg;
-	struct tee_shm *shm;
-	u_int offs;
-
-	msg_arg = optee_get_msg_arg(ctx, 0, &entry, &shm, &offs);
-	if (IS_ERR(msg_arg))
-		return PTR_ERR(msg_arg);
-
-	msg_arg->cmd = cmd;
-	optee_smc_do_call_with_arg(ctx, shm, offs);
-
-	optee_free_msg_arg(ctx, entry, offs);
-	return 0;
-}
-
-static int optee_smc_do_bottom_half(struct tee_context *ctx)
-{
-	return simple_call_with_arg(ctx, OPTEE_MSG_CMD_DO_BOTTOM_HALF);
-}
-
-static int optee_smc_stop_async_notif(struct tee_context *ctx)
-{
-	return simple_call_with_arg(ctx, OPTEE_MSG_CMD_STOP_ASYNC_NOTIF);
-}
-
 /*
  * 5. Asynchronous notification
  */
@@ -1048,7 +1023,7 @@ static irqreturn_t notif_irq_thread_fn(int irq, void *dev_id)
 {
 	struct optee *optee = dev_id;
 
-	optee_smc_do_bottom_half(optee->ctx);
+	optee_do_bottom_half(optee->ctx);
 
 	return IRQ_HANDLED;
 }
@@ -1086,7 +1061,7 @@ static void notif_pcpu_irq_work_fn(struct work_struct *work)
 						   notif_pcpu_work);
 	struct optee *optee = container_of(optee_smc, struct optee, smc);
 
-	optee_smc_do_bottom_half(optee->ctx);
+	optee_do_bottom_half(optee->ctx);
 }
 
 static int init_pcpu_irq(struct optee *optee, u_int irq)
@@ -1158,7 +1133,7 @@ static void uninit_pcpu_irq(struct optee *optee)
 static void optee_smc_notif_uninit_irq(struct optee *optee)
 {
 	if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) {
-		optee_smc_stop_async_notif(optee->ctx);
+		optee_stop_async_notif(optee->ctx);
 		if (optee->smc.notif_irq) {
 			if (irq_is_percpu_devid(optee->smc.notif_irq))
 				uninit_pcpu_irq(optee);
@@ -1210,6 +1185,7 @@ static const struct tee_driver_ops optee_clnt_ops = {
 	.release = optee_release,
 	.open_session = optee_open_session,
 	.close_session = optee_close_session,
+	.system_session = optee_system_session,
 	.invoke_func = optee_invoke_func,
 	.cancel_req = optee_cancel_req,
 	.shm_register = optee_shm_register,
@@ -1357,6 +1333,16 @@ static bool optee_msg_exchange_capabilities(optee_invoke_fn *invoke_fn,
 	return true;
 }
 
+static unsigned int optee_msg_get_thread_count(optee_invoke_fn *invoke_fn)
+{
+	struct arm_smccc_res res;
+
+	invoke_fn(OPTEE_SMC_GET_THREAD_COUNT, 0, 0, 0, 0, 0, 0, 0, &res);
+	if (res.a0)
+		return 0;
+	return res.a1;
+}
+
 static struct tee_shm_pool *
 optee_config_shm_memremap(optee_invoke_fn *invoke_fn, void **memremaped_shm)
 {
@@ -1609,6 +1595,7 @@ static int optee_probe(struct platform_device *pdev)
 	struct optee *optee = NULL;
 	void *memremaped_shm = NULL;
 	unsigned int rpc_param_count;
+	unsigned int thread_count;
 	struct tee_device *teedev;
 	struct tee_context *ctx;
 	u32 max_notif_value;
@@ -1636,6 +1623,7 @@ static int optee_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
+	thread_count = optee_msg_get_thread_count(invoke_fn);
 	if (!optee_msg_exchange_capabilities(invoke_fn, &sec_caps,
 					     &max_notif_value,
 					     &rpc_param_count)) {
@@ -1725,8 +1713,7 @@ static int optee_probe(struct platform_device *pdev)
 	if (rc)
 		goto err_unreg_supp_teedev;
 
-	mutex_init(&optee->call_queue.mutex);
-	INIT_LIST_HEAD(&optee->call_queue.waiters);
+	optee_cq_init(&optee->call_queue, thread_count);
 	optee_supp_init(&optee->supp);
 	optee->smc.memremaped_shm = memremaped_shm;
 	optee->pool = pool;
diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c
index 5ddfd5d9ac7f..792d6fae4354 100644
--- a/drivers/tee/tee_core.c
+++ b/drivers/tee/tee_core.c
@@ -1173,6 +1173,14 @@ int tee_client_close_session(struct tee_context *ctx, u32 session)
 }
 EXPORT_SYMBOL_GPL(tee_client_close_session);
 
+int tee_client_system_session(struct tee_context *ctx, u32 session)
+{
+	if (!ctx->teedev->desc->ops->system_session)
+		return -EINVAL;
+	return ctx->teedev->desc->ops->system_session(ctx, session);
+}
+EXPORT_SYMBOL_GPL(tee_client_system_session);
+
 int tee_client_invoke_func(struct tee_context *ctx,
 			   struct tee_ioctl_invoke_arg *arg,
 			   struct tee_param *param)
diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c
index 9c17d35ccbbd..625ba07cbe2f 100644
--- a/drivers/thermal/thermal_core.c
+++ b/drivers/thermal/thermal_core.c
@@ -345,22 +345,51 @@ static void handle_critical_trips(struct thermal_zone_device *tz,
 }
 
 static void handle_thermal_trip(struct thermal_zone_device *tz,
-				const struct thermal_trip *trip)
+				struct thermal_trip *trip)
 {
 	if (trip->temperature == THERMAL_TEMP_INVALID)
 		return;
 
-	if (tz->last_temperature != THERMAL_TEMP_INVALID) {
-		if (tz->last_temperature < trip->temperature &&
-		    tz->temperature >= trip->temperature)
+	if (tz->last_temperature == THERMAL_TEMP_INVALID) {
+		/* Initialization. */
+		trip->threshold = trip->temperature;
+		if (tz->temperature >= trip->threshold)
+			trip->threshold -= trip->hysteresis;
+	} else if (tz->last_temperature < trip->threshold) {
+		/*
+		 * The trip threshold is equal to the trip temperature, unless
+		 * the latter has changed in the meantime.  In either case,
+		 * the trip is crossed if the current zone temperature is at
+		 * least equal to its temperature, but otherwise ensure that
+		 * the threshold and the trip temperature will be equal.
+		 */
+		if (tz->temperature >= trip->temperature) {
 			thermal_notify_tz_trip_up(tz->id,
 						  thermal_zone_trip_id(tz, trip),
 						  tz->temperature);
-		if (tz->last_temperature >= trip->temperature &&
-		    tz->temperature < trip->temperature - trip->hysteresis)
+			trip->threshold = trip->temperature - trip->hysteresis;
+		} else {
+			trip->threshold = trip->temperature;
+		}
+	} else {
+		/*
+		 * The previous zone temperature was above or equal to the trip
+		 * threshold, which would be equal to the "low temperature" of
+		 * the trip (its temperature minus its hysteresis), unless the
+		 * trip temperature or hysteresis had changed.  In either case,
+		 * the trip is crossed if the current zone temperature is below
+		 * the low temperature of the trip, but otherwise ensure that
+		 * the trip threshold will be equal to the low temperature of
+		 * the trip.
+		 */
+		if (tz->temperature < trip->temperature - trip->hysteresis) {
 			thermal_notify_tz_trip_down(tz->id,
 						    thermal_zone_trip_id(tz, trip),
 						    tz->temperature);
+			trip->threshold = trip->temperature;
+		} else {
+			trip->threshold = trip->temperature - trip->hysteresis;
+		}
 	}
 
 	if (trip->type == THERMAL_TRIP_CRITICAL || trip->type == THERMAL_TRIP_HOT)
@@ -403,7 +432,7 @@ static void thermal_zone_device_init(struct thermal_zone_device *tz)
 void __thermal_zone_device_update(struct thermal_zone_device *tz,
 				  enum thermal_notify_event event)
 {
-	const struct thermal_trip *trip;
+	struct thermal_trip *trip;
 
 	if (atomic_read(&in_suspend))
 		return;
diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c
index 1e15ffa79295..44e9b09de47a 100644
--- a/drivers/thunderbolt/switch.c
+++ b/drivers/thunderbolt/switch.c
@@ -1143,7 +1143,7 @@ int tb_port_lane_bonding_enable(struct tb_port *port)
 	 * Only set bonding if the link was not already bonded. This
 	 * avoids the lane adapter to re-enter bonding state.
 	 */
-	if (width == TB_LINK_WIDTH_SINGLE) {
+	if (width == TB_LINK_WIDTH_SINGLE && !tb_is_upstream_port(port)) {
 		ret = tb_port_set_lane_bonding(port, true);
 		if (ret)
 			goto err_lane1;
@@ -2880,6 +2880,7 @@ static int tb_switch_lane_bonding_disable(struct tb_switch *sw)
 	return tb_port_wait_for_link_width(down, TB_LINK_WIDTH_SINGLE, 100);
 }
 
+/* Note updating sw->link_width done in tb_switch_update_link_attributes() */
 static int tb_switch_asym_enable(struct tb_switch *sw, enum tb_link_width width)
 {
 	struct tb_port *up, *down, *port;
@@ -2919,10 +2920,10 @@ static int tb_switch_asym_enable(struct tb_switch *sw, enum tb_link_width width)
 			return ret;
 	}
 
-	sw->link_width = width;
 	return 0;
 }
 
+/* Note updating sw->link_width done in tb_switch_update_link_attributes() */
 static int tb_switch_asym_disable(struct tb_switch *sw)
 {
 	struct tb_port *up, *down;
@@ -2957,7 +2958,6 @@ static int tb_switch_asym_disable(struct tb_switch *sw)
 			return ret;
 	}
 
-	sw->link_width = TB_LINK_WIDTH_DUAL;
 	return 0;
 }
 
diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c
index 5acdeb766860..fd49f86e0353 100644
--- a/drivers/thunderbolt/tb.c
+++ b/drivers/thunderbolt/tb.c
@@ -213,7 +213,17 @@ static void tb_add_dp_resources(struct tb_switch *sw)
 		if (!tb_switch_query_dp_resource(sw, port))
 			continue;
 
-		list_add(&port->list, &tcm->dp_resources);
+		/*
+		 * If DP IN on device router exist, position it at the
+		 * beginning of the DP resources list, so that it is used
+		 * before DP IN of the host router. This way external GPU(s)
+		 * will be prioritized when pairing DP IN to a DP OUT.
+		 */
+		if (tb_route(sw))
+			list_add(&port->list, &tcm->dp_resources);
+		else
+			list_add_tail(&port->list, &tcm->dp_resources);
+
 		tb_port_dbg(port, "DP IN resource available\n");
 	}
 }
diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c
index 7534cd3a81f4..8d45cb4f04ab 100644
--- a/drivers/thunderbolt/tunnel.c
+++ b/drivers/thunderbolt/tunnel.c
@@ -1313,8 +1313,6 @@ static void tb_dp_dump(struct tb_tunnel *tunnel)
 		      "DP IN maximum supported bandwidth %u Mb/s x%u = %u Mb/s\n",
 		      rate, lanes, tb_dp_bandwidth(rate, lanes));
 
-	out = tunnel->dst_port;
-
 	if (tb_port_read(out, &dp_cap, TB_CFG_PORT,
 			 out->cap_adap + DP_LOCAL_CAP, 1))
 		return;
diff --git a/drivers/ufs/core/ufs-sysfs.c b/drivers/ufs/core/ufs-sysfs.c
index c95906443d5f..05b10ca90b50 100644
--- a/drivers/ufs/core/ufs-sysfs.c
+++ b/drivers/ufs/core/ufs-sysfs.c
@@ -7,9 +7,56 @@
 #include <asm/unaligned.h>
 
 #include <ufs/ufs.h>
+#include <ufs/unipro.h>
 #include "ufs-sysfs.h"
 #include "ufshcd-priv.h"
 
+static const char *ufs_pa_pwr_mode_to_string(enum ufs_pa_pwr_mode mode)
+{
+	switch (mode) {
+	case FAST_MODE:		return "FAST_MODE";
+	case SLOW_MODE:		return "SLOW_MODE";
+	case FASTAUTO_MODE:	return "FASTAUTO_MODE";
+	case SLOWAUTO_MODE:	return "SLOWAUTO_MODE";
+	default:		return "UNKNOWN";
+	}
+}
+
+static const char *ufs_hs_gear_rate_to_string(enum ufs_hs_gear_rate rate)
+{
+	switch (rate) {
+	case PA_HS_MODE_A:	return "HS_RATE_A";
+	case PA_HS_MODE_B:	return "HS_RATE_B";
+	default:		return "UNKNOWN";
+	}
+}
+
+static const char *ufs_pwm_gear_to_string(enum ufs_pwm_gear_tag gear)
+{
+	switch (gear) {
+	case UFS_PWM_G1:	return "PWM_GEAR1";
+	case UFS_PWM_G2:	return "PWM_GEAR2";
+	case UFS_PWM_G3:	return "PWM_GEAR3";
+	case UFS_PWM_G4:	return "PWM_GEAR4";
+	case UFS_PWM_G5:	return "PWM_GEAR5";
+	case UFS_PWM_G6:	return "PWM_GEAR6";
+	case UFS_PWM_G7:	return "PWM_GEAR7";
+	default:		return "UNKNOWN";
+	}
+}
+
+static const char *ufs_hs_gear_to_string(enum ufs_hs_gear_tag gear)
+{
+	switch (gear) {
+	case UFS_HS_G1:	return "HS_GEAR1";
+	case UFS_HS_G2:	return "HS_GEAR2";
+	case UFS_HS_G3:	return "HS_GEAR3";
+	case UFS_HS_G4:	return "HS_GEAR4";
+	case UFS_HS_G5:	return "HS_GEAR5";
+	default:	return "UNKNOWN";
+	}
+}
+
 static const char *ufshcd_uic_link_state_to_string(
 			enum uic_link_state state)
 {
@@ -628,6 +675,78 @@ static const struct attribute_group ufs_sysfs_monitor_group = {
 	.attrs = ufs_sysfs_monitor_attrs,
 };
 
+static ssize_t lane_show(struct device *dev, struct device_attribute *attr,
+			 char *buf)
+{
+	struct ufs_hba *hba = dev_get_drvdata(dev);
+
+	return sysfs_emit(buf, "%u\n", hba->pwr_info.lane_rx);
+}
+
+static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
+			 char *buf)
+{
+	struct ufs_hba *hba = dev_get_drvdata(dev);
+
+	return sysfs_emit(buf, "%s\n", ufs_pa_pwr_mode_to_string(hba->pwr_info.pwr_rx));
+}
+
+static ssize_t rate_show(struct device *dev, struct device_attribute *attr,
+			 char *buf)
+{
+	struct ufs_hba *hba = dev_get_drvdata(dev);
+
+	return sysfs_emit(buf, "%s\n", ufs_hs_gear_rate_to_string(hba->pwr_info.hs_rate));
+}
+
+static ssize_t gear_show(struct device *dev, struct device_attribute *attr,
+			 char *buf)
+{
+	struct ufs_hba *hba = dev_get_drvdata(dev);
+
+	return sysfs_emit(buf, "%s\n", hba->pwr_info.hs_rate ?
+			  ufs_hs_gear_to_string(hba->pwr_info.gear_rx) :
+			  ufs_pwm_gear_to_string(hba->pwr_info.gear_rx));
+}
+
+static ssize_t dev_pm_show(struct device *dev, struct device_attribute *attr,
+			   char *buf)
+{
+	struct ufs_hba *hba = dev_get_drvdata(dev);
+
+	return sysfs_emit(buf, "%s\n", ufshcd_ufs_dev_pwr_mode_to_string(hba->curr_dev_pwr_mode));
+}
+
+static ssize_t link_state_show(struct device *dev,
+			       struct device_attribute *attr, char *buf)
+{
+	struct ufs_hba *hba = dev_get_drvdata(dev);
+
+	return sysfs_emit(buf, "%s\n", ufshcd_uic_link_state_to_string(hba->uic_link_state));
+}
+
+static DEVICE_ATTR_RO(lane);
+static DEVICE_ATTR_RO(mode);
+static DEVICE_ATTR_RO(rate);
+static DEVICE_ATTR_RO(gear);
+static DEVICE_ATTR_RO(dev_pm);
+static DEVICE_ATTR_RO(link_state);
+
+static struct attribute *ufs_power_info_attrs[] = {
+	&dev_attr_lane.attr,
+	&dev_attr_mode.attr,
+	&dev_attr_rate.attr,
+	&dev_attr_gear.attr,
+	&dev_attr_dev_pm.attr,
+	&dev_attr_link_state.attr,
+	NULL
+};
+
+static const struct attribute_group ufs_sysfs_power_info_group = {
+	.name = "power_info",
+	.attrs = ufs_power_info_attrs,
+};
+
 static ssize_t ufs_sysfs_read_desc_param(struct ufs_hba *hba,
 				  enum desc_idn desc_id,
 				  u8 desc_index,
@@ -1233,6 +1352,7 @@ static const struct attribute_group *ufs_sysfs_groups[] = {
 	&ufs_sysfs_default_group,
 	&ufs_sysfs_capabilities_group,
 	&ufs_sysfs_monitor_group,
+	&ufs_sysfs_power_info_group,
 	&ufs_sysfs_device_descriptor_group,
 	&ufs_sysfs_interconnect_descriptor_group,
 	&ufs_sysfs_geometry_descriptor_group,
diff --git a/drivers/usb/cdns3/cdnsp-ring.c b/drivers/usb/cdns3/cdnsp-ring.c
index af981778382d..02f297f5637d 100644
--- a/drivers/usb/cdns3/cdnsp-ring.c
+++ b/drivers/usb/cdns3/cdnsp-ring.c
@@ -1529,6 +1529,7 @@ irqreturn_t cdnsp_thread_irq_handler(int irq, void *data)
 	unsigned long flags;
 	int counter = 0;
 
+	local_bh_disable();
 	spin_lock_irqsave(&pdev->lock, flags);
 
 	if (pdev->cdnsp_state & (CDNSP_STATE_HALTED | CDNSP_STATE_DYING)) {
@@ -1541,6 +1542,7 @@ irqreturn_t cdnsp_thread_irq_handler(int irq, void *data)
 			cdnsp_died(pdev);
 
 		spin_unlock_irqrestore(&pdev->lock, flags);
+		local_bh_enable();
 		return IRQ_HANDLED;
 	}
 
@@ -1557,6 +1559,7 @@ irqreturn_t cdnsp_thread_irq_handler(int irq, void *data)
 	cdnsp_update_erst_dequeue(pdev, event_ring_deq, 1);
 
 	spin_unlock_irqrestore(&pdev->lock, flags);
+	local_bh_enable();
 
 	return IRQ_HANDLED;
 }
diff --git a/drivers/usb/core/config.c b/drivers/usb/core/config.c
index b19e38d5fd10..7f8d33f92ddb 100644
--- a/drivers/usb/core/config.c
+++ b/drivers/usb/core/config.c
@@ -1047,7 +1047,7 @@ int usb_get_bos_descriptor(struct usb_device *dev)
 
 		if (cap->bDescriptorType != USB_DT_DEVICE_CAPABILITY) {
 			dev_notice(ddev, "descriptor type invalid, skip\n");
-			continue;
+			goto skip_to_next_descriptor;
 		}
 
 		switch (cap_type) {
@@ -1078,6 +1078,7 @@ int usb_get_bos_descriptor(struct usb_device *dev)
 			break;
 		}
 
+skip_to_next_descriptor:
 		total_len -= length;
 		buffer += length;
 	}
diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c
index b4584a0cd484..87480a6e6d93 100644
--- a/drivers/usb/core/hub.c
+++ b/drivers/usb/core/hub.c
@@ -622,29 +622,6 @@ static int hub_ext_port_status(struct usb_hub *hub, int port1, int type,
 		ret = 0;
 	}
 	mutex_unlock(&hub->status_mutex);
-
-	/*
-	 * There is no need to lock status_mutex here, because status_mutex
-	 * protects hub->status, and the phy driver only checks the port
-	 * status without changing the status.
-	 */
-	if (!ret) {
-		struct usb_device *hdev = hub->hdev;
-
-		/*
-		 * Only roothub will be notified of port state changes,
-		 * since the USB PHY only cares about changes at the next
-		 * level.
-		 */
-		if (is_root_hub(hdev)) {
-			struct usb_hcd *hcd = bus_to_hcd(hdev->bus);
-
-			if (hcd->usb_phy)
-				usb_phy_notify_port_status(hcd->usb_phy,
-							   port1 - 1, *status, *change);
-		}
-	}
-
 	return ret;
 }
 
diff --git a/drivers/usb/dwc2/hcd_intr.c b/drivers/usb/dwc2/hcd_intr.c
index 0144ca8350c3..5c7538d498dd 100644
--- a/drivers/usb/dwc2/hcd_intr.c
+++ b/drivers/usb/dwc2/hcd_intr.c
@@ -2015,15 +2015,17 @@ static void dwc2_hc_n_intr(struct dwc2_hsotg *hsotg, int chnum)
 {
 	struct dwc2_qtd *qtd;
 	struct dwc2_host_chan *chan;
-	u32 hcint, hcintmsk;
+	u32 hcint, hcintraw, hcintmsk;
 
 	chan = hsotg->hc_ptr_array[chnum];
 
-	hcint = dwc2_readl(hsotg, HCINT(chnum));
+	hcintraw = dwc2_readl(hsotg, HCINT(chnum));
 	hcintmsk = dwc2_readl(hsotg, HCINTMSK(chnum));
+	hcint = hcintraw & hcintmsk;
+	dwc2_writel(hsotg, hcint, HCINT(chnum));
+
 	if (!chan) {
 		dev_err(hsotg->dev, "## hc_ptr_array for channel is NULL ##\n");
-		dwc2_writel(hsotg, hcint, HCINT(chnum));
 		return;
 	}
 
@@ -2032,11 +2034,9 @@ static void dwc2_hc_n_intr(struct dwc2_hsotg *hsotg, int chnum)
 			 chnum);
 		dev_vdbg(hsotg->dev,
 			 "  hcint 0x%08x, hcintmsk 0x%08x, hcint&hcintmsk 0x%08x\n",
-			 hcint, hcintmsk, hcint & hcintmsk);
+			 hcintraw, hcintmsk, hcint);
 	}
 
-	dwc2_writel(hsotg, hcint, HCINT(chnum));
-
 	/*
 	 * If we got an interrupt after someone called
 	 * dwc2_hcd_endpoint_disable() we don't want to crash below
@@ -2046,8 +2046,7 @@ static void dwc2_hc_n_intr(struct dwc2_hsotg *hsotg, int chnum)
 		return;
 	}
 
-	chan->hcint = hcint;
-	hcint &= hcintmsk;
+	chan->hcint = hcintraw;
 
 	/*
 	 * If the channel was halted due to a dequeue, the qtd list might
diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c
index 0328c86ef806..b101dbf8c5dc 100644
--- a/drivers/usb/dwc3/core.c
+++ b/drivers/usb/dwc3/core.c
@@ -2034,6 +2034,8 @@ static int dwc3_probe(struct platform_device *pdev)
 
 	pm_runtime_put(dev);
 
+	dma_set_max_seg_size(dev, UINT_MAX);
+
 	return 0;
 
 err_exit_debugfs:
diff --git a/drivers/usb/dwc3/drd.c b/drivers/usb/dwc3/drd.c
index 039bf241769a..57ddd2e43022 100644
--- a/drivers/usb/dwc3/drd.c
+++ b/drivers/usb/dwc3/drd.c
@@ -505,6 +505,7 @@ static int dwc3_setup_role_switch(struct dwc3 *dwc)
 		dwc->role_switch_default_mode = USB_DR_MODE_PERIPHERAL;
 		mode = DWC3_GCTL_PRTCAP_DEVICE;
 	}
+	dwc3_set_mode(dwc, mode);
 
 	dwc3_role_switch.fwnode = dev_fwnode(dwc->dev);
 	dwc3_role_switch.set = dwc3_usb_role_switch_set;
@@ -526,7 +527,6 @@ static int dwc3_setup_role_switch(struct dwc3 *dwc)
 		}
 	}
 
-	dwc3_set_mode(dwc, mode);
 	return 0;
 }
 #else
diff --git a/drivers/usb/dwc3/dwc3-rtk.c b/drivers/usb/dwc3/dwc3-rtk.c
index 590028e8fdcb..3cd6b184551c 100644
--- a/drivers/usb/dwc3/dwc3-rtk.c
+++ b/drivers/usb/dwc3/dwc3-rtk.c
@@ -183,10 +183,13 @@ static enum usb_device_speed __get_dwc3_maximum_speed(struct device_node *np)
 
 	ret = of_property_read_string(dwc3_np, "maximum-speed", &maximum_speed);
 	if (ret < 0)
-		return USB_SPEED_UNKNOWN;
+		goto out;
 
 	ret = match_string(speed_names, ARRAY_SIZE(speed_names), maximum_speed);
 
+out:
+	of_node_put(dwc3_np);
+
 	return (ret < 0) ? USB_SPEED_UNKNOWN : ret;
 }
 
@@ -339,6 +342,9 @@ static int dwc3_rtk_probe_dwc3_core(struct dwc3_rtk *rtk)
 
 	switch_usb2_role(rtk, rtk->cur_role);
 
+	platform_device_put(dwc3_pdev);
+	of_node_put(dwc3_node);
+
 	return 0;
 
 err_pdev_put:
diff --git a/drivers/usb/gadget/udc/max3420_udc.c b/drivers/usb/gadget/udc/max3420_udc.c
index 2d57786d3db7..89e8cf2a2a7d 100644
--- a/drivers/usb/gadget/udc/max3420_udc.c
+++ b/drivers/usb/gadget/udc/max3420_udc.c
@@ -1201,7 +1201,7 @@ static int max3420_probe(struct spi_device *spi)
 	int err, irq;
 	u8 reg[8];
 
-	if (spi->master->flags & SPI_MASTER_HALF_DUPLEX) {
+	if (spi->master->flags & SPI_CONTROLLER_HALF_DUPLEX) {
 		dev_err(&spi->dev, "UDC needs full duplex to work\n");
 		return -EINVAL;
 	}
diff --git a/drivers/usb/host/xhci-mtk-sch.c b/drivers/usb/host/xhci-mtk-sch.c
index 5b3cd455adec..61f3f8bbdcea 100644
--- a/drivers/usb/host/xhci-mtk-sch.c
+++ b/drivers/usb/host/xhci-mtk-sch.c
@@ -650,9 +650,8 @@ static int check_isoc_ss_overlap(struct mu3h_sch_ep_info *sch_ep, u32 offset)
 
 		if (sch_ep->ep_type == ISOC_OUT_EP) {
 			for (j = 0; j < sch_ep->num_budget_microframes; j++) {
-				k = XHCI_MTK_BW_INDEX(base + j + CS_OFFSET);
-				/* use cs to indicate existence of in-ss @(base+j) */
-				if (tt->fs_bus_bw_in[k])
+				k = XHCI_MTK_BW_INDEX(base + j);
+				if (tt->in_ss_cnt[k])
 					return -ESCH_SS_OVERLAP;
 			}
 		} else if (sch_ep->ep_type == ISOC_IN_EP || sch_ep->ep_type == INT_IN_EP) {
@@ -769,6 +768,14 @@ static void update_sch_tt(struct mu3h_sch_ep_info *sch_ep, bool used)
 				tt->fs_frame_bw[f] -= (u16)sch_ep->bw_budget_table[j];
 			}
 		}
+
+		if (sch_ep->ep_type == ISOC_IN_EP || sch_ep->ep_type == INT_IN_EP) {
+			k = XHCI_MTK_BW_INDEX(base);
+			if (used)
+				tt->in_ss_cnt[k]++;
+			else
+				tt->in_ss_cnt[k]--;
+		}
 	}
 
 	if (used)
diff --git a/drivers/usb/host/xhci-mtk.h b/drivers/usb/host/xhci-mtk.h
index 865b55e23b15..39f7ae7d3087 100644
--- a/drivers/usb/host/xhci-mtk.h
+++ b/drivers/usb/host/xhci-mtk.h
@@ -38,6 +38,7 @@
  * @fs_bus_bw_in: save bandwidth used by FS/LS IN eps in each uframes
  * @ls_bus_bw: save bandwidth used by LS eps in each uframes
  * @fs_frame_bw: save bandwidth used by FS/LS eps in each FS frames
+ * @in_ss_cnt: the count of Start-Split for IN eps
  * @ep_list: Endpoints using this TT
  */
 struct mu3h_sch_tt {
@@ -45,6 +46,7 @@ struct mu3h_sch_tt {
 	u16 fs_bus_bw_in[XHCI_MTK_MAX_ESIT];
 	u8 ls_bus_bw[XHCI_MTK_MAX_ESIT];
 	u16 fs_frame_bw[XHCI_MTK_FRAMES_CNT];
+	u8 in_ss_cnt[XHCI_MTK_MAX_ESIT];
 	struct list_head ep_list;
 };
 
diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c
index b93161374293..732cdeb73920 100644
--- a/drivers/usb/host/xhci-plat.c
+++ b/drivers/usb/host/xhci-plat.c
@@ -13,6 +13,7 @@
 #include <linux/module.h>
 #include <linux/pci.h>
 #include <linux/of.h>
+#include <linux/of_device.h>
 #include <linux/platform_device.h>
 #include <linux/usb/phy.h>
 #include <linux/slab.h>
@@ -148,7 +149,7 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s
 	int			ret;
 	int			irq;
 	struct xhci_plat_priv	*priv = NULL;
-
+	bool			of_match;
 
 	if (usb_disabled())
 		return -ENODEV;
@@ -253,16 +254,23 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s
 					 &xhci->imod_interval);
 	}
 
-	hcd->usb_phy = devm_usb_get_phy_by_phandle(sysdev, "usb-phy", 0);
-	if (IS_ERR(hcd->usb_phy)) {
-		ret = PTR_ERR(hcd->usb_phy);
-		if (ret == -EPROBE_DEFER)
-			goto disable_clk;
-		hcd->usb_phy = NULL;
-	} else {
-		ret = usb_phy_init(hcd->usb_phy);
-		if (ret)
-			goto disable_clk;
+	/*
+	 * Drivers such as dwc3 manages PHYs themself (and rely on driver name
+	 * matching for the xhci platform device).
+	 */
+	of_match = of_match_device(pdev->dev.driver->of_match_table, &pdev->dev);
+	if (of_match) {
+		hcd->usb_phy = devm_usb_get_phy_by_phandle(sysdev, "usb-phy", 0);
+		if (IS_ERR(hcd->usb_phy)) {
+			ret = PTR_ERR(hcd->usb_phy);
+			if (ret == -EPROBE_DEFER)
+				goto disable_clk;
+			hcd->usb_phy = NULL;
+		} else {
+			ret = usb_phy_init(hcd->usb_phy);
+			if (ret)
+				goto disable_clk;
+		}
 	}
 
 	hcd->tpl_support = of_usb_host_tpl_support(sysdev->of_node);
@@ -285,15 +293,17 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s
 			goto dealloc_usb2_hcd;
 		}
 
-		xhci->shared_hcd->usb_phy = devm_usb_get_phy_by_phandle(sysdev,
-			    "usb-phy", 1);
-		if (IS_ERR(xhci->shared_hcd->usb_phy)) {
-			xhci->shared_hcd->usb_phy = NULL;
-		} else {
-			ret = usb_phy_init(xhci->shared_hcd->usb_phy);
-			if (ret)
-				dev_err(sysdev, "%s init usb3phy fail (ret=%d)\n",
-					    __func__, ret);
+		if (of_match) {
+			xhci->shared_hcd->usb_phy = devm_usb_get_phy_by_phandle(sysdev,
+										"usb-phy", 1);
+			if (IS_ERR(xhci->shared_hcd->usb_phy)) {
+				xhci->shared_hcd->usb_phy = NULL;
+			} else {
+				ret = usb_phy_init(xhci->shared_hcd->usb_phy);
+				if (ret)
+					dev_err(sysdev, "%s init usb3phy fail (ret=%d)\n",
+						__func__, ret);
+			}
 		}
 
 		xhci->shared_hcd->tpl_support = hcd->tpl_support;
diff --git a/drivers/usb/misc/usb-ljca.c b/drivers/usb/misc/usb-ljca.c
index c9decd0396d4..7f0deebebc13 100644
--- a/drivers/usb/misc/usb-ljca.c
+++ b/drivers/usb/misc/usb-ljca.c
@@ -457,8 +457,8 @@ static void ljca_auxdev_acpi_bind(struct ljca_adapter *adap,
 				  u64 adr, u8 id)
 {
 	struct ljca_match_ids_walk_data wd = { 0 };
-	struct acpi_device *parent, *adev;
 	struct device *dev = adap->dev;
+	struct acpi_device *parent;
 	char uid[4];
 
 	parent = ACPI_COMPANION(dev);
@@ -466,17 +466,7 @@ static void ljca_auxdev_acpi_bind(struct ljca_adapter *adap,
 		return;
 
 	/*
-	 * get auxdev ACPI handle from the ACPI device directly
-	 * under the parent that matches _ADR.
-	 */
-	adev = acpi_find_child_device(parent, adr, false);
-	if (adev) {
-		ACPI_COMPANION_SET(&auxdev->dev, adev);
-		return;
-	}
-
-	/*
-	 * _ADR is a grey area in the ACPI specification, some
+	 * Currently LJCA hw doesn't use _ADR instead the shipped
 	 * platforms use _HID to distinguish children devices.
 	 */
 	switch (adr) {
diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c
index 45dcfaadaf98..9c76095ebfe1 100644
--- a/drivers/usb/serial/option.c
+++ b/drivers/usb/serial/option.c
@@ -203,8 +203,8 @@ static void option_instat_callback(struct urb *urb);
 #define DELL_PRODUCT_5829E_ESIM			0x81e4
 #define DELL_PRODUCT_5829E			0x81e6
 
-#define DELL_PRODUCT_FM101R			0x8213
-#define DELL_PRODUCT_FM101R_ESIM		0x8215
+#define DELL_PRODUCT_FM101R_ESIM		0x8213
+#define DELL_PRODUCT_FM101R			0x8215
 
 #define KYOCERA_VENDOR_ID			0x0c88
 #define KYOCERA_PRODUCT_KPC650			0x17da
@@ -1546,7 +1546,8 @@ static const struct usb_device_id option_ids[] = {
 	{ USB_DEVICE_AND_INTERFACE_INFO(ZTE_VENDOR_ID, 0x0165, 0xff, 0xff, 0xff) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(ZTE_VENDOR_ID, 0x0167, 0xff, 0xff, 0xff),
 	  .driver_info = RSVD(4) },
-	{ USB_DEVICE_AND_INTERFACE_INFO(ZTE_VENDOR_ID, 0x0189, 0xff, 0xff, 0xff) },
+	{ USB_DEVICE_AND_INTERFACE_INFO(ZTE_VENDOR_ID, 0x0189, 0xff, 0xff, 0xff),
+	  .driver_info = RSVD(4) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(ZTE_VENDOR_ID, 0x0191, 0xff, 0xff, 0xff), /* ZTE EuFi890 */
 	  .driver_info = RSVD(4) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(ZTE_VENDOR_ID, 0x0196, 0xff, 0xff, 0xff) },
diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c
index 058d5b853b57..bfb6f9481e87 100644
--- a/drivers/usb/typec/tcpm/tcpm.c
+++ b/drivers/usb/typec/tcpm/tcpm.c
@@ -4273,7 +4273,8 @@ static void run_state_machine(struct tcpm_port *port)
 				current_lim = PD_P_SNK_STDBY_MW / 5;
 			tcpm_set_current_limit(port, current_lim, 5000);
 			/* Not sink vbus if operational current is 0mA */
-			tcpm_set_charge(port, !!pdo_max_current(port->snk_pdo[0]));
+			tcpm_set_charge(port, !port->pd_supported ||
+					pdo_max_current(port->snk_pdo[0]));
 
 			if (!port->pd_supported)
 				tcpm_set_state(port, SNK_READY, 0);
@@ -5391,6 +5392,15 @@ static void _tcpm_pd_hard_reset(struct tcpm_port *port)
 	if (port->bist_request == BDO_MODE_TESTDATA && port->tcpc->set_bist_data)
 		port->tcpc->set_bist_data(port->tcpc, false);
 
+	switch (port->state) {
+	case ERROR_RECOVERY:
+	case PORT_RESET:
+	case PORT_RESET_WAIT_OFF:
+		return;
+	default:
+		break;
+	}
+
 	if (port->ams != NONE_AMS)
 		port->ams = NONE_AMS;
 	if (port->hard_reset_count < PD_N_HARD_RESET_COUNT)
diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c
index 0e867f531d34..196535ad996d 100644
--- a/drivers/usb/typec/tipd/core.c
+++ b/drivers/usb/typec/tipd/core.c
@@ -968,16 +968,17 @@ static int tps25750_start_patch_burst_mode(struct tps6598x *tps)
 	ret = of_property_match_string(np, "reg-names", "patch-address");
 	if (ret < 0) {
 		dev_err(tps->dev, "failed to get patch-address %d\n", ret);
-		return ret;
+		goto release_fw;
 	}
 
 	ret = of_property_read_u32_index(np, "reg", ret, &addr);
 	if (ret)
-		return ret;
+		goto release_fw;
 
 	if (addr == 0 || (addr >= 0x20 && addr <= 0x23)) {
 		dev_err(tps->dev, "wrong patch address %u\n", addr);
-		return -EINVAL;
+		ret = -EINVAL;
+		goto release_fw;
 	}
 
 	bpms_data.addr = (u8)addr;
@@ -1226,7 +1227,10 @@ static int tps6598x_probe(struct i2c_client *client)
 			TPS_REG_INT_PLUG_EVENT;
 	}
 
-	tps->data = device_get_match_data(tps->dev);
+	if (dev_fwnode(tps->dev))
+		tps->data = device_get_match_data(tps->dev);
+	else
+		tps->data = i2c_get_match_data(client);
 	if (!tps->data)
 		return -EINVAL;
 
@@ -1425,7 +1429,7 @@ static const struct of_device_id tps6598x_of_match[] = {
 MODULE_DEVICE_TABLE(of, tps6598x_of_match);
 
 static const struct i2c_device_id tps6598x_id[] = {
-	{ "tps6598x" },
+	{ "tps6598x", (kernel_ulong_t)&tps6598x_data },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tps6598x_id);
diff --git a/drivers/video/fbdev/imxfb.c b/drivers/video/fbdev/imxfb.c
index 84201c9608d3..a4dbc72f93c3 100644
--- a/drivers/video/fbdev/imxfb.c
+++ b/drivers/video/fbdev/imxfb.c
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0-only
 /*
  *  Freescale i.MX Frame Buffer device driver
  *
  *  Copyright (C) 2004 Sascha Hauer, Pengutronix
  *   Based on acornfb.c Copyright (C) Russell King.
  *
- * This file is subject to the terms and conditions of the GNU General Public
- * License.  See the file COPYING in the main directory of this archive for
- * more details.
- *
  * Please direct your questions and comments on this driver to the following
  * email address:
  *
@@ -34,6 +31,7 @@
 #include <linux/math64.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
+#include <linux/bitfield.h>
 
 #include <linux/regulator/consumer.h>
 
@@ -41,12 +39,6 @@
 #include <video/of_videomode.h>
 #include <video/videomode.h>
 
-#define PCR_TFT		(1 << 31)
-#define PCR_BPIX_8	(3 << 25)
-#define PCR_BPIX_12	(4 << 25)
-#define PCR_BPIX_16	(5 << 25)
-#define PCR_BPIX_18	(6 << 25)
-
 struct imx_fb_videomode {
 	struct fb_videomode mode;
 	u32 pcr;
@@ -64,42 +56,50 @@ struct imx_fb_videomode {
 #define LCDC_SSA	0x00
 
 #define LCDC_SIZE	0x04
-#define SIZE_XMAX(x)	((((x) >> 4) & 0x3f) << 20)
+#define SIZE_XMAX_MASK	GENMASK(25, 20)
 
-#define YMAX_MASK_IMX1	0x1ff
-#define YMAX_MASK_IMX21	0x3ff
+#define YMAX_MASK_IMX1	GENMASK(8, 0)
+#define YMAX_MASK_IMX21	GENMASK(9, 0)
 
 #define LCDC_VPW	0x08
-#define VPW_VPW(x)	((x) & 0x3ff)
+#define VPW_VPW_MASK	GENMASK(9, 0)
 
 #define LCDC_CPOS	0x0C
-#define CPOS_CC1	(1<<31)
-#define CPOS_CC0	(1<<30)
-#define CPOS_OP		(1<<28)
-#define CPOS_CXP(x)	(((x) & 3ff) << 16)
+#define CPOS_CC1	BIT(31)
+#define CPOS_CC0	BIT(30)
+#define CPOS_OP		BIT(28)
+#define CPOS_CXP_MASK	GENMASK(25, 16)
 
 #define LCDC_LCWHB	0x10
-#define LCWHB_BK_EN	(1<<31)
-#define LCWHB_CW(w)	(((w) & 0x1f) << 24)
-#define LCWHB_CH(h)	(((h) & 0x1f) << 16)
-#define LCWHB_BD(x)	((x) & 0xff)
+#define LCWHB_BK_EN	BIT(31)
+#define LCWHB_CW_MASK	GENMASK(28, 24)
+#define LCWHB_CH_MASK	GENMASK(20, 16)
+#define LCWHB_BD_MASK	GENMASK(7, 0)
 
 #define LCDC_LCHCC	0x14
 
 #define LCDC_PCR	0x18
+#define PCR_TFT		BIT(31)
+#define PCR_COLOR	BIT(30)
+#define PCR_BPIX_MASK	GENMASK(27, 25)
+#define PCR_BPIX_8	3
+#define PCR_BPIX_12	4
+#define PCR_BPIX_16	5
+#define PCR_BPIX_18	6
+#define PCR_PCD_MASK	GENMASK(5, 0)
 
 #define LCDC_HCR	0x1C
-#define HCR_H_WIDTH(x)	(((x) & 0x3f) << 26)
-#define HCR_H_WAIT_1(x)	(((x) & 0xff) << 8)
-#define HCR_H_WAIT_2(x)	((x) & 0xff)
+#define HCR_H_WIDTH_MASK	GENMASK(31, 26)
+#define HCR_H_WAIT_1_MASK	GENMASK(15, 8)
+#define HCR_H_WAIT_2_MASK	GENMASK(7, 0)
 
 #define LCDC_VCR	0x20
-#define VCR_V_WIDTH(x)	(((x) & 0x3f) << 26)
-#define VCR_V_WAIT_1(x)	(((x) & 0xff) << 8)
-#define VCR_V_WAIT_2(x)	((x) & 0xff)
+#define VCR_V_WIDTH_MASK	GENMASK(31, 26)
+#define VCR_V_WAIT_1_MASK	GENMASK(15, 8)
+#define VCR_V_WAIT_2_MASK	GENMASK(7, 0)
 
 #define LCDC_POS	0x24
-#define POS_POS(x)	((x) & 1f)
+#define POS_POS_MASK	GENMASK(4, 0)
 
 #define LCDC_LSCR1	0x28
 /* bit fields in imxfb.h */
@@ -112,24 +112,24 @@ struct imx_fb_videomode {
 
 #define LCDC_RMCR	0x34
 
-#define RMCR_LCDC_EN_MX1	(1<<1)
+#define RMCR_LCDC_EN_MX1	BIT(1)
 
-#define RMCR_SELF_REF	(1<<0)
+#define RMCR_SELF_REF	BIT(0)
 
 #define LCDC_LCDICR	0x38
-#define LCDICR_INT_SYN	(1<<2)
-#define LCDICR_INT_CON	(1)
+#define LCDICR_INT_SYN	BIT(2)
+#define LCDICR_INT_CON	BIT(0)
 
 #define LCDC_LCDISR	0x40
-#define LCDISR_UDR_ERR	(1<<3)
-#define LCDISR_ERR_RES	(1<<2)
-#define LCDISR_EOF	(1<<1)
-#define LCDISR_BOF	(1<<0)
+#define LCDISR_UDR_ERR	BIT(3)
+#define LCDISR_ERR_RES	BIT(2)
+#define LCDISR_EOF	BIT(1)
+#define LCDISR_BOF	BIT(0)
 
 #define IMXFB_LSCR1_DEFAULT 0x00120300
 
 #define LCDC_LAUSCR	0x80
-#define LAUSCR_AUS_MODE	(1<<31)
+#define LAUSCR_AUS_MODE	BIT(31)
 
 /* Used fb-mode. Can be set on kernel command line, therefore file-static. */
 static const char *fb_mode;
@@ -150,6 +150,12 @@ enum imxfb_type {
 	IMX21_FB,
 };
 
+enum imxfb_panel_type {
+	PANEL_TYPE_MONOCHROME,
+	PANEL_TYPE_CSTN,
+	PANEL_TYPE_TFT,
+};
+
 struct imxfb_info {
 	struct platform_device  *pdev;
 	void __iomem		*regs;
@@ -157,6 +163,7 @@ struct imxfb_info {
 	struct clk		*clk_ahb;
 	struct clk		*clk_per;
 	enum imxfb_type		devtype;
+	enum imxfb_panel_type	panel_type;
 	bool			enabled;
 
 	/*
@@ -273,10 +280,10 @@ static int imxfb_setpalettereg(u_int regno, u_int red, u_int green, u_int blue,
 	struct imxfb_info *fbi = info->par;
 	u_int val, ret = 1;
 
-#define CNVT_TOHW(val,width) ((((val)<<(width))+0x7FFF-(val))>>16)
+#define CNVT_TOHW(val, width) ((((val)<<(width))+0x7FFF-(val))>>16)
 	if (regno < fbi->palette_size) {
 		val = (CNVT_TOHW(red, 4) << 8) |
-		      (CNVT_TOHW(green,4) << 4) |
+		      (CNVT_TOHW(green, 4) << 4) |
 		      CNVT_TOHW(blue,  4);
 
 		writel(val, fbi->regs + 0x800 + (regno << 2));
@@ -405,23 +412,23 @@ static int imxfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info)
 
 	pcr = (unsigned int)tmp;
 
-	if (--pcr > 0x3F) {
-		pcr = 0x3F;
-		printk(KERN_WARNING "Must limit pixel clock to %luHz\n",
-				lcd_clk / pcr);
+	if (--pcr > PCR_PCD_MASK) {
+		pcr = PCR_PCD_MASK;
+		dev_warn(&fbi->pdev->dev, "Must limit pixel clock to %luHz\n",
+			 lcd_clk / pcr);
 	}
 
 	switch (var->bits_per_pixel) {
 	case 32:
-		pcr |= PCR_BPIX_18;
+		pcr |= FIELD_PREP(PCR_BPIX_MASK, PCR_BPIX_18);
 		rgb = &def_rgb_18;
 		break;
 	case 16:
 	default:
 		if (is_imx1_fb(fbi))
-			pcr |= PCR_BPIX_12;
+			pcr |= FIELD_PREP(PCR_BPIX_MASK, PCR_BPIX_12);
 		else
-			pcr |= PCR_BPIX_16;
+			pcr |= FIELD_PREP(PCR_BPIX_MASK, PCR_BPIX_16);
 
 		if (imxfb_mode->pcr & PCR_TFT)
 			rgb = &def_rgb_16_tft;
@@ -429,13 +436,13 @@ static int imxfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info)
 			rgb = &def_rgb_16_stn;
 		break;
 	case 8:
-		pcr |= PCR_BPIX_8;
+		pcr |= FIELD_PREP(PCR_BPIX_MASK, PCR_BPIX_8);
 		rgb = &def_rgb_8;
 		break;
 	}
 
 	/* add sync polarities */
-	pcr |= imxfb_mode->pcr & ~(0x3f | (7 << 25));
+	pcr |= imxfb_mode->pcr & ~(PCR_PCD_MASK | PCR_BPIX_MASK);
 
 	fbi->pcr = pcr;
 	/*
@@ -444,6 +451,13 @@ static int imxfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info)
 	if (!is_imx1_fb(fbi) && imxfb_mode->aus_mode)
 		fbi->lauscr = LAUSCR_AUS_MODE;
 
+	if (imxfb_mode->pcr & PCR_TFT)
+		fbi->panel_type = PANEL_TYPE_TFT;
+	else if (imxfb_mode->pcr & PCR_COLOR)
+		fbi->panel_type = PANEL_TYPE_CSTN;
+	else
+		fbi->panel_type = PANEL_TYPE_MONOCHROME;
+
 	/*
 	 * Copy the RGB parameters for this display
 	 * from the machine specific parameters.
@@ -506,7 +520,7 @@ static int imxfb_enable_controller(struct imxfb_info *fbi)
 	writel(fbi->map_dma, fbi->regs + LCDC_SSA);
 
 	/* panning offset 0 (0 pixel offset)        */
-	writel(0x00000000, fbi->regs + LCDC_POS);
+	writel(FIELD_PREP(POS_POS_MASK, 0), fbi->regs + LCDC_POS);
 
 	/* disable hardware cursor */
 	writel(readl(fbi->regs + LCDC_CPOS) & ~(CPOS_CC0 | CPOS_CC1),
@@ -562,7 +576,7 @@ static int imxfb_blank(int blank, struct fb_info *info)
 {
 	struct imxfb_info *fbi = info->par;
 
-	pr_debug("imxfb_blank: blank=%d\n", blank);
+	pr_debug("%s: blank=%d\n", __func__, blank);
 
 	switch (blank) {
 	case FB_BLANK_POWERDOWN:
@@ -596,6 +610,7 @@ static int imxfb_activate_var(struct fb_var_screeninfo *var, struct fb_info *inf
 {
 	struct imxfb_info *fbi = info->par;
 	u32 ymax_mask = is_imx1_fb(fbi) ? YMAX_MASK_IMX1 : YMAX_MASK_IMX21;
+	u8 left_margin_low;
 
 	pr_debug("var: xres=%d hslen=%d lm=%d rm=%d\n",
 		var->xres, var->hsync_len,
@@ -604,49 +619,59 @@ static int imxfb_activate_var(struct fb_var_screeninfo *var, struct fb_info *inf
 		var->yres, var->vsync_len,
 		var->upper_margin, var->lower_margin);
 
+	if (fbi->panel_type == PANEL_TYPE_TFT)
+		left_margin_low = 3;
+	else if (fbi->panel_type == PANEL_TYPE_CSTN)
+		left_margin_low = 2;
+	else
+		left_margin_low = 0;
+
 #if DEBUG_VAR
 	if (var->xres < 16        || var->xres > 1024)
-		printk(KERN_ERR "%s: invalid xres %d\n",
+		dev_err(&fbi->pdev->dev, "%s: invalid xres %d\n",
 			info->fix.id, var->xres);
 	if (var->hsync_len < 1    || var->hsync_len > 64)
-		printk(KERN_ERR "%s: invalid hsync_len %d\n",
+		dev_err(&fbi->pdev->dev, "%s: invalid hsync_len %d\n",
 			info->fix.id, var->hsync_len);
-	if (var->left_margin < 3  || var->left_margin > 255)
-		printk(KERN_ERR "%s: invalid left_margin %d\n",
+	if (var->left_margin < left_margin_low  || var->left_margin > 255)
+		dev_err(&fbi->pdev->dev, "%s: invalid left_margin %d\n",
 			info->fix.id, var->left_margin);
 	if (var->right_margin < 1 || var->right_margin > 255)
-		printk(KERN_ERR "%s: invalid right_margin %d\n",
+		dev_err(&fbi->pdev->dev, "%s: invalid right_margin %d\n",
 			info->fix.id, var->right_margin);
 	if (var->yres < 1 || var->yres > ymax_mask)
-		printk(KERN_ERR "%s: invalid yres %d\n",
+		dev_err(&fbi->pdev->dev, "%s: invalid yres %d\n",
 			info->fix.id, var->yres);
 	if (var->vsync_len > 100)
-		printk(KERN_ERR "%s: invalid vsync_len %d\n",
+		dev_err(&fbi->pdev->dev, "%s: invalid vsync_len %d\n",
 			info->fix.id, var->vsync_len);
 	if (var->upper_margin > 63)
-		printk(KERN_ERR "%s: invalid upper_margin %d\n",
+		dev_err(&fbi->pdev->dev, "%s: invalid upper_margin %d\n",
 			info->fix.id, var->upper_margin);
 	if (var->lower_margin > 255)
-		printk(KERN_ERR "%s: invalid lower_margin %d\n",
+		dev_err(&fbi->pdev->dev, "%s: invalid lower_margin %d\n",
 			info->fix.id, var->lower_margin);
 #endif
 
 	/* physical screen start address	    */
-	writel(VPW_VPW(var->xres * var->bits_per_pixel / 8 / 4),
-		fbi->regs + LCDC_VPW);
+	writel(FIELD_PREP(VPW_VPW_MASK,
+			  var->xres * var->bits_per_pixel / 8 / 4),
+	       fbi->regs + LCDC_VPW);
 
-	writel(HCR_H_WIDTH(var->hsync_len - 1) |
-		HCR_H_WAIT_1(var->right_margin - 1) |
-		HCR_H_WAIT_2(var->left_margin - 3),
-		fbi->regs + LCDC_HCR);
+	writel(FIELD_PREP(HCR_H_WIDTH_MASK, var->hsync_len - 1) |
+	       FIELD_PREP(HCR_H_WAIT_1_MASK, var->right_margin - 1) |
+	       FIELD_PREP(HCR_H_WAIT_2_MASK,
+			  var->left_margin - left_margin_low),
+	       fbi->regs + LCDC_HCR);
 
-	writel(VCR_V_WIDTH(var->vsync_len) |
-		VCR_V_WAIT_1(var->lower_margin) |
-		VCR_V_WAIT_2(var->upper_margin),
-		fbi->regs + LCDC_VCR);
+	writel(FIELD_PREP(VCR_V_WIDTH_MASK, var->vsync_len) |
+	       FIELD_PREP(VCR_V_WAIT_1_MASK, var->lower_margin) |
+	       FIELD_PREP(VCR_V_WAIT_2_MASK, var->upper_margin),
+	       fbi->regs + LCDC_VCR);
 
-	writel(SIZE_XMAX(var->xres) | (var->yres & ymax_mask),
-			fbi->regs + LCDC_SIZE);
+	writel(FIELD_PREP(SIZE_XMAX_MASK, var->xres >> 4) |
+	       (var->yres & ymax_mask),
+	       fbi->regs + LCDC_SIZE);
 
 	writel(fbi->pcr, fbi->regs + LCDC_PCR);
 	if (fbi->pwmr)
@@ -669,8 +694,6 @@ static int imxfb_init_fbinfo(struct platform_device *pdev)
 	struct imxfb_info *fbi = info->par;
 	struct device_node *np;
 
-	pr_debug("%s\n",__func__);
-
 	info->pseudo_palette = devm_kmalloc_array(&pdev->dev, 16,
 						  sizeof(u32), GFP_KERNEL);
 	if (!info->pseudo_palette)
@@ -678,6 +701,7 @@ static int imxfb_init_fbinfo(struct platform_device *pdev)
 
 	memset(fbi, 0, sizeof(struct imxfb_info));
 
+	fbi->pdev = pdev;
 	fbi->devtype = pdev->id_entry->driver_data;
 
 	strscpy(info->fix.id, IMX_NAME, sizeof(info->fix.id));
@@ -922,8 +946,10 @@ static int imxfb_probe(struct platform_device *pdev)
 	if (ret)
 		goto failed_init;
 
-	/* Calculate maximum bytes used per pixel. In most cases this should
-	 * be the same as m->bpp/8 */
+	/*
+	 * Calculate maximum bytes used per pixel. In most cases this should
+	 * be the same as m->bpp/8
+	 */
 	m = &fbi->mode[0];
 	bytes_per_pixel = (m->bpp + 7) / 8;
 	for (i = 0; i < fbi->num_modes; i++, m++)
@@ -1021,7 +1047,6 @@ static int imxfb_probe(struct platform_device *pdev)
 	lcd->props.max_contrast = 0xff;
 
 	imxfb_enable_controller(fbi);
-	fbi->pdev = pdev;
 
 	return 0;
 
diff --git a/drivers/video/fbdev/simplefb.c b/drivers/video/fbdev/simplefb.c
index 62f99f6fccd3..fe682af63827 100644
--- a/drivers/video/fbdev/simplefb.c
+++ b/drivers/video/fbdev/simplefb.c
@@ -21,9 +21,11 @@
 #include <linux/platform_device.h>
 #include <linux/clk.h>
 #include <linux/of.h>
+#include <linux/of_address.h>
 #include <linux/of_clk.h>
 #include <linux/of_platform.h>
 #include <linux/parser.h>
+#include <linux/pm_domain.h>
 #include <linux/regulator/consumer.h>
 
 static const struct fb_fix_screeninfo simplefb_fix = {
@@ -77,6 +79,11 @@ struct simplefb_par {
 	unsigned int clk_count;
 	struct clk **clks;
 #endif
+#if defined CONFIG_OF && defined CONFIG_PM_GENERIC_DOMAINS
+	unsigned int num_genpds;
+	struct device **genpds;
+	struct device_link **genpd_links;
+#endif
 #if defined CONFIG_OF && defined CONFIG_REGULATOR
 	bool regulators_enabled;
 	u32 regulator_count;
@@ -121,12 +128,13 @@ struct simplefb_params {
 	u32 height;
 	u32 stride;
 	struct simplefb_format *format;
+	struct resource memory;
 };
 
 static int simplefb_parse_dt(struct platform_device *pdev,
 			   struct simplefb_params *params)
 {
-	struct device_node *np = pdev->dev.of_node;
+	struct device_node *np = pdev->dev.of_node, *mem;
 	int ret;
 	const char *format;
 	int i;
@@ -166,6 +174,23 @@ static int simplefb_parse_dt(struct platform_device *pdev,
 		return -EINVAL;
 	}
 
+	mem = of_parse_phandle(np, "memory-region", 0);
+	if (mem) {
+		ret = of_address_to_resource(mem, 0, &params->memory);
+		if (ret < 0) {
+			dev_err(&pdev->dev, "failed to parse memory-region\n");
+			of_node_put(mem);
+			return ret;
+		}
+
+		if (of_property_present(np, "reg"))
+			dev_warn(&pdev->dev, "preferring \"memory-region\" over \"reg\" property\n");
+
+		of_node_put(mem);
+	} else {
+		memset(&params->memory, 0, sizeof(params->memory));
+	}
+
 	return 0;
 }
 
@@ -193,6 +218,8 @@ static int simplefb_parse_pd(struct platform_device *pdev,
 		return -EINVAL;
 	}
 
+	memset(&params->memory, 0, sizeof(params->memory));
+
 	return 0;
 }
 
@@ -411,6 +438,89 @@ static void simplefb_regulators_enable(struct simplefb_par *par,
 static void simplefb_regulators_destroy(struct simplefb_par *par) { }
 #endif
 
+#if defined CONFIG_OF && defined CONFIG_PM_GENERIC_DOMAINS
+static void simplefb_detach_genpds(void *res)
+{
+	struct simplefb_par *par = res;
+	unsigned int i = par->num_genpds;
+
+	if (par->num_genpds <= 1)
+		return;
+
+	while (i--) {
+		if (par->genpd_links[i])
+			device_link_del(par->genpd_links[i]);
+
+		if (!IS_ERR_OR_NULL(par->genpds[i]))
+			dev_pm_domain_detach(par->genpds[i], true);
+	}
+}
+
+static int simplefb_attach_genpds(struct simplefb_par *par,
+				  struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	unsigned int i;
+	int err;
+
+	err = of_count_phandle_with_args(dev->of_node, "power-domains",
+					 "#power-domain-cells");
+	if (err < 0) {
+		dev_info(dev, "failed to parse power-domains: %d\n", err);
+		return err;
+	}
+
+	par->num_genpds = err;
+
+	/*
+	 * Single power-domain devices are handled by the driver core, so
+	 * nothing to do here.
+	 */
+	if (par->num_genpds <= 1)
+		return 0;
+
+	par->genpds = devm_kcalloc(dev, par->num_genpds, sizeof(*par->genpds),
+				   GFP_KERNEL);
+	if (!par->genpds)
+		return -ENOMEM;
+
+	par->genpd_links = devm_kcalloc(dev, par->num_genpds,
+					sizeof(*par->genpd_links),
+					GFP_KERNEL);
+	if (!par->genpd_links)
+		return -ENOMEM;
+
+	for (i = 0; i < par->num_genpds; i++) {
+		par->genpds[i] = dev_pm_domain_attach_by_id(dev, i);
+		if (IS_ERR(par->genpds[i])) {
+			err = PTR_ERR(par->genpds[i]);
+			if (err == -EPROBE_DEFER) {
+				simplefb_detach_genpds(par);
+				return err;
+			}
+
+			dev_warn(dev, "failed to attach domain %u: %d\n", i, err);
+			continue;
+		}
+
+		par->genpd_links[i] = device_link_add(dev, par->genpds[i],
+						      DL_FLAG_STATELESS |
+						      DL_FLAG_PM_RUNTIME |
+						      DL_FLAG_RPM_ACTIVE);
+		if (!par->genpd_links[i])
+			dev_warn(dev, "failed to link power-domain %u\n", i);
+	}
+
+	return devm_add_action_or_reset(dev, simplefb_detach_genpds, par);
+}
+#else
+static int simplefb_attach_genpds(struct simplefb_par *par,
+				  struct platform_device *pdev)
+{
+	return 0;
+}
+#endif
+
 static int simplefb_probe(struct platform_device *pdev)
 {
 	int ret;
@@ -431,10 +541,14 @@ static int simplefb_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res) {
-		dev_err(&pdev->dev, "No memory resource\n");
-		return -EINVAL;
+	if (params.memory.start == 0 && params.memory.end == 0) {
+		res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+		if (!res) {
+			dev_err(&pdev->dev, "No memory resource\n");
+			return -EINVAL;
+		}
+	} else {
+		res = &params.memory;
 	}
 
 	mem = request_mem_region(res->start, resource_size(res), "simplefb");
@@ -493,6 +607,10 @@ static int simplefb_probe(struct platform_device *pdev)
 	if (ret < 0)
 		goto error_clocks;
 
+	ret = simplefb_attach_genpds(par, pdev);
+	if (ret < 0)
+		goto error_regulators;
+
 	simplefb_clocks_enable(par, pdev);
 	simplefb_regulators_enable(par, pdev);
 
diff --git a/drivers/w1/masters/Kconfig b/drivers/w1/masters/Kconfig
index ad316573288a..513c0b114337 100644
--- a/drivers/w1/masters/Kconfig
+++ b/drivers/w1/masters/Kconfig
@@ -5,6 +5,17 @@
 
 menu "1-wire Bus Masters"
 
+config W1_MASTER_AMD_AXI
+	tristate "AMD AXI 1-wire bus host"
+	help
+	  Say Y here is you want to support the AMD AXI 1-wire IP core.
+	  This driver makes use of the programmable logic IP to perform
+	  correctly timed 1 wire transactions without relying on GPIO timing
+	  through the kernel.
+
+	  This driver can also be built as a module.  If so, the module will be
+	  called amd_w1_axi.
+
 config W1_MASTER_MATROX
 	tristate "Matrox G400 transport layer for 1-wire"
 	depends on PCI
diff --git a/drivers/w1/masters/Makefile b/drivers/w1/masters/Makefile
index c5d85a827e52..6c5a21f9b88c 100644
--- a/drivers/w1/masters/Makefile
+++ b/drivers/w1/masters/Makefile
@@ -3,6 +3,7 @@
 # Makefile for 1-wire bus master drivers.
 #
 
+obj-$(CONFIG_W1_MASTER_AMD_AXI)		+= amd_axi_w1.o
 obj-$(CONFIG_W1_MASTER_MATROX)		+= matrox_w1.o
 obj-$(CONFIG_W1_MASTER_DS2490)		+= ds2490.o
 obj-$(CONFIG_W1_MASTER_DS2482)		+= ds2482.o
diff --git a/drivers/w1/masters/amd_axi_w1.c b/drivers/w1/masters/amd_axi_w1.c
new file mode 100644
index 000000000000..24a05c2de5f1
--- /dev/null
+++ b/drivers/w1/masters/amd_axi_w1.c
@@ -0,0 +1,395 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * amd_axi_w1 - AMD 1Wire programmable logic bus host driver
+ *
+ * Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All Rights Reserved.
+ */
+
+#include <linux/atomic.h>
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/jiffies.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/types.h>
+#include <linux/wait.h>
+
+#include <linux/w1.h>
+
+/* 1-wire AMD IP definition */
+#define AXIW1_IPID	0x10ee4453
+/* Registers offset */
+#define AXIW1_INST_REG	0x0
+#define AXIW1_CTRL_REG	0x4
+#define AXIW1_IRQE_REG	0x8
+#define AXIW1_STAT_REG	0xC
+#define AXIW1_DATA_REG	0x10
+#define AXIW1_IPVER_REG	0x18
+#define AXIW1_IPID_REG	0x1C
+/* Instructions */
+#define AXIW1_INITPRES	0x0800
+#define AXIW1_READBIT	0x0C00
+#define AXIW1_WRITEBIT	0x0E00
+#define AXIW1_READBYTE	0x0D00
+#define AXIW1_WRITEBYTE	0x0F00
+/* Status flag masks */
+#define AXIW1_DONE	BIT(0)
+#define AXIW1_READY	BIT(4)
+#define AXIW1_PRESENCE	BIT(31)
+#define AXIW1_MAJORVER_MASK	GENMASK(23, 8)
+#define AXIW1_MINORVER_MASK	GENMASK(7, 0)
+/* Control flag */
+#define AXIW1_GO	BIT(0)
+#define AXI_CLEAR	0
+#define AXI_RESET	BIT(31)
+#define AXIW1_READDATA	BIT(0)
+/* Interrupt Enable */
+#define AXIW1_READY_IRQ_EN	BIT(4)
+#define AXIW1_DONE_IRQ_EN	BIT(0)
+
+#define AXIW1_TIMEOUT	msecs_to_jiffies(100)
+
+#define DRIVER_NAME	"amd_axi_w1"
+
+struct amd_axi_w1_local {
+	struct device *dev;
+	void __iomem *base_addr;
+	int irq;
+	atomic_t flag;			/* Set on IRQ, cleared once serviced */
+	wait_queue_head_t wait_queue;
+	struct w1_bus_master bus_host;
+};
+
+/**
+ * amd_axi_w1_wait_irq_interruptible_timeout() - Wait for IRQ with timeout.
+ *
+ * @amd_axi_w1_local:	Pointer to device structure
+ * @IRQ:		IRQ channel to wait on
+ *
+ * Return:		%0 - OK, %-EINTR - Interrupted, %-EBUSY - Timed out
+ */
+static int amd_axi_w1_wait_irq_interruptible_timeout(struct amd_axi_w1_local *amd_axi_w1_local,
+						     u32 IRQ)
+{
+	int ret;
+
+	/* Enable the IRQ requested and wait for flag to indicate it's been triggered */
+	iowrite32(IRQ, amd_axi_w1_local->base_addr + AXIW1_IRQE_REG);
+	ret = wait_event_interruptible_timeout(amd_axi_w1_local->wait_queue,
+					       atomic_read(&amd_axi_w1_local->flag) != 0,
+					       AXIW1_TIMEOUT);
+	if (ret < 0) {
+		dev_err(amd_axi_w1_local->dev, "Wait IRQ Interrupted\n");
+		return -EINTR;
+	}
+
+	if (!ret) {
+		dev_err(amd_axi_w1_local->dev, "Wait IRQ Timeout\n");
+		return -EBUSY;
+	}
+
+	atomic_set(&amd_axi_w1_local->flag, 0);
+	return 0;
+}
+
+/**
+ * amd_axi_w1_touch_bit() - Performs the touch-bit function - write a 0 or 1 and reads the level.
+ *
+ * @data:	Pointer to device structure
+ * @bit:	The level to write
+ *
+ * Return:	The level read
+ */
+static u8 amd_axi_w1_touch_bit(void *data, u8 bit)
+{
+	struct amd_axi_w1_local *amd_axi_w1_local = data;
+	u8 val = 0;
+	int rc;
+
+	/* Wait for READY signal to be 1 to ensure 1-wire IP is ready */
+	while ((ioread32(amd_axi_w1_local->base_addr + AXIW1_STAT_REG) & AXIW1_READY) == 0) {
+		rc = amd_axi_w1_wait_irq_interruptible_timeout(amd_axi_w1_local,
+							       AXIW1_READY_IRQ_EN);
+		if (rc < 0)
+			return 1; /* Callee doesn't test for error. Return inactive bus state */
+	}
+
+	if (bit)
+		/* Read. Write read Bit command in register 0 */
+		iowrite32(AXIW1_READBIT, amd_axi_w1_local->base_addr + AXIW1_INST_REG);
+	else
+		/* Write. Write tx Bit command in instruction register with bit to transmit */
+		iowrite32(AXIW1_WRITEBIT + (bit & 0x01),
+			  amd_axi_w1_local->base_addr + AXIW1_INST_REG);
+
+	/* Write Go signal and clear control reset signal in control register */
+	iowrite32(AXIW1_GO, amd_axi_w1_local->base_addr + AXIW1_CTRL_REG);
+
+	/* Wait for done signal to be 1 */
+	while ((ioread32(amd_axi_w1_local->base_addr + AXIW1_STAT_REG) & AXIW1_DONE) != 1) {
+		rc = amd_axi_w1_wait_irq_interruptible_timeout(amd_axi_w1_local, AXIW1_DONE_IRQ_EN);
+		if (rc < 0)
+			return 1; /* Callee doesn't test for error. Return inactive bus state */
+	}
+
+	/* If read, Retrieve data from register */
+	if (bit)
+		val = (u8)(ioread32(amd_axi_w1_local->base_addr + AXIW1_DATA_REG) & AXIW1_READDATA);
+
+	/* Clear Go signal in register 1 */
+	iowrite32(AXI_CLEAR, amd_axi_w1_local->base_addr + AXIW1_CTRL_REG);
+
+	return val;
+}
+
+/**
+ * amd_axi_w1_read_byte - Performs the read byte function.
+ *
+ * @data:	Pointer to device structure
+ * Return:	The value read
+ */
+static u8 amd_axi_w1_read_byte(void *data)
+{
+	struct amd_axi_w1_local *amd_axi_w1_local = data;
+	u8 val = 0;
+	int rc;
+
+	/* Wait for READY signal to be 1 to ensure 1-wire IP is ready */
+	while ((ioread32(amd_axi_w1_local->base_addr + AXIW1_STAT_REG) & AXIW1_READY) == 0) {
+		rc = amd_axi_w1_wait_irq_interruptible_timeout(amd_axi_w1_local,
+							       AXIW1_READY_IRQ_EN);
+		if (rc < 0)
+			return 0xFF; /* Return inactive bus state */
+	}
+
+	/* Write read Byte command in instruction register*/
+	iowrite32(AXIW1_READBYTE, amd_axi_w1_local->base_addr + AXIW1_INST_REG);
+
+	/* Write Go signal and clear control reset signal in control register */
+	iowrite32(AXIW1_GO, amd_axi_w1_local->base_addr + AXIW1_CTRL_REG);
+
+	/* Wait for done signal to be 1 */
+	while ((ioread32(amd_axi_w1_local->base_addr + AXIW1_STAT_REG) & AXIW1_DONE) != 1) {
+		rc = amd_axi_w1_wait_irq_interruptible_timeout(amd_axi_w1_local, AXIW1_DONE_IRQ_EN);
+		if (rc < 0)
+			return 0xFF; /* Return inactive bus state */
+	}
+
+	/* Retrieve LSB bit in data register to get RX byte */
+	val = (u8)(ioread32(amd_axi_w1_local->base_addr + AXIW1_DATA_REG) & 0x000000FF);
+
+	/* Clear Go signal in control register */
+	iowrite32(AXI_CLEAR, amd_axi_w1_local->base_addr + AXIW1_CTRL_REG);
+
+	return val;
+}
+
+/**
+ * amd_axi_w1_write_byte - Performs the write byte function.
+ *
+ * @data:	The ds2482 channel pointer
+ * @val:	The value to write
+ */
+static void amd_axi_w1_write_byte(void *data, u8 val)
+{
+	struct amd_axi_w1_local *amd_axi_w1_local = data;
+	int rc;
+
+	/* Wait for READY signal to be 1 to ensure 1-wire IP is ready */
+	while ((ioread32(amd_axi_w1_local->base_addr + AXIW1_STAT_REG) & AXIW1_READY) == 0) {
+		rc = amd_axi_w1_wait_irq_interruptible_timeout(amd_axi_w1_local,
+							       AXIW1_READY_IRQ_EN);
+		if (rc < 0)
+			return;
+	}
+
+	/* Write tx Byte command in instruction register with bit to transmit */
+	iowrite32(AXIW1_WRITEBYTE + val, amd_axi_w1_local->base_addr + AXIW1_INST_REG);
+
+	/* Write Go signal and clear control reset signal in register 1 */
+	iowrite32(AXIW1_GO, amd_axi_w1_local->base_addr + AXIW1_CTRL_REG);
+
+	/* Wait for done signal to be 1 */
+	while ((ioread32(amd_axi_w1_local->base_addr + AXIW1_STAT_REG) & AXIW1_DONE) != 1) {
+		rc = amd_axi_w1_wait_irq_interruptible_timeout(amd_axi_w1_local,
+							       AXIW1_DONE_IRQ_EN);
+		if (rc < 0)
+			return;
+	}
+
+	/* Clear Go signal in control register */
+	iowrite32(AXI_CLEAR, amd_axi_w1_local->base_addr + AXIW1_CTRL_REG);
+}
+
+/**
+ * amd_axi_w1_reset_bus() - Issues a reset bus sequence.
+ *
+ * @data:	the bus host data struct
+ * Return:	0=Device present, 1=No device present or error
+ */
+static u8 amd_axi_w1_reset_bus(void *data)
+{
+	struct amd_axi_w1_local *amd_axi_w1_local = data;
+	u8 val = 0;
+	int rc;
+
+	/* Reset 1-wire Axi IP */
+	iowrite32(AXI_RESET, amd_axi_w1_local->base_addr + AXIW1_CTRL_REG);
+
+	/* Wait for READY signal to be 1 to ensure 1-wire IP is ready */
+	while ((ioread32(amd_axi_w1_local->base_addr + AXIW1_STAT_REG) & AXIW1_READY) == 0) {
+		rc = amd_axi_w1_wait_irq_interruptible_timeout(amd_axi_w1_local,
+							       AXIW1_READY_IRQ_EN);
+		if (rc < 0)
+			return 1; /* Something went wrong with the hardware */
+	}
+	/* Write Initialization command in instruction register */
+	iowrite32(AXIW1_INITPRES, amd_axi_w1_local->base_addr + AXIW1_INST_REG);
+
+	/* Write Go signal and clear control reset signal in register 1 */
+	iowrite32(AXIW1_GO, amd_axi_w1_local->base_addr + AXIW1_CTRL_REG);
+
+	/* Wait for done signal to be 1 */
+	while ((ioread32(amd_axi_w1_local->base_addr + AXIW1_STAT_REG) & AXIW1_DONE) != 1) {
+		rc = amd_axi_w1_wait_irq_interruptible_timeout(amd_axi_w1_local, AXIW1_DONE_IRQ_EN);
+		if (rc < 0)
+			return 1; /* Something went wrong with the hardware */
+	}
+	/* Retrieve MSB bit in status register to get failure bit */
+	if ((ioread32(amd_axi_w1_local->base_addr + AXIW1_STAT_REG) & AXIW1_PRESENCE) != 0)
+		val = 1;
+
+	/* Clear Go signal in control register */
+	iowrite32(AXI_CLEAR, amd_axi_w1_local->base_addr + AXIW1_CTRL_REG);
+
+	return val;
+}
+
+/* Reset the 1-wire AXI IP. Put the IP in reset state and clear registers */
+static void amd_axi_w1_reset(struct amd_axi_w1_local *amd_axi_w1_local)
+{
+	iowrite32(AXI_RESET, amd_axi_w1_local->base_addr + AXIW1_CTRL_REG);
+	iowrite32(AXI_CLEAR, amd_axi_w1_local->base_addr + AXIW1_INST_REG);
+	iowrite32(AXI_CLEAR, amd_axi_w1_local->base_addr + AXIW1_IRQE_REG);
+	iowrite32(AXI_CLEAR, amd_axi_w1_local->base_addr + AXIW1_STAT_REG);
+	iowrite32(AXI_CLEAR, amd_axi_w1_local->base_addr + AXIW1_DATA_REG);
+}
+
+static irqreturn_t amd_axi_w1_irq(int irq, void *lp)
+{
+	struct amd_axi_w1_local *amd_axi_w1_local = lp;
+
+	/* Reset interrupt trigger */
+	iowrite32(AXI_CLEAR, amd_axi_w1_local->base_addr + AXIW1_IRQE_REG);
+
+	atomic_set(&amd_axi_w1_local->flag, 1);
+	wake_up_interruptible(&amd_axi_w1_local->wait_queue);
+
+	return IRQ_HANDLED;
+}
+
+static int amd_axi_w1_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct amd_axi_w1_local *lp;
+	struct clk *clk;
+	u32 ver_major, ver_minor;
+	int val, rc = 0;
+
+	lp = devm_kzalloc(dev, sizeof(*lp), GFP_KERNEL);
+	if (!lp)
+		return -ENOMEM;
+
+	lp->dev = dev;
+	lp->base_addr = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(lp->base_addr))
+		return PTR_ERR(lp->base_addr);
+
+	lp->irq = platform_get_irq(pdev, 0);
+	if (lp->irq < 0)
+		return lp->irq;
+
+	rc = devm_request_irq(dev, lp->irq, &amd_axi_w1_irq, IRQF_TRIGGER_HIGH, DRIVER_NAME, lp);
+	if (rc)
+		return rc;
+
+	/* Initialize wait queue and flag */
+	init_waitqueue_head(&lp->wait_queue);
+
+	clk = devm_clk_get_enabled(dev, NULL);
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
+
+	/* Verify IP presence in HW */
+	if (ioread32(lp->base_addr + AXIW1_IPID_REG) != AXIW1_IPID) {
+		dev_err(dev, "AMD 1-wire IP not detected in hardware\n");
+		return -ENODEV;
+	}
+
+	/*
+	 * Allow for future driver expansion supporting new hardware features
+	 * This driver currently only supports hardware 1.x, but include logic
+	 * to detect if a potentially incompatible future version is used
+	 * by reading major version ID. It is highly undesirable for new IP versions
+	 * to break the API, but this code will at least allow for graceful failure
+	 * should that happen. Future new features can be enabled by hardware
+	 * incrementing the minor version and augmenting the driver to detect capability
+	 * using the minor version number
+	 */
+	val = ioread32(lp->base_addr + AXIW1_IPVER_REG);
+	ver_major = FIELD_GET(AXIW1_MAJORVER_MASK, val);
+	ver_minor = FIELD_GET(AXIW1_MINORVER_MASK, val);
+
+	if (ver_major != 1) {
+		dev_err(dev, "AMD AXI W1 host version %u.%u is not supported by this driver",
+			ver_major, ver_minor);
+		return -ENODEV;
+	}
+
+	lp->bus_host.data = lp;
+	lp->bus_host.touch_bit = amd_axi_w1_touch_bit;
+	lp->bus_host.read_byte = amd_axi_w1_read_byte;
+	lp->bus_host.write_byte = amd_axi_w1_write_byte;
+	lp->bus_host.reset_bus = amd_axi_w1_reset_bus;
+
+	amd_axi_w1_reset(lp);
+
+	platform_set_drvdata(pdev, lp);
+	rc = w1_add_master_device(&lp->bus_host);
+	if (rc) {
+		dev_err(dev, "Could not add host device\n");
+		return rc;
+	}
+
+	return 0;
+}
+
+static void amd_axi_w1_remove(struct platform_device *pdev)
+{
+	struct amd_axi_w1_local *lp = platform_get_drvdata(pdev);
+
+	w1_remove_master_device(&lp->bus_host);
+}
+
+static const struct of_device_id amd_axi_w1_of_match[] = {
+	{ .compatible = "amd,axi-1wire-host" },
+	{ /* end of list */ },
+};
+MODULE_DEVICE_TABLE(of, amd_axi_w1_of_match);
+
+static struct platform_driver amd_axi_w1_driver = {
+	.probe = amd_axi_w1_probe,
+	.remove_new = amd_axi_w1_remove,
+	.driver = {
+		.name = DRIVER_NAME,
+		.of_match_table = amd_axi_w1_of_match,
+	},
+};
+module_platform_driver(amd_axi_w1_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Kris Chaplin <kris.chaplin@amd.com>");
+MODULE_DESCRIPTION("Driver for AMD AXI 1 Wire IP core");
diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c
index 1ce7f3c7a950..0eb337a8ec0f 100644
--- a/drivers/xen/privcmd.c
+++ b/drivers/xen/privcmd.c
@@ -1115,7 +1115,7 @@ struct privcmd_kernel_ioreq {
 	spinlock_t lock; /* Protects ioeventfds list */
 	struct list_head ioeventfds;
 	struct list_head list;
-	struct ioreq_port ports[0];
+	struct ioreq_port ports[] __counted_by(vcpus);
 };
 
 static irqreturn_t ioeventfd_interrupt(int irq, void *dev_id)
diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
index 946bd56f0ac5..0e6c6c25d154 100644
--- a/drivers/xen/swiotlb-xen.c
+++ b/drivers/xen/swiotlb-xen.c
@@ -405,4 +405,5 @@ const struct dma_map_ops xen_swiotlb_dma_ops = {
 	.get_sgtable = dma_common_get_sgtable,
 	.alloc_pages = dma_common_alloc_pages,
 	.free_pages = dma_common_free_pages,
+	.max_mapping_size = swiotlb_max_mapping_size,
 };
diff --git a/fs/9p/vfs_dentry.c b/fs/9p/vfs_dentry.c
index f16f73581634..de679d9505db 100644
--- a/fs/9p/vfs_dentry.c
+++ b/fs/9p/vfs_dentry.c
@@ -56,7 +56,8 @@ static void v9fs_dentry_release(struct dentry *dentry)
 	dentry->d_fsdata = NULL;
 }
 
-static int v9fs_lookup_revalidate(struct dentry *dentry, unsigned int flags)
+static int v9fs_lookup_revalidate(struct dentry *dentry,
+				  const struct qstr *name, unsigned int flags)
 {
 	struct p9_fid *fid;
 	struct inode *inode;
@@ -94,9 +95,15 @@ static int v9fs_lookup_revalidate(struct dentry *dentry, unsigned int flags)
 	return 1;
 }
 
+static int v9fs_lookup_weak_revalidate(struct dentry *dentry,
+				       unsigned int flags)
+{
+	return v9fs_lookup_revalidate(dentry, NULL, flags);
+}
+
 const struct dentry_operations v9fs_cached_dentry_operations = {
 	.d_revalidate = v9fs_lookup_revalidate,
-	.d_weak_revalidate = v9fs_lookup_revalidate,
+	.d_weak_revalidate = v9fs_lookup_weak_revalidate,
 	.d_delete = v9fs_cached_dentry_delete,
 	.d_release = v9fs_dentry_release,
 };
diff --git a/fs/afs/dir.c b/fs/afs/dir.c
index 5219182e52e1..e3ba14512715 100644
--- a/fs/afs/dir.c
+++ b/fs/afs/dir.c
@@ -21,7 +21,8 @@ static struct dentry *afs_lookup(struct inode *dir, struct dentry *dentry,
 				 unsigned int flags);
 static int afs_dir_open(struct inode *inode, struct file *file);
 static int afs_readdir(struct file *file, struct dir_context *ctx);
-static int afs_d_revalidate(struct dentry *dentry, unsigned int flags);
+static int afs_d_revalidate(struct dentry *dentry, const struct qstr *name,
+			    unsigned int flags);
 static int afs_d_delete(const struct dentry *dentry);
 static void afs_d_iput(struct dentry *dentry, struct inode *inode);
 static bool afs_lookup_one_filldir(struct dir_context *ctx, const char *name, int nlen,
@@ -1084,7 +1085,8 @@ static int afs_d_revalidate_rcu(struct dentry *dentry)
  * - NOTE! the hit can be a negative hit too, so we can't assume we have an
  *   inode
  */
-static int afs_d_revalidate(struct dentry *dentry, unsigned int flags)
+static int afs_d_revalidate(struct dentry *dentry, const struct qstr *name,
+			    unsigned int flags)
 {
 	struct afs_vnode *vnode, *dir;
 	struct afs_fid fid;
diff --git a/fs/afs/dynroot.c b/fs/afs/dynroot.c
index 4d04ef2d3ae7..4b79a7ed973c 100644
--- a/fs/afs/dynroot.c
+++ b/fs/afs/dynroot.c
@@ -247,7 +247,9 @@ const struct inode_operations afs_dynroot_inode_operations = {
 /*
  * Dirs in the dynamic root don't need revalidation.
  */
-static int afs_dynroot_d_revalidate(struct dentry *dentry, unsigned int flags)
+static int afs_dynroot_d_revalidate(struct dentry *dentry,
+				    const struct qstr *name,
+				    unsigned int flags)
 {
 	return 1;
 }
diff --git a/fs/afs/write.c b/fs/afs/write.c
index 4a168781936b..e87b52b1f34c 100644
--- a/fs/afs/write.c
+++ b/fs/afs/write.c
@@ -242,7 +242,7 @@ static void afs_kill_pages(struct address_space *mapping,
 		folio_clear_uptodate(folio);
 		folio_end_writeback(folio);
 		folio_lock(folio);
-		generic_error_remove_page(mapping, &folio->page);
+		generic_error_remove_folio(mapping, folio);
 		folio_unlock(folio);
 		folio_put(folio);
 
@@ -559,8 +559,7 @@ static void afs_extend_writeback(struct address_space *mapping,
 
 			if (!folio_clear_dirty_for_io(folio))
 				BUG();
-			if (folio_start_writeback(folio))
-				BUG();
+			folio_start_writeback(folio);
 			afs_folio_start_fscache(caching, folio);
 
 			*_count -= folio_nr_pages(folio);
@@ -595,8 +594,7 @@ static ssize_t afs_write_back_from_locked_folio(struct address_space *mapping,
 
 	_enter(",%lx,%llx-%llx", folio_index(folio), start, end);
 
-	if (folio_start_writeback(folio))
-		BUG();
+	folio_start_writeback(folio);
 	afs_folio_start_fscache(caching, folio);
 
 	count -= folio_nr_pages(folio);
diff --git a/fs/aio.c b/fs/aio.c
index f8589caef9c1..d02842156b35 100644
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -266,7 +266,7 @@ static struct file *aio_private_file(struct kioctx *ctx, loff_t nr_pages)
 		return ERR_CAST(inode);
 
 	inode->i_mapping->a_ops = &aio_ctx_aops;
-	inode->i_mapping->private_data = ctx;
+	inode->i_mapping->i_private_data = ctx;
 	inode->i_size = PAGE_SIZE * nr_pages;
 
 	file = alloc_file_pseudo(inode, aio_mnt, "[aio]",
@@ -316,10 +316,10 @@ static void put_aio_ring_file(struct kioctx *ctx)
 
 		/* Prevent further access to the kioctx from migratepages */
 		i_mapping = aio_ring_file->f_mapping;
-		spin_lock(&i_mapping->private_lock);
-		i_mapping->private_data = NULL;
+		spin_lock(&i_mapping->i_private_lock);
+		i_mapping->i_private_data = NULL;
 		ctx->aio_ring_file = NULL;
-		spin_unlock(&i_mapping->private_lock);
+		spin_unlock(&i_mapping->i_private_lock);
 
 		fput(aio_ring_file);
 	}
@@ -422,9 +422,9 @@ static int aio_migrate_folio(struct address_space *mapping, struct folio *dst,
 
 	rc = 0;
 
-	/* mapping->private_lock here protects against the kioctx teardown.  */
-	spin_lock(&mapping->private_lock);
-	ctx = mapping->private_data;
+	/* mapping->i_private_lock here protects against the kioctx teardown.  */
+	spin_lock(&mapping->i_private_lock);
+	ctx = mapping->i_private_data;
 	if (!ctx) {
 		rc = -EINVAL;
 		goto out;
@@ -476,7 +476,7 @@ static int aio_migrate_folio(struct address_space *mapping, struct folio *dst,
 out_unlock:
 	mutex_unlock(&ctx->ring_lock);
 out:
-	spin_unlock(&mapping->private_lock);
+	spin_unlock(&mapping->i_private_lock);
 	return rc;
 }
 #else
diff --git a/fs/anon_inodes.c b/fs/anon_inodes.c
index d26222b7eefe..0496cb5b6eab 100644
--- a/fs/anon_inodes.c
+++ b/fs/anon_inodes.c
@@ -79,7 +79,7 @@ static struct file *__anon_inode_getfile(const char *name,
 					 const struct file_operations *fops,
 					 void *priv, int flags,
 					 const struct inode *context_inode,
-					 bool secure)
+					 bool make_inode)
 {
 	struct inode *inode;
 	struct file *file;
@@ -87,7 +87,7 @@ static struct file *__anon_inode_getfile(const char *name,
 	if (fops->owner && !try_module_get(fops->owner))
 		return ERR_PTR(-ENOENT);
 
-	if (secure) {
+	if (make_inode) {
 		inode =	anon_inode_make_secure_inode(name, context_inode);
 		if (IS_ERR(inode)) {
 			file = ERR_CAST(inode);
@@ -149,13 +149,10 @@ struct file *anon_inode_getfile(const char *name,
 EXPORT_SYMBOL_GPL(anon_inode_getfile);
 
 /**
- * anon_inode_getfile_secure - Like anon_inode_getfile(), but creates a new
+ * anon_inode_create_getfile - Like anon_inode_getfile(), but creates a new
  *                             !S_PRIVATE anon inode rather than reuse the
  *                             singleton anon inode and calls the
- *                             inode_init_security_anon() LSM hook.  This
- *                             allows for both the inode to have its own
- *                             security context and for the LSM to enforce
- *                             policy on the inode's creation.
+ *                             inode_init_security_anon() LSM hook.
  *
  * @name:    [in]    name of the "class" of the new file
  * @fops:    [in]    file operations for the new file
@@ -164,11 +161,21 @@ EXPORT_SYMBOL_GPL(anon_inode_getfile);
  * @context_inode:
  *           [in]    the logical relationship with the new inode (optional)
  *
+ * Create a new anonymous inode and file pair.  This can be done for two
+ * reasons:
+ *
+ * - for the inode to have its own security context, so that LSMs can enforce
+ *   policy on the inode's creation;
+ *
+ * - if the caller needs a unique inode, for example in order to customize
+ *   the size returned by fstat()
+ *
  * The LSM may use @context_inode in inode_init_security_anon(), but a
- * reference to it is not held.  Returns the newly created file* or an error
- * pointer.  See the anon_inode_getfile() documentation for more information.
+ * reference to it is not held.
+ *
+ * Returns the newly created file* or an error pointer.
  */
-struct file *anon_inode_getfile_secure(const char *name,
+struct file *anon_inode_create_getfile(const char *name,
 				       const struct file_operations *fops,
 				       void *priv, int flags,
 				       const struct inode *context_inode)
@@ -176,12 +183,13 @@ struct file *anon_inode_getfile_secure(const char *name,
 	return __anon_inode_getfile(name, fops, priv, flags,
 				    context_inode, true);
 }
+EXPORT_SYMBOL_GPL(anon_inode_create_getfile);
 
 static int __anon_inode_getfd(const char *name,
 			      const struct file_operations *fops,
 			      void *priv, int flags,
 			      const struct inode *context_inode,
-			      bool secure)
+			      bool make_inode)
 {
 	int error, fd;
 	struct file *file;
@@ -192,7 +200,7 @@ static int __anon_inode_getfd(const char *name,
 	fd = error;
 
 	file = __anon_inode_getfile(name, fops, priv, flags, context_inode,
-				    secure);
+				    make_inode);
 	if (IS_ERR(file)) {
 		error = PTR_ERR(file);
 		goto err_put_unused_fd;
@@ -231,10 +239,9 @@ int anon_inode_getfd(const char *name, const struct file_operations *fops,
 EXPORT_SYMBOL_GPL(anon_inode_getfd);
 
 /**
- * anon_inode_getfd_secure - Like anon_inode_getfd(), but creates a new
+ * anon_inode_create_getfd - Like anon_inode_getfd(), but creates a new
  * !S_PRIVATE anon inode rather than reuse the singleton anon inode, and calls
- * the inode_init_security_anon() LSM hook. This allows the inode to have its
- * own security context and for a LSM to reject creation of the inode.
+ * the inode_init_security_anon() LSM hook.
  *
  * @name:    [in]    name of the "class" of the new file
  * @fops:    [in]    file operations for the new file
@@ -243,16 +250,26 @@ EXPORT_SYMBOL_GPL(anon_inode_getfd);
  * @context_inode:
  *           [in]    the logical relationship with the new inode (optional)
  *
+ * Create a new anonymous inode and file pair.  This can be done for two
+ * reasons:
+ *
+ * - for the inode to have its own security context, so that LSMs can enforce
+ *   policy on the inode's creation;
+ *
+ * - if the caller needs a unique inode, for example in order to customize
+ *   the size returned by fstat()
+ *
  * The LSM may use @context_inode in inode_init_security_anon(), but a
  * reference to it is not held.
+ *
+ * Returns a newly created file descriptor or an error code.
  */
-int anon_inode_getfd_secure(const char *name, const struct file_operations *fops,
+int anon_inode_create_getfd(const char *name, const struct file_operations *fops,
 			    void *priv, int flags,
 			    const struct inode *context_inode)
 {
 	return __anon_inode_getfd(name, fops, priv, flags, context_inode, true);
 }
-EXPORT_SYMBOL_GPL(anon_inode_getfd_secure);
 
 static int __init anon_inode_init(void)
 {
diff --git a/fs/autofs/inode.c b/fs/autofs/inode.c
index a5083d447a62..1f5db6863663 100644
--- a/fs/autofs/inode.c
+++ b/fs/autofs/inode.c
@@ -309,9 +309,7 @@ static int autofs_fill_super(struct super_block *s, struct fs_context *fc)
 	struct autofs_fs_context *ctx = fc->fs_private;
 	struct autofs_sb_info *sbi = s->s_fs_info;
 	struct inode *root_inode;
-	struct dentry *root;
 	struct autofs_info *ino;
-	int ret = -ENOMEM;
 
 	pr_debug("starting up, sbi = %p\n", sbi);
 
@@ -328,56 +326,44 @@ static int autofs_fill_super(struct super_block *s, struct fs_context *fc)
 	 */
 	ino = autofs_new_ino(sbi);
 	if (!ino)
-		goto fail;
+		return -ENOMEM;
 
 	root_inode = autofs_get_inode(s, S_IFDIR | 0755);
+	if (!root_inode)
+		return -ENOMEM;
+
 	root_inode->i_uid = ctx->uid;
 	root_inode->i_gid = ctx->gid;
+	root_inode->i_fop = &autofs_root_operations;
+	root_inode->i_op = &autofs_dir_inode_operations;
 
-	root = d_make_root(root_inode);
-	if (!root)
-		goto fail_ino;
-
-	root->d_fsdata = ino;
+	s->s_root = d_make_root(root_inode);
+	if (unlikely(!s->s_root)) {
+		autofs_free_ino(ino);
+		return -ENOMEM;
+	}
+	s->s_root->d_fsdata = ino;
 
 	if (ctx->pgrp_set) {
 		sbi->oz_pgrp = find_get_pid(ctx->pgrp);
-		if (!sbi->oz_pgrp) {
-			ret = invalf(fc, "Could not find process group %d",
-				     ctx->pgrp);
-			goto fail_dput;
-		}
-	} else {
+		if (!sbi->oz_pgrp)
+			return invalf(fc, "Could not find process group %d",
+				      ctx->pgrp);
+	} else
 		sbi->oz_pgrp = get_task_pid(current, PIDTYPE_PGID);
-	}
 
 	if (autofs_type_trigger(sbi->type))
-		__managed_dentry_set_managed(root);
-
-	root_inode->i_fop = &autofs_root_operations;
-	root_inode->i_op = &autofs_dir_inode_operations;
+		/* s->s_root won't be contended so there's little to
+		 * be gained by not taking the d_lock when setting
+		 * d_flags, even when a lot mounts are being done.
+		 */
+		managed_dentry_set_managed(s->s_root);
 
 	pr_debug("pipe fd = %d, pgrp = %u\n",
 		 sbi->pipefd, pid_nr(sbi->oz_pgrp));
 
 	sbi->flags &= ~AUTOFS_SBI_CATATONIC;
-
-	/*
-	 * Success! Install the root dentry now to indicate completion.
-	 */
-	s->s_root = root;
 	return 0;
-
-	/*
-	 * Failure ... clean up.
-	 */
-fail_dput:
-	dput(root);
-	goto fail;
-fail_ino:
-	autofs_free_ino(ino);
-fail:
-	return ret;
 }
 
 /*
diff --git a/fs/bcachefs/Kconfig b/fs/bcachefs/Kconfig
index c08c2c7d6fbb..8f564dcdd5dd 100644
--- a/fs/bcachefs/Kconfig
+++ b/fs/bcachefs/Kconfig
@@ -73,6 +73,16 @@ config BCACHEFS_NO_LATENCY_ACCT
 	help
 	This disables device latency tracking and time stats, only for performance testing
 
+config BCACHEFS_SIX_OPTIMISTIC_SPIN
+	bool "Optimistic spinning for six locks"
+	depends on BCACHEFS_FS
+	depends on SMP
+	default y
+	help
+	Instead of immediately sleeping when attempting to take a six lock that
+	is held by another thread, spin for a short while, as long as the
+	thread owning the lock is running.
+
 config MEAN_AND_VARIANCE_UNIT_TEST
 	tristate "mean_and_variance unit tests" if !KUNIT_ALL_TESTS
 	depends on KUNIT
diff --git a/fs/bcachefs/Makefile b/fs/bcachefs/Makefile
index 45b64f89258c..ed550a40463e 100644
--- a/fs/bcachefs/Makefile
+++ b/fs/bcachefs/Makefile
@@ -28,6 +28,7 @@ bcachefs-y		:=	\
 	clock.o			\
 	compress.o		\
 	counters.o		\
+	darray.o		\
 	debug.o			\
 	dirent.o		\
 	disk_groups.o		\
diff --git a/fs/bcachefs/alloc_background.c b/fs/bcachefs/alloc_background.c
index 1fec0e67891f..1ed8506c33c7 100644
--- a/fs/bcachefs/alloc_background.c
+++ b/fs/bcachefs/alloc_background.c
@@ -561,8 +561,8 @@ int bch2_bucket_gens_init(struct bch_fs *c)
 
 		if (have_bucket_gens_key && bkey_cmp(iter.pos, pos)) {
 			ret = commit_do(trans, NULL, NULL,
-					BTREE_INSERT_NOFAIL|
-					BTREE_INSERT_LAZY_RW,
+					BCH_TRANS_COMMIT_no_enospc|
+					BCH_TRANS_COMMIT_lazy_rw,
 				bch2_btree_insert_trans(trans, BTREE_ID_bucket_gens, &g.k_i, 0));
 			if (ret)
 				break;
@@ -581,8 +581,8 @@ int bch2_bucket_gens_init(struct bch_fs *c)
 
 	if (have_bucket_gens_key && !ret)
 		ret = commit_do(trans, NULL, NULL,
-				BTREE_INSERT_NOFAIL|
-				BTREE_INSERT_LAZY_RW,
+				BCH_TRANS_COMMIT_no_enospc|
+				BCH_TRANS_COMMIT_lazy_rw,
 			bch2_btree_insert_trans(trans, BTREE_ID_bucket_gens, &g.k_i, 0));
 
 	bch2_trans_put(trans);
@@ -847,6 +847,19 @@ int bch2_trans_mark_alloc(struct btree_trans *trans,
 			return ret;
 	}
 
+	/*
+	 * need to know if we're getting called from the invalidate path or
+	 * not:
+	 */
+
+	if ((flags & BTREE_TRIGGER_BUCKET_INVALIDATE) &&
+	    old_a->cached_sectors) {
+		ret = bch2_update_cached_sectors_list(trans, new->k.p.inode,
+						      -((s64) old_a->cached_sectors));
+		if (ret)
+			return ret;
+	}
+
 	return 0;
 }
 
@@ -1212,7 +1225,7 @@ int bch2_check_alloc_hole_bucket_gens(struct btree_trans *trans,
 	return ret;
 }
 
-static noinline_for_stack int __bch2_check_discard_freespace_key(struct btree_trans *trans,
+static noinline_for_stack int bch2_check_discard_freespace_key(struct btree_trans *trans,
 					      struct btree_iter *iter)
 {
 	struct bch_fs *c = trans->c;
@@ -1267,28 +1280,10 @@ static noinline_for_stack int __bch2_check_discard_freespace_key(struct btree_tr
 	ret =   bch2_btree_delete_extent_at(trans, iter,
 			iter->btree_id == BTREE_ID_freespace ? 1 : 0, 0) ?:
 		bch2_trans_commit(trans, NULL, NULL,
-			BTREE_INSERT_NOFAIL|BTREE_INSERT_LAZY_RW);
+			BCH_TRANS_COMMIT_no_enospc|BCH_TRANS_COMMIT_lazy_rw);
 	goto out;
 }
 
-static int bch2_check_discard_freespace_key(struct btree_trans *trans,
-					    struct btree_iter *iter,
-					    struct bpos end)
-{
-	if (!btree_id_is_extents(iter->btree_id)) {
-		return __bch2_check_discard_freespace_key(trans, iter);
-	} else {
-		int ret = 0;
-
-		while (!bkey_eq(iter->pos, end) &&
-		       !(ret = btree_trans_too_many_iters(trans) ?:
-			       __bch2_check_discard_freespace_key(trans, iter)))
-			bch2_btree_iter_set_pos(iter, bpos_nosnap_successor(iter->pos));
-
-		return ret;
-	}
-}
-
 /*
  * We've already checked that generation numbers in the bucket_gens btree are
  * valid for buckets that exist; this just checks for keys for nonexistent
@@ -1422,8 +1417,8 @@ int bch2_check_alloc_info(struct bch_fs *c)
 		}
 
 		ret = bch2_trans_commit(trans, NULL, NULL,
-					BTREE_INSERT_NOFAIL|
-					BTREE_INSERT_LAZY_RW);
+					BCH_TRANS_COMMIT_no_enospc|
+					BCH_TRANS_COMMIT_lazy_rw);
 		if (ret)
 			goto bkey_err;
 
@@ -1445,15 +1440,43 @@ int bch2_check_alloc_info(struct bch_fs *c)
 	ret = for_each_btree_key2(trans, iter,
 			BTREE_ID_need_discard, POS_MIN,
 			BTREE_ITER_PREFETCH, k,
-		bch2_check_discard_freespace_key(trans, &iter, k.k->p)) ?:
-	      for_each_btree_key2(trans, iter,
-			BTREE_ID_freespace, POS_MIN,
-			BTREE_ITER_PREFETCH, k,
-		bch2_check_discard_freespace_key(trans, &iter, k.k->p)) ?:
-	      for_each_btree_key_commit(trans, iter,
+		bch2_check_discard_freespace_key(trans, &iter));
+	if (ret)
+		goto err;
+
+	bch2_trans_iter_init(trans, &iter, BTREE_ID_freespace, POS_MIN,
+			     BTREE_ITER_PREFETCH);
+	while (1) {
+		bch2_trans_begin(trans);
+		k = bch2_btree_iter_peek(&iter);
+		if (!k.k)
+			break;
+
+		ret = bkey_err(k) ?:
+			bch2_check_discard_freespace_key(trans, &iter);
+		if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) {
+			ret = 0;
+			continue;
+		}
+		if (ret) {
+			struct printbuf buf = PRINTBUF;
+			bch2_bkey_val_to_text(&buf, c, k);
+
+			bch_err(c, "while checking %s", buf.buf);
+			printbuf_exit(&buf);
+			break;
+		}
+
+		bch2_btree_iter_set_pos(&iter, bpos_nosnap_successor(iter.pos));
+	}
+	bch2_trans_iter_exit(trans, &iter);
+	if (ret)
+		goto err;
+
+	ret = for_each_btree_key_commit(trans, iter,
 			BTREE_ID_bucket_gens, POS_MIN,
 			BTREE_ITER_PREFETCH, k,
-			NULL, NULL, BTREE_INSERT_NOFAIL|BTREE_INSERT_LAZY_RW,
+			NULL, NULL, BCH_TRANS_COMMIT_no_enospc|BCH_TRANS_COMMIT_lazy_rw,
 		bch2_check_bucket_gens_key(trans, &iter, k));
 err:
 	bch2_trans_put(trans);
@@ -1546,7 +1569,7 @@ int bch2_check_alloc_to_lru_refs(struct bch_fs *c)
 	ret = bch2_trans_run(c,
 		for_each_btree_key_commit(trans, iter, BTREE_ID_alloc,
 				POS_MIN, BTREE_ITER_PREFETCH, k,
-				NULL, NULL, BTREE_INSERT_NOFAIL|BTREE_INSERT_LAZY_RW,
+				NULL, NULL, BCH_TRANS_COMMIT_no_enospc|BCH_TRANS_COMMIT_lazy_rw,
 			bch2_check_alloc_to_lru_ref(trans, &iter)));
 	if (ret)
 		bch_err_fn(c, ret);
@@ -1655,7 +1678,7 @@ static int bch2_discard_one_bucket(struct btree_trans *trans,
 	ret =   bch2_trans_update(trans, &iter, &a->k_i, 0) ?:
 		bch2_trans_commit(trans, NULL, NULL,
 				  BCH_WATERMARK_btree|
-				  BTREE_INSERT_NOFAIL);
+				  BCH_TRANS_COMMIT_no_enospc);
 	if (ret)
 		goto out;
 
@@ -1760,7 +1783,7 @@ static int invalidate_one_bucket(struct btree_trans *trans,
 				BTREE_TRIGGER_BUCKET_INVALIDATE) ?:
 		bch2_trans_commit(trans, NULL, NULL,
 				  BCH_WATERMARK_btree|
-				  BTREE_INSERT_NOFAIL);
+				  BCH_TRANS_COMMIT_no_enospc);
 	if (ret)
 		goto out;
 
@@ -1802,7 +1825,7 @@ static void bch2_do_invalidates_work(struct work_struct *work)
 	unsigned i;
 	int ret = 0;
 
-	ret = bch2_btree_write_buffer_flush(trans);
+	ret = bch2_btree_write_buffer_tryflush(trans);
 	if (ret)
 		goto err;
 
@@ -1884,8 +1907,8 @@ int bch2_dev_freespace_init(struct bch_fs *c, struct bch_dev *ca,
 
 			ret =   bch2_bucket_do_index(trans, k, a, true) ?:
 				bch2_trans_commit(trans, NULL, NULL,
-						  BTREE_INSERT_LAZY_RW|
-						  BTREE_INSERT_NOFAIL);
+						  BCH_TRANS_COMMIT_lazy_rw|
+						  BCH_TRANS_COMMIT_no_enospc);
 			if (ret)
 				goto bkey_err;
 
@@ -1905,8 +1928,8 @@ int bch2_dev_freespace_init(struct bch_fs *c, struct bch_dev *ca,
 
 			ret = bch2_btree_insert_trans(trans, BTREE_ID_freespace, freespace, 0) ?:
 				bch2_trans_commit(trans, NULL, NULL,
-						  BTREE_INSERT_LAZY_RW|
-						  BTREE_INSERT_NOFAIL);
+						  BCH_TRANS_COMMIT_lazy_rw|
+						  BCH_TRANS_COMMIT_no_enospc);
 			if (ret)
 				goto bkey_err;
 
diff --git a/fs/bcachefs/alloc_foreground.c b/fs/bcachefs/alloc_foreground.c
index b85c7765272f..eef6fa8d0f9f 100644
--- a/fs/bcachefs/alloc_foreground.c
+++ b/fs/bcachefs/alloc_foreground.c
@@ -1297,6 +1297,30 @@ static struct write_point *writepoint_find(struct btree_trans *trans,
 	return wp;
 }
 
+static noinline void
+deallocate_extra_replicas(struct bch_fs *c,
+			  struct open_buckets *ptrs,
+			  struct open_buckets *ptrs_no_use,
+			  unsigned extra_replicas)
+{
+	struct open_buckets ptrs2 = { 0 };
+	struct open_bucket *ob;
+	unsigned i;
+
+	open_bucket_for_each(c, ptrs, ob, i) {
+		unsigned d = bch_dev_bkey_exists(c, ob->dev)->mi.durability;
+
+		if (d && d <= extra_replicas) {
+			extra_replicas -= d;
+			ob_push(c, ptrs_no_use, ob);
+		} else {
+			ob_push(c, &ptrs2, ob);
+		}
+	}
+
+	*ptrs = ptrs2;
+}
+
 /*
  * Get us an open_bucket we can allocate from, return with it locked:
  */
@@ -1382,6 +1406,9 @@ int bch2_alloc_sectors_start_trans(struct btree_trans *trans,
 	if (ret)
 		goto err;
 
+	if (nr_effective > nr_replicas)
+		deallocate_extra_replicas(c, &ptrs, &wp->ptrs, nr_effective - nr_replicas);
+
 	/* Free buckets we didn't use: */
 	open_bucket_for_each(c, &wp->ptrs, ob, i)
 		open_bucket_free_unused(c, ob);
diff --git a/fs/bcachefs/backpointers.c b/fs/bcachefs/backpointers.c
index 23c0834a97a4..4c8bcf23bb27 100644
--- a/fs/bcachefs/backpointers.c
+++ b/fs/bcachefs/backpointers.c
@@ -382,7 +382,7 @@ int bch2_check_btree_backpointers(struct bch_fs *c)
 	ret = bch2_trans_run(c,
 		for_each_btree_key_commit(trans, iter,
 			BTREE_ID_backpointers, POS_MIN, 0, k,
-			NULL, NULL, BTREE_INSERT_LAZY_RW|BTREE_INSERT_NOFAIL,
+			NULL, NULL, BCH_TRANS_COMMIT_lazy_rw|BCH_TRANS_COMMIT_no_enospc,
 		  bch2_check_btree_backpointer(trans, &iter, k)));
 	if (ret)
 		bch_err_fn(c, ret);
@@ -457,25 +457,18 @@ static int check_bp_exists(struct btree_trans *trans,
 }
 
 static int check_extent_to_backpointers(struct btree_trans *trans,
-					struct btree_iter *iter,
+					enum btree_id btree, unsigned level,
 					struct bpos bucket_start,
 					struct bpos bucket_end,
-					struct bpos_level *last_flushed)
+					struct bpos_level *last_flushed,
+					struct bkey_s_c k)
 {
 	struct bch_fs *c = trans->c;
 	struct bkey_ptrs_c ptrs;
 	const union bch_extent_entry *entry;
 	struct extent_ptr_decoded p;
-	struct bkey_s_c k;
 	int ret;
 
-	k = bch2_btree_iter_peek_all_levels(iter);
-	ret = bkey_err(k);
-	if (ret)
-		return ret;
-	if (!k.k)
-		return 0;
-
 	ptrs = bch2_bkey_ptrs_c(k);
 	bkey_for_each_ptr_decode(k.k, ptrs, p, entry) {
 		struct bpos bucket_pos;
@@ -484,7 +477,7 @@ static int check_extent_to_backpointers(struct btree_trans *trans,
 		if (p.ptr.cached)
 			continue;
 
-		bch2_extent_ptr_to_bp(c, iter->btree_id, iter->path->level,
+		bch2_extent_ptr_to_bp(c, btree, level,
 				      k, p, &bucket_pos, &bp);
 
 		ret = check_bp_exists(trans, bucket_pos, bp, k,
@@ -501,44 +494,33 @@ static int check_btree_root_to_backpointers(struct btree_trans *trans,
 					    enum btree_id btree_id,
 					    struct bpos bucket_start,
 					    struct bpos bucket_end,
-					    struct bpos_level *last_flushed)
+					    struct bpos_level *last_flushed,
+					    int *level)
 {
 	struct bch_fs *c = trans->c;
-	struct btree_root *r = bch2_btree_id_root(c, btree_id);
 	struct btree_iter iter;
 	struct btree *b;
 	struct bkey_s_c k;
-	struct bkey_ptrs_c ptrs;
-	struct extent_ptr_decoded p;
-	const union bch_extent_entry *entry;
 	int ret;
-
-	bch2_trans_node_iter_init(trans, &iter, btree_id, POS_MIN, 0, r->level, 0);
+retry:
+	bch2_trans_node_iter_init(trans, &iter, btree_id, POS_MIN,
+				  0, bch2_btree_id_root(c, btree_id)->b->c.level, 0);
 	b = bch2_btree_iter_peek_node(&iter);
 	ret = PTR_ERR_OR_ZERO(b);
 	if (ret)
 		goto err;
 
-	BUG_ON(b != btree_node_root(c, b));
-
-	k = bkey_i_to_s_c(&b->key);
-	ptrs = bch2_bkey_ptrs_c(k);
-	bkey_for_each_ptr_decode(k.k, ptrs, p, entry) {
-		struct bpos bucket_pos;
-		struct bch_backpointer bp;
-
-		if (p.ptr.cached)
-			continue;
+	if (b != btree_node_root(c, b)) {
+		bch2_trans_iter_exit(trans, &iter);
+		goto retry;
+	}
 
-		bch2_extent_ptr_to_bp(c, iter.btree_id, b->c.level + 1,
-				      k, p, &bucket_pos, &bp);
+	*level = b->c.level;
 
-		ret = check_bp_exists(trans, bucket_pos, bp, k,
+	k = bkey_i_to_s_c(&b->key);
+	ret = check_extent_to_backpointers(trans, btree_id, b->c.level + 1,
 				      bucket_start, bucket_end,
-				      last_flushed);
-		if (ret)
-			goto err;
-	}
+				      last_flushed, k);
 err:
 	bch2_trans_iter_exit(trans, &iter);
 	return ret;
@@ -616,43 +598,49 @@ static int bch2_check_extents_to_backpointers_pass(struct btree_trans *trans,
 	struct bch_fs *c = trans->c;
 	struct btree_iter iter;
 	enum btree_id btree_id;
+	struct bkey_s_c k;
 	struct bpos_level last_flushed = { UINT_MAX, POS_MIN };
 	int ret = 0;
 
 	for (btree_id = 0; btree_id < btree_id_nr_alive(c); btree_id++) {
-		unsigned depth = btree_type_has_ptrs(btree_id) ? 0 : 1;
-
-		bch2_trans_node_iter_init(trans, &iter, btree_id, POS_MIN, 0,
-					  depth,
-					  BTREE_ITER_ALL_LEVELS|
-					  BTREE_ITER_PREFETCH);
-
-		do {
-			ret = commit_do(trans, NULL, NULL,
-					BTREE_INSERT_LAZY_RW|
-					BTREE_INSERT_NOFAIL,
-					check_extent_to_backpointers(trans, &iter,
-								bucket_start, bucket_end,
-								&last_flushed));
-			if (ret)
-				break;
-		} while (!bch2_btree_iter_advance(&iter));
-
-		bch2_trans_iter_exit(trans, &iter);
-
-		if (ret)
-			break;
+		int level, depth = btree_type_has_ptrs(btree_id) ? 0 : 1;
 
 		ret = commit_do(trans, NULL, NULL,
-				BTREE_INSERT_LAZY_RW|
-				BTREE_INSERT_NOFAIL,
+				BCH_TRANS_COMMIT_lazy_rw|
+				BCH_TRANS_COMMIT_no_enospc,
 				check_btree_root_to_backpointers(trans, btree_id,
 							bucket_start, bucket_end,
-							&last_flushed));
+							&last_flushed, &level));
 		if (ret)
-			break;
+			return ret;
+
+		while (level >= depth) {
+			bch2_trans_node_iter_init(trans, &iter, btree_id, POS_MIN, 0,
+						  level,
+						  BTREE_ITER_PREFETCH);
+			for_each_btree_key_continue(trans, iter, BTREE_ITER_PREFETCH, k, ret) {
+				ret = commit_do(trans, NULL, NULL,
+						BCH_TRANS_COMMIT_lazy_rw|
+						BCH_TRANS_COMMIT_no_enospc,
+					check_extent_to_backpointers(trans, btree_id, level,
+								     bucket_start, bucket_end,
+								     &last_flushed, k));
+				if (ret)
+					break;
+
+				if (bpos_eq(iter.pos, SPOS_MAX))
+					break;
+			}
+			bch2_trans_iter_exit(trans, &iter);
+
+			if (ret)
+				return ret;
+
+			--level;
+		}
 	}
-	return ret;
+
+	return 0;
 }
 
 static struct bpos bucket_pos_to_bp_safe(const struct bch_fs *c,
@@ -807,7 +795,7 @@ static int bch2_check_backpointers_to_extents_pass(struct btree_trans *trans,
 
 	return for_each_btree_key_commit(trans, iter, BTREE_ID_backpointers,
 				  POS_MIN, BTREE_ITER_PREFETCH, k,
-				  NULL, NULL, BTREE_INSERT_LAZY_RW|BTREE_INSERT_NOFAIL,
+				  NULL, NULL, BCH_TRANS_COMMIT_lazy_rw|BCH_TRANS_COMMIT_no_enospc,
 		check_one_backpointer(trans, start, end,
 				      bkey_s_c_to_backpointer(k),
 				      &last_flushed_pos));
diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h
index 403aa3389fcc..53f93f03f088 100644
--- a/fs/bcachefs/bcachefs.h
+++ b/fs/bcachefs/bcachefs.h
@@ -401,9 +401,12 @@ BCH_DEBUG_PARAMS_DEBUG()
 	x(journal_flush_write)			\
 	x(journal_noflush_write)		\
 	x(journal_flush_seq)			\
-	x(blocked_journal)			\
+	x(blocked_journal_low_on_space)		\
+	x(blocked_journal_low_on_pin)		\
+	x(blocked_journal_max_in_flight)	\
 	x(blocked_allocate)			\
 	x(blocked_allocate_open_bucket)		\
+	x(blocked_write_buffer_full)		\
 	x(nocow_lock_contended)
 
 enum bch_time_stats {
@@ -638,6 +641,8 @@ struct journal_keys {
 	size_t			gap;
 	size_t			nr;
 	size_t			size;
+	atomic_t		ref;
+	bool			initial_ref_held;
 };
 
 struct btree_trans_buf {
@@ -662,7 +667,8 @@ struct btree_trans_buf {
 	x(invalidate)							\
 	x(delete_dead_snapshots)					\
 	x(snapshot_delete_pagecache)					\
-	x(sysfs)
+	x(sysfs)							\
+	x(btree_write_buffer)
 
 enum bch_write_ref {
 #define x(n) BCH_WRITE_REF_##n,
@@ -1062,6 +1068,16 @@ static inline void bch2_write_ref_get(struct bch_fs *c, enum bch_write_ref ref)
 #endif
 }
 
+static inline bool __bch2_write_ref_tryget(struct bch_fs *c, enum bch_write_ref ref)
+{
+#ifdef BCH_WRITE_REF_DEBUG
+	return !test_bit(BCH_FS_GOING_RO, &c->flags) &&
+		atomic_long_inc_not_zero(&c->writes[ref]);
+#else
+	return percpu_ref_tryget(&c->writes);
+#endif
+}
+
 static inline bool bch2_write_ref_tryget(struct bch_fs *c, enum bch_write_ref ref)
 {
 #ifdef BCH_WRITE_REF_DEBUG
diff --git a/fs/bcachefs/bcachefs_format.h b/fs/bcachefs/bcachefs_format.h
index 0a750953ff92..ad0f298c87c3 100644
--- a/fs/bcachefs/bcachefs_format.h
+++ b/fs/bcachefs/bcachefs_format.h
@@ -303,6 +303,13 @@ struct bkey_i {
 	struct bch_val	v;
 };
 
+#define POS_KEY(_pos)							\
+((struct bkey) {							\
+	.u64s		= BKEY_U64s,					\
+	.format		= KEY_FORMAT_CURRENT,				\
+	.p		= _pos,						\
+})
+
 #define KEY(_inode, _offset, _size)					\
 ((struct bkey) {							\
 	.u64s		= BKEY_U64s,					\
@@ -1436,7 +1443,7 @@ struct bch_sb_field_replicas_v0 {
 	struct bch_replicas_entry_v0 entries[];
 } __packed __aligned(8);
 
-struct bch_replicas_entry {
+struct bch_replicas_entry_v1 {
 	__u8			data_type;
 	__u8			nr_devs;
 	__u8			nr_required;
@@ -1448,7 +1455,7 @@ struct bch_replicas_entry {
 
 struct bch_sb_field_replicas {
 	struct bch_sb_field	field;
-	struct bch_replicas_entry entries[];
+	struct bch_replicas_entry_v1 entries[];
 } __packed __aligned(8);
 
 /* BCH_SB_FIELD_quota: */
@@ -2124,7 +2131,8 @@ static inline __u64 __bset_magic(struct bch_sb *sb)
 	x(clock,		7)		\
 	x(dev_usage,		8)		\
 	x(log,			9)		\
-	x(overwrite,		10)
+	x(overwrite,		10)		\
+	x(write_buffer_keys,	11)
 
 enum {
 #define x(f, nr)	BCH_JSET_ENTRY_##f	= nr,
@@ -2174,7 +2182,7 @@ struct jset_entry_usage {
 struct jset_entry_data_usage {
 	struct jset_entry	entry;
 	__le64			v;
-	struct bch_replicas_entry r;
+	struct bch_replicas_entry_v1 r;
 } __packed;
 
 struct jset_entry_clock {
diff --git a/fs/bcachefs/bcachefs_ioctl.h b/fs/bcachefs/bcachefs_ioctl.h
index f05881f7e113..18eb325401cf 100644
--- a/fs/bcachefs/bcachefs_ioctl.h
+++ b/fs/bcachefs/bcachefs_ioctl.h
@@ -173,12 +173,18 @@ struct bch_ioctl_disk_set_state {
 	__u64			dev;
 };
 
+#define BCH_DATA_OPS()			\
+	x(scrub,		0)	\
+	x(rereplicate,		1)	\
+	x(migrate,		2)	\
+	x(rewrite_old_nodes,	3)	\
+	x(drop_extra_replicas,	4)
+
 enum bch_data_ops {
-	BCH_DATA_OP_SCRUB		= 0,
-	BCH_DATA_OP_REREPLICATE		= 1,
-	BCH_DATA_OP_MIGRATE		= 2,
-	BCH_DATA_OP_REWRITE_OLD_NODES	= 3,
-	BCH_DATA_OP_NR			= 4,
+#define x(t, n) BCH_DATA_OP_##t = n,
+	BCH_DATA_OPS()
+#undef x
+	BCH_DATA_OP_NR
 };
 
 /*
@@ -237,7 +243,7 @@ struct bch_ioctl_data_event {
 
 struct bch_replicas_usage {
 	__u64			sectors;
-	struct bch_replicas_entry r;
+	struct bch_replicas_entry_v1 r;
 } __packed;
 
 static inline struct bch_replicas_usage *
diff --git a/fs/bcachefs/bkey_methods.h b/fs/bcachefs/bkey_methods.h
index 3a370b7087ac..912adadfb4dd 100644
--- a/fs/bcachefs/bkey_methods.h
+++ b/fs/bcachefs/bkey_methods.h
@@ -93,7 +93,6 @@ static inline int bch2_mark_key(struct btree_trans *trans,
 enum btree_update_flags {
 	__BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE = __BTREE_ITER_FLAGS_END,
 	__BTREE_UPDATE_NOJOURNAL,
-	__BTREE_UPDATE_PREJOURNAL,
 	__BTREE_UPDATE_KEY_CACHE_RECLAIM,
 
 	__BTREE_TRIGGER_NORUN,		/* Don't run triggers at all */
@@ -108,7 +107,6 @@ enum btree_update_flags {
 
 #define BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE (1U << __BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE)
 #define BTREE_UPDATE_NOJOURNAL		(1U << __BTREE_UPDATE_NOJOURNAL)
-#define BTREE_UPDATE_PREJOURNAL		(1U << __BTREE_UPDATE_PREJOURNAL)
 #define BTREE_UPDATE_KEY_CACHE_RECLAIM	(1U << __BTREE_UPDATE_KEY_CACHE_RECLAIM)
 
 #define BTREE_TRIGGER_NORUN		(1U << __BTREE_TRIGGER_NORUN)
diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c
index 0b5d09c8475d..136df94e9f84 100644
--- a/fs/bcachefs/btree_gc.c
+++ b/fs/bcachefs/btree_gc.c
@@ -1284,7 +1284,7 @@ static int bch2_gc_done(struct bch_fs *c,
 		}
 
 		for (i = 0; i < c->replicas.nr; i++) {
-			struct bch_replicas_entry *e =
+			struct bch_replicas_entry_v1 *e =
 				cpu_replicas_entry(&c->replicas, i);
 
 			if (metadata_only &&
@@ -1499,7 +1499,7 @@ static int bch2_gc_alloc_done(struct bch_fs *c, bool metadata_only)
 		ret = for_each_btree_key_commit(trans, iter, BTREE_ID_alloc,
 				POS(ca->dev_idx, ca->mi.first_bucket),
 				BTREE_ITER_SLOTS|BTREE_ITER_PREFETCH, k,
-				NULL, NULL, BTREE_INSERT_LAZY_RW,
+				NULL, NULL, BCH_TRANS_COMMIT_lazy_rw,
 			bch2_alloc_write_key(trans, &iter, k, metadata_only));
 
 		if (ret < 0) {
@@ -1656,7 +1656,7 @@ static int bch2_gc_reflink_done(struct bch_fs *c, bool metadata_only)
 	ret = for_each_btree_key_commit(trans, iter,
 			BTREE_ID_reflink, POS_MIN,
 			BTREE_ITER_PREFETCH, k,
-			NULL, NULL, BTREE_INSERT_NOFAIL,
+			NULL, NULL, BCH_TRANS_COMMIT_no_enospc,
 		bch2_gc_write_reflink_key(trans, &iter, k, &idx));
 
 	c->reflink_gc_nr = 0;
@@ -1780,7 +1780,7 @@ static int bch2_gc_stripes_done(struct bch_fs *c, bool metadata_only)
 	ret = for_each_btree_key_commit(trans, iter,
 			BTREE_ID_stripes, POS_MIN,
 			BTREE_ITER_PREFETCH, k,
-			NULL, NULL, BTREE_INSERT_NOFAIL,
+			NULL, NULL, BCH_TRANS_COMMIT_no_enospc,
 		bch2_gc_write_stripes_key(trans, &iter, k));
 
 	bch2_trans_put(trans);
@@ -2016,7 +2016,7 @@ int bch2_gc_gens(struct bch_fs *c)
 					BTREE_ITER_PREFETCH|BTREE_ITER_ALL_SNAPSHOTS,
 					k,
 					NULL, NULL,
-					BTREE_INSERT_NOFAIL,
+					BCH_TRANS_COMMIT_no_enospc,
 				gc_btree_gens_key(trans, &iter, k));
 			if (ret && !bch2_err_matches(ret, EROFS))
 				bch_err_fn(c, ret);
@@ -2029,7 +2029,7 @@ int bch2_gc_gens(struct bch_fs *c)
 			BTREE_ITER_PREFETCH,
 			k,
 			NULL, NULL,
-			BTREE_INSERT_NOFAIL,
+			BCH_TRANS_COMMIT_no_enospc,
 		bch2_alloc_write_oldest_gen(trans, &iter, k));
 	if (ret && !bch2_err_matches(ret, EROFS))
 		bch_err_fn(c, ret);
diff --git a/fs/bcachefs/btree_io.c b/fs/bcachefs/btree_io.c
index 37d896edb06e..3c663c596b46 100644
--- a/fs/bcachefs/btree_io.c
+++ b/fs/bcachefs/btree_io.c
@@ -1358,10 +1358,9 @@ static bool btree_node_has_extra_bsets(struct bch_fs *c, unsigned offset, void *
 	return offset;
 }
 
-static void btree_node_read_all_replicas_done(struct closure *cl)
+static CLOSURE_CALLBACK(btree_node_read_all_replicas_done)
 {
-	struct btree_node_read_all *ra =
-		container_of(cl, struct btree_node_read_all, cl);
+	closure_type(ra, struct btree_node_read_all, cl);
 	struct bch_fs *c = ra->c;
 	struct btree *b = ra->b;
 	struct printbuf buf = PRINTBUF;
@@ -1567,7 +1566,7 @@ static int btree_node_read_all_replicas(struct bch_fs *c, struct btree *b, bool
 
 	if (sync) {
 		closure_sync(&ra->cl);
-		btree_node_read_all_replicas_done(&ra->cl);
+		btree_node_read_all_replicas_done(&ra->cl.work);
 	} else {
 		continue_at(&ra->cl, btree_node_read_all_replicas_done,
 			    c->io_complete_wq);
@@ -1801,9 +1800,9 @@ static void btree_node_write_work(struct work_struct *work)
 		ret = bch2_trans_do(c, NULL, NULL, 0,
 			bch2_btree_node_update_key_get_iter(trans, b, &wbio->key,
 					BCH_WATERMARK_reclaim|
-					BTREE_INSERT_JOURNAL_RECLAIM|
-					BTREE_INSERT_NOFAIL|
-					BTREE_INSERT_NOCHECK_RW,
+					BCH_TRANS_COMMIT_journal_reclaim|
+					BCH_TRANS_COMMIT_no_enospc|
+					BCH_TRANS_COMMIT_no_check_rw,
 					!wbio->wbio.failed.nr));
 		if (ret)
 			goto err;
diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c
index 6fa90bcd7016..1d734e297eb4 100644
--- a/fs/bcachefs/btree_iter.c
+++ b/fs/bcachefs/btree_iter.c
@@ -1214,8 +1214,6 @@ __bch2_btree_path_set_pos(struct btree_trans *trans,
 		   struct btree_path *path, struct bpos new_pos,
 		   bool intent, unsigned long ip, int cmp)
 {
-	unsigned level = path->level;
-
 	bch2_trans_verify_not_in_restart(trans);
 	EBUG_ON(!path->ref);
 
@@ -1231,7 +1229,7 @@ __bch2_btree_path_set_pos(struct btree_trans *trans,
 		goto out;
 	}
 
-	level = btree_path_up_until_good_node(trans, path, cmp);
+	unsigned level = btree_path_up_until_good_node(trans, path, cmp);
 
 	if (btree_path_node(path, level)) {
 		struct btree_path_level *l = &path->l[level];
@@ -1799,23 +1797,15 @@ struct btree *bch2_btree_iter_next_node(struct btree_iter *iter)
 
 inline bool bch2_btree_iter_advance(struct btree_iter *iter)
 {
-	if (likely(!(iter->flags & BTREE_ITER_ALL_LEVELS))) {
-		struct bpos pos = iter->k.p;
-		bool ret = !(iter->flags & BTREE_ITER_ALL_SNAPSHOTS
-			     ? bpos_eq(pos, SPOS_MAX)
-			     : bkey_eq(pos, SPOS_MAX));
-
-		if (ret && !(iter->flags & BTREE_ITER_IS_EXTENTS))
-			pos = bkey_successor(iter, pos);
-		bch2_btree_iter_set_pos(iter, pos);
-		return ret;
-	} else {
-		if (!btree_path_node(iter->path, iter->path->level))
-			return true;
+	struct bpos pos = iter->k.p;
+	bool ret = !(iter->flags & BTREE_ITER_ALL_SNAPSHOTS
+		     ? bpos_eq(pos, SPOS_MAX)
+		     : bkey_eq(pos, SPOS_MAX));
 
-		iter->advanced = true;
-		return false;
-	}
+	if (ret && !(iter->flags & BTREE_ITER_IS_EXTENTS))
+		pos = bkey_successor(iter, pos);
+	bch2_btree_iter_set_pos(iter, pos);
+	return ret;
 }
 
 inline bool bch2_btree_iter_rewind(struct btree_iter *iter)
@@ -1864,19 +1854,11 @@ static struct bkey_i *bch2_btree_journal_peek(struct btree_trans *trans,
 					      struct btree_iter *iter,
 					      struct bpos end_pos)
 {
-	struct bkey_i *k;
-
-	if (bpos_lt(iter->path->pos, iter->journal_pos))
-		iter->journal_idx = 0;
-
-	k = bch2_journal_keys_peek_upto(trans->c, iter->btree_id,
-					iter->path->level,
-					iter->path->pos,
-					end_pos,
-					&iter->journal_idx);
-
-	iter->journal_pos = k ? k->k.p : end_pos;
-	return k;
+	return bch2_journal_keys_peek_upto(trans->c, iter->btree_id,
+					   iter->path->level,
+					   iter->path->pos,
+					   end_pos,
+					   &iter->journal_idx);
 }
 
 static noinline
@@ -2066,7 +2048,6 @@ struct bkey_s_c bch2_btree_iter_peek_upto(struct btree_iter *iter, struct bpos e
 	struct bpos iter_pos;
 	int ret;
 
-	EBUG_ON(iter->flags & BTREE_ITER_ALL_LEVELS);
 	EBUG_ON((iter->flags & BTREE_ITER_FILTER_SNAPSHOTS) && bkey_eq(end, POS_MAX));
 
 	if (iter->update_path) {
@@ -2192,103 +2173,6 @@ struct bkey_s_c bch2_btree_iter_peek_upto(struct btree_iter *iter, struct bpos e
 	goto out_no_locked;
 }
 
-/**
- * bch2_btree_iter_peek_all_levels() - returns the first key greater than or
- * equal to iterator's current position, returning keys from every level of the
- * btree. For keys at different levels of the btree that compare equal, the key
- * from the lower level (leaf) is returned first.
- * @iter:	iterator to peek from
- *
- * Returns:	key if found, or an error extractable with bkey_err().
- */
-struct bkey_s_c bch2_btree_iter_peek_all_levels(struct btree_iter *iter)
-{
-	struct btree_trans *trans = iter->trans;
-	struct bkey_s_c k;
-	int ret;
-
-	EBUG_ON(iter->path->cached);
-	bch2_btree_iter_verify(iter);
-	BUG_ON(iter->path->level < iter->min_depth);
-	BUG_ON(!(iter->flags & BTREE_ITER_ALL_SNAPSHOTS));
-	EBUG_ON(!(iter->flags & BTREE_ITER_ALL_LEVELS));
-
-	while (1) {
-		iter->path = bch2_btree_path_set_pos(trans, iter->path, iter->pos,
-					iter->flags & BTREE_ITER_INTENT,
-					btree_iter_ip_allocated(iter));
-
-		ret = bch2_btree_path_traverse(trans, iter->path, iter->flags);
-		if (unlikely(ret)) {
-			/* ensure that iter->k is consistent with iter->pos: */
-			bch2_btree_iter_set_pos(iter, iter->pos);
-			k = bkey_s_c_err(ret);
-			goto out_no_locked;
-		}
-
-		/* Already at end? */
-		if (!btree_path_node(iter->path, iter->path->level)) {
-			k = bkey_s_c_null;
-			goto out_no_locked;
-		}
-
-		k = btree_path_level_peek_all(trans->c,
-				&iter->path->l[iter->path->level], &iter->k);
-
-		/* Check if we should go up to the parent node: */
-		if (!k.k ||
-		    (iter->advanced &&
-		     bpos_eq(path_l(iter->path)->b->key.k.p, iter->pos))) {
-			iter->pos = path_l(iter->path)->b->key.k.p;
-			btree_path_set_level_up(trans, iter->path);
-			iter->advanced = false;
-			continue;
-		}
-
-		/*
-		 * Check if we should go back down to a leaf:
-		 * If we're not in a leaf node, we only return the current key
-		 * if it exactly matches iter->pos - otherwise we first have to
-		 * go back to the leaf:
-		 */
-		if (iter->path->level != iter->min_depth &&
-		    (iter->advanced ||
-		     !k.k ||
-		     !bpos_eq(iter->pos, k.k->p))) {
-			btree_path_set_level_down(trans, iter->path, iter->min_depth);
-			iter->pos = bpos_successor(iter->pos);
-			iter->advanced = false;
-			continue;
-		}
-
-		/* Check if we should go to the next key: */
-		if (iter->path->level == iter->min_depth &&
-		    iter->advanced &&
-		    k.k &&
-		    bpos_eq(iter->pos, k.k->p)) {
-			iter->pos = bpos_successor(iter->pos);
-			iter->advanced = false;
-			continue;
-		}
-
-		if (iter->advanced &&
-		    iter->path->level == iter->min_depth &&
-		    !bpos_eq(k.k->p, iter->pos))
-			iter->advanced = false;
-
-		BUG_ON(iter->advanced);
-		BUG_ON(!k.k);
-		break;
-	}
-
-	iter->pos = k.k->p;
-	btree_path_set_should_be_locked(iter->path);
-out_no_locked:
-	bch2_btree_iter_verify(iter);
-
-	return k;
-}
-
 /**
  * bch2_btree_iter_next() - returns first key greater than iterator's current
  * position
@@ -2455,7 +2339,6 @@ struct bkey_s_c bch2_btree_iter_peek_slot(struct btree_iter *iter)
 
 	bch2_btree_iter_verify(iter);
 	bch2_btree_iter_verify_entry_exit(iter);
-	EBUG_ON(iter->flags & BTREE_ITER_ALL_LEVELS);
 	EBUG_ON(iter->path->level && (iter->flags & BTREE_ITER_WITH_KEY_CACHE));
 
 	/* extents can't span inode numbers: */
@@ -2981,7 +2864,8 @@ struct btree_trans *__bch2_trans_get(struct bch_fs *c, unsigned fn_idx)
 	trans->fn_idx		= fn_idx;
 	trans->locking_wait.task = current;
 	trans->journal_replay_not_finished =
-		!test_bit(JOURNAL_REPLAY_DONE, &c->journal.flags);
+		unlikely(!test_bit(JOURNAL_REPLAY_DONE, &c->journal.flags)) &&
+		atomic_inc_not_zero(&c->journal_keys.ref);
 	closure_init_stack(&trans->ref);
 
 	s = btree_trans_stats(trans);
@@ -3098,6 +2982,9 @@ void bch2_trans_put(struct btree_trans *trans)
 			kfree(trans->fs_usage_deltas);
 	}
 
+	if (unlikely(trans->journal_replay_not_finished))
+		bch2_journal_keys_put(c);
+
 	if (trans->mem_bytes == BTREE_TRANS_MEM_MAX)
 		mempool_free(trans->mem, &c->btree_trans_mem_pool);
 	else
diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h
index 85e7cb52f6b6..a4fec7cce911 100644
--- a/fs/bcachefs/btree_iter.h
+++ b/fs/bcachefs/btree_iter.h
@@ -348,8 +348,6 @@ struct btree *bch2_btree_iter_next_node(struct btree_iter *);
 struct bkey_s_c bch2_btree_iter_peek_upto(struct btree_iter *, struct bpos);
 struct bkey_s_c bch2_btree_iter_next(struct btree_iter *);
 
-struct bkey_s_c bch2_btree_iter_peek_all_levels(struct btree_iter *);
-
 static inline struct bkey_s_c bch2_btree_iter_peek(struct btree_iter *iter)
 {
 	return bch2_btree_iter_peek_upto(iter, SPOS_MAX);
@@ -408,9 +406,6 @@ static inline unsigned __bch2_btree_iter_flags(struct btree_trans *trans,
 					       unsigned btree_id,
 					       unsigned flags)
 {
-	if (flags & BTREE_ITER_ALL_LEVELS)
-		flags |= BTREE_ITER_ALL_SNAPSHOTS|__BTREE_ITER_ALL_SNAPSHOTS;
-
 	if (!(flags & (BTREE_ITER_ALL_SNAPSHOTS|BTREE_ITER_NOT_EXTENTS)) &&
 	    btree_id_is_extents(btree_id))
 		flags |= BTREE_ITER_IS_EXTENTS;
@@ -450,14 +445,16 @@ static inline void bch2_trans_iter_init_common(struct btree_trans *trans,
 					  unsigned flags,
 					  unsigned long ip)
 {
-	memset(iter, 0, sizeof(*iter));
-	iter->trans	= trans;
-	iter->btree_id	= btree_id;
-	iter->flags	= flags;
-	iter->snapshot	= pos.snapshot;
-	iter->pos	= pos;
-	iter->k.p	= pos;
-
+	iter->trans		= trans;
+	iter->update_path	= NULL;
+	iter->key_cache_path	= NULL;
+	iter->btree_id		= btree_id;
+	iter->min_depth		= 0;
+	iter->flags		= flags;
+	iter->snapshot		= pos.snapshot;
+	iter->pos		= pos;
+	iter->k			= POS_KEY(pos);
+	iter->journal_idx	= 0;
 #ifdef CONFIG_BCACHEFS_DEBUG
 	iter->ip_allocated = ip;
 #endif
@@ -606,8 +603,6 @@ u32 bch2_trans_begin(struct btree_trans *);
 static inline struct bkey_s_c bch2_btree_iter_peek_prev_type(struct btree_iter *iter,
 							     unsigned flags)
 {
-	BUG_ON(flags & BTREE_ITER_ALL_LEVELS);
-
 	return  flags & BTREE_ITER_SLOTS      ? bch2_btree_iter_peek_slot(iter) :
 						bch2_btree_iter_peek_prev(iter);
 }
@@ -615,8 +610,7 @@ static inline struct bkey_s_c bch2_btree_iter_peek_prev_type(struct btree_iter *
 static inline struct bkey_s_c bch2_btree_iter_peek_type(struct btree_iter *iter,
 							unsigned flags)
 {
-	return  flags & BTREE_ITER_ALL_LEVELS ? bch2_btree_iter_peek_all_levels(iter) :
-		flags & BTREE_ITER_SLOTS      ? bch2_btree_iter_peek_slot(iter) :
+	return  flags & BTREE_ITER_SLOTS      ? bch2_btree_iter_peek_slot(iter) :
 						bch2_btree_iter_peek(iter);
 }
 
diff --git a/fs/bcachefs/btree_journal_iter.c b/fs/bcachefs/btree_journal_iter.c
index 58a981bcf3aa..7a5e0a893df9 100644
--- a/fs/bcachefs/btree_journal_iter.c
+++ b/fs/bcachefs/btree_journal_iter.c
@@ -73,6 +73,7 @@ static size_t bch2_journal_key_search(struct journal_keys *keys,
 	return idx_to_pos(keys, __bch2_journal_key_search(keys, id, level, pos));
 }
 
+/* Returns first non-overwritten key >= search key: */
 struct bkey_i *bch2_journal_keys_peek_upto(struct bch_fs *c, enum btree_id btree_id,
 					   unsigned level, struct bpos pos,
 					   struct bpos end_pos, size_t *idx)
@@ -80,16 +81,32 @@ struct bkey_i *bch2_journal_keys_peek_upto(struct bch_fs *c, enum btree_id btree
 	struct journal_keys *keys = &c->journal_keys;
 	unsigned iters = 0;
 	struct journal_key *k;
+
+	BUG_ON(*idx > keys->nr);
 search:
 	if (!*idx)
 		*idx = __bch2_journal_key_search(keys, btree_id, level, pos);
 
+	while (*idx &&
+	       __journal_key_cmp(btree_id, level, end_pos, idx_to_key(keys, *idx - 1)) <= 0) {
+		--(*idx);
+		iters++;
+		if (iters == 10) {
+			*idx = 0;
+			goto search;
+		}
+	}
+
 	while ((k = *idx < keys->nr ? idx_to_key(keys, *idx) : NULL)) {
 		if (__journal_key_cmp(btree_id, level, end_pos, k) < 0)
 			return NULL;
 
-		if (__journal_key_cmp(btree_id, level, pos, k) <= 0 &&
-		    !k->overwritten)
+		if (k->overwritten) {
+			(*idx)++;
+			continue;
+		}
+
+		if (__journal_key_cmp(btree_id, level, pos, k) <= 0)
 			return k->k;
 
 		(*idx)++;
@@ -189,10 +206,12 @@ int bch2_journal_key_insert_take(struct bch_fs *c, enum btree_id id,
 		/* Since @keys was full, there was no gap: */
 		memcpy(new_keys.d, keys->d, sizeof(keys->d[0]) * keys->nr);
 		kvfree(keys->d);
-		*keys = new_keys;
+		keys->d		= new_keys.d;
+		keys->nr	= new_keys.nr;
+		keys->size	= new_keys.size;
 
 		/* And now the gap is at the end: */
-		keys->gap = keys->nr;
+		keys->gap	= keys->nr;
 	}
 
 	journal_iters_move_gap(c, keys->gap, idx);
@@ -415,10 +434,16 @@ static int journal_sort_key_cmp(const void *_l, const void *_r)
 		cmp_int(l->journal_offset, r->journal_offset);
 }
 
-void bch2_journal_keys_free(struct journal_keys *keys)
+void bch2_journal_keys_put(struct bch_fs *c)
 {
+	struct journal_keys *keys = &c->journal_keys;
 	struct journal_key *i;
 
+	BUG_ON(atomic_read(&keys->ref) <= 0);
+
+	if (!atomic_dec_and_test(&keys->ref))
+		return;
+
 	move_gap(keys->d, keys->nr, keys->size, keys->gap, keys->nr);
 	keys->gap = keys->nr;
 
@@ -429,6 +454,8 @@ void bch2_journal_keys_free(struct journal_keys *keys)
 	kvfree(keys->d);
 	keys->d = NULL;
 	keys->nr = keys->gap = keys->size = 0;
+
+	bch2_journal_entries_free(c);
 }
 
 static void __journal_keys_sort(struct journal_keys *keys)
diff --git a/fs/bcachefs/btree_journal_iter.h b/fs/bcachefs/btree_journal_iter.h
index 5d64e7e22f26..8ca4c100b2e3 100644
--- a/fs/bcachefs/btree_journal_iter.h
+++ b/fs/bcachefs/btree_journal_iter.h
@@ -49,7 +49,15 @@ void bch2_btree_and_journal_iter_init_node_iter(struct btree_and_journal_iter *,
 						struct bch_fs *,
 						struct btree *);
 
-void bch2_journal_keys_free(struct journal_keys *);
+void bch2_journal_keys_put(struct bch_fs *);
+
+static inline void bch2_journal_keys_put_initial(struct bch_fs *c)
+{
+	if (c->journal_keys.initial_ref_held)
+		bch2_journal_keys_put(c);
+	c->journal_keys.initial_ref_held = false;
+}
+
 void bch2_journal_entries_free(struct bch_fs *);
 
 int bch2_journal_keys_sort(struct bch_fs *);
diff --git a/fs/bcachefs/btree_key_cache.c b/fs/bcachefs/btree_key_cache.c
index 37fbf22de8fc..4402cf068c56 100644
--- a/fs/bcachefs/btree_key_cache.c
+++ b/fs/bcachefs/btree_key_cache.c
@@ -645,22 +645,27 @@ static int btree_key_cache_flush_pos(struct btree_trans *trans,
 	if (journal_seq && ck->journal.seq != journal_seq)
 		goto out;
 
+	trans->journal_res.seq = ck->journal.seq;
+
 	/*
-	 * Since journal reclaim depends on us making progress here, and the
-	 * allocator/copygc depend on journal reclaim making progress, we need
-	 * to be using alloc reserves:
+	 * If we're at the end of the journal, we really want to free up space
+	 * in the journal right away - we don't want to pin that old journal
+	 * sequence number with a new btree node write, we want to re-journal
+	 * the update
 	 */
+	if (ck->journal.seq == journal_last_seq(j))
+		commit_flags |= BCH_WATERMARK_reclaim;
+	else
+		commit_flags |= BCH_TRANS_COMMIT_no_journal_res;
+
 	ret   = bch2_btree_iter_traverse(&b_iter) ?:
 		bch2_trans_update(trans, &b_iter, ck->k,
 				  BTREE_UPDATE_KEY_CACHE_RECLAIM|
 				  BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE|
 				  BTREE_TRIGGER_NORUN) ?:
 		bch2_trans_commit(trans, NULL, NULL,
-				  BTREE_INSERT_NOCHECK_RW|
-				  BTREE_INSERT_NOFAIL|
-				  (ck->journal.seq == journal_last_seq(j)
-				   ? BCH_WATERMARK_reclaim
-				   : 0)|
+				  BCH_TRANS_COMMIT_no_check_rw|
+				  BCH_TRANS_COMMIT_no_enospc|
 				  commit_flags);
 
 	bch2_fs_fatal_err_on(ret &&
@@ -734,7 +739,7 @@ int bch2_btree_key_cache_journal_flush(struct journal *j,
 
 	ret = commit_do(trans, NULL, NULL, 0,
 		btree_key_cache_flush_pos(trans, key, seq,
-				BTREE_INSERT_JOURNAL_RECLAIM, false));
+				BCH_TRANS_COMMIT_journal_reclaim, false));
 unlock:
 	srcu_read_unlock(&c->btree_trans_barrier, srcu_idx);
 
diff --git a/fs/bcachefs/btree_locking.c b/fs/bcachefs/btree_locking.c
index 3d48834d091f..6039278121dc 100644
--- a/fs/bcachefs/btree_locking.c
+++ b/fs/bcachefs/btree_locking.c
@@ -665,7 +665,7 @@ void __bch2_btree_path_downgrade(struct btree_trans *trans,
 				 struct btree_path *path,
 				 unsigned new_locks_want)
 {
-	unsigned l;
+	unsigned l, old_locks_want = path->locks_want;
 
 	if (trans->restarted)
 		return;
@@ -690,7 +690,7 @@ void __bch2_btree_path_downgrade(struct btree_trans *trans,
 	bch2_btree_path_verify_locks(path);
 
 	path->downgrade_seq++;
-	trace_path_downgrade(trans, _RET_IP_, path);
+	trace_path_downgrade(trans, _RET_IP_, path, old_locks_want);
 }
 
 /* Btree transaction locking: */
diff --git a/fs/bcachefs/btree_trans_commit.c b/fs/bcachefs/btree_trans_commit.c
index 12907beda98c..09e94cc460ef 100644
--- a/fs/bcachefs/btree_trans_commit.c
+++ b/fs/bcachefs/btree_trans_commit.c
@@ -409,7 +409,7 @@ static int btree_key_can_insert_cached(struct btree_trans *trans, unsigned flags
 
 	if (!test_bit(BKEY_CACHED_DIRTY, &ck->flags) &&
 	    bch2_btree_key_cache_must_wait(c) &&
-	    !(flags & BTREE_INSERT_JOURNAL_RECLAIM))
+	    !(flags & BCH_TRANS_COMMIT_journal_reclaim))
 		return -BCH_ERR_btree_insert_need_journal_reclaim;
 
 	/*
@@ -656,17 +656,15 @@ bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags,
 			*stopped_at = i;
 			return ret;
 		}
-	}
 
-	if (trans->nr_wb_updates &&
-	    trans->nr_wb_updates + c->btree_write_buffer.state.nr > c->btree_write_buffer.size)
-		return -BCH_ERR_btree_insert_need_flush_buffer;
+		i->k->k.needs_whiteout = false;
+	}
 
 	/*
 	 * Don't get journal reservation until after we know insert will
 	 * succeed:
 	 */
-	if (likely(!(flags & BTREE_INSERT_JOURNAL_REPLAY))) {
+	if (likely(!(flags & BCH_TRANS_COMMIT_no_journal_res))) {
 		ret = bch2_trans_journal_res_get(trans,
 				(flags & BCH_WATERMARK_MASK)|
 				JOURNAL_RES_GET_NONBLOCK);
@@ -675,8 +673,6 @@ bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags,
 
 		if (unlikely(trans->journal_transaction_names))
 			journal_transaction_name(trans);
-	} else {
-		trans->journal_res.seq = c->journal.replay_journal_seq;
 	}
 
 	/*
@@ -685,7 +681,7 @@ bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags,
 	 */
 
 	if (IS_ENABLED(CONFIG_BCACHEFS_DEBUG) &&
-	    !(flags & BTREE_INSERT_JOURNAL_REPLAY)) {
+	    !(flags & BCH_TRANS_COMMIT_no_journal_res)) {
 		if (bch2_journal_seq_verify)
 			trans_for_each_update(trans, i)
 				i->k->k.version.lo = trans->journal_res.seq;
@@ -698,14 +694,6 @@ bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags,
 	    bch2_trans_fs_usage_apply(trans, trans->fs_usage_deltas))
 		return -BCH_ERR_btree_insert_need_mark_replicas;
 
-	if (trans->nr_wb_updates) {
-		EBUG_ON(flags & BTREE_INSERT_JOURNAL_REPLAY);
-
-		ret = bch2_btree_insert_keys_write_buffer(trans);
-		if (ret)
-			goto revert_fs_usage;
-	}
-
 	h = trans->hooks;
 	while (h) {
 		ret = h->fn(trans, h);
@@ -736,7 +724,7 @@ bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags,
 		trans->journal_res.u64s		-= trans->extra_journal_entries.nr;
 	}
 
-	if (likely(!(flags & BTREE_INSERT_JOURNAL_REPLAY))) {
+	if (likely(!(flags & BCH_TRANS_COMMIT_no_journal_res))) {
 		struct journal *j = &c->journal;
 		struct jset_entry *entry;
 
@@ -767,7 +755,7 @@ bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags,
 
 		trans_for_each_wb_update(trans, wb) {
 			entry = bch2_journal_add_entry(j, &trans->journal_res,
-					       BCH_JSET_ENTRY_btree_keys,
+					       BCH_JSET_ENTRY_write_buffer_keys,
 					       wb->btree, 0,
 					       wb->k.k.u64s);
 			bkey_copy((struct bkey_i *) entry->start, &wb->k);
@@ -778,15 +766,8 @@ bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags,
 	}
 
 	trans_for_each_update(trans, i) {
-		i->k->k.needs_whiteout = false;
-
 		if (!i->cached) {
-			u64 seq = trans->journal_res.seq;
-
-			if (i->flags & BTREE_UPDATE_PREJOURNAL)
-				seq = i->seq;
-
-			bch2_btree_insert_key_leaf(trans, i->path, i->k, seq);
+			bch2_btree_insert_key_leaf(trans, i->path, i->k, trans->journal_res.seq);
 		} else if (!i->key_cache_already_flushed)
 			bch2_btree_insert_key_cached(trans, flags, i);
 		else {
@@ -841,6 +822,12 @@ static noinline int bch2_trans_commit_bkey_invalid(struct btree_trans *trans,
 	return -EINVAL;
 }
 
+static int bch2_trans_commit_journal_pin_flush(struct journal *j,
+				struct journal_entry_pin *_pin, u64 seq)
+{
+	return 0;
+}
+
 /*
  * Get journal reservation, take write locks, and attempt to do btree update(s):
  */
@@ -884,13 +871,15 @@ static inline int do_bch2_trans_commit(struct btree_trans *trans, unsigned flags
 
 	if (!ret && trans->journal_pin)
 		bch2_journal_pin_add(&c->journal, trans->journal_res.seq,
-				     trans->journal_pin, NULL);
+				     trans->journal_pin,
+				     bch2_trans_commit_journal_pin_flush);
 
 	/*
 	 * Drop journal reservation after dropping write locks, since dropping
 	 * the journal reservation may kick off a journal write:
 	 */
-	bch2_journal_res_put(&c->journal, &trans->journal_res);
+	if (likely(!(flags & BCH_TRANS_COMMIT_no_journal_res)))
+		bch2_journal_res_put(&c->journal, &trans->journal_res);
 
 	return ret;
 }
@@ -927,7 +916,7 @@ int bch2_trans_commit_error(struct btree_trans *trans, unsigned flags,
 		 * XXX: this should probably be a separate BTREE_INSERT_NONBLOCK
 		 * flag
 		 */
-		if ((flags & BTREE_INSERT_JOURNAL_RECLAIM) &&
+		if ((flags & BCH_TRANS_COMMIT_journal_reclaim) &&
 		    (flags & BCH_WATERMARK_MASK) != BCH_WATERMARK_reclaim) {
 			ret = -BCH_ERR_journal_reclaim_would_deadlock;
 			break;
@@ -950,30 +939,6 @@ int bch2_trans_commit_error(struct btree_trans *trans, unsigned flags,
 
 		ret = bch2_trans_relock(trans);
 		break;
-	case -BCH_ERR_btree_insert_need_flush_buffer: {
-		struct btree_write_buffer *wb = &c->btree_write_buffer;
-
-		ret = 0;
-
-		if (wb->state.nr > wb->size * 3 / 4) {
-			bch2_trans_unlock(trans);
-			mutex_lock(&wb->flush_lock);
-
-			if (wb->state.nr > wb->size * 3 / 4) {
-				bch2_trans_begin(trans);
-				ret = __bch2_btree_write_buffer_flush(trans,
-						flags|BTREE_INSERT_NOCHECK_RW, true);
-				if (!ret) {
-					trace_and_count(c, trans_restart_write_buffer_flush, trans, _THIS_IP_);
-					ret = btree_trans_restart(trans, BCH_ERR_transaction_restart_write_buffer_flush);
-				}
-			} else {
-				mutex_unlock(&wb->flush_lock);
-				ret = bch2_trans_relock(trans);
-			}
-		}
-		break;
-	}
 	default:
 		BUG_ON(ret >= 0);
 		break;
@@ -982,8 +947,7 @@ int bch2_trans_commit_error(struct btree_trans *trans, unsigned flags,
 	BUG_ON(bch2_err_matches(ret, BCH_ERR_transaction_restart) != !!trans->restarted);
 
 	bch2_fs_inconsistent_on(bch2_err_matches(ret, ENOSPC) &&
-				!(flags & BTREE_INSERT_NOWAIT) &&
-				(flags & BTREE_INSERT_NOFAIL), c,
+				(flags & BCH_TRANS_COMMIT_no_enospc), c,
 		"%s: incorrectly got %s\n", __func__, bch2_err_str(ret));
 
 	return ret;
@@ -995,7 +959,7 @@ bch2_trans_commit_get_rw_cold(struct btree_trans *trans, unsigned flags)
 	struct bch_fs *c = trans->c;
 	int ret;
 
-	if (likely(!(flags & BTREE_INSERT_LAZY_RW)) ||
+	if (likely(!(flags & BCH_TRANS_COMMIT_lazy_rw)) ||
 	    test_bit(BCH_FS_STARTED, &c->flags))
 		return -BCH_ERR_erofs_trans_commit;
 
@@ -1040,9 +1004,6 @@ int __bch2_trans_commit(struct btree_trans *trans, unsigned flags)
 	    !trans->extra_journal_entries.nr)
 		goto out_reset;
 
-	if (flags & BTREE_INSERT_GC_LOCK_HELD)
-		lockdep_assert_held(&c->gc_lock);
-
 	ret = bch2_trans_commit_run_triggers(trans);
 	if (ret)
 		goto out_reset;
@@ -1051,7 +1012,7 @@ int __bch2_trans_commit(struct btree_trans *trans, unsigned flags)
 		struct printbuf buf = PRINTBUF;
 		enum bkey_invalid_flags invalid_flags = 0;
 
-		if (!(flags & BTREE_INSERT_JOURNAL_REPLAY))
+		if (!(flags & BCH_TRANS_COMMIT_no_journal_res))
 			invalid_flags |= BKEY_INVALID_WRITE|BKEY_INVALID_COMMIT;
 
 		if (unlikely(bch2_bkey_invalid(c, bkey_i_to_s_c(i->k),
@@ -1069,27 +1030,13 @@ int __bch2_trans_commit(struct btree_trans *trans, unsigned flags)
 		goto out_reset;
 	}
 
-	if (!(flags & BTREE_INSERT_NOCHECK_RW) &&
+	if (!(flags & BCH_TRANS_COMMIT_no_check_rw) &&
 	    unlikely(!bch2_write_ref_tryget(c, BCH_WRITE_REF_trans))) {
 		ret = bch2_trans_commit_get_rw_cold(trans, flags);
 		if (ret)
 			goto out_reset;
 	}
 
-	if (c->btree_write_buffer.state.nr > c->btree_write_buffer.size / 2 &&
-	    mutex_trylock(&c->btree_write_buffer.flush_lock)) {
-		bch2_trans_begin(trans);
-		bch2_trans_unlock(trans);
-
-		ret = __bch2_btree_write_buffer_flush(trans,
-					flags|BTREE_INSERT_NOCHECK_RW, true);
-		if (!ret) {
-			trace_and_count(c, trans_restart_write_buffer_flush, trans, _THIS_IP_);
-			ret = btree_trans_restart(trans, BCH_ERR_transaction_restart_write_buffer_flush);
-		}
-		goto out;
-	}
-
 	EBUG_ON(test_bit(BCH_FS_CLEAN_SHUTDOWN, &c->flags));
 
 	trans->journal_u64s		= trans->extra_journal_entries.nr;
@@ -1126,14 +1073,15 @@ int __bch2_trans_commit(struct btree_trans *trans, unsigned flags)
 	if (trans->extra_journal_res) {
 		ret = bch2_disk_reservation_add(c, trans->disk_res,
 				trans->extra_journal_res,
-				(flags & BTREE_INSERT_NOFAIL)
+				(flags & BCH_TRANS_COMMIT_no_enospc)
 				? BCH_DISK_RESERVATION_NOFAIL : 0);
 		if (ret)
 			goto err;
 	}
 retry:
 	bch2_trans_verify_not_in_restart(trans);
-	memset(&trans->journal_res, 0, sizeof(trans->journal_res));
+	if (likely(!(flags & BCH_TRANS_COMMIT_no_journal_res)))
+		memset(&trans->journal_res, 0, sizeof(trans->journal_res));
 
 	ret = do_bch2_trans_commit(trans, flags, &i, _RET_IP_);
 
@@ -1145,7 +1093,7 @@ int __bch2_trans_commit(struct btree_trans *trans, unsigned flags)
 
 	trace_and_count(c, transaction_commit, trans, _RET_IP_);
 out:
-	if (likely(!(flags & BTREE_INSERT_NOCHECK_RW)))
+	if (likely(!(flags & BCH_TRANS_COMMIT_no_check_rw)))
 		bch2_write_ref_put(c, BCH_WRITE_REF_trans);
 out_reset:
 	if (!ret)
@@ -1158,5 +1106,17 @@ int __bch2_trans_commit(struct btree_trans *trans, unsigned flags)
 	if (ret)
 		goto out;
 
+	/*
+	 * We might have done another transaction commit in the error path -
+	 * i.e. btree write buffer flush - which will have made use of
+	 * trans->journal_res, but with BCH_TRANS_COMMIT_no_journal_res that is
+	 * how the journal sequence number to pin is passed in - so we must
+	 * restart:
+	 */
+	if (flags & BCH_TRANS_COMMIT_no_journal_res) {
+		ret = -BCH_ERR_transaction_restart_nested;
+		goto out;
+	}
+
 	goto retry;
 }
diff --git a/fs/bcachefs/btree_types.h b/fs/bcachefs/btree_types.h
index 60453ba86c4b..ca7526603d06 100644
--- a/fs/bcachefs/btree_types.h
+++ b/fs/bcachefs/btree_types.h
@@ -185,33 +185,32 @@ struct btree_node_iter {
  * Iterate over all possible positions, synthesizing deleted keys for holes:
  */
 static const __maybe_unused u16 BTREE_ITER_SLOTS		= 1 << 0;
-static const __maybe_unused u16 BTREE_ITER_ALL_LEVELS		= 1 << 1;
 /*
  * Indicates that intent locks should be taken on leaf nodes, because we expect
  * to be doing updates:
  */
-static const __maybe_unused u16 BTREE_ITER_INTENT		= 1 << 2;
+static const __maybe_unused u16 BTREE_ITER_INTENT		= 1 << 1;
 /*
  * Causes the btree iterator code to prefetch additional btree nodes from disk:
  */
-static const __maybe_unused u16 BTREE_ITER_PREFETCH		= 1 << 3;
+static const __maybe_unused u16 BTREE_ITER_PREFETCH		= 1 << 2;
 /*
  * Used in bch2_btree_iter_traverse(), to indicate whether we're searching for
  * @pos or the first key strictly greater than @pos
  */
-static const __maybe_unused u16 BTREE_ITER_IS_EXTENTS		= 1 << 4;
-static const __maybe_unused u16 BTREE_ITER_NOT_EXTENTS		= 1 << 5;
-static const __maybe_unused u16 BTREE_ITER_CACHED		= 1 << 6;
-static const __maybe_unused u16 BTREE_ITER_WITH_KEY_CACHE	= 1 << 7;
-static const __maybe_unused u16 BTREE_ITER_WITH_UPDATES		= 1 << 8;
-static const __maybe_unused u16 BTREE_ITER_WITH_JOURNAL		= 1 << 9;
-static const __maybe_unused u16 __BTREE_ITER_ALL_SNAPSHOTS	= 1 << 10;
-static const __maybe_unused u16 BTREE_ITER_ALL_SNAPSHOTS	= 1 << 11;
-static const __maybe_unused u16 BTREE_ITER_FILTER_SNAPSHOTS	= 1 << 12;
-static const __maybe_unused u16 BTREE_ITER_NOPRESERVE		= 1 << 13;
-static const __maybe_unused u16 BTREE_ITER_CACHED_NOFILL	= 1 << 14;
-static const __maybe_unused u16 BTREE_ITER_KEY_CACHE_FILL	= 1 << 15;
-#define __BTREE_ITER_FLAGS_END					       16
+static const __maybe_unused u16 BTREE_ITER_IS_EXTENTS		= 1 << 3;
+static const __maybe_unused u16 BTREE_ITER_NOT_EXTENTS		= 1 << 4;
+static const __maybe_unused u16 BTREE_ITER_CACHED		= 1 << 5;
+static const __maybe_unused u16 BTREE_ITER_WITH_KEY_CACHE	= 1 << 6;
+static const __maybe_unused u16 BTREE_ITER_WITH_UPDATES		= 1 << 7;
+static const __maybe_unused u16 BTREE_ITER_WITH_JOURNAL		= 1 << 8;
+static const __maybe_unused u16 __BTREE_ITER_ALL_SNAPSHOTS	= 1 << 9;
+static const __maybe_unused u16 BTREE_ITER_ALL_SNAPSHOTS	= 1 << 10;
+static const __maybe_unused u16 BTREE_ITER_FILTER_SNAPSHOTS	= 1 << 11;
+static const __maybe_unused u16 BTREE_ITER_NOPRESERVE		= 1 << 12;
+static const __maybe_unused u16 BTREE_ITER_CACHED_NOFILL	= 1 << 13;
+static const __maybe_unused u16 BTREE_ITER_KEY_CACHE_FILL	= 1 << 14;
+#define __BTREE_ITER_FLAGS_END					       15
 
 enum btree_path_uptodate {
 	BTREE_ITER_UPTODATE		= 0,
@@ -288,8 +287,7 @@ struct btree_iter {
 	struct btree_path	*key_cache_path;
 
 	enum btree_id		btree_id:8;
-	unsigned		min_depth:3;
-	unsigned		advanced:1;
+	u8			min_depth;
 
 	/* btree_iter_copy starts here: */
 	u16			flags;
@@ -306,7 +304,6 @@ struct btree_iter {
 
 	/* BTREE_ITER_WITH_JOURNAL: */
 	size_t			journal_idx;
-	struct bpos		journal_pos;
 #ifdef TRACK_PATH_ALLOCATED
 	unsigned long		ip_allocated;
 #endif
@@ -356,7 +353,6 @@ struct btree_insert_entry {
 	u8			old_btree_u64s;
 	struct bkey_i		*k;
 	struct btree_path	*path;
-	u64			seq;
 	/* key being overwritten: */
 	struct bkey		old_k;
 	const struct bch_val	*old_v;
diff --git a/fs/bcachefs/btree_update.c b/fs/bcachefs/btree_update.c
index 324767c0ddcc..1837f8484569 100644
--- a/fs/bcachefs/btree_update.c
+++ b/fs/bcachefs/btree_update.c
@@ -380,21 +380,12 @@ bch2_trans_update_by_path(struct btree_trans *trans, struct btree_path *path,
 {
 	struct bch_fs *c = trans->c;
 	struct btree_insert_entry *i, n;
-	u64 seq = 0;
 	int cmp;
 
 	EBUG_ON(!path->should_be_locked);
 	EBUG_ON(trans->nr_updates >= BTREE_ITER_MAX);
 	EBUG_ON(!bpos_eq(k->k.p, path->pos));
 
-	/*
-	 * The transaction journal res hasn't been allocated at this point.
-	 * That occurs at commit time. Reuse the seq field to pass in the seq
-	 * of a prejournaled key.
-	 */
-	if (flags & BTREE_UPDATE_PREJOURNAL)
-		seq = trans->journal_res.seq;
-
 	n = (struct btree_insert_entry) {
 		.flags		= flags,
 		.bkey_type	= __btree_node_type(path->level, path->btree_id),
@@ -403,7 +394,6 @@ bch2_trans_update_by_path(struct btree_trans *trans, struct btree_path *path,
 		.cached		= path->cached,
 		.path		= path,
 		.k		= k,
-		.seq		= seq,
 		.ip_allocated	= ip,
 	};
 
@@ -431,7 +421,6 @@ bch2_trans_update_by_path(struct btree_trans *trans, struct btree_path *path,
 		i->cached	= n.cached;
 		i->k		= n.k;
 		i->path		= n.path;
-		i->seq		= n.seq;
 		i->ip_allocated	= n.ip_allocated;
 	} else {
 		array_insert_item(trans->updates, trans->nr_updates,
@@ -542,18 +531,6 @@ int __must_check bch2_trans_update(struct btree_trans *trans, struct btree_iter
 	return bch2_trans_update_by_path(trans, path, k, flags, _RET_IP_);
 }
 
-/*
- * Add a transaction update for a key that has already been journaled.
- */
-int __must_check bch2_trans_update_seq(struct btree_trans *trans, u64 seq,
-				       struct btree_iter *iter, struct bkey_i *k,
-				       enum btree_update_flags flags)
-{
-	trans->journal_res.seq = seq;
-	return bch2_trans_update(trans, iter, k, flags|BTREE_UPDATE_NOJOURNAL|
-						 BTREE_UPDATE_PREJOURNAL);
-}
-
 int __must_check bch2_trans_update_buffered(struct btree_trans *trans,
 					    enum btree_id btree,
 					    struct bkey_i *k)
@@ -792,7 +769,7 @@ int bch2_btree_delete_range_trans(struct btree_trans *trans, enum btree_id id,
 
 		ret   = bch2_trans_update(trans, &iter, &delete, update_flags) ?:
 			bch2_trans_commit(trans, &disk_res, journal_seq,
-					  BTREE_INSERT_NOFAIL);
+					  BCH_TRANS_COMMIT_no_enospc);
 		bch2_disk_reservation_put(trans->c, &disk_res);
 err:
 		/*
@@ -897,7 +874,7 @@ __bch2_fs_log_msg(struct bch_fs *c, unsigned commit_flags, const char *fmt,
 		ret = __bch2_trans_log_msg(&c->journal.early_journal_entries, fmt, args);
 	} else {
 		ret = bch2_trans_do(c, NULL, NULL,
-			BTREE_INSERT_LAZY_RW|commit_flags,
+			BCH_TRANS_COMMIT_lazy_rw|commit_flags,
 			__bch2_trans_log_msg(&trans->extra_journal_entries, fmt, args));
 	}
 
diff --git a/fs/bcachefs/btree_update.h b/fs/bcachefs/btree_update.h
index 9816d2286540..14a2315aa88e 100644
--- a/fs/bcachefs/btree_update.h
+++ b/fs/bcachefs/btree_update.h
@@ -21,37 +21,28 @@ void bch2_btree_add_journal_pin(struct bch_fs *, struct btree *, u64);
 void bch2_btree_insert_key_leaf(struct btree_trans *, struct btree_path *,
 				struct bkey_i *, u64);
 
-enum btree_insert_flags {
+#define BCH_TRANS_COMMIT_FLAGS()							\
+	x(no_enospc,	"don't check for enospc")					\
+	x(no_check_rw,	"don't attempt to take a ref on c->writes")			\
+	x(lazy_rw,	"go read-write if we haven't yet - only for use in recovery")	\
+	x(no_journal_res, "don't take a journal reservation, instead "			\
+			"pin journal entry referred to by trans->journal_res.seq")	\
+	x(journal_reclaim, "operation required for journal reclaim; may return error"	\
+			"instead of deadlocking if BCH_WATERMARK_reclaim not specified")\
+
+enum __bch_trans_commit_flags {
 	/* First bits for bch_watermark: */
-	__BTREE_INSERT_NOFAIL = BCH_WATERMARK_BITS,
-	__BTREE_INSERT_NOCHECK_RW,
-	__BTREE_INSERT_LAZY_RW,
-	__BTREE_INSERT_JOURNAL_REPLAY,
-	__BTREE_INSERT_JOURNAL_RECLAIM,
-	__BTREE_INSERT_NOWAIT,
-	__BTREE_INSERT_GC_LOCK_HELD,
-	__BCH_HASH_SET_MUST_CREATE,
-	__BCH_HASH_SET_MUST_REPLACE,
+	__BCH_TRANS_COMMIT_FLAGS_START = BCH_WATERMARK_BITS,
+#define x(n, ...)	__BCH_TRANS_COMMIT_##n,
+	BCH_TRANS_COMMIT_FLAGS()
+#undef x
 };
 
-/* Don't check for -ENOSPC: */
-#define BTREE_INSERT_NOFAIL		BIT(__BTREE_INSERT_NOFAIL)
-
-#define BTREE_INSERT_NOCHECK_RW		BIT(__BTREE_INSERT_NOCHECK_RW)
-#define BTREE_INSERT_LAZY_RW		BIT(__BTREE_INSERT_LAZY_RW)
-
-/* Insert is for journal replay - don't get journal reservations: */
-#define BTREE_INSERT_JOURNAL_REPLAY	BIT(__BTREE_INSERT_JOURNAL_REPLAY)
-
-/* Insert is being called from journal reclaim path: */
-#define BTREE_INSERT_JOURNAL_RECLAIM	BIT(__BTREE_INSERT_JOURNAL_RECLAIM)
-
-/* Don't block on allocation failure (for new btree nodes: */
-#define BTREE_INSERT_NOWAIT		BIT(__BTREE_INSERT_NOWAIT)
-#define BTREE_INSERT_GC_LOCK_HELD	BIT(__BTREE_INSERT_GC_LOCK_HELD)
-
-#define BCH_HASH_SET_MUST_CREATE	BIT(__BCH_HASH_SET_MUST_CREATE)
-#define BCH_HASH_SET_MUST_REPLACE	BIT(__BCH_HASH_SET_MUST_REPLACE)
+enum bch_trans_commit_flags {
+#define x(n, ...)	BCH_TRANS_COMMIT_##n = BIT(__BCH_TRANS_COMMIT_##n),
+	BCH_TRANS_COMMIT_FLAGS()
+#undef x
+};
 
 int bch2_btree_delete_extent_at(struct btree_trans *, struct btree_iter *,
 				unsigned, unsigned);
diff --git a/fs/bcachefs/btree_update_interior.c b/fs/bcachefs/btree_update_interior.c
index 76f27bc9fa24..bfe4d7975bd8 100644
--- a/fs/bcachefs/btree_update_interior.c
+++ b/fs/bcachefs/btree_update_interior.c
@@ -475,9 +475,6 @@ static int bch2_btree_reserve_get(struct btree_trans *trans,
 	/*
 	 * Protects reaping from the btree node cache and using the btree node
 	 * open bucket reserve:
-	 *
-	 * BTREE_INSERT_NOWAIT only applies to btree node allocation, not
-	 * blocking on this lock:
 	 */
 	ret = bch2_btree_cache_cannibalize_lock(c, cl);
 	if (ret)
@@ -487,9 +484,8 @@ static int bch2_btree_reserve_get(struct btree_trans *trans,
 		struct prealloc_nodes *p = as->prealloc_nodes + interior;
 
 		while (p->nr < nr_nodes[interior]) {
-			b = __bch2_btree_node_alloc(trans, &as->disk_res,
-					flags & BTREE_INSERT_NOWAIT ? NULL : cl,
-					interior, flags);
+			b = __bch2_btree_node_alloc(trans, &as->disk_res, cl,
+						    interior, flags);
 			if (IS_ERR(b)) {
 				ret = PTR_ERR(b);
 				goto err;
@@ -644,9 +640,9 @@ static void btree_update_nodes_written(struct btree_update *as)
 	 */
 	ret = commit_do(trans, &as->disk_res, &journal_seq,
 			BCH_WATERMARK_reclaim|
-			BTREE_INSERT_NOFAIL|
-			BTREE_INSERT_NOCHECK_RW|
-			BTREE_INSERT_JOURNAL_RECLAIM,
+			BCH_TRANS_COMMIT_no_enospc|
+			BCH_TRANS_COMMIT_no_check_rw|
+			BCH_TRANS_COMMIT_journal_reclaim,
 			btree_update_nodes_written_trans(trans, as));
 	bch2_trans_unlock(trans);
 
@@ -778,9 +774,9 @@ static void btree_interior_update_work(struct work_struct *work)
 	}
 }
 
-static void btree_update_set_nodes_written(struct closure *cl)
+static CLOSURE_CALLBACK(btree_update_set_nodes_written)
 {
-	struct btree_update *as = container_of(cl, struct btree_update, cl);
+	closure_type(as, struct btree_update, cl);
 	struct bch_fs *c = as->c;
 
 	mutex_lock(&c->btree_interior_update_lock);
@@ -814,6 +810,12 @@ static void btree_update_updated_node(struct btree_update *as, struct btree *b)
 	mutex_unlock(&c->btree_interior_update_lock);
 }
 
+static int bch2_update_reparent_journal_pin_flush(struct journal *j,
+				struct journal_entry_pin *_pin, u64 seq)
+{
+	return 0;
+}
+
 static void btree_update_reparent(struct btree_update *as,
 				  struct btree_update *child)
 {
@@ -824,7 +826,8 @@ static void btree_update_reparent(struct btree_update *as,
 	child->b = NULL;
 	child->mode = BTREE_INTERIOR_UPDATING_AS;
 
-	bch2_journal_pin_copy(&c->journal, &as->journal, &child->journal, NULL);
+	bch2_journal_pin_copy(&c->journal, &as->journal, &child->journal,
+			      bch2_update_reparent_journal_pin_flush);
 }
 
 static void btree_update_updated_root(struct btree_update *as, struct btree *b)
@@ -933,6 +936,12 @@ static void bch2_btree_update_get_open_buckets(struct btree_update *as, struct b
 			b->ob.v[--b->ob.nr];
 }
 
+static int bch2_btree_update_will_free_node_journal_pin_flush(struct journal *j,
+				struct journal_entry_pin *_pin, u64 seq)
+{
+	return 0;
+}
+
 /*
  * @b is being split/rewritten: it may have pointers to not-yet-written btree
  * nodes and thus outstanding btree_updates - redirect @b's
@@ -984,11 +993,13 @@ static void bch2_btree_interior_update_will_free_node(struct btree_update *as,
 	 * when the new nodes are persistent and reachable on disk:
 	 */
 	w = btree_current_write(b);
-	bch2_journal_pin_copy(&c->journal, &as->journal, &w->journal, NULL);
+	bch2_journal_pin_copy(&c->journal, &as->journal, &w->journal,
+			      bch2_btree_update_will_free_node_journal_pin_flush);
 	bch2_journal_pin_drop(&c->journal, &w->journal);
 
 	w = btree_prev_write(b);
-	bch2_journal_pin_copy(&c->journal, &as->journal, &w->journal, NULL);
+	bch2_journal_pin_copy(&c->journal, &as->journal, &w->journal,
+			      bch2_btree_update_will_free_node_journal_pin_flush);
 	bch2_journal_pin_drop(&c->journal, &w->journal);
 
 	mutex_unlock(&c->btree_interior_update_lock);
@@ -1038,7 +1049,7 @@ bch2_btree_update_start(struct btree_trans *trans, struct btree_path *path,
 	struct bch_fs *c = trans->c;
 	struct btree_update *as;
 	u64 start_time = local_clock();
-	int disk_res_flags = (flags & BTREE_INSERT_NOFAIL)
+	int disk_res_flags = (flags & BCH_TRANS_COMMIT_no_enospc)
 		? BCH_DISK_RESERVATION_NOFAIL : 0;
 	unsigned nr_nodes[2] = { 0, 0 };
 	unsigned update_level = level;
@@ -1078,9 +1089,7 @@ bch2_btree_update_start(struct btree_trans *trans, struct btree_path *path,
 		split = path->l[update_level].b->nr.live_u64s > BTREE_SPLIT_THRESHOLD(c);
 	}
 
-	if (flags & BTREE_INSERT_GC_LOCK_HELD)
-		lockdep_assert_held(&c->gc_lock);
-	else if (!down_read_trylock(&c->gc_lock)) {
+	if (!down_read_trylock(&c->gc_lock)) {
 		ret = drop_locks_do(trans, (down_read(&c->gc_lock), 0));
 		if (ret) {
 			up_read(&c->gc_lock);
@@ -1094,7 +1103,7 @@ bch2_btree_update_start(struct btree_trans *trans, struct btree_path *path,
 	as->c		= c;
 	as->start_time	= start_time;
 	as->mode	= BTREE_INTERIOR_NO_UPDATE;
-	as->took_gc_lock = !(flags & BTREE_INSERT_GC_LOCK_HELD);
+	as->took_gc_lock = true;
 	as->btree_id	= path->btree_id;
 	as->update_level = update_level;
 	INIT_LIST_HEAD(&as->list);
@@ -1137,7 +1146,7 @@ bch2_btree_update_start(struct btree_trans *trans, struct btree_path *path,
 		 * flag
 		 */
 		if (bch2_err_matches(ret, ENOSPC) &&
-		    (flags & BTREE_INSERT_JOURNAL_RECLAIM) &&
+		    (flags & BCH_TRANS_COMMIT_journal_reclaim) &&
 		    watermark != BCH_WATERMARK_reclaim) {
 			ret = -BCH_ERR_journal_reclaim_would_deadlock;
 			goto err;
@@ -1825,7 +1834,7 @@ int __bch2_foreground_maybe_merge(struct btree_trans *trans,
 
 	parent = btree_node_parent(path, b);
 	as = bch2_btree_update_start(trans, path, level, false,
-				     BTREE_INSERT_NOFAIL|flags);
+				     BCH_TRANS_COMMIT_no_enospc|flags);
 	ret = PTR_ERR_OR_ZERO(as);
 	if (ret)
 		goto err;
@@ -1911,7 +1920,7 @@ int bch2_btree_node_rewrite(struct btree_trans *trans,
 	struct btree_update *as;
 	int ret;
 
-	flags |= BTREE_INSERT_NOFAIL;
+	flags |= BCH_TRANS_COMMIT_no_enospc;
 
 	parent = btree_node_parent(iter->path, b);
 	as = bch2_btree_update_start(trans, iter->path, b->c.level,
diff --git a/fs/bcachefs/btree_write_buffer.c b/fs/bcachefs/btree_write_buffer.c
index 4e6241db518b..0c2db1fab38a 100644
--- a/fs/bcachefs/btree_write_buffer.c
+++ b/fs/bcachefs/btree_write_buffer.c
@@ -7,44 +7,143 @@
 #include "btree_write_buffer.h"
 #include "error.h"
 #include "journal.h"
+#include "journal_io.h"
 #include "journal_reclaim.h"
 
-#include <linux/sort.h>
+#include <linux/prefetch.h>
 
-static int btree_write_buffered_key_cmp(const void *_l, const void *_r)
+static int bch2_btree_write_buffer_journal_flush(struct journal *,
+				struct journal_entry_pin *, u64);
+
+static int bch2_journal_keys_to_write_buffer(struct bch_fs *, struct journal_buf *);
+
+static inline bool __wb_key_cmp(const struct wb_key_ref *l, const struct wb_key_ref *r)
+{
+	return (cmp_int(l->hi, r->hi) ?:
+		cmp_int(l->mi, r->mi) ?:
+		cmp_int(l->lo, r->lo)) >= 0;
+}
+
+static inline bool wb_key_cmp(const struct wb_key_ref *l, const struct wb_key_ref *r)
+{
+#ifdef CONFIG_X86_64
+	int cmp;
+
+	asm(".intel_syntax noprefix;"
+	    "mov rax, [%[l]];"
+	    "sub rax, [%[r]];"
+	    "mov rax, [%[l] + 8];"
+	    "sbb rax, [%[r] + 8];"
+	    "mov rax, [%[l] + 16];"
+	    "sbb rax, [%[r] + 16];"
+	    ".att_syntax prefix;"
+	    : "=@ccae" (cmp)
+	    : [l] "r" (l), [r] "r" (r)
+	    : "rax", "cc");
+
+	EBUG_ON(cmp != __wb_key_cmp(l, r));
+	return cmp;
+#else
+	return __wb_key_cmp(l, r);
+#endif
+}
+
+/* Compare excluding idx, the low 24 bits: */
+static inline bool wb_key_eq(const void *_l, const void *_r)
+{
+	const struct wb_key_ref *l = _l;
+	const struct wb_key_ref *r = _r;
+
+	return !((l->hi ^ r->hi)|
+		 (l->mi ^ r->mi)|
+		 ((l->lo >> 24) ^ (r->lo >> 24)));
+}
+
+static noinline void wb_sort(struct wb_key_ref *base, size_t num)
 {
-	const struct btree_write_buffered_key *l = _l;
-	const struct btree_write_buffered_key *r = _r;
+	size_t n = num, a = num / 2;
+
+	if (!a)		/* num < 2 || size == 0 */
+		return;
+
+	for (;;) {
+		size_t b, c, d;
 
-	return  cmp_int(l->btree, r->btree) ?:
-		bpos_cmp(l->k.k.p, r->k.k.p) ?:
-		cmp_int(l->journal_seq, r->journal_seq) ?:
-		cmp_int(l->journal_offset, r->journal_offset);
+		if (a)			/* Building heap: sift down --a */
+			--a;
+		else if (--n)		/* Sorting: Extract root to --n */
+			swap(base[0], base[n]);
+		else			/* Sort complete */
+			break;
+
+		/*
+		 * Sift element at "a" down into heap.  This is the
+		 * "bottom-up" variant, which significantly reduces
+		 * calls to cmp_func(): we find the sift-down path all
+		 * the way to the leaves (one compare per level), then
+		 * backtrack to find where to insert the target element.
+		 *
+		 * Because elements tend to sift down close to the leaves,
+		 * this uses fewer compares than doing two per level
+		 * on the way down.  (A bit more than half as many on
+		 * average, 3/4 worst-case.)
+		 */
+		for (b = a; c = 2*b + 1, (d = c + 1) < n;)
+			b = wb_key_cmp(base + c, base + d) ? c : d;
+		if (d == n)		/* Special case last leaf with no sibling */
+			b = c;
+
+		/* Now backtrack from "b" to the correct location for "a" */
+		while (b != a && wb_key_cmp(base + a, base + b))
+			b = (b - 1) / 2;
+		c = b;			/* Where "a" belongs */
+		while (b != a) {	/* Shift it into place */
+			b = (b - 1) / 2;
+			swap(base[b], base[c]);
+		}
+	}
 }
 
-static int btree_write_buffered_journal_cmp(const void *_l, const void *_r)
+static noinline int wb_flush_one_slowpath(struct btree_trans *trans,
+					  struct btree_iter *iter,
+					  struct btree_write_buffered_key *wb)
 {
-	const struct btree_write_buffered_key *l = _l;
-	const struct btree_write_buffered_key *r = _r;
+	bch2_btree_node_unlock_write(trans, iter->path, iter->path->l[0].b);
 
-	return  cmp_int(l->journal_seq, r->journal_seq);
+	trans->journal_res.seq = wb->journal_seq;
+
+	return bch2_trans_update(trans, iter, &wb->k,
+				 BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE) ?:
+		bch2_trans_commit(trans, NULL, NULL,
+				  BCH_TRANS_COMMIT_no_enospc|
+				  BCH_TRANS_COMMIT_no_check_rw|
+				  BCH_TRANS_COMMIT_no_journal_res|
+				  BCH_TRANS_COMMIT_journal_reclaim);
 }
 
-static int bch2_btree_write_buffer_flush_one(struct btree_trans *trans,
-					     struct btree_iter *iter,
-					     struct btree_write_buffered_key *wb,
-					     unsigned commit_flags,
-					     bool *write_locked,
-					     size_t *fast)
+static inline int wb_flush_one(struct btree_trans *trans, struct btree_iter *iter,
+			       struct btree_write_buffered_key *wb,
+			       bool *write_locked, size_t *fast)
 {
 	struct bch_fs *c = trans->c;
 	struct btree_path *path;
 	int ret;
 
+	EBUG_ON(!wb->journal_seq);
+	EBUG_ON(!c->btree_write_buffer.flushing.pin.seq);
+	EBUG_ON(c->btree_write_buffer.flushing.pin.seq > wb->journal_seq);
+
 	ret = bch2_btree_iter_traverse(iter);
 	if (ret)
 		return ret;
 
+	/*
+	 * We can't clone a path that has write locks: unshare it now, before
+	 * set_pos and traverse():
+	 */
+	if (iter->path->ref > 1)
+		iter->path = __bch2_btree_path_make_mut(trans, iter->path, true, _THIS_IP_);
+
 	path = iter->path;
 
 	if (!*write_locked) {
@@ -56,52 +155,14 @@ static int bch2_btree_write_buffer_flush_one(struct btree_trans *trans,
 		*write_locked = true;
 	}
 
-	if (!bch2_btree_node_insert_fits(c, path->l[0].b, wb->k.k.u64s)) {
-		bch2_btree_node_unlock_write(trans, path, path->l[0].b);
+	if (unlikely(!bch2_btree_node_insert_fits(c, path->l[0].b, wb->k.k.u64s))) {
 		*write_locked = false;
-		goto trans_commit;
+		return wb_flush_one_slowpath(trans, iter, wb);
 	}
 
 	bch2_btree_insert_key_leaf(trans, path, &wb->k, wb->journal_seq);
 	(*fast)++;
-
-	if (path->ref > 1) {
-		/*
-		 * We can't clone a path that has write locks: if the path is
-		 * shared, unlock before set_pos(), traverse():
-		 */
-		bch2_btree_node_unlock_write(trans, path, path->l[0].b);
-		*write_locked = false;
-	}
 	return 0;
-trans_commit:
-	return  bch2_trans_update_seq(trans, wb->journal_seq, iter, &wb->k,
-				      BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE) ?:
-		bch2_trans_commit(trans, NULL, NULL,
-				  commit_flags|
-				  BTREE_INSERT_NOCHECK_RW|
-				  BTREE_INSERT_NOFAIL|
-				  BTREE_INSERT_JOURNAL_RECLAIM);
-}
-
-static union btree_write_buffer_state btree_write_buffer_switch(struct btree_write_buffer *wb)
-{
-	union btree_write_buffer_state old, new;
-	u64 v = READ_ONCE(wb->state.v);
-
-	do {
-		old.v = new.v = v;
-
-		new.nr = 0;
-		new.idx++;
-	} while ((v = atomic64_cmpxchg_acquire(&wb->state.counter, old.v, new.v)) != old.v);
-
-	while (old.idx == 0 ? wb->state.ref0 : wb->state.ref1)
-		cpu_relax();
-
-	smp_mb();
-
-	return old;
 }
 
 /*
@@ -124,41 +185,88 @@ btree_write_buffered_insert(struct btree_trans *trans,
 	bch2_trans_iter_init(trans, &iter, wb->btree, bkey_start_pos(&wb->k.k),
 			     BTREE_ITER_CACHED|BTREE_ITER_INTENT);
 
+	trans->journal_res.seq = wb->journal_seq;
+
 	ret   = bch2_btree_iter_traverse(&iter) ?:
-		bch2_trans_update_seq(trans, wb->journal_seq, &iter, &wb->k,
-				      BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE);
+		bch2_trans_update(trans, &iter, &wb->k,
+				  BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE);
 	bch2_trans_iter_exit(trans, &iter);
 	return ret;
 }
 
-int __bch2_btree_write_buffer_flush(struct btree_trans *trans, unsigned commit_flags,
-				    bool locked)
+static void move_keys_from_inc_to_flushing(struct btree_write_buffer *wb)
+{
+	struct bch_fs *c = container_of(wb, struct bch_fs, btree_write_buffer);
+	struct journal *j = &c->journal;
+
+	if (!wb->inc.keys.nr)
+		return;
+
+	bch2_journal_pin_add(j, wb->inc.keys.data[0].journal_seq, &wb->flushing.pin,
+			     bch2_btree_write_buffer_journal_flush);
+
+	darray_resize(&wb->flushing.keys, min_t(size_t, 1U << 20, wb->flushing.keys.nr + wb->inc.keys.nr));
+	darray_resize(&wb->sorted, wb->flushing.keys.size);
+
+	if (!wb->flushing.keys.nr && wb->sorted.size >= wb->inc.keys.nr) {
+		swap(wb->flushing.keys, wb->inc.keys);
+		goto out;
+	}
+
+	size_t nr = min(darray_room(wb->flushing.keys),
+			wb->sorted.size - wb->flushing.keys.nr);
+	nr = min(nr, wb->inc.keys.nr);
+
+	memcpy(&darray_top(wb->flushing.keys),
+	       wb->inc.keys.data,
+	       sizeof(wb->inc.keys.data[0]) * nr);
+
+	memmove(wb->inc.keys.data,
+		wb->inc.keys.data + nr,
+	       sizeof(wb->inc.keys.data[0]) * (wb->inc.keys.nr - nr));
+
+	wb->flushing.keys.nr	+= nr;
+	wb->inc.keys.nr		-= nr;
+out:
+	if (!wb->inc.keys.nr)
+		bch2_journal_pin_drop(j, &wb->inc.pin);
+	else
+		bch2_journal_pin_update(j, wb->inc.keys.data[0].journal_seq, &wb->inc.pin,
+					bch2_btree_write_buffer_journal_flush);
+
+	if (j->watermark) {
+		spin_lock(&j->lock);
+		bch2_journal_set_watermark(j);
+		spin_unlock(&j->lock);
+	}
+
+	BUG_ON(wb->sorted.size < wb->flushing.keys.nr);
+}
+
+static int bch2_btree_write_buffer_flush_locked(struct btree_trans *trans)
 {
 	struct bch_fs *c = trans->c;
 	struct journal *j = &c->journal;
 	struct btree_write_buffer *wb = &c->btree_write_buffer;
-	struct journal_entry_pin pin;
-	struct btree_write_buffered_key *i, *keys;
+	struct wb_key_ref *i;
 	struct btree_iter iter = { NULL };
-	size_t nr = 0, skipped = 0, fast = 0, slowpath = 0;
+	size_t skipped = 0, fast = 0, slowpath = 0;
 	bool write_locked = false;
-	union btree_write_buffer_state s;
 	int ret = 0;
 
-	memset(&pin, 0, sizeof(pin));
-
-	if (!locked && !mutex_trylock(&wb->flush_lock))
-		return 0;
-
-	bch2_journal_pin_copy(j, &pin, &wb->journal_pin, NULL);
-	bch2_journal_pin_drop(j, &wb->journal_pin);
+	bch2_trans_unlock(trans);
+	bch2_trans_begin(trans);
 
-	s = btree_write_buffer_switch(wb);
-	keys = wb->keys[s.idx];
-	nr = s.nr;
+	mutex_lock(&wb->inc.lock);
+	move_keys_from_inc_to_flushing(wb);
+	mutex_unlock(&wb->inc.lock);
 
-	if (race_fault())
-		goto slowpath;
+	for (size_t i = 0; i < wb->flushing.keys.nr; i++) {
+		wb->sorted.data[i].idx = i;
+		wb->sorted.data[i].btree = wb->flushing.keys.data[i].btree;
+		memcpy(&wb->sorted.data[i].pos, &wb->flushing.keys.data[i].k.k.p, sizeof(struct bpos));
+	}
+	wb->sorted.nr = wb->flushing.keys.nr;
 
 	/*
 	 * We first sort so that we can detect and skip redundant updates, and
@@ -168,123 +276,184 @@ int __bch2_btree_write_buffer_flush(struct btree_trans *trans, unsigned commit_f
 	 * However, since we're not flushing in the order they appear in the
 	 * journal we won't be able to drop our journal pin until everything is
 	 * flushed - which means this could deadlock the journal if we weren't
-	 * passing BTREE_INSERT_JOURNAL_RECLAIM. This causes the update to fail
+	 * passing BCH_TRANS_COMMIT_journal_reclaim. This causes the update to fail
 	 * if it would block taking a journal reservation.
 	 *
 	 * If that happens, simply skip the key so we can optimistically insert
 	 * as many keys as possible in the fast path.
 	 */
-	sort(keys, nr, sizeof(keys[0]),
-	     btree_write_buffered_key_cmp, NULL);
+	wb_sort(wb->sorted.data, wb->sorted.nr);
+
+	darray_for_each(wb->sorted, i) {
+		struct btree_write_buffered_key *k = &wb->flushing.keys.data[i->idx];
+
+		for (struct wb_key_ref *n = i + 1; n < min(i + 4, &darray_top(wb->sorted)); n++)
+			prefetch(&wb->flushing.keys.data[n->idx]);
+
+		BUG_ON(!k->journal_seq);
+
+		if (i + 1 < &darray_top(wb->sorted) &&
+		    wb_key_eq(i, i + 1)) {
+			struct btree_write_buffered_key *n = &wb->flushing.keys.data[i[1].idx];
 
-	for (i = keys; i < keys + nr; i++) {
-		if (i + 1 < keys + nr &&
-		    i[0].btree == i[1].btree &&
-		    bpos_eq(i[0].k.k.p, i[1].k.k.p)) {
 			skipped++;
-			i->journal_seq = 0;
+			n->journal_seq = min_t(u64, n->journal_seq, k->journal_seq);;
+			k->journal_seq = 0;
 			continue;
 		}
 
 		if (write_locked &&
-		    (iter.path->btree_id != i->btree ||
-		     bpos_gt(i->k.k.p, iter.path->l[0].b->key.k.p))) {
+		    (iter.path->btree_id != k->btree ||
+		     bpos_gt(k->k.k.p, iter.path->l[0].b->key.k.p))) {
 			bch2_btree_node_unlock_write(trans, iter.path, iter.path->l[0].b);
 			write_locked = false;
 		}
 
-		if (!iter.path || iter.path->btree_id != i->btree) {
+		if (!iter.path || iter.path->btree_id != k->btree) {
 			bch2_trans_iter_exit(trans, &iter);
-			bch2_trans_iter_init(trans, &iter, i->btree, i->k.k.p,
+			bch2_trans_iter_init(trans, &iter, k->btree, k->k.k.p,
 					     BTREE_ITER_INTENT|BTREE_ITER_ALL_SNAPSHOTS);
 		}
 
-		bch2_btree_iter_set_pos(&iter, i->k.k.p);
+		bch2_btree_iter_set_pos(&iter, k->k.k.p);
 		iter.path->preserve = false;
 
 		do {
-			ret = bch2_btree_write_buffer_flush_one(trans, &iter, i,
-						commit_flags, &write_locked, &fast);
+			if (race_fault()) {
+				ret = -BCH_ERR_journal_reclaim_would_deadlock;
+				break;
+			}
+
+			ret = wb_flush_one(trans, &iter, k, &write_locked, &fast);
 			if (!write_locked)
 				bch2_trans_begin(trans);
 		} while (bch2_err_matches(ret, BCH_ERR_transaction_restart));
 
-		if (ret == -BCH_ERR_journal_reclaim_would_deadlock) {
+		if (!ret) {
+			k->journal_seq = 0;
+		} else if (ret == -BCH_ERR_journal_reclaim_would_deadlock) {
 			slowpath++;
-			continue;
-		}
-		if (ret)
+			ret = 0;
+		} else
 			break;
-
-		i->journal_seq = 0;
 	}
 
 	if (write_locked)
 		bch2_btree_node_unlock_write(trans, iter.path, iter.path->l[0].b);
 	bch2_trans_iter_exit(trans, &iter);
 
-	trace_write_buffer_flush(trans, nr, skipped, fast, wb->size);
-
-	if (slowpath)
-		goto slowpath;
+	if (ret)
+		goto err;
 
+	if (slowpath) {
+		/*
+		 * Flush in the order they were present in the journal, so that
+		 * we can release journal pins:
+		 * The fastpath zapped the seq of keys that were successfully flushed so
+		 * we can skip those here.
+		 */
+		trace_write_buffer_flush_slowpath(trans, slowpath, wb->flushing.keys.nr);
+
+		struct btree_write_buffered_key *i;
+		darray_for_each(wb->flushing.keys, i) {
+			if (!i->journal_seq)
+				continue;
+
+			bch2_journal_pin_update(j, i->journal_seq, &wb->flushing.pin,
+						bch2_btree_write_buffer_journal_flush);
+
+			bch2_trans_begin(trans);
+
+			ret = commit_do(trans, NULL, NULL,
+					BCH_WATERMARK_reclaim|
+					BCH_TRANS_COMMIT_no_check_rw|
+					BCH_TRANS_COMMIT_no_enospc|
+					BCH_TRANS_COMMIT_no_journal_res|
+					BCH_TRANS_COMMIT_journal_reclaim,
+					btree_write_buffered_insert(trans, i));
+			if (ret)
+				goto err;
+		}
+	}
+err:
 	bch2_fs_fatal_err_on(ret, c, "%s: insert error %s", __func__, bch2_err_str(ret));
-out:
-	bch2_journal_pin_drop(j, &pin);
-	mutex_unlock(&wb->flush_lock);
+	trace_write_buffer_flush(trans, wb->flushing.keys.nr, skipped, fast, 0);
+	bch2_journal_pin_drop(j, &wb->flushing.pin);
+	wb->flushing.keys.nr = 0;
 	return ret;
-slowpath:
-	trace_write_buffer_flush_slowpath(trans, i - keys, nr);
+}
 
-	/*
-	 * Now sort the rest by journal seq and bump the journal pin as we go.
-	 * The slowpath zapped the seq of keys that were successfully flushed so
-	 * we can skip those here.
-	 */
-	sort(keys, nr, sizeof(keys[0]),
-	     btree_write_buffered_journal_cmp,
-	     NULL);
+static int fetch_wb_keys_from_journal(struct bch_fs *c, u64 seq)
+{
+	struct journal *j = &c->journal;
+	struct journal_buf *buf;
+	int ret = 0;
 
-	commit_flags &= ~BCH_WATERMARK_MASK;
-	commit_flags |= BCH_WATERMARK_reclaim;
+	mutex_lock(&j->buf_lock);
+	while ((buf = bch2_next_write_buffer_flush_journal_buf(j, seq)))
+		if (bch2_journal_keys_to_write_buffer(c, buf)) {
+			ret = -ENOMEM;
+			break;
+		}
+	mutex_unlock(&j->buf_lock);
 
-	for (i = keys; i < keys + nr; i++) {
-		if (!i->journal_seq)
-			continue;
+	return ret;
+}
+
+int bch2_btree_write_buffer_flush_sync(struct btree_trans *trans)
+{
+	struct bch_fs *c = trans->c;
+	struct btree_write_buffer *wb = &c->btree_write_buffer;
+	int ret = 0, fetch_from_journal_err;
 
-		if (i->journal_seq > pin.seq) {
-			struct journal_entry_pin pin2;
+	trace_write_buffer_flush_sync(trans, _RET_IP_);
+retry:
+	bch2_trans_unlock(trans);
 
-			memset(&pin2, 0, sizeof(pin2));
+	bch2_journal_block_reservations(&c->journal);
+	fetch_from_journal_err = fetch_wb_keys_from_journal(c, U64_MAX);
+	bch2_journal_unblock(&c->journal);
 
-			bch2_journal_pin_add(j, i->journal_seq, &pin2, NULL);
-			bch2_journal_pin_drop(j, &pin);
-			bch2_journal_pin_copy(j, &pin, &pin2, NULL);
-			bch2_journal_pin_drop(j, &pin2);
-		}
+	/*
+	 * On memory allocation failure, bch2_btree_write_buffer_flush_locked()
+	 * is not guaranteed to empty wb->inc:
+	 */
+	mutex_lock(&wb->flushing.lock);
+	while (!ret &&
+	       (wb->flushing.keys.nr || wb->inc.keys.nr))
+		ret = bch2_btree_write_buffer_flush_locked(trans);
+	mutex_unlock(&wb->flushing.lock);
 
-		ret = commit_do(trans, NULL, NULL,
-				commit_flags|
-				BTREE_INSERT_NOFAIL|
-				BTREE_INSERT_JOURNAL_RECLAIM,
-				btree_write_buffered_insert(trans, i));
-		if (bch2_fs_fatal_err_on(ret, c, "%s: insert error %s", __func__, bch2_err_str(ret)))
-			break;
-	}
+	if (!ret && fetch_from_journal_err)
+		goto retry;
 
-	goto out;
+	return ret;
 }
 
-int bch2_btree_write_buffer_flush_sync(struct btree_trans *trans)
+int bch2_btree_write_buffer_flush_nocheck_rw(struct btree_trans *trans)
 {
-	bch2_trans_unlock(trans);
-	mutex_lock(&trans->c->btree_write_buffer.flush_lock);
-	return __bch2_btree_write_buffer_flush(trans, 0, true);
+	struct bch_fs *c = trans->c;
+	struct btree_write_buffer *wb = &c->btree_write_buffer;
+	int ret = 0;
+
+	if (mutex_trylock(&wb->flushing.lock)) {
+		ret = bch2_btree_write_buffer_flush_locked(trans);
+		mutex_unlock(&wb->flushing.lock);
+	}
+
+	return ret;
 }
 
-int bch2_btree_write_buffer_flush(struct btree_trans *trans)
+int bch2_btree_write_buffer_tryflush(struct btree_trans *trans)
 {
-	return __bch2_btree_write_buffer_flush(trans, 0, false);
+	struct bch_fs *c = trans->c;
+
+	if (!bch2_write_ref_tryget(c, BCH_WRITE_REF_btree_write_buffer))
+		return -BCH_ERR_erofs_no_writes;
+
+	int ret = bch2_btree_write_buffer_flush_nocheck_rw(trans);
+	bch2_write_ref_put(c, BCH_WRITE_REF_btree_write_buffer);
+	return ret;
 }
 
 static int bch2_btree_write_buffer_journal_flush(struct journal *j,
@@ -292,84 +461,195 @@ static int bch2_btree_write_buffer_journal_flush(struct journal *j,
 {
 	struct bch_fs *c = container_of(j, struct bch_fs, journal);
 	struct btree_write_buffer *wb = &c->btree_write_buffer;
+	int ret, fetch_from_journal_err;
+
+	do {
+		fetch_from_journal_err = fetch_wb_keys_from_journal(c, seq);
 
-	mutex_lock(&wb->flush_lock);
+		mutex_lock(&wb->flushing.lock);
+		ret = bch2_trans_run(c, bch2_btree_write_buffer_flush_locked(trans));
+		mutex_unlock(&wb->flushing.lock);
+	} while (!ret &&
+		 (fetch_from_journal_err ||
+		  (wb->flushing.pin.seq && wb->flushing.pin.seq <= seq) ||
+		  (wb->inc.pin.seq && wb->inc.pin.seq <= seq)));
 
-	return bch2_trans_run(c,
-			__bch2_btree_write_buffer_flush(trans, BTREE_INSERT_NOCHECK_RW, true));
+	return ret;
 }
 
-static inline u64 btree_write_buffer_ref(int idx)
+static void bch2_btree_write_buffer_flush_work(struct work_struct *work)
 {
-	return ((union btree_write_buffer_state) {
-		.ref0 = idx == 0,
-		.ref1 = idx == 1,
-	}).v;
+	struct bch_fs *c = container_of(work, struct bch_fs, btree_write_buffer.flush_work);
+	struct btree_write_buffer *wb = &c->btree_write_buffer;
+	int ret;
+
+	mutex_lock(&wb->flushing.lock);
+	do {
+		ret = bch2_trans_run(c, bch2_btree_write_buffer_flush_locked(trans));
+	} while (!ret && bch2_btree_write_buffer_should_flush(c));
+	mutex_unlock(&wb->flushing.lock);
+
+	bch2_write_ref_put(c, BCH_WRITE_REF_btree_write_buffer);
 }
 
-int bch2_btree_insert_keys_write_buffer(struct btree_trans *trans)
+int __bch2_journal_key_to_wb(struct bch_fs *c,
+			     struct journal_keys_to_wb *dst,
+			     enum btree_id btree, struct bkey_i *k)
 {
-	struct bch_fs *c = trans->c;
 	struct btree_write_buffer *wb = &c->btree_write_buffer;
-	struct btree_write_buffered_key *i;
-	union btree_write_buffer_state old, new;
-	int ret = 0;
-	u64 v;
-
-	trans_for_each_wb_update(trans, i) {
-		EBUG_ON(i->k.k.u64s > BTREE_WRITE_BUFERED_U64s_MAX);
+	int ret;
+retry:
+	ret = darray_make_room_gfp(&dst->wb->keys, 1, GFP_KERNEL);
+	if (!ret && dst->wb == &wb->flushing)
+		ret = darray_resize(&wb->sorted, wb->flushing.keys.size);
+
+	if (unlikely(ret)) {
+		if (dst->wb == &c->btree_write_buffer.flushing) {
+			mutex_unlock(&dst->wb->lock);
+			dst->wb = &c->btree_write_buffer.inc;
+			bch2_journal_pin_add(&c->journal, dst->seq, &dst->wb->pin,
+					     bch2_btree_write_buffer_journal_flush);
+			goto retry;
+		}
 
-		i->journal_seq		= trans->journal_res.seq;
-		i->journal_offset	= trans->journal_res.offset;
+		return ret;
 	}
 
-	preempt_disable();
-	v = READ_ONCE(wb->state.v);
-	do {
-		old.v = new.v = v;
+	dst->room = darray_room(dst->wb->keys);
+	if (dst->wb == &wb->flushing)
+		dst->room = min(dst->room, wb->sorted.size - wb->flushing.keys.nr);
+	BUG_ON(!dst->room);
+	BUG_ON(!dst->seq);
+
+	struct btree_write_buffered_key *wb_k = &darray_top(dst->wb->keys);
+	wb_k->journal_seq	= dst->seq;
+	wb_k->btree		= btree;
+	bkey_copy(&wb_k->k, k);
+	dst->wb->keys.nr++;
+	dst->room--;
+	return 0;
+}
 
-		new.v += btree_write_buffer_ref(new.idx);
-		new.nr += trans->nr_wb_updates;
-		if (new.nr > wb->size) {
-			ret = -BCH_ERR_btree_insert_need_flush_buffer;
-			goto out;
+void bch2_journal_keys_to_write_buffer_start(struct bch_fs *c, struct journal_keys_to_wb *dst, u64 seq)
+{
+	struct btree_write_buffer *wb = &c->btree_write_buffer;
+
+	if (mutex_trylock(&wb->flushing.lock)) {
+		mutex_lock(&wb->inc.lock);
+		move_keys_from_inc_to_flushing(wb);
+
+		/*
+		 * Attempt to skip wb->inc, and add keys directly to
+		 * wb->flushing, saving us a copy later:
+		 */
+
+		if (!wb->inc.keys.nr) {
+			dst->wb = &wb->flushing;
+		} else {
+			mutex_unlock(&wb->flushing.lock);
+			dst->wb = &wb->inc;
 		}
-	} while ((v = atomic64_cmpxchg_acquire(&wb->state.counter, old.v, new.v)) != old.v);
+	} else {
+		mutex_lock(&wb->inc.lock);
+		dst->wb = &wb->inc;
+	}
 
-	memcpy(wb->keys[new.idx] + old.nr,
-	       trans->wb_updates,
-	       sizeof(trans->wb_updates[0]) * trans->nr_wb_updates);
+	dst->room = darray_room(dst->wb->keys);
+	if (dst->wb == &wb->flushing)
+		dst->room = min(dst->room, wb->sorted.size - wb->flushing.keys.nr);
+	dst->seq = seq;
 
-	bch2_journal_pin_add(&c->journal, trans->journal_res.seq, &wb->journal_pin,
+	bch2_journal_pin_add(&c->journal, seq, &dst->wb->pin,
 			     bch2_btree_write_buffer_journal_flush);
+}
+
+void bch2_journal_keys_to_write_buffer_end(struct bch_fs *c, struct journal_keys_to_wb *dst)
+{
+	struct btree_write_buffer *wb = &c->btree_write_buffer;
+
+	if (!dst->wb->keys.nr)
+		bch2_journal_pin_drop(&c->journal, &dst->wb->pin);
+
+	if (bch2_btree_write_buffer_should_flush(c) &&
+	    __bch2_write_ref_tryget(c, BCH_WRITE_REF_btree_write_buffer) &&
+	    !queue_work(system_unbound_wq, &c->btree_write_buffer.flush_work))
+		bch2_write_ref_put(c, BCH_WRITE_REF_btree_write_buffer);
+
+	if (dst->wb == &wb->flushing)
+		mutex_unlock(&wb->flushing.lock);
+	mutex_unlock(&wb->inc.lock);
+}
 
-	atomic64_sub_return_release(btree_write_buffer_ref(new.idx), &wb->state.counter);
+static int bch2_journal_keys_to_write_buffer(struct bch_fs *c, struct journal_buf *buf)
+{
+	struct journal_keys_to_wb dst;
+	struct jset_entry *entry;
+	struct bkey_i *k;
+	int ret = 0;
+
+	bch2_journal_keys_to_write_buffer_start(c, &dst, le64_to_cpu(buf->data->seq));
+
+	for_each_jset_entry_type(entry, buf->data, BCH_JSET_ENTRY_write_buffer_keys) {
+		jset_entry_for_each_key(entry, k) {
+			ret = bch2_journal_key_to_wb(c, &dst, entry->btree_id, k);
+			if (ret)
+				goto out;
+		}
+
+		entry->type = BCH_JSET_ENTRY_btree_keys;
+	}
+
+	buf->need_flush_to_write_buffer = false;
 out:
-	preempt_enable();
+	bch2_journal_keys_to_write_buffer_end(c, &dst);
 	return ret;
 }
 
+static int wb_keys_resize(struct btree_write_buffer_keys *wb, size_t new_size)
+{
+	if (wb->keys.size >= new_size)
+		return 0;
+
+	if (!mutex_trylock(&wb->lock))
+		return -EINTR;
+
+	int ret = darray_resize(&wb->keys, new_size);
+	mutex_unlock(&wb->lock);
+	return ret;
+}
+
+int bch2_btree_write_buffer_resize(struct bch_fs *c, size_t new_size)
+{
+	struct btree_write_buffer *wb = &c->btree_write_buffer;
+
+	return wb_keys_resize(&wb->flushing, new_size) ?:
+		wb_keys_resize(&wb->inc, new_size);
+}
+
 void bch2_fs_btree_write_buffer_exit(struct bch_fs *c)
 {
 	struct btree_write_buffer *wb = &c->btree_write_buffer;
 
-	BUG_ON(wb->state.nr && !bch2_journal_error(&c->journal));
+	BUG_ON((wb->inc.keys.nr || wb->flushing.keys.nr) &&
+	       !bch2_journal_error(&c->journal));
 
-	kvfree(wb->keys[1]);
-	kvfree(wb->keys[0]);
+	darray_exit(&wb->sorted);
+	darray_exit(&wb->flushing.keys);
+	darray_exit(&wb->inc.keys);
 }
 
 int bch2_fs_btree_write_buffer_init(struct bch_fs *c)
 {
 	struct btree_write_buffer *wb = &c->btree_write_buffer;
 
-	mutex_init(&wb->flush_lock);
-	wb->size = c->opts.btree_write_buffer_size;
+	mutex_init(&wb->inc.lock);
+	mutex_init(&wb->flushing.lock);
+	INIT_WORK(&wb->flush_work, bch2_btree_write_buffer_flush_work);
 
-	wb->keys[0] = kvmalloc_array(wb->size, sizeof(*wb->keys[0]), GFP_KERNEL);
-	wb->keys[1] = kvmalloc_array(wb->size, sizeof(*wb->keys[1]), GFP_KERNEL);
-	if (!wb->keys[0] || !wb->keys[1])
-		return -BCH_ERR_ENOMEM_fs_btree_write_buffer_init;
+	/* Will be resized by journal as needed: */
+	unsigned initial_size = 1 << 16;
 
-	return 0;
+	return  darray_make_room(&wb->inc.keys, initial_size) ?:
+		darray_make_room(&wb->flushing.keys, initial_size) ?:
+		darray_make_room(&wb->sorted, initial_size);
 }
diff --git a/fs/bcachefs/btree_write_buffer.h b/fs/bcachefs/btree_write_buffer.h
index 322df1c8304e..1f645f529ed2 100644
--- a/fs/bcachefs/btree_write_buffer.h
+++ b/fs/bcachefs/btree_write_buffer.h
@@ -2,12 +2,59 @@
 #ifndef _BCACHEFS_BTREE_WRITE_BUFFER_H
 #define _BCACHEFS_BTREE_WRITE_BUFFER_H
 
-int __bch2_btree_write_buffer_flush(struct btree_trans *, unsigned, bool);
+#include "bkey.h"
+
+static inline bool bch2_btree_write_buffer_should_flush(struct bch_fs *c)
+{
+	struct btree_write_buffer *wb = &c->btree_write_buffer;
+
+	return wb->inc.keys.nr + wb->flushing.keys.nr > wb->inc.keys.size / 4;
+}
+
+static inline bool bch2_btree_write_buffer_must_wait(struct bch_fs *c)
+{
+	struct btree_write_buffer *wb = &c->btree_write_buffer;
+
+	return wb->inc.keys.nr > wb->inc.keys.size * 3 / 4;
+}
+
+struct btree_trans;
 int bch2_btree_write_buffer_flush_sync(struct btree_trans *);
-int bch2_btree_write_buffer_flush(struct btree_trans *);
+int bch2_btree_write_buffer_flush_nocheck_rw(struct btree_trans *);
+int bch2_btree_write_buffer_tryflush(struct btree_trans *);
+
+struct journal_keys_to_wb {
+	struct btree_write_buffer_keys	*wb;
+	size_t				room;
+	u64				seq;
+};
+
+int __bch2_journal_key_to_wb(struct bch_fs *,
+			     struct journal_keys_to_wb *,
+			     enum btree_id, struct bkey_i *);
+
+static inline int bch2_journal_key_to_wb(struct bch_fs *c,
+			     struct journal_keys_to_wb *dst,
+			     enum btree_id btree, struct bkey_i *k)
+{
+	EBUG_ON(!dst->seq);
+
+	if (unlikely(!dst->room))
+		return __bch2_journal_key_to_wb(c, dst, btree, k);
+
+	struct btree_write_buffered_key *wb_k = &darray_top(dst->wb->keys);
+	wb_k->journal_seq	= dst->seq;
+	wb_k->btree		= btree;
+	bkey_copy(&wb_k->k, k);
+	dst->wb->keys.nr++;
+	dst->room--;
+	return 0;
+}
 
-int bch2_btree_insert_keys_write_buffer(struct btree_trans *);
+void bch2_journal_keys_to_write_buffer_start(struct bch_fs *, struct journal_keys_to_wb *, u64);
+void bch2_journal_keys_to_write_buffer_end(struct bch_fs *, struct journal_keys_to_wb *);
 
+int bch2_btree_write_buffer_resize(struct bch_fs *, size_t);
 void bch2_fs_btree_write_buffer_exit(struct bch_fs *);
 int bch2_fs_btree_write_buffer_init(struct bch_fs *);
 
diff --git a/fs/bcachefs/btree_write_buffer_types.h b/fs/bcachefs/btree_write_buffer_types.h
index 99993ba77aea..9b9433de9c36 100644
--- a/fs/bcachefs/btree_write_buffer_types.h
+++ b/fs/bcachefs/btree_write_buffer_types.h
@@ -2,43 +2,56 @@
 #ifndef _BCACHEFS_BTREE_WRITE_BUFFER_TYPES_H
 #define _BCACHEFS_BTREE_WRITE_BUFFER_TYPES_H
 
+#include "darray.h"
 #include "journal_types.h"
 
 #define BTREE_WRITE_BUFERED_VAL_U64s_MAX	4
 #define BTREE_WRITE_BUFERED_U64s_MAX	(BKEY_U64s + BTREE_WRITE_BUFERED_VAL_U64s_MAX)
 
-struct btree_write_buffered_key {
-	u64			journal_seq;
-	unsigned		journal_offset;
-	enum btree_id		btree;
-	__BKEY_PADDED(k, BTREE_WRITE_BUFERED_VAL_U64s_MAX);
-};
-
-union btree_write_buffer_state {
+struct wb_key_ref {
+union {
 	struct {
-		atomic64_t	counter;
-	};
-
-	struct {
-		u64		v;
-	};
-
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+		unsigned			idx:24;
+		u8				pos[sizeof(struct bpos)];
+		enum btree_id			btree:8;
+#else
+		enum btree_id			btree:8;
+		u8				pos[sizeof(struct bpos)];
+		unsigned			idx:24;
+#endif
+	} __packed;
 	struct {
-		u64			nr:23;
-		u64			idx:1;
-		u64			ref0:20;
-		u64			ref1:20;
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+		u64 lo;
+		u64 mi;
+		u64 hi;
+#else
+		u64 hi;
+		u64 mi;
+		u64 lo;
+#endif
 	};
 };
+};
 
-struct btree_write_buffer {
-	struct mutex			flush_lock;
-	struct journal_entry_pin	journal_pin;
+struct btree_write_buffered_key {
+	enum btree_id			btree:8;
+	u64				journal_seq:56;
+	__BKEY_PADDED(k, BTREE_WRITE_BUFERED_VAL_U64s_MAX);
+};
 
-	union btree_write_buffer_state	state;
-	size_t				size;
+struct btree_write_buffer_keys {
+	DARRAY(struct btree_write_buffered_key) keys;
+	struct journal_entry_pin	pin;
+	struct mutex			lock;
+};
 
-	struct btree_write_buffered_key	*keys[2];
+struct btree_write_buffer {
+	DARRAY(struct wb_key_ref)	sorted;
+	struct btree_write_buffer_keys	inc;
+	struct btree_write_buffer_keys	flushing;
+	struct work_struct		flush_work;
 };
 
 #endif /* _BCACHEFS_BTREE_WRITE_BUFFER_TYPES_H */
diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c
index 58d8c6ffd955..5bfa102a0438 100644
--- a/fs/bcachefs/buckets.c
+++ b/fs/bcachefs/buckets.c
@@ -61,7 +61,7 @@ void bch2_fs_usage_initialize(struct bch_fs *c)
 		usage->reserved += usage->persistent_reserved[i];
 
 	for (i = 0; i < c->replicas.nr; i++) {
-		struct bch_replicas_entry *e =
+		struct bch_replicas_entry_v1 *e =
 			cpu_replicas_entry(&c->replicas, i);
 
 		fs_usage_data_type_to_base(usage, e->data_type, usage->replicas[i]);
@@ -214,7 +214,7 @@ void bch2_fs_usage_to_text(struct printbuf *out,
 	}
 
 	for (i = 0; i < c->replicas.nr; i++) {
-		struct bch_replicas_entry *e =
+		struct bch_replicas_entry_v1 *e =
 			cpu_replicas_entry(&c->replicas, i);
 
 		prt_printf(out, "\t");
@@ -345,7 +345,7 @@ static void bch2_dev_usage_update_m(struct bch_fs *c, struct bch_dev *ca,
 
 static inline int __update_replicas(struct bch_fs *c,
 				    struct bch_fs_usage *fs_usage,
-				    struct bch_replicas_entry *r,
+				    struct bch_replicas_entry_v1 *r,
 				    s64 sectors)
 {
 	int idx = bch2_replicas_entry_idx(c, r);
@@ -359,7 +359,7 @@ static inline int __update_replicas(struct bch_fs *c,
 }
 
 static inline int update_replicas(struct bch_fs *c, struct bkey_s_c k,
-			struct bch_replicas_entry *r, s64 sectors,
+			struct bch_replicas_entry_v1 *r, s64 sectors,
 			unsigned journal_seq, bool gc)
 {
 	struct bch_fs_usage *fs_usage;
@@ -453,9 +453,9 @@ int bch2_replicas_deltas_realloc(struct btree_trans *trans, unsigned more)
 				__replicas_deltas_realloc(trans, more, _gfp));
 }
 
-static inline int update_replicas_list(struct btree_trans *trans,
-					struct bch_replicas_entry *r,
-					s64 sectors)
+int bch2_update_replicas_list(struct btree_trans *trans,
+			 struct bch_replicas_entry_v1 *r,
+			 s64 sectors)
 {
 	struct replicas_delta_list *d;
 	struct replicas_delta *n;
@@ -481,14 +481,13 @@ static inline int update_replicas_list(struct btree_trans *trans,
 	return 0;
 }
 
-static inline int update_cached_sectors_list(struct btree_trans *trans,
-					      unsigned dev, s64 sectors)
+int bch2_update_cached_sectors_list(struct btree_trans *trans, unsigned dev, s64 sectors)
 {
 	struct bch_replicas_padded r;
 
 	bch2_replicas_entry_cached(&r.e, dev);
 
-	return update_replicas_list(trans, &r.e, sectors);
+	return bch2_update_replicas_list(trans, &r.e, sectors);
 }
 
 int bch2_mark_alloc(struct btree_trans *trans,
@@ -580,23 +579,6 @@ int bch2_mark_alloc(struct btree_trans *trans,
 	}
 	percpu_up_read(&c->mark_lock);
 
-	/*
-	 * need to know if we're getting called from the invalidate path or
-	 * not:
-	 */
-
-	if ((flags & BTREE_TRIGGER_BUCKET_INVALIDATE) &&
-	    old_a->cached_sectors) {
-		ret = update_cached_sectors(c, new, ca->dev_idx,
-					    -((s64) old_a->cached_sectors),
-					    journal_seq, gc);
-		if (ret) {
-			bch2_fs_fatal_error(c, "%s(): no replicas entry while updating cached sectors",
-					    __func__);
-			return ret;
-		}
-	}
-
 	if (new_a->data_type == BCH_DATA_free &&
 	    (!new_a->journal_seq || new_a->journal_seq < c->journal.flushed_seq_ondisk))
 		closure_wake_up(&c->freelist_wait);
@@ -1470,7 +1452,7 @@ static int bch2_trans_mark_stripe_ptr(struct btree_trans *trans,
 
 	bch2_bkey_to_replicas(&r.e, bkey_i_to_s_c(&s->k_i));
 	r.e.data_type = data_type;
-	ret = update_replicas_list(trans, &r.e, sectors);
+	ret = bch2_update_replicas_list(trans, &r.e, sectors);
 err:
 	bch2_trans_iter_exit(trans, &iter);
 	return ret;
@@ -1513,8 +1495,8 @@ static int __trans_mark_extent(struct btree_trans *trans,
 
 		if (p.ptr.cached) {
 			if (!stale) {
-				ret = update_cached_sectors_list(trans, p.ptr.dev,
-								 disk_sectors);
+				ret = bch2_update_cached_sectors_list(trans, p.ptr.dev,
+								      disk_sectors);
 				if (ret)
 					return ret;
 			}
@@ -1532,7 +1514,7 @@ static int __trans_mark_extent(struct btree_trans *trans,
 	}
 
 	if (r.e.nr_devs)
-		ret = update_replicas_list(trans, &r.e, dirty_sectors);
+		ret = bch2_update_replicas_list(trans, &r.e, dirty_sectors);
 
 	return ret;
 }
@@ -1669,7 +1651,7 @@ int bch2_trans_mark_stripe(struct btree_trans *trans,
 		s64 sectors = le16_to_cpu(new_s->sectors);
 
 		bch2_bkey_to_replicas(&r.e, bkey_i_to_s_c(new));
-		ret = update_replicas_list(trans, &r.e, sectors * new_s->nr_redundant);
+		ret = bch2_update_replicas_list(trans, &r.e, sectors * new_s->nr_redundant);
 		if (ret)
 			return ret;
 	}
@@ -1678,7 +1660,7 @@ int bch2_trans_mark_stripe(struct btree_trans *trans,
 		s64 sectors = -((s64) le16_to_cpu(old_s->sectors));
 
 		bch2_bkey_to_replicas(&r.e, old);
-		ret = update_replicas_list(trans, &r.e, sectors * old_s->nr_redundant);
+		ret = bch2_update_replicas_list(trans, &r.e, sectors * old_s->nr_redundant);
 		if (ret)
 			return ret;
 	}
diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h
index 21f6cb356921..5574b62e0553 100644
--- a/fs/bcachefs/buckets.h
+++ b/fs/bcachefs/buckets.h
@@ -315,6 +315,9 @@ static inline struct bch_fs_usage *fs_usage_ptr(struct bch_fs *c,
 			    : c->usage[journal_seq & JOURNAL_BUF_MASK]);
 }
 
+int bch2_update_replicas_list(struct btree_trans *,
+			 struct bch_replicas_entry_v1 *, s64);
+int bch2_update_cached_sectors_list(struct btree_trans *, unsigned, s64);
 int bch2_replicas_deltas_realloc(struct btree_trans *, unsigned);
 
 void bch2_fs_usage_initialize(struct bch_fs *);
diff --git a/fs/bcachefs/chardev.c b/fs/bcachefs/chardev.c
index 4bb88aefed12..de3d82de9d29 100644
--- a/fs/bcachefs/chardev.c
+++ b/fs/bcachefs/chardev.c
@@ -444,7 +444,7 @@ static long bch2_ioctl_fs_usage(struct bch_fs *c,
 	dst_end = (void *) arg->replicas + replica_entries_bytes;
 
 	for (i = 0; i < c->replicas.nr; i++) {
-		struct bch_replicas_entry *src_e =
+		struct bch_replicas_entry_v1 *src_e =
 			cpu_replicas_entry(&c->replicas, i);
 
 		/* check that we have enough space for one replicas entry */
diff --git a/fs/bcachefs/clock.c b/fs/bcachefs/clock.c
index f41889093a2c..c36bfc627a98 100644
--- a/fs/bcachefs/clock.c
+++ b/fs/bcachefs/clock.c
@@ -95,7 +95,6 @@ void bch2_kthread_io_clock_wait(struct io_clock *clock,
 				unsigned long io_until,
 				unsigned long cpu_timeout)
 {
-	bool kthread = (current->flags & PF_KTHREAD) != 0;
 	struct io_clock_wait wait;
 
 	wait.io_timer.expire	= io_until;
@@ -111,7 +110,7 @@ void bch2_kthread_io_clock_wait(struct io_clock *clock,
 
 	while (1) {
 		set_current_state(TASK_INTERRUPTIBLE);
-		if (kthread && kthread_should_stop())
+		if (kthread_should_stop())
 			break;
 
 		if (wait.expired)
diff --git a/fs/bcachefs/darray.c b/fs/bcachefs/darray.c
new file mode 100644
index 000000000000..4c900c853268
--- /dev/null
+++ b/fs/bcachefs/darray.c
@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/log2.h>
+#include <linux/slab.h>
+#include "darray.h"
+
+int __bch2_darray_resize(darray_void *d, size_t element_size, size_t new_size, gfp_t gfp)
+{
+	if (new_size > d->size) {
+		new_size = roundup_pow_of_two(new_size);
+
+		void *data = kvmalloc_array(new_size, element_size, gfp);
+		if (!data)
+			return -ENOMEM;
+
+		memcpy(data, d->data, d->size * element_size);
+		kvfree(d->data);
+		d->data	= data;
+		d->size = new_size;
+	}
+
+	return 0;
+}
diff --git a/fs/bcachefs/darray.h b/fs/bcachefs/darray.h
index 87b4b2d1ec76..6157c53d5bf0 100644
--- a/fs/bcachefs/darray.h
+++ b/fs/bcachefs/darray.h
@@ -8,7 +8,6 @@
  * Inspired by CCAN's darray
  */
 
-#include "util.h"
 #include <linux/slab.h>
 
 #define DARRAY(type)							\
@@ -19,20 +18,25 @@ struct {								\
 
 typedef DARRAY(void) darray_void;
 
-static inline int __darray_make_room(darray_void *d, size_t t_size, size_t more, gfp_t gfp)
+int __bch2_darray_resize(darray_void *, size_t, size_t, gfp_t);
+
+static inline int __darray_resize(darray_void *d, size_t element_size,
+				  size_t new_size, gfp_t gfp)
 {
-	if (d->nr + more > d->size) {
-		size_t new_size = roundup_pow_of_two(d->nr + more);
-		void *data = krealloc_array(d->data, new_size, t_size, gfp);
+	return unlikely(new_size > d->size)
+		? __bch2_darray_resize(d, element_size, new_size, gfp)
+		: 0;
+}
 
-		if (!data)
-			return -ENOMEM;
+#define darray_resize_gfp(_d, _new_size, _gfp)				\
+	__darray_resize((darray_void *) (_d), sizeof((_d)->data[0]), (_new_size), _gfp)
 
-		d->data	= data;
-		d->size = new_size;
-	}
+#define darray_resize(_d, _new_size)					\
+	darray_resize_gfp(_d, _new_size, GFP_KERNEL)
 
-	return 0;
+static inline int __darray_make_room(darray_void *d, size_t t_size, size_t more, gfp_t gfp)
+{
+	return __darray_resize(d, t_size, d->nr + more, gfp);
 }
 
 #define darray_make_room_gfp(_d, _more, _gfp)				\
@@ -41,6 +45,8 @@ static inline int __darray_make_room(darray_void *d, size_t t_size, size_t more,
 #define darray_make_room(_d, _more)					\
 	darray_make_room_gfp(_d, _more, GFP_KERNEL)
 
+#define darray_room(_d)		((_d).size - (_d).nr)
+
 #define darray_top(_d)		((_d).data[(_d).nr])
 
 #define darray_push_gfp(_d, _item, _gfp)				\
@@ -86,7 +92,7 @@ do {									\
 
 #define darray_exit(_d)							\
 do {									\
-	kfree((_d)->data);						\
+	kvfree((_d)->data);						\
 	darray_init(_d);						\
 } while (0)
 
diff --git a/fs/bcachefs/data_update.c b/fs/bcachefs/data_update.c
index 5ed66202c226..55769d77e6e7 100644
--- a/fs/bcachefs/data_update.c
+++ b/fs/bcachefs/data_update.c
@@ -278,8 +278,8 @@ static int __bch2_data_update_index_update(struct btree_trans *trans,
 				BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE) ?:
 			bch2_trans_commit(trans, &op->res,
 				NULL,
-				BTREE_INSERT_NOCHECK_RW|
-				BTREE_INSERT_NOFAIL|
+				BCH_TRANS_COMMIT_no_check_rw|
+				BCH_TRANS_COMMIT_no_enospc|
 				m->data_opts.btree_insert_flags);
 		if (!ret) {
 			bch2_btree_iter_set_pos(&iter, next_pos);
diff --git a/fs/bcachefs/dirent.c b/fs/bcachefs/dirent.c
index 1a0f2d571569..0542d9948c24 100644
--- a/fs/bcachefs/dirent.c
+++ b/fs/bcachefs/dirent.c
@@ -201,7 +201,8 @@ static struct bkey_i_dirent *dirent_create_key(struct btree_trans *trans,
 int bch2_dirent_create(struct btree_trans *trans, subvol_inum dir,
 		       const struct bch_hash_info *hash_info,
 		       u8 type, const struct qstr *name, u64 dst_inum,
-		       u64 *dir_offset, int flags)
+		       u64 *dir_offset,
+		       bch_str_hash_flags_t str_hash_flags)
 {
 	struct bkey_i_dirent *dirent;
 	int ret;
@@ -212,7 +213,7 @@ int bch2_dirent_create(struct btree_trans *trans, subvol_inum dir,
 		return ret;
 
 	ret = bch2_hash_set(trans, bch2_dirent_hash_desc, hash_info,
-			    dir, &dirent->k_i, flags);
+			    dir, &dirent->k_i, str_hash_flags);
 	*dir_offset = dirent->k.p.offset;
 
 	return ret;
diff --git a/fs/bcachefs/dirent.h b/fs/bcachefs/dirent.h
index cd262bf4d9c5..8a55245547ba 100644
--- a/fs/bcachefs/dirent.h
+++ b/fs/bcachefs/dirent.h
@@ -37,7 +37,8 @@ int bch2_dirent_read_target(struct btree_trans *, subvol_inum,
 
 int bch2_dirent_create(struct btree_trans *, subvol_inum,
 		       const struct bch_hash_info *, u8,
-		       const struct qstr *, u64, u64 *, int);
+		       const struct qstr *, u64, u64 *,
+		       bch_str_hash_flags_t);
 
 static inline unsigned vfs_d_type(unsigned type)
 {
diff --git a/fs/bcachefs/ec.c b/fs/bcachefs/ec.c
index 2a77de18c004..2a02bf00b67f 100644
--- a/fs/bcachefs/ec.c
+++ b/fs/bcachefs/ec.c
@@ -803,7 +803,7 @@ static void ec_stripe_delete_work(struct work_struct *work)
 		if (!idx)
 			break;
 
-		ret = commit_do(trans, NULL, NULL, BTREE_INSERT_NOFAIL,
+		ret = commit_do(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc,
 				ec_stripe_delete(trans, idx));
 		if (ret) {
 			bch_err_fn(c, ret);
@@ -983,8 +983,8 @@ static int ec_stripe_update_bucket(struct btree_trans *trans, struct ec_stripe_b
 
 	while (1) {
 		ret = commit_do(trans, NULL, NULL,
-				BTREE_INSERT_NOCHECK_RW|
-				BTREE_INSERT_NOFAIL,
+				BCH_TRANS_COMMIT_no_check_rw|
+				BCH_TRANS_COMMIT_no_enospc,
 			ec_stripe_update_extent(trans, bucket_pos, bucket.gen,
 						s, &bp_pos));
 		if (ret)
@@ -1005,7 +1005,7 @@ static int ec_stripe_update_extents(struct bch_fs *c, struct ec_stripe_buf *s)
 	unsigned i, nr_data = v->nr_blocks - v->nr_redundant;
 	int ret = 0;
 
-	ret = bch2_btree_write_buffer_flush(trans);
+	ret = bch2_btree_write_buffer_flush_sync(trans);
 	if (ret)
 		goto err;
 
@@ -1121,8 +1121,8 @@ static void ec_stripe_create(struct ec_stripe_new *s)
 	}
 
 	ret = bch2_trans_do(c, &s->res, NULL,
-			    BTREE_INSERT_NOCHECK_RW|
-			    BTREE_INSERT_NOFAIL,
+			    BCH_TRANS_COMMIT_no_check_rw|
+			    BCH_TRANS_COMMIT_no_enospc,
 			    ec_stripe_key_update(trans,
 					bkey_i_to_stripe(&s->new_stripe.key),
 					!s->have_existing_stripe));
diff --git a/fs/bcachefs/ec_types.h b/fs/bcachefs/ec_types.h
index e2b02a82de32..976426da3a12 100644
--- a/fs/bcachefs/ec_types.h
+++ b/fs/bcachefs/ec_types.h
@@ -5,7 +5,7 @@
 #include "bcachefs_format.h"
 
 struct bch_replicas_padded {
-	struct bch_replicas_entry	e;
+	struct bch_replicas_entry_v1	e;
 	u8				pad[BCH_BKEY_PTRS_MAX];
 };
 
diff --git a/fs/bcachefs/errcode.h b/fs/bcachefs/errcode.h
index 68a1a96bb7ca..e42b45293bbd 100644
--- a/fs/bcachefs/errcode.h
+++ b/fs/bcachefs/errcode.h
@@ -73,7 +73,6 @@
 	x(ENOMEM,			ENOMEM_fsck_add_nlink)			\
 	x(ENOMEM,			ENOMEM_journal_key_insert)		\
 	x(ENOMEM,			ENOMEM_journal_keys_sort)		\
-	x(ENOMEM,			ENOMEM_journal_replay)			\
 	x(ENOMEM,			ENOMEM_read_superblock_clean)		\
 	x(ENOMEM,			ENOMEM_fs_alloc)			\
 	x(ENOMEM,			ENOMEM_fs_name_alloc)			\
@@ -151,7 +150,6 @@
 	x(BCH_ERR_btree_insert_fail,	btree_insert_need_mark_replicas)	\
 	x(BCH_ERR_btree_insert_fail,	btree_insert_need_journal_res)		\
 	x(BCH_ERR_btree_insert_fail,	btree_insert_need_journal_reclaim)	\
-	x(BCH_ERR_btree_insert_fail,	btree_insert_need_flush_buffer)		\
 	x(0,				backpointer_to_overwritten_btree_node)	\
 	x(0,				lock_fail_root_changed)			\
 	x(0,				journal_reclaim_would_deadlock)		\
diff --git a/fs/bcachefs/fs-io-direct.c b/fs/bcachefs/fs-io-direct.c
index 5b42a76c4796..9a479e4de6b3 100644
--- a/fs/bcachefs/fs-io-direct.c
+++ b/fs/bcachefs/fs-io-direct.c
@@ -35,9 +35,9 @@ static void bio_check_or_release(struct bio *bio, bool check_dirty)
 	}
 }
 
-static void bch2_dio_read_complete(struct closure *cl)
+static CLOSURE_CALLBACK(bch2_dio_read_complete)
 {
-	struct dio_read *dio = container_of(cl, struct dio_read, cl);
+	closure_type(dio, struct dio_read, cl);
 
 	dio->req->ki_complete(dio->req, dio->ret);
 	bio_check_or_release(&dio->rbio.bio, dio->should_dirty);
@@ -325,9 +325,9 @@ static noinline int bch2_dio_write_copy_iov(struct dio_write *dio)
 	return 0;
 }
 
-static void bch2_dio_write_flush_done(struct closure *cl)
+static CLOSURE_CALLBACK(bch2_dio_write_flush_done)
 {
-	struct dio_write *dio = container_of(cl, struct dio_write, op.cl);
+	closure_type(dio, struct dio_write, op.cl);
 	struct bch_fs *c = dio->op.c;
 
 	closure_debug_destroy(cl);
diff --git a/fs/bcachefs/fs-io.c b/fs/bcachefs/fs-io.c
index b0e8144ec550..31f40e587a4f 100644
--- a/fs/bcachefs/fs-io.c
+++ b/fs/bcachefs/fs-io.c
@@ -861,7 +861,8 @@ loff_t bch2_remap_file_range(struct file *file_src, loff_t pos_src,
 	    abs(pos_src - pos_dst) < len)
 		return -EINVAL;
 
-	bch2_lock_inodes(INODE_LOCK|INODE_PAGECACHE_BLOCK, src, dst);
+	lock_two_nondirectories(&src->v, &dst->v);
+	bch2_lock_inodes(INODE_PAGECACHE_BLOCK, src, dst);
 
 	inode_dio_wait(&src->v);
 	inode_dio_wait(&dst->v);
@@ -914,7 +915,8 @@ loff_t bch2_remap_file_range(struct file *file_src, loff_t pos_src,
 		ret = bch2_flush_inode(c, dst);
 err:
 	bch2_quota_reservation_put(c, dst, &quota_res);
-	bch2_unlock_inodes(INODE_LOCK|INODE_PAGECACHE_BLOCK, src, dst);
+	bch2_unlock_inodes(INODE_PAGECACHE_BLOCK, src, dst);
+	unlock_two_nondirectories(&src->v, &dst->v);
 
 	return bch2_err_class(ret);
 }
diff --git a/fs/bcachefs/fs-ioctl.c b/fs/bcachefs/fs-ioctl.c
index 5a39bcb597a3..e0e24e47dd1b 100644
--- a/fs/bcachefs/fs-ioctl.c
+++ b/fs/bcachefs/fs-ioctl.c
@@ -291,14 +291,14 @@ static int bch2_ioc_goingdown(struct bch_fs *c, u32 __user *arg)
 
 	switch (flags) {
 	case FSOP_GOING_FLAGS_DEFAULT:
-		ret = freeze_bdev(c->vfs_sb->s_bdev);
+		ret = bdev_freeze(c->vfs_sb->s_bdev);
 		if (ret)
 			goto err;
 
 		bch2_journal_flush(&c->journal);
 		c->vfs_sb->s_flags |= SB_RDONLY;
 		bch2_fs_emergency_read_only(c);
-		thaw_bdev(c->vfs_sb->s_bdev);
+		bdev_thaw(c->vfs_sb->s_bdev);
 		break;
 
 	case FSOP_GOING_FLAGS_LOGFLUSH:
@@ -453,33 +453,36 @@ static long bch2_ioctl_subvolume_create(struct bch_fs *c, struct file *filp,
 static long bch2_ioctl_subvolume_destroy(struct bch_fs *c, struct file *filp,
 				struct bch_ioctl_subvolume arg)
 {
+	const char __user *name = (void __user *)(unsigned long)arg.dst_ptr;
 	struct path path;
 	struct inode *dir;
+	struct dentry *victim;
 	int ret = 0;
 
 	if (arg.flags)
 		return -EINVAL;
 
-	ret = user_path_at(arg.dirfd,
-			(const char __user *)(unsigned long)arg.dst_ptr,
-			LOOKUP_FOLLOW, &path);
-	if (ret)
-		return ret;
+	victim = user_path_locked_at(arg.dirfd, name, &path);
+	if (IS_ERR(victim))
+		return PTR_ERR(victim);
 
-	if (path.dentry->d_sb->s_fs_info != c) {
+	if (victim->d_sb->s_fs_info != c) {
 		ret = -EXDEV;
 		goto err;
 	}
-
-	dir = path.dentry->d_parent->d_inode;
-
-	ret = __bch2_unlink(dir, path.dentry, true);
-	if (ret)
+	if (!d_is_positive(victim)) {
+		ret = -ENOENT;
 		goto err;
-
-	fsnotify_rmdir(dir, path.dentry);
-	d_delete(path.dentry);
+	}
+	dir = d_inode(path.dentry);
+	ret = __bch2_unlink(dir, victim, true);
+	if (!ret) {
+		fsnotify_rmdir(dir, victim);
+		d_delete(victim);
+	}
+	inode_unlock(dir);
 err:
+	dput(victim);
 	path_put(&path);
 	return ret;
 }
diff --git a/fs/bcachefs/fs.c b/fs/bcachefs/fs.c
index 8ef817304e4a..5bb9832ec718 100644
--- a/fs/bcachefs/fs.c
+++ b/fs/bcachefs/fs.c
@@ -93,7 +93,7 @@ int __must_check bch2_write_inode(struct bch_fs *c,
 				BTREE_ITER_INTENT) ?:
 		(set ? set(trans, inode, &inode_u, p) : 0) ?:
 		bch2_inode_write(trans, &iter, &inode_u) ?:
-		bch2_trans_commit(trans, NULL, NULL, BTREE_INSERT_NOFAIL);
+		bch2_trans_commit(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc);
 
 	/*
 	 * the btree node lock protects inode->ei_inode, not ei_update_lock;
@@ -452,7 +452,7 @@ int __bch2_unlink(struct inode *vdir, struct dentry *dentry,
 	bch2_lock_inodes(INODE_UPDATE_LOCK, dir, inode);
 
 	ret = commit_do(trans, NULL, NULL,
-			BTREE_INSERT_NOFAIL,
+			BCH_TRANS_COMMIT_no_enospc,
 		bch2_unlink_trans(trans,
 				  inode_inum(dir), &dir_u,
 				  &inode_u, &dentry->d_name,
@@ -717,7 +717,7 @@ int bch2_setattr_nonsize(struct mnt_idmap *idmap,
 
 	ret =   bch2_inode_write(trans, &inode_iter, &inode_u) ?:
 		bch2_trans_commit(trans, NULL, NULL,
-				  BTREE_INSERT_NOFAIL);
+				  BCH_TRANS_COMMIT_no_enospc);
 btree_err:
 	bch2_trans_iter_exit(trans, &inode_iter);
 
@@ -1103,7 +1103,7 @@ static const struct address_space_operations bch_address_space_operations = {
 #ifdef CONFIG_MIGRATION
 	.migrate_folio	= filemap_migrate_folio,
 #endif
-	.error_remove_page = generic_error_remove_page,
+	.error_remove_folio = generic_error_remove_folio,
 };
 
 struct bcachefs_fid {
@@ -1667,8 +1667,7 @@ static int bch2_show_devname(struct seq_file *seq, struct dentry *root)
 		if (!first)
 			seq_putc(seq, ':');
 		first = false;
-		seq_puts(seq, "/dev/");
-		seq_puts(seq, ca->name);
+		seq_puts(seq, ca->disk_sb.sb_name);
 	}
 
 	return 0;
diff --git a/fs/bcachefs/fs.h b/fs/bcachefs/fs.h
index 5edf1d4b9e6b..c3af7225ff69 100644
--- a/fs/bcachefs/fs.h
+++ b/fs/bcachefs/fs.h
@@ -77,9 +77,8 @@ static inline int ptrcmp(void *l, void *r)
 }
 
 enum bch_inode_lock_op {
-	INODE_LOCK		= (1U << 0),
-	INODE_PAGECACHE_BLOCK	= (1U << 1),
-	INODE_UPDATE_LOCK	= (1U << 2),
+	INODE_PAGECACHE_BLOCK	= (1U << 0),
+	INODE_UPDATE_LOCK	= (1U << 1),
 };
 
 #define bch2_lock_inodes(_locks, ...)					\
@@ -91,8 +90,6 @@ do {									\
 									\
 	for (i = 1; i < ARRAY_SIZE(a); i++)				\
 		if (a[i] != a[i - 1]) {					\
-			if ((_locks) & INODE_LOCK)			\
-				down_write_nested(&a[i]->v.i_rwsem, i);	\
 			if ((_locks) & INODE_PAGECACHE_BLOCK)		\
 				bch2_pagecache_block_get(a[i]);\
 			if ((_locks) & INODE_UPDATE_LOCK)			\
@@ -109,8 +106,6 @@ do {									\
 									\
 	for (i = 1; i < ARRAY_SIZE(a); i++)				\
 		if (a[i] != a[i - 1]) {					\
-			if ((_locks) & INODE_LOCK)			\
-				up_write(&a[i]->v.i_rwsem);		\
 			if ((_locks) & INODE_PAGECACHE_BLOCK)		\
 				bch2_pagecache_block_put(a[i]);\
 			if ((_locks) & INODE_UPDATE_LOCK)			\
diff --git a/fs/bcachefs/fsck.c b/fs/bcachefs/fsck.c
index e0c5cd119acc..b1c89a4821f5 100644
--- a/fs/bcachefs/fsck.c
+++ b/fs/bcachefs/fsck.c
@@ -208,8 +208,8 @@ static int fsck_write_inode(struct btree_trans *trans,
 			    u32 snapshot)
 {
 	int ret = commit_do(trans, NULL, NULL,
-				  BTREE_INSERT_NOFAIL|
-				  BTREE_INSERT_LAZY_RW,
+				  BCH_TRANS_COMMIT_no_enospc|
+				  BCH_TRANS_COMMIT_lazy_rw,
 				  __write_inode(trans, inode, snapshot));
 	if (ret)
 		bch_err_fn(trans->c, ret);
@@ -354,8 +354,8 @@ static int reattach_inode(struct btree_trans *trans,
 			  u32 inode_snapshot)
 {
 	int ret = commit_do(trans, NULL, NULL,
-				  BTREE_INSERT_LAZY_RW|
-				  BTREE_INSERT_NOFAIL,
+				  BCH_TRANS_COMMIT_lazy_rw|
+				  BCH_TRANS_COMMIT_no_enospc,
 			__reattach_inode(trans, inode, inode_snapshot));
 	bch_err_msg(trans->c, ret, "reattaching inode %llu", inode->bi_inum);
 	return ret;
@@ -757,8 +757,8 @@ static int hash_redo_key(struct btree_trans *trans,
 				       BCH_HASH_SET_MUST_CREATE,
 				       BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE) ?:
 		bch2_trans_commit(trans, NULL, NULL,
-				  BTREE_INSERT_NOFAIL|
-				  BTREE_INSERT_LAZY_RW);
+				  BCH_TRANS_COMMIT_no_enospc|
+				  BCH_TRANS_COMMIT_lazy_rw);
 }
 
 static int hash_check_key(struct btree_trans *trans,
@@ -826,6 +826,18 @@ static int hash_check_key(struct btree_trans *trans,
 	goto out;
 }
 
+static int check_inode_deleted_list(struct btree_trans *trans, struct bpos p)
+{
+	struct btree_iter iter;
+	struct bkey_s_c k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_deleted_inodes, p, 0);
+	int ret = bkey_err(k);
+	if (ret)
+		return ret;
+
+	bch2_trans_iter_exit(trans, &iter);
+	return k.k->type == KEY_TYPE_set;
+}
+
 static int check_inode(struct btree_trans *trans,
 		       struct btree_iter *iter,
 		       struct bkey_s_c k,
@@ -890,6 +902,17 @@ static int check_inode(struct btree_trans *trans,
 		return 0;
 	}
 
+	if (u.bi_flags & BCH_INODE_unlinked &&
+	    c->sb.version >= bcachefs_metadata_version_deleted_inodes) {
+		ret = check_inode_deleted_list(trans, k.k->p);
+		if (ret)
+			return ret;
+
+		fsck_err_on(ret, c, unlinked_inode_not_on_deleted_list,
+			    "inode %llu:%u unlinked, but not on deleted list",
+			    u.bi_inum, k.k->p.snapshot);
+	}
+
 	if (u.bi_flags & BCH_INODE_unlinked &&
 	    (!c->sb.clean ||
 	     fsck_err(c, inode_unlinked_but_clean,
@@ -992,7 +1015,7 @@ int bch2_check_inodes(struct bch_fs *c)
 	ret = for_each_btree_key_commit(trans, iter, BTREE_ID_inodes,
 			POS_MIN,
 			BTREE_ITER_PREFETCH|BTREE_ITER_ALL_SNAPSHOTS, k,
-			NULL, NULL, BTREE_INSERT_LAZY_RW|BTREE_INSERT_NOFAIL,
+			NULL, NULL, BCH_TRANS_COMMIT_lazy_rw|BCH_TRANS_COMMIT_no_enospc,
 		check_inode(trans, &iter, k, &prev, &s, full));
 
 	snapshots_seen_exit(&s);
@@ -1023,25 +1046,6 @@ static bool dirent_points_to_inode(struct bkey_s_c_dirent d,
 		: le64_to_cpu(d.v->d_inum)		== inode->bi_inum;
 }
 
-static int inode_backpointer_exists(struct btree_trans *trans,
-				    struct bch_inode_unpacked *inode,
-				    u32 snapshot)
-{
-	struct btree_iter iter;
-	struct bkey_s_c_dirent d;
-	int ret;
-
-	d = dirent_get_by_pos(trans, &iter,
-			SPOS(inode->bi_dir, inode->bi_dir_offset, snapshot));
-	ret = bkey_err(d);
-	if (ret)
-		return bch2_err_matches(ret, ENOENT) ? 0 : ret;
-
-	ret = dirent_points_to_inode(d, inode);
-	bch2_trans_iter_exit(trans, &iter);
-	return ret;
-}
-
 static int check_i_sectors(struct btree_trans *trans, struct inode_walker *w)
 {
 	struct bch_fs *c = trans->c;
@@ -1226,7 +1230,7 @@ static int overlapping_extents_found(struct btree_trans *trans,
 				BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE,
 				k1, k2) ?:
 			bch2_trans_commit(trans, &res, NULL,
-				BTREE_INSERT_LAZY_RW|BTREE_INSERT_NOFAIL);
+				BCH_TRANS_COMMIT_lazy_rw|BCH_TRANS_COMMIT_no_enospc);
 		bch2_disk_reservation_put(c, &res);
 
 		if (ret)
@@ -1465,7 +1469,7 @@ int bch2_check_extents(struct bch_fs *c)
 			POS(BCACHEFS_ROOT_INO, 0),
 			BTREE_ITER_PREFETCH|BTREE_ITER_ALL_SNAPSHOTS, k,
 			&res, NULL,
-			BTREE_INSERT_LAZY_RW|BTREE_INSERT_NOFAIL, ({
+			BCH_TRANS_COMMIT_lazy_rw|BCH_TRANS_COMMIT_no_enospc, ({
 		bch2_disk_reservation_put(c, &res);
 		check_extent(trans, &iter, k, &w, &s, &extent_ends) ?:
 		check_extent_overbig(trans, &iter, k);
@@ -1494,7 +1498,7 @@ int bch2_check_indirect_extents(struct bch_fs *c)
 			POS_MIN,
 			BTREE_ITER_PREFETCH, k,
 			&res, NULL,
-			BTREE_INSERT_LAZY_RW|BTREE_INSERT_NOFAIL, ({
+			BCH_TRANS_COMMIT_lazy_rw|BCH_TRANS_COMMIT_no_enospc, ({
 		bch2_disk_reservation_put(c, &res);
 		check_extent_overbig(trans, &iter, k);
 	}));
@@ -1553,8 +1557,8 @@ static int check_dirent_target(struct btree_trans *trans,
 {
 	struct bch_fs *c = trans->c;
 	struct bkey_i_dirent *n;
-	bool backpointer_exists = true;
 	struct printbuf buf = PRINTBUF;
+	struct btree_iter bp_iter = { NULL };
 	int ret = 0;
 
 	if (!target->bi_dir &&
@@ -1568,25 +1572,37 @@ static int check_dirent_target(struct btree_trans *trans,
 	}
 
 	if (!inode_points_to_dirent(target, d)) {
-		ret = inode_backpointer_exists(trans, target, d.k->p.snapshot);
-		if (ret < 0)
+		struct bkey_s_c_dirent bp_dirent = dirent_get_by_pos(trans, &bp_iter,
+				      SPOS(target->bi_dir, target->bi_dir_offset, target_snapshot));
+		ret = bkey_err(bp_dirent);
+		if (ret && !bch2_err_matches(ret, ENOENT))
 			goto err;
 
-		backpointer_exists = ret;
+		bool backpointer_exists = !ret;
 		ret = 0;
 
+		bch2_bkey_val_to_text(&buf, c, d.s_c);
+		prt_newline(&buf);
+		if (backpointer_exists)
+			bch2_bkey_val_to_text(&buf, c, bp_dirent.s_c);
+
 		if (fsck_err_on(S_ISDIR(target->bi_mode) && backpointer_exists,
 				c, inode_dir_multiple_links,
-				"directory %llu with multiple links",
-				target->bi_inum)) {
+				"directory %llu:%u with multiple links\n%s",
+				target->bi_inum, target_snapshot, buf.buf)) {
 			ret = __remove_dirent(trans, d.k->p);
 			goto out;
 		}
 
+		/*
+		 * hardlinked file with nlink 0:
+		 * We're just adjusting nlink here so check_nlinks() will pick
+		 * it up, it ignores inodes with nlink 0
+		 */
 		if (fsck_err_on(backpointer_exists && !target->bi_nlink,
 				c, inode_multiple_links_but_nlink_0,
-				"inode %llu type %s has multiple links but i_nlink 0",
-				target->bi_inum, bch2_d_types[d.v->d_type])) {
+				"inode %llu:%u type %s has multiple links but i_nlink 0\n%s",
+				target->bi_inum, target_snapshot, bch2_d_types[d.v->d_type], buf.buf)) {
 			target->bi_nlink++;
 			target->bi_flags &= ~BCH_INODE_unlinked;
 
@@ -1660,6 +1676,7 @@ static int check_dirent_target(struct btree_trans *trans,
 out:
 err:
 fsck_err:
+	bch2_trans_iter_exit(trans, &bp_iter);
 	printbuf_exit(&buf);
 	bch_err_fn(c, ret);
 	return ret;
@@ -1854,7 +1871,7 @@ int bch2_check_dirents(struct bch_fs *c)
 			BTREE_ITER_PREFETCH|BTREE_ITER_ALL_SNAPSHOTS,
 			k,
 			NULL, NULL,
-			BTREE_INSERT_LAZY_RW|BTREE_INSERT_NOFAIL,
+			BCH_TRANS_COMMIT_lazy_rw|BCH_TRANS_COMMIT_no_enospc,
 		check_dirent(trans, &iter, k, &hash_info, &dir, &target, &s));
 
 	bch2_trans_put(trans);
@@ -1918,7 +1935,7 @@ int bch2_check_xattrs(struct bch_fs *c)
 			BTREE_ITER_PREFETCH|BTREE_ITER_ALL_SNAPSHOTS,
 			k,
 			NULL, NULL,
-			BTREE_INSERT_LAZY_RW|BTREE_INSERT_NOFAIL,
+			BCH_TRANS_COMMIT_lazy_rw|BCH_TRANS_COMMIT_no_enospc,
 		check_xattr(trans, &iter, k, &hash_info, &inode)));
 	bch_err_fn(c, ret);
 	return ret;
@@ -1949,8 +1966,8 @@ static int check_root_trans(struct btree_trans *trans)
 		root_subvol.v.snapshot	= cpu_to_le32(snapshot);
 		root_subvol.v.inode	= cpu_to_le64(inum);
 		ret = commit_do(trans, NULL, NULL,
-				      BTREE_INSERT_NOFAIL|
-				      BTREE_INSERT_LAZY_RW,
+				      BCH_TRANS_COMMIT_no_enospc|
+				      BCH_TRANS_COMMIT_lazy_rw,
 			bch2_btree_insert_trans(trans, BTREE_ID_subvolumes,
 					    &root_subvol.k_i, 0));
 		bch_err_msg(c, ret, "writing root subvol");
@@ -1986,8 +2003,8 @@ int bch2_check_root(struct bch_fs *c)
 	int ret;
 
 	ret = bch2_trans_do(c, NULL, NULL,
-			     BTREE_INSERT_NOFAIL|
-			     BTREE_INSERT_LAZY_RW,
+			     BCH_TRANS_COMMIT_no_enospc|
+			     BCH_TRANS_COMMIT_lazy_rw,
 		check_root_trans(trans));
 	bch_err_fn(c, ret);
 	return ret;
@@ -2116,8 +2133,8 @@ static int check_path(struct btree_trans *trans,
 				return 0;
 
 			ret = commit_do(trans, NULL, NULL,
-					      BTREE_INSERT_NOFAIL|
-					      BTREE_INSERT_LAZY_RW,
+					      BCH_TRANS_COMMIT_no_enospc|
+					      BCH_TRANS_COMMIT_lazy_rw,
 					remove_backpointer(trans, inode));
 			if (ret) {
 				bch_err(c, "error removing dirent: %i", ret);
@@ -2398,10 +2415,10 @@ static int check_nlinks_update_hardlinks(struct bch_fs *c,
 		for_each_btree_key_commit(trans, iter, BTREE_ID_inodes,
 				POS(0, range_start),
 				BTREE_ITER_INTENT|BTREE_ITER_PREFETCH|BTREE_ITER_ALL_SNAPSHOTS, k,
-				NULL, NULL, BTREE_INSERT_LAZY_RW|BTREE_INSERT_NOFAIL,
+				NULL, NULL, BCH_TRANS_COMMIT_lazy_rw|BCH_TRANS_COMMIT_no_enospc,
 			check_nlinks_update_inode(trans, &iter, k, links, &idx, range_end)));
 	if (ret < 0) {
-		bch_err(c, "error in fsck: btree error %i while walking inodes", ret);
+		bch_err(c, "error in fsck walking inodes: %s", bch2_err_str(ret));
 		return ret;
 	}
 
@@ -2483,7 +2500,7 @@ int bch2_fix_reflink_p(struct bch_fs *c)
 				BTREE_ID_extents, POS_MIN,
 				BTREE_ITER_INTENT|BTREE_ITER_PREFETCH|
 				BTREE_ITER_ALL_SNAPSHOTS, k,
-				NULL, NULL, BTREE_INSERT_NOFAIL|BTREE_INSERT_LAZY_RW,
+				NULL, NULL, BCH_TRANS_COMMIT_no_enospc|BCH_TRANS_COMMIT_lazy_rw,
 			fix_reflink_p_key(trans, &iter, k)));
 	bch_err_fn(c, ret);
 	return ret;
diff --git a/fs/bcachefs/inode.c b/fs/bcachefs/inode.c
index c7849b0753e7..a2c96f7c193f 100644
--- a/fs/bcachefs/inode.c
+++ b/fs/bcachefs/inode.c
@@ -830,7 +830,7 @@ static int bch2_inode_delete_keys(struct btree_trans *trans,
 
 		ret = bch2_trans_update(trans, &iter, &delete, 0) ?:
 		      bch2_trans_commit(trans, NULL, NULL,
-					BTREE_INSERT_NOFAIL);
+					BCH_TRANS_COMMIT_no_enospc);
 err:
 		if (ret && !bch2_err_matches(ret, BCH_ERR_transaction_restart))
 			break;
@@ -893,7 +893,7 @@ int bch2_inode_rm(struct bch_fs *c, subvol_inum inum)
 
 	ret   = bch2_trans_update(trans, &iter, &delete.k_i, 0) ?:
 		bch2_trans_commit(trans, NULL, NULL,
-				BTREE_INSERT_NOFAIL);
+				BCH_TRANS_COMMIT_no_enospc);
 err:
 	bch2_trans_iter_exit(trans, &iter);
 	if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
@@ -1057,7 +1057,7 @@ int bch2_inode_rm_snapshot(struct btree_trans *trans, u64 inum, u32 snapshot)
 
 	ret   = bch2_trans_update(trans, &iter, &delete.k_i, 0) ?:
 		bch2_trans_commit(trans, NULL, NULL,
-				BTREE_INSERT_NOFAIL);
+				BCH_TRANS_COMMIT_no_enospc);
 err:
 	bch2_trans_iter_exit(trans, &iter);
 	if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
@@ -1157,10 +1157,6 @@ int bch2_delete_dead_inodes(struct bch_fs *c)
 again:
 	need_another_pass = false;
 
-	ret = bch2_btree_write_buffer_flush_sync(trans);
-	if (ret)
-		goto err;
-
 	/*
 	 * Weird transaction restart handling here because on successful delete,
 	 * bch2_inode_rm_snapshot() will return a nested transaction restart,
@@ -1170,8 +1166,8 @@ int bch2_delete_dead_inodes(struct bch_fs *c)
 	for_each_btree_key(trans, iter, BTREE_ID_deleted_inodes, POS_MIN,
 			   BTREE_ITER_PREFETCH|BTREE_ITER_ALL_SNAPSHOTS, k, ret) {
 		ret = commit_do(trans, NULL, NULL,
-				BTREE_INSERT_NOFAIL|
-				BTREE_INSERT_LAZY_RW,
+				BCH_TRANS_COMMIT_no_enospc|
+				BCH_TRANS_COMMIT_lazy_rw,
 			may_delete_deleted_inode(trans, &iter, k.k->p, &need_another_pass));
 		if (ret < 0)
 			break;
@@ -1191,8 +1187,12 @@ int bch2_delete_dead_inodes(struct bch_fs *c)
 	}
 	bch2_trans_iter_exit(trans, &iter);
 
-	if (!ret && need_another_pass)
+	if (!ret && need_another_pass) {
+		ret = bch2_btree_write_buffer_flush_sync(trans);
+		if (ret)
+			goto err;
 		goto again;
+	}
 err:
 	bch2_trans_put(trans);
 
diff --git a/fs/bcachefs/io_misc.c b/fs/bcachefs/io_misc.c
index bebc11444ef5..eab0c8c57785 100644
--- a/fs/bcachefs/io_misc.c
+++ b/fs/bcachefs/io_misc.c
@@ -256,7 +256,7 @@ static int __bch2_resume_logged_op_truncate(struct btree_trans *trans,
 	u64 new_i_size = le64_to_cpu(op->v.new_i_size);
 	int ret;
 
-	ret = commit_do(trans, NULL, NULL, BTREE_INSERT_NOFAIL,
+	ret = commit_do(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc,
 			truncate_set_isize(trans, inum, new_i_size));
 	if (ret)
 		goto err;
@@ -378,7 +378,7 @@ case LOGGED_OP_FINSERT_start:
 	op->v.state = LOGGED_OP_FINSERT_shift_extents;
 
 	if (insert) {
-		ret = commit_do(trans, NULL, NULL, BTREE_INSERT_NOFAIL,
+		ret = commit_do(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc,
 				adjust_i_size(trans, inum, src_offset, len) ?:
 				bch2_logged_op_update(trans, &op->k_i));
 		if (ret)
@@ -390,7 +390,7 @@ case LOGGED_OP_FINSERT_start:
 		if (ret && !bch2_err_matches(ret, BCH_ERR_transaction_restart))
 			goto err;
 
-		ret = commit_do(trans, NULL, NULL, BTREE_INSERT_NOFAIL,
+		ret = commit_do(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc,
 				bch2_logged_op_update(trans, &op->k_i));
 	}
 
@@ -455,7 +455,7 @@ case LOGGED_OP_FINSERT_shift_extents:
 			bch2_btree_insert_trans(trans, BTREE_ID_extents, &delete, 0) ?:
 			bch2_btree_insert_trans(trans, BTREE_ID_extents, copy, 0) ?:
 			bch2_logged_op_update(trans, &op->k_i) ?:
-			bch2_trans_commit(trans, &disk_res, NULL, BTREE_INSERT_NOFAIL);
+			bch2_trans_commit(trans, &disk_res, NULL, BCH_TRANS_COMMIT_no_enospc);
 btree_err:
 		bch2_disk_reservation_put(c, &disk_res);
 
@@ -470,12 +470,12 @@ case LOGGED_OP_FINSERT_shift_extents:
 	op->v.state = LOGGED_OP_FINSERT_finish;
 
 	if (!insert) {
-		ret = commit_do(trans, NULL, NULL, BTREE_INSERT_NOFAIL,
+		ret = commit_do(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc,
 				adjust_i_size(trans, inum, src_offset, shift) ?:
 				bch2_logged_op_update(trans, &op->k_i));
 	} else {
 		/* We need an inode update to update bi_journal_seq for fsync: */
-		ret = commit_do(trans, NULL, NULL, BTREE_INSERT_NOFAIL,
+		ret = commit_do(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc,
 				adjust_i_size(trans, inum, 0, 0) ?:
 				bch2_logged_op_update(trans, &op->k_i));
 	}
diff --git a/fs/bcachefs/io_read.c b/fs/bcachefs/io_read.c
index a56ed553dc15..b833409c7865 100644
--- a/fs/bcachefs/io_read.c
+++ b/fs/bcachefs/io_read.c
@@ -526,7 +526,7 @@ static int __bch2_rbio_narrow_crcs(struct btree_trans *trans,
 
 static noinline void bch2_rbio_narrow_crcs(struct bch_read_bio *rbio)
 {
-	bch2_trans_do(rbio->c, NULL, NULL, BTREE_INSERT_NOFAIL,
+	bch2_trans_do(rbio->c, NULL, NULL, BCH_TRANS_COMMIT_no_enospc,
 		      __bch2_rbio_narrow_crcs(trans, rbio));
 }
 
diff --git a/fs/bcachefs/io_write.c b/fs/bcachefs/io_write.c
index d704a8f829c8..d6bd8f788d3a 100644
--- a/fs/bcachefs/io_write.c
+++ b/fs/bcachefs/io_write.c
@@ -316,8 +316,8 @@ int bch2_extent_update(struct btree_trans *trans,
 						  i_sectors_delta) ?:
 		bch2_trans_update(trans, iter, k, 0) ?:
 		bch2_trans_commit(trans, disk_res, NULL,
-				BTREE_INSERT_NOCHECK_RW|
-				BTREE_INSERT_NOFAIL);
+				BCH_TRANS_COMMIT_no_check_rw|
+				BCH_TRANS_COMMIT_no_enospc);
 	if (unlikely(ret))
 		return ret;
 
@@ -580,9 +580,9 @@ static inline void wp_update_state(struct write_point *wp, bool running)
 	__wp_update_state(wp, state);
 }
 
-static void bch2_write_index(struct closure *cl)
+static CLOSURE_CALLBACK(bch2_write_index)
 {
-	struct bch_write_op *op = container_of(cl, struct bch_write_op, cl);
+	closure_type(op, struct bch_write_op, cl);
 	struct write_point *wp = op->wp;
 	struct workqueue_struct *wq = index_update_wq(op);
 	unsigned long flags;
@@ -1176,7 +1176,7 @@ static void bch2_nocow_write_convert_unwritten(struct bch_write_op *op)
 		ret = for_each_btree_key_upto_commit(trans, iter, BTREE_ID_extents,
 				     bkey_start_pos(&orig->k), orig->k.p,
 				     BTREE_ITER_INTENT, k,
-				     NULL, NULL, BTREE_INSERT_NOFAIL, ({
+				     NULL, NULL, BCH_TRANS_COMMIT_no_enospc, ({
 			bch2_nocow_write_convert_one_unwritten(trans, &iter, orig, k, op->new_i_size);
 		}));
 
@@ -1208,9 +1208,9 @@ static void __bch2_nocow_write_done(struct bch_write_op *op)
 		bch2_nocow_write_convert_unwritten(op);
 }
 
-static void bch2_nocow_write_done(struct closure *cl)
+static CLOSURE_CALLBACK(bch2_nocow_write_done)
 {
-	struct bch_write_op *op = container_of(cl, struct bch_write_op, cl);
+	closure_type(op, struct bch_write_op, cl);
 
 	__bch2_nocow_write_done(op);
 	bch2_write_done(cl);
@@ -1363,7 +1363,7 @@ static void bch2_nocow_write(struct bch_write_op *op)
 		op->insert_keys.top = op->insert_keys.keys;
 	} else if (op->flags & BCH_WRITE_SYNC) {
 		closure_sync(&op->cl);
-		bch2_nocow_write_done(&op->cl);
+		bch2_nocow_write_done(&op->cl.work);
 	} else {
 		/*
 		 * XXX
@@ -1566,9 +1566,9 @@ static void bch2_write_data_inline(struct bch_write_op *op, unsigned data_len)
  * If op->discard is true, instead of inserting the data it invalidates the
  * region of the cache represented by op->bio and op->inode.
  */
-void bch2_write(struct closure *cl)
+CLOSURE_CALLBACK(bch2_write)
 {
-	struct bch_write_op *op = container_of(cl, struct bch_write_op, cl);
+	closure_type(op, struct bch_write_op, cl);
 	struct bio *bio = &op->wbio.bio;
 	struct bch_fs *c = op->c;
 	unsigned data_len;
diff --git a/fs/bcachefs/io_write.h b/fs/bcachefs/io_write.h
index 9323167229ee..6c276a48f95d 100644
--- a/fs/bcachefs/io_write.h
+++ b/fs/bcachefs/io_write.h
@@ -90,8 +90,7 @@ static inline void bch2_write_op_init(struct bch_write_op *op, struct bch_fs *c,
 	op->devs_need_flush	= NULL;
 }
 
-void bch2_write(struct closure *);
-
+CLOSURE_CALLBACK(bch2_write);
 void bch2_write_point_do_index_updates(struct work_struct *);
 
 static inline struct bch_write_bio *wbio_init(struct bio *bio)
diff --git a/fs/bcachefs/journal.c b/fs/bcachefs/journal.c
index 23a9b7845d11..86b148d9bea3 100644
--- a/fs/bcachefs/journal.c
+++ b/fs/bcachefs/journal.c
@@ -10,6 +10,7 @@
 #include "bkey_methods.h"
 #include "btree_gc.h"
 #include "btree_update.h"
+#include "btree_write_buffer.h"
 #include "buckets.h"
 #include "error.h"
 #include "journal.h"
@@ -147,6 +148,7 @@ void bch2_journal_buf_put_final(struct journal *j, u64 seq, bool write)
 		bch2_journal_reclaim_fast(j);
 	if (write)
 		closure_call(&j->io, bch2_journal_write, c->io_complete_wq, NULL);
+	wake_up(&j->wait);
 }
 
 /*
@@ -184,6 +186,8 @@ static void __journal_entry_close(struct journal *j, unsigned closed_val)
 	/* Close out old buffer: */
 	buf->data->u64s		= cpu_to_le32(old.cur_entry_offset);
 
+	trace_journal_entry_close(c, vstruct_bytes(buf->data));
+
 	sectors = vstruct_blocks_plus(buf->data, c->block_bits,
 				      buf->u64s_reserved) << c->block_bits;
 	BUG_ON(sectors > buf->sectors);
@@ -328,6 +332,7 @@ static int journal_entry_open(struct journal *j)
 	buf->must_flush	= false;
 	buf->separate_flush = false;
 	buf->flush_time	= 0;
+	buf->need_flush_to_write_buffer = true;
 
 	memset(buf->data, 0, sizeof(*buf->data));
 	buf->data->seq	= cpu_to_le64(journal_cur_seq(j));
@@ -361,11 +366,6 @@ static int journal_entry_open(struct journal *j)
 	} while ((v = atomic64_cmpxchg(&j->reservations.counter,
 				       old.v, new.v)) != old.v);
 
-	if (j->res_get_blocked_start)
-		bch2_time_stats_update(j->blocked_time,
-				       j->res_get_blocked_start);
-	j->res_get_blocked_start = 0;
-
 	mod_delayed_work(c->io_complete_wq,
 			 &j->write_work,
 			 msecs_to_jiffies(c->opts.journal_flush_delay));
@@ -465,15 +465,12 @@ static int __journal_res_get(struct journal *j, struct journal_res *res,
 	__journal_entry_close(j, JOURNAL_ENTRY_CLOSED_VAL);
 	ret = journal_entry_open(j);
 
-	if (ret == JOURNAL_ERR_max_in_flight)
+	if (ret == JOURNAL_ERR_max_in_flight) {
+		track_event_change(&c->times[BCH_TIME_blocked_journal_max_in_flight],
+				   &j->max_in_flight_start, true);
 		trace_and_count(c, journal_entry_full, c);
-unlock:
-	if ((ret && ret != JOURNAL_ERR_insufficient_devices) &&
-	    !j->res_get_blocked_start) {
-		j->res_get_blocked_start = local_clock() ?: 1;
-		trace_and_count(c, journal_full, c);
 	}
-
+unlock:
 	can_discard = j->can_discard;
 	spin_unlock(&j->lock);
 
@@ -772,6 +769,75 @@ void bch2_journal_block(struct journal *j)
 	journal_quiesce(j);
 }
 
+/*
+ * XXX: ideally this would not be closing the current journal entry, but
+ * otherwise we do not have a way to avoid racing with res_get() - j->blocked
+ * will race.
+ */
+static bool journal_reservations_stopped(struct journal *j)
+{
+	union journal_res_state s;
+
+	journal_entry_close(j);
+
+	s.v = atomic64_read_acquire(&j->reservations.counter);
+
+	return  s.buf0_count == 0 &&
+		s.buf1_count == 0 &&
+		s.buf2_count == 0 &&
+		s.buf3_count == 0;
+}
+
+void bch2_journal_block_reservations(struct journal *j)
+{
+	spin_lock(&j->lock);
+	j->blocked++;
+	spin_unlock(&j->lock);
+
+	wait_event(j->wait, journal_reservations_stopped(j));
+}
+
+static struct journal_buf *__bch2_next_write_buffer_flush_journal_buf(struct journal *j, u64 max_seq)
+{
+	spin_lock(&j->lock);
+	max_seq = min(max_seq, journal_cur_seq(j));
+
+	for (u64 seq = journal_last_unwritten_seq(j);
+	     seq <= max_seq;
+	     seq++) {
+		unsigned idx = seq & JOURNAL_BUF_MASK;
+		struct journal_buf *buf = j->buf + idx;
+		union journal_res_state s;
+
+		if (!buf->need_flush_to_write_buffer)
+			continue;
+
+		if (seq == journal_cur_seq(j))
+			__journal_entry_close(j, JOURNAL_ENTRY_CLOSED_VAL);
+
+		s.v = atomic64_read_acquire(&j->reservations.counter);
+
+		if (journal_state_count(s, idx)) {
+			spin_unlock(&j->lock);
+			return ERR_PTR(-EAGAIN);
+		}
+
+		spin_unlock(&j->lock);
+		return buf;
+	}
+
+	spin_unlock(&j->lock);
+	return NULL;
+}
+
+struct journal_buf *bch2_next_write_buffer_flush_journal_buf(struct journal *j, u64 max_seq)
+{
+	struct journal_buf *ret;
+
+	wait_event(j->wait, (ret = __bch2_next_write_buffer_flush_journal_buf(j, max_seq)) != ERR_PTR(-EAGAIN));
+	return ret;
+}
+
 /* allocate journal on a device: */
 
 static int __bch2_set_nr_journal_buckets(struct bch_dev *ca, unsigned nr,
@@ -1223,6 +1289,7 @@ int bch2_fs_journal_init(struct journal *j)
 	static struct lock_class_key res_key;
 	unsigned i;
 
+	mutex_init(&j->buf_lock);
 	spin_lock_init(&j->lock);
 	spin_lock_init(&j->err_lock);
 	init_waitqueue_head(&j->wait);
@@ -1260,6 +1327,7 @@ void __bch2_journal_debug_to_text(struct printbuf *out, struct journal *j)
 	union journal_res_state s;
 	struct bch_dev *ca;
 	unsigned long now = jiffies;
+	u64 nr_writes = j->nr_flush_writes + j->nr_noflush_writes;
 	u64 seq;
 	unsigned i;
 
@@ -1273,20 +1341,23 @@ void __bch2_journal_debug_to_text(struct printbuf *out, struct journal *j)
 	prt_printf(out, "dirty journal entries:\t%llu/%llu\n",	fifo_used(&j->pin), j->pin.size);
 	prt_printf(out, "seq:\t\t\t%llu\n",			journal_cur_seq(j));
 	prt_printf(out, "seq_ondisk:\t\t%llu\n",		j->seq_ondisk);
-	prt_printf(out, "last_seq:\t\t%llu\n",		journal_last_seq(j));
+	prt_printf(out, "last_seq:\t\t%llu\n",			journal_last_seq(j));
 	prt_printf(out, "last_seq_ondisk:\t%llu\n",		j->last_seq_ondisk);
-	prt_printf(out, "flushed_seq_ondisk:\t%llu\n",	j->flushed_seq_ondisk);
-	prt_printf(out, "watermark:\t\t%s\n",		bch2_watermarks[j->watermark]);
-	prt_printf(out, "each entry reserved:\t%u\n",	j->entry_u64s_reserved);
+	prt_printf(out, "flushed_seq_ondisk:\t%llu\n",		j->flushed_seq_ondisk);
+	prt_printf(out, "watermark:\t\t%s\n",			bch2_watermarks[j->watermark]);
+	prt_printf(out, "each entry reserved:\t%u\n",		j->entry_u64s_reserved);
 	prt_printf(out, "nr flush writes:\t%llu\n",		j->nr_flush_writes);
-	prt_printf(out, "nr noflush writes:\t%llu\n",	j->nr_noflush_writes);
-	prt_printf(out, "nr direct reclaim:\t%llu\n",	j->nr_direct_reclaim);
+	prt_printf(out, "nr noflush writes:\t%llu\n",		j->nr_noflush_writes);
+	prt_printf(out, "average write size:\t");
+	prt_human_readable_u64(out, nr_writes ? div64_u64(j->entry_bytes_written, nr_writes) : 0);
+	prt_newline(out);
+	prt_printf(out, "nr direct reclaim:\t%llu\n",		j->nr_direct_reclaim);
 	prt_printf(out, "nr background reclaim:\t%llu\n",	j->nr_background_reclaim);
 	prt_printf(out, "reclaim kicked:\t\t%u\n",		j->reclaim_kicked);
-	prt_printf(out, "reclaim runs in:\t%u ms\n",	time_after(j->next_reclaim, now)
+	prt_printf(out, "reclaim runs in:\t%u ms\n",		time_after(j->next_reclaim, now)
 	       ? jiffies_to_msecs(j->next_reclaim - jiffies) : 0);
-	prt_printf(out, "current entry sectors:\t%u\n",	j->cur_entry_sectors);
-	prt_printf(out, "current entry error:\t%s\n",	bch2_journal_errors[j->cur_entry_error]);
+	prt_printf(out, "current entry sectors:\t%u\n",		j->cur_entry_sectors);
+	prt_printf(out, "current entry error:\t%s\n",		bch2_journal_errors[j->cur_entry_error]);
 	prt_printf(out, "current entry:\t\t");
 
 	switch (s.cur_entry_offset) {
diff --git a/fs/bcachefs/journal.h b/fs/bcachefs/journal.h
index c85d01cf4948..b5185f97af0f 100644
--- a/fs/bcachefs/journal.h
+++ b/fs/bcachefs/journal.h
@@ -259,7 +259,7 @@ static inline union journal_res_state journal_state_buf_put(struct journal *j, u
 {
 	union journal_res_state s;
 
-	s.v = atomic64_sub_return(((union journal_res_state) {
+	s.v = atomic64_sub_return_release(((union journal_res_state) {
 				    .buf0_count = idx == 0,
 				    .buf1_count = idx == 1,
 				    .buf2_count = idx == 2,
@@ -427,6 +427,8 @@ static inline void bch2_journal_set_replay_done(struct journal *j)
 
 void bch2_journal_unblock(struct journal *);
 void bch2_journal_block(struct journal *);
+void bch2_journal_block_reservations(struct journal *);
+struct journal_buf *bch2_next_write_buffer_flush_journal_buf(struct journal *j, u64 max_seq);
 
 void __bch2_journal_debug_to_text(struct printbuf *, struct journal *);
 void bch2_journal_debug_to_text(struct printbuf *, struct journal *);
diff --git a/fs/bcachefs/journal_io.c b/fs/bcachefs/journal_io.c
index 786a09285509..4ec5d5d38abc 100644
--- a/fs/bcachefs/journal_io.c
+++ b/fs/bcachefs/journal_io.c
@@ -4,6 +4,7 @@
 #include "alloc_foreground.h"
 #include "btree_io.h"
 #include "btree_update_interior.h"
+#include "btree_write_buffer.h"
 #include "buckets.h"
 #include "checksum.h"
 #include "disk_groups.h"
@@ -713,6 +714,22 @@ static void journal_entry_overwrite_to_text(struct printbuf *out, struct bch_fs
 	journal_entry_btree_keys_to_text(out, c, entry);
 }
 
+static int journal_entry_write_buffer_keys_validate(struct bch_fs *c,
+				struct jset *jset,
+				struct jset_entry *entry,
+				unsigned version, int big_endian,
+				enum bkey_invalid_flags flags)
+{
+	return journal_entry_btree_keys_validate(c, jset, entry,
+				version, big_endian, READ);
+}
+
+static void journal_entry_write_buffer_keys_to_text(struct printbuf *out, struct bch_fs *c,
+					    struct jset_entry *entry)
+{
+	journal_entry_btree_keys_to_text(out, c, entry);
+}
+
 struct jset_entry_ops {
 	int (*validate)(struct bch_fs *, struct jset *,
 			struct jset_entry *, unsigned, int,
@@ -1025,10 +1042,9 @@ static int journal_read_bucket(struct bch_dev *ca,
 	return 0;
 }
 
-static void bch2_journal_read_device(struct closure *cl)
+static CLOSURE_CALLBACK(bch2_journal_read_device)
 {
-	struct journal_device *ja =
-		container_of(cl, struct journal_device, read);
+	closure_type(ja, struct journal_device, read);
 	struct bch_dev *ca = container_of(ja, struct bch_dev, journal);
 	struct bch_fs *c = ca->fs;
 	struct journal_list *jlist =
@@ -1494,6 +1510,8 @@ static int journal_write_alloc(struct journal *j, struct journal_buf *w)
 
 static void journal_buf_realloc(struct journal *j, struct journal_buf *buf)
 {
+	struct bch_fs *c = container_of(j, struct bch_fs, journal);
+
 	/* we aren't holding j->lock: */
 	unsigned new_size = READ_ONCE(j->buf_size_want);
 	void *new_buf;
@@ -1501,6 +1519,11 @@ static void journal_buf_realloc(struct journal *j, struct journal_buf *buf)
 	if (buf->buf_size >= new_size)
 		return;
 
+	size_t btree_write_buffer_size = new_size / 64;
+
+	if (bch2_btree_write_buffer_resize(c, btree_write_buffer_size))
+		return;
+
 	new_buf = kvpmalloc(new_size, GFP_NOFS|__GFP_NOWARN);
 	if (!new_buf)
 		return;
@@ -1520,9 +1543,9 @@ static inline struct journal_buf *journal_last_unwritten_buf(struct journal *j)
 	return j->buf + (journal_last_unwritten_seq(j) & JOURNAL_BUF_MASK);
 }
 
-static void journal_write_done(struct closure *cl)
+static CLOSURE_CALLBACK(journal_write_done)
 {
-	struct journal *j = container_of(cl, struct journal, io);
+	closure_type(j, struct journal, io);
 	struct bch_fs *c = container_of(j, struct bch_fs, journal);
 	struct journal_buf *w = journal_last_unwritten_buf(j);
 	struct bch_replicas_padded replicas;
@@ -1590,8 +1613,12 @@ static void journal_write_done(struct closure *cl)
 	} while ((v = atomic64_cmpxchg(&j->reservations.counter,
 				       old.v, new.v)) != old.v);
 
+	bch2_journal_reclaim_fast(j);
 	bch2_journal_space_available(j);
 
+	track_event_change(&c->times[BCH_TIME_blocked_journal_max_in_flight],
+			   &j->max_in_flight_start, false);
+
 	closure_wake_up(&w->wait);
 	journal_wake(j);
 
@@ -1638,9 +1665,9 @@ static void journal_write_endio(struct bio *bio)
 	percpu_ref_put(&ca->io_ref);
 }
 
-static void do_journal_write(struct closure *cl)
+static CLOSURE_CALLBACK(do_journal_write)
 {
-	struct journal *j = container_of(cl, struct journal, io);
+	closure_type(j, struct journal, io);
 	struct bch_fs *c = container_of(j, struct bch_fs, journal);
 	struct bch_dev *ca;
 	struct journal_buf *w = journal_last_unwritten_buf(j);
@@ -1690,9 +1717,11 @@ static int bch2_journal_write_prep(struct journal *j, struct journal_buf *w)
 	struct bch_fs *c = container_of(j, struct bch_fs, journal);
 	struct jset_entry *start, *end, *i, *next, *prev = NULL;
 	struct jset *jset = w->data;
+	struct journal_keys_to_wb wb = { NULL };
 	unsigned sectors, bytes, u64s;
-	bool validate_before_checksum = false;
 	unsigned long btree_roots_have = 0;
+	bool validate_before_checksum = false;
+	u64 seq = le64_to_cpu(jset->seq);
 	int ret;
 
 	/*
@@ -1720,9 +1749,28 @@ static int bch2_journal_write_prep(struct journal *j, struct journal_buf *w)
 		 * to c->btree_roots we have to get any missing btree roots and
 		 * add them to this journal entry:
 		 */
-		if (i->type == BCH_JSET_ENTRY_btree_root) {
+		switch (i->type) {
+		case BCH_JSET_ENTRY_btree_root:
 			bch2_journal_entry_to_btree_root(c, i);
 			__set_bit(i->btree_id, &btree_roots_have);
+			break;
+		case BCH_JSET_ENTRY_write_buffer_keys:
+			EBUG_ON(!w->need_flush_to_write_buffer);
+
+			if (!wb.wb)
+				bch2_journal_keys_to_write_buffer_start(c, &wb, seq);
+
+			struct bkey_i *k;
+			jset_entry_for_each_key(i, k) {
+				ret = bch2_journal_key_to_wb(c, &wb, i->btree_id, k);
+				if (ret) {
+					bch2_fs_fatal_error(c, "-ENOMEM flushing journal keys to btree write buffer");
+					bch2_journal_keys_to_write_buffer_end(c, &wb);
+					return ret;
+				}
+			}
+			i->type = BCH_JSET_ENTRY_btree_keys;
+			break;
 		}
 
 		/* Can we merge with previous entry? */
@@ -1745,6 +1793,10 @@ static int bch2_journal_write_prep(struct journal *j, struct journal_buf *w)
 			memmove_u64s_down(prev, i, jset_u64s(u64s));
 	}
 
+	if (wb.wb)
+		bch2_journal_keys_to_write_buffer_end(c, &wb);
+	w->need_flush_to_write_buffer = false;
+
 	prev = prev ? vstruct_next(prev) : jset->start;
 	jset->u64s = cpu_to_le32((u64 *) prev - jset->_data);
 
@@ -1752,8 +1804,7 @@ static int bch2_journal_write_prep(struct journal *j, struct journal_buf *w)
 
 	end	= bch2_btree_roots_to_journal_entries(c, end, btree_roots_have);
 
-	bch2_journal_super_entries_add_common(c, &end,
-				le64_to_cpu(jset->seq));
+	bch2_journal_super_entries_add_common(c, &end, seq);
 	u64s	= (u64 *) end - (u64 *) start;
 	BUG_ON(u64s > j->entry_u64s_reserved);
 
@@ -1776,7 +1827,7 @@ static int bch2_journal_write_prep(struct journal *j, struct journal_buf *w)
 	SET_JSET_CSUM_TYPE(jset, bch2_meta_checksum_type(c));
 
 	if (!JSET_NO_FLUSH(jset) && journal_entry_empty(jset))
-		j->last_empty_seq = le64_to_cpu(jset->seq);
+		j->last_empty_seq = seq;
 
 	if (bch2_csum_type_is_encryption(JSET_CSUM_TYPE(jset)))
 		validate_before_checksum = true;
@@ -1835,7 +1886,7 @@ static int bch2_journal_write_pick_flush(struct journal *j, struct journal_buf *
 	    (!w->must_flush &&
 	     (jiffies - j->last_flush_write) < msecs_to_jiffies(c->opts.journal_flush_delay) &&
 	     test_bit(JOURNAL_MAY_SKIP_FLUSH, &j->flags))) {
-		     w->noflush = true;
+		w->noflush = true;
 		SET_JSET_NO_FLUSH(w->data, true);
 		w->data->last_seq	= 0;
 		w->last_seq		= 0;
@@ -1850,9 +1901,9 @@ static int bch2_journal_write_pick_flush(struct journal *j, struct journal_buf *
 	return 0;
 }
 
-void bch2_journal_write(struct closure *cl)
+CLOSURE_CALLBACK(bch2_journal_write)
 {
-	struct journal *j = container_of(cl, struct journal, io);
+	closure_type(j, struct journal, io);
 	struct bch_fs *c = container_of(j, struct bch_fs, journal);
 	struct bch_dev *ca;
 	struct journal_buf *w = journal_last_unwritten_buf(j);
@@ -1872,12 +1923,16 @@ void bch2_journal_write(struct closure *cl)
 	if (ret)
 		goto err;
 
+	mutex_lock(&j->buf_lock);
 	journal_buf_realloc(j, w);
 
 	ret = bch2_journal_write_prep(j, w);
+	mutex_unlock(&j->buf_lock);
 	if (ret)
 		goto err;
 
+	j->entry_bytes_written += vstruct_bytes(w->data);
+
 	while (1) {
 		spin_lock(&j->lock);
 		ret = journal_write_alloc(j, w);
diff --git a/fs/bcachefs/journal_io.h b/fs/bcachefs/journal_io.h
index a88d097b13f1..c035e7c108e1 100644
--- a/fs/bcachefs/journal_io.h
+++ b/fs/bcachefs/journal_io.h
@@ -60,6 +60,6 @@ void bch2_journal_ptrs_to_text(struct printbuf *, struct bch_fs *,
 
 int bch2_journal_read(struct bch_fs *, u64 *, u64 *, u64 *);
 
-void bch2_journal_write(struct closure *);
+CLOSURE_CALLBACK(bch2_journal_write);
 
 #endif /* _BCACHEFS_JOURNAL_IO_H */
diff --git a/fs/bcachefs/journal_reclaim.c b/fs/bcachefs/journal_reclaim.c
index e63c6eda86af..2aa4c0c6bbba 100644
--- a/fs/bcachefs/journal_reclaim.c
+++ b/fs/bcachefs/journal_reclaim.c
@@ -3,6 +3,7 @@
 #include "bcachefs.h"
 #include "btree_key_cache.h"
 #include "btree_update.h"
+#include "btree_write_buffer.h"
 #include "buckets.h"
 #include "errcode.h"
 #include "error.h"
@@ -50,17 +51,24 @@ unsigned bch2_journal_dev_buckets_available(struct journal *j,
 	return available;
 }
 
-static inline void journal_set_watermark(struct journal *j, bool low_on_space)
+void bch2_journal_set_watermark(struct journal *j)
 {
-	unsigned watermark = BCH_WATERMARK_stripe;
-
-	if (low_on_space)
-		watermark = max_t(unsigned, watermark, BCH_WATERMARK_reclaim);
-	if (fifo_free(&j->pin) < j->pin.size / 4)
-		watermark = max_t(unsigned, watermark, BCH_WATERMARK_reclaim);
-
-	if (watermark == j->watermark)
-		return;
+	struct bch_fs *c = container_of(j, struct bch_fs, journal);
+	bool low_on_space = j->space[journal_space_clean].total * 4 <=
+		j->space[journal_space_total].total;
+	bool low_on_pin = fifo_free(&j->pin) < j->pin.size / 4;
+	bool low_on_wb = bch2_btree_write_buffer_must_wait(c);
+	unsigned watermark = low_on_space || low_on_pin || low_on_wb
+		? BCH_WATERMARK_reclaim
+		: BCH_WATERMARK_stripe;
+
+	if (track_event_change(&c->times[BCH_TIME_blocked_journal_low_on_space],
+			       &j->low_on_space_start, low_on_space) ||
+	    track_event_change(&c->times[BCH_TIME_blocked_journal_low_on_pin],
+			       &j->low_on_pin_start, low_on_pin) ||
+	    track_event_change(&c->times[BCH_TIME_blocked_write_buffer_full],
+			       &j->write_buffer_full_start, low_on_wb))
+		trace_and_count(c, journal_full, c);
 
 	swap(watermark, j->watermark);
 	if (watermark > j->watermark)
@@ -226,7 +234,7 @@ void bch2_journal_space_available(struct journal *j)
 	else
 		clear_bit(JOURNAL_MAY_SKIP_FLUSH, &j->flags);
 
-	journal_set_watermark(j, clean * 4 <= total);
+	bch2_journal_set_watermark(j);
 out:
 	j->cur_entry_sectors	= !ret ? j->space[journal_space_discarded].next_entry : 0;
 	j->cur_entry_error	= ret;
@@ -299,6 +307,7 @@ void bch2_journal_reclaim_fast(struct journal *j)
 	 * all btree nodes got written out
 	 */
 	while (!fifo_empty(&j->pin) &&
+	       j->pin.front <= j->seq_ondisk &&
 	       !atomic_read(&fifo_peek_front(&j->pin).count)) {
 		j->pin.front++;
 		popped = true;
@@ -367,15 +376,36 @@ static enum journal_pin_type journal_pin_type(journal_pin_flush_fn fn)
 		return JOURNAL_PIN_other;
 }
 
-void bch2_journal_pin_set(struct journal *j, u64 seq,
+static inline void bch2_journal_pin_set_locked(struct journal *j, u64 seq,
 			  struct journal_entry_pin *pin,
-			  journal_pin_flush_fn flush_fn)
+			  journal_pin_flush_fn flush_fn,
+			  enum journal_pin_type type)
+{
+	struct journal_entry_pin_list *pin_list = journal_seq_pin(j, seq);
+
+	/*
+	 * flush_fn is how we identify journal pins in debugfs, so must always
+	 * exist, even if it doesn't do anything:
+	 */
+	BUG_ON(!flush_fn);
+
+	atomic_inc(&pin_list->count);
+	pin->seq	= seq;
+	pin->flush	= flush_fn;
+	list_add(&pin->list, &pin_list->list[type]);
+}
+
+void bch2_journal_pin_copy(struct journal *j,
+			   struct journal_entry_pin *dst,
+			   struct journal_entry_pin *src,
+			   journal_pin_flush_fn flush_fn)
 {
-	struct journal_entry_pin_list *pin_list;
 	bool reclaim;
 
 	spin_lock(&j->lock);
 
+	u64 seq = READ_ONCE(src->seq);
+
 	if (seq < journal_last_seq(j)) {
 		/*
 		 * bch2_journal_pin_copy() raced with bch2_journal_pin_drop() on
@@ -387,18 +417,34 @@ void bch2_journal_pin_set(struct journal *j, u64 seq,
 		return;
 	}
 
-	pin_list = journal_seq_pin(j, seq);
+	reclaim = __journal_pin_drop(j, dst);
 
-	reclaim = __journal_pin_drop(j, pin);
+	bch2_journal_pin_set_locked(j, seq, dst, flush_fn, journal_pin_type(flush_fn));
 
-	atomic_inc(&pin_list->count);
-	pin->seq	= seq;
-	pin->flush	= flush_fn;
+	if (reclaim)
+		bch2_journal_reclaim_fast(j);
+	spin_unlock(&j->lock);
 
-	if (flush_fn)
-		list_add(&pin->list, &pin_list->list[journal_pin_type(flush_fn)]);
-	else
-		list_add(&pin->list, &pin_list->flushed);
+	/*
+	 * If the journal is currently full,  we might want to call flush_fn
+	 * immediately:
+	 */
+	journal_wake(j);
+}
+
+void bch2_journal_pin_set(struct journal *j, u64 seq,
+			  struct journal_entry_pin *pin,
+			  journal_pin_flush_fn flush_fn)
+{
+	bool reclaim;
+
+	spin_lock(&j->lock);
+
+	BUG_ON(seq < journal_last_seq(j));
+
+	reclaim = __journal_pin_drop(j, pin);
+
+	bch2_journal_pin_set_locked(j, seq, pin, flush_fn, journal_pin_type(flush_fn));
 
 	if (reclaim)
 		bch2_journal_reclaim_fast(j);
@@ -594,7 +640,6 @@ static u64 journal_seq_to_flush(struct journal *j)
 static int __bch2_journal_reclaim(struct journal *j, bool direct, bool kicked)
 {
 	struct bch_fs *c = container_of(j, struct bch_fs, journal);
-	bool kthread = (current->flags & PF_KTHREAD) != 0;
 	u64 seq_to_flush;
 	size_t min_nr, min_key_cache, nr_flushed;
 	unsigned flags;
@@ -610,7 +655,7 @@ static int __bch2_journal_reclaim(struct journal *j, bool direct, bool kicked)
 	flags = memalloc_noreclaim_save();
 
 	do {
-		if (kthread && kthread_should_stop())
+		if (kthread_should_stop())
 			break;
 
 		if (bch2_journal_error(j)) {
@@ -793,6 +838,7 @@ static int journal_flush_done(struct journal *j, u64 seq_to_flush,
 
 bool bch2_journal_flush_pins(struct journal *j, u64 seq_to_flush)
 {
+	/* time_stats this */
 	bool did_work = false;
 
 	if (!test_bit(JOURNAL_STARTED, &j->flags))
diff --git a/fs/bcachefs/journal_reclaim.h b/fs/bcachefs/journal_reclaim.h
index 494d1a6eddb0..ec84c3345281 100644
--- a/fs/bcachefs/journal_reclaim.h
+++ b/fs/bcachefs/journal_reclaim.h
@@ -16,6 +16,7 @@ static inline void journal_reclaim_kick(struct journal *j)
 unsigned bch2_journal_dev_buckets_available(struct journal *,
 					    struct journal_device *,
 					    enum journal_space_from);
+void bch2_journal_set_watermark(struct journal *);
 void bch2_journal_space_available(struct journal *);
 
 static inline bool journal_pin_active(struct journal_entry_pin *pin)
@@ -47,17 +48,10 @@ static inline void bch2_journal_pin_add(struct journal *j, u64 seq,
 		bch2_journal_pin_set(j, seq, pin, flush_fn);
 }
 
-static inline void bch2_journal_pin_copy(struct journal *j,
-					 struct journal_entry_pin *dst,
-					 struct journal_entry_pin *src,
-					 journal_pin_flush_fn flush_fn)
-{
-	/* Guard against racing with journal_pin_drop(src): */
-	u64 seq = READ_ONCE(src->seq);
-
-	if (seq)
-		bch2_journal_pin_add(j, seq, dst, flush_fn);
-}
+void bch2_journal_pin_copy(struct journal *,
+			   struct journal_entry_pin *,
+			   struct journal_entry_pin *,
+			   journal_pin_flush_fn);
 
 static inline void bch2_journal_pin_update(struct journal *j, u64 seq,
 					   struct journal_entry_pin *pin,
diff --git a/fs/bcachefs/journal_types.h b/fs/bcachefs/journal_types.h
index a756b69582e3..85c543af60e5 100644
--- a/fs/bcachefs/journal_types.h
+++ b/fs/bcachefs/journal_types.h
@@ -36,6 +36,7 @@ struct journal_buf {
 	bool			noflush;	/* write has already been kicked off, and was noflush */
 	bool			must_flush;	/* something wants a flush */
 	bool			separate_flush;
+	bool			need_flush_to_write_buffer;
 };
 
 /*
@@ -181,6 +182,12 @@ struct journal {
 	 */
 	darray_u64		early_journal_entries;
 
+	/*
+	 * Protects journal_buf->data, when accessing without a jorunal
+	 * reservation: for synchronization between the btree write buffer code
+	 * and the journal write path:
+	 */
+	struct mutex		buf_lock;
 	/*
 	 * Two journal entries -- one is currently open for new entries, the
 	 * other is possibly being written out.
@@ -262,15 +269,19 @@ struct journal {
 
 	unsigned long		last_flush_write;
 
-	u64			res_get_blocked_start;
 	u64			write_start_time;
 
 	u64			nr_flush_writes;
 	u64			nr_noflush_writes;
+	u64			entry_bytes_written;
+
+	u64			low_on_space_start;
+	u64			low_on_pin_start;
+	u64			max_in_flight_start;
+	u64			write_buffer_full_start;
 
 	struct bch2_time_stats	*flush_write_time;
 	struct bch2_time_stats	*noflush_write_time;
-	struct bch2_time_stats	*blocked_time;
 	struct bch2_time_stats	*flush_seq_time;
 
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
diff --git a/fs/bcachefs/logged_ops.c b/fs/bcachefs/logged_ops.c
index 8640f7dee0de..9a76a9aab5c3 100644
--- a/fs/bcachefs/logged_ops.c
+++ b/fs/bcachefs/logged_ops.c
@@ -85,13 +85,13 @@ static int __bch2_logged_op_start(struct btree_trans *trans, struct bkey_i *k)
 
 int bch2_logged_op_start(struct btree_trans *trans, struct bkey_i *k)
 {
-	return commit_do(trans, NULL, NULL, BTREE_INSERT_NOFAIL,
+	return commit_do(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc,
 			 __bch2_logged_op_start(trans, k));
 }
 
 void bch2_logged_op_finish(struct btree_trans *trans, struct bkey_i *k)
 {
-	int ret = commit_do(trans, NULL, NULL, BTREE_INSERT_NOFAIL,
+	int ret = commit_do(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc,
 			    bch2_btree_delete(trans, BTREE_ID_logged_ops, k->k.p, 0));
 	/*
 	 * This needs to be a fatal error because we've left an unfinished
diff --git a/fs/bcachefs/lru.c b/fs/bcachefs/lru.c
index a5cc0ed195d6..e6d081c0592c 100644
--- a/fs/bcachefs/lru.c
+++ b/fs/bcachefs/lru.c
@@ -155,7 +155,7 @@ int bch2_check_lrus(struct bch_fs *c)
 	ret = bch2_trans_run(c,
 		for_each_btree_key_commit(trans, iter,
 				BTREE_ID_lru, POS_MIN, BTREE_ITER_PREFETCH, k,
-				NULL, NULL, BTREE_INSERT_NOFAIL|BTREE_INSERT_LAZY_RW,
+				NULL, NULL, BCH_TRANS_COMMIT_no_enospc|BCH_TRANS_COMMIT_lazy_rw,
 			bch2_check_lru_key(trans, &iter, k, &last_flushed_pos)));
 	if (ret)
 		bch_err_fn(c, ret);
diff --git a/fs/bcachefs/migrate.c b/fs/bcachefs/migrate.c
index e3a51f6d6c9b..8e5688d0a8ca 100644
--- a/fs/bcachefs/migrate.c
+++ b/fs/bcachefs/migrate.c
@@ -90,7 +90,7 @@ static int bch2_dev_usrdata_drop(struct bch_fs *c, unsigned dev_idx, int flags)
 
 		ret = for_each_btree_key_commit(trans, iter, id, POS_MIN,
 				BTREE_ITER_PREFETCH|BTREE_ITER_ALL_SNAPSHOTS, k,
-				NULL, NULL, BTREE_INSERT_NOFAIL,
+				NULL, NULL, BCH_TRANS_COMMIT_no_enospc,
 			bch2_dev_usrdata_drop_key(trans, &iter, k, dev_idx, flags));
 		if (ret)
 			break;
diff --git a/fs/bcachefs/move.c b/fs/bcachefs/move.c
index ab749bf2fcbc..cf36f2b0738f 100644
--- a/fs/bcachefs/move.c
+++ b/fs/bcachefs/move.c
@@ -27,6 +27,13 @@
 #include <linux/ioprio.h>
 #include <linux/kthread.h>
 
+const char * const bch2_data_ops_strs[] = {
+#define x(t, n, ...) [n] = #t,
+	BCH_DATA_OPS()
+#undef x
+	NULL
+};
+
 static void trace_move_extent2(struct bch_fs *c, struct bkey_s_c k)
 {
 	if (trace_move_extent_enabled()) {
@@ -163,12 +170,17 @@ void bch2_move_ctxt_wait_for_io(struct moving_context *ctxt)
 		atomic_read(&ctxt->write_sectors) != sectors_pending);
 }
 
+static void bch2_moving_ctxt_flush_all(struct moving_context *ctxt)
+{
+	move_ctxt_wait_event(ctxt, list_empty(&ctxt->reads));
+	closure_sync(&ctxt->cl);
+}
+
 void bch2_moving_ctxt_exit(struct moving_context *ctxt)
 {
 	struct bch_fs *c = ctxt->trans->c;
 
-	move_ctxt_wait_event(ctxt, list_empty(&ctxt->reads));
-	closure_sync(&ctxt->cl);
+	bch2_moving_ctxt_flush_all(ctxt);
 
 	EBUG_ON(atomic_read(&ctxt->write_sectors));
 	EBUG_ON(atomic_read(&ctxt->write_ios));
@@ -216,7 +228,7 @@ void bch2_move_stats_exit(struct bch_move_stats *stats, struct bch_fs *c)
 	trace_move_data(c, stats);
 }
 
-void bch2_move_stats_init(struct bch_move_stats *stats, char *name)
+void bch2_move_stats_init(struct bch_move_stats *stats, const char *name)
 {
 	memset(stats, 0, sizeof(*stats));
 	stats->data_type = BCH_DATA_user;
@@ -263,7 +275,7 @@ static int bch2_extent_drop_ptrs(struct btree_trans *trans,
 
 	return bch2_trans_relock(trans) ?:
 		bch2_trans_update(trans, iter, n, BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE) ?:
-		bch2_trans_commit(trans, NULL, NULL, BTREE_INSERT_NOFAIL);
+		bch2_trans_commit(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc);
 }
 
 int bch2_move_extent(struct moving_context *ctxt,
@@ -484,8 +496,8 @@ int bch2_move_ratelimit(struct moving_context *ctxt)
 	struct bch_fs *c = ctxt->trans->c;
 	u64 delay;
 
-	if (ctxt->wait_on_copygc && !c->copygc_running) {
-		bch2_trans_unlock_long(ctxt->trans);
+	if (ctxt->wait_on_copygc && c->copygc_running) {
+		bch2_moving_ctxt_flush_all(ctxt);
 		wait_event_killable(c->copygc_running_wq,
 				    !c->copygc_running ||
 				    kthread_should_stop());
@@ -503,7 +515,7 @@ int bch2_move_ratelimit(struct moving_context *ctxt)
 			set_current_state(TASK_INTERRUPTIBLE);
 		}
 
-		if ((current->flags & PF_KTHREAD) && kthread_should_stop()) {
+		if (kthread_should_stop()) {
 			__set_current_state(TASK_RUNNING);
 			return 1;
 		}
@@ -512,7 +524,7 @@ int bch2_move_ratelimit(struct moving_context *ctxt)
 			schedule_timeout(delay);
 
 		if (unlikely(freezing(current))) {
-			move_ctxt_wait_event(ctxt, list_empty(&ctxt->reads));
+			bch2_moving_ctxt_flush_all(ctxt);
 			try_to_freeze();
 		}
 	} while (delay);
@@ -721,11 +733,10 @@ int __bch2_evacuate_bucket(struct moving_context *ctxt,
 	bucket_size = bch_dev_bkey_exists(c, bucket.inode)->mi.bucket_size;
 	fragmentation = a->fragmentation_lru;
 
-	ret = bch2_btree_write_buffer_flush(trans);
-	if (ret) {
-		bch_err_msg(c, ret, "flushing btree write buffer");
+	ret = bch2_btree_write_buffer_tryflush(trans);
+	bch_err_msg(c, ret, "flushing btree write buffer");
+	if (ret)
 		goto err;
-	}
 
 	while (!(ret = bch2_move_ratelimit(ctxt))) {
 		bch2_trans_begin(trans);
@@ -856,18 +867,17 @@ typedef bool (*move_btree_pred)(struct bch_fs *, void *,
 				struct data_update_opts *);
 
 static int bch2_move_btree(struct bch_fs *c,
-			   enum btree_id start_btree_id, struct bpos start_pos,
-			   enum btree_id end_btree_id,   struct bpos end_pos,
+			   struct bbpos start,
+			   struct bbpos end,
 			   move_btree_pred pred, void *arg,
 			   struct bch_move_stats *stats)
 {
-	bool kthread = (current->flags & PF_KTHREAD) != 0;
 	struct bch_io_opts io_opts = bch2_opts_to_inode_opts(c->opts);
 	struct moving_context ctxt;
 	struct btree_trans *trans;
 	struct btree_iter iter;
 	struct btree *b;
-	enum btree_id id;
+	enum btree_id btree;
 	struct data_update_opts data_opts;
 	int ret = 0;
 
@@ -878,26 +888,26 @@ static int bch2_move_btree(struct bch_fs *c,
 
 	stats->data_type = BCH_DATA_btree;
 
-	for (id = start_btree_id;
-	     id <= min_t(unsigned, end_btree_id, btree_id_nr_alive(c) - 1);
-	     id++) {
-		stats->pos = BBPOS(id, POS_MIN);
+	for (btree = start.btree;
+	     btree <= min_t(unsigned, end.btree, btree_id_nr_alive(c) - 1);
+	     btree ++) {
+		stats->pos = BBPOS(btree, POS_MIN);
 
-		if (!bch2_btree_id_root(c, id)->b)
+		if (!bch2_btree_id_root(c, btree)->b)
 			continue;
 
-		bch2_trans_node_iter_init(trans, &iter, id, POS_MIN, 0, 0,
+		bch2_trans_node_iter_init(trans, &iter, btree, POS_MIN, 0, 0,
 					  BTREE_ITER_PREFETCH);
 retry:
 		ret = 0;
 		while (bch2_trans_begin(trans),
 		       (b = bch2_btree_iter_peek_node(&iter)) &&
 		       !(ret = PTR_ERR_OR_ZERO(b))) {
-			if (kthread && kthread_should_stop())
+			if (kthread_should_stop())
 				break;
 
-			if ((cmp_int(id, end_btree_id) ?:
-			     bpos_cmp(b->key.k.p, end_pos)) > 0)
+			if ((cmp_int(btree, end.btree) ?:
+			     bpos_cmp(b->key.k.p, end.pos)) > 0)
 				break;
 
 			stats->pos = BBPOS(iter.btree_id, iter.pos);
@@ -918,7 +928,7 @@ static int bch2_move_btree(struct bch_fs *c,
 
 		bch2_trans_iter_exit(trans, &iter);
 
-		if (kthread && kthread_should_stop())
+		if (kthread_should_stop())
 			break;
 	}
 
@@ -1034,8 +1044,8 @@ int bch2_scan_old_btree_nodes(struct bch_fs *c, struct bch_move_stats *stats)
 	int ret;
 
 	ret = bch2_move_btree(c,
-			      0,		POS_MIN,
-			      BTREE_ID_NR,	SPOS_MAX,
+			      BBPOS_MIN,
+			      BBPOS_MAX,
 			      rewrite_old_nodes_pred, c, stats);
 	if (!ret) {
 		mutex_lock(&c->sb_lock);
@@ -1050,71 +1060,101 @@ int bch2_scan_old_btree_nodes(struct bch_fs *c, struct bch_move_stats *stats)
 	return ret;
 }
 
+static bool drop_extra_replicas_pred(struct bch_fs *c, void *arg,
+			     struct bkey_s_c k,
+			     struct bch_io_opts *io_opts,
+			     struct data_update_opts *data_opts)
+{
+	unsigned durability = bch2_bkey_durability(c, k);
+	unsigned replicas = bkey_is_btree_ptr(k.k)
+		? c->opts.metadata_replicas
+		: io_opts->data_replicas;
+	const union bch_extent_entry *entry;
+	struct extent_ptr_decoded p;
+	unsigned i = 0;
+
+	bkey_for_each_ptr_decode(k.k, bch2_bkey_ptrs_c(k), p, entry) {
+		unsigned d = bch2_extent_ptr_durability(c, &p);
+
+		if (d && durability - d >= replicas) {
+			data_opts->kill_ptrs |= BIT(i);
+			durability -= d;
+		}
+
+		i++;
+	}
+
+	return data_opts->kill_ptrs != 0;
+}
+
+static bool drop_extra_replicas_btree_pred(struct bch_fs *c, void *arg,
+				   struct btree *b,
+				   struct bch_io_opts *io_opts,
+				   struct data_update_opts *data_opts)
+{
+	return drop_extra_replicas_pred(c, arg, bkey_i_to_s_c(&b->key), io_opts, data_opts);
+}
+
 int bch2_data_job(struct bch_fs *c,
 		  struct bch_move_stats *stats,
 		  struct bch_ioctl_data op)
 {
+	struct bbpos start	= BBPOS(op.start_btree, op.start_pos);
+	struct bbpos end	= BBPOS(op.end_btree, op.end_pos);
 	int ret = 0;
 
+	if (op.op >= BCH_DATA_OP_NR)
+		return -EINVAL;
+
+	bch2_move_stats_init(stats, bch2_data_ops_strs[op.op]);
+
 	switch (op.op) {
-	case BCH_DATA_OP_REREPLICATE:
-		bch2_move_stats_init(stats, "rereplicate");
+	case BCH_DATA_OP_rereplicate:
 		stats->data_type = BCH_DATA_journal;
 		ret = bch2_journal_flush_device_pins(&c->journal, -1);
-
-		ret = bch2_move_btree(c,
-				      op.start_btree,	op.start_pos,
-				      op.end_btree,	op.end_pos,
+		ret = bch2_move_btree(c, start, end,
 				      rereplicate_btree_pred, c, stats) ?: ret;
-		ret = bch2_replicas_gc2(c) ?: ret;
-
-		ret = bch2_move_data(c,
-				     (struct bbpos) { op.start_btree,	op.start_pos },
-				     (struct bbpos) { op.end_btree,	op.end_pos },
+		ret = bch2_move_data(c, start, end,
 				     NULL,
 				     stats,
 				     writepoint_hashed((unsigned long) current),
 				     true,
 				     rereplicate_pred, c) ?: ret;
 		ret = bch2_replicas_gc2(c) ?: ret;
-
-		bch2_move_stats_exit(stats, c);
 		break;
-	case BCH_DATA_OP_MIGRATE:
+	case BCH_DATA_OP_migrate:
 		if (op.migrate.dev >= c->sb.nr_devices)
 			return -EINVAL;
 
-		bch2_move_stats_init(stats, "migrate");
 		stats->data_type = BCH_DATA_journal;
 		ret = bch2_journal_flush_device_pins(&c->journal, op.migrate.dev);
-
-		ret = bch2_move_btree(c,
-				      op.start_btree,	op.start_pos,
-				      op.end_btree,	op.end_pos,
+		ret = bch2_move_btree(c, start, end,
 				      migrate_btree_pred, &op, stats) ?: ret;
-		ret = bch2_replicas_gc2(c) ?: ret;
-
-		ret = bch2_move_data(c,
-				     (struct bbpos) { op.start_btree,	op.start_pos },
-				     (struct bbpos) { op.end_btree,	op.end_pos },
+		ret = bch2_move_data(c, start, end,
 				     NULL,
 				     stats,
 				     writepoint_hashed((unsigned long) current),
 				     true,
 				     migrate_pred, &op) ?: ret;
 		ret = bch2_replicas_gc2(c) ?: ret;
-
-		bch2_move_stats_exit(stats, c);
 		break;
-	case BCH_DATA_OP_REWRITE_OLD_NODES:
-		bch2_move_stats_init(stats, "rewrite_old_nodes");
+	case BCH_DATA_OP_rewrite_old_nodes:
 		ret = bch2_scan_old_btree_nodes(c, stats);
-		bch2_move_stats_exit(stats, c);
+		break;
+	case BCH_DATA_OP_drop_extra_replicas:
+		ret = bch2_move_btree(c, start, end,
+				drop_extra_replicas_btree_pred, c, stats) ?: ret;
+		ret = bch2_move_data(c, start, end, NULL, stats,
+				writepoint_hashed((unsigned long) current),
+				true,
+				drop_extra_replicas_pred, c) ?: ret;
+		ret = bch2_replicas_gc2(c) ?: ret;
 		break;
 	default:
 		ret = -EINVAL;
 	}
 
+	bch2_move_stats_exit(stats, c);
 	return ret;
 }
 
diff --git a/fs/bcachefs/move.h b/fs/bcachefs/move.h
index 07cf9d42643b..cedde6ee99d0 100644
--- a/fs/bcachefs/move.h
+++ b/fs/bcachefs/move.h
@@ -56,6 +56,8 @@ do {									\
 typedef bool (*move_pred_fn)(struct bch_fs *, void *, struct bkey_s_c,
 			     struct bch_io_opts *, struct data_update_opts *);
 
+extern const char * const bch2_data_ops_strs[];
+
 void bch2_moving_ctxt_exit(struct moving_context *);
 void bch2_moving_ctxt_init(struct moving_context *, struct bch_fs *,
 			   struct bch_ratelimit *, struct bch_move_stats *,
@@ -130,7 +132,7 @@ int bch2_data_job(struct bch_fs *,
 
 void bch2_move_stats_to_text(struct printbuf *, struct bch_move_stats *);
 void bch2_move_stats_exit(struct bch_move_stats *, struct bch_fs *);
-void bch2_move_stats_init(struct bch_move_stats *, char *);
+void bch2_move_stats_init(struct bch_move_stats *, const char *);
 
 void bch2_fs_moving_ctxts_to_text(struct printbuf *, struct bch_fs *);
 
diff --git a/fs/bcachefs/movinggc.c b/fs/bcachefs/movinggc.c
index 0a0576326c5b..e884324bd2fa 100644
--- a/fs/bcachefs/movinggc.c
+++ b/fs/bcachefs/movinggc.c
@@ -153,8 +153,11 @@ static int bch2_copygc_get_buckets(struct moving_context *ctxt,
 
 	move_buckets_wait(ctxt, buckets_in_flight, false);
 
-	ret = bch2_btree_write_buffer_flush(trans);
-	if (bch2_fs_fatal_err_on(ret, c, "%s: error %s from bch2_btree_write_buffer_flush()",
+	ret = bch2_btree_write_buffer_tryflush(trans);
+	if (bch2_err_matches(ret, EROFS))
+		return ret;
+
+	if (bch2_fs_fatal_err_on(ret, c, "%s: error %s from bch2_btree_write_buffer_tryflush()",
 				 __func__, bch2_err_str(ret)))
 		return ret;
 
diff --git a/fs/bcachefs/opts.h b/fs/bcachefs/opts.h
index 8526f177450a..b7f9990c5848 100644
--- a/fs/bcachefs/opts.h
+++ b/fs/bcachefs/opts.h
@@ -233,11 +233,6 @@ enum fsck_err_opts {
 	  OPT_BOOL(),							\
 	  BCH2_NO_SB_OPT,		true,				\
 	  NULL,		"Stash pointer to in memory btree node in btree ptr")\
-	x(btree_write_buffer_size, u32,					\
-	  OPT_FS|OPT_MOUNT,						\
-	  OPT_UINT(16, (1U << 20) - 1),					\
-	  BCH2_NO_SB_OPT,		1U << 13,			\
-	  NULL,		"Number of btree write buffer entries")		\
 	x(gc_reserve_percent,		u8,				\
 	  OPT_FS|OPT_FORMAT|OPT_MOUNT|OPT_RUNTIME,			\
 	  OPT_UINT(5, 21),						\
diff --git a/fs/bcachefs/rebalance.c b/fs/bcachefs/rebalance.c
index 3319190b8d9c..db2139c0545d 100644
--- a/fs/bcachefs/rebalance.c
+++ b/fs/bcachefs/rebalance.c
@@ -69,7 +69,7 @@ static int __bch2_set_rebalance_needs_scan(struct btree_trans *trans, u64 inum)
 
 int bch2_set_rebalance_needs_scan(struct bch_fs *c, u64 inum)
 {
-	int ret = bch2_trans_do(c, NULL, NULL, BTREE_INSERT_NOFAIL|BTREE_INSERT_LAZY_RW,
+	int ret = bch2_trans_do(c, NULL, NULL, BCH_TRANS_COMMIT_no_enospc|BCH_TRANS_COMMIT_lazy_rw,
 			    __bch2_set_rebalance_needs_scan(trans, inum));
 	rebalance_wakeup(c);
 	return ret;
@@ -125,7 +125,7 @@ static int bch2_bkey_clear_needs_rebalance(struct btree_trans *trans,
 
 	extent_entry_drop(bkey_i_to_s(n),
 			  (void *) bch2_bkey_rebalance_opts(bkey_i_to_s_c(n)));
-	return bch2_trans_commit(trans, NULL, NULL, BTREE_INSERT_NOFAIL);
+	return bch2_trans_commit(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc);
 }
 
 static struct bkey_s_c next_rebalance_extent(struct btree_trans *trans,
@@ -273,7 +273,7 @@ static int do_rebalance_scan(struct moving_context *ctxt, u64 inum, u64 cookie)
 	r->state = BCH_REBALANCE_scanning;
 
 	ret = __bch2_move_data(ctxt, r->scan_start, r->scan_end, rebalance_pred, NULL) ?:
-		commit_do(trans, NULL, NULL, BTREE_INSERT_NOFAIL,
+		commit_do(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc,
 			  bch2_clear_rebalance_needs_scan(trans, inum, cookie));
 
 	bch2_move_stats_exit(&r->scan_stats, trans->c);
diff --git a/fs/bcachefs/recovery.c b/fs/bcachefs/recovery.c
index 9c30500ce920..a94f2b5ed055 100644
--- a/fs/bcachefs/recovery.c
+++ b/fs/bcachefs/recovery.c
@@ -98,6 +98,11 @@ static int bch2_journal_replay_key(struct btree_trans *trans,
 	unsigned update_flags = BTREE_TRIGGER_NORUN;
 	int ret;
 
+	if (k->overwritten)
+		return 0;
+
+	trans->journal_res.seq = k->journal_seq;
+
 	/*
 	 * BTREE_UPDATE_KEY_CACHE_RECLAIM disables key cache lookup/update to
 	 * keep the key cache coherent with the underlying btree. Nothing
@@ -139,27 +144,14 @@ static int journal_sort_seq_cmp(const void *_l, const void *_r)
 static int bch2_journal_replay(struct bch_fs *c)
 {
 	struct journal_keys *keys = &c->journal_keys;
-	struct journal_key **keys_sorted, *k;
+	DARRAY(struct journal_key *) keys_sorted = { 0 };
+	struct journal_key **kp;
 	struct journal *j = &c->journal;
 	u64 start_seq	= c->journal_replay_seq_start;
 	u64 end_seq	= c->journal_replay_seq_start;
-	size_t i;
+	struct btree_trans *trans = bch2_trans_get(c);
 	int ret;
 
-	move_gap(keys->d, keys->nr, keys->size, keys->gap, keys->nr);
-	keys->gap = keys->nr;
-
-	keys_sorted = kvmalloc_array(keys->nr, sizeof(*keys_sorted), GFP_KERNEL);
-	if (!keys_sorted)
-		return -BCH_ERR_ENOMEM_journal_replay;
-
-	for (i = 0; i < keys->nr; i++)
-		keys_sorted[i] = &keys->d[i];
-
-	sort(keys_sorted, keys->nr,
-	     sizeof(keys_sorted[0]),
-	     journal_sort_seq_cmp, NULL);
-
 	if (keys->nr) {
 		ret = bch2_journal_log_msg(c, "Starting journal replay (%zu keys in entries %llu-%llu)",
 					   keys->nr, start_seq, end_seq);
@@ -167,41 +159,82 @@ static int bch2_journal_replay(struct bch_fs *c)
 			goto err;
 	}
 
-	for (i = 0; i < keys->nr; i++) {
-		k = keys_sorted[i];
+	BUG_ON(!atomic_read(&keys->ref));
 
+	/*
+	 * First, attempt to replay keys in sorted order. This is more
+	 * efficient - better locality of btree access -  but some might fail if
+	 * that would cause a journal deadlock.
+	 */
+	for (size_t i = 0; i < keys->nr; i++) {
 		cond_resched();
 
-		replay_now_at(j, k->journal_seq);
+		struct journal_key *k = keys->d + i;
 
-		ret = bch2_trans_do(c, NULL, NULL,
-				    BTREE_INSERT_LAZY_RW|
-				    BTREE_INSERT_NOFAIL|
-				    (!k->allocated
-				     ? BTREE_INSERT_JOURNAL_REPLAY|BCH_WATERMARK_reclaim
-				     : 0),
+		ret = commit_do(trans, NULL, NULL,
+				BCH_TRANS_COMMIT_no_enospc|
+				BCH_TRANS_COMMIT_journal_reclaim|
+				(!k->allocated ? BCH_TRANS_COMMIT_no_journal_res : 0),
 			     bch2_journal_replay_key(trans, k));
+		BUG_ON(!ret && !k->overwritten);
 		if (ret) {
-			bch_err(c, "journal replay: error while replaying key at btree %s level %u: %s",
-				bch2_btree_id_str(k->btree_id), k->level, bch2_err_str(ret));
-			goto err;
+			ret = darray_push(&keys_sorted, k);
+			if (ret)
+				goto err;
 		}
 	}
 
+	/*
+	 * Now, replay any remaining keys in the order in which they appear in
+	 * the journal, unpinning those journal entries as we go:
+	 */
+	sort(keys_sorted.data, keys_sorted.nr,
+	     sizeof(keys_sorted.data[0]),
+	     journal_sort_seq_cmp, NULL);
+
+	darray_for_each(keys_sorted, kp) {
+		cond_resched();
+
+		struct journal_key *k = *kp;
+
+		replay_now_at(j, k->journal_seq);
+
+		ret = commit_do(trans, NULL, NULL,
+				BCH_TRANS_COMMIT_no_enospc|
+				(!k->allocated
+				 ? BCH_TRANS_COMMIT_no_journal_res|BCH_WATERMARK_reclaim
+				 : 0),
+			     bch2_journal_replay_key(trans, k));
+		bch_err_msg(c, ret, "while replaying key at btree %s level %u:",
+			    bch2_btree_id_str(k->btree_id), k->level);
+		if (ret)
+			goto err;
+
+		BUG_ON(!k->overwritten);
+	}
+
+	/*
+	 * We need to put our btree_trans before calling flush_all_pins(), since
+	 * that will use a btree_trans internally
+	 */
+	bch2_trans_put(trans);
+	trans = NULL;
+
+	if (!c->opts.keep_journal)
+		bch2_journal_keys_put_initial(c);
+
 	replay_now_at(j, j->replay_journal_seq_end);
 	j->replay_journal_seq = 0;
 
 	bch2_journal_set_replay_done(j);
-	bch2_journal_flush_all_pins(j);
-	ret = bch2_journal_error(j);
 
-	if (keys->nr && !ret)
+	if (keys->nr)
 		bch2_journal_log_msg(c, "journal replay finished");
 err:
-	kvfree(keys_sorted);
-
-	if (ret)
-		bch_err_fn(c, ret);
+	if (trans)
+		bch2_trans_put(trans);
+	darray_exit(&keys_sorted);
+	bch_err_fn(c, ret);
 	return ret;
 }
 
@@ -468,7 +501,7 @@ static int __bch2_fs_upgrade_for_subvolumes(struct btree_trans *trans)
 noinline_for_stack
 static int bch2_fs_upgrade_for_subvolumes(struct bch_fs *c)
 {
-	int ret = bch2_trans_do(c, NULL, NULL, BTREE_INSERT_LAZY_RW,
+	int ret = bch2_trans_do(c, NULL, NULL, BCH_TRANS_COMMIT_lazy_rw,
 				__bch2_fs_upgrade_for_subvolumes(trans));
 	if (ret)
 		bch_err_fn(c, ret);
@@ -489,7 +522,19 @@ static int bch2_check_allocations(struct bch_fs *c)
 
 static int bch2_set_may_go_rw(struct bch_fs *c)
 {
+	struct journal_keys *keys = &c->journal_keys;
+
+	/*
+	 * After we go RW, the journal keys buffer can't be modified (except for
+	 * setting journal_key->overwritten: it will be accessed by multiple
+	 * threads
+	 */
+	move_gap(keys->d, keys->nr, keys->size, keys->gap, keys->nr);
+	keys->gap = keys->nr;
+
 	set_bit(BCH_FS_MAY_GO_RW, &c->flags);
+	if (keys->nr)
+		return bch2_fs_read_write_early(c);
 	return 0;
 }
 
@@ -836,6 +881,8 @@ int bch2_fs_recovery(struct bch_fs *c)
 	    test_bit(BCH_FS_ERRORS_FIXED, &c->flags) &&
 	    !test_bit(BCH_FS_ERRORS_NOT_FIXED, &c->flags) &&
 	    !test_bit(BCH_FS_ERROR, &c->flags)) {
+		bch2_flush_fsck_errs(c);
+
 		bch_info(c, "Fixed errors, running fsck a second time to verify fs is clean");
 		clear_bit(BCH_FS_ERRORS_FIXED, &c->flags);
 
@@ -891,8 +938,12 @@ int bch2_fs_recovery(struct bch_fs *c)
 
 		bch2_move_stats_init(&stats, "recovery");
 
-		bch_info(c, "scanning for old btree nodes");
-		ret =   bch2_fs_read_write(c) ?:
+		struct printbuf buf = PRINTBUF;
+		bch2_version_to_text(&buf, c->sb.version_min);
+		bch_info(c, "scanning for old btree nodes: min_version %s", buf.buf);
+		printbuf_exit(&buf);
+
+		ret =   bch2_fs_read_write_early(c) ?:
 			bch2_scan_old_btree_nodes(c, &stats);
 		if (ret)
 			goto err;
@@ -909,10 +960,8 @@ int bch2_fs_recovery(struct bch_fs *c)
 	bch2_flush_fsck_errs(c);
 
 	if (!c->opts.keep_journal &&
-	    test_bit(JOURNAL_REPLAY_DONE, &c->journal.flags)) {
-		bch2_journal_keys_free(&c->journal_keys);
-		bch2_journal_entries_free(c);
-	}
+	    test_bit(JOURNAL_REPLAY_DONE, &c->journal.flags))
+		bch2_journal_keys_put_initial(c);
 	kfree(clean);
 
 	if (!ret && test_bit(BCH_FS_NEED_DELETE_DEAD_SNAPSHOTS, &c->flags)) {
diff --git a/fs/bcachefs/reflink.c b/fs/bcachefs/reflink.c
index 6e1bfe9feb59..07ddf3e85ee4 100644
--- a/fs/bcachefs/reflink.c
+++ b/fs/bcachefs/reflink.c
@@ -390,7 +390,7 @@ s64 bch2_remap_range(struct bch_fs *c,
 			inode_u.bi_size = new_i_size;
 			ret2  = bch2_inode_write(trans, &inode_iter, &inode_u) ?:
 				bch2_trans_commit(trans, NULL, NULL,
-						  BTREE_INSERT_NOFAIL);
+						  BCH_TRANS_COMMIT_no_enospc);
 		}
 
 		bch2_trans_iter_exit(trans, &inode_iter);
diff --git a/fs/bcachefs/replicas.c b/fs/bcachefs/replicas.c
index 1c3ae13bfced..820f99898a16 100644
--- a/fs/bcachefs/replicas.c
+++ b/fs/bcachefs/replicas.c
@@ -11,7 +11,7 @@ static int bch2_cpu_replicas_to_sb_replicas(struct bch_fs *,
 
 /* Replicas tracking - in memory: */
 
-static void verify_replicas_entry(struct bch_replicas_entry *e)
+static void verify_replicas_entry(struct bch_replicas_entry_v1 *e)
 {
 #ifdef CONFIG_BCACHEFS_DEBUG
 	unsigned i;
@@ -26,7 +26,7 @@ static void verify_replicas_entry(struct bch_replicas_entry *e)
 #endif
 }
 
-void bch2_replicas_entry_sort(struct bch_replicas_entry *e)
+void bch2_replicas_entry_sort(struct bch_replicas_entry_v1 *e)
 {
 	bubble_sort(e->devs, e->nr_devs, u8_cmp);
 }
@@ -53,7 +53,7 @@ static void bch2_replicas_entry_v0_to_text(struct printbuf *out,
 }
 
 void bch2_replicas_entry_to_text(struct printbuf *out,
-				 struct bch_replicas_entry *e)
+				 struct bch_replicas_entry_v1 *e)
 {
 	unsigned i;
 
@@ -71,7 +71,7 @@ void bch2_replicas_entry_to_text(struct printbuf *out,
 void bch2_cpu_replicas_to_text(struct printbuf *out,
 			       struct bch_replicas_cpu *r)
 {
-	struct bch_replicas_entry *e;
+	struct bch_replicas_entry_v1 *e;
 	bool first = true;
 
 	for_each_cpu_replicas_entry(r, e) {
@@ -84,7 +84,7 @@ void bch2_cpu_replicas_to_text(struct printbuf *out,
 }
 
 static void extent_to_replicas(struct bkey_s_c k,
-			       struct bch_replicas_entry *r)
+			       struct bch_replicas_entry_v1 *r)
 {
 	struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
 	const union bch_extent_entry *entry;
@@ -104,7 +104,7 @@ static void extent_to_replicas(struct bkey_s_c k,
 }
 
 static void stripe_to_replicas(struct bkey_s_c k,
-			       struct bch_replicas_entry *r)
+			       struct bch_replicas_entry_v1 *r)
 {
 	struct bkey_s_c_stripe s = bkey_s_c_to_stripe(k);
 	const struct bch_extent_ptr *ptr;
@@ -117,7 +117,7 @@ static void stripe_to_replicas(struct bkey_s_c k,
 		r->devs[r->nr_devs++] = ptr->dev;
 }
 
-void bch2_bkey_to_replicas(struct bch_replicas_entry *e,
+void bch2_bkey_to_replicas(struct bch_replicas_entry_v1 *e,
 			   struct bkey_s_c k)
 {
 	e->nr_devs = 0;
@@ -142,7 +142,7 @@ void bch2_bkey_to_replicas(struct bch_replicas_entry *e,
 	bch2_replicas_entry_sort(e);
 }
 
-void bch2_devlist_to_replicas(struct bch_replicas_entry *e,
+void bch2_devlist_to_replicas(struct bch_replicas_entry_v1 *e,
 			      enum bch_data_type data_type,
 			      struct bch_devs_list devs)
 {
@@ -164,7 +164,7 @@ void bch2_devlist_to_replicas(struct bch_replicas_entry *e,
 
 static struct bch_replicas_cpu
 cpu_replicas_add_entry(struct bch_replicas_cpu *old,
-		       struct bch_replicas_entry *new_entry)
+		       struct bch_replicas_entry_v1 *new_entry)
 {
 	unsigned i;
 	struct bch_replicas_cpu new = {
@@ -194,7 +194,7 @@ cpu_replicas_add_entry(struct bch_replicas_cpu *old,
 }
 
 static inline int __replicas_entry_idx(struct bch_replicas_cpu *r,
-				       struct bch_replicas_entry *search)
+				       struct bch_replicas_entry_v1 *search)
 {
 	int idx, entry_size = replicas_entry_bytes(search);
 
@@ -212,7 +212,7 @@ static inline int __replicas_entry_idx(struct bch_replicas_cpu *r,
 }
 
 int bch2_replicas_entry_idx(struct bch_fs *c,
-			    struct bch_replicas_entry *search)
+			    struct bch_replicas_entry_v1 *search)
 {
 	bch2_replicas_entry_sort(search);
 
@@ -220,13 +220,13 @@ int bch2_replicas_entry_idx(struct bch_fs *c,
 }
 
 static bool __replicas_has_entry(struct bch_replicas_cpu *r,
-				 struct bch_replicas_entry *search)
+				 struct bch_replicas_entry_v1 *search)
 {
 	return __replicas_entry_idx(r, search) >= 0;
 }
 
 bool bch2_replicas_marked(struct bch_fs *c,
-			  struct bch_replicas_entry *search)
+			  struct bch_replicas_entry_v1 *search)
 {
 	bool marked;
 
@@ -343,7 +343,7 @@ static int replicas_table_update(struct bch_fs *c,
 static unsigned reserve_journal_replicas(struct bch_fs *c,
 				     struct bch_replicas_cpu *r)
 {
-	struct bch_replicas_entry *e;
+	struct bch_replicas_entry_v1 *e;
 	unsigned journal_res_u64s = 0;
 
 	/* nr_inodes: */
@@ -368,7 +368,7 @@ static unsigned reserve_journal_replicas(struct bch_fs *c,
 
 noinline
 static int bch2_mark_replicas_slowpath(struct bch_fs *c,
-				struct bch_replicas_entry *new_entry)
+				struct bch_replicas_entry_v1 *new_entry)
 {
 	struct bch_replicas_cpu new_r, new_gc;
 	int ret = 0;
@@ -433,7 +433,7 @@ static int bch2_mark_replicas_slowpath(struct bch_fs *c,
 	goto out;
 }
 
-int bch2_mark_replicas(struct bch_fs *c, struct bch_replicas_entry *r)
+int bch2_mark_replicas(struct bch_fs *c, struct bch_replicas_entry_v1 *r)
 {
 	return likely(bch2_replicas_marked(c, r))
 		? 0 : bch2_mark_replicas_slowpath(c, r);
@@ -484,7 +484,7 @@ int bch2_replicas_gc_end(struct bch_fs *c, int ret)
 
 int bch2_replicas_gc_start(struct bch_fs *c, unsigned typemask)
 {
-	struct bch_replicas_entry *e;
+	struct bch_replicas_entry_v1 *e;
 	unsigned i = 0;
 
 	lockdep_assert_held(&c->replicas_gc_lock);
@@ -559,7 +559,7 @@ int bch2_replicas_gc2(struct bch_fs *c)
 	}
 
 	for (i = 0; i < c->replicas.nr; i++) {
-		struct bch_replicas_entry *e =
+		struct bch_replicas_entry_v1 *e =
 			cpu_replicas_entry(&c->replicas, i);
 
 		if (e->data_type == BCH_DATA_journal ||
@@ -590,7 +590,7 @@ int bch2_replicas_gc2(struct bch_fs *c)
 }
 
 int bch2_replicas_set_usage(struct bch_fs *c,
-			    struct bch_replicas_entry *r,
+			    struct bch_replicas_entry_v1 *r,
 			    u64 sectors)
 {
 	int ret, idx = bch2_replicas_entry_idx(c, r);
@@ -623,7 +623,7 @@ static int
 __bch2_sb_replicas_to_cpu_replicas(struct bch_sb_field_replicas *sb_r,
 				   struct bch_replicas_cpu *cpu_r)
 {
-	struct bch_replicas_entry *e, *dst;
+	struct bch_replicas_entry_v1 *e, *dst;
 	unsigned nr = 0, entry_size = 0, idx = 0;
 
 	for_each_replicas_entry(sb_r, e) {
@@ -661,7 +661,7 @@ __bch2_sb_replicas_v0_to_cpu_replicas(struct bch_sb_field_replicas_v0 *sb_r,
 		nr++;
 	}
 
-	entry_size += sizeof(struct bch_replicas_entry) -
+	entry_size += sizeof(struct bch_replicas_entry_v1) -
 		sizeof(struct bch_replicas_entry_v0);
 
 	cpu_r->entries = kcalloc(nr, entry_size, GFP_KERNEL);
@@ -672,7 +672,7 @@ __bch2_sb_replicas_v0_to_cpu_replicas(struct bch_sb_field_replicas_v0 *sb_r,
 	cpu_r->entry_size	= entry_size;
 
 	for_each_replicas_entry(sb_r, e) {
-		struct bch_replicas_entry *dst =
+		struct bch_replicas_entry_v1 *dst =
 			cpu_replicas_entry(cpu_r, idx++);
 
 		dst->data_type	= e->data_type;
@@ -716,7 +716,7 @@ static int bch2_cpu_replicas_to_sb_replicas_v0(struct bch_fs *c,
 {
 	struct bch_sb_field_replicas_v0 *sb_r;
 	struct bch_replicas_entry_v0 *dst;
-	struct bch_replicas_entry *src;
+	struct bch_replicas_entry_v1 *src;
 	size_t bytes;
 
 	bytes = sizeof(struct bch_sb_field_replicas);
@@ -754,7 +754,7 @@ static int bch2_cpu_replicas_to_sb_replicas(struct bch_fs *c,
 					    struct bch_replicas_cpu *r)
 {
 	struct bch_sb_field_replicas *sb_r;
-	struct bch_replicas_entry *dst, *src;
+	struct bch_replicas_entry_v1 *dst, *src;
 	bool need_v1 = false;
 	size_t bytes;
 
@@ -805,7 +805,7 @@ static int bch2_cpu_replicas_validate(struct bch_replicas_cpu *cpu_r,
 		      memcmp, NULL);
 
 	for (i = 0; i < cpu_r->nr; i++) {
-		struct bch_replicas_entry *e =
+		struct bch_replicas_entry_v1 *e =
 			cpu_replicas_entry(cpu_r, i);
 
 		if (e->data_type >= BCH_DATA_NR) {
@@ -835,7 +835,7 @@ static int bch2_cpu_replicas_validate(struct bch_replicas_cpu *cpu_r,
 			}
 
 		if (i + 1 < cpu_r->nr) {
-			struct bch_replicas_entry *n =
+			struct bch_replicas_entry_v1 *n =
 				cpu_replicas_entry(cpu_r, i + 1);
 
 			BUG_ON(memcmp(e, n, cpu_r->entry_size) > 0);
@@ -872,7 +872,7 @@ static void bch2_sb_replicas_to_text(struct printbuf *out,
 				     struct bch_sb_field *f)
 {
 	struct bch_sb_field_replicas *r = field_to_type(f, replicas);
-	struct bch_replicas_entry *e;
+	struct bch_replicas_entry_v1 *e;
 	bool first = true;
 
 	for_each_replicas_entry(r, e) {
@@ -934,7 +934,7 @@ const struct bch_sb_field_ops bch_sb_field_ops_replicas_v0 = {
 bool bch2_have_enough_devs(struct bch_fs *c, struct bch_devs_mask devs,
 			   unsigned flags, bool print)
 {
-	struct bch_replicas_entry *e;
+	struct bch_replicas_entry_v1 *e;
 	bool ret = true;
 
 	percpu_down_read(&c->mark_lock);
@@ -994,7 +994,7 @@ unsigned bch2_sb_dev_has_data(struct bch_sb *sb, unsigned dev)
 	replicas_v0 = bch2_sb_field_get(sb, replicas_v0);
 
 	if (replicas) {
-		struct bch_replicas_entry *r;
+		struct bch_replicas_entry_v1 *r;
 
 		for_each_replicas_entry(replicas, r)
 			for (i = 0; i < r->nr_devs; i++)
diff --git a/fs/bcachefs/replicas.h b/fs/bcachefs/replicas.h
index 4887675a86f0..b2bb12a9b533 100644
--- a/fs/bcachefs/replicas.h
+++ b/fs/bcachefs/replicas.h
@@ -6,26 +6,26 @@
 #include "eytzinger.h"
 #include "replicas_types.h"
 
-void bch2_replicas_entry_sort(struct bch_replicas_entry *);
+void bch2_replicas_entry_sort(struct bch_replicas_entry_v1 *);
 void bch2_replicas_entry_to_text(struct printbuf *,
-				 struct bch_replicas_entry *);
+				 struct bch_replicas_entry_v1 *);
 void bch2_cpu_replicas_to_text(struct printbuf *, struct bch_replicas_cpu *);
 
-static inline struct bch_replicas_entry *
+static inline struct bch_replicas_entry_v1 *
 cpu_replicas_entry(struct bch_replicas_cpu *r, unsigned i)
 {
 	return (void *) r->entries + r->entry_size * i;
 }
 
 int bch2_replicas_entry_idx(struct bch_fs *,
-			    struct bch_replicas_entry *);
+			    struct bch_replicas_entry_v1 *);
 
-void bch2_devlist_to_replicas(struct bch_replicas_entry *,
+void bch2_devlist_to_replicas(struct bch_replicas_entry_v1 *,
 			      enum bch_data_type,
 			      struct bch_devs_list);
-bool bch2_replicas_marked(struct bch_fs *, struct bch_replicas_entry *);
+bool bch2_replicas_marked(struct bch_fs *, struct bch_replicas_entry_v1 *);
 int bch2_mark_replicas(struct bch_fs *,
-		       struct bch_replicas_entry *);
+		       struct bch_replicas_entry_v1 *);
 
 static inline struct replicas_delta *
 replicas_delta_next(struct replicas_delta *d)
@@ -35,9 +35,9 @@ replicas_delta_next(struct replicas_delta *d)
 
 int bch2_replicas_delta_list_mark(struct bch_fs *, struct replicas_delta_list *);
 
-void bch2_bkey_to_replicas(struct bch_replicas_entry *, struct bkey_s_c);
+void bch2_bkey_to_replicas(struct bch_replicas_entry_v1 *, struct bkey_s_c);
 
-static inline void bch2_replicas_entry_cached(struct bch_replicas_entry *e,
+static inline void bch2_replicas_entry_cached(struct bch_replicas_entry_v1 *e,
 					      unsigned dev)
 {
 	e->data_type	= BCH_DATA_cached;
@@ -57,7 +57,7 @@ int bch2_replicas_gc_start(struct bch_fs *, unsigned);
 int bch2_replicas_gc2(struct bch_fs *);
 
 int bch2_replicas_set_usage(struct bch_fs *,
-			    struct bch_replicas_entry *,
+			    struct bch_replicas_entry_v1 *,
 			    u64);
 
 #define for_each_cpu_replicas_entry(_r, _i)				\
diff --git a/fs/bcachefs/replicas_types.h b/fs/bcachefs/replicas_types.h
index 5cfff489bbc3..030324078bba 100644
--- a/fs/bcachefs/replicas_types.h
+++ b/fs/bcachefs/replicas_types.h
@@ -5,12 +5,12 @@
 struct bch_replicas_cpu {
 	unsigned		nr;
 	unsigned		entry_size;
-	struct bch_replicas_entry *entries;
+	struct bch_replicas_entry_v1 *entries;
 };
 
 struct replicas_delta {
 	s64			delta;
-	struct bch_replicas_entry r;
+	struct bch_replicas_entry_v1 r;
 } __packed;
 
 struct replicas_delta_list {
diff --git a/fs/bcachefs/sb-clean.c b/fs/bcachefs/sb-clean.c
index e151ada1c8bd..fedc9e102edd 100644
--- a/fs/bcachefs/sb-clean.c
+++ b/fs/bcachefs/sb-clean.c
@@ -235,7 +235,7 @@ void bch2_journal_super_entries_add_common(struct bch_fs *c,
 	}
 
 	for (i = 0; i < c->replicas.nr; i++) {
-		struct bch_replicas_entry *e =
+		struct bch_replicas_entry_v1 *e =
 			cpu_replicas_entry(&c->replicas, i);
 		struct jset_entry_data_usage *u =
 			container_of(jset_entry_init(end, sizeof(*u) + e->nr_devs),
diff --git a/fs/bcachefs/sb-errors.h b/fs/bcachefs/sb-errors.h
index 5a09a53966be..57bce14c9d71 100644
--- a/fs/bcachefs/sb-errors.h
+++ b/fs/bcachefs/sb-errors.h
@@ -248,7 +248,8 @@
 	x(root_inode_not_dir,					240)	\
 	x(dir_loop,						241)	\
 	x(hash_table_key_duplicate,				242)	\
-	x(hash_table_key_wrong_offset,				243)
+	x(hash_table_key_wrong_offset,				243)	\
+	x(unlinked_inode_not_on_deleted_list,			244)
 
 enum bch_sb_error_id {
 #define x(t, n) BCH_FSCK_ERR_##t = n,
diff --git a/fs/bcachefs/six.c b/fs/bcachefs/six.c
index 97790445e67a..3a494c5d1247 100644
--- a/fs/bcachefs/six.c
+++ b/fs/bcachefs/six.c
@@ -324,101 +324,57 @@ bool six_relock_ip(struct six_lock *lock, enum six_lock_type type,
 }
 EXPORT_SYMBOL_GPL(six_relock_ip);
 
-#ifdef CONFIG_SIX_LOCK_SPIN_ON_OWNER
+#ifdef CONFIG_BCACHEFS_SIX_OPTIMISTIC_SPIN
 
-static inline bool six_can_spin_on_owner(struct six_lock *lock)
+static inline bool six_owner_running(struct six_lock *lock)
 {
-	struct task_struct *owner;
-	bool ret;
-
-	if (need_resched())
-		return false;
-
+	/*
+	 * When there's no owner, we might have preempted between the owner
+	 * acquiring the lock and setting the owner field. If we're an RT task
+	 * that will live-lock because we won't let the owner complete.
+	 */
 	rcu_read_lock();
-	owner = READ_ONCE(lock->owner);
-	ret = !owner || owner_on_cpu(owner);
+	struct task_struct *owner = READ_ONCE(lock->owner);
+	bool ret = owner ? owner_on_cpu(owner) : !rt_task(current);
 	rcu_read_unlock();
 
 	return ret;
 }
 
-static inline bool six_spin_on_owner(struct six_lock *lock,
-				     struct task_struct *owner,
-				     u64 end_time)
+static inline bool six_optimistic_spin(struct six_lock *lock,
+				       struct six_lock_waiter *wait,
+				       enum six_lock_type type)
 {
-	bool ret = true;
 	unsigned loop = 0;
-
-	rcu_read_lock();
-	while (lock->owner == owner) {
-		/*
-		 * Ensure we emit the owner->on_cpu, dereference _after_
-		 * checking lock->owner still matches owner. If that fails,
-		 * owner might point to freed memory. If it still matches,
-		 * the rcu_read_lock() ensures the memory stays valid.
-		 */
-		barrier();
-
-		if (!owner_on_cpu(owner) || need_resched()) {
-			ret = false;
-			break;
-		}
-
-		if (!(++loop & 0xf) && (time_after64(sched_clock(), end_time))) {
-			six_set_bitmask(lock, SIX_LOCK_NOSPIN);
-			ret = false;
-			break;
-		}
-
-		cpu_relax();
-	}
-	rcu_read_unlock();
-
-	return ret;
-}
-
-static inline bool six_optimistic_spin(struct six_lock *lock, enum six_lock_type type)
-{
-	struct task_struct *task = current;
 	u64 end_time;
 
 	if (type == SIX_LOCK_write)
 		return false;
 
-	preempt_disable();
-	if (!six_can_spin_on_owner(lock))
-		goto fail;
+	if (lock->wait_list.next != &wait->list)
+		return false;
 
-	if (!osq_lock(&lock->osq))
-		goto fail;
+	if (atomic_read(&lock->state) & SIX_LOCK_NOSPIN)
+		return false;
 
+	preempt_disable();
 	end_time = sched_clock() + 10 * NSEC_PER_USEC;
 
-	while (1) {
-		struct task_struct *owner;
-
+	while (!need_resched() && six_owner_running(lock)) {
 		/*
-		 * If there's an owner, wait for it to either
-		 * release the lock or go to sleep.
+		 * Ensures that writes to the waitlist entry happen after we see
+		 * wait->lock_acquired: pairs with the smp_store_release in
+		 * __six_lock_wakeup
 		 */
-		owner = READ_ONCE(lock->owner);
-		if (owner && !six_spin_on_owner(lock, owner, end_time))
-			break;
-
-		if (do_six_trylock(lock, type, false)) {
-			osq_unlock(&lock->osq);
+		if (smp_load_acquire(&wait->lock_acquired)) {
 			preempt_enable();
 			return true;
 		}
 
-		/*
-		 * When there's no owner, we might have preempted between the
-		 * owner acquiring the lock and setting the owner field. If
-		 * we're an RT task that will live-lock because we won't let
-		 * the owner complete.
-		 */
-		if (!owner && (need_resched() || rt_task(task)))
+		if (!(++loop & 0xf) && (time_after64(sched_clock(), end_time))) {
+			six_set_bitmask(lock, SIX_LOCK_NOSPIN);
 			break;
+		}
 
 		/*
 		 * The cpu_relax() call is a compiler barrier which forces
@@ -429,24 +385,15 @@ static inline bool six_optimistic_spin(struct six_lock *lock, enum six_lock_type
 		cpu_relax();
 	}
 
-	osq_unlock(&lock->osq);
-fail:
 	preempt_enable();
-
-	/*
-	 * If we fell out of the spin path because of need_resched(),
-	 * reschedule now, before we try-lock again. This avoids getting
-	 * scheduled out right after we obtained the lock.
-	 */
-	if (need_resched())
-		schedule();
-
 	return false;
 }
 
-#else /* CONFIG_SIX_LOCK_SPIN_ON_OWNER */
+#else /* CONFIG_LOCK_SPIN_ON_OWNER */
 
-static inline bool six_optimistic_spin(struct six_lock *lock, enum six_lock_type type)
+static inline bool six_optimistic_spin(struct six_lock *lock,
+				       struct six_lock_waiter *wait,
+				       enum six_lock_type type)
 {
 	return false;
 }
@@ -470,9 +417,6 @@ static int six_lock_slowpath(struct six_lock *lock, enum six_lock_type type,
 	trace_contention_begin(lock, 0);
 	lock_contended(&lock->dep_map, ip);
 
-	if (six_optimistic_spin(lock, type))
-		goto out;
-
 	wait->task		= current;
 	wait->lock_want		= type;
 	wait->lock_acquired	= false;
@@ -510,6 +454,9 @@ static int six_lock_slowpath(struct six_lock *lock, enum six_lock_type type,
 		ret = 0;
 	}
 
+	if (six_optimistic_spin(lock, wait, type))
+		goto out;
+
 	while (1) {
 		set_current_state(TASK_UNINTERRUPTIBLE);
 
diff --git a/fs/bcachefs/six.h b/fs/bcachefs/six.h
index 4c268b0b8316..a7104ac1d35c 100644
--- a/fs/bcachefs/six.h
+++ b/fs/bcachefs/six.h
@@ -127,10 +127,6 @@
 #include <linux/sched.h>
 #include <linux/types.h>
 
-#ifdef CONFIG_SIX_LOCK_SPIN_ON_OWNER
-#include <linux/osq_lock.h>
-#endif
-
 enum six_lock_type {
 	SIX_LOCK_read,
 	SIX_LOCK_intent,
@@ -143,9 +139,6 @@ struct six_lock {
 	unsigned		intent_lock_recurse;
 	struct task_struct	*owner;
 	unsigned __percpu	*readers;
-#ifdef CONFIG_SIX_LOCK_SPIN_ON_OWNER
-	struct optimistic_spin_queue osq;
-#endif
 	raw_spinlock_t		wait_lock;
 	struct list_head	wait_list;
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
diff --git a/fs/bcachefs/snapshot.c b/fs/bcachefs/snapshot.c
index e9af77b384c7..b23550b44098 100644
--- a/fs/bcachefs/snapshot.c
+++ b/fs/bcachefs/snapshot.c
@@ -590,7 +590,7 @@ int bch2_check_snapshot_trees(struct bch_fs *c)
 		for_each_btree_key_commit(trans, iter,
 			BTREE_ID_snapshot_trees, POS_MIN,
 			BTREE_ITER_PREFETCH, k,
-			NULL, NULL, BTREE_INSERT_LAZY_RW|BTREE_INSERT_NOFAIL,
+			NULL, NULL, BCH_TRANS_COMMIT_lazy_rw|BCH_TRANS_COMMIT_no_enospc,
 		check_snapshot_tree(trans, &iter, k)));
 
 	if (ret)
@@ -868,7 +868,7 @@ int bch2_check_snapshots(struct bch_fs *c)
 		for_each_btree_key_reverse_commit(trans, iter,
 			BTREE_ID_snapshots, POS_MAX,
 			BTREE_ITER_PREFETCH, k,
-			NULL, NULL, BTREE_INSERT_LAZY_RW|BTREE_INSERT_NOFAIL,
+			NULL, NULL, BCH_TRANS_COMMIT_lazy_rw|BCH_TRANS_COMMIT_no_enospc,
 		check_snapshot(trans, &iter, k)));
 	if (ret)
 		bch_err_fn(c, ret);
@@ -959,7 +959,7 @@ static int bch2_snapshot_node_delete(struct btree_trans *trans, u32 id)
 					parent_id, id))
 			goto err;
 
-		parent->v.children[i] = le32_to_cpu(child_id);
+		parent->v.children[i] = cpu_to_le32(child_id);
 
 		normalize_snapshot_child_pointers(&parent->v);
 	}
@@ -1449,12 +1449,12 @@ int bch2_delete_dead_snapshots(struct bch_fs *c)
 		ret = for_each_btree_key_commit(trans, iter,
 				id, POS_MIN,
 				BTREE_ITER_PREFETCH|BTREE_ITER_ALL_SNAPSHOTS, k,
-				&res, NULL, BTREE_INSERT_NOFAIL,
+				&res, NULL, BCH_TRANS_COMMIT_no_enospc,
 			snapshot_delete_key(trans, &iter, k, &deleted, &equiv_seen, &last_pos)) ?:
 		      for_each_btree_key_commit(trans, iter,
 				id, POS_MIN,
 				BTREE_ITER_PREFETCH|BTREE_ITER_ALL_SNAPSHOTS, k,
-				&res, NULL, BTREE_INSERT_NOFAIL,
+				&res, NULL, BCH_TRANS_COMMIT_no_enospc,
 			move_key_to_correct_snapshot(trans, &iter, k));
 
 		bch2_disk_reservation_put(c, &res);
@@ -1489,7 +1489,7 @@ int bch2_delete_dead_snapshots(struct bch_fs *c)
 	 */
 	ret = for_each_btree_key_commit(trans, iter, BTREE_ID_snapshots, POS_MIN,
 				  BTREE_ITER_INTENT, k,
-				  NULL, NULL, BTREE_INSERT_NOFAIL,
+				  NULL, NULL, BCH_TRANS_COMMIT_no_enospc,
 		bch2_fix_child_of_deleted_snapshot(trans, &iter, k, &deleted_interior));
 	if (ret)
 		goto err_create_lock;
diff --git a/fs/bcachefs/str_hash.h b/fs/bcachefs/str_hash.h
index ae21a8cca1b4..89fdb7c21134 100644
--- a/fs/bcachefs/str_hash.h
+++ b/fs/bcachefs/str_hash.h
@@ -15,6 +15,16 @@
 #include <crypto/hash.h>
 #include <crypto/sha2.h>
 
+typedef unsigned __bitwise bch_str_hash_flags_t;
+
+enum bch_str_hash_flags {
+	__BCH_HASH_SET_MUST_CREATE,
+	__BCH_HASH_SET_MUST_REPLACE,
+};
+
+#define BCH_HASH_SET_MUST_CREATE	(__force bch_str_hash_flags_t) BIT(__BCH_HASH_SET_MUST_CREATE)
+#define BCH_HASH_SET_MUST_REPLACE	(__force bch_str_hash_flags_t) BIT(__BCH_HASH_SET_MUST_REPLACE)
+
 static inline enum bch_str_hash_type
 bch2_str_hash_opt_to_type(struct bch_fs *c, enum bch_str_hash_opts opt)
 {
@@ -246,7 +256,7 @@ int bch2_hash_set_snapshot(struct btree_trans *trans,
 			   const struct bch_hash_info *info,
 			   subvol_inum inum, u32 snapshot,
 			   struct bkey_i *insert,
-			   int flags,
+			   bch_str_hash_flags_t str_hash_flags,
 			   int update_flags)
 {
 	struct btree_iter iter, slot = { NULL };
@@ -269,7 +279,7 @@ int bch2_hash_set_snapshot(struct btree_trans *trans,
 		}
 
 		if (!slot.path &&
-		    !(flags & BCH_HASH_SET_MUST_REPLACE))
+		    !(str_hash_flags & BCH_HASH_SET_MUST_REPLACE))
 			bch2_trans_copy_iter(&slot, &iter);
 
 		if (k.k->type != KEY_TYPE_hash_whiteout)
@@ -287,16 +297,16 @@ int bch2_hash_set_snapshot(struct btree_trans *trans,
 	found = true;
 not_found:
 
-	if (!found && (flags & BCH_HASH_SET_MUST_REPLACE)) {
+	if (!found && (str_hash_flags & BCH_HASH_SET_MUST_REPLACE)) {
 		ret = -BCH_ERR_ENOENT_str_hash_set_must_replace;
-	} else if (found && (flags & BCH_HASH_SET_MUST_CREATE)) {
+	} else if (found && (str_hash_flags & BCH_HASH_SET_MUST_CREATE)) {
 		ret = -EEXIST;
 	} else {
 		if (!found && slot.path)
 			swap(iter, slot);
 
 		insert->k.p = iter.pos;
-		ret = bch2_trans_update(trans, &iter, insert, 0);
+		ret = bch2_trans_update(trans, &iter, insert, update_flags);
 	}
 
 	goto out;
@@ -307,7 +317,8 @@ int bch2_hash_set(struct btree_trans *trans,
 		  const struct bch_hash_desc desc,
 		  const struct bch_hash_info *info,
 		  subvol_inum inum,
-		  struct bkey_i *insert, int flags)
+		  struct bkey_i *insert,
+		  bch_str_hash_flags_t str_hash_flags)
 {
 	u32 snapshot;
 	int ret;
@@ -319,7 +330,7 @@ int bch2_hash_set(struct btree_trans *trans,
 	insert->k.p.inode = inum.inum;
 
 	return bch2_hash_set_snapshot(trans, desc, info, inum,
-				      snapshot, insert, flags, 0);
+				      snapshot, insert, str_hash_flags, 0);
 }
 
 static __always_inline
diff --git a/fs/bcachefs/subvolume.c b/fs/bcachefs/subvolume.c
index fccd25aa3242..1cbf9e3a09ec 100644
--- a/fs/bcachefs/subvolume.c
+++ b/fs/bcachefs/subvolume.c
@@ -89,7 +89,7 @@ int bch2_check_subvols(struct bch_fs *c)
 	ret = bch2_trans_run(c,
 		for_each_btree_key_commit(trans, iter,
 			BTREE_ID_subvolumes, POS_MIN, BTREE_ITER_PREFETCH, k,
-			NULL, NULL, BTREE_INSERT_LAZY_RW|BTREE_INSERT_NOFAIL,
+			NULL, NULL, BCH_TRANS_COMMIT_lazy_rw|BCH_TRANS_COMMIT_no_enospc,
 		check_subvol(trans, &iter, k)));
 	if (ret)
 		bch_err_fn(c, ret);
@@ -219,7 +219,7 @@ static int bch2_subvolumes_reparent(struct btree_trans *trans, u32 subvolid_to_d
 				   BTREE_ITER_CACHED, &s)) ?:
 		for_each_btree_key_commit(trans, iter,
 				BTREE_ID_subvolumes, POS_MIN, BTREE_ITER_PREFETCH, k,
-				NULL, NULL, BTREE_INSERT_NOFAIL,
+				NULL, NULL, BCH_TRANS_COMMIT_no_enospc,
 			bch2_subvolume_reparent(trans, &iter, k,
 					subvolid_to_delete, le32_to_cpu(s.parent)));
 }
@@ -256,7 +256,7 @@ static int __bch2_subvolume_delete(struct btree_trans *trans, u32 subvolid)
 static int bch2_subvolume_delete(struct btree_trans *trans, u32 subvolid)
 {
 	return bch2_subvolumes_reparent(trans, subvolid) ?:
-		commit_do(trans, NULL, NULL, BTREE_INSERT_NOFAIL,
+		commit_do(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc,
 			  __bch2_subvolume_delete(trans, subvolid));
 }
 
diff --git a/fs/bcachefs/super-io.c b/fs/bcachefs/super-io.c
index f4cad903f4d6..f90fc7a4c26f 100644
--- a/fs/bcachefs/super-io.c
+++ b/fs/bcachefs/super-io.c
@@ -163,9 +163,10 @@ void bch2_sb_field_delete(struct bch_sb_handle *sb,
 void bch2_free_super(struct bch_sb_handle *sb)
 {
 	kfree(sb->bio);
-	if (!IS_ERR_OR_NULL(sb->bdev))
-		blkdev_put(sb->bdev, sb->holder);
+	if (!IS_ERR_OR_NULL(sb->bdev_handle))
+		bdev_release(sb->bdev_handle);
 	kfree(sb->holder);
+	kfree(sb->sb_name);
 
 	kfree(sb->sb);
 	memset(sb, 0, sizeof(*sb));
@@ -657,12 +658,13 @@ static int read_one_super(struct bch_sb_handle *sb, u64 offset, struct printbuf
 	return 0;
 }
 
-int bch2_read_super(const char *path, struct bch_opts *opts,
-		    struct bch_sb_handle *sb)
+int __bch2_read_super(const char *path, struct bch_opts *opts,
+		    struct bch_sb_handle *sb, bool ignore_notbchfs_msg)
 {
 	u64 offset = opt_get(*opts, sb);
 	struct bch_sb_layout layout;
 	struct printbuf err = PRINTBUF;
+	struct printbuf err2 = PRINTBUF;
 	__le64 *i;
 	int ret;
 #ifndef __KERNEL__
@@ -675,6 +677,10 @@ int bch2_read_super(const char *path, struct bch_opts *opts,
 	if (!sb->holder)
 		return -ENOMEM;
 
+	sb->sb_name = kstrdup(path, GFP_KERNEL);
+	if (!sb->sb_name)
+		return -ENOMEM;
+
 #ifndef __KERNEL__
 	if (opt_get(*opts, direct_io) == false)
 		sb->mode |= BLK_OPEN_BUFFERED;
@@ -686,21 +692,22 @@ int bch2_read_super(const char *path, struct bch_opts *opts,
 	if (!opt_get(*opts, nochanges))
 		sb->mode |= BLK_OPEN_WRITE;
 
-	sb->bdev = blkdev_get_by_path(path, sb->mode, sb->holder, &bch2_sb_handle_bdev_ops);
-	if (IS_ERR(sb->bdev) &&
-	    PTR_ERR(sb->bdev) == -EACCES &&
+	sb->bdev_handle = bdev_open_by_path(path, sb->mode, sb->holder, &bch2_sb_handle_bdev_ops);
+	if (IS_ERR(sb->bdev_handle) &&
+	    PTR_ERR(sb->bdev_handle) == -EACCES &&
 	    opt_get(*opts, read_only)) {
 		sb->mode &= ~BLK_OPEN_WRITE;
 
-		sb->bdev = blkdev_get_by_path(path, sb->mode, sb->holder, &bch2_sb_handle_bdev_ops);
-		if (!IS_ERR(sb->bdev))
+		sb->bdev_handle = bdev_open_by_path(path, sb->mode, sb->holder, &bch2_sb_handle_bdev_ops);
+		if (!IS_ERR(sb->bdev_handle))
 			opt_set(*opts, nochanges, true);
 	}
 
-	if (IS_ERR(sb->bdev)) {
-		ret = PTR_ERR(sb->bdev);
+	if (IS_ERR(sb->bdev_handle)) {
+		ret = PTR_ERR(sb->bdev_handle);
 		goto out;
 	}
+	sb->bdev = sb->bdev_handle->bdev;
 
 	ret = bch2_sb_realloc(sb, 0);
 	if (ret) {
@@ -721,8 +728,14 @@ int bch2_read_super(const char *path, struct bch_opts *opts,
 	if (opt_defined(*opts, sb))
 		goto err;
 
-	printk(KERN_ERR "bcachefs (%s): error reading default superblock: %s\n",
+	prt_printf(&err2, "bcachefs (%s): error reading default superblock: %s\n",
 	       path, err.buf);
+	if (ret == -BCH_ERR_invalid_sb_magic && ignore_notbchfs_msg)
+		printk(KERN_INFO "%s", err2.buf);
+	else
+		printk(KERN_ERR "%s", err2.buf);
+
+	printbuf_exit(&err2);
 	printbuf_reset(&err);
 
 	/*
@@ -798,6 +811,20 @@ int bch2_read_super(const char *path, struct bch_opts *opts,
 	goto out;
 }
 
+int bch2_read_super(const char *path, struct bch_opts *opts,
+		    struct bch_sb_handle *sb)
+{
+	return __bch2_read_super(path, opts, sb, false);
+}
+
+/* provide a silenced version for mount.bcachefs */
+
+int bch2_read_super_silent(const char *path, struct bch_opts *opts,
+		    struct bch_sb_handle *sb)
+{
+	return __bch2_read_super(path, opts, sb, true);
+}
+
 /* write superblock: */
 
 static void write_super_endio(struct bio *bio)
diff --git a/fs/bcachefs/super-io.h b/fs/bcachefs/super-io.h
index f5abd102bff7..e6f40a054319 100644
--- a/fs/bcachefs/super-io.h
+++ b/fs/bcachefs/super-io.h
@@ -74,6 +74,7 @@ void bch2_free_super(struct bch_sb_handle *);
 int bch2_sb_realloc(struct bch_sb_handle *, unsigned);
 
 int bch2_read_super(const char *, struct bch_opts *, struct bch_sb_handle *);
+int bch2_read_super_silent(const char *, struct bch_opts *, struct bch_sb_handle *);
 int bch2_write_super(struct bch_fs *);
 void __bch2_check_set_feature(struct bch_fs *, unsigned);
 
diff --git a/fs/bcachefs/super.c b/fs/bcachefs/super.c
index 24672bb31cbe..91f757173ef0 100644
--- a/fs/bcachefs/super.c
+++ b/fs/bcachefs/super.c
@@ -314,7 +314,8 @@ void bch2_fs_read_only(struct bch_fs *c)
 		BUG_ON(c->journal.last_empty_seq != journal_cur_seq(&c->journal));
 		BUG_ON(atomic_read(&c->btree_cache.dirty));
 		BUG_ON(atomic_long_read(&c->btree_key_cache.nr_dirty));
-		BUG_ON(c->btree_write_buffer.state.nr);
+		BUG_ON(c->btree_write_buffer.inc.keys.nr);
+		BUG_ON(c->btree_write_buffer.flushing.keys.nr);
 
 		bch_verbose(c, "marking filesystem clean");
 		bch2_fs_mark_clean(c);
@@ -504,8 +505,8 @@ static void __bch2_fs_free(struct bch_fs *c)
 	bch2_io_clock_exit(&c->io_clock[WRITE]);
 	bch2_io_clock_exit(&c->io_clock[READ]);
 	bch2_fs_compress_exit(c);
-	bch2_journal_keys_free(&c->journal_keys);
-	bch2_journal_entries_free(c);
+	bch2_journal_keys_put_initial(c);
+	BUG_ON(atomic_read(&c->journal_keys.ref));
 	bch2_fs_btree_write_buffer_exit(c);
 	percpu_free_rwsem(&c->mark_lock);
 	free_percpu(c->online_reserved);
@@ -641,7 +642,9 @@ static int bch2_fs_online(struct bch_fs *c)
 	ret = kobject_add(&c->kobj, NULL, "%pU", c->sb.user_uuid.b) ?:
 	    kobject_add(&c->internal, &c->kobj, "internal") ?:
 	    kobject_add(&c->opts_dir, &c->kobj, "options") ?:
+#ifndef CONFIG_BCACHEFS_NO_LATENCY_ACCT
 	    kobject_add(&c->time_stats, &c->kobj, "time_stats") ?:
+#endif
 	    kobject_add(&c->counters_kobj, &c->kobj, "counters") ?:
 	    bch2_opts_create_sysfs_files(&c->opts_dir);
 	if (ret) {
@@ -702,6 +705,8 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts)
 
 	init_rwsem(&c->gc_lock);
 	mutex_init(&c->gc_gens_lock);
+	atomic_set(&c->journal_keys.ref, 1);
+	c->journal_keys.initial_ref_held = true;
 
 	for (i = 0; i < BCH_TIME_STAT_NR; i++)
 		bch2_time_stats_init(&c->times[i]);
@@ -750,7 +755,6 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts)
 
 	c->journal.flush_write_time	= &c->times[BCH_TIME_journal_flush_write];
 	c->journal.noflush_write_time	= &c->times[BCH_TIME_journal_noflush_write];
-	c->journal.blocked_time		= &c->times[BCH_TIME_blocked_journal];
 	c->journal.flush_seq_time	= &c->times[BCH_TIME_journal_flush_seq];
 
 	bch2_fs_btree_cache_init_early(&c->btree_cache);
diff --git a/fs/bcachefs/super_types.h b/fs/bcachefs/super_types.h
index 7dda4985b99f..b2119686e2e1 100644
--- a/fs/bcachefs/super_types.h
+++ b/fs/bcachefs/super_types.h
@@ -4,7 +4,9 @@
 
 struct bch_sb_handle {
 	struct bch_sb		*sb;
+	struct bdev_handle	*bdev_handle;
 	struct block_device	*bdev;
+	char			*sb_name;
 	struct bio		*bio;
 	void			*holder;
 	size_t			buffer_size;
diff --git a/fs/bcachefs/trace.h b/fs/bcachefs/trace.h
index 7857671159b4..7b24e7fe3b53 100644
--- a/fs/bcachefs/trace.h
+++ b/fs/bcachefs/trace.h
@@ -188,6 +188,25 @@ DEFINE_EVENT(bch_fs, journal_entry_full,
 	TP_ARGS(c)
 );
 
+TRACE_EVENT(journal_entry_close,
+	TP_PROTO(struct bch_fs *c, unsigned bytes),
+	TP_ARGS(c, bytes),
+
+	TP_STRUCT__entry(
+		__field(dev_t,		dev			)
+		__field(u32,		bytes			)
+	),
+
+	TP_fast_assign(
+		__entry->dev			= c->dev;
+		__entry->bytes			= bytes;
+	),
+
+	TP_printk("%d,%d entry bytes %u",
+		  MAJOR(__entry->dev), MINOR(__entry->dev),
+		  __entry->bytes)
+);
+
 DEFINE_EVENT(bio, journal_write,
 	TP_PROTO(struct bio *bio),
 	TP_ARGS(bio)
@@ -1252,22 +1271,37 @@ TRACE_EVENT(trans_restart_key_cache_key_realloced,
 TRACE_EVENT(path_downgrade,
 	TP_PROTO(struct btree_trans *trans,
 		 unsigned long caller_ip,
-		 struct btree_path *path),
-	TP_ARGS(trans, caller_ip, path),
+		 struct btree_path *path,
+		 unsigned old_locks_want),
+	TP_ARGS(trans, caller_ip, path, old_locks_want),
 
 	TP_STRUCT__entry(
 		__array(char,			trans_fn, 32	)
 		__field(unsigned long,		caller_ip	)
+		__field(unsigned,		old_locks_want	)
+		__field(unsigned,		new_locks_want	)
+		__field(unsigned,		btree		)
+		TRACE_BPOS_entries(pos)
 	),
 
 	TP_fast_assign(
 		strscpy(__entry->trans_fn, trans->fn, sizeof(__entry->trans_fn));
 		__entry->caller_ip		= caller_ip;
+		__entry->old_locks_want		= old_locks_want;
+		__entry->new_locks_want		= path->locks_want;
+		__entry->btree			= path->btree_id;
+		TRACE_BPOS_assign(pos, path->pos);
 	),
 
-	TP_printk("%s %pS",
+	TP_printk("%s %pS locks_want %u -> %u %s %llu:%llu:%u",
 		  __entry->trans_fn,
-		  (void *) __entry->caller_ip)
+		  (void *) __entry->caller_ip,
+		  __entry->old_locks_want,
+		  __entry->new_locks_want,
+		  bch2_btree_id_str(__entry->btree),
+		  __entry->pos_inode,
+		  __entry->pos_offset,
+		  __entry->pos_snapshot)
 );
 
 DEFINE_EVENT(transaction_event,	trans_restart_write_buffer_flush,
@@ -1298,21 +1332,38 @@ TRACE_EVENT(write_buffer_flush,
 		  __entry->nr, __entry->size, __entry->skipped, __entry->fast)
 );
 
+TRACE_EVENT(write_buffer_flush_sync,
+	TP_PROTO(struct btree_trans *trans, unsigned long caller_ip),
+	TP_ARGS(trans, caller_ip),
+
+	TP_STRUCT__entry(
+		__array(char,			trans_fn, 32	)
+		__field(unsigned long,		caller_ip	)
+	),
+
+	TP_fast_assign(
+		strscpy(__entry->trans_fn, trans->fn, sizeof(__entry->trans_fn));
+		__entry->caller_ip		= caller_ip;
+	),
+
+	TP_printk("%s %pS", __entry->trans_fn, (void *) __entry->caller_ip)
+);
+
 TRACE_EVENT(write_buffer_flush_slowpath,
-	TP_PROTO(struct btree_trans *trans, size_t nr, size_t size),
-	TP_ARGS(trans, nr, size),
+	TP_PROTO(struct btree_trans *trans, size_t slowpath, size_t total),
+	TP_ARGS(trans, slowpath, total),
 
 	TP_STRUCT__entry(
-		__field(size_t,		nr		)
-		__field(size_t,		size		)
+		__field(size_t,		slowpath	)
+		__field(size_t,		total		)
 	),
 
 	TP_fast_assign(
-		__entry->nr	= nr;
-		__entry->size	= size;
+		__entry->slowpath	= slowpath;
+		__entry->total		= total;
 	),
 
-	TP_printk("%zu/%zu", __entry->nr, __entry->size)
+	TP_printk("%zu/%zu", __entry->slowpath, __entry->total)
 );
 
 #endif /* _TRACE_BCACHEFS_H */
diff --git a/fs/bcachefs/util.c b/fs/bcachefs/util.c
index 84b142fcc3df..e85b1f467295 100644
--- a/fs/bcachefs/util.c
+++ b/fs/bcachefs/util.c
@@ -315,6 +315,58 @@ int bch2_prt_task_backtrace(struct printbuf *out, struct task_struct *task)
 	return ret;
 }
 
+#ifndef __KERNEL__
+#include <time.h>
+void bch2_prt_datetime(struct printbuf *out, time64_t sec)
+{
+	time_t t = sec;
+	char buf[64];
+	ctime_r(&t, buf);
+	strim(buf);
+	prt_str(out, buf);
+}
+#else
+void bch2_prt_datetime(struct printbuf *out, time64_t sec)
+{
+	char buf[64];
+	snprintf(buf, sizeof(buf), "%ptT", &sec);
+	prt_u64(out, sec);
+}
+#endif
+
+static const struct time_unit {
+	const char	*name;
+	u64		nsecs;
+} time_units[] = {
+	{ "ns",		1		 },
+	{ "us",		NSEC_PER_USEC	 },
+	{ "ms",		NSEC_PER_MSEC	 },
+	{ "s",		NSEC_PER_SEC	 },
+	{ "m",          (u64) NSEC_PER_SEC * 60},
+	{ "h",          (u64) NSEC_PER_SEC * 3600},
+	{ "eon",        U64_MAX          },
+};
+
+static const struct time_unit *pick_time_units(u64 ns)
+{
+	const struct time_unit *u;
+
+	for (u = time_units;
+	     u + 1 < time_units + ARRAY_SIZE(time_units) &&
+	     ns >= u[1].nsecs << 1;
+	     u++)
+		;
+
+	return u;
+}
+
+void bch2_pr_time_units(struct printbuf *out, u64 ns)
+{
+	const struct time_unit *u = pick_time_units(ns);
+
+	prt_printf(out, "%llu %s", div_u64(ns, u->nsecs), u->name);
+}
+
 /* time stats: */
 
 #ifndef CONFIG_BCACHEFS_NO_LATENCY_ACCT
@@ -359,6 +411,7 @@ static inline void bch2_time_stats_update_one(struct bch2_time_stats *stats,
 		mean_and_variance_weighted_update(&stats->duration_stats_weighted, duration);
 		stats->max_duration = max(stats->max_duration, duration);
 		stats->min_duration = min(stats->min_duration, duration);
+		stats->total_duration += duration;
 		bch2_quantiles_update(&stats->quantiles, duration);
 	}
 
@@ -372,20 +425,24 @@ static inline void bch2_time_stats_update_one(struct bch2_time_stats *stats,
 	}
 }
 
+static void __bch2_time_stats_clear_buffer(struct bch2_time_stats *stats,
+					   struct bch2_time_stat_buffer *b)
+{
+	for (struct bch2_time_stat_buffer_entry *i = b->entries;
+	     i < b->entries + ARRAY_SIZE(b->entries);
+	     i++)
+		bch2_time_stats_update_one(stats, i->start, i->end);
+	b->nr = 0;
+}
+
 static noinline void bch2_time_stats_clear_buffer(struct bch2_time_stats *stats,
 						  struct bch2_time_stat_buffer *b)
 {
-	struct bch2_time_stat_buffer_entry *i;
 	unsigned long flags;
 
 	spin_lock_irqsave(&stats->lock, flags);
-	for (i = b->entries;
-	     i < b->entries + ARRAY_SIZE(b->entries);
-	     i++)
-		bch2_time_stats_update_one(stats, i->start, i->end);
+	__bch2_time_stats_clear_buffer(stats, b);
 	spin_unlock_irqrestore(&stats->lock, flags);
-
-	b->nr = 0;
 }
 
 void __bch2_time_stats_update(struct bch2_time_stats *stats, u64 start, u64 end)
@@ -423,40 +480,6 @@ void __bch2_time_stats_update(struct bch2_time_stats *stats, u64 start, u64 end)
 		preempt_enable();
 	}
 }
-#endif
-
-static const struct time_unit {
-	const char	*name;
-	u64		nsecs;
-} time_units[] = {
-	{ "ns",		1		 },
-	{ "us",		NSEC_PER_USEC	 },
-	{ "ms",		NSEC_PER_MSEC	 },
-	{ "s",		NSEC_PER_SEC	 },
-	{ "m",          (u64) NSEC_PER_SEC * 60},
-	{ "h",          (u64) NSEC_PER_SEC * 3600},
-	{ "eon",        U64_MAX          },
-};
-
-static const struct time_unit *pick_time_units(u64 ns)
-{
-	const struct time_unit *u;
-
-	for (u = time_units;
-	     u + 1 < time_units + ARRAY_SIZE(time_units) &&
-	     ns >= u[1].nsecs << 1;
-	     u++)
-		;
-
-	return u;
-}
-
-void bch2_pr_time_units(struct printbuf *out, u64 ns)
-{
-	const struct time_unit *u = pick_time_units(ns);
-
-	prt_printf(out, "%llu %s", div_u64(ns, u->nsecs), u->name);
-}
 
 static void bch2_pr_time_units_aligned(struct printbuf *out, u64 ns)
 {
@@ -467,26 +490,6 @@ static void bch2_pr_time_units_aligned(struct printbuf *out, u64 ns)
 	prt_printf(out, "%s", u->name);
 }
 
-#ifndef __KERNEL__
-#include <time.h>
-void bch2_prt_datetime(struct printbuf *out, time64_t sec)
-{
-	time_t t = sec;
-	char buf[64];
-	ctime_r(&t, buf);
-	prt_str(out, buf);
-}
-#else
-void bch2_prt_datetime(struct printbuf *out, time64_t sec)
-{
-	char buf[64];
-	snprintf(buf, sizeof(buf), "%ptT", &sec);
-	prt_u64(out, sec);
-}
-#endif
-
-#define TABSTOP_SIZE 12
-
 static inline void pr_name_and_units(struct printbuf *out, const char *name, u64 ns)
 {
 	prt_str(out, name);
@@ -495,12 +498,24 @@ static inline void pr_name_and_units(struct printbuf *out, const char *name, u64
 	prt_newline(out);
 }
 
+#define TABSTOP_SIZE 12
+
 void bch2_time_stats_to_text(struct printbuf *out, struct bch2_time_stats *stats)
 {
 	const struct time_unit *u;
 	s64 f_mean = 0, d_mean = 0;
 	u64 q, last_q = 0, f_stddev = 0, d_stddev = 0;
 	int i;
+
+	if (stats->buffer) {
+		int cpu;
+
+		spin_lock_irq(&stats->lock);
+		for_each_possible_cpu(cpu)
+			__bch2_time_stats_clear_buffer(stats, per_cpu_ptr(stats->buffer, cpu));
+		spin_unlock_irq(&stats->lock);
+	}
+
 	/*
 	 * avoid divide by zero
 	 */
@@ -546,6 +561,7 @@ void bch2_time_stats_to_text(struct printbuf *out, struct bch2_time_stats *stats
 
 	pr_name_and_units(out, "min:", stats->min_duration);
 	pr_name_and_units(out, "max:", stats->max_duration);
+	pr_name_and_units(out, "total:", stats->total_duration);
 
 	prt_printf(out, "mean:");
 	prt_tab(out);
@@ -603,6 +619,9 @@ void bch2_time_stats_to_text(struct printbuf *out, struct bch2_time_stats *stats
 		last_q = q;
 	}
 }
+#else
+void bch2_time_stats_to_text(struct printbuf *out, struct bch2_time_stats *stats) {}
+#endif
 
 void bch2_time_stats_exit(struct bch2_time_stats *stats)
 {
diff --git a/fs/bcachefs/util.h b/fs/bcachefs/util.h
index 2984b57b2958..b93d5f481c7e 100644
--- a/fs/bcachefs/util.h
+++ b/fs/bcachefs/util.h
@@ -373,8 +373,9 @@ struct bch2_time_stat_buffer {
 struct bch2_time_stats {
 	spinlock_t	lock;
 	/* all fields are in nanoseconds */
-	u64		max_duration;
 	u64             min_duration;
+	u64		max_duration;
+	u64		total_duration;
 	u64             max_freq;
 	u64             min_freq;
 	u64		last_event;
@@ -389,15 +390,39 @@ struct bch2_time_stats {
 
 #ifndef CONFIG_BCACHEFS_NO_LATENCY_ACCT
 void __bch2_time_stats_update(struct bch2_time_stats *stats, u64, u64);
-#else
-static inline void __bch2_time_stats_update(struct bch2_time_stats *stats, u64 start, u64 end) {}
-#endif
 
 static inline void bch2_time_stats_update(struct bch2_time_stats *stats, u64 start)
 {
 	__bch2_time_stats_update(stats, start, local_clock());
 }
 
+static inline bool track_event_change(struct bch2_time_stats *stats,
+				      u64 *start, bool v)
+{
+	if (v != !!*start) {
+		if (!v) {
+			bch2_time_stats_update(stats, *start);
+			*start = 0;
+		} else {
+			*start = local_clock() ?: 1;
+			return true;
+		}
+	}
+
+	return false;
+}
+#else
+static inline void __bch2_time_stats_update(struct bch2_time_stats *stats, u64 start, u64 end) {}
+static inline void bch2_time_stats_update(struct bch2_time_stats *stats, u64 start) {}
+static inline bool track_event_change(struct bch2_time_stats *stats,
+				      u64 *start, bool v)
+{
+	bool ret = v && !*start;
+	*start = v;
+	return ret;
+}
+#endif
+
 void bch2_time_stats_to_text(struct printbuf *, struct bch2_time_stats *);
 
 void bch2_time_stats_exit(struct bch2_time_stats *);
diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c
index 35c1d24d4a78..137c4eb24c28 100644
--- a/fs/btrfs/ctree.c
+++ b/fs/btrfs/ctree.c
@@ -370,33 +370,41 @@ int btrfs_copy_root(struct btrfs_trans_handle *trans,
 /*
  * check if the tree block can be shared by multiple trees
  */
-int btrfs_block_can_be_shared(struct btrfs_trans_handle *trans,
-			      struct btrfs_root *root,
-			      struct extent_buffer *buf)
+bool btrfs_block_can_be_shared(struct btrfs_trans_handle *trans,
+			       struct btrfs_root *root,
+			       struct extent_buffer *buf)
 {
+	const u64 buf_gen = btrfs_header_generation(buf);
+
 	/*
 	 * Tree blocks not in shareable trees and tree roots are never shared.
 	 * If a block was allocated after the last snapshot and the block was
 	 * not allocated by tree relocation, we know the block is not shared.
 	 */
-	if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) &&
-	    buf != root->node &&
-	    (btrfs_header_generation(buf) <=
-	     btrfs_root_last_snapshot(&root->root_item) ||
-	     btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC))) {
-		if (buf != root->commit_root)
-			return 1;
-		/*
-		 * An extent buffer that used to be the commit root may still be
-		 * shared because the tree height may have increased and it
-		 * became a child of a higher level root. This can happen when
-		 * snapshotting a subvolume created in the current transaction.
-		 */
-		if (btrfs_header_generation(buf) == trans->transid)
-			return 1;
-	}
 
-	return 0;
+	if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state))
+		return false;
+
+	if (buf == root->node)
+		return false;
+
+	if (buf_gen > btrfs_root_last_snapshot(&root->root_item) &&
+	    !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC))
+		return false;
+
+	if (buf != root->commit_root)
+		return true;
+
+	/*
+	 * An extent buffer that used to be the commit root may still be shared
+	 * because the tree height may have increased and it became a child of a
+	 * higher level root. This can happen when snapshotting a subvolume
+	 * created in the current transaction.
+	 */
+	if (buf_gen == trans->transid)
+		return true;
+
+	return false;
 }
 
 static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans,
diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
index 196c005c31f6..9c0800f5bdcb 100644
--- a/fs/btrfs/ctree.h
+++ b/fs/btrfs/ctree.h
@@ -224,9 +224,6 @@ struct btrfs_root {
 
 	struct list_head root_list;
 
-	spinlock_t log_extents_lock[2];
-	struct list_head logged_list[2];
-
 	spinlock_t inode_lock;
 	/* red-black tree that keeps track of in-memory inodes */
 	struct rb_root inode_tree;
@@ -561,9 +558,9 @@ int btrfs_copy_root(struct btrfs_trans_handle *trans,
 		      struct btrfs_root *root,
 		      struct extent_buffer *buf,
 		      struct extent_buffer **cow_ret, u64 new_root_objectid);
-int btrfs_block_can_be_shared(struct btrfs_trans_handle *trans,
-			      struct btrfs_root *root,
-			      struct extent_buffer *buf);
+bool btrfs_block_can_be_shared(struct btrfs_trans_handle *trans,
+			       struct btrfs_root *root,
+			       struct extent_buffer *buf);
 int btrfs_del_ptr(struct btrfs_trans_handle *trans, struct btrfs_root *root,
 		  struct btrfs_path *path, int level, int slot);
 void btrfs_extend_item(struct btrfs_trans_handle *trans,
diff --git a/fs/btrfs/delayed-inode.c b/fs/btrfs/delayed-inode.c
index 7381241334e8..91159dd7355b 100644
--- a/fs/btrfs/delayed-inode.c
+++ b/fs/btrfs/delayed-inode.c
@@ -1036,14 +1036,33 @@ static int __btrfs_update_delayed_inode(struct btrfs_trans_handle *trans,
 	if (!test_bit(BTRFS_DELAYED_NODE_DEL_IREF, &node->flags))
 		goto out;
 
-	path->slots[0]++;
-	if (path->slots[0] >= btrfs_header_nritems(leaf))
-		goto search;
-again:
+	/*
+	 * Now we're going to delete the INODE_REF/EXTREF, which should be the
+	 * only one ref left.  Check if the next item is an INODE_REF/EXTREF.
+	 *
+	 * But if we're the last item already, release and search for the last
+	 * INODE_REF/EXTREF.
+	 */
+	if (path->slots[0] + 1 >= btrfs_header_nritems(leaf)) {
+		key.objectid = node->inode_id;
+		key.type = BTRFS_INODE_EXTREF_KEY;
+		key.offset = (u64)-1;
+
+		btrfs_release_path(path);
+		ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
+		if (ret < 0)
+			goto err_out;
+		ASSERT(ret > 0);
+		ASSERT(path->slots[0] > 0);
+		ret = 0;
+		path->slots[0]--;
+		leaf = path->nodes[0];
+	} else {
+		path->slots[0]++;
+	}
 	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
 	if (key.objectid != node->inode_id)
 		goto out;
-
 	if (key.type != BTRFS_INODE_REF_KEY &&
 	    key.type != BTRFS_INODE_EXTREF_KEY)
 		goto out;
@@ -1070,22 +1089,6 @@ static int __btrfs_update_delayed_inode(struct btrfs_trans_handle *trans,
 		btrfs_abort_transaction(trans, ret);
 
 	return ret;
-
-search:
-	btrfs_release_path(path);
-
-	key.type = BTRFS_INODE_EXTREF_KEY;
-	key.offset = -1;
-
-	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
-	if (ret < 0)
-		goto err_out;
-	ASSERT(ret);
-
-	ret = 0;
-	leaf = path->nodes[0];
-	path->slots[0]--;
-	goto again;
 }
 
 static inline int btrfs_update_delayed_inode(struct btrfs_trans_handle *trans,
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index 401ea09ae4b8..5ac6789ca55f 100644
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -650,14 +650,10 @@ static void __setup_root(struct btrfs_root *root, struct btrfs_fs_info *fs_info,
 	INIT_LIST_HEAD(&root->ordered_extents);
 	INIT_LIST_HEAD(&root->ordered_root);
 	INIT_LIST_HEAD(&root->reloc_dirty_list);
-	INIT_LIST_HEAD(&root->logged_list[0]);
-	INIT_LIST_HEAD(&root->logged_list[1]);
 	spin_lock_init(&root->inode_lock);
 	spin_lock_init(&root->delalloc_lock);
 	spin_lock_init(&root->ordered_extent_lock);
 	spin_lock_init(&root->accounting_lock);
-	spin_lock_init(&root->log_extents_lock[0]);
-	spin_lock_init(&root->log_extents_lock[1]);
 	spin_lock_init(&root->qgroup_meta_rsv_lock);
 	mutex_init(&root->objectid_mutex);
 	mutex_init(&root->log_mutex);
@@ -3213,6 +3209,7 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device
 		goto fail_alloc;
 	}
 
+	btrfs_info(fs_info, "first mount of filesystem %pU", disk_super->fsid);
 	/*
 	 * Verify the type first, if that or the checksum value are
 	 * corrupted, we'll find out
diff --git a/fs/btrfs/disk-io.h b/fs/btrfs/disk-io.h
index 50dab8f639dc..e589359e6a68 100644
--- a/fs/btrfs/disk-io.h
+++ b/fs/btrfs/disk-io.h
@@ -37,8 +37,6 @@ struct extent_buffer *btrfs_find_create_tree_block(
 						struct btrfs_fs_info *fs_info,
 						u64 bytenr, u64 owner_root,
 						int level);
-void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans,
-			      struct extent_buffer *buf);
 void btrfs_clear_oneshot_options(struct btrfs_fs_info *fs_info);
 int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info);
 int btrfs_check_super_csum(struct btrfs_fs_info *fs_info,
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index 03cef28d9e37..3431a53bf3fd 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -870,7 +870,7 @@ static int attach_extent_buffer_page(struct extent_buffer *eb,
 	 * will not race with any other ebs.
 	 */
 	if (page->mapping)
-		lockdep_assert_held(&page->mapping->private_lock);
+		lockdep_assert_held(&page->mapping->i_private_lock);
 
 	if (fs_info->nodesize >= PAGE_SIZE) {
 		if (!PagePrivate(page))
@@ -1736,16 +1736,16 @@ static int submit_eb_subpage(struct page *page, struct writeback_control *wbc)
 		 * Take private lock to ensure the subpage won't be detached
 		 * in the meantime.
 		 */
-		spin_lock(&page->mapping->private_lock);
+		spin_lock(&page->mapping->i_private_lock);
 		if (!PagePrivate(page)) {
-			spin_unlock(&page->mapping->private_lock);
+			spin_unlock(&page->mapping->i_private_lock);
 			break;
 		}
 		spin_lock_irqsave(&subpage->lock, flags);
 		if (!test_bit(bit_start + fs_info->subpage_info->dirty_offset,
 			      subpage->bitmaps)) {
 			spin_unlock_irqrestore(&subpage->lock, flags);
-			spin_unlock(&page->mapping->private_lock);
+			spin_unlock(&page->mapping->i_private_lock);
 			bit_start++;
 			continue;
 		}
@@ -1759,7 +1759,7 @@ static int submit_eb_subpage(struct page *page, struct writeback_control *wbc)
 		 */
 		eb = find_extent_buffer_nolock(fs_info, start);
 		spin_unlock_irqrestore(&subpage->lock, flags);
-		spin_unlock(&page->mapping->private_lock);
+		spin_unlock(&page->mapping->i_private_lock);
 
 		/*
 		 * The eb has already reached 0 refs thus find_extent_buffer()
@@ -1811,9 +1811,9 @@ static int submit_eb_page(struct page *page, struct btrfs_eb_write_context *ctx)
 	if (btrfs_sb(page->mapping->host->i_sb)->nodesize < PAGE_SIZE)
 		return submit_eb_subpage(page, wbc);
 
-	spin_lock(&mapping->private_lock);
+	spin_lock(&mapping->i_private_lock);
 	if (!PagePrivate(page)) {
-		spin_unlock(&mapping->private_lock);
+		spin_unlock(&mapping->i_private_lock);
 		return 0;
 	}
 
@@ -1824,16 +1824,16 @@ static int submit_eb_page(struct page *page, struct btrfs_eb_write_context *ctx)
 	 * crashing the machine for something we can survive anyway.
 	 */
 	if (WARN_ON(!eb)) {
-		spin_unlock(&mapping->private_lock);
+		spin_unlock(&mapping->i_private_lock);
 		return 0;
 	}
 
 	if (eb == ctx->eb) {
-		spin_unlock(&mapping->private_lock);
+		spin_unlock(&mapping->i_private_lock);
 		return 0;
 	}
 	ret = atomic_inc_not_zero(&eb->refs);
-	spin_unlock(&mapping->private_lock);
+	spin_unlock(&mapping->i_private_lock);
 	if (!ret)
 		return 0;
 
@@ -3056,7 +3056,7 @@ static bool page_range_has_eb(struct btrfs_fs_info *fs_info, struct page *page)
 {
 	struct btrfs_subpage *subpage;
 
-	lockdep_assert_held(&page->mapping->private_lock);
+	lockdep_assert_held(&page->mapping->i_private_lock);
 
 	if (PagePrivate(page)) {
 		subpage = (struct btrfs_subpage *)page->private;
@@ -3079,14 +3079,14 @@ static void detach_extent_buffer_page(struct extent_buffer *eb, struct page *pag
 
 	/*
 	 * For mapped eb, we're going to change the page private, which should
-	 * be done under the private_lock.
+	 * be done under the i_private_lock.
 	 */
 	if (mapped)
-		spin_lock(&page->mapping->private_lock);
+		spin_lock(&page->mapping->i_private_lock);
 
 	if (!PagePrivate(page)) {
 		if (mapped)
-			spin_unlock(&page->mapping->private_lock);
+			spin_unlock(&page->mapping->i_private_lock);
 		return;
 	}
 
@@ -3110,7 +3110,7 @@ static void detach_extent_buffer_page(struct extent_buffer *eb, struct page *pag
 			detach_page_private(page);
 		}
 		if (mapped)
-			spin_unlock(&page->mapping->private_lock);
+			spin_unlock(&page->mapping->i_private_lock);
 		return;
 	}
 
@@ -3133,7 +3133,7 @@ static void detach_extent_buffer_page(struct extent_buffer *eb, struct page *pag
 	if (!page_range_has_eb(fs_info, page))
 		btrfs_detach_subpage(fs_info, page);
 
-	spin_unlock(&page->mapping->private_lock);
+	spin_unlock(&page->mapping->i_private_lock);
 }
 
 /* Release all pages attached to the extent buffer */
@@ -3514,7 +3514,7 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
 
 	/*
 	 * Preallocate page->private for subpage case, so that we won't
-	 * allocate memory with private_lock nor page lock hold.
+	 * allocate memory with i_private_lock nor page lock hold.
 	 *
 	 * The memory will be freed by attach_extent_buffer_page() or freed
 	 * manually if we exit earlier.
@@ -3535,10 +3535,10 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
 			goto free_eb;
 		}
 
-		spin_lock(&mapping->private_lock);
+		spin_lock(&mapping->i_private_lock);
 		exists = grab_extent_buffer(fs_info, p);
 		if (exists) {
-			spin_unlock(&mapping->private_lock);
+			spin_unlock(&mapping->i_private_lock);
 			unlock_page(p);
 			put_page(p);
 			mark_extent_buffer_accessed(exists, p);
@@ -3558,7 +3558,7 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
 		 * Thus needs no special handling in error path.
 		 */
 		btrfs_page_inc_eb_refs(fs_info, p);
-		spin_unlock(&mapping->private_lock);
+		spin_unlock(&mapping->i_private_lock);
 
 		WARN_ON(btrfs_page_test_dirty(fs_info, p, eb->start, eb->len));
 		eb->pages[i] = p;
@@ -4563,12 +4563,12 @@ static int try_release_subpage_extent_buffer(struct page *page)
 	 * Finally to check if we have cleared page private, as if we have
 	 * released all ebs in the page, the page private should be cleared now.
 	 */
-	spin_lock(&page->mapping->private_lock);
+	spin_lock(&page->mapping->i_private_lock);
 	if (!PagePrivate(page))
 		ret = 1;
 	else
 		ret = 0;
-	spin_unlock(&page->mapping->private_lock);
+	spin_unlock(&page->mapping->i_private_lock);
 	return ret;
 
 }
@@ -4584,9 +4584,9 @@ int try_release_extent_buffer(struct page *page)
 	 * We need to make sure nobody is changing page->private, as we rely on
 	 * page->private as the pointer to extent buffer.
 	 */
-	spin_lock(&page->mapping->private_lock);
+	spin_lock(&page->mapping->i_private_lock);
 	if (!PagePrivate(page)) {
-		spin_unlock(&page->mapping->private_lock);
+		spin_unlock(&page->mapping->i_private_lock);
 		return 1;
 	}
 
@@ -4601,10 +4601,10 @@ int try_release_extent_buffer(struct page *page)
 	spin_lock(&eb->refs_lock);
 	if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
 		spin_unlock(&eb->refs_lock);
-		spin_unlock(&page->mapping->private_lock);
+		spin_unlock(&page->mapping->i_private_lock);
 		return 0;
 	}
-	spin_unlock(&page->mapping->private_lock);
+	spin_unlock(&page->mapping->i_private_lock);
 
 	/*
 	 * If tree ref isn't set then we know the ref on this eb is a real ref,
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index 9f5a9894f88f..ff7b4efca24f 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -10930,7 +10930,7 @@ static const struct address_space_operations btrfs_aops = {
 	.release_folio	= btrfs_release_folio,
 	.migrate_folio	= btrfs_migrate_folio,
 	.dirty_folio	= filemap_dirty_folio,
-	.error_remove_page = generic_error_remove_page,
+	.error_remove_folio = generic_error_remove_folio,
 	.swap_activate	= btrfs_swap_activate,
 	.swap_deactivate = btrfs_swap_deactivate,
 };
diff --git a/fs/btrfs/subpage.c b/fs/btrfs/subpage.c
index 1b999c6e4193..2347cf15278b 100644
--- a/fs/btrfs/subpage.c
+++ b/fs/btrfs/subpage.c
@@ -200,7 +200,7 @@ void btrfs_page_inc_eb_refs(const struct btrfs_fs_info *fs_info,
 		return;
 
 	ASSERT(PagePrivate(page) && page->mapping);
-	lockdep_assert_held(&page->mapping->private_lock);
+	lockdep_assert_held(&page->mapping->i_private_lock);
 
 	subpage = (struct btrfs_subpage *)page->private;
 	atomic_inc(&subpage->eb_refs);
@@ -215,7 +215,7 @@ void btrfs_page_dec_eb_refs(const struct btrfs_fs_info *fs_info,
 		return;
 
 	ASSERT(PagePrivate(page) && page->mapping);
-	lockdep_assert_held(&page->mapping->private_lock);
+	lockdep_assert_held(&page->mapping->i_private_lock);
 
 	subpage = (struct btrfs_subpage *)page->private;
 	ASSERT(atomic_read(&subpage->eb_refs));
diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
index f638dc339693..2a3a5bf102dc 100644
--- a/fs/btrfs/super.c
+++ b/fs/btrfs/super.c
@@ -80,7 +80,10 @@ static int btrfs_remount(struct super_block *sb, int *flags, char *data);
 
 static void btrfs_put_super(struct super_block *sb)
 {
-	close_ctree(btrfs_sb(sb));
+	struct btrfs_fs_info *fs_info = btrfs_sb(sb);
+
+	btrfs_info(fs_info, "last unmount of filesystem %pU", fs_info->fs_devices->fsid);
+	close_ctree(fs_info);
 }
 
 enum {
@@ -1403,6 +1406,8 @@ static struct dentry *btrfs_mount_root(struct file_system_type *fs_type,
 			return ERR_PTR(error);
 	}
 
+	/* No support for restricting writes to btrfs devices yet... */
+	mode &= ~BLK_OPEN_RESTRICT_WRITES;
 	/*
 	 * Setup a dummy root and fs_info for test/set super.  This is because
 	 * we don't actually fill this stuff out until open_ctree, but we need
diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c
index 6e63816dddcb..bfc0eb5e3b7c 100644
--- a/fs/btrfs/transaction.c
+++ b/fs/btrfs/transaction.c
@@ -1774,7 +1774,7 @@ static noinline int create_pending_snapshot(struct btrfs_trans_handle *trans,
 	btrfs_release_path(path);
 
 	ret = btrfs_create_qgroup(trans, objectid);
-	if (ret) {
+	if (ret && ret != -EEXIST) {
 		btrfs_abort_transaction(trans, ret);
 		goto fail;
 	}
diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c
index a416cbea75d1..50fdc69fdddf 100644
--- a/fs/btrfs/tree-checker.c
+++ b/fs/btrfs/tree-checker.c
@@ -31,6 +31,7 @@
 #include "inode-item.h"
 #include "dir-item.h"
 #include "raid-stripe-tree.h"
+#include "extent-tree.h"
 
 /*
  * Error message should follow the following format:
@@ -1276,6 +1277,8 @@ static int check_extent_item(struct extent_buffer *leaf,
 	unsigned long ptr;	/* Current pointer inside inline refs */
 	unsigned long end;	/* Extent item end */
 	const u32 item_size = btrfs_item_size(leaf, slot);
+	u8 last_type = 0;
+	u64 last_seq = U64_MAX;
 	u64 flags;
 	u64 generation;
 	u64 total_refs;		/* Total refs in btrfs_extent_item */
@@ -1322,6 +1325,18 @@ static int check_extent_item(struct extent_buffer *leaf,
 	 *    2.2) Ref type specific data
 	 *         Either using btrfs_extent_inline_ref::offset, or specific
 	 *         data structure.
+	 *
+	 *    All above inline items should follow the order:
+	 *
+	 *    - All btrfs_extent_inline_ref::type should be in an ascending
+	 *      order
+	 *
+	 *    - Within the same type, the items should follow a descending
+	 *      order by their sequence number. The sequence number is
+	 *      determined by:
+	 *      * btrfs_extent_inline_ref::offset for all types  other than
+	 *        EXTENT_DATA_REF
+	 *      * hash_extent_data_ref() for EXTENT_DATA_REF
 	 */
 	if (unlikely(item_size < sizeof(*ei))) {
 		extent_err(leaf, slot,
@@ -1403,6 +1418,7 @@ static int check_extent_item(struct extent_buffer *leaf,
 		struct btrfs_extent_inline_ref *iref;
 		struct btrfs_extent_data_ref *dref;
 		struct btrfs_shared_data_ref *sref;
+		u64 seq;
 		u64 dref_offset;
 		u64 inline_offset;
 		u8 inline_type;
@@ -1416,6 +1432,7 @@ static int check_extent_item(struct extent_buffer *leaf,
 		iref = (struct btrfs_extent_inline_ref *)ptr;
 		inline_type = btrfs_extent_inline_ref_type(leaf, iref);
 		inline_offset = btrfs_extent_inline_ref_offset(leaf, iref);
+		seq = inline_offset;
 		if (unlikely(ptr + btrfs_extent_inline_ref_size(inline_type) > end)) {
 			extent_err(leaf, slot,
 "inline ref item overflows extent item, ptr %lu iref size %u end %lu",
@@ -1446,6 +1463,10 @@ static int check_extent_item(struct extent_buffer *leaf,
 		case BTRFS_EXTENT_DATA_REF_KEY:
 			dref = (struct btrfs_extent_data_ref *)(&iref->offset);
 			dref_offset = btrfs_extent_data_ref_offset(leaf, dref);
+			seq = hash_extent_data_ref(
+					btrfs_extent_data_ref_root(leaf, dref),
+					btrfs_extent_data_ref_objectid(leaf, dref),
+					btrfs_extent_data_ref_offset(leaf, dref));
 			if (unlikely(!IS_ALIGNED(dref_offset,
 						 fs_info->sectorsize))) {
 				extent_err(leaf, slot,
@@ -1475,6 +1496,24 @@ static int check_extent_item(struct extent_buffer *leaf,
 				   inline_type);
 			return -EUCLEAN;
 		}
+		if (inline_type < last_type) {
+			extent_err(leaf, slot,
+				   "inline ref out-of-order: has type %u, prev type %u",
+				   inline_type, last_type);
+			return -EUCLEAN;
+		}
+		/* Type changed, allow the sequence starts from U64_MAX again. */
+		if (inline_type > last_type)
+			last_seq = U64_MAX;
+		if (seq > last_seq) {
+			extent_err(leaf, slot,
+"inline ref out-of-order: has type %u offset %llu seq 0x%llx, prev type %u seq 0x%llx",
+				   inline_type, inline_offset, seq,
+				   last_type, last_seq);
+			return -EUCLEAN;
+		}
+		last_type = inline_type;
+		last_seq = seq;
 		ptr += btrfs_extent_inline_ref_size(inline_type);
 	}
 	/* No padding is allowed */
diff --git a/fs/buffer.c b/fs/buffer.c
index 967f34b70aa8..f653add0aed5 100644
--- a/fs/buffer.c
+++ b/fs/buffer.c
@@ -180,11 +180,11 @@ EXPORT_SYMBOL(end_buffer_write_sync);
  * Various filesystems appear to want __find_get_block to be non-blocking.
  * But it's the page lock which protects the buffers.  To get around this,
  * we get exclusion from try_to_free_buffers with the blockdev mapping's
- * private_lock.
+ * i_private_lock.
  *
- * Hack idea: for the blockdev mapping, private_lock contention
+ * Hack idea: for the blockdev mapping, i_private_lock contention
  * may be quite high.  This code could TryLock the page, and if that
- * succeeds, there is no need to take private_lock.
+ * succeeds, there is no need to take i_private_lock.
  */
 static struct buffer_head *
 __find_get_block_slow(struct block_device *bdev, sector_t block)
@@ -199,12 +199,12 @@ __find_get_block_slow(struct block_device *bdev, sector_t block)
 	int all_mapped = 1;
 	static DEFINE_RATELIMIT_STATE(last_warned, HZ, 1);
 
-	index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
+	index = ((loff_t)block << bd_inode->i_blkbits) / PAGE_SIZE;
 	folio = __filemap_get_folio(bd_mapping, index, FGP_ACCESSED, 0);
 	if (IS_ERR(folio))
 		goto out;
 
-	spin_lock(&bd_mapping->private_lock);
+	spin_lock(&bd_mapping->i_private_lock);
 	head = folio_buffers(folio);
 	if (!head)
 		goto out_unlock;
@@ -236,7 +236,7 @@ __find_get_block_slow(struct block_device *bdev, sector_t block)
 		       1 << bd_inode->i_blkbits);
 	}
 out_unlock:
-	spin_unlock(&bd_mapping->private_lock);
+	spin_unlock(&bd_mapping->i_private_lock);
 	folio_put(folio);
 out:
 	return ret;
@@ -467,25 +467,25 @@ EXPORT_SYMBOL(mark_buffer_async_write);
  *
  * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
  * inode_has_buffers() and invalidate_inode_buffers() are provided for the
- * management of a list of dependent buffers at ->i_mapping->private_list.
+ * management of a list of dependent buffers at ->i_mapping->i_private_list.
  *
  * Locking is a little subtle: try_to_free_buffers() will remove buffers
  * from their controlling inode's queue when they are being freed.  But
  * try_to_free_buffers() will be operating against the *blockdev* mapping
  * at the time, not against the S_ISREG file which depends on those buffers.
- * So the locking for private_list is via the private_lock in the address_space
+ * So the locking for i_private_list is via the i_private_lock in the address_space
  * which backs the buffers.  Which is different from the address_space 
  * against which the buffers are listed.  So for a particular address_space,
- * mapping->private_lock does *not* protect mapping->private_list!  In fact,
- * mapping->private_list will always be protected by the backing blockdev's
- * ->private_lock.
+ * mapping->i_private_lock does *not* protect mapping->i_private_list!  In fact,
+ * mapping->i_private_list will always be protected by the backing blockdev's
+ * ->i_private_lock.
  *
  * Which introduces a requirement: all buffers on an address_space's
- * ->private_list must be from the same address_space: the blockdev's.
+ * ->i_private_list must be from the same address_space: the blockdev's.
  *
- * address_spaces which do not place buffers at ->private_list via these
- * utility functions are free to use private_lock and private_list for
- * whatever they want.  The only requirement is that list_empty(private_list)
+ * address_spaces which do not place buffers at ->i_private_list via these
+ * utility functions are free to use i_private_lock and i_private_list for
+ * whatever they want.  The only requirement is that list_empty(i_private_list)
  * be true at clear_inode() time.
  *
  * FIXME: clear_inode should not call invalidate_inode_buffers().  The
@@ -508,7 +508,7 @@ EXPORT_SYMBOL(mark_buffer_async_write);
  */
 
 /*
- * The buffer's backing address_space's private_lock must be held
+ * The buffer's backing address_space's i_private_lock must be held
  */
 static void __remove_assoc_queue(struct buffer_head *bh)
 {
@@ -519,7 +519,7 @@ static void __remove_assoc_queue(struct buffer_head *bh)
 
 int inode_has_buffers(struct inode *inode)
 {
-	return !list_empty(&inode->i_data.private_list);
+	return !list_empty(&inode->i_data.i_private_list);
 }
 
 /*
@@ -561,7 +561,7 @@ static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
  * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
  * @mapping: the mapping which wants those buffers written
  *
- * Starts I/O against the buffers at mapping->private_list, and waits upon
+ * Starts I/O against the buffers at mapping->i_private_list, and waits upon
  * that I/O.
  *
  * Basically, this is a convenience function for fsync().
@@ -570,13 +570,13 @@ static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
  */
 int sync_mapping_buffers(struct address_space *mapping)
 {
-	struct address_space *buffer_mapping = mapping->private_data;
+	struct address_space *buffer_mapping = mapping->i_private_data;
 
-	if (buffer_mapping == NULL || list_empty(&mapping->private_list))
+	if (buffer_mapping == NULL || list_empty(&mapping->i_private_list))
 		return 0;
 
-	return fsync_buffers_list(&buffer_mapping->private_lock,
-					&mapping->private_list);
+	return fsync_buffers_list(&buffer_mapping->i_private_lock,
+					&mapping->i_private_list);
 }
 EXPORT_SYMBOL(sync_mapping_buffers);
 
@@ -673,17 +673,17 @@ void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
 	struct address_space *buffer_mapping = bh->b_folio->mapping;
 
 	mark_buffer_dirty(bh);
-	if (!mapping->private_data) {
-		mapping->private_data = buffer_mapping;
+	if (!mapping->i_private_data) {
+		mapping->i_private_data = buffer_mapping;
 	} else {
-		BUG_ON(mapping->private_data != buffer_mapping);
+		BUG_ON(mapping->i_private_data != buffer_mapping);
 	}
 	if (!bh->b_assoc_map) {
-		spin_lock(&buffer_mapping->private_lock);
+		spin_lock(&buffer_mapping->i_private_lock);
 		list_move_tail(&bh->b_assoc_buffers,
-				&mapping->private_list);
+				&mapping->i_private_list);
 		bh->b_assoc_map = mapping;
-		spin_unlock(&buffer_mapping->private_lock);
+		spin_unlock(&buffer_mapping->i_private_lock);
 	}
 }
 EXPORT_SYMBOL(mark_buffer_dirty_inode);
@@ -706,7 +706,7 @@ EXPORT_SYMBOL(mark_buffer_dirty_inode);
  * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
  * page on the dirty page list.
  *
- * We use private_lock to lock against try_to_free_buffers while using the
+ * We use i_private_lock to lock against try_to_free_buffers while using the
  * page's buffer list.  Also use this to protect against clean buffers being
  * added to the page after it was set dirty.
  *
@@ -718,7 +718,7 @@ bool block_dirty_folio(struct address_space *mapping, struct folio *folio)
 	struct buffer_head *head;
 	bool newly_dirty;
 
-	spin_lock(&mapping->private_lock);
+	spin_lock(&mapping->i_private_lock);
 	head = folio_buffers(folio);
 	if (head) {
 		struct buffer_head *bh = head;
@@ -734,7 +734,7 @@ bool block_dirty_folio(struct address_space *mapping, struct folio *folio)
 	 */
 	folio_memcg_lock(folio);
 	newly_dirty = !folio_test_set_dirty(folio);
-	spin_unlock(&mapping->private_lock);
+	spin_unlock(&mapping->i_private_lock);
 
 	if (newly_dirty)
 		__folio_mark_dirty(folio, mapping, 1);
@@ -827,7 +827,7 @@ static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
 		smp_mb();
 		if (buffer_dirty(bh)) {
 			list_add(&bh->b_assoc_buffers,
-				 &mapping->private_list);
+				 &mapping->i_private_list);
 			bh->b_assoc_map = mapping;
 		}
 		spin_unlock(lock);
@@ -851,7 +851,7 @@ static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
  * probably unmounting the fs, but that doesn't mean we have already
  * done a sync().  Just drop the buffers from the inode list.
  *
- * NOTE: we take the inode's blockdev's mapping's private_lock.  Which
+ * NOTE: we take the inode's blockdev's mapping's i_private_lock.  Which
  * assumes that all the buffers are against the blockdev.  Not true
  * for reiserfs.
  */
@@ -859,13 +859,13 @@ void invalidate_inode_buffers(struct inode *inode)
 {
 	if (inode_has_buffers(inode)) {
 		struct address_space *mapping = &inode->i_data;
-		struct list_head *list = &mapping->private_list;
-		struct address_space *buffer_mapping = mapping->private_data;
+		struct list_head *list = &mapping->i_private_list;
+		struct address_space *buffer_mapping = mapping->i_private_data;
 
-		spin_lock(&buffer_mapping->private_lock);
+		spin_lock(&buffer_mapping->i_private_lock);
 		while (!list_empty(list))
 			__remove_assoc_queue(BH_ENTRY(list->next));
-		spin_unlock(&buffer_mapping->private_lock);
+		spin_unlock(&buffer_mapping->i_private_lock);
 	}
 }
 EXPORT_SYMBOL(invalidate_inode_buffers);
@@ -882,10 +882,10 @@ int remove_inode_buffers(struct inode *inode)
 
 	if (inode_has_buffers(inode)) {
 		struct address_space *mapping = &inode->i_data;
-		struct list_head *list = &mapping->private_list;
-		struct address_space *buffer_mapping = mapping->private_data;
+		struct list_head *list = &mapping->i_private_list;
+		struct address_space *buffer_mapping = mapping->i_private_data;
 
-		spin_lock(&buffer_mapping->private_lock);
+		spin_lock(&buffer_mapping->i_private_lock);
 		while (!list_empty(list)) {
 			struct buffer_head *bh = BH_ENTRY(list->next);
 			if (buffer_dirty(bh)) {
@@ -894,7 +894,7 @@ int remove_inode_buffers(struct inode *inode)
 			}
 			__remove_assoc_queue(bh);
 		}
-		spin_unlock(&buffer_mapping->private_lock);
+		spin_unlock(&buffer_mapping->i_private_lock);
 	}
 	return ret;
 }
@@ -995,11 +995,12 @@ static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
  * Initialise the state of a blockdev folio's buffers.
  */ 
 static sector_t folio_init_buffers(struct folio *folio,
-		struct block_device *bdev, sector_t block, int size)
+		struct block_device *bdev, unsigned size)
 {
 	struct buffer_head *head = folio_buffers(folio);
 	struct buffer_head *bh = head;
 	bool uptodate = folio_test_uptodate(folio);
+	sector_t block = div_u64(folio_pos(folio), size);
 	sector_t end_block = blkdev_max_block(bdev, size);
 
 	do {
@@ -1024,86 +1025,81 @@ static sector_t folio_init_buffers(struct folio *folio,
 }
 
 /*
- * Create the page-cache page that contains the requested block.
+ * Create the page-cache folio that contains the requested block.
  *
  * This is used purely for blockdev mappings.
+ *
+ * Returns false if we have a 'permanent' failure.  Returns true if
+ * we succeeded, or the caller should retry.
  */
-static int
-grow_dev_page(struct block_device *bdev, sector_t block,
-	      pgoff_t index, int size, int sizebits, gfp_t gfp)
+static bool grow_dev_folio(struct block_device *bdev, sector_t block,
+		pgoff_t index, unsigned size, gfp_t gfp)
 {
 	struct inode *inode = bdev->bd_inode;
 	struct folio *folio;
 	struct buffer_head *bh;
-	sector_t end_block;
-	int ret = 0;
+	sector_t end_block = 0;
 
 	folio = __filemap_get_folio(inode->i_mapping, index,
 			FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp);
 	if (IS_ERR(folio))
-		return PTR_ERR(folio);
+		return false;
 
 	bh = folio_buffers(folio);
 	if (bh) {
 		if (bh->b_size == size) {
-			end_block = folio_init_buffers(folio, bdev,
-					(sector_t)index << sizebits, size);
-			goto done;
+			end_block = folio_init_buffers(folio, bdev, size);
+			goto unlock;
 		}
+
+		/* Caller should retry if this call fails */
+		end_block = ~0ULL;
 		if (!try_to_free_buffers(folio))
-			goto failed;
+			goto unlock;
 	}
 
-	ret = -ENOMEM;
 	bh = folio_alloc_buffers(folio, size, gfp | __GFP_ACCOUNT);
 	if (!bh)
-		goto failed;
+		goto unlock;
 
 	/*
 	 * Link the folio to the buffers and initialise them.  Take the
 	 * lock to be atomic wrt __find_get_block(), which does not
 	 * run under the folio lock.
 	 */
-	spin_lock(&inode->i_mapping->private_lock);
+	spin_lock(&inode->i_mapping->i_private_lock);
 	link_dev_buffers(folio, bh);
-	end_block = folio_init_buffers(folio, bdev,
-			(sector_t)index << sizebits, size);
-	spin_unlock(&inode->i_mapping->private_lock);
-done:
-	ret = (block < end_block) ? 1 : -ENXIO;
-failed:
+	end_block = folio_init_buffers(folio, bdev, size);
+	spin_unlock(&inode->i_mapping->i_private_lock);
+unlock:
 	folio_unlock(folio);
 	folio_put(folio);
-	return ret;
+	return block < end_block;
 }
 
 /*
- * Create buffers for the specified block device block's page.  If
- * that page was dirty, the buffers are set dirty also.
+ * Create buffers for the specified block device block's folio.  If
+ * that folio was dirty, the buffers are set dirty also.  Returns false
+ * if we've hit a permanent error.
  */
-static int
-grow_buffers(struct block_device *bdev, sector_t block, int size, gfp_t gfp)
+static bool grow_buffers(struct block_device *bdev, sector_t block,
+		unsigned size, gfp_t gfp)
 {
-	pgoff_t index;
-	int sizebits;
-
-	sizebits = PAGE_SHIFT - __ffs(size);
-	index = block >> sizebits;
+	loff_t pos;
 
 	/*
-	 * Check for a block which wants to lie outside our maximum possible
-	 * pagecache index.  (this comparison is done using sector_t types).
+	 * Check for a block which lies outside our maximum possible
+	 * pagecache index.
 	 */
-	if (unlikely(index != block >> sizebits)) {
-		printk(KERN_ERR "%s: requested out-of-range block %llu for "
-			"device %pg\n",
+	if (check_mul_overflow(block, size, &pos) || pos > MAX_LFS_FILESIZE) {
+		printk(KERN_ERR "%s: requested out-of-range block %llu for device %pg\n",
 			__func__, (unsigned long long)block,
 			bdev);
-		return -EIO;
+		return false;
 	}
 
-	/* Create a page with the proper size buffers.. */
-	return grow_dev_page(bdev, block, index, size, sizebits, gfp);
+	/* Create a folio with the proper size buffers */
+	return grow_dev_folio(bdev, block, pos / PAGE_SIZE, size, gfp);
 }
 
 static struct buffer_head *
@@ -1124,14 +1120,12 @@ __getblk_slow(struct block_device *bdev, sector_t block,
 
 	for (;;) {
 		struct buffer_head *bh;
-		int ret;
 
 		bh = __find_get_block(bdev, block, size);
 		if (bh)
 			return bh;
 
-		ret = grow_buffers(bdev, block, size, gfp);
-		if (ret < 0)
+		if (!grow_buffers(bdev, block, size, gfp))
 			return NULL;
 	}
 }
@@ -1168,7 +1162,7 @@ __getblk_slow(struct block_device *bdev, sector_t block,
  * and then attach the address_space's inode to its superblock's dirty
  * inode list.
  *
- * mark_buffer_dirty() is atomic.  It takes bh->b_folio->mapping->private_lock,
+ * mark_buffer_dirty() is atomic.  It takes bh->b_folio->mapping->i_private_lock,
  * i_pages lock and mapping->host->i_lock.
  */
 void mark_buffer_dirty(struct buffer_head *bh)
@@ -1246,10 +1240,10 @@ void __bforget(struct buffer_head *bh)
 	if (bh->b_assoc_map) {
 		struct address_space *buffer_mapping = bh->b_folio->mapping;
 
-		spin_lock(&buffer_mapping->private_lock);
+		spin_lock(&buffer_mapping->i_private_lock);
 		list_del_init(&bh->b_assoc_buffers);
 		bh->b_assoc_map = NULL;
-		spin_unlock(&buffer_mapping->private_lock);
+		spin_unlock(&buffer_mapping->i_private_lock);
 	}
 	__brelse(bh);
 }
@@ -1638,7 +1632,7 @@ EXPORT_SYMBOL(block_invalidate_folio);
 
 /*
  * We attach and possibly dirty the buffers atomically wrt
- * block_dirty_folio() via private_lock.  try_to_free_buffers
+ * block_dirty_folio() via i_private_lock.  try_to_free_buffers
  * is already excluded via the folio lock.
  */
 struct buffer_head *create_empty_buffers(struct folio *folio,
@@ -1656,7 +1650,7 @@ struct buffer_head *create_empty_buffers(struct folio *folio,
 	} while (bh);
 	tail->b_this_page = head;
 
-	spin_lock(&folio->mapping->private_lock);
+	spin_lock(&folio->mapping->i_private_lock);
 	if (folio_test_uptodate(folio) || folio_test_dirty(folio)) {
 		bh = head;
 		do {
@@ -1668,7 +1662,7 @@ struct buffer_head *create_empty_buffers(struct folio *folio,
 		} while (bh != head);
 	}
 	folio_attach_private(folio, head);
-	spin_unlock(&folio->mapping->private_lock);
+	spin_unlock(&folio->mapping->i_private_lock);
 
 	return head;
 }
@@ -1699,13 +1693,13 @@ void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len)
 	struct inode *bd_inode = bdev->bd_inode;
 	struct address_space *bd_mapping = bd_inode->i_mapping;
 	struct folio_batch fbatch;
-	pgoff_t index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
+	pgoff_t index = ((loff_t)block << bd_inode->i_blkbits) / PAGE_SIZE;
 	pgoff_t end;
 	int i, count;
 	struct buffer_head *bh;
 	struct buffer_head *head;
 
-	end = (block + len - 1) >> (PAGE_SHIFT - bd_inode->i_blkbits);
+	end = ((loff_t)(block + len - 1) << bd_inode->i_blkbits) / PAGE_SIZE;
 	folio_batch_init(&fbatch);
 	while (filemap_get_folios(bd_mapping, &index, end, &fbatch)) {
 		count = folio_batch_count(&fbatch);
@@ -1715,7 +1709,7 @@ void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len)
 			if (!folio_buffers(folio))
 				continue;
 			/*
-			 * We use folio lock instead of bd_mapping->private_lock
+			 * We use folio lock instead of bd_mapping->i_private_lock
 			 * to pin buffers here since we can afford to sleep and
 			 * it scales better than a global spinlock lock.
 			 */
@@ -1748,19 +1742,6 @@ void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len)
 }
 EXPORT_SYMBOL(clean_bdev_aliases);
 
-/*
- * Size is a power-of-two in the range 512..PAGE_SIZE,
- * and the case we care about most is PAGE_SIZE.
- *
- * So this *could* possibly be written with those
- * constraints in mind (relevant mostly if some
- * architecture has a slow bit-scan instruction)
- */
-static inline int block_size_bits(unsigned int blocksize)
-{
-	return ilog2(blocksize);
-}
-
 static struct buffer_head *folio_create_buffers(struct folio *folio,
 						struct inode *inode,
 						unsigned int b_state)
@@ -1813,7 +1794,7 @@ int __block_write_full_folio(struct inode *inode, struct folio *folio,
 	sector_t block;
 	sector_t last_block;
 	struct buffer_head *bh, *head;
-	unsigned int blocksize, bbits;
+	size_t blocksize;
 	int nr_underway = 0;
 	blk_opf_t write_flags = wbc_to_write_flags(wbc);
 
@@ -1832,10 +1813,9 @@ int __block_write_full_folio(struct inode *inode, struct folio *folio,
 
 	bh = head;
 	blocksize = bh->b_size;
-	bbits = block_size_bits(blocksize);
 
-	block = (sector_t)folio->index << (PAGE_SHIFT - bbits);
-	last_block = (i_size_read(inode) - 1) >> bbits;
+	block = div_u64(folio_pos(folio), blocksize);
+	last_block = div_u64(i_size_read(inode) - 1, blocksize);
 
 	/*
 	 * Get all the dirty buffers mapped to disk addresses and
@@ -2014,7 +1994,7 @@ static int
 iomap_to_bh(struct inode *inode, sector_t block, struct buffer_head *bh,
 		const struct iomap *iomap)
 {
-	loff_t offset = block << inode->i_blkbits;
+	loff_t offset = (loff_t)block << inode->i_blkbits;
 
 	bh->b_bdev = iomap->bdev;
 
@@ -2081,27 +2061,24 @@ iomap_to_bh(struct inode *inode, sector_t block, struct buffer_head *bh,
 int __block_write_begin_int(struct folio *folio, loff_t pos, unsigned len,
 		get_block_t *get_block, const struct iomap *iomap)
 {
-	unsigned from = pos & (PAGE_SIZE - 1);
-	unsigned to = from + len;
+	size_t from = offset_in_folio(folio, pos);
+	size_t to = from + len;
 	struct inode *inode = folio->mapping->host;
-	unsigned block_start, block_end;
+	size_t block_start, block_end;
 	sector_t block;
 	int err = 0;
-	unsigned blocksize, bbits;
+	size_t blocksize;
 	struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
 
 	BUG_ON(!folio_test_locked(folio));
-	BUG_ON(from > PAGE_SIZE);
-	BUG_ON(to > PAGE_SIZE);
+	BUG_ON(to > folio_size(folio));
 	BUG_ON(from > to);
 
 	head = folio_create_buffers(folio, inode, 0);
 	blocksize = head->b_size;
-	bbits = block_size_bits(blocksize);
+	block = div_u64(folio_pos(folio), blocksize);
 
-	block = (sector_t)folio->index << (PAGE_SHIFT - bbits);
-
-	for(bh = head, block_start = 0; bh != head || !block_start;
+	for (bh = head, block_start = 0; bh != head || !block_start;
 	    block++, block_start=block_end, bh = bh->b_this_page) {
 		block_end = block_start + blocksize;
 		if (block_end <= from || block_start >= to) {
@@ -2364,7 +2341,7 @@ int block_read_full_folio(struct folio *folio, get_block_t *get_block)
 	struct inode *inode = folio->mapping->host;
 	sector_t iblock, lblock;
 	struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
-	unsigned int blocksize, bbits;
+	size_t blocksize;
 	int nr, i;
 	int fully_mapped = 1;
 	bool page_error = false;
@@ -2378,10 +2355,9 @@ int block_read_full_folio(struct folio *folio, get_block_t *get_block)
 
 	head = folio_create_buffers(folio, inode, 0);
 	blocksize = head->b_size;
-	bbits = block_size_bits(blocksize);
 
-	iblock = (sector_t)folio->index << (PAGE_SHIFT - bbits);
-	lblock = (limit+blocksize-1) >> bbits;
+	iblock = div_u64(folio_pos(folio), blocksize);
+	lblock = div_u64(limit + blocksize - 1, blocksize);
 	bh = head;
 	nr = 0;
 	i = 0;
@@ -2666,8 +2642,8 @@ int block_truncate_page(struct address_space *mapping,
 		return 0;
 
 	length = blocksize - length;
-	iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
-	
+	iblock = ((loff_t)index * PAGE_SIZE) >> inode->i_blkbits;
+
 	folio = filemap_grab_folio(mapping, index);
 	if (IS_ERR(folio))
 		return PTR_ERR(folio);
@@ -2883,7 +2859,7 @@ EXPORT_SYMBOL(sync_dirty_buffer);
  * are unused, and releases them if so.
  *
  * Exclusion against try_to_free_buffers may be obtained by either
- * locking the folio or by holding its mapping's private_lock.
+ * locking the folio or by holding its mapping's i_private_lock.
  *
  * If the folio is dirty but all the buffers are clean then we need to
  * be sure to mark the folio clean as well.  This is because the folio
@@ -2894,7 +2870,7 @@ EXPORT_SYMBOL(sync_dirty_buffer);
  * The same applies to regular filesystem folios: if all the buffers are
  * clean then we set the folio clean and proceed.  To do that, we require
  * total exclusion from block_dirty_folio().  That is obtained with
- * private_lock.
+ * i_private_lock.
  *
  * try_to_free_buffers() is non-blocking.
  */
@@ -2946,7 +2922,7 @@ bool try_to_free_buffers(struct folio *folio)
 		goto out;
 	}
 
-	spin_lock(&mapping->private_lock);
+	spin_lock(&mapping->i_private_lock);
 	ret = drop_buffers(folio, &buffers_to_free);
 
 	/*
@@ -2959,13 +2935,13 @@ bool try_to_free_buffers(struct folio *folio)
 	 * the folio's buffers clean.  We discover that here and clean
 	 * the folio also.
 	 *
-	 * private_lock must be held over this entire operation in order
+	 * i_private_lock must be held over this entire operation in order
 	 * to synchronise against block_dirty_folio and prevent the
 	 * dirty bit from being lost.
 	 */
 	if (ret)
 		folio_cancel_dirty(folio);
-	spin_unlock(&mapping->private_lock);
+	spin_unlock(&mapping->i_private_lock);
 out:
 	if (buffers_to_free) {
 		struct buffer_head *bh = buffers_to_free;
diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c
index 85be3bf18cdf..13af429ab030 100644
--- a/fs/ceph/addr.c
+++ b/fs/ceph/addr.c
@@ -907,8 +907,8 @@ static void writepages_finish(struct ceph_osd_request *req)
 			doutc(cl, "unlocking %p\n", page);
 
 			if (remove_page)
-				generic_error_remove_page(inode->i_mapping,
-							  page);
+				generic_error_remove_folio(inode->i_mapping,
+							  page_folio(page));
 
 			unlock_page(page);
 		}
diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c
index 2c0b8dc3dd0d..9c02f328c966 100644
--- a/fs/ceph/caps.c
+++ b/fs/ceph/caps.c
@@ -4887,13 +4887,15 @@ int ceph_encode_dentry_release(void **p, struct dentry *dentry,
 			       struct inode *dir,
 			       int mds, int drop, int unless)
 {
-	struct dentry *parent = NULL;
 	struct ceph_mds_request_release *rel = *p;
 	struct ceph_dentry_info *di = ceph_dentry(dentry);
 	struct ceph_client *cl;
 	int force = 0;
 	int ret;
 
+	/* This shouldn't happen */
+	BUG_ON(!dir);
+
 	/*
 	 * force an record for the directory caps if we have a dentry lease.
 	 * this is racy (can't take i_ceph_lock and d_lock together), but it
@@ -4903,14 +4905,9 @@ int ceph_encode_dentry_release(void **p, struct dentry *dentry,
 	spin_lock(&dentry->d_lock);
 	if (di->lease_session && di->lease_session->s_mds == mds)
 		force = 1;
-	if (!dir) {
-		parent = dget(dentry->d_parent);
-		dir = d_inode(parent);
-	}
 	spin_unlock(&dentry->d_lock);
 
 	ret = ceph_encode_inode_release(p, dir, mds, drop, unless, force);
-	dput(parent);
 
 	cl = ceph_inode_to_client(dir);
 	spin_lock(&dentry->d_lock);
diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c
index 91709934c8b1..61297690789c 100644
--- a/fs/ceph/dir.c
+++ b/fs/ceph/dir.c
@@ -1908,7 +1908,8 @@ static int dir_lease_is_valid(struct inode *dir, struct dentry *dentry,
 /*
  * Check if cached dentry can be trusted.
  */
-static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags)
+static int ceph_d_revalidate(struct dentry *dentry, const struct qstr *name,
+			     unsigned int flags)
 {
 	struct ceph_mds_client *mdsc = ceph_sb_to_fs_client(dentry->d_sb)->mdsc;
 	struct ceph_client *cl = mdsc->fsc->client;
@@ -1916,7 +1917,7 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags)
 	struct dentry *parent;
 	struct inode *dir, *inode;
 
-	valid = fscrypt_d_revalidate(dentry, flags);
+	valid = fscrypt_d_revalidate(dentry, name, flags);
 	if (valid <= 0)
 		return valid;
 
diff --git a/fs/coda/dir.c b/fs/coda/dir.c
index 4e552ba7bd43..6d6b18f02664 100644
--- a/fs/coda/dir.c
+++ b/fs/coda/dir.c
@@ -445,7 +445,8 @@ static int coda_readdir(struct file *coda_file, struct dir_context *ctx)
 }
 
 /* called when a cache lookup succeeds */
-static int coda_dentry_revalidate(struct dentry *de, unsigned int flags)
+static int coda_dentry_revalidate(struct dentry *de, const struct qstr *name,
+				  unsigned int flags)
 {
 	struct inode *inode;
 	struct coda_inode_info *cii;
diff --git a/fs/configfs/configfs_internal.h b/fs/configfs/configfs_internal.h
index e710a1782382..0b969d0eb8ff 100644
--- a/fs/configfs/configfs_internal.h
+++ b/fs/configfs/configfs_internal.h
@@ -55,6 +55,8 @@ struct configfs_dirent {
 #define CONFIGFS_USET_IN_MKDIR	0x0200
 #define CONFIGFS_USET_CREATING	0x0400
 #define CONFIGFS_NOT_PINNED	(CONFIGFS_ITEM_ATTR | CONFIGFS_ITEM_BIN_ATTR)
+#define CONFIGFS_PINNED \
+	(CONFIGFS_ROOT | CONFIGFS_DIR | CONFIGFS_ITEM_LINK)
 
 extern struct mutex configfs_symlink_mutex;
 extern spinlock_t configfs_dirent_lock;
@@ -73,8 +75,6 @@ extern int configfs_make_dirent(struct configfs_dirent *, struct dentry *,
 				void *, umode_t, int, struct configfs_fragment *);
 extern int configfs_dirent_is_ready(struct configfs_dirent *);
 
-extern void configfs_hash_and_remove(struct dentry * dir, const char * name);
-
 extern const unsigned char * configfs_get_name(struct configfs_dirent *sd);
 extern void configfs_drop_dentry(struct configfs_dirent *sd, struct dentry *parent);
 extern int configfs_setattr(struct mnt_idmap *idmap,
diff --git a/fs/configfs/dir.c b/fs/configfs/dir.c
index 18677cd4e62f..7d6cd4b366d5 100644
--- a/fs/configfs/dir.c
+++ b/fs/configfs/dir.c
@@ -207,7 +207,17 @@ static struct configfs_dirent *configfs_new_dirent(struct configfs_dirent *paren
 		return ERR_PTR(-ENOENT);
 	}
 	sd->s_frag = get_fragment(frag);
-	list_add(&sd->s_sibling, &parent_sd->s_children);
+
+	/*
+	 * configfs_lookup scans only for unpinned items. s_children is
+	 * partitioned so that configfs_lookup can bail out early.
+	 * CONFIGFS_PINNED and CONFIGFS_NOT_PINNED are not symmetrical.  readdir
+	 * cursors still need to be inserted at the front of the list.
+	 */
+	if (sd->s_type & CONFIGFS_PINNED)
+		list_add_tail(&sd->s_sibling, &parent_sd->s_children);
+	else
+		list_add(&sd->s_sibling, &parent_sd->s_children);
 	spin_unlock(&configfs_dirent_lock);
 
 	return sd;
@@ -220,10 +230,11 @@ static struct configfs_dirent *configfs_new_dirent(struct configfs_dirent *paren
  *
  * called with parent inode's i_mutex held
  */
-static int configfs_dirent_exists(struct configfs_dirent *parent_sd,
-				  const unsigned char *new)
+static int configfs_dirent_exists(struct dentry *dentry)
 {
-	struct configfs_dirent * sd;
+	struct configfs_dirent *parent_sd = dentry->d_parent->d_fsdata;
+	const unsigned char *new = dentry->d_name.name;
+	struct configfs_dirent *sd;
 
 	list_for_each_entry(sd, &parent_sd->s_children, s_sibling) {
 		if (sd->s_element) {
@@ -289,10 +300,6 @@ static int configfs_create_dir(struct config_item *item, struct dentry *dentry,
 
 	BUG_ON(!item);
 
-	error = configfs_dirent_exists(p->d_fsdata, dentry->d_name.name);
-	if (unlikely(error))
-		return error;
-
 	error = configfs_make_dirent(p->d_fsdata, dentry, item, mode,
 				     CONFIGFS_DIR | CONFIGFS_USET_CREATING,
 				     frag);
@@ -451,6 +458,18 @@ static struct dentry * configfs_lookup(struct inode *dir,
 
 	spin_lock(&configfs_dirent_lock);
 	list_for_each_entry(sd, &parent_sd->s_children, s_sibling) {
+
+		/*
+		 * s_children is partitioned, see configfs_new_dirent. The first
+		 * pinned item indicates we can stop scanning.
+		 */
+		if (sd->s_type & CONFIGFS_PINNED)
+			break;
+
+		/*
+		 * Note: CONFIGFS_PINNED and CONFIGFS_NOT_PINNED are asymmetric.
+		 * there may be a readdir cursor in this list
+		 */
 		if ((sd->s_type & CONFIGFS_NOT_PINNED) &&
 		    !strcmp(configfs_get_name(sd), dentry->d_name.name)) {
 			struct configfs_attribute *attr = sd->s_element;
@@ -1875,8 +1894,11 @@ int configfs_register_subsystem(struct configfs_subsystem *subsys)
 	if (dentry) {
 		d_add(dentry, NULL);
 
-		err = configfs_attach_group(sd->s_element, &group->cg_item,
-					    dentry, frag);
+		err = configfs_dirent_exists(dentry);
+		if (!err)
+			err = configfs_attach_group(sd->s_element,
+						    &group->cg_item,
+						    dentry, frag);
 		if (err) {
 			BUG_ON(d_inode(dentry));
 			d_drop(dentry);
diff --git a/fs/configfs/inode.c b/fs/configfs/inode.c
index dcc22f593e43..0937f7c4ab0e 100644
--- a/fs/configfs/inode.c
+++ b/fs/configfs/inode.c
@@ -217,27 +217,3 @@ void configfs_drop_dentry(struct configfs_dirent * sd, struct dentry * parent)
 	}
 }
 
-void configfs_hash_and_remove(struct dentry * dir, const char * name)
-{
-	struct configfs_dirent * sd;
-	struct configfs_dirent * parent_sd = dir->d_fsdata;
-
-	if (d_really_is_negative(dir))
-		/* no inode means this hasn't been made visible yet */
-		return;
-
-	inode_lock(d_inode(dir));
-	list_for_each_entry(sd, &parent_sd->s_children, s_sibling) {
-		if (!sd->s_element)
-			continue;
-		if (!strcmp(configfs_get_name(sd), name)) {
-			spin_lock(&configfs_dirent_lock);
-			list_del_init(&sd->s_sibling);
-			spin_unlock(&configfs_dirent_lock);
-			configfs_drop_dentry(sd, dir);
-			configfs_put(sd);
-			break;
-		}
-	}
-	inode_unlock(d_inode(dir));
-}
diff --git a/fs/crypto/fname.c b/fs/crypto/fname.c
index 7b3fc189593a..e0521bd97651 100644
--- a/fs/crypto/fname.c
+++ b/fs/crypto/fname.c
@@ -580,7 +580,8 @@ EXPORT_SYMBOL_GPL(fscrypt_fname_siphash);
  * Validate dentries in encrypted directories to make sure we aren't potentially
  * caching stale dentries after a key has been added.
  */
-int fscrypt_d_revalidate(struct dentry *dentry, unsigned int flags)
+int fscrypt_d_revalidate(struct dentry *dentry, const struct qstr *name,
+			 unsigned int flags)
 {
 	struct dentry *dir;
 	int err;
diff --git a/fs/dax.c b/fs/dax.c
index 3380b43cb6bb..423fc1607dfa 100644
--- a/fs/dax.c
+++ b/fs/dax.c
@@ -1128,7 +1128,7 @@ static int dax_iomap_copy_around(loff_t pos, uint64_t length, size_t align_size,
 	/* zero the edges if srcmap is a HOLE or IOMAP_UNWRITTEN */
 	bool zero_edge = srcmap->flags & IOMAP_F_SHARED ||
 			 srcmap->type == IOMAP_UNWRITTEN;
-	void *saddr = 0;
+	void *saddr = NULL;
 	int ret = 0;
 
 	if (!zero_edge) {
diff --git a/fs/dcache.c b/fs/dcache.c
index c82ae731df9a..26f46a52cd23 100644
--- a/fs/dcache.c
+++ b/fs/dcache.c
@@ -1957,6 +1957,14 @@ void d_set_fallthru(struct dentry *dentry)
 }
 EXPORT_SYMBOL(d_set_fallthru);
 
+void d_set_casefolded_name(struct dentry *dentry)
+{
+	spin_lock(&dentry->d_lock);
+	dentry->d_flags |= DCACHE_CASEFOLDED_NAME;
+	spin_unlock(&dentry->d_lock);
+}
+EXPORT_SYMBOL(d_set_casefolded_name);
+
 static unsigned d_flags_for_inode(struct inode *inode)
 {
 	unsigned add_flags = DCACHE_REGULAR_TYPE;
diff --git a/fs/direct-io.c b/fs/direct-io.c
index 20533266ade6..60456263a338 100644
--- a/fs/direct-io.c
+++ b/fs/direct-io.c
@@ -1114,7 +1114,7 @@ ssize_t __blockdev_direct_IO(struct kiocb *iocb, struct inode *inode,
 	loff_t offset = iocb->ki_pos;
 	const loff_t end = offset + count;
 	struct dio *dio;
-	struct dio_submit sdio = { 0, };
+	struct dio_submit sdio = { NULL, };
 	struct buffer_head map_bh = { 0, };
 	struct blk_plug plug;
 	unsigned long align = offset | iov_iter_alignment(iter);
diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c
index 67f8dd8a05ef..6296c62c10fa 100644
--- a/fs/dlm/lowcomms.c
+++ b/fs/dlm/lowcomms.c
@@ -1817,8 +1817,8 @@ static int dlm_tcp_bind(struct socket *sock)
 	memcpy(&src_addr, &dlm_local_addr[0], sizeof(src_addr));
 	make_sockaddr(&src_addr, 0, &addr_len);
 
-	result = sock->ops->bind(sock, (struct sockaddr *)&src_addr,
-				 addr_len);
+	result = kernel_bind(sock, (struct sockaddr *)&src_addr,
+			     addr_len);
 	if (result < 0) {
 		/* This *may* not indicate a critical error */
 		log_print("could not bind for connect: %d", result);
@@ -1830,7 +1830,7 @@ static int dlm_tcp_bind(struct socket *sock)
 static int dlm_tcp_connect(struct connection *con, struct socket *sock,
 			   struct sockaddr *addr, int addr_len)
 {
-	return sock->ops->connect(sock, addr, addr_len, O_NONBLOCK);
+	return kernel_connect(sock, addr, addr_len, O_NONBLOCK);
 }
 
 static int dlm_tcp_listen_validate(void)
@@ -1862,8 +1862,8 @@ static int dlm_tcp_listen_bind(struct socket *sock)
 
 	/* Bind to our port */
 	make_sockaddr(&dlm_local_addr[0], dlm_config.ci_tcp_port, &addr_len);
-	return sock->ops->bind(sock, (struct sockaddr *)&dlm_local_addr[0],
-			       addr_len);
+	return kernel_bind(sock, (struct sockaddr *)&dlm_local_addr[0],
+			   addr_len);
 }
 
 static const struct dlm_proto_ops dlm_tcp_ops = {
@@ -1888,12 +1888,12 @@ static int dlm_sctp_connect(struct connection *con, struct socket *sock,
 	int ret;
 
 	/*
-	 * Make sock->ops->connect() function return in specified time,
+	 * Make kernel_connect() function return in specified time,
 	 * since O_NONBLOCK argument in connect() function does not work here,
 	 * then, we should restore the default value of this attribute.
 	 */
 	sock_set_sndtimeo(sock->sk, 5);
-	ret = sock->ops->connect(sock, addr, addr_len, 0);
+	ret = kernel_connect(sock, addr, addr_len, 0);
 	sock_set_sndtimeo(sock->sk, 0);
 	return ret;
 }
diff --git a/fs/dlm/plock.c b/fs/dlm/plock.c
index e6b4c1a21446..d814c5121367 100644
--- a/fs/dlm/plock.c
+++ b/fs/dlm/plock.c
@@ -140,11 +140,12 @@ int dlm_posix_lock(dlm_lockspace_t *lockspace, u64 number, struct file *file,
 	op->info.optype		= DLM_PLOCK_OP_LOCK;
 	op->info.pid		= fl->fl_pid;
 	op->info.ex		= (fl->fl_type == F_WRLCK);
-	op->info.wait		= IS_SETLKW(cmd);
+	op->info.wait		= !!(fl->fl_flags & FL_SLEEP);
 	op->info.fsid		= ls->ls_global_id;
 	op->info.number		= number;
 	op->info.start		= fl->fl_start;
 	op->info.end		= fl->fl_end;
+	op->info.owner = (__u64)(long)fl->fl_owner;
 	/* async handling */
 	if (fl->fl_lmops && fl->fl_lmops->lm_grant) {
 		op_data = kzalloc(sizeof(*op_data), GFP_NOFS);
@@ -154,9 +155,6 @@ int dlm_posix_lock(dlm_lockspace_t *lockspace, u64 number, struct file *file,
 			goto out;
 		}
 
-		/* fl_owner is lockd which doesn't distinguish
-		   processes on the nfs client */
-		op->info.owner	= (__u64) fl->fl_pid;
 		op_data->callback = fl->fl_lmops->lm_grant;
 		locks_init_lock(&op_data->flc);
 		locks_copy_lock(&op_data->flc, fl);
@@ -168,8 +166,6 @@ int dlm_posix_lock(dlm_lockspace_t *lockspace, u64 number, struct file *file,
 		send_op(op);
 		rv = FILE_LOCK_DEFERRED;
 		goto out;
-	} else {
-		op->info.owner	= (__u64)(long) fl->fl_owner;
 	}
 
 	send_op(op);
@@ -326,10 +322,7 @@ int dlm_posix_unlock(dlm_lockspace_t *lockspace, u64 number, struct file *file,
 	op->info.number		= number;
 	op->info.start		= fl->fl_start;
 	op->info.end		= fl->fl_end;
-	if (fl->fl_lmops && fl->fl_lmops->lm_grant)
-		op->info.owner	= (__u64) fl->fl_pid;
-	else
-		op->info.owner	= (__u64)(long) fl->fl_owner;
+	op->info.owner = (__u64)(long)fl->fl_owner;
 
 	if (fl->fl_flags & FL_CLOSE) {
 		op->info.flags |= DLM_PLOCK_FL_CLOSE;
@@ -389,7 +382,7 @@ int dlm_posix_cancel(dlm_lockspace_t *lockspace, u64 number, struct file *file,
 	info.number = number;
 	info.start = fl->fl_start;
 	info.end = fl->fl_end;
-	info.owner = (__u64)fl->fl_pid;
+	info.owner = (__u64)(long)fl->fl_owner;
 
 	rv = do_lock_cancel(&info);
 	switch (rv) {
@@ -450,10 +443,7 @@ int dlm_posix_get(dlm_lockspace_t *lockspace, u64 number, struct file *file,
 	op->info.number		= number;
 	op->info.start		= fl->fl_start;
 	op->info.end		= fl->fl_end;
-	if (fl->fl_lmops && fl->fl_lmops->lm_grant)
-		op->info.owner	= (__u64) fl->fl_pid;
-	else
-		op->info.owner	= (__u64)(long) fl->fl_owner;
+	op->info.owner = (__u64)(long)fl->fl_owner;
 
 	send_op(op);
 	wait_event(recv_wq, (op->done != 0));
diff --git a/fs/ecryptfs/crypto.c b/fs/ecryptfs/crypto.c
index 03bd55069d86..044863c0d824 100644
--- a/fs/ecryptfs/crypto.c
+++ b/fs/ecryptfs/crypto.c
@@ -1289,7 +1289,7 @@ static int ecryptfs_read_headers_virt(char *page_virt,
 
 /**
  * ecryptfs_read_xattr_region
- * @page_virt: The vitual address into which to read the xattr data
+ * @page_virt: The virtual address into which to read the xattr data
  * @ecryptfs_inode: The eCryptfs inode
  *
  * Attempts to read the crypto metadata from the extended attribute
diff --git a/fs/ecryptfs/dentry.c b/fs/ecryptfs/dentry.c
index acaa0825e9bb..ce86891a1711 100644
--- a/fs/ecryptfs/dentry.c
+++ b/fs/ecryptfs/dentry.c
@@ -18,6 +18,7 @@
 /**
  * ecryptfs_d_revalidate - revalidate an ecryptfs dentry
  * @dentry: The ecryptfs dentry
+ * @name: The name under lookup
  * @flags: lookup flags
  *
  * Called when the VFS needs to revalidate a dentry. This
@@ -28,7 +29,8 @@
  * Returns 1 if valid, 0 otherwise.
  *
  */
-static int ecryptfs_d_revalidate(struct dentry *dentry, unsigned int flags)
+static int ecryptfs_d_revalidate(struct dentry *dentry,
+				 const struct qstr *name, unsigned int flags)
 {
 	struct dentry *lower_dentry = ecryptfs_dentry_to_lower(dentry);
 	int rc = 1;
@@ -37,7 +39,7 @@ static int ecryptfs_d_revalidate(struct dentry *dentry, unsigned int flags)
 		return -ECHILD;
 
 	if (lower_dentry->d_flags & DCACHE_OP_REVALIDATE)
-		rc = lower_dentry->d_op->d_revalidate(lower_dentry, flags);
+		rc = lower_dentry->d_op->d_revalidate(lower_dentry, name, flags);
 
 	if (d_really_is_positive(dentry)) {
 		struct inode *inode = d_inode(dentry);
diff --git a/fs/ecryptfs/inode.c b/fs/ecryptfs/inode.c
index a25dd3d20008..d7193687b9b4 100644
--- a/fs/ecryptfs/inode.c
+++ b/fs/ecryptfs/inode.c
@@ -78,6 +78,14 @@ static struct inode *__ecryptfs_get_inode(struct inode *lower_inode,
 
 	if (lower_inode->i_sb != ecryptfs_superblock_to_lower(sb))
 		return ERR_PTR(-EXDEV);
+
+	/* Reject dealing with casefold directories. */
+	if (IS_CASEFOLDED(lower_inode)) {
+		pr_err_ratelimited("%s: Can't handle casefolded directory.\n",
+				   __func__);
+		return ERR_PTR(-EREMOTE);
+	}
+
 	if (!igrab(lower_inode))
 		return ERR_PTR(-ESTALE);
 	inode = iget5_locked(sb, (unsigned long)lower_inode,
@@ -998,6 +1006,14 @@ static int ecryptfs_getattr_link(struct mnt_idmap *idmap,
 	return rc;
 }
 
+static int ecryptfs_do_getattr(const struct path *path, struct kstat *stat,
+			       u32 request_mask, unsigned int flags)
+{
+	if (flags & AT_GETATTR_NOSEC)
+		return vfs_getattr_nosec(path, stat, request_mask, flags);
+	return vfs_getattr(path, stat, request_mask, flags);
+}
+
 static int ecryptfs_getattr(struct mnt_idmap *idmap,
 			    const struct path *path, struct kstat *stat,
 			    u32 request_mask, unsigned int flags)
@@ -1006,8 +1022,8 @@ static int ecryptfs_getattr(struct mnt_idmap *idmap,
 	struct kstat lower_stat;
 	int rc;
 
-	rc = vfs_getattr(ecryptfs_dentry_to_lower_path(dentry), &lower_stat,
-			 request_mask, flags);
+	rc = ecryptfs_do_getattr(ecryptfs_dentry_to_lower_path(dentry),
+				 &lower_stat, request_mask, flags);
 	if (!rc) {
 		fsstack_copy_attr_all(d_inode(dentry),
 				      ecryptfs_inode_to_lower(d_inode(dentry)));
diff --git a/fs/ecryptfs/keystore.c b/fs/ecryptfs/keystore.c
index 3fe41964c0d8..2452d6fd7062 100644
--- a/fs/ecryptfs/keystore.c
+++ b/fs/ecryptfs/keystore.c
@@ -878,7 +878,7 @@ struct ecryptfs_parse_tag_70_packet_silly_stack {
  * @filename: This function kmalloc's the memory for the filename
  * @filename_size: This function sets this to the amount of memory
  *                 kmalloc'd for the filename
- * @packet_size: This function sets this to the the number of octets
+ * @packet_size: This function sets this to the number of octets
  *               in the packet parsed
  * @mount_crypt_stat: The mount-wide cryptographic context
  * @data: The memory location containing the start of the tag 70
diff --git a/fs/erofs/Kconfig b/fs/erofs/Kconfig
index e540648dedc2..1d318f85232d 100644
--- a/fs/erofs/Kconfig
+++ b/fs/erofs/Kconfig
@@ -21,7 +21,7 @@ config EROFS_FS
 	  performance under extremely memory pressure without extra cost.
 
 	  See the documentation at <file:Documentation/filesystems/erofs.rst>
-	  for more details.
+	  and the web pages at <https://erofs.docs.kernel.org> for more details.
 
 	  If unsure, say N.
 
diff --git a/fs/erofs/data.c b/fs/erofs/data.c
index 029c761670bf..c98aeda8abb2 100644
--- a/fs/erofs/data.c
+++ b/fs/erofs/data.c
@@ -220,7 +220,7 @@ int erofs_map_dev(struct super_block *sb, struct erofs_map_dev *map)
 			up_read(&devs->rwsem);
 			return 0;
 		}
-		map->m_bdev = dif->bdev_handle->bdev;
+		map->m_bdev = dif->bdev_handle ? dif->bdev_handle->bdev : NULL;
 		map->m_daxdev = dif->dax_dev;
 		map->m_dax_part_off = dif->dax_part_off;
 		map->m_fscache = dif->fscache;
@@ -238,7 +238,8 @@ int erofs_map_dev(struct super_block *sb, struct erofs_map_dev *map)
 			if (map->m_pa >= startoff &&
 			    map->m_pa < startoff + length) {
 				map->m_pa -= startoff;
-				map->m_bdev = dif->bdev_handle->bdev;
+				map->m_bdev = dif->bdev_handle ?
+					      dif->bdev_handle->bdev : NULL;
 				map->m_daxdev = dif->dax_dev;
 				map->m_dax_part_off = dif->dax_part_off;
 				map->m_fscache = dif->fscache;
diff --git a/fs/erofs/inode.c b/fs/erofs/inode.c
index b8ad05b4509d..14a79d3226ab 100644
--- a/fs/erofs/inode.c
+++ b/fs/erofs/inode.c
@@ -15,11 +15,11 @@ static void *erofs_read_inode(struct erofs_buf *buf,
 	struct erofs_sb_info *sbi = EROFS_SB(sb);
 	struct erofs_inode *vi = EROFS_I(inode);
 	const erofs_off_t inode_loc = erofs_iloc(inode);
-
 	erofs_blk_t blkaddr, nblks = 0;
 	void *kaddr;
 	struct erofs_inode_compact *dic;
 	struct erofs_inode_extended *die, *copied = NULL;
+	union erofs_inode_i_u iu;
 	unsigned int ifmt;
 	int err;
 
@@ -35,9 +35,8 @@ static void *erofs_read_inode(struct erofs_buf *buf,
 
 	dic = kaddr + *ofs;
 	ifmt = le16_to_cpu(dic->i_format);
-
 	if (ifmt & ~EROFS_I_ALL) {
-		erofs_err(inode->i_sb, "unsupported i_format %u of nid %llu",
+		erofs_err(sb, "unsupported i_format %u of nid %llu",
 			  ifmt, vi->nid);
 		err = -EOPNOTSUPP;
 		goto err_out;
@@ -45,7 +44,7 @@ static void *erofs_read_inode(struct erofs_buf *buf,
 
 	vi->datalayout = erofs_inode_datalayout(ifmt);
 	if (vi->datalayout >= EROFS_INODE_DATALAYOUT_MAX) {
-		erofs_err(inode->i_sb, "unsupported datalayout %u of nid %llu",
+		erofs_err(sb, "unsupported datalayout %u of nid %llu",
 			  vi->datalayout, vi->nid);
 		err = -EOPNOTSUPP;
 		goto err_out;
@@ -82,40 +81,15 @@ static void *erofs_read_inode(struct erofs_buf *buf,
 		vi->xattr_isize = erofs_xattr_ibody_size(die->i_xattr_icount);
 
 		inode->i_mode = le16_to_cpu(die->i_mode);
-		switch (inode->i_mode & S_IFMT) {
-		case S_IFREG:
-		case S_IFDIR:
-		case S_IFLNK:
-			vi->raw_blkaddr = le32_to_cpu(die->i_u.raw_blkaddr);
-			break;
-		case S_IFCHR:
-		case S_IFBLK:
-			inode->i_rdev =
-				new_decode_dev(le32_to_cpu(die->i_u.rdev));
-			break;
-		case S_IFIFO:
-		case S_IFSOCK:
-			inode->i_rdev = 0;
-			break;
-		default:
-			goto bogusimode;
-		}
+		iu = die->i_u;
 		i_uid_write(inode, le32_to_cpu(die->i_uid));
 		i_gid_write(inode, le32_to_cpu(die->i_gid));
 		set_nlink(inode, le32_to_cpu(die->i_nlink));
-
-		/* extended inode has its own timestamp */
+		/* each extended inode has its own timestamp */
 		inode_set_ctime(inode, le64_to_cpu(die->i_mtime),
 				le32_to_cpu(die->i_mtime_nsec));
 
 		inode->i_size = le64_to_cpu(die->i_size);
-
-		/* total blocks for compressed files */
-		if (erofs_inode_is_data_compressed(vi->datalayout))
-			nblks = le32_to_cpu(die->i_u.compressed_blocks);
-		else if (vi->datalayout == EROFS_INODE_CHUNK_BASED)
-			/* fill chunked inode summary info */
-			vi->chunkformat = le16_to_cpu(die->i_u.c.format);
 		kfree(copied);
 		copied = NULL;
 		break;
@@ -125,49 +99,51 @@ static void *erofs_read_inode(struct erofs_buf *buf,
 		vi->xattr_isize = erofs_xattr_ibody_size(dic->i_xattr_icount);
 
 		inode->i_mode = le16_to_cpu(dic->i_mode);
-		switch (inode->i_mode & S_IFMT) {
-		case S_IFREG:
-		case S_IFDIR:
-		case S_IFLNK:
-			vi->raw_blkaddr = le32_to_cpu(dic->i_u.raw_blkaddr);
-			break;
-		case S_IFCHR:
-		case S_IFBLK:
-			inode->i_rdev =
-				new_decode_dev(le32_to_cpu(dic->i_u.rdev));
-			break;
-		case S_IFIFO:
-		case S_IFSOCK:
-			inode->i_rdev = 0;
-			break;
-		default:
-			goto bogusimode;
-		}
+		iu = dic->i_u;
 		i_uid_write(inode, le16_to_cpu(dic->i_uid));
 		i_gid_write(inode, le16_to_cpu(dic->i_gid));
 		set_nlink(inode, le16_to_cpu(dic->i_nlink));
-
 		/* use build time for compact inodes */
 		inode_set_ctime(inode, sbi->build_time, sbi->build_time_nsec);
 
 		inode->i_size = le32_to_cpu(dic->i_size);
-		if (erofs_inode_is_data_compressed(vi->datalayout))
-			nblks = le32_to_cpu(dic->i_u.compressed_blocks);
-		else if (vi->datalayout == EROFS_INODE_CHUNK_BASED)
-			vi->chunkformat = le16_to_cpu(dic->i_u.c.format);
 		break;
 	default:
-		erofs_err(inode->i_sb,
-			  "unsupported on-disk inode version %u of nid %llu",
+		erofs_err(sb, "unsupported on-disk inode version %u of nid %llu",
 			  erofs_inode_version(ifmt), vi->nid);
 		err = -EOPNOTSUPP;
 		goto err_out;
 	}
 
-	if (vi->datalayout == EROFS_INODE_CHUNK_BASED) {
+	switch (inode->i_mode & S_IFMT) {
+	case S_IFREG:
+	case S_IFDIR:
+	case S_IFLNK:
+		vi->raw_blkaddr = le32_to_cpu(iu.raw_blkaddr);
+		break;
+	case S_IFCHR:
+	case S_IFBLK:
+		inode->i_rdev = new_decode_dev(le32_to_cpu(iu.rdev));
+		break;
+	case S_IFIFO:
+	case S_IFSOCK:
+		inode->i_rdev = 0;
+		break;
+	default:
+		erofs_err(sb, "bogus i_mode (%o) @ nid %llu", inode->i_mode,
+			  vi->nid);
+		err = -EFSCORRUPTED;
+		goto err_out;
+	}
+
+	/* total blocks for compressed files */
+	if (erofs_inode_is_data_compressed(vi->datalayout)) {
+		nblks = le32_to_cpu(iu.compressed_blocks);
+	} else if (vi->datalayout == EROFS_INODE_CHUNK_BASED) {
+		/* fill chunked inode summary info */
+		vi->chunkformat = le16_to_cpu(iu.c.format);
 		if (vi->chunkformat & ~EROFS_CHUNK_FORMAT_ALL) {
-			erofs_err(inode->i_sb,
-				  "unsupported chunk format %x of nid %llu",
+			erofs_err(sb, "unsupported chunk format %x of nid %llu",
 				  vi->chunkformat, vi->nid);
 			err = -EOPNOTSUPP;
 			goto err_out;
@@ -191,10 +167,6 @@ static void *erofs_read_inode(struct erofs_buf *buf,
 		inode->i_blocks = nblks << (sb->s_blocksize_bits - 9);
 	return kaddr;
 
-bogusimode:
-	erofs_err(inode->i_sb, "bogus i_mode (%o) @ nid %llu",
-		  inode->i_mode, vi->nid);
-	err = -EFSCORRUPTED;
 err_out:
 	DBG_BUGON(1);
 	kfree(copied);
diff --git a/fs/exec.c b/fs/exec.c
index 4aa19b24f281..ee43597cb453 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -1578,11 +1578,10 @@ static void check_unsafe_exec(struct linux_binprm *bprm)
 	 * will be able to manipulate the current directory, etc.
 	 * It would be nice to force an unshare instead...
 	 */
-	t = p;
 	n_fs = 1;
 	spin_lock(&p->fs->lock);
 	rcu_read_lock();
-	while_each_thread(p, t) {
+	for_other_threads(p, t) {
 		if (t->fs == p->fs)
 			n_fs++;
 	}
diff --git a/fs/exfat/namei.c b/fs/exfat/namei.c
index 5d737e0b639a..1b069d91eb83 100644
--- a/fs/exfat/namei.c
+++ b/fs/exfat/namei.c
@@ -31,7 +31,8 @@ static inline void exfat_d_version_set(struct dentry *dentry,
  * If it happened, the negative dentry isn't actually negative anymore.  So,
  * drop it.
  */
-static int exfat_d_revalidate(struct dentry *dentry, unsigned int flags)
+static int exfat_d_revalidate(struct dentry *dentry, const struct qstr *name,
+			      unsigned int flags)
 {
 	int ret;
 
diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c
index 464faf6c217e..5a4272b2c6b0 100644
--- a/fs/ext2/inode.c
+++ b/fs/ext2/inode.c
@@ -969,7 +969,7 @@ const struct address_space_operations ext2_aops = {
 	.writepages		= ext2_writepages,
 	.migrate_folio		= buffer_migrate_folio,
 	.is_partially_uptodate	= block_is_partially_uptodate,
-	.error_remove_page	= generic_error_remove_page,
+	.error_remove_folio	= generic_error_remove_folio,
 };
 
 static const struct address_space_operations ext2_dax_aops = {
diff --git a/fs/ext4/inline.c b/fs/ext4/inline.c
index 9a84a5f9fef4..d5bd1e3a5d36 100644
--- a/fs/ext4/inline.c
+++ b/fs/ext4/inline.c
@@ -502,9 +502,8 @@ static int ext4_read_inline_folio(struct inode *inode, struct folio *folio)
 	BUG_ON(len > PAGE_SIZE);
 	kaddr = kmap_local_folio(folio, 0);
 	ret = ext4_read_inline_data(inode, kaddr, len, &iloc);
-	flush_dcache_folio(folio);
+	kaddr = folio_zero_tail(folio, len, kaddr + len);
 	kunmap_local(kaddr);
-	folio_zero_segment(folio, len, folio_size(folio));
 	folio_mark_uptodate(folio);
 	brelse(iloc.bh);
 
diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
index 61277f7f8722..83ee4e0f46f4 100644
--- a/fs/ext4/inode.c
+++ b/fs/ext4/inode.c
@@ -1261,7 +1261,7 @@ static int write_end_fn(handle_t *handle, struct inode *inode,
  * We need to pick up the new inode size which generic_commit_write gave us
  * `file' can be NULL - eg, when called from page_symlink().
  *
- * ext4 never places buffers on inode->i_mapping->private_list.  metadata
+ * ext4 never places buffers on inode->i_mapping->i_private_list.  metadata
  * buffers are managed internally.
  */
 static int ext4_write_end(struct file *file,
@@ -3213,7 +3213,7 @@ static bool ext4_inode_datasync_dirty(struct inode *inode)
 	}
 
 	/* Any metadata buffers to write? */
-	if (!list_empty(&inode->i_mapping->private_list))
+	if (!list_empty(&inode->i_mapping->i_private_list))
 		return true;
 	return inode->i_state & I_DIRTY_DATASYNC;
 }
@@ -3564,7 +3564,7 @@ static const struct address_space_operations ext4_aops = {
 	.direct_IO		= noop_direct_IO,
 	.migrate_folio		= buffer_migrate_folio,
 	.is_partially_uptodate  = block_is_partially_uptodate,
-	.error_remove_page	= generic_error_remove_page,
+	.error_remove_folio	= generic_error_remove_folio,
 	.swap_activate		= ext4_iomap_swap_activate,
 };
 
@@ -3581,7 +3581,7 @@ static const struct address_space_operations ext4_journalled_aops = {
 	.direct_IO		= noop_direct_IO,
 	.migrate_folio		= buffer_migrate_folio_norefs,
 	.is_partially_uptodate  = block_is_partially_uptodate,
-	.error_remove_page	= generic_error_remove_page,
+	.error_remove_folio	= generic_error_remove_folio,
 	.swap_activate		= ext4_iomap_swap_activate,
 };
 
@@ -3598,7 +3598,7 @@ static const struct address_space_operations ext4_da_aops = {
 	.direct_IO		= noop_direct_IO,
 	.migrate_folio		= buffer_migrate_folio,
 	.is_partially_uptodate  = block_is_partially_uptodate,
-	.error_remove_page	= generic_error_remove_page,
+	.error_remove_folio	= generic_error_remove_folio,
 	.swap_activate		= ext4_iomap_swap_activate,
 };
 
diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c
index 4f931f80cb34..aa6be510eb8f 100644
--- a/fs/ext4/ioctl.c
+++ b/fs/ext4/ioctl.c
@@ -819,11 +819,11 @@ int ext4_force_shutdown(struct super_block *sb, u32 flags)
 
 	switch (flags) {
 	case EXT4_GOING_FLAGS_DEFAULT:
-		ret = freeze_bdev(sb->s_bdev);
+		ret = bdev_freeze(sb->s_bdev);
 		if (ret)
 			return ret;
 		set_bit(EXT4_FLAGS_SHUTDOWN, &sbi->s_ext4_flags);
-		thaw_bdev(sb->s_bdev);
+		bdev_thaw(sb->s_bdev);
 		break;
 	case EXT4_GOING_FLAGS_LOGFLUSH:
 		set_bit(EXT4_FLAGS_SHUTDOWN, &sbi->s_ext4_flags);
diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c
index d252935f9c8a..57a66b15b5e6 100644
--- a/fs/ext4/namei.c
+++ b/fs/ext4/namei.c
@@ -1763,6 +1763,10 @@ static struct buffer_head *ext4_lookup_entry(struct inode *dir,
 
 	err = ext4_fname_prepare_lookup(dir, dentry, &fname);
 	generic_set_encrypted_ci_d_ops(dentry);
+
+	if (IS_CASEFOLDED(dir))
+		d_set_casefolded_name(dentry);
+
 	if (err == -ENOENT)
 		return NULL;
 	if (err)
@@ -1870,16 +1874,6 @@ static struct dentry *ext4_lookup(struct inode *dir, struct dentry *dentry, unsi
 		}
 	}
 
-#if IS_ENABLED(CONFIG_UNICODE)
-	if (!inode && IS_CASEFOLDED(dir)) {
-		/* Eventually we want to call d_add_ci(dentry, NULL)
-		 * for negative dentries in the encoding case as
-		 * well.  For now, prevent the negative dentry
-		 * from being cached.
-		 */
-		return NULL;
-	}
-#endif
 	return d_splice_alias(inode, dentry);
 }
 
@@ -3211,17 +3205,6 @@ static int ext4_rmdir(struct inode *dir, struct dentry *dentry)
 	ext4_fc_track_unlink(handle, dentry);
 	retval = ext4_mark_inode_dirty(handle, dir);
 
-#if IS_ENABLED(CONFIG_UNICODE)
-	/* VFS negative dentries are incompatible with Encoding and
-	 * Case-insensitiveness. Eventually we'll want avoid
-	 * invalidating the dentries here, alongside with returning the
-	 * negative dentries at ext4_lookup(), when it is better
-	 * supported by the VFS for the CI case.
-	 */
-	if (IS_CASEFOLDED(dir))
-		d_invalidate(dentry);
-#endif
-
 end_rmdir:
 	brelse(bh);
 	if (handle)
@@ -3322,16 +3305,6 @@ static int ext4_unlink(struct inode *dir, struct dentry *dentry)
 		goto out_trace;
 
 	retval = __ext4_unlink(dir, &dentry->d_name, d_inode(dentry), dentry);
-#if IS_ENABLED(CONFIG_UNICODE)
-	/* VFS negative dentries are incompatible with Encoding and
-	 * Case-insensitiveness. Eventually we'll want avoid
-	 * invalidating the dentries here, alongside with returning the
-	 * negative dentries at ext4_lookup(), when it is  better
-	 * supported by the VFS for the CI case.
-	 */
-	if (IS_CASEFOLDED(dir))
-		d_invalidate(dentry);
-#endif
 
 out_trace:
 	trace_ext4_unlink_exit(dentry, retval);
diff --git a/fs/ext4/super.c b/fs/ext4/super.c
index c5fcf377ab1f..0980845c8b8f 100644
--- a/fs/ext4/super.c
+++ b/fs/ext4/super.c
@@ -5864,11 +5864,9 @@ static struct bdev_handle *ext4_get_journal_blkdev(struct super_block *sb,
 	struct ext4_super_block *es;
 	int errno;
 
-	/* see get_tree_bdev why this is needed and safe */
-	up_write(&sb->s_umount);
-	bdev_handle = bdev_open_by_dev(j_dev, BLK_OPEN_READ | BLK_OPEN_WRITE,
-				       sb, &fs_holder_ops);
-	down_write(&sb->s_umount);
+	bdev_handle = bdev_open_by_dev(j_dev,
+		BLK_OPEN_READ | BLK_OPEN_WRITE | BLK_OPEN_RESTRICT_WRITES,
+		sb, &fs_holder_ops);
 	if (IS_ERR(bdev_handle)) {
 		ext4_msg(sb, KERN_ERR,
 			 "failed to open journal device unknown-block(%u,%u) %ld",
diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c
index 36e5dab6baae..6b2af514660d 100644
--- a/fs/f2fs/compress.c
+++ b/fs/f2fs/compress.c
@@ -1944,7 +1944,7 @@ void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino)
 				continue;
 			}
 
-			generic_error_remove_page(mapping, &folio->page);
+			generic_error_remove_folio(mapping, folio);
 			folio_unlock(folio);
 		}
 		folio_batch_release(&fbatch);
diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
index e50363583f01..4580dfefd5e9 100644
--- a/fs/f2fs/file.c
+++ b/fs/f2fs/file.c
@@ -2239,11 +2239,11 @@ static int f2fs_ioc_shutdown(struct file *filp, unsigned long arg)
 
 	switch (in) {
 	case F2FS_GOING_DOWN_FULLSYNC:
-		ret = freeze_bdev(sb->s_bdev);
+		ret = bdev_freeze(sb->s_bdev);
 		if (ret)
 			goto out;
 		f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_SHUTDOWN);
-		thaw_bdev(sb->s_bdev);
+		bdev_thaw(sb->s_bdev);
 		break;
 	case F2FS_GOING_DOWN_METASYNC:
 		/* do checkpoint only */
diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c
index 560bfcad1af2..a9eb3891f417 100644
--- a/fs/f2fs/inode.c
+++ b/fs/f2fs/inode.c
@@ -600,7 +600,7 @@ struct inode *f2fs_iget(struct super_block *sb, unsigned long ino)
 #ifdef CONFIG_F2FS_FS_COMPRESSION
 		inode->i_mapping->a_ops = &f2fs_compress_aops;
 		/*
-		 * generic_error_remove_page only truncates pages of regular
+		 * generic_error_remove_folio only truncates pages of regular
 		 * inode
 		 */
 		inode->i_mode |= S_IFREG;
diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c
index d0053b0284d8..c5a78a07b7a1 100644
--- a/fs/f2fs/namei.c
+++ b/fs/f2fs/namei.c
@@ -533,6 +533,10 @@ static struct dentry *f2fs_lookup(struct inode *dir, struct dentry *dentry,
 
 	err = f2fs_prepare_lookup(dir, dentry, &fname);
 	generic_set_encrypted_ci_d_ops(dentry);
+
+	if (IS_CASEFOLDED(dir))
+		d_set_casefolded_name(dentry);
+
 	if (err == -ENOENT)
 		goto out_splice;
 	if (err)
@@ -578,17 +582,6 @@ static struct dentry *f2fs_lookup(struct inode *dir, struct dentry *dentry,
 		goto out_iput;
 	}
 out_splice:
-#if IS_ENABLED(CONFIG_UNICODE)
-	if (!inode && IS_CASEFOLDED(dir)) {
-		/* Eventually we want to call d_add_ci(dentry, NULL)
-		 * for negative dentries in the encoding case as
-		 * well.  For now, prevent the negative dentry
-		 * from being cached.
-		 */
-		trace_f2fs_lookup_end(dir, dentry, ino, err);
-		return NULL;
-	}
-#endif
 	new = d_splice_alias(inode, dentry);
 	trace_f2fs_lookup_end(dir, !IS_ERR_OR_NULL(new) ? new : dentry,
 				ino, IS_ERR(new) ? PTR_ERR(new) : err);
@@ -641,16 +634,6 @@ static int f2fs_unlink(struct inode *dir, struct dentry *dentry)
 	f2fs_delete_entry(de, page, dir, inode);
 	f2fs_unlock_op(sbi);
 
-#if IS_ENABLED(CONFIG_UNICODE)
-	/* VFS negative dentries are incompatible with Encoding and
-	 * Case-insensitiveness. Eventually we'll want avoid
-	 * invalidating the dentries here, alongside with returning the
-	 * negative dentries at f2fs_lookup(), when it is better
-	 * supported by the VFS for the CI case.
-	 */
-	if (IS_CASEFOLDED(dir))
-		d_invalidate(dentry);
-#endif
 	if (IS_DIRSYNC(dir))
 		f2fs_sync_fs(sbi->sb, 1);
 fail:
diff --git a/fs/fat/namei_vfat.c b/fs/fat/namei_vfat.c
index c4d00999a433..73981b0e4ea7 100644
--- a/fs/fat/namei_vfat.c
+++ b/fs/fat/namei_vfat.c
@@ -53,7 +53,8 @@ static int vfat_revalidate_shortname(struct dentry *dentry)
 	return ret;
 }
 
-static int vfat_revalidate(struct dentry *dentry, unsigned int flags)
+static int vfat_revalidate(struct dentry *dentry, const struct qstr *name,
+			   unsigned int flags)
 {
 	if (flags & LOOKUP_RCU)
 		return -ECHILD;
@@ -64,7 +65,8 @@ static int vfat_revalidate(struct dentry *dentry, unsigned int flags)
 	return vfat_revalidate_shortname(dentry);
 }
 
-static int vfat_revalidate_ci(struct dentry *dentry, unsigned int flags)
+static int vfat_revalidate_ci(struct dentry *dentry, const struct qstr *name,
+			      unsigned int flags)
 {
 	if (flags & LOOKUP_RCU)
 		return -ECHILD;
diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c
index d19cbf34c634..50548e4d404b 100644
--- a/fs/fuse/dir.c
+++ b/fs/fuse/dir.c
@@ -192,7 +192,8 @@ static void fuse_lookup_init(struct fuse_conn *fc, struct fuse_args *args,
  * the lookup once more.  If the lookup results in the same inode,
  * then refresh the attributes, timeouts and mark the dentry valid.
  */
-static int fuse_dentry_revalidate(struct dentry *entry, unsigned int flags)
+static int fuse_dentry_revalidate(struct dentry *entry,
+				  const struct qstr *name, unsigned int flags)
 {
 	struct inode *inode;
 	struct dentry *parent;
diff --git a/fs/fuse/file.c b/fs/fuse/file.c
index 1cdb6327511e..89e870d1a526 100644
--- a/fs/fuse/file.c
+++ b/fs/fuse/file.c
@@ -1448,7 +1448,7 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,
 	if (!ia)
 		return -ENOMEM;
 
-	if (fopen_direct_io && fc->direct_io_relax) {
+	if (fopen_direct_io && fc->direct_io_allow_mmap) {
 		res = filemap_write_and_wait_range(mapping, pos, pos + count - 1);
 		if (res) {
 			fuse_io_free(ia);
@@ -2466,9 +2466,9 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma)
 
 	if (ff->open_flags & FOPEN_DIRECT_IO) {
 		/* Can't provide the coherency needed for MAP_SHARED
-		 * if FUSE_DIRECT_IO_RELAX isn't set.
+		 * if FUSE_DIRECT_IO_ALLOW_MMAP isn't set.
 		 */
-		if ((vma->vm_flags & VM_MAYSHARE) && !fc->direct_io_relax)
+		if ((vma->vm_flags & VM_MAYSHARE) && !fc->direct_io_allow_mmap)
 			return -ENODEV;
 
 		invalidate_inode_pages2(file->f_mapping);
diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h
index 6e6e721f421b..1df83eebda92 100644
--- a/fs/fuse/fuse_i.h
+++ b/fs/fuse/fuse_i.h
@@ -63,6 +63,19 @@ struct fuse_forget_link {
 	struct fuse_forget_link *next;
 };
 
+/* Submount lookup tracking */
+struct fuse_submount_lookup {
+	/** Refcount */
+	refcount_t count;
+
+	/** Unique ID, which identifies the inode between userspace
+	 * and kernel */
+	u64 nodeid;
+
+	/** The request used for sending the FORGET message */
+	struct fuse_forget_link *forget;
+};
+
 /** FUSE inode */
 struct fuse_inode {
 	/** Inode data */
@@ -158,6 +171,8 @@ struct fuse_inode {
 	 */
 	struct fuse_inode_dax *dax;
 #endif
+	/** Submount specific lookup tracking */
+	struct fuse_submount_lookup *submount_lookup;
 };
 
 /** FUSE inode state bits */
@@ -797,8 +812,8 @@ struct fuse_conn {
 	/* Is tmpfile not implemented by fs? */
 	unsigned int no_tmpfile:1;
 
-	/* relax restrictions in FOPEN_DIRECT_IO mode */
-	unsigned int direct_io_relax:1;
+	/* Relax restrictions to allow shared mmap in FOPEN_DIRECT_IO mode */
+	unsigned int direct_io_allow_mmap:1;
 
 	/* Is statx not implemented by fs? */
 	unsigned int no_statx:1;
diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c
index 74d4f09d5827..2a6d44f91729 100644
--- a/fs/fuse/inode.c
+++ b/fs/fuse/inode.c
@@ -68,6 +68,24 @@ struct fuse_forget_link *fuse_alloc_forget(void)
 	return kzalloc(sizeof(struct fuse_forget_link), GFP_KERNEL_ACCOUNT);
 }
 
+static struct fuse_submount_lookup *fuse_alloc_submount_lookup(void)
+{
+	struct fuse_submount_lookup *sl;
+
+	sl = kzalloc(sizeof(struct fuse_submount_lookup), GFP_KERNEL_ACCOUNT);
+	if (!sl)
+		return NULL;
+	sl->forget = fuse_alloc_forget();
+	if (!sl->forget)
+		goto out_free;
+
+	return sl;
+
+out_free:
+	kfree(sl);
+	return NULL;
+}
+
 static struct inode *fuse_alloc_inode(struct super_block *sb)
 {
 	struct fuse_inode *fi;
@@ -83,6 +101,7 @@ static struct inode *fuse_alloc_inode(struct super_block *sb)
 	fi->attr_version = 0;
 	fi->orig_ino = 0;
 	fi->state = 0;
+	fi->submount_lookup = NULL;
 	mutex_init(&fi->mutex);
 	spin_lock_init(&fi->lock);
 	fi->forget = fuse_alloc_forget();
@@ -113,6 +132,17 @@ static void fuse_free_inode(struct inode *inode)
 	kmem_cache_free(fuse_inode_cachep, fi);
 }
 
+static void fuse_cleanup_submount_lookup(struct fuse_conn *fc,
+					 struct fuse_submount_lookup *sl)
+{
+	if (!refcount_dec_and_test(&sl->count))
+		return;
+
+	fuse_queue_forget(fc, sl->forget, sl->nodeid, 1);
+	sl->forget = NULL;
+	kfree(sl);
+}
+
 static void fuse_evict_inode(struct inode *inode)
 {
 	struct fuse_inode *fi = get_fuse_inode(inode);
@@ -132,6 +162,11 @@ static void fuse_evict_inode(struct inode *inode)
 					  fi->nlookup);
 			fi->forget = NULL;
 		}
+
+		if (fi->submount_lookup) {
+			fuse_cleanup_submount_lookup(fc, fi->submount_lookup);
+			fi->submount_lookup = NULL;
+		}
 	}
 	if (S_ISREG(inode->i_mode) && !fuse_is_bad(inode)) {
 		WARN_ON(!list_empty(&fi->write_files));
@@ -330,6 +365,13 @@ void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr,
 		fuse_dax_dontcache(inode, attr->flags);
 }
 
+static void fuse_init_submount_lookup(struct fuse_submount_lookup *sl,
+				      u64 nodeid)
+{
+	sl->nodeid = nodeid;
+	refcount_set(&sl->count, 1);
+}
+
 static void fuse_init_inode(struct inode *inode, struct fuse_attr *attr,
 			    struct fuse_conn *fc)
 {
@@ -392,12 +434,22 @@ struct inode *fuse_iget(struct super_block *sb, u64 nodeid,
 	 */
 	if (fc->auto_submounts && (attr->flags & FUSE_ATTR_SUBMOUNT) &&
 	    S_ISDIR(attr->mode)) {
+		struct fuse_inode *fi;
+
 		inode = new_inode(sb);
 		if (!inode)
 			return NULL;
 
 		fuse_init_inode(inode, attr, fc);
-		get_fuse_inode(inode)->nodeid = nodeid;
+		fi = get_fuse_inode(inode);
+		fi->nodeid = nodeid;
+		fi->submount_lookup = fuse_alloc_submount_lookup();
+		if (!fi->submount_lookup) {
+			iput(inode);
+			return NULL;
+		}
+		/* Sets nlookup = 1 on fi->submount_lookup->nlookup */
+		fuse_init_submount_lookup(fi->submount_lookup, nodeid);
 		inode->i_flags |= S_AUTOMOUNT;
 		goto done;
 	}
@@ -420,11 +472,11 @@ struct inode *fuse_iget(struct super_block *sb, u64 nodeid,
 		iput(inode);
 		goto retry;
 	}
-done:
 	fi = get_fuse_inode(inode);
 	spin_lock(&fi->lock);
 	fi->nlookup++;
 	spin_unlock(&fi->lock);
+done:
 	fuse_change_attributes(inode, attr, NULL, attr_valid, attr_version);
 
 	return inode;
@@ -1230,8 +1282,8 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args,
 				fc->init_security = 1;
 			if (flags & FUSE_CREATE_SUPP_GROUP)
 				fc->create_supp_group = 1;
-			if (flags & FUSE_DIRECT_IO_RELAX)
-				fc->direct_io_relax = 1;
+			if (flags & FUSE_DIRECT_IO_ALLOW_MMAP)
+				fc->direct_io_allow_mmap = 1;
 		} else {
 			ra_pages = fc->max_read / PAGE_SIZE;
 			fc->no_lock = 1;
@@ -1278,7 +1330,7 @@ void fuse_send_init(struct fuse_mount *fm)
 		FUSE_NO_OPENDIR_SUPPORT | FUSE_EXPLICIT_INVAL_DATA |
 		FUSE_HANDLE_KILLPRIV_V2 | FUSE_SETXATTR_EXT | FUSE_INIT_EXT |
 		FUSE_SECURITY_CTX | FUSE_CREATE_SUPP_GROUP |
-		FUSE_HAS_EXPIRE_ONLY | FUSE_DIRECT_IO_RELAX;
+		FUSE_HAS_EXPIRE_ONLY | FUSE_DIRECT_IO_ALLOW_MMAP;
 #ifdef CONFIG_FUSE_DAX
 	if (fm->fc->dax)
 		flags |= FUSE_MAP_ALIGNMENT;
@@ -1465,6 +1517,8 @@ static int fuse_fill_super_submount(struct super_block *sb,
 	struct super_block *parent_sb = parent_fi->inode.i_sb;
 	struct fuse_attr root_attr;
 	struct inode *root;
+	struct fuse_submount_lookup *sl;
+	struct fuse_inode *fi;
 
 	fuse_sb_defaults(sb);
 	fm->sb = sb;
@@ -1487,12 +1541,27 @@ static int fuse_fill_super_submount(struct super_block *sb,
 	 * its nlookup should not be incremented.  fuse_iget() does
 	 * that, though, so undo it here.
 	 */
-	get_fuse_inode(root)->nlookup--;
+	fi = get_fuse_inode(root);
+	fi->nlookup--;
+
 	sb->s_d_op = &fuse_dentry_operations;
 	sb->s_root = d_make_root(root);
 	if (!sb->s_root)
 		return -ENOMEM;
 
+	/*
+	 * Grab the parent's submount_lookup pointer and take a
+	 * reference on the shared nlookup from the parent.  This is to
+	 * prevent the last forget for this nodeid from getting
+	 * triggered until all users have finished with it.
+	 */
+	sl = parent_fi->submount_lookup;
+	WARN_ON(!sl);
+	if (sl) {
+		refcount_inc(&sl->count);
+		fi->submount_lookup = sl;
+	}
+
 	return 0;
 }
 
diff --git a/fs/gfs2/aops.c b/fs/gfs2/aops.c
index 9611bfceda4b..5cffb079b87c 100644
--- a/fs/gfs2/aops.c
+++ b/fs/gfs2/aops.c
@@ -403,18 +403,18 @@ static int gfs2_jdata_writepages(struct address_space *mapping,
 }
 
 /**
- * stuffed_readpage - Fill in a Linux folio with stuffed file data
+ * stuffed_read_folio - Fill in a Linux folio with stuffed file data
  * @ip: the inode
  * @folio: the folio
  *
  * Returns: errno
  */
-static int stuffed_readpage(struct gfs2_inode *ip, struct folio *folio)
+static int stuffed_read_folio(struct gfs2_inode *ip, struct folio *folio)
 {
-	struct buffer_head *dibh;
-	size_t i_size = i_size_read(&ip->i_inode);
-	void *data;
-	int error;
+	struct buffer_head *dibh = NULL;
+	size_t dsize = i_size_read(&ip->i_inode);
+	void *from = NULL;
+	int error = 0;
 
 	/*
 	 * Due to the order of unstuffing files and ->fault(), we can be
@@ -422,22 +422,20 @@ static int stuffed_readpage(struct gfs2_inode *ip, struct folio *folio)
 	 * so we need to supply one here. It doesn't happen often.
 	 */
 	if (unlikely(folio->index)) {
-		folio_zero_range(folio, 0, folio_size(folio));
-		folio_mark_uptodate(folio);
-		return 0;
+		dsize = 0;
+	} else {
+		error = gfs2_meta_inode_buffer(ip, &dibh);
+		if (error)
+			goto out;
+		from = dibh->b_data + sizeof(struct gfs2_dinode);
 	}
 
-	error = gfs2_meta_inode_buffer(ip, &dibh);
-	if (error)
-		return error;
-
-	data = dibh->b_data + sizeof(struct gfs2_dinode);
-	memcpy_to_folio(folio, 0, data, i_size);
-	folio_zero_range(folio, i_size, folio_size(folio) - i_size);
+	folio_fill_tail(folio, 0, from, dsize);
 	brelse(dibh);
-	folio_mark_uptodate(folio);
+out:
+	folio_end_read(folio, error == 0);
 
-	return 0;
+	return error;
 }
 
 /**
@@ -456,8 +454,7 @@ static int gfs2_read_folio(struct file *file, struct folio *folio)
 	    (i_blocksize(inode) == PAGE_SIZE && !folio_buffers(folio))) {
 		error = iomap_read_folio(folio, &gfs2_iomap_ops);
 	} else if (gfs2_is_stuffed(ip)) {
-		error = stuffed_readpage(ip, folio);
-		folio_unlock(folio);
+		error = stuffed_read_folio(ip, folio);
 	} else {
 		error = mpage_read_folio(folio, gfs2_block_map);
 	}
@@ -748,7 +745,7 @@ static const struct address_space_operations gfs2_aops = {
 	.bmap = gfs2_bmap,
 	.migrate_folio = filemap_migrate_folio,
 	.is_partially_uptodate = iomap_is_partially_uptodate,
-	.error_remove_page = generic_error_remove_page,
+	.error_remove_folio = generic_error_remove_folio,
 };
 
 static const struct address_space_operations gfs2_jdata_aops = {
@@ -761,7 +758,7 @@ static const struct address_space_operations gfs2_jdata_aops = {
 	.invalidate_folio = gfs2_invalidate_folio,
 	.release_folio = gfs2_release_folio,
 	.is_partially_uptodate = block_is_partially_uptodate,
-	.error_remove_page = generic_error_remove_page,
+	.error_remove_folio = generic_error_remove_folio,
 };
 
 void gfs2_set_aops(struct inode *inode)
diff --git a/fs/gfs2/dentry.c b/fs/gfs2/dentry.c
index 2e215e8c3c88..c3fd1286805b 100644
--- a/fs/gfs2/dentry.c
+++ b/fs/gfs2/dentry.c
@@ -22,6 +22,7 @@
 /**
  * gfs2_drevalidate - Check directory lookup consistency
  * @dentry: the mapping to check
+ * @name: The name under lookup
  * @flags: lookup flags
  *
  * Check to make sure the lookup necessary to arrive at this inode from its
@@ -30,7 +31,8 @@
  * Returns: 1 if the dentry is ok, 0 if it isn't
  */
 
-static int gfs2_drevalidate(struct dentry *dentry, unsigned int flags)
+static int gfs2_drevalidate(struct dentry *dentry, const struct qstr *name,
+			    unsigned int flags)
 {
 	struct dentry *parent;
 	struct gfs2_sbd *sdp;
@@ -41,9 +43,6 @@ static int gfs2_drevalidate(struct dentry *dentry, unsigned int flags)
 	int error, valid = 0;
 	int had_lock = 0;
 
-	if (flags & LOOKUP_RCU)
-		return -ECHILD;
-
 	parent = dget_parent(dentry);
 	sdp = GFS2_SB(d_inode(parent));
 	dip = GFS2_I(d_inode(parent));
@@ -62,7 +61,8 @@ static int gfs2_drevalidate(struct dentry *dentry, unsigned int flags)
 
 	had_lock = (gfs2_glock_is_locked_by_me(dip->i_gl) != NULL);
 	if (!had_lock) {
-		error = gfs2_glock_nq_init(dip->i_gl, LM_ST_SHARED, 0, &d_gh);
+		error = gfs2_glock_nq_init(dip->i_gl, LM_ST_SHARED,
+					   flags & LOOKUP_RCU ? GL_NOBLOCK : 0, &d_gh);
 		if (error)
 			goto out;
 	}
diff --git a/fs/gfs2/export.c b/fs/gfs2/export.c
index cf40895233f5..ef1013eff936 100644
--- a/fs/gfs2/export.c
+++ b/fs/gfs2/export.c
@@ -192,5 +192,6 @@ const struct export_operations gfs2_export_ops = {
 	.fh_to_parent = gfs2_fh_to_parent,
 	.get_name = gfs2_get_name,
 	.get_parent = gfs2_get_parent,
+	.flags = EXPORT_OP_ASYNC_LOCK,
 };
 
diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c
index d6bf1f8c25dc..f28c67181230 100644
--- a/fs/gfs2/glock.c
+++ b/fs/gfs2/glock.c
@@ -516,6 +516,23 @@ static inline struct gfs2_holder *find_first_waiter(const struct gfs2_glock *gl)
 	return NULL;
 }
 
+/**
+ * find_last_waiter - find the last gh that's waiting for the glock
+ * @gl: the glock
+ *
+ * This also is a fast way of finding out if there are any waiters.
+ */
+
+static inline struct gfs2_holder *find_last_waiter(const struct gfs2_glock *gl)
+{
+	struct gfs2_holder *gh;
+
+	if (list_empty(&gl->gl_holders))
+		return NULL;
+	gh = list_last_entry(&gl->gl_holders, struct gfs2_holder, gh_list);
+	return test_bit(HIF_HOLDER, &gh->gh_iflags) ? NULL : gh;
+}
+
 /**
  * state_change - record that the glock is now in a different state
  * @gl: the glock
@@ -1213,7 +1230,7 @@ int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number,
 		mapping->host = s->s_bdev->bd_inode;
 		mapping->flags = 0;
 		mapping_set_gfp_mask(mapping, GFP_NOFS);
-		mapping->private_data = NULL;
+		mapping->i_private_data = NULL;
 		mapping->writeback_index = 0;
 	}
 
@@ -1555,11 +1572,30 @@ __acquires(&gl->gl_lockref.lock)
 int gfs2_glock_nq(struct gfs2_holder *gh)
 {
 	struct gfs2_glock *gl = gh->gh_gl;
-	int error = 0;
+	int error;
 
 	if (glock_blocked_by_withdraw(gl) && !(gh->gh_flags & LM_FLAG_NOEXP))
 		return -EIO;
 
+	if (gh->gh_flags & GL_NOBLOCK) {
+		struct gfs2_holder *current_gh;
+
+		error = -ECHILD;
+		spin_lock(&gl->gl_lockref.lock);
+		if (find_last_waiter(gl))
+			goto unlock;
+		current_gh = find_first_holder(gl);
+		if (!may_grant(gl, current_gh, gh))
+			goto unlock;
+		set_bit(HIF_HOLDER, &gh->gh_iflags);
+		list_add_tail(&gh->gh_list, &gl->gl_holders);
+		trace_gfs2_promote(gh);
+		error = 0;
+unlock:
+		spin_unlock(&gl->gl_lockref.lock);
+		return error;
+	}
+
 	if (test_bit(GLF_LRU, &gl->gl_flags))
 		gfs2_glock_remove_from_lru(gl);
 
@@ -1575,6 +1611,7 @@ int gfs2_glock_nq(struct gfs2_holder *gh)
 	run_queue(gl, 1);
 	spin_unlock(&gl->gl_lockref.lock);
 
+	error = 0;
 	if (!(gh->gh_flags & GL_ASYNC))
 		error = gfs2_glock_wait(gh);
 
diff --git a/fs/gfs2/glock.h b/fs/gfs2/glock.h
index 61197598abfd..0114f3e0ebe0 100644
--- a/fs/gfs2/glock.h
+++ b/fs/gfs2/glock.h
@@ -84,6 +84,7 @@ enum {
 #define GL_SKIP			0x0100
 #define GL_NOPID		0x0200
 #define GL_NOCACHE		0x0400
+#define GL_NOBLOCK		0x0800
   
 /*
  * lm_async_cb return flags
diff --git a/fs/gfs2/glops.c b/fs/gfs2/glops.c
index b41c78bd2cc0..15d0e653fd2b 100644
--- a/fs/gfs2/glops.c
+++ b/fs/gfs2/glops.c
@@ -494,7 +494,7 @@ int gfs2_inode_refresh(struct gfs2_inode *ip)
 
 /**
  * inode_go_instantiate - read in an inode if necessary
- * @gh: The glock holder
+ * @gl: The glock
  *
  * Returns: errno
  */
diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c
index 1b95db2c3aac..6bfc9383b7b8 100644
--- a/fs/gfs2/inode.c
+++ b/fs/gfs2/inode.c
@@ -1882,10 +1882,10 @@ int gfs2_permission(struct mnt_idmap *idmap, struct inode *inode,
 		WARN_ON_ONCE(!may_not_block);
 		return -ECHILD;
         }
-	if (gfs2_glock_is_locked_by_me(gl) == NULL) {
-		if (may_not_block)
-			return -ECHILD;
-		error = gfs2_glock_nq_init(gl, LM_ST_SHARED, LM_FLAG_ANY, &i_gh);
+	if (gfs2_glock_is_locked_by_me(ip->i_gl) == NULL) {
+		int noblock = may_not_block ? GL_NOBLOCK : 0;
+		error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED,
+					   LM_FLAG_ANY | noblock, &i_gh);
 		if (error)
 			return error;
 	}
diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c
index b108c5d26839..00ce89bdf32c 100644
--- a/fs/gfs2/ops_fstype.c
+++ b/fs/gfs2/ops_fstype.c
@@ -117,7 +117,7 @@ static struct gfs2_sbd *init_sbd(struct super_block *sb)
 	mapping->host = sb->s_bdev->bd_inode;
 	mapping->flags = 0;
 	mapping_set_gfp_mask(mapping, GFP_NOFS);
-	mapping->private_data = NULL;
+	mapping->i_private_data = NULL;
 	mapping->writeback_index = 0;
 
 	spin_lock_init(&sdp->sd_log_lock);
diff --git a/fs/hfs/sysdep.c b/fs/hfs/sysdep.c
index 76fa02e3835b..86da5c7f03cc 100644
--- a/fs/hfs/sysdep.c
+++ b/fs/hfs/sysdep.c
@@ -13,7 +13,8 @@
 
 /* dentry case-handling: just lowercase everything */
 
-static int hfs_revalidate_dentry(struct dentry *dentry, unsigned int flags)
+static int hfs_revalidate_dentry(struct dentry *dentry,
+				 const struct qstr *name, unsigned int flags)
 {
 	struct inode *inode;
 	int diff;
diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
index f757d4f7ad98..ea5b8e57d904 100644
--- a/fs/hugetlbfs/inode.c
+++ b/fs/hugetlbfs/inode.c
@@ -686,7 +686,7 @@ static void hugetlbfs_evict_inode(struct inode *inode)
 	 * at inode creation time.  If this is a device special inode,
 	 * i_mapping may not point to the original address space.
 	 */
-	resv_map = (struct resv_map *)(&inode->i_data)->private_data;
+	resv_map = (struct resv_map *)(&inode->i_data)->i_private_data;
 	/* Only regular and link inodes have associated reserve maps */
 	if (resv_map)
 		resv_map_release(&resv_map->refs);
@@ -1000,7 +1000,7 @@ static struct inode *hugetlbfs_get_inode(struct super_block *sb,
 				&hugetlbfs_i_mmap_rwsem_key);
 		inode->i_mapping->a_ops = &hugetlbfs_aops;
 		simple_inode_init_ts(inode);
-		inode->i_mapping->private_data = resv_map;
+		inode->i_mapping->i_private_data = resv_map;
 		info->seals = F_SEAL_SEAL;
 		switch (mode & S_IFMT) {
 		default:
@@ -1129,8 +1129,8 @@ static int hugetlbfs_migrate_folio(struct address_space *mapping,
 #define hugetlbfs_migrate_folio NULL
 #endif
 
-static int hugetlbfs_error_remove_page(struct address_space *mapping,
-				struct page *page)
+static int hugetlbfs_error_remove_folio(struct address_space *mapping,
+				struct folio *folio)
 {
 	return 0;
 }
@@ -1277,7 +1277,7 @@ static const struct address_space_operations hugetlbfs_aops = {
 	.write_end	= hugetlbfs_write_end,
 	.dirty_folio	= noop_dirty_folio,
 	.migrate_folio  = hugetlbfs_migrate_folio,
-	.error_remove_page	= hugetlbfs_error_remove_page,
+	.error_remove_folio	= hugetlbfs_error_remove_folio,
 };
 
 
diff --git a/fs/inode.c b/fs/inode.c
index edcd8a61975f..2940ca8a3f32 100644
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -209,12 +209,14 @@ int inode_init_always(struct super_block *sb, struct inode *inode)
 	atomic_set(&mapping->nr_thps, 0);
 #endif
 	mapping_set_gfp_mask(mapping, GFP_HIGHUSER_MOVABLE);
-	mapping->private_data = NULL;
+	mapping->i_private_data = NULL;
 	mapping->writeback_index = 0;
 	init_rwsem(&mapping->invalidate_lock);
 	lockdep_set_class_and_name(&mapping->invalidate_lock,
 				   &sb->s_type->invalidate_lock_key,
 				   "mapping.invalidate_lock");
+	if (sb->s_iflags & SB_I_STABLE_WRITES)
+		mapping_set_stable_writes(mapping);
 	inode->i_private = NULL;
 	inode->i_mapping = mapping;
 	INIT_HLIST_HEAD(&inode->i_dentry);	/* buggered by rcu freeing */
@@ -396,8 +398,8 @@ static void __address_space_init_once(struct address_space *mapping)
 {
 	xa_init_flags(&mapping->i_pages, XA_FLAGS_LOCK_IRQ | XA_FLAGS_ACCOUNT);
 	init_rwsem(&mapping->i_mmap_rwsem);
-	INIT_LIST_HEAD(&mapping->private_list);
-	spin_lock_init(&mapping->private_lock);
+	INIT_LIST_HEAD(&mapping->i_private_list);
+	spin_lock_init(&mapping->i_private_lock);
 	mapping->i_mmap = RB_ROOT_CACHED;
 }
 
@@ -618,7 +620,7 @@ void clear_inode(struct inode *inode)
 	 * nor even WARN_ON(!mapping_empty).
 	 */
 	xa_unlock_irq(&inode->i_data.i_pages);
-	BUG_ON(!list_empty(&inode->i_data.private_list));
+	BUG_ON(!list_empty(&inode->i_data.i_private_list));
 	BUG_ON(!(inode->i_state & I_FREEING));
 	BUG_ON(inode->i_state & I_CLEAR);
 	BUG_ON(!list_empty(&inode->i_wb_list));
diff --git a/fs/internal.h b/fs/internal.h
index 58e43341aebf..16bfe50747c6 100644
--- a/fs/internal.h
+++ b/fs/internal.h
@@ -83,6 +83,8 @@ int path_mount(const char *dev_name, struct path *path,
 		const char *type_page, unsigned long flags, void *data_page);
 int path_umount(struct path *path, int flags);
 
+int show_path(struct seq_file *m, struct dentry *root);
+
 /*
  * fs_struct.c
  */
diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c
index f72df2babe56..093c4515b22a 100644
--- a/fs/iomap/buffered-io.c
+++ b/fs/iomap/buffered-io.c
@@ -305,28 +305,18 @@ static int iomap_read_inline_data(const struct iomap_iter *iter,
 {
 	const struct iomap *iomap = iomap_iter_srcmap(iter);
 	size_t size = i_size_read(iter->inode) - iomap->offset;
-	size_t poff = offset_in_page(iomap->offset);
 	size_t offset = offset_in_folio(folio, iomap->offset);
-	void *addr;
 
 	if (folio_test_uptodate(folio))
 		return 0;
 
-	if (WARN_ON_ONCE(size > PAGE_SIZE - poff))
-		return -EIO;
-	if (WARN_ON_ONCE(size > PAGE_SIZE -
-			 offset_in_page(iomap->inline_data)))
-		return -EIO;
 	if (WARN_ON_ONCE(size > iomap->length))
 		return -EIO;
 	if (offset > 0)
 		ifs_alloc(iter->inode, folio, iter->flags);
 
-	addr = kmap_local_folio(folio, offset);
-	memcpy(addr, iomap->inline_data, size);
-	memset(addr + size, 0, PAGE_SIZE - poff - size);
-	kunmap_local(addr);
-	iomap_set_range_uptodate(folio, offset, PAGE_SIZE - poff);
+	folio_fill_tail(folio, offset, iomap->inline_data, size);
+	iomap_set_range_uptodate(folio, offset, folio_size(folio) - offset);
 	return 0;
 }
 
diff --git a/fs/jfs/jfs_dmap.c b/fs/jfs/jfs_dmap.c
index 11c77757ead9..8eec84c651bf 100644
--- a/fs/jfs/jfs_dmap.c
+++ b/fs/jfs/jfs_dmap.c
@@ -63,10 +63,10 @@
  */
 static void dbAllocBits(struct bmap * bmp, struct dmap * dp, s64 blkno,
 			int nblocks);
-static void dbSplit(dmtree_t * tp, int leafno, int splitsz, int newval);
-static int dbBackSplit(dmtree_t * tp, int leafno);
-static int dbJoin(dmtree_t * tp, int leafno, int newval);
-static void dbAdjTree(dmtree_t * tp, int leafno, int newval);
+static void dbSplit(dmtree_t *tp, int leafno, int splitsz, int newval, bool is_ctl);
+static int dbBackSplit(dmtree_t *tp, int leafno, bool is_ctl);
+static int dbJoin(dmtree_t *tp, int leafno, int newval, bool is_ctl);
+static void dbAdjTree(dmtree_t *tp, int leafno, int newval, bool is_ctl);
 static int dbAdjCtl(struct bmap * bmp, s64 blkno, int newval, int alloc,
 		    int level);
 static int dbAllocAny(struct bmap * bmp, s64 nblocks, int l2nb, s64 * results);
@@ -2103,7 +2103,7 @@ static int dbFreeDmap(struct bmap * bmp, struct dmap * dp, s64 blkno,
 		 * system.
 		 */
 		if (dp->tree.stree[word] == NOFREE)
-			dbBackSplit((dmtree_t *) & dp->tree, word);
+			dbBackSplit((dmtree_t *)&dp->tree, word, false);
 
 		dbAllocBits(bmp, dp, blkno, nblocks);
 	}
@@ -2189,7 +2189,7 @@ static void dbAllocBits(struct bmap * bmp, struct dmap * dp, s64 blkno,
 			 * the binary system of the leaves if need be.
 			 */
 			dbSplit(tp, word, BUDMIN,
-				dbMaxBud((u8 *) & dp->wmap[word]));
+				dbMaxBud((u8 *)&dp->wmap[word]), false);
 
 			word += 1;
 		} else {
@@ -2229,7 +2229,7 @@ static void dbAllocBits(struct bmap * bmp, struct dmap * dp, s64 blkno,
 				 * system of the leaves to reflect the current
 				 * allocation (size).
 				 */
-				dbSplit(tp, word, size, NOFREE);
+				dbSplit(tp, word, size, NOFREE, false);
 
 				/* get the number of dmap words handled */
 				nw = BUDSIZE(size, BUDMIN);
@@ -2336,7 +2336,7 @@ static int dbFreeBits(struct bmap * bmp, struct dmap * dp, s64 blkno,
 			/* update the leaf for this dmap word.
 			 */
 			rc = dbJoin(tp, word,
-				    dbMaxBud((u8 *) & dp->wmap[word]));
+				    dbMaxBud((u8 *)&dp->wmap[word]), false);
 			if (rc)
 				return rc;
 
@@ -2369,7 +2369,7 @@ static int dbFreeBits(struct bmap * bmp, struct dmap * dp, s64 blkno,
 
 				/* update the leaf.
 				 */
-				rc = dbJoin(tp, word, size);
+				rc = dbJoin(tp, word, size, false);
 				if (rc)
 					return rc;
 
@@ -2521,16 +2521,16 @@ dbAdjCtl(struct bmap * bmp, s64 blkno, int newval, int alloc, int level)
 		 * that it is at the front of a binary buddy system.
 		 */
 		if (oldval == NOFREE) {
-			rc = dbBackSplit((dmtree_t *) dcp, leafno);
+			rc = dbBackSplit((dmtree_t *)dcp, leafno, true);
 			if (rc) {
 				release_metapage(mp);
 				return rc;
 			}
 			oldval = dcp->stree[ti];
 		}
-		dbSplit((dmtree_t *) dcp, leafno, dcp->budmin, newval);
+		dbSplit((dmtree_t *) dcp, leafno, dcp->budmin, newval, true);
 	} else {
-		rc = dbJoin((dmtree_t *) dcp, leafno, newval);
+		rc = dbJoin((dmtree_t *) dcp, leafno, newval, true);
 		if (rc) {
 			release_metapage(mp);
 			return rc;
@@ -2561,7 +2561,7 @@ dbAdjCtl(struct bmap * bmp, s64 blkno, int newval, int alloc, int level)
 				 */
 				if (alloc) {
 					dbJoin((dmtree_t *) dcp, leafno,
-					       oldval);
+					       oldval, true);
 				} else {
 					/* the dbJoin() above might have
 					 * caused a larger binary buddy system
@@ -2571,9 +2571,9 @@ dbAdjCtl(struct bmap * bmp, s64 blkno, int newval, int alloc, int level)
 					 */
 					if (dcp->stree[ti] == NOFREE)
 						dbBackSplit((dmtree_t *)
-							    dcp, leafno);
+							    dcp, leafno, true);
 					dbSplit((dmtree_t *) dcp, leafno,
-						dcp->budmin, oldval);
+						dcp->budmin, oldval, true);
 				}
 
 				/* release the buffer and return the error.
@@ -2621,7 +2621,7 @@ dbAdjCtl(struct bmap * bmp, s64 blkno, int newval, int alloc, int level)
  *
  * serialization: IREAD_LOCK(ipbmap) or IWRITE_LOCK(ipbmap) held on entry/exit;
  */
-static void dbSplit(dmtree_t * tp, int leafno, int splitsz, int newval)
+static void dbSplit(dmtree_t *tp, int leafno, int splitsz, int newval, bool is_ctl)
 {
 	int budsz;
 	int cursz;
@@ -2643,7 +2643,7 @@ static void dbSplit(dmtree_t * tp, int leafno, int splitsz, int newval)
 		while (cursz >= splitsz) {
 			/* update the buddy's leaf with its new value.
 			 */
-			dbAdjTree(tp, leafno ^ budsz, cursz);
+			dbAdjTree(tp, leafno ^ budsz, cursz, is_ctl);
 
 			/* on to the next size and buddy.
 			 */
@@ -2655,7 +2655,7 @@ static void dbSplit(dmtree_t * tp, int leafno, int splitsz, int newval)
 	/* adjust the dmap tree to reflect the specified leaf's new
 	 * value.
 	 */
-	dbAdjTree(tp, leafno, newval);
+	dbAdjTree(tp, leafno, newval, is_ctl);
 }
 
 
@@ -2686,7 +2686,7 @@ static void dbSplit(dmtree_t * tp, int leafno, int splitsz, int newval)
  *
  * serialization: IREAD_LOCK(ipbmap) or IWRITE_LOCK(ipbmap) held on entry/exit;
  */
-static int dbBackSplit(dmtree_t * tp, int leafno)
+static int dbBackSplit(dmtree_t *tp, int leafno, bool is_ctl)
 {
 	int budsz, bud, w, bsz, size;
 	int cursz;
@@ -2737,7 +2737,7 @@ static int dbBackSplit(dmtree_t * tp, int leafno)
 				 * system in two.
 				 */
 				cursz = leaf[bud] - 1;
-				dbSplit(tp, bud, cursz, cursz);
+				dbSplit(tp, bud, cursz, cursz, is_ctl);
 				break;
 			}
 		}
@@ -2763,9 +2763,11 @@ static int dbBackSplit(dmtree_t * tp, int leafno)
  *	leafno	- the number of the leaf to be updated.
  *	newval	- the new value for the leaf.
  *
- * RETURN VALUES: none
+ * RETURN VALUES:
+ *  0		- success
+ *	-EIO	- i/o error
  */
-static int dbJoin(dmtree_t * tp, int leafno, int newval)
+static int dbJoin(dmtree_t *tp, int leafno, int newval, bool is_ctl)
 {
 	int budsz, buddy;
 	s8 *leaf;
@@ -2790,6 +2792,10 @@ static int dbJoin(dmtree_t * tp, int leafno, int newval)
 		 * get the buddy size (number of words covered) of
 		 * the new value.
 		 */
+
+		if ((newval - tp->dmt_budmin) > BUDMIN)
+			return -EIO;
+
 		budsz = BUDSIZE(newval, tp->dmt_budmin);
 
 		/* try to join.
@@ -2820,12 +2826,12 @@ static int dbJoin(dmtree_t * tp, int leafno, int newval)
 			if (leafno < buddy) {
 				/* leafno is the left buddy.
 				 */
-				dbAdjTree(tp, buddy, NOFREE);
+				dbAdjTree(tp, buddy, NOFREE, is_ctl);
 			} else {
 				/* buddy is the left buddy and becomes
 				 * leafno.
 				 */
-				dbAdjTree(tp, leafno, NOFREE);
+				dbAdjTree(tp, leafno, NOFREE, is_ctl);
 				leafno = buddy;
 			}
 
@@ -2838,7 +2844,7 @@ static int dbJoin(dmtree_t * tp, int leafno, int newval)
 
 	/* update the leaf value.
 	 */
-	dbAdjTree(tp, leafno, newval);
+	dbAdjTree(tp, leafno, newval, is_ctl);
 
 	return 0;
 }
@@ -2859,15 +2865,20 @@ static int dbJoin(dmtree_t * tp, int leafno, int newval)
  *
  * RETURN VALUES: none
  */
-static void dbAdjTree(dmtree_t * tp, int leafno, int newval)
+static void dbAdjTree(dmtree_t *tp, int leafno, int newval, bool is_ctl)
 {
 	int lp, pp, k;
-	int max;
+	int max, size;
+
+	size = is_ctl ? CTLTREESIZE : TREESIZE;
 
 	/* pick up the index of the leaf for this leafno.
 	 */
 	lp = leafno + le32_to_cpu(tp->dmt_leafidx);
 
+	if (WARN_ON_ONCE(lp >= size || lp < 0))
+		return;
+
 	/* is the current value the same as the old value ?  if so,
 	 * there is nothing to do.
 	 */
diff --git a/fs/jfs/jfs_dtree.c b/fs/jfs/jfs_dtree.c
index 92b7c533407c..031d8f570f58 100644
--- a/fs/jfs/jfs_dtree.c
+++ b/fs/jfs/jfs_dtree.c
@@ -633,6 +633,11 @@ int dtSearch(struct inode *ip, struct component_name * key, ino_t * data,
 		for (base = 0, lim = p->header.nextindex; lim; lim >>= 1) {
 			index = base + (lim >> 1);
 
+			if (stbl[index] < 0) {
+				rc = -EIO;
+				goto out;
+			}
+
 			if (p->header.flag & BT_LEAF) {
 				/* uppercase leaf name to compare */
 				cmp =
@@ -1970,7 +1975,7 @@ static int dtSplitRoot(tid_t tid,
 		do {
 			f = &rp->slot[fsi];
 			fsi = f->next;
-		} while (fsi != -1);
+		} while (fsi >= 0);
 
 		f->next = n;
 	}
diff --git a/fs/jfs/jfs_mount.c b/fs/jfs/jfs_mount.c
index 415eb65a36ff..9b5c6a20b30c 100644
--- a/fs/jfs/jfs_mount.c
+++ b/fs/jfs/jfs_mount.c
@@ -172,15 +172,15 @@ int jfs_mount(struct super_block *sb)
 	}
 	jfs_info("jfs_mount: ipimap:0x%p", ipimap);
 
-	/* map further access of per fileset inodes by the fileset inode */
-	sbi->ipimap = ipimap;
-
 	/* initialize fileset inode allocation map */
 	if ((rc = diMount(ipimap))) {
 		jfs_err("jfs_mount: diMount failed w/rc = %d", rc);
 		goto err_ipimap;
 	}
 
+	/* map further access of per fileset inodes by the fileset inode */
+	sbi->ipimap = ipimap;
+
 	return rc;
 
 	/*
diff --git a/fs/jfs/namei.c b/fs/jfs/namei.c
index d68a4e6ac345..4e402c05959c 100644
--- a/fs/jfs/namei.c
+++ b/fs/jfs/namei.c
@@ -1576,7 +1576,8 @@ static int jfs_ci_compare(const struct dentry *dentry,
 	return result;
 }
 
-static int jfs_ci_revalidate(struct dentry *dentry, unsigned int flags)
+static int jfs_ci_revalidate(struct dentry *dentry, const struct qstr *name,
+			     unsigned int flags)
 {
 	/*
 	 * This is not negative dentry. Always valid.
diff --git a/fs/kernfs/dir.c b/fs/kernfs/dir.c
index 8b2bd65d70e7..7e689b65b081 100644
--- a/fs/kernfs/dir.c
+++ b/fs/kernfs/dir.c
@@ -1088,7 +1088,8 @@ struct kernfs_node *kernfs_create_empty_dir(struct kernfs_node *parent,
 	return ERR_PTR(rc);
 }
 
-static int kernfs_dop_revalidate(struct dentry *dentry, unsigned int flags)
+static int kernfs_dop_revalidate(struct dentry *dentry,
+				 const struct qstr *name, unsigned int flags)
 {
 	struct kernfs_node *kn;
 	struct kernfs_root *root;
diff --git a/fs/libfs.c b/fs/libfs.c
index e9440d55073c..1bb5c37b1df2 100644
--- a/fs/libfs.c
+++ b/fs/libfs.c
@@ -399,6 +399,8 @@ static loff_t offset_dir_llseek(struct file *file, loff_t offset, int whence)
 		return -EINVAL;
 	}
 
+	/* In this case, ->private_data is protected by f_pos_lock */
+	file->private_data = NULL;
 	return vfs_setpos(file, offset, U32_MAX);
 }
 
@@ -428,7 +430,7 @@ static bool offset_dir_emit(struct dir_context *ctx, struct dentry *dentry)
 			  inode->i_ino, fs_umode_to_dtype(inode->i_mode));
 }
 
-static void offset_iterate_dir(struct inode *inode, struct dir_context *ctx)
+static void *offset_iterate_dir(struct inode *inode, struct dir_context *ctx)
 {
 	struct offset_ctx *so_ctx = inode->i_op->get_offset_ctx(inode);
 	XA_STATE(xas, &so_ctx->xa, ctx->pos);
@@ -437,7 +439,7 @@ static void offset_iterate_dir(struct inode *inode, struct dir_context *ctx)
 	while (true) {
 		dentry = offset_find_next(&xas);
 		if (!dentry)
-			break;
+			return ERR_PTR(-ENOENT);
 
 		if (!offset_dir_emit(ctx, dentry)) {
 			dput(dentry);
@@ -447,6 +449,7 @@ static void offset_iterate_dir(struct inode *inode, struct dir_context *ctx)
 		dput(dentry);
 		ctx->pos = xas.xa_index + 1;
 	}
+	return NULL;
 }
 
 /**
@@ -479,7 +482,12 @@ static int offset_readdir(struct file *file, struct dir_context *ctx)
 	if (!dir_emit_dots(file, ctx))
 		return 0;
 
-	offset_iterate_dir(d_inode(dir), ctx);
+	/* In this case, ->private_data is protected by f_pos_lock */
+	if (ctx->pos == 2)
+		file->private_data = NULL;
+	else if (file->private_data == ERR_PTR(-ENOENT))
+		return 0;
+	file->private_data = offset_iterate_dir(d_inode(dir), ctx);
 	return 0;
 }
 
@@ -1765,9 +1773,71 @@ static int generic_ci_d_hash(const struct dentry *dentry, struct qstr *str)
 	return 0;
 }
 
-static const struct dentry_operations generic_ci_dentry_ops = {
+static int ci_d_revalidate(struct dentry *dentry, const struct qstr *name,
+			   unsigned int flags)
+{
+	const struct dentry *parent;
+	const struct inode *dir;
+
+	if (!d_is_negative(dentry))
+		return 1;
+
+	parent = READ_ONCE(dentry->d_parent);
+	dir = READ_ONCE(parent->d_inode);
+
+	if (!dir || !IS_CASEFOLDED(dir))
+		return 1;
+
+	/*
+	 * Negative dentries created prior to turning the directory
+	 * case-insensitive cannot be trusted, since they don't ensure
+	 * any possible case version of the filename doesn't exist.
+	 */
+	if (!d_is_casefolded_name(dentry))
+		return 0;
+
+	/*
+	 * If the lookup is for creation, then a negative dentry can only be
+	 * reused if it's a case-sensitive match, not just a case-insensitive
+	 * one.  This is needed to make the new file be created with the name
+	 * the user specified, preserving case.
+	 *
+	 * LOOKUP_CREATE or LOOKUP_RENAME_TARGET cover most creations.  In these
+	 * cases, ->d_name is stable and can be compared to 'name' without
+	 * taking ->d_lock because the caller must hold dir->i_rwsem.  (This
+	 * is because the directory lock blocks the dentry from being
+	 * concurrently instantiated, and negative dentries are never moved.)
+	 *
+	 * All other creations actually use flags==0.  These come from the edge
+	 * case of filesystems calling functions like lookup_one() that do a
+	 * lookup without setting the lookup flags at all.  Such lookups might
+	 * or might not be for creation, and if not don't guarantee stable
+	 * ->d_name.  Therefore, invalidate all negative dentries when flags==0.
+	 */
+	if (flags & (LOOKUP_CREATE | LOOKUP_RENAME_TARGET)) {
+		if (dentry->d_name.len != name->len ||
+		    memcmp(dentry->d_name.name, name->name, name->len))
+			return 0;
+	} else if (!flags) {
+		return 0;
+	}
+
+	return 1;
+}
+
+static int generic_ci_d_revalidate(struct dentry *dentry,
+				   const struct qstr *name,
+				   unsigned int flags)
+{
+	if (!ci_d_revalidate(dentry, name, flags))
+		return 0;
+	return fscrypt_d_revalidate(dentry, name, flags);
+}
+
+static const struct dentry_operations generic_encrypted_ci_dentry_ops = {
 	.d_hash = generic_ci_d_hash,
 	.d_compare = generic_ci_d_compare,
+	.d_revalidate = generic_ci_d_revalidate,
 };
 #endif
 
@@ -1777,26 +1847,19 @@ static const struct dentry_operations generic_encrypted_dentry_ops = {
 };
 #endif
 
-#if defined(CONFIG_FS_ENCRYPTION) && IS_ENABLED(CONFIG_UNICODE)
-static const struct dentry_operations generic_encrypted_ci_dentry_ops = {
-	.d_hash = generic_ci_d_hash,
-	.d_compare = generic_ci_d_compare,
-	.d_revalidate = fscrypt_d_revalidate,
-};
-#endif
-
 /**
  * generic_set_encrypted_ci_d_ops - helper for setting d_ops for given dentry
  * @dentry:	dentry to set ops on
  *
- * Casefolded directories need d_hash and d_compare set, so that the dentries
- * contained in them are handled case-insensitively.  Note that these operations
- * are needed on the parent directory rather than on the dentries in it, and
- * while the casefolding flag can be toggled on and off on an empty directory,
- * dentry_operations can't be changed later.  As a result, if the filesystem has
- * casefolding support enabled at all, we have to give all dentries the
- * casefolding operations even if their inode doesn't have the casefolding flag
- * currently (and thus the casefolding ops would be no-ops for now).
+ * Casefolded directories need some dentry_operations set, so that the dentries
+ * contained in them are handled case-insensitively.  Note that d_hash and
+ * d_compare are needed on the parent directory rather than on the dentries in
+ * it, and while the casefolding flag can be toggled on and off on an empty
+ * directory, dentry_operations can't be changed later.  As a result, if the
+ * filesystem has casefolding support enabled at all, we have to give all
+ * dentries the casefolding operations even if their inode doesn't have the
+ * casefolding flag currently (and thus the casefolding ops would be no-ops for
+ * now).
  *
  * Encryption works differently in that the only dentry operation it needs is
  * d_revalidate, which it only needs on dentries that have the no-key name flag.
@@ -1805,34 +1868,22 @@ static const struct dentry_operations generic_encrypted_ci_dentry_ops = {
  * Finally, to maximize compatibility with overlayfs (which isn't compatible
  * with certain dentry operations) and to avoid taking an unnecessary
  * performance hit, we use custom dentry_operations for each possible
- * combination rather than always installing all operations.
+ * combination of operations rather than always installing them.
  */
 void generic_set_encrypted_ci_d_ops(struct dentry *dentry)
 {
-#ifdef CONFIG_FS_ENCRYPTION
-	bool needs_encrypt_ops = dentry->d_flags & DCACHE_NOKEY_NAME;
-#endif
 #if IS_ENABLED(CONFIG_UNICODE)
-	bool needs_ci_ops = dentry->d_sb->s_encoding;
-#endif
-#if defined(CONFIG_FS_ENCRYPTION) && IS_ENABLED(CONFIG_UNICODE)
-	if (needs_encrypt_ops && needs_ci_ops) {
+	if (dentry->d_sb->s_encoding) {
 		d_set_d_op(dentry, &generic_encrypted_ci_dentry_ops);
 		return;
 	}
 #endif
 #ifdef CONFIG_FS_ENCRYPTION
-	if (needs_encrypt_ops) {
+	if (dentry->d_flags & DCACHE_NOKEY_NAME) {
 		d_set_d_op(dentry, &generic_encrypted_dentry_ops);
 		return;
 	}
 #endif
-#if IS_ENABLED(CONFIG_UNICODE)
-	if (needs_ci_ops) {
-		d_set_d_op(dentry, &generic_ci_dentry_ops);
-		return;
-	}
-#endif
 }
 EXPORT_SYMBOL(generic_set_encrypted_ci_d_ops);
 
diff --git a/fs/mount.h b/fs/mount.h
index 130c07c2f8d2..4a42fc68f4cc 100644
--- a/fs/mount.h
+++ b/fs/mount.h
@@ -8,19 +8,13 @@
 struct mnt_namespace {
 	struct ns_common	ns;
 	struct mount *	root;
-	/*
-	 * Traversal and modification of .list is protected by either
-	 * - taking namespace_sem for write, OR
-	 * - taking namespace_sem for read AND taking .ns_lock.
-	 */
-	struct list_head	list;
-	spinlock_t		ns_lock;
+	struct rb_root		mounts; /* Protected by namespace_sem */
 	struct user_namespace	*user_ns;
 	struct ucounts		*ucounts;
 	u64			seq;	/* Sequence number to prevent loops */
 	wait_queue_head_t poll;
 	u64 event;
-	unsigned int		mounts; /* # of mounts in the namespace */
+	unsigned int		nr_mounts; /* # of mounts in the namespace */
 	unsigned int		pending_mounts;
 } __randomize_layout;
 
@@ -55,7 +49,10 @@ struct mount {
 	struct list_head mnt_child;	/* and going through their mnt_child */
 	struct list_head mnt_instance;	/* mount instance on sb->s_mounts */
 	const char *mnt_devname;	/* Name of device e.g. /dev/dsk/hda1 */
-	struct list_head mnt_list;
+	union {
+		struct rb_node mnt_node;	/* Under ns->mounts */
+		struct list_head mnt_list;
+	};
 	struct list_head mnt_expire;	/* link in fs-specific expiry list */
 	struct list_head mnt_share;	/* circular list of shared mounts */
 	struct list_head mnt_slave_list;/* list of slave mounts */
@@ -72,7 +69,8 @@ struct mount {
 	struct fsnotify_mark_connector __rcu *mnt_fsnotify_marks;
 	__u32 mnt_fsnotify_mask;
 #endif
-	int mnt_id;			/* mount identifier */
+	int mnt_id;			/* mount identifier, reused */
+	u64 mnt_id_unique;		/* mount ID unique until reboot */
 	int mnt_group_id;		/* peer group identifier */
 	int mnt_expiry_mark;		/* true if marked for expiry */
 	struct hlist_head mnt_pins;
@@ -127,7 +125,6 @@ struct proc_mounts {
 	struct mnt_namespace *ns;
 	struct path root;
 	int (*show)(struct seq_file *, struct vfsmount *);
-	struct mount cursor;
 };
 
 extern const struct seq_operations mounts_op;
@@ -146,4 +143,12 @@ static inline bool is_anon_ns(struct mnt_namespace *ns)
 	return ns->seq == 0;
 }
 
+static inline void move_from_ns(struct mount *mnt, struct list_head *dt_list)
+{
+	WARN_ON(!(mnt->mnt.mnt_flags & MNT_ONRB));
+	mnt->mnt.mnt_flags &= ~MNT_ONRB;
+	rb_erase(&mnt->mnt_node, &mnt->mnt_ns->mounts);
+	list_add_tail(&mnt->mnt_list, dt_list);
+}
+
 extern void mnt_cursor_del(struct mnt_namespace *ns, struct mount *cursor);
diff --git a/fs/namei.c b/fs/namei.c
index 71c13b2990b4..490ef5d1c883 100644
--- a/fs/namei.c
+++ b/fs/namei.c
@@ -856,10 +856,12 @@ static bool try_to_unlazy_next(struct nameidata *nd, struct dentry *dentry)
 	return false;
 }
 
-static inline int d_revalidate(struct dentry *dentry, unsigned int flags)
+static inline int d_revalidate(struct dentry *dentry,
+			       const struct qstr *name,
+			       unsigned int flags)
 {
 	if (unlikely(dentry->d_flags & DCACHE_OP_REVALIDATE))
-		return dentry->d_op->d_revalidate(dentry, flags);
+		return dentry->d_op->d_revalidate(dentry, name, flags);
 	else
 		return 1;
 }
@@ -1569,7 +1571,7 @@ static struct dentry *lookup_dcache(const struct qstr *name,
 {
 	struct dentry *dentry = d_lookup(dir, name);
 	if (dentry) {
-		int error = d_revalidate(dentry, flags);
+		int error = d_revalidate(dentry, name, flags);
 		if (unlikely(error <= 0)) {
 			if (!error)
 				d_invalidate(dentry);
@@ -1640,19 +1642,19 @@ static struct dentry *lookup_fast(struct nameidata *nd)
 		if (read_seqcount_retry(&parent->d_seq, nd->seq))
 			return ERR_PTR(-ECHILD);
 
-		status = d_revalidate(dentry, nd->flags);
+		status = d_revalidate(dentry, &nd->last, nd->flags);
 		if (likely(status > 0))
 			return dentry;
 		if (!try_to_unlazy_next(nd, dentry))
 			return ERR_PTR(-ECHILD);
 		if (status == -ECHILD)
 			/* we'd been told to redo it in non-rcu mode */
-			status = d_revalidate(dentry, nd->flags);
+			status = d_revalidate(dentry, &nd->last, nd->flags);
 	} else {
 		dentry = __d_lookup(parent, &nd->last);
 		if (unlikely(!dentry))
 			return NULL;
-		status = d_revalidate(dentry, nd->flags);
+		status = d_revalidate(dentry, &nd->last, nd->flags);
 	}
 	if (unlikely(status <= 0)) {
 		if (!status)
@@ -1680,7 +1682,7 @@ static struct dentry *__lookup_slow(const struct qstr *name,
 	if (IS_ERR(dentry))
 		return dentry;
 	if (unlikely(!d_in_lookup(dentry))) {
-		int error = d_revalidate(dentry, flags);
+		int error = d_revalidate(dentry, name, flags);
 		if (unlikely(error <= 0)) {
 			if (!error) {
 				d_invalidate(dentry);
@@ -2467,7 +2469,7 @@ static int handle_lookup_down(struct nameidata *nd)
 	return PTR_ERR(step_into(nd, WALK_NOFOLLOW, nd->path.dentry));
 }
 
-/* Returns 0 and nd will be valid on success; Retuns error, otherwise. */
+/* Returns 0 and nd will be valid on success; Returns error, otherwise. */
 static int path_lookupat(struct nameidata *nd, unsigned flags, struct path *path)
 {
 	const char *s = path_init(nd, flags);
@@ -2522,7 +2524,7 @@ int filename_lookup(int dfd, struct filename *name, unsigned flags,
 	return retval;
 }
 
-/* Returns 0 and nd will be valid on success; Retuns error, otherwise. */
+/* Returns 0 and nd will be valid on success; Returns error, otherwise. */
 static int path_parentat(struct nameidata *nd, unsigned flags,
 				struct path *parent)
 {
@@ -2573,13 +2575,13 @@ static int filename_parentat(int dfd, struct filename *name,
 }
 
 /* does lookup, returns the object with parent locked */
-static struct dentry *__kern_path_locked(struct filename *name, struct path *path)
+static struct dentry *__kern_path_locked(int dfd, struct filename *name, struct path *path)
 {
 	struct dentry *d;
 	struct qstr last;
 	int type, error;
 
-	error = filename_parentat(AT_FDCWD, name, 0, path, &last, &type);
+	error = filename_parentat(dfd, name, 0, path, &last, &type);
 	if (error)
 		return ERR_PTR(error);
 	if (unlikely(type != LAST_NORM)) {
@@ -2598,12 +2600,22 @@ static struct dentry *__kern_path_locked(struct filename *name, struct path *pat
 struct dentry *kern_path_locked(const char *name, struct path *path)
 {
 	struct filename *filename = getname_kernel(name);
-	struct dentry *res = __kern_path_locked(filename, path);
+	struct dentry *res = __kern_path_locked(AT_FDCWD, filename, path);
 
 	putname(filename);
 	return res;
 }
 
+struct dentry *user_path_locked_at(int dfd, const char __user *name, struct path *path)
+{
+	struct filename *filename = getname(name);
+	struct dentry *res = __kern_path_locked(dfd, filename, path);
+
+	putname(filename);
+	return res;
+}
+EXPORT_SYMBOL(user_path_locked_at);
+
 int kern_path(const char *name, unsigned int flags, struct path *path)
 {
 	struct filename *filename = getname_kernel(name);
@@ -3406,7 +3418,7 @@ static struct dentry *lookup_open(struct nameidata *nd, struct file *file,
 		if (d_in_lookup(dentry))
 			break;
 
-		error = d_revalidate(dentry, nd->flags);
+		error = d_revalidate(dentry, &nd->last, nd->flags);
 		if (likely(error > 0))
 			break;
 		if (error)
diff --git a/fs/namespace.c b/fs/namespace.c
index fbf0e596fcd3..2ec1edd05fdd 100644
--- a/fs/namespace.c
+++ b/fs/namespace.c
@@ -32,6 +32,7 @@
 #include <linux/fs_context.h>
 #include <linux/shmem_fs.h>
 #include <linux/mnt_idmapping.h>
+#include <linux/nospec.h>
 
 #include "pnode.h"
 #include "internal.h"
@@ -68,6 +69,9 @@ static u64 event;
 static DEFINE_IDA(mnt_id_ida);
 static DEFINE_IDA(mnt_group_ida);
 
+/* Don't allow confusion with old 32bit mount ID */
+static atomic64_t mnt_id_ctr = ATOMIC64_INIT(1ULL << 32);
+
 static struct hlist_head *mount_hashtable __ro_after_init;
 static struct hlist_head *mountpoint_hashtable __ro_after_init;
 static struct kmem_cache *mnt_cache __ro_after_init;
@@ -131,6 +135,7 @@ static int mnt_alloc_id(struct mount *mnt)
 	if (res < 0)
 		return res;
 	mnt->mnt_id = res;
+	mnt->mnt_id_unique = atomic64_inc_return(&mnt_id_ctr);
 	return 0;
 }
 
@@ -730,21 +735,6 @@ struct vfsmount *lookup_mnt(const struct path *path)
 	return m;
 }
 
-static inline void lock_ns_list(struct mnt_namespace *ns)
-{
-	spin_lock(&ns->ns_lock);
-}
-
-static inline void unlock_ns_list(struct mnt_namespace *ns)
-{
-	spin_unlock(&ns->ns_lock);
-}
-
-static inline bool mnt_is_cursor(struct mount *mnt)
-{
-	return mnt->mnt.mnt_flags & MNT_CURSOR;
-}
-
 /*
  * __is_local_mountpoint - Test to see if dentry is a mountpoint in the
  *                         current mount namespace.
@@ -763,19 +753,15 @@ static inline bool mnt_is_cursor(struct mount *mnt)
 bool __is_local_mountpoint(struct dentry *dentry)
 {
 	struct mnt_namespace *ns = current->nsproxy->mnt_ns;
-	struct mount *mnt;
+	struct mount *mnt, *n;
 	bool is_covered = false;
 
 	down_read(&namespace_sem);
-	lock_ns_list(ns);
-	list_for_each_entry(mnt, &ns->list, mnt_list) {
-		if (mnt_is_cursor(mnt))
-			continue;
+	rbtree_postorder_for_each_entry_safe(mnt, n, &ns->mounts, mnt_node) {
 		is_covered = (mnt->mnt_mountpoint == dentry);
 		if (is_covered)
 			break;
 	}
-	unlock_ns_list(ns);
 	up_read(&namespace_sem);
 
 	return is_covered;
@@ -1022,6 +1008,30 @@ void mnt_change_mountpoint(struct mount *parent, struct mountpoint *mp, struct m
 	mnt_add_count(old_parent, -1);
 }
 
+static inline struct mount *node_to_mount(struct rb_node *node)
+{
+	return rb_entry(node, struct mount, mnt_node);
+}
+
+static void mnt_add_to_ns(struct mnt_namespace *ns, struct mount *mnt)
+{
+	struct rb_node **link = &ns->mounts.rb_node;
+	struct rb_node *parent = NULL;
+
+	WARN_ON(mnt->mnt.mnt_flags & MNT_ONRB);
+	mnt->mnt_ns = ns;
+	while (*link) {
+		parent = *link;
+		if (mnt->mnt_id_unique < node_to_mount(parent)->mnt_id_unique)
+			link = &parent->rb_left;
+		else
+			link = &parent->rb_right;
+	}
+	rb_link_node(&mnt->mnt_node, parent, link);
+	rb_insert_color(&mnt->mnt_node, &ns->mounts);
+	mnt->mnt.mnt_flags |= MNT_ONRB;
+}
+
 /*
  * vfsmount lock must be held for write
  */
@@ -1035,12 +1045,13 @@ static void commit_tree(struct mount *mnt)
 	BUG_ON(parent == mnt);
 
 	list_add_tail(&head, &mnt->mnt_list);
-	list_for_each_entry(m, &head, mnt_list)
-		m->mnt_ns = n;
-
-	list_splice(&head, n->list.prev);
+	while (!list_empty(&head)) {
+		m = list_first_entry(&head, typeof(*m), mnt_list);
+		list_del(&m->mnt_list);
 
-	n->mounts += n->pending_mounts;
+		mnt_add_to_ns(n, m);
+	}
+	n->nr_mounts += n->pending_mounts;
 	n->pending_mounts = 0;
 
 	__attach_mnt(mnt, parent);
@@ -1188,7 +1199,7 @@ static struct mount *clone_mnt(struct mount *old, struct dentry *root,
 	}
 
 	mnt->mnt.mnt_flags = old->mnt.mnt_flags;
-	mnt->mnt.mnt_flags &= ~(MNT_WRITE_HOLD|MNT_MARKED|MNT_INTERNAL);
+	mnt->mnt.mnt_flags &= ~(MNT_WRITE_HOLD|MNT_MARKED|MNT_INTERNAL|MNT_ONRB);
 
 	atomic_inc(&sb->s_active);
 	mnt->mnt.mnt_idmap = mnt_idmap_get(mnt_idmap(&old->mnt));
@@ -1413,65 +1424,57 @@ struct vfsmount *mnt_clone_internal(const struct path *path)
 	return &p->mnt;
 }
 
-#ifdef CONFIG_PROC_FS
-static struct mount *mnt_list_next(struct mnt_namespace *ns,
-				   struct list_head *p)
+/*
+ * Returns the mount which either has the specified mnt_id, or has the next
+ * smallest id afer the specified one.
+ */
+static struct mount *mnt_find_id_at(struct mnt_namespace *ns, u64 mnt_id)
 {
-	struct mount *mnt, *ret = NULL;
+	struct rb_node *node = ns->mounts.rb_node;
+	struct mount *ret = NULL;
 
-	lock_ns_list(ns);
-	list_for_each_continue(p, &ns->list) {
-		mnt = list_entry(p, typeof(*mnt), mnt_list);
-		if (!mnt_is_cursor(mnt)) {
-			ret = mnt;
-			break;
+	while (node) {
+		struct mount *m = node_to_mount(node);
+
+		if (mnt_id <= m->mnt_id_unique) {
+			ret = node_to_mount(node);
+			if (mnt_id == m->mnt_id_unique)
+				break;
+			node = node->rb_left;
+		} else {
+			node = node->rb_right;
 		}
 	}
-	unlock_ns_list(ns);
-
 	return ret;
 }
 
+#ifdef CONFIG_PROC_FS
+
 /* iterator; we want it to have access to namespace_sem, thus here... */
 static void *m_start(struct seq_file *m, loff_t *pos)
 {
 	struct proc_mounts *p = m->private;
-	struct list_head *prev;
 
 	down_read(&namespace_sem);
-	if (!*pos) {
-		prev = &p->ns->list;
-	} else {
-		prev = &p->cursor.mnt_list;
-
-		/* Read after we'd reached the end? */
-		if (list_empty(prev))
-			return NULL;
-	}
 
-	return mnt_list_next(p->ns, prev);
+	return mnt_find_id_at(p->ns, *pos);
 }
 
 static void *m_next(struct seq_file *m, void *v, loff_t *pos)
 {
-	struct proc_mounts *p = m->private;
-	struct mount *mnt = v;
+	struct mount *next = NULL, *mnt = v;
+	struct rb_node *node = rb_next(&mnt->mnt_node);
 
 	++*pos;
-	return mnt_list_next(p->ns, &mnt->mnt_list);
+	if (node) {
+		next = node_to_mount(node);
+		*pos = next->mnt_id_unique;
+	}
+	return next;
 }
 
 static void m_stop(struct seq_file *m, void *v)
 {
-	struct proc_mounts *p = m->private;
-	struct mount *mnt = v;
-
-	lock_ns_list(p->ns);
-	if (mnt)
-		list_move_tail(&p->cursor.mnt_list, &mnt->mnt_list);
-	else
-		list_del_init(&p->cursor.mnt_list);
-	unlock_ns_list(p->ns);
 	up_read(&namespace_sem);
 }
 
@@ -1489,14 +1492,6 @@ const struct seq_operations mounts_op = {
 	.show	= m_show,
 };
 
-void mnt_cursor_del(struct mnt_namespace *ns, struct mount *cursor)
-{
-	down_read(&namespace_sem);
-	lock_ns_list(ns);
-	list_del(&cursor->mnt_list);
-	unlock_ns_list(ns);
-	up_read(&namespace_sem);
-}
 #endif  /* CONFIG_PROC_FS */
 
 /**
@@ -1638,7 +1633,10 @@ static void umount_tree(struct mount *mnt, enum umount_tree_flags how)
 	/* Gather the mounts to umount */
 	for (p = mnt; p; p = next_mnt(p, mnt)) {
 		p->mnt.mnt_flags |= MNT_UMOUNT;
-		list_move(&p->mnt_list, &tmp_list);
+		if (p->mnt.mnt_flags & MNT_ONRB)
+			move_from_ns(p, &tmp_list);
+		else
+			list_move(&p->mnt_list, &tmp_list);
 	}
 
 	/* Hide the mounts from mnt_mounts */
@@ -1658,7 +1656,7 @@ static void umount_tree(struct mount *mnt, enum umount_tree_flags how)
 		list_del_init(&p->mnt_list);
 		ns = p->mnt_ns;
 		if (ns) {
-			ns->mounts--;
+			ns->nr_mounts--;
 			__touch_mnt_namespace(ns);
 		}
 		p->mnt_ns = NULL;
@@ -1784,14 +1782,16 @@ static int do_umount(struct mount *mnt, int flags)
 
 	event++;
 	if (flags & MNT_DETACH) {
-		if (!list_empty(&mnt->mnt_list))
+		if (mnt->mnt.mnt_flags & MNT_ONRB ||
+		    !list_empty(&mnt->mnt_list))
 			umount_tree(mnt, UMOUNT_PROPAGATE);
 		retval = 0;
 	} else {
 		shrink_submounts(mnt);
 		retval = -EBUSY;
 		if (!propagate_mount_busy(mnt, 2)) {
-			if (!list_empty(&mnt->mnt_list))
+			if (mnt->mnt.mnt_flags & MNT_ONRB ||
+			    !list_empty(&mnt->mnt_list))
 				umount_tree(mnt, UMOUNT_PROPAGATE|UMOUNT_SYNC);
 			retval = 0;
 		}
@@ -2209,9 +2209,9 @@ int count_mounts(struct mnt_namespace *ns, struct mount *mnt)
 	unsigned int mounts = 0;
 	struct mount *p;
 
-	if (ns->mounts >= max)
+	if (ns->nr_mounts >= max)
 		return -ENOSPC;
-	max -= ns->mounts;
+	max -= ns->nr_mounts;
 	if (ns->pending_mounts >= max)
 		return -ENOSPC;
 	max -= ns->pending_mounts;
@@ -2355,8 +2355,12 @@ static int attach_recursive_mnt(struct mount *source_mnt,
 		touch_mnt_namespace(source_mnt->mnt_ns);
 	} else {
 		if (source_mnt->mnt_ns) {
+			LIST_HEAD(head);
+
 			/* move from anon - the caller will destroy */
-			list_del_init(&source_mnt->mnt_ns->list);
+			for (p = source_mnt; p; p = next_mnt(p, source_mnt))
+				move_from_ns(p, &head);
+			list_del_init(&head);
 		}
 		if (beneath)
 			mnt_set_mountpoint_beneath(source_mnt, top_mnt, smp);
@@ -2667,11 +2671,10 @@ static struct file *open_detached_copy(struct path *path, bool recursive)
 
 	lock_mount_hash();
 	for (p = mnt; p; p = next_mnt(p, mnt)) {
-		p->mnt_ns = ns;
-		ns->mounts++;
+		mnt_add_to_ns(ns, p);
+		ns->nr_mounts++;
 	}
 	ns->root = mnt;
-	list_add_tail(&ns->list, &mnt->mnt_list);
 	mntget(&mnt->mnt);
 	unlock_mount_hash();
 	namespace_unlock();
@@ -3026,6 +3029,7 @@ static inline bool path_overmounted(const struct path *path)
  * can_move_mount_beneath - check that we can mount beneath the top mount
  * @from: mount to mount beneath
  * @to:   mount under which to mount
+ * @mp:   mountpoint of @to
  *
  * - Make sure that @to->dentry is actually the root of a mount under
  *   which we can mount another mount.
@@ -3734,9 +3738,8 @@ static struct mnt_namespace *alloc_mnt_ns(struct user_namespace *user_ns, bool a
 	if (!anon)
 		new_ns->seq = atomic64_add_return(1, &mnt_ns_seq);
 	refcount_set(&new_ns->ns.count, 1);
-	INIT_LIST_HEAD(&new_ns->list);
+	new_ns->mounts = RB_ROOT;
 	init_waitqueue_head(&new_ns->poll);
-	spin_lock_init(&new_ns->ns_lock);
 	new_ns->user_ns = get_user_ns(user_ns);
 	new_ns->ucounts = ucounts;
 	return new_ns;
@@ -3783,7 +3786,6 @@ struct mnt_namespace *copy_mnt_ns(unsigned long flags, struct mnt_namespace *ns,
 		unlock_mount_hash();
 	}
 	new_ns->root = new;
-	list_add_tail(&new_ns->list, &new->mnt_list);
 
 	/*
 	 * Second pass: switch the tsk->fs->* elements and mark new vfsmounts
@@ -3793,8 +3795,8 @@ struct mnt_namespace *copy_mnt_ns(unsigned long flags, struct mnt_namespace *ns,
 	p = old;
 	q = new;
 	while (p) {
-		q->mnt_ns = new_ns;
-		new_ns->mounts++;
+		mnt_add_to_ns(new_ns, q);
+		new_ns->nr_mounts++;
 		if (new_fs) {
 			if (&p->mnt == new_fs->root.mnt) {
 				new_fs->root.mnt = mntget(&q->mnt);
@@ -3836,10 +3838,9 @@ struct dentry *mount_subtree(struct vfsmount *m, const char *name)
 		mntput(m);
 		return ERR_CAST(ns);
 	}
-	mnt->mnt_ns = ns;
 	ns->root = mnt;
-	ns->mounts++;
-	list_add(&mnt->mnt_list, &ns->list);
+	ns->nr_mounts++;
+	mnt_add_to_ns(ns, mnt);
 
 	err = vfs_path_lookup(m->mnt_root, m,
 			name, LOOKUP_FOLLOW|LOOKUP_AUTOMOUNT, &path);
@@ -4017,10 +4018,9 @@ SYSCALL_DEFINE3(fsmount, int, fs_fd, unsigned int, flags,
 		goto err_path;
 	}
 	mnt = real_mount(newmount.mnt);
-	mnt->mnt_ns = ns;
 	ns->root = mnt;
-	ns->mounts = 1;
-	list_add(&mnt->mnt_list, &ns->list);
+	ns->nr_mounts = 1;
+	mnt_add_to_ns(ns, mnt);
 	mntget(newmount.mnt);
 
 	/* Attach to an apparent O_PATH fd with a note that we need to unmount
@@ -4676,6 +4676,441 @@ SYSCALL_DEFINE5(mount_setattr, int, dfd, const char __user *, path,
 	return err;
 }
 
+int show_path(struct seq_file *m, struct dentry *root)
+{
+	if (root->d_sb->s_op->show_path)
+		return root->d_sb->s_op->show_path(m, root);
+
+	seq_dentry(m, root, " \t\n\\");
+	return 0;
+}
+
+static struct vfsmount *lookup_mnt_in_ns(u64 id, struct mnt_namespace *ns)
+{
+	struct mount *mnt = mnt_find_id_at(ns, id);
+
+	if (!mnt || mnt->mnt_id_unique != id)
+		return NULL;
+
+	return &mnt->mnt;
+}
+
+struct kstatmount {
+	struct statmount __user *const buf;
+	size_t const bufsize;
+	struct vfsmount *const mnt;
+	u64 const mask;
+	struct path root;
+	struct statmount sm;
+	char *fs_type;
+	size_t fs_type_len;
+	char *mnt_root;
+	size_t mnt_root_len;
+	char *mnt_point;
+	size_t mnt_point_len;
+	size_t pos;
+};
+
+static u64 mnt_to_attr_flags(struct vfsmount *mnt)
+{
+	unsigned int mnt_flags = READ_ONCE(mnt->mnt_flags);
+	u64 attr_flags = 0;
+
+	if (mnt_flags & MNT_READONLY)
+		attr_flags |= MOUNT_ATTR_RDONLY;
+	if (mnt_flags & MNT_NOSUID)
+		attr_flags |= MOUNT_ATTR_NOSUID;
+	if (mnt_flags & MNT_NODEV)
+		attr_flags |= MOUNT_ATTR_NODEV;
+	if (mnt_flags & MNT_NOEXEC)
+		attr_flags |= MOUNT_ATTR_NOEXEC;
+	if (mnt_flags & MNT_NODIRATIME)
+		attr_flags |= MOUNT_ATTR_NODIRATIME;
+	if (mnt_flags & MNT_NOSYMFOLLOW)
+		attr_flags |= MOUNT_ATTR_NOSYMFOLLOW;
+
+	if (mnt_flags & MNT_NOATIME)
+		attr_flags |= MOUNT_ATTR_NOATIME;
+	else if (mnt_flags & MNT_RELATIME)
+		attr_flags |= MOUNT_ATTR_RELATIME;
+	else
+		attr_flags |= MOUNT_ATTR_STRICTATIME;
+
+	if (is_idmapped_mnt(mnt))
+		attr_flags |= MOUNT_ATTR_IDMAP;
+
+	return attr_flags;
+}
+
+static u64 mnt_to_propagation_flags(struct mount *m)
+{
+	u64 propagation = 0;
+
+	if (IS_MNT_SHARED(m))
+		propagation |= MS_SHARED;
+	if (IS_MNT_SLAVE(m))
+		propagation |= MS_SLAVE;
+	if (IS_MNT_UNBINDABLE(m))
+		propagation |= MS_UNBINDABLE;
+	if (!propagation)
+		propagation |= MS_PRIVATE;
+
+	return propagation;
+}
+
+static void statmount_sb_basic(struct kstatmount *s)
+{
+	struct super_block *sb = s->mnt->mnt_sb;
+
+	s->sm.mask |= STATMOUNT_SB_BASIC;
+	s->sm.sb_dev_major = MAJOR(sb->s_dev);
+	s->sm.sb_dev_minor = MINOR(sb->s_dev);
+	s->sm.sb_magic = sb->s_magic;
+	s->sm.sb_flags = sb->s_flags & (SB_RDONLY|SB_SYNCHRONOUS|SB_DIRSYNC|SB_LAZYTIME);
+}
+
+static void statmount_mnt_basic(struct kstatmount *s)
+{
+	struct mount *m = real_mount(s->mnt);
+
+	s->sm.mask |= STATMOUNT_MNT_BASIC;
+	s->sm.mnt_id = m->mnt_id_unique;
+	s->sm.mnt_parent_id = m->mnt_parent->mnt_id_unique;
+	s->sm.mnt_id_old = m->mnt_id;
+	s->sm.mnt_parent_id_old = m->mnt_parent->mnt_id;
+	s->sm.mnt_attr = mnt_to_attr_flags(&m->mnt);
+	s->sm.mnt_propagation = mnt_to_propagation_flags(m);
+	s->sm.mnt_peer_group = IS_MNT_SHARED(m) ? m->mnt_group_id : 0;
+	s->sm.mnt_master = IS_MNT_SLAVE(m) ? m->mnt_master->mnt_group_id : 0;
+}
+
+static void statmount_propagate_from(struct kstatmount *s)
+{
+	struct mount *m = real_mount(s->mnt);
+
+	s->sm.mask |= STATMOUNT_PROPAGATE_FROM;
+	if (IS_MNT_SLAVE(m))
+		s->sm.propagate_from = get_dominating_id(m, &current->fs->root);
+}
+
+static int statmount_mnt_root(struct kstatmount *s, struct seq_file *seq)
+{
+	int err;
+
+	err = show_path(seq, s->mnt->mnt_root);
+	if (!err && !seq_has_overflowed(seq)) {
+		seq->buf[seq->count] = '\0';
+		seq->count = string_unescape_inplace(seq->buf, UNESCAPE_OCTAL);
+	}
+	return err;
+}
+
+static int statmount_mnt_point(struct kstatmount *s, struct seq_file *seq)
+{
+	struct vfsmount *mnt = s->mnt;
+	struct path mnt_path = { .dentry = mnt->mnt_root, .mnt = mnt };
+	int err;
+
+	err = seq_path_root(seq, &mnt_path, &s->root, "");
+	return err == SEQ_SKIP ? 0 : err;
+}
+
+static int statmount_fs_type(struct kstatmount *s, struct seq_file *seq)
+{
+	struct super_block *sb = s->mnt->mnt_sb;
+
+	seq_puts(seq, sb->s_type->name);
+	return 0;
+}
+
+static int statmount_string(struct kstatmount *s, u64 flag)
+{
+	struct seq_file seq = {};
+	char *buf;
+	struct statmount *sm = &s->sm;
+	int ret;
+
+	if (s->pos + sizeof(s->sm) >= s->bufsize)
+		return -EOVERFLOW;
+
+	buf = __getname();
+	if (!buf)
+		return -ENOMEM;
+
+	seq.count = 0;
+	seq.size = PATH_MAX;
+	seq.buf = buf;
+
+	switch (flag) {
+	case STATMOUNT_FS_TYPE:
+		ret = statmount_fs_type(s, &seq);
+		break;
+	case STATMOUNT_MNT_ROOT:
+		ret = statmount_mnt_root(s, &seq);
+		break;
+	case STATMOUNT_MNT_POINT:
+		ret = statmount_mnt_point(s, &seq);
+		break;
+	default:
+		WARN_ON_ONCE(true);
+		ret = -EINVAL;
+	}
+	if (unlikely(seq_has_overflowed(&seq)))
+		ret = -EOVERFLOW;
+
+	if (ret) {
+		__putname(buf);
+		return ret;
+	}
+
+	/* Use the last byte as the \0 byte. */
+	seq.buf[seq.count++] = '\0';
+
+	switch (flag) {
+	case STATMOUNT_FS_TYPE:
+		sm->fs_type = s->pos;
+		s->fs_type = buf;
+		s->fs_type_len = seq.count;
+		break;
+	case STATMOUNT_MNT_ROOT:
+		sm->mnt_root = s->pos;
+		s->mnt_root = buf;
+		s->mnt_root_len = seq.count;
+		break;
+	case STATMOUNT_MNT_POINT:
+		sm->mnt_point = s->pos;
+		s->mnt_point = buf;
+		s->mnt_point_len = seq.count;
+		break;
+	}
+
+	s->pos += seq.count;
+	sm->mask |= flag;
+	return 0;
+}
+
+static int copy_statmount_to_user(struct kstatmount *s)
+{
+	struct statmount *sm = &s->sm;
+	size_t copysize = min_t(size_t, s->bufsize, sizeof(*sm));
+	int ret = 0;
+
+	if (sm->mask & STATMOUNT_FS_TYPE)
+		ret = copy_to_user(s->buf->str + sm->fs_type,
+				   s->fs_type, s->fs_type_len);
+
+	if (!ret && sm->mask & STATMOUNT_MNT_ROOT)
+		ret = copy_to_user(s->buf->str + sm->mnt_root,
+				   s->mnt_root, s->mnt_root_len);
+
+	if (!ret && sm->mask & STATMOUNT_MNT_POINT)
+		ret = copy_to_user(s->buf->str + sm->mnt_point,
+				   s->mnt_point, s->mnt_point_len);
+
+	if (ret)
+		return -EFAULT;
+
+	/* Return the number of bytes copied to the buffer */
+	sm->size = copysize + s->pos;
+	if (copy_to_user(s->buf, sm, copysize))
+		return -EFAULT;
+
+	return 0;
+}
+
+static int do_statmount(struct kstatmount *s)
+{
+	struct mount *m = real_mount(s->mnt);
+	int err;
+
+	if (!capable(CAP_SYS_ADMIN) &&
+	    !is_path_reachable(m, m->mnt.mnt_root, &s->root))
+		return -EPERM;
+
+	err = security_sb_statfs(s->mnt->mnt_root);
+	if (err)
+		return err;
+
+	if (s->mask & STATMOUNT_SB_BASIC)
+		statmount_sb_basic(s);
+
+	if (s->mask & STATMOUNT_MNT_BASIC)
+		statmount_mnt_basic(s);
+
+	if (s->mask & STATMOUNT_PROPAGATE_FROM)
+		statmount_propagate_from(s);
+
+	if (s->mask & STATMOUNT_FS_TYPE)
+		err = statmount_string(s, STATMOUNT_FS_TYPE);
+
+	if (!err && s->mask & STATMOUNT_MNT_ROOT)
+		err = statmount_string(s, STATMOUNT_MNT_ROOT);
+
+	if (!err && s->mask & STATMOUNT_MNT_POINT)
+		err = statmount_string(s, STATMOUNT_MNT_POINT);
+
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static inline void drop_kstatmount(struct kstatmount *ks)
+{
+	if (ks->fs_type)
+		__putname(ks->fs_type);
+	if (ks->mnt_root)
+		__putname(ks->mnt_root);
+	if (ks->mnt_point)
+		__putname(ks->mnt_point);
+}
+
+SYSCALL_DEFINE4(statmount, const struct mnt_id_req __user *, req,
+		struct statmount __user *, buf, size_t, bufsize,
+		unsigned int, flags)
+{
+	struct vfsmount *mnt;
+	struct mnt_id_req kreq;
+	struct kstatmount *ks;
+	int ret;
+
+	if (flags)
+		return -EINVAL;
+
+	if (copy_from_user(&kreq, req, sizeof(kreq)))
+		return -EFAULT;
+
+	down_read(&namespace_sem);
+	mnt = lookup_mnt_in_ns(kreq.mnt_id, current->nsproxy->mnt_ns);
+	if (!mnt) {
+		up_read(&namespace_sem);
+		return -ENOENT;
+	}
+
+	ks = &(struct kstatmount){
+		.mask		= kreq.request_mask,
+		.buf		= buf,
+		.bufsize	= bufsize,
+		.mnt		= mnt,
+	};
+
+	get_fs_root(current->fs, &ks->root);
+	ret = do_statmount(ks);
+	path_put(&ks->root);
+	up_read(&namespace_sem);
+
+	if (!ret)
+		ret = copy_statmount_to_user(ks);
+	drop_kstatmount(ks);
+	return ret;
+}
+
+static struct mount *listmnt_first(struct mount *root)
+{
+	return list_first_entry_or_null(&root->mnt_mounts, struct mount, mnt_child);
+}
+
+static struct mount *listmnt_next(struct mount *curr, struct mount *root, bool recurse)
+{
+	if (recurse)
+		return next_mnt(curr, root);
+	if (!list_is_head(curr->mnt_child.next, &root->mnt_mounts))
+		return list_next_entry(curr, mnt_child);
+	return NULL;
+}
+
+static ssize_t do_listmount(struct vfsmount *mnt, u64 __user *buf,
+			    size_t bufsize, const struct path *root,
+			    unsigned int flags)
+{
+	struct mount *r, *m = real_mount(mnt);
+	struct path path_mnt_root = {
+		.mnt = root->mnt,
+		.dentry = root->mnt->mnt_root
+	};
+	ssize_t ctr;
+	bool unreachable = (flags & LISTMOUNT_UNREACHABLE);
+	bool recurse = flags & LISTMOUNT_RECURSIVE;
+	int err;
+
+	/*
+	 * The caller explicitly requested to see all mounts including those
+	 * that aren't reachable from the caller's mount root so check that
+	 * they have the privileges to do so.
+	 */
+	if (unreachable && !capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
+	/*
+	 * The caller requested to only list reachable mounts so ensure that
+	 * the mount whose children we want to see is actually reachable from
+	 * the caller's mount root.
+	 *
+	 * If not, check whether the caller does have sufficient privileges to
+	 * also list unreachable mounts. If they lack privileges we need to
+	 * return a permission error and if they do, we need to return success.
+	 */
+	if (!unreachable && !is_path_reachable(m, mnt->mnt_root, &path_mnt_root))
+		return capable(CAP_SYS_ADMIN) ? 0 : -EPERM;
+
+	err = security_sb_statfs(mnt->mnt_root);
+	if (err)
+		return err;
+
+	for (ctr = 0, r = listmnt_first(m); r; r = listmnt_next(r, m, recurse)) {
+		if (!unreachable && !is_path_reachable(r, r->mnt.mnt_root, root))
+			continue;
+
+		if (ctr >= bufsize)
+			return -EOVERFLOW;
+		ctr = array_index_nospec(ctr, bufsize);
+		if (put_user(r->mnt_id_unique, buf + ctr))
+			return -EFAULT;
+		if (check_add_overflow(ctr, 1, &ctr))
+			return -ERANGE;
+	}
+	return ctr;
+}
+
+SYSCALL_DEFINE4(listmount, const struct mnt_id_req __user *, req,
+		u64 __user *, buf, size_t, bufsize, unsigned int, flags)
+{
+	struct mnt_id_req kreq;
+	struct vfsmount *mnt;
+	struct path root;
+	u64 mnt_id;
+	ssize_t ret;
+
+	if (flags & ~(LISTMOUNT_UNREACHABLE | LISTMOUNT_RECURSIVE))
+		return -EINVAL;
+
+	if (copy_from_user(&kreq, req, sizeof(kreq)))
+		return -EFAULT;
+	if (kreq.request_mask != 0)
+		return -EINVAL;
+	mnt_id = kreq.mnt_id;
+
+	down_read(&namespace_sem);
+	if (mnt_id == LSMT_ROOT)
+		mnt = &current->nsproxy->mnt_ns->root->mnt;
+	else
+		mnt = lookup_mnt_in_ns(mnt_id, current->nsproxy->mnt_ns);
+	if (!mnt) {
+		up_read(&namespace_sem);
+		return -ENOENT;
+	}
+
+	get_fs_root(current->fs, &root);
+	/* Skip unreachable for LSMT_ROOT */
+	if (mnt_id == LSMT_ROOT && !(flags & LISTMOUNT_UNREACHABLE))
+		mnt = root.mnt;
+	ret = do_listmount(mnt, buf, bufsize, &root, flags);
+	path_put(&root);
+	up_read(&namespace_sem);
+	return ret;
+}
+
+
 static void __init init_mount_tree(void)
 {
 	struct vfsmount *mnt;
@@ -4691,10 +5126,9 @@ static void __init init_mount_tree(void)
 	if (IS_ERR(ns))
 		panic("Can't allocate initial namespace");
 	m = real_mount(mnt);
-	m->mnt_ns = ns;
 	ns->root = m;
-	ns->mounts = 1;
-	list_add(&m->mnt_list, &ns->list);
+	ns->nr_mounts = 1;
+	mnt_add_to_ns(ns, m);
 	init_task.nsproxy->mnt_ns = ns;
 	get_mnt_ns(ns);
 
@@ -4821,18 +5255,14 @@ static bool mnt_already_visible(struct mnt_namespace *ns,
 				int *new_mnt_flags)
 {
 	int new_flags = *new_mnt_flags;
-	struct mount *mnt;
+	struct mount *mnt, *n;
 	bool visible = false;
 
 	down_read(&namespace_sem);
-	lock_ns_list(ns);
-	list_for_each_entry(mnt, &ns->list, mnt_list) {
+	rbtree_postorder_for_each_entry_safe(mnt, n, &ns->mounts, mnt_node) {
 		struct mount *child;
 		int mnt_flags;
 
-		if (mnt_is_cursor(mnt))
-			continue;
-
 		if (mnt->mnt.mnt_sb->s_type != sb->s_type)
 			continue;
 
@@ -4880,7 +5310,6 @@ static bool mnt_already_visible(struct mnt_namespace *ns,
 	next:	;
 	}
 found:
-	unlock_ns_list(ns);
 	up_read(&namespace_sem);
 	return visible;
 }
diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c
index 13dffe4201e6..b6e63916cb45 100644
--- a/fs/nfs/dir.c
+++ b/fs/nfs/dir.c
@@ -1812,7 +1812,8 @@ __nfs_lookup_revalidate(struct dentry *dentry, unsigned int flags,
 	return ret;
 }
 
-static int nfs_lookup_revalidate(struct dentry *dentry, unsigned int flags)
+static int nfs_lookup_revalidate(struct dentry *dentry,
+				 const struct qstr *name, unsigned int flags)
 {
 	return __nfs_lookup_revalidate(dentry, flags, nfs_do_lookup_revalidate);
 }
@@ -2000,7 +2001,8 @@ void nfs_d_prune_case_insensitive_aliases(struct inode *inode)
 EXPORT_SYMBOL_GPL(nfs_d_prune_case_insensitive_aliases);
 
 #if IS_ENABLED(CONFIG_NFS_V4)
-static int nfs4_lookup_revalidate(struct dentry *, unsigned int);
+static int nfs4_lookup_revalidate(struct dentry *, const struct qstr *name,
+				  unsigned int);
 
 const struct dentry_operations nfs4_dentry_operations = {
 	.d_revalidate	= nfs4_lookup_revalidate,
@@ -2233,7 +2235,8 @@ nfs4_do_lookup_revalidate(struct inode *dir, struct dentry *dentry,
 	return nfs_do_lookup_revalidate(dir, dentry, flags);
 }
 
-static int nfs4_lookup_revalidate(struct dentry *dentry, unsigned int flags)
+static int nfs4_lookup_revalidate(struct dentry *dentry,
+				  const struct qstr *name, unsigned int flags)
 {
 	return __nfs_lookup_revalidate(dentry, flags,
 			nfs4_do_lookup_revalidate);
diff --git a/fs/nfs/file.c b/fs/nfs/file.c
index 3f9768810427..e8cccb94b927 100644
--- a/fs/nfs/file.c
+++ b/fs/nfs/file.c
@@ -567,7 +567,7 @@ const struct address_space_operations nfs_file_aops = {
 	.migrate_folio = nfs_migrate_folio,
 	.launder_folio = nfs_launder_folio,
 	.is_dirty_writeback = nfs_check_dirty_writeback,
-	.error_remove_page = generic_error_remove_page,
+	.error_remove_folio = generic_error_remove_folio,
 	.swap_activate = nfs_swap_activate,
 	.swap_deactivate = nfs_swap_deactivate,
 	.swap_rw = nfs_swap_rw,
diff --git a/fs/nfs/write.c b/fs/nfs/write.c
index b664caea8b4e..7248705faef4 100644
--- a/fs/nfs/write.c
+++ b/fs/nfs/write.c
@@ -192,13 +192,13 @@ static struct nfs_page *nfs_folio_find_private_request(struct folio *folio)
 
 	if (!folio_test_private(folio))
 		return NULL;
-	spin_lock(&mapping->private_lock);
+	spin_lock(&mapping->i_private_lock);
 	req = nfs_folio_private_request(folio);
 	if (req) {
 		WARN_ON_ONCE(req->wb_head != req);
 		kref_get(&req->wb_kref);
 	}
-	spin_unlock(&mapping->private_lock);
+	spin_unlock(&mapping->i_private_lock);
 	return req;
 }
 
@@ -769,13 +769,13 @@ static void nfs_inode_add_request(struct nfs_page *req)
 	 * Swap-space should not get truncated. Hence no need to plug the race
 	 * with invalidate/truncate.
 	 */
-	spin_lock(&mapping->private_lock);
+	spin_lock(&mapping->i_private_lock);
 	if (likely(!folio_test_swapcache(folio))) {
 		set_bit(PG_MAPPED, &req->wb_flags);
 		folio_set_private(folio);
 		folio->private = req;
 	}
-	spin_unlock(&mapping->private_lock);
+	spin_unlock(&mapping->i_private_lock);
 	atomic_long_inc(&nfsi->nrequests);
 	/* this a head request for a page group - mark it as having an
 	 * extra reference so sub groups can follow suit.
@@ -796,13 +796,13 @@ static void nfs_inode_remove_request(struct nfs_page *req)
 		struct folio *folio = nfs_page_to_folio(req->wb_head);
 		struct address_space *mapping = folio_file_mapping(folio);
 
-		spin_lock(&mapping->private_lock);
+		spin_lock(&mapping->i_private_lock);
 		if (likely(folio && !folio_test_swapcache(folio))) {
 			folio->private = NULL;
 			folio_clear_private(folio);
 			clear_bit(PG_MAPPED, &req->wb_head->wb_flags);
 		}
-		spin_unlock(&mapping->private_lock);
+		spin_unlock(&mapping->i_private_lock);
 	}
 
 	if (test_and_clear_bit(PG_INODE_REF, &req->wb_flags)) {
diff --git a/fs/nfsd/Kconfig b/fs/nfsd/Kconfig
index 43b88eaf0673..272ab8d5c4d7 100644
--- a/fs/nfsd/Kconfig
+++ b/fs/nfsd/Kconfig
@@ -158,3 +158,19 @@ config NFSD_V4_SECURITY_LABEL
 
 	If you do not wish to enable fine-grained security labels SELinux or
 	Smack policies on NFSv4 files, say N.
+
+config NFSD_LEGACY_CLIENT_TRACKING
+	bool "Support legacy NFSv4 client tracking methods (DEPRECATED)"
+	depends on NFSD_V4
+	default n
+	help
+	  The NFSv4 server needs to store a small amount of information on
+	  stable storage in order to handle state recovery after reboot. Most
+	  modern deployments upcall to a userland daemon for this (nfsdcld),
+	  but older NFS servers may store information directly in a
+	  recoverydir, or spawn a process directly using a usermodehelper
+	  upcall.
+
+	  These legacy client tracking methods have proven to be probelmatic
+	  and will be removed in the future. Say Y here if you need support
+	  for them in the interim.
diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c
index ef063f93fde9..66dd5059f1bb 100644
--- a/fs/nfsd/filecache.c
+++ b/fs/nfsd/filecache.c
@@ -717,7 +717,7 @@ nfsd_file_cache_init(void)
 		return ret;
 
 	ret = -ENOMEM;
-	nfsd_filecache_wq = alloc_workqueue("nfsd_filecache", 0, 0);
+	nfsd_filecache_wq = alloc_workqueue("nfsd_filecache", WQ_UNBOUND, 0);
 	if (!nfsd_filecache_wq)
 		goto out;
 
diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c
index 3509e73abe1f..2c060e0b1604 100644
--- a/fs/nfsd/nfs4recover.c
+++ b/fs/nfsd/nfs4recover.c
@@ -66,6 +66,7 @@ struct nfsd4_client_tracking_ops {
 static const struct nfsd4_client_tracking_ops nfsd4_cld_tracking_ops;
 static const struct nfsd4_client_tracking_ops nfsd4_cld_tracking_ops_v2;
 
+#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING
 /* Globals */
 static char user_recovery_dirname[PATH_MAX] = "/var/lib/nfs/v4recovery";
 
@@ -720,6 +721,7 @@ static const struct nfsd4_client_tracking_ops nfsd4_legacy_tracking_ops = {
 	.version	= 1,
 	.msglen		= 0,
 };
+#endif /* CONFIG_NFSD_LEGACY_CLIENT_TRACKING */
 
 /* Globals */
 #define NFSD_PIPE_DIR		"nfsd"
@@ -731,8 +733,10 @@ struct cld_net {
 	spinlock_t		 cn_lock;
 	struct list_head	 cn_list;
 	unsigned int		 cn_xid;
-	bool			 cn_has_legacy;
 	struct crypto_shash	*cn_tfm;
+#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING
+	bool			 cn_has_legacy;
+#endif
 };
 
 struct cld_upcall {
@@ -793,7 +797,6 @@ __cld_pipe_inprogress_downcall(const struct cld_msg_v2 __user *cmsg,
 	uint8_t cmd, princhashlen;
 	struct xdr_netobj name, princhash = { .len = 0, .data = NULL };
 	uint16_t namelen;
-	struct cld_net *cn = nn->cld_net;
 
 	if (get_user(cmd, &cmsg->cm_cmd)) {
 		dprintk("%s: error when copying cmd from userspace", __func__);
@@ -833,11 +836,15 @@ __cld_pipe_inprogress_downcall(const struct cld_msg_v2 __user *cmsg,
 				return PTR_ERR(name.data);
 			name.len = namelen;
 		}
+#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING
 		if (name.len > 5 && memcmp(name.data, "hash:", 5) == 0) {
+			struct cld_net *cn = nn->cld_net;
+
 			name.len = name.len - 5;
 			memmove(name.data, name.data + 5, name.len);
 			cn->cn_has_legacy = true;
 		}
+#endif
 		if (!nfs4_client_to_reclaim(name, princhash, nn)) {
 			kfree(name.data);
 			kfree(princhash.data);
@@ -1010,7 +1017,9 @@ __nfsd4_init_cld_pipe(struct net *net)
 	}
 
 	cn->cn_pipe->dentry = dentry;
+#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING
 	cn->cn_has_legacy = false;
+#endif
 	nn->cld_net = cn;
 	return 0;
 
@@ -1282,10 +1291,6 @@ nfsd4_cld_check(struct nfs4_client *clp)
 {
 	struct nfs4_client_reclaim *crp;
 	struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
-	struct cld_net *cn = nn->cld_net;
-	int status;
-	char dname[HEXDIR_LEN];
-	struct xdr_netobj name;
 
 	/* did we already find that this client is stable? */
 	if (test_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags))
@@ -1296,7 +1301,12 @@ nfsd4_cld_check(struct nfs4_client *clp)
 	if (crp)
 		goto found;
 
-	if (cn->cn_has_legacy) {
+#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING
+	if (nn->cld_net->cn_has_legacy) {
+		int status;
+		char dname[HEXDIR_LEN];
+		struct xdr_netobj name;
+
 		status = nfs4_make_rec_clidname(dname, &clp->cl_name);
 		if (status)
 			return -ENOENT;
@@ -1314,6 +1324,7 @@ nfsd4_cld_check(struct nfs4_client *clp)
 			goto found;
 
 	}
+#endif
 	return -ENOENT;
 found:
 	crp->cr_clp = clp;
@@ -1327,8 +1338,6 @@ nfsd4_cld_check_v2(struct nfs4_client *clp)
 	struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
 	struct cld_net *cn = nn->cld_net;
 	int status;
-	char dname[HEXDIR_LEN];
-	struct xdr_netobj name;
 	struct crypto_shash *tfm = cn->cn_tfm;
 	struct xdr_netobj cksum;
 	char *principal = NULL;
@@ -1342,7 +1351,11 @@ nfsd4_cld_check_v2(struct nfs4_client *clp)
 	if (crp)
 		goto found;
 
+#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING
 	if (cn->cn_has_legacy) {
+		struct xdr_netobj name;
+		char dname[HEXDIR_LEN];
+
 		status = nfs4_make_rec_clidname(dname, &clp->cl_name);
 		if (status)
 			return -ENOENT;
@@ -1360,6 +1373,7 @@ nfsd4_cld_check_v2(struct nfs4_client *clp)
 			goto found;
 
 	}
+#endif
 	return -ENOENT;
 found:
 	if (crp->cr_princhash.len) {
@@ -1663,6 +1677,7 @@ static const struct nfsd4_client_tracking_ops nfsd4_cld_tracking_ops_v2 = {
 	.msglen		= sizeof(struct cld_msg_v2),
 };
 
+#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING
 /* upcall via usermodehelper */
 static char cltrack_prog[PATH_MAX] = "/sbin/nfsdcltrack";
 module_param_string(cltrack_prog, cltrack_prog, sizeof(cltrack_prog),
@@ -2007,28 +2022,10 @@ static const struct nfsd4_client_tracking_ops nfsd4_umh_tracking_ops = {
 	.msglen		= 0,
 };
 
-int
-nfsd4_client_tracking_init(struct net *net)
+static inline int check_for_legacy_methods(int status, struct net *net)
 {
-	int status;
-	struct path path;
 	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
-
-	/* just run the init if it the method is already decided */
-	if (nn->client_tracking_ops)
-		goto do_init;
-
-	/* First, try to use nfsdcld */
-	nn->client_tracking_ops = &nfsd4_cld_tracking_ops;
-	status = nn->client_tracking_ops->init(net);
-	if (!status)
-		return status;
-	if (status != -ETIMEDOUT) {
-		nn->client_tracking_ops = &nfsd4_cld_tracking_ops_v0;
-		status = nn->client_tracking_ops->init(net);
-		if (!status)
-			return status;
-	}
+	struct path path;
 
 	/*
 	 * Next, try the UMH upcall.
@@ -2045,14 +2042,46 @@ nfsd4_client_tracking_init(struct net *net)
 	nn->client_tracking_ops = &nfsd4_legacy_tracking_ops;
 	status = kern_path(nfs4_recoverydir(), LOOKUP_FOLLOW, &path);
 	if (!status) {
-		status = d_is_dir(path.dentry);
+		status = !d_is_dir(path.dentry);
 		path_put(&path);
-		if (!status) {
-			status = -EINVAL;
-			goto out;
-		}
+		if (status)
+			return -ENOTDIR;
+		status = nn->client_tracking_ops->init(net);
+	}
+	return status;
+}
+#else
+static inline int check_for_legacy_methods(int status, struct net *net)
+{
+	return status;
+}
+#endif /* CONFIG_LEGACY_NFSD_CLIENT_TRACKING */
+
+int
+nfsd4_client_tracking_init(struct net *net)
+{
+	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
+	int status;
+
+	/* just run the init if it the method is already decided */
+	if (nn->client_tracking_ops)
+		goto do_init;
+
+	/* First, try to use nfsdcld */
+	nn->client_tracking_ops = &nfsd4_cld_tracking_ops;
+	status = nn->client_tracking_ops->init(net);
+	if (!status)
+		return status;
+	if (status != -ETIMEDOUT) {
+		nn->client_tracking_ops = &nfsd4_cld_tracking_ops_v0;
+		status = nn->client_tracking_ops->init(net);
+		if (!status)
+			return status;
 	}
 
+	status = check_for_legacy_methods(status, net);
+	if (status)
+		goto out;
 do_init:
 	status = nn->client_tracking_ops->init(net);
 out:
diff --git a/fs/nfsd/nfscache.c b/fs/nfsd/nfscache.c
index d3273a396659..5c1a4a0aa605 100644
--- a/fs/nfsd/nfscache.c
+++ b/fs/nfsd/nfscache.c
@@ -364,8 +364,6 @@ nfsd_reply_cache_scan(struct shrinker *shrink, struct shrink_control *sc)
 		if (freed > sc->nr_to_scan)
 			break;
 	}
-
-	trace_nfsd_drc_gc(nn, freed);
 	return freed;
 }
 
@@ -508,7 +506,6 @@ int nfsd_cache_lookup(struct svc_rqst *rqstp, unsigned int start,
 	__wsum			csum;
 	struct nfsd_drc_bucket	*b;
 	int type = rqstp->rq_cachetype;
-	unsigned long freed;
 	LIST_HEAD(dispose);
 	int rtn = RC_DOIT;
 
@@ -538,8 +535,7 @@ int nfsd_cache_lookup(struct svc_rqst *rqstp, unsigned int start,
 	nfsd_prune_bucket_locked(nn, b, 3, &dispose);
 	spin_unlock(&b->cache_lock);
 
-	freed = nfsd_cacherep_dispose(&dispose);
-	trace_nfsd_drc_gc(nn, freed);
+	nfsd_cacherep_dispose(&dispose);
 
 	nfsd_stats_rc_misses_inc();
 	atomic_inc(&nn->num_drc_entries);
diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
index 3e15b72f421d..d6eeee149370 100644
--- a/fs/nfsd/nfsctl.c
+++ b/fs/nfsd/nfsctl.c
@@ -76,7 +76,9 @@ static ssize_t write_maxconn(struct file *file, char *buf, size_t size);
 #ifdef CONFIG_NFSD_V4
 static ssize_t write_leasetime(struct file *file, char *buf, size_t size);
 static ssize_t write_gracetime(struct file *file, char *buf, size_t size);
+#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING
 static ssize_t write_recoverydir(struct file *file, char *buf, size_t size);
+#endif
 static ssize_t write_v4_end_grace(struct file *file, char *buf, size_t size);
 #endif
 
@@ -93,7 +95,9 @@ static ssize_t (*const write_op[])(struct file *, char *, size_t) = {
 #ifdef CONFIG_NFSD_V4
 	[NFSD_Leasetime] = write_leasetime,
 	[NFSD_Gracetime] = write_gracetime,
+#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING
 	[NFSD_RecoveryDir] = write_recoverydir,
+#endif
 	[NFSD_V4EndGrace] = write_v4_end_grace,
 #endif
 };
@@ -1013,6 +1017,7 @@ static ssize_t write_gracetime(struct file *file, char *buf, size_t size)
 	return nfsd4_write_time(file, buf, size, &nn->nfsd4_grace, nn);
 }
 
+#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING
 static ssize_t __write_recoverydir(struct file *file, char *buf, size_t size,
 				   struct nfsd_net *nn)
 {
@@ -1073,6 +1078,7 @@ static ssize_t write_recoverydir(struct file *file, char *buf, size_t size)
 	mutex_unlock(&nfsd_mutex);
 	return rv;
 }
+#endif
 
 /*
  * write_v4_end_grace - release grace period for nfsd's v4.x lock manager
diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c
index fe61d9bbcc1f..66ca50b38b27 100644
--- a/fs/nfsd/nfssvc.c
+++ b/fs/nfsd/nfssvc.c
@@ -359,13 +359,12 @@ static bool nfsd_needs_lockd(struct nfsd_net *nn)
  */
 void nfsd_copy_write_verifier(__be32 verf[2], struct nfsd_net *nn)
 {
-	int seq = 0;
+	unsigned int seq;
 
 	do {
-		read_seqbegin_or_lock(&nn->writeverf_lock, &seq);
+		seq = read_seqbegin(&nn->writeverf_lock);
 		memcpy(verf, nn->writeverf, sizeof(nn->writeverf));
-	} while (need_seqretry(&nn->writeverf_lock, seq));
-	done_seqretry(&nn->writeverf_lock, seq);
+	} while (read_seqretry(&nn->writeverf_lock, seq));
 }
 
 static void nfsd_reset_write_verifier_locked(struct nfsd_net *nn)
diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h
index fbc0ccb40424..d1e8cf079b0f 100644
--- a/fs/nfsd/trace.h
+++ b/fs/nfsd/trace.h
@@ -1262,28 +1262,6 @@ TRACE_EVENT(nfsd_drc_mismatch,
 		__entry->ingress)
 );
 
-TRACE_EVENT_CONDITION(nfsd_drc_gc,
-	TP_PROTO(
-		const struct nfsd_net *nn,
-		unsigned long freed
-	),
-	TP_ARGS(nn, freed),
-	TP_CONDITION(freed > 0),
-	TP_STRUCT__entry(
-		__field(unsigned long long, boot_time)
-		__field(unsigned long, freed)
-		__field(int, total)
-	),
-	TP_fast_assign(
-		__entry->boot_time = nn->boot_time;
-		__entry->freed = freed;
-		__entry->total = atomic_read(&nn->num_drc_entries);
-	),
-	TP_printk("boot_time=%16llx total=%d freed=%lu",
-		__entry->boot_time, __entry->total, __entry->freed
-	)
-);
-
 TRACE_EVENT(nfsd_cb_args,
 	TP_PROTO(
 		const struct nfs4_client *clp,
diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c
index fbbea7498f02..87a5953c6bf0 100644
--- a/fs/nfsd/vfs.c
+++ b/fs/nfsd/vfs.c
@@ -2109,9 +2109,23 @@ static __be32 nfsd_buffered_readdir(struct file *file, struct svc_fh *fhp,
 	return cdp->err;
 }
 
-/*
- * Read entries from a directory.
- * The  NFSv3/4 verifier we ignore for now.
+/**
+ * nfsd_readdir - Read entries from a directory
+ * @rqstp: RPC transaction context
+ * @fhp: NFS file handle of directory to be read
+ * @offsetp: OUT: seek offset of final entry that was read
+ * @cdp: OUT: an eof error value
+ * @func: entry filler actor
+ *
+ * This implementation ignores the NFSv3/4 verifier cookie.
+ *
+ * NB: normal system calls hold file->f_pos_lock when calling
+ * ->iterate_shared and ->llseek, but nfsd_readdir() does not.
+ * Because the struct file acquired here is not visible to other
+ * threads, it's internal state does not need mutex protection.
+ *
+ * Returns nfs_ok on success, otherwise an nfsstat code is
+ * returned.
  */
 __be32
 nfsd_readdir(struct svc_rqst *rqstp, struct svc_fh *fhp, loff_t *offsetp, 
diff --git a/fs/nilfs2/btnode.c b/fs/nilfs2/btnode.c
index 5710833ac1cc..1204dd06ead8 100644
--- a/fs/nilfs2/btnode.c
+++ b/fs/nilfs2/btnode.c
@@ -64,8 +64,8 @@ nilfs_btnode_create_block(struct address_space *btnc, __u64 blocknr)
 	set_buffer_mapped(bh);
 	set_buffer_uptodate(bh);
 
-	unlock_page(bh->b_page);
-	put_page(bh->b_page);
+	folio_unlock(bh->b_folio);
+	folio_put(bh->b_folio);
 	return bh;
 }
 
@@ -75,7 +75,7 @@ int nilfs_btnode_submit_block(struct address_space *btnc, __u64 blocknr,
 {
 	struct buffer_head *bh;
 	struct inode *inode = btnc->host;
-	struct page *page;
+	struct folio *folio;
 	int err;
 
 	bh = nilfs_grab_buffer(inode, btnc, blocknr, BIT(BH_NILFS_Node));
@@ -83,7 +83,7 @@ int nilfs_btnode_submit_block(struct address_space *btnc, __u64 blocknr,
 		return -ENOMEM;
 
 	err = -EEXIST; /* internal code */
-	page = bh->b_page;
+	folio = bh->b_folio;
 
 	if (buffer_uptodate(bh) || buffer_dirty(bh))
 		goto found;
@@ -130,8 +130,8 @@ int nilfs_btnode_submit_block(struct address_space *btnc, __u64 blocknr,
 	*pbh = bh;
 
 out_locked:
-	unlock_page(page);
-	put_page(page);
+	folio_unlock(folio);
+	folio_put(folio);
 	return err;
 }
 
@@ -145,19 +145,19 @@ int nilfs_btnode_submit_block(struct address_space *btnc, __u64 blocknr,
 void nilfs_btnode_delete(struct buffer_head *bh)
 {
 	struct address_space *mapping;
-	struct page *page = bh->b_page;
-	pgoff_t index = page_index(page);
+	struct folio *folio = bh->b_folio;
+	pgoff_t index = folio->index;
 	int still_dirty;
 
-	get_page(page);
-	lock_page(page);
-	wait_on_page_writeback(page);
+	folio_get(folio);
+	folio_lock(folio);
+	folio_wait_writeback(folio);
 
 	nilfs_forget_buffer(bh);
-	still_dirty = PageDirty(page);
-	mapping = page->mapping;
-	unlock_page(page);
-	put_page(page);
+	still_dirty = folio_test_dirty(folio);
+	mapping = folio->mapping;
+	folio_unlock(folio);
+	folio_put(folio);
 
 	if (!still_dirty && mapping)
 		invalidate_inode_pages2_range(mapping, index, index);
@@ -185,23 +185,23 @@ int nilfs_btnode_prepare_change_key(struct address_space *btnc,
 	ctxt->newbh = NULL;
 
 	if (inode->i_blkbits == PAGE_SHIFT) {
-		struct page *opage = obh->b_page;
-		lock_page(opage);
+		struct folio *ofolio = obh->b_folio;
+		folio_lock(ofolio);
 retry:
 		/* BUG_ON(oldkey != obh->b_folio->index); */
-		if (unlikely(oldkey != opage->index))
-			NILFS_PAGE_BUG(opage,
+		if (unlikely(oldkey != ofolio->index))
+			NILFS_PAGE_BUG(&ofolio->page,
 				       "invalid oldkey %lld (newkey=%lld)",
 				       (unsigned long long)oldkey,
 				       (unsigned long long)newkey);
 
 		xa_lock_irq(&btnc->i_pages);
-		err = __xa_insert(&btnc->i_pages, newkey, opage, GFP_NOFS);
+		err = __xa_insert(&btnc->i_pages, newkey, ofolio, GFP_NOFS);
 		xa_unlock_irq(&btnc->i_pages);
 		/*
-		 * Note: page->index will not change to newkey until
+		 * Note: folio->index will not change to newkey until
 		 * nilfs_btnode_commit_change_key() will be called.
-		 * To protect the page in intermediate state, the page lock
+		 * To protect the folio in intermediate state, the folio lock
 		 * is held.
 		 */
 		if (!err)
@@ -213,7 +213,7 @@ int nilfs_btnode_prepare_change_key(struct address_space *btnc,
 		if (!err)
 			goto retry;
 		/* fallback to copy mode */
-		unlock_page(opage);
+		folio_unlock(ofolio);
 	}
 
 	nbh = nilfs_btnode_create_block(btnc, newkey);
@@ -225,7 +225,7 @@ int nilfs_btnode_prepare_change_key(struct address_space *btnc,
 	return 0;
 
  failed_unlock:
-	unlock_page(obh->b_page);
+	folio_unlock(obh->b_folio);
 	return err;
 }
 
@@ -238,15 +238,15 @@ void nilfs_btnode_commit_change_key(struct address_space *btnc,
 {
 	struct buffer_head *obh = ctxt->bh, *nbh = ctxt->newbh;
 	__u64 oldkey = ctxt->oldkey, newkey = ctxt->newkey;
-	struct page *opage;
+	struct folio *ofolio;
 
 	if (oldkey == newkey)
 		return;
 
 	if (nbh == NULL) {	/* blocksize == pagesize */
-		opage = obh->b_page;
-		if (unlikely(oldkey != opage->index))
-			NILFS_PAGE_BUG(opage,
+		ofolio = obh->b_folio;
+		if (unlikely(oldkey != ofolio->index))
+			NILFS_PAGE_BUG(&ofolio->page,
 				       "invalid oldkey %lld (newkey=%lld)",
 				       (unsigned long long)oldkey,
 				       (unsigned long long)newkey);
@@ -257,8 +257,8 @@ void nilfs_btnode_commit_change_key(struct address_space *btnc,
 		__xa_set_mark(&btnc->i_pages, newkey, PAGECACHE_TAG_DIRTY);
 		xa_unlock_irq(&btnc->i_pages);
 
-		opage->index = obh->b_blocknr = newkey;
-		unlock_page(opage);
+		ofolio->index = obh->b_blocknr = newkey;
+		folio_unlock(ofolio);
 	} else {
 		nilfs_copy_buffer(nbh, obh);
 		mark_buffer_dirty(nbh);
@@ -284,7 +284,7 @@ void nilfs_btnode_abort_change_key(struct address_space *btnc,
 
 	if (nbh == NULL) {	/* blocksize == pagesize */
 		xa_erase_irq(&btnc->i_pages, newkey);
-		unlock_page(ctxt->bh->b_page);
+		folio_unlock(ctxt->bh->b_folio);
 	} else {
 		/*
 		 * When canceling a buffer that a prepare operation has
diff --git a/fs/nilfs2/file.c b/fs/nilfs2/file.c
index 740ce26d1e76..bec33b89a075 100644
--- a/fs/nilfs2/file.c
+++ b/fs/nilfs2/file.c
@@ -45,34 +45,36 @@ int nilfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync)
 static vm_fault_t nilfs_page_mkwrite(struct vm_fault *vmf)
 {
 	struct vm_area_struct *vma = vmf->vma;
-	struct page *page = vmf->page;
+	struct folio *folio = page_folio(vmf->page);
 	struct inode *inode = file_inode(vma->vm_file);
 	struct nilfs_transaction_info ti;
+	struct buffer_head *bh, *head;
 	int ret = 0;
 
 	if (unlikely(nilfs_near_disk_full(inode->i_sb->s_fs_info)))
 		return VM_FAULT_SIGBUS; /* -ENOSPC */
 
 	sb_start_pagefault(inode->i_sb);
-	lock_page(page);
-	if (page->mapping != inode->i_mapping ||
-	    page_offset(page) >= i_size_read(inode) || !PageUptodate(page)) {
-		unlock_page(page);
+	folio_lock(folio);
+	if (folio->mapping != inode->i_mapping ||
+	    folio_pos(folio) >= i_size_read(inode) ||
+	    !folio_test_uptodate(folio)) {
+		folio_unlock(folio);
 		ret = -EFAULT;	/* make the VM retry the fault */
 		goto out;
 	}
 
 	/*
-	 * check to see if the page is mapped already (no holes)
+	 * check to see if the folio is mapped already (no holes)
 	 */
-	if (PageMappedToDisk(page))
+	if (folio_test_mappedtodisk(folio))
 		goto mapped;
 
-	if (page_has_buffers(page)) {
-		struct buffer_head *bh, *head;
+	head = folio_buffers(folio);
+	if (head) {
 		int fully_mapped = 1;
 
-		bh = head = page_buffers(page);
+		bh = head;
 		do {
 			if (!buffer_mapped(bh)) {
 				fully_mapped = 0;
@@ -81,11 +83,11 @@ static vm_fault_t nilfs_page_mkwrite(struct vm_fault *vmf)
 		} while (bh = bh->b_this_page, bh != head);
 
 		if (fully_mapped) {
-			SetPageMappedToDisk(page);
+			folio_set_mappedtodisk(folio);
 			goto mapped;
 		}
 	}
-	unlock_page(page);
+	folio_unlock(folio);
 
 	/*
 	 * fill hole blocks
@@ -105,7 +107,7 @@ static vm_fault_t nilfs_page_mkwrite(struct vm_fault *vmf)
 	nilfs_transaction_commit(inode->i_sb);
 
  mapped:
-	wait_for_stable_page(page);
+	folio_wait_stable(folio);
  out:
 	sb_end_pagefault(inode->i_sb);
 	return vmf_fs_error(ret);
diff --git a/fs/nilfs2/gcinode.c b/fs/nilfs2/gcinode.c
index 8beb2730929d..bf9a11d58817 100644
--- a/fs/nilfs2/gcinode.c
+++ b/fs/nilfs2/gcinode.c
@@ -98,8 +98,8 @@ int nilfs_gccache_submit_read_data(struct inode *inode, sector_t blkoff,
 	*out_bh = bh;
 
  failed:
-	unlock_page(bh->b_page);
-	put_page(bh->b_page);
+	folio_unlock(bh->b_folio);
+	folio_put(bh->b_folio);
 	if (unlikely(err))
 		brelse(bh);
 	return err;
diff --git a/fs/nilfs2/inode.c b/fs/nilfs2/inode.c
index f861f3a0bf5c..9c334c722fc1 100644
--- a/fs/nilfs2/inode.c
+++ b/fs/nilfs2/inode.c
@@ -175,7 +175,8 @@ static int nilfs_writepages(struct address_space *mapping,
 
 static int nilfs_writepage(struct page *page, struct writeback_control *wbc)
 {
-	struct inode *inode = page->mapping->host;
+	struct folio *folio = page_folio(page);
+	struct inode *inode = folio->mapping->host;
 	int err;
 
 	if (sb_rdonly(inode->i_sb)) {
@@ -185,13 +186,13 @@ static int nilfs_writepage(struct page *page, struct writeback_control *wbc)
 		 * have dirty pages that try to be flushed in background.
 		 * So, here we simply discard this dirty page.
 		 */
-		nilfs_clear_dirty_page(page, false);
-		unlock_page(page);
+		nilfs_clear_folio_dirty(folio, false);
+		folio_unlock(folio);
 		return -EROFS;
 	}
 
-	redirty_page_for_writepage(wbc, page);
-	unlock_page(page);
+	folio_redirty_for_writepage(wbc, folio);
+	folio_unlock(folio);
 
 	if (wbc->sync_mode == WB_SYNC_ALL) {
 		err = nilfs_construct_segment(inode->i_sb);
@@ -214,7 +215,7 @@ static bool nilfs_dirty_folio(struct address_space *mapping,
 	/*
 	 * The page may not be locked, eg if called from try_to_unmap_one()
 	 */
-	spin_lock(&mapping->private_lock);
+	spin_lock(&mapping->i_private_lock);
 	head = folio_buffers(folio);
 	if (head) {
 		struct buffer_head *bh = head;
@@ -230,7 +231,7 @@ static bool nilfs_dirty_folio(struct address_space *mapping,
 	} else if (ret) {
 		nr_dirty = 1 << (folio_shift(folio) - inode->i_blkbits);
 	}
-	spin_unlock(&mapping->private_lock);
+	spin_unlock(&mapping->i_private_lock);
 
 	if (nr_dirty)
 		nilfs_set_file_dirty(inode, nr_dirty);
diff --git a/fs/nilfs2/ioctl.c b/fs/nilfs2/ioctl.c
index 40ffade49f38..cfb6aca5ec38 100644
--- a/fs/nilfs2/ioctl.c
+++ b/fs/nilfs2/ioctl.c
@@ -872,16 +872,14 @@ static int nilfs_ioctl_clean_segments(struct inode *inode, struct file *filp,
 	nsegs = argv[4].v_nmembs;
 	if (argv[4].v_size != argsz[4])
 		goto out;
-	if (nsegs > UINT_MAX / sizeof(__u64))
-		goto out;
 
 	/*
 	 * argv[4] points to segment numbers this ioctl cleans.  We
-	 * use kmalloc() for its buffer because memory used for the
-	 * segment numbers is enough small.
+	 * use kmalloc() for its buffer because the memory used for the
+	 * segment numbers is small enough.
 	 */
-	kbufs[4] = memdup_user((void __user *)(unsigned long)argv[4].v_base,
-			       nsegs * sizeof(__u64));
+	kbufs[4] = memdup_array_user((void __user *)(unsigned long)argv[4].v_base,
+				     nsegs, sizeof(__u64));
 	if (IS_ERR(kbufs[4])) {
 		ret = PTR_ERR(kbufs[4]);
 		goto out;
diff --git a/fs/nilfs2/mdt.c b/fs/nilfs2/mdt.c
index c97c77a39668..e45c01a559c0 100644
--- a/fs/nilfs2/mdt.c
+++ b/fs/nilfs2/mdt.c
@@ -97,8 +97,8 @@ static int nilfs_mdt_create_block(struct inode *inode, unsigned long block,
 	}
 
  failed_bh:
-	unlock_page(bh->b_page);
-	put_page(bh->b_page);
+	folio_unlock(bh->b_folio);
+	folio_put(bh->b_folio);
 	brelse(bh);
 
  failed_unlock:
@@ -158,8 +158,8 @@ nilfs_mdt_submit_block(struct inode *inode, unsigned long blkoff, blk_opf_t opf,
 	*out_bh = bh;
 
  failed_bh:
-	unlock_page(bh->b_page);
-	put_page(bh->b_page);
+	folio_unlock(bh->b_folio);
+	folio_put(bh->b_folio);
 	brelse(bh);
  failed:
 	return ret;
@@ -399,7 +399,8 @@ int nilfs_mdt_fetch_dirty(struct inode *inode)
 static int
 nilfs_mdt_write_page(struct page *page, struct writeback_control *wbc)
 {
-	struct inode *inode = page->mapping->host;
+	struct folio *folio = page_folio(page);
+	struct inode *inode = folio->mapping->host;
 	struct super_block *sb;
 	int err = 0;
 
@@ -407,16 +408,16 @@ nilfs_mdt_write_page(struct page *page, struct writeback_control *wbc)
 		/*
 		 * It means that filesystem was remounted in read-only
 		 * mode because of error or metadata corruption. But we
-		 * have dirty pages that try to be flushed in background.
-		 * So, here we simply discard this dirty page.
+		 * have dirty folios that try to be flushed in background.
+		 * So, here we simply discard this dirty folio.
 		 */
-		nilfs_clear_dirty_page(page, false);
-		unlock_page(page);
+		nilfs_clear_folio_dirty(folio, false);
+		folio_unlock(folio);
 		return -EROFS;
 	}
 
-	redirty_page_for_writepage(wbc, page);
-	unlock_page(page);
+	folio_redirty_for_writepage(wbc, folio);
+	folio_unlock(folio);
 
 	if (!inode)
 		return 0;
diff --git a/fs/nilfs2/page.c b/fs/nilfs2/page.c
index 06b04758f289..94e11bcee05b 100644
--- a/fs/nilfs2/page.c
+++ b/fs/nilfs2/page.c
@@ -73,7 +73,7 @@ struct buffer_head *nilfs_grab_buffer(struct inode *inode,
  */
 void nilfs_forget_buffer(struct buffer_head *bh)
 {
-	struct page *page = bh->b_page;
+	struct folio *folio = bh->b_folio;
 	const unsigned long clear_bits =
 		(BIT(BH_Uptodate) | BIT(BH_Dirty) | BIT(BH_Mapped) |
 		 BIT(BH_Async_Write) | BIT(BH_NILFS_Volatile) |
@@ -81,12 +81,12 @@ void nilfs_forget_buffer(struct buffer_head *bh)
 
 	lock_buffer(bh);
 	set_mask_bits(&bh->b_state, clear_bits, 0);
-	if (nilfs_page_buffers_clean(page))
-		__nilfs_clear_page_dirty(page);
+	if (nilfs_folio_buffers_clean(folio))
+		__nilfs_clear_folio_dirty(folio);
 
 	bh->b_blocknr = -1;
-	ClearPageUptodate(page);
-	ClearPageMappedToDisk(page);
+	folio_clear_uptodate(folio);
+	folio_clear_mappedtodisk(folio);
 	unlock_buffer(bh);
 	brelse(bh);
 }
@@ -131,23 +131,23 @@ void nilfs_copy_buffer(struct buffer_head *dbh, struct buffer_head *sbh)
 }
 
 /**
- * nilfs_page_buffers_clean - check if a page has dirty buffers or not.
- * @page: page to be checked
+ * nilfs_folio_buffers_clean - Check if a folio has dirty buffers or not.
+ * @folio: Folio to be checked.
  *
- * nilfs_page_buffers_clean() returns zero if the page has dirty buffers.
- * Otherwise, it returns non-zero value.
+ * nilfs_folio_buffers_clean() returns false if the folio has dirty buffers.
+ * Otherwise, it returns true.
  */
-int nilfs_page_buffers_clean(struct page *page)
+bool nilfs_folio_buffers_clean(struct folio *folio)
 {
 	struct buffer_head *bh, *head;
 
-	bh = head = page_buffers(page);
+	bh = head = folio_buffers(folio);
 	do {
 		if (buffer_dirty(bh))
-			return 0;
+			return false;
 		bh = bh->b_this_page;
 	} while (bh != head);
-	return 1;
+	return true;
 }
 
 void nilfs_page_bug(struct page *page)
@@ -379,7 +379,7 @@ void nilfs_clear_dirty_pages(struct address_space *mapping, bool silent)
 			 * was acquired.  Skip processing in that case.
 			 */
 			if (likely(folio->mapping == mapping))
-				nilfs_clear_dirty_page(&folio->page, silent);
+				nilfs_clear_folio_dirty(folio, silent);
 
 			folio_unlock(folio);
 		}
@@ -389,32 +389,33 @@ void nilfs_clear_dirty_pages(struct address_space *mapping, bool silent)
 }
 
 /**
- * nilfs_clear_dirty_page - discard dirty page
- * @page: dirty page that will be discarded
+ * nilfs_clear_folio_dirty - discard dirty folio
+ * @folio: dirty folio that will be discarded
  * @silent: suppress [true] or print [false] warning messages
  */
-void nilfs_clear_dirty_page(struct page *page, bool silent)
+void nilfs_clear_folio_dirty(struct folio *folio, bool silent)
 {
-	struct inode *inode = page->mapping->host;
+	struct inode *inode = folio->mapping->host;
 	struct super_block *sb = inode->i_sb;
+	struct buffer_head *bh, *head;
 
-	BUG_ON(!PageLocked(page));
+	BUG_ON(!folio_test_locked(folio));
 
 	if (!silent)
 		nilfs_warn(sb, "discard dirty page: offset=%lld, ino=%lu",
-			   page_offset(page), inode->i_ino);
+			   folio_pos(folio), inode->i_ino);
 
-	ClearPageUptodate(page);
-	ClearPageMappedToDisk(page);
+	folio_clear_uptodate(folio);
+	folio_clear_mappedtodisk(folio);
 
-	if (page_has_buffers(page)) {
-		struct buffer_head *bh, *head;
+	head = folio_buffers(folio);
+	if (head) {
 		const unsigned long clear_bits =
 			(BIT(BH_Uptodate) | BIT(BH_Dirty) | BIT(BH_Mapped) |
 			 BIT(BH_Async_Write) | BIT(BH_NILFS_Volatile) |
 			 BIT(BH_NILFS_Checked) | BIT(BH_NILFS_Redirected));
 
-		bh = head = page_buffers(page);
+		bh = head;
 		do {
 			lock_buffer(bh);
 			if (!silent)
@@ -427,7 +428,7 @@ void nilfs_clear_dirty_page(struct page *page, bool silent)
 		} while (bh = bh->b_this_page, bh != head);
 	}
 
-	__nilfs_clear_page_dirty(page);
+	__nilfs_clear_folio_dirty(folio);
 }
 
 unsigned int nilfs_page_count_clean_buffers(struct page *page,
@@ -457,22 +458,23 @@ unsigned int nilfs_page_count_clean_buffers(struct page *page,
  * 2) Some B-tree operations like insertion or deletion may dispose buffers
  *    in dirty state, and this needs to cancel the dirty state of their pages.
  */
-int __nilfs_clear_page_dirty(struct page *page)
+void __nilfs_clear_folio_dirty(struct folio *folio)
 {
-	struct address_space *mapping = page->mapping;
+	struct address_space *mapping = folio->mapping;
 
 	if (mapping) {
 		xa_lock_irq(&mapping->i_pages);
-		if (test_bit(PG_dirty, &page->flags)) {
-			__xa_clear_mark(&mapping->i_pages, page_index(page),
+		if (folio_test_dirty(folio)) {
+			__xa_clear_mark(&mapping->i_pages, folio->index,
 					     PAGECACHE_TAG_DIRTY);
 			xa_unlock_irq(&mapping->i_pages);
-			return clear_page_dirty_for_io(page);
+			folio_clear_dirty_for_io(folio);
+			return;
 		}
 		xa_unlock_irq(&mapping->i_pages);
-		return 0;
+		return;
 	}
-	return TestClearPageDirty(page);
+	folio_clear_dirty(folio);
 }
 
 /**
diff --git a/fs/nilfs2/page.h b/fs/nilfs2/page.h
index d249ea1cefff..968b311d265b 100644
--- a/fs/nilfs2/page.h
+++ b/fs/nilfs2/page.h
@@ -30,18 +30,18 @@ BUFFER_FNS(NILFS_Checked, nilfs_checked)	/* buffer is verified */
 BUFFER_FNS(NILFS_Redirected, nilfs_redirected)	/* redirected to a copy */
 
 
-int __nilfs_clear_page_dirty(struct page *);
+void __nilfs_clear_folio_dirty(struct folio *);
 
 struct buffer_head *nilfs_grab_buffer(struct inode *, struct address_space *,
 				      unsigned long, unsigned long);
 void nilfs_forget_buffer(struct buffer_head *);
 void nilfs_copy_buffer(struct buffer_head *, struct buffer_head *);
-int nilfs_page_buffers_clean(struct page *);
+bool nilfs_folio_buffers_clean(struct folio *);
 void nilfs_page_bug(struct page *);
 
 int nilfs_copy_dirty_pages(struct address_space *, struct address_space *);
 void nilfs_copy_back_pages(struct address_space *, struct address_space *);
-void nilfs_clear_dirty_page(struct page *, bool);
+void nilfs_clear_folio_dirty(struct folio *, bool);
 void nilfs_clear_dirty_pages(struct address_space *, bool);
 unsigned int nilfs_page_count_clean_buffers(struct page *, unsigned int,
 					    unsigned int);
diff --git a/fs/nilfs2/segment.c b/fs/nilfs2/segment.c
index 55e31cc903d1..52995838f2de 100644
--- a/fs/nilfs2/segment.c
+++ b/fs/nilfs2/segment.c
@@ -1665,39 +1665,39 @@ static int nilfs_segctor_assign(struct nilfs_sc_info *sci, int mode)
 	return 0;
 }
 
-static void nilfs_begin_page_io(struct page *page)
+static void nilfs_begin_folio_io(struct folio *folio)
 {
-	if (!page || PageWriteback(page))
+	if (!folio || folio_test_writeback(folio))
 		/*
 		 * For split b-tree node pages, this function may be called
 		 * twice.  We ignore the 2nd or later calls by this check.
 		 */
 		return;
 
-	lock_page(page);
-	clear_page_dirty_for_io(page);
-	set_page_writeback(page);
-	unlock_page(page);
+	folio_lock(folio);
+	folio_clear_dirty_for_io(folio);
+	folio_start_writeback(folio);
+	folio_unlock(folio);
 }
 
 static void nilfs_segctor_prepare_write(struct nilfs_sc_info *sci)
 {
 	struct nilfs_segment_buffer *segbuf;
-	struct page *bd_page = NULL, *fs_page = NULL;
+	struct folio *bd_folio = NULL, *fs_folio = NULL;
 
 	list_for_each_entry(segbuf, &sci->sc_segbufs, sb_list) {
 		struct buffer_head *bh;
 
 		list_for_each_entry(bh, &segbuf->sb_segsum_buffers,
 				    b_assoc_buffers) {
-			if (bh->b_page != bd_page) {
-				if (bd_page) {
-					lock_page(bd_page);
-					clear_page_dirty_for_io(bd_page);
-					set_page_writeback(bd_page);
-					unlock_page(bd_page);
+			if (bh->b_folio != bd_folio) {
+				if (bd_folio) {
+					folio_lock(bd_folio);
+					folio_clear_dirty_for_io(bd_folio);
+					folio_start_writeback(bd_folio);
+					folio_unlock(bd_folio);
 				}
-				bd_page = bh->b_page;
+				bd_folio = bh->b_folio;
 			}
 		}
 
@@ -1705,28 +1705,28 @@ static void nilfs_segctor_prepare_write(struct nilfs_sc_info *sci)
 				    b_assoc_buffers) {
 			set_buffer_async_write(bh);
 			if (bh == segbuf->sb_super_root) {
-				if (bh->b_page != bd_page) {
-					lock_page(bd_page);
-					clear_page_dirty_for_io(bd_page);
-					set_page_writeback(bd_page);
-					unlock_page(bd_page);
-					bd_page = bh->b_page;
+				if (bh->b_folio != bd_folio) {
+					folio_lock(bd_folio);
+					folio_clear_dirty_for_io(bd_folio);
+					folio_start_writeback(bd_folio);
+					folio_unlock(bd_folio);
+					bd_folio = bh->b_folio;
 				}
 				break;
 			}
-			if (bh->b_page != fs_page) {
-				nilfs_begin_page_io(fs_page);
-				fs_page = bh->b_page;
+			if (bh->b_folio != fs_folio) {
+				nilfs_begin_folio_io(fs_folio);
+				fs_folio = bh->b_folio;
 			}
 		}
 	}
-	if (bd_page) {
-		lock_page(bd_page);
-		clear_page_dirty_for_io(bd_page);
-		set_page_writeback(bd_page);
-		unlock_page(bd_page);
+	if (bd_folio) {
+		folio_lock(bd_folio);
+		folio_clear_dirty_for_io(bd_folio);
+		folio_start_writeback(bd_folio);
+		folio_unlock(bd_folio);
 	}
-	nilfs_begin_page_io(fs_page);
+	nilfs_begin_folio_io(fs_folio);
 }
 
 static int nilfs_segctor_write(struct nilfs_sc_info *sci,
@@ -1739,17 +1739,18 @@ static int nilfs_segctor_write(struct nilfs_sc_info *sci,
 	return ret;
 }
 
-static void nilfs_end_page_io(struct page *page, int err)
+static void nilfs_end_folio_io(struct folio *folio, int err)
 {
-	if (!page)
+	if (!folio)
 		return;
 
-	if (buffer_nilfs_node(page_buffers(page)) && !PageWriteback(page)) {
+	if (buffer_nilfs_node(folio_buffers(folio)) &&
+			!folio_test_writeback(folio)) {
 		/*
 		 * For b-tree node pages, this function may be called twice
 		 * or more because they might be split in a segment.
 		 */
-		if (PageDirty(page)) {
+		if (folio_test_dirty(folio)) {
 			/*
 			 * For pages holding split b-tree node buffers, dirty
 			 * flag on the buffers may be cleared discretely.
@@ -1757,30 +1758,30 @@ static void nilfs_end_page_io(struct page *page, int err)
 			 * remaining buffers, and it must be cancelled if
 			 * all the buffers get cleaned later.
 			 */
-			lock_page(page);
-			if (nilfs_page_buffers_clean(page))
-				__nilfs_clear_page_dirty(page);
-			unlock_page(page);
+			folio_lock(folio);
+			if (nilfs_folio_buffers_clean(folio))
+				__nilfs_clear_folio_dirty(folio);
+			folio_unlock(folio);
 		}
 		return;
 	}
 
 	if (!err) {
-		if (!nilfs_page_buffers_clean(page))
-			__set_page_dirty_nobuffers(page);
-		ClearPageError(page);
+		if (!nilfs_folio_buffers_clean(folio))
+			filemap_dirty_folio(folio->mapping, folio);
+		folio_clear_error(folio);
 	} else {
-		__set_page_dirty_nobuffers(page);
-		SetPageError(page);
+		filemap_dirty_folio(folio->mapping, folio);
+		folio_set_error(folio);
 	}
 
-	end_page_writeback(page);
+	folio_end_writeback(folio);
 }
 
 static void nilfs_abort_logs(struct list_head *logs, int err)
 {
 	struct nilfs_segment_buffer *segbuf;
-	struct page *bd_page = NULL, *fs_page = NULL;
+	struct folio *bd_folio = NULL, *fs_folio = NULL;
 	struct buffer_head *bh;
 
 	if (list_empty(logs))
@@ -1790,10 +1791,10 @@ static void nilfs_abort_logs(struct list_head *logs, int err)
 		list_for_each_entry(bh, &segbuf->sb_segsum_buffers,
 				    b_assoc_buffers) {
 			clear_buffer_uptodate(bh);
-			if (bh->b_page != bd_page) {
-				if (bd_page)
-					end_page_writeback(bd_page);
-				bd_page = bh->b_page;
+			if (bh->b_folio != bd_folio) {
+				if (bd_folio)
+					folio_end_writeback(bd_folio);
+				bd_folio = bh->b_folio;
 			}
 		}
 
@@ -1802,22 +1803,22 @@ static void nilfs_abort_logs(struct list_head *logs, int err)
 			clear_buffer_async_write(bh);
 			if (bh == segbuf->sb_super_root) {
 				clear_buffer_uptodate(bh);
-				if (bh->b_page != bd_page) {
-					end_page_writeback(bd_page);
-					bd_page = bh->b_page;
+				if (bh->b_folio != bd_folio) {
+					folio_end_writeback(bd_folio);
+					bd_folio = bh->b_folio;
 				}
 				break;
 			}
-			if (bh->b_page != fs_page) {
-				nilfs_end_page_io(fs_page, err);
-				fs_page = bh->b_page;
+			if (bh->b_folio != fs_folio) {
+				nilfs_end_folio_io(fs_folio, err);
+				fs_folio = bh->b_folio;
 			}
 		}
 	}
-	if (bd_page)
-		end_page_writeback(bd_page);
+	if (bd_folio)
+		folio_end_writeback(bd_folio);
 
-	nilfs_end_page_io(fs_page, err);
+	nilfs_end_folio_io(fs_folio, err);
 }
 
 static void nilfs_segctor_abort_construction(struct nilfs_sc_info *sci,
@@ -1859,7 +1860,7 @@ static void nilfs_set_next_segment(struct the_nilfs *nilfs,
 static void nilfs_segctor_complete_write(struct nilfs_sc_info *sci)
 {
 	struct nilfs_segment_buffer *segbuf;
-	struct page *bd_page = NULL, *fs_page = NULL;
+	struct folio *bd_folio = NULL, *fs_folio = NULL;
 	struct the_nilfs *nilfs = sci->sc_super->s_fs_info;
 	int update_sr = false;
 
@@ -1870,21 +1871,21 @@ static void nilfs_segctor_complete_write(struct nilfs_sc_info *sci)
 				    b_assoc_buffers) {
 			set_buffer_uptodate(bh);
 			clear_buffer_dirty(bh);
-			if (bh->b_page != bd_page) {
-				if (bd_page)
-					end_page_writeback(bd_page);
-				bd_page = bh->b_page;
+			if (bh->b_folio != bd_folio) {
+				if (bd_folio)
+					folio_end_writeback(bd_folio);
+				bd_folio = bh->b_folio;
 			}
 		}
 		/*
-		 * We assume that the buffers which belong to the same page
+		 * We assume that the buffers which belong to the same folio
 		 * continue over the buffer list.
-		 * Under this assumption, the last BHs of pages is
-		 * identifiable by the discontinuity of bh->b_page
-		 * (page != fs_page).
+		 * Under this assumption, the last BHs of folios is
+		 * identifiable by the discontinuity of bh->b_folio
+		 * (folio != fs_folio).
 		 *
 		 * For B-tree node blocks, however, this assumption is not
-		 * guaranteed.  The cleanup code of B-tree node pages needs
+		 * guaranteed.  The cleanup code of B-tree node folios needs
 		 * special care.
 		 */
 		list_for_each_entry(bh, &segbuf->sb_payload_buffers,
@@ -1897,16 +1898,16 @@ static void nilfs_segctor_complete_write(struct nilfs_sc_info *sci)
 
 			set_mask_bits(&bh->b_state, clear_bits, set_bits);
 			if (bh == segbuf->sb_super_root) {
-				if (bh->b_page != bd_page) {
-					end_page_writeback(bd_page);
-					bd_page = bh->b_page;
+				if (bh->b_folio != bd_folio) {
+					folio_end_writeback(bd_folio);
+					bd_folio = bh->b_folio;
 				}
 				update_sr = true;
 				break;
 			}
-			if (bh->b_page != fs_page) {
-				nilfs_end_page_io(fs_page, 0);
-				fs_page = bh->b_page;
+			if (bh->b_folio != fs_folio) {
+				nilfs_end_folio_io(fs_folio, 0);
+				fs_folio = bh->b_folio;
 			}
 		}
 
@@ -1920,13 +1921,13 @@ static void nilfs_segctor_complete_write(struct nilfs_sc_info *sci)
 		}
 	}
 	/*
-	 * Since pages may continue over multiple segment buffers,
-	 * end of the last page must be checked outside of the loop.
+	 * Since folios may continue over multiple segment buffers,
+	 * end of the last folio must be checked outside of the loop.
 	 */
-	if (bd_page)
-		end_page_writeback(bd_page);
+	if (bd_folio)
+		folio_end_writeback(bd_folio);
 
-	nilfs_end_page_io(fs_page, 0);
+	nilfs_end_folio_io(fs_folio, 0);
 
 	nilfs_drop_collected_inodes(&sci->sc_dirty_files);
 
diff --git a/fs/nilfs2/super.c b/fs/nilfs2/super.c
index a5d1fa4e7552..df8674173b22 100644
--- a/fs/nilfs2/super.c
+++ b/fs/nilfs2/super.c
@@ -1314,15 +1314,7 @@ nilfs_mount(struct file_system_type *fs_type, int flags,
 		return ERR_CAST(s);
 
 	if (!s->s_root) {
-		/*
-		 * We drop s_umount here because we need to open the bdev and
-		 * bdev->open_mutex ranks above s_umount (blkdev_put() ->
-		 * __invalidate_device()). It is safe because we have active sb
-		 * reference and SB_BORN is not set yet.
-		 */
-		up_write(&s->s_umount);
 		err = setup_bdev_super(s, flags, NULL);
-		down_write(&s->s_umount);
 		if (!err)
 			err = nilfs_fill_super(s, data,
 					       flags & SB_SILENT ? 1 : 0);
diff --git a/fs/ntfs/aops.c b/fs/ntfs/aops.c
index 71e31e789b29..1c747a3baa3e 100644
--- a/fs/ntfs/aops.c
+++ b/fs/ntfs/aops.c
@@ -1644,7 +1644,7 @@ const struct address_space_operations ntfs_normal_aops = {
 	.bmap		= ntfs_bmap,
 	.migrate_folio	= buffer_migrate_folio,
 	.is_partially_uptodate = block_is_partially_uptodate,
-	.error_remove_page = generic_error_remove_page,
+	.error_remove_folio = generic_error_remove_folio,
 };
 
 /*
@@ -1658,7 +1658,7 @@ const struct address_space_operations ntfs_compressed_aops = {
 #endif /* NTFS_RW */
 	.migrate_folio	= buffer_migrate_folio,
 	.is_partially_uptodate = block_is_partially_uptodate,
-	.error_remove_page = generic_error_remove_page,
+	.error_remove_folio = generic_error_remove_folio,
 };
 
 /*
@@ -1673,7 +1673,7 @@ const struct address_space_operations ntfs_mst_aops = {
 #endif /* NTFS_RW */
 	.migrate_folio	= buffer_migrate_folio,
 	.is_partially_uptodate	= block_is_partially_uptodate,
-	.error_remove_page = generic_error_remove_page,
+	.error_remove_folio = generic_error_remove_folio,
 };
 
 #ifdef NTFS_RW
@@ -1690,7 +1690,7 @@ const struct address_space_operations ntfs_mst_aops = {
  *
  * If the page does not have buffers, we create them and set them uptodate.
  * The page may not be locked which is why we need to handle the buffers under
- * the mapping->private_lock.  Once the buffers are marked dirty we no longer
+ * the mapping->i_private_lock.  Once the buffers are marked dirty we no longer
  * need the lock since try_to_free_buffers() does not free dirty buffers.
  */
 void mark_ntfs_record_dirty(struct page *page, const unsigned int ofs) {
@@ -1702,11 +1702,11 @@ void mark_ntfs_record_dirty(struct page *page, const unsigned int ofs) {
 	BUG_ON(!PageUptodate(page));
 	end = ofs + ni->itype.index.block_size;
 	bh_size = VFS_I(ni)->i_sb->s_blocksize;
-	spin_lock(&mapping->private_lock);
+	spin_lock(&mapping->i_private_lock);
 	if (unlikely(!page_has_buffers(page))) {
-		spin_unlock(&mapping->private_lock);
+		spin_unlock(&mapping->i_private_lock);
 		bh = head = alloc_page_buffers(page, bh_size, true);
-		spin_lock(&mapping->private_lock);
+		spin_lock(&mapping->i_private_lock);
 		if (likely(!page_has_buffers(page))) {
 			struct buffer_head *tail;
 
@@ -1730,7 +1730,7 @@ void mark_ntfs_record_dirty(struct page *page, const unsigned int ofs) {
 			break;
 		set_buffer_dirty(bh);
 	} while ((bh = bh->b_this_page) != head);
-	spin_unlock(&mapping->private_lock);
+	spin_unlock(&mapping->i_private_lock);
 	filemap_dirty_folio(mapping, page_folio(page));
 	if (unlikely(buffers_to_free)) {
 		do {
diff --git a/fs/ocfs2/aops.c b/fs/ocfs2/aops.c
index ba790219d528..795997806326 100644
--- a/fs/ocfs2/aops.c
+++ b/fs/ocfs2/aops.c
@@ -2480,5 +2480,5 @@ const struct address_space_operations ocfs2_aops = {
 	.release_folio		= ocfs2_release_folio,
 	.migrate_folio		= buffer_migrate_folio,
 	.is_partially_uptodate	= block_is_partially_uptodate,
-	.error_remove_page	= generic_error_remove_page,
+	.error_remove_folio	= generic_error_remove_folio,
 };
diff --git a/fs/ocfs2/dcache.c b/fs/ocfs2/dcache.c
index 04fc8344063a..277757f4fd2d 100644
--- a/fs/ocfs2/dcache.c
+++ b/fs/ocfs2/dcache.c
@@ -32,7 +32,9 @@ void ocfs2_dentry_attach_gen(struct dentry *dentry)
 }
 
 
-static int ocfs2_dentry_revalidate(struct dentry *dentry, unsigned int flags)
+static int ocfs2_dentry_revalidate(struct dentry *dentry,
+				   const struct qstr *name,
+				   unsigned int flags)
 {
 	struct inode *inode;
 	int ret = 0;    /* if all else fails, just return false */
diff --git a/fs/ocfs2/export.c b/fs/ocfs2/export.c
index eaa8c80ace3c..b8b6a191b5cb 100644
--- a/fs/ocfs2/export.c
+++ b/fs/ocfs2/export.c
@@ -280,4 +280,5 @@ const struct export_operations ocfs2_export_ops = {
 	.fh_to_dentry	= ocfs2_fh_to_dentry,
 	.fh_to_parent	= ocfs2_fh_to_parent,
 	.get_parent	= ocfs2_get_parent,
+	.flags		= EXPORT_OP_ASYNC_LOCK,
 };
diff --git a/fs/open.c b/fs/open.c
index 02dc608d40d8..0bd7fce21cbf 100644
--- a/fs/open.c
+++ b/fs/open.c
@@ -442,7 +442,8 @@ static const struct cred *access_override_creds(void)
 	 * 'get_current_cred()' function), that will clear the
 	 * non_rcu field, because now that other user may be
 	 * expecting RCU freeing. But normal thread-synchronous
-	 * cred accesses will keep things non-RCY.
+	 * cred accesses will keep things non-racy to avoid RCU
+	 * freeing.
 	 */
 	override_cred->non_rcu = 1;
 
diff --git a/fs/orangefs/dcache.c b/fs/orangefs/dcache.c
index 8bbe9486e3a6..435b88007809 100644
--- a/fs/orangefs/dcache.c
+++ b/fs/orangefs/dcache.c
@@ -94,7 +94,8 @@ static int orangefs_revalidate_lookup(struct dentry *dentry)
  *
  * Should return 1 if dentry can still be trusted, else 0.
  */
-static int orangefs_d_revalidate(struct dentry *dentry, unsigned int flags)
+static int orangefs_d_revalidate(struct dentry *dentry,
+				 const struct qstr *name, unsigned int flags)
 {
 	int ret;
 	unsigned long time = (unsigned long) dentry->d_fsdata;
diff --git a/fs/overlayfs/export.c b/fs/overlayfs/export.c
index 7e16bbcad95e..9e316d5f936e 100644
--- a/fs/overlayfs/export.c
+++ b/fs/overlayfs/export.c
@@ -289,7 +289,6 @@ static struct dentry *ovl_obtain_alias(struct super_block *sb,
 {
 	struct dentry *lower = lowerpath ? lowerpath->dentry : NULL;
 	struct dentry *upper = upper_alias ?: index;
-	struct dentry *dentry;
 	struct inode *inode = NULL;
 	struct ovl_entry *oe;
 	struct ovl_inode_params oip = {
@@ -320,27 +319,7 @@ static struct dentry *ovl_obtain_alias(struct super_block *sb,
 	if (upper)
 		ovl_set_flag(OVL_UPPERDATA, inode);
 
-	dentry = d_find_any_alias(inode);
-	if (dentry)
-		goto out_iput;
-
-	dentry = d_alloc_anon(inode->i_sb);
-	if (unlikely(!dentry))
-		goto nomem;
-
-	if (upper_alias)
-		ovl_dentry_set_upper_alias(dentry);
-
-	ovl_dentry_init_reval(dentry, upper, OVL_I_E(inode));
-
-	return d_instantiate_anon(dentry, inode);
-
-nomem:
-	dput(dentry);
-	dentry = ERR_PTR(-ENOMEM);
-out_iput:
-	iput(inode);
-	return dentry;
+	return d_obtain_alias(inode);
 }
 
 /* Get the upper or lower dentry in stack whose on layer @idx */
diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c
index 345b8f161ca4..c63b31a460be 100644
--- a/fs/overlayfs/inode.c
+++ b/fs/overlayfs/inode.c
@@ -171,7 +171,7 @@ int ovl_getattr(struct mnt_idmap *idmap, const struct path *path,
 
 	type = ovl_path_real(dentry, &realpath);
 	old_cred = ovl_override_creds(dentry->d_sb);
-	err = vfs_getattr(&realpath, stat, request_mask, flags);
+	err = ovl_do_getattr(&realpath, stat, request_mask, flags);
 	if (err)
 		goto out;
 
@@ -196,8 +196,8 @@ int ovl_getattr(struct mnt_idmap *idmap, const struct path *path,
 					(!is_dir ? STATX_NLINK : 0);
 
 			ovl_path_lower(dentry, &realpath);
-			err = vfs_getattr(&realpath, &lowerstat,
-					  lowermask, flags);
+			err = ovl_do_getattr(&realpath, &lowerstat, lowermask,
+					     flags);
 			if (err)
 				goto out;
 
@@ -249,8 +249,8 @@ int ovl_getattr(struct mnt_idmap *idmap, const struct path *path,
 
 			ovl_path_lowerdata(dentry, &realpath);
 			if (realpath.dentry) {
-				err = vfs_getattr(&realpath, &lowerdatastat,
-						  lowermask, flags);
+				err = ovl_do_getattr(&realpath, &lowerdatastat,
+						     lowermask, flags);
 				if (err)
 					goto out;
 			} else {
diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h
index ca88b2636a57..05c3dd597fa8 100644
--- a/fs/overlayfs/overlayfs.h
+++ b/fs/overlayfs/overlayfs.h
@@ -408,6 +408,14 @@ static inline bool ovl_open_flags_need_copy_up(int flags)
 	return ((OPEN_FMODE(flags) & FMODE_WRITE) || (flags & O_TRUNC));
 }
 
+static inline int ovl_do_getattr(const struct path *path, struct kstat *stat,
+				 u32 request_mask, unsigned int flags)
+{
+	if (flags & AT_GETATTR_NOSEC)
+		return vfs_getattr_nosec(path, stat, request_mask, flags);
+	return vfs_getattr(path, stat, request_mask, flags);
+}
+
 /* util.c */
 int ovl_get_write_access(struct dentry *dentry);
 void ovl_put_write_access(struct dentry *dentry);
diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c
index a0967bb25003..ba7811a394ab 100644
--- a/fs/overlayfs/super.c
+++ b/fs/overlayfs/super.c
@@ -88,7 +88,8 @@ static struct dentry *ovl_d_real(struct dentry *dentry,
 	return dentry;
 }
 
-static int ovl_revalidate_real(struct dentry *d, unsigned int flags, bool weak)
+static int ovl_revalidate_real(struct dentry *d, const struct qstr *name,
+			       unsigned int flags, bool weak)
 {
 	int ret = 1;
 
@@ -99,7 +100,7 @@ static int ovl_revalidate_real(struct dentry *d, unsigned int flags, bool weak)
 		if (d->d_flags & DCACHE_OP_WEAK_REVALIDATE)
 			ret =  d->d_op->d_weak_revalidate(d, flags);
 	} else if (d->d_flags & DCACHE_OP_REVALIDATE) {
-		ret = d->d_op->d_revalidate(d, flags);
+		ret = d->d_op->d_revalidate(d, name, flags);
 		if (!ret) {
 			if (!(flags & LOOKUP_RCU))
 				d_invalidate(d);
@@ -110,6 +111,7 @@ static int ovl_revalidate_real(struct dentry *d, unsigned int flags, bool weak)
 }
 
 static int ovl_dentry_revalidate_common(struct dentry *dentry,
+					const struct qstr *name,
 					unsigned int flags, bool weak)
 {
 	struct ovl_entry *oe;
@@ -127,22 +129,24 @@ static int ovl_dentry_revalidate_common(struct dentry *dentry,
 	lowerstack = ovl_lowerstack(oe);
 	upper = ovl_i_dentry_upper(inode);
 	if (upper)
-		ret = ovl_revalidate_real(upper, flags, weak);
+		ret = ovl_revalidate_real(upper, name, flags, weak);
 
 	for (i = 0; ret > 0 && i < ovl_numlower(oe); i++)
-		ret = ovl_revalidate_real(lowerstack[i].dentry, flags, weak);
+		ret = ovl_revalidate_real(lowerstack[i].dentry, name, flags, weak);
 
 	return ret;
 }
 
-static int ovl_dentry_revalidate(struct dentry *dentry, unsigned int flags)
+static int ovl_dentry_revalidate(struct dentry *dentry,
+				 const struct qstr *name, unsigned int flags)
 {
-	return ovl_dentry_revalidate_common(dentry, flags, false);
+	return ovl_dentry_revalidate_common(dentry, name, flags, false);
 }
 
-static int ovl_dentry_weak_revalidate(struct dentry *dentry, unsigned int flags)
+static int ovl_dentry_weak_revalidate(struct dentry *dentry,
+				      unsigned int flags)
 {
-	return ovl_dentry_revalidate_common(dentry, flags, true);
+	return ovl_dentry_revalidate_common(dentry, NULL, flags, true);
 }
 
 static const struct dentry_operations ovl_dentry_operations = {
diff --git a/fs/pnode.c b/fs/pnode.c
index e4d0340393d5..a799e0315cc9 100644
--- a/fs/pnode.c
+++ b/fs/pnode.c
@@ -468,7 +468,7 @@ static void umount_one(struct mount *mnt, struct list_head *to_umount)
 	mnt->mnt.mnt_flags |= MNT_UMOUNT;
 	list_del_init(&mnt->mnt_child);
 	list_del_init(&mnt->mnt_umounting);
-	list_move_tail(&mnt->mnt_list, to_umount);
+	move_from_ns(mnt, to_umount);
 }
 
 /*
diff --git a/fs/proc/base.c b/fs/proc/base.c
index dd31e3b6bf77..28e6af0cca66 100644
--- a/fs/proc/base.c
+++ b/fs/proc/base.c
@@ -97,6 +97,7 @@
 #include <linux/resctrl.h>
 #include <linux/cn_proc.h>
 #include <linux/ksm.h>
+#include <uapi/linux/lsm.h>
 #include <trace/events/oom.h>
 #include "internal.h"
 #include "fd.h"
@@ -146,10 +147,10 @@ struct pid_entry {
 	NOD(NAME, (S_IFREG|(MODE)),			\
 		NULL, &proc_single_file_operations,	\
 		{ .proc_show = show } )
-#define ATTR(LSM, NAME, MODE)				\
+#define ATTR(LSMID, NAME, MODE)				\
 	NOD(NAME, (S_IFREG|(MODE)),			\
 		NULL, &proc_pid_attr_operations,	\
-		{ .lsm = LSM })
+		{ .lsmid = LSMID })
 
 /*
  * Count the number of hardlinks for the pid_entry table, excluding the .
@@ -2001,7 +2002,8 @@ void pid_update_inode(struct task_struct *task, struct inode *inode)
  * performed a setuid(), etc.
  *
  */
-static int pid_revalidate(struct dentry *dentry, unsigned int flags)
+static int pid_revalidate(struct dentry *dentry, const struct qstr *name,
+			  unsigned int flags)
 {
 	struct inode *inode;
 	struct task_struct *task;
@@ -2134,7 +2136,8 @@ static int dname_to_vma_addr(struct dentry *dentry,
 	return 0;
 }
 
-static int map_files_d_revalidate(struct dentry *dentry, unsigned int flags)
+static int map_files_d_revalidate(struct dentry *dentry,
+				  const struct qstr *name, unsigned int flags)
 {
 	unsigned long vm_start, vm_end;
 	bool exact_vma_exists = false;
@@ -2726,7 +2729,7 @@ static ssize_t proc_pid_attr_read(struct file * file, char __user * buf,
 	if (!task)
 		return -ESRCH;
 
-	length = security_getprocattr(task, PROC_I(inode)->op.lsm,
+	length = security_getprocattr(task, PROC_I(inode)->op.lsmid,
 				      file->f_path.dentry->d_name.name,
 				      &p);
 	put_task_struct(task);
@@ -2784,7 +2787,7 @@ static ssize_t proc_pid_attr_write(struct file * file, const char __user * buf,
 	if (rv < 0)
 		goto out_free;
 
-	rv = security_setprocattr(PROC_I(inode)->op.lsm,
+	rv = security_setprocattr(PROC_I(inode)->op.lsmid,
 				  file->f_path.dentry->d_name.name, page,
 				  count);
 	mutex_unlock(&current->signal->cred_guard_mutex);
@@ -2833,27 +2836,27 @@ static const struct inode_operations proc_##LSM##_attr_dir_inode_ops = { \
 
 #ifdef CONFIG_SECURITY_SMACK
 static const struct pid_entry smack_attr_dir_stuff[] = {
-	ATTR("smack", "current",	0666),
+	ATTR(LSM_ID_SMACK, "current",	0666),
 };
 LSM_DIR_OPS(smack);
 #endif
 
 #ifdef CONFIG_SECURITY_APPARMOR
 static const struct pid_entry apparmor_attr_dir_stuff[] = {
-	ATTR("apparmor", "current",	0666),
-	ATTR("apparmor", "prev",	0444),
-	ATTR("apparmor", "exec",	0666),
+	ATTR(LSM_ID_APPARMOR, "current",	0666),
+	ATTR(LSM_ID_APPARMOR, "prev",		0444),
+	ATTR(LSM_ID_APPARMOR, "exec",		0666),
 };
 LSM_DIR_OPS(apparmor);
 #endif
 
 static const struct pid_entry attr_dir_stuff[] = {
-	ATTR(NULL, "current",		0666),
-	ATTR(NULL, "prev",		0444),
-	ATTR(NULL, "exec",		0666),
-	ATTR(NULL, "fscreate",		0666),
-	ATTR(NULL, "keycreate",		0666),
-	ATTR(NULL, "sockcreate",	0666),
+	ATTR(LSM_ID_UNDEF, "current",	0666),
+	ATTR(LSM_ID_UNDEF, "prev",		0444),
+	ATTR(LSM_ID_UNDEF, "exec",		0666),
+	ATTR(LSM_ID_UNDEF, "fscreate",	0666),
+	ATTR(LSM_ID_UNDEF, "keycreate",	0666),
+	ATTR(LSM_ID_UNDEF, "sockcreate",	0666),
 #ifdef CONFIG_SECURITY_SMACK
 	DIR("smack",			0555,
 	    proc_smack_attr_dir_inode_ops, proc_smack_attr_dir_ops),
diff --git a/fs/proc/fd.c b/fs/proc/fd.c
index 6e72e5ad42bc..035d0297705a 100644
--- a/fs/proc/fd.c
+++ b/fs/proc/fd.c
@@ -138,7 +138,8 @@ static void tid_fd_update_inode(struct task_struct *task, struct inode *inode,
 	security_task_to_inode(task, inode);
 }
 
-static int tid_fd_revalidate(struct dentry *dentry, unsigned int flags)
+static int tid_fd_revalidate(struct dentry *dentry, const struct qstr *name,
+			     unsigned int flags)
 {
 	struct task_struct *task;
 	struct inode *inode;
diff --git a/fs/proc/generic.c b/fs/proc/generic.c
index 775ce0bcf08c..015f72b07927 100644
--- a/fs/proc/generic.c
+++ b/fs/proc/generic.c
@@ -216,7 +216,8 @@ void proc_free_inum(unsigned int inum)
 	ida_simple_remove(&proc_inum_ida, inum - PROC_DYNAMIC_FIRST);
 }
 
-static int proc_misc_d_revalidate(struct dentry *dentry, unsigned int flags)
+static int proc_misc_d_revalidate(struct dentry *dentry,
+				  const struct qstr *name, unsigned int flags)
 {
 	if (flags & LOOKUP_RCU)
 		return -ECHILD;
@@ -343,7 +344,8 @@ static const struct file_operations proc_dir_operations = {
 	.iterate_shared		= proc_readdir,
 };
 
-static int proc_net_d_revalidate(struct dentry *dentry, unsigned int flags)
+static int proc_net_d_revalidate(struct dentry *dentry,
+				 const struct qstr *name, unsigned int flags)
 {
 	return 0;
 }
diff --git a/fs/proc/internal.h b/fs/proc/internal.h
index 9a8f32f21ff5..a71ac5379584 100644
--- a/fs/proc/internal.h
+++ b/fs/proc/internal.h
@@ -92,7 +92,7 @@ union proc_op {
 	int (*proc_show)(struct seq_file *m,
 		struct pid_namespace *ns, struct pid *pid,
 		struct task_struct *task);
-	const char *lsm;
+	int lsmid;
 };
 
 struct proc_inode {
diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c
index 8064ea76f80b..6caf5cd25611 100644
--- a/fs/proc/proc_sysctl.c
+++ b/fs/proc/proc_sysctl.c
@@ -888,7 +888,8 @@ static const struct inode_operations proc_sys_dir_operations = {
 	.getattr	= proc_sys_getattr,
 };
 
-static int proc_sys_revalidate(struct dentry *dentry, unsigned int flags)
+static int proc_sys_revalidate(struct dentry *dentry, const struct qstr *name,
+			       unsigned int flags)
 {
 	if (flags & LOOKUP_RCU)
 		return -ECHILD;
diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index ef2eb12906da..d19924bf0a39 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -1761,7 +1761,7 @@ static int pagemap_release(struct inode *inode, struct file *file)
 #define PM_SCAN_CATEGORIES	(PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN |	\
 				 PAGE_IS_FILE |	PAGE_IS_PRESENT |	\
 				 PAGE_IS_SWAPPED | PAGE_IS_PFNZERO |	\
-				 PAGE_IS_HUGE)
+				 PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY)
 #define PM_SCAN_FLAGS		(PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC)
 
 struct pagemap_scan_private {
@@ -1793,6 +1793,8 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p,
 
 		if (is_zero_pfn(pte_pfn(pte)))
 			categories |= PAGE_IS_PFNZERO;
+		if (pte_soft_dirty(pte))
+			categories |= PAGE_IS_SOFT_DIRTY;
 	} else if (is_swap_pte(pte)) {
 		swp_entry_t swp;
 
@@ -1806,6 +1808,8 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p,
 			    !PageAnon(pfn_swap_entry_to_page(swp)))
 				categories |= PAGE_IS_FILE;
 		}
+		if (pte_swp_soft_dirty(pte))
+			categories |= PAGE_IS_SOFT_DIRTY;
 	}
 
 	return categories;
@@ -1853,12 +1857,16 @@ static unsigned long pagemap_thp_category(struct pagemap_scan_private *p,
 
 		if (is_zero_pfn(pmd_pfn(pmd)))
 			categories |= PAGE_IS_PFNZERO;
+		if (pmd_soft_dirty(pmd))
+			categories |= PAGE_IS_SOFT_DIRTY;
 	} else if (is_swap_pmd(pmd)) {
 		swp_entry_t swp;
 
 		categories |= PAGE_IS_SWAPPED;
 		if (!pmd_swp_uffd_wp(pmd))
 			categories |= PAGE_IS_WRITTEN;
+		if (pmd_swp_soft_dirty(pmd))
+			categories |= PAGE_IS_SOFT_DIRTY;
 
 		if (p->masks_of_interest & PAGE_IS_FILE) {
 			swp = pmd_to_swp_entry(pmd);
@@ -1905,10 +1913,14 @@ static unsigned long pagemap_hugetlb_category(pte_t pte)
 			categories |= PAGE_IS_FILE;
 		if (is_zero_pfn(pte_pfn(pte)))
 			categories |= PAGE_IS_PFNZERO;
+		if (pte_soft_dirty(pte))
+			categories |= PAGE_IS_SOFT_DIRTY;
 	} else if (is_swap_pte(pte)) {
 		categories |= PAGE_IS_SWAPPED;
 		if (!pte_swp_uffd_wp_any(pte))
 			categories |= PAGE_IS_WRITTEN;
+		if (pte_swp_soft_dirty(pte))
+			categories |= PAGE_IS_SOFT_DIRTY;
 	}
 
 	return categories;
@@ -1982,15 +1994,34 @@ static int pagemap_scan_test_walk(unsigned long start, unsigned long end,
 	struct pagemap_scan_private *p = walk->private;
 	struct vm_area_struct *vma = walk->vma;
 	unsigned long vma_category = 0;
+	bool wp_allowed = userfaultfd_wp_async(vma) &&
+	    userfaultfd_wp_use_markers(vma);
 
-	if (userfaultfd_wp_async(vma) && userfaultfd_wp_use_markers(vma))
-		vma_category |= PAGE_IS_WPALLOWED;
-	else if (p->arg.flags & PM_SCAN_CHECK_WPASYNC)
-		return -EPERM;
+	if (!wp_allowed) {
+		/* User requested explicit failure over wp-async capability */
+		if (p->arg.flags & PM_SCAN_CHECK_WPASYNC)
+			return -EPERM;
+		/*
+		 * User requires wr-protect, and allows silently skipping
+		 * unsupported vmas.
+		 */
+		if (p->arg.flags & PM_SCAN_WP_MATCHING)
+			return 1;
+		/*
+		 * Then the request doesn't involve wr-protects at all,
+		 * fall through to the rest checks, and allow vma walk.
+		 */
+	}
 
 	if (vma->vm_flags & VM_PFNMAP)
 		return 1;
 
+	if (wp_allowed)
+		vma_category |= PAGE_IS_WPALLOWED;
+
+	if (vma->vm_flags & VM_SOFTDIRTY)
+		vma_category |= PAGE_IS_SOFT_DIRTY;
+
 	if (!pagemap_scan_is_interesting_vma(vma_category, p))
 		return 1;
 
@@ -2140,7 +2171,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start,
 		return 0;
 	}
 
-	if (!p->vec_out) {
+	if ((p->arg.flags & PM_SCAN_WP_MATCHING) && !p->vec_out) {
 		/* Fast path for performing exclusive WP */
 		for (addr = start; addr != end; pte++, addr += PAGE_SIZE) {
 			if (pte_uffd_wp(ptep_get(pte)))
diff --git a/fs/proc_namespace.c b/fs/proc_namespace.c
index 250eb5bf7b52..0a808951b7d3 100644
--- a/fs/proc_namespace.c
+++ b/fs/proc_namespace.c
@@ -142,13 +142,9 @@ static int show_mountinfo(struct seq_file *m, struct vfsmount *mnt)
 
 	seq_printf(m, "%i %i %u:%u ", r->mnt_id, r->mnt_parent->mnt_id,
 		   MAJOR(sb->s_dev), MINOR(sb->s_dev));
-	if (sb->s_op->show_path) {
-		err = sb->s_op->show_path(m, mnt->mnt_root);
-		if (err)
-			goto out;
-	} else {
-		seq_dentry(m, mnt->mnt_root, " \t\n\\");
-	}
+	err = show_path(m, mnt->mnt_root);
+	if (err)
+		goto out;
 	seq_putc(m, ' ');
 
 	/* mountpoints outside of chroot jail will give SEQ_SKIP on this */
@@ -283,8 +279,6 @@ static int mounts_open_common(struct inode *inode, struct file *file,
 	p->ns = ns;
 	p->root = root;
 	p->show = show;
-	INIT_LIST_HEAD(&p->cursor.mnt_list);
-	p->cursor.mnt.mnt_flags = MNT_CURSOR;
 
 	return 0;
 
@@ -301,7 +295,6 @@ static int mounts_release(struct inode *inode, struct file *file)
 	struct seq_file *m = file->private_data;
 	struct proc_mounts *p = m->private;
 	path_put(&p->root);
-	mnt_cursor_del(p->ns, &p->cursor);
 	put_mnt_ns(p->ns);
 	return seq_release_private(inode, file);
 }
diff --git a/fs/pstore/ram.c b/fs/pstore/ram.c
index d36702c7ab3c..88b34fdbf759 100644
--- a/fs/pstore/ram.c
+++ b/fs/pstore/ram.c
@@ -529,6 +529,7 @@ static int ramoops_init_przs(const char *name,
 	}
 
 	zone_sz = mem_sz / *cnt;
+	zone_sz = ALIGN_DOWN(zone_sz, 2);
 	if (!zone_sz) {
 		dev_err(dev, "%s zone size == 0\n", name);
 		goto fail;
diff --git a/fs/reiserfs/xattr.c b/fs/reiserfs/xattr.c
index 998035a6388e..c7cb4b7ff142 100644
--- a/fs/reiserfs/xattr.c
+++ b/fs/reiserfs/xattr.c
@@ -958,7 +958,8 @@ int reiserfs_permission(struct mnt_idmap *idmap, struct inode *inode,
 	return generic_permission(&nop_mnt_idmap, inode, mask);
 }
 
-static int xattr_hide_revalidate(struct dentry *dentry, unsigned int flags)
+static int xattr_hide_revalidate(struct dentry *dentry,
+				 const struct qstr *name, unsigned int flags)
 {
 	return -EPERM;
 }
diff --git a/fs/smb/client/dir.c b/fs/smb/client/dir.c
index 580a27a3a7e6..00aefa7a6fe6 100644
--- a/fs/smb/client/dir.c
+++ b/fs/smb/client/dir.c
@@ -714,7 +714,8 @@ cifs_lookup(struct inode *parent_dir_inode, struct dentry *direntry,
 }
 
 static int
-cifs_d_revalidate(struct dentry *direntry, unsigned int flags)
+cifs_d_revalidate(struct dentry *direntry, const struct qstr *name,
+		  unsigned int flags)
 {
 	struct inode *inode;
 	int rc;
diff --git a/fs/smb/client/file.c b/fs/smb/client/file.c
index cf17e3dd703e..45ca492c141c 100644
--- a/fs/smb/client/file.c
+++ b/fs/smb/client/file.c
@@ -2706,8 +2706,7 @@ static void cifs_extend_writeback(struct address_space *mapping,
 			 */
 			if (!folio_clear_dirty_for_io(folio))
 				WARN_ON(1);
-			if (folio_start_writeback(folio))
-				WARN_ON(1);
+			folio_start_writeback(folio);
 
 			*_count -= folio_nr_pages(folio);
 			folio_unlock(folio);
@@ -2742,8 +2741,7 @@ static ssize_t cifs_write_back_from_locked_folio(struct address_space *mapping,
 	int rc;
 
 	/* The folio should be locked, dirty and not undergoing writeback. */
-	if (folio_start_writeback(folio))
-		WARN_ON(1);
+	folio_start_writeback(folio);
 
 	count -= folio_nr_pages(folio);
 	len = folio_size(folio);
diff --git a/fs/squashfs/block.c b/fs/squashfs/block.c
index 581ce9519339..2dc730800f44 100644
--- a/fs/squashfs/block.c
+++ b/fs/squashfs/block.c
@@ -321,7 +321,7 @@ int squashfs_read_data(struct super_block *sb, u64 index, int length,
 		TRACE("Block @ 0x%llx, %scompressed size %d\n", index - 2,
 		      compressed ? "" : "un", length);
 	}
-	if (length < 0 || length > output->length ||
+	if (length <= 0 || length > output->length ||
 			(index + length) > msblk->bytes_used) {
 		res = -EIO;
 		goto out;
diff --git a/fs/squashfs/file.c b/fs/squashfs/file.c
index 8ba8c4c50770..e8df6430444b 100644
--- a/fs/squashfs/file.c
+++ b/fs/squashfs/file.c
@@ -544,7 +544,8 @@ static void squashfs_readahead(struct readahead_control *ractl)
 	struct squashfs_page_actor *actor;
 	unsigned int nr_pages = 0;
 	struct page **pages;
-	int i, file_end = i_size_read(inode) >> msblk->block_log;
+	int i;
+	loff_t file_end = i_size_read(inode) >> msblk->block_log;
 	unsigned int max_pages = 1UL << shift;
 
 	readahead_expand(ractl, start, (len | mask) + 1);
diff --git a/fs/squashfs/file_direct.c b/fs/squashfs/file_direct.c
index f1ccad519e28..763a3f7a75f6 100644
--- a/fs/squashfs/file_direct.c
+++ b/fs/squashfs/file_direct.c
@@ -26,10 +26,10 @@ int squashfs_readpage_block(struct page *target_page, u64 block, int bsize,
 	struct inode *inode = target_page->mapping->host;
 	struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info;
 
-	int file_end = (i_size_read(inode) - 1) >> PAGE_SHIFT;
+	loff_t file_end = (i_size_read(inode) - 1) >> PAGE_SHIFT;
 	int mask = (1 << (msblk->block_log - PAGE_SHIFT)) - 1;
-	int start_index = target_page->index & ~mask;
-	int end_index = start_index | mask;
+	loff_t start_index = target_page->index & ~mask;
+	loff_t end_index = start_index | mask;
 	int i, n, pages, bytes, res = -ENOMEM;
 	struct page **page;
 	struct squashfs_page_actor *actor;
diff --git a/fs/stat.c b/fs/stat.c
index 24bb0209e459..d8c44c599cf2 100644
--- a/fs/stat.c
+++ b/fs/stat.c
@@ -133,7 +133,8 @@ int vfs_getattr_nosec(const struct path *path, struct kstat *stat,
 	idmap = mnt_idmap(path->mnt);
 	if (inode->i_op->getattr)
 		return inode->i_op->getattr(idmap, path, stat,
-					    request_mask, query_flags);
+					    request_mask,
+					    query_flags | AT_GETATTR_NOSEC);
 
 	generic_fillattr(idmap, request_mask, inode, stat);
 	return 0;
@@ -166,6 +167,9 @@ int vfs_getattr(const struct path *path, struct kstat *stat,
 {
 	int retval;
 
+	if (WARN_ON_ONCE(query_flags & AT_GETATTR_NOSEC))
+		return -EPERM;
+
 	retval = security_inode_getattr(path);
 	if (retval)
 		return retval;
@@ -243,8 +247,13 @@ static int vfs_statx(int dfd, struct filename *filename, int flags,
 
 	error = vfs_getattr(&path, stat, request_mask, flags);
 
-	stat->mnt_id = real_mount(path.mnt)->mnt_id;
-	stat->result_mask |= STATX_MNT_ID;
+	if (request_mask & STATX_MNT_ID_UNIQUE) {
+		stat->mnt_id = real_mount(path.mnt)->mnt_id_unique;
+		stat->result_mask |= STATX_MNT_ID_UNIQUE;
+	} else {
+		stat->mnt_id = real_mount(path.mnt)->mnt_id;
+		stat->result_mask |= STATX_MNT_ID;
+	}
 
 	if (path.mnt->mnt_root == path.dentry)
 		stat->attributes |= STATX_ATTR_MOUNT_ROOT;
diff --git a/fs/super.c b/fs/super.c
index 076392396e72..aa4e5c11ee32 100644
--- a/fs/super.c
+++ b/fs/super.c
@@ -105,8 +105,9 @@ static inline bool wait_born(struct super_block *sb)
  *
  * The caller must have acquired a temporary reference on @sb->s_count.
  *
- * Return: This returns true if SB_BORN was set, false if SB_DYING was
- *         set. The function acquires s_umount and returns with it held.
+ * Return: The function returns true if SB_BORN was set and with
+ *         s_umount held. The function returns false if SB_DYING was
+ *         set and without s_umount held.
  */
 static __must_check bool super_lock(struct super_block *sb, bool excl)
 {
@@ -121,8 +122,10 @@ static __must_check bool super_lock(struct super_block *sb, bool excl)
 	 * @sb->s_root is NULL and @sb->s_active is 0. No one needs to
 	 * grab a reference to this. Tell them so.
 	 */
-	if (sb->s_flags & SB_DYING)
+	if (sb->s_flags & SB_DYING) {
+		super_unlock(sb, excl);
 		return false;
+	}
 
 	/* Has called ->get_tree() successfully. */
 	if (sb->s_flags & SB_BORN)
@@ -140,13 +143,13 @@ static __must_check bool super_lock(struct super_block *sb, bool excl)
 	goto relock;
 }
 
-/* wait and acquire read-side of @sb->s_umount */
+/* wait and try to acquire read-side of @sb->s_umount */
 static inline bool super_lock_shared(struct super_block *sb)
 {
 	return super_lock(sb, false);
 }
 
-/* wait and acquire write-side of @sb->s_umount */
+/* wait and try to acquire write-side of @sb->s_umount */
 static inline bool super_lock_excl(struct super_block *sb)
 {
 	return super_lock(sb, true);
@@ -520,35 +523,6 @@ void deactivate_super(struct super_block *s)
 
 EXPORT_SYMBOL(deactivate_super);
 
-/**
- *	grab_super - acquire an active reference
- *	@s: reference we are trying to make active
- *
- *	Tries to acquire an active reference.  grab_super() is used when we
- * 	had just found a superblock in super_blocks or fs_type->fs_supers
- *	and want to turn it into a full-blown active reference.  grab_super()
- *	is called with sb_lock held and drops it.  Returns 1 in case of
- *	success, 0 if we had failed (superblock contents was already dead or
- *	dying when grab_super() had been called).  Note that this is only
- *	called for superblocks not in rundown mode (== ones still on ->fs_supers
- *	of their type), so increment of ->s_count is OK here.
- */
-static int grab_super(struct super_block *s) __releases(sb_lock)
-{
-	bool born;
-
-	s->s_count++;
-	spin_unlock(&sb_lock);
-	born = super_lock_excl(s);
-	if (born && atomic_inc_not_zero(&s->s_active)) {
-		put_super(s);
-		return 1;
-	}
-	super_unlock_excl(s);
-	put_super(s);
-	return 0;
-}
-
 static inline bool wait_dead(struct super_block *sb)
 {
 	unsigned int flags;
@@ -562,7 +536,7 @@ static inline bool wait_dead(struct super_block *sb)
 }
 
 /**
- * grab_super_dead - acquire an active reference to a superblock
+ * grab_super - acquire an active reference to a superblock
  * @sb: superblock to acquire
  *
  * Acquire a temporary reference on a superblock and try to trade it for
@@ -573,17 +547,21 @@ static inline bool wait_dead(struct super_block *sb)
  * Return: This returns true if an active reference could be acquired,
  *         false if not.
  */
-static bool grab_super_dead(struct super_block *sb)
+static bool grab_super(struct super_block *sb)
 {
+	bool locked;
 
 	sb->s_count++;
-	if (grab_super(sb)) {
-		put_super(sb);
-		lockdep_assert_held(&sb->s_umount);
-		return true;
+	spin_unlock(&sb_lock);
+	locked = super_lock_excl(sb);
+	if (locked) {
+		if (atomic_inc_not_zero(&sb->s_active)) {
+			put_super(sb);
+			return true;
+		}
+		super_unlock_excl(sb);
 	}
 	wait_var_event(&sb->s_flags, wait_dead(sb));
-	lockdep_assert_not_held(&sb->s_umount);
 	put_super(sb);
 	return false;
 }
@@ -834,7 +812,7 @@ struct super_block *sget_fc(struct fs_context *fc,
 			warnfc(fc, "reusing existing filesystem in another namespace not allowed");
 		return ERR_PTR(-EBUSY);
 	}
-	if (!grab_super_dead(old))
+	if (!grab_super(old))
 		goto retry;
 	destroy_unused_super(s);
 	return old;
@@ -878,7 +856,7 @@ struct super_block *sget(struct file_system_type *type,
 				destroy_unused_super(s);
 				return ERR_PTR(-EBUSY);
 			}
-			if (!grab_super_dead(old))
+			if (!grab_super(old))
 				goto retry;
 			destroy_unused_super(s);
 			return old;
@@ -961,15 +939,17 @@ void iterate_supers(void (*f)(struct super_block *, void *), void *arg)
 
 	spin_lock(&sb_lock);
 	list_for_each_entry(sb, &super_blocks, s_list) {
-		bool born;
+		bool locked;
 
 		sb->s_count++;
 		spin_unlock(&sb_lock);
 
-		born = super_lock_shared(sb);
-		if (born && sb->s_root)
-			f(sb, arg);
-		super_unlock_shared(sb);
+		locked = super_lock_shared(sb);
+		if (locked) {
+			if (sb->s_root)
+				f(sb, arg);
+			super_unlock_shared(sb);
+		}
 
 		spin_lock(&sb_lock);
 		if (p)
@@ -997,15 +977,17 @@ void iterate_supers_type(struct file_system_type *type,
 
 	spin_lock(&sb_lock);
 	hlist_for_each_entry(sb, &type->fs_supers, s_instances) {
-		bool born;
+		bool locked;
 
 		sb->s_count++;
 		spin_unlock(&sb_lock);
 
-		born = super_lock_shared(sb);
-		if (born && sb->s_root)
-			f(sb, arg);
-		super_unlock_shared(sb);
+		locked = super_lock_shared(sb);
+		if (locked) {
+			if (sb->s_root)
+				f(sb, arg);
+			super_unlock_shared(sb);
+		}
 
 		spin_lock(&sb_lock);
 		if (p)
@@ -1019,34 +1001,6 @@ void iterate_supers_type(struct file_system_type *type,
 
 EXPORT_SYMBOL(iterate_supers_type);
 
-/**
- * get_active_super - get an active reference to the superblock of a device
- * @bdev: device to get the superblock for
- *
- * Scans the superblock list and finds the superblock of the file system
- * mounted on the device given.  Returns the superblock with an active
- * reference or %NULL if none was found.
- */
-struct super_block *get_active_super(struct block_device *bdev)
-{
-	struct super_block *sb;
-
-	if (!bdev)
-		return NULL;
-
-	spin_lock(&sb_lock);
-	list_for_each_entry(sb, &super_blocks, s_list) {
-		if (sb->s_bdev == bdev) {
-			if (!grab_super(sb))
-				return NULL;
-			super_unlock_excl(sb);
-			return sb;
-		}
-	}
-	spin_unlock(&sb_lock);
-	return NULL;
-}
-
 struct super_block *user_get_super(dev_t dev, bool excl)
 {
 	struct super_block *sb;
@@ -1054,15 +1008,17 @@ struct super_block *user_get_super(dev_t dev, bool excl)
 	spin_lock(&sb_lock);
 	list_for_each_entry(sb, &super_blocks, s_list) {
 		if (sb->s_dev ==  dev) {
-			bool born;
+			bool locked;
 
 			sb->s_count++;
 			spin_unlock(&sb_lock);
 			/* still alive? */
-			born = super_lock(sb, excl);
-			if (born && sb->s_root)
-				return sb;
-			super_unlock(sb, excl);
+			locked = super_lock(sb, excl);
+			if (locked) {
+				if (sb->s_root)
+					return sb;
+				super_unlock(sb, excl);
+			}
 			/* nope, got unmounted */
 			spin_lock(&sb_lock);
 			__put_super(sb);
@@ -1173,9 +1129,9 @@ int reconfigure_super(struct fs_context *fc)
 
 static void do_emergency_remount_callback(struct super_block *sb)
 {
-	bool born = super_lock_excl(sb);
+	bool locked = super_lock_excl(sb);
 
-	if (born && sb->s_root && sb->s_bdev && !sb_rdonly(sb)) {
+	if (locked && sb->s_root && sb->s_bdev && !sb_rdonly(sb)) {
 		struct fs_context *fc;
 
 		fc = fs_context_for_reconfigure(sb->s_root,
@@ -1186,7 +1142,8 @@ static void do_emergency_remount_callback(struct super_block *sb)
 			put_fs_context(fc);
 		}
 	}
-	super_unlock_excl(sb);
+	if (locked)
+		super_unlock_excl(sb);
 }
 
 static void do_emergency_remount(struct work_struct *work)
@@ -1209,16 +1166,17 @@ void emergency_remount(void)
 
 static void do_thaw_all_callback(struct super_block *sb)
 {
-	bool born = super_lock_excl(sb);
+	bool locked = super_lock_excl(sb);
 
-	if (born && sb->s_root) {
+	if (locked && sb->s_root) {
 		if (IS_ENABLED(CONFIG_BLOCK))
-			while (sb->s_bdev && !thaw_bdev(sb->s_bdev))
+			while (sb->s_bdev && !bdev_thaw(sb->s_bdev))
 				pr_warn("Emergency Thaw on %pg\n", sb->s_bdev);
 		thaw_super_locked(sb, FREEZE_HOLDER_USERSPACE);
-	} else {
-		super_unlock_excl(sb);
+		return;
 	}
+	if (locked)
+		super_unlock_excl(sb);
 }
 
 static void do_thaw_all(struct work_struct *work)
@@ -1428,11 +1386,11 @@ EXPORT_SYMBOL(sget_dev);
  *
  * The function must be called with bdev->bd_holder_lock and releases it.
  */
-static struct super_block *bdev_super_lock_shared(struct block_device *bdev)
+static struct super_block *bdev_super_lock(struct block_device *bdev, bool excl)
 	__releases(&bdev->bd_holder_lock)
 {
 	struct super_block *sb = bdev->bd_holder;
-	bool born;
+	bool locked;
 
 	lockdep_assert_held(&bdev->bd_holder_lock);
 	lockdep_assert_not_held(&sb->s_umount);
@@ -1442,19 +1400,25 @@ static struct super_block *bdev_super_lock_shared(struct block_device *bdev)
 	spin_lock(&sb_lock);
 	sb->s_count++;
 	spin_unlock(&sb_lock);
+
 	mutex_unlock(&bdev->bd_holder_lock);
 
-	born = super_lock_shared(sb);
-	if (!born || !sb->s_root || !(sb->s_flags & SB_ACTIVE)) {
-		super_unlock_shared(sb);
-		put_super(sb);
-		return NULL;
-	}
+	locked = super_lock(sb, excl);
+
 	/*
-	 * The superblock is active and we hold s_umount, we can drop our
-	 * temporary reference now.
-	 */
+	 * If the superblock wasn't already SB_DYING then we hold
+	 * s_umount and can safely drop our temporary reference.
+         */
 	put_super(sb);
+
+	if (!locked)
+		return NULL;
+
+	if (!sb->s_root || !(sb->s_flags & SB_ACTIVE)) {
+		super_unlock(sb, excl);
+		return NULL;
+	}
+
 	return sb;
 }
 
@@ -1462,7 +1426,7 @@ static void fs_bdev_mark_dead(struct block_device *bdev, bool surprise)
 {
 	struct super_block *sb;
 
-	sb = bdev_super_lock_shared(bdev);
+	sb = bdev_super_lock(bdev, false);
 	if (!sb)
 		return;
 
@@ -1480,16 +1444,110 @@ static void fs_bdev_sync(struct block_device *bdev)
 {
 	struct super_block *sb;
 
-	sb = bdev_super_lock_shared(bdev);
+	sb = bdev_super_lock(bdev, false);
 	if (!sb)
 		return;
+
 	sync_filesystem(sb);
 	super_unlock_shared(sb);
 }
 
+static struct super_block *get_bdev_super(struct block_device *bdev)
+{
+	bool active = false;
+	struct super_block *sb;
+
+	sb = bdev_super_lock(bdev, true);
+	if (sb) {
+		active = atomic_inc_not_zero(&sb->s_active);
+		super_unlock_excl(sb);
+	}
+	if (!active)
+		return NULL;
+	return sb;
+}
+
+/**
+ * fs_bdev_freeze - freeze owning filesystem of block device
+ * @bdev: block device
+ *
+ * Freeze the filesystem that owns this block device if it is still
+ * active.
+ *
+ * A filesystem that owns multiple block devices may be frozen from each
+ * block device and won't be unfrozen until all block devices are
+ * unfrozen. Each block device can only freeze the filesystem once as we
+ * nest freezes for block devices in the block layer.
+ *
+ * Return: If the freeze was successful zero is returned. If the freeze
+ *         failed a negative error code is returned.
+ */
+static int fs_bdev_freeze(struct block_device *bdev)
+{
+	struct super_block *sb;
+	int error = 0;
+
+	lockdep_assert_held(&bdev->bd_fsfreeze_mutex);
+
+	sb = get_bdev_super(bdev);
+	if (!sb)
+		return -EINVAL;
+
+	if (sb->s_op->freeze_super)
+		error = sb->s_op->freeze_super(sb,
+				FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE);
+	else
+		error = freeze_super(sb,
+				FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE);
+	if (!error)
+		error = sync_blockdev(bdev);
+	deactivate_super(sb);
+	return error;
+}
+
+/**
+ * fs_bdev_thaw - thaw owning filesystem of block device
+ * @bdev: block device
+ *
+ * Thaw the filesystem that owns this block device.
+ *
+ * A filesystem that owns multiple block devices may be frozen from each
+ * block device and won't be unfrozen until all block devices are
+ * unfrozen. Each block device can only freeze the filesystem once as we
+ * nest freezes for block devices in the block layer.
+ *
+ * Return: If the thaw was successful zero is returned. If the thaw
+ *         failed a negative error code is returned. If this function
+ *         returns zero it doesn't mean that the filesystem is unfrozen
+ *         as it may have been frozen multiple times (kernel may hold a
+ *         freeze or might be frozen from other block devices).
+ */
+static int fs_bdev_thaw(struct block_device *bdev)
+{
+	struct super_block *sb;
+	int error;
+
+	lockdep_assert_held(&bdev->bd_fsfreeze_mutex);
+
+	sb = get_bdev_super(bdev);
+	if (WARN_ON_ONCE(!sb))
+		return -EINVAL;
+
+	if (sb->s_op->thaw_super)
+		error = sb->s_op->thaw_super(sb,
+				FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE);
+	else
+		error = thaw_super(sb,
+				FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE);
+	deactivate_super(sb);
+	return error;
+}
+
 const struct blk_holder_ops fs_holder_ops = {
 	.mark_dead		= fs_bdev_mark_dead,
 	.sync			= fs_bdev_sync,
+	.freeze			= fs_bdev_freeze,
+	.thaw			= fs_bdev_thaw,
 };
 EXPORT_SYMBOL_GPL(fs_holder_ops);
 
@@ -1519,15 +1577,10 @@ int setup_bdev_super(struct super_block *sb, int sb_flags,
 	}
 
 	/*
-	 * Until SB_BORN flag is set, there can be no active superblock
-	 * references and thus no filesystem freezing. get_active_super() will
-	 * just loop waiting for SB_BORN so even freeze_bdev() cannot proceed.
-	 *
-	 * It is enough to check bdev was not frozen before we set s_bdev.
+	 * It is enough to check bdev was not frozen before we set
+	 * s_bdev as freezing will wait until SB_BORN is set.
 	 */
-	mutex_lock(&bdev->bd_fsfreeze_mutex);
-	if (bdev->bd_fsfreeze_count > 0) {
-		mutex_unlock(&bdev->bd_fsfreeze_mutex);
+	if (atomic_read(&bdev->bd_fsfreeze_count) > 0) {
 		if (fc)
 			warnf(fc, "%pg: Can't mount, blockdev is frozen", bdev);
 		bdev_release(bdev_handle);
@@ -1540,7 +1593,6 @@ int setup_bdev_super(struct super_block *sb, int sb_flags,
 	if (bdev_stable_writes(bdev))
 		sb->s_iflags |= SB_I_STABLE_WRITES;
 	spin_unlock(&sb_lock);
-	mutex_unlock(&bdev->bd_fsfreeze_mutex);
 
 	snprintf(sb->s_id, sizeof(sb->s_id), "%pg", bdev);
 	shrinker_debugfs_rename(sb->s_shrink, "sb-%s:%s", sb->s_type->name,
@@ -1585,15 +1637,7 @@ int get_tree_bdev(struct fs_context *fc,
 			return -EBUSY;
 		}
 	} else {
-		/*
-		 * We drop s_umount here because we need to open the bdev and
-		 * bdev->open_mutex ranks above s_umount (blkdev_put() ->
-		 * bdev_mark_dead()). It is safe because we have active sb
-		 * reference and SB_BORN is not set yet.
-		 */
-		super_unlock_excl(s);
 		error = setup_bdev_super(s, fc->sb_flags, fc);
-		__super_lock_excl(s);
 		if (!error)
 			error = fill_super(s, fc);
 		if (error) {
@@ -1637,15 +1681,7 @@ struct dentry *mount_bdev(struct file_system_type *fs_type,
 			return ERR_PTR(-EBUSY);
 		}
 	} else {
-		/*
-		 * We drop s_umount here because we need to open the bdev and
-		 * bdev->open_mutex ranks above s_umount (blkdev_put() ->
-		 * bdev_mark_dead()). It is safe because we have active sb
-		 * reference and SB_BORN is not set yet.
-		 */
-		super_unlock_excl(s);
 		error = setup_bdev_super(s, flags, NULL);
-		__super_lock_excl(s);
 		if (!error)
 			error = fill_super(s, data, flags & SB_SILENT ? 1 : 0);
 		if (error) {
@@ -1914,6 +1950,47 @@ static int wait_for_partially_frozen(struct super_block *sb)
 	return ret;
 }
 
+#define FREEZE_HOLDERS (FREEZE_HOLDER_KERNEL | FREEZE_HOLDER_USERSPACE)
+#define FREEZE_FLAGS (FREEZE_HOLDERS | FREEZE_MAY_NEST)
+
+static inline int freeze_inc(struct super_block *sb, enum freeze_holder who)
+{
+	WARN_ON_ONCE((who & ~FREEZE_FLAGS));
+	WARN_ON_ONCE(hweight32(who & FREEZE_HOLDERS) > 1);
+
+	if (who & FREEZE_HOLDER_KERNEL)
+		++sb->s_writers.freeze_kcount;
+	if (who & FREEZE_HOLDER_USERSPACE)
+		++sb->s_writers.freeze_ucount;
+	return sb->s_writers.freeze_kcount + sb->s_writers.freeze_ucount;
+}
+
+static inline int freeze_dec(struct super_block *sb, enum freeze_holder who)
+{
+	WARN_ON_ONCE((who & ~FREEZE_FLAGS));
+	WARN_ON_ONCE(hweight32(who & FREEZE_HOLDERS) > 1);
+
+	if ((who & FREEZE_HOLDER_KERNEL) && sb->s_writers.freeze_kcount)
+		--sb->s_writers.freeze_kcount;
+	if ((who & FREEZE_HOLDER_USERSPACE) && sb->s_writers.freeze_ucount)
+		--sb->s_writers.freeze_ucount;
+	return sb->s_writers.freeze_kcount + sb->s_writers.freeze_ucount;
+}
+
+static inline bool may_freeze(struct super_block *sb, enum freeze_holder who)
+{
+	WARN_ON_ONCE((who & ~FREEZE_FLAGS));
+	WARN_ON_ONCE(hweight32(who & FREEZE_HOLDERS) > 1);
+
+	if (who & FREEZE_HOLDER_KERNEL)
+		return (who & FREEZE_MAY_NEST) ||
+		       sb->s_writers.freeze_kcount == 0;
+	if (who & FREEZE_HOLDER_USERSPACE)
+		return (who & FREEZE_MAY_NEST) ||
+		       sb->s_writers.freeze_ucount == 0;
+	return false;
+}
+
 /**
  * freeze_super - lock the filesystem and force it into a consistent state
  * @sb: the super to lock
@@ -1926,6 +2003,7 @@ static int wait_for_partially_frozen(struct super_block *sb)
  * @who should be:
  * * %FREEZE_HOLDER_USERSPACE if userspace wants to freeze the fs;
  * * %FREEZE_HOLDER_KERNEL if the kernel wants to freeze the fs.
+ * * %FREEZE_MAY_NEST whether nesting freeze and thaw requests is allowed.
  *
  * The @who argument distinguishes between the kernel and userspace trying to
  * freeze the filesystem.  Although there cannot be multiple kernel freezes or
@@ -1933,6 +2011,13 @@ static int wait_for_partially_frozen(struct super_block *sb)
  * userspace can both hold a filesystem frozen.  The filesystem remains frozen
  * until there are no kernel or userspace freezes in effect.
  *
+ * A filesystem may hold multiple devices and thus a filesystems may be
+ * frozen through the block layer via multiple block devices. In this
+ * case the request is marked as being allowed to nest by passing
+ * FREEZE_MAY_NEST. The filesystem remains frozen until all block
+ * devices are unfrozen. If multiple freezes are attempted without
+ * FREEZE_MAY_NEST -EBUSY will be returned.
+ *
  * During this function, sb->s_writers.frozen goes through these values:
  *
  * SB_UNFROZEN: File system is normal, all writes progress as usual.
@@ -1957,31 +2042,29 @@ static int wait_for_partially_frozen(struct super_block *sb)
  * mostly auxiliary for filesystems to verify they do not modify frozen fs.
  *
  * sb->s_writers.frozen is protected by sb->s_umount.
+ *
+ * Return: If the freeze was successful zero is returned. If the freeze
+ *         failed a negative error code is returned.
  */
 int freeze_super(struct super_block *sb, enum freeze_holder who)
 {
 	int ret;
 
+	if (!super_lock_excl(sb)) {
+		WARN_ON_ONCE("Dying superblock while freezing!");
+		return -EINVAL;
+	}
 	atomic_inc(&sb->s_active);
-	if (!super_lock_excl(sb))
-		WARN(1, "Dying superblock while freezing!");
 
 retry:
 	if (sb->s_writers.frozen == SB_FREEZE_COMPLETE) {
-		if (sb->s_writers.freeze_holders & who) {
-			deactivate_locked_super(sb);
-			return -EBUSY;
-		}
-
-		WARN_ON(sb->s_writers.freeze_holders == 0);
-
-		/*
-		 * Someone else already holds this type of freeze; share the
-		 * freeze and assign the active ref to the freeze.
-		 */
-		sb->s_writers.freeze_holders |= who;
-		super_unlock_excl(sb);
-		return 0;
+		if (may_freeze(sb, who))
+			ret = !!WARN_ON_ONCE(freeze_inc(sb, who) == 1);
+		else
+			ret = -EBUSY;
+		/* All freezers share a single active reference. */
+		deactivate_locked_super(sb);
+		return ret;
 	}
 
 	if (sb->s_writers.frozen != SB_UNFROZEN) {
@@ -1994,14 +2077,9 @@ int freeze_super(struct super_block *sb, enum freeze_holder who)
 		goto retry;
 	}
 
-	if (!(sb->s_flags & SB_BORN)) {
-		super_unlock_excl(sb);
-		return 0;	/* sic - it's "nothing to do" */
-	}
-
 	if (sb_rdonly(sb)) {
 		/* Nothing to do really... */
-		sb->s_writers.freeze_holders |= who;
+		WARN_ON_ONCE(freeze_inc(sb, who) > 1);
 		sb->s_writers.frozen = SB_FREEZE_COMPLETE;
 		wake_up_var(&sb->s_writers.frozen);
 		super_unlock_excl(sb);
@@ -2012,8 +2090,10 @@ int freeze_super(struct super_block *sb, enum freeze_holder who)
 	/* Release s_umount to preserve sb_start_write -> s_umount ordering */
 	super_unlock_excl(sb);
 	sb_wait_write(sb, SB_FREEZE_WRITE);
-	if (!super_lock_excl(sb))
-		WARN(1, "Dying superblock while freezing!");
+	if (!super_lock_excl(sb)) {
+		WARN_ON_ONCE("Dying superblock while freezing!");
+		return -EINVAL;
+	}
 
 	/* Now we go and block page faults... */
 	sb->s_writers.frozen = SB_FREEZE_PAGEFAULT;
@@ -2049,7 +2129,7 @@ int freeze_super(struct super_block *sb, enum freeze_holder who)
 	 * For debugging purposes so that fs can warn if it sees write activity
 	 * when frozen is set to SB_FREEZE_COMPLETE, and for thaw_super().
 	 */
-	sb->s_writers.freeze_holders |= who;
+	WARN_ON_ONCE(freeze_inc(sb, who) > 1);
 	sb->s_writers.frozen = SB_FREEZE_COMPLETE;
 	wake_up_var(&sb->s_writers.frozen);
 	lockdep_sb_freeze_release(sb);
@@ -2066,34 +2146,22 @@ EXPORT_SYMBOL(freeze_super);
  */
 static int thaw_super_locked(struct super_block *sb, enum freeze_holder who)
 {
-	int error;
+	int error = -EINVAL;
 
-	if (sb->s_writers.frozen == SB_FREEZE_COMPLETE) {
-		if (!(sb->s_writers.freeze_holders & who)) {
-			super_unlock_excl(sb);
-			return -EINVAL;
-		}
+	if (sb->s_writers.frozen != SB_FREEZE_COMPLETE)
+		goto out_unlock;
 
-		/*
-		 * Freeze is shared with someone else.  Release our hold and
-		 * drop the active ref that freeze_super assigned to the
-		 * freezer.
-		 */
-		if (sb->s_writers.freeze_holders & ~who) {
-			sb->s_writers.freeze_holders &= ~who;
-			deactivate_locked_super(sb);
-			return 0;
-		}
-	} else {
-		super_unlock_excl(sb);
-		return -EINVAL;
-	}
+	/*
+	 * All freezers share a single active reference.
+	 * So just unlock in case there are any left.
+	 */
+	if (freeze_dec(sb, who))
+		goto out_unlock;
 
 	if (sb_rdonly(sb)) {
-		sb->s_writers.freeze_holders &= ~who;
 		sb->s_writers.frozen = SB_UNFROZEN;
 		wake_up_var(&sb->s_writers.frozen);
-		goto out;
+		goto out_deactivate;
 	}
 
 	lockdep_sb_freeze_acquire(sb);
@@ -2101,20 +2169,23 @@ static int thaw_super_locked(struct super_block *sb, enum freeze_holder who)
 	if (sb->s_op->unfreeze_fs) {
 		error = sb->s_op->unfreeze_fs(sb);
 		if (error) {
-			printk(KERN_ERR "VFS:Filesystem thaw failed\n");
+			pr_err("VFS: Filesystem thaw failed\n");
+			freeze_inc(sb, who);
 			lockdep_sb_freeze_release(sb);
-			super_unlock_excl(sb);
-			return error;
+			goto out_unlock;
 		}
 	}
 
-	sb->s_writers.freeze_holders &= ~who;
 	sb->s_writers.frozen = SB_UNFROZEN;
 	wake_up_var(&sb->s_writers.frozen);
 	sb_freeze_unlock(sb, SB_FREEZE_FS);
-out:
+out_deactivate:
 	deactivate_locked_super(sb);
 	return 0;
+
+out_unlock:
+	super_unlock_excl(sb);
+	return error;
 }
 
 /**
@@ -2128,11 +2199,18 @@ static int thaw_super_locked(struct super_block *sb, enum freeze_holder who)
  * @who should be:
  * * %FREEZE_HOLDER_USERSPACE if userspace wants to thaw the fs;
  * * %FREEZE_HOLDER_KERNEL if the kernel wants to thaw the fs.
+ * * %FREEZE_MAY_NEST whether nesting freeze and thaw requests is allowed
+ *
+ * A filesystem may hold multiple devices and thus a filesystems may
+ * have been frozen through the block layer via multiple block devices.
+ * The filesystem remains frozen until all block devices are unfrozen.
  */
 int thaw_super(struct super_block *sb, enum freeze_holder who)
 {
-	if (!super_lock_excl(sb))
-		WARN(1, "Dying superblock while thawing!");
+	if (!super_lock_excl(sb)) {
+		WARN_ON_ONCE("Dying superblock while thawing!");
+		return -EINVAL;
+	}
 	return thaw_super_locked(sb, who);
 }
 EXPORT_SYMBOL(thaw_super);
diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c
index e8af40b05549..38b10390a8aa 100644
--- a/fs/userfaultfd.c
+++ b/fs/userfaultfd.c
@@ -1033,7 +1033,7 @@ static int resolve_userfault_fork(struct userfaultfd_ctx *new,
 {
 	int fd;
 
-	fd = anon_inode_getfd_secure("[userfaultfd]", &userfaultfd_fops, new,
+	fd = anon_inode_create_getfd("[userfaultfd]", &userfaultfd_fops, new,
 			O_RDONLY | (new->flags & UFFD_SHARED_FCNTL_FLAGS), inode);
 	if (fd < 0)
 		return fd;
@@ -2189,7 +2189,8 @@ static int new_userfaultfd(int flags)
 	/* prevent the mm struct to be freed */
 	mmgrab(ctx->mm);
 
-	fd = anon_inode_getfd_secure("[userfaultfd]", &userfaultfd_fops, ctx,
+	/* Create a new inode so that the LSM can block the creation.  */
+	fd = anon_inode_create_getfd("[userfaultfd]", &userfaultfd_fops, ctx,
 			O_RDONLY | (flags & UFFD_SHARED_FCNTL_FLAGS), NULL);
 	if (fd < 0) {
 		mmdrop(ctx->mm);
diff --git a/fs/vboxsf/dir.c b/fs/vboxsf/dir.c
index 5f1a14d5b927..792d9aa83561 100644
--- a/fs/vboxsf/dir.c
+++ b/fs/vboxsf/dir.c
@@ -192,7 +192,9 @@ const struct file_operations vboxsf_dir_fops = {
  * This is called during name resolution/lookup to check if the @dentry in
  * the cache is still valid. the job is handled by vboxsf_inode_revalidate.
  */
-static int vboxsf_dentry_revalidate(struct dentry *dentry, unsigned int flags)
+static int vboxsf_dentry_revalidate(struct dentry *dentry,
+				    const struct qstr *name,
+				    unsigned int flags)
 {
 	if (flags & LOOKUP_RCU)
 		return -ECHILD;
diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c
index 465d7630bb21..813f85156b0c 100644
--- a/fs/xfs/xfs_aops.c
+++ b/fs/xfs/xfs_aops.c
@@ -584,7 +584,7 @@ const struct address_space_operations xfs_address_space_operations = {
 	.bmap			= xfs_vm_bmap,
 	.migrate_folio		= filemap_migrate_folio,
 	.is_partially_uptodate  = iomap_is_partially_uptodate,
-	.error_remove_page	= generic_error_remove_page,
+	.error_remove_folio	= generic_error_remove_folio,
 	.swap_activate		= xfs_iomap_swapfile_activate,
 };
 
diff --git a/fs/xfs/xfs_fsops.c b/fs/xfs/xfs_fsops.c
index 7cb75cb6b8e9..57076a25f17d 100644
--- a/fs/xfs/xfs_fsops.c
+++ b/fs/xfs/xfs_fsops.c
@@ -482,9 +482,9 @@ xfs_fs_goingdown(
 {
 	switch (inflags) {
 	case XFS_FSOP_GOING_FLAGS_DEFAULT: {
-		if (!freeze_bdev(mp->m_super->s_bdev)) {
+		if (!bdev_freeze(mp->m_super->s_bdev)) {
 			xfs_force_shutdown(mp, SHUTDOWN_FORCE_UMOUNT);
-			thaw_bdev(mp->m_super->s_bdev);
+			bdev_thaw(mp->m_super->s_bdev);
 		}
 		break;
 	}
diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h
index 3dc47937da5d..3beb470f1892 100644
--- a/fs/xfs/xfs_inode.h
+++ b/fs/xfs/xfs_inode.h
@@ -569,6 +569,14 @@ extern void xfs_setup_inode(struct xfs_inode *ip);
 extern void xfs_setup_iops(struct xfs_inode *ip);
 extern void xfs_diflags_to_iflags(struct xfs_inode *ip, bool init);
 
+static inline void xfs_update_stable_writes(struct xfs_inode *ip)
+{
+	if (bdev_stable_writes(xfs_inode_buftarg(ip)->bt_bdev))
+		mapping_set_stable_writes(VFS_I(ip)->i_mapping);
+	else
+		mapping_clear_stable_writes(VFS_I(ip)->i_mapping);
+}
+
 /*
  * When setting up a newly allocated inode, we need to call
  * xfs_finish_inode_setup() once the inode is fully instantiated at
diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c
index a82470e027f7..6c3919687ea6 100644
--- a/fs/xfs/xfs_ioctl.c
+++ b/fs/xfs/xfs_ioctl.c
@@ -1121,23 +1121,25 @@ xfs_ioctl_setattr_xflags(
 	struct fileattr		*fa)
 {
 	struct xfs_mount	*mp = ip->i_mount;
+	bool			rtflag = (fa->fsx_xflags & FS_XFLAG_REALTIME);
 	uint64_t		i_flags2;
 
-	/* Can't change realtime flag if any extents are allocated. */
-	if ((ip->i_df.if_nextents || ip->i_delayed_blks) &&
-	    XFS_IS_REALTIME_INODE(ip) != (fa->fsx_xflags & FS_XFLAG_REALTIME))
-		return -EINVAL;
+	if (rtflag != XFS_IS_REALTIME_INODE(ip)) {
+		/* Can't change realtime flag if any extents are allocated. */
+		if (ip->i_df.if_nextents || ip->i_delayed_blks)
+			return -EINVAL;
+	}
 
-	/* If realtime flag is set then must have realtime device */
-	if (fa->fsx_xflags & FS_XFLAG_REALTIME) {
+	if (rtflag) {
+		/* If realtime flag is set then must have realtime device */
 		if (mp->m_sb.sb_rblocks == 0 || mp->m_sb.sb_rextsize == 0 ||
 		    xfs_extlen_to_rtxmod(mp, ip->i_extsize))
 			return -EINVAL;
-	}
 
-	/* Clear reflink if we are actually able to set the rt flag. */
-	if ((fa->fsx_xflags & FS_XFLAG_REALTIME) && xfs_is_reflink_inode(ip))
-		ip->i_diflags2 &= ~XFS_DIFLAG2_REFLINK;
+		/* Clear reflink if we are actually able to set the rt flag. */
+		if (xfs_is_reflink_inode(ip))
+			ip->i_diflags2 &= ~XFS_DIFLAG2_REFLINK;
+	}
 
 	/* diflags2 only valid for v3 inodes. */
 	i_flags2 = xfs_flags2diflags2(ip, fa->fsx_xflags);
@@ -1148,6 +1150,14 @@ xfs_ioctl_setattr_xflags(
 	ip->i_diflags2 = i_flags2;
 
 	xfs_diflags_to_iflags(ip, false);
+
+	/*
+	 * Make the stable writes flag match that of the device the inode
+	 * resides on when flipping the RT flag.
+	 */
+	if (rtflag != XFS_IS_REALTIME_INODE(ip) && S_ISREG(VFS_I(ip)->i_mode))
+		xfs_update_stable_writes(ip);
+
 	xfs_trans_ichgtime(tp, ip, XFS_ICHGTIME_CHG);
 	xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
 	XFS_STATS_INC(mp, xs_ig_attrchg);
diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c
index fdfda4fba12b..a0d77f5f512e 100644
--- a/fs/xfs/xfs_iops.c
+++ b/fs/xfs/xfs_iops.c
@@ -1298,6 +1298,13 @@ xfs_setup_inode(
 	gfp_mask = mapping_gfp_mask(inode->i_mapping);
 	mapping_set_gfp_mask(inode->i_mapping, (gfp_mask & ~(__GFP_FS)));
 
+	/*
+	 * For real-time inodes update the stable write flags to that of the RT
+	 * device instead of the data device.
+	 */
+	if (S_ISREG(inode->i_mode) && XFS_IS_REALTIME_INODE(ip))
+		xfs_update_stable_writes(ip);
+
 	/*
 	 * If there is no attribute fork no ACL can exist on this inode,
 	 * and it can't have any file capabilities attached to it either.
diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c
index 764304595e8b..07857d967ee8 100644
--- a/fs/xfs/xfs_super.c
+++ b/fs/xfs/xfs_super.c
@@ -366,8 +366,9 @@ xfs_blkdev_get(
 {
 	int			error = 0;
 
-	*handlep = bdev_open_by_path(name, BLK_OPEN_READ | BLK_OPEN_WRITE,
-				     mp->m_super, &fs_holder_ops);
+	*handlep = bdev_open_by_path(name,
+		BLK_OPEN_READ | BLK_OPEN_WRITE | BLK_OPEN_RESTRICT_WRITES,
+		mp->m_super, &fs_holder_ops);
 	if (IS_ERR(*handlep)) {
 		error = PTR_ERR(*handlep);
 		*handlep = NULL;
@@ -438,19 +439,13 @@ xfs_open_devices(
 	struct bdev_handle	*logdev_handle = NULL, *rtdev_handle = NULL;
 	int			error;
 
-	/*
-	 * blkdev_put() can't be called under s_umount, see the comment
-	 * in get_tree_bdev() for more details
-	 */
-	up_write(&sb->s_umount);
-
 	/*
 	 * Open real time and log devices - order is important.
 	 */
 	if (mp->m_logname) {
 		error = xfs_blkdev_get(mp, mp->m_logname, &logdev_handle);
 		if (error)
-			goto out_relock;
+			return error;
 	}
 
 	if (mp->m_rtname) {
@@ -493,10 +488,7 @@ xfs_open_devices(
 			bdev_release(logdev_handle);
 	}
 
-	error = 0;
-out_relock:
-	down_write(&sb->s_umount);
-	return error;
+	return 0;
 
  out_free_rtdev_targ:
 	if (mp->m_rtdev_targp)
@@ -509,7 +501,7 @@ xfs_open_devices(
  out_close_logdev:
 	if (logdev_handle)
 		bdev_release(logdev_handle);
-	goto out_relock;
+	return error;
 }
 
 /*
@@ -759,10 +751,6 @@ static void
 xfs_mount_free(
 	struct xfs_mount	*mp)
 {
-	/*
-	 * Free the buftargs here because blkdev_put needs to be called outside
-	 * of sb->s_umount, which is held around the call to ->put_super.
-	 */
 	if (mp->m_logdev_targp && mp->m_logdev_targp != mp->m_ddev_targp)
 		xfs_free_buftarg(mp->m_logdev_targp);
 	if (mp->m_rtdev_targp)
diff --git a/fs/zonefs/file.c b/fs/zonefs/file.c
index b2c9b35df8f7..6ab2318a9c8e 100644
--- a/fs/zonefs/file.c
+++ b/fs/zonefs/file.c
@@ -180,7 +180,7 @@ const struct address_space_operations zonefs_file_aops = {
 	.invalidate_folio	= iomap_invalidate_folio,
 	.migrate_folio		= filemap_migrate_folio,
 	.is_partially_uptodate	= iomap_is_partially_uptodate,
-	.error_remove_page	= generic_error_remove_page,
+	.error_remove_folio	= generic_error_remove_folio,
 	.swap_activate		= zonefs_swap_activate,
 };
 
diff --git a/include/acpi/acpi_bus.h b/include/acpi/acpi_bus.h
index afeed6e72049..ec6a673dcb95 100644
--- a/include/acpi/acpi_bus.h
+++ b/include/acpi/acpi_bus.h
@@ -366,6 +366,98 @@ struct acpi_device_data {
 
 struct acpi_gpio_mapping;
 
+#define ACPI_DEVICE_SWNODE_ROOT			0
+
+/*
+ * The maximum expected number of CSI-2 data lanes.
+ *
+ * This number is not expected to ever have to be equal to or greater than the
+ * number of bits in an unsigned long variable, but if it needs to be increased
+ * above that limit, code will need to be adjusted accordingly.
+ */
+#define ACPI_DEVICE_CSI2_DATA_LANES		8
+
+#define ACPI_DEVICE_SWNODE_PORT_NAME_LENGTH	8
+
+enum acpi_device_swnode_dev_props {
+	ACPI_DEVICE_SWNODE_DEV_ROTATION,
+	ACPI_DEVICE_SWNODE_DEV_CLOCK_FREQUENCY,
+	ACPI_DEVICE_SWNODE_DEV_LED_MAX_MICROAMP,
+	ACPI_DEVICE_SWNODE_DEV_FLASH_MAX_MICROAMP,
+	ACPI_DEVICE_SWNODE_DEV_FLASH_MAX_TIMEOUT_US,
+	ACPI_DEVICE_SWNODE_DEV_NUM_OF,
+	ACPI_DEVICE_SWNODE_DEV_NUM_ENTRIES
+};
+
+enum acpi_device_swnode_port_props {
+	ACPI_DEVICE_SWNODE_PORT_REG,
+	ACPI_DEVICE_SWNODE_PORT_NUM_OF,
+	ACPI_DEVICE_SWNODE_PORT_NUM_ENTRIES
+};
+
+enum acpi_device_swnode_ep_props {
+	ACPI_DEVICE_SWNODE_EP_REMOTE_EP,
+	ACPI_DEVICE_SWNODE_EP_BUS_TYPE,
+	ACPI_DEVICE_SWNODE_EP_REG,
+	ACPI_DEVICE_SWNODE_EP_CLOCK_LANES,
+	ACPI_DEVICE_SWNODE_EP_DATA_LANES,
+	ACPI_DEVICE_SWNODE_EP_LANE_POLARITIES,
+	/* TX only */
+	ACPI_DEVICE_SWNODE_EP_LINK_FREQUENCIES,
+	ACPI_DEVICE_SWNODE_EP_NUM_OF,
+	ACPI_DEVICE_SWNODE_EP_NUM_ENTRIES
+};
+
+/*
+ * Each device has a root software node plus two times as many nodes as the
+ * number of CSI-2 ports.
+ */
+#define ACPI_DEVICE_SWNODE_PORT(port)	(2 * (port) + 1)
+#define ACPI_DEVICE_SWNODE_EP(endpoint)	\
+		(ACPI_DEVICE_SWNODE_PORT(endpoint) + 1)
+
+/**
+ * struct acpi_device_software_node_port - MIPI DisCo for Imaging CSI-2 port
+ * @port_name: Port name.
+ * @data_lanes: "data-lanes" property values.
+ * @lane_polarities: "lane-polarities" property values.
+ * @link_frequencies: "link_frequencies" property values.
+ * @port_nr: Port number.
+ * @crs_crs2_local: _CRS CSI2 record present (i.e. this is a transmitter one).
+ * @port_props: Port properties.
+ * @ep_props: Endpoint properties.
+ * @remote_ep: Reference to the remote endpoint.
+ */
+struct acpi_device_software_node_port {
+	char port_name[ACPI_DEVICE_SWNODE_PORT_NAME_LENGTH + 1];
+	u32 data_lanes[ACPI_DEVICE_CSI2_DATA_LANES];
+	u32 lane_polarities[ACPI_DEVICE_CSI2_DATA_LANES + 1 /* clock lane */];
+	u64 link_frequencies[ACPI_DEVICE_CSI2_DATA_LANES];
+	unsigned int port_nr;
+	bool crs_csi2_local;
+
+	struct property_entry port_props[ACPI_DEVICE_SWNODE_PORT_NUM_ENTRIES];
+	struct property_entry ep_props[ACPI_DEVICE_SWNODE_EP_NUM_ENTRIES];
+
+	struct software_node_ref_args remote_ep[1];
+};
+
+/**
+ * struct acpi_device_software_nodes - Software nodes for an ACPI device
+ * @dev_props: Device properties.
+ * @nodes: Software nodes for root as well as ports and endpoints.
+ * @nodeprts: Array of software node pointers, for (un)registering them.
+ * @ports: Information related to each port and endpoint within a port.
+ * @num_ports: The number of ports.
+ */
+struct acpi_device_software_nodes {
+	struct property_entry dev_props[ACPI_DEVICE_SWNODE_DEV_NUM_ENTRIES];
+	struct software_node *nodes;
+	const struct software_node **nodeptrs;
+	struct acpi_device_software_node_port *ports;
+	unsigned int num_ports;
+};
+
 /* Device */
 struct acpi_device {
 	u32 pld_crc;
@@ -384,6 +476,7 @@ struct acpi_device {
 	struct acpi_device_data data;
 	struct acpi_scan_handler *handler;
 	struct acpi_hotplug_context *hp;
+	struct acpi_device_software_nodes *swnodes;
 	const struct acpi_gpio_mapping *driver_gpios;
 	void *driver_data;
 	struct device dev;
@@ -542,6 +635,7 @@ int acpi_device_set_power(struct acpi_device *device, int state);
 int acpi_bus_init_power(struct acpi_device *device);
 int acpi_device_fix_up_power(struct acpi_device *device);
 void acpi_device_fix_up_power_extended(struct acpi_device *adev);
+void acpi_device_fix_up_power_children(struct acpi_device *adev);
 int acpi_bus_update_power(acpi_handle handle, int *state_p);
 int acpi_device_update_power(struct acpi_device *device, int *state_p);
 bool acpi_bus_power_manageable(acpi_handle handle);
diff --git a/include/acpi/video.h b/include/acpi/video.h
index 4230392b5b0b..3d538d4178ab 100644
--- a/include/acpi/video.h
+++ b/include/acpi/video.h
@@ -75,6 +75,15 @@ static inline enum acpi_backlight_type acpi_video_get_backlight_type(void)
 	return __acpi_video_get_backlight_type(false, NULL);
 }
 
+/*
+ * This function MUST only be called by GPU drivers to check if the driver
+ * should register a backlight class device. This function not only checks
+ * if a GPU native backlight device should be registered it *also* tells
+ * the ACPI video-detect code that native GPU backlight control is available.
+ * Therefor calling this from any place other then the GPU driver is wrong!
+ * To check if GPU native backlight control is used in other places instead use:
+ *   if (acpi_video_get_backlight_type() == acpi_backlight_native) { ... }
+ */
 static inline bool acpi_video_backlight_use_native(void)
 {
 	return __acpi_video_get_backlight_type(true, NULL) == acpi_backlight_native;
diff --git a/include/drm/display/drm_dp.h b/include/drm/display/drm_dp.h
index e69cece404b3..83d2039c018b 100644
--- a/include/drm/display/drm_dp.h
+++ b/include/drm/display/drm_dp.h
@@ -148,6 +148,7 @@
 #define DP_RECEIVE_PORT_0_CAP_0		    0x008
 # define DP_LOCAL_EDID_PRESENT		    (1 << 1)
 # define DP_ASSOCIATED_TO_PRECEDING_PORT    (1 << 2)
+# define DP_HBLANK_EXPANSION_CAPABLE        (1 << 3)
 
 #define DP_RECEIVE_PORT_0_BUFFER_SIZE	    0x009
 
@@ -543,6 +544,10 @@
 /* DFP Capability Extension */
 #define DP_DFP_CAPABILITY_EXTENSION_SUPPORT	0x0a3	/* 2.0 */
 
+#define DP_PANEL_REPLAY_CAP                 0x0b0  /* DP 2.0 */
+# define DP_PANEL_REPLAY_SUPPORT            (1 << 0)
+# define DP_PANEL_REPLAY_SU_SUPPORT         (1 << 1)
+
 /* Link Configuration */
 #define	DP_LINK_BW_SET		            0x100
 # define DP_LINK_RATE_TABLE		    0x00    /* eDP 1.4 */
@@ -699,6 +704,7 @@
 
 #define DP_DSC_ENABLE                       0x160   /* DP 1.4 */
 # define DP_DECOMPRESSION_EN                (1 << 0)
+# define DP_DSC_PASSTHROUGH_EN		    (1 << 1)
 #define DP_DSC_CONFIGURATION				0x161	/* DP 2.0 */
 
 #define DP_PSR_EN_CFG				0x170   /* XXX 1.2? */
@@ -716,6 +722,13 @@
 #define DP_BRANCH_DEVICE_CTRL		    0x1a1
 # define DP_BRANCH_DEVICE_IRQ_HPD	    (1 << 0)
 
+#define PANEL_REPLAY_CONFIG                             0x1b0  /* DP 2.0 */
+# define DP_PANEL_REPLAY_ENABLE                         (1 << 0)
+# define DP_PANEL_REPLAY_UNRECOVERABLE_ERROR_EN         (1 << 3)
+# define DP_PANEL_REPLAY_RFB_STORAGE_ERROR_EN           (1 << 4)
+# define DP_PANEL_REPLAY_ACTIVE_FRAME_CRC_ERROR_EN      (1 << 5)
+# define DP_PANEL_REPLAY_SU_ENABLE                      (1 << 6)
+
 #define DP_PAYLOAD_ALLOCATE_SET		    0x1c0
 #define DP_PAYLOAD_ALLOCATE_START_TIME_SLOT 0x1c1
 #define DP_PAYLOAD_ALLOCATE_TIME_SLOT_COUNT 0x1c2
@@ -1105,6 +1118,18 @@
 #define DP_LANE_ALIGN_STATUS_UPDATED_ESI       0x200e /* status same as 0x204 */
 #define DP_SINK_STATUS_ESI                     0x200f /* status same as 0x205 */
 
+#define DP_PANEL_REPLAY_ERROR_STATUS                   0x2020  /* DP 2.1*/
+# define DP_PANEL_REPLAY_LINK_CRC_ERROR                (1 << 0)
+# define DP_PANEL_REPLAY_RFB_STORAGE_ERROR             (1 << 1)
+# define DP_PANEL_REPLAY_VSC_SDP_UNCORRECTABLE_ERROR   (1 << 2)
+
+#define DP_SINK_DEVICE_PR_AND_FRAME_LOCK_STATUS        0x2022  /* DP 2.1 */
+# define DP_SINK_DEVICE_PANEL_REPLAY_STATUS_MASK       (7 << 0)
+# define DP_SINK_FRAME_LOCKED_SHIFT                    3
+# define DP_SINK_FRAME_LOCKED_MASK                     (3 << 3)
+# define DP_SINK_FRAME_LOCKED_STATUS_VALID_SHIFT       5
+# define DP_SINK_FRAME_LOCKED_STATUS_VALID_MASK        (1 << 5)
+
 /* Extended Receiver Capability: See DP_DPCD_REV for definitions */
 #define DP_DP13_DPCD_REV                    0x2200
 
diff --git a/include/drm/display/drm_dp_helper.h b/include/drm/display/drm_dp_helper.h
index 3d74b2cec72f..863b2e7add29 100644
--- a/include/drm/display/drm_dp_helper.h
+++ b/include/drm/display/drm_dp_helper.h
@@ -164,6 +164,7 @@ drm_dp_is_branch(const u8 dpcd[DP_RECEIVER_CAP_SIZE])
 }
 
 /* DP/eDP DSC support */
+u8 drm_dp_dsc_sink_bpp_incr(const u8 dsc_dpcd[DP_DSC_RECEIVER_CAP_SIZE]);
 u8 drm_dp_dsc_sink_max_slice_count(const u8 dsc_dpcd[DP_DSC_RECEIVER_CAP_SIZE],
 				   bool is_edp);
 u8 drm_dp_dsc_sink_line_buf_depth(const u8 dsc_dpcd[DP_DSC_RECEIVER_CAP_SIZE]);
@@ -251,6 +252,19 @@ drm_edp_backlight_supported(const u8 edp_dpcd[EDP_DISPLAY_CTL_CAP_SIZE])
 	return !!(edp_dpcd[1] & DP_EDP_TCON_BACKLIGHT_ADJUSTMENT_CAP);
 }
 
+/**
+ * drm_dp_is_uhbr_rate - Determine if a link rate is UHBR
+ * @link_rate: link rate in 10kbits/s units
+ *
+ * Determine if the provided link rate is an UHBR rate.
+ *
+ * Returns: %True if @link_rate is an UHBR rate.
+ */
+static inline bool drm_dp_is_uhbr_rate(int link_rate)
+{
+	return link_rate >= 1000000;
+}
+
 /*
  * DisplayPort AUX channel
  */
@@ -632,6 +646,13 @@ enum drm_dp_quirk {
 	 * the DP_MAX_LINK_RATE register reporting a lower max multiplier.
 	 */
 	DP_DPCD_QUIRK_CAN_DO_MAX_LINK_RATE_3_24_GBPS,
+	/**
+	 * @DP_DPCD_QUIRK_HBLANK_EXPANSION_REQUIRES_DSC:
+	 *
+	 * The device applies HBLANK expansion for some modes, but this
+	 * requires enabling DSC.
+	 */
+	DP_DPCD_QUIRK_HBLANK_EXPANSION_REQUIRES_DSC,
 };
 
 /**
@@ -781,4 +802,15 @@ bool drm_dp_downstream_rgb_to_ycbcr_conversion(const u8 dpcd[DP_RECEIVER_CAP_SIZ
 					       const u8 port_cap[4], u8 color_spc);
 int drm_dp_pcon_convert_rgb_to_ycbcr(struct drm_dp_aux *aux, u8 color_spc);
 
+#define DRM_DP_BW_OVERHEAD_MST		BIT(0)
+#define DRM_DP_BW_OVERHEAD_UHBR		BIT(1)
+#define DRM_DP_BW_OVERHEAD_SSC_REF_CLK	BIT(2)
+#define DRM_DP_BW_OVERHEAD_FEC		BIT(3)
+#define DRM_DP_BW_OVERHEAD_DSC		BIT(4)
+
+int drm_dp_bw_overhead(int lane_count, int hactive,
+		       int dsc_slice_count,
+		       int bpp_x16, unsigned long flags);
+int drm_dp_bw_channel_coding_efficiency(bool is_uhbr);
+
 #endif /* _DRM_DP_HELPER_H_ */
diff --git a/include/drm/display/drm_dp_mst_helper.h b/include/drm/display/drm_dp_mst_helper.h
index 4429d3b1745b..9b19d8bd520a 100644
--- a/include/drm/display/drm_dp_mst_helper.h
+++ b/include/drm/display/drm_dp_mst_helper.h
@@ -25,6 +25,7 @@
 #include <linux/types.h>
 #include <drm/display/drm_dp_helper.h>
 #include <drm/drm_atomic.h>
+#include <drm/drm_fixed.h>
 
 #if IS_ENABLED(CONFIG_DRM_DEBUG_DP_MST_TOPOLOGY_REFS)
 #include <linux/stackdepot.h>
@@ -617,7 +618,7 @@ struct drm_dp_mst_topology_state {
 	 * @pbn_div: The current PBN divisor for this topology. The driver is expected to fill this
 	 * out itself.
 	 */
-	int pbn_div;
+	fixed20_12 pbn_div;
 };
 
 #define to_dp_mst_topology_mgr(x) container_of(x, struct drm_dp_mst_topology_mgr, base)
@@ -839,10 +840,10 @@ struct edid *drm_dp_mst_get_edid(struct drm_connector *connector,
 				 struct drm_dp_mst_topology_mgr *mgr,
 				 struct drm_dp_mst_port *port);
 
-int drm_dp_get_vc_payload_bw(const struct drm_dp_mst_topology_mgr *mgr,
-			     int link_rate, int link_lane_count);
+fixed20_12 drm_dp_get_vc_payload_bw(const struct drm_dp_mst_topology_mgr *mgr,
+				    int link_rate, int link_lane_count);
 
-int drm_dp_calc_pbn_mode(int clock, int bpp, bool dsc);
+int drm_dp_calc_pbn_mode(int clock, int bpp);
 
 void drm_dp_mst_update_slots(struct drm_dp_mst_topology_state *mst_state, uint8_t link_encoding_cap);
 
@@ -892,6 +893,9 @@ drm_atomic_get_new_mst_topology_state(struct drm_atomic_state *state,
 struct drm_dp_mst_atomic_payload *
 drm_atomic_get_mst_payload_state(struct drm_dp_mst_topology_state *state,
 				 struct drm_dp_mst_port *port);
+bool drm_dp_mst_port_downstream_of_parent(struct drm_dp_mst_topology_mgr *mgr,
+					  struct drm_dp_mst_port *port,
+					  struct drm_dp_mst_port *parent);
 int __must_check
 drm_dp_atomic_find_time_slots(struct drm_atomic_state *state,
 			      struct drm_dp_mst_topology_mgr *mgr,
@@ -913,6 +917,10 @@ int drm_dp_send_power_updown_phy(struct drm_dp_mst_topology_mgr *mgr,
 int drm_dp_send_query_stream_enc_status(struct drm_dp_mst_topology_mgr *mgr,
 		struct drm_dp_mst_port *port,
 		struct drm_dp_query_stream_enc_status_ack_reply *status);
+int __must_check drm_dp_mst_atomic_check_mgr(struct drm_atomic_state *state,
+					     struct drm_dp_mst_topology_mgr *mgr,
+					     struct drm_dp_mst_topology_state *mst_state,
+					     struct drm_dp_mst_port **failing_port);
 int __must_check drm_dp_mst_atomic_check(struct drm_atomic_state *state);
 int __must_check drm_dp_mst_root_conn_atomic_check(struct drm_connector_state *new_conn_state,
 						   struct drm_dp_mst_topology_mgr *mgr);
diff --git a/include/drm/drm_edid.h b/include/drm/drm_edid.h
index 882d2638708e..518d1b8106c7 100644
--- a/include/drm/drm_edid.h
+++ b/include/drm/drm_edid.h
@@ -269,64 +269,6 @@ struct detailed_timing {
 #define DRM_EDID_DSC_MAX_SLICES			0xf
 #define DRM_EDID_DSC_TOTAL_CHUNK_KBYTES		0x3f
 
-/* ELD Header Block */
-#define DRM_ELD_HEADER_BLOCK_SIZE	4
-
-#define DRM_ELD_VER			0
-# define DRM_ELD_VER_SHIFT		3
-# define DRM_ELD_VER_MASK		(0x1f << 3)
-# define DRM_ELD_VER_CEA861D		(2 << 3) /* supports 861D or below */
-# define DRM_ELD_VER_CANNED		(0x1f << 3)
-
-#define DRM_ELD_BASELINE_ELD_LEN	2	/* in dwords! */
-
-/* ELD Baseline Block for ELD_Ver == 2 */
-#define DRM_ELD_CEA_EDID_VER_MNL	4
-# define DRM_ELD_CEA_EDID_VER_SHIFT	5
-# define DRM_ELD_CEA_EDID_VER_MASK	(7 << 5)
-# define DRM_ELD_CEA_EDID_VER_NONE	(0 << 5)
-# define DRM_ELD_CEA_EDID_VER_CEA861	(1 << 5)
-# define DRM_ELD_CEA_EDID_VER_CEA861A	(2 << 5)
-# define DRM_ELD_CEA_EDID_VER_CEA861BCD	(3 << 5)
-# define DRM_ELD_MNL_SHIFT		0
-# define DRM_ELD_MNL_MASK		(0x1f << 0)
-
-#define DRM_ELD_SAD_COUNT_CONN_TYPE	5
-# define DRM_ELD_SAD_COUNT_SHIFT	4
-# define DRM_ELD_SAD_COUNT_MASK		(0xf << 4)
-# define DRM_ELD_CONN_TYPE_SHIFT	2
-# define DRM_ELD_CONN_TYPE_MASK		(3 << 2)
-# define DRM_ELD_CONN_TYPE_HDMI		(0 << 2)
-# define DRM_ELD_CONN_TYPE_DP		(1 << 2)
-# define DRM_ELD_SUPPORTS_AI		(1 << 1)
-# define DRM_ELD_SUPPORTS_HDCP		(1 << 0)
-
-#define DRM_ELD_AUD_SYNCH_DELAY		6	/* in units of 2 ms */
-# define DRM_ELD_AUD_SYNCH_DELAY_MAX	0xfa	/* 500 ms */
-
-#define DRM_ELD_SPEAKER			7
-# define DRM_ELD_SPEAKER_MASK		0x7f
-# define DRM_ELD_SPEAKER_RLRC		(1 << 6)
-# define DRM_ELD_SPEAKER_FLRC		(1 << 5)
-# define DRM_ELD_SPEAKER_RC		(1 << 4)
-# define DRM_ELD_SPEAKER_RLR		(1 << 3)
-# define DRM_ELD_SPEAKER_FC		(1 << 2)
-# define DRM_ELD_SPEAKER_LFE		(1 << 1)
-# define DRM_ELD_SPEAKER_FLR		(1 << 0)
-
-#define DRM_ELD_PORT_ID			8	/* offsets 8..15 inclusive */
-# define DRM_ELD_PORT_ID_LEN		8
-
-#define DRM_ELD_MANUFACTURER_NAME0	16
-#define DRM_ELD_MANUFACTURER_NAME1	17
-
-#define DRM_ELD_PRODUCT_CODE0		18
-#define DRM_ELD_PRODUCT_CODE1		19
-
-#define DRM_ELD_MONITOR_NAME_STRING	20	/* offsets 20..(20+mnl-1) inclusive */
-
-#define DRM_ELD_CEA_SAD(mnl, sad)	(20 + (mnl) + 3 * (sad))
-
 struct edid {
 	u8 header[8];
 	/* Vendor & product info */
@@ -387,11 +329,6 @@ int drm_edid_to_speaker_allocation(const struct edid *edid, u8 **sadb);
 int drm_av_sync_delay(struct drm_connector *connector,
 		      const struct drm_display_mode *mode);
 
-#ifdef CONFIG_DRM_LOAD_EDID_FIRMWARE
-int __drm_set_edid_firmware_path(const char *path);
-int __drm_get_edid_firmware_path(char *buf, size_t bufsize);
-#endif
-
 bool drm_edid_are_equal(const struct edid *edid1, const struct edid *edid2);
 
 int
@@ -409,96 +346,6 @@ drm_hdmi_avi_infoframe_quant_range(struct hdmi_avi_infoframe *frame,
 				   const struct drm_display_mode *mode,
 				   enum hdmi_quantization_range rgb_quant_range);
 
-/**
- * drm_eld_mnl - Get ELD monitor name length in bytes.
- * @eld: pointer to an eld memory structure with mnl set
- */
-static inline int drm_eld_mnl(const uint8_t *eld)
-{
-	return (eld[DRM_ELD_CEA_EDID_VER_MNL] & DRM_ELD_MNL_MASK) >> DRM_ELD_MNL_SHIFT;
-}
-
-/**
- * drm_eld_sad - Get ELD SAD structures.
- * @eld: pointer to an eld memory structure with sad_count set
- */
-static inline const uint8_t *drm_eld_sad(const uint8_t *eld)
-{
-	unsigned int ver, mnl;
-
-	ver = (eld[DRM_ELD_VER] & DRM_ELD_VER_MASK) >> DRM_ELD_VER_SHIFT;
-	if (ver != 2 && ver != 31)
-		return NULL;
-
-	mnl = drm_eld_mnl(eld);
-	if (mnl > 16)
-		return NULL;
-
-	return eld + DRM_ELD_CEA_SAD(mnl, 0);
-}
-
-/**
- * drm_eld_sad_count - Get ELD SAD count.
- * @eld: pointer to an eld memory structure with sad_count set
- */
-static inline int drm_eld_sad_count(const uint8_t *eld)
-{
-	return (eld[DRM_ELD_SAD_COUNT_CONN_TYPE] & DRM_ELD_SAD_COUNT_MASK) >>
-		DRM_ELD_SAD_COUNT_SHIFT;
-}
-
-/**
- * drm_eld_calc_baseline_block_size - Calculate baseline block size in bytes
- * @eld: pointer to an eld memory structure with mnl and sad_count set
- *
- * This is a helper for determining the payload size of the baseline block, in
- * bytes, for e.g. setting the Baseline_ELD_Len field in the ELD header block.
- */
-static inline int drm_eld_calc_baseline_block_size(const uint8_t *eld)
-{
-	return DRM_ELD_MONITOR_NAME_STRING - DRM_ELD_HEADER_BLOCK_SIZE +
-		drm_eld_mnl(eld) + drm_eld_sad_count(eld) * 3;
-}
-
-/**
- * drm_eld_size - Get ELD size in bytes
- * @eld: pointer to a complete eld memory structure
- *
- * The returned value does not include the vendor block. It's vendor specific,
- * and comprises of the remaining bytes in the ELD memory buffer after
- * drm_eld_size() bytes of header and baseline block.
- *
- * The returned value is guaranteed to be a multiple of 4.
- */
-static inline int drm_eld_size(const uint8_t *eld)
-{
-	return DRM_ELD_HEADER_BLOCK_SIZE + eld[DRM_ELD_BASELINE_ELD_LEN] * 4;
-}
-
-/**
- * drm_eld_get_spk_alloc - Get speaker allocation
- * @eld: pointer to an ELD memory structure
- *
- * The returned value is the speakers mask. User has to use %DRM_ELD_SPEAKER
- * field definitions to identify speakers.
- */
-static inline u8 drm_eld_get_spk_alloc(const uint8_t *eld)
-{
-	return eld[DRM_ELD_SPEAKER] & DRM_ELD_SPEAKER_MASK;
-}
-
-/**
- * drm_eld_get_conn_type - Get device type hdmi/dp connected
- * @eld: pointer to an ELD memory structure
- *
- * The caller need to use %DRM_ELD_CONN_TYPE_HDMI or %DRM_ELD_CONN_TYPE_DP to
- * identify the display type connected.
- */
-static inline u8 drm_eld_get_conn_type(const uint8_t *eld)
-{
-	return eld[DRM_ELD_SAD_COUNT_CONN_TYPE] & DRM_ELD_CONN_TYPE_MASK;
-}
-
 /**
  * drm_edid_decode_mfg_id - Decode the manufacturer ID
  * @mfg_id: The manufacturer ID
diff --git a/include/drm/drm_eld.h b/include/drm/drm_eld.h
new file mode 100644
index 000000000000..0a88d10b28b0
--- /dev/null
+++ b/include/drm/drm_eld.h
@@ -0,0 +1,164 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2023 Intel Corporation
+ */
+
+#ifndef __DRM_ELD_H__
+#define __DRM_ELD_H__
+
+#include <linux/types.h>
+
+struct cea_sad;
+
+/* ELD Header Block */
+#define DRM_ELD_HEADER_BLOCK_SIZE	4
+
+#define DRM_ELD_VER			0
+# define DRM_ELD_VER_SHIFT		3
+# define DRM_ELD_VER_MASK		(0x1f << 3)
+# define DRM_ELD_VER_CEA861D		(2 << 3) /* supports 861D or below */
+# define DRM_ELD_VER_CANNED		(0x1f << 3)
+
+#define DRM_ELD_BASELINE_ELD_LEN	2	/* in dwords! */
+
+/* ELD Baseline Block for ELD_Ver == 2 */
+#define DRM_ELD_CEA_EDID_VER_MNL	4
+# define DRM_ELD_CEA_EDID_VER_SHIFT	5
+# define DRM_ELD_CEA_EDID_VER_MASK	(7 << 5)
+# define DRM_ELD_CEA_EDID_VER_NONE	(0 << 5)
+# define DRM_ELD_CEA_EDID_VER_CEA861	(1 << 5)
+# define DRM_ELD_CEA_EDID_VER_CEA861A	(2 << 5)
+# define DRM_ELD_CEA_EDID_VER_CEA861BCD	(3 << 5)
+# define DRM_ELD_MNL_SHIFT		0
+# define DRM_ELD_MNL_MASK		(0x1f << 0)
+
+#define DRM_ELD_SAD_COUNT_CONN_TYPE	5
+# define DRM_ELD_SAD_COUNT_SHIFT	4
+# define DRM_ELD_SAD_COUNT_MASK		(0xf << 4)
+# define DRM_ELD_CONN_TYPE_SHIFT	2
+# define DRM_ELD_CONN_TYPE_MASK		(3 << 2)
+# define DRM_ELD_CONN_TYPE_HDMI		(0 << 2)
+# define DRM_ELD_CONN_TYPE_DP		(1 << 2)
+# define DRM_ELD_SUPPORTS_AI		(1 << 1)
+# define DRM_ELD_SUPPORTS_HDCP		(1 << 0)
+
+#define DRM_ELD_AUD_SYNCH_DELAY		6	/* in units of 2 ms */
+# define DRM_ELD_AUD_SYNCH_DELAY_MAX	0xfa	/* 500 ms */
+
+#define DRM_ELD_SPEAKER			7
+# define DRM_ELD_SPEAKER_MASK		0x7f
+# define DRM_ELD_SPEAKER_RLRC		(1 << 6)
+# define DRM_ELD_SPEAKER_FLRC		(1 << 5)
+# define DRM_ELD_SPEAKER_RC		(1 << 4)
+# define DRM_ELD_SPEAKER_RLR		(1 << 3)
+# define DRM_ELD_SPEAKER_FC		(1 << 2)
+# define DRM_ELD_SPEAKER_LFE		(1 << 1)
+# define DRM_ELD_SPEAKER_FLR		(1 << 0)
+
+#define DRM_ELD_PORT_ID			8	/* offsets 8..15 inclusive */
+# define DRM_ELD_PORT_ID_LEN		8
+
+#define DRM_ELD_MANUFACTURER_NAME0	16
+#define DRM_ELD_MANUFACTURER_NAME1	17
+
+#define DRM_ELD_PRODUCT_CODE0		18
+#define DRM_ELD_PRODUCT_CODE1		19
+
+#define DRM_ELD_MONITOR_NAME_STRING	20	/* offsets 20..(20+mnl-1) inclusive */
+
+#define DRM_ELD_CEA_SAD(mnl, sad)	(20 + (mnl) + 3 * (sad))
+
+/**
+ * drm_eld_mnl - Get ELD monitor name length in bytes.
+ * @eld: pointer to an eld memory structure with mnl set
+ */
+static inline int drm_eld_mnl(const u8 *eld)
+{
+	return (eld[DRM_ELD_CEA_EDID_VER_MNL] & DRM_ELD_MNL_MASK) >> DRM_ELD_MNL_SHIFT;
+}
+
+int drm_eld_sad_get(const u8 *eld, int sad_index, struct cea_sad *cta_sad);
+int drm_eld_sad_set(u8 *eld, int sad_index, const struct cea_sad *cta_sad);
+
+/**
+ * drm_eld_sad - Get ELD SAD structures.
+ * @eld: pointer to an eld memory structure with sad_count set
+ */
+static inline const u8 *drm_eld_sad(const u8 *eld)
+{
+	unsigned int ver, mnl;
+
+	ver = (eld[DRM_ELD_VER] & DRM_ELD_VER_MASK) >> DRM_ELD_VER_SHIFT;
+	if (ver != 2 && ver != 31)
+		return NULL;
+
+	mnl = drm_eld_mnl(eld);
+	if (mnl > 16)
+		return NULL;
+
+	return eld + DRM_ELD_CEA_SAD(mnl, 0);
+}
+
+/**
+ * drm_eld_sad_count - Get ELD SAD count.
+ * @eld: pointer to an eld memory structure with sad_count set
+ */
+static inline int drm_eld_sad_count(const u8 *eld)
+{
+	return (eld[DRM_ELD_SAD_COUNT_CONN_TYPE] & DRM_ELD_SAD_COUNT_MASK) >>
+		DRM_ELD_SAD_COUNT_SHIFT;
+}
+
+/**
+ * drm_eld_calc_baseline_block_size - Calculate baseline block size in bytes
+ * @eld: pointer to an eld memory structure with mnl and sad_count set
+ *
+ * This is a helper for determining the payload size of the baseline block, in
+ * bytes, for e.g. setting the Baseline_ELD_Len field in the ELD header block.
+ */
+static inline int drm_eld_calc_baseline_block_size(const u8 *eld)
+{
+	return DRM_ELD_MONITOR_NAME_STRING - DRM_ELD_HEADER_BLOCK_SIZE +
+		drm_eld_mnl(eld) + drm_eld_sad_count(eld) * 3;
+}
+
+/**
+ * drm_eld_size - Get ELD size in bytes
+ * @eld: pointer to a complete eld memory structure
+ *
+ * The returned value does not include the vendor block. It's vendor specific,
+ * and comprises of the remaining bytes in the ELD memory buffer after
+ * drm_eld_size() bytes of header and baseline block.
+ *
+ * The returned value is guaranteed to be a multiple of 4.
+ */
+static inline int drm_eld_size(const u8 *eld)
+{
+	return DRM_ELD_HEADER_BLOCK_SIZE + eld[DRM_ELD_BASELINE_ELD_LEN] * 4;
+}
+
+/**
+ * drm_eld_get_spk_alloc - Get speaker allocation
+ * @eld: pointer to an ELD memory structure
+ *
+ * The returned value is the speakers mask. User has to use %DRM_ELD_SPEAKER
+ * field definitions to identify speakers.
+ */
+static inline u8 drm_eld_get_spk_alloc(const u8 *eld)
+{
+	return eld[DRM_ELD_SPEAKER] & DRM_ELD_SPEAKER_MASK;
+}
+
+/**
+ * drm_eld_get_conn_type - Get device type hdmi/dp connected
+ * @eld: pointer to an ELD memory structure
+ *
+ * The caller need to use %DRM_ELD_CONN_TYPE_HDMI or %DRM_ELD_CONN_TYPE_DP to
+ * identify the display type connected.
+ */
+static inline u8 drm_eld_get_conn_type(const u8 *eld)
+{
+	return eld[DRM_ELD_SAD_COUNT_CONN_TYPE] & DRM_ELD_CONN_TYPE_MASK;
+}
+
+#endif /* __DRM_ELD_H__ */
diff --git a/include/drm/drm_flip_work.h b/include/drm/drm_flip_work.h
index 21c3d512d25c..1eef3283a109 100644
--- a/include/drm/drm_flip_work.h
+++ b/include/drm/drm_flip_work.h
@@ -31,11 +31,10 @@
 /**
  * DOC: flip utils
  *
- * Util to queue up work to run from work-queue context after flip/vblank.
+ * Utility to queue up work to run from work-queue context after flip/vblank.
  * Typically this can be used to defer unref of framebuffer's, cursor
- * bo's, etc until after vblank.  The APIs are all thread-safe.
- * Moreover, drm_flip_work_queue_task and drm_flip_work_queue can be called
- * in atomic context.
+ * bo's, etc until after vblank. The APIs are all thread-safe. Moreover,
+ * drm_flip_work_commit() can be called in atomic context.
  */
 
 struct drm_flip_work;
@@ -51,16 +50,6 @@ struct drm_flip_work;
  */
 typedef void (*drm_flip_func_t)(struct drm_flip_work *work, void *val);
 
-/**
- * struct drm_flip_task - flip work task
- * @node: list entry element
- * @data: data to pass to &drm_flip_work.func
- */
-struct drm_flip_task {
-	struct list_head node;
-	void *data;
-};
-
 /**
  * struct drm_flip_work - flip work queue
  * @name: debug name
@@ -79,9 +68,6 @@ struct drm_flip_work {
 	spinlock_t lock;
 };
 
-struct drm_flip_task *drm_flip_work_allocate_task(void *data, gfp_t flags);
-void drm_flip_work_queue_task(struct drm_flip_work *work,
-			      struct drm_flip_task *task);
 void drm_flip_work_queue(struct drm_flip_work *work, void *val);
 void drm_flip_work_commit(struct drm_flip_work *work,
 		struct workqueue_struct *wq);
diff --git a/include/drm/drm_format_helper.h b/include/drm/drm_format_helper.h
index 291deb09475b..f13b34e0b752 100644
--- a/include/drm/drm_format_helper.h
+++ b/include/drm/drm_format_helper.h
@@ -15,6 +15,57 @@ struct drm_rect;
 
 struct iosys_map;
 
+/**
+ * struct drm_format_conv_state - Stores format-conversion state
+ *
+ * DRM helpers for format conversion store temporary state in
+ * struct drm_xfrm_buf. The buffer's resources can be reused
+ * among multiple conversion operations.
+ *
+ * All fields are considered private.
+ */
+struct drm_format_conv_state {
+	struct {
+		void *mem;
+		size_t size;
+		bool preallocated;
+	} tmp;
+};
+
+#define __DRM_FORMAT_CONV_STATE_INIT(_mem, _size, _preallocated) { \
+		.tmp = { \
+			.mem = (_mem), \
+			.size = (_size), \
+			.preallocated = (_preallocated), \
+		} \
+	}
+
+/**
+ * DRM_FORMAT_CONV_STATE_INIT - Initializer for struct drm_format_conv_state
+ *
+ * Initializes an instance of struct drm_format_conv_state to default values.
+ */
+#define DRM_FORMAT_CONV_STATE_INIT \
+	__DRM_FORMAT_CONV_STATE_INIT(NULL, 0, false)
+
+/**
+ * DRM_FORMAT_CONV_STATE_INIT_PREALLOCATED - Initializer for struct drm_format_conv_state
+ * @_mem: The preallocated memory area
+ * @_size: The number of bytes in _mem
+ *
+ * Initializes an instance of struct drm_format_conv_state to preallocated
+ * storage. The caller is responsible for releasing the provided memory range.
+ */
+#define DRM_FORMAT_CONV_STATE_INIT_PREALLOCATED(_mem, _size) \
+	__DRM_FORMAT_CONV_STATE_INIT(_mem, _size, true)
+
+void drm_format_conv_state_init(struct drm_format_conv_state *state);
+void drm_format_conv_state_copy(struct drm_format_conv_state *state,
+				const struct drm_format_conv_state *old_state);
+void *drm_format_conv_state_reserve(struct drm_format_conv_state *state,
+				    size_t new_size, gfp_t flags);
+void drm_format_conv_state_release(struct drm_format_conv_state *state);
+
 unsigned int drm_fb_clip_offset(unsigned int pitch, const struct drm_format_info *format,
 				const struct drm_rect *clip);
 
@@ -23,45 +74,49 @@ void drm_fb_memcpy(struct iosys_map *dst, const unsigned int *dst_pitch,
 		   const struct drm_rect *clip);
 void drm_fb_swab(struct iosys_map *dst, const unsigned int *dst_pitch,
 		 const struct iosys_map *src, const struct drm_framebuffer *fb,
-		 const struct drm_rect *clip, bool cached);
+		 const struct drm_rect *clip, bool cached,
+		 struct drm_format_conv_state *state);
 void drm_fb_xrgb8888_to_rgb332(struct iosys_map *dst, const unsigned int *dst_pitch,
 			       const struct iosys_map *src, const struct drm_framebuffer *fb,
-			       const struct drm_rect *clip);
+			       const struct drm_rect *clip, struct drm_format_conv_state *state);
 void drm_fb_xrgb8888_to_rgb565(struct iosys_map *dst, const unsigned int *dst_pitch,
 			       const struct iosys_map *src, const struct drm_framebuffer *fb,
-			       const struct drm_rect *clip, bool swab);
+			       const struct drm_rect *clip, struct drm_format_conv_state *state,
+			       bool swab);
 void drm_fb_xrgb8888_to_xrgb1555(struct iosys_map *dst, const unsigned int *dst_pitch,
 				 const struct iosys_map *src, const struct drm_framebuffer *fb,
-				 const struct drm_rect *clip);
+				 const struct drm_rect *clip, struct drm_format_conv_state *state);
 void drm_fb_xrgb8888_to_argb1555(struct iosys_map *dst, const unsigned int *dst_pitch,
 				 const struct iosys_map *src, const struct drm_framebuffer *fb,
-				 const struct drm_rect *clip);
+				 const struct drm_rect *clip, struct drm_format_conv_state *state);
 void drm_fb_xrgb8888_to_rgba5551(struct iosys_map *dst, const unsigned int *dst_pitch,
 				 const struct iosys_map *src, const struct drm_framebuffer *fb,
-				 const struct drm_rect *clip);
+				 const struct drm_rect *clip, struct drm_format_conv_state *state);
 void drm_fb_xrgb8888_to_rgb888(struct iosys_map *dst, const unsigned int *dst_pitch,
 			       const struct iosys_map *src, const struct drm_framebuffer *fb,
-			       const struct drm_rect *clip);
+			       const struct drm_rect *clip, struct drm_format_conv_state *state);
 void drm_fb_xrgb8888_to_argb8888(struct iosys_map *dst, const unsigned int *dst_pitch,
 				 const struct iosys_map *src, const struct drm_framebuffer *fb,
-				 const struct drm_rect *clip);
+				 const struct drm_rect *clip, struct drm_format_conv_state *state);
 void drm_fb_xrgb8888_to_xrgb2101010(struct iosys_map *dst, const unsigned int *dst_pitch,
 				    const struct iosys_map *src, const struct drm_framebuffer *fb,
-				    const struct drm_rect *clip);
+				    const struct drm_rect *clip,
+				    struct drm_format_conv_state *state);
 void drm_fb_xrgb8888_to_argb2101010(struct iosys_map *dst, const unsigned int *dst_pitch,
 				    const struct iosys_map *src, const struct drm_framebuffer *fb,
-				    const struct drm_rect *clip);
+				    const struct drm_rect *clip,
+				    struct drm_format_conv_state *state);
 void drm_fb_xrgb8888_to_gray8(struct iosys_map *dst, const unsigned int *dst_pitch,
 			      const struct iosys_map *src, const struct drm_framebuffer *fb,
-			      const struct drm_rect *clip);
+			      const struct drm_rect *clip, struct drm_format_conv_state *state);
 
 int drm_fb_blit(struct iosys_map *dst, const unsigned int *dst_pitch, uint32_t dst_format,
 		const struct iosys_map *src, const struct drm_framebuffer *fb,
-		const struct drm_rect *rect);
+		const struct drm_rect *clip, struct drm_format_conv_state *state);
 
 void drm_fb_xrgb8888_to_mono(struct iosys_map *dst, const unsigned int *dst_pitch,
 			     const struct iosys_map *src, const struct drm_framebuffer *fb,
-			     const struct drm_rect *clip);
+			     const struct drm_rect *clip, struct drm_format_conv_state *state);
 
 size_t drm_fb_build_fourcc_list(struct drm_device *dev,
 				const u32 *native_fourccs, size_t native_nfourccs,
diff --git a/include/drm/drm_gem.h b/include/drm/drm_gem.h
index 16364487fde9..369505447acd 100644
--- a/include/drm/drm_gem.h
+++ b/include/drm/drm_gem.h
@@ -580,7 +580,7 @@ int drm_gem_evict(struct drm_gem_object *obj);
  * drm_gem_gpuva_init() - initialize the gpuva list of a GEM object
  * @obj: the &drm_gem_object
  *
- * This initializes the &drm_gem_object's &drm_gpuva list.
+ * This initializes the &drm_gem_object's &drm_gpuvm_bo list.
  *
  * Calling this function is only necessary for drivers intending to support the
  * &drm_driver_feature DRIVER_GEM_GPUVA.
@@ -593,28 +593,28 @@ static inline void drm_gem_gpuva_init(struct drm_gem_object *obj)
 }
 
 /**
- * drm_gem_for_each_gpuva() - iternator to walk over a list of gpuvas
- * @entry__: &drm_gpuva structure to assign to in each iteration step
- * @obj__: the &drm_gem_object the &drm_gpuvas to walk are associated with
+ * drm_gem_for_each_gpuvm_bo() - iterator to walk over a list of &drm_gpuvm_bo
+ * @entry__: &drm_gpuvm_bo structure to assign to in each iteration step
+ * @obj__: the &drm_gem_object the &drm_gpuvm_bo to walk are associated with
  *
- * This iterator walks over all &drm_gpuva structures associated with the
- * &drm_gpuva_manager.
+ * This iterator walks over all &drm_gpuvm_bo structures associated with the
+ * &drm_gem_object.
  */
-#define drm_gem_for_each_gpuva(entry__, obj__) \
-	list_for_each_entry(entry__, &(obj__)->gpuva.list, gem.entry)
+#define drm_gem_for_each_gpuvm_bo(entry__, obj__) \
+	list_for_each_entry(entry__, &(obj__)->gpuva.list, list.entry.gem)
 
 /**
- * drm_gem_for_each_gpuva_safe() - iternator to safely walk over a list of
- * gpuvas
- * @entry__: &drm_gpuva structure to assign to in each iteration step
- * @next__: &next &drm_gpuva to store the next step
- * @obj__: the &drm_gem_object the &drm_gpuvas to walk are associated with
+ * drm_gem_for_each_gpuvm_bo_safe() - iterator to safely walk over a list of
+ * &drm_gpuvm_bo
+ * @entry__: &drm_gpuvm_bostructure to assign to in each iteration step
+ * @next__: &next &drm_gpuvm_bo to store the next step
+ * @obj__: the &drm_gem_object the &drm_gpuvm_bo to walk are associated with
  *
- * This iterator walks over all &drm_gpuva structures associated with the
+ * This iterator walks over all &drm_gpuvm_bo structures associated with the
  * &drm_gem_object. It is implemented with list_for_each_entry_safe(), hence
  * it is save against removal of elements.
  */
-#define drm_gem_for_each_gpuva_safe(entry__, next__, obj__) \
-	list_for_each_entry_safe(entry__, next__, &(obj__)->gpuva.list, gem.entry)
+#define drm_gem_for_each_gpuvm_bo_safe(entry__, next__, obj__) \
+	list_for_each_entry_safe(entry__, next__, &(obj__)->gpuva.list, list.entry.gem)
 
 #endif /* __DRM_GEM_H__ */
diff --git a/include/drm/drm_gem_atomic_helper.h b/include/drm/drm_gem_atomic_helper.h
index 40b8b039518e..3e01c619a25e 100644
--- a/include/drm/drm_gem_atomic_helper.h
+++ b/include/drm/drm_gem_atomic_helper.h
@@ -5,6 +5,7 @@
 
 #include <linux/iosys-map.h>
 
+#include <drm/drm_format_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_plane.h>
 
@@ -49,6 +50,15 @@ struct drm_shadow_plane_state {
 	/** @base: plane state */
 	struct drm_plane_state base;
 
+	/**
+	 * @fmtcnv_state: Format-conversion state
+	 *
+	 * Per-plane state for format conversion.
+	 * Flags for copying shadow buffers into backend storage. Also holds
+	 * temporary storage for format conversion.
+	 */
+	struct drm_format_conv_state fmtcnv_state;
+
 	/* Transitional state - do not export or duplicate */
 
 	/**
diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h
index bdfafc4a7705..8ca10461d8ac 100644
--- a/include/drm/drm_gpuvm.h
+++ b/include/drm/drm_gpuvm.h
@@ -25,13 +25,17 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#include <linux/dma-resv.h>
 #include <linux/list.h>
 #include <linux/rbtree.h>
 #include <linux/types.h>
 
+#include <drm/drm_device.h>
 #include <drm/drm_gem.h>
+#include <drm/drm_exec.h>
 
 struct drm_gpuvm;
+struct drm_gpuvm_bo;
 struct drm_gpuvm_ops;
 
 /**
@@ -72,6 +76,12 @@ struct drm_gpuva {
 	 */
 	struct drm_gpuvm *vm;
 
+	/**
+	 * @vm_bo: the &drm_gpuvm_bo abstraction for the mapped
+	 * &drm_gem_object
+	 */
+	struct drm_gpuvm_bo *vm_bo;
+
 	/**
 	 * @flags: the &drm_gpuva_flags for this mapping
 	 */
@@ -107,7 +117,7 @@ struct drm_gpuva {
 		struct drm_gem_object *obj;
 
 		/**
-		 * @entry: the &list_head to attach this object to a &drm_gem_object
+		 * @entry: the &list_head to attach this object to a &drm_gpuvm_bo
 		 */
 		struct list_head entry;
 	} gem;
@@ -140,7 +150,7 @@ struct drm_gpuva {
 int drm_gpuva_insert(struct drm_gpuvm *gpuvm, struct drm_gpuva *va);
 void drm_gpuva_remove(struct drm_gpuva *va);
 
-void drm_gpuva_link(struct drm_gpuva *va);
+void drm_gpuva_link(struct drm_gpuva *va, struct drm_gpuvm_bo *vm_bo);
 void drm_gpuva_unlink(struct drm_gpuva *va);
 
 struct drm_gpuva *drm_gpuva_find(struct drm_gpuvm *gpuvm,
@@ -183,6 +193,22 @@ static inline bool drm_gpuva_invalidated(struct drm_gpuva *va)
 	return va->flags & DRM_GPUVA_INVALIDATED;
 }
 
+/**
+ * enum drm_gpuvm_flags - flags for struct drm_gpuvm
+ */
+enum drm_gpuvm_flags {
+	/**
+	 * @DRM_GPUVM_RESV_PROTECTED: GPUVM is protected externally by the
+	 * GPUVM's &dma_resv lock
+	 */
+	DRM_GPUVM_RESV_PROTECTED = BIT(0),
+
+	/**
+	 * @DRM_GPUVM_USERBITS: user defined bits
+	 */
+	DRM_GPUVM_USERBITS = BIT(1),
+};
+
 /**
  * struct drm_gpuvm - DRM GPU VA Manager
  *
@@ -201,6 +227,16 @@ struct drm_gpuvm {
 	 */
 	const char *name;
 
+	/**
+	 * @flags: the &drm_gpuvm_flags of this GPUVM
+	 */
+	enum drm_gpuvm_flags flags;
+
+	/**
+	 * @drm: the &drm_device this VM lives in
+	 */
+	struct drm_device *drm;
+
 	/**
 	 * @mm_start: start of the VA space
 	 */
@@ -226,6 +262,11 @@ struct drm_gpuvm {
 		struct list_head list;
 	} rb;
 
+	/**
+	 * @kref: reference count of this object
+	 */
+	struct kref kref;
+
 	/**
 	 * @kernel_alloc_node:
 	 *
@@ -238,16 +279,147 @@ struct drm_gpuvm {
 	 * @ops: &drm_gpuvm_ops providing the split/merge steps to drivers
 	 */
 	const struct drm_gpuvm_ops *ops;
+
+	/**
+	 * @r_obj: Resv GEM object; representing the GPUVM's common &dma_resv.
+	 */
+	struct drm_gem_object *r_obj;
+
+	/**
+	 * @extobj: structure holding the extobj list
+	 */
+	struct {
+		/**
+		 * @list: &list_head storing &drm_gpuvm_bos serving as
+		 * external object
+		 */
+		struct list_head list;
+
+		/**
+		 * @local_list: pointer to the local list temporarily storing
+		 * entries from the external object list
+		 */
+		struct list_head *local_list;
+
+		/**
+		 * @lock: spinlock to protect the extobj list
+		 */
+		spinlock_t lock;
+	} extobj;
+
+	/**
+	 * @evict: structure holding the evict list and evict list lock
+	 */
+	struct {
+		/**
+		 * @list: &list_head storing &drm_gpuvm_bos currently being
+		 * evicted
+		 */
+		struct list_head list;
+
+		/**
+		 * @local_list: pointer to the local list temporarily storing
+		 * entries from the evicted object list
+		 */
+		struct list_head *local_list;
+
+		/**
+		 * @lock: spinlock to protect the evict list
+		 */
+		spinlock_t lock;
+	} evict;
 };
 
 void drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name,
+		    enum drm_gpuvm_flags flags,
+		    struct drm_device *drm,
+		    struct drm_gem_object *r_obj,
 		    u64 start_offset, u64 range,
 		    u64 reserve_offset, u64 reserve_range,
 		    const struct drm_gpuvm_ops *ops);
-void drm_gpuvm_destroy(struct drm_gpuvm *gpuvm);
 
+/**
+ * drm_gpuvm_get() - acquire a struct drm_gpuvm reference
+ * @gpuvm: the &drm_gpuvm to acquire the reference of
+ *
+ * This function acquires an additional reference to @gpuvm. It is illegal to
+ * call this without already holding a reference. No locks required.
+ */
+static inline struct drm_gpuvm *
+drm_gpuvm_get(struct drm_gpuvm *gpuvm)
+{
+	kref_get(&gpuvm->kref);
+
+	return gpuvm;
+}
+
+void drm_gpuvm_put(struct drm_gpuvm *gpuvm);
+
+bool drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm, u64 addr, u64 range);
 bool drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range);
 
+struct drm_gem_object *
+drm_gpuvm_resv_object_alloc(struct drm_device *drm);
+
+/**
+ * drm_gpuvm_resv_protected() - indicates whether &DRM_GPUVM_RESV_PROTECTED is
+ * set
+ * @gpuvm: the &drm_gpuvm
+ *
+ * Returns: true if &DRM_GPUVM_RESV_PROTECTED is set, false otherwise.
+ */
+static inline bool
+drm_gpuvm_resv_protected(struct drm_gpuvm *gpuvm)
+{
+	return gpuvm->flags & DRM_GPUVM_RESV_PROTECTED;
+}
+
+/**
+ * drm_gpuvm_resv() - returns the &drm_gpuvm's &dma_resv
+ * @gpuvm__: the &drm_gpuvm
+ *
+ * Returns: a pointer to the &drm_gpuvm's shared &dma_resv
+ */
+#define drm_gpuvm_resv(gpuvm__) ((gpuvm__)->r_obj->resv)
+
+/**
+ * drm_gpuvm_resv_obj() - returns the &drm_gem_object holding the &drm_gpuvm's
+ * &dma_resv
+ * @gpuvm__: the &drm_gpuvm
+ *
+ * Returns: a pointer to the &drm_gem_object holding the &drm_gpuvm's shared
+ * &dma_resv
+ */
+#define drm_gpuvm_resv_obj(gpuvm__) ((gpuvm__)->r_obj)
+
+#define drm_gpuvm_resv_held(gpuvm__) \
+	dma_resv_held(drm_gpuvm_resv(gpuvm__))
+
+#define drm_gpuvm_resv_assert_held(gpuvm__) \
+	dma_resv_assert_held(drm_gpuvm_resv(gpuvm__))
+
+#define drm_gpuvm_resv_held(gpuvm__) \
+	dma_resv_held(drm_gpuvm_resv(gpuvm__))
+
+#define drm_gpuvm_resv_assert_held(gpuvm__) \
+	dma_resv_assert_held(drm_gpuvm_resv(gpuvm__))
+
+/**
+ * drm_gpuvm_is_extobj() - indicates whether the given &drm_gem_object is an
+ * external object
+ * @gpuvm: the &drm_gpuvm to check
+ * @obj: the &drm_gem_object to check
+ *
+ * Returns: true if the &drm_gem_object &dma_resv differs from the
+ * &drm_gpuvms &dma_resv, false otherwise
+ */
+static inline bool
+drm_gpuvm_is_extobj(struct drm_gpuvm *gpuvm,
+		    struct drm_gem_object *obj)
+{
+	return obj && obj->resv != drm_gpuvm_resv(gpuvm);
+}
+
 static inline struct drm_gpuva *
 __drm_gpuva_next(struct drm_gpuva *va)
 {
@@ -326,6 +498,302 @@ __drm_gpuva_next(struct drm_gpuva *va)
 #define drm_gpuvm_for_each_va_safe(va__, next__, gpuvm__) \
 	list_for_each_entry_safe(va__, next__, &(gpuvm__)->rb.list, rb.entry)
 
+/**
+ * struct drm_gpuvm_exec - &drm_gpuvm abstraction of &drm_exec
+ *
+ * This structure should be created on the stack as &drm_exec should be.
+ *
+ * Optionally, @extra can be set in order to lock additional &drm_gem_objects.
+ */
+struct drm_gpuvm_exec {
+	/**
+	 * @exec: the &drm_exec structure
+	 */
+	struct drm_exec exec;
+
+	/**
+	 * @flags: the flags for the struct drm_exec
+	 */
+	uint32_t flags;
+
+	/**
+	 * @vm: the &drm_gpuvm to lock its DMA reservations
+	 */
+	struct drm_gpuvm *vm;
+
+	/**
+	 * @num_fences: the number of fences to reserve for the &dma_resv of the
+	 * locked &drm_gem_objects
+	 */
+	unsigned int num_fences;
+
+	/**
+	 * @extra: Callback and corresponding private data for the driver to
+	 * lock arbitrary additional &drm_gem_objects.
+	 */
+	struct {
+		/**
+		 * @fn: The driver callback to lock additional &drm_gem_objects.
+		 */
+		int (*fn)(struct drm_gpuvm_exec *vm_exec);
+
+		/**
+		 * @priv: driver private data for the @fn callback
+		 */
+		void *priv;
+	} extra;
+};
+
+/**
+ * drm_gpuvm_prepare_vm() - prepare the GPUVMs common dma-resv
+ * @gpuvm: the &drm_gpuvm
+ * @exec: the &drm_exec context
+ * @num_fences: the amount of &dma_fences to reserve
+ *
+ * Calls drm_exec_prepare_obj() for the GPUVMs dummy &drm_gem_object.
+ *
+ * Using this function directly, it is the drivers responsibility to call
+ * drm_exec_init() and drm_exec_fini() accordingly.
+ *
+ * Returns: 0 on success, negative error code on failure.
+ */
+static inline int
+drm_gpuvm_prepare_vm(struct drm_gpuvm *gpuvm,
+		     struct drm_exec *exec,
+		     unsigned int num_fences)
+{
+	return drm_exec_prepare_obj(exec, gpuvm->r_obj, num_fences);
+}
+
+int drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm,
+			      struct drm_exec *exec,
+			      unsigned int num_fences);
+
+int drm_gpuvm_prepare_range(struct drm_gpuvm *gpuvm,
+			    struct drm_exec *exec,
+			    u64 addr, u64 range,
+			    unsigned int num_fences);
+
+int drm_gpuvm_exec_lock(struct drm_gpuvm_exec *vm_exec);
+
+int drm_gpuvm_exec_lock_array(struct drm_gpuvm_exec *vm_exec,
+			      struct drm_gem_object **objs,
+			      unsigned int num_objs);
+
+int drm_gpuvm_exec_lock_range(struct drm_gpuvm_exec *vm_exec,
+			      u64 addr, u64 range);
+
+/**
+ * drm_gpuvm_exec_unlock() - lock all dma-resv of all assoiciated BOs
+ * @vm_exec: the &drm_gpuvm_exec wrapper
+ *
+ * Releases all dma-resv locks of all &drm_gem_objects previously acquired
+ * through drm_gpuvm_exec_lock() or its variants.
+ *
+ * Returns: 0 on success, negative error code on failure.
+ */
+static inline void
+drm_gpuvm_exec_unlock(struct drm_gpuvm_exec *vm_exec)
+{
+	drm_exec_fini(&vm_exec->exec);
+}
+
+int drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec);
+void drm_gpuvm_resv_add_fence(struct drm_gpuvm *gpuvm,
+			      struct drm_exec *exec,
+			      struct dma_fence *fence,
+			      enum dma_resv_usage private_usage,
+			      enum dma_resv_usage extobj_usage);
+
+/**
+ * drm_gpuvm_exec_resv_add_fence()
+ * @vm_exec: the &drm_gpuvm_exec wrapper
+ * @fence: fence to add
+ * @private_usage: private dma-resv usage
+ * @extobj_usage: extobj dma-resv usage
+ *
+ * See drm_gpuvm_resv_add_fence().
+ */
+static inline void
+drm_gpuvm_exec_resv_add_fence(struct drm_gpuvm_exec *vm_exec,
+			      struct dma_fence *fence,
+			      enum dma_resv_usage private_usage,
+			      enum dma_resv_usage extobj_usage)
+{
+	drm_gpuvm_resv_add_fence(vm_exec->vm, &vm_exec->exec, fence,
+				 private_usage, extobj_usage);
+}
+
+/**
+ * drm_gpuvm_exec_validate()
+ * @vm_exec: the &drm_gpuvm_exec wrapper
+ *
+ * See drm_gpuvm_validate().
+ */
+static inline int
+drm_gpuvm_exec_validate(struct drm_gpuvm_exec *vm_exec)
+{
+	return drm_gpuvm_validate(vm_exec->vm, &vm_exec->exec);
+}
+
+/**
+ * struct drm_gpuvm_bo - structure representing a &drm_gpuvm and
+ * &drm_gem_object combination
+ *
+ * This structure is an abstraction representing a &drm_gpuvm and
+ * &drm_gem_object combination. It serves as an indirection to accelerate
+ * iterating all &drm_gpuvas within a &drm_gpuvm backed by the same
+ * &drm_gem_object.
+ *
+ * Furthermore it is used cache evicted GEM objects for a certain GPU-VM to
+ * accelerate validation.
+ *
+ * Typically, drivers want to create an instance of a struct drm_gpuvm_bo once
+ * a GEM object is mapped first in a GPU-VM and release the instance once the
+ * last mapping of the GEM object in this GPU-VM is unmapped.
+ */
+struct drm_gpuvm_bo {
+	/**
+	 * @vm: The &drm_gpuvm the @obj is mapped in. This is a reference
+	 * counted pointer.
+	 */
+	struct drm_gpuvm *vm;
+
+	/**
+	 * @obj: The &drm_gem_object being mapped in @vm. This is a reference
+	 * counted pointer.
+	 */
+	struct drm_gem_object *obj;
+
+	/**
+	 * @evicted: Indicates whether the &drm_gem_object is evicted; field
+	 * protected by the &drm_gem_object's dma-resv lock.
+	 */
+	bool evicted;
+
+	/**
+	 * @kref: The reference count for this &drm_gpuvm_bo.
+	 */
+	struct kref kref;
+
+	/**
+	 * @list: Structure containing all &list_heads.
+	 */
+	struct {
+		/**
+		 * @gpuva: The list of linked &drm_gpuvas.
+		 *
+		 * It is safe to access entries from this list as long as the
+		 * GEM's gpuva lock is held. See also struct drm_gem_object.
+		 */
+		struct list_head gpuva;
+
+		/**
+		 * @entry: Structure containing all &list_heads serving as
+		 * entry.
+		 */
+		struct {
+			/**
+			 * @gem: List entry to attach to the &drm_gem_objects
+			 * gpuva list.
+			 */
+			struct list_head gem;
+
+			/**
+			 * @evict: List entry to attach to the &drm_gpuvms
+			 * extobj list.
+			 */
+			struct list_head extobj;
+
+			/**
+			 * @evict: List entry to attach to the &drm_gpuvms evict
+			 * list.
+			 */
+			struct list_head evict;
+		} entry;
+	} list;
+};
+
+struct drm_gpuvm_bo *
+drm_gpuvm_bo_create(struct drm_gpuvm *gpuvm,
+		    struct drm_gem_object *obj);
+
+struct drm_gpuvm_bo *
+drm_gpuvm_bo_obtain(struct drm_gpuvm *gpuvm,
+		    struct drm_gem_object *obj);
+struct drm_gpuvm_bo *
+drm_gpuvm_bo_obtain_prealloc(struct drm_gpuvm_bo *vm_bo);
+
+/**
+ * drm_gpuvm_bo_get() - acquire a struct drm_gpuvm_bo reference
+ * @vm_bo: the &drm_gpuvm_bo to acquire the reference of
+ *
+ * This function acquires an additional reference to @vm_bo. It is illegal to
+ * call this without already holding a reference. No locks required.
+ */
+static inline struct drm_gpuvm_bo *
+drm_gpuvm_bo_get(struct drm_gpuvm_bo *vm_bo)
+{
+	kref_get(&vm_bo->kref);
+	return vm_bo;
+}
+
+void drm_gpuvm_bo_put(struct drm_gpuvm_bo *vm_bo);
+
+struct drm_gpuvm_bo *
+drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm,
+		  struct drm_gem_object *obj);
+
+void drm_gpuvm_bo_evict(struct drm_gpuvm_bo *vm_bo, bool evict);
+
+/**
+ * drm_gpuvm_bo_gem_evict()
+ * @obj: the &drm_gem_object
+ * @evict: indicates whether @obj is evicted
+ *
+ * See drm_gpuvm_bo_evict().
+ */
+static inline void
+drm_gpuvm_bo_gem_evict(struct drm_gem_object *obj, bool evict)
+{
+	struct drm_gpuvm_bo *vm_bo;
+
+	drm_gem_gpuva_assert_lock_held(obj);
+	drm_gem_for_each_gpuvm_bo(vm_bo, obj)
+		drm_gpuvm_bo_evict(vm_bo, evict);
+}
+
+void drm_gpuvm_bo_extobj_add(struct drm_gpuvm_bo *vm_bo);
+
+/**
+ * drm_gpuvm_bo_for_each_va() - iterator to walk over a list of &drm_gpuva
+ * @va__: &drm_gpuva structure to assign to in each iteration step
+ * @vm_bo__: the &drm_gpuvm_bo the &drm_gpuva to walk are associated with
+ *
+ * This iterator walks over all &drm_gpuva structures associated with the
+ * &drm_gpuvm_bo.
+ *
+ * The caller must hold the GEM's gpuva lock.
+ */
+#define drm_gpuvm_bo_for_each_va(va__, vm_bo__) \
+	list_for_each_entry(va__, &(vm_bo)->list.gpuva, gem.entry)
+
+/**
+ * drm_gpuvm_bo_for_each_va_safe() - iterator to safely walk over a list of
+ * &drm_gpuva
+ * @va__: &drm_gpuva structure to assign to in each iteration step
+ * @next__: &next &drm_gpuva to store the next step
+ * @vm_bo__: the &drm_gpuvm_bo the &drm_gpuva to walk are associated with
+ *
+ * This iterator walks over all &drm_gpuva structures associated with the
+ * &drm_gpuvm_bo. It is implemented with list_for_each_entry_safe(), hence
+ * it is save against removal of elements.
+ *
+ * The caller must hold the GEM's gpuva lock.
+ */
+#define drm_gpuvm_bo_for_each_va_safe(va__, next__, vm_bo__) \
+	list_for_each_entry_safe(va__, next__, &(vm_bo)->list.gpuva, gem.entry)
+
 /**
  * enum drm_gpuva_op_type - GPU VA operation type
  *
@@ -595,8 +1063,7 @@ drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm,
 				 u64 addr, u64 range);
 
 struct drm_gpuva_ops *
-drm_gpuvm_gem_unmap_ops_create(struct drm_gpuvm *gpuvm,
-			       struct drm_gem_object *obj);
+drm_gpuvm_bo_unmap_ops_create(struct drm_gpuvm_bo *vm_bo);
 
 void drm_gpuva_ops_free(struct drm_gpuvm *gpuvm,
 			struct drm_gpuva_ops *ops);
@@ -616,6 +1083,14 @@ static inline void drm_gpuva_init_from_op(struct drm_gpuva *va,
  * operations to drivers.
  */
 struct drm_gpuvm_ops {
+	/**
+	 * @vm_free: called when the last reference of a struct drm_gpuvm is
+	 * dropped
+	 *
+	 * This callback is mandatory.
+	 */
+	void (*vm_free)(struct drm_gpuvm *gpuvm);
+
 	/**
 	 * @op_alloc: called when the &drm_gpuvm allocates
 	 * a struct drm_gpuva_op
@@ -640,6 +1115,42 @@ struct drm_gpuvm_ops {
 	 */
 	void (*op_free)(struct drm_gpuva_op *op);
 
+	/**
+	 * @vm_bo_alloc: called when the &drm_gpuvm allocates
+	 * a struct drm_gpuvm_bo
+	 *
+	 * Some drivers may want to embed struct drm_gpuvm_bo into driver
+	 * specific structures. By implementing this callback drivers can
+	 * allocate memory accordingly.
+	 *
+	 * This callback is optional.
+	 */
+	struct drm_gpuvm_bo *(*vm_bo_alloc)(void);
+
+	/**
+	 * @vm_bo_free: called when the &drm_gpuvm frees a
+	 * struct drm_gpuvm_bo
+	 *
+	 * Some drivers may want to embed struct drm_gpuvm_bo into driver
+	 * specific structures. By implementing this callback drivers can
+	 * free the previously allocated memory accordingly.
+	 *
+	 * This callback is optional.
+	 */
+	void (*vm_bo_free)(struct drm_gpuvm_bo *vm_bo);
+
+	/**
+	 * @vm_bo_validate: called from drm_gpuvm_validate()
+	 *
+	 * Drivers receive this callback for every evicted &drm_gem_object being
+	 * mapped in the corresponding &drm_gpuvm.
+	 *
+	 * Typically, drivers would call their driver specific variant of
+	 * ttm_bo_validate() from within this callback.
+	 */
+	int (*vm_bo_validate)(struct drm_gpuvm_bo *vm_bo,
+			      struct drm_exec *exec);
+
 	/**
 	 * @sm_step_map: called from &drm_gpuvm_sm_map to finally insert the
 	 * mapping once all previous steps were completed
diff --git a/include/drm/drm_mipi_dbi.h b/include/drm/drm_mipi_dbi.h
index 816f196b3d4c..e8e0f8d39f3a 100644
--- a/include/drm/drm_mipi_dbi.h
+++ b/include/drm/drm_mipi_dbi.h
@@ -12,6 +12,7 @@
 #include <drm/drm_device.h>
 #include <drm/drm_simple_kms_helper.h>
 
+struct drm_format_conv_state;
 struct drm_rect;
 struct gpio_desc;
 struct iosys_map;
@@ -192,7 +193,8 @@ int mipi_dbi_command_buf(struct mipi_dbi *dbi, u8 cmd, u8 *data, size_t len);
 int mipi_dbi_command_stackbuf(struct mipi_dbi *dbi, u8 cmd, const u8 *data,
 			      size_t len);
 int mipi_dbi_buf_copy(void *dst, struct iosys_map *src, struct drm_framebuffer *fb,
-		      struct drm_rect *clip, bool swap);
+		      struct drm_rect *clip, bool swap,
+		      struct drm_format_conv_state *fmtcnv_state);
 
 /**
  * mipi_dbi_command - MIPI DCS command with optional parameter(s)
diff --git a/include/drm/drm_print.h b/include/drm/drm_print.h
index a93a387f8a1a..dd4883df876a 100644
--- a/include/drm/drm_print.h
+++ b/include/drm/drm_print.h
@@ -453,7 +453,7 @@ void __drm_dev_dbg(struct _ddebug *desc, const struct device *dev,
 
 /* Helper for struct drm_device based logging. */
 #define __drm_printk(drm, level, type, fmt, ...)			\
-	dev_##level##type((drm)->dev, "[drm] " fmt, ##__VA_ARGS__)
+	dev_##level##type((drm) ? (drm)->dev : NULL, "[drm] " fmt, ##__VA_ARGS__)
 
 
 #define drm_info(drm, fmt, ...)					\
diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
index d2fb81e34174..9a50348bd5c0 100644
--- a/include/drm/gpu_scheduler.h
+++ b/include/drm/gpu_scheduler.h
@@ -320,6 +320,7 @@ struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f);
  * @sched: the scheduler instance on which this job is scheduled.
  * @s_fence: contains the fences for the scheduling of job.
  * @finish_cb: the callback for the finished fence.
+ * @credits: the number of credits this job contributes to the scheduler
  * @work: Helper to reschdeule job kill to different context.
  * @id: a unique id assigned to each job scheduled on the scheduler.
  * @karma: increment on every hang caused by this job. If this exceeds the hang
@@ -339,6 +340,8 @@ struct drm_sched_job {
 	struct drm_gpu_scheduler	*sched;
 	struct drm_sched_fence		*s_fence;
 
+	u32				credits;
+
 	/*
 	 * work is used only after finish_cb has been used and will not be
 	 * accessed anymore.
@@ -462,29 +465,42 @@ struct drm_sched_backend_ops {
          * and it's time to clean it up.
 	 */
 	void (*free_job)(struct drm_sched_job *sched_job);
+
+	/**
+	 * @update_job_credits: Called when the scheduler is considering this
+	 * job for execution.
+	 *
+	 * This callback returns the number of credits the job would take if
+	 * pushed to the hardware. Drivers may use this to dynamically update
+	 * the job's credit count. For instance, deduct the number of credits
+	 * for already signalled native fences.
+	 *
+	 * This callback is optional.
+	 */
+	u32 (*update_job_credits)(struct drm_sched_job *sched_job);
 };
 
 /**
  * struct drm_gpu_scheduler - scheduler instance-specific data
  *
  * @ops: backend operations provided by the driver.
- * @hw_submission_limit: the max size of the hardware queue.
+ * @credit_limit: the credit limit of this scheduler
+ * @credit_count: the current credit count of this scheduler
  * @timeout: the time after which a job is removed from the scheduler.
  * @name: name of the ring for which this scheduler is being used.
  * @num_rqs: Number of run-queues. This is at most DRM_SCHED_PRIORITY_COUNT,
  *           as there's usually one run-queue per priority, but could be less.
  * @sched_rq: An allocated array of run-queues of size @num_rqs;
- * @wake_up_worker: the wait queue on which the scheduler sleeps until a job
- *                  is ready to be scheduled.
  * @job_scheduled: once @drm_sched_entity_do_release is called the scheduler
  *                 waits on this wait queue until all the scheduled jobs are
  *                 finished.
- * @hw_rq_count: the number of jobs currently in the hardware queue.
  * @job_id_count: used to assign unique id to the each job.
+ * @submit_wq: workqueue used to queue @work_run_job and @work_free_job
  * @timeout_wq: workqueue used to queue @work_tdr
+ * @work_run_job: work which calls run_job op of each scheduler.
+ * @work_free_job: work which calls free_job op of each scheduler.
  * @work_tdr: schedules a delayed call to @drm_sched_job_timedout after the
  *            timeout interval is over.
- * @thread: the kthread on which the scheduler which run.
  * @pending_list: the list of jobs which are currently in the job queue.
  * @job_list_lock: lock to protect the pending_list.
  * @hang_limit: once the hangs by a job crosses this limit then it is marked
@@ -493,24 +509,27 @@ struct drm_sched_backend_ops {
  * @_score: score used when the driver doesn't provide one
  * @ready: marks if the underlying HW is ready to work
  * @free_guilty: A hit to time out handler to free the guilty job.
+ * @pause_submit: pause queuing of @work_run_job on @submit_wq
+ * @own_submit_wq: scheduler owns allocation of @submit_wq
  * @dev: system &struct device
  *
  * One scheduler is implemented for each hardware ring.
  */
 struct drm_gpu_scheduler {
 	const struct drm_sched_backend_ops	*ops;
-	uint32_t			hw_submission_limit;
+	u32				credit_limit;
+	atomic_t			credit_count;
 	long				timeout;
 	const char			*name;
 	u32                             num_rqs;
 	struct drm_sched_rq             **sched_rq;
-	wait_queue_head_t		wake_up_worker;
 	wait_queue_head_t		job_scheduled;
-	atomic_t			hw_rq_count;
 	atomic64_t			job_id_count;
+	struct workqueue_struct		*submit_wq;
 	struct workqueue_struct		*timeout_wq;
+	struct work_struct		work_run_job;
+	struct work_struct		work_free_job;
 	struct delayed_work		work_tdr;
-	struct task_struct		*thread;
 	struct list_head		pending_list;
 	spinlock_t			job_list_lock;
 	int				hang_limit;
@@ -518,19 +537,22 @@ struct drm_gpu_scheduler {
 	atomic_t                        _score;
 	bool				ready;
 	bool				free_guilty;
+	bool				pause_submit;
+	bool				own_submit_wq;
 	struct device			*dev;
 };
 
 int drm_sched_init(struct drm_gpu_scheduler *sched,
 		   const struct drm_sched_backend_ops *ops,
-		   u32 num_rqs, uint32_t hw_submission, unsigned int hang_limit,
+		   struct workqueue_struct *submit_wq,
+		   u32 num_rqs, u32 credit_limit, unsigned int hang_limit,
 		   long timeout, struct workqueue_struct *timeout_wq,
 		   atomic_t *score, const char *name, struct device *dev);
 
 void drm_sched_fini(struct drm_gpu_scheduler *sched);
 int drm_sched_job_init(struct drm_sched_job *job,
 		       struct drm_sched_entity *entity,
-		       void *owner);
+		       u32 credits, void *owner);
 void drm_sched_job_arm(struct drm_sched_job *job);
 int drm_sched_job_add_dependency(struct drm_sched_job *job,
 				 struct dma_fence *fence);
@@ -550,8 +572,12 @@ void drm_sched_entity_modify_sched(struct drm_sched_entity *entity,
 				    struct drm_gpu_scheduler **sched_list,
                                    unsigned int num_sched_list);
 
+void drm_sched_tdr_queue_imm(struct drm_gpu_scheduler *sched);
 void drm_sched_job_cleanup(struct drm_sched_job *job);
-void drm_sched_wakeup_if_can_queue(struct drm_gpu_scheduler *sched);
+void drm_sched_wakeup(struct drm_gpu_scheduler *sched, struct drm_sched_entity *entity);
+bool drm_sched_wqueue_ready(struct drm_gpu_scheduler *sched);
+void drm_sched_wqueue_stop(struct drm_gpu_scheduler *sched);
+void drm_sched_wqueue_start(struct drm_gpu_scheduler *sched);
 void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad);
 void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery);
 void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
diff --git a/include/linux/anon_inodes.h b/include/linux/anon_inodes.h
index 5deaddbd7927..93a5f16d03f3 100644
--- a/include/linux/anon_inodes.h
+++ b/include/linux/anon_inodes.h
@@ -15,13 +15,13 @@ struct inode;
 struct file *anon_inode_getfile(const char *name,
 				const struct file_operations *fops,
 				void *priv, int flags);
-struct file *anon_inode_getfile_secure(const char *name,
+struct file *anon_inode_create_getfile(const char *name,
 				       const struct file_operations *fops,
 				       void *priv, int flags,
 				       const struct inode *context_inode);
 int anon_inode_getfd(const char *name, const struct file_operations *fops,
 		     void *priv, int flags);
-int anon_inode_getfd_secure(const char *name,
+int anon_inode_create_getfd(const char *name,
 			    const struct file_operations *fops,
 			    void *priv, int flags,
 			    const struct inode *context_inode);
diff --git a/include/linux/arm_ffa.h b/include/linux/arm_ffa.h
index 1abedb5b2e48..3d0fde57ba90 100644
--- a/include/linux/arm_ffa.h
+++ b/include/linux/arm_ffa.h
@@ -209,6 +209,8 @@ bool ffa_device_is_valid(struct ffa_device *ffa_dev) { return false; }
 #define module_ffa_driver(__ffa_driver)	\
 	module_driver(__ffa_driver, ffa_register, ffa_unregister)
 
+extern struct bus_type ffa_bus_type;
+
 /* FFA transport related */
 struct ffa_partition_info {
 	u16 id;
diff --git a/include/linux/blk-pm.h b/include/linux/blk-pm.h
index 2580e05a8ab6..004b38a538ff 100644
--- a/include/linux/blk-pm.h
+++ b/include/linux/blk-pm.h
@@ -15,7 +15,6 @@ extern int blk_pre_runtime_suspend(struct request_queue *q);
 extern void blk_post_runtime_suspend(struct request_queue *q, int err);
 extern void blk_pre_runtime_resume(struct request_queue *q);
 extern void blk_post_runtime_resume(struct request_queue *q);
-extern void blk_set_runtime_active(struct request_queue *q);
 #else
 static inline void blk_pm_runtime_init(struct request_queue *q,
 				       struct device *dev) {}
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index d5c5e59ddbd2..52e264d5a830 100644
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -56,19 +56,17 @@ struct block_device {
 	void *			bd_holder;
 	const struct blk_holder_ops *bd_holder_ops;
 	struct mutex		bd_holder_lock;
-	/* The counter of freeze processes */
-	int			bd_fsfreeze_count;
 	int			bd_holders;
 	struct kobject		*bd_holder_dir;
 
-	/* Mutex for freeze */
-	struct mutex		bd_fsfreeze_mutex;
-	struct super_block	*bd_fsfreeze_sb;
+	atomic_t		bd_fsfreeze_count; /* number of freeze requests */
+	struct mutex		bd_fsfreeze_mutex; /* serialize freeze/thaw */
 
 	struct partition_meta_info *bd_meta_info;
 #ifdef CONFIG_FAIL_MAKE_REQUEST
 	bool			bd_make_it_fail;
 #endif
+	int			bd_writers;
 	/*
 	 * keep this out-of-line as it's both big and not needed in the fast
 	 * path
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 51fa7ffdee83..9f6c3373f9fc 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -124,6 +124,8 @@ typedef unsigned int __bitwise blk_mode_t;
 #define BLK_OPEN_NDELAY		((__force blk_mode_t)(1 << 3))
 /* open for "writes" only for ioctls (specialy hack for floppy.c) */
 #define BLK_OPEN_WRITE_IOCTL	((__force blk_mode_t)(1 << 4))
+/* open is exclusive wrt all other BLK_OPEN_WRITE opens to the device */
+#define BLK_OPEN_RESTRICT_WRITES	((__force blk_mode_t)(1 << 5))
 
 struct gendisk {
 	/*
@@ -1468,8 +1470,23 @@ struct blk_holder_ops {
 	 * Sync the file system mounted on the block device.
 	 */
 	void (*sync)(struct block_device *bdev);
+
+	/*
+	 * Freeze the file system mounted on the block device.
+	 */
+	int (*freeze)(struct block_device *bdev);
+
+	/*
+	 * Thaw the file system mounted on the block device.
+	 */
+	int (*thaw)(struct block_device *bdev);
 };
 
+/*
+ * For filesystems using @fs_holder_ops, the @holder argument passed to
+ * helpers used to open and claim block devices via
+ * bd_prepare_to_claim() must point to a superblock.
+ */
 extern const struct blk_holder_ops fs_holder_ops;
 
 /*
@@ -1477,7 +1494,8 @@ extern const struct blk_holder_ops fs_holder_ops;
  * as stored in sb->s_flags.
  */
 #define sb_open_mode(flags) \
-	(BLK_OPEN_READ | (((flags) & SB_RDONLY) ? 0 : BLK_OPEN_WRITE))
+	(BLK_OPEN_READ | BLK_OPEN_RESTRICT_WRITES | \
+	 (((flags) & SB_RDONLY) ? 0 : BLK_OPEN_WRITE))
 
 struct bdev_handle {
 	struct block_device *bdev;
@@ -1485,10 +1503,6 @@ struct bdev_handle {
 	blk_mode_t mode;
 };
 
-struct block_device *blkdev_get_by_dev(dev_t dev, blk_mode_t mode, void *holder,
-		const struct blk_holder_ops *hops);
-struct block_device *blkdev_get_by_path(const char *path, blk_mode_t mode,
-		void *holder, const struct blk_holder_ops *hops);
 struct bdev_handle *bdev_open_by_dev(dev_t dev, blk_mode_t mode, void *holder,
 		const struct blk_holder_ops *hops);
 struct bdev_handle *bdev_open_by_path(const char *path, blk_mode_t mode,
@@ -1496,7 +1510,6 @@ struct bdev_handle *bdev_open_by_path(const char *path, blk_mode_t mode,
 int bd_prepare_to_claim(struct block_device *bdev, void *holder,
 		const struct blk_holder_ops *hops);
 void bd_abort_claiming(struct block_device *bdev, void *holder);
-void blkdev_put(struct block_device *bdev, void *holder);
 void bdev_release(struct bdev_handle *handle);
 
 /* just for blk-cgroup, don't use elsewhere */
@@ -1541,8 +1554,8 @@ static inline int early_lookup_bdev(const char *pathname, dev_t *dev)
 }
 #endif /* CONFIG_BLOCK */
 
-int freeze_bdev(struct block_device *bdev);
-int thaw_bdev(struct block_device *bdev);
+int bdev_freeze(struct block_device *bdev);
+int bdev_thaw(struct block_device *bdev);
 
 struct io_comp_batch {
 	struct request *req_list;
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 6762dac3ef76..258ba232e302 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -186,8 +186,8 @@ enum btf_field_type {
 	BPF_LIST_NODE  = (1 << 6),
 	BPF_RB_ROOT    = (1 << 7),
 	BPF_RB_NODE    = (1 << 8),
-	BPF_GRAPH_NODE_OR_ROOT = BPF_LIST_NODE | BPF_LIST_HEAD |
-				 BPF_RB_NODE | BPF_RB_ROOT,
+	BPF_GRAPH_NODE = BPF_RB_NODE | BPF_LIST_NODE,
+	BPF_GRAPH_ROOT = BPF_RB_ROOT | BPF_LIST_HEAD,
 	BPF_REFCOUNT   = (1 << 9),
 };
 
@@ -1226,6 +1226,8 @@ enum bpf_dynptr_type {
 
 int bpf_dynptr_check_size(u32 size);
 u32 __bpf_dynptr_size(const struct bpf_dynptr_kern *ptr);
+const void *__bpf_dynptr_data(const struct bpf_dynptr_kern *ptr, u32 len);
+void *__bpf_dynptr_data_rw(const struct bpf_dynptr_kern *ptr, u32 len);
 
 #ifdef CONFIG_BPF_JIT
 int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 24213a99cc79..d99a636d36a7 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -301,6 +301,17 @@ struct bpf_func_state {
 	struct tnum callback_ret_range;
 	bool in_async_callback_fn;
 	bool in_exception_callback_fn;
+	/* For callback calling functions that limit number of possible
+	 * callback executions (e.g. bpf_loop) keeps track of current
+	 * simulated iteration number.
+	 * Value in frame N refers to number of times callback with frame
+	 * N+1 was simulated, e.g. for the following call:
+	 *
+	 *   bpf_loop(..., fn, ...); | suppose current frame is N
+	 *                           | fn would be simulated in frame N+1
+	 *                           | number of simulations is tracked in frame N
+	 */
+	u32 callback_depth;
 
 	/* The following fields should be last. See copy_func_state() */
 	int acquired_refs;
@@ -400,6 +411,7 @@ struct bpf_verifier_state {
 	struct bpf_idx_pair *jmp_history;
 	u32 jmp_history_cnt;
 	u32 dfs_depth;
+	u32 callback_unroll_depth;
 };
 
 #define bpf_get_spilled_reg(slot, frame, mask)				\
@@ -511,6 +523,10 @@ struct bpf_insn_aux_data {
 	 * this instruction, regardless of any heuristics
 	 */
 	bool force_checkpoint;
+	/* true if instruction is a call to a helper function that
+	 * accepts callback function as a parameter.
+	 */
+	bool calls_callback;
 };
 
 #define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */
@@ -602,6 +618,7 @@ struct bpf_verifier_env {
 	int stack_size;			/* number of states to be processed */
 	bool strict_alignment;		/* perform strict pointer alignment checks */
 	bool test_state_freq;		/* test verifier with different pruning frequency */
+	bool test_reg_invariants;	/* fail verification on register invariants violations */
 	struct bpf_verifier_state *cur_state; /* current verifier state */
 	struct bpf_verifier_state_list **explored_states; /* search pruning optimization */
 	struct bpf_verifier_state_list *free_list;
@@ -679,6 +696,10 @@ int bpf_vlog_init(struct bpf_verifier_log *log, u32 log_level,
 void bpf_vlog_reset(struct bpf_verifier_log *log, u64 new_pos);
 int bpf_vlog_finalize(struct bpf_verifier_log *log, u32 *log_size_actual);
 
+__printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env,
+				  u32 insn_off,
+				  const char *prefix_fmt, ...);
+
 static inline struct bpf_func_state *cur_func(struct bpf_verifier_env *env)
 {
 	struct bpf_verifier_state *cur = env->cur_state;
@@ -778,4 +799,76 @@ static inline bool bpf_type_has_unsafe_modifiers(u32 type)
 	return type_flag(type) & ~BPF_REG_TRUSTED_MODIFIERS;
 }
 
+static inline bool type_is_ptr_alloc_obj(u32 type)
+{
+	return base_type(type) == PTR_TO_BTF_ID && type_flag(type) & MEM_ALLOC;
+}
+
+static inline bool type_is_non_owning_ref(u32 type)
+{
+	return type_is_ptr_alloc_obj(type) && type_flag(type) & NON_OWN_REF;
+}
+
+static inline bool type_is_pkt_pointer(enum bpf_reg_type type)
+{
+	type = base_type(type);
+	return type == PTR_TO_PACKET ||
+	       type == PTR_TO_PACKET_META;
+}
+
+static inline bool type_is_sk_pointer(enum bpf_reg_type type)
+{
+	return type == PTR_TO_SOCKET ||
+		type == PTR_TO_SOCK_COMMON ||
+		type == PTR_TO_TCP_SOCK ||
+		type == PTR_TO_XDP_SOCK;
+}
+
+static inline void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno)
+{
+	env->scratched_regs |= 1U << regno;
+}
+
+static inline void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi)
+{
+	env->scratched_stack_slots |= 1ULL << spi;
+}
+
+static inline bool reg_scratched(const struct bpf_verifier_env *env, u32 regno)
+{
+	return (env->scratched_regs >> regno) & 1;
+}
+
+static inline bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno)
+{
+	return (env->scratched_stack_slots >> regno) & 1;
+}
+
+static inline bool verifier_state_scratched(const struct bpf_verifier_env *env)
+{
+	return env->scratched_regs || env->scratched_stack_slots;
+}
+
+static inline void mark_verifier_state_clean(struct bpf_verifier_env *env)
+{
+	env->scratched_regs = 0U;
+	env->scratched_stack_slots = 0ULL;
+}
+
+/* Used for printing the entire verifier state. */
+static inline void mark_verifier_state_scratched(struct bpf_verifier_env *env)
+{
+	env->scratched_regs = ~0U;
+	env->scratched_stack_slots = ~0ULL;
+}
+
+const char *reg_type_str(struct bpf_verifier_env *env, enum bpf_reg_type type);
+const char *dynptr_type_str(enum bpf_dynptr_type type);
+const char *iter_type_str(const struct btf *btf, u32 btf_id);
+const char *iter_state_str(enum bpf_iter_state state);
+
+void print_verifier_state(struct bpf_verifier_env *env,
+			  const struct bpf_func_state *state, bool print_all);
+void print_insn_state(struct bpf_verifier_env *env, const struct bpf_func_state *state);
+
 #endif /* _LINUX_BPF_VERIFIER_H */
diff --git a/include/linux/ceph/osd_client.h b/include/linux/ceph/osd_client.h
index b8610e9d2471..5edf9fffa097 100644
--- a/include/linux/ceph/osd_client.h
+++ b/include/linux/ceph/osd_client.h
@@ -45,6 +45,7 @@ enum ceph_sparse_read_state {
 	CEPH_SPARSE_READ_HDR	= 0,
 	CEPH_SPARSE_READ_EXTENTS,
 	CEPH_SPARSE_READ_DATA_LEN,
+	CEPH_SPARSE_READ_DATA_PRE,
 	CEPH_SPARSE_READ_DATA,
 };
 
@@ -64,7 +65,7 @@ struct ceph_sparse_read {
 	u64				sr_req_len;  /* orig request length */
 	u64				sr_pos;      /* current pos in buffer */
 	int				sr_index;    /* current extent index */
-	__le32				sr_datalen;  /* length of actual data */
+	u32				sr_datalen;  /* length of actual data */
 	u32				sr_count;    /* extent count in reply */
 	int				sr_ext_len;  /* length of extent array */
 	struct ceph_sparse_extent	*sr_extent;  /* extent array */
diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h
index 4a6b6b77ccb6..37518436cfe7 100644
--- a/include/linux/cgroup-defs.h
+++ b/include/linux/cgroup-defs.h
@@ -496,6 +496,13 @@ struct cgroup {
 	struct cgroup_rstat_cpu __percpu *rstat_cpu;
 	struct list_head rstat_css_list;
 
+	/*
+	 * A singly-linked list of cgroup structures to be rstat flushed.
+	 * This is a scratch field to be used exclusively by
+	 * cgroup_rstat_flush_locked() and protected by cgroup_rstat_lock.
+	 */
+	struct cgroup	*rstat_flush_next;
+
 	/* cgroup basic resource statistics */
 	struct cgroup_base_stat last_bstat;
 	struct cgroup_base_stat bstat;
@@ -563,6 +570,7 @@ struct cgroup_root {
 
 	/* A list running through the active hierarchies */
 	struct list_head root_list;
+	struct rcu_head rcu;
 
 	/* Hierarchy-specific flags */
 	unsigned int flags;
diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
index 0ef0af66080e..34aaf0e87def 100644
--- a/include/linux/cgroup.h
+++ b/include/linux/cgroup.h
@@ -69,6 +69,7 @@ struct css_task_iter {
 extern struct file_system_type cgroup_fs_type;
 extern struct cgroup_root cgrp_dfl_root;
 extern struct css_set init_css_set;
+extern spinlock_t css_set_lock;
 
 #define SUBSYS(_x) extern struct cgroup_subsys _x ## _cgrp_subsys;
 #include <linux/cgroup_subsys.h>
@@ -386,7 +387,6 @@ static inline void cgroup_unlock(void)
  * as locks used during the cgroup_subsys::attach() methods.
  */
 #ifdef CONFIG_PROVE_RCU
-extern spinlock_t css_set_lock;
 #define task_css_set_check(task, __c)					\
 	rcu_dereference_check((task)->cgroups,				\
 		rcu_read_lock_sched_held() ||				\
@@ -853,4 +853,6 @@ static inline void cgroup_bpf_put(struct cgroup *cgrp) {}
 
 #endif /* CONFIG_CGROUP_BPF */
 
+struct cgroup *task_get_cgroup1(struct task_struct *tsk, int hierarchy_id);
+
 #endif /* _LINUX_CGROUP_H */
diff --git a/include/linux/cleanup.h b/include/linux/cleanup.h
index 9f1a9c455b68..c2d09bc4f976 100644
--- a/include/linux/cleanup.h
+++ b/include/linux/cleanup.h
@@ -125,25 +125,55 @@ static inline class_##_name##_t class_##_name##ext##_constructor(_init_args) \
  *	trivial wrapper around DEFINE_CLASS() above specifically
  *	for locks.
  *
+ * DEFINE_GUARD_COND(name, ext, condlock)
+ *	wrapper around EXTEND_CLASS above to add conditional lock
+ *	variants to a base class, eg. mutex_trylock() or
+ *	mutex_lock_interruptible().
+ *
  * guard(name):
- *	an anonymous instance of the (guard) class
+ *	an anonymous instance of the (guard) class, not recommended for
+ *	conditional locks.
  *
  * scoped_guard (name, args...) { }:
  *	similar to CLASS(name, scope)(args), except the variable (with the
  *	explicit name 'scope') is declard in a for-loop such that its scope is
  *	bound to the next (compound) statement.
  *
+ *	for conditional locks the loop body is skipped when the lock is not
+ *	acquired.
+ *
+ * scoped_cond_guard (name, fail, args...) { }:
+ *      similar to scoped_guard(), except it does fail when the lock
+ *      acquire fails.
+ *
  */
 
 #define DEFINE_GUARD(_name, _type, _lock, _unlock) \
-	DEFINE_CLASS(_name, _type, _unlock, ({ _lock; _T; }), _type _T)
+	DEFINE_CLASS(_name, _type, if (_T) { _unlock; }, ({ _lock; _T; }), _type _T); \
+	static inline void * class_##_name##_lock_ptr(class_##_name##_t *_T) \
+	{ return *_T; }
+
+#define DEFINE_GUARD_COND(_name, _ext, _condlock) \
+	EXTEND_CLASS(_name, _ext, \
+		     ({ void *_t = _T; if (_T && !(_condlock)) _t = NULL; _t; }), \
+		     class_##_name##_t _T) \
+	static inline void * class_##_name##_ext##_lock_ptr(class_##_name##_t *_T) \
+	{ return class_##_name##_lock_ptr(_T); }
 
 #define guard(_name) \
 	CLASS(_name, __UNIQUE_ID(guard))
 
+#define __guard_ptr(_name) class_##_name##_lock_ptr
+
 #define scoped_guard(_name, args...)					\
 	for (CLASS(_name, scope)(args),					\
-	     *done = NULL; !done; done = (void *)1)
+	     *done = NULL; __guard_ptr(_name)(&scope) && !done; done = (void *)1)
+
+#define scoped_cond_guard(_name, _fail, args...) \
+	for (CLASS(_name, scope)(args), \
+	     *done = NULL; !done; done = (void *)1) \
+		if (!__guard_ptr(_name)(&scope)) _fail; \
+		else
 
 /*
  * Additional helper macros for generating lock guards with types, either for
@@ -152,6 +182,7 @@ static inline class_##_name##_t class_##_name##ext##_constructor(_init_args) \
  *
  * DEFINE_LOCK_GUARD_0(name, lock, unlock, ...)
  * DEFINE_LOCK_GUARD_1(name, type, lock, unlock, ...)
+ * DEFINE_LOCK_GUARD_1_COND(name, ext, condlock)
  *
  * will result in the following type:
  *
@@ -173,6 +204,11 @@ typedef struct {							\
 static inline void class_##_name##_destructor(class_##_name##_t *_T)	\
 {									\
 	if (_T->lock) { _unlock; }					\
+}									\
+									\
+static inline void *class_##_name##_lock_ptr(class_##_name##_t *_T)	\
+{									\
+	return _T->lock;						\
 }
 
 
@@ -201,4 +237,14 @@ __DEFINE_LOCK_GUARD_1(_name, _type, _lock)
 __DEFINE_UNLOCK_GUARD(_name, void, _unlock, __VA_ARGS__)		\
 __DEFINE_LOCK_GUARD_0(_name, _lock)
 
+#define DEFINE_LOCK_GUARD_1_COND(_name, _ext, _condlock)		\
+	EXTEND_CLASS(_name, _ext,					\
+		     ({ class_##_name##_t _t = { .lock = l }, *_T = &_t;\
+		        if (_T->lock && !(_condlock)) _T->lock = NULL;	\
+			_t; }),						\
+		     typeof_member(class_##_name##_t, lock) l)		\
+	static inline void * class_##_name##_ext##_lock_ptr(class_##_name##_t *_T) \
+	{ return class_##_name##_lock_ptr(_T); }
+
+
 #endif /* __LINUX_GUARDS_H */
diff --git a/include/linux/closure.h b/include/linux/closure.h
index de7bb47d8a46..c554c6a08768 100644
--- a/include/linux/closure.h
+++ b/include/linux/closure.h
@@ -104,7 +104,7 @@
 
 struct closure;
 struct closure_syncer;
-typedef void (closure_fn) (struct closure *);
+typedef void (closure_fn) (struct work_struct *);
 extern struct dentry *bcache_debug;
 
 struct closure_waitlist {
@@ -254,7 +254,7 @@ static inline void closure_queue(struct closure *cl)
 		INIT_WORK(&cl->work, cl->work.func);
 		BUG_ON(!queue_work(wq, &cl->work));
 	} else
-		cl->fn(cl);
+		cl->fn(&cl->work);
 }
 
 /**
@@ -309,6 +309,11 @@ static inline void closure_wake_up(struct closure_waitlist *list)
 	__closure_wake_up(list);
 }
 
+#define CLOSURE_CALLBACK(name)	void name(struct work_struct *ws)
+#define closure_type(name, type, member)				\
+	struct closure *cl = container_of(ws, struct closure, work);	\
+	type *name = container_of(cl, type, member)
+
 /**
  * continue_at - jump to another function with barrier
  *
diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h
index 2ceba3fe4ec1..aebb65bf95a7 100644
--- a/include/linux/compiler-gcc.h
+++ b/include/linux/compiler-gcc.h
@@ -136,7 +136,7 @@
 #endif
 
 #define __diag_ignore_all(option, comment) \
-	__diag_GCC(8, ignore, option)
+	__diag(__diag_GCC_ignore option)
 
 /*
  * Prior to 9.1, -Wno-alloc-size-larger-than (and therefore the "alloc_size"
diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index bb1339c7057b..cdcdaa48b4d2 100644
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -209,7 +209,7 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
  */
 #define ___ADDRESSABLE(sym, __attrs) \
 	static void * __used __attrs \
-		__UNIQUE_ID(__PASTE(__addressable_,sym)) = (void *)&sym;
+	__UNIQUE_ID(__PASTE(__addressable_,sym)) = (void *)(uintptr_t)&sym;
 #define __ADDRESSABLE(sym) \
 	___ADDRESSABLE(sym, __section(".discard.addressable"))
 
diff --git a/include/linux/coresight.h b/include/linux/coresight.h
index a269fffaf991..a4cb7dd6ca23 100644
--- a/include/linux/coresight.h
+++ b/include/linux/coresight.h
@@ -64,6 +64,7 @@ enum coresight_dev_subtype_source {
 	CORESIGHT_DEV_SUBTYPE_SOURCE_PROC,
 	CORESIGHT_DEV_SUBTYPE_SOURCE_BUS,
 	CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE,
+	CORESIGHT_DEV_SUBTYPE_SOURCE_TPDM,
 	CORESIGHT_DEV_SUBTYPE_SOURCE_OTHERS,
 };
 
diff --git a/include/linux/dcache.h b/include/linux/dcache.h
index 3da2f0545d5d..e601e9c30138 100644
--- a/include/linux/dcache.h
+++ b/include/linux/dcache.h
@@ -126,7 +126,7 @@ enum dentry_d_lock_class
 };
 
 struct dentry_operations {
-	int (*d_revalidate)(struct dentry *, unsigned int);
+	int (*d_revalidate)(struct dentry *, const struct qstr *, unsigned int);
 	int (*d_weak_revalidate)(struct dentry *, unsigned int);
 	int (*d_hash)(const struct dentry *, struct qstr *);
 	int (*d_compare)(const struct dentry *,
@@ -208,6 +208,7 @@ struct dentry_operations {
 #define DCACHE_FALLTHRU			0x01000000 /* Fall through to lower layer */
 #define DCACHE_NOKEY_NAME		0x02000000 /* Encrypted name encoded without key */
 #define DCACHE_OP_REAL			0x04000000
+#define DCACHE_CASEFOLDED_NAME		0x08000000 /* Dentry comes from a casefold directory */
 
 #define DCACHE_PAR_LOOKUP		0x10000000 /* being looked up (with parent locked shared) */
 #define DCACHE_DENTRY_CURSOR		0x20000000
@@ -497,6 +498,13 @@ static inline bool d_is_fallthru(const struct dentry *dentry)
 	return dentry->d_flags & DCACHE_FALLTHRU;
 }
 
+extern void d_set_casefolded_name(struct dentry *dentry);
+
+static inline bool d_is_casefolded_name(const struct dentry *dentry)
+{
+	return dentry->d_flags & DCACHE_CASEFOLDED_NAME;
+}
+
 
 extern int sysctl_vfs_cache_pressure;
 
diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h
index 689028257fcc..c2bb74143eda 100644
--- a/include/linux/ethtool.h
+++ b/include/linux/ethtool.h
@@ -1043,6 +1043,14 @@ static inline int ethtool_mm_frag_size_min_to_add(u32 val_min, u32 *val_add,
 	return -EINVAL;
 }
 
+/**
+ * ethtool_get_ts_info_by_layer - Obtains time stamping capabilities from the MAC or PHY layer.
+ * @dev: pointer to net_device structure
+ * @info: buffer to hold the result
+ * Returns zero on success, non-zero otherwise.
+ */
+int ethtool_get_ts_info_by_layer(struct net_device *dev, struct ethtool_ts_info *info);
+
 /**
  * ethtool_sprintf - Write formatted string to ethtool string data
  * @data: Pointer to a pointer to the start of string to update
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 98b7a7a8c42e..2b05fabfdaa9 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -434,7 +434,7 @@ struct address_space_operations {
 	bool (*is_partially_uptodate) (struct folio *, size_t from,
 			size_t count);
 	void (*is_dirty_writeback) (struct folio *, bool *dirty, bool *wb);
-	int (*error_remove_page)(struct address_space *, struct page *);
+	int (*error_remove_folio)(struct address_space *, struct folio *);
 
 	/* swapfile support */
 	int (*swap_activate)(struct swap_info_struct *sis, struct file *file,
@@ -463,9 +463,9 @@ extern const struct address_space_operations empty_aops;
  * @a_ops: Methods.
  * @flags: Error bits and flags (AS_*).
  * @wb_err: The most recent error which has occurred.
- * @private_lock: For use by the owner of the address_space.
- * @private_list: For use by the owner of the address_space.
- * @private_data: For use by the owner of the address_space.
+ * @i_private_lock: For use by the owner of the address_space.
+ * @i_private_list: For use by the owner of the address_space.
+ * @i_private_data: For use by the owner of the address_space.
  */
 struct address_space {
 	struct inode		*host;
@@ -484,9 +484,9 @@ struct address_space {
 	unsigned long		flags;
 	struct rw_semaphore	i_mmap_rwsem;
 	errseq_t		wb_err;
-	spinlock_t		private_lock;
-	struct list_head	private_list;
-	void			*private_data;
+	spinlock_t		i_private_lock;
+	struct list_head	i_private_list;
+	void *			i_private_data;
 } __attribute__((aligned(sizeof(long)))) __randomize_layout;
 	/*
 	 * On most architectures that alignment is already the case; but
@@ -1185,7 +1185,8 @@ enum {
 
 struct sb_writers {
 	unsigned short			frozen;		/* Is sb frozen? */
-	unsigned short			freeze_holders;	/* Who froze fs? */
+	int				freeze_kcount;	/* How many kernel freeze requests? */
+	int				freeze_ucount;	/* How many userspace freeze requests? */
 	struct percpu_rw_semaphore	rw_sem[SB_FREEZE_LEVELS];
 };
 
@@ -2051,9 +2052,24 @@ extern loff_t vfs_dedupe_file_range_one(struct file *src_file, loff_t src_pos,
 					struct file *dst_file, loff_t dst_pos,
 					loff_t len, unsigned int remap_flags);
 
+/**
+ * enum freeze_holder - holder of the freeze
+ * @FREEZE_HOLDER_KERNEL: kernel wants to freeze or thaw filesystem
+ * @FREEZE_HOLDER_USERSPACE: userspace wants to freeze or thaw filesystem
+ * @FREEZE_MAY_NEST: whether nesting freeze and thaw requests is allowed
+ *
+ * Indicate who the owner of the freeze or thaw request is and whether
+ * the freeze needs to be exclusive or can nest.
+ * Without @FREEZE_MAY_NEST, multiple freeze and thaw requests from the
+ * same holder aren't allowed. It is however allowed to hold a single
+ * @FREEZE_HOLDER_USERSPACE and a single @FREEZE_HOLDER_KERNEL freeze at
+ * the same time. This is relied upon by some filesystems during online
+ * repair or similar.
+ */
 enum freeze_holder {
 	FREEZE_HOLDER_KERNEL	= (1U << 0),
 	FREEZE_HOLDER_USERSPACE	= (1U << 1),
+	FREEZE_MAY_NEST		= (1U << 2),
 };
 
 struct super_operations {
@@ -3121,7 +3137,6 @@ extern int vfs_readlink(struct dentry *, char __user *, int);
 extern struct file_system_type *get_filesystem(struct file_system_type *fs);
 extern void put_filesystem(struct file_system_type *fs);
 extern struct file_system_type *get_fs_type(const char *name);
-extern struct super_block *get_active_super(struct block_device *bdev);
 extern void drop_super(struct super_block *sb);
 extern void drop_super_exclusive(struct super_block *sb);
 extern void iterate_supers(void (*)(struct super_block *, void *), void *);
diff --git a/include/linux/fscrypt.h b/include/linux/fscrypt.h
index 12f9e455d569..2a8d592a6556 100644
--- a/include/linux/fscrypt.h
+++ b/include/linux/fscrypt.h
@@ -368,7 +368,8 @@ int fscrypt_fname_disk_to_usr(const struct inode *inode,
 bool fscrypt_match_name(const struct fscrypt_name *fname,
 			const u8 *de_name, u32 de_name_len);
 u64 fscrypt_fname_siphash(const struct inode *dir, const struct qstr *name);
-int fscrypt_d_revalidate(struct dentry *dentry, unsigned int flags);
+int fscrypt_d_revalidate(struct dentry *dentry, const struct qstr *name,
+			 unsigned int flags);
 
 /* bio.c */
 bool fscrypt_decrypt_bio(struct bio *bio);
@@ -663,6 +664,7 @@ static inline u64 fscrypt_fname_siphash(const struct inode *dir,
 }
 
 static inline int fscrypt_d_revalidate(struct dentry *dentry,
+				       const struct qstr *name,
 				       unsigned int flags)
 {
 	return 1;
diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h
index 6583a58670c5..ae994534a12a 100644
--- a/include/linux/gfp_types.h
+++ b/include/linux/gfp_types.h
@@ -274,7 +274,8 @@ typedef unsigned int __bitwise gfp_t;
  * accounted to kmemcg.
  *
  * %GFP_NOWAIT is for kernel allocations that should not stall for direct
- * reclaim, start physical IO or use any filesystem callback.
+ * reclaim, start physical IO or use any filesystem callback.  It is very
+ * likely to fail to allocate memory, even for very small allocations.
  *
  * %GFP_NOIO will use direct reclaim to discard clean pages or slab pages
  * that do not require the starting of any physical IO.
@@ -325,7 +326,7 @@ typedef unsigned int __bitwise gfp_t;
 #define GFP_ATOMIC	(__GFP_HIGH|__GFP_KSWAPD_RECLAIM)
 #define GFP_KERNEL	(__GFP_RECLAIM | __GFP_IO | __GFP_FS)
 #define GFP_KERNEL_ACCOUNT (GFP_KERNEL | __GFP_ACCOUNT)
-#define GFP_NOWAIT	(__GFP_KSWAPD_RECLAIM)
+#define GFP_NOWAIT	(__GFP_KSWAPD_RECLAIM | __GFP_NOWARN)
 #define GFP_NOIO	(__GFP_RECLAIM)
 #define GFP_NOFS	(__GFP_RECLAIM | __GFP_IO)
 #define GFP_USER	(__GFP_RECLAIM | __GFP_IO | __GFP_FS | __GFP_HARDWALL)
diff --git a/include/linux/hid.h b/include/linux/hid.h
index 5a8387a4a712..bf43f3ff6664 100644
--- a/include/linux/hid.h
+++ b/include/linux/hid.h
@@ -679,6 +679,7 @@ struct hid_device {							/* device report descriptor */
 	struct list_head debug_list;
 	spinlock_t  debug_list_lock;
 	wait_queue_head_t debug_wait;
+	struct kref			ref;
 
 	unsigned int id;						/* system unique id */
 
@@ -687,6 +688,8 @@ struct hid_device {							/* device report descriptor */
 #endif /* CONFIG_BPF */
 };
 
+void hiddev_free(struct kref *ref);
+
 #define to_hid_device(pdev) \
 	container_of(pdev, struct hid_device, dev)
 
diff --git a/include/linux/highmem.h b/include/linux/highmem.h
index 4cacc0e43b51..6e4075f0c3c0 100644
--- a/include/linux/highmem.h
+++ b/include/linux/highmem.h
@@ -483,6 +483,82 @@ static inline void memcpy_to_folio(struct folio *folio, size_t offset,
 	flush_dcache_folio(folio);
 }
 
+/**
+ * folio_zero_tail - Zero the tail of a folio.
+ * @folio: The folio to zero.
+ * @offset: The byte offset in the folio to start zeroing at.
+ * @kaddr: The address the folio is currently mapped to.
+ *
+ * If you have already used kmap_local_folio() to map a folio, written
+ * some data to it and now need to zero the end of the folio (and flush
+ * the dcache), you can use this function.  If you do not have the
+ * folio kmapped (eg the folio has been partially populated by DMA),
+ * use folio_zero_range() or folio_zero_segment() instead.
+ *
+ * Return: An address which can be passed to kunmap_local().
+ */
+static inline __must_check void *folio_zero_tail(struct folio *folio,
+		size_t offset, void *kaddr)
+{
+	size_t len = folio_size(folio) - offset;
+
+	if (folio_test_highmem(folio)) {
+		size_t max = PAGE_SIZE - offset_in_page(offset);
+
+		while (len > max) {
+			memset(kaddr, 0, max);
+			kunmap_local(kaddr);
+			len -= max;
+			offset += max;
+			max = PAGE_SIZE;
+			kaddr = kmap_local_folio(folio, offset);
+		}
+	}
+
+	memset(kaddr, 0, len);
+	flush_dcache_folio(folio);
+
+	return kaddr;
+}
+
+/**
+ * folio_fill_tail - Copy some data to a folio and pad with zeroes.
+ * @folio: The destination folio.
+ * @offset: The offset into @folio at which to start copying.
+ * @from: The data to copy.
+ * @len: How many bytes of data to copy.
+ *
+ * This function is most useful for filesystems which support inline data.
+ * When they want to copy data from the inode into the page cache, this
+ * function does everything for them.  It supports large folios even on
+ * HIGHMEM configurations.
+ */
+static inline void folio_fill_tail(struct folio *folio, size_t offset,
+		const char *from, size_t len)
+{
+	char *to = kmap_local_folio(folio, offset);
+
+	VM_BUG_ON(offset + len > folio_size(folio));
+
+	if (folio_test_highmem(folio)) {
+		size_t max = PAGE_SIZE - offset_in_page(offset);
+
+		while (len > max) {
+			memcpy(to, from, max);
+			kunmap_local(to);
+			len -= max;
+			from += max;
+			offset += max;
+			max = PAGE_SIZE;
+			to = kmap_local_folio(folio, offset);
+		}
+	}
+
+	memcpy(to, from, len);
+	to = folio_zero_tail(folio, offset + len, to + len);
+	kunmap_local(to);
+}
+
 /**
  * memcpy_from_file_folio - Copy some bytes from a file folio.
  * @to: The destination buffer.
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index d3acecc5db4b..236ec7b63c54 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -1268,10 +1268,7 @@ static inline bool __vma_shareable_lock(struct vm_area_struct *vma)
 	return (vma->vm_flags & VM_MAYSHARE) && vma->vm_private_data;
 }
 
-static inline bool __vma_private_lock(struct vm_area_struct *vma)
-{
-	return (!(vma->vm_flags & VM_MAYSHARE)) && vma->vm_private_data;
-}
+bool __vma_private_lock(struct vm_area_struct *vma);
 
 /*
  * Safe version of huge_pte_offset() to check the locks.  See comments
diff --git a/include/linux/indirect_call_wrapper.h b/include/linux/indirect_call_wrapper.h
index c1c76a70a6ce..adb83a42a6b9 100644
--- a/include/linux/indirect_call_wrapper.h
+++ b/include/linux/indirect_call_wrapper.h
@@ -11,7 +11,7 @@
  *  @__VA_ARGS__: arguments for @f
  *
  * Avoid retpoline overhead for known builtin, checking @f vs each of them and
- * eventually invoking directly the builtin function. The functions are check
+ * eventually invoking directly the builtin function. The functions are checked
  * in the given order. Fallback to the indirect call.
  */
 #define INDIRECT_CALL_1(f, f1, ...)					\
diff --git a/include/linux/init_task.h b/include/linux/init_task.h
index 40fc5813cf93..bccb3f1f6262 100644
--- a/include/linux/init_task.h
+++ b/include/linux/init_task.h
@@ -37,13 +37,6 @@ extern struct cred init_cred;
 
 #define INIT_TASK_COMM "swapper"
 
-/* Attach to the init_task data structure for proper alignment */
-#ifdef CONFIG_ARCH_TASK_STRUCT_ON_STACK
-#define __init_task_data __section(".data..init_task")
-#else
-#define __init_task_data /**/
-#endif
-
 /* Attach to the thread_info data structure for proper alignment */
 #define __init_thread_info __section(".data..init_thread_info")
 
diff --git a/include/linux/iosys-map.h b/include/linux/iosys-map.h
index 1b06d074ade0..e3649a6563dd 100644
--- a/include/linux/iosys-map.h
+++ b/include/linux/iosys-map.h
@@ -168,9 +168,9 @@ struct iosys_map {
  * about the use of uninitialized variable.
  */
 #define IOSYS_MAP_INIT_OFFSET(map_, offset_) ({				\
-	struct iosys_map copy = *map_;					\
-	iosys_map_incr(&copy, offset_);					\
-	copy;								\
+	struct iosys_map copy_ = *map_;					\
+	iosys_map_incr(&copy_, offset_);				\
+	copy_;								\
 })
 
 /**
@@ -391,14 +391,14 @@ static inline void iosys_map_memset(struct iosys_map *dst, size_t offset,
  * Returns:
  * The value read from the mapping.
  */
-#define iosys_map_rd(map__, offset__, type__) ({				\
-	type__ val;								\
-	if ((map__)->is_iomem) {						\
-		__iosys_map_rd_io(val, (map__)->vaddr_iomem + (offset__), type__);\
-	} else {								\
-		__iosys_map_rd_sys(val, (map__)->vaddr + (offset__), type__);	\
-	}									\
-	val;									\
+#define iosys_map_rd(map__, offset__, type__) ({					\
+	type__ val_;									\
+	if ((map__)->is_iomem) {							\
+		__iosys_map_rd_io(val_, (map__)->vaddr_iomem + (offset__), type__);	\
+	} else {									\
+		__iosys_map_rd_sys(val_, (map__)->vaddr + (offset__), type__);		\
+	}										\
+	val_;										\
 })
 
 /**
@@ -413,13 +413,13 @@ static inline void iosys_map_memset(struct iosys_map *dst, size_t offset,
  * or if pointer may be unaligned (and problematic for the architecture
  * supported), use iosys_map_memcpy_to()
  */
-#define iosys_map_wr(map__, offset__, type__, val__) ({				\
-	type__ val = (val__);							\
-	if ((map__)->is_iomem) {						\
-		__iosys_map_wr_io(val, (map__)->vaddr_iomem + (offset__), type__);\
-	} else {								\
-		__iosys_map_wr_sys(val, (map__)->vaddr + (offset__), type__);	\
-	}									\
+#define iosys_map_wr(map__, offset__, type__, val__) ({					\
+	type__ val_ = (val__);								\
+	if ((map__)->is_iomem) {							\
+		__iosys_map_wr_io(val_, (map__)->vaddr_iomem + (offset__), type__);	\
+	} else {									\
+		__iosys_map_wr_sys(val_, (map__)->vaddr + (offset__), type__);		\
+	}										\
 })
 
 /**
@@ -485,9 +485,9 @@ static inline void iosys_map_memset(struct iosys_map *dst, size_t offset,
  * The value read from the mapping.
  */
 #define iosys_map_rd_field(map__, struct_offset__, struct_type__, field__) ({	\
-	struct_type__ *s;							\
+	struct_type__ *s_;							\
 	iosys_map_rd(map__, struct_offset__ + offsetof(struct_type__, field__),	\
-		     typeof(s->field__));					\
+		     typeof(s_->field__));					\
 })
 
 /**
@@ -508,9 +508,9 @@ static inline void iosys_map_memset(struct iosys_map *dst, size_t offset,
  * usage and memory layout.
  */
 #define iosys_map_wr_field(map__, struct_offset__, struct_type__, field__, val__) ({	\
-	struct_type__ *s;								\
+	struct_type__ *s_;								\
 	iosys_map_wr(map__, struct_offset__ + offsetof(struct_type__, field__),		\
-		     typeof(s->field__), val__);					\
+		     typeof(s_->field__), val__);					\
 })
 
 #endif /* __IOSYS_MAP_H__ */
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 4944136efaa2..ea1523a7b83a 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -80,8 +80,8 @@
 /* Two fragments for cross MMIO pages. */
 #define KVM_MAX_MMIO_FRAGMENTS	2
 
-#ifndef KVM_ADDRESS_SPACE_NUM
-#define KVM_ADDRESS_SPACE_NUM	1
+#ifndef KVM_MAX_NR_ADDRESS_SPACES
+#define KVM_MAX_NR_ADDRESS_SPACES	1
 #endif
 
 /*
@@ -253,9 +253,10 @@ bool kvm_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
 int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu);
 #endif
 
-#ifdef KVM_ARCH_WANT_MMU_NOTIFIER
+#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER
 union kvm_mmu_notifier_arg {
 	pte_t pte;
+	unsigned long attributes;
 };
 
 struct kvm_gfn_range {
@@ -588,8 +589,20 @@ struct kvm_memory_slot {
 	u32 flags;
 	short id;
 	u16 as_id;
+
+#ifdef CONFIG_KVM_PRIVATE_MEM
+	struct {
+		struct file __rcu *file;
+		pgoff_t pgoff;
+	} gmem;
+#endif
 };
 
+static inline bool kvm_slot_can_be_private(const struct kvm_memory_slot *slot)
+{
+	return slot && (slot->flags & KVM_MEM_GUEST_MEMFD);
+}
+
 static inline bool kvm_slot_dirty_track_enabled(const struct kvm_memory_slot *slot)
 {
 	return slot->flags & KVM_MEM_LOG_DIRTY_PAGES;
@@ -677,13 +690,29 @@ bool kvm_arch_irqchip_in_kernel(struct kvm *kvm);
 #define KVM_MEM_SLOTS_NUM SHRT_MAX
 #define KVM_USER_MEM_SLOTS (KVM_MEM_SLOTS_NUM - KVM_INTERNAL_MEM_SLOTS)
 
-#ifndef __KVM_VCPU_MULTIPLE_ADDRESS_SPACE
+#if KVM_MAX_NR_ADDRESS_SPACES == 1
+static inline int kvm_arch_nr_memslot_as_ids(struct kvm *kvm)
+{
+	return KVM_MAX_NR_ADDRESS_SPACES;
+}
+
 static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu)
 {
 	return 0;
 }
 #endif
 
+/*
+ * Arch code must define kvm_arch_has_private_mem if support for private memory
+ * is enabled.
+ */
+#if !defined(kvm_arch_has_private_mem) && !IS_ENABLED(CONFIG_KVM_PRIVATE_MEM)
+static inline bool kvm_arch_has_private_mem(struct kvm *kvm)
+{
+	return false;
+}
+#endif
+
 struct kvm_memslots {
 	u64 generation;
 	atomic_long_t last_used_slot;
@@ -721,9 +750,9 @@ struct kvm {
 	struct mm_struct *mm; /* userspace tied to this vm */
 	unsigned long nr_memslot_pages;
 	/* The two memslot sets - active and inactive (per address space) */
-	struct kvm_memslots __memslots[KVM_ADDRESS_SPACE_NUM][2];
+	struct kvm_memslots __memslots[KVM_MAX_NR_ADDRESS_SPACES][2];
 	/* The current active memslot set for each address space */
-	struct kvm_memslots __rcu *memslots[KVM_ADDRESS_SPACE_NUM];
+	struct kvm_memslots __rcu *memslots[KVM_MAX_NR_ADDRESS_SPACES];
 	struct xarray vcpu_array;
 	/*
 	 * Protected by slots_lock, but can be read outside if an
@@ -783,12 +812,12 @@ struct kvm {
 	struct hlist_head irq_ack_notifier_list;
 #endif
 
-#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
+#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER
 	struct mmu_notifier mmu_notifier;
 	unsigned long mmu_invalidate_seq;
 	long mmu_invalidate_in_progress;
-	unsigned long mmu_invalidate_range_start;
-	unsigned long mmu_invalidate_range_end;
+	gfn_t mmu_invalidate_range_start;
+	gfn_t mmu_invalidate_range_end;
 #endif
 	struct list_head devices;
 	u64 manual_dirty_log_protect;
@@ -806,6 +835,10 @@ struct kvm {
 
 #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER
 	struct notifier_block pm_notifier;
+#endif
+#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
+	/* Protected by slots_locks (for writes) and RCU (for reads) */
+	struct xarray mem_attr_array;
 #endif
 	char stats_id[KVM_STATS_NAME_SIZE];
 };
@@ -989,7 +1022,7 @@ void kvm_put_kvm_no_destroy(struct kvm *kvm);
 
 static inline struct kvm_memslots *__kvm_memslots(struct kvm *kvm, int as_id)
 {
-	as_id = array_index_nospec(as_id, KVM_ADDRESS_SPACE_NUM);
+	as_id = array_index_nospec(as_id, KVM_MAX_NR_ADDRESS_SPACES);
 	return srcu_dereference_check(kvm->memslots[as_id], &kvm->srcu,
 			lockdep_is_held(&kvm->slots_lock) ||
 			!refcount_read(&kvm->users_count));
@@ -1146,9 +1179,9 @@ enum kvm_mr_change {
 };
 
 int kvm_set_memory_region(struct kvm *kvm,
-			  const struct kvm_userspace_memory_region *mem);
+			  const struct kvm_userspace_memory_region2 *mem);
 int __kvm_set_memory_region(struct kvm *kvm,
-			    const struct kvm_userspace_memory_region *mem);
+			    const struct kvm_userspace_memory_region2 *mem);
 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot);
 void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen);
 int kvm_arch_prepare_memory_region(struct kvm *kvm,
@@ -1392,10 +1425,10 @@ void kvm_mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc);
 void *kvm_mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc);
 #endif
 
-void kvm_mmu_invalidate_begin(struct kvm *kvm, unsigned long start,
-			      unsigned long end);
-void kvm_mmu_invalidate_end(struct kvm *kvm, unsigned long start,
-			    unsigned long end);
+void kvm_mmu_invalidate_begin(struct kvm *kvm);
+void kvm_mmu_invalidate_range_add(struct kvm *kvm, gfn_t start, gfn_t end);
+void kvm_mmu_invalidate_end(struct kvm *kvm);
+bool kvm_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range);
 
 long kvm_arch_dev_ioctl(struct file *filp,
 			unsigned int ioctl, unsigned long arg);
@@ -1947,7 +1980,7 @@ extern const struct _kvm_stats_desc kvm_vm_stats_desc[];
 extern const struct kvm_stats_header kvm_vcpu_stats_header;
 extern const struct _kvm_stats_desc kvm_vcpu_stats_desc[];
 
-#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
+#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER
 static inline int mmu_invalidate_retry(struct kvm *kvm, unsigned long mmu_seq)
 {
 	if (unlikely(kvm->mmu_invalidate_in_progress))
@@ -1970,9 +2003,9 @@ static inline int mmu_invalidate_retry(struct kvm *kvm, unsigned long mmu_seq)
 	return 0;
 }
 
-static inline int mmu_invalidate_retry_hva(struct kvm *kvm,
+static inline int mmu_invalidate_retry_gfn(struct kvm *kvm,
 					   unsigned long mmu_seq,
-					   unsigned long hva)
+					   gfn_t gfn)
 {
 	lockdep_assert_held(&kvm->mmu_lock);
 	/*
@@ -1981,10 +2014,20 @@ static inline int mmu_invalidate_retry_hva(struct kvm *kvm,
 	 * that might be being invalidated. Note that it may include some false
 	 * positives, due to shortcuts when handing concurrent invalidations.
 	 */
-	if (unlikely(kvm->mmu_invalidate_in_progress) &&
-	    hva >= kvm->mmu_invalidate_range_start &&
-	    hva < kvm->mmu_invalidate_range_end)
-		return 1;
+	if (unlikely(kvm->mmu_invalidate_in_progress)) {
+		/*
+		 * Dropping mmu_lock after bumping mmu_invalidate_in_progress
+		 * but before updating the range is a KVM bug.
+		 */
+		if (WARN_ON_ONCE(kvm->mmu_invalidate_range_start == INVALID_GPA ||
+				 kvm->mmu_invalidate_range_end == INVALID_GPA))
+			return 1;
+
+		if (gfn >= kvm->mmu_invalidate_range_start &&
+		    gfn < kvm->mmu_invalidate_range_end)
+			return 1;
+	}
+
 	if (kvm->mmu_invalidate_seq != mmu_seq)
 		return 1;
 	return 0;
@@ -2318,4 +2361,57 @@ static inline void kvm_account_pgtable_pages(void *virt, int nr)
 /* Max number of entries allowed for each kvm dirty ring */
 #define  KVM_DIRTY_RING_MAX_ENTRIES  65536
 
+static inline void kvm_prepare_memory_fault_exit(struct kvm_vcpu *vcpu,
+						 gpa_t gpa, gpa_t size,
+						 bool is_write, bool is_exec,
+						 bool is_private)
+{
+	vcpu->run->exit_reason = KVM_EXIT_MEMORY_FAULT;
+	vcpu->run->memory_fault.gpa = gpa;
+	vcpu->run->memory_fault.size = size;
+
+	/* RWX flags are not (yet) defined or communicated to userspace. */
+	vcpu->run->memory_fault.flags = 0;
+	if (is_private)
+		vcpu->run->memory_fault.flags |= KVM_MEMORY_EXIT_FLAG_PRIVATE;
+}
+
+#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
+static inline unsigned long kvm_get_memory_attributes(struct kvm *kvm, gfn_t gfn)
+{
+	return xa_to_value(xa_load(&kvm->mem_attr_array, gfn));
+}
+
+bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
+				     unsigned long attrs);
+bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm,
+					struct kvm_gfn_range *range);
+bool kvm_arch_post_set_memory_attributes(struct kvm *kvm,
+					 struct kvm_gfn_range *range);
+
+static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn)
+{
+	return IS_ENABLED(CONFIG_KVM_PRIVATE_MEM) &&
+	       kvm_get_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE;
+}
+#else
+static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn)
+{
+	return false;
+}
+#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */
+
+#ifdef CONFIG_KVM_PRIVATE_MEM
+int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot,
+		     gfn_t gfn, kvm_pfn_t *pfn, int *max_order);
+#else
+static inline int kvm_gmem_get_pfn(struct kvm *kvm,
+				   struct kvm_memory_slot *slot, gfn_t gfn,
+				   kvm_pfn_t *pfn, int *max_order)
+{
+	KVM_BUG_ON(1, kvm);
+	return -EIO;
+}
+#endif /* CONFIG_KVM_PRIVATE_MEM */
+
 #endif
diff --git a/include/linux/kvm_types.h b/include/linux/kvm_types.h
index 6f4737d5046a..9d1f7835d8c1 100644
--- a/include/linux/kvm_types.h
+++ b/include/linux/kvm_types.h
@@ -6,6 +6,7 @@
 struct kvm;
 struct kvm_async_pf;
 struct kvm_device_ops;
+struct kvm_gfn_range;
 struct kvm_interrupt;
 struct kvm_irq_routing_table;
 struct kvm_memory_slot;
diff --git a/include/linux/linkmode.h b/include/linux/linkmode.h
index 7303b4bc2ce0..287f590ed56b 100644
--- a/include/linux/linkmode.h
+++ b/include/linux/linkmode.h
@@ -10,6 +10,11 @@ static inline void linkmode_zero(unsigned long *dst)
 	bitmap_zero(dst, __ETHTOOL_LINK_MODE_MASK_NBITS);
 }
 
+static inline void linkmode_fill(unsigned long *dst)
+{
+	bitmap_fill(dst, __ETHTOOL_LINK_MODE_MASK_NBITS);
+}
+
 static inline void linkmode_copy(unsigned long *dst, const unsigned long *src)
 {
 	bitmap_copy(dst, src, __ETHTOOL_LINK_MODE_MASK_NBITS);
diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h
index ff217a5ce552..c925a0d26edf 100644
--- a/include/linux/lsm_hook_defs.h
+++ b/include/linux/lsm_hook_defs.h
@@ -262,6 +262,10 @@ LSM_HOOK(int, 0, sem_semop, struct kern_ipc_perm *perm, struct sembuf *sops,
 LSM_HOOK(int, 0, netlink_send, struct sock *sk, struct sk_buff *skb)
 LSM_HOOK(void, LSM_RET_VOID, d_instantiate, struct dentry *dentry,
 	 struct inode *inode)
+LSM_HOOK(int, -EOPNOTSUPP, getselfattr, unsigned int attr,
+	 struct lsm_ctx __user *ctx, size_t *size, u32 flags)
+LSM_HOOK(int, -EOPNOTSUPP, setselfattr, unsigned int attr,
+	 struct lsm_ctx *ctx, size_t size, u32 flags)
 LSM_HOOK(int, -EINVAL, getprocattr, struct task_struct *p, const char *name,
 	 char **value)
 LSM_HOOK(int, -EINVAL, setprocattr, const char *name, void *value, size_t size)
diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h
index dcb5e5b5eb13..a2ade0ffe9e7 100644
--- a/include/linux/lsm_hooks.h
+++ b/include/linux/lsm_hooks.h
@@ -25,6 +25,7 @@
 #ifndef __LINUX_LSM_HOOKS_H
 #define __LINUX_LSM_HOOKS_H
 
+#include <uapi/linux/lsm.h>
 #include <linux/security.h>
 #include <linux/init.h>
 #include <linux/rculist.h>
@@ -42,6 +43,18 @@ struct security_hook_heads {
 	#undef LSM_HOOK
 } __randomize_layout;
 
+/**
+ * struct lsm_id - Identify a Linux Security Module.
+ * @lsm: name of the LSM, must be approved by the LSM maintainers
+ * @id: LSM ID number from uapi/linux/lsm.h
+ *
+ * Contains the information that identifies the LSM.
+ */
+struct lsm_id {
+	const char	*name;
+	u64		id;
+};
+
 /*
  * Security module hook list structure.
  * For use with generic list macros for common operations.
@@ -50,7 +63,7 @@ struct security_hook_list {
 	struct hlist_node		list;
 	struct hlist_head		*head;
 	union security_list_options	hook;
-	const char			*lsm;
+	const struct lsm_id		*lsmid;
 } __randomize_layout;
 
 /*
@@ -104,7 +117,7 @@ extern struct security_hook_heads security_hook_heads;
 extern char *lsm_names;
 
 extern void security_add_hooks(struct security_hook_list *hooks, int count,
-				const char *lsm);
+			       const struct lsm_id *lsmid);
 
 #define LSM_FLAG_LEGACY_MAJOR	BIT(0)
 #define LSM_FLAG_EXCLUSIVE	BIT(1)
diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h
index d01e850b570f..b3d63123b945 100644
--- a/include/linux/maple_tree.h
+++ b/include/linux/maple_tree.h
@@ -256,6 +256,8 @@ struct maple_tree {
 	struct maple_tree name = MTREE_INIT(name, 0)
 
 #define mtree_lock(mt)		spin_lock((&(mt)->ma_lock))
+#define mtree_lock_nested(mas, subclass) \
+		spin_lock_nested((&(mt)->ma_lock), subclass)
 #define mtree_unlock(mt)	spin_unlock((&(mt)->ma_lock))
 
 /*
@@ -327,6 +329,9 @@ int mtree_store(struct maple_tree *mt, unsigned long index,
 		void *entry, gfp_t gfp);
 void *mtree_erase(struct maple_tree *mt, unsigned long index);
 
+int mtree_dup(struct maple_tree *mt, struct maple_tree *new, gfp_t gfp);
+int __mt_dup(struct maple_tree *mt, struct maple_tree *new, gfp_t gfp);
+
 void mtree_destroy(struct maple_tree *mt);
 void __mt_destroy(struct maple_tree *mt);
 
@@ -344,6 +349,36 @@ static inline bool mtree_empty(const struct maple_tree *mt)
 
 /* Advanced API */
 
+/*
+ * Maple State Status
+ * ma_active means the maple state is pointing to a node and offset and can
+ * continue operating on the tree.
+ * ma_start means we have not searched the tree.
+ * ma_root means we have searched the tree and the entry we found lives in
+ * the root of the tree (ie it has index 0, length 1 and is the only entry in
+ * the tree).
+ * ma_none means we have searched the tree and there is no node in the
+ * tree for this entry.  For example, we searched for index 1 in an empty
+ * tree.  Or we have a tree which points to a full leaf node and we
+ * searched for an entry which is larger than can be contained in that
+ * leaf node.
+ * ma_pause means the data within the maple state may be stale, restart the
+ * operation
+ * ma_overflow means the search has reached the upper limit of the search
+ * ma_underflow means the search has reached the lower limit of the search
+ * ma_error means there was an error, check the node for the error number.
+ */
+enum maple_status {
+	ma_active,
+	ma_start,
+	ma_root,
+	ma_none,
+	ma_pause,
+	ma_overflow,
+	ma_underflow,
+	ma_error,
+};
+
 /*
  * The maple state is defined in the struct ma_state and is used to keep track
  * of information during operations, and even between operations when using the
@@ -376,6 +411,13 @@ static inline bool mtree_empty(const struct maple_tree *mt)
  * When returning a value the maple state index and last respectively contain
  * the start and end of the range for the entry.  Ranges are inclusive in the
  * Maple Tree.
+ *
+ * The status of the state is used to determine how the next action should treat
+ * the state.  For instance, if the status is ma_start then the next action
+ * should start at the root of the tree and walk down.  If the status is
+ * ma_pause then the node may be stale data and should be discarded.  If the
+ * status is ma_overflow, then the last action hit the upper limit.
+ *
  */
 struct ma_state {
 	struct maple_tree *tree;	/* The tree we're operating in */
@@ -385,9 +427,11 @@ struct ma_state {
 	unsigned long min;		/* The minimum index of this node - implied pivot min */
 	unsigned long max;		/* The maximum index of this node - implied pivot max */
 	struct maple_alloc *alloc;	/* Allocated nodes for this operation */
+	enum maple_status status;	/* The status of the state (active, start, none, etc) */
 	unsigned char depth;		/* depth of tree descent during write */
 	unsigned char offset;
 	unsigned char mas_flags;
+	unsigned char end;		/* The end of the node */
 };
 
 struct ma_wr_state {
@@ -397,7 +441,6 @@ struct ma_wr_state {
 	unsigned long r_max;		/* range max */
 	enum maple_type type;		/* mas->node type */
 	unsigned char offset_end;	/* The offset where the write ends */
-	unsigned char node_end;		/* mas->node end */
 	unsigned long *pivots;		/* mas->node->pivots pointer */
 	unsigned long end_piv;		/* The pivot at the offset end */
 	void __rcu **slots;		/* mas->node->slots pointer */
@@ -406,30 +449,16 @@ struct ma_wr_state {
 };
 
 #define mas_lock(mas)           spin_lock(&((mas)->tree->ma_lock))
+#define mas_lock_nested(mas, subclass) \
+		spin_lock_nested(&((mas)->tree->ma_lock), subclass)
 #define mas_unlock(mas)         spin_unlock(&((mas)->tree->ma_lock))
 
-
 /*
  * Special values for ma_state.node.
- * MAS_START means we have not searched the tree.
- * MAS_ROOT means we have searched the tree and the entry we found lives in
- * the root of the tree (ie it has index 0, length 1 and is the only entry in
- * the tree).
- * MAS_NONE means we have searched the tree and there is no node in the
- * tree for this entry.  For example, we searched for index 1 in an empty
- * tree.  Or we have a tree which points to a full leaf node and we
- * searched for an entry which is larger than can be contained in that
- * leaf node.
  * MA_ERROR represents an errno.  After dropping the lock and attempting
  * to resolve the error, the walk would have to be restarted from the
  * top of the tree as the tree may have been modified.
  */
-#define MAS_START	((struct maple_enode *)1UL)
-#define MAS_ROOT	((struct maple_enode *)5UL)
-#define MAS_NONE	((struct maple_enode *)9UL)
-#define MAS_PAUSE	((struct maple_enode *)17UL)
-#define MAS_OVERFLOW	((struct maple_enode *)33UL)
-#define MAS_UNDERFLOW	((struct maple_enode *)65UL)
 #define MA_ERROR(err) \
 		((struct maple_enode *)(((unsigned long)err << 2) | 2UL))
 
@@ -438,7 +467,8 @@ struct ma_wr_state {
 		.tree = mt,						\
 		.index = first,						\
 		.last = end,						\
-		.node = MAS_START,					\
+		.node = NULL,						\
+		.status = ma_start,					\
 		.min = 0,						\
 		.max = ULONG_MAX,					\
 		.alloc = NULL,						\
@@ -469,7 +499,6 @@ void *mas_find_range(struct ma_state *mas, unsigned long max);
 void *mas_find_rev(struct ma_state *mas, unsigned long min);
 void *mas_find_range_rev(struct ma_state *mas, unsigned long max);
 int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp);
-bool mas_is_err(struct ma_state *mas);
 
 bool mas_nomem(struct ma_state *mas, gfp_t gfp);
 void mas_pause(struct ma_state *mas);
@@ -498,28 +527,18 @@ static inline void mas_init(struct ma_state *mas, struct maple_tree *tree,
 	mas->tree = tree;
 	mas->index = mas->last = addr;
 	mas->max = ULONG_MAX;
-	mas->node = MAS_START;
-}
-
-/* Checks if a mas has not found anything */
-static inline bool mas_is_none(const struct ma_state *mas)
-{
-	return mas->node == MAS_NONE;
+	mas->status = ma_start;
+	mas->node = NULL;
 }
 
-/* Checks if a mas has been paused */
-static inline bool mas_is_paused(const struct ma_state *mas)
+static inline bool mas_is_active(struct ma_state *mas)
 {
-	return mas->node == MAS_PAUSE;
+	return mas->status == ma_active;
 }
 
-/* Check if the mas is pointing to a node or not */
-static inline bool mas_is_active(struct ma_state *mas)
+static inline bool mas_is_err(struct ma_state *mas)
 {
-	if ((unsigned long)mas->node >= MAPLE_RESERVED_RANGE)
-		return true;
-
-	return false;
+	return mas->status == ma_error;
 }
 
 /**
@@ -532,9 +551,10 @@ static inline bool mas_is_active(struct ma_state *mas)
  *
  * Context: Any context.
  */
-static inline void mas_reset(struct ma_state *mas)
+static __always_inline void mas_reset(struct ma_state *mas)
 {
-	mas->node = MAS_START;
+	mas->status = ma_start;
+	mas->node = NULL;
 }
 
 /**
@@ -550,6 +570,131 @@ static inline void mas_reset(struct ma_state *mas)
  */
 #define mas_for_each(__mas, __entry, __max) \
 	while (((__entry) = mas_find((__mas), (__max))) != NULL)
+
+#ifdef CONFIG_DEBUG_MAPLE_TREE
+enum mt_dump_format {
+	mt_dump_dec,
+	mt_dump_hex,
+};
+
+extern atomic_t maple_tree_tests_run;
+extern atomic_t maple_tree_tests_passed;
+
+void mt_dump(const struct maple_tree *mt, enum mt_dump_format format);
+void mas_dump(const struct ma_state *mas);
+void mas_wr_dump(const struct ma_wr_state *wr_mas);
+void mt_validate(struct maple_tree *mt);
+void mt_cache_shrink(void);
+#define MT_BUG_ON(__tree, __x) do {					\
+	atomic_inc(&maple_tree_tests_run);				\
+	if (__x) {							\
+		pr_info("BUG at %s:%d (%u)\n",				\
+		__func__, __LINE__, __x);				\
+		mt_dump(__tree, mt_dump_hex);				\
+		pr_info("Pass: %u Run:%u\n",				\
+			atomic_read(&maple_tree_tests_passed),		\
+			atomic_read(&maple_tree_tests_run));		\
+		dump_stack();						\
+	} else {							\
+		atomic_inc(&maple_tree_tests_passed);			\
+	}								\
+} while (0)
+
+#define MAS_BUG_ON(__mas, __x) do {					\
+	atomic_inc(&maple_tree_tests_run);				\
+	if (__x) {							\
+		pr_info("BUG at %s:%d (%u)\n",				\
+		__func__, __LINE__, __x);				\
+		mas_dump(__mas);					\
+		mt_dump((__mas)->tree, mt_dump_hex);			\
+		pr_info("Pass: %u Run:%u\n",				\
+			atomic_read(&maple_tree_tests_passed),		\
+			atomic_read(&maple_tree_tests_run));		\
+		dump_stack();						\
+	} else {							\
+		atomic_inc(&maple_tree_tests_passed);			\
+	}								\
+} while (0)
+
+#define MAS_WR_BUG_ON(__wrmas, __x) do {				\
+	atomic_inc(&maple_tree_tests_run);				\
+	if (__x) {							\
+		pr_info("BUG at %s:%d (%u)\n",				\
+		__func__, __LINE__, __x);				\
+		mas_wr_dump(__wrmas);					\
+		mas_dump((__wrmas)->mas);				\
+		mt_dump((__wrmas)->mas->tree, mt_dump_hex);		\
+		pr_info("Pass: %u Run:%u\n",				\
+			atomic_read(&maple_tree_tests_passed),		\
+			atomic_read(&maple_tree_tests_run));		\
+		dump_stack();						\
+	} else {							\
+		atomic_inc(&maple_tree_tests_passed);			\
+	}								\
+} while (0)
+
+#define MT_WARN_ON(__tree, __x)  ({					\
+	int ret = !!(__x);						\
+	atomic_inc(&maple_tree_tests_run);				\
+	if (ret) {							\
+		pr_info("WARN at %s:%d (%u)\n",				\
+		__func__, __LINE__, __x);				\
+		mt_dump(__tree, mt_dump_hex);				\
+		pr_info("Pass: %u Run:%u\n",				\
+			atomic_read(&maple_tree_tests_passed),		\
+			atomic_read(&maple_tree_tests_run));		\
+		dump_stack();						\
+	} else {							\
+		atomic_inc(&maple_tree_tests_passed);			\
+	}								\
+	unlikely(ret);							\
+})
+
+#define MAS_WARN_ON(__mas, __x) ({					\
+	int ret = !!(__x);						\
+	atomic_inc(&maple_tree_tests_run);				\
+	if (ret) {							\
+		pr_info("WARN at %s:%d (%u)\n",				\
+		__func__, __LINE__, __x);				\
+		mas_dump(__mas);					\
+		mt_dump((__mas)->tree, mt_dump_hex);			\
+		pr_info("Pass: %u Run:%u\n",				\
+			atomic_read(&maple_tree_tests_passed),		\
+			atomic_read(&maple_tree_tests_run));		\
+		dump_stack();						\
+	} else {							\
+		atomic_inc(&maple_tree_tests_passed);			\
+	}								\
+	unlikely(ret);							\
+})
+
+#define MAS_WR_WARN_ON(__wrmas, __x) ({					\
+	int ret = !!(__x);						\
+	atomic_inc(&maple_tree_tests_run);				\
+	if (ret) {							\
+		pr_info("WARN at %s:%d (%u)\n",				\
+		__func__, __LINE__, __x);				\
+		mas_wr_dump(__wrmas);					\
+		mas_dump((__wrmas)->mas);				\
+		mt_dump((__wrmas)->mas->tree, mt_dump_hex);		\
+		pr_info("Pass: %u Run:%u\n",				\
+			atomic_read(&maple_tree_tests_passed),		\
+			atomic_read(&maple_tree_tests_run));		\
+		dump_stack();						\
+	} else {							\
+		atomic_inc(&maple_tree_tests_passed);			\
+	}								\
+	unlikely(ret);							\
+})
+#else
+#define MT_BUG_ON(__tree, __x)		BUG_ON(__x)
+#define MAS_BUG_ON(__mas, __x)		BUG_ON(__x)
+#define MAS_WR_BUG_ON(__mas, __x)	BUG_ON(__x)
+#define MT_WARN_ON(__tree, __x)		WARN_ON(__x)
+#define MAS_WARN_ON(__mas, __x)		WARN_ON(__x)
+#define MAS_WR_WARN_ON(__mas, __x)	WARN_ON(__x)
+#endif /* CONFIG_DEBUG_MAPLE_TREE */
+
 /**
  * __mas_set_range() - Set up Maple Tree operation state to a sub-range of the
  * current location.
@@ -563,6 +708,9 @@ static inline void mas_reset(struct ma_state *mas)
 static inline void __mas_set_range(struct ma_state *mas, unsigned long start,
 		unsigned long last)
 {
+	/* Ensure the range starts within the current slot */
+	MAS_WARN_ON(mas, mas_is_active(mas) &&
+		   (mas->index > start || mas->last < start));
 	mas->index = start;
 	mas->last = last;
 }
@@ -580,8 +728,8 @@ static inline void __mas_set_range(struct ma_state *mas, unsigned long start,
 static inline
 void mas_set_range(struct ma_state *mas, unsigned long start, unsigned long last)
 {
+	mas_reset(mas);
 	__mas_set_range(mas, start, last);
-	mas->node = MAS_START;
 }
 
 /**
@@ -706,129 +854,4 @@ void *mt_next(struct maple_tree *mt, unsigned long index, unsigned long max);
 	for (__entry = mt_find(__tree, &(__index), __max); \
 		__entry; __entry = mt_find_after(__tree, &(__index), __max))
 
-
-#ifdef CONFIG_DEBUG_MAPLE_TREE
-enum mt_dump_format {
-	mt_dump_dec,
-	mt_dump_hex,
-};
-
-extern atomic_t maple_tree_tests_run;
-extern atomic_t maple_tree_tests_passed;
-
-void mt_dump(const struct maple_tree *mt, enum mt_dump_format format);
-void mas_dump(const struct ma_state *mas);
-void mas_wr_dump(const struct ma_wr_state *wr_mas);
-void mt_validate(struct maple_tree *mt);
-void mt_cache_shrink(void);
-#define MT_BUG_ON(__tree, __x) do {					\
-	atomic_inc(&maple_tree_tests_run);				\
-	if (__x) {							\
-		pr_info("BUG at %s:%d (%u)\n",				\
-		__func__, __LINE__, __x);				\
-		mt_dump(__tree, mt_dump_hex);				\
-		pr_info("Pass: %u Run:%u\n",				\
-			atomic_read(&maple_tree_tests_passed),		\
-			atomic_read(&maple_tree_tests_run));		\
-		dump_stack();						\
-	} else {							\
-		atomic_inc(&maple_tree_tests_passed);			\
-	}								\
-} while (0)
-
-#define MAS_BUG_ON(__mas, __x) do {					\
-	atomic_inc(&maple_tree_tests_run);				\
-	if (__x) {							\
-		pr_info("BUG at %s:%d (%u)\n",				\
-		__func__, __LINE__, __x);				\
-		mas_dump(__mas);					\
-		mt_dump((__mas)->tree, mt_dump_hex);			\
-		pr_info("Pass: %u Run:%u\n",				\
-			atomic_read(&maple_tree_tests_passed),		\
-			atomic_read(&maple_tree_tests_run));		\
-		dump_stack();						\
-	} else {							\
-		atomic_inc(&maple_tree_tests_passed);			\
-	}								\
-} while (0)
-
-#define MAS_WR_BUG_ON(__wrmas, __x) do {				\
-	atomic_inc(&maple_tree_tests_run);				\
-	if (__x) {							\
-		pr_info("BUG at %s:%d (%u)\n",				\
-		__func__, __LINE__, __x);				\
-		mas_wr_dump(__wrmas);					\
-		mas_dump((__wrmas)->mas);				\
-		mt_dump((__wrmas)->mas->tree, mt_dump_hex);		\
-		pr_info("Pass: %u Run:%u\n",				\
-			atomic_read(&maple_tree_tests_passed),		\
-			atomic_read(&maple_tree_tests_run));		\
-		dump_stack();						\
-	} else {							\
-		atomic_inc(&maple_tree_tests_passed);			\
-	}								\
-} while (0)
-
-#define MT_WARN_ON(__tree, __x)  ({					\
-	int ret = !!(__x);						\
-	atomic_inc(&maple_tree_tests_run);				\
-	if (ret) {							\
-		pr_info("WARN at %s:%d (%u)\n",				\
-		__func__, __LINE__, __x);				\
-		mt_dump(__tree, mt_dump_hex);				\
-		pr_info("Pass: %u Run:%u\n",				\
-			atomic_read(&maple_tree_tests_passed),		\
-			atomic_read(&maple_tree_tests_run));		\
-		dump_stack();						\
-	} else {							\
-		atomic_inc(&maple_tree_tests_passed);			\
-	}								\
-	unlikely(ret);							\
-})
-
-#define MAS_WARN_ON(__mas, __x) ({					\
-	int ret = !!(__x);						\
-	atomic_inc(&maple_tree_tests_run);				\
-	if (ret) {							\
-		pr_info("WARN at %s:%d (%u)\n",				\
-		__func__, __LINE__, __x);				\
-		mas_dump(__mas);					\
-		mt_dump((__mas)->tree, mt_dump_hex);			\
-		pr_info("Pass: %u Run:%u\n",				\
-			atomic_read(&maple_tree_tests_passed),		\
-			atomic_read(&maple_tree_tests_run));		\
-		dump_stack();						\
-	} else {							\
-		atomic_inc(&maple_tree_tests_passed);			\
-	}								\
-	unlikely(ret);							\
-})
-
-#define MAS_WR_WARN_ON(__wrmas, __x) ({					\
-	int ret = !!(__x);						\
-	atomic_inc(&maple_tree_tests_run);				\
-	if (ret) {							\
-		pr_info("WARN at %s:%d (%u)\n",				\
-		__func__, __LINE__, __x);				\
-		mas_wr_dump(__wrmas);					\
-		mas_dump((__wrmas)->mas);				\
-		mt_dump((__wrmas)->mas->tree, mt_dump_hex);		\
-		pr_info("Pass: %u Run:%u\n",				\
-			atomic_read(&maple_tree_tests_passed),		\
-			atomic_read(&maple_tree_tests_run));		\
-		dump_stack();						\
-	} else {							\
-		atomic_inc(&maple_tree_tests_passed);			\
-	}								\
-	unlikely(ret);							\
-})
-#else
-#define MT_BUG_ON(__tree, __x)		BUG_ON(__x)
-#define MAS_BUG_ON(__mas, __x)		BUG_ON(__x)
-#define MAS_WR_BUG_ON(__mas, __x)	BUG_ON(__x)
-#define MT_WARN_ON(__tree, __x)		WARN_ON(__x)
-#define MAS_WARN_ON(__mas, __x)		WARN_ON(__x)
-#define MAS_WR_WARN_ON(__mas, __x)	WARN_ON(__x)
-#endif /* CONFIG_DEBUG_MAPLE_TREE */
-
 #endif /*_LINUX_MAPLE_TREE_H */
diff --git a/include/linux/memblock.h b/include/linux/memblock.h
index ae3bde302f70..b695f9e946da 100644
--- a/include/linux/memblock.h
+++ b/include/linux/memblock.h
@@ -123,6 +123,7 @@ int memblock_physmem_add(phys_addr_t base, phys_addr_t size);
 void memblock_trim_memory(phys_addr_t align);
 bool memblock_overlaps_region(struct memblock_type *type,
 			      phys_addr_t base, phys_addr_t size);
+bool memblock_validate_numa_coverage(unsigned long threshold_bytes);
 int memblock_mark_hotplug(phys_addr_t base, phys_addr_t size);
 int memblock_clear_hotplug(phys_addr_t base, phys_addr_t size);
 int memblock_mark_mirror(phys_addr_t base, phys_addr_t size);
diff --git a/include/linux/memremap.h b/include/linux/memremap.h
index 1314d9c5f05b..744c830f4b13 100644
--- a/include/linux/memremap.h
+++ b/include/linux/memremap.h
@@ -196,8 +196,6 @@ struct dev_pagemap *get_dev_pagemap(unsigned long pfn,
 		struct dev_pagemap *pgmap);
 bool pgmap_pfn_valid(struct dev_pagemap *pgmap, unsigned long pfn);
 
-unsigned long vmem_altmap_offset(struct vmem_altmap *altmap);
-void vmem_altmap_free(struct vmem_altmap *altmap, unsigned long nr_pfns);
 unsigned long memremap_compat_align(void);
 #else
 static inline void *devm_memremap_pages(struct device *dev,
@@ -228,16 +226,6 @@ static inline bool pgmap_pfn_valid(struct dev_pagemap *pgmap, unsigned long pfn)
 	return false;
 }
 
-static inline unsigned long vmem_altmap_offset(struct vmem_altmap *altmap)
-{
-	return 0;
-}
-
-static inline void vmem_altmap_free(struct vmem_altmap *altmap,
-		unsigned long nr_pfns)
-{
-}
-
 /* when memremap_pages() is disabled all archs can remap a single page */
 static inline unsigned long memremap_compat_align(void)
 {
diff --git a/include/linux/mhi.h b/include/linux/mhi.h
index 039943ec4d4e..d0f9b522f328 100644
--- a/include/linux/mhi.h
+++ b/include/linux/mhi.h
@@ -266,6 +266,7 @@ struct mhi_event_config {
  * struct mhi_controller_config - Root MHI controller configuration
  * @max_channels: Maximum number of channels supported
  * @timeout_ms: Timeout value for operations. 0 means use default
+ * @ready_timeout_ms: Timeout value for waiting device to be ready (optional)
  * @buf_len: Size of automatically allocated buffers. 0 means use default
  * @num_channels: Number of channels defined in @ch_cfg
  * @ch_cfg: Array of defined channels
@@ -277,6 +278,7 @@ struct mhi_event_config {
 struct mhi_controller_config {
 	u32 max_channels;
 	u32 timeout_ms;
+	u32 ready_timeout_ms;
 	u32 buf_len;
 	u32 num_channels;
 	const struct mhi_channel_config *ch_cfg;
@@ -330,6 +332,7 @@ struct mhi_controller_config {
  * @pm_mutex: Mutex for suspend/resume operation
  * @pm_lock: Lock for protecting MHI power management state
  * @timeout_ms: Timeout in ms for state transitions
+ * @ready_timeout_ms: Timeout in ms for waiting device to be ready (optional)
  * @pm_state: MHI power management state
  * @db_access: DB access states
  * @ee: MHI device execution environment
@@ -419,6 +422,7 @@ struct mhi_controller {
 	struct mutex pm_mutex;
 	rwlock_t pm_lock;
 	u32 timeout_ms;
+	u32 ready_timeout_ms;
 	u32 pm_state;
 	u32 db_access;
 	enum mhi_ee_type ee;
diff --git a/include/linux/mii_timestamper.h b/include/linux/mii_timestamper.h
index fa940bbaf8ae..26b04f73f214 100644
--- a/include/linux/mii_timestamper.h
+++ b/include/linux/mii_timestamper.h
@@ -9,6 +9,7 @@
 #include <linux/device.h>
 #include <linux/ethtool.h>
 #include <linux/skbuff.h>
+#include <linux/net_tstamp.h>
 
 struct phy_device;
 
@@ -51,7 +52,8 @@ struct mii_timestamper {
 			 struct sk_buff *skb, int type);
 
 	int  (*hwtstamp)(struct mii_timestamper *mii_ts,
-			 struct ifreq *ifreq);
+			 struct kernel_hwtstamp_config *kernel_config,
+			 struct netlink_ext_ack *extack);
 
 	void (*link_state)(struct mii_timestamper *mii_ts,
 			   struct phy_device *phydev);
diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h
index 6f3631425f38..ce2e71cd6d2a 100644
--- a/include/linux/mlx5/mlx5_ifc.h
+++ b/include/linux/mlx5/mlx5_ifc.h
@@ -10103,7 +10103,10 @@ enum {
 struct mlx5_ifc_mtutc_reg_bits {
 	u8         reserved_at_0[0x5];
 	u8         freq_adj_units[0x3];
-	u8         reserved_at_8[0x14];
+	u8         reserved_at_8[0x3];
+	u8         log_max_freq_adjustment[0x5];
+
+	u8         reserved_at_10[0xc];
 	u8         operation[0x4];
 
 	u8         freq_adjustment[0x20];
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 418d26608ece..a422cc123a2d 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -994,6 +994,17 @@ static inline int vma_iter_bulk_alloc(struct vma_iterator *vmi,
 	return mas_expected_entries(&vmi->mas, count);
 }
 
+static inline int vma_iter_clear_gfp(struct vma_iterator *vmi,
+			unsigned long start, unsigned long end, gfp_t gfp)
+{
+	__mas_set_range(&vmi->mas, start, end - 1);
+	mas_store_gfp(&vmi->mas, NULL, gfp);
+	if (unlikely(mas_is_err(&vmi->mas)))
+		return -ENOMEM;
+
+	return 0;
+}
+
 /* Free any unused preallocations */
 static inline void vma_iter_free(struct vma_iterator *vmi)
 {
@@ -2373,7 +2384,8 @@ extern void truncate_pagecache(struct inode *inode, loff_t new);
 extern void truncate_setsize(struct inode *inode, loff_t newsize);
 void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);
 void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);
-int generic_error_remove_page(struct address_space *mapping, struct page *page);
+int generic_error_remove_folio(struct address_space *mapping,
+		struct folio *folio);
 
 struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm,
 		unsigned long address, struct pt_regs *regs);
@@ -3859,6 +3871,32 @@ void vmemmap_free(unsigned long start, unsigned long end,
 		struct vmem_altmap *altmap);
 #endif
 
+#ifdef CONFIG_SPARSEMEM_VMEMMAP
+static inline unsigned long vmem_altmap_offset(struct vmem_altmap *altmap)
+{
+	/* number of pfns from base where pfn_to_page() is valid */
+	if (altmap)
+		return altmap->reserve + altmap->free;
+	return 0;
+}
+
+static inline void vmem_altmap_free(struct vmem_altmap *altmap,
+				    unsigned long nr_pfns)
+{
+	altmap->alloc -= nr_pfns;
+}
+#else
+static inline unsigned long vmem_altmap_offset(struct vmem_altmap *altmap)
+{
+	return 0;
+}
+
+static inline void vmem_altmap_free(struct vmem_altmap *altmap,
+				    unsigned long nr_pfns)
+{
+}
+#endif
+
 #define VMEMMAP_RESERVE_NR	2
 #ifdef CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAP
 static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap,
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 957ce38768b2..fcfa1ea8d11e 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -600,6 +600,9 @@ struct vma_numab_state {
 	 */
 	unsigned long pids_active[2];
 
+	/* MM scan sequence ID when scan first started after VMA creation */
+	int start_scan_seq;
+
 	/*
 	 * MM scan sequence ID when the VMA was last completely scanned.
 	 * A VMA is not eligible for scanning if prev_scan_seq == numa_scan_seq
@@ -1071,7 +1074,8 @@ struct vma_iterator {
 		.mas = {						\
 			.tree = &(__mm)->mm_mt,				\
 			.index = __addr,				\
-			.node = MAS_START,				\
+			.node = NULL,					\
+			.status = ma_start,				\
 		},							\
 	}
 
diff --git a/include/linux/mmc/card.h b/include/linux/mmc/card.h
index 7b12eebc5586..47eeb122524c 100644
--- a/include/linux/mmc/card.h
+++ b/include/linux/mmc/card.h
@@ -52,9 +52,6 @@ struct mmc_ext_csd {
 	u8			part_config;
 	u8			cache_ctrl;
 	u8			rst_n_function;
-	u8			max_packed_writes;
-	u8			max_packed_reads;
-	u8			packed_event_en;
 	unsigned int		part_time;		/* Units: ms */
 	unsigned int		sa_timeout;		/* Units: 100ns */
 	unsigned int		generic_cmd6_time;	/* Units: 10ms */
diff --git a/include/linux/mmc/core.h b/include/linux/mmc/core.h
index 6efec0b9820c..2c7928a50907 100644
--- a/include/linux/mmc/core.h
+++ b/include/linux/mmc/core.h
@@ -27,7 +27,6 @@ struct mmc_command {
 	u32			opcode;
 	u32			arg;
 #define MMC_CMD23_ARG_REL_WR	(1 << 31)
-#define MMC_CMD23_ARG_PACKED	((0 << 31) | (1 << 30))
 #define MMC_CMD23_ARG_TAG_REQ	(1 << 29)
 	u32			resp[4];
 	unsigned int		flags;		/* expected response type */
diff --git a/include/linux/mmc/mmc.h b/include/linux/mmc/mmc.h
index 6f7993803ee7..cf2bcb5da30d 100644
--- a/include/linux/mmc/mmc.h
+++ b/include/linux/mmc/mmc.h
@@ -257,8 +257,6 @@ static inline bool mmc_ready_for_data(u32 status)
 #define EXT_CSD_FLUSH_CACHE		32      /* W */
 #define EXT_CSD_CACHE_CTRL		33      /* R/W */
 #define EXT_CSD_POWER_OFF_NOTIFICATION	34	/* R/W */
-#define EXT_CSD_PACKED_FAILURE_INDEX	35	/* RO */
-#define EXT_CSD_PACKED_CMD_STATUS	36	/* RO */
 #define EXT_CSD_EXP_EVENTS_STATUS	54	/* RO, 2 bytes */
 #define EXT_CSD_EXP_EVENTS_CTRL		56	/* R/W, 2 bytes */
 #define EXT_CSD_DATA_SECTOR_SIZE	61	/* R */
@@ -321,8 +319,6 @@ static inline bool mmc_ready_for_data(u32 status)
 #define EXT_CSD_SUPPORTED_MODE		493	/* RO */
 #define EXT_CSD_TAG_UNIT_SIZE		498	/* RO */
 #define EXT_CSD_DATA_TAG_SUPPORT	499	/* RO */
-#define EXT_CSD_MAX_PACKED_WRITES	500	/* RO */
-#define EXT_CSD_MAX_PACKED_READS	501	/* RO */
 #define EXT_CSD_BKOPS_SUPPORT		502	/* RO */
 #define EXT_CSD_HPI_FEATURES		503	/* RO */
 
@@ -402,18 +398,12 @@ static inline bool mmc_ready_for_data(u32 status)
 #define EXT_CSD_PWR_CL_8BIT_SHIFT	4
 #define EXT_CSD_PWR_CL_4BIT_SHIFT	0
 
-#define EXT_CSD_PACKED_EVENT_EN	BIT(3)
-
 /*
  * EXCEPTION_EVENT_STATUS field
  */
 #define EXT_CSD_URGENT_BKOPS		BIT(0)
 #define EXT_CSD_DYNCAP_NEEDED		BIT(1)
 #define EXT_CSD_SYSPOOL_EXHAUSTED	BIT(2)
-#define EXT_CSD_PACKED_FAILURE		BIT(3)
-
-#define EXT_CSD_PACKED_GENERIC_ERROR	BIT(0)
-#define EXT_CSD_PACKED_INDEXED_ERROR	BIT(1)
 
 /*
  * BKOPS status level
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 3c25226beeed..7b1816450bfc 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -206,6 +206,10 @@ enum node_stat_item {
 #ifdef CONFIG_NUMA_BALANCING
 	PGPROMOTE_SUCCESS,	/* promote successfully */
 	PGPROMOTE_CANDIDATE,	/* candidate pages to promote */
+	/* PGDEMOTE_*: pages demoted */
+	PGDEMOTE_KSWAPD,
+	PGDEMOTE_DIRECT,
+	PGDEMOTE_KHUGEPAGED,
 #endif
 	NR_VM_NODE_STAT_ITEMS
 };
@@ -1789,6 +1793,7 @@ static inline unsigned long section_nr_to_pfn(unsigned long sec)
 #define SUBSECTION_ALIGN_DOWN(pfn) ((pfn) & PAGE_SUBSECTION_MASK)
 
 struct mem_section_usage {
+	struct rcu_head rcu;
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 	DECLARE_BITMAP(subsection_map, SUBSECTIONS_PER_SECTION);
 #endif
@@ -1982,7 +1987,7 @@ static inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn)
 {
 	int idx = subsection_map_index(pfn);
 
-	return test_bit(idx, ms->usage->subsection_map);
+	return test_bit(idx, READ_ONCE(ms->usage)->subsection_map);
 }
 #else
 static inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn)
@@ -2006,6 +2011,7 @@ static inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn)
 static inline int pfn_valid(unsigned long pfn)
 {
 	struct mem_section *ms;
+	int ret;
 
 	/*
 	 * Ensure the upper PAGE_SHIFT bits are clear in the
@@ -2019,13 +2025,19 @@ static inline int pfn_valid(unsigned long pfn)
 	if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS)
 		return 0;
 	ms = __pfn_to_section(pfn);
-	if (!valid_section(ms))
+	rcu_read_lock();
+	if (!valid_section(ms)) {
+		rcu_read_unlock();
 		return 0;
+	}
 	/*
 	 * Traditionally early sections always returned pfn_valid() for
 	 * the entire section-sized span.
 	 */
-	return early_section(ms) || pfn_section_valid(ms, pfn);
+	ret = early_section(ms) || pfn_section_valid(ms, pfn);
+	rcu_read_unlock();
+
+	return ret;
 }
 #endif
 
diff --git a/include/linux/mount.h b/include/linux/mount.h
index ac3dd2876197..c34c18b4e8f3 100644
--- a/include/linux/mount.h
+++ b/include/linux/mount.h
@@ -50,8 +50,7 @@ struct path;
 #define MNT_ATIME_MASK (MNT_NOATIME | MNT_NODIRATIME | MNT_RELATIME )
 
 #define MNT_INTERNAL_FLAGS (MNT_SHARED | MNT_WRITE_HOLD | MNT_INTERNAL | \
-			    MNT_DOOMED | MNT_SYNC_UMOUNT | MNT_MARKED | \
-			    MNT_CURSOR)
+			    MNT_DOOMED | MNT_SYNC_UMOUNT | MNT_MARKED | MNT_ONRB)
 
 #define MNT_INTERNAL	0x4000
 
@@ -65,7 +64,7 @@ struct path;
 #define MNT_SYNC_UMOUNT		0x2000000
 #define MNT_MARKED		0x4000000
 #define MNT_UMOUNT		0x8000000
-#define MNT_CURSOR		0x10000000
+#define MNT_ONRB		0x10000000
 
 struct vfsmount {
 	struct dentry *mnt_root;	/* root of the mounted tree */
diff --git a/include/linux/mutex.h b/include/linux/mutex.h
index a33aa9eb9fc3..95d11308f995 100644
--- a/include/linux/mutex.h
+++ b/include/linux/mutex.h
@@ -221,6 +221,7 @@ extern void mutex_unlock(struct mutex *lock);
 extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock);
 
 DEFINE_GUARD(mutex, struct mutex *, mutex_lock(_T), mutex_unlock(_T))
-DEFINE_FREE(mutex, struct mutex *, if (_T) mutex_unlock(_T))
+DEFINE_GUARD_COND(mutex, _try, mutex_trylock(_T))
+DEFINE_GUARD_COND(mutex, _intr, mutex_lock_interruptible(_T) == 0)
 
 #endif /* __LINUX_MUTEX_H */
diff --git a/include/linux/namei.h b/include/linux/namei.h
index 3100371b5e32..74e0cc14ebf8 100644
--- a/include/linux/namei.h
+++ b/include/linux/namei.h
@@ -66,6 +66,7 @@ extern struct dentry *kern_path_create(int, const char *, struct path *, unsigne
 extern struct dentry *user_path_create(int, const char __user *, struct path *, unsigned int);
 extern void done_path_create(struct path *, struct dentry *);
 extern struct dentry *kern_path_locked(const char *, struct path *);
+extern struct dentry *user_path_locked_at(int , const char __user *, struct path *);
 int vfs_path_parent_lookup(struct filename *filename, unsigned int flags,
 			   struct path *parent, struct qstr *last, int *type,
 			   const struct path *root);
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index a16c9cc063fe..e87caa81f70c 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -1797,6 +1797,13 @@ enum netdev_ml_priv_type {
 	ML_PRIV_CAN,
 };
 
+enum netdev_stat_type {
+	NETDEV_PCPU_STAT_NONE,
+	NETDEV_PCPU_STAT_LSTATS, /* struct pcpu_lstats */
+	NETDEV_PCPU_STAT_TSTATS, /* struct pcpu_sw_netstats */
+	NETDEV_PCPU_STAT_DSTATS, /* struct pcpu_dstats */
+};
+
 /**
  *	struct net_device - The DEVICE structure.
  *
@@ -1991,10 +1998,14 @@ enum netdev_ml_priv_type {
  *
  * 	@ml_priv:	Mid-layer private
  *	@ml_priv_type:  Mid-layer private type
- * 	@lstats:	Loopback statistics
- * 	@tstats:	Tunnel statistics
- * 	@dstats:	Dummy statistics
- * 	@vstats:	Virtual ethernet statistics
+ *
+ *	@pcpu_stat_type:	Type of device statistics which the core should
+ *				allocate/free: none, lstats, tstats, dstats. none
+ *				means the driver is handling statistics allocation/
+ *				freeing internally.
+ *	@lstats:		Loopback statistics: packets, bytes
+ *	@tstats:		Tunnel statistics: RX/TX packets, RX/TX bytes
+ *	@dstats:		Dummy statistics: RX/TX/drop packets, RX/TX bytes
  *
  *	@garp_port:	GARP
  *	@mrp_port:	MRP
@@ -2354,6 +2365,7 @@ struct net_device {
 	void				*ml_priv;
 	enum netdev_ml_priv_type	ml_priv_type;
 
+	enum netdev_stat_type		pcpu_stat_type:8;
 	union {
 		struct pcpu_lstats __percpu		*lstats;
 		struct pcpu_sw_netstats __percpu	*tstats;
@@ -2755,6 +2767,16 @@ struct pcpu_sw_netstats {
 	struct u64_stats_sync   syncp;
 } __aligned(4 * sizeof(u64));
 
+struct pcpu_dstats {
+	u64			rx_packets;
+	u64			rx_bytes;
+	u64			rx_drops;
+	u64			tx_packets;
+	u64			tx_bytes;
+	u64			tx_drops;
+	struct u64_stats_sync	syncp;
+} __aligned(8 * sizeof(u64));
+
 struct pcpu_lstats {
 	u64_stats_t packets;
 	u64_stats_t bytes;
@@ -3942,6 +3964,9 @@ int generic_hwtstamp_get_lower(struct net_device *dev,
 int generic_hwtstamp_set_lower(struct net_device *dev,
 			       struct kernel_hwtstamp_config *kernel_cfg,
 			       struct netlink_ext_ack *extack);
+int dev_set_hwtstamp_phylib(struct net_device *dev,
+			    struct kernel_hwtstamp_config *cfg,
+			    struct netlink_ext_ack *extack);
 int dev_ethtool(struct net *net, struct ifreq *ifr, void __user *userdata);
 unsigned int dev_get_flags(const struct net_device *);
 int __dev_change_flags(struct net_device *dev, unsigned int flags,
diff --git a/include/linux/netlink.h b/include/linux/netlink.h
index 75d7de34c908..abe91ed6b9aa 100644
--- a/include/linux/netlink.h
+++ b/include/linux/netlink.h
@@ -351,5 +351,6 @@ bool netlink_ns_capable(const struct sk_buff *skb,
 			struct user_namespace *ns, int cap);
 bool netlink_capable(const struct sk_buff *skb, int cap);
 bool netlink_net_capable(const struct sk_buff *skb, int cap);
+struct sk_buff *netlink_alloc_large_skb(unsigned int size, int broadcast);
 
 #endif	/* __LINUX_NETLINK_H */
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index a88e64acebfe..735cddc13d20 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -772,19 +772,14 @@ static __always_inline void SetPageUptodate(struct page *page)
 
 CLEARPAGEFLAG(Uptodate, uptodate, PF_NO_TAIL)
 
-bool __folio_start_writeback(struct folio *folio, bool keep_write);
-bool set_page_writeback(struct page *page);
+void __folio_start_writeback(struct folio *folio, bool keep_write);
+void set_page_writeback(struct page *page);
 
 #define folio_start_writeback(folio)			\
 	__folio_start_writeback(folio, false)
 #define folio_start_writeback_keepwrite(folio)	\
 	__folio_start_writeback(folio, true)
 
-static inline bool test_set_page_writeback(struct page *page)
-{
-	return set_page_writeback(page);
-}
-
 static __always_inline bool folio_test_head(struct folio *folio)
 {
 	return test_bit(PG_head, folio_flags(folio, FOLIO_PF_ANY));
diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index bcc1ea44b4e8..2b51e4e3bd36 100644
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -203,7 +203,10 @@ enum mapping_flags {
 	/* writeback related tags are not used */
 	AS_NO_WRITEBACK_TAGS = 5,
 	AS_LARGE_FOLIO_SUPPORT = 6,
-	AS_RELEASE_ALWAYS,	/* Call ->release_folio(), even if no private data */
+	AS_RELEASE_ALWAYS = 7,	/* Call ->release_folio(), even if no private data */
+	AS_UNMOVABLE	= 8,	/* The mapping cannot be moved, ever */
+	AS_STABLE_WRITES,	/* must wait for writeback before modifying
+				   folio contents */
 };
 
 /**
@@ -289,6 +292,37 @@ static inline void mapping_clear_release_always(struct address_space *mapping)
 	clear_bit(AS_RELEASE_ALWAYS, &mapping->flags);
 }
 
+static inline void mapping_set_unmovable(struct address_space *mapping)
+{
+	/*
+	 * It's expected unmovable mappings are also unevictable. Compaction
+	 * migrate scanner (isolate_migratepages_block()) relies on this to
+	 * reduce page locking.
+	 */
+	set_bit(AS_UNEVICTABLE, &mapping->flags);
+	set_bit(AS_UNMOVABLE, &mapping->flags);
+}
+
+static inline bool mapping_unmovable(struct address_space *mapping)
+{
+	return test_bit(AS_UNMOVABLE, &mapping->flags);
+}
+
+static inline bool mapping_stable_writes(const struct address_space *mapping)
+{
+	return test_bit(AS_STABLE_WRITES, &mapping->flags);
+}
+
+static inline void mapping_set_stable_writes(struct address_space *mapping)
+{
+	set_bit(AS_STABLE_WRITES, &mapping->flags);
+}
+
+static inline void mapping_clear_stable_writes(struct address_space *mapping)
+{
+	clear_bit(AS_STABLE_WRITES, &mapping->flags);
+}
+
 static inline gfp_t mapping_gfp_mask(struct address_space * mapping)
 {
 	return mapping->gfp_mask;
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 5547ba68e6e4..d2a15c0c6f8a 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -1143,6 +1143,15 @@ static inline bool branch_sample_priv(const struct perf_event *event)
 	return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_PRIV_SAVE;
 }
 
+static inline bool branch_sample_counters(const struct perf_event *event)
+{
+	return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS;
+}
+
+static inline bool branch_sample_call_stack(const struct perf_event *event)
+{
+	return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK;
+}
 
 struct perf_sample_data {
 	/*
@@ -1177,6 +1186,7 @@ struct perf_sample_data {
 	struct perf_callchain_entry	*callchain;
 	struct perf_raw_record		*raw;
 	struct perf_branch_stack	*br_stack;
+	u64				*br_stack_cntr;
 	union perf_sample_weight	weight;
 	union  perf_mem_data_src	data_src;
 	u64				txn;
@@ -1254,7 +1264,8 @@ static inline void perf_sample_save_raw_data(struct perf_sample_data *data,
 
 static inline void perf_sample_save_brstack(struct perf_sample_data *data,
 					    struct perf_event *event,
-					    struct perf_branch_stack *brs)
+					    struct perf_branch_stack *brs,
+					    u64 *brs_cntr)
 {
 	int size = sizeof(u64); /* nr */
 
@@ -1262,7 +1273,16 @@ static inline void perf_sample_save_brstack(struct perf_sample_data *data,
 		size += sizeof(u64);
 	size += brs->nr * sizeof(struct perf_branch_entry);
 
+	/*
+	 * The extension space for counters is appended after the
+	 * struct perf_branch_stack. It is used to store the occurrences
+	 * of events of each branch.
+	 */
+	if (brs_cntr)
+		size += brs->nr * sizeof(u64);
+
 	data->br_stack = brs;
+	data->br_stack_cntr = brs_cntr;
 	data->dyn_size += size;
 	data->sample_flags |= PERF_SAMPLE_BRANCH_STACK;
 }
diff --git a/include/linux/phy.h b/include/linux/phy.h
index 3cc52826f18e..e5f1f41e399c 100644
--- a/include/linux/phy.h
+++ b/include/linux/phy.h
@@ -1560,9 +1560,11 @@ static inline bool phy_has_txtstamp(struct phy_device *phydev)
 	return phydev && phydev->mii_ts && phydev->mii_ts->txtstamp;
 }
 
-static inline int phy_hwtstamp(struct phy_device *phydev, struct ifreq *ifr)
+static inline int phy_hwtstamp(struct phy_device *phydev,
+			       struct kernel_hwtstamp_config *cfg,
+			       struct netlink_ext_ack *extack)
 {
-	return phydev->mii_ts->hwtstamp(phydev->mii_ts, ifr);
+	return phydev->mii_ts->hwtstamp(phydev->mii_ts, cfg, extack);
 }
 
 static inline bool phy_rxtstamp(struct phy_device *phydev, struct sk_buff *skb,
diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h
index ccd97bcef269..af53101a1383 100644
--- a/include/linux/pm_opp.h
+++ b/include/linux/pm_opp.h
@@ -92,9 +92,12 @@ struct dev_pm_opp_config {
 	struct device ***virt_devs;
 };
 
+#define OPP_LEVEL_UNSET			U32_MAX
+
 /**
  * struct dev_pm_opp_data - The data to use to initialize an OPP.
- * @level: The performance level for the OPP.
+ * @level: The performance level for the OPP. Set level to OPP_LEVEL_UNSET if
+ * level field isn't used.
  * @freq: The clock rate in Hz for the OPP.
  * @u_volt: The voltage in uV for the OPP.
  */
diff --git a/include/linux/power/bq27xxx_battery.h b/include/linux/power/bq27xxx_battery.h
index 7c8d65414a70..7d8025fb74b7 100644
--- a/include/linux/power/bq27xxx_battery.h
+++ b/include/linux/power/bq27xxx_battery.h
@@ -83,5 +83,6 @@ struct bq27xxx_device_info {
 void bq27xxx_battery_update(struct bq27xxx_device_info *di);
 int bq27xxx_battery_setup(struct bq27xxx_device_info *di);
 void bq27xxx_battery_teardown(struct bq27xxx_device_info *di);
+extern const struct dev_pm_ops bq27xxx_battery_battery_pm_ops;
 
 #endif
diff --git a/include/linux/property.h b/include/linux/property.h
index 9f2585d705a8..9040346c620c 100644
--- a/include/linux/property.h
+++ b/include/linux/property.h
@@ -98,6 +98,18 @@ static inline bool device_is_compatible(const struct device *dev, const char *co
 	return fwnode_device_is_compatible(dev_fwnode(dev), compat);
 }
 
+int fwnode_property_match_property_string(const struct fwnode_handle *fwnode,
+					  const char *propname,
+					  const char * const *array, size_t n);
+
+static inline
+int device_property_match_property_string(const struct device *dev,
+					  const char *propname,
+					  const char * const *array, size_t n)
+{
+	return fwnode_property_match_property_string(dev_fwnode(dev), propname, array, n);
+}
+
 int fwnode_property_get_reference_args(const struct fwnode_handle *fwnode,
 				       const char *prop, const char *nargs_prop,
 				       unsigned int nargs, unsigned int index,
@@ -489,6 +501,13 @@ struct software_node {
 	const struct property_entry *properties;
 };
 
+#define SOFTWARE_NODE(_name_, _properties_, _parent_)	\
+	(struct software_node) {			\
+		.name = _name_,				\
+		.properties = _properties_,		\
+		.parent = _parent_,			\
+	}
+
 bool is_software_node(const struct fwnode_handle *fwnode);
 const struct software_node *
 to_software_node(const struct fwnode_handle *fwnode);
diff --git a/include/linux/rcu_notifier.h b/include/linux/rcu_notifier.h
index ebf371364581..5640f024773b 100644
--- a/include/linux/rcu_notifier.h
+++ b/include/linux/rcu_notifier.h
@@ -13,7 +13,7 @@
 #define RCU_STALL_NOTIFY_NORM	1
 #define RCU_STALL_NOTIFY_EXP	2
 
-#ifdef CONFIG_RCU_STALL_COMMON
+#if defined(CONFIG_RCU_STALL_COMMON) && defined(CONFIG_RCU_CPU_STALL_NOTIFIER)
 
 #include <linux/notifier.h>
 #include <linux/types.h>
@@ -21,12 +21,12 @@
 int rcu_stall_chain_notifier_register(struct notifier_block *n);
 int rcu_stall_chain_notifier_unregister(struct notifier_block *n);
 
-#else // #ifdef CONFIG_RCU_STALL_COMMON
+#else // #if defined(CONFIG_RCU_STALL_COMMON) && defined(CONFIG_RCU_CPU_STALL_NOTIFIER)
 
 // No RCU CPU stall warnings in Tiny RCU.
 static inline int rcu_stall_chain_notifier_register(struct notifier_block *n) { return -EEXIST; }
 static inline int rcu_stall_chain_notifier_unregister(struct notifier_block *n) { return -ENOENT; }
 
-#endif // #else // #ifdef CONFIG_RCU_STALL_COMMON
+#endif // #else // #if defined(CONFIG_RCU_STALL_COMMON) && defined(CONFIG_RCU_CPU_STALL_NOTIFIER)
 
 #endif /* __LINUX_RCU_NOTIFIER_H */
diff --git a/include/linux/regulator/machine.h b/include/linux/regulator/machine.h
index 621b7f4a3639..0cd76d264727 100644
--- a/include/linux/regulator/machine.h
+++ b/include/linux/regulator/machine.h
@@ -49,6 +49,13 @@ struct regulator;
 #define DISABLE_IN_SUSPEND	1
 #define ENABLE_IN_SUSPEND	2
 
+/*
+ * Default time window (in milliseconds) following a critical under-voltage
+ * event during which less critical actions can be safely carried out by the
+ * system.
+ */
+#define REGULATOR_DEF_UV_LESS_CRITICAL_WINDOW_MS	10
+
 /* Regulator active discharge flags */
 enum regulator_active_discharge {
 	REGULATOR_ACTIVE_DISCHARGE_DEFAULT,
@@ -127,6 +134,8 @@ struct notification_limit {
  * @ramp_disable: Disable ramp delay when initialising or when setting voltage.
  * @soft_start: Enable soft start so that voltage ramps slowly.
  * @pull_down: Enable pull down when regulator is disabled.
+ * @system_critical: Set if the regulator is critical to system stability or
+ *                   functionality.
  * @over_current_protection: Auto disable on over current event.
  *
  * @over_current_detection: Configure over current limits.
@@ -153,6 +162,13 @@ struct notification_limit {
  *		      regulator_active_discharge values are used for
  *		      initialisation.
  * @enable_time: Turn-on time of the rails (unit: microseconds)
+ * @uv_less_critical_window_ms: Specifies the time window (in milliseconds)
+ *                              following a critical under-voltage (UV) event
+ *                              during which less critical actions can be
+ *                              safely carried out by the system (for example
+ *                              logging). After this time window more critical
+ *                              actions should be done (for example prevent
+ *                              HW damage).
  */
 struct regulation_constraints {
 
@@ -204,6 +220,7 @@ struct regulation_constraints {
 	unsigned int settling_time_up;
 	unsigned int settling_time_down;
 	unsigned int enable_time;
+	unsigned int uv_less_critical_window_ms;
 
 	unsigned int active_discharge;
 
@@ -214,6 +231,7 @@ struct regulation_constraints {
 	unsigned ramp_disable:1; /* disable ramp delay */
 	unsigned soft_start:1;	/* ramp voltage slowly */
 	unsigned pull_down:1;	/* pull down resistor when regulator off */
+	unsigned system_critical:1;	/* critical to system stability */
 	unsigned over_current_protection:1; /* auto disable on over current */
 	unsigned over_current_detection:1; /* notify on over current */
 	unsigned over_voltage_detection:1; /* notify on over voltage */
diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h
index 1dd530ce8b45..9c29689ff505 100644
--- a/include/linux/rwsem.h
+++ b/include/linux/rwsem.h
@@ -203,11 +203,11 @@ extern void up_read(struct rw_semaphore *sem);
 extern void up_write(struct rw_semaphore *sem);
 
 DEFINE_GUARD(rwsem_read, struct rw_semaphore *, down_read(_T), up_read(_T))
-DEFINE_GUARD(rwsem_write, struct rw_semaphore *, down_write(_T), up_write(_T))
-
-DEFINE_FREE(up_read, struct rw_semaphore *, if (_T) up_read(_T))
-DEFINE_FREE(up_write, struct rw_semaphore *, if (_T) up_write(_T))
+DEFINE_GUARD_COND(rwsem_read, _try, down_read_trylock(_T))
+DEFINE_GUARD_COND(rwsem_read, _intr, down_read_interruptible(_T) == 0)
 
+DEFINE_GUARD(rwsem_write, struct rw_semaphore *, down_write(_T), up_write(_T))
+DEFINE_GUARD_COND(rwsem_write, _try, down_write_trylock(_T))
 
 /*
  * downgrade write lock to read lock
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 292c31697248..81a6206a3190 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -63,11 +63,13 @@ struct robust_list_head;
 struct root_domain;
 struct rq;
 struct sched_attr;
+struct sched_dl_entity;
 struct seq_file;
 struct sighand_struct;
 struct signal_struct;
 struct task_delay_info;
 struct task_group;
+struct task_struct;
 struct user_event_mm;
 
 /*
@@ -523,7 +525,7 @@ struct sched_statistics {
 	u64				block_max;
 	s64				sum_block_runtime;
 
-	u64				exec_max;
+	s64				exec_max;
 	u64				slice_max;
 
 	u64				nr_migrations_cold;
@@ -553,7 +555,7 @@ struct sched_entity {
 	struct load_weight		load;
 	struct rb_node			run_node;
 	u64				deadline;
-	u64				min_deadline;
+	u64				min_vruntime;
 
 	struct list_head		group_node;
 	unsigned int			on_rq;
@@ -607,6 +609,9 @@ struct sched_rt_entity {
 #endif
 } __randomize_layout;
 
+typedef bool (*dl_server_has_tasks_f)(struct sched_dl_entity *);
+typedef struct task_struct *(*dl_server_pick_f)(struct sched_dl_entity *);
+
 struct sched_dl_entity {
 	struct rb_node			rb_node;
 
@@ -654,6 +659,7 @@ struct sched_dl_entity {
 	unsigned int			dl_yielded        : 1;
 	unsigned int			dl_non_contending : 1;
 	unsigned int			dl_overrun	  : 1;
+	unsigned int			dl_server         : 1;
 
 	/*
 	 * Bandwidth enforcement timer. Each -deadline task has its
@@ -668,7 +674,20 @@ struct sched_dl_entity {
 	 * timer is needed to decrease the active utilization at the correct
 	 * time.
 	 */
-	struct hrtimer inactive_timer;
+	struct hrtimer			inactive_timer;
+
+	/*
+	 * Bits for DL-server functionality. Also see the comment near
+	 * dl_server_update().
+	 *
+	 * @rq the runqueue this server is for
+	 *
+	 * @server_has_tasks() returns true if @server_pick return a
+	 * runnable task.
+	 */
+	struct rq			*rq;
+	dl_server_has_tasks_f		server_has_tasks;
+	dl_server_pick_f		server_pick;
 
 #ifdef CONFIG_RT_MUTEXES
 	/*
@@ -795,6 +814,7 @@ struct task_struct {
 	struct sched_entity		se;
 	struct sched_rt_entity		rt;
 	struct sched_dl_entity		dl;
+	struct sched_dl_entity		*dl_server;
 	const struct sched_class	*sched_class;
 
 #ifdef CONFIG_SCHED_CORE
@@ -1955,9 +1975,7 @@ extern void ia64_set_curr_task(int cpu, struct task_struct *p);
 void yield(void);
 
 union thread_union {
-#ifndef CONFIG_ARCH_TASK_STRUCT_ON_STACK
 	struct task_struct task;
-#endif
 #ifndef CONFIG_THREAD_INFO_IN_TASK
 	struct thread_info thread_info;
 #endif
diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h
index 3499c1a8b929..015c0e3a3e1d 100644
--- a/include/linux/sched/signal.h
+++ b/include/linux/sched/signal.h
@@ -432,7 +432,6 @@ static inline bool fault_signal_pending(vm_fault_t fault_flags,
  * This is required every time the blocked sigset_t changes.
  * callers must hold sighand->siglock.
  */
-extern void recalc_sigpending_and_wake(struct task_struct *t);
 extern void recalc_sigpending(void);
 extern void calculate_sigpending(void);
 
@@ -646,6 +645,9 @@ extern bool current_is_single_threaded(void);
 #define while_each_thread(g, t) \
 	while ((t = next_thread(t)) != g)
 
+#define for_other_threads(p, t)	\
+	for (t = p; (t = next_thread(t)) != p; )
+
 #define __for_each_thread(signal, t)	\
 	list_for_each_entry_rcu(t, &(signal)->thread_head, thread_node, \
 		lockdep_is_held(&tasklist_lock))
diff --git a/include/linux/security.h b/include/linux/security.h
index 1d1df326c881..750130a7b9dd 100644
--- a/include/linux/security.h
+++ b/include/linux/security.h
@@ -32,6 +32,7 @@
 #include <linux/string.h>
 #include <linux/mm.h>
 #include <linux/sockptr.h>
+#include <uapi/linux/lsm.h>
 
 struct linux_binprm;
 struct cred;
@@ -60,6 +61,7 @@ struct fs_parameter;
 enum fs_value_type;
 struct watch;
 struct watch_notification;
+struct lsm_ctx;
 
 /* Default (no) options for the capable function */
 #define CAP_OPT_NONE 0x0
@@ -138,6 +140,8 @@ enum lockdown_reason {
 };
 
 extern const char *const lockdown_reasons[LOCKDOWN_CONFIDENTIALITY_MAX+1];
+extern u32 lsm_active_cnt;
+extern const struct lsm_id *lsm_idlist[];
 
 /* These functions are in security/commoncap.c */
 extern int cap_capable(const struct cred *cred, struct user_namespace *ns,
@@ -261,6 +265,7 @@ int unregister_blocking_lsm_notifier(struct notifier_block *nb);
 /* prototypes */
 extern int security_init(void);
 extern int early_security_init(void);
+extern u64 lsm_name_to_attr(const char *name);
 
 /* Security operations */
 int security_binder_set_context_mgr(const struct cred *mgr);
@@ -470,10 +475,13 @@ int security_sem_semctl(struct kern_ipc_perm *sma, int cmd);
 int security_sem_semop(struct kern_ipc_perm *sma, struct sembuf *sops,
 			unsigned nsops, int alter);
 void security_d_instantiate(struct dentry *dentry, struct inode *inode);
-int security_getprocattr(struct task_struct *p, const char *lsm, const char *name,
+int security_getselfattr(unsigned int attr, struct lsm_ctx __user *ctx,
+			 size_t __user *size, u32 flags);
+int security_setselfattr(unsigned int attr, struct lsm_ctx __user *ctx,
+			 size_t size, u32 flags);
+int security_getprocattr(struct task_struct *p, int lsmid, const char *name,
 			 char **value);
-int security_setprocattr(const char *lsm, const char *name, void *value,
-			 size_t size);
+int security_setprocattr(int lsmid, const char *name, void *value, size_t size);
 int security_netlink_send(struct sock *sk, struct sk_buff *skb);
 int security_ismaclabel(const char *name);
 int security_secid_to_secctx(u32 secid, char **secdata, u32 *seclen);
@@ -484,6 +492,8 @@ int security_inode_notifysecctx(struct inode *inode, void *ctx, u32 ctxlen);
 int security_inode_setsecctx(struct dentry *dentry, void *ctx, u32 ctxlen);
 int security_inode_getsecctx(struct inode *inode, void **ctx, u32 *ctxlen);
 int security_locked_down(enum lockdown_reason what);
+int lsm_fill_user_ctx(struct lsm_ctx __user *uctx, size_t *uctx_len,
+		      void *val, size_t val_len, u64 id, u64 flags);
 #else /* CONFIG_SECURITY */
 
 static inline int call_blocking_lsm_notifier(enum lsm_event event, void *data)
@@ -501,6 +511,11 @@ static inline  int unregister_blocking_lsm_notifier(struct notifier_block *nb)
 	return 0;
 }
 
+static inline u64 lsm_name_to_attr(const char *name)
+{
+	return LSM_ATTR_UNDEF;
+}
+
 static inline void security_free_mnt_opts(void **mnt_opts)
 {
 }
@@ -1337,14 +1352,28 @@ static inline void security_d_instantiate(struct dentry *dentry,
 					  struct inode *inode)
 { }
 
-static inline int security_getprocattr(struct task_struct *p, const char *lsm,
+static inline int security_getselfattr(unsigned int attr,
+				       struct lsm_ctx __user *ctx,
+				       size_t __user *size, u32 flags)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int security_setselfattr(unsigned int attr,
+				       struct lsm_ctx __user *ctx,
+				       size_t size, u32 flags)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int security_getprocattr(struct task_struct *p, int lsmid,
 				       const char *name, char **value)
 {
 	return -EINVAL;
 }
 
-static inline int security_setprocattr(const char *lsm, char *name,
-				       void *value, size_t size)
+static inline int security_setprocattr(int lsmid, char *name, void *value,
+				       size_t size)
 {
 	return -EINVAL;
 }
@@ -1395,6 +1424,12 @@ static inline int security_locked_down(enum lockdown_reason what)
 {
 	return 0;
 }
+static inline int lsm_fill_user_ctx(struct lsm_ctx __user *uctx,
+				    size_t *uctx_len, void *val, size_t val_len,
+				    u64 id, u64 flags)
+{
+	return -EOPNOTSUPP;
+}
 #endif	/* CONFIG_SECURITY */
 
 #if defined(CONFIG_SECURITY) && defined(CONFIG_WATCH_QUEUE)
diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h
index 255a0562aea5..7b4baff63c5c 100644
--- a/include/linux/spi/spi.h
+++ b/include/linux/spi/spi.h
@@ -1638,8 +1638,6 @@ spi_transfer_is_last(struct spi_controller *ctlr, struct spi_transfer *xfer)
 /* Compatibility layer */
 #define spi_master			spi_controller
 
-#define SPI_MASTER_HALF_DUPLEX		SPI_CONTROLLER_HALF_DUPLEX
-
 #define spi_master_get_devdata(_ctlr)	spi_controller_get_devdata(_ctlr)
 #define spi_master_set_devdata(_ctlr, _data)	\
 	spi_controller_set_devdata(_ctlr, _data)
diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h
index 31d3d747a9db..ceb56b39c70f 100644
--- a/include/linux/spinlock.h
+++ b/include/linux/spinlock.h
@@ -507,6 +507,8 @@ DEFINE_LOCK_GUARD_1(raw_spinlock, raw_spinlock_t,
 		    raw_spin_lock(_T->lock),
 		    raw_spin_unlock(_T->lock))
 
+DEFINE_LOCK_GUARD_1_COND(raw_spinlock, _try, raw_spin_trylock(_T->lock))
+
 DEFINE_LOCK_GUARD_1(raw_spinlock_nested, raw_spinlock_t,
 		    raw_spin_lock_nested(_T->lock, SINGLE_DEPTH_NESTING),
 		    raw_spin_unlock(_T->lock))
@@ -515,23 +517,36 @@ DEFINE_LOCK_GUARD_1(raw_spinlock_irq, raw_spinlock_t,
 		    raw_spin_lock_irq(_T->lock),
 		    raw_spin_unlock_irq(_T->lock))
 
+DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irq, _try, raw_spin_trylock_irq(_T->lock))
+
 DEFINE_LOCK_GUARD_1(raw_spinlock_irqsave, raw_spinlock_t,
 		    raw_spin_lock_irqsave(_T->lock, _T->flags),
 		    raw_spin_unlock_irqrestore(_T->lock, _T->flags),
 		    unsigned long flags)
 
+DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irqsave, _try,
+			 raw_spin_trylock_irqsave(_T->lock, _T->flags))
+
 DEFINE_LOCK_GUARD_1(spinlock, spinlock_t,
 		    spin_lock(_T->lock),
 		    spin_unlock(_T->lock))
 
+DEFINE_LOCK_GUARD_1_COND(spinlock, _try, spin_trylock(_T->lock))
+
 DEFINE_LOCK_GUARD_1(spinlock_irq, spinlock_t,
 		    spin_lock_irq(_T->lock),
 		    spin_unlock_irq(_T->lock))
 
+DEFINE_LOCK_GUARD_1_COND(spinlock_irq, _try,
+			 spin_trylock_irq(_T->lock))
+
 DEFINE_LOCK_GUARD_1(spinlock_irqsave, spinlock_t,
 		    spin_lock_irqsave(_T->lock, _T->flags),
 		    spin_unlock_irqrestore(_T->lock, _T->flags),
 		    unsigned long flags)
 
+DEFINE_LOCK_GUARD_1_COND(spinlock_irqsave, _try,
+			 spin_trylock_irqsave(_T->lock, _T->flags))
+
 #undef __LINUX_INSIDE_SPINLOCK_H
 #endif /* __LINUX_SPINLOCK_H */
diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h
index e58306783d8e..a6796f178913 100644
--- a/include/linux/stackdepot.h
+++ b/include/linux/stackdepot.h
@@ -32,6 +32,18 @@ typedef u32 depot_stack_handle_t;
  */
 #define STACK_DEPOT_EXTRA_BITS 5
 
+typedef u32 depot_flags_t;
+
+/*
+ * Flags that can be passed to stack_depot_save_flags(); see the comment next
+ * to its declaration for more details.
+ */
+#define STACK_DEPOT_FLAG_CAN_ALLOC	((depot_flags_t)0x0001)
+#define STACK_DEPOT_FLAG_GET		((depot_flags_t)0x0002)
+
+#define STACK_DEPOT_FLAGS_NUM	2
+#define STACK_DEPOT_FLAGS_MASK	((depot_flags_t)((1 << STACK_DEPOT_FLAGS_NUM) - 1))
+
 /*
  * Using stack depot requires its initialization, which can be done in 3 ways:
  *
@@ -69,31 +81,39 @@ static inline int stack_depot_early_init(void)	{ return 0; }
 #endif
 
 /**
- * __stack_depot_save - Save a stack trace to stack depot
+ * stack_depot_save_flags - Save a stack trace to stack depot
  *
  * @entries:		Pointer to the stack trace
  * @nr_entries:		Number of frames in the stack
  * @alloc_flags:	Allocation GFP flags
- * @can_alloc:		Allocate stack pools (increased chance of failure if false)
+ * @depot_flags:	Stack depot flags
+ *
+ * Saves a stack trace from @entries array of size @nr_entries.
+ *
+ * If STACK_DEPOT_FLAG_CAN_ALLOC is set in @depot_flags, stack depot can
+ * replenish the stack pools in case no space is left (allocates using GFP
+ * flags of @alloc_flags). Otherwise, stack depot avoids any allocations and
+ * fails if no space is left to store the stack trace.
  *
- * Saves a stack trace from @entries array of size @nr_entries. If @can_alloc is
- * %true, stack depot can replenish the stack pools in case no space is left
- * (allocates using GFP flags of @alloc_flags). If @can_alloc is %false, avoids
- * any allocations and fails if no space is left to store the stack trace.
+ * If STACK_DEPOT_FLAG_GET is set in @depot_flags, stack depot will increment
+ * the refcount on the saved stack trace if it already exists in stack depot.
+ * Users of this flag must also call stack_depot_put() when keeping the stack
+ * trace is no longer required to avoid overflowing the refcount.
  *
  * If the provided stack trace comes from the interrupt context, only the part
  * up to the interrupt entry is saved.
  *
- * Context: Any context, but setting @can_alloc to %false is required if
+ * Context: Any context, but setting STACK_DEPOT_FLAG_CAN_ALLOC is required if
  *          alloc_pages() cannot be used from the current context. Currently
  *          this is the case for contexts where neither %GFP_ATOMIC nor
  *          %GFP_NOWAIT can be used (NMI, raw_spin_lock).
  *
  * Return: Handle of the stack struct stored in depot, 0 on failure
  */
-depot_stack_handle_t __stack_depot_save(unsigned long *entries,
-					unsigned int nr_entries,
-					gfp_t gfp_flags, bool can_alloc);
+depot_stack_handle_t stack_depot_save_flags(unsigned long *entries,
+					    unsigned int nr_entries,
+					    gfp_t gfp_flags,
+					    depot_flags_t depot_flags);
 
 /**
  * stack_depot_save - Save a stack trace to stack depot
@@ -102,8 +122,11 @@ depot_stack_handle_t __stack_depot_save(unsigned long *entries,
  * @nr_entries:		Number of frames in the stack
  * @alloc_flags:	Allocation GFP flags
  *
- * Context: Contexts where allocations via alloc_pages() are allowed.
- *          See __stack_depot_save() for more details.
+ * Does not increment the refcount on the saved stack trace; see
+ * stack_depot_save_flags() for more details.
+ *
+ * Context: Contexts where allocations via alloc_pages() are allowed;
+ *          see stack_depot_save_flags() for more details.
  *
  * Return: Handle of the stack trace stored in depot, 0 on failure
  */
@@ -141,6 +164,18 @@ void stack_depot_print(depot_stack_handle_t stack);
 int stack_depot_snprint(depot_stack_handle_t handle, char *buf, size_t size,
 		       int spaces);
 
+/**
+ * stack_depot_put - Drop a reference to a stack trace from stack depot
+ *
+ * @handle:	Stack depot handle returned from stack_depot_save()
+ *
+ * The stack trace is evicted from stack depot once all references to it have
+ * been dropped (once the number of stack_depot_evict() calls matches the
+ * number of stack_depot_save_flags() calls with STACK_DEPOT_FLAG_GET set for
+ * this stack trace).
+ */
+void stack_depot_put(depot_stack_handle_t handle);
+
 /**
  * stack_depot_set_extra_bits - Set extra bits in a stack depot handle
  *
diff --git a/include/linux/swap.h b/include/linux/swap.h
index f6dd6575b905..3ba146ae7cf5 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -659,6 +659,7 @@ static inline void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_p
 }
 
 extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg);
+extern long mem_cgroup_get_nr_swapcache_pages(struct mem_cgroup *memcg);
 extern bool mem_cgroup_swap_full(struct folio *folio);
 #else
 static inline void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry)
@@ -681,6 +682,11 @@ static inline long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
 	return get_nr_swap_pages();
 }
 
+static inline long mem_cgroup_get_nr_swapcache_pages(struct mem_cgroup *memcg)
+{
+	return total_swapcache_pages();
+}
+
 static inline bool mem_cgroup_swap_full(struct folio *folio)
 {
 	return vm_swap_full();
diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h
index fd9d12de7e92..5c0dbef55792 100644
--- a/include/linux/syscalls.h
+++ b/include/linux/syscalls.h
@@ -71,9 +71,12 @@ struct clone_args;
 struct open_how;
 struct mount_attr;
 struct landlock_ruleset_attr;
+struct lsm_ctx;
 enum landlock_rule_type;
 struct cachestat_range;
 struct cachestat;
+struct statmount;
+struct mnt_id_req;
 
 #include <linux/types.h>
 #include <linux/aio_abi.h>
@@ -407,6 +410,12 @@ asmlinkage long sys_statfs64(const char __user *path, size_t sz,
 asmlinkage long sys_fstatfs(unsigned int fd, struct statfs __user *buf);
 asmlinkage long sys_fstatfs64(unsigned int fd, size_t sz,
 				struct statfs64 __user *buf);
+asmlinkage long sys_statmount(const struct mnt_id_req __user *req,
+			      struct statmount __user *buf, size_t bufsize,
+			      unsigned int flags);
+asmlinkage long sys_listmount(const struct mnt_id_req __user *req,
+			      u64 __user *buf, size_t bufsize,
+			      unsigned int flags);
 asmlinkage long sys_truncate(const char __user *path, long length);
 asmlinkage long sys_ftruncate(unsigned int fd, unsigned long length);
 #if BITS_PER_LONG == 32
@@ -949,6 +958,11 @@ asmlinkage long sys_cachestat(unsigned int fd,
 		struct cachestat_range __user *cstat_range,
 		struct cachestat __user *cstat, unsigned int flags);
 asmlinkage long sys_map_shadow_stack(unsigned long addr, unsigned long size, unsigned int flags);
+asmlinkage long sys_lsm_get_self_attr(unsigned int attr, struct lsm_ctx *ctx,
+				      size_t *size, __u32 flags);
+asmlinkage long sys_lsm_set_self_attr(unsigned int attr, struct lsm_ctx *ctx,
+				      size_t size, __u32 flags);
+asmlinkage long sys_lsm_list_modules(u64 *ids, size_t *size, u32 flags);
 
 /*
  * Architecture-specific system calls
diff --git a/include/linux/tee_drv.h b/include/linux/tee_drv.h
index 17eb1c5205d3..911ddf92dcee 100644
--- a/include/linux/tee_drv.h
+++ b/include/linux/tee_drv.h
@@ -84,6 +84,7 @@ struct tee_param {
  * @release:		release this open file
  * @open_session:	open a new session
  * @close_session:	close a session
+ * @system_session:	declare session as a system session
  * @invoke_func:	invoke a trusted function
  * @cancel_req:		request cancel of an ongoing invoke or open
  * @supp_recv:		called for supplicant to get a command
@@ -100,6 +101,7 @@ struct tee_driver_ops {
 			    struct tee_ioctl_open_session_arg *arg,
 			    struct tee_param *param);
 	int (*close_session)(struct tee_context *ctx, u32 session);
+	int (*system_session)(struct tee_context *ctx, u32 session);
 	int (*invoke_func)(struct tee_context *ctx,
 			   struct tee_ioctl_invoke_arg *arg,
 			   struct tee_param *param);
@@ -429,6 +431,20 @@ int tee_client_open_session(struct tee_context *ctx,
  */
 int tee_client_close_session(struct tee_context *ctx, u32 session);
 
+/**
+ * tee_client_system_session() - Declare session as a system session
+ * @ctx:	TEE Context
+ * @session:	Session id
+ *
+ * This function requests TEE to provision an entry context ready to use for
+ * that session only. The provisioned entry context is used for command
+ * invocation and session closure, not for command cancelling requests.
+ * TEE releases the provisioned context upon session closure.
+ *
+ * Return < 0 on error else 0 if an entry context has been provisioned.
+ */
+int tee_client_system_session(struct tee_context *ctx, u32 session);
+
 /**
  * tee_client_invoke_func() - Invoke a function in a Trusted Application
  * @ctx:	TEE Context
diff --git a/include/linux/thermal.h b/include/linux/thermal.h
index cee814d5d1ac..1f9ee869f9f9 100644
--- a/include/linux/thermal.h
+++ b/include/linux/thermal.h
@@ -57,12 +57,14 @@ enum thermal_notify_event {
  * struct thermal_trip - representation of a point in temperature domain
  * @temperature: temperature value in miliCelsius
  * @hysteresis: relative hysteresis in miliCelsius
+ * @threshold: trip crossing notification threshold miliCelsius
  * @type: trip point type
  * @priv: pointer to driver data associated with this trip
  */
 struct thermal_trip {
 	int temperature;
 	int hysteresis;
+	int threshold;
 	enum thermal_trip_type type;
 	void *priv;
 };
diff --git a/include/linux/tnum.h b/include/linux/tnum.h
index 1c3948a1d6ad..3c13240077b8 100644
--- a/include/linux/tnum.h
+++ b/include/linux/tnum.h
@@ -106,6 +106,10 @@ int tnum_sbin(char *str, size_t size, struct tnum a);
 struct tnum tnum_subreg(struct tnum a);
 /* Returns the tnum with the lower 32-bit subreg cleared */
 struct tnum tnum_clear_subreg(struct tnum a);
+/* Returns the tnum with the lower 32-bit subreg in *reg* set to the lower
+ * 32-bit subreg in *subreg*
+ */
+struct tnum tnum_with_subreg(struct tnum reg, struct tnum subreg);
 /* Returns the tnum with the lower 32-bit subreg set to value */
 struct tnum tnum_const_subreg(struct tnum a, u32 value);
 /* Returns true if 32-bit subreg @a is a known constant*/
diff --git a/include/linux/usb/phy.h b/include/linux/usb/phy.h
index b513749582d7..e4de6bc1f69b 100644
--- a/include/linux/usb/phy.h
+++ b/include/linux/usb/phy.h
@@ -144,10 +144,6 @@ struct usb_phy {
 	 */
 	int	(*set_wakeup)(struct usb_phy *x, bool enabled);
 
-	/* notify phy port status change */
-	int	(*notify_port_status)(struct usb_phy *x, int port,
-				      u16 portstatus, u16 portchange);
-
 	/* notify phy connect status change */
 	int	(*notify_connect)(struct usb_phy *x,
 			enum usb_device_speed speed);
@@ -320,15 +316,6 @@ usb_phy_set_wakeup(struct usb_phy *x, bool enabled)
 		return 0;
 }
 
-static inline int
-usb_phy_notify_port_status(struct usb_phy *x, int port, u16 portstatus, u16 portchange)
-{
-	if (x && x->notify_port_status)
-		return x->notify_port_status(x, port, portstatus, portchange);
-	else
-		return 0;
-}
-
 static inline int
 usb_phy_notify_connect(struct usb_phy *x, enum usb_device_speed speed)
 {
diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index 8abfa1240040..d1b847502f09 100644
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -41,9 +41,6 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
 		PGSTEAL_KSWAPD,
 		PGSTEAL_DIRECT,
 		PGSTEAL_KHUGEPAGED,
-		PGDEMOTE_KSWAPD,
-		PGDEMOTE_DIRECT,
-		PGDEMOTE_KHUGEPAGED,
 		PGSCAN_KSWAPD,
 		PGSCAN_DIRECT,
 		PGSCAN_KHUGEPAGED,
diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
index 24b1e5070f4d..b0b9604b76b8 100644
--- a/include/linux/workqueue.h
+++ b/include/linux/workqueue.h
@@ -491,7 +491,7 @@ struct workqueue_attrs *alloc_workqueue_attrs(void);
 void free_workqueue_attrs(struct workqueue_attrs *attrs);
 int apply_workqueue_attrs(struct workqueue_struct *wq,
 			  const struct workqueue_attrs *attrs);
-int workqueue_set_unbound_cpumask(cpumask_var_t cpumask);
+extern int workqueue_unbound_exclude_cpumask(cpumask_var_t cpumask);
 
 extern bool queue_work_on(int cpu, struct workqueue_struct *wq,
 			struct work_struct *work);
diff --git a/include/linux/writeback.h b/include/linux/writeback.h
index 083387c00f0c..6d0a14f7019d 100644
--- a/include/linux/writeback.h
+++ b/include/linux/writeback.h
@@ -193,7 +193,6 @@ void inode_io_list_del(struct inode *inode);
 /* writeback.h requires fs.h; it, too, is not included from here. */
 static inline void wait_on_inode(struct inode *inode)
 {
-	might_sleep();
 	wait_on_bit(&inode->i_state, __I_NEW, TASK_UNINTERRUPTIBLE);
 }
 
diff --git a/include/linux/zstd.h b/include/linux/zstd.h
index 113408eef6ec..f109d49f43f8 100644
--- a/include/linux/zstd.h
+++ b/include/linux/zstd.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/include/linux/zstd_errors.h b/include/linux/zstd_errors.h
index 58b6dd45a969..6d5cf55f0bf3 100644
--- a/include/linux/zstd_errors.h
+++ b/include/linux/zstd_errors.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -17,8 +18,17 @@
 
 
 /* =====   ZSTDERRORLIB_API : control library symbols visibility   ===== */
-#define ZSTDERRORLIB_VISIBILITY 
-#define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY
+#define ZSTDERRORLIB_VISIBLE 
+
+#ifndef ZSTDERRORLIB_HIDDEN
+#  if (__GNUC__ >= 4) && !defined(__MINGW32__)
+#    define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden")))
+#  else
+#    define ZSTDERRORLIB_HIDDEN
+#  endif
+#endif
+
+#define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE
 
 /*-*********************************************
  *  Error codes list
@@ -43,14 +53,17 @@ typedef enum {
   ZSTD_error_frameParameter_windowTooLarge = 16,
   ZSTD_error_corruption_detected = 20,
   ZSTD_error_checksum_wrong      = 22,
+  ZSTD_error_literals_headerWrong = 24,
   ZSTD_error_dictionary_corrupted      = 30,
   ZSTD_error_dictionary_wrong          = 32,
   ZSTD_error_dictionaryCreation_failed = 34,
   ZSTD_error_parameter_unsupported   = 40,
+  ZSTD_error_parameter_combination_unsupported = 41,
   ZSTD_error_parameter_outOfBound    = 42,
   ZSTD_error_tableLog_tooLarge       = 44,
   ZSTD_error_maxSymbolValue_tooLarge = 46,
   ZSTD_error_maxSymbolValue_tooSmall = 48,
+  ZSTD_error_stabilityCondition_notRespected = 50,
   ZSTD_error_stage_wrong       = 60,
   ZSTD_error_init_missing      = 62,
   ZSTD_error_memory_allocation = 64,
@@ -58,11 +71,15 @@ typedef enum {
   ZSTD_error_dstSize_tooSmall = 70,
   ZSTD_error_srcSize_wrong    = 72,
   ZSTD_error_dstBuffer_null   = 74,
+  ZSTD_error_noForwardProgress_destFull = 80,
+  ZSTD_error_noForwardProgress_inputEmpty = 82,
   /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */
   ZSTD_error_frameIndex_tooLarge = 100,
   ZSTD_error_seekableIO          = 102,
   ZSTD_error_dstBuffer_wrong     = 104,
   ZSTD_error_srcBuffer_wrong     = 105,
+  ZSTD_error_sequenceProducer_failed = 106,
+  ZSTD_error_externalSequences_invalid = 107,
   ZSTD_error_maxCode = 120  /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */
 } ZSTD_ErrorCode;
 
diff --git a/include/linux/zstd_lib.h b/include/linux/zstd_lib.h
index 79d55465d5c1..8b4ffe649df5 100644
--- a/include/linux/zstd_lib.h
+++ b/include/linux/zstd_lib.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -11,23 +12,42 @@
 #ifndef ZSTD_H_235446
 #define ZSTD_H_235446
 
-/* ======   Dependency   ======*/
+/* ======   Dependencies   ======*/
 #include <linux/limits.h>   /* INT_MAX */
 #include <linux/types.h>   /* size_t */
 
 
 /* =====   ZSTDLIB_API : control library symbols visibility   ===== */
-#ifndef ZSTDLIB_VISIBLE
+#define ZSTDLIB_VISIBLE 
+
+#ifndef ZSTDLIB_HIDDEN
 #  if (__GNUC__ >= 4) && !defined(__MINGW32__)
-#    define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default")))
 #    define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden")))
 #  else
-#    define ZSTDLIB_VISIBLE
 #    define ZSTDLIB_HIDDEN
 #  endif
 #endif
+
 #define ZSTDLIB_API ZSTDLIB_VISIBLE
 
+/* Deprecation warnings :
+ * Should these warnings be a problem, it is generally possible to disable them,
+ * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual.
+ * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS.
+ */
+#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS
+#  define ZSTD_DEPRECATED(message) /* disable deprecation warnings */
+#else
+#  if (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__)
+#    define ZSTD_DEPRECATED(message) __attribute__((deprecated(message)))
+#  elif (__GNUC__ >= 3)
+#    define ZSTD_DEPRECATED(message) __attribute__((deprecated))
+#  else
+#    pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler")
+#    define ZSTD_DEPRECATED(message)
+#  endif
+#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */
+
 
 /* *****************************************************************************
   Introduction
@@ -65,7 +85,7 @@
 /*------   Version   ------*/
 #define ZSTD_VERSION_MAJOR    1
 #define ZSTD_VERSION_MINOR    5
-#define ZSTD_VERSION_RELEASE  2
+#define ZSTD_VERSION_RELEASE  5
 #define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
 
 /*! ZSTD_versionNumber() :
@@ -107,7 +127,8 @@ ZSTDLIB_API const char* ZSTD_versionString(void);
 ***************************************/
 /*! ZSTD_compress() :
  *  Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
- *  Hint : compression runs faster if `dstCapacity` >=  `ZSTD_compressBound(srcSize)`.
+ *  NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have
+ *        enough space to successfully compress the data.
  *  @return : compressed size written into `dst` (<= `dstCapacity),
  *            or an error code if it fails (which can be tested using ZSTD_isError()). */
 ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
@@ -156,7 +177,9 @@ ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t
  *  "empty", "unknown" and "error" results to the same return value (0),
  *  while ZSTD_getFrameContentSize() gives them separate return values.
  * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */
-ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
+ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize")
+ZSTDLIB_API
+unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
 
 /*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+
  * `src` should point to the start of a ZSTD frame or skippable frame.
@@ -168,8 +191,30 @@ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
 
 
 /*======  Helper functions  ======*/
-#define ZSTD_COMPRESSBOUND(srcSize)   ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
-ZSTDLIB_API size_t      ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
+/* ZSTD_compressBound() :
+ * maximum compressed size in worst case single-pass scenario.
+ * When invoking `ZSTD_compress()` or any other one-pass compression function,
+ * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize)
+ * as it eliminates one potential failure scenario,
+ * aka not enough room in dst buffer to write the compressed frame.
+ * Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE .
+ *        In which case, ZSTD_compressBound() will return an error code
+ *        which can be tested using ZSTD_isError().
+ *
+ * ZSTD_COMPRESSBOUND() :
+ * same as ZSTD_compressBound(), but as a macro.
+ * It can be used to produce constants, which can be useful for static allocation,
+ * for example to size a static array on stack.
+ * Will produce constant value 0 if srcSize too large.
+ */
+#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00LLU : 0xFF00FF00U)
+#define ZSTD_COMPRESSBOUND(srcSize)   (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0))  /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */
+ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
+/* ZSTD_isError() :
+ * Most ZSTD_* functions returning a size_t value can be tested for error,
+ * using ZSTD_isError().
+ * @return 1 if error, 0 otherwise
+ */
 ZSTDLIB_API unsigned    ZSTD_isError(size_t code);          /*!< tells if a `size_t` function result is an error code */
 ZSTDLIB_API const char* ZSTD_getErrorName(size_t code);     /*!< provides readable string from an error code */
 ZSTDLIB_API int         ZSTD_minCLevel(void);               /*!< minimum negative compression level allowed, requires v1.4.0+ */
@@ -412,6 +457,9 @@ typedef enum {
      * ZSTD_c_validateSequences
      * ZSTD_c_useBlockSplitter
      * ZSTD_c_useRowMatchFinder
+     * ZSTD_c_prefetchCDictTables
+     * ZSTD_c_enableSeqProducerFallback
+     * ZSTD_c_maxBlockSize
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly;
      *        also, the enums values themselves are unstable and can still change.
@@ -430,7 +478,11 @@ typedef enum {
      ZSTD_c_experimentalParam12=1009,
      ZSTD_c_experimentalParam13=1010,
      ZSTD_c_experimentalParam14=1011,
-     ZSTD_c_experimentalParam15=1012
+     ZSTD_c_experimentalParam15=1012,
+     ZSTD_c_experimentalParam16=1013,
+     ZSTD_c_experimentalParam17=1014,
+     ZSTD_c_experimentalParam18=1015,
+     ZSTD_c_experimentalParam19=1016
 } ZSTD_cParameter;
 
 typedef struct {
@@ -493,7 +545,7 @@ typedef enum {
  *                  They will be used to compress next frame.
  *                  Resetting session never fails.
  *  - The parameters : changes all parameters back to "default".
- *                  This removes any reference to any dictionary too.
+ *                  This also removes any reference to any dictionary or external sequence producer.
  *                  Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing)
  *                  otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError())
  *  - Both : similar to resetting the session, followed by resetting parameters.
@@ -506,7 +558,8 @@ ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset);
  *  Should cctx hold data from a previously unfinished frame, everything about it is forgotten.
  *  - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
  *  - The function is always blocking, returns when compression is completed.
- *  Hint : compression runs faster if `dstCapacity` >=  `ZSTD_compressBound(srcSize)`.
+ *  NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have
+ *        enough space to successfully compress the data, though it is possible it fails for other reasons.
  * @return : compressed size written into `dst` (<= `dstCapacity),
  *           or an error code if it fails (which can be tested using ZSTD_isError()).
  */
@@ -543,13 +596,15 @@ typedef enum {
      * ZSTD_d_stableOutBuffer
      * ZSTD_d_forceIgnoreChecksum
      * ZSTD_d_refMultipleDDicts
+     * ZSTD_d_disableHuffmanAssembly
      * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
      * note : never ever use experimentalParam? names directly
      */
      ZSTD_d_experimentalParam1=1000,
      ZSTD_d_experimentalParam2=1001,
      ZSTD_d_experimentalParam3=1002,
-     ZSTD_d_experimentalParam4=1003
+     ZSTD_d_experimentalParam4=1003,
+     ZSTD_d_experimentalParam5=1004
 
 } ZSTD_dParameter;
 
@@ -728,8 +783,6 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /*< recommended size for output
  * This following is a legacy streaming API, available since v1.0+ .
  * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2().
  * It is redundant, but remains fully supported.
- * Streaming in combination with advanced parameters and dictionary compression
- * can only be used through the new API.
  ******************************************************************************/
 
 /*!
@@ -738,6 +791,9 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /*< recommended size for output
  *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
  *     ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any)
  *     ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel);
+ *
+ * Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API
+ * to compress with a dictionary.
  */
 ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
 /*!
@@ -788,13 +844,31 @@ ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);  /* accept NULL pointer
 
 /*===== Streaming decompression functions =====*/
 
-/* This function is redundant with the advanced API and equivalent to:
+/*! ZSTD_initDStream() :
+ * Initialize/reset DStream state for new decompression operation.
+ * Call before new decompression operation using same DStream.
  *
+ * Note : This function is redundant with the advanced API and equivalent to:
  *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
  *     ZSTD_DCtx_refDDict(zds, NULL);
  */
 ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
 
+/*! ZSTD_decompressStream() :
+ * Streaming decompression function.
+ * Call repetitively to consume full input updating it as necessary.
+ * Function will update both input and output `pos` fields exposing current state via these fields:
+ * - `input.pos < input.size`, some input remaining and caller should provide remaining input
+ *   on the next call.
+ * - `output.pos < output.size`, decoder finished and flushed all remaining buffers.
+ * - `output.pos == output.size`, potentially uncflushed data present in the internal buffers,
+ *   call ZSTD_decompressStream() again to flush remaining data to output.
+ * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX.
+ *
+ * @return : 0 when a frame is completely decoded and fully flushed,
+ *           or an error code, which can be tested using ZSTD_isError(),
+ *           or any other value > 0, which means there is some decoding or flushing to do to complete current frame.
+ */
 ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
 
 ZSTDLIB_API size_t ZSTD_DStreamInSize(void);    /*!< recommended size for input buffer */
@@ -913,7 +987,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
  *  If @return == 0, the dictID could not be decoded.
  *  This could for one of the following reasons :
  *  - The frame does not require a dictionary to be decoded (most common case).
- *  - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
+ *  - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden piece of information.
  *    Note : this use case also happens when using a non-conformant dictionary.
  *  - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
  *  - This is not a Zstandard frame.
@@ -925,9 +999,11 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
  * Advanced dictionary and prefix API (Requires v1.4.0+)
  *
  * This API allows dictionaries to be used with ZSTD_compress2(),
- * ZSTD_compressStream2(), and ZSTD_decompressDCtx(). Dictionaries are sticky, and
- * only reset with the context is reset with ZSTD_reset_parameters or
- * ZSTD_reset_session_and_parameters. Prefixes are single-use.
+ * ZSTD_compressStream2(), and ZSTD_decompressDCtx().
+ * Dictionaries are sticky, they remain valid when same context is re-used,
+ * they only reset when the context is reset
+ * with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters.
+ * In contrast, Prefixes are single-use.
  ******************************************************************************/
 
 
@@ -937,8 +1013,9 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  *  Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary,
  *           meaning "return to no-dictionary mode".
- *  Note 1 : Dictionary is sticky, it will be used for all future compressed frames.
- *           To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters).
+ *  Note 1 : Dictionary is sticky, it will be used for all future compressed frames,
+ *           until parameters are reset, a new dictionary is loaded, or the dictionary
+ *           is explicitly invalidated by loading a NULL dictionary.
  *  Note 2 : Loading a dictionary involves building tables.
  *           It's also a CPU consuming operation, with non-negligible impact on latency.
  *           Tables are dependent on compression parameters, and for this reason,
@@ -947,11 +1024,15 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
  *           Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead.
  *           In such a case, dictionary buffer must outlive its users.
  *  Note 4 : Use ZSTD_CCtx_loadDictionary_advanced()
- *           to precisely select how dictionary content must be interpreted. */
+ *           to precisely select how dictionary content must be interpreted.
+ *  Note 5 : This method does not benefit from LDM (long distance mode).
+ *           If you want to employ LDM on some large dictionary content,
+ *           prefer employing ZSTD_CCtx_refPrefix() described below.
+ */
 ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
 
 /*! ZSTD_CCtx_refCDict() : Requires v1.4.0+
- *  Reference a prepared dictionary, to be used for all next compressed frames.
+ *  Reference a prepared dictionary, to be used for all future compressed frames.
  *  Note that compression parameters are enforced from within CDict,
  *  and supersede any compression parameter previously set within CCtx.
  *  The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs.
@@ -970,6 +1051,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
  *  Decompression will need same prefix to properly regenerate data.
  *  Compressing with a prefix is similar in outcome as performing a diff and compressing it,
  *  but performs much faster, especially during decompression (compression speed is tunable with compression level).
+ *  This method is compatible with LDM (long distance mode).
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  *  Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary
  *  Note 1 : Prefix buffer is referenced. It **must** outlive compression.
@@ -986,9 +1068,9 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
                                  const void* prefix, size_t prefixSize);
 
 /*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+
- *  Create an internal DDict from dict buffer,
- *  to be used to decompress next frames.
- *  The dictionary remains valid for all future frames, until explicitly invalidated.
+ *  Create an internal DDict from dict buffer, to be used to decompress all future frames.
+ *  The dictionary remains valid for all future frames, until explicitly invalidated, or
+ *  a new dictionary is loaded.
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
  *  Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary,
  *            meaning "return to no-dictionary mode".
@@ -1012,9 +1094,10 @@ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, s
  *  The memory for the table is allocated on the first call to refDDict, and can be
  *  freed with ZSTD_freeDCtx().
  *
+ *  If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary
+ *  will be managed, and referencing a dictionary effectively "discards" any previous one.
+ *
  * @result : 0, or an error code (which can be tested with ZSTD_isError()).
- *  Note 1 : Currently, only one dictionary can be managed.
- *           Referencing a new dictionary effectively "discards" any previous one.
  *  Special: referencing a NULL DDict means "return to no-dictionary mode".
  *  Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx.
  */
@@ -1071,24 +1154,6 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 #define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE
 #endif
 
-/* Deprecation warnings :
- * Should these warnings be a problem, it is generally possible to disable them,
- * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual.
- * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS.
- */
-#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS
-#  define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API  /* disable deprecation warnings */
-#else
-#  if (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__)
-#    define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __attribute__((deprecated(message)))
-#  elif (__GNUC__ >= 3)
-#    define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __attribute__((deprecated))
-#  else
-#    pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler")
-#    define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API
-#  endif
-#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */
-
 /* **************************************************************************************
  *   experimental API (static linking only)
  ****************************************************************************************
@@ -1123,6 +1188,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 #define ZSTD_TARGETLENGTH_MIN     0   /* note : comparing this constant to an unsigned results in a tautological test */
 #define ZSTD_STRATEGY_MIN        ZSTD_fast
 #define ZSTD_STRATEGY_MAX        ZSTD_btultra2
+#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */
 
 
 #define ZSTD_OVERLAPLOG_MIN       0
@@ -1303,7 +1369,7 @@ typedef enum {
 } ZSTD_paramSwitch_e;
 
 /* *************************************
-*  Frame size functions
+*  Frame header and size functions
 ***************************************/
 
 /*! ZSTD_findDecompressedSize() :
@@ -1350,29 +1416,109 @@ ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size
  *           or an error code (if srcSize is too small) */
 ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize);
 
+typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
+typedef struct {
+    unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */
+    unsigned long long windowSize;       /* can be very large, up to <= frameContentSize */
+    unsigned blockSizeMax;
+    ZSTD_frameType_e frameType;          /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
+    unsigned headerSize;
+    unsigned dictID;
+    unsigned checksumFlag;
+    unsigned _reserved1;
+    unsigned _reserved2;
+} ZSTD_frameHeader;
+
+/*! ZSTD_getFrameHeader() :
+ *  decode Frame Header, or requires larger `srcSize`.
+ * @return : 0, `zfhPtr` is correctly filled,
+ *          >0, `srcSize` is too small, value is wanted `srcSize` amount,
+ *           or an error code, which can be tested using ZSTD_isError() */
+ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize);   /*< doesn't consume input */
+/*! ZSTD_getFrameHeader_advanced() :
+ *  same as ZSTD_getFrameHeader(),
+ *  with added capability to select a format (like ZSTD_f_zstd1_magicless) */
+ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
+
+/*! ZSTD_decompressionMargin() :
+ * Zstd supports in-place decompression, where the input and output buffers overlap.
+ * In this case, the output buffer must be at least (Margin + Output_Size) bytes large,
+ * and the input buffer must be at the end of the output buffer.
+ *
+ *  _______________________ Output Buffer ________________________
+ * |                                                              |
+ * |                                        ____ Input Buffer ____|
+ * |                                       |                      |
+ * v                                       v                      v
+ * |---------------------------------------|-----------|----------|
+ * ^                                                   ^          ^
+ * |___________________ Output_Size ___________________|_ Margin _|
+ *
+ * NOTE: See also ZSTD_DECOMPRESSION_MARGIN().
+ * NOTE: This applies only to single-pass decompression through ZSTD_decompress() or
+ * ZSTD_decompressDCtx().
+ * NOTE: This function supports multi-frame input.
+ *
+ * @param src The compressed frame(s)
+ * @param srcSize The size of the compressed frame(s)
+ * @returns The decompression margin or an error that can be checked with ZSTD_isError().
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize);
+
+/*! ZSTD_DECOMPRESS_MARGIN() :
+ * Similar to ZSTD_decompressionMargin(), but instead of computing the margin from
+ * the compressed frame, compute it from the original size and the blockSizeLog.
+ * See ZSTD_decompressionMargin() for details.
+ *
+ * WARNING: This macro does not support multi-frame input, the input must be a single
+ * zstd frame. If you need that support use the function, or implement it yourself.
+ *
+ * @param originalSize The original uncompressed size of the data.
+ * @param blockSize    The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX).
+ *                     Unless you explicitly set the windowLog smaller than
+ *                     ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX.
+ */
+#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)(                                              \
+        ZSTD_FRAMEHEADERSIZE_MAX                                                              /* Frame header */ + \
+        4                                                                                         /* checksum */ + \
+        ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \
+        (blockSize)                                                                    /* One block of margin */   \
+    ))
+
 typedef enum {
   ZSTD_sf_noBlockDelimiters = 0,         /* Representation of ZSTD_Sequence has no block delimiters, sequences only */
   ZSTD_sf_explicitBlockDelimiters = 1    /* Representation of ZSTD_Sequence contains explicit block delimiters */
 } ZSTD_sequenceFormat_e;
 
+/*! ZSTD_sequenceBound() :
+ * `srcSize` : size of the input buffer
+ *  @return : upper-bound for the number of sequences that can be generated
+ *            from a buffer of srcSize bytes
+ *
+ *  note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
+
 /*! ZSTD_generateSequences() :
- * Generate sequences using ZSTD_compress2, given a source buffer.
+ * Generate sequences using ZSTD_compress2(), given a source buffer.
  *
  * Each block will end with a dummy sequence
  * with offset == 0, matchLength == 0, and litLength == length of last literals.
  * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0)
  * simply acts as a block delimiter.
  *
- * zc can be used to insert custom compression params.
- * This function invokes ZSTD_compress2
+ * @zc can be used to insert custom compression params.
+ * This function invokes ZSTD_compress2().
  *
  * The output of this function can be fed into ZSTD_compressSequences() with CCtx
  * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters
  * @return : number of sequences generated
  */
 
-ZSTDLIB_STATIC_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
-                                          size_t outSeqsSize, const void* src, size_t srcSize);
+ZSTDLIB_STATIC_API size_t
+ZSTD_generateSequences( ZSTD_CCtx* zc,
+                        ZSTD_Sequence* outSeqs, size_t outSeqsSize,
+                        const void* src, size_t srcSize);
 
 /*! ZSTD_mergeBlockDelimiters() :
  * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals
@@ -1388,7 +1534,9 @@ ZSTDLIB_STATIC_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* o
 ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize);
 
 /*! ZSTD_compressSequences() :
- * Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst.
+ * Compress an array of ZSTD_Sequence, associated with @src buffer, into dst.
+ * @src contains the entire input (not just the literals).
+ * If @srcSize > sum(sequence.length), the remaining bytes are considered all literals
  * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.)
  * The entire source is compressed into a single frame.
  *
@@ -1413,11 +1561,12 @@ ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, si
  * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused.
  * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly,
  *         and cannot emit an RLE block that disagrees with the repcode history
- * @return : final compressed size or a ZSTD error.
+ * @return : final compressed size, or a ZSTD error code.
  */
-ZSTDLIB_STATIC_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize,
-                                  const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
-                                  const void* src, size_t srcSize);
+ZSTDLIB_STATIC_API size_t
+ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize,
+                        const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
+                        const void* src, size_t srcSize);
 
 
 /*! ZSTD_writeSkippableFrame() :
@@ -1481,8 +1630,11 @@ ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size);
  *  and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter().
  *  Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits.
  *
- *  Note 2 : only single-threaded compression is supported.
+ *  Note : only single-threaded compression is supported.
  *  ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
+ *
+ *  Note 2 : ZSTD_estimateCCtxSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
+ *  Size estimates assume that no external sequence producer is registered.
  */
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams);
@@ -1501,7 +1653,12 @@ ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void);
  *  or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame();
  *  Note : if streaming is init with function ZSTD_init?Stream_usingDict(),
  *         an internal ?Dict will be created, which additional size is not estimated here.
- *         In this case, get total size by adding ZSTD_estimate?DictSize */
+ *         In this case, get total size by adding ZSTD_estimate?DictSize
+ *  Note 2 : only single-threaded compression is supported.
+ *  ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1.
+ *  Note 3 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time.
+ *  Size estimates assume that no external sequence producer is registered.
+ */
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams);
 ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params);
@@ -1649,22 +1806,45 @@ ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params);
  *  This function never fails (wide contract) */
 ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize);
 
+/*! ZSTD_CCtx_setCParams() :
+ *  Set all parameters provided within @p cparams into the working @p cctx.
+ *  Note : if modifying parameters during compression (MT mode only),
+ *         note that changes to the .windowLog parameter will be ignored.
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()).
+ *         On failure, no parameters are updated.
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams);
+
+/*! ZSTD_CCtx_setFParams() :
+ *  Set all parameters provided within @p fparams into the working @p cctx.
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams);
+
+/*! ZSTD_CCtx_setParams() :
+ *  Set all parameters provided within @p params into the working @p cctx.
+ * @return 0 on success, or an error code (can be checked with ZSTD_isError()).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params);
+
 /*! ZSTD_compress_advanced() :
  *  Note : this function is now DEPRECATED.
  *         It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters.
  *  This prototype will generate compilation warnings. */
 ZSTD_DEPRECATED("use ZSTD_compress2")
+ZSTDLIB_STATIC_API
 size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx,
-                                          void* dst, size_t dstCapacity,
-                                    const void* src, size_t srcSize,
-                                    const void* dict,size_t dictSize,
-                                          ZSTD_parameters params);
+                              void* dst, size_t dstCapacity,
+                        const void* src, size_t srcSize,
+                        const void* dict,size_t dictSize,
+                              ZSTD_parameters params);
 
 /*! ZSTD_compress_usingCDict_advanced() :
  *  Note : this function is now DEPRECATED.
  *         It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters.
  *  This prototype will generate compilation warnings. */
 ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary")
+ZSTDLIB_STATIC_API
 size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx,
                                               void* dst, size_t dstCapacity,
                                         const void* src, size_t srcSize,
@@ -1808,13 +1988,16 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
  * Experimental parameter.
  * Default is 0 == disabled. Set to 1 to enable.
  *
- * Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same
- * between calls, except for the modifications that zstd makes to pos (the
- * caller must not modify pos). This is checked by the compressor, and
- * compression will fail if it ever changes. This means the only flush
- * mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end
- * is not used. The data in the ZSTD_inBuffer in the range [src, src + pos)
- * MUST not be modified during compression or you will get data corruption.
+ * Tells the compressor that input data presented with ZSTD_inBuffer
+ * will ALWAYS be the same between calls.
+ * Technically, the @src pointer must never be changed,
+ * and the @pos field can only be updated by zstd.
+ * However, it's possible to increase the @size field,
+ * allowing scenarios where more data can be appended after compressions starts.
+ * These conditions are checked by the compressor,
+ * and compression will fail if they are not respected.
+ * Also, data in the ZSTD_inBuffer within the range [src, src + pos)
+ * MUST not be modified during compression or it will result in data corruption.
  *
  * When this flag is enabled zstd won't allocate an input window buffer,
  * because the user guarantees it can reference the ZSTD_inBuffer until
@@ -1822,18 +2005,15 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
  * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also
  * avoid the memcpy() from the input buffer to the input window buffer.
  *
- * NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used.
- * That means this flag cannot be used with ZSTD_compressStream().
- *
  * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using
  * this flag is ALWAYS memory safe, and will never access out-of-bounds
- * memory. However, compression WILL fail if you violate the preconditions.
+ * memory. However, compression WILL fail if conditions are not respected.
  *
- * WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST
- * not be modified during compression or you will get data corruption. This
- * is because zstd needs to reference data in the ZSTD_inBuffer to find
+ * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST
+ * not be modified during compression or it will result in data corruption.
+ * This is because zstd needs to reference data in the ZSTD_inBuffer to find
  * matches. Normally zstd maintains its own window buffer for this purpose,
- * but passing this flag tells zstd to use the user provided buffer.
+ * but passing this flag tells zstd to rely on user provided buffer instead.
  */
 #define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9
 
@@ -1878,7 +2058,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
  * Without validation, providing a sequence that does not conform to the zstd spec will cause
  * undefined behavior, and may produce a corrupted block.
  *
- * With validation enabled, a if sequence is invalid (see doc/zstd_compression_format.md for
+ * With validation enabled, if sequence is invalid (see doc/zstd_compression_format.md for
  * specifics regarding offset/matchlength requirements) then the function will bail out and
  * return an error.
  *
@@ -1928,6 +2108,79 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo
  */
 #define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15
 
+/* ZSTD_c_prefetchCDictTables
+ * Controlled with ZSTD_paramSwitch_e enum. Default is ZSTD_ps_auto.
+ *
+ * In some situations, zstd uses CDict tables in-place rather than copying them
+ * into the working context. (See docs on ZSTD_dictAttachPref_e above for details).
+ * In such situations, compression speed is seriously impacted when CDict tables are
+ * "cold" (outside CPU cache). This parameter instructs zstd to prefetch CDict tables
+ * when they are used in-place.
+ *
+ * For sufficiently small inputs, the cost of the prefetch will outweigh the benefit.
+ * For sufficiently large inputs, zstd will by default memcpy() CDict tables
+ * into the working context, so there is no need to prefetch. This parameter is
+ * targeted at a middle range of input sizes, where a prefetch is cheap enough to be
+ * useful but memcpy() is too expensive. The exact range of input sizes where this
+ * makes sense is best determined by careful experimentation.
+ *
+ * Note: for this parameter, ZSTD_ps_auto is currently equivalent to ZSTD_ps_disable,
+ * but in the future zstd may conditionally enable this feature via an auto-detection
+ * heuristic for cold CDicts.
+ * Use ZSTD_ps_disable to opt out of prefetching under any circumstances.
+ */
+#define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16
+
+/* ZSTD_c_enableSeqProducerFallback
+ * Allowed values are 0 (disable) and 1 (enable). The default setting is 0.
+ *
+ * Controls whether zstd will fall back to an internal sequence producer if an
+ * external sequence producer is registered and returns an error code. This fallback
+ * is block-by-block: the internal sequence producer will only be called for blocks
+ * where the external sequence producer returns an error code. Fallback parsing will
+ * follow any other cParam settings, such as compression level, the same as in a
+ * normal (fully-internal) compression operation.
+ *
+ * The user is strongly encouraged to read the full Block-Level Sequence Producer API
+ * documentation (below) before setting this parameter. */
+#define ZSTD_c_enableSeqProducerFallback ZSTD_c_experimentalParam17
+
+/* ZSTD_c_maxBlockSize
+ * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB).
+ * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default.
+ *
+ * This parameter can be used to set an upper bound on the blocksize
+ * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper
+ * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make
+ * compressBound() inaccurate). Only currently meant to be used for testing.
+ *
+ */
+#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18
+
+/* ZSTD_c_searchForExternalRepcodes
+ * This parameter affects how zstd parses external sequences, such as sequences
+ * provided through the compressSequences() API or from an external block-level
+ * sequence producer.
+ *
+ * If set to ZSTD_ps_enable, the library will check for repeated offsets in
+ * external sequences, even if those repcodes are not explicitly indicated in
+ * the "rep" field. Note that this is the only way to exploit repcode matches
+ * while using compressSequences() or an external sequence producer, since zstd
+ * currently ignores the "rep" field of external sequences.
+ *
+ * If set to ZSTD_ps_disable, the library will not exploit repeated offsets in
+ * external sequences, regardless of whether the "rep" field has been set. This
+ * reduces sequence compression overhead by about 25% while sacrificing some
+ * compression ratio.
+ *
+ * The default value is ZSTD_ps_auto, for which the library will enable/disable
+ * based on compression level.
+ *
+ * Note: for now, this param only has an effect if ZSTD_c_blockDelimiters is
+ * set to ZSTD_sf_explicitBlockDelimiters. That may change in the future.
+ */
+#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19
+
 /*! ZSTD_CCtx_getParameter() :
  *  Get the requested compression parameter value, selected by enum ZSTD_cParameter,
  *  and store it into int* value.
@@ -2084,7 +2337,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParamete
  * in the range [dst, dst + pos) MUST not be modified during decompression
  * or you will get data corruption.
  *
- * When this flags is enabled zstd won't allocate an output buffer, because
+ * When this flag is enabled zstd won't allocate an output buffer, because
  * it can write directly to the ZSTD_outBuffer, but it will still allocate
  * an input buffer large enough to fit any compressed block. This will also
  * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer.
@@ -2137,6 +2390,17 @@ ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParamete
  */
 #define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4
 
+/* ZSTD_d_disableHuffmanAssembly
+ * Set to 1 to disable the Huffman assembly implementation.
+ * The default value is 0, which allows zstd to use the Huffman assembly
+ * implementation if available.
+ *
+ * This parameter can be used to disable Huffman assembly at runtime.
+ * If you want to disable it at compile time you can define the macro
+ * ZSTD_DISABLE_ASM.
+ */
+#define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5
+
 
 /*! ZSTD_DCtx_setFormat() :
  *  This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter().
@@ -2145,6 +2409,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParamete
  *  such ZSTD_f_zstd1_magicless for example.
  * @return : 0, or an error code (which can be tested using ZSTD_isError()). */
 ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead")
+ZSTDLIB_STATIC_API
 size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format);
 
 /*! ZSTD_decompressStream_simpleArgs() :
@@ -2181,6 +2446,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs (
  * This prototype will generate compilation warnings.
  */
 ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
                          int compressionLevel,
                          unsigned long long pledgedSrcSize);
@@ -2198,17 +2464,15 @@ size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs,
  * This prototype will generate compilation warnings.
  */
 ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
                      const void* dict, size_t dictSize,
                            int compressionLevel);
 
 /*! ZSTD_initCStream_advanced() :
- * This function is DEPRECATED, and is approximately equivalent to:
+ * This function is DEPRECATED, and is equivalent to:
  *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
- *     // Pseudocode: Set each zstd parameter and leave the rest as-is.
- *     for ((param, value) : params) {
- *         ZSTD_CCtx_setParameter(zcs, param, value);
- *     }
+ *     ZSTD_CCtx_setParams(zcs, params);
  *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
  *     ZSTD_CCtx_loadDictionary(zcs, dict, dictSize);
  *
@@ -2218,6 +2482,7 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs,
  * This prototype will generate compilation warnings.
  */
 ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
                     const void* dict, size_t dictSize,
                           ZSTD_parameters params,
@@ -2232,15 +2497,13 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
  * This prototype will generate compilation warnings.
  */
 ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
 
 /*! ZSTD_initCStream_usingCDict_advanced() :
- *   This function is DEPRECATED, and is approximately equivalent to:
+ *   This function is DEPRECATED, and is equivalent to:
  *     ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only);
- *     // Pseudocode: Set each zstd frame parameter and leave the rest as-is.
- *     for ((fParam, value) : fParams) {
- *         ZSTD_CCtx_setParameter(zcs, fParam, value);
- *     }
+ *     ZSTD_CCtx_setFParams(zcs, fParams);
  *     ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize);
  *     ZSTD_CCtx_refCDict(zcs, cdict);
  *
@@ -2250,6 +2513,7 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);
  * This prototype will generate compilation warnings.
  */
 ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
                                const ZSTD_CDict* cdict,
                                      ZSTD_frameParameters fParams,
@@ -2274,6 +2538,7 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs,
  *  This prototype will generate compilation warnings.
  */
 ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions")
+ZSTDLIB_STATIC_API
 size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);
 
 
@@ -2319,8 +2584,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx);
  *     ZSTD_DCtx_loadDictionary(zds, dict, dictSize);
  *
  * note: no dictionary will be used if dict == NULL or dictSize < 8
- * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  */
+ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions")
 ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
 
 /*!
@@ -2330,8 +2595,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const vo
  *     ZSTD_DCtx_refDDict(zds, ddict);
  *
  * note : ddict is referenced, it must outlive decompression session
- * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  */
+ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions")
 ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);
 
 /*!
@@ -2340,17 +2605,185 @@ ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const Z
  *     ZSTD_DCtx_reset(zds, ZSTD_reset_session_only);
  *
  * re-use decompression parameters from previous init; saves dictionary loading
- * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x
  */
+ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions")
 ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
 
 
+/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API *********************
+ *
+ * *** OVERVIEW ***
+ * The Block-Level Sequence Producer API allows users to provide their own custom
+ * sequence producer which libzstd invokes to process each block. The produced list
+ * of sequences (literals and matches) is then post-processed by libzstd to produce
+ * valid compressed blocks.
+ *
+ * This block-level offload API is a more granular complement of the existing
+ * frame-level offload API compressSequences() (introduced in v1.5.1). It offers
+ * an easier migration story for applications already integrated with libzstd: the
+ * user application continues to invoke the same compression functions
+ * ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits
+ * from the specific advantages of the external sequence producer. For example,
+ * the sequence producer could be tuned to take advantage of known characteristics
+ * of the input, to offer better speed / ratio, or could leverage hardware
+ * acceleration not available within libzstd itself.
+ *
+ * See contrib/externalSequenceProducer for an example program employing the
+ * Block-Level Sequence Producer API.
+ *
+ * *** USAGE ***
+ * The user is responsible for implementing a function of type
+ * ZSTD_sequenceProducer_F. For each block, zstd will pass the following
+ * arguments to the user-provided function:
+ *
+ *   - sequenceProducerState: a pointer to a user-managed state for the sequence
+ *     producer.
+ *
+ *   - outSeqs, outSeqsCapacity: an output buffer for the sequence producer.
+ *     outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory
+ *     backing outSeqs is managed by the CCtx.
+ *
+ *   - src, srcSize: an input buffer for the sequence producer to parse.
+ *     srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX.
+ *
+ *   - dict, dictSize: a history buffer, which may be empty, which the sequence
+ *     producer may reference as it parses the src buffer. Currently, zstd will
+ *     always pass dictSize == 0 into external sequence producers, but this will
+ *     change in the future.
+ *
+ *   - compressionLevel: a signed integer representing the zstd compression level
+ *     set by the user for the current operation. The sequence producer may choose
+ *     to use this information to change its compression strategy and speed/ratio
+ *     tradeoff. Note: the compression level does not reflect zstd parameters set
+ *     through the advanced API.
+ *
+ *   - windowSize: a size_t representing the maximum allowed offset for external
+ *     sequences. Note that sequence offsets are sometimes allowed to exceed the
+ *     windowSize if a dictionary is present, see doc/zstd_compression_format.md
+ *     for details.
+ *
+ * The user-provided function shall return a size_t representing the number of
+ * sequences written to outSeqs. This return value will be treated as an error
+ * code if it is greater than outSeqsCapacity. The return value must be non-zero
+ * if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided
+ * for convenience, but any value greater than outSeqsCapacity will be treated as
+ * an error code.
+ *
+ * If the user-provided function does not return an error code, the sequences
+ * written to outSeqs must be a valid parse of the src buffer. Data corruption may
+ * occur if the parse is not valid. A parse is defined to be valid if the
+ * following conditions hold:
+ *   - The sum of matchLengths and literalLengths must equal srcSize.
+ *   - All sequences in the parse, except for the final sequence, must have
+ *     matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have
+ *     matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0.
+ *   - All offsets must respect the windowSize parameter as specified in
+ *     doc/zstd_compression_format.md.
+ *   - If the final sequence has matchLength == 0, it must also have offset == 0.
+ *
+ * zstd will only validate these conditions (and fail compression if they do not
+ * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence
+ * validation has a performance cost.
+ *
+ * If the user-provided function returns an error, zstd will either fall back
+ * to an internal sequence producer or fail the compression operation. The user can
+ * choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback
+ * cParam. Fallback compression will follow any other cParam settings, such as
+ * compression level, the same as in a normal compression operation.
+ *
+ * The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F
+ * function by calling
+ *         ZSTD_registerSequenceProducer(cctx,
+ *                                       sequenceProducerState,
+ *                                       sequenceProducer)
+ * This setting will persist until the next parameter reset of the CCtx.
+ *
+ * The sequenceProducerState must be initialized by the user before calling
+ * ZSTD_registerSequenceProducer(). The user is responsible for destroying the
+ * sequenceProducerState.
+ *
+ * *** LIMITATIONS ***
+ * This API is compatible with all zstd compression APIs which respect advanced parameters.
+ * However, there are three limitations:
+ *
+ * First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported.
+ * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level
+ * external sequence producer.
+ *   - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some
+ *     cases (see its documentation for details). Users must explicitly set
+ *     ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external
+ *     sequence producer is registered.
+ *   - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default
+ *     whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should
+ *     check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence
+ *     Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog).
+ *
+ * Second, history buffers are not currently supported. Concretely, zstd will always pass
+ * dictSize == 0 to the external sequence producer (for now). This has two implications:
+ *   - Dictionaries are not currently supported. Compression will *not* fail if the user
+ *     references a dictionary, but the dictionary won't have any effect.
+ *   - Stream history is not currently supported. All advanced compression APIs, including
+ *     streaming APIs, work with external sequence producers, but each block is treated as
+ *     an independent chunk without history from previous blocks.
+ *
+ * Third, multi-threading within a single compression is not currently supported. In other words,
+ * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered.
+ * Multi-threading across compressions is fine: simply create one CCtx per thread.
+ *
+ * Long-term, we plan to overcome all three limitations. There is no technical blocker to
+ * overcoming them. It is purely a question of engineering effort.
+ */
+
+#define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1))
+
+typedef size_t ZSTD_sequenceProducer_F (
+  void* sequenceProducerState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+);
+
+/*! ZSTD_registerSequenceProducer() :
+ * Instruct zstd to use a block-level external sequence producer function.
+ *
+ * The sequenceProducerState must be initialized by the caller, and the caller is
+ * responsible for managing its lifetime. This parameter is sticky across
+ * compressions. It will remain set until the user explicitly resets compression
+ * parameters.
+ *
+ * Sequence producer registration is considered to be an "advanced parameter",
+ * part of the "advanced API". This means it will only have an effect on compression
+ * APIs which respect advanced parameters, such as compress2() and compressStream2().
+ * Older compression APIs such as compressCCtx(), which predate the introduction of
+ * "advanced parameters", will ignore any external sequence producer setting.
+ *
+ * The sequence producer can be "cleared" by registering a NULL function pointer. This
+ * removes all limitations described above in the "LIMITATIONS" section of the API docs.
+ *
+ * The user is strongly encouraged to read the full API documentation (above) before
+ * calling this function. */
+ZSTDLIB_STATIC_API void
+ZSTD_registerSequenceProducer(
+  ZSTD_CCtx* cctx,
+  void* sequenceProducerState,
+  ZSTD_sequenceProducer_F* sequenceProducer
+);
+
+
 /* *******************************************************************
-*  Buffer-less and synchronous inner streaming functions
+*  Buffer-less and synchronous inner streaming functions (DEPRECATED)
+*
+*  This API is deprecated, and will be removed in a future version.
+*  It allows streaming (de)compression with user allocated buffers.
+*  However, it is hard to use, and not as well tested as the rest of
+*  our API.
 *
-*  This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
-*  But it's also a complex one, with several restrictions, documented below.
-*  Prefer normal streaming API for an easier experience.
+*  Please use the normal streaming API instead: ZSTD_compressStream2,
+*  and ZSTD_decompressStream.
+*  If there is functionality that you need, but it doesn't provide,
+*  please open an issue on our GitHub.
 ********************************************************************* */
 
 /*
@@ -2362,7 +2795,6 @@ ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
 
   Start by initializing a context.
   Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression.
-  It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx()
 
   Then, consume your input using ZSTD_compressContinue().
   There are some important considerations to keep in mind when using this advanced function :
@@ -2384,18 +2816,28 @@ ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds);
 */
 
 /*=====   Buffer-less streaming compression functions  =====*/
+ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
 ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
+ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
 ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
+ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
 ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /*< note: fails if cdict==NULL */
-ZSTDLIB_STATIC_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /*<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
 
+ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.")
+ZSTDLIB_STATIC_API
+size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /*<  note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */
+
+ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
 ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.")
 ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 
 /* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */
 ZSTD_DEPRECATED("use advanced API to access custom parameters")
+ZSTDLIB_STATIC_API
 size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /*< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */
 ZSTD_DEPRECATED("use advanced API to access custom parameters")
+ZSTDLIB_STATIC_API
 size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize);   /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */
 /*
   Buffer-less streaming decompression (synchronous mode)
@@ -2408,8 +2850,8 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
   Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough.
   Data fragment must be large enough to ensure successful decoding.
  `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough.
-  @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
-           >0 : `srcSize` is too small, please provide at least @result bytes on next attempt.
+  result  : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled.
+           >0 : `srcSize` is too small, please provide at least result bytes on next attempt.
            errorCode, which can be tested using ZSTD_isError().
 
   It fills a ZSTD_frameHeader structure with important information to correctly decode the frame,
@@ -2428,7 +2870,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
 
   The most memory efficient way is to use a round buffer of sufficient size.
   Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(),
-  which can @return an error code if required value is too large for current system (in 32-bits mode).
+  which can return an error code if required value is too large for current system (in 32-bits mode).
   In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one,
   up to the moment there is not enough room left in the buffer to guarantee decoding another full block,
   which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`.
@@ -2448,7 +2890,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
   ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue().
   ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail.
 
- @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
+  result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
   It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item.
   It can also be an error code, which can be tested with ZSTD_isError().
 
@@ -2471,27 +2913,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_
 */
 
 /*=====   Buffer-less streaming decompression functions  =====*/
-typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e;
-typedef struct {
-    unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */
-    unsigned long long windowSize;       /* can be very large, up to <= frameContentSize */
-    unsigned blockSizeMax;
-    ZSTD_frameType_e frameType;          /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */
-    unsigned headerSize;
-    unsigned dictID;
-    unsigned checksumFlag;
-} ZSTD_frameHeader;
 
-/*! ZSTD_getFrameHeader() :
- *  decode Frame Header, or requires larger `srcSize`.
- * @return : 0, `zfhPtr` is correctly filled,
- *          >0, `srcSize` is too small, value is wanted `srcSize` amount,
- *           or an error code, which can be tested using ZSTD_isError() */
-ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize);   /*< doesn't consume input */
-/*! ZSTD_getFrameHeader_advanced() :
- *  same as ZSTD_getFrameHeader(),
- *  with added capability to select a format (like ZSTD_f_zstd1_magicless) */
-ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format);
 ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize);  /*< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */
 
 ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
@@ -2502,6 +2924,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
 ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 
 /* misc */
+ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.")
 ZSTDLIB_STATIC_API void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
 typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
 ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
@@ -2509,11 +2932,23 @@ ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
 
 
 
-/* ============================ */
-/*       Block level API       */
-/* ============================ */
+/* ========================================= */
+/*       Block level API (DEPRECATED)       */
+/* ========================================= */
 
 /*!
+
+    This API is deprecated in favor of the regular compression API.
+    You can get the frame header down to 2 bytes by setting:
+      - ZSTD_c_format = ZSTD_f_zstd1_magicless
+      - ZSTD_c_contentSizeFlag = 0
+      - ZSTD_c_checksumFlag = 0
+      - ZSTD_c_dictIDFlag = 0
+
+    This API is not as well tested as our normal API, so we recommend not using it.
+    We will be removing it in a future version. If the normal API doesn't provide
+    the functionality you need, please open a GitHub issue.
+
     Block functions produce and decode raw zstd blocks, without frame metadata.
     Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes).
     But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes.
@@ -2524,7 +2959,6 @@ ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
     - It is necessary to init context before starting
       + compression : any ZSTD_compressBegin*() variant, including with dictionary
       + decompression : any ZSTD_decompressBegin*() variant, including with dictionary
-      + copyCCtx() and copyDCtx() can be used too
     - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB
       + If input is larger than a block size, it's necessary to split input data into multiple blocks
       + For inputs larger than a single block, consider using regular ZSTD_compress() instead.
@@ -2541,11 +2975,14 @@ ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
 */
 
 /*=====   Raw zstd block functions  =====*/
+ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
 ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize   (const ZSTD_CCtx* cctx);
+ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
 ZSTDLIB_STATIC_API size_t ZSTD_compressBlock  (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
 ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.")
 ZSTDLIB_STATIC_API size_t ZSTD_insertBlock    (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize);  /*< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */
 
-
 #endif   /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */
 
diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h
index 20988623c5cc..201c0809540a 100644
--- a/include/net/bluetooth/hci_core.h
+++ b/include/net/bluetooth/hci_core.h
@@ -1292,6 +1292,30 @@ static inline struct hci_conn *hci_conn_hash_lookup_big(struct hci_dev *hdev,
 	return NULL;
 }
 
+static inline struct hci_conn *
+hci_conn_hash_lookup_big_state(struct hci_dev *hdev, __u8 handle,  __u16 state)
+{
+	struct hci_conn_hash *h = &hdev->conn_hash;
+	struct hci_conn  *c;
+
+	rcu_read_lock();
+
+	list_for_each_entry_rcu(c, &h->list, list) {
+		if (bacmp(&c->dst, BDADDR_ANY) || c->type != ISO_LINK ||
+			c->state != state)
+			continue;
+
+		if (handle == c->iso_qos.bcast.big) {
+			rcu_read_unlock();
+			return c;
+		}
+	}
+
+	rcu_read_unlock();
+
+	return NULL;
+}
+
 static inline struct hci_conn *
 hci_conn_hash_lookup_pa_sync_big_handle(struct hci_dev *hdev, __u8 big)
 {
diff --git a/include/net/ip_tunnels.h b/include/net/ip_tunnels.h
index f346b4efbc30..2d746f4c9a0a 100644
--- a/include/net/ip_tunnels.h
+++ b/include/net/ip_tunnels.h
@@ -416,6 +416,17 @@ static inline u8 ip_tunnel_get_dsfield(const struct iphdr *iph,
 		return 0;
 }
 
+static inline __be32 ip_tunnel_get_flowlabel(const struct iphdr *iph,
+					     const struct sk_buff *skb)
+{
+	__be16 payload_protocol = skb_protocol(skb, true);
+
+	if (payload_protocol == htons(ETH_P_IPV6))
+		return ip6_flowlabel((const struct ipv6hdr *)iph);
+	else
+		return 0;
+}
+
 static inline u8 ip_tunnel_get_ttl(const struct iphdr *iph,
 				       const struct sk_buff *skb)
 {
diff --git a/include/net/netkit.h b/include/net/netkit.h
index 0ba2e6b847ca..9ec0163739f4 100644
--- a/include/net/netkit.h
+++ b/include/net/netkit.h
@@ -10,6 +10,7 @@ int netkit_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog);
 int netkit_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
 int netkit_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog);
 int netkit_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr);
+INDIRECT_CALLABLE_DECLARE(struct net_device *netkit_peer_dev(struct net_device *dev));
 #else
 static inline int netkit_prog_attach(const union bpf_attr *attr,
 				     struct bpf_prog *prog)
@@ -34,5 +35,10 @@ static inline int netkit_prog_query(const union bpf_attr *attr,
 {
 	return -EINVAL;
 }
+
+static inline struct net_device *netkit_peer_dev(struct net_device *dev)
+{
+	return NULL;
+}
 #endif /* CONFIG_NETKIT */
 #endif /* __NET_NETKIT_H */
diff --git a/include/net/netlink.h b/include/net/netlink.h
index 83bdf787aeee..167b91348e57 100644
--- a/include/net/netlink.h
+++ b/include/net/netlink.h
@@ -1010,6 +1010,20 @@ static inline struct sk_buff *nlmsg_new(size_t payload, gfp_t flags)
 	return alloc_skb(nlmsg_total_size(payload), flags);
 }
 
+/**
+ * nlmsg_new_large - Allocate a new netlink message with non-contiguous
+ * physical memory
+ * @payload: size of the message payload
+ *
+ * The allocated skb is unable to have frag page for shinfo->frags*,
+ * as the NULL setting for skb->head in netlink_skb_destructor() will
+ * bypass most of the handling in skb_release_data()
+ */
+static inline struct sk_buff *nlmsg_new_large(size_t payload)
+{
+	return netlink_alloc_large_skb(nlmsg_total_size(payload), 0);
+}
+
 /**
  * nlmsg_end - Finalize a netlink message
  * @skb: socket buffer the message is stored in
diff --git a/include/net/vxlan.h b/include/net/vxlan.h
index 6a9f8a5f387c..33ba6fc151cf 100644
--- a/include/net/vxlan.h
+++ b/include/net/vxlan.h
@@ -210,22 +210,23 @@ struct vxlan_rdst {
 };
 
 struct vxlan_config {
-	union vxlan_addr	remote_ip;
-	union vxlan_addr	saddr;
-	__be32			vni;
-	int			remote_ifindex;
-	int			mtu;
-	__be16			dst_port;
-	u16			port_min;
-	u16			port_max;
-	u8			tos;
-	u8			ttl;
-	__be32			label;
-	u32			flags;
-	unsigned long		age_interval;
-	unsigned int		addrmax;
-	bool			no_share;
-	enum ifla_vxlan_df	df;
+	union vxlan_addr		remote_ip;
+	union vxlan_addr		saddr;
+	__be32				vni;
+	int				remote_ifindex;
+	int				mtu;
+	__be16				dst_port;
+	u16				port_min;
+	u16				port_max;
+	u8				tos;
+	u8				ttl;
+	__be32				label;
+	enum ifla_vxlan_label_policy	label_policy;
+	u32				flags;
+	unsigned long			age_interval;
+	unsigned int			addrmax;
+	bool				no_share;
+	enum ifla_vxlan_df		df;
 };
 
 enum {
diff --git a/include/soc/tegra/mc.h b/include/soc/tegra/mc.h
index 71ae37d3bedd..af1d73a7f0cd 100644
--- a/include/soc/tegra/mc.h
+++ b/include/soc/tegra/mc.h
@@ -162,6 +162,7 @@ struct tegra_mc_ops {
 	 */
 	int (*probe)(struct tegra_mc *mc);
 	void (*remove)(struct tegra_mc *mc);
+	int (*resume)(struct tegra_mc *mc);
 	irqreturn_t (*handle_irq)(int irq, void *data);
 	int (*probe_device)(struct tegra_mc *mc, struct device *dev);
 };
diff --git a/include/sound/cs35l41.h b/include/sound/cs35l41.h
index 043f8ac65dbf..68e053fe7340 100644
--- a/include/sound/cs35l41.h
+++ b/include/sound/cs35l41.h
@@ -906,6 +906,6 @@ int cs35l41_init_boost(struct device *dev, struct regmap *regmap,
 bool cs35l41_safe_reset(struct regmap *regmap, enum cs35l41_boost_type b_type);
 int cs35l41_mdsync_up(struct regmap *regmap);
 int cs35l41_global_enable(struct device *dev, struct regmap *regmap, enum cs35l41_boost_type b_type,
-			  int enable, bool firmware_running);
+			  int enable, struct cs_dsp *dsp);
 
 #endif /* __CS35L41_H */
diff --git a/include/sound/rt5682s.h b/include/sound/rt5682s.h
index 66ca0c75b914..006e6003d11c 100644
--- a/include/sound/rt5682s.h
+++ b/include/sound/rt5682s.h
@@ -31,6 +31,13 @@ enum rt5682s_dai_clks {
 	RT5682S_DAI_NUM_CLKS,
 };
 
+enum {
+	RT5682S_LDO_1_607V,
+	RT5682S_LDO_1_5V,
+	RT5682S_LDO_1_406V,
+	RT5682S_LDO_1_731V,
+};
+
 struct rt5682s_platform_data {
 	enum rt5682s_dmic1_data_pin dmic1_data_pin;
 	enum rt5682s_dmic1_clk_pin dmic1_clk_pin;
@@ -38,6 +45,7 @@ struct rt5682s_platform_data {
 	unsigned int dmic_clk_rate;
 	unsigned int dmic_delay;
 	unsigned int amic_delay;
+	unsigned int ldo_dacref;
 	bool dmic_clk_driving_high;
 
 	const char *dai_clk_names[RT5682S_DAI_NUM_CLKS];
diff --git a/include/sound/sof/dai-imx.h b/include/sound/sof/dai-imx.h
index ca8325353d41..6bc987bd4761 100644
--- a/include/sound/sof/dai-imx.h
+++ b/include/sound/sof/dai-imx.h
@@ -51,4 +51,11 @@ struct sof_ipc_dai_sai_params {
 	uint16_t tdm_slot_width;
 	uint16_t reserved2;	/* alignment */
 } __packed;
+
+/* MICFIL Configuration Request - SOF_IPC_DAI_MICFIL_CONFIG */
+struct sof_ipc_dai_micfil_params {
+	uint32_t pdm_rate;
+	uint32_t pdm_ch;
+} __packed;
+
 #endif
diff --git a/include/sound/sof/dai.h b/include/sound/sof/dai.h
index 3041f5805b7b..4773a5f616a4 100644
--- a/include/sound/sof/dai.h
+++ b/include/sound/sof/dai.h
@@ -88,6 +88,7 @@ enum sof_ipc_dai_type {
 	SOF_DAI_AMD_HS,			/**< Amd HS */
 	SOF_DAI_AMD_SP_VIRTUAL,		/**< AMD ACP SP VIRTUAL */
 	SOF_DAI_AMD_HS_VIRTUAL,		/**< AMD ACP HS VIRTUAL */
+	SOF_DAI_IMX_MICFIL,		/** < i.MX MICFIL PDM */
 };
 
 /* general purpose DAI configuration */
@@ -117,6 +118,7 @@ struct sof_ipc_dai_config {
 		struct sof_ipc_dai_acpdmic_params acpdmic;
 		struct sof_ipc_dai_acp_params acphs;
 		struct sof_ipc_dai_mtk_afe_params afe;
+		struct sof_ipc_dai_micfil_params micfil;
 	};
 } __packed;
 
diff --git a/include/trace/events/afs.h b/include/trace/events/afs.h
index e9d412d19dbb..597677acc6b1 100644
--- a/include/trace/events/afs.h
+++ b/include/trace/events/afs.h
@@ -18,97 +18,6 @@
 #ifndef __AFS_DECLARE_TRACE_ENUMS_ONCE_ONLY
 #define __AFS_DECLARE_TRACE_ENUMS_ONCE_ONLY
 
-enum afs_call_trace {
-	afs_call_trace_alloc,
-	afs_call_trace_free,
-	afs_call_trace_get,
-	afs_call_trace_put,
-	afs_call_trace_wake,
-	afs_call_trace_work,
-};
-
-enum afs_server_trace {
-	afs_server_trace_alloc,
-	afs_server_trace_callback,
-	afs_server_trace_destroy,
-	afs_server_trace_free,
-	afs_server_trace_gc,
-	afs_server_trace_get_by_addr,
-	afs_server_trace_get_by_uuid,
-	afs_server_trace_get_caps,
-	afs_server_trace_get_install,
-	afs_server_trace_get_new_cbi,
-	afs_server_trace_get_probe,
-	afs_server_trace_give_up_cb,
-	afs_server_trace_purging,
-	afs_server_trace_put_call,
-	afs_server_trace_put_cbi,
-	afs_server_trace_put_find_rsq,
-	afs_server_trace_put_probe,
-	afs_server_trace_put_slist,
-	afs_server_trace_put_slist_isort,
-	afs_server_trace_put_uuid_rsq,
-	afs_server_trace_update,
-};
-
-
-enum afs_volume_trace {
-	afs_volume_trace_alloc,
-	afs_volume_trace_free,
-	afs_volume_trace_get_alloc_sbi,
-	afs_volume_trace_get_cell_insert,
-	afs_volume_trace_get_new_op,
-	afs_volume_trace_get_query_alias,
-	afs_volume_trace_put_cell_dup,
-	afs_volume_trace_put_cell_root,
-	afs_volume_trace_put_destroy_sbi,
-	afs_volume_trace_put_free_fc,
-	afs_volume_trace_put_put_op,
-	afs_volume_trace_put_query_alias,
-	afs_volume_trace_put_validate_fc,
-	afs_volume_trace_remove,
-};
-
-enum afs_cell_trace {
-	afs_cell_trace_alloc,
-	afs_cell_trace_free,
-	afs_cell_trace_get_queue_dns,
-	afs_cell_trace_get_queue_manage,
-	afs_cell_trace_get_queue_new,
-	afs_cell_trace_get_vol,
-	afs_cell_trace_insert,
-	afs_cell_trace_manage,
-	afs_cell_trace_put_candidate,
-	afs_cell_trace_put_destroy,
-	afs_cell_trace_put_queue_fail,
-	afs_cell_trace_put_queue_work,
-	afs_cell_trace_put_vol,
-	afs_cell_trace_see_source,
-	afs_cell_trace_see_ws,
-	afs_cell_trace_unuse_alias,
-	afs_cell_trace_unuse_check_alias,
-	afs_cell_trace_unuse_delete,
-	afs_cell_trace_unuse_fc,
-	afs_cell_trace_unuse_lookup,
-	afs_cell_trace_unuse_mntpt,
-	afs_cell_trace_unuse_no_pin,
-	afs_cell_trace_unuse_parse,
-	afs_cell_trace_unuse_pin,
-	afs_cell_trace_unuse_probe,
-	afs_cell_trace_unuse_sbi,
-	afs_cell_trace_unuse_ws,
-	afs_cell_trace_use_alias,
-	afs_cell_trace_use_check_alias,
-	afs_cell_trace_use_fc,
-	afs_cell_trace_use_fc_alias,
-	afs_cell_trace_use_lookup,
-	afs_cell_trace_use_mntpt,
-	afs_cell_trace_use_pin,
-	afs_cell_trace_use_probe,
-	afs_cell_trace_use_sbi,
-	afs_cell_trace_wait,
-};
-
 enum afs_fs_operation {
 	afs_FS_FetchData		= 130,	/* AFS Fetch file data */
 	afs_FS_FetchACL			= 131,	/* AFS Fetch file ACL */
@@ -202,121 +111,6 @@ enum yfs_cm_operation {
 	yfs_CB_CallBack			= 64204,
 };
 
-enum afs_edit_dir_op {
-	afs_edit_dir_create,
-	afs_edit_dir_create_error,
-	afs_edit_dir_create_inval,
-	afs_edit_dir_create_nospc,
-	afs_edit_dir_delete,
-	afs_edit_dir_delete_error,
-	afs_edit_dir_delete_inval,
-	afs_edit_dir_delete_noent,
-};
-
-enum afs_edit_dir_reason {
-	afs_edit_dir_for_create,
-	afs_edit_dir_for_link,
-	afs_edit_dir_for_mkdir,
-	afs_edit_dir_for_rename_0,
-	afs_edit_dir_for_rename_1,
-	afs_edit_dir_for_rename_2,
-	afs_edit_dir_for_rmdir,
-	afs_edit_dir_for_silly_0,
-	afs_edit_dir_for_silly_1,
-	afs_edit_dir_for_symlink,
-	afs_edit_dir_for_unlink,
-};
-
-enum afs_eproto_cause {
-	afs_eproto_bad_status,
-	afs_eproto_cb_count,
-	afs_eproto_cb_fid_count,
-	afs_eproto_cellname_len,
-	afs_eproto_file_type,
-	afs_eproto_ibulkst_cb_count,
-	afs_eproto_ibulkst_count,
-	afs_eproto_motd_len,
-	afs_eproto_offline_msg_len,
-	afs_eproto_volname_len,
-	afs_eproto_yvl_fsendpt4_len,
-	afs_eproto_yvl_fsendpt6_len,
-	afs_eproto_yvl_fsendpt_num,
-	afs_eproto_yvl_fsendpt_type,
-	afs_eproto_yvl_vlendpt4_len,
-	afs_eproto_yvl_vlendpt6_len,
-	afs_eproto_yvl_vlendpt_type,
-};
-
-enum afs_io_error {
-	afs_io_error_cm_reply,
-	afs_io_error_extract,
-	afs_io_error_fs_probe_fail,
-	afs_io_error_vl_lookup_fail,
-	afs_io_error_vl_probe_fail,
-};
-
-enum afs_file_error {
-	afs_file_error_dir_bad_magic,
-	afs_file_error_dir_big,
-	afs_file_error_dir_missing_page,
-	afs_file_error_dir_name_too_long,
-	afs_file_error_dir_over_end,
-	afs_file_error_dir_small,
-	afs_file_error_dir_unmarked_ext,
-	afs_file_error_mntpt,
-	afs_file_error_writeback_fail,
-};
-
-enum afs_flock_event {
-	afs_flock_acquired,
-	afs_flock_callback_break,
-	afs_flock_defer_unlock,
-	afs_flock_extend_fail,
-	afs_flock_fail_other,
-	afs_flock_fail_perm,
-	afs_flock_no_lockers,
-	afs_flock_release_fail,
-	afs_flock_silly_delete,
-	afs_flock_timestamp,
-	afs_flock_try_to_lock,
-	afs_flock_vfs_lock,
-	afs_flock_vfs_locking,
-	afs_flock_waited,
-	afs_flock_waiting,
-	afs_flock_work_extending,
-	afs_flock_work_retry,
-	afs_flock_work_unlocking,
-	afs_flock_would_block,
-};
-
-enum afs_flock_operation {
-	afs_flock_op_copy_lock,
-	afs_flock_op_flock,
-	afs_flock_op_grant,
-	afs_flock_op_lock,
-	afs_flock_op_release_lock,
-	afs_flock_op_return_ok,
-	afs_flock_op_return_eagain,
-	afs_flock_op_return_edeadlk,
-	afs_flock_op_return_error,
-	afs_flock_op_set_lock,
-	afs_flock_op_unlock,
-	afs_flock_op_wake,
-};
-
-enum afs_cb_break_reason {
-	afs_cb_break_no_break,
-	afs_cb_break_no_promise,
-	afs_cb_break_for_callback,
-	afs_cb_break_for_deleted,
-	afs_cb_break_for_lapsed,
-	afs_cb_break_for_s_reinit,
-	afs_cb_break_for_unlink,
-	afs_cb_break_for_v_break,
-	afs_cb_break_for_volume_callback,
-	afs_cb_break_for_zap,
-};
-
 #endif /* end __AFS_DECLARE_TRACE_ENUMS_ONCE_ONLY */
 
 /*
@@ -391,6 +185,7 @@ enum afs_cb_break_reason {
 	EM(afs_cell_trace_unuse_fc,		"UNU fc    ") \
 	EM(afs_cell_trace_unuse_lookup,		"UNU lookup") \
 	EM(afs_cell_trace_unuse_mntpt,		"UNU mntpt ") \
+	EM(afs_cell_trace_unuse_no_pin,		"UNU no-pin") \
 	EM(afs_cell_trace_unuse_parse,		"UNU parse ") \
 	EM(afs_cell_trace_unuse_pin,		"UNU pin   ") \
 	EM(afs_cell_trace_unuse_probe,		"UNU probe ") \
@@ -614,6 +409,32 @@ enum afs_cb_break_reason {
 	EM(afs_cb_break_for_volume_callback,	"break-v-cb")		\
 	E_(afs_cb_break_for_zap,		"break-zap")
 
+/*
+ * Generate enums for tracing information.
+ */
+#ifndef __AFS_GENERATE_TRACE_ENUMS_ONCE_ONLY
+#define __AFS_GENERATE_TRACE_ENUMS_ONCE_ONLY
+
+#undef EM
+#undef E_
+#define EM(a, b) a,
+#define E_(a, b) a
+
+enum afs_call_trace		{ afs_call_traces } __mode(byte);
+enum afs_cb_break_reason	{ afs_cb_break_reasons } __mode(byte);
+enum afs_cell_trace		{ afs_cell_traces } __mode(byte);
+enum afs_edit_dir_op		{ afs_edit_dir_ops } __mode(byte);
+enum afs_edit_dir_reason	{ afs_edit_dir_reasons } __mode(byte);
+enum afs_eproto_cause		{ afs_eproto_causes } __mode(byte);
+enum afs_file_error		{ afs_file_errors } __mode(byte);
+enum afs_flock_event		{ afs_flock_events } __mode(byte);
+enum afs_flock_operation	{ afs_flock_operations } __mode(byte);
+enum afs_io_error		{ afs_io_errors } __mode(byte);
+enum afs_server_trace		{ afs_server_traces } __mode(byte);
+enum afs_volume_trace		{ afs_volume_traces } __mode(byte);
+
+#endif /* end __AFS_GENERATE_TRACE_ENUMS_ONCE_ONLY */
+
 /*
  * Export enum symbols via userspace.
  */
@@ -654,12 +475,12 @@ TRACE_EVENT(afs_receive_data,
 	    TP_ARGS(call, iter, want_more, ret),
 
 	    TP_STRUCT__entry(
-		    __field(loff_t,			remain		)
-		    __field(unsigned int,		call		)
-		    __field(enum afs_call_state,	state		)
-		    __field(unsigned short,		unmarshall	)
-		    __field(bool,			want_more	)
-		    __field(int,			ret		)
+		    __field(loff_t,			remain)
+		    __field(unsigned int,		call)
+		    __field(enum afs_call_state,	state)
+		    __field(unsigned short,		unmarshall)
+		    __field(bool,			want_more)
+		    __field(int,			ret)
 			     ),
 
 	    TP_fast_assign(
@@ -686,9 +507,9 @@ TRACE_EVENT(afs_notify_call,
 	    TP_ARGS(rxcall, call),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,		call		)
-		    __field(enum afs_call_state,	state		)
-		    __field(unsigned short,		unmarshall	)
+		    __field(unsigned int,		call)
+		    __field(enum afs_call_state,	state)
+		    __field(unsigned short,		unmarshall)
 			     ),
 
 	    TP_fast_assign(
@@ -708,9 +529,9 @@ TRACE_EVENT(afs_cb_call,
 	    TP_ARGS(call),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,		call		)
-		    __field(u32,			op		)
-		    __field(u16,			service_id	)
+		    __field(unsigned int,		call)
+		    __field(u32,			op)
+		    __field(u16,			service_id)
 			     ),
 
 	    TP_fast_assign(
@@ -733,11 +554,11 @@ TRACE_EVENT(afs_call,
 	    TP_ARGS(call_debug_id, op, ref, outstanding, where),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,		call		)
-		    __field(int,			op		)
-		    __field(int,			ref		)
-		    __field(int,			outstanding	)
-		    __field(const void *,		where		)
+		    __field(unsigned int,		call)
+		    __field(int,			op)
+		    __field(int,			ref)
+		    __field(int,			outstanding)
+		    __field(const void *,		where)
 			     ),
 
 	    TP_fast_assign(
@@ -762,9 +583,9 @@ TRACE_EVENT(afs_make_fs_call,
 	    TP_ARGS(call, fid),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,		call		)
-		    __field(enum afs_fs_operation,	op		)
-		    __field_struct(struct afs_fid,	fid		)
+		    __field(unsigned int,		call)
+		    __field(enum afs_fs_operation,	op)
+		    __field_struct(struct afs_fid,	fid)
 			     ),
 
 	    TP_fast_assign(
@@ -794,10 +615,10 @@ TRACE_EVENT(afs_make_fs_calli,
 	    TP_ARGS(call, fid, i),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,		call		)
-		    __field(unsigned int,		i		)
-		    __field(enum afs_fs_operation,	op		)
-		    __field_struct(struct afs_fid,	fid		)
+		    __field(unsigned int,		call)
+		    __field(unsigned int,		i)
+		    __field(enum afs_fs_operation,	op)
+		    __field_struct(struct afs_fid,	fid)
 			     ),
 
 	    TP_fast_assign(
@@ -829,10 +650,10 @@ TRACE_EVENT(afs_make_fs_call1,
 	    TP_ARGS(call, fid, name),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,		call		)
-		    __field(enum afs_fs_operation,	op		)
-		    __field_struct(struct afs_fid,	fid		)
-		    __array(char,			name, 24	)
+		    __field(unsigned int,		call)
+		    __field(enum afs_fs_operation,	op)
+		    __field_struct(struct afs_fid,	fid)
+		    __array(char,			name, 24)
 			     ),
 
 	    TP_fast_assign(
@@ -866,11 +687,11 @@ TRACE_EVENT(afs_make_fs_call2,
 	    TP_ARGS(call, fid, name, name2),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,		call		)
-		    __field(enum afs_fs_operation,	op		)
-		    __field_struct(struct afs_fid,	fid		)
-		    __array(char,			name, 24	)
-		    __array(char,			name2, 24	)
+		    __field(unsigned int,		call)
+		    __field(enum afs_fs_operation,	op)
+		    __field_struct(struct afs_fid,	fid)
+		    __array(char,			name, 24)
+		    __array(char,			name2, 24)
 			     ),
 
 	    TP_fast_assign(
@@ -907,8 +728,8 @@ TRACE_EVENT(afs_make_vl_call,
 	    TP_ARGS(call),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,		call		)
-		    __field(enum afs_vl_operation,	op		)
+		    __field(unsigned int,		call)
+		    __field(enum afs_vl_operation,	op)
 			     ),
 
 	    TP_fast_assign(
@@ -927,10 +748,10 @@ TRACE_EVENT(afs_call_done,
 	    TP_ARGS(call),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,		call		)
-		    __field(struct rxrpc_call *,	rx_call		)
-		    __field(int,			ret		)
-		    __field(u32,			abort_code	)
+		    __field(unsigned int,		call)
+		    __field(struct rxrpc_call *,	rx_call)
+		    __field(int,			ret)
+		    __field(u32,			abort_code)
 			     ),
 
 	    TP_fast_assign(
@@ -953,10 +774,10 @@ TRACE_EVENT(afs_send_data,
 	    TP_ARGS(call, msg),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,		call		)
-		    __field(unsigned int,		flags		)
-		    __field(loff_t,			offset		)
-		    __field(loff_t,			count		)
+		    __field(unsigned int,		call)
+		    __field(unsigned int,		flags)
+		    __field(loff_t,			offset)
+		    __field(loff_t,			count)
 			     ),
 
 	    TP_fast_assign(
@@ -977,10 +798,10 @@ TRACE_EVENT(afs_sent_data,
 	    TP_ARGS(call, msg, ret),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,		call		)
-		    __field(int,			ret		)
-		    __field(loff_t,			offset		)
-		    __field(loff_t,			count		)
+		    __field(unsigned int,		call)
+		    __field(int,			ret)
+		    __field(loff_t,			offset)
+		    __field(loff_t,			count)
 			     ),
 
 	    TP_fast_assign(
@@ -1001,9 +822,9 @@ TRACE_EVENT(afs_dir_check_failed,
 	    TP_ARGS(vnode, off, i_size),
 
 	    TP_STRUCT__entry(
-		    __field(struct afs_vnode *,		vnode		)
-		    __field(loff_t,			off		)
-		    __field(loff_t,			i_size		)
+		    __field(struct afs_vnode *,		vnode)
+		    __field(loff_t,			off)
+		    __field(loff_t,			i_size)
 			     ),
 
 	    TP_fast_assign(
@@ -1022,11 +843,11 @@ TRACE_EVENT(afs_folio_dirty,
 	    TP_ARGS(vnode, where, folio),
 
 	    TP_STRUCT__entry(
-		    __field(struct afs_vnode *,		vnode		)
-		    __field(const char *,		where		)
-		    __field(pgoff_t,			index		)
-		    __field(unsigned long,		from		)
-		    __field(unsigned long,		to		)
+		    __field(struct afs_vnode *,		vnode)
+		    __field(const char *,		where)
+		    __field(pgoff_t,			index)
+		    __field(unsigned long,		from)
+		    __field(unsigned long,		to)
 			     ),
 
 	    TP_fast_assign(
@@ -1056,11 +877,11 @@ TRACE_EVENT(afs_call_state,
 	    TP_ARGS(call, from, to, ret, remote_abort),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,		call		)
-		    __field(enum afs_call_state,	from		)
-		    __field(enum afs_call_state,	to		)
-		    __field(int,			ret		)
-		    __field(u32,			abort		)
+		    __field(unsigned int,		call)
+		    __field(enum afs_call_state,	from)
+		    __field(enum afs_call_state,	to)
+		    __field(int,			ret)
+		    __field(u32,			abort)
 			     ),
 
 	    TP_fast_assign(
@@ -1084,9 +905,9 @@ TRACE_EVENT(afs_lookup,
 	    TP_ARGS(dvnode, name, fid),
 
 	    TP_STRUCT__entry(
-		    __field_struct(struct afs_fid,	dfid		)
-		    __field_struct(struct afs_fid,	fid		)
-		    __array(char,			name, 24	)
+		    __field_struct(struct afs_fid,	dfid)
+		    __field_struct(struct afs_fid,	fid)
+		    __array(char,			name, 24)
 			     ),
 
 	    TP_fast_assign(
@@ -1116,15 +937,15 @@ TRACE_EVENT(afs_edit_dir,
 	    TP_ARGS(dvnode, why, op, block, slot, f_vnode, f_unique, name),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,		vnode		)
-		    __field(unsigned int,		unique		)
-		    __field(enum afs_edit_dir_reason,	why		)
-		    __field(enum afs_edit_dir_op,	op		)
-		    __field(unsigned int,		block		)
-		    __field(unsigned short,		slot		)
-		    __field(unsigned int,		f_vnode		)
-		    __field(unsigned int,		f_unique	)
-		    __array(char,			name, 24	)
+		    __field(unsigned int,		vnode)
+		    __field(unsigned int,		unique)
+		    __field(enum afs_edit_dir_reason,	why)
+		    __field(enum afs_edit_dir_op,	op)
+		    __field(unsigned int,		block)
+		    __field(unsigned short,		slot)
+		    __field(unsigned int,		f_vnode)
+		    __field(unsigned int,		f_unique)
+		    __array(char,			name, 24)
 			     ),
 
 	    TP_fast_assign(
@@ -1157,8 +978,8 @@ TRACE_EVENT(afs_protocol_error,
 	    TP_ARGS(call, cause),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,		call		)
-		    __field(enum afs_eproto_cause,	cause		)
+		    __field(unsigned int,		call)
+		    __field(enum afs_eproto_cause,	cause)
 			     ),
 
 	    TP_fast_assign(
@@ -1177,9 +998,9 @@ TRACE_EVENT(afs_io_error,
 	    TP_ARGS(call, error, where),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,	call		)
-		    __field(int,		error		)
-		    __field(enum afs_io_error,	where		)
+		    __field(unsigned int,	call)
+		    __field(int,		error)
+		    __field(enum afs_io_error,	where)
 			     ),
 
 	    TP_fast_assign(
@@ -1199,9 +1020,9 @@ TRACE_EVENT(afs_file_error,
 	    TP_ARGS(vnode, error, where),
 
 	    TP_STRUCT__entry(
-		    __field_struct(struct afs_fid,	fid		)
-		    __field(int,			error		)
-		    __field(enum afs_file_error,	where		)
+		    __field_struct(struct afs_fid,	fid)
+		    __field(int,			error)
+		    __field(enum afs_file_error,	where)
 			     ),
 
 	    TP_fast_assign(
@@ -1222,9 +1043,9 @@ TRACE_EVENT(afs_cm_no_server,
 	    TP_ARGS(call, srx),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,			call	)
-		    __field(unsigned int,			op_id	)
-		    __field_struct(struct sockaddr_rxrpc,	srx	)
+		    __field(unsigned int,			call)
+		    __field(unsigned int,			op_id)
+		    __field_struct(struct sockaddr_rxrpc,	srx)
 			     ),
 
 	    TP_fast_assign(
@@ -1243,9 +1064,9 @@ TRACE_EVENT(afs_cm_no_server_u,
 	    TP_ARGS(call, uuid),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,			call	)
-		    __field(unsigned int,			op_id	)
-		    __field_struct(uuid_t,			uuid	)
+		    __field(unsigned int,			call)
+		    __field(unsigned int,			op_id)
+		    __field_struct(uuid_t,			uuid)
 			     ),
 
 	    TP_fast_assign(
@@ -1265,11 +1086,11 @@ TRACE_EVENT(afs_flock_ev,
 	    TP_ARGS(vnode, fl, event, error),
 
 	    TP_STRUCT__entry(
-		    __field_struct(struct afs_fid,	fid		)
-		    __field(enum afs_flock_event,	event		)
-		    __field(enum afs_lock_state,	state		)
-		    __field(int,			error		)
-		    __field(unsigned int,		debug_id	)
+		    __field_struct(struct afs_fid,	fid)
+		    __field(enum afs_flock_event,	event)
+		    __field(enum afs_lock_state,	state)
+		    __field(int,			error)
+		    __field(unsigned int,		debug_id)
 			     ),
 
 	    TP_fast_assign(
@@ -1295,13 +1116,13 @@ TRACE_EVENT(afs_flock_op,
 	    TP_ARGS(vnode, fl, op),
 
 	    TP_STRUCT__entry(
-		    __field_struct(struct afs_fid,	fid		)
-		    __field(loff_t,			from		)
-		    __field(loff_t,			len		)
-		    __field(enum afs_flock_operation,	op		)
-		    __field(unsigned char,		type		)
-		    __field(unsigned int,		flags		)
-		    __field(unsigned int,		debug_id	)
+		    __field_struct(struct afs_fid,	fid)
+		    __field(loff_t,			from)
+		    __field(loff_t,			len)
+		    __field(enum afs_flock_operation,	op)
+		    __field(unsigned char,		type)
+		    __field(unsigned int,		flags)
+		    __field(unsigned int,		debug_id)
 			     ),
 
 	    TP_fast_assign(
@@ -1328,7 +1149,7 @@ TRACE_EVENT(afs_reload_dir,
 	    TP_ARGS(vnode),
 
 	    TP_STRUCT__entry(
-		    __field_struct(struct afs_fid,	fid		)
+		    __field_struct(struct afs_fid,	fid)
 			     ),
 
 	    TP_fast_assign(
@@ -1345,8 +1166,8 @@ TRACE_EVENT(afs_silly_rename,
 	    TP_ARGS(vnode, done),
 
 	    TP_STRUCT__entry(
-		    __field_struct(struct afs_fid,	fid		)
-		    __field(bool,			done		)
+		    __field_struct(struct afs_fid,	fid)
+		    __field(bool,			done)
 			     ),
 
 	    TP_fast_assign(
@@ -1365,9 +1186,9 @@ TRACE_EVENT(afs_get_tree,
 	    TP_ARGS(cell, volume),
 
 	    TP_STRUCT__entry(
-		    __field(u64,			vid		)
-		    __array(char,			cell, 24	)
-		    __array(char,			volume, 24	)
+		    __field(u64,			vid)
+		    __array(char,			cell, 24)
+		    __array(char,			volume, 24)
 			     ),
 
 	    TP_fast_assign(
@@ -1392,10 +1213,10 @@ TRACE_EVENT(afs_cb_break,
 	    TP_ARGS(fid, cb_break, reason, skipped),
 
 	    TP_STRUCT__entry(
-		    __field_struct(struct afs_fid,	fid		)
-		    __field(unsigned int,		cb_break	)
-		    __field(enum afs_cb_break_reason,	reason		)
-		    __field(bool,			skipped		)
+		    __field_struct(struct afs_fid,	fid)
+		    __field(unsigned int,		cb_break)
+		    __field(enum afs_cb_break_reason,	reason)
+		    __field(bool,			skipped)
 			     ),
 
 	    TP_fast_assign(
@@ -1418,8 +1239,8 @@ TRACE_EVENT(afs_cb_miss,
 	    TP_ARGS(fid, reason),
 
 	    TP_STRUCT__entry(
-		    __field_struct(struct afs_fid,	fid		)
-		    __field(enum afs_cb_break_reason,	reason		)
+		    __field_struct(struct afs_fid,	fid)
+		    __field(enum afs_cb_break_reason,	reason)
 			     ),
 
 	    TP_fast_assign(
@@ -1439,10 +1260,10 @@ TRACE_EVENT(afs_server,
 	    TP_ARGS(server_debug_id, ref, active, reason),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,		server		)
-		    __field(int,			ref		)
-		    __field(int,			active		)
-		    __field(int,			reason		)
+		    __field(unsigned int,		server)
+		    __field(int,			ref)
+		    __field(int,			active)
+		    __field(int,			reason)
 			     ),
 
 	    TP_fast_assign(
@@ -1465,9 +1286,9 @@ TRACE_EVENT(afs_volume,
 	    TP_ARGS(vid, ref, reason),
 
 	    TP_STRUCT__entry(
-		    __field(afs_volid_t,		vid		)
-		    __field(int,			ref		)
-		    __field(enum afs_volume_trace,	reason		)
+		    __field(afs_volid_t,		vid)
+		    __field(int,			ref)
+		    __field(enum afs_volume_trace,	reason)
 			     ),
 
 	    TP_fast_assign(
@@ -1489,10 +1310,10 @@ TRACE_EVENT(afs_cell,
 	    TP_ARGS(cell_debug_id, ref, active, reason),
 
 	    TP_STRUCT__entry(
-		    __field(unsigned int,		cell		)
-		    __field(int,			ref		)
-		    __field(int,			active		)
-		    __field(int,			reason		)
+		    __field(unsigned int,		cell)
+		    __field(int,			ref)
+		    __field(int,			active)
+		    __field(int,			reason)
 			     ),
 
 	    TP_fast_assign(
diff --git a/include/trace/events/rxrpc.h b/include/trace/events/rxrpc.h
index 4c53a5ef6257..f7e537f64db4 100644
--- a/include/trace/events/rxrpc.h
+++ b/include/trace/events/rxrpc.h
@@ -328,7 +328,7 @@
 	E_(rxrpc_rtt_tx_ping,			"PING")
 
 #define rxrpc_rtt_rx_traces \
-	EM(rxrpc_rtt_rx_cancel,			"CNCL") \
+	EM(rxrpc_rtt_rx_other_ack,		"OACK") \
 	EM(rxrpc_rtt_rx_obsolete,		"OBSL") \
 	EM(rxrpc_rtt_rx_lost,			"LOST") \
 	EM(rxrpc_rtt_rx_ping_response,		"PONG") \
diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h
index 6188ad0d9e0d..dbb01b4b7451 100644
--- a/include/trace/events/sched.h
+++ b/include/trace/events/sched.h
@@ -493,33 +493,30 @@ DEFINE_EVENT_SCHEDSTAT(sched_stat_template, sched_stat_blocked,
  */
 DECLARE_EVENT_CLASS(sched_stat_runtime,
 
-	TP_PROTO(struct task_struct *tsk, u64 runtime, u64 vruntime),
+	TP_PROTO(struct task_struct *tsk, u64 runtime),
 
-	TP_ARGS(tsk, __perf_count(runtime), vruntime),
+	TP_ARGS(tsk, __perf_count(runtime)),
 
 	TP_STRUCT__entry(
 		__array( char,	comm,	TASK_COMM_LEN	)
 		__field( pid_t,	pid			)
 		__field( u64,	runtime			)
-		__field( u64,	vruntime			)
 	),
 
 	TP_fast_assign(
 		memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN);
 		__entry->pid		= tsk->pid;
 		__entry->runtime	= runtime;
-		__entry->vruntime	= vruntime;
 	),
 
-	TP_printk("comm=%s pid=%d runtime=%Lu [ns] vruntime=%Lu [ns]",
+	TP_printk("comm=%s pid=%d runtime=%Lu [ns]",
 			__entry->comm, __entry->pid,
-			(unsigned long long)__entry->runtime,
-			(unsigned long long)__entry->vruntime)
+			(unsigned long long)__entry->runtime)
 );
 
 DEFINE_EVENT(sched_stat_runtime, sched_stat_runtime,
-	     TP_PROTO(struct task_struct *tsk, u64 runtime, u64 vruntime),
-	     TP_ARGS(tsk, runtime, vruntime));
+	     TP_PROTO(struct task_struct *tsk, u64 runtime),
+	     TP_ARGS(tsk, runtime));
 
 /*
  * Tracepoint for showing priority inheritance modifying a tasks
diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h
index 756b013fb832..75f00965ab15 100644
--- a/include/uapi/asm-generic/unistd.h
+++ b/include/uapi/asm-generic/unistd.h
@@ -829,8 +829,21 @@ __SYSCALL(__NR_futex_wait, sys_futex_wait)
 #define __NR_futex_requeue 456
 __SYSCALL(__NR_futex_requeue, sys_futex_requeue)
 
+#define __NR_statmount   457
+__SYSCALL(__NR_statmount, sys_statmount)
+
+#define __NR_listmount   458
+__SYSCALL(__NR_listmount, sys_listmount)
+
+#define __NR_lsm_get_self_attr 459
+__SYSCALL(__NR_lsm_get_self_attr, sys_lsm_get_self_attr)
+#define __NR_lsm_set_self_attr 460
+__SYSCALL(__NR_lsm_set_self_attr, sys_lsm_set_self_attr)
+#define __NR_lsm_list_modules 461
+__SYSCALL(__NR_lsm_list_modules, sys_lsm_list_modules)
+
 #undef __NR_syscalls
-#define __NR_syscalls 457
+#define __NR_syscalls 462
 
 /*
  * 32 bit systems traditionally used different
diff --git a/include/uapi/drm/drm.h b/include/uapi/drm/drm.h
index de723566c5ae..8662b5aeea0c 100644
--- a/include/uapi/drm/drm.h
+++ b/include/uapi/drm/drm.h
@@ -1218,6 +1218,26 @@ extern "C" {
 
 #define DRM_IOCTL_SYNCOBJ_EVENTFD	DRM_IOWR(0xCF, struct drm_syncobj_eventfd)
 
+/**
+ * DRM_IOCTL_MODE_CLOSEFB - Close a framebuffer.
+ *
+ * This closes a framebuffer previously added via ADDFB/ADDFB2. The IOCTL
+ * argument is a framebuffer object ID.
+ *
+ * This IOCTL is similar to &DRM_IOCTL_MODE_RMFB, except it doesn't disable
+ * planes and CRTCs. As long as the framebuffer is used by a plane, it's kept
+ * alive. When the plane no longer uses the framebuffer (because the
+ * framebuffer is replaced with another one, or the plane is disabled), the
+ * framebuffer is cleaned up.
+ *
+ * This is useful to implement flicker-free transitions between two processes.
+ *
+ * Depending on the threat model, user-space may want to ensure that the
+ * framebuffer doesn't expose any sensitive user information: closed
+ * framebuffers attached to a plane can be read back by the next DRM master.
+ */
+#define DRM_IOCTL_MODE_CLOSEFB		DRM_IOWR(0xD0, struct drm_mode_closefb)
+
 /*
  * Device specific ioctls should only be in their respective headers
  * The device specific ioctl range is from 0x40 to 0x9f.
diff --git a/include/uapi/drm/drm_mode.h b/include/uapi/drm/drm_mode.h
index 128d09138ceb..09e7a471ee30 100644
--- a/include/uapi/drm/drm_mode.h
+++ b/include/uapi/drm/drm_mode.h
@@ -1323,6 +1323,16 @@ struct drm_mode_rect {
 	__s32 y2;
 };
 
+/**
+ * struct drm_mode_closefb
+ * @fb_id: Framebuffer ID.
+ * @pad: Must be zero.
+ */
+struct drm_mode_closefb {
+	__u32 fb_id;
+	__u32 pad;
+};
+
 #if defined(__cplusplus)
 }
 #endif
diff --git a/include/uapi/drm/ivpu_accel.h b/include/uapi/drm/ivpu_accel.h
index 262db0c3beee..de1944e42c65 100644
--- a/include/uapi/drm/ivpu_accel.h
+++ b/include/uapi/drm/ivpu_accel.h
@@ -196,7 +196,7 @@ struct drm_ivpu_bo_create {
 	 *
 	 * %DRM_IVPU_BO_UNCACHED:
 	 *
-	 * Allocated BO will not be cached on host side nor snooped on the VPU side.
+	 * Not supported. Use DRM_IVPU_BO_WC instead.
 	 *
 	 * %DRM_IVPU_BO_WC:
 	 *
diff --git a/include/uapi/drm/qaic_accel.h b/include/uapi/drm/qaic_accel.h
index 43ac5d864512..9dab32316aee 100644
--- a/include/uapi/drm/qaic_accel.h
+++ b/include/uapi/drm/qaic_accel.h
@@ -287,8 +287,9 @@ struct qaic_execute_entry {
  * struct qaic_partial_execute_entry - Defines a BO to resize and submit.
  * @handle: In. GEM handle of the BO to commit to the device.
  * @dir: In. Direction of data. 1 = to device, 2 = from device.
- * @resize: In. New size of the BO.  Must be <= the original BO size.  0 is
- *	    short for no resize.
+ * @resize: In. New size of the BO.  Must be <= the original BO size.
+ *	    @resize as 0 would be interpreted as no DMA transfer is
+ *	    involved.
  */
 struct qaic_partial_execute_entry {
 	__u32 handle;
diff --git a/include/uapi/drm/v3d_drm.h b/include/uapi/drm/v3d_drm.h
index 3dfc0af8756a..1a7d7a689de3 100644
--- a/include/uapi/drm/v3d_drm.h
+++ b/include/uapi/drm/v3d_drm.h
@@ -319,6 +319,11 @@ struct drm_v3d_submit_tfu {
 
 	/* Pointer to an array of ioctl extensions*/
 	__u64 extensions;
+
+	struct {
+		__u32 ioc;
+		__u32 pad;
+	} v71;
 };
 
 /* Submits a compute shader for dispatch.  This job will block on any
diff --git a/include/uapi/drm/virtgpu_drm.h b/include/uapi/drm/virtgpu_drm.h
index b1d0e56565bc..c2ce71987e9b 100644
--- a/include/uapi/drm/virtgpu_drm.h
+++ b/include/uapi/drm/virtgpu_drm.h
@@ -97,6 +97,7 @@ struct drm_virtgpu_execbuffer {
 #define VIRTGPU_PARAM_CROSS_DEVICE 5 /* Cross virtio-device resource sharing  */
 #define VIRTGPU_PARAM_CONTEXT_INIT 6 /* DRM_VIRTGPU_CONTEXT_INIT */
 #define VIRTGPU_PARAM_SUPPORTED_CAPSET_IDs 7 /* Bitmask of supported capability set ids */
+#define VIRTGPU_PARAM_EXPLICIT_DEBUG_NAME 8 /* Ability to set debug name from userspace */
 
 struct drm_virtgpu_getparam {
 	__u64 param;
@@ -198,6 +199,7 @@ struct drm_virtgpu_resource_create_blob {
 #define VIRTGPU_CONTEXT_PARAM_CAPSET_ID       0x0001
 #define VIRTGPU_CONTEXT_PARAM_NUM_RINGS       0x0002
 #define VIRTGPU_CONTEXT_PARAM_POLL_RINGS_MASK 0x0003
+#define VIRTGPU_CONTEXT_PARAM_DEBUG_NAME      0x0004
 struct drm_virtgpu_context_set_param {
 	__u64 param;
 	__u64 value;
diff --git a/include/uapi/linux/batadv_packet.h b/include/uapi/linux/batadv_packet.h
index 9204e4494b25..6e25753015df 100644
--- a/include/uapi/linux/batadv_packet.h
+++ b/include/uapi/linux/batadv_packet.h
@@ -116,6 +116,9 @@ enum batadv_icmp_packettype {
  * only need routable IPv4 multicast packets we signed up for explicitly
  * @BATADV_MCAST_WANT_NO_RTR6: we have no IPv6 multicast router and therefore
  * only need routable IPv6 multicast packets we signed up for explicitly
+ * @BATADV_MCAST_HAVE_MC_PTYPE_CAPA: we can parse, receive and forward
+ * batman-adv multicast packets with a multicast tracker TVLV. And all our
+ * hard interfaces have an MTU of at least 1280 bytes.
  */
 enum batadv_mcast_flags {
 	BATADV_MCAST_WANT_ALL_UNSNOOPABLES	= 1UL << 0,
@@ -123,6 +126,7 @@ enum batadv_mcast_flags {
 	BATADV_MCAST_WANT_ALL_IPV6		= 1UL << 2,
 	BATADV_MCAST_WANT_NO_RTR4		= 1UL << 3,
 	BATADV_MCAST_WANT_NO_RTR6		= 1UL << 4,
+	BATADV_MCAST_HAVE_MC_PTYPE_CAPA		= 1UL << 5,
 };
 
 /* tt data subtypes */
@@ -174,14 +178,16 @@ enum batadv_bla_claimframe {
  * @BATADV_TVLV_TT: translation table tvlv
  * @BATADV_TVLV_ROAM: roaming advertisement tvlv
  * @BATADV_TVLV_MCAST: multicast capability tvlv
+ * @BATADV_TVLV_MCAST_TRACKER: multicast tracker tvlv
  */
 enum batadv_tvlv_type {
-	BATADV_TVLV_GW		= 0x01,
-	BATADV_TVLV_DAT		= 0x02,
-	BATADV_TVLV_NC		= 0x03,
-	BATADV_TVLV_TT		= 0x04,
-	BATADV_TVLV_ROAM	= 0x05,
-	BATADV_TVLV_MCAST	= 0x06,
+	BATADV_TVLV_GW			= 0x01,
+	BATADV_TVLV_DAT			= 0x02,
+	BATADV_TVLV_NC			= 0x03,
+	BATADV_TVLV_TT			= 0x04,
+	BATADV_TVLV_ROAM		= 0x05,
+	BATADV_TVLV_MCAST		= 0x06,
+	BATADV_TVLV_MCAST_TRACKER	= 0x07,
 };
 
 #pragma pack(2)
@@ -487,6 +493,25 @@ struct batadv_bcast_packet {
 	 */
 };
 
+/**
+ * struct batadv_mcast_packet - multicast packet for network payload
+ * @packet_type: batman-adv packet type, part of the general header
+ * @version: batman-adv protocol version, part of the general header
+ * @ttl: time to live for this packet, part of the general header
+ * @reserved: reserved byte for alignment
+ * @tvlv_len: length of the appended tvlv buffer (in bytes)
+ */
+struct batadv_mcast_packet {
+	__u8 packet_type;
+	__u8 version;
+	__u8 ttl;
+	__u8 reserved;
+	__be16 tvlv_len;
+	/* "4 bytes boundary + 2 bytes" long to make the payload after the
+	 * following ethernet header again 4 bytes boundary aligned
+	 */
+};
+
 /**
  * struct batadv_coded_packet - network coded packet
  * @packet_type: batman-adv packet type, part of the general header
@@ -628,6 +653,14 @@ struct batadv_tvlv_mcast_data {
 	__u8 reserved[3];
 };
 
+/**
+ * struct batadv_tvlv_mcast_tracker - payload of a multicast tracker tvlv
+ * @num_dests: number of subsequent destination originator MAC addresses
+ */
+struct batadv_tvlv_mcast_tracker {
+	__be16	num_dests;
+};
+
 #pragma pack()
 
 #endif /* _UAPI_LINUX_BATADV_PACKET_H_ */
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 0f6cdf52b1da..7a5498242eaa 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -1200,6 +1200,9 @@ enum bpf_perf_event_type {
  */
 #define BPF_F_XDP_DEV_BOUND_ONLY	(1U << 6)
 
+/* The verifier internal test flag. Behavior is undefined */
+#define BPF_F_TEST_REG_INVARIANTS	(1U << 7)
+
 /* link_create.kprobe_multi.flags used in LINK_CREATE command for
  * BPF_TRACE_KPROBE_MULTI attach type to create return probe.
  */
@@ -4517,6 +4520,8 @@ union bpf_attr {
  * long bpf_get_task_stack(struct task_struct *task, void *buf, u32 size, u64 flags)
  *	Description
  *		Return a user or a kernel stack in bpf program provided buffer.
+ *		Note: the user stack will only be populated if the *task* is
+ *		the current task; all other tasks will return -EOPNOTSUPP.
  *		To achieve this, the helper needs *task*, which is a valid
  *		pointer to **struct task_struct**. To store the stacktrace, the
  *		bpf program provides *buf* with a nonnegative *size*.
@@ -4528,6 +4533,7 @@ union bpf_attr {
  *
  *		**BPF_F_USER_STACK**
  *			Collect a user space stack instead of a kernel stack.
+ *			The *task* must be the current task.
  *		**BPF_F_USER_BUILD_ID**
  *			Collect buildid+offset instead of ips for user stack,
  *			only valid if **BPF_F_USER_STACK** is also specified.
@@ -7151,40 +7157,31 @@ struct bpf_spin_lock {
 };
 
 struct bpf_timer {
-	__u64 :64;
-	__u64 :64;
+	__u64 __opaque[2];
 } __attribute__((aligned(8)));
 
 struct bpf_dynptr {
-	__u64 :64;
-	__u64 :64;
+	__u64 __opaque[2];
 } __attribute__((aligned(8)));
 
 struct bpf_list_head {
-	__u64 :64;
-	__u64 :64;
+	__u64 __opaque[2];
 } __attribute__((aligned(8)));
 
 struct bpf_list_node {
-	__u64 :64;
-	__u64 :64;
-	__u64 :64;
+	__u64 __opaque[3];
 } __attribute__((aligned(8)));
 
 struct bpf_rb_root {
-	__u64 :64;
-	__u64 :64;
+	__u64 __opaque[2];
 } __attribute__((aligned(8)));
 
 struct bpf_rb_node {
-	__u64 :64;
-	__u64 :64;
-	__u64 :64;
-	__u64 :64;
+	__u64 __opaque[4];
 } __attribute__((aligned(8)));
 
 struct bpf_refcount {
-	__u32 :32;
+	__u32 __opaque[1];
 } __attribute__((aligned(4)));
 
 struct bpf_sysctl {
diff --git a/include/uapi/linux/fcntl.h b/include/uapi/linux/fcntl.h
index 6c80f96049bd..282e90aeb163 100644
--- a/include/uapi/linux/fcntl.h
+++ b/include/uapi/linux/fcntl.h
@@ -116,5 +116,8 @@
 #define AT_HANDLE_FID		AT_REMOVEDIR	/* file handle is needed to
 					compare object identity and may not
 					be usable to open_by_handle_at(2) */
+#if defined(__KERNEL__)
+#define AT_GETATTR_NOSEC	0x80000000
+#endif
 
 #endif /* _UAPI_LINUX_FCNTL_H */
diff --git a/include/uapi/linux/fs.h b/include/uapi/linux/fs.h
index da43810b7485..48ad69f7722e 100644
--- a/include/uapi/linux/fs.h
+++ b/include/uapi/linux/fs.h
@@ -316,6 +316,7 @@ typedef int __bitwise __kernel_rwf_t;
 #define PAGE_IS_SWAPPED		(1 << 4)
 #define PAGE_IS_PFNZERO		(1 << 5)
 #define PAGE_IS_HUGE		(1 << 6)
+#define PAGE_IS_SOFT_DIRTY	(1 << 7)
 
 /*
  * struct page_region - Page region with flags
diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h
index db92a7202b34..e7418d15fe39 100644
--- a/include/uapi/linux/fuse.h
+++ b/include/uapi/linux/fuse.h
@@ -209,7 +209,7 @@
  *  - add FUSE_HAS_EXPIRE_ONLY
  *
  *  7.39
- *  - add FUSE_DIRECT_IO_RELAX
+ *  - add FUSE_DIRECT_IO_ALLOW_MMAP
  *  - add FUSE_STATX and related structures
  */
 
@@ -409,8 +409,7 @@ struct fuse_file_lock {
  * FUSE_CREATE_SUPP_GROUP: add supplementary group info to create, mkdir,
  *			symlink and mknod (single group that matches parent)
  * FUSE_HAS_EXPIRE_ONLY: kernel supports expiry-only entry invalidation
- * FUSE_DIRECT_IO_RELAX: relax restrictions in FOPEN_DIRECT_IO mode, for now
- *                       allow shared mmap
+ * FUSE_DIRECT_IO_ALLOW_MMAP: allow shared mmap in FOPEN_DIRECT_IO mode.
  */
 #define FUSE_ASYNC_READ		(1 << 0)
 #define FUSE_POSIX_LOCKS	(1 << 1)
@@ -449,7 +448,10 @@ struct fuse_file_lock {
 #define FUSE_HAS_INODE_DAX	(1ULL << 33)
 #define FUSE_CREATE_SUPP_GROUP	(1ULL << 34)
 #define FUSE_HAS_EXPIRE_ONLY	(1ULL << 35)
-#define FUSE_DIRECT_IO_RELAX	(1ULL << 36)
+#define FUSE_DIRECT_IO_ALLOW_MMAP (1ULL << 36)
+
+/* Obsolete alias for FUSE_DIRECT_IO_ALLOW_MMAP */
+#define FUSE_DIRECT_IO_RELAX	FUSE_DIRECT_IO_ALLOW_MMAP
 
 /**
  * CUSE INIT request/reply flags
diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h
index 29ff80da2775..8181ef23a7a2 100644
--- a/include/uapi/linux/if_link.h
+++ b/include/uapi/linux/if_link.h
@@ -856,6 +856,7 @@ enum {
 	IFLA_VXLAN_DF,
 	IFLA_VXLAN_VNIFILTER, /* only applicable with COLLECT_METADATA mode */
 	IFLA_VXLAN_LOCALBYPASS,
+	IFLA_VXLAN_LABEL_POLICY, /* IPv6 flow label policy; ifla_vxlan_label_policy */
 	__IFLA_VXLAN_MAX
 };
 #define IFLA_VXLAN_MAX	(__IFLA_VXLAN_MAX - 1)
@@ -873,6 +874,13 @@ enum ifla_vxlan_df {
 	VXLAN_DF_MAX = __VXLAN_DF_END - 1,
 };
 
+enum ifla_vxlan_label_policy {
+	VXLAN_LABEL_FIXED = 0,
+	VXLAN_LABEL_INHERIT = 1,
+	__VXLAN_LABEL_END,
+	VXLAN_LABEL_MAX = __VXLAN_LABEL_END - 1,
+};
+
 /* GENEVE section */
 enum {
 	IFLA_GENEVE_UNSPEC,
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index 211b86de35ac..e9cb2df67a1d 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -95,6 +95,19 @@ struct kvm_userspace_memory_region {
 	__u64 userspace_addr; /* start of the userspace allocated memory */
 };
 
+/* for KVM_SET_USER_MEMORY_REGION2 */
+struct kvm_userspace_memory_region2 {
+	__u32 slot;
+	__u32 flags;
+	__u64 guest_phys_addr;
+	__u64 memory_size;
+	__u64 userspace_addr;
+	__u64 guest_memfd_offset;
+	__u32 guest_memfd;
+	__u32 pad1;
+	__u64 pad2[14];
+};
+
 /*
  * The bit 0 ~ bit 15 of kvm_userspace_memory_region::flags are visible for
  * userspace, other bits are reserved for kvm internal use which are defined
@@ -102,6 +115,7 @@ struct kvm_userspace_memory_region {
  */
 #define KVM_MEM_LOG_DIRTY_PAGES	(1UL << 0)
 #define KVM_MEM_READONLY	(1UL << 1)
+#define KVM_MEM_GUEST_MEMFD	(1UL << 2)
 
 /* for KVM_IRQ_LINE */
 struct kvm_irq_level {
@@ -265,6 +279,7 @@ struct kvm_xen_exit {
 #define KVM_EXIT_RISCV_CSR        36
 #define KVM_EXIT_NOTIFY           37
 #define KVM_EXIT_LOONGARCH_IOCSR  38
+#define KVM_EXIT_MEMORY_FAULT     39
 
 /* For KVM_EXIT_INTERNAL_ERROR */
 /* Emulate instruction failed. */
@@ -518,6 +533,13 @@ struct kvm_run {
 #define KVM_NOTIFY_CONTEXT_INVALID	(1 << 0)
 			__u32 flags;
 		} notify;
+		/* KVM_EXIT_MEMORY_FAULT */
+		struct {
+#define KVM_MEMORY_EXIT_FLAG_PRIVATE	(1ULL << 3)
+			__u64 flags;
+			__u64 gpa;
+			__u64 size;
+		} memory_fault;
 		/* Fix the size of the union. */
 		char padding[256];
 	};
@@ -1201,6 +1223,11 @@ struct kvm_ppc_resize_hpt {
 #define KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE 228
 #define KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES 229
 #define KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES 230
+#define KVM_CAP_USER_MEMORY2 231
+#define KVM_CAP_MEMORY_FAULT_INFO 232
+#define KVM_CAP_MEMORY_ATTRIBUTES 233
+#define KVM_CAP_GUEST_MEMFD 234
+#define KVM_CAP_VM_TYPES 235
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -1483,6 +1510,8 @@ struct kvm_vfio_spapr_tce {
 					struct kvm_userspace_memory_region)
 #define KVM_SET_TSS_ADDR          _IO(KVMIO,   0x47)
 #define KVM_SET_IDENTITY_MAP_ADDR _IOW(KVMIO,  0x48, __u64)
+#define KVM_SET_USER_MEMORY_REGION2 _IOW(KVMIO, 0x49, \
+					 struct kvm_userspace_memory_region2)
 
 /* enable ucontrol for s390 */
 struct kvm_s390_ucas_mapping {
@@ -2267,4 +2296,24 @@ struct kvm_s390_zpci_op {
 /* flags for kvm_s390_zpci_op->u.reg_aen.flags */
 #define KVM_S390_ZPCIOP_REGAEN_HOST    (1 << 0)
 
+/* Available with KVM_CAP_MEMORY_ATTRIBUTES */
+#define KVM_SET_MEMORY_ATTRIBUTES              _IOW(KVMIO,  0xd2, struct kvm_memory_attributes)
+
+struct kvm_memory_attributes {
+	__u64 address;
+	__u64 size;
+	__u64 attributes;
+	__u64 flags;
+};
+
+#define KVM_MEMORY_ATTRIBUTE_PRIVATE           (1ULL << 3)
+
+#define KVM_CREATE_GUEST_MEMFD	_IOWR(KVMIO,  0xd4, struct kvm_create_guest_memfd)
+
+struct kvm_create_guest_memfd {
+	__u64 size;
+	__u64 flags;
+	__u64 reserved[6];
+};
+
 #endif /* __LINUX_KVM_H */
diff --git a/include/uapi/linux/landlock.h b/include/uapi/linux/landlock.h
index 25c8d7677539..578f268b084b 100644
--- a/include/uapi/linux/landlock.h
+++ b/include/uapi/linux/landlock.h
@@ -128,7 +128,7 @@ struct landlock_net_port_attr {
  * files and directories.  Files or directories opened before the sandboxing
  * are not subject to these restrictions.
  *
- * A file can only receive these access rights:
+ * The following access rights apply only to files:
  *
  * - %LANDLOCK_ACCESS_FS_EXECUTE: Execute a file.
  * - %LANDLOCK_ACCESS_FS_WRITE_FILE: Open a file with write access. Note that
@@ -138,12 +138,13 @@ struct landlock_net_port_attr {
  * - %LANDLOCK_ACCESS_FS_READ_FILE: Open a file with read access.
  * - %LANDLOCK_ACCESS_FS_TRUNCATE: Truncate a file with :manpage:`truncate(2)`,
  *   :manpage:`ftruncate(2)`, :manpage:`creat(2)`, or :manpage:`open(2)` with
- *   ``O_TRUNC``. Whether an opened file can be truncated with
- *   :manpage:`ftruncate(2)` is determined during :manpage:`open(2)`, in the
- *   same way as read and write permissions are checked during
- *   :manpage:`open(2)` using %LANDLOCK_ACCESS_FS_READ_FILE and
- *   %LANDLOCK_ACCESS_FS_WRITE_FILE. This access right is available since the
- *   third version of the Landlock ABI.
+ *   ``O_TRUNC``.  This access right is available since the third version of the
+ *   Landlock ABI.
+ *
+ * Whether an opened file can be truncated with :manpage:`ftruncate(2)` or used
+ * with `ioctl(2)` is determined during :manpage:`open(2)`, in the same way as
+ * read and write permissions are checked during :manpage:`open(2)` using
+ * %LANDLOCK_ACCESS_FS_READ_FILE and %LANDLOCK_ACCESS_FS_WRITE_FILE.
  *
  * A directory can receive access rights related to files or directories.  The
  * following access right is applied to the directory itself, and the
@@ -198,13 +199,53 @@ struct landlock_net_port_attr {
  *   If multiple requirements are not met, the ``EACCES`` error code takes
  *   precedence over ``EXDEV``.
  *
+ * The following access right applies both to files and directories:
+ *
+ * - %LANDLOCK_ACCESS_FS_IOCTL: Invoke :manpage:`ioctl(2)` commands on an opened
+ *   file or directory.
+ *
+ *   This access right applies to all :manpage:`ioctl(2)` commands, except of
+ *   ``FIOCLEX``, ``FIONCLEX``, ``FIONBIO`` and ``FIOASYNC``.  These commands
+ *   continue to be invokable independent of the %LANDLOCK_ACCESS_FS_IOCTL
+ *   access right.
+ *
+ *   When certain other access rights are handled in the ruleset, in addition to
+ *   %LANDLOCK_ACCESS_FS_IOCTL, granting these access rights will unlock access
+ *   to additional groups of IOCTL commands, on the affected files:
+ *
+ *   * %LANDLOCK_ACCESS_FS_READ_FILE unlocks access to ``FIOQSIZE``,
+ *     ``FS_IOC_FIEMAP``, ``FIBMAP``, ``FIGETBSZ``, ``FIONREAD``,
+ *     ``FIDEDUPRANGE``.
+ *
+ *   * %LANDLOCK_ACCESS_FS_WRITE_FILE unlocks access to ``FIOQSIZE``,
+ *     ``FS_IOC_FIEMAP``, ``FIBMAP``, ``FIGETBSZ``, ``FICLONE``,
+ *     ``FICLONERANGE``, ``FS_IOC_RESVSP``, ``FS_IOC_RESVSP64``,
+ *     ``FS_IOC_UNRESVSP``, ``FS_IOC_UNRESVSP64``, ``FS_IOC_ZERO_RANGE``.
+ *
+ *   * %LANDLOCK_ACCESS_FS_READ_DIR unlocks access to ``FIOQSIZE``,
+ *     ``FS_IOC_FIEMAP``, ``FIBMAP``, ``FIGETBSZ``.
+ *
+ *   When these access rights are handled in the ruleset, the availability of
+ *   the affected IOCTL commands is not governed by %LANDLOCK_ACCESS_FS_IOCTL
+ *   any more, but by the respective access right.
+ *
+ *   All other IOCTL commands are not handled specially, and are governed by
+ *   %LANDLOCK_ACCESS_FS_IOCTL.  This includes %FS_IOC_GETFLAGS and
+ *   %FS_IOC_SETFLAGS for manipulating inode flags (:manpage:`ioctl_iflags(2)`),
+ *   %FS_IOC_FSFETXATTR and %FS_IOC_FSSETXATTR for manipulating extended
+ *   attributes, as well as %FIFREEZE and %FITHAW for freezing and thawing file
+ *   systems.
+ *
+ *   This access right is available since the fifth version of the Landlock
+ *   ABI.
+ *
  * .. warning::
  *
  *   It is currently not possible to restrict some file-related actions
  *   accessible through these syscall families: :manpage:`chdir(2)`,
  *   :manpage:`stat(2)`, :manpage:`flock(2)`, :manpage:`chmod(2)`,
  *   :manpage:`chown(2)`, :manpage:`setxattr(2)`, :manpage:`utime(2)`,
- *   :manpage:`ioctl(2)`, :manpage:`fcntl(2)`, :manpage:`access(2)`.
+ *   :manpage:`fcntl(2)`, :manpage:`access(2)`.
  *   Future Landlock evolutions will enable to restrict them.
  */
 /* clang-format off */
@@ -223,6 +264,7 @@ struct landlock_net_port_attr {
 #define LANDLOCK_ACCESS_FS_MAKE_SYM			(1ULL << 12)
 #define LANDLOCK_ACCESS_FS_REFER			(1ULL << 13)
 #define LANDLOCK_ACCESS_FS_TRUNCATE			(1ULL << 14)
+#define LANDLOCK_ACCESS_FS_IOCTL			(1ULL << 15)
 /* clang-format on */
 
 /**
diff --git a/include/uapi/linux/lsm.h b/include/uapi/linux/lsm.h
new file mode 100644
index 000000000000..f0386880a78e
--- /dev/null
+++ b/include/uapi/linux/lsm.h
@@ -0,0 +1,89 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * Linux Security Modules (LSM) - User space API
+ *
+ * Copyright (C) 2022 Casey Schaufler <casey@schaufler-ca.com>
+ * Copyright (C) 2022 Intel Corporation
+ */
+
+#ifndef _UAPI_LINUX_LSM_H
+#define _UAPI_LINUX_LSM_H
+
+#include <linux/types.h>
+#include <linux/unistd.h>
+
+/**
+ * struct lsm_ctx - LSM context information
+ * @id: the LSM id number, see LSM_ID_XXX
+ * @flags: LSM specific flags
+ * @len: length of the lsm_ctx struct, @ctx and any other data or padding
+ * @ctx_len: the size of @ctx
+ * @ctx: the LSM context value
+ *
+ * The @len field MUST be equal to the size of the lsm_ctx struct
+ * plus any additional padding and/or data placed after @ctx.
+ *
+ * In all cases @ctx_len MUST be equal to the length of @ctx.
+ * If @ctx is a string value it should be nul terminated with
+ * @ctx_len equal to `strlen(@ctx) + 1`.  Binary values are
+ * supported.
+ *
+ * The @flags and @ctx fields SHOULD only be interpreted by the
+ * LSM specified by @id; they MUST be set to zero/0 when not used.
+ */
+struct lsm_ctx {
+	__u64 id;
+	__u64 flags;
+	__u64 len;
+	__u64 ctx_len;
+	__u8 ctx[];
+};
+
+/*
+ * ID tokens to identify Linux Security Modules (LSMs)
+ *
+ * These token values are used to uniquely identify specific LSMs
+ * in the kernel as well as in the kernel's LSM userspace API.
+ *
+ * A value of zero/0 is considered undefined and should not be used
+ * outside the kernel. Values 1-99 are reserved for potential
+ * future use.
+ */
+#define LSM_ID_UNDEF		0
+#define LSM_ID_CAPABILITY	100
+#define LSM_ID_SELINUX		101
+#define LSM_ID_SMACK		102
+#define LSM_ID_TOMOYO		103
+#define LSM_ID_APPARMOR		104
+#define LSM_ID_YAMA		105
+#define LSM_ID_LOADPIN		106
+#define LSM_ID_SAFESETID	107
+#define LSM_ID_LOCKDOWN		108
+#define LSM_ID_BPF		109
+#define LSM_ID_LANDLOCK		110
+
+/*
+ * LSM_ATTR_XXX definitions identify different LSM attributes
+ * which are used in the kernel's LSM userspace API. Support
+ * for these attributes vary across the different LSMs. None
+ * are required.
+ *
+ * A value of zero/0 is considered undefined and should not be used
+ * outside the kernel. Values 1-99 are reserved for potential
+ * future use.
+ */
+#define LSM_ATTR_UNDEF		0
+#define LSM_ATTR_CURRENT	100
+#define LSM_ATTR_EXEC		101
+#define LSM_ATTR_FSCREATE	102
+#define LSM_ATTR_KEYCREATE	103
+#define LSM_ATTR_PREV		104
+#define LSM_ATTR_SOCKCREATE	105
+
+/*
+ * LSM_FLAG_XXX definitions identify special handling instructions
+ * for the API.
+ */
+#define LSM_FLAG_SINGLE	0x0001
+
+#endif /* _UAPI_LINUX_LSM_H */
diff --git a/include/uapi/linux/mount.h b/include/uapi/linux/mount.h
index bb242fdcfe6b..7a5bd0b24a62 100644
--- a/include/uapi/linux/mount.h
+++ b/include/uapi/linux/mount.h
@@ -138,4 +138,66 @@ struct mount_attr {
 /* List of all mount_attr versions. */
 #define MOUNT_ATTR_SIZE_VER0	32 /* sizeof first published struct */
 
+
+/*
+ * Structure for getting mount/superblock/filesystem info with statmount(2).
+ *
+ * The interface is similar to statx(2): individual fields or groups can be
+ * selected with the @mask argument of statmount().  Kernel will set the @mask
+ * field according to the supported fields.
+ *
+ * If string fields are selected, then the caller needs to pass a buffer that
+ * has space after the fixed part of the structure.  Nul terminated strings are
+ * copied there and offsets relative to @str are stored in the relevant fields.
+ * If the buffer is too small, then EOVERFLOW is returned.  The actually used
+ * size is returned in @size.
+ */
+struct statmount {
+	__u32 size;		/* Total size, including strings */
+	__u32 __spare1;
+	__u64 mask;		/* What results were written */
+	__u32 sb_dev_major;	/* Device ID */
+	__u32 sb_dev_minor;
+	__u64 sb_magic;		/* ..._SUPER_MAGIC */
+	__u32 sb_flags;		/* SB_{RDONLY,SYNCHRONOUS,DIRSYNC,LAZYTIME} */
+	__u32 fs_type;		/* [str] Filesystem type */
+	__u64 mnt_id;		/* Unique ID of mount */
+	__u64 mnt_parent_id;	/* Unique ID of parent (for root == mnt_id) */
+	__u32 mnt_id_old;	/* Reused IDs used in proc/.../mountinfo */
+	__u32 mnt_parent_id_old;
+	__u64 mnt_attr;		/* MOUNT_ATTR_... */
+	__u64 mnt_propagation;	/* MS_{SHARED,SLAVE,PRIVATE,UNBINDABLE} */
+	__u64 mnt_peer_group;	/* ID of shared peer group */
+	__u64 mnt_master;	/* Mount receives propagation from this ID */
+	__u64 propagate_from;	/* Propagation from in current namespace */
+	__u32 mnt_root;		/* [str] Root of mount relative to root of fs */
+	__u32 mnt_point;	/* [str] Mountpoint relative to current root */
+	__u64 __spare2[50];
+	char str[];		/* Variable size part containing strings */
+};
+
+struct mnt_id_req {
+	__u64 mnt_id;
+	__u64 request_mask;
+};
+
+/*
+ * @mask bits for statmount(2)
+ */
+#define STATMOUNT_SB_BASIC		0x00000001U     /* Want/got sb_... */
+#define STATMOUNT_MNT_BASIC		0x00000002U	/* Want/got mnt_... */
+#define STATMOUNT_PROPAGATE_FROM	0x00000004U	/* Want/got propagate_from */
+#define STATMOUNT_MNT_ROOT		0x00000008U	/* Want/got mnt_root  */
+#define STATMOUNT_MNT_POINT		0x00000010U	/* Want/got mnt_point */
+#define STATMOUNT_FS_TYPE		0x00000020U	/* Want/got fs_type */
+
+/* listmount(2) flags */
+#define LISTMOUNT_UNREACHABLE	0x01U	/* List unreachable mounts too */
+#define LISTMOUNT_RECURSIVE	0x02U	/* List a mount tree */
+
+/*
+ * Special @mnt_id values that can be passed to listmount
+ */
+#define LSMT_ROOT		0xffffffffffffffff	/* root mount */
+
 #endif /* _UAPI_LINUX_MOUNT_H */
diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h
index 39c6a250dd1b..3a64499b0f5d 100644
--- a/include/uapi/linux/perf_event.h
+++ b/include/uapi/linux/perf_event.h
@@ -204,6 +204,8 @@ enum perf_branch_sample_type_shift {
 
 	PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT	= 18, /* save privilege mode */
 
+	PERF_SAMPLE_BRANCH_COUNTERS_SHIFT	= 19, /* save occurrences of events on a branch */
+
 	PERF_SAMPLE_BRANCH_MAX_SHIFT		/* non-ABI */
 };
 
@@ -235,6 +237,8 @@ enum perf_branch_sample_type {
 
 	PERF_SAMPLE_BRANCH_PRIV_SAVE	= 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT,
 
+	PERF_SAMPLE_BRANCH_COUNTERS	= 1U << PERF_SAMPLE_BRANCH_COUNTERS_SHIFT,
+
 	PERF_SAMPLE_BRANCH_MAX		= 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT,
 };
 
@@ -982,6 +986,12 @@ enum perf_event_type {
 	 *	{ u64                   nr;
 	 *	  { u64	hw_idx; } && PERF_SAMPLE_BRANCH_HW_INDEX
 	 *        { u64 from, to, flags } lbr[nr];
+	 *        #
+	 *        # The format of the counters is decided by the
+	 *        # "branch_counter_nr" and "branch_counter_width",
+	 *        # which are defined in the ABI.
+	 *        #
+	 *        { u64 counters; } cntr[nr] && PERF_SAMPLE_BRANCH_COUNTERS
 	 *      } && PERF_SAMPLE_BRANCH_STACK
 	 *
 	 * 	{ u64			abi; # enum perf_sample_regs_abi
@@ -1427,6 +1437,9 @@ struct perf_branch_entry {
 		reserved:31;
 };
 
+/* Size of used info bits in struct perf_branch_entry */
+#define PERF_BRANCH_ENTRY_INFO_BITS_MAX		33
+
 union perf_sample_weight {
 	__u64		full;
 #if defined(__LITTLE_ENDIAN_BITFIELD)
diff --git a/include/uapi/linux/stat.h b/include/uapi/linux/stat.h
index 7cab2c65d3d7..2f2ee82d5517 100644
--- a/include/uapi/linux/stat.h
+++ b/include/uapi/linux/stat.h
@@ -154,6 +154,7 @@ struct statx {
 #define STATX_BTIME		0x00000800U	/* Want/got stx_btime */
 #define STATX_MNT_ID		0x00001000U	/* Got stx_mnt_id */
 #define STATX_DIOALIGN		0x00002000U	/* Want/got direct I/O alignment info */
+#define STATX_MNT_ID_UNIQUE	0x00004000U	/* Want/got extended stx_mount_id */
 
 #define STATX__RESERVED		0x80000000U	/* Reserved for future struct statx expansion */
 
diff --git a/include/uapi/linux/v4l2-subdev.h b/include/uapi/linux/v4l2-subdev.h
index 4a195b68f28f..b383c2fe0cf3 100644
--- a/include/uapi/linux/v4l2-subdev.h
+++ b/include/uapi/linux/v4l2-subdev.h
@@ -239,7 +239,7 @@ struct v4l2_subdev_routing {
  * set (which is the default), the 'stream' fields will be forced to 0 by the
  * kernel.
  */
- #define V4L2_SUBDEV_CLIENT_CAP_STREAMS		(1U << 0)
+ #define V4L2_SUBDEV_CLIENT_CAP_STREAMS		(1ULL << 0)
 
 /**
  * struct v4l2_subdev_client_capability - Capabilities of the client accessing
diff --git a/include/uapi/rdma/bnxt_re-abi.h b/include/uapi/rdma/bnxt_re-abi.h
index 6e7c67a0cca3..a1b896d6d940 100644
--- a/include/uapi/rdma/bnxt_re-abi.h
+++ b/include/uapi/rdma/bnxt_re-abi.h
@@ -54,6 +54,7 @@ enum {
 	BNXT_RE_UCNTX_CMASK_HAVE_MODE = 0x02ULL,
 	BNXT_RE_UCNTX_CMASK_WC_DPI_ENABLED = 0x04ULL,
 	BNXT_RE_UCNTX_CMASK_DBR_PACING_ENABLED = 0x08ULL,
+	BNXT_RE_UCNTX_CMASK_POW2_DISABLED = 0x10ULL,
 };
 
 enum bnxt_re_wqe_mode {
@@ -62,6 +63,14 @@ enum bnxt_re_wqe_mode {
 	BNXT_QPLIB_WQE_MODE_INVALID	= 0x02,
 };
 
+enum {
+	BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT = 0x01,
+};
+
+struct bnxt_re_uctx_req {
+	__aligned_u64 comp_mask;
+};
+
 struct bnxt_re_uctx_resp {
 	__u32 dev_id;
 	__u32 max_qp;
diff --git a/include/uapi/sound/sof/tokens.h b/include/uapi/sound/sof/tokens.h
index 453cab2a1209..0fb39780f9bd 100644
--- a/include/uapi/sound/sof/tokens.h
+++ b/include/uapi/sound/sof/tokens.h
@@ -213,4 +213,8 @@
 #define SOF_TKN_AMD_ACPI2S_CH			1701
 #define SOF_TKN_AMD_ACPI2S_TDM_MODE		1702
 
+/* MICFIL PDM */
+#define SOF_TKN_IMX_MICFIL_RATE			2000
+#define SOF_TKN_IMX_MICFIL_CH			2001
+
 #endif
diff --git a/include/ufs/unipro.h b/include/ufs/unipro.h
index 256eb3a43f54..360e1245fb40 100644
--- a/include/ufs/unipro.h
+++ b/include/ufs/unipro.h
@@ -193,7 +193,7 @@
 #define DME_LocalAFC0ReqTimeOutVal		0xD043
 
 /* PA power modes */
-enum {
+enum ufs_pa_pwr_mode {
 	FAST_MODE	= 1,
 	SLOW_MODE	= 2,
 	FASTAUTO_MODE	= 4,
@@ -205,7 +205,7 @@ enum {
 #define PWRMODE_RX_OFFSET	4
 
 /* PA TX/RX Frequency Series */
-enum {
+enum ufs_hs_gear_rate {
 	PA_HS_MODE_A	= 1,
 	PA_HS_MODE_B	= 2,
 };
diff --git a/init/init_task.c b/init/init_task.c
index 5727d42149c3..6f6485d554df 100644
--- a/init/init_task.c
+++ b/init/init_task.c
@@ -51,8 +51,7 @@ static struct sighand_struct init_sighand = {
 };
 
 #ifdef CONFIG_SHADOW_CALL_STACK
-unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)]
-		__init_task_data = {
+unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)] = {
 	[(SCS_SIZE / sizeof(long)) - 1] = SCS_END_MAGIC
 };
 #endif
@@ -61,12 +60,7 @@ unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)]
  * Set up the first task table, touch at your own risk!. Base=0,
  * limit=0x1fffff (=2MB)
  */
-struct task_struct init_task
-#ifdef CONFIG_ARCH_TASK_STRUCT_ON_STACK
-	__init_task_data
-#endif
-	__aligned(L1_CACHE_BYTES)
-= {
+struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
 #ifdef CONFIG_THREAD_INFO_IN_TASK
 	.thread_info	= INIT_THREAD_INFO(init_task),
 	.stack_refcount	= REFCOUNT_INIT(1),
diff --git a/io_uring/fs.c b/io_uring/fs.c
index 08e3b175469c..eccea851dd5a 100644
--- a/io_uring/fs.c
+++ b/io_uring/fs.c
@@ -254,7 +254,7 @@ int io_linkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 	newf = u64_to_user_ptr(READ_ONCE(sqe->addr2));
 	lnk->flags = READ_ONCE(sqe->hardlink_flags);
 
-	lnk->oldpath = getname(oldf);
+	lnk->oldpath = getname_uflags(oldf, lnk->flags);
 	if (IS_ERR(lnk->oldpath))
 		return PTR_ERR(lnk->oldpath);
 
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index ed254076c723..db3f545ddcac 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -3877,7 +3877,8 @@ static struct file *io_uring_get_file(struct io_ring_ctx *ctx)
 		return ERR_PTR(ret);
 #endif
 
-	file = anon_inode_getfile_secure("[io_uring]", &io_uring_fops, ctx,
+	/* Create a new inode so that the LSM can block the creation.  */
+	file = anon_inode_create_getfile("[io_uring]", &io_uring_fops, ctx,
 					 O_RDWR | O_CLOEXEC, NULL);
 #if defined(CONFIG_UNIX)
 	if (IS_ERR(file)) {
diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 7034be555334..f521c5965a93 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -1258,7 +1258,7 @@ int io_import_fixed(int ddir, struct iov_iter *iter,
 		 */
 		const struct bio_vec *bvec = imu->bvec;
 
-		if (offset <= bvec->bv_len) {
+		if (offset < bvec->bv_len) {
 			/*
 			 * Note, huge pages buffers consists of one large
 			 * bvec entry and should always go this way. The other
diff --git a/ipc/ipc_sysctl.c b/ipc/ipc_sysctl.c
index 8c62e443f78b..01c4a50d22b2 100644
--- a/ipc/ipc_sysctl.c
+++ b/ipc/ipc_sysctl.c
@@ -14,6 +14,7 @@
 #include <linux/ipc_namespace.h>
 #include <linux/msg.h>
 #include <linux/slab.h>
+#include <linux/cred.h>
 #include "util.h"
 
 static int proc_ipc_dointvec_minmax_orphans(struct ctl_table *table, int write,
@@ -190,25 +191,57 @@ static int set_is_seen(struct ctl_table_set *set)
 	return &current->nsproxy->ipc_ns->ipc_set == set;
 }
 
+static void ipc_set_ownership(struct ctl_table_header *head,
+			      struct ctl_table *table,
+			      kuid_t *uid, kgid_t *gid)
+{
+	struct ipc_namespace *ns =
+		container_of(head->set, struct ipc_namespace, ipc_set);
+
+	kuid_t ns_root_uid = make_kuid(ns->user_ns, 0);
+	kgid_t ns_root_gid = make_kgid(ns->user_ns, 0);
+
+	*uid = uid_valid(ns_root_uid) ? ns_root_uid : GLOBAL_ROOT_UID;
+	*gid = gid_valid(ns_root_gid) ? ns_root_gid : GLOBAL_ROOT_GID;
+}
+
 static int ipc_permissions(struct ctl_table_header *head, struct ctl_table *table)
 {
 	int mode = table->mode;
 
 #ifdef CONFIG_CHECKPOINT_RESTORE
-	struct ipc_namespace *ns = current->nsproxy->ipc_ns;
+	struct ipc_namespace *ns =
+		container_of(head->set, struct ipc_namespace, ipc_set);
 
 	if (((table->data == &ns->ids[IPC_SEM_IDS].next_id) ||
 	     (table->data == &ns->ids[IPC_MSG_IDS].next_id) ||
 	     (table->data == &ns->ids[IPC_SHM_IDS].next_id)) &&
 	    checkpoint_restore_ns_capable(ns->user_ns))
 		mode = 0666;
+	else
 #endif
-	return mode;
+	{
+		kuid_t ns_root_uid;
+		kgid_t ns_root_gid;
+
+		ipc_set_ownership(head, table, &ns_root_uid, &ns_root_gid);
+
+		if (uid_eq(current_euid(), ns_root_uid))
+			mode >>= 6;
+
+		else if (in_egroup_p(ns_root_gid))
+			mode >>= 3;
+	}
+
+	mode &= 7;
+
+	return (mode << 6) | (mode << 3) | mode;
 }
 
 static struct ctl_table_root set_root = {
 	.lookup = set_lookup,
 	.permissions = ipc_permissions,
+	.set_ownership = ipc_set_ownership,
 };
 
 bool setup_ipc_sysctls(struct ipc_namespace *ns)
diff --git a/ipc/mq_sysctl.c b/ipc/mq_sysctl.c
index ebb5ed81c151..21fba3a6edaf 100644
--- a/ipc/mq_sysctl.c
+++ b/ipc/mq_sysctl.c
@@ -12,6 +12,7 @@
 #include <linux/stat.h>
 #include <linux/capability.h>
 #include <linux/slab.h>
+#include <linux/cred.h>
 
 static int msg_max_limit_min = MIN_MSGMAX;
 static int msg_max_limit_max = HARD_MSGMAX;
@@ -76,8 +77,43 @@ static int set_is_seen(struct ctl_table_set *set)
 	return &current->nsproxy->ipc_ns->mq_set == set;
 }
 
+static void mq_set_ownership(struct ctl_table_header *head,
+			     struct ctl_table *table,
+			     kuid_t *uid, kgid_t *gid)
+{
+	struct ipc_namespace *ns =
+		container_of(head->set, struct ipc_namespace, mq_set);
+
+	kuid_t ns_root_uid = make_kuid(ns->user_ns, 0);
+	kgid_t ns_root_gid = make_kgid(ns->user_ns, 0);
+
+	*uid = uid_valid(ns_root_uid) ? ns_root_uid : GLOBAL_ROOT_UID;
+	*gid = gid_valid(ns_root_gid) ? ns_root_gid : GLOBAL_ROOT_GID;
+}
+
+static int mq_permissions(struct ctl_table_header *head, struct ctl_table *table)
+{
+	int mode = table->mode;
+	kuid_t ns_root_uid;
+	kgid_t ns_root_gid;
+
+	mq_set_ownership(head, table, &ns_root_uid, &ns_root_gid);
+
+	if (uid_eq(current_euid(), ns_root_uid))
+		mode >>= 6;
+
+	else if (in_egroup_p(ns_root_gid))
+		mode >>= 3;
+
+	mode &= 7;
+
+	return (mode << 6) | (mode << 3) | mode;
+}
+
 static struct ctl_table_root set_root = {
 	.lookup = set_lookup,
+	.permissions = mq_permissions,
+	.set_ownership = mq_set_ownership,
 };
 
 bool setup_mq_sysctls(struct ipc_namespace *ns)
diff --git a/kernel/Kconfig.kexec b/kernel/Kconfig.kexec
index 7aff28ded2f4..51f719af10e7 100644
--- a/kernel/Kconfig.kexec
+++ b/kernel/Kconfig.kexec
@@ -36,6 +36,8 @@ config KEXEC
 config KEXEC_FILE
 	bool "Enable kexec file based system call"
 	depends on ARCH_SUPPORTS_KEXEC_FILE
+	select CRYPTO
+	select CRYPTO_SHA256
 	select KEXEC_CORE
 	help
 	  This is new version of kexec system call. This system call is
diff --git a/kernel/audit.c b/kernel/audit.c
index 16205dd29843..9c8e5f732c4c 100644
--- a/kernel/audit.c
+++ b/kernel/audit.c
@@ -487,15 +487,19 @@ static void auditd_conn_free(struct rcu_head *rcu)
  * @pid: auditd PID
  * @portid: auditd netlink portid
  * @net: auditd network namespace pointer
+ * @skb: the netlink command from the audit daemon
+ * @ack: netlink ack flag, cleared if ack'd here
  *
  * Description:
  * This function will obtain and drop network namespace references as
  * necessary.  Returns zero on success, negative values on failure.
  */
-static int auditd_set(struct pid *pid, u32 portid, struct net *net)
+static int auditd_set(struct pid *pid, u32 portid, struct net *net,
+		      struct sk_buff *skb, bool *ack)
 {
 	unsigned long flags;
 	struct auditd_connection *ac_old, *ac_new;
+	struct nlmsghdr *nlh;
 
 	if (!pid || !net)
 		return -EINVAL;
@@ -507,6 +511,13 @@ static int auditd_set(struct pid *pid, u32 portid, struct net *net)
 	ac_new->portid = portid;
 	ac_new->net = get_net(net);
 
+	/* send the ack now to avoid a race with the queue backlog */
+	if (*ack) {
+		nlh = nlmsg_hdr(skb);
+		netlink_ack(skb, nlh, 0, NULL);
+		*ack = false;
+	}
+
 	spin_lock_irqsave(&auditd_conn_lock, flags);
 	ac_old = rcu_dereference_protected(auditd_conn,
 					   lockdep_is_held(&auditd_conn_lock));
@@ -1200,7 +1211,8 @@ static int audit_replace(struct pid *pid)
 	return auditd_send_unicast_skb(skb);
 }
 
-static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh)
+static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh,
+			     bool *ack)
 {
 	u32			seq;
 	void			*data;
@@ -1293,7 +1305,8 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh)
 				/* register a new auditd connection */
 				err = auditd_set(req_pid,
 						 NETLINK_CB(skb).portid,
-						 sock_net(NETLINK_CB(skb).sk));
+						 sock_net(NETLINK_CB(skb).sk),
+						 skb, ack);
 				if (audit_enabled != AUDIT_OFF)
 					audit_log_config_change("audit_pid",
 								new_pid,
@@ -1538,9 +1551,10 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh)
  * Parse the provided skb and deal with any messages that may be present,
  * malformed skbs are discarded.
  */
-static void audit_receive(struct sk_buff  *skb)
+static void audit_receive(struct sk_buff *skb)
 {
 	struct nlmsghdr *nlh;
+	bool ack;
 	/*
 	 * len MUST be signed for nlmsg_next to be able to dec it below 0
 	 * if the nlmsg_len was not aligned
@@ -1553,9 +1567,12 @@ static void audit_receive(struct sk_buff  *skb)
 
 	audit_ctl_lock();
 	while (nlmsg_ok(nlh, len)) {
-		err = audit_receive_msg(skb, nlh);
-		/* if err or if this message says it wants a response */
-		if (err || (nlh->nlmsg_flags & NLM_F_ACK))
+		ack = nlh->nlmsg_flags & NLM_F_ACK;
+		err = audit_receive_msg(skb, nlh, &ack);
+
+		/* send an ack if the user asked for one and audit_receive_msg
+		 * didn't already do it, or if there was an error. */
+		if (ack || err)
 			netlink_ack(skb, nlh, err, NULL);
 
 		nlh = nlmsg_next(nlh, &len);
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 15d71d2986d3..63cf4128fc05 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -3840,9 +3840,6 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type
 	return ERR_PTR(ret);
 }
 
-#define GRAPH_ROOT_MASK (BPF_LIST_HEAD | BPF_RB_ROOT)
-#define GRAPH_NODE_MASK (BPF_LIST_NODE | BPF_RB_NODE)
-
 int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec)
 {
 	int i;
@@ -3855,13 +3852,13 @@ int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec)
 	 * Hence we only need to ensure that bpf_{list_head,rb_root} ownership
 	 * does not form cycles.
 	 */
-	if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & GRAPH_ROOT_MASK))
+	if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & BPF_GRAPH_ROOT))
 		return 0;
 	for (i = 0; i < rec->cnt; i++) {
 		struct btf_struct_meta *meta;
 		u32 btf_id;
 
-		if (!(rec->fields[i].type & GRAPH_ROOT_MASK))
+		if (!(rec->fields[i].type & BPF_GRAPH_ROOT))
 			continue;
 		btf_id = rec->fields[i].graph_root.value_btf_id;
 		meta = btf_find_struct_meta(btf, btf_id);
@@ -3873,7 +3870,7 @@ int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec)
 		 * to check ownership cycle for a type unless it's also a
 		 * node type.
 		 */
-		if (!(rec->field_mask & GRAPH_NODE_MASK))
+		if (!(rec->field_mask & BPF_GRAPH_NODE))
 			continue;
 
 		/* We need to ensure ownership acyclicity among all types. The
@@ -3909,7 +3906,7 @@ int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec)
 		 * - A is both an root and node.
 		 * - B is only an node.
 		 */
-		if (meta->record->field_mask & GRAPH_ROOT_MASK)
+		if (meta->record->field_mask & BPF_GRAPH_ROOT)
 			return -ELOOP;
 	}
 	return 0;
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 56b0c1f678ee..b45a8381f9bd 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -1937,10 +1937,7 @@ void __bpf_obj_drop_impl(void *p, const struct btf_record *rec, bool percpu)
 		ma = &bpf_global_percpu_ma;
 	else
 		ma = &bpf_global_ma;
-	if (rec && rec->refcount_off >= 0)
-		bpf_mem_free_rcu(ma, p);
-	else
-		bpf_mem_free(ma, p);
+	bpf_mem_free_rcu(ma, p);
 }
 
 __bpf_kfunc void bpf_obj_drop_impl(void *p__alloc, void *meta__ign)
@@ -2231,6 +2228,25 @@ __bpf_kfunc long bpf_task_under_cgroup(struct task_struct *task,
 	rcu_read_unlock();
 	return ret;
 }
+
+/**
+ * bpf_task_get_cgroup1 - Acquires the associated cgroup of a task within a
+ * specific cgroup1 hierarchy. The cgroup1 hierarchy is identified by its
+ * hierarchy ID.
+ * @task: The target task
+ * @hierarchy_id: The ID of a cgroup1 hierarchy
+ *
+ * On success, the cgroup is returen. On failure, NULL is returned.
+ */
+__bpf_kfunc struct cgroup *
+bpf_task_get_cgroup1(struct task_struct *task, int hierarchy_id)
+{
+	struct cgroup *cgrp = task_get_cgroup1(task, hierarchy_id);
+
+	if (IS_ERR(cgrp))
+		return NULL;
+	return cgrp;
+}
 #endif /* CONFIG_CGROUPS */
 
 /**
@@ -2520,7 +2536,7 @@ BTF_ID_FLAGS(func, bpf_obj_new_impl, KF_ACQUIRE | KF_RET_NULL)
 BTF_ID_FLAGS(func, bpf_percpu_obj_new_impl, KF_ACQUIRE | KF_RET_NULL)
 BTF_ID_FLAGS(func, bpf_obj_drop_impl, KF_RELEASE)
 BTF_ID_FLAGS(func, bpf_percpu_obj_drop_impl, KF_RELEASE)
-BTF_ID_FLAGS(func, bpf_refcount_acquire_impl, KF_ACQUIRE | KF_RET_NULL)
+BTF_ID_FLAGS(func, bpf_refcount_acquire_impl, KF_ACQUIRE | KF_RET_NULL | KF_RCU)
 BTF_ID_FLAGS(func, bpf_list_push_front_impl)
 BTF_ID_FLAGS(func, bpf_list_push_back_impl)
 BTF_ID_FLAGS(func, bpf_list_pop_front, KF_ACQUIRE | KF_RET_NULL)
@@ -2537,6 +2553,7 @@ BTF_ID_FLAGS(func, bpf_cgroup_release, KF_RELEASE)
 BTF_ID_FLAGS(func, bpf_cgroup_ancestor, KF_ACQUIRE | KF_RCU | KF_RET_NULL)
 BTF_ID_FLAGS(func, bpf_cgroup_from_id, KF_ACQUIRE | KF_RET_NULL)
 BTF_ID_FLAGS(func, bpf_task_under_cgroup, KF_RCU)
+BTF_ID_FLAGS(func, bpf_task_get_cgroup1, KF_ACQUIRE | KF_RCU | KF_RET_NULL)
 #endif
 BTF_ID_FLAGS(func, bpf_task_from_pid, KF_ACQUIRE | KF_RET_NULL)
 BTF_ID_FLAGS(func, bpf_throw)
@@ -2618,3 +2635,22 @@ static int __init kfunc_init(void)
 }
 
 late_initcall(kfunc_init);
+
+/* Get a pointer to dynptr data up to len bytes for read only access. If
+ * the dynptr doesn't have continuous data up to len bytes, return NULL.
+ */
+const void *__bpf_dynptr_data(const struct bpf_dynptr_kern *ptr, u32 len)
+{
+	return bpf_dynptr_slice(ptr, 0, NULL, len);
+}
+
+/* Get a pointer to dynptr data up to len bytes for read write access. If
+ * the dynptr doesn't have continuous data up to len bytes, or the dynptr
+ * is read only, return NULL.
+ */
+void *__bpf_dynptr_data_rw(const struct bpf_dynptr_kern *ptr, u32 len)
+{
+	if (__bpf_dynptr_is_rdonly(ptr))
+		return NULL;
+	return (void *)__bpf_dynptr_data(ptr, len);
+}
diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c
index 850494423530..3505f3e5ae96 100644
--- a/kernel/bpf/log.c
+++ b/kernel/bpf/log.c
@@ -10,6 +10,8 @@
 #include <linux/bpf_verifier.h>
 #include <linux/math64.h>
 
+#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
+
 static bool bpf_verifier_log_attr_valid(const struct bpf_verifier_log *log)
 {
 	/* ubuf and len_total should both be specified (or not) together */
@@ -325,3 +327,481 @@ __printf(2, 3) void bpf_log(struct bpf_verifier_log *log,
 	va_end(args);
 }
 EXPORT_SYMBOL_GPL(bpf_log);
+
+static const struct bpf_line_info *
+find_linfo(const struct bpf_verifier_env *env, u32 insn_off)
+{
+	const struct bpf_line_info *linfo;
+	const struct bpf_prog *prog;
+	u32 i, nr_linfo;
+
+	prog = env->prog;
+	nr_linfo = prog->aux->nr_linfo;
+
+	if (!nr_linfo || insn_off >= prog->len)
+		return NULL;
+
+	linfo = prog->aux->linfo;
+	for (i = 1; i < nr_linfo; i++)
+		if (insn_off < linfo[i].insn_off)
+			break;
+
+	return &linfo[i - 1];
+}
+
+static const char *ltrim(const char *s)
+{
+	while (isspace(*s))
+		s++;
+
+	return s;
+}
+
+__printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env,
+				  u32 insn_off,
+				  const char *prefix_fmt, ...)
+{
+	const struct bpf_line_info *linfo;
+
+	if (!bpf_verifier_log_needed(&env->log))
+		return;
+
+	linfo = find_linfo(env, insn_off);
+	if (!linfo || linfo == env->prev_linfo)
+		return;
+
+	if (prefix_fmt) {
+		va_list args;
+
+		va_start(args, prefix_fmt);
+		bpf_verifier_vlog(&env->log, prefix_fmt, args);
+		va_end(args);
+	}
+
+	verbose(env, "%s\n",
+		ltrim(btf_name_by_offset(env->prog->aux->btf,
+					 linfo->line_off)));
+
+	env->prev_linfo = linfo;
+}
+
+static const char *btf_type_name(const struct btf *btf, u32 id)
+{
+	return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);
+}
+
+/* string representation of 'enum bpf_reg_type'
+ *
+ * Note that reg_type_str() can not appear more than once in a single verbose()
+ * statement.
+ */
+const char *reg_type_str(struct bpf_verifier_env *env, enum bpf_reg_type type)
+{
+	char postfix[16] = {0}, prefix[64] = {0};
+	static const char * const str[] = {
+		[NOT_INIT]		= "?",
+		[SCALAR_VALUE]		= "scalar",
+		[PTR_TO_CTX]		= "ctx",
+		[CONST_PTR_TO_MAP]	= "map_ptr",
+		[PTR_TO_MAP_VALUE]	= "map_value",
+		[PTR_TO_STACK]		= "fp",
+		[PTR_TO_PACKET]		= "pkt",
+		[PTR_TO_PACKET_META]	= "pkt_meta",
+		[PTR_TO_PACKET_END]	= "pkt_end",
+		[PTR_TO_FLOW_KEYS]	= "flow_keys",
+		[PTR_TO_SOCKET]		= "sock",
+		[PTR_TO_SOCK_COMMON]	= "sock_common",
+		[PTR_TO_TCP_SOCK]	= "tcp_sock",
+		[PTR_TO_TP_BUFFER]	= "tp_buffer",
+		[PTR_TO_XDP_SOCK]	= "xdp_sock",
+		[PTR_TO_BTF_ID]		= "ptr_",
+		[PTR_TO_MEM]		= "mem",
+		[PTR_TO_BUF]		= "buf",
+		[PTR_TO_FUNC]		= "func",
+		[PTR_TO_MAP_KEY]	= "map_key",
+		[CONST_PTR_TO_DYNPTR]	= "dynptr_ptr",
+	};
+
+	if (type & PTR_MAYBE_NULL) {
+		if (base_type(type) == PTR_TO_BTF_ID)
+			strncpy(postfix, "or_null_", 16);
+		else
+			strncpy(postfix, "_or_null", 16);
+	}
+
+	snprintf(prefix, sizeof(prefix), "%s%s%s%s%s%s%s",
+		 type & MEM_RDONLY ? "rdonly_" : "",
+		 type & MEM_RINGBUF ? "ringbuf_" : "",
+		 type & MEM_USER ? "user_" : "",
+		 type & MEM_PERCPU ? "percpu_" : "",
+		 type & MEM_RCU ? "rcu_" : "",
+		 type & PTR_UNTRUSTED ? "untrusted_" : "",
+		 type & PTR_TRUSTED ? "trusted_" : ""
+	);
+
+	snprintf(env->tmp_str_buf, TMP_STR_BUF_LEN, "%s%s%s",
+		 prefix, str[base_type(type)], postfix);
+	return env->tmp_str_buf;
+}
+
+const char *dynptr_type_str(enum bpf_dynptr_type type)
+{
+	switch (type) {
+	case BPF_DYNPTR_TYPE_LOCAL:
+		return "local";
+	case BPF_DYNPTR_TYPE_RINGBUF:
+		return "ringbuf";
+	case BPF_DYNPTR_TYPE_SKB:
+		return "skb";
+	case BPF_DYNPTR_TYPE_XDP:
+		return "xdp";
+	case BPF_DYNPTR_TYPE_INVALID:
+		return "<invalid>";
+	default:
+		WARN_ONCE(1, "unknown dynptr type %d\n", type);
+		return "<unknown>";
+	}
+}
+
+const char *iter_type_str(const struct btf *btf, u32 btf_id)
+{
+	if (!btf || btf_id == 0)
+		return "<invalid>";
+
+	/* we already validated that type is valid and has conforming name */
+	return btf_type_name(btf, btf_id) + sizeof(ITER_PREFIX) - 1;
+}
+
+const char *iter_state_str(enum bpf_iter_state state)
+{
+	switch (state) {
+	case BPF_ITER_STATE_ACTIVE:
+		return "active";
+	case BPF_ITER_STATE_DRAINED:
+		return "drained";
+	case BPF_ITER_STATE_INVALID:
+		return "<invalid>";
+	default:
+		WARN_ONCE(1, "unknown iter state %d\n", state);
+		return "<unknown>";
+	}
+}
+
+static char slot_type_char[] = {
+	[STACK_INVALID]	= '?',
+	[STACK_SPILL]	= 'r',
+	[STACK_MISC]	= 'm',
+	[STACK_ZERO]	= '0',
+	[STACK_DYNPTR]	= 'd',
+	[STACK_ITER]	= 'i',
+};
+
+static void print_liveness(struct bpf_verifier_env *env,
+			   enum bpf_reg_liveness live)
+{
+	if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE))
+	    verbose(env, "_");
+	if (live & REG_LIVE_READ)
+		verbose(env, "r");
+	if (live & REG_LIVE_WRITTEN)
+		verbose(env, "w");
+	if (live & REG_LIVE_DONE)
+		verbose(env, "D");
+}
+
+#define UNUM_MAX_DECIMAL U16_MAX
+#define SNUM_MAX_DECIMAL S16_MAX
+#define SNUM_MIN_DECIMAL S16_MIN
+
+static bool is_unum_decimal(u64 num)
+{
+	return num <= UNUM_MAX_DECIMAL;
+}
+
+static bool is_snum_decimal(s64 num)
+{
+	return num >= SNUM_MIN_DECIMAL && num <= SNUM_MAX_DECIMAL;
+}
+
+static void verbose_unum(struct bpf_verifier_env *env, u64 num)
+{
+	if (is_unum_decimal(num))
+		verbose(env, "%llu", num);
+	else
+		verbose(env, "%#llx", num);
+}
+
+static void verbose_snum(struct bpf_verifier_env *env, s64 num)
+{
+	if (is_snum_decimal(num))
+		verbose(env, "%lld", num);
+	else
+		verbose(env, "%#llx", num);
+}
+
+static void print_scalar_ranges(struct bpf_verifier_env *env,
+				const struct bpf_reg_state *reg,
+				const char **sep)
+{
+	/* For signed ranges, we want to unify 64-bit and 32-bit values in the
+	 * output as much as possible, but there is a bit of a complication.
+	 * If we choose to print values as decimals, this is natural to do,
+	 * because negative 64-bit and 32-bit values >= -S32_MIN have the same
+	 * representation due to sign extension. But if we choose to print
+	 * them in hex format (see is_snum_decimal()), then sign extension is
+	 * misleading.
+	 * E.g., smin=-2 and smin32=-2 are exactly the same in decimal, but in
+	 * hex they will be smin=0xfffffffffffffffe and smin32=0xfffffffe, two
+	 * very different numbers.
+	 * So we avoid sign extension if we choose to print values in hex.
+	 */
+	struct {
+		const char *name;
+		u64 val;
+		bool omit;
+	} minmaxs[] = {
+		{"smin",   reg->smin_value,         reg->smin_value == S64_MIN},
+		{"smax",   reg->smax_value,         reg->smax_value == S64_MAX},
+		{"umin",   reg->umin_value,         reg->umin_value == 0},
+		{"umax",   reg->umax_value,         reg->umax_value == U64_MAX},
+		{"smin32",
+		 is_snum_decimal((s64)reg->s32_min_value)
+			 ? (s64)reg->s32_min_value
+			 : (u32)reg->s32_min_value, reg->s32_min_value == S32_MIN},
+		{"smax32",
+		 is_snum_decimal((s64)reg->s32_max_value)
+			 ? (s64)reg->s32_max_value
+			 : (u32)reg->s32_max_value, reg->s32_max_value == S32_MAX},
+		{"umin32", reg->u32_min_value,      reg->u32_min_value == 0},
+		{"umax32", reg->u32_max_value,      reg->u32_max_value == U32_MAX},
+	}, *m1, *m2, *mend = &minmaxs[ARRAY_SIZE(minmaxs)];
+	bool neg1, neg2;
+
+	for (m1 = &minmaxs[0]; m1 < mend; m1++) {
+		if (m1->omit)
+			continue;
+
+		neg1 = m1->name[0] == 's' && (s64)m1->val < 0;
+
+		verbose(env, "%s%s=", *sep, m1->name);
+		*sep = ",";
+
+		for (m2 = m1 + 2; m2 < mend; m2 += 2) {
+			if (m2->omit || m2->val != m1->val)
+				continue;
+			/* don't mix negatives with positives */
+			neg2 = m2->name[0] == 's' && (s64)m2->val < 0;
+			if (neg2 != neg1)
+				continue;
+			m2->omit = true;
+			verbose(env, "%s=", m2->name);
+		}
+
+		if (m1->name[0] == 's')
+			verbose_snum(env, m1->val);
+		else
+			verbose_unum(env, m1->val);
+	}
+}
+
+static bool type_is_map_ptr(enum bpf_reg_type t) {
+	switch (base_type(t)) {
+	case CONST_PTR_TO_MAP:
+	case PTR_TO_MAP_KEY:
+	case PTR_TO_MAP_VALUE:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static void print_reg_state(struct bpf_verifier_env *env,
+			    const struct bpf_func_state *state,
+			    const struct bpf_reg_state *reg)
+{
+	enum bpf_reg_type t;
+	const char *sep = "";
+
+	t = reg->type;
+	if (t == SCALAR_VALUE && reg->precise)
+		verbose(env, "P");
+	if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) {
+		/* reg->off should be 0 for SCALAR_VALUE */
+		verbose_snum(env, reg->var_off.value + reg->off);
+		return;
+	}
+/*
+ * _a stands for append, was shortened to avoid multiline statements below.
+ * This macro is used to output a comma separated list of attributes.
+ */
+#define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, ##__VA_ARGS__); sep = ","; })
+
+	verbose(env, "%s", reg_type_str(env, t));
+	if (t == PTR_TO_STACK) {
+		if (state->frameno != reg->frameno)
+			verbose(env, "[%d]", reg->frameno);
+		if (tnum_is_const(reg->var_off)) {
+			verbose_snum(env, reg->var_off.value + reg->off);
+			return;
+		}
+	}
+	if (base_type(t) == PTR_TO_BTF_ID)
+		verbose(env, "%s", btf_type_name(reg->btf, reg->btf_id));
+	verbose(env, "(");
+	if (reg->id)
+		verbose_a("id=%d", reg->id);
+	if (reg->ref_obj_id)
+		verbose_a("ref_obj_id=%d", reg->ref_obj_id);
+	if (type_is_non_owning_ref(reg->type))
+		verbose_a("%s", "non_own_ref");
+	if (type_is_map_ptr(t)) {
+		if (reg->map_ptr->name[0])
+			verbose_a("map=%s", reg->map_ptr->name);
+		verbose_a("ks=%d,vs=%d",
+			  reg->map_ptr->key_size,
+			  reg->map_ptr->value_size);
+	}
+	if (t != SCALAR_VALUE && reg->off) {
+		verbose_a("off=");
+		verbose_snum(env, reg->off);
+	}
+	if (type_is_pkt_pointer(t)) {
+		verbose_a("r=");
+		verbose_unum(env, reg->range);
+	}
+	if (tnum_is_const(reg->var_off)) {
+		/* a pointer register with fixed offset */
+		if (reg->var_off.value) {
+			verbose_a("imm=");
+			verbose_snum(env, reg->var_off.value);
+		}
+	} else {
+		print_scalar_ranges(env, reg, &sep);
+		if (!tnum_is_unknown(reg->var_off)) {
+			char tn_buf[48];
+
+			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
+			verbose_a("var_off=%s", tn_buf);
+		}
+	}
+	verbose(env, ")");
+
+#undef verbose_a
+}
+
+void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_func_state *state,
+			  bool print_all)
+{
+	const struct bpf_reg_state *reg;
+	int i;
+
+	if (state->frameno)
+		verbose(env, " frame%d:", state->frameno);
+	for (i = 0; i < MAX_BPF_REG; i++) {
+		reg = &state->regs[i];
+		if (reg->type == NOT_INIT)
+			continue;
+		if (!print_all && !reg_scratched(env, i))
+			continue;
+		verbose(env, " R%d", i);
+		print_liveness(env, reg->live);
+		verbose(env, "=");
+		print_reg_state(env, state, reg);
+	}
+	for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
+		char types_buf[BPF_REG_SIZE + 1];
+		bool valid = false;
+		u8 slot_type;
+		int j;
+
+		if (!print_all && !stack_slot_scratched(env, i))
+			continue;
+
+		for (j = 0; j < BPF_REG_SIZE; j++) {
+			slot_type = state->stack[i].slot_type[j];
+			if (slot_type != STACK_INVALID)
+				valid = true;
+			types_buf[j] = slot_type_char[slot_type];
+		}
+		types_buf[BPF_REG_SIZE] = 0;
+		if (!valid)
+			continue;
+
+		reg = &state->stack[i].spilled_ptr;
+		switch (state->stack[i].slot_type[BPF_REG_SIZE - 1]) {
+		case STACK_SPILL:
+			/* print MISC/ZERO/INVALID slots above subreg spill */
+			for (j = 0; j < BPF_REG_SIZE; j++)
+				if (state->stack[i].slot_type[j] == STACK_SPILL)
+					break;
+			types_buf[j] = '\0';
+
+			verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
+			print_liveness(env, reg->live);
+			verbose(env, "=%s", types_buf);
+			print_reg_state(env, state, reg);
+			break;
+		case STACK_DYNPTR:
+			/* skip to main dynptr slot */
+			i += BPF_DYNPTR_NR_SLOTS - 1;
+			reg = &state->stack[i].spilled_ptr;
+
+			verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
+			print_liveness(env, reg->live);
+			verbose(env, "=dynptr_%s", dynptr_type_str(reg->dynptr.type));
+			if (reg->ref_obj_id)
+				verbose(env, "(ref_id=%d)", reg->ref_obj_id);
+			break;
+		case STACK_ITER:
+			/* only main slot has ref_obj_id set; skip others */
+			if (!reg->ref_obj_id)
+				continue;
+
+			verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
+			print_liveness(env, reg->live);
+			verbose(env, "=iter_%s(ref_id=%d,state=%s,depth=%u)",
+				iter_type_str(reg->iter.btf, reg->iter.btf_id),
+				reg->ref_obj_id, iter_state_str(reg->iter.state),
+				reg->iter.depth);
+			break;
+		case STACK_MISC:
+		case STACK_ZERO:
+		default:
+			verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
+			print_liveness(env, reg->live);
+			verbose(env, "=%s", types_buf);
+			break;
+		}
+	}
+	if (state->acquired_refs && state->refs[0].id) {
+		verbose(env, " refs=%d", state->refs[0].id);
+		for (i = 1; i < state->acquired_refs; i++)
+			if (state->refs[i].id)
+				verbose(env, ",%d", state->refs[i].id);
+	}
+	if (state->in_callback_fn)
+		verbose(env, " cb");
+	if (state->in_async_callback_fn)
+		verbose(env, " async_cb");
+	verbose(env, "\n");
+	if (!print_all)
+		mark_verifier_state_clean(env);
+}
+
+static inline u32 vlog_alignment(u32 pos)
+{
+	return round_up(max(pos + BPF_LOG_MIN_ALIGNMENT / 2, BPF_LOG_ALIGNMENT),
+			BPF_LOG_MIN_ALIGNMENT) - pos - 1;
+}
+
+void print_insn_state(struct bpf_verifier_env *env, const struct bpf_func_state *state)
+{
+	if (env->prev_log_pos && env->prev_log_pos == env->log.end_pos) {
+		/* remove new line character */
+		bpf_vlog_reset(&env->log, env->prev_log_pos - 1);
+		verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_pos), ' ');
+	} else {
+		verbose(env, "%d:", env->insn_idx);
+	}
+	print_verifier_state(env, state, false);
+}
diff --git a/kernel/bpf/lpm_trie.c b/kernel/bpf/lpm_trie.c
index 17c7e7782a1f..b32be680da6c 100644
--- a/kernel/bpf/lpm_trie.c
+++ b/kernel/bpf/lpm_trie.c
@@ -231,6 +231,9 @@ static void *trie_lookup_elem(struct bpf_map *map, void *_key)
 	struct lpm_trie_node *node, *found = NULL;
 	struct bpf_lpm_trie_key *key = _key;
 
+	if (key->prefixlen > trie->max_prefixlen)
+		return NULL;
+
 	/* Start walking the trie from the root node ... */
 
 	for (node = rcu_dereference_check(trie->root, rcu_read_lock_bh_held());
diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
index d6b277482085..dff7ba539701 100644
--- a/kernel/bpf/stackmap.c
+++ b/kernel/bpf/stackmap.c
@@ -388,6 +388,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 {
 	u32 trace_nr, copy_len, elem_size, num_elem, max_depth;
 	bool user_build_id = flags & BPF_F_USER_BUILD_ID;
+	bool crosstask = task && task != current;
 	u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
 	bool user = flags & BPF_F_USER_STACK;
 	struct perf_callchain_entry *trace;
@@ -410,6 +411,14 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 	if (task && user && !user_mode(regs))
 		goto err_fault;
 
+	/* get_perf_callchain does not support crosstask user stack walking
+	 * but returns an empty stack instead of NULL.
+	 */
+	if (crosstask && user) {
+		err = -EOPNOTSUPP;
+		goto clear;
+	}
+
 	num_elem = size / elem_size;
 	max_depth = num_elem + skip;
 	if (sysctl_perf_event_max_stack < max_depth)
@@ -421,7 +430,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
 		trace = get_callchain_entry_for_task(task, max_depth);
 	else
 		trace = get_perf_callchain(regs, 0, kernel, user, max_depth,
-					   false, false);
+					   crosstask, false);
 	if (unlikely(!trace))
 		goto err_fault;
 
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 0ed286b8a0f0..5e43ddd1b83f 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -2573,7 +2573,8 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size)
 				 BPF_F_SLEEPABLE |
 				 BPF_F_TEST_RND_HI32 |
 				 BPF_F_XDP_HAS_FRAGS |
-				 BPF_F_XDP_DEV_BOUND_ONLY))
+				 BPF_F_XDP_DEV_BOUND_ONLY |
+				 BPF_F_TEST_REG_INVARIANTS))
 		return -EINVAL;
 
 	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c
index 26082b97894d..e5c3500443c6 100644
--- a/kernel/bpf/task_iter.c
+++ b/kernel/bpf/task_iter.c
@@ -70,15 +70,13 @@ static struct task_struct *task_group_seq_get_next(struct bpf_iter_seq_task_comm
 		return NULL;
 
 retry:
-	task = next_thread(task);
+	task = __next_thread(task);
+	if (!task)
+		return NULL;
 
 	next_tid = __task_pid_nr_ns(task, PIDTYPE_PID, common->ns);
-	if (!next_tid || next_tid == common->pid) {
-		/* Run out of tasks of a process.  The tasks of a
-		 * thread_group are linked as circular linked list.
-		 */
-		return NULL;
-	}
+	if (!next_tid)
+		goto retry;
 
 	if (skip_if_dup_files && task->files == task->group_leader->files)
 		goto retry;
@@ -980,7 +978,6 @@ __bpf_kfunc int bpf_iter_task_new(struct bpf_iter_task *it,
 	BUILD_BUG_ON(__alignof__(struct bpf_iter_task_kern) !=
 					__alignof__(struct bpf_iter_task));
 
-	kit->task = kit->pos = NULL;
 	switch (flags) {
 	case BPF_TASK_ITER_ALL_THREADS:
 	case BPF_TASK_ITER_ALL_PROCS:
@@ -1017,20 +1014,16 @@ __bpf_kfunc struct task_struct *bpf_iter_task_next(struct bpf_iter_task *it)
 	if (flags == BPF_TASK_ITER_ALL_PROCS)
 		goto get_next_task;
 
-	kit->pos = next_thread(kit->pos);
-	if (kit->pos == kit->task) {
-		if (flags == BPF_TASK_ITER_PROC_THREADS) {
-			kit->pos = NULL;
-			return pos;
-		}
-	} else
+	kit->pos = __next_thread(kit->pos);
+	if (kit->pos || flags == BPF_TASK_ITER_PROC_THREADS)
 		return pos;
 
 get_next_task:
-	kit->pos = next_task(kit->pos);
-	kit->task = kit->pos;
-	if (kit->pos == &init_task)
+	kit->task = next_task(kit->task);
+	if (kit->task == &init_task)
 		kit->pos = NULL;
+	else
+		kit->pos = kit->task;
 
 	return pos;
 }
diff --git a/kernel/bpf/tnum.c b/kernel/bpf/tnum.c
index 3d7127f439a1..f4c91c9b27d7 100644
--- a/kernel/bpf/tnum.c
+++ b/kernel/bpf/tnum.c
@@ -208,7 +208,12 @@ struct tnum tnum_clear_subreg(struct tnum a)
 	return tnum_lshift(tnum_rshift(a, 32), 32);
 }
 
+struct tnum tnum_with_subreg(struct tnum reg, struct tnum subreg)
+{
+	return tnum_or(tnum_clear_subreg(reg), tnum_subreg(subreg));
+}
+
 struct tnum tnum_const_subreg(struct tnum a, u32 value)
 {
-	return tnum_or(tnum_clear_subreg(a), tnum_const(value));
+	return tnum_with_subreg(a, tnum_const(value));
 }
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 6da370a047fe..405da1f9e724 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -342,27 +342,6 @@ struct btf *btf_vmlinux;
 static DEFINE_MUTEX(bpf_verifier_lock);
 static DEFINE_MUTEX(bpf_percpu_ma_lock);
 
-static const struct bpf_line_info *
-find_linfo(const struct bpf_verifier_env *env, u32 insn_off)
-{
-	const struct bpf_line_info *linfo;
-	const struct bpf_prog *prog;
-	u32 i, nr_linfo;
-
-	prog = env->prog;
-	nr_linfo = prog->aux->nr_linfo;
-
-	if (!nr_linfo || insn_off >= prog->len)
-		return NULL;
-
-	linfo = prog->aux->linfo;
-	for (i = 1; i < nr_linfo; i++)
-		if (insn_off < linfo[i].insn_off)
-			break;
-
-	return &linfo[i - 1];
-}
-
 __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...)
 {
 	struct bpf_verifier_env *env = private_data;
@@ -376,42 +355,6 @@ __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...)
 	va_end(args);
 }
 
-static const char *ltrim(const char *s)
-{
-	while (isspace(*s))
-		s++;
-
-	return s;
-}
-
-__printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env,
-					 u32 insn_off,
-					 const char *prefix_fmt, ...)
-{
-	const struct bpf_line_info *linfo;
-
-	if (!bpf_verifier_log_needed(&env->log))
-		return;
-
-	linfo = find_linfo(env, insn_off);
-	if (!linfo || linfo == env->prev_linfo)
-		return;
-
-	if (prefix_fmt) {
-		va_list args;
-
-		va_start(args, prefix_fmt);
-		bpf_verifier_vlog(&env->log, prefix_fmt, args);
-		va_end(args);
-	}
-
-	verbose(env, "%s\n",
-		ltrim(btf_name_by_offset(env->prog->aux->btf,
-					 linfo->line_off)));
-
-	env->prev_linfo = linfo;
-}
-
 static void verbose_invalid_scalar(struct bpf_verifier_env *env,
 				   struct bpf_reg_state *reg,
 				   struct tnum *range, const char *ctx,
@@ -430,21 +373,6 @@ static void verbose_invalid_scalar(struct bpf_verifier_env *env,
 	verbose(env, " should have been in %s\n", tn_buf);
 }
 
-static bool type_is_pkt_pointer(enum bpf_reg_type type)
-{
-	type = base_type(type);
-	return type == PTR_TO_PACKET ||
-	       type == PTR_TO_PACKET_META;
-}
-
-static bool type_is_sk_pointer(enum bpf_reg_type type)
-{
-	return type == PTR_TO_SOCKET ||
-		type == PTR_TO_SOCK_COMMON ||
-		type == PTR_TO_TCP_SOCK ||
-		type == PTR_TO_XDP_SOCK;
-}
-
 static bool type_may_be_null(u32 type)
 {
 	return type & PTR_MAYBE_NULL;
@@ -468,16 +396,6 @@ static bool reg_not_null(const struct bpf_reg_state *reg)
 		type == PTR_TO_MEM;
 }
 
-static bool type_is_ptr_alloc_obj(u32 type)
-{
-	return base_type(type) == PTR_TO_BTF_ID && type_flag(type) & MEM_ALLOC;
-}
-
-static bool type_is_non_owning_ref(u32 type)
-{
-	return type_is_ptr_alloc_obj(type) && type_flag(type) & NON_OWN_REF;
-}
-
 static struct btf_record *reg_btf_record(const struct bpf_reg_state *reg)
 {
 	struct btf_record *rec = NULL;
@@ -547,13 +465,12 @@ static bool is_dynptr_ref_function(enum bpf_func_id func_id)
 	return func_id == BPF_FUNC_dynptr_data;
 }
 
-static bool is_callback_calling_kfunc(u32 btf_id);
+static bool is_sync_callback_calling_kfunc(u32 btf_id);
 static bool is_bpf_throw_kfunc(struct bpf_insn *insn);
 
-static bool is_callback_calling_function(enum bpf_func_id func_id)
+static bool is_sync_callback_calling_function(enum bpf_func_id func_id)
 {
 	return func_id == BPF_FUNC_for_each_map_elem ||
-	       func_id == BPF_FUNC_timer_set_callback ||
 	       func_id == BPF_FUNC_find_vma ||
 	       func_id == BPF_FUNC_loop ||
 	       func_id == BPF_FUNC_user_ringbuf_drain;
@@ -564,6 +481,18 @@ static bool is_async_callback_calling_function(enum bpf_func_id func_id)
 	return func_id == BPF_FUNC_timer_set_callback;
 }
 
+static bool is_callback_calling_function(enum bpf_func_id func_id)
+{
+	return is_sync_callback_calling_function(func_id) ||
+	       is_async_callback_calling_function(func_id);
+}
+
+static bool is_sync_callback_calling_insn(struct bpf_insn *insn)
+{
+	return (bpf_helper_call(insn) && is_sync_callback_calling_function(insn->imm)) ||
+	       (bpf_pseudo_kfunc_call(insn) && is_sync_callback_calling_kfunc(insn->imm));
+}
+
 static bool is_storage_get_function(enum bpf_func_id func_id)
 {
 	return func_id == BPF_FUNC_sk_storage_get ||
@@ -594,83 +523,6 @@ static bool is_cmpxchg_insn(const struct bpf_insn *insn)
 	       insn->imm == BPF_CMPXCHG;
 }
 
-/* string representation of 'enum bpf_reg_type'
- *
- * Note that reg_type_str() can not appear more than once in a single verbose()
- * statement.
- */
-static const char *reg_type_str(struct bpf_verifier_env *env,
-				enum bpf_reg_type type)
-{
-	char postfix[16] = {0}, prefix[64] = {0};
-	static const char * const str[] = {
-		[NOT_INIT]		= "?",
-		[SCALAR_VALUE]		= "scalar",
-		[PTR_TO_CTX]		= "ctx",
-		[CONST_PTR_TO_MAP]	= "map_ptr",
-		[PTR_TO_MAP_VALUE]	= "map_value",
-		[PTR_TO_STACK]		= "fp",
-		[PTR_TO_PACKET]		= "pkt",
-		[PTR_TO_PACKET_META]	= "pkt_meta",
-		[PTR_TO_PACKET_END]	= "pkt_end",
-		[PTR_TO_FLOW_KEYS]	= "flow_keys",
-		[PTR_TO_SOCKET]		= "sock",
-		[PTR_TO_SOCK_COMMON]	= "sock_common",
-		[PTR_TO_TCP_SOCK]	= "tcp_sock",
-		[PTR_TO_TP_BUFFER]	= "tp_buffer",
-		[PTR_TO_XDP_SOCK]	= "xdp_sock",
-		[PTR_TO_BTF_ID]		= "ptr_",
-		[PTR_TO_MEM]		= "mem",
-		[PTR_TO_BUF]		= "buf",
-		[PTR_TO_FUNC]		= "func",
-		[PTR_TO_MAP_KEY]	= "map_key",
-		[CONST_PTR_TO_DYNPTR]	= "dynptr_ptr",
-	};
-
-	if (type & PTR_MAYBE_NULL) {
-		if (base_type(type) == PTR_TO_BTF_ID)
-			strncpy(postfix, "or_null_", 16);
-		else
-			strncpy(postfix, "_or_null", 16);
-	}
-
-	snprintf(prefix, sizeof(prefix), "%s%s%s%s%s%s%s",
-		 type & MEM_RDONLY ? "rdonly_" : "",
-		 type & MEM_RINGBUF ? "ringbuf_" : "",
-		 type & MEM_USER ? "user_" : "",
-		 type & MEM_PERCPU ? "percpu_" : "",
-		 type & MEM_RCU ? "rcu_" : "",
-		 type & PTR_UNTRUSTED ? "untrusted_" : "",
-		 type & PTR_TRUSTED ? "trusted_" : ""
-	);
-
-	snprintf(env->tmp_str_buf, TMP_STR_BUF_LEN, "%s%s%s",
-		 prefix, str[base_type(type)], postfix);
-	return env->tmp_str_buf;
-}
-
-static char slot_type_char[] = {
-	[STACK_INVALID]	= '?',
-	[STACK_SPILL]	= 'r',
-	[STACK_MISC]	= 'm',
-	[STACK_ZERO]	= '0',
-	[STACK_DYNPTR]	= 'd',
-	[STACK_ITER]	= 'i',
-};
-
-static void print_liveness(struct bpf_verifier_env *env,
-			   enum bpf_reg_liveness live)
-{
-	if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE))
-	    verbose(env, "_");
-	if (live & REG_LIVE_READ)
-		verbose(env, "r");
-	if (live & REG_LIVE_WRITTEN)
-		verbose(env, "w");
-	if (live & REG_LIVE_DONE)
-		verbose(env, "D");
-}
-
 static int __get_spi(s32 off)
 {
 	return (-off - 1) / BPF_REG_SIZE;
@@ -740,87 +592,6 @@ static const char *btf_type_name(const struct btf *btf, u32 id)
 	return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);
 }
 
-static const char *dynptr_type_str(enum bpf_dynptr_type type)
-{
-	switch (type) {
-	case BPF_DYNPTR_TYPE_LOCAL:
-		return "local";
-	case BPF_DYNPTR_TYPE_RINGBUF:
-		return "ringbuf";
-	case BPF_DYNPTR_TYPE_SKB:
-		return "skb";
-	case BPF_DYNPTR_TYPE_XDP:
-		return "xdp";
-	case BPF_DYNPTR_TYPE_INVALID:
-		return "<invalid>";
-	default:
-		WARN_ONCE(1, "unknown dynptr type %d\n", type);
-		return "<unknown>";
-	}
-}
-
-static const char *iter_type_str(const struct btf *btf, u32 btf_id)
-{
-	if (!btf || btf_id == 0)
-		return "<invalid>";
-
-	/* we already validated that type is valid and has conforming name */
-	return btf_type_name(btf, btf_id) + sizeof(ITER_PREFIX) - 1;
-}
-
-static const char *iter_state_str(enum bpf_iter_state state)
-{
-	switch (state) {
-	case BPF_ITER_STATE_ACTIVE:
-		return "active";
-	case BPF_ITER_STATE_DRAINED:
-		return "drained";
-	case BPF_ITER_STATE_INVALID:
-		return "<invalid>";
-	default:
-		WARN_ONCE(1, "unknown iter state %d\n", state);
-		return "<unknown>";
-	}
-}
-
-static void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno)
-{
-	env->scratched_regs |= 1U << regno;
-}
-
-static void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi)
-{
-	env->scratched_stack_slots |= 1ULL << spi;
-}
-
-static bool reg_scratched(const struct bpf_verifier_env *env, u32 regno)
-{
-	return (env->scratched_regs >> regno) & 1;
-}
-
-static bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno)
-{
-	return (env->scratched_stack_slots >> regno) & 1;
-}
-
-static bool verifier_state_scratched(const struct bpf_verifier_env *env)
-{
-	return env->scratched_regs || env->scratched_stack_slots;
-}
-
-static void mark_verifier_state_clean(struct bpf_verifier_env *env)
-{
-	env->scratched_regs = 0U;
-	env->scratched_stack_slots = 0ULL;
-}
-
-/* Used for printing the entire verifier state. */
-static void mark_verifier_state_scratched(struct bpf_verifier_env *env)
-{
-	env->scratched_regs = ~0U;
-	env->scratched_stack_slots = ~0ULL;
-}
-
 static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type)
 {
 	switch (arg_type & DYNPTR_TYPE_FLAG_MASK) {
@@ -1360,226 +1131,6 @@ static void scrub_spilled_slot(u8 *stype)
 		*stype = STACK_MISC;
 }
 
-static void print_scalar_ranges(struct bpf_verifier_env *env,
-				const struct bpf_reg_state *reg,
-				const char **sep)
-{
-	struct {
-		const char *name;
-		u64 val;
-		bool omit;
-	} minmaxs[] = {
-		{"smin",   reg->smin_value,         reg->smin_value == S64_MIN},
-		{"smax",   reg->smax_value,         reg->smax_value == S64_MAX},
-		{"umin",   reg->umin_value,         reg->umin_value == 0},
-		{"umax",   reg->umax_value,         reg->umax_value == U64_MAX},
-		{"smin32", (s64)reg->s32_min_value, reg->s32_min_value == S32_MIN},
-		{"smax32", (s64)reg->s32_max_value, reg->s32_max_value == S32_MAX},
-		{"umin32", reg->u32_min_value,      reg->u32_min_value == 0},
-		{"umax32", reg->u32_max_value,      reg->u32_max_value == U32_MAX},
-	}, *m1, *m2, *mend = &minmaxs[ARRAY_SIZE(minmaxs)];
-	bool neg1, neg2;
-
-	for (m1 = &minmaxs[0]; m1 < mend; m1++) {
-		if (m1->omit)
-			continue;
-
-		neg1 = m1->name[0] == 's' && (s64)m1->val < 0;
-
-		verbose(env, "%s%s=", *sep, m1->name);
-		*sep = ",";
-
-		for (m2 = m1 + 2; m2 < mend; m2 += 2) {
-			if (m2->omit || m2->val != m1->val)
-				continue;
-			/* don't mix negatives with positives */
-			neg2 = m2->name[0] == 's' && (s64)m2->val < 0;
-			if (neg2 != neg1)
-				continue;
-			m2->omit = true;
-			verbose(env, "%s=", m2->name);
-		}
-
-		verbose(env, m1->name[0] == 's' ? "%lld" : "%llu", m1->val);
-	}
-}
-
-static void print_verifier_state(struct bpf_verifier_env *env,
-				 const struct bpf_func_state *state,
-				 bool print_all)
-{
-	const struct bpf_reg_state *reg;
-	enum bpf_reg_type t;
-	int i;
-
-	if (state->frameno)
-		verbose(env, " frame%d:", state->frameno);
-	for (i = 0; i < MAX_BPF_REG; i++) {
-		reg = &state->regs[i];
-		t = reg->type;
-		if (t == NOT_INIT)
-			continue;
-		if (!print_all && !reg_scratched(env, i))
-			continue;
-		verbose(env, " R%d", i);
-		print_liveness(env, reg->live);
-		verbose(env, "=");
-		if (t == SCALAR_VALUE && reg->precise)
-			verbose(env, "P");
-		if ((t == SCALAR_VALUE || t == PTR_TO_STACK) &&
-		    tnum_is_const(reg->var_off)) {
-			/* reg->off should be 0 for SCALAR_VALUE */
-			verbose(env, "%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t));
-			verbose(env, "%lld", reg->var_off.value + reg->off);
-		} else {
-			const char *sep = "";
-
-			verbose(env, "%s", reg_type_str(env, t));
-			if (base_type(t) == PTR_TO_BTF_ID)
-				verbose(env, "%s", btf_type_name(reg->btf, reg->btf_id));
-			verbose(env, "(");
-/*
- * _a stands for append, was shortened to avoid multiline statements below.
- * This macro is used to output a comma separated list of attributes.
- */
-#define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, __VA_ARGS__); sep = ","; })
-
-			if (reg->id)
-				verbose_a("id=%d", reg->id);
-			if (reg->ref_obj_id)
-				verbose_a("ref_obj_id=%d", reg->ref_obj_id);
-			if (type_is_non_owning_ref(reg->type))
-				verbose_a("%s", "non_own_ref");
-			if (t != SCALAR_VALUE)
-				verbose_a("off=%d", reg->off);
-			if (type_is_pkt_pointer(t))
-				verbose_a("r=%d", reg->range);
-			else if (base_type(t) == CONST_PTR_TO_MAP ||
-				 base_type(t) == PTR_TO_MAP_KEY ||
-				 base_type(t) == PTR_TO_MAP_VALUE)
-				verbose_a("ks=%d,vs=%d",
-					  reg->map_ptr->key_size,
-					  reg->map_ptr->value_size);
-			if (tnum_is_const(reg->var_off)) {
-				/* Typically an immediate SCALAR_VALUE, but
-				 * could be a pointer whose offset is too big
-				 * for reg->off
-				 */
-				verbose_a("imm=%llx", reg->var_off.value);
-			} else {
-				print_scalar_ranges(env, reg, &sep);
-				if (!tnum_is_unknown(reg->var_off)) {
-					char tn_buf[48];
-
-					tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
-					verbose_a("var_off=%s", tn_buf);
-				}
-			}
-#undef verbose_a
-
-			verbose(env, ")");
-		}
-	}
-	for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
-		char types_buf[BPF_REG_SIZE + 1];
-		bool valid = false;
-		int j;
-
-		for (j = 0; j < BPF_REG_SIZE; j++) {
-			if (state->stack[i].slot_type[j] != STACK_INVALID)
-				valid = true;
-			types_buf[j] = slot_type_char[state->stack[i].slot_type[j]];
-		}
-		types_buf[BPF_REG_SIZE] = 0;
-		if (!valid)
-			continue;
-		if (!print_all && !stack_slot_scratched(env, i))
-			continue;
-		switch (state->stack[i].slot_type[BPF_REG_SIZE - 1]) {
-		case STACK_SPILL:
-			reg = &state->stack[i].spilled_ptr;
-			t = reg->type;
-
-			verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
-			print_liveness(env, reg->live);
-			verbose(env, "=%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t));
-			if (t == SCALAR_VALUE && reg->precise)
-				verbose(env, "P");
-			if (t == SCALAR_VALUE && tnum_is_const(reg->var_off))
-				verbose(env, "%lld", reg->var_off.value + reg->off);
-			break;
-		case STACK_DYNPTR:
-			i += BPF_DYNPTR_NR_SLOTS - 1;
-			reg = &state->stack[i].spilled_ptr;
-
-			verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
-			print_liveness(env, reg->live);
-			verbose(env, "=dynptr_%s", dynptr_type_str(reg->dynptr.type));
-			if (reg->ref_obj_id)
-				verbose(env, "(ref_id=%d)", reg->ref_obj_id);
-			break;
-		case STACK_ITER:
-			/* only main slot has ref_obj_id set; skip others */
-			reg = &state->stack[i].spilled_ptr;
-			if (!reg->ref_obj_id)
-				continue;
-
-			verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
-			print_liveness(env, reg->live);
-			verbose(env, "=iter_%s(ref_id=%d,state=%s,depth=%u)",
-				iter_type_str(reg->iter.btf, reg->iter.btf_id),
-				reg->ref_obj_id, iter_state_str(reg->iter.state),
-				reg->iter.depth);
-			break;
-		case STACK_MISC:
-		case STACK_ZERO:
-		default:
-			reg = &state->stack[i].spilled_ptr;
-
-			for (j = 0; j < BPF_REG_SIZE; j++)
-				types_buf[j] = slot_type_char[state->stack[i].slot_type[j]];
-			types_buf[BPF_REG_SIZE] = 0;
-
-			verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
-			print_liveness(env, reg->live);
-			verbose(env, "=%s", types_buf);
-			break;
-		}
-	}
-	if (state->acquired_refs && state->refs[0].id) {
-		verbose(env, " refs=%d", state->refs[0].id);
-		for (i = 1; i < state->acquired_refs; i++)
-			if (state->refs[i].id)
-				verbose(env, ",%d", state->refs[i].id);
-	}
-	if (state->in_callback_fn)
-		verbose(env, " cb");
-	if (state->in_async_callback_fn)
-		verbose(env, " async_cb");
-	verbose(env, "\n");
-	if (!print_all)
-		mark_verifier_state_clean(env);
-}
-
-static inline u32 vlog_alignment(u32 pos)
-{
-	return round_up(max(pos + BPF_LOG_MIN_ALIGNMENT / 2, BPF_LOG_ALIGNMENT),
-			BPF_LOG_MIN_ALIGNMENT) - pos - 1;
-}
-
-static void print_insn_state(struct bpf_verifier_env *env,
-			     const struct bpf_func_state *state)
-{
-	if (env->prev_log_pos && env->prev_log_pos == env->log.end_pos) {
-		/* remove new line character */
-		bpf_vlog_reset(&env->log, env->prev_log_pos - 1);
-		verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_pos), ' ');
-	} else {
-		verbose(env, "%d:", env->insn_idx);
-	}
-	print_verifier_state(env, state, false);
-}
-
 /* copy array src of length n * size bytes to dst. dst is reallocated if it's too
  * small to hold src. This is different from krealloc since we don't want to preserve
  * the contents of dst.
@@ -1808,6 +1359,7 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state,
 	dst_state->first_insn_idx = src->first_insn_idx;
 	dst_state->last_insn_idx = src->last_insn_idx;
 	dst_state->dfs_depth = src->dfs_depth;
+	dst_state->callback_unroll_depth = src->callback_unroll_depth;
 	dst_state->used_as_loop_entry = src->used_as_loop_entry;
 	for (i = 0; i <= src->curframe; i++) {
 		dst = dst_state->frame[i];
@@ -2329,69 +1881,214 @@ static void __update_reg_bounds(struct bpf_reg_state *reg)
 /* Uses signed min/max values to inform unsigned, and vice-versa */
 static void __reg32_deduce_bounds(struct bpf_reg_state *reg)
 {
-	/* Learn sign from signed bounds.
-	 * If we cannot cross the sign boundary, then signed and unsigned bounds
-	 * are the same, so combine.  This works even in the negative case, e.g.
-	 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff.
+	/* If upper 32 bits of u64/s64 range don't change, we can use lower 32
+	 * bits to improve our u32/s32 boundaries.
+	 *
+	 * E.g., the case where we have upper 32 bits as zero ([10, 20] in
+	 * u64) is pretty trivial, it's obvious that in u32 we'll also have
+	 * [10, 20] range. But this property holds for any 64-bit range as
+	 * long as upper 32 bits in that entire range of values stay the same.
+	 *
+	 * E.g., u64 range [0x10000000A, 0x10000000F] ([4294967306, 4294967311]
+	 * in decimal) has the same upper 32 bits throughout all the values in
+	 * that range. As such, lower 32 bits form a valid [0xA, 0xF] ([10, 15])
+	 * range.
+	 *
+	 * Note also, that [0xA, 0xF] is a valid range both in u32 and in s32,
+	 * following the rules outlined below about u64/s64 correspondence
+	 * (which equally applies to u32 vs s32 correspondence). In general it
+	 * depends on actual hexadecimal values of 32-bit range. They can form
+	 * only valid u32, or only valid s32 ranges in some cases.
+	 *
+	 * So we use all these insights to derive bounds for subregisters here.
 	 */
-	if (reg->s32_min_value >= 0 || reg->s32_max_value < 0) {
-		reg->s32_min_value = reg->u32_min_value =
-			max_t(u32, reg->s32_min_value, reg->u32_min_value);
-		reg->s32_max_value = reg->u32_max_value =
-			min_t(u32, reg->s32_max_value, reg->u32_max_value);
-		return;
+	if ((reg->umin_value >> 32) == (reg->umax_value >> 32)) {
+		/* u64 to u32 casting preserves validity of low 32 bits as
+		 * a range, if upper 32 bits are the same
+		 */
+		reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)reg->umin_value);
+		reg->u32_max_value = min_t(u32, reg->u32_max_value, (u32)reg->umax_value);
+
+		if ((s32)reg->umin_value <= (s32)reg->umax_value) {
+			reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->umin_value);
+			reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->umax_value);
+		}
+	}
+	if ((reg->smin_value >> 32) == (reg->smax_value >> 32)) {
+		/* low 32 bits should form a proper u32 range */
+		if ((u32)reg->smin_value <= (u32)reg->smax_value) {
+			reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)reg->smin_value);
+			reg->u32_max_value = min_t(u32, reg->u32_max_value, (u32)reg->smax_value);
+		}
+		/* low 32 bits should form a proper s32 range */
+		if ((s32)reg->smin_value <= (s32)reg->smax_value) {
+			reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->smin_value);
+			reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->smax_value);
+		}
+	}
+	/* Special case where upper bits form a small sequence of two
+	 * sequential numbers (in 32-bit unsigned space, so 0xffffffff to
+	 * 0x00000000 is also valid), while lower bits form a proper s32 range
+	 * going from negative numbers to positive numbers. E.g., let's say we
+	 * have s64 range [-1, 1] ([0xffffffffffffffff, 0x0000000000000001]).
+	 * Possible s64 values are {-1, 0, 1} ({0xffffffffffffffff,
+	 * 0x0000000000000000, 0x00000000000001}). Ignoring upper 32 bits,
+	 * we still get a valid s32 range [-1, 1] ([0xffffffff, 0x00000001]).
+	 * Note that it doesn't have to be 0xffffffff going to 0x00000000 in
+	 * upper 32 bits. As a random example, s64 range
+	 * [0xfffffff0fffffff0; 0xfffffff100000010], forms a valid s32 range
+	 * [-16, 16] ([0xfffffff0; 0x00000010]) in its 32 bit subregister.
+	 */
+	if ((u32)(reg->umin_value >> 32) + 1 == (u32)(reg->umax_value >> 32) &&
+	    (s32)reg->umin_value < 0 && (s32)reg->umax_value >= 0) {
+		reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->umin_value);
+		reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->umax_value);
+	}
+	if ((u32)(reg->smin_value >> 32) + 1 == (u32)(reg->smax_value >> 32) &&
+	    (s32)reg->smin_value < 0 && (s32)reg->smax_value >= 0) {
+		reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->smin_value);
+		reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->smax_value);
+	}
+	/* if u32 range forms a valid s32 range (due to matching sign bit),
+	 * try to learn from that
+	 */
+	if ((s32)reg->u32_min_value <= (s32)reg->u32_max_value) {
+		reg->s32_min_value = max_t(s32, reg->s32_min_value, reg->u32_min_value);
+		reg->s32_max_value = min_t(s32, reg->s32_max_value, reg->u32_max_value);
 	}
-	/* Learn sign from unsigned bounds.  Signed bounds cross the sign
-	 * boundary, so we must be careful.
+	/* If we cannot cross the sign boundary, then signed and unsigned bounds
+	 * are the same, so combine.  This works even in the negative case, e.g.
+	 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff.
 	 */
-	if ((s32)reg->u32_max_value >= 0) {
-		/* Positive.  We can't learn anything from the smin, but smax
-		 * is positive, hence safe.
-		 */
-		reg->s32_min_value = reg->u32_min_value;
-		reg->s32_max_value = reg->u32_max_value =
-			min_t(u32, reg->s32_max_value, reg->u32_max_value);
-	} else if ((s32)reg->u32_min_value < 0) {
-		/* Negative.  We can't learn anything from the smax, but smin
-		 * is negative, hence safe.
-		 */
-		reg->s32_min_value = reg->u32_min_value =
-			max_t(u32, reg->s32_min_value, reg->u32_min_value);
-		reg->s32_max_value = reg->u32_max_value;
+	if ((u32)reg->s32_min_value <= (u32)reg->s32_max_value) {
+		reg->u32_min_value = max_t(u32, reg->s32_min_value, reg->u32_min_value);
+		reg->u32_max_value = min_t(u32, reg->s32_max_value, reg->u32_max_value);
 	}
 }
 
 static void __reg64_deduce_bounds(struct bpf_reg_state *reg)
 {
-	/* Learn sign from signed bounds.
-	 * If we cannot cross the sign boundary, then signed and unsigned bounds
+	/* If u64 range forms a valid s64 range (due to matching sign bit),
+	 * try to learn from that. Let's do a bit of ASCII art to see when
+	 * this is happening. Let's take u64 range first:
+	 *
+	 * 0             0x7fffffffffffffff 0x8000000000000000        U64_MAX
+	 * |-------------------------------|--------------------------------|
+	 *
+	 * Valid u64 range is formed when umin and umax are anywhere in the
+	 * range [0, U64_MAX], and umin <= umax. u64 case is simple and
+	 * straightforward. Let's see how s64 range maps onto the same range
+	 * of values, annotated below the line for comparison:
+	 *
+	 * 0             0x7fffffffffffffff 0x8000000000000000        U64_MAX
+	 * |-------------------------------|--------------------------------|
+	 * 0                        S64_MAX S64_MIN                        -1
+	 *
+	 * So s64 values basically start in the middle and they are logically
+	 * contiguous to the right of it, wrapping around from -1 to 0, and
+	 * then finishing as S64_MAX (0x7fffffffffffffff) right before
+	 * S64_MIN. We can try drawing the continuity of u64 vs s64 values
+	 * more visually as mapped to sign-agnostic range of hex values.
+	 *
+	 *  u64 start                                               u64 end
+	 *  _______________________________________________________________
+	 * /                                                               \
+	 * 0             0x7fffffffffffffff 0x8000000000000000        U64_MAX
+	 * |-------------------------------|--------------------------------|
+	 * 0                        S64_MAX S64_MIN                        -1
+	 *                                / \
+	 * >------------------------------   ------------------------------->
+	 * s64 continues...        s64 end   s64 start          s64 "midpoint"
+	 *
+	 * What this means is that, in general, we can't always derive
+	 * something new about u64 from any random s64 range, and vice versa.
+	 *
+	 * But we can do that in two particular cases. One is when entire
+	 * u64/s64 range is *entirely* contained within left half of the above
+	 * diagram or when it is *entirely* contained in the right half. I.e.:
+	 *
+	 * |-------------------------------|--------------------------------|
+	 *     ^                   ^            ^                 ^
+	 *     A                   B            C                 D
+	 *
+	 * [A, B] and [C, D] are contained entirely in their respective halves
+	 * and form valid contiguous ranges as both u64 and s64 values. [A, B]
+	 * will be non-negative both as u64 and s64 (and in fact it will be
+	 * identical ranges no matter the signedness). [C, D] treated as s64
+	 * will be a range of negative values, while in u64 it will be
+	 * non-negative range of values larger than 0x8000000000000000.
+	 *
+	 * Now, any other range here can't be represented in both u64 and s64
+	 * simultaneously. E.g., [A, C], [A, D], [B, C], [B, D] are valid
+	 * contiguous u64 ranges, but they are discontinuous in s64. [B, C]
+	 * in s64 would be properly presented as [S64_MIN, C] and [B, S64_MAX],
+	 * for example. Similarly, valid s64 range [D, A] (going from negative
+	 * to positive values), would be two separate [D, U64_MAX] and [0, A]
+	 * ranges as u64. Currently reg_state can't represent two segments per
+	 * numeric domain, so in such situations we can only derive maximal
+	 * possible range ([0, U64_MAX] for u64, and [S64_MIN, S64_MAX] for s64).
+	 *
+	 * So we use these facts to derive umin/umax from smin/smax and vice
+	 * versa only if they stay within the same "half". This is equivalent
+	 * to checking sign bit: lower half will have sign bit as zero, upper
+	 * half have sign bit 1. Below in code we simplify this by just
+	 * casting umin/umax as smin/smax and checking if they form valid
+	 * range, and vice versa. Those are equivalent checks.
+	 */
+	if ((s64)reg->umin_value <= (s64)reg->umax_value) {
+		reg->smin_value = max_t(s64, reg->smin_value, reg->umin_value);
+		reg->smax_value = min_t(s64, reg->smax_value, reg->umax_value);
+	}
+	/* If we cannot cross the sign boundary, then signed and unsigned bounds
 	 * are the same, so combine.  This works even in the negative case, e.g.
 	 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff.
 	 */
-	if (reg->smin_value >= 0 || reg->smax_value < 0) {
-		reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value,
-							  reg->umin_value);
-		reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value,
-							  reg->umax_value);
-		return;
+	if ((u64)reg->smin_value <= (u64)reg->smax_value) {
+		reg->umin_value = max_t(u64, reg->smin_value, reg->umin_value);
+		reg->umax_value = min_t(u64, reg->smax_value, reg->umax_value);
 	}
-	/* Learn sign from unsigned bounds.  Signed bounds cross the sign
-	 * boundary, so we must be careful.
+}
+
+static void __reg_deduce_mixed_bounds(struct bpf_reg_state *reg)
+{
+	/* Try to tighten 64-bit bounds from 32-bit knowledge, using 32-bit
+	 * values on both sides of 64-bit range in hope to have tigher range.
+	 * E.g., if r1 is [0x1'00000000, 0x3'80000000], and we learn from
+	 * 32-bit signed > 0 operation that s32 bounds are now [1; 0x7fffffff].
+	 * With this, we can substitute 1 as low 32-bits of _low_ 64-bit bound
+	 * (0x100000000 -> 0x100000001) and 0x7fffffff as low 32-bits of
+	 * _high_ 64-bit bound (0x380000000 -> 0x37fffffff) and arrive at a
+	 * better overall bounds for r1 as [0x1'000000001; 0x3'7fffffff].
+	 * We just need to make sure that derived bounds we are intersecting
+	 * with are well-formed ranges in respecitve s64 or u64 domain, just
+	 * like we do with similar kinds of 32-to-64 or 64-to-32 adjustments.
 	 */
-	if ((s64)reg->umax_value >= 0) {
-		/* Positive.  We can't learn anything from the smin, but smax
-		 * is positive, hence safe.
-		 */
-		reg->smin_value = reg->umin_value;
-		reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value,
-							  reg->umax_value);
-	} else if ((s64)reg->umin_value < 0) {
-		/* Negative.  We can't learn anything from the smax, but smin
-		 * is negative, hence safe.
-		 */
-		reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value,
-							  reg->umin_value);
-		reg->smax_value = reg->umax_value;
+	__u64 new_umin, new_umax;
+	__s64 new_smin, new_smax;
+
+	/* u32 -> u64 tightening, it's always well-formed */
+	new_umin = (reg->umin_value & ~0xffffffffULL) | reg->u32_min_value;
+	new_umax = (reg->umax_value & ~0xffffffffULL) | reg->u32_max_value;
+	reg->umin_value = max_t(u64, reg->umin_value, new_umin);
+	reg->umax_value = min_t(u64, reg->umax_value, new_umax);
+	/* u32 -> s64 tightening, u32 range embedded into s64 preserves range validity */
+	new_smin = (reg->smin_value & ~0xffffffffULL) | reg->u32_min_value;
+	new_smax = (reg->smax_value & ~0xffffffffULL) | reg->u32_max_value;
+	reg->smin_value = max_t(s64, reg->smin_value, new_smin);
+	reg->smax_value = min_t(s64, reg->smax_value, new_smax);
+
+	/* if s32 can be treated as valid u32 range, we can use it as well */
+	if ((u32)reg->s32_min_value <= (u32)reg->s32_max_value) {
+		/* s32 -> u64 tightening */
+		new_umin = (reg->umin_value & ~0xffffffffULL) | (u32)reg->s32_min_value;
+		new_umax = (reg->umax_value & ~0xffffffffULL) | (u32)reg->s32_max_value;
+		reg->umin_value = max_t(u64, reg->umin_value, new_umin);
+		reg->umax_value = min_t(u64, reg->umax_value, new_umax);
+		/* s32 -> s64 tightening */
+		new_smin = (reg->smin_value & ~0xffffffffULL) | (u32)reg->s32_min_value;
+		new_smax = (reg->smax_value & ~0xffffffffULL) | (u32)reg->s32_max_value;
+		reg->smin_value = max_t(s64, reg->smin_value, new_smin);
+		reg->smax_value = min_t(s64, reg->smax_value, new_smax);
 	}
 }
 
@@ -2399,6 +2096,7 @@ static void __reg_deduce_bounds(struct bpf_reg_state *reg)
 {
 	__reg32_deduce_bounds(reg);
 	__reg64_deduce_bounds(reg);
+	__reg_deduce_mixed_bounds(reg);
 }
 
 /* Attempts to improve var_off based on unsigned min/max information */
@@ -2420,6 +2118,7 @@ static void reg_bounds_sync(struct bpf_reg_state *reg)
 	__update_reg_bounds(reg);
 	/* We might have learned something about the sign bit. */
 	__reg_deduce_bounds(reg);
+	__reg_deduce_bounds(reg);
 	/* We might have learned some bits from the bounds. */
 	__reg_bound_offset(reg);
 	/* Intersecting with the old var_off might have improved our bounds
@@ -2429,6 +2128,56 @@ static void reg_bounds_sync(struct bpf_reg_state *reg)
 	__update_reg_bounds(reg);
 }
 
+static int reg_bounds_sanity_check(struct bpf_verifier_env *env,
+				   struct bpf_reg_state *reg, const char *ctx)
+{
+	const char *msg;
+
+	if (reg->umin_value > reg->umax_value ||
+	    reg->smin_value > reg->smax_value ||
+	    reg->u32_min_value > reg->u32_max_value ||
+	    reg->s32_min_value > reg->s32_max_value) {
+		    msg = "range bounds violation";
+		    goto out;
+	}
+
+	if (tnum_is_const(reg->var_off)) {
+		u64 uval = reg->var_off.value;
+		s64 sval = (s64)uval;
+
+		if (reg->umin_value != uval || reg->umax_value != uval ||
+		    reg->smin_value != sval || reg->smax_value != sval) {
+			msg = "const tnum out of sync with range bounds";
+			goto out;
+		}
+	}
+
+	if (tnum_subreg_is_const(reg->var_off)) {
+		u32 uval32 = tnum_subreg(reg->var_off).value;
+		s32 sval32 = (s32)uval32;
+
+		if (reg->u32_min_value != uval32 || reg->u32_max_value != uval32 ||
+		    reg->s32_min_value != sval32 || reg->s32_max_value != sval32) {
+			msg = "const subreg tnum out of sync with range bounds";
+			goto out;
+		}
+	}
+
+	return 0;
+out:
+	verbose(env, "REG INVARIANTS VIOLATION (%s): %s u64=[%#llx, %#llx] "
+		"s64=[%#llx, %#llx] u32=[%#x, %#x] s32=[%#x, %#x] var_off=(%#llx, %#llx)\n",
+		ctx, msg, reg->umin_value, reg->umax_value,
+		reg->smin_value, reg->smax_value,
+		reg->u32_min_value, reg->u32_max_value,
+		reg->s32_min_value, reg->s32_max_value,
+		reg->var_off.value, reg->var_off.mask);
+	if (env->test_reg_invariants)
+		return -EFAULT;
+	__mark_reg_unbounded(reg);
+	return 0;
+}
+
 static bool __reg32_bound_s64(s32 a)
 {
 	return a >= 0 && a <= S32_MAX;
@@ -2453,51 +2202,6 @@ static void __reg_assign_32_into_64(struct bpf_reg_state *reg)
 	}
 }
 
-static void __reg_combine_32_into_64(struct bpf_reg_state *reg)
-{
-	/* special case when 64-bit register has upper 32-bit register
-	 * zeroed. Typically happens after zext or <<32, >>32 sequence
-	 * allowing us to use 32-bit bounds directly,
-	 */
-	if (tnum_equals_const(tnum_clear_subreg(reg->var_off), 0)) {
-		__reg_assign_32_into_64(reg);
-	} else {
-		/* Otherwise the best we can do is push lower 32bit known and
-		 * unknown bits into register (var_off set from jmp logic)
-		 * then learn as much as possible from the 64-bit tnum
-		 * known and unknown bits. The previous smin/smax bounds are
-		 * invalid here because of jmp32 compare so mark them unknown
-		 * so they do not impact tnum bounds calculation.
-		 */
-		__mark_reg64_unbounded(reg);
-	}
-	reg_bounds_sync(reg);
-}
-
-static bool __reg64_bound_s32(s64 a)
-{
-	return a >= S32_MIN && a <= S32_MAX;
-}
-
-static bool __reg64_bound_u32(u64 a)
-{
-	return a >= U32_MIN && a <= U32_MAX;
-}
-
-static void __reg_combine_64_into_32(struct bpf_reg_state *reg)
-{
-	__mark_reg32_unbounded(reg);
-	if (__reg64_bound_s32(reg->smin_value) && __reg64_bound_s32(reg->smax_value)) {
-		reg->s32_min_value = (s32)reg->smin_value;
-		reg->s32_max_value = (s32)reg->smax_value;
-	}
-	if (__reg64_bound_u32(reg->umin_value) && __reg64_bound_u32(reg->umax_value)) {
-		reg->u32_min_value = (u32)reg->umin_value;
-		reg->u32_max_value = (u32)reg->umax_value;
-	}
-	reg_bounds_sync(reg);
-}
-
 /* Mark a register as having a completely unknown (scalar) value. */
 static void __mark_reg_unknown(const struct bpf_verifier_env *env,
 			       struct bpf_reg_state *reg)
@@ -3439,13 +3143,11 @@ static void mark_insn_zext(struct bpf_verifier_env *env,
 	reg->subreg_def = DEF_NOT_SUBREG;
 }
 
-static int check_reg_arg(struct bpf_verifier_env *env, u32 regno,
-			 enum reg_arg_type t)
+static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno,
+			   enum reg_arg_type t)
 {
-	struct bpf_verifier_state *vstate = env->cur_state;
-	struct bpf_func_state *state = vstate->frame[vstate->curframe];
 	struct bpf_insn *insn = env->prog->insnsi + env->insn_idx;
-	struct bpf_reg_state *reg, *regs = state->regs;
+	struct bpf_reg_state *reg;
 	bool rw64;
 
 	if (regno >= MAX_BPF_REG) {
@@ -3486,6 +3188,15 @@ static int check_reg_arg(struct bpf_verifier_env *env, u32 regno,
 	return 0;
 }
 
+static int check_reg_arg(struct bpf_verifier_env *env, u32 regno,
+			 enum reg_arg_type t)
+{
+	struct bpf_verifier_state *vstate = env->cur_state;
+	struct bpf_func_state *state = vstate->frame[vstate->curframe];
+
+	return __check_reg_arg(env, state->regs, regno, t);
+}
+
 static void mark_jmp_point(struct bpf_verifier_env *env, int idx)
 {
 	env->insn_aux_data[idx].jmp_point = true;
@@ -3724,6 +3435,8 @@ static void fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask)
 	}
 }
 
+static bool calls_callback(struct bpf_verifier_env *env, int insn_idx);
+
 /* For given verifier state backtrack_insn() is called from the last insn to
  * the first insn. Its purpose is to compute a bitmask of registers and
  * stack slots that needs precision in the parent verifier state.
@@ -3899,16 +3612,13 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
 					return -EFAULT;
 				return 0;
 			}
-		} else if ((bpf_helper_call(insn) &&
-			    is_callback_calling_function(insn->imm) &&
-			    !is_async_callback_calling_function(insn->imm)) ||
-			   (bpf_pseudo_kfunc_call(insn) && is_callback_calling_kfunc(insn->imm))) {
-			/* callback-calling helper or kfunc call, which means
-			 * we are exiting from subprog, but unlike the subprog
-			 * call handling above, we shouldn't propagate
-			 * precision of r1-r5 (if any requested), as they are
-			 * not actually arguments passed directly to callback
-			 * subprogs
+		} else if (is_sync_callback_calling_insn(insn) && idx != subseq_idx - 1) {
+			/* exit from callback subprog to callback-calling helper or
+			 * kfunc call. Use idx/subseq_idx check to discern it from
+			 * straight line code backtracking.
+			 * Unlike the subprog call handling above, we shouldn't
+			 * propagate precision of r1-r5 (if any requested), as they are
+			 * not actually arguments passed directly to callback subprogs
 			 */
 			if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) {
 				verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
@@ -3943,10 +3653,18 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
 		} else if (opcode == BPF_EXIT) {
 			bool r0_precise;
 
+			/* Backtracking to a nested function call, 'idx' is a part of
+			 * the inner frame 'subseq_idx' is a part of the outer frame.
+			 * In case of a regular function call, instructions giving
+			 * precision to registers R1-R5 should have been found already.
+			 * In case of a callback, it is ok to have R1-R5 marked for
+			 * backtracking, as these registers are set by the function
+			 * invoking callback.
+			 */
+			if (subseq_idx >= 0 && calls_callback(env, subseq_idx))
+				for (i = BPF_REG_1; i <= BPF_REG_5; i++)
+					bt_clear_reg(bt, i);
 			if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
-				/* if backtracing was looking for registers R1-R5
-				 * they should have been found already.
-				 */
 				verbose(env, "BUG regs %x\n", bt_reg_mask(bt));
 				WARN_ONCE(1, "verifier backtracking bug");
 				return -EFAULT;
@@ -4566,9 +4284,17 @@ static bool register_is_null(struct bpf_reg_state *reg)
 	return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0);
 }
 
-static bool register_is_const(struct bpf_reg_state *reg)
+/* check if register is a constant scalar value */
+static bool is_reg_const(struct bpf_reg_state *reg, bool subreg32)
+{
+	return reg->type == SCALAR_VALUE &&
+	       tnum_is_const(subreg32 ? tnum_subreg(reg->var_off) : reg->var_off);
+}
+
+/* assuming is_reg_const() is true, return constant value of a register */
+static u64 reg_const_value(struct bpf_reg_state *reg, bool subreg32)
 {
-	return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off);
+	return subreg32 ? tnum_subreg(reg->var_off).value : reg->var_off.value;
 }
 
 static bool __is_scalar_unbounded(struct bpf_reg_state *reg)
@@ -5425,10 +5151,23 @@ BTF_SET_END(rcu_protected_types)
 static bool rcu_protected_object(const struct btf *btf, u32 btf_id)
 {
 	if (!btf_is_kernel(btf))
-		return false;
+		return true;
 	return btf_id_set_contains(&rcu_protected_types, btf_id);
 }
 
+static struct btf_record *kptr_pointee_btf_record(struct btf_field *kptr_field)
+{
+	struct btf_struct_meta *meta;
+
+	if (btf_is_kernel(kptr_field->kptr.btf))
+		return NULL;
+
+	meta = btf_find_struct_meta(kptr_field->kptr.btf,
+				    kptr_field->kptr.btf_id);
+
+	return meta ? meta->record : NULL;
+}
+
 static bool rcu_safe_kptr(const struct btf_field *field)
 {
 	const struct btf_field_kptr *kptr = &field->kptr;
@@ -5439,12 +5178,25 @@ static bool rcu_safe_kptr(const struct btf_field *field)
 
 static u32 btf_ld_kptr_type(struct bpf_verifier_env *env, struct btf_field *kptr_field)
 {
+	struct btf_record *rec;
+	u32 ret;
+
+	ret = PTR_MAYBE_NULL;
 	if (rcu_safe_kptr(kptr_field) && in_rcu_cs(env)) {
-		if (kptr_field->type != BPF_KPTR_PERCPU)
-			return PTR_MAYBE_NULL | MEM_RCU;
-		return PTR_MAYBE_NULL | MEM_RCU | MEM_PERCPU;
+		ret |= MEM_RCU;
+		if (kptr_field->type == BPF_KPTR_PERCPU)
+			ret |= MEM_PERCPU;
+		else if (!btf_is_kernel(kptr_field->kptr.btf))
+			ret |= MEM_ALLOC;
+
+		rec = kptr_pointee_btf_record(kptr_field);
+		if (rec && btf_record_has_field(rec, BPF_GRAPH_NODE))
+			ret |= NON_OWN_REF;
+	} else {
+		ret |= PTR_UNTRUSTED;
 	}
-	return PTR_MAYBE_NULL | PTR_UNTRUSTED;
+
+	return ret;
 }
 
 static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno,
@@ -6218,9 +5970,10 @@ static void coerce_reg_to_size(struct bpf_reg_state *reg, int size)
 	 * values are also truncated so we push 64-bit bounds into
 	 * 32-bit bounds. Above were truncated < 32-bits already.
 	 */
-	if (size >= 4)
-		return;
-	__reg_combine_64_into_32(reg);
+	if (size < 4) {
+		__mark_reg32_unbounded(reg);
+		reg_bounds_sync(reg);
+	}
 }
 
 static void set_sext64_default_val(struct bpf_reg_state *reg, int size)
@@ -8600,6 +8353,54 @@ static enum bpf_dynptr_type dynptr_get_type(struct bpf_verifier_env *env,
 	return state->stack[spi].spilled_ptr.dynptr.type;
 }
 
+static int check_reg_const_str(struct bpf_verifier_env *env,
+			       struct bpf_reg_state *reg, u32 regno)
+{
+	struct bpf_map *map = reg->map_ptr;
+	int err;
+	int map_off;
+	u64 map_addr;
+	char *str_ptr;
+
+	if (reg->type != PTR_TO_MAP_VALUE)
+		return -EINVAL;
+
+	if (!bpf_map_is_rdonly(map)) {
+		verbose(env, "R%d does not point to a readonly map'\n", regno);
+		return -EACCES;
+	}
+
+	if (!tnum_is_const(reg->var_off)) {
+		verbose(env, "R%d is not a constant address'\n", regno);
+		return -EACCES;
+	}
+
+	if (!map->ops->map_direct_value_addr) {
+		verbose(env, "no direct value access support for this map type\n");
+		return -EACCES;
+	}
+
+	err = check_map_access(env, regno, reg->off,
+			       map->value_size - reg->off, false,
+			       ACCESS_HELPER);
+	if (err)
+		return err;
+
+	map_off = reg->off + reg->var_off.value;
+	err = map->ops->map_direct_value_addr(map, &map_addr, map_off);
+	if (err) {
+		verbose(env, "direct value access on string failed\n");
+		return err;
+	}
+
+	str_ptr = (char *)(long)(map_addr);
+	if (!strnchr(str_ptr + map_off, map->value_size - map_off, 0)) {
+		verbose(env, "string is not zero-terminated\n");
+		return -EINVAL;
+	}
+	return 0;
+}
+
 static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
 			  struct bpf_call_arg_meta *meta,
 			  const struct bpf_func_proto *fn,
@@ -8844,44 +8645,9 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
 	}
 	case ARG_PTR_TO_CONST_STR:
 	{
-		struct bpf_map *map = reg->map_ptr;
-		int map_off;
-		u64 map_addr;
-		char *str_ptr;
-
-		if (!bpf_map_is_rdonly(map)) {
-			verbose(env, "R%d does not point to a readonly map'\n", regno);
-			return -EACCES;
-		}
-
-		if (!tnum_is_const(reg->var_off)) {
-			verbose(env, "R%d is not a constant address'\n", regno);
-			return -EACCES;
-		}
-
-		if (!map->ops->map_direct_value_addr) {
-			verbose(env, "no direct value access support for this map type\n");
-			return -EACCES;
-		}
-
-		err = check_map_access(env, regno, reg->off,
-				       map->value_size - reg->off, false,
-				       ACCESS_HELPER);
+		err = check_reg_const_str(env, reg, regno);
 		if (err)
 			return err;
-
-		map_off = reg->off + reg->var_off.value;
-		err = map->ops->map_direct_value_addr(map, &map_addr, map_off);
-		if (err) {
-			verbose(env, "direct value access on string failed\n");
-			return err;
-		}
-
-		str_ptr = (char *)(long)(map_addr);
-		if (!strnchr(str_ptr + map_off, map->value_size - map_off, 0)) {
-			verbose(env, "string is not zero-terminated\n");
-			return -EINVAL;
-		}
 		break;
 	}
 	case ARG_PTR_TO_KPTR:
@@ -9350,7 +9116,7 @@ static void clear_caller_saved_regs(struct bpf_verifier_env *env,
 	/* after the call registers r0 - r5 were scratched */
 	for (i = 0; i < CALLER_SAVED_REGS; i++) {
 		mark_reg_not_init(env, regs, caller_saved[i]);
-		check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
+		__check_reg_arg(env, regs, caller_saved[i], DST_OP_NO_MARK);
 	}
 }
 
@@ -9363,11 +9129,10 @@ static int set_callee_state(struct bpf_verifier_env *env,
 			    struct bpf_func_state *caller,
 			    struct bpf_func_state *callee, int insn_idx);
 
-static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
-			     int *insn_idx, int subprog,
-			     set_callee_state_fn set_callee_state_cb)
+static int setup_func_entry(struct bpf_verifier_env *env, int subprog, int callsite,
+			    set_callee_state_fn set_callee_state_cb,
+			    struct bpf_verifier_state *state)
 {
-	struct bpf_verifier_state *state = env->cur_state;
 	struct bpf_func_state *caller, *callee;
 	int err;
 
@@ -9377,54 +9142,72 @@ static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 		return -E2BIG;
 	}
 
-	caller = state->frame[state->curframe];
 	if (state->frame[state->curframe + 1]) {
 		verbose(env, "verifier bug. Frame %d already allocated\n",
 			state->curframe + 1);
 		return -EFAULT;
 	}
 
+	caller = state->frame[state->curframe];
+	callee = kzalloc(sizeof(*callee), GFP_KERNEL);
+	if (!callee)
+		return -ENOMEM;
+	state->frame[state->curframe + 1] = callee;
+
+	/* callee cannot access r0, r6 - r9 for reading and has to write
+	 * into its own stack before reading from it.
+	 * callee can read/write into caller's stack
+	 */
+	init_func_state(env, callee,
+			/* remember the callsite, it will be used by bpf_exit */
+			callsite,
+			state->curframe + 1 /* frameno within this callchain */,
+			subprog /* subprog number within this prog */);
+	/* Transfer references to the callee */
+	err = copy_reference_state(callee, caller);
+	err = err ?: set_callee_state_cb(env, caller, callee, callsite);
+	if (err)
+		goto err_out;
+
+	/* only increment it after check_reg_arg() finished */
+	state->curframe++;
+
+	return 0;
+
+err_out:
+	free_func_state(callee);
+	state->frame[state->curframe + 1] = NULL;
+	return err;
+}
+
+static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
+			      int insn_idx, int subprog,
+			      set_callee_state_fn set_callee_state_cb)
+{
+	struct bpf_verifier_state *state = env->cur_state, *callback_state;
+	struct bpf_func_state *caller, *callee;
+	int err;
+
+	caller = state->frame[state->curframe];
 	err = btf_check_subprog_call(env, subprog, caller->regs);
 	if (err == -EFAULT)
 		return err;
-	if (subprog_is_global(env, subprog)) {
-		if (err) {
-			verbose(env, "Caller passes invalid args into func#%d\n",
-				subprog);
-			return err;
-		} else {
-			if (env->log.level & BPF_LOG_LEVEL)
-				verbose(env,
-					"Func#%d is global and valid. Skipping.\n",
-					subprog);
-			clear_caller_saved_regs(env, caller->regs);
-
-			/* All global functions return a 64-bit SCALAR_VALUE */
-			mark_reg_unknown(env, caller->regs, BPF_REG_0);
-			caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
-
-			/* continue with next insn after call */
-			return 0;
-		}
-	}
 
 	/* set_callee_state is used for direct subprog calls, but we are
 	 * interested in validating only BPF helpers that can call subprogs as
 	 * callbacks
 	 */
-	if (set_callee_state_cb != set_callee_state) {
-		env->subprog_info[subprog].is_cb = true;
-		if (bpf_pseudo_kfunc_call(insn) &&
-		    !is_callback_calling_kfunc(insn->imm)) {
-			verbose(env, "verifier bug: kfunc %s#%d not marked as callback-calling\n",
-				func_id_name(insn->imm), insn->imm);
-			return -EFAULT;
-		} else if (!bpf_pseudo_kfunc_call(insn) &&
-			   !is_callback_calling_function(insn->imm)) { /* helper */
-			verbose(env, "verifier bug: helper %s#%d not marked as callback-calling\n",
-				func_id_name(insn->imm), insn->imm);
-			return -EFAULT;
-		}
+	env->subprog_info[subprog].is_cb = true;
+	if (bpf_pseudo_kfunc_call(insn) &&
+	    !is_sync_callback_calling_kfunc(insn->imm)) {
+		verbose(env, "verifier bug: kfunc %s#%d not marked as callback-calling\n",
+			func_id_name(insn->imm), insn->imm);
+		return -EFAULT;
+	} else if (!bpf_pseudo_kfunc_call(insn) &&
+		   !is_callback_calling_function(insn->imm)) { /* helper */
+		verbose(env, "verifier bug: helper %s#%d not marked as callback-calling\n",
+			func_id_name(insn->imm), insn->imm);
+		return -EFAULT;
 	}
 
 	if (insn->code == (BPF_JMP | BPF_CALL) &&
@@ -9435,53 +9218,83 @@ static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 		/* there is no real recursion here. timer callbacks are async */
 		env->subprog_info[subprog].is_async_cb = true;
 		async_cb = push_async_cb(env, env->subprog_info[subprog].start,
-					 *insn_idx, subprog);
+					 insn_idx, subprog);
 		if (!async_cb)
 			return -EFAULT;
 		callee = async_cb->frame[0];
 		callee->async_entry_cnt = caller->async_entry_cnt + 1;
 
 		/* Convert bpf_timer_set_callback() args into timer callback args */
-		err = set_callee_state_cb(env, caller, callee, *insn_idx);
+		err = set_callee_state_cb(env, caller, callee, insn_idx);
 		if (err)
 			return err;
 
+		return 0;
+	}
+
+	/* for callback functions enqueue entry to callback and
+	 * proceed with next instruction within current frame.
+	 */
+	callback_state = push_stack(env, env->subprog_info[subprog].start, insn_idx, false);
+	if (!callback_state)
+		return -ENOMEM;
+
+	err = setup_func_entry(env, subprog, insn_idx, set_callee_state_cb,
+			       callback_state);
+	if (err)
+		return err;
+
+	callback_state->callback_unroll_depth++;
+	callback_state->frame[callback_state->curframe - 1]->callback_depth++;
+	caller->callback_depth = 0;
+	return 0;
+}
+
+static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
+			   int *insn_idx)
+{
+	struct bpf_verifier_state *state = env->cur_state;
+	struct bpf_func_state *caller;
+	int err, subprog, target_insn;
+
+	target_insn = *insn_idx + insn->imm + 1;
+	subprog = find_subprog(env, target_insn);
+	if (subprog < 0) {
+		verbose(env, "verifier bug. No program starts at insn %d\n", target_insn);
+		return -EFAULT;
+	}
+
+	caller = state->frame[state->curframe];
+	err = btf_check_subprog_call(env, subprog, caller->regs);
+	if (err == -EFAULT)
+		return err;
+	if (subprog_is_global(env, subprog)) {
+		if (err) {
+			verbose(env, "Caller passes invalid args into func#%d\n", subprog);
+			return err;
+		}
+
+		if (env->log.level & BPF_LOG_LEVEL)
+			verbose(env, "Func#%d is global and valid. Skipping.\n", subprog);
 		clear_caller_saved_regs(env, caller->regs);
+
+		/* All global functions return a 64-bit SCALAR_VALUE */
 		mark_reg_unknown(env, caller->regs, BPF_REG_0);
 		caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
+
 		/* continue with next insn after call */
 		return 0;
 	}
 
-	callee = kzalloc(sizeof(*callee), GFP_KERNEL);
-	if (!callee)
-		return -ENOMEM;
-	state->frame[state->curframe + 1] = callee;
-
-	/* callee cannot access r0, r6 - r9 for reading and has to write
-	 * into its own stack before reading from it.
-	 * callee can read/write into caller's stack
+	/* for regular function entry setup new frame and continue
+	 * from that frame.
 	 */
-	init_func_state(env, callee,
-			/* remember the callsite, it will be used by bpf_exit */
-			*insn_idx /* callsite */,
-			state->curframe + 1 /* frameno within this callchain */,
-			subprog /* subprog number within this prog */);
-
-	/* Transfer references to the callee */
-	err = copy_reference_state(callee, caller);
-	if (err)
-		goto err_out;
-
-	err = set_callee_state_cb(env, caller, callee, *insn_idx);
+	err = setup_func_entry(env, subprog, *insn_idx, set_callee_state, state);
 	if (err)
-		goto err_out;
+		return err;
 
 	clear_caller_saved_regs(env, caller->regs);
 
-	/* only increment it after check_reg_arg() finished */
-	state->curframe++;
-
 	/* and go analyze first insn of the callee */
 	*insn_idx = env->subprog_info[subprog].start - 1;
 
@@ -9489,14 +9302,10 @@ static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 		verbose(env, "caller:\n");
 		print_verifier_state(env, caller, true);
 		verbose(env, "callee:\n");
-		print_verifier_state(env, callee, true);
+		print_verifier_state(env, state->frame[state->curframe], true);
 	}
-	return 0;
 
-err_out:
-	free_func_state(callee);
-	state->frame[state->curframe + 1] = NULL;
-	return err;
+	return 0;
 }
 
 int map_set_for_each_callback_args(struct bpf_verifier_env *env,
@@ -9540,22 +9349,6 @@ static int set_callee_state(struct bpf_verifier_env *env,
 	return 0;
 }
 
-static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
-			   int *insn_idx)
-{
-	int subprog, target_insn;
-
-	target_insn = *insn_idx + insn->imm + 1;
-	subprog = find_subprog(env, target_insn);
-	if (subprog < 0) {
-		verbose(env, "verifier bug. No program starts at insn %d\n",
-			target_insn);
-		return -EFAULT;
-	}
-
-	return __check_func_call(env, insn, insn_idx, subprog, set_callee_state);
-}
-
 static int set_map_elem_callback_state(struct bpf_verifier_env *env,
 				       struct bpf_func_state *caller,
 				       struct bpf_func_state *callee,
@@ -9748,9 +9541,10 @@ static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env)
 
 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
 {
-	struct bpf_verifier_state *state = env->cur_state;
+	struct bpf_verifier_state *state = env->cur_state, *prev_st;
 	struct bpf_func_state *caller, *callee;
 	struct bpf_reg_state *r0;
+	bool in_callback_fn;
 	int err;
 
 	callee = state->frame[state->curframe];
@@ -9779,6 +9573,11 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
 			verbose_invalid_scalar(env, r0, &range, "callback return", "R0");
 			return -EINVAL;
 		}
+		if (!calls_callback(env, callee->callsite)) {
+			verbose(env, "BUG: in callback at %d, callsite %d !calls_callback\n",
+				*insn_idx, callee->callsite);
+			return -EFAULT;
+		}
 	} else {
 		/* return to the caller whatever r0 had in the callee */
 		caller->regs[BPF_REG_0] = *r0;
@@ -9796,7 +9595,16 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
 			return err;
 	}
 
-	*insn_idx = callee->callsite + 1;
+	/* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite,
+	 * there function call logic would reschedule callback visit. If iteration
+	 * converges is_state_visited() would prune that visit eventually.
+	 */
+	in_callback_fn = callee->in_callback_fn;
+	if (in_callback_fn)
+		*insn_idx = callee->callsite;
+	else
+		*insn_idx = callee->callsite + 1;
+
 	if (env->log.level & BPF_LOG_LEVEL) {
 		verbose(env, "returning from callee:\n");
 		print_verifier_state(env, callee, true);
@@ -9807,17 +9615,36 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
 	 * bpf_throw, this will be done by copy_verifier_state for extra frames. */
 	free_func_state(callee);
 	state->frame[state->curframe--] = NULL;
+
+	/* for callbacks widen imprecise scalars to make programs like below verify:
+	 *
+	 *   struct ctx { int i; }
+	 *   void cb(int idx, struct ctx *ctx) { ctx->i++; ... }
+	 *   ...
+	 *   struct ctx = { .i = 0; }
+	 *   bpf_loop(100, cb, &ctx, 0);
+	 *
+	 * This is similar to what is done in process_iter_next_call() for open
+	 * coded iterators.
+	 */
+	prev_st = in_callback_fn ? find_prev_entry(env, state, *insn_idx) : NULL;
+	if (prev_st) {
+		err = widen_imprecise_scalars(env, prev_st, state);
+		if (err)
+			return err;
+	}
 	return 0;
 }
 
-static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type,
-				   int func_id,
-				   struct bpf_call_arg_meta *meta)
+static int do_refine_retval_range(struct bpf_verifier_env *env,
+				  struct bpf_reg_state *regs, int ret_type,
+				  int func_id,
+				  struct bpf_call_arg_meta *meta)
 {
 	struct bpf_reg_state *ret_reg = &regs[BPF_REG_0];
 
 	if (ret_type != RET_INTEGER)
-		return;
+		return 0;
 
 	switch (func_id) {
 	case BPF_FUNC_get_stack:
@@ -9843,6 +9670,8 @@ static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type,
 		reg_bounds_sync(ret_reg);
 		break;
 	}
+
+	return reg_bounds_sanity_check(env, ret_reg, "retval");
 }
 
 static int
@@ -9912,7 +9741,7 @@ record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
 	val = reg->var_off.value;
 	max = map->max_entries;
 
-	if (!(register_is_const(reg) && val < max)) {
+	if (!(is_reg_const(reg, false) && val < max)) {
 		bpf_map_key_store(aux, BPF_MAP_KEY_POISON);
 		return 0;
 	}
@@ -10209,24 +10038,37 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 		}
 		break;
 	case BPF_FUNC_for_each_map_elem:
-		err = __check_func_call(env, insn, insn_idx_p, meta.subprogno,
-					set_map_elem_callback_state);
+		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
+					 set_map_elem_callback_state);
 		break;
 	case BPF_FUNC_timer_set_callback:
-		err = __check_func_call(env, insn, insn_idx_p, meta.subprogno,
-					set_timer_callback_state);
+		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
+					 set_timer_callback_state);
 		break;
 	case BPF_FUNC_find_vma:
-		err = __check_func_call(env, insn, insn_idx_p, meta.subprogno,
-					set_find_vma_callback_state);
+		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
+					 set_find_vma_callback_state);
 		break;
 	case BPF_FUNC_snprintf:
 		err = check_bpf_snprintf_call(env, regs);
 		break;
 	case BPF_FUNC_loop:
 		update_loop_inline_state(env, meta.subprogno);
-		err = __check_func_call(env, insn, insn_idx_p, meta.subprogno,
-					set_loop_callback_state);
+		/* Verifier relies on R1 value to determine if bpf_loop() iteration
+		 * is finished, thus mark it precise.
+		 */
+		err = mark_chain_precision(env, BPF_REG_1);
+		if (err)
+			return err;
+		if (cur_func(env)->callback_depth < regs[BPF_REG_1].umax_value) {
+			err = push_callback_call(env, insn, insn_idx, meta.subprogno,
+						 set_loop_callback_state);
+		} else {
+			cur_func(env)->callback_depth = 0;
+			if (env->log.level & BPF_LOG_LEVEL2)
+				verbose(env, "frame%d bpf_loop iteration limit reached\n",
+					env->cur_state->curframe);
+		}
 		break;
 	case BPF_FUNC_dynptr_from_mem:
 		if (regs[BPF_REG_1].type != PTR_TO_MAP_VALUE) {
@@ -10322,8 +10164,8 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 		break;
 	}
 	case BPF_FUNC_user_ringbuf_drain:
-		err = __check_func_call(env, insn, insn_idx_p, meta.subprogno,
-					set_user_ringbuf_callback_state);
+		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
+					 set_user_ringbuf_callback_state);
 		break;
 	}
 
@@ -10494,7 +10336,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 		regs[BPF_REG_0].ref_obj_id = id;
 	}
 
-	do_refine_retval_range(regs, fn->ret_type, func_id, &meta);
+	err = do_refine_retval_range(env, regs, fn->ret_type, func_id, &meta);
+	if (err)
+		return err;
 
 	err = check_map_func_compatibility(env, meta.map_ptr, func_id);
 	if (err)
@@ -10672,6 +10516,11 @@ static bool is_kfunc_arg_nullable(const struct btf *btf, const struct btf_param
 	return __kfunc_param_match_suffix(btf, arg, "__nullable");
 }
 
+static bool is_kfunc_arg_const_str(const struct btf *btf, const struct btf_param *arg)
+{
+	return __kfunc_param_match_suffix(btf, arg, "__str");
+}
+
 static bool is_kfunc_arg_scalar_with_name(const struct btf *btf,
 					  const struct btf_param *arg,
 					  const char *name)
@@ -10815,6 +10664,7 @@ enum kfunc_ptr_arg_type {
 	KF_ARG_PTR_TO_RB_ROOT,
 	KF_ARG_PTR_TO_RB_NODE,
 	KF_ARG_PTR_TO_NULL,
+	KF_ARG_PTR_TO_CONST_STR,
 };
 
 enum special_kfunc_type {
@@ -10965,6 +10815,9 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env,
 	if (is_kfunc_arg_rbtree_node(meta->btf, &args[argno]))
 		return KF_ARG_PTR_TO_RB_NODE;
 
+	if (is_kfunc_arg_const_str(meta->btf, &args[argno]))
+		return KF_ARG_PTR_TO_CONST_STR;
+
 	if ((base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)])) {
 		if (!btf_type_is_struct(ref_t)) {
 			verbose(env, "kernel function %s args#%d pointer type %s %s is not supported\n",
@@ -11211,7 +11064,7 @@ static bool is_bpf_graph_api_kfunc(u32 btf_id)
 	       btf_id == special_kfunc_list[KF_bpf_refcount_acquire_impl];
 }
 
-static bool is_callback_calling_kfunc(u32 btf_id)
+static bool is_sync_callback_calling_kfunc(u32 btf_id)
 {
 	return btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl];
 }
@@ -11596,6 +11449,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 		case KF_ARG_PTR_TO_MEM_SIZE:
 		case KF_ARG_PTR_TO_CALLBACK:
 		case KF_ARG_PTR_TO_REFCOUNTED_KPTR:
+		case KF_ARG_PTR_TO_CONST_STR:
 			/* Trusted by default */
 			break;
 		default:
@@ -11867,6 +11721,15 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 			meta->arg_btf = reg->btf;
 			meta->arg_btf_id = reg->btf_id;
 			break;
+		case KF_ARG_PTR_TO_CONST_STR:
+			if (reg->type != PTR_TO_MAP_VALUE) {
+				verbose(env, "arg#%d doesn't point to a const string\n", i);
+				return -EINVAL;
+			}
+			ret = check_reg_const_str(env, reg, regno);
+			if (ret)
+				return ret;
+			break;
 		}
 	}
 
@@ -11963,6 +11826,21 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		return -EACCES;
 	}
 
+	/* Check the arguments */
+	err = check_kfunc_args(env, &meta, insn_idx);
+	if (err < 0)
+		return err;
+
+	if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
+		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
+					 set_rbtree_add_callback_state);
+		if (err) {
+			verbose(env, "kfunc %s#%d failed callback verification\n",
+				func_name, meta.func_id);
+			return err;
+		}
+	}
+
 	rcu_lock = is_kfunc_bpf_rcu_read_lock(&meta);
 	rcu_unlock = is_kfunc_bpf_rcu_read_unlock(&meta);
 
@@ -11998,10 +11876,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		return -EINVAL;
 	}
 
-	/* Check the arguments */
-	err = check_kfunc_args(env, &meta, insn_idx);
-	if (err < 0)
-		return err;
 	/* In case of release function, we get register number of refcounted
 	 * PTR_TO_BTF_ID in bpf_kfunc_arg_meta, do the release now.
 	 */
@@ -12035,16 +11909,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		}
 	}
 
-	if (meta.func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) {
-		err = __check_func_call(env, insn, insn_idx_p, meta.subprogno,
-					set_rbtree_add_callback_state);
-		if (err) {
-			verbose(env, "kfunc %s#%d failed callback verification\n",
-				func_name, meta.func_id);
-			return err;
-		}
-	}
-
 	if (meta.func_id == special_kfunc_list[KF_bpf_throw]) {
 		if (!bpf_jit_supports_exceptions()) {
 			verbose(env, "JIT does not support calling kfunc %s#%d\n",
@@ -13986,13 +13850,12 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
 
 		/* check dest operand */
 		err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
+		err = err ?: adjust_reg_min_max_vals(env, insn);
 		if (err)
 			return err;
-
-		return adjust_reg_min_max_vals(env, insn);
 	}
 
-	return 0;
+	return reg_bounds_sanity_check(env, &regs[insn->dst_reg], "alu");
 }
 
 static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
@@ -14074,161 +13937,130 @@ static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
 	}));
 }
 
-static int is_branch32_taken(struct bpf_reg_state *reg, u32 val, u8 opcode)
-{
-	struct tnum subreg = tnum_subreg(reg->var_off);
-	s32 sval = (s32)val;
-
-	switch (opcode) {
-	case BPF_JEQ:
-		if (tnum_is_const(subreg))
-			return !!tnum_equals_const(subreg, val);
-		else if (val < reg->u32_min_value || val > reg->u32_max_value)
-			return 0;
-		else if (sval < reg->s32_min_value || sval > reg->s32_max_value)
-			return 0;
-		break;
-	case BPF_JNE:
-		if (tnum_is_const(subreg))
-			return !tnum_equals_const(subreg, val);
-		else if (val < reg->u32_min_value || val > reg->u32_max_value)
-			return 1;
-		else if (sval < reg->s32_min_value || sval > reg->s32_max_value)
-			return 1;
-		break;
-	case BPF_JSET:
-		if ((~subreg.mask & subreg.value) & val)
-			return 1;
-		if (!((subreg.mask | subreg.value) & val))
-			return 0;
-		break;
-	case BPF_JGT:
-		if (reg->u32_min_value > val)
-			return 1;
-		else if (reg->u32_max_value <= val)
-			return 0;
-		break;
-	case BPF_JSGT:
-		if (reg->s32_min_value > sval)
-			return 1;
-		else if (reg->s32_max_value <= sval)
-			return 0;
-		break;
-	case BPF_JLT:
-		if (reg->u32_max_value < val)
-			return 1;
-		else if (reg->u32_min_value >= val)
-			return 0;
-		break;
-	case BPF_JSLT:
-		if (reg->s32_max_value < sval)
-			return 1;
-		else if (reg->s32_min_value >= sval)
-			return 0;
-		break;
-	case BPF_JGE:
-		if (reg->u32_min_value >= val)
-			return 1;
-		else if (reg->u32_max_value < val)
-			return 0;
-		break;
-	case BPF_JSGE:
-		if (reg->s32_min_value >= sval)
-			return 1;
-		else if (reg->s32_max_value < sval)
-			return 0;
-		break;
-	case BPF_JLE:
-		if (reg->u32_max_value <= val)
-			return 1;
-		else if (reg->u32_min_value > val)
-			return 0;
-		break;
-	case BPF_JSLE:
-		if (reg->s32_max_value <= sval)
-			return 1;
-		else if (reg->s32_min_value > sval)
-			return 0;
-		break;
-	}
-
-	return -1;
-}
-
-
-static int is_branch64_taken(struct bpf_reg_state *reg, u64 val, u8 opcode)
-{
-	s64 sval = (s64)val;
+/*
+ * <reg1> <op> <reg2>, currently assuming reg2 is a constant
+ */
+static int is_scalar_branch_taken(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2,
+				  u8 opcode, bool is_jmp32)
+{
+	struct tnum t1 = is_jmp32 ? tnum_subreg(reg1->var_off) : reg1->var_off;
+	struct tnum t2 = is_jmp32 ? tnum_subreg(reg2->var_off) : reg2->var_off;
+	u64 umin1 = is_jmp32 ? (u64)reg1->u32_min_value : reg1->umin_value;
+	u64 umax1 = is_jmp32 ? (u64)reg1->u32_max_value : reg1->umax_value;
+	s64 smin1 = is_jmp32 ? (s64)reg1->s32_min_value : reg1->smin_value;
+	s64 smax1 = is_jmp32 ? (s64)reg1->s32_max_value : reg1->smax_value;
+	u64 umin2 = is_jmp32 ? (u64)reg2->u32_min_value : reg2->umin_value;
+	u64 umax2 = is_jmp32 ? (u64)reg2->u32_max_value : reg2->umax_value;
+	s64 smin2 = is_jmp32 ? (s64)reg2->s32_min_value : reg2->smin_value;
+	s64 smax2 = is_jmp32 ? (s64)reg2->s32_max_value : reg2->smax_value;
 
 	switch (opcode) {
 	case BPF_JEQ:
-		if (tnum_is_const(reg->var_off))
-			return !!tnum_equals_const(reg->var_off, val);
-		else if (val < reg->umin_value || val > reg->umax_value)
+		/* constants, umin/umax and smin/smax checks would be
+		 * redundant in this case because they all should match
+		 */
+		if (tnum_is_const(t1) && tnum_is_const(t2))
+			return t1.value == t2.value;
+		/* non-overlapping ranges */
+		if (umin1 > umax2 || umax1 < umin2)
 			return 0;
-		else if (sval < reg->smin_value || sval > reg->smax_value)
+		if (smin1 > smax2 || smax1 < smin2)
 			return 0;
+		if (!is_jmp32) {
+			/* if 64-bit ranges are inconclusive, see if we can
+			 * utilize 32-bit subrange knowledge to eliminate
+			 * branches that can't be taken a priori
+			 */
+			if (reg1->u32_min_value > reg2->u32_max_value ||
+			    reg1->u32_max_value < reg2->u32_min_value)
+				return 0;
+			if (reg1->s32_min_value > reg2->s32_max_value ||
+			    reg1->s32_max_value < reg2->s32_min_value)
+				return 0;
+		}
 		break;
 	case BPF_JNE:
-		if (tnum_is_const(reg->var_off))
-			return !tnum_equals_const(reg->var_off, val);
-		else if (val < reg->umin_value || val > reg->umax_value)
+		/* constants, umin/umax and smin/smax checks would be
+		 * redundant in this case because they all should match
+		 */
+		if (tnum_is_const(t1) && tnum_is_const(t2))
+			return t1.value != t2.value;
+		/* non-overlapping ranges */
+		if (umin1 > umax2 || umax1 < umin2)
 			return 1;
-		else if (sval < reg->smin_value || sval > reg->smax_value)
+		if (smin1 > smax2 || smax1 < smin2)
 			return 1;
+		if (!is_jmp32) {
+			/* if 64-bit ranges are inconclusive, see if we can
+			 * utilize 32-bit subrange knowledge to eliminate
+			 * branches that can't be taken a priori
+			 */
+			if (reg1->u32_min_value > reg2->u32_max_value ||
+			    reg1->u32_max_value < reg2->u32_min_value)
+				return 1;
+			if (reg1->s32_min_value > reg2->s32_max_value ||
+			    reg1->s32_max_value < reg2->s32_min_value)
+				return 1;
+		}
 		break;
 	case BPF_JSET:
-		if ((~reg->var_off.mask & reg->var_off.value) & val)
+		if (!is_reg_const(reg2, is_jmp32)) {
+			swap(reg1, reg2);
+			swap(t1, t2);
+		}
+		if (!is_reg_const(reg2, is_jmp32))
+			return -1;
+		if ((~t1.mask & t1.value) & t2.value)
 			return 1;
-		if (!((reg->var_off.mask | reg->var_off.value) & val))
+		if (!((t1.mask | t1.value) & t2.value))
 			return 0;
 		break;
 	case BPF_JGT:
-		if (reg->umin_value > val)
+		if (umin1 > umax2)
 			return 1;
-		else if (reg->umax_value <= val)
+		else if (umax1 <= umin2)
 			return 0;
 		break;
 	case BPF_JSGT:
-		if (reg->smin_value > sval)
+		if (smin1 > smax2)
 			return 1;
-		else if (reg->smax_value <= sval)
+		else if (smax1 <= smin2)
 			return 0;
 		break;
 	case BPF_JLT:
-		if (reg->umax_value < val)
+		if (umax1 < umin2)
 			return 1;
-		else if (reg->umin_value >= val)
+		else if (umin1 >= umax2)
 			return 0;
 		break;
 	case BPF_JSLT:
-		if (reg->smax_value < sval)
+		if (smax1 < smin2)
 			return 1;
-		else if (reg->smin_value >= sval)
+		else if (smin1 >= smax2)
 			return 0;
 		break;
 	case BPF_JGE:
-		if (reg->umin_value >= val)
+		if (umin1 >= umax2)
 			return 1;
-		else if (reg->umax_value < val)
+		else if (umax1 < umin2)
 			return 0;
 		break;
 	case BPF_JSGE:
-		if (reg->smin_value >= sval)
+		if (smin1 >= smax2)
 			return 1;
-		else if (reg->smax_value < sval)
+		else if (smax1 < smin2)
 			return 0;
 		break;
 	case BPF_JLE:
-		if (reg->umax_value <= val)
+		if (umax1 <= umin2)
 			return 1;
-		else if (reg->umin_value > val)
+		else if (umin1 > umax2)
 			return 0;
 		break;
 	case BPF_JSLE:
-		if (reg->smax_value <= sval)
+		if (smax1 <= smin2)
 			return 1;
-		else if (reg->smin_value > sval)
+		else if (smin1 > smax2)
 			return 0;
 		break;
 	}
@@ -14236,41 +14068,6 @@ static int is_branch64_taken(struct bpf_reg_state *reg, u64 val, u8 opcode)
 	return -1;
 }
 
-/* compute branch direction of the expression "if (reg opcode val) goto target;"
- * and return:
- *  1 - branch will be taken and "goto target" will be executed
- *  0 - branch will not be taken and fall-through to next insn
- * -1 - unknown. Example: "if (reg < 5)" is unknown when register value
- *      range [0,10]
- */
-static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode,
-			   bool is_jmp32)
-{
-	if (__is_pointer_value(false, reg)) {
-		if (!reg_not_null(reg))
-			return -1;
-
-		/* If pointer is valid tests against zero will fail so we can
-		 * use this to direct branch taken.
-		 */
-		if (val != 0)
-			return -1;
-
-		switch (opcode) {
-		case BPF_JEQ:
-			return 0;
-		case BPF_JNE:
-			return 1;
-		default:
-			return -1;
-		}
-	}
-
-	if (is_jmp32)
-		return is_branch32_taken(reg, val, opcode);
-	return is_branch64_taken(reg, val, opcode);
-}
-
 static int flip_opcode(u32 opcode)
 {
 	/* How can we transform "a <op> b" into "b <op> a"? */
@@ -14332,216 +14129,244 @@ static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg,
 	return -1;
 }
 
-/* Adjusts the register min/max values in the case that the dst_reg is the
- * variable register that we are working on, and src_reg is a constant or we're
- * simply doing a BPF_K check.
- * In JEQ/JNE cases we also adjust the var_off values.
+/* compute branch direction of the expression "if (<reg1> opcode <reg2>) goto target;"
+ * and return:
+ *  1 - branch will be taken and "goto target" will be executed
+ *  0 - branch will not be taken and fall-through to next insn
+ * -1 - unknown. Example: "if (reg1 < 5)" is unknown when register value
+ *      range [0,10]
  */
-static void reg_set_min_max(struct bpf_reg_state *true_reg,
-			    struct bpf_reg_state *false_reg,
-			    u64 val, u32 val32,
-			    u8 opcode, bool is_jmp32)
-{
-	struct tnum false_32off = tnum_subreg(false_reg->var_off);
-	struct tnum false_64off = false_reg->var_off;
-	struct tnum true_32off = tnum_subreg(true_reg->var_off);
-	struct tnum true_64off = true_reg->var_off;
-	s64 sval = (s64)val;
-	s32 sval32 = (s32)val32;
-
-	/* If the dst_reg is a pointer, we can't learn anything about its
-	 * variable offset from the compare (unless src_reg were a pointer into
-	 * the same object, but we don't bother with that.
-	 * Since false_reg and true_reg have the same type by construction, we
-	 * only need to check one of them for pointerness.
-	 */
-	if (__is_pointer_value(false, false_reg))
-		return;
+static int is_branch_taken(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2,
+			   u8 opcode, bool is_jmp32)
+{
+	if (reg_is_pkt_pointer_any(reg1) && reg_is_pkt_pointer_any(reg2) && !is_jmp32)
+		return is_pkt_ptr_branch_taken(reg1, reg2, opcode);
+
+	if (__is_pointer_value(false, reg1) || __is_pointer_value(false, reg2)) {
+		u64 val;
+
+		/* arrange that reg2 is a scalar, and reg1 is a pointer */
+		if (!is_reg_const(reg2, is_jmp32)) {
+			opcode = flip_opcode(opcode);
+			swap(reg1, reg2);
+		}
+		/* and ensure that reg2 is a constant */
+		if (!is_reg_const(reg2, is_jmp32))
+			return -1;
+
+		if (!reg_not_null(reg1))
+			return -1;
+
+		/* If pointer is valid tests against zero will fail so we can
+		 * use this to direct branch taken.
+		 */
+		val = reg_const_value(reg2, is_jmp32);
+		if (val != 0)
+			return -1;
+
+		switch (opcode) {
+		case BPF_JEQ:
+			return 0;
+		case BPF_JNE:
+			return 1;
+		default:
+			return -1;
+		}
+	}
+
+	/* now deal with two scalars, but not necessarily constants */
+	return is_scalar_branch_taken(reg1, reg2, opcode, is_jmp32);
+}
 
+/* Opcode that corresponds to a *false* branch condition.
+ * E.g., if r1 < r2, then reverse (false) condition is r1 >= r2
+ */
+static u8 rev_opcode(u8 opcode)
+{
 	switch (opcode) {
-	/* JEQ/JNE comparison doesn't change the register equivalence.
-	 *
-	 * r1 = r2;
-	 * if (r1 == 42) goto label;
-	 * ...
-	 * label: // here both r1 and r2 are known to be 42.
-	 *
-	 * Hence when marking register as known preserve it's ID.
+	case BPF_JEQ:		return BPF_JNE;
+	case BPF_JNE:		return BPF_JEQ;
+	/* JSET doesn't have it's reverse opcode in BPF, so add
+	 * BPF_X flag to denote the reverse of that operation
 	 */
+	case BPF_JSET:		return BPF_JSET | BPF_X;
+	case BPF_JSET | BPF_X:	return BPF_JSET;
+	case BPF_JGE:		return BPF_JLT;
+	case BPF_JGT:		return BPF_JLE;
+	case BPF_JLE:		return BPF_JGT;
+	case BPF_JLT:		return BPF_JGE;
+	case BPF_JSGE:		return BPF_JSLT;
+	case BPF_JSGT:		return BPF_JSLE;
+	case BPF_JSLE:		return BPF_JSGT;
+	case BPF_JSLT:		return BPF_JSGE;
+	default:		return 0;
+	}
+}
+
+/* Refine range knowledge for <reg1> <op> <reg>2 conditional operation. */
+static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state *reg2,
+				u8 opcode, bool is_jmp32)
+{
+	struct tnum t;
+	u64 val;
+
+again:
+	switch (opcode) {
 	case BPF_JEQ:
 		if (is_jmp32) {
-			__mark_reg32_known(true_reg, val32);
-			true_32off = tnum_subreg(true_reg->var_off);
+			reg1->u32_min_value = max(reg1->u32_min_value, reg2->u32_min_value);
+			reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value);
+			reg1->s32_min_value = max(reg1->s32_min_value, reg2->s32_min_value);
+			reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value);
+			reg2->u32_min_value = reg1->u32_min_value;
+			reg2->u32_max_value = reg1->u32_max_value;
+			reg2->s32_min_value = reg1->s32_min_value;
+			reg2->s32_max_value = reg1->s32_max_value;
+
+			t = tnum_intersect(tnum_subreg(reg1->var_off), tnum_subreg(reg2->var_off));
+			reg1->var_off = tnum_with_subreg(reg1->var_off, t);
+			reg2->var_off = tnum_with_subreg(reg2->var_off, t);
 		} else {
-			___mark_reg_known(true_reg, val);
-			true_64off = true_reg->var_off;
+			reg1->umin_value = max(reg1->umin_value, reg2->umin_value);
+			reg1->umax_value = min(reg1->umax_value, reg2->umax_value);
+			reg1->smin_value = max(reg1->smin_value, reg2->smin_value);
+			reg1->smax_value = min(reg1->smax_value, reg2->smax_value);
+			reg2->umin_value = reg1->umin_value;
+			reg2->umax_value = reg1->umax_value;
+			reg2->smin_value = reg1->smin_value;
+			reg2->smax_value = reg1->smax_value;
+
+			reg1->var_off = tnum_intersect(reg1->var_off, reg2->var_off);
+			reg2->var_off = reg1->var_off;
 		}
 		break;
 	case BPF_JNE:
-		if (is_jmp32) {
-			__mark_reg32_known(false_reg, val32);
-			false_32off = tnum_subreg(false_reg->var_off);
-		} else {
-			___mark_reg_known(false_reg, val);
-			false_64off = false_reg->var_off;
-		}
+		/* we don't derive any new information for inequality yet */
 		break;
 	case BPF_JSET:
+		if (!is_reg_const(reg2, is_jmp32))
+			swap(reg1, reg2);
+		if (!is_reg_const(reg2, is_jmp32))
+			break;
+		val = reg_const_value(reg2, is_jmp32);
+		/* BPF_JSET (i.e., TRUE branch, *not* BPF_JSET | BPF_X)
+		 * requires single bit to learn something useful. E.g., if we
+		 * know that `r1 & 0x3` is true, then which bits (0, 1, or both)
+		 * are actually set? We can learn something definite only if
+		 * it's a single-bit value to begin with.
+		 *
+		 * BPF_JSET | BPF_X (i.e., negation of BPF_JSET) doesn't have
+		 * this restriction. I.e., !(r1 & 0x3) means neither bit 0 nor
+		 * bit 1 is set, which we can readily use in adjustments.
+		 */
+		if (!is_power_of_2(val))
+			break;
 		if (is_jmp32) {
-			false_32off = tnum_and(false_32off, tnum_const(~val32));
-			if (is_power_of_2(val32))
-				true_32off = tnum_or(true_32off,
-						     tnum_const(val32));
+			t = tnum_or(tnum_subreg(reg1->var_off), tnum_const(val));
+			reg1->var_off = tnum_with_subreg(reg1->var_off, t);
 		} else {
-			false_64off = tnum_and(false_64off, tnum_const(~val));
-			if (is_power_of_2(val))
-				true_64off = tnum_or(true_64off,
-						     tnum_const(val));
+			reg1->var_off = tnum_or(reg1->var_off, tnum_const(val));
 		}
 		break;
-	case BPF_JGE:
-	case BPF_JGT:
-	{
+	case BPF_JSET | BPF_X: /* reverse of BPF_JSET, see rev_opcode() */
+		if (!is_reg_const(reg2, is_jmp32))
+			swap(reg1, reg2);
+		if (!is_reg_const(reg2, is_jmp32))
+			break;
+		val = reg_const_value(reg2, is_jmp32);
 		if (is_jmp32) {
-			u32 false_umax = opcode == BPF_JGT ? val32  : val32 - 1;
-			u32 true_umin = opcode == BPF_JGT ? val32 + 1 : val32;
-
-			false_reg->u32_max_value = min(false_reg->u32_max_value,
-						       false_umax);
-			true_reg->u32_min_value = max(true_reg->u32_min_value,
-						      true_umin);
+			t = tnum_and(tnum_subreg(reg1->var_off), tnum_const(~val));
+			reg1->var_off = tnum_with_subreg(reg1->var_off, t);
 		} else {
-			u64 false_umax = opcode == BPF_JGT ? val    : val - 1;
-			u64 true_umin = opcode == BPF_JGT ? val + 1 : val;
-
-			false_reg->umax_value = min(false_reg->umax_value, false_umax);
-			true_reg->umin_value = max(true_reg->umin_value, true_umin);
+			reg1->var_off = tnum_and(reg1->var_off, tnum_const(~val));
 		}
 		break;
-	}
-	case BPF_JSGE:
-	case BPF_JSGT:
-	{
+	case BPF_JLE:
 		if (is_jmp32) {
-			s32 false_smax = opcode == BPF_JSGT ? sval32    : sval32 - 1;
-			s32 true_smin = opcode == BPF_JSGT ? sval32 + 1 : sval32;
-
-			false_reg->s32_max_value = min(false_reg->s32_max_value, false_smax);
-			true_reg->s32_min_value = max(true_reg->s32_min_value, true_smin);
+			reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value);
+			reg2->u32_min_value = max(reg1->u32_min_value, reg2->u32_min_value);
 		} else {
-			s64 false_smax = opcode == BPF_JSGT ? sval    : sval - 1;
-			s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval;
-
-			false_reg->smax_value = min(false_reg->smax_value, false_smax);
-			true_reg->smin_value = max(true_reg->smin_value, true_smin);
+			reg1->umax_value = min(reg1->umax_value, reg2->umax_value);
+			reg2->umin_value = max(reg1->umin_value, reg2->umin_value);
 		}
 		break;
-	}
-	case BPF_JLE:
 	case BPF_JLT:
-	{
 		if (is_jmp32) {
-			u32 false_umin = opcode == BPF_JLT ? val32  : val32 + 1;
-			u32 true_umax = opcode == BPF_JLT ? val32 - 1 : val32;
-
-			false_reg->u32_min_value = max(false_reg->u32_min_value,
-						       false_umin);
-			true_reg->u32_max_value = min(true_reg->u32_max_value,
-						      true_umax);
+			reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value - 1);
+			reg2->u32_min_value = max(reg1->u32_min_value + 1, reg2->u32_min_value);
 		} else {
-			u64 false_umin = opcode == BPF_JLT ? val    : val + 1;
-			u64 true_umax = opcode == BPF_JLT ? val - 1 : val;
-
-			false_reg->umin_value = max(false_reg->umin_value, false_umin);
-			true_reg->umax_value = min(true_reg->umax_value, true_umax);
+			reg1->umax_value = min(reg1->umax_value, reg2->umax_value - 1);
+			reg2->umin_value = max(reg1->umin_value + 1, reg2->umin_value);
 		}
 		break;
-	}
 	case BPF_JSLE:
+		if (is_jmp32) {
+			reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value);
+			reg2->s32_min_value = max(reg1->s32_min_value, reg2->s32_min_value);
+		} else {
+			reg1->smax_value = min(reg1->smax_value, reg2->smax_value);
+			reg2->smin_value = max(reg1->smin_value, reg2->smin_value);
+		}
+		break;
 	case BPF_JSLT:
-	{
 		if (is_jmp32) {
-			s32 false_smin = opcode == BPF_JSLT ? sval32    : sval32 + 1;
-			s32 true_smax = opcode == BPF_JSLT ? sval32 - 1 : sval32;
-
-			false_reg->s32_min_value = max(false_reg->s32_min_value, false_smin);
-			true_reg->s32_max_value = min(true_reg->s32_max_value, true_smax);
+			reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value - 1);
+			reg2->s32_min_value = max(reg1->s32_min_value + 1, reg2->s32_min_value);
 		} else {
-			s64 false_smin = opcode == BPF_JSLT ? sval    : sval + 1;
-			s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval;
-
-			false_reg->smin_value = max(false_reg->smin_value, false_smin);
-			true_reg->smax_value = min(true_reg->smax_value, true_smax);
+			reg1->smax_value = min(reg1->smax_value, reg2->smax_value - 1);
+			reg2->smin_value = max(reg1->smin_value + 1, reg2->smin_value);
 		}
 		break;
-	}
+	case BPF_JGE:
+	case BPF_JGT:
+	case BPF_JSGE:
+	case BPF_JSGT:
+		/* just reuse LE/LT logic above */
+		opcode = flip_opcode(opcode);
+		swap(reg1, reg2);
+		goto again;
 	default:
 		return;
 	}
-
-	if (is_jmp32) {
-		false_reg->var_off = tnum_or(tnum_clear_subreg(false_64off),
-					     tnum_subreg(false_32off));
-		true_reg->var_off = tnum_or(tnum_clear_subreg(true_64off),
-					    tnum_subreg(true_32off));
-		__reg_combine_32_into_64(false_reg);
-		__reg_combine_32_into_64(true_reg);
-	} else {
-		false_reg->var_off = false_64off;
-		true_reg->var_off = true_64off;
-		__reg_combine_64_into_32(false_reg);
-		__reg_combine_64_into_32(true_reg);
-	}
 }
 
-/* Same as above, but for the case that dst_reg holds a constant and src_reg is
- * the variable reg.
+/* Adjusts the register min/max values in the case that the dst_reg and
+ * src_reg are both SCALAR_VALUE registers (or we are simply doing a BPF_K
+ * check, in which case we havea fake SCALAR_VALUE representing insn->imm).
+ * Technically we can do similar adjustments for pointers to the same object,
+ * but we don't support that right now.
  */
-static void reg_set_min_max_inv(struct bpf_reg_state *true_reg,
-				struct bpf_reg_state *false_reg,
-				u64 val, u32 val32,
-				u8 opcode, bool is_jmp32)
+static int reg_set_min_max(struct bpf_verifier_env *env,
+			   struct bpf_reg_state *true_reg1,
+			   struct bpf_reg_state *true_reg2,
+			   struct bpf_reg_state *false_reg1,
+			   struct bpf_reg_state *false_reg2,
+			   u8 opcode, bool is_jmp32)
 {
-	opcode = flip_opcode(opcode);
-	/* This uses zero as "not present in table"; luckily the zero opcode,
-	 * BPF_JA, can't get here.
+	int err;
+
+	/* If either register is a pointer, we can't learn anything about its
+	 * variable offset from the compare (unless they were a pointer into
+	 * the same object, but we don't bother with that).
 	 */
-	if (opcode)
-		reg_set_min_max(true_reg, false_reg, val, val32, opcode, is_jmp32);
-}
-
-/* Regs are known to be equal, so intersect their min/max/var_off */
-static void __reg_combine_min_max(struct bpf_reg_state *src_reg,
-				  struct bpf_reg_state *dst_reg)
-{
-	src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value,
-							dst_reg->umin_value);
-	src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value,
-							dst_reg->umax_value);
-	src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value,
-							dst_reg->smin_value);
-	src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value,
-							dst_reg->smax_value);
-	src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off,
-							     dst_reg->var_off);
-	reg_bounds_sync(src_reg);
-	reg_bounds_sync(dst_reg);
-}
+	if (false_reg1->type != SCALAR_VALUE || false_reg2->type != SCALAR_VALUE)
+		return 0;
 
-static void reg_combine_min_max(struct bpf_reg_state *true_src,
-				struct bpf_reg_state *true_dst,
-				struct bpf_reg_state *false_src,
-				struct bpf_reg_state *false_dst,
-				u8 opcode)
-{
-	switch (opcode) {
-	case BPF_JEQ:
-		__reg_combine_min_max(true_src, true_dst);
-		break;
-	case BPF_JNE:
-		__reg_combine_min_max(false_src, false_dst);
-		break;
-	}
+	/* fallthrough (FALSE) branch */
+	regs_refine_cond_op(false_reg1, false_reg2, rev_opcode(opcode), is_jmp32);
+	reg_bounds_sync(false_reg1);
+	reg_bounds_sync(false_reg2);
+
+	/* jump (TRUE) branch */
+	regs_refine_cond_op(true_reg1, true_reg2, opcode, is_jmp32);
+	reg_bounds_sync(true_reg1);
+	reg_bounds_sync(true_reg2);
+
+	err = reg_bounds_sanity_check(env, true_reg1, "true_reg1");
+	err = err ?: reg_bounds_sanity_check(env, true_reg2, "true_reg2");
+	err = err ?: reg_bounds_sanity_check(env, false_reg1, "false_reg1");
+	err = err ?: reg_bounds_sanity_check(env, false_reg2, "false_reg2");
+	return err;
 }
 
 static void mark_ptr_or_null_reg(struct bpf_func_state *state,
@@ -14739,6 +14564,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
 	struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs;
 	struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL;
 	struct bpf_reg_state *eq_branch_regs;
+	struct bpf_reg_state fake_reg = {};
 	u8 opcode = BPF_OP(insn->code);
 	bool is_jmp32;
 	int pred = -1;
@@ -14779,42 +14605,13 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
 			verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
 			return -EINVAL;
 		}
+		src_reg = &fake_reg;
+		src_reg->type = SCALAR_VALUE;
+		__mark_reg_known(src_reg, insn->imm);
 	}
 
 	is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
-
-	if (BPF_SRC(insn->code) == BPF_K) {
-		pred = is_branch_taken(dst_reg, insn->imm, opcode, is_jmp32);
-	} else if (src_reg->type == SCALAR_VALUE &&
-		   is_jmp32 && tnum_is_const(tnum_subreg(src_reg->var_off))) {
-		pred = is_branch_taken(dst_reg,
-				       tnum_subreg(src_reg->var_off).value,
-				       opcode,
-				       is_jmp32);
-	} else if (src_reg->type == SCALAR_VALUE &&
-		   !is_jmp32 && tnum_is_const(src_reg->var_off)) {
-		pred = is_branch_taken(dst_reg,
-				       src_reg->var_off.value,
-				       opcode,
-				       is_jmp32);
-	} else if (dst_reg->type == SCALAR_VALUE &&
-		   is_jmp32 && tnum_is_const(tnum_subreg(dst_reg->var_off))) {
-		pred = is_branch_taken(src_reg,
-				       tnum_subreg(dst_reg->var_off).value,
-				       flip_opcode(opcode),
-				       is_jmp32);
-	} else if (dst_reg->type == SCALAR_VALUE &&
-		   !is_jmp32 && tnum_is_const(dst_reg->var_off)) {
-		pred = is_branch_taken(src_reg,
-				       dst_reg->var_off.value,
-				       flip_opcode(opcode),
-				       is_jmp32);
-	} else if (reg_is_pkt_pointer_any(dst_reg) &&
-		   reg_is_pkt_pointer_any(src_reg) &&
-		   !is_jmp32) {
-		pred = is_pkt_ptr_branch_taken(dst_reg, src_reg, opcode);
-	}
-
+	pred = is_branch_taken(dst_reg, src_reg, opcode, is_jmp32);
 	if (pred >= 0) {
 		/* If we get here with a dst_reg pointer type it is because
 		 * above is_branch_taken() special cased the 0 comparison.
@@ -14862,53 +14659,27 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
 		return -EFAULT;
 	other_branch_regs = other_branch->frame[other_branch->curframe]->regs;
 
-	/* detect if we are comparing against a constant value so we can adjust
-	 * our min/max values for our dst register.
-	 * this is only legit if both are scalars (or pointers to the same
-	 * object, I suppose, see the PTR_MAYBE_NULL related if block below),
-	 * because otherwise the different base pointers mean the offsets aren't
-	 * comparable.
-	 */
 	if (BPF_SRC(insn->code) == BPF_X) {
-		struct bpf_reg_state *src_reg = &regs[insn->src_reg];
-
-		if (dst_reg->type == SCALAR_VALUE &&
-		    src_reg->type == SCALAR_VALUE) {
-			if (tnum_is_const(src_reg->var_off) ||
-			    (is_jmp32 &&
-			     tnum_is_const(tnum_subreg(src_reg->var_off))))
-				reg_set_min_max(&other_branch_regs[insn->dst_reg],
-						dst_reg,
-						src_reg->var_off.value,
-						tnum_subreg(src_reg->var_off).value,
-						opcode, is_jmp32);
-			else if (tnum_is_const(dst_reg->var_off) ||
-				 (is_jmp32 &&
-				  tnum_is_const(tnum_subreg(dst_reg->var_off))))
-				reg_set_min_max_inv(&other_branch_regs[insn->src_reg],
-						    src_reg,
-						    dst_reg->var_off.value,
-						    tnum_subreg(dst_reg->var_off).value,
-						    opcode, is_jmp32);
-			else if (!is_jmp32 &&
-				 (opcode == BPF_JEQ || opcode == BPF_JNE))
-				/* Comparing for equality, we can combine knowledge */
-				reg_combine_min_max(&other_branch_regs[insn->src_reg],
-						    &other_branch_regs[insn->dst_reg],
-						    src_reg, dst_reg, opcode);
-			if (src_reg->id &&
-			    !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) {
-				find_equal_scalars(this_branch, src_reg);
-				find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]);
-			}
-
-		}
-	} else if (dst_reg->type == SCALAR_VALUE) {
-		reg_set_min_max(&other_branch_regs[insn->dst_reg],
-					dst_reg, insn->imm, (u32)insn->imm,
-					opcode, is_jmp32);
+		err = reg_set_min_max(env,
+				      &other_branch_regs[insn->dst_reg],
+				      &other_branch_regs[insn->src_reg],
+				      dst_reg, src_reg, opcode, is_jmp32);
+	} else /* BPF_SRC(insn->code) == BPF_K */ {
+		err = reg_set_min_max(env,
+				      &other_branch_regs[insn->dst_reg],
+				      src_reg /* fake one */,
+				      dst_reg, src_reg /* same fake one */,
+				      opcode, is_jmp32);
 	}
+	if (err)
+		return err;
 
+	if (BPF_SRC(insn->code) == BPF_X &&
+	    src_reg->type == SCALAR_VALUE && src_reg->id &&
+	    !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) {
+		find_equal_scalars(this_branch, src_reg);
+		find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]);
+	}
 	if (dst_reg->type == SCALAR_VALUE && dst_reg->id &&
 	    !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) {
 		find_equal_scalars(this_branch, dst_reg);
@@ -15408,6 +15179,15 @@ static bool is_force_checkpoint(struct bpf_verifier_env *env, int insn_idx)
 	return env->insn_aux_data[insn_idx].force_checkpoint;
 }
 
+static void mark_calls_callback(struct bpf_verifier_env *env, int idx)
+{
+	env->insn_aux_data[idx].calls_callback = true;
+}
+
+static bool calls_callback(struct bpf_verifier_env *env, int insn_idx)
+{
+	return env->insn_aux_data[insn_idx].calls_callback;
+}
 
 enum {
 	DONE_EXPLORING = 0,
@@ -15521,6 +15301,21 @@ static int visit_insn(int t, struct bpf_verifier_env *env)
 			 * async state will be pushed for further exploration.
 			 */
 			mark_prune_point(env, t);
+		/* For functions that invoke callbacks it is not known how many times
+		 * callback would be called. Verifier models callback calling functions
+		 * by repeatedly visiting callback bodies and returning to origin call
+		 * instruction.
+		 * In order to stop such iteration verifier needs to identify when a
+		 * state identical some state from a previous iteration is reached.
+		 * Check below forces creation of checkpoint before callback calling
+		 * instruction to allow search for such identical states.
+		 */
+		if (is_sync_callback_calling_insn(insn)) {
+			mark_calls_callback(env, t);
+			mark_force_checkpoint(env, t);
+			mark_prune_point(env, t);
+			mark_jmp_point(env, t);
+		}
 		if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
 			struct bpf_kfunc_call_arg_meta meta;
 
@@ -16990,10 +16785,16 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)
 				}
 				goto skip_inf_loop_check;
 			}
+			if (calls_callback(env, insn_idx)) {
+				if (states_equal(env, &sl->state, cur, true))
+					goto hit;
+				goto skip_inf_loop_check;
+			}
 			/* attempt to detect infinite loop to avoid unnecessary doomed work */
 			if (states_maybe_looping(&sl->state, cur) &&
 			    states_equal(env, &sl->state, cur, false) &&
-			    !iter_active_depths_differ(&sl->state, cur)) {
+			    !iter_active_depths_differ(&sl->state, cur) &&
+			    sl->state.callback_unroll_depth == cur->callback_unroll_depth) {
 				verbose_linfo(env, insn_idx, "; ");
 				verbose(env, "infinite loop detected at insn %d\n", insn_idx);
 				verbose(env, "cur state:");
@@ -17411,10 +17212,8 @@ static int do_check(struct bpf_verifier_env *env)
 					       insn->off, BPF_SIZE(insn->code),
 					       BPF_READ, insn->dst_reg, false,
 					       BPF_MODE(insn->code) == BPF_MEMSX);
-			if (err)
-				return err;
-
-			err = save_aux_ptr_type(env, src_reg_type, true);
+			err = err ?: save_aux_ptr_type(env, src_reg_type, true);
+			err = err ?: reg_bounds_sanity_check(env, &regs[insn->dst_reg], "ldx");
 			if (err)
 				return err;
 		} else if (class == BPF_STX) {
@@ -20701,6 +20500,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
 
 	if (is_priv)
 		env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ;
+	env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS;
 
 	env->explored_states = kvcalloc(state_htab_size(env),
 				       sizeof(struct bpf_verifier_state_list *),
diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h
index c56071f150f2..520b90dd97ec 100644
--- a/kernel/cgroup/cgroup-internal.h
+++ b/kernel/cgroup/cgroup-internal.h
@@ -164,13 +164,13 @@ struct cgroup_mgctx {
 #define DEFINE_CGROUP_MGCTX(name)						\
 	struct cgroup_mgctx name = CGROUP_MGCTX_INIT(name)
 
-extern spinlock_t css_set_lock;
 extern struct cgroup_subsys *cgroup_subsys[];
 extern struct list_head cgroup_roots;
 
 /* iterate across the hierarchies */
 #define for_each_root(root)						\
-	list_for_each_entry((root), &cgroup_roots, root_list)
+	list_for_each_entry_rcu((root), &cgroup_roots, root_list,	\
+				lockdep_is_held(&cgroup_mutex))
 
 /**
  * for_each_subsys - iterate all enabled cgroup subsystems
diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c
index 76db6c67e39a..04d11a7dd95f 100644
--- a/kernel/cgroup/cgroup-v1.c
+++ b/kernel/cgroup/cgroup-v1.c
@@ -1262,6 +1262,40 @@ int cgroup1_get_tree(struct fs_context *fc)
 	return ret;
 }
 
+/**
+ * task_get_cgroup1 - Acquires the associated cgroup of a task within a
+ * specific cgroup1 hierarchy. The cgroup1 hierarchy is identified by its
+ * hierarchy ID.
+ * @tsk: The target task
+ * @hierarchy_id: The ID of a cgroup1 hierarchy
+ *
+ * On success, the cgroup is returned. On failure, ERR_PTR is returned.
+ * We limit it to cgroup1 only.
+ */
+struct cgroup *task_get_cgroup1(struct task_struct *tsk, int hierarchy_id)
+{
+	struct cgroup *cgrp = ERR_PTR(-ENOENT);
+	struct cgroup_root *root;
+	unsigned long flags;
+
+	rcu_read_lock();
+	for_each_root(root) {
+		/* cgroup1 only*/
+		if (root == &cgrp_dfl_root)
+			continue;
+		if (root->hierarchy_id != hierarchy_id)
+			continue;
+		spin_lock_irqsave(&css_set_lock, flags);
+		cgrp = task_cgroup_from_root(tsk, root);
+		if (!cgrp || !cgroup_tryget(cgrp))
+			cgrp = ERR_PTR(-ENOENT);
+		spin_unlock_irqrestore(&css_set_lock, flags);
+		break;
+	}
+	rcu_read_unlock();
+	return cgrp;
+}
+
 static int __init cgroup1_wq_init(void)
 {
 	/*
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 4b9ff41ca603..8f3cef1a4d8a 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -1315,7 +1315,7 @@ static void cgroup_exit_root_id(struct cgroup_root *root)
 
 void cgroup_free_root(struct cgroup_root *root)
 {
-	kfree(root);
+	kfree_rcu(root, rcu);
 }
 
 static void cgroup_destroy_root(struct cgroup_root *root)
@@ -1347,10 +1347,9 @@ static void cgroup_destroy_root(struct cgroup_root *root)
 
 	spin_unlock_irq(&css_set_lock);
 
-	if (!list_empty(&root->root_list)) {
-		list_del(&root->root_list);
-		cgroup_root_count--;
-	}
+	WARN_ON_ONCE(list_empty(&root->root_list));
+	list_del_rcu(&root->root_list);
+	cgroup_root_count--;
 
 	if (!have_favordynmods)
 		cgroup_favor_dynmods(root, false);
@@ -1390,7 +1389,15 @@ static inline struct cgroup *__cset_cgroup_from_root(struct css_set *cset,
 		}
 	}
 
-	BUG_ON(!res_cgroup);
+	/*
+	 * If cgroup_mutex is not held, the cgrp_cset_link will be freed
+	 * before we remove the cgroup root from the root_list. Consequently,
+	 * when accessing a cgroup root, the cset_link may have already been
+	 * freed, resulting in a NULL res_cgroup. However, by holding the
+	 * cgroup_mutex, we ensure that res_cgroup can't be NULL.
+	 * If we don't hold cgroup_mutex in the caller, we must do the NULL
+	 * check.
+	 */
 	return res_cgroup;
 }
 
@@ -1413,6 +1420,11 @@ current_cgns_cgroup_from_root(struct cgroup_root *root)
 
 	rcu_read_unlock();
 
+	/*
+	 * The namespace_sem is held by current, so the root cgroup can't
+	 * be umounted. Therefore, we can ensure that the res is non-NULL.
+	 */
+	WARN_ON_ONCE(!res);
 	return res;
 }
 
@@ -1449,7 +1461,6 @@ static struct cgroup *current_cgns_cgroup_dfl(void)
 static struct cgroup *cset_cgroup_from_root(struct css_set *cset,
 					    struct cgroup_root *root)
 {
-	lockdep_assert_held(&cgroup_mutex);
 	lockdep_assert_held(&css_set_lock);
 
 	return __cset_cgroup_from_root(cset, root);
@@ -1457,7 +1468,9 @@ static struct cgroup *cset_cgroup_from_root(struct css_set *cset,
 
 /*
  * Return the cgroup for "task" from the given hierarchy. Must be
- * called with cgroup_mutex and css_set_lock held.
+ * called with css_set_lock held to prevent task's groups from being modified.
+ * Must be called with either cgroup_mutex or rcu read lock to prevent the
+ * cgroup root from being destroyed.
  */
 struct cgroup *task_cgroup_from_root(struct task_struct *task,
 				     struct cgroup_root *root)
@@ -2032,7 +2045,7 @@ void init_cgroup_root(struct cgroup_fs_context *ctx)
 	struct cgroup_root *root = ctx->root;
 	struct cgroup *cgrp = &root->cgrp;
 
-	INIT_LIST_HEAD(&root->root_list);
+	INIT_LIST_HEAD_RCU(&root->root_list);
 	atomic_set(&root->nr_cgrps, 1);
 	cgrp->root = root;
 	init_cgroup_housekeeping(cgrp);
@@ -2115,7 +2128,7 @@ int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask)
 	 * care of subsystems' refcounts, which are explicitly dropped in
 	 * the failure exit path.
 	 */
-	list_add(&root->root_list, &cgroup_roots);
+	list_add_rcu(&root->root_list, &cgroup_roots);
 	cgroup_root_count++;
 
 	/*
@@ -6265,7 +6278,7 @@ int proc_cgroup_show(struct seq_file *m, struct pid_namespace *ns,
 	if (!buf)
 		goto out;
 
-	cgroup_lock();
+	rcu_read_lock();
 	spin_lock_irq(&css_set_lock);
 
 	for_each_root(root) {
@@ -6276,6 +6289,11 @@ int proc_cgroup_show(struct seq_file *m, struct pid_namespace *ns,
 		if (root == &cgrp_dfl_root && !READ_ONCE(cgrp_dfl_visible))
 			continue;
 
+		cgrp = task_cgroup_from_root(tsk, root);
+		/* The root has already been unmounted. */
+		if (!cgrp)
+			continue;
+
 		seq_printf(m, "%d:", root->hierarchy_id);
 		if (root != &cgrp_dfl_root)
 			for_each_subsys(ss, ssid)
@@ -6286,9 +6304,6 @@ int proc_cgroup_show(struct seq_file *m, struct pid_namespace *ns,
 			seq_printf(m, "%sname=%s", count ? "," : "",
 				   root->name);
 		seq_putc(m, ':');
-
-		cgrp = task_cgroup_from_root(tsk, root);
-
 		/*
 		 * On traditional hierarchies, all zombie tasks show up as
 		 * belonging to the root cgroup.  On the default hierarchy,
@@ -6320,7 +6335,7 @@ int proc_cgroup_show(struct seq_file *m, struct pid_namespace *ns,
 	retval = 0;
 out_unlock:
 	spin_unlock_irq(&css_set_lock);
-	cgroup_unlock();
+	rcu_read_unlock();
 	kfree(buf);
 out:
 	return retval;
diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
index 615daaf87f1f..1bad4007ff4b 100644
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -25,6 +25,7 @@
 #include <linux/cpu.h>
 #include <linux/cpumask.h>
 #include <linux/cpuset.h>
+#include <linux/delay.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
@@ -43,6 +44,7 @@
 #include <linux/sched/isolation.h>
 #include <linux/cgroup.h>
 #include <linux/wait.h>
+#include <linux/workqueue.h>
 
 DEFINE_STATIC_KEY_FALSE(cpusets_pre_enable_key);
 DEFINE_STATIC_KEY_FALSE(cpusets_enabled_key);
@@ -204,6 +206,11 @@ struct cpuset {
  */
 static cpumask_var_t	subpartitions_cpus;
 
+/*
+ * Exclusive CPUs in isolated partitions
+ */
+static cpumask_var_t	isolated_cpus;
+
 /* List of remote partition root children */
 static struct list_head remote_children;
 
@@ -1317,6 +1324,7 @@ static void compute_effective_cpumask(struct cpumask *new_cpus,
  */
 enum partition_cmd {
 	partcmd_enable,		/* Enable partition root	  */
+	partcmd_enablei,	/* Enable isolated partition root */
 	partcmd_disable,	/* Disable partition root	  */
 	partcmd_update,		/* Update parent's effective_cpus */
 	partcmd_invalidate,	/* Make partition invalid	  */
@@ -1418,6 +1426,98 @@ static void reset_partition_data(struct cpuset *cs)
 	}
 }
 
+/*
+ * partition_xcpus_newstate - Exclusive CPUs state change
+ * @old_prs: old partition_root_state
+ * @new_prs: new partition_root_state
+ * @xcpus: exclusive CPUs with state change
+ */
+static void partition_xcpus_newstate(int old_prs, int new_prs, struct cpumask *xcpus)
+{
+	WARN_ON_ONCE(old_prs == new_prs);
+	if (new_prs == PRS_ISOLATED)
+		cpumask_or(isolated_cpus, isolated_cpus, xcpus);
+	else
+		cpumask_andnot(isolated_cpus, isolated_cpus, xcpus);
+}
+
+/*
+ * partition_xcpus_add - Add new exclusive CPUs to partition
+ * @new_prs: new partition_root_state
+ * @parent: parent cpuset
+ * @xcpus: exclusive CPUs to be added
+ * Return: true if isolated_cpus modified, false otherwise
+ *
+ * Remote partition if parent == NULL
+ */
+static bool partition_xcpus_add(int new_prs, struct cpuset *parent,
+				struct cpumask *xcpus)
+{
+	bool isolcpus_updated;
+
+	WARN_ON_ONCE(new_prs < 0);
+	lockdep_assert_held(&callback_lock);
+	if (!parent)
+		parent = &top_cpuset;
+
+
+	if (parent == &top_cpuset)
+		cpumask_or(subpartitions_cpus, subpartitions_cpus, xcpus);
+
+	isolcpus_updated = (new_prs != parent->partition_root_state);
+	if (isolcpus_updated)
+		partition_xcpus_newstate(parent->partition_root_state, new_prs,
+					 xcpus);
+
+	cpumask_andnot(parent->effective_cpus, parent->effective_cpus, xcpus);
+	return isolcpus_updated;
+}
+
+/*
+ * partition_xcpus_del - Remove exclusive CPUs from partition
+ * @old_prs: old partition_root_state
+ * @parent: parent cpuset
+ * @xcpus: exclusive CPUs to be removed
+ * Return: true if isolated_cpus modified, false otherwise
+ *
+ * Remote partition if parent == NULL
+ */
+static bool partition_xcpus_del(int old_prs, struct cpuset *parent,
+				struct cpumask *xcpus)
+{
+	bool isolcpus_updated;
+
+	WARN_ON_ONCE(old_prs < 0);
+	lockdep_assert_held(&callback_lock);
+	if (!parent)
+		parent = &top_cpuset;
+
+	if (parent == &top_cpuset)
+		cpumask_andnot(subpartitions_cpus, subpartitions_cpus, xcpus);
+
+	isolcpus_updated = (old_prs != parent->partition_root_state);
+	if (isolcpus_updated)
+		partition_xcpus_newstate(old_prs, parent->partition_root_state,
+					 xcpus);
+
+	cpumask_and(xcpus, xcpus, cpu_active_mask);
+	cpumask_or(parent->effective_cpus, parent->effective_cpus, xcpus);
+	return isolcpus_updated;
+}
+
+static void update_unbound_workqueue_cpumask(bool isolcpus_updated)
+{
+	int ret;
+
+	lockdep_assert_cpus_held();
+
+	if (!isolcpus_updated)
+		return;
+
+	ret = workqueue_unbound_exclude_cpumask(isolated_cpus);
+	WARN_ON_ONCE(ret < 0);
+}
+
 /*
  * compute_effective_exclusive_cpumask - compute effective exclusive CPUs
  * @cs: cpuset
@@ -1456,14 +1556,18 @@ static inline bool is_local_partition(struct cpuset *cs)
 /*
  * remote_partition_enable - Enable current cpuset as a remote partition root
  * @cs: the cpuset to update
+ * @new_prs: new partition_root_state
  * @tmp: temparary masks
  * Return: 1 if successful, 0 if error
  *
  * Enable the current cpuset to become a remote partition root taking CPUs
  * directly from the top cpuset. cpuset_mutex must be held by the caller.
  */
-static int remote_partition_enable(struct cpuset *cs, struct tmpmasks *tmp)
+static int remote_partition_enable(struct cpuset *cs, int new_prs,
+				   struct tmpmasks *tmp)
 {
+	bool isolcpus_updated;
+
 	/*
 	 * The user must have sysadmin privilege.
 	 */
@@ -1485,26 +1589,22 @@ static int remote_partition_enable(struct cpuset *cs, struct tmpmasks *tmp)
 		return 0;
 
 	spin_lock_irq(&callback_lock);
-	cpumask_andnot(top_cpuset.effective_cpus,
-		       top_cpuset.effective_cpus, tmp->new_cpus);
-	cpumask_or(subpartitions_cpus,
-		   subpartitions_cpus, tmp->new_cpus);
-
+	isolcpus_updated = partition_xcpus_add(new_prs, NULL, tmp->new_cpus);
+	list_add(&cs->remote_sibling, &remote_children);
 	if (cs->use_parent_ecpus) {
 		struct cpuset *parent = parent_cs(cs);
 
 		cs->use_parent_ecpus = false;
 		parent->child_ecpus_count--;
 	}
-	list_add(&cs->remote_sibling, &remote_children);
 	spin_unlock_irq(&callback_lock);
+	update_unbound_workqueue_cpumask(isolcpus_updated);
 
 	/*
 	 * Proprogate changes in top_cpuset's effective_cpus down the hierarchy.
 	 */
 	update_tasks_cpumask(&top_cpuset, tmp->new_cpus);
 	update_sibling_cpumasks(&top_cpuset, NULL, tmp);
-
 	return 1;
 }
 
@@ -1519,23 +1619,22 @@ static int remote_partition_enable(struct cpuset *cs, struct tmpmasks *tmp)
  */
 static void remote_partition_disable(struct cpuset *cs, struct tmpmasks *tmp)
 {
+	bool isolcpus_updated;
+
 	compute_effective_exclusive_cpumask(cs, tmp->new_cpus);
 	WARN_ON_ONCE(!is_remote_partition(cs));
 	WARN_ON_ONCE(!cpumask_subset(tmp->new_cpus, subpartitions_cpus));
 
 	spin_lock_irq(&callback_lock);
-	cpumask_andnot(subpartitions_cpus,
-		       subpartitions_cpus, tmp->new_cpus);
-	cpumask_and(tmp->new_cpus,
-		    tmp->new_cpus, cpu_active_mask);
-	cpumask_or(top_cpuset.effective_cpus,
-		   top_cpuset.effective_cpus, tmp->new_cpus);
 	list_del_init(&cs->remote_sibling);
+	isolcpus_updated = partition_xcpus_del(cs->partition_root_state,
+					       NULL, tmp->new_cpus);
 	cs->partition_root_state = -cs->partition_root_state;
 	if (!cs->prs_err)
 		cs->prs_err = PERR_INVCPUS;
 	reset_partition_data(cs);
 	spin_unlock_irq(&callback_lock);
+	update_unbound_workqueue_cpumask(isolcpus_updated);
 
 	/*
 	 * Proprogate changes in top_cpuset's effective_cpus down the hierarchy.
@@ -1557,6 +1656,8 @@ static void remote_cpus_update(struct cpuset *cs, struct cpumask *newmask,
 			       struct tmpmasks *tmp)
 {
 	bool adding, deleting;
+	int prs = cs->partition_root_state;
+	int isolcpus_updated = 0;
 
 	if (WARN_ON_ONCE(!is_remote_partition(cs)))
 		return;
@@ -1580,21 +1681,12 @@ static void remote_cpus_update(struct cpuset *cs, struct cpumask *newmask,
 		goto invalidate;
 
 	spin_lock_irq(&callback_lock);
-	if (adding) {
-		cpumask_or(subpartitions_cpus,
-			   subpartitions_cpus, tmp->addmask);
-		cpumask_andnot(top_cpuset.effective_cpus,
-			       top_cpuset.effective_cpus, tmp->addmask);
-	}
-	if (deleting) {
-		cpumask_andnot(subpartitions_cpus,
-			       subpartitions_cpus, tmp->delmask);
-		cpumask_and(tmp->delmask,
-			    tmp->delmask, cpu_active_mask);
-		cpumask_or(top_cpuset.effective_cpus,
-			   top_cpuset.effective_cpus, tmp->delmask);
-	}
+	if (adding)
+		isolcpus_updated += partition_xcpus_add(prs, NULL, tmp->addmask);
+	if (deleting)
+		isolcpus_updated += partition_xcpus_del(prs, NULL, tmp->delmask);
 	spin_unlock_irq(&callback_lock);
+	update_unbound_workqueue_cpumask(isolcpus_updated);
 
 	/*
 	 * Proprogate changes in top_cpuset's effective_cpus down the hierarchy.
@@ -1676,11 +1768,11 @@ static bool prstate_housekeeping_conflict(int prstate, struct cpumask *new_cpus)
  * @tmp:     Temporary addmask and delmask
  * Return:   0 or a partition root state error code
  *
- * For partcmd_enable, the cpuset is being transformed from a non-partition
- * root to a partition root. The effective_xcpus (cpus_allowed if effective_xcpus
- * not set) mask of the given cpuset will be taken away from parent's
- * effective_cpus. The function will return 0 if all the CPUs listed in
- * effective_xcpus can be granted or an error code will be returned.
+ * For partcmd_enable*, the cpuset is being transformed from a non-partition
+ * root to a partition root. The effective_xcpus (cpus_allowed if
+ * effective_xcpus not set) mask of the given cpuset will be taken away from
+ * parent's effective_cpus. The function will return 0 if all the CPUs listed
+ * in effective_xcpus can be granted or an error code will be returned.
  *
  * For partcmd_disable, the cpuset is being transformed from a partition
  * root back to a non-partition root. Any CPUs in effective_xcpus will be
@@ -1695,7 +1787,7 @@ static bool prstate_housekeeping_conflict(int prstate, struct cpumask *new_cpus)
  *
  * For partcmd_invalidate, the current partition will be made invalid.
  *
- * The partcmd_enable and partcmd_disable commands are used by
+ * The partcmd_enable* and partcmd_disable commands are used by
  * update_prstate(). An error code may be returned and the caller will check
  * for error.
  *
@@ -1716,6 +1808,7 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
 	int part_error = PERR_NONE;	/* Partition error? */
 	int subparts_delta = 0;
 	struct cpumask *xcpus;		/* cs effective_xcpus */
+	int isolcpus_updated = 0;
 	bool nocpu;
 
 	lockdep_assert_held(&cpuset_mutex);
@@ -1760,7 +1853,7 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
 
 	nocpu = tasks_nocpu_error(parent, cs, xcpus);
 
-	if (cmd == partcmd_enable) {
+	if ((cmd == partcmd_enable) || (cmd == partcmd_enablei)) {
 		/*
 		 * Enabling partition root is not allowed if its
 		 * effective_xcpus is empty or doesn't overlap with
@@ -1783,6 +1876,7 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
 		cpumask_copy(tmp->delmask, xcpus);
 		deleting = true;
 		subparts_delta++;
+		new_prs = (cmd == partcmd_enable) ? PRS_ROOT : PRS_ISOLATED;
 	} else if (cmd == partcmd_disable) {
 		/*
 		 * May need to add cpus to parent's effective_cpus for
@@ -1792,6 +1886,7 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
 			  cpumask_and(tmp->addmask, xcpus, parent->effective_xcpus);
 		if (adding)
 			subparts_delta--;
+		new_prs = PRS_MEMBER;
 	} else if (newmask) {
 		/*
 		 * Empty cpumask is not allowed
@@ -1940,38 +2035,28 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd,
 	 * newly deleted ones will be added back to effective_cpus.
 	 */
 	spin_lock_irq(&callback_lock);
-	if (adding) {
-		if (parent == &top_cpuset)
-			cpumask_andnot(subpartitions_cpus,
-				       subpartitions_cpus, tmp->addmask);
-		/*
-		 * Some of the CPUs in effective_xcpus might have been offlined.
-		 */
-		cpumask_or(parent->effective_cpus,
-			   parent->effective_cpus, tmp->addmask);
-		cpumask_and(parent->effective_cpus,
-			    parent->effective_cpus, cpu_active_mask);
-	}
-	if (deleting) {
-		if (parent == &top_cpuset)
-			cpumask_or(subpartitions_cpus,
-				   subpartitions_cpus, tmp->delmask);
-		cpumask_andnot(parent->effective_cpus,
-			       parent->effective_cpus, tmp->delmask);
-	}
-
-	if (is_partition_valid(parent)) {
-		parent->nr_subparts += subparts_delta;
-		WARN_ON_ONCE(parent->nr_subparts < 0);
-	}
-
 	if (old_prs != new_prs) {
 		cs->partition_root_state = new_prs;
 		if (new_prs <= 0)
 			cs->nr_subparts = 0;
 	}
+	/*
+	 * Adding to parent's effective_cpus means deletion CPUs from cs
+	 * and vice versa.
+	 */
+	if (adding)
+		isolcpus_updated += partition_xcpus_del(old_prs, parent,
+							tmp->addmask);
+	if (deleting)
+		isolcpus_updated += partition_xcpus_add(new_prs, parent,
+							tmp->delmask);
 
+	if (is_partition_valid(parent)) {
+		parent->nr_subparts += subparts_delta;
+		WARN_ON_ONCE(parent->nr_subparts < 0);
+	}
 	spin_unlock_irq(&callback_lock);
+	update_unbound_workqueue_cpumask(isolcpus_updated);
 
 	if ((old_prs != new_prs) && (cmd == partcmd_update))
 		update_partition_exclusive(cs, new_prs);
@@ -2948,6 +3033,7 @@ static int update_prstate(struct cpuset *cs, int new_prs)
 	int err = PERR_NONE, old_prs = cs->partition_root_state;
 	struct cpuset *parent = parent_cs(cs);
 	struct tmpmasks tmpmask;
+	bool new_xcpus_state = false;
 
 	if (old_prs == new_prs)
 		return 0;
@@ -2977,6 +3063,9 @@ static int update_prstate(struct cpuset *cs, int new_prs)
 		goto out;
 
 	if (!old_prs) {
+		enum partition_cmd cmd = (new_prs == PRS_ROOT)
+				       ? partcmd_enable : partcmd_enablei;
+
 		/*
 		 * cpus_allowed cannot be empty.
 		 */
@@ -2985,19 +3074,18 @@ static int update_prstate(struct cpuset *cs, int new_prs)
 			goto out;
 		}
 
-		err = update_parent_effective_cpumask(cs, partcmd_enable,
-						      NULL, &tmpmask);
+		err = update_parent_effective_cpumask(cs, cmd, NULL, &tmpmask);
 		/*
 		 * If an attempt to become local partition root fails,
 		 * try to become a remote partition root instead.
 		 */
-		if (err && remote_partition_enable(cs, &tmpmask))
+		if (err && remote_partition_enable(cs, new_prs, &tmpmask))
 			err = 0;
 	} else if (old_prs && new_prs) {
 		/*
 		 * A change in load balance state only, no change in cpumasks.
 		 */
-		;
+		new_xcpus_state = true;
 	} else {
 		/*
 		 * Switching back to member is always allowed even if it
@@ -3029,7 +3117,10 @@ static int update_prstate(struct cpuset *cs, int new_prs)
 	WRITE_ONCE(cs->prs_err, err);
 	if (!is_partition_valid(cs))
 		reset_partition_data(cs);
+	else if (new_xcpus_state)
+		partition_xcpus_newstate(old_prs, new_prs, cs->effective_xcpus);
 	spin_unlock_irq(&callback_lock);
+	update_unbound_workqueue_cpumask(new_xcpus_state);
 
 	/* Force update if switching back to member */
 	update_cpumasks_hier(cs, &tmpmask, !new_prs ? HIER_CHECKALL : 0);
@@ -3386,6 +3477,7 @@ typedef enum {
 	FILE_SUBPARTS_CPULIST,
 	FILE_EXCLUSIVE_CPULIST,
 	FILE_EFFECTIVE_XCPULIST,
+	FILE_ISOLATED_CPULIST,
 	FILE_CPU_EXCLUSIVE,
 	FILE_MEM_EXCLUSIVE,
 	FILE_MEM_HARDWALL,
@@ -3582,6 +3674,9 @@ static int cpuset_common_seq_show(struct seq_file *sf, void *v)
 	case FILE_SUBPARTS_CPULIST:
 		seq_printf(sf, "%*pbl\n", cpumask_pr_args(subpartitions_cpus));
 		break;
+	case FILE_ISOLATED_CPULIST:
+		seq_printf(sf, "%*pbl\n", cpumask_pr_args(isolated_cpus));
+		break;
 	default:
 		ret = -EINVAL;
 	}
@@ -3875,6 +3970,13 @@ static struct cftype dfl_files[] = {
 		.flags = CFTYPE_ONLY_ON_ROOT | CFTYPE_DEBUG,
 	},
 
+	{
+		.name = "cpus.isolated",
+		.seq_show = cpuset_common_seq_show,
+		.private = FILE_ISOLATED_CPULIST,
+		.flags = CFTYPE_ONLY_ON_ROOT | CFTYPE_DEBUG,
+	},
+
 	{ }	/* terminate */
 };
 
@@ -4194,6 +4296,7 @@ int __init cpuset_init(void)
 	BUG_ON(!alloc_cpumask_var(&top_cpuset.effective_xcpus, GFP_KERNEL));
 	BUG_ON(!alloc_cpumask_var(&top_cpuset.exclusive_cpus, GFP_KERNEL));
 	BUG_ON(!zalloc_cpumask_var(&subpartitions_cpus, GFP_KERNEL));
+	BUG_ON(!zalloc_cpumask_var(&isolated_cpus, GFP_KERNEL));
 
 	cpumask_setall(top_cpuset.cpus_allowed);
 	nodes_setall(top_cpuset.mems_allowed);
@@ -4306,6 +4409,30 @@ void cpuset_force_rebuild(void)
 	force_rebuild = true;
 }
 
+/*
+ * Attempt to acquire a cpus_read_lock while a hotplug operation may be in
+ * progress.
+ * Return: true if successful, false otherwise
+ *
+ * To avoid circular lock dependency between cpuset_mutex and cpus_read_lock,
+ * cpus_read_trylock() is used here to acquire the lock.
+ */
+static bool cpuset_hotplug_cpus_read_trylock(void)
+{
+	int retries = 0;
+
+	while (!cpus_read_trylock()) {
+		/*
+		 * CPU hotplug still in progress. Retry 5 times
+		 * with a 10ms wait before bailing out.
+		 */
+		if (++retries > 5)
+			return false;
+		msleep(10);
+	}
+	return true;
+}
+
 /**
  * cpuset_hotplug_update_tasks - update tasks in a cpuset for hotunplug
  * @cs: cpuset in interest
@@ -4322,6 +4449,7 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp)
 	bool cpus_updated;
 	bool mems_updated;
 	bool remote;
+	int partcmd = -1;
 	struct cpuset *parent;
 retry:
 	wait_event(cpuset_attach_wq, cs->attach_in_progress == 0);
@@ -4353,11 +4481,13 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp)
 		compute_partition_effective_cpumask(cs, &new_cpus);
 
 	if (remote && cpumask_empty(&new_cpus) &&
-	    partition_is_populated(cs, NULL)) {
+	    partition_is_populated(cs, NULL) &&
+	    cpuset_hotplug_cpus_read_trylock()) {
 		remote_partition_disable(cs, tmp);
 		compute_effective_cpumask(&new_cpus, cs, parent);
 		remote = false;
 		cpuset_force_rebuild();
+		cpus_read_unlock();
 	}
 
 	/*
@@ -4368,18 +4498,28 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp)
 	 *    partitions.
 	 */
 	if (is_local_partition(cs) && (!is_partition_valid(parent) ||
-				tasks_nocpu_error(parent, cs, &new_cpus))) {
-		update_parent_effective_cpumask(cs, partcmd_invalidate, NULL, tmp);
-		compute_effective_cpumask(&new_cpus, cs, parent);
-		cpuset_force_rebuild();
-	}
+				tasks_nocpu_error(parent, cs, &new_cpus)))
+		partcmd = partcmd_invalidate;
 	/*
 	 * On the other hand, an invalid partition root may be transitioned
 	 * back to a regular one.
 	 */
-	else if (is_partition_valid(parent) && is_partition_invalid(cs)) {
-		update_parent_effective_cpumask(cs, partcmd_update, NULL, tmp);
-		if (is_partition_valid(cs)) {
+	else if (is_partition_valid(parent) && is_partition_invalid(cs))
+		partcmd = partcmd_update;
+
+	/*
+	 * cpus_read_lock needs to be held before calling
+	 * update_parent_effective_cpumask(). To avoid circular lock
+	 * dependency between cpuset_mutex and cpus_read_lock,
+	 * cpus_read_trylock() is used here to acquire the lock.
+	 */
+	if (partcmd >= 0) {
+		if (!cpuset_hotplug_cpus_read_trylock())
+			goto update_tasks;
+
+		update_parent_effective_cpumask(cs, partcmd, NULL, tmp);
+		cpus_read_unlock();
+		if ((partcmd == partcmd_invalidate) || is_partition_valid(cs)) {
 			compute_partition_effective_cpumask(cs, &new_cpus);
 			cpuset_force_rebuild();
 		}
diff --git a/kernel/cgroup/legacy_freezer.c b/kernel/cgroup/legacy_freezer.c
index 122dacb3a443..66d1708042a7 100644
--- a/kernel/cgroup/legacy_freezer.c
+++ b/kernel/cgroup/legacy_freezer.c
@@ -66,9 +66,15 @@ static struct freezer *parent_freezer(struct freezer *freezer)
 bool cgroup_freezing(struct task_struct *task)
 {
 	bool ret;
+	unsigned int state;
 
 	rcu_read_lock();
-	ret = task_freezer(task)->state & CGROUP_FREEZING;
+	/* Check if the cgroup is still FREEZING, but not FROZEN. The extra
+	 * !FROZEN check is required, because the FREEZING bit is not cleared
+	 * when the state FROZEN is reached.
+	 */
+	state = task_freezer(task)->state;
+	ret = (state & CGROUP_FREEZING) && !(state & CGROUP_FROZEN);
 	rcu_read_unlock();
 
 	return ret;
diff --git a/kernel/cgroup/rstat.c b/kernel/cgroup/rstat.c
index c0adb7254b45..37fe8011d8a4 100644
--- a/kernel/cgroup/rstat.c
+++ b/kernel/cgroup/rstat.c
@@ -145,6 +145,32 @@ static struct cgroup *cgroup_rstat_cpu_pop_updated(struct cgroup *pos,
 	return pos;
 }
 
+/* Return a list of updated cgroups to be flushed */
+static struct cgroup *cgroup_rstat_updated_list(struct cgroup *root, int cpu)
+{
+	raw_spinlock_t *cpu_lock = per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu);
+	struct cgroup *head, *tail, *next;
+	unsigned long flags;
+
+	/*
+	 * The _irqsave() is needed because cgroup_rstat_lock is
+	 * spinlock_t which is a sleeping lock on PREEMPT_RT. Acquiring
+	 * this lock with the _irq() suffix only disables interrupts on
+	 * a non-PREEMPT_RT kernel. The raw_spinlock_t below disables
+	 * interrupts on both configurations. The _irqsave() ensures
+	 * that interrupts are always disabled and later restored.
+	 */
+	raw_spin_lock_irqsave(cpu_lock, flags);
+	head = tail = cgroup_rstat_cpu_pop_updated(NULL, root, cpu);
+	while (tail) {
+		next = cgroup_rstat_cpu_pop_updated(tail, root, cpu);
+		tail->rstat_flush_next = next;
+		tail = next;
+	}
+	raw_spin_unlock_irqrestore(cpu_lock, flags);
+	return head;
+}
+
 /*
  * A hook for bpf stat collectors to attach to and flush their stats.
  * Together with providing bpf kfuncs for cgroup_rstat_updated() and
@@ -176,21 +202,9 @@ static void cgroup_rstat_flush_locked(struct cgroup *cgrp)
 	lockdep_assert_held(&cgroup_rstat_lock);
 
 	for_each_possible_cpu(cpu) {
-		raw_spinlock_t *cpu_lock = per_cpu_ptr(&cgroup_rstat_cpu_lock,
-						       cpu);
-		struct cgroup *pos = NULL;
-		unsigned long flags;
+		struct cgroup *pos = cgroup_rstat_updated_list(cgrp, cpu);
 
-		/*
-		 * The _irqsave() is needed because cgroup_rstat_lock is
-		 * spinlock_t which is a sleeping lock on PREEMPT_RT. Acquiring
-		 * this lock with the _irq() suffix only disables interrupts on
-		 * a non-PREEMPT_RT kernel. The raw_spinlock_t below disables
-		 * interrupts on both configurations. The _irqsave() ensures
-		 * that interrupts are always disabled and later restored.
-		 */
-		raw_spin_lock_irqsave(cpu_lock, flags);
-		while ((pos = cgroup_rstat_cpu_pop_updated(pos, cgrp, cpu))) {
+		for (; pos; pos = pos->rstat_flush_next) {
 			struct cgroup_subsys_state *css;
 
 			cgroup_base_stat_flush(pos, cpu);
@@ -202,7 +216,6 @@ static void cgroup_rstat_flush_locked(struct cgroup *cgrp)
 				css->ss->css_rstat_flush(css, cpu);
 			rcu_read_unlock();
 		}
-		raw_spin_unlock_irqrestore(cpu_lock, flags);
 
 		/* play nice and yield if necessary */
 		if (need_resched() || spin_needbreak(&cgroup_rstat_lock)) {
diff --git a/kernel/events/core.c b/kernel/events/core.c
index b704d83a28b2..4f0c45ab8d7d 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -7372,6 +7372,14 @@ void perf_output_sample(struct perf_output_handle *handle,
 			if (branch_sample_hw_index(event))
 				perf_output_put(handle, data->br_stack->hw_idx);
 			perf_output_copy(handle, data->br_stack->entries, size);
+			/*
+			 * Add the extension space which is appended
+			 * right after the struct perf_branch_stack.
+			 */
+			if (data->br_stack_cntr) {
+				size = data->br_stack->nr * sizeof(u64);
+				perf_output_copy(handle, data->br_stack_cntr, size);
+			}
 		} else {
 			/*
 			 * we always store at least the value of nr
@@ -11400,9 +11408,32 @@ static DEVICE_ATTR_RW(perf_event_mux_interval_ms);
 static struct attribute *pmu_dev_attrs[] = {
 	&dev_attr_type.attr,
 	&dev_attr_perf_event_mux_interval_ms.attr,
+	&dev_attr_nr_addr_filters.attr,
+	NULL,
+};
+
+static umode_t pmu_dev_is_visible(struct kobject *kobj, struct attribute *a, int n)
+{
+	struct device *dev = kobj_to_dev(kobj);
+	struct pmu *pmu = dev_get_drvdata(dev);
+
+	if (!pmu->nr_addr_filters)
+		return 0;
+
+	return a->mode;
+
+	return 0;
+}
+
+static struct attribute_group pmu_dev_attr_group = {
+	.is_visible = pmu_dev_is_visible,
+	.attrs = pmu_dev_attrs,
+};
+
+static const struct attribute_group *pmu_dev_groups[] = {
+	&pmu_dev_attr_group,
 	NULL,
 };
-ATTRIBUTE_GROUPS(pmu_dev);
 
 static int pmu_bus_running;
 static struct bus_type pmu_bus = {
@@ -11439,18 +11470,11 @@ static int pmu_dev_alloc(struct pmu *pmu)
 	if (ret)
 		goto free_dev;
 
-	/* For PMUs with address filters, throw in an extra attribute: */
-	if (pmu->nr_addr_filters)
-		ret = device_create_file(pmu->dev, &dev_attr_nr_addr_filters);
-
-	if (ret)
-		goto del_dev;
-
-	if (pmu->attr_update)
+	if (pmu->attr_update) {
 		ret = sysfs_update_groups(&pmu->dev->kobj, pmu->attr_update);
-
-	if (ret)
-		goto del_dev;
+		if (ret)
+			goto del_dev;
+	}
 
 out:
 	return ret;
diff --git a/kernel/fork.c b/kernel/fork.c
index 10917c3e1f03..abb534e78e8a 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -165,7 +165,6 @@ void __weak arch_release_task_struct(struct task_struct *tsk)
 {
 }
 
-#ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
 static struct kmem_cache *task_struct_cachep;
 
 static inline struct task_struct *alloc_task_struct_node(int node)
@@ -177,9 +176,6 @@ static inline void free_task_struct(struct task_struct *tsk)
 {
 	kmem_cache_free(task_struct_cachep, tsk);
 }
-#endif
-
-#ifndef CONFIG_ARCH_THREAD_STACK_ALLOCATOR
 
 /*
  * Allocate pages if THREAD_SIZE is >= PAGE_SIZE, otherwise use a
@@ -412,24 +408,6 @@ void thread_stack_cache_init(void)
 }
 
 # endif /* THREAD_SIZE >= PAGE_SIZE || defined(CONFIG_VMAP_STACK) */
-#else /* CONFIG_ARCH_THREAD_STACK_ALLOCATOR */
-
-static int alloc_thread_stack_node(struct task_struct *tsk, int node)
-{
-	unsigned long *stack;
-
-	stack = arch_alloc_thread_stack_node(tsk, node);
-	tsk->stack = stack;
-	return stack ? 0 : -ENOMEM;
-}
-
-static void free_thread_stack(struct task_struct *tsk)
-{
-	arch_free_thread_stack(tsk);
-	tsk->stack = NULL;
-}
-
-#endif /* !CONFIG_ARCH_THREAD_STACK_ALLOCATOR */
 
 /* SLAB cache for signal_struct structures (tsk->signal) */
 static struct kmem_cache *signal_cachep;
@@ -650,7 +628,6 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
 	int retval;
 	unsigned long charge = 0;
 	LIST_HEAD(uf);
-	VMA_ITERATOR(old_vmi, oldmm, 0);
 	VMA_ITERATOR(vmi, mm, 0);
 
 	uprobe_start_dup_mmap();
@@ -678,16 +655,22 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
 		goto out;
 	khugepaged_fork(mm, oldmm);
 
-	retval = vma_iter_bulk_alloc(&vmi, oldmm->map_count);
-	if (retval)
+	/* Use __mt_dup() to efficiently build an identical maple tree. */
+	retval = __mt_dup(&oldmm->mm_mt, &mm->mm_mt, GFP_KERNEL);
+	if (unlikely(retval))
 		goto out;
 
 	mt_clear_in_rcu(vmi.mas.tree);
-	for_each_vma(old_vmi, mpnt) {
+	for_each_vma(vmi, mpnt) {
 		struct file *file;
 
 		vma_start_write(mpnt);
 		if (mpnt->vm_flags & VM_DONTCOPY) {
+			retval = vma_iter_clear_gfp(&vmi, mpnt->vm_start,
+						    mpnt->vm_end, GFP_KERNEL);
+			if (retval)
+				goto loop_out;
+
 			vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt));
 			continue;
 		}
@@ -749,9 +732,11 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
 		if (is_vm_hugetlb_page(tmp))
 			hugetlb_dup_vma_private(tmp);
 
-		/* Link the vma into the MT */
-		if (vma_iter_bulk_store(&vmi, tmp))
-			goto fail_nomem_vmi_store;
+		/*
+		 * Link the vma into the MT. After using __mt_dup(), memory
+		 * allocation is not necessary here, so it cannot fail.
+		 */
+		vma_iter_bulk_store(&vmi, tmp);
 
 		mm->map_count++;
 		if (!(tmp->vm_flags & VM_WIPEONFORK))
@@ -760,15 +745,28 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
 		if (tmp->vm_ops && tmp->vm_ops->open)
 			tmp->vm_ops->open(tmp);
 
-		if (retval)
+		if (retval) {
+			mpnt = vma_next(&vmi);
 			goto loop_out;
+		}
 	}
 	/* a new mm has just been created */
 	retval = arch_dup_mmap(oldmm, mm);
 loop_out:
 	vma_iter_free(&vmi);
-	if (!retval)
+	if (!retval) {
 		mt_set_in_rcu(vmi.mas.tree);
+	} else if (mpnt) {
+		/*
+		 * The entire maple tree has already been duplicated. If the
+		 * mmap duplication fails, mark the failure point with
+		 * XA_ZERO_ENTRY. In exit_mmap(), if this marker is encountered,
+		 * stop releasing VMAs that have not been duplicated after this
+		 * point.
+		 */
+		mas_set_range(&vmi.mas, mpnt->vm_start, mpnt->vm_end - 1);
+		mas_store(&vmi.mas, XA_ZERO_ENTRY);
+	}
 out:
 	mmap_write_unlock(mm);
 	flush_tlb_mm(oldmm);
@@ -778,8 +776,6 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
 	uprobe_end_dup_mmap();
 	return retval;
 
-fail_nomem_vmi_store:
-	unlink_anon_vmas(tmp);
 fail_nomem_anon_vma_fork:
 	mpol_put(vma_policy(tmp));
 fail_nomem_policy:
@@ -1021,7 +1017,6 @@ static void set_max_threads(unsigned int max_threads_suggested)
 int arch_task_struct_size __read_mostly;
 #endif
 
-#ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
 static void task_struct_whitelist(unsigned long *offset, unsigned long *size)
 {
 	/* Fetch thread_struct whitelist for the architecture. */
@@ -1036,12 +1031,10 @@ static void task_struct_whitelist(unsigned long *offset, unsigned long *size)
 	else
 		*offset += offsetof(struct task_struct, thread);
 }
-#endif /* CONFIG_ARCH_TASK_STRUCT_ALLOCATOR */
 
 void __init fork_init(void)
 {
 	int i;
-#ifndef CONFIG_ARCH_TASK_STRUCT_ALLOCATOR
 #ifndef ARCH_MIN_TASKALIGN
 #define ARCH_MIN_TASKALIGN	0
 #endif
@@ -1054,7 +1047,6 @@ void __init fork_init(void)
 			arch_task_struct_size, align,
 			SLAB_PANIC|SLAB_ACCOUNT,
 			useroffset, usersize, NULL);
-#endif
 
 	/* do the arch specific task caches init */
 	arch_task_cache_init();
diff --git a/kernel/kthread.c b/kernel/kthread.c
index c5e40830c1f2..25742c005bde 100644
--- a/kernel/kthread.c
+++ b/kernel/kthread.c
@@ -155,7 +155,8 @@ void free_kthread_struct(struct task_struct *k)
  */
 bool kthread_should_stop(void)
 {
-	return test_bit(KTHREAD_SHOULD_STOP, &to_kthread(current)->flags);
+	return (current->flags & PF_KTHREAD) &&
+		test_bit(KTHREAD_SHOULD_STOP, &to_kthread(current)->flags);
 }
 EXPORT_SYMBOL(kthread_should_stop);
 
diff --git a/kernel/rcu/Kconfig.debug b/kernel/rcu/Kconfig.debug
index 2984de629f74..9b0b52e1836f 100644
--- a/kernel/rcu/Kconfig.debug
+++ b/kernel/rcu/Kconfig.debug
@@ -105,6 +105,31 @@ config RCU_CPU_STALL_CPUTIME
 	  The boot option rcupdate.rcu_cpu_stall_cputime has the same function
 	  as this one, but will override this if it exists.
 
+config RCU_CPU_STALL_NOTIFIER
+	bool "Provide RCU CPU-stall notifiers"
+	depends on RCU_STALL_COMMON
+	depends on DEBUG_KERNEL
+	depends on RCU_EXPERT
+	default n
+	help
+	  WARNING:  You almost certainly do not want this!!!
+
+	  Enable RCU CPU-stall notifiers, which are invoked just before
+	  printing the RCU CPU stall warning.  As such, bugs in notifier
+	  callbacks can prevent stall warnings from being printed.
+	  And the whole reason that a stall warning is being printed is
+	  that something is hung up somewhere.	Therefore, the notifier
+	  callbacks must be written extremely carefully, preferably
+	  containing only lockless code.  After all, it is quite possible
+	  that the whole reason that the RCU CPU stall is happening in
+	  the first place is that someone forgot to release whatever lock
+	  that you are thinking of acquiring.  In which case, having your
+	  notifier callback acquire that lock will hang, preventing the
+	  RCU CPU stall warning from appearing.
+
+	  Say Y here if you want RCU CPU stall notifiers (you don't want them)
+	  Say N if you are unsure.
+
 config RCU_TRACE
 	bool "Enable tracing for RCU"
 	depends on DEBUG_KERNEL
diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h
index b531c33e9545..f94f65877f2b 100644
--- a/kernel/rcu/rcu.h
+++ b/kernel/rcu/rcu.h
@@ -262,6 +262,8 @@ static inline bool rcu_stall_is_suppressed_at_boot(void)
 	return rcu_cpu_stall_suppress_at_boot && !rcu_inkernel_boot_has_ended();
 }
 
+extern int rcu_cpu_stall_notifiers;
+
 #ifdef CONFIG_RCU_STALL_COMMON
 
 extern int rcu_cpu_stall_ftrace_dump;
@@ -659,10 +661,10 @@ static inline bool rcu_cpu_beenfullyonline(int cpu) { return true; }
 bool rcu_cpu_beenfullyonline(int cpu);
 #endif
 
-#ifdef CONFIG_RCU_STALL_COMMON
+#if defined(CONFIG_RCU_STALL_COMMON) && defined(CONFIG_RCU_CPU_STALL_NOTIFIER)
 int rcu_stall_notifier_call_chain(unsigned long val, void *v);
-#else // #ifdef CONFIG_RCU_STALL_COMMON
+#else // #if defined(CONFIG_RCU_STALL_COMMON) && defined(CONFIG_RCU_CPU_STALL_NOTIFIER)
 static inline int rcu_stall_notifier_call_chain(unsigned long val, void *v) { return NOTIFY_DONE; }
-#endif // #else // #ifdef CONFIG_RCU_STALL_COMMON
+#endif // #else // #if defined(CONFIG_RCU_STALL_COMMON) && defined(CONFIG_RCU_CPU_STALL_NOTIFIER)
 
 #endif /* __LINUX_RCU_H */
diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index 30fc9d34e329..07a6a183c555 100644
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -2450,10 +2450,12 @@ static int rcu_torture_stall(void *args)
 	unsigned long stop_at;
 
 	VERBOSE_TOROUT_STRING("rcu_torture_stall task started");
-	ret = rcu_stall_chain_notifier_register(&rcu_torture_stall_block);
-	if (ret)
-		pr_info("%s: rcu_stall_chain_notifier_register() returned %d, %sexpected.\n",
-			__func__, ret, !IS_ENABLED(CONFIG_RCU_STALL_COMMON) ? "un" : "");
+	if (rcu_cpu_stall_notifiers) {
+		ret = rcu_stall_chain_notifier_register(&rcu_torture_stall_block);
+		if (ret)
+			pr_info("%s: rcu_stall_chain_notifier_register() returned %d, %sexpected.\n",
+				__func__, ret, !IS_ENABLED(CONFIG_RCU_STALL_COMMON) ? "un" : "");
+	}
 	if (stall_cpu_holdoff > 0) {
 		VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff");
 		schedule_timeout_interruptible(stall_cpu_holdoff * HZ);
@@ -2497,7 +2499,7 @@ static int rcu_torture_stall(void *args)
 		cur_ops->readunlock(idx);
 	}
 	pr_alert("%s end.\n", __func__);
-	if (!ret) {
+	if (rcu_cpu_stall_notifiers && !ret) {
 		ret = rcu_stall_chain_notifier_unregister(&rcu_torture_stall_block);
 		if (ret)
 			pr_info("%s: rcu_stall_chain_notifier_unregister() returned %d.\n", __func__, ret);
diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h
index ac8e86babe44..5d666428546b 100644
--- a/kernel/rcu/tree_stall.h
+++ b/kernel/rcu/tree_stall.h
@@ -1061,6 +1061,7 @@ static int __init rcu_sysrq_init(void)
 }
 early_initcall(rcu_sysrq_init);
 
+#ifdef CONFIG_RCU_CPU_STALL_NOTIFIER
 
 //////////////////////////////////////////////////////////////////////////////
 //
@@ -1081,7 +1082,13 @@ static ATOMIC_NOTIFIER_HEAD(rcu_cpu_stall_notifier_list);
  */
 int rcu_stall_chain_notifier_register(struct notifier_block *n)
 {
-	return atomic_notifier_chain_register(&rcu_cpu_stall_notifier_list, n);
+	int rcsn = rcu_cpu_stall_notifiers;
+
+	WARN(1, "Adding %pS() to RCU stall notifier list (%s).\n", n->notifier_call,
+	     rcsn ? "possibly suppressing RCU CPU stall warnings" : "failed, so all is well");
+	if (rcsn)
+		return atomic_notifier_chain_register(&rcu_cpu_stall_notifier_list, n);
+	return -EEXIST;
 }
 EXPORT_SYMBOL_GPL(rcu_stall_chain_notifier_register);
 
@@ -1115,3 +1122,5 @@ int rcu_stall_notifier_call_chain(unsigned long val, void *v)
 {
 	return atomic_notifier_call_chain(&rcu_cpu_stall_notifier_list, val, v);
 }
+
+#endif // #ifdef CONFIG_RCU_CPU_STALL_NOTIFIER
diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c
index c534d6806d3d..46aaaa9fe339 100644
--- a/kernel/rcu/update.c
+++ b/kernel/rcu/update.c
@@ -538,9 +538,15 @@ long torture_sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
 EXPORT_SYMBOL_GPL(torture_sched_setaffinity);
 #endif
 
+int rcu_cpu_stall_notifiers __read_mostly; // !0 = provide stall notifiers (rarely useful)
+EXPORT_SYMBOL_GPL(rcu_cpu_stall_notifiers);
+
 #ifdef CONFIG_RCU_STALL_COMMON
 int rcu_cpu_stall_ftrace_dump __read_mostly;
 module_param(rcu_cpu_stall_ftrace_dump, int, 0644);
+#ifdef CONFIG_RCU_CPU_STALL_NOTIFIER
+module_param(rcu_cpu_stall_notifiers, int, 0444);
+#endif // #ifdef CONFIG_RCU_CPU_STALL_NOTIFIER
 int rcu_cpu_stall_suppress __read_mostly; // !0 = suppress stall warnings.
 EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress);
 module_param(rcu_cpu_stall_suppress, int, 0644);
diff --git a/kernel/reboot.c b/kernel/reboot.c
index 395a0ea3c7a8..c3a3b82c4f64 100644
--- a/kernel/reboot.c
+++ b/kernel/reboot.c
@@ -58,6 +58,14 @@ struct sys_off_handler {
 	struct device *dev;
 };
 
+/*
+ * This variable is used to indicate if a halt was initiated instead of a
+ * reboot when the reboot call was invoked with LINUX_REBOOT_CMD_POWER_OFF, but
+ * the system cannot be powered off. This allowes kernel_halt() to notify users
+ * of that.
+ */
+static bool poweroff_fallback_to_halt;
+
 /*
  * Temporary stub that prevents linkage failure while we're in process
  * of removing all uses of legacy pm_power_off() around the kernel.
@@ -297,7 +305,10 @@ void kernel_halt(void)
 	kernel_shutdown_prepare(SYSTEM_HALT);
 	migrate_to_reboot_cpu();
 	syscore_shutdown();
-	pr_emerg("System halted\n");
+	if (poweroff_fallback_to_halt)
+		pr_emerg("Power off not available: System halted instead\n");
+	else
+		pr_emerg("System halted\n");
 	kmsg_dump(KMSG_DUMP_SHUTDOWN);
 	machine_halt();
 }
@@ -732,8 +743,10 @@ SYSCALL_DEFINE4(reboot, int, magic1, int, magic2, unsigned int, cmd,
 	/* Instead of trying to make the power_off code look like
 	 * halt when pm_power_off is not set do it the easy way.
 	 */
-	if ((cmd == LINUX_REBOOT_CMD_POWER_OFF) && !kernel_can_power_off())
+	if ((cmd == LINUX_REBOOT_CMD_POWER_OFF) && !kernel_can_power_off()) {
+		poweroff_fallback_to_halt = true;
 		cmd = LINUX_REBOOT_CMD_HALT;
+	}
 
 	mutex_lock(&system_transition_mutex);
 	switch (cmd) {
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index a708d225c28e..2de77a6d5ef8 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1131,6 +1131,28 @@ static void wake_up_idle_cpu(int cpu)
 	if (cpu == smp_processor_id())
 		return;
 
+	/*
+	 * Set TIF_NEED_RESCHED and send an IPI if in the non-polling
+	 * part of the idle loop. This forces an exit from the idle loop
+	 * and a round trip to schedule(). Now this could be optimized
+	 * because a simple new idle loop iteration is enough to
+	 * re-evaluate the next tick. Provided some re-ordering of tick
+	 * nohz functions that would need to follow TIF_NR_POLLING
+	 * clearing:
+	 *
+	 * - On most archs, a simple fetch_or on ti::flags with a
+	 *   "0" value would be enough to know if an IPI needs to be sent.
+	 *
+	 * - x86 needs to perform a last need_resched() check between
+	 *   monitor and mwait which doesn't take timers into account.
+	 *   There a dedicated TIF_TIMER flag would be required to
+	 *   fetch_or here and be checked along with TIF_NEED_RESCHED
+	 *   before mwait().
+	 *
+	 * However, remote timer enqueue is not such a frequent event
+	 * and testing of the above solutions didn't appear to report
+	 * much benefits.
+	 */
 	if (set_nr_and_not_polling(rq->idle))
 		smp_send_reschedule(cpu);
 	else
@@ -2124,12 +2146,14 @@ void activate_task(struct rq *rq, struct task_struct *p, int flags)
 
 	enqueue_task(rq, p, flags);
 
-	p->on_rq = TASK_ON_RQ_QUEUED;
+	WRITE_ONCE(p->on_rq, TASK_ON_RQ_QUEUED);
+	ASSERT_EXCLUSIVE_WRITER(p->on_rq);
 }
 
 void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
 {
-	p->on_rq = (flags & DEQUEUE_SLEEP) ? 0 : TASK_ON_RQ_MIGRATING;
+	WRITE_ONCE(p->on_rq, (flags & DEQUEUE_SLEEP) ? 0 : TASK_ON_RQ_MIGRATING);
+	ASSERT_EXCLUSIVE_WRITER(p->on_rq);
 
 	dequeue_task(rq, p, flags);
 }
@@ -3795,6 +3819,8 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
 		rq->idle_stamp = 0;
 	}
 #endif
+
+	p->dl_server = NULL;
 }
 
 /*
@@ -4509,10 +4535,7 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
 	memset(&p->stats, 0, sizeof(p->stats));
 #endif
 
-	RB_CLEAR_NODE(&p->dl.rb_node);
-	init_dl_task_timer(&p->dl);
-	init_dl_inactive_task_timer(&p->dl);
-	__dl_clear_params(p);
+	init_dl_entity(&p->dl);
 
 	INIT_LIST_HEAD(&p->rt.run_list);
 	p->rt.timeout		= 0;
@@ -6004,12 +6027,27 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 			p = pick_next_task_idle(rq);
 		}
 
+		/*
+		 * This is the fast path; it cannot be a DL server pick;
+		 * therefore even if @p == @prev, ->dl_server must be NULL.
+		 */
+		if (p->dl_server)
+			p->dl_server = NULL;
+
 		return p;
 	}
 
 restart:
 	put_prev_task_balance(rq, prev, rf);
 
+	/*
+	 * We've updated @prev and no longer need the server link, clear it.
+	 * Must be done before ->pick_next_task() because that can (re)set
+	 * ->dl_server.
+	 */
+	if (prev->dl_server)
+		prev->dl_server = NULL;
+
 	for_each_class(class) {
 		p = class->pick_next_task(rq);
 		if (p)
diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index b28114478b82..a04a436af8cc 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -54,8 +54,14 @@ static int __init sched_dl_sysctl_init(void)
 late_initcall(sched_dl_sysctl_init);
 #endif
 
+static bool dl_server(struct sched_dl_entity *dl_se)
+{
+	return dl_se->dl_server;
+}
+
 static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se)
 {
+	BUG_ON(dl_server(dl_se));
 	return container_of(dl_se, struct task_struct, dl);
 }
 
@@ -64,12 +70,19 @@ static inline struct rq *rq_of_dl_rq(struct dl_rq *dl_rq)
 	return container_of(dl_rq, struct rq, dl);
 }
 
-static inline struct dl_rq *dl_rq_of_se(struct sched_dl_entity *dl_se)
+static inline struct rq *rq_of_dl_se(struct sched_dl_entity *dl_se)
 {
-	struct task_struct *p = dl_task_of(dl_se);
-	struct rq *rq = task_rq(p);
+	struct rq *rq = dl_se->rq;
+
+	if (!dl_server(dl_se))
+		rq = task_rq(dl_task_of(dl_se));
+
+	return rq;
+}
 
-	return &rq->dl;
+static inline struct dl_rq *dl_rq_of_se(struct sched_dl_entity *dl_se)
+{
+	return &rq_of_dl_se(dl_se)->dl;
 }
 
 static inline int on_dl_rq(struct sched_dl_entity *dl_se)
@@ -335,6 +348,8 @@ static void dl_change_utilization(struct task_struct *p, u64 new_bw)
 	__add_rq_bw(new_bw, &rq->dl);
 }
 
+static void __dl_clear_params(struct sched_dl_entity *dl_se);
+
 /*
  * The utilization of a task cannot be immediately removed from
  * the rq active utilization (running_bw) when the task blocks.
@@ -389,12 +404,11 @@ static void dl_change_utilization(struct task_struct *p, u64 new_bw)
  * up, and checks if the task is still in the "ACTIVE non contending"
  * state or not (in the second case, it updates running_bw).
  */
-static void task_non_contending(struct task_struct *p)
+static void task_non_contending(struct sched_dl_entity *dl_se)
 {
-	struct sched_dl_entity *dl_se = &p->dl;
 	struct hrtimer *timer = &dl_se->inactive_timer;
-	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);
-	struct rq *rq = rq_of_dl_rq(dl_rq);
+	struct rq *rq = rq_of_dl_se(dl_se);
+	struct dl_rq *dl_rq = &rq->dl;
 	s64 zerolag_time;
 
 	/*
@@ -424,24 +438,33 @@ static void task_non_contending(struct task_struct *p)
 	 * utilization now, instead of starting a timer
 	 */
 	if ((zerolag_time < 0) || hrtimer_active(&dl_se->inactive_timer)) {
-		if (dl_task(p))
+		if (dl_server(dl_se)) {
 			sub_running_bw(dl_se, dl_rq);
-		if (!dl_task(p) || READ_ONCE(p->__state) == TASK_DEAD) {
-			struct dl_bw *dl_b = dl_bw_of(task_cpu(p));
-
-			if (READ_ONCE(p->__state) == TASK_DEAD)
-				sub_rq_bw(&p->dl, &rq->dl);
-			raw_spin_lock(&dl_b->lock);
-			__dl_sub(dl_b, p->dl.dl_bw, dl_bw_cpus(task_cpu(p)));
-			raw_spin_unlock(&dl_b->lock);
-			__dl_clear_params(p);
+		} else {
+			struct task_struct *p = dl_task_of(dl_se);
+
+			if (dl_task(p))
+				sub_running_bw(dl_se, dl_rq);
+
+			if (!dl_task(p) || READ_ONCE(p->__state) == TASK_DEAD) {
+				struct dl_bw *dl_b = dl_bw_of(task_cpu(p));
+
+				if (READ_ONCE(p->__state) == TASK_DEAD)
+					sub_rq_bw(dl_se, &rq->dl);
+				raw_spin_lock(&dl_b->lock);
+				__dl_sub(dl_b, dl_se->dl_bw, dl_bw_cpus(task_cpu(p)));
+				raw_spin_unlock(&dl_b->lock);
+				__dl_clear_params(dl_se);
+			}
 		}
 
 		return;
 	}
 
 	dl_se->dl_non_contending = 1;
-	get_task_struct(p);
+	if (!dl_server(dl_se))
+		get_task_struct(dl_task_of(dl_se));
+
 	hrtimer_start(timer, ns_to_ktime(zerolag_time), HRTIMER_MODE_REL_HARD);
 }
 
@@ -468,8 +491,10 @@ static void task_contending(struct sched_dl_entity *dl_se, int flags)
 		 * will not touch the rq's active utilization,
 		 * so we are still safe.
 		 */
-		if (hrtimer_try_to_cancel(&dl_se->inactive_timer) == 1)
-			put_task_struct(dl_task_of(dl_se));
+		if (hrtimer_try_to_cancel(&dl_se->inactive_timer) == 1) {
+			if (!dl_server(dl_se))
+				put_task_struct(dl_task_of(dl_se));
+		}
 	} else {
 		/*
 		 * Since "dl_non_contending" is not set, the
@@ -482,10 +507,8 @@ static void task_contending(struct sched_dl_entity *dl_se, int flags)
 	}
 }
 
-static inline int is_leftmost(struct task_struct *p, struct dl_rq *dl_rq)
+static inline int is_leftmost(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)
 {
-	struct sched_dl_entity *dl_se = &p->dl;
-
 	return rb_first_cached(&dl_rq->root) == &dl_se->rb_node;
 }
 
@@ -737,8 +760,10 @@ static inline void deadline_queue_pull_task(struct rq *rq)
 }
 #endif /* CONFIG_SMP */
 
+static void
+enqueue_dl_entity(struct sched_dl_entity *dl_se, int flags);
 static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags);
-static void __dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags);
+static void dequeue_dl_entity(struct sched_dl_entity *dl_se, int flags);
 static void wakeup_preempt_dl(struct rq *rq, struct task_struct *p, int flags);
 
 static inline void replenish_dl_new_period(struct sched_dl_entity *dl_se,
@@ -986,8 +1011,7 @@ static inline bool dl_is_implicit(struct sched_dl_entity *dl_se)
  */
 static void update_dl_entity(struct sched_dl_entity *dl_se)
 {
-	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);
-	struct rq *rq = rq_of_dl_rq(dl_rq);
+	struct rq *rq = rq_of_dl_se(dl_se);
 
 	if (dl_time_before(dl_se->deadline, rq_clock(rq)) ||
 	    dl_entity_overflow(dl_se, rq_clock(rq))) {
@@ -1018,11 +1042,11 @@ static inline u64 dl_next_period(struct sched_dl_entity *dl_se)
  * actually started or not (i.e., the replenishment instant is in
  * the future or in the past).
  */
-static int start_dl_timer(struct task_struct *p)
+static int start_dl_timer(struct sched_dl_entity *dl_se)
 {
-	struct sched_dl_entity *dl_se = &p->dl;
 	struct hrtimer *timer = &dl_se->dl_timer;
-	struct rq *rq = task_rq(p);
+	struct dl_rq *dl_rq = dl_rq_of_se(dl_se);
+	struct rq *rq = rq_of_dl_rq(dl_rq);
 	ktime_t now, act;
 	s64 delta;
 
@@ -1056,13 +1080,33 @@ static int start_dl_timer(struct task_struct *p)
 	 * and observe our state.
 	 */
 	if (!hrtimer_is_queued(timer)) {
-		get_task_struct(p);
+		if (!dl_server(dl_se))
+			get_task_struct(dl_task_of(dl_se));
 		hrtimer_start(timer, act, HRTIMER_MODE_ABS_HARD);
 	}
 
 	return 1;
 }
 
+static void __push_dl_task(struct rq *rq, struct rq_flags *rf)
+{
+#ifdef CONFIG_SMP
+	/*
+	 * Queueing this task back might have overloaded rq, check if we need
+	 * to kick someone away.
+	 */
+	if (has_pushable_dl_tasks(rq)) {
+		/*
+		 * Nothing relies on rq->lock after this, so its safe to drop
+		 * rq->lock.
+		 */
+		rq_unpin_lock(rq, rf);
+		push_dl_task(rq);
+		rq_repin_lock(rq, rf);
+	}
+#endif
+}
+
 /*
  * This is the bandwidth enforcement timer callback. If here, we know
  * a task is not on its dl_rq, since the fact that the timer was running
@@ -1081,10 +1125,34 @@ static enum hrtimer_restart dl_task_timer(struct hrtimer *timer)
 	struct sched_dl_entity *dl_se = container_of(timer,
 						     struct sched_dl_entity,
 						     dl_timer);
-	struct task_struct *p = dl_task_of(dl_se);
+	struct task_struct *p;
 	struct rq_flags rf;
 	struct rq *rq;
 
+	if (dl_server(dl_se)) {
+		struct rq *rq = rq_of_dl_se(dl_se);
+		struct rq_flags rf;
+
+		rq_lock(rq, &rf);
+		if (dl_se->dl_throttled) {
+			sched_clock_tick();
+			update_rq_clock(rq);
+
+			if (dl_se->server_has_tasks(dl_se)) {
+				enqueue_dl_entity(dl_se, ENQUEUE_REPLENISH);
+				resched_curr(rq);
+				__push_dl_task(rq, &rf);
+			} else {
+				replenish_dl_entity(dl_se);
+			}
+
+		}
+		rq_unlock(rq, &rf);
+
+		return HRTIMER_NORESTART;
+	}
+
+	p = dl_task_of(dl_se);
 	rq = task_rq_lock(p, &rf);
 
 	/*
@@ -1155,21 +1223,7 @@ static enum hrtimer_restart dl_task_timer(struct hrtimer *timer)
 	else
 		resched_curr(rq);
 
-#ifdef CONFIG_SMP
-	/*
-	 * Queueing this task back might have overloaded rq, check if we need
-	 * to kick someone away.
-	 */
-	if (has_pushable_dl_tasks(rq)) {
-		/*
-		 * Nothing relies on rq->lock after this, so its safe to drop
-		 * rq->lock.
-		 */
-		rq_unpin_lock(rq, &rf);
-		push_dl_task(rq);
-		rq_repin_lock(rq, &rf);
-	}
-#endif
+	__push_dl_task(rq, &rf);
 
 unlock:
 	task_rq_unlock(rq, p, &rf);
@@ -1183,7 +1237,7 @@ static enum hrtimer_restart dl_task_timer(struct hrtimer *timer)
 	return HRTIMER_NORESTART;
 }
 
-void init_dl_task_timer(struct sched_dl_entity *dl_se)
+static void init_dl_task_timer(struct sched_dl_entity *dl_se)
 {
 	struct hrtimer *timer = &dl_se->dl_timer;
 
@@ -1211,12 +1265,11 @@ void init_dl_task_timer(struct sched_dl_entity *dl_se)
  */
 static inline void dl_check_constrained_dl(struct sched_dl_entity *dl_se)
 {
-	struct task_struct *p = dl_task_of(dl_se);
-	struct rq *rq = rq_of_dl_rq(dl_rq_of_se(dl_se));
+	struct rq *rq = rq_of_dl_se(dl_se);
 
 	if (dl_time_before(dl_se->deadline, rq_clock(rq)) &&
 	    dl_time_before(rq_clock(rq), dl_next_period(dl_se))) {
-		if (unlikely(is_dl_boosted(dl_se) || !start_dl_timer(p)))
+		if (unlikely(is_dl_boosted(dl_se) || !start_dl_timer(dl_se)))
 			return;
 		dl_se->dl_throttled = 1;
 		if (dl_se->runtime > 0)
@@ -1267,44 +1320,19 @@ static u64 grub_reclaim(u64 delta, struct rq *rq, struct sched_dl_entity *dl_se)
 	return (delta * u_act) >> BW_SHIFT;
 }
 
-/*
- * Update the current task's runtime statistics (provided it is still
- * a -deadline task and has not been removed from the dl_rq).
- */
-static void update_curr_dl(struct rq *rq)
+static inline void
+update_stats_dequeue_dl(struct dl_rq *dl_rq, struct sched_dl_entity *dl_se,
+                        int flags);
+static void update_curr_dl_se(struct rq *rq, struct sched_dl_entity *dl_se, s64 delta_exec)
 {
-	struct task_struct *curr = rq->curr;
-	struct sched_dl_entity *dl_se = &curr->dl;
-	u64 delta_exec, scaled_delta_exec;
-	int cpu = cpu_of(rq);
-	u64 now;
-
-	if (!dl_task(curr) || !on_dl_rq(dl_se))
-		return;
+	s64 scaled_delta_exec;
 
-	/*
-	 * Consumed budget is computed considering the time as
-	 * observed by schedulable tasks (excluding time spent
-	 * in hardirq context, etc.). Deadlines are instead
-	 * computed using hard walltime. This seems to be the more
-	 * natural solution, but the full ramifications of this
-	 * approach need further study.
-	 */
-	now = rq_clock_task(rq);
-	delta_exec = now - curr->se.exec_start;
-	if (unlikely((s64)delta_exec <= 0)) {
+	if (unlikely(delta_exec <= 0)) {
 		if (unlikely(dl_se->dl_yielded))
 			goto throttle;
 		return;
 	}
 
-	schedstat_set(curr->stats.exec_max,
-		      max(curr->stats.exec_max, delta_exec));
-
-	trace_sched_stat_runtime(curr, delta_exec, 0);
-
-	update_current_exec_runtime(curr, now, delta_exec);
-
 	if (dl_entity_is_special(dl_se))
 		return;
 
@@ -1316,10 +1344,9 @@ static void update_curr_dl(struct rq *rq)
 	 * according to current frequency and CPU maximum capacity.
 	 */
 	if (unlikely(dl_se->flags & SCHED_FLAG_RECLAIM)) {
-		scaled_delta_exec = grub_reclaim(delta_exec,
-						 rq,
-						 &curr->dl);
+		scaled_delta_exec = grub_reclaim(delta_exec, rq, dl_se);
 	} else {
+		int cpu = cpu_of(rq);
 		unsigned long scale_freq = arch_scale_freq_capacity(cpu);
 		unsigned long scale_cpu = arch_scale_cpu_capacity(cpu);
 
@@ -1338,11 +1365,20 @@ static void update_curr_dl(struct rq *rq)
 		    (dl_se->flags & SCHED_FLAG_DL_OVERRUN))
 			dl_se->dl_overrun = 1;
 
-		__dequeue_task_dl(rq, curr, 0);
-		if (unlikely(is_dl_boosted(dl_se) || !start_dl_timer(curr)))
-			enqueue_task_dl(rq, curr, ENQUEUE_REPLENISH);
+		dequeue_dl_entity(dl_se, 0);
+		if (!dl_server(dl_se)) {
+			update_stats_dequeue_dl(&rq->dl, dl_se, 0);
+			dequeue_pushable_dl_task(rq, dl_task_of(dl_se));
+		}
+
+		if (unlikely(is_dl_boosted(dl_se) || !start_dl_timer(dl_se))) {
+			if (dl_server(dl_se))
+				enqueue_dl_entity(dl_se, ENQUEUE_REPLENISH);
+			else
+				enqueue_task_dl(rq, dl_task_of(dl_se), ENQUEUE_REPLENISH);
+		}
 
-		if (!is_leftmost(curr, &rq->dl))
+		if (!is_leftmost(dl_se, &rq->dl))
 			resched_curr(rq);
 	}
 
@@ -1372,20 +1408,82 @@ static void update_curr_dl(struct rq *rq)
 	}
 }
 
+void dl_server_update(struct sched_dl_entity *dl_se, s64 delta_exec)
+{
+	update_curr_dl_se(dl_se->rq, dl_se, delta_exec);
+}
+
+void dl_server_start(struct sched_dl_entity *dl_se)
+{
+	if (!dl_server(dl_se)) {
+		dl_se->dl_server = 1;
+		setup_new_dl_entity(dl_se);
+	}
+	enqueue_dl_entity(dl_se, ENQUEUE_WAKEUP);
+}
+
+void dl_server_stop(struct sched_dl_entity *dl_se)
+{
+	dequeue_dl_entity(dl_se, DEQUEUE_SLEEP);
+}
+
+void dl_server_init(struct sched_dl_entity *dl_se, struct rq *rq,
+		    dl_server_has_tasks_f has_tasks,
+		    dl_server_pick_f pick)
+{
+	dl_se->rq = rq;
+	dl_se->server_has_tasks = has_tasks;
+	dl_se->server_pick = pick;
+}
+
+/*
+ * Update the current task's runtime statistics (provided it is still
+ * a -deadline task and has not been removed from the dl_rq).
+ */
+static void update_curr_dl(struct rq *rq)
+{
+	struct task_struct *curr = rq->curr;
+	struct sched_dl_entity *dl_se = &curr->dl;
+	s64 delta_exec;
+
+	if (!dl_task(curr) || !on_dl_rq(dl_se))
+		return;
+
+	/*
+	 * Consumed budget is computed considering the time as
+	 * observed by schedulable tasks (excluding time spent
+	 * in hardirq context, etc.). Deadlines are instead
+	 * computed using hard walltime. This seems to be the more
+	 * natural solution, but the full ramifications of this
+	 * approach need further study.
+	 */
+	delta_exec = update_curr_common(rq);
+	update_curr_dl_se(rq, dl_se, delta_exec);
+}
+
 static enum hrtimer_restart inactive_task_timer(struct hrtimer *timer)
 {
 	struct sched_dl_entity *dl_se = container_of(timer,
 						     struct sched_dl_entity,
 						     inactive_timer);
-	struct task_struct *p = dl_task_of(dl_se);
+	struct task_struct *p = NULL;
 	struct rq_flags rf;
 	struct rq *rq;
 
-	rq = task_rq_lock(p, &rf);
+	if (!dl_server(dl_se)) {
+		p = dl_task_of(dl_se);
+		rq = task_rq_lock(p, &rf);
+	} else {
+		rq = dl_se->rq;
+		rq_lock(rq, &rf);
+	}
 
 	sched_clock_tick();
 	update_rq_clock(rq);
 
+	if (dl_server(dl_se))
+		goto no_task;
+
 	if (!dl_task(p) || READ_ONCE(p->__state) == TASK_DEAD) {
 		struct dl_bw *dl_b = dl_bw_of(task_cpu(p));
 
@@ -1398,23 +1496,30 @@ static enum hrtimer_restart inactive_task_timer(struct hrtimer *timer)
 		raw_spin_lock(&dl_b->lock);
 		__dl_sub(dl_b, p->dl.dl_bw, dl_bw_cpus(task_cpu(p)));
 		raw_spin_unlock(&dl_b->lock);
-		__dl_clear_params(p);
+		__dl_clear_params(dl_se);
 
 		goto unlock;
 	}
+
+no_task:
 	if (dl_se->dl_non_contending == 0)
 		goto unlock;
 
 	sub_running_bw(dl_se, &rq->dl);
 	dl_se->dl_non_contending = 0;
 unlock:
-	task_rq_unlock(rq, p, &rf);
-	put_task_struct(p);
+
+	if (!dl_server(dl_se)) {
+		task_rq_unlock(rq, p, &rf);
+		put_task_struct(p);
+	} else {
+		rq_unlock(rq, &rf);
+	}
 
 	return HRTIMER_NORESTART;
 }
 
-void init_dl_inactive_task_timer(struct sched_dl_entity *dl_se)
+static void init_dl_inactive_task_timer(struct sched_dl_entity *dl_se)
 {
 	struct hrtimer *timer = &dl_se->inactive_timer;
 
@@ -1472,10 +1577,8 @@ static inline void dec_dl_deadline(struct dl_rq *dl_rq, u64 deadline) {}
 static inline
 void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)
 {
-	int prio = dl_task_of(dl_se)->prio;
 	u64 deadline = dl_se->deadline;
 
-	WARN_ON(!dl_prio(prio));
 	dl_rq->dl_nr_running++;
 	add_nr_running(rq_of_dl_rq(dl_rq), 1);
 
@@ -1485,9 +1588,6 @@ void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)
 static inline
 void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)
 {
-	int prio = dl_task_of(dl_se)->prio;
-
-	WARN_ON(!dl_prio(prio));
 	WARN_ON(!dl_rq->dl_nr_running);
 	dl_rq->dl_nr_running--;
 	sub_nr_running(rq_of_dl_rq(dl_rq), 1);
@@ -1608,6 +1708,41 @@ enqueue_dl_entity(struct sched_dl_entity *dl_se, int flags)
 
 	update_stats_enqueue_dl(dl_rq_of_se(dl_se), dl_se, flags);
 
+	/*
+	 * Check if a constrained deadline task was activated
+	 * after the deadline but before the next period.
+	 * If that is the case, the task will be throttled and
+	 * the replenishment timer will be set to the next period.
+	 */
+	if (!dl_se->dl_throttled && !dl_is_implicit(dl_se))
+		dl_check_constrained_dl(dl_se);
+
+	if (flags & (ENQUEUE_RESTORE|ENQUEUE_MIGRATING)) {
+		struct dl_rq *dl_rq = dl_rq_of_se(dl_se);
+
+		add_rq_bw(dl_se, dl_rq);
+		add_running_bw(dl_se, dl_rq);
+	}
+
+	/*
+	 * If p is throttled, we do not enqueue it. In fact, if it exhausted
+	 * its budget it needs a replenishment and, since it now is on
+	 * its rq, the bandwidth timer callback (which clearly has not
+	 * run yet) will take care of this.
+	 * However, the active utilization does not depend on the fact
+	 * that the task is on the runqueue or not (but depends on the
+	 * task's state - in GRUB parlance, "inactive" vs "active contending").
+	 * In other words, even if a task is throttled its utilization must
+	 * be counted in the active utilization; hence, we need to call
+	 * add_running_bw().
+	 */
+	if (dl_se->dl_throttled && !(flags & ENQUEUE_REPLENISH)) {
+		if (flags & ENQUEUE_WAKEUP)
+			task_contending(dl_se, flags);
+
+		return;
+	}
+
 	/*
 	 * If this is a wakeup or a new instance, the scheduling
 	 * parameters of the task might need updating. Otherwise,
@@ -1619,17 +1754,35 @@ enqueue_dl_entity(struct sched_dl_entity *dl_se, int flags)
 	} else if (flags & ENQUEUE_REPLENISH) {
 		replenish_dl_entity(dl_se);
 	} else if ((flags & ENQUEUE_RESTORE) &&
-		  dl_time_before(dl_se->deadline,
-				 rq_clock(rq_of_dl_rq(dl_rq_of_se(dl_se))))) {
+		   dl_time_before(dl_se->deadline, rq_clock(rq_of_dl_se(dl_se)))) {
 		setup_new_dl_entity(dl_se);
 	}
 
 	__enqueue_dl_entity(dl_se);
 }
 
-static void dequeue_dl_entity(struct sched_dl_entity *dl_se)
+static void dequeue_dl_entity(struct sched_dl_entity *dl_se, int flags)
 {
 	__dequeue_dl_entity(dl_se);
+
+	if (flags & (DEQUEUE_SAVE|DEQUEUE_MIGRATING)) {
+		struct dl_rq *dl_rq = dl_rq_of_se(dl_se);
+
+		sub_running_bw(dl_se, dl_rq);
+		sub_rq_bw(dl_se, dl_rq);
+	}
+
+	/*
+	 * This check allows to start the inactive timer (or to immediately
+	 * decrease the active utilization, if needed) in two cases:
+	 * when the task blocks and when it is terminating
+	 * (p->state == TASK_DEAD). We can handle the two cases in the same
+	 * way, because from GRUB's point of view the same thing is happening
+	 * (the task moves from "active contending" to "active non contending"
+	 * or "inactive")
+	 */
+	if (flags & DEQUEUE_SLEEP)
+		task_non_contending(dl_se);
 }
 
 static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags)
@@ -1674,76 +1827,31 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags)
 		return;
 	}
 
-	/*
-	 * Check if a constrained deadline task was activated
-	 * after the deadline but before the next period.
-	 * If that is the case, the task will be throttled and
-	 * the replenishment timer will be set to the next period.
-	 */
-	if (!p->dl.dl_throttled && !dl_is_implicit(&p->dl))
-		dl_check_constrained_dl(&p->dl);
-
-	if (p->on_rq == TASK_ON_RQ_MIGRATING || flags & ENQUEUE_RESTORE) {
-		add_rq_bw(&p->dl, &rq->dl);
-		add_running_bw(&p->dl, &rq->dl);
-	}
-
-	/*
-	 * If p is throttled, we do not enqueue it. In fact, if it exhausted
-	 * its budget it needs a replenishment and, since it now is on
-	 * its rq, the bandwidth timer callback (which clearly has not
-	 * run yet) will take care of this.
-	 * However, the active utilization does not depend on the fact
-	 * that the task is on the runqueue or not (but depends on the
-	 * task's state - in GRUB parlance, "inactive" vs "active contending").
-	 * In other words, even if a task is throttled its utilization must
-	 * be counted in the active utilization; hence, we need to call
-	 * add_running_bw().
-	 */
-	if (p->dl.dl_throttled && !(flags & ENQUEUE_REPLENISH)) {
-		if (flags & ENQUEUE_WAKEUP)
-			task_contending(&p->dl, flags);
-
-		return;
-	}
-
 	check_schedstat_required();
 	update_stats_wait_start_dl(dl_rq_of_se(&p->dl), &p->dl);
 
+	if (p->on_rq == TASK_ON_RQ_MIGRATING)
+		flags |= ENQUEUE_MIGRATING;
+
 	enqueue_dl_entity(&p->dl, flags);
 
-	if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
-		enqueue_pushable_dl_task(rq, p);
-}
+	if (dl_server(&p->dl))
+		return;
 
-static void __dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags)
-{
-	update_stats_dequeue_dl(&rq->dl, &p->dl, flags);
-	dequeue_dl_entity(&p->dl);
-	dequeue_pushable_dl_task(rq, p);
+	if (!task_current(rq, p) && !p->dl.dl_throttled && p->nr_cpus_allowed > 1)
+		enqueue_pushable_dl_task(rq, p);
 }
 
 static void dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags)
 {
 	update_curr_dl(rq);
-	__dequeue_task_dl(rq, p, flags);
 
-	if (p->on_rq == TASK_ON_RQ_MIGRATING || flags & DEQUEUE_SAVE) {
-		sub_running_bw(&p->dl, &rq->dl);
-		sub_rq_bw(&p->dl, &rq->dl);
-	}
+	if (p->on_rq == TASK_ON_RQ_MIGRATING)
+		flags |= DEQUEUE_MIGRATING;
 
-	/*
-	 * This check allows to start the inactive timer (or to immediately
-	 * decrease the active utilization, if needed) in two cases:
-	 * when the task blocks and when it is terminating
-	 * (p->state == TASK_DEAD). We can handle the two cases in the same
-	 * way, because from GRUB's point of view the same thing is happening
-	 * (the task moves from "active contending" to "active non contending"
-	 * or "inactive")
-	 */
-	if (flags & DEQUEUE_SLEEP)
-		task_non_contending(p);
+	dequeue_dl_entity(&p->dl, flags);
+	if (!p->dl.dl_throttled && !dl_server(&p->dl))
+		dequeue_pushable_dl_task(rq, p);
 }
 
 /*
@@ -1933,12 +2041,12 @@ static void wakeup_preempt_dl(struct rq *rq, struct task_struct *p,
 }
 
 #ifdef CONFIG_SCHED_HRTICK
-static void start_hrtick_dl(struct rq *rq, struct task_struct *p)
+static void start_hrtick_dl(struct rq *rq, struct sched_dl_entity *dl_se)
 {
-	hrtick_start(rq, p->dl.runtime);
+	hrtick_start(rq, dl_se->runtime);
 }
 #else /* !CONFIG_SCHED_HRTICK */
-static void start_hrtick_dl(struct rq *rq, struct task_struct *p)
+static void start_hrtick_dl(struct rq *rq, struct sched_dl_entity *dl_se)
 {
 }
 #endif
@@ -1958,9 +2066,6 @@ static void set_next_task_dl(struct rq *rq, struct task_struct *p, bool first)
 	if (!first)
 		return;
 
-	if (hrtick_enabled_dl(rq))
-		start_hrtick_dl(rq, p);
-
 	if (rq->curr->sched_class != &dl_sched_class)
 		update_dl_rq_load_avg(rq_clock_pelt(rq), rq, 0);
 
@@ -1983,12 +2088,25 @@ static struct task_struct *pick_task_dl(struct rq *rq)
 	struct dl_rq *dl_rq = &rq->dl;
 	struct task_struct *p;
 
+again:
 	if (!sched_dl_runnable(rq))
 		return NULL;
 
 	dl_se = pick_next_dl_entity(dl_rq);
 	WARN_ON_ONCE(!dl_se);
-	p = dl_task_of(dl_se);
+
+	if (dl_server(dl_se)) {
+		p = dl_se->server_pick(dl_se);
+		if (!p) {
+			WARN_ON_ONCE(1);
+			dl_se->dl_yielded = 1;
+			update_curr_dl_se(rq, dl_se, 0);
+			goto again;
+		}
+		p->dl_server = dl_se;
+	} else {
+		p = dl_task_of(dl_se);
+	}
 
 	return p;
 }
@@ -1998,9 +2116,15 @@ static struct task_struct *pick_next_task_dl(struct rq *rq)
 	struct task_struct *p;
 
 	p = pick_task_dl(rq);
-	if (p)
+	if (!p)
+		return p;
+
+	if (!p->dl_server)
 		set_next_task_dl(rq, p, true);
 
+	if (hrtick_enabled(rq))
+		start_hrtick_dl(rq, &p->dl);
+
 	return p;
 }
 
@@ -2038,8 +2162,8 @@ static void task_tick_dl(struct rq *rq, struct task_struct *p, int queued)
 	 * be set and schedule() will start a new hrtick for the next task.
 	 */
 	if (hrtick_enabled_dl(rq) && queued && p->dl.runtime > 0 &&
-	    is_leftmost(p, &rq->dl))
-		start_hrtick_dl(rq, p);
+	    is_leftmost(&p->dl, &rq->dl))
+		start_hrtick_dl(rq, &p->dl);
 }
 
 static void task_fork_dl(struct task_struct *p)
@@ -2558,7 +2682,7 @@ static void switched_from_dl(struct rq *rq, struct task_struct *p)
 	 * will reset the task parameters.
 	 */
 	if (task_on_rq_queued(p) && p->dl.dl_runtime)
-		task_non_contending(p);
+		task_non_contending(&p->dl);
 
 	/*
 	 * In case a task is setscheduled out from SCHED_DEADLINE we need to
@@ -2966,10 +3090,8 @@ bool __checkparam_dl(const struct sched_attr *attr)
 /*
  * This function clears the sched_dl_entity static params.
  */
-void __dl_clear_params(struct task_struct *p)
+static void __dl_clear_params(struct sched_dl_entity *dl_se)
 {
-	struct sched_dl_entity *dl_se = &p->dl;
-
 	dl_se->dl_runtime		= 0;
 	dl_se->dl_deadline		= 0;
 	dl_se->dl_period		= 0;
@@ -2981,12 +3103,21 @@ void __dl_clear_params(struct task_struct *p)
 	dl_se->dl_yielded		= 0;
 	dl_se->dl_non_contending	= 0;
 	dl_se->dl_overrun		= 0;
+	dl_se->dl_server		= 0;
 
 #ifdef CONFIG_RT_MUTEXES
 	dl_se->pi_se			= dl_se;
 #endif
 }
 
+void init_dl_entity(struct sched_dl_entity *dl_se)
+{
+	RB_CLEAR_NODE(&dl_se->rb_node);
+	init_dl_task_timer(dl_se);
+	init_dl_inactive_task_timer(dl_se);
+	__dl_clear_params(dl_se);
+}
+
 bool dl_param_changed(struct task_struct *p, const struct sched_attr *attr)
 {
 	struct sched_dl_entity *dl_se = &p->dl;
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 4580a450700e..168eecc209b4 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -628,8 +628,8 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu)
 
 void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
 {
-	s64 left_vruntime = -1, min_vruntime, right_vruntime = -1, spread;
-	struct sched_entity *last, *first;
+	s64 left_vruntime = -1, min_vruntime, right_vruntime = -1, left_deadline = -1, spread;
+	struct sched_entity *last, *first, *root;
 	struct rq *rq = cpu_rq(cpu);
 	unsigned long flags;
 
@@ -644,15 +644,20 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
 			SPLIT_NS(cfs_rq->exec_clock));
 
 	raw_spin_rq_lock_irqsave(rq, flags);
+	root = __pick_root_entity(cfs_rq);
+	if (root)
+		left_vruntime = root->min_vruntime;
 	first = __pick_first_entity(cfs_rq);
 	if (first)
-		left_vruntime = first->vruntime;
+		left_deadline = first->deadline;
 	last = __pick_last_entity(cfs_rq);
 	if (last)
 		right_vruntime = last->vruntime;
 	min_vruntime = cfs_rq->min_vruntime;
 	raw_spin_rq_unlock_irqrestore(rq, flags);
 
+	SEQ_printf(m, "  .%-30s: %Ld.%06ld\n", "left_deadline",
+			SPLIT_NS(left_deadline));
 	SEQ_printf(m, "  .%-30s: %Ld.%06ld\n", "left_vruntime",
 			SPLIT_NS(left_vruntime));
 	SEQ_printf(m, "  .%-30s: %Ld.%06ld\n", "min_vruntime",
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index d7a3c63a2171..07f555857698 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -551,7 +551,11 @@ static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime)
 static inline bool entity_before(const struct sched_entity *a,
 				 const struct sched_entity *b)
 {
-	return (s64)(a->vruntime - b->vruntime) < 0;
+	/*
+	 * Tiebreak on vruntime seems unnecessary since it can
+	 * hardly happen.
+	 */
+	return (s64)(a->deadline - b->deadline) < 0;
 }
 
 static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se)
@@ -720,7 +724,7 @@ static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se)
  * Note: using 'avg_vruntime() > se->vruntime' is inacurate due
  *       to the loss in precision caused by the division.
  */
-int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se)
+static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime)
 {
 	struct sched_entity *curr = cfs_rq->curr;
 	s64 avg = cfs_rq->avg_vruntime;
@@ -733,7 +737,12 @@ int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se)
 		load += weight;
 	}
 
-	return avg >= entity_key(cfs_rq, se) * load;
+	return avg >= (s64)(vruntime - cfs_rq->min_vruntime) * load;
+}
+
+int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se)
+{
+	return vruntime_eligible(cfs_rq, se->vruntime);
 }
 
 static u64 __update_min_vruntime(struct cfs_rq *cfs_rq, u64 vruntime)
@@ -752,9 +761,8 @@ static u64 __update_min_vruntime(struct cfs_rq *cfs_rq, u64 vruntime)
 
 static void update_min_vruntime(struct cfs_rq *cfs_rq)
 {
-	struct sched_entity *se = __pick_first_entity(cfs_rq);
+	struct sched_entity *se = __pick_root_entity(cfs_rq);
 	struct sched_entity *curr = cfs_rq->curr;
-
 	u64 vruntime = cfs_rq->min_vruntime;
 
 	if (curr) {
@@ -766,9 +774,9 @@ static void update_min_vruntime(struct cfs_rq *cfs_rq)
 
 	if (se) {
 		if (!curr)
-			vruntime = se->vruntime;
+			vruntime = se->min_vruntime;
 		else
-			vruntime = min_vruntime(vruntime, se->vruntime);
+			vruntime = min_vruntime(vruntime, se->min_vruntime);
 	}
 
 	/* ensure we never gain time by being placed backwards. */
@@ -781,34 +789,34 @@ static inline bool __entity_less(struct rb_node *a, const struct rb_node *b)
 	return entity_before(__node_2_se(a), __node_2_se(b));
 }
 
-#define deadline_gt(field, lse, rse) ({ (s64)((lse)->field - (rse)->field) > 0; })
+#define vruntime_gt(field, lse, rse) ({ (s64)((lse)->field - (rse)->field) > 0; })
 
-static inline void __update_min_deadline(struct sched_entity *se, struct rb_node *node)
+static inline void __min_vruntime_update(struct sched_entity *se, struct rb_node *node)
 {
 	if (node) {
 		struct sched_entity *rse = __node_2_se(node);
-		if (deadline_gt(min_deadline, se, rse))
-			se->min_deadline = rse->min_deadline;
+		if (vruntime_gt(min_vruntime, se, rse))
+			se->min_vruntime = rse->min_vruntime;
 	}
 }
 
 /*
- * se->min_deadline = min(se->deadline, left->min_deadline, right->min_deadline)
+ * se->min_vruntime = min(se->vruntime, {left,right}->min_vruntime)
  */
-static inline bool min_deadline_update(struct sched_entity *se, bool exit)
+static inline bool min_vruntime_update(struct sched_entity *se, bool exit)
 {
-	u64 old_min_deadline = se->min_deadline;
+	u64 old_min_vruntime = se->min_vruntime;
 	struct rb_node *node = &se->run_node;
 
-	se->min_deadline = se->deadline;
-	__update_min_deadline(se, node->rb_right);
-	__update_min_deadline(se, node->rb_left);
+	se->min_vruntime = se->vruntime;
+	__min_vruntime_update(se, node->rb_right);
+	__min_vruntime_update(se, node->rb_left);
 
-	return se->min_deadline == old_min_deadline;
+	return se->min_vruntime == old_min_vruntime;
 }
 
-RB_DECLARE_CALLBACKS(static, min_deadline_cb, struct sched_entity,
-		     run_node, min_deadline, min_deadline_update);
+RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity,
+		     run_node, min_vruntime, min_vruntime_update);
 
 /*
  * Enqueue an entity into the rb-tree:
@@ -816,18 +824,28 @@ RB_DECLARE_CALLBACKS(static, min_deadline_cb, struct sched_entity,
 static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
 	avg_vruntime_add(cfs_rq, se);
-	se->min_deadline = se->deadline;
+	se->min_vruntime = se->vruntime;
 	rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline,
-				__entity_less, &min_deadline_cb);
+				__entity_less, &min_vruntime_cb);
 }
 
 static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
 	rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline,
-				  &min_deadline_cb);
+				  &min_vruntime_cb);
 	avg_vruntime_sub(cfs_rq, se);
 }
 
+struct sched_entity *__pick_root_entity(struct cfs_rq *cfs_rq)
+{
+	struct rb_node *root = cfs_rq->tasks_timeline.rb_root.rb_node;
+
+	if (!root)
+		return NULL;
+
+	return __node_2_se(root);
+}
+
 struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq)
 {
 	struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline);
@@ -850,23 +868,29 @@ struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq)
  *     with the earliest virtual deadline.
  *
  * We can do this in O(log n) time due to an augmented RB-tree. The
- * tree keeps the entries sorted on service, but also functions as a
- * heap based on the deadline by keeping:
+ * tree keeps the entries sorted on deadline, but also functions as a
+ * heap based on the vruntime by keeping:
  *
- *  se->min_deadline = min(se->deadline, se->{left,right}->min_deadline)
+ *  se->min_vruntime = min(se->vruntime, se->{left,right}->min_vruntime)
  *
- * Which allows an EDF like search on (sub)trees.
+ * Which allows tree pruning through eligibility.
  */
-static struct sched_entity *__pick_eevdf(struct cfs_rq *cfs_rq)
+static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq)
 {
 	struct rb_node *node = cfs_rq->tasks_timeline.rb_root.rb_node;
+	struct sched_entity *se = __pick_first_entity(cfs_rq);
 	struct sched_entity *curr = cfs_rq->curr;
 	struct sched_entity *best = NULL;
-	struct sched_entity *best_left = NULL;
+
+	/*
+	 * We can safely skip eligibility check if there is only one entity
+	 * in this cfs_rq, saving some cycles.
+	 */
+	if (cfs_rq->nr_running == 1)
+		return curr && curr->on_rq ? curr : se;
 
 	if (curr && (!curr->on_rq || !entity_eligible(cfs_rq, curr)))
 		curr = NULL;
-	best = curr;
 
 	/*
 	 * Once selected, run a task until it either becomes non-eligible or
@@ -875,95 +899,45 @@ static struct sched_entity *__pick_eevdf(struct cfs_rq *cfs_rq)
 	if (sched_feat(RUN_TO_PARITY) && curr && curr->vlag == curr->deadline)
 		return curr;
 
+	/* Pick the leftmost entity if it's eligible */
+	if (se && entity_eligible(cfs_rq, se)) {
+		best = se;
+		goto found;
+	}
+
+	/* Heap search for the EEVD entity */
 	while (node) {
-		struct sched_entity *se = __node_2_se(node);
+		struct rb_node *left = node->rb_left;
 
 		/*
-		 * If this entity is not eligible, try the left subtree.
+		 * Eligible entities in left subtree are always better
+		 * choices, since they have earlier deadlines.
 		 */
-		if (!entity_eligible(cfs_rq, se)) {
-			node = node->rb_left;
+		if (left && vruntime_eligible(cfs_rq,
+					__node_2_se(left)->min_vruntime)) {
+			node = left;
 			continue;
 		}
 
-		/*
-		 * Now we heap search eligible trees for the best (min_)deadline
-		 */
-		if (!best || deadline_gt(deadline, best, se))
-			best = se;
+		se = __node_2_se(node);
 
 		/*
-		 * Every se in a left branch is eligible, keep track of the
-		 * branch with the best min_deadline
+		 * The left subtree either is empty or has no eligible
+		 * entity, so check the current node since it is the one
+		 * with earliest deadline that might be eligible.
 		 */
-		if (node->rb_left) {
-			struct sched_entity *left = __node_2_se(node->rb_left);
-
-			if (!best_left || deadline_gt(min_deadline, best_left, left))
-				best_left = left;
-
-			/*
-			 * min_deadline is in the left branch. rb_left and all
-			 * descendants are eligible, so immediately switch to the second
-			 * loop.
-			 */
-			if (left->min_deadline == se->min_deadline)
-				break;
-		}
-
-		/* min_deadline is at this node, no need to look right */
-		if (se->deadline == se->min_deadline)
+		if (entity_eligible(cfs_rq, se)) {
+			best = se;
 			break;
-
-		/* else min_deadline is in the right branch. */
-		node = node->rb_right;
-	}
-
-	/*
-	 * We ran into an eligible node which is itself the best.
-	 * (Or nr_running == 0 and both are NULL)
-	 */
-	if (!best_left || (s64)(best_left->min_deadline - best->deadline) > 0)
-		return best;
-
-	/*
-	 * Now best_left and all of its children are eligible, and we are just
-	 * looking for deadline == min_deadline
-	 */
-	node = &best_left->run_node;
-	while (node) {
-		struct sched_entity *se = __node_2_se(node);
-
-		/* min_deadline is the current node */
-		if (se->deadline == se->min_deadline)
-			return se;
-
-		/* min_deadline is in the left branch */
-		if (node->rb_left &&
-		    __node_2_se(node->rb_left)->min_deadline == se->min_deadline) {
-			node = node->rb_left;
-			continue;
 		}
 
-		/* else min_deadline is in the right branch */
 		node = node->rb_right;
 	}
-	return NULL;
-}
-
-static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq)
-{
-	struct sched_entity *se = __pick_eevdf(cfs_rq);
-
-	if (!se) {
-		struct sched_entity *left = __pick_first_entity(cfs_rq);
-		if (left) {
-			pr_err("EEVDF scheduling fail, picking leftmost\n");
-			return left;
-		}
-	}
+found:
+	if (!best || (curr && entity_before(curr, best)))
+		best = curr;
 
-	return se;
+	return best;
 }
 
 #ifdef CONFIG_SCHED_DEBUG
@@ -1129,23 +1103,17 @@ static void update_tg_load_avg(struct cfs_rq *cfs_rq)
 }
 #endif /* CONFIG_SMP */
 
-/*
- * Update the current task's runtime statistics.
- */
-static void update_curr(struct cfs_rq *cfs_rq)
+static s64 update_curr_se(struct rq *rq, struct sched_entity *curr)
 {
-	struct sched_entity *curr = cfs_rq->curr;
-	u64 now = rq_clock_task(rq_of(cfs_rq));
-	u64 delta_exec;
-
-	if (unlikely(!curr))
-		return;
+	u64 now = rq_clock_task(rq);
+	s64 delta_exec;
 
 	delta_exec = now - curr->exec_start;
-	if (unlikely((s64)delta_exec <= 0))
-		return;
+	if (unlikely(delta_exec <= 0))
+		return delta_exec;
 
 	curr->exec_start = now;
+	curr->sum_exec_runtime += delta_exec;
 
 	if (schedstat_enabled()) {
 		struct sched_statistics *stats;
@@ -1155,20 +1123,54 @@ static void update_curr(struct cfs_rq *cfs_rq)
 				max(delta_exec, stats->exec_max));
 	}
 
-	curr->sum_exec_runtime += delta_exec;
-	schedstat_add(cfs_rq->exec_clock, delta_exec);
+	return delta_exec;
+}
+
+static inline void update_curr_task(struct task_struct *p, s64 delta_exec)
+{
+	trace_sched_stat_runtime(p, delta_exec);
+	account_group_exec_runtime(p, delta_exec);
+	cgroup_account_cputime(p, delta_exec);
+	if (p->dl_server)
+		dl_server_update(p->dl_server, delta_exec);
+}
+
+/*
+ * Used by other classes to account runtime.
+ */
+s64 update_curr_common(struct rq *rq)
+{
+	struct task_struct *curr = rq->curr;
+	s64 delta_exec;
+
+	delta_exec = update_curr_se(rq, &curr->se);
+	if (likely(delta_exec > 0))
+		update_curr_task(curr, delta_exec);
+
+	return delta_exec;
+}
+
+/*
+ * Update the current task's runtime statistics.
+ */
+static void update_curr(struct cfs_rq *cfs_rq)
+{
+	struct sched_entity *curr = cfs_rq->curr;
+	s64 delta_exec;
+
+	if (unlikely(!curr))
+		return;
+
+	delta_exec = update_curr_se(rq_of(cfs_rq), curr);
+	if (unlikely(delta_exec <= 0))
+		return;
 
 	curr->vruntime += calc_delta_fair(delta_exec, curr);
 	update_deadline(cfs_rq, curr);
 	update_min_vruntime(cfs_rq);
 
-	if (entity_is_task(curr)) {
-		struct task_struct *curtask = task_of(curr);
-
-		trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime);
-		cgroup_account_cputime(curtask, delta_exec);
-		account_group_exec_runtime(curtask, delta_exec);
-	}
+	if (entity_is_task(curr))
+		update_curr_task(task_of(curr), delta_exec);
 
 	account_cfs_rq_runtime(cfs_rq, delta_exec);
 }
@@ -3164,7 +3166,7 @@ static bool vma_is_accessed(struct mm_struct *mm, struct vm_area_struct *vma)
 	 * This is also done to avoid any side effect of task scanning
 	 * amplifying the unfairness of disjoint set of VMAs' access.
 	 */
-	if (READ_ONCE(current->mm->numa_scan_seq) < 2)
+	if ((READ_ONCE(current->mm->numa_scan_seq) - vma->numab_state->start_scan_seq) < 2)
 		return true;
 
 	pids = vma->numab_state->pids_active[0] | vma->numab_state->pids_active[1];
@@ -3307,6 +3309,8 @@ static void task_numa_work(struct callback_head *work)
 			if (!vma->numab_state)
 				continue;
 
+			vma->numab_state->start_scan_seq = mm->numa_scan_seq;
+
 			vma->numab_state->next_scan = now +
 				msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
 
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
index 565f8374ddbb..31231925f1ec 100644
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -258,6 +258,36 @@ static void do_idle(void)
 	while (!need_resched()) {
 		rmb();
 
+		/*
+		 * Interrupts shouldn't be re-enabled from that point on until
+		 * the CPU sleeping instruction is reached. Otherwise an interrupt
+		 * may fire and queue a timer that would be ignored until the CPU
+		 * wakes from the sleeping instruction. And testing need_resched()
+		 * doesn't tell about pending needed timer reprogram.
+		 *
+		 * Several cases to consider:
+		 *
+		 * - SLEEP-UNTIL-PENDING-INTERRUPT based instructions such as
+		 *   "wfi" or "mwait" are fine because they can be entered with
+		 *   interrupt disabled.
+		 *
+		 * - sti;mwait() couple is fine because the interrupts are
+		 *   re-enabled only upon the execution of mwait, leaving no gap
+		 *   in-between.
+		 *
+		 * - ROLLBACK based idle handlers with the sleeping instruction
+		 *   called with interrupts enabled are NOT fine. In this scheme
+		 *   when the interrupt detects it has interrupted an idle handler,
+		 *   it rolls back to its beginning which performs the
+		 *   need_resched() check before re-executing the sleeping
+		 *   instruction. This can leak a pending needed timer reprogram.
+		 *   If such a scheme is really mandatory due to the lack of an
+		 *   appropriate CPU sleeping instruction, then a FAST-FORWARD
+		 *   must instead be applied: when the interrupt detects it has
+		 *   interrupted an idle handler, it must resume to the end of
+		 *   this idle handler so that the generic idle loop is iterated
+		 *   again to reprogram the tick.
+		 */
 		local_irq_disable();
 
 		if (cpu_is_offline(cpu)) {
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 6aaf0a3d6081..3261b067b67e 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -1002,24 +1002,15 @@ static void update_curr_rt(struct rq *rq)
 {
 	struct task_struct *curr = rq->curr;
 	struct sched_rt_entity *rt_se = &curr->rt;
-	u64 delta_exec;
-	u64 now;
+	s64 delta_exec;
 
 	if (curr->sched_class != &rt_sched_class)
 		return;
 
-	now = rq_clock_task(rq);
-	delta_exec = now - curr->se.exec_start;
-	if (unlikely((s64)delta_exec <= 0))
+	delta_exec = update_curr_common(rq);
+	if (unlikely(delta_exec <= 0))
 		return;
 
-	schedstat_set(curr->stats.exec_max,
-		      max(curr->stats.exec_max, delta_exec));
-
-	trace_sched_stat_runtime(curr, delta_exec, 0);
-
-	update_current_exec_runtime(curr, now, delta_exec);
-
 	if (!rt_bandwidth_enabled())
 		return;
 
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 2e5a95486a42..8a70d51ffa33 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -273,8 +273,6 @@ struct rt_bandwidth {
 	unsigned int		rt_period_active;
 };
 
-void __dl_clear_params(struct task_struct *p);
-
 static inline int dl_bandwidth_enabled(void)
 {
 	return sysctl_sched_rt_runtime >= 0;
@@ -315,6 +313,33 @@ extern bool dl_param_changed(struct task_struct *p, const struct sched_attr *att
 extern int  dl_cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);
 extern int  dl_bw_check_overflow(int cpu);
 
+/*
+ * SCHED_DEADLINE supports servers (nested scheduling) with the following
+ * interface:
+ *
+ *   dl_se::rq -- runqueue we belong to.
+ *
+ *   dl_se::server_has_tasks() -- used on bandwidth enforcement; we 'stop' the
+ *                                server when it runs out of tasks to run.
+ *
+ *   dl_se::server_pick() -- nested pick_next_task(); we yield the period if this
+ *                           returns NULL.
+ *
+ *   dl_server_update() -- called from update_curr_common(), propagates runtime
+ *                         to the server.
+ *
+ *   dl_server_start()
+ *   dl_server_stop()  -- start/stop the server when it has (no) tasks.
+ *
+ *   dl_server_init() -- initializes the server.
+ */
+extern void dl_server_update(struct sched_dl_entity *dl_se, s64 delta_exec);
+extern void dl_server_start(struct sched_dl_entity *dl_se);
+extern void dl_server_stop(struct sched_dl_entity *dl_se);
+extern void dl_server_init(struct sched_dl_entity *dl_se, struct rq *rq,
+		    dl_server_has_tasks_f has_tasks,
+		    dl_server_pick_f pick);
+
 #ifdef CONFIG_CGROUP_SCHED
 
 struct cfs_rq;
@@ -2179,6 +2204,10 @@ extern const u32		sched_prio_to_wmult[40];
  * MOVE - paired with SAVE/RESTORE, explicitly does not preserve the location
  *        in the runqueue.
  *
+ * NOCLOCK - skip the update_rq_clock() (avoids double updates)
+ *
+ * MIGRATION - p->on_rq == TASK_ON_RQ_MIGRATING (used for DEADLINE)
+ *
  * ENQUEUE_HEAD      - place at front of runqueue (tail if not specified)
  * ENQUEUE_REPLENISH - CBS (replenish runtime and postpone deadline)
  * ENQUEUE_MIGRATED  - the task was migrated during wakeup
@@ -2189,6 +2218,7 @@ extern const u32		sched_prio_to_wmult[40];
 #define DEQUEUE_SAVE		0x02 /* Matches ENQUEUE_RESTORE */
 #define DEQUEUE_MOVE		0x04 /* Matches ENQUEUE_MOVE */
 #define DEQUEUE_NOCLOCK		0x08 /* Matches ENQUEUE_NOCLOCK */
+#define DEQUEUE_MIGRATING	0x100 /* Matches ENQUEUE_MIGRATING */
 
 #define ENQUEUE_WAKEUP		0x01
 #define ENQUEUE_RESTORE		0x02
@@ -2203,6 +2233,7 @@ extern const u32		sched_prio_to_wmult[40];
 #define ENQUEUE_MIGRATED	0x00
 #endif
 #define ENQUEUE_INITIAL		0x80
+#define ENQUEUE_MIGRATING	0x100
 
 #define RETRY_TASK		((void *)-1UL)
 
@@ -2212,6 +2243,8 @@ struct affinity_context {
 	unsigned int flags;
 };
 
+extern s64 update_curr_common(struct rq *rq);
+
 struct sched_class {
 
 #ifdef CONFIG_UCLAMP_TASK
@@ -2425,8 +2458,7 @@ extern struct rt_bandwidth def_rt_bandwidth;
 extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime);
 extern bool sched_rt_bandwidth_account(struct rt_rq *rt_rq);
 
-extern void init_dl_task_timer(struct sched_dl_entity *dl_se);
-extern void init_dl_inactive_task_timer(struct sched_dl_entity *dl_se);
+extern void init_dl_entity(struct sched_dl_entity *dl_se);
 
 #define BW_SHIFT		20
 #define BW_UNIT			(1 << BW_SHIFT)
@@ -2822,6 +2854,7 @@ DEFINE_LOCK_GUARD_2(double_rq_lock, struct rq,
 		    double_rq_lock(_T->lock, _T->lock2),
 		    double_rq_unlock(_T->lock, _T->lock2))
 
+extern struct sched_entity *__pick_root_entity(struct cfs_rq *cfs_rq);
 extern struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq);
 extern struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq);
 
@@ -3261,16 +3294,6 @@ extern int sched_dynamic_mode(const char *str);
 extern void sched_dynamic_update(int mode);
 #endif
 
-static inline void update_current_exec_runtime(struct task_struct *curr,
-						u64 now, u64 delta_exec)
-{
-	curr->se.sum_exec_runtime += delta_exec;
-	account_group_exec_runtime(curr, delta_exec);
-
-	curr->se.exec_start = now;
-	cgroup_account_cputime(curr, delta_exec);
-}
-
 #ifdef CONFIG_SCHED_MM_CID
 
 #define SCHED_MM_CID_PERIOD_NS	(100ULL * 1000000)	/* 100ms */
diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c
index 6cf7304e6449..b1b8fe61c532 100644
--- a/kernel/sched/stop_task.c
+++ b/kernel/sched/stop_task.c
@@ -70,18 +70,7 @@ static void yield_task_stop(struct rq *rq)
 
 static void put_prev_task_stop(struct rq *rq, struct task_struct *prev)
 {
-	struct task_struct *curr = rq->curr;
-	u64 now, delta_exec;
-
-	now = rq_clock_task(rq);
-	delta_exec = now - curr->se.exec_start;
-	if (unlikely((s64)delta_exec < 0))
-		delta_exec = 0;
-
-	schedstat_set(curr->stats.exec_max,
-		      max(curr->stats.exec_max, delta_exec));
-
-	update_current_exec_runtime(curr, now, delta_exec);
+	update_curr_common(rq);
 }
 
 /*
diff --git a/kernel/signal.c b/kernel/signal.c
index 47a7602dfe8d..c9c57d053ce4 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -171,16 +171,6 @@ static bool recalc_sigpending_tsk(struct task_struct *t)
 	return false;
 }
 
-/*
- * After recalculating TIF_SIGPENDING, we need to make sure the task wakes up.
- * This is superfluous when called on current, the wakeup is a harmless no-op.
- */
-void recalc_sigpending_and_wake(struct task_struct *t)
-{
-	if (recalc_sigpending_tsk(t))
-		signal_wake_up(t, 0);
-}
-
 void recalc_sigpending(void)
 {
 	if (!recalc_sigpending_tsk(current) && !freezing(current))
@@ -1348,10 +1338,8 @@ force_sig_info_to_task(struct kernel_siginfo *info, struct task_struct *t,
 		action->sa.sa_handler = SIG_DFL;
 		if (handler == HANDLER_EXIT)
 			action->sa.sa_flags |= SA_IMMUTABLE;
-		if (blocked) {
+		if (blocked)
 			sigdelset(&t->blocked, sig);
-			recalc_sigpending_and_wake(t);
-		}
 	}
 	/*
 	 * Don't clear SIGNAL_UNKILLABLE for traced tasks, users won't expect
@@ -1361,6 +1349,9 @@ force_sig_info_to_task(struct kernel_siginfo *info, struct task_struct *t,
 	    (!t->ptrace || (handler == HANDLER_EXIT)))
 		t->signal->flags &= ~SIGNAL_UNKILLABLE;
 	ret = send_signal_locked(sig, info, t, PIDTYPE_PID);
+	/* This can happen if the signal was already pending and blocked */
+	if (!task_sigpending(t))
+		signal_wake_up(t, 0);
 	spin_unlock_irqrestore(&t->sighand->siglock, flags);
 
 	return ret;
@@ -1376,12 +1367,12 @@ int force_sig_info(struct kernel_siginfo *info)
  */
 int zap_other_threads(struct task_struct *p)
 {
-	struct task_struct *t = p;
+	struct task_struct *t;
 	int count = 0;
 
 	p->signal->group_stop_count = 0;
 
-	while_each_thread(p, t) {
+	for_other_threads(p, t) {
 		task_clear_jobctl_pending(t, JOBCTL_PENDING_MASK);
 		/* Don't require de_thread to wait for the vhost_worker */
 		if ((t->flags & (PF_IO_WORKER | PF_USER_WORKER)) != PF_USER_WORKER)
@@ -2465,12 +2456,10 @@ static bool do_signal_stop(int signr)
 			sig->group_exit_code = signr;
 
 		sig->group_stop_count = 0;
-
 		if (task_set_jobctl_pending(current, signr | gstop))
 			sig->group_stop_count++;
 
-		t = current;
-		while_each_thread(current, t) {
+		for_other_threads(current, t) {
 			/*
 			 * Setting state to TASK_STOPPED for a group
 			 * stop is always done with the siglock held,
@@ -2966,8 +2955,7 @@ static void retarget_shared_pending(struct task_struct *tsk, sigset_t *which)
 	if (sigisemptyset(&retarget))
 		return;
 
-	t = tsk;
-	while_each_thread(tsk, t) {
+	for_other_threads(tsk, t) {
 		if (t->flags & PF_EXITING)
 			continue;
 
diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c
index e1a6e3c675c0..9fa5989bf2ce 100644
--- a/kernel/sys_ni.c
+++ b/kernel/sys_ni.c
@@ -171,6 +171,9 @@ COND_SYSCALL(landlock_add_rule);
 COND_SYSCALL(landlock_restrict_self);
 COND_SYSCALL(fadvise64_64);
 COND_SYSCALL_COMPAT(fadvise64_64);
+COND_SYSCALL(lsm_get_self_attr);
+COND_SYSCALL(lsm_set_self_attr);
+COND_SYSCALL(lsm_list_modules);
 
 /* CONFIG_MMU only */
 COND_SYSCALL(swapon);
diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index 84e8a0f6e4e0..f0b8b7c29126 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -1376,6 +1376,8 @@ __bpf_kfunc int bpf_verify_pkcs7_signature(struct bpf_dynptr_kern *data_ptr,
 			       struct bpf_dynptr_kern *sig_ptr,
 			       struct bpf_key *trusted_keyring)
 {
+	const void *data, *sig;
+	u32 data_len, sig_len;
 	int ret;
 
 	if (trusted_keyring->has_ref) {
@@ -1392,10 +1394,12 @@ __bpf_kfunc int bpf_verify_pkcs7_signature(struct bpf_dynptr_kern *data_ptr,
 			return ret;
 	}
 
-	return verify_pkcs7_signature(data_ptr->data,
-				      __bpf_dynptr_size(data_ptr),
-				      sig_ptr->data,
-				      __bpf_dynptr_size(sig_ptr),
+	data_len = __bpf_dynptr_size(data_ptr);
+	data = __bpf_dynptr_data(data_ptr, data_len);
+	sig_len = __bpf_dynptr_size(sig_ptr);
+	sig = __bpf_dynptr_data(sig_ptr, sig_len);
+
+	return verify_pkcs7_signature(data, data_len, sig, sig_len,
 				      trusted_keyring->key,
 				      VERIFYING_UNSPECIFIED_SIGNATURE, NULL,
 				      NULL);
diff --git a/kernel/ucount.c b/kernel/ucount.c
index 4aa6166cb856..e188c25ed2b3 100644
--- a/kernel/ucount.c
+++ b/kernel/ucount.c
@@ -213,18 +213,16 @@ void put_ucounts(struct ucounts *ucounts)
 	}
 }
 
-static inline bool atomic_long_inc_below(atomic_long_t *v, int u)
+static inline bool atomic_long_inc_below(atomic_long_t *v, long u)
 {
-	long c, old;
-	c = atomic_long_read(v);
-	for (;;) {
+	long c = atomic_long_read(v);
+
+	do {
 		if (unlikely(c >= u))
 			return false;
-		old = atomic_long_cmpxchg(v, c, c+1);
-		if (likely(old == c))
-			return true;
-		c = old;
-	}
+	} while (!atomic_long_try_cmpxchg(v, &c, c+1));
+
+	return true;
 }
 
 struct ucounts *inc_ucount(struct user_namespace *ns, kuid_t uid,
diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c
index eabe8bcc7042..625101249e4d 100644
--- a/kernel/user_namespace.c
+++ b/kernel/user_namespace.c
@@ -231,7 +231,7 @@ void __put_user_ns(struct user_namespace *ns)
 }
 EXPORT_SYMBOL(__put_user_ns);
 
-/**
+/*
  * struct idmap_key - holds the information necessary to find an idmapping in a
  * sorted idmap array. It is passed to cmp_map_id() as first argument.
  */
@@ -241,7 +241,7 @@ struct idmap_key {
 	u32 count; /* == 0 unless used with map_id_range_down() */
 };
 
-/**
+/*
  * cmp_map_id - Function to be passed to bsearch() to find the requested
  * idmapping. Expects struct idmap_key to be passed via @k.
  */
@@ -271,7 +271,7 @@ static int cmp_map_id(const void *k, const void *e)
 	return 1;
 }
 
-/**
+/*
  * map_id_range_down_max - Find idmap via binary search in ordered idmap array.
  * Can only be called if number of mappings exceeds UID_GID_MAP_MAX_BASE_EXTENTS.
  */
@@ -288,7 +288,7 @@ map_id_range_down_max(unsigned extents, struct uid_gid_map *map, u32 id, u32 cou
 		       sizeof(struct uid_gid_extent), cmp_map_id);
 }
 
-/**
+/*
  * map_id_range_down_base - Find idmap via binary search in static extent array.
  * Can only be called if number of mappings is equal or less than
  * UID_GID_MAP_MAX_BASE_EXTENTS.
@@ -337,7 +337,7 @@ static u32 map_id_down(struct uid_gid_map *map, u32 id)
 	return map_id_range_down(map, id, 1);
 }
 
-/**
+/*
  * map_id_up_base - Find idmap via binary search in static extent array.
  * Can only be called if number of mappings is equal or less than
  * UID_GID_MAP_MAX_BASE_EXTENTS.
@@ -358,7 +358,7 @@ map_id_up_base(unsigned extents, struct uid_gid_map *map, u32 id)
 	return NULL;
 }
 
-/**
+/*
  * map_id_up_max - Find idmap via binary search in ordered idmap array.
  * Can only be called if number of mappings exceeds UID_GID_MAP_MAX_BASE_EXTENTS.
  */
@@ -770,7 +770,7 @@ static bool mappings_overlap(struct uid_gid_map *new_map,
 	return false;
 }
 
-/**
+/*
  * insert_extent - Safely insert a new idmap extent into struct uid_gid_map.
  * Takes care to allocate a 4K block of memory if the number of mappings exceeds
  * UID_GID_MAP_MAX_BASE_EXTENTS.
@@ -839,7 +839,7 @@ static int cmp_extents_reverse(const void *a, const void *b)
 	return 0;
 }
 
-/**
+/*
  * sort_idmaps - Sorts an array of idmap entries.
  * Can only be called if number of mappings exceeds UID_GID_MAP_MAX_BASE_EXTENTS.
  */
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 6e578f576a6f..2fc585d3d6ca 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -381,6 +381,12 @@ static bool workqueue_freezing;		/* PL: have wqs started freezing? */
 /* PL&A: allowable cpus for unbound wqs and work items */
 static cpumask_var_t wq_unbound_cpumask;
 
+/* PL: user requested unbound cpumask via sysfs */
+static cpumask_var_t wq_requested_unbound_cpumask;
+
+/* PL: isolated cpumask to be excluded from unbound cpumask */
+static cpumask_var_t wq_isolated_cpumask;
+
 /* for further constrain wq_unbound_cpumask by cmdline parameter*/
 static struct cpumask wq_cmdline_cpumask __initdata;
 
@@ -4411,19 +4417,6 @@ static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx)
 	mutex_unlock(&ctx->wq->mutex);
 }
 
-static void apply_wqattrs_lock(void)
-{
-	/* CPUs should stay stable across pwq creations and installations */
-	cpus_read_lock();
-	mutex_lock(&wq_pool_mutex);
-}
-
-static void apply_wqattrs_unlock(void)
-{
-	mutex_unlock(&wq_pool_mutex);
-	cpus_read_unlock();
-}
-
 static int apply_workqueue_attrs_locked(struct workqueue_struct *wq,
 					const struct workqueue_attrs *attrs)
 {
@@ -5828,39 +5821,40 @@ static int workqueue_apply_unbound_cpumask(const cpumask_var_t unbound_cpumask)
 }
 
 /**
- *  workqueue_set_unbound_cpumask - Set the low-level unbound cpumask
- *  @cpumask: the cpumask to set
- *
- *  The low-level workqueues cpumask is a global cpumask that limits
- *  the affinity of all unbound workqueues.  This function check the @cpumask
- *  and apply it to all unbound workqueues and updates all pwqs of them.
+ * workqueue_unbound_exclude_cpumask - Exclude given CPUs from unbound cpumask
+ * @exclude_cpumask: the cpumask to be excluded from wq_unbound_cpumask
  *
- *  Return:	0	- Success
- *  		-EINVAL	- Invalid @cpumask
- *  		-ENOMEM	- Failed to allocate memory for attrs or pwqs.
+ * This function can be called from cpuset code to provide a set of isolated
+ * CPUs that should be excluded from wq_unbound_cpumask. The caller must hold
+ * either cpus_read_lock or cpus_write_lock.
  */
-int workqueue_set_unbound_cpumask(cpumask_var_t cpumask)
+int workqueue_unbound_exclude_cpumask(cpumask_var_t exclude_cpumask)
 {
-	int ret = -EINVAL;
+	cpumask_var_t cpumask;
+	int ret = 0;
+
+	if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
+		return -ENOMEM;
+
+	lockdep_assert_cpus_held();
+	mutex_lock(&wq_pool_mutex);
+
+	/* Save the current isolated cpumask & export it via sysfs */
+	cpumask_copy(wq_isolated_cpumask, exclude_cpumask);
 
 	/*
-	 * Not excluding isolated cpus on purpose.
-	 * If the user wishes to include them, we allow that.
+	 * If the operation fails, it will fall back to
+	 * wq_requested_unbound_cpumask which is initially set to
+	 * (HK_TYPE_WQ ∩ HK_TYPE_DOMAIN) house keeping mask and rewritten
+	 * by any subsequent write to workqueue/cpumask sysfs file.
 	 */
-	cpumask_and(cpumask, cpumask, cpu_possible_mask);
-	if (!cpumask_empty(cpumask)) {
-		apply_wqattrs_lock();
-		if (cpumask_equal(cpumask, wq_unbound_cpumask)) {
-			ret = 0;
-			goto out_unlock;
-		}
-
+	if (!cpumask_andnot(cpumask, wq_requested_unbound_cpumask, exclude_cpumask))
+		cpumask_copy(cpumask, wq_requested_unbound_cpumask);
+	if (!cpumask_equal(cpumask, wq_unbound_cpumask))
 		ret = workqueue_apply_unbound_cpumask(cpumask);
 
-out_unlock:
-		apply_wqattrs_unlock();
-	}
-
+	mutex_unlock(&wq_pool_mutex);
+	free_cpumask_var(cpumask);
 	return ret;
 }
 
@@ -5982,6 +5976,19 @@ static struct attribute *wq_sysfs_attrs[] = {
 };
 ATTRIBUTE_GROUPS(wq_sysfs);
 
+static void apply_wqattrs_lock(void)
+{
+	/* CPUs should stay stable across pwq creations and installations */
+	cpus_read_lock();
+	mutex_lock(&wq_pool_mutex);
+}
+
+static void apply_wqattrs_unlock(void)
+{
+	mutex_unlock(&wq_pool_mutex);
+	cpus_read_unlock();
+}
+
 static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr,
 			    char *buf)
 {
@@ -6158,19 +6165,74 @@ static struct bus_type wq_subsys = {
 	.dev_groups			= wq_sysfs_groups,
 };
 
-static ssize_t wq_unbound_cpumask_show(struct device *dev,
-		struct device_attribute *attr, char *buf)
+/**
+ *  workqueue_set_unbound_cpumask - Set the low-level unbound cpumask
+ *  @cpumask: the cpumask to set
+ *
+ *  The low-level workqueues cpumask is a global cpumask that limits
+ *  the affinity of all unbound workqueues.  This function check the @cpumask
+ *  and apply it to all unbound workqueues and updates all pwqs of them.
+ *
+ *  Return:	0	- Success
+ *		-EINVAL	- Invalid @cpumask
+ *		-ENOMEM	- Failed to allocate memory for attrs or pwqs.
+ */
+static int workqueue_set_unbound_cpumask(cpumask_var_t cpumask)
+{
+	int ret = -EINVAL;
+
+	/*
+	 * Not excluding isolated cpus on purpose.
+	 * If the user wishes to include them, we allow that.
+	 */
+	cpumask_and(cpumask, cpumask, cpu_possible_mask);
+	if (!cpumask_empty(cpumask)) {
+		apply_wqattrs_lock();
+		cpumask_copy(wq_requested_unbound_cpumask, cpumask);
+		if (cpumask_equal(cpumask, wq_unbound_cpumask)) {
+			ret = 0;
+			goto out_unlock;
+		}
+
+		ret = workqueue_apply_unbound_cpumask(cpumask);
+
+out_unlock:
+		apply_wqattrs_unlock();
+	}
+
+	return ret;
+}
+
+static ssize_t __wq_cpumask_show(struct device *dev,
+		struct device_attribute *attr, char *buf, cpumask_var_t mask)
 {
 	int written;
 
 	mutex_lock(&wq_pool_mutex);
-	written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
-			    cpumask_pr_args(wq_unbound_cpumask));
+	written = scnprintf(buf, PAGE_SIZE, "%*pb\n", cpumask_pr_args(mask));
 	mutex_unlock(&wq_pool_mutex);
 
 	return written;
 }
 
+static ssize_t wq_unbound_cpumask_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	return __wq_cpumask_show(dev, attr, buf, wq_unbound_cpumask);
+}
+
+static ssize_t wq_requested_cpumask_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	return __wq_cpumask_show(dev, attr, buf, wq_requested_unbound_cpumask);
+}
+
+static ssize_t wq_isolated_cpumask_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	return __wq_cpumask_show(dev, attr, buf, wq_isolated_cpumask);
+}
+
 static ssize_t wq_unbound_cpumask_store(struct device *dev,
 		struct device_attribute *attr, const char *buf, size_t count)
 {
@@ -6188,9 +6250,13 @@ static ssize_t wq_unbound_cpumask_store(struct device *dev,
 	return ret ? ret : count;
 }
 
-static struct device_attribute wq_sysfs_cpumask_attr =
+static struct device_attribute wq_sysfs_cpumask_attrs[] = {
 	__ATTR(cpumask, 0644, wq_unbound_cpumask_show,
-	       wq_unbound_cpumask_store);
+	       wq_unbound_cpumask_store),
+	__ATTR(cpumask_requested, 0444, wq_requested_cpumask_show, NULL),
+	__ATTR(cpumask_isolated, 0444, wq_isolated_cpumask_show, NULL),
+	__ATTR_NULL,
+};
 
 static int __init wq_sysfs_init(void)
 {
@@ -6203,7 +6269,13 @@ static int __init wq_sysfs_init(void)
 
 	dev_root = bus_get_dev_root(&wq_subsys);
 	if (dev_root) {
-		err = device_create_file(dev_root, &wq_sysfs_cpumask_attr);
+		struct device_attribute *attr;
+
+		for (attr = wq_sysfs_cpumask_attrs; attr->attr.name; attr++) {
+			err = device_create_file(dev_root, attr);
+			if (err)
+				break;
+		}
 		put_device(dev_root);
 	}
 	return err;
@@ -6534,11 +6606,14 @@ void __init workqueue_init_early(void)
 	BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
 
 	BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));
+	BUG_ON(!alloc_cpumask_var(&wq_requested_unbound_cpumask, GFP_KERNEL));
+	BUG_ON(!zalloc_cpumask_var(&wq_isolated_cpumask, GFP_KERNEL));
 	cpumask_copy(wq_unbound_cpumask, housekeeping_cpumask(HK_TYPE_WQ));
 	cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, housekeeping_cpumask(HK_TYPE_DOMAIN));
 
 	if (!cpumask_empty(&wq_cmdline_cpumask))
 		cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, &wq_cmdline_cpumask);
+	cpumask_copy(wq_requested_unbound_cpumask, wq_unbound_cpumask);
 
 	pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
 
diff --git a/lib/Kconfig b/lib/Kconfig
index 3ea1c830efab..5ddda7c2ed9b 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -713,10 +713,20 @@ config ARCH_STACKWALK
 config STACKDEPOT
 	bool
 	select STACKTRACE
+	help
+	  Stack depot: stack trace storage that avoids duplication
 
 config STACKDEPOT_ALWAYS_INIT
 	bool
 	select STACKDEPOT
+	help
+	  Always initialize stack depot during early boot
+
+config STACKDEPOT_MAX_FRAMES
+	int "Maximum number of frames in trace saved in stack depot"
+	range 1 256
+	default 64
+	depends on STACKDEPOT
 
 config REF_TRACKER
 	bool
diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan
index fdca89c05745..935eda08b1e1 100644
--- a/lib/Kconfig.kasan
+++ b/lib/Kconfig.kasan
@@ -134,7 +134,7 @@ endchoice
 choice
 	prompt "Instrumentation type"
 	depends on KASAN_GENERIC || KASAN_SW_TAGS
-	default KASAN_OUTLINE
+	default KASAN_INLINE if !ARCH_DISABLE_KASAN_INLINE
 
 config KASAN_OUTLINE
 	bool "Outline instrumentation"
diff --git a/lib/closure.c b/lib/closure.c
index f86c9eeafb35..c16540552d61 100644
--- a/lib/closure.c
+++ b/lib/closure.c
@@ -36,7 +36,7 @@ static inline void closure_put_after_sub(struct closure *cl, int flags)
 			closure_debug_destroy(cl);
 
 			if (destructor)
-				destructor(cl);
+				destructor(&cl->work);
 
 			if (parent)
 				closure_put(parent);
@@ -108,8 +108,9 @@ struct closure_syncer {
 	int			done;
 };
 
-static void closure_sync_fn(struct closure *cl)
+static CLOSURE_CALLBACK(closure_sync_fn)
 {
+	struct closure *cl = container_of(ws, struct closure, work);
 	struct closure_syncer *s = cl->s;
 	struct task_struct *p;
 
diff --git a/lib/crypto/aesgcm.c b/lib/crypto/aesgcm.c
index c632d6e17af8..6bba6473fdf3 100644
--- a/lib/crypto/aesgcm.c
+++ b/lib/crypto/aesgcm.c
@@ -73,6 +73,19 @@ static void aesgcm_ghash(be128 *ghash, const be128 *key, const void *src,
 	}
 }
 
+/**
+ * aesgcm_mac - Generates the authentication tag using AES-GCM algorithm.
+ * @ctx: The data structure that will hold the AES-GCM key schedule
+ * @src: The input source data.
+ * @src_len: Length of the source data.
+ * @assoc: Points to the associated data.
+ * @assoc_len: Length of the associated data values.
+ * @ctr: Points to the counter value.
+ * @authtag: The output buffer for the authentication tag.
+ *
+ * It takes in the AES-GCM context, source data, associated data, counter value,
+ * and an output buffer for the authentication tag.
+ */
 static void aesgcm_mac(const struct aesgcm_ctx *ctx, const u8 *src, int src_len,
 		       const u8 *assoc, int assoc_len, __be32 *ctr, u8 *authtag)
 {
diff --git a/lib/group_cpus.c b/lib/group_cpus.c
index aa3f6815bb12..ee272c4cefcc 100644
--- a/lib/group_cpus.c
+++ b/lib/group_cpus.c
@@ -366,13 +366,25 @@ struct cpumask *group_cpus_evenly(unsigned int numgrps)
 	if (!masks)
 		goto fail_node_to_cpumask;
 
-	/* Stabilize the cpumasks */
-	cpus_read_lock();
 	build_node_to_cpumask(node_to_cpumask);
 
+	/*
+	 * Make a local cache of 'cpu_present_mask', so the two stages
+	 * spread can observe consistent 'cpu_present_mask' without holding
+	 * cpu hotplug lock, then we can reduce deadlock risk with cpu
+	 * hotplug code.
+	 *
+	 * Here CPU hotplug may happen when reading `cpu_present_mask`, and
+	 * we can live with the case because it only affects that hotplug
+	 * CPU is handled in the 1st or 2nd stage, and either way is correct
+	 * from API user viewpoint since 2-stage spread is sort of
+	 * optimization.
+	 */
+	cpumask_copy(npresmsk, data_race(cpu_present_mask));
+
 	/* grouping present CPUs first */
 	ret = __group_cpus_evenly(curgrp, numgrps, node_to_cpumask,
-				  cpu_present_mask, nmsk, masks);
+				  npresmsk, nmsk, masks);
 	if (ret < 0)
 		goto fail_build_affinity;
 	nr_present = ret;
@@ -387,15 +399,13 @@ struct cpumask *group_cpus_evenly(unsigned int numgrps)
 		curgrp = 0;
 	else
 		curgrp = nr_present;
-	cpumask_andnot(npresmsk, cpu_possible_mask, cpu_present_mask);
+	cpumask_andnot(npresmsk, cpu_possible_mask, npresmsk);
 	ret = __group_cpus_evenly(curgrp, numgrps, node_to_cpumask,
 				  npresmsk, nmsk, masks);
 	if (ret >= 0)
 		nr_others = ret;
 
  fail_build_affinity:
-	cpus_read_unlock();
-
 	if (ret >= 0)
 		WARN_ON(nr_present + nr_others < numgrps);
 
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index de7d11cf4c63..8ff6824a1005 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -409,7 +409,7 @@ size_t copy_page_to_iter_nofault(struct page *page, unsigned offset, size_t byte
 		void *kaddr = kmap_local_page(page);
 		size_t n = min(bytes, (size_t)PAGE_SIZE - offset);
 
-		n = iterate_and_advance(i, bytes, kaddr,
+		n = iterate_and_advance(i, n, kaddr + offset,
 					copy_to_user_iter_nofault,
 					memcpy_to_iter);
 		kunmap_local(kaddr);
diff --git a/lib/kunit/kunit-test.c b/lib/kunit/kunit-test.c
index 99d2a3a528e1..de2113a58fa0 100644
--- a/lib/kunit/kunit-test.c
+++ b/lib/kunit/kunit-test.c
@@ -562,7 +562,7 @@ static void kunit_log_test(struct kunit *test)
 	KUNIT_EXPECT_TRUE(test, test->log->append_newlines);
 
 	full_log = string_stream_get_string(test->log);
-	kunit_add_action(test, (kunit_action_t *)kfree, full_log);
+	kunit_add_action(test, kfree_wrapper, full_log);
 	KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
 				     strstr(full_log, "put this in log."));
 	KUNIT_EXPECT_NOT_ERR_OR_NULL(test,
diff --git a/lib/kunit/test.c b/lib/kunit/test.c
index f2eb71f1a66c..7aceb07a1af9 100644
--- a/lib/kunit/test.c
+++ b/lib/kunit/test.c
@@ -338,6 +338,36 @@ void kunit_init_test(struct kunit *test, const char *name, struct string_stream
 }
 EXPORT_SYMBOL_GPL(kunit_init_test);
 
+/* Only warn when a test takes more than twice the threshold */
+#define KUNIT_SPEED_WARNING_MULTIPLIER	2
+
+/* Slow tests are defined as taking more than 1s */
+#define KUNIT_SPEED_SLOW_THRESHOLD_S	1
+
+#define KUNIT_SPEED_SLOW_WARNING_THRESHOLD_S	\
+	(KUNIT_SPEED_WARNING_MULTIPLIER * KUNIT_SPEED_SLOW_THRESHOLD_S)
+
+#define s_to_timespec64(s) ns_to_timespec64((s) * NSEC_PER_SEC)
+
+static void kunit_run_case_check_speed(struct kunit *test,
+				       struct kunit_case *test_case,
+				       struct timespec64 duration)
+{
+	struct timespec64 slow_thr =
+		s_to_timespec64(KUNIT_SPEED_SLOW_WARNING_THRESHOLD_S);
+	enum kunit_speed speed = test_case->attr.speed;
+
+	if (timespec64_compare(&duration, &slow_thr) < 0)
+		return;
+
+	if (speed == KUNIT_SPEED_VERY_SLOW || speed == KUNIT_SPEED_SLOW)
+		return;
+
+	kunit_warn(test,
+		   "Test should be marked slow (runtime: %lld.%09lds)",
+		   duration.tv_sec, duration.tv_nsec);
+}
+
 /*
  * Initializes and runs test case. Does not clean up or do post validations.
  */
@@ -345,6 +375,8 @@ static void kunit_run_case_internal(struct kunit *test,
 				    struct kunit_suite *suite,
 				    struct kunit_case *test_case)
 {
+	struct timespec64 start, end;
+
 	if (suite->init) {
 		int ret;
 
@@ -356,7 +388,13 @@ static void kunit_run_case_internal(struct kunit *test,
 		}
 	}
 
+	ktime_get_ts64(&start);
+
 	test_case->run_case(test);
+
+	ktime_get_ts64(&end);
+
+	kunit_run_case_check_speed(test, test_case, timespec64_sub(end, start));
 }
 
 static void kunit_case_internal_cleanup(struct kunit *test)
@@ -670,6 +708,8 @@ int __kunit_test_suites_init(struct kunit_suite * const * const suites, int num_
 		return 0;
 	}
 
+	kunit_suite_counter = 1;
+
 	static_branch_inc(&kunit_running);
 
 	for (i = 0; i < num_suites; i++) {
@@ -696,8 +736,6 @@ void __kunit_test_suites_exit(struct kunit_suite **suites, int num_suites)
 
 	for (i = 0; i < num_suites; i++)
 		kunit_exit_suite(suites[i]);
-
-	kunit_suite_counter = 1;
 }
 EXPORT_SYMBOL_GPL(__kunit_test_suites_exit);
 
diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index bb24d84a4922..89f8d2160277 100644
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -4,6 +4,8 @@
  * Copyright (c) 2018-2022 Oracle Corporation
  * Authors: Liam R. Howlett <Liam.Howlett@oracle.com>
  *	    Matthew Wilcox <willy@infradead.org>
+ * Copyright (c) 2023 ByteDance
+ * Author: Peng Zhang <zhangpeng.00@bytedance.com>
  */
 
 /*
@@ -165,6 +167,11 @@ static inline int mt_alloc_bulk(gfp_t gfp, size_t size, void **nodes)
 	return kmem_cache_alloc_bulk(maple_node_cache, gfp, size, nodes);
 }
 
+static inline void mt_free_one(struct maple_node *node)
+{
+	kmem_cache_free(maple_node_cache, node);
+}
+
 static inline void mt_free_bulk(size_t size, void __rcu **nodes)
 {
 	kmem_cache_free_bulk(maple_node_cache, size, (void **)nodes);
@@ -205,23 +212,29 @@ static unsigned int mas_mt_height(struct ma_state *mas)
 	return mt_height(mas->tree);
 }
 
-static inline enum maple_type mte_node_type(const struct maple_enode *entry)
+static inline unsigned int mt_attr(struct maple_tree *mt)
+{
+	return mt->ma_flags & ~MT_FLAGS_HEIGHT_MASK;
+}
+
+static __always_inline enum maple_type mte_node_type(
+		const struct maple_enode *entry)
 {
 	return ((unsigned long)entry >> MAPLE_NODE_TYPE_SHIFT) &
 		MAPLE_NODE_TYPE_MASK;
 }
 
-static inline bool ma_is_dense(const enum maple_type type)
+static __always_inline bool ma_is_dense(const enum maple_type type)
 {
 	return type < maple_leaf_64;
 }
 
-static inline bool ma_is_leaf(const enum maple_type type)
+static __always_inline bool ma_is_leaf(const enum maple_type type)
 {
 	return type < maple_range_64;
 }
 
-static inline bool mte_is_leaf(const struct maple_enode *entry)
+static __always_inline bool mte_is_leaf(const struct maple_enode *entry)
 {
 	return ma_is_leaf(mte_node_type(entry));
 }
@@ -230,60 +243,50 @@ static inline bool mte_is_leaf(const struct maple_enode *entry)
  * We also reserve values with the bottom two bits set to '10' which are
  * below 4096
  */
-static inline bool mt_is_reserved(const void *entry)
+static __always_inline bool mt_is_reserved(const void *entry)
 {
 	return ((unsigned long)entry < MAPLE_RESERVED_RANGE) &&
 		xa_is_internal(entry);
 }
 
-static inline void mas_set_err(struct ma_state *mas, long err)
+static __always_inline void mas_set_err(struct ma_state *mas, long err)
 {
 	mas->node = MA_ERROR(err);
+	mas->status = ma_error;
 }
 
-static inline bool mas_is_ptr(const struct ma_state *mas)
+static __always_inline bool mas_is_ptr(const struct ma_state *mas)
 {
-	return mas->node == MAS_ROOT;
+	return mas->status == ma_root;
 }
 
-static inline bool mas_is_start(const struct ma_state *mas)
+static __always_inline bool mas_is_start(const struct ma_state *mas)
 {
-	return mas->node == MAS_START;
+	return mas->status == ma_start;
 }
 
-bool mas_is_err(struct ma_state *mas)
+static __always_inline bool mas_is_none(const struct ma_state *mas)
 {
-	return xa_is_err(mas->node);
+	return mas->status == ma_none;
 }
 
-static __always_inline bool mas_is_overflow(struct ma_state *mas)
+static __always_inline bool mas_is_paused(const struct ma_state *mas)
 {
-	if (unlikely(mas->node == MAS_OVERFLOW))
-		return true;
-
-	return false;
+	return mas->status == ma_pause;
 }
 
-static __always_inline bool mas_is_underflow(struct ma_state *mas)
+static __always_inline bool mas_is_overflow(struct ma_state *mas)
 {
-	if (unlikely(mas->node == MAS_UNDERFLOW))
-		return true;
-
-	return false;
+	return mas->status == ma_overflow;
 }
 
-static inline bool mas_searchable(struct ma_state *mas)
+static inline bool mas_is_underflow(struct ma_state *mas)
 {
-	if (mas_is_none(mas))
-		return false;
-
-	if (mas_is_ptr(mas))
-		return false;
-
-	return true;
+	return mas->status == ma_underflow;
 }
 
-static inline struct maple_node *mte_to_node(const struct maple_enode *entry)
+static __always_inline struct maple_node *mte_to_node(
+		const struct maple_enode *entry)
 {
 	return (struct maple_node *)((unsigned long)entry & ~MAPLE_NODE_MASK);
 }
@@ -360,12 +363,12 @@ static inline bool mte_has_null(const struct maple_enode *node)
 	return (unsigned long)node & MAPLE_ENODE_NULL;
 }
 
-static inline bool ma_is_root(struct maple_node *node)
+static __always_inline bool ma_is_root(struct maple_node *node)
 {
 	return ((unsigned long)node->parent & MA_ROOT_PARENT);
 }
 
-static inline bool mte_is_root(const struct maple_enode *node)
+static __always_inline bool mte_is_root(const struct maple_enode *node)
 {
 	return ma_is_root(mte_to_node(node));
 }
@@ -375,7 +378,7 @@ static inline bool mas_is_root_limits(const struct ma_state *mas)
 	return !mas->min && mas->max == ULONG_MAX;
 }
 
-static inline bool mt_is_alloc(struct maple_tree *mt)
+static __always_inline bool mt_is_alloc(struct maple_tree *mt)
 {
 	return (mt->ma_flags & MT_FLAGS_ALLOC_RANGE);
 }
@@ -514,11 +517,12 @@ void mas_set_parent(struct ma_state *mas, struct maple_enode *enode,
  *
  * Return: The slot in the parent node where @enode resides.
  */
-static inline unsigned int mte_parent_slot(const struct maple_enode *enode)
+static __always_inline
+unsigned int mte_parent_slot(const struct maple_enode *enode)
 {
 	unsigned long val = (unsigned long)mte_to_node(enode)->parent;
 
-	if (val & MA_ROOT_PARENT)
+	if (unlikely(val & MA_ROOT_PARENT))
 		return 0;
 
 	/*
@@ -534,7 +538,8 @@ static inline unsigned int mte_parent_slot(const struct maple_enode *enode)
  *
  * Return: The parent maple node.
  */
-static inline struct maple_node *mte_parent(const struct maple_enode *enode)
+static __always_inline
+struct maple_node *mte_parent(const struct maple_enode *enode)
 {
 	return (void *)((unsigned long)
 			(mte_to_node(enode)->parent) & ~MAPLE_NODE_MASK);
@@ -546,7 +551,7 @@ static inline struct maple_node *mte_parent(const struct maple_enode *enode)
  *
  * Return: true if dead, false otherwise.
  */
-static inline bool ma_dead_node(const struct maple_node *node)
+static __always_inline bool ma_dead_node(const struct maple_node *node)
 {
 	struct maple_node *parent;
 
@@ -562,7 +567,7 @@ static inline bool ma_dead_node(const struct maple_node *node)
  *
  * Return: true if dead, false otherwise.
  */
-static inline bool mte_dead_node(const struct maple_enode *enode)
+static __always_inline bool mte_dead_node(const struct maple_enode *enode)
 {
 	struct maple_node *parent, *node;
 
@@ -679,35 +684,6 @@ static inline unsigned long *ma_gaps(struct maple_node *node,
 	return NULL;
 }
 
-/*
- * mas_pivot() - Get the pivot at @piv of the maple encoded node.
- * @mas: The maple state.
- * @piv: The pivot.
- *
- * Return: the pivot at @piv of @mn.
- */
-static inline unsigned long mas_pivot(struct ma_state *mas, unsigned char piv)
-{
-	struct maple_node *node = mas_mn(mas);
-	enum maple_type type = mte_node_type(mas->node);
-
-	if (MAS_WARN_ON(mas, piv >= mt_pivots[type])) {
-		mas_set_err(mas, -EIO);
-		return 0;
-	}
-
-	switch (type) {
-	case maple_arange_64:
-		return node->ma64.pivot[piv];
-	case maple_range_64:
-	case maple_leaf_64:
-		return node->mr64.pivot[piv];
-	case maple_dense:
-		return 0;
-	}
-	return 0;
-}
-
 /*
  * mas_safe_pivot() - get the pivot at @piv or mas->max.
  * @mas: The maple state
@@ -718,7 +694,7 @@ static inline unsigned long mas_pivot(struct ma_state *mas, unsigned char piv)
  * Return: The pivot at @piv within the limit of the @pivots array, @mas->max
  * otherwise.
  */
-static inline unsigned long
+static __always_inline unsigned long
 mas_safe_pivot(const struct ma_state *mas, unsigned long *pivots,
 	       unsigned char piv, enum maple_type type)
 {
@@ -759,7 +735,6 @@ static inline void mte_set_pivot(struct maple_enode *mn, unsigned char piv,
 
 	BUG_ON(piv >= mt_pivots[type]);
 	switch (type) {
-	default:
 	case maple_range_64:
 	case maple_leaf_64:
 		node->mr64.pivot[piv] = val;
@@ -783,7 +758,6 @@ static inline void mte_set_pivot(struct maple_enode *mn, unsigned char piv,
 static inline void __rcu **ma_slots(struct maple_node *mn, enum maple_type mt)
 {
 	switch (mt) {
-	default:
 	case maple_arange_64:
 		return mn->ma64.slot;
 	case maple_range_64:
@@ -792,6 +766,8 @@ static inline void __rcu **ma_slots(struct maple_node *mn, enum maple_type mt)
 	case maple_dense:
 		return mn->slot;
 	}
+
+	return NULL;
 }
 
 static inline bool mt_write_locked(const struct maple_tree *mt)
@@ -800,20 +776,20 @@ static inline bool mt_write_locked(const struct maple_tree *mt)
 		lockdep_is_held(&mt->ma_lock);
 }
 
-static inline bool mt_locked(const struct maple_tree *mt)
+static __always_inline bool mt_locked(const struct maple_tree *mt)
 {
 	return mt_external_lock(mt) ? mt_lock_is_held(mt) :
 		lockdep_is_held(&mt->ma_lock);
 }
 
-static inline void *mt_slot(const struct maple_tree *mt,
+static __always_inline void *mt_slot(const struct maple_tree *mt,
 		void __rcu **slots, unsigned char offset)
 {
 	return rcu_dereference_check(slots[offset], mt_locked(mt));
 }
 
-static inline void *mt_slot_locked(struct maple_tree *mt, void __rcu **slots,
-				   unsigned char offset)
+static __always_inline void *mt_slot_locked(struct maple_tree *mt,
+		void __rcu **slots, unsigned char offset)
 {
 	return rcu_dereference_protected(slots[offset], mt_write_locked(mt));
 }
@@ -825,8 +801,8 @@ static inline void *mt_slot_locked(struct maple_tree *mt, void __rcu **slots,
  *
  * Return: The entry stored in @slots at the @offset.
  */
-static inline void *mas_slot_locked(struct ma_state *mas, void __rcu **slots,
-				       unsigned char offset)
+static __always_inline void *mas_slot_locked(struct ma_state *mas,
+		void __rcu **slots, unsigned char offset)
 {
 	return mt_slot_locked(mas->tree, slots, offset);
 }
@@ -839,8 +815,8 @@ static inline void *mas_slot_locked(struct ma_state *mas, void __rcu **slots,
  *
  * Return: The entry stored in @slots at the @offset
  */
-static inline void *mas_slot(struct ma_state *mas, void __rcu **slots,
-			     unsigned char offset)
+static __always_inline void *mas_slot(struct ma_state *mas, void __rcu **slots,
+		unsigned char offset)
 {
 	return mt_slot(mas->tree, slots, offset);
 }
@@ -851,7 +827,7 @@ static inline void *mas_slot(struct ma_state *mas, void __rcu **slots,
  *
  * Return: The pointer to the root of the tree
  */
-static inline void *mas_root(struct ma_state *mas)
+static __always_inline void *mas_root(struct ma_state *mas)
 {
 	return rcu_dereference_check(mas->tree->ma_root, mt_locked(mas->tree));
 }
@@ -954,10 +930,8 @@ static inline unsigned char ma_meta_end(struct maple_node *mn,
 /*
  * ma_meta_gap() - Get the largest gap location of a node from the metadata
  * @mn: The maple node
- * @mt: The maple node type
  */
-static inline unsigned char ma_meta_gap(struct maple_node *mn,
-					enum maple_type mt)
+static inline unsigned char ma_meta_gap(struct maple_node *mn)
 {
 	return mn->ma64.meta.gap;
 }
@@ -1112,14 +1086,16 @@ static int mas_ascend(struct ma_state *mas)
 		return 0;
 	}
 
-	if (!mas->min)
+	min = 0;
+	max = ULONG_MAX;
+	if (!mas->offset) {
+		min = mas->min;
 		set_min = true;
+	}
 
 	if (mas->max == ULONG_MAX)
 		set_max = true;
 
-	min = 0;
-	max = ULONG_MAX;
 	do {
 		p_enode = a_enode;
 		a_type = mas_parent_type(mas, p_enode);
@@ -1258,6 +1234,7 @@ static inline void mas_alloc_nodes(struct ma_state *mas, gfp_t gfp)
 	if (mas->mas_flags & MA_STATE_PREALLOC) {
 		if (allocated)
 			return;
+		BUG_ON(!allocated);
 		WARN_ON(!allocated);
 	}
 
@@ -1363,14 +1340,14 @@ static void mas_node_count(struct ma_state *mas, int count)
  * mas_start() - Sets up maple state for operations.
  * @mas: The maple state.
  *
- * If mas->node == MAS_START, then set the min, max and depth to
+ * If mas->status == mas_start, then set the min, max and depth to
  * defaults.
  *
  * Return:
- * - If mas->node is an error or not MAS_START, return NULL.
- * - If it's an empty tree:     NULL & mas->node == MAS_NONE
- * - If it's a single entry:    The entry & mas->node == MAS_ROOT
- * - If it's a tree:            NULL & mas->node == safe root node.
+ * - If mas->node is an error or not mas_start, return NULL.
+ * - If it's an empty tree:     NULL & mas->status == ma_none
+ * - If it's a single entry:    The entry & mas->status == mas_root
+ * - If it's a tree:            NULL & mas->status == safe root node.
  */
 static inline struct maple_enode *mas_start(struct ma_state *mas)
 {
@@ -1386,6 +1363,7 @@ static inline struct maple_enode *mas_start(struct ma_state *mas)
 		/* Tree with nodes */
 		if (likely(xa_is_node(root))) {
 			mas->depth = 1;
+			mas->status = ma_active;
 			mas->node = mte_safe_root(root);
 			mas->offset = 0;
 			if (mte_dead_node(mas->node))
@@ -1396,13 +1374,14 @@ static inline struct maple_enode *mas_start(struct ma_state *mas)
 
 		/* empty tree */
 		if (unlikely(!root)) {
-			mas->node = MAS_NONE;
+			mas->node = NULL;
+			mas->status = ma_none;
 			mas->offset = MAPLE_NODE_SLOTS;
 			return NULL;
 		}
 
 		/* Single entry tree */
-		mas->node = MAS_ROOT;
+		mas->status = ma_root;
 		mas->offset = MAPLE_NODE_SLOTS;
 
 		/* Single entry tree. */
@@ -1425,10 +1404,8 @@ static inline struct maple_enode *mas_start(struct ma_state *mas)
  * Uses metadata to find the end of the data when possible.
  * Return: The zero indexed last slot with data (may be null).
  */
-static inline unsigned char ma_data_end(struct maple_node *node,
-					enum maple_type type,
-					unsigned long *pivots,
-					unsigned long max)
+static __always_inline unsigned char ma_data_end(struct maple_node *node,
+		enum maple_type type, unsigned long *pivots, unsigned long max)
 {
 	unsigned char offset;
 
@@ -1608,7 +1585,7 @@ static inline unsigned long mas_max_gap(struct ma_state *mas)
 
 	node = mas_mn(mas);
 	MAS_BUG_ON(mas, mt != maple_arange_64);
-	offset = ma_meta_gap(node, mt);
+	offset = ma_meta_gap(node);
 	gaps = ma_gaps(node, mt);
 	return gaps[offset];
 }
@@ -1639,7 +1616,7 @@ static inline void mas_parent_gap(struct ma_state *mas, unsigned char offset,
 
 ascend:
 	MAS_BUG_ON(mas, pmt != maple_arange_64);
-	meta_offset = ma_meta_gap(pnode, pmt);
+	meta_offset = ma_meta_gap(pnode);
 	meta_gap = pgaps[meta_offset];
 
 	pgaps[offset] = new;
@@ -1987,27 +1964,13 @@ static inline void mas_mab_cp(struct ma_state *mas, unsigned char mas_start,
 
 /*
  * mas_leaf_set_meta() - Set the metadata of a leaf if possible.
- * @mas: The maple state
  * @node: The maple node
- * @pivots: pointer to the maple node pivots
  * @mt: The maple type
- * @end: The assumed end
- *
- * Note, end may be incremented within this function but not modified at the
- * source.  This is fine since the metadata is the last thing to be stored in a
- * node during a write.
+ * @end: The node end
  */
-static inline void mas_leaf_set_meta(struct ma_state *mas,
-		struct maple_node *node, unsigned long *pivots,
+static inline void mas_leaf_set_meta(struct maple_node *node,
 		enum maple_type mt, unsigned char end)
 {
-	/* There is no room for metadata already */
-	if (mt_pivots[mt] <= end)
-		return;
-
-	if (pivots[end] && pivots[end] < mas->max)
-		end++;
-
 	if (end < mt_slots[mt] - 1)
 		ma_set_meta(node, mt, 0, end);
 }
@@ -2064,7 +2027,7 @@ static inline void mab_mas_cp(struct maple_big_node *b_node,
 
 		ma_set_meta(node, mt, offset, end);
 	} else {
-		mas_leaf_set_meta(mas, node, pivots, mt, end);
+		mas_leaf_set_meta(node, mt, end);
 	}
 }
 
@@ -2152,11 +2115,11 @@ static noinline_for_kasan void mas_store_b_node(struct ma_wr_state *wr_mas,
 	}
 
 	slot = offset_end + 1;
-	if (slot > wr_mas->node_end)
+	if (slot > mas->end)
 		goto b_end;
 
 	/* Copy end data to the end of the node. */
-	mas_mab_cp(mas, slot, wr_mas->node_end + 1, b_node, ++b_end);
+	mas_mab_cp(mas, slot, mas->end + 1, b_node, ++b_end);
 	b_node->b_end--;
 	return;
 
@@ -2211,19 +2174,21 @@ static inline bool mas_next_sibling(struct ma_state *mas)
 }
 
 /*
- * mte_node_or_node() - Return the encoded node or MAS_NONE.
+ * mte_node_or_none() - Set the enode and state.
  * @enode: The encoded maple node.
  *
- * Shorthand to avoid setting %NULLs in the tree or maple_subtree_state.
- *
- * Return: @enode or MAS_NONE
+ * Set the node to the enode and the status.
  */
-static inline struct maple_enode *mte_node_or_none(struct maple_enode *enode)
+static inline void mas_node_or_none(struct ma_state *mas,
+		struct maple_enode *enode)
 {
-	if (enode)
-		return enode;
-
-	return ma_enode_ptr(MAS_NONE);
+	if (enode) {
+		mas->node = enode;
+		mas->status = ma_active;
+	} else {
+		mas->node = NULL;
+		mas->status = ma_none;
+	}
 }
 
 /*
@@ -2245,8 +2210,8 @@ static inline void mas_wr_node_walk(struct ma_wr_state *wr_mas)
 
 	wr_mas->node = mas_mn(wr_mas->mas);
 	wr_mas->pivots = ma_pivots(wr_mas->node, wr_mas->type);
-	count = wr_mas->node_end = ma_data_end(wr_mas->node, wr_mas->type,
-					       wr_mas->pivots, mas->max);
+	count = mas->end = ma_data_end(wr_mas->node, wr_mas->type,
+				       wr_mas->pivots, mas->max);
 	offset = mas->offset;
 
 	while (offset < count && mas->index > wr_mas->pivots[offset])
@@ -2543,13 +2508,15 @@ static inline void mast_set_split_parents(struct maple_subtree_state *mast,
  * The node will either be RCU freed or pushed back on the maple state.
  */
 static inline void mas_topiary_node(struct ma_state *mas,
-		struct maple_enode *enode, bool in_rcu)
+		struct ma_state *tmp_mas, bool in_rcu)
 {
 	struct maple_node *tmp;
+	struct maple_enode *enode;
 
-	if (enode == MAS_NONE)
+	if (mas_is_none(tmp_mas))
 		return;
 
+	enode = tmp_mas->node;
 	tmp = mte_to_node(enode);
 	mte_set_node_dead(enode);
 	if (in_rcu)
@@ -2589,8 +2556,8 @@ static inline void mas_topiary_replace(struct ma_state *mas,
 	/* Update the parent pointers in the tree */
 	tmp[0] = *mas;
 	tmp[0].offset = 0;
-	tmp[1].node = MAS_NONE;
-	tmp[2].node = MAS_NONE;
+	tmp[1].status = ma_none;
+	tmp[2].status = ma_none;
 	while (!mte_is_leaf(tmp[0].node)) {
 		n = 0;
 		for (i = 0; i < 3; i++) {
@@ -2610,7 +2577,7 @@ static inline void mas_topiary_replace(struct ma_state *mas,
 			break;
 
 		while (n < 3)
-			tmp_next[n++].node = MAS_NONE;
+			tmp_next[n++].status = ma_none;
 
 		for (i = 0; i < 3; i++)
 			tmp[i] = tmp_next[i];
@@ -2623,8 +2590,8 @@ static inline void mas_topiary_replace(struct ma_state *mas,
 	tmp[0] = *mas;
 	tmp[0].offset = 0;
 	tmp[0].node = old_enode;
-	tmp[1].node = MAS_NONE;
-	tmp[2].node = MAS_NONE;
+	tmp[1].status = ma_none;
+	tmp[2].status = ma_none;
 	in_rcu = mt_in_rcu(mas->tree);
 	do {
 		n = 0;
@@ -2639,7 +2606,7 @@ static inline void mas_topiary_replace(struct ma_state *mas,
 				if ((tmp_next[n].min >= tmp_next->index) &&
 				    (tmp_next[n].max <= tmp_next->last)) {
 					mat_add(&subtrees, tmp_next[n].node);
-					tmp_next[n].node = MAS_NONE;
+					tmp_next[n].status = ma_none;
 				} else {
 					n++;
 				}
@@ -2650,16 +2617,16 @@ static inline void mas_topiary_replace(struct ma_state *mas,
 			break;
 
 		while (n < 3)
-			tmp_next[n++].node = MAS_NONE;
+			tmp_next[n++].status = ma_none;
 
 		for (i = 0; i < 3; i++) {
-			mas_topiary_node(mas, tmp[i].node, in_rcu);
+			mas_topiary_node(mas, &tmp[i], in_rcu);
 			tmp[i] = tmp_next[i];
 		}
 	} while (!mte_is_leaf(tmp[0].node));
 
 	for (i = 0; i < 3; i++)
-		mas_topiary_node(mas, tmp[i].node, in_rcu);
+		mas_topiary_node(mas, &tmp[i], in_rcu);
 
 	mas_mat_destroy(mas, &subtrees);
 }
@@ -2698,9 +2665,9 @@ static inline void mast_cp_to_nodes(struct maple_subtree_state *mast,
 {
 	bool new_lmax = true;
 
-	mast->l->node = mte_node_or_none(left);
-	mast->m->node = mte_node_or_none(middle);
-	mast->r->node = mte_node_or_none(right);
+	mas_node_or_none(mast->l, left);
+	mas_node_or_none(mast->m, middle);
+	mas_node_or_none(mast->r, right);
 
 	mast->l->min = mast->orig_l->min;
 	if (split == mast->bn->b_end) {
@@ -2796,32 +2763,29 @@ static inline void *mtree_range_walk(struct ma_state *mas)
 	min = mas->min;
 	max = mas->max;
 	do {
-		offset = 0;
 		last = next;
 		node = mte_to_node(next);
 		type = mte_node_type(next);
 		pivots = ma_pivots(node, type);
 		end = ma_data_end(node, type, pivots, max);
-		if (unlikely(ma_dead_node(node)))
-			goto dead_node;
-
-		if (pivots[offset] >= mas->index) {
-			prev_max = max;
-			prev_min = min;
-			max = pivots[offset];
+		prev_min = min;
+		prev_max = max;
+		if (pivots[0] >= mas->index) {
+			offset = 0;
+			max = pivots[0];
 			goto next;
 		}
 
-		do {
+		offset = 1;
+		while (offset < end) {
+			if (pivots[offset] >= mas->index) {
+				max = pivots[offset];
+				break;
+			}
 			offset++;
-		} while ((offset < end) && (pivots[offset] < mas->index));
+		}
 
-		prev_min = min;
 		min = pivots[offset - 1] + 1;
-		prev_max = max;
-		if (likely(offset < end && pivots[offset]))
-			max = pivots[offset];
-
 next:
 		slots = ma_slots(node, type);
 		next = mt_slot(mas->tree, slots, offset);
@@ -2829,6 +2793,7 @@ static inline void *mtree_range_walk(struct ma_state *mas)
 			goto dead_node;
 	} while (!ma_is_leaf(type));
 
+	mas->end = end;
 	mas->offset = offset;
 	mas->index = min;
 	mas->last = max;
@@ -2879,7 +2844,7 @@ static int mas_spanning_rebalance(struct ma_state *mas,
 	mast->l = &l_mas;
 	mast->m = &m_mas;
 	mast->r = &r_mas;
-	l_mas.node = r_mas.node = m_mas.node = MAS_NONE;
+	l_mas.status = r_mas.status = m_mas.status = ma_none;
 
 	/* Check if this is not root and has sufficient data.  */
 	if (((mast->orig_l->min != 0) || (mast->orig_r->max != ULONG_MAX)) &&
@@ -3406,7 +3371,6 @@ static int mas_split(struct ma_state *mas, struct maple_big_node *b_node)
 		/* Try to push left. */
 		if (mas_push_data(mas, height, &mast, true))
 			break;
-
 		/* Try to push right. */
 		if (mas_push_data(mas, height, &mast, false))
 			break;
@@ -3495,6 +3459,7 @@ static noinline_for_kasan int mas_commit_b_node(struct ma_wr_state *wr_mas,
 	mas_replace_node(wr_mas->mas, old_enode);
 reuse_node:
 	mas_update_gap(wr_mas->mas);
+	wr_mas->mas->end = b_end;
 	return 1;
 }
 
@@ -3521,6 +3486,7 @@ static inline int mas_root_expand(struct ma_state *mas, void *entry)
 	slots = ma_slots(node, type);
 	node->parent = ma_parent_ptr(mas_tree_parent(mas));
 	mas->node = mt_mk_node(node, type);
+	mas->status = ma_active;
 
 	if (mas->index) {
 		if (contents) {
@@ -3553,7 +3519,7 @@ static inline void mas_store_root(struct ma_state *mas, void *entry)
 		mas_root_expand(mas, entry);
 	else {
 		rcu_assign_pointer(mas->tree->ma_root, entry);
-		mas->node = MAS_START;
+		mas->status = ma_start;
 	}
 }
 
@@ -3730,23 +3696,17 @@ static inline void *mtree_lookup_walk(struct ma_state *mas)
 	enum maple_type type;
 	void __rcu **slots;
 	unsigned char end;
-	unsigned long max;
 
 	next = mas->node;
-	max = ULONG_MAX;
 	do {
-		offset = 0;
 		node = mte_to_node(next);
 		type = mte_node_type(next);
 		pivots = ma_pivots(node, type);
-		end = ma_data_end(node, type, pivots, max);
-		if (unlikely(ma_dead_node(node)))
-			goto dead_node;
+		end = mt_pivots[type];
+		offset = 0;
 		do {
-			if (pivots[offset] >= mas->index) {
-				max = pivots[offset];
+			if (pivots[offset] >= mas->index)
 				break;
-			}
 		} while (++offset < end);
 
 		slots = ma_slots(node, type);
@@ -3785,7 +3745,7 @@ static inline int mas_new_root(struct ma_state *mas, void *entry)
 		mas->depth = 0;
 		mas_set_height(mas);
 		rcu_assign_pointer(mas->tree->ma_root, entry);
-		mas->node = MAS_START;
+		mas->status = ma_start;
 		goto done;
 	}
 
@@ -3798,6 +3758,7 @@ static inline int mas_new_root(struct ma_state *mas, void *entry)
 	slots = ma_slots(node, type);
 	node->parent = ma_parent_ptr(mas_tree_parent(mas));
 	mas->node = mt_mk_node(node, type);
+	mas->status = ma_active;
 	rcu_assign_pointer(slots[0], entry);
 	pivots[0] = mas->last;
 	mas->depth = 1;
@@ -3891,10 +3852,10 @@ static inline int mas_wr_spanning_store(struct ma_wr_state *wr_mas)
 
 	memset(&b_node, 0, sizeof(struct maple_big_node));
 	/* Copy l_mas and store the value in b_node. */
-	mas_store_b_node(&l_wr_mas, &b_node, l_wr_mas.node_end);
+	mas_store_b_node(&l_wr_mas, &b_node, l_mas.end);
 	/* Copy r_mas into b_node. */
-	if (r_mas.offset <= r_wr_mas.node_end)
-		mas_mab_cp(&r_mas, r_mas.offset, r_wr_mas.node_end,
+	if (r_mas.offset <= r_mas.end)
+		mas_mab_cp(&r_mas, r_mas.offset, r_mas.end,
 			   &b_node, b_node.b_end + 1);
 	else
 		b_node.b_end++;
@@ -3936,7 +3897,7 @@ static inline bool mas_wr_node_store(struct ma_wr_state *wr_mas,
 	if (mas->last == wr_mas->end_piv)
 		offset_end++; /* don't copy this offset */
 	else if (unlikely(wr_mas->r_max == ULONG_MAX))
-		mas_bulk_rebalance(mas, wr_mas->node_end, wr_mas->type);
+		mas_bulk_rebalance(mas, mas->end, wr_mas->type);
 
 	/* set up node. */
 	if (in_rcu) {
@@ -3972,12 +3933,12 @@ static inline bool mas_wr_node_store(struct ma_wr_state *wr_mas,
 	 * this range wrote to the end of the node or it overwrote the rest of
 	 * the data
 	 */
-	if (offset_end > wr_mas->node_end)
+	if (offset_end > mas->end)
 		goto done;
 
 	dst_offset = mas->offset + 1;
 	/* Copy to the end of node if necessary. */
-	copy_size = wr_mas->node_end - offset_end + 1;
+	copy_size = mas->end - offset_end + 1;
 	memcpy(dst_slots + dst_offset, wr_mas->slots + offset_end,
 	       sizeof(void *) * copy_size);
 	memcpy(dst_pivots + dst_offset, wr_mas->pivots + offset_end,
@@ -3987,7 +3948,7 @@ static inline bool mas_wr_node_store(struct ma_wr_state *wr_mas,
 		dst_pivots[new_end] = mas->max;
 
 done:
-	mas_leaf_set_meta(mas, newnode, dst_pivots, maple_leaf_64, new_end);
+	mas_leaf_set_meta(newnode, maple_leaf_64, new_end);
 	if (in_rcu) {
 		struct maple_enode *old_enode = mas->node;
 
@@ -3998,6 +3959,7 @@ static inline bool mas_wr_node_store(struct ma_wr_state *wr_mas,
 	}
 	trace_ma_write(__func__, mas, 0, wr_mas->entry);
 	mas_update_gap(mas);
+	mas->end = new_end;
 	return true;
 }
 
@@ -4063,10 +4025,10 @@ static inline void mas_wr_extend_null(struct ma_wr_state *wr_mas)
 	} else {
 		/* Check next slot(s) if we are overwriting the end */
 		if ((mas->last == wr_mas->end_piv) &&
-		    (wr_mas->node_end != wr_mas->offset_end) &&
+		    (mas->end != wr_mas->offset_end) &&
 		    !wr_mas->slots[wr_mas->offset_end + 1]) {
 			wr_mas->offset_end++;
-			if (wr_mas->offset_end == wr_mas->node_end)
+			if (wr_mas->offset_end == mas->end)
 				mas->last = mas->max;
 			else
 				mas->last = wr_mas->pivots[wr_mas->offset_end];
@@ -4091,11 +4053,11 @@ static inline void mas_wr_extend_null(struct ma_wr_state *wr_mas)
 
 static inline void mas_wr_end_piv(struct ma_wr_state *wr_mas)
 {
-	while ((wr_mas->offset_end < wr_mas->node_end) &&
+	while ((wr_mas->offset_end < wr_mas->mas->end) &&
 	       (wr_mas->mas->last > wr_mas->pivots[wr_mas->offset_end]))
 		wr_mas->offset_end++;
 
-	if (wr_mas->offset_end < wr_mas->node_end)
+	if (wr_mas->offset_end < wr_mas->mas->end)
 		wr_mas->end_piv = wr_mas->pivots[wr_mas->offset_end];
 	else
 		wr_mas->end_piv = wr_mas->mas->max;
@@ -4107,7 +4069,7 @@ static inline void mas_wr_end_piv(struct ma_wr_state *wr_mas)
 static inline unsigned char mas_wr_new_end(struct ma_wr_state *wr_mas)
 {
 	struct ma_state *mas = wr_mas->mas;
-	unsigned char new_end = wr_mas->node_end + 2;
+	unsigned char new_end = mas->end + 2;
 
 	new_end -= wr_mas->offset_end - mas->offset;
 	if (wr_mas->r_min == mas->index)
@@ -4141,10 +4103,7 @@ static inline bool mas_wr_append(struct ma_wr_state *wr_mas,
 	if (mt_in_rcu(mas->tree))
 		return false;
 
-	if (mas->offset != wr_mas->node_end)
-		return false;
-
-	end = wr_mas->node_end;
+	end = mas->end;
 	if (mas->offset != end)
 		return false;
 
@@ -4178,6 +4137,7 @@ static inline bool mas_wr_append(struct ma_wr_state *wr_mas,
 	if (!wr_mas->content || !wr_mas->entry)
 		mas_update_gap(mas);
 
+	mas->end = new_end;
 	trace_ma_write(__func__, mas, new_end, wr_mas->entry);
 	return  true;
 }
@@ -4195,7 +4155,7 @@ static void mas_wr_bnode(struct ma_wr_state *wr_mas)
 	trace_ma_write(__func__, wr_mas->mas, 0, wr_mas->entry);
 	memset(&b_node, 0, sizeof(struct maple_big_node));
 	mas_store_b_node(wr_mas, &b_node, wr_mas->offset_end);
-	mas_commit_b_node(wr_mas, &b_node, wr_mas->node_end);
+	mas_commit_b_node(wr_mas, &b_node, wr_mas->mas->end);
 }
 
 static inline void mas_wr_modify(struct ma_wr_state *wr_mas)
@@ -4223,7 +4183,7 @@ static inline void mas_wr_modify(struct ma_wr_state *wr_mas)
 	if (mas_wr_append(wr_mas, new_end))
 		return;
 
-	if (new_end == wr_mas->node_end && mas_wr_slot_store(wr_mas))
+	if (new_end == mas->end && mas_wr_slot_store(wr_mas))
 		return;
 
 	if (mas_wr_node_store(wr_mas, new_end))
@@ -4328,7 +4288,7 @@ static inline void *mas_insert(struct ma_state *mas, void *entry)
 
 }
 
-static inline void mas_rewalk(struct ma_state *mas, unsigned long index)
+static __always_inline void mas_rewalk(struct ma_state *mas, unsigned long index)
 {
 retry:
 	mas_set(mas, index);
@@ -4337,7 +4297,7 @@ static inline void mas_rewalk(struct ma_state *mas, unsigned long index)
 		goto retry;
 }
 
-static inline bool mas_rewalk_if_dead(struct ma_state *mas,
+static __always_inline bool mas_rewalk_if_dead(struct ma_state *mas,
 		struct maple_node *node, const unsigned long index)
 {
 	if (unlikely(ma_dead_node(node))) {
@@ -4349,14 +4309,16 @@ static inline bool mas_rewalk_if_dead(struct ma_state *mas,
 
 /*
  * mas_prev_node() - Find the prev non-null entry at the same level in the
- * tree.  The prev value will be mas->node[mas->offset] or MAS_NONE.
+ * tree.  The prev value will be mas->node[mas->offset] or the status will be
+ * ma_none.
  * @mas: The maple state
  * @min: The lower limit to search
  *
- * The prev node value will be mas->node[mas->offset] or MAS_NONE.
+ * The prev node value will be mas->node[mas->offset] or the status will be
+ * ma_none.
  * Return: 1 if the node is dead, 0 otherwise.
  */
-static inline int mas_prev_node(struct ma_state *mas, unsigned long min)
+static int mas_prev_node(struct ma_state *mas, unsigned long min)
 {
 	enum maple_type mt;
 	int offset, level;
@@ -4416,13 +4378,14 @@ static inline int mas_prev_node(struct ma_state *mas, unsigned long min)
 	if (unlikely(mte_dead_node(mas->node)))
 		return 1;
 
+	mas->end = mas->offset;
 	return 0;
 
 no_entry:
 	if (unlikely(ma_dead_node(node)))
 		return 1;
 
-	mas->node = MAS_NONE;
+	mas->status = ma_underflow;
 	return 0;
 }
 
@@ -4436,8 +4399,7 @@ static inline int mas_prev_node(struct ma_state *mas, unsigned long min)
  *
  * Return: The entry in the previous slot which is possibly NULL
  */
-static void *mas_prev_slot(struct ma_state *mas, unsigned long min, bool empty,
-			   bool set_underflow)
+static void *mas_prev_slot(struct ma_state *mas, unsigned long min, bool empty)
 {
 	void *entry;
 	void __rcu **slots;
@@ -4475,8 +4437,8 @@ static void *mas_prev_slot(struct ma_state *mas, unsigned long min, bool empty,
 			goto retry;
 		}
 
-		if (mas_is_none(mas))
-			goto underflow;
+		if (WARN_ON_ONCE(mas_is_underflow(mas)))
+			return NULL;
 
 		mas->last = mas->max;
 		node = mas_mn(mas);
@@ -4490,12 +4452,15 @@ static void *mas_prev_slot(struct ma_state *mas, unsigned long min, bool empty,
 	if (unlikely(mas_rewalk_if_dead(mas, node, save_point)))
 		goto retry;
 
+	if (mas->index <= min)
+		mas->status = ma_underflow;
+
 	if (likely(entry))
 		return entry;
 
 	if (!empty) {
-		if (mas->index <= min)
-			goto underflow;
+		if (mas_is_underflow(mas))
+			return NULL;
 
 		goto again;
 	}
@@ -4503,8 +4468,7 @@ static void *mas_prev_slot(struct ma_state *mas, unsigned long min, bool empty,
 	return entry;
 
 underflow:
-	if (set_underflow)
-		mas->node = MAS_UNDERFLOW;
+	mas->status = ma_underflow;
 	return NULL;
 }
 
@@ -4513,28 +4477,30 @@ static void *mas_prev_slot(struct ma_state *mas, unsigned long min, bool empty,
  * @mas: The maple state
  * @max: The maximum pivot value to check.
  *
- * The next value will be mas->node[mas->offset] or MAS_NONE.
+ * The next value will be mas->node[mas->offset] or the status will have
+ * overflowed.
  * Return: 1 on dead node, 0 otherwise.
  */
-static inline int mas_next_node(struct ma_state *mas, struct maple_node *node,
-				unsigned long max)
+static int mas_next_node(struct ma_state *mas, struct maple_node *node,
+		unsigned long max)
 {
 	unsigned long min;
 	unsigned long *pivots;
 	struct maple_enode *enode;
+	struct maple_node *tmp;
 	int level = 0;
 	unsigned char node_end;
 	enum maple_type mt;
 	void __rcu **slots;
 
 	if (mas->max >= max)
-		goto no_entry;
+		goto overflow;
 
 	min = mas->max + 1;
 	level = 0;
 	do {
 		if (ma_is_root(node))
-			goto no_entry;
+			goto overflow;
 
 		/* Walk up. */
 		if (unlikely(mas_ascend(mas)))
@@ -4574,6 +4540,10 @@ static inline int mas_next_node(struct ma_state *mas, struct maple_node *node,
 		pivots = ma_pivots(node, mt);
 
 	mas->max = mas_safe_pivot(mas, pivots, mas->offset, mt);
+	tmp = mte_to_node(enode);
+	mt = mte_node_type(enode);
+	pivots = ma_pivots(tmp, mt);
+	mas->end = ma_data_end(tmp, mt, pivots, mas->max);
 	if (unlikely(ma_dead_node(node)))
 		return 1;
 
@@ -4581,11 +4551,11 @@ static inline int mas_next_node(struct ma_state *mas, struct maple_node *node,
 	mas->min = min;
 	return 0;
 
-no_entry:
+overflow:
 	if (unlikely(ma_dead_node(node)))
 		return 1;
 
-	mas->node = MAS_NONE;
+	mas->status = ma_overflow;
 	return 0;
 }
 
@@ -4600,15 +4570,13 @@ static inline int mas_next_node(struct ma_state *mas, struct maple_node *node,
  *
  * Return: The entry in the next slot which is possibly NULL
  */
-static void *mas_next_slot(struct ma_state *mas, unsigned long max, bool empty,
-			   bool set_overflow)
+static void *mas_next_slot(struct ma_state *mas, unsigned long max, bool empty)
 {
 	void __rcu **slots;
 	unsigned long *pivots;
 	unsigned long pivot;
 	enum maple_type type;
 	struct maple_node *node;
-	unsigned char data_end;
 	unsigned long save_point = mas->last;
 	void *entry;
 
@@ -4616,28 +4584,29 @@ static void *mas_next_slot(struct ma_state *mas, unsigned long max, bool empty,
 	node = mas_mn(mas);
 	type = mte_node_type(mas->node);
 	pivots = ma_pivots(node, type);
-	data_end = ma_data_end(node, type, pivots, mas->max);
 	if (unlikely(mas_rewalk_if_dead(mas, node, save_point)))
 		goto retry;
 
 	if (mas->max >= max) {
-		if (likely(mas->offset < data_end))
+		if (likely(mas->offset < mas->end))
 			pivot = pivots[mas->offset];
 		else
-			goto overflow;
+			pivot = mas->max;
 
 		if (unlikely(mas_rewalk_if_dead(mas, node, save_point)))
 			goto retry;
 
-		if (pivot >= max)
-			goto overflow;
+		if (pivot >= max) {
+			mas->status = ma_overflow;
+			return NULL;
+		}
 	}
 
-	if (likely(mas->offset < data_end)) {
+	if (likely(mas->offset < mas->end)) {
 		mas->index = pivots[mas->offset] + 1;
 again:
 		mas->offset++;
-		if (likely(mas->offset < data_end))
+		if (likely(mas->offset < mas->end))
 			mas->last = pivots[mas->offset];
 		else
 			mas->last = mas->max;
@@ -4647,11 +4616,8 @@ static void *mas_next_slot(struct ma_state *mas, unsigned long max, bool empty,
 			goto retry;
 		}
 
-		if (WARN_ON_ONCE(mas_is_none(mas))) {
-			mas->node = MAS_OVERFLOW;
+		if (WARN_ON_ONCE(mas_is_overflow(mas)))
 			return NULL;
-			goto overflow;
-		}
 
 		mas->offset = 0;
 		mas->index = mas->min;
@@ -4669,21 +4635,18 @@ static void *mas_next_slot(struct ma_state *mas, unsigned long max, bool empty,
 	if (entry)
 		return entry;
 
+
 	if (!empty) {
-		if (mas->last >= max)
-			goto overflow;
+		if (mas->last >= max) {
+			mas->status = ma_overflow;
+			return NULL;
+		}
 
 		mas->index = mas->last + 1;
-		/* Node cannot end on NULL, so it's safe to short-cut here */
 		goto again;
 	}
 
 	return entry;
-
-overflow:
-	if (set_overflow)
-		mas->node = MAS_OVERFLOW;
-	return NULL;
 }
 
 /*
@@ -4702,11 +4665,11 @@ static void *mas_next_slot(struct ma_state *mas, unsigned long max, bool empty,
 static inline void *mas_next_entry(struct ma_state *mas, unsigned long limit)
 {
 	if (mas->last >= limit) {
-		mas->node = MAS_OVERFLOW;
+		mas->status = ma_overflow;
 		return NULL;
 	}
 
-	return mas_next_slot(mas, limit, false, true);
+	return mas_next_slot(mas, limit, false);
 }
 
 /*
@@ -4874,7 +4837,7 @@ static inline bool mas_anode_descend(struct ma_state *mas, unsigned long size)
  * @mas: The maple state.
  *
  * mas->index and mas->last will be set to the range if there is a value.  If
- * mas->node is MAS_NONE, reset to MAS_START.
+ * mas->status is ma_none, reset to ma_start
  *
  * Return: the entry at the location or %NULL.
  */
@@ -4883,7 +4846,7 @@ void *mas_walk(struct ma_state *mas)
 	void *entry;
 
 	if (!mas_is_active(mas) || !mas_is_start(mas))
-		mas->node = MAS_START;
+		mas->status = ma_start;
 retry:
 	entry = mas_state_walk(mas);
 	if (mas_is_start(mas)) {
@@ -4899,7 +4862,7 @@ void *mas_walk(struct ma_state *mas)
 
 		mas->index = 1;
 		mas->last = ULONG_MAX;
-		mas->node = MAS_NONE;
+		mas->status = ma_none;
 		return NULL;
 	}
 
@@ -5026,6 +4989,7 @@ int mas_empty_area(struct ma_state *mas, unsigned long min,
 	unsigned char offset;
 	unsigned long *pivots;
 	enum maple_type mt;
+	struct maple_node *node;
 
 	if (min > max)
 		return -EINVAL;
@@ -5056,12 +5020,14 @@ int mas_empty_area(struct ma_state *mas, unsigned long min,
 	if (unlikely(offset == MAPLE_NODE_SLOTS))
 		return -EBUSY;
 
+	node = mas_mn(mas);
 	mt = mte_node_type(mas->node);
-	pivots = ma_pivots(mas_mn(mas), mt);
+	pivots = ma_pivots(node, mt);
 	min = mas_safe_min(mas, pivots, offset);
 	if (mas->index < min)
 		mas->index = min;
 	mas->last = mas->index + size - 1;
+	mas->end = ma_data_end(node, mt, pivots, mas->max);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(mas_empty_area);
@@ -5122,6 +5088,7 @@ int mas_empty_area_rev(struct ma_state *mas, unsigned long min,
 		mas->last = max;
 
 	mas->index = mas->last - size + 1;
+	mas->end = mas_data_end(mas);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(mas_empty_area_rev);
@@ -5555,7 +5522,7 @@ void mas_destroy(struct ma_state *mas)
 
 		mas_start(mas);
 		mtree_range_walk(mas);
-		end = mas_data_end(mas) + 1;
+		end = mas->end + 1;
 		if (end < mt_min_slot_count(mas->node) - 1)
 			mas_destroy_rebalance(mas, end);
 
@@ -5573,7 +5540,7 @@ void mas_destroy(struct ma_state *mas)
 			mt_free_bulk(count, (void __rcu **)&node->slot[1]);
 			total -= count;
 		}
-		kmem_cache_free(maple_node_cache, node);
+		mt_free_one(ma_mnode_ptr(node));
 		total--;
 	}
 
@@ -5643,33 +5610,46 @@ int mas_expected_entries(struct ma_state *mas, unsigned long nr_entries)
 }
 EXPORT_SYMBOL_GPL(mas_expected_entries);
 
-static inline bool mas_next_setup(struct ma_state *mas, unsigned long max,
+static bool mas_next_setup(struct ma_state *mas, unsigned long max,
 		void **entry)
 {
 	bool was_none = mas_is_none(mas);
 
 	if (unlikely(mas->last >= max)) {
-		mas->node = MAS_OVERFLOW;
+		mas->status = ma_overflow;
 		return true;
 	}
 
-	if (mas_is_active(mas))
+	switch (mas->status) {
+	case ma_active:
 		return false;
-
-	if (mas_is_none(mas) || mas_is_paused(mas)) {
-		mas->node = MAS_START;
-	} else if (mas_is_overflow(mas)) {
+	case ma_none:
+		fallthrough;
+	case ma_pause:
+		mas->status = ma_start;
+		fallthrough;
+	case ma_start:
+		mas_walk(mas); /* Retries on dead nodes handled by mas_walk */
+		break;
+	case ma_overflow:
 		/* Overflowed before, but the max changed */
-		mas->node = MAS_START;
-	} else if (mas_is_underflow(mas)) {
-		mas->node = MAS_START;
+		mas->status = ma_active;
+		break;
+	case ma_underflow:
+		/* The user expects the mas to be one before where it is */
+		mas->status = ma_active;
 		*entry = mas_walk(mas);
 		if (*entry)
 			return true;
+		break;
+	case ma_root:
+		break;
+	case ma_error:
+		return true;
 	}
 
-	if (mas_is_start(mas))
-		*entry = mas_walk(mas); /* Retries on dead nodes handled by mas_walk */
+	if (likely(mas_is_active(mas))) /* Fast path */
+		return false;
 
 	if (mas_is_ptr(mas)) {
 		*entry = NULL;
@@ -5679,7 +5659,7 @@ static inline bool mas_next_setup(struct ma_state *mas, unsigned long max,
 		}
 		mas->index = 1;
 		mas->last = ULONG_MAX;
-		mas->node = MAS_NONE;
+		mas->status = ma_none;
 		return true;
 	}
 
@@ -5708,7 +5688,7 @@ void *mas_next(struct ma_state *mas, unsigned long max)
 		return entry;
 
 	/* Retries on dead nodes handled by mas_next_slot */
-	return mas_next_slot(mas, max, false, true);
+	return mas_next_slot(mas, max, false);
 }
 EXPORT_SYMBOL_GPL(mas_next);
 
@@ -5731,7 +5711,7 @@ void *mas_next_range(struct ma_state *mas, unsigned long max)
 		return entry;
 
 	/* Retries on dead nodes handled by mas_next_slot */
-	return mas_next_slot(mas, max, true, true);
+	return mas_next_slot(mas, max, true);
 }
 EXPORT_SYMBOL_GPL(mas_next_range);
 
@@ -5759,37 +5739,48 @@ void *mt_next(struct maple_tree *mt, unsigned long index, unsigned long max)
 }
 EXPORT_SYMBOL_GPL(mt_next);
 
-static inline bool mas_prev_setup(struct ma_state *mas, unsigned long min,
-		void **entry)
+static bool mas_prev_setup(struct ma_state *mas, unsigned long min, void **entry)
 {
 	if (unlikely(mas->index <= min)) {
-		mas->node = MAS_UNDERFLOW;
+		mas->status = ma_underflow;
 		return true;
 	}
 
-	if (mas_is_active(mas))
+	switch (mas->status) {
+	case ma_active:
 		return false;
-
-	if (mas_is_overflow(mas)) {
-		mas->node = MAS_START;
+	case ma_start:
+		break;
+	case ma_none:
+		fallthrough;
+	case ma_pause:
+		mas->status = ma_start;
+		break;
+	case ma_underflow:
+		/* underflowed before but the min changed */
+		mas->status = ma_active;
+		break;
+	case ma_overflow:
+		/* User expects mas to be one after where it is */
+		mas->status = ma_active;
 		*entry = mas_walk(mas);
 		if (*entry)
 			return true;
-	}
-
-	if (mas_is_none(mas) || mas_is_paused(mas)) {
-		mas->node = MAS_START;
-	} else if (mas_is_underflow(mas)) {
-		/* underflowed before but the min changed */
-		mas->node = MAS_START;
+		break;
+	case ma_root:
+		break;
+	case ma_error:
+		return true;
 	}
 
 	if (mas_is_start(mas))
 		mas_walk(mas);
 
 	if (unlikely(mas_is_ptr(mas))) {
-		if (!mas->index)
-			goto none;
+		if (!mas->index) {
+			mas->status = ma_none;
+			return true;
+		}
 		mas->index = mas->last = 0;
 		*entry = mas_root(mas);
 		return true;
@@ -5799,7 +5790,7 @@ static inline bool mas_prev_setup(struct ma_state *mas, unsigned long min,
 		if (mas->index) {
 			/* Walked to out-of-range pointer? */
 			mas->index = mas->last = 0;
-			mas->node = MAS_ROOT;
+			mas->status = ma_root;
 			*entry = mas_root(mas);
 			return true;
 		}
@@ -5807,10 +5798,6 @@ static inline bool mas_prev_setup(struct ma_state *mas, unsigned long min,
 	}
 
 	return false;
-
-none:
-	mas->node = MAS_NONE;
-	return true;
 }
 
 /**
@@ -5819,7 +5806,7 @@ static inline bool mas_prev_setup(struct ma_state *mas, unsigned long min,
  * @min: The minimum value to check.
  *
  * Must hold rcu_read_lock or the write lock.
- * Will reset mas to MAS_START if the node is MAS_NONE.  Will stop on not
+ * Will reset mas to ma_start if the status is ma_none.  Will stop on not
  * searchable nodes.
  *
  * Return: the previous value or %NULL.
@@ -5831,7 +5818,7 @@ void *mas_prev(struct ma_state *mas, unsigned long min)
 	if (mas_prev_setup(mas, min, &entry))
 		return entry;
 
-	return mas_prev_slot(mas, min, false, true);
+	return mas_prev_slot(mas, min, false);
 }
 EXPORT_SYMBOL_GPL(mas_prev);
 
@@ -5842,7 +5829,7 @@ EXPORT_SYMBOL_GPL(mas_prev);
  *
  * Sets @mas->index and @mas->last to the range.
  * Must hold rcu_read_lock or the write lock.
- * Will reset mas to MAS_START if the node is MAS_NONE.  Will stop on not
+ * Will reset mas to ma_start if the node is ma_none.  Will stop on not
  * searchable nodes.
  *
  * Return: the previous value or %NULL.
@@ -5854,7 +5841,7 @@ void *mas_prev_range(struct ma_state *mas, unsigned long min)
 	if (mas_prev_setup(mas, min, &entry))
 		return entry;
 
-	return mas_prev_slot(mas, min, true, true);
+	return mas_prev_slot(mas, min, true);
 }
 EXPORT_SYMBOL_GPL(mas_prev_range);
 
@@ -5897,7 +5884,8 @@ EXPORT_SYMBOL_GPL(mt_prev);
  */
 void mas_pause(struct ma_state *mas)
 {
-	mas->node = MAS_PAUSE;
+	mas->status = ma_pause;
+	mas->node = NULL;
 }
 EXPORT_SYMBOL_GPL(mas_pause);
 
@@ -5909,35 +5897,54 @@ EXPORT_SYMBOL_GPL(mas_pause);
  *
  * Returns: True if entry is the answer, false otherwise.
  */
-static inline bool mas_find_setup(struct ma_state *mas, unsigned long max,
-		void **entry)
+static __always_inline bool mas_find_setup(struct ma_state *mas, unsigned long max, void **entry)
 {
-	if (mas_is_active(mas)) {
+	switch (mas->status) {
+	case ma_active:
 		if (mas->last < max)
 			return false;
-
 		return true;
-	}
-
-	if (mas_is_paused(mas)) {
+	case ma_start:
+		break;
+	case ma_pause:
 		if (unlikely(mas->last >= max))
 			return true;
 
 		mas->index = ++mas->last;
-		mas->node = MAS_START;
-	} else if (mas_is_none(mas)) {
+		mas->status = ma_start;
+		break;
+	case ma_none:
 		if (unlikely(mas->last >= max))
 			return true;
 
 		mas->index = mas->last;
-		mas->node = MAS_START;
-	} else if (mas_is_overflow(mas) || mas_is_underflow(mas)) {
-		if (mas->index > max) {
-			mas->node = MAS_OVERFLOW;
+		mas->status = ma_start;
+		break;
+	case ma_underflow:
+		/* mas is pointing at entry before unable to go lower */
+		if (unlikely(mas->index >= max)) {
+			mas->status = ma_overflow;
 			return true;
 		}
 
-		mas->node = MAS_START;
+		mas->status = ma_active;
+		*entry = mas_walk(mas);
+		if (*entry)
+			return true;
+		break;
+	case ma_overflow:
+		if (unlikely(mas->last >= max))
+			return true;
+
+		mas->status = ma_active;
+		*entry = mas_walk(mas);
+		if (*entry)
+			return true;
+		break;
+	case ma_root:
+		break;
+	case ma_error:
+		return true;
 	}
 
 	if (mas_is_start(mas)) {
@@ -5951,12 +5958,11 @@ static inline bool mas_find_setup(struct ma_state *mas, unsigned long max,
 
 	}
 
-	if (unlikely(!mas_searchable(mas))) {
-		if (unlikely(mas_is_ptr(mas)))
-			goto ptr_out_of_range;
+	if (unlikely(mas_is_ptr(mas)))
+		goto ptr_out_of_range;
 
+	if (unlikely(mas_is_none(mas)))
 		return true;
-	}
 
 	if (mas->index == max)
 		return true;
@@ -5964,7 +5970,7 @@ static inline bool mas_find_setup(struct ma_state *mas, unsigned long max,
 	return false;
 
 ptr_out_of_range:
-	mas->node = MAS_NONE;
+	mas->status = ma_none;
 	mas->index = 1;
 	mas->last = ULONG_MAX;
 	return true;
@@ -5978,7 +5984,7 @@ static inline bool mas_find_setup(struct ma_state *mas, unsigned long max,
  *
  * Must hold rcu_read_lock or the write lock.
  * If an entry exists, last and index are updated accordingly.
- * May set @mas->node to MAS_NONE.
+ * May set @mas->status to ma_overflow.
  *
  * Return: The entry or %NULL.
  */
@@ -5990,7 +5996,10 @@ void *mas_find(struct ma_state *mas, unsigned long max)
 		return entry;
 
 	/* Retries on dead nodes handled by mas_next_slot */
-	return mas_next_slot(mas, max, false, false);
+	entry = mas_next_slot(mas, max, false);
+	/* Ignore overflow */
+	mas->status = ma_active;
+	return entry;
 }
 EXPORT_SYMBOL_GPL(mas_find);
 
@@ -6002,7 +6011,7 @@ EXPORT_SYMBOL_GPL(mas_find);
  *
  * Must hold rcu_read_lock or the write lock.
  * If an entry exists, last and index are updated accordingly.
- * May set @mas->node to MAS_NONE.
+ * May set @mas->status to ma_overflow.
  *
  * Return: The entry or %NULL.
  */
@@ -6014,7 +6023,7 @@ void *mas_find_range(struct ma_state *mas, unsigned long max)
 		return entry;
 
 	/* Retries on dead nodes handled by mas_next_slot */
-	return mas_next_slot(mas, max, true, false);
+	return mas_next_slot(mas, max, true);
 }
 EXPORT_SYMBOL_GPL(mas_find_range);
 
@@ -6026,36 +6035,48 @@ EXPORT_SYMBOL_GPL(mas_find_range);
  *
  * Returns: True if entry is the answer, false otherwise.
  */
-static inline bool mas_find_rev_setup(struct ma_state *mas, unsigned long min,
+static bool mas_find_rev_setup(struct ma_state *mas, unsigned long min,
 		void **entry)
 {
-	if (mas_is_active(mas)) {
-		if (mas->index > min)
-			return false;
-
-		return true;
-	}
 
-	if (mas_is_paused(mas)) {
+	switch (mas->status) {
+	case ma_active:
+		goto active;
+	case ma_start:
+		break;
+	case ma_pause:
 		if (unlikely(mas->index <= min)) {
-			mas->node = MAS_NONE;
+			mas->status = ma_underflow;
 			return true;
 		}
-		mas->node = MAS_START;
 		mas->last = --mas->index;
-	} else if (mas_is_none(mas)) {
+		mas->status = ma_start;
+		break;
+	case ma_none:
 		if (mas->index <= min)
 			goto none;
 
 		mas->last = mas->index;
-		mas->node = MAS_START;
-	} else if (mas_is_underflow(mas) || mas_is_overflow(mas)) {
-		if (mas->last <= min) {
-			mas->node = MAS_UNDERFLOW;
+		mas->status = ma_start;
+		break;
+	case ma_overflow: /* user expects the mas to be one after where it is */
+		if (unlikely(mas->index <= min)) {
+			mas->status = ma_underflow;
 			return true;
 		}
 
-		mas->node = MAS_START;
+		mas->status = ma_active;
+		break;
+	case ma_underflow: /* user expects the mas to be one before where it is */
+		if (unlikely(mas->index <= min))
+			return true;
+
+		mas->status = ma_active;
+		break;
+	case ma_root:
+		break;
+	case ma_error:
+		return true;
 	}
 
 	if (mas_is_start(mas)) {
@@ -6068,29 +6089,28 @@ static inline bool mas_find_rev_setup(struct ma_state *mas, unsigned long min,
 			return true;
 	}
 
-	if (unlikely(!mas_searchable(mas))) {
-		if (mas_is_ptr(mas))
-			goto none;
+	if (unlikely(mas_is_ptr(mas)))
+		goto none;
 
-		if (mas_is_none(mas)) {
-			/*
-			 * Walked to the location, and there was nothing so the
-			 * previous location is 0.
-			 */
-			mas->last = mas->index = 0;
-			mas->node = MAS_ROOT;
-			*entry = mas_root(mas);
-			return true;
-		}
+	if (unlikely(mas_is_none(mas))) {
+		/*
+		 * Walked to the location, and there was nothing so the previous
+		 * location is 0.
+		 */
+		mas->last = mas->index = 0;
+		mas->status = ma_root;
+		*entry = mas_root(mas);
+		return true;
 	}
 
+active:
 	if (mas->index < min)
 		return true;
 
 	return false;
 
 none:
-	mas->node = MAS_NONE;
+	mas->status = ma_none;
 	return true;
 }
 
@@ -6103,7 +6123,7 @@ static inline bool mas_find_rev_setup(struct ma_state *mas, unsigned long min,
  *
  * Must hold rcu_read_lock or the write lock.
  * If an entry exists, last and index are updated accordingly.
- * May set @mas->node to MAS_NONE.
+ * May set @mas->status to ma_underflow.
  *
  * Return: The entry or %NULL.
  */
@@ -6115,7 +6135,7 @@ void *mas_find_rev(struct ma_state *mas, unsigned long min)
 		return entry;
 
 	/* Retries on dead nodes handled by mas_prev_slot */
-	return mas_prev_slot(mas, min, false, false);
+	return mas_prev_slot(mas, min, false);
 
 }
 EXPORT_SYMBOL_GPL(mas_find_rev);
@@ -6129,7 +6149,7 @@ EXPORT_SYMBOL_GPL(mas_find_rev);
  *
  * Must hold rcu_read_lock or the write lock.
  * If an entry exists, last and index are updated accordingly.
- * May set @mas->node to MAS_NONE.
+ * May set @mas->status to ma_underflow.
  *
  * Return: The entry or %NULL.
  */
@@ -6141,7 +6161,7 @@ void *mas_find_range_rev(struct ma_state *mas, unsigned long min)
 		return entry;
 
 	/* Retries on dead nodes handled by mas_prev_slot */
-	return mas_prev_slot(mas, min, true, false);
+	return mas_prev_slot(mas, min, true);
 }
 EXPORT_SYMBOL_GPL(mas_find_range_rev);
 
@@ -6161,8 +6181,8 @@ void *mas_erase(struct ma_state *mas)
 	void *entry;
 	MA_WR_STATE(wr_mas, mas, NULL);
 
-	if (mas_is_none(mas) || mas_is_paused(mas))
-		mas->node = MAS_START;
+	if (!mas_is_active(mas) || !mas_is_start(mas))
+		mas->status = ma_start;
 
 	/* Retry unnecessary when holding the write lock. */
 	entry = mas_state_walk(mas);
@@ -6207,7 +6227,7 @@ bool mas_nomem(struct ma_state *mas, gfp_t gfp)
 	if (!mas_allocated(mas))
 		return false;
 
-	mas->node = MAS_START;
+	mas->status = ma_start;
 	return true;
 }
 
@@ -6465,6 +6485,278 @@ void *mtree_erase(struct maple_tree *mt, unsigned long index)
 }
 EXPORT_SYMBOL(mtree_erase);
 
+/*
+ * mas_dup_free() - Free an incomplete duplication of a tree.
+ * @mas: The maple state of a incomplete tree.
+ *
+ * The parameter @mas->node passed in indicates that the allocation failed on
+ * this node. This function frees all nodes starting from @mas->node in the
+ * reverse order of mas_dup_build(). There is no need to hold the source tree
+ * lock at this time.
+ */
+static void mas_dup_free(struct ma_state *mas)
+{
+	struct maple_node *node;
+	enum maple_type type;
+	void __rcu **slots;
+	unsigned char count, i;
+
+	/* Maybe the first node allocation failed. */
+	if (mas_is_none(mas))
+		return;
+
+	while (!mte_is_root(mas->node)) {
+		mas_ascend(mas);
+		if (mas->offset) {
+			mas->offset--;
+			do {
+				mas_descend(mas);
+				mas->offset = mas_data_end(mas);
+			} while (!mte_is_leaf(mas->node));
+
+			mas_ascend(mas);
+		}
+
+		node = mte_to_node(mas->node);
+		type = mte_node_type(mas->node);
+		slots = ma_slots(node, type);
+		count = mas_data_end(mas) + 1;
+		for (i = 0; i < count; i++)
+			((unsigned long *)slots)[i] &= ~MAPLE_NODE_MASK;
+		mt_free_bulk(count, slots);
+	}
+
+	node = mte_to_node(mas->node);
+	mt_free_one(node);
+}
+
+/*
+ * mas_copy_node() - Copy a maple node and replace the parent.
+ * @mas: The maple state of source tree.
+ * @new_mas: The maple state of new tree.
+ * @parent: The parent of the new node.
+ *
+ * Copy @mas->node to @new_mas->node, set @parent to be the parent of
+ * @new_mas->node. If memory allocation fails, @mas is set to -ENOMEM.
+ */
+static inline void mas_copy_node(struct ma_state *mas, struct ma_state *new_mas,
+		struct maple_pnode *parent)
+{
+	struct maple_node *node = mte_to_node(mas->node);
+	struct maple_node *new_node = mte_to_node(new_mas->node);
+	unsigned long val;
+
+	/* Copy the node completely. */
+	memcpy(new_node, node, sizeof(struct maple_node));
+	/* Update the parent node pointer. */
+	val = (unsigned long)node->parent & MAPLE_NODE_MASK;
+	new_node->parent = ma_parent_ptr(val | (unsigned long)parent);
+}
+
+/*
+ * mas_dup_alloc() - Allocate child nodes for a maple node.
+ * @mas: The maple state of source tree.
+ * @new_mas: The maple state of new tree.
+ * @gfp: The GFP_FLAGS to use for allocations.
+ *
+ * This function allocates child nodes for @new_mas->node during the duplication
+ * process. If memory allocation fails, @mas is set to -ENOMEM.
+ */
+static inline void mas_dup_alloc(struct ma_state *mas, struct ma_state *new_mas,
+		gfp_t gfp)
+{
+	struct maple_node *node = mte_to_node(mas->node);
+	struct maple_node *new_node = mte_to_node(new_mas->node);
+	enum maple_type type;
+	unsigned char request, count, i;
+	void __rcu **slots;
+	void __rcu **new_slots;
+	unsigned long val;
+
+	/* Allocate memory for child nodes. */
+	type = mte_node_type(mas->node);
+	new_slots = ma_slots(new_node, type);
+	request = mas_data_end(mas) + 1;
+	count = mt_alloc_bulk(gfp, request, (void **)new_slots);
+	if (unlikely(count < request)) {
+		memset(new_slots, 0, request * sizeof(void *));
+		mas_set_err(mas, -ENOMEM);
+		return;
+	}
+
+	/* Restore node type information in slots. */
+	slots = ma_slots(node, type);
+	for (i = 0; i < count; i++) {
+		val = (unsigned long)mt_slot_locked(mas->tree, slots, i);
+		val &= MAPLE_NODE_MASK;
+		((unsigned long *)new_slots)[i] |= val;
+	}
+}
+
+/*
+ * mas_dup_build() - Build a new maple tree from a source tree
+ * @mas: The maple state of source tree, need to be in MAS_START state.
+ * @new_mas: The maple state of new tree, need to be in MAS_START state.
+ * @gfp: The GFP_FLAGS to use for allocations.
+ *
+ * This function builds a new tree in DFS preorder. If the memory allocation
+ * fails, the error code -ENOMEM will be set in @mas, and @new_mas points to the
+ * last node. mas_dup_free() will free the incomplete duplication of a tree.
+ *
+ * Note that the attributes of the two trees need to be exactly the same, and the
+ * new tree needs to be empty, otherwise -EINVAL will be set in @mas.
+ */
+static inline void mas_dup_build(struct ma_state *mas, struct ma_state *new_mas,
+		gfp_t gfp)
+{
+	struct maple_node *node;
+	struct maple_pnode *parent = NULL;
+	struct maple_enode *root;
+	enum maple_type type;
+
+	if (unlikely(mt_attr(mas->tree) != mt_attr(new_mas->tree)) ||
+	    unlikely(!mtree_empty(new_mas->tree))) {
+		mas_set_err(mas, -EINVAL);
+		return;
+	}
+
+	root = mas_start(mas);
+	if (mas_is_ptr(mas) || mas_is_none(mas))
+		goto set_new_tree;
+
+	node = mt_alloc_one(gfp);
+	if (!node) {
+		new_mas->status = ma_none;
+		mas_set_err(mas, -ENOMEM);
+		return;
+	}
+
+	type = mte_node_type(mas->node);
+	root = mt_mk_node(node, type);
+	new_mas->node = root;
+	new_mas->min = 0;
+	new_mas->max = ULONG_MAX;
+	root = mte_mk_root(root);
+	while (1) {
+		mas_copy_node(mas, new_mas, parent);
+		if (!mte_is_leaf(mas->node)) {
+			/* Only allocate child nodes for non-leaf nodes. */
+			mas_dup_alloc(mas, new_mas, gfp);
+			if (unlikely(mas_is_err(mas)))
+				return;
+		} else {
+			/*
+			 * This is the last leaf node and duplication is
+			 * completed.
+			 */
+			if (mas->max == ULONG_MAX)
+				goto done;
+
+			/* This is not the last leaf node and needs to go up. */
+			do {
+				mas_ascend(mas);
+				mas_ascend(new_mas);
+			} while (mas->offset == mas_data_end(mas));
+
+			/* Move to the next subtree. */
+			mas->offset++;
+			new_mas->offset++;
+		}
+
+		mas_descend(mas);
+		parent = ma_parent_ptr(mte_to_node(new_mas->node));
+		mas_descend(new_mas);
+		mas->offset = 0;
+		new_mas->offset = 0;
+	}
+done:
+	/* Specially handle the parent of the root node. */
+	mte_to_node(root)->parent = ma_parent_ptr(mas_tree_parent(new_mas));
+set_new_tree:
+	/* Make them the same height */
+	new_mas->tree->ma_flags = mas->tree->ma_flags;
+	rcu_assign_pointer(new_mas->tree->ma_root, root);
+}
+
+/**
+ * __mt_dup(): Duplicate an entire maple tree
+ * @mt: The source maple tree
+ * @new: The new maple tree
+ * @gfp: The GFP_FLAGS to use for allocations
+ *
+ * This function duplicates a maple tree in Depth-First Search (DFS) pre-order
+ * traversal. It uses memcpy() to copy nodes in the source tree and allocate
+ * new child nodes in non-leaf nodes. The new node is exactly the same as the
+ * source node except for all the addresses stored in it. It will be faster than
+ * traversing all elements in the source tree and inserting them one by one into
+ * the new tree.
+ * The user needs to ensure that the attributes of the source tree and the new
+ * tree are the same, and the new tree needs to be an empty tree, otherwise
+ * -EINVAL will be returned.
+ * Note that the user needs to manually lock the source tree and the new tree.
+ *
+ * Return: 0 on success, -ENOMEM if memory could not be allocated, -EINVAL If
+ * the attributes of the two trees are different or the new tree is not an empty
+ * tree.
+ */
+int __mt_dup(struct maple_tree *mt, struct maple_tree *new, gfp_t gfp)
+{
+	int ret = 0;
+	MA_STATE(mas, mt, 0, 0);
+	MA_STATE(new_mas, new, 0, 0);
+
+	mas_dup_build(&mas, &new_mas, gfp);
+	if (unlikely(mas_is_err(&mas))) {
+		ret = xa_err(mas.node);
+		if (ret == -ENOMEM)
+			mas_dup_free(&new_mas);
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL(__mt_dup);
+
+/**
+ * mtree_dup(): Duplicate an entire maple tree
+ * @mt: The source maple tree
+ * @new: The new maple tree
+ * @gfp: The GFP_FLAGS to use for allocations
+ *
+ * This function duplicates a maple tree in Depth-First Search (DFS) pre-order
+ * traversal. It uses memcpy() to copy nodes in the source tree and allocate
+ * new child nodes in non-leaf nodes. The new node is exactly the same as the
+ * source node except for all the addresses stored in it. It will be faster than
+ * traversing all elements in the source tree and inserting them one by one into
+ * the new tree.
+ * The user needs to ensure that the attributes of the source tree and the new
+ * tree are the same, and the new tree needs to be an empty tree, otherwise
+ * -EINVAL will be returned.
+ *
+ * Return: 0 on success, -ENOMEM if memory could not be allocated, -EINVAL If
+ * the attributes of the two trees are different or the new tree is not an empty
+ * tree.
+ */
+int mtree_dup(struct maple_tree *mt, struct maple_tree *new, gfp_t gfp)
+{
+	int ret = 0;
+	MA_STATE(mas, mt, 0, 0);
+	MA_STATE(new_mas, new, 0, 0);
+
+	mas_lock(&new_mas);
+	mas_lock_nested(&mas, SINGLE_DEPTH_NESTING);
+	mas_dup_build(&mas, &new_mas, gfp);
+	mas_unlock(&mas);
+	if (unlikely(mas_is_err(&mas))) {
+		ret = xa_err(mas.node);
+		if (ret == -ENOMEM)
+			mas_dup_free(&new_mas);
+	}
+
+	mas_unlock(&new_mas);
+	return ret;
+}
+EXPORT_SYMBOL(mtree_dup);
+
 /**
  * __mt_destroy() - Walk and free all nodes of a locked maple tree.
  * @mt: The maple tree
@@ -6479,7 +6771,7 @@ void __mt_destroy(struct maple_tree *mt)
 	if (xa_is_node(root))
 		mte_destroy_walk(root, mt);
 
-	mt->ma_flags = 0;
+	mt->ma_flags = mt_attr(mt);
 }
 EXPORT_SYMBOL_GPL(__mt_destroy);
 
@@ -6538,7 +6830,7 @@ void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max)
 	if (entry)
 		goto unlock;
 
-	while (mas_searchable(&mas) && (mas.last < max)) {
+	while (mas_is_active(&mas) && (mas.last < max)) {
 		entry = mas_next_entry(&mas, max);
 		if (likely(entry && !xa_is_zero(entry)))
 			break;
@@ -6620,26 +6912,6 @@ unsigned int mt_nr_allocated(void)
 	return kmem_cache_nr_allocated(maple_node_cache);
 }
 
-/*
- * mas_dead_node() - Check if the maple state is pointing to a dead node.
- * @mas: The maple state
- * @index: The index to restore in @mas.
- *
- * Used in test code.
- * Return: 1 if @mas has been reset to MAS_START, 0 otherwise.
- */
-static inline int mas_dead_node(struct ma_state *mas, unsigned long index)
-{
-	if (unlikely(!mas_searchable(mas) || mas_is_start(mas)))
-		return 0;
-
-	if (likely(!mte_dead_node(mas->node)))
-		return 0;
-
-	mas_rewalk(mas, index);
-	return 1;
-}
-
 void mt_cache_shrink(void)
 {
 }
@@ -6678,11 +6950,11 @@ static inline struct maple_enode *mas_get_slot(struct ma_state *mas,
 static void mas_dfs_postorder(struct ma_state *mas, unsigned long max)
 {
 
-	struct maple_enode *p = MAS_NONE, *mn = mas->node;
+	struct maple_enode *p, *mn = mas->node;
 	unsigned long p_min, p_max;
 
 	mas_next_node(mas, mas_mn(mas), max);
-	if (!mas_is_none(mas))
+	if (!mas_is_overflow(mas))
 		return;
 
 	if (mte_is_root(mn))
@@ -6695,7 +6967,7 @@ static void mas_dfs_postorder(struct ma_state *mas, unsigned long max)
 		p_min = mas->min;
 		p_max = mas->max;
 		mas_prev_node(mas, 0);
-	} while (!mas_is_none(mas));
+	} while (!mas_is_underflow(mas));
 
 	mas->node = p;
 	mas->max = p_max;
@@ -6718,7 +6990,6 @@ static void mt_dump_range(unsigned long min, unsigned long max,
 		else
 			pr_info("%.*s%lx-%lx: ", depth * 2, spaces, min, max);
 		break;
-	default:
 	case mt_dump_dec:
 		if (min == max)
 			pr_info("%.*s%lu: ", depth * 2, spaces, min);
@@ -6758,7 +7029,6 @@ static void mt_dump_range64(const struct maple_tree *mt, void *entry,
 		case mt_dump_hex:
 			pr_cont("%p %lX ", node->slot[i], node->pivot[i]);
 			break;
-		default:
 		case mt_dump_dec:
 			pr_cont("%p %lu ", node->slot[i], node->pivot[i]);
 		}
@@ -6788,7 +7058,6 @@ static void mt_dump_range64(const struct maple_tree *mt, void *entry,
 				pr_err("node %p last (%lx) > max (%lx) at pivot %d!\n",
 					node, last, max, i);
 				break;
-			default:
 			case mt_dump_dec:
 				pr_err("node %p last (%lu) > max (%lu) at pivot %d!\n",
 					node, last, max, i);
@@ -6813,7 +7082,6 @@ static void mt_dump_arange64(const struct maple_tree *mt, void *entry,
 		case mt_dump_hex:
 			pr_cont("%lx ", node->gap[i]);
 			break;
-		default:
 		case mt_dump_dec:
 			pr_cont("%lu ", node->gap[i]);
 		}
@@ -6824,7 +7092,6 @@ static void mt_dump_arange64(const struct maple_tree *mt, void *entry,
 		case mt_dump_hex:
 			pr_cont("%p %lX ", node->slot[i], node->pivot[i]);
 			break;
-		default:
 		case mt_dump_dec:
 			pr_cont("%p %lu ", node->slot[i], node->pivot[i]);
 		}
@@ -6965,7 +7232,8 @@ static void mas_validate_gaps(struct ma_state *mas)
 
 counted:
 	if (mt == maple_arange_64) {
-		offset = ma_meta_gap(node, mt);
+		MT_BUG_ON(mas->tree, !gaps);
+		offset = ma_meta_gap(node);
 		if (offset > i) {
 			pr_err("gap offset %p[%u] is invalid\n", node, offset);
 			MT_BUG_ON(mas->tree, 1);
@@ -6977,7 +7245,6 @@ static void mas_validate_gaps(struct ma_state *mas)
 			MT_BUG_ON(mas->tree, 1);
 		}
 
-		MT_BUG_ON(mas->tree, !gaps);
 		for (i++ ; i < mt_slot_count(mte); i++) {
 			if (gaps[i] != 0) {
 				pr_err("gap %p[%u] beyond node limit != 0\n",
@@ -7155,7 +7422,7 @@ static void mt_validate_nulls(struct maple_tree *mt)
 	MA_STATE(mas, mt, 0, 0);
 
 	mas_start(&mas);
-	if (mas_is_none(&mas) || (mas.node == MAS_ROOT))
+	if (mas_is_none(&mas) || (mas_is_ptr(&mas)))
 		return;
 
 	while (!mte_is_leaf(mas.node))
@@ -7172,7 +7439,7 @@ static void mt_validate_nulls(struct maple_tree *mt)
 		last = entry;
 		if (offset == mas_data_end(&mas)) {
 			mas_next_node(&mas, mas_mn(&mas), ULONG_MAX);
-			if (mas_is_none(&mas))
+			if (mas_is_overflow(&mas))
 				return;
 			offset = 0;
 			slots = ma_slots(mte_to_node(mas.node),
@@ -7181,7 +7448,7 @@ static void mt_validate_nulls(struct maple_tree *mt)
 			offset++;
 		}
 
-	} while (!mas_is_none(&mas));
+	} while (!mas_is_overflow(&mas));
 }
 
 /*
@@ -7196,13 +7463,13 @@ void mt_validate(struct maple_tree *mt)
 	MA_STATE(mas, mt, 0, 0);
 	rcu_read_lock();
 	mas_start(&mas);
-	if (!mas_searchable(&mas))
+	if (!mas_is_active(&mas))
 		goto done;
 
 	while (!mte_is_leaf(mas.node))
 		mas_descend(&mas);
 
-	while (!mas_is_none(&mas)) {
+	while (!mas_is_overflow(&mas)) {
 		MAS_WARN_ON(&mas, mte_dead_node(mas.node));
 		end = mas_data_end(&mas);
 		if (MAS_WARN_ON(&mas, (end < mt_min_slot_count(mas.node)) &&
@@ -7227,16 +7494,35 @@ EXPORT_SYMBOL_GPL(mt_validate);
 void mas_dump(const struct ma_state *mas)
 {
 	pr_err("MAS: tree=%p enode=%p ", mas->tree, mas->node);
-	if (mas_is_none(mas))
-		pr_err("(MAS_NONE) ");
-	else if (mas_is_ptr(mas))
-		pr_err("(MAS_ROOT) ");
-	else if (mas_is_start(mas))
-		 pr_err("(MAS_START) ");
-	else if (mas_is_paused(mas))
-		pr_err("(MAS_PAUSED) ");
-
-	pr_err("[%u] index=%lx last=%lx\n", mas->offset, mas->index, mas->last);
+	switch (mas->status) {
+	case ma_active:
+		pr_err("(ma_active)");
+		break;
+	case ma_none:
+		pr_err("(ma_none)");
+		break;
+	case ma_root:
+		pr_err("(ma_root)");
+		break;
+	case ma_start:
+		pr_err("(ma_start) ");
+		break;
+	case ma_pause:
+		pr_err("(ma_pause) ");
+		break;
+	case ma_overflow:
+		pr_err("(ma_overflow) ");
+		break;
+	case ma_underflow:
+		pr_err("(ma_underflow) ");
+		break;
+	case ma_error:
+		pr_err("(ma_error) ");
+		break;
+	}
+
+	pr_err("[%u/%u] index=%lx last=%lx\n", mas->offset, mas->end,
+	       mas->index, mas->last);
 	pr_err("     min=%lx max=%lx alloc=%p, depth=%u, flags=%x\n",
 	       mas->min, mas->max, mas->alloc, mas->depth, mas->mas_flags);
 	if (mas->index > mas->last)
@@ -7249,7 +7535,7 @@ void mas_wr_dump(const struct ma_wr_state *wr_mas)
 	pr_err("WR_MAS: node=%p r_min=%lx r_max=%lx\n",
 	       wr_mas->node, wr_mas->r_min, wr_mas->r_max);
 	pr_err("        type=%u off_end=%u, node_end=%u, end_piv=%lx\n",
-	       wr_mas->type, wr_mas->offset_end, wr_mas->node_end,
+	       wr_mas->type, wr_mas->offset_end, wr_mas->mas->end,
 	       wr_mas->end_piv);
 }
 EXPORT_SYMBOL_GPL(mas_wr_dump);
diff --git a/lib/stackdepot.c b/lib/stackdepot.c
index 2f5aa851834e..870cce2f4cbd 100644
--- a/lib/stackdepot.c
+++ b/lib/stackdepot.c
@@ -18,11 +18,14 @@
 #include <linux/jhash.h>
 #include <linux/kernel.h>
 #include <linux/kmsan.h>
+#include <linux/list.h>
 #include <linux/mm.h>
 #include <linux/mutex.h>
 #include <linux/percpu.h>
 #include <linux/printk.h>
+#include <linux/refcount.h>
 #include <linux/slab.h>
+#include <linux/spinlock.h>
 #include <linux/stacktrace.h>
 #include <linux/stackdepot.h>
 #include <linux/string.h>
@@ -32,14 +35,23 @@
 
 #define DEPOT_HANDLE_BITS (sizeof(depot_stack_handle_t) * 8)
 
-#define DEPOT_VALID_BITS 1
 #define DEPOT_POOL_ORDER 2 /* Pool size order, 4 pages */
 #define DEPOT_POOL_SIZE (1LL << (PAGE_SHIFT + DEPOT_POOL_ORDER))
 #define DEPOT_STACK_ALIGN 4
 #define DEPOT_OFFSET_BITS (DEPOT_POOL_ORDER + PAGE_SHIFT - DEPOT_STACK_ALIGN)
-#define DEPOT_POOL_INDEX_BITS (DEPOT_HANDLE_BITS - DEPOT_VALID_BITS - \
-			       DEPOT_OFFSET_BITS - STACK_DEPOT_EXTRA_BITS)
+#define DEPOT_POOL_INDEX_BITS (DEPOT_HANDLE_BITS - DEPOT_OFFSET_BITS - \
+			       STACK_DEPOT_EXTRA_BITS)
+#if IS_ENABLED(CONFIG_KMSAN) && CONFIG_STACKDEPOT_MAX_FRAMES >= 32
+/*
+ * KMSAN is frequently used in fuzzing scenarios and thus saves a lot of stack
+ * traces. As KMSAN does not support evicting stack traces from the stack
+ * depot, the stack depot capacity might be reached quickly with large stack
+ * records. Adjust the maximum number of stack depot pools for this case.
+ */
+#define DEPOT_POOLS_CAP (8192 * (CONFIG_STACKDEPOT_MAX_FRAMES / 16))
+#else
 #define DEPOT_POOLS_CAP 8192
+#endif
 #define DEPOT_MAX_POOLS \
 	(((1LL << (DEPOT_POOL_INDEX_BITS)) < DEPOT_POOLS_CAP) ? \
 	 (1LL << (DEPOT_POOL_INDEX_BITS)) : DEPOT_POOLS_CAP)
@@ -50,19 +62,22 @@ union handle_parts {
 	struct {
 		u32 pool_index	: DEPOT_POOL_INDEX_BITS;
 		u32 offset	: DEPOT_OFFSET_BITS;
-		u32 valid	: DEPOT_VALID_BITS;
 		u32 extra	: STACK_DEPOT_EXTRA_BITS;
 	};
 };
 
 struct stack_record {
-	struct stack_record *next;	/* Link in the hash table */
-	u32 hash;			/* Hash in the hash table */
+	struct list_head list;		/* Links in hash table or freelist */
+	u32 hash;			/* Hash in hash table */
 	u32 size;			/* Number of stored frames */
 	union handle_parts handle;
-	unsigned long entries[];	/* Variable-sized array of frames */
+	refcount_t count;
+	unsigned long entries[CONFIG_STACKDEPOT_MAX_FRAMES];	/* Frames */
 };
 
+#define DEPOT_STACK_RECORD_SIZE \
+	ALIGN(sizeof(struct stack_record), 1 << DEPOT_STACK_ALIGN)
+
 static bool stack_depot_disabled;
 static bool __stack_depot_early_init_requested __initdata = IS_ENABLED(CONFIG_STACKDEPOT_ALWAYS_INIT);
 static bool __stack_depot_early_init_passed __initdata;
@@ -75,40 +90,34 @@ static bool __stack_depot_early_init_passed __initdata;
 /* Initial seed for jhash2. */
 #define STACK_HASH_SEED 0x9747b28c
 
-/* Hash table of pointers to stored stack traces. */
-static struct stack_record **stack_table;
+/* Hash table of stored stack records. */
+static struct list_head *stack_table;
 /* Fixed order of the number of table buckets. Used when KASAN is enabled. */
 static unsigned int stack_bucket_number_order;
 /* Hash mask for indexing the table. */
 static unsigned int stack_hash_mask;
 
-/* Array of memory regions that store stack traces. */
+/* Array of memory regions that store stack records. */
 static void *stack_pools[DEPOT_MAX_POOLS];
-/* Currently used pool in stack_pools. */
-static int pool_index;
-/* Offset to the unused space in the currently used pool. */
-static size_t pool_offset;
-/* Lock that protects the variables above. */
-static DEFINE_RAW_SPINLOCK(pool_lock);
+/* Newly allocated pool that is not yet added to stack_pools. */
+static void *new_pool;
+/* Number of pools in stack_pools. */
+static int pools_num;
+/* Freelist of stack records within stack_pools. */
+static LIST_HEAD(free_stacks);
 /*
  * Stack depot tries to keep an extra pool allocated even before it runs out
- * of space in the currently used pool.
- * This flag marks that this next extra pool needs to be allocated and
- * initialized. It has the value 0 when either the next pool is not yet
- * initialized or the limit on the number of pools is reached.
+ * of space in the currently used pool. This flag marks whether this extra pool
+ * needs to be allocated. It has the value 0 when either an extra pool is not
+ * yet allocated or if the limit on the number of pools is reached.
  */
-static int next_pool_required = 1;
+static bool new_pool_required = true;
+/* Lock that protects the variables above. */
+static DEFINE_RWLOCK(pool_rwlock);
 
 static int __init disable_stack_depot(char *str)
 {
-	int ret;
-
-	ret = kstrtobool(str, &stack_depot_disabled);
-	if (!ret && stack_depot_disabled) {
-		pr_info("disabled\n");
-		stack_table = NULL;
-	}
-	return 0;
+	return kstrtobool(str, &stack_depot_disabled);
 }
 early_param("stack_depot_disable", disable_stack_depot);
 
@@ -120,6 +129,15 @@ void __init stack_depot_request_early_init(void)
 	__stack_depot_early_init_requested = true;
 }
 
+/* Initialize list_head's within the hash table. */
+static void init_stack_table(unsigned long entries)
+{
+	unsigned long i;
+
+	for (i = 0; i < entries; i++)
+		INIT_LIST_HEAD(&stack_table[i]);
+}
+
 /* Allocates a hash table via memblock. Can only be used during early boot. */
 int __init stack_depot_early_init(void)
 {
@@ -130,6 +148,15 @@ int __init stack_depot_early_init(void)
 		return 0;
 	__stack_depot_early_init_passed = true;
 
+	/*
+	 * Print disabled message even if early init has not been requested:
+	 * stack_depot_init() will not print one.
+	 */
+	if (stack_depot_disabled) {
+		pr_info("disabled\n");
+		return 0;
+	}
+
 	/*
 	 * If KASAN is enabled, use the maximum order: KASAN is frequently used
 	 * in fuzzing scenarios, which leads to a large number of different
@@ -138,21 +165,25 @@ int __init stack_depot_early_init(void)
 	if (kasan_enabled() && !stack_bucket_number_order)
 		stack_bucket_number_order = STACK_BUCKET_NUMBER_ORDER_MAX;
 
-	if (!__stack_depot_early_init_requested || stack_depot_disabled)
+	/*
+	 * Check if early init has been requested after setting
+	 * stack_bucket_number_order: stack_depot_init() uses its value.
+	 */
+	if (!__stack_depot_early_init_requested)
 		return 0;
 
 	/*
 	 * If stack_bucket_number_order is not set, leave entries as 0 to rely
-	 * on the automatic calculations performed by alloc_large_system_hash.
+	 * on the automatic calculations performed by alloc_large_system_hash().
 	 */
 	if (stack_bucket_number_order)
 		entries = 1UL << stack_bucket_number_order;
 	pr_info("allocating hash table via alloc_large_system_hash\n");
 	stack_table = alloc_large_system_hash("stackdepot",
-						sizeof(struct stack_record *),
+						sizeof(struct list_head),
 						entries,
 						STACK_HASH_TABLE_SCALE,
-						HASH_EARLY | HASH_ZERO,
+						HASH_EARLY,
 						NULL,
 						&stack_hash_mask,
 						1UL << STACK_BUCKET_NUMBER_ORDER_MIN,
@@ -162,6 +193,14 @@ int __init stack_depot_early_init(void)
 		stack_depot_disabled = true;
 		return -ENOMEM;
 	}
+	if (!entries) {
+		/*
+		 * Obtain the number of entries that was calculated by
+		 * alloc_large_system_hash().
+		 */
+		entries = stack_hash_mask + 1;
+	}
+	init_stack_table(entries);
 
 	return 0;
 }
@@ -202,7 +241,7 @@ int stack_depot_init(void)
 		entries = 1UL << STACK_BUCKET_NUMBER_ORDER_MAX;
 
 	pr_info("allocating hash table of %lu entries via kvcalloc\n", entries);
-	stack_table = kvcalloc(entries, sizeof(struct stack_record *), GFP_KERNEL);
+	stack_table = kvcalloc(entries, sizeof(struct list_head), GFP_KERNEL);
 	if (!stack_table) {
 		pr_err("hash table allocation failed, disabling\n");
 		stack_depot_disabled = true;
@@ -210,6 +249,7 @@ int stack_depot_init(void)
 		goto out_unlock;
 	}
 	stack_hash_mask = entries - 1;
+	init_stack_table(entries);
 
 out_unlock:
 	mutex_unlock(&stack_depot_init_mutex);
@@ -218,41 +258,103 @@ int stack_depot_init(void)
 }
 EXPORT_SYMBOL_GPL(stack_depot_init);
 
-/* Uses preallocated memory to initialize a new stack depot pool. */
-static void depot_init_pool(void **prealloc)
+/* Initializes a stack depol pool. */
+static void depot_init_pool(void *pool)
 {
+	int offset;
+
+	lockdep_assert_held_write(&pool_rwlock);
+
+	WARN_ON(!list_empty(&free_stacks));
+
+	/* Initialize handles and link stack records into the freelist. */
+	for (offset = 0; offset <= DEPOT_POOL_SIZE - DEPOT_STACK_RECORD_SIZE;
+	     offset += DEPOT_STACK_RECORD_SIZE) {
+		struct stack_record *stack = pool + offset;
+
+		stack->handle.pool_index = pools_num;
+		stack->handle.offset = offset >> DEPOT_STACK_ALIGN;
+		stack->handle.extra = 0;
+
+		list_add(&stack->list, &free_stacks);
+	}
+
+	/* Save reference to the pool to be used by depot_fetch_stack(). */
+	stack_pools[pools_num] = pool;
+	pools_num++;
+}
+
+/* Keeps the preallocated memory to be used for a new stack depot pool. */
+static void depot_keep_new_pool(void **prealloc)
+{
+	lockdep_assert_held_write(&pool_rwlock);
+
 	/*
-	 * If the next pool is already initialized or the maximum number of
+	 * If a new pool is already saved or the maximum number of
 	 * pools is reached, do not use the preallocated memory.
-	 * smp_load_acquire() here pairs with smp_store_release() below and
-	 * in depot_alloc_stack().
 	 */
-	if (!smp_load_acquire(&next_pool_required))
+	if (!new_pool_required)
 		return;
 
-	/* Check if the current pool is not yet allocated. */
-	if (stack_pools[pool_index] == NULL) {
-		/* Use the preallocated memory for the current pool. */
-		stack_pools[pool_index] = *prealloc;
+	/*
+	 * Use the preallocated memory for the new pool
+	 * as long as we do not exceed the maximum number of pools.
+	 */
+	if (pools_num < DEPOT_MAX_POOLS) {
+		new_pool = *prealloc;
 		*prealloc = NULL;
-	} else {
-		/*
-		 * Otherwise, use the preallocated memory for the next pool
-		 * as long as we do not exceed the maximum number of pools.
-		 */
-		if (pool_index + 1 < DEPOT_MAX_POOLS) {
-			stack_pools[pool_index + 1] = *prealloc;
-			*prealloc = NULL;
-		}
-		/*
-		 * At this point, either the next pool is initialized or the
-		 * maximum number of pools is reached. In either case, take
-		 * note that initializing another pool is not required.
-		 * This smp_store_release pairs with smp_load_acquire() above
-		 * and in stack_depot_save().
-		 */
-		smp_store_release(&next_pool_required, 0);
 	}
+
+	/*
+	 * At this point, either a new pool is kept or the maximum
+	 * number of pools is reached. In either case, take note that
+	 * keeping another pool is not required.
+	 */
+	new_pool_required = false;
+}
+
+/* Updates references to the current and the next stack depot pools. */
+static bool depot_update_pools(void **prealloc)
+{
+	lockdep_assert_held_write(&pool_rwlock);
+
+	/* Check if we still have objects in the freelist. */
+	if (!list_empty(&free_stacks))
+		goto out_keep_prealloc;
+
+	/* Check if we have a new pool saved and use it. */
+	if (new_pool) {
+		depot_init_pool(new_pool);
+		new_pool = NULL;
+
+		/* Take note that we might need a new new_pool. */
+		if (pools_num < DEPOT_MAX_POOLS)
+			new_pool_required = true;
+
+		/* Try keeping the preallocated memory for new_pool. */
+		goto out_keep_prealloc;
+	}
+
+	/* Bail out if we reached the pool limit. */
+	if (unlikely(pools_num >= DEPOT_MAX_POOLS)) {
+		WARN_ONCE(1, "Stack depot reached limit capacity");
+		return false;
+	}
+
+	/* Check if we have preallocated memory and use it. */
+	if (*prealloc) {
+		depot_init_pool(*prealloc);
+		*prealloc = NULL;
+		return true;
+	}
+
+	return false;
+
+out_keep_prealloc:
+	/* Keep the preallocated memory for a new pool if required. */
+	if (*prealloc)
+		depot_keep_new_pool(prealloc);
+	return true;
 }
 
 /* Allocates a new stack in a stack depot pool. */
@@ -260,62 +362,72 @@ static struct stack_record *
 depot_alloc_stack(unsigned long *entries, int size, u32 hash, void **prealloc)
 {
 	struct stack_record *stack;
-	size_t required_size = struct_size(stack, entries, size);
 
-	required_size = ALIGN(required_size, 1 << DEPOT_STACK_ALIGN);
+	lockdep_assert_held_write(&pool_rwlock);
 
-	/* Check if there is not enough space in the current pool. */
-	if (unlikely(pool_offset + required_size > DEPOT_POOL_SIZE)) {
-		/* Bail out if we reached the pool limit. */
-		if (unlikely(pool_index + 1 >= DEPOT_MAX_POOLS)) {
-			WARN_ONCE(1, "Stack depot reached limit capacity");
-			return NULL;
-		}
+	/* Update current and new pools if required and possible. */
+	if (!depot_update_pools(prealloc))
+		return NULL;
 
-		/*
-		 * Move on to the next pool.
-		 * WRITE_ONCE pairs with potential concurrent read in
-		 * stack_depot_fetch().
-		 */
-		WRITE_ONCE(pool_index, pool_index + 1);
-		pool_offset = 0;
-		/*
-		 * If the maximum number of pools is not reached, take note
-		 * that the next pool needs to initialized.
-		 * smp_store_release() here pairs with smp_load_acquire() in
-		 * stack_depot_save() and depot_init_pool().
-		 */
-		if (pool_index + 1 < DEPOT_MAX_POOLS)
-			smp_store_release(&next_pool_required, 1);
-	}
+	/* Check if we have a stack record to save the stack trace. */
+	if (list_empty(&free_stacks))
+		return NULL;
 
-	/* Assign the preallocated memory to a pool if required. */
-	if (*prealloc)
-		depot_init_pool(prealloc);
+	/* Get and unlink the first entry from the freelist. */
+	stack = list_first_entry(&free_stacks, struct stack_record, list);
+	list_del(&stack->list);
 
-	/* Check if we have a pool to save the stack trace. */
-	if (stack_pools[pool_index] == NULL)
-		return NULL;
+	/* Limit number of saved frames to CONFIG_STACKDEPOT_MAX_FRAMES. */
+	if (size > CONFIG_STACKDEPOT_MAX_FRAMES)
+		size = CONFIG_STACKDEPOT_MAX_FRAMES;
 
 	/* Save the stack trace. */
-	stack = stack_pools[pool_index] + pool_offset;
 	stack->hash = hash;
 	stack->size = size;
-	stack->handle.pool_index = pool_index;
-	stack->handle.offset = pool_offset >> DEPOT_STACK_ALIGN;
-	stack->handle.valid = 1;
-	stack->handle.extra = 0;
+	/* stack->handle is already filled in by depot_init_pool(). */
+	refcount_set(&stack->count, 1);
 	memcpy(stack->entries, entries, flex_array_size(stack, entries, size));
-	pool_offset += required_size;
+
 	/*
 	 * Let KMSAN know the stored stack record is initialized. This shall
 	 * prevent false positive reports if instrumented code accesses it.
 	 */
-	kmsan_unpoison_memory(stack, required_size);
+	kmsan_unpoison_memory(stack, DEPOT_STACK_RECORD_SIZE);
 
 	return stack;
 }
 
+static struct stack_record *depot_fetch_stack(depot_stack_handle_t handle)
+{
+	union handle_parts parts = { .handle = handle };
+	void *pool;
+	size_t offset = parts.offset << DEPOT_STACK_ALIGN;
+	struct stack_record *stack;
+
+	lockdep_assert_held(&pool_rwlock);
+
+	if (parts.pool_index > pools_num) {
+		WARN(1, "pool index %d out of bounds (%d) for stack id %08x\n",
+		     parts.pool_index, pools_num, handle);
+		return NULL;
+	}
+
+	pool = stack_pools[parts.pool_index];
+	if (!pool)
+		return NULL;
+
+	stack = pool + offset;
+	return stack;
+}
+
+/* Links stack into the freelist. */
+static void depot_free_stack(struct stack_record *stack)
+{
+	lockdep_assert_held_write(&pool_rwlock);
+
+	list_add(&stack->list, &free_stacks);
+}
+
 /* Calculates the hash for a stack. */
 static inline u32 hash_stack(unsigned long *entries, unsigned int size)
 {
@@ -340,13 +452,17 @@ int stackdepot_memcmp(const unsigned long *u1, const unsigned long *u2,
 }
 
 /* Finds a stack in a bucket of the hash table. */
-static inline struct stack_record *find_stack(struct stack_record *bucket,
+static inline struct stack_record *find_stack(struct list_head *bucket,
 					     unsigned long *entries, int size,
 					     u32 hash)
 {
+	struct list_head *pos;
 	struct stack_record *found;
 
-	for (found = bucket; found; found = found->next) {
+	lockdep_assert_held(&pool_rwlock);
+
+	list_for_each(pos, bucket) {
+		found = list_entry(pos, struct stack_record, list);
 		if (found->hash == hash &&
 		    found->size == size &&
 		    !stackdepot_memcmp(entries, found->entries, size))
@@ -355,17 +471,24 @@ static inline struct stack_record *find_stack(struct stack_record *bucket,
 	return NULL;
 }
 
-depot_stack_handle_t __stack_depot_save(unsigned long *entries,
-					unsigned int nr_entries,
-					gfp_t alloc_flags, bool can_alloc)
+depot_stack_handle_t stack_depot_save_flags(unsigned long *entries,
+					    unsigned int nr_entries,
+					    gfp_t alloc_flags,
+					    depot_flags_t depot_flags)
 {
-	struct stack_record *found = NULL, **bucket;
-	union handle_parts retval = { .handle = 0 };
+	struct list_head *bucket;
+	struct stack_record *found = NULL;
+	depot_stack_handle_t handle = 0;
 	struct page *page = NULL;
 	void *prealloc = NULL;
+	bool can_alloc = depot_flags & STACK_DEPOT_FLAG_CAN_ALLOC;
+	bool need_alloc = false;
 	unsigned long flags;
 	u32 hash;
 
+	if (WARN_ON(depot_flags & ~STACK_DEPOT_FLAGS_MASK))
+		return 0;
+
 	/*
 	 * If this stack trace is from an interrupt, including anything before
 	 * interrupt entry usually leads to unbounded stack depot growth.
@@ -377,28 +500,33 @@ depot_stack_handle_t __stack_depot_save(unsigned long *entries,
 	nr_entries = filter_irq_stacks(entries, nr_entries);
 
 	if (unlikely(nr_entries == 0) || stack_depot_disabled)
-		goto fast_exit;
+		return 0;
 
 	hash = hash_stack(entries, nr_entries);
 	bucket = &stack_table[hash & stack_hash_mask];
 
-	/*
-	 * Fast path: look the stack trace up without locking.
-	 * The smp_load_acquire() here pairs with smp_store_release() to
-	 * |bucket| below.
-	 */
-	found = find_stack(smp_load_acquire(bucket), entries, nr_entries, hash);
-	if (found)
+	read_lock_irqsave(&pool_rwlock, flags);
+
+	/* Fast path: look the stack trace up without full locking. */
+	found = find_stack(bucket, entries, nr_entries, hash);
+	if (found) {
+		if (depot_flags & STACK_DEPOT_FLAG_GET)
+			refcount_inc(&found->count);
+		read_unlock_irqrestore(&pool_rwlock, flags);
 		goto exit;
+	}
+
+	/* Take note if another stack pool needs to be allocated. */
+	if (new_pool_required)
+		need_alloc = true;
+
+	read_unlock_irqrestore(&pool_rwlock, flags);
 
 	/*
-	 * Check if another stack pool needs to be initialized. If so, allocate
-	 * the memory now - we won't be able to do that under the lock.
-	 *
-	 * The smp_load_acquire() here pairs with smp_store_release() to
-	 * |next_pool_inited| in depot_alloc_stack() and depot_init_pool().
+	 * Allocate memory for a new pool if required now:
+	 * we won't be able to do that under the lock.
 	 */
-	if (unlikely(can_alloc && smp_load_acquire(&next_pool_required))) {
+	if (unlikely(can_alloc && need_alloc)) {
 		/*
 		 * Zero out zone modifiers, as we don't have specific zone
 		 * requirements. Keep the flags related to allocation in atomic
@@ -412,63 +540,54 @@ depot_stack_handle_t __stack_depot_save(unsigned long *entries,
 			prealloc = page_address(page);
 	}
 
-	raw_spin_lock_irqsave(&pool_lock, flags);
+	write_lock_irqsave(&pool_rwlock, flags);
 
-	found = find_stack(*bucket, entries, nr_entries, hash);
+	found = find_stack(bucket, entries, nr_entries, hash);
 	if (!found) {
 		struct stack_record *new =
 			depot_alloc_stack(entries, nr_entries, hash, &prealloc);
 
 		if (new) {
-			new->next = *bucket;
-			/*
-			 * This smp_store_release() pairs with
-			 * smp_load_acquire() from |bucket| above.
-			 */
-			smp_store_release(bucket, new);
+			list_add(&new->list, bucket);
 			found = new;
 		}
-	} else if (prealloc) {
+	} else {
+		if (depot_flags & STACK_DEPOT_FLAG_GET)
+			refcount_inc(&found->count);
 		/*
 		 * Stack depot already contains this stack trace, but let's
-		 * keep the preallocated memory for the future.
+		 * keep the preallocated memory for future.
 		 */
-		depot_init_pool(&prealloc);
+		if (prealloc)
+			depot_keep_new_pool(&prealloc);
 	}
 
-	raw_spin_unlock_irqrestore(&pool_lock, flags);
+	write_unlock_irqrestore(&pool_rwlock, flags);
 exit:
 	if (prealloc) {
 		/* Stack depot didn't use this memory, free it. */
 		free_pages((unsigned long)prealloc, DEPOT_POOL_ORDER);
 	}
 	if (found)
-		retval.handle = found->handle.handle;
-fast_exit:
-	return retval.handle;
+		handle = found->handle.handle;
+	return handle;
 }
-EXPORT_SYMBOL_GPL(__stack_depot_save);
+EXPORT_SYMBOL_GPL(stack_depot_save_flags);
 
 depot_stack_handle_t stack_depot_save(unsigned long *entries,
 				      unsigned int nr_entries,
 				      gfp_t alloc_flags)
 {
-	return __stack_depot_save(entries, nr_entries, alloc_flags, true);
+	return stack_depot_save_flags(entries, nr_entries, alloc_flags,
+				      STACK_DEPOT_FLAG_CAN_ALLOC);
 }
 EXPORT_SYMBOL_GPL(stack_depot_save);
 
 unsigned int stack_depot_fetch(depot_stack_handle_t handle,
 			       unsigned long **entries)
 {
-	union handle_parts parts = { .handle = handle };
-	/*
-	 * READ_ONCE pairs with potential concurrent write in
-	 * depot_alloc_stack.
-	 */
-	int pool_index_cached = READ_ONCE(pool_index);
-	void *pool;
-	size_t offset = parts.offset << DEPOT_STACK_ALIGN;
 	struct stack_record *stack;
+	unsigned long flags;
 
 	*entries = NULL;
 	/*
@@ -477,24 +596,47 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle,
 	 */
 	kmsan_unpoison_memory(entries, sizeof(*entries));
 
-	if (!handle)
+	if (!handle || stack_depot_disabled)
 		return 0;
 
-	if (parts.pool_index > pool_index_cached) {
-		WARN(1, "pool index %d out of bounds (%d) for stack id %08x\n",
-			parts.pool_index, pool_index_cached, handle);
-		return 0;
-	}
-	pool = stack_pools[parts.pool_index];
-	if (!pool)
-		return 0;
-	stack = pool + offset;
+	read_lock_irqsave(&pool_rwlock, flags);
+
+	stack = depot_fetch_stack(handle);
+
+	read_unlock_irqrestore(&pool_rwlock, flags);
 
 	*entries = stack->entries;
 	return stack->size;
 }
 EXPORT_SYMBOL_GPL(stack_depot_fetch);
 
+void stack_depot_put(depot_stack_handle_t handle)
+{
+	struct stack_record *stack;
+	unsigned long flags;
+
+	if (!handle || stack_depot_disabled)
+		return;
+
+	write_lock_irqsave(&pool_rwlock, flags);
+
+	stack = depot_fetch_stack(handle);
+	if (WARN_ON(!stack))
+		goto out;
+
+	if (refcount_dec_and_test(&stack->count)) {
+		/* Unlink stack from the hash table. */
+		list_del(&stack->list);
+
+		/* Free stack. */
+		depot_free_stack(stack);
+	}
+
+out:
+	write_unlock_irqrestore(&pool_rwlock, flags);
+}
+EXPORT_SYMBOL_GPL(stack_depot_put);
+
 void stack_depot_print(depot_stack_handle_t stack)
 {
 	unsigned long *entries;
diff --git a/lib/test_bpf.c b/lib/test_bpf.c
index 7916503e6a6a..c148f8d1e564 100644
--- a/lib/test_bpf.c
+++ b/lib/test_bpf.c
@@ -5144,22 +5144,6 @@ static struct bpf_test tests[] = {
 		{ },
 		{ { 0, 0x1 } },
 	},
-	{
-		"ALU_MOVSX | BPF_W",
-		.u.insns_int = {
-			BPF_LD_IMM64(R2, 0x00000000deadbeefLL),
-			BPF_LD_IMM64(R3, 0xdeadbeefdeadbeefLL),
-			BPF_MOVSX32_REG(R1, R3, 32),
-			BPF_JMP_REG(BPF_JEQ, R2, R1, 2),
-			BPF_MOV32_IMM(R0, 2),
-			BPF_EXIT_INSN(),
-			BPF_MOV32_IMM(R0, 1),
-			BPF_EXIT_INSN(),
-		},
-		INTERNAL,
-		{ },
-		{ { 0, 0x1 } },
-	},
 	/* MOVSX64 REG */
 	{
 		"ALU64_MOVSX | BPF_B",
diff --git a/lib/test_maple_tree.c b/lib/test_maple_tree.c
index 464eeb90d5ad..15fbeb788f3a 100644
--- a/lib/test_maple_tree.c
+++ b/lib/test_maple_tree.c
@@ -43,6 +43,7 @@ atomic_t maple_tree_tests_passed;
 /* #define BENCH_NODE_STORE */
 /* #define BENCH_AWALK */
 /* #define BENCH_WALK */
+/* #define BENCH_LOAD */
 /* #define BENCH_MT_FOR_EACH */
 /* #define BENCH_FORK */
 /* #define BENCH_MAS_FOR_EACH */
@@ -54,6 +55,11 @@ atomic_t maple_tree_tests_passed;
 #else
 #define cond_resched()			do {} while (0)
 #endif
+
+#define mas_is_none(x)		((x)->status == ma_none)
+#define mas_is_overflow(x)	((x)->status == ma_overflow)
+#define mas_is_underflow(x)	((x)->status == ma_underflow)
+
 static int __init mtree_insert_index(struct maple_tree *mt,
 				     unsigned long index, gfp_t gfp)
 {
@@ -582,7 +588,7 @@ static noinline void __init check_find(struct maple_tree *mt)
 	MT_BUG_ON(mt, last != mas.last);
 
 
-	mas.node = MAS_NONE;
+	mas.status = ma_none;
 	mas.index = ULONG_MAX;
 	mas.last = ULONG_MAX;
 	entry2 = mas_prev(&mas, 0);
@@ -1749,6 +1755,19 @@ static noinline void __init bench_walk(struct maple_tree *mt)
 }
 #endif
 
+#if defined(BENCH_LOAD)
+static noinline void __init bench_load(struct maple_tree *mt)
+{
+	int i, max = 2500, count = 550000000;
+
+	for (i = 0; i < max; i += 10)
+		mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL);
+
+	for (i = 0; i < count; i++)
+		mtree_load(mt, 1470);
+}
+#endif
+
 #if defined(BENCH_MT_FOR_EACH)
 static noinline void __init bench_mt_for_each(struct maple_tree *mt)
 {
@@ -1834,47 +1853,48 @@ static noinline void __init bench_mas_prev(struct maple_tree *mt)
 }
 #endif
 /* check_forking - simulate the kernel forking sequence with the tree. */
-static noinline void __init check_forking(struct maple_tree *mt)
+static noinline void __init check_forking(void)
 {
-
-	struct maple_tree newmt;
-	int i, nr_entries = 134;
+	struct maple_tree mt, newmt;
+	int i, nr_entries = 134, ret;
 	void *val;
-	MA_STATE(mas, mt, 0, 0);
-	MA_STATE(newmas, mt, 0, 0);
-	struct rw_semaphore newmt_lock;
+	MA_STATE(mas, &mt, 0, 0);
+	MA_STATE(newmas, &newmt, 0, 0);
+	struct rw_semaphore mt_lock, newmt_lock;
 
+	init_rwsem(&mt_lock);
 	init_rwsem(&newmt_lock);
 
-	for (i = 0; i <= nr_entries; i++)
-		mtree_store_range(mt, i*10, i*10 + 5,
-				  xa_mk_value(i), GFP_KERNEL);
+	mt_init_flags(&mt, MT_FLAGS_ALLOC_RANGE | MT_FLAGS_LOCK_EXTERN);
+	mt_set_external_lock(&mt, &mt_lock);
 
-	mt_set_non_kernel(99999);
 	mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE | MT_FLAGS_LOCK_EXTERN);
 	mt_set_external_lock(&newmt, &newmt_lock);
-	newmas.tree = &newmt;
-	mas_reset(&newmas);
-	mas_reset(&mas);
-	down_write(&newmt_lock);
-	mas.index = 0;
-	mas.last = 0;
-	if (mas_expected_entries(&newmas, nr_entries)) {
+
+	down_write(&mt_lock);
+	for (i = 0; i <= nr_entries; i++) {
+		mas_set_range(&mas, i*10, i*10 + 5);
+		mas_store_gfp(&mas, xa_mk_value(i), GFP_KERNEL);
+	}
+
+	down_write_nested(&newmt_lock, SINGLE_DEPTH_NESTING);
+	ret = __mt_dup(&mt, &newmt, GFP_KERNEL);
+	if (ret) {
 		pr_err("OOM!");
 		BUG_ON(1);
 	}
-	rcu_read_lock();
-	mas_for_each(&mas, val, ULONG_MAX) {
-		newmas.index = mas.index;
-		newmas.last = mas.last;
+
+	mas_set(&newmas, 0);
+	mas_for_each(&newmas, val, ULONG_MAX)
 		mas_store(&newmas, val);
-	}
-	rcu_read_unlock();
+
 	mas_destroy(&newmas);
+	mas_destroy(&mas);
 	mt_validate(&newmt);
-	mt_set_non_kernel(0);
 	__mt_destroy(&newmt);
+	__mt_destroy(&mt);
 	up_write(&newmt_lock);
+	up_write(&mt_lock);
 }
 
 static noinline void __init check_iteration(struct maple_tree *mt)
@@ -1977,49 +1997,51 @@ static noinline void __init check_mas_store_gfp(struct maple_tree *mt)
 }
 
 #if defined(BENCH_FORK)
-static noinline void __init bench_forking(struct maple_tree *mt)
+static noinline void __init bench_forking(void)
 {
-
-	struct maple_tree newmt;
-	int i, nr_entries = 134, nr_fork = 80000;
+	struct maple_tree mt, newmt;
+	int i, nr_entries = 134, nr_fork = 80000, ret;
 	void *val;
-	MA_STATE(mas, mt, 0, 0);
-	MA_STATE(newmas, mt, 0, 0);
-	struct rw_semaphore newmt_lock;
+	MA_STATE(mas, &mt, 0, 0);
+	MA_STATE(newmas, &newmt, 0, 0);
+	struct rw_semaphore mt_lock, newmt_lock;
 
+	init_rwsem(&mt_lock);
 	init_rwsem(&newmt_lock);
-	mt_set_external_lock(&newmt, &newmt_lock);
 
-	for (i = 0; i <= nr_entries; i++)
-		mtree_store_range(mt, i*10, i*10 + 5,
-				  xa_mk_value(i), GFP_KERNEL);
+	mt_init_flags(&mt, MT_FLAGS_ALLOC_RANGE | MT_FLAGS_LOCK_EXTERN);
+	mt_set_external_lock(&mt, &mt_lock);
+
+	down_write(&mt_lock);
+	for (i = 0; i <= nr_entries; i++) {
+		mas_set_range(&mas, i*10, i*10 + 5);
+		mas_store_gfp(&mas, xa_mk_value(i), GFP_KERNEL);
+	}
 
 	for (i = 0; i < nr_fork; i++) {
-		mt_set_non_kernel(99999);
-		mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE);
-		newmas.tree = &newmt;
-		mas_reset(&newmas);
-		mas_reset(&mas);
-		mas.index = 0;
-		mas.last = 0;
-		rcu_read_lock();
-		down_write(&newmt_lock);
-		if (mas_expected_entries(&newmas, nr_entries)) {
-			printk("OOM!");
+		mt_init_flags(&newmt,
+			      MT_FLAGS_ALLOC_RANGE | MT_FLAGS_LOCK_EXTERN);
+		mt_set_external_lock(&newmt, &newmt_lock);
+
+		down_write_nested(&newmt_lock, SINGLE_DEPTH_NESTING);
+		ret = __mt_dup(&mt, &newmt, GFP_KERNEL);
+		if (ret) {
+			pr_err("OOM!");
 			BUG_ON(1);
 		}
-		mas_for_each(&mas, val, ULONG_MAX) {
-			newmas.index = mas.index;
-			newmas.last = mas.last;
+
+		mas_set(&newmas, 0);
+		mas_for_each(&newmas, val, ULONG_MAX)
 			mas_store(&newmas, val);
-		}
+
 		mas_destroy(&newmas);
-		rcu_read_unlock();
 		mt_validate(&newmt);
-		mt_set_non_kernel(0);
 		__mt_destroy(&newmt);
 		up_write(&newmt_lock);
 	}
+	mas_destroy(&mas);
+	__mt_destroy(&mt);
+	up_write(&mt_lock);
 }
 #endif
 
@@ -2175,7 +2197,7 @@ static noinline void __init next_prev_test(struct maple_tree *mt)
 	MT_BUG_ON(mt, val != NULL);
 	MT_BUG_ON(mt, mas.index != 0);
 	MT_BUG_ON(mt, mas.last != 5);
-	MT_BUG_ON(mt, mas.node != MAS_UNDERFLOW);
+	MT_BUG_ON(mt, !mas_is_underflow(&mas));
 
 	mas.index = 0;
 	mas.last = 5;
@@ -3039,10 +3061,6 @@ static noinline void __init check_empty_area_fill(struct maple_tree *mt)
  *		DNE	active		active		range of NULL
  */
 
-#define mas_active(x)		(((x).node != MAS_ROOT) && \
-				 ((x).node != MAS_START) && \
-				 ((x).node != MAS_PAUSE) && \
-				 ((x).node != MAS_NONE))
 static noinline void __init check_state_handling(struct maple_tree *mt)
 {
 	MA_STATE(mas, mt, 0, 0);
@@ -3057,7 +3075,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	/* prev: Start -> underflow*/
 	entry = mas_prev(&mas, 0);
 	MT_BUG_ON(mt, entry != NULL);
-	MT_BUG_ON(mt, mas.node != MAS_UNDERFLOW);
+	MT_BUG_ON(mt, mas.status != ma_underflow);
 
 	/* prev: Start -> root */
 	mas_set(&mas, 10);
@@ -3065,7 +3083,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0);
 	MT_BUG_ON(mt, mas.last != 0);
-	MT_BUG_ON(mt, mas.node != MAS_ROOT);
+	MT_BUG_ON(mt, mas.status != ma_root);
 
 	/* prev: pause -> root */
 	mas_set(&mas, 10);
@@ -3074,7 +3092,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0);
 	MT_BUG_ON(mt, mas.last != 0);
-	MT_BUG_ON(mt, mas.node != MAS_ROOT);
+	MT_BUG_ON(mt, mas.status != ma_root);
 
 	/* next: start -> none */
 	mas_set(&mas, 0);
@@ -3082,7 +3100,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, mas.index != 1);
 	MT_BUG_ON(mt, mas.last != ULONG_MAX);
 	MT_BUG_ON(mt, entry != NULL);
-	MT_BUG_ON(mt, mas.node != MAS_NONE);
+	MT_BUG_ON(mt, mas.status != ma_none);
 
 	/* next: start -> none*/
 	mas_set(&mas, 10);
@@ -3090,7 +3108,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, mas.index != 1);
 	MT_BUG_ON(mt, mas.last != ULONG_MAX);
 	MT_BUG_ON(mt, entry != NULL);
-	MT_BUG_ON(mt, mas.node != MAS_NONE);
+	MT_BUG_ON(mt, mas.status != ma_none);
 
 	/* find: start -> root */
 	mas_set(&mas, 0);
@@ -3098,21 +3116,21 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0);
 	MT_BUG_ON(mt, mas.last != 0);
-	MT_BUG_ON(mt, mas.node != MAS_ROOT);
+	MT_BUG_ON(mt, mas.status != ma_root);
 
 	/* find: root -> none */
 	entry = mas_find(&mas, ULONG_MAX);
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 1);
 	MT_BUG_ON(mt, mas.last != ULONG_MAX);
-	MT_BUG_ON(mt, mas.node != MAS_NONE);
+	MT_BUG_ON(mt, mas.status != ma_none);
 
 	/* find: none -> none */
 	entry = mas_find(&mas, ULONG_MAX);
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 1);
 	MT_BUG_ON(mt, mas.last != ULONG_MAX);
-	MT_BUG_ON(mt, mas.node != MAS_NONE);
+	MT_BUG_ON(mt, mas.status != ma_none);
 
 	/* find: start -> none */
 	mas_set(&mas, 10);
@@ -3120,14 +3138,14 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 1);
 	MT_BUG_ON(mt, mas.last != ULONG_MAX);
-	MT_BUG_ON(mt, mas.node != MAS_NONE);
+	MT_BUG_ON(mt, mas.status != ma_none);
 
 	/* find_rev: none -> root */
 	entry = mas_find_rev(&mas, 0);
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0);
 	MT_BUG_ON(mt, mas.last != 0);
-	MT_BUG_ON(mt, mas.node != MAS_ROOT);
+	MT_BUG_ON(mt, mas.status != ma_root);
 
 	/* find_rev: start -> root */
 	mas_set(&mas, 0);
@@ -3135,21 +3153,21 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0);
 	MT_BUG_ON(mt, mas.last != 0);
-	MT_BUG_ON(mt, mas.node != MAS_ROOT);
+	MT_BUG_ON(mt, mas.status != ma_root);
 
 	/* find_rev: root -> none */
 	entry = mas_find_rev(&mas, 0);
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 0);
 	MT_BUG_ON(mt, mas.last != 0);
-	MT_BUG_ON(mt, mas.node != MAS_NONE);
+	MT_BUG_ON(mt, mas.status != ma_none);
 
 	/* find_rev: none -> none */
 	entry = mas_find_rev(&mas, 0);
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 0);
 	MT_BUG_ON(mt, mas.last != 0);
-	MT_BUG_ON(mt, mas.node != MAS_NONE);
+	MT_BUG_ON(mt, mas.status != ma_none);
 
 	/* find_rev: start -> root */
 	mas_set(&mas, 10);
@@ -3157,7 +3175,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0);
 	MT_BUG_ON(mt, mas.last != 0);
-	MT_BUG_ON(mt, mas.node != MAS_ROOT);
+	MT_BUG_ON(mt, mas.status != ma_root);
 
 	/* walk: start -> none */
 	mas_set(&mas, 10);
@@ -3165,7 +3183,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 1);
 	MT_BUG_ON(mt, mas.last != ULONG_MAX);
-	MT_BUG_ON(mt, mas.node != MAS_NONE);
+	MT_BUG_ON(mt, mas.status != ma_none);
 
 	/* walk: pause -> none*/
 	mas_set(&mas, 10);
@@ -3174,7 +3192,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 1);
 	MT_BUG_ON(mt, mas.last != ULONG_MAX);
-	MT_BUG_ON(mt, mas.node != MAS_NONE);
+	MT_BUG_ON(mt, mas.status != ma_none);
 
 	/* walk: none -> none */
 	mas.index = mas.last = 10;
@@ -3182,14 +3200,14 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 1);
 	MT_BUG_ON(mt, mas.last != ULONG_MAX);
-	MT_BUG_ON(mt, mas.node != MAS_NONE);
+	MT_BUG_ON(mt, mas.status != ma_none);
 
 	/* walk: none -> none */
 	entry = mas_walk(&mas);
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 1);
 	MT_BUG_ON(mt, mas.last != ULONG_MAX);
-	MT_BUG_ON(mt, mas.node != MAS_NONE);
+	MT_BUG_ON(mt, mas.status != ma_none);
 
 	/* walk: start -> root */
 	mas_set(&mas, 0);
@@ -3197,7 +3215,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0);
 	MT_BUG_ON(mt, mas.last != 0);
-	MT_BUG_ON(mt, mas.node != MAS_ROOT);
+	MT_BUG_ON(mt, mas.status != ma_root);
 
 	/* walk: pause -> root */
 	mas_set(&mas, 0);
@@ -3206,22 +3224,22 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0);
 	MT_BUG_ON(mt, mas.last != 0);
-	MT_BUG_ON(mt, mas.node != MAS_ROOT);
+	MT_BUG_ON(mt, mas.status != ma_root);
 
 	/* walk: none -> root */
-	mas.node = MAS_NONE;
+	mas.status = ma_none;
 	entry = mas_walk(&mas);
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0);
 	MT_BUG_ON(mt, mas.last != 0);
-	MT_BUG_ON(mt, mas.node != MAS_ROOT);
+	MT_BUG_ON(mt, mas.status != ma_root);
 
 	/* walk: root -> root */
 	entry = mas_walk(&mas);
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0);
 	MT_BUG_ON(mt, mas.last != 0);
-	MT_BUG_ON(mt, mas.node != MAS_ROOT);
+	MT_BUG_ON(mt, mas.status != ma_root);
 
 	/* walk: root -> none */
 	mas_set(&mas, 10);
@@ -3229,7 +3247,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 1);
 	MT_BUG_ON(mt, mas.last != ULONG_MAX);
-	MT_BUG_ON(mt, mas.node != MAS_NONE);
+	MT_BUG_ON(mt, mas.status != ma_none);
 
 	/* walk: none -> root */
 	mas.index = mas.last = 0;
@@ -3237,7 +3255,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0);
 	MT_BUG_ON(mt, mas.last != 0);
-	MT_BUG_ON(mt, mas.node != MAS_ROOT);
+	MT_BUG_ON(mt, mas.status != ma_root);
 
 	mas_unlock(&mas);
 
@@ -3255,7 +3273,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0x1000);
 	MT_BUG_ON(mt, mas.last != 0x1500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* next: pause ->active */
 	mas_set(&mas, 0);
@@ -3264,126 +3282,127 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0x1000);
 	MT_BUG_ON(mt, mas.last != 0x1500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* next: none ->active */
 	mas.index = mas.last = 0;
 	mas.offset = 0;
-	mas.node = MAS_NONE;
+	mas.status = ma_none;
 	entry = mas_next(&mas, ULONG_MAX);
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0x1000);
 	MT_BUG_ON(mt, mas.last != 0x1500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* next:active ->active */
 	entry = mas_next(&mas, ULONG_MAX);
 	MT_BUG_ON(mt, entry != ptr2);
 	MT_BUG_ON(mt, mas.index != 0x2000);
 	MT_BUG_ON(mt, mas.last != 0x2500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
-	/* next:active -> active beyond data */
+	/* next:active -> overflow (limit reached) beyond data */
 	entry = mas_next(&mas, 0x2999);
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 0x2501);
 	MT_BUG_ON(mt, mas.last != 0x2fff);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_overflow(&mas));
 
-	/* Continue after last range ends after max */
+	/* next:overflow -> active (limit changed) */
 	entry = mas_next(&mas, ULONG_MAX);
 	MT_BUG_ON(mt, entry != ptr3);
 	MT_BUG_ON(mt, mas.index != 0x3000);
 	MT_BUG_ON(mt, mas.last != 0x3500);
-	MT_BUG_ON(mt, !mas_active(mas));
-
-	/* next:active -> active continued */
-	entry = mas_next(&mas, ULONG_MAX);
-	MT_BUG_ON(mt, entry != NULL);
-	MT_BUG_ON(mt, mas.index != 0x3501);
-	MT_BUG_ON(mt, mas.last != ULONG_MAX);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
-	/* next:active -> overflow  */
+	/* next:active ->  overflow (limit reached) */
 	entry = mas_next(&mas, ULONG_MAX);
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 0x3501);
 	MT_BUG_ON(mt, mas.last != ULONG_MAX);
-	MT_BUG_ON(mt, mas.node != MAS_OVERFLOW);
+	MT_BUG_ON(mt, !mas_is_overflow(&mas));
 
 	/* next:overflow -> overflow  */
 	entry = mas_next(&mas, ULONG_MAX);
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 0x3501);
 	MT_BUG_ON(mt, mas.last != ULONG_MAX);
-	MT_BUG_ON(mt, mas.node != MAS_OVERFLOW);
+	MT_BUG_ON(mt, !mas_is_overflow(&mas));
 
 	/* prev:overflow -> active  */
 	entry = mas_prev(&mas, 0);
 	MT_BUG_ON(mt, entry != ptr3);
 	MT_BUG_ON(mt, mas.index != 0x3000);
 	MT_BUG_ON(mt, mas.last != 0x3500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* next: none -> active, skip value at location */
 	mas_set(&mas, 0);
 	entry = mas_next(&mas, ULONG_MAX);
-	mas.node = MAS_NONE;
+	mas.status = ma_none;
 	mas.offset = 0;
 	entry = mas_next(&mas, ULONG_MAX);
 	MT_BUG_ON(mt, entry != ptr2);
 	MT_BUG_ON(mt, mas.index != 0x2000);
 	MT_BUG_ON(mt, mas.last != 0x2500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* prev:active ->active */
 	entry = mas_prev(&mas, 0);
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0x1000);
 	MT_BUG_ON(mt, mas.last != 0x1500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
-	/* prev:active -> active spanning end range */
+	/* prev:active -> underflow (span limit) */
+	mas_next(&mas, ULONG_MAX);
+	entry = mas_prev(&mas, 0x1200);
+	MT_BUG_ON(mt, entry != ptr);
+	MT_BUG_ON(mt, mas.index != 0x1000);
+	MT_BUG_ON(mt, mas.last != 0x1500);
+	MT_BUG_ON(mt, !mas_is_underflow(&mas));
+
+	/* prev:underflow -> underflow (lower limit) spanning end range */
 	entry = mas_prev(&mas, 0x0100);
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 0);
 	MT_BUG_ON(mt, mas.last != 0x0FFF);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_underflow(&mas));
 
-	/* prev:active -> underflow */
+	/* prev:underflow -> underflow */
 	entry = mas_prev(&mas, 0);
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 0);
 	MT_BUG_ON(mt, mas.last != 0x0FFF);
-	MT_BUG_ON(mt, mas.node != MAS_UNDERFLOW);
+	MT_BUG_ON(mt, !mas_is_underflow(&mas));
 
 	/* prev:underflow -> underflow */
 	entry = mas_prev(&mas, 0);
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 0);
 	MT_BUG_ON(mt, mas.last != 0x0FFF);
-	MT_BUG_ON(mt, mas.node != MAS_UNDERFLOW);
+	MT_BUG_ON(mt, !mas_is_underflow(&mas));
 
 	/* next:underflow -> active */
 	entry = mas_next(&mas, ULONG_MAX);
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0x1000);
 	MT_BUG_ON(mt, mas.last != 0x1500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* prev:first value -> underflow */
 	entry = mas_prev(&mas, 0x1000);
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 0x1000);
 	MT_BUG_ON(mt, mas.last != 0x1500);
-	MT_BUG_ON(mt, mas.node != MAS_UNDERFLOW);
+	MT_BUG_ON(mt, !mas_is_underflow(&mas));
 
 	/* find:underflow -> first value */
 	entry = mas_find(&mas, ULONG_MAX);
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0x1000);
 	MT_BUG_ON(mt, mas.last != 0x1500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* prev: pause ->active */
 	mas_set(&mas, 0x3600);
@@ -3394,21 +3413,21 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != ptr2);
 	MT_BUG_ON(mt, mas.index != 0x2000);
 	MT_BUG_ON(mt, mas.last != 0x2500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
-	/* prev:active -> active spanning min */
+	/* prev:active -> underflow spanning min */
 	entry = mas_prev(&mas, 0x1600);
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 0x1501);
 	MT_BUG_ON(mt, mas.last != 0x1FFF);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_underflow(&mas));
 
 	/* prev: active ->active, continue */
 	entry = mas_prev(&mas, 0);
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0x1000);
 	MT_BUG_ON(mt, mas.last != 0x1500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* find: start ->active */
 	mas_set(&mas, 0);
@@ -3416,7 +3435,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0x1000);
 	MT_BUG_ON(mt, mas.last != 0x1500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* find: pause ->active */
 	mas_set(&mas, 0);
@@ -3425,7 +3444,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0x1000);
 	MT_BUG_ON(mt, mas.last != 0x1500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* find: start ->active on value */;
 	mas_set(&mas, 1200);
@@ -3433,14 +3452,14 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0x1000);
 	MT_BUG_ON(mt, mas.last != 0x1500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* find:active ->active */
 	entry = mas_find(&mas, ULONG_MAX);
 	MT_BUG_ON(mt, entry != ptr2);
 	MT_BUG_ON(mt, mas.index != 0x2000);
 	MT_BUG_ON(mt, mas.last != 0x2500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 
 	/* find:active -> active (NULL)*/
@@ -3448,35 +3467,35 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 0x2501);
 	MT_BUG_ON(mt, mas.last != 0x2FFF);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MAS_BUG_ON(&mas, !mas_is_active(&mas));
 
 	/* find: overflow ->active */
 	entry = mas_find(&mas, 0x5000);
 	MT_BUG_ON(mt, entry != ptr3);
 	MT_BUG_ON(mt, mas.index != 0x3000);
 	MT_BUG_ON(mt, mas.last != 0x3500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* find:active -> active (NULL) end*/
 	entry = mas_find(&mas, ULONG_MAX);
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 0x3501);
 	MT_BUG_ON(mt, mas.last != ULONG_MAX);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MAS_BUG_ON(&mas, !mas_is_active(&mas));
 
 	/* find_rev: active (END) ->active */
 	entry = mas_find_rev(&mas, 0);
 	MT_BUG_ON(mt, entry != ptr3);
 	MT_BUG_ON(mt, mas.index != 0x3000);
 	MT_BUG_ON(mt, mas.last != 0x3500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* find_rev:active ->active */
 	entry = mas_find_rev(&mas, 0);
 	MT_BUG_ON(mt, entry != ptr2);
 	MT_BUG_ON(mt, mas.index != 0x2000);
 	MT_BUG_ON(mt, mas.last != 0x2500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* find_rev: pause ->active */
 	mas_pause(&mas);
@@ -3484,14 +3503,14 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0x1000);
 	MT_BUG_ON(mt, mas.last != 0x1500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
-	/* find_rev:active -> active */
+	/* find_rev:active -> underflow */
 	entry = mas_find_rev(&mas, 0);
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 0);
 	MT_BUG_ON(mt, mas.last != 0x0FFF);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_underflow(&mas));
 
 	/* find_rev: start ->active */
 	mas_set(&mas, 0x1200);
@@ -3499,7 +3518,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0x1000);
 	MT_BUG_ON(mt, mas.last != 0x1500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* mas_walk start ->active */
 	mas_set(&mas, 0x1200);
@@ -3507,7 +3526,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0x1000);
 	MT_BUG_ON(mt, mas.last != 0x1500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* mas_walk start ->active */
 	mas_set(&mas, 0x1600);
@@ -3515,7 +3534,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 0x1501);
 	MT_BUG_ON(mt, mas.last != 0x1fff);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* mas_walk pause ->active */
 	mas_set(&mas, 0x1200);
@@ -3524,7 +3543,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0x1000);
 	MT_BUG_ON(mt, mas.last != 0x1500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* mas_walk pause -> active */
 	mas_set(&mas, 0x1600);
@@ -3533,25 +3552,25 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 0x1501);
 	MT_BUG_ON(mt, mas.last != 0x1fff);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* mas_walk none -> active */
 	mas_set(&mas, 0x1200);
-	mas.node = MAS_NONE;
+	mas.status = ma_none;
 	entry = mas_walk(&mas);
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0x1000);
 	MT_BUG_ON(mt, mas.last != 0x1500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* mas_walk none -> active */
 	mas_set(&mas, 0x1600);
-	mas.node = MAS_NONE;
+	mas.status = ma_none;
 	entry = mas_walk(&mas);
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 0x1501);
 	MT_BUG_ON(mt, mas.last != 0x1fff);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* mas_walk active -> active */
 	mas.index = 0x1200;
@@ -3561,7 +3580,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != ptr);
 	MT_BUG_ON(mt, mas.index != 0x1000);
 	MT_BUG_ON(mt, mas.last != 0x1500);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	/* mas_walk active -> active */
 	mas.index = 0x1600;
@@ -3570,7 +3589,7 @@ static noinline void __init check_state_handling(struct maple_tree *mt)
 	MT_BUG_ON(mt, entry != NULL);
 	MT_BUG_ON(mt, mas.index != 0x1501);
 	MT_BUG_ON(mt, mas.last != 0x1fff);
-	MT_BUG_ON(mt, !mas_active(mas));
+	MT_BUG_ON(mt, !mas_is_active(&mas));
 
 	mas_unlock(&mas);
 }
@@ -3585,10 +3604,6 @@ static int __init maple_tree_seed(void)
 
 	pr_info("\nTEST STARTING\n\n");
 
-	mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
-	check_root_expand(&tree);
-	mtree_destroy(&tree);
-
 #if defined(BENCH_SLOT_STORE)
 #define BENCH
 	mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
@@ -3617,13 +3632,18 @@ static int __init maple_tree_seed(void)
 	mtree_destroy(&tree);
 	goto skip;
 #endif
-#if defined(BENCH_FORK)
+#if defined(BENCH_LOAD)
 #define BENCH
 	mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
-	bench_forking(&tree);
+	bench_load(&tree);
 	mtree_destroy(&tree);
 	goto skip;
 #endif
+#if defined(BENCH_FORK)
+#define BENCH
+	bench_forking();
+	goto skip;
+#endif
 #if defined(BENCH_MT_FOR_EACH)
 #define BENCH
 	mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
@@ -3647,13 +3667,15 @@ static int __init maple_tree_seed(void)
 #endif
 
 	mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
-	check_iteration(&tree);
+	check_root_expand(&tree);
 	mtree_destroy(&tree);
 
 	mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
-	check_forking(&tree);
+	check_iteration(&tree);
 	mtree_destroy(&tree);
 
+	check_forking();
+
 	mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE);
 	check_mas_store_gfp(&tree);
 	mtree_destroy(&tree);
diff --git a/lib/zstd/Makefile b/lib/zstd/Makefile
index 20f08c644b71..464c410b2768 100644
--- a/lib/zstd/Makefile
+++ b/lib/zstd/Makefile
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 # ################################################################
-# Copyright (c) Facebook, Inc.
+# Copyright (c) Meta Platforms, Inc. and affiliates.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/zstd/common/allocations.h b/lib/zstd/common/allocations.h
new file mode 100644
index 000000000000..05adbbeccaa9
--- /dev/null
+++ b/lib/zstd/common/allocations.h
@@ -0,0 +1,56 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* This file provides custom allocation primitives
+ */
+
+#define ZSTD_DEPS_NEED_MALLOC
+#include "zstd_deps.h"   /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */
+
+#include "mem.h" /* MEM_STATIC */
+#define ZSTD_STATIC_LINKING_ONLY
+#include <linux/zstd.h> /* ZSTD_customMem */
+
+#ifndef ZSTD_ALLOCATIONS_H
+#define ZSTD_ALLOCATIONS_H
+
+/* custom memory allocation functions */
+
+MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem)
+{
+    if (customMem.customAlloc)
+        return customMem.customAlloc(customMem.opaque, size);
+    return ZSTD_malloc(size);
+}
+
+MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem)
+{
+    if (customMem.customAlloc) {
+        /* calloc implemented as malloc+memset;
+         * not as efficient as calloc, but next best guess for custom malloc */
+        void* const ptr = customMem.customAlloc(customMem.opaque, size);
+        ZSTD_memset(ptr, 0, size);
+        return ptr;
+    }
+    return ZSTD_calloc(1, size);
+}
+
+MEM_STATIC void ZSTD_customFree(void* ptr, ZSTD_customMem customMem)
+{
+    if (ptr!=NULL) {
+        if (customMem.customFree)
+            customMem.customFree(customMem.opaque, ptr);
+        else
+            ZSTD_free(ptr);
+    }
+}
+
+#endif /* ZSTD_ALLOCATIONS_H */
diff --git a/lib/zstd/common/bits.h b/lib/zstd/common/bits.h
new file mode 100644
index 000000000000..aa3487ec4b6a
--- /dev/null
+++ b/lib/zstd/common/bits.h
@@ -0,0 +1,149 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef ZSTD_BITS_H
+#define ZSTD_BITS_H
+
+#include "mem.h"
+
+MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val)
+{
+    assert(val != 0);
+    {
+        static const U32 DeBruijnBytePos[32] = {0, 1, 28, 2, 29, 14, 24, 3,
+                                                30, 22, 20, 15, 25, 17, 4, 8,
+                                                31, 27, 13, 23, 21, 19, 16, 7,
+                                                26, 12, 18, 6, 11, 5, 10, 9};
+        return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >> 27];
+    }
+}
+
+MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val)
+{
+    assert(val != 0);
+#   if (__GNUC__ >= 4)
+        return (unsigned)__builtin_ctz(val);
+#   else
+        return ZSTD_countTrailingZeros32_fallback(val);
+#   endif
+}
+
+MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) {
+    assert(val != 0);
+    {
+        static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29,
+                                            11, 14, 16, 18, 22, 25, 3, 30,
+                                            8, 12, 20, 28, 15, 17, 24, 7,
+                                            19, 27, 23, 6, 26, 5, 4, 31};
+        val |= val >> 1;
+        val |= val >> 2;
+        val |= val >> 4;
+        val |= val >> 8;
+        val |= val >> 16;
+        return 31 - DeBruijnClz[(val * 0x07C4ACDDU) >> 27];
+    }
+}
+
+MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val)
+{
+    assert(val != 0);
+#   if (__GNUC__ >= 4)
+        return (unsigned)__builtin_clz(val);
+#   else
+        return ZSTD_countLeadingZeros32_fallback(val);
+#   endif
+}
+
+MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val)
+{
+    assert(val != 0);
+#   if (__GNUC__ >= 4) && defined(__LP64__)
+        return (unsigned)__builtin_ctzll(val);
+#   else
+        {
+            U32 mostSignificantWord = (U32)(val >> 32);
+            U32 leastSignificantWord = (U32)val;
+            if (leastSignificantWord == 0) {
+                return 32 + ZSTD_countTrailingZeros32(mostSignificantWord);
+            } else {
+                return ZSTD_countTrailingZeros32(leastSignificantWord);
+            }
+        }
+#   endif
+}
+
+MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val)
+{
+    assert(val != 0);
+#   if (__GNUC__ >= 4)
+        return (unsigned)(__builtin_clzll(val));
+#   else
+        {
+            U32 mostSignificantWord = (U32)(val >> 32);
+            U32 leastSignificantWord = (U32)val;
+            if (mostSignificantWord == 0) {
+                return 32 + ZSTD_countLeadingZeros32(leastSignificantWord);
+            } else {
+                return ZSTD_countLeadingZeros32(mostSignificantWord);
+            }
+        }
+#   endif
+}
+
+MEM_STATIC unsigned ZSTD_NbCommonBytes(size_t val)
+{
+    if (MEM_isLittleEndian()) {
+        if (MEM_64bits()) {
+            return ZSTD_countTrailingZeros64((U64)val) >> 3;
+        } else {
+            return ZSTD_countTrailingZeros32((U32)val) >> 3;
+        }
+    } else {  /* Big Endian CPU */
+        if (MEM_64bits()) {
+            return ZSTD_countLeadingZeros64((U64)val) >> 3;
+        } else {
+            return ZSTD_countLeadingZeros32((U32)val) >> 3;
+        }
+    }
+}
+
+MEM_STATIC unsigned ZSTD_highbit32(U32 val)   /* compress, dictBuilder, decodeCorpus */
+{
+    assert(val != 0);
+    return 31 - ZSTD_countLeadingZeros32(val);
+}
+
+/* ZSTD_rotateRight_*():
+ * Rotates a bitfield to the right by "count" bits.
+ * https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts
+ */
+MEM_STATIC
+U64 ZSTD_rotateRight_U64(U64 const value, U32 count) {
+    assert(count < 64);
+    count &= 0x3F; /* for fickle pattern recognition */
+    return (value >> count) | (U64)(value << ((0U - count) & 0x3F));
+}
+
+MEM_STATIC
+U32 ZSTD_rotateRight_U32(U32 const value, U32 count) {
+    assert(count < 32);
+    count &= 0x1F; /* for fickle pattern recognition */
+    return (value >> count) | (U32)(value << ((0U - count) & 0x1F));
+}
+
+MEM_STATIC
+U16 ZSTD_rotateRight_U16(U16 const value, U32 count) {
+    assert(count < 16);
+    count &= 0x0F; /* for fickle pattern recognition */
+    return (value >> count) | (U16)(value << ((0U - count) & 0x0F));
+}
+
+#endif /* ZSTD_BITS_H */
diff --git a/lib/zstd/common/bitstream.h b/lib/zstd/common/bitstream.h
index feef3a1b1d60..444dc4f85c64 100644
--- a/lib/zstd/common/bitstream.h
+++ b/lib/zstd/common/bitstream.h
@@ -1,7 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /* ******************************************************************
  * bitstream
  * Part of FSE library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -27,6 +28,7 @@
 #include "compiler.h"       /* UNLIKELY() */
 #include "debug.h"          /* assert(), DEBUGLOG(), RAWLOG() */
 #include "error_private.h"  /* error codes and messages */
+#include "bits.h"           /* ZSTD_highbit32 */
 
 
 /*=========================================
@@ -122,33 +124,6 @@ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
 MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
 /* faster, but works only if nbBits >= 1 */
 
-
-
-/*-**************************************************************
-*  Internal functions
-****************************************************************/
-MEM_STATIC unsigned BIT_highbit32 (U32 val)
-{
-    assert(val != 0);
-    {
-#   if (__GNUC__ >= 3)   /* Use GCC Intrinsic */
-        return __builtin_clz (val) ^ 31;
-#   else   /* Software version */
-        static const unsigned DeBruijnClz[32] = { 0,  9,  1, 10, 13, 21,  2, 29,
-                                                 11, 14, 16, 18, 22, 25,  3, 30,
-                                                  8, 12, 20, 28, 15, 17, 24,  7,
-                                                 19, 27, 23,  6, 26,  5,  4, 31 };
-        U32 v = val;
-        v |= v >> 1;
-        v |= v >> 2;
-        v |= v >> 4;
-        v |= v >> 8;
-        v |= v >> 16;
-        return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
-#   endif
-    }
-}
-
 /*=====    Local Constants   =====*/
 static const unsigned BIT_mask[] = {
     0,          1,         3,         7,         0xF,       0x1F,
@@ -178,6 +153,12 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
     return 0;
 }
 
+MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
+{
+    assert(nbBits < BIT_MASK_SIZE);
+    return bitContainer & BIT_mask[nbBits];
+}
+
 /*! BIT_addBits() :
  *  can add up to 31 bits into `bitC`.
  *  Note : does not check for register overflow ! */
@@ -187,7 +168,7 @@ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
     DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32);
     assert(nbBits < BIT_MASK_SIZE);
     assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
-    bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos;
+    bitC->bitContainer |= BIT_getLowerBits(value, nbBits) << bitC->bitPos;
     bitC->bitPos += nbBits;
 }
 
@@ -266,7 +247,7 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
         bitD->ptr   = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
         bitD->bitContainer = MEM_readLEST(bitD->ptr);
         { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
-          bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;  /* ensures bitsConsumed is always set */
+          bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0;  /* ensures bitsConsumed is always set */
           if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
     } else {
         bitD->ptr   = bitD->start;
@@ -294,7 +275,7 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si
         default: break;
         }
         {   BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
-            bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
+            bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0;
             if (lastByte == 0) return ERROR(corruption_detected);  /* endMark not present */
         }
         bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8;
@@ -325,12 +306,6 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 c
 #endif
 }
 
-MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
-{
-    assert(nbBits < BIT_MASK_SIZE);
-    return bitContainer & BIT_mask[nbBits];
-}
-
 /*! BIT_lookBits() :
  *  Provides next n bits from local register.
  *  local register is not modified.
@@ -377,7 +352,7 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned n
 }
 
 /*! BIT_readBitsFast() :
- *  unsafe version; only works only if nbBits >= 1 */
+ *  unsafe version; only works if nbBits >= 1 */
 MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
 {
     size_t const value = BIT_lookBitsFast(bitD, nbBits);
@@ -408,7 +383,7 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD)
  *  This function is safe, it guarantees it will not read beyond src buffer.
  * @return : status of `BIT_DStream_t` internal register.
  *           when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */
-MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
+MEM_STATIC FORCE_INLINE_ATTR BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
 {
     if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))  /* overflow detected, like end of stream */
         return BIT_DStream_overflow;
diff --git a/lib/zstd/common/compiler.h b/lib/zstd/common/compiler.h
index c42d39faf9bd..c437e0975575 100644
--- a/lib/zstd/common/compiler.h
+++ b/lib/zstd/common/compiler.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -179,6 +180,17 @@
 *  Sanitizer
 *****************************************************************/
 
+/* Issue #3240 reports an ASAN failure on an llvm-mingw build. Out of an
+ * abundance of caution, disable our custom poisoning on mingw. */
+#ifdef __MINGW32__
+#ifndef ZSTD_ASAN_DONT_POISON_WORKSPACE
+#define ZSTD_ASAN_DONT_POISON_WORKSPACE 1
+#endif
+#ifndef ZSTD_MSAN_DONT_POISON_WORKSPACE
+#define ZSTD_MSAN_DONT_POISON_WORKSPACE 1
+#endif
+#endif
+
 
 
 #endif /* ZSTD_COMPILER_H */
diff --git a/lib/zstd/common/cpu.h b/lib/zstd/common/cpu.h
index 0db7b42407ee..d8319a2bef4c 100644
--- a/lib/zstd/common/cpu.h
+++ b/lib/zstd/common/cpu.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/zstd/common/debug.c b/lib/zstd/common/debug.c
index bb863c9ea616..d77926cbad14 100644
--- a/lib/zstd/common/debug.c
+++ b/lib/zstd/common/debug.c
@@ -1,7 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /* ******************************************************************
  * debug
  * Part of FSE library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -21,4 +22,6 @@
 
 #include "debug.h"
 
+#if (DEBUGLEVEL>=2)
 int g_debuglevel = DEBUGLEVEL;
+#endif
diff --git a/lib/zstd/common/debug.h b/lib/zstd/common/debug.h
index 6dd88d1fbd02..da0dbfc614b8 100644
--- a/lib/zstd/common/debug.h
+++ b/lib/zstd/common/debug.h
@@ -1,7 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /* ******************************************************************
  * debug
  * Part of FSE library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/zstd/common/entropy_common.c b/lib/zstd/common/entropy_common.c
index fef67056f052..6cdd82233fb5 100644
--- a/lib/zstd/common/entropy_common.c
+++ b/lib/zstd/common/entropy_common.c
@@ -1,6 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /* ******************************************************************
  * Common functions of New Generation Entropy library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -19,8 +20,8 @@
 #include "error_private.h"       /* ERR_*, ERROR */
 #define FSE_STATIC_LINKING_ONLY  /* FSE_MIN_TABLELOG */
 #include "fse.h"
-#define HUF_STATIC_LINKING_ONLY  /* HUF_TABLELOG_ABSOLUTEMAX */
 #include "huf.h"
+#include "bits.h"                /* ZSDT_highbit32, ZSTD_countTrailingZeros32 */
 
 
 /*===   Version   ===*/
@@ -38,23 +39,6 @@ const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
 /*-**************************************************************
 *  FSE NCount encoding-decoding
 ****************************************************************/
-static U32 FSE_ctz(U32 val)
-{
-    assert(val != 0);
-    {
-#   if (__GNUC__ >= 3)   /* GCC Intrinsic */
-        return __builtin_ctz(val);
-#   else   /* Software version */
-        U32 count = 0;
-        while ((val & 1) == 0) {
-            val >>= 1;
-            ++count;
-        }
-        return count;
-#   endif
-    }
-}
-
 FORCE_INLINE_TEMPLATE
 size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
                            const void* headerBuffer, size_t hbSize)
@@ -102,7 +86,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne
              * repeat.
              * Avoid UB by setting the high bit to 1.
              */
-            int repeats = FSE_ctz(~bitStream | 0x80000000) >> 1;
+            int repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1;
             while (repeats >= 12) {
                 charnum += 3 * 12;
                 if (LIKELY(ip <= iend-7)) {
@@ -113,7 +97,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne
                     ip = iend - 4;
                 }
                 bitStream = MEM_readLE32(ip) >> bitCount;
-                repeats = FSE_ctz(~bitStream | 0x80000000) >> 1;
+                repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1;
             }
             charnum += 3 * repeats;
             bitStream >>= 2 * repeats;
@@ -178,7 +162,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne
                  * know that threshold > 1.
                  */
                 if (remaining <= 1) break;
-                nbBits = BIT_highbit32(remaining) + 1;
+                nbBits = ZSTD_highbit32(remaining) + 1;
                 threshold = 1 << (nbBits - 1);
             }
             if (charnum >= maxSV1) break;
@@ -253,7 +237,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
                      const void* src, size_t srcSize)
 {
     U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
-    return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0);
+    return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* flags */ 0);
 }
 
 FORCE_INLINE_TEMPLATE size_t
@@ -301,14 +285,14 @@ HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats,
     if (weightTotal == 0) return ERROR(corruption_detected);
 
     /* get last non-null symbol weight (implied, total must be 2^n) */
-    {   U32 const tableLog = BIT_highbit32(weightTotal) + 1;
+    {   U32 const tableLog = ZSTD_highbit32(weightTotal) + 1;
         if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected);
         *tableLogPtr = tableLog;
         /* determine last weight */
         {   U32 const total = 1 << tableLog;
             U32 const rest = total - weightTotal;
-            U32 const verif = 1 << BIT_highbit32(rest);
-            U32 const lastWeight = BIT_highbit32(rest) + 1;
+            U32 const verif = 1 << ZSTD_highbit32(rest);
+            U32 const lastWeight = ZSTD_highbit32(rest) + 1;
             if (verif != rest) return ERROR(corruption_detected);    /* last value must be a clean power of 2 */
             huffWeight[oSize] = (BYTE)lastWeight;
             rankStats[lastWeight]++;
@@ -345,13 +329,13 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats,
                      U32* nbSymbolsPtr, U32* tableLogPtr,
                      const void* src, size_t srcSize,
                      void* workSpace, size_t wkspSize,
-                     int bmi2)
+                     int flags)
 {
 #if DYNAMIC_BMI2
-    if (bmi2) {
+    if (flags & HUF_flags_bmi2) {
         return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
     }
 #endif
-    (void)bmi2;
+    (void)flags;
     return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize);
 }
diff --git a/lib/zstd/common/error_private.c b/lib/zstd/common/error_private.c
index 6d1135f8c373..a4062d30d170 100644
--- a/lib/zstd/common/error_private.c
+++ b/lib/zstd/common/error_private.c
@@ -1,5 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -27,9 +28,11 @@ const char* ERR_getErrorString(ERR_enum code)
     case PREFIX(version_unsupported): return "Version not supported";
     case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
     case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
-    case PREFIX(corruption_detected): return "Corrupted block detected";
+    case PREFIX(corruption_detected): return "Data corruption detected";
     case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
+    case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification";
     case PREFIX(parameter_unsupported): return "Unsupported parameter";
+    case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters";
     case PREFIX(parameter_outOfBound): return "Parameter is out of bound";
     case PREFIX(init_missing): return "Context should be init first";
     case PREFIX(memory_allocation): return "Allocation error : not enough memory";
@@ -38,17 +41,22 @@ const char* ERR_getErrorString(ERR_enum code)
     case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
     case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
     case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
+    case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected";
     case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
     case PREFIX(dictionary_wrong): return "Dictionary mismatch";
     case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
     case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
     case PREFIX(srcSize_wrong): return "Src size is incorrect";
     case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer";
+    case PREFIX(noForwardProgress_destFull): return "Operation made no progress over multiple calls, due to output buffer being full";
+    case PREFIX(noForwardProgress_inputEmpty): return "Operation made no progress over multiple calls, due to input being empty";
         /* following error codes are not stable and may be removed or changed in a future version */
     case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
     case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
     case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong";
     case PREFIX(srcBuffer_wrong): return "Source buffer is wrong";
+    case PREFIX(sequenceProducer_failed): return "Block-level external sequence producer returned an error code";
+    case PREFIX(externalSequences_invalid): return "External sequences are not valid";
     case PREFIX(maxCode):
     default: return notErrorCode;
     }
diff --git a/lib/zstd/common/error_private.h b/lib/zstd/common/error_private.h
index ca5101e542fa..9a4699a38a88 100644
--- a/lib/zstd/common/error_private.h
+++ b/lib/zstd/common/error_private.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/zstd/common/fse.h b/lib/zstd/common/fse.h
index 4507043b2287..c4e25a219142 100644
--- a/lib/zstd/common/fse.h
+++ b/lib/zstd/common/fse.h
@@ -1,7 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /* ******************************************************************
  * FSE : Finite State Entropy codec
  * Public Prototypes declaration
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -50,34 +51,6 @@
 FSE_PUBLIC_API unsigned FSE_versionNumber(void);   /*< library version number; to be used when checking dll version */
 
 
-/*-****************************************
-*  FSE simple functions
-******************************************/
-/*! FSE_compress() :
-    Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
-    'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize).
-    @return : size of compressed data (<= dstCapacity).
-    Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
-                     if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
-                     if FSE_isError(return), compression failed (more details using FSE_getErrorName())
-*/
-FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity,
-                             const void* src, size_t srcSize);
-
-/*! FSE_decompress():
-    Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
-    into already allocated destination buffer 'dst', of size 'dstCapacity'.
-    @return : size of regenerated data (<= maxDstSize),
-              or an error code, which can be tested using FSE_isError() .
-
-    ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!!
-    Why ? : making this distinction requires a header.
-    Header management is intentionally delegated to the user layer, which can better manage special cases.
-*/
-FSE_PUBLIC_API size_t FSE_decompress(void* dst,  size_t dstCapacity,
-                               const void* cSrc, size_t cSrcSize);
-
-
 /*-*****************************************
 *  Tool functions
 ******************************************/
@@ -88,20 +61,6 @@ FSE_PUBLIC_API unsigned    FSE_isError(size_t code);        /* tells if a return
 FSE_PUBLIC_API const char* FSE_getErrorName(size_t code);   /* provides error code string (useful for debugging) */
 
 
-/*-*****************************************
-*  FSE advanced functions
-******************************************/
-/*! FSE_compress2() :
-    Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog'
-    Both parameters can be defined as '0' to mean : use default value
-    @return : size of compressed data
-    Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
-                     if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
-                     if FSE_isError(return), it's an error code.
-*/
-FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
-
-
 /*-*****************************************
 *  FSE detailed API
 ******************************************/
@@ -161,8 +120,6 @@ FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize,
 /*! Constructor and Destructor of FSE_CTable.
     Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
 typedef unsigned FSE_CTable;   /* don't allocate that. It's only meant to be more restrictive than void* */
-FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog);
-FSE_PUBLIC_API void        FSE_freeCTable (FSE_CTable* ct);
 
 /*! FSE_buildCTable():
     Builds `ct`, which must be already allocated, using FSE_createCTable().
@@ -238,23 +195,7 @@ FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter,
                            unsigned* maxSymbolValuePtr, unsigned* tableLogPtr,
                            const void* rBuffer, size_t rBuffSize, int bmi2);
 
-/*! Constructor and Destructor of FSE_DTable.
-    Note that its size depends on 'tableLog' */
 typedef unsigned FSE_DTable;   /* don't allocate that. It's just a way to be more restrictive than void* */
-FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog);
-FSE_PUBLIC_API void        FSE_freeDTable(FSE_DTable* dt);
-
-/*! FSE_buildDTable():
-    Builds 'dt', which must be already allocated, using FSE_createDTable().
-    return : 0, or an errorCode, which can be tested using FSE_isError() */
-FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
-
-/*! FSE_decompress_usingDTable():
-    Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
-    into `dst` which must be already allocated.
-    @return : size of regenerated data (necessarily <= `dstCapacity`),
-              or an errorCode, which can be tested using FSE_isError() */
-FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
 
 /*!
 Tutorial :
@@ -317,16 +258,6 @@ If there is an error, the function will return an error code, which can be teste
 unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
 /*< same as FSE_optimalTableLog(), which used `minus==2` */
 
-/* FSE_compress_wksp() :
- * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`).
- * FSE_COMPRESS_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable.
- */
-#define FSE_COMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue)   ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) )
-size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
-
-size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
-/*< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */
-
 size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
 /*< build a fake FSE_CTable, designed to compress always the same symbolValue */
 
@@ -344,19 +275,11 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi
 FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
 /*< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */
 
-size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
-/*< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */
-
-size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
-/*< build a fake FSE_DTable, designed to always generate the same symbolValue */
-
-#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1)
+#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + 1 + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1)
 #define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned))
-size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize);
-/*< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)` */
-
 size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2);
-/*< Same as FSE_decompress_wksp() but with dynamic BMI2 support. Pass 1 if your CPU supports BMI2 or 0 if it doesn't. */
+/*< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)`.
+ * Set bmi2 to 1 if your CPU supports BMI2 or 0 if it doesn't */
 
 typedef enum {
    FSE_repeat_none,  /*< Cannot use the previous table */
@@ -552,7 +475,7 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt
 
 /* FSE_getMaxNbBits() :
  * Approximate maximum cost of a symbol, in bits.
- * Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)
+ * Fractional get rounded up (i.e. a symbol with a normalized frequency of 3 gives the same result as a frequency of 2)
  * note 1 : assume symbolValue is valid (<= maxSymbolValue)
  * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */
 MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue)
diff --git a/lib/zstd/common/fse_decompress.c b/lib/zstd/common/fse_decompress.c
index 8dcb8ca39767..99ce8fa54d08 100644
--- a/lib/zstd/common/fse_decompress.c
+++ b/lib/zstd/common/fse_decompress.c
@@ -1,6 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /* ******************************************************************
  * FSE : Finite State Entropy decoder
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -24,6 +25,7 @@
 #include "error_private.h"
 #define ZSTD_DEPS_NEED_MALLOC
 #include "zstd_deps.h"
+#include "bits.h"       /* ZSTD_highbit32 */
 
 
 /* **************************************************************
@@ -55,19 +57,6 @@
 #define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
 #define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
 
-
-/* Function templates */
-FSE_DTable* FSE_createDTable (unsigned tableLog)
-{
-    if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
-    return (FSE_DTable*)ZSTD_malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
-}
-
-void FSE_freeDTable (FSE_DTable* dt)
-{
-    ZSTD_free(dt);
-}
-
 static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize)
 {
     void* const tdPtr = dt+1;   /* because *dt is unsigned, 32-bits aligned on 32-bits */
@@ -127,10 +116,10 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo
             }
         }
         /* Now we spread those positions across the table.
-         * The benefit of doing it in two stages is that we avoid the the
+         * The benefit of doing it in two stages is that we avoid the
          * variable size inner loop, which caused lots of branch misses.
          * Now we can run through all the positions without any branch misses.
-         * We unroll the loop twice, since that is what emperically worked best.
+         * We unroll the loop twice, since that is what empirically worked best.
          */
         {
             size_t position = 0;
@@ -166,7 +155,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo
         for (u=0; u<tableSize; u++) {
             FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
             U32 const nextState = symbolNext[symbol]++;
-            tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32(nextState) );
+            tableDecode[u].nbBits = (BYTE) (tableLog - ZSTD_highbit32(nextState) );
             tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
     }   }
 
@@ -184,49 +173,6 @@ size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsi
 /*-*******************************************************
 *  Decompression (Byte symbols)
 *********************************************************/
-size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
-{
-    void* ptr = dt;
-    FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
-    void* dPtr = dt + 1;
-    FSE_decode_t* const cell = (FSE_decode_t*)dPtr;
-
-    DTableH->tableLog = 0;
-    DTableH->fastMode = 0;
-
-    cell->newState = 0;
-    cell->symbol = symbolValue;
-    cell->nbBits = 0;
-
-    return 0;
-}
-
-
-size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
-{
-    void* ptr = dt;
-    FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
-    void* dPtr = dt + 1;
-    FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr;
-    const unsigned tableSize = 1 << nbBits;
-    const unsigned tableMask = tableSize - 1;
-    const unsigned maxSV1 = tableMask+1;
-    unsigned s;
-
-    /* Sanity checks */
-    if (nbBits < 1) return ERROR(GENERIC);         /* min size */
-
-    /* Build Decoding Table */
-    DTableH->tableLog = (U16)nbBits;
-    DTableH->fastMode = 1;
-    for (s=0; s<maxSV1; s++) {
-        dinfo[s].newState = 0;
-        dinfo[s].symbol = (BYTE)s;
-        dinfo[s].nbBits = (BYTE)nbBits;
-    }
-
-    return 0;
-}
 
 FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic(
           void* dst, size_t maxDstSize,
@@ -290,26 +236,6 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_usingDTable_generic(
     return op-ostart;
 }
 
-
-size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
-                            const void* cSrc, size_t cSrcSize,
-                            const FSE_DTable* dt)
-{
-    const void* ptr = dt;
-    const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
-    const U32 fastMode = DTableH->fastMode;
-
-    /* select fast mode (static) */
-    if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
-    return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
-}
-
-
-size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize)
-{
-    return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, /* bmi2 */ 0);
-}
-
 typedef struct {
     short ncount[FSE_MAX_SYMBOL_VALUE + 1];
     FSE_DTable dtable[]; /* Dynamically sized */
@@ -342,7 +268,8 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body(
     }
 
     if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge);
-    workSpace = wksp->dtable + FSE_DTABLE_SIZE_U32(tableLog);
+    assert(sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog) <= wkspSize);
+    workSpace = (BYTE*)workSpace + sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog);
     wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog);
 
     CHECK_F( FSE_buildDTable_internal(wksp->dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) );
@@ -382,9 +309,4 @@ size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc,
     return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize);
 }
 
-
-typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
-
-
-
 #endif   /* FSE_COMMONDEFS_ONLY */
diff --git a/lib/zstd/common/huf.h b/lib/zstd/common/huf.h
index 5042ff870308..8e7943092ed1 100644
--- a/lib/zstd/common/huf.h
+++ b/lib/zstd/common/huf.h
@@ -1,7 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /* ******************************************************************
  * huff0 huffman codec,
  * part of Finite State Entropy library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * You can contact the author at :
  * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -18,99 +19,22 @@
 
 /* *** Dependencies *** */
 #include "zstd_deps.h"    /* size_t */
-
-
-/* *** library symbols visibility *** */
-/* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual,
- *        HUF symbols remain "private" (internal symbols for library only).
- *        Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */
-#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4)
-#  define HUF_PUBLIC_API __attribute__ ((visibility ("default")))
-#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1)   /* Visual expected */
-#  define HUF_PUBLIC_API __declspec(dllexport)
-#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1)
-#  define HUF_PUBLIC_API __declspec(dllimport)  /* not required, just to generate faster code (saves a function pointer load from IAT and an indirect jump) */
-#else
-#  define HUF_PUBLIC_API
-#endif
-
-
-/* ========================== */
-/* ***  simple functions  *** */
-/* ========================== */
-
-/* HUF_compress() :
- *  Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
- * 'dst' buffer must be already allocated.
- *  Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
- * `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
- * @return : size of compressed data (<= `dstCapacity`).
- *  Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
- *                   if HUF_isError(return), compression failed (more details using HUF_getErrorName())
- */
-HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity,
-                             const void* src, size_t srcSize);
-
-/* HUF_decompress() :
- *  Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
- *  into already allocated buffer 'dst', of minimum size 'dstSize'.
- * `originalSize` : **must** be the ***exact*** size of original (uncompressed) data.
- *  Note : in contrast with FSE, HUF_decompress can regenerate
- *         RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
- *         because it knows size to regenerate (originalSize).
- * @return : size of regenerated data (== originalSize),
- *           or an error code, which can be tested using HUF_isError()
- */
-HUF_PUBLIC_API size_t HUF_decompress(void* dst,  size_t originalSize,
-                               const void* cSrc, size_t cSrcSize);
+#include "mem.h"          /* U32 */
+#define FSE_STATIC_LINKING_ONLY
+#include "fse.h"
 
 
 /* ***   Tool functions *** */
-#define HUF_BLOCKSIZE_MAX (128 * 1024)                  /*< maximum input size for a single block compressed with HUF_compress */
-HUF_PUBLIC_API size_t HUF_compressBound(size_t size);   /*< maximum compressed size (worst case) */
+#define HUF_BLOCKSIZE_MAX (128 * 1024)   /*< maximum input size for a single block compressed with HUF_compress */
+size_t HUF_compressBound(size_t size);   /*< maximum compressed size (worst case) */
 
 /* Error Management */
-HUF_PUBLIC_API unsigned    HUF_isError(size_t code);       /*< tells if a return value is an error code */
-HUF_PUBLIC_API const char* HUF_getErrorName(size_t code);  /*< provides error code string (useful for debugging) */
+unsigned    HUF_isError(size_t code);       /*< tells if a return value is an error code */
+const char* HUF_getErrorName(size_t code);  /*< provides error code string (useful for debugging) */
 
 
-/* ***   Advanced function   *** */
-
-/* HUF_compress2() :
- *  Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`.
- * `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX .
- * `tableLog` must be `<= HUF_TABLELOG_MAX` . */
-HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity,
-                               const void* src, size_t srcSize,
-                               unsigned maxSymbolValue, unsigned tableLog);
-
-/* HUF_compress4X_wksp() :
- *  Same as HUF_compress2(), but uses externally allocated `workSpace`.
- * `workspace` must be at least as large as HUF_WORKSPACE_SIZE */
 #define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */)
 #define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64))
-HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity,
-                                     const void* src, size_t srcSize,
-                                     unsigned maxSymbolValue, unsigned tableLog,
-                                     void* workSpace, size_t wkspSize);
-
-#endif   /* HUF_H_298734234 */
-
-/* ******************************************************************
- *  WARNING !!
- *  The following section contains advanced and experimental definitions
- *  which shall never be used in the context of a dynamic library,
- *  because they are not guaranteed to remain stable in the future.
- *  Only consider them in association with static linking.
- * *****************************************************************/
-#if !defined(HUF_H_HUF_STATIC_LINKING_ONLY)
-#define HUF_H_HUF_STATIC_LINKING_ONLY
-
-/* *** Dependencies *** */
-#include "mem.h"   /* U32 */
-#define FSE_STATIC_LINKING_ONLY
-#include "fse.h"
-
 
 /* *** Constants *** */
 #define HUF_TABLELOG_MAX      12      /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */
@@ -151,25 +75,49 @@ typedef U32 HUF_DTable;
 /* ****************************************
 *  Advanced decompression functions
 ******************************************/
-size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /*< single-symbol decoder */
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /*< double-symbols decoder */
-#endif
 
-size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /*< decodes RLE and uncompressed */
-size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /*< considers RLE and uncompressed as errors */
-size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /*< considers RLE and uncompressed as errors */
-size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /*< single-symbol decoder */
-size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);   /*< single-symbol decoder */
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /*< double-symbols decoder */
-size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);   /*< double-symbols decoder */
-#endif
+/*
+ * Huffman flags bitset.
+ * For all flags, 0 is the default value.
+ */
+typedef enum {
+    /*
+     * If compiled with DYNAMIC_BMI2: Set flag only if the CPU supports BMI2 at runtime.
+     * Otherwise: Ignored.
+     */
+    HUF_flags_bmi2 = (1 << 0),
+    /*
+     * If set: Test possible table depths to find the one that produces the smallest header + encoded size.
+     * If unset: Use heuristic to find the table depth.
+     */
+    HUF_flags_optimalDepth = (1 << 1),
+    /*
+     * If set: If the previous table can encode the input, always reuse the previous table.
+     * If unset: If the previous table can encode the input, reuse the previous table if it results in a smaller output.
+     */
+    HUF_flags_preferRepeat = (1 << 2),
+    /*
+     * If set: Sample the input and check if the sample is uncompressible, if it is then don't attempt to compress.
+     * If unset: Always histogram the entire input.
+     */
+    HUF_flags_suspectUncompressible = (1 << 3),
+    /*
+     * If set: Don't use assembly implementations
+     * If unset: Allow using assembly implementations
+     */
+    HUF_flags_disableAsm = (1 << 4),
+    /*
+     * If set: Don't use the fast decoding loop, always use the fallback decoding loop.
+     * If unset: Use the fast decoding loop when possible.
+     */
+    HUF_flags_disableFast = (1 << 5)
+} HUF_flags_e;
 
 
 /* ****************************************
  *  HUF detailed API
  * ****************************************/
+#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra
 
 /*! HUF_compress() does the following:
  *  1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h")
@@ -182,12 +130,12 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
  *  For example, it's possible to compress several blocks using the same 'CTable',
  *  or to save and regenerate 'CTable' using external methods.
  */
-unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
-size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);   /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */
-size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
+unsigned HUF_minTableLog(unsigned symbolCardinality);
+unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue);
+unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace,
+ size_t wkspSize, HUF_CElt* table, const unsigned* count, int flags); /* table is used as scratch space for building and testing tables, not a return value */
 size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize);
-size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
-size_t HUF_compress4X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2);
+size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags);
 size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
 int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue);
 
@@ -196,6 +144,7 @@ typedef enum {
    HUF_repeat_check, /*< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
    HUF_repeat_valid  /*< Can use the previous table and it is assumed to be valid */
  } HUF_repeat;
+
 /* HUF_compress4X_repeat() :
  *  Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
  *  If it uses hufTable it does not modify hufTable or repeat.
@@ -206,13 +155,13 @@ size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
                        const void* src, size_t srcSize,
                        unsigned maxSymbolValue, unsigned tableLog,
                        void* workSpace, size_t wkspSize,    /*< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
-                       HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible);
+                       HUF_CElt* hufTable, HUF_repeat* repeat, int flags);
 
 /* HUF_buildCTable_wksp() :
  *  Same as HUF_buildCTable(), but using externally allocated scratch buffer.
  * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE.
  */
-#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1)
+#define HUF_CTABLE_WORKSPACE_SIZE_U32 ((4 * (HUF_SYMBOLVALUE_MAX + 1)) + 192)
 #define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
 size_t HUF_buildCTable_wksp (HUF_CElt* tree,
                        const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
@@ -238,7 +187,7 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize,
                           U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr,
                           const void* src, size_t srcSize,
                           void* workspace, size_t wkspSize,
-                          int bmi2);
+                          int flags);
 
 /* HUF_readCTable() :
  *  Loading a CTable saved with HUF_writeCTable() */
@@ -276,32 +225,12 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
 #define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9))
 #define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
 
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize);
-size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
-#endif
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
-size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
-#endif
-
-size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
-#endif
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
-#endif
-
 
 /* ====================== */
 /* single stream variants */
 /* ====================== */
 
-size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
-size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);  /*< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U64 U64 */
-size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
-size_t HUF_compress1X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2);
+size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags);
 /* HUF_compress1X_repeat() :
  *  Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
  *  If it uses hufTable it does not modify hufTable or repeat.
@@ -312,47 +241,28 @@ size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
                        const void* src, size_t srcSize,
                        unsigned maxSymbolValue, unsigned tableLog,
                        void* workSpace, size_t wkspSize,   /*< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */
-                       HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible);
-
-size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* single-symbol decoder */
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* double-symbol decoder */
-#endif
-
-size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
-size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /*< single-symbol decoder */
-size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);   /*< single-symbol decoder */
-#endif
-#ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /*< double-symbols decoder */
-size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);   /*< double-symbols decoder */
-#endif
+                       HUF_CElt* hufTable, HUF_repeat* repeat, int flags);
 
-size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);   /*< automatic selection of sing or double symbol decoder, based on DTable */
-#ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
-#endif
+size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
 #ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
+size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);   /*< double-symbols decoder */
 #endif
 
 /* BMI2 variants.
  * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
  */
-size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
+size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags);
 #ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
+size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
 #endif
-size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
-size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
+size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags);
+size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags);
 #ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2);
+size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags);
 #endif
 #ifndef HUF_FORCE_DECOMPRESS_X1
-size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2);
+size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags);
 #endif
 
-#endif /* HUF_STATIC_LINKING_ONLY */
+#endif   /* HUF_H_298734234 */
 
diff --git a/lib/zstd/common/mem.h b/lib/zstd/common/mem.h
index 1d9cc03924ca..a7231822b6e3 100644
--- a/lib/zstd/common/mem.h
+++ b/lib/zstd/common/mem.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/zstd/common/portability_macros.h b/lib/zstd/common/portability_macros.h
index 0e3b2c0a527d..7ede8cf1ffe5 100644
--- a/lib/zstd/common/portability_macros.h
+++ b/lib/zstd/common/portability_macros.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -12,7 +13,7 @@
 #define ZSTD_PORTABILITY_MACROS_H
 
 /*
- * This header file contains macro defintions to support portability.
+ * This header file contains macro definitions to support portability.
  * This header is shared between C and ASM code, so it MUST only
  * contain macro definitions. It MUST not contain any C code.
  *
@@ -65,7 +66,7 @@
 #endif
 
 /*
- * Only enable assembly for GNUC comptabile compilers,
+ * Only enable assembly for GNUC compatible compilers,
  * because other platforms may not support GAS assembly syntax.
  *
  * Only enable assembly for Linux / MacOS, other platforms may
@@ -90,4 +91,23 @@
  */
 #define ZSTD_ENABLE_ASM_X86_64_BMI2 0
 
+/*
+ * For x86 ELF targets, add .note.gnu.property section for Intel CET in
+ * assembly sources when CET is enabled.
+ *
+ * Additionally, any function that may be called indirectly must begin
+ * with ZSTD_CET_ENDBRANCH.
+ */
+#if defined(__ELF__) && (defined(__x86_64__) || defined(__i386__)) \
+    && defined(__has_include)
+# if __has_include(<cet.h>)
+#  include <cet.h>
+#  define ZSTD_CET_ENDBRANCH _CET_ENDBR
+# endif
+#endif
+
+#ifndef ZSTD_CET_ENDBRANCH
+# define ZSTD_CET_ENDBRANCH
+#endif
+
 #endif /* ZSTD_PORTABILITY_MACROS_H */
diff --git a/lib/zstd/common/zstd_common.c b/lib/zstd/common/zstd_common.c
index 3d7e35b309b5..44b95b25344a 100644
--- a/lib/zstd/common/zstd_common.c
+++ b/lib/zstd/common/zstd_common.c
@@ -1,5 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -14,7 +15,6 @@
 *  Dependencies
 ***************************************/
 #define ZSTD_DEPS_NEED_MALLOC
-#include "zstd_deps.h"   /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */
 #include "error_private.h"
 #include "zstd_internal.h"
 
@@ -47,37 +47,3 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
 /*! ZSTD_getErrorString() :
  *  provides error code string from enum */
 const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); }
-
-
-
-/*=**************************************************************
-*  Custom allocator
-****************************************************************/
-void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem)
-{
-    if (customMem.customAlloc)
-        return customMem.customAlloc(customMem.opaque, size);
-    return ZSTD_malloc(size);
-}
-
-void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem)
-{
-    if (customMem.customAlloc) {
-        /* calloc implemented as malloc+memset;
-         * not as efficient as calloc, but next best guess for custom malloc */
-        void* const ptr = customMem.customAlloc(customMem.opaque, size);
-        ZSTD_memset(ptr, 0, size);
-        return ptr;
-    }
-    return ZSTD_calloc(1, size);
-}
-
-void ZSTD_customFree(void* ptr, ZSTD_customMem customMem)
-{
-    if (ptr!=NULL) {
-        if (customMem.customFree)
-            customMem.customFree(customMem.opaque, ptr);
-        else
-            ZSTD_free(ptr);
-    }
-}
diff --git a/lib/zstd/common/zstd_deps.h b/lib/zstd/common/zstd_deps.h
index 2c34e8a33a1c..f931f7d0e294 100644
--- a/lib/zstd/common/zstd_deps.h
+++ b/lib/zstd/common/zstd_deps.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -105,3 +105,17 @@ static uint64_t ZSTD_div64(uint64_t dividend, uint32_t divisor) {
 
 #endif /* ZSTD_DEPS_IO */
 #endif /* ZSTD_DEPS_NEED_IO */
+
+/*
+ * Only requested when MSAN is enabled.
+ * Need:
+ * intptr_t
+ */
+#ifdef ZSTD_DEPS_NEED_STDINT
+#ifndef ZSTD_DEPS_STDINT
+#define ZSTD_DEPS_STDINT
+
+/* intptr_t already provided by ZSTD_DEPS_COMMON */
+
+#endif /* ZSTD_DEPS_STDINT */
+#endif /* ZSTD_DEPS_NEED_STDINT */
diff --git a/lib/zstd/common/zstd_internal.h b/lib/zstd/common/zstd_internal.h
index 93305d9b41bb..7f023e4d4774 100644
--- a/lib/zstd/common/zstd_internal.h
+++ b/lib/zstd/common/zstd_internal.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -28,7 +29,6 @@
 #include <linux/zstd.h>
 #define FSE_STATIC_LINKING_ONLY
 #include "fse.h"
-#define HUF_STATIC_LINKING_ONLY
 #include "huf.h"
 #include <linux/xxhash.h>                /* XXH_reset, update, digest */
 #define ZSTD_TRACE 0
@@ -83,9 +83,9 @@ typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
 #define ZSTD_FRAMECHECKSUMSIZE 4
 
 #define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
-#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */)   /* for a non-null block */
+#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */)   /* for a non-null block */
+#define MIN_LITERALS_FOR_4_STREAMS 6
 
-#define HufLog 12
 typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e;
 
 #define LONGNBSEQ 0x7F00
@@ -93,6 +93,7 @@ typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingTy
 #define MINMATCH 3
 
 #define Litbits  8
+#define LitHufLog 11
 #define MaxLit ((1<<Litbits) - 1)
 #define MaxML   52
 #define MaxLL   35
@@ -103,6 +104,8 @@ typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingTy
 #define LLFSELog    9
 #define OffFSELog   8
 #define MaxFSELog  MAX(MAX(MLFSELog, LLFSELog), OffFSELog)
+#define MaxMLBits 16
+#define MaxLLBits 16
 
 #define ZSTD_MAX_HUF_HEADER_SIZE 128 /* header + <= 127 byte tree description */
 /* Each table cannot take more than #symbols * FSELog bits */
@@ -225,12 +228,6 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e
          * one COPY16() in the first call. Then, do two calls per loop since
          * at that point it is more likely to have a high trip count.
          */
-#ifdef __aarch64__
-        do {
-            COPY16(op, ip);
-        }
-        while (op < oend);
-#else
         ZSTD_copy16(op, ip);
         if (16 >= length) return;
         op += 16;
@@ -240,7 +237,6 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e
             COPY16(op, ip);
         }
         while (op < oend);
-#endif
     }
 }
 
@@ -289,11 +285,11 @@ typedef enum {
 typedef struct {
     seqDef* sequencesStart;
     seqDef* sequences;      /* ptr to end of sequences */
-    BYTE* litStart;
-    BYTE* lit;              /* ptr to end of literals */
-    BYTE* llCode;
-    BYTE* mlCode;
-    BYTE* ofCode;
+    BYTE*  litStart;
+    BYTE*  lit;             /* ptr to end of literals */
+    BYTE*  llCode;
+    BYTE*  mlCode;
+    BYTE*  ofCode;
     size_t maxNbSeq;
     size_t maxNbLit;
 
@@ -301,8 +297,8 @@ typedef struct {
      * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment
      * the existing value of the litLength or matchLength by 0x10000.
      */
-    ZSTD_longLengthType_e   longLengthType;
-    U32                     longLengthPos;  /* Index of the sequence to apply long length modification to */
+    ZSTD_longLengthType_e longLengthType;
+    U32                   longLengthPos;  /* Index of the sequence to apply long length modification to */
 } seqStore_t;
 
 typedef struct {
@@ -321,10 +317,10 @@ MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore
     seqLen.matchLength = seq->mlBase + MINMATCH;
     if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) {
         if (seqStore->longLengthType == ZSTD_llt_literalLength) {
-            seqLen.litLength += 0xFFFF;
+            seqLen.litLength += 0x10000;
         }
         if (seqStore->longLengthType == ZSTD_llt_matchLength) {
-            seqLen.matchLength += 0xFFFF;
+            seqLen.matchLength += 0x10000;
         }
     }
     return seqLen;
@@ -337,72 +333,13 @@ MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore
  *          `decompressedBound != ZSTD_CONTENTSIZE_ERROR`
  */
 typedef struct {
+    size_t nbBlocks;
     size_t compressedSize;
     unsigned long long decompressedBound;
 } ZSTD_frameSizeInfo;   /* decompress & legacy */
 
 const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);   /* compress & dictBuilder */
-void ZSTD_seqToCodes(const seqStore_t* seqStorePtr);   /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
-
-/* custom memory allocation functions */
-void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem);
-void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem);
-void ZSTD_customFree(void* ptr, ZSTD_customMem customMem);
-
-
-MEM_STATIC U32 ZSTD_highbit32(U32 val)   /* compress, dictBuilder, decodeCorpus */
-{
-    assert(val != 0);
-    {
-#   if (__GNUC__ >= 3)   /* GCC Intrinsic */
-        return __builtin_clz (val) ^ 31;
-#   else   /* Software version */
-        static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
-        U32 v = val;
-        v |= v >> 1;
-        v |= v >> 2;
-        v |= v >> 4;
-        v |= v >> 8;
-        v |= v >> 16;
-        return DeBruijnClz[(v * 0x07C4ACDDU) >> 27];
-#   endif
-    }
-}
-
-/*
- * Counts the number of trailing zeros of a `size_t`.
- * Most compilers should support CTZ as a builtin. A backup
- * implementation is provided if the builtin isn't supported, but
- * it may not be terribly efficient.
- */
-MEM_STATIC unsigned ZSTD_countTrailingZeros(size_t val)
-{
-    if (MEM_64bits()) {
-#       if (__GNUC__ >= 4)
-            return __builtin_ctzll((U64)val);
-#       else
-            static const int DeBruijnBytePos[64] = {  0,  1,  2,  7,  3, 13,  8, 19,
-                                                      4, 25, 14, 28,  9, 34, 20, 56,
-                                                      5, 17, 26, 54, 15, 41, 29, 43,
-                                                      10, 31, 38, 35, 21, 45, 49, 57,
-                                                      63,  6, 12, 18, 24, 27, 33, 55,
-                                                      16, 53, 40, 42, 30, 37, 44, 48,
-                                                      62, 11, 23, 32, 52, 39, 36, 47,
-                                                      61, 22, 51, 46, 60, 50, 59, 58 };
-            return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
-#       endif
-    } else { /* 32 bits */
-#       if (__GNUC__ >= 3)
-            return __builtin_ctz((U32)val);
-#       else
-            static const int DeBruijnBytePos[32] = {  0,  1, 28,  2, 29, 14, 24,  3,
-                                                     30, 22, 20, 15, 25, 17,  4,  8,
-                                                     31, 27, 13, 23, 21, 19, 16,  7,
-                                                     26, 12, 18,  6, 11,  5, 10,  9 };
-            return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
-#       endif
-    }
-}
+int ZSTD_seqToCodes(const seqStore_t* seqStorePtr);   /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */
 
 
 /* ZSTD_invalidateRepCodes() :
diff --git a/lib/zstd/compress/clevels.h b/lib/zstd/compress/clevels.h
index d9a76112ec3a..6ab8be6532ef 100644
--- a/lib/zstd/compress/clevels.h
+++ b/lib/zstd/compress/clevels.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/zstd/compress/fse_compress.c b/lib/zstd/compress/fse_compress.c
index ec5b1ca6d71a..e46ca6621b48 100644
--- a/lib/zstd/compress/fse_compress.c
+++ b/lib/zstd/compress/fse_compress.c
@@ -1,6 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /* ******************************************************************
  * FSE : Finite State Entropy encoder
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -26,6 +27,7 @@
 #define ZSTD_DEPS_NEED_MALLOC
 #define ZSTD_DEPS_NEED_MATH64
 #include "../common/zstd_deps.h"  /* ZSTD_malloc, ZSTD_free, ZSTD_memcpy, ZSTD_memset */
+#include "../common/bits.h" /* ZSTD_highbit32 */
 
 
 /* **************************************************************
@@ -90,7 +92,7 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
     assert(tableLog < 16);   /* required for threshold strategy to work */
 
     /* For explanations on how to distribute symbol values over the table :
-     * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
+     * https://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
 
      #ifdef __clang_analyzer__
      ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize);   /* useless initialization, just to keep scan-build happy */
@@ -191,7 +193,7 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct,
                 break;
             default :
                 assert(normalizedCounter[s] > 1);
-                {   U32 const maxBitsOut = tableLog - BIT_highbit32 ((U32)normalizedCounter[s]-1);
+                {   U32 const maxBitsOut = tableLog - ZSTD_highbit32 ((U32)normalizedCounter[s]-1);
                     U32 const minStatePlus = (U32)normalizedCounter[s] << maxBitsOut;
                     symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
                     symbolTT[s].deltaFindState = (int)(total - (unsigned)normalizedCounter[s]);
@@ -342,21 +344,11 @@ size_t FSE_writeNCount (void* buffer, size_t bufferSize,
 *  FSE Compression Code
 ****************************************************************/
 
-FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog)
-{
-    size_t size;
-    if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
-    size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
-    return (FSE_CTable*)ZSTD_malloc(size);
-}
-
-void FSE_freeCTable (FSE_CTable* ct) { ZSTD_free(ct); }
-
 /* provides the minimum logSize to safely represent a distribution */
 static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
 {
-    U32 minBitsSrc = BIT_highbit32((U32)(srcSize)) + 1;
-    U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2;
+    U32 minBitsSrc = ZSTD_highbit32((U32)(srcSize)) + 1;
+    U32 minBitsSymbols = ZSTD_highbit32(maxSymbolValue) + 2;
     U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
     assert(srcSize > 1); /* Not supported, RLE should be used instead */
     return minBits;
@@ -364,7 +356,7 @@ static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
 
 unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus)
 {
-    U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus;
+    U32 maxBitsSrc = ZSTD_highbit32((U32)(srcSize - 1)) - minus;
     U32 tableLog = maxTableLog;
     U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
     assert(srcSize > 1); /* Not supported, RLE should be used instead */
@@ -532,40 +524,6 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
     return tableLog;
 }
 
-
-/* fake FSE_CTable, for raw (uncompressed) input */
-size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
-{
-    const unsigned tableSize = 1 << nbBits;
-    const unsigned tableMask = tableSize - 1;
-    const unsigned maxSymbolValue = tableMask;
-    void* const ptr = ct;
-    U16* const tableU16 = ( (U16*) ptr) + 2;
-    void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableSize>>1);   /* assumption : tableLog >= 1 */
-    FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
-    unsigned s;
-
-    /* Sanity checks */
-    if (nbBits < 1) return ERROR(GENERIC);             /* min size */
-
-    /* header */
-    tableU16[-2] = (U16) nbBits;
-    tableU16[-1] = (U16) maxSymbolValue;
-
-    /* Build table */
-    for (s=0; s<tableSize; s++)
-        tableU16[s] = (U16)(tableSize + s);
-
-    /* Build Symbol Transformation Table */
-    {   const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits);
-        for (s=0; s<=maxSymbolValue; s++) {
-            symbolTT[s].deltaNbBits = deltaNbBits;
-            symbolTT[s].deltaFindState = s-1;
-    }   }
-
-    return 0;
-}
-
 /* fake FSE_CTable, for rle input (always same symbol) */
 size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue)
 {
@@ -664,5 +622,4 @@ size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
 
 size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
 
-
 #endif   /* FSE_COMMONDEFS_ONLY */
diff --git a/lib/zstd/compress/hist.c b/lib/zstd/compress/hist.c
index 3ddc6dfb6894..0b12587cc14b 100644
--- a/lib/zstd/compress/hist.c
+++ b/lib/zstd/compress/hist.c
@@ -1,7 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /* ******************************************************************
  * hist : Histogram functions
  * part of Finite State Entropy project
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/zstd/compress/hist.h b/lib/zstd/compress/hist.h
index fc1830abc9c6..f7687b0fc20a 100644
--- a/lib/zstd/compress/hist.h
+++ b/lib/zstd/compress/hist.h
@@ -1,7 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /* ******************************************************************
  * hist : Histogram functions
  * part of Finite State Entropy project
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
diff --git a/lib/zstd/compress/huf_compress.c b/lib/zstd/compress/huf_compress.c
index 74ef0db47621..83241abafe35 100644
--- a/lib/zstd/compress/huf_compress.c
+++ b/lib/zstd/compress/huf_compress.c
@@ -1,6 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /* ******************************************************************
  * Huffman encoder, part of New Generation Entropy library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -26,9 +27,9 @@
 #include "hist.h"
 #define FSE_STATIC_LINKING_ONLY   /* FSE_optimalTableLog_internal */
 #include "../common/fse.h"        /* header compression */
-#define HUF_STATIC_LINKING_ONLY
 #include "../common/huf.h"
 #include "../common/error_private.h"
+#include "../common/bits.h"       /* ZSTD_highbit32 */
 
 
 /* **************************************************************
@@ -39,13 +40,67 @@
 
 
 /* **************************************************************
-*  Utils
+*  Required declarations
 ****************************************************************/
-unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
+typedef struct nodeElt_s {
+    U32 count;
+    U16 parent;
+    BYTE byte;
+    BYTE nbBits;
+} nodeElt;
+
+
+/* **************************************************************
+*  Debug Traces
+****************************************************************/
+
+#if DEBUGLEVEL >= 2
+
+static size_t showU32(const U32* arr, size_t size)
+{
+    size_t u;
+    for (u=0; u<size; u++) {
+        RAWLOG(6, " %u", arr[u]); (void)arr;
+    }
+    RAWLOG(6, " \n");
+    return size;
+}
+
+static size_t HUF_getNbBits(HUF_CElt elt);
+
+static size_t showCTableBits(const HUF_CElt* ctable, size_t size)
 {
-    return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
+    size_t u;
+    for (u=0; u<size; u++) {
+        RAWLOG(6, " %zu", HUF_getNbBits(ctable[u])); (void)ctable;
+    }
+    RAWLOG(6, " \n");
+    return size;
+
 }
 
+static size_t showHNodeSymbols(const nodeElt* hnode, size_t size)
+{
+    size_t u;
+    for (u=0; u<size; u++) {
+        RAWLOG(6, " %u", hnode[u].byte); (void)hnode;
+    }
+    RAWLOG(6, " \n");
+    return size;
+}
+
+static size_t showHNodeBits(const nodeElt* hnode, size_t size)
+{
+    size_t u;
+    for (u=0; u<size; u++) {
+        RAWLOG(6, " %u", hnode[u].nbBits); (void)hnode;
+    }
+    RAWLOG(6, " \n");
+    return size;
+}
+
+#endif
+
 
 /* *******************************************************
 *  HUF : Huffman block compression
@@ -86,7 +141,10 @@ typedef struct {
     S16 norm[HUF_TABLELOG_MAX+1];
 } HUF_CompressWeightsWksp;
 
-static size_t HUF_compressWeights(void* dst, size_t dstSize, const void* weightTable, size_t wtSize, void* workspace, size_t workspaceSize)
+static size_t
+HUF_compressWeights(void* dst, size_t dstSize,
+              const void* weightTable, size_t wtSize,
+                    void* workspace, size_t workspaceSize)
 {
     BYTE* const ostart = (BYTE*) dst;
     BYTE* op = ostart;
@@ -137,7 +195,7 @@ static size_t HUF_getNbBitsFast(HUF_CElt elt)
 
 static size_t HUF_getValue(HUF_CElt elt)
 {
-    return elt & ~0xFF;
+    return elt & ~(size_t)0xFF;
 }
 
 static size_t HUF_getValueFast(HUF_CElt elt)
@@ -175,6 +233,8 @@ size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize,
     U32 n;
     HUF_WriteCTableWksp* wksp = (HUF_WriteCTableWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32));
 
+    HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE >= sizeof(HUF_WriteCTableWksp));
+
     /* check conditions */
     if (workspaceSize < sizeof(HUF_WriteCTableWksp)) return ERROR(GENERIC);
     if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge);
@@ -204,16 +264,6 @@ size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize,
     return ((maxSymbolValue+1)/2) + 1;
 }
 
-/*! HUF_writeCTable() :
-    `CTable` : Huffman tree to save, using huf representation.
-    @return : size of saved CTable */
-size_t HUF_writeCTable (void* dst, size_t maxDstSize,
-                        const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog)
-{
-    HUF_WriteCTableWksp wksp;
-    return HUF_writeCTable_wksp(dst, maxDstSize, CTable, maxSymbolValue, huffLog, &wksp, sizeof(wksp));
-}
-
 
 size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* hasZeroWeights)
 {
@@ -269,68 +319,64 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void
 
 U32 HUF_getNbBitsFromCTable(HUF_CElt const* CTable, U32 symbolValue)
 {
-    const HUF_CElt* ct = CTable + 1;
+    const HUF_CElt* const ct = CTable + 1;
     assert(symbolValue <= HUF_SYMBOLVALUE_MAX);
     return (U32)HUF_getNbBits(ct[symbolValue]);
 }
 
 
-typedef struct nodeElt_s {
-    U32 count;
-    U16 parent;
-    BYTE byte;
-    BYTE nbBits;
-} nodeElt;
-
 /*
  * HUF_setMaxHeight():
- * Enforces maxNbBits on the Huffman tree described in huffNode.
+ * Try to enforce @targetNbBits on the Huffman tree described in @huffNode.
  *
- * It sets all nodes with nbBits > maxNbBits to be maxNbBits. Then it adjusts
- * the tree to so that it is a valid canonical Huffman tree.
+ * It attempts to convert all nodes with nbBits > @targetNbBits
+ * to employ @targetNbBits instead. Then it adjusts the tree
+ * so that it remains a valid canonical Huffman tree.
  *
  * @pre               The sum of the ranks of each symbol == 2^largestBits,
  *                    where largestBits == huffNode[lastNonNull].nbBits.
  * @post              The sum of the ranks of each symbol == 2^largestBits,
- *                    where largestBits is the return value <= maxNbBits.
+ *                    where largestBits is the return value (expected <= targetNbBits).
  *
- * @param huffNode    The Huffman tree modified in place to enforce maxNbBits.
+ * @param huffNode    The Huffman tree modified in place to enforce targetNbBits.
+ *                    It's presumed sorted, from most frequent to rarest symbol.
  * @param lastNonNull The symbol with the lowest count in the Huffman tree.
- * @param maxNbBits   The maximum allowed number of bits, which the Huffman tree
+ * @param targetNbBits  The allowed number of bits, which the Huffman tree
  *                    may not respect. After this function the Huffman tree will
- *                    respect maxNbBits.
- * @return            The maximum number of bits of the Huffman tree after adjustment,
- *                    necessarily no more than maxNbBits.
+ *                    respect targetNbBits.
+ * @return            The maximum number of bits of the Huffman tree after adjustment.
  */
-static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
+static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 targetNbBits)
 {
     const U32 largestBits = huffNode[lastNonNull].nbBits;
-    /* early exit : no elt > maxNbBits, so the tree is already valid. */
-    if (largestBits <= maxNbBits) return largestBits;
+    /* early exit : no elt > targetNbBits, so the tree is already valid. */
+    if (largestBits <= targetNbBits) return largestBits;
+
+    DEBUGLOG(5, "HUF_setMaxHeight (targetNbBits = %u)", targetNbBits);
 
     /* there are several too large elements (at least >= 2) */
     {   int totalCost = 0;
-        const U32 baseCost = 1 << (largestBits - maxNbBits);
+        const U32 baseCost = 1 << (largestBits - targetNbBits);
         int n = (int)lastNonNull;
 
-        /* Adjust any ranks > maxNbBits to maxNbBits.
+        /* Adjust any ranks > targetNbBits to targetNbBits.
          * Compute totalCost, which is how far the sum of the ranks is
          * we are over 2^largestBits after adjust the offending ranks.
          */
-        while (huffNode[n].nbBits > maxNbBits) {
+        while (huffNode[n].nbBits > targetNbBits) {
             totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits));
-            huffNode[n].nbBits = (BYTE)maxNbBits;
+            huffNode[n].nbBits = (BYTE)targetNbBits;
             n--;
         }
-        /* n stops at huffNode[n].nbBits <= maxNbBits */
-        assert(huffNode[n].nbBits <= maxNbBits);
-        /* n end at index of smallest symbol using < maxNbBits */
-        while (huffNode[n].nbBits == maxNbBits) --n;
+        /* n stops at huffNode[n].nbBits <= targetNbBits */
+        assert(huffNode[n].nbBits <= targetNbBits);
+        /* n end at index of smallest symbol using < targetNbBits */
+        while (huffNode[n].nbBits == targetNbBits) --n;
 
-        /* renorm totalCost from 2^largestBits to 2^maxNbBits
+        /* renorm totalCost from 2^largestBits to 2^targetNbBits
          * note : totalCost is necessarily a multiple of baseCost */
-        assert((totalCost & (baseCost - 1)) == 0);
-        totalCost >>= (largestBits - maxNbBits);
+        assert(((U32)totalCost & (baseCost - 1)) == 0);
+        totalCost >>= (largestBits - targetNbBits);
         assert(totalCost > 0);
 
         /* repay normalized cost */
@@ -339,19 +385,19 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
 
             /* Get pos of last (smallest = lowest cum. count) symbol per rank */
             ZSTD_memset(rankLast, 0xF0, sizeof(rankLast));
-            {   U32 currentNbBits = maxNbBits;
+            {   U32 currentNbBits = targetNbBits;
                 int pos;
                 for (pos=n ; pos >= 0; pos--) {
                     if (huffNode[pos].nbBits >= currentNbBits) continue;
-                    currentNbBits = huffNode[pos].nbBits;   /* < maxNbBits */
-                    rankLast[maxNbBits-currentNbBits] = (U32)pos;
+                    currentNbBits = huffNode[pos].nbBits;   /* < targetNbBits */
+                    rankLast[targetNbBits-currentNbBits] = (U32)pos;
             }   }
 
             while (totalCost > 0) {
                 /* Try to reduce the next power of 2 above totalCost because we
                  * gain back half the rank.
                  */
-                U32 nBitsToDecrease = BIT_highbit32((U32)totalCost) + 1;
+                U32 nBitsToDecrease = ZSTD_highbit32((U32)totalCost) + 1;
                 for ( ; nBitsToDecrease > 1; nBitsToDecrease--) {
                     U32 const highPos = rankLast[nBitsToDecrease];
                     U32 const lowPos = rankLast[nBitsToDecrease-1];
@@ -391,7 +437,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
                     rankLast[nBitsToDecrease] = noSymbol;
                 else {
                     rankLast[nBitsToDecrease]--;
-                    if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease)
+                    if (huffNode[rankLast[nBitsToDecrease]].nbBits != targetNbBits-nBitsToDecrease)
                         rankLast[nBitsToDecrease] = noSymbol;   /* this rank is now empty */
                 }
             }   /* while (totalCost > 0) */
@@ -403,11 +449,11 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
              * TODO.
              */
             while (totalCost < 0) {  /* Sometimes, cost correction overshoot */
-                /* special case : no rank 1 symbol (using maxNbBits-1);
-                 * let's create one from largest rank 0 (using maxNbBits).
+                /* special case : no rank 1 symbol (using targetNbBits-1);
+                 * let's create one from largest rank 0 (using targetNbBits).
                  */
                 if (rankLast[1] == noSymbol) {
-                    while (huffNode[n].nbBits == maxNbBits) n--;
+                    while (huffNode[n].nbBits == targetNbBits) n--;
                     huffNode[n+1].nbBits--;
                     assert(n >= 0);
                     rankLast[1] = (U32)(n+1);
@@ -421,7 +467,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
         }   /* repay normalized cost */
     }   /* there are several too large elements (at least >= 2) */
 
-    return maxNbBits;
+    return targetNbBits;
 }
 
 typedef struct {
@@ -429,7 +475,7 @@ typedef struct {
     U16 curr;
 } rankPos;
 
-typedef nodeElt huffNodeTable[HUF_CTABLE_WORKSPACE_SIZE_U32];
+typedef nodeElt huffNodeTable[2 * (HUF_SYMBOLVALUE_MAX + 1)];
 
 /* Number of buckets available for HUF_sort() */
 #define RANK_POSITION_TABLE_SIZE 192
@@ -448,8 +494,8 @@ typedef struct {
  * Let buckets 166 to 192 represent all remaining counts up to RANK_POSITION_MAX_COUNT_LOG using log2 bucketing.
  */
 #define RANK_POSITION_MAX_COUNT_LOG 32
-#define RANK_POSITION_LOG_BUCKETS_BEGIN (RANK_POSITION_TABLE_SIZE - 1) - RANK_POSITION_MAX_COUNT_LOG - 1 /* == 158 */
-#define RANK_POSITION_DISTINCT_COUNT_CUTOFF RANK_POSITION_LOG_BUCKETS_BEGIN + BIT_highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN) /* == 166 */
+#define RANK_POSITION_LOG_BUCKETS_BEGIN ((RANK_POSITION_TABLE_SIZE - 1) - RANK_POSITION_MAX_COUNT_LOG - 1 /* == 158 */)
+#define RANK_POSITION_DISTINCT_COUNT_CUTOFF (RANK_POSITION_LOG_BUCKETS_BEGIN + ZSTD_highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN) /* == 166 */)
 
 /* Return the appropriate bucket index for a given count. See definition of
  * RANK_POSITION_DISTINCT_COUNT_CUTOFF for explanation of bucketing strategy.
@@ -457,7 +503,7 @@ typedef struct {
 static U32 HUF_getIndex(U32 const count) {
     return (count < RANK_POSITION_DISTINCT_COUNT_CUTOFF)
         ? count
-        : BIT_highbit32(count) + RANK_POSITION_LOG_BUCKETS_BEGIN;
+        : ZSTD_highbit32(count) + RANK_POSITION_LOG_BUCKETS_BEGIN;
 }
 
 /* Helper swap function for HUF_quickSortPartition() */
@@ -580,7 +626,7 @@ static void HUF_sort(nodeElt huffNode[], const unsigned count[], U32 const maxSy
 
     /* Sort each bucket. */
     for (n = RANK_POSITION_DISTINCT_COUNT_CUTOFF; n < RANK_POSITION_TABLE_SIZE - 1; ++n) {
-        U32 const bucketSize = rankPosition[n].curr-rankPosition[n].base;
+        int const bucketSize = rankPosition[n].curr - rankPosition[n].base;
         U32 const bucketStartIdx = rankPosition[n].base;
         if (bucketSize > 1) {
             assert(bucketStartIdx < maxSymbolValue1);
@@ -591,6 +637,7 @@ static void HUF_sort(nodeElt huffNode[], const unsigned count[], U32 const maxSy
     assert(HUF_isSorted(huffNode, maxSymbolValue1));
 }
 
+
 /* HUF_buildCTable_wksp() :
  *  Same as HUF_buildCTable(), but using externally allocated scratch buffer.
  *  `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as sizeof(HUF_buildCTable_wksp_tables).
@@ -611,6 +658,7 @@ static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue)
     int lowS, lowN;
     int nodeNb = STARTNODE;
     int n, nodeRoot;
+    DEBUGLOG(5, "HUF_buildTree (alphabet size = %u)", maxSymbolValue + 1);
     /* init for parents */
     nonNullRank = (int)maxSymbolValue;
     while(huffNode[nonNullRank].count == 0) nonNullRank--;
@@ -637,6 +685,8 @@ static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue)
     for (n=0; n<=nonNullRank; n++)
         huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
 
+    DEBUGLOG(6, "Initial distribution of bits completed (%zu sorted symbols)", showHNodeBits(huffNode, maxSymbolValue+1));
+
     return nonNullRank;
 }
 
@@ -674,28 +724,36 @@ static void HUF_buildCTableFromTree(HUF_CElt* CTable, nodeElt const* huffNode, i
     CTable[0] = maxNbBits;
 }
 
-size_t HUF_buildCTable_wksp (HUF_CElt* CTable, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize)
+size_t
+HUF_buildCTable_wksp(HUF_CElt* CTable, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
+                     void* workSpace, size_t wkspSize)
 {
-    HUF_buildCTable_wksp_tables* const wksp_tables = (HUF_buildCTable_wksp_tables*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(U32));
+    HUF_buildCTable_wksp_tables* const wksp_tables =
+        (HUF_buildCTable_wksp_tables*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(U32));
     nodeElt* const huffNode0 = wksp_tables->huffNodeTbl;
     nodeElt* const huffNode = huffNode0+1;
     int nonNullRank;
 
+    HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE == sizeof(HUF_buildCTable_wksp_tables));
+
+    DEBUGLOG(5, "HUF_buildCTable_wksp (alphabet size = %u)", maxSymbolValue+1);
+
     /* safety checks */
     if (wkspSize < sizeof(HUF_buildCTable_wksp_tables))
-      return ERROR(workSpace_tooSmall);
+        return ERROR(workSpace_tooSmall);
     if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT;
     if (maxSymbolValue > HUF_SYMBOLVALUE_MAX)
-      return ERROR(maxSymbolValue_tooLarge);
+        return ERROR(maxSymbolValue_tooLarge);
     ZSTD_memset(huffNode0, 0, sizeof(huffNodeTable));
 
     /* sort, decreasing order */
     HUF_sort(huffNode, count, maxSymbolValue, wksp_tables->rankPosition);
+    DEBUGLOG(6, "sorted symbols completed (%zu symbols)", showHNodeSymbols(huffNode, maxSymbolValue+1));
 
     /* build tree */
     nonNullRank = HUF_buildTree(huffNode, maxSymbolValue);
 
-    /* enforce maxTableLog */
+    /* determine and enforce maxTableLog */
     maxNbBits = HUF_setMaxHeight(huffNode, (U32)nonNullRank, maxNbBits);
     if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC);   /* check fit into table */
 
@@ -804,7 +862,7 @@ FORCE_INLINE_TEMPLATE void HUF_addBits(HUF_CStream_t* bitC, HUF_CElt elt, int id
 #if DEBUGLEVEL >= 1
     {
         size_t const nbBits = HUF_getNbBits(elt);
-        size_t const dirtyBits = nbBits == 0 ? 0 : BIT_highbit32((U32)nbBits) + 1;
+        size_t const dirtyBits = nbBits == 0 ? 0 : ZSTD_highbit32((U32)nbBits) + 1;
         (void)dirtyBits;
         /* Middle bits are 0. */
         assert(((elt >> dirtyBits) << (dirtyBits + nbBits)) == 0);
@@ -884,7 +942,7 @@ static size_t HUF_closeCStream(HUF_CStream_t* bitC)
     {
         size_t const nbBits = bitC->bitPos[0] & 0xFF;
         if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */
-        return (bitC->ptr - bitC->startPtr) + (nbBits > 0);
+        return (size_t)(bitC->ptr - bitC->startPtr) + (nbBits > 0);
     }
 }
 
@@ -1045,9 +1103,9 @@ HUF_compress1X_usingCTable_internal_default(void* dst, size_t dstSize,
 static size_t
 HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize,
                               const void* src, size_t srcSize,
-                              const HUF_CElt* CTable, const int bmi2)
+                              const HUF_CElt* CTable, const int flags)
 {
-    if (bmi2) {
+    if (flags & HUF_flags_bmi2) {
         return HUF_compress1X_usingCTable_internal_bmi2(dst, dstSize, src, srcSize, CTable);
     }
     return HUF_compress1X_usingCTable_internal_default(dst, dstSize, src, srcSize, CTable);
@@ -1058,28 +1116,23 @@ HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize,
 static size_t
 HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize,
                               const void* src, size_t srcSize,
-                              const HUF_CElt* CTable, const int bmi2)
+                              const HUF_CElt* CTable, const int flags)
 {
-    (void)bmi2;
+    (void)flags;
     return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable);
 }
 
 #endif
 
-size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
-{
-    return HUF_compress1X_usingCTable_bmi2(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0);
-}
-
-size_t HUF_compress1X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2)
+size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags)
 {
-    return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, bmi2);
+    return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags);
 }
 
 static size_t
 HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
                               const void* src, size_t srcSize,
-                              const HUF_CElt* CTable, int bmi2)
+                              const HUF_CElt* CTable, int flags)
 {
     size_t const segmentSize = (srcSize+3)/4;   /* first 3 segments */
     const BYTE* ip = (const BYTE*) src;
@@ -1093,7 +1146,7 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
     op += 6;   /* jumpTable */
 
     assert(op <= oend);
-    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) );
+    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) );
         if (cSize == 0 || cSize > 65535) return 0;
         MEM_writeLE16(ostart, (U16)cSize);
         op += cSize;
@@ -1101,7 +1154,7 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
 
     ip += segmentSize;
     assert(op <= oend);
-    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) );
+    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) );
         if (cSize == 0 || cSize > 65535) return 0;
         MEM_writeLE16(ostart+2, (U16)cSize);
         op += cSize;
@@ -1109,7 +1162,7 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
 
     ip += segmentSize;
     assert(op <= oend);
-    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) );
+    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) );
         if (cSize == 0 || cSize > 65535) return 0;
         MEM_writeLE16(ostart+4, (U16)cSize);
         op += cSize;
@@ -1118,7 +1171,7 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
     ip += segmentSize;
     assert(op <= oend);
     assert(ip <= iend);
-    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, bmi2) );
+    {   CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, flags) );
         if (cSize == 0 || cSize > 65535) return 0;
         op += cSize;
     }
@@ -1126,14 +1179,9 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize,
     return (size_t)(op-ostart);
 }
 
-size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
+size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags)
 {
-    return HUF_compress4X_usingCTable_bmi2(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0);
-}
-
-size_t HUF_compress4X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2)
-{
-    return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, bmi2);
+    return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags);
 }
 
 typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e;
@@ -1141,11 +1189,11 @@ typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e;
 static size_t HUF_compressCTable_internal(
                 BYTE* const ostart, BYTE* op, BYTE* const oend,
                 const void* src, size_t srcSize,
-                HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int bmi2)
+                HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int flags)
 {
     size_t const cSize = (nbStreams==HUF_singleStream) ?
-                         HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2) :
-                         HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2);
+                         HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags) :
+                         HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags);
     if (HUF_isError(cSize)) { return cSize; }
     if (cSize==0) { return 0; }   /* uncompressible */
     op += cSize;
@@ -1168,6 +1216,79 @@ typedef struct {
 #define SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE 4096
 #define SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO 10  /* Must be >= 2 */
 
+unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue)
+{
+    unsigned cardinality = 0;
+    unsigned i;
+
+    for (i = 0; i < maxSymbolValue + 1; i++) {
+        if (count[i] != 0) cardinality += 1;
+    }
+
+    return cardinality;
+}
+
+unsigned HUF_minTableLog(unsigned symbolCardinality)
+{
+    U32 minBitsSymbols = ZSTD_highbit32(symbolCardinality) + 1;
+    return minBitsSymbols;
+}
+
+unsigned HUF_optimalTableLog(
+            unsigned maxTableLog,
+            size_t srcSize,
+            unsigned maxSymbolValue,
+            void* workSpace, size_t wkspSize,
+            HUF_CElt* table,
+      const unsigned* count,
+            int flags)
+{
+    assert(srcSize > 1); /* Not supported, RLE should be used instead */
+    assert(wkspSize >= sizeof(HUF_buildCTable_wksp_tables));
+
+    if (!(flags & HUF_flags_optimalDepth)) {
+        /* cheap evaluation, based on FSE */
+        return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
+    }
+
+    {   BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp);
+        size_t dstSize = wkspSize - sizeof(HUF_WriteCTableWksp);
+        size_t maxBits, hSize, newSize;
+        const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue);
+        const unsigned minTableLog = HUF_minTableLog(symbolCardinality);
+        size_t optSize = ((size_t) ~0) - 1;
+        unsigned optLog = maxTableLog, optLogGuess;
+
+        DEBUGLOG(6, "HUF_optimalTableLog: probing huf depth (srcSize=%zu)", srcSize);
+
+        /* Search until size increases */
+        for (optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) {
+            DEBUGLOG(7, "checking for huffLog=%u", optLogGuess);
+            maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize);
+            if (ERR_isError(maxBits)) continue;
+
+            if (maxBits < optLogGuess && optLogGuess > minTableLog) break;
+
+            hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize);
+
+            if (ERR_isError(hSize)) continue;
+
+            newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize;
+
+            if (newSize > optSize + 1) {
+                break;
+            }
+
+            if (newSize < optSize) {
+                optSize = newSize;
+                optLog = optLogGuess;
+            }
+        }
+        assert(optLog <= HUF_TABLELOG_MAX);
+        return optLog;
+    }
+}
+
 /* HUF_compress_internal() :
  * `workSpace_align4` must be aligned on 4-bytes boundaries,
  * and occupies the same space as a table of HUF_WORKSPACE_SIZE_U64 unsigned */
@@ -1177,14 +1298,14 @@ HUF_compress_internal (void* dst, size_t dstSize,
                        unsigned maxSymbolValue, unsigned huffLog,
                        HUF_nbStreams_e nbStreams,
                        void* workSpace, size_t wkspSize,
-                       HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat,
-                 const int bmi2, unsigned suspectUncompressible)
+                       HUF_CElt* oldHufTable, HUF_repeat* repeat, int flags)
 {
     HUF_compress_tables_t* const table = (HUF_compress_tables_t*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(size_t));
     BYTE* const ostart = (BYTE*)dst;
     BYTE* const oend = ostart + dstSize;
     BYTE* op = ostart;
 
+    DEBUGLOG(5, "HUF_compress_internal (srcSize=%zu)", srcSize);
     HUF_STATIC_ASSERT(sizeof(*table) + HUF_WORKSPACE_MAX_ALIGNMENT <= HUF_WORKSPACE_SIZE);
 
     /* checks & inits */
@@ -1198,16 +1319,17 @@ HUF_compress_internal (void* dst, size_t dstSize,
     if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
 
     /* Heuristic : If old table is valid, use it for small inputs */
-    if (preferRepeat && repeat && *repeat == HUF_repeat_valid) {
+    if ((flags & HUF_flags_preferRepeat) && repeat && *repeat == HUF_repeat_valid) {
         return HUF_compressCTable_internal(ostart, op, oend,
                                            src, srcSize,
-                                           nbStreams, oldHufTable, bmi2);
+                                           nbStreams, oldHufTable, flags);
     }
 
     /* If uncompressible data is suspected, do a smaller sampling first */
     DEBUG_STATIC_ASSERT(SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO >= 2);
-    if (suspectUncompressible && srcSize >= (SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO)) {
+    if ((flags & HUF_flags_suspectUncompressible) && srcSize >= (SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO)) {
         size_t largestTotal = 0;
+        DEBUGLOG(5, "input suspected incompressible : sampling to check");
         {   unsigned maxSymbolValueBegin = maxSymbolValue;
             CHECK_V_F(largestBegin, HIST_count_simple (table->count, &maxSymbolValueBegin, (const BYTE*)src, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) );
             largestTotal += largestBegin;
@@ -1224,6 +1346,7 @@ HUF_compress_internal (void* dst, size_t dstSize,
         if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; }   /* single symbol, rle */
         if (largest <= (srcSize >> 7)+4) return 0;   /* heuristic : probably not compressible enough */
     }
+    DEBUGLOG(6, "histogram detail completed (%zu symbols)", showU32(table->count, maxSymbolValue+1));
 
     /* Check validity of previous table */
     if ( repeat
@@ -1232,19 +1355,20 @@ HUF_compress_internal (void* dst, size_t dstSize,
         *repeat = HUF_repeat_none;
     }
     /* Heuristic : use existing table for small inputs */
-    if (preferRepeat && repeat && *repeat != HUF_repeat_none) {
+    if ((flags & HUF_flags_preferRepeat) && repeat && *repeat != HUF_repeat_none) {
         return HUF_compressCTable_internal(ostart, op, oend,
                                            src, srcSize,
-                                           nbStreams, oldHufTable, bmi2);
+                                           nbStreams, oldHufTable, flags);
     }
 
     /* Build Huffman Tree */
-    huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
+    huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, &table->wksps, sizeof(table->wksps), table->CTable, table->count, flags);
     {   size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count,
                                             maxSymbolValue, huffLog,
                                             &table->wksps.buildCTable_wksp, sizeof(table->wksps.buildCTable_wksp));
         CHECK_F(maxBits);
         huffLog = (U32)maxBits;
+        DEBUGLOG(6, "bit distribution completed (%zu symbols)", showCTableBits(table->CTable + 1, maxSymbolValue+1));
     }
     /* Zero unused symbols in CTable, so we can check it for validity */
     {
@@ -1263,7 +1387,7 @@ HUF_compress_internal (void* dst, size_t dstSize,
             if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) {
                 return HUF_compressCTable_internal(ostart, op, oend,
                                                    src, srcSize,
-                                                   nbStreams, oldHufTable, bmi2);
+                                                   nbStreams, oldHufTable, flags);
         }   }
 
         /* Use the new huffman table */
@@ -1275,46 +1399,20 @@ HUF_compress_internal (void* dst, size_t dstSize,
     }
     return HUF_compressCTable_internal(ostart, op, oend,
                                        src, srcSize,
-                                       nbStreams, table->CTable, bmi2);
-}
-
-
-size_t HUF_compress1X_wksp (void* dst, size_t dstSize,
-                      const void* src, size_t srcSize,
-                      unsigned maxSymbolValue, unsigned huffLog,
-                      void* workSpace, size_t wkspSize)
-{
-    return HUF_compress_internal(dst, dstSize, src, srcSize,
-                                 maxSymbolValue, huffLog, HUF_singleStream,
-                                 workSpace, wkspSize,
-                                 NULL, NULL, 0, 0 /*bmi2*/, 0);
+                                       nbStreams, table->CTable, flags);
 }
 
 size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
                       const void* src, size_t srcSize,
                       unsigned maxSymbolValue, unsigned huffLog,
                       void* workSpace, size_t wkspSize,
-                      HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat,
-                      int bmi2, unsigned suspectUncompressible)
+                      HUF_CElt* hufTable, HUF_repeat* repeat, int flags)
 {
+    DEBUGLOG(5, "HUF_compress1X_repeat (srcSize = %zu)", srcSize);
     return HUF_compress_internal(dst, dstSize, src, srcSize,
                                  maxSymbolValue, huffLog, HUF_singleStream,
                                  workSpace, wkspSize, hufTable,
-                                 repeat, preferRepeat, bmi2, suspectUncompressible);
-}
-
-/* HUF_compress4X_repeat():
- * compress input using 4 streams.
- * provide workspace to generate compression tables */
-size_t HUF_compress4X_wksp (void* dst, size_t dstSize,
-                      const void* src, size_t srcSize,
-                      unsigned maxSymbolValue, unsigned huffLog,
-                      void* workSpace, size_t wkspSize)
-{
-    return HUF_compress_internal(dst, dstSize, src, srcSize,
-                                 maxSymbolValue, huffLog, HUF_fourStreams,
-                                 workSpace, wkspSize,
-                                 NULL, NULL, 0, 0 /*bmi2*/, 0);
+                                 repeat, flags);
 }
 
 /* HUF_compress4X_repeat():
@@ -1325,11 +1423,11 @@ size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
                       const void* src, size_t srcSize,
                       unsigned maxSymbolValue, unsigned huffLog,
                       void* workSpace, size_t wkspSize,
-                      HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible)
+                      HUF_CElt* hufTable, HUF_repeat* repeat, int flags)
 {
+    DEBUGLOG(5, "HUF_compress4X_repeat (srcSize = %zu)", srcSize);
     return HUF_compress_internal(dst, dstSize, src, srcSize,
                                  maxSymbolValue, huffLog, HUF_fourStreams,
                                  workSpace, wkspSize,
-                                 hufTable, repeat, preferRepeat, bmi2, suspectUncompressible);
+                                 hufTable, repeat, flags);
 }
-
diff --git a/lib/zstd/compress/zstd_compress.c b/lib/zstd/compress/zstd_compress.c
index f620cafca633..c1c316e9e289 100644
--- a/lib/zstd/compress/zstd_compress.c
+++ b/lib/zstd/compress/zstd_compress.c
@@ -1,5 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -11,12 +12,12 @@
 /*-*************************************
 *  Dependencies
 ***************************************/
+#include "../common/allocations.h"  /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */
 #include "../common/zstd_deps.h"  /* INT_MAX, ZSTD_memset, ZSTD_memcpy */
 #include "../common/mem.h"
 #include "hist.h"           /* HIST_countFast_wksp */
 #define FSE_STATIC_LINKING_ONLY   /* FSE_encodeSymbol */
 #include "../common/fse.h"
-#define HUF_STATIC_LINKING_ONLY
 #include "../common/huf.h"
 #include "zstd_compress_internal.h"
 #include "zstd_compress_sequences.h"
@@ -27,6 +28,7 @@
 #include "zstd_opt.h"
 #include "zstd_ldm.h"
 #include "zstd_compress_superblock.h"
+#include  "../common/bits.h"      /* ZSTD_highbit32, ZSTD_rotateRight_U64 */
 
 /* ***************************************************************
 *  Tuning parameters
@@ -55,14 +57,17 @@
 *  Helper functions
 ***************************************/
 /* ZSTD_compressBound()
- * Note that the result from this function is only compatible with the "normal"
- * full-block strategy.
- * When there are a lot of small blocks due to frequent flush in streaming mode
- * the overhead of headers can make the compressed data to be larger than the
- * return value of ZSTD_compressBound().
+ * Note that the result from this function is only valid for
+ * the one-pass compression functions.
+ * When employing the streaming mode,
+ * if flushes are frequently altering the size of blocks,
+ * the overhead from block headers can make the compressed data larger
+ * than the return value of ZSTD_compressBound().
  */
 size_t ZSTD_compressBound(size_t srcSize) {
-    return ZSTD_COMPRESSBOUND(srcSize);
+    size_t const r = ZSTD_COMPRESSBOUND(srcSize);
+    if (r==0) return ERROR(srcSize_wrong);
+    return r;
 }
 
 
@@ -171,12 +176,9 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
     if (cctx==NULL) return 0;   /* support free on NULL */
     RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
                     "not compatible with static CCtx");
-    {
-        int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx);
+    {   int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx);
         ZSTD_freeCCtxContent(cctx);
-        if (!cctxInWorkspace) {
-            ZSTD_customFree(cctx, cctx->customMem);
-        }
+        if (!cctxInWorkspace) ZSTD_customFree(cctx, cctx->customMem);
     }
     return 0;
 }
@@ -257,9 +259,9 @@ static int ZSTD_allocateChainTable(const ZSTD_strategy strategy,
     return forDDSDict || ((strategy != ZSTD_fast) && !ZSTD_rowMatchFinderUsed(strategy, useRowMatchFinder));
 }
 
-/* Returns 1 if compression parameters are such that we should
+/* Returns ZSTD_ps_enable if compression parameters are such that we should
  * enable long distance matching (wlog >= 27, strategy >= btopt).
- * Returns 0 otherwise.
+ * Returns ZSTD_ps_disable otherwise.
  */
 static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode,
                                  const ZSTD_compressionParameters* const cParams) {
@@ -267,6 +269,34 @@ static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode,
     return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27) ? ZSTD_ps_enable : ZSTD_ps_disable;
 }
 
+static int ZSTD_resolveExternalSequenceValidation(int mode) {
+    return mode;
+}
+
+/* Resolves maxBlockSize to the default if no value is present. */
+static size_t ZSTD_resolveMaxBlockSize(size_t maxBlockSize) {
+    if (maxBlockSize == 0) {
+        return ZSTD_BLOCKSIZE_MAX;
+    } else {
+        return maxBlockSize;
+    }
+}
+
+static ZSTD_paramSwitch_e ZSTD_resolveExternalRepcodeSearch(ZSTD_paramSwitch_e value, int cLevel) {
+    if (value != ZSTD_ps_auto) return value;
+    if (cLevel < 10) {
+        return ZSTD_ps_disable;
+    } else {
+        return ZSTD_ps_enable;
+    }
+}
+
+/* Returns 1 if compression parameters are such that CDict hashtable and chaintable indices are tagged.
+ * If so, the tags need to be removed in ZSTD_resetCCtx_byCopyingCDict. */
+static int ZSTD_CDictIndicesAreTagged(const ZSTD_compressionParameters* const cParams) {
+    return cParams->strategy == ZSTD_fast || cParams->strategy == ZSTD_dfast;
+}
+
 static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
         ZSTD_compressionParameters cParams)
 {
@@ -284,6 +314,10 @@ static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
     }
     cctxParams.useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.useBlockSplitter, &cParams);
     cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams);
+    cctxParams.validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams.validateSequences);
+    cctxParams.maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams.maxBlockSize);
+    cctxParams.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams.searchForExternalRepcodes,
+                                                                             cctxParams.compressionLevel);
     assert(!ZSTD_checkCParams(cParams));
     return cctxParams;
 }
@@ -329,10 +363,13 @@ size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel)
 #define ZSTD_NO_CLEVEL 0
 
 /*
- * Initializes the cctxParams from params and compressionLevel.
+ * Initializes `cctxParams` from `params` and `compressionLevel`.
  * @param compressionLevel If params are derived from a compression level then that compression level, otherwise ZSTD_NO_CLEVEL.
  */
-static void ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, ZSTD_parameters const* params, int compressionLevel)
+static void
+ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams,
+                        const ZSTD_parameters* params,
+                              int compressionLevel)
 {
     assert(!ZSTD_checkCParams(params->cParams));
     ZSTD_memset(cctxParams, 0, sizeof(*cctxParams));
@@ -345,6 +382,9 @@ static void ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, ZSTD_par
     cctxParams->useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams->useRowMatchFinder, &params->cParams);
     cctxParams->useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->useBlockSplitter, &params->cParams);
     cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, &params->cParams);
+    cctxParams->validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams->validateSequences);
+    cctxParams->maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams->maxBlockSize);
+    cctxParams->searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams->searchForExternalRepcodes, compressionLevel);
     DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d ldm=%d",
                 cctxParams->useRowMatchFinder, cctxParams->useBlockSplitter, cctxParams->ldmParams.enableLdm);
 }
@@ -359,7 +399,7 @@ size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_paramete
 
 /*
  * Sets cctxParams' cParams and fParams from params, but otherwise leaves them alone.
- * @param param Validated zstd parameters.
+ * @param params Validated zstd parameters.
  */
 static void ZSTD_CCtxParams_setZstdParams(
         ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params)
@@ -455,8 +495,8 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
         return bounds;
 
     case ZSTD_c_enableLongDistanceMatching:
-        bounds.lowerBound = 0;
-        bounds.upperBound = 1;
+        bounds.lowerBound = (int)ZSTD_ps_auto;
+        bounds.upperBound = (int)ZSTD_ps_disable;
         return bounds;
 
     case ZSTD_c_ldmHashLog:
@@ -549,6 +589,26 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
         bounds.upperBound = 1;
         return bounds;
 
+    case ZSTD_c_prefetchCDictTables:
+        bounds.lowerBound = (int)ZSTD_ps_auto;
+        bounds.upperBound = (int)ZSTD_ps_disable;
+        return bounds;
+
+    case ZSTD_c_enableSeqProducerFallback:
+        bounds.lowerBound = 0;
+        bounds.upperBound = 1;
+        return bounds;
+
+    case ZSTD_c_maxBlockSize:
+        bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN;
+        bounds.upperBound = ZSTD_BLOCKSIZE_MAX;
+        return bounds;
+
+    case ZSTD_c_searchForExternalRepcodes:
+        bounds.lowerBound = (int)ZSTD_ps_auto;
+        bounds.upperBound = (int)ZSTD_ps_disable;
+        return bounds;
+
     default:
         bounds.error = ERROR(parameter_unsupported);
         return bounds;
@@ -613,6 +673,10 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
     case ZSTD_c_useBlockSplitter:
     case ZSTD_c_useRowMatchFinder:
     case ZSTD_c_deterministicRefPrefix:
+    case ZSTD_c_prefetchCDictTables:
+    case ZSTD_c_enableSeqProducerFallback:
+    case ZSTD_c_maxBlockSize:
+    case ZSTD_c_searchForExternalRepcodes:
     default:
         return 0;
     }
@@ -625,7 +689,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
         if (ZSTD_isUpdateAuthorized(param)) {
             cctx->cParamsChanged = 1;
         } else {
-            RETURN_ERROR(stage_wrong, "can only set params in ctx init stage");
+            RETURN_ERROR(stage_wrong, "can only set params in cctx init stage");
     }   }
 
     switch(param)
@@ -668,6 +732,10 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
     case ZSTD_c_useBlockSplitter:
     case ZSTD_c_useRowMatchFinder:
     case ZSTD_c_deterministicRefPrefix:
+    case ZSTD_c_prefetchCDictTables:
+    case ZSTD_c_enableSeqProducerFallback:
+    case ZSTD_c_maxBlockSize:
+    case ZSTD_c_searchForExternalRepcodes:
         break;
 
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
@@ -723,12 +791,12 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
     case ZSTD_c_minMatch :
         if (value!=0)   /* 0 => use default */
             BOUNDCHECK(ZSTD_c_minMatch, value);
-        CCtxParams->cParams.minMatch = value;
+        CCtxParams->cParams.minMatch = (U32)value;
         return CCtxParams->cParams.minMatch;
 
     case ZSTD_c_targetLength :
         BOUNDCHECK(ZSTD_c_targetLength, value);
-        CCtxParams->cParams.targetLength = value;
+        CCtxParams->cParams.targetLength = (U32)value;
         return CCtxParams->cParams.targetLength;
 
     case ZSTD_c_strategy :
@@ -741,12 +809,12 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
         /* Content size written in frame header _when known_ (default:1) */
         DEBUGLOG(4, "set content size flag = %u", (value!=0));
         CCtxParams->fParams.contentSizeFlag = value != 0;
-        return CCtxParams->fParams.contentSizeFlag;
+        return (size_t)CCtxParams->fParams.contentSizeFlag;
 
     case ZSTD_c_checksumFlag :
         /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */
         CCtxParams->fParams.checksumFlag = value != 0;
-        return CCtxParams->fParams.checksumFlag;
+        return (size_t)CCtxParams->fParams.checksumFlag;
 
     case ZSTD_c_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */
         DEBUGLOG(4, "set dictIDFlag = %u", (value!=0));
@@ -755,18 +823,18 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
 
     case ZSTD_c_forceMaxWindow :
         CCtxParams->forceWindow = (value != 0);
-        return CCtxParams->forceWindow;
+        return (size_t)CCtxParams->forceWindow;
 
     case ZSTD_c_forceAttachDict : {
         const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value;
-        BOUNDCHECK(ZSTD_c_forceAttachDict, pref);
+        BOUNDCHECK(ZSTD_c_forceAttachDict, (int)pref);
         CCtxParams->attachDictPref = pref;
         return CCtxParams->attachDictPref;
     }
 
     case ZSTD_c_literalCompressionMode : {
         const ZSTD_paramSwitch_e lcm = (ZSTD_paramSwitch_e)value;
-        BOUNDCHECK(ZSTD_c_literalCompressionMode, lcm);
+        BOUNDCHECK(ZSTD_c_literalCompressionMode, (int)lcm);
         CCtxParams->literalCompressionMode = lcm;
         return CCtxParams->literalCompressionMode;
     }
@@ -789,47 +857,48 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
 
     case ZSTD_c_enableDedicatedDictSearch :
         CCtxParams->enableDedicatedDictSearch = (value!=0);
-        return CCtxParams->enableDedicatedDictSearch;
+        return (size_t)CCtxParams->enableDedicatedDictSearch;
 
     case ZSTD_c_enableLongDistanceMatching :
+        BOUNDCHECK(ZSTD_c_enableLongDistanceMatching, value);
         CCtxParams->ldmParams.enableLdm = (ZSTD_paramSwitch_e)value;
         return CCtxParams->ldmParams.enableLdm;
 
     case ZSTD_c_ldmHashLog :
         if (value!=0)   /* 0 ==> auto */
             BOUNDCHECK(ZSTD_c_ldmHashLog, value);
-        CCtxParams->ldmParams.hashLog = value;
+        CCtxParams->ldmParams.hashLog = (U32)value;
         return CCtxParams->ldmParams.hashLog;
 
     case ZSTD_c_ldmMinMatch :
         if (value!=0)   /* 0 ==> default */
             BOUNDCHECK(ZSTD_c_ldmMinMatch, value);
-        CCtxParams->ldmParams.minMatchLength = value;
+        CCtxParams->ldmParams.minMatchLength = (U32)value;
         return CCtxParams->ldmParams.minMatchLength;
 
     case ZSTD_c_ldmBucketSizeLog :
         if (value!=0)   /* 0 ==> default */
             BOUNDCHECK(ZSTD_c_ldmBucketSizeLog, value);
-        CCtxParams->ldmParams.bucketSizeLog = value;
+        CCtxParams->ldmParams.bucketSizeLog = (U32)value;
         return CCtxParams->ldmParams.bucketSizeLog;
 
     case ZSTD_c_ldmHashRateLog :
         if (value!=0)   /* 0 ==> default */
             BOUNDCHECK(ZSTD_c_ldmHashRateLog, value);
-        CCtxParams->ldmParams.hashRateLog = value;
+        CCtxParams->ldmParams.hashRateLog = (U32)value;
         return CCtxParams->ldmParams.hashRateLog;
 
     case ZSTD_c_targetCBlockSize :
         if (value!=0)   /* 0 ==> default */
             BOUNDCHECK(ZSTD_c_targetCBlockSize, value);
-        CCtxParams->targetCBlockSize = value;
+        CCtxParams->targetCBlockSize = (U32)value;
         return CCtxParams->targetCBlockSize;
 
     case ZSTD_c_srcSizeHint :
         if (value!=0)    /* 0 ==> default */
             BOUNDCHECK(ZSTD_c_srcSizeHint, value);
         CCtxParams->srcSizeHint = value;
-        return CCtxParams->srcSizeHint;
+        return (size_t)CCtxParams->srcSizeHint;
 
     case ZSTD_c_stableInBuffer:
         BOUNDCHECK(ZSTD_c_stableInBuffer, value);
@@ -866,6 +935,27 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
         CCtxParams->deterministicRefPrefix = !!value;
         return CCtxParams->deterministicRefPrefix;
 
+    case ZSTD_c_prefetchCDictTables:
+        BOUNDCHECK(ZSTD_c_prefetchCDictTables, value);
+        CCtxParams->prefetchCDictTables = (ZSTD_paramSwitch_e)value;
+        return CCtxParams->prefetchCDictTables;
+
+    case ZSTD_c_enableSeqProducerFallback:
+        BOUNDCHECK(ZSTD_c_enableSeqProducerFallback, value);
+        CCtxParams->enableMatchFinderFallback = value;
+        return CCtxParams->enableMatchFinderFallback;
+
+    case ZSTD_c_maxBlockSize:
+        if (value!=0)    /* 0 ==> default */
+            BOUNDCHECK(ZSTD_c_maxBlockSize, value);
+        CCtxParams->maxBlockSize = value;
+        return CCtxParams->maxBlockSize;
+
+    case ZSTD_c_searchForExternalRepcodes:
+        BOUNDCHECK(ZSTD_c_searchForExternalRepcodes, value);
+        CCtxParams->searchForExternalRepcodes = (ZSTD_paramSwitch_e)value;
+        return CCtxParams->searchForExternalRepcodes;
+
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
     }
 }
@@ -980,6 +1070,18 @@ size_t ZSTD_CCtxParams_getParameter(
     case ZSTD_c_deterministicRefPrefix:
         *value = (int)CCtxParams->deterministicRefPrefix;
         break;
+    case ZSTD_c_prefetchCDictTables:
+        *value = (int)CCtxParams->prefetchCDictTables;
+        break;
+    case ZSTD_c_enableSeqProducerFallback:
+        *value = CCtxParams->enableMatchFinderFallback;
+        break;
+    case ZSTD_c_maxBlockSize:
+        *value = (int)CCtxParams->maxBlockSize;
+        break;
+    case ZSTD_c_searchForExternalRepcodes:
+        *value = (int)CCtxParams->searchForExternalRepcodes;
+        break;
     default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
     }
     return 0;
@@ -1006,9 +1108,47 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams(
     return 0;
 }
 
+size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams)
+{
+    ZSTD_STATIC_ASSERT(sizeof(cparams) == 7 * 4 /* all params are listed below */);
+    DEBUGLOG(4, "ZSTD_CCtx_setCParams");
+    /* only update if all parameters are valid */
+    FORWARD_IF_ERROR(ZSTD_checkCParams(cparams), "");
+    FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, cparams.windowLog), "");
+    FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_chainLog, cparams.chainLog), "");
+    FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, cparams.hashLog), "");
+    FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_searchLog, cparams.searchLog), "");
+    FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, cparams.minMatch), "");
+    FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetLength, cparams.targetLength), "");
+    FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, cparams.strategy), "");
+    return 0;
+}
+
+size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams)
+{
+    ZSTD_STATIC_ASSERT(sizeof(fparams) == 3 * 4 /* all params are listed below */);
+    DEBUGLOG(4, "ZSTD_CCtx_setFParams");
+    FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, fparams.contentSizeFlag != 0), "");
+    FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, fparams.checksumFlag != 0), "");
+    FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_dictIDFlag, fparams.noDictIDFlag == 0), "");
+    return 0;
+}
+
+size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params)
+{
+    DEBUGLOG(4, "ZSTD_CCtx_setParams");
+    /* First check cParams, because we want to update all or none. */
+    FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), "");
+    /* Next set fParams, because this could fail if the cctx isn't in init stage. */
+    FORWARD_IF_ERROR(ZSTD_CCtx_setFParams(cctx, params.fParams), "");
+    /* Finally set cParams, which should succeed. */
+    FORWARD_IF_ERROR(ZSTD_CCtx_setCParams(cctx, params.cParams), "");
+    return 0;
+}
+
 size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize)
 {
-    DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %u bytes", (U32)pledgedSrcSize);
+    DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %llu bytes", pledgedSrcSize);
     RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
                     "Can't set pledgedSrcSize when not in init stage.");
     cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1;
@@ -1024,9 +1164,9 @@ static void ZSTD_dedicatedDictSearch_revertCParams(
         ZSTD_compressionParameters* cParams);
 
 /*
- * Initializes the local dict using the requested parameters.
- * NOTE: This does not use the pledged src size, because it may be used for more
- * than one compression.
+ * Initializes the local dictionary using requested parameters.
+ * NOTE: Initialization does not employ the pledged src size,
+ * because the dictionary may be used for multiple compressions.
  */
 static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx)
 {
@@ -1039,8 +1179,8 @@ static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx)
         return 0;
     }
     if (dl->cdict != NULL) {
-        assert(cctx->cdict == dl->cdict);
         /* Local dictionary already initialized. */
+        assert(cctx->cdict == dl->cdict);
         return 0;
     }
     assert(dl->dictSize > 0);
@@ -1060,26 +1200,30 @@ static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx)
 }
 
 size_t ZSTD_CCtx_loadDictionary_advanced(
-        ZSTD_CCtx* cctx, const void* dict, size_t dictSize,
-        ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType)
+        ZSTD_CCtx* cctx,
+        const void* dict, size_t dictSize,
+        ZSTD_dictLoadMethod_e dictLoadMethod,
+        ZSTD_dictContentType_e dictContentType)
 {
-    RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
-                    "Can't load a dictionary when ctx is not in init stage.");
     DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize);
-    ZSTD_clearAllDicts(cctx);  /* in case one already exists */
-    if (dict == NULL || dictSize == 0)  /* no dictionary mode */
+    RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
+                    "Can't load a dictionary when cctx is not in init stage.");
+    ZSTD_clearAllDicts(cctx);  /* erase any previously set dictionary */
+    if (dict == NULL || dictSize == 0)  /* no dictionary */
         return 0;
     if (dictLoadMethod == ZSTD_dlm_byRef) {
         cctx->localDict.dict = dict;
     } else {
+        /* copy dictionary content inside CCtx to own its lifetime */
         void* dictBuffer;
         RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
-                        "no malloc for static CCtx");
+                        "static CCtx can't allocate for an internal copy of dictionary");
         dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem);
-        RETURN_ERROR_IF(!dictBuffer, memory_allocation, "NULL pointer!");
+        RETURN_ERROR_IF(dictBuffer==NULL, memory_allocation,
+                        "allocation failed for dictionary content");
         ZSTD_memcpy(dictBuffer, dict, dictSize);
-        cctx->localDict.dictBuffer = dictBuffer;
-        cctx->localDict.dict = dictBuffer;
+        cctx->localDict.dictBuffer = dictBuffer;  /* owned ptr to free */
+        cctx->localDict.dict = dictBuffer;        /* read-only reference */
     }
     cctx->localDict.dictSize = dictSize;
     cctx->localDict.dictContentType = dictContentType;
@@ -1149,8 +1293,9 @@ size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset)
     if ( (reset == ZSTD_reset_parameters)
       || (reset == ZSTD_reset_session_and_parameters) ) {
         RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong,
-                        "Can't reset parameters only when not in init stage.");
+                        "Reset parameters is only possible during init stage.");
         ZSTD_clearAllDicts(cctx);
+        ZSTD_memset(&cctx->externalMatchCtx, 0, sizeof(cctx->externalMatchCtx));
         return ZSTD_CCtxParams_reset(&cctx->requestedParams);
     }
     return 0;
@@ -1247,7 +1392,8 @@ static ZSTD_compressionParameters
 ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
                             unsigned long long srcSize,
                             size_t dictSize,
-                            ZSTD_cParamMode_e mode)
+                            ZSTD_cParamMode_e mode,
+                            ZSTD_paramSwitch_e useRowMatchFinder)
 {
     const U64 minSrcSize = 513; /* (1<<9) + 1 */
     const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1);
@@ -1281,8 +1427,8 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
     }
 
     /* resize windowLog if input is small enough, to use less memory */
-    if ( (srcSize < maxWindowResize)
-      && (dictSize < maxWindowResize) )  {
+    if ( (srcSize <= maxWindowResize)
+      && (dictSize <= maxWindowResize) )  {
         U32 const tSize = (U32)(srcSize + dictSize);
         static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN;
         U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN :
@@ -1300,6 +1446,42 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar,
     if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN)
         cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN;  /* minimum wlog required for valid frame header */
 
+    /* We can't use more than 32 bits of hash in total, so that means that we require:
+     * (hashLog + 8) <= 32 && (chainLog + 8) <= 32
+     */
+    if (mode == ZSTD_cpm_createCDict && ZSTD_CDictIndicesAreTagged(&cPar)) {
+        U32 const maxShortCacheHashLog = 32 - ZSTD_SHORT_CACHE_TAG_BITS;
+        if (cPar.hashLog > maxShortCacheHashLog) {
+            cPar.hashLog = maxShortCacheHashLog;
+        }
+        if (cPar.chainLog > maxShortCacheHashLog) {
+            cPar.chainLog = maxShortCacheHashLog;
+        }
+    }
+
+
+    /* At this point, we aren't 100% sure if we are using the row match finder.
+     * Unless it is explicitly disabled, conservatively assume that it is enabled.
+     * In this case it will only be disabled for small sources, so shrinking the
+     * hash log a little bit shouldn't result in any ratio loss.
+     */
+    if (useRowMatchFinder == ZSTD_ps_auto)
+        useRowMatchFinder = ZSTD_ps_enable;
+
+    /* We can't hash more than 32-bits in total. So that means that we require:
+     * (hashLog - rowLog + 8) <= 32
+     */
+    if (ZSTD_rowMatchFinderUsed(cPar.strategy, useRowMatchFinder)) {
+        /* Switch to 32-entry rows if searchLog is 5 (or more) */
+        U32 const rowLog = BOUNDED(4, cPar.searchLog, 6);
+        U32 const maxRowHashLog = 32 - ZSTD_ROW_HASH_TAG_BITS;
+        U32 const maxHashLog = maxRowHashLog + rowLog;
+        assert(cPar.hashLog >= rowLog);
+        if (cPar.hashLog > maxHashLog) {
+            cPar.hashLog = maxHashLog;
+        }
+    }
+
     return cPar;
 }
 
@@ -1310,7 +1492,7 @@ ZSTD_adjustCParams(ZSTD_compressionParameters cPar,
 {
     cPar = ZSTD_clampCParams(cPar);   /* resulting cPar is necessarily valid (all parameters within range) */
     if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN;
-    return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown);
+    return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown, ZSTD_ps_auto);
 }
 
 static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode);
@@ -1341,7 +1523,7 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
     ZSTD_overrideCParams(&cParams, &CCtxParams->cParams);
     assert(!ZSTD_checkCParams(cParams));
     /* srcSizeHint == 0 means 0 */
-    return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode);
+    return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode, CCtxParams->useRowMatchFinder);
 }
 
 static size_t
@@ -1370,7 +1552,7 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
       + ZSTD_cwksp_aligned_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t))
       + ZSTD_cwksp_aligned_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t));
     size_t const lazyAdditionalSpace = ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)
-                                            ? ZSTD_cwksp_aligned_alloc_size(hSize*sizeof(U16))
+                                            ? ZSTD_cwksp_aligned_alloc_size(hSize)
                                             : 0;
     size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt))
                                 ? optPotentialSpace
@@ -1386,6 +1568,13 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
     return tableSpace + optSpace + slackSpace + lazyAdditionalSpace;
 }
 
+/* Helper function for calculating memory requirements.
+ * Gives a tighter bound than ZSTD_sequenceBound() by taking minMatch into account. */
+static size_t ZSTD_maxNbSeq(size_t blockSize, unsigned minMatch, int useSequenceProducer) {
+    U32 const divider = (minMatch==3 || useSequenceProducer) ? 3 : 4;
+    return blockSize / divider;
+}
+
 static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
         const ZSTD_compressionParameters* cParams,
         const ldmParams_t* ldmParams,
@@ -1393,12 +1582,13 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
         const ZSTD_paramSwitch_e useRowMatchFinder,
         const size_t buffInSize,
         const size_t buffOutSize,
-        const U64 pledgedSrcSize)
+        const U64 pledgedSrcSize,
+        int useSequenceProducer,
+        size_t maxBlockSize)
 {
     size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize);
-    size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
-    U32    const divider = (cParams->minMatch==3) ? 3 : 4;
-    size_t const maxNbSeq = blockSize / divider;
+    size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(maxBlockSize), windowSize);
+    size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useSequenceProducer);
     size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
                             + ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef))
                             + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE));
@@ -1417,6 +1607,11 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
 
     size_t const cctxSpace = isStatic ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0;
 
+    size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
+    size_t const externalSeqSpace = useSequenceProducer
+        ? ZSTD_cwksp_aligned_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence))
+        : 0;
+
     size_t const neededSpace =
         cctxSpace +
         entropySpace +
@@ -1425,7 +1620,8 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
         ldmSeqSpace +
         matchStateSize +
         tokenSpace +
-        bufferSpace;
+        bufferSpace +
+        externalSeqSpace;
 
     DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace);
     return neededSpace;
@@ -1443,7 +1639,7 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
      * be needed. However, we still allocate two 0-sized buffers, which can
      * take space under ASAN. */
     return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
-        &cParams, &params->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN);
+        &cParams, &params->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, params->useSequenceProducer, params->maxBlockSize);
 }
 
 size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams)
@@ -1493,7 +1689,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
     RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
     {   ZSTD_compressionParameters const cParams =
                 ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
-        size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog);
+        size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(params->maxBlockSize), (size_t)1 << cParams.windowLog);
         size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered)
                 ? ((size_t)1 << cParams.windowLog) + blockSize
                 : 0;
@@ -1504,7 +1700,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
 
         return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
             &cParams, &params->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize,
-            ZSTD_CONTENTSIZE_UNKNOWN);
+            ZSTD_CONTENTSIZE_UNKNOWN, params->useSequenceProducer, params->maxBlockSize);
     }
 }
 
@@ -1637,6 +1833,19 @@ typedef enum {
     ZSTD_resetTarget_CCtx
 } ZSTD_resetTarget_e;
 
+/* Mixes bits in a 64 bits in a value, based on XXH3_rrmxmx */
+static U64 ZSTD_bitmix(U64 val, U64 len) {
+    val ^= ZSTD_rotateRight_U64(val, 49) ^ ZSTD_rotateRight_U64(val, 24);
+    val *= 0x9FB21C651E98DF25ULL;
+    val ^= (val >> 35) + len ;
+    val *= 0x9FB21C651E98DF25ULL;
+    return val ^ (val >> 28);
+}
+
+/* Mixes in the hashSalt and hashSaltEntropy to create a new hashSalt */
+static void ZSTD_advanceHashSalt(ZSTD_matchState_t* ms) {
+    ms->hashSalt = ZSTD_bitmix(ms->hashSalt, 8) ^ ZSTD_bitmix((U64) ms->hashSaltEntropy, 4);
+}
 
 static size_t
 ZSTD_reset_matchState(ZSTD_matchState_t* ms,
@@ -1664,6 +1873,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
     }
 
     ms->hashLog3 = hashLog3;
+    ms->lazySkipping = 0;
 
     ZSTD_invalidateMatchState(ms);
 
@@ -1685,6 +1895,27 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
         ZSTD_cwksp_clean_tables(ws);
     }
 
+    if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) {
+        /* Row match finder needs an additional table of hashes ("tags") */
+        size_t const tagTableSize = hSize;
+        /* We want to generate a new salt in case we reset a Cctx, but we always want to use
+         * 0 when we reset a Cdict */
+        if(forWho == ZSTD_resetTarget_CCtx) {
+            ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned_init_once(ws, tagTableSize);
+            ZSTD_advanceHashSalt(ms);
+        } else {
+            /* When we are not salting we want to always memset the memory */
+            ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned(ws, tagTableSize);
+            ZSTD_memset(ms->tagTable, 0, tagTableSize);
+            ms->hashSalt = 0;
+        }
+        {   /* Switch to 32-entry rows if searchLog is 5 (or more) */
+            U32 const rowLog = BOUNDED(4, cParams->searchLog, 6);
+            assert(cParams->hashLog >= rowLog);
+            ms->rowHashLog = cParams->hashLog - rowLog;
+        }
+    }
+
     /* opt parser space */
     if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) {
         DEBUGLOG(4, "reserving optimal parser space");
@@ -1696,19 +1927,6 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
         ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t));
     }
 
-    if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) {
-        {   /* Row match finder needs an additional table of hashes ("tags") */
-            size_t const tagTableSize = hSize*sizeof(U16);
-            ms->tagTable = (U16*)ZSTD_cwksp_reserve_aligned(ws, tagTableSize);
-            if (ms->tagTable) ZSTD_memset(ms->tagTable, 0, tagTableSize);
-        }
-        {   /* Switch to 32-entry rows if searchLog is 5 (or more) */
-            U32 const rowLog = BOUNDED(4, cParams->searchLog, 6);
-            assert(cParams->hashLog >= rowLog);
-            ms->rowHashLog = cParams->hashLog - rowLog;
-        }
-    }
-
     ms->cParams = *cParams;
 
     RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation,
@@ -1768,6 +1986,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
     assert(params->useRowMatchFinder != ZSTD_ps_auto);
     assert(params->useBlockSplitter != ZSTD_ps_auto);
     assert(params->ldmParams.enableLdm != ZSTD_ps_auto);
+    assert(params->maxBlockSize != 0);
     if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
         /* Adjust long distance matching parameters */
         ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, &params->cParams);
@@ -1776,9 +1995,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
     }
 
     {   size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize));
-        size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize);
-        U32    const divider = (params->cParams.minMatch==3) ? 3 : 4;
-        size_t const maxNbSeq = blockSize / divider;
+        size_t const blockSize = MIN(params->maxBlockSize, windowSize);
+        size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, params->useSequenceProducer);
         size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered)
                 ? ZSTD_compressBound(blockSize) + 1
                 : 0;
@@ -1795,7 +2013,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         size_t const neededSpace =
             ZSTD_estimateCCtxSize_usingCCtxParams_internal(
                 &params->cParams, &params->ldmParams, zc->staticSize != 0, params->useRowMatchFinder,
-                buffInSize, buffOutSize, pledgedSrcSize);
+                buffInSize, buffOutSize, pledgedSrcSize, params->useSequenceProducer, params->maxBlockSize);
         int resizeWorkspace;
 
         FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!");
@@ -1838,6 +2056,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
 
         /* init params */
         zc->blockState.matchState.cParams = params->cParams;
+        zc->blockState.matchState.prefetchCDictTables = params->prefetchCDictTables == ZSTD_ps_enable;
         zc->pledgedSrcSizePlusOne = pledgedSrcSize+1;
         zc->consumedSrcSize = 0;
         zc->producedCSize = 0;
@@ -1854,13 +2073,46 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
 
         ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock);
 
+        FORWARD_IF_ERROR(ZSTD_reset_matchState(
+                &zc->blockState.matchState,
+                ws,
+                &params->cParams,
+                params->useRowMatchFinder,
+                crp,
+                needsIndexReset,
+                ZSTD_resetTarget_CCtx), "");
+
+        zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef));
+
+        /* ldm hash table */
+        if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
+            /* TODO: avoid memset? */
+            size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog;
+            zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t));
+            ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t));
+            zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq));
+            zc->maxNbLdmSequences = maxNbLdmSeq;
+
+            ZSTD_window_init(&zc->ldmState.window);
+            zc->ldmState.loadedDictEnd = 0;
+        }
+
+        /* reserve space for block-level external sequences */
+        if (params->useSequenceProducer) {
+            size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize);
+            zc->externalMatchCtx.seqBufferCapacity = maxNbExternalSeq;
+            zc->externalMatchCtx.seqBuffer =
+                (ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence));
+        }
+
+        /* buffers */
+
         /* ZSTD_wildcopy() is used to copy into the literals buffer,
          * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes.
          */
         zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH);
         zc->seqStore.maxNbLit = blockSize;
 
-        /* buffers */
         zc->bufferedPolicy = zbuff;
         zc->inBuffSize = buffInSize;
         zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize);
@@ -1883,32 +2135,9 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
         zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
         zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
         zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE));
-        zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef));
-
-        FORWARD_IF_ERROR(ZSTD_reset_matchState(
-            &zc->blockState.matchState,
-            ws,
-            &params->cParams,
-            params->useRowMatchFinder,
-            crp,
-            needsIndexReset,
-            ZSTD_resetTarget_CCtx), "");
-
-        /* ldm hash table */
-        if (params->ldmParams.enableLdm == ZSTD_ps_enable) {
-            /* TODO: avoid memset? */
-            size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog;
-            zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t));
-            ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t));
-            zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq));
-            zc->maxNbLdmSequences = maxNbLdmSeq;
-
-            ZSTD_window_init(&zc->ldmState.window);
-            zc->ldmState.loadedDictEnd = 0;
-        }
 
         DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws));
-        assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace, resizeWorkspace));
+        assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace));
 
         zc->initialized = 1;
 
@@ -1980,7 +2209,8 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
         }
 
         params.cParams = ZSTD_adjustCParams_internal(adjusted_cdict_cParams, pledgedSrcSize,
-                                                     cdict->dictContentSize, ZSTD_cpm_attachDict);
+                                                     cdict->dictContentSize, ZSTD_cpm_attachDict,
+                                                     params.useRowMatchFinder);
         params.cParams.windowLog = windowLog;
         params.useRowMatchFinder = cdict->useRowMatchFinder;    /* cdict overrides */
         FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, &params, pledgedSrcSize,
@@ -2019,6 +2249,22 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
     return 0;
 }
 
+static void ZSTD_copyCDictTableIntoCCtx(U32* dst, U32 const* src, size_t tableSize,
+                                        ZSTD_compressionParameters const* cParams) {
+    if (ZSTD_CDictIndicesAreTagged(cParams)){
+        /* Remove tags from the CDict table if they are present.
+         * See docs on "short cache" in zstd_compress_internal.h for context. */
+        size_t i;
+        for (i = 0; i < tableSize; i++) {
+            U32 const taggedIndex = src[i];
+            U32 const index = taggedIndex >> ZSTD_SHORT_CACHE_TAG_BITS;
+            dst[i] = index;
+        }
+    } else {
+        ZSTD_memcpy(dst, src, tableSize * sizeof(U32));
+    }
+}
+
 static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx,
                             const ZSTD_CDict* cdict,
                             ZSTD_CCtx_params params,
@@ -2054,21 +2300,23 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx,
                                                             : 0;
         size_t const hSize =  (size_t)1 << cdict_cParams->hashLog;
 
-        ZSTD_memcpy(cctx->blockState.matchState.hashTable,
-               cdict->matchState.hashTable,
-               hSize * sizeof(U32));
+        ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.hashTable,
+                                cdict->matchState.hashTable,
+                                hSize, cdict_cParams);
+
         /* Do not copy cdict's chainTable if cctx has parameters such that it would not use chainTable */
         if (ZSTD_allocateChainTable(cctx->appliedParams.cParams.strategy, cctx->appliedParams.useRowMatchFinder, 0 /* forDDSDict */)) {
-            ZSTD_memcpy(cctx->blockState.matchState.chainTable,
-               cdict->matchState.chainTable,
-               chainSize * sizeof(U32));
+            ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.chainTable,
+                                    cdict->matchState.chainTable,
+                                    chainSize, cdict_cParams);
         }
         /* copy tag table */
         if (ZSTD_rowMatchFinderUsed(cdict_cParams->strategy, cdict->useRowMatchFinder)) {
-            size_t const tagTableSize = hSize*sizeof(U16);
+            size_t const tagTableSize = hSize;
             ZSTD_memcpy(cctx->blockState.matchState.tagTable,
-                cdict->matchState.tagTable,
-                tagTableSize);
+                        cdict->matchState.tagTable,
+                        tagTableSize);
+            cctx->blockState.matchState.hashSalt = cdict->matchState.hashSalt;
         }
     }
 
@@ -2147,6 +2395,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
         params.useBlockSplitter = srcCCtx->appliedParams.useBlockSplitter;
         params.ldmParams = srcCCtx->appliedParams.ldmParams;
         params.fParams = fParams;
+        params.maxBlockSize = srcCCtx->appliedParams.maxBlockSize;
         ZSTD_resetCCtx_internal(dstCCtx, &params, pledgedSrcSize,
                                 /* loadedDictSize */ 0,
                                 ZSTDcrp_leaveDirty, zbuff);
@@ -2294,7 +2543,7 @@ static void ZSTD_reduceIndex (ZSTD_matchState_t* ms, ZSTD_CCtx_params const* par
 
 /* See doc/zstd_compression_format.md for detailed format description */
 
-void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
+int ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
 {
     const seqDef* const sequences = seqStorePtr->sequencesStart;
     BYTE* const llCodeTable = seqStorePtr->llCode;
@@ -2302,18 +2551,24 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
     BYTE* const mlCodeTable = seqStorePtr->mlCode;
     U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
     U32 u;
+    int longOffsets = 0;
     assert(nbSeq <= seqStorePtr->maxNbSeq);
     for (u=0; u<nbSeq; u++) {
         U32 const llv = sequences[u].litLength;
+        U32 const ofCode = ZSTD_highbit32(sequences[u].offBase);
         U32 const mlv = sequences[u].mlBase;
         llCodeTable[u] = (BYTE)ZSTD_LLcode(llv);
-        ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offBase);
+        ofCodeTable[u] = (BYTE)ofCode;
         mlCodeTable[u] = (BYTE)ZSTD_MLcode(mlv);
+        assert(!(MEM_64bits() && ofCode >= STREAM_ACCUMULATOR_MIN));
+        if (MEM_32bits() && ofCode >= STREAM_ACCUMULATOR_MIN)
+            longOffsets = 1;
     }
     if (seqStorePtr->longLengthType==ZSTD_llt_literalLength)
         llCodeTable[seqStorePtr->longLengthPos] = MaxLL;
     if (seqStorePtr->longLengthType==ZSTD_llt_matchLength)
         mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
+    return longOffsets;
 }
 
 /* ZSTD_useTargetCBlockSize():
@@ -2347,6 +2602,7 @@ typedef struct {
     U32 MLtype;
     size_t size;
     size_t lastCountSize; /* Accounts for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */
+    int longOffsets;
 } ZSTD_symbolEncodingTypeStats_t;
 
 /* ZSTD_buildSequencesStatistics():
@@ -2357,11 +2613,13 @@ typedef struct {
  * entropyWkspSize must be of size at least ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1)*sizeof(U32)
  */
 static ZSTD_symbolEncodingTypeStats_t
-ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq,
-                        const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy,
-                              BYTE* dst, const BYTE* const dstEnd,
-                              ZSTD_strategy strategy, unsigned* countWorkspace,
-                              void* entropyWorkspace, size_t entropyWkspSize) {
+ZSTD_buildSequencesStatistics(
+                const seqStore_t* seqStorePtr, size_t nbSeq,
+                const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy,
+                      BYTE* dst, const BYTE* const dstEnd,
+                      ZSTD_strategy strategy, unsigned* countWorkspace,
+                      void* entropyWorkspace, size_t entropyWkspSize)
+{
     BYTE* const ostart = dst;
     const BYTE* const oend = dstEnd;
     BYTE* op = ostart;
@@ -2375,7 +2633,7 @@ ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq,
 
     stats.lastCountSize = 0;
     /* convert length/distances into codes */
-    ZSTD_seqToCodes(seqStorePtr);
+    stats.longOffsets = ZSTD_seqToCodes(seqStorePtr);
     assert(op <= oend);
     assert(nbSeq != 0); /* ZSTD_selectEncodingType() divides by nbSeq */
     /* build CTable for Literal Lengths */
@@ -2480,22 +2738,22 @@ ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq,
  */
 #define SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO 20
 MEM_STATIC size_t
-ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
-                          const ZSTD_entropyCTables_t* prevEntropy,
-                                ZSTD_entropyCTables_t* nextEntropy,
-                          const ZSTD_CCtx_params* cctxParams,
-                                void* dst, size_t dstCapacity,
-                                void* entropyWorkspace, size_t entropyWkspSize,
-                          const int bmi2)
+ZSTD_entropyCompressSeqStore_internal(
+                        const seqStore_t* seqStorePtr,
+                        const ZSTD_entropyCTables_t* prevEntropy,
+                              ZSTD_entropyCTables_t* nextEntropy,
+                        const ZSTD_CCtx_params* cctxParams,
+                              void* dst, size_t dstCapacity,
+                              void* entropyWorkspace, size_t entropyWkspSize,
+                        const int bmi2)
 {
-    const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN;
     ZSTD_strategy const strategy = cctxParams->cParams.strategy;
     unsigned* count = (unsigned*)entropyWorkspace;
     FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable;
     FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable;
     FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable;
     const seqDef* const sequences = seqStorePtr->sequencesStart;
-    const size_t nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
+    const size_t nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
     const BYTE* const ofCodeTable = seqStorePtr->ofCode;
     const BYTE* const llCodeTable = seqStorePtr->llCode;
     const BYTE* const mlCodeTable = seqStorePtr->mlCode;
@@ -2503,29 +2761,31 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
     BYTE* const oend = ostart + dstCapacity;
     BYTE* op = ostart;
     size_t lastCountSize;
+    int longOffsets = 0;
 
     entropyWorkspace = count + (MaxSeq + 1);
     entropyWkspSize -= (MaxSeq + 1) * sizeof(*count);
 
-    DEBUGLOG(4, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu)", nbSeq);
+    DEBUGLOG(5, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu, dstCapacity=%zu)", nbSeq, dstCapacity);
     ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog)));
     assert(entropyWkspSize >= HUF_WORKSPACE_SIZE);
 
     /* Compress literals */
     {   const BYTE* const literals = seqStorePtr->litStart;
-        size_t const numSequences = seqStorePtr->sequences - seqStorePtr->sequencesStart;
-        size_t const numLiterals = seqStorePtr->lit - seqStorePtr->litStart;
+        size_t const numSequences = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
+        size_t const numLiterals = (size_t)(seqStorePtr->lit - seqStorePtr->litStart);
         /* Base suspicion of uncompressibility on ratio of literals to sequences */
         unsigned const suspectUncompressible = (numSequences == 0) || (numLiterals / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO);
         size_t const litSize = (size_t)(seqStorePtr->lit - literals);
+
         size_t const cSize = ZSTD_compressLiterals(
-                                    &prevEntropy->huf, &nextEntropy->huf,
-                                    cctxParams->cParams.strategy,
-                                    ZSTD_literalsCompressionIsDisabled(cctxParams),
                                     op, dstCapacity,
                                     literals, litSize,
                                     entropyWorkspace, entropyWkspSize,
-                                    bmi2, suspectUncompressible);
+                                    &prevEntropy->huf, &nextEntropy->huf,
+                                    cctxParams->cParams.strategy,
+                                    ZSTD_literalsCompressionIsDisabled(cctxParams),
+                                    suspectUncompressible, bmi2);
         FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed");
         assert(cSize <= dstCapacity);
         op += cSize;
@@ -2551,11 +2811,10 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
         ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse));
         return (size_t)(op - ostart);
     }
-    {
-        ZSTD_symbolEncodingTypeStats_t stats;
-        BYTE* seqHead = op++;
+    {   BYTE* const seqHead = op++;
         /* build stats for sequences */
-        stats = ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq,
+        const ZSTD_symbolEncodingTypeStats_t stats =
+                ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq,
                                              &prevEntropy->fse, &nextEntropy->fse,
                                               op, oend,
                                               strategy, count,
@@ -2564,6 +2823,7 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
         *seqHead = (BYTE)((stats.LLtype<<6) + (stats.Offtype<<4) + (stats.MLtype<<2));
         lastCountSize = stats.lastCountSize;
         op += stats.size;
+        longOffsets = stats.longOffsets;
     }
 
     {   size_t const bitstreamSize = ZSTD_encodeSequences(
@@ -2598,14 +2858,15 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
 }
 
 MEM_STATIC size_t
-ZSTD_entropyCompressSeqStore(seqStore_t* seqStorePtr,
-                       const ZSTD_entropyCTables_t* prevEntropy,
-                             ZSTD_entropyCTables_t* nextEntropy,
-                       const ZSTD_CCtx_params* cctxParams,
-                             void* dst, size_t dstCapacity,
-                             size_t srcSize,
-                             void* entropyWorkspace, size_t entropyWkspSize,
-                             int bmi2)
+ZSTD_entropyCompressSeqStore(
+                    const seqStore_t* seqStorePtr,
+                    const ZSTD_entropyCTables_t* prevEntropy,
+                          ZSTD_entropyCTables_t* nextEntropy,
+                    const ZSTD_CCtx_params* cctxParams,
+                          void* dst, size_t dstCapacity,
+                          size_t srcSize,
+                          void* entropyWorkspace, size_t entropyWkspSize,
+                          int bmi2)
 {
     size_t const cSize = ZSTD_entropyCompressSeqStore_internal(
                             seqStorePtr, prevEntropy, nextEntropy, cctxParams,
@@ -2615,15 +2876,21 @@ ZSTD_entropyCompressSeqStore(seqStore_t* seqStorePtr,
     /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block.
      * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block.
      */
-    if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity))
+    if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity)) {
+        DEBUGLOG(4, "not enough dstCapacity (%zu) for ZSTD_entropyCompressSeqStore_internal()=> do not compress block", dstCapacity);
         return 0;  /* block not compressed */
+    }
     FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSeqStore_internal failed");
 
     /* Check compressibility */
     {   size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy);
         if (cSize >= maxCSize) return 0;  /* block not compressed */
     }
-    DEBUGLOG(4, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize);
+    DEBUGLOG(5, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize);
+    /* libzstd decoder before  > v1.5.4 is not compatible with compressed blocks of size ZSTD_BLOCKSIZE_MAX exactly.
+     * This restriction is indirectly already fulfilled by respecting ZSTD_minGain() condition above.
+     */
+    assert(cSize < ZSTD_BLOCKSIZE_MAX);
     return cSize;
 }
 
@@ -2718,6 +2985,72 @@ void ZSTD_resetSeqStore(seqStore_t* ssPtr)
     ssPtr->longLengthType = ZSTD_llt_none;
 }
 
+/* ZSTD_postProcessSequenceProducerResult() :
+ * Validates and post-processes sequences obtained through the external matchfinder API:
+ *   - Checks whether nbExternalSeqs represents an error condition.
+ *   - Appends a block delimiter to outSeqs if one is not already present.
+ *     See zstd.h for context regarding block delimiters.
+ * Returns the number of sequences after post-processing, or an error code. */
+static size_t ZSTD_postProcessSequenceProducerResult(
+    ZSTD_Sequence* outSeqs, size_t nbExternalSeqs, size_t outSeqsCapacity, size_t srcSize
+) {
+    RETURN_ERROR_IF(
+        nbExternalSeqs > outSeqsCapacity,
+        sequenceProducer_failed,
+        "External sequence producer returned error code %lu",
+        (unsigned long)nbExternalSeqs
+    );
+
+    RETURN_ERROR_IF(
+        nbExternalSeqs == 0 && srcSize > 0,
+        sequenceProducer_failed,
+        "Got zero sequences from external sequence producer for a non-empty src buffer!"
+    );
+
+    if (srcSize == 0) {
+        ZSTD_memset(&outSeqs[0], 0, sizeof(ZSTD_Sequence));
+        return 1;
+    }
+
+    {
+        ZSTD_Sequence const lastSeq = outSeqs[nbExternalSeqs - 1];
+
+        /* We can return early if lastSeq is already a block delimiter. */
+        if (lastSeq.offset == 0 && lastSeq.matchLength == 0) {
+            return nbExternalSeqs;
+        }
+
+        /* This error condition is only possible if the external matchfinder
+         * produced an invalid parse, by definition of ZSTD_sequenceBound(). */
+        RETURN_ERROR_IF(
+            nbExternalSeqs == outSeqsCapacity,
+            sequenceProducer_failed,
+            "nbExternalSeqs == outSeqsCapacity but lastSeq is not a block delimiter!"
+        );
+
+        /* lastSeq is not a block delimiter, so we need to append one. */
+        ZSTD_memset(&outSeqs[nbExternalSeqs], 0, sizeof(ZSTD_Sequence));
+        return nbExternalSeqs + 1;
+    }
+}
+
+/* ZSTD_fastSequenceLengthSum() :
+ * Returns sum(litLen) + sum(matchLen) + lastLits for *seqBuf*.
+ * Similar to another function in zstd_compress.c (determine_blockSize),
+ * except it doesn't check for a block delimiter to end summation.
+ * Removing the early exit allows the compiler to auto-vectorize (https://godbolt.org/z/cY1cajz9P).
+ * This function can be deleted and replaced by determine_blockSize after we resolve issue #3456. */
+static size_t ZSTD_fastSequenceLengthSum(ZSTD_Sequence const* seqBuf, size_t seqBufSize) {
+    size_t matchLenSum, litLenSum, i;
+    matchLenSum = 0;
+    litLenSum = 0;
+    for (i = 0; i < seqBufSize; i++) {
+        litLenSum += seqBuf[i].litLength;
+        matchLenSum += seqBuf[i].matchLength;
+    }
+    return litLenSum + matchLenSum;
+}
+
 typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e;
 
 static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
@@ -2727,7 +3060,9 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
     assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
     /* Assert that we have correctly flushed the ctx params into the ms's copy */
     ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams);
-    if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) {
+    /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding
+     * additional 1. We need to revisit and change this logic to be more consistent */
+    if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) {
         if (zc->appliedParams.cParams.strategy >= ZSTD_btopt) {
             ZSTD_ldm_skipRawSeqStoreBytes(&zc->externSeqStore, srcSize);
         } else {
@@ -2763,6 +3098,15 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
         }
         if (zc->externSeqStore.pos < zc->externSeqStore.size) {
             assert(zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_disable);
+
+            /* External matchfinder + LDM is technically possible, just not implemented yet.
+             * We need to revisit soon and implement it. */
+            RETURN_ERROR_IF(
+                zc->appliedParams.useSequenceProducer,
+                parameter_combination_unsupported,
+                "Long-distance matching with external sequence producer enabled is not currently supported."
+            );
+
             /* Updates ldmSeqStore.pos */
             lastLLSize =
                 ZSTD_ldm_blockCompress(&zc->externSeqStore,
@@ -2774,6 +3118,14 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
         } else if (zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) {
             rawSeqStore_t ldmSeqStore = kNullRawSeqStore;
 
+            /* External matchfinder + LDM is technically possible, just not implemented yet.
+             * We need to revisit soon and implement it. */
+            RETURN_ERROR_IF(
+                zc->appliedParams.useSequenceProducer,
+                parameter_combination_unsupported,
+                "Long-distance matching with external sequence producer enabled is not currently supported."
+            );
+
             ldmSeqStore.seq = zc->ldmSequences;
             ldmSeqStore.capacity = zc->maxNbLdmSequences;
             /* Updates ldmSeqStore.size */
@@ -2788,7 +3140,68 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
                                        zc->appliedParams.useRowMatchFinder,
                                        src, srcSize);
             assert(ldmSeqStore.pos == ldmSeqStore.size);
-        } else {   /* not long range mode */
+        } else if (zc->appliedParams.useSequenceProducer) {
+            assert(
+                zc->externalMatchCtx.seqBufferCapacity >= ZSTD_sequenceBound(srcSize)
+            );
+            assert(zc->externalMatchCtx.mFinder != NULL);
+
+            {   U32 const windowSize = (U32)1 << zc->appliedParams.cParams.windowLog;
+
+                size_t const nbExternalSeqs = (zc->externalMatchCtx.mFinder)(
+                    zc->externalMatchCtx.mState,
+                    zc->externalMatchCtx.seqBuffer,
+                    zc->externalMatchCtx.seqBufferCapacity,
+                    src, srcSize,
+                    NULL, 0,  /* dict and dictSize, currently not supported */
+                    zc->appliedParams.compressionLevel,
+                    windowSize
+                );
+
+                size_t const nbPostProcessedSeqs = ZSTD_postProcessSequenceProducerResult(
+                    zc->externalMatchCtx.seqBuffer,
+                    nbExternalSeqs,
+                    zc->externalMatchCtx.seqBufferCapacity,
+                    srcSize
+                );
+
+                /* Return early if there is no error, since we don't need to worry about last literals */
+                if (!ZSTD_isError(nbPostProcessedSeqs)) {
+                    ZSTD_sequencePosition seqPos = {0,0,0};
+                    size_t const seqLenSum = ZSTD_fastSequenceLengthSum(zc->externalMatchCtx.seqBuffer, nbPostProcessedSeqs);
+                    RETURN_ERROR_IF(seqLenSum > srcSize, externalSequences_invalid, "External sequences imply too large a block!");
+                    FORWARD_IF_ERROR(
+                        ZSTD_copySequencesToSeqStoreExplicitBlockDelim(
+                            zc, &seqPos,
+                            zc->externalMatchCtx.seqBuffer, nbPostProcessedSeqs,
+                            src, srcSize,
+                            zc->appliedParams.searchForExternalRepcodes
+                        ),
+                        "Failed to copy external sequences to seqStore!"
+                    );
+                    ms->ldmSeqStore = NULL;
+                    DEBUGLOG(5, "Copied %lu sequences from external sequence producer to internal seqStore.", (unsigned long)nbExternalSeqs);
+                    return ZSTDbss_compress;
+                }
+
+                /* Propagate the error if fallback is disabled */
+                if (!zc->appliedParams.enableMatchFinderFallback) {
+                    return nbPostProcessedSeqs;
+                }
+
+                /* Fallback to software matchfinder */
+                {   ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy,
+                                                                                            zc->appliedParams.useRowMatchFinder,
+                                                                                            dictMode);
+                    ms->ldmSeqStore = NULL;
+                    DEBUGLOG(
+                        5,
+                        "External sequence producer returned error code %lu. Falling back to internal parser.",
+                        (unsigned long)nbExternalSeqs
+                    );
+                    lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
+            }   }
+        } else {   /* not long range mode and no external matchfinder */
             ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy,
                                                                                     zc->appliedParams.useRowMatchFinder,
                                                                                     dictMode);
@@ -2849,7 +3262,7 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc)
         /* seqStoreSeqs[i].offset == offCode+1, and ZSTD_updateRep() expects offCode
            so we provide seqStoreSeqs[i].offset - 1 */
         ZSTD_updateRep(updatedRepcodes.rep,
-                       seqStoreSeqs[i].offBase - 1,
+                       seqStoreSeqs[i].offBase,
                        seqStoreSeqs[i].litLength == 0);
         literalsRead += outSeqs[i].litLength;
     }
@@ -2865,6 +3278,10 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc)
     zc->seqCollector.seqIndex += seqStoreSeqSize;
 }
 
+size_t ZSTD_sequenceBound(size_t srcSize) {
+    return (srcSize / ZSTD_MINMATCH_MIN) + 1;
+}
+
 size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
                               size_t outSeqsSize, const void* src, size_t srcSize)
 {
@@ -2910,19 +3327,17 @@ static int ZSTD_isRLE(const BYTE* src, size_t length) {
     const size_t unrollMask = unrollSize - 1;
     const size_t prefixLength = length & unrollMask;
     size_t i;
-    size_t u;
     if (length == 1) return 1;
     /* Check if prefix is RLE first before using unrolled loop */
     if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) {
         return 0;
     }
     for (i = prefixLength; i != length; i += unrollSize) {
+        size_t u;
         for (u = 0; u < unrollSize; u += sizeof(size_t)) {
             if (MEM_readST(ip + i + u) != valueST) {
                 return 0;
-            }
-        }
-    }
+    }   }   }
     return 1;
 }
 
@@ -2938,7 +3353,8 @@ static int ZSTD_maybeRLE(seqStore_t const* seqStore)
     return nbSeqs < 4 && nbLits < 10;
 }
 
-static void ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs)
+static void
+ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs)
 {
     ZSTD_compressedBlockState_t* const tmp = bs->prevCBlock;
     bs->prevCBlock = bs->nextCBlock;
@@ -2946,7 +3362,9 @@ static void ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* c
 }
 
 /* Writes the block header */
-static void writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock) {
+static void
+writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock)
+{
     U32 const cBlockHeader = cSize == 1 ?
                         lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) :
                         lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
@@ -2959,13 +3377,16 @@ static void writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastB
  *  Stores literals block type (raw, rle, compressed, repeat) and
  *  huffman description table to hufMetadata.
  *  Requires ENTROPY_WORKSPACE_SIZE workspace
- *  @return : size of huffman description table or error code */
-static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize,
-                                            const ZSTD_hufCTables_t* prevHuf,
-                                                  ZSTD_hufCTables_t* nextHuf,
-                                                  ZSTD_hufCTablesMetadata_t* hufMetadata,
-                                                  const int literalsCompressionIsDisabled,
-                                                  void* workspace, size_t wkspSize)
+ * @return : size of huffman description table, or an error code
+ */
+static size_t
+ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize,
+                               const ZSTD_hufCTables_t* prevHuf,
+                                     ZSTD_hufCTables_t* nextHuf,
+                                     ZSTD_hufCTablesMetadata_t* hufMetadata,
+                               const int literalsCompressionIsDisabled,
+                                     void* workspace, size_t wkspSize,
+                                     int hufFlags)
 {
     BYTE* const wkspStart = (BYTE*)workspace;
     BYTE* const wkspEnd = wkspStart + wkspSize;
@@ -2973,9 +3394,9 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi
     unsigned* const countWksp = (unsigned*)workspace;
     const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned);
     BYTE* const nodeWksp = countWkspStart + countWkspSize;
-    const size_t nodeWkspSize = wkspEnd-nodeWksp;
+    const size_t nodeWkspSize = (size_t)(wkspEnd - nodeWksp);
     unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX;
-    unsigned huffLog = HUF_TABLELOG_DEFAULT;
+    unsigned huffLog = LitHufLog;
     HUF_repeat repeat = prevHuf->repeatMode;
     DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_literals (srcSize=%zu)", srcSize);
 
@@ -2990,73 +3411,77 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi
 
     /* small ? don't even attempt compression (speed opt) */
 #ifndef COMPRESS_LITERALS_SIZE_MIN
-#define COMPRESS_LITERALS_SIZE_MIN 63
+# define COMPRESS_LITERALS_SIZE_MIN 63  /* heuristic */
 #endif
     {   size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN;
         if (srcSize <= minLitSize) {
             DEBUGLOG(5, "set_basic - too small");
             hufMetadata->hType = set_basic;
             return 0;
-        }
-    }
+    }   }
 
     /* Scan input and build symbol stats */
-    {   size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)src, srcSize, workspace, wkspSize);
+    {   size_t const largest =
+            HIST_count_wksp (countWksp, &maxSymbolValue,
+                            (const BYTE*)src, srcSize,
+                            workspace, wkspSize);
         FORWARD_IF_ERROR(largest, "HIST_count_wksp failed");
         if (largest == srcSize) {
+            /* only one literal symbol */
             DEBUGLOG(5, "set_rle");
             hufMetadata->hType = set_rle;
             return 0;
         }
         if (largest <= (srcSize >> 7)+4) {
+            /* heuristic: likely not compressible */
             DEBUGLOG(5, "set_basic - no gain");
             hufMetadata->hType = set_basic;
             return 0;
-        }
-    }
+    }   }
 
     /* Validate the previous Huffman table */
-    if (repeat == HUF_repeat_check && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) {
+    if (repeat == HUF_repeat_check
+      && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) {
         repeat = HUF_repeat_none;
     }
 
     /* Build Huffman Tree */
     ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable));
-    huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
+    huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, nodeWksp, nodeWkspSize, nextHuf->CTable, countWksp, hufFlags);
+    assert(huffLog <= LitHufLog);
     {   size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp,
                                                     maxSymbolValue, huffLog,
                                                     nodeWksp, nodeWkspSize);
         FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp");
         huffLog = (U32)maxBits;
-        {   /* Build and write the CTable */
-            size_t const newCSize = HUF_estimateCompressedSize(
-                    (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue);
-            size_t const hSize = HUF_writeCTable_wksp(
-                    hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer),
-                    (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog,
-                    nodeWksp, nodeWkspSize);
-            /* Check against repeating the previous CTable */
-            if (repeat != HUF_repeat_none) {
-                size_t const oldCSize = HUF_estimateCompressedSize(
-                        (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue);
-                if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) {
-                    DEBUGLOG(5, "set_repeat - smaller");
-                    ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
-                    hufMetadata->hType = set_repeat;
-                    return 0;
-                }
-            }
-            if (newCSize + hSize >= srcSize) {
-                DEBUGLOG(5, "set_basic - no gains");
+    }
+    {   /* Build and write the CTable */
+        size_t const newCSize = HUF_estimateCompressedSize(
+                (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue);
+        size_t const hSize = HUF_writeCTable_wksp(
+                hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer),
+                (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog,
+                nodeWksp, nodeWkspSize);
+        /* Check against repeating the previous CTable */
+        if (repeat != HUF_repeat_none) {
+            size_t const oldCSize = HUF_estimateCompressedSize(
+                    (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue);
+            if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) {
+                DEBUGLOG(5, "set_repeat - smaller");
                 ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
-                hufMetadata->hType = set_basic;
+                hufMetadata->hType = set_repeat;
                 return 0;
-            }
-            DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize);
-            hufMetadata->hType = set_compressed;
-            nextHuf->repeatMode = HUF_repeat_check;
-            return hSize;
+        }   }
+        if (newCSize + hSize >= srcSize) {
+            DEBUGLOG(5, "set_basic - no gains");
+            ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+            hufMetadata->hType = set_basic;
+            return 0;
         }
+        DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize);
+        hufMetadata->hType = set_compressed;
+        nextHuf->repeatMode = HUF_repeat_check;
+        return hSize;
     }
 }
 
@@ -3066,8 +3491,9 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi
  * and updates nextEntropy to the appropriate repeatMode.
  */
 static ZSTD_symbolEncodingTypeStats_t
-ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy) {
-    ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0};
+ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy)
+{
+    ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0, 0};
     nextEntropy->litlength_repeatMode = FSE_repeat_none;
     nextEntropy->offcode_repeatMode = FSE_repeat_none;
     nextEntropy->matchlength_repeatMode = FSE_repeat_none;
@@ -3078,16 +3504,18 @@ ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy) {
  *  Builds entropy for the sequences.
  *  Stores symbol compression modes and fse table to fseMetadata.
  *  Requires ENTROPY_WORKSPACE_SIZE wksp.
- *  @return : size of fse tables or error code */
-static size_t ZSTD_buildBlockEntropyStats_sequences(seqStore_t* seqStorePtr,
-                                              const ZSTD_fseCTables_t* prevEntropy,
-                                                    ZSTD_fseCTables_t* nextEntropy,
-                                              const ZSTD_CCtx_params* cctxParams,
-                                                    ZSTD_fseCTablesMetadata_t* fseMetadata,
-                                                    void* workspace, size_t wkspSize)
+ * @return : size of fse tables or error code */
+static size_t
+ZSTD_buildBlockEntropyStats_sequences(
+                const seqStore_t* seqStorePtr,
+                const ZSTD_fseCTables_t* prevEntropy,
+                      ZSTD_fseCTables_t* nextEntropy,
+                const ZSTD_CCtx_params* cctxParams,
+                      ZSTD_fseCTablesMetadata_t* fseMetadata,
+                      void* workspace, size_t wkspSize)
 {
     ZSTD_strategy const strategy = cctxParams->cParams.strategy;
-    size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
+    size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
     BYTE* const ostart = fseMetadata->fseTablesBuffer;
     BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer);
     BYTE* op = ostart;
@@ -3114,23 +3542,28 @@ static size_t ZSTD_buildBlockEntropyStats_sequences(seqStore_t* seqStorePtr,
 /* ZSTD_buildBlockEntropyStats() :
  *  Builds entropy for the block.
  *  Requires workspace size ENTROPY_WORKSPACE_SIZE
- *
- *  @return : 0 on success or error code
+ * @return : 0 on success, or an error code
+ *  Note : also employed in superblock
  */
-size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr,
-                             const ZSTD_entropyCTables_t* prevEntropy,
-                                   ZSTD_entropyCTables_t* nextEntropy,
-                             const ZSTD_CCtx_params* cctxParams,
-                                   ZSTD_entropyCTablesMetadata_t* entropyMetadata,
-                                   void* workspace, size_t wkspSize)
-{
-    size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart;
+size_t ZSTD_buildBlockEntropyStats(
+            const seqStore_t* seqStorePtr,
+            const ZSTD_entropyCTables_t* prevEntropy,
+                  ZSTD_entropyCTables_t* nextEntropy,
+            const ZSTD_CCtx_params* cctxParams,
+                  ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+                  void* workspace, size_t wkspSize)
+{
+    size_t const litSize = (size_t)(seqStorePtr->lit - seqStorePtr->litStart);
+    int const huf_useOptDepth = (cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD);
+    int const hufFlags = huf_useOptDepth ? HUF_flags_optimalDepth : 0;
+
     entropyMetadata->hufMetadata.hufDesSize =
         ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize,
                                             &prevEntropy->huf, &nextEntropy->huf,
                                             &entropyMetadata->hufMetadata,
                                             ZSTD_literalsCompressionIsDisabled(cctxParams),
-                                            workspace, wkspSize);
+                                            workspace, wkspSize, hufFlags);
+
     FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed");
     entropyMetadata->fseMetadata.fseTablesSize =
         ZSTD_buildBlockEntropyStats_sequences(seqStorePtr,
@@ -3143,11 +3576,12 @@ size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr,
 }
 
 /* Returns the size estimate for the literals section (header + content) of a block */
-static size_t ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize,
-                                                const ZSTD_hufCTables_t* huf,
-                                                const ZSTD_hufCTablesMetadata_t* hufMetadata,
-                                                void* workspace, size_t wkspSize,
-                                                int writeEntropy)
+static size_t
+ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize,
+                               const ZSTD_hufCTables_t* huf,
+                               const ZSTD_hufCTablesMetadata_t* hufMetadata,
+                               void* workspace, size_t wkspSize,
+                               int writeEntropy)
 {
     unsigned* const countWksp = (unsigned*)workspace;
     unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX;
@@ -3169,12 +3603,13 @@ static size_t ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSiz
 }
 
 /* Returns the size estimate for the FSE-compressed symbols (of, ml, ll) of a block */
-static size_t ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type,
-                        const BYTE* codeTable, size_t nbSeq, unsigned maxCode,
-                        const FSE_CTable* fseCTable,
-                        const U8* additionalBits,
-                        short const* defaultNorm, U32 defaultNormLog, U32 defaultMax,
-                        void* workspace, size_t wkspSize)
+static size_t
+ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type,
+                    const BYTE* codeTable, size_t nbSeq, unsigned maxCode,
+                    const FSE_CTable* fseCTable,
+                    const U8* additionalBits,
+                    short const* defaultNorm, U32 defaultNormLog, U32 defaultMax,
+                    void* workspace, size_t wkspSize)
 {
     unsigned* const countWksp = (unsigned*)workspace;
     const BYTE* ctp = codeTable;
@@ -3206,99 +3641,107 @@ static size_t ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type,
 }
 
 /* Returns the size estimate for the sequences section (header + content) of a block */
-static size_t ZSTD_estimateBlockSize_sequences(const BYTE* ofCodeTable,
-                                                  const BYTE* llCodeTable,
-                                                  const BYTE* mlCodeTable,
-                                                  size_t nbSeq,
-                                                  const ZSTD_fseCTables_t* fseTables,
-                                                  const ZSTD_fseCTablesMetadata_t* fseMetadata,
-                                                  void* workspace, size_t wkspSize,
-                                                  int writeEntropy)
+static size_t
+ZSTD_estimateBlockSize_sequences(const BYTE* ofCodeTable,
+                                 const BYTE* llCodeTable,
+                                 const BYTE* mlCodeTable,
+                                 size_t nbSeq,
+                                 const ZSTD_fseCTables_t* fseTables,
+                                 const ZSTD_fseCTablesMetadata_t* fseMetadata,
+                                 void* workspace, size_t wkspSize,
+                                 int writeEntropy)
 {
     size_t sequencesSectionHeaderSize = 1 /* seqHead */ + 1 /* min seqSize size */ + (nbSeq >= 128) + (nbSeq >= LONGNBSEQ);
     size_t cSeqSizeEstimate = 0;
     cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, nbSeq, MaxOff,
-                                         fseTables->offcodeCTable, NULL,
-                                         OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
-                                         workspace, wkspSize);
+                                    fseTables->offcodeCTable, NULL,
+                                    OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
+                                    workspace, wkspSize);
     cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->llType, llCodeTable, nbSeq, MaxLL,
-                                         fseTables->litlengthCTable, LL_bits,
-                                         LL_defaultNorm, LL_defaultNormLog, MaxLL,
-                                         workspace, wkspSize);
+                                    fseTables->litlengthCTable, LL_bits,
+                                    LL_defaultNorm, LL_defaultNormLog, MaxLL,
+                                    workspace, wkspSize);
     cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, nbSeq, MaxML,
-                                         fseTables->matchlengthCTable, ML_bits,
-                                         ML_defaultNorm, ML_defaultNormLog, MaxML,
-                                         workspace, wkspSize);
+                                    fseTables->matchlengthCTable, ML_bits,
+                                    ML_defaultNorm, ML_defaultNormLog, MaxML,
+                                    workspace, wkspSize);
     if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize;
     return cSeqSizeEstimate + sequencesSectionHeaderSize;
 }
 
 /* Returns the size estimate for a given stream of literals, of, ll, ml */
-static size_t ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize,
-                                     const BYTE* ofCodeTable,
-                                     const BYTE* llCodeTable,
-                                     const BYTE* mlCodeTable,
-                                     size_t nbSeq,
-                                     const ZSTD_entropyCTables_t* entropy,
-                                     const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
-                                     void* workspace, size_t wkspSize,
-                                     int writeLitEntropy, int writeSeqEntropy) {
+static size_t
+ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize,
+                       const BYTE* ofCodeTable,
+                       const BYTE* llCodeTable,
+                       const BYTE* mlCodeTable,
+                       size_t nbSeq,
+                       const ZSTD_entropyCTables_t* entropy,
+                       const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+                       void* workspace, size_t wkspSize,
+                       int writeLitEntropy, int writeSeqEntropy)
+{
     size_t const literalsSize = ZSTD_estimateBlockSize_literal(literals, litSize,
-                                                         &entropy->huf, &entropyMetadata->hufMetadata,
-                                                         workspace, wkspSize, writeLitEntropy);
+                                    &entropy->huf, &entropyMetadata->hufMetadata,
+                                    workspace, wkspSize, writeLitEntropy);
     size_t const seqSize = ZSTD_estimateBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable,
-                                                         nbSeq, &entropy->fse, &entropyMetadata->fseMetadata,
-                                                         workspace, wkspSize, writeSeqEntropy);
+                                    nbSeq, &entropy->fse, &entropyMetadata->fseMetadata,
+                                    workspace, wkspSize, writeSeqEntropy);
     return seqSize + literalsSize + ZSTD_blockHeaderSize;
 }
 
 /* Builds entropy statistics and uses them for blocksize estimation.
  *
- * Returns the estimated compressed size of the seqStore, or a zstd error.
+ * @return: estimated compressed size of the seqStore, or a zstd error.
  */
-static size_t ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, ZSTD_CCtx* zc) {
-    ZSTD_entropyCTablesMetadata_t* entropyMetadata = &zc->blockSplitCtx.entropyMetadata;
+static size_t
+ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, ZSTD_CCtx* zc)
+{
+    ZSTD_entropyCTablesMetadata_t* const entropyMetadata = &zc->blockSplitCtx.entropyMetadata;
     DEBUGLOG(6, "ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()");
     FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(seqStore,
                     &zc->blockState.prevCBlock->entropy,
                     &zc->blockState.nextCBlock->entropy,
                     &zc->appliedParams,
                     entropyMetadata,
-                    zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), "");
-    return ZSTD_estimateBlockSize(seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart),
+                    zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE), "");
+    return ZSTD_estimateBlockSize(
+                    seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart),
                     seqStore->ofCode, seqStore->llCode, seqStore->mlCode,
                     (size_t)(seqStore->sequences - seqStore->sequencesStart),
-                    &zc->blockState.nextCBlock->entropy, entropyMetadata, zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE,
+                    &zc->blockState.nextCBlock->entropy,
+                    entropyMetadata,
+                    zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE,
                     (int)(entropyMetadata->hufMetadata.hType == set_compressed), 1);
 }
 
 /* Returns literals bytes represented in a seqStore */
-static size_t ZSTD_countSeqStoreLiteralsBytes(const seqStore_t* const seqStore) {
+static size_t ZSTD_countSeqStoreLiteralsBytes(const seqStore_t* const seqStore)
+{
     size_t literalsBytes = 0;
-    size_t const nbSeqs = seqStore->sequences - seqStore->sequencesStart;
+    size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
     size_t i;
     for (i = 0; i < nbSeqs; ++i) {
-        seqDef seq = seqStore->sequencesStart[i];
+        seqDef const seq = seqStore->sequencesStart[i];
         literalsBytes += seq.litLength;
         if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_literalLength) {
             literalsBytes += 0x10000;
-        }
-    }
+    }   }
     return literalsBytes;
 }
 
 /* Returns match bytes represented in a seqStore */
-static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore) {
+static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore)
+{
     size_t matchBytes = 0;
-    size_t const nbSeqs = seqStore->sequences - seqStore->sequencesStart;
+    size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart);
     size_t i;
     for (i = 0; i < nbSeqs; ++i) {
         seqDef seq = seqStore->sequencesStart[i];
         matchBytes += seq.mlBase + MINMATCH;
         if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_matchLength) {
             matchBytes += 0x10000;
-        }
-    }
+    }   }
     return matchBytes;
 }
 
@@ -3307,15 +3750,12 @@ static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore) {
  */
 static void ZSTD_deriveSeqStoreChunk(seqStore_t* resultSeqStore,
                                const seqStore_t* originalSeqStore,
-                                     size_t startIdx, size_t endIdx) {
-    BYTE* const litEnd = originalSeqStore->lit;
-    size_t literalsBytes;
-    size_t literalsBytesPreceding = 0;
-
+                                     size_t startIdx, size_t endIdx)
+{
     *resultSeqStore = *originalSeqStore;
     if (startIdx > 0) {
         resultSeqStore->sequences = originalSeqStore->sequencesStart + startIdx;
-        literalsBytesPreceding = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore);
+        resultSeqStore->litStart += ZSTD_countSeqStoreLiteralsBytes(resultSeqStore);
     }
 
     /* Move longLengthPos into the correct position if necessary */
@@ -3328,13 +3768,12 @@ static void ZSTD_deriveSeqStoreChunk(seqStore_t* resultSeqStore,
     }
     resultSeqStore->sequencesStart = originalSeqStore->sequencesStart + startIdx;
     resultSeqStore->sequences = originalSeqStore->sequencesStart + endIdx;
-    literalsBytes = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore);
-    resultSeqStore->litStart += literalsBytesPreceding;
     if (endIdx == (size_t)(originalSeqStore->sequences - originalSeqStore->sequencesStart)) {
         /* This accounts for possible last literals if the derived chunk reaches the end of the block */
-        resultSeqStore->lit = litEnd;
+        assert(resultSeqStore->lit == originalSeqStore->lit);
     } else {
-        resultSeqStore->lit = resultSeqStore->litStart+literalsBytes;
+        size_t const literalsBytes = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore);
+        resultSeqStore->lit = resultSeqStore->litStart + literalsBytes;
     }
     resultSeqStore->llCode += startIdx;
     resultSeqStore->mlCode += startIdx;
@@ -3342,20 +3781,26 @@ static void ZSTD_deriveSeqStoreChunk(seqStore_t* resultSeqStore,
 }
 
 /*
- * Returns the raw offset represented by the combination of offCode, ll0, and repcode history.
- * offCode must represent a repcode in the numeric representation of ZSTD_storeSeq().
+ * Returns the raw offset represented by the combination of offBase, ll0, and repcode history.
+ * offBase must represent a repcode in the numeric representation of ZSTD_storeSeq().
  */
 static U32
-ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offCode, const U32 ll0)
-{
-    U32 const adjustedOffCode = STORED_REPCODE(offCode) - 1 + ll0;  /* [ 0 - 3 ] */
-    assert(STORED_IS_REPCODE(offCode));
-    if (adjustedOffCode == ZSTD_REP_NUM) {
-        /* litlength == 0 and offCode == 2 implies selection of first repcode - 1 */
-        assert(rep[0] > 0);
+ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offBase, const U32 ll0)
+{
+    U32 const adjustedRepCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0;  /* [ 0 - 3 ] */
+    assert(OFFBASE_IS_REPCODE(offBase));
+    if (adjustedRepCode == ZSTD_REP_NUM) {
+        assert(ll0);
+        /* litlength == 0 and offCode == 2 implies selection of first repcode - 1
+         * This is only valid if it results in a valid offset value, aka > 0.
+         * Note : it may happen that `rep[0]==1` in exceptional circumstances.
+         * In which case this function will return 0, which is an invalid offset.
+         * It's not an issue though, since this value will be
+         * compared and discarded within ZSTD_seqStore_resolveOffCodes().
+         */
         return rep[0] - 1;
     }
-    return rep[adjustedOffCode];
+    return rep[adjustedRepCode];
 }
 
 /*
@@ -3371,30 +3816,33 @@ ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offCode, c
  *        1-3 : repcode 1-3
  *        4+ : real_offset+3
  */
-static void ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes,
-                                          seqStore_t* const seqStore, U32 const nbSeq) {
+static void
+ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes,
+                        const seqStore_t* const seqStore, U32 const nbSeq)
+{
     U32 idx = 0;
+    U32 const longLitLenIdx = seqStore->longLengthType == ZSTD_llt_literalLength ? seqStore->longLengthPos : nbSeq;
     for (; idx < nbSeq; ++idx) {
         seqDef* const seq = seqStore->sequencesStart + idx;
-        U32 const ll0 = (seq->litLength == 0);
-        U32 const offCode = OFFBASE_TO_STORED(seq->offBase);
-        assert(seq->offBase > 0);
-        if (STORED_IS_REPCODE(offCode)) {
-            U32 const dRawOffset = ZSTD_resolveRepcodeToRawOffset(dRepcodes->rep, offCode, ll0);
-            U32 const cRawOffset = ZSTD_resolveRepcodeToRawOffset(cRepcodes->rep, offCode, ll0);
+        U32 const ll0 = (seq->litLength == 0) && (idx != longLitLenIdx);
+        U32 const offBase = seq->offBase;
+        assert(offBase > 0);
+        if (OFFBASE_IS_REPCODE(offBase)) {
+            U32 const dRawOffset = ZSTD_resolveRepcodeToRawOffset(dRepcodes->rep, offBase, ll0);
+            U32 const cRawOffset = ZSTD_resolveRepcodeToRawOffset(cRepcodes->rep, offBase, ll0);
             /* Adjust simulated decompression repcode history if we come across a mismatch. Replace
              * the repcode with the offset it actually references, determined by the compression
              * repcode history.
              */
             if (dRawOffset != cRawOffset) {
-                seq->offBase = cRawOffset + ZSTD_REP_NUM;
+                seq->offBase = OFFSET_TO_OFFBASE(cRawOffset);
             }
         }
         /* Compression repcode history is always updated with values directly from the unmodified seqStore.
          * Decompression repcode history may use modified seq->offset value taken from compression repcode history.
          */
-        ZSTD_updateRep(dRepcodes->rep, OFFBASE_TO_STORED(seq->offBase), ll0);
-        ZSTD_updateRep(cRepcodes->rep, offCode, ll0);
+        ZSTD_updateRep(dRepcodes->rep, seq->offBase, ll0);
+        ZSTD_updateRep(cRepcodes->rep, offBase, ll0);
     }
 }
 
@@ -3404,10 +3852,11 @@ static void ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_
  * Returns the total size of that block (including header) or a ZSTD error code.
  */
 static size_t
-ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc, seqStore_t* const seqStore,
+ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc,
+                            const seqStore_t* const seqStore,
                                   repcodes_t* const dRep, repcodes_t* const cRep,
                                   void* dst, size_t dstCapacity,
-                                  const void* src, size_t srcSize,
+                            const void* src, size_t srcSize,
                                   U32 lastBlock, U32 isPartition)
 {
     const U32 rleMaxLength = 25;
@@ -3481,45 +3930,49 @@ typedef struct {
 
 /* Helper function to perform the recursive search for block splits.
  * Estimates the cost of seqStore prior to split, and estimates the cost of splitting the sequences in half.
- * If advantageous to split, then we recurse down the two sub-blocks. If not, or if an error occurred in estimation, then
- * we do not recurse.
+ * If advantageous to split, then we recurse down the two sub-blocks.
+ * If not, or if an error occurred in estimation, then we do not recurse.
  *
- * Note: The recursion depth is capped by a heuristic minimum number of sequences, defined by MIN_SEQUENCES_BLOCK_SPLITTING.
+ * Note: The recursion depth is capped by a heuristic minimum number of sequences,
+ * defined by MIN_SEQUENCES_BLOCK_SPLITTING.
  * In theory, this means the absolute largest recursion depth is 10 == log2(maxNbSeqInBlock/MIN_SEQUENCES_BLOCK_SPLITTING).
  * In practice, recursion depth usually doesn't go beyond 4.
  *
- * Furthermore, the number of splits is capped by ZSTD_MAX_NB_BLOCK_SPLITS. At ZSTD_MAX_NB_BLOCK_SPLITS == 196 with the current existing blockSize
+ * Furthermore, the number of splits is capped by ZSTD_MAX_NB_BLOCK_SPLITS.
+ * At ZSTD_MAX_NB_BLOCK_SPLITS == 196 with the current existing blockSize
  * maximum of 128 KB, this value is actually impossible to reach.
  */
 static void
 ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t endIdx,
                              ZSTD_CCtx* zc, const seqStore_t* origSeqStore)
 {
-    seqStore_t* fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk;
-    seqStore_t* firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore;
-    seqStore_t* secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore;
+    seqStore_t* const fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk;
+    seqStore_t* const firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore;
+    seqStore_t* const secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore;
     size_t estimatedOriginalSize;
     size_t estimatedFirstHalfSize;
     size_t estimatedSecondHalfSize;
     size_t midIdx = (startIdx + endIdx)/2;
 
+    DEBUGLOG(5, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx);
+    assert(endIdx >= startIdx);
     if (endIdx - startIdx < MIN_SEQUENCES_BLOCK_SPLITTING || splits->idx >= ZSTD_MAX_NB_BLOCK_SPLITS) {
-        DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences");
+        DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences (%zu)", endIdx - startIdx);
         return;
     }
-    DEBUGLOG(4, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx);
     ZSTD_deriveSeqStoreChunk(fullSeqStoreChunk, origSeqStore, startIdx, endIdx);
     ZSTD_deriveSeqStoreChunk(firstHalfSeqStore, origSeqStore, startIdx, midIdx);
     ZSTD_deriveSeqStoreChunk(secondHalfSeqStore, origSeqStore, midIdx, endIdx);
     estimatedOriginalSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(fullSeqStoreChunk, zc);
     estimatedFirstHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(firstHalfSeqStore, zc);
     estimatedSecondHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(secondHalfSeqStore, zc);
-    DEBUGLOG(4, "Estimated original block size: %zu -- First half split: %zu -- Second half split: %zu",
+    DEBUGLOG(5, "Estimated original block size: %zu -- First half split: %zu -- Second half split: %zu",
              estimatedOriginalSize, estimatedFirstHalfSize, estimatedSecondHalfSize);
     if (ZSTD_isError(estimatedOriginalSize) || ZSTD_isError(estimatedFirstHalfSize) || ZSTD_isError(estimatedSecondHalfSize)) {
         return;
     }
     if (estimatedFirstHalfSize + estimatedSecondHalfSize < estimatedOriginalSize) {
+        DEBUGLOG(5, "split decided at seqNb:%zu", midIdx);
         ZSTD_deriveBlockSplitsHelper(splits, startIdx, midIdx, zc, origSeqStore);
         splits->splitLocations[splits->idx] = (U32)midIdx;
         splits->idx++;
@@ -3527,14 +3980,18 @@ ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t end
     }
 }
 
-/* Base recursive function. Populates a table with intra-block partition indices that can improve compression ratio.
+/* Base recursive function.
+ * Populates a table with intra-block partition indices that can improve compression ratio.
  *
- * Returns the number of splits made (which equals the size of the partition table - 1).
+ * @return: number of splits made (which equals the size of the partition table - 1).
  */
-static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq) {
-    seqStoreSplits splits = {partitions, 0};
+static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq)
+{
+    seqStoreSplits splits;
+    splits.splitLocations = partitions;
+    splits.idx = 0;
     if (nbSeq <= 4) {
-        DEBUGLOG(4, "ZSTD_deriveBlockSplits: Too few sequences to split");
+        DEBUGLOG(5, "ZSTD_deriveBlockSplits: Too few sequences to split (%u <= 4)", nbSeq);
         /* Refuse to try and split anything with less than 4 sequences */
         return 0;
     }
@@ -3550,18 +4007,20 @@ static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq)
  * Returns combined size of all blocks (which includes headers), or a ZSTD error code.
  */
 static size_t
-ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity,
-                                       const void* src, size_t blockSize, U32 lastBlock, U32 nbSeq)
+ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc,
+                                    void* dst, size_t dstCapacity,
+                              const void* src, size_t blockSize,
+                                    U32 lastBlock, U32 nbSeq)
 {
     size_t cSize = 0;
     const BYTE* ip = (const BYTE*)src;
     BYTE* op = (BYTE*)dst;
     size_t i = 0;
     size_t srcBytesTotal = 0;
-    U32* partitions = zc->blockSplitCtx.partitions; /* size == ZSTD_MAX_NB_BLOCK_SPLITS */
-    seqStore_t* nextSeqStore = &zc->blockSplitCtx.nextSeqStore;
-    seqStore_t* currSeqStore = &zc->blockSplitCtx.currSeqStore;
-    size_t numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq);
+    U32* const partitions = zc->blockSplitCtx.partitions; /* size == ZSTD_MAX_NB_BLOCK_SPLITS */
+    seqStore_t* const nextSeqStore = &zc->blockSplitCtx.nextSeqStore;
+    seqStore_t* const currSeqStore = &zc->blockSplitCtx.currSeqStore;
+    size_t const numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq);
 
     /* If a block is split and some partitions are emitted as RLE/uncompressed, then repcode history
      * may become invalid. In order to reconcile potentially invalid repcodes, we keep track of two
@@ -3583,30 +4042,31 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapac
     ZSTD_memcpy(cRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t));
     ZSTD_memset(nextSeqStore, 0, sizeof(seqStore_t));
 
-    DEBUGLOG(4, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
+    DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
                 (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit,
                 (unsigned)zc->blockState.matchState.nextToUpdate);
 
     if (numSplits == 0) {
-        size_t cSizeSingleBlock = ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore,
-                                                                   &dRep, &cRep,
-                                                                    op, dstCapacity,
-                                                                    ip, blockSize,
-                                                                    lastBlock, 0 /* isPartition */);
+        size_t cSizeSingleBlock =
+            ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore,
+                                            &dRep, &cRep,
+                                            op, dstCapacity,
+                                            ip, blockSize,
+                                            lastBlock, 0 /* isPartition */);
         FORWARD_IF_ERROR(cSizeSingleBlock, "Compressing single block from splitBlock_internal() failed!");
         DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal: No splits");
-        assert(cSizeSingleBlock <= ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize);
+        assert(zc->blockSize <= ZSTD_BLOCKSIZE_MAX);
+        assert(cSizeSingleBlock <= zc->blockSize + ZSTD_blockHeaderSize);
         return cSizeSingleBlock;
     }
 
     ZSTD_deriveSeqStoreChunk(currSeqStore, &zc->seqStore, 0, partitions[0]);
     for (i = 0; i <= numSplits; ++i) {
-        size_t srcBytes;
         size_t cSizeChunk;
         U32 const lastPartition = (i == numSplits);
         U32 lastBlockEntireSrc = 0;
 
-        srcBytes = ZSTD_countSeqStoreLiteralsBytes(currSeqStore) + ZSTD_countSeqStoreMatchBytes(currSeqStore);
+        size_t srcBytes = ZSTD_countSeqStoreLiteralsBytes(currSeqStore) + ZSTD_countSeqStoreMatchBytes(currSeqStore);
         srcBytesTotal += srcBytes;
         if (lastPartition) {
             /* This is the final partition, need to account for possible last literals */
@@ -3621,7 +4081,8 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapac
                                                        op, dstCapacity,
                                                        ip, srcBytes,
                                                        lastBlockEntireSrc, 1 /* isPartition */);
-        DEBUGLOG(5, "Estimated size: %zu actual size: %zu", ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(currSeqStore, zc), cSizeChunk);
+        DEBUGLOG(5, "Estimated size: %zu vs %zu : actual size",
+                    ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(currSeqStore, zc), cSizeChunk);
         FORWARD_IF_ERROR(cSizeChunk, "Compressing chunk failed!");
 
         ip += srcBytes;
@@ -3629,10 +4090,10 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapac
         dstCapacity -= cSizeChunk;
         cSize += cSizeChunk;
         *currSeqStore = *nextSeqStore;
-        assert(cSizeChunk <= ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize);
+        assert(cSizeChunk <= zc->blockSize + ZSTD_blockHeaderSize);
     }
-    /* cRep and dRep may have diverged during the compression. If so, we use the dRep repcodes
-     * for the next block.
+    /* cRep and dRep may have diverged during the compression.
+     * If so, we use the dRep repcodes for the next block.
      */
     ZSTD_memcpy(zc->blockState.prevCBlock->rep, dRep.rep, sizeof(repcodes_t));
     return cSize;
@@ -3643,8 +4104,6 @@ ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc,
                               void* dst, size_t dstCapacity,
                               const void* src, size_t srcSize, U32 lastBlock)
 {
-    const BYTE* ip = (const BYTE*)src;
-    BYTE* op = (BYTE*)dst;
     U32 nbSeq;
     size_t cSize;
     DEBUGLOG(4, "ZSTD_compressBlock_splitBlock");
@@ -3655,7 +4114,7 @@ ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc,
         if (bss == ZSTDbss_noCompress) {
             if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
                 zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
-            cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock);
+            cSize = ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock);
             FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed");
             DEBUGLOG(4, "ZSTD_compressBlock_splitBlock: Nocompress block");
             return cSize;
@@ -3673,9 +4132,9 @@ ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
                             void* dst, size_t dstCapacity,
                             const void* src, size_t srcSize, U32 frame)
 {
-    /* This the upper bound for the length of an rle block.
-     * This isn't the actual upper bound. Finding the real threshold
-     * needs further investigation.
+    /* This is an estimated upper bound for the length of an rle block.
+     * This isn't the actual upper bound.
+     * Finding the real threshold needs further investigation.
      */
     const U32 rleMaxLength = 25;
     size_t cSize;
@@ -3767,10 +4226,11 @@ static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc,
          *   * cSize >= blockBound(srcSize): We have expanded the block too much so
          *     emit an uncompressed block.
          */
-        {
-            size_t const cSize = ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock);
+        {   size_t const cSize =
+                ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock);
             if (cSize != ERROR(dstSize_tooSmall)) {
-                size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy);
+                size_t const maxCSize =
+                    srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy);
                 FORWARD_IF_ERROR(cSize, "ZSTD_compressSuperBlock failed");
                 if (cSize != 0 && cSize < maxCSize + ZSTD_blockHeaderSize) {
                     ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
@@ -3778,7 +4238,7 @@ static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc,
                 }
             }
         }
-    }
+    } /* if (bss == ZSTDbss_compress)*/
 
     DEBUGLOG(6, "Resorting to ZSTD_noCompressBlock()");
     /* Superblock compression failed, attempt to emit a single no compress block.
@@ -3836,7 +4296,7 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms,
 *   All blocks will be terminated, all input will be consumed.
 *   Function will issue an error if there is not enough `dstCapacity` to hold the compressed content.
 *   Frame is supposed already started (header already produced)
-*   @return : compressed size, or an error code
+*  @return : compressed size, or an error code
 */
 static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx,
                                      void* dst, size_t dstCapacity,
@@ -3860,7 +4320,9 @@ static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx,
         ZSTD_matchState_t* const ms = &cctx->blockState.matchState;
         U32 const lastBlock = lastFrameChunk & (blockSize >= remaining);
 
-        RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE,
+        /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding
+         * additional 1. We need to revisit and change this logic to be more consistent */
+        RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE + 1,
                         dstSize_tooSmall,
                         "not enough space to store compressed block");
         if (remaining < blockSize) blockSize = remaining;
@@ -3899,7 +4361,7 @@ static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx,
                     MEM_writeLE24(op, cBlockHeader);
                     cSize += ZSTD_blockHeaderSize;
                 }
-            }
+            }  /* if (ZSTD_useTargetCBlockSize(&cctx->appliedParams))*/
 
 
             ip += blockSize;
@@ -4078,31 +4540,51 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
     }
 }
 
-size_t ZSTD_compressContinue (ZSTD_CCtx* cctx,
-                              void* dst, size_t dstCapacity,
-                        const void* src, size_t srcSize)
+size_t ZSTD_compressContinue_public(ZSTD_CCtx* cctx,
+                                        void* dst, size_t dstCapacity,
+                                  const void* src, size_t srcSize)
 {
     DEBUGLOG(5, "ZSTD_compressContinue (srcSize=%u)", (unsigned)srcSize);
     return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */);
 }
 
+/* NOTE: Must just wrap ZSTD_compressContinue_public() */
+size_t ZSTD_compressContinue(ZSTD_CCtx* cctx,
+                             void* dst, size_t dstCapacity,
+                       const void* src, size_t srcSize)
+{
+    return ZSTD_compressContinue_public(cctx, dst, dstCapacity, src, srcSize);
+}
 
-size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
+static size_t ZSTD_getBlockSize_deprecated(const ZSTD_CCtx* cctx)
 {
     ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams;
     assert(!ZSTD_checkCParams(cParams));
-    return MIN (ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog);
+    return MIN(cctx->appliedParams.maxBlockSize, (size_t)1 << cParams.windowLog);
 }
 
-size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+/* NOTE: Must just wrap ZSTD_getBlockSize_deprecated() */
+size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx)
+{
+    return ZSTD_getBlockSize_deprecated(cctx);
+}
+
+/* NOTE: Must just wrap ZSTD_compressBlock_deprecated() */
+size_t ZSTD_compressBlock_deprecated(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
     DEBUGLOG(5, "ZSTD_compressBlock: srcSize = %u", (unsigned)srcSize);
-    { size_t const blockSizeMax = ZSTD_getBlockSize(cctx);
+    { size_t const blockSizeMax = ZSTD_getBlockSize_deprecated(cctx);
       RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong, "input is larger than a block"); }
 
     return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */);
 }
 
+/* NOTE: Must just wrap ZSTD_compressBlock_deprecated() */
+size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize)
+{
+    return ZSTD_compressBlock_deprecated(cctx, dst, dstCapacity, src, srcSize);
+}
+
 /*! ZSTD_loadDictionaryContent() :
  *  @return : 0, or an error code
  */
@@ -4111,25 +4593,36 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
                                          ZSTD_cwksp* ws,
                                          ZSTD_CCtx_params const* params,
                                          const void* src, size_t srcSize,
-                                         ZSTD_dictTableLoadMethod_e dtlm)
+                                         ZSTD_dictTableLoadMethod_e dtlm,
+                                         ZSTD_tableFillPurpose_e tfp)
 {
     const BYTE* ip = (const BYTE*) src;
     const BYTE* const iend = ip + srcSize;
     int const loadLdmDict = params->ldmParams.enableLdm == ZSTD_ps_enable && ls != NULL;
 
-    /* Assert that we the ms params match the params we're being given */
+    /* Assert that the ms params match the params we're being given */
     ZSTD_assertEqualCParams(params->cParams, ms->cParams);
 
-    if (srcSize > ZSTD_CHUNKSIZE_MAX) {
+    {   /* Ensure large dictionaries can't cause index overflow */
+
         /* Allow the dictionary to set indices up to exactly ZSTD_CURRENT_MAX.
          * Dictionaries right at the edge will immediately trigger overflow
          * correction, but I don't want to insert extra constraints here.
          */
-        U32 const maxDictSize = ZSTD_CURRENT_MAX - 1;
-        /* We must have cleared our windows when our source is this large. */
-        assert(ZSTD_window_isEmpty(ms->window));
-        if (loadLdmDict)
-            assert(ZSTD_window_isEmpty(ls->window));
+        U32 maxDictSize = ZSTD_CURRENT_MAX - ZSTD_WINDOW_START_INDEX;
+
+        int const CDictTaggedIndices = ZSTD_CDictIndicesAreTagged(&params->cParams);
+        if (CDictTaggedIndices && tfp == ZSTD_tfp_forCDict) {
+            /* Some dictionary matchfinders in zstd use "short cache",
+             * which treats the lower ZSTD_SHORT_CACHE_TAG_BITS of each
+             * CDict hashtable entry as a tag rather than as part of an index.
+             * When short cache is used, we need to truncate the dictionary
+             * so that its indices don't overlap with the tag. */
+            U32 const shortCacheMaxDictSize = (1u << (32 - ZSTD_SHORT_CACHE_TAG_BITS)) - ZSTD_WINDOW_START_INDEX;
+            maxDictSize = MIN(maxDictSize, shortCacheMaxDictSize);
+            assert(!loadLdmDict);
+        }
+
         /* If the dictionary is too large, only load the suffix of the dictionary. */
         if (srcSize > maxDictSize) {
             ip = iend - maxDictSize;
@@ -4138,30 +4631,46 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
         }
     }
 
-    DEBUGLOG(4, "ZSTD_loadDictionaryContent(): useRowMatchFinder=%d", (int)params->useRowMatchFinder);
+    if (srcSize > ZSTD_CHUNKSIZE_MAX) {
+        /* We must have cleared our windows when our source is this large. */
+        assert(ZSTD_window_isEmpty(ms->window));
+        if (loadLdmDict) assert(ZSTD_window_isEmpty(ls->window));
+    }
     ZSTD_window_update(&ms->window, src, srcSize, /* forceNonContiguous */ 0);
-    ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base);
-    ms->forceNonContiguous = params->deterministicRefPrefix;
 
-    if (loadLdmDict) {
+    DEBUGLOG(4, "ZSTD_loadDictionaryContent(): useRowMatchFinder=%d", (int)params->useRowMatchFinder);
+
+    if (loadLdmDict) { /* Load the entire dict into LDM matchfinders. */
         ZSTD_window_update(&ls->window, src, srcSize, /* forceNonContiguous */ 0);
         ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base);
+        ZSTD_ldm_fillHashTable(ls, ip, iend, &params->ldmParams);
+    }
+
+    /* If the dict is larger than we can reasonably index in our tables, only load the suffix. */
+    if (params->cParams.strategy < ZSTD_btultra) {
+        U32 maxDictSize = 8U << MIN(MAX(params->cParams.hashLog, params->cParams.chainLog), 28);
+        if (srcSize > maxDictSize) {
+            ip = iend - maxDictSize;
+            src = ip;
+            srcSize = maxDictSize;
+        }
     }
 
+    ms->nextToUpdate = (U32)(ip - ms->window.base);
+    ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base);
+    ms->forceNonContiguous = params->deterministicRefPrefix;
+
     if (srcSize <= HASH_READ_SIZE) return 0;
 
     ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, iend);
 
-    if (loadLdmDict)
-        ZSTD_ldm_fillHashTable(ls, ip, iend, &params->ldmParams);
-
     switch(params->cParams.strategy)
     {
     case ZSTD_fast:
-        ZSTD_fillHashTable(ms, iend, dtlm);
+        ZSTD_fillHashTable(ms, iend, dtlm, tfp);
         break;
     case ZSTD_dfast:
-        ZSTD_fillDoubleHashTable(ms, iend, dtlm);
+        ZSTD_fillDoubleHashTable(ms, iend, dtlm, tfp);
         break;
 
     case ZSTD_greedy:
@@ -4174,7 +4683,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
         } else {
             assert(params->useRowMatchFinder != ZSTD_ps_auto);
             if (params->useRowMatchFinder == ZSTD_ps_enable) {
-                size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog) * sizeof(U16);
+                size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog);
                 ZSTD_memset(ms->tagTable, 0, tagTableSize);
                 ZSTD_row_update(ms, iend-HASH_READ_SIZE);
                 DEBUGLOG(4, "Using row-based hash table for lazy dict");
@@ -4327,6 +4836,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
                                       ZSTD_CCtx_params const* params,
                                       const void* dict, size_t dictSize,
                                       ZSTD_dictTableLoadMethod_e dtlm,
+                                      ZSTD_tableFillPurpose_e tfp,
                                       void* workspace)
 {
     const BYTE* dictPtr = (const BYTE*)dict;
@@ -4345,7 +4855,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
     {
         size_t const dictContentSize = (size_t)(dictEnd - dictPtr);
         FORWARD_IF_ERROR(ZSTD_loadDictionaryContent(
-            ms, NULL, ws, params, dictPtr, dictContentSize, dtlm), "");
+            ms, NULL, ws, params, dictPtr, dictContentSize, dtlm, tfp), "");
     }
     return dictID;
 }
@@ -4361,6 +4871,7 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs,
                          const void* dict, size_t dictSize,
                                ZSTD_dictContentType_e dictContentType,
                                ZSTD_dictTableLoadMethod_e dtlm,
+                               ZSTD_tableFillPurpose_e tfp,
                                void* workspace)
 {
     DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize);
@@ -4373,13 +4884,13 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs,
 
     /* dict restricted modes */
     if (dictContentType == ZSTD_dct_rawContent)
-        return ZSTD_loadDictionaryContent(ms, ls, ws, params, dict, dictSize, dtlm);
+        return ZSTD_loadDictionaryContent(ms, ls, ws, params, dict, dictSize, dtlm, tfp);
 
     if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) {
         if (dictContentType == ZSTD_dct_auto) {
             DEBUGLOG(4, "raw content dictionary detected");
             return ZSTD_loadDictionaryContent(
-                ms, ls, ws, params, dict, dictSize, dtlm);
+                ms, ls, ws, params, dict, dictSize, dtlm, tfp);
         }
         RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, "");
         assert(0);   /* impossible */
@@ -4387,13 +4898,14 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs,
 
     /* dict as full zstd dictionary */
     return ZSTD_loadZstdDictionary(
-        bs, ms, ws, params, dict, dictSize, dtlm, workspace);
+        bs, ms, ws, params, dict, dictSize, dtlm, tfp, workspace);
 }
 
 #define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB)
 #define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6ULL)
 
 /*! ZSTD_compressBegin_internal() :
+ * Assumption : either @dict OR @cdict (or none) is non-NULL, never both
  * @return : 0, or an error code */
 static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
                                     const void* dict, size_t dictSize,
@@ -4426,11 +4938,11 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
                         cctx->blockState.prevCBlock, &cctx->blockState.matchState,
                         &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent,
                         cdict->dictContentSize, cdict->dictContentType, dtlm,
-                        cctx->entropyWorkspace)
+                        ZSTD_tfp_forCCtx, cctx->entropyWorkspace)
               : ZSTD_compress_insertDictionary(
                         cctx->blockState.prevCBlock, &cctx->blockState.matchState,
                         &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize,
-                        dictContentType, dtlm, cctx->entropyWorkspace);
+                        dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->entropyWorkspace);
         FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed");
         assert(dictID <= UINT_MAX);
         cctx->dictID = (U32)dictID;
@@ -4471,11 +4983,11 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
                                             &cctxParams, pledgedSrcSize);
 }
 
-size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
+static size_t
+ZSTD_compressBegin_usingDict_deprecated(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
 {
     ZSTD_CCtx_params cctxParams;
-    {
-        ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict);
+    {   ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict);
         ZSTD_CCtxParams_init_internal(&cctxParams, &params, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel);
     }
     DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize);
@@ -4483,9 +4995,15 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t di
                                        &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered);
 }
 
+size_t
+ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel)
+{
+    return ZSTD_compressBegin_usingDict_deprecated(cctx, dict, dictSize, compressionLevel);
+}
+
 size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel)
 {
-    return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel);
+    return ZSTD_compressBegin_usingDict_deprecated(cctx, NULL, 0, compressionLevel);
 }
 
 
@@ -4537,9 +5055,9 @@ void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize)
     (void)extraCSize;
 }
 
-size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
-                         void* dst, size_t dstCapacity,
-                   const void* src, size_t srcSize)
+size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx,
+                               void* dst, size_t dstCapacity,
+                         const void* src, size_t srcSize)
 {
     size_t endResult;
     size_t const cSize = ZSTD_compressContinue_internal(cctx,
@@ -4563,6 +5081,14 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx,
     return cSize + endResult;
 }
 
+/* NOTE: Must just wrap ZSTD_compressEnd_public() */
+size_t ZSTD_compressEnd(ZSTD_CCtx* cctx,
+                        void* dst, size_t dstCapacity,
+                  const void* src, size_t srcSize)
+{
+    return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize);
+}
+
 size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx,
                                void* dst, size_t dstCapacity,
                          const void* src, size_t srcSize,
@@ -4591,7 +5117,7 @@ size_t ZSTD_compress_advanced_internal(
     FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx,
                          dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL,
                          params, srcSize, ZSTDb_not_buffered) , "");
-    return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
+    return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize);
 }
 
 size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx,
@@ -4709,7 +5235,7 @@ static size_t ZSTD_initCDict_internal(
         {   size_t const dictID = ZSTD_compress_insertDictionary(
                     &cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace,
                     &params, cdict->dictContent, cdict->dictContentSize,
-                    dictContentType, ZSTD_dtlm_full, cdict->entropyWorkspace);
+                    dictContentType, ZSTD_dtlm_full, ZSTD_tfp_forCDict, cdict->entropyWorkspace);
             FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed");
             assert(dictID <= (size_t)(U32)-1);
             cdict->dictID = (U32)dictID;
@@ -4906,6 +5432,7 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
     params.cParams = cParams;
     params.useRowMatchFinder = useRowMatchFinder;
     cdict->useRowMatchFinder = useRowMatchFinder;
+    cdict->compressionLevel = ZSTD_NO_CLEVEL;
 
     if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
                                               dict, dictSize,
@@ -4985,12 +5512,17 @@ size_t ZSTD_compressBegin_usingCDict_advanced(
 
 /* ZSTD_compressBegin_usingCDict() :
  * cdict must be != NULL */
-size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
+size_t ZSTD_compressBegin_usingCDict_deprecated(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
 {
     ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ };
     return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN);
 }
 
+size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict)
+{
+    return ZSTD_compressBegin_usingCDict_deprecated(cctx, cdict);
+}
+
 /*! ZSTD_compress_usingCDict_internal():
  * Implementation of various ZSTD_compress_usingCDict* functions.
  */
@@ -5000,7 +5532,7 @@ static size_t ZSTD_compress_usingCDict_internal(ZSTD_CCtx* cctx,
                                 const ZSTD_CDict* cdict, ZSTD_frameParameters fParams)
 {
     FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */
-    return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
+    return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize);
 }
 
 /*! ZSTD_compress_usingCDict_advanced():
@@ -5197,30 +5729,41 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
 
 static size_t ZSTD_nextInputSizeHint(const ZSTD_CCtx* cctx)
 {
-    size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos;
-    if (hintInSize==0) hintInSize = cctx->blockSize;
-    return hintInSize;
+    if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) {
+        return cctx->blockSize - cctx->stableIn_notConsumed;
+    }
+    assert(cctx->appliedParams.inBufferMode == ZSTD_bm_buffered);
+    {   size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos;
+        if (hintInSize==0) hintInSize = cctx->blockSize;
+        return hintInSize;
+    }
 }
 
 /* ZSTD_compressStream_generic():
  *  internal function for all *compressStream*() variants
- *  non-static, because can be called from zstdmt_compress.c
- * @return : hint size for next input */
+ * @return : hint size for next input to complete ongoing block */
 static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                                           ZSTD_outBuffer* output,
                                           ZSTD_inBuffer* input,
                                           ZSTD_EndDirective const flushMode)
 {
-    const char* const istart = (const char*)input->src;
-    const char* const iend = input->size != 0 ? istart + input->size : istart;
-    const char* ip = input->pos != 0 ? istart + input->pos : istart;
-    char* const ostart = (char*)output->dst;
-    char* const oend = output->size != 0 ? ostart + output->size : ostart;
-    char* op = output->pos != 0 ? ostart + output->pos : ostart;
+    const char* const istart = (assert(input != NULL), (const char*)input->src);
+    const char* const iend = (istart != NULL) ? istart + input->size : istart;
+    const char* ip = (istart != NULL) ? istart + input->pos : istart;
+    char* const ostart = (assert(output != NULL), (char*)output->dst);
+    char* const oend = (ostart != NULL) ? ostart + output->size : ostart;
+    char* op = (ostart != NULL) ? ostart + output->pos : ostart;
     U32 someMoreWork = 1;
 
     /* check expectations */
-    DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%u", (unsigned)flushMode);
+    DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%i, srcSize = %zu", (int)flushMode, input->size - input->pos);
+    assert(zcs != NULL);
+    if (zcs->appliedParams.inBufferMode == ZSTD_bm_stable) {
+        assert(input->pos >= zcs->stableIn_notConsumed);
+        input->pos -= zcs->stableIn_notConsumed;
+        ip -= zcs->stableIn_notConsumed;
+        zcs->stableIn_notConsumed = 0;
+    }
     if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) {
         assert(zcs->inBuff != NULL);
         assert(zcs->inBuffSize > 0);
@@ -5229,8 +5772,10 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
         assert(zcs->outBuff !=  NULL);
         assert(zcs->outBuffSize > 0);
     }
-    assert(output->pos <= output->size);
+    if (input->src == NULL) assert(input->size == 0);
     assert(input->pos <= input->size);
+    if (output->dst == NULL) assert(output->size == 0);
+    assert(output->pos <= output->size);
     assert((U32)flushMode <= (U32)ZSTD_e_end);
 
     while (someMoreWork) {
@@ -5245,7 +5790,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                 || zcs->appliedParams.outBufferMode == ZSTD_bm_stable)  /* OR we are allowed to return dstSizeTooSmall */
               && (zcs->inBuffPos == 0) ) {
                 /* shortcut to compression pass directly into output buffer */
-                size_t const cSize = ZSTD_compressEnd(zcs,
+                size_t const cSize = ZSTD_compressEnd_public(zcs,
                                                 op, oend-op, ip, iend-ip);
                 DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize);
                 FORWARD_IF_ERROR(cSize, "ZSTD_compressEnd failed");
@@ -5262,8 +5807,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                                         zcs->inBuff + zcs->inBuffPos, toLoad,
                                         ip, iend-ip);
                 zcs->inBuffPos += loaded;
-                if (loaded != 0)
-                    ip += loaded;
+                if (ip) ip += loaded;
                 if ( (flushMode == ZSTD_e_continue)
                   && (zcs->inBuffPos < zcs->inBuffTarget) ) {
                     /* not enough input to fill full block : stop here */
@@ -5274,6 +5818,20 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                     /* empty */
                     someMoreWork = 0; break;
                 }
+            } else {
+                assert(zcs->appliedParams.inBufferMode == ZSTD_bm_stable);
+                if ( (flushMode == ZSTD_e_continue)
+                  && ( (size_t)(iend - ip) < zcs->blockSize) ) {
+                    /* can't compress a full block : stop here */
+                    zcs->stableIn_notConsumed = (size_t)(iend - ip);
+                    ip = iend;  /* pretend to have consumed input */
+                    someMoreWork = 0; break;
+                }
+                if ( (flushMode == ZSTD_e_flush)
+                  && (ip == iend) ) {
+                    /* empty */
+                    someMoreWork = 0; break;
+                }
             }
             /* compress current block (note : this stage cannot be stopped in the middle) */
             DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode);
@@ -5281,9 +5839,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                 void* cDst;
                 size_t cSize;
                 size_t oSize = oend-op;
-                size_t const iSize = inputBuffered
-                    ? zcs->inBuffPos - zcs->inToCompress
-                    : MIN((size_t)(iend - ip), zcs->blockSize);
+                size_t const iSize = inputBuffered ? zcs->inBuffPos - zcs->inToCompress
+                                                   : MIN((size_t)(iend - ip), zcs->blockSize);
                 if (oSize >= ZSTD_compressBound(iSize) || zcs->appliedParams.outBufferMode == ZSTD_bm_stable)
                     cDst = op;   /* compress into output buffer, to skip flush stage */
                 else
@@ -5291,9 +5848,9 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                 if (inputBuffered) {
                     unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend);
                     cSize = lastBlock ?
-                            ZSTD_compressEnd(zcs, cDst, oSize,
+                            ZSTD_compressEnd_public(zcs, cDst, oSize,
                                         zcs->inBuff + zcs->inToCompress, iSize) :
-                            ZSTD_compressContinue(zcs, cDst, oSize,
+                            ZSTD_compressContinue_public(zcs, cDst, oSize,
                                         zcs->inBuff + zcs->inToCompress, iSize);
                     FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed");
                     zcs->frameEnded = lastBlock;
@@ -5306,19 +5863,16 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
                     if (!lastBlock)
                         assert(zcs->inBuffTarget <= zcs->inBuffSize);
                     zcs->inToCompress = zcs->inBuffPos;
-                } else {
-                    unsigned const lastBlock = (ip + iSize == iend);
-                    assert(flushMode == ZSTD_e_end /* Already validated */);
+                } else { /* !inputBuffered, hence ZSTD_bm_stable */
+                    unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip + iSize == iend);
                     cSize = lastBlock ?
-                            ZSTD_compressEnd(zcs, cDst, oSize, ip, iSize) :
-                            ZSTD_compressContinue(zcs, cDst, oSize, ip, iSize);
+                            ZSTD_compressEnd_public(zcs, cDst, oSize, ip, iSize) :
+                            ZSTD_compressContinue_public(zcs, cDst, oSize, ip, iSize);
                     /* Consume the input prior to error checking to mirror buffered mode. */
-                    if (iSize > 0)
-                        ip += iSize;
+                    if (ip) ip += iSize;
                     FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed");
                     zcs->frameEnded = lastBlock;
-                    if (lastBlock)
-                        assert(ip == iend);
+                    if (lastBlock) assert(ip == iend);
                 }
                 if (cDst == op) {  /* no need to flush */
                     op += cSize;
@@ -5388,8 +5942,10 @@ size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuf
 /* After a compression call set the expected input/output buffer.
  * This is validated at the start of the next compression call.
  */
-static void ZSTD_setBufferExpectations(ZSTD_CCtx* cctx, ZSTD_outBuffer const* output, ZSTD_inBuffer const* input)
+static void
+ZSTD_setBufferExpectations(ZSTD_CCtx* cctx, const ZSTD_outBuffer* output, const ZSTD_inBuffer* input)
 {
+    DEBUGLOG(5, "ZSTD_setBufferExpectations (for advanced stable in/out modes)");
     if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) {
         cctx->expectedInBuffer = *input;
     }
@@ -5408,22 +5964,22 @@ static size_t ZSTD_checkBufferStability(ZSTD_CCtx const* cctx,
 {
     if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) {
         ZSTD_inBuffer const expect = cctx->expectedInBuffer;
-        if (expect.src != input->src || expect.pos != input->pos || expect.size != input->size)
-            RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer enabled but input differs!");
-        if (endOp != ZSTD_e_end)
-            RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer can only be used with ZSTD_e_end!");
+        if (expect.src != input->src || expect.pos != input->pos)
+            RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableInBuffer enabled but input differs!");
     }
+    (void)endOp;
     if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) {
         size_t const outBufferSize = output->size - output->pos;
         if (cctx->expectedOutBufferSize != outBufferSize)
-            RETURN_ERROR(dstBuffer_wrong, "ZSTD_c_stableOutBuffer enabled but output size differs!");
+            RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableOutBuffer enabled but output size differs!");
     }
     return 0;
 }
 
 static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
                                              ZSTD_EndDirective endOp,
-                                             size_t inSize) {
+                                             size_t inSize)
+{
     ZSTD_CCtx_params params = cctx->requestedParams;
     ZSTD_prefixDict const prefixDict = cctx->prefixDict;
     FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */
@@ -5437,9 +5993,9 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
         params.compressionLevel = cctx->cdict->compressionLevel;
     }
     DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage");
-    if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1;  /* auto-fix pledgedSrcSize */
-    {
-        size_t const dictSize = prefixDict.dict
+    if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1;  /* auto-determine pledgedSrcSize */
+
+    {   size_t const dictSize = prefixDict.dict
                 ? prefixDict.dictSize
                 : (cctx->cdict ? cctx->cdict->dictContentSize : 0);
         ZSTD_cParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, &params, cctx->pledgedSrcSizePlusOne - 1);
@@ -5451,6 +6007,9 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
     params.useBlockSplitter = ZSTD_resolveBlockSplitterMode(params.useBlockSplitter, &params.cParams);
     params.ldmParams.enableLdm = ZSTD_resolveEnableLdm(params.ldmParams.enableLdm, &params.cParams);
     params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, &params.cParams);
+    params.validateSequences = ZSTD_resolveExternalSequenceValidation(params.validateSequences);
+    params.maxBlockSize = ZSTD_resolveMaxBlockSize(params.maxBlockSize);
+    params.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(params.searchForExternalRepcodes, params.compressionLevel);
 
     {   U64 const pledgedSrcSize = cctx->pledgedSrcSizePlusOne - 1;
         assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
@@ -5477,6 +6036,8 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
     return 0;
 }
 
+/* @return provides a minimum amount of data remaining to be flushed from internal buffers
+ */
 size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
                              ZSTD_outBuffer* output,
                              ZSTD_inBuffer* input,
@@ -5491,8 +6052,27 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx,
 
     /* transparent initialization stage */
     if (cctx->streamStage == zcss_init) {
-        FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, input->size), "CompressStream2 initialization failed");
-        ZSTD_setBufferExpectations(cctx, output, input);    /* Set initial buffer expectations now that we've initialized */
+        size_t const inputSize = input->size - input->pos;  /* no obligation to start from pos==0 */
+        size_t const totalInputSize = inputSize + cctx->stableIn_notConsumed;
+        if ( (cctx->requestedParams.inBufferMode == ZSTD_bm_stable) /* input is presumed stable, across invocations */
+          && (endOp == ZSTD_e_continue)                             /* no flush requested, more input to come */
+          && (totalInputSize < ZSTD_BLOCKSIZE_MAX) ) {              /* not even reached one block yet */
+            if (cctx->stableIn_notConsumed) {  /* not the first time */
+                /* check stable source guarantees */
+                RETURN_ERROR_IF(input->src != cctx->expectedInBuffer.src, stabilityCondition_notRespected, "stableInBuffer condition not respected: wrong src pointer");
+                RETURN_ERROR_IF(input->pos != cctx->expectedInBuffer.size, stabilityCondition_notRespected, "stableInBuffer condition not respected: externally modified pos");
+            }
+            /* pretend input was consumed, to give a sense forward progress */
+            input->pos = input->size;
+            /* save stable inBuffer, for later control, and flush/end */
+            cctx->expectedInBuffer = *input;
+            /* but actually input wasn't consumed, so keep track of position from where compression shall resume */
+            cctx->stableIn_notConsumed += inputSize;
+            /* don't initialize yet, wait for the first block of flush() order, for better parameters adaptation */
+            return ZSTD_FRAMEHEADERSIZE_MIN(cctx->requestedParams.format);  /* at least some header to produce */
+        }
+        FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, totalInputSize), "compressStream2 initialization failed");
+        ZSTD_setBufferExpectations(cctx, output, input);   /* Set initial buffer expectations now that we've initialized */
     }
     /* end of transparent initialization stage */
 
@@ -5510,13 +6090,20 @@ size_t ZSTD_compressStream2_simpleArgs (
                       const void* src, size_t srcSize, size_t* srcPos,
                             ZSTD_EndDirective endOp)
 {
-    ZSTD_outBuffer output = { dst, dstCapacity, *dstPos };
-    ZSTD_inBuffer  input  = { src, srcSize, *srcPos };
+    ZSTD_outBuffer output;
+    ZSTD_inBuffer  input;
+    output.dst = dst;
+    output.size = dstCapacity;
+    output.pos = *dstPos;
+    input.src = src;
+    input.size = srcSize;
+    input.pos = *srcPos;
     /* ZSTD_compressStream2() will check validity of dstPos and srcPos */
-    size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp);
-    *dstPos = output.pos;
-    *srcPos = input.pos;
-    return cErr;
+    {   size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp);
+        *dstPos = output.pos;
+        *srcPos = input.pos;
+        return cErr;
+    }
 }
 
 size_t ZSTD_compress2(ZSTD_CCtx* cctx,
@@ -5539,6 +6126,7 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx,
         /* Reset to the original values. */
         cctx->requestedParams.inBufferMode = originalInBufferMode;
         cctx->requestedParams.outBufferMode = originalOutBufferMode;
+
         FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed");
         if (result != 0) {  /* compression not completed, due to lack of output space */
             assert(oPos == dstCapacity);
@@ -5549,64 +6137,61 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx,
     }
 }
 
-typedef struct {
-    U32 idx;             /* Index in array of ZSTD_Sequence */
-    U32 posInSequence;   /* Position within sequence at idx */
-    size_t posInSrc;        /* Number of bytes given by sequences provided so far */
-} ZSTD_sequencePosition;
-
 /* ZSTD_validateSequence() :
  * @offCode : is presumed to follow format required by ZSTD_storeSeq()
  * @returns a ZSTD error code if sequence is not valid
  */
 static size_t
-ZSTD_validateSequence(U32 offCode, U32 matchLength,
-                      size_t posInSrc, U32 windowLog, size_t dictSize)
+ZSTD_validateSequence(U32 offCode, U32 matchLength, U32 minMatch,
+                      size_t posInSrc, U32 windowLog, size_t dictSize, int useSequenceProducer)
 {
-    U32 const windowSize = 1 << windowLog;
+    U32 const windowSize = 1u << windowLog;
     /* posInSrc represents the amount of data the decoder would decode up to this point.
      * As long as the amount of data decoded is less than or equal to window size, offsets may be
      * larger than the total length of output decoded in order to reference the dict, even larger than
      * window size. After output surpasses windowSize, we're limited to windowSize offsets again.
      */
     size_t const offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize;
-    RETURN_ERROR_IF(offCode > STORE_OFFSET(offsetBound), corruption_detected, "Offset too large!");
-    RETURN_ERROR_IF(matchLength < MINMATCH, corruption_detected, "Matchlength too small");
+    size_t const matchLenLowerBound = (minMatch == 3 || useSequenceProducer) ? 3 : 4;
+    RETURN_ERROR_IF(offCode > OFFSET_TO_OFFBASE(offsetBound), externalSequences_invalid, "Offset too large!");
+    /* Validate maxNbSeq is large enough for the given matchLength and minMatch */
+    RETURN_ERROR_IF(matchLength < matchLenLowerBound, externalSequences_invalid, "Matchlength too small for the minMatch");
     return 0;
 }
 
 /* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */
-static U32 ZSTD_finalizeOffCode(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0)
+static U32 ZSTD_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0)
 {
-    U32 offCode = STORE_OFFSET(rawOffset);
+    U32 offBase = OFFSET_TO_OFFBASE(rawOffset);
 
     if (!ll0 && rawOffset == rep[0]) {
-        offCode = STORE_REPCODE_1;
+        offBase = REPCODE1_TO_OFFBASE;
     } else if (rawOffset == rep[1]) {
-        offCode = STORE_REPCODE(2 - ll0);
+        offBase = REPCODE_TO_OFFBASE(2 - ll0);
     } else if (rawOffset == rep[2]) {
-        offCode = STORE_REPCODE(3 - ll0);
+        offBase = REPCODE_TO_OFFBASE(3 - ll0);
     } else if (ll0 && rawOffset == rep[0] - 1) {
-        offCode = STORE_REPCODE_3;
+        offBase = REPCODE3_TO_OFFBASE;
     }
-    return offCode;
+    return offBase;
 }
 
-/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of
- * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter.
- */
-static size_t
+size_t
 ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
                                               ZSTD_sequencePosition* seqPos,
                                         const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
-                                        const void* src, size_t blockSize)
+                                        const void* src, size_t blockSize,
+                                        ZSTD_paramSwitch_e externalRepSearch)
 {
     U32 idx = seqPos->idx;
+    U32 const startIdx = idx;
     BYTE const* ip = (BYTE const*)(src);
     const BYTE* const iend = ip + blockSize;
     repcodes_t updatedRepcodes;
     U32 dictSize;
 
+    DEBUGLOG(5, "ZSTD_copySequencesToSeqStoreExplicitBlockDelim (blockSize = %zu)", blockSize);
+
     if (cctx->cdict) {
         dictSize = (U32)cctx->cdict->dictContentSize;
     } else if (cctx->prefixDict.dict) {
@@ -5615,25 +6200,55 @@ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
         dictSize = 0;
     }
     ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t));
-    for (; (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0) && idx < inSeqsSize; ++idx) {
+    for (; idx < inSeqsSize && (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0); ++idx) {
         U32 const litLength = inSeqs[idx].litLength;
-        U32 const ll0 = (litLength == 0);
         U32 const matchLength = inSeqs[idx].matchLength;
-        U32 const offCode = ZSTD_finalizeOffCode(inSeqs[idx].offset, updatedRepcodes.rep, ll0);
-        ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0);
+        U32 offBase;
 
-        DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength);
+        if (externalRepSearch == ZSTD_ps_disable) {
+            offBase = OFFSET_TO_OFFBASE(inSeqs[idx].offset);
+        } else {
+            U32 const ll0 = (litLength == 0);
+            offBase = ZSTD_finalizeOffBase(inSeqs[idx].offset, updatedRepcodes.rep, ll0);
+            ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0);
+        }
+
+        DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength);
         if (cctx->appliedParams.validateSequences) {
             seqPos->posInSrc += litLength + matchLength;
-            FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc,
-                                                cctx->appliedParams.cParams.windowLog, dictSize),
+            FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc,
+                                                cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useSequenceProducer),
                                                 "Sequence validation failed");
         }
-        RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation,
+        RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid,
                         "Not enough memory allocated. Try adjusting ZSTD_c_minMatch.");
-        ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offCode, matchLength);
+        ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength);
         ip += matchLength + litLength;
     }
+
+    /* If we skipped repcode search while parsing, we need to update repcodes now */
+    assert(externalRepSearch != ZSTD_ps_auto);
+    assert(idx >= startIdx);
+    if (externalRepSearch == ZSTD_ps_disable && idx != startIdx) {
+        U32* const rep = updatedRepcodes.rep;
+        U32 lastSeqIdx = idx - 1; /* index of last non-block-delimiter sequence */
+
+        if (lastSeqIdx >= startIdx + 2) {
+            rep[2] = inSeqs[lastSeqIdx - 2].offset;
+            rep[1] = inSeqs[lastSeqIdx - 1].offset;
+            rep[0] = inSeqs[lastSeqIdx].offset;
+        } else if (lastSeqIdx == startIdx + 1) {
+            rep[2] = rep[0];
+            rep[1] = inSeqs[lastSeqIdx - 1].offset;
+            rep[0] = inSeqs[lastSeqIdx].offset;
+        } else {
+            assert(lastSeqIdx == startIdx);
+            rep[2] = rep[1];
+            rep[1] = rep[0];
+            rep[0] = inSeqs[lastSeqIdx].offset;
+        }
+    }
+
     ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t));
 
     if (inSeqs[idx].litLength) {
@@ -5642,26 +6257,15 @@ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
         ip += inSeqs[idx].litLength;
         seqPos->posInSrc += inSeqs[idx].litLength;
     }
-    RETURN_ERROR_IF(ip != iend, corruption_detected, "Blocksize doesn't agree with block delimiter!");
+    RETURN_ERROR_IF(ip != iend, externalSequences_invalid, "Blocksize doesn't agree with block delimiter!");
     seqPos->idx = idx+1;
     return 0;
 }
 
-/* Returns the number of bytes to move the current read position back by. Only non-zero
- * if we ended up splitting a sequence. Otherwise, it may return a ZSTD error if something
- * went wrong.
- *
- * This function will attempt to scan through blockSize bytes represented by the sequences
- * in inSeqs, storing any (partial) sequences.
- *
- * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to
- * avoid splitting a match, or to avoid splitting a match such that it would produce a match
- * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block.
- */
-static size_t
+size_t
 ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
                                    const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
-                                   const void* src, size_t blockSize)
+                                   const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch)
 {
     U32 idx = seqPos->idx;
     U32 startPosInSequence = seqPos->posInSequence;
@@ -5673,6 +6277,9 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition*
     U32 bytesAdjustment = 0;
     U32 finalMatchSplit = 0;
 
+    /* TODO(embg) support fast parsing mode in noBlockDelim mode */
+    (void)externalRepSearch;
+
     if (cctx->cdict) {
         dictSize = cctx->cdict->dictContentSize;
     } else if (cctx->prefixDict.dict) {
@@ -5680,7 +6287,7 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition*
     } else {
         dictSize = 0;
     }
-    DEBUGLOG(5, "ZSTD_copySequencesToSeqStore: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize);
+    DEBUGLOG(5, "ZSTD_copySequencesToSeqStoreNoBlockDelim: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize);
     DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength);
     ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t));
     while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) {
@@ -5688,7 +6295,7 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition*
         U32 litLength = currSeq.litLength;
         U32 matchLength = currSeq.matchLength;
         U32 const rawOffset = currSeq.offset;
-        U32 offCode;
+        U32 offBase;
 
         /* Modify the sequence depending on where endPosInSequence lies */
         if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) {
@@ -5702,7 +6309,6 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition*
             /* Move to the next sequence */
             endPosInSequence -= currSeq.litLength + currSeq.matchLength;
             startPosInSequence = 0;
-            idx++;
         } else {
             /* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence
                does not reach the end of the match. So, we have to split the sequence */
@@ -5742,21 +6348,23 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition*
         }
         /* Check if this offset can be represented with a repcode */
         {   U32 const ll0 = (litLength == 0);
-            offCode = ZSTD_finalizeOffCode(rawOffset, updatedRepcodes.rep, ll0);
-            ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0);
+            offBase = ZSTD_finalizeOffBase(rawOffset, updatedRepcodes.rep, ll0);
+            ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0);
         }
 
         if (cctx->appliedParams.validateSequences) {
             seqPos->posInSrc += litLength + matchLength;
-            FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc,
-                                                   cctx->appliedParams.cParams.windowLog, dictSize),
+            FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc,
+                                                   cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useSequenceProducer),
                                                    "Sequence validation failed");
         }
-        DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength);
-        RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation,
+        DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength);
+        RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid,
                         "Not enough memory allocated. Try adjusting ZSTD_c_minMatch.");
-        ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offCode, matchLength);
+        ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength);
         ip += matchLength + litLength;
+        if (!finalMatchSplit)
+            idx++; /* Next Sequence */
     }
     DEBUGLOG(5, "Ending seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength);
     assert(idx == inSeqsSize || endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength);
@@ -5779,7 +6387,7 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition*
 
 typedef size_t (*ZSTD_sequenceCopier) (ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
                                        const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
-                                       const void* src, size_t blockSize);
+                                       const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch);
 static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode)
 {
     ZSTD_sequenceCopier sequenceCopier = NULL;
@@ -5793,6 +6401,57 @@ static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode)
     return sequenceCopier;
 }
 
+/* Discover the size of next block by searching for the delimiter.
+ * Note that a block delimiter **must** exist in this mode,
+ * otherwise it's an input error.
+ * The block size retrieved will be later compared to ensure it remains within bounds */
+static size_t
+blockSize_explicitDelimiter(const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_sequencePosition seqPos)
+{
+    int end = 0;
+    size_t blockSize = 0;
+    size_t spos = seqPos.idx;
+    DEBUGLOG(6, "blockSize_explicitDelimiter : seq %zu / %zu", spos, inSeqsSize);
+    assert(spos <= inSeqsSize);
+    while (spos < inSeqsSize) {
+        end = (inSeqs[spos].offset == 0);
+        blockSize += inSeqs[spos].litLength + inSeqs[spos].matchLength;
+        if (end) {
+            if (inSeqs[spos].matchLength != 0)
+                RETURN_ERROR(externalSequences_invalid, "delimiter format error : both matchlength and offset must be == 0");
+            break;
+        }
+        spos++;
+    }
+    if (!end)
+        RETURN_ERROR(externalSequences_invalid, "Reached end of sequences without finding a block delimiter");
+    return blockSize;
+}
+
+/* More a "target" block size */
+static size_t blockSize_noDelimiter(size_t blockSize, size_t remaining)
+{
+    int const lastBlock = (remaining <= blockSize);
+    return lastBlock ? remaining : blockSize;
+}
+
+static size_t determine_blockSize(ZSTD_sequenceFormat_e mode,
+                           size_t blockSize, size_t remaining,
+                     const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_sequencePosition seqPos)
+{
+    DEBUGLOG(6, "determine_blockSize : remainingSize = %zu", remaining);
+    if (mode == ZSTD_sf_noBlockDelimiters)
+        return blockSize_noDelimiter(blockSize, remaining);
+    {   size_t const explicitBlockSize = blockSize_explicitDelimiter(inSeqs, inSeqsSize, seqPos);
+        FORWARD_IF_ERROR(explicitBlockSize, "Error while determining block size with explicit delimiters");
+        if (explicitBlockSize > blockSize)
+            RETURN_ERROR(externalSequences_invalid, "sequences incorrectly define a too large block");
+        if (explicitBlockSize > remaining)
+            RETURN_ERROR(externalSequences_invalid, "sequences define a frame longer than source");
+        return explicitBlockSize;
+    }
+}
+
 /* Compress, block-by-block, all of the sequences given.
  *
  * Returns the cumulative size of all compressed blocks (including their headers),
@@ -5805,9 +6464,6 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
                           const void* src, size_t srcSize)
 {
     size_t cSize = 0;
-    U32 lastBlock;
-    size_t blockSize;
-    size_t compressedSeqsSize;
     size_t remaining = srcSize;
     ZSTD_sequencePosition seqPos = {0, 0, 0};
 
@@ -5827,22 +6483,29 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
     }
 
     while (remaining) {
+        size_t compressedSeqsSize;
         size_t cBlockSize;
         size_t additionalByteAdjustment;
-        lastBlock = remaining <= cctx->blockSize;
-        blockSize = lastBlock ? (U32)remaining : (U32)cctx->blockSize;
+        size_t blockSize = determine_blockSize(cctx->appliedParams.blockDelimiters,
+                                        cctx->blockSize, remaining,
+                                        inSeqs, inSeqsSize, seqPos);
+        U32 const lastBlock = (blockSize == remaining);
+        FORWARD_IF_ERROR(blockSize, "Error while trying to determine block size");
+        assert(blockSize <= remaining);
         ZSTD_resetSeqStore(&cctx->seqStore);
-        DEBUGLOG(4, "Working on new block. Blocksize: %zu", blockSize);
+        DEBUGLOG(5, "Working on new block. Blocksize: %zu (total:%zu)", blockSize, (ip - (const BYTE*)src) + blockSize);
 
-        additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize);
+        additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize, cctx->appliedParams.searchForExternalRepcodes);
         FORWARD_IF_ERROR(additionalByteAdjustment, "Bad sequence copy");
         blockSize -= additionalByteAdjustment;
 
         /* If blocks are too small, emit as a nocompress block */
-        if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) {
+        /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding
+         * additional 1. We need to revisit and change this logic to be more consistent */
+        if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) {
             cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
             FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed");
-            DEBUGLOG(4, "Block too small, writing out nocompress block: cSize: %zu", cBlockSize);
+            DEBUGLOG(5, "Block too small, writing out nocompress block: cSize: %zu", cBlockSize);
             cSize += cBlockSize;
             ip += blockSize;
             op += cBlockSize;
@@ -5851,6 +6514,7 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
             continue;
         }
 
+        RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "not enough dstCapacity to write a new compressed block");
         compressedSeqsSize = ZSTD_entropyCompressSeqStore(&cctx->seqStore,
                                 &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy,
                                 &cctx->appliedParams,
@@ -5859,11 +6523,11 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
                                 cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */,
                                 cctx->bmi2);
         FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed");
-        DEBUGLOG(4, "Compressed sequences size: %zu", compressedSeqsSize);
+        DEBUGLOG(5, "Compressed sequences size: %zu", compressedSeqsSize);
 
         if (!cctx->isFirstBlock &&
             ZSTD_maybeRLE(&cctx->seqStore) &&
-            ZSTD_isRLE((BYTE const*)src, srcSize)) {
+            ZSTD_isRLE(ip, blockSize)) {
             /* We don't want to emit our first block as a RLE even if it qualifies because
             * doing so will cause the decoder (cli only) to throw a "should consume all input error."
             * This is only an issue for zstd <= v1.4.3
@@ -5874,12 +6538,12 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
         if (compressedSeqsSize == 0) {
             /* ZSTD_noCompressBlock writes the block header as well */
             cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock);
-            FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed");
-            DEBUGLOG(4, "Writing out nocompress block, size: %zu", cBlockSize);
+            FORWARD_IF_ERROR(cBlockSize, "ZSTD_noCompressBlock failed");
+            DEBUGLOG(5, "Writing out nocompress block, size: %zu", cBlockSize);
         } else if (compressedSeqsSize == 1) {
             cBlockSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, blockSize, lastBlock);
-            FORWARD_IF_ERROR(cBlockSize, "RLE compress block failed");
-            DEBUGLOG(4, "Writing out RLE block, size: %zu", cBlockSize);
+            FORWARD_IF_ERROR(cBlockSize, "ZSTD_rleCompressBlock failed");
+            DEBUGLOG(5, "Writing out RLE block, size: %zu", cBlockSize);
         } else {
             U32 cBlockHeader;
             /* Error checking and repcodes update */
@@ -5891,11 +6555,10 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
             cBlockHeader = lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3);
             MEM_writeLE24(op, cBlockHeader);
             cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize;
-            DEBUGLOG(4, "Writing out compressed block, size: %zu", cBlockSize);
+            DEBUGLOG(5, "Writing out compressed block, size: %zu", cBlockSize);
         }
 
         cSize += cBlockSize;
-        DEBUGLOG(4, "cSize running total: %zu", cSize);
 
         if (lastBlock) {
             break;
@@ -5906,12 +6569,15 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
             dstCapacity -= cBlockSize;
             cctx->isFirstBlock = 0;
         }
+        DEBUGLOG(5, "cSize running total: %zu (remaining dstCapacity=%zu)", cSize, dstCapacity);
     }
 
+    DEBUGLOG(4, "cSize final total: %zu", cSize);
     return cSize;
 }
 
-size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstCapacity,
+size_t ZSTD_compressSequences(ZSTD_CCtx* cctx,
+                              void* dst, size_t dstCapacity,
                               const ZSTD_Sequence* inSeqs, size_t inSeqsSize,
                               const void* src, size_t srcSize)
 {
@@ -5921,7 +6587,7 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstCapaci
     size_t frameHeaderSize = 0;
 
     /* Transparent initialization stage, same as compressStream2() */
-    DEBUGLOG(3, "ZSTD_compressSequences()");
+    DEBUGLOG(4, "ZSTD_compressSequences (dstCapacity=%zu)", dstCapacity);
     assert(cctx != NULL);
     FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed");
     /* Begin writing output, starting with frame header */
@@ -5949,26 +6615,34 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstCapaci
         cSize += 4;
     }
 
-    DEBUGLOG(3, "Final compressed size: %zu", cSize);
+    DEBUGLOG(4, "Final compressed size: %zu", cSize);
     return cSize;
 }
 
 /*======   Finalize   ======*/
 
+static ZSTD_inBuffer inBuffer_forEndFlush(const ZSTD_CStream* zcs)
+{
+    const ZSTD_inBuffer nullInput = { NULL, 0, 0 };
+    const int stableInput = (zcs->appliedParams.inBufferMode == ZSTD_bm_stable);
+    return stableInput ? zcs->expectedInBuffer : nullInput;
+}
+
 /*! ZSTD_flushStream() :
  * @return : amount of data remaining to flush */
 size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
 {
-    ZSTD_inBuffer input = { NULL, 0, 0 };
+    ZSTD_inBuffer input = inBuffer_forEndFlush(zcs);
+    input.size = input.pos; /* do not ingest more input during flush */
     return ZSTD_compressStream2(zcs, output, &input, ZSTD_e_flush);
 }
 
 
 size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
 {
-    ZSTD_inBuffer input = { NULL, 0, 0 };
+    ZSTD_inBuffer input = inBuffer_forEndFlush(zcs);
     size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end);
-    FORWARD_IF_ERROR( remainingToFlush , "ZSTD_compressStream2 failed");
+    FORWARD_IF_ERROR(remainingToFlush , "ZSTD_compressStream2(,,ZSTD_e_end) failed");
     if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush;   /* minimal estimation */
     /* single thread mode : attempt to calculate remaining to flush more precisely */
     {   size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE;
@@ -6090,7 +6764,7 @@ static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel,
             cp.targetLength = (unsigned)(-clampedCompressionLevel);
         }
         /* refine parameters based on srcSize & dictSize */
-        return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode);
+        return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode, ZSTD_ps_auto);
     }
 }
 
@@ -6125,3 +6799,21 @@ ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeH
     if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN;
     return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown);
 }
+
+void ZSTD_registerSequenceProducer(
+    ZSTD_CCtx* zc, void* mState,
+    ZSTD_sequenceProducer_F* mFinder
+) {
+    if (mFinder != NULL) {
+        ZSTD_externalMatchCtx emctx;
+        emctx.mState = mState;
+        emctx.mFinder = mFinder;
+        emctx.seqBuffer = NULL;
+        emctx.seqBufferCapacity = 0;
+        zc->externalMatchCtx = emctx;
+        zc->requestedParams.useSequenceProducer = 1;
+    } else {
+        ZSTD_memset(&zc->externalMatchCtx, 0, sizeof(zc->externalMatchCtx));
+        zc->requestedParams.useSequenceProducer = 0;
+    }
+}
diff --git a/lib/zstd/compress/zstd_compress_internal.h b/lib/zstd/compress/zstd_compress_internal.h
index 71697a11ae30..899f5e2de8e9 100644
--- a/lib/zstd/compress/zstd_compress_internal.h
+++ b/lib/zstd/compress/zstd_compress_internal.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -20,6 +21,7 @@
 ***************************************/
 #include "../common/zstd_internal.h"
 #include "zstd_cwksp.h"
+#include "../common/bits.h" /* ZSTD_highbit32, ZSTD_NbCommonBytes */
 
 
 /*-*************************************
@@ -111,12 +113,13 @@ typedef struct {
 /* ZSTD_buildBlockEntropyStats() :
  *  Builds entropy for the block.
  *  @return : 0 on success or error code */
-size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr,
-                             const ZSTD_entropyCTables_t* prevEntropy,
-                                   ZSTD_entropyCTables_t* nextEntropy,
-                             const ZSTD_CCtx_params* cctxParams,
-                                   ZSTD_entropyCTablesMetadata_t* entropyMetadata,
-                                   void* workspace, size_t wkspSize);
+size_t ZSTD_buildBlockEntropyStats(
+                    const seqStore_t* seqStorePtr,
+                    const ZSTD_entropyCTables_t* prevEntropy,
+                          ZSTD_entropyCTables_t* nextEntropy,
+                    const ZSTD_CCtx_params* cctxParams,
+                          ZSTD_entropyCTablesMetadata_t* entropyMetadata,
+                          void* workspace, size_t wkspSize);
 
 /* *******************************
 *  Compression internals structs *
@@ -142,6 +145,12 @@ typedef struct {
   size_t capacity;      /* The capacity starting from `seq` pointer */
 } rawSeqStore_t;
 
+typedef struct {
+    U32 idx;            /* Index in array of ZSTD_Sequence */
+    U32 posInSequence;  /* Position within sequence at idx */
+    size_t posInSrc;    /* Number of bytes given by sequences provided so far */
+} ZSTD_sequencePosition;
+
 UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0};
 
 typedef struct {
@@ -212,8 +221,10 @@ struct ZSTD_matchState_t {
     U32 hashLog3;           /* dispatch table for matches of len==3 : larger == faster, more memory */
 
     U32 rowHashLog;                          /* For row-based matchfinder: Hashlog based on nb of rows in the hashTable.*/
-    U16* tagTable;                           /* For row-based matchFinder: A row-based table containing the hashes and head index. */
+    BYTE* tagTable;                          /* For row-based matchFinder: A row-based table containing the hashes and head index. */
     U32 hashCache[ZSTD_ROW_HASH_CACHE_SIZE]; /* For row-based matchFinder: a cache of hashes to improve speed */
+    U64 hashSalt;                            /* For row-based matchFinder: salts the hash for re-use of tag table */
+    U32 hashSaltEntropy;                     /* For row-based matchFinder: collects entropy for salt generation */
 
     U32* hashTable;
     U32* hashTable3;
@@ -228,6 +239,18 @@ struct ZSTD_matchState_t {
     const ZSTD_matchState_t* dictMatchState;
     ZSTD_compressionParameters cParams;
     const rawSeqStore_t* ldmSeqStore;
+
+    /* Controls prefetching in some dictMatchState matchfinders.
+     * This behavior is controlled from the cctx ms.
+     * This parameter has no effect in the cdict ms. */
+    int prefetchCDictTables;
+
+    /* When == 0, lazy match finders insert every position.
+     * When != 0, lazy match finders only insert positions they search.
+     * This allows them to skip much faster over incompressible data,
+     * at a small cost to compression ratio.
+     */
+    int lazySkipping;
 };
 
 typedef struct {
@@ -324,6 +347,24 @@ struct ZSTD_CCtx_params_s {
 
     /* Internal use, for createCCtxParams() and freeCCtxParams() only */
     ZSTD_customMem customMem;
+
+    /* Controls prefetching in some dictMatchState matchfinders */
+    ZSTD_paramSwitch_e prefetchCDictTables;
+
+    /* Controls whether zstd will fall back to an internal matchfinder
+     * if the external matchfinder returns an error code. */
+    int enableMatchFinderFallback;
+
+    /* Indicates whether an external matchfinder has been referenced.
+     * Users can't set this externally.
+     * It is set internally in ZSTD_registerSequenceProducer(). */
+    int useSequenceProducer;
+
+    /* Adjust the max block size*/
+    size_t maxBlockSize;
+
+    /* Controls repcode search in external sequence parsing */
+    ZSTD_paramSwitch_e searchForExternalRepcodes;
 };  /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
 
 #define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2))
@@ -355,6 +396,14 @@ typedef struct {
     ZSTD_entropyCTablesMetadata_t entropyMetadata;
 } ZSTD_blockSplitCtx;
 
+/* Context for block-level external matchfinder API */
+typedef struct {
+  void* mState;
+  ZSTD_sequenceProducer_F* mFinder;
+  ZSTD_Sequence* seqBuffer;
+  size_t seqBufferCapacity;
+} ZSTD_externalMatchCtx;
+
 struct ZSTD_CCtx_s {
     ZSTD_compressionStage_e stage;
     int cParamsChanged;                  /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */
@@ -404,6 +453,7 @@ struct ZSTD_CCtx_s {
 
     /* Stable in/out buffer verification */
     ZSTD_inBuffer expectedInBuffer;
+    size_t stableIn_notConsumed; /* nb bytes within stable input buffer that are said to be consumed but are not */
     size_t expectedOutBufferSize;
 
     /* Dictionary */
@@ -417,9 +467,13 @@ struct ZSTD_CCtx_s {
 
     /* Workspace for block splitter */
     ZSTD_blockSplitCtx blockSplitCtx;
+
+    /* Workspace for external matchfinder */
+    ZSTD_externalMatchCtx externalMatchCtx;
 };
 
 typedef enum { ZSTD_dtlm_fast, ZSTD_dtlm_full } ZSTD_dictTableLoadMethod_e;
+typedef enum { ZSTD_tfp_forCCtx, ZSTD_tfp_forCDict } ZSTD_tableFillPurpose_e;
 
 typedef enum {
     ZSTD_noDict = 0,
@@ -441,7 +495,7 @@ typedef enum {
                                  * In this mode we take both the source size and the dictionary size
                                  * into account when selecting and adjusting the parameters.
                                  */
-    ZSTD_cpm_unknown = 3,       /* ZSTD_getCParams, ZSTD_getParams, ZSTD_adjustParams.
+    ZSTD_cpm_unknown = 3        /* ZSTD_getCParams, ZSTD_getParams, ZSTD_adjustParams.
                                  * We don't know what these parameters are for. We default to the legacy
                                  * behavior of taking both the source size and the dict size into account
                                  * when selecting and adjusting parameters.
@@ -500,9 +554,11 @@ MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value)
 /* ZSTD_noCompressBlock() :
  * Writes uncompressed block to dst buffer from given src.
  * Returns the size of the block */
-MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock)
+MEM_STATIC size_t
+ZSTD_noCompressBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock)
 {
     U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3);
+    DEBUGLOG(5, "ZSTD_noCompressBlock (srcSize=%zu, dstCapacity=%zu)", srcSize, dstCapacity);
     RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity,
                     dstSize_tooSmall, "dst buf too small for uncompressed block");
     MEM_writeLE24(dst, cBlockHeader24);
@@ -510,7 +566,8 @@ MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const voi
     return ZSTD_blockHeaderSize + srcSize;
 }
 
-MEM_STATIC size_t ZSTD_rleCompressBlock (void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock)
+MEM_STATIC size_t
+ZSTD_rleCompressBlock(void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock)
 {
     BYTE* const op = (BYTE*)dst;
     U32 const cBlockHeader = lastBlock + (((U32)bt_rle)<<1) + (U32)(srcSize << 3);
@@ -529,7 +586,7 @@ MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat)
 {
     U32 const minlog = (strat>=ZSTD_btultra) ? (U32)(strat) - 1 : 6;
     ZSTD_STATIC_ASSERT(ZSTD_btultra == 8);
-    assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat));
+    assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, (int)strat));
     return (srcSize >> minlog) + 2;
 }
 
@@ -565,29 +622,27 @@ ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE con
     while (ip < iend) *op++ = *ip++;
 }
 
-#define ZSTD_REP_MOVE     (ZSTD_REP_NUM-1)
-#define STORE_REPCODE_1 STORE_REPCODE(1)
-#define STORE_REPCODE_2 STORE_REPCODE(2)
-#define STORE_REPCODE_3 STORE_REPCODE(3)
-#define STORE_REPCODE(r) (assert((r)>=1), assert((r)<=3), (r)-1)
-#define STORE_OFFSET(o)  (assert((o)>0), o + ZSTD_REP_MOVE)
-#define STORED_IS_OFFSET(o)  ((o) > ZSTD_REP_MOVE)
-#define STORED_IS_REPCODE(o) ((o) <= ZSTD_REP_MOVE)
-#define STORED_OFFSET(o)  (assert(STORED_IS_OFFSET(o)), (o)-ZSTD_REP_MOVE)
-#define STORED_REPCODE(o) (assert(STORED_IS_REPCODE(o)), (o)+1)  /* returns ID 1,2,3 */
-#define STORED_TO_OFFBASE(o) ((o)+1)
-#define OFFBASE_TO_STORED(o) ((o)-1)
+
+#define REPCODE1_TO_OFFBASE REPCODE_TO_OFFBASE(1)
+#define REPCODE2_TO_OFFBASE REPCODE_TO_OFFBASE(2)
+#define REPCODE3_TO_OFFBASE REPCODE_TO_OFFBASE(3)
+#define REPCODE_TO_OFFBASE(r) (assert((r)>=1), assert((r)<=ZSTD_REP_NUM), (r)) /* accepts IDs 1,2,3 */
+#define OFFSET_TO_OFFBASE(o)  (assert((o)>0), o + ZSTD_REP_NUM)
+#define OFFBASE_IS_OFFSET(o)  ((o) > ZSTD_REP_NUM)
+#define OFFBASE_IS_REPCODE(o) ( 1 <= (o) && (o) <= ZSTD_REP_NUM)
+#define OFFBASE_TO_OFFSET(o)  (assert(OFFBASE_IS_OFFSET(o)), (o) - ZSTD_REP_NUM)
+#define OFFBASE_TO_REPCODE(o) (assert(OFFBASE_IS_REPCODE(o)), (o))  /* returns ID 1,2,3 */
 
 /*! ZSTD_storeSeq() :
- *  Store a sequence (litlen, litPtr, offCode and matchLength) into seqStore_t.
- *  @offBase_minus1 : Users should use employ macros STORE_REPCODE_X and STORE_OFFSET().
+ *  Store a sequence (litlen, litPtr, offBase and matchLength) into seqStore_t.
+ *  @offBase : Users should employ macros REPCODE_TO_OFFBASE() and OFFSET_TO_OFFBASE().
  *  @matchLength : must be >= MINMATCH
- *  Allowed to overread literals up to litLimit.
+ *  Allowed to over-read literals up to litLimit.
 */
 HINT_INLINE UNUSED_ATTR void
 ZSTD_storeSeq(seqStore_t* seqStorePtr,
               size_t litLength, const BYTE* literals, const BYTE* litLimit,
-              U32 offBase_minus1,
+              U32 offBase,
               size_t matchLength)
 {
     BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH;
@@ -596,8 +651,8 @@ ZSTD_storeSeq(seqStore_t* seqStorePtr,
     static const BYTE* g_start = NULL;
     if (g_start==NULL) g_start = (const BYTE*)literals;  /* note : index only works for compression within a single segment */
     {   U32 const pos = (U32)((const BYTE*)literals - g_start);
-        DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offCode%7u",
-               pos, (U32)litLength, (U32)matchLength, (U32)offBase_minus1);
+        DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offBase%7u",
+               pos, (U32)litLength, (U32)matchLength, (U32)offBase);
     }
 #endif
     assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq);
@@ -607,9 +662,9 @@ ZSTD_storeSeq(seqStore_t* seqStorePtr,
     assert(literals + litLength <= litLimit);
     if (litEnd <= litLimit_w) {
         /* Common case we can use wildcopy.
-	 * First copy 16 bytes, because literals are likely short.
-	 */
-        assert(WILDCOPY_OVERLENGTH >= 16);
+         * First copy 16 bytes, because literals are likely short.
+         */
+        ZSTD_STATIC_ASSERT(WILDCOPY_OVERLENGTH >= 16);
         ZSTD_copy16(seqStorePtr->lit, literals);
         if (litLength > 16) {
             ZSTD_wildcopy(seqStorePtr->lit+16, literals+16, (ptrdiff_t)litLength-16, ZSTD_no_overlap);
@@ -628,7 +683,7 @@ ZSTD_storeSeq(seqStore_t* seqStorePtr,
     seqStorePtr->sequences[0].litLength = (U16)litLength;
 
     /* match offset */
-    seqStorePtr->sequences[0].offBase = STORED_TO_OFFBASE(offBase_minus1);
+    seqStorePtr->sequences[0].offBase = offBase;
 
     /* match Length */
     assert(matchLength >= MINMATCH);
@@ -646,17 +701,17 @@ ZSTD_storeSeq(seqStore_t* seqStorePtr,
 
 /* ZSTD_updateRep() :
  * updates in-place @rep (array of repeat offsets)
- * @offBase_minus1 : sum-type, with same numeric representation as ZSTD_storeSeq()
+ * @offBase : sum-type, using numeric representation of ZSTD_storeSeq()
  */
 MEM_STATIC void
-ZSTD_updateRep(U32 rep[ZSTD_REP_NUM], U32 const offBase_minus1, U32 const ll0)
+ZSTD_updateRep(U32 rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0)
 {
-    if (STORED_IS_OFFSET(offBase_minus1)) {  /* full offset */
+    if (OFFBASE_IS_OFFSET(offBase)) {  /* full offset */
         rep[2] = rep[1];
         rep[1] = rep[0];
-        rep[0] = STORED_OFFSET(offBase_minus1);
+        rep[0] = OFFBASE_TO_OFFSET(offBase);
     } else {   /* repcode */
-        U32 const repCode = STORED_REPCODE(offBase_minus1) - 1 + ll0;
+        U32 const repCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0;
         if (repCode > 0) {  /* note : if repCode==0, no change */
             U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode];
             rep[2] = (repCode >= 2) ? rep[1] : rep[2];
@@ -673,11 +728,11 @@ typedef struct repcodes_s {
 } repcodes_t;
 
 MEM_STATIC repcodes_t
-ZSTD_newRep(U32 const rep[ZSTD_REP_NUM], U32 const offBase_minus1, U32 const ll0)
+ZSTD_newRep(U32 const rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0)
 {
     repcodes_t newReps;
     ZSTD_memcpy(&newReps, rep, sizeof(newReps));
-    ZSTD_updateRep(newReps.rep, offBase_minus1, ll0);
+    ZSTD_updateRep(newReps.rep, offBase, ll0);
     return newReps;
 }
 
@@ -685,59 +740,6 @@ ZSTD_newRep(U32 const rep[ZSTD_REP_NUM], U32 const offBase_minus1, U32 const ll0
 /*-*************************************
 *  Match length counter
 ***************************************/
-static unsigned ZSTD_NbCommonBytes (size_t val)
-{
-    if (MEM_isLittleEndian()) {
-        if (MEM_64bits()) {
-#       if (__GNUC__ >= 4)
-            return (__builtin_ctzll((U64)val) >> 3);
-#       else
-            static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
-                                                     0, 3, 1, 3, 1, 4, 2, 7,
-                                                     0, 2, 3, 6, 1, 5, 3, 5,
-                                                     1, 3, 4, 4, 2, 5, 6, 7,
-                                                     7, 0, 1, 2, 3, 3, 4, 6,
-                                                     2, 6, 5, 5, 3, 4, 5, 6,
-                                                     7, 1, 2, 4, 6, 4, 4, 5,
-                                                     7, 2, 6, 5, 7, 6, 7, 7 };
-            return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
-#       endif
-        } else { /* 32 bits */
-#       if (__GNUC__ >= 3)
-            return (__builtin_ctz((U32)val) >> 3);
-#       else
-            static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
-                                                     3, 2, 2, 1, 3, 2, 0, 1,
-                                                     3, 3, 1, 2, 2, 2, 2, 0,
-                                                     3, 1, 2, 0, 1, 0, 1, 1 };
-            return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
-#       endif
-        }
-    } else {  /* Big Endian CPU */
-        if (MEM_64bits()) {
-#       if (__GNUC__ >= 4)
-            return (__builtin_clzll(val) >> 3);
-#       else
-            unsigned r;
-            const unsigned n32 = sizeof(size_t)*4;   /* calculate this way due to compiler complaining in 32-bits mode */
-            if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
-            if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
-            r += (!val);
-            return r;
-#       endif
-        } else { /* 32 bits */
-#       if (__GNUC__ >= 3)
-            return (__builtin_clz((U32)val) >> 3);
-#       else
-            unsigned r;
-            if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
-            r += (!val);
-            return r;
-#       endif
-    }   }
-}
-
-
 MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit)
 {
     const BYTE* const pStart = pIn;
@@ -783,32 +785,43 @@ ZSTD_count_2segments(const BYTE* ip, const BYTE* match,
  *  Hashes
  ***************************************/
 static const U32 prime3bytes = 506832829U;
-static U32    ZSTD_hash3(U32 u, U32 h) { return ((u << (32-24)) * prime3bytes)  >> (32-h) ; }
-MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); } /* only in zstd_opt.h */
+static U32    ZSTD_hash3(U32 u, U32 h, U32 s) { assert(h <= 32); return (((u << (32-24)) * prime3bytes) ^ s)  >> (32-h) ; }
+MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h, 0); } /* only in zstd_opt.h */
+MEM_STATIC size_t ZSTD_hash3PtrS(const void* ptr, U32 h, U32 s) { return ZSTD_hash3(MEM_readLE32(ptr), h, s); }
 
 static const U32 prime4bytes = 2654435761U;
-static U32    ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; }
-static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_read32(ptr), h); }
+static U32    ZSTD_hash4(U32 u, U32 h, U32 s) { assert(h <= 32); return ((u * prime4bytes) ^ s) >> (32-h) ; }
+static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_readLE32(ptr), h, 0); }
+static size_t ZSTD_hash4PtrS(const void* ptr, U32 h, U32 s) { return ZSTD_hash4(MEM_readLE32(ptr), h, s); }
 
 static const U64 prime5bytes = 889523592379ULL;
-static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)(((u  << (64-40)) * prime5bytes) >> (64-h)) ; }
-static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h); }
+static size_t ZSTD_hash5(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u  << (64-40)) * prime5bytes) ^ s) >> (64-h)) ; }
+static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h, 0); }
+static size_t ZSTD_hash5PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash5(MEM_readLE64(p), h, s); }
 
 static const U64 prime6bytes = 227718039650203ULL;
-static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u  << (64-48)) * prime6bytes) >> (64-h)) ; }
-static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); }
+static size_t ZSTD_hash6(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u  << (64-48)) * prime6bytes) ^ s) >> (64-h)) ; }
+static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h, 0); }
+static size_t ZSTD_hash6PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash6(MEM_readLE64(p), h, s); }
 
 static const U64 prime7bytes = 58295818150454627ULL;
-static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u  << (64-56)) * prime7bytes) >> (64-h)) ; }
-static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h); }
+static size_t ZSTD_hash7(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u  << (64-56)) * prime7bytes) ^ s) >> (64-h)) ; }
+static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h, 0); }
+static size_t ZSTD_hash7PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash7(MEM_readLE64(p), h, s); }
 
 static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL;
-static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; }
-static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); }
+static size_t ZSTD_hash8(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u) * prime8bytes)  ^ s) >> (64-h)) ; }
+static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h, 0); }
+static size_t ZSTD_hash8PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash8(MEM_readLE64(p), h, s); }
+
 
 MEM_STATIC FORCE_INLINE_ATTR
 size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
 {
+    /* Although some of these hashes do support hBits up to 64, some do not.
+     * To be on the safe side, always avoid hBits > 32. */
+    assert(hBits <= 32);
+
     switch(mls)
     {
     default:
@@ -820,6 +833,24 @@ size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
     }
 }
 
+MEM_STATIC FORCE_INLINE_ATTR
+size_t ZSTD_hashPtrSalted(const void* p, U32 hBits, U32 mls, const U64 hashSalt) {
+    /* Although some of these hashes do support hBits up to 64, some do not.
+     * To be on the safe side, always avoid hBits > 32. */
+    assert(hBits <= 32);
+
+    switch(mls)
+    {
+        default:
+        case 4: return ZSTD_hash4PtrS(p, hBits, (U32)hashSalt);
+        case 5: return ZSTD_hash5PtrS(p, hBits, hashSalt);
+        case 6: return ZSTD_hash6PtrS(p, hBits, hashSalt);
+        case 7: return ZSTD_hash7PtrS(p, hBits, hashSalt);
+        case 8: return ZSTD_hash8PtrS(p, hBits, hashSalt);
+    }
+}
+
+
 /* ZSTD_ipow() :
  * Return base^exponent.
  */
@@ -1167,10 +1198,15 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window,
                     (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd);
         assert(blockEndIdx >= loadedDictEnd);
 
-        if (blockEndIdx > loadedDictEnd + maxDist) {
+        if (blockEndIdx > loadedDictEnd + maxDist || loadedDictEnd != window->dictLimit) {
             /* On reaching window size, dictionaries are invalidated.
              * For simplification, if window size is reached anywhere within next block,
              * the dictionary is invalidated for the full block.
+             *
+             * We also have to invalidate the dictionary if ZSTD_window_update() has detected
+             * non-contiguous segments, which means that loadedDictEnd != window->dictLimit.
+             * loadedDictEnd may be 0, if forceWindow is true, but in that case we never use
+             * dictMatchState, so setting it to NULL is not a problem.
              */
             DEBUGLOG(6, "invalidating dictionary for current block (distance > windowSize)");
             *loadedDictEndPtr = 0;
@@ -1302,6 +1338,42 @@ MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max)
 
 #endif
 
+/* Short Cache */
+
+/* Normally, zstd matchfinders follow this flow:
+ *     1. Compute hash at ip
+ *     2. Load index from hashTable[hash]
+ *     3. Check if *ip == *(base + index)
+ * In dictionary compression, loading *(base + index) is often an L2 or even L3 miss.
+ *
+ * Short cache is an optimization which allows us to avoid step 3 most of the time
+ * when the data doesn't actually match. With short cache, the flow becomes:
+ *     1. Compute (hash, currentTag) at ip. currentTag is an 8-bit independent hash at ip.
+ *     2. Load (index, matchTag) from hashTable[hash]. See ZSTD_writeTaggedIndex to understand how this works.
+ *     3. Only if currentTag == matchTag, check *ip == *(base + index). Otherwise, continue.
+ *
+ * Currently, short cache is only implemented in CDict hashtables. Thus, its use is limited to
+ * dictMatchState matchfinders.
+ */
+#define ZSTD_SHORT_CACHE_TAG_BITS 8
+#define ZSTD_SHORT_CACHE_TAG_MASK ((1u << ZSTD_SHORT_CACHE_TAG_BITS) - 1)
+
+/* Helper function for ZSTD_fillHashTable and ZSTD_fillDoubleHashTable.
+ * Unpacks hashAndTag into (hash, tag), then packs (index, tag) into hashTable[hash]. */
+MEM_STATIC void ZSTD_writeTaggedIndex(U32* const hashTable, size_t hashAndTag, U32 index) {
+    size_t const hash = hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS;
+    U32 const tag = (U32)(hashAndTag & ZSTD_SHORT_CACHE_TAG_MASK);
+    assert(index >> (32 - ZSTD_SHORT_CACHE_TAG_BITS) == 0);
+    hashTable[hash] = (index << ZSTD_SHORT_CACHE_TAG_BITS) | tag;
+}
+
+/* Helper function for short cache matchfinders.
+ * Unpacks tag1 and tag2 from lower bits of packedTag1 and packedTag2, then checks if the tags match. */
+MEM_STATIC int ZSTD_comparePackedTags(size_t packedTag1, size_t packedTag2) {
+    U32 const tag1 = packedTag1 & ZSTD_SHORT_CACHE_TAG_MASK;
+    U32 const tag2 = packedTag2 & ZSTD_SHORT_CACHE_TAG_MASK;
+    return tag1 == tag2;
+}
 
 
 /* ===============================================================
@@ -1396,4 +1468,51 @@ U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat);
  */
 void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize);
 
+/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of
+ * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter.
+ * Note that the block delimiter must include the last literals of the block.
+ */
+size_t
+ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx,
+                                              ZSTD_sequencePosition* seqPos,
+                                        const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
+                                        const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch);
+
+/* Returns the number of bytes to move the current read position back by.
+ * Only non-zero if we ended up splitting a sequence.
+ * Otherwise, it may return a ZSTD error if something went wrong.
+ *
+ * This function will attempt to scan through blockSize bytes
+ * represented by the sequences in @inSeqs,
+ * storing any (partial) sequences.
+ *
+ * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to
+ * avoid splitting a match, or to avoid splitting a match such that it would produce a match
+ * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block.
+ */
+size_t
+ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos,
+                                   const ZSTD_Sequence* const inSeqs, size_t inSeqsSize,
+                                   const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch);
+
+
+/* ===============================================================
+ * Deprecated definitions that are still used internally to avoid
+ * deprecation warnings. These functions are exactly equivalent to
+ * their public variants, but avoid the deprecation warnings.
+ * =============================================================== */
+
+size_t ZSTD_compressBegin_usingCDict_deprecated(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
+
+size_t ZSTD_compressContinue_public(ZSTD_CCtx* cctx,
+                                    void* dst, size_t dstCapacity,
+                              const void* src, size_t srcSize);
+
+size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx,
+                               void* dst, size_t dstCapacity,
+                         const void* src, size_t srcSize);
+
+size_t ZSTD_compressBlock_deprecated(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
+
+
 #endif /* ZSTD_COMPRESS_H */
diff --git a/lib/zstd/compress/zstd_compress_literals.c b/lib/zstd/compress/zstd_compress_literals.c
index 52b0a8059aba..3e9ea46a670a 100644
--- a/lib/zstd/compress/zstd_compress_literals.c
+++ b/lib/zstd/compress/zstd_compress_literals.c
@@ -1,5 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -13,11 +14,36 @@
  ***************************************/
 #include "zstd_compress_literals.h"
 
+
+/* **************************************************************
+*  Debug Traces
+****************************************************************/
+#if DEBUGLEVEL >= 2
+
+static size_t showHexa(const void* src, size_t srcSize)
+{
+    const BYTE* const ip = (const BYTE*)src;
+    size_t u;
+    for (u=0; u<srcSize; u++) {
+        RAWLOG(5, " %02X", ip[u]); (void)ip;
+    }
+    RAWLOG(5, " \n");
+    return srcSize;
+}
+
+#endif
+
+
+/* **************************************************************
+*  Literals compression - special cases
+****************************************************************/
 size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
     BYTE* const ostart = (BYTE*)dst;
     U32   const flSize = 1 + (srcSize>31) + (srcSize>4095);
 
+    DEBUGLOG(5, "ZSTD_noCompressLiterals: srcSize=%zu, dstCapacity=%zu", srcSize, dstCapacity);
+
     RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, "");
 
     switch(flSize)
@@ -36,16 +62,30 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src,
     }
 
     ZSTD_memcpy(ostart + flSize, src, srcSize);
-    DEBUGLOG(5, "Raw literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize));
+    DEBUGLOG(5, "Raw (uncompressed) literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize));
     return srcSize + flSize;
 }
 
+static int allBytesIdentical(const void* src, size_t srcSize)
+{
+    assert(srcSize >= 1);
+    assert(src != NULL);
+    {   const BYTE b = ((const BYTE*)src)[0];
+        size_t p;
+        for (p=1; p<srcSize; p++) {
+            if (((const BYTE*)src)[p] != b) return 0;
+        }
+        return 1;
+    }
+}
+
 size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 {
     BYTE* const ostart = (BYTE*)dst;
     U32   const flSize = 1 + (srcSize>31) + (srcSize>4095);
 
-    (void)dstCapacity;  /* dstCapacity already guaranteed to be >=4, hence large enough */
+    assert(dstCapacity >= 4); (void)dstCapacity;
+    assert(allBytesIdentical(src, srcSize));
 
     switch(flSize)
     {
@@ -63,28 +103,51 @@ size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void*
     }
 
     ostart[flSize] = *(const BYTE*)src;
-    DEBUGLOG(5, "RLE literals: %u -> %u", (U32)srcSize, (U32)flSize + 1);
+    DEBUGLOG(5, "RLE : Repeated Literal (%02X: %u times) -> %u bytes encoded", ((const BYTE*)src)[0], (U32)srcSize, (U32)flSize + 1);
     return flSize+1;
 }
 
-size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
-                              ZSTD_hufCTables_t* nextHuf,
-                              ZSTD_strategy strategy, int disableLiteralCompression,
-                              void* dst, size_t dstCapacity,
-                        const void* src, size_t srcSize,
-                              void* entropyWorkspace, size_t entropyWorkspaceSize,
-                        const int bmi2,
-                        unsigned suspectUncompressible)
+/* ZSTD_minLiteralsToCompress() :
+ * returns minimal amount of literals
+ * for literal compression to even be attempted.
+ * Minimum is made tighter as compression strategy increases.
+ */
+static size_t
+ZSTD_minLiteralsToCompress(ZSTD_strategy strategy, HUF_repeat huf_repeat)
+{
+    assert((int)strategy >= 0);
+    assert((int)strategy <= 9);
+    /* btultra2 : min 8 bytes;
+     * then 2x larger for each successive compression strategy
+     * max threshold 64 bytes */
+    {   int const shift = MIN(9-(int)strategy, 3);
+        size_t const mintc = (huf_repeat == HUF_repeat_valid) ? 6 : (size_t)8 << shift;
+        DEBUGLOG(7, "minLiteralsToCompress = %zu", mintc);
+        return mintc;
+    }
+}
+
+size_t ZSTD_compressLiterals (
+                  void* dst, size_t dstCapacity,
+            const void* src, size_t srcSize,
+                  void* entropyWorkspace, size_t entropyWorkspaceSize,
+            const ZSTD_hufCTables_t* prevHuf,
+                  ZSTD_hufCTables_t* nextHuf,
+                  ZSTD_strategy strategy,
+                  int disableLiteralCompression,
+                  int suspectUncompressible,
+                  int bmi2)
 {
-    size_t const minGain = ZSTD_minGain(srcSize, strategy);
     size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB);
     BYTE*  const ostart = (BYTE*)dst;
     U32 singleStream = srcSize < 256;
     symbolEncodingType_e hType = set_compressed;
     size_t cLitSize;
 
-    DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i srcSize=%u)",
-                disableLiteralCompression, (U32)srcSize);
+    DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i, srcSize=%u, dstCapacity=%zu)",
+                disableLiteralCompression, (U32)srcSize, dstCapacity);
+
+    DEBUGLOG(6, "Completed literals listing (%zu bytes)", showHexa(src, srcSize));
 
     /* Prepare nextEntropy assuming reusing the existing table */
     ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
@@ -92,40 +155,51 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
     if (disableLiteralCompression)
         return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
 
-    /* small ? don't even attempt compression (speed opt) */
-#   define COMPRESS_LITERALS_SIZE_MIN 63
-    {   size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN;
-        if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
-    }
+    /* if too small, don't even attempt compression (speed opt) */
+    if (srcSize < ZSTD_minLiteralsToCompress(strategy, prevHuf->repeatMode))
+        return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
 
     RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression");
     {   HUF_repeat repeat = prevHuf->repeatMode;
-        int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
+        int const flags = 0
+            | (bmi2 ? HUF_flags_bmi2 : 0)
+            | (strategy < ZSTD_lazy && srcSize <= 1024 ? HUF_flags_preferRepeat : 0)
+            | (strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_flags_optimalDepth : 0)
+            | (suspectUncompressible ? HUF_flags_suspectUncompressible : 0);
+
+        typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int);
+        huf_compress_f huf_compress;
         if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
-        cLitSize = singleStream ?
-            HUF_compress1X_repeat(
-                ostart+lhSize, dstCapacity-lhSize, src, srcSize,
-                HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize,
-                (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2, suspectUncompressible) :
-            HUF_compress4X_repeat(
-                ostart+lhSize, dstCapacity-lhSize, src, srcSize,
-                HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize,
-                (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2, suspectUncompressible);
+        huf_compress = singleStream ? HUF_compress1X_repeat : HUF_compress4X_repeat;
+        cLitSize = huf_compress(ostart+lhSize, dstCapacity-lhSize,
+                                src, srcSize,
+                                HUF_SYMBOLVALUE_MAX, LitHufLog,
+                                entropyWorkspace, entropyWorkspaceSize,
+                                (HUF_CElt*)nextHuf->CTable,
+                                &repeat, flags);
+        DEBUGLOG(5, "%zu literals compressed into %zu bytes (before header)", srcSize, cLitSize);
         if (repeat != HUF_repeat_none) {
             /* reused the existing table */
-            DEBUGLOG(5, "Reusing previous huffman table");
+            DEBUGLOG(5, "reusing statistics from previous huffman block");
             hType = set_repeat;
         }
     }
 
-    if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) {
-        ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
-        return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
-    }
+    {   size_t const minGain = ZSTD_minGain(srcSize, strategy);
+        if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) {
+            ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+            return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
+    }   }
     if (cLitSize==1) {
-        ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
-        return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
-    }
+        /* A return value of 1 signals that the alphabet consists of a single symbol.
+         * However, in some rare circumstances, it could be the compressed size (a single byte).
+         * For that outcome to have a chance to happen, it's necessary that `srcSize < 8`.
+         * (it's also necessary to not generate statistics).
+         * Therefore, in such a case, actively check that all bytes are identical. */
+        if ((srcSize >= 8) || allBytesIdentical(src, srcSize)) {
+            ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
+            return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
+    }   }
 
     if (hType == set_compressed) {
         /* using a newly constructed table */
@@ -136,16 +210,19 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
     switch(lhSize)
     {
     case 3: /* 2 - 2 - 10 - 10 */
-        {   U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14);
+        if (!singleStream) assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
+        {   U32 const lhc = hType + ((U32)(!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14);
             MEM_writeLE24(ostart, lhc);
             break;
         }
     case 4: /* 2 - 2 - 14 - 14 */
+        assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
         {   U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18);
             MEM_writeLE32(ostart, lhc);
             break;
         }
     case 5: /* 2 - 2 - 18 - 18 */
+        assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS);
         {   U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22);
             MEM_writeLE32(ostart, lhc);
             ostart[4] = (BYTE)(cLitSize >> 10);
diff --git a/lib/zstd/compress/zstd_compress_literals.h b/lib/zstd/compress/zstd_compress_literals.h
index 9775fb97cb70..a2a85d6b69e5 100644
--- a/lib/zstd/compress/zstd_compress_literals.h
+++ b/lib/zstd/compress/zstd_compress_literals.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -16,16 +17,24 @@
 
 size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 
+/* ZSTD_compressRleLiteralsBlock() :
+ * Conditions :
+ * - All bytes in @src are identical
+ * - dstCapacity >= 4 */
 size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 
-/* If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */
-size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
-                              ZSTD_hufCTables_t* nextHuf,
-                              ZSTD_strategy strategy, int disableLiteralCompression,
-                              void* dst, size_t dstCapacity,
+/* ZSTD_compressLiterals():
+ * @entropyWorkspace: must be aligned on 4-bytes boundaries
+ * @entropyWorkspaceSize : must be >= HUF_WORKSPACE_SIZE
+ * @suspectUncompressible: sampling checks, to potentially skip huffman coding
+ */
+size_t ZSTD_compressLiterals (void* dst, size_t dstCapacity,
                         const void* src, size_t srcSize,
                               void* entropyWorkspace, size_t entropyWorkspaceSize,
-                        const int bmi2,
-                        unsigned suspectUncompressible);
+                        const ZSTD_hufCTables_t* prevHuf,
+                              ZSTD_hufCTables_t* nextHuf,
+                              ZSTD_strategy strategy, int disableLiteralCompression,
+                              int suspectUncompressible,
+                              int bmi2);
 
 #endif /* ZSTD_COMPRESS_LITERALS_H */
diff --git a/lib/zstd/compress/zstd_compress_sequences.c b/lib/zstd/compress/zstd_compress_sequences.c
index 21ddc1b37acf..5c028c78d889 100644
--- a/lib/zstd/compress/zstd_compress_sequences.c
+++ b/lib/zstd/compress/zstd_compress_sequences.c
@@ -1,5 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -58,7 +59,7 @@ static unsigned ZSTD_useLowProbCount(size_t const nbSeq)
 {
     /* Heuristic: This should cover most blocks <= 16K and
      * start to fade out after 16K to about 32K depending on
-     * comprssibility.
+     * compressibility.
      */
     return nbSeq >= 2048;
 }
@@ -166,7 +167,7 @@ ZSTD_selectEncodingType(
     if (mostFrequent == nbSeq) {
         *repeatMode = FSE_repeat_none;
         if (isDefaultAllowed && nbSeq <= 2) {
-            /* Prefer set_basic over set_rle when there are 2 or less symbols,
+            /* Prefer set_basic over set_rle when there are 2 or fewer symbols,
              * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol.
              * If basic encoding isn't possible, always choose RLE.
              */
diff --git a/lib/zstd/compress/zstd_compress_sequences.h b/lib/zstd/compress/zstd_compress_sequences.h
index 7991364c2f71..7fe6f4ff5cf2 100644
--- a/lib/zstd/compress/zstd_compress_sequences.h
+++ b/lib/zstd/compress/zstd_compress_sequences.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/zstd/compress/zstd_compress_superblock.c b/lib/zstd/compress/zstd_compress_superblock.c
index 17d836cc84e8..dbacbaf72733 100644
--- a/lib/zstd/compress/zstd_compress_superblock.c
+++ b/lib/zstd/compress/zstd_compress_superblock.c
@@ -1,5 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -36,13 +37,14 @@
  *      If it is set_compressed, first sub-block's literals section will be Treeless_Literals_Block
  *      and the following sub-blocks' literals sections will be Treeless_Literals_Block.
  *  @return : compressed size of literals section of a sub-block
- *            Or 0 if it unable to compress.
+ *            Or 0 if unable to compress.
  *            Or error code */
-static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
-                                    const ZSTD_hufCTablesMetadata_t* hufMetadata,
-                                    const BYTE* literals, size_t litSize,
-                                    void* dst, size_t dstSize,
-                                    const int bmi2, int writeEntropy, int* entropyWritten)
+static size_t
+ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
+                              const ZSTD_hufCTablesMetadata_t* hufMetadata,
+                              const BYTE* literals, size_t litSize,
+                              void* dst, size_t dstSize,
+                              const int bmi2, int writeEntropy, int* entropyWritten)
 {
     size_t const header = writeEntropy ? 200 : 0;
     size_t const lhSize = 3 + (litSize >= (1 KB - header)) + (litSize >= (16 KB - header));
@@ -53,8 +55,6 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
     symbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat;
     size_t cLitSize = 0;
 
-    (void)bmi2; /* TODO bmi2... */
-
     DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy);
 
     *entropyWritten = 0;
@@ -76,9 +76,9 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
         DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize);
     }
 
-    /* TODO bmi2 */
-    {   const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable)
-                                          : HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable);
+    {   int const flags = bmi2 ? HUF_flags_bmi2 : 0;
+        const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable, flags)
+                                          : HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable, flags);
         op += cSize;
         cLitSize += cSize;
         if (cSize == 0 || ERR_isError(cSize)) {
@@ -126,7 +126,11 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
     return op-ostart;
 }
 
-static size_t ZSTD_seqDecompressedSize(seqStore_t const* seqStore, const seqDef* sequences, size_t nbSeq, size_t litSize, int lastSequence) {
+static size_t
+ZSTD_seqDecompressedSize(seqStore_t const* seqStore,
+                   const seqDef* sequences, size_t nbSeq,
+                         size_t litSize, int lastSequence)
+{
     const seqDef* const sstart = sequences;
     const seqDef* const send = sequences + nbSeq;
     const seqDef* sp = sstart;
@@ -156,13 +160,14 @@ static size_t ZSTD_seqDecompressedSize(seqStore_t const* seqStore, const seqDef*
  *  @return : compressed size of sequences section of a sub-block
  *            Or 0 if it is unable to compress
  *            Or error code. */
-static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables,
-                                              const ZSTD_fseCTablesMetadata_t* fseMetadata,
-                                              const seqDef* sequences, size_t nbSeq,
-                                              const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode,
-                                              const ZSTD_CCtx_params* cctxParams,
-                                              void* dst, size_t dstCapacity,
-                                              const int bmi2, int writeEntropy, int* entropyWritten)
+static size_t
+ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables,
+                                const ZSTD_fseCTablesMetadata_t* fseMetadata,
+                                const seqDef* sequences, size_t nbSeq,
+                                const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode,
+                                const ZSTD_CCtx_params* cctxParams,
+                                void* dst, size_t dstCapacity,
+                                const int bmi2, int writeEntropy, int* entropyWritten)
 {
     const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN;
     BYTE* const ostart = (BYTE*)dst;
@@ -539,7 +544,7 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
             repcodes_t rep;
             ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep));
             for (seq = sstart; seq < sp; ++seq) {
-                ZSTD_updateRep(rep.rep, seq->offBase - 1, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0);
+                ZSTD_updateRep(rep.rep, seq->offBase, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0);
             }
             ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep));
         }
diff --git a/lib/zstd/compress/zstd_compress_superblock.h b/lib/zstd/compress/zstd_compress_superblock.h
index 224ece79546e..826bbc9e029b 100644
--- a/lib/zstd/compress/zstd_compress_superblock.h
+++ b/lib/zstd/compress/zstd_compress_superblock.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/zstd/compress/zstd_cwksp.h b/lib/zstd/compress/zstd_cwksp.h
index 349fc923c355..65ea53b62844 100644
--- a/lib/zstd/compress/zstd_cwksp.h
+++ b/lib/zstd/compress/zstd_cwksp.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -14,7 +15,9 @@
 /*-*************************************
 *  Dependencies
 ***************************************/
+#include "../common/allocations.h"  /* ZSTD_customMalloc, ZSTD_customFree */
 #include "../common/zstd_internal.h"
+#include "../common/portability_macros.h"
 
 
 /*-*************************************
@@ -41,8 +44,9 @@
 ***************************************/
 typedef enum {
     ZSTD_cwksp_alloc_objects,
-    ZSTD_cwksp_alloc_buffers,
-    ZSTD_cwksp_alloc_aligned
+    ZSTD_cwksp_alloc_aligned_init_once,
+    ZSTD_cwksp_alloc_aligned,
+    ZSTD_cwksp_alloc_buffers
 } ZSTD_cwksp_alloc_phase_e;
 
 /*
@@ -95,8 +99,8 @@ typedef enum {
  *
  * Workspace Layout:
  *
- * [                        ... workspace ...                         ]
- * [objects][tables ... ->] free space [<- ... aligned][<- ... buffers]
+ * [                        ... workspace ...                           ]
+ * [objects][tables ->] free space [<- buffers][<- aligned][<- init once]
  *
  * The various objects that live in the workspace are divided into the
  * following categories, and are allocated separately:
@@ -120,9 +124,18 @@ typedef enum {
  *   uint32_t arrays, all of whose values are between 0 and (nextSrc - base).
  *   Their sizes depend on the cparams. These tables are 64-byte aligned.
  *
- * - Aligned: these buffers are used for various purposes that require 4 byte
- *   alignment, but don't require any initialization before they're used. These
- *   buffers are each aligned to 64 bytes.
+ * - Init once: these buffers require to be initialized at least once before
+ *   use. They should be used when we want to skip memory initialization
+ *   while not triggering memory checkers (like Valgrind) when reading from
+ *   from this memory without writing to it first.
+ *   These buffers should be used carefully as they might contain data
+ *   from previous compressions.
+ *   Buffers are aligned to 64 bytes.
+ *
+ * - Aligned: these buffers don't require any initialization before they're
+ *   used. The user of the buffer should make sure they write into a buffer
+ *   location before reading from it.
+ *   Buffers are aligned to 64 bytes.
  *
  * - Buffers: these buffers are used for various purposes that don't require
  *   any alignment or initialization before they're used. This means they can
@@ -134,8 +147,9 @@ typedef enum {
  * correctly packed into the workspace buffer. That order is:
  *
  * 1. Objects
- * 2. Buffers
- * 3. Aligned/Tables
+ * 2. Init once / Tables
+ * 3. Aligned / Tables
+ * 4. Buffers / Tables
  *
  * Attempts to reserve objects of different types out of order will fail.
  */
@@ -147,6 +161,7 @@ typedef struct {
     void* tableEnd;
     void* tableValidEnd;
     void* allocStart;
+    void* initOnceStart;
 
     BYTE allocFailed;
     int workspaceOversizedDuration;
@@ -159,6 +174,7 @@ typedef struct {
 ***************************************/
 
 MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws);
+MEM_STATIC void*  ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws);
 
 MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) {
     (void)ws;
@@ -168,6 +184,8 @@ MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) {
     assert(ws->tableEnd <= ws->allocStart);
     assert(ws->tableValidEnd <= ws->allocStart);
     assert(ws->allocStart <= ws->workspaceEnd);
+    assert(ws->initOnceStart <= ZSTD_cwksp_initialAllocStart(ws));
+    assert(ws->workspace <= ws->initOnceStart);
 }
 
 /*
@@ -210,14 +228,10 @@ MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size) {
  * for internal purposes (currently only alignment).
  */
 MEM_STATIC size_t ZSTD_cwksp_slack_space_required(void) {
-    /* For alignment, the wksp will always allocate an additional n_1=[1, 64] bytes
-     * to align the beginning of tables section, as well as another n_2=[0, 63] bytes
-     * to align the beginning of the aligned section.
-     *
-     * n_1 + n_2 == 64 bytes if the cwksp is freshly allocated, due to tables and
-     * aligneds being sized in multiples of 64 bytes.
+    /* For alignment, the wksp will always allocate an additional 2*ZSTD_CWKSP_ALIGNMENT_BYTES
+     * bytes to align the beginning of tables section and end of buffers;
      */
-    size_t const slackSpace = ZSTD_CWKSP_ALIGNMENT_BYTES;
+    size_t const slackSpace = ZSTD_CWKSP_ALIGNMENT_BYTES * 2;
     return slackSpace;
 }
 
@@ -230,10 +244,18 @@ MEM_STATIC size_t ZSTD_cwksp_bytes_to_align_ptr(void* ptr, const size_t alignByt
     size_t const alignBytesMask = alignBytes - 1;
     size_t const bytes = (alignBytes - ((size_t)ptr & (alignBytesMask))) & alignBytesMask;
     assert((alignBytes & alignBytesMask) == 0);
-    assert(bytes != ZSTD_CWKSP_ALIGNMENT_BYTES);
+    assert(bytes < alignBytes);
     return bytes;
 }
 
+/*
+ * Returns the initial value for allocStart which is used to determine the position from
+ * which we can allocate from the end of the workspace.
+ */
+MEM_STATIC void*  ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws) {
+    return (void*)((size_t)ws->workspaceEnd & ~(ZSTD_CWKSP_ALIGNMENT_BYTES-1));
+}
+
 /*
  * Internal function. Do not use directly.
  * Reserves the given number of bytes within the aligned/buffer segment of the wksp,
@@ -274,27 +296,16 @@ ZSTD_cwksp_internal_advance_phase(ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase
 {
     assert(phase >= ws->phase);
     if (phase > ws->phase) {
-        /* Going from allocating objects to allocating buffers */
-        if (ws->phase < ZSTD_cwksp_alloc_buffers &&
-                phase >= ZSTD_cwksp_alloc_buffers) {
+        /* Going from allocating objects to allocating initOnce / tables */
+        if (ws->phase < ZSTD_cwksp_alloc_aligned_init_once &&
+            phase >= ZSTD_cwksp_alloc_aligned_init_once) {
             ws->tableValidEnd = ws->objectEnd;
-        }
+            ws->initOnceStart = ZSTD_cwksp_initialAllocStart(ws);
 
-        /* Going from allocating buffers to allocating aligneds/tables */
-        if (ws->phase < ZSTD_cwksp_alloc_aligned &&
-                phase >= ZSTD_cwksp_alloc_aligned) {
-            {   /* Align the start of the "aligned" to 64 bytes. Use [1, 64] bytes. */
-                size_t const bytesToAlign =
-                    ZSTD_CWKSP_ALIGNMENT_BYTES - ZSTD_cwksp_bytes_to_align_ptr(ws->allocStart, ZSTD_CWKSP_ALIGNMENT_BYTES);
-                DEBUGLOG(5, "reserving aligned alignment addtl space: %zu", bytesToAlign);
-                ZSTD_STATIC_ASSERT((ZSTD_CWKSP_ALIGNMENT_BYTES & (ZSTD_CWKSP_ALIGNMENT_BYTES - 1)) == 0); /* power of 2 */
-                RETURN_ERROR_IF(!ZSTD_cwksp_reserve_internal_buffer_space(ws, bytesToAlign),
-                                memory_allocation, "aligned phase - alignment initial allocation failed!");
-            }
             {   /* Align the start of the tables to 64 bytes. Use [0, 63] bytes */
-                void* const alloc = ws->objectEnd;
+                void *const alloc = ws->objectEnd;
                 size_t const bytesToAlign = ZSTD_cwksp_bytes_to_align_ptr(alloc, ZSTD_CWKSP_ALIGNMENT_BYTES);
-                void* const objectEnd = (BYTE*)alloc + bytesToAlign;
+                void *const objectEnd = (BYTE *) alloc + bytesToAlign;
                 DEBUGLOG(5, "reserving table alignment addtl space: %zu", bytesToAlign);
                 RETURN_ERROR_IF(objectEnd > ws->workspaceEnd, memory_allocation,
                                 "table phase - alignment initial allocation failed!");
@@ -302,7 +313,9 @@ ZSTD_cwksp_internal_advance_phase(ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase
                 ws->tableEnd = objectEnd;  /* table area starts being empty */
                 if (ws->tableValidEnd < ws->tableEnd) {
                     ws->tableValidEnd = ws->tableEnd;
-        }   }   }
+                }
+            }
+        }
         ws->phase = phase;
         ZSTD_cwksp_assert_internal_consistency(ws);
     }
@@ -314,7 +327,7 @@ ZSTD_cwksp_internal_advance_phase(ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase
  */
 MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr)
 {
-    return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd);
+    return (ptr != NULL) && (ws->workspace <= ptr) && (ptr < ws->workspaceEnd);
 }
 
 /*
@@ -343,6 +356,33 @@ MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes)
     return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers);
 }
 
+/*
+ * Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes).
+ * This memory has been initialized at least once in the past.
+ * This doesn't mean it has been initialized this time, and it might contain data from previous
+ * operations.
+ * The main usage is for algorithms that might need read access into uninitialized memory.
+ * The algorithm must maintain safety under these conditions and must make sure it doesn't
+ * leak any of the past data (directly or in side channels).
+ */
+MEM_STATIC void* ZSTD_cwksp_reserve_aligned_init_once(ZSTD_cwksp* ws, size_t bytes)
+{
+    size_t const alignedBytes = ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES);
+    void* ptr = ZSTD_cwksp_reserve_internal(ws, alignedBytes, ZSTD_cwksp_alloc_aligned_init_once);
+    assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0);
+    if(ptr && ptr < ws->initOnceStart) {
+        /* We assume the memory following the current allocation is either:
+         * 1. Not usable as initOnce memory (end of workspace)
+         * 2. Another initOnce buffer that has been allocated before (and so was previously memset)
+         * 3. An ASAN redzone, in which case we don't want to write on it
+         * For these reasons it should be fine to not explicitly zero every byte up to ws->initOnceStart.
+         * Note that we assume here that MSAN and ASAN cannot run in the same time. */
+        ZSTD_memset(ptr, 0, MIN((size_t)((U8*)ws->initOnceStart - (U8*)ptr), alignedBytes));
+        ws->initOnceStart = ptr;
+    }
+    return ptr;
+}
+
 /*
  * Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes).
  */
@@ -361,13 +401,17 @@ MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes)
  */
 MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes)
 {
-    const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned;
+    const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned_init_once;
     void* alloc;
     void* end;
     void* top;
 
-    if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase))) {
-        return NULL;
+    /* We can only start allocating tables after we are done reserving space for objects at the
+     * start of the workspace */
+    if(ws->phase < phase) {
+        if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase))) {
+            return NULL;
+        }
     }
     alloc = ws->tableEnd;
     end = (BYTE *)alloc + bytes;
@@ -451,7 +495,7 @@ MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) {
     assert(ws->tableValidEnd >= ws->objectEnd);
     assert(ws->tableValidEnd <= ws->allocStart);
     if (ws->tableValidEnd < ws->tableEnd) {
-        ZSTD_memset(ws->tableValidEnd, 0, (BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd);
+        ZSTD_memset(ws->tableValidEnd, 0, (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd));
     }
     ZSTD_cwksp_mark_tables_clean(ws);
 }
@@ -478,10 +522,10 @@ MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) {
 
 
     ws->tableEnd = ws->objectEnd;
-    ws->allocStart = ws->workspaceEnd;
+    ws->allocStart = ZSTD_cwksp_initialAllocStart(ws);
     ws->allocFailed = 0;
-    if (ws->phase > ZSTD_cwksp_alloc_buffers) {
-        ws->phase = ZSTD_cwksp_alloc_buffers;
+    if (ws->phase > ZSTD_cwksp_alloc_aligned_init_once) {
+        ws->phase = ZSTD_cwksp_alloc_aligned_init_once;
     }
     ZSTD_cwksp_assert_internal_consistency(ws);
 }
@@ -498,6 +542,7 @@ MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size, ZSTD_c
     ws->workspaceEnd = (BYTE*)start + size;
     ws->objectEnd = ws->workspace;
     ws->tableValidEnd = ws->objectEnd;
+    ws->initOnceStart = ZSTD_cwksp_initialAllocStart(ws);
     ws->phase = ZSTD_cwksp_alloc_objects;
     ws->isStatic = isStatic;
     ZSTD_cwksp_clear(ws);
@@ -550,17 +595,11 @@ MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) {
  * Returns if the estimated space needed for a wksp is within an acceptable limit of the
  * actual amount of space used.
  */
-MEM_STATIC int ZSTD_cwksp_estimated_space_within_bounds(const ZSTD_cwksp* const ws,
-                                                        size_t const estimatedSpace, int resizedWorkspace) {
-    if (resizedWorkspace) {
-        /* Resized/newly allocated wksp should have exact bounds */
-        return ZSTD_cwksp_used(ws) == estimatedSpace;
-    } else {
-        /* Due to alignment, when reusing a workspace, we can actually consume 63 fewer or more bytes
-         * than estimatedSpace. See the comments in zstd_cwksp.h for details.
-         */
-        return (ZSTD_cwksp_used(ws) >= estimatedSpace - 63) && (ZSTD_cwksp_used(ws) <= estimatedSpace + 63);
-    }
+MEM_STATIC int ZSTD_cwksp_estimated_space_within_bounds(const ZSTD_cwksp *const ws, size_t const estimatedSpace) {
+    /* We have an alignment space between objects and tables between tables and buffers, so we can have up to twice
+     * the alignment bytes difference between estimation and actual usage */
+    return (estimatedSpace - ZSTD_cwksp_slack_space_required()) <= ZSTD_cwksp_used(ws) &&
+           ZSTD_cwksp_used(ws) <= estimatedSpace;
 }
 
 
diff --git a/lib/zstd/compress/zstd_double_fast.c b/lib/zstd/compress/zstd_double_fast.c
index 76933dea2624..ab9440a99603 100644
--- a/lib/zstd/compress/zstd_double_fast.c
+++ b/lib/zstd/compress/zstd_double_fast.c
@@ -1,5 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -11,8 +12,43 @@
 #include "zstd_compress_internal.h"
 #include "zstd_double_fast.h"
 
+static void ZSTD_fillDoubleHashTableForCDict(ZSTD_matchState_t* ms,
+                              void const* end, ZSTD_dictTableLoadMethod_e dtlm)
+{
+    const ZSTD_compressionParameters* const cParams = &ms->cParams;
+    U32* const hashLarge = ms->hashTable;
+    U32  const hBitsL = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS;
+    U32  const mls = cParams->minMatch;
+    U32* const hashSmall = ms->chainTable;
+    U32  const hBitsS = cParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS;
+    const BYTE* const base = ms->window.base;
+    const BYTE* ip = base + ms->nextToUpdate;
+    const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
+    const U32 fastHashFillStep = 3;
 
-void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
+    /* Always insert every fastHashFillStep position into the hash tables.
+     * Insert the other positions into the large hash table if their entry
+     * is empty.
+     */
+    for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) {
+        U32 const curr = (U32)(ip - base);
+        U32 i;
+        for (i = 0; i < fastHashFillStep; ++i) {
+            size_t const smHashAndTag = ZSTD_hashPtr(ip + i, hBitsS, mls);
+            size_t const lgHashAndTag = ZSTD_hashPtr(ip + i, hBitsL, 8);
+            if (i == 0) {
+                ZSTD_writeTaggedIndex(hashSmall, smHashAndTag, curr + i);
+            }
+            if (i == 0 || hashLarge[lgHashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) {
+                ZSTD_writeTaggedIndex(hashLarge, lgHashAndTag, curr + i);
+            }
+            /* Only load extra positions for ZSTD_dtlm_full */
+            if (dtlm == ZSTD_dtlm_fast)
+                break;
+    }   }
+}
+
+static void ZSTD_fillDoubleHashTableForCCtx(ZSTD_matchState_t* ms,
                               void const* end, ZSTD_dictTableLoadMethod_e dtlm)
 {
     const ZSTD_compressionParameters* const cParams = &ms->cParams;
@@ -43,7 +79,19 @@ void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
             /* Only load extra positions for ZSTD_dtlm_full */
             if (dtlm == ZSTD_dtlm_fast)
                 break;
-    }   }
+        }   }
+}
+
+void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
+                        const void* const end,
+                        ZSTD_dictTableLoadMethod_e dtlm,
+                        ZSTD_tableFillPurpose_e tfp)
+{
+    if (tfp == ZSTD_tfp_forCDict) {
+        ZSTD_fillDoubleHashTableForCDict(ms, end, dtlm);
+    } else {
+        ZSTD_fillDoubleHashTableForCCtx(ms, end, dtlm);
+    }
 }
 
 
@@ -67,7 +115,7 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic(
     const BYTE* const iend = istart + srcSize;
     const BYTE* const ilimit = iend - HASH_READ_SIZE;
     U32 offset_1=rep[0], offset_2=rep[1];
-    U32 offsetSaved = 0;
+    U32 offsetSaved1 = 0, offsetSaved2 = 0;
 
     size_t mLength;
     U32 offset;
@@ -100,8 +148,8 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic(
         U32 const current = (U32)(ip - base);
         U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, current, cParams->windowLog);
         U32 const maxRep = current - windowLow;
-        if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0;
-        if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0;
+        if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0;
+        if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0;
     }
 
     /* Outer Loop: one iteration per match found and stored */
@@ -131,7 +179,7 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic(
             if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) {
                 mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4;
                 ip++;
-                ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength);
+                ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
                 goto _match_stored;
             }
 
@@ -175,9 +223,13 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic(
         } while (ip1 <= ilimit);
 
 _cleanup:
+        /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0),
+         * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */
+        offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2;
+
         /* save reps for next block */
-        rep[0] = offset_1 ? offset_1 : offsetSaved;
-        rep[1] = offset_2 ? offset_2 : offsetSaved;
+        rep[0] = offset_1 ? offset_1 : offsetSaved1;
+        rep[1] = offset_2 ? offset_2 : offsetSaved2;
 
         /* Return the last literals size */
         return (size_t)(iend - anchor);
@@ -217,7 +269,7 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic(
             hashLong[hl1] = (U32)(ip1 - base);
         }
 
-        ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
+        ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
 
 _match_stored:
         /* match found */
@@ -243,7 +295,7 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic(
                 U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff;  /* swap offset_2 <=> offset_1 */
                 hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base);
                 hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base);
-                ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, rLength);
+                ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, rLength);
                 ip += rLength;
                 anchor = ip;
                 continue;   /* faster when present ... (?) */
@@ -275,7 +327,6 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
     const BYTE* const iend = istart + srcSize;
     const BYTE* const ilimit = iend - HASH_READ_SIZE;
     U32 offset_1=rep[0], offset_2=rep[1];
-    U32 offsetSaved = 0;
 
     const ZSTD_matchState_t* const dms = ms->dictMatchState;
     const ZSTD_compressionParameters* const dictCParams = &dms->cParams;
@@ -286,8 +337,8 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
     const BYTE* const dictStart    = dictBase + dictStartIndex;
     const BYTE* const dictEnd      = dms->window.nextSrc;
     const U32 dictIndexDelta       = prefixLowestIndex - (U32)(dictEnd - dictBase);
-    const U32 dictHBitsL           = dictCParams->hashLog;
-    const U32 dictHBitsS           = dictCParams->chainLog;
+    const U32 dictHBitsL           = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS;
+    const U32 dictHBitsS           = dictCParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS;
     const U32 dictAndPrefixLength  = (U32)((ip - prefixLowest) + (dictEnd - dictStart));
 
     DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_dictMatchState_generic");
@@ -295,6 +346,13 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
     /* if a dictionary is attached, it must be within window range */
     assert(ms->window.dictLimit + (1U << cParams->windowLog) >= endIndex);
 
+    if (ms->prefetchCDictTables) {
+        size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32);
+        size_t const chainTableBytes = (((size_t)1) << dictCParams->chainLog) * sizeof(U32);
+        PREFETCH_AREA(dictHashLong, hashTableBytes)
+        PREFETCH_AREA(dictHashSmall, chainTableBytes)
+    }
+
     /* init */
     ip += (dictAndPrefixLength == 0);
 
@@ -309,8 +367,12 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
         U32 offset;
         size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8);
         size_t const h = ZSTD_hashPtr(ip, hBitsS, mls);
-        size_t const dictHL = ZSTD_hashPtr(ip, dictHBitsL, 8);
-        size_t const dictHS = ZSTD_hashPtr(ip, dictHBitsS, mls);
+        size_t const dictHashAndTagL = ZSTD_hashPtr(ip, dictHBitsL, 8);
+        size_t const dictHashAndTagS = ZSTD_hashPtr(ip, dictHBitsS, mls);
+        U32 const dictMatchIndexAndTagL = dictHashLong[dictHashAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS];
+        U32 const dictMatchIndexAndTagS = dictHashSmall[dictHashAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS];
+        int const dictTagsMatchL = ZSTD_comparePackedTags(dictMatchIndexAndTagL, dictHashAndTagL);
+        int const dictTagsMatchS = ZSTD_comparePackedTags(dictMatchIndexAndTagS, dictHashAndTagS);
         U32 const curr = (U32)(ip-base);
         U32 const matchIndexL = hashLong[h2];
         U32 matchIndexS = hashSmall[h];
@@ -328,7 +390,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
             const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
             mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
             ip++;
-            ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength);
+            ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
             goto _match_stored;
         }
 
@@ -340,9 +402,9 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
                 while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */
                 goto _match_found;
             }
-        } else {
+        } else if (dictTagsMatchL) {
             /* check dictMatchState long match */
-            U32 const dictMatchIndexL = dictHashLong[dictHL];
+            U32 const dictMatchIndexL = dictMatchIndexAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS;
             const BYTE* dictMatchL = dictBase + dictMatchIndexL;
             assert(dictMatchL < dictEnd);
 
@@ -358,9 +420,9 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
             if (MEM_read32(match) == MEM_read32(ip)) {
                 goto _search_next_long;
             }
-        } else {
+        } else if (dictTagsMatchS) {
             /* check dictMatchState short match */
-            U32 const dictMatchIndexS = dictHashSmall[dictHS];
+            U32 const dictMatchIndexS = dictMatchIndexAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS;
             match = dictBase + dictMatchIndexS;
             matchIndexS = dictMatchIndexS + dictIndexDelta;
 
@@ -375,10 +437,11 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
         continue;
 
 _search_next_long:
-
         {   size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
-            size_t const dictHLNext = ZSTD_hashPtr(ip+1, dictHBitsL, 8);
+            size_t const dictHashAndTagL3 = ZSTD_hashPtr(ip+1, dictHBitsL, 8);
             U32 const matchIndexL3 = hashLong[hl3];
+            U32 const dictMatchIndexAndTagL3 = dictHashLong[dictHashAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS];
+            int const dictTagsMatchL3 = ZSTD_comparePackedTags(dictMatchIndexAndTagL3, dictHashAndTagL3);
             const BYTE* matchL3 = base + matchIndexL3;
             hashLong[hl3] = curr + 1;
 
@@ -391,9 +454,9 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
                     while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */
                     goto _match_found;
                 }
-            } else {
+            } else if (dictTagsMatchL3) {
                 /* check dict long +1 match */
-                U32 const dictMatchIndexL3 = dictHashLong[dictHLNext];
+                U32 const dictMatchIndexL3 = dictMatchIndexAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS;
                 const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3;
                 assert(dictMatchL3 < dictEnd);
                 if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) {
@@ -419,7 +482,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
         offset_2 = offset_1;
         offset_1 = offset;
 
-        ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
+        ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
 
 _match_stored:
         /* match found */
@@ -448,7 +511,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
                     const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend;
                     size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4;
                     U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset;   /* swap offset_2 <=> offset_1 */
-                    ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, repLength2);
+                    ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2);
                     hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
                     hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
                     ip += repLength2;
@@ -461,8 +524,8 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic(
     }   /* while (ip < ilimit) */
 
     /* save reps for next block */
-    rep[0] = offset_1 ? offset_1 : offsetSaved;
-    rep[1] = offset_2 ? offset_2 : offsetSaved;
+    rep[0] = offset_1;
+    rep[1] = offset_2;
 
     /* Return the last literals size */
     return (size_t)(iend - anchor);
@@ -585,7 +648,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
             const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
             mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
             ip++;
-            ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength);
+            ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
         } else {
             if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) {
                 const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend;
@@ -596,7 +659,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
                 while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; }   /* catch up */
                 offset_2 = offset_1;
                 offset_1 = offset;
-                ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
+                ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
 
             } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) {
                 size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8);
@@ -621,7 +684,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
                 }
                 offset_2 = offset_1;
                 offset_1 = offset;
-                ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
+                ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
 
             } else {
                 ip += ((ip-anchor) >> kSearchStrength) + 1;
@@ -653,7 +716,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
                     const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
                     size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
                     U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset;   /* swap offset_2 <=> offset_1 */
-                    ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, repLength2);
+                    ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2);
                     hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2;
                     hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2;
                     ip += repLength2;
diff --git a/lib/zstd/compress/zstd_double_fast.h b/lib/zstd/compress/zstd_double_fast.h
index 6822bde65a1d..0204f12e4cf7 100644
--- a/lib/zstd/compress/zstd_double_fast.h
+++ b/lib/zstd/compress/zstd_double_fast.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -16,7 +17,8 @@
 #include "zstd_compress_internal.h"     /* ZSTD_CCtx, size_t */
 
 void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
-                              void const* end, ZSTD_dictTableLoadMethod_e dtlm);
+                              void const* end, ZSTD_dictTableLoadMethod_e dtlm,
+                              ZSTD_tableFillPurpose_e tfp);
 size_t ZSTD_compressBlock_doubleFast(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
         void const* src, size_t srcSize);
diff --git a/lib/zstd/compress/zstd_fast.c b/lib/zstd/compress/zstd_fast.c
index a752e6beab52..3399b39c5dbc 100644
--- a/lib/zstd/compress/zstd_fast.c
+++ b/lib/zstd/compress/zstd_fast.c
@@ -1,5 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -11,8 +12,42 @@
 #include "zstd_compress_internal.h"  /* ZSTD_hashPtr, ZSTD_count, ZSTD_storeSeq */
 #include "zstd_fast.h"
 
+static void ZSTD_fillHashTableForCDict(ZSTD_matchState_t* ms,
+                        const void* const end,
+                        ZSTD_dictTableLoadMethod_e dtlm)
+{
+    const ZSTD_compressionParameters* const cParams = &ms->cParams;
+    U32* const hashTable = ms->hashTable;
+    U32  const hBits = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS;
+    U32  const mls = cParams->minMatch;
+    const BYTE* const base = ms->window.base;
+    const BYTE* ip = base + ms->nextToUpdate;
+    const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
+    const U32 fastHashFillStep = 3;
 
-void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
+    /* Currently, we always use ZSTD_dtlm_full for filling CDict tables.
+     * Feel free to remove this assert if there's a good reason! */
+    assert(dtlm == ZSTD_dtlm_full);
+
+    /* Always insert every fastHashFillStep position into the hash table.
+     * Insert the other positions if their hash entry is empty.
+     */
+    for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) {
+        U32 const curr = (U32)(ip - base);
+        {   size_t const hashAndTag = ZSTD_hashPtr(ip, hBits, mls);
+            ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr);   }
+
+        if (dtlm == ZSTD_dtlm_fast) continue;
+        /* Only load extra positions for ZSTD_dtlm_full */
+        {   U32 p;
+            for (p = 1; p < fastHashFillStep; ++p) {
+                size_t const hashAndTag = ZSTD_hashPtr(ip + p, hBits, mls);
+                if (hashTable[hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) {  /* not yet filled */
+                    ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr + p);
+                }   }   }   }
+}
+
+static void ZSTD_fillHashTableForCCtx(ZSTD_matchState_t* ms,
                         const void* const end,
                         ZSTD_dictTableLoadMethod_e dtlm)
 {
@@ -25,6 +60,10 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
     const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
     const U32 fastHashFillStep = 3;
 
+    /* Currently, we always use ZSTD_dtlm_fast for filling CCtx tables.
+     * Feel free to remove this assert if there's a good reason! */
+    assert(dtlm == ZSTD_dtlm_fast);
+
     /* Always insert every fastHashFillStep position into the hash table.
      * Insert the other positions if their hash entry is empty.
      */
@@ -42,6 +81,18 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
     }   }   }   }
 }
 
+void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
+                        const void* const end,
+                        ZSTD_dictTableLoadMethod_e dtlm,
+                        ZSTD_tableFillPurpose_e tfp)
+{
+    if (tfp == ZSTD_tfp_forCDict) {
+        ZSTD_fillHashTableForCDict(ms, end, dtlm);
+    } else {
+        ZSTD_fillHashTableForCCtx(ms, end, dtlm);
+    }
+}
+
 
 /*
  * If you squint hard enough (and ignore repcodes), the search operation at any
@@ -117,7 +168,7 @@ ZSTD_compressBlock_fast_noDict_generic(
 
     U32 rep_offset1 = rep[0];
     U32 rep_offset2 = rep[1];
-    U32 offsetSaved = 0;
+    U32 offsetSaved1 = 0, offsetSaved2 = 0;
 
     size_t hash0; /* hash for ip0 */
     size_t hash1; /* hash for ip1 */
@@ -141,8 +192,8 @@ ZSTD_compressBlock_fast_noDict_generic(
     {   U32 const curr = (U32)(ip0 - base);
         U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, cParams->windowLog);
         U32 const maxRep = curr - windowLow;
-        if (rep_offset2 > maxRep) offsetSaved = rep_offset2, rep_offset2 = 0;
-        if (rep_offset1 > maxRep) offsetSaved = rep_offset1, rep_offset1 = 0;
+        if (rep_offset2 > maxRep) offsetSaved2 = rep_offset2, rep_offset2 = 0;
+        if (rep_offset1 > maxRep) offsetSaved1 = rep_offset1, rep_offset1 = 0;
     }
 
     /* start each op */
@@ -180,8 +231,14 @@ ZSTD_compressBlock_fast_noDict_generic(
             mLength = ip0[-1] == match0[-1];
             ip0 -= mLength;
             match0 -= mLength;
-            offcode = STORE_REPCODE_1;
+            offcode = REPCODE1_TO_OFFBASE;
             mLength += 4;
+
+            /* First write next hash table entry; we've already calculated it.
+             * This write is known to be safe because the ip1 is before the
+             * repcode (ip2). */
+            hashTable[hash1] = (U32)(ip1 - base);
+
             goto _match;
         }
 
@@ -195,6 +252,12 @@ ZSTD_compressBlock_fast_noDict_generic(
         /* check match at ip[0] */
         if (MEM_read32(ip0) == mval) {
             /* found a match! */
+
+            /* First write next hash table entry; we've already calculated it.
+             * This write is known to be safe because the ip1 == ip0 + 1, so
+             * we know we will resume searching after ip1 */
+            hashTable[hash1] = (U32)(ip1 - base);
+
             goto _offset;
         }
 
@@ -224,6 +287,21 @@ ZSTD_compressBlock_fast_noDict_generic(
         /* check match at ip[0] */
         if (MEM_read32(ip0) == mval) {
             /* found a match! */
+
+            /* first write next hash table entry; we've already calculated it */
+            if (step <= 4) {
+                /* We need to avoid writing an index into the hash table >= the
+                 * position at which we will pick up our searching after we've
+                 * taken this match.
+                 *
+                 * The minimum possible match has length 4, so the earliest ip0
+                 * can be after we take this match will be the current ip0 + 4.
+                 * ip1 is ip0 + step - 1. If ip1 is >= ip0 + 4, we can't safely
+                 * write this position.
+                 */
+                hashTable[hash1] = (U32)(ip1 - base);
+            }
+
             goto _offset;
         }
 
@@ -254,9 +332,24 @@ ZSTD_compressBlock_fast_noDict_generic(
      * However, it seems to be a meaningful performance hit to try to search
      * them. So let's not. */
 
+    /* When the repcodes are outside of the prefix, we set them to zero before the loop.
+     * When the offsets are still zero, we need to restore them after the block to have a correct
+     * repcode history. If only one offset was invalid, it is easy. The tricky case is when both
+     * offsets were invalid. We need to figure out which offset to refill with.
+     *     - If both offsets are zero they are in the same order.
+     *     - If both offsets are non-zero, we won't restore the offsets from `offsetSaved[12]`.
+     *     - If only one is zero, we need to decide which offset to restore.
+     *         - If rep_offset1 is non-zero, then rep_offset2 must be offsetSaved1.
+     *         - It is impossible for rep_offset2 to be non-zero.
+     *
+     * So if rep_offset1 started invalid (offsetSaved1 != 0) and became valid (rep_offset1 != 0), then
+     * set rep[0] = rep_offset1 and rep[1] = offsetSaved1.
+     */
+    offsetSaved2 = ((offsetSaved1 != 0) && (rep_offset1 != 0)) ? offsetSaved1 : offsetSaved2;
+
     /* save reps for next block */
-    rep[0] = rep_offset1 ? rep_offset1 : offsetSaved;
-    rep[1] = rep_offset2 ? rep_offset2 : offsetSaved;
+    rep[0] = rep_offset1 ? rep_offset1 : offsetSaved1;
+    rep[1] = rep_offset2 ? rep_offset2 : offsetSaved2;
 
     /* Return the last literals size */
     return (size_t)(iend - anchor);
@@ -267,7 +360,7 @@ ZSTD_compressBlock_fast_noDict_generic(
     match0 = base + idx;
     rep_offset2 = rep_offset1;
     rep_offset1 = (U32)(ip0-match0);
-    offcode = STORE_OFFSET(rep_offset1);
+    offcode = OFFSET_TO_OFFBASE(rep_offset1);
     mLength = 4;
 
     /* Count the backwards match length. */
@@ -287,11 +380,6 @@ ZSTD_compressBlock_fast_noDict_generic(
     ip0 += mLength;
     anchor = ip0;
 
-    /* write next hash table entry */
-    if (ip1 < ip0) {
-        hashTable[hash1] = (U32)(ip1 - base);
-    }
-
     /* Fill table and check for immediate repcode. */
     if (ip0 <= ilimit) {
         /* Fill Table */
@@ -306,7 +394,7 @@ ZSTD_compressBlock_fast_noDict_generic(
                 { U32 const tmpOff = rep_offset2; rep_offset2 = rep_offset1; rep_offset1 = tmpOff; } /* swap rep_offset2 <=> rep_offset1 */
                 hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base);
                 ip0 += rLength;
-                ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, STORE_REPCODE_1, rLength);
+                ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, REPCODE1_TO_OFFBASE, rLength);
                 anchor = ip0;
                 continue;   /* faster when present (confirmed on gcc-8) ... (?) */
     }   }   }
@@ -380,14 +468,14 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
     U32 const stepSize = cParams->targetLength + !(cParams->targetLength);
     const BYTE* const base = ms->window.base;
     const BYTE* const istart = (const BYTE*)src;
-    const BYTE* ip = istart;
+    const BYTE* ip0 = istart;
+    const BYTE* ip1 = ip0 + stepSize; /* we assert below that stepSize >= 1 */
     const BYTE* anchor = istart;
     const U32   prefixStartIndex = ms->window.dictLimit;
     const BYTE* const prefixStart = base + prefixStartIndex;
     const BYTE* const iend = istart + srcSize;
     const BYTE* const ilimit = iend - HASH_READ_SIZE;
     U32 offset_1=rep[0], offset_2=rep[1];
-    U32 offsetSaved = 0;
 
     const ZSTD_matchState_t* const dms = ms->dictMatchState;
     const ZSTD_compressionParameters* const dictCParams = &dms->cParams ;
@@ -397,13 +485,13 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
     const BYTE* const dictStart    = dictBase + dictStartIndex;
     const BYTE* const dictEnd      = dms->window.nextSrc;
     const U32 dictIndexDelta       = prefixStartIndex - (U32)(dictEnd - dictBase);
-    const U32 dictAndPrefixLength  = (U32)(ip - prefixStart + dictEnd - dictStart);
-    const U32 dictHLog             = dictCParams->hashLog;
+    const U32 dictAndPrefixLength  = (U32)(istart - prefixStart + dictEnd - dictStart);
+    const U32 dictHBits            = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS;
 
     /* if a dictionary is still attached, it necessarily means that
      * it is within window size. So we just check it. */
     const U32 maxDistance = 1U << cParams->windowLog;
-    const U32 endIndex = (U32)((size_t)(ip - base) + srcSize);
+    const U32 endIndex = (U32)((size_t)(istart - base) + srcSize);
     assert(endIndex - prefixStartIndex <= maxDistance);
     (void)maxDistance; (void)endIndex;   /* these variables are not used when assert() is disabled */
 
@@ -413,106 +501,155 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
      * when translating a dict index into a local index */
     assert(prefixStartIndex >= (U32)(dictEnd - dictBase));
 
+    if (ms->prefetchCDictTables) {
+        size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32);
+        PREFETCH_AREA(dictHashTable, hashTableBytes)
+    }
+
     /* init */
     DEBUGLOG(5, "ZSTD_compressBlock_fast_dictMatchState_generic");
-    ip += (dictAndPrefixLength == 0);
+    ip0 += (dictAndPrefixLength == 0);
     /* dictMatchState repCode checks don't currently handle repCode == 0
      * disabling. */
     assert(offset_1 <= dictAndPrefixLength);
     assert(offset_2 <= dictAndPrefixLength);
 
-    /* Main Search Loop */
-    while (ip < ilimit) {   /* < instead of <=, because repcode check at (ip+1) */
+    /* Outer search loop */
+    assert(stepSize >= 1);
+    while (ip1 <= ilimit) {   /* repcode check at (ip0 + 1) is safe because ip0 < ip1 */
         size_t mLength;
-        size_t const h = ZSTD_hashPtr(ip, hlog, mls);
-        U32 const curr = (U32)(ip-base);
-        U32 const matchIndex = hashTable[h];
-        const BYTE* match = base + matchIndex;
-        const U32 repIndex = curr + 1 - offset_1;
-        const BYTE* repMatch = (repIndex < prefixStartIndex) ?
-                               dictBase + (repIndex - dictIndexDelta) :
-                               base + repIndex;
-        hashTable[h] = curr;   /* update hash table */
-
-        if ( ((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */
-          && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
-            const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
-            mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
-            ip++;
-            ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength);
-        } else if ( (matchIndex <= prefixStartIndex) ) {
-            size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls);
-            U32 const dictMatchIndex = dictHashTable[dictHash];
-            const BYTE* dictMatch = dictBase + dictMatchIndex;
-            if (dictMatchIndex <= dictStartIndex ||
-                MEM_read32(dictMatch) != MEM_read32(ip)) {
-                assert(stepSize >= 1);
-                ip += ((ip-anchor) >> kSearchStrength) + stepSize;
-                continue;
-            } else {
-                /* found a dict match */
-                U32 const offset = (U32)(curr-dictMatchIndex-dictIndexDelta);
-                mLength = ZSTD_count_2segments(ip+4, dictMatch+4, iend, dictEnd, prefixStart) + 4;
-                while (((ip>anchor) & (dictMatch>dictStart))
-                     && (ip[-1] == dictMatch[-1])) {
-                    ip--; dictMatch--; mLength++;
+        size_t hash0 = ZSTD_hashPtr(ip0, hlog, mls);
+
+        size_t const dictHashAndTag0 = ZSTD_hashPtr(ip0, dictHBits, mls);
+        U32 dictMatchIndexAndTag = dictHashTable[dictHashAndTag0 >> ZSTD_SHORT_CACHE_TAG_BITS];
+        int dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag0);
+
+        U32 matchIndex = hashTable[hash0];
+        U32 curr = (U32)(ip0 - base);
+        size_t step = stepSize;
+        const size_t kStepIncr = 1 << kSearchStrength;
+        const BYTE* nextStep = ip0 + kStepIncr;
+
+        /* Inner search loop */
+        while (1) {
+            const BYTE* match = base + matchIndex;
+            const U32 repIndex = curr + 1 - offset_1;
+            const BYTE* repMatch = (repIndex < prefixStartIndex) ?
+                                   dictBase + (repIndex - dictIndexDelta) :
+                                   base + repIndex;
+            const size_t hash1 = ZSTD_hashPtr(ip1, hlog, mls);
+            size_t const dictHashAndTag1 = ZSTD_hashPtr(ip1, dictHBits, mls);
+            hashTable[hash0] = curr;   /* update hash table */
+
+            if (((U32) ((prefixStartIndex - 1) - repIndex) >=
+                 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */
+                && (MEM_read32(repMatch) == MEM_read32(ip0 + 1))) {
+                const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
+                mLength = ZSTD_count_2segments(ip0 + 1 + 4, repMatch + 4, iend, repMatchEnd, prefixStart) + 4;
+                ip0++;
+                ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength);
+                break;
+            }
+
+            if (dictTagsMatch) {
+                /* Found a possible dict match */
+                const U32 dictMatchIndex = dictMatchIndexAndTag >> ZSTD_SHORT_CACHE_TAG_BITS;
+                const BYTE* dictMatch = dictBase + dictMatchIndex;
+                if (dictMatchIndex > dictStartIndex &&
+                    MEM_read32(dictMatch) == MEM_read32(ip0)) {
+                    /* To replicate extDict parse behavior, we only use dict matches when the normal matchIndex is invalid */
+                    if (matchIndex <= prefixStartIndex) {
+                        U32 const offset = (U32) (curr - dictMatchIndex - dictIndexDelta);
+                        mLength = ZSTD_count_2segments(ip0 + 4, dictMatch + 4, iend, dictEnd, prefixStart) + 4;
+                        while (((ip0 > anchor) & (dictMatch > dictStart))
+                            && (ip0[-1] == dictMatch[-1])) {
+                            ip0--;
+                            dictMatch--;
+                            mLength++;
+                        } /* catch up */
+                        offset_2 = offset_1;
+                        offset_1 = offset;
+                        ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
+                        break;
+                    }
+                }
+            }
+
+            if (matchIndex > prefixStartIndex && MEM_read32(match) == MEM_read32(ip0)) {
+                /* found a regular match */
+                U32 const offset = (U32) (ip0 - match);
+                mLength = ZSTD_count(ip0 + 4, match + 4, iend) + 4;
+                while (((ip0 > anchor) & (match > prefixStart))
+                       && (ip0[-1] == match[-1])) {
+                    ip0--;
+                    match--;
+                    mLength++;
                 } /* catch up */
                 offset_2 = offset_1;
                 offset_1 = offset;
-                ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
+                ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength);
+                break;
             }
-        } else if (MEM_read32(match) != MEM_read32(ip)) {
-            /* it's not a match, and we're not going to check the dictionary */
-            assert(stepSize >= 1);
-            ip += ((ip-anchor) >> kSearchStrength) + stepSize;
-            continue;
-        } else {
-            /* found a regular match */
-            U32 const offset = (U32)(ip-match);
-            mLength = ZSTD_count(ip+4, match+4, iend) + 4;
-            while (((ip>anchor) & (match>prefixStart))
-                 && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */
-            offset_2 = offset_1;
-            offset_1 = offset;
-            ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
-        }
+
+            /* Prepare for next iteration */
+            dictMatchIndexAndTag = dictHashTable[dictHashAndTag1 >> ZSTD_SHORT_CACHE_TAG_BITS];
+            dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag1);
+            matchIndex = hashTable[hash1];
+
+            if (ip1 >= nextStep) {
+                step++;
+                nextStep += kStepIncr;
+            }
+            ip0 = ip1;
+            ip1 = ip1 + step;
+            if (ip1 > ilimit) goto _cleanup;
+
+            curr = (U32)(ip0 - base);
+            hash0 = hash1;
+        }   /* end inner search loop */
 
         /* match found */
-        ip += mLength;
-        anchor = ip;
+        assert(mLength);
+        ip0 += mLength;
+        anchor = ip0;
 
-        if (ip <= ilimit) {
+        if (ip0 <= ilimit) {
             /* Fill Table */
             assert(base+curr+2 > istart);  /* check base overflow */
             hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2;  /* here because curr+2 could be > iend-8 */
-            hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base);
+            hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base);
 
             /* check immediate repcode */
-            while (ip <= ilimit) {
-                U32 const current2 = (U32)(ip-base);
+            while (ip0 <= ilimit) {
+                U32 const current2 = (U32)(ip0-base);
                 U32 const repIndex2 = current2 - offset_2;
                 const BYTE* repMatch2 = repIndex2 < prefixStartIndex ?
                         dictBase - dictIndexDelta + repIndex2 :
                         base + repIndex2;
                 if ( ((U32)((prefixStartIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */)
-                   && (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
+                   && (MEM_read32(repMatch2) == MEM_read32(ip0))) {
                     const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
-                    size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
+                    size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
                     U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset;   /* swap offset_2 <=> offset_1 */
-                    ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, repLength2);
-                    hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2;
-                    ip += repLength2;
-                    anchor = ip;
+                    ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2);
+                    hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = current2;
+                    ip0 += repLength2;
+                    anchor = ip0;
                     continue;
                 }
                 break;
             }
         }
+
+        /* Prepare for next iteration */
+        assert(ip0 == anchor);
+        ip1 = ip0 + stepSize;
     }
 
+_cleanup:
     /* save reps for next block */
-    rep[0] = offset_1 ? offset_1 : offsetSaved;
-    rep[1] = offset_2 ? offset_2 : offsetSaved;
+    rep[0] = offset_1;
+    rep[1] = offset_2;
 
     /* Return the last literals size */
     return (size_t)(iend - anchor);
@@ -553,11 +690,10 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
     U32* const hashTable = ms->hashTable;
     U32 const hlog = cParams->hashLog;
     /* support stepSize of 0 */
-    U32 const stepSize = cParams->targetLength + !(cParams->targetLength);
+    size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1;
     const BYTE* const base = ms->window.base;
     const BYTE* const dictBase = ms->window.dictBase;
     const BYTE* const istart = (const BYTE*)src;
-    const BYTE* ip = istart;
     const BYTE* anchor = istart;
     const U32   endIndex = (U32)((size_t)(istart - base) + srcSize);
     const U32   lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog);
@@ -570,6 +706,28 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
     const BYTE* const iend = istart + srcSize;
     const BYTE* const ilimit = iend - 8;
     U32 offset_1=rep[0], offset_2=rep[1];
+    U32 offsetSaved1 = 0, offsetSaved2 = 0;
+
+    const BYTE* ip0 = istart;
+    const BYTE* ip1;
+    const BYTE* ip2;
+    const BYTE* ip3;
+    U32 current0;
+
+
+    size_t hash0; /* hash for ip0 */
+    size_t hash1; /* hash for ip1 */
+    U32 idx; /* match idx for ip0 */
+    const BYTE* idxBase; /* base pointer for idx */
+
+    U32 offcode;
+    const BYTE* match0;
+    size_t mLength;
+    const BYTE* matchEnd = 0; /* initialize to avoid warning, assert != 0 later */
+
+    size_t step;
+    const BYTE* nextStep;
+    const size_t kStepIncr = (1 << (kSearchStrength - 1));
 
     (void)hasStep; /* not currently specialized on whether it's accelerated */
 
@@ -579,75 +737,202 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
     if (prefixStartIndex == dictStartIndex)
         return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize);
 
-    /* Search Loop */
-    while (ip < ilimit) {  /* < instead of <=, because (ip+1) */
-        const size_t h = ZSTD_hashPtr(ip, hlog, mls);
-        const U32    matchIndex = hashTable[h];
-        const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base;
-        const BYTE*  match = matchBase + matchIndex;
-        const U32    curr = (U32)(ip-base);
-        const U32    repIndex = curr + 1 - offset_1;
-        const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base;
-        const BYTE* const repMatch = repBase + repIndex;
-        hashTable[h] = curr;   /* update hash table */
-        DEBUGLOG(7, "offset_1 = %u , curr = %u", offset_1, curr);
-
-        if ( ( ((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow */
-             & (offset_1 <= curr+1 - dictStartIndex) ) /* note: we are searching at curr+1 */
-           && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
-            const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
-            size_t const rLength = ZSTD_count_2segments(ip+1 +4, repMatch +4, iend, repMatchEnd, prefixStart) + 4;
-            ip++;
-            ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, rLength);
-            ip += rLength;
-            anchor = ip;
-        } else {
-            if ( (matchIndex < dictStartIndex) ||
-                 (MEM_read32(match) != MEM_read32(ip)) ) {
-                assert(stepSize >= 1);
-                ip += ((ip-anchor) >> kSearchStrength) + stepSize;
-                continue;
+    {   U32 const curr = (U32)(ip0 - base);
+        U32 const maxRep = curr - dictStartIndex;
+        if (offset_2 >= maxRep) offsetSaved2 = offset_2, offset_2 = 0;
+        if (offset_1 >= maxRep) offsetSaved1 = offset_1, offset_1 = 0;
+    }
+
+    /* start each op */
+_start: /* Requires: ip0 */
+
+    step = stepSize;
+    nextStep = ip0 + kStepIncr;
+
+    /* calculate positions, ip0 - anchor == 0, so we skip step calc */
+    ip1 = ip0 + 1;
+    ip2 = ip0 + step;
+    ip3 = ip2 + 1;
+
+    if (ip3 >= ilimit) {
+        goto _cleanup;
+    }
+
+    hash0 = ZSTD_hashPtr(ip0, hlog, mls);
+    hash1 = ZSTD_hashPtr(ip1, hlog, mls);
+
+    idx = hashTable[hash0];
+    idxBase = idx < prefixStartIndex ? dictBase : base;
+
+    do {
+        {   /* load repcode match for ip[2] */
+            U32 const current2 = (U32)(ip2 - base);
+            U32 const repIndex = current2 - offset_1;
+            const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base;
+            U32 rval;
+            if ( ((U32)(prefixStartIndex - repIndex) >= 4) /* intentional underflow */
+                 & (offset_1 > 0) ) {
+                rval = MEM_read32(repBase + repIndex);
+            } else {
+                rval = MEM_read32(ip2) ^ 1; /* guaranteed to not match. */
             }
-            {   const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend;
-                const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart;
-                U32 const offset = curr - matchIndex;
-                size_t mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4;
-                while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; }   /* catch up */
-                offset_2 = offset_1; offset_1 = offset;  /* update offset history */
-                ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength);
-                ip += mLength;
-                anchor = ip;
+
+            /* write back hash table entry */
+            current0 = (U32)(ip0 - base);
+            hashTable[hash0] = current0;
+
+            /* check repcode at ip[2] */
+            if (MEM_read32(ip2) == rval) {
+                ip0 = ip2;
+                match0 = repBase + repIndex;
+                matchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
+                assert((match0 != prefixStart) & (match0 != dictStart));
+                mLength = ip0[-1] == match0[-1];
+                ip0 -= mLength;
+                match0 -= mLength;
+                offcode = REPCODE1_TO_OFFBASE;
+                mLength += 4;
+                goto _match;
         }   }
 
-        if (ip <= ilimit) {
-            /* Fill Table */
-            hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2;
-            hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base);
-            /* check immediate repcode */
-            while (ip <= ilimit) {
-                U32 const current2 = (U32)(ip-base);
-                U32 const repIndex2 = current2 - offset_2;
-                const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
-                if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (offset_2 <= curr - dictStartIndex))  /* intentional overflow */
-                   && (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
-                    const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
-                    size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
-                    { U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; }  /* swap offset_2 <=> offset_1 */
-                    ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, STORE_REPCODE_1, repLength2);
-                    hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2;
-                    ip += repLength2;
-                    anchor = ip;
-                    continue;
-                }
-                break;
-    }   }   }
+        {   /* load match for ip[0] */
+            U32 const mval = idx >= dictStartIndex ?
+                    MEM_read32(idxBase + idx) :
+                    MEM_read32(ip0) ^ 1; /* guaranteed not to match */
+
+            /* check match at ip[0] */
+            if (MEM_read32(ip0) == mval) {
+                /* found a match! */
+                goto _offset;
+        }   }
+
+        /* lookup ip[1] */
+        idx = hashTable[hash1];
+        idxBase = idx < prefixStartIndex ? dictBase : base;
+
+        /* hash ip[2] */
+        hash0 = hash1;
+        hash1 = ZSTD_hashPtr(ip2, hlog, mls);
+
+        /* advance to next positions */
+        ip0 = ip1;
+        ip1 = ip2;
+        ip2 = ip3;
+
+        /* write back hash table entry */
+        current0 = (U32)(ip0 - base);
+        hashTable[hash0] = current0;
+
+        {   /* load match for ip[0] */
+            U32 const mval = idx >= dictStartIndex ?
+                    MEM_read32(idxBase + idx) :
+                    MEM_read32(ip0) ^ 1; /* guaranteed not to match */
+
+            /* check match at ip[0] */
+            if (MEM_read32(ip0) == mval) {
+                /* found a match! */
+                goto _offset;
+        }   }
+
+        /* lookup ip[1] */
+        idx = hashTable[hash1];
+        idxBase = idx < prefixStartIndex ? dictBase : base;
+
+        /* hash ip[2] */
+        hash0 = hash1;
+        hash1 = ZSTD_hashPtr(ip2, hlog, mls);
+
+        /* advance to next positions */
+        ip0 = ip1;
+        ip1 = ip2;
+        ip2 = ip0 + step;
+        ip3 = ip1 + step;
+
+        /* calculate step */
+        if (ip2 >= nextStep) {
+            step++;
+            PREFETCH_L1(ip1 + 64);
+            PREFETCH_L1(ip1 + 128);
+            nextStep += kStepIncr;
+        }
+    } while (ip3 < ilimit);
+
+_cleanup:
+    /* Note that there are probably still a couple positions we could search.
+     * However, it seems to be a meaningful performance hit to try to search
+     * them. So let's not. */
+
+    /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0),
+     * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */
+    offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2;
 
     /* save reps for next block */
-    rep[0] = offset_1;
-    rep[1] = offset_2;
+    rep[0] = offset_1 ? offset_1 : offsetSaved1;
+    rep[1] = offset_2 ? offset_2 : offsetSaved2;
 
     /* Return the last literals size */
     return (size_t)(iend - anchor);
+
+_offset: /* Requires: ip0, idx, idxBase */
+
+    /* Compute the offset code. */
+    {   U32 const offset = current0 - idx;
+        const BYTE* const lowMatchPtr = idx < prefixStartIndex ? dictStart : prefixStart;
+        matchEnd = idx < prefixStartIndex ? dictEnd : iend;
+        match0 = idxBase + idx;
+        offset_2 = offset_1;
+        offset_1 = offset;
+        offcode = OFFSET_TO_OFFBASE(offset);
+        mLength = 4;
+
+        /* Count the backwards match length. */
+        while (((ip0>anchor) & (match0>lowMatchPtr)) && (ip0[-1] == match0[-1])) {
+            ip0--;
+            match0--;
+            mLength++;
+    }   }
+
+_match: /* Requires: ip0, match0, offcode, matchEnd */
+
+    /* Count the forward length. */
+    assert(matchEnd != 0);
+    mLength += ZSTD_count_2segments(ip0 + mLength, match0 + mLength, iend, matchEnd, prefixStart);
+
+    ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength);
+
+    ip0 += mLength;
+    anchor = ip0;
+
+    /* write next hash table entry */
+    if (ip1 < ip0) {
+        hashTable[hash1] = (U32)(ip1 - base);
+    }
+
+    /* Fill table and check for immediate repcode. */
+    if (ip0 <= ilimit) {
+        /* Fill Table */
+        assert(base+current0+2 > istart);  /* check base overflow */
+        hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2;  /* here because current+2 could be > iend-8 */
+        hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base);
+
+        while (ip0 <= ilimit) {
+            U32 const repIndex2 = (U32)(ip0-base) - offset_2;
+            const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
+            if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (offset_2 > 0))  /* intentional underflow */
+                 && (MEM_read32(repMatch2) == MEM_read32(ip0)) ) {
+                const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
+                size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
+                { U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; }  /* swap offset_2 <=> offset_1 */
+                ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, REPCODE1_TO_OFFBASE, repLength2);
+                hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base);
+                ip0 += repLength2;
+                anchor = ip0;
+                continue;
+            }
+            break;
+    }   }
+
+    goto _start;
 }
 
 ZSTD_GEN_FAST_FN(extDict, 4, 0)
@@ -660,6 +945,7 @@ size_t ZSTD_compressBlock_fast_extDict(
         void const* src, size_t srcSize)
 {
     U32 const mls = ms->cParams.minMatch;
+    assert(ms->dictMatchState == NULL);
     switch(mls)
     {
     default: /* includes case 3 */
diff --git a/lib/zstd/compress/zstd_fast.h b/lib/zstd/compress/zstd_fast.h
index fddc2f532d21..e64d9e1b2d39 100644
--- a/lib/zstd/compress/zstd_fast.h
+++ b/lib/zstd/compress/zstd_fast.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -16,7 +17,8 @@
 #include "zstd_compress_internal.h"
 
 void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
-                        void const* end, ZSTD_dictTableLoadMethod_e dtlm);
+                        void const* end, ZSTD_dictTableLoadMethod_e dtlm,
+                        ZSTD_tableFillPurpose_e tfp);
 size_t ZSTD_compressBlock_fast(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
         void const* src, size_t srcSize);
diff --git a/lib/zstd/compress/zstd_lazy.c b/lib/zstd/compress/zstd_lazy.c
index 0298a01a7504..f6b4978ceba7 100644
--- a/lib/zstd/compress/zstd_lazy.c
+++ b/lib/zstd/compress/zstd_lazy.c
@@ -1,5 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -10,6 +11,9 @@
 
 #include "zstd_compress_internal.h"
 #include "zstd_lazy.h"
+#include "../common/bits.h" /* ZSTD_countTrailingZeros64 */
+
+#define kLazySkippingStep 8
 
 
 /*-*************************************
@@ -197,8 +201,8 @@ ZSTD_DUBT_findBetterDictMatch (
             U32 matchIndex = dictMatchIndex + dictIndexDelta;
             if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) {
                 DEBUGLOG(9, "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)",
-                    curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, STORE_OFFSET(curr - matchIndex), dictMatchIndex, matchIndex);
-                bestLength = matchLength, *offsetPtr = STORE_OFFSET(curr - matchIndex);
+                    curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, OFFSET_TO_OFFBASE(curr - matchIndex), dictMatchIndex, matchIndex);
+                bestLength = matchLength, *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex);
             }
             if (ip+matchLength == iend) {   /* reached end of input : ip[matchLength] is not valid, no way to know if it's larger or smaller than match */
                 break;   /* drop, to guarantee consistency (miss a little bit of compression) */
@@ -218,7 +222,7 @@ ZSTD_DUBT_findBetterDictMatch (
     }
 
     if (bestLength >= MINMATCH) {
-        U32 const mIndex = curr - (U32)STORED_OFFSET(*offsetPtr); (void)mIndex;
+        U32 const mIndex = curr - (U32)OFFBASE_TO_OFFSET(*offsetPtr); (void)mIndex;
         DEBUGLOG(8, "ZSTD_DUBT_findBetterDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)",
                     curr, (U32)bestLength, (U32)*offsetPtr, mIndex);
     }
@@ -230,7 +234,7 @@ ZSTD_DUBT_findBetterDictMatch (
 static size_t
 ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
                         const BYTE* const ip, const BYTE* const iend,
-                        size_t* offsetPtr,
+                        size_t* offBasePtr,
                         U32 const mls,
                         const ZSTD_dictMode_e dictMode)
 {
@@ -327,8 +331,8 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
             if (matchLength > bestLength) {
                 if (matchLength > matchEndIdx - matchIndex)
                     matchEndIdx = matchIndex + (U32)matchLength;
-                if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) )
-                    bestLength = matchLength, *offsetPtr = STORE_OFFSET(curr - matchIndex);
+                if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr - matchIndex + 1) - ZSTD_highbit32((U32)*offBasePtr)) )
+                    bestLength = matchLength, *offBasePtr = OFFSET_TO_OFFBASE(curr - matchIndex);
                 if (ip+matchLength == iend) {   /* equal : no way to know if inf or sup */
                     if (dictMode == ZSTD_dictMatchState) {
                         nbCompares = 0; /* in addition to avoiding checking any
@@ -361,16 +365,16 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
         if (dictMode == ZSTD_dictMatchState && nbCompares) {
             bestLength = ZSTD_DUBT_findBetterDictMatch(
                     ms, ip, iend,
-                    offsetPtr, bestLength, nbCompares,
+                    offBasePtr, bestLength, nbCompares,
                     mls, dictMode);
         }
 
         assert(matchEndIdx > curr+8); /* ensure nextToUpdate is increased */
         ms->nextToUpdate = matchEndIdx - 8;   /* skip repetitive patterns */
         if (bestLength >= MINMATCH) {
-            U32 const mIndex = curr - (U32)STORED_OFFSET(*offsetPtr); (void)mIndex;
+            U32 const mIndex = curr - (U32)OFFBASE_TO_OFFSET(*offBasePtr); (void)mIndex;
             DEBUGLOG(8, "ZSTD_DUBT_findBestMatch(%u) : found match of length %u and offsetCode %u (pos %u)",
-                        curr, (U32)bestLength, (U32)*offsetPtr, mIndex);
+                        curr, (U32)bestLength, (U32)*offBasePtr, mIndex);
         }
         return bestLength;
     }
@@ -381,14 +385,14 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
 FORCE_INLINE_TEMPLATE size_t
 ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms,
                 const BYTE* const ip, const BYTE* const iLimit,
-                      size_t* offsetPtr,
+                      size_t* offBasePtr,
                 const U32 mls /* template */,
                 const ZSTD_dictMode_e dictMode)
 {
     DEBUGLOG(7, "ZSTD_BtFindBestMatch");
     if (ip < ms->window.base + ms->nextToUpdate) return 0;   /* skipped area */
     ZSTD_updateDUBT(ms, ip, iLimit, mls);
-    return ZSTD_DUBT_findBestMatch(ms, ip, iLimit, offsetPtr, mls, dictMode);
+    return ZSTD_DUBT_findBestMatch(ms, ip, iLimit, offBasePtr, mls, dictMode);
 }
 
 /* *********************************
@@ -561,7 +565,7 @@ size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nb
         /* save best solution */
         if (currentMl > ml) {
             ml = currentMl;
-            *offsetPtr = STORE_OFFSET(curr - (matchIndex + ddsIndexDelta));
+            *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + ddsIndexDelta));
             if (ip+currentMl == iLimit) {
                 /* best possible, avoids read overflow on next attempt */
                 return ml;
@@ -598,7 +602,7 @@ size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nb
             /* save best solution */
             if (currentMl > ml) {
                 ml = currentMl;
-                *offsetPtr = STORE_OFFSET(curr - (matchIndex + ddsIndexDelta));
+                *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + ddsIndexDelta));
                 if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
             }
         }
@@ -617,7 +621,7 @@ size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nb
 FORCE_INLINE_TEMPLATE U32 ZSTD_insertAndFindFirstIndex_internal(
                         ZSTD_matchState_t* ms,
                         const ZSTD_compressionParameters* const cParams,
-                        const BYTE* ip, U32 const mls)
+                        const BYTE* ip, U32 const mls, U32 const lazySkipping)
 {
     U32* const hashTable  = ms->hashTable;
     const U32 hashLog = cParams->hashLog;
@@ -632,6 +636,9 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_insertAndFindFirstIndex_internal(
         NEXT_IN_CHAIN(idx, chainMask) = hashTable[h];
         hashTable[h] = idx;
         idx++;
+        /* Stop inserting every position when in the lazy skipping mode. */
+        if (lazySkipping)
+            break;
     }
 
     ms->nextToUpdate = target;
@@ -640,7 +647,7 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_insertAndFindFirstIndex_internal(
 
 U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) {
     const ZSTD_compressionParameters* const cParams = &ms->cParams;
-    return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch);
+    return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch, /* lazySkipping*/ 0);
 }
 
 /* inlining is important to hardwire a hot branch (template emulation) */
@@ -684,14 +691,15 @@ size_t ZSTD_HcFindBestMatch(
     }
 
     /* HC4 match finder */
-    matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls);
+    matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls, ms->lazySkipping);
 
     for ( ; (matchIndex>=lowLimit) & (nbAttempts>0) ; nbAttempts--) {
         size_t currentMl=0;
         if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) {
             const BYTE* const match = base + matchIndex;
             assert(matchIndex >= dictLimit);   /* ensures this is true if dictMode != ZSTD_extDict */
-            if (match[ml] == ip[ml])   /* potentially better */
+            /* read 4B starting from (match + ml + 1 - sizeof(U32)) */
+            if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3))   /* potentially better */
                 currentMl = ZSTD_count(ip, match, iLimit);
         } else {
             const BYTE* const match = dictBase + matchIndex;
@@ -703,7 +711,7 @@ size_t ZSTD_HcFindBestMatch(
         /* save best solution */
         if (currentMl > ml) {
             ml = currentMl;
-            *offsetPtr = STORE_OFFSET(curr - matchIndex);
+            *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex);
             if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
         }
 
@@ -739,7 +747,7 @@ size_t ZSTD_HcFindBestMatch(
             if (currentMl > ml) {
                 ml = currentMl;
                 assert(curr > matchIndex + dmsIndexDelta);
-                *offsetPtr = STORE_OFFSET(curr - (matchIndex + dmsIndexDelta));
+                *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + dmsIndexDelta));
                 if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
             }
 
@@ -756,8 +764,6 @@ size_t ZSTD_HcFindBestMatch(
 * (SIMD) Row-based matchfinder
 ***********************************/
 /* Constants for row-based hash */
-#define ZSTD_ROW_HASH_TAG_OFFSET 16     /* byte offset of hashes in the match state's tagTable from the beginning of a row */
-#define ZSTD_ROW_HASH_TAG_BITS 8        /* nb bits to use for the tag */
 #define ZSTD_ROW_HASH_TAG_MASK ((1u << ZSTD_ROW_HASH_TAG_BITS) - 1)
 #define ZSTD_ROW_HASH_MAX_ENTRIES 64    /* absolute maximum number of entries per row, for all configurations */
 
@@ -769,64 +775,19 @@ typedef U64 ZSTD_VecMask;   /* Clarifies when we are interacting with a U64 repr
  * Starting from the LSB, returns the idx of the next non-zero bit.
  * Basically counting the nb of trailing zeroes.
  */
-static U32 ZSTD_VecMask_next(ZSTD_VecMask val) {
-    assert(val != 0);
-#   if (defined(__GNUC__) && ((__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4))))
-    if (sizeof(size_t) == 4) {
-        U32 mostSignificantWord = (U32)(val >> 32);
-        U32 leastSignificantWord = (U32)val;
-        if (leastSignificantWord == 0) {
-            return 32 + (U32)__builtin_ctz(mostSignificantWord);
-        } else {
-            return (U32)__builtin_ctz(leastSignificantWord);
-        }
-    } else {
-        return (U32)__builtin_ctzll(val);
-    }
-#   else
-    /* Software ctz version: http://aggregate.org/MAGIC/#Trailing%20Zero%20Count
-     * and: https://stackoverflow.com/questions/2709430/count-number-of-bits-in-a-64-bit-long-big-integer
-     */
-    val = ~val & (val - 1ULL); /* Lowest set bit mask */
-    val = val - ((val >> 1) & 0x5555555555555555);
-    val = (val & 0x3333333333333333ULL) + ((val >> 2) & 0x3333333333333333ULL);
-    return (U32)((((val + (val >> 4)) & 0xF0F0F0F0F0F0F0FULL) * 0x101010101010101ULL) >> 56);
-#   endif
-}
-
-/* ZSTD_rotateRight_*():
- * Rotates a bitfield to the right by "count" bits.
- * https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts
- */
-FORCE_INLINE_TEMPLATE
-U64 ZSTD_rotateRight_U64(U64 const value, U32 count) {
-    assert(count < 64);
-    count &= 0x3F; /* for fickle pattern recognition */
-    return (value >> count) | (U64)(value << ((0U - count) & 0x3F));
-}
-
-FORCE_INLINE_TEMPLATE
-U32 ZSTD_rotateRight_U32(U32 const value, U32 count) {
-    assert(count < 32);
-    count &= 0x1F; /* for fickle pattern recognition */
-    return (value >> count) | (U32)(value << ((0U - count) & 0x1F));
-}
-
-FORCE_INLINE_TEMPLATE
-U16 ZSTD_rotateRight_U16(U16 const value, U32 count) {
-    assert(count < 16);
-    count &= 0x0F; /* for fickle pattern recognition */
-    return (value >> count) | (U16)(value << ((0U - count) & 0x0F));
+MEM_STATIC U32 ZSTD_VecMask_next(ZSTD_VecMask val) {
+    return ZSTD_countTrailingZeros64(val);
 }
 
 /* ZSTD_row_nextIndex():
  * Returns the next index to insert at within a tagTable row, and updates the "head"
- * value to reflect the update. Essentially cycles backwards from [0, {entries per row})
+ * value to reflect the update. Essentially cycles backwards from [1, {entries per row})
  */
 FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextIndex(BYTE* const tagRow, U32 const rowMask) {
-  U32 const next = (*tagRow - 1) & rowMask;
-  *tagRow = (BYTE)next;
-  return next;
+    U32 next = (*tagRow-1) & rowMask;
+    next += (next == 0) ? rowMask : 0; /* skip first position */
+    *tagRow = (BYTE)next;
+    return next;
 }
 
 /* ZSTD_isAligned():
@@ -840,7 +801,7 @@ MEM_STATIC int ZSTD_isAligned(void const* ptr, size_t align) {
 /* ZSTD_row_prefetch():
  * Performs prefetching for the hashTable and tagTable at a given row.
  */
-FORCE_INLINE_TEMPLATE void ZSTD_row_prefetch(U32 const* hashTable, U16 const* tagTable, U32 const relRow, U32 const rowLog) {
+FORCE_INLINE_TEMPLATE void ZSTD_row_prefetch(U32 const* hashTable, BYTE const* tagTable, U32 const relRow, U32 const rowLog) {
     PREFETCH_L1(hashTable + relRow);
     if (rowLog >= 5) {
         PREFETCH_L1(hashTable + relRow + 16);
@@ -864,13 +825,13 @@ FORCE_INLINE_TEMPLATE void ZSTD_row_fillHashCache(ZSTD_matchState_t* ms, const B
                                    U32 idx, const BYTE* const iLimit)
 {
     U32 const* const hashTable = ms->hashTable;
-    U16 const* const tagTable = ms->tagTable;
+    BYTE const* const tagTable = ms->tagTable;
     U32 const hashLog = ms->rowHashLog;
     U32 const maxElemsToPrefetch = (base + idx) > iLimit ? 0 : (U32)(iLimit - (base + idx) + 1);
     U32 const lim = idx + MIN(ZSTD_ROW_HASH_CACHE_SIZE, maxElemsToPrefetch);
 
     for (; idx < lim; ++idx) {
-        U32 const hash = (U32)ZSTD_hashPtr(base + idx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls);
+        U32 const hash = (U32)ZSTD_hashPtrSalted(base + idx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt);
         U32 const row = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
         ZSTD_row_prefetch(hashTable, tagTable, row, rowLog);
         ms->hashCache[idx & ZSTD_ROW_HASH_CACHE_MASK] = hash;
@@ -886,11 +847,12 @@ FORCE_INLINE_TEMPLATE void ZSTD_row_fillHashCache(ZSTD_matchState_t* ms, const B
  * base + idx + ZSTD_ROW_HASH_CACHE_SIZE. Also prefetches the appropriate rows from hashTable and tagTable.
  */
 FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextCachedHash(U32* cache, U32 const* hashTable,
-                                                  U16 const* tagTable, BYTE const* base,
+                                                  BYTE const* tagTable, BYTE const* base,
                                                   U32 idx, U32 const hashLog,
-                                                  U32 const rowLog, U32 const mls)
+                                                  U32 const rowLog, U32 const mls,
+                                                  U64 const hashSalt)
 {
-    U32 const newHash = (U32)ZSTD_hashPtr(base+idx+ZSTD_ROW_HASH_CACHE_SIZE, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls);
+    U32 const newHash = (U32)ZSTD_hashPtrSalted(base+idx+ZSTD_ROW_HASH_CACHE_SIZE, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, hashSalt);
     U32 const row = (newHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
     ZSTD_row_prefetch(hashTable, tagTable, row, rowLog);
     {   U32 const hash = cache[idx & ZSTD_ROW_HASH_CACHE_MASK];
@@ -908,22 +870,21 @@ FORCE_INLINE_TEMPLATE void ZSTD_row_update_internalImpl(ZSTD_matchState_t* ms,
                                                         U32 const rowMask, U32 const useCache)
 {
     U32* const hashTable = ms->hashTable;
-    U16* const tagTable = ms->tagTable;
+    BYTE* const tagTable = ms->tagTable;
     U32 const hashLog = ms->rowHashLog;
     const BYTE* const base = ms->window.base;
 
     DEBUGLOG(6, "ZSTD_row_update_internalImpl(): updateStartIdx=%u, updateEndIdx=%u", updateStartIdx, updateEndIdx);
     for (; updateStartIdx < updateEndIdx; ++updateStartIdx) {
-        U32 const hash = useCache ? ZSTD_row_nextCachedHash(ms->hashCache, hashTable, tagTable, base, updateStartIdx, hashLog, rowLog, mls)
-                                  : (U32)ZSTD_hashPtr(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls);
+        U32 const hash = useCache ? ZSTD_row_nextCachedHash(ms->hashCache, hashTable, tagTable, base, updateStartIdx, hashLog, rowLog, mls, ms->hashSalt)
+                                  : (U32)ZSTD_hashPtrSalted(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt);
         U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
         U32* const row = hashTable + relRow;
-        BYTE* tagRow = (BYTE*)(tagTable + relRow);  /* Though tagTable is laid out as a table of U16, each tag is only 1 byte.
-                                                       Explicit cast allows us to get exact desired position within each row */
+        BYTE* tagRow = tagTable + relRow;
         U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask);
 
-        assert(hash == ZSTD_hashPtr(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls));
-        ((BYTE*)tagRow)[pos + ZSTD_ROW_HASH_TAG_OFFSET] = hash & ZSTD_ROW_HASH_TAG_MASK;
+        assert(hash == ZSTD_hashPtrSalted(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt));
+        tagRow[pos] = hash & ZSTD_ROW_HASH_TAG_MASK;
         row[pos] = updateStartIdx;
     }
 }
@@ -971,7 +932,35 @@ void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip) {
     const U32 mls = MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */);
 
     DEBUGLOG(5, "ZSTD_row_update(), rowLog=%u", rowLog);
-    ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 0 /* dont use cache */);
+    ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 0 /* don't use cache */);
+}
+
+/* Returns the mask width of bits group of which will be set to 1. Given not all
+ * architectures have easy movemask instruction, this helps to iterate over
+ * groups of bits easier and faster.
+ */
+FORCE_INLINE_TEMPLATE U32
+ZSTD_row_matchMaskGroupWidth(const U32 rowEntries)
+{
+    assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64);
+    assert(rowEntries <= ZSTD_ROW_HASH_MAX_ENTRIES);
+    (void)rowEntries;
+#if defined(ZSTD_ARCH_ARM_NEON)
+    /* NEON path only works for little endian */
+    if (!MEM_isLittleEndian()) {
+        return 1;
+    }
+    if (rowEntries == 16) {
+        return 4;
+    }
+    if (rowEntries == 32) {
+        return 2;
+    }
+    if (rowEntries == 64) {
+        return 1;
+    }
+#endif
+    return 1;
 }
 
 #if defined(ZSTD_ARCH_X86_SSE2)
@@ -994,71 +983,82 @@ ZSTD_row_getSSEMask(int nbChunks, const BYTE* const src, const BYTE tag, const U
 }
 #endif
 
-/* Returns a ZSTD_VecMask (U32) that has the nth bit set to 1 if the newly-computed "tag" matches
- * the hash at the nth position in a row of the tagTable.
- * Each row is a circular buffer beginning at the value of "head". So we must rotate the "matches" bitfield
- * to match up with the actual layout of the entries within the hashTable */
+#if defined(ZSTD_ARCH_ARM_NEON)
+FORCE_INLINE_TEMPLATE ZSTD_VecMask
+ZSTD_row_getNEONMask(const U32 rowEntries, const BYTE* const src, const BYTE tag, const U32 headGrouped)
+{
+    assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64);
+    if (rowEntries == 16) {
+        /* vshrn_n_u16 shifts by 4 every u16 and narrows to 8 lower bits.
+         * After that groups of 4 bits represent the equalMask. We lower
+         * all bits except the highest in these groups by doing AND with
+         * 0x88 = 0b10001000.
+         */
+        const uint8x16_t chunk = vld1q_u8(src);
+        const uint16x8_t equalMask = vreinterpretq_u16_u8(vceqq_u8(chunk, vdupq_n_u8(tag)));
+        const uint8x8_t res = vshrn_n_u16(equalMask, 4);
+        const U64 matches = vget_lane_u64(vreinterpret_u64_u8(res), 0);
+        return ZSTD_rotateRight_U64(matches, headGrouped) & 0x8888888888888888ull;
+    } else if (rowEntries == 32) {
+        /* Same idea as with rowEntries == 16 but doing AND with
+         * 0x55 = 0b01010101.
+         */
+        const uint16x8x2_t chunk = vld2q_u16((const uint16_t*)(const void*)src);
+        const uint8x16_t chunk0 = vreinterpretq_u8_u16(chunk.val[0]);
+        const uint8x16_t chunk1 = vreinterpretq_u8_u16(chunk.val[1]);
+        const uint8x16_t dup = vdupq_n_u8(tag);
+        const uint8x8_t t0 = vshrn_n_u16(vreinterpretq_u16_u8(vceqq_u8(chunk0, dup)), 6);
+        const uint8x8_t t1 = vshrn_n_u16(vreinterpretq_u16_u8(vceqq_u8(chunk1, dup)), 6);
+        const uint8x8_t res = vsli_n_u8(t0, t1, 4);
+        const U64 matches = vget_lane_u64(vreinterpret_u64_u8(res), 0) ;
+        return ZSTD_rotateRight_U64(matches, headGrouped) & 0x5555555555555555ull;
+    } else { /* rowEntries == 64 */
+        const uint8x16x4_t chunk = vld4q_u8(src);
+        const uint8x16_t dup = vdupq_n_u8(tag);
+        const uint8x16_t cmp0 = vceqq_u8(chunk.val[0], dup);
+        const uint8x16_t cmp1 = vceqq_u8(chunk.val[1], dup);
+        const uint8x16_t cmp2 = vceqq_u8(chunk.val[2], dup);
+        const uint8x16_t cmp3 = vceqq_u8(chunk.val[3], dup);
+
+        const uint8x16_t t0 = vsriq_n_u8(cmp1, cmp0, 1);
+        const uint8x16_t t1 = vsriq_n_u8(cmp3, cmp2, 1);
+        const uint8x16_t t2 = vsriq_n_u8(t1, t0, 2);
+        const uint8x16_t t3 = vsriq_n_u8(t2, t2, 4);
+        const uint8x8_t t4 = vshrn_n_u16(vreinterpretq_u16_u8(t3), 4);
+        const U64 matches = vget_lane_u64(vreinterpret_u64_u8(t4), 0);
+        return ZSTD_rotateRight_U64(matches, headGrouped);
+    }
+}
+#endif
+
+/* Returns a ZSTD_VecMask (U64) that has the nth group (determined by
+ * ZSTD_row_matchMaskGroupWidth) of bits set to 1 if the newly-computed "tag"
+ * matches the hash at the nth position in a row of the tagTable.
+ * Each row is a circular buffer beginning at the value of "headGrouped". So we
+ * must rotate the "matches" bitfield to match up with the actual layout of the
+ * entries within the hashTable */
 FORCE_INLINE_TEMPLATE ZSTD_VecMask
-ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 head, const U32 rowEntries)
+ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 headGrouped, const U32 rowEntries)
 {
-    const BYTE* const src = tagRow + ZSTD_ROW_HASH_TAG_OFFSET;
+    const BYTE* const src = tagRow;
     assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64);
     assert(rowEntries <= ZSTD_ROW_HASH_MAX_ENTRIES);
+    assert(ZSTD_row_matchMaskGroupWidth(rowEntries) * rowEntries <= sizeof(ZSTD_VecMask) * 8);
 
 #if defined(ZSTD_ARCH_X86_SSE2)
 
-    return ZSTD_row_getSSEMask(rowEntries / 16, src, tag, head);
+    return ZSTD_row_getSSEMask(rowEntries / 16, src, tag, headGrouped);
 
 #else /* SW or NEON-LE */
 
 # if defined(ZSTD_ARCH_ARM_NEON)
   /* This NEON path only works for little endian - otherwise use SWAR below */
     if (MEM_isLittleEndian()) {
-        if (rowEntries == 16) {
-            const uint8x16_t chunk = vld1q_u8(src);
-            const uint16x8_t equalMask = vreinterpretq_u16_u8(vceqq_u8(chunk, vdupq_n_u8(tag)));
-            const uint16x8_t t0 = vshlq_n_u16(equalMask, 7);
-            const uint32x4_t t1 = vreinterpretq_u32_u16(vsriq_n_u16(t0, t0, 14));
-            const uint64x2_t t2 = vreinterpretq_u64_u32(vshrq_n_u32(t1, 14));
-            const uint8x16_t t3 = vreinterpretq_u8_u64(vsraq_n_u64(t2, t2, 28));
-            const U16 hi = (U16)vgetq_lane_u8(t3, 8);
-            const U16 lo = (U16)vgetq_lane_u8(t3, 0);
-            return ZSTD_rotateRight_U16((hi << 8) | lo, head);
-        } else if (rowEntries == 32) {
-            const uint16x8x2_t chunk = vld2q_u16((const U16*)(const void*)src);
-            const uint8x16_t chunk0 = vreinterpretq_u8_u16(chunk.val[0]);
-            const uint8x16_t chunk1 = vreinterpretq_u8_u16(chunk.val[1]);
-            const uint8x16_t equalMask0 = vceqq_u8(chunk0, vdupq_n_u8(tag));
-            const uint8x16_t equalMask1 = vceqq_u8(chunk1, vdupq_n_u8(tag));
-            const int8x8_t pack0 = vqmovn_s16(vreinterpretq_s16_u8(equalMask0));
-            const int8x8_t pack1 = vqmovn_s16(vreinterpretq_s16_u8(equalMask1));
-            const uint8x8_t t0 = vreinterpret_u8_s8(pack0);
-            const uint8x8_t t1 = vreinterpret_u8_s8(pack1);
-            const uint8x8_t t2 = vsri_n_u8(t1, t0, 2);
-            const uint8x8x2_t t3 = vuzp_u8(t2, t0);
-            const uint8x8_t t4 = vsri_n_u8(t3.val[1], t3.val[0], 4);
-            const U32 matches = vget_lane_u32(vreinterpret_u32_u8(t4), 0);
-            return ZSTD_rotateRight_U32(matches, head);
-        } else { /* rowEntries == 64 */
-            const uint8x16x4_t chunk = vld4q_u8(src);
-            const uint8x16_t dup = vdupq_n_u8(tag);
-            const uint8x16_t cmp0 = vceqq_u8(chunk.val[0], dup);
-            const uint8x16_t cmp1 = vceqq_u8(chunk.val[1], dup);
-            const uint8x16_t cmp2 = vceqq_u8(chunk.val[2], dup);
-            const uint8x16_t cmp3 = vceqq_u8(chunk.val[3], dup);
-
-            const uint8x16_t t0 = vsriq_n_u8(cmp1, cmp0, 1);
-            const uint8x16_t t1 = vsriq_n_u8(cmp3, cmp2, 1);
-            const uint8x16_t t2 = vsriq_n_u8(t1, t0, 2);
-            const uint8x16_t t3 = vsriq_n_u8(t2, t2, 4);
-            const uint8x8_t t4 = vshrn_n_u16(vreinterpretq_u16_u8(t3), 4);
-            const U64 matches = vget_lane_u64(vreinterpret_u64_u8(t4), 0);
-            return ZSTD_rotateRight_U64(matches, head);
-        }
+        return ZSTD_row_getNEONMask(rowEntries, src, tag, headGrouped);
     }
 # endif /* ZSTD_ARCH_ARM_NEON */
     /* SWAR */
-    {   const size_t chunkSize = sizeof(size_t);
+    {   const int chunkSize = sizeof(size_t);
         const size_t shiftAmount = ((chunkSize * 8) - chunkSize);
         const size_t xFF = ~((size_t)0);
         const size_t x01 = xFF / 0xFF;
@@ -1091,11 +1091,11 @@ ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 head,
         }
         matches = ~matches;
         if (rowEntries == 16) {
-            return ZSTD_rotateRight_U16((U16)matches, head);
+            return ZSTD_rotateRight_U16((U16)matches, headGrouped);
         } else if (rowEntries == 32) {
-            return ZSTD_rotateRight_U32((U32)matches, head);
+            return ZSTD_rotateRight_U32((U32)matches, headGrouped);
         } else {
-            return ZSTD_rotateRight_U64((U64)matches, head);
+            return ZSTD_rotateRight_U64((U64)matches, headGrouped);
         }
     }
 #endif
@@ -1125,7 +1125,7 @@ size_t ZSTD_RowFindBestMatch(
                         const U32 rowLog)
 {
     U32* const hashTable = ms->hashTable;
-    U16* const tagTable = ms->tagTable;
+    BYTE* const tagTable = ms->tagTable;
     U32* const hashCache = ms->hashCache;
     const U32 hashLog = ms->rowHashLog;
     const ZSTD_compressionParameters* const cParams = &ms->cParams;
@@ -1143,8 +1143,11 @@ size_t ZSTD_RowFindBestMatch(
     const U32 rowEntries = (1U << rowLog);
     const U32 rowMask = rowEntries - 1;
     const U32 cappedSearchLog = MIN(cParams->searchLog, rowLog); /* nb of searches is capped at nb entries per row */
+    const U32 groupWidth = ZSTD_row_matchMaskGroupWidth(rowEntries);
+    const U64 hashSalt = ms->hashSalt;
     U32 nbAttempts = 1U << cappedSearchLog;
     size_t ml=4-1;
+    U32 hash;
 
     /* DMS/DDS variables that may be referenced laster */
     const ZSTD_matchState_t* const dms = ms->dictMatchState;
@@ -1168,7 +1171,7 @@ size_t ZSTD_RowFindBestMatch(
     if (dictMode == ZSTD_dictMatchState) {
         /* Prefetch DMS rows */
         U32* const dmsHashTable = dms->hashTable;
-        U16* const dmsTagTable = dms->tagTable;
+        BYTE* const dmsTagTable = dms->tagTable;
         U32 const dmsHash = (U32)ZSTD_hashPtr(ip, dms->rowHashLog + ZSTD_ROW_HASH_TAG_BITS, mls);
         U32 const dmsRelRow = (dmsHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
         dmsTag = dmsHash & ZSTD_ROW_HASH_TAG_MASK;
@@ -1178,23 +1181,34 @@ size_t ZSTD_RowFindBestMatch(
     }
 
     /* Update the hashTable and tagTable up to (but not including) ip */
-    ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 1 /* useCache */);
+    if (!ms->lazySkipping) {
+        ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 1 /* useCache */);
+        hash = ZSTD_row_nextCachedHash(hashCache, hashTable, tagTable, base, curr, hashLog, rowLog, mls, hashSalt);
+    } else {
+        /* Stop inserting every position when in the lazy skipping mode.
+         * The hash cache is also not kept up to date in this mode.
+         */
+        hash = (U32)ZSTD_hashPtrSalted(ip, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, hashSalt);
+        ms->nextToUpdate = curr;
+    }
+    ms->hashSaltEntropy += hash; /* collect salt entropy */
+
     {   /* Get the hash for ip, compute the appropriate row */
-        U32 const hash = ZSTD_row_nextCachedHash(hashCache, hashTable, tagTable, base, curr, hashLog, rowLog, mls);
         U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
         U32 const tag = hash & ZSTD_ROW_HASH_TAG_MASK;
         U32* const row = hashTable + relRow;
         BYTE* tagRow = (BYTE*)(tagTable + relRow);
-        U32 const head = *tagRow & rowMask;
+        U32 const headGrouped = (*tagRow & rowMask) * groupWidth;
         U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES];
         size_t numMatches = 0;
         size_t currMatch = 0;
-        ZSTD_VecMask matches = ZSTD_row_getMatchMask(tagRow, (BYTE)tag, head, rowEntries);
+        ZSTD_VecMask matches = ZSTD_row_getMatchMask(tagRow, (BYTE)tag, headGrouped, rowEntries);
 
         /* Cycle through the matches and prefetch */
-        for (; (matches > 0) && (nbAttempts > 0); --nbAttempts, matches &= (matches - 1)) {
-            U32 const matchPos = (head + ZSTD_VecMask_next(matches)) & rowMask;
+        for (; (matches > 0) && (nbAttempts > 0); matches &= (matches - 1)) {
+            U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask;
             U32 const matchIndex = row[matchPos];
+            if(matchPos == 0) continue;
             assert(numMatches < rowEntries);
             if (matchIndex < lowLimit)
                 break;
@@ -1204,13 +1218,14 @@ size_t ZSTD_RowFindBestMatch(
                 PREFETCH_L1(dictBase + matchIndex);
             }
             matchBuffer[numMatches++] = matchIndex;
+            --nbAttempts;
         }
 
         /* Speed opt: insert current byte into hashtable too. This allows us to avoid one iteration of the loop
            in ZSTD_row_update_internal() at the next search. */
         {
             U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask);
-            tagRow[pos + ZSTD_ROW_HASH_TAG_OFFSET] = (BYTE)tag;
+            tagRow[pos] = (BYTE)tag;
             row[pos] = ms->nextToUpdate++;
         }
 
@@ -1224,7 +1239,8 @@ size_t ZSTD_RowFindBestMatch(
             if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) {
                 const BYTE* const match = base + matchIndex;
                 assert(matchIndex >= dictLimit);   /* ensures this is true if dictMode != ZSTD_extDict */
-                if (match[ml] == ip[ml])   /* potentially better */
+                /* read 4B starting from (match + ml + 1 - sizeof(U32)) */
+                if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3))   /* potentially better */
                     currentMl = ZSTD_count(ip, match, iLimit);
             } else {
                 const BYTE* const match = dictBase + matchIndex;
@@ -1236,7 +1252,7 @@ size_t ZSTD_RowFindBestMatch(
             /* Save best solution */
             if (currentMl > ml) {
                 ml = currentMl;
-                *offsetPtr = STORE_OFFSET(curr - matchIndex);
+                *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex);
                 if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
             }
         }
@@ -1254,19 +1270,21 @@ size_t ZSTD_RowFindBestMatch(
         const U32 dmsSize              = (U32)(dmsEnd - dmsBase);
         const U32 dmsIndexDelta        = dictLimit - dmsSize;
 
-        {   U32 const head = *dmsTagRow & rowMask;
+        {   U32 const headGrouped = (*dmsTagRow & rowMask) * groupWidth;
             U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES];
             size_t numMatches = 0;
             size_t currMatch = 0;
-            ZSTD_VecMask matches = ZSTD_row_getMatchMask(dmsTagRow, (BYTE)dmsTag, head, rowEntries);
+            ZSTD_VecMask matches = ZSTD_row_getMatchMask(dmsTagRow, (BYTE)dmsTag, headGrouped, rowEntries);
 
-            for (; (matches > 0) && (nbAttempts > 0); --nbAttempts, matches &= (matches - 1)) {
-                U32 const matchPos = (head + ZSTD_VecMask_next(matches)) & rowMask;
+            for (; (matches > 0) && (nbAttempts > 0); matches &= (matches - 1)) {
+                U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask;
                 U32 const matchIndex = dmsRow[matchPos];
+                if(matchPos == 0) continue;
                 if (matchIndex < dmsLowestIndex)
                     break;
                 PREFETCH_L1(dmsBase + matchIndex);
                 matchBuffer[numMatches++] = matchIndex;
+                --nbAttempts;
             }
 
             /* Return the longest match */
@@ -1285,7 +1303,7 @@ size_t ZSTD_RowFindBestMatch(
                 if (currentMl > ml) {
                     ml = currentMl;
                     assert(curr > matchIndex + dmsIndexDelta);
-                    *offsetPtr = STORE_OFFSET(curr - (matchIndex + dmsIndexDelta));
+                    *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + dmsIndexDelta));
                     if (ip+currentMl == iLimit) break;
                 }
             }
@@ -1491,7 +1509,8 @@ ZSTD_compressBlock_lazy_generic(
     const U32 mls = BOUNDED(4, ms->cParams.minMatch, 6);
     const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6);
 
-    U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0;
+    U32 offset_1 = rep[0], offset_2 = rep[1];
+    U32 offsetSaved1 = 0, offsetSaved2 = 0;
 
     const int isDMS = dictMode == ZSTD_dictMatchState;
     const int isDDS = dictMode == ZSTD_dedicatedDictSearch;
@@ -1512,8 +1531,8 @@ ZSTD_compressBlock_lazy_generic(
         U32 const curr = (U32)(ip - base);
         U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, ms->cParams.windowLog);
         U32 const maxRep = curr - windowLow;
-        if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0;
-        if (offset_1 > maxRep) savedOffset = offset_1, offset_1 = 0;
+        if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0;
+        if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0;
     }
     if (isDxS) {
         /* dictMatchState repCode checks don't currently handle repCode == 0
@@ -1522,10 +1541,11 @@ ZSTD_compressBlock_lazy_generic(
         assert(offset_2 <= dictAndPrefixLength);
     }
 
+    /* Reset the lazy skipping state */
+    ms->lazySkipping = 0;
+
     if (searchMethod == search_rowHash) {
-        ZSTD_row_fillHashCache(ms, base, rowLog,
-                            MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */),
-                            ms->nextToUpdate, ilimit);
+        ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit);
     }
 
     /* Match Loop */
@@ -1537,7 +1557,7 @@ ZSTD_compressBlock_lazy_generic(
 #endif
     while (ip < ilimit) {
         size_t matchLength=0;
-        size_t offcode=STORE_REPCODE_1;
+        size_t offBase = REPCODE1_TO_OFFBASE;
         const BYTE* start=ip+1;
         DEBUGLOG(7, "search baseline (depth 0)");
 
@@ -1562,14 +1582,23 @@ ZSTD_compressBlock_lazy_generic(
         }
 
         /* first search (depth 0) */
-        {   size_t offsetFound = 999999999;
-            size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &offsetFound, mls, rowLog, searchMethod, dictMode);
+        {   size_t offbaseFound = 999999999;
+            size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &offbaseFound, mls, rowLog, searchMethod, dictMode);
             if (ml2 > matchLength)
-                matchLength = ml2, start = ip, offcode=offsetFound;
+                matchLength = ml2, start = ip, offBase = offbaseFound;
         }
 
         if (matchLength < 4) {
-            ip += ((ip-anchor) >> kSearchStrength) + 1;   /* jump faster over incompressible sections */
+            size_t const step = ((size_t)(ip-anchor) >> kSearchStrength) + 1;   /* jump faster over incompressible sections */;
+            ip += step;
+            /* Enter the lazy skipping mode once we are skipping more than 8 bytes at a time.
+             * In this mode we stop inserting every position into our tables, and only insert
+             * positions that we search, which is one in step positions.
+             * The exact cutoff is flexible, I've just chosen a number that is reasonably high,
+             * so we minimize the compression ratio loss in "normal" scenarios. This mode gets
+             * triggered once we've gone 2KB without finding any matches.
+             */
+            ms->lazySkipping = step > kLazySkippingStep;
             continue;
         }
 
@@ -1579,12 +1608,12 @@ ZSTD_compressBlock_lazy_generic(
             DEBUGLOG(7, "search depth 1");
             ip ++;
             if ( (dictMode == ZSTD_noDict)
-              && (offcode) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) {
+              && (offBase) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) {
                 size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4;
                 int const gain2 = (int)(mlRep * 3);
-                int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1);
+                int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1);
                 if ((mlRep >= 4) && (gain2 > gain1))
-                    matchLength = mlRep, offcode = STORE_REPCODE_1, start = ip;
+                    matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip;
             }
             if (isDxS) {
                 const U32 repIndex = (U32)(ip - base) - offset_1;
@@ -1596,17 +1625,17 @@ ZSTD_compressBlock_lazy_generic(
                     const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
                     size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
                     int const gain2 = (int)(mlRep * 3);
-                    int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1);
+                    int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1);
                     if ((mlRep >= 4) && (gain2 > gain1))
-                        matchLength = mlRep, offcode = STORE_REPCODE_1, start = ip;
+                        matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip;
                 }
             }
-            {   size_t offset2=999999999;
-                size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &offset2, mls, rowLog, searchMethod, dictMode);
-                int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offset2)));   /* raw approx */
-                int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 4);
+            {   size_t ofbCandidate=999999999;
+                size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode);
+                int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate));   /* raw approx */
+                int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4);
                 if ((ml2 >= 4) && (gain2 > gain1)) {
-                    matchLength = ml2, offcode = offset2, start = ip;
+                    matchLength = ml2, offBase = ofbCandidate, start = ip;
                     continue;   /* search a better one */
             }   }
 
@@ -1615,12 +1644,12 @@ ZSTD_compressBlock_lazy_generic(
                 DEBUGLOG(7, "search depth 2");
                 ip ++;
                 if ( (dictMode == ZSTD_noDict)
-                  && (offcode) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) {
+                  && (offBase) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) {
                     size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4;
                     int const gain2 = (int)(mlRep * 4);
-                    int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1);
+                    int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1);
                     if ((mlRep >= 4) && (gain2 > gain1))
-                        matchLength = mlRep, offcode = STORE_REPCODE_1, start = ip;
+                        matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip;
                 }
                 if (isDxS) {
                     const U32 repIndex = (U32)(ip - base) - offset_1;
@@ -1632,17 +1661,17 @@ ZSTD_compressBlock_lazy_generic(
                         const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
                         size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
                         int const gain2 = (int)(mlRep * 4);
-                        int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1);
+                        int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1);
                         if ((mlRep >= 4) && (gain2 > gain1))
-                            matchLength = mlRep, offcode = STORE_REPCODE_1, start = ip;
+                            matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip;
                     }
                 }
-                {   size_t offset2=999999999;
-                    size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &offset2, mls, rowLog, searchMethod, dictMode);
-                    int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offset2)));   /* raw approx */
-                    int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 7);
+                {   size_t ofbCandidate=999999999;
+                    size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode);
+                    int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate));   /* raw approx */
+                    int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7);
                     if ((ml2 >= 4) && (gain2 > gain1)) {
-                        matchLength = ml2, offcode = offset2, start = ip;
+                        matchLength = ml2, offBase = ofbCandidate, start = ip;
                         continue;
             }   }   }
             break;  /* nothing found : store previous solution */
@@ -1653,26 +1682,33 @@ ZSTD_compressBlock_lazy_generic(
          * notably if `value` is unsigned, resulting in a large positive `-value`.
          */
         /* catch up */
-        if (STORED_IS_OFFSET(offcode)) {
+        if (OFFBASE_IS_OFFSET(offBase)) {
             if (dictMode == ZSTD_noDict) {
-                while ( ((start > anchor) & (start - STORED_OFFSET(offcode) > prefixLowest))
-                     && (start[-1] == (start-STORED_OFFSET(offcode))[-1]) )  /* only search for offset within prefix */
+                while ( ((start > anchor) & (start - OFFBASE_TO_OFFSET(offBase) > prefixLowest))
+                     && (start[-1] == (start-OFFBASE_TO_OFFSET(offBase))[-1]) )  /* only search for offset within prefix */
                     { start--; matchLength++; }
             }
             if (isDxS) {
-                U32 const matchIndex = (U32)((size_t)(start-base) - STORED_OFFSET(offcode));
+                U32 const matchIndex = (U32)((size_t)(start-base) - OFFBASE_TO_OFFSET(offBase));
                 const BYTE* match = (matchIndex < prefixLowestIndex) ? dictBase + matchIndex - dictIndexDelta : base + matchIndex;
                 const BYTE* const mStart = (matchIndex < prefixLowestIndex) ? dictLowest : prefixLowest;
                 while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; }  /* catch up */
             }
-            offset_2 = offset_1; offset_1 = (U32)STORED_OFFSET(offcode);
+            offset_2 = offset_1; offset_1 = (U32)OFFBASE_TO_OFFSET(offBase);
         }
         /* store sequence */
 _storeSequence:
         {   size_t const litLength = (size_t)(start - anchor);
-            ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offcode, matchLength);
+            ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength);
             anchor = ip = start + matchLength;
         }
+        if (ms->lazySkipping) {
+            /* We've found a match, disable lazy skipping mode, and refill the hash cache. */
+            if (searchMethod == search_rowHash) {
+                ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit);
+            }
+            ms->lazySkipping = 0;
+        }
 
         /* check immediate repcode */
         if (isDxS) {
@@ -1686,8 +1722,8 @@ ZSTD_compressBlock_lazy_generic(
                    && (MEM_read32(repMatch) == MEM_read32(ip)) ) {
                     const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend;
                     matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4;
-                    offcode = offset_2; offset_2 = offset_1; offset_1 = (U32)offcode;   /* swap offset_2 <=> offset_1 */
-                    ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, matchLength);
+                    offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase;   /* swap offset_2 <=> offset_1 */
+                    ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength);
                     ip += matchLength;
                     anchor = ip;
                     continue;
@@ -1701,16 +1737,20 @@ ZSTD_compressBlock_lazy_generic(
                  && (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) {
                 /* store sequence */
                 matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
-                offcode = offset_2; offset_2 = offset_1; offset_1 = (U32)offcode; /* swap repcodes */
-                ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, matchLength);
+                offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap repcodes */
+                ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength);
                 ip += matchLength;
                 anchor = ip;
                 continue;   /* faster when present ... (?) */
     }   }   }
 
-    /* Save reps for next block */
-    rep[0] = offset_1 ? offset_1 : savedOffset;
-    rep[1] = offset_2 ? offset_2 : savedOffset;
+    /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0),
+     * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */
+    offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2;
+
+    /* save reps for next block */
+    rep[0] = offset_1 ? offset_1 : offsetSaved1;
+    rep[1] = offset_2 ? offset_2 : offsetSaved2;
 
     /* Return the last literals size */
     return (size_t)(iend - anchor);
@@ -1886,12 +1926,13 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
 
     DEBUGLOG(5, "ZSTD_compressBlock_lazy_extDict_generic (searchFunc=%u)", (U32)searchMethod);
 
+    /* Reset the lazy skipping state */
+    ms->lazySkipping = 0;
+
     /* init */
     ip += (ip == prefixStart);
     if (searchMethod == search_rowHash) {
-        ZSTD_row_fillHashCache(ms, base, rowLog,
-                               MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */),
-                               ms->nextToUpdate, ilimit);
+        ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit);
     }
 
     /* Match Loop */
@@ -1903,7 +1944,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
 #endif
     while (ip < ilimit) {
         size_t matchLength=0;
-        size_t offcode=STORE_REPCODE_1;
+        size_t offBase = REPCODE1_TO_OFFBASE;
         const BYTE* start=ip+1;
         U32 curr = (U32)(ip-base);
 
@@ -1922,14 +1963,23 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
         }   }
 
         /* first search (depth 0) */
-        {   size_t offsetFound = 999999999;
-            size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &offsetFound, mls, rowLog, searchMethod, ZSTD_extDict);
+        {   size_t ofbCandidate = 999999999;
+            size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict);
             if (ml2 > matchLength)
-                matchLength = ml2, start = ip, offcode=offsetFound;
+                matchLength = ml2, start = ip, offBase = ofbCandidate;
         }
 
         if (matchLength < 4) {
-            ip += ((ip-anchor) >> kSearchStrength) + 1;   /* jump faster over incompressible sections */
+            size_t const step = ((size_t)(ip-anchor) >> kSearchStrength);
+            ip += step + 1;   /* jump faster over incompressible sections */
+            /* Enter the lazy skipping mode once we are skipping more than 8 bytes at a time.
+             * In this mode we stop inserting every position into our tables, and only insert
+             * positions that we search, which is one in step positions.
+             * The exact cutoff is flexible, I've just chosen a number that is reasonably high,
+             * so we minimize the compression ratio loss in "normal" scenarios. This mode gets
+             * triggered once we've gone 2KB without finding any matches.
+             */
+            ms->lazySkipping = step > kLazySkippingStep;
             continue;
         }
 
@@ -1939,7 +1989,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
             ip ++;
             curr++;
             /* check repCode */
-            if (offcode) {
+            if (offBase) {
                 const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr, windowLog);
                 const U32 repIndex = (U32)(curr - offset_1);
                 const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
@@ -1951,18 +2001,18 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
                     const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
                     size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4;
                     int const gain2 = (int)(repLength * 3);
-                    int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1);
+                    int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1);
                     if ((repLength >= 4) && (gain2 > gain1))
-                        matchLength = repLength, offcode = STORE_REPCODE_1, start = ip;
+                        matchLength = repLength, offBase = REPCODE1_TO_OFFBASE, start = ip;
             }   }
 
             /* search match, depth 1 */
-            {   size_t offset2=999999999;
-                size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &offset2, mls, rowLog, searchMethod, ZSTD_extDict);
-                int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offset2)));   /* raw approx */
-                int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 4);
+            {   size_t ofbCandidate = 999999999;
+                size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict);
+                int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate));   /* raw approx */
+                int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4);
                 if ((ml2 >= 4) && (gain2 > gain1)) {
-                    matchLength = ml2, offcode = offset2, start = ip;
+                    matchLength = ml2, offBase = ofbCandidate, start = ip;
                     continue;   /* search a better one */
             }   }
 
@@ -1971,7 +2021,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
                 ip ++;
                 curr++;
                 /* check repCode */
-                if (offcode) {
+                if (offBase) {
                     const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr, windowLog);
                     const U32 repIndex = (U32)(curr - offset_1);
                     const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
@@ -1983,38 +2033,45 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
                         const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
                         size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4;
                         int const gain2 = (int)(repLength * 4);
-                        int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1);
+                        int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1);
                         if ((repLength >= 4) && (gain2 > gain1))
-                            matchLength = repLength, offcode = STORE_REPCODE_1, start = ip;
+                            matchLength = repLength, offBase = REPCODE1_TO_OFFBASE, start = ip;
                 }   }
 
                 /* search match, depth 2 */
-                {   size_t offset2=999999999;
-                    size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &offset2, mls, rowLog, searchMethod, ZSTD_extDict);
-                    int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offset2)));   /* raw approx */
-                    int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 7);
+                {   size_t ofbCandidate = 999999999;
+                    size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict);
+                    int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate));   /* raw approx */
+                    int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7);
                     if ((ml2 >= 4) && (gain2 > gain1)) {
-                        matchLength = ml2, offcode = offset2, start = ip;
+                        matchLength = ml2, offBase = ofbCandidate, start = ip;
                         continue;
             }   }   }
             break;  /* nothing found : store previous solution */
         }
 
         /* catch up */
-        if (STORED_IS_OFFSET(offcode)) {
-            U32 const matchIndex = (U32)((size_t)(start-base) - STORED_OFFSET(offcode));
+        if (OFFBASE_IS_OFFSET(offBase)) {
+            U32 const matchIndex = (U32)((size_t)(start-base) - OFFBASE_TO_OFFSET(offBase));
             const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
             const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart;
             while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; }  /* catch up */
-            offset_2 = offset_1; offset_1 = (U32)STORED_OFFSET(offcode);
+            offset_2 = offset_1; offset_1 = (U32)OFFBASE_TO_OFFSET(offBase);
         }
 
         /* store sequence */
 _storeSequence:
         {   size_t const litLength = (size_t)(start - anchor);
-            ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offcode, matchLength);
+            ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength);
             anchor = ip = start + matchLength;
         }
+        if (ms->lazySkipping) {
+            /* We've found a match, disable lazy skipping mode, and refill the hash cache. */
+            if (searchMethod == search_rowHash) {
+                ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit);
+            }
+            ms->lazySkipping = 0;
+        }
 
         /* check immediate repcode */
         while (ip <= ilimit) {
@@ -2029,8 +2086,8 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
                 /* repcode detected we should take it */
                 const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
                 matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4;
-                offcode = offset_2; offset_2 = offset_1; offset_1 = (U32)offcode;   /* swap offset history */
-                ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, matchLength);
+                offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase;   /* swap offset history */
+                ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength);
                 ip += matchLength;
                 anchor = ip;
                 continue;   /* faster when present ... (?) */
@@ -2096,7 +2153,6 @@ size_t ZSTD_compressBlock_lazy_extDict_row(
 size_t ZSTD_compressBlock_lazy2_extDict_row(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
         void const* src, size_t srcSize)
-
 {
     return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2);
 }
diff --git a/lib/zstd/compress/zstd_lazy.h b/lib/zstd/compress/zstd_lazy.h
index e5bdf4df8dde..9505bed93c03 100644
--- a/lib/zstd/compress/zstd_lazy.h
+++ b/lib/zstd/compress/zstd_lazy.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -22,6 +23,8 @@
  */
 #define ZSTD_LAZY_DDSS_BUCKET_LOG 2
 
+#define ZSTD_ROW_HASH_TAG_BITS 8        /* nb bits to use for the tag */
+
 U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip);
 void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip);
 
@@ -113,7 +116,7 @@ size_t ZSTD_compressBlock_lazy2_extDict_row(
 size_t ZSTD_compressBlock_btlazy2_extDict(
         ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
         void const* src, size_t srcSize);
-        
+
 
 
 #endif /* ZSTD_LAZY_H */
diff --git a/lib/zstd/compress/zstd_ldm.c b/lib/zstd/compress/zstd_ldm.c
index dd86fc83e7dd..b7da76b0db7c 100644
--- a/lib/zstd/compress/zstd_ldm.c
+++ b/lib/zstd/compress/zstd_ldm.c
@@ -1,5 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -242,11 +243,11 @@ static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms,
     switch(ms->cParams.strategy)
     {
     case ZSTD_fast:
-        ZSTD_fillHashTable(ms, iend, ZSTD_dtlm_fast);
+        ZSTD_fillHashTable(ms, iend, ZSTD_dtlm_fast, ZSTD_tfp_forCCtx);
         break;
 
     case ZSTD_dfast:
-        ZSTD_fillDoubleHashTable(ms, iend, ZSTD_dtlm_fast);
+        ZSTD_fillDoubleHashTable(ms, iend, ZSTD_dtlm_fast, ZSTD_tfp_forCCtx);
         break;
 
     case ZSTD_greedy:
@@ -549,7 +550,7 @@ size_t ZSTD_ldm_generateSequences(
          * the window through early invalidation.
          * TODO: * Test the chunk size.
          *       * Try invalidation after the sequence generation and test the
-         *         the offset against maxDist directly.
+         *         offset against maxDist directly.
          *
          * NOTE: Because of dictionaries + sequence splitting we MUST make sure
          * that any offset used is valid at the END of the sequence, since it may
@@ -711,7 +712,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
             rep[0] = sequence.offset;
             /* Store the sequence */
             ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend,
-                          STORE_OFFSET(sequence.offset),
+                          OFFSET_TO_OFFBASE(sequence.offset),
                           sequence.matchLength);
             ip += sequence.matchLength;
         }
diff --git a/lib/zstd/compress/zstd_ldm.h b/lib/zstd/compress/zstd_ldm.h
index fbc6a5e88fd7..c540731abde7 100644
--- a/lib/zstd/compress/zstd_ldm.h
+++ b/lib/zstd/compress/zstd_ldm.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/zstd/compress/zstd_ldm_geartab.h b/lib/zstd/compress/zstd_ldm_geartab.h
index 647f865be290..cfccfc46f6f7 100644
--- a/lib/zstd/compress/zstd_ldm_geartab.h
+++ b/lib/zstd/compress/zstd_ldm_geartab.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/zstd/compress/zstd_opt.c b/lib/zstd/compress/zstd_opt.c
index fd82acfda62f..1e41cb04f482 100644
--- a/lib/zstd/compress/zstd_opt.c
+++ b/lib/zstd/compress/zstd_opt.c
@@ -1,5 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -16,7 +17,7 @@
 #define ZSTD_LITFREQ_ADD    2   /* scaling factor for litFreq, so that frequencies adapt faster to new stats */
 #define ZSTD_MAX_PRICE     (1<<30)
 
-#define ZSTD_PREDEF_THRESHOLD 1024   /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */
+#define ZSTD_PREDEF_THRESHOLD 8   /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */
 
 
 /*-*************************************
@@ -26,27 +27,35 @@
 #if 0    /* approximation at bit level (for tests) */
 #  define BITCOST_ACCURACY 0
 #  define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
-#  define WEIGHT(stat, opt) ((void)opt, ZSTD_bitWeight(stat))
+#  define WEIGHT(stat, opt) ((void)(opt), ZSTD_bitWeight(stat))
 #elif 0  /* fractional bit accuracy (for tests) */
 #  define BITCOST_ACCURACY 8
 #  define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
-#  define WEIGHT(stat,opt) ((void)opt, ZSTD_fracWeight(stat))
+#  define WEIGHT(stat,opt) ((void)(opt), ZSTD_fracWeight(stat))
 #else    /* opt==approx, ultra==accurate */
 #  define BITCOST_ACCURACY 8
 #  define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY)
-#  define WEIGHT(stat,opt) (opt ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat))
+#  define WEIGHT(stat,opt) ((opt) ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat))
 #endif
 
+/* ZSTD_bitWeight() :
+ * provide estimated "cost" of a stat in full bits only */
 MEM_STATIC U32 ZSTD_bitWeight(U32 stat)
 {
     return (ZSTD_highbit32(stat+1) * BITCOST_MULTIPLIER);
 }
 
+/* ZSTD_fracWeight() :
+ * provide fractional-bit "cost" of a stat,
+ * using linear interpolation approximation */
 MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat)
 {
     U32 const stat = rawStat + 1;
     U32 const hb = ZSTD_highbit32(stat);
     U32 const BWeight = hb * BITCOST_MULTIPLIER;
+    /* Fweight was meant for "Fractional weight"
+     * but it's effectively a value between 1 and 2
+     * using fixed point arithmetic */
     U32 const FWeight = (stat << BITCOST_ACCURACY) >> hb;
     U32 const weight = BWeight + FWeight;
     assert(hb + BITCOST_ACCURACY < 31);
@@ -57,7 +66,7 @@ MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat)
 /* debugging function,
  * @return price in bytes as fractional value
  * for debug messages only */
-MEM_STATIC double ZSTD_fCost(U32 price)
+MEM_STATIC double ZSTD_fCost(int price)
 {
     return (double)price / (BITCOST_MULTIPLIER*8);
 }
@@ -88,20 +97,26 @@ static U32 sum_u32(const unsigned table[], size_t nbElts)
     return total;
 }
 
-static U32 ZSTD_downscaleStats(unsigned* table, U32 lastEltIndex, U32 shift)
+typedef enum { base_0possible=0, base_1guaranteed=1 } base_directive_e;
+
+static U32
+ZSTD_downscaleStats(unsigned* table, U32 lastEltIndex, U32 shift, base_directive_e base1)
 {
     U32 s, sum=0;
-    DEBUGLOG(5, "ZSTD_downscaleStats (nbElts=%u, shift=%u)", (unsigned)lastEltIndex+1, (unsigned)shift);
+    DEBUGLOG(5, "ZSTD_downscaleStats (nbElts=%u, shift=%u)",
+            (unsigned)lastEltIndex+1, (unsigned)shift );
     assert(shift < 30);
     for (s=0; s<lastEltIndex+1; s++) {
-        table[s] = 1 + (table[s] >> shift);
-        sum += table[s];
+        unsigned const base = base1 ? 1 : (table[s]>0);
+        unsigned const newStat = base + (table[s] >> shift);
+        sum += newStat;
+        table[s] = newStat;
     }
     return sum;
 }
 
 /* ZSTD_scaleStats() :
- * reduce all elements in table is sum too large
+ * reduce all elt frequencies in table if sum too large
  * return the resulting sum of elements */
 static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget)
 {
@@ -110,7 +125,7 @@ static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget)
     DEBUGLOG(5, "ZSTD_scaleStats (nbElts=%u, target=%u)", (unsigned)lastEltIndex+1, (unsigned)logTarget);
     assert(logTarget < 30);
     if (factor <= 1) return prevsum;
-    return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor));
+    return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor), base_1guaranteed);
 }
 
 /* ZSTD_rescaleFreqs() :
@@ -129,18 +144,22 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
     DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize);
     optPtr->priceType = zop_dynamic;
 
-    if (optPtr->litLengthSum == 0) {  /* first block : init */
-        if (srcSize <= ZSTD_PREDEF_THRESHOLD) {  /* heuristic */
-            DEBUGLOG(5, "(srcSize <= ZSTD_PREDEF_THRESHOLD) => zop_predef");
+    if (optPtr->litLengthSum == 0) {  /* no literals stats collected -> first block assumed -> init */
+
+        /* heuristic: use pre-defined stats for too small inputs */
+        if (srcSize <= ZSTD_PREDEF_THRESHOLD) {
+            DEBUGLOG(5, "srcSize <= %i : use predefined stats", ZSTD_PREDEF_THRESHOLD);
             optPtr->priceType = zop_predef;
         }
 
         assert(optPtr->symbolCosts != NULL);
         if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) {
-            /* huffman table presumed generated by dictionary */
+
+            /* huffman stats covering the full value set : table presumed generated by dictionary */
             optPtr->priceType = zop_dynamic;
 
             if (compressedLiterals) {
+                /* generate literals statistics from huffman table */
                 unsigned lit;
                 assert(optPtr->litFreq != NULL);
                 optPtr->litSum = 0;
@@ -188,13 +207,14 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
                     optPtr->offCodeSum += optPtr->offCodeFreq[of];
             }   }
 
-        } else {  /* not a dictionary */
+        } else {  /* first block, no dictionary */
 
             assert(optPtr->litFreq != NULL);
             if (compressedLiterals) {
+                /* base initial cost of literals on direct frequency within src */
                 unsigned lit = MaxLit;
                 HIST_count_simple(optPtr->litFreq, &lit, src, srcSize);   /* use raw first block to init statistics */
-                optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8);
+                optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8, base_0possible);
             }
 
             {   unsigned const baseLLfreqs[MaxLL+1] = {
@@ -224,10 +244,9 @@ ZSTD_rescaleFreqs(optState_t* const optPtr,
                 optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1);
             }
 
-
         }
 
-    } else {   /* new block : re-use previous statistics, scaled down */
+    } else {   /* new block : scale down accumulated statistics */
 
         if (compressedLiterals)
             optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 12);
@@ -255,11 +274,14 @@ static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength,
         return (litLength*6) * BITCOST_MULTIPLIER;  /* 6 bit per literal - no statistic used */
 
     /* dynamic statistics */
-    {   U32 price = litLength * optPtr->litSumBasePrice;
+    {   U32 price = optPtr->litSumBasePrice * litLength;
+        U32 const litPriceMax = optPtr->litSumBasePrice - BITCOST_MULTIPLIER;
         U32 u;
+        assert(optPtr->litSumBasePrice >= BITCOST_MULTIPLIER);
         for (u=0; u < litLength; u++) {
-            assert(WEIGHT(optPtr->litFreq[literals[u]], optLevel) <= optPtr->litSumBasePrice);   /* literal cost should never be negative */
-            price -= WEIGHT(optPtr->litFreq[literals[u]], optLevel);
+            U32 litPrice = WEIGHT(optPtr->litFreq[literals[u]], optLevel);
+            if (UNLIKELY(litPrice > litPriceMax)) litPrice = litPriceMax;
+            price -= litPrice;
         }
         return price;
     }
@@ -272,10 +294,11 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP
     assert(litLength <= ZSTD_BLOCKSIZE_MAX);
     if (optPtr->priceType == zop_predef)
         return WEIGHT(litLength, optLevel);
-    /* We can't compute the litLength price for sizes >= ZSTD_BLOCKSIZE_MAX
-     * because it isn't representable in the zstd format. So instead just
-     * call it 1 bit more than ZSTD_BLOCKSIZE_MAX - 1. In this case the block
-     * would be all literals.
+
+    /* ZSTD_LLcode() can't compute litLength price for sizes >= ZSTD_BLOCKSIZE_MAX
+     * because it isn't representable in the zstd format.
+     * So instead just pretend it would cost 1 bit more than ZSTD_BLOCKSIZE_MAX - 1.
+     * In such a case, the block would be all literals.
      */
     if (litLength == ZSTD_BLOCKSIZE_MAX)
         return BITCOST_MULTIPLIER + ZSTD_litLengthPrice(ZSTD_BLOCKSIZE_MAX - 1, optPtr, optLevel);
@@ -289,24 +312,25 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP
 }
 
 /* ZSTD_getMatchPrice() :
- * Provides the cost of the match part (offset + matchLength) of a sequence
+ * Provides the cost of the match part (offset + matchLength) of a sequence.
  * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence.
- * @offcode : expects a scale where 0,1,2 are repcodes 1-3, and 3+ are real_offsets+2
+ * @offBase : sumtype, representing an offset or a repcode, and using numeric representation of ZSTD_storeSeq()
  * @optLevel: when <2, favors small offset for decompression speed (improved cache efficiency)
  */
 FORCE_INLINE_TEMPLATE U32
-ZSTD_getMatchPrice(U32 const offcode,
+ZSTD_getMatchPrice(U32 const offBase,
                    U32 const matchLength,
              const optState_t* const optPtr,
                    int const optLevel)
 {
     U32 price;
-    U32 const offCode = ZSTD_highbit32(STORED_TO_OFFBASE(offcode));
+    U32 const offCode = ZSTD_highbit32(offBase);
     U32 const mlBase = matchLength - MINMATCH;
     assert(matchLength >= MINMATCH);
 
-    if (optPtr->priceType == zop_predef)  /* fixed scheme, do not use statistics */
-        return WEIGHT(mlBase, optLevel) + ((16 + offCode) * BITCOST_MULTIPLIER);
+    if (optPtr->priceType == zop_predef)  /* fixed scheme, does not use statistics */
+        return WEIGHT(mlBase, optLevel)
+             + ((16 + offCode) * BITCOST_MULTIPLIER); /* emulated offset cost */
 
     /* dynamic statistics */
     price = (offCode * BITCOST_MULTIPLIER) + (optPtr->offCodeSumBasePrice - WEIGHT(optPtr->offCodeFreq[offCode], optLevel));
@@ -325,10 +349,10 @@ ZSTD_getMatchPrice(U32 const offcode,
 }
 
 /* ZSTD_updateStats() :
- * assumption : literals + litLengtn <= iend */
+ * assumption : literals + litLength <= iend */
 static void ZSTD_updateStats(optState_t* const optPtr,
                              U32 litLength, const BYTE* literals,
-                             U32 offsetCode, U32 matchLength)
+                             U32 offBase, U32 matchLength)
 {
     /* literals */
     if (ZSTD_compressedLiterals(optPtr)) {
@@ -344,8 +368,8 @@ static void ZSTD_updateStats(optState_t* const optPtr,
         optPtr->litLengthSum++;
     }
 
-    /* offset code : expected to follow storeSeq() numeric representation */
-    {   U32 const offCode = ZSTD_highbit32(STORED_TO_OFFBASE(offsetCode));
+    /* offset code : follows storeSeq() numeric representation */
+    {   U32 const offCode = ZSTD_highbit32(offBase);
         assert(offCode <= MaxOff);
         optPtr->offCodeFreq[offCode]++;
         optPtr->offCodeSum++;
@@ -552,16 +576,17 @@ void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) {
     ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict);
 }
 
-FORCE_INLINE_TEMPLATE
-U32 ZSTD_insertBtAndGetAllMatches (
-                    ZSTD_match_t* matches,   /* store result (found matches) in this table (presumed large enough) */
-                    ZSTD_matchState_t* ms,
-                    U32* nextToUpdate3,
-                    const BYTE* const ip, const BYTE* const iLimit, const ZSTD_dictMode_e dictMode,
-                    const U32 rep[ZSTD_REP_NUM],
-                    U32 const ll0,   /* tells if associated literal length is 0 or not. This value must be 0 or 1 */
-                    const U32 lengthToBeat,
-                    U32 const mls /* template */)
+FORCE_INLINE_TEMPLATE U32
+ZSTD_insertBtAndGetAllMatches (
+                ZSTD_match_t* matches,  /* store result (found matches) in this table (presumed large enough) */
+                ZSTD_matchState_t* ms,
+                U32* nextToUpdate3,
+                const BYTE* const ip, const BYTE* const iLimit,
+                const ZSTD_dictMode_e dictMode,
+                const U32 rep[ZSTD_REP_NUM],
+                const U32 ll0,  /* tells if associated literal length is 0 or not. This value must be 0 or 1 */
+                const U32 lengthToBeat,
+                const U32 mls /* template */)
 {
     const ZSTD_compressionParameters* const cParams = &ms->cParams;
     U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
@@ -644,7 +669,7 @@ U32 ZSTD_insertBtAndGetAllMatches (
                 DEBUGLOG(8, "found repCode %u (ll0:%u, offset:%u) of length %u",
                             repCode, ll0, repOffset, repLen);
                 bestLength = repLen;
-                matches[mnum].off = STORE_REPCODE(repCode - ll0 + 1);  /* expect value between 1 and 3 */
+                matches[mnum].off = REPCODE_TO_OFFBASE(repCode - ll0 + 1);  /* expect value between 1 and 3 */
                 matches[mnum].len = (U32)repLen;
                 mnum++;
                 if ( (repLen > sufficient_len)
@@ -673,7 +698,7 @@ U32 ZSTD_insertBtAndGetAllMatches (
                 bestLength = mlen;
                 assert(curr > matchIndex3);
                 assert(mnum==0);  /* no prior solution */
-                matches[0].off = STORE_OFFSET(curr - matchIndex3);
+                matches[0].off = OFFSET_TO_OFFBASE(curr - matchIndex3);
                 matches[0].len = (U32)mlen;
                 mnum = 1;
                 if ( (mlen > sufficient_len) |
@@ -706,13 +731,13 @@ U32 ZSTD_insertBtAndGetAllMatches (
         }
 
         if (matchLength > bestLength) {
-            DEBUGLOG(8, "found match of length %u at distance %u (offCode=%u)",
-                    (U32)matchLength, curr - matchIndex, STORE_OFFSET(curr - matchIndex));
+            DEBUGLOG(8, "found match of length %u at distance %u (offBase=%u)",
+                    (U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex));
             assert(matchEndIdx > matchIndex);
             if (matchLength > matchEndIdx - matchIndex)
                 matchEndIdx = matchIndex + (U32)matchLength;
             bestLength = matchLength;
-            matches[mnum].off = STORE_OFFSET(curr - matchIndex);
+            matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex);
             matches[mnum].len = (U32)matchLength;
             mnum++;
             if ( (matchLength > ZSTD_OPT_NUM)
@@ -754,12 +779,12 @@ U32 ZSTD_insertBtAndGetAllMatches (
 
             if (matchLength > bestLength) {
                 matchIndex = dictMatchIndex + dmsIndexDelta;
-                DEBUGLOG(8, "found dms match of length %u at distance %u (offCode=%u)",
-                        (U32)matchLength, curr - matchIndex, STORE_OFFSET(curr - matchIndex));
+                DEBUGLOG(8, "found dms match of length %u at distance %u (offBase=%u)",
+                        (U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex));
                 if (matchLength > matchEndIdx - matchIndex)
                     matchEndIdx = matchIndex + (U32)matchLength;
                 bestLength = matchLength;
-                matches[mnum].off = STORE_OFFSET(curr - matchIndex);
+                matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex);
                 matches[mnum].len = (U32)matchLength;
                 mnum++;
                 if ( (matchLength > ZSTD_OPT_NUM)
@@ -960,7 +985,7 @@ static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches,
                                       const ZSTD_optLdm_t* optLdm, U32 currPosInBlock)
 {
     U32 const posDiff = currPosInBlock - optLdm->startPosInBlock;
-    /* Note: ZSTD_match_t actually contains offCode and matchLength (before subtracting MINMATCH) */
+    /* Note: ZSTD_match_t actually contains offBase and matchLength (before subtracting MINMATCH) */
     U32 const candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff;
 
     /* Ensure that current block position is not outside of the match */
@@ -971,11 +996,11 @@ static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches,
     }
 
     if (*nbMatches == 0 || ((candidateMatchLength > matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM)) {
-        U32 const candidateOffCode = STORE_OFFSET(optLdm->offset);
-        DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offCode: %u matchLength %u) at block position=%u",
-                 candidateOffCode, candidateMatchLength, currPosInBlock);
+        U32 const candidateOffBase = OFFSET_TO_OFFBASE(optLdm->offset);
+        DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offBase: %u matchLength %u) at block position=%u",
+                 candidateOffBase, candidateMatchLength, currPosInBlock);
         matches[*nbMatches].len = candidateMatchLength;
-        matches[*nbMatches].off = candidateOffCode;
+        matches[*nbMatches].off = candidateOffBase;
         (*nbMatches)++;
     }
 }
@@ -1062,6 +1087,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
     ZSTD_optimal_t lastSequence;
     ZSTD_optLdm_t optLdm;
 
+    ZSTD_memset(&lastSequence, 0, sizeof(ZSTD_optimal_t));
+
     optLdm.seqStore = ms->ldmSeqStore ? *ms->ldmSeqStore : kNullRawSeqStore;
     optLdm.endPosInBlock = optLdm.startPosInBlock = optLdm.offset = 0;
     ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip));
@@ -1098,14 +1125,14 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
 
             /* large match -> immediate encoding */
             {   U32 const maxML = matches[nbMatches-1].len;
-                U32 const maxOffcode = matches[nbMatches-1].off;
-                DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffCode=%u at cPos=%u => start new series",
-                            nbMatches, maxML, maxOffcode, (U32)(ip-prefixStart));
+                U32 const maxOffBase = matches[nbMatches-1].off;
+                DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffBase=%u at cPos=%u => start new series",
+                            nbMatches, maxML, maxOffBase, (U32)(ip-prefixStart));
 
                 if (maxML > sufficient_len) {
                     lastSequence.litlen = litlen;
                     lastSequence.mlen = maxML;
-                    lastSequence.off = maxOffcode;
+                    lastSequence.off = maxOffBase;
                     DEBUGLOG(6, "large match (%u>%u), immediate encoding",
                                 maxML, sufficient_len);
                     cur = 0;
@@ -1122,15 +1149,15 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
                     opt[pos].price = ZSTD_MAX_PRICE;   /* mlen, litlen and price will be fixed during forward scanning */
                 }
                 for (matchNb = 0; matchNb < nbMatches; matchNb++) {
-                    U32 const offcode = matches[matchNb].off;
+                    U32 const offBase = matches[matchNb].off;
                     U32 const end = matches[matchNb].len;
                     for ( ; pos <= end ; pos++ ) {
-                        U32 const matchPrice = ZSTD_getMatchPrice(offcode, pos, optStatePtr, optLevel);
+                        U32 const matchPrice = ZSTD_getMatchPrice(offBase, pos, optStatePtr, optLevel);
                         U32 const sequencePrice = literalsPrice + matchPrice;
                         DEBUGLOG(7, "rPos:%u => set initial price : %.2f",
-                                    pos, ZSTD_fCost(sequencePrice));
+                                    pos, ZSTD_fCost((int)sequencePrice));
                         opt[pos].mlen = pos;
-                        opt[pos].off = offcode;
+                        opt[pos].off = offBase;
                         opt[pos].litlen = litlen;
                         opt[pos].price = (int)sequencePrice;
                 }   }
@@ -1230,7 +1257,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
                     U32 const startML = (matchNb>0) ? matches[matchNb-1].len+1 : minMatch;
                     U32 mlen;
 
-                    DEBUGLOG(7, "testing match %u => offCode=%4u, mlen=%2u, llen=%2u",
+                    DEBUGLOG(7, "testing match %u => offBase=%4u, mlen=%2u, llen=%2u",
                                 matchNb, matches[matchNb].off, lastML, litlen);
 
                     for (mlen = lastML; mlen >= startML; mlen--) {  /* scan downward */
@@ -1296,7 +1323,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
                 for (storePos=storeStart; storePos <= storeEnd; storePos++) {
                     U32 const llen = opt[storePos].litlen;
                     U32 const mlen = opt[storePos].mlen;
-                    U32 const offCode = opt[storePos].off;
+                    U32 const offBase = opt[storePos].off;
                     U32 const advance = llen + mlen;
                     DEBUGLOG(6, "considering seq starting at %zi, llen=%u, mlen=%u",
                                 anchor - istart, (unsigned)llen, (unsigned)mlen);
@@ -1308,8 +1335,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
                     }
 
                     assert(anchor + llen <= iend);
-                    ZSTD_updateStats(optStatePtr, llen, anchor, offCode, mlen);
-                    ZSTD_storeSeq(seqStore, llen, anchor, iend, offCode, mlen);
+                    ZSTD_updateStats(optStatePtr, llen, anchor, offBase, mlen);
+                    ZSTD_storeSeq(seqStore, llen, anchor, iend, offBase, mlen);
                     anchor += advance;
                     ip = anchor;
             }   }
@@ -1349,7 +1376,7 @@ size_t ZSTD_compressBlock_btopt(
 /* ZSTD_initStats_ultra():
  * make a first compression pass, just to seed stats with more accurate starting values.
  * only works on first block, with no dictionary and no ldm.
- * this function cannot error, hence its contract must be respected.
+ * this function cannot error out, its narrow contract must be respected.
  */
 static void
 ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
@@ -1368,7 +1395,7 @@ ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
 
     ZSTD_compressBlock_opt2(ms, seqStore, tmpRep, src, srcSize, ZSTD_noDict);   /* generate stats into ms->opt*/
 
-    /* invalidate first scan from history */
+    /* invalidate first scan from history, only keep entropy stats */
     ZSTD_resetSeqStore(seqStore);
     ms->window.base -= srcSize;
     ms->window.dictLimit += (U32)srcSize;
@@ -1392,20 +1419,20 @@ size_t ZSTD_compressBlock_btultra2(
     U32 const curr = (U32)((const BYTE*)src - ms->window.base);
     DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize);
 
-    /* 2-pass strategy:
+    /* 2-passes strategy:
      * this strategy makes a first pass over first block to collect statistics
-     * and seed next round's statistics with it.
-     * After 1st pass, function forgets everything, and starts a new block.
+     * in order to seed next round's statistics with it.
+     * After 1st pass, function forgets history, and starts a new block.
      * Consequently, this can only work if no data has been previously loaded in tables,
      * aka, no dictionary, no prefix, no ldm preprocessing.
      * The compression ratio gain is generally small (~0.5% on first block),
-     * the cost is 2x cpu time on first block. */
+    ** the cost is 2x cpu time on first block. */
     assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
     if ( (ms->opt.litLengthSum==0)   /* first block */
       && (seqStore->sequences == seqStore->sequencesStart)  /* no ldm */
       && (ms->window.dictLimit == ms->window.lowLimit)   /* no dictionary */
-      && (curr == ms->window.dictLimit)   /* start of frame, nothing already loaded nor skipped */
-      && (srcSize > ZSTD_PREDEF_THRESHOLD)
+      && (curr == ms->window.dictLimit)    /* start of frame, nothing already loaded nor skipped */
+      && (srcSize > ZSTD_PREDEF_THRESHOLD) /* input large enough to not employ default stats */
       ) {
         ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize);
     }
diff --git a/lib/zstd/compress/zstd_opt.h b/lib/zstd/compress/zstd_opt.h
index 22b862858ba7..faa73ff4b03d 100644
--- a/lib/zstd/compress/zstd_opt.h
+++ b/lib/zstd/compress/zstd_opt.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/zstd/decompress/huf_decompress.c b/lib/zstd/decompress/huf_decompress.c
index 60958afebc41..db670d71fdab 100644
--- a/lib/zstd/decompress/huf_decompress.c
+++ b/lib/zstd/decompress/huf_decompress.c
@@ -1,7 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /* ******************************************************************
  * huff0 huffman decoder,
  * part of Finite State Entropy library
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  *  You can contact the author at :
  *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -19,10 +20,10 @@
 #include "../common/compiler.h"
 #include "../common/bitstream.h"  /* BIT_* */
 #include "../common/fse.h"        /* to compress headers */
-#define HUF_STATIC_LINKING_ONLY
 #include "../common/huf.h"
 #include "../common/error_private.h"
 #include "../common/zstd_internal.h"
+#include "../common/bits.h"       /* ZSTD_highbit32, ZSTD_countTrailingZeros64 */
 
 /* **************************************************************
 *  Constants
@@ -34,6 +35,12 @@
 *  Macros
 ****************************************************************/
 
+#ifdef HUF_DISABLE_FAST_DECODE
+# define HUF_ENABLE_FAST_DECODE 0
+#else
+# define HUF_ENABLE_FAST_DECODE 1
+#endif
+
 /* These two optional macros force the use one way or another of the two
  * Huffman decompression implementations. You can't force in both directions
  * at the same time.
@@ -43,27 +50,25 @@
 #error "Cannot force the use of the X1 and X2 decoders at the same time!"
 #endif
 
-#if ZSTD_ENABLE_ASM_X86_64_BMI2 && DYNAMIC_BMI2
-# define HUF_ASM_X86_64_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE
+/* When DYNAMIC_BMI2 is enabled, fast decoders are only called when bmi2 is
+ * supported at runtime, so we can add the BMI2 target attribute.
+ * When it is disabled, we will still get BMI2 if it is enabled statically.
+ */
+#if DYNAMIC_BMI2
+# define HUF_FAST_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE
 #else
-# define HUF_ASM_X86_64_BMI2_ATTRS
+# define HUF_FAST_BMI2_ATTRS
 #endif
 
 #define HUF_EXTERN_C
 #define HUF_ASM_DECL HUF_EXTERN_C
 
-#if DYNAMIC_BMI2 || (ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__))
+#if DYNAMIC_BMI2
 # define HUF_NEED_BMI2_FUNCTION 1
 #else
 # define HUF_NEED_BMI2_FUNCTION 0
 #endif
 
-#if !(ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__))
-# define HUF_NEED_DEFAULT_FUNCTION 1
-#else
-# define HUF_NEED_DEFAULT_FUNCTION 0
-#endif
-
 /* **************************************************************
 *  Error Management
 ****************************************************************/
@@ -80,6 +85,11 @@
 /* **************************************************************
 *  BMI2 Variant Wrappers
 ****************************************************************/
+typedef size_t (*HUF_DecompressUsingDTableFn)(void *dst, size_t dstSize,
+                                              const void *cSrc,
+                                              size_t cSrcSize,
+                                              const HUF_DTable *DTable);
+
 #if DYNAMIC_BMI2
 
 #define HUF_DGEN(fn)                                                        \
@@ -101,9 +111,9 @@
     }                                                                       \
                                                                             \
     static size_t fn(void* dst, size_t dstSize, void const* cSrc,           \
-                     size_t cSrcSize, HUF_DTable const* DTable, int bmi2)   \
+                     size_t cSrcSize, HUF_DTable const* DTable, int flags)  \
     {                                                                       \
-        if (bmi2) {                                                         \
+        if (flags & HUF_flags_bmi2) {                                       \
             return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);         \
         }                                                                   \
         return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable);          \
@@ -113,9 +123,9 @@
 
 #define HUF_DGEN(fn)                                                        \
     static size_t fn(void* dst, size_t dstSize, void const* cSrc,           \
-                     size_t cSrcSize, HUF_DTable const* DTable, int bmi2)   \
+                     size_t cSrcSize, HUF_DTable const* DTable, int flags)  \
     {                                                                       \
-        (void)bmi2;                                                         \
+        (void)flags;                                                        \
         return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable);             \
     }
 
@@ -134,15 +144,28 @@ static DTableDesc HUF_getDTableDesc(const HUF_DTable* table)
     return dtd;
 }
 
-#if ZSTD_ENABLE_ASM_X86_64_BMI2
-
-static size_t HUF_initDStream(BYTE const* ip) {
+static size_t HUF_initFastDStream(BYTE const* ip) {
     BYTE const lastByte = ip[7];
-    size_t const bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
+    size_t const bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0;
     size_t const value = MEM_readLEST(ip) | 1;
     assert(bitsConsumed <= 8);
+    assert(sizeof(size_t) == 8);
     return value << bitsConsumed;
 }
+
+
+/*
+ * The input/output arguments to the Huffman fast decoding loop:
+ *
+ * ip [in/out] - The input pointers, must be updated to reflect what is consumed.
+ * op [in/out] - The output pointers, must be updated to reflect what is written.
+ * bits [in/out] - The bitstream containers, must be updated to reflect the current state.
+ * dt [in] - The decoding table.
+ * ilimit [in] - The input limit, stop when any input pointer is below ilimit.
+ * oend [in] - The end of the output stream. op[3] must not cross oend.
+ * iend [in] - The end of each input stream. ip[i] may cross iend[i],
+ *             as long as it is above ilimit, but that indicates corruption.
+ */
 typedef struct {
     BYTE const* ip[4];
     BYTE* op[4];
@@ -151,15 +174,17 @@ typedef struct {
     BYTE const* ilimit;
     BYTE* oend;
     BYTE const* iend[4];
-} HUF_DecompressAsmArgs;
+} HUF_DecompressFastArgs;
+
+typedef void (*HUF_DecompressFastLoopFn)(HUF_DecompressFastArgs*);
 
 /*
- * Initializes args for the asm decoding loop.
- * @returns 0 on success
- *          1 if the fallback implementation should be used.
+ * Initializes args for the fast decoding loop.
+ * @returns 1 on success
+ *          0 if the fallback implementation should be used.
  *          Or an error code on failure.
  */
-static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable)
+static size_t HUF_DecompressFastArgs_init(HUF_DecompressFastArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable)
 {
     void const* dt = DTable + 1;
     U32 const dtLog = HUF_getDTableDesc(DTable).tableLog;
@@ -168,9 +193,11 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst,
 
     BYTE* const oend = (BYTE*)dst + dstSize;
 
-    /* The following condition is false on x32 platform,
-     * but HUF_asm is not compatible with this ABI */
-    if (!(MEM_isLittleEndian() && !MEM_32bits())) return 1;
+    /* The fast decoding loop assumes 64-bit little-endian.
+     * This condition is false on x32.
+     */
+    if (!MEM_isLittleEndian() || MEM_32bits())
+        return 0;
 
     /* strict minimum : jump table + 1 byte per stream */
     if (srcSize < 10)
@@ -181,7 +208,7 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst,
      * On small inputs we don't have enough data to trigger the fast loop, so use the old decoder.
      */
     if (dtLog != HUF_DECODER_FAST_TABLELOG)
-        return 1;
+        return 0;
 
     /* Read the jump table. */
     {
@@ -195,13 +222,13 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst,
         args->iend[2] = args->iend[1] + length2;
         args->iend[3] = args->iend[2] + length3;
 
-        /* HUF_initDStream() requires this, and this small of an input
+        /* HUF_initFastDStream() requires this, and this small of an input
          * won't benefit from the ASM loop anyways.
          * length1 must be >= 16 so that ip[0] >= ilimit before the loop
          * starts.
          */
         if (length1 < 16 || length2 < 8 || length3 < 8 || length4 < 8)
-            return 1;
+            return 0;
         if (length4 > srcSize) return ERROR(corruption_detected);   /* overflow */
     }
     /* ip[] contains the position that is currently loaded into bits[]. */
@@ -218,7 +245,7 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst,
 
     /* No point to call the ASM loop for tiny outputs. */
     if (args->op[3] >= oend)
-        return 1;
+        return 0;
 
     /* bits[] is the bit container.
         * It is read from the MSB down to the LSB.
@@ -227,10 +254,10 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst,
         * set, so that CountTrailingZeros(bits[]) can be used
         * to count how many bits we've consumed.
         */
-    args->bits[0] = HUF_initDStream(args->ip[0]);
-    args->bits[1] = HUF_initDStream(args->ip[1]);
-    args->bits[2] = HUF_initDStream(args->ip[2]);
-    args->bits[3] = HUF_initDStream(args->ip[3]);
+    args->bits[0] = HUF_initFastDStream(args->ip[0]);
+    args->bits[1] = HUF_initFastDStream(args->ip[1]);
+    args->bits[2] = HUF_initFastDStream(args->ip[2]);
+    args->bits[3] = HUF_initFastDStream(args->ip[3]);
 
     /* If ip[] >= ilimit, it is guaranteed to be safe to
         * reload bits[]. It may be beyond its section, but is
@@ -241,10 +268,10 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst,
     args->oend = oend;
     args->dt = dt;
 
-    return 0;
+    return 1;
 }
 
-static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressAsmArgs const* args, int stream, BYTE* segmentEnd)
+static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressFastArgs const* args, int stream, BYTE* segmentEnd)
 {
     /* Validate that we haven't overwritten. */
     if (args->op[stream] > segmentEnd)
@@ -258,15 +285,33 @@ static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressAsmArgs
         return ERROR(corruption_detected);
 
     /* Construct the BIT_DStream_t. */
-    bit->bitContainer = MEM_readLE64(args->ip[stream]);
-    bit->bitsConsumed = ZSTD_countTrailingZeros((size_t)args->bits[stream]);
+    assert(sizeof(size_t) == 8);
+    bit->bitContainer = MEM_readLEST(args->ip[stream]);
+    bit->bitsConsumed = ZSTD_countTrailingZeros64(args->bits[stream]);
     bit->start = (const char*)args->iend[0];
     bit->limitPtr = bit->start + sizeof(size_t);
     bit->ptr = (const char*)args->ip[stream];
 
     return 0;
 }
-#endif
+
+/* Calls X(N) for each stream 0, 1, 2, 3. */
+#define HUF_4X_FOR_EACH_STREAM(X) \
+    {                             \
+        X(0)                      \
+        X(1)                      \
+        X(2)                      \
+        X(3)                      \
+    }
+
+/* Calls X(N, var) for each stream 0, 1, 2, 3. */
+#define HUF_4X_FOR_EACH_STREAM_WITH_VAR(X, var) \
+    {                                           \
+        X(0, (var))                             \
+        X(1, (var))                             \
+        X(2, (var))                             \
+        X(3, (var))                             \
+    }
 
 
 #ifndef HUF_FORCE_DECOMPRESS_X2
@@ -283,10 +328,11 @@ typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1;   /* single-symbol decodi
 static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) {
     U64 D4;
     if (MEM_isLittleEndian()) {
-        D4 = (symbol << 8) + nbBits;
+        D4 = (U64)((symbol << 8) + nbBits);
     } else {
-        D4 = symbol + (nbBits << 8);
+        D4 = (U64)(symbol + (nbBits << 8));
     }
+    assert(D4 < (1U << 16));
     D4 *= 0x0001000100010001ULL;
     return D4;
 }
@@ -329,13 +375,7 @@ typedef struct {
         BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
 } HUF_ReadDTableX1_Workspace;
 
-
-size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize)
-{
-    return HUF_readDTableX1_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0);
-}
-
-size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2)
+size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags)
 {
     U32 tableLog = 0;
     U32 nbSymbols = 0;
@@ -350,7 +390,7 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr
     DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable));
     /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */   /* is not necessary, even though some analyzer complain ... */
 
-    iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), bmi2);
+    iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), flags);
     if (HUF_isError(iSize)) return iSize;
 
 
@@ -377,9 +417,8 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr
      * rankStart[0] is not filled because there are no entries in the table for
      * weight 0.
      */
-    {
-        int n;
-        int nextRankStart = 0;
+    {   int n;
+        U32 nextRankStart = 0;
         int const unroll = 4;
         int const nLimit = (int)nbSymbols - unroll + 1;
         for (n=0; n<(int)tableLog+1; n++) {
@@ -406,10 +445,9 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr
      * We can switch based on the length to a different inner loop which is
      * optimized for that particular case.
      */
-    {
-        U32 w;
-        int symbol=wksp->rankVal[0];
-        int rankStart=0;
+    {   U32 w;
+        int symbol = wksp->rankVal[0];
+        int rankStart = 0;
         for (w=1; w<tableLog+1; ++w) {
             int const symbolCount = wksp->rankVal[w];
             int const length = (1 << w) >> 1;
@@ -519,7 +557,7 @@ HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, cons
     while (p < pEnd)
         HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
 
-    return pEnd-pStart;
+    return (size_t)(pEnd-pStart);
 }
 
 FORCE_INLINE_TEMPLATE size_t
@@ -545,6 +583,10 @@ HUF_decompress1X1_usingDTable_internal_body(
     return dstSize;
 }
 
+/* HUF_decompress4X1_usingDTable_internal_body():
+ * Conditions :
+ * @dstSize >= 6
+ */
 FORCE_INLINE_TEMPLATE size_t
 HUF_decompress4X1_usingDTable_internal_body(
           void* dst,  size_t dstSize,
@@ -588,6 +630,7 @@ HUF_decompress4X1_usingDTable_internal_body(
 
         if (length4 > cSrcSize) return ERROR(corruption_detected);   /* overflow */
         if (opStart4 > oend) return ERROR(corruption_detected);      /* overflow */
+        if (dstSize < 6) return ERROR(corruption_detected);         /* stream 4-split doesn't work */
         CHECK_F( BIT_initDStream(&bitD1, istart1, length1) );
         CHECK_F( BIT_initDStream(&bitD2, istart2, length2) );
         CHECK_F( BIT_initDStream(&bitD3, istart3, length3) );
@@ -650,38 +693,156 @@ size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, vo
 }
 #endif
 
-#if HUF_NEED_DEFAULT_FUNCTION
 static
 size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc,
                     size_t cSrcSize, HUF_DTable const* DTable) {
     return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
 }
-#endif
 
 #if ZSTD_ENABLE_ASM_X86_64_BMI2
 
-HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args) ZSTDLIB_HIDDEN;
+HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN;
+
+#endif
+
+static HUF_FAST_BMI2_ATTRS
+void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args)
+{
+    U64 bits[4];
+    BYTE const* ip[4];
+    BYTE* op[4];
+    U16 const* const dtable = (U16 const*)args->dt;
+    BYTE* const oend = args->oend;
+    BYTE const* const ilimit = args->ilimit;
+
+    /* Copy the arguments to local variables */
+    ZSTD_memcpy(&bits, &args->bits, sizeof(bits));
+    ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip));
+    ZSTD_memcpy(&op, &args->op, sizeof(op));
+
+    assert(MEM_isLittleEndian());
+    assert(!MEM_32bits());
+
+    for (;;) {
+        BYTE* olimit;
+        int stream;
+
+        /* Assert loop preconditions */
+#ifndef NDEBUG
+        for (stream = 0; stream < 4; ++stream) {
+            assert(op[stream] <= (stream == 3 ? oend : op[stream + 1]));
+            assert(ip[stream] >= ilimit);
+        }
+#endif
+        /* Compute olimit */
+        {
+            /* Each iteration produces 5 output symbols per stream */
+            size_t const oiters = (size_t)(oend - op[3]) / 5;
+            /* Each iteration consumes up to 11 bits * 5 = 55 bits < 7 bytes
+             * per stream.
+             */
+            size_t const iiters = (size_t)(ip[0] - ilimit) / 7;
+            /* We can safely run iters iterations before running bounds checks */
+            size_t const iters = MIN(oiters, iiters);
+            size_t const symbols = iters * 5;
+
+            /* We can simply check that op[3] < olimit, instead of checking all
+             * of our bounds, since we can't hit the other bounds until we've run
+             * iters iterations, which only happens when op[3] == olimit.
+             */
+            olimit = op[3] + symbols;
+
+            /* Exit fast decoding loop once we get close to the end. */
+            if (op[3] + 20 > olimit)
+                break;
+
+            /* Exit the decoding loop if any input pointer has crossed the
+             * previous one. This indicates corruption, and a precondition
+             * to our loop is that ip[i] >= ip[0].
+             */
+            for (stream = 1; stream < 4; ++stream) {
+                if (ip[stream] < ip[stream - 1])
+                    goto _out;
+            }
+        }
+
+#ifndef NDEBUG
+        for (stream = 1; stream < 4; ++stream) {
+            assert(ip[stream] >= ip[stream - 1]);
+        }
+#endif
+
+#define HUF_4X1_DECODE_SYMBOL(_stream, _symbol)                 \
+    {                                                           \
+        int const index = (int)(bits[(_stream)] >> 53);         \
+        int const entry = (int)dtable[index];                   \
+        bits[(_stream)] <<= (entry & 0x3F);                     \
+        op[(_stream)][(_symbol)] = (BYTE)((entry >> 8) & 0xFF); \
+    }
+
+#define HUF_4X1_RELOAD_STREAM(_stream)                              \
+    {                                                               \
+        int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \
+        int const nbBits = ctz & 7;                                 \
+        int const nbBytes = ctz >> 3;                               \
+        op[(_stream)] += 5;                                         \
+        ip[(_stream)] -= nbBytes;                                   \
+        bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1;            \
+        bits[(_stream)] <<= nbBits;                                 \
+    }
+
+        /* Manually unroll the loop because compilers don't consistently
+         * unroll the inner loops, which destroys performance.
+         */
+        do {
+            /* Decode 5 symbols in each of the 4 streams */
+            HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 0)
+            HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 1)
+            HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 2)
+            HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 3)
+            HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 4)
+
+            /* Reload each of the 4 the bitstreams */
+            HUF_4X_FOR_EACH_STREAM(HUF_4X1_RELOAD_STREAM)
+        } while (op[3] < olimit);
+
+#undef HUF_4X1_DECODE_SYMBOL
+#undef HUF_4X1_RELOAD_STREAM
+    }
+
+_out:
 
-static HUF_ASM_X86_64_BMI2_ATTRS
+    /* Save the final values of each of the state variables back to args. */
+    ZSTD_memcpy(&args->bits, &bits, sizeof(bits));
+    ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip));
+    ZSTD_memcpy(&args->op, &op, sizeof(op));
+}
+
+/*
+ * @returns @p dstSize on success (>= 6)
+ *          0 if the fallback implementation should be used
+ *          An error if an error occurred
+ */
+static HUF_FAST_BMI2_ATTRS
 size_t
-HUF_decompress4X1_usingDTable_internal_bmi2_asm(
+HUF_decompress4X1_usingDTable_internal_fast(
           void* dst,  size_t dstSize,
     const void* cSrc, size_t cSrcSize,
-    const HUF_DTable* DTable)
+    const HUF_DTable* DTable,
+    HUF_DecompressFastLoopFn loopFn)
 {
     void const* dt = DTable + 1;
     const BYTE* const iend = (const BYTE*)cSrc + 6;
     BYTE* const oend = (BYTE*)dst + dstSize;
-    HUF_DecompressAsmArgs args;
-    {
-        size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
-        FORWARD_IF_ERROR(ret, "Failed to init asm args");
-        if (ret != 0)
-            return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
+    HUF_DecompressFastArgs args;
+    {   size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
+        FORWARD_IF_ERROR(ret, "Failed to init fast loop args");
+        if (ret == 0)
+            return 0;
     }
 
     assert(args.ip[0] >= args.ilimit);
-    HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop(&args);
+    loopFn(&args);
 
     /* Our loop guarantees that ip[] >= ilimit and that we haven't
     * overwritten any op[].
@@ -694,8 +855,7 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm(
     (void)iend;
 
     /* finish bit streams one by one. */
-    {
-        size_t const segmentSize = (dstSize+3) / 4;
+    {   size_t const segmentSize = (dstSize+3) / 4;
         BYTE* segmentEnd = (BYTE*)dst;
         int i;
         for (i = 0; i < 4; ++i) {
@@ -712,97 +872,59 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm(
     }
 
     /* decoded size */
+    assert(dstSize != 0);
     return dstSize;
 }
-#endif /* ZSTD_ENABLE_ASM_X86_64_BMI2 */
-
-typedef size_t (*HUF_decompress_usingDTable_t)(void *dst, size_t dstSize,
-                                               const void *cSrc,
-                                               size_t cSrcSize,
-                                               const HUF_DTable *DTable);
 
 HUF_DGEN(HUF_decompress1X1_usingDTable_internal)
 
 static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc,
-                    size_t cSrcSize, HUF_DTable const* DTable, int bmi2)
+                    size_t cSrcSize, HUF_DTable const* DTable, int flags)
 {
+    HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X1_usingDTable_internal_default;
+    HUF_DecompressFastLoopFn loopFn = HUF_decompress4X1_usingDTable_internal_fast_c_loop;
+
 #if DYNAMIC_BMI2
-    if (bmi2) {
+    if (flags & HUF_flags_bmi2) {
+        fallbackFn = HUF_decompress4X1_usingDTable_internal_bmi2;
 # if ZSTD_ENABLE_ASM_X86_64_BMI2
-        return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
-# else
-        return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
+        if (!(flags & HUF_flags_disableAsm)) {
+            loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop;
+        }
 # endif
+    } else {
+        return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
     }
-#else
-    (void)bmi2;
 #endif
 
 #if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
-    return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
-#else
-    return HUF_decompress4X1_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable);
+    if (!(flags & HUF_flags_disableAsm)) {
+        loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop;
+    }
 #endif
-}
-
-
-size_t HUF_decompress1X1_usingDTable(
-          void* dst,  size_t dstSize,
-    const void* cSrc, size_t cSrcSize,
-    const HUF_DTable* DTable)
-{
-    DTableDesc dtd = HUF_getDTableDesc(DTable);
-    if (dtd.tableType != 0) return ERROR(GENERIC);
-    return HUF_decompress1X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-}
-
-size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize,
-                                   const void* cSrc, size_t cSrcSize,
-                                   void* workSpace, size_t wkspSize)
-{
-    const BYTE* ip = (const BYTE*) cSrc;
 
-    size_t const hSize = HUF_readDTableX1_wksp(DCtx, cSrc, cSrcSize, workSpace, wkspSize);
-    if (HUF_isError(hSize)) return hSize;
-    if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
-    ip += hSize; cSrcSize -= hSize;
-
-    return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0);
-}
-
-
-size_t HUF_decompress4X1_usingDTable(
-          void* dst,  size_t dstSize,
-    const void* cSrc, size_t cSrcSize,
-    const HUF_DTable* DTable)
-{
-    DTableDesc dtd = HUF_getDTableDesc(DTable);
-    if (dtd.tableType != 0) return ERROR(GENERIC);
-    return HUF_decompress4X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
+    if (HUF_ENABLE_FAST_DECODE && !(flags & HUF_flags_disableFast)) {
+        size_t const ret = HUF_decompress4X1_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn);
+        if (ret != 0)
+            return ret;
+    }
+    return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
 }
 
-static size_t HUF_decompress4X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize,
+static size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
                                    const void* cSrc, size_t cSrcSize,
-                                   void* workSpace, size_t wkspSize, int bmi2)
+                                   void* workSpace, size_t wkspSize, int flags)
 {
     const BYTE* ip = (const BYTE*) cSrc;
 
-    size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
+    size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags);
     if (HUF_isError(hSize)) return hSize;
     if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
     ip += hSize; cSrcSize -= hSize;
 
-    return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2);
-}
-
-size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
-                                   const void* cSrc, size_t cSrcSize,
-                                   void* workSpace, size_t wkspSize)
-{
-    return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, 0);
+    return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags);
 }
 
-
 #endif /* HUF_FORCE_DECOMPRESS_X2 */
 
 
@@ -985,7 +1107,7 @@ static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32
 
 static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog,
                            const sortedSymbol_t* sortedList,
-                           const U32* rankStart, rankValCol_t *rankValOrigin, const U32 maxWeight,
+                           const U32* rankStart, rankValCol_t* rankValOrigin, const U32 maxWeight,
                            const U32 nbBitsBaseline)
 {
     U32* const rankVal = rankValOrigin[0];
@@ -1040,14 +1162,7 @@ typedef struct {
 
 size_t HUF_readDTableX2_wksp(HUF_DTable* DTable,
                        const void* src, size_t srcSize,
-                             void* workSpace, size_t wkspSize)
-{
-    return HUF_readDTableX2_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0);
-}
-
-size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable,
-                       const void* src, size_t srcSize,
-                             void* workSpace, size_t wkspSize, int bmi2)
+                             void* workSpace, size_t wkspSize, int flags)
 {
     U32 tableLog, maxW, nbSymbols;
     DTableDesc dtd = HUF_getDTableDesc(DTable);
@@ -1069,7 +1184,7 @@ size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable,
     if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
     /* ZSTD_memset(weightList, 0, sizeof(weightList)); */  /* is not necessary, even though some analyzer complain ... */
 
-    iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), bmi2);
+    iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), flags);
     if (HUF_isError(iSize)) return iSize;
 
     /* check result */
@@ -1240,6 +1355,11 @@ HUF_decompress1X2_usingDTable_internal_body(
     /* decoded size */
     return dstSize;
 }
+
+/* HUF_decompress4X2_usingDTable_internal_body():
+ * Conditions:
+ * @dstSize >= 6
+ */
 FORCE_INLINE_TEMPLATE size_t
 HUF_decompress4X2_usingDTable_internal_body(
           void* dst,  size_t dstSize,
@@ -1280,8 +1400,9 @@ HUF_decompress4X2_usingDTable_internal_body(
         DTableDesc const dtd = HUF_getDTableDesc(DTable);
         U32 const dtLog = dtd.tableLog;
 
-        if (length4 > cSrcSize) return ERROR(corruption_detected);   /* overflow */
-        if (opStart4 > oend) return ERROR(corruption_detected);      /* overflow */
+        if (length4 > cSrcSize) return ERROR(corruption_detected);  /* overflow */
+        if (opStart4 > oend) return ERROR(corruption_detected);     /* overflow */
+        if (dstSize < 6) return ERROR(corruption_detected);         /* stream 4-split doesn't work */
         CHECK_F( BIT_initDStream(&bitD1, istart1, length1) );
         CHECK_F( BIT_initDStream(&bitD2, istart2, length2) );
         CHECK_F( BIT_initDStream(&bitD3, istart3, length3) );
@@ -1366,36 +1487,178 @@ size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, vo
 }
 #endif
 
-#if HUF_NEED_DEFAULT_FUNCTION
 static
 size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc,
                     size_t cSrcSize, HUF_DTable const* DTable) {
     return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
 }
-#endif
 
 #if ZSTD_ENABLE_ASM_X86_64_BMI2
 
-HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args) ZSTDLIB_HIDDEN;
+HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN;
+
+#endif
+
+static HUF_FAST_BMI2_ATTRS
+void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args)
+{
+    U64 bits[4];
+    BYTE const* ip[4];
+    BYTE* op[4];
+    BYTE* oend[4];
+    HUF_DEltX2 const* const dtable = (HUF_DEltX2 const*)args->dt;
+    BYTE const* const ilimit = args->ilimit;
+
+    /* Copy the arguments to local registers. */
+    ZSTD_memcpy(&bits, &args->bits, sizeof(bits));
+    ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip));
+    ZSTD_memcpy(&op, &args->op, sizeof(op));
+
+    oend[0] = op[1];
+    oend[1] = op[2];
+    oend[2] = op[3];
+    oend[3] = args->oend;
+
+    assert(MEM_isLittleEndian());
+    assert(!MEM_32bits());
+
+    for (;;) {
+        BYTE* olimit;
+        int stream;
+
+        /* Assert loop preconditions */
+#ifndef NDEBUG
+        for (stream = 0; stream < 4; ++stream) {
+            assert(op[stream] <= oend[stream]);
+            assert(ip[stream] >= ilimit);
+        }
+#endif
+        /* Compute olimit */
+        {
+            /* Each loop does 5 table lookups for each of the 4 streams.
+             * Each table lookup consumes up to 11 bits of input, and produces
+             * up to 2 bytes of output.
+             */
+            /* We can consume up to 7 bytes of input per iteration per stream.
+             * We also know that each input pointer is >= ip[0]. So we can run
+             * iters loops before running out of input.
+             */
+            size_t iters = (size_t)(ip[0] - ilimit) / 7;
+            /* Each iteration can produce up to 10 bytes of output per stream.
+             * Each output stream my advance at different rates. So take the
+             * minimum number of safe iterations among all the output streams.
+             */
+            for (stream = 0; stream < 4; ++stream) {
+                size_t const oiters = (size_t)(oend[stream] - op[stream]) / 10;
+                iters = MIN(iters, oiters);
+            }
+
+            /* Each iteration produces at least 5 output symbols. So until
+             * op[3] crosses olimit, we know we haven't executed iters
+             * iterations yet. This saves us maintaining an iters counter,
+             * at the expense of computing the remaining # of iterations
+             * more frequently.
+             */
+            olimit = op[3] + (iters * 5);
+
+            /* Exit the fast decoding loop if we are too close to the end. */
+            if (op[3] + 10 > olimit)
+                break;
+
+            /* Exit the decoding loop if any input pointer has crossed the
+             * previous one. This indicates corruption, and a precondition
+             * to our loop is that ip[i] >= ip[0].
+             */
+            for (stream = 1; stream < 4; ++stream) {
+                if (ip[stream] < ip[stream - 1])
+                    goto _out;
+            }
+        }
+
+#ifndef NDEBUG
+        for (stream = 1; stream < 4; ++stream) {
+            assert(ip[stream] >= ip[stream - 1]);
+        }
+#endif
+
+#define HUF_4X2_DECODE_SYMBOL(_stream, _decode3)        \
+    if ((_decode3) || (_stream) != 3) {                 \
+        int const index = (int)(bits[(_stream)] >> 53); \
+        HUF_DEltX2 const entry = dtable[index];         \
+        MEM_write16(op[(_stream)], entry.sequence);     \
+        bits[(_stream)] <<= (entry.nbBits) & 0x3F;      \
+        op[(_stream)] += (entry.length);                \
+    }
+
+#define HUF_4X2_RELOAD_STREAM(_stream)                                  \
+    {                                                                   \
+        HUF_4X2_DECODE_SYMBOL(3, 1)                                     \
+        {                                                               \
+            int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \
+            int const nbBits = ctz & 7;                                 \
+            int const nbBytes = ctz >> 3;                               \
+            ip[(_stream)] -= nbBytes;                                   \
+            bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1;            \
+            bits[(_stream)] <<= nbBits;                                 \
+        }                                                               \
+    }
+
+        /* Manually unroll the loop because compilers don't consistently
+         * unroll the inner loops, which destroys performance.
+         */
+        do {
+            /* Decode 5 symbols from each of the first 3 streams.
+             * The final stream will be decoded during the reload phase
+             * to reduce register pressure.
+             */
+            HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0)
+            HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0)
+            HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0)
+            HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0)
+            HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0)
+
+            /* Decode one symbol from the final stream */
+            HUF_4X2_DECODE_SYMBOL(3, 1)
+
+            /* Decode 4 symbols from the final stream & reload bitstreams.
+             * The final stream is reloaded last, meaning that all 5 symbols
+             * are decoded from the final stream before it is reloaded.
+             */
+            HUF_4X_FOR_EACH_STREAM(HUF_4X2_RELOAD_STREAM)
+        } while (op[3] < olimit);
+    }
 
-static HUF_ASM_X86_64_BMI2_ATTRS size_t
-HUF_decompress4X2_usingDTable_internal_bmi2_asm(
+#undef HUF_4X2_DECODE_SYMBOL
+#undef HUF_4X2_RELOAD_STREAM
+
+_out:
+
+    /* Save the final values of each of the state variables back to args. */
+    ZSTD_memcpy(&args->bits, &bits, sizeof(bits));
+    ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip));
+    ZSTD_memcpy(&args->op, &op, sizeof(op));
+}
+
+
+static HUF_FAST_BMI2_ATTRS size_t
+HUF_decompress4X2_usingDTable_internal_fast(
           void* dst,  size_t dstSize,
     const void* cSrc, size_t cSrcSize,
-    const HUF_DTable* DTable) {
+    const HUF_DTable* DTable,
+    HUF_DecompressFastLoopFn loopFn) {
     void const* dt = DTable + 1;
     const BYTE* const iend = (const BYTE*)cSrc + 6;
     BYTE* const oend = (BYTE*)dst + dstSize;
-    HUF_DecompressAsmArgs args;
+    HUF_DecompressFastArgs args;
     {
-        size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
+        size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
         FORWARD_IF_ERROR(ret, "Failed to init asm args");
-        if (ret != 0)
-            return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
+        if (ret == 0)
+            return 0;
     }
 
     assert(args.ip[0] >= args.ilimit);
-    HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop(&args);
+    loopFn(&args);
 
     /* note : op4 already verified within main loop */
     assert(args.ip[0] >= iend);
@@ -1426,91 +1689,72 @@ HUF_decompress4X2_usingDTable_internal_bmi2_asm(
     /* decoded size */
     return dstSize;
 }
-#endif /* ZSTD_ENABLE_ASM_X86_64_BMI2 */
 
 static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc,
-                    size_t cSrcSize, HUF_DTable const* DTable, int bmi2)
+                    size_t cSrcSize, HUF_DTable const* DTable, int flags)
 {
+    HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X2_usingDTable_internal_default;
+    HUF_DecompressFastLoopFn loopFn = HUF_decompress4X2_usingDTable_internal_fast_c_loop;
+
 #if DYNAMIC_BMI2
-    if (bmi2) {
+    if (flags & HUF_flags_bmi2) {
+        fallbackFn = HUF_decompress4X2_usingDTable_internal_bmi2;
 # if ZSTD_ENABLE_ASM_X86_64_BMI2
-        return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
-# else
-        return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);
+        if (!(flags & HUF_flags_disableAsm)) {
+            loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop;
+        }
 # endif
+    } else {
+        return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
     }
-#else
-    (void)bmi2;
 #endif
 
 #if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
-    return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable);
-#else
-    return HUF_decompress4X2_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable);
+    if (!(flags & HUF_flags_disableAsm)) {
+        loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop;
+    }
 #endif
+
+    if (HUF_ENABLE_FAST_DECODE && !(flags & HUF_flags_disableFast)) {
+        size_t const ret = HUF_decompress4X2_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn);
+        if (ret != 0)
+            return ret;
+    }
+    return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
 }
 
 HUF_DGEN(HUF_decompress1X2_usingDTable_internal)
 
-size_t HUF_decompress1X2_usingDTable(
-          void* dst,  size_t dstSize,
-    const void* cSrc, size_t cSrcSize,
-    const HUF_DTable* DTable)
-{
-    DTableDesc dtd = HUF_getDTableDesc(DTable);
-    if (dtd.tableType != 1) return ERROR(GENERIC);
-    return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-}
-
 size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize,
                                    const void* cSrc, size_t cSrcSize,
-                                   void* workSpace, size_t wkspSize)
+                                   void* workSpace, size_t wkspSize, int flags)
 {
     const BYTE* ip = (const BYTE*) cSrc;
 
     size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize,
-                                               workSpace, wkspSize);
+                                               workSpace, wkspSize, flags);
     if (HUF_isError(hSize)) return hSize;
     if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
     ip += hSize; cSrcSize -= hSize;
 
-    return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0);
+    return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, flags);
 }
 
-
-size_t HUF_decompress4X2_usingDTable(
-          void* dst,  size_t dstSize,
-    const void* cSrc, size_t cSrcSize,
-    const HUF_DTable* DTable)
-{
-    DTableDesc dtd = HUF_getDTableDesc(DTable);
-    if (dtd.tableType != 1) return ERROR(GENERIC);
-    return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-}
-
-static size_t HUF_decompress4X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize,
+static size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
                                    const void* cSrc, size_t cSrcSize,
-                                   void* workSpace, size_t wkspSize, int bmi2)
+                                   void* workSpace, size_t wkspSize, int flags)
 {
     const BYTE* ip = (const BYTE*) cSrc;
 
     size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize,
-                                         workSpace, wkspSize);
+                                         workSpace, wkspSize, flags);
     if (HUF_isError(hSize)) return hSize;
     if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
     ip += hSize; cSrcSize -= hSize;
 
-    return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2);
+    return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags);
 }
 
-size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
-                                   const void* cSrc, size_t cSrcSize,
-                                   void* workSpace, size_t wkspSize)
-{
-    return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, /* bmi2 */ 0);
-}
-
-
 #endif /* HUF_FORCE_DECOMPRESS_X1 */
 
 
@@ -1518,44 +1762,6 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
 /* Universal decompression selectors */
 /* ***********************************/
 
-size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize,
-                                    const void* cSrc, size_t cSrcSize,
-                                    const HUF_DTable* DTable)
-{
-    DTableDesc const dtd = HUF_getDTableDesc(DTable);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
-    (void)dtd;
-    assert(dtd.tableType == 0);
-    return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
-    (void)dtd;
-    assert(dtd.tableType == 1);
-    return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#else
-    return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) :
-                           HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#endif
-}
-
-size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize,
-                                    const void* cSrc, size_t cSrcSize,
-                                    const HUF_DTable* DTable)
-{
-    DTableDesc const dtd = HUF_getDTableDesc(DTable);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
-    (void)dtd;
-    assert(dtd.tableType == 0);
-    return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
-    (void)dtd;
-    assert(dtd.tableType == 1);
-    return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#else
-    return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) :
-                           HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
-#endif
-}
-
 
 #if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2)
 typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t;
@@ -1610,36 +1816,9 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
 #endif
 }
 
-
-size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst,
-                                     size_t dstSize, const void* cSrc,
-                                     size_t cSrcSize, void* workSpace,
-                                     size_t wkspSize)
-{
-    /* validation checks */
-    if (dstSize == 0) return ERROR(dstSize_tooSmall);
-    if (cSrcSize == 0) return ERROR(corruption_detected);
-
-    {   U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
-#if defined(HUF_FORCE_DECOMPRESS_X1)
-        (void)algoNb;
-        assert(algoNb == 0);
-        return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
-#elif defined(HUF_FORCE_DECOMPRESS_X2)
-        (void)algoNb;
-        assert(algoNb == 1);
-        return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
-#else
-        return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
-                            cSrcSize, workSpace, wkspSize):
-                        HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
-#endif
-    }
-}
-
 size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
                                   const void* cSrc, size_t cSrcSize,
-                                  void* workSpace, size_t wkspSize)
+                                  void* workSpace, size_t wkspSize, int flags)
 {
     /* validation checks */
     if (dstSize == 0) return ERROR(dstSize_tooSmall);
@@ -1652,71 +1831,71 @@ size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
         (void)algoNb;
         assert(algoNb == 0);
         return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
-                                cSrcSize, workSpace, wkspSize);
+                                cSrcSize, workSpace, wkspSize, flags);
 #elif defined(HUF_FORCE_DECOMPRESS_X2)
         (void)algoNb;
         assert(algoNb == 1);
         return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
-                                cSrcSize, workSpace, wkspSize);
+                                cSrcSize, workSpace, wkspSize, flags);
 #else
         return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
-                                cSrcSize, workSpace, wkspSize):
+                                cSrcSize, workSpace, wkspSize, flags):
                         HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
-                                cSrcSize, workSpace, wkspSize);
+                                cSrcSize, workSpace, wkspSize, flags);
 #endif
     }
 }
 
 
-size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2)
+size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags)
 {
     DTableDesc const dtd = HUF_getDTableDesc(DTable);
 #if defined(HUF_FORCE_DECOMPRESS_X1)
     (void)dtd;
     assert(dtd.tableType == 0);
-    return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
+    return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
 #elif defined(HUF_FORCE_DECOMPRESS_X2)
     (void)dtd;
     assert(dtd.tableType == 1);
-    return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
+    return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
 #else
-    return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) :
-                           HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
+    return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) :
+                           HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
 #endif
 }
 
 #ifndef HUF_FORCE_DECOMPRESS_X2
-size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2)
+size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags)
 {
     const BYTE* ip = (const BYTE*) cSrc;
 
-    size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
+    size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags);
     if (HUF_isError(hSize)) return hSize;
     if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
     ip += hSize; cSrcSize -= hSize;
 
-    return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2);
+    return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags);
 }
 #endif
 
-size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2)
+size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags)
 {
     DTableDesc const dtd = HUF_getDTableDesc(DTable);
 #if defined(HUF_FORCE_DECOMPRESS_X1)
     (void)dtd;
     assert(dtd.tableType == 0);
-    return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
+    return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
 #elif defined(HUF_FORCE_DECOMPRESS_X2)
     (void)dtd;
     assert(dtd.tableType == 1);
-    return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
+    return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
 #else
-    return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) :
-                           HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
+    return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) :
+                           HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
 #endif
 }
 
-size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2)
+size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags)
 {
     /* validation checks */
     if (dstSize == 0) return ERROR(dstSize_tooSmall);
@@ -1726,15 +1905,14 @@ size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t ds
 #if defined(HUF_FORCE_DECOMPRESS_X1)
         (void)algoNb;
         assert(algoNb == 0);
-        return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
+        return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags);
 #elif defined(HUF_FORCE_DECOMPRESS_X2)
         (void)algoNb;
         assert(algoNb == 1);
-        return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
+        return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags);
 #else
-        return algoNb ? HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2) :
-                        HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
+        return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags) :
+                        HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags);
 #endif
     }
 }
-
diff --git a/lib/zstd/decompress/zstd_ddict.c b/lib/zstd/decompress/zstd_ddict.c
index dbbc7919de53..30ef65e1ab5c 100644
--- a/lib/zstd/decompress/zstd_ddict.c
+++ b/lib/zstd/decompress/zstd_ddict.c
@@ -1,5 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -14,12 +15,12 @@
 /*-*******************************************************
 *  Dependencies
 *********************************************************/
+#include "../common/allocations.h"  /* ZSTD_customMalloc, ZSTD_customFree */
 #include "../common/zstd_deps.h"   /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
 #include "../common/cpu.h"         /* bmi2 */
 #include "../common/mem.h"         /* low level memory routines */
 #define FSE_STATIC_LINKING_ONLY
 #include "../common/fse.h"
-#define HUF_STATIC_LINKING_ONLY
 #include "../common/huf.h"
 #include "zstd_decompress_internal.h"
 #include "zstd_ddict.h"
@@ -131,7 +132,7 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
         ZSTD_memcpy(internalBuffer, dict, dictSize);
     }
     ddict->dictSize = dictSize;
-    ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);  /* cover both little and big endian */
+    ddict->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001);  /* cover both little and big endian */
 
     /* parse dictionary content */
     FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , "");
@@ -237,5 +238,5 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
 unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
 {
     if (ddict==NULL) return 0;
-    return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
+    return ddict->dictID;
 }
diff --git a/lib/zstd/decompress/zstd_ddict.h b/lib/zstd/decompress/zstd_ddict.h
index 8c1a79d666f8..de459a0dacd1 100644
--- a/lib/zstd/decompress/zstd_ddict.h
+++ b/lib/zstd/decompress/zstd_ddict.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/zstd/decompress/zstd_decompress.c b/lib/zstd/decompress/zstd_decompress.c
index 6b3177c94711..03dbdf39109f 100644
--- a/lib/zstd/decompress/zstd_decompress.c
+++ b/lib/zstd/decompress/zstd_decompress.c
@@ -1,5 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -52,17 +53,18 @@
 /*-*******************************************************
 *  Dependencies
 *********************************************************/
+#include "../common/allocations.h"  /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */
 #include "../common/zstd_deps.h"   /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
 #include "../common/mem.h"         /* low level memory routines */
 #define FSE_STATIC_LINKING_ONLY
 #include "../common/fse.h"
-#define HUF_STATIC_LINKING_ONLY
 #include "../common/huf.h"
 #include <linux/xxhash.h> /* xxh64_reset, xxh64_update, xxh64_digest, XXH64 */
 #include "../common/zstd_internal.h"  /* blockProperties_t */
 #include "zstd_decompress_internal.h"   /* ZSTD_DCtx */
 #include "zstd_ddict.h"  /* ZSTD_DDictDictContent */
 #include "zstd_decompress_block.h"   /* ZSTD_decompressBlock_internal */
+#include "../common/bits.h"  /* ZSTD_highbit32 */
 
 
 
@@ -72,11 +74,11 @@
  *************************************/
 
 #define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4
-#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3   /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float.
-                                                     * Currently, that means a 0.75 load factor.
-                                                     * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded
-                                                     * the load factor of the ddict hash set.
-                                                     */
+#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3  /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float.
+                                                    * Currently, that means a 0.75 load factor.
+                                                    * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded
+                                                    * the load factor of the ddict hash set.
+                                                    */
 
 #define DDICT_HASHSET_TABLE_BASE_SIZE 64
 #define DDICT_HASHSET_RESIZE_FACTOR 2
@@ -237,6 +239,7 @@ static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx)
     dctx->outBufferMode = ZSTD_bm_buffered;
     dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum;
     dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict;
+    dctx->disableHufAsm = 0;
 }
 
 static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx)
@@ -421,16 +424,40 @@ size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
  *  note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless
  * @return : 0, `zfhPtr` is correctly filled,
  *          >0, `srcSize` is too small, value is wanted `srcSize` amount,
- *           or an error code, which can be tested using ZSTD_isError() */
+**           or an error code, which can be tested using ZSTD_isError() */
 size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format)
 {
     const BYTE* ip = (const BYTE*)src;
     size_t const minInputSize = ZSTD_startingInputLength(format);
 
-    ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr));   /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */
-    if (srcSize < minInputSize) return minInputSize;
-    RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter");
+    DEBUGLOG(5, "ZSTD_getFrameHeader_advanced: minInputSize = %zu, srcSize = %zu", minInputSize, srcSize);
+
+    if (srcSize > 0) {
+        /* note : technically could be considered an assert(), since it's an invalid entry */
+        RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter : src==NULL, but srcSize>0");
+    }
+    if (srcSize < minInputSize) {
+        if (srcSize > 0 && format != ZSTD_f_zstd1_magicless) {
+            /* when receiving less than @minInputSize bytes,
+             * control these bytes at least correspond to a supported magic number
+             * in order to error out early if they don't.
+            **/
+            size_t const toCopy = MIN(4, srcSize);
+            unsigned char hbuf[4]; MEM_writeLE32(hbuf, ZSTD_MAGICNUMBER);
+            assert(src != NULL);
+            ZSTD_memcpy(hbuf, src, toCopy);
+            if ( MEM_readLE32(hbuf) != ZSTD_MAGICNUMBER ) {
+                /* not a zstd frame : let's check if it's a skippable frame */
+                MEM_writeLE32(hbuf, ZSTD_MAGIC_SKIPPABLE_START);
+                ZSTD_memcpy(hbuf, src, toCopy);
+                if ((MEM_readLE32(hbuf) & ZSTD_MAGIC_SKIPPABLE_MASK) != ZSTD_MAGIC_SKIPPABLE_START) {
+                    RETURN_ERROR(prefix_unknown,
+                                "first bytes don't correspond to any supported magic number");
+        }   }   }
+        return minInputSize;
+    }
 
+    ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr));   /* not strictly necessary, but static analyzers may not understand that zfhPtr will be read only if return value is zero, since they are 2 different signals */
     if ( (format != ZSTD_f_zstd1_magicless)
       && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) {
         if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
@@ -540,49 +567,52 @@ static size_t readSkippableFrameSize(void const* src, size_t srcSize)
     sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE);
     RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32,
                     frameParameter_unsupported, "");
-    {
-        size_t const skippableSize = skippableHeaderSize + sizeU32;
+    {   size_t const skippableSize = skippableHeaderSize + sizeU32;
         RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, "");
         return skippableSize;
     }
 }
 
 /*! ZSTD_readSkippableFrame() :
- * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer.
+ * Retrieves content of a skippable frame, and writes it to dst buffer.
  *
  * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written,
  * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START.  This can be NULL if the caller is not interested
  * in the magicVariant.
  *
- * Returns an error if destination buffer is not large enough, or if the frame is not skippable.
+ * Returns an error if destination buffer is not large enough, or if this is not a valid skippable frame.
  *
  * @return : number of bytes written or a ZSTD error.
  */
-ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant,
-                                            const void* src, size_t srcSize)
+size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity,
+                               unsigned* magicVariant,  /* optional, can be NULL */
+                         const void* src, size_t srcSize)
 {
-    U32 const magicNumber = MEM_readLE32(src);
-    size_t skippableFrameSize = readSkippableFrameSize(src, srcSize);
-    size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE;
-
-    /* check input validity */
-    RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, "");
-    RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, "");
-    RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, "");
+    RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, "");
 
-    /* deliver payload */
-    if (skippableContentSize > 0  && dst != NULL)
-        ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize);
-    if (magicVariant != NULL)
-        *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START;
-    return skippableContentSize;
+    {   U32 const magicNumber = MEM_readLE32(src);
+        size_t skippableFrameSize = readSkippableFrameSize(src, srcSize);
+        size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE;
+
+        /* check input validity */
+        RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, "");
+        RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, "");
+        RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, "");
+
+        /* deliver payload */
+        if (skippableContentSize > 0  && dst != NULL)
+            ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize);
+        if (magicVariant != NULL)
+            *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START;
+        return skippableContentSize;
+    }
 }
 
 /* ZSTD_findDecompressedSize() :
- *  compatible with legacy mode
  *  `srcSize` must be the exact length of some number of ZSTD compressed and/or
  *      skippable frames
- *  @return : decompressed size of the frames contained */
+ *  note: compatible with legacy mode
+ * @return : decompressed size of the frames contained */
 unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
 {
     unsigned long long totalDstSize = 0;
@@ -592,9 +622,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
 
         if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
             size_t const skippableSize = readSkippableFrameSize(src, srcSize);
-            if (ZSTD_isError(skippableSize)) {
-                return ZSTD_CONTENTSIZE_ERROR;
-            }
+            if (ZSTD_isError(skippableSize)) return ZSTD_CONTENTSIZE_ERROR;
             assert(skippableSize <= srcSize);
 
             src = (const BYTE *)src + skippableSize;
@@ -602,17 +630,17 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
             continue;
         }
 
-        {   unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
-            if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret;
+        {   unsigned long long const fcs = ZSTD_getFrameContentSize(src, srcSize);
+            if (fcs >= ZSTD_CONTENTSIZE_ERROR) return fcs;
 
-            /* check for overflow */
-            if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR;
-            totalDstSize += ret;
+            if (totalDstSize + fcs < totalDstSize)
+                return ZSTD_CONTENTSIZE_ERROR; /* check for overflow */
+            totalDstSize += fcs;
         }
+        /* skip to next frame */
         {   size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize);
-            if (ZSTD_isError(frameSrcSize)) {
-                return ZSTD_CONTENTSIZE_ERROR;
-            }
+            if (ZSTD_isError(frameSrcSize)) return ZSTD_CONTENTSIZE_ERROR;
+            assert(frameSrcSize <= srcSize);
 
             src = (const BYTE *)src + frameSrcSize;
             srcSize -= frameSrcSize;
@@ -730,10 +758,11 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize
             ip += 4;
         }
 
+        frameSizeInfo.nbBlocks = nbBlocks;
         frameSizeInfo.compressedSize = (size_t)(ip - ipstart);
         frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN)
                                         ? zfh.frameContentSize
-                                        : nbBlocks * zfh.blockSizeMax;
+                                        : (unsigned long long)nbBlocks * zfh.blockSizeMax;
         return frameSizeInfo;
     }
 }
@@ -773,6 +802,48 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize)
     return bound;
 }
 
+size_t ZSTD_decompressionMargin(void const* src, size_t srcSize)
+{
+    size_t margin = 0;
+    unsigned maxBlockSize = 0;
+
+    /* Iterate over each frame */
+    while (srcSize > 0) {
+        ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize);
+        size_t const compressedSize = frameSizeInfo.compressedSize;
+        unsigned long long const decompressedBound = frameSizeInfo.decompressedBound;
+        ZSTD_frameHeader zfh;
+
+        FORWARD_IF_ERROR(ZSTD_getFrameHeader(&zfh, src, srcSize), "");
+        if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR)
+            return ERROR(corruption_detected);
+
+        if (zfh.frameType == ZSTD_frame) {
+            /* Add the frame header to our margin */
+            margin += zfh.headerSize;
+            /* Add the checksum to our margin */
+            margin += zfh.checksumFlag ? 4 : 0;
+            /* Add 3 bytes per block */
+            margin += 3 * frameSizeInfo.nbBlocks;
+
+            /* Compute the max block size */
+            maxBlockSize = MAX(maxBlockSize, zfh.blockSizeMax);
+        } else {
+            assert(zfh.frameType == ZSTD_skippableFrame);
+            /* Add the entire skippable frame size to our margin. */
+            margin += compressedSize;
+        }
+
+        assert(srcSize >= compressedSize);
+        src = (const BYTE*)src + compressedSize;
+        srcSize -= compressedSize;
+    }
+
+    /* Add the max block size back to the margin. */
+    margin += maxBlockSize;
+
+    return margin;
+}
 
 /*-*************************************************************
  *   Frame decoding
@@ -930,6 +1001,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
     }
     ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0);
     /* Allow caller to get size read */
+    DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %zi, consuming %zi bytes of input", op-ostart, ip - (const BYTE*)*srcPtr);
     *srcPtr = ip;
     *srcSizePtr = remainingSrcSize;
     return (size_t)(op-ostart);
@@ -955,17 +1027,18 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
     while (srcSize >= ZSTD_startingInputLength(dctx->format)) {
 
 
-        {   U32 const magicNumber = MEM_readLE32(src);
-            DEBUGLOG(4, "reading magic number %08X (expecting %08X)",
-                        (unsigned)magicNumber, ZSTD_MAGICNUMBER);
+        if (srcSize >= 4) {
+            U32 const magicNumber = MEM_readLE32(src);
+            DEBUGLOG(5, "reading magic number %08X", (unsigned)magicNumber);
             if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
+                /* skippable frame detected : skip it */
                 size_t const skippableSize = readSkippableFrameSize(src, srcSize);
-                FORWARD_IF_ERROR(skippableSize, "readSkippableFrameSize failed");
+                FORWARD_IF_ERROR(skippableSize, "invalid skippable frame");
                 assert(skippableSize <= srcSize);
 
                 src = (const BYTE *)src + skippableSize;
                 srcSize -= skippableSize;
-                continue;
+                continue; /* check next frame */
         }   }
 
         if (ddict) {
@@ -1061,8 +1134,8 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr
 size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; }
 
 /*
- * Similar to ZSTD_nextSrcSizeToDecompress(), but when a block input can be streamed,
- * we allow taking a partial block as the input. Currently only raw uncompressed blocks can
+ * Similar to ZSTD_nextSrcSizeToDecompress(), but when a block input can be streamed, we
+ * allow taking a partial block as the input. Currently only raw uncompressed blocks can
  * be streamed.
  *
  * For blocks that can be streamed, this allows us to reduce the latency until we produce
@@ -1262,7 +1335,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
 
     default:
         assert(0);   /* impossible */
-        RETURN_ERROR(GENERIC, "impossible to reach");   /* some compiler require default to do something */
+        RETURN_ERROR(GENERIC, "impossible to reach");   /* some compilers require default to do something */
     }
 }
 
@@ -1303,11 +1376,11 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
         /* in minimal huffman, we always use X1 variants */
         size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable,
                                                 dictPtr, dictEnd - dictPtr,
-                                                workspace, workspaceSize);
+                                                workspace, workspaceSize, /* flags */ 0);
 #else
         size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable,
                                                 dictPtr, (size_t)(dictEnd - dictPtr),
-                                                workspace, workspaceSize);
+                                                workspace, workspaceSize, /* flags */ 0);
 #endif
         RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, "");
         dictPtr += hSize;
@@ -1403,7 +1476,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
     dctx->prefixStart = NULL;
     dctx->virtualStart = NULL;
     dctx->dictEnd = NULL;
-    dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);  /* cover both little and big endian */
+    dctx->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001);  /* cover both little and big endian */
     dctx->litEntropy = dctx->fseEntropy = 0;
     dctx->dictID = 0;
     dctx->bType = bt_reserved;
@@ -1465,7 +1538,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
  *  This could for one of the following reasons :
  *  - The frame does not require a dictionary (most common case).
  *  - The frame was built with dictID intentionally removed.
- *    Needed dictionary is a hidden information.
+ *    Needed dictionary is a hidden piece of information.
  *    Note : this use case also happens when using a non-conformant dictionary.
  *  - `srcSize` is too small, and as a result, frame header could not be decoded.
  *    Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`.
@@ -1474,7 +1547,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
  *  ZSTD_getFrameHeader(), which will provide a more precise error code. */
 unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
 {
-    ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0 };
+    ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 };
     size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize);
     if (ZSTD_isError(hError)) return 0;
     return zfp.dictID;
@@ -1581,7 +1654,9 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
 size_t ZSTD_initDStream(ZSTD_DStream* zds)
 {
     DEBUGLOG(4, "ZSTD_initDStream");
-    return ZSTD_initDStream_usingDDict(zds, NULL);
+    FORWARD_IF_ERROR(ZSTD_DCtx_reset(zds, ZSTD_reset_session_only), "");
+    FORWARD_IF_ERROR(ZSTD_DCtx_refDDict(zds, NULL), "");
+    return ZSTD_startingInputLength(zds->format);
 }
 
 /* ZSTD_initDStream_usingDDict() :
@@ -1589,6 +1664,7 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds)
  * this function cannot fail */
 size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
 {
+    DEBUGLOG(4, "ZSTD_initDStream_usingDDict");
     FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , "");
     FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , "");
     return ZSTD_startingInputLength(dctx->format);
@@ -1599,6 +1675,7 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict)
  * this function cannot fail */
 size_t ZSTD_resetDStream(ZSTD_DStream* dctx)
 {
+    DEBUGLOG(4, "ZSTD_resetDStream");
     FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), "");
     return ZSTD_startingInputLength(dctx->format);
 }
@@ -1670,6 +1747,11 @@ ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam)
             bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict;
             bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts;
             return bounds;
+        case ZSTD_d_disableHuffmanAssembly:
+            bounds.lowerBound = 0;
+            bounds.upperBound = 1;
+            return bounds;
+
         default:;
     }
     bounds.error = ERROR(parameter_unsupported);
@@ -1710,6 +1792,9 @@ size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value
         case ZSTD_d_refMultipleDDicts:
             *value = (int)dctx->refMultipleDDicts;
             return 0;
+        case ZSTD_d_disableHuffmanAssembly:
+            *value = (int)dctx->disableHufAsm;
+            return 0;
         default:;
     }
     RETURN_ERROR(parameter_unsupported, "");
@@ -1743,6 +1828,10 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value
             }
             dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value;
             return 0;
+        case ZSTD_d_disableHuffmanAssembly:
+            CHECK_DBOUNDS(ZSTD_d_disableHuffmanAssembly, value);
+            dctx->disableHufAsm = value != 0;
+            return 0;
         default:;
     }
     RETURN_ERROR(parameter_unsupported, "");
@@ -1918,7 +2007,6 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                 if (zds->refMultipleDDicts && zds->ddictSet) {
                     ZSTD_DCtx_selectFrameDDict(zds);
                 }
-                DEBUGLOG(5, "header size : %u", (U32)hSize);
                 if (ZSTD_isError(hSize)) {
                     return hSize;   /* error */
                 }
@@ -1932,6 +2020,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                             zds->lhSize += remainingInput;
                         }
                         input->pos = input->size;
+                        /* check first few bytes */
+                        FORWARD_IF_ERROR(
+                            ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format),
+                            "First few bytes detected incorrect" );
+                        /* return hint input size */
                         return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize;   /* remaining header bytes + next block header */
                     }
                     assert(ip != NULL);
@@ -1949,8 +2042,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                     size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds));
                     if (ZSTD_isError(decompressedSize)) return decompressedSize;
                     DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()")
+                    assert(istart != NULL);
                     ip = istart + cSize;
-                    op += decompressedSize;
+                    op = op ? op + decompressedSize : op; /* can occur if frameContentSize = 0 (empty frame) */
                     zds->expected = 0;
                     zds->streamStage = zdss_init;
                     someMoreWork = 0;
@@ -2034,6 +2128,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                 }
                 if ((size_t)(iend-ip) >= neededInSize) {  /* decode directly from src */
                     FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), "");
+                    assert(ip != NULL);
                     ip += neededInSize;
                     /* Function modifies the stage so we must break */
                     break;
@@ -2048,7 +2143,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                 int const isSkipFrame = ZSTD_isSkipFrame(zds);
                 size_t loadedSize;
                 /* At this point we shouldn't be decompressing a block that we can stream. */
-                assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip));
+                assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip)));
                 if (isSkipFrame) {
                     loadedSize = MIN(toLoad, (size_t)(iend-ip));
                 } else {
@@ -2057,8 +2152,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                                     "should never happen");
                     loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip));
                 }
-                ip += loadedSize;
-                zds->inPos += loadedSize;
+                if (loadedSize != 0) {
+                    /* ip may be NULL */
+                    ip += loadedSize;
+                    zds->inPos += loadedSize;
+                }
                 if (loadedSize < toLoad) { someMoreWork = 0; break; }   /* not enough input, wait for more */
 
                 /* decode loaded input */
@@ -2068,14 +2166,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                 break;
             }
         case zdss_flush:
-            {   size_t const toFlushSize = zds->outEnd - zds->outStart;
+            {
+                size_t const toFlushSize = zds->outEnd - zds->outStart;
                 size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize);
-                op += flushedSize;
+
+                op = op ? op + flushedSize : op;
+
                 zds->outStart += flushedSize;
                 if (flushedSize == toFlushSize) {  /* flush completed */
                     zds->streamStage = zdss_read;
                     if ( (zds->outBuffSize < zds->fParams.frameContentSize)
-                      && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) {
+                        && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) {
                         DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)",
                                 (int)(zds->outBuffSize - zds->outStart),
                                 (U32)zds->fParams.blockSizeMax);
@@ -2089,7 +2190,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
 
         default:
             assert(0);    /* impossible */
-            RETURN_ERROR(GENERIC, "impossible to reach");   /* some compiler require default to do something */
+            RETURN_ERROR(GENERIC, "impossible to reach");   /* some compilers require default to do something */
     }   }
 
     /* result */
@@ -2102,8 +2203,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
     if ((ip==istart) && (op==ostart)) {  /* no forward progress */
         zds->noForwardProgress ++;
         if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) {
-            RETURN_ERROR_IF(op==oend, dstSize_tooSmall, "");
-            RETURN_ERROR_IF(ip==iend, srcSize_wrong, "");
+            RETURN_ERROR_IF(op==oend, noForwardProgress_destFull, "");
+            RETURN_ERROR_IF(ip==iend, noForwardProgress_inputEmpty, "");
             assert(0);
         }
     } else {
@@ -2140,11 +2241,17 @@ size_t ZSTD_decompressStream_simpleArgs (
                             void* dst, size_t dstCapacity, size_t* dstPos,
                       const void* src, size_t srcSize, size_t* srcPos)
 {
-    ZSTD_outBuffer output = { dst, dstCapacity, *dstPos };
-    ZSTD_inBuffer  input  = { src, srcSize, *srcPos };
-    /* ZSTD_compress_generic() will check validity of dstPos and srcPos */
-    size_t const cErr = ZSTD_decompressStream(dctx, &output, &input);
-    *dstPos = output.pos;
-    *srcPos = input.pos;
-    return cErr;
+    ZSTD_outBuffer output;
+    ZSTD_inBuffer  input;
+    output.dst = dst;
+    output.size = dstCapacity;
+    output.pos = *dstPos;
+    input.src = src;
+    input.size = srcSize;
+    input.pos = *srcPos;
+    {   size_t const cErr = ZSTD_decompressStream(dctx, &output, &input);
+        *dstPos = output.pos;
+        *srcPos = input.pos;
+        return cErr;
+    }
 }
diff --git a/lib/zstd/decompress/zstd_decompress_block.c b/lib/zstd/decompress/zstd_decompress_block.c
index c1913b8e7c89..9f5577e5bc19 100644
--- a/lib/zstd/decompress/zstd_decompress_block.c
+++ b/lib/zstd/decompress/zstd_decompress_block.c
@@ -1,5 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -20,12 +21,12 @@
 #include "../common/mem.h"         /* low level memory routines */
 #define FSE_STATIC_LINKING_ONLY
 #include "../common/fse.h"
-#define HUF_STATIC_LINKING_ONLY
 #include "../common/huf.h"
 #include "../common/zstd_internal.h"
 #include "zstd_decompress_internal.h"   /* ZSTD_DCtx */
 #include "zstd_ddict.h"  /* ZSTD_DDictDictContent */
 #include "zstd_decompress_block.h"
+#include "../common/bits.h"  /* ZSTD_highbit32 */
 
 /*_*******************************************************
 *  Macros
@@ -89,7 +90,7 @@ static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const
             dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE;
         }
         else {
-            /* initially this will be stored entirely in dst during huffman decoding, it will partially shifted to litExtraBuffer after */
+            /* initially this will be stored entirely in dst during huffman decoding, it will partially be shifted to litExtraBuffer after */
             dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize;
             dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize;
         }
@@ -134,13 +135,16 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
             ZSTD_FALLTHROUGH;
 
         case set_compressed:
-            RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3");
+            RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need up to 5 for case 3");
             {   size_t lhSize, litSize, litCSize;
                 U32 singleStream=0;
                 U32 const lhlCode = (istart[0] >> 2) & 3;
                 U32 const lhc = MEM_readLE32(istart);
                 size_t hufSuccess;
                 size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity);
+                int const flags = 0
+                    | (ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0)
+                    | (dctx->disableHufAsm ? HUF_flags_disableAsm : 0);
                 switch(lhlCode)
                 {
                 case 0: case 1: default:   /* note : default is impossible, since lhlCode into [0..3] */
@@ -165,6 +169,10 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                 }
                 RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
                 RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, "");
+                if (!singleStream)
+                    RETURN_ERROR_IF(litSize < MIN_LITERALS_FOR_4_STREAMS, literals_headerWrong,
+                        "Not enough literals (%zu) for the 4-streams mode (min %u)",
+                        litSize, MIN_LITERALS_FOR_4_STREAMS);
                 RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, "");
                 RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, "");
                 ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0);
@@ -176,13 +184,14 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
 
                 if (litEncType==set_repeat) {
                     if (singleStream) {
-                        hufSuccess = HUF_decompress1X_usingDTable_bmi2(
+                        hufSuccess = HUF_decompress1X_usingDTable(
                             dctx->litBuffer, litSize, istart+lhSize, litCSize,
-                            dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx));
+                            dctx->HUFptr, flags);
                     } else {
-                        hufSuccess = HUF_decompress4X_usingDTable_bmi2(
+                        assert(litSize >= MIN_LITERALS_FOR_4_STREAMS);
+                        hufSuccess = HUF_decompress4X_usingDTable(
                             dctx->litBuffer, litSize, istart+lhSize, litCSize,
-                            dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx));
+                            dctx->HUFptr, flags);
                     }
                 } else {
                     if (singleStream) {
@@ -190,18 +199,18 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                         hufSuccess = HUF_decompress1X_DCtx_wksp(
                             dctx->entropy.hufTable, dctx->litBuffer, litSize,
                             istart+lhSize, litCSize, dctx->workspace,
-                            sizeof(dctx->workspace));
+                            sizeof(dctx->workspace), flags);
 #else
-                        hufSuccess = HUF_decompress1X1_DCtx_wksp_bmi2(
+                        hufSuccess = HUF_decompress1X1_DCtx_wksp(
                             dctx->entropy.hufTable, dctx->litBuffer, litSize,
                             istart+lhSize, litCSize, dctx->workspace,
-                            sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx));
+                            sizeof(dctx->workspace), flags);
 #endif
                     } else {
-                        hufSuccess = HUF_decompress4X_hufOnly_wksp_bmi2(
+                        hufSuccess = HUF_decompress4X_hufOnly_wksp(
                             dctx->entropy.hufTable, dctx->litBuffer, litSize,
                             istart+lhSize, litCSize, dctx->workspace,
-                            sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx));
+                            sizeof(dctx->workspace), flags);
                     }
                 }
                 if (dctx->litBufferLocation == ZSTD_split)
@@ -237,6 +246,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                     break;
                 case 3:
                     lhSize = 3;
+                    RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize = 3");
                     litSize = MEM_readLE24(istart) >> 4;
                     break;
                 }
@@ -279,12 +289,13 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                     break;
                 case 1:
                     lhSize = 2;
+                    RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 3");
                     litSize = MEM_readLE16(istart) >> 4;
                     break;
                 case 3:
                     lhSize = 3;
+                    RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 4");
                     litSize = MEM_readLE24(istart) >> 4;
-                    RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4");
                     break;
                 }
                 RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled");
@@ -506,14 +517,15 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt,
                 for (i = 8; i < n; i += 8) {
                     MEM_write64(spread + pos + i, sv);
                 }
-                pos += n;
+                assert(n>=0);
+                pos += (size_t)n;
             }
         }
         /* Now we spread those positions across the table.
-         * The benefit of doing it in two stages is that we avoid the the
+         * The benefit of doing it in two stages is that we avoid the
          * variable size inner loop, which caused lots of branch misses.
          * Now we can run through all the positions without any branch misses.
-         * We unroll the loop twice, since that is what emperically worked best.
+         * We unroll the loop twice, since that is what empirically worked best.
          */
         {
             size_t position = 0;
@@ -540,7 +552,7 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt,
             for (i=0; i<n; i++) {
                 tableDecode[position].baseValue = s;
                 position = (position + step) & tableMask;
-                while (position > highThreshold) position = (position + step) & tableMask;   /* lowprob area */
+                while (UNLIKELY(position > highThreshold)) position = (position + step) & tableMask;   /* lowprob area */
         }   }
         assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
     }
@@ -551,7 +563,7 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt,
         for (u=0; u<tableSize; u++) {
             U32 const symbol = tableDecode[u].baseValue;
             U32 const nextState = symbolNext[symbol]++;
-            tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32(nextState) );
+            tableDecode[u].nbBits = (BYTE) (tableLog - ZSTD_highbit32(nextState) );
             tableDecode[u].nextState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
             assert(nbAdditionalBits[symbol] < 255);
             tableDecode[u].nbAdditionalBits = nbAdditionalBits[symbol];
@@ -964,6 +976,11 @@ size_t ZSTD_execSequence(BYTE* op,
 
     assert(op != NULL /* Precondition */);
     assert(oend_w < oend /* No underflow */);
+
+#if defined(__aarch64__)
+    /* prefetch sequence starting from match that will be used for copy later */
+    PREFETCH_L1(match);
+#endif
     /* Handle edge cases in a slow path:
      *   - Read beyond end of literals
      *   - Match end is within WILDCOPY_OVERLIMIT of oend
@@ -1154,7 +1171,7 @@ ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16
 }
 
 /* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum
- * offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1)
+ * offset bits. But we can only read at most STREAM_ACCUMULATOR_MIN_32
  * bits before reloading. This value is the maximum number of bytes we read
  * after reloading when we are decoding long offsets.
  */
@@ -1169,9 +1186,27 @@ FORCE_INLINE_TEMPLATE seq_t
 ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets)
 {
     seq_t seq;
+    /*
+     * ZSTD_seqSymbol is a structure with a total of 64 bits wide. So it can be
+     * loaded in one operation and extracted its fields by simply shifting or
+     * bit-extracting on aarch64.
+     * GCC doesn't recognize this and generates more unnecessary ldr/ldrb/ldrh
+     * operations that cause performance drop. This can be avoided by using this
+     * ZSTD_memcpy hack.
+     */
+#if defined(__aarch64__) && (defined(__GNUC__) && !defined(__clang__))
+    ZSTD_seqSymbol llDInfoS, mlDInfoS, ofDInfoS;
+    ZSTD_seqSymbol* const llDInfo = &llDInfoS;
+    ZSTD_seqSymbol* const mlDInfo = &mlDInfoS;
+    ZSTD_seqSymbol* const ofDInfo = &ofDInfoS;
+    ZSTD_memcpy(llDInfo, seqState->stateLL.table + seqState->stateLL.state, sizeof(ZSTD_seqSymbol));
+    ZSTD_memcpy(mlDInfo, seqState->stateML.table + seqState->stateML.state, sizeof(ZSTD_seqSymbol));
+    ZSTD_memcpy(ofDInfo, seqState->stateOffb.table + seqState->stateOffb.state, sizeof(ZSTD_seqSymbol));
+#else
     const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state;
     const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state;
     const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state;
+#endif
     seq.matchLength = mlDInfo->baseValue;
     seq.litLength = llDInfo->baseValue;
     {   U32 const ofBase = ofDInfo->baseValue;
@@ -1186,28 +1221,31 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets)
         U32 const llnbBits = llDInfo->nbBits;
         U32 const mlnbBits = mlDInfo->nbBits;
         U32 const ofnbBits = ofDInfo->nbBits;
+
+        assert(llBits <= MaxLLBits);
+        assert(mlBits <= MaxMLBits);
+        assert(ofBits <= MaxOff);
         /*
          * As gcc has better branch and block analyzers, sometimes it is only
-         * valuable to mark likelyness for clang, it gives around 3-4% of
+         * valuable to mark likeliness for clang, it gives around 3-4% of
          * performance.
          */
 
         /* sequence */
         {   size_t offset;
-    #if defined(__clang__)
-            if (LIKELY(ofBits > 1)) {
-    #else
             if (ofBits > 1) {
-    #endif
                 ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1);
                 ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5);
-                assert(ofBits <= MaxOff);
+                ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32);
+                ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits);
                 if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) {
-                    U32 const extraBits = ofBits - MIN(ofBits, 32 - seqState->DStream.bitsConsumed);
+                    /* Always read extra bits, this keeps the logic simple,
+                     * avoids branches, and avoids accidentally reading 0 bits.
+                     */
+                    U32 const extraBits = LONG_OFFSETS_MAX_EXTRA_BITS_32;
                     offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
                     BIT_reloadDStream(&seqState->DStream);
-                    if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits);
-                    assert(extraBits <= LONG_OFFSETS_MAX_EXTRA_BITS_32);   /* to avoid another reload */
+                    offset += BIT_readBitsFast(&seqState->DStream, extraBits);
                 } else {
                     offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/);   /* <=  (ZSTD_WINDOWLOG_MAX-1) bits */
                     if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);
@@ -1232,11 +1270,7 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets)
             seq.offset = offset;
         }
 
-    #if defined(__clang__)
-        if (UNLIKELY(mlBits > 0))
-    #else
         if (mlBits > 0)
-    #endif
             seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/);
 
         if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32))
@@ -1246,11 +1280,7 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets)
         /* Ensure there are enough bits to read the rest of data in 64-bit mode. */
         ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64);
 
-    #if defined(__clang__)
-        if (UNLIKELY(llBits > 0))
-    #else
         if (llBits > 0)
-    #endif
             seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/);
 
         if (MEM_32bits())
@@ -1552,7 +1582,7 @@ ZSTD_decompressSequences_body(ZSTD_DCtx* dctx,
     const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart);
     const BYTE* const vBase = (const BYTE*)(dctx->virtualStart);
     const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd);
-    DEBUGLOG(5, "ZSTD_decompressSequences_body");
+    DEBUGLOG(5, "ZSTD_decompressSequences_body: nbSeq = %d", nbSeq);
     (void)frame;
 
     /* Regen sequences */
@@ -1945,34 +1975,79 @@ ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx,
 #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */
 
 
+/*
+ * @returns The total size of the history referenceable by zstd, including
+ * both the prefix and the extDict. At @p op any offset larger than this
+ * is invalid.
+ */
+static size_t ZSTD_totalHistorySize(BYTE* op, BYTE const* virtualStart)
+{
+    return (size_t)(op - virtualStart);
+}
+
+typedef struct {
+    unsigned longOffsetShare;
+    unsigned maxNbAdditionalBits;
+} ZSTD_OffsetInfo;
 
-#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
-    !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
-/* ZSTD_getLongOffsetsShare() :
+/* ZSTD_getOffsetInfo() :
  * condition : offTable must be valid
  * @return : "share" of long offsets (arbitrarily defined as > (1<<23))
- *           compared to maximum possible of (1<<OffFSELog) */
-static unsigned
-ZSTD_getLongOffsetsShare(const ZSTD_seqSymbol* offTable)
+ *           compared to maximum possible of (1<<OffFSELog),
+ *           as well as the maximum number additional bits required.
+ */
+static ZSTD_OffsetInfo
+ZSTD_getOffsetInfo(const ZSTD_seqSymbol* offTable, int nbSeq)
 {
-    const void* ptr = offTable;
-    U32 const tableLog = ((const ZSTD_seqSymbol_header*)ptr)[0].tableLog;
-    const ZSTD_seqSymbol* table = offTable + 1;
-    U32 const max = 1 << tableLog;
-    U32 u, total = 0;
-    DEBUGLOG(5, "ZSTD_getLongOffsetsShare: (tableLog=%u)", tableLog);
-
-    assert(max <= (1 << OffFSELog));  /* max not too large */
-    for (u=0; u<max; u++) {
-        if (table[u].nbAdditionalBits > 22) total += 1;
+    ZSTD_OffsetInfo info = {0, 0};
+    /* If nbSeq == 0, then the offTable is uninitialized, but we have
+     * no sequences, so both values should be 0.
+     */
+    if (nbSeq != 0) {
+        const void* ptr = offTable;
+        U32 const tableLog = ((const ZSTD_seqSymbol_header*)ptr)[0].tableLog;
+        const ZSTD_seqSymbol* table = offTable + 1;
+        U32 const max = 1 << tableLog;
+        U32 u;
+        DEBUGLOG(5, "ZSTD_getLongOffsetsShare: (tableLog=%u)", tableLog);
+
+        assert(max <= (1 << OffFSELog));  /* max not too large */
+        for (u=0; u<max; u++) {
+            info.maxNbAdditionalBits = MAX(info.maxNbAdditionalBits, table[u].nbAdditionalBits);
+            if (table[u].nbAdditionalBits > 22) info.longOffsetShare += 1;
+        }
+
+        assert(tableLog <= OffFSELog);
+        info.longOffsetShare <<= (OffFSELog - tableLog);  /* scale to OffFSELog */
     }
 
-    assert(tableLog <= OffFSELog);
-    total <<= (OffFSELog - tableLog);  /* scale to OffFSELog */
+    return info;
+}
 
-    return total;
+/*
+ * @returns The maximum offset we can decode in one read of our bitstream, without
+ * reloading more bits in the middle of the offset bits read. Any offsets larger
+ * than this must use the long offset decoder.
+ */
+static size_t ZSTD_maxShortOffset(void)
+{
+    if (MEM_64bits()) {
+        /* We can decode any offset without reloading bits.
+         * This might change if the max window size grows.
+         */
+        ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31);
+        return (size_t)-1;
+    } else {
+        /* The maximum offBase is (1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1.
+         * This offBase would require STREAM_ACCUMULATOR_MIN extra bits.
+         * Then we have to subtract ZSTD_REP_NUM to get the maximum possible offset.
+         */
+        size_t const maxOffbase = ((size_t)1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1;
+        size_t const maxOffset = maxOffbase - ZSTD_REP_NUM;
+        assert(ZSTD_highbit32((U32)maxOffbase) == STREAM_ACCUMULATOR_MIN);
+        return maxOffset;
+    }
 }
-#endif
 
 size_t
 ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
@@ -1980,20 +2055,21 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
                         const void* src, size_t srcSize, const int frame, const streaming_operation streaming)
 {   /* blockType == blockCompressed */
     const BYTE* ip = (const BYTE*)src;
-    /* isLongOffset must be true if there are long offsets.
-     * Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN.
-     * We don't expect that to be the case in 64-bit mode.
-     * In block mode, window size is not known, so we have to be conservative.
-     * (note: but it could be evaluated from current-lowLimit)
-     */
-    ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN))));
     DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize);
 
-    RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong, "");
+    /* Note : the wording of the specification
+     * allows compressed block to be sized exactly ZSTD_BLOCKSIZE_MAX.
+     * This generally does not happen, as it makes little sense,
+     * since an uncompressed block would feature same size and have no decompression cost.
+     * Also, note that decoder from reference libzstd before < v1.5.4
+     * would consider this edge case as an error.
+     * As a consequence, avoid generating compressed blocks of size ZSTD_BLOCKSIZE_MAX
+     * for broader compatibility with the deployed ecosystem of zstd decoders */
+    RETURN_ERROR_IF(srcSize > ZSTD_BLOCKSIZE_MAX, srcSize_wrong, "");
 
     /* Decode literals section */
     {   size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming);
-        DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize);
+        DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : cSize=%u, nbLiterals=%zu", (U32)litCSize, dctx->litSize);
         if (ZSTD_isError(litCSize)) return litCSize;
         ip += litCSize;
         srcSize -= litCSize;
@@ -2001,6 +2077,23 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
 
     /* Build Decoding Tables */
     {
+        /* Compute the maximum block size, which must also work when !frame and fParams are unset.
+         * Additionally, take the min with dstCapacity to ensure that the totalHistorySize fits in a size_t.
+         */
+        size_t const blockSizeMax = MIN(dstCapacity, (frame ? dctx->fParams.blockSizeMax : ZSTD_BLOCKSIZE_MAX));
+        size_t const totalHistorySize = ZSTD_totalHistorySize((BYTE*)dst + blockSizeMax, (BYTE const*)dctx->virtualStart);
+        /* isLongOffset must be true if there are long offsets.
+         * Offsets are long if they are larger than ZSTD_maxShortOffset().
+         * We don't expect that to be the case in 64-bit mode.
+         *
+         * We check here to see if our history is large enough to allow long offsets.
+         * If it isn't, then we can't possible have (valid) long offsets. If the offset
+         * is invalid, then it is okay to read it incorrectly.
+         *
+         * If isLongOffsets is true, then we will later check our decoding table to see
+         * if it is even possible to generate long offsets.
+         */
+        ZSTD_longOffset_e isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (totalHistorySize > ZSTD_maxShortOffset()));
         /* These macros control at build-time which decompressor implementation
          * we use. If neither is defined, we do some inspection and dispatch at
          * runtime.
@@ -2008,6 +2101,11 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
 #if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
     !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
         int usePrefetchDecoder = dctx->ddictIsCold;
+#else
+        /* Set to 1 to avoid computing offset info if we don't need to.
+         * Otherwise this value is ignored.
+         */
+        int usePrefetchDecoder = 1;
 #endif
         int nbSeq;
         size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize);
@@ -2015,28 +2113,42 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
         ip += seqHSize;
         srcSize -= seqHSize;
 
-        RETURN_ERROR_IF(dst == NULL && nbSeq > 0, dstSize_tooSmall, "NULL not handled");
+        RETURN_ERROR_IF((dst == NULL || dstCapacity == 0) && nbSeq > 0, dstSize_tooSmall, "NULL not handled");
+        RETURN_ERROR_IF(MEM_64bits() && sizeof(size_t) == sizeof(void*) && (size_t)(-1) - (size_t)dst < (size_t)(1 << 20), dstSize_tooSmall,
+                "invalid dst");
 
-#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
-    !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
-        if ( !usePrefetchDecoder
-          && (!frame || (dctx->fParams.windowSize > (1<<24)))
-          && (nbSeq>ADVANCED_SEQS) ) {  /* could probably use a larger nbSeq limit */
-            U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr);
-            U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */
-            usePrefetchDecoder = (shareLongOffsets >= minShare);
+        /* If we could potentially have long offsets, or we might want to use the prefetch decoder,
+         * compute information about the share of long offsets, and the maximum nbAdditionalBits.
+         * NOTE: could probably use a larger nbSeq limit
+         */
+        if (isLongOffset || (!usePrefetchDecoder && (totalHistorySize > (1u << 24)) && (nbSeq > 8))) {
+            ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr, nbSeq);
+            if (isLongOffset && info.maxNbAdditionalBits <= STREAM_ACCUMULATOR_MIN) {
+                /* If isLongOffset, but the maximum number of additional bits that we see in our table is small
+                 * enough, then we know it is impossible to have too long an offset in this block, so we can
+                 * use the regular offset decoder.
+                 */
+                isLongOffset = ZSTD_lo_isRegularOffset;
+            }
+            if (!usePrefetchDecoder) {
+                U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */
+                usePrefetchDecoder = (info.longOffsetShare >= minShare);
+            }
         }
-#endif
 
         dctx->ddictIsCold = 0;
 
 #if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \
     !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG)
-        if (usePrefetchDecoder)
+        if (usePrefetchDecoder) {
+#else
+        (void)usePrefetchDecoder;
+        {
 #endif
 #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT
             return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame);
 #endif
+        }
 
 #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG
         /* else */
@@ -2060,9 +2172,9 @@ void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize)
 }
 
 
-size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
-                            void* dst, size_t dstCapacity,
-                      const void* src, size_t srcSize)
+size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx,
+                                       void* dst, size_t dstCapacity,
+                                 const void* src, size_t srcSize)
 {
     size_t dSize;
     ZSTD_checkContinuity(dctx, dst, dstCapacity);
@@ -2070,3 +2182,12 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
     dctx->previousDstEnd = (char*)dst + dSize;
     return dSize;
 }
+
+
+/* NOTE: Must just wrap ZSTD_decompressBlock_deprecated() */
+size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx,
+                            void* dst, size_t dstCapacity,
+                      const void* src, size_t srcSize)
+{
+    return ZSTD_decompressBlock_deprecated(dctx, dst, dstCapacity, src, srcSize);
+}
diff --git a/lib/zstd/decompress/zstd_decompress_block.h b/lib/zstd/decompress/zstd_decompress_block.h
index 3d2d57a5d25a..5888e6cc788b 100644
--- a/lib/zstd/decompress/zstd_decompress_block.h
+++ b/lib/zstd/decompress/zstd_decompress_block.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -64,5 +65,10 @@ void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
                    unsigned tableLog, void* wksp, size_t wkspSize,
                    int bmi2);
 
+/* Internal definition of ZSTD_decompressBlock() to avoid deprecation warnings. */
+size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx,
+                            void* dst, size_t dstCapacity,
+                      const void* src, size_t srcSize);
+
 
 #endif /* ZSTD_DEC_BLOCK_H */
diff --git a/lib/zstd/decompress/zstd_decompress_internal.h b/lib/zstd/decompress/zstd_decompress_internal.h
index 98102edb6a83..32f79fb2873d 100644
--- a/lib/zstd/decompress/zstd_decompress_internal.h
+++ b/lib/zstd/decompress/zstd_decompress_internal.h
@@ -1,5 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Yann Collet, Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -75,12 +76,13 @@ static UNUSED_ATTR const U32 ML_base[MaxML+1] = {
 
 #define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64))
 #define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32))
+#define ZSTD_HUFFDTABLE_CAPACITY_LOG 12
 
 typedef struct {
     ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)];    /* Note : Space reserved for FSE Tables */
     ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)];   /* is also used as temporary workspace while building hufTable during DDict creation */
     ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)];    /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */
-    HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)];  /* can accommodate HUF_decompress4X */
+    HUF_DTable hufTable[HUF_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)];  /* can accommodate HUF_decompress4X */
     U32 rep[ZSTD_REP_NUM];
     U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32];
 } ZSTD_entropyDTables_t;
@@ -164,6 +166,7 @@ struct ZSTD_DCtx_s
     ZSTD_dictUses_e dictUses;
     ZSTD_DDictHashSet* ddictSet;                    /* Hash set for multiple ddicts */
     ZSTD_refMultipleDDicts_e refMultipleDDicts;     /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */
+    int disableHufAsm;
 
     /* streaming */
     ZSTD_dStreamStage streamStage;
diff --git a/lib/zstd/decompress_sources.h b/lib/zstd/decompress_sources.h
index a06ca187aab5..8a47eb2a4514 100644
--- a/lib/zstd/decompress_sources.h
+++ b/lib/zstd/decompress_sources.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/zstd/zstd_common_module.c b/lib/zstd/zstd_common_module.c
index 22686e367e6f..466828e35752 100644
--- a/lib/zstd/zstd_common_module.c
+++ b/lib/zstd/zstd_common_module.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -24,9 +24,6 @@ EXPORT_SYMBOL_GPL(HUF_readStats_wksp);
 EXPORT_SYMBOL_GPL(ZSTD_isError);
 EXPORT_SYMBOL_GPL(ZSTD_getErrorName);
 EXPORT_SYMBOL_GPL(ZSTD_getErrorCode);
-EXPORT_SYMBOL_GPL(ZSTD_customMalloc);
-EXPORT_SYMBOL_GPL(ZSTD_customCalloc);
-EXPORT_SYMBOL_GPL(ZSTD_customFree);
 
 MODULE_LICENSE("Dual BSD/GPL");
 MODULE_DESCRIPTION("Zstd Common");
diff --git a/lib/zstd/zstd_compress_module.c b/lib/zstd/zstd_compress_module.c
index 04e1b5c01d9b..8ecf43226af2 100644
--- a/lib/zstd/zstd_compress_module.c
+++ b/lib/zstd/zstd_compress_module.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
diff --git a/lib/zstd/zstd_decompress_module.c b/lib/zstd/zstd_decompress_module.c
index f4ed952ed485..7d31518e9d5a 100644
--- a/lib/zstd/zstd_decompress_module.c
+++ b/lib/zstd/zstd_decompress_module.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause
 /*
- * Copyright (c) Facebook, Inc.
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
  * All rights reserved.
  *
  * This source code is licensed under both the BSD-style license (found in the
@@ -77,7 +77,7 @@ EXPORT_SYMBOL(zstd_init_dstream);
 
 size_t zstd_reset_dstream(zstd_dstream *dstream)
 {
-	return ZSTD_resetDStream(dstream);
+	return ZSTD_DCtx_reset(dstream, ZSTD_reset_session_only);
 }
 EXPORT_SYMBOL(zstd_reset_dstream);
 
diff --git a/localversion-next b/localversion-next
new file mode 100644
index 000000000000..bc1591e00ac4
--- /dev/null
+++ b/localversion-next
@@ -0,0 +1 @@
+-next-20231122
diff --git a/mm/compaction.c b/mm/compaction.c
index 01ba298739dd..e2d0c604ae42 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -882,6 +882,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
 
 	/* Time to isolate some pages for migration */
 	for (; low_pfn < end_pfn; low_pfn++) {
+		bool is_dirty, is_unevictable;
 
 		if (skip_on_failure && low_pfn >= next_skip_pfn) {
 			/*
@@ -1079,8 +1080,10 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
 		if (!folio_test_lru(folio))
 			goto isolate_fail_put;
 
+		is_unevictable = folio_test_unevictable(folio);
+
 		/* Compaction might skip unevictable pages but CMA takes them */
-		if (!(mode & ISOLATE_UNEVICTABLE) && folio_test_unevictable(folio))
+		if (!(mode & ISOLATE_UNEVICTABLE) && is_unevictable)
 			goto isolate_fail_put;
 
 		/*
@@ -1092,26 +1095,42 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
 		if ((mode & ISOLATE_ASYNC_MIGRATE) && folio_test_writeback(folio))
 			goto isolate_fail_put;
 
-		if ((mode & ISOLATE_ASYNC_MIGRATE) && folio_test_dirty(folio)) {
-			bool migrate_dirty;
+		is_dirty = folio_test_dirty(folio);
+
+		if (((mode & ISOLATE_ASYNC_MIGRATE) && is_dirty) ||
+		    (mapping && is_unevictable)) {
+			bool migrate_dirty = true;
+			bool is_unmovable;
 
 			/*
 			 * Only folios without mappings or that have
-			 * a ->migrate_folio callback are possible to
-			 * migrate without blocking.  However, we may
-			 * be racing with truncation, which can free
-			 * the mapping.  Truncation holds the folio lock
-			 * until after the folio is removed from the page
-			 * cache so holding it ourselves is sufficient.
+			 * a ->migrate_folio callback are possible to migrate
+			 * without blocking.
+			 *
+			 * Folios from unmovable mappings are not migratable.
+			 *
+			 * However, we can be racing with truncation, which can
+			 * free the mapping that we need to check. Truncation
+			 * holds the folio lock until after the folio is removed
+			 * from the page so holding it ourselves is sufficient.
+			 *
+			 * To avoid locking the folio just to check unmovable,
+			 * assume every unmovable folio is also unevictable,
+			 * which is a cheaper test.  If our assumption goes
+			 * wrong, it's not a correctness bug, just potentially
+			 * wasted cycles.
 			 */
 			if (!folio_trylock(folio))
 				goto isolate_fail_put;
 
 			mapping = folio_mapping(folio);
-			migrate_dirty = !mapping ||
-					mapping->a_ops->migrate_folio;
+			if ((mode & ISOLATE_ASYNC_MIGRATE) && is_dirty) {
+				migrate_dirty = !mapping ||
+						mapping->a_ops->migrate_folio;
+			}
+			is_unmovable = mapping && mapping_unmovable(mapping);
 			folio_unlock(folio);
-			if (!migrate_dirty)
+			if (!migrate_dirty || is_unmovable)
 				goto isolate_fail_put;
 		}
 
diff --git a/mm/damon/core-test.h b/mm/damon/core-test.h
index 649adf91ebc5..e6a01ea2ec54 100644
--- a/mm/damon/core-test.h
+++ b/mm/damon/core-test.h
@@ -122,18 +122,25 @@ static void damon_test_split_at(struct kunit *test)
 {
 	struct damon_ctx *c = damon_new_ctx();
 	struct damon_target *t;
-	struct damon_region *r;
+	struct damon_region *r, *r_new;
 
 	t = damon_new_target();
 	r = damon_new_region(0, 100);
+	r->nr_accesses_bp = 420000;
+	r->nr_accesses = 42;
+	r->last_nr_accesses = 15;
 	damon_add_region(r, t);
 	damon_split_region_at(t, r, 25);
 	KUNIT_EXPECT_EQ(test, r->ar.start, 0ul);
 	KUNIT_EXPECT_EQ(test, r->ar.end, 25ul);
 
-	r = damon_next_region(r);
-	KUNIT_EXPECT_EQ(test, r->ar.start, 25ul);
-	KUNIT_EXPECT_EQ(test, r->ar.end, 100ul);
+	r_new = damon_next_region(r);
+	KUNIT_EXPECT_EQ(test, r_new->ar.start, 25ul);
+	KUNIT_EXPECT_EQ(test, r_new->ar.end, 100ul);
+
+	KUNIT_EXPECT_EQ(test, r->nr_accesses_bp, r_new->nr_accesses_bp);
+	KUNIT_EXPECT_EQ(test, r->nr_accesses, r_new->nr_accesses);
+	KUNIT_EXPECT_EQ(test, r->last_nr_accesses, r_new->last_nr_accesses);
 
 	damon_free_target(t);
 	damon_destroy_ctx(c);
diff --git a/mm/damon/core.c b/mm/damon/core.c
index 6262d55904e7..ce1562783e7e 100644
--- a/mm/damon/core.c
+++ b/mm/damon/core.c
@@ -1225,6 +1225,7 @@ static void damon_split_region_at(struct damon_target *t,
 	new->age = r->age;
 	new->last_nr_accesses = r->last_nr_accesses;
 	new->nr_accesses_bp = r->nr_accesses_bp;
+	new->nr_accesses = r->nr_accesses;
 
 	damon_insert_region(new, r, damon_next_region(r), t);
 }
diff --git a/mm/filemap.c b/mm/filemap.c
index 32eedf3afd45..71f00539ac00 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -2173,7 +2173,7 @@ unsigned filemap_get_folios_contig(struct address_space *mapping,
 
 	if (nr) {
 		folio = fbatch->folios[nr - 1];
-		*start = folio->index + folio_nr_pages(folio);
+		*start = folio_next_index(folio);
 	}
 out:
 	rcu_read_unlock();
@@ -3371,7 +3371,7 @@ static bool filemap_map_pmd(struct vm_fault *vmf, struct folio *folio,
 		}
 	}
 
-	if (pmd_none(*vmf->pmd))
+	if (pmd_none(*vmf->pmd) && vmf->prealloc_pte)
 		pmd_install(mm, vmf->pmd, &vmf->prealloc_pte);
 
 	return false;
diff --git a/mm/folio-compat.c b/mm/folio-compat.c
index 10c3247542cb..aee3b9a16828 100644
--- a/mm/folio-compat.c
+++ b/mm/folio-compat.c
@@ -46,9 +46,9 @@ void mark_page_accessed(struct page *page)
 }
 EXPORT_SYMBOL(mark_page_accessed);
 
-bool set_page_writeback(struct page *page)
+void set_page_writeback(struct page *page)
 {
-	return folio_start_writeback(page_folio(page));
+	folio_start_writeback(page_folio(page));
 }
 EXPORT_SYMBOL(set_page_writeback);
 
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 4f542444a91f..6eb55f97a3d2 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -2379,7 +2379,7 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma,
 static void unmap_folio(struct folio *folio)
 {
 	enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD |
-		TTU_SYNC;
+		TTU_SYNC | TTU_BATCH_FLUSH;
 
 	VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);
 
@@ -2392,6 +2392,8 @@ static void unmap_folio(struct folio *folio)
 		try_to_migrate(folio, ttu_flags);
 	else
 		try_to_unmap(folio, ttu_flags | TTU_IGNORE_MLOCK);
+
+	try_to_unmap_flush();
 }
 
 static void remap_page(struct folio *folio, unsigned long nr)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 1169ef2f2176..c466551e2fd9 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -1141,7 +1141,7 @@ static inline struct resv_map *inode_resv_map(struct inode *inode)
 	 * The VERY common case is inode->mapping == &inode->i_data but,
 	 * this may not be true for device special inodes.
 	 */
-	return (struct resv_map *)(&inode->i_data)->private_data;
+	return (struct resv_map *)(&inode->i_data)->i_private_data;
 }
 
 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
@@ -1182,6 +1182,13 @@ static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
 	return (get_vma_private_data(vma) & flag) != 0;
 }
 
+bool __vma_private_lock(struct vm_area_struct *vma)
+{
+	return !(vma->vm_flags & VM_MAYSHARE) &&
+		get_vma_private_data(vma) & ~HPAGE_RESV_MASK &&
+		is_vma_resv_set(vma, HPAGE_RESV_OWNER);
+}
+
 void hugetlb_dup_vma_private(struct vm_area_struct *vma)
 {
 	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
diff --git a/mm/internal.h b/mm/internal.h
index b61034bd50f5..8450562744cf 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -138,7 +138,7 @@ void filemap_free_folio(struct address_space *mapping, struct folio *folio);
 int truncate_inode_folio(struct address_space *mapping, struct folio *folio);
 bool truncate_inode_partial_folio(struct folio *folio, loff_t start,
 		loff_t end);
-long invalidate_inode_page(struct page *page);
+long mapping_evict_folio(struct address_space *mapping, struct folio *folio);
 unsigned long mapping_try_invalidate(struct address_space *mapping,
 		pgoff_t start, pgoff_t end, unsigned long *nr_failed);
 
@@ -1135,8 +1135,6 @@ static inline bool vma_soft_dirty_enabled(struct vm_area_struct *vma)
 static inline void vma_iter_config(struct vma_iterator *vmi,
 		unsigned long index, unsigned long last)
 {
-	MAS_BUG_ON(&vmi->mas, vmi->mas.node != MAS_START &&
-		   (vmi->mas.index > index || vmi->mas.last < index));
 	__mas_set_range(&vmi->mas, index, last - 1);
 }
 
@@ -1154,17 +1152,6 @@ static inline void vma_iter_clear(struct vma_iterator *vmi)
 	mas_store_prealloc(&vmi->mas, NULL);
 }
 
-static inline int vma_iter_clear_gfp(struct vma_iterator *vmi,
-			unsigned long start, unsigned long end, gfp_t gfp)
-{
-	__mas_set_range(&vmi->mas, start, end - 1);
-	mas_store_gfp(&vmi->mas, NULL, gfp);
-	if (unlikely(mas_is_err(&vmi->mas)))
-		return -ENOMEM;
-
-	return 0;
-}
-
 static inline struct vm_area_struct *vma_iter_load(struct vma_iterator *vmi)
 {
 	return mas_walk(&vmi->mas);
@@ -1176,13 +1163,13 @@ static inline void vma_iter_store(struct vma_iterator *vmi,
 {
 
 #if defined(CONFIG_DEBUG_VM_MAPLE_TREE)
-	if (MAS_WARN_ON(&vmi->mas, vmi->mas.node != MAS_START &&
+	if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
 			vmi->mas.index > vma->vm_start)) {
 		pr_warn("%lx > %lx\n store vma %lx-%lx\n into slot %lx-%lx\n",
 			vmi->mas.index, vma->vm_start, vma->vm_start,
 			vma->vm_end, vmi->mas.index, vmi->mas.last);
 	}
-	if (MAS_WARN_ON(&vmi->mas, vmi->mas.node != MAS_START &&
+	if (MAS_WARN_ON(&vmi->mas, vmi->mas.status != ma_start &&
 			vmi->mas.last <  vma->vm_start)) {
 		pr_warn("%lx < %lx\nstore vma %lx-%lx\ninto slot %lx-%lx\n",
 		       vmi->mas.last, vma->vm_start, vma->vm_start, vma->vm_end,
@@ -1190,7 +1177,7 @@ static inline void vma_iter_store(struct vma_iterator *vmi,
 	}
 #endif
 
-	if (vmi->mas.node != MAS_START &&
+	if (vmi->mas.status != ma_start &&
 	    ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
 		vma_iter_invalidate(vmi);
 
@@ -1201,7 +1188,7 @@ static inline void vma_iter_store(struct vma_iterator *vmi,
 static inline int vma_iter_store_gfp(struct vma_iterator *vmi,
 			struct vm_area_struct *vma, gfp_t gfp)
 {
-	if (vmi->mas.node != MAS_START &&
+	if (vmi->mas.status != ma_start &&
 	    ((vmi->mas.index > vma->vm_start) || (vmi->mas.last < vma->vm_start)))
 		vma_iter_invalidate(vmi);
 
diff --git a/mm/kasan/common.c b/mm/kasan/common.c
index 256930da578a..b5d8bd26fced 100644
--- a/mm/kasan/common.c
+++ b/mm/kasan/common.c
@@ -22,6 +22,7 @@
 #include <linux/sched.h>
 #include <linux/sched/task_stack.h>
 #include <linux/slab.h>
+#include <linux/stackdepot.h>
 #include <linux/stacktrace.h>
 #include <linux/string.h>
 #include <linux/types.h>
@@ -37,19 +38,20 @@ struct slab *kasan_addr_to_slab(const void *addr)
 	return NULL;
 }
 
-depot_stack_handle_t kasan_save_stack(gfp_t flags, bool can_alloc)
+depot_stack_handle_t kasan_save_stack(gfp_t flags, depot_flags_t depot_flags)
 {
 	unsigned long entries[KASAN_STACK_DEPTH];
 	unsigned int nr_entries;
 
 	nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0);
-	return __stack_depot_save(entries, nr_entries, flags, can_alloc);
+	return stack_depot_save_flags(entries, nr_entries, flags, depot_flags);
 }
 
 void kasan_set_track(struct kasan_track *track, gfp_t flags)
 {
 	track->pid = current->pid;
-	track->stack = kasan_save_stack(flags, true);
+	track->stack = kasan_save_stack(flags,
+			STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_GET);
 }
 
 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
diff --git a/mm/kasan/generic.c b/mm/kasan/generic.c
index 4d837ab83f08..7ca5d0b91585 100644
--- a/mm/kasan/generic.c
+++ b/mm/kasan/generic.c
@@ -25,6 +25,7 @@
 #include <linux/sched.h>
 #include <linux/sched/task_stack.h>
 #include <linux/slab.h>
+#include <linux/stackdepot.h>
 #include <linux/stacktrace.h>
 #include <linux/string.h>
 #include <linux/types.h>
@@ -361,6 +362,8 @@ void kasan_cache_create(struct kmem_cache *cache, unsigned int *size,
 {
 	unsigned int ok_size;
 	unsigned int optimal_size;
+	unsigned int rem_free_meta_size;
+	unsigned int orig_alloc_meta_offset;
 
 	if (!kasan_requires_meta())
 		return;
@@ -394,6 +397,9 @@ void kasan_cache_create(struct kmem_cache *cache, unsigned int *size,
 		/* Continue, since free meta might still fit. */
 	}
 
+	ok_size = *size;
+	orig_alloc_meta_offset = cache->kasan_info.alloc_meta_offset;
+
 	/*
 	 * Add free meta into redzone when it's not possible to store
 	 * it in the object. This is the case when:
@@ -401,21 +407,26 @@ void kasan_cache_create(struct kmem_cache *cache, unsigned int *size,
 	 *    be touched after it was freed, or
 	 * 2. Object has a constructor, which means it's expected to
 	 *    retain its content until the next allocation, or
-	 * 3. Object is too small.
 	 * Otherwise cache->kasan_info.free_meta_offset = 0 is implied.
+	 * Even if the object is smaller than free meta, it is still
+	 * possible to store part of the free meta in the object.
 	 */
-	if ((cache->flags & SLAB_TYPESAFE_BY_RCU) || cache->ctor ||
-	    cache->object_size < sizeof(struct kasan_free_meta)) {
-		ok_size = *size;
-
+	if ((cache->flags & SLAB_TYPESAFE_BY_RCU) || cache->ctor) {
 		cache->kasan_info.free_meta_offset = *size;
 		*size += sizeof(struct kasan_free_meta);
+	} else if (cache->object_size < sizeof(struct kasan_free_meta)) {
+		rem_free_meta_size = sizeof(struct kasan_free_meta) -
+								cache->object_size;
+		*size += rem_free_meta_size;
+		if (cache->kasan_info.alloc_meta_offset != 0)
+			cache->kasan_info.alloc_meta_offset += rem_free_meta_size;
+	}
 
-		/* If free meta doesn't fit, don't add it. */
-		if (*size > KMALLOC_MAX_SIZE) {
-			cache->kasan_info.free_meta_offset = KASAN_NO_FREE_META;
-			*size = ok_size;
-		}
+	/* If free meta doesn't fit, don't add it. */
+	if (*size > KMALLOC_MAX_SIZE) {
+		cache->kasan_info.free_meta_offset = KASAN_NO_FREE_META;
+		cache->kasan_info.alloc_meta_offset = orig_alloc_meta_offset;
+		*size = ok_size;
 	}
 
 	/* Calculate size with optimal redzone. */
@@ -448,10 +459,14 @@ struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache,
 void kasan_init_object_meta(struct kmem_cache *cache, const void *object)
 {
 	struct kasan_alloc_meta *alloc_meta;
+	struct kasan_free_meta *free_meta;
 
 	alloc_meta = kasan_get_alloc_meta(cache, object);
 	if (alloc_meta)
 		__memset(alloc_meta, 0, sizeof(*alloc_meta));
+	free_meta = kasan_get_free_meta(cache, object);
+	if (free_meta)
+		__memset(free_meta, 0, sizeof(*free_meta));
 }
 
 size_t kasan_metadata_size(struct kmem_cache *cache, bool in_object)
@@ -472,7 +487,7 @@ size_t kasan_metadata_size(struct kmem_cache *cache, bool in_object)
 			sizeof(struct kasan_free_meta) : 0);
 }
 
-static void __kasan_record_aux_stack(void *addr, bool can_alloc)
+static void __kasan_record_aux_stack(void *addr, depot_flags_t depot_flags)
 {
 	struct slab *slab = kasan_addr_to_slab(addr);
 	struct kmem_cache *cache;
@@ -488,18 +503,20 @@ static void __kasan_record_aux_stack(void *addr, bool can_alloc)
 	if (!alloc_meta)
 		return;
 
+	stack_depot_put(alloc_meta->aux_stack[1]);
 	alloc_meta->aux_stack[1] = alloc_meta->aux_stack[0];
-	alloc_meta->aux_stack[0] = kasan_save_stack(0, can_alloc);
+	alloc_meta->aux_stack[0] = kasan_save_stack(0, depot_flags);
 }
 
 void kasan_record_aux_stack(void *addr)
 {
-	return __kasan_record_aux_stack(addr, true);
+	return __kasan_record_aux_stack(addr,
+			STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_GET);
 }
 
 void kasan_record_aux_stack_noalloc(void *addr)
 {
-	return __kasan_record_aux_stack(addr, false);
+	return __kasan_record_aux_stack(addr, STACK_DEPOT_FLAG_GET);
 }
 
 void kasan_save_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags)
@@ -507,8 +524,16 @@ void kasan_save_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags)
 	struct kasan_alloc_meta *alloc_meta;
 
 	alloc_meta = kasan_get_alloc_meta(cache, object);
-	if (alloc_meta)
-		kasan_set_track(&alloc_meta->alloc_track, flags);
+	if (!alloc_meta)
+		return;
+
+	/* Evict previous stack traces (might exist for krealloc). */
+	stack_depot_put(alloc_meta->alloc_track.stack);
+	stack_depot_put(alloc_meta->aux_stack[0]);
+	stack_depot_put(alloc_meta->aux_stack[1]);
+	__memset(alloc_meta, 0, sizeof(*alloc_meta));
+
+	kasan_set_track(&alloc_meta->alloc_track, flags);
 }
 
 void kasan_save_free_info(struct kmem_cache *cache, void *object)
diff --git a/mm/kasan/kasan.h b/mm/kasan/kasan.h
index 8b06bab5c406..b29d46b83d1f 100644
--- a/mm/kasan/kasan.h
+++ b/mm/kasan/kasan.h
@@ -368,7 +368,7 @@ static inline void kasan_init_cache_meta(struct kmem_cache *cache, unsigned int
 static inline void kasan_init_object_meta(struct kmem_cache *cache, const void *object) { }
 #endif
 
-depot_stack_handle_t kasan_save_stack(gfp_t flags, bool can_alloc);
+depot_stack_handle_t kasan_save_stack(gfp_t flags, depot_flags_t depot_flags);
 void kasan_set_track(struct kasan_track *track, gfp_t flags);
 void kasan_save_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags);
 void kasan_save_free_info(struct kmem_cache *cache, void *object);
diff --git a/mm/kasan/quarantine.c b/mm/kasan/quarantine.c
index ca4529156735..265ca2bbe2dd 100644
--- a/mm/kasan/quarantine.c
+++ b/mm/kasan/quarantine.c
@@ -143,11 +143,22 @@ static void *qlink_to_object(struct qlist_node *qlink, struct kmem_cache *cache)
 static void qlink_free(struct qlist_node *qlink, struct kmem_cache *cache)
 {
 	void *object = qlink_to_object(qlink, cache);
-	struct kasan_free_meta *meta = kasan_get_free_meta(cache, object);
+	struct kasan_alloc_meta *alloc_meta = kasan_get_alloc_meta(cache, object);
+	struct kasan_free_meta *free_meta = kasan_get_free_meta(cache, object);
 	unsigned long flags;
 
-	if (IS_ENABLED(CONFIG_SLAB))
-		local_irq_save(flags);
+	if (alloc_meta) {
+		stack_depot_put(alloc_meta->alloc_track.stack);
+		stack_depot_put(alloc_meta->aux_stack[0]);
+		stack_depot_put(alloc_meta->aux_stack[1]);
+		__memset(alloc_meta, 0, sizeof(*alloc_meta));
+	}
+
+	if (free_meta &&
+	    *(u8 *)kasan_mem_to_shadow(object) == KASAN_SLAB_FREETRACK) {
+		stack_depot_put(free_meta->free_track.stack);
+		free_meta->free_track.stack = 0;
+	}
 
 	/*
 	 * If init_on_free is enabled and KASAN's free metadata is stored in
@@ -157,14 +168,17 @@ static void qlink_free(struct qlist_node *qlink, struct kmem_cache *cache)
 	 */
 	if (slab_want_init_on_free(cache) &&
 	    cache->kasan_info.free_meta_offset == 0)
-		memzero_explicit(meta, sizeof(*meta));
+		memzero_explicit(free_meta, sizeof(*free_meta));
 
 	/*
-	 * As the object now gets freed from the quarantine, assume that its
-	 * free track is no longer valid.
+	 * As the object now gets freed from the quarantine,
+	 * take note that its free track is no longer exists.
 	 */
 	*(u8 *)kasan_mem_to_shadow(object) = KASAN_SLAB_FREE;
 
+	if (IS_ENABLED(CONFIG_SLAB))
+		local_irq_save(flags);
+
 	___cache_free(cache, object, _THIS_IP_);
 
 	if (IS_ENABLED(CONFIG_SLAB))
diff --git a/mm/kasan/report_tags.c b/mm/kasan/report_tags.c
index 8b8bfdb3cfdb..55154743f915 100644
--- a/mm/kasan/report_tags.c
+++ b/mm/kasan/report_tags.c
@@ -7,6 +7,7 @@
 #include <linux/atomic.h>
 
 #include "kasan.h"
+#include "../slab.h"
 
 extern struct kasan_stack_ring stack_ring;
 
@@ -31,10 +32,6 @@ void kasan_complete_mode_report_info(struct kasan_report_info *info)
 	unsigned long flags;
 	u64 pos;
 	struct kasan_stack_ring_entry *entry;
-	void *ptr;
-	u32 pid;
-	depot_stack_handle_t stack;
-	bool is_free;
 	bool alloc_found = false, free_found = false;
 
 	if ((!info->cache || !info->object) && !info->bug_type) {
@@ -61,18 +58,12 @@ void kasan_complete_mode_report_info(struct kasan_report_info *info)
 
 		entry = &stack_ring.entries[i % stack_ring.size];
 
-		/* Paired with smp_store_release() in save_stack_info(). */
-		ptr = (void *)smp_load_acquire(&entry->ptr);
-
-		if (kasan_reset_tag(ptr) != info->object ||
-		    get_tag(ptr) != get_tag(info->access_addr))
+		if (kasan_reset_tag(entry->ptr) != info->object ||
+		    get_tag(entry->ptr) != get_tag(info->access_addr) ||
+		    info->cache->object_size != entry->size)
 			continue;
 
-		pid = READ_ONCE(entry->pid);
-		stack = READ_ONCE(entry->stack);
-		is_free = READ_ONCE(entry->is_free);
-
-		if (is_free) {
+		if (entry->is_free) {
 			/*
 			 * Second free of the same object.
 			 * Give up on trying to find the alloc entry.
@@ -80,8 +71,8 @@ void kasan_complete_mode_report_info(struct kasan_report_info *info)
 			if (free_found)
 				break;
 
-			info->free_track.pid = pid;
-			info->free_track.stack = stack;
+			info->free_track.pid = entry->pid;
+			info->free_track.stack = entry->stack;
 			free_found = true;
 
 			/*
@@ -95,8 +86,8 @@ void kasan_complete_mode_report_info(struct kasan_report_info *info)
 			if (alloc_found)
 				break;
 
-			info->alloc_track.pid = pid;
-			info->alloc_track.stack = stack;
+			info->alloc_track.pid = entry->pid;
+			info->alloc_track.stack = entry->stack;
 			alloc_found = true;
 
 			/*
diff --git a/mm/kasan/tags.c b/mm/kasan/tags.c
index 7dcfe341d48e..739ae997463d 100644
--- a/mm/kasan/tags.c
+++ b/mm/kasan/tags.c
@@ -13,6 +13,7 @@
 #include <linux/memblock.h>
 #include <linux/memory.h>
 #include <linux/mm.h>
+#include <linux/stackdepot.h>
 #include <linux/static_key.h>
 #include <linux/string.h>
 #include <linux/types.h>
@@ -96,12 +97,13 @@ static void save_stack_info(struct kmem_cache *cache, void *object,
 			gfp_t gfp_flags, bool is_free)
 {
 	unsigned long flags;
-	depot_stack_handle_t stack;
+	depot_stack_handle_t stack, old_stack;
 	u64 pos;
 	struct kasan_stack_ring_entry *entry;
 	void *old_ptr;
 
-	stack = kasan_save_stack(gfp_flags, true);
+	stack = kasan_save_stack(gfp_flags,
+			STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_GET);
 
 	/*
 	 * Prevent save_stack_info() from modifying stack ring
@@ -120,17 +122,19 @@ static void save_stack_info(struct kmem_cache *cache, void *object,
 	if (!try_cmpxchg(&entry->ptr, &old_ptr, STACK_RING_BUSY_PTR))
 		goto next; /* Busy slot. */
 
-	WRITE_ONCE(entry->size, cache->object_size);
-	WRITE_ONCE(entry->pid, current->pid);
-	WRITE_ONCE(entry->stack, stack);
-	WRITE_ONCE(entry->is_free, is_free);
+	old_stack = entry->stack;
 
-	/*
-	 * Paired with smp_load_acquire() in kasan_complete_mode_report_info().
-	 */
-	smp_store_release(&entry->ptr, (s64)object);
+	entry->size = cache->object_size;
+	entry->pid = current->pid;
+	entry->stack = stack;
+	entry->is_free = is_free;
+
+	entry->ptr = object;
 
 	read_unlock_irqrestore(&stack_ring.lock, flags);
+
+	if (old_stack)
+		stack_depot_put(old_stack);
 }
 
 void kasan_save_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags)
diff --git a/mm/kmemleak.c b/mm/kmemleak.c
index 1eacca03bedd..0fb4dcc3b06a 100644
--- a/mm/kmemleak.c
+++ b/mm/kmemleak.c
@@ -355,16 +355,14 @@ static void print_unreferenced(struct seq_file *seq,
 	int i;
 	unsigned long *entries;
 	unsigned int nr_entries;
-	unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);
 
 	nr_entries = stack_depot_fetch(object->trace_handle, &entries);
 	warn_or_seq_printf(seq, "unreferenced object 0x%08lx (size %zu):\n",
 			  object->pointer, object->size);
-	warn_or_seq_printf(seq, "  comm \"%s\", pid %d, jiffies %lu (age %d.%03ds)\n",
-			   object->comm, object->pid, object->jiffies,
-			   msecs_age / 1000, msecs_age % 1000);
+	warn_or_seq_printf(seq, "  comm \"%s\", pid %d, jiffies %lu\n",
+			   object->comm, object->pid, object->jiffies);
 	hex_dump_object(seq, object);
-	warn_or_seq_printf(seq, "  backtrace:\n");
+	warn_or_seq_printf(seq, "  backtrace (crc %x):\n", object->checksum);
 
 	for (i = 0; i < nr_entries; i++) {
 		void *ptr = (void *)entries[i];
@@ -642,32 +640,16 @@ static struct kmemleak_object *__alloc_object(gfp_t gfp)
 	if (!object) {
 		pr_warn("Cannot allocate a kmemleak_object structure\n");
 		kmemleak_disable();
+		return NULL;
 	}
 
-	return object;
-}
-
-static int __link_object(struct kmemleak_object *object, unsigned long ptr,
-			 size_t size, int min_count, bool is_phys)
-{
-
-	struct kmemleak_object *parent;
-	struct rb_node **link, *rb_parent;
-	unsigned long untagged_ptr;
-	unsigned long untagged_objp;
-
 	INIT_LIST_HEAD(&object->object_list);
 	INIT_LIST_HEAD(&object->gray_list);
 	INIT_HLIST_HEAD(&object->area_list);
 	raw_spin_lock_init(&object->lock);
 	atomic_set(&object->use_count, 1);
-	object->flags = OBJECT_ALLOCATED | (is_phys ? OBJECT_PHYS : 0);
-	object->pointer = ptr;
-	object->size = kfence_ksize((void *)ptr) ?: size;
 	object->excess_ref = 0;
-	object->min_count = min_count;
 	object->count = 0;			/* white color initially */
-	object->jiffies = jiffies;
 	object->checksum = 0;
 	object->del_state = 0;
 
@@ -692,6 +674,24 @@ static int __link_object(struct kmemleak_object *object, unsigned long ptr,
 	/* kernel backtrace */
 	object->trace_handle = set_track_prepare();
 
+	return object;
+}
+
+static int __link_object(struct kmemleak_object *object, unsigned long ptr,
+			 size_t size, int min_count, bool is_phys)
+{
+
+	struct kmemleak_object *parent;
+	struct rb_node **link, *rb_parent;
+	unsigned long untagged_ptr;
+	unsigned long untagged_objp;
+
+	object->flags = OBJECT_ALLOCATED | (is_phys ? OBJECT_PHYS : 0);
+	object->pointer = ptr;
+	object->size = kfence_ksize((void *)ptr) ?: size;
+	object->min_count = min_count;
+	object->jiffies = jiffies;
+
 	untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr);
 	/*
 	 * Only update min_addr and max_addr with object
@@ -1150,6 +1150,7 @@ EXPORT_SYMBOL_GPL(kmemleak_free_percpu);
 void __ref kmemleak_update_trace(const void *ptr)
 {
 	struct kmemleak_object *object;
+	depot_stack_handle_t trace_handle;
 	unsigned long flags;
 
 	pr_debug("%s(0x%px)\n", __func__, ptr);
@@ -1166,8 +1167,9 @@ void __ref kmemleak_update_trace(const void *ptr)
 		return;
 	}
 
+	trace_handle = set_track_prepare();
 	raw_spin_lock_irqsave(&object->lock, flags);
-	object->trace_handle = set_track_prepare();
+	object->trace_handle = trace_handle;
 	raw_spin_unlock_irqrestore(&object->lock, flags);
 
 	put_object(object);
diff --git a/mm/kmsan/core.c b/mm/kmsan/core.c
index c19f47af0424..cf2d70e9c9a5 100644
--- a/mm/kmsan/core.c
+++ b/mm/kmsan/core.c
@@ -76,7 +76,7 @@ depot_stack_handle_t kmsan_save_stack_with_flags(gfp_t flags,
 	/* Don't sleep. */
 	flags &= ~(__GFP_DIRECT_RECLAIM | __GFP_KSWAPD_RECLAIM);
 
-	handle = __stack_depot_save(entries, nr_entries, flags, true);
+	handle = stack_depot_save(entries, nr_entries, flags);
 	return stack_depot_set_extra_bits(handle, extra);
 }
 
@@ -185,11 +185,10 @@ depot_stack_handle_t kmsan_internal_chain_origin(depot_stack_handle_t id)
 	/*
 	 * @entries is a local var in non-instrumented code, so KMSAN does not
 	 * know it is initialized. Explicitly unpoison it to avoid false
-	 * positives when __stack_depot_save() passes it to instrumented code.
+	 * positives when stack_depot_save() passes it to instrumented code.
 	 */
 	kmsan_internal_unpoison_memory(entries, sizeof(entries), false);
-	handle = __stack_depot_save(entries, ARRAY_SIZE(entries), __GFP_HIGH,
-				    true);
+	handle = stack_depot_save(entries, ARRAY_SIZE(entries), __GFP_HIGH);
 	return stack_depot_set_extra_bits(handle, extra_bits);
 }
 
diff --git a/mm/ksm.c b/mm/ksm.c
index 6a831009b4cb..5d60d5385de6 100644
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -1099,9 +1099,9 @@ static int unmerge_and_remove_all_rmap_items(void)
 static u32 calc_checksum(struct page *page)
 {
 	u32 checksum;
-	void *addr = kmap_atomic(page);
+	void *addr = kmap_local_page(page);
 	checksum = xxhash(addr, PAGE_SIZE, 0);
-	kunmap_atomic(addr);
+	kunmap_local(addr);
 	return checksum;
 }
 
diff --git a/mm/madvise.c b/mm/madvise.c
index cf4d694280e9..6214a1ab5654 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -335,6 +335,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
 	struct folio *folio = NULL;
 	LIST_HEAD(folio_list);
 	bool pageout_anon_only_filter;
+	unsigned int batch_count = 0;
 
 	if (fatal_signal_pending(current))
 		return -EINTR;
@@ -416,6 +417,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
 regular_folio:
 #endif
 	tlb_change_page_size(tlb, PAGE_SIZE);
+restart:
 	start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
 	if (!start_pte)
 		return 0;
@@ -424,6 +426,15 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd,
 	for (; addr < end; pte++, addr += PAGE_SIZE) {
 		ptent = ptep_get(pte);
 
+		if (++batch_count == SWAP_CLUSTER_MAX) {
+			batch_count = 0;
+			if (need_resched()) {
+				pte_unmap_unlock(start_pte, ptl);
+				cond_resched();
+				goto restart;
+			}
+		}
+
 		if (pte_none(ptent))
 			continue;
 
diff --git a/mm/memblock.c b/mm/memblock.c
index 5a88d6d24d79..4a62f7774b65 100644
--- a/mm/memblock.c
+++ b/mm/memblock.c
@@ -734,6 +734,40 @@ int __init_memblock memblock_add(phys_addr_t base, phys_addr_t size)
 	return memblock_add_range(&memblock.memory, base, size, MAX_NUMNODES, 0);
 }
 
+/**
+ * memblock_validate_numa_coverage - check if amount of memory with
+ * no node ID assigned is less than a threshold
+ * @threshold_bytes: maximal number of pages that can have unassigned node
+ * ID (in bytes).
+ *
+ * A buggy firmware may report memory that does not belong to any node.
+ * Check if amount of such memory is below @threshold_bytes.
+ *
+ * Return: true on success, false on failure.
+ */
+bool __init_memblock memblock_validate_numa_coverage(unsigned long threshold_bytes)
+{
+	unsigned long nr_pages = 0;
+	unsigned long start_pfn, end_pfn, mem_size_mb;
+	int nid, i;
+
+	/* calculate lose page */
+	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
+		if (nid == NUMA_NO_NODE)
+			nr_pages += end_pfn - start_pfn;
+	}
+
+	if ((nr_pages << PAGE_SHIFT) >= threshold_bytes) {
+		mem_size_mb = memblock_phys_mem_size() >> 20;
+		pr_err("NUMA: no nodes coverage for %luMB of %luMB RAM\n",
+		       (nr_pages << PAGE_SHIFT) >> 20, mem_size_mb);
+		return false;
+	}
+
+	return true;
+}
+
+
 /**
  * memblock_isolate_range - isolate given range into disjoint memblocks
  * @type: memblock type to isolate range for
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 1c1061df9cd1..564aa8f25b71 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -3166,6 +3166,7 @@ __always_inline struct obj_cgroup *current_obj_cgroup(void)
 	return NULL;
 
 from_memcg:
+	objcg = NULL;
 	for (; !mem_cgroup_is_root(memcg); memcg = parent_mem_cgroup(memcg)) {
 		/*
 		 * Memcg pointer is protected by scope (see set_active_memcg())
@@ -3176,7 +3177,6 @@ __always_inline struct obj_cgroup *current_obj_cgroup(void)
 		objcg = rcu_dereference_check(memcg->objcg, 1);
 		if (likely(objcg))
 			break;
-		objcg = NULL;
 	}
 
 	return objcg;
@@ -7866,6 +7866,14 @@ long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
 	return nr_swap_pages;
 }
 
+long mem_cgroup_get_nr_swapcache_pages(struct mem_cgroup *memcg)
+{
+	if (mem_cgroup_disabled())
+		return total_swapcache_pages();
+
+	return memcg_page_state(memcg, NR_SWAPCACHE);
+}
+
 bool mem_cgroup_swap_full(struct folio *folio)
 {
 	struct mem_cgroup *memcg;
diff --git a/mm/memory-failure.c b/mm/memory-failure.c
index 660c21859118..d8c853b35dbb 100644
--- a/mm/memory-failure.c
+++ b/mm/memory-failure.c
@@ -902,39 +902,38 @@ static const char * const action_page_types[] = {
  * The page count will stop it from being freed by unpoison.
  * Stress tests should be aware of this memory leak problem.
  */
-static int delete_from_lru_cache(struct page *p)
+static int delete_from_lru_cache(struct folio *folio)
 {
-	if (isolate_lru_page(p)) {
+	if (folio_isolate_lru(folio)) {
 		/*
 		 * Clear sensible page flags, so that the buddy system won't
-		 * complain when the page is unpoison-and-freed.
+		 * complain when the folio is unpoison-and-freed.
 		 */
-		ClearPageActive(p);
-		ClearPageUnevictable(p);
+		folio_clear_active(folio);
+		folio_clear_unevictable(folio);
 
 		/*
 		 * Poisoned page might never drop its ref count to 0 so we have
 		 * to uncharge it manually from its memcg.
 		 */
-		mem_cgroup_uncharge(page_folio(p));
+		mem_cgroup_uncharge(folio);
 
 		/*
-		 * drop the page count elevated by isolate_lru_page()
+		 * drop the refcount elevated by folio_isolate_lru()
 		 */
-		put_page(p);
+		folio_put(folio);
 		return 0;
 	}
 	return -EIO;
 }
 
-static int truncate_error_page(struct page *p, unsigned long pfn,
+static int truncate_error_folio(struct folio *folio, unsigned long pfn,
 				struct address_space *mapping)
 {
 	int ret = MF_FAILED;
 
-	if (mapping->a_ops->error_remove_page) {
-		struct folio *folio = page_folio(p);
-		int err = mapping->a_ops->error_remove_page(mapping, p);
+	if (mapping->a_ops->error_remove_folio) {
+		int err = mapping->a_ops->error_remove_folio(mapping, folio);
 
 		if (err != 0)
 			pr_info("%#lx: Failed to punch page: %d\n", pfn, err);
@@ -947,7 +946,7 @@ static int truncate_error_page(struct page *p, unsigned long pfn,
 		 * If the file system doesn't support it just invalidate
 		 * This fails on dirty or anything with private pages
 		 */
-		if (invalidate_inode_page(p))
+		if (mapping_evict_folio(mapping, folio))
 			ret = MF_RECOVERED;
 		else
 			pr_info("%#lx: Failed to invalidate\n",	pfn);
@@ -1014,17 +1013,18 @@ static int me_unknown(struct page_state *ps, struct page *p)
  */
 static int me_pagecache_clean(struct page_state *ps, struct page *p)
 {
+	struct folio *folio = page_folio(p);
 	int ret;
 	struct address_space *mapping;
 	bool extra_pins;
 
-	delete_from_lru_cache(p);
+	delete_from_lru_cache(folio);
 
 	/*
-	 * For anonymous pages we're done the only reference left
+	 * For anonymous folios the only reference left
 	 * should be the one m_f() holds.
 	 */
-	if (PageAnon(p)) {
+	if (folio_test_anon(folio)) {
 		ret = MF_RECOVERED;
 		goto out;
 	}
@@ -1036,11 +1036,9 @@ static int me_pagecache_clean(struct page_state *ps, struct page *p)
 	 * has a reference, because it could be file system metadata
 	 * and that's not safe to truncate.
 	 */
-	mapping = page_mapping(p);
+	mapping = folio_mapping(folio);
 	if (!mapping) {
-		/*
-		 * Page has been teared down in the meanwhile
-		 */
+		/* Folio has been torn down in the meantime */
 		ret = MF_FAILED;
 		goto out;
 	}
@@ -1056,12 +1054,12 @@ static int me_pagecache_clean(struct page_state *ps, struct page *p)
 	 *
 	 * Open: to take i_rwsem or not for this? Right now we don't.
 	 */
-	ret = truncate_error_page(p, page_to_pfn(p), mapping);
+	ret = truncate_error_folio(folio, page_to_pfn(p), mapping);
 	if (has_extra_refcount(ps, p, extra_pins))
 		ret = MF_FAILED;
 
 out:
-	unlock_page(p);
+	folio_unlock(folio);
 
 	return ret;
 }
@@ -1139,15 +1137,16 @@ static int me_pagecache_dirty(struct page_state *ps, struct page *p)
  */
 static int me_swapcache_dirty(struct page_state *ps, struct page *p)
 {
+	struct folio *folio = page_folio(p);
 	int ret;
 	bool extra_pins = false;
 
-	ClearPageDirty(p);
+	folio_clear_dirty(folio);
 	/* Trigger EIO in shmem: */
-	ClearPageUptodate(p);
+	folio_clear_uptodate(folio);
 
-	ret = delete_from_lru_cache(p) ? MF_FAILED : MF_DELAYED;
-	unlock_page(p);
+	ret = delete_from_lru_cache(folio) ? MF_FAILED : MF_DELAYED;
+	folio_unlock(folio);
 
 	if (ret == MF_DELAYED)
 		extra_pins = true;
@@ -1165,7 +1164,7 @@ static int me_swapcache_clean(struct page_state *ps, struct page *p)
 
 	delete_from_swap_cache(folio);
 
-	ret = delete_from_lru_cache(p) ? MF_FAILED : MF_RECOVERED;
+	ret = delete_from_lru_cache(folio) ? MF_FAILED : MF_RECOVERED;
 	folio_unlock(folio);
 
 	if (has_extra_refcount(ps, p, false))
@@ -1182,25 +1181,25 @@ static int me_swapcache_clean(struct page_state *ps, struct page *p)
  */
 static int me_huge_page(struct page_state *ps, struct page *p)
 {
+	struct folio *folio = page_folio(p);
 	int res;
-	struct page *hpage = compound_head(p);
 	struct address_space *mapping;
 	bool extra_pins = false;
 
-	mapping = page_mapping(hpage);
+	mapping = folio_mapping(folio);
 	if (mapping) {
-		res = truncate_error_page(hpage, page_to_pfn(p), mapping);
+		res = truncate_error_folio(folio, page_to_pfn(p), mapping);
 		/* The page is kept in page cache. */
 		extra_pins = true;
-		unlock_page(hpage);
+		folio_unlock(folio);
 	} else {
-		unlock_page(hpage);
+		folio_unlock(folio);
 		/*
 		 * migration entry prevents later access on error hugepage,
 		 * so we can free and dissolve it into buddy to save healthy
 		 * subpages.
 		 */
-		put_page(hpage);
+		folio_put(folio);
 		if (__page_handle_poison(p) >= 0) {
 			page_ref_inc(p);
 			res = MF_RECOVERED;
@@ -2602,37 +2601,37 @@ int unpoison_memory(unsigned long pfn)
 }
 EXPORT_SYMBOL(unpoison_memory);
 
-static bool isolate_page(struct page *page, struct list_head *pagelist)
+static bool mf_isolate_folio(struct folio *folio, struct list_head *pagelist)
 {
 	bool isolated = false;
 
-	if (PageHuge(page)) {
-		isolated = isolate_hugetlb(page_folio(page), pagelist);
+	if (folio_test_hugetlb(folio)) {
+		isolated = isolate_hugetlb(folio, pagelist);
 	} else {
-		bool lru = !__PageMovable(page);
+		bool lru = !__folio_test_movable(folio);
 
 		if (lru)
-			isolated = isolate_lru_page(page);
+			isolated = folio_isolate_lru(folio);
 		else
-			isolated = isolate_movable_page(page,
+			isolated = isolate_movable_page(&folio->page,
 							ISOLATE_UNEVICTABLE);
 
 		if (isolated) {
-			list_add(&page->lru, pagelist);
+			list_add(&folio->lru, pagelist);
 			if (lru)
-				inc_node_page_state(page, NR_ISOLATED_ANON +
-						    page_is_file_lru(page));
+				node_stat_add_folio(folio, NR_ISOLATED_ANON +
+						    folio_is_file_lru(folio));
 		}
 	}
 
 	/*
-	 * If we succeed to isolate the page, we grabbed another refcount on
-	 * the page, so we can safely drop the one we got from get_any_page().
-	 * If we failed to isolate the page, it means that we cannot go further
+	 * If we succeed to isolate the folio, we grabbed another refcount on
+	 * the folio, so we can safely drop the one we got from get_any_page().
+	 * If we failed to isolate the folio, it means that we cannot go further
 	 * and we will return an error, so drop the reference we got from
 	 * get_any_page() as well.
 	 */
-	put_page(page);
+	folio_put(folio);
 	return isolated;
 }
 
@@ -2645,40 +2644,40 @@ static int soft_offline_in_use_page(struct page *page)
 {
 	long ret = 0;
 	unsigned long pfn = page_to_pfn(page);
-	struct page *hpage = compound_head(page);
+	struct folio *folio = page_folio(page);
 	char const *msg_page[] = {"page", "hugepage"};
-	bool huge = PageHuge(page);
+	bool huge = folio_test_hugetlb(folio);
 	LIST_HEAD(pagelist);
 	struct migration_target_control mtc = {
 		.nid = NUMA_NO_NODE,
 		.gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL,
 	};
 
-	if (!huge && PageTransHuge(hpage)) {
+	if (!huge && folio_test_large(folio)) {
 		if (try_to_split_thp_page(page)) {
 			pr_info("soft offline: %#lx: thp split failed\n", pfn);
 			return -EBUSY;
 		}
-		hpage = page;
+		folio = page_folio(page);
 	}
 
-	lock_page(page);
+	folio_lock(folio);
 	if (!huge)
-		wait_on_page_writeback(page);
+		folio_wait_writeback(folio);
 	if (PageHWPoison(page)) {
-		unlock_page(page);
-		put_page(page);
+		folio_unlock(folio);
+		folio_put(folio);
 		pr_info("soft offline: %#lx page already poisoned\n", pfn);
 		return 0;
 	}
 
-	if (!huge && PageLRU(page) && !PageSwapCache(page))
+	if (!huge && folio_test_lru(folio) && !folio_test_swapcache(folio))
 		/*
 		 * Try to invalidate first. This should work for
 		 * non dirty unmapped page cache pages.
 		 */
-		ret = invalidate_inode_page(page);
-	unlock_page(page);
+		ret = mapping_evict_folio(folio_mapping(folio), folio);
+	folio_unlock(folio);
 
 	if (ret) {
 		pr_info("soft_offline: %#lx: invalidated\n", pfn);
@@ -2686,7 +2685,7 @@ static int soft_offline_in_use_page(struct page *page)
 		return 0;
 	}
 
-	if (isolate_page(hpage, &pagelist)) {
+	if (mf_isolate_folio(folio, &pagelist)) {
 		ret = migrate_pages(&pagelist, alloc_migration_target, NULL,
 			(unsigned long)&mtc, MIGRATE_SYNC, MR_MEMORY_FAILURE, NULL);
 		if (!ret) {
diff --git a/mm/memory.c b/mm/memory.c
index 1f18ed4a5497..a8ff3489211b 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -374,6 +374,8 @@ void free_pgtables(struct mmu_gather *tlb, struct ma_state *mas,
 		 * be 0.  This will underflow and is okay.
 		 */
 		next = mas_find(mas, ceiling - 1);
+		if (unlikely(xa_is_zero(next)))
+			next = NULL;
 
 		/*
 		 * Hide vma from rmap and truncate_pagecache before freeing
@@ -395,6 +397,8 @@ void free_pgtables(struct mmu_gather *tlb, struct ma_state *mas,
 			       && !is_vm_hugetlb_page(next)) {
 				vma = next;
 				next = mas_find(mas, ceiling - 1);
+				if (unlikely(xa_is_zero(next)))
+					next = NULL;
 				if (mm_wr_locked)
 					vma_start_write(vma);
 				unlink_anon_vmas(vma);
@@ -1517,6 +1521,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb,
 				continue;
 		} else {
 			/* We should have covered all the swap entry types */
+			pr_alert("unrecognized swap entry 0x%lx\n", entry.val);
 			WARN_ON_ONCE(1);
 		}
 		pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
@@ -1743,7 +1748,8 @@ void unmap_vmas(struct mmu_gather *tlb, struct ma_state *mas,
 		unmap_single_vma(tlb, vma, start, end, &details,
 				 mm_wr_locked);
 		hugetlb_zap_end(vma, &details);
-	} while ((vma = mas_find(mas, tree_end - 1)) != NULL);
+		vma = mas_find(mas, tree_end - 1);
+	} while (vma && likely(!xa_is_zero(vma)));
 	mmu_notifier_invalidate_range_end(&range);
 }
 
@@ -2835,7 +2841,8 @@ static inline int __wp_page_copy_user(struct page *dst, struct page *src,
 	 * just copying from the original user address. If that
 	 * fails, we just zero-fill it. Live with it.
 	 */
-	kaddr = kmap_atomic(dst);
+	kaddr = kmap_local_page(dst);
+	pagefault_disable();
 	uaddr = (void __user *)(addr & PAGE_MASK);
 
 	/*
@@ -2903,7 +2910,8 @@ static inline int __wp_page_copy_user(struct page *dst, struct page *src,
 pte_unlock:
 	if (vmf->pte)
 		pte_unmap_unlock(vmf->pte, vmf->ptl);
-	kunmap_atomic(kaddr);
+	pagefault_enable();
+	kunmap_local(kaddr);
 	flush_dcache_page(dst);
 
 	return ret;
@@ -4239,6 +4247,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
 static vm_fault_t __do_fault(struct vm_fault *vmf)
 {
 	struct vm_area_struct *vma = vmf->vma;
+	struct folio *folio;
 	vm_fault_t ret;
 
 	/*
@@ -4267,27 +4276,26 @@ static vm_fault_t __do_fault(struct vm_fault *vmf)
 			    VM_FAULT_DONE_COW)))
 		return ret;
 
+	folio = page_folio(vmf->page);
 	if (unlikely(PageHWPoison(vmf->page))) {
-		struct page *page = vmf->page;
 		vm_fault_t poisonret = VM_FAULT_HWPOISON;
 		if (ret & VM_FAULT_LOCKED) {
-			if (page_mapped(page))
-				unmap_mapping_pages(page_mapping(page),
-						    page->index, 1, false);
-			/* Retry if a clean page was removed from the cache. */
-			if (invalidate_inode_page(page))
+			if (page_mapped(vmf->page))
+				unmap_mapping_folio(folio);
+			/* Retry if a clean folio was removed from the cache. */
+			if (mapping_evict_folio(folio->mapping, folio))
 				poisonret = VM_FAULT_NOPAGE;
-			unlock_page(page);
+			folio_unlock(folio);
 		}
-		put_page(page);
+		folio_put(folio);
 		vmf->page = NULL;
 		return poisonret;
 	}
 
 	if (unlikely(!(ret & VM_FAULT_LOCKED)))
-		lock_page(vmf->page);
+		folio_lock(folio);
 	else
-		VM_BUG_ON_PAGE(!PageLocked(vmf->page), vmf->page);
+		VM_BUG_ON_PAGE(!folio_test_locked(folio), vmf->page);
 
 	return ret;
 }
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index ab41a511e20a..926e1cfb10e9 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -1129,6 +1129,9 @@ void mhp_deinit_memmap_on_memory(unsigned long pfn, unsigned long nr_pages)
 	kasan_remove_zero_shadow(__va(PFN_PHYS(pfn)), PFN_PHYS(nr_pages));
 }
 
+/*
+ * Must be called with mem_hotplug_lock in write mode.
+ */
 int __ref online_pages(unsigned long pfn, unsigned long nr_pages,
 		       struct zone *zone, struct memory_group *group)
 {
@@ -1149,7 +1152,6 @@ int __ref online_pages(unsigned long pfn, unsigned long nr_pages,
 			 !IS_ALIGNED(pfn + nr_pages, PAGES_PER_SECTION)))
 		return -EINVAL;
 
-	mem_hotplug_begin();
 
 	/* associate pfn range with the zone */
 	move_pfn_range_to_zone(zone, pfn, nr_pages, NULL, MIGRATE_ISOLATE);
@@ -1208,7 +1210,6 @@ int __ref online_pages(unsigned long pfn, unsigned long nr_pages,
 	writeback_set_ratelimit();
 
 	memory_notify(MEM_ONLINE, &arg);
-	mem_hotplug_done();
 	return 0;
 
 failed_addition:
@@ -1217,7 +1218,6 @@ int __ref online_pages(unsigned long pfn, unsigned long nr_pages,
 		 (((unsigned long long) pfn + nr_pages) << PAGE_SHIFT) - 1);
 	memory_notify(MEM_CANCEL_ONLINE, &arg);
 	remove_pfn_range_from_zone(zone, pfn, nr_pages);
-	mem_hotplug_done();
 	return ret;
 }
 
@@ -1380,6 +1380,85 @@ static bool mhp_supports_memmap_on_memory(unsigned long size)
 	return arch_supports_memmap_on_memory(vmemmap_size);
 }
 
+static void __ref remove_memory_blocks_and_altmaps(u64 start, u64 size)
+{
+	unsigned long memblock_size = memory_block_size_bytes();
+	u64 cur_start;
+
+	/*
+	 * For memmap_on_memory, the altmaps were added on a per-memblock
+	 * basis; we have to process each individual memory block.
+	 */
+	for (cur_start = start; cur_start < start + size;
+	     cur_start += memblock_size) {
+		struct vmem_altmap *altmap = NULL;
+		struct memory_block *mem;
+
+		mem = find_memory_block(pfn_to_section_nr(PFN_DOWN(cur_start)));
+		if (WARN_ON_ONCE(!mem))
+			continue;
+
+		altmap = mem->altmap;
+		mem->altmap = NULL;
+
+		remove_memory_block_devices(cur_start, memblock_size);
+
+		arch_remove_memory(cur_start, memblock_size, altmap);
+
+		/* Verify that all vmemmap pages have actually been freed. */
+		WARN(altmap->alloc, "Altmap not fully unmapped");
+		kfree(altmap);
+	}
+}
+
+static int create_altmaps_and_memory_blocks(int nid, struct memory_group *group,
+					    u64 start, u64 size)
+{
+	unsigned long memblock_size = memory_block_size_bytes();
+	u64 cur_start;
+	int ret;
+
+	for (cur_start = start; cur_start < start + size;
+	     cur_start += memblock_size) {
+		struct mhp_params params = { .pgprot =
+						     pgprot_mhp(PAGE_KERNEL) };
+		struct vmem_altmap mhp_altmap = {
+			.base_pfn = PHYS_PFN(cur_start),
+			.end_pfn = PHYS_PFN(cur_start + memblock_size - 1),
+		};
+
+		mhp_altmap.free = memory_block_memmap_on_memory_pages();
+		params.altmap = kmemdup(&mhp_altmap, sizeof(struct vmem_altmap),
+					GFP_KERNEL);
+		if (!params.altmap) {
+			ret = -ENOMEM;
+			goto out;
+		}
+
+		/* call arch's memory hotadd */
+		ret = arch_add_memory(nid, cur_start, memblock_size, &params);
+		if (ret < 0) {
+			kfree(params.altmap);
+			goto out;
+		}
+
+		/* create memory block devices after memory was added */
+		ret = create_memory_block_devices(cur_start, memblock_size,
+						  params.altmap, group);
+		if (ret) {
+			arch_remove_memory(cur_start, memblock_size, NULL);
+			kfree(params.altmap);
+			goto out;
+		}
+	}
+
+	return 0;
+out:
+	if (ret && cur_start != start)
+		remove_memory_blocks_and_altmaps(start, cur_start - start);
+	return ret;
+}
+
 /*
  * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
  * and online/offline operations (triggered e.g. by sysfs).
@@ -1390,10 +1469,6 @@ int __ref add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags)
 {
 	struct mhp_params params = { .pgprot = pgprot_mhp(PAGE_KERNEL) };
 	enum memblock_flags memblock_flags = MEMBLOCK_NONE;
-	struct vmem_altmap mhp_altmap = {
-		.base_pfn =  PHYS_PFN(res->start),
-		.end_pfn  =  PHYS_PFN(res->end),
-	};
 	struct memory_group *group = NULL;
 	u64 start, size;
 	bool new_node = false;
@@ -1436,30 +1511,22 @@ int __ref add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags)
 	/*
 	 * Self hosted memmap array
 	 */
-	if (mhp_flags & MHP_MEMMAP_ON_MEMORY) {
-		if (mhp_supports_memmap_on_memory(size)) {
-			mhp_altmap.free = memory_block_memmap_on_memory_pages();
-			params.altmap = kmalloc(sizeof(struct vmem_altmap), GFP_KERNEL);
-			if (!params.altmap) {
-				ret = -ENOMEM;
-				goto error;
-			}
+	if ((mhp_flags & MHP_MEMMAP_ON_MEMORY) &&
+	    mhp_supports_memmap_on_memory(memory_block_size_bytes())) {
+		ret = create_altmaps_and_memory_blocks(nid, group, start, size);
+		if (ret)
+			goto error;
+	} else {
+		ret = arch_add_memory(nid, start, size, &params);
+		if (ret < 0)
+			goto error;
 
-			memcpy(params.altmap, &mhp_altmap, sizeof(mhp_altmap));
+		/* create memory block devices after memory was added */
+		ret = create_memory_block_devices(start, size, NULL, group);
+		if (ret) {
+			arch_remove_memory(start, size, params.altmap);
+			goto error;
 		}
-		/* fallback to not using altmap  */
-	}
-
-	/* call arch's memory hotadd */
-	ret = arch_add_memory(nid, start, size, &params);
-	if (ret < 0)
-		goto error_free;
-
-	/* create memory block devices after memory was added */
-	ret = create_memory_block_devices(start, size, params.altmap, group);
-	if (ret) {
-		arch_remove_memory(start, size, NULL);
-		goto error_free;
 	}
 
 	if (new_node) {
@@ -1496,8 +1563,6 @@ int __ref add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags)
 		walk_memory_blocks(start, size, NULL, online_memory_block);
 
 	return ret;
-error_free:
-	kfree(params.altmap);
 error:
 	if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK))
 		memblock_remove(start, size);
@@ -1863,6 +1928,9 @@ static int count_system_ram_pages_cb(unsigned long start_pfn,
 	return 0;
 }
 
+/*
+ * Must be called with mem_hotplug_lock in write mode.
+ */
 int __ref offline_pages(unsigned long start_pfn, unsigned long nr_pages,
 			struct zone *zone, struct memory_group *group)
 {
@@ -1885,8 +1953,6 @@ int __ref offline_pages(unsigned long start_pfn, unsigned long nr_pages,
 			 !IS_ALIGNED(start_pfn + nr_pages, PAGES_PER_SECTION)))
 		return -EINVAL;
 
-	mem_hotplug_begin();
-
 	/*
 	 * Don't allow to offline memory blocks that contain holes.
 	 * Consequently, memory blocks with holes can never get onlined
@@ -2031,7 +2097,6 @@ int __ref offline_pages(unsigned long start_pfn, unsigned long nr_pages,
 
 	memory_notify(MEM_OFFLINE, &arg);
 	remove_pfn_range_from_zone(zone, start_pfn, nr_pages);
-	mem_hotplug_done();
 	return 0;
 
 failed_removal_isolated:
@@ -2046,7 +2111,6 @@ int __ref offline_pages(unsigned long start_pfn, unsigned long nr_pages,
 		 (unsigned long long) start_pfn << PAGE_SHIFT,
 		 ((unsigned long long) end_pfn << PAGE_SHIFT) - 1,
 		 reason);
-	mem_hotplug_done();
 	return ret;
 }
 
@@ -2068,17 +2132,13 @@ static int check_memblock_offlined_cb(struct memory_block *mem, void *arg)
 	return 0;
 }
 
-static int test_has_altmap_cb(struct memory_block *mem, void *arg)
+static int count_memory_range_altmaps_cb(struct memory_block *mem, void *arg)
 {
-	struct memory_block **mem_ptr = (struct memory_block **)arg;
-	/*
-	 * return the memblock if we have altmap
-	 * and break callback.
-	 */
-	if (mem->altmap) {
-		*mem_ptr = mem;
-		return 1;
-	}
+	u64 *num_altmaps = (u64 *)arg;
+
+	if (mem->altmap)
+		*num_altmaps += 1;
+
 	return 0;
 }
 
@@ -2152,11 +2212,29 @@ void try_offline_node(int nid)
 }
 EXPORT_SYMBOL(try_offline_node);
 
+static int memory_blocks_have_altmaps(u64 start, u64 size)
+{
+	u64 num_memblocks = size / memory_block_size_bytes();
+	u64 num_altmaps = 0;
+
+	if (!mhp_memmap_on_memory())
+		return 0;
+
+	walk_memory_blocks(start, size, &num_altmaps,
+			   count_memory_range_altmaps_cb);
+
+	if (num_altmaps == 0)
+		return 0;
+
+	if (WARN_ON_ONCE(num_memblocks != num_altmaps))
+		return -EINVAL;
+
+	return 1;
+}
+
 static int __ref try_remove_memory(u64 start, u64 size)
 {
-	struct memory_block *mem;
-	int rc = 0, nid = NUMA_NO_NODE;
-	struct vmem_altmap *altmap = NULL;
+	int rc, nid = NUMA_NO_NODE;
 
 	BUG_ON(check_hotplug_memory_range(start, size));
 
@@ -2173,45 +2251,26 @@ static int __ref try_remove_memory(u64 start, u64 size)
 	if (rc)
 		return rc;
 
-	/*
-	 * We only support removing memory added with MHP_MEMMAP_ON_MEMORY in
-	 * the same granularity it was added - a single memory block.
-	 */
-	if (mhp_memmap_on_memory()) {
-		rc = walk_memory_blocks(start, size, &mem, test_has_altmap_cb);
-		if (rc) {
-			if (size != memory_block_size_bytes()) {
-				pr_warn("Refuse to remove %#llx - %#llx,"
-					"wrong granularity\n",
-					start, start + size);
-				return -EINVAL;
-			}
-			altmap = mem->altmap;
-			/*
-			 * Mark altmap NULL so that we can add a debug
-			 * check on memblock free.
-			 */
-			mem->altmap = NULL;
-		}
-	}
-
 	/* remove memmap entry */
 	firmware_map_remove(start, start + size, "System RAM");
 
-	/*
-	 * Memory block device removal under the device_hotplug_lock is
-	 * a barrier against racing online attempts.
-	 */
-	remove_memory_block_devices(start, size);
-
 	mem_hotplug_begin();
 
-	arch_remove_memory(start, size, altmap);
-
-	/* Verify that all vmemmap pages have actually been freed. */
-	if (altmap) {
-		WARN(altmap->alloc, "Altmap not fully unmapped");
-		kfree(altmap);
+	rc = memory_blocks_have_altmaps(start, size);
+	if (rc < 0) {
+		mem_hotplug_done();
+		return rc;
+	} else if (!rc) {
+		/*
+		 * Memory block device removal under the device_hotplug_lock is
+		 * a barrier against racing online attempts.
+		 * No altmaps present, do the removal directly
+		 */
+		remove_memory_block_devices(start, size);
+		arch_remove_memory(start, size, NULL);
+	} else {
+		/* all memblocks in the range have altmaps */
+		remove_memory_blocks_and_altmaps(start, size);
 	}
 
 	if (IS_ENABLED(CONFIG_ARCH_KEEP_MEMBLOCK)) {
diff --git a/mm/mempool.c b/mm/mempool.c
index 734bcf5afbb7..b3d2084fd989 100644
--- a/mm/mempool.c
+++ b/mm/mempool.c
@@ -64,10 +64,10 @@ static void check_element(mempool_t *pool, void *element)
 	} else if (pool->free == mempool_free_pages) {
 		/* Mempools backed by page allocator */
 		int order = (int)(long)pool->pool_data;
-		void *addr = kmap_atomic((struct page *)element);
+		void *addr = kmap_local_page((struct page *)element);
 
 		__check_element(pool, addr, 1UL << (PAGE_SHIFT + order));
-		kunmap_atomic(addr);
+		kunmap_local(addr);
 	}
 }
 
@@ -89,10 +89,10 @@ static void poison_element(mempool_t *pool, void *element)
 	} else if (pool->alloc == mempool_alloc_pages) {
 		/* Mempools backed by page allocator */
 		int order = (int)(long)pool->pool_data;
-		void *addr = kmap_atomic((struct page *)element);
+		void *addr = kmap_local_page((struct page *)element);
 
 		__poison_element(addr, 1UL << (PAGE_SHIFT + order));
-		kunmap_atomic(addr);
+		kunmap_local(addr);
 	}
 }
 #else /* CONFIG_DEBUG_SLAB || CONFIG_SLUB_DEBUG_ON */
diff --git a/mm/memremap.c b/mm/memremap.c
index bee85560a243..9531faa92a7c 100644
--- a/mm/memremap.c
+++ b/mm/memremap.c
@@ -7,6 +7,7 @@
 #include <linux/memremap.h>
 #include <linux/pfn_t.h>
 #include <linux/swap.h>
+#include <linux/mm.h>
 #include <linux/mmzone.h>
 #include <linux/swapops.h>
 #include <linux/types.h>
@@ -422,19 +423,6 @@ void devm_memunmap_pages(struct device *dev, struct dev_pagemap *pgmap)
 }
 EXPORT_SYMBOL_GPL(devm_memunmap_pages);
 
-unsigned long vmem_altmap_offset(struct vmem_altmap *altmap)
-{
-	/* number of pfns from base where pfn_to_page() is valid */
-	if (altmap)
-		return altmap->reserve + altmap->free;
-	return 0;
-}
-
-void vmem_altmap_free(struct vmem_altmap *altmap, unsigned long nr_pfns)
-{
-	altmap->alloc -= nr_pfns;
-}
-
 /**
  * get_dev_pagemap() - take a new live reference on the dev_pagemap for @pfn
  * @pfn: page frame number to lookup page_map
diff --git a/mm/migrate.c b/mm/migrate.c
index 35a88334bb3c..d9d2b9432e81 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -746,7 +746,7 @@ static int __buffer_migrate_folio(struct address_space *mapping,
 
 recheck_buffers:
 		busy = false;
-		spin_lock(&mapping->private_lock);
+		spin_lock(&mapping->i_private_lock);
 		bh = head;
 		do {
 			if (atomic_read(&bh->b_count)) {
@@ -760,7 +760,7 @@ static int __buffer_migrate_folio(struct address_space *mapping,
 				rc = -EAGAIN;
 				goto unlock_buffers;
 			}
-			spin_unlock(&mapping->private_lock);
+			spin_unlock(&mapping->i_private_lock);
 			invalidate_bh_lrus();
 			invalidated = true;
 			goto recheck_buffers;
@@ -787,7 +787,7 @@ static int __buffer_migrate_folio(struct address_space *mapping,
 	rc = MIGRATEPAGE_SUCCESS;
 unlock_buffers:
 	if (check_refs)
-		spin_unlock(&mapping->private_lock);
+		spin_unlock(&mapping->i_private_lock);
 	bh = head;
 	do {
 		unlock_buffer(bh);
@@ -955,6 +955,8 @@ static int move_to_new_folio(struct folio *dst, struct folio *src,
 
 		if (!mapping)
 			rc = migrate_folio(mapping, dst, src, mode);
+		else if (mapping_unmovable(mapping))
+			rc = -EOPNOTSUPP;
 		else if (mapping->a_ops->migrate_folio)
 			/*
 			 * Most folios have a mapping and most filesystems
diff --git a/mm/mmap.c b/mm/mmap.c
index 1971bfffcc03..e8470b6b678c 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -2207,42 +2207,7 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned lon
 }
 #endif
 
-/*
- * IA64 has some horrid mapping rules: it can expand both up and down,
- * but with various special rules.
- *
- * We'll get rid of this architecture eventually, so the ugliness is
- * temporary.
- */
-#ifdef CONFIG_IA64
-static inline bool vma_expand_ok(struct vm_area_struct *vma, unsigned long addr)
-{
-	return REGION_NUMBER(addr) == REGION_NUMBER(vma->vm_start) &&
-		REGION_OFFSET(addr) < RGN_MAP_LIMIT;
-}
-
-/*
- * IA64 stacks grow down, but there's a special register backing store
- * that can grow up. Only sequentially, though, so the new address must
- * match vm_end.
- */
-static inline int vma_expand_up(struct vm_area_struct *vma, unsigned long addr)
-{
-	if (!vma_expand_ok(vma, addr))
-		return -EFAULT;
-	if (vma->vm_end != (addr & PAGE_MASK))
-		return -EFAULT;
-	return expand_upwards(vma, addr);
-}
-
-static inline bool vma_expand_down(struct vm_area_struct *vma, unsigned long addr)
-{
-	if (!vma_expand_ok(vma, addr))
-		return -EFAULT;
-	return expand_downwards(vma, addr);
-}
-
-#elif defined(CONFIG_STACK_GROWSUP)
+#if defined(CONFIG_STACK_GROWSUP)
 
 #define vma_expand_up(vma,addr) expand_upwards(vma, addr)
 #define vma_expand_down(vma, addr) (-EFAULT)
@@ -3294,10 +3259,11 @@ void exit_mmap(struct mm_struct *mm)
 	arch_exit_mmap(mm);
 
 	vma = mas_find(&mas, ULONG_MAX);
-	if (!vma) {
+	if (!vma || unlikely(xa_is_zero(vma))) {
 		/* Can happen if dup_mmap() received an OOM */
 		mmap_read_unlock(mm);
-		return;
+		mmap_write_lock(mm);
+		goto destroy;
 	}
 
 	lru_add_drain();
@@ -3332,11 +3298,13 @@ void exit_mmap(struct mm_struct *mm)
 		remove_vma(vma, true);
 		count++;
 		cond_resched();
-	} while ((vma = mas_find(&mas, ULONG_MAX)) != NULL);
+		vma = mas_find(&mas, ULONG_MAX);
+	} while (vma && likely(!xa_is_zero(vma)));
 
 	BUG_ON(count != mm->map_count);
 
 	trace_exit_mmap(mm);
+destroy:
 	__mt_destroy(&mm->mm_mt);
 	mmap_write_unlock(mm);
 	vm_unacct_memory(nr_accounted);
diff --git a/mm/page-writeback.c b/mm/page-writeback.c
index 46f2f5d3d183..ca64bd513fa2 100644
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -2982,67 +2982,63 @@ bool __folio_end_writeback(struct folio *folio)
 	return ret;
 }
 
-bool __folio_start_writeback(struct folio *folio, bool keep_write)
+void __folio_start_writeback(struct folio *folio, bool keep_write)
 {
 	long nr = folio_nr_pages(folio);
 	struct address_space *mapping = folio_mapping(folio);
-	bool ret;
 	int access_ret;
 
+	VM_BUG_ON_FOLIO(folio_test_writeback(folio), folio);
+
 	folio_memcg_lock(folio);
 	if (mapping && mapping_use_writeback_tags(mapping)) {
 		XA_STATE(xas, &mapping->i_pages, folio_index(folio));
 		struct inode *inode = mapping->host;
 		struct backing_dev_info *bdi = inode_to_bdi(inode);
 		unsigned long flags;
+		bool on_wblist;
 
 		xas_lock_irqsave(&xas, flags);
 		xas_load(&xas);
-		ret = folio_test_set_writeback(folio);
-		if (!ret) {
-			bool on_wblist;
+		folio_test_set_writeback(folio);
 
-			on_wblist = mapping_tagged(mapping,
-						   PAGECACHE_TAG_WRITEBACK);
+		on_wblist = mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK);
 
-			xas_set_mark(&xas, PAGECACHE_TAG_WRITEBACK);
-			if (bdi->capabilities & BDI_CAP_WRITEBACK_ACCT) {
-				struct bdi_writeback *wb = inode_to_wb(inode);
-
-				wb_stat_mod(wb, WB_WRITEBACK, nr);
-				if (!on_wblist)
-					wb_inode_writeback_start(wb);
-			}
+		xas_set_mark(&xas, PAGECACHE_TAG_WRITEBACK);
+		if (bdi->capabilities & BDI_CAP_WRITEBACK_ACCT) {
+			struct bdi_writeback *wb = inode_to_wb(inode);
 
-			/*
-			 * We can come through here when swapping
-			 * anonymous folios, so we don't necessarily
-			 * have an inode to track for sync.
-			 */
-			if (mapping->host && !on_wblist)
-				sb_mark_inode_writeback(mapping->host);
+			wb_stat_mod(wb, WB_WRITEBACK, nr);
+			if (!on_wblist)
+				wb_inode_writeback_start(wb);
 		}
+
+		/*
+		 * We can come through here when swapping anonymous
+		 * folios, so we don't necessarily have an inode to
+		 * track for sync.
+		 */
+		if (mapping->host && !on_wblist)
+			sb_mark_inode_writeback(mapping->host);
 		if (!folio_test_dirty(folio))
 			xas_clear_mark(&xas, PAGECACHE_TAG_DIRTY);
 		if (!keep_write)
 			xas_clear_mark(&xas, PAGECACHE_TAG_TOWRITE);
 		xas_unlock_irqrestore(&xas, flags);
 	} else {
-		ret = folio_test_set_writeback(folio);
-	}
-	if (!ret) {
-		lruvec_stat_mod_folio(folio, NR_WRITEBACK, nr);
-		zone_stat_mod_folio(folio, NR_ZONE_WRITE_PENDING, nr);
+		folio_test_set_writeback(folio);
 	}
+
+	lruvec_stat_mod_folio(folio, NR_WRITEBACK, nr);
+	zone_stat_mod_folio(folio, NR_ZONE_WRITE_PENDING, nr);
 	folio_memcg_unlock(folio);
+
 	access_ret = arch_make_folio_accessible(folio);
 	/*
 	 * If writeback has been triggered on a page that cannot be made
 	 * accessible, it is too late to recover here.
 	 */
 	VM_BUG_ON_FOLIO(access_ret != 0, folio);
-
-	return ret;
 }
 EXPORT_SYMBOL(__folio_start_writeback);
 
@@ -3107,7 +3103,7 @@ EXPORT_SYMBOL_GPL(folio_wait_writeback_killable);
  */
 void folio_wait_stable(struct folio *folio)
 {
-	if (folio_inode(folio)->i_sb->s_iflags & SB_I_STABLE_WRITES)
+	if (mapping_stable_writes(folio_mapping(folio)))
 		folio_wait_writeback(folio);
 }
 EXPORT_SYMBOL_GPL(folio_wait_stable);
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 733732e7e0ba..7a8dac0c1c74 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1086,13 +1086,11 @@ static __always_inline bool free_pages_prepare(struct page *page,
 	trace_mm_page_free(page, order);
 	kmsan_free_page(page, order);
 
+	if (memcg_kmem_online() && PageMemcgKmem(page))
+		__memcg_kmem_uncharge_page(page, order);
+
 	if (unlikely(PageHWPoison(page)) && !order) {
-		/*
-		 * Do not let hwpoison pages hit pcplists/buddy
-		 * Untie memcg state and reset page's owner
-		 */
-		if (memcg_kmem_online() && PageMemcgKmem(page))
-			__memcg_kmem_uncharge_page(page, order);
+		/* Do not let hwpoison pages hit pcplists/buddy */
 		reset_page_owner(page, order);
 		page_table_check_free(page, order);
 		return false;
@@ -1123,8 +1121,6 @@ static __always_inline bool free_pages_prepare(struct page *page,
 	}
 	if (PageMappingFlags(page))
 		page->mapping = NULL;
-	if (memcg_kmem_online() && PageMemcgKmem(page))
-		__memcg_kmem_uncharge_page(page, order);
 	if (is_check_pages_enabled()) {
 		if (free_page_is_bad(page))
 			bad++;
@@ -2079,6 +2075,43 @@ __rmqueue_fallback(struct zone *zone, int order, int start_migratetype,
 
 }
 
+#ifdef CONFIG_CMA
+/*
+ * GFP_MOVABLE allocation could drain UNMOVABLE & RECLAIMABLE page blocks via
+ * the help of CMA which makes GFP_KERNEL failed. Checking if zone_watermark_ok
+ * again without ALLOC_CMA to see if to use CMA first.
+ */
+static bool use_cma_first(struct zone *zone, unsigned int order, unsigned int alloc_flags)
+{
+	unsigned long watermark;
+	bool cma_first = false;
+
+	watermark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
+	/* check if GFP_MOVABLE pass previous zone_watermark_ok via the help of CMA */
+	if (zone_watermark_ok(zone, order, watermark, 0, alloc_flags & (~ALLOC_CMA))) {
+		/*
+		 * Balance movable allocations between regular and CMA areas by
+		 * allocating from CMA when over half of the zone's free memory
+		 * is in the CMA area.
+		 */
+		cma_first = (zone_page_state(zone, NR_FREE_CMA_PAGES) >
+				zone_page_state(zone, NR_FREE_PAGES) / 2);
+	} else {
+		/*
+		 * watermark failed means UNMOVABLE & RECLAIMBLE is not enough
+		 * now, we should use cma first to keep them stay around the
+		 * corresponding watermark
+		 */
+		cma_first = true;
+	}
+	return cma_first;
+}
+#else
+static bool use_cma_first(struct zone *zone, unsigned int order, unsigned int alloc_flags)
+{
+	return false;
+}
+#endif
 /*
  * Do the hard work of removing an element from the buddy allocator.
  * Call me with the zone->lock already held.
@@ -2092,12 +2125,11 @@ __rmqueue(struct zone *zone, unsigned int order, int migratetype,
 	if (IS_ENABLED(CONFIG_CMA)) {
 		/*
 		 * Balance movable allocations between regular and CMA areas by
-		 * allocating from CMA when over half of the zone's free memory
-		 * is in the CMA area.
+		 * allocating from CMA base on judging zone_watermark_ok again
+		 * to see if the latest check got pass via the help of CMA
 		 */
 		if (alloc_flags & ALLOC_CMA &&
-		    zone_page_state(zone, NR_FREE_CMA_PAGES) >
-		    zone_page_state(zone, NR_FREE_PAGES) / 2) {
+			use_cma_first(zone, order, alloc_flags)) {
 			page = __rmqueue_cma_fallback(zone, order);
 			if (page)
 				return page;
diff --git a/mm/page_owner.c b/mm/page_owner.c
index 4f13ce7d2452..e7eba7688881 100644
--- a/mm/page_owner.c
+++ b/mm/page_owner.c
@@ -32,6 +32,8 @@ struct page_owner {
 	char comm[TASK_COMM_LEN];
 	pid_t pid;
 	pid_t tgid;
+	pid_t free_pid;
+	pid_t free_tgid;
 };
 
 static bool page_owner_enabled __initdata;
@@ -152,6 +154,8 @@ void __reset_page_owner(struct page *page, unsigned short order)
 		page_owner = get_page_owner(page_ext);
 		page_owner->free_handle = handle;
 		page_owner->free_ts_nsec = free_ts_nsec;
+		page_owner->free_pid = current->pid;
+		page_owner->free_tgid = current->tgid;
 		page_ext = page_ext_next(page_ext);
 	}
 	page_ext_put(page_ext);
@@ -253,6 +257,8 @@ void __folio_copy_owner(struct folio *newfolio, struct folio *old)
 	new_page_owner->handle = old_page_owner->handle;
 	new_page_owner->pid = old_page_owner->pid;
 	new_page_owner->tgid = old_page_owner->tgid;
+	new_page_owner->free_pid = old_page_owner->free_pid;
+	new_page_owner->free_tgid = old_page_owner->free_tgid;
 	new_page_owner->ts_nsec = old_page_owner->ts_nsec;
 	new_page_owner->free_ts_nsec = old_page_owner->ts_nsec;
 	strcpy(new_page_owner->comm, old_page_owner->comm);
@@ -495,7 +501,8 @@ void __dump_page_owner(const struct page *page)
 	if (!handle) {
 		pr_alert("page_owner free stack trace missing\n");
 	} else {
-		pr_alert("page last free stack trace:\n");
+		pr_alert("page last free pid %d tgid %d stack trace:\n",
+			  page_owner->free_pid, page_owner->free_tgid);
 		stack_depot_print(handle);
 	}
 
diff --git a/mm/page_poison.c b/mm/page_poison.c
index b4f456437b7e..3e9037363cf9 100644
--- a/mm/page_poison.c
+++ b/mm/page_poison.c
@@ -21,13 +21,13 @@ early_param("page_poison", early_page_poison_param);
 
 static void poison_page(struct page *page)
 {
-	void *addr = kmap_atomic(page);
+	void *addr = kmap_local_page(page);
 
 	/* KASAN still think the page is in-use, so skip it. */
 	kasan_disable_current();
 	memset(kasan_reset_tag(addr), PAGE_POISON, PAGE_SIZE);
 	kasan_enable_current();
-	kunmap_atomic(addr);
+	kunmap_local(addr);
 }
 
 void __kernel_poison_pages(struct page *page, int n)
@@ -77,7 +77,7 @@ static void unpoison_page(struct page *page)
 {
 	void *addr;
 
-	addr = kmap_atomic(page);
+	addr = kmap_local_page(page);
 	kasan_disable_current();
 	/*
 	 * Page poisoning when enabled poisons each and every page
@@ -86,7 +86,7 @@ static void unpoison_page(struct page *page)
 	 */
 	check_poison_mem(page, kasan_reset_tag(addr), PAGE_SIZE);
 	kasan_enable_current();
-	kunmap_atomic(addr);
+	kunmap_local(addr);
 }
 
 void __kernel_unpoison_pages(struct page *page, int n)
diff --git a/mm/process_vm_access.c b/mm/process_vm_access.c
index 0523edab03a6..b308e96cd05a 100644
--- a/mm/process_vm_access.c
+++ b/mm/process_vm_access.c
@@ -53,7 +53,10 @@ static int process_vm_rw_pages(struct page **pages,
 }
 
 /* Maximum number of pages kmalloc'd to hold struct page's during copy */
-#define PVM_MAX_KMALLOC_PAGES (PAGE_SIZE * 2)
+#define PVM_MAX_KMALLOC_PAGES 2
+
+/* Maximum number of pages that can be stored at a time */
+#define PVM_MAX_USER_PAGES (PVM_MAX_KMALLOC_PAGES * PAGE_SIZE / sizeof(struct page *))
 
 /**
  * process_vm_rw_single_vec - read/write pages from task specified
@@ -79,8 +82,6 @@ static int process_vm_rw_single_vec(unsigned long addr,
 	unsigned long start_offset = addr - pa;
 	unsigned long nr_pages;
 	ssize_t rc = 0;
-	unsigned long max_pages_per_loop = PVM_MAX_KMALLOC_PAGES
-		/ sizeof(struct pages *);
 	unsigned int flags = 0;
 
 	/* Work out address and page range required */
@@ -92,7 +93,7 @@ static int process_vm_rw_single_vec(unsigned long addr,
 		flags |= FOLL_WRITE;
 
 	while (!rc && nr_pages && iov_iter_count(iter)) {
-		int pinned_pages = min(nr_pages, max_pages_per_loop);
+		int pinned_pages = min_t(unsigned long, nr_pages, PVM_MAX_USER_PAGES);
 		int locked = 1;
 		size_t bytes;
 
@@ -171,7 +172,7 @@ static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter,
 		iov_len = rvec[i].iov_len;
 		if (iov_len > 0) {
 			nr_pages_iov = ((unsigned long)rvec[i].iov_base
-					+ iov_len)
+					+ iov_len - 1)
 				/ PAGE_SIZE - (unsigned long)rvec[i].iov_base
 				/ PAGE_SIZE + 1;
 			nr_pages = max(nr_pages, nr_pages_iov);
@@ -184,8 +185,8 @@ static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter,
 	if (nr_pages > PVM_MAX_PP_ARRAY_COUNT) {
 		/* For reliability don't try to kmalloc more than
 		   2 pages worth */
-		process_pages = kmalloc(min_t(size_t, PVM_MAX_KMALLOC_PAGES,
-					      sizeof(struct pages *)*nr_pages),
+		process_pages = kmalloc(min_t(size_t, PVM_MAX_KMALLOC_PAGES * PAGE_SIZE,
+					      sizeof(struct page *)*nr_pages),
 					GFP_KERNEL);
 
 		if (!process_pages)
diff --git a/mm/shmem.c b/mm/shmem.c
index 91e2620148b2..c62f904ba1ca 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -1080,7 +1080,24 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
 				}
 				VM_BUG_ON_FOLIO(folio_test_writeback(folio),
 						folio);
-				truncate_inode_folio(mapping, folio);
+
+				if (!folio_test_large(folio)) {
+					truncate_inode_folio(mapping, folio);
+				} else if (truncate_inode_partial_folio(folio, lstart, lend)) {
+					/*
+					 * If we split a page, reset the loop so
+					 * that we pick up the new sub pages.
+					 * Otherwise the THP was entirely
+					 * dropped or the target range was
+					 * zeroed, so just continue the loop as
+					 * is.
+					 */
+					if (!folio_test_large(folio)) {
+						folio_unlock(folio);
+						index = start;
+						break;
+					}
+				}
 			}
 			folio_unlock(folio);
 		}
@@ -4445,8 +4462,8 @@ static void __init shmem_destroy_inodecache(void)
 }
 
 /* Keep the page in page cache instead of truncating it */
-static int shmem_error_remove_page(struct address_space *mapping,
-				   struct page *page)
+static int shmem_error_remove_folio(struct address_space *mapping,
+				   struct folio *folio)
 {
 	return 0;
 }
@@ -4461,7 +4478,7 @@ const struct address_space_operations shmem_aops = {
 #ifdef CONFIG_MIGRATION
 	.migrate_folio	= migrate_folio,
 #endif
-	.error_remove_page = shmem_error_remove_page,
+	.error_remove_folio = shmem_error_remove_folio,
 };
 EXPORT_SYMBOL(shmem_aops);
 
diff --git a/mm/slub.c b/mm/slub.c
index 63d281dfacdb..a307d319e82c 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -76,13 +76,22 @@
  *
  *   Frozen slabs
  *
- *   If a slab is frozen then it is exempt from list management. It is not
- *   on any list except per cpu partial list. The processor that froze the
+ *   If a slab is frozen then it is exempt from list management. It is
+ *   the cpu slab which is actively allocated from by the processor that
+ *   froze it and it is not on any list. The processor that froze the
  *   slab is the one who can perform list operations on the slab. Other
  *   processors may put objects onto the freelist but the processor that
  *   froze the slab is the only one that can retrieve the objects from the
  *   slab's freelist.
  *
+ *   CPU partial slabs
+ *
+ *   The partially empty slabs cached on the CPU partial list are used
+ *   for performance reasons, which speeds up the allocation process.
+ *   These slabs are not frozen, but are also exempt from list management,
+ *   by clearing the PG_workingset flag when moving out of the node
+ *   partial list. Please see __slab_free() for more details.
+ *
  *   list_lock
  *
  *   The list_lock protects the partial and full list on each node and
@@ -204,9 +213,9 @@ DEFINE_STATIC_KEY_FALSE(slub_debug_enabled);
 
 /* Structure holding parameters for get_partial() call chain */
 struct partial_context {
-	struct slab **slab;
 	gfp_t flags;
 	unsigned int orig_size;
+	void *object;
 };
 
 static inline bool kmem_cache_debug(struct kmem_cache *s)
@@ -2116,6 +2125,25 @@ static void discard_slab(struct kmem_cache *s, struct slab *slab)
 	free_slab(s, slab);
 }
 
+/*
+ * SLUB reuses PG_workingset bit to keep track of whether it's on
+ * the per-node partial list.
+ */
+static inline bool slab_test_node_partial(const struct slab *slab)
+{
+	return folio_test_workingset((struct folio *)slab_folio(slab));
+}
+
+static inline void slab_set_node_partial(struct slab *slab)
+{
+	__set_bit(PG_workingset, folio_flags(slab_folio(slab), 0));
+}
+
+static inline void slab_clear_node_partial(struct slab *slab)
+{
+	__clear_bit(PG_workingset, folio_flags(slab_folio(slab), 0));
+}
+
 /*
  * Management of partially allocated slabs.
  */
@@ -2127,6 +2155,7 @@ __add_partial(struct kmem_cache_node *n, struct slab *slab, int tail)
 		list_add_tail(&slab->slab_list, &n->partial);
 	else
 		list_add(&slab->slab_list, &n->partial);
+	slab_set_node_partial(slab);
 }
 
 static inline void add_partial(struct kmem_cache_node *n,
@@ -2141,6 +2170,7 @@ static inline void remove_partial(struct kmem_cache_node *n,
 {
 	lockdep_assert_held(&n->list_lock);
 	list_del(&slab->slab_list);
+	slab_clear_node_partial(slab);
 	n->nr_partial--;
 }
 
@@ -2213,51 +2243,6 @@ static void *alloc_single_from_new_slab(struct kmem_cache *s,
 	return object;
 }
 
-/*
- * Remove slab from the partial list, freeze it and
- * return the pointer to the freelist.
- *
- * Returns a list of objects or NULL if it fails.
- */
-static inline void *acquire_slab(struct kmem_cache *s,
-		struct kmem_cache_node *n, struct slab *slab,
-		int mode)
-{
-	void *freelist;
-	unsigned long counters;
-	struct slab new;
-
-	lockdep_assert_held(&n->list_lock);
-
-	/*
-	 * Zap the freelist and set the frozen bit.
-	 * The old freelist is the list of objects for the
-	 * per cpu allocation list.
-	 */
-	freelist = slab->freelist;
-	counters = slab->counters;
-	new.counters = counters;
-	if (mode) {
-		new.inuse = slab->objects;
-		new.freelist = NULL;
-	} else {
-		new.freelist = freelist;
-	}
-
-	VM_BUG_ON(new.frozen);
-	new.frozen = 1;
-
-	if (!__slab_update_freelist(s, slab,
-			freelist, counters,
-			new.freelist, new.counters,
-			"acquire_slab"))
-		return NULL;
-
-	remove_partial(n, slab);
-	WARN_ON(!freelist);
-	return freelist;
-}
-
 #ifdef CONFIG_SLUB_CPU_PARTIAL
 static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain);
 #else
@@ -2269,11 +2254,11 @@ static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags);
 /*
  * Try to allocate a partial slab from a specific node.
  */
-static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
-			      struct partial_context *pc)
+static struct slab *get_partial_node(struct kmem_cache *s,
+				     struct kmem_cache_node *n,
+				     struct partial_context *pc)
 {
-	struct slab *slab, *slab2;
-	void *object = NULL;
+	struct slab *slab, *slab2, *partial = NULL;
 	unsigned long flags;
 	unsigned int partial_slabs = 0;
 
@@ -2288,27 +2273,25 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
 
 	spin_lock_irqsave(&n->list_lock, flags);
 	list_for_each_entry_safe(slab, slab2, &n->partial, slab_list) {
-		void *t;
-
 		if (!pfmemalloc_match(slab, pc->flags))
 			continue;
 
 		if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) {
-			object = alloc_single_from_partial(s, n, slab,
+			void *object = alloc_single_from_partial(s, n, slab,
 							pc->orig_size);
-			if (object)
+			if (object) {
+				partial = slab;
+				pc->object = object;
 				break;
+			}
 			continue;
 		}
 
-		t = acquire_slab(s, n, slab, object == NULL);
-		if (!t)
-			break;
+		remove_partial(n, slab);
 
-		if (!object) {
-			*pc->slab = slab;
+		if (!partial) {
+			partial = slab;
 			stat(s, ALLOC_FROM_PARTIAL);
-			object = t;
 		} else {
 			put_cpu_partial(s, slab, 0);
 			stat(s, CPU_PARTIAL_NODE);
@@ -2324,20 +2307,21 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
 
 	}
 	spin_unlock_irqrestore(&n->list_lock, flags);
-	return object;
+	return partial;
 }
 
 /*
  * Get a slab from somewhere. Search in increasing NUMA distances.
  */
-static void *get_any_partial(struct kmem_cache *s, struct partial_context *pc)
+static struct slab *get_any_partial(struct kmem_cache *s,
+				    struct partial_context *pc)
 {
 #ifdef CONFIG_NUMA
 	struct zonelist *zonelist;
 	struct zoneref *z;
 	struct zone *zone;
 	enum zone_type highest_zoneidx = gfp_zone(pc->flags);
-	void *object;
+	struct slab *slab;
 	unsigned int cpuset_mems_cookie;
 
 	/*
@@ -2372,8 +2356,8 @@ static void *get_any_partial(struct kmem_cache *s, struct partial_context *pc)
 
 			if (n && cpuset_zone_allowed(zone, pc->flags) &&
 					n->nr_partial > s->min_partial) {
-				object = get_partial_node(s, n, pc);
-				if (object) {
+				slab = get_partial_node(s, n, pc);
+				if (slab) {
 					/*
 					 * Don't check read_mems_allowed_retry()
 					 * here - if mems_allowed was updated in
@@ -2381,7 +2365,7 @@ static void *get_any_partial(struct kmem_cache *s, struct partial_context *pc)
 					 * between allocation and the cpuset
 					 * update
 					 */
-					return object;
+					return slab;
 				}
 			}
 		}
@@ -2393,17 +2377,18 @@ static void *get_any_partial(struct kmem_cache *s, struct partial_context *pc)
 /*
  * Get a partial slab, lock it and return it.
  */
-static void *get_partial(struct kmem_cache *s, int node, struct partial_context *pc)
+static struct slab *get_partial(struct kmem_cache *s, int node,
+				struct partial_context *pc)
 {
-	void *object;
+	struct slab *slab;
 	int searchnode = node;
 
 	if (node == NUMA_NO_NODE)
 		searchnode = numa_mem_id();
 
-	object = get_partial_node(s, get_node(s, searchnode), pc);
-	if (object || node != NUMA_NO_NODE)
-		return object;
+	slab = get_partial_node(s, get_node(s, searchnode), pc);
+	if (slab || node != NUMA_NO_NODE)
+		return slab;
 
 	return get_any_partial(s, pc);
 }
@@ -2492,10 +2477,8 @@ static void init_kmem_cache_cpus(struct kmem_cache *s)
 static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
 			    void *freelist)
 {
-	enum slab_modes { M_NONE, M_PARTIAL, M_FREE, M_FULL_NOLIST };
 	struct kmem_cache_node *n = get_node(s, slab_nid(slab));
 	int free_delta = 0;
-	enum slab_modes mode = M_NONE;
 	void *nextfree, *freelist_iter, *freelist_tail;
 	int tail = DEACTIVATE_TO_HEAD;
 	unsigned long flags = 0;
@@ -2533,80 +2516,52 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
 	/*
 	 * Stage two: Unfreeze the slab while splicing the per-cpu
 	 * freelist to the head of slab's freelist.
-	 *
-	 * Ensure that the slab is unfrozen while the list presence
-	 * reflects the actual number of objects during unfreeze.
-	 *
-	 * We first perform cmpxchg holding lock and insert to list
-	 * when it succeed. If there is mismatch then the slab is not
-	 * unfrozen and number of objects in the slab may have changed.
-	 * Then release lock and retry cmpxchg again.
 	 */
-redo:
-
-	old.freelist = READ_ONCE(slab->freelist);
-	old.counters = READ_ONCE(slab->counters);
-	VM_BUG_ON(!old.frozen);
-
-	/* Determine target state of the slab */
-	new.counters = old.counters;
-	if (freelist_tail) {
-		new.inuse -= free_delta;
-		set_freepointer(s, freelist_tail, old.freelist);
-		new.freelist = freelist;
-	} else
-		new.freelist = old.freelist;
-
-	new.frozen = 0;
+	do {
+		old.freelist = READ_ONCE(slab->freelist);
+		old.counters = READ_ONCE(slab->counters);
+		VM_BUG_ON(!old.frozen);
+
+		/* Determine target state of the slab */
+		new.counters = old.counters;
+		new.frozen = 0;
+		if (freelist_tail) {
+			new.inuse -= free_delta;
+			set_freepointer(s, freelist_tail, old.freelist);
+			new.freelist = freelist;
+		} else {
+			new.freelist = old.freelist;
+		}
+	} while (!slab_update_freelist(s, slab,
+		old.freelist, old.counters,
+		new.freelist, new.counters,
+		"unfreezing slab"));
 
+	/*
+	 * Stage three: Manipulate the slab list based on the updated state.
+	 */
 	if (!new.inuse && n->nr_partial >= s->min_partial) {
-		mode = M_FREE;
+		stat(s, DEACTIVATE_EMPTY);
+		discard_slab(s, slab);
+		stat(s, FREE_SLAB);
 	} else if (new.freelist) {
-		mode = M_PARTIAL;
-		/*
-		 * Taking the spinlock removes the possibility that
-		 * acquire_slab() will see a slab that is frozen
-		 */
 		spin_lock_irqsave(&n->list_lock, flags);
-	} else {
-		mode = M_FULL_NOLIST;
-	}
-
-
-	if (!slab_update_freelist(s, slab,
-				old.freelist, old.counters,
-				new.freelist, new.counters,
-				"unfreezing slab")) {
-		if (mode == M_PARTIAL)
-			spin_unlock_irqrestore(&n->list_lock, flags);
-		goto redo;
-	}
-
-
-	if (mode == M_PARTIAL) {
 		add_partial(n, slab, tail);
 		spin_unlock_irqrestore(&n->list_lock, flags);
 		stat(s, tail);
-	} else if (mode == M_FREE) {
-		stat(s, DEACTIVATE_EMPTY);
-		discard_slab(s, slab);
-		stat(s, FREE_SLAB);
-	} else if (mode == M_FULL_NOLIST) {
+	} else {
 		stat(s, DEACTIVATE_FULL);
 	}
 }
 
 #ifdef CONFIG_SLUB_CPU_PARTIAL
-static void __unfreeze_partials(struct kmem_cache *s, struct slab *partial_slab)
+static void __put_partials(struct kmem_cache *s, struct slab *partial_slab)
 {
 	struct kmem_cache_node *n = NULL, *n2 = NULL;
 	struct slab *slab, *slab_to_discard = NULL;
 	unsigned long flags = 0;
 
 	while (partial_slab) {
-		struct slab new;
-		struct slab old;
-
 		slab = partial_slab;
 		partial_slab = slab->next;
 
@@ -2619,23 +2574,7 @@ static void __unfreeze_partials(struct kmem_cache *s, struct slab *partial_slab)
 			spin_lock_irqsave(&n->list_lock, flags);
 		}
 
-		do {
-
-			old.freelist = slab->freelist;
-			old.counters = slab->counters;
-			VM_BUG_ON(!old.frozen);
-
-			new.counters = old.counters;
-			new.freelist = old.freelist;
-
-			new.frozen = 0;
-
-		} while (!__slab_update_freelist(s, slab,
-				old.freelist, old.counters,
-				new.freelist, new.counters,
-				"unfreezing slab"));
-
-		if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) {
+		if (unlikely(!slab->inuse && n->nr_partial >= s->min_partial)) {
 			slab->next = slab_to_discard;
 			slab_to_discard = slab;
 		} else {
@@ -2658,9 +2597,9 @@ static void __unfreeze_partials(struct kmem_cache *s, struct slab *partial_slab)
 }
 
 /*
- * Unfreeze all the cpu partial slabs.
+ * Put all the cpu partial slabs to the node partial list.
  */
-static void unfreeze_partials(struct kmem_cache *s)
+static void put_partials(struct kmem_cache *s)
 {
 	struct slab *partial_slab;
 	unsigned long flags;
@@ -2671,11 +2610,11 @@ static void unfreeze_partials(struct kmem_cache *s)
 	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
 
 	if (partial_slab)
-		__unfreeze_partials(s, partial_slab);
+		__put_partials(s, partial_slab);
 }
 
-static void unfreeze_partials_cpu(struct kmem_cache *s,
-				  struct kmem_cache_cpu *c)
+static void put_partials_cpu(struct kmem_cache *s,
+			     struct kmem_cache_cpu *c)
 {
 	struct slab *partial_slab;
 
@@ -2683,12 +2622,11 @@ static void unfreeze_partials_cpu(struct kmem_cache *s,
 	c->partial = NULL;
 
 	if (partial_slab)
-		__unfreeze_partials(s, partial_slab);
+		__put_partials(s, partial_slab);
 }
 
 /*
- * Put a slab that was just frozen (in __slab_free|get_partial_node) into a
- * partial slab slot if available.
+ * Put a slab into a partial slab slot if available.
  *
  * If we did not find a slot then simply move all the partials to the
  * per node partial list.
@@ -2696,7 +2634,7 @@ static void unfreeze_partials_cpu(struct kmem_cache *s,
 static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain)
 {
 	struct slab *oldslab;
-	struct slab *slab_to_unfreeze = NULL;
+	struct slab *slab_to_put = NULL;
 	unsigned long flags;
 	int slabs = 0;
 
@@ -2711,7 +2649,7 @@ static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain)
 			 * per node partial list. Postpone the actual unfreezing
 			 * outside of the critical section.
 			 */
-			slab_to_unfreeze = oldslab;
+			slab_to_put = oldslab;
 			oldslab = NULL;
 		} else {
 			slabs = oldslab->slabs;
@@ -2727,17 +2665,17 @@ static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain)
 
 	local_unlock_irqrestore(&s->cpu_slab->lock, flags);
 
-	if (slab_to_unfreeze) {
-		__unfreeze_partials(s, slab_to_unfreeze);
+	if (slab_to_put) {
+		__put_partials(s, slab_to_put);
 		stat(s, CPU_PARTIAL_DRAIN);
 	}
 }
 
 #else	/* CONFIG_SLUB_CPU_PARTIAL */
 
-static inline void unfreeze_partials(struct kmem_cache *s) { }
-static inline void unfreeze_partials_cpu(struct kmem_cache *s,
-				  struct kmem_cache_cpu *c) { }
+static inline void put_partials(struct kmem_cache *s) { }
+static inline void put_partials_cpu(struct kmem_cache *s,
+				    struct kmem_cache_cpu *c) { }
 
 #endif	/* CONFIG_SLUB_CPU_PARTIAL */
 
@@ -2779,7 +2717,7 @@ static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
 		stat(s, CPUSLAB_FLUSH);
 	}
 
-	unfreeze_partials_cpu(s, c);
+	put_partials_cpu(s, c);
 }
 
 struct slub_flush_work {
@@ -2807,7 +2745,7 @@ static void flush_cpu_slab(struct work_struct *w)
 	if (c->slab)
 		flush_slab(s, c);
 
-	unfreeze_partials(s);
+	put_partials(s);
 }
 
 static bool has_cpu_slab(int cpu, struct kmem_cache *s)
@@ -3073,6 +3011,33 @@ static inline void *get_freelist(struct kmem_cache *s, struct slab *slab)
 	return freelist;
 }
 
+/*
+ * Freeze the partial slab and return the pointer to the freelist.
+ */
+static inline void *freeze_slab(struct kmem_cache *s, struct slab *slab)
+{
+	struct slab new;
+	unsigned long counters;
+	void *freelist;
+
+	do {
+		freelist = slab->freelist;
+		counters = slab->counters;
+
+		new.counters = counters;
+		VM_BUG_ON(new.frozen);
+
+		new.inuse = slab->objects;
+		new.frozen = 1;
+
+	} while (!slab_update_freelist(s, slab,
+		freelist, counters,
+		NULL, new.counters,
+		"freeze_slab"));
+
+	return freelist;
+}
+
 /*
  * Slow path. The lockless freelist is empty or we need to perform
  * debugging duties.
@@ -3115,7 +3080,6 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
 			node = NUMA_NO_NODE;
 		goto new_slab;
 	}
-redo:
 
 	if (unlikely(!node_match(slab, node))) {
 		/*
@@ -3191,7 +3155,8 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
 
 new_slab:
 
-	if (slub_percpu_partial(c)) {
+#ifdef CONFIG_SLUB_CPU_PARTIAL
+	while (slub_percpu_partial(c)) {
 		local_lock_irqsave(&s->cpu_slab->lock, flags);
 		if (unlikely(c->slab)) {
 			local_unlock_irqrestore(&s->cpu_slab->lock, flags);
@@ -3203,21 +3168,45 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
 			goto new_objects;
 		}
 
-		slab = c->slab = slub_percpu_partial(c);
+		slab = slub_percpu_partial(c);
 		slub_set_percpu_partial(c, slab);
 		local_unlock_irqrestore(&s->cpu_slab->lock, flags);
 		stat(s, CPU_PARTIAL_ALLOC);
-		goto redo;
+
+		if (unlikely(!node_match(slab, node) ||
+			     !pfmemalloc_match(slab, gfpflags))) {
+			slab->next = NULL;
+			__put_partials(s, slab);
+			continue;
+		}
+
+		freelist = freeze_slab(s, slab);
+		goto retry_load_slab;
 	}
+#endif
 
 new_objects:
 
 	pc.flags = gfpflags;
-	pc.slab = &slab;
 	pc.orig_size = orig_size;
-	freelist = get_partial(s, node, &pc);
-	if (freelist)
-		goto check_new_slab;
+	slab = get_partial(s, node, &pc);
+	if (slab) {
+		if (kmem_cache_debug(s)) {
+			freelist = pc.object;
+			/*
+			 * For debug caches here we had to go through
+			 * alloc_single_from_partial() so just store the
+			 * tracking info and return the object.
+			 */
+			if (s->flags & SLAB_STORE_USER)
+				set_track(s, freelist, TRACK_ALLOC, addr);
+
+			return freelist;
+		}
+
+		freelist = freeze_slab(s, slab);
+		goto retry_load_slab;
+	}
 
 	slub_put_cpu_ptr(s->cpu_slab);
 	slab = new_slab(s, gfpflags, node);
@@ -3253,20 +3242,6 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
 
 	inc_slabs_node(s, slab_nid(slab), slab->objects);
 
-check_new_slab:
-
-	if (kmem_cache_debug(s)) {
-		/*
-		 * For debug caches here we had to go through
-		 * alloc_single_from_partial() so just store the tracking info
-		 * and return the object
-		 */
-		if (s->flags & SLAB_STORE_USER)
-			set_track(s, freelist, TRACK_ALLOC, addr);
-
-		return freelist;
-	}
-
 	if (unlikely(!pfmemalloc_match(slab, gfpflags))) {
 		/*
 		 * For !pfmemalloc_match() case we don't load freelist so that
@@ -3409,12 +3384,11 @@ static void *__slab_alloc_node(struct kmem_cache *s,
 	void *object;
 
 	pc.flags = gfpflags;
-	pc.slab = &slab;
 	pc.orig_size = orig_size;
-	object = get_partial(s, node, &pc);
+	slab = get_partial(s, node, &pc);
 
-	if (object)
-		return object;
+	if (slab)
+		return pc.object;
 
 	slab = new_slab(s, gfpflags, node);
 	if (unlikely(!slab)) {
@@ -3608,6 +3582,7 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
 	unsigned long counters;
 	struct kmem_cache_node *n = NULL;
 	unsigned long flags;
+	bool on_node_partial;
 
 	stat(s, FREE_SLOWPATH);
 
@@ -3631,18 +3606,8 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
 		was_frozen = new.frozen;
 		new.inuse -= cnt;
 		if ((!new.inuse || !prior) && !was_frozen) {
-
-			if (kmem_cache_has_cpu_partial(s) && !prior) {
-
-				/*
-				 * Slab was on no list before and will be
-				 * partially empty
-				 * We can defer the list move and instead
-				 * freeze it.
-				 */
-				new.frozen = 1;
-
-			} else { /* Needs to be taken off a list */
+			/* Needs to be taken off a list */
+			if (!kmem_cache_has_cpu_partial(s) || prior) {
 
 				n = get_node(s, slab_nid(slab));
 				/*
@@ -3655,6 +3620,7 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
 				 */
 				spin_lock_irqsave(&n->list_lock, flags);
 
+				on_node_partial = slab_test_node_partial(slab);
 			}
 		}
 
@@ -3671,9 +3637,9 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
 			 * activity can be necessary.
 			 */
 			stat(s, FREE_FROZEN);
-		} else if (new.frozen) {
+		} else if (kmem_cache_has_cpu_partial(s) && !prior) {
 			/*
-			 * If we just froze the slab then put it onto the
+			 * If we started with a full slab then put it onto the
 			 * per cpu partial list.
 			 */
 			put_cpu_partial(s, slab, 1);
@@ -3683,6 +3649,15 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
 		return;
 	}
 
+	/*
+	 * This slab was partially empty but not on the per-node partial list,
+	 * in which case we shouldn't manipulate its list, just return.
+	 */
+	if (prior && !on_node_partial) {
+		spin_unlock_irqrestore(&n->list_lock, flags);
+		return;
+	}
+
 	if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
 		goto slab_empty;
 
@@ -4831,6 +4806,7 @@ static int __kmem_cache_do_shrink(struct kmem_cache *s)
 
 			if (free == slab->objects) {
 				list_move(&slab->slab_list, &discard);
+				slab_clear_node_partial(slab);
 				n->nr_partial--;
 				dec_slabs_node(s, node, slab->objects);
 			} else if (free <= SHRINK_PROMOTE_MAX)
diff --git a/mm/sparse.c b/mm/sparse.c
index 77d91e565045..338cf946dee8 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -791,6 +791,13 @@ static void section_deactivate(unsigned long pfn, unsigned long nr_pages,
 	if (empty) {
 		unsigned long section_nr = pfn_to_section_nr(pfn);
 
+		/*
+		 * Mark the section invalid so that valid_section()
+		 * return false. This prevents code from dereferencing
+		 * ms->usage array.
+		 */
+		ms->section_mem_map &= ~SECTION_HAS_MEM_MAP;
+
 		/*
 		 * When removing an early section, the usage map is kept (as the
 		 * usage maps of other sections fall into the same page). It
@@ -799,16 +806,10 @@ static void section_deactivate(unsigned long pfn, unsigned long nr_pages,
 		 * was allocated during boot.
 		 */
 		if (!PageReserved(virt_to_page(ms->usage))) {
-			kfree(ms->usage);
-			ms->usage = NULL;
+			kfree_rcu(ms->usage, rcu);
+			WRITE_ONCE(ms->usage, NULL);
 		}
 		memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
-		/*
-		 * Mark the section invalid so that valid_section()
-		 * return false. This prevents code from dereferencing
-		 * ms->usage array.
-		 */
-		ms->section_mem_map &= ~SECTION_HAS_MEM_MAP;
 	}
 
 	/*
diff --git a/mm/truncate.c b/mm/truncate.c
index 8e3aa9e8618e..725b150e47ac 100644
--- a/mm/truncate.c
+++ b/mm/truncate.c
@@ -250,10 +250,9 @@ bool truncate_inode_partial_folio(struct folio *folio, loff_t start, loff_t end)
 /*
  * Used to get rid of pages on hardware memory corruption.
  */
-int generic_error_remove_page(struct address_space *mapping, struct page *page)
+int generic_error_remove_folio(struct address_space *mapping,
+		struct folio *folio)
 {
-	VM_BUG_ON_PAGE(PageTail(page), page);
-
 	if (!mapping)
 		return -EINVAL;
 	/*
@@ -262,13 +261,26 @@ int generic_error_remove_page(struct address_space *mapping, struct page *page)
 	 */
 	if (!S_ISREG(mapping->host->i_mode))
 		return -EIO;
-	return truncate_inode_folio(mapping, page_folio(page));
+	return truncate_inode_folio(mapping, folio);
 }
-EXPORT_SYMBOL(generic_error_remove_page);
+EXPORT_SYMBOL(generic_error_remove_folio);
 
-static long mapping_evict_folio(struct address_space *mapping,
-		struct folio *folio)
+/**
+ * mapping_evict_folio() - Remove an unused folio from the page-cache.
+ * @mapping: The mapping this folio belongs to.
+ * @folio: The folio to remove.
+ *
+ * Safely remove one folio from the page cache.
+ * It only drops clean, unused folios.
+ *
+ * Context: Folio must be locked.
+ * Return: The number of pages successfully removed.
+ */
+long mapping_evict_folio(struct address_space *mapping, struct folio *folio)
 {
+	/* The page may have been truncated before it was locked */
+	if (!mapping)
+		return 0;
 	if (folio_test_dirty(folio) || folio_test_writeback(folio))
 		return 0;
 	/* The refcount will be elevated if any page in the folio is mapped */
@@ -281,27 +293,6 @@ static long mapping_evict_folio(struct address_space *mapping,
 	return remove_mapping(mapping, folio);
 }
 
-/**
- * invalidate_inode_page() - Remove an unused page from the pagecache.
- * @page: The page to remove.
- *
- * Safely invalidate one page from its pagecache mapping.
- * It only drops clean, unused pages.
- *
- * Context: Page must be locked.
- * Return: The number of pages successfully removed.
- */
-long invalidate_inode_page(struct page *page)
-{
-	struct folio *folio = page_folio(page);
-	struct address_space *mapping = folio_mapping(folio);
-
-	/* The page may have been truncated before it was locked */
-	if (!mapping)
-		return 0;
-	return mapping_evict_folio(mapping, folio);
-}
-
 /**
  * truncate_inode_pages_range - truncate range of pages specified by start & end byte offsets
  * @mapping: mapping to truncate
@@ -560,9 +551,9 @@ unsigned long invalidate_mapping_pages(struct address_space *mapping,
 EXPORT_SYMBOL(invalidate_mapping_pages);
 
 /*
- * This is like invalidate_inode_page(), except it ignores the page's
+ * This is like mapping_evict_folio(), except it ignores the folio's
  * refcount.  We do this because invalidate_inode_pages2() needs stronger
- * invalidation guarantees, and cannot afford to leave pages behind because
+ * invalidation guarantees, and cannot afford to leave folios behind because
  * shrink_page_list() has a temp ref on them, or because they're transiently
  * sitting in the folio_add_lru() caches.
  */
diff --git a/mm/util.c b/mm/util.c
index 744b4d7e3fae..5a6a9802583b 100644
--- a/mm/util.c
+++ b/mm/util.c
@@ -1047,11 +1047,11 @@ int __weak memcmp_pages(struct page *page1, struct page *page2)
 	char *addr1, *addr2;
 	int ret;
 
-	addr1 = kmap_atomic(page1);
-	addr2 = kmap_atomic(page2);
+	addr1 = kmap_local_page(page1);
+	addr2 = kmap_local_page(page2);
 	ret = memcmp(addr1, addr2, PAGE_SIZE);
-	kunmap_atomic(addr2);
-	kunmap_atomic(addr1);
+	kunmap_local(addr2);
+	kunmap_local(addr1);
 	return ret;
 }
 
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 506f8220c5fe..96abaa5a973e 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -136,6 +136,9 @@ struct scan_control {
 	/* Always discard instead of demoting to lower tier memory */
 	unsigned int no_demotion:1;
 
+	/* Swap space is exhausted, only reclaim swapcache for anon LRU */
+	unsigned int swapcache_only:1;
+
 	/* Allocation order */
 	s8 order;
 
@@ -319,10 +322,22 @@ static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg,
 		 */
 		if (get_nr_swap_pages() > 0)
 			return true;
+		/* Is there any swapcache pages to reclaim? */
+		if (total_swapcache_pages() > 0) {
+			if (sc)
+				sc->swapcache_only = 1;
+			return true;
+		}
 	} else {
 		/* Is the memcg below its swap limit? */
 		if (mem_cgroup_get_nr_swap_pages(memcg) > 0)
 			return true;
+		/* Is there any swapcache pages in memcg to reclaim? */
+		if (mem_cgroup_get_nr_swapcache_pages(memcg) > 0) {
+			if (sc)
+				sc->swapcache_only = 1;
+			return true;
+		}
 	}
 
 	/*
@@ -409,12 +424,14 @@ void drop_slab(void)
 
 static int reclaimer_offset(void)
 {
+#ifdef CONFIG_NUMA_BALANCING
 	BUILD_BUG_ON(PGSTEAL_DIRECT - PGSTEAL_KSWAPD !=
 			PGDEMOTE_DIRECT - PGDEMOTE_KSWAPD);
-	BUILD_BUG_ON(PGSTEAL_DIRECT - PGSTEAL_KSWAPD !=
-			PGSCAN_DIRECT - PGSCAN_KSWAPD);
 	BUILD_BUG_ON(PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD !=
 			PGDEMOTE_KHUGEPAGED - PGDEMOTE_KSWAPD);
+#endif
+	BUILD_BUG_ON(PGSTEAL_DIRECT - PGSTEAL_KSWAPD !=
+			PGSCAN_DIRECT - PGSCAN_KSWAPD);
 	BUILD_BUG_ON(PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD !=
 			PGSCAN_KHUGEPAGED - PGSCAN_KSWAPD);
 
@@ -976,8 +993,10 @@ static unsigned int demote_folio_list(struct list_head *demote_folios,
 	migrate_pages(demote_folios, alloc_demote_folio, NULL,
 		      (unsigned long)&mtc, MIGRATE_ASYNC, MR_DEMOTION,
 		      &nr_succeeded);
-
-	__count_vm_events(PGDEMOTE_KSWAPD + reclaimer_offset(), nr_succeeded);
+#ifdef CONFIG_NUMA_BALANCING
+	mod_node_page_state(pgdat, PGDEMOTE_KSWAPD + reclaimer_offset(),
+			    nr_succeeded);
+#endif
 
 	return nr_succeeded;
 }
@@ -1642,6 +1661,15 @@ static unsigned long isolate_lru_folios(unsigned long nr_to_scan,
 		 */
 		scan += nr_pages;
 
+		/*
+		 * Count non-swapcache too because the swapcache pages may
+		 * be rare and it takes too much times here if not count
+		 * the non-swapcache pages.
+		 */
+		if (unlikely(sc->swapcache_only && !is_file_lru(lru) &&
+		    !folio_test_swapcache(folio)))
+			goto move;
+
 		if (!folio_test_lru(folio))
 			goto move;
 		if (!sc->may_unmap && folio_mapped(folio))
diff --git a/mm/vmstat.c b/mm/vmstat.c
index 359460deb377..afa5a38fcc9c 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -1248,6 +1248,9 @@ const char * const vmstat_text[] = {
 #ifdef CONFIG_NUMA_BALANCING
 	"pgpromote_success",
 	"pgpromote_candidate",
+	"pgdemote_kswapd",
+	"pgdemote_direct",
+	"pgdemote_khugepaged",
 #endif
 
 	/* enum writeback_stat_item counters */
@@ -1279,9 +1282,6 @@ const char * const vmstat_text[] = {
 	"pgsteal_kswapd",
 	"pgsteal_direct",
 	"pgsteal_khugepaged",
-	"pgdemote_kswapd",
-	"pgdemote_direct",
-	"pgdemote_khugepaged",
 	"pgscan_kswapd",
 	"pgscan_direct",
 	"pgscan_khugepaged",
diff --git a/mm/zswap.c b/mm/zswap.c
index 74411dfdad92..030cc137138f 100644
--- a/mm/zswap.c
+++ b/mm/zswap.c
@@ -1144,6 +1144,11 @@ static int zswap_writeback_entry(struct zswap_entry *entry,
 	/* move it to the tail of the inactive list after end_writeback */
 	SetPageReclaim(page);
 
+	if (!PageLRU(page)) {
+		/* drain lru cache to help folio_rotate_reclaimable() */
+		lru_add_drain();
+	}
+
 	/* start writeback */
 	__swap_writepage(page, &wbc);
 	put_page(page);
diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c
index 2a7f1b15714a..407b2335f091 100644
--- a/net/8021q/vlan_dev.c
+++ b/net/8021q/vlan_dev.c
@@ -702,20 +702,7 @@ static int vlan_ethtool_get_ts_info(struct net_device *dev,
 				    struct ethtool_ts_info *info)
 {
 	const struct vlan_dev_priv *vlan = vlan_dev_priv(dev);
-	const struct ethtool_ops *ops = vlan->real_dev->ethtool_ops;
-	struct phy_device *phydev = vlan->real_dev->phydev;
-
-	if (phy_has_tsinfo(phydev)) {
-		return phy_ts_info(phydev, info);
-	} else if (ops->get_ts_info) {
-		return ops->get_ts_info(vlan->real_dev, info);
-	} else {
-		info->so_timestamping = SOF_TIMESTAMPING_RX_SOFTWARE |
-			SOF_TIMESTAMPING_SOFTWARE;
-		info->phc_index = -1;
-	}
-
-	return 0;
+	return ethtool_get_ts_info_by_layer(vlan->real_dev, info);
 }
 
 static void vlan_dev_get_stats64(struct net_device *dev,
diff --git a/net/batman-adv/Makefile b/net/batman-adv/Makefile
index 3bd0760c76a2..b51d8b071b56 100644
--- a/net/batman-adv/Makefile
+++ b/net/batman-adv/Makefile
@@ -20,6 +20,7 @@ batman-adv-y += hash.o
 batman-adv-$(CONFIG_BATMAN_ADV_DEBUG) += log.o
 batman-adv-y += main.o
 batman-adv-$(CONFIG_BATMAN_ADV_MCAST) += multicast.o
+batman-adv-$(CONFIG_BATMAN_ADV_MCAST) += multicast_forw.o
 batman-adv-y += netlink.o
 batman-adv-$(CONFIG_BATMAN_ADV_NC) += network-coding.o
 batman-adv-y += originator.o
diff --git a/net/batman-adv/bridge_loop_avoidance.c b/net/batman-adv/bridge_loop_avoidance.c
index 37ce6cfb3520..5f46ca3d4bb8 100644
--- a/net/batman-adv/bridge_loop_avoidance.c
+++ b/net/batman-adv/bridge_loop_avoidance.c
@@ -20,7 +20,6 @@
 #include <linux/if_vlan.h>
 #include <linux/jhash.h>
 #include <linux/jiffies.h>
-#include <linux/kernel.h>
 #include <linux/kref.h>
 #include <linux/list.h>
 #include <linux/lockdep.h>
@@ -31,6 +30,7 @@
 #include <linux/skbuff.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
+#include <linux/sprintf.h>
 #include <linux/stddef.h>
 #include <linux/string.h>
 #include <linux/workqueue.h>
diff --git a/net/batman-adv/fragmentation.c b/net/batman-adv/fragmentation.c
index c120c7c6d25f..757c084ac2d1 100644
--- a/net/batman-adv/fragmentation.c
+++ b/net/batman-adv/fragmentation.c
@@ -25,7 +25,6 @@
 
 #include "hard-interface.h"
 #include "originator.h"
-#include "routing.h"
 #include "send.h"
 
 /**
@@ -351,18 +350,14 @@ bool batadv_frag_skb_fwd(struct sk_buff *skb,
 			 struct batadv_orig_node *orig_node_src)
 {
 	struct batadv_priv *bat_priv = netdev_priv(recv_if->soft_iface);
-	struct batadv_orig_node *orig_node_dst;
 	struct batadv_neigh_node *neigh_node = NULL;
 	struct batadv_frag_packet *packet;
 	u16 total_size;
 	bool ret = false;
 
 	packet = (struct batadv_frag_packet *)skb->data;
-	orig_node_dst = batadv_orig_hash_find(bat_priv, packet->dest);
-	if (!orig_node_dst)
-		goto out;
 
-	neigh_node = batadv_find_router(bat_priv, orig_node_dst, recv_if);
+	neigh_node = batadv_orig_to_router(bat_priv, packet->dest, recv_if);
 	if (!neigh_node)
 		goto out;
 
@@ -381,7 +376,6 @@ bool batadv_frag_skb_fwd(struct sk_buff *skb,
 	}
 
 out:
-	batadv_orig_node_put(orig_node_dst);
 	batadv_neigh_node_put(neigh_node);
 	return ret;
 }
diff --git a/net/batman-adv/gateway_client.c b/net/batman-adv/gateway_client.c
index d26124bc27e1..0ddd8b4b3f4c 100644
--- a/net/batman-adv/gateway_client.c
+++ b/net/batman-adv/gateway_client.c
@@ -18,7 +18,6 @@
 #include <linux/in.h>
 #include <linux/ip.h>
 #include <linux/ipv6.h>
-#include <linux/kernel.h>
 #include <linux/kref.h>
 #include <linux/list.h>
 #include <linux/lockdep.h>
@@ -29,6 +28,7 @@
 #include <linux/skbuff.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
+#include <linux/sprintf.h>
 #include <linux/stddef.h>
 #include <linux/udp.h>
 #include <net/sock.h>
diff --git a/net/batman-adv/main.c b/net/batman-adv/main.c
index e8a449915566..5fc754b0b3f7 100644
--- a/net/batman-adv/main.c
+++ b/net/batman-adv/main.c
@@ -6,6 +6,7 @@
 
 #include "main.h"
 
+#include <linux/array_size.h>
 #include <linux/atomic.h>
 #include <linux/build_bug.h>
 #include <linux/byteorder/generic.h>
@@ -20,7 +21,6 @@
 #include <linux/init.h>
 #include <linux/ip.h>
 #include <linux/ipv6.h>
-#include <linux/kernel.h>
 #include <linux/kobject.h>
 #include <linux/kref.h>
 #include <linux/list.h>
@@ -33,6 +33,7 @@
 #include <linux/skbuff.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
+#include <linux/sprintf.h>
 #include <linux/stddef.h>
 #include <linux/string.h>
 #include <linux/workqueue.h>
@@ -532,6 +533,8 @@ static void batadv_recv_handler_init(void)
 
 	/* broadcast packet */
 	batadv_rx_handler[BATADV_BCAST] = batadv_recv_bcast_packet;
+	/* multicast packet */
+	batadv_rx_handler[BATADV_MCAST] = batadv_recv_mcast_packet;
 
 	/* unicast packets ... */
 	/* unicast with 4 addresses packet */
diff --git a/net/batman-adv/main.h b/net/batman-adv/main.h
index 10007c5894a1..870dcd7f1786 100644
--- a/net/batman-adv/main.h
+++ b/net/batman-adv/main.h
@@ -13,7 +13,7 @@
 #define BATADV_DRIVER_DEVICE "batman-adv"
 
 #ifndef BATADV_SOURCE_VERSION
-#define BATADV_SOURCE_VERSION "2023.3"
+#define BATADV_SOURCE_VERSION "2024.0"
 #endif
 
 /* B.A.T.M.A.N. parameters */
diff --git a/net/batman-adv/multicast.c b/net/batman-adv/multicast.c
index 315394f12c55..d982daea8329 100644
--- a/net/batman-adv/multicast.c
+++ b/net/batman-adv/multicast.c
@@ -25,7 +25,6 @@
 #include <linux/ip.h>
 #include <linux/ipv6.h>
 #include <linux/jiffies.h>
-#include <linux/kernel.h>
 #include <linux/list.h>
 #include <linux/lockdep.h>
 #include <linux/netdevice.h>
@@ -36,6 +35,7 @@
 #include <linux/skbuff.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
+#include <linux/sprintf.h>
 #include <linux/stddef.h>
 #include <linux/string.h>
 #include <linux/types.h>
@@ -235,6 +235,37 @@ static u8 batadv_mcast_mla_rtr_flags_get(struct batadv_priv *bat_priv,
 	return flags;
 }
 
+/**
+ * batadv_mcast_mla_forw_flags_get() - get multicast forwarding flags
+ * @bat_priv: the bat priv with all the soft interface information
+ *
+ * Checks if all active hard interfaces have an MTU larger or equal to 1280
+ * bytes (IPv6 minimum MTU).
+ *
+ * Return: BATADV_MCAST_HAVE_MC_PTYPE_CAPA if yes, BATADV_NO_FLAGS otherwise.
+ */
+static u8 batadv_mcast_mla_forw_flags_get(struct batadv_priv *bat_priv)
+{
+	const struct batadv_hard_iface *hard_iface;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(hard_iface, &batadv_hardif_list, list) {
+		if (hard_iface->if_status != BATADV_IF_ACTIVE)
+			continue;
+
+		if (hard_iface->soft_iface != bat_priv->soft_iface)
+			continue;
+
+		if (hard_iface->net_dev->mtu < IPV6_MIN_MTU) {
+			rcu_read_unlock();
+			return BATADV_NO_FLAGS;
+		}
+	}
+	rcu_read_unlock();
+
+	return BATADV_MCAST_HAVE_MC_PTYPE_CAPA;
+}
+
 /**
  * batadv_mcast_mla_flags_get() - get the new multicast flags
  * @bat_priv: the bat priv with all the soft interface information
@@ -256,6 +287,7 @@ batadv_mcast_mla_flags_get(struct batadv_priv *bat_priv)
 	mla_flags.enabled = 1;
 	mla_flags.tvlv_flags |= batadv_mcast_mla_rtr_flags_get(bat_priv,
 							       bridge);
+	mla_flags.tvlv_flags |= batadv_mcast_mla_forw_flags_get(bat_priv);
 
 	if (!bridge)
 		return mla_flags;
@@ -806,23 +838,25 @@ static void batadv_mcast_flags_log(struct batadv_priv *bat_priv, u8 flags)
 {
 	bool old_enabled = bat_priv->mcast.mla_flags.enabled;
 	u8 old_flags = bat_priv->mcast.mla_flags.tvlv_flags;
-	char str_old_flags[] = "[.... . ]";
+	char str_old_flags[] = "[.... . .]";
 
-	sprintf(str_old_flags, "[%c%c%c%s%s]",
+	sprintf(str_old_flags, "[%c%c%c%s%s%c]",
 		(old_flags & BATADV_MCAST_WANT_ALL_UNSNOOPABLES) ? 'U' : '.',
 		(old_flags & BATADV_MCAST_WANT_ALL_IPV4) ? '4' : '.',
 		(old_flags & BATADV_MCAST_WANT_ALL_IPV6) ? '6' : '.',
 		!(old_flags & BATADV_MCAST_WANT_NO_RTR4) ? "R4" : ". ",
-		!(old_flags & BATADV_MCAST_WANT_NO_RTR6) ? "R6" : ". ");
+		!(old_flags & BATADV_MCAST_WANT_NO_RTR6) ? "R6" : ". ",
+		!(old_flags & BATADV_MCAST_HAVE_MC_PTYPE_CAPA) ? 'P' : '.');
 
 	batadv_dbg(BATADV_DBG_MCAST, bat_priv,
-		   "Changing multicast flags from '%s' to '[%c%c%c%s%s]'\n",
+		   "Changing multicast flags from '%s' to '[%c%c%c%s%s%c]'\n",
 		   old_enabled ? str_old_flags : "<undefined>",
 		   (flags & BATADV_MCAST_WANT_ALL_UNSNOOPABLES) ? 'U' : '.',
 		   (flags & BATADV_MCAST_WANT_ALL_IPV4) ? '4' : '.',
 		   (flags & BATADV_MCAST_WANT_ALL_IPV6) ? '6' : '.',
 		   !(flags & BATADV_MCAST_WANT_NO_RTR4) ? "R4" : ". ",
-		   !(flags & BATADV_MCAST_WANT_NO_RTR6) ? "R6" : ". ");
+		   !(flags & BATADV_MCAST_WANT_NO_RTR6) ? "R6" : ". ",
+		   !(flags & BATADV_MCAST_HAVE_MC_PTYPE_CAPA) ? 'P' : '.');
 }
 
 /**
@@ -1135,17 +1169,62 @@ static int batadv_mcast_forw_rtr_count(struct batadv_priv *bat_priv,
 	}
 }
 
+/**
+ * batadv_mcast_forw_mode_by_count() - get forwarding mode by count
+ * @bat_priv: the bat priv with all the soft interface information
+ * @skb: the multicast packet to check
+ * @vid: the vlan identifier
+ * @is_routable: stores whether the destination is routable
+ * @count: the number of originators the multicast packet need to be sent to
+ *
+ * For a multicast packet with multiple destination originators, checks which
+ * mode to use. For BATADV_FORW_MCAST it also encapsulates the packet with a
+ * complete batman-adv multicast header.
+ *
+ * Return:
+ *	BATADV_FORW_MCAST: If all nodes have multicast packet routing
+ *	capabilities and an MTU >= 1280 on all hard interfaces (including us)
+ *	and the encapsulated multicast packet with all destination addresses
+ *	would still fit into an 1280 bytes batman-adv multicast packet
+ *	(excluding the outer ethernet frame) and we could successfully push
+ *	the full batman-adv multicast packet header.
+ *	BATADV_FORW_UCASTS: If the packet cannot be sent in a batman-adv
+ *	multicast packet and the amount of batman-adv unicast packets needed
+ *	is smaller or equal to the configured multicast fanout.
+ *	BATADV_FORW_BCAST: Otherwise.
+ */
+static enum batadv_forw_mode
+batadv_mcast_forw_mode_by_count(struct batadv_priv *bat_priv,
+				struct sk_buff *skb, unsigned short vid,
+				int is_routable, int count)
+{
+	unsigned int mcast_hdrlen = batadv_mcast_forw_packet_hdrlen(count);
+	u8 own_tvlv_flags = bat_priv->mcast.mla_flags.tvlv_flags;
+
+	if (!atomic_read(&bat_priv->mcast.num_no_mc_ptype_capa) &&
+	    own_tvlv_flags & BATADV_MCAST_HAVE_MC_PTYPE_CAPA &&
+	    skb->len + mcast_hdrlen <= IPV6_MIN_MTU &&
+	    batadv_mcast_forw_push(bat_priv, skb, vid, is_routable, count))
+		return BATADV_FORW_MCAST;
+
+	if (count <= atomic_read(&bat_priv->multicast_fanout))
+		return BATADV_FORW_UCASTS;
+
+	return BATADV_FORW_BCAST;
+}
+
 /**
  * batadv_mcast_forw_mode() - check on how to forward a multicast packet
  * @bat_priv: the bat priv with all the soft interface information
  * @skb: the multicast packet to check
+ * @vid: the vlan identifier
  * @is_routable: stores whether the destination is routable
  *
  * Return: The forwarding mode as enum batadv_forw_mode.
  */
 enum batadv_forw_mode
 batadv_mcast_forw_mode(struct batadv_priv *bat_priv, struct sk_buff *skb,
-		       int *is_routable)
+		       unsigned short vid, int *is_routable)
 {
 	int ret, tt_count, ip_count, unsnoop_count, total_count;
 	bool is_unsnoopable = false;
@@ -1175,10 +1254,8 @@ batadv_mcast_forw_mode(struct batadv_priv *bat_priv, struct sk_buff *skb,
 	else if (unsnoop_count)
 		return BATADV_FORW_BCAST;
 
-	if (total_count <= atomic_read(&bat_priv->multicast_fanout))
-		return BATADV_FORW_UCASTS;
-
-	return BATADV_FORW_BCAST;
+	return batadv_mcast_forw_mode_by_count(bat_priv, skb, vid, *is_routable,
+					       total_count);
 }
 
 /**
@@ -1738,6 +1815,31 @@ static void batadv_mcast_want_rtr6_update(struct batadv_priv *bat_priv,
 	}
 }
 
+/**
+ * batadv_mcast_have_mc_ptype_update() - update multicast packet type counter
+ * @bat_priv: the bat priv with all the soft interface information
+ * @orig: the orig_node which multicast state might have changed of
+ * @mcast_flags: flags indicating the new multicast state
+ *
+ * If the BATADV_MCAST_HAVE_MC_PTYPE_CAPA flag of this originator, orig, has
+ * toggled then this method updates the counter accordingly.
+ */
+static void batadv_mcast_have_mc_ptype_update(struct batadv_priv *bat_priv,
+					      struct batadv_orig_node *orig,
+					      u8 mcast_flags)
+{
+	lockdep_assert_held(&orig->mcast_handler_lock);
+
+	/* switched from flag set to unset */
+	if (!(mcast_flags & BATADV_MCAST_HAVE_MC_PTYPE_CAPA) &&
+	    orig->mcast_flags & BATADV_MCAST_HAVE_MC_PTYPE_CAPA)
+		atomic_inc(&bat_priv->mcast.num_no_mc_ptype_capa);
+	/* switched from flag unset to set */
+	else if (mcast_flags & BATADV_MCAST_HAVE_MC_PTYPE_CAPA &&
+		 !(orig->mcast_flags & BATADV_MCAST_HAVE_MC_PTYPE_CAPA))
+		atomic_dec(&bat_priv->mcast.num_no_mc_ptype_capa);
+}
+
 /**
  * batadv_mcast_tvlv_flags_get() - get multicast flags from an OGM TVLV
  * @enabled: whether the originator has multicast TVLV support enabled
@@ -1806,6 +1908,7 @@ static void batadv_mcast_tvlv_ogm_handler(struct batadv_priv *bat_priv,
 	batadv_mcast_want_ipv6_update(bat_priv, orig, mcast_flags);
 	batadv_mcast_want_rtr4_update(bat_priv, orig, mcast_flags);
 	batadv_mcast_want_rtr6_update(bat_priv, orig, mcast_flags);
+	batadv_mcast_have_mc_ptype_update(bat_priv, orig, mcast_flags);
 
 	orig->mcast_flags = mcast_flags;
 	spin_unlock_bh(&orig->mcast_handler_lock);
@@ -1820,6 +1923,10 @@ void batadv_mcast_init(struct batadv_priv *bat_priv)
 	batadv_tvlv_handler_register(bat_priv, batadv_mcast_tvlv_ogm_handler,
 				     NULL, NULL, BATADV_TVLV_MCAST, 2,
 				     BATADV_TVLV_HANDLER_OGM_CIFNOTFND);
+	batadv_tvlv_handler_register(bat_priv, NULL, NULL,
+				     batadv_mcast_forw_tracker_tvlv_handler,
+				     BATADV_TVLV_MCAST_TRACKER, 1,
+				     BATADV_TVLV_HANDLER_OGM_CIFNOTFND);
 
 	INIT_DELAYED_WORK(&bat_priv->mcast.work, batadv_mcast_mla_update);
 	batadv_mcast_start_timer(bat_priv);
diff --git a/net/batman-adv/multicast.h b/net/batman-adv/multicast.h
index a9770d8d6d36..d97ee51d26f2 100644
--- a/net/batman-adv/multicast.h
+++ b/net/batman-adv/multicast.h
@@ -11,6 +11,7 @@
 
 #include <linux/netlink.h>
 #include <linux/skbuff.h>
+#include <linux/types.h>
 
 /**
  * enum batadv_forw_mode - the way a packet should be forwarded as
@@ -28,6 +29,12 @@ enum batadv_forw_mode {
 	 */
 	BATADV_FORW_UCASTS,
 
+	/**
+	 * @BATADV_FORW_MCAST: forward the packet to some nodes via a
+	 *  batman-adv multicast packet
+	 */
+	BATADV_FORW_MCAST,
+
 	/** @BATADV_FORW_NONE: don't forward, drop it */
 	BATADV_FORW_NONE,
 };
@@ -36,7 +43,7 @@ enum batadv_forw_mode {
 
 enum batadv_forw_mode
 batadv_mcast_forw_mode(struct batadv_priv *bat_priv, struct sk_buff *skb,
-		       int *is_routable);
+		       unsigned short vid, int *is_routable);
 
 int batadv_mcast_forw_send(struct batadv_priv *bat_priv, struct sk_buff *skb,
 			   unsigned short vid, int is_routable);
@@ -52,11 +59,23 @@ void batadv_mcast_free(struct batadv_priv *bat_priv);
 
 void batadv_mcast_purge_orig(struct batadv_orig_node *orig_node);
 
+/* multicast_forw.c */
+
+int batadv_mcast_forw_tracker_tvlv_handler(struct batadv_priv *bat_priv,
+					   struct sk_buff *skb);
+
+unsigned int batadv_mcast_forw_packet_hdrlen(unsigned int num_dests);
+
+bool batadv_mcast_forw_push(struct batadv_priv *bat_priv, struct sk_buff *skb,
+			    unsigned short vid, int is_routable, int count);
+
+int batadv_mcast_forw_mcsend(struct batadv_priv *bat_priv, struct sk_buff *skb);
+
 #else
 
 static inline enum batadv_forw_mode
 batadv_mcast_forw_mode(struct batadv_priv *bat_priv, struct sk_buff *skb,
-		       int *is_routable)
+		       unsigned short vid, int *is_routable)
 {
 	return BATADV_FORW_BCAST;
 }
@@ -94,6 +113,13 @@ static inline void batadv_mcast_purge_orig(struct batadv_orig_node *orig_node)
 {
 }
 
+static inline int batadv_mcast_forw_mcsend(struct batadv_priv *bat_priv,
+					   struct sk_buff *skb)
+{
+	kfree_skb(skb);
+	return NET_XMIT_DROP;
+}
+
 #endif /* CONFIG_BATMAN_ADV_MCAST */
 
 #endif /* _NET_BATMAN_ADV_MULTICAST_H_ */
diff --git a/net/batman-adv/multicast_forw.c b/net/batman-adv/multicast_forw.c
new file mode 100644
index 000000000000..fafd6ba8c056
--- /dev/null
+++ b/net/batman-adv/multicast_forw.c
@@ -0,0 +1,1178 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (C) B.A.T.M.A.N. contributors:
+ *
+ * Linus Lüssing
+ */
+
+#include "multicast.h"
+#include "main.h"
+
+#include <linux/bug.h>
+#include <linux/build_bug.h>
+#include <linux/byteorder/generic.h>
+#include <linux/compiler.h>
+#include <linux/errno.h>
+#include <linux/etherdevice.h>
+#include <linux/gfp.h>
+#include <linux/if_ether.h>
+#include <linux/if_vlan.h>
+#include <linux/ipv6.h>
+#include <linux/limits.h>
+#include <linux/netdevice.h>
+#include <linux/rculist.h>
+#include <linux/rcupdate.h>
+#include <linux/skbuff.h>
+#include <linux/stddef.h>
+#include <linux/string.h>
+#include <linux/types.h>
+#include <uapi/linux/batadv_packet.h>
+
+#include "bridge_loop_avoidance.h"
+#include "originator.h"
+#include "send.h"
+#include "translation-table.h"
+
+#define batadv_mcast_forw_tracker_for_each_dest(dest, num_dests) \
+	for (; num_dests; num_dests--, (dest) += ETH_ALEN)
+
+#define batadv_mcast_forw_tracker_for_each_dest2(dest1, dest2, num_dests) \
+	for (; num_dests; num_dests--, (dest1) += ETH_ALEN, (dest2) += ETH_ALEN)
+
+/**
+ * batadv_mcast_forw_skb_push() - skb_push and memorize amount of pushed bytes
+ * @skb: the skb to push onto
+ * @size: the amount of bytes to push
+ * @len: stores the total amount of bytes pushed
+ *
+ * Performs an skb_push() onto the given skb and adds the amount of pushed bytes
+ * to the given len pointer.
+ *
+ * Return: the return value of the skb_push() call.
+ */
+static void *batadv_mcast_forw_skb_push(struct sk_buff *skb, size_t size,
+					unsigned short *len)
+{
+	*len += size;
+	return skb_push(skb, size);
+}
+
+/**
+ * batadv_mcast_forw_push_padding() - push 2 padding bytes to skb's front
+ * @skb: the skb to push onto
+ * @tvlv_len: stores the amount of currently pushed TVLV bytes
+ *
+ * Pushes two padding bytes to the front of the given skb.
+ *
+ * Return: On success a pointer to the first byte of the two pushed padding
+ * bytes within the skb. NULL otherwise.
+ */
+static char *
+batadv_mcast_forw_push_padding(struct sk_buff *skb, unsigned short *tvlv_len)
+{
+	const int pad_len = 2;
+	char *padding;
+
+	if (skb_headroom(skb) < pad_len)
+		return NULL;
+
+	padding = batadv_mcast_forw_skb_push(skb, pad_len, tvlv_len);
+	memset(padding, 0, pad_len);
+
+	return padding;
+}
+
+/**
+ * batadv_mcast_forw_push_est_padding() - push padding bytes if necessary
+ * @skb: the skb to potentially push the padding onto
+ * @count: the (estimated) number of originators the multicast packet needs to
+ *  be sent to
+ * @tvlv_len: stores the amount of currently pushed TVLV bytes
+ *
+ * If the number of destination entries is even then this adds two
+ * padding bytes to the end of the tracker TVLV.
+ *
+ * Return: true on success or if no padding is needed, false otherwise.
+ */
+static bool
+batadv_mcast_forw_push_est_padding(struct sk_buff *skb, int count,
+				   unsigned short *tvlv_len)
+{
+	if (!(count % 2) && !batadv_mcast_forw_push_padding(skb, tvlv_len))
+		return false;
+
+	return true;
+}
+
+/**
+ * batadv_mcast_forw_orig_entry() - get orig_node from an hlist node
+ * @node: the hlist node to get the orig_node from
+ * @entry_offset: the offset of the hlist node within the orig_node struct
+ *
+ * Return: The orig_node containing the hlist node on success, NULL on error.
+ */
+static struct batadv_orig_node *
+batadv_mcast_forw_orig_entry(struct hlist_node *node,
+			     size_t entry_offset)
+{
+	/* sanity check */
+	switch (entry_offset) {
+	case offsetof(struct batadv_orig_node, mcast_want_all_ipv4_node):
+	case offsetof(struct batadv_orig_node, mcast_want_all_ipv6_node):
+	case offsetof(struct batadv_orig_node, mcast_want_all_rtr4_node):
+	case offsetof(struct batadv_orig_node, mcast_want_all_rtr6_node):
+		break;
+	default:
+		WARN_ON(1);
+		return NULL;
+	}
+
+	return (struct batadv_orig_node *)((void *)node - entry_offset);
+}
+
+/**
+ * batadv_mcast_forw_push_dest() - push an originator MAC address onto an skb
+ * @bat_priv: the bat priv with all the soft interface information
+ * @skb: the skb to push the destination address onto
+ * @vid: the vlan identifier
+ * @orig_node: the originator node to get the MAC address from
+ * @num_dests: a pointer to store the number of pushed addresses in
+ * @tvlv_len: stores the amount of currently pushed TVLV bytes
+ *
+ * If the orig_node is a BLA backbone gateway, if there is not enough skb
+ * headroom available or if num_dests is already at its maximum (65535) then
+ * neither the skb nor num_dests is changed. Otherwise the originator's MAC
+ * address is pushed onto the given skb and num_dests incremented by one.
+ *
+ * Return: true if the orig_node is a backbone gateway or if an orig address
+ *  was pushed successfully, false otherwise.
+ */
+static bool batadv_mcast_forw_push_dest(struct batadv_priv *bat_priv,
+					struct sk_buff *skb, unsigned short vid,
+					struct batadv_orig_node *orig_node,
+					unsigned short *num_dests,
+					unsigned short *tvlv_len)
+{
+	BUILD_BUG_ON(sizeof_field(struct batadv_tvlv_mcast_tracker, num_dests)
+		     != sizeof(__be16));
+
+	/* Avoid sending to other BLA gateways - they already got the frame from
+	 * the LAN side we share with them.
+	 * TODO: Refactor to take BLA into account earlier in mode check.
+	 */
+	if (batadv_bla_is_backbone_gw_orig(bat_priv, orig_node->orig, vid))
+		return true;
+
+	if (skb_headroom(skb) < ETH_ALEN || *num_dests == U16_MAX)
+		return false;
+
+	batadv_mcast_forw_skb_push(skb, ETH_ALEN, tvlv_len);
+	ether_addr_copy(skb->data, orig_node->orig);
+	(*num_dests)++;
+
+	return true;
+}
+
+/**
+ * batadv_mcast_forw_push_dests_list() - push originators from list onto an skb
+ * @bat_priv: the bat priv with all the soft interface information
+ * @skb: the skb to push the destination addresses onto
+ * @vid: the vlan identifier
+ * @head: the list to gather originators from
+ * @entry_offset: offset of an hlist node in an orig_node structure
+ * @num_dests: a pointer to store the number of pushed addresses in
+ * @tvlv_len: stores the amount of currently pushed TVLV bytes
+ *
+ * Push the MAC addresses of all originators in the given list onto the given
+ * skb.
+ *
+ * Return: true on success, false otherwise.
+ */
+static int batadv_mcast_forw_push_dests_list(struct batadv_priv *bat_priv,
+					     struct sk_buff *skb,
+					     unsigned short vid,
+					     struct hlist_head *head,
+					     size_t entry_offset,
+					     unsigned short *num_dests,
+					     unsigned short *tvlv_len)
+{
+	struct hlist_node *node;
+	struct batadv_orig_node *orig_node;
+
+	rcu_read_lock();
+	__hlist_for_each_rcu(node, head) {
+		orig_node = batadv_mcast_forw_orig_entry(node, entry_offset);
+		if (!orig_node ||
+		    !batadv_mcast_forw_push_dest(bat_priv, skb, vid, orig_node,
+						 num_dests, tvlv_len)) {
+			rcu_read_unlock();
+			return false;
+		}
+	}
+	rcu_read_unlock();
+
+	return true;
+}
+
+/**
+ * batadv_mcast_forw_push_tt() - push originators with interest through TT
+ * @bat_priv: the bat priv with all the soft interface information
+ * @skb: the skb to push the destination addresses onto
+ * @vid: the vlan identifier
+ * @num_dests: a pointer to store the number of pushed addresses in
+ * @tvlv_len: stores the amount of currently pushed TVLV bytes
+ *
+ * Push the MAC addresses of all originators which have indicated interest in
+ * this multicast packet through the translation table onto the given skb.
+ *
+ * Return: true on success, false otherwise.
+ */
+static bool
+batadv_mcast_forw_push_tt(struct batadv_priv *bat_priv, struct sk_buff *skb,
+			  unsigned short vid, unsigned short *num_dests,
+			  unsigned short *tvlv_len)
+{
+	struct batadv_tt_orig_list_entry *orig_entry;
+
+	struct batadv_tt_global_entry *tt_global;
+	const u8 *addr = eth_hdr(skb)->h_dest;
+
+	/* ok */
+	int ret = true;
+
+	tt_global = batadv_tt_global_hash_find(bat_priv, addr, vid);
+	if (!tt_global)
+		goto out;
+
+	rcu_read_lock();
+	hlist_for_each_entry_rcu(orig_entry, &tt_global->orig_list, list) {
+		if (!batadv_mcast_forw_push_dest(bat_priv, skb, vid,
+						 orig_entry->orig_node,
+						 num_dests, tvlv_len)) {
+			ret = false;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	batadv_tt_global_entry_put(tt_global);
+
+out:
+	return ret;
+}
+
+/**
+ * batadv_mcast_forw_push_want_all() - push originators with want-all flag
+ * @bat_priv: the bat priv with all the soft interface information
+ * @skb: the skb to push the destination addresses onto
+ * @vid: the vlan identifier
+ * @num_dests: a pointer to store the number of pushed addresses in
+ * @tvlv_len: stores the amount of currently pushed TVLV bytes
+ *
+ * Push the MAC addresses of all originators which have indicated interest in
+ * this multicast packet through the want-all flag onto the given skb.
+ *
+ * Return: true on success, false otherwise.
+ */
+static bool batadv_mcast_forw_push_want_all(struct batadv_priv *bat_priv,
+					    struct sk_buff *skb,
+					    unsigned short vid,
+					    unsigned short *num_dests,
+					    unsigned short *tvlv_len)
+{
+	struct hlist_head *head = NULL;
+	size_t offset;
+	int ret;
+
+	switch (eth_hdr(skb)->h_proto) {
+	case htons(ETH_P_IP):
+		head = &bat_priv->mcast.want_all_ipv4_list;
+		offset = offsetof(struct batadv_orig_node,
+				  mcast_want_all_ipv4_node);
+		break;
+	case htons(ETH_P_IPV6):
+		head = &bat_priv->mcast.want_all_ipv6_list;
+		offset = offsetof(struct batadv_orig_node,
+				  mcast_want_all_ipv6_node);
+		break;
+	default:
+		return false;
+	}
+
+	ret = batadv_mcast_forw_push_dests_list(bat_priv, skb, vid, head,
+						offset, num_dests, tvlv_len);
+	if (!ret)
+		return false;
+
+	return true;
+}
+
+/**
+ * batadv_mcast_forw_push_want_rtr() - push originators with want-router flag
+ * @bat_priv: the bat priv with all the soft interface information
+ * @skb: the skb to push the destination addresses onto
+ * @vid: the vlan identifier
+ * @num_dests: a pointer to store the number of pushed addresses in
+ * @tvlv_len: stores the amount of currently pushed TVLV bytes
+ *
+ * Push the MAC addresses of all originators which have indicated interest in
+ * this multicast packet through the want-all-rtr flag onto the given skb.
+ *
+ * Return: true on success, false otherwise.
+ */
+static bool batadv_mcast_forw_push_want_rtr(struct batadv_priv *bat_priv,
+					    struct sk_buff *skb,
+					    unsigned short vid,
+					    unsigned short *num_dests,
+					    unsigned short *tvlv_len)
+{
+	struct hlist_head *head = NULL;
+	size_t offset;
+	int ret;
+
+	switch (eth_hdr(skb)->h_proto) {
+	case htons(ETH_P_IP):
+		head = &bat_priv->mcast.want_all_rtr4_list;
+		offset = offsetof(struct batadv_orig_node,
+				  mcast_want_all_rtr4_node);
+		break;
+	case htons(ETH_P_IPV6):
+		head = &bat_priv->mcast.want_all_rtr6_list;
+		offset = offsetof(struct batadv_orig_node,
+				  mcast_want_all_rtr6_node);
+		break;
+	default:
+		return false;
+	}
+
+	ret = batadv_mcast_forw_push_dests_list(bat_priv, skb, vid, head,
+						offset, num_dests, tvlv_len);
+	if (!ret)
+		return false;
+
+	return true;
+}
+
+/**
+ * batadv_mcast_forw_scrape() - remove bytes within skb data
+ * @skb: the skb to remove bytes from
+ * @offset: the offset from the skb data from which to scrape
+ * @len: the amount of bytes to scrape starting from the offset
+ *
+ * Scrapes/removes len bytes from the given skb at the given offset from the
+ * skb data.
+ *
+ * Caller needs to ensure that the region from the skb data's start up
+ * to/including the to be removed bytes are linearized.
+ */
+static void batadv_mcast_forw_scrape(struct sk_buff *skb,
+				     unsigned short offset,
+				     unsigned short len)
+{
+	char *to, *from;
+
+	SKB_LINEAR_ASSERT(skb);
+
+	to = skb_pull(skb, len);
+	from = to - len;
+
+	memmove(to, from, offset);
+}
+
+/**
+ * batadv_mcast_forw_push_scrape_padding() - remove TVLV padding
+ * @skb: the skb to potentially adjust the TVLV's padding on
+ * @tvlv_len: stores the amount of currently pushed TVLV bytes
+ *
+ * Remove two padding bytes from the end of the multicast tracker TVLV,
+ * from before the payload data.
+ *
+ * Caller needs to ensure that the TVLV bytes are linearized.
+ */
+static void batadv_mcast_forw_push_scrape_padding(struct sk_buff *skb,
+						  unsigned short *tvlv_len)
+{
+	const int pad_len = 2;
+
+	batadv_mcast_forw_scrape(skb, *tvlv_len - pad_len, pad_len);
+	*tvlv_len -= pad_len;
+}
+
+/**
+ * batadv_mcast_forw_push_insert_padding() - insert TVLV padding
+ * @skb: the skb to potentially adjust the TVLV's padding on
+ * @tvlv_len: stores the amount of currently pushed TVLV bytes
+ *
+ * Inserts two padding bytes at the end of the multicast tracker TVLV,
+ * before the payload data in the given skb.
+ *
+ * Return: true on success, false otherwise.
+ */
+static bool batadv_mcast_forw_push_insert_padding(struct sk_buff *skb,
+						  unsigned short *tvlv_len)
+{
+	unsigned short offset =	*tvlv_len;
+	char *to, *from = skb->data;
+
+	to = batadv_mcast_forw_push_padding(skb, tvlv_len);
+	if (!to)
+		return false;
+
+	memmove(to, from, offset);
+	memset(to + offset, 0, *tvlv_len - offset);
+	return true;
+}
+
+/**
+ * batadv_mcast_forw_push_adjust_padding() - adjust padding if necessary
+ * @skb: the skb to potentially adjust the TVLV's padding on
+ * @count: the estimated number of originators the multicast packet needs to
+ *  be sent to
+ * @num_dests_pushed: the number of originators that were actually added to the
+ *  multicast packet's tracker TVLV
+ * @tvlv_len: stores the amount of currently pushed TVLV bytes
+ *
+ * Adjusts the padding in the multicast packet's tracker TVLV depending on the
+ * initially estimated amount of destinations versus the amount of destinations
+ * that were actually added to the tracker TVLV.
+ *
+ * If the initial estimate was correct or at least the oddness was the same then
+ * no padding adjustment is performed.
+ * If the initially estimated number was even, so padding was initially added,
+ * but it turned out to be odd then padding is removed.
+ * If the initially estimated number was odd, so no padding was initially added,
+ * but it turned out to be even then padding is added.
+ *
+ * Return: true if no padding adjustment is needed or the adjustment was
+ * successful, false otherwise.
+ */
+static bool
+batadv_mcast_forw_push_adjust_padding(struct sk_buff *skb, int *count,
+				      unsigned short num_dests_pushed,
+				      unsigned short *tvlv_len)
+{
+	int ret = true;
+
+	if (likely((num_dests_pushed % 2) == (*count % 2)))
+		goto out;
+
+	/**
+	 * estimated even number of destinations, but turned out to be odd
+	 * -> remove padding
+	 */
+	if (!(*count % 2) && (num_dests_pushed % 2))
+		batadv_mcast_forw_push_scrape_padding(skb, tvlv_len);
+	/**
+	 * estimated odd number of destinations, but turned out to be even
+	 * -> add padding
+	 */
+	else if ((*count % 2) && (!(num_dests_pushed % 2)))
+		ret = batadv_mcast_forw_push_insert_padding(skb, tvlv_len);
+
+out:
+	*count = num_dests_pushed;
+	return ret;
+}
+
+/**
+ * batadv_mcast_forw_push_dests() - push originator addresses onto an skb
+ * @bat_priv: the bat priv with all the soft interface information
+ * @skb: the skb to push the destination addresses onto
+ * @vid: the vlan identifier
+ * @is_routable: indicates whether the destination is routable
+ * @count: the number of originators the multicast packet needs to be sent to
+ * @tvlv_len: stores the amount of currently pushed TVLV bytes
+ *
+ * Push the MAC addresses of all originators which have indicated interest in
+ * this multicast packet onto the given skb.
+ *
+ * Return: -ENOMEM if there is not enough skb headroom available. Otherwise, on
+ * success 0.
+ */
+static int
+batadv_mcast_forw_push_dests(struct batadv_priv *bat_priv, struct sk_buff *skb,
+			     unsigned short vid, int is_routable, int *count,
+			     unsigned short *tvlv_len)
+{
+	unsigned short num_dests = 0;
+
+	if (!batadv_mcast_forw_push_est_padding(skb, *count, tvlv_len))
+		goto err;
+
+	if (!batadv_mcast_forw_push_tt(bat_priv, skb, vid, &num_dests,
+				       tvlv_len))
+		goto err;
+
+	if (!batadv_mcast_forw_push_want_all(bat_priv, skb, vid, &num_dests,
+					     tvlv_len))
+		goto err;
+
+	if (is_routable &&
+	    !batadv_mcast_forw_push_want_rtr(bat_priv, skb, vid, &num_dests,
+					     tvlv_len))
+		goto err;
+
+	if (!batadv_mcast_forw_push_adjust_padding(skb, count, num_dests,
+						   tvlv_len))
+		goto err;
+
+	return 0;
+err:
+	return -ENOMEM;
+}
+
+/**
+ * batadv_mcast_forw_push_tracker() - push a multicast tracker TVLV header
+ * @skb: the skb to push the tracker TVLV onto
+ * @num_dests: the number of destination addresses to set in the header
+ * @tvlv_len: stores the amount of currently pushed TVLV bytes
+ *
+ * Pushes a multicast tracker TVLV header onto the given skb, including the
+ * generic TVLV header but excluding the destination MAC addresses.
+ *
+ * The provided num_dests value is taken into consideration to set the
+ * num_dests field in the tracker header and to set the appropriate TVLV length
+ * value fields.
+ *
+ * Return: -ENOMEM if there is not enough skb headroom available. Otherwise, on
+ * success 0.
+ */
+static int batadv_mcast_forw_push_tracker(struct sk_buff *skb, int num_dests,
+					  unsigned short *tvlv_len)
+{
+	struct batadv_tvlv_mcast_tracker *mcast_tracker;
+	struct batadv_tvlv_hdr *tvlv_hdr;
+	unsigned int tvlv_value_len;
+
+	if (skb_headroom(skb) < sizeof(*mcast_tracker) + sizeof(*tvlv_hdr))
+		return -ENOMEM;
+
+	tvlv_value_len = sizeof(*mcast_tracker) + *tvlv_len;
+	if (tvlv_value_len + sizeof(*tvlv_hdr) > U16_MAX)
+		return -ENOMEM;
+
+	batadv_mcast_forw_skb_push(skb, sizeof(*mcast_tracker), tvlv_len);
+	mcast_tracker = (struct batadv_tvlv_mcast_tracker *)skb->data;
+	mcast_tracker->num_dests = htons(num_dests);
+
+	skb_reset_network_header(skb);
+
+	batadv_mcast_forw_skb_push(skb, sizeof(*tvlv_hdr), tvlv_len);
+	tvlv_hdr = (struct batadv_tvlv_hdr *)skb->data;
+	tvlv_hdr->type = BATADV_TVLV_MCAST_TRACKER;
+	tvlv_hdr->version = 1;
+	tvlv_hdr->len = htons(tvlv_value_len);
+
+	return 0;
+}
+
+/**
+ * batadv_mcast_forw_push_tvlvs() - push a multicast tracker TVLV onto an skb
+ * @bat_priv: the bat priv with all the soft interface information
+ * @skb: the skb to push the tracker TVLV onto
+ * @vid: the vlan identifier
+ * @is_routable: indicates whether the destination is routable
+ * @count: the number of originators the multicast packet needs to be sent to
+ * @tvlv_len: stores the amount of currently pushed TVLV bytes
+ *
+ * Pushes a multicast tracker TVLV onto the given skb, including the collected
+ * destination MAC addresses and the generic TVLV header.
+ *
+ * Return: -ENOMEM if there is not enough skb headroom available. Otherwise, on
+ * success 0.
+ */
+static int
+batadv_mcast_forw_push_tvlvs(struct batadv_priv *bat_priv, struct sk_buff *skb,
+			     unsigned short vid, int is_routable, int count,
+			     unsigned short *tvlv_len)
+{
+	int ret;
+
+	ret = batadv_mcast_forw_push_dests(bat_priv, skb, vid, is_routable,
+					   &count, tvlv_len);
+	if (ret < 0)
+		return ret;
+
+	ret = batadv_mcast_forw_push_tracker(skb, count, tvlv_len);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+/**
+ * batadv_mcast_forw_push_hdr() - push a multicast packet header onto an skb
+ * @skb: the skb to push the header onto
+ * @tvlv_len: the total TVLV length value to set in the header
+ *
+ * Pushes a batman-adv multicast packet header onto the given skb and sets
+ * the provided total TVLV length value in it.
+ *
+ * Caller needs to ensure enough skb headroom is available.
+ *
+ * Return: -ENOMEM if there is not enough skb headroom available. Otherwise, on
+ * success 0.
+ */
+static int
+batadv_mcast_forw_push_hdr(struct sk_buff *skb, unsigned short tvlv_len)
+{
+	struct batadv_mcast_packet *mcast_packet;
+
+	if (skb_headroom(skb) < sizeof(*mcast_packet))
+		return -ENOMEM;
+
+	skb_push(skb, sizeof(*mcast_packet));
+
+	mcast_packet = (struct batadv_mcast_packet *)skb->data;
+	mcast_packet->version = BATADV_COMPAT_VERSION;
+	mcast_packet->ttl = BATADV_TTL;
+	mcast_packet->packet_type = BATADV_MCAST;
+	mcast_packet->reserved = 0;
+	mcast_packet->tvlv_len = htons(tvlv_len);
+
+	return 0;
+}
+
+/**
+ * batadv_mcast_forw_scrub_dests() - scrub destinations in a tracker TVLV
+ * @bat_priv: the bat priv with all the soft interface information
+ * @comp_neigh: next hop neighbor to scrub+collect destinations for
+ * @dest: start MAC entry in original skb's tracker TVLV
+ * @next_dest: start MAC entry in to be sent skb's tracker TVLV
+ * @num_dests: number of remaining destination MAC entries to iterate over
+ *
+ * This sorts destination entries into either the original batman-adv
+ * multicast packet or the skb (copy) that is going to be sent to comp_neigh
+ * next.
+ *
+ * In preparation for the next, to be (unicast) transmitted batman-adv multicast
+ * packet skb to be sent to the given neighbor node, tries to collect all
+ * originator MAC addresses that have the given neighbor node as their next hop
+ * in the to be transmitted skb (copy), which next_dest points into. That is we
+ * zero all destination entries in next_dest which do not have comp_neigh as
+ * their next hop. And zero all destination entries in the original skb that
+ * would have comp_neigh as their next hop (to avoid redundant transmissions and
+ * duplicated payload later).
+ */
+static void
+batadv_mcast_forw_scrub_dests(struct batadv_priv *bat_priv,
+			      struct batadv_neigh_node *comp_neigh, u8 *dest,
+			      u8 *next_dest, u16 num_dests)
+{
+	struct batadv_neigh_node *next_neigh;
+
+	/* skip first entry, this is what we are comparing with */
+	eth_zero_addr(dest);
+	dest += ETH_ALEN;
+	next_dest += ETH_ALEN;
+	num_dests--;
+
+	batadv_mcast_forw_tracker_for_each_dest2(dest, next_dest, num_dests) {
+		if (is_zero_ether_addr(next_dest))
+			continue;
+
+		/* sanity check, we expect unicast destinations */
+		if (is_multicast_ether_addr(next_dest)) {
+			eth_zero_addr(dest);
+			eth_zero_addr(next_dest);
+			continue;
+		}
+
+		next_neigh = batadv_orig_to_router(bat_priv, next_dest, NULL);
+		if (!next_neigh) {
+			eth_zero_addr(next_dest);
+			continue;
+		}
+
+		if (!batadv_compare_eth(next_neigh->addr, comp_neigh->addr)) {
+			eth_zero_addr(next_dest);
+			batadv_neigh_node_put(next_neigh);
+			continue;
+		}
+
+		/* found an entry for our next packet to transmit, so remove it
+		 * from the original packet
+		 */
+		eth_zero_addr(dest);
+		batadv_neigh_node_put(next_neigh);
+	}
+}
+
+/**
+ * batadv_mcast_forw_shrink_fill() - swap slot with next non-zero destination
+ * @slot: the to be filled zero-MAC destination entry in a tracker TVLV
+ * @num_dests_slot: remaining entries in tracker TVLV from/including slot
+ *
+ * Searches for the next non-zero-MAC destination entry in a tracker TVLV after
+ * the given slot pointer. And if found, swaps it with the zero-MAC destination
+ * entry which the slot points to.
+ *
+ * Return: true if slot was swapped/filled successfully, false otherwise.
+ */
+static bool batadv_mcast_forw_shrink_fill(u8 *slot, u16 num_dests_slot)
+{
+	u16 num_dests_filler;
+	u8 *filler;
+
+	/* sanity check, should not happen */
+	if (!num_dests_slot)
+		return false;
+
+	num_dests_filler = num_dests_slot - 1;
+	filler = slot + ETH_ALEN;
+
+	/* find a candidate to fill the empty slot */
+	batadv_mcast_forw_tracker_for_each_dest(filler, num_dests_filler) {
+		if (is_zero_ether_addr(filler))
+			continue;
+
+		ether_addr_copy(slot, filler);
+		eth_zero_addr(filler);
+		return true;
+	}
+
+	return false;
+}
+
+/**
+ * batadv_mcast_forw_shrink_pack_dests() - pack destinations of a tracker TVLV
+ * @skb: the batman-adv multicast packet to compact destinations in
+ *
+ * Compacts the originator destination MAC addresses in the multicast tracker
+ * TVLV of the given multicast packet. This is done by moving all non-zero
+ * MAC addresses in direction of the skb head and all zero MAC addresses in skb
+ * tail direction, within the multicast tracker TVLV.
+ *
+ * Return: The number of consecutive zero MAC address destinations which are
+ * now at the end of the multicast tracker TVLV.
+ */
+static int batadv_mcast_forw_shrink_pack_dests(struct sk_buff *skb)
+{
+	struct batadv_tvlv_mcast_tracker *mcast_tracker;
+	unsigned char *skb_net_hdr;
+	u16 num_dests_slot;
+	u8 *slot;
+
+	skb_net_hdr = skb_network_header(skb);
+	mcast_tracker = (struct batadv_tvlv_mcast_tracker *)skb_net_hdr;
+	num_dests_slot = ntohs(mcast_tracker->num_dests);
+
+	slot = (u8 *)mcast_tracker + sizeof(*mcast_tracker);
+
+	batadv_mcast_forw_tracker_for_each_dest(slot, num_dests_slot) {
+		/* find an empty slot */
+		if (!is_zero_ether_addr(slot))
+			continue;
+
+		if (!batadv_mcast_forw_shrink_fill(slot, num_dests_slot))
+			/* could not find a filler, so we successfully packed
+			 * and can stop - and must not reduce num_dests_slot!
+			 */
+			break;
+	}
+
+	/* num_dests_slot is now the amount of reduced, zeroed
+	 * destinations at the end of the tracker TVLV
+	 */
+	return num_dests_slot;
+}
+
+/**
+ * batadv_mcast_forw_shrink_align_offset() - get new alignment offset
+ * @num_dests_old: the old, to be updated amount of destination nodes
+ * @num_dests_reduce: the number of destinations that were removed
+ *
+ * Calculates the amount of potential extra alignment offset that is needed to
+ * adjust the TVLV padding after the change in destination nodes.
+ *
+ * Return:
+ *	0: If no change to padding is needed.
+ *	2: If padding needs to be removed.
+ *	-2: If padding needs to be added.
+ */
+static short
+batadv_mcast_forw_shrink_align_offset(unsigned int num_dests_old,
+				      unsigned int num_dests_reduce)
+{
+	/* even amount of removed destinations -> no alignment change */
+	if (!(num_dests_reduce % 2))
+		return 0;
+
+	/* even to odd amount of destinations -> remove padding */
+	if (!(num_dests_old % 2))
+		return 2;
+
+	/* odd to even amount of destinations -> add padding */
+	return -2;
+}
+
+/**
+ * batadv_mcast_forw_shrink_update_headers() - update shrunk mc packet headers
+ * @skb: the batman-adv multicast packet to update headers of
+ * @num_dests_reduce: the number of destinations that were removed
+ *
+ * This updates any fields of a batman-adv multicast packet that are affected
+ * by the reduced number of destinations in the multicast tracket TVLV. In
+ * particular this updates:
+ *
+ * The num_dest field of the multicast tracker TVLV.
+ * The TVLV length field of the according generic TVLV header.
+ * The batman-adv multicast packet's total TVLV length field.
+ *
+ * Return: The offset in skb's tail direction at which the new batman-adv
+ * multicast packet header needs to start.
+ */
+static unsigned int
+batadv_mcast_forw_shrink_update_headers(struct sk_buff *skb,
+					unsigned int num_dests_reduce)
+{
+	struct batadv_tvlv_mcast_tracker *mcast_tracker;
+	struct batadv_mcast_packet *mcast_packet;
+	struct batadv_tvlv_hdr *tvlv_hdr;
+	unsigned char *skb_net_hdr;
+	unsigned int offset;
+	short align_offset;
+	u16 num_dests;
+
+	skb_net_hdr = skb_network_header(skb);
+	mcast_tracker = (struct batadv_tvlv_mcast_tracker *)skb_net_hdr;
+	num_dests = ntohs(mcast_tracker->num_dests);
+
+	align_offset = batadv_mcast_forw_shrink_align_offset(num_dests,
+							     num_dests_reduce);
+	offset = ETH_ALEN * num_dests_reduce + align_offset;
+	num_dests -= num_dests_reduce;
+
+	/* update tracker header */
+	mcast_tracker->num_dests = htons(num_dests);
+
+	/* update tracker's tvlv header's length field */
+	tvlv_hdr = (struct batadv_tvlv_hdr *)(skb_network_header(skb) -
+					      sizeof(*tvlv_hdr));
+	tvlv_hdr->len = htons(ntohs(tvlv_hdr->len) - offset);
+
+	/* update multicast packet header's tvlv length field */
+	mcast_packet = (struct batadv_mcast_packet *)skb->data;
+	mcast_packet->tvlv_len = htons(ntohs(mcast_packet->tvlv_len) - offset);
+
+	return offset;
+}
+
+/**
+ * batadv_mcast_forw_shrink_move_headers() - move multicast headers by offset
+ * @skb: the batman-adv multicast packet to move headers for
+ * @offset: a non-negative offset to move headers by, towards the skb tail
+ *
+ * Moves the batman-adv multicast packet header, its multicast tracker TVLV and
+ * any TVLVs in between by the given offset in direction towards the tail.
+ */
+static void
+batadv_mcast_forw_shrink_move_headers(struct sk_buff *skb, unsigned int offset)
+{
+	struct batadv_tvlv_mcast_tracker *mcast_tracker;
+	unsigned char *skb_net_hdr;
+	unsigned int len;
+	u16 num_dests;
+
+	skb_net_hdr = skb_network_header(skb);
+	mcast_tracker = (struct batadv_tvlv_mcast_tracker *)skb_net_hdr;
+	num_dests = ntohs(mcast_tracker->num_dests);
+	len = skb_network_offset(skb) + sizeof(*mcast_tracker);
+	len += num_dests * ETH_ALEN;
+
+	batadv_mcast_forw_scrape(skb, len, offset);
+}
+
+/**
+ * batadv_mcast_forw_shrink_tracker() - remove zero addresses in a tracker tvlv
+ * @skb: the batman-adv multicast packet to (potentially) shrink
+ *
+ * Removes all destinations with a zero MAC addresses (00:00:00:00:00:00) from
+ * the given batman-adv multicast packet's tracker TVLV and updates headers
+ * accordingly to maintain a valid batman-adv multicast packet.
+ */
+static void batadv_mcast_forw_shrink_tracker(struct sk_buff *skb)
+{
+	unsigned int offset;
+	u16 dests_reduced;
+
+	dests_reduced = batadv_mcast_forw_shrink_pack_dests(skb);
+	if (!dests_reduced)
+		return;
+
+	offset = batadv_mcast_forw_shrink_update_headers(skb, dests_reduced);
+	batadv_mcast_forw_shrink_move_headers(skb, offset);
+}
+
+/**
+ * batadv_mcast_forw_packet() - forward a batman-adv multicast packet
+ * @bat_priv: the bat priv with all the soft interface information
+ * @skb: the received or locally generated batman-adv multicast packet
+ * @local_xmit: indicates that the packet was locally generated and not received
+ *
+ * Parses the tracker TVLV of a batman-adv multicast packet and forwards the
+ * packet as indicated in this TVLV.
+ *
+ * Caller needs to set the skb network header to the start of the multicast
+ * tracker TVLV (excluding the generic TVLV header) and the skb transport header
+ * to the next byte after this multicast tracker TVLV.
+ *
+ * Caller needs to free the skb.
+ *
+ * Return: NET_RX_SUCCESS or NET_RX_DROP on success or a negative error
+ * code on failure. NET_RX_SUCCESS if the received packet is supposed to be
+ * decapsulated and forwarded to the own soft interface, NET_RX_DROP otherwise.
+ */
+static int batadv_mcast_forw_packet(struct batadv_priv *bat_priv,
+				    struct sk_buff *skb, bool local_xmit)
+{
+	struct batadv_tvlv_mcast_tracker *mcast_tracker;
+	struct batadv_neigh_node *neigh_node;
+	unsigned long offset, num_dests_off;
+	struct sk_buff *nexthop_skb;
+	unsigned char *skb_net_hdr;
+	bool local_recv = false;
+	unsigned int tvlv_len;
+	bool xmitted = false;
+	u8 *dest, *next_dest;
+	u16 num_dests;
+	int ret;
+
+	/* (at least) TVLV part needs to be linearized */
+	SKB_LINEAR_ASSERT(skb);
+
+	/* check if num_dests is within skb length */
+	num_dests_off = offsetof(struct batadv_tvlv_mcast_tracker, num_dests);
+	if (num_dests_off > skb_network_header_len(skb))
+		return -EINVAL;
+
+	skb_net_hdr = skb_network_header(skb);
+	mcast_tracker = (struct batadv_tvlv_mcast_tracker *)skb_net_hdr;
+	num_dests = ntohs(mcast_tracker->num_dests);
+
+	dest = (u8 *)mcast_tracker + sizeof(*mcast_tracker);
+
+	/* check if full tracker tvlv is within skb length */
+	tvlv_len = sizeof(*mcast_tracker) + ETH_ALEN * num_dests;
+	if (tvlv_len > skb_network_header_len(skb))
+		return -EINVAL;
+
+	/* invalidate checksum: */
+	skb->ip_summed = CHECKSUM_NONE;
+
+	batadv_mcast_forw_tracker_for_each_dest(dest, num_dests) {
+		if (is_zero_ether_addr(dest))
+			continue;
+
+		/* only unicast originator addresses supported */
+		if (is_multicast_ether_addr(dest)) {
+			eth_zero_addr(dest);
+			continue;
+		}
+
+		if (batadv_is_my_mac(bat_priv, dest)) {
+			eth_zero_addr(dest);
+			local_recv = true;
+			continue;
+		}
+
+		neigh_node = batadv_orig_to_router(bat_priv, dest, NULL);
+		if (!neigh_node) {
+			eth_zero_addr(dest);
+			continue;
+		}
+
+		nexthop_skb = skb_copy(skb, GFP_ATOMIC);
+		if (!nexthop_skb) {
+			batadv_neigh_node_put(neigh_node);
+			return -ENOMEM;
+		}
+
+		offset = dest - skb->data;
+		next_dest = nexthop_skb->data + offset;
+
+		batadv_mcast_forw_scrub_dests(bat_priv, neigh_node, dest,
+					      next_dest, num_dests);
+		batadv_mcast_forw_shrink_tracker(nexthop_skb);
+
+		batadv_inc_counter(bat_priv, BATADV_CNT_MCAST_TX);
+		batadv_add_counter(bat_priv, BATADV_CNT_MCAST_TX_BYTES,
+				   nexthop_skb->len + ETH_HLEN);
+		xmitted = true;
+		ret = batadv_send_unicast_skb(nexthop_skb, neigh_node);
+
+		batadv_neigh_node_put(neigh_node);
+
+		if (ret < 0)
+			return ret;
+	}
+
+	if (xmitted) {
+		if (local_xmit) {
+			batadv_inc_counter(bat_priv, BATADV_CNT_MCAST_TX_LOCAL);
+			batadv_add_counter(bat_priv,
+					   BATADV_CNT_MCAST_TX_LOCAL_BYTES,
+					   skb->len -
+					   skb_transport_offset(skb));
+		} else {
+			batadv_inc_counter(bat_priv, BATADV_CNT_MCAST_FWD);
+			batadv_add_counter(bat_priv, BATADV_CNT_MCAST_FWD_BYTES,
+					   skb->len + ETH_HLEN);
+		}
+	}
+
+	if (local_recv)
+		return NET_RX_SUCCESS;
+	else
+		return NET_RX_DROP;
+}
+
+/**
+ * batadv_mcast_forw_tracker_tvlv_handler() - handle an mcast tracker tvlv
+ * @bat_priv: the bat priv with all the soft interface information
+ * @skb: the received batman-adv multicast packet
+ *
+ * Parses the tracker TVLV of an incoming batman-adv multicast packet and
+ * forwards the packet as indicated in this TVLV.
+ *
+ * Caller needs to set the skb network header to the start of the multicast
+ * tracker TVLV (excluding the generic TVLV header) and the skb transport header
+ * to the next byte after this multicast tracker TVLV.
+ *
+ * Caller needs to free the skb.
+ *
+ * Return: NET_RX_SUCCESS or NET_RX_DROP on success or a negative error
+ * code on failure. NET_RX_SUCCESS if the received packet is supposed to be
+ * decapsulated and forwarded to the own soft interface, NET_RX_DROP otherwise.
+ */
+int batadv_mcast_forw_tracker_tvlv_handler(struct batadv_priv *bat_priv,
+					   struct sk_buff *skb)
+{
+	return batadv_mcast_forw_packet(bat_priv, skb, false);
+}
+
+/**
+ * batadv_mcast_forw_packet_hdrlen() - multicast packet header length
+ * @num_dests: number of destination nodes
+ *
+ * Calculates the total batman-adv multicast packet header length for a given
+ * number of destination nodes (excluding the outer ethernet frame).
+ *
+ * Return: The calculated total batman-adv multicast packet header length.
+ */
+unsigned int batadv_mcast_forw_packet_hdrlen(unsigned int num_dests)
+{
+	/**
+	 * If the number of destination entries is even then we need to add
+	 * two byte padding to the tracker TVLV.
+	 */
+	int padding = (!(num_dests % 2)) ? 2 : 0;
+
+	return padding + num_dests * ETH_ALEN +
+	       sizeof(struct batadv_tvlv_mcast_tracker) +
+	       sizeof(struct batadv_tvlv_hdr) +
+	       sizeof(struct batadv_mcast_packet);
+}
+
+/**
+ * batadv_mcast_forw_expand_head() - expand headroom for an mcast packet
+ * @bat_priv: the bat priv with all the soft interface information
+ * @skb: the multicast packet to send
+ *
+ * Tries to expand an skb's headroom so that its head to tail is 1298
+ * bytes (minimum IPv6 MTU + vlan ethernet header size) large.
+ *
+ * Return: -EINVAL if the given skb's length is too large or -ENOMEM on memory
+ * allocation failure. Otherwise, on success, zero is returned.
+ */
+static int batadv_mcast_forw_expand_head(struct batadv_priv *bat_priv,
+					 struct sk_buff *skb)
+{
+	int hdr_size = VLAN_ETH_HLEN + IPV6_MIN_MTU - skb->len;
+
+	 /* TODO: Could be tightened to actual number of destination nodes?
+	  * But it's tricky, number of destinations might have increased since
+	  * we last checked.
+	  */
+	if (hdr_size < 0) {
+		/* batadv_mcast_forw_mode_check_count() should ensure we do not
+		 * end up here
+		 */
+		WARN_ON(1);
+		return -EINVAL;
+	}
+
+	if (skb_headroom(skb) < hdr_size &&
+	    pskb_expand_head(skb, hdr_size, 0, GFP_ATOMIC) < 0)
+		return -ENOMEM;
+
+	return 0;
+}
+
+/**
+ * batadv_mcast_forw_push() - encapsulate skb in a batman-adv multicast packet
+ * @bat_priv: the bat priv with all the soft interface information
+ * @skb: the multicast packet to encapsulate and send
+ * @vid: the vlan identifier
+ * @is_routable: indicates whether the destination is routable
+ * @count: the number of originators the multicast packet needs to be sent to
+ *
+ * Encapsulates the given multicast packet in a batman-adv multicast packet.
+ * A multicast tracker TVLV with destination originator addresses for any node
+ * that signaled interest in it, that is either via the translation table or the
+ * according want-all flags, is attached accordingly.
+ *
+ * Return: true on success, false otherwise.
+ */
+bool batadv_mcast_forw_push(struct batadv_priv *bat_priv, struct sk_buff *skb,
+			    unsigned short vid, int is_routable, int count)
+{
+	unsigned short tvlv_len = 0;
+	int ret;
+
+	if (batadv_mcast_forw_expand_head(bat_priv, skb) < 0)
+		goto err;
+
+	skb_reset_transport_header(skb);
+
+	ret = batadv_mcast_forw_push_tvlvs(bat_priv, skb, vid, is_routable,
+					   count, &tvlv_len);
+	if (ret < 0)
+		goto err;
+
+	ret = batadv_mcast_forw_push_hdr(skb, tvlv_len);
+	if (ret < 0)
+		goto err;
+
+	return true;
+
+err:
+	if (tvlv_len)
+		skb_pull(skb, tvlv_len);
+
+	return false;
+}
+
+/**
+ * batadv_mcast_forw_mcsend() - send a self prepared batman-adv multicast packet
+ * @bat_priv: the bat priv with all the soft interface information
+ * @skb: the multicast packet to encapsulate and send
+ *
+ * Transmits a batman-adv multicast packet that was locally prepared and
+ * consumes/frees it.
+ *
+ * Return: NET_XMIT_DROP on memory allocation failure. NET_XMIT_SUCCESS
+ * otherwise.
+ */
+int batadv_mcast_forw_mcsend(struct batadv_priv *bat_priv,
+			     struct sk_buff *skb)
+{
+	int ret = batadv_mcast_forw_packet(bat_priv, skb, true);
+
+	if (ret < 0) {
+		kfree_skb(skb);
+		return NET_XMIT_DROP;
+	}
+
+	consume_skb(skb);
+	return NET_XMIT_SUCCESS;
+}
diff --git a/net/batman-adv/netlink.c b/net/batman-adv/netlink.c
index 0c64d81a7761..1f7ed9d4f6fd 100644
--- a/net/batman-adv/netlink.c
+++ b/net/batman-adv/netlink.c
@@ -7,6 +7,7 @@
 #include "netlink.h"
 #include "main.h"
 
+#include <linux/array_size.h>
 #include <linux/atomic.h>
 #include <linux/bitops.h>
 #include <linux/bug.h>
@@ -20,7 +21,6 @@
 #include <linux/if_ether.h>
 #include <linux/if_vlan.h>
 #include <linux/init.h>
-#include <linux/kernel.h>
 #include <linux/limits.h>
 #include <linux/list.h>
 #include <linux/minmax.h>
diff --git a/net/batman-adv/originator.c b/net/batman-adv/originator.c
index 34903df4fe93..71c143d4b6d0 100644
--- a/net/batman-adv/originator.c
+++ b/net/batman-adv/originator.c
@@ -311,6 +311,33 @@ batadv_orig_router_get(struct batadv_orig_node *orig_node,
 	return router;
 }
 
+/**
+ * batadv_orig_to_router() - get next hop neighbor to an orig address
+ * @bat_priv: the bat priv with all the soft interface information
+ * @orig_addr: the originator MAC address to search the best next hop router for
+ * @if_outgoing: the interface where the payload packet has been received or
+ *  the OGM should be sent to
+ *
+ * Return: A neighbor node which is the best router towards the given originator
+ * address.
+ */
+struct batadv_neigh_node *
+batadv_orig_to_router(struct batadv_priv *bat_priv, u8 *orig_addr,
+		      struct batadv_hard_iface *if_outgoing)
+{
+	struct batadv_neigh_node *neigh_node;
+	struct batadv_orig_node *orig_node;
+
+	orig_node = batadv_orig_hash_find(bat_priv, orig_addr);
+	if (!orig_node)
+		return NULL;
+
+	neigh_node = batadv_find_router(bat_priv, orig_node, if_outgoing);
+	batadv_orig_node_put(orig_node);
+
+	return neigh_node;
+}
+
 /**
  * batadv_orig_ifinfo_get() - find the ifinfo from an orig_node
  * @orig_node: the orig node to be queried
@@ -942,6 +969,7 @@ struct batadv_orig_node *batadv_orig_node_new(struct batadv_priv *bat_priv,
 #ifdef CONFIG_BATMAN_ADV_MCAST
 	orig_node->mcast_flags = BATADV_MCAST_WANT_NO_RTR4;
 	orig_node->mcast_flags |= BATADV_MCAST_WANT_NO_RTR6;
+	orig_node->mcast_flags |= BATADV_MCAST_HAVE_MC_PTYPE_CAPA;
 	INIT_HLIST_NODE(&orig_node->mcast_want_all_unsnoopables_node);
 	INIT_HLIST_NODE(&orig_node->mcast_want_all_ipv4_node);
 	INIT_HLIST_NODE(&orig_node->mcast_want_all_ipv6_node);
diff --git a/net/batman-adv/originator.h b/net/batman-adv/originator.h
index ea3d69e4e670..db0c55128170 100644
--- a/net/batman-adv/originator.h
+++ b/net/batman-adv/originator.h
@@ -36,6 +36,9 @@ void batadv_neigh_node_release(struct kref *ref);
 struct batadv_neigh_node *
 batadv_orig_router_get(struct batadv_orig_node *orig_node,
 		       const struct batadv_hard_iface *if_outgoing);
+struct batadv_neigh_node *
+batadv_orig_to_router(struct batadv_priv *bat_priv, u8 *orig_addr,
+		      struct batadv_hard_iface *if_outgoing);
 struct batadv_neigh_ifinfo *
 batadv_neigh_ifinfo_new(struct batadv_neigh_node *neigh,
 			struct batadv_hard_iface *if_outgoing);
diff --git a/net/batman-adv/routing.c b/net/batman-adv/routing.c
index 163cd43c4821..f1061985149f 100644
--- a/net/batman-adv/routing.c
+++ b/net/batman-adv/routing.c
@@ -1270,3 +1270,73 @@ int batadv_recv_bcast_packet(struct sk_buff *skb,
 	batadv_orig_node_put(orig_node);
 	return ret;
 }
+
+#ifdef CONFIG_BATMAN_ADV_MCAST
+/**
+ * batadv_recv_mcast_packet() - process received batman-adv multicast packet
+ * @skb: the received batman-adv multicast packet
+ * @recv_if: interface that the skb is received on
+ *
+ * Parses the given, received batman-adv multicast packet. Depending on the
+ * contents of its TVLV forwards it and/or decapsulates it to hand it to the
+ * soft interface.
+ *
+ * Return: NET_RX_DROP if the skb is not consumed, NET_RX_SUCCESS otherwise.
+ */
+int batadv_recv_mcast_packet(struct sk_buff *skb,
+			     struct batadv_hard_iface *recv_if)
+{
+	struct batadv_priv *bat_priv = netdev_priv(recv_if->soft_iface);
+	struct batadv_mcast_packet *mcast_packet;
+	int hdr_size = sizeof(*mcast_packet);
+	unsigned char *tvlv_buff;
+	int ret = NET_RX_DROP;
+	u16 tvlv_buff_len;
+
+	if (batadv_check_unicast_packet(bat_priv, skb, hdr_size) < 0)
+		goto free_skb;
+
+	/* create a copy of the skb, if needed, to modify it. */
+	if (skb_cow(skb, ETH_HLEN) < 0)
+		goto free_skb;
+
+	/* packet needs to be linearized to access the tvlv content */
+	if (skb_linearize(skb) < 0)
+		goto free_skb;
+
+	mcast_packet = (struct batadv_mcast_packet *)skb->data;
+	if (mcast_packet->ttl-- < 2)
+		goto free_skb;
+
+	tvlv_buff = (unsigned char *)(skb->data + hdr_size);
+	tvlv_buff_len = ntohs(mcast_packet->tvlv_len);
+
+	if (tvlv_buff_len > skb->len - hdr_size)
+		goto free_skb;
+
+	ret = batadv_tvlv_containers_process(bat_priv, BATADV_MCAST, NULL, skb,
+					     tvlv_buff, tvlv_buff_len);
+	if (ret >= 0) {
+		batadv_inc_counter(bat_priv, BATADV_CNT_MCAST_RX);
+		batadv_add_counter(bat_priv, BATADV_CNT_MCAST_RX_BYTES,
+				   skb->len + ETH_HLEN);
+	}
+
+	hdr_size += tvlv_buff_len;
+
+	if (ret == NET_RX_SUCCESS && (skb->len - hdr_size >= ETH_HLEN)) {
+		batadv_inc_counter(bat_priv, BATADV_CNT_MCAST_RX_LOCAL);
+		batadv_add_counter(bat_priv, BATADV_CNT_MCAST_RX_LOCAL_BYTES,
+				   skb->len - hdr_size);
+
+		batadv_interface_rx(bat_priv->soft_iface, skb, hdr_size, NULL);
+		/* skb was consumed */
+		skb = NULL;
+	}
+
+free_skb:
+	kfree_skb(skb);
+
+	return ret;
+}
+#endif /* CONFIG_BATMAN_ADV_MCAST */
diff --git a/net/batman-adv/routing.h b/net/batman-adv/routing.h
index afd15b3879f1..e9849f032a24 100644
--- a/net/batman-adv/routing.h
+++ b/net/batman-adv/routing.h
@@ -27,6 +27,17 @@ int batadv_recv_frag_packet(struct sk_buff *skb,
 			    struct batadv_hard_iface *iface);
 int batadv_recv_bcast_packet(struct sk_buff *skb,
 			     struct batadv_hard_iface *recv_if);
+#ifdef CONFIG_BATMAN_ADV_MCAST
+int batadv_recv_mcast_packet(struct sk_buff *skb,
+			     struct batadv_hard_iface *recv_if);
+#else
+static inline int batadv_recv_mcast_packet(struct sk_buff *skb,
+					   struct batadv_hard_iface *recv_if)
+{
+	kfree_skb(skb);
+	return NET_RX_DROP;
+}
+#endif
 int batadv_recv_unicast_tvlv(struct sk_buff *skb,
 			     struct batadv_hard_iface *recv_if);
 int batadv_recv_unhandled_unicast_packet(struct sk_buff *skb,
diff --git a/net/batman-adv/soft-interface.c b/net/batman-adv/soft-interface.c
index 1bf1232a4f75..89c51b3cf430 100644
--- a/net/batman-adv/soft-interface.c
+++ b/net/batman-adv/soft-interface.c
@@ -301,12 +301,13 @@ static netdev_tx_t batadv_interface_tx(struct sk_buff *skb,
 
 send:
 		if (do_bcast && !is_broadcast_ether_addr(ethhdr->h_dest)) {
-			forw_mode = batadv_mcast_forw_mode(bat_priv, skb,
+			forw_mode = batadv_mcast_forw_mode(bat_priv, skb, vid,
 							   &mcast_is_routable);
 			switch (forw_mode) {
 			case BATADV_FORW_BCAST:
 				break;
 			case BATADV_FORW_UCASTS:
+			case BATADV_FORW_MCAST:
 				do_bcast = false;
 				break;
 			case BATADV_FORW_NONE:
@@ -365,6 +366,8 @@ static netdev_tx_t batadv_interface_tx(struct sk_buff *skb,
 		} else if (forw_mode == BATADV_FORW_UCASTS) {
 			ret = batadv_mcast_forw_send(bat_priv, skb, vid,
 						     mcast_is_routable);
+		} else if (forw_mode == BATADV_FORW_MCAST) {
+			ret = batadv_mcast_forw_mcsend(bat_priv, skb);
 		} else {
 			if (batadv_dat_snoop_outgoing_arp_request(bat_priv,
 								  skb))
@@ -762,6 +765,7 @@ static int batadv_softif_init_late(struct net_device *dev)
 	atomic_set(&bat_priv->mcast.num_want_all_unsnoopables, 0);
 	atomic_set(&bat_priv->mcast.num_want_all_ipv4, 0);
 	atomic_set(&bat_priv->mcast.num_want_all_ipv6, 0);
+	atomic_set(&bat_priv->mcast.num_no_mc_ptype_capa, 0);
 #endif
 	atomic_set(&bat_priv->gw.mode, BATADV_GW_MODE_OFF);
 	atomic_set(&bat_priv->gw.bandwidth_down, 100);
@@ -925,6 +929,18 @@ static const struct {
 	{ "tt_response_rx" },
 	{ "tt_roam_adv_tx" },
 	{ "tt_roam_adv_rx" },
+#ifdef CONFIG_BATMAN_ADV_MCAST
+	{ "mcast_tx" },
+	{ "mcast_tx_bytes" },
+	{ "mcast_tx_local" },
+	{ "mcast_tx_local_bytes" },
+	{ "mcast_rx" },
+	{ "mcast_rx_bytes" },
+	{ "mcast_rx_local" },
+	{ "mcast_rx_local_bytes" },
+	{ "mcast_fwd" },
+	{ "mcast_fwd_bytes" },
+#endif
 #ifdef CONFIG_BATMAN_ADV_DAT
 	{ "dat_get_tx" },
 	{ "dat_get_rx" },
diff --git a/net/batman-adv/types.h b/net/batman-adv/types.h
index 17d5ea1d8e84..00840d5784fe 100644
--- a/net/batman-adv/types.h
+++ b/net/batman-adv/types.h
@@ -862,6 +862,70 @@ enum batadv_counters {
 	 */
 	BATADV_CNT_TT_ROAM_ADV_RX,
 
+#ifdef CONFIG_BATMAN_ADV_MCAST
+	/**
+	 * @BATADV_CNT_MCAST_TX: transmitted batman-adv multicast packets
+	 *  counter
+	 */
+	BATADV_CNT_MCAST_TX,
+
+	/**
+	 * @BATADV_CNT_MCAST_TX_BYTES: transmitted batman-adv multicast packets
+	 *  bytes counter
+	 */
+	BATADV_CNT_MCAST_TX_BYTES,
+
+	/**
+	 * @BATADV_CNT_MCAST_TX_LOCAL: counter for multicast packets which
+	 *  were locally encapsulated and transmitted as batman-adv multicast
+	 *  packets
+	 */
+	BATADV_CNT_MCAST_TX_LOCAL,
+
+	/**
+	 * @BATADV_CNT_MCAST_TX_LOCAL_BYTES: bytes counter for multicast packets
+	 *  which were locally encapsulated and transmitted as batman-adv
+	 *  multicast packets
+	 */
+	BATADV_CNT_MCAST_TX_LOCAL_BYTES,
+
+	/**
+	 * @BATADV_CNT_MCAST_RX: received batman-adv multicast packet counter
+	 */
+	BATADV_CNT_MCAST_RX,
+
+	/**
+	 * @BATADV_CNT_MCAST_RX_BYTES: received batman-adv multicast packet
+	 *  bytes counter
+	 */
+	BATADV_CNT_MCAST_RX_BYTES,
+
+	/**
+	 * @BATADV_CNT_MCAST_RX_LOCAL: counter for received batman-adv multicast
+	 *  packets which were forwarded to the local soft interface
+	 */
+	BATADV_CNT_MCAST_RX_LOCAL,
+
+	/**
+	 * @BATADV_CNT_MCAST_RX_LOCAL_BYTES: bytes counter for received
+	 *  batman-adv multicast packets which were forwarded to the local soft
+	 *  interface
+	 */
+	BATADV_CNT_MCAST_RX_LOCAL_BYTES,
+
+	/**
+	 * @BATADV_CNT_MCAST_FWD: counter for received batman-adv multicast
+	 *  packets which were forwarded to other, neighboring nodes
+	 */
+	BATADV_CNT_MCAST_FWD,
+
+	/**
+	 * @BATADV_CNT_MCAST_FWD_BYTES: bytes counter for received batman-adv
+	 *  multicast packets which were forwarded to other, neighboring nodes
+	 */
+	BATADV_CNT_MCAST_FWD_BYTES,
+#endif
+
 #ifdef CONFIG_BATMAN_ADV_DAT
 	/**
 	 * @BATADV_CNT_DAT_GET_TX: transmitted dht GET traffic packet counter
@@ -1278,6 +1342,12 @@ struct batadv_priv_mcast {
 	/** @num_want_all_rtr6: counter for items in want_all_rtr6_list */
 	atomic_t num_want_all_rtr6;
 
+	/**
+	 * @num_no_mc_ptype_capa: counter for number of nodes without the
+	 *  BATADV_MCAST_HAVE_MC_PTYPE_CAPA flag
+	 */
+	atomic_t num_no_mc_ptype_capa;
+
 	/**
 	 * @want_lists_lock: lock for protecting modifications to mcasts
 	 *  want_all_{unsnoopables,ipv4,ipv6}_list (traversals are rcu-locked)
diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c
index 2cee330188ce..a09071059214 100644
--- a/net/bluetooth/hci_conn.c
+++ b/net/bluetooth/hci_conn.c
@@ -1086,8 +1086,9 @@ static void hci_conn_cleanup_child(struct hci_conn *conn, u8 reason)
 			hci_conn_failed(conn, reason);
 		break;
 	case ISO_LINK:
-		if (conn->state != BT_CONNECTED &&
-		    !test_bit(HCI_CONN_CREATE_CIS, &conn->flags))
+		if ((conn->state != BT_CONNECTED &&
+		    !test_bit(HCI_CONN_CREATE_CIS, &conn->flags)) ||
+		    test_bit(HCI_CONN_BIG_CREATED, &conn->flags))
 			hci_conn_failed(conn, reason);
 		break;
 	}
@@ -2228,7 +2229,17 @@ struct hci_conn *hci_bind_bis(struct hci_dev *hdev, bdaddr_t *dst,
 			      __u8 base_len, __u8 *base)
 {
 	struct hci_conn *conn;
+	struct hci_conn *parent;
 	__u8 eir[HCI_MAX_PER_AD_LENGTH];
+	struct hci_link *link;
+
+	/* Look for any BIS that is open for rebinding */
+	conn = hci_conn_hash_lookup_big_state(hdev, qos->bcast.big, BT_OPEN);
+	if (conn) {
+		memcpy(qos, &conn->iso_qos, sizeof(*qos));
+		conn->state = BT_CONNECTED;
+		return conn;
+	}
 
 	if (base_len && base)
 		base_len = eir_append_service_data(eir, 0,  0x1851,
@@ -2256,6 +2267,20 @@ struct hci_conn *hci_bind_bis(struct hci_dev *hdev, bdaddr_t *dst,
 	conn->iso_qos = *qos;
 	conn->state = BT_BOUND;
 
+	/* Link BISes together */
+	parent = hci_conn_hash_lookup_big(hdev,
+					  conn->iso_qos.bcast.big);
+	if (parent && parent != conn) {
+		link = hci_conn_link(parent, conn);
+		if (!link) {
+			hci_conn_drop(conn);
+			return ERR_PTR(-ENOLINK);
+		}
+
+		/* Link takes the refcount */
+		hci_conn_drop(conn);
+	}
+
 	return conn;
 }
 
@@ -2287,6 +2312,9 @@ struct hci_conn *hci_connect_bis(struct hci_dev *hdev, bdaddr_t *dst,
 	if (IS_ERR(conn))
 		return conn;
 
+	if (conn->state == BT_CONNECTED)
+		return conn;
+
 	data.big = qos->bcast.big;
 	data.bis = qos->bcast.bis;
 
diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c
index 0849e0dafa95..5b6fd625fc09 100644
--- a/net/bluetooth/hci_event.c
+++ b/net/bluetooth/hci_event.c
@@ -820,8 +820,6 @@ static u8 hci_cc_write_auth_payload_timeout(struct hci_dev *hdev, void *data,
 	if (!rp->status)
 		conn->auth_payload_timeout = get_unaligned_le16(sent + 2);
 
-	hci_encrypt_cfm(conn, 0);
-
 unlock:
 	hci_dev_unlock(hdev);
 
@@ -3683,12 +3681,8 @@ static void hci_encrypt_change_evt(struct hci_dev *hdev, void *data,
 		cp.handle = cpu_to_le16(conn->handle);
 		cp.timeout = cpu_to_le16(hdev->auth_payload_timeout);
 		if (hci_send_cmd(conn->hdev, HCI_OP_WRITE_AUTH_PAYLOAD_TO,
-				 sizeof(cp), &cp)) {
+				 sizeof(cp), &cp))
 			bt_dev_err(hdev, "write auth payload timeout failed");
-			goto notify;
-		}
-
-		goto unlock;
 	}
 
 notify:
diff --git a/net/bluetooth/iso.c b/net/bluetooth/iso.c
index 07b80e97aead..e49f00e8a6a6 100644
--- a/net/bluetooth/iso.c
+++ b/net/bluetooth/iso.c
@@ -574,19 +574,68 @@ static struct sock *iso_get_sock_listen(bdaddr_t *src, bdaddr_t *dst,
 			continue;
 
 		/* Exact match. */
-		if (!bacmp(&iso_pi(sk)->src, src))
+		if (!bacmp(&iso_pi(sk)->src, src)) {
+			sock_hold(sk);
 			break;
+		}
 
 		/* Closest match */
-		if (!bacmp(&iso_pi(sk)->src, BDADDR_ANY))
+		if (!bacmp(&iso_pi(sk)->src, BDADDR_ANY)) {
+			if (sk1)
+				sock_put(sk1);
+
 			sk1 = sk;
+			sock_hold(sk1);
+		}
 	}
 
+	if (sk && sk1)
+		sock_put(sk1);
+
 	read_unlock(&iso_sk_list.lock);
 
 	return sk ? sk : sk1;
 }
 
+static struct sock *iso_get_sock_big(struct sock *match_sk, bdaddr_t *src,
+				     bdaddr_t *dst, uint8_t big)
+{
+	struct sock *sk = NULL;
+
+	read_lock(&iso_sk_list.lock);
+
+	sk_for_each(sk, &iso_sk_list.head) {
+		if (match_sk == sk)
+			continue;
+
+		/* Look for sockets that have already been
+		 * connected to the BIG
+		 */
+		if (sk->sk_state != BT_CONNECTED &&
+		    sk->sk_state != BT_CONNECT)
+			continue;
+
+		/* Match Broadcast destination */
+		if (bacmp(&iso_pi(sk)->dst, dst))
+			continue;
+
+		/* Match BIG handle */
+		if (iso_pi(sk)->qos.bcast.big != big)
+			continue;
+
+		/* Match source address */
+		if (bacmp(&iso_pi(sk)->src, src))
+			continue;
+
+		sock_hold(sk);
+		break;
+	}
+
+	read_unlock(&iso_sk_list.lock);
+
+	return sk;
+}
+
 static void iso_sock_destruct(struct sock *sk)
 {
 	BT_DBG("sk %p", sk);
@@ -639,6 +688,28 @@ static void iso_sock_kill(struct sock *sk)
 
 static void iso_sock_disconn(struct sock *sk)
 {
+	struct sock *bis_sk;
+	struct hci_conn *hcon = iso_pi(sk)->conn->hcon;
+
+	if (test_bit(HCI_CONN_BIG_CREATED, &hcon->flags)) {
+		bis_sk = iso_get_sock_big(sk, &iso_pi(sk)->src,
+					  &iso_pi(sk)->dst,
+					  iso_pi(sk)->qos.bcast.big);
+
+		/* If there are any other connected sockets for the
+		 * same BIG, just delete the sk and leave the bis
+		 * hcon active, in case later rebinding is needed.
+		 */
+		if (bis_sk) {
+			hcon->state = BT_OPEN;
+			iso_pi(sk)->conn->hcon = NULL;
+			iso_sock_clear_timer(sk);
+			iso_chan_del(sk, bt_to_errno(hcon->abort_reason));
+			sock_put(bis_sk);
+			return;
+		}
+	}
+
 	sk->sk_state = BT_DISCONN;
 	iso_sock_set_timer(sk, ISO_DISCONN_TIMEOUT);
 	iso_conn_lock(iso_pi(sk)->conn);
@@ -792,27 +863,75 @@ static int iso_sock_bind_bc(struct socket *sock, struct sockaddr *addr,
 	BT_DBG("sk %p bc_sid %u bc_num_bis %u", sk, sa->iso_bc->bc_sid,
 	       sa->iso_bc->bc_num_bis);
 
-	if (addr_len > sizeof(*sa) + sizeof(*sa->iso_bc))
+	if (addr_len != sizeof(*sa) + sizeof(*sa->iso_bc))
 		return -EINVAL;
 
 	bacpy(&iso_pi(sk)->dst, &sa->iso_bc->bc_bdaddr);
+
+	/* Check if the address type is of LE type */
+	if (!bdaddr_type_is_le(sa->iso_bc->bc_bdaddr_type))
+		return -EINVAL;
+
 	iso_pi(sk)->dst_type = sa->iso_bc->bc_bdaddr_type;
 	iso_pi(sk)->sync_handle = -1;
+
+	if (sa->iso_bc->bc_sid > 0x0f)
+		return -EINVAL;
+
 	iso_pi(sk)->bc_sid = sa->iso_bc->bc_sid;
+
+	if (sa->iso_bc->bc_num_bis > ISO_MAX_NUM_BIS)
+		return -EINVAL;
+
 	iso_pi(sk)->bc_num_bis = sa->iso_bc->bc_num_bis;
 
-	for (i = 0; i < iso_pi(sk)->bc_num_bis; i++) {
+	for (i = 0; i < iso_pi(sk)->bc_num_bis; i++)
 		if (sa->iso_bc->bc_bis[i] < 0x01 ||
 		    sa->iso_bc->bc_bis[i] > 0x1f)
 			return -EINVAL;
 
-		memcpy(iso_pi(sk)->bc_bis, sa->iso_bc->bc_bis,
-		       iso_pi(sk)->bc_num_bis);
-	}
+	memcpy(iso_pi(sk)->bc_bis, sa->iso_bc->bc_bis,
+	       iso_pi(sk)->bc_num_bis);
 
 	return 0;
 }
 
+static int iso_sock_bind_pa_sk(struct sock *sk, struct sockaddr_iso *sa,
+			       int addr_len)
+{
+	int err = 0;
+
+	if (sk->sk_type != SOCK_SEQPACKET) {
+		err = -EINVAL;
+		goto done;
+	}
+
+	if (addr_len != sizeof(*sa) + sizeof(*sa->iso_bc)) {
+		err = -EINVAL;
+		goto done;
+	}
+
+	if (sa->iso_bc->bc_num_bis > ISO_MAX_NUM_BIS) {
+		err = -EINVAL;
+		goto done;
+	}
+
+	iso_pi(sk)->bc_num_bis = sa->iso_bc->bc_num_bis;
+
+	for (int i = 0; i < iso_pi(sk)->bc_num_bis; i++)
+		if (sa->iso_bc->bc_bis[i] < 0x01 ||
+		    sa->iso_bc->bc_bis[i] > 0x1f) {
+			err = -EINVAL;
+			goto done;
+		}
+
+	memcpy(iso_pi(sk)->bc_bis, sa->iso_bc->bc_bis,
+	       iso_pi(sk)->bc_num_bis);
+
+done:
+	return err;
+}
+
 static int iso_sock_bind(struct socket *sock, struct sockaddr *addr,
 			 int addr_len)
 {
@@ -828,6 +947,15 @@ static int iso_sock_bind(struct socket *sock, struct sockaddr *addr,
 
 	lock_sock(sk);
 
+	/* Allow the user to bind a PA sync socket to a number
+	 * of BISes to sync to.
+	 */
+	if (sk->sk_state == BT_CONNECT2 &&
+	    test_bit(BT_SK_PA_SYNC, &iso_pi(sk)->flags)) {
+		err = iso_sock_bind_pa_sk(sk, sa, addr_len);
+		goto done;
+	}
+
 	if (sk->sk_state != BT_OPEN) {
 		err = -EBADFD;
 		goto done;
@@ -1694,6 +1822,7 @@ static void iso_conn_ready(struct iso_conn *conn)
 		parent->sk_data_ready(parent);
 
 		release_sock(parent);
+		sock_put(parent);
 	}
 }
 
@@ -1778,6 +1907,7 @@ int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 *flags)
 				if (err) {
 					bt_dev_err(hdev, "hci_le_big_create_sync: %d",
 						   err);
+					sock_put(sk);
 					sk = NULL;
 				}
 			}
@@ -1810,6 +1940,8 @@ int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 *flags)
 	if (test_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags))
 		*flags |= HCI_PROTO_DEFER;
 
+	sock_put(sk);
+
 	return lm;
 }
 
diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
index d3a759e052c8..6db2880e928a 100644
--- a/net/ceph/osd_client.c
+++ b/net/ceph/osd_client.c
@@ -5850,8 +5850,6 @@ static inline void convert_extent_map(struct ceph_sparse_read *sr)
 }
 #endif
 
-#define MAX_EXTENTS 4096
-
 static int osd_sparse_read(struct ceph_connection *con,
 			   struct ceph_msg_data_cursor *cursor,
 			   char **pbuf)
@@ -5882,23 +5880,16 @@ static int osd_sparse_read(struct ceph_connection *con,
 
 		if (count > 0) {
 			if (!sr->sr_extent || count > sr->sr_ext_len) {
-				/*
-				 * Apply a hard cap to the number of extents.
-				 * If we have more, assume something is wrong.
-				 */
-				if (count > MAX_EXTENTS) {
-					dout("%s: OSD returned 0x%x extents in a single reply!\n",
-					     __func__, count);
-					return -EREMOTEIO;
-				}
-
 				/* no extent array provided, or too short */
 				kfree(sr->sr_extent);
 				sr->sr_extent = kmalloc_array(count,
 							      sizeof(*sr->sr_extent),
 							      GFP_NOIO);
-				if (!sr->sr_extent)
+				if (!sr->sr_extent) {
+					pr_err("%s: failed to allocate %u extents\n",
+					       __func__, count);
 					return -ENOMEM;
+				}
 				sr->sr_ext_len = count;
 			}
 			ret = count * sizeof(*sr->sr_extent);
@@ -5912,10 +5903,21 @@ static int osd_sparse_read(struct ceph_connection *con,
 		convert_extent_map(sr);
 		ret = sizeof(sr->sr_datalen);
 		*pbuf = (char *)&sr->sr_datalen;
-		sr->sr_state = CEPH_SPARSE_READ_DATA;
+		sr->sr_state = CEPH_SPARSE_READ_DATA_PRE;
 		break;
+	case CEPH_SPARSE_READ_DATA_PRE:
+		/* Convert sr_datalen to host-endian */
+		sr->sr_datalen = le32_to_cpu((__force __le32)sr->sr_datalen);
+		sr->sr_state = CEPH_SPARSE_READ_DATA;
+		fallthrough;
 	case CEPH_SPARSE_READ_DATA:
 		if (sr->sr_index >= count) {
+			if (sr->sr_datalen && count) {
+				pr_err("%s: datalen mismatch, %u bytes left\n",
+				       __func__, sr->sr_datalen);
+				return -EREMOTEIO;
+			}
+
 			sr->sr_state = CEPH_SPARSE_READ_HDR;
 			goto next_op;
 		}
@@ -5932,6 +5934,8 @@ static int osd_sparse_read(struct ceph_connection *con,
 			return -EREMOTEIO;
 		}
 
+		sr->sr_datalen -= elen;
+
 		/* zero out anything from sr_pos to start of extent */
 		if (sr->sr_pos < eoff)
 			advance_cursor(cursor, eoff - sr->sr_pos, true);
diff --git a/net/core/dev.c b/net/core/dev.c
index af53f6d838ce..3950ced396b5 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -10051,6 +10051,54 @@ void netif_tx_stop_all_queues(struct net_device *dev)
 }
 EXPORT_SYMBOL(netif_tx_stop_all_queues);
 
+static int netdev_do_alloc_pcpu_stats(struct net_device *dev)
+{
+	void __percpu *v;
+
+	/* Drivers implementing ndo_get_peer_dev must support tstat
+	 * accounting, so that skb_do_redirect() can bump the dev's
+	 * RX stats upon network namespace switch.
+	 */
+	if (dev->netdev_ops->ndo_get_peer_dev &&
+	    dev->pcpu_stat_type != NETDEV_PCPU_STAT_TSTATS)
+		return -EOPNOTSUPP;
+
+	switch (dev->pcpu_stat_type) {
+	case NETDEV_PCPU_STAT_NONE:
+		return 0;
+	case NETDEV_PCPU_STAT_LSTATS:
+		v = dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats);
+		break;
+	case NETDEV_PCPU_STAT_TSTATS:
+		v = dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
+		break;
+	case NETDEV_PCPU_STAT_DSTATS:
+		v = dev->dstats = netdev_alloc_pcpu_stats(struct pcpu_dstats);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return v ? 0 : -ENOMEM;
+}
+
+static void netdev_do_free_pcpu_stats(struct net_device *dev)
+{
+	switch (dev->pcpu_stat_type) {
+	case NETDEV_PCPU_STAT_NONE:
+		return;
+	case NETDEV_PCPU_STAT_LSTATS:
+		free_percpu(dev->lstats);
+		break;
+	case NETDEV_PCPU_STAT_TSTATS:
+		free_percpu(dev->tstats);
+		break;
+	case NETDEV_PCPU_STAT_DSTATS:
+		free_percpu(dev->dstats);
+		break;
+	}
+}
+
 /**
  * register_netdevice() - register a network device
  * @dev: device to register
@@ -10111,9 +10159,13 @@ int register_netdevice(struct net_device *dev)
 		goto err_uninit;
 	}
 
+	ret = netdev_do_alloc_pcpu_stats(dev);
+	if (ret)
+		goto err_uninit;
+
 	ret = dev_index_reserve(net, dev->ifindex);
 	if (ret < 0)
-		goto err_uninit;
+		goto err_free_pcpu;
 	dev->ifindex = ret;
 
 	/* Transfer changeable features to wanted_features and enable
@@ -10219,6 +10271,8 @@ int register_netdevice(struct net_device *dev)
 	call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev);
 err_ifindex_release:
 	dev_index_release(net, dev->ifindex);
+err_free_pcpu:
+	netdev_do_free_pcpu_stats(dev);
 err_uninit:
 	if (dev->netdev_ops->ndo_uninit)
 		dev->netdev_ops->ndo_uninit(dev);
@@ -10471,6 +10525,7 @@ void netdev_run_todo(void)
 		WARN_ON(rcu_access_pointer(dev->ip_ptr));
 		WARN_ON(rcu_access_pointer(dev->ip6_ptr));
 
+		netdev_do_free_pcpu_stats(dev);
 		if (dev->priv_destructor)
 			dev->priv_destructor(dev);
 		if (dev->needs_free_netdev)
@@ -11181,17 +11236,19 @@ int __dev_change_net_namespace(struct net_device *dev, struct net *net,
 	dev_net_set(dev, net);
 	dev->ifindex = new_ifindex;
 
-	/* Send a netdev-add uevent to the new namespace */
-	kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
-	netdev_adjacent_add_links(dev);
-
 	if (new_name[0]) /* Rename the netdev to prepared name */
 		strscpy(dev->name, new_name, IFNAMSIZ);
 
 	/* Fixup kobjects */
+	dev_set_uevent_suppress(&dev->dev, 1);
 	err = device_rename(&dev->dev, dev->name);
+	dev_set_uevent_suppress(&dev->dev, 0);
 	WARN_ON(err);
 
+	/* Send a netdev-add uevent to the new namespace */
+	kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
+	netdev_adjacent_add_links(dev);
+
 	/* Adapt owner in case owning user namespace of target network
 	 * namespace is different from the original one.
 	 */
diff --git a/net/core/dev_ioctl.c b/net/core/dev_ioctl.c
index feeddf95f450..9a66cf5015f2 100644
--- a/net/core/dev_ioctl.c
+++ b/net/core/dev_ioctl.c
@@ -322,9 +322,9 @@ static int dev_get_hwtstamp(struct net_device *dev, struct ifreq *ifr)
  * frames and not forward them), it must set IFF_SEE_ALL_HWTSTAMP_REQUESTS in
  * dev->priv_flags.
  */
-static int dev_set_hwtstamp_phylib(struct net_device *dev,
-				   struct kernel_hwtstamp_config *cfg,
-				   struct netlink_ext_ack *extack)
+int dev_set_hwtstamp_phylib(struct net_device *dev,
+			    struct kernel_hwtstamp_config *cfg,
+			    struct netlink_ext_ack *extack)
 {
 	const struct net_device_ops *ops = dev->netdev_ops;
 	bool phy_ts = phy_has_hwtstamp(dev->phydev);
@@ -363,6 +363,7 @@ static int dev_set_hwtstamp_phylib(struct net_device *dev,
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(dev_set_hwtstamp_phylib);
 
 static int dev_set_hwtstamp(struct net_device *dev, struct ifreq *ifr)
 {
diff --git a/net/core/filter.c b/net/core/filter.c
index 383f96b0a1c7..7e4d7c3bcc84 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -81,6 +81,7 @@
 #include <net/xdp.h>
 #include <net/mptcp.h>
 #include <net/netfilter/nf_conntrack_bpf.h>
+#include <net/netkit.h>
 #include <linux/un.h>
 
 #include "dev.h"
@@ -2468,6 +2469,16 @@ static const struct bpf_func_proto bpf_clone_redirect_proto = {
 DEFINE_PER_CPU(struct bpf_redirect_info, bpf_redirect_info);
 EXPORT_PER_CPU_SYMBOL_GPL(bpf_redirect_info);
 
+static struct net_device *skb_get_peer_dev(struct net_device *dev)
+{
+	const struct net_device_ops *ops = dev->netdev_ops;
+
+	if (likely(ops->ndo_get_peer_dev))
+		return INDIRECT_CALL_1(ops->ndo_get_peer_dev,
+				       netkit_peer_dev, dev);
+	return NULL;
+}
+
 int skb_do_redirect(struct sk_buff *skb)
 {
 	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
@@ -2481,17 +2492,15 @@ int skb_do_redirect(struct sk_buff *skb)
 	if (unlikely(!dev))
 		goto out_drop;
 	if (flags & BPF_F_PEER) {
-		const struct net_device_ops *ops = dev->netdev_ops;
-
-		if (unlikely(!ops->ndo_get_peer_dev ||
-			     !skb_at_tc_ingress(skb)))
+		if (unlikely(!skb_at_tc_ingress(skb)))
 			goto out_drop;
-		dev = ops->ndo_get_peer_dev(dev);
+		dev = skb_get_peer_dev(dev);
 		if (unlikely(!dev ||
 			     !(dev->flags & IFF_UP) ||
 			     net_eq(net, dev_net(dev))))
 			goto out_drop;
 		skb->dev = dev;
+		dev_sw_netstats_rx_add(dev, skb->len);
 		return -EAGAIN;
 	}
 	return flags & BPF_F_NEIGH ?
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index e8431c6c8490..592164c2a540 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -3849,7 +3849,7 @@ static int rtnl_getlink(struct sk_buff *skb, struct nlmsghdr *nlh,
 		goto out;
 
 	err = -ENOBUFS;
-	nskb = nlmsg_new(if_nlmsg_size(dev, ext_filter_mask), GFP_KERNEL);
+	nskb = nlmsg_new_large(if_nlmsg_size(dev, ext_filter_mask));
 	if (nskb == NULL)
 		goto out;
 
diff --git a/net/devlink/core.c b/net/devlink/core.c
index 6984877e9f10..4275a2bc6d8e 100644
--- a/net/devlink/core.c
+++ b/net/devlink/core.c
@@ -503,14 +503,14 @@ static void __net_exit devlink_pernet_pre_exit(struct net *net)
 	 * all devlink instances from this namespace into init_net.
 	 */
 	devlinks_xa_for_each_registered_get(net, index, devlink) {
-		devl_lock(devlink);
+		devl_dev_lock(devlink, true);
 		err = 0;
 		if (devl_is_registered(devlink))
 			err = devlink_reload(devlink, &init_net,
 					     DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
 					     DEVLINK_RELOAD_LIMIT_UNSPEC,
 					     &actions_performed, NULL);
-		devl_unlock(devlink);
+		devl_dev_unlock(devlink, true);
 		devlink_put(devlink);
 		if (err && err != -EOPNOTSUPP)
 			pr_warn("Failed to reload devlink instance into init_net\n");
diff --git a/net/devlink/dev.c b/net/devlink/dev.c
index 4fc7adb32663..ea6a92f2e6a2 100644
--- a/net/devlink/dev.c
+++ b/net/devlink/dev.c
@@ -4,6 +4,7 @@
  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
  */
 
+#include <linux/device.h>
 #include <net/genetlink.h>
 #include <net/sock.h>
 #include "devl_internal.h"
@@ -433,6 +434,13 @@ int devlink_reload(struct devlink *devlink, struct net *dest_net,
 	struct net *curr_net;
 	int err;
 
+	/* Make sure the reload operations are invoked with the device lock
+	 * held to allow drivers to trigger functionality that expects it
+	 * (e.g., PCI reset) and to close possible races between these
+	 * operations and probe/remove.
+	 */
+	device_lock_assert(devlink->dev);
+
 	memcpy(remote_reload_stats, devlink->stats.remote_reload_stats,
 	       sizeof(remote_reload_stats));
 
diff --git a/net/devlink/devl_internal.h b/net/devlink/devl_internal.h
index 183dbe3807ab..5ea2e2012e93 100644
--- a/net/devlink/devl_internal.h
+++ b/net/devlink/devl_internal.h
@@ -3,6 +3,7 @@
  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
  */
 
+#include <linux/device.h>
 #include <linux/etherdevice.h>
 #include <linux/mutex.h>
 #include <linux/netdevice.h>
@@ -96,6 +97,20 @@ static inline bool devl_is_registered(struct devlink *devlink)
 	return xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
 }
 
+static inline void devl_dev_lock(struct devlink *devlink, bool dev_lock)
+{
+	if (dev_lock)
+		device_lock(devlink->dev);
+	devl_lock(devlink);
+}
+
+static inline void devl_dev_unlock(struct devlink *devlink, bool dev_lock)
+{
+	devl_unlock(devlink);
+	if (dev_lock)
+		device_unlock(devlink->dev);
+}
+
 typedef void devlink_rel_notify_cb_t(struct devlink *devlink, u32 obj_index);
 typedef void devlink_rel_cleanup_cb_t(struct devlink *devlink, u32 obj_index,
 				      u32 rel_index);
@@ -111,9 +126,6 @@ int devlink_rel_devlink_handle_put(struct sk_buff *msg, struct devlink *devlink,
 				   bool *msg_updated);
 
 /* Netlink */
-#define DEVLINK_NL_FLAG_NEED_PORT		BIT(0)
-#define DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT	BIT(1)
-
 enum devlink_multicast_groups {
 	DEVLINK_MCGRP_CONFIG,
 };
@@ -140,7 +152,8 @@ typedef int devlink_nl_dump_one_func_t(struct sk_buff *msg,
 				       int flags);
 
 struct devlink *
-devlink_get_from_attrs_lock(struct net *net, struct nlattr **attrs);
+devlink_get_from_attrs_lock(struct net *net, struct nlattr **attrs,
+			    bool dev_lock);
 
 int devlink_nl_dumpit(struct sk_buff *msg, struct netlink_callback *cb,
 		      devlink_nl_dump_one_func_t *dump_one);
diff --git a/net/devlink/health.c b/net/devlink/health.c
index 695df61f8ac2..71ae121dc739 100644
--- a/net/devlink/health.c
+++ b/net/devlink/health.c
@@ -1151,7 +1151,8 @@ devlink_health_reporter_get_from_cb_lock(struct netlink_callback *cb)
 	struct nlattr **attrs = info->attrs;
 	struct devlink *devlink;
 
-	devlink = devlink_get_from_attrs_lock(sock_net(cb->skb->sk), attrs);
+	devlink = devlink_get_from_attrs_lock(sock_net(cb->skb->sk), attrs,
+					      false);
 	if (IS_ERR(devlink))
 		return NULL;
 
diff --git a/net/devlink/netlink.c b/net/devlink/netlink.c
index d0b90ebc8b15..fa9afe3e6d9b 100644
--- a/net/devlink/netlink.c
+++ b/net/devlink/netlink.c
@@ -9,6 +9,10 @@
 
 #include "devl_internal.h"
 
+#define DEVLINK_NL_FLAG_NEED_PORT		BIT(0)
+#define DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT	BIT(1)
+#define DEVLINK_NL_FLAG_NEED_DEV_LOCK		BIT(2)
+
 static const struct genl_multicast_group devlink_nl_mcgrps[] = {
 	[DEVLINK_MCGRP_CONFIG] = { .name = DEVLINK_GENL_MCGRP_CONFIG_NAME },
 };
@@ -61,7 +65,8 @@ int devlink_nl_msg_reply_and_new(struct sk_buff **msg, struct genl_info *info)
 }
 
 struct devlink *
-devlink_get_from_attrs_lock(struct net *net, struct nlattr **attrs)
+devlink_get_from_attrs_lock(struct net *net, struct nlattr **attrs,
+			    bool dev_lock)
 {
 	struct devlink *devlink;
 	unsigned long index;
@@ -75,12 +80,12 @@ devlink_get_from_attrs_lock(struct net *net, struct nlattr **attrs)
 	devname = nla_data(attrs[DEVLINK_ATTR_DEV_NAME]);
 
 	devlinks_xa_for_each_registered_get(net, index, devlink) {
-		devl_lock(devlink);
+		devl_dev_lock(devlink, dev_lock);
 		if (devl_is_registered(devlink) &&
 		    strcmp(devlink->dev->bus->name, busname) == 0 &&
 		    strcmp(dev_name(devlink->dev), devname) == 0)
 			return devlink;
-		devl_unlock(devlink);
+		devl_dev_unlock(devlink, dev_lock);
 		devlink_put(devlink);
 	}
 
@@ -90,11 +95,13 @@ devlink_get_from_attrs_lock(struct net *net, struct nlattr **attrs)
 static int __devlink_nl_pre_doit(struct sk_buff *skb, struct genl_info *info,
 				 u8 flags)
 {
+	bool dev_lock = flags & DEVLINK_NL_FLAG_NEED_DEV_LOCK;
 	struct devlink_port *devlink_port;
 	struct devlink *devlink;
 	int err;
 
-	devlink = devlink_get_from_attrs_lock(genl_info_net(info), info->attrs);
+	devlink = devlink_get_from_attrs_lock(genl_info_net(info), info->attrs,
+					      dev_lock);
 	if (IS_ERR(devlink))
 		return PTR_ERR(devlink);
 
@@ -114,7 +121,7 @@ static int __devlink_nl_pre_doit(struct sk_buff *skb, struct genl_info *info,
 	return 0;
 
 unlock:
-	devl_unlock(devlink);
+	devl_dev_unlock(devlink, dev_lock);
 	devlink_put(devlink);
 	return err;
 }
@@ -131,6 +138,12 @@ int devlink_nl_pre_doit_port(const struct genl_split_ops *ops,
 	return __devlink_nl_pre_doit(skb, info, DEVLINK_NL_FLAG_NEED_PORT);
 }
 
+int devlink_nl_pre_doit_dev_lock(const struct genl_split_ops *ops,
+				 struct sk_buff *skb, struct genl_info *info)
+{
+	return __devlink_nl_pre_doit(skb, info, DEVLINK_NL_FLAG_NEED_DEV_LOCK);
+}
+
 int devlink_nl_pre_doit_port_optional(const struct genl_split_ops *ops,
 				      struct sk_buff *skb,
 				      struct genl_info *info)
@@ -138,16 +151,30 @@ int devlink_nl_pre_doit_port_optional(const struct genl_split_ops *ops,
 	return __devlink_nl_pre_doit(skb, info, DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT);
 }
 
-void devlink_nl_post_doit(const struct genl_split_ops *ops,
-			  struct sk_buff *skb, struct genl_info *info)
+static void __devlink_nl_post_doit(struct sk_buff *skb, struct genl_info *info,
+				   u8 flags)
 {
+	bool dev_lock = flags & DEVLINK_NL_FLAG_NEED_DEV_LOCK;
 	struct devlink *devlink;
 
 	devlink = info->user_ptr[0];
-	devl_unlock(devlink);
+	devl_dev_unlock(devlink, dev_lock);
 	devlink_put(devlink);
 }
 
+void devlink_nl_post_doit(const struct genl_split_ops *ops,
+			  struct sk_buff *skb, struct genl_info *info)
+{
+	__devlink_nl_post_doit(skb, info, 0);
+}
+
+void
+devlink_nl_post_doit_dev_lock(const struct genl_split_ops *ops,
+			      struct sk_buff *skb, struct genl_info *info)
+{
+	__devlink_nl_post_doit(skb, info, DEVLINK_NL_FLAG_NEED_DEV_LOCK);
+}
+
 static int devlink_nl_inst_single_dumpit(struct sk_buff *msg,
 					 struct netlink_callback *cb, int flags,
 					 devlink_nl_dump_one_func_t *dump_one,
@@ -156,7 +183,7 @@ static int devlink_nl_inst_single_dumpit(struct sk_buff *msg,
 	struct devlink *devlink;
 	int err;
 
-	devlink = devlink_get_from_attrs_lock(sock_net(msg->sk), attrs);
+	devlink = devlink_get_from_attrs_lock(sock_net(msg->sk), attrs, false);
 	if (IS_ERR(devlink))
 		return PTR_ERR(devlink);
 	err = dump_one(msg, devlink, cb, flags | NLM_F_DUMP_FILTERED);
diff --git a/net/devlink/netlink_gen.c b/net/devlink/netlink_gen.c
index 788dfdc498a9..95f9b4350ab7 100644
--- a/net/devlink/netlink_gen.c
+++ b/net/devlink/netlink_gen.c
@@ -846,9 +846,9 @@ const struct genl_split_ops devlink_nl_ops[73] = {
 	{
 		.cmd		= DEVLINK_CMD_RELOAD,
 		.validate	= GENL_DONT_VALIDATE_STRICT,
-		.pre_doit	= devlink_nl_pre_doit,
+		.pre_doit	= devlink_nl_pre_doit_dev_lock,
 		.doit		= devlink_nl_reload_doit,
-		.post_doit	= devlink_nl_post_doit,
+		.post_doit	= devlink_nl_post_doit_dev_lock,
 		.policy		= devlink_reload_nl_policy,
 		.maxattr	= DEVLINK_ATTR_RELOAD_LIMITS,
 		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
diff --git a/net/devlink/netlink_gen.h b/net/devlink/netlink_gen.h
index 0e9e89c31c31..02f3c0bfae0e 100644
--- a/net/devlink/netlink_gen.h
+++ b/net/devlink/netlink_gen.h
@@ -22,12 +22,17 @@ int devlink_nl_pre_doit(const struct genl_split_ops *ops, struct sk_buff *skb,
 			struct genl_info *info);
 int devlink_nl_pre_doit_port(const struct genl_split_ops *ops,
 			     struct sk_buff *skb, struct genl_info *info);
+int devlink_nl_pre_doit_dev_lock(const struct genl_split_ops *ops,
+				 struct sk_buff *skb, struct genl_info *info);
 int devlink_nl_pre_doit_port_optional(const struct genl_split_ops *ops,
 				      struct sk_buff *skb,
 				      struct genl_info *info);
 void
 devlink_nl_post_doit(const struct genl_split_ops *ops, struct sk_buff *skb,
 		     struct genl_info *info);
+void
+devlink_nl_post_doit_dev_lock(const struct genl_split_ops *ops,
+			      struct sk_buff *skb, struct genl_info *info);
 
 int devlink_nl_get_doit(struct sk_buff *skb, struct genl_info *info);
 int devlink_nl_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb);
diff --git a/net/devlink/region.c b/net/devlink/region.c
index 0aab7b82d678..e3bab458db94 100644
--- a/net/devlink/region.c
+++ b/net/devlink/region.c
@@ -883,7 +883,8 @@ int devlink_nl_region_read_dumpit(struct sk_buff *skb,
 
 	start_offset = state->start_offset;
 
-	devlink = devlink_get_from_attrs_lock(sock_net(cb->skb->sk), attrs);
+	devlink = devlink_get_from_attrs_lock(sock_net(cb->skb->sk), attrs,
+					      false);
 	if (IS_ERR(devlink))
 		return PTR_ERR(devlink);
 
diff --git a/net/dsa/tag_rtl4_a.c b/net/dsa/tag_rtl4_a.c
index 4da5bad1a7aa..a019226ec6d2 100644
--- a/net/dsa/tag_rtl4_a.c
+++ b/net/dsa/tag_rtl4_a.c
@@ -23,7 +23,6 @@
 #define RTL4_A_NAME		"rtl4a"
 
 #define RTL4_A_HDR_LEN		4
-#define RTL4_A_ETHERTYPE	0x8899
 #define RTL4_A_PROTOCOL_SHIFT	12
 /*
  * 0x1 = Realtek Remote Control protocol (RRCP)
@@ -54,7 +53,7 @@ static struct sk_buff *rtl4a_tag_xmit(struct sk_buff *skb,
 
 	/* Set Ethertype */
 	p = (__be16 *)tag;
-	*p = htons(RTL4_A_ETHERTYPE);
+	*p = htons(ETH_P_REALTEK);
 
 	out = (RTL4_A_PROTOCOL_RTL8366RB << RTL4_A_PROTOCOL_SHIFT);
 	/* The lower bits indicate the port number */
@@ -82,7 +81,7 @@ static struct sk_buff *rtl4a_tag_rcv(struct sk_buff *skb,
 	tag = dsa_etype_header_pos_rx(skb);
 	p = (__be16 *)tag;
 	etype = ntohs(*p);
-	if (etype != RTL4_A_ETHERTYPE) {
+	if (etype != ETH_P_REALTEK) {
 		/* Not custom, just pass through */
 		netdev_dbg(dev, "non-realtek ethertype 0x%04x\n", etype);
 		return skb;
diff --git a/net/ethtool/common.c b/net/ethtool/common.c
index b4419fb6df6a..11d8797f63f6 100644
--- a/net/ethtool/common.c
+++ b/net/ethtool/common.c
@@ -661,6 +661,12 @@ int ethtool_get_phc_vclocks(struct net_device *dev, int **vclock_index)
 }
 EXPORT_SYMBOL(ethtool_get_phc_vclocks);
 
+int ethtool_get_ts_info_by_layer(struct net_device *dev, struct ethtool_ts_info *info)
+{
+	return __ethtool_get_ts_info(dev, info);
+}
+EXPORT_SYMBOL(ethtool_get_ts_info_by_layer);
+
 const struct ethtool_phy_ops *ethtool_phy_ops;
 
 void ethtool_set_ethtool_phy_ops(const struct ethtool_phy_ops *ops)
diff --git a/net/ipv4/inet_diag.c b/net/ipv4/inet_diag.c
index f01aee832aab..7d0e7aaa71e0 100644
--- a/net/ipv4/inet_diag.c
+++ b/net/ipv4/inet_diag.c
@@ -1481,5 +1481,6 @@ static void __exit inet_diag_exit(void)
 module_init(inet_diag_init);
 module_exit(inet_diag_exit);
 MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("INET/INET6: socket monitoring via SOCK_DIAG");
 MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG, 2 /* AF_INET */);
 MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG, 10 /* AF_INET6 */);
diff --git a/net/ipv4/raw_diag.c b/net/ipv4/raw_diag.c
index 63a40e4b678f..fe2140c8375c 100644
--- a/net/ipv4/raw_diag.c
+++ b/net/ipv4/raw_diag.c
@@ -257,5 +257,6 @@ static void __exit raw_diag_exit(void)
 module_init(raw_diag_init);
 module_exit(raw_diag_exit);
 MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("RAW socket monitoring via SOCK_DIAG");
 MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG, 2-255 /* AF_INET - IPPROTO_RAW */);
 MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG, 10-255 /* AF_INET6 - IPPROTO_RAW */);
diff --git a/net/ipv4/route.c b/net/ipv4/route.c
index 3290a4442b4a..16615d107cf0 100644
--- a/net/ipv4/route.c
+++ b/net/ipv4/route.c
@@ -780,7 +780,7 @@ static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flow
 			goto reject_redirect;
 	}
 
-	n = __ipv4_neigh_lookup(rt->dst.dev, new_gw);
+	n = __ipv4_neigh_lookup(rt->dst.dev, (__force u32)new_gw);
 	if (!n)
 		n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev);
 	if (!IS_ERR(n)) {
diff --git a/net/ipv4/tcp_diag.c b/net/ipv4/tcp_diag.c
index 01b50fa79189..4cbe4b44425a 100644
--- a/net/ipv4/tcp_diag.c
+++ b/net/ipv4/tcp_diag.c
@@ -247,4 +247,5 @@ static void __exit tcp_diag_exit(void)
 module_init(tcp_diag_init);
 module_exit(tcp_diag_exit);
 MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("TCP socket monitoring via SOCK_DIAG");
 MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG, 2-6 /* AF_INET - IPPROTO_TCP */);
diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 5f693bbd578d..86cc6d36f818 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -482,6 +482,7 @@ int tcp_v4_err(struct sk_buff *skb, u32 info)
 	const int code = icmp_hdr(skb)->code;
 	struct sock *sk;
 	struct request_sock *fastopen;
+	bool harderr = false;
 	u32 seq, snd_una;
 	int err;
 	struct net *net = dev_net(skb->dev);
@@ -555,6 +556,7 @@ int tcp_v4_err(struct sk_buff *skb, u32 info)
 		goto out;
 	case ICMP_PARAMETERPROB:
 		err = EPROTO;
+		harderr = true;
 		break;
 	case ICMP_DEST_UNREACH:
 		if (code > NR_ICMP_UNREACH)
@@ -579,6 +581,7 @@ int tcp_v4_err(struct sk_buff *skb, u32 info)
 		}
 
 		err = icmp_err_convert[code].errno;
+		harderr = icmp_err_convert[code].fatal;
 		/* check if this ICMP message allows revert of backoff.
 		 * (see RFC 6069)
 		 */
@@ -604,6 +607,9 @@ int tcp_v4_err(struct sk_buff *skb, u32 info)
 
 		ip_icmp_error(sk, skb, err, th->dest, info, (u8 *)th);
 
+		if (!harderr)
+			break;
+
 		if (!sock_owned_by_user(sk)) {
 			WRITE_ONCE(sk->sk_err, err);
 
diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c
index 1f9f6c1c196b..d1ad20ce1c8c 100644
--- a/net/ipv4/tcp_timer.c
+++ b/net/ipv4/tcp_timer.c
@@ -626,7 +626,6 @@ void tcp_retransmit_timer(struct sock *sk)
 	 * implemented ftp to mars will work nicely. We will have to fix
 	 * the 120 second clamps though!
 	 */
-	icsk->icsk_backoff++;
 
 out_reset_timer:
 	/* If stream is thin, use linear timeouts. Since 'icsk_backoff' is
@@ -647,11 +646,12 @@ void tcp_retransmit_timer(struct sock *sk)
 				       tcp_rto_min(sk),
 				       TCP_RTO_MAX);
 	} else if (sk->sk_state != TCP_SYN_SENT ||
-		   icsk->icsk_backoff >
+		   tp->total_rto >
 		   READ_ONCE(net->ipv4.sysctl_tcp_syn_linear_timeouts)) {
 		/* Use normal (exponential) backoff unless linear timeouts are
 		 * activated.
 		 */
+		icsk->icsk_backoff++;
 		icsk->icsk_rto = min(icsk->icsk_rto << 1, TCP_RTO_MAX);
 	}
 	inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
diff --git a/net/ipv4/udp_diag.c b/net/ipv4/udp_diag.c
index de3f2d31f510..dc41a22ee80e 100644
--- a/net/ipv4/udp_diag.c
+++ b/net/ipv4/udp_diag.c
@@ -296,5 +296,6 @@ static void __exit udp_diag_exit(void)
 module_init(udp_diag_init);
 module_exit(udp_diag_exit);
 MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("UDP socket monitoring via SOCK_DIAG");
 MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG, 2-17 /* AF_INET - IPPROTO_UDP */);
 MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG, 2-136 /* AF_INET - IPPROTO_UDPLITE */);
diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
index 937a02c2e534..43deda49cc52 100644
--- a/net/ipv6/tcp_ipv6.c
+++ b/net/ipv6/tcp_ipv6.c
@@ -381,7 +381,7 @@ static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
 	struct tcp_sock *tp;
 	__u32 seq, snd_una;
 	struct sock *sk;
-	bool fatal;
+	bool harderr;
 	int err;
 
 	sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
@@ -402,9 +402,9 @@ static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
 		return 0;
 	}
 	seq = ntohl(th->seq);
-	fatal = icmpv6_err_convert(type, code, &err);
+	harderr = icmpv6_err_convert(type, code, &err);
 	if (sk->sk_state == TCP_NEW_SYN_RECV) {
-		tcp_req_err(sk, seq, fatal);
+		tcp_req_err(sk, seq, harderr);
 		return 0;
 	}
 
@@ -489,6 +489,9 @@ static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
 
 		ipv6_icmp_error(sk, skb, err, th->dest, ntohl(info), (u8 *)th);
 
+		if (!harderr)
+			break;
+
 		if (!sock_owned_by_user(sk)) {
 			WRITE_ONCE(sk->sk_err, err);
 			sk_error_report(sk);		/* Wake people up to see the error (see connect in sock.c) */
diff --git a/net/mptcp/mptcp_diag.c b/net/mptcp/mptcp_diag.c
index 8df1bdb647e2..5409c2ea3f57 100644
--- a/net/mptcp/mptcp_diag.c
+++ b/net/mptcp/mptcp_diag.c
@@ -245,4 +245,5 @@ static void __exit mptcp_diag_exit(void)
 module_init(mptcp_diag_init);
 module_exit(mptcp_diag_exit);
 MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("MPTCP socket monitoring via SOCK_DIAG");
 MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG, 2-262 /* AF_INET - IPPROTO_MPTCP */);
diff --git a/net/ncsi/internal.h b/net/ncsi/internal.h
index 03757e76bb6b..374412ed780b 100644
--- a/net/ncsi/internal.h
+++ b/net/ncsi/internal.h
@@ -105,8 +105,11 @@ enum {
 
 
 struct ncsi_channel_version {
-	u32 version;		/* Supported BCD encoded NCSI version */
-	u32 alpha2;		/* Supported BCD encoded NCSI version */
+	u8   major;		/* NCSI version major */
+	u8   minor;		/* NCSI version minor */
+	u8   update;		/* NCSI version update */
+	char alpha1;		/* NCSI version alpha1 */
+	char alpha2;		/* NCSI version alpha2 */
 	u8  fw_name[12];	/* Firmware name string                */
 	u32 fw_version;		/* Firmware version                   */
 	u16 pci_ids[4];		/* PCI identification                 */
diff --git a/net/ncsi/ncsi-cmd.c b/net/ncsi/ncsi-cmd.c
index fd2236ee9a79..b3ff37a181d7 100644
--- a/net/ncsi/ncsi-cmd.c
+++ b/net/ncsi/ncsi-cmd.c
@@ -270,7 +270,8 @@ static struct ncsi_cmd_handler {
 	{ NCSI_PKT_CMD_GPS,    0, ncsi_cmd_handler_default },
 	{ NCSI_PKT_CMD_OEM,   -1, ncsi_cmd_handler_oem     },
 	{ NCSI_PKT_CMD_PLDM,   0, NULL                     },
-	{ NCSI_PKT_CMD_GPUUID, 0, ncsi_cmd_handler_default }
+	{ NCSI_PKT_CMD_GPUUID, 0, ncsi_cmd_handler_default },
+	{ NCSI_PKT_CMD_GMCMA,  0, ncsi_cmd_handler_default }
 };
 
 static struct ncsi_request *ncsi_alloc_command(struct ncsi_cmd_arg *nca)
diff --git a/net/ncsi/ncsi-manage.c b/net/ncsi/ncsi-manage.c
index d9da942ad53d..745c788f1d1d 100644
--- a/net/ncsi/ncsi-manage.c
+++ b/net/ncsi/ncsi-manage.c
@@ -689,8 +689,6 @@ static int set_one_vid(struct ncsi_dev_priv *ndp, struct ncsi_channel *nc,
 	return 0;
 }
 
-#if IS_ENABLED(CONFIG_NCSI_OEM_CMD_KEEP_PHY)
-
 static int ncsi_oem_keep_phy_intel(struct ncsi_cmd_arg *nca)
 {
 	unsigned char data[NCSI_OEM_INTEL_CMD_KEEP_PHY_LEN];
@@ -716,10 +714,6 @@ static int ncsi_oem_keep_phy_intel(struct ncsi_cmd_arg *nca)
 	return ret;
 }
 
-#endif
-
-#if IS_ENABLED(CONFIG_NCSI_OEM_CMD_GET_MAC)
-
 /* NCSI OEM Command APIs */
 static int ncsi_oem_gma_handler_bcm(struct ncsi_cmd_arg *nca)
 {
@@ -856,8 +850,6 @@ static int ncsi_gma_handler(struct ncsi_cmd_arg *nca, unsigned int mf_id)
 	return nch->handler(nca);
 }
 
-#endif /* CONFIG_NCSI_OEM_CMD_GET_MAC */
-
 /* Determine if a given channel from the channel_queue should be used for Tx */
 static bool ncsi_channel_is_tx(struct ncsi_dev_priv *ndp,
 			       struct ncsi_channel *nc)
@@ -1039,20 +1031,23 @@ static void ncsi_configure_channel(struct ncsi_dev_priv *ndp)
 			goto error;
 		}
 
-		nd->state = ncsi_dev_state_config_oem_gma;
+		nd->state = IS_ENABLED(CONFIG_NCSI_OEM_CMD_GET_MAC)
+			  ? ncsi_dev_state_config_oem_gma
+			  : ncsi_dev_state_config_clear_vids;
 		break;
 	case ncsi_dev_state_config_oem_gma:
 		nd->state = ncsi_dev_state_config_clear_vids;
-		ret = -1;
 
-#if IS_ENABLED(CONFIG_NCSI_OEM_CMD_GET_MAC)
-		nca.type = NCSI_PKT_CMD_OEM;
 		nca.package = np->id;
 		nca.channel = nc->id;
 		ndp->pending_req_num = 1;
-		ret = ncsi_gma_handler(&nca, nc->version.mf_id);
-#endif /* CONFIG_NCSI_OEM_CMD_GET_MAC */
-
+		if (nc->version.major >= 1 && nc->version.minor >= 2) {
+			nca.type = NCSI_PKT_CMD_GMCMA;
+			ret = ncsi_xmit_cmd(&nca);
+		} else {
+			nca.type = NCSI_PKT_CMD_OEM;
+			ret = ncsi_gma_handler(&nca, nc->version.mf_id);
+		}
 		if (ret < 0)
 			schedule_work(&ndp->work);
 
@@ -1404,7 +1399,6 @@ static void ncsi_probe_channel(struct ncsi_dev_priv *ndp)
 
 		schedule_work(&ndp->work);
 		break;
-#if IS_ENABLED(CONFIG_NCSI_OEM_CMD_GET_MAC)
 	case ncsi_dev_state_probe_mlx_gma:
 		ndp->pending_req_num = 1;
 
@@ -1429,7 +1423,6 @@ static void ncsi_probe_channel(struct ncsi_dev_priv *ndp)
 
 		nd->state = ncsi_dev_state_probe_cis;
 		break;
-#endif /* CONFIG_NCSI_OEM_CMD_GET_MAC */
 	case ncsi_dev_state_probe_cis:
 		ndp->pending_req_num = NCSI_RESERVED_CHANNEL;
 
@@ -1447,7 +1440,6 @@ static void ncsi_probe_channel(struct ncsi_dev_priv *ndp)
 		if (IS_ENABLED(CONFIG_NCSI_OEM_CMD_KEEP_PHY))
 			nd->state = ncsi_dev_state_probe_keep_phy;
 		break;
-#if IS_ENABLED(CONFIG_NCSI_OEM_CMD_KEEP_PHY)
 	case ncsi_dev_state_probe_keep_phy:
 		ndp->pending_req_num = 1;
 
@@ -1460,7 +1452,6 @@ static void ncsi_probe_channel(struct ncsi_dev_priv *ndp)
 
 		nd->state = ncsi_dev_state_probe_gvi;
 		break;
-#endif /* CONFIG_NCSI_OEM_CMD_KEEP_PHY */
 	case ncsi_dev_state_probe_gvi:
 	case ncsi_dev_state_probe_gc:
 	case ncsi_dev_state_probe_gls:
diff --git a/net/ncsi/ncsi-netlink.c b/net/ncsi/ncsi-netlink.c
index a3a6753a1db7..2f872d064396 100644
--- a/net/ncsi/ncsi-netlink.c
+++ b/net/ncsi/ncsi-netlink.c
@@ -71,8 +71,8 @@ static int ncsi_write_channel_info(struct sk_buff *skb,
 	if (nc == nc->package->preferred_channel)
 		nla_put_flag(skb, NCSI_CHANNEL_ATTR_FORCED);
 
-	nla_put_u32(skb, NCSI_CHANNEL_ATTR_VERSION_MAJOR, nc->version.version);
-	nla_put_u32(skb, NCSI_CHANNEL_ATTR_VERSION_MINOR, nc->version.alpha2);
+	nla_put_u32(skb, NCSI_CHANNEL_ATTR_VERSION_MAJOR, nc->version.major);
+	nla_put_u32(skb, NCSI_CHANNEL_ATTR_VERSION_MINOR, nc->version.minor);
 	nla_put_string(skb, NCSI_CHANNEL_ATTR_VERSION_STR, nc->version.fw_name);
 
 	vid_nest = nla_nest_start_noflag(skb, NCSI_CHANNEL_ATTR_VLAN_LIST);
diff --git a/net/ncsi/ncsi-pkt.h b/net/ncsi/ncsi-pkt.h
index ba66c7dc3a21..f2f3b5c1b941 100644
--- a/net/ncsi/ncsi-pkt.h
+++ b/net/ncsi/ncsi-pkt.h
@@ -197,9 +197,12 @@ struct ncsi_rsp_gls_pkt {
 /* Get Version ID */
 struct ncsi_rsp_gvi_pkt {
 	struct ncsi_rsp_pkt_hdr rsp;          /* Response header */
-	__be32                  ncsi_version; /* NCSI version    */
+	unsigned char           major;        /* NCSI version major */
+	unsigned char           minor;        /* NCSI version minor */
+	unsigned char           update;       /* NCSI version update */
+	unsigned char           alpha1;       /* NCSI version alpha1 */
 	unsigned char           reserved[3];  /* Reserved        */
-	unsigned char           alpha2;       /* NCSI version    */
+	unsigned char           alpha2;       /* NCSI version alpha2 */
 	unsigned char           fw_name[12];  /* f/w name string */
 	__be32                  fw_version;   /* f/w version     */
 	__be16                  pci_ids[4];   /* PCI IDs         */
@@ -335,6 +338,14 @@ struct ncsi_rsp_gpuuid_pkt {
 	__be32                  checksum;
 };
 
+/* Get MC MAC Address */
+struct ncsi_rsp_gmcma_pkt {
+	struct ncsi_rsp_pkt_hdr rsp;
+	unsigned char           address_count;
+	unsigned char           reserved[3];
+	unsigned char           addresses[][ETH_ALEN];
+};
+
 /* AEN: Link State Change */
 struct ncsi_aen_lsc_pkt {
 	struct ncsi_aen_pkt_hdr aen;        /* AEN header      */
@@ -395,6 +406,7 @@ struct ncsi_aen_hncdsc_pkt {
 #define NCSI_PKT_CMD_GPUUID	0x52 /* Get package UUID                 */
 #define NCSI_PKT_CMD_QPNPR	0x56 /* Query Pending NC PLDM request */
 #define NCSI_PKT_CMD_SNPR	0x57 /* Send NC PLDM Reply  */
+#define NCSI_PKT_CMD_GMCMA	0x58 /* Get MC MAC Address */
 
 
 /* NCSI packet responses */
@@ -430,6 +442,7 @@ struct ncsi_aen_hncdsc_pkt {
 #define NCSI_PKT_RSP_GPUUID	(NCSI_PKT_CMD_GPUUID + 0x80)
 #define NCSI_PKT_RSP_QPNPR	(NCSI_PKT_CMD_QPNPR   + 0x80)
 #define NCSI_PKT_RSP_SNPR	(NCSI_PKT_CMD_SNPR   + 0x80)
+#define NCSI_PKT_RSP_GMCMA	(NCSI_PKT_CMD_GMCMA  + 0x80)
 
 /* NCSI response code/reason */
 #define NCSI_PKT_RSP_C_COMPLETED	0x0000 /* Command Completed        */
diff --git a/net/ncsi/ncsi-rsp.c b/net/ncsi/ncsi-rsp.c
index 069c2659074b..bee290d0f48b 100644
--- a/net/ncsi/ncsi-rsp.c
+++ b/net/ncsi/ncsi-rsp.c
@@ -19,6 +19,19 @@
 #include "ncsi-pkt.h"
 #include "ncsi-netlink.h"
 
+/* Nibbles within [0xA, 0xF] add zero "0" to the returned value.
+ * Optional fields (encoded as 0xFF) will default to zero.
+ */
+static u8 decode_bcd_u8(u8 x)
+{
+	int lo = x & 0xF;
+	int hi = x >> 4;
+
+	lo = lo < 0xA ? lo : 0;
+	hi = hi < 0xA ? hi : 0;
+	return lo + hi * 10;
+}
+
 static int ncsi_validate_rsp_pkt(struct ncsi_request *nr,
 				 unsigned short payload)
 {
@@ -755,9 +768,18 @@ static int ncsi_rsp_handler_gvi(struct ncsi_request *nr)
 	if (!nc)
 		return -ENODEV;
 
-	/* Update to channel's version info */
+	/* Update channel's version info
+	 *
+	 * Major, minor, and update fields are supposed to be
+	 * unsigned integers encoded as packed BCD.
+	 *
+	 * Alpha1 and alpha2 are ISO/IEC 8859-1 characters.
+	 */
 	ncv = &nc->version;
-	ncv->version = ntohl(rsp->ncsi_version);
+	ncv->major = decode_bcd_u8(rsp->major);
+	ncv->minor = decode_bcd_u8(rsp->minor);
+	ncv->update = decode_bcd_u8(rsp->update);
+	ncv->alpha1 = rsp->alpha1;
 	ncv->alpha2 = rsp->alpha2;
 	memcpy(ncv->fw_name, rsp->fw_name, 12);
 	ncv->fw_version = ntohl(rsp->fw_version);
@@ -1069,6 +1091,44 @@ static int ncsi_rsp_handler_netlink(struct ncsi_request *nr)
 	return ret;
 }
 
+static int ncsi_rsp_handler_gmcma(struct ncsi_request *nr)
+{
+	struct ncsi_dev_priv *ndp = nr->ndp;
+	struct net_device *ndev = ndp->ndev.dev;
+	struct ncsi_rsp_gmcma_pkt *rsp;
+	struct sockaddr saddr;
+	int ret = -1;
+	int i;
+
+	rsp = (struct ncsi_rsp_gmcma_pkt *)skb_network_header(nr->rsp);
+	saddr.sa_family = ndev->type;
+	ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
+
+	netdev_info(ndev, "NCSI: Received %d provisioned MAC addresses\n",
+		    rsp->address_count);
+	for (i = 0; i < rsp->address_count; i++) {
+		netdev_info(ndev, "NCSI: MAC address %d: %02x:%02x:%02x:%02x:%02x:%02x\n",
+			    i, rsp->addresses[i][0], rsp->addresses[i][1],
+			    rsp->addresses[i][2], rsp->addresses[i][3],
+			    rsp->addresses[i][4], rsp->addresses[i][5]);
+	}
+
+	for (i = 0; i < rsp->address_count; i++) {
+		memcpy(saddr.sa_data, &rsp->addresses[i], ETH_ALEN);
+		ret = ndev->netdev_ops->ndo_set_mac_address(ndev, &saddr);
+		if (ret < 0) {
+			netdev_warn(ndev, "NCSI: Unable to assign %pM to device\n",
+				    saddr.sa_data);
+			continue;
+		}
+		netdev_warn(ndev, "NCSI: Set MAC address to %pM\n", saddr.sa_data);
+		break;
+	}
+
+	ndp->gma_flag = ret == 0;
+	return ret;
+}
+
 static struct ncsi_rsp_handler {
 	unsigned char	type;
 	int             payload;
@@ -1105,7 +1165,8 @@ static struct ncsi_rsp_handler {
 	{ NCSI_PKT_RSP_PLDM,   -1, ncsi_rsp_handler_pldm    },
 	{ NCSI_PKT_RSP_GPUUID, 20, ncsi_rsp_handler_gpuuid  },
 	{ NCSI_PKT_RSP_QPNPR,  -1, ncsi_rsp_handler_pldm    },
-	{ NCSI_PKT_RSP_SNPR,   -1, ncsi_rsp_handler_pldm    }
+	{ NCSI_PKT_RSP_SNPR,   -1, ncsi_rsp_handler_pldm    },
+	{ NCSI_PKT_RSP_GMCMA,  -1, ncsi_rsp_handler_gmcma   },
 };
 
 int ncsi_rcv_rsp(struct sk_buff *skb, struct net_device *dev,
diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c
index eb086b06d60d..177126fb0484 100644
--- a/net/netlink/af_netlink.c
+++ b/net/netlink/af_netlink.c
@@ -1204,8 +1204,7 @@ struct sock *netlink_getsockbyfilp(struct file *filp)
 	return sock;
 }
 
-static struct sk_buff *netlink_alloc_large_skb(unsigned int size,
-					       int broadcast)
+struct sk_buff *netlink_alloc_large_skb(unsigned int size, int broadcast)
 {
 	struct sk_buff *skb;
 	void *data;
diff --git a/net/packet/diag.c b/net/packet/diag.c
index f6b200cb3c06..9a7980e3309d 100644
--- a/net/packet/diag.c
+++ b/net/packet/diag.c
@@ -262,4 +262,5 @@ static void __exit packet_diag_exit(void)
 module_init(packet_diag_init);
 module_exit(packet_diag_exit);
 MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("PACKET socket monitoring via SOCK_DIAG");
 MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG, 17 /* AF_PACKET */);
diff --git a/net/rxrpc/conn_client.c b/net/rxrpc/conn_client.c
index 981ca5b98bcb..1d95f8bc769f 100644
--- a/net/rxrpc/conn_client.c
+++ b/net/rxrpc/conn_client.c
@@ -73,6 +73,7 @@ static void rxrpc_destroy_client_conn_ids(struct rxrpc_local *local)
 static struct rxrpc_bundle *rxrpc_alloc_bundle(struct rxrpc_call *call,
 					       gfp_t gfp)
 {
+	static atomic_t rxrpc_bundle_id;
 	struct rxrpc_bundle *bundle;
 
 	bundle = kzalloc(sizeof(*bundle), gfp);
@@ -85,6 +86,7 @@ static struct rxrpc_bundle *rxrpc_alloc_bundle(struct rxrpc_call *call,
 		bundle->upgrade		= test_bit(RXRPC_CALL_UPGRADE, &call->flags);
 		bundle->service_id	= call->dest_srx.srx_service;
 		bundle->security_level	= call->security_level;
+		bundle->debug_id	= atomic_inc_return(&rxrpc_bundle_id);
 		refcount_set(&bundle->ref, 1);
 		atomic_set(&bundle->active, 1);
 		INIT_LIST_HEAD(&bundle->waiting_calls);
@@ -105,7 +107,8 @@ struct rxrpc_bundle *rxrpc_get_bundle(struct rxrpc_bundle *bundle,
 
 static void rxrpc_free_bundle(struct rxrpc_bundle *bundle)
 {
-	trace_rxrpc_bundle(bundle->debug_id, 1, rxrpc_bundle_free);
+	trace_rxrpc_bundle(bundle->debug_id, refcount_read(&bundle->ref),
+			   rxrpc_bundle_free);
 	rxrpc_put_peer(bundle->peer, rxrpc_peer_put_bundle);
 	key_put(bundle->key);
 	kfree(bundle);
@@ -239,7 +242,6 @@ static bool rxrpc_may_reuse_conn(struct rxrpc_connection *conn)
  */
 int rxrpc_look_up_bundle(struct rxrpc_call *call, gfp_t gfp)
 {
-	static atomic_t rxrpc_bundle_id;
 	struct rxrpc_bundle *bundle, *candidate;
 	struct rxrpc_local *local = call->local;
 	struct rb_node *p, **pp, *parent;
@@ -306,7 +308,6 @@ int rxrpc_look_up_bundle(struct rxrpc_call *call, gfp_t gfp)
 	}
 
 	_debug("new bundle");
-	candidate->debug_id = atomic_inc_return(&rxrpc_bundle_id);
 	rb_link_node(&candidate->local_node, parent, pp);
 	rb_insert_color(&candidate->local_node, &local->client_bundles);
 	call->bundle = rxrpc_get_bundle(candidate, rxrpc_bundle_get_client_call);
diff --git a/net/rxrpc/input.c b/net/rxrpc/input.c
index 030d64f282f3..92495e73b869 100644
--- a/net/rxrpc/input.c
+++ b/net/rxrpc/input.c
@@ -643,12 +643,8 @@ static void rxrpc_complete_rtt_probe(struct rxrpc_call *call,
 			clear_bit(i + RXRPC_CALL_RTT_PEND_SHIFT, &call->rtt_avail);
 			smp_mb(); /* Read data before setting avail bit */
 			set_bit(i, &call->rtt_avail);
-			if (type != rxrpc_rtt_rx_cancel)
-				rxrpc_peer_add_rtt(call, type, i, acked_serial, ack_serial,
-						   sent_at, resp_time);
-			else
-				trace_rxrpc_rtt_rx(call, rxrpc_rtt_rx_cancel, i,
-						   orig_serial, acked_serial, 0, 0);
+			rxrpc_peer_add_rtt(call, type, i, acked_serial, ack_serial,
+					   sent_at, resp_time);
 			matched = true;
 		}
 
@@ -801,28 +797,21 @@ static void rxrpc_input_ack(struct rxrpc_call *call, struct sk_buff *skb)
 			   summary.ack_reason, nr_acks);
 	rxrpc_inc_stat(call->rxnet, stat_rx_acks[ack.reason]);
 
-	switch (ack.reason) {
-	case RXRPC_ACK_PING_RESPONSE:
-		rxrpc_complete_rtt_probe(call, skb->tstamp, acked_serial, ack_serial,
-					 rxrpc_rtt_rx_ping_response);
-		break;
-	case RXRPC_ACK_REQUESTED:
-		rxrpc_complete_rtt_probe(call, skb->tstamp, acked_serial, ack_serial,
-					 rxrpc_rtt_rx_requested_ack);
-		break;
-	default:
-		if (acked_serial != 0)
+	if (acked_serial != 0) {
+		switch (ack.reason) {
+		case RXRPC_ACK_PING_RESPONSE:
 			rxrpc_complete_rtt_probe(call, skb->tstamp, acked_serial, ack_serial,
-						 rxrpc_rtt_rx_cancel);
-		break;
-	}
-
-	if (ack.reason == RXRPC_ACK_PING) {
-		rxrpc_send_ACK(call, RXRPC_ACK_PING_RESPONSE, ack_serial,
-			       rxrpc_propose_ack_respond_to_ping);
-	} else if (sp->hdr.flags & RXRPC_REQUEST_ACK) {
-		rxrpc_send_ACK(call, RXRPC_ACK_REQUESTED, ack_serial,
-			       rxrpc_propose_ack_respond_to_ack);
+						 rxrpc_rtt_rx_ping_response);
+			break;
+		case RXRPC_ACK_REQUESTED:
+			rxrpc_complete_rtt_probe(call, skb->tstamp, acked_serial, ack_serial,
+						 rxrpc_rtt_rx_requested_ack);
+			break;
+		default:
+			rxrpc_complete_rtt_probe(call, skb->tstamp, acked_serial, ack_serial,
+						 rxrpc_rtt_rx_other_ack);
+			break;
+		}
 	}
 
 	/* If we get an EXCEEDS_WINDOW ACK from the server, it probably
@@ -835,7 +824,7 @@ static void rxrpc_input_ack(struct rxrpc_call *call, struct sk_buff *skb)
 	    rxrpc_is_client_call(call)) {
 		rxrpc_set_call_completion(call, RXRPC_CALL_REMOTELY_ABORTED,
 					  0, -ENETRESET);
-		return;
+		goto send_response;
 	}
 
 	/* If we get an OUT_OF_SEQUENCE ACK from the server, that can also
@@ -849,7 +838,7 @@ static void rxrpc_input_ack(struct rxrpc_call *call, struct sk_buff *skb)
 	    rxrpc_is_client_call(call)) {
 		rxrpc_set_call_completion(call, RXRPC_CALL_REMOTELY_ABORTED,
 					  0, -ENETRESET);
-		return;
+		goto send_response;
 	}
 
 	/* Discard any out-of-order or duplicate ACKs (outside lock). */
@@ -857,7 +846,7 @@ static void rxrpc_input_ack(struct rxrpc_call *call, struct sk_buff *skb)
 		trace_rxrpc_rx_discard_ack(call->debug_id, ack_serial,
 					   first_soft_ack, call->acks_first_seq,
 					   prev_pkt, call->acks_prev_seq);
-		return;
+		goto send_response;
 	}
 
 	info.rxMTU = 0;
@@ -897,7 +886,7 @@ static void rxrpc_input_ack(struct rxrpc_call *call, struct sk_buff *skb)
 	case RXRPC_CALL_SERVER_AWAIT_ACK:
 		break;
 	default:
-		return;
+		goto send_response;
 	}
 
 	if (before(hard_ack, call->acks_hard_ack) ||
@@ -909,7 +898,7 @@ static void rxrpc_input_ack(struct rxrpc_call *call, struct sk_buff *skb)
 	if (after(hard_ack, call->acks_hard_ack)) {
 		if (rxrpc_rotate_tx_window(call, hard_ack, &summary)) {
 			rxrpc_end_tx_phase(call, false, rxrpc_eproto_unexpected_ack);
-			return;
+			goto send_response;
 		}
 	}
 
@@ -927,6 +916,14 @@ static void rxrpc_input_ack(struct rxrpc_call *call, struct sk_buff *skb)
 				   rxrpc_propose_ack_ping_for_lost_reply);
 
 	rxrpc_congestion_management(call, skb, &summary, acked_serial);
+
+send_response:
+	if (ack.reason == RXRPC_ACK_PING)
+		rxrpc_send_ACK(call, RXRPC_ACK_PING_RESPONSE, ack_serial,
+			       rxrpc_propose_ack_respond_to_ping);
+	else if (sp->hdr.flags & RXRPC_REQUEST_ACK)
+		rxrpc_send_ACK(call, RXRPC_ACK_REQUESTED, ack_serial,
+			       rxrpc_propose_ack_respond_to_ack);
 }
 
 /*
diff --git a/net/sched/cls_u32.c b/net/sched/cls_u32.c
index d5bdfd4a7655..289e1755c26b 100644
--- a/net/sched/cls_u32.c
+++ b/net/sched/cls_u32.c
@@ -71,7 +71,7 @@ struct tc_u_hnode {
 	struct tc_u_hnode __rcu	*next;
 	u32			handle;
 	u32			prio;
-	int			refcnt;
+	refcount_t		refcnt;
 	unsigned int		divisor;
 	struct idr		handle_idr;
 	bool			is_root;
@@ -86,7 +86,7 @@ struct tc_u_hnode {
 struct tc_u_common {
 	struct tc_u_hnode __rcu	*hlist;
 	void			*ptr;
-	int			refcnt;
+	refcount_t		refcnt;
 	struct idr		handle_idr;
 	struct hlist_node	hnode;
 	long			knodes;
@@ -359,7 +359,7 @@ static int u32_init(struct tcf_proto *tp)
 	if (root_ht == NULL)
 		return -ENOBUFS;
 
-	root_ht->refcnt++;
+	refcount_set(&root_ht->refcnt, 1);
 	root_ht->handle = tp_c ? gen_new_htid(tp_c, root_ht) : 0x80000000;
 	root_ht->prio = tp->prio;
 	root_ht->is_root = true;
@@ -371,18 +371,20 @@ static int u32_init(struct tcf_proto *tp)
 			kfree(root_ht);
 			return -ENOBUFS;
 		}
+		refcount_set(&tp_c->refcnt, 1);
 		tp_c->ptr = key;
 		INIT_HLIST_NODE(&tp_c->hnode);
 		idr_init(&tp_c->handle_idr);
 
 		hlist_add_head(&tp_c->hnode, tc_u_hash(key));
+	} else {
+		refcount_inc(&tp_c->refcnt);
 	}
 
-	tp_c->refcnt++;
 	RCU_INIT_POINTER(root_ht->next, tp_c->hlist);
 	rcu_assign_pointer(tp_c->hlist, root_ht);
 
-	root_ht->refcnt++;
+	/* root_ht must be destroyed when tcf_proto is destroyed */
 	rcu_assign_pointer(tp->root, root_ht);
 	tp->data = tp_c;
 	return 0;
@@ -393,7 +395,7 @@ static void __u32_destroy_key(struct tc_u_knode *n)
 	struct tc_u_hnode *ht = rtnl_dereference(n->ht_down);
 
 	tcf_exts_destroy(&n->exts);
-	if (ht && --ht->refcnt == 0)
+	if (ht && refcount_dec_and_test(&ht->refcnt))
 		kfree(ht);
 	kfree(n);
 }
@@ -601,8 +603,6 @@ static int u32_destroy_hnode(struct tcf_proto *tp, struct tc_u_hnode *ht,
 	struct tc_u_hnode __rcu **hn;
 	struct tc_u_hnode *phn;
 
-	WARN_ON(--ht->refcnt);
-
 	u32_clear_hnode(tp, ht, extack);
 
 	hn = &tp_c->hlist;
@@ -630,10 +630,10 @@ static void u32_destroy(struct tcf_proto *tp, bool rtnl_held,
 
 	WARN_ON(root_ht == NULL);
 
-	if (root_ht && --root_ht->refcnt == 1)
+	if (root_ht && refcount_dec_and_test(&root_ht->refcnt))
 		u32_destroy_hnode(tp, root_ht, extack);
 
-	if (--tp_c->refcnt == 0) {
+	if (refcount_dec_and_test(&tp_c->refcnt)) {
 		struct tc_u_hnode *ht;
 
 		hlist_del(&tp_c->hnode);
@@ -645,7 +645,7 @@ static void u32_destroy(struct tcf_proto *tp, bool rtnl_held,
 			/* u32_destroy_key() will later free ht for us, if it's
 			 * still referenced by some knode
 			 */
-			if (--ht->refcnt == 0)
+			if (refcount_dec_and_test(&ht->refcnt))
 				kfree_rcu(ht, rcu);
 		}
 
@@ -674,7 +674,7 @@ static int u32_delete(struct tcf_proto *tp, void *arg, bool *last,
 		return -EINVAL;
 	}
 
-	if (ht->refcnt == 1) {
+	if (refcount_dec_if_one(&ht->refcnt)) {
 		u32_destroy_hnode(tp, ht, extack);
 	} else {
 		NL_SET_ERR_MSG_MOD(extack, "Can not delete in-use filter");
@@ -682,7 +682,7 @@ static int u32_delete(struct tcf_proto *tp, void *arg, bool *last,
 	}
 
 out:
-	*last = tp_c->refcnt == 1 && tp_c->knodes == 0;
+	*last = refcount_read(&tp_c->refcnt) == 1 && tp_c->knodes == 0;
 	return ret;
 }
 
@@ -766,14 +766,14 @@ static int u32_set_parms(struct net *net, struct tcf_proto *tp,
 				NL_SET_ERR_MSG_MOD(extack, "Not linking to root node");
 				return -EINVAL;
 			}
-			ht_down->refcnt++;
+			refcount_inc(&ht_down->refcnt);
 		}
 
 		ht_old = rtnl_dereference(n->ht_down);
 		rcu_assign_pointer(n->ht_down, ht_down);
 
 		if (ht_old)
-			ht_old->refcnt--;
+			refcount_dec(&ht_old->refcnt);
 	}
 
 	if (ifindex >= 0)
@@ -852,7 +852,7 @@ static struct tc_u_knode *u32_init_knode(struct net *net, struct tcf_proto *tp,
 
 	/* bump reference count as long as we hold pointer to structure */
 	if (ht)
-		ht->refcnt++;
+		refcount_inc(&ht->refcnt);
 
 	return new;
 }
@@ -932,7 +932,7 @@ static int u32_change(struct net *net, struct sk_buff *in_skb,
 
 				ht_old = rtnl_dereference(n->ht_down);
 				if (ht_old)
-					ht_old->refcnt++;
+					refcount_inc(&ht_old->refcnt);
 			}
 			__u32_destroy_key(new);
 			return err;
@@ -980,7 +980,7 @@ static int u32_change(struct net *net, struct sk_buff *in_skb,
 				return err;
 			}
 		}
-		ht->refcnt = 1;
+		refcount_set(&ht->refcnt, 1);
 		ht->divisor = divisor;
 		ht->handle = handle;
 		ht->prio = tp->prio;
diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c
index 4195a4bc26ca..8dd0e5925342 100644
--- a/net/sched/sch_generic.c
+++ b/net/sched/sch_generic.c
@@ -522,8 +522,9 @@ static void dev_watchdog(struct timer_list *t)
 
 			if (unlikely(timedout_ms)) {
 				trace_net_dev_xmit_timeout(dev, i);
-				WARN_ONCE(1, "NETDEV WATCHDOG: %s (%s): transmit queue %u timed out %u ms\n",
-					  dev->name, netdev_drivername(dev), i, timedout_ms);
+				netdev_crit(dev, "NETDEV WATCHDOG: CPU: %d: transmit queue %u timed out %u ms\n",
+					    raw_smp_processor_id(),
+					    i, timedout_ms);
 				netif_freeze_queues(dev);
 				dev->netdev_ops->ndo_tx_timeout(dev, i);
 				netif_unfreeze_queues(dev);
diff --git a/net/sctp/diag.c b/net/sctp/diag.c
index c3d6b92dd386..eb05131ff1dd 100644
--- a/net/sctp/diag.c
+++ b/net/sctp/diag.c
@@ -527,4 +527,5 @@ static void __exit sctp_diag_exit(void)
 module_init(sctp_diag_init);
 module_exit(sctp_diag_exit);
 MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("SCTP socket monitoring via SOCK_DIAG");
 MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG, 2-132);
diff --git a/net/smc/smc_diag.c b/net/smc/smc_diag.c
index 7ff2152971a5..a584613aca12 100644
--- a/net/smc/smc_diag.c
+++ b/net/smc/smc_diag.c
@@ -268,5 +268,6 @@ static void __exit smc_diag_exit(void)
 module_init(smc_diag_init);
 module_exit(smc_diag_exit);
 MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("SMC socket monitoring via SOCK_DIAG");
 MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG, 43 /* AF_SMC */);
 MODULE_ALIAS_GENL_FAMILY(SMCR_GENL_FAMILY_NAME);
diff --git a/net/tipc/diag.c b/net/tipc/diag.c
index 73137f4aeb68..18733451c9e0 100644
--- a/net/tipc/diag.c
+++ b/net/tipc/diag.c
@@ -113,4 +113,5 @@ module_init(tipc_diag_init);
 module_exit(tipc_diag_exit);
 
 MODULE_LICENSE("Dual BSD/GPL");
+MODULE_DESCRIPTION("TIPC socket monitoring via SOCK_DIAG");
 MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG, AF_TIPC);
diff --git a/net/tipc/netlink_compat.c b/net/tipc/netlink_compat.c
index c763008a8adb..079aebb16ed8 100644
--- a/net/tipc/netlink_compat.c
+++ b/net/tipc/netlink_compat.c
@@ -168,7 +168,7 @@ static struct sk_buff *tipc_get_err_tlv(char *str)
 	int str_len = strlen(str) + 1;
 	struct sk_buff *buf;
 
-	buf = tipc_tlv_alloc(TLV_SPACE(str_len));
+	buf = tipc_tlv_alloc(str_len);
 	if (buf)
 		tipc_add_tlv(buf, TIPC_TLV_ERROR_STRING, str, str_len);
 
diff --git a/net/unix/diag.c b/net/unix/diag.c
index 616b55c5b890..bec09a3a1d44 100644
--- a/net/unix/diag.c
+++ b/net/unix/diag.c
@@ -339,4 +339,5 @@ static void __exit unix_diag_exit(void)
 module_init(unix_diag_init);
 module_exit(unix_diag_exit);
 MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("UNIX socket monitoring via SOCK_DIAG");
 MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG, 1 /* AF_LOCAL */);
diff --git a/net/vmw_vsock/diag.c b/net/vmw_vsock/diag.c
index a2823b1c5e28..2e29994f92ff 100644
--- a/net/vmw_vsock/diag.c
+++ b/net/vmw_vsock/diag.c
@@ -174,5 +174,6 @@ static void __exit vsock_diag_exit(void)
 module_init(vsock_diag_init);
 module_exit(vsock_diag_exit);
 MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("VMware Virtual Sockets monitoring via SOCK_DIAG");
 MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG,
 			       40 /* AF_VSOCK */);
diff --git a/net/xdp/xsk_diag.c b/net/xdp/xsk_diag.c
index 22b36c8143cf..9f8955367275 100644
--- a/net/xdp/xsk_diag.c
+++ b/net/xdp/xsk_diag.c
@@ -211,4 +211,5 @@ static void __exit xsk_diag_exit(void)
 module_init(xsk_diag_init);
 module_exit(xsk_diag_exit);
 MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("XDP socket monitoring via SOCK_DIAG");
 MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG, AF_XDP);
diff --git a/samples/landlock/sandboxer.c b/samples/landlock/sandboxer.c
index 08596c0ef070..d7323e5526be 100644
--- a/samples/landlock/sandboxer.c
+++ b/samples/landlock/sandboxer.c
@@ -81,7 +81,8 @@ static int parse_path(char *env_path, const char ***const path_list)
 	LANDLOCK_ACCESS_FS_EXECUTE | \
 	LANDLOCK_ACCESS_FS_WRITE_FILE | \
 	LANDLOCK_ACCESS_FS_READ_FILE | \
-	LANDLOCK_ACCESS_FS_TRUNCATE)
+	LANDLOCK_ACCESS_FS_TRUNCATE | \
+	LANDLOCK_ACCESS_FS_IOCTL)
 
 /* clang-format on */
 
@@ -199,11 +200,12 @@ static int populate_ruleset_net(const char *const env_var, const int ruleset_fd,
 	LANDLOCK_ACCESS_FS_MAKE_BLOCK | \
 	LANDLOCK_ACCESS_FS_MAKE_SYM | \
 	LANDLOCK_ACCESS_FS_REFER | \
-	LANDLOCK_ACCESS_FS_TRUNCATE)
+	LANDLOCK_ACCESS_FS_TRUNCATE | \
+	LANDLOCK_ACCESS_FS_IOCTL)
 
 /* clang-format on */
 
-#define LANDLOCK_ABI_LAST 4
+#define LANDLOCK_ABI_LAST 5
 
 int main(const int argc, char *const argv[], char *const *const envp)
 {
@@ -317,6 +319,11 @@ int main(const int argc, char *const argv[], char *const *const envp)
 		ruleset_attr.handled_access_net &=
 			~(LANDLOCK_ACCESS_NET_BIND_TCP |
 			  LANDLOCK_ACCESS_NET_CONNECT_TCP);
+		__attribute__((fallthrough));
+	case 4:
+		/* Removes LANDLOCK_ACCESS_FS_IOCTL for ABI < 5 */
+		ruleset_attr.handled_access_fs &= ~LANDLOCK_ACCESS_FS_IOCTL;
+
 		fprintf(stderr,
 			"Hint: You should update the running kernel "
 			"to leverage Landlock features "
diff --git a/scripts/Makefile.extrawarn b/scripts/Makefile.extrawarn
index 2fe6f2828d37..1527199161d7 100644
--- a/scripts/Makefile.extrawarn
+++ b/scripts/Makefile.extrawarn
@@ -106,7 +106,6 @@ KBUILD_CFLAGS += $(call cc-option, -Wunused-const-variable)
 KBUILD_CFLAGS += $(call cc-option, -Wpacked-not-aligned)
 KBUILD_CFLAGS += $(call cc-option, -Wformat-overflow)
 KBUILD_CFLAGS += $(call cc-option, -Wformat-truncation)
-KBUILD_CFLAGS += $(call cc-option, -Wstringop-overflow)
 KBUILD_CFLAGS += $(call cc-option, -Wstringop-truncation)
 
 KBUILD_CPPFLAGS += -Wundef
@@ -122,7 +121,6 @@ KBUILD_CFLAGS += $(call cc-disable-warning, restrict)
 KBUILD_CFLAGS += $(call cc-disable-warning, packed-not-aligned)
 KBUILD_CFLAGS += $(call cc-disable-warning, format-overflow)
 KBUILD_CFLAGS += $(call cc-disable-warning, format-truncation)
-KBUILD_CFLAGS += $(call cc-disable-warning, stringop-overflow)
 KBUILD_CFLAGS += $(call cc-disable-warning, stringop-truncation)
 
 ifdef CONFIG_CC_IS_CLANG
diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl
index 25fdb7fda112..a94ed6c46a6d 100755
--- a/scripts/checkpatch.pl
+++ b/scripts/checkpatch.pl
@@ -4054,7 +4054,7 @@ sub process {
 		if ($prevline =~ /^[\+ ]};?\s*$/ &&
 		    $line =~ /^\+/ &&
 		    !($line =~ /^\+\s*$/ ||
-		      $line =~ /^\+\s*(?:EXPORT_SYMBOL|early_param)/ ||
+		      $line =~ /^\+\s*(?:EXPORT_SYMBOL|early_param|ALLOW_ERROR_INJECTION)/ ||
 		      $line =~ /^\+\s*MODULE_/i ||
 		      $line =~ /^\+\s*\#\s*(?:end|elif|else)/ ||
 		      $line =~ /^\+[a-z_]*init/ ||
diff --git a/scripts/checkstack.pl b/scripts/checkstack.pl
index 84f5fb7f1cec..5d0eabfc4ec7 100755
--- a/scripts/checkstack.pl
+++ b/scripts/checkstack.pl
@@ -139,15 +139,11 @@ $total_size = 0;
 while (my $line = <STDIN>) {
 	if ($line =~ m/$funcre/) {
 		$func = $1;
-		next if $line !~ m/^($xs*)/;
+		next if $line !~ m/^($x*)/;
 		if ($total_size > $min_stack) {
 			push @stack, "$intro$total_size\n";
 		}
-
-		$addr = $1;
-		$addr =~ s/ /0/g;
-		$addr = "0x$addr";
-
+		$addr = "0x$1";
 		$intro = "$addr $func [$file]:";
 		my $padlen = 56 - length($intro);
 		while ($padlen > 0) {
@@ -194,5 +190,20 @@ if ($total_size > $min_stack) {
 	push @stack, "$intro$total_size\n";
 }
 
-# Sort output by size (last field)
-print sort { ($b =~ /:\t*(\d+)$/)[0] <=> ($a =~ /:\t*(\d+)$/)[0] } @stack;
+# Sort output by size (last field) and function name if size is the same
+sub sort_lines {
+	my ($a, $b) = @_;
+
+	my $num_a = $1 if $a =~ /:\t*(\d+)$/;
+	my $num_b = $1 if $b =~ /:\t*(\d+)$/;
+	my $func_a = $1 if $a =~ / (.*):/;
+	my $func_b = $1 if $b =~ / (.*):/;
+
+	if ($num_a != $num_b) {
+		return $num_b <=> $num_a;
+	} else {
+		return $func_a cmp $func_b;
+	}
+}
+
+print sort { sort_lines($a, $b) } @stack;
diff --git a/scripts/dtc/dt-extract-compatibles b/scripts/dtc/dt-extract-compatibles
index bd07477dd144..5ffb2364409b 100755
--- a/scripts/dtc/dt-extract-compatibles
+++ b/scripts/dtc/dt-extract-compatibles
@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 # SPDX-License-Identifier: GPL-2.0-only
 
+import fnmatch
 import os
-import glob
 import re
 import argparse
 
@@ -81,10 +81,20 @@ def print_compat(filename, compatibles):
 	else:
 		print(*compatibles, sep='\n')
 
+def glob_without_symlinks(root, glob):
+	for path, dirs, files in os.walk(root):
+		# Ignore hidden directories
+		for d in dirs:
+			if fnmatch.fnmatch(d, ".*"):
+				dirs.remove(d)
+		for f in files:
+			if fnmatch.fnmatch(f, glob):
+				yield os.path.join(path, f)
+
 def files_to_parse(path_args):
 	for f in path_args:
 		if os.path.isdir(f):
-			for filename in glob.iglob(f + "/**/*.c", recursive=True):
+			for filename in glob_without_symlinks(f, "*.c"):
 				yield filename
 		else:
 			yield f
diff --git a/scripts/genksyms/genksyms.c b/scripts/genksyms/genksyms.c
index f5dfdb9d80e9..f3901c55df23 100644
--- a/scripts/genksyms/genksyms.c
+++ b/scripts/genksyms/genksyms.c
@@ -16,9 +16,7 @@
 #include <unistd.h>
 #include <assert.h>
 #include <stdarg.h>
-#ifdef __GNU_LIBRARY__
 #include <getopt.h>
-#endif				/* __GNU_LIBRARY__ */
 
 #include "genksyms.h"
 /*----------------------------------------------------------------------*/
@@ -718,8 +716,6 @@ void error_with_pos(const char *fmt, ...)
 static void genksyms_usage(void)
 {
 	fputs("Usage:\n" "genksyms [-adDTwqhVR] > /path/to/.tmp_obj.ver\n" "\n"
-#ifdef __GNU_LIBRARY__
-	      "  -s, --symbol-prefix   Select symbol prefix\n"
 	      "  -d, --debug           Increment the debug level (repeatable)\n"
 	      "  -D, --dump            Dump expanded symbol defs (for debugging only)\n"
 	      "  -r, --reference file  Read reference symbols from a file\n"
@@ -729,18 +725,6 @@ static void genksyms_usage(void)
 	      "  -q, --quiet           Disable warnings (default)\n"
 	      "  -h, --help            Print this message\n"
 	      "  -V, --version         Print the release version\n"
-#else				/* __GNU_LIBRARY__ */
-	      "  -s                    Select symbol prefix\n"
-	      "  -d                    Increment the debug level (repeatable)\n"
-	      "  -D                    Dump expanded symbol defs (for debugging only)\n"
-	      "  -r file               Read reference symbols from a file\n"
-	      "  -T file               Dump expanded types into file\n"
-	      "  -p                    Preserve reference modversions or fail\n"
-	      "  -w                    Enable warnings\n"
-	      "  -q                    Disable warnings (default)\n"
-	      "  -h                    Print this message\n"
-	      "  -V                    Print the release version\n"
-#endif				/* __GNU_LIBRARY__ */
 	      , stderr);
 }
 
@@ -749,7 +733,6 @@ int main(int argc, char **argv)
 	FILE *dumpfile = NULL, *ref_file = NULL;
 	int o;
 
-#ifdef __GNU_LIBRARY__
 	struct option long_opts[] = {
 		{"debug", 0, 0, 'd'},
 		{"warnings", 0, 0, 'w'},
@@ -763,11 +746,8 @@ int main(int argc, char **argv)
 		{0, 0, 0, 0}
 	};
 
-	while ((o = getopt_long(argc, argv, "s:dwqVDr:T:ph",
+	while ((o = getopt_long(argc, argv, "dwqVDr:T:ph",
 				&long_opts[0], NULL)) != EOF)
-#else				/* __GNU_LIBRARY__ */
-	while ((o = getopt(argc, argv, "s:dwqVDr:T:ph")) != EOF)
-#endif				/* __GNU_LIBRARY__ */
 		switch (o) {
 		case 'd':
 			flag_debug++;
diff --git a/scripts/kconfig/confdata.c b/scripts/kconfig/confdata.c
index 4a6811d77d18..7fca9cc3ae74 100644
--- a/scripts/kconfig/confdata.c
+++ b/scripts/kconfig/confdata.c
@@ -594,7 +594,7 @@ int conf_read(const char *name)
 				/* Reset a string value if it's out of range */
 				if (sym_string_within_range(sym, sym->def[S_DEF_USER].val))
 					break;
-				sym->flags &= ~(SYMBOL_VALID|SYMBOL_DEF_USER);
+				sym->flags &= ~SYMBOL_VALID;
 				conf_unsaved++;
 				break;
 			default:
diff --git a/security/Makefile b/security/Makefile
index 18121f8f85cd..59f238490665 100644
--- a/security/Makefile
+++ b/security/Makefile
@@ -7,6 +7,7 @@ obj-$(CONFIG_KEYS)			+= keys/
 
 # always enable default capabilities
 obj-y					+= commoncap.o
+obj-$(CONFIG_SECURITY) 			+= lsm_syscalls.o
 obj-$(CONFIG_MMU)			+= min_addr.o
 
 # Object file lists
diff --git a/security/apparmor/Kconfig b/security/apparmor/Kconfig
index e0d1dd0a192a..64cc3044a42c 100644
--- a/security/apparmor/Kconfig
+++ b/security/apparmor/Kconfig
@@ -57,10 +57,10 @@ config SECURITY_APPARMOR_INTROSPECT_POLICY
 	  cpu is paramount.
 
 config SECURITY_APPARMOR_HASH
-	bool "Enable introspection of sha1 hashes for loaded profiles"
+	bool "Enable introspection of sha256 hashes for loaded profiles"
 	depends on SECURITY_APPARMOR_INTROSPECT_POLICY
 	select CRYPTO
-	select CRYPTO_SHA1
+	select CRYPTO_SHA256
 	default y
 	help
 	  This option selects whether introspection of loaded policy
@@ -74,10 +74,10 @@ config SECURITY_APPARMOR_HASH_DEFAULT
        depends on SECURITY_APPARMOR_HASH
        default y
        help
-         This option selects whether sha1 hashing of loaded policy
-	 is enabled by default. The generation of sha1 hashes for
-	 loaded policy provide system administrators a quick way
-	 to verify that policy in the kernel matches what is expected,
+	 This option selects whether sha256 hashing of loaded policy
+	 is enabled by default. The generation of sha256 hashes for
+	 loaded policy provide system administrators a quick way to
+	 verify that policy in the kernel matches what is expected,
 	 however it can slow down policy load on some devices. In
 	 these cases policy hashing can be disabled by default and
 	 enabled only if needed.
diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c
index 38650e52ef57..21a6413b8472 100644
--- a/security/apparmor/apparmorfs.c
+++ b/security/apparmor/apparmorfs.c
@@ -1474,7 +1474,7 @@ int __aa_fs_create_rawdata(struct aa_ns *ns, struct aa_loaddata *rawdata)
 	rawdata->dents[AAFS_LOADDATA_REVISION] = dent;
 
 	if (aa_g_hash_policy) {
-		dent = aafs_create_file("sha1", S_IFREG | 0444, dir,
+		dent = aafs_create_file("sha256", S_IFREG | 0444, dir,
 					      rawdata, &seq_rawdata_hash_fops);
 		if (IS_ERR(dent))
 			goto fail;
@@ -1648,11 +1648,11 @@ static const char *rawdata_get_link_base(struct dentry *dentry,
 	return target;
 }
 
-static const char *rawdata_get_link_sha1(struct dentry *dentry,
+static const char *rawdata_get_link_sha256(struct dentry *dentry,
 					 struct inode *inode,
 					 struct delayed_call *done)
 {
-	return rawdata_get_link_base(dentry, inode, done, "sha1");
+	return rawdata_get_link_base(dentry, inode, done, "sha256");
 }
 
 static const char *rawdata_get_link_abi(struct dentry *dentry,
@@ -1669,8 +1669,8 @@ static const char *rawdata_get_link_data(struct dentry *dentry,
 	return rawdata_get_link_base(dentry, inode, done, "raw_data");
 }
 
-static const struct inode_operations rawdata_link_sha1_iops = {
-	.get_link	= rawdata_get_link_sha1,
+static const struct inode_operations rawdata_link_sha256_iops = {
+	.get_link	= rawdata_get_link_sha256,
 };
 
 static const struct inode_operations rawdata_link_abi_iops = {
@@ -1743,7 +1743,7 @@ int __aafs_profile_mkdir(struct aa_profile *profile, struct dentry *parent)
 	profile->dents[AAFS_PROF_ATTACH] = dent;
 
 	if (profile->hash) {
-		dent = create_profile_file(dir, "sha1", profile,
+		dent = create_profile_file(dir, "sha256", profile,
 					   &seq_profile_hash_fops);
 		if (IS_ERR(dent))
 			goto fail;
@@ -1753,9 +1753,9 @@ int __aafs_profile_mkdir(struct aa_profile *profile, struct dentry *parent)
 #ifdef CONFIG_SECURITY_APPARMOR_EXPORT_BINARY
 	if (profile->rawdata) {
 		if (aa_g_hash_policy) {
-			dent = aafs_create("raw_sha1", S_IFLNK | 0444, dir,
+			dent = aafs_create("raw_sha256", S_IFLNK | 0444, dir,
 					   profile->label.proxy, NULL, NULL,
-					   &rawdata_link_sha1_iops);
+					   &rawdata_link_sha256_iops);
 			if (IS_ERR(dent))
 				goto fail;
 			aa_get_proxy(profile->label.proxy);
diff --git a/security/apparmor/crypto.c b/security/apparmor/crypto.c
index 6724e2ff6da8..aad486b2fca6 100644
--- a/security/apparmor/crypto.c
+++ b/security/apparmor/crypto.c
@@ -106,16 +106,16 @@ static int __init init_profile_hash(void)
 	if (!apparmor_initialized)
 		return 0;
 
-	tfm = crypto_alloc_shash("sha1", 0, 0);
+	tfm = crypto_alloc_shash("sha256", 0, 0);
 	if (IS_ERR(tfm)) {
 		int error = PTR_ERR(tfm);
-		AA_ERROR("failed to setup profile sha1 hashing: %d\n", error);
+		AA_ERROR("failed to setup profile sha256 hashing: %d\n", error);
 		return error;
 	}
 	apparmor_tfm = tfm;
 	apparmor_hash_size = crypto_shash_digestsize(apparmor_tfm);
 
-	aa_info_message("AppArmor sha1 policy hashing enabled");
+	aa_info_message("AppArmor sha256 policy hashing enabled");
 
 	return 0;
 }
diff --git a/security/apparmor/domain.c b/security/apparmor/domain.c
index 89fbeab4b33b..571158ec6188 100644
--- a/security/apparmor/domain.c
+++ b/security/apparmor/domain.c
@@ -1311,7 +1311,7 @@ static int change_profile_perms_wrapper(const char *op, const char *name,
 	return error;
 }
 
-const char *stack_msg = "change_profile unprivileged unconfined converted to stacking";
+static const char *stack_msg = "change_profile unprivileged unconfined converted to stacking";
 
 /**
  * aa_change_profile - perform a one-way profile transition
diff --git a/security/apparmor/include/procattr.h b/security/apparmor/include/procattr.h
index 31689437e0e1..03dbfdb2f2c0 100644
--- a/security/apparmor/include/procattr.h
+++ b/security/apparmor/include/procattr.h
@@ -11,7 +11,7 @@
 #ifndef __AA_PROCATTR_H
 #define __AA_PROCATTR_H
 
-int aa_getprocattr(struct aa_label *label, char **string);
+int aa_getprocattr(struct aa_label *label, char **string, bool newline);
 int aa_setprocattr_changehat(char *args, size_t size, int flags);
 
 #endif /* __AA_PROCATTR_H */
diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c
index 4981bdf02993..816dda0229c1 100644
--- a/security/apparmor/lsm.c
+++ b/security/apparmor/lsm.c
@@ -24,6 +24,7 @@
 #include <linux/zstd.h>
 #include <net/sock.h>
 #include <uapi/linux/mount.h>
+#include <uapi/linux/lsm.h>
 
 #include "include/apparmor.h"
 #include "include/apparmorfs.h"
@@ -775,6 +776,46 @@ static int apparmor_sb_pivotroot(const struct path *old_path,
 	return error;
 }
 
+static int apparmor_getselfattr(unsigned int attr, struct lsm_ctx __user *lx,
+				size_t *size, u32 flags)
+{
+	int error = -ENOENT;
+	struct aa_task_ctx *ctx = task_ctx(current);
+	struct aa_label *label = NULL;
+	char *value;
+
+	switch (attr) {
+	case LSM_ATTR_CURRENT:
+		label = aa_get_newest_label(cred_label(current_cred()));
+		break;
+	case LSM_ATTR_PREV:
+		if (ctx->previous)
+			label = aa_get_newest_label(ctx->previous);
+		break;
+	case LSM_ATTR_EXEC:
+		if (ctx->onexec)
+			label = aa_get_newest_label(ctx->onexec);
+		break;
+	default:
+		error = -EOPNOTSUPP;
+		break;
+	}
+
+	if (label) {
+		error = aa_getprocattr(label, &value, false);
+		if (error > 0)
+			error = lsm_fill_user_ctx(lx, size, value, error,
+						  LSM_ID_APPARMOR, 0);
+		kfree(value);
+	}
+
+	aa_put_label(label);
+
+	if (error < 0)
+		return error;
+	return 1;
+}
+
 static int apparmor_getprocattr(struct task_struct *task, const char *name,
 				char **value)
 {
@@ -794,7 +835,7 @@ static int apparmor_getprocattr(struct task_struct *task, const char *name,
 		error = -EINVAL;
 
 	if (label)
-		error = aa_getprocattr(label, value);
+		error = aa_getprocattr(label, value, true);
 
 	aa_put_label(label);
 	put_cred(cred);
@@ -802,8 +843,7 @@ static int apparmor_getprocattr(struct task_struct *task, const char *name,
 	return error;
 }
 
-static int apparmor_setprocattr(const char *name, void *value,
-				size_t size)
+static int do_setattr(u64 attr, void *value, size_t size)
 {
 	char *command, *largs = NULL, *args = value;
 	size_t arg_size;
@@ -834,7 +874,7 @@ static int apparmor_setprocattr(const char *name, void *value,
 		goto out;
 
 	arg_size = size - (args - (largs ? largs : (char *) value));
-	if (strcmp(name, "current") == 0) {
+	if (attr == LSM_ATTR_CURRENT) {
 		if (strcmp(command, "changehat") == 0) {
 			error = aa_setprocattr_changehat(args, arg_size,
 							 AA_CHANGE_NOFLAGS);
@@ -849,7 +889,7 @@ static int apparmor_setprocattr(const char *name, void *value,
 			error = aa_change_profile(args, AA_CHANGE_STACK);
 		} else
 			goto fail;
-	} else if (strcmp(name, "exec") == 0) {
+	} else if (attr == LSM_ATTR_EXEC) {
 		if (strcmp(command, "exec") == 0)
 			error = aa_change_profile(args, AA_CHANGE_ONEXEC);
 		else if (strcmp(command, "stack") == 0)
@@ -869,13 +909,42 @@ static int apparmor_setprocattr(const char *name, void *value,
 
 fail:
 	ad.subj_label = begin_current_label_crit_section();
-	ad.info = name;
+	if (attr == LSM_ATTR_CURRENT)
+		ad.info = "current";
+	else if (attr == LSM_ATTR_EXEC)
+		ad.info = "exec";
+	else
+		ad.info = "invalid";
 	ad.error = error = -EINVAL;
 	aa_audit_msg(AUDIT_APPARMOR_DENIED, &ad, NULL);
 	end_current_label_crit_section(ad.subj_label);
 	goto out;
 }
 
+static int apparmor_setselfattr(unsigned int attr, struct lsm_ctx *ctx,
+				size_t size, u32 flags)
+{
+	int rc;
+
+	if (attr != LSM_ATTR_CURRENT && attr != LSM_ATTR_EXEC)
+		return -EOPNOTSUPP;
+
+	rc = do_setattr(attr, ctx->ctx, ctx->ctx_len);
+	if (rc > 0)
+		return 0;
+	return rc;
+}
+
+static int apparmor_setprocattr(const char *name, void *value,
+				size_t size)
+{
+	int attr = lsm_name_to_attr(name);
+
+	if (attr)
+		return do_setattr(attr, value, size);
+	return -EINVAL;
+}
+
 /**
  * apparmor_bprm_committing_creds - do task cleanup on committing new creds
  * @bprm: binprm for the exec  (NOT NULL)
@@ -1385,6 +1454,11 @@ struct lsm_blob_sizes apparmor_blob_sizes __ro_after_init = {
 	.lbs_task = sizeof(struct aa_task_ctx),
 };
 
+static const struct lsm_id apparmor_lsmid = {
+	.name = "apparmor",
+	.id = LSM_ID_APPARMOR,
+};
+
 static struct security_hook_list apparmor_hooks[] __ro_after_init = {
 	LSM_HOOK_INIT(ptrace_access_check, apparmor_ptrace_access_check),
 	LSM_HOOK_INIT(ptrace_traceme, apparmor_ptrace_traceme),
@@ -1418,6 +1492,8 @@ static struct security_hook_list apparmor_hooks[] __ro_after_init = {
 	LSM_HOOK_INIT(file_lock, apparmor_file_lock),
 	LSM_HOOK_INIT(file_truncate, apparmor_file_truncate),
 
+	LSM_HOOK_INIT(getselfattr, apparmor_getselfattr),
+	LSM_HOOK_INIT(setselfattr, apparmor_setselfattr),
 	LSM_HOOK_INIT(getprocattr, apparmor_getprocattr),
 	LSM_HOOK_INIT(setprocattr, apparmor_setprocattr),
 
@@ -2106,7 +2182,7 @@ __initcall(apparmor_nf_ip_init);
 static char nulldfa_src[] = {
 	#include "nulldfa.in"
 };
-struct aa_dfa *nulldfa;
+static struct aa_dfa *nulldfa;
 
 static char stacksplitdfa_src[] = {
 	#include "stacksplitdfa.in"
@@ -2202,7 +2278,7 @@ static int __init apparmor_init(void)
 		goto buffers_out;
 	}
 	security_add_hooks(apparmor_hooks, ARRAY_SIZE(apparmor_hooks),
-				"apparmor");
+				&apparmor_lsmid);
 
 	/* Report that AppArmor successfully initialized */
 	apparmor_initialized = 1;
diff --git a/security/apparmor/procattr.c b/security/apparmor/procattr.c
index 197d41f9c32b..e3857e3d7c6c 100644
--- a/security/apparmor/procattr.c
+++ b/security/apparmor/procattr.c
@@ -20,6 +20,7 @@
  * aa_getprocattr - Return the label information for @label
  * @label: the label to print label info about  (NOT NULL)
  * @string: Returns - string containing the label info (NOT NULL)
+ * @newline: indicates that a newline should be added
  *
  * Requires: label != NULL && string != NULL
  *
@@ -27,7 +28,7 @@
  *
  * Returns: size of string placed in @string else error code on failure
  */
-int aa_getprocattr(struct aa_label *label, char **string)
+int aa_getprocattr(struct aa_label *label, char **string, bool newline)
 {
 	struct aa_ns *ns = labels_ns(label);
 	struct aa_ns *current_ns = aa_get_current_ns();
@@ -57,11 +58,12 @@ int aa_getprocattr(struct aa_label *label, char **string)
 		return len;
 	}
 
-	(*string)[len] = '\n';
-	(*string)[len + 1] = 0;
+	if (newline)
+		(*string)[len++] = '\n';
+	(*string)[len] = 0;
 
 	aa_put_ns(current_ns);
-	return len + 1;
+	return len;
 }
 
 /**
diff --git a/security/apparmor/task.c b/security/apparmor/task.c
index f29a2e80e6bf..c87fb9f4ac18 100644
--- a/security/apparmor/task.c
+++ b/security/apparmor/task.c
@@ -278,7 +278,9 @@ static int profile_tracer_perm(const struct cred *cred,
 
 /**
  * aa_may_ptrace - test if tracer task can trace the tracee
+ * @tracer_cred: cred of task doing the tracing  (NOT NULL)
  * @tracer: label of the task doing the tracing  (NOT NULL)
+ * @tracee_cred: cred of task to be traced
  * @tracee: task label to be traced
  * @request: permission request
  *
diff --git a/security/bpf/hooks.c b/security/bpf/hooks.c
index cfaf1d0e6a5f..57b9ffd53c98 100644
--- a/security/bpf/hooks.c
+++ b/security/bpf/hooks.c
@@ -5,6 +5,7 @@
  */
 #include <linux/lsm_hooks.h>
 #include <linux/bpf_lsm.h>
+#include <uapi/linux/lsm.h>
 
 static struct security_hook_list bpf_lsm_hooks[] __ro_after_init = {
 	#define LSM_HOOK(RET, DEFAULT, NAME, ...) \
@@ -15,9 +16,15 @@ static struct security_hook_list bpf_lsm_hooks[] __ro_after_init = {
 	LSM_HOOK_INIT(task_free, bpf_task_storage_free),
 };
 
+static const struct lsm_id bpf_lsmid = {
+	.name = "bpf",
+	.id = LSM_ID_BPF,
+};
+
 static int __init bpf_lsm_init(void)
 {
-	security_add_hooks(bpf_lsm_hooks, ARRAY_SIZE(bpf_lsm_hooks), "bpf");
+	security_add_hooks(bpf_lsm_hooks, ARRAY_SIZE(bpf_lsm_hooks),
+			   &bpf_lsmid);
 	pr_info("LSM support for eBPF active\n");
 	return 0;
 }
diff --git a/security/commoncap.c b/security/commoncap.c
index 8e8c630ce204..162d96b3a676 100644
--- a/security/commoncap.c
+++ b/security/commoncap.c
@@ -25,6 +25,7 @@
 #include <linux/binfmts.h>
 #include <linux/personality.h>
 #include <linux/mnt_idmapping.h>
+#include <uapi/linux/lsm.h>
 
 /*
  * If a non-root user executes a setuid-root binary in
@@ -1440,6 +1441,11 @@ int cap_mmap_file(struct file *file, unsigned long reqprot,
 
 #ifdef CONFIG_SECURITY
 
+static const struct lsm_id capability_lsmid = {
+	.name = "capability",
+	.id = LSM_ID_CAPABILITY,
+};
+
 static struct security_hook_list capability_hooks[] __ro_after_init = {
 	LSM_HOOK_INIT(capable, cap_capable),
 	LSM_HOOK_INIT(settime, cap_settime),
@@ -1464,7 +1470,7 @@ static struct security_hook_list capability_hooks[] __ro_after_init = {
 static int __init capability_init(void)
 {
 	security_add_hooks(capability_hooks, ARRAY_SIZE(capability_hooks),
-				"capability");
+			   &capability_lsmid);
 	return 0;
 }
 
diff --git a/security/landlock/cred.c b/security/landlock/cred.c
index 13dff2a31545..786af18c4a1c 100644
--- a/security/landlock/cred.c
+++ b/security/landlock/cred.c
@@ -42,5 +42,5 @@ static struct security_hook_list landlock_hooks[] __ro_after_init = {
 __init void landlock_add_cred_hooks(void)
 {
 	security_add_hooks(landlock_hooks, ARRAY_SIZE(landlock_hooks),
-			   LANDLOCK_NAME);
+			   &landlock_lsmid);
 }
diff --git a/security/landlock/fs.c b/security/landlock/fs.c
index bc7c126deea2..5d56e94de081 100644
--- a/security/landlock/fs.c
+++ b/security/landlock/fs.c
@@ -7,12 +7,14 @@
  * Copyright © 2021-2022 Microsoft Corporation
  */
 
+#include <asm/ioctls.h>
 #include <linux/atomic.h>
 #include <linux/bitops.h>
 #include <linux/bits.h>
 #include <linux/compiler_types.h>
 #include <linux/dcache.h>
 #include <linux/err.h>
+#include <linux/falloc.h>
 #include <linux/fs.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
@@ -28,6 +30,7 @@
 #include <linux/types.h>
 #include <linux/wait_bit.h>
 #include <linux/workqueue.h>
+#include <uapi/linux/fiemap.h>
 #include <uapi/linux/landlock.h>
 
 #include "common.h"
@@ -83,6 +86,68 @@ static const struct landlock_object_underops landlock_fs_underops = {
 	.release = release_inode
 };
 
+/* IOCTL helpers */
+
+/**
+ * expand_ioctl() - Return the dst flags from either the src flag or the
+ * %LANDLOCK_ACCESS_FS_IOCTL flag, depending on whether the
+ * %LANDLOCK_ACCESS_FS_IOCTL and src access rights are handled or not.
+ *
+ * @handled: Handled access rights
+ * @access: The access mask to copy values from
+ * @src: A single access right to copy from in @access.
+ * @dst: One or more access rights to copy to
+ *
+ * Returns: @dst, or 0
+ */
+static inline access_mask_t expand_ioctl(const access_mask_t handled,
+					 const access_mask_t access,
+					 const access_mask_t src,
+					 const access_mask_t dst)
+{
+	if (!(handled & LANDLOCK_ACCESS_FS_IOCTL))
+		return 0;
+
+	access_mask_t copy_from = (handled & src) ? src :
+						    LANDLOCK_ACCESS_FS_IOCTL;
+	if (access & copy_from)
+		return dst;
+
+	return 0;
+}
+
+/**
+ * landlock_expand_access_fs() - Returns @access with the synthetic IOCTL group
+ * flags enabled if necessary.
+ *
+ * @handled: Handled FS access rights.
+ * @access: FS access rights to expand.
+ *
+ * Returns: @access expanded by the necessary flags for the synthetic IOCTL
+ * access rights.
+ */
+static inline access_mask_t
+landlock_expand_access_fs(const access_mask_t handled,
+			  const access_mask_t access)
+{
+	return access |
+	       expand_ioctl(handled, access, LANDLOCK_ACCESS_FS_WRITE_FILE,
+			    LANDLOCK_ACCESS_FS_IOCTL_GROUP1 |
+				    LANDLOCK_ACCESS_FS_IOCTL_GROUP2 |
+				    LANDLOCK_ACCESS_FS_IOCTL_GROUP4) |
+	       expand_ioctl(handled, access, LANDLOCK_ACCESS_FS_READ_FILE,
+			    LANDLOCK_ACCESS_FS_IOCTL_GROUP1 |
+				    LANDLOCK_ACCESS_FS_IOCTL_GROUP2 |
+				    LANDLOCK_ACCESS_FS_IOCTL_GROUP3) |
+	       expand_ioctl(handled, access, LANDLOCK_ACCESS_FS_READ_DIR,
+			    LANDLOCK_ACCESS_FS_IOCTL_GROUP1);
+}
+
+access_mask_t landlock_expand_handled_access_fs(const access_mask_t handled)
+{
+	return landlock_expand_access_fs(handled, handled);
+}
+
 /* Ruleset management */
 
 static struct landlock_object *get_inode_object(struct inode *const inode)
@@ -147,7 +212,8 @@ static struct landlock_object *get_inode_object(struct inode *const inode)
 	LANDLOCK_ACCESS_FS_EXECUTE | \
 	LANDLOCK_ACCESS_FS_WRITE_FILE | \
 	LANDLOCK_ACCESS_FS_READ_FILE | \
-	LANDLOCK_ACCESS_FS_TRUNCATE)
+	LANDLOCK_ACCESS_FS_TRUNCATE | \
+	LANDLOCK_ACCESS_FS_IOCTL)
 /* clang-format on */
 
 /*
@@ -157,6 +223,7 @@ int landlock_append_fs_rule(struct landlock_ruleset *const ruleset,
 			    const struct path *const path,
 			    access_mask_t access_rights)
 {
+	access_mask_t handled;
 	int err;
 	struct landlock_id id = {
 		.type = LANDLOCK_KEY_INODE,
@@ -169,9 +236,11 @@ int landlock_append_fs_rule(struct landlock_ruleset *const ruleset,
 	if (WARN_ON_ONCE(ruleset->num_layers != 1))
 		return -EINVAL;
 
+	handled = landlock_get_fs_access_mask(ruleset, 0);
+	/* Expands the synthetic IOCTL groups. */
+	access_rights |= landlock_expand_access_fs(handled, access_rights);
 	/* Transforms relative access rights to absolute ones. */
-	access_rights |= LANDLOCK_MASK_ACCESS_FS &
-			 ~landlock_get_fs_access_mask(ruleset, 0);
+	access_rights |= LANDLOCK_MASK_ACCESS_FS & ~handled;
 	id.key.object = get_inode_object(d_backing_inode(path->dentry));
 	if (IS_ERR(id.key.object))
 		return PTR_ERR(id.key.object);
@@ -1119,11 +1188,17 @@ static int hook_file_alloc_security(struct file *const file)
 	return 0;
 }
 
+static const access_mask_t ioctl_groups =
+	LANDLOCK_ACCESS_FS_IOCTL_GROUP1 | LANDLOCK_ACCESS_FS_IOCTL_GROUP2 |
+	LANDLOCK_ACCESS_FS_IOCTL_GROUP3 | LANDLOCK_ACCESS_FS_IOCTL_GROUP4;
+
 static int hook_file_open(struct file *const file)
 {
 	layer_mask_t layer_masks[LANDLOCK_NUM_ACCESS_FS] = {};
 	access_mask_t open_access_request, full_access_request, allowed_access;
-	const access_mask_t optional_access = LANDLOCK_ACCESS_FS_TRUNCATE;
+	const access_mask_t optional_access = LANDLOCK_ACCESS_FS_TRUNCATE |
+					      LANDLOCK_ACCESS_FS_IOCTL |
+					      ioctl_groups;
 	const struct landlock_ruleset *const dom = get_current_fs_domain();
 
 	if (!dom)
@@ -1196,6 +1271,72 @@ static int hook_file_truncate(struct file *const file)
 	return -EACCES;
 }
 
+/**
+ * required_ioctl_access(): Determine required IOCTL access rights.
+ *
+ * @cmd: The IOCTL command that is supposed to be run.
+ *
+ * Returns: The access rights that must be granted on an opened file in order to
+ * use the given @cmd.
+ */
+static access_mask_t required_ioctl_access(unsigned int cmd)
+{
+	switch (cmd) {
+	case FIOCLEX:
+	case FIONCLEX:
+	case FIONBIO:
+	case FIOASYNC:
+		/*
+		 * FIOCLEX, FIONCLEX, FIONBIO and FIOASYNC manipulate the FD's
+		 * close-on-exec and the file's buffered-IO and async flags.
+		 * These operations are also available through fcntl(2),
+		 * and are unconditionally permitted in Landlock.
+		 */
+		return 0;
+	case FIOQSIZE:
+		return LANDLOCK_ACCESS_FS_IOCTL_GROUP1;
+	case FS_IOC_FIEMAP:
+	case FIBMAP:
+	case FIGETBSZ:
+		return LANDLOCK_ACCESS_FS_IOCTL_GROUP2;
+	case FIONREAD:
+	case FIDEDUPERANGE:
+		return LANDLOCK_ACCESS_FS_IOCTL_GROUP3;
+	case FICLONE:
+	case FICLONERANGE:
+	case FS_IOC_RESVSP:
+	case FS_IOC_RESVSP64:
+	case FS_IOC_UNRESVSP:
+	case FS_IOC_UNRESVSP64:
+	case FS_IOC_ZERO_RANGE:
+		return LANDLOCK_ACCESS_FS_IOCTL_GROUP4;
+	default:
+		/*
+		 * Other commands are guarded by the catch-all access right.
+		 */
+		return LANDLOCK_ACCESS_FS_IOCTL;
+	}
+}
+
+static int hook_file_ioctl(struct file *file, unsigned int cmd,
+			   unsigned long arg)
+{
+	const access_mask_t required_access = required_ioctl_access(cmd);
+	const access_mask_t allowed_access =
+		landlock_file(file)->allowed_access;
+
+	/*
+	 * It is the access rights at the time of opening the file which
+	 * determine whether ioctl can be used on the opened file later.
+	 *
+	 * The access right is attached to the opened file in hook_file_open().
+	 */
+	if ((allowed_access & required_access) == required_access)
+		return 0;
+
+	return -EACCES;
+}
+
 static struct security_hook_list landlock_hooks[] __ro_after_init = {
 	LSM_HOOK_INIT(inode_free_security, hook_inode_free_security),
 
@@ -1218,10 +1359,11 @@ static struct security_hook_list landlock_hooks[] __ro_after_init = {
 	LSM_HOOK_INIT(file_alloc_security, hook_file_alloc_security),
 	LSM_HOOK_INIT(file_open, hook_file_open),
 	LSM_HOOK_INIT(file_truncate, hook_file_truncate),
+	LSM_HOOK_INIT(file_ioctl, hook_file_ioctl),
 };
 
 __init void landlock_add_fs_hooks(void)
 {
 	security_add_hooks(landlock_hooks, ARRAY_SIZE(landlock_hooks),
-			   LANDLOCK_NAME);
+			   &landlock_lsmid);
 }
diff --git a/security/landlock/fs.h b/security/landlock/fs.h
index 488e4813680a..b3ef11968160 100644
--- a/security/landlock/fs.h
+++ b/security/landlock/fs.h
@@ -92,4 +92,15 @@ int landlock_append_fs_rule(struct landlock_ruleset *const ruleset,
 			    const struct path *const path,
 			    access_mask_t access_hierarchy);
 
+/**
+ * landlock_expand_handled_access_fs() - add synthetic IOCTL access rights to an
+ * access mask of handled accesses.
+ *
+ * @handled: The handled accesses of a ruleset that is being created
+ *
+ * Returns: @handled, with the bits for the synthetic IOCTL access rights set,
+ * if %LANDLOCK_ACCESS_FS_IOCTL is handled
+ */
+access_mask_t landlock_expand_handled_access_fs(const access_mask_t handled);
+
 #endif /* _SECURITY_LANDLOCK_FS_H */
diff --git a/security/landlock/limits.h b/security/landlock/limits.h
index 93c9c6f91556..75e822f878e0 100644
--- a/security/landlock/limits.h
+++ b/security/landlock/limits.h
@@ -18,7 +18,20 @@
 #define LANDLOCK_MAX_NUM_LAYERS		16
 #define LANDLOCK_MAX_NUM_RULES		U32_MAX
 
-#define LANDLOCK_LAST_ACCESS_FS		LANDLOCK_ACCESS_FS_TRUNCATE
+#define LANDLOCK_LAST_PUBLIC_ACCESS_FS	LANDLOCK_ACCESS_FS_IOCTL
+#define LANDLOCK_MASK_PUBLIC_ACCESS_FS	((LANDLOCK_LAST_PUBLIC_ACCESS_FS << 1) - 1)
+
+/*
+ * These are synthetic access rights, which are only used within the kernel, but
+ * not exposed to callers in userspace.  The mapping between these access rights
+ * and IOCTL commands is defined in the required_ioctl_access() helper function.
+ */
+#define LANDLOCK_ACCESS_FS_IOCTL_GROUP1	(LANDLOCK_LAST_PUBLIC_ACCESS_FS << 1)
+#define LANDLOCK_ACCESS_FS_IOCTL_GROUP2	(LANDLOCK_LAST_PUBLIC_ACCESS_FS << 2)
+#define LANDLOCK_ACCESS_FS_IOCTL_GROUP3	(LANDLOCK_LAST_PUBLIC_ACCESS_FS << 3)
+#define LANDLOCK_ACCESS_FS_IOCTL_GROUP4	(LANDLOCK_LAST_PUBLIC_ACCESS_FS << 4)
+
+#define LANDLOCK_LAST_ACCESS_FS		LANDLOCK_ACCESS_FS_IOCTL_GROUP4
 #define LANDLOCK_MASK_ACCESS_FS		((LANDLOCK_LAST_ACCESS_FS << 1) - 1)
 #define LANDLOCK_NUM_ACCESS_FS		__const_hweight64(LANDLOCK_MASK_ACCESS_FS)
 #define LANDLOCK_SHIFT_ACCESS_FS	0
diff --git a/security/landlock/net.c b/security/landlock/net.c
index aaa92c2b1f08..efa1b644a4af 100644
--- a/security/landlock/net.c
+++ b/security/landlock/net.c
@@ -196,5 +196,5 @@ static struct security_hook_list landlock_hooks[] __ro_after_init = {
 __init void landlock_add_net_hooks(void)
 {
 	security_add_hooks(landlock_hooks, ARRAY_SIZE(landlock_hooks),
-			   LANDLOCK_NAME);
+			   &landlock_lsmid);
 }
diff --git a/security/landlock/ptrace.c b/security/landlock/ptrace.c
index 8a06d6c492bf..2bfc533d36e4 100644
--- a/security/landlock/ptrace.c
+++ b/security/landlock/ptrace.c
@@ -116,5 +116,5 @@ static struct security_hook_list landlock_hooks[] __ro_after_init = {
 __init void landlock_add_ptrace_hooks(void)
 {
 	security_add_hooks(landlock_hooks, ARRAY_SIZE(landlock_hooks),
-			   LANDLOCK_NAME);
+			   &landlock_lsmid);
 }
diff --git a/security/landlock/ruleset.c b/security/landlock/ruleset.c
index ffedc99f2b68..23819457bc3f 100644
--- a/security/landlock/ruleset.c
+++ b/security/landlock/ruleset.c
@@ -723,11 +723,12 @@ landlock_init_layer_masks(const struct landlock_ruleset *const domain,
 	/* Saves all handled accesses per layer. */
 	for (layer_level = 0; layer_level < domain->num_layers; layer_level++) {
 		const unsigned long access_req = access_request;
+		const access_mask_t access_mask =
+			get_access_mask(domain, layer_level);
 		unsigned long access_bit;
 
 		for_each_set_bit(access_bit, &access_req, num_access) {
-			if (BIT_ULL(access_bit) &
-			    get_access_mask(domain, layer_level)) {
+			if (BIT_ULL(access_bit) & access_mask) {
 				(*layer_masks)[access_bit] |=
 					BIT_ULL(layer_level);
 				handled_accesses |= BIT_ULL(access_bit);
diff --git a/security/landlock/ruleset.h b/security/landlock/ruleset.h
index c7f1526784fd..5a28ea8e1c3d 100644
--- a/security/landlock/ruleset.h
+++ b/security/landlock/ruleset.h
@@ -30,7 +30,7 @@
 	LANDLOCK_ACCESS_FS_REFER)
 /* clang-format on */
 
-typedef u16 access_mask_t;
+typedef u32 access_mask_t;
 /* Makes sure all filesystem access rights can be stored. */
 static_assert(BITS_PER_TYPE(access_mask_t) >= LANDLOCK_NUM_ACCESS_FS);
 /* Makes sure all network access rights can be stored. */
diff --git a/security/landlock/setup.c b/security/landlock/setup.c
index 3e11d303542f..f6dd33143b7f 100644
--- a/security/landlock/setup.c
+++ b/security/landlock/setup.c
@@ -8,6 +8,7 @@
 
 #include <linux/init.h>
 #include <linux/lsm_hooks.h>
+#include <uapi/linux/lsm.h>
 
 #include "common.h"
 #include "cred.h"
@@ -25,6 +26,11 @@ struct lsm_blob_sizes landlock_blob_sizes __ro_after_init = {
 	.lbs_superblock = sizeof(struct landlock_superblock_security),
 };
 
+const struct lsm_id landlock_lsmid = {
+	.name = LANDLOCK_NAME,
+	.id = LSM_ID_LANDLOCK,
+};
+
 static int __init landlock_init(void)
 {
 	landlock_add_cred_hooks();
diff --git a/security/landlock/setup.h b/security/landlock/setup.h
index 1daffab1ab4b..c4252d46d49d 100644
--- a/security/landlock/setup.h
+++ b/security/landlock/setup.h
@@ -14,5 +14,6 @@
 extern bool landlock_initialized;
 
 extern struct lsm_blob_sizes landlock_blob_sizes;
+extern const struct lsm_id landlock_lsmid;
 
 #endif /* _SECURITY_LANDLOCK_SETUP_H */
diff --git a/security/landlock/syscalls.c b/security/landlock/syscalls.c
index 898358f57fa0..c196cac2a5fb 100644
--- a/security/landlock/syscalls.c
+++ b/security/landlock/syscalls.c
@@ -137,7 +137,7 @@ static const struct file_operations ruleset_fops = {
 	.write = fop_dummy_write,
 };
 
-#define LANDLOCK_ABI_VERSION 4
+#define LANDLOCK_ABI_VERSION 5
 
 /**
  * sys_landlock_create_ruleset - Create a new ruleset
@@ -192,8 +192,8 @@ SYSCALL_DEFINE3(landlock_create_ruleset,
 		return err;
 
 	/* Checks content (and 32-bits cast). */
-	if ((ruleset_attr.handled_access_fs | LANDLOCK_MASK_ACCESS_FS) !=
-	    LANDLOCK_MASK_ACCESS_FS)
+	if ((ruleset_attr.handled_access_fs | LANDLOCK_MASK_PUBLIC_ACCESS_FS) !=
+	    LANDLOCK_MASK_PUBLIC_ACCESS_FS)
 		return -EINVAL;
 
 	/* Checks network content (and 32-bits cast). */
@@ -201,6 +201,10 @@ SYSCALL_DEFINE3(landlock_create_ruleset,
 	    LANDLOCK_MASK_ACCESS_NET)
 		return -EINVAL;
 
+	/* Expands synthetic IOCTL groups. */
+	ruleset_attr.handled_access_fs = landlock_expand_handled_access_fs(
+		ruleset_attr.handled_access_fs);
+
 	/* Checks arguments and transforms to kernel struct. */
 	ruleset = landlock_create_ruleset(ruleset_attr.handled_access_fs,
 					  ruleset_attr.handled_access_net);
diff --git a/security/loadpin/loadpin.c b/security/loadpin/loadpin.c
index a9d40456a064..8e93cda130f1 100644
--- a/security/loadpin/loadpin.c
+++ b/security/loadpin/loadpin.c
@@ -20,6 +20,7 @@
 #include <linux/string_helpers.h>
 #include <linux/dm-verity-loadpin.h>
 #include <uapi/linux/loadpin.h>
+#include <uapi/linux/lsm.h>
 
 #define VERITY_DIGEST_FILE_HEADER "# LOADPIN_TRUSTED_VERITY_ROOT_DIGESTS"
 
@@ -208,6 +209,11 @@ static int loadpin_load_data(enum kernel_load_data_id id, bool contents)
 	return loadpin_check(NULL, (enum kernel_read_file_id) id);
 }
 
+static const struct lsm_id loadpin_lsmid = {
+	.name = "loadpin",
+	.id = LSM_ID_LOADPIN,
+};
+
 static struct security_hook_list loadpin_hooks[] __ro_after_init = {
 	LSM_HOOK_INIT(sb_free_security, loadpin_sb_free_security),
 	LSM_HOOK_INIT(kernel_read_file, loadpin_read_file),
@@ -259,7 +265,8 @@ static int __init loadpin_init(void)
 	if (!register_sysctl("kernel/loadpin", loadpin_sysctl_table))
 		pr_notice("sysctl registration failed!\n");
 #endif
-	security_add_hooks(loadpin_hooks, ARRAY_SIZE(loadpin_hooks), "loadpin");
+	security_add_hooks(loadpin_hooks, ARRAY_SIZE(loadpin_hooks),
+			   &loadpin_lsmid);
 
 	return 0;
 }
diff --git a/security/lockdown/lockdown.c b/security/lockdown/lockdown.c
index 68d19632aeb7..cd84d8ea1dfb 100644
--- a/security/lockdown/lockdown.c
+++ b/security/lockdown/lockdown.c
@@ -13,6 +13,7 @@
 #include <linux/security.h>
 #include <linux/export.h>
 #include <linux/lsm_hooks.h>
+#include <uapi/linux/lsm.h>
 
 static enum lockdown_reason kernel_locked_down;
 
@@ -75,6 +76,11 @@ static struct security_hook_list lockdown_hooks[] __ro_after_init = {
 	LSM_HOOK_INIT(locked_down, lockdown_is_locked_down),
 };
 
+const struct lsm_id lockdown_lsmid = {
+	.name = "lockdown",
+	.id = LSM_ID_LOCKDOWN,
+};
+
 static int __init lockdown_lsm_init(void)
 {
 #if defined(CONFIG_LOCK_DOWN_KERNEL_FORCE_INTEGRITY)
@@ -83,7 +89,7 @@ static int __init lockdown_lsm_init(void)
 	lock_kernel_down("Kernel configuration", LOCKDOWN_CONFIDENTIALITY_MAX);
 #endif
 	security_add_hooks(lockdown_hooks, ARRAY_SIZE(lockdown_hooks),
-			   "lockdown");
+			   &lockdown_lsmid);
 	return 0;
 }
 
diff --git a/security/lsm_syscalls.c b/security/lsm_syscalls.c
new file mode 100644
index 000000000000..5d391b1f7e69
--- /dev/null
+++ b/security/lsm_syscalls.c
@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * System calls implementing the Linux Security Module API.
+ *
+ *  Copyright (C) 2022 Casey Schaufler <casey@schaufler-ca.com>
+ *  Copyright (C) 2022 Intel Corporation
+ */
+
+#include <asm/current.h>
+#include <linux/compiler_types.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/security.h>
+#include <linux/stddef.h>
+#include <linux/syscalls.h>
+#include <linux/types.h>
+#include <linux/lsm_hooks.h>
+#include <uapi/linux/lsm.h>
+
+/**
+ * lsm_name_to_attr - map an LSM attribute name to its ID
+ * @name: name of the attribute
+ *
+ * Returns the LSM attribute value associated with @name, or 0 if
+ * there is no mapping.
+ */
+u64 lsm_name_to_attr(const char *name)
+{
+	if (!strcmp(name, "current"))
+		return LSM_ATTR_CURRENT;
+	if (!strcmp(name, "exec"))
+		return LSM_ATTR_EXEC;
+	if (!strcmp(name, "fscreate"))
+		return LSM_ATTR_FSCREATE;
+	if (!strcmp(name, "keycreate"))
+		return LSM_ATTR_KEYCREATE;
+	if (!strcmp(name, "prev"))
+		return LSM_ATTR_PREV;
+	if (!strcmp(name, "sockcreate"))
+		return LSM_ATTR_SOCKCREATE;
+	return LSM_ATTR_UNDEF;
+}
+
+/**
+ * sys_lsm_set_self_attr - Set current task's security module attribute
+ * @attr: which attribute to set
+ * @ctx: the LSM contexts
+ * @size: size of @ctx
+ * @flags: reserved for future use
+ *
+ * Sets the calling task's LSM context. On success this function
+ * returns 0. If the attribute specified cannot be set a negative
+ * value indicating the reason for the error is returned.
+ */
+SYSCALL_DEFINE4(lsm_set_self_attr, unsigned int, attr, struct lsm_ctx __user *,
+		ctx, size_t, size, u32, flags)
+{
+	return security_setselfattr(attr, ctx, size, flags);
+}
+
+/**
+ * sys_lsm_get_self_attr - Return current task's security module attributes
+ * @attr: which attribute to return
+ * @ctx: the user-space destination for the information, or NULL
+ * @size: pointer to the size of space available to receive the data
+ * @flags: special handling options. LSM_FLAG_SINGLE indicates that only
+ * attributes associated with the LSM identified in the passed @ctx be
+ * reported.
+ *
+ * Returns the calling task's LSM contexts. On success this
+ * function returns the number of @ctx array elements. This value
+ * may be zero if there are no LSM contexts assigned. If @size is
+ * insufficient to contain the return data -E2BIG is returned and
+ * @size is set to the minimum required size. In all other cases
+ * a negative value indicating the error is returned.
+ */
+SYSCALL_DEFINE4(lsm_get_self_attr, unsigned int, attr, struct lsm_ctx __user *,
+		ctx, size_t __user *, size, u32, flags)
+{
+	return security_getselfattr(attr, ctx, size, flags);
+}
+
+/**
+ * sys_lsm_list_modules - Return a list of the active security modules
+ * @ids: the LSM module ids
+ * @size: pointer to size of @ids, updated on return
+ * @flags: reserved for future use, must be zero
+ *
+ * Returns a list of the active LSM ids. On success this function
+ * returns the number of @ids array elements. This value may be zero
+ * if there are no LSMs active. If @size is insufficient to contain
+ * the return data -E2BIG is returned and @size is set to the minimum
+ * required size. In all other cases a negative value indicating the
+ * error is returned.
+ */
+SYSCALL_DEFINE3(lsm_list_modules, u64 __user *, ids, size_t __user *, size,
+		u32, flags)
+{
+	size_t total_size = lsm_active_cnt * sizeof(*ids);
+	size_t usize;
+	int i;
+
+	if (flags)
+		return -EINVAL;
+
+	if (get_user(usize, size))
+		return -EFAULT;
+
+	if (put_user(total_size, size) != 0)
+		return -EFAULT;
+
+	if (usize < total_size)
+		return -E2BIG;
+
+	for (i = 0; i < lsm_active_cnt; i++)
+		if (put_user(lsm_idlist[i]->id, ids++))
+			return -EFAULT;
+
+	return lsm_active_cnt;
+}
diff --git a/security/safesetid/lsm.c b/security/safesetid/lsm.c
index 5be5894aa0ea..1ba564f097f5 100644
--- a/security/safesetid/lsm.c
+++ b/security/safesetid/lsm.c
@@ -19,6 +19,7 @@
 #include <linux/ptrace.h>
 #include <linux/sched/task_stack.h>
 #include <linux/security.h>
+#include <uapi/linux/lsm.h>
 #include "lsm.h"
 
 /* Flag indicating whether initialization completed */
@@ -261,6 +262,11 @@ static int safesetid_task_fix_setgroups(struct cred *new, const struct cred *old
 	return 0;
 }
 
+static const struct lsm_id safesetid_lsmid = {
+	.name = "safesetid",
+	.id = LSM_ID_SAFESETID,
+};
+
 static struct security_hook_list safesetid_security_hooks[] = {
 	LSM_HOOK_INIT(task_fix_setuid, safesetid_task_fix_setuid),
 	LSM_HOOK_INIT(task_fix_setgid, safesetid_task_fix_setgid),
@@ -271,7 +277,8 @@ static struct security_hook_list safesetid_security_hooks[] = {
 static int __init safesetid_security_init(void)
 {
 	security_add_hooks(safesetid_security_hooks,
-			   ARRAY_SIZE(safesetid_security_hooks), "safesetid");
+			   ARRAY_SIZE(safesetid_security_hooks),
+			   &safesetid_lsmid);
 
 	/* Report that SafeSetID successfully initialized */
 	safesetid_initialized = 1;
diff --git a/security/security.c b/security/security.c
index dcb3e7014f9b..d7b15ea67c3f 100644
--- a/security/security.c
+++ b/security/security.c
@@ -34,6 +34,24 @@
 /* How many LSMs were built into the kernel? */
 #define LSM_COUNT (__end_lsm_info - __start_lsm_info)
 
+/*
+ * How many LSMs are built into the kernel as determined at
+ * build time. Used to determine fixed array sizes.
+ * The capability module is accounted for by CONFIG_SECURITY
+ */
+#define LSM_CONFIG_COUNT ( \
+	(IS_ENABLED(CONFIG_SECURITY) ? 1 : 0) + \
+	(IS_ENABLED(CONFIG_SECURITY_SELINUX) ? 1 : 0) + \
+	(IS_ENABLED(CONFIG_SECURITY_SMACK) ? 1 : 0) + \
+	(IS_ENABLED(CONFIG_SECURITY_TOMOYO) ? 1 : 0) + \
+	(IS_ENABLED(CONFIG_SECURITY_APPARMOR) ? 1 : 0) + \
+	(IS_ENABLED(CONFIG_SECURITY_YAMA) ? 1 : 0) + \
+	(IS_ENABLED(CONFIG_SECURITY_LOADPIN) ? 1 : 0) + \
+	(IS_ENABLED(CONFIG_SECURITY_SAFESETID) ? 1 : 0) + \
+	(IS_ENABLED(CONFIG_SECURITY_LOCKDOWN_LSM) ? 1 : 0) + \
+	(IS_ENABLED(CONFIG_BPF_LSM) ? 1 : 0) + \
+	(IS_ENABLED(CONFIG_SECURITY_LANDLOCK) ? 1 : 0))
+
 /*
  * These are descriptions of the reasons that can be passed to the
  * security_locked_down() LSM hook. Placing this array here allows
@@ -245,6 +263,12 @@ static void __init initialize_lsm(struct lsm_info *lsm)
 	}
 }
 
+/*
+ * Current index to use while initializing the lsm id list.
+ */
+u32 lsm_active_cnt __ro_after_init;
+const struct lsm_id *lsm_idlist[LSM_CONFIG_COUNT];
+
 /* Populate ordered LSMs list from comma-separated LSM name list. */
 static void __init ordered_lsm_parse(const char *order, const char *origin)
 {
@@ -513,17 +537,29 @@ static int lsm_append(const char *new, char **result)
  * security_add_hooks - Add a modules hooks to the hook lists.
  * @hooks: the hooks to add
  * @count: the number of hooks to add
- * @lsm: the name of the security module
+ * @lsmid: the identification information for the security module
  *
  * Each LSM has to register its hooks with the infrastructure.
  */
 void __init security_add_hooks(struct security_hook_list *hooks, int count,
-			       const char *lsm)
+			       const struct lsm_id *lsmid)
 {
 	int i;
 
+	/*
+	 * A security module may call security_add_hooks() more
+	 * than once during initialization, and LSM initialization
+	 * is serialized. Landlock is one such case.
+	 * Look at the previous entry, if there is one, for duplication.
+	 */
+	if (lsm_active_cnt == 0 || lsm_idlist[lsm_active_cnt - 1] != lsmid) {
+		if (lsm_active_cnt >= LSM_CONFIG_COUNT)
+			panic("%s Too many LSMs registered.\n", __func__);
+		lsm_idlist[lsm_active_cnt++] = lsmid;
+	}
+
 	for (i = 0; i < count; i++) {
-		hooks[i].lsm = lsm;
+		hooks[i].lsmid = lsmid;
 		hlist_add_tail_rcu(&hooks[i].list, hooks[i].head);
 	}
 
@@ -532,7 +568,7 @@ void __init security_add_hooks(struct security_hook_list *hooks, int count,
 	 * and fix this up afterwards.
 	 */
 	if (slab_is_available()) {
-		if (lsm_append(lsm, &lsm_names) < 0)
+		if (lsm_append(lsmid->name, &lsm_names) < 0)
 			panic("%s - Cannot get early memory.\n", __func__);
 	}
 }
@@ -734,6 +770,54 @@ static int lsm_superblock_alloc(struct super_block *sb)
 	return 0;
 }
 
+/**
+ * lsm_fill_user_ctx - Fill a user space lsm_ctx structure
+ * @uctx: a userspace LSM context to be filled
+ * @uctx_len: available uctx size (input), used uctx size (output)
+ * @val: the new LSM context value
+ * @val_len: the size of the new LSM context value
+ * @id: LSM id
+ * @flags: LSM defined flags
+ *
+ * Fill all of the fields in a userspace lsm_ctx structure.
+ *
+ * Returns 0 on success, -E2BIG if userspace buffer is not large enough,
+ * -EFAULT on a copyout error, -ENOMEM if memory can't be allocated.
+ */
+int lsm_fill_user_ctx(struct lsm_ctx __user *uctx, size_t *uctx_len,
+		      void *val, size_t val_len,
+		      u64 id, u64 flags)
+{
+	struct lsm_ctx *nctx = NULL;
+	size_t nctx_len;
+	int rc = 0;
+
+	nctx_len = ALIGN(struct_size(nctx, ctx, val_len), sizeof(void *));
+	if (nctx_len > *uctx_len) {
+		rc = -E2BIG;
+		goto out;
+	}
+
+	nctx = kzalloc(nctx_len, GFP_KERNEL);
+	if (nctx == NULL) {
+		rc = -ENOMEM;
+		goto out;
+	}
+	nctx->id = id;
+	nctx->flags = flags;
+	nctx->len = nctx_len;
+	nctx->ctx_len = val_len;
+	memcpy(nctx->ctx, val, val_len);
+
+	if (copy_to_user(uctx, nctx, nctx_len))
+		rc = -EFAULT;
+
+out:
+	kfree(nctx);
+	*uctx_len = nctx_len;
+	return rc;
+}
+
 /*
  * The default value of the LSM hook is defined in linux/lsm_hook_defs.h and
  * can be accessed with:
@@ -3800,10 +3884,158 @@ void security_d_instantiate(struct dentry *dentry, struct inode *inode)
 }
 EXPORT_SYMBOL(security_d_instantiate);
 
+/*
+ * Please keep this in sync with it's counterpart in security/lsm_syscalls.c
+ */
+
+/**
+ * security_getselfattr - Read an LSM attribute of the current process.
+ * @attr: which attribute to return
+ * @uctx: the user-space destination for the information, or NULL
+ * @size: pointer to the size of space available to receive the data
+ * @flags: special handling options. LSM_FLAG_SINGLE indicates that only
+ * attributes associated with the LSM identified in the passed @ctx be
+ * reported.
+ *
+ * A NULL value for @uctx can be used to get both the number of attributes
+ * and the size of the data.
+ *
+ * Returns the number of attributes found on success, negative value
+ * on error. @size is reset to the total size of the data.
+ * If @size is insufficient to contain the data -E2BIG is returned.
+ */
+int security_getselfattr(unsigned int attr, struct lsm_ctx __user *uctx,
+			 size_t __user *size, u32 flags)
+{
+	struct security_hook_list *hp;
+	struct lsm_ctx lctx = { .id = LSM_ID_UNDEF, };
+	u8 __user *base = (u8 __user *)uctx;
+	size_t total = 0;
+	size_t entrysize;
+	size_t left;
+	bool toobig = false;
+	bool single = false;
+	int count = 0;
+	int rc;
+
+	if (attr == LSM_ATTR_UNDEF)
+		return -EINVAL;
+	if (size == NULL)
+		return -EINVAL;
+	if (get_user(left, size))
+		return -EFAULT;
+
+	if (flags) {
+		/*
+		 * Only flag supported is LSM_FLAG_SINGLE
+		 */
+		if (flags != LSM_FLAG_SINGLE || !uctx)
+			return -EINVAL;
+		if (copy_from_user(&lctx, uctx, sizeof(lctx)))
+			return -EFAULT;
+		/*
+		 * If the LSM ID isn't specified it is an error.
+		 */
+		if (lctx.id == LSM_ID_UNDEF)
+			return -EINVAL;
+		single = true;
+	}
+
+	/*
+	 * In the usual case gather all the data from the LSMs.
+	 * In the single case only get the data from the LSM specified.
+	 */
+	hlist_for_each_entry(hp, &security_hook_heads.getselfattr, list) {
+		if (single && lctx.id != hp->lsmid->id)
+			continue;
+		entrysize = left;
+		if (base)
+			uctx = (struct lsm_ctx __user *)(base + total);
+		rc = hp->hook.getselfattr(attr, uctx, &entrysize, flags);
+		if (rc == -EOPNOTSUPP) {
+			rc = 0;
+			continue;
+		}
+		if (rc == -E2BIG) {
+			rc = 0;
+			left = 0;
+			toobig = true;
+		} else if (rc < 0)
+			return rc;
+		else
+			left -= entrysize;
+
+		total += entrysize;
+		count += rc;
+		if (single)
+			break;
+	}
+	if (put_user(total, size))
+		return -EFAULT;
+	if (toobig)
+		return -E2BIG;
+	if (count == 0)
+		return LSM_RET_DEFAULT(getselfattr);
+	return count;
+}
+
+/*
+ * Please keep this in sync with it's counterpart in security/lsm_syscalls.c
+ */
+
+/**
+ * security_setselfattr - Set an LSM attribute on the current process.
+ * @attr: which attribute to set
+ * @uctx: the user-space source for the information
+ * @size: the size of the data
+ * @flags: reserved for future use, must be 0
+ *
+ * Set an LSM attribute for the current process. The LSM, attribute
+ * and new value are included in @uctx.
+ *
+ * Returns 0 on success, -EINVAL if the input is inconsistent, -EFAULT
+ * if the user buffer is inaccessible, E2BIG if size is too big, or an
+ * LSM specific failure.
+ */
+int security_setselfattr(unsigned int attr, struct lsm_ctx __user *uctx,
+			 size_t size, u32 flags)
+{
+	struct security_hook_list *hp;
+	struct lsm_ctx *lctx;
+	int rc = LSM_RET_DEFAULT(setselfattr);
+
+	if (flags)
+		return -EINVAL;
+	if (size < sizeof(*lctx))
+		return -EINVAL;
+	if (size > PAGE_SIZE)
+		return -E2BIG;
+
+	lctx = memdup_user(uctx, size);
+	if (IS_ERR(lctx))
+		return PTR_ERR(lctx);
+
+	if (size < lctx->len || size < lctx->ctx_len + sizeof(*lctx) ||
+	    lctx->len < lctx->ctx_len + sizeof(*lctx)) {
+		rc = -EINVAL;
+		goto free_out;
+	}
+
+	hlist_for_each_entry(hp, &security_hook_heads.setselfattr, list)
+		if ((hp->lsmid->id) == lctx->id) {
+			rc = hp->hook.setselfattr(attr, lctx, size, flags);
+			break;
+		}
+
+free_out:
+	kfree(lctx);
+	return rc;
+}
+
 /**
  * security_getprocattr() - Read an attribute for a task
  * @p: the task
- * @lsm: LSM name
+ * @lsmid: LSM identification
  * @name: attribute name
  * @value: attribute value
  *
@@ -3811,13 +4043,13 @@ EXPORT_SYMBOL(security_d_instantiate);
  *
  * Return: Returns the length of @value on success, a negative value otherwise.
  */
-int security_getprocattr(struct task_struct *p, const char *lsm,
-			 const char *name, char **value)
+int security_getprocattr(struct task_struct *p, int lsmid, const char *name,
+			 char **value)
 {
 	struct security_hook_list *hp;
 
 	hlist_for_each_entry(hp, &security_hook_heads.getprocattr, list) {
-		if (lsm != NULL && strcmp(lsm, hp->lsm))
+		if (lsmid != 0 && lsmid != hp->lsmid->id)
 			continue;
 		return hp->hook.getprocattr(p, name, value);
 	}
@@ -3826,7 +4058,7 @@ int security_getprocattr(struct task_struct *p, const char *lsm,
 
 /**
  * security_setprocattr() - Set an attribute for a task
- * @lsm: LSM name
+ * @lsmid: LSM identification
  * @name: attribute name
  * @value: attribute value
  * @size: attribute value size
@@ -3836,13 +4068,12 @@ int security_getprocattr(struct task_struct *p, const char *lsm,
  *
  * Return: Returns bytes written on success, a negative value otherwise.
  */
-int security_setprocattr(const char *lsm, const char *name, void *value,
-			 size_t size)
+int security_setprocattr(int lsmid, const char *name, void *value, size_t size)
 {
 	struct security_hook_list *hp;
 
 	hlist_for_each_entry(hp, &security_hook_heads.setprocattr, list) {
-		if (lsm != NULL && strcmp(lsm, hp->lsm))
+		if (lsmid != 0 && lsmid != hp->lsmid->id)
 			continue;
 		return hp->hook.setprocattr(name, value, size);
 	}
diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c
index feda711c6b7b..091fa0183625 100644
--- a/security/selinux/hooks.c
+++ b/security/selinux/hooks.c
@@ -92,6 +92,7 @@
 #include <linux/fsnotify.h>
 #include <linux/fanotify.h>
 #include <linux/io_uring.h>
+#include <uapi/linux/lsm.h>
 
 #include "avc.h"
 #include "objsec.h"
@@ -2315,6 +2316,19 @@ static int selinux_bprm_creds_for_exec(struct linux_binprm *bprm)
 	new_tsec->keycreate_sid = 0;
 	new_tsec->sockcreate_sid = 0;
 
+	/*
+	 * Before policy is loaded, label any task outside kernel space
+	 * as SECINITSID_INIT, so that any userspace tasks surviving from
+	 * early boot end up with a label different from SECINITSID_KERNEL
+	 * (if the policy chooses to set SECINITSID_INIT != SECINITSID_KERNEL).
+	 */
+	if (!selinux_initialized()) {
+		new_tsec->sid = SECINITSID_INIT;
+		/* also clear the exec_sid just in case */
+		new_tsec->exec_sid = 0;
+		return 0;
+	}
+
 	if (old_tsec->exec_sid) {
 		new_tsec->sid = old_tsec->exec_sid;
 		/* Reset exec SID on execve. */
@@ -4553,6 +4567,21 @@ static int sock_has_perm(struct sock *sk, u32 perms)
 	if (sksec->sid == SECINITSID_KERNEL)
 		return 0;
 
+	/*
+	 * Before POLICYDB_CAP_USERSPACE_INITIAL_CONTEXT, sockets that
+	 * inherited the kernel context from early boot used to be skipped
+	 * here, so preserve that behavior unless the capability is set.
+	 *
+	 * By setting the capability the policy signals that it is ready
+	 * for this quirk to be fixed. Note that sockets created by a kernel
+	 * thread or a usermode helper executed without a transition will
+	 * still be skipped in this check regardless of the policycap
+	 * setting.
+	 */
+	if (!selinux_policycap_userspace_initial_context() &&
+	    sksec->sid == SECINITSID_INIT)
+		return 0;
+
 	ad_net_init_from_sk(&ad, &net, sk);
 
 	return avc_has_perm(current_sid(), sksec->sid, sksec->sclass, perms,
@@ -6284,8 +6313,8 @@ static void selinux_d_instantiate(struct dentry *dentry, struct inode *inode)
 		inode_doinit_with_dentry(inode, dentry);
 }
 
-static int selinux_getprocattr(struct task_struct *p,
-			       const char *name, char **value)
+static int selinux_lsm_getattr(unsigned int attr, struct task_struct *p,
+			       char **value)
 {
 	const struct task_security_struct *__tsec;
 	u32 sid;
@@ -6302,20 +6331,27 @@ static int selinux_getprocattr(struct task_struct *p,
 			goto bad;
 	}
 
-	if (!strcmp(name, "current"))
+	switch (attr) {
+	case LSM_ATTR_CURRENT:
 		sid = __tsec->sid;
-	else if (!strcmp(name, "prev"))
+		break;
+	case LSM_ATTR_PREV:
 		sid = __tsec->osid;
-	else if (!strcmp(name, "exec"))
+		break;
+	case LSM_ATTR_EXEC:
 		sid = __tsec->exec_sid;
-	else if (!strcmp(name, "fscreate"))
+		break;
+	case LSM_ATTR_FSCREATE:
 		sid = __tsec->create_sid;
-	else if (!strcmp(name, "keycreate"))
+		break;
+	case LSM_ATTR_KEYCREATE:
 		sid = __tsec->keycreate_sid;
-	else if (!strcmp(name, "sockcreate"))
+		break;
+	case LSM_ATTR_SOCKCREATE:
 		sid = __tsec->sockcreate_sid;
-	else {
-		error = -EINVAL;
+		break;
+	default:
+		error = -EOPNOTSUPP;
 		goto bad;
 	}
 	rcu_read_unlock();
@@ -6333,7 +6369,7 @@ static int selinux_getprocattr(struct task_struct *p,
 	return error;
 }
 
-static int selinux_setprocattr(const char *name, void *value, size_t size)
+static int selinux_lsm_setattr(u64 attr, void *value, size_t size)
 {
 	struct task_security_struct *tsec;
 	struct cred *new;
@@ -6344,23 +6380,31 @@ static int selinux_setprocattr(const char *name, void *value, size_t size)
 	/*
 	 * Basic control over ability to set these attributes at all.
 	 */
-	if (!strcmp(name, "exec"))
+	switch (attr) {
+	case LSM_ATTR_EXEC:
 		error = avc_has_perm(mysid, mysid, SECCLASS_PROCESS,
 				     PROCESS__SETEXEC, NULL);
-	else if (!strcmp(name, "fscreate"))
+		break;
+	case LSM_ATTR_FSCREATE:
 		error = avc_has_perm(mysid, mysid, SECCLASS_PROCESS,
 				     PROCESS__SETFSCREATE, NULL);
-	else if (!strcmp(name, "keycreate"))
+		break;
+	case LSM_ATTR_KEYCREATE:
 		error = avc_has_perm(mysid, mysid, SECCLASS_PROCESS,
 				     PROCESS__SETKEYCREATE, NULL);
-	else if (!strcmp(name, "sockcreate"))
+		break;
+	case LSM_ATTR_SOCKCREATE:
 		error = avc_has_perm(mysid, mysid, SECCLASS_PROCESS,
 				     PROCESS__SETSOCKCREATE, NULL);
-	else if (!strcmp(name, "current"))
+		break;
+	case LSM_ATTR_CURRENT:
 		error = avc_has_perm(mysid, mysid, SECCLASS_PROCESS,
 				     PROCESS__SETCURRENT, NULL);
-	else
-		error = -EINVAL;
+		break;
+	default:
+		error = -EOPNOTSUPP;
+		break;
+	}
 	if (error)
 		return error;
 
@@ -6372,13 +6416,14 @@ static int selinux_setprocattr(const char *name, void *value, size_t size)
 		}
 		error = security_context_to_sid(value, size,
 						&sid, GFP_KERNEL);
-		if (error == -EINVAL && !strcmp(name, "fscreate")) {
+		if (error == -EINVAL && attr == LSM_ATTR_FSCREATE) {
 			if (!has_cap_mac_admin(true)) {
 				struct audit_buffer *ab;
 				size_t audit_size;
 
-				/* We strip a nul only if it is at the end, otherwise the
-				 * context contains a nul and we should audit that */
+				/* We strip a nul only if it is at the end,
+				 * otherwise the context contains a nul and
+				 * we should audit that */
 				if (str[size - 1] == '\0')
 					audit_size = size - 1;
 				else
@@ -6389,7 +6434,8 @@ static int selinux_setprocattr(const char *name, void *value, size_t size)
 				if (!ab)
 					return error;
 				audit_log_format(ab, "op=fscreate invalid_context=");
-				audit_log_n_untrustedstring(ab, value, audit_size);
+				audit_log_n_untrustedstring(ab, value,
+							    audit_size);
 				audit_log_end(ab);
 
 				return error;
@@ -6412,11 +6458,11 @@ static int selinux_setprocattr(const char *name, void *value, size_t size)
 	   checks and may_create for the file creation checks. The
 	   operation will then fail if the context is not permitted. */
 	tsec = selinux_cred(new);
-	if (!strcmp(name, "exec")) {
+	if (attr == LSM_ATTR_EXEC) {
 		tsec->exec_sid = sid;
-	} else if (!strcmp(name, "fscreate")) {
+	} else if (attr == LSM_ATTR_FSCREATE) {
 		tsec->create_sid = sid;
-	} else if (!strcmp(name, "keycreate")) {
+	} else if (attr == LSM_ATTR_KEYCREATE) {
 		if (sid) {
 			error = avc_has_perm(mysid, sid,
 					     SECCLASS_KEY, KEY__CREATE, NULL);
@@ -6424,9 +6470,9 @@ static int selinux_setprocattr(const char *name, void *value, size_t size)
 				goto abort_change;
 		}
 		tsec->keycreate_sid = sid;
-	} else if (!strcmp(name, "sockcreate")) {
+	} else if (attr == LSM_ATTR_SOCKCREATE) {
 		tsec->sockcreate_sid = sid;
-	} else if (!strcmp(name, "current")) {
+	} else if (attr == LSM_ATTR_CURRENT) {
 		error = -EINVAL;
 		if (sid == 0)
 			goto abort_change;
@@ -6468,6 +6514,69 @@ static int selinux_setprocattr(const char *name, void *value, size_t size)
 	return error;
 }
 
+/**
+ * selinux_getselfattr - Get SELinux current task attributes
+ * @attr: the requested attribute
+ * @ctx: buffer to receive the result
+ * @size: buffer size (input), buffer size used (output)
+ * @flags: unused
+ *
+ * Fill the passed user space @ctx with the details of the requested
+ * attribute.
+ *
+ * Returns the number of attributes on success, an error code otherwise.
+ * There will only ever be one attribute.
+ */
+static int selinux_getselfattr(unsigned int attr, struct lsm_ctx __user *ctx,
+			       size_t *size, u32 flags)
+{
+	int rc;
+	char *val;
+	int val_len;
+
+	val_len = selinux_lsm_getattr(attr, current, &val);
+	if (val_len < 0)
+		return val_len;
+	rc = lsm_fill_user_ctx(ctx, size, val, val_len, LSM_ID_SELINUX, 0);
+	kfree(val);
+	return (!rc ? 1 : rc);
+}
+
+static int selinux_setselfattr(unsigned int attr, struct lsm_ctx *ctx,
+			       size_t size, u32 flags)
+{
+	int rc;
+
+	rc = selinux_lsm_setattr(attr, ctx->ctx, ctx->ctx_len);
+	if (rc > 0)
+		return 0;
+	return rc;
+}
+
+static int selinux_getprocattr(struct task_struct *p,
+			       const char *name, char **value)
+{
+	unsigned int attr = lsm_name_to_attr(name);
+	int rc;
+
+	if (attr) {
+		rc = selinux_lsm_getattr(attr, p, value);
+		if (rc != -EOPNOTSUPP)
+			return rc;
+	}
+
+	return -EINVAL;
+}
+
+static int selinux_setprocattr(const char *name, void *value, size_t size)
+{
+	int attr = lsm_name_to_attr(name);
+
+	if (attr)
+		return selinux_lsm_setattr(attr, value, size);
+	return -EINVAL;
+}
+
 static int selinux_ismaclabel(const char *name)
 {
 	return (strcmp(name, XATTR_SELINUX_SUFFIX) == 0);
@@ -6950,6 +7059,11 @@ static int selinux_uring_cmd(struct io_uring_cmd *ioucmd)
 }
 #endif /* CONFIG_IO_URING */
 
+static const struct lsm_id selinux_lsmid = {
+	.name = "selinux",
+	.id = LSM_ID_SELINUX,
+};
+
 /*
  * IMPORTANT NOTE: When adding new hooks, please be careful to keep this order:
  * 1. any hooks that don't belong to (2.) or (3.) below,
@@ -7091,6 +7205,8 @@ static struct security_hook_list selinux_hooks[] __ro_after_init = {
 
 	LSM_HOOK_INIT(d_instantiate, selinux_d_instantiate),
 
+	LSM_HOOK_INIT(getselfattr, selinux_getselfattr),
+	LSM_HOOK_INIT(setselfattr, selinux_setselfattr),
 	LSM_HOOK_INIT(getprocattr, selinux_getprocattr),
 	LSM_HOOK_INIT(setprocattr, selinux_setprocattr),
 
@@ -7270,7 +7386,8 @@ static __init int selinux_init(void)
 
 	hashtab_cache_init();
 
-	security_add_hooks(selinux_hooks, ARRAY_SIZE(selinux_hooks), "selinux");
+	security_add_hooks(selinux_hooks, ARRAY_SIZE(selinux_hooks),
+			   &selinux_lsmid);
 
 	if (avc_add_callback(selinux_netcache_avc_callback, AVC_CALLBACK_RESET))
 		panic("SELinux: Unable to register AVC netcache callback\n");
diff --git a/security/selinux/include/initial_sid_to_string.h b/security/selinux/include/initial_sid_to_string.h
index ecc6e74fa09b..5e5f0993dac2 100644
--- a/security/selinux/include/initial_sid_to_string.h
+++ b/security/selinux/include/initial_sid_to_string.h
@@ -10,7 +10,7 @@ static const char *const initial_sid_to_string[] = {
 	NULL,
 	"file",
 	NULL,
-	NULL,
+	"init",
 	"any_socket",
 	"port",
 	"netif",
diff --git a/security/selinux/include/policycap.h b/security/selinux/include/policycap.h
index f35d3458e71d..c7373e6effe5 100644
--- a/security/selinux/include/policycap.h
+++ b/security/selinux/include/policycap.h
@@ -12,6 +12,7 @@ enum {
 	POLICYDB_CAP_NNP_NOSUID_TRANSITION,
 	POLICYDB_CAP_GENFS_SECLABEL_SYMLINKS,
 	POLICYDB_CAP_IOCTL_SKIP_CLOEXEC,
+	POLICYDB_CAP_USERSPACE_INITIAL_CONTEXT,
 	__POLICYDB_CAP_MAX
 };
 #define POLICYDB_CAP_MAX (__POLICYDB_CAP_MAX - 1)
diff --git a/security/selinux/include/policycap_names.h b/security/selinux/include/policycap_names.h
index 49bbe120d173..28e4c9ee2399 100644
--- a/security/selinux/include/policycap_names.h
+++ b/security/selinux/include/policycap_names.h
@@ -14,6 +14,7 @@ const char *const selinux_policycap_names[__POLICYDB_CAP_MAX] = {
 	"nnp_nosuid_transition",
 	"genfs_seclabel_symlinks",
 	"ioctl_skip_cloexec",
+	"userspace_initial_context",
 };
 
 #endif /* _SELINUX_POLICYCAP_NAMES_H_ */
diff --git a/security/selinux/include/security.h b/security/selinux/include/security.h
index a9de89af8fdc..074d439fe9ad 100644
--- a/security/selinux/include/security.h
+++ b/security/selinux/include/security.h
@@ -189,6 +189,12 @@ static inline bool selinux_policycap_ioctl_skip_cloexec(void)
 		selinux_state.policycap[POLICYDB_CAP_IOCTL_SKIP_CLOEXEC]);
 }
 
+static inline bool selinux_policycap_userspace_initial_context(void)
+{
+	return READ_ONCE(
+		selinux_state.policycap[POLICYDB_CAP_USERSPACE_INITIAL_CONTEXT]);
+}
+
 struct selinux_policy_convert_data;
 
 struct selinux_load_state {
diff --git a/security/selinux/selinuxfs.c b/security/selinux/selinuxfs.c
index 6c596ae7fef9..0619a1cbbfbe 100644
--- a/security/selinux/selinuxfs.c
+++ b/security/selinux/selinuxfs.c
@@ -336,12 +336,9 @@ static struct dentry *sel_make_dir(struct dentry *dir, const char *name,
 			unsigned long *ino);
 
 /* declaration for sel_make_policy_nodes */
-static struct dentry *sel_make_disconnected_dir(struct super_block *sb,
+static struct dentry *sel_make_swapover_dir(struct super_block *sb,
 						unsigned long *ino);
 
-/* declaration for sel_make_policy_nodes */
-static void sel_remove_entries(struct dentry *de);
-
 static ssize_t sel_read_mls(struct file *filp, char __user *buf,
 				size_t count, loff_t *ppos)
 {
@@ -508,13 +505,13 @@ static int sel_make_policy_nodes(struct selinux_fs_info *fsi,
 				struct selinux_policy *newpolicy)
 {
 	int ret = 0;
-	struct dentry *tmp_parent, *tmp_bool_dir, *tmp_class_dir, *old_dentry;
-	unsigned int tmp_bool_num, old_bool_num;
-	char **tmp_bool_names, **old_bool_names;
-	int *tmp_bool_values, *old_bool_values;
+	struct dentry *tmp_parent, *tmp_bool_dir, *tmp_class_dir;
+	unsigned int bool_num = 0;
+	char **bool_names = NULL;
+	int *bool_values = NULL;
 	unsigned long tmp_ino = fsi->last_ino; /* Don't increment last_ino in this function */
 
-	tmp_parent = sel_make_disconnected_dir(fsi->sb, &tmp_ino);
+	tmp_parent = sel_make_swapover_dir(fsi->sb, &tmp_ino);
 	if (IS_ERR(tmp_parent))
 		return PTR_ERR(tmp_parent);
 
@@ -532,8 +529,8 @@ static int sel_make_policy_nodes(struct selinux_fs_info *fsi,
 		goto out;
 	}
 
-	ret = sel_make_bools(newpolicy, tmp_bool_dir, &tmp_bool_num,
-			     &tmp_bool_names, &tmp_bool_values);
+	ret = sel_make_bools(newpolicy, tmp_bool_dir, &bool_num,
+			     &bool_names, &bool_values);
 	if (ret)
 		goto out;
 
@@ -542,38 +539,30 @@ static int sel_make_policy_nodes(struct selinux_fs_info *fsi,
 	if (ret)
 		goto out;
 
+	lock_rename(tmp_parent, fsi->sb->s_root);
+
 	/* booleans */
-	old_dentry = fsi->bool_dir;
-	lock_rename(tmp_bool_dir, old_dentry);
 	d_exchange(tmp_bool_dir, fsi->bool_dir);
 
-	old_bool_num = fsi->bool_num;
-	old_bool_names = fsi->bool_pending_names;
-	old_bool_values = fsi->bool_pending_values;
-
-	fsi->bool_num = tmp_bool_num;
-	fsi->bool_pending_names = tmp_bool_names;
-	fsi->bool_pending_values = tmp_bool_values;
-
-	sel_remove_old_bool_data(old_bool_num, old_bool_names, old_bool_values);
+	swap(fsi->bool_num, bool_num);
+	swap(fsi->bool_pending_names, bool_names);
+	swap(fsi->bool_pending_values, bool_values);
 
 	fsi->bool_dir = tmp_bool_dir;
-	unlock_rename(tmp_bool_dir, old_dentry);
 
 	/* classes */
-	old_dentry = fsi->class_dir;
-	lock_rename(tmp_class_dir, old_dentry);
 	d_exchange(tmp_class_dir, fsi->class_dir);
 	fsi->class_dir = tmp_class_dir;
-	unlock_rename(tmp_class_dir, old_dentry);
+
+	unlock_rename(tmp_parent, fsi->sb->s_root);
 
 out:
+	sel_remove_old_bool_data(bool_num, bool_names, bool_values);
 	/* Since the other temporary dirs are children of tmp_parent
 	 * this will handle all the cleanup in the case of a failure before
 	 * the swapover
 	 */
-	sel_remove_entries(tmp_parent);
-	dput(tmp_parent); /* d_genocide() only handles the children */
+	simple_recursive_removal(tmp_parent, NULL);
 
 	return ret;
 }
@@ -1351,54 +1340,48 @@ static const struct file_operations sel_commit_bools_ops = {
 	.llseek		= generic_file_llseek,
 };
 
-static void sel_remove_entries(struct dentry *de)
-{
-	d_genocide(de);
-	shrink_dcache_parent(de);
-}
-
 static int sel_make_bools(struct selinux_policy *newpolicy, struct dentry *bool_dir,
 			  unsigned int *bool_num, char ***bool_pending_names,
 			  int **bool_pending_values)
 {
 	int ret;
-	ssize_t len;
-	struct dentry *dentry = NULL;
-	struct inode *inode = NULL;
-	struct inode_security_struct *isec;
-	char **names = NULL, *page;
+	char **names, *page;
 	u32 i, num;
-	int *values = NULL;
-	u32 sid;
 
-	ret = -ENOMEM;
 	page = (char *)get_zeroed_page(GFP_KERNEL);
 	if (!page)
-		goto out;
+		return -ENOMEM;
 
-	ret = security_get_bools(newpolicy, &num, &names, &values);
+	ret = security_get_bools(newpolicy, &num, &names, bool_pending_values);
 	if (ret)
 		goto out;
 
+	*bool_num = num;
+	*bool_pending_names = names;
+
 	for (i = 0; i < num; i++) {
-		ret = -ENOMEM;
+		struct dentry *dentry;
+		struct inode *inode;
+		struct inode_security_struct *isec;
+		ssize_t len;
+		u32 sid;
+
+		len = snprintf(page, PAGE_SIZE, "/%s/%s", BOOL_DIR_NAME, names[i]);
+		if (len >= PAGE_SIZE) {
+			ret = -ENAMETOOLONG;
+			break;
+		}
 		dentry = d_alloc_name(bool_dir, names[i]);
-		if (!dentry)
-			goto out;
+		if (!dentry) {
+			ret = -ENOMEM;
+			break;
+		}
 
-		ret = -ENOMEM;
 		inode = sel_make_inode(bool_dir->d_sb, S_IFREG | S_IRUGO | S_IWUSR);
 		if (!inode) {
 			dput(dentry);
-			goto out;
-		}
-
-		ret = -ENAMETOOLONG;
-		len = snprintf(page, PAGE_SIZE, "/%s/%s", BOOL_DIR_NAME, names[i]);
-		if (len >= PAGE_SIZE) {
-			dput(dentry);
-			iput(inode);
-			goto out;
+			ret = -ENOMEM;
+			break;
 		}
 
 		isec = selinux_inode(inode);
@@ -1416,23 +1399,8 @@ static int sel_make_bools(struct selinux_policy *newpolicy, struct dentry *bool_
 		inode->i_ino = i|SEL_BOOL_INO_OFFSET;
 		d_add(dentry, inode);
 	}
-	*bool_num = num;
-	*bool_pending_names = names;
-	*bool_pending_values = values;
-
-	free_page((unsigned long)page);
-	return 0;
 out:
 	free_page((unsigned long)page);
-
-	if (names) {
-		for (i = 0; i < num; i++)
-			kfree(names[i]);
-		kfree(names);
-	}
-	kfree(values);
-	sel_remove_entries(bool_dir);
-
 	return ret;
 }
 
@@ -1961,20 +1929,40 @@ static struct dentry *sel_make_dir(struct dentry *dir, const char *name,
 	return dentry;
 }
 
-static struct dentry *sel_make_disconnected_dir(struct super_block *sb,
+static int reject_all(struct mnt_idmap *idmap, struct inode *inode, int mask)
+{
+	return -EPERM;	// no access for anyone, root or no root.
+}
+
+static const struct inode_operations swapover_dir_inode_operations = {
+	.lookup		= simple_lookup,
+	.permission	= reject_all,
+};
+
+static struct dentry *sel_make_swapover_dir(struct super_block *sb,
 						unsigned long *ino)
 {
-	struct inode *inode = sel_make_inode(sb, S_IFDIR | S_IRUGO | S_IXUGO);
+	struct dentry *dentry = d_alloc_name(sb->s_root, ".swapover");
+	struct inode *inode;
 
-	if (!inode)
+	if (!dentry)
 		return ERR_PTR(-ENOMEM);
 
-	inode->i_op = &simple_dir_inode_operations;
-	inode->i_fop = &simple_dir_operations;
+	inode = sel_make_inode(sb, S_IFDIR);
+	if (!inode) {
+		dput(dentry);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	inode->i_op = &swapover_dir_inode_operations;
 	inode->i_ino = ++(*ino);
 	/* directory inodes start off with i_nlink == 2 (for "." entry) */
 	inc_nlink(inode);
-	return d_obtain_alias(inode);
+	inode_lock(sb->s_root->d_inode);
+	d_add(dentry, inode);
+	inc_nlink(sb->s_root->d_inode);
+	inode_unlock(sb->s_root->d_inode);
+	return dentry;
 }
 
 #define NULL_FILE_NAME "null"
diff --git a/security/selinux/ss/avtab.c b/security/selinux/ss/avtab.c
index 8751a602ead2..697eb4352439 100644
--- a/security/selinux/ss/avtab.c
+++ b/security/selinux/ss/avtab.c
@@ -96,12 +96,34 @@ avtab_insert_node(struct avtab *h, struct avtab_node **dst,
 	return newnode;
 }
 
+static int avtab_node_cmp(const struct avtab_key *key1,
+			  const struct avtab_key *key2)
+{
+	u16 specified = key1->specified & ~(AVTAB_ENABLED|AVTAB_ENABLED_OLD);
+
+	if (key1->source_type == key2->source_type &&
+	    key1->target_type == key2->target_type &&
+	    key1->target_class == key2->target_class &&
+	    (specified & key2->specified))
+		return 0;
+	if (key1->source_type < key2->source_type)
+		return -1;
+	if (key1->source_type == key2->source_type &&
+	    key1->target_type < key2->target_type)
+		return -1;
+	if (key1->source_type == key2->source_type &&
+	    key1->target_type == key2->target_type &&
+	    key1->target_class < key2->target_class)
+		return -1;
+	return 1;
+}
+
 static int avtab_insert(struct avtab *h, const struct avtab_key *key,
 			const struct avtab_datum *datum)
 {
 	u32 hvalue;
 	struct avtab_node *prev, *cur, *newnode;
-	u16 specified = key->specified & ~(AVTAB_ENABLED|AVTAB_ENABLED_OLD);
+	int cmp;
 
 	if (!h || !h->nslot || h->nel == U32_MAX)
 		return -EINVAL;
@@ -110,23 +132,11 @@ static int avtab_insert(struct avtab *h, const struct avtab_key *key,
 	for (prev = NULL, cur = h->htable[hvalue];
 	     cur;
 	     prev = cur, cur = cur->next) {
-		if (key->source_type == cur->key.source_type &&
-		    key->target_type == cur->key.target_type &&
-		    key->target_class == cur->key.target_class &&
-		    (specified & cur->key.specified)) {
-			/* extended perms may not be unique */
-			if (specified & AVTAB_XPERMS)
-				break;
+		cmp = avtab_node_cmp(key, &cur->key);
+		/* extended perms may not be unique */
+		if (cmp == 0 && !(key->specified & AVTAB_XPERMS))
 			return -EEXIST;
-		}
-		if (key->source_type < cur->key.source_type)
-			break;
-		if (key->source_type == cur->key.source_type &&
-		    key->target_type < cur->key.target_type)
-			break;
-		if (key->source_type == cur->key.source_type &&
-		    key->target_type == cur->key.target_type &&
-		    key->target_class < cur->key.target_class)
+		if (cmp <= 0)
 			break;
 	}
 
@@ -148,7 +158,7 @@ struct avtab_node *avtab_insert_nonunique(struct avtab *h,
 {
 	u32 hvalue;
 	struct avtab_node *prev, *cur;
-	u16 specified = key->specified & ~(AVTAB_ENABLED|AVTAB_ENABLED_OLD);
+	int cmp;
 
 	if (!h || !h->nslot || h->nel == U32_MAX)
 		return NULL;
@@ -156,19 +166,8 @@ struct avtab_node *avtab_insert_nonunique(struct avtab *h,
 	for (prev = NULL, cur = h->htable[hvalue];
 	     cur;
 	     prev = cur, cur = cur->next) {
-		if (key->source_type == cur->key.source_type &&
-		    key->target_type == cur->key.target_type &&
-		    key->target_class == cur->key.target_class &&
-		    (specified & cur->key.specified))
-			break;
-		if (key->source_type < cur->key.source_type)
-			break;
-		if (key->source_type == cur->key.source_type &&
-		    key->target_type < cur->key.target_type)
-			break;
-		if (key->source_type == cur->key.source_type &&
-		    key->target_type == cur->key.target_type &&
-		    key->target_class < cur->key.target_class)
+		cmp = avtab_node_cmp(key, &cur->key);
+		if (cmp <= 0)
 			break;
 	}
 	return avtab_insert_node(h, prev ? &prev->next : &h->htable[hvalue],
@@ -183,7 +182,7 @@ struct avtab_node *avtab_search_node(struct avtab *h,
 {
 	u32 hvalue;
 	struct avtab_node *cur;
-	u16 specified = key->specified & ~(AVTAB_ENABLED|AVTAB_ENABLED_OLD);
+	int cmp;
 
 	if (!h || !h->nslot)
 		return NULL;
@@ -191,20 +190,10 @@ struct avtab_node *avtab_search_node(struct avtab *h,
 	hvalue = avtab_hash(key, h->mask);
 	for (cur = h->htable[hvalue]; cur;
 	     cur = cur->next) {
-		if (key->source_type == cur->key.source_type &&
-		    key->target_type == cur->key.target_type &&
-		    key->target_class == cur->key.target_class &&
-		    (specified & cur->key.specified))
+		cmp = avtab_node_cmp(key, &cur->key);
+		if (cmp == 0)
 			return cur;
-
-		if (key->source_type < cur->key.source_type)
-			break;
-		if (key->source_type == cur->key.source_type &&
-		    key->target_type < cur->key.target_type)
-			break;
-		if (key->source_type == cur->key.source_type &&
-		    key->target_type == cur->key.target_type &&
-		    key->target_class < cur->key.target_class)
+		if (cmp < 0)
 			break;
 	}
 	return NULL;
@@ -213,27 +202,19 @@ struct avtab_node *avtab_search_node(struct avtab *h,
 struct avtab_node*
 avtab_search_node_next(struct avtab_node *node, u16 specified)
 {
+	struct avtab_key tmp_key;
 	struct avtab_node *cur;
+	int cmp;
 
 	if (!node)
 		return NULL;
-
-	specified &= ~(AVTAB_ENABLED|AVTAB_ENABLED_OLD);
+	tmp_key = node->key;
+	tmp_key.specified = specified;
 	for (cur = node->next; cur; cur = cur->next) {
-		if (node->key.source_type == cur->key.source_type &&
-		    node->key.target_type == cur->key.target_type &&
-		    node->key.target_class == cur->key.target_class &&
-		    (specified & cur->key.specified))
+		cmp = avtab_node_cmp(&tmp_key, &cur->key);
+		if (cmp == 0)
 			return cur;
-
-		if (node->key.source_type < cur->key.source_type)
-			break;
-		if (node->key.source_type == cur->key.source_type &&
-		    node->key.target_type < cur->key.target_type)
-			break;
-		if (node->key.source_type == cur->key.source_type &&
-		    node->key.target_type == cur->key.target_type &&
-		    node->key.target_class < cur->key.target_class)
+		if (cmp < 0)
 			break;
 	}
 	return NULL;
diff --git a/security/selinux/ss/policydb.c b/security/selinux/ss/policydb.c
index 595a435ea9c8..3b19ad28c922 100644
--- a/security/selinux/ss/policydb.c
+++ b/security/selinux/ss/policydb.c
@@ -409,16 +409,9 @@ static int roles_init(struct policydb *p)
 static u32 filenametr_hash(const void *k)
 {
 	const struct filename_trans_key *ft = k;
-	unsigned long hash;
-	unsigned int byte_num;
-	unsigned char focus;
+	unsigned long salt = ft->ttype ^ ft->tclass;
 
-	hash = ft->ttype ^ ft->tclass;
-
-	byte_num = 0;
-	while ((focus = ft->name[byte_num++]))
-		hash = partial_name_hash(focus, hash);
-	return hash;
+	return full_name_hash((void *)salt, ft->name, strlen(ft->name));
 }
 
 static int filenametr_cmp(const void *k1, const void *k2)
@@ -864,6 +857,8 @@ void policydb_destroy(struct policydb *p)
 int policydb_load_isids(struct policydb *p, struct sidtab *s)
 {
 	struct ocontext *head, *c;
+	bool isid_init_supported = ebitmap_get_bit(&p->policycaps,
+						   POLICYDB_CAP_USERSPACE_INITIAL_CONTEXT);
 	int rc;
 
 	rc = sidtab_init(s);
@@ -887,6 +882,13 @@ int policydb_load_isids(struct policydb *p, struct sidtab *s)
 		if (!name)
 			continue;
 
+		/*
+		 * Also ignore SECINITSID_INIT if the policy doesn't declare
+		 * support for it
+		 */
+		if (sid == SECINITSID_INIT && !isid_init_supported)
+			continue;
+
 		rc = sidtab_set_initial(s, sid, &c->context[0]);
 		if (rc) {
 			pr_err("SELinux:  unable to load initial SID %s.\n",
@@ -894,6 +896,24 @@ int policydb_load_isids(struct policydb *p, struct sidtab *s)
 			sidtab_destroy(s);
 			return rc;
 		}
+
+		/*
+		 * If the policy doesn't support the "userspace_initial_context"
+		 * capability, set SECINITSID_INIT to the same context as
+		 * SECINITSID_KERNEL. This ensures the same behavior as before
+		 * the reintroduction of SECINITSID_INIT, where all tasks
+		 * started before policy load would initially get the context
+		 * corresponding to SECINITSID_KERNEL.
+		 */
+		if (sid == SECINITSID_KERNEL && !isid_init_supported) {
+			rc = sidtab_set_initial(s, SECINITSID_INIT, &c->context[0]);
+			if (rc) {
+				pr_err("SELinux:  unable to load initial SID %s.\n",
+				       name);
+				sidtab_destroy(s);
+				return rc;
+			}
+		}
 	}
 	return 0;
 }
diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c
index 1eeffc66ea7d..e88b1b6c4adb 100644
--- a/security/selinux/ss/services.c
+++ b/security/selinux/ss/services.c
@@ -1322,8 +1322,19 @@ static int security_sid_to_context_core(u32 sid, char **scontext,
 	if (!selinux_initialized()) {
 		if (sid <= SECINITSID_NUM) {
 			char *scontextp;
-			const char *s = initial_sid_to_string[sid];
+			const char *s;
 
+			/*
+			 * Before the policy is loaded, translate
+			 * SECINITSID_INIT to "kernel", because systemd and
+			 * libselinux < 2.6 take a getcon_raw() result that is
+			 * both non-null and not "kernel" to mean that a policy
+			 * is already loaded.
+			 */
+			if (sid == SECINITSID_INIT)
+				sid = SECINITSID_KERNEL;
+
+			s = initial_sid_to_string[sid];
 			if (!s)
 				return -EINVAL;
 			*scontext_len = strlen(s) + 1;
diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c
index 65130a791f57..53336d7daa93 100644
--- a/security/smack/smack_lsm.c
+++ b/security/smack/smack_lsm.c
@@ -43,6 +43,7 @@
 #include <linux/fs_parser.h>
 #include <linux/watch_queue.h>
 #include <linux/io_uring.h>
+#include <uapi/linux/lsm.h>
 #include "smack.h"
 
 #define TRANS_TRUE	"TRUE"
@@ -3625,6 +3626,35 @@ static void smack_d_instantiate(struct dentry *opt_dentry, struct inode *inode)
 	return;
 }
 
+/**
+ * smack_getselfattr - Smack current process attribute
+ * @attr: which attribute to fetch
+ * @ctx: buffer to receive the result
+ * @size: available size in, actual size out
+ * @flags: unused
+ *
+ * Fill the passed user space @ctx with the details of the requested
+ * attribute.
+ *
+ * Returns the number of attributes on success, an error code otherwise.
+ * There will only ever be one attribute.
+ */
+static int smack_getselfattr(unsigned int attr, struct lsm_ctx __user *ctx,
+			     size_t *size, u32 flags)
+{
+	int rc;
+	struct smack_known *skp;
+
+	if (attr != LSM_ATTR_CURRENT)
+		return -EOPNOTSUPP;
+
+	skp = smk_of_current();
+	rc = lsm_fill_user_ctx(ctx, size,
+			       skp->smk_known, strlen(skp->smk_known) + 1,
+			       LSM_ID_SMACK, 0);
+	return (!rc ? 1 : rc);
+}
+
 /**
  * smack_getprocattr - Smack process attribute access
  * @p: the object task
@@ -3654,8 +3684,8 @@ static int smack_getprocattr(struct task_struct *p, const char *name, char **val
 }
 
 /**
- * smack_setprocattr - Smack process attribute setting
- * @name: the name of the attribute in /proc/.../attr
+ * do_setattr - Smack process attribute setting
+ * @attr: the ID of the attribute
  * @value: the value to set
  * @size: the size of the value
  *
@@ -3664,7 +3694,7 @@ static int smack_getprocattr(struct task_struct *p, const char *name, char **val
  *
  * Returns the length of the smack label or an error code
  */
-static int smack_setprocattr(const char *name, void *value, size_t size)
+static int do_setattr(u64 attr, void *value, size_t size)
 {
 	struct task_smack *tsp = smack_cred(current_cred());
 	struct cred *new;
@@ -3678,8 +3708,8 @@ static int smack_setprocattr(const char *name, void *value, size_t size)
 	if (value == NULL || size == 0 || size >= SMK_LONGLABEL)
 		return -EINVAL;
 
-	if (strcmp(name, "current") != 0)
-		return -EINVAL;
+	if (attr != LSM_ATTR_CURRENT)
+		return -EOPNOTSUPP;
 
 	skp = smk_import_entry(value, size);
 	if (IS_ERR(skp))
@@ -3718,6 +3748,49 @@ static int smack_setprocattr(const char *name, void *value, size_t size)
 	return size;
 }
 
+/**
+ * smack_setselfattr - Set a Smack process attribute
+ * @attr: which attribute to set
+ * @ctx: buffer containing the data
+ * @size: size of @ctx
+ * @flags: unused
+ *
+ * Fill the passed user space @ctx with the details of the requested
+ * attribute.
+ *
+ * Returns 0 on success, an error code otherwise.
+ */
+static int smack_setselfattr(unsigned int attr, struct lsm_ctx *ctx,
+			     size_t size, u32 flags)
+{
+	int rc;
+
+	rc = do_setattr(attr, ctx->ctx, ctx->ctx_len);
+	if (rc > 0)
+		return 0;
+	return rc;
+}
+
+/**
+ * smack_setprocattr - Smack process attribute setting
+ * @name: the name of the attribute in /proc/.../attr
+ * @value: the value to set
+ * @size: the size of the value
+ *
+ * Sets the Smack value of the task. Only setting self
+ * is permitted and only with privilege
+ *
+ * Returns the length of the smack label or an error code
+ */
+static int smack_setprocattr(const char *name, void *value, size_t size)
+{
+	int attr = lsm_name_to_attr(name);
+
+	if (attr != LSM_ATTR_UNDEF)
+		return do_setattr(attr, value, size);
+	return -EINVAL;
+}
+
 /**
  * smack_unix_stream_connect - Smack access on UDS
  * @sock: one sock
@@ -4933,6 +5006,11 @@ struct lsm_blob_sizes smack_blob_sizes __ro_after_init = {
 	.lbs_xattr_count = SMACK_INODE_INIT_XATTRS,
 };
 
+static const struct lsm_id smack_lsmid = {
+	.name = "smack",
+	.id = LSM_ID_SMACK,
+};
+
 static struct security_hook_list smack_hooks[] __ro_after_init = {
 	LSM_HOOK_INIT(ptrace_access_check, smack_ptrace_access_check),
 	LSM_HOOK_INIT(ptrace_traceme, smack_ptrace_traceme),
@@ -5027,6 +5105,8 @@ static struct security_hook_list smack_hooks[] __ro_after_init = {
 
 	LSM_HOOK_INIT(d_instantiate, smack_d_instantiate),
 
+	LSM_HOOK_INIT(getselfattr, smack_getselfattr),
+	LSM_HOOK_INIT(setselfattr, smack_setselfattr),
 	LSM_HOOK_INIT(getprocattr, smack_getprocattr),
 	LSM_HOOK_INIT(setprocattr, smack_setprocattr),
 
@@ -5140,7 +5220,7 @@ static __init int smack_init(void)
 	/*
 	 * Register with LSM
 	 */
-	security_add_hooks(smack_hooks, ARRAY_SIZE(smack_hooks), "smack");
+	security_add_hooks(smack_hooks, ARRAY_SIZE(smack_hooks), &smack_lsmid);
 	smack_enabled = 1;
 
 	pr_info("Smack:  Initializing.\n");
diff --git a/security/tomoyo/tomoyo.c b/security/tomoyo/tomoyo.c
index 255f1b470295..e10491f155a5 100644
--- a/security/tomoyo/tomoyo.c
+++ b/security/tomoyo/tomoyo.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/lsm_hooks.h>
+#include <uapi/linux/lsm.h>
 #include "common.h"
 
 /**
@@ -542,6 +543,11 @@ static void tomoyo_task_free(struct task_struct *task)
 	}
 }
 
+static const struct lsm_id tomoyo_lsmid = {
+	.name = "tomoyo",
+	.id = LSM_ID_TOMOYO,
+};
+
 /*
  * tomoyo_security_ops is a "struct security_operations" which is used for
  * registering TOMOYO.
@@ -595,7 +601,8 @@ static int __init tomoyo_init(void)
 	struct tomoyo_task *s = tomoyo_task(current);
 
 	/* register ourselves with the security framework */
-	security_add_hooks(tomoyo_hooks, ARRAY_SIZE(tomoyo_hooks), "tomoyo");
+	security_add_hooks(tomoyo_hooks, ARRAY_SIZE(tomoyo_hooks),
+			   &tomoyo_lsmid);
 	pr_info("TOMOYO Linux initialized\n");
 	s->domain_info = &tomoyo_kernel_domain;
 	atomic_inc(&tomoyo_kernel_domain.users);
diff --git a/security/yama/yama_lsm.c b/security/yama/yama_lsm.c
index 2503cf153d4a..49dc52b454ef 100644
--- a/security/yama/yama_lsm.c
+++ b/security/yama/yama_lsm.c
@@ -18,6 +18,7 @@
 #include <linux/task_work.h>
 #include <linux/sched.h>
 #include <linux/spinlock.h>
+#include <uapi/linux/lsm.h>
 
 #define YAMA_SCOPE_DISABLED	0
 #define YAMA_SCOPE_RELATIONAL	1
@@ -421,6 +422,11 @@ static int yama_ptrace_traceme(struct task_struct *parent)
 	return rc;
 }
 
+static const struct lsm_id yama_lsmid = {
+	.name = "yama",
+	.id = LSM_ID_YAMA,
+};
+
 static struct security_hook_list yama_hooks[] __ro_after_init = {
 	LSM_HOOK_INIT(ptrace_access_check, yama_ptrace_access_check),
 	LSM_HOOK_INIT(ptrace_traceme, yama_ptrace_traceme),
@@ -471,7 +477,7 @@ static inline void yama_init_sysctl(void) { }
 static int __init yama_init(void)
 {
 	pr_info("Yama: becoming mindful.\n");
-	security_add_hooks(yama_hooks, ARRAY_SIZE(yama_hooks), "yama");
+	security_add_hooks(yama_hooks, ARRAY_SIZE(yama_hooks), &yama_lsmid);
 	yama_init_sysctl();
 	return 0;
 }
diff --git a/sound/core/pcm_drm_eld.c b/sound/core/pcm_drm_eld.c
index 07075071972d..1cdca4d4fc9c 100644
--- a/sound/core/pcm_drm_eld.c
+++ b/sound/core/pcm_drm_eld.c
@@ -6,6 +6,7 @@
 #include <linux/export.h>
 #include <linux/hdmi.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_eld.h>
 #include <sound/pcm.h>
 #include <sound/pcm_drm_eld.h>
 
diff --git a/sound/isa/wavefront/wavefront_fx.c b/sound/isa/wavefront/wavefront_fx.c
index 3c21324b2a0e..0273b7dfaf12 100644
--- a/sound/isa/wavefront/wavefront_fx.c
+++ b/sound/isa/wavefront/wavefront_fx.c
@@ -191,9 +191,9 @@ snd_wavefront_fx_ioctl (struct snd_hwdep *sdev, struct file *file,
 					    "> 512 bytes to FX\n");
 				return -EIO;
 			}
-			page_data = memdup_user((unsigned char __user *)
-						r.data[3],
-						r.data[2] * sizeof(short));
+			page_data = memdup_array_user((unsigned char __user *)
+						      r.data[3],
+						      r.data[2], sizeof(short));
 			if (IS_ERR(page_data))
 				return PTR_ERR(page_data);
 			pd = page_data;
diff --git a/sound/pci/hda/cs35l41_hda.c b/sound/pci/hda/cs35l41_hda.c
index b2db8091f0ed..cbd7d8badf91 100644
--- a/sound/pci/hda/cs35l41_hda.c
+++ b/sound/pci/hda/cs35l41_hda.c
@@ -504,7 +504,6 @@ static void cs35l41_shutdown_dsp(struct cs35l41_hda *cs35l41)
 
 	cs_dsp_stop(dsp);
 	cs_dsp_power_down(dsp);
-	cs35l41->firmware_running = false;
 	dev_dbg(cs35l41->dev, "Unloaded Firmware\n");
 }
 
@@ -550,7 +549,7 @@ static void cs35l41_hda_play_start(struct device *dev)
 
 	cs35l41->playback_started = true;
 
-	if (cs35l41->firmware_running) {
+	if (cs35l41->cs_dsp.running) {
 		regmap_multi_reg_write(reg, cs35l41_hda_config_dsp,
 				       ARRAY_SIZE(cs35l41_hda_config_dsp));
 		regmap_update_bits(reg, CS35L41_PWR_CTRL2,
@@ -580,7 +579,7 @@ static void cs35l41_mute(struct device *dev, bool mute)
 			regmap_multi_reg_write(reg, cs35l41_hda_mute, ARRAY_SIZE(cs35l41_hda_mute));
 		} else {
 			dev_dbg(dev, "Unmuting\n");
-			if (cs35l41->firmware_running) {
+			if (cs35l41->cs_dsp.running) {
 				regmap_multi_reg_write(reg, cs35l41_hda_unmute_dsp,
 						ARRAY_SIZE(cs35l41_hda_unmute_dsp));
 			} else {
@@ -599,7 +598,7 @@ static void cs35l41_hda_play_done(struct device *dev)
 	dev_dbg(dev, "Play (Complete)\n");
 
 	cs35l41_global_enable(dev, reg, cs35l41->hw_cfg.bst_type, 1,
-			      cs35l41->firmware_running);
+			      &cs35l41->cs_dsp);
 	cs35l41_mute(dev, false);
 }
 
@@ -612,7 +611,7 @@ static void cs35l41_hda_pause_start(struct device *dev)
 
 	cs35l41_mute(dev, true);
 	cs35l41_global_enable(dev, reg, cs35l41->hw_cfg.bst_type, 0,
-			      cs35l41->firmware_running);
+			      &cs35l41->cs_dsp);
 }
 
 static void cs35l41_hda_pause_done(struct device *dev)
@@ -625,7 +624,7 @@ static void cs35l41_hda_pause_done(struct device *dev)
 	regmap_update_bits(reg, CS35L41_PWR_CTRL2, CS35L41_AMP_EN_MASK, 0 << CS35L41_AMP_EN_SHIFT);
 	if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST)
 		regmap_write(reg, CS35L41_GPIO1_CTRL1, 0x00000001);
-	if (cs35l41->firmware_running) {
+	if (cs35l41->cs_dsp.running) {
 		cs35l41_set_cspl_mbox_cmd(dev, reg, CSPL_MBOX_CMD_PAUSE);
 		regmap_update_bits(reg, CS35L41_PWR_CTRL2,
 				   CS35L41_VMON_EN_MASK | CS35L41_IMON_EN_MASK,
@@ -675,7 +674,7 @@ static void cs35l41_hda_playback_hook(struct device *dev, int action)
 		break;
 	case HDA_GEN_PCM_ACT_CLOSE:
 		mutex_lock(&cs35l41->fw_mutex);
-		if (!cs35l41->firmware_running && cs35l41->request_fw_load &&
+		if (!cs35l41->cs_dsp.running && cs35l41->request_fw_load &&
 		    !cs35l41->fw_request_ongoing) {
 			dev_info(dev, "Requesting Firmware Load after HDA_GEN_PCM_ACT_CLOSE\n");
 			cs35l41->fw_request_ongoing = true;
@@ -761,10 +760,9 @@ static int cs35l41_verify_id(struct cs35l41_hda *cs35l41, unsigned int *regid, u
 static int cs35l41_ready_for_reset(struct cs35l41_hda *cs35l41)
 {
 	mutex_lock(&cs35l41->fw_mutex);
-	if (cs35l41->firmware_running) {
+	if (cs35l41->cs_dsp.running) {
 		cs35l41->cs_dsp.running = false;
 		cs35l41->cs_dsp.booted = false;
-		cs35l41->firmware_running = false;
 	}
 	regcache_mark_dirty(cs35l41->regmap);
 	mutex_unlock(&cs35l41->fw_mutex);
@@ -925,7 +923,7 @@ static int cs35l41_runtime_suspend(struct device *dev)
 
 	mutex_lock(&cs35l41->fw_mutex);
 
-	if (cs35l41->firmware_running) {
+	if (cs35l41->cs_dsp.running) {
 		ret = cs35l41_enter_hibernate(cs35l41->dev, cs35l41->regmap,
 					      cs35l41->hw_cfg.bst_type);
 		if (ret)
@@ -960,7 +958,7 @@ static int cs35l41_runtime_resume(struct device *dev)
 
 	regcache_cache_only(cs35l41->regmap, false);
 
-	if (cs35l41->firmware_running)	{
+	if (cs35l41->cs_dsp.running)	{
 		ret = cs35l41_exit_hibernate(cs35l41->dev, cs35l41->regmap);
 		if (ret) {
 			dev_warn(cs35l41->dev, "Unable to exit Hibernate.");
@@ -1052,8 +1050,6 @@ static int cs35l41_smart_amp(struct cs35l41_hda *cs35l41)
 		goto clean_dsp;
 	}
 
-	cs35l41->firmware_running = true;
-
 	return 0;
 
 clean_dsp:
@@ -1063,10 +1059,10 @@ static int cs35l41_smart_amp(struct cs35l41_hda *cs35l41)
 
 static void cs35l41_load_firmware(struct cs35l41_hda *cs35l41, bool load)
 {
-	if (cs35l41->firmware_running && !load) {
+	if (cs35l41->cs_dsp.running && !load) {
 		dev_dbg(cs35l41->dev, "Unloading Firmware\n");
 		cs35l41_shutdown_dsp(cs35l41);
-	} else if (!cs35l41->firmware_running && load) {
+	} else if (!cs35l41->cs_dsp.running && load) {
 		dev_dbg(cs35l41->dev, "Loading Firmware\n");
 		cs35l41_smart_amp(cs35l41);
 	} else {
@@ -1346,7 +1342,7 @@ static int cs35l41_hda_bind(struct device *dev, struct device *master, void *mas
 		 cs35l41->acpi_subsystem_id, cs35l41->hw_cfg.bst_type,
 		 cs35l41->hw_cfg.gpio1.func == CS35l41_VSPK_SWITCH,
 		 cs35l41->hw_cfg.spk_pos ? 'R' : 'L',
-		 cs35l41->firmware_running, cs35l41->speaker_id);
+		 cs35l41->cs_dsp.running, cs35l41->speaker_id);
 
 	return ret;
 }
diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c
index 5618b1d9bfd1..079876b7b3e7 100644
--- a/sound/pci/hda/patch_realtek.c
+++ b/sound/pci/hda/patch_realtek.c
@@ -7249,6 +7249,7 @@ enum {
 	ALC290_FIXUP_SUBWOOFER_HSJACK,
 	ALC269_FIXUP_THINKPAD_ACPI,
 	ALC269_FIXUP_DMIC_THINKPAD_ACPI,
+	ALC269VB_FIXUP_CHUWI_COREBOOK_XPRO,
 	ALC255_FIXUP_ACER_MIC_NO_PRESENCE,
 	ALC255_FIXUP_ASUS_MIC_NO_PRESENCE,
 	ALC255_FIXUP_DELL1_MIC_NO_PRESENCE,
@@ -7601,6 +7602,14 @@ static const struct hda_fixup alc269_fixups[] = {
 		.type = HDA_FIXUP_FUNC,
 		.v.func = alc269_fixup_pincfg_U7x7_headset_mic,
 	},
+	[ALC269VB_FIXUP_CHUWI_COREBOOK_XPRO] = {
+		.type = HDA_FIXUP_PINS,
+		.v.pins = (const struct hda_pintbl[]) {
+			{ 0x18, 0x03a19020 }, /* headset mic */
+			{ 0x1b, 0x90170150 }, /* speaker */
+			{ }
+		},
+	},
 	[ALC269_FIXUP_AMIC] = {
 		.type = HDA_FIXUP_PINS,
 		.v.pins = (const struct hda_pintbl[]) {
@@ -10244,6 +10253,7 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x1d72, 0x1901, "RedmiBook 14", ALC256_FIXUP_ASUS_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1d72, 0x1945, "Redmi G", ALC256_FIXUP_ASUS_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1d72, 0x1947, "RedmiBook Air", ALC255_FIXUP_XIAOMI_HEADSET_MIC),
+	SND_PCI_QUIRK(0x2782, 0x0232, "CHUWI CoreBook XPro", ALC269VB_FIXUP_CHUWI_COREBOOK_XPRO),
 	SND_PCI_QUIRK(0x8086, 0x2074, "Intel NUC 8", ALC233_FIXUP_INTEL_NUC8_DMIC),
 	SND_PCI_QUIRK(0x8086, 0x2080, "Intel NUC 8 Rugged", ALC256_FIXUP_INTEL_NUC8_RUGGED),
 	SND_PCI_QUIRK(0x8086, 0x2081, "Intel NUC 10", ALC256_FIXUP_INTEL_NUC10),
diff --git a/sound/soc/amd/acp/Kconfig b/sound/soc/amd/acp/Kconfig
index 5fb322212938..84c963241dc5 100644
--- a/sound/soc/amd/acp/Kconfig
+++ b/sound/soc/amd/acp/Kconfig
@@ -73,6 +73,19 @@ config SND_AMD_ASOC_ACP63
 	  Say Y if you want to enable AUDIO on ACP6.3
 	  If unsure select "N".
 
+config SND_AMD_ASOC_ACP70
+	tristate "AMD ACP ASOC Acp7.0 Support"
+	depends on X86 && PCI
+	depends on ACPI
+	select SND_SOC_AMD_ACP_PCM
+	select SND_SOC_AMD_ACP_I2S
+	select SND_SOC_AMD_ACP_PDM
+	select SND_SOC_AMD_ACP_LEGACY_COMMON
+	help
+	This option enables Acp7.0 PDM support on AMD platform.
+	Say Y if you want to enable AUDIO on ACP7.0
+          If unsure select "N".
+
 config SND_SOC_AMD_MACH_COMMON
 	tristate
 	depends on X86 && PCI && I2C
diff --git a/sound/soc/amd/acp/Makefile b/sound/soc/amd/acp/Makefile
index dd85700f1c5f..ff5f7893b81e 100644
--- a/sound/soc/amd/acp/Makefile
+++ b/sound/soc/amd/acp/Makefile
@@ -15,6 +15,7 @@ snd-acp-pci-objs     := acp-pci.o
 snd-acp-renoir-objs     := acp-renoir.o
 snd-acp-rembrandt-objs  := acp-rembrandt.o
 snd-acp63-objs := acp63.o
+snd-acp70-objs := acp70.o
 
 #machine specific driver
 snd-acp-mach-objs     := acp-mach-common.o
@@ -30,6 +31,7 @@ obj-$(CONFIG_SND_SOC_AMD_ACP_PCI) += snd-acp-pci.o
 obj-$(CONFIG_SND_AMD_ASOC_RENOIR) += snd-acp-renoir.o
 obj-$(CONFIG_SND_AMD_ASOC_REMBRANDT) += snd-acp-rembrandt.o
 obj-$(CONFIG_SND_AMD_ASOC_ACP63) += snd-acp63.o
+obj-$(CONFIG_SND_AMD_ASOC_ACP70) += snd-acp70.o
 
 obj-$(CONFIG_SND_SOC_AMD_MACH_COMMON) += snd-acp-mach.o
 obj-$(CONFIG_SND_SOC_AMD_LEGACY_MACH) += snd-acp-legacy-mach.o
diff --git a/sound/soc/amd/acp/acp63.c b/sound/soc/amd/acp/acp63.c
index b871a216a6af..4d342441a650 100644
--- a/sound/soc/amd/acp/acp63.c
+++ b/sound/soc/amd/acp/acp63.c
@@ -283,18 +283,16 @@ static int __maybe_unused acp63_pcm_resume(struct device *dev)
 
 	spin_lock(&adata->acp_lock);
 	list_for_each_entry(stream, &adata->stream_list, list) {
-		if (stream) {
-			substream = stream->substream;
-			if (substream && substream->runtime) {
-				buf_in_frames = (substream->runtime->buffer_size);
-				buf_size = frames_to_bytes(substream->runtime, buf_in_frames);
-				config_pte_for_stream(adata, stream);
-				config_acp_dma(adata, stream, buf_size);
-				if (stream->dai_id)
-					restore_acp_i2s_params(substream, adata, stream);
-				else
-					restore_acp_pdm_params(substream, adata);
-			}
+		substream = stream->substream;
+		if (substream && substream->runtime) {
+			buf_in_frames = (substream->runtime->buffer_size);
+			buf_size = frames_to_bytes(substream->runtime, buf_in_frames);
+			config_pte_for_stream(adata, stream);
+			config_acp_dma(adata, stream, buf_size);
+			if (stream->dai_id)
+				restore_acp_i2s_params(substream, adata, stream);
+			else
+				restore_acp_pdm_params(substream, adata);
 		}
 	}
 	spin_unlock(&adata->acp_lock);
diff --git a/sound/soc/amd/acp/acp70.c b/sound/soc/amd/acp/acp70.c
index dd384c966ae9..0d7cdd4017e5 100644
--- a/sound/soc/amd/acp/acp70.c
+++ b/sound/soc/amd/acp/acp70.c
@@ -52,8 +52,8 @@ static struct snd_soc_dai_driver acp70_dai[] = {
 	.playback = {
 		.stream_name = "I2S SP Playback",
 		.rates = SNDRV_PCM_RATE_8000_96000,
-		.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 |
-			   SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE,
+		.formats = SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S8 |
+			   SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S32_LE,
 		.channels_min = 2,
 		.channels_max = 8,
 		.rate_min = 8000,
@@ -62,8 +62,8 @@ static struct snd_soc_dai_driver acp70_dai[] = {
 	.capture = {
 		.stream_name = "I2S SP Capture",
 		.rates = SNDRV_PCM_RATE_8000_48000,
-		.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 |
-			   SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE,
+		.formats = SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S8 |
+			   SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S32_LE,
 		.channels_min = 2,
 		.channels_max = 2,
 		.rate_min = 8000,
@@ -77,8 +77,8 @@ static struct snd_soc_dai_driver acp70_dai[] = {
 	.playback = {
 		.stream_name = "I2S BT Playback",
 		.rates = SNDRV_PCM_RATE_8000_96000,
-		.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 |
-			   SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE,
+		.formats = SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S8 |
+			   SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S32_LE,
 		.channels_min = 2,
 		.channels_max = 8,
 		.rate_min = 8000,
@@ -87,8 +87,8 @@ static struct snd_soc_dai_driver acp70_dai[] = {
 	.capture = {
 		.stream_name = "I2S BT Capture",
 		.rates = SNDRV_PCM_RATE_8000_48000,
-		.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 |
-			   SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE,
+		.formats = SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S8 |
+			   SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S32_LE,
 		.channels_min = 2,
 		.channels_max = 2,
 		.rate_min = 8000,
@@ -102,8 +102,8 @@ static struct snd_soc_dai_driver acp70_dai[] = {
 	.playback = {
 		.stream_name = "I2S HS Playback",
 		.rates = SNDRV_PCM_RATE_8000_96000,
-		.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 |
-			   SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE,
+		.formats = SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S8 |
+			   SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S32_LE,
 		.channels_min = 2,
 		.channels_max = 8,
 		.rate_min = 8000,
@@ -112,8 +112,8 @@ static struct snd_soc_dai_driver acp70_dai[] = {
 	.capture = {
 		.stream_name = "I2S HS Capture",
 		.rates = SNDRV_PCM_RATE_8000_48000,
-		.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 |
-			   SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE,
+		.formats = SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S8 |
+			   SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S32_LE,
 		.channels_min = 2,
 		.channels_max = 8,
 		.rate_min = 8000,
@@ -229,8 +229,8 @@ static int __maybe_unused acp70_pcm_resume(struct device *dev)
 			}
 		}
 	}
-		spin_unlock(&adata->acp_lock);
-		return 0;
+	spin_unlock(&adata->acp_lock);
+	return 0;
 }
 
 static const struct dev_pm_ops acp70_dma_pm_ops = {
diff --git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c
index 15a864dcd7bd..e2a510443bf1 100644
--- a/sound/soc/amd/yc/acp6x-mach.c
+++ b/sound/soc/amd/yc/acp6x-mach.c
@@ -367,6 +367,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = {
 			DMI_MATCH(DMI_BOARD_NAME, "8A3E"),
 		}
 	},
+	{
+		.driver_data = &acp6x_card,
+		.matches = {
+			DMI_MATCH(DMI_BOARD_VENDOR, "HP"),
+			DMI_MATCH(DMI_BOARD_NAME, "8B2F"),
+		}
+	},
 	{
 		.driver_data = &acp6x_card,
 		.matches = {
diff --git a/sound/soc/codecs/aw88395/aw88395_device.h b/sound/soc/codecs/aw88395/aw88395_device.h
index 791c8c106557..0f750f654f3e 100644
--- a/sound/soc/codecs/aw88395/aw88395_device.h
+++ b/sound/soc/codecs/aw88395/aw88395_device.h
@@ -146,6 +146,7 @@ struct aw_device {
 
 	unsigned int channel;
 	unsigned int fade_step;
+	unsigned int prof_data_type;
 
 	struct i2c_client *i2c;
 	struct device *dev;
diff --git a/sound/soc/codecs/aw88395/aw88395_lib.c b/sound/soc/codecs/aw88395/aw88395_lib.c
index 9ebe7c510109..f25f6e0d4428 100644
--- a/sound/soc/codecs/aw88395/aw88395_lib.c
+++ b/sound/soc/codecs/aw88395/aw88395_lib.c
@@ -11,7 +11,6 @@
 #include <linux/i2c.h>
 #include "aw88395_lib.h"
 #include "aw88395_device.h"
-#include "aw88395_reg.h"
 
 #define AW88395_CRC8_POLYNOMIAL 0x8C
 DECLARE_CRC8_TABLE(aw_crc8_table);
@@ -456,14 +455,6 @@ static int aw_dev_parse_reg_bin_with_hdr(struct aw_device *aw_dev,
 		goto parse_bin_failed;
 	}
 
-	if (aw_dev->chip_id == AW88261_CHIP_ID) {
-		if (aw_bin->header_info[0].valid_data_len % 4) {
-			dev_err(aw_dev->dev, "bin data len get error!");
-			ret = -EINVAL;
-			goto parse_bin_failed;
-		}
-	}
-
 	prof_desc->sec_desc[AW88395_DATA_TYPE_REG].data =
 				data + aw_bin->header_info[0].valid_data_addr;
 	prof_desc->sec_desc[AW88395_DATA_TYPE_REG].len =
@@ -528,7 +519,7 @@ static int aw_dev_parse_dev_type(struct aw_device *aw_dev,
 							cfg_dde[i].dev_profile);
 				return -EINVAL;
 			}
-
+			aw_dev->prof_data_type = cfg_dde[i].data_type;
 			ret = aw_dev_parse_data_by_sec_type(aw_dev, prof_hdr, &cfg_dde[i],
 					&all_prof_info->prof_desc[cfg_dde[i].dev_profile]);
 			if (ret < 0) {
@@ -564,6 +555,7 @@ static int aw_dev_parse_dev_default_type(struct aw_device *aw_dev,
 					cfg_dde[i].dev_profile);
 				return -EINVAL;
 			}
+			aw_dev->prof_data_type = cfg_dde[i].data_type;
 			ret = aw_dev_parse_data_by_sec_type(aw_dev, prof_hdr, &cfg_dde[i],
 					&all_prof_info->prof_desc[cfg_dde[i].dev_profile]);
 			if (ret < 0) {
@@ -582,7 +574,7 @@ static int aw_dev_parse_dev_default_type(struct aw_device *aw_dev,
 	return 0;
 }
 
-static int aw88261_dev_cfg_get_valid_prof(struct aw_device *aw_dev,
+static int aw_dev_cfg_get_reg_valid_prof(struct aw_device *aw_dev,
 				struct aw_all_prof_info *all_prof_info)
 {
 	struct aw_prof_desc *prof_desc = all_prof_info->prof_desc;
@@ -624,7 +616,7 @@ static int aw88261_dev_cfg_get_valid_prof(struct aw_device *aw_dev,
 	return 0;
 }
 
-static int aw88395_dev_cfg_get_valid_prof(struct aw_device *aw_dev,
+static int aw_dev_cfg_get_multiple_valid_prof(struct aw_device *aw_dev,
 				struct aw_all_prof_info *all_prof_info)
 {
 	struct aw_prof_desc *prof_desc = all_prof_info->prof_desc;
@@ -703,26 +695,20 @@ static int aw_dev_load_cfg_by_hdr(struct aw_device *aw_dev,
 			goto exit;
 	}
 
-	switch (aw_dev->chip_id) {
-	case AW88395_CHIP_ID:
-	case AW88399_CHIP_ID:
-		ret = aw88395_dev_cfg_get_valid_prof(aw_dev, all_prof_info);
-		if (ret < 0)
-			goto exit;
+	switch (aw_dev->prof_data_type) {
+	case ACF_SEC_TYPE_MULTIPLE_BIN:
+		ret = aw_dev_cfg_get_multiple_valid_prof(aw_dev, all_prof_info);
 		break;
-	case AW88261_CHIP_ID:
-	case AW87390_CHIP_ID:
-		ret = aw88261_dev_cfg_get_valid_prof(aw_dev, all_prof_info);
-		if (ret < 0)
-			goto exit;
+	case ACF_SEC_TYPE_HDR_REG:
+		ret = aw_dev_cfg_get_reg_valid_prof(aw_dev, all_prof_info);
 		break;
 	default:
-		dev_err(aw_dev->dev, "valid prof unsupported");
+		dev_err(aw_dev->dev, "unsupport data type\n");
 		ret = -EINVAL;
 		break;
 	}
-
-	aw_dev->prof_info.prof_name_list = profile_name;
+	if (!ret)
+		aw_dev->prof_info.prof_name_list = profile_name;
 
 exit:
 	devm_kfree(aw_dev->dev, all_prof_info);
@@ -791,39 +777,23 @@ static int aw_get_dev_scene_count_v1(struct aw_device *aw_dev, struct aw_contain
 	struct aw_cfg_dde_v1 *cfg_dde =
 		(struct aw_cfg_dde_v1 *)(aw_cfg->data + cfg_hdr->hdr_offset);
 	unsigned int i;
-	int ret;
 
-	switch (aw_dev->chip_id) {
-	case AW88395_CHIP_ID:
-	case AW88399_CHIP_ID:
-		for (i = 0; i < cfg_hdr->ddt_num; ++i) {
-			if ((cfg_dde[i].data_type == ACF_SEC_TYPE_MULTIPLE_BIN) &&
-			    (aw_dev->chip_id == cfg_dde[i].chip_id) &&
-			    (aw_dev->i2c->adapter->nr == cfg_dde[i].dev_bus) &&
-			    (aw_dev->i2c->addr == cfg_dde[i].dev_addr))
-				(*scene_num)++;
-		}
-		ret = 0;
-		break;
-	case AW88261_CHIP_ID:
-	case AW87390_CHIP_ID:
-		for (i = 0; i < cfg_hdr->ddt_num; ++i) {
-			if (((cfg_dde[i].data_type == ACF_SEC_TYPE_REG) ||
-			     (cfg_dde[i].data_type == ACF_SEC_TYPE_HDR_REG)) &&
-			    (aw_dev->chip_id == cfg_dde[i].chip_id) &&
-			    (aw_dev->i2c->adapter->nr == cfg_dde[i].dev_bus) &&
-			    (aw_dev->i2c->addr == cfg_dde[i].dev_addr))
-				(*scene_num)++;
-		}
-		ret = 0;
-		break;
-	default:
-		dev_err(aw_dev->dev, "unsupported device");
-		ret = -EINVAL;
-		break;
+	for (i = 0; i < cfg_hdr->ddt_num; ++i) {
+		if (((cfg_dde[i].data_type == ACF_SEC_TYPE_REG) ||
+		     (cfg_dde[i].data_type == ACF_SEC_TYPE_HDR_REG) ||
+		     (cfg_dde[i].data_type == ACF_SEC_TYPE_MULTIPLE_BIN)) &&
+		    (aw_dev->chip_id == cfg_dde[i].chip_id) &&
+		    (aw_dev->i2c->adapter->nr == cfg_dde[i].dev_bus) &&
+		    (aw_dev->i2c->addr == cfg_dde[i].dev_addr))
+			(*scene_num)++;
 	}
 
-	return ret;
+	if ((*scene_num) == 0) {
+		dev_err(aw_dev->dev, "failed to obtain scene, scenu_num = %d\n", (*scene_num));
+		return -EINVAL;
+	}
+
+	return 0;
 }
 
 static int aw_get_default_scene_count_v1(struct aw_device *aw_dev,
@@ -834,37 +804,23 @@ static int aw_get_default_scene_count_v1(struct aw_device *aw_dev,
 	struct aw_cfg_dde_v1 *cfg_dde =
 		(struct aw_cfg_dde_v1 *)(aw_cfg->data + cfg_hdr->hdr_offset);
 	unsigned int i;
-	int ret;
 
-	switch (aw_dev->chip_id) {
-	case AW88395_CHIP_ID:
-	case AW88399_CHIP_ID:
-		for (i = 0; i < cfg_hdr->ddt_num; ++i) {
-			if ((cfg_dde[i].data_type == ACF_SEC_TYPE_MULTIPLE_BIN) &&
-			    (aw_dev->chip_id == cfg_dde[i].chip_id) &&
-			    (aw_dev->channel == cfg_dde[i].dev_index))
-				(*scene_num)++;
-		}
-		ret = 0;
-		break;
-	case AW88261_CHIP_ID:
-	case AW87390_CHIP_ID:
-		for (i = 0; i < cfg_hdr->ddt_num; ++i) {
-			if (((cfg_dde[i].data_type == ACF_SEC_TYPE_REG) ||
-			     (cfg_dde[i].data_type == ACF_SEC_TYPE_HDR_REG)) &&
-			    (aw_dev->chip_id == cfg_dde[i].chip_id) &&
-			    (aw_dev->channel == cfg_dde[i].dev_index))
-				(*scene_num)++;
-		}
-		ret = 0;
-		break;
-	default:
-		dev_err(aw_dev->dev, "unsupported device");
-		ret = -EINVAL;
-		break;
+
+	for (i = 0; i < cfg_hdr->ddt_num; ++i) {
+		if (((cfg_dde[i].data_type == ACF_SEC_TYPE_MULTIPLE_BIN) ||
+		     (cfg_dde[i].data_type == ACF_SEC_TYPE_REG) ||
+		     (cfg_dde[i].data_type == ACF_SEC_TYPE_HDR_REG)) &&
+		    (aw_dev->chip_id == cfg_dde[i].chip_id) &&
+		    (aw_dev->channel == cfg_dde[i].dev_index))
+			(*scene_num)++;
 	}
 
-	return ret;
+	if ((*scene_num) == 0) {
+		dev_err(aw_dev->dev, "failed to obtain scene, scenu_num = %d\n", (*scene_num));
+		return -EINVAL;
+	}
+
+	return 0;
 }
 
 static int aw_dev_parse_scene_count_v1(struct aw_device *aw_dev,
diff --git a/sound/soc/codecs/aw88395/aw88395_reg.h b/sound/soc/codecs/aw88395/aw88395_reg.h
index ede7deab6a9c..e64f24e97150 100644
--- a/sound/soc/codecs/aw88395/aw88395_reg.h
+++ b/sound/soc/codecs/aw88395/aw88395_reg.h
@@ -95,10 +95,7 @@
 #define AW88395_TM_REG			(0x7C)
 
 enum aw88395_id {
-	AW88399_CHIP_ID = 0x2183,
 	AW88395_CHIP_ID = 0x2049,
-	AW88261_CHIP_ID = 0x2113,
-	AW87390_CHIP_ID = 0x76,
 };
 
 #define AW88395_REG_MAX		(0x7D)
diff --git a/sound/soc/codecs/aw88399.c b/sound/soc/codecs/aw88399.c
index 54f8457e8497..9fcb805bf971 100644
--- a/sound/soc/codecs/aw88399.c
+++ b/sound/soc/codecs/aw88399.c
@@ -15,7 +15,6 @@
 #include <sound/soc.h>
 #include "aw88399.h"
 #include "aw88395/aw88395_device.h"
-#include "aw88395/aw88395_reg.h"
 
 static const struct regmap_config aw88399_remap_config = {
 	.val_bits = 16,
diff --git a/sound/soc/codecs/aw88399.h b/sound/soc/codecs/aw88399.h
index 4f391099d0f2..5e9cdf725d3d 100644
--- a/sound/soc/codecs/aw88399.h
+++ b/sound/soc/codecs/aw88399.h
@@ -491,6 +491,7 @@
 #define AW88399_CRC_FW_BASE_ADDR		(0x4C0)
 #define AW88399_ACF_FILE			"aw88399_acf.bin"
 #define AW88399_DEV_SYSST_CHECK_MAX		(10)
+#define AW88399_CHIP_ID			0x2183
 
 #define AW88399_I2C_NAME			"aw88399"
 
diff --git a/sound/soc/codecs/cs35l41-lib.c b/sound/soc/codecs/cs35l41-lib.c
index 4569e4f7cf7e..e9993a39f7d0 100644
--- a/sound/soc/codecs/cs35l41-lib.c
+++ b/sound/soc/codecs/cs35l41-lib.c
@@ -16,6 +16,8 @@
 
 #include <sound/cs35l41.h>
 
+#define CS35L41_FIRMWARE_OLD_VERSION 0x001C00 /* v0.28.0 */
+
 static const struct reg_default cs35l41_reg[] = {
 	{ CS35L41_PWR_CTRL1,			0x00000000 },
 	{ CS35L41_PWR_CTRL2,			0x00000000 },
@@ -1214,7 +1216,7 @@ EXPORT_SYMBOL_GPL(cs35l41_safe_reset);
  * the PLL Lock interrupt, in the IRQ handler.
  */
 int cs35l41_global_enable(struct device *dev, struct regmap *regmap, enum cs35l41_boost_type b_type,
-			  int enable, bool firmware_running)
+			  int enable, struct cs_dsp *dsp)
 {
 	int ret;
 	unsigned int gpio1_func, pad_control, pwr_ctrl1, pwr_ctrl3, int_status, pup_pdn_mask;
@@ -1309,7 +1311,7 @@ int cs35l41_global_enable(struct device *dev, struct regmap *regmap, enum cs35l4
 			}
 			regmap_write(regmap, CS35L41_IRQ1_STATUS1, CS35L41_PUP_DONE_MASK);
 
-			if (firmware_running)
+			if (dsp->running && dsp->fw_id_version > CS35L41_FIRMWARE_OLD_VERSION)
 				ret = cs35l41_set_cspl_mbox_cmd(dev, regmap,
 								CSPL_MBOX_CMD_SPK_OUT_ENABLE);
 			else
diff --git a/sound/soc/codecs/cs35l41.c b/sound/soc/codecs/cs35l41.c
index d0e9128ac6d0..dfb4ce53491b 100644
--- a/sound/soc/codecs/cs35l41.c
+++ b/sound/soc/codecs/cs35l41.c
@@ -519,11 +519,11 @@ static int cs35l41_main_amp_event(struct snd_soc_dapm_widget *w,
 						ARRAY_SIZE(cs35l41_pup_patch));
 
 		ret = cs35l41_global_enable(cs35l41->dev, cs35l41->regmap, cs35l41->hw_cfg.bst_type,
-					    1, cs35l41->dsp.cs_dsp.running);
+					    1, &cs35l41->dsp.cs_dsp);
 		break;
 	case SND_SOC_DAPM_POST_PMD:
 		ret = cs35l41_global_enable(cs35l41->dev, cs35l41->regmap, cs35l41->hw_cfg.bst_type,
-					    0, cs35l41->dsp.cs_dsp.running);
+					    0, &cs35l41->dsp.cs_dsp);
 
 		regmap_multi_reg_write_bypassed(cs35l41->regmap,
 						cs35l41_pdn_patch,
diff --git a/sound/soc/codecs/cs42l43.c b/sound/soc/codecs/cs42l43.c
index d62c9f26c632..5c98343ebf71 100644
--- a/sound/soc/codecs/cs42l43.c
+++ b/sound/soc/codecs/cs42l43.c
@@ -2251,8 +2251,8 @@ static int cs42l43_codec_runtime_resume(struct device *dev)
 	return 0;
 }
 
-DEFINE_RUNTIME_DEV_PM_OPS(cs42l43_codec_pm_ops, NULL,
-			  cs42l43_codec_runtime_resume, NULL);
+static DEFINE_RUNTIME_DEV_PM_OPS(cs42l43_codec_pm_ops, NULL,
+				 cs42l43_codec_runtime_resume, NULL);
 
 static const struct platform_device_id cs42l43_codec_id_table[] = {
 	{ "cs42l43-codec", },
diff --git a/sound/soc/codecs/cs43130.c b/sound/soc/codecs/cs43130.c
index 0b40fdfb1825..845611afed85 100644
--- a/sound/soc/codecs/cs43130.c
+++ b/sound/soc/codecs/cs43130.c
@@ -16,7 +16,7 @@
 #include <linux/platform_device.h>
 #include <linux/pm.h>
 #include <linux/i2c.h>
-#include <linux/of.h>
+#include <linux/property.h>
 #include <linux/regmap.h>
 #include <linux/slab.h>
 #include <sound/core.h>
@@ -238,7 +238,7 @@ static int cs43130_pll_config(struct snd_soc_component *component)
 	struct cs43130_private *cs43130 = snd_soc_component_get_drvdata(component);
 	const struct cs43130_pll_params *pll_entry;
 
-	dev_dbg(component->dev, "cs43130->mclk = %u, cs43130->mclk_int = %u\n",
+	dev_dbg(cs43130->dev, "cs43130->mclk = %u, cs43130->mclk_int = %u\n",
 		cs43130->mclk, cs43130->mclk_int);
 
 	pll_entry = cs43130_get_pll_table(cs43130->mclk, cs43130->mclk_int);
@@ -303,7 +303,7 @@ static int cs43130_set_pll(struct snd_soc_component *component, int pll_id, int
 		cs43130->mclk = freq_in;
 		break;
 	default:
-		dev_err(component->dev,
+		dev_err(cs43130->dev,
 			"unsupported pll input reference clock:%d\n", freq_in);
 		return -EINVAL;
 	}
@@ -316,13 +316,13 @@ static int cs43130_set_pll(struct snd_soc_component *component, int pll_id, int
 		cs43130->mclk_int = freq_out;
 		break;
 	default:
-		dev_err(component->dev,
+		dev_err(cs43130->dev,
 			"unsupported pll output ref clock: %u\n", freq_out);
 		return -EINVAL;
 	}
 
 	ret = cs43130_pll_config(component);
-	dev_dbg(component->dev, "cs43130->pll_bypass = %d", cs43130->pll_bypass);
+	dev_dbg(cs43130->dev, "cs43130->pll_bypass = %d", cs43130->pll_bypass);
 	return ret;
 }
 
@@ -346,7 +346,7 @@ static int cs43130_change_clksrc(struct snd_soc_component *component,
 		mclk_int_decoded = CS43130_MCLK_24P5;
 		break;
 	default:
-		dev_err(component->dev, "Invalid MCLK INT freq: %u\n", cs43130->mclk_int);
+		dev_err(cs43130->dev, "Invalid MCLK INT freq: %u\n", cs43130->mclk_int);
 		return -EINVAL;
 	}
 
@@ -370,7 +370,7 @@ static int cs43130_change_clksrc(struct snd_soc_component *component,
 					   CS43130_XTAL_RDY_INT_MASK,
 					   1 << CS43130_XTAL_RDY_INT_SHIFT);
 			if (ret == 0) {
-				dev_err(component->dev, "Timeout waiting for XTAL_READY interrupt\n");
+				dev_err(cs43130->dev, "Timeout waiting for XTAL_READY interrupt\n");
 				return -ETIMEDOUT;
 			}
 		}
@@ -406,7 +406,7 @@ static int cs43130_change_clksrc(struct snd_soc_component *component,
 					   CS43130_XTAL_RDY_INT_MASK,
 					   1 << CS43130_XTAL_RDY_INT_SHIFT);
 			if (ret == 0) {
-				dev_err(component->dev, "Timeout waiting for XTAL_READY interrupt\n");
+				dev_err(cs43130->dev, "Timeout waiting for XTAL_READY interrupt\n");
 				return -ETIMEDOUT;
 			}
 		}
@@ -422,7 +422,7 @@ static int cs43130_change_clksrc(struct snd_soc_component *component,
 				   CS43130_PLL_RDY_INT_MASK,
 				   1 << CS43130_PLL_RDY_INT_SHIFT);
 		if (ret == 0) {
-			dev_err(component->dev, "Timeout waiting for PLL_READY interrupt\n");
+			dev_err(cs43130->dev, "Timeout waiting for PLL_READY interrupt\n");
 			return -ETIMEDOUT;
 		}
 
@@ -453,7 +453,7 @@ static int cs43130_change_clksrc(struct snd_soc_component *component,
 				   1 << CS43130_PDN_PLL_SHIFT);
 		break;
 	default:
-		dev_err(component->dev, "Invalid MCLK source value\n");
+		dev_err(cs43130->dev, "Invalid MCLK source value\n");
 		return -EINVAL;
 	}
 
@@ -578,7 +578,7 @@ static int cs43130_set_sp_fmt(int dai_id, unsigned int bitwidth_sclk,
 		break;
 	case SND_SOC_DAIFMT_LEFT_J:
 		hi_size = bitwidth_sclk;
-		frm_delay = 2;
+		frm_delay = 0;
 		frm_phase = 1;
 		break;
 	case SND_SOC_DAIFMT_DSP_A:
@@ -804,7 +804,7 @@ static int cs43130_dsd_hw_params(struct snd_pcm_substream *substream,
 		dsd_speed = 1;
 		break;
 	default:
-		dev_err(component->dev, "Rate(%u) not supported\n",
+		dev_err(cs43130->dev, "Rate(%u) not supported\n",
 			params_rate(params));
 		return -EINVAL;
 	}
@@ -875,7 +875,7 @@ static int cs43130_hw_params(struct snd_pcm_substream *substream,
 			dsd_speed = 1;
 			break;
 		default:
-			dev_err(component->dev, "Rate(%u) not supported\n",
+			dev_err(cs43130->dev, "Rate(%u) not supported\n",
 				params_rate(params));
 			return -EINVAL;
 		}
@@ -892,7 +892,7 @@ static int cs43130_hw_params(struct snd_pcm_substream *substream,
 		regmap_write(cs43130->regmap, CS43130_SP_SRATE, rate_map->val);
 		break;
 	default:
-		dev_err(component->dev, "Invalid DAI (%d)\n", dai->id);
+		dev_err(cs43130->dev, "Invalid DAI (%d)\n", dai->id);
 		return -EINVAL;
 	}
 
@@ -916,21 +916,21 @@ static int cs43130_hw_params(struct snd_pcm_substream *substream,
 
 	if (!sclk) {
 		/* at this point, SCLK must be set */
-		dev_err(component->dev, "SCLK freq is not set\n");
+		dev_err(cs43130->dev, "SCLK freq is not set\n");
 		return -EINVAL;
 	}
 
 	bitwidth_sclk = (sclk / params_rate(params)) / params_channels(params);
 	if (bitwidth_sclk < bitwidth_dai) {
-		dev_err(component->dev, "Format not supported: SCLK freq is too low\n");
+		dev_err(cs43130->dev, "Format not supported: SCLK freq is too low\n");
 		return -EINVAL;
 	}
 
-	dev_dbg(component->dev,
+	dev_dbg(cs43130->dev,
 		"sclk = %u, fs = %d, bitwidth_dai = %u\n",
 		sclk, params_rate(params), bitwidth_dai);
 
-	dev_dbg(component->dev,
+	dev_dbg(cs43130->dev,
 		"bitwidth_sclk = %u, num_ch = %u\n",
 		bitwidth_sclk, params_channels(params));
 
@@ -1189,7 +1189,7 @@ static int cs43130_dsd_event(struct snd_soc_dapm_widget *w,
 		}
 		break;
 	default:
-		dev_err(component->dev, "Invalid event = 0x%x\n", event);
+		dev_err(cs43130->dev, "Invalid event = 0x%x\n", event);
 		return -EINVAL;
 	}
 	return 0;
@@ -1246,7 +1246,7 @@ static int cs43130_pcm_event(struct snd_soc_dapm_widget *w,
 		}
 		break;
 	default:
-		dev_err(component->dev, "Invalid event = 0x%x\n", event);
+		dev_err(cs43130->dev, "Invalid event = 0x%x\n", event);
 		return -EINVAL;
 	}
 	return 0;
@@ -1322,7 +1322,7 @@ static int cs43130_dac_event(struct snd_soc_dapm_widget *w,
 		}
 		break;
 	default:
-		dev_err(component->dev, "Invalid DAC event = 0x%x\n", event);
+		dev_err(cs43130->dev, "Invalid DAC event = 0x%x\n", event);
 		return -EINVAL;
 	}
 	return 0;
@@ -1360,13 +1360,21 @@ static int cs43130_hpin_event(struct snd_soc_dapm_widget *w,
 				       ARRAY_SIZE(hpin_postpmu_seq));
 		break;
 	default:
-		dev_err(component->dev, "Invalid HPIN event = 0x%x\n", event);
+		dev_err(cs43130->dev, "Invalid HPIN event = 0x%x\n", event);
 		return -EINVAL;
 	}
 	return 0;
 }
 
+static const char * const bypass_mux_text[] = {
+	"Internal",
+	"Alternative",
+};
+static SOC_ENUM_SINGLE_DECL(bypass_enum, SND_SOC_NOPM, 0, bypass_mux_text);
+static const struct snd_kcontrol_new bypass_ctrl = SOC_DAPM_ENUM("Switch", bypass_enum);
+
 static const struct snd_soc_dapm_widget digital_hp_widgets[] = {
+	SND_SOC_DAPM_MUX("Bypass Switch", SND_SOC_NOPM, 0, 0, &bypass_ctrl),
 	SND_SOC_DAPM_OUTPUT("HPOUTA"),
 	SND_SOC_DAPM_OUTPUT("HPOUTB"),
 
@@ -1419,13 +1427,13 @@ static const struct snd_soc_dapm_route digital_hp_routes[] = {
 	{"DSD", NULL, "XSPIN DSD"},
 	{"HiFi DAC", NULL, "ASPIN PCM"},
 	{"HiFi DAC", NULL, "DSD"},
-	{"HPOUTA", NULL, "HiFi DAC"},
-	{"HPOUTB", NULL, "HiFi DAC"},
+	{"Bypass Switch", "Internal", "HiFi DAC"},
+	{"HPOUTA", NULL, "Bypass Switch"},
+	{"HPOUTB", NULL, "Bypass Switch"},
 };
 
 static const struct snd_soc_dapm_route analog_hp_routes[] = {
-	{"HPOUTA", NULL, "Analog Playback"},
-	{"HPOUTB", NULL, "Analog Playback"},
+	{"Bypass Switch", "Alternative", "Analog Playback"},
 };
 
 static struct snd_soc_dapm_route all_hp_routes[
@@ -1479,7 +1487,7 @@ static int cs43130_pcm_set_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt)
 		cs43130->dais[codec_dai->id].dai_mode = SND_SOC_DAIFMT_CBM_CFM;
 		break;
 	default:
-		dev_err(component->dev, "unsupported mode\n");
+		dev_err(cs43130->dev, "unsupported mode\n");
 		return -EINVAL;
 	}
 
@@ -1497,12 +1505,12 @@ static int cs43130_pcm_set_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt)
 		cs43130->dais[codec_dai->id].dai_format = SND_SOC_DAIFMT_DSP_B;
 		break;
 	default:
-		dev_err(component->dev,
+		dev_err(cs43130->dev,
 			"unsupported audio format\n");
 		return -EINVAL;
 	}
 
-	dev_dbg(component->dev, "dai_id = %d,  dai_mode = %u, dai_format = %u\n",
+	dev_dbg(cs43130->dev, "dai_id = %d,  dai_mode = %u, dai_format = %u\n",
 		codec_dai->id,
 		cs43130->dais[codec_dai->id].dai_mode,
 		cs43130->dais[codec_dai->id].dai_format);
@@ -1523,11 +1531,11 @@ static int cs43130_dsd_set_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt)
 		cs43130->dais[codec_dai->id].dai_mode = SND_SOC_DAIFMT_CBM_CFM;
 		break;
 	default:
-		dev_err(component->dev, "Unsupported DAI format.\n");
+		dev_err(cs43130->dev, "Unsupported DAI format.\n");
 		return -EINVAL;
 	}
 
-	dev_dbg(component->dev, "dai_mode = 0x%x\n",
+	dev_dbg(cs43130->dev, "dai_mode = 0x%x\n",
 		cs43130->dais[codec_dai->id].dai_mode);
 
 	return 0;
@@ -1540,7 +1548,7 @@ static int cs43130_set_sysclk(struct snd_soc_dai *codec_dai,
 	struct cs43130_private *cs43130 = snd_soc_component_get_drvdata(component);
 
 	cs43130->dais[codec_dai->id].sclk = freq;
-	dev_dbg(component->dev, "dai_id = %d,  sclk = %u\n", codec_dai->id,
+	dev_dbg(cs43130->dev, "dai_id = %d,  sclk = %u\n", codec_dai->id,
 		cs43130->dais[codec_dai->id].sclk);
 
 	return 0;
@@ -1630,7 +1638,7 @@ static int cs43130_component_set_sysclk(struct snd_soc_component *component,
 {
 	struct cs43130_private *cs43130 = snd_soc_component_get_drvdata(component);
 
-	dev_dbg(component->dev, "clk_id = %d, source = %d, freq = %d, dir = %d\n",
+	dev_dbg(cs43130->dev, "clk_id = %d, source = %d, freq = %d, dir = %d\n",
 		clk_id, source, freq, dir);
 
 	switch (freq) {
@@ -1639,14 +1647,14 @@ static int cs43130_component_set_sysclk(struct snd_soc_component *component,
 		cs43130->mclk = freq;
 		break;
 	default:
-		dev_err(component->dev, "Invalid MCLK INT freq: %u\n", freq);
+		dev_err(cs43130->dev, "Invalid MCLK INT freq: %u\n", freq);
 		return -EINVAL;
 	}
 
 	if (source == CS43130_MCLK_SRC_EXT) {
 		cs43130->pll_bypass = true;
 	} else {
-		dev_err(component->dev, "Invalid MCLK source\n");
+		dev_err(cs43130->dev, "Invalid MCLK source\n");
 		return -EINVAL;
 	}
 
@@ -1682,7 +1690,7 @@ static ssize_t hpload_dc_r_show(struct device *dev,
 	return cs43130_show_dc(dev, buf, HP_RIGHT);
 }
 
-static u16 const cs43130_ac_freq[CS43130_AC_FREQ] = {
+static const u16 cs43130_ac_freq[CS43130_AC_FREQ] = {
 	24,
 	43,
 	93,
@@ -1933,7 +1941,6 @@ static int cs43130_update_hpload(unsigned int msk, int ac_idx,
 	unsigned int reg;
 	u32 addr;
 	u16 impedance;
-	struct snd_soc_component *component = cs43130->component;
 
 	switch (msk) {
 	case CS43130_HPLOAD_DC_INT:
@@ -1963,7 +1970,7 @@ static int cs43130_update_hpload(unsigned int msk, int ac_idx,
 		else
 			cs43130->hpload_dc[HP_RIGHT] = impedance;
 
-		dev_dbg(component->dev, "HP DC impedance (Ch %u): %u\n", !left_ch,
+		dev_dbg(cs43130->dev, "HP DC impedance (Ch %u): %u\n", !left_ch,
 			impedance);
 	} else {
 		if (left_ch)
@@ -1971,7 +1978,7 @@ static int cs43130_update_hpload(unsigned int msk, int ac_idx,
 		else
 			cs43130->hpload_ac[ac_idx][HP_RIGHT] = impedance;
 
-		dev_dbg(component->dev, "HP AC (%u Hz) impedance (Ch %u): %u\n",
+		dev_dbg(cs43130->dev, "HP AC (%u Hz) impedance (Ch %u): %u\n",
 			cs43130->ac_freq[ac_idx], !left_ch, impedance);
 	}
 
@@ -1985,7 +1992,6 @@ static int cs43130_hpload_proc(struct cs43130_private *cs43130,
 	int ret;
 	unsigned int msk;
 	u16 ac_reg_val;
-	struct snd_soc_component *component = cs43130->component;
 
 	reinit_completion(&cs43130->hpload_evt);
 
@@ -2008,17 +2014,17 @@ static int cs43130_hpload_proc(struct cs43130_private *cs43130,
 					  msecs_to_jiffies(1000));
 	regmap_read(cs43130->regmap, CS43130_INT_MASK_4, &msk);
 	if (!ret) {
-		dev_err(component->dev, "Timeout waiting for HPLOAD interrupt\n");
+		dev_err(cs43130->dev, "Timeout waiting for HPLOAD interrupt\n");
 		return -1;
 	}
 
-	dev_dbg(component->dev, "HP load stat: %x, INT_MASK_4: %x\n",
+	dev_dbg(cs43130->dev, "HP load stat: %x, INT_MASK_4: %x\n",
 		cs43130->hpload_stat, msk);
 	if ((cs43130->hpload_stat & (CS43130_HPLOAD_NO_DC_INT |
 				     CS43130_HPLOAD_UNPLUG_INT |
 				     CS43130_HPLOAD_OOR_INT)) ||
 	    !(cs43130->hpload_stat & rslt_msk)) {
-		dev_dbg(component->dev, "HP load measure failed\n");
+		dev_dbg(cs43130->dev, "HP load measure failed\n");
 		return -1;
 	}
 
@@ -2129,9 +2135,9 @@ static void cs43130_imp_meas(struct work_struct *wk)
 		snd_soc_jack_report(&cs43130->jack, CS43130_JACK_HEADPHONE,
 				    CS43130_JACK_MASK);
 
-	dev_dbg(component->dev, "Set HP output control. DC threshold\n");
+	dev_dbg(cs43130->dev, "Set HP output control. DC threshold\n");
 	for (i = 0; i < CS43130_DC_THRESHOLD; i++)
-		dev_dbg(component->dev, "DC threshold[%d]: %u.\n", i,
+		dev_dbg(cs43130->dev, "DC threshold[%d]: %u.\n", i,
 			cs43130->dc_threshold[i]);
 
 	cs43130_set_hv(cs43130->regmap, cs43130->hpload_dc[HP_LEFT],
@@ -2165,7 +2171,6 @@ static void cs43130_imp_meas(struct work_struct *wk)
 static irqreturn_t cs43130_irq_thread(int irq, void *data)
 {
 	struct cs43130_private *cs43130 = (struct cs43130_private *)data;
-	struct snd_soc_component *component = cs43130->component;
 	unsigned int stickies[CS43130_NUM_INT];
 	unsigned int irq_occurrence = 0;
 	unsigned int masks[CS43130_NUM_INT];
@@ -2183,8 +2188,6 @@ static irqreturn_t cs43130_irq_thread(int irq, void *data)
 		for (j = 0; j < 8; j++)
 			irq_occurrence += (stickies[i] >> j) & 1;
 	}
-	dev_dbg(component->dev, "number of interrupts occurred (%u)\n",
-		irq_occurrence);
 
 	if (!irq_occurrence)
 		return IRQ_NONE;
@@ -2201,7 +2204,7 @@ static irqreturn_t cs43130_irq_thread(int irq, void *data)
 
 	if (stickies[3] & CS43130_HPLOAD_NO_DC_INT) {
 		cs43130->hpload_stat = stickies[3];
-		dev_err(component->dev,
+		dev_err(cs43130->dev,
 			"DC load has not completed before AC load (%x)\n",
 			cs43130->hpload_stat);
 		complete(&cs43130->hpload_evt);
@@ -2210,7 +2213,7 @@ static irqreturn_t cs43130_irq_thread(int irq, void *data)
 
 	if (stickies[3] & CS43130_HPLOAD_UNPLUG_INT) {
 		cs43130->hpload_stat = stickies[3];
-		dev_err(component->dev, "HP unplugged during measurement (%x)\n",
+		dev_err(cs43130->dev, "HP unplugged during measurement (%x)\n",
 			cs43130->hpload_stat);
 		complete(&cs43130->hpload_evt);
 		return IRQ_HANDLED;
@@ -2218,7 +2221,7 @@ static irqreturn_t cs43130_irq_thread(int irq, void *data)
 
 	if (stickies[3] & CS43130_HPLOAD_OOR_INT) {
 		cs43130->hpload_stat = stickies[3];
-		dev_err(component->dev, "HP load out of range (%x)\n",
+		dev_err(cs43130->dev, "HP load out of range (%x)\n",
 			cs43130->hpload_stat);
 		complete(&cs43130->hpload_evt);
 		return IRQ_HANDLED;
@@ -2226,7 +2229,7 @@ static irqreturn_t cs43130_irq_thread(int irq, void *data)
 
 	if (stickies[3] & CS43130_HPLOAD_AC_INT) {
 		cs43130->hpload_stat = stickies[3];
-		dev_dbg(component->dev, "HP AC load measurement done (%x)\n",
+		dev_dbg(cs43130->dev, "HP AC load measurement done (%x)\n",
 			cs43130->hpload_stat);
 		complete(&cs43130->hpload_evt);
 		return IRQ_HANDLED;
@@ -2234,7 +2237,7 @@ static irqreturn_t cs43130_irq_thread(int irq, void *data)
 
 	if (stickies[3] & CS43130_HPLOAD_DC_INT) {
 		cs43130->hpload_stat = stickies[3];
-		dev_dbg(component->dev, "HP DC load measurement done (%x)\n",
+		dev_dbg(cs43130->dev, "HP DC load measurement done (%x)\n",
 			cs43130->hpload_stat);
 		complete(&cs43130->hpload_evt);
 		return IRQ_HANDLED;
@@ -2242,7 +2245,7 @@ static irqreturn_t cs43130_irq_thread(int irq, void *data)
 
 	if (stickies[3] & CS43130_HPLOAD_ON_INT) {
 		cs43130->hpload_stat = stickies[3];
-		dev_dbg(component->dev, "HP load state machine on done (%x)\n",
+		dev_dbg(cs43130->dev, "HP load state machine on done (%x)\n",
 			cs43130->hpload_stat);
 		complete(&cs43130->hpload_evt);
 		return IRQ_HANDLED;
@@ -2250,19 +2253,19 @@ static irqreturn_t cs43130_irq_thread(int irq, void *data)
 
 	if (stickies[3] & CS43130_HPLOAD_OFF_INT) {
 		cs43130->hpload_stat = stickies[3];
-		dev_dbg(component->dev, "HP load state machine off done (%x)\n",
+		dev_dbg(cs43130->dev, "HP load state machine off done (%x)\n",
 			cs43130->hpload_stat);
 		complete(&cs43130->hpload_evt);
 		return IRQ_HANDLED;
 	}
 
 	if (stickies[0] & CS43130_XTAL_ERR_INT) {
-		dev_err(component->dev, "Crystal err: clock is not running\n");
+		dev_err(cs43130->dev, "Crystal err: clock is not running\n");
 		return IRQ_HANDLED;
 	}
 
 	if (stickies[0] & CS43130_HP_UNPLUG_INT) {
-		dev_dbg(component->dev, "HP unplugged\n");
+		dev_dbg(cs43130->dev, "HP unplugged\n");
 		cs43130->hpload_done = false;
 		snd_soc_jack_report(&cs43130->jack, 0, CS43130_JACK_MASK);
 		return IRQ_HANDLED;
@@ -2271,7 +2274,7 @@ static irqreturn_t cs43130_irq_thread(int irq, void *data)
 	if (stickies[0] & CS43130_HP_PLUG_INT) {
 		if (cs43130->dc_meas && !cs43130->hpload_done &&
 		    !work_busy(&cs43130->work)) {
-			dev_dbg(component->dev, "HP load queue work\n");
+			dev_dbg(cs43130->dev, "HP load queue work\n");
 			queue_work(cs43130->wq, &cs43130->work);
 		}
 
@@ -2303,19 +2306,19 @@ static int cs43130_probe(struct snd_soc_component *component)
 	ret = snd_soc_card_jack_new(card, "Headphone", CS43130_JACK_MASK,
 				    &cs43130->jack);
 	if (ret < 0) {
-		dev_err(component->dev, "Cannot create jack\n");
+		dev_err(cs43130->dev, "Cannot create jack\n");
 		return ret;
 	}
 
 	cs43130->hpload_done = false;
 	if (cs43130->dc_meas) {
-		ret = sysfs_create_groups(&component->dev->kobj, hpload_groups);
+		ret = sysfs_create_groups(&cs43130->dev->kobj, hpload_groups);
 		if (ret)
 			return ret;
 
 		cs43130->wq = create_singlethread_workqueue("cs43130_hp");
 		if (!cs43130->wq) {
-			sysfs_remove_groups(&component->dev->kobj, hpload_groups);
+			sysfs_remove_groups(&cs43130->dev->kobj, hpload_groups);
 			return -ENOMEM;
 		}
 		INIT_WORK(&cs43130->work, cs43130_imp_meas);
@@ -2362,19 +2365,17 @@ static const struct regmap_config cs43130_regmap = {
 	.use_single_write	= true,
 };
 
-static u16 const cs43130_dc_threshold[CS43130_DC_THRESHOLD] = {
+static const u16 cs43130_dc_threshold[CS43130_DC_THRESHOLD] = {
 	50,
 	120,
 };
 
-static int cs43130_handle_device_data(struct i2c_client *i2c_client,
-				      struct cs43130_private *cs43130)
+static int cs43130_handle_device_data(struct cs43130_private *cs43130)
 {
-	struct device_node *np = i2c_client->dev.of_node;
 	unsigned int val;
 	int i;
 
-	if (of_property_read_u32(np, "cirrus,xtal-ibias", &val) < 0) {
+	if (device_property_read_u32(cs43130->dev, "cirrus,xtal-ibias", &val) < 0) {
 		/* Crystal is unused. System clock is used for external MCLK */
 		cs43130->xtal_ibias = CS43130_XTAL_UNUSED;
 		return 0;
@@ -2391,23 +2392,23 @@ static int cs43130_handle_device_data(struct i2c_client *i2c_client,
 		cs43130->xtal_ibias = CS43130_XTAL_IBIAS_15UA;
 		break;
 	default:
-		dev_err(&i2c_client->dev,
+		dev_err(cs43130->dev,
 			"Invalid cirrus,xtal-ibias value: %d\n", val);
 		return -EINVAL;
 	}
 
-	cs43130->dc_meas = of_property_read_bool(np, "cirrus,dc-measure");
-	cs43130->ac_meas = of_property_read_bool(np, "cirrus,ac-measure");
+	cs43130->dc_meas = device_property_read_bool(cs43130->dev, "cirrus,dc-measure");
+	cs43130->ac_meas = device_property_read_bool(cs43130->dev, "cirrus,ac-measure");
 
-	if (of_property_read_u16_array(np, "cirrus,ac-freq", cs43130->ac_freq,
-					CS43130_AC_FREQ) < 0) {
+	if (!device_property_read_u16_array(cs43130->dev, "cirrus,ac-freq", cs43130->ac_freq,
+					CS43130_AC_FREQ)) {
 		for (i = 0; i < CS43130_AC_FREQ; i++)
 			cs43130->ac_freq[i] = cs43130_ac_freq[i];
 	}
 
-	if (of_property_read_u16_array(np, "cirrus,dc-threshold",
+	if (!device_property_read_u16_array(cs43130->dev, "cirrus,dc-threshold",
 				       cs43130->dc_threshold,
-				       CS43130_DC_THRESHOLD) < 0) {
+				       CS43130_DC_THRESHOLD)) {
 		for (i = 0; i < CS43130_DC_THRESHOLD; i++)
 			cs43130->dc_threshold[i] = cs43130_dc_threshold[i];
 	}
@@ -2426,6 +2427,8 @@ static int cs43130_i2c_probe(struct i2c_client *client)
 	if (!cs43130)
 		return -ENOMEM;
 
+	cs43130->dev = &client->dev;
+
 	i2c_set_clientdata(client, cs43130);
 
 	cs43130->regmap = devm_regmap_init_i2c(client, &cs43130_regmap);
@@ -2434,29 +2437,30 @@ static int cs43130_i2c_probe(struct i2c_client *client)
 		return ret;
 	}
 
-	if (client->dev.of_node) {
-		ret = cs43130_handle_device_data(client, cs43130);
+	if (dev_fwnode(cs43130->dev)) {
+		ret = cs43130_handle_device_data(cs43130);
 		if (ret != 0)
 			return ret;
 	}
+
 	for (i = 0; i < ARRAY_SIZE(cs43130->supplies); i++)
 		cs43130->supplies[i].supply = cs43130_supply_names[i];
 
-	ret = devm_regulator_bulk_get(&client->dev,
+	ret = devm_regulator_bulk_get(cs43130->dev,
 				      ARRAY_SIZE(cs43130->supplies),
 				      cs43130->supplies);
 	if (ret != 0) {
-		dev_err(&client->dev, "Failed to request supplies: %d\n", ret);
+		dev_err(cs43130->dev, "Failed to request supplies: %d\n", ret);
 		return ret;
 	}
 	ret = regulator_bulk_enable(ARRAY_SIZE(cs43130->supplies),
 				    cs43130->supplies);
 	if (ret != 0) {
-		dev_err(&client->dev, "Failed to enable supplies: %d\n", ret);
+		dev_err(cs43130->dev, "Failed to enable supplies: %d\n", ret);
 		return ret;
 	}
 
-	cs43130->reset_gpio = devm_gpiod_get_optional(&client->dev,
+	cs43130->reset_gpio = devm_gpiod_get_optional(cs43130->dev,
 						      "reset", GPIOD_OUT_LOW);
 	if (IS_ERR(cs43130->reset_gpio)) {
 		ret = PTR_ERR(cs43130->reset_gpio);
@@ -2470,7 +2474,7 @@ static int cs43130_i2c_probe(struct i2c_client *client)
 	devid = cirrus_read_device_id(cs43130->regmap, CS43130_DEVID_AB);
 	if (devid < 0) {
 		ret = devid;
-		dev_err(&client->dev, "Failed to read device ID: %d\n", ret);
+		dev_err(cs43130->dev, "Failed to read device ID: %d\n", ret);
 		goto err;
 	}
 
@@ -2481,7 +2485,7 @@ static int cs43130_i2c_probe(struct i2c_client *client)
 	case CS43198_CHIP_ID:
 		break;
 	default:
-		dev_err(&client->dev,
+		dev_err(cs43130->dev,
 			"CS43130 Device ID %X. Expected ID %X, %X, %X or %X\n",
 			devid, CS43130_CHIP_ID, CS4399_CHIP_ID,
 			CS43131_CHIP_ID, CS43198_CHIP_ID);
@@ -2492,11 +2496,11 @@ static int cs43130_i2c_probe(struct i2c_client *client)
 	cs43130->dev_id = devid;
 	ret = regmap_read(cs43130->regmap, CS43130_REV_ID, &reg);
 	if (ret < 0) {
-		dev_err(&client->dev, "Get Revision ID failed\n");
+		dev_err(cs43130->dev, "Get Revision ID failed\n");
 		goto err;
 	}
 
-	dev_info(&client->dev,
+	dev_info(cs43130->dev,
 		 "Cirrus Logic CS43130 (%x), Revision: %02X\n", devid,
 		 reg & 0xFF);
 
@@ -2506,21 +2510,21 @@ static int cs43130_i2c_probe(struct i2c_client *client)
 	init_completion(&cs43130->pll_rdy);
 	init_completion(&cs43130->hpload_evt);
 
-	ret = devm_request_threaded_irq(&client->dev, client->irq,
+	ret = devm_request_threaded_irq(cs43130->dev, client->irq,
 					NULL, cs43130_irq_thread,
 					IRQF_ONESHOT | IRQF_TRIGGER_LOW,
 					"cs43130", cs43130);
 	if (ret != 0) {
-		dev_err(&client->dev, "Failed to request IRQ: %d\n", ret);
+		dev_err(cs43130->dev, "Failed to request IRQ: %d\n", ret);
 		goto err;
 	}
 
 	cs43130->mclk_int_src = CS43130_MCLK_SRC_RCO;
 
-	pm_runtime_set_autosuspend_delay(&client->dev, 100);
-	pm_runtime_use_autosuspend(&client->dev);
-	pm_runtime_set_active(&client->dev);
-	pm_runtime_enable(&client->dev);
+	pm_runtime_set_autosuspend_delay(cs43130->dev, 100);
+	pm_runtime_use_autosuspend(cs43130->dev);
+	pm_runtime_set_active(cs43130->dev);
+	pm_runtime_enable(cs43130->dev);
 
 	switch (cs43130->dev_id) {
 	case CS43130_CHIP_ID:
@@ -2556,11 +2560,11 @@ static int cs43130_i2c_probe(struct i2c_client *client)
 		break;
 	}
 
-	ret = devm_snd_soc_register_component(&client->dev,
+	ret = devm_snd_soc_register_component(cs43130->dev,
 				     &soc_component_dev_cs43130,
 				     cs43130_dai, ARRAY_SIZE(cs43130_dai));
 	if (ret < 0) {
-		dev_err(&client->dev,
+		dev_err(cs43130->dev,
 			"snd_soc_register_component failed with ret = %d\n", ret);
 		goto err;
 	}
@@ -2598,15 +2602,15 @@ static void cs43130_i2c_remove(struct i2c_client *client)
 		cancel_work_sync(&cs43130->work);
 		flush_workqueue(cs43130->wq);
 
-		device_remove_file(&client->dev, &dev_attr_hpload_dc_l);
-		device_remove_file(&client->dev, &dev_attr_hpload_dc_r);
-		device_remove_file(&client->dev, &dev_attr_hpload_ac_l);
-		device_remove_file(&client->dev, &dev_attr_hpload_ac_r);
+		device_remove_file(cs43130->dev, &dev_attr_hpload_dc_l);
+		device_remove_file(cs43130->dev, &dev_attr_hpload_dc_r);
+		device_remove_file(cs43130->dev, &dev_attr_hpload_ac_l);
+		device_remove_file(cs43130->dev, &dev_attr_hpload_ac_r);
 	}
 
 	gpiod_set_value_cansleep(cs43130->reset_gpio, 0);
 
-	pm_runtime_disable(&client->dev);
+	pm_runtime_disable(cs43130->dev);
 	regulator_bulk_disable(CS43130_NUM_SUPPLIES, cs43130->supplies);
 }
 
@@ -2669,6 +2673,7 @@ static const struct dev_pm_ops cs43130_runtime_pm = {
 			   NULL)
 };
 
+#if IS_ENABLED(CONFIG_OF)
 static const struct of_device_id cs43130_of_match[] = {
 	{.compatible = "cirrus,cs43130",},
 	{.compatible = "cirrus,cs4399",},
@@ -2678,6 +2683,17 @@ static const struct of_device_id cs43130_of_match[] = {
 };
 
 MODULE_DEVICE_TABLE(of, cs43130_of_match);
+#endif
+
+#if IS_ENABLED(CONFIG_ACPI)
+static const struct acpi_device_id cs43130_acpi_match[] = {
+	{ "CSC4399", 0 },
+	{}
+};
+
+MODULE_DEVICE_TABLE(acpi, cs43130_acpi_match);
+#endif
+
 
 static const struct i2c_device_id cs43130_i2c_id[] = {
 	{"cs43130", 0},
@@ -2691,9 +2707,10 @@ MODULE_DEVICE_TABLE(i2c, cs43130_i2c_id);
 
 static struct i2c_driver cs43130_i2c_driver = {
 	.driver = {
-		.name		= "cs43130",
-		.of_match_table	= cs43130_of_match,
-		.pm             = &cs43130_runtime_pm,
+		.name			= "cs43130",
+		.of_match_table		= of_match_ptr(cs43130_of_match),
+		.acpi_match_table	= ACPI_PTR(cs43130_acpi_match),
+		.pm			= &cs43130_runtime_pm,
 	},
 	.id_table	= cs43130_i2c_id,
 	.probe		= cs43130_i2c_probe,
diff --git a/sound/soc/codecs/cs43130.h b/sound/soc/codecs/cs43130.h
index 90e8895275e7..2f5ec3888103 100644
--- a/sound/soc/codecs/cs43130.h
+++ b/sound/soc/codecs/cs43130.h
@@ -500,6 +500,7 @@ struct cs43130_dai {
 };
 
 struct	cs43130_private {
+	struct device			*dev;
 	struct snd_soc_component	*component;
 	struct regmap			*regmap;
 	struct regulator_bulk_data	supplies[CS43130_NUM_SUPPLIES];
diff --git a/sound/soc/codecs/es8326.c b/sound/soc/codecs/es8326.c
old mode 100644
new mode 100755
index 6c263086c44d..fa890f6205e2
--- a/sound/soc/codecs/es8326.c
+++ b/sound/soc/codecs/es8326.c
@@ -132,6 +132,11 @@ static const struct snd_soc_dapm_widget es8326_dapm_widgets[] = {
 	SND_SOC_DAPM_PGA("LHPMIX", ES8326_DAC2HPMIX, 7, 0, NULL, 0),
 	SND_SOC_DAPM_PGA("RHPMIX", ES8326_DAC2HPMIX, 3, 0, NULL, 0),
 
+	SND_SOC_DAPM_REG(snd_soc_dapm_supply, "HPOR Supply", ES8326_HP_CAL,
+			 4, 7, 0, 0),
+	SND_SOC_DAPM_REG(snd_soc_dapm_supply, "HPOL Supply", ES8326_HP_CAL,
+			 0, 7, 0, 0),
+
 	SND_SOC_DAPM_OUTPUT("HPOL"),
 	SND_SOC_DAPM_OUTPUT("HPOR"),
 };
@@ -156,6 +161,9 @@ static const struct snd_soc_dapm_route es8326_dapm_routes[] = {
 	{"LHPMIX", NULL, "Left DAC"},
 	{"RHPMIX", NULL, "Right DAC"},
 
+	{"HPOR", NULL, "HPOR Supply"},
+	{"HPOL", NULL, "HPOL Supply"},
+
 	{"HPOL", NULL, "LHPMIX"},
 	{"HPOR", NULL, "RHPMIX"},
 };
@@ -198,77 +206,108 @@ struct _coeff_div {
 
 /* codec hifi mclk clock divider coefficients */
 /* {ratio, LRCK, MCLK, REG04, REG05, REG06, REG07, REG08, REG09, REG10, REG11} */
-static const struct _coeff_div coeff_div[] = {
-	{32, 8000, 256000, 0x60, 0x00, 0x0F, 0x75, 0x0A, 0x1B, 0x1F, 0x7F},
-	{32, 16000, 512000, 0x20, 0x00, 0x0D, 0x75, 0x0A, 0x1B, 0x1F, 0x3F},
-	{32, 44100, 1411200, 0x00, 0x00, 0x13, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
-	{32, 48000, 1536000, 0x00, 0x00, 0x13, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
-	{36, 8000, 288000, 0x20, 0x00, 0x0D, 0x75, 0x0A, 0x1B, 0x23, 0x47},
-	{36, 16000, 576000, 0x20, 0x00, 0x0D, 0x75, 0x0A, 0x1B, 0x23, 0x47},
-	{48, 8000, 384000, 0x60, 0x02, 0x1F, 0x75, 0x0A, 0x1B, 0x1F, 0x7F},
-	{48, 16000, 768000, 0x20, 0x02, 0x0F, 0x75, 0x0A, 0x1B, 0x1F, 0x3F},
-	{48, 48000, 2304000, 0x00, 0x02, 0x0D, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
-	{64, 8000, 512000, 0x60, 0x00, 0x0D, 0x75, 0x0A, 0x1B, 0x1F, 0x7F},
-	{64, 16000, 1024000, 0x20, 0x00, 0x05, 0x75, 0x0A, 0x1B, 0x1F, 0x3F},
-
-	{64, 44100, 2822400, 0x00, 0x00, 0x11, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
-	{64, 48000, 3072000, 0x00, 0x00, 0x11, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
-	{72, 8000, 576000, 0x20, 0x00, 0x13, 0x35, 0x0A, 0x1B, 0x23, 0x47},
-	{72, 16000, 1152000, 0x20, 0x00, 0x05, 0x75, 0x0A, 0x1B, 0x23, 0x47},
-	{96, 8000, 768000, 0x60, 0x02, 0x1D, 0x75, 0x0A, 0x1B, 0x1F, 0x7F},
-	{96, 16000, 1536000, 0x20, 0x02, 0x0D, 0x75, 0x0A, 0x1B, 0x1F, 0x3F},
-	{100, 48000, 4800000, 0x04, 0x04, 0x3F, 0x6D, 0x38, 0x08, 0x4f, 0x1f},
-	{125, 48000, 6000000, 0x04, 0x04, 0x1F, 0x2D, 0x0A, 0x0A, 0x27, 0x27},
-	{128, 8000, 1024000, 0x60, 0x00, 0x13, 0x35, 0x0A, 0x1B, 0x1F, 0x7F},
-	{128, 16000, 2048000, 0x20, 0x00, 0x11, 0x35, 0x0A, 0x1B, 0x1F, 0x3F},
-
-	{128, 44100, 5644800, 0x00, 0x00, 0x01, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
-	{128, 48000, 6144000, 0x00, 0x00, 0x01, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
-	{144, 8000, 1152000, 0x20, 0x00, 0x03, 0x35, 0x0A, 0x1B, 0x23, 0x47},
-	{144, 16000, 2304000, 0x20, 0x00, 0x11, 0x35, 0x0A, 0x1B, 0x23, 0x47},
-	{192, 8000, 1536000, 0x60, 0x02, 0x0D, 0x75, 0x0A, 0x1B, 0x1F, 0x7F},
-	{192, 16000, 3072000, 0x20, 0x02, 0x05, 0x75, 0x0A, 0x1B, 0x1F, 0x3F},
-	{200, 48000, 9600000, 0x04, 0x04, 0x0F, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
-	{250, 48000, 12000000, 0x04, 0x04, 0x0F, 0x2D, 0x0A, 0x0A, 0x27, 0x27},
-	{256, 8000, 2048000, 0x60, 0x00, 0x11, 0x35, 0x0A, 0x1B, 0x1F, 0x7F},
-	{256, 16000, 4096000, 0x20, 0x00, 0x01, 0x35, 0x0A, 0x1B, 0x1F, 0x3F},
-
-	{256, 44100, 11289600, 0x00, 0x00, 0x10, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
-	{256, 48000, 12288000, 0x00, 0x00, 0x30, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
-	{288, 8000, 2304000, 0x20, 0x00, 0x01, 0x35, 0x0A, 0x1B, 0x23, 0x47},
-	{384, 8000, 3072000, 0x60, 0x02, 0x05, 0x75, 0x0A, 0x1B, 0x1F, 0x7F},
-	{384, 16000, 6144000, 0x20, 0x02, 0x03, 0x35, 0x0A, 0x1B, 0x1F, 0x3F},
-	{384, 48000, 18432000, 0x00, 0x02, 0x01, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
-	{400, 48000, 19200000, 0x09, 0x04, 0x0f, 0x6d, 0x3a, 0x0A, 0x4F, 0x1F},
-	{500, 48000, 24000000, 0x18, 0x04, 0x1F, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
-	{512, 8000, 4096000, 0x60, 0x00, 0x01, 0x35, 0x0A, 0x1B, 0x1F, 0x7F},
-	{512, 16000, 8192000, 0x20, 0x00, 0x10, 0x35, 0x0A, 0x1B, 0x1F, 0x3F},
-
-	{512, 44100, 22579200, 0x00, 0x00, 0x00, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
-	{512, 48000, 24576000, 0x00, 0x00, 0x00, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
-	{768, 8000, 6144000, 0x60, 0x02, 0x11, 0x35, 0x0A, 0x1B, 0x1F, 0x7F},
-	{768, 16000, 12288000, 0x20, 0x02, 0x01, 0x35, 0x0A, 0x1B, 0x1F, 0x3F},
-	{800, 48000, 38400000, 0x00, 0x18, 0x13, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
-	{1024, 8000, 8192000, 0x60, 0x00, 0x10, 0x35, 0x0A, 0x1B, 0x1F, 0x7F},
+static const struct _coeff_div coeff_div_v0[] = {
+	{64, 8000, 512000, 0x60, 0x01, 0x0F, 0x75, 0x0A, 0x1B, 0x1F, 0x7F},
+	{64, 16000, 1024000, 0x20, 0x00, 0x33, 0x35, 0x0A, 0x1B, 0x1F, 0x3F},
+	{64, 44100, 2822400, 0xE0, 0x00, 0x03, 0x2D, 0x4A, 0x0A, 0x1F, 0x1F},
+	{64, 48000, 3072000, 0xE0, 0x00, 0x03, 0x2D, 0x4A, 0x0A, 0x1F, 0x1F},
+	{128, 8000, 1024000, 0x60, 0x00, 0x33, 0x35, 0x0A, 0x1B, 0x1F, 0x7F},
+	{128, 16000, 2048000, 0x20, 0x00, 0x03, 0x35, 0x0A, 0x1B, 0x1F, 0x3F},
+	{128, 44100, 5644800, 0xE0, 0x01, 0x03, 0x2D, 0x4A, 0x0A, 0x1F, 0x1F},
+	{128, 48000, 6144000, 0xE0, 0x01, 0x03, 0x2D, 0x4A, 0x0A, 0x1F, 0x1F},
+
+	{192, 32000, 6144000, 0xE0, 0x02, 0x03, 0x2D, 0x4A, 0x0A, 0x1F, 0x1F},
+	{256, 8000, 2048000, 0x60, 0x00, 0x03, 0x35, 0x0A, 0x1B, 0x1F, 0x7F},
+	{256, 16000, 4096000, 0x20, 0x01, 0x03, 0x35, 0x0A, 0x1B, 0x1F, 0x3F},
+	{256, 44100, 11289600, 0xE0, 0x00, 0x30, 0x2D, 0x4A, 0x0A, 0x1F, 0x1F},
+	{256, 48000, 12288000, 0xE0, 0x00, 0x30, 0x2D, 0x4A, 0x0A, 0x1F, 0x1F},
+	{384, 32000, 12288000, 0xE0, 0x05, 0x03, 0x2D, 0x4A, 0x0A, 0x1F, 0x1F},
+	{400, 48000, 19200000, 0xE9, 0x04, 0x0F, 0x6d, 0x4A, 0x0A, 0x1F, 0x1F},
+
+	{500, 48000, 24000000, 0xF8, 0x04, 0x3F, 0x6D, 0x4A, 0x0A, 0x1F, 0x1F},
+	{512, 8000, 4096000, 0x60, 0x01, 0x03, 0x35, 0x0A, 0x1B, 0x1F, 0x7F},
+	{512, 16000, 8192000, 0x20, 0x00, 0x30, 0x35, 0x0A, 0x1B, 0x1F, 0x3F},
+	{512, 44100, 22579200, 0xE0, 0x00, 0x00, 0x2D, 0x4A, 0x0A, 0x1F, 0x1F},
+	{512, 48000, 24576000, 0xE0, 0x00, 0x00, 0x2D, 0x4A, 0x0A, 0x1F, 0x1F},
+	{768, 32000, 24576000, 0xE0, 0x02, 0x30, 0x2D, 0x4A, 0x0A, 0x1F, 0x1F},
+	{1024, 8000, 8192000, 0x60, 0x00, 0x30, 0x35, 0x0A, 0x1B, 0x1F, 0x7F},
 	{1024, 16000, 16384000, 0x20, 0x00, 0x00, 0x35, 0x0A, 0x1B, 0x1F, 0x3F},
-	{1152, 16000, 18432000, 0x20, 0x08, 0x11, 0x35, 0x0A, 0x1B, 0x1F, 0x3F},
-	{1536, 8000, 12288000, 0x60, 0x02, 0x01, 0x35, 0x0A, 0x1B, 0x1F, 0x7F},
-
-	{1536, 16000, 24576000, 0x20, 0x02, 0x10, 0x35, 0x0A, 0x1B, 0x1F, 0x3F},
-	{1625, 8000, 13000000, 0x0C, 0x18, 0x1F, 0x2D, 0x0A, 0x0A, 0x27, 0x27},
-	{1625, 16000, 26000000, 0x0C, 0x18, 0x1F, 0x2D, 0x0A, 0x0A, 0x27, 0x27},
-	{2048, 8000, 16384000, 0x60, 0x00, 0x00, 0x35, 0x0A, 0x1B, 0x1F, 0x7F},
-	{2304, 8000, 18432000, 0x40, 0x02, 0x10, 0x35, 0x0A, 0x1B, 0x1F, 0x5F},
-	{3072, 8000, 24576000, 0x60, 0x02, 0x10, 0x35, 0x0A, 0x1B, 0x1F, 0x7F},
-	{3250, 8000, 26000000, 0x0C, 0x18, 0x0F, 0x2D, 0x0A, 0x0A, 0x27, 0x27},
+};
 
+static const struct _coeff_div coeff_div_v3[] = {
+	{32, 8000, 256000, 0x60, 0x00, 0x0F, 0x75, 0x8A, 0x1B, 0x1F, 0x7F},
+	{32, 16000, 512000, 0x20, 0x00, 0x0D, 0x75, 0x8A, 0x1B, 0x1F, 0x3F},
+	{32, 44100, 1411200, 0x00, 0x00, 0x13, 0x2D, 0x8A, 0x0A, 0x1F, 0x1F},
+	{32, 48000, 1536000, 0x00, 0x00, 0x13, 0x2D, 0x8A, 0x0A, 0x1F, 0x1F},
+	{36, 8000, 288000, 0x20, 0x00, 0x0D, 0x75, 0x8A, 0x1B, 0x23, 0x47},
+	{36, 16000, 576000, 0x20, 0x00, 0x0D, 0x75, 0x8A, 0x1B, 0x23, 0x47},
+	{48, 8000, 384000, 0x60, 0x02, 0x1F, 0x75, 0x8A, 0x1B, 0x1F, 0x7F},
+	{48, 16000, 768000, 0x20, 0x02, 0x0F, 0x75, 0x8A, 0x1B, 0x1F, 0x3F},
+	{48, 48000, 2304000, 0x00, 0x02, 0x0D, 0x2D, 0x8A, 0x0A, 0x1F, 0x1F},
+
+	{64, 8000, 512000, 0x60, 0x00, 0x35, 0x75, 0x8A, 0x1B, 0x1F, 0x7F},
+	{64, 16000, 1024000, 0x20, 0x00, 0x05, 0x75, 0x8A, 0x1B, 0x1F, 0x3F},
+	{64, 44100, 2822400, 0xE0, 0x00, 0x31, 0x2D, 0xCA, 0x0A, 0x1F, 0x1F},
+	{64, 48000, 3072000, 0xE0, 0x00, 0x31, 0x2D, 0xCA, 0x0A, 0x1F, 0x1F},
+	{72, 8000, 576000, 0x20, 0x00, 0x13, 0x35, 0x8A, 0x1B, 0x23, 0x47},
+	{72, 16000, 1152000, 0x20, 0x00, 0x05, 0x75, 0x8A, 0x1B, 0x23, 0x47},
+	{96, 8000, 768000, 0x60, 0x02, 0x1D, 0x75, 0x8A, 0x1B, 0x1F, 0x7F},
+	{96, 16000, 1536000, 0x20, 0x02, 0x0D, 0x75, 0x8A, 0x1B, 0x1F, 0x3F},
+	{100, 48000, 4800000, 0x04, 0x04, 0x3F, 0x6D, 0xB8, 0x08, 0x4f, 0x1f},
+	{125, 48000, 6000000, 0x04, 0x04, 0x1F, 0x2D, 0x8A, 0x0A, 0x27, 0x27},
+
+	{128, 8000, 1024000, 0x60, 0x00, 0x05, 0x75, 0x8A, 0x1B, 0x1F, 0x7F},
+	{128, 16000, 2048000, 0x20, 0x00, 0x31, 0x35, 0x8A, 0x1B, 0x1F, 0x3F},
+	{128, 44100, 5644800, 0xE0, 0x00, 0x01, 0x2D, 0xCA, 0x0A, 0x1F, 0x1F},
+	{128, 48000, 6144000, 0xE0, 0x00, 0x01, 0x2D, 0xCA, 0x0A, 0x1F, 0x1F},
+	{144, 8000, 1152000, 0x20, 0x00, 0x03, 0x35, 0x8A, 0x1B, 0x23, 0x47},
+	{144, 16000, 2304000, 0x20, 0x00, 0x11, 0x35, 0x8A, 0x1B, 0x23, 0x47},
+	{192, 8000, 1536000, 0x60, 0x02, 0x0D, 0x75, 0x8A, 0x1B, 0x1F, 0x7F},
+	{192, 32000, 6144000, 0xE0, 0x02, 0x31, 0x2D, 0xCA, 0x0A, 0x1F, 0x1F},
+	{192, 16000, 3072000, 0x20, 0x02, 0x05, 0x75, 0xCA, 0x1B, 0x1F, 0x3F},
+
+	{200, 48000, 9600000, 0x04, 0x04, 0x0F, 0x2D, 0xCA, 0x0A, 0x1F, 0x1F},
+	{250, 48000, 12000000, 0x04, 0x04, 0x0F, 0x2D, 0xCA, 0x0A, 0x27, 0x27},
+	{256, 8000, 2048000, 0x60, 0x00, 0x31, 0x35, 0x8A, 0x1B, 0x1F, 0x7F},
+	{256, 16000, 4096000, 0x20, 0x00, 0x01, 0x35, 0x8A, 0x1B, 0x1F, 0x3F},
+	{256, 44100, 11289600, 0xE0, 0x00, 0x30, 0x2D, 0xCA, 0x0A, 0x1F, 0x1F},
+	{256, 48000, 12288000, 0xE0, 0x00, 0x30, 0x2D, 0xCA, 0x0A, 0x1F, 0x1F},
+	{288, 8000, 2304000, 0x20, 0x00, 0x01, 0x35, 0x8A, 0x1B, 0x23, 0x47},
+	{384, 8000, 3072000, 0x60, 0x02, 0x05, 0x75, 0x8A, 0x1B, 0x1F, 0x7F},
+	{384, 16000, 6144000, 0x20, 0x02, 0x03, 0x35, 0x8A, 0x1B, 0x1F, 0x3F},
+	{384, 32000, 12288000, 0xE0, 0x02, 0x01, 0x2D, 0xCA, 0x0A, 0x1F, 0x1F},
+	{384, 48000, 18432000, 0x00, 0x02, 0x01, 0x2D, 0x8A, 0x0A, 0x1F, 0x1F},
+
+	{400, 48000, 19200000, 0xE4, 0x04, 0x35, 0x6d, 0xCA, 0x0A, 0x1F, 0x1F},
+	{500, 48000, 24000000, 0xF8, 0x04, 0x3F, 0x6D, 0xCA, 0x0A, 0x1F, 0x1F},
+	{512, 8000, 4096000, 0x60, 0x00, 0x01, 0x35, 0x8A, 0x1B, 0x1F, 0x7F},
+	{512, 16000, 8192000, 0x20, 0x00, 0x30, 0x35, 0x8A, 0x1B, 0x1F, 0x3F},
+	{512, 44100, 22579200, 0xE0, 0x00, 0x00, 0x2D, 0xCA, 0x0A, 0x1F, 0x1F},
+	{512, 48000, 24576000, 0xE0, 0x00, 0x00, 0x2D, 0xCA, 0x0A, 0x1F, 0x1F},
+	{768, 8000, 6144000, 0x60, 0x02, 0x11, 0x35, 0x8A, 0x1B, 0x1F, 0x7F},
+	{768, 16000, 12288000, 0x20, 0x02, 0x01, 0x35, 0x8A, 0x1B, 0x1F, 0x3F},
+	{768, 32000, 24576000, 0xE0, 0x02, 0x30, 0x2D, 0xCA, 0x0A, 0x1F, 0x1F},
+	{800, 48000, 38400000, 0x00, 0x18, 0x13, 0x2D, 0x8A, 0x0A, 0x1F, 0x1F},
+
+	{1024, 8000, 8192000, 0x60, 0x00, 0x30, 0x35, 0x8A, 0x1B, 0x1F, 0x7F},
+	{1024, 16000, 16384000, 0x20, 0x00, 0x00, 0x35, 0x8A, 0x1B, 0x1F, 0x3F},
+	{1152, 16000, 18432000, 0x20, 0x08, 0x11, 0x35, 0x8A, 0x1B, 0x1F, 0x3F},
+	{1536, 8000, 12288000, 0x60, 0x02, 0x01, 0x35, 0x8A, 0x1B, 0x1F, 0x7F},
+	{1536, 16000, 24576000, 0x20, 0x02, 0x10, 0x35, 0x8A, 0x1B, 0x1F, 0x3F},
+	{1625, 8000, 13000000, 0x0C, 0x18, 0x1F, 0x2D, 0x8A, 0x0A, 0x27, 0x27},
+	{1625, 16000, 26000000, 0x0C, 0x18, 0x1F, 0x2D, 0x8A, 0x0A, 0x27, 0x27},
+	{2048, 8000, 16384000, 0x60, 0x00, 0x00, 0x35, 0x8A, 0x1B, 0x1F, 0x7F},
+	{2304, 8000, 18432000, 0x40, 0x02, 0x10, 0x35, 0x8A, 0x1B, 0x1F, 0x5F},
+	{3072, 8000, 24576000, 0x60, 0x02, 0x10, 0x35, 0x8A, 0x1B, 0x1F, 0x7F},
+	{3250, 8000, 26000000, 0x0C, 0x18, 0x0F, 0x2D, 0x8A, 0x0A, 0x27, 0x27},
 };
 
-static inline int get_coeff(int mclk, int rate)
+static inline int get_coeff(int mclk, int rate, int array,
+				const struct _coeff_div *coeff_div)
 {
 	int i;
 
-	for (i = 0; i < ARRAY_SIZE(coeff_div); i++) {
+	for (i = 0; i < array; i++) {
 		if (coeff_div[i].rate == rate && coeff_div[i].mclk == mclk)
 			return i;
 	}
@@ -333,11 +372,19 @@ static int es8326_pcm_hw_params(struct snd_pcm_substream *substream,
 				struct snd_soc_dai *dai)
 {
 	struct snd_soc_component *component = dai->component;
+	const struct _coeff_div *coeff_div;
 	struct es8326_priv *es8326 = snd_soc_component_get_drvdata(component);
 	u8 srate = 0;
-	int coeff;
+	int coeff, array;
 
-	coeff = get_coeff(es8326->sysclk, params_rate(params));
+	if (es8326->version == 0) {
+		coeff_div =  coeff_div_v0;
+		array = ARRAY_SIZE(coeff_div_v0);
+	} else {
+		coeff_div =  coeff_div_v3;
+		array = ARRAY_SIZE(coeff_div_v3);
+	}
+	coeff = get_coeff(es8326->sysclk, params_rate(params), array, coeff_div);
 	/* bit size */
 	switch (params_format(params)) {
 	case SNDRV_PCM_FORMAT_S16_LE:
@@ -409,8 +456,8 @@ static int es8326_mute(struct snd_soc_dai *dai, int mute, int direction)
 			regmap_write(es8326->regmap, ES8326_HPR_OFFSET_INI, offset_r);
 			es8326->calibrated = true;
 		}
-		regmap_write(es8326->regmap, ES8326_HP_DRIVER, 0xa0);
-		regmap_write(es8326->regmap, ES8326_HP_VOL, 0x80);
+		regmap_write(es8326->regmap, ES8326_HP_DRIVER, 0xa1);
+		regmap_write(es8326->regmap, ES8326_HP_VOL, 0x91);
 		regmap_write(es8326->regmap, ES8326_HP_CAL, ES8326_HP_ON);
 		regmap_update_bits(es8326->regmap, ES8326_DAC_MUTE,
 				ES8326_MUTE_MASK, ~(ES8326_MUTE));
@@ -430,8 +477,6 @@ static int es8326_set_bias_level(struct snd_soc_component *codec,
 		if (ret)
 			return ret;
 
-		regmap_write(es8326->regmap, ES8326_RESET, 0x9f);
-		msleep(20);
 		regmap_update_bits(es8326->regmap, ES8326_DAC_DSM, 0x01, 0x00);
 		regmap_write(es8326->regmap, ES8326_INTOUT_IO, es8326->interrupt_clk);
 		regmap_write(es8326->regmap, ES8326_SDINOUT1_IO,
@@ -440,19 +485,21 @@ static int es8326_set_bias_level(struct snd_soc_component *codec,
 		regmap_write(es8326->regmap, ES8326_PGA_PDN, 0x40);
 		regmap_write(es8326->regmap, ES8326_ANA_PDN, 0x00);
 		regmap_update_bits(es8326->regmap,  ES8326_CLK_CTL, 0x20, 0x20);
-		regmap_write(es8326->regmap, ES8326_RESET, ES8326_CSM_ON);
+
+		regmap_update_bits(es8326->regmap, ES8326_RESET,
+				ES8326_CSM_ON, ES8326_CSM_ON);
 		break;
 	case SND_SOC_BIAS_PREPARE:
 		break;
 	case SND_SOC_BIAS_STANDBY:
-		break;
-	case SND_SOC_BIAS_OFF:
-		clk_disable_unprepare(es8326->mclk);
 		regmap_write(es8326->regmap, ES8326_ANA_PDN, 0x3b);
 		regmap_write(es8326->regmap, ES8326_VMIDSEL, 0x00);
 		regmap_update_bits(es8326->regmap, ES8326_CLK_CTL, 0x20, 0x00);
 		regmap_write(es8326->regmap, ES8326_SDINOUT1_IO, ES8326_IO_INPUT);
 		break;
+	case SND_SOC_BIAS_OFF:
+		clk_disable_unprepare(es8326->mclk);
+		break;
 	}
 
 	return 0;
@@ -594,7 +641,7 @@ static void es8326_jack_detect_handler(struct work_struct *work)
 	iface = snd_soc_component_read(comp, ES8326_HPDET_STA);
 	dev_dbg(comp->dev, "gpio flag %#04x", iface);
 
-	if (es8326->jack_remove_retry == 1) {
+	if ((es8326->jack_remove_retry == 1) && (es8326->version != ES8326_VERSION_B)) {
 		if (iface & ES8326_HPINSERT_FLAG)
 			es8326->jack_remove_retry = 2;
 		else
@@ -628,7 +675,7 @@ static void es8326_jack_detect_handler(struct work_struct *work)
 		/*
 		 * Inverted HPJACK_POL bit to trigger one IRQ to double check HP Removal event
 		 */
-		if (es8326->jack_remove_retry == 0) {
+		if ((es8326->jack_remove_retry == 0) && (es8326->version != ES8326_VERSION_B)) {
 			es8326->jack_remove_retry = 1;
 			dev_dbg(comp->dev, "remove event check, invert HPJACK_POL, cnt = %d\n",
 					es8326->jack_remove_retry);
@@ -644,14 +691,14 @@ static void es8326_jack_detect_handler(struct work_struct *work)
 		if (es8326->hp == 0) {
 			dev_dbg(comp->dev, "First insert, start OMTP/CTIA type check\n");
 			/*
-			 * set auto-check mode, then restart jack_detect_work after 100ms.
+			 * set auto-check mode, then restart jack_detect_work after 400ms.
 			 * Don't report jack status.
 			 */
 			regmap_update_bits(es8326->regmap, ES8326_HPDET_TYPE, 0x03, 0x01);
 			usleep_range(50000, 70000);
 			regmap_update_bits(es8326->regmap, ES8326_HPDET_TYPE, 0x03, 0x00);
 			queue_delayed_work(system_wq, &es8326->jack_detect_work,
-					msecs_to_jiffies(100));
+					msecs_to_jiffies(400));
 			es8326->hp = 1;
 			goto exit;
 		}
@@ -701,7 +748,7 @@ static irqreturn_t es8326_irq(int irq, void *dev_id)
 				   msecs_to_jiffies(10));
 	else
 		queue_delayed_work(system_wq, &es8326->jack_detect_work,
-				   msecs_to_jiffies(600));
+				   msecs_to_jiffies(300));
 
 out:
 	return IRQ_HANDLED;
@@ -722,13 +769,15 @@ static int es8326_calibrate(struct snd_soc_component *component)
 		regmap_write(es8326->regmap, ES8326_CLK_DIV1, 0x01);
 		regmap_write(es8326->regmap, ES8326_CLK_DLL, 0x30);
 		regmap_write(es8326->regmap, ES8326_CLK_MUX, 0xed);
+		regmap_write(es8326->regmap, ES8326_CLK_DAC_SEL, 0x08);
 		regmap_write(es8326->regmap, ES8326_CLK_TRI, 0xc1);
 		regmap_write(es8326->regmap, ES8326_DAC_MUTE, 0x03);
 		regmap_write(es8326->regmap, ES8326_ANA_VSEL, 0x7f);
-		regmap_write(es8326->regmap, ES8326_VMIDLOW, 0x33);
+		regmap_write(es8326->regmap, ES8326_VMIDLOW, 0x03);
 		regmap_write(es8326->regmap, ES8326_DAC2HPMIX, 0x88);
-		regmap_write(es8326->regmap, ES8326_HP_VOL, 0x80);
+		usleep_range(15000, 20000);
 		regmap_write(es8326->regmap, ES8326_HP_OFFSET_CAL, 0x8c);
+		usleep_range(15000, 20000);
 		regmap_write(es8326->regmap, ES8326_RESET, 0xc0);
 		usleep_range(15000, 20000);
 
@@ -766,27 +815,27 @@ static int es8326_resume(struct snd_soc_component *component)
 	regmap_write(es8326->regmap, ES8326_RESET, 0x1f);
 	regmap_write(es8326->regmap, ES8326_VMIDSEL, 0x0E);
 	usleep_range(10000, 15000);
-	regmap_write(es8326->regmap, ES8326_HPJACK_TIMER, 0x88);
+	regmap_write(es8326->regmap, ES8326_HPJACK_TIMER, 0xe9);
+	regmap_write(es8326->regmap, ES8326_ANA_MICBIAS, 0x4b);
 	/* set headphone default type and detect pin */
-	regmap_write(es8326->regmap, ES8326_HPDET_TYPE, 0x81);
+	regmap_write(es8326->regmap, ES8326_HPDET_TYPE, 0x83);
 	regmap_write(es8326->regmap, ES8326_CLK_RESAMPLE, 0x05);
+	regmap_write(es8326->regmap, ES8326_HP_MISC, 0x30);
 
 	/* set internal oscillator as clock source of headpone cp */
-	regmap_write(es8326->regmap, ES8326_CLK_DIV_CPC, 0x84);
+	regmap_write(es8326->regmap, ES8326_CLK_DIV_CPC, 0x89);
 	regmap_write(es8326->regmap, ES8326_CLK_CTL, ES8326_CLK_ON);
 	/* clock manager reset release */
 	regmap_write(es8326->regmap, ES8326_RESET, 0x17);
 	/* set headphone detection as half scan mode */
-	regmap_write(es8326->regmap, ES8326_HP_MISC, 0x08);
+	regmap_write(es8326->regmap, ES8326_HP_MISC, 0x30);
 	regmap_write(es8326->regmap, ES8326_PULLUP_CTL, 0x00);
 
 	/* enable headphone driver */
 	regmap_write(es8326->regmap, ES8326_HP_DRIVER, 0xa7);
 	usleep_range(2000, 5000);
-	regmap_write(es8326->regmap, ES8326_HP_DRIVER_REF, 0xab);
-	usleep_range(2000, 5000);
-	regmap_write(es8326->regmap, ES8326_HP_DRIVER_REF, 0xbb);
-	usleep_range(2000, 5000);
+	regmap_write(es8326->regmap, ES8326_HP_DRIVER_REF, 0xa3);
+	regmap_write(es8326->regmap, ES8326_HP_DRIVER_REF, 0xb3);
 	regmap_write(es8326->regmap, ES8326_HP_DRIVER, 0xa1);
 
 	regmap_write(es8326->regmap, ES8326_CLK_INV, 0x00);
@@ -800,9 +849,6 @@ static int es8326_resume(struct snd_soc_component *component)
 	/* set ADC and DAC in low power mode */
 	regmap_write(es8326->regmap, ES8326_ANA_LP, 0xf0);
 
-	/* force micbias on */
-	regmap_write(es8326->regmap, ES8326_ANA_MICBIAS, 0x4f);
-	regmap_write(es8326->regmap, ES8326_SYS_BIAS, 0x08);
 	regmap_write(es8326->regmap, ES8326_ANA_VSEL, 0x7F);
 	/* select vdda as micbias source */
 	regmap_write(es8326->regmap, ES8326_VMIDLOW, 0x23);
@@ -830,6 +876,7 @@ static int es8326_resume(struct snd_soc_component *component)
 			((es8326->version == ES8326_VERSION_B) ?
 			(ES8326_HP_DET_SRC_PIN9 | es8326->jack_pol) :
 			(ES8326_HP_DET_SRC_PIN9 | es8326->jack_pol | 0x04)));
+	regmap_write(es8326->regmap, ES8326_HP_VOL, 0x11);
 
 	es8326->jack_remove_retry = 0;
 	es8326->hp = 0;
diff --git a/sound/soc/codecs/hdac_hdmi.c b/sound/soc/codecs/hdac_hdmi.c
index b9c5ffbfb5ba..a188a89e40d9 100644
--- a/sound/soc/codecs/hdac_hdmi.c
+++ b/sound/soc/codecs/hdac_hdmi.c
@@ -16,6 +16,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/hdmi.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_eld.h>
 #include <sound/pcm_params.h>
 #include <sound/jack.h>
 #include <sound/soc.h>
diff --git a/sound/soc/codecs/hdmi-codec.c b/sound/soc/codecs/hdmi-codec.c
index 20da1eaa4f1c..a7bd1796aed6 100644
--- a/sound/soc/codecs/hdmi-codec.c
+++ b/sound/soc/codecs/hdmi-codec.c
@@ -17,6 +17,7 @@
 #include <sound/pcm_iec958.h>
 
 #include <drm/drm_crtc.h> /* This is only to get MAX_ELD_BYTES */
+#include <drm/drm_eld.h>
 
 #define HDMI_CODEC_CHMAP_IDX_UNKNOWN  -1
 
diff --git a/sound/soc/codecs/nau8810.c b/sound/soc/codecs/nau8810.c
index 47f000cd4d99..97a54059474c 100644
--- a/sound/soc/codecs/nau8810.c
+++ b/sound/soc/codecs/nau8810.c
@@ -169,6 +169,7 @@ static int nau8810_eq_get(struct snd_kcontrol *kcontrol,
 	struct soc_bytes_ext *params = (void *)kcontrol->private_value;
 	int i, reg, reg_val;
 	u16 *val;
+	__be16 tmp;
 
 	val = (u16 *)ucontrol->value.bytes.data;
 	reg = NAU8810_REG_EQ1;
@@ -177,8 +178,8 @@ static int nau8810_eq_get(struct snd_kcontrol *kcontrol,
 		/* conversion of 16-bit integers between native CPU format
 		 * and big endian format
 		 */
-		reg_val = cpu_to_be16(reg_val);
-		memcpy(val + i, &reg_val, sizeof(reg_val));
+		tmp = cpu_to_be16(reg_val);
+		memcpy(val + i, &tmp, sizeof(tmp));
 	}
 
 	return 0;
@@ -201,6 +202,7 @@ static int nau8810_eq_put(struct snd_kcontrol *kcontrol,
 	void *data;
 	u16 *val, value;
 	int i, reg, ret;
+	__be16 *tmp;
 
 	data = kmemdup(ucontrol->value.bytes.data,
 		params->max, GFP_KERNEL | GFP_DMA);
@@ -213,7 +215,8 @@ static int nau8810_eq_put(struct snd_kcontrol *kcontrol,
 		/* conversion of 16-bit integers between native CPU format
 		 * and big endian format
 		 */
-		value = be16_to_cpu(*(val + i));
+		tmp = (__be16 *)(val + i);
+		value = be16_to_cpup(tmp);
 		ret = regmap_write(nau8810->regmap, reg + i, value);
 		if (ret) {
 			dev_err(component->dev, "EQ configuration fail, register: %x ret: %d\n",
diff --git a/sound/soc/codecs/nau8821.c b/sound/soc/codecs/nau8821.c
index 6e1b6b26298a..012e347e6391 100644
--- a/sound/soc/codecs/nau8821.c
+++ b/sound/soc/codecs/nau8821.c
@@ -1738,6 +1738,10 @@ static int nau8821_read_device_properties(struct device *dev,
 		&nau8821->dmic_clk_threshold);
 	if (ret)
 		nau8821->dmic_clk_threshold = 3072000;
+	ret = device_property_read_u32(dev, "nuvoton,dmic-slew-rate",
+		&nau8821->dmic_slew_rate);
+	if (ret)
+		nau8821->dmic_slew_rate = 0;
 
 	return 0;
 }
@@ -1797,6 +1801,9 @@ static void nau8821_init_regs(struct nau8821 *nau8821)
 		NAU8821_ADC_SYNC_DOWN_MASK, NAU8821_ADC_SYNC_DOWN_64);
 	regmap_update_bits(regmap, NAU8821_R2C_DAC_CTRL1,
 		NAU8821_DAC_OVERSAMPLE_MASK, NAU8821_DAC_OVERSAMPLE_64);
+	regmap_update_bits(regmap, NAU8821_R13_DMIC_CTRL,
+		NAU8821_DMIC_SLEW_MASK, nau8821->dmic_slew_rate <<
+		NAU8821_DMIC_SLEW_SFT);
 	if (nau8821->left_input_single_end) {
 		regmap_update_bits(regmap, NAU8821_R6B_PGA_MUTE,
 			NAU8821_MUTE_MICNL_EN, NAU8821_MUTE_MICNL_EN);
diff --git a/sound/soc/codecs/nau8821.h b/sound/soc/codecs/nau8821.h
index 00a888ed07ce..62eaad130b2e 100644
--- a/sound/soc/codecs/nau8821.h
+++ b/sound/soc/codecs/nau8821.h
@@ -236,6 +236,8 @@
 #define NAU8821_DMIC_SRC_MASK	(0x3 << NAU8821_DMIC_SRC_SFT)
 #define NAU8821_CLK_DMIC_SRC	(0x2 << NAU8821_DMIC_SRC_SFT)
 #define NAU8821_DMIC_EN_SFT	0
+#define NAU8821_DMIC_SLEW_SFT  8
+#define NAU8821_DMIC_SLEW_MASK (0x7 << NAU8821_DMIC_SLEW_SFT)
 
 /* GPIO12_CTRL (0x1a) */
 #define NAU8821_JKDET_PULL_UP	(0x1 << 11) /* 0 - pull down, 1 - pull up */
@@ -573,6 +575,7 @@ struct nau8821 {
 	int jack_eject_debounce;
 	int fs;
 	int dmic_clk_threshold;
+	int dmic_slew_rate;
 	int key_enable;
 };
 
diff --git a/sound/soc/codecs/nau8822.c b/sound/soc/codecs/nau8822.c
index ff3024899f45..7199d734c79f 100644
--- a/sound/soc/codecs/nau8822.c
+++ b/sound/soc/codecs/nau8822.c
@@ -184,6 +184,7 @@ static int nau8822_eq_get(struct snd_kcontrol *kcontrol,
 	struct soc_bytes_ext *params = (void *)kcontrol->private_value;
 	int i, reg;
 	u16 reg_val, *val;
+	__be16 tmp;
 
 	val = (u16 *)ucontrol->value.bytes.data;
 	reg = NAU8822_REG_EQ1;
@@ -192,8 +193,8 @@ static int nau8822_eq_get(struct snd_kcontrol *kcontrol,
 		/* conversion of 16-bit integers between native CPU format
 		 * and big endian format
 		 */
-		reg_val = cpu_to_be16(reg_val);
-		memcpy(val + i, &reg_val, sizeof(reg_val));
+		tmp = cpu_to_be16(reg_val);
+		memcpy(val + i, &tmp, sizeof(tmp));
 	}
 
 	return 0;
@@ -216,6 +217,7 @@ static int nau8822_eq_put(struct snd_kcontrol *kcontrol,
 	void *data;
 	u16 *val, value;
 	int i, reg, ret;
+	__be16 *tmp;
 
 	data = kmemdup(ucontrol->value.bytes.data,
 		params->max, GFP_KERNEL | GFP_DMA);
@@ -228,7 +230,8 @@ static int nau8822_eq_put(struct snd_kcontrol *kcontrol,
 		/* conversion of 16-bit integers between native CPU format
 		 * and big endian format
 		 */
-		value = be16_to_cpu(*(val + i));
+		tmp = (__be16 *)(val + i);
+		value = be16_to_cpup(tmp);
 		ret = snd_soc_component_write(component, reg + i, value);
 		if (ret) {
 			dev_err(component->dev,
diff --git a/sound/soc/codecs/rt5682s.c b/sound/soc/codecs/rt5682s.c
index c261c33c4be7..3322056bbb3b 100644
--- a/sound/soc/codecs/rt5682s.c
+++ b/sound/soc/codecs/rt5682s.c
@@ -2971,6 +2971,8 @@ static int rt5682s_parse_dt(struct rt5682s_priv *rt5682s, struct device *dev)
 		&rt5682s->pdata.dmic_delay);
 	device_property_read_u32(dev, "realtek,amic-delay-ms",
 		&rt5682s->pdata.amic_delay);
+	device_property_read_u32(dev, "realtek,ldo-sel",
+		&rt5682s->pdata.ldo_dacref);
 
 	if (device_property_read_string_array(dev, "clock-output-names",
 					      rt5682s->pdata.dai_clk_names,
@@ -3250,6 +3252,27 @@ static int rt5682s_i2c_probe(struct i2c_client *i2c)
 		break;
 	}
 
+	/* LDO output voltage control */
+	switch (rt5682s->pdata.ldo_dacref) {
+	case RT5682S_LDO_1_607V:
+		break;
+	case RT5682S_LDO_1_5V:
+		regmap_update_bits(rt5682s->regmap, RT5682S_BIAS_CUR_CTRL_7,
+			RT5682S_LDO_DACREF_MASK, RT5682S_LDO_DACREF_1_5V);
+		break;
+	case RT5682S_LDO_1_406V:
+		regmap_update_bits(rt5682s->regmap, RT5682S_BIAS_CUR_CTRL_7,
+			RT5682S_LDO_DACREF_MASK, RT5682S_LDO_DACREF_1_406V);
+		break;
+	case RT5682S_LDO_1_731V:
+		regmap_update_bits(rt5682s->regmap, RT5682S_BIAS_CUR_CTRL_7,
+			RT5682S_LDO_DACREF_MASK, RT5682S_LDO_DACREF_1_731V);
+		break;
+	default:
+		dev_warn(&i2c->dev, "invalid LDO output setting.\n");
+		break;
+	}
+
 	INIT_DELAYED_WORK(&rt5682s->jack_detect_work, rt5682s_jack_detect_handler);
 	INIT_DELAYED_WORK(&rt5682s->jd_check_work, rt5682s_jd_check_handler);
 
diff --git a/sound/soc/codecs/rt5682s.h b/sound/soc/codecs/rt5682s.h
index 1d79d432d0d8..67f42898de96 100644
--- a/sound/soc/codecs/rt5682s.h
+++ b/sound/soc/codecs/rt5682s.h
@@ -1263,6 +1263,13 @@
 #define RT5682S_JDH_NO_PLUG			(0x1 << 4)
 #define RT5682S_JDH_PLUG			(0x0 << 4)
 
+/* Bias current control 7  (0x0110) */
+#define RT5682S_LDO_DACREF_MASK			(0x3 << 4)
+#define RT5682S_LDO_DACREF_1_607V		(0x0 << 4)
+#define RT5682S_LDO_DACREF_1_5V			(0x1 << 4)
+#define RT5682S_LDO_DACREF_1_406V		(0x2 << 4)
+#define RT5682S_LDO_DACREF_1_731V		(0x3 << 4)
+
 /* Charge Pump Internal Register1 (0x0125) */
 #define RT5682S_CP_CLK_HP_MASK			(0x3 << 4)
 #define RT5682S_CP_CLK_HP_100KHZ		(0x0 << 4)
diff --git a/sound/soc/codecs/wm8974.c b/sound/soc/codecs/wm8974.c
index 044b6f604c09..260bac695b20 100644
--- a/sound/soc/codecs/wm8974.c
+++ b/sound/soc/codecs/wm8974.c
@@ -186,7 +186,7 @@ SOC_DAPM_SINGLE("PCM Playback Switch", WM8974_MONOMIX, 0, 1, 0),
 
 /* Boost mixer */
 static const struct snd_kcontrol_new wm8974_boost_mixer[] = {
-SOC_DAPM_SINGLE("Aux Switch", WM8974_INPPGA, 6, 1, 1),
+SOC_DAPM_SINGLE("PGA Switch", WM8974_INPPGA, 6, 1, 1),
 };
 
 /* Input PGA */
@@ -246,8 +246,8 @@ static const struct snd_soc_dapm_route wm8974_dapm_routes[] = {
 
 	/* Boost Mixer */
 	{"ADC", NULL, "Boost Mixer"},
-	{"Boost Mixer", "Aux Switch", "Aux Input"},
-	{"Boost Mixer", NULL, "Input PGA"},
+	{"Boost Mixer", NULL, "Aux Input"},
+	{"Boost Mixer", "PGA Switch", "Input PGA"},
 	{"Boost Mixer", NULL, "MICP"},
 
 	/* Input PGA */
diff --git a/sound/soc/fsl/fsl_mqs.c b/sound/soc/fsl/fsl_mqs.c
index f2d74ec05cdf..86704ba5f6f0 100644
--- a/sound/soc/fsl/fsl_mqs.c
+++ b/sound/soc/fsl/fsl_mqs.c
@@ -10,7 +10,6 @@
 #include <linux/moduleparam.h>
 #include <linux/mfd/syscon.h>
 #include <linux/mfd/syscon/imx6q-iomuxc-gpr.h>
-#include <linux/of.h>
 #include <linux/pm_runtime.h>
 #include <linux/of.h>
 #include <linux/pm.h>
diff --git a/sound/soc/fsl/fsl_sai.c b/sound/soc/fsl/fsl_sai.c
index 79e7c6b98a75..32bbe5056a63 100644
--- a/sound/soc/fsl/fsl_sai.c
+++ b/sound/soc/fsl/fsl_sai.c
@@ -673,6 +673,20 @@ static int fsl_sai_hw_params(struct snd_pcm_substream *substream,
 			   FSL_SAI_CR3_TRCE_MASK,
 			   FSL_SAI_CR3_TRCE((dl_cfg[dl_cfg_idx].mask[tx] & trce_mask)));
 
+	/*
+	 * When the TERE and FSD_MSTR enabled before configuring the word width
+	 * There will be no frame sync clock issue, because word width impact
+	 * the generation of frame sync clock.
+	 *
+	 * TERE enabled earlier only for i.MX8MP case for the hardware limitation,
+	 * We need to disable FSD_MSTR before configuring word width, then enable
+	 * FSD_MSTR bit for this specific case.
+	 */
+	if (sai->soc_data->mclk_with_tere && sai->mclk_direction_output &&
+	    !sai->is_consumer_mode)
+		regmap_update_bits(sai->regmap, FSL_SAI_xCR4(tx, ofs),
+				   FSL_SAI_CR4_FSD_MSTR, 0);
+
 	regmap_update_bits(sai->regmap, FSL_SAI_xCR4(tx, ofs),
 			   FSL_SAI_CR4_SYWD_MASK | FSL_SAI_CR4_FRSZ_MASK |
 			   FSL_SAI_CR4_CHMOD_MASK,
@@ -680,6 +694,13 @@ static int fsl_sai_hw_params(struct snd_pcm_substream *substream,
 	regmap_update_bits(sai->regmap, FSL_SAI_xCR5(tx, ofs),
 			   FSL_SAI_CR5_WNW_MASK | FSL_SAI_CR5_W0W_MASK |
 			   FSL_SAI_CR5_FBT_MASK, val_cr5);
+
+	/* Enable FSD_MSTR after configuring word width */
+	if (sai->soc_data->mclk_with_tere && sai->mclk_direction_output &&
+	    !sai->is_consumer_mode)
+		regmap_update_bits(sai->regmap, FSL_SAI_xCR4(tx, ofs),
+				   FSL_SAI_CR4_FSD_MSTR, FSL_SAI_CR4_FSD_MSTR);
+
 	regmap_write(sai->regmap, FSL_SAI_xMR(tx),
 		     ~0UL - ((1 << min(channels, slots)) - 1));
 
diff --git a/sound/soc/fsl/imx-rpmsg.c b/sound/soc/fsl/imx-rpmsg.c
index a0c5c35817dd..e5bd63dab10c 100644
--- a/sound/soc/fsl/imx-rpmsg.c
+++ b/sound/soc/fsl/imx-rpmsg.c
@@ -2,9 +2,8 @@
 // Copyright 2017-2020 NXP
 
 #include <linux/module.h>
-#include <linux/of.h>
+#include <linux/of_platform.h>
 #include <linux/of_reserved_mem.h>
-#include <linux/platform_device.h>
 #include <linux/i2c.h>
 #include <linux/of_gpio.h>
 #include <linux/slab.h>
@@ -21,8 +20,11 @@ struct imx_rpmsg {
 	struct snd_soc_dai_link dai;
 	struct snd_soc_card card;
 	unsigned long sysclk;
+	bool lpa;
 };
 
+static struct dev_pm_ops lpa_pm;
+
 static const struct snd_soc_dapm_widget imx_rpmsg_dapm_widgets[] = {
 	SND_SOC_DAPM_HP("Headphone Jack", NULL),
 	SND_SOC_DAPM_SPK("Ext Spk", NULL),
@@ -39,6 +41,58 @@ static int imx_rpmsg_late_probe(struct snd_soc_card *card)
 	struct device *dev = card->dev;
 	int ret;
 
+	if (data->lpa) {
+		struct snd_soc_component *codec_comp;
+		struct device_node *codec_np;
+		struct device_driver *codec_drv;
+		struct device *codec_dev = NULL;
+
+		codec_np = data->dai.codecs->of_node;
+		if (codec_np) {
+			struct platform_device *codec_pdev;
+			struct i2c_client *codec_i2c;
+
+			codec_i2c = of_find_i2c_device_by_node(codec_np);
+			if (codec_i2c)
+				codec_dev = &codec_i2c->dev;
+			if (!codec_dev) {
+				codec_pdev = of_find_device_by_node(codec_np);
+				if (codec_pdev)
+					codec_dev = &codec_pdev->dev;
+			}
+		}
+		if (codec_dev) {
+			codec_comp = snd_soc_lookup_component_nolocked(codec_dev, NULL);
+			if (codec_comp) {
+				int i, num_widgets;
+				const char *widgets;
+				struct snd_soc_dapm_context *dapm;
+
+				num_widgets = of_property_count_strings(data->card.dev->of_node,
+									"ignore-suspend-widgets");
+				for (i = 0; i < num_widgets; i++) {
+					of_property_read_string_index(data->card.dev->of_node,
+								      "ignore-suspend-widgets",
+								      i, &widgets);
+					dapm = snd_soc_component_get_dapm(codec_comp);
+					snd_soc_dapm_ignore_suspend(dapm, widgets);
+				}
+			}
+			codec_drv = codec_dev->driver;
+			if (codec_drv->pm) {
+				memcpy(&lpa_pm, codec_drv->pm, sizeof(lpa_pm));
+				lpa_pm.suspend = NULL;
+				lpa_pm.resume = NULL;
+				lpa_pm.freeze = NULL;
+				lpa_pm.thaw = NULL;
+				lpa_pm.poweroff = NULL;
+				lpa_pm.restore = NULL;
+				codec_drv->pm = &lpa_pm;
+			}
+			put_device(codec_dev);
+		}
+	}
+
 	if (!data->sysclk)
 		return 0;
 
@@ -138,6 +192,9 @@ static int imx_rpmsg_probe(struct platform_device *pdev)
 		goto fail;
 	}
 
+	if (of_property_read_bool(np, "fsl,enable-lpa"))
+		data->lpa = true;
+
 	data->card.dev = &pdev->dev;
 	data->card.owner = THIS_MODULE;
 	data->card.dapm_widgets = imx_rpmsg_dapm_widgets;
diff --git a/sound/soc/intel/avs/boards/da7219.c b/sound/soc/intel/avs/boards/da7219.c
index 6060894954df..c018f84fe025 100644
--- a/sound/soc/intel/avs/boards/da7219.c
+++ b/sound/soc/intel/avs/boards/da7219.c
@@ -277,16 +277,24 @@ static int avs_da7219_probe(struct platform_device *pdev)
 	return devm_snd_soc_register_card(dev, card);
 }
 
+static const struct platform_device_id avs_da7219_driver_ids[] = {
+	{
+		.name = "avs_da7219",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(platform, avs_da7219_driver_ids);
+
 static struct platform_driver avs_da7219_driver = {
 	.probe = avs_da7219_probe,
 	.driver = {
 		.name = "avs_da7219",
 		.pm = &snd_soc_pm_ops,
 	},
+	.id_table = avs_da7219_driver_ids,
 };
 
 module_platform_driver(avs_da7219_driver);
 
 MODULE_AUTHOR("Cezary Rojewski <cezary.rojewski@intel.com>");
 MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:avs_da7219");
diff --git a/sound/soc/intel/avs/boards/dmic.c b/sound/soc/intel/avs/boards/dmic.c
index c270646faf86..ba2bc7f689eb 100644
--- a/sound/soc/intel/avs/boards/dmic.c
+++ b/sound/soc/intel/avs/boards/dmic.c
@@ -77,15 +77,23 @@ static int avs_dmic_probe(struct platform_device *pdev)
 	return devm_snd_soc_register_card(dev, card);
 }
 
+static const struct platform_device_id avs_dmic_driver_ids[] = {
+	{
+		.name = "avs_dmic",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(platform, avs_dmic_driver_ids);
+
 static struct platform_driver avs_dmic_driver = {
 	.probe = avs_dmic_probe,
 	.driver = {
 		.name = "avs_dmic",
 		.pm = &snd_soc_pm_ops,
 	},
+	.id_table = avs_dmic_driver_ids,
 };
 
 module_platform_driver(avs_dmic_driver);
 
 MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:avs_dmic");
diff --git a/sound/soc/intel/avs/boards/es8336.c b/sound/soc/intel/avs/boards/es8336.c
index f972ef64d284..1090082e7d5b 100644
--- a/sound/soc/intel/avs/boards/es8336.c
+++ b/sound/soc/intel/avs/boards/es8336.c
@@ -307,15 +307,23 @@ static int avs_es8336_probe(struct platform_device *pdev)
 	return devm_snd_soc_register_card(dev, card);
 }
 
+static const struct platform_device_id avs_es8336_driver_ids[] = {
+	{
+		.name = "avs_es8336",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(platform, avs_es8336_driver_ids);
+
 static struct platform_driver avs_es8336_driver = {
 	.probe = avs_es8336_probe,
 	.driver = {
 		.name = "avs_es8336",
 		.pm = &snd_soc_pm_ops,
 	},
+	.id_table = avs_es8336_driver_ids,
 };
 
 module_platform_driver(avs_es8336_driver);
 
 MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:avs_es8336");
diff --git a/sound/soc/intel/avs/boards/hdaudio.c b/sound/soc/intel/avs/boards/hdaudio.c
index 8876558f19a1..844a918f9a81 100644
--- a/sound/soc/intel/avs/boards/hdaudio.c
+++ b/sound/soc/intel/avs/boards/hdaudio.c
@@ -218,12 +218,21 @@ static int avs_hdaudio_probe(struct platform_device *pdev)
 	return devm_snd_soc_register_card(dev, card);
 }
 
+static const struct platform_device_id avs_hdaudio_driver_ids[] = {
+	{
+		.name = "avs_hdaudio",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(platform, avs_hdaudio_driver_ids);
+
 static struct platform_driver avs_hdaudio_driver = {
 	.probe = avs_hdaudio_probe,
 	.driver = {
 		.name = "avs_hdaudio",
 		.pm = &snd_soc_pm_ops,
 	},
+	.id_table = avs_hdaudio_driver_ids,
 };
 
 module_platform_driver(avs_hdaudio_driver)
@@ -231,4 +240,3 @@ module_platform_driver(avs_hdaudio_driver)
 MODULE_DESCRIPTION("Intel HD-Audio machine driver");
 MODULE_AUTHOR("Cezary Rojewski <cezary.rojewski@intel.com>");
 MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:avs_hdaudio");
diff --git a/sound/soc/intel/avs/boards/i2s_test.c b/sound/soc/intel/avs/boards/i2s_test.c
index 3d03e1eed3a9..28f254eb0d03 100644
--- a/sound/soc/intel/avs/boards/i2s_test.c
+++ b/sound/soc/intel/avs/boards/i2s_test.c
@@ -185,15 +185,23 @@ static int avs_i2s_test_probe(struct platform_device *pdev)
 	return devm_snd_soc_register_card(dev, card);
 }
 
+static const struct platform_device_id avs_i2s_test_driver_ids[] = {
+	{
+		.name = "avs_i2s_test",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(platform, avs_i2s_test_driver_ids);
+
 static struct platform_driver avs_i2s_test_driver = {
 	.probe = avs_i2s_test_probe,
 	.driver = {
 		.name = "avs_i2s_test",
 		.pm = &snd_soc_pm_ops,
 	},
+	.id_table = avs_i2s_test_driver_ids,
 };
 
 module_platform_driver(avs_i2s_test_driver);
 
 MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:avs_i2s_test");
diff --git a/sound/soc/intel/avs/boards/max98357a.c b/sound/soc/intel/avs/boards/max98357a.c
index 6ba7b6564279..a83b95f25129 100644
--- a/sound/soc/intel/avs/boards/max98357a.c
+++ b/sound/soc/intel/avs/boards/max98357a.c
@@ -135,15 +135,23 @@ static int avs_max98357a_probe(struct platform_device *pdev)
 	return devm_snd_soc_register_card(dev, card);
 }
 
+static const struct platform_device_id avs_max98357a_driver_ids[] = {
+	{
+		.name = "avs_max98357a",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(platform, avs_max98357a_driver_ids);
+
 static struct platform_driver avs_max98357a_driver = {
 	.probe = avs_max98357a_probe,
 	.driver = {
 		.name = "avs_max98357a",
 		.pm = &snd_soc_pm_ops,
 	},
+	.id_table = avs_max98357a_driver_ids,
 };
 
 module_platform_driver(avs_max98357a_driver)
 
 MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:avs_max98357a");
diff --git a/sound/soc/intel/avs/boards/max98373.c b/sound/soc/intel/avs/boards/max98373.c
index cc7dfdf72083..3b980a025e6f 100644
--- a/sound/soc/intel/avs/boards/max98373.c
+++ b/sound/soc/intel/avs/boards/max98373.c
@@ -192,15 +192,23 @@ static int avs_max98373_probe(struct platform_device *pdev)
 	return devm_snd_soc_register_card(dev, card);
 }
 
+static const struct platform_device_id avs_max98373_driver_ids[] = {
+	{
+		.name = "avs_max98373",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(platform, avs_max98373_driver_ids);
+
 static struct platform_driver avs_max98373_driver = {
 	.probe = avs_max98373_probe,
 	.driver = {
 		.name = "avs_max98373",
 		.pm = &snd_soc_pm_ops,
 	},
+	.id_table = avs_max98373_driver_ids,
 };
 
 module_platform_driver(avs_max98373_driver)
 
 MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:avs_max98373");
diff --git a/sound/soc/intel/avs/boards/max98927.c b/sound/soc/intel/avs/boards/max98927.c
index fb0175f37d61..86dd2b228df3 100644
--- a/sound/soc/intel/avs/boards/max98927.c
+++ b/sound/soc/intel/avs/boards/max98927.c
@@ -189,15 +189,23 @@ static int avs_max98927_probe(struct platform_device *pdev)
 	return devm_snd_soc_register_card(dev, card);
 }
 
+static const struct platform_device_id avs_max98927_driver_ids[] = {
+	{
+		.name = "avs_max98927",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(platform, avs_max98927_driver_ids);
+
 static struct platform_driver avs_max98927_driver = {
 	.probe = avs_max98927_probe,
 	.driver = {
 		.name = "avs_max98927",
 		.pm = &snd_soc_pm_ops,
 	},
+	.id_table = avs_max98927_driver_ids,
 };
 
 module_platform_driver(avs_max98927_driver)
 
 MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:avs_max98927");
diff --git a/sound/soc/intel/avs/boards/nau8825.c b/sound/soc/intel/avs/boards/nau8825.c
index d98b5deb78c9..1c1e2083f474 100644
--- a/sound/soc/intel/avs/boards/nau8825.c
+++ b/sound/soc/intel/avs/boards/nau8825.c
@@ -294,15 +294,23 @@ static int avs_nau8825_probe(struct platform_device *pdev)
 	return devm_snd_soc_register_card(dev, card);
 }
 
+static const struct platform_device_id avs_nau8825_driver_ids[] = {
+	{
+		.name = "avs_nau8825",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(platform, avs_nau8825_driver_ids);
+
 static struct platform_driver avs_nau8825_driver = {
 	.probe = avs_nau8825_probe,
 	.driver = {
 		.name = "avs_nau8825",
 		.pm = &snd_soc_pm_ops,
 	},
+	.id_table = avs_nau8825_driver_ids,
 };
 
 module_platform_driver(avs_nau8825_driver)
 
 MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:avs_nau8825");
diff --git a/sound/soc/intel/avs/boards/probe.c b/sound/soc/intel/avs/boards/probe.c
index 411acaee74f9..a9469b5ecb40 100644
--- a/sound/soc/intel/avs/boards/probe.c
+++ b/sound/soc/intel/avs/boards/probe.c
@@ -50,15 +50,23 @@ static int avs_probe_mb_probe(struct platform_device *pdev)
 	return devm_snd_soc_register_card(dev, card);
 }
 
+static const struct platform_device_id avs_probe_mb_driver_ids[] = {
+	{
+		.name = "avs_probe_mb",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(platform, avs_probe_mb_driver_ids);
+
 static struct platform_driver avs_probe_mb_driver = {
 	.probe = avs_probe_mb_probe,
 	.driver = {
 		.name = "avs_probe_mb",
 		.pm = &snd_soc_pm_ops,
 	},
+	.id_table = avs_probe_mb_driver_ids,
 };
 
 module_platform_driver(avs_probe_mb_driver);
 
 MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:avs_probe_mb");
diff --git a/sound/soc/intel/avs/boards/rt274.c b/sound/soc/intel/avs/boards/rt274.c
index 157183b1de24..bfcb8845fd15 100644
--- a/sound/soc/intel/avs/boards/rt274.c
+++ b/sound/soc/intel/avs/boards/rt274.c
@@ -257,15 +257,23 @@ static int avs_rt274_probe(struct platform_device *pdev)
 	return devm_snd_soc_register_card(dev, card);
 }
 
+static const struct platform_device_id avs_rt274_driver_ids[] = {
+	{
+		.name = "avs_rt274",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(platform, avs_rt274_driver_ids);
+
 static struct platform_driver avs_rt274_driver = {
 	.probe = avs_rt274_probe,
 	.driver = {
 		.name = "avs_rt274",
 		.pm = &snd_soc_pm_ops,
 	},
+	.id_table = avs_rt274_driver_ids,
 };
 
 module_platform_driver(avs_rt274_driver);
 
 MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:avs_rt274");
diff --git a/sound/soc/intel/avs/boards/rt286.c b/sound/soc/intel/avs/boards/rt286.c
index 131237471e3e..28d7d86b1cc9 100644
--- a/sound/soc/intel/avs/boards/rt286.c
+++ b/sound/soc/intel/avs/boards/rt286.c
@@ -228,15 +228,23 @@ static int avs_rt286_probe(struct platform_device *pdev)
 	return devm_snd_soc_register_card(dev, card);
 }
 
+static const struct platform_device_id avs_rt286_driver_ids[] = {
+	{
+		.name = "avs_rt286",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(platform, avs_rt286_driver_ids);
+
 static struct platform_driver avs_rt286_driver = {
 	.probe = avs_rt286_probe,
 	.driver = {
 		.name = "avs_rt286",
 		.pm = &snd_soc_pm_ops,
 	},
+	.id_table = avs_rt286_driver_ids,
 };
 
 module_platform_driver(avs_rt286_driver);
 
 MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:avs_rt286");
diff --git a/sound/soc/intel/avs/boards/rt298.c b/sound/soc/intel/avs/boards/rt298.c
index ea32a7690c8a..80f490b9e118 100644
--- a/sound/soc/intel/avs/boards/rt298.c
+++ b/sound/soc/intel/avs/boards/rt298.c
@@ -247,15 +247,23 @@ static int avs_rt298_probe(struct platform_device *pdev)
 	return devm_snd_soc_register_card(dev, card);
 }
 
+static const struct platform_device_id avs_rt298_driver_ids[] = {
+	{
+		.name = "avs_rt298",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(platform, avs_rt298_driver_ids);
+
 static struct platform_driver avs_rt298_driver = {
 	.probe = avs_rt298_probe,
 	.driver = {
 		.name = "avs_rt298",
 		.pm = &snd_soc_pm_ops,
 	},
+	.id_table = avs_rt298_driver_ids,
 };
 
 module_platform_driver(avs_rt298_driver);
 
 MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:avs_rt298");
diff --git a/sound/soc/intel/avs/boards/rt5514.c b/sound/soc/intel/avs/boards/rt5514.c
index ad486a52e5e3..60105f453ae2 100644
--- a/sound/soc/intel/avs/boards/rt5514.c
+++ b/sound/soc/intel/avs/boards/rt5514.c
@@ -173,15 +173,23 @@ static int avs_rt5514_probe(struct platform_device *pdev)
 	return devm_snd_soc_register_card(dev, card);
 }
 
+static const struct platform_device_id avs_rt5514_driver_ids[] = {
+	{
+		.name = "avs_rt5514",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(platform, avs_rt5514_driver_ids);
+
 static struct platform_driver avs_rt5514_driver = {
 	.probe = avs_rt5514_probe,
 	.driver = {
 		.name = "avs_rt5514",
 		.pm = &snd_soc_pm_ops,
 	},
+	.id_table = avs_rt5514_driver_ids,
 };
 
 module_platform_driver(avs_rt5514_driver);
 
 MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:avs_rt5514");
diff --git a/sound/soc/intel/avs/boards/rt5663.c b/sound/soc/intel/avs/boards/rt5663.c
index 3effd789a45e..b4762c2a7bf2 100644
--- a/sound/soc/intel/avs/boards/rt5663.c
+++ b/sound/soc/intel/avs/boards/rt5663.c
@@ -246,15 +246,23 @@ static int avs_rt5663_probe(struct platform_device *pdev)
 	return devm_snd_soc_register_card(dev, card);
 }
 
+static const struct platform_device_id avs_rt5663_driver_ids[] = {
+	{
+		.name = "avs_rt5663",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(platform, avs_rt5663_driver_ids);
+
 static struct platform_driver avs_rt5663_driver = {
 	.probe = avs_rt5663_probe,
 	.driver = {
 		.name = "avs_rt5663",
 		.pm = &snd_soc_pm_ops,
 	},
+	.id_table = avs_rt5663_driver_ids,
 };
 
 module_platform_driver(avs_rt5663_driver);
 
 MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:avs_rt5663");
diff --git a/sound/soc/intel/avs/boards/rt5682.c b/sound/soc/intel/avs/boards/rt5682.c
index 84e850c0b085..243f979fda98 100644
--- a/sound/soc/intel/avs/boards/rt5682.c
+++ b/sound/soc/intel/avs/boards/rt5682.c
@@ -322,16 +322,24 @@ static int avs_rt5682_probe(struct platform_device *pdev)
 	return devm_snd_soc_register_card(dev, card);
 }
 
+static const struct platform_device_id avs_rt5682_driver_ids[] = {
+	{
+		.name = "avs_rt5682",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(platform, avs_rt5682_driver_ids);
+
 static struct platform_driver avs_rt5682_driver = {
 	.probe = avs_rt5682_probe,
 	.driver = {
 		.name = "avs_rt5682",
 		.pm = &snd_soc_pm_ops,
 	},
+	.id_table = avs_rt5682_driver_ids,
 };
 
 module_platform_driver(avs_rt5682_driver)
 
 MODULE_AUTHOR("Cezary Rojewski <cezary.rojewski@intel.com>");
 MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:avs_rt5682");
diff --git a/sound/soc/intel/avs/boards/ssm4567.c b/sound/soc/intel/avs/boards/ssm4567.c
index 6bcab9deae5c..4a0e136835ff 100644
--- a/sound/soc/intel/avs/boards/ssm4567.c
+++ b/sound/soc/intel/avs/boards/ssm4567.c
@@ -181,15 +181,23 @@ static int avs_ssm4567_probe(struct platform_device *pdev)
 	return devm_snd_soc_register_card(dev, card);
 }
 
+static const struct platform_device_id avs_ssm4567_driver_ids[] = {
+	{
+		.name = "avs_ssm4567",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(platform, avs_ssm4567_driver_ids);
+
 static struct platform_driver avs_ssm4567_driver = {
 	.probe = avs_ssm4567_probe,
 	.driver = {
 		.name = "avs_ssm4567",
 		.pm = &snd_soc_pm_ops,
 	},
+	.id_table = avs_ssm4567_driver_ids,
 };
 
 module_platform_driver(avs_ssm4567_driver)
 
 MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:avs_ssm4567");
diff --git a/sound/soc/intel/boards/sof_rt5682.c b/sound/soc/intel/boards/sof_rt5682.c
index 06ad15af46de..9723479f43da 100644
--- a/sound/soc/intel/boards/sof_rt5682.c
+++ b/sound/soc/intel/boards/sof_rt5682.c
@@ -1147,6 +1147,15 @@ static const struct platform_device_id board_ids[] = {
 					SOF_RT5682_SSP_AMP(0) |
 					SOF_RT5682_NUM_HDMIDEV(3)),
 	},
+	{
+		.name = "mtl_rt5650",
+		.driver_data = (kernel_ulong_t)(SOF_RT5682_MCLK_EN |
+					SOF_RT5682_SSP_CODEC(2) |
+					SOF_RT5682_SSP_AMP(0) |
+					SOF_RT5682_NUM_HDMIDEV(3) |
+					SOF_BT_OFFLOAD_SSP(1) |
+					SOF_SSP_BT_OFFLOAD_PRESENT),
+	},
 	{ }
 };
 MODULE_DEVICE_TABLE(platform, board_ids);
diff --git a/sound/soc/intel/common/soc-acpi-intel-mtl-match.c b/sound/soc/intel/common/soc-acpi-intel-mtl-match.c
index 301b8142d554..af4224bff718 100644
--- a/sound/soc/intel/common/soc-acpi-intel-mtl-match.c
+++ b/sound/soc/intel/common/soc-acpi-intel-mtl-match.c
@@ -40,6 +40,11 @@ static const struct snd_soc_acpi_codecs mtl_lt6911_hdmi = {
 	.codecs = {"INTC10B0"}
 };
 
+static const struct snd_soc_acpi_codecs mtl_rt5650_amp = {
+	.num_codecs = 1,
+	.codecs = {"10EC5650"}
+};
+
 struct snd_soc_acpi_mach snd_soc_acpi_intel_mtl_machines[] = {
 	{
 		.comp_ids = &mtl_rt5682_rt5682s_hp,
@@ -77,6 +82,13 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_mtl_machines[] = {
 					SND_SOC_ACPI_TPLG_INTEL_SSP_MSB |
 					SND_SOC_ACPI_TPLG_INTEL_DMIC_NUMBER,
 	},
+	{
+		.id = "10EC5650",
+		.drv_name = "mtl_rt5650",
+		.machine_quirk = snd_soc_acpi_codec_list,
+		.quirk_data = &mtl_rt5650_amp,
+		.sof_tplg_filename = "sof-mtl-rt5650.tplg",
+	},
 	/* place amp-only boards in the end of table */
 	{
 		.id = "INTC10B0",
diff --git a/sound/soc/intel/skylake/skl-pcm.c b/sound/soc/intel/skylake/skl-pcm.c
index d0c02e8a6785..174aae6e0398 100644
--- a/sound/soc/intel/skylake/skl-pcm.c
+++ b/sound/soc/intel/skylake/skl-pcm.c
@@ -240,8 +240,10 @@ static int skl_pcm_open(struct snd_pcm_substream *substream,
 	snd_pcm_set_sync(substream);
 
 	mconfig = skl_tplg_fe_get_cpr_module(dai, substream->stream);
-	if (!mconfig)
+	if (!mconfig) {
+		kfree(dma_params);
 		return -EINVAL;
+	}
 
 	skl_tplg_d0i3_get(skl, mconfig->d0i3_caps);
 
@@ -1462,6 +1464,7 @@ int skl_platform_register(struct device *dev)
 		dais = krealloc(skl->dais, sizeof(skl_fe_dai) +
 				sizeof(skl_platform_dai), GFP_KERNEL);
 		if (!dais) {
+			kfree(skl->dais);
 			ret = -ENOMEM;
 			goto err;
 		}
@@ -1474,8 +1477,10 @@ int skl_platform_register(struct device *dev)
 
 	ret = devm_snd_soc_register_component(dev, &skl_component,
 					 skl->dais, num_dais);
-	if (ret)
+	if (ret) {
+		kfree(skl->dais);
 		dev_err(dev, "soc component registration failed %d\n", ret);
+	}
 err:
 	return ret;
 }
diff --git a/sound/soc/intel/skylake/skl-sst-ipc.c b/sound/soc/intel/skylake/skl-sst-ipc.c
index 7a425271b08b..fd9624ad5f72 100644
--- a/sound/soc/intel/skylake/skl-sst-ipc.c
+++ b/sound/soc/intel/skylake/skl-sst-ipc.c
@@ -1003,8 +1003,10 @@ int skl_ipc_get_large_config(struct sst_generic_ipc *ipc,
 
 	reply.size = (reply.header >> 32) & IPC_DATA_OFFSET_SZ_MASK;
 	buf = krealloc(reply.data, reply.size, GFP_KERNEL);
-	if (!buf)
+	if (!buf) {
+		kfree(reply.data);
 		return -ENOMEM;
+	}
 	*payload = buf;
 	*bytes = reply.size;
 
diff --git a/sound/soc/soc-dapm.c b/sound/soc/soc-dapm.c
index 3844f777c87b..407a26af3eaf 100644
--- a/sound/soc/soc-dapm.c
+++ b/sound/soc/soc-dapm.c
@@ -320,7 +320,8 @@ EXPORT_SYMBOL_GPL(dapm_mark_endpoints_dirty);
 
 /* create a new dapm widget */
 static inline struct snd_soc_dapm_widget *dapm_cnew_widget(
-	const struct snd_soc_dapm_widget *_widget)
+	const struct snd_soc_dapm_widget *_widget,
+	const char *prefix)
 {
 	struct snd_soc_dapm_widget *w;
 
@@ -328,13 +329,19 @@ static inline struct snd_soc_dapm_widget *dapm_cnew_widget(
 	if (!w)
 		return NULL;
 
-	/*
-	 * w->name is duplicated in caller, but w->sname isn't.
-	 * Duplicate it here if defined
-	 */
+	if (prefix)
+		w->name = kasprintf(GFP_KERNEL, "%s %s", prefix, _widget->name);
+	else
+		w->name = kstrdup_const(_widget->name, GFP_KERNEL);
+	if (!w->name) {
+		kfree(w);
+		return NULL;
+	}
+
 	if (_widget->sname) {
 		w->sname = kstrdup_const(_widget->sname, GFP_KERNEL);
 		if (!w->sname) {
+			kfree_const(w->name);
 			kfree(w);
 			return NULL;
 		}
@@ -3629,20 +3636,12 @@ snd_soc_dapm_new_control_unlocked(struct snd_soc_dapm_context *dapm,
 {
 	enum snd_soc_dapm_direction dir;
 	struct snd_soc_dapm_widget *w;
-	const char *prefix;
 	int ret = -ENOMEM;
 
-	if ((w = dapm_cnew_widget(widget)) == NULL)
+	w = dapm_cnew_widget(widget, soc_dapm_prefix(dapm));
+	if (!w)
 		goto cnew_failed;
 
-	prefix = soc_dapm_prefix(dapm);
-	if (prefix)
-		w->name = kasprintf(GFP_KERNEL, "%s %s", prefix, widget->name);
-	else
-		w->name = kstrdup_const(widget->name, GFP_KERNEL);
-	if (!w->name)
-		goto name_failed;
-
 	switch (w->id) {
 	case snd_soc_dapm_regulator_supply:
 		w->regulator = devm_regulator_get(dapm->dev, widget->name);
@@ -3767,7 +3766,6 @@ snd_soc_dapm_new_control_unlocked(struct snd_soc_dapm_context *dapm,
 	dev_err_probe(dapm->dev, ret, "ASoC: Failed to request %s\n",
 		      w->name);
 	kfree_const(w->name);
-name_failed:
 	kfree_const(w->sname);
 	kfree(w);
 cnew_failed:
diff --git a/sound/soc/sof/imx/imx8m.c b/sound/soc/sof/imx/imx8m.c
index 2680f061ba42..f088fd1a672b 100644
--- a/sound/soc/sof/imx/imx8m.c
+++ b/sound/soc/sof/imx/imx8m.c
@@ -313,6 +313,13 @@ static struct snd_soc_dai_driver imx8m_dai[] = {
 		.channels_max = 32,
 	},
 },
+{
+	.name = "micfil",
+	.capture = {
+		.channels_min = 1,
+		.channels_max = 8,
+	},
+},
 };
 
 static int imx8m_dsp_set_power_state(struct snd_sof_dev *sdev,
diff --git a/sound/soc/sof/ipc3-pcm.c b/sound/soc/sof/ipc3-pcm.c
index 2d0addcbc819..330f04bcd75d 100644
--- a/sound/soc/sof/ipc3-pcm.c
+++ b/sound/soc/sof/ipc3-pcm.c
@@ -384,6 +384,17 @@ static int sof_ipc3_pcm_dai_link_fixup(struct snd_soc_pcm_runtime *rtd,
 		dev_dbg(component->dev, "AMD_DMIC channels_min: %d channels_max: %d\n",
 			channels->min, channels->max);
 		break;
+	case SOF_DAI_IMX_MICFIL:
+		rate->min = private->dai_config->micfil.pdm_rate;
+		rate->max = private->dai_config->micfil.pdm_rate;
+		channels->min = private->dai_config->micfil.pdm_ch;
+		channels->max = private->dai_config->micfil.pdm_ch;
+
+		dev_dbg(component->dev,
+			"MICFIL PDM rate_min: %d rate_max: %d\n", rate->min, rate->max);
+		dev_dbg(component->dev, "MICFIL PDM channels_min: %d channels_max: %d\n",
+			channels->min, channels->max);
+		break;
 	default:
 		dev_err(component->dev, "Invalid DAI type %d\n", private->dai_config->type);
 		break;
diff --git a/sound/soc/sof/ipc3-topology.c b/sound/soc/sof/ipc3-topology.c
index ba4ef290b634..7a4932c152a9 100644
--- a/sound/soc/sof/ipc3-topology.c
+++ b/sound/soc/sof/ipc3-topology.c
@@ -286,6 +286,16 @@ static const struct sof_topology_token acpi2s_tokens[] = {
 		offsetof(struct sof_ipc_dai_acp_params, tdm_mode)},
 };
 
+/* MICFIL PDM */
+static const struct sof_topology_token micfil_pdm_tokens[] = {
+	{SOF_TKN_IMX_MICFIL_RATE,
+		SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_dai_micfil_params, pdm_rate)},
+	{SOF_TKN_IMX_MICFIL_CH,
+		SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_dai_micfil_params, pdm_ch)},
+};
+
 /* Core tokens */
 static const struct sof_topology_token core_tokens[] = {
 	{SOF_TKN_COMP_CORE_ID, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
@@ -322,6 +332,8 @@ static const struct sof_token_info ipc3_token_list[SOF_TOKEN_COUNT] = {
 	[SOF_AFE_TOKENS] = {"AFE tokens", afe_tokens, ARRAY_SIZE(afe_tokens)},
 	[SOF_ACPDMIC_TOKENS] = {"ACPDMIC tokens", acpdmic_tokens, ARRAY_SIZE(acpdmic_tokens)},
 	[SOF_ACPI2S_TOKENS]   = {"ACPI2S tokens", acpi2s_tokens, ARRAY_SIZE(acpi2s_tokens)},
+	[SOF_MICFIL_TOKENS] = {"MICFIL PDM tokens",
+		micfil_pdm_tokens, ARRAY_SIZE(micfil_pdm_tokens)},
 };
 
 /**
@@ -1136,6 +1148,37 @@ static int sof_link_esai_load(struct snd_soc_component *scomp, struct snd_sof_da
 	return 0;
 }
 
+static int sof_link_micfil_load(struct snd_soc_component *scomp, struct snd_sof_dai_link *slink,
+				struct sof_ipc_dai_config *config, struct snd_sof_dai *dai)
+{
+	struct snd_soc_tplg_hw_config *hw_config = slink->hw_configs;
+	struct sof_dai_private_data *private = dai->private;
+	u32 size = sizeof(*config);
+	int ret;
+
+       /* handle master/slave and inverted clocks */
+	sof_dai_set_format(hw_config, config);
+
+	config->hdr.size = size;
+
+	/* parse the required set of MICFIL PDM tokens based on num_hw_cfgs */
+	ret = sof_update_ipc_object(scomp, &config->micfil, SOF_MICFIL_TOKENS, slink->tuples,
+				    slink->num_tuples, size, slink->num_hw_configs);
+	if (ret < 0)
+		return ret;
+
+	dev_info(scomp->dev, "MICFIL PDM config dai_index %d channel %d rate %d\n",
+		 config->dai_index, config->micfil.pdm_ch, config->micfil.pdm_rate);
+
+	dai->number_configs = 1;
+	dai->current_config = 0;
+	private->dai_config = kmemdup(config, size, GFP_KERNEL);
+	if (!private->dai_config)
+		return -ENOMEM;
+
+	return 0;
+}
+
 static int sof_link_acp_dmic_load(struct snd_soc_component *scomp, struct snd_sof_dai_link *slink,
 				  struct sof_ipc_dai_config *config, struct snd_sof_dai *dai)
 {
@@ -1559,6 +1602,9 @@ static int sof_ipc3_widget_setup_comp_dai(struct snd_sof_widget *swidget)
 		case SOF_DAI_IMX_ESAI:
 			ret = sof_link_esai_load(scomp, slink, config, dai);
 			break;
+		case SOF_DAI_IMX_MICFIL:
+			ret = sof_link_micfil_load(scomp, slink, config, dai);
+			break;
 		case SOF_DAI_AMD_BT:
 			ret = sof_link_acp_bt_load(scomp, slink, config, dai);
 			break;
diff --git a/sound/soc/sof/mediatek/adsp_helper.h b/sound/soc/sof/mediatek/adsp_helper.h
index d41e904e6614..35527567962e 100644
--- a/sound/soc/sof/mediatek/adsp_helper.h
+++ b/sound/soc/sof/mediatek/adsp_helper.h
@@ -15,17 +15,13 @@
 struct mtk_adsp_chip_info {
 	phys_addr_t pa_sram;
 	phys_addr_t pa_dram; /* adsp dram physical base */
-	phys_addr_t pa_shared_dram; /* adsp dram physical base */
 	phys_addr_t pa_cfgreg;
 	u32 sramsize;
 	u32 dramsize;
 	u32 cfgregsize;
-	u32 shared_size;
 	void __iomem *va_sram; /* corresponding to pa_sram */
 	void __iomem *va_dram; /* corresponding to pa_dram */
 	void __iomem *va_cfgreg;
-	void __iomem *shared_sram; /* part of  va_sram */
-	void __iomem *shared_dram; /* part of  va_dram */
 	phys_addr_t adsp_bootup_addr;
 	int dram_offset; /*dram offset between system and dsp view*/
 
diff --git a/sound/soc/sof/mediatek/mt8186/mt8186.c b/sound/soc/sof/mediatek/mt8186/mt8186.c
index b69fa788b16f..0d2d7d697de0 100644
--- a/sound/soc/sof/mediatek/mt8186/mt8186.c
+++ b/sound/soc/sof/mediatek/mt8186/mt8186.c
@@ -96,29 +96,6 @@ static int platform_parse_resource(struct platform_device *pdev, void *data)
 	struct mtk_adsp_chip_info *adsp = data;
 	int ret;
 
-	mem_region = of_parse_phandle(dev->of_node, "memory-region", 0);
-	if (!mem_region) {
-		dev_err(dev, "no dma memory-region phandle\n");
-		return -ENODEV;
-	}
-
-	ret = of_address_to_resource(mem_region, 0, &res);
-	of_node_put(mem_region);
-	if (ret) {
-		dev_err(dev, "of_address_to_resource dma failed\n");
-		return ret;
-	}
-
-	dev_dbg(dev, "DMA %pR\n", &res);
-
-	adsp->pa_shared_dram = (phys_addr_t)res.start;
-	adsp->shared_size = resource_size(&res);
-	if (adsp->pa_shared_dram & DRAM_REMAP_MASK) {
-		dev_err(dev, "adsp shared dma memory(%#x) is not 4K-aligned\n",
-			(u32)adsp->pa_shared_dram);
-		return -EINVAL;
-	}
-
 	ret = of_reserved_mem_device_init(dev);
 	if (ret) {
 		dev_err(dev, "of_reserved_mem_device_init failed\n");
@@ -248,26 +225,6 @@ static int adsp_memory_remap_init(struct snd_sof_dev *sdev, struct mtk_adsp_chip
 	return 0;
 }
 
-static int adsp_shared_base_ioremap(struct platform_device *pdev, void *data)
-{
-	struct device *dev = &pdev->dev;
-	struct mtk_adsp_chip_info *adsp = data;
-
-	/* remap shared-dram base to be non-cachable */
-	adsp->shared_dram = devm_ioremap(dev, adsp->pa_shared_dram,
-					 adsp->shared_size);
-	if (!adsp->shared_dram) {
-		dev_err(dev, "failed to ioremap base %pa size %#x\n",
-			adsp->shared_dram, adsp->shared_size);
-		return -ENOMEM;
-	}
-
-	dev_dbg(dev, "shared-dram vbase=%p, phy addr :%pa,  size=%#x\n",
-		adsp->shared_dram, &adsp->pa_shared_dram, adsp->shared_size);
-
-	return 0;
-}
-
 static int mt8186_run(struct snd_sof_dev *sdev)
 {
 	u32 adsp_bootup_addr;
@@ -324,12 +281,6 @@ static int mt8186_dsp_probe(struct snd_sof_dev *sdev)
 
 	priv->adsp->va_dram = sdev->bar[SOF_FW_BLK_TYPE_SRAM];
 
-	ret = adsp_shared_base_ioremap(pdev, priv->adsp);
-	if (ret) {
-		dev_err(sdev->dev, "adsp_shared_base_ioremap fail!\n");
-		return ret;
-	}
-
 	sdev->bar[DSP_REG_BAR] = priv->adsp->va_cfgreg;
 	sdev->bar[DSP_SECREG_BAR] = priv->adsp->va_secreg;
 	sdev->bar[DSP_BUSREG_BAR] = priv->adsp->va_busreg;
diff --git a/sound/soc/sof/mediatek/mt8195/mt8195.c b/sound/soc/sof/mediatek/mt8195/mt8195.c
index cac0a085f60a..8ee7ee246344 100644
--- a/sound/soc/sof/mediatek/mt8195/mt8195.c
+++ b/sound/soc/sof/mediatek/mt8195/mt8195.c
@@ -96,29 +96,6 @@ static int platform_parse_resource(struct platform_device *pdev, void *data)
 	struct mtk_adsp_chip_info *adsp = data;
 	int ret;
 
-	mem_region = of_parse_phandle(dev->of_node, "memory-region", 0);
-	if (!mem_region) {
-		dev_err(dev, "no dma memory-region phandle\n");
-		return -ENODEV;
-	}
-
-	ret = of_address_to_resource(mem_region, 0, &res);
-	of_node_put(mem_region);
-	if (ret) {
-		dev_err(dev, "of_address_to_resource dma failed\n");
-		return ret;
-	}
-
-	dev_dbg(dev, "DMA %pR\n", &res);
-
-	adsp->pa_shared_dram = (phys_addr_t)res.start;
-	adsp->shared_size = resource_size(&res);
-	if (adsp->pa_shared_dram & DRAM_REMAP_MASK) {
-		dev_err(dev, "adsp shared dma memory(%#x) is not 4K-aligned\n",
-			(u32)adsp->pa_shared_dram);
-		return -EINVAL;
-	}
-
 	ret = of_reserved_mem_device_init(dev);
 	if (ret) {
 		dev_err(dev, "of_reserved_mem_device_init failed\n");
@@ -238,26 +215,6 @@ static int adsp_memory_remap_init(struct device *dev, struct mtk_adsp_chip_info
 	return 0;
 }
 
-static int adsp_shared_base_ioremap(struct platform_device *pdev, void *data)
-{
-	struct device *dev = &pdev->dev;
-	struct mtk_adsp_chip_info *adsp = data;
-
-	/* remap shared-dram base to be non-cachable */
-	adsp->shared_dram = devm_ioremap(dev, adsp->pa_shared_dram,
-					 adsp->shared_size);
-	if (!adsp->shared_dram) {
-		dev_err(dev, "failed to ioremap base %pa size %#x\n",
-			adsp->shared_dram, adsp->shared_size);
-		return -ENOMEM;
-	}
-
-	dev_dbg(dev, "shared-dram vbase=%p, phy addr :%pa,  size=%#x\n",
-		adsp->shared_dram, &adsp->pa_shared_dram, adsp->shared_size);
-
-	return 0;
-}
-
 static int mt8195_run(struct snd_sof_dev *sdev)
 {
 	u32 adsp_bootup_addr;
@@ -338,12 +295,6 @@ static int mt8195_dsp_probe(struct snd_sof_dev *sdev)
 	}
 	priv->adsp->va_dram = sdev->bar[SOF_FW_BLK_TYPE_SRAM];
 
-	ret = adsp_shared_base_ioremap(pdev, priv->adsp);
-	if (ret) {
-		dev_err(sdev->dev, "adsp_shared_base_ioremap fail!\n");
-		goto err_adsp_sram_power_off;
-	}
-
 	sdev->bar[DSP_REG_BAR] = priv->adsp->va_cfgreg;
 
 	sdev->mmio_bar = SOF_FW_BLK_TYPE_SRAM;
diff --git a/sound/soc/sof/sof-audio.h b/sound/soc/sof/sof-audio.h
index 5d5eeb1a1a6f..99c940b22538 100644
--- a/sound/soc/sof/sof-audio.h
+++ b/sound/soc/sof/sof-audio.h
@@ -275,6 +275,7 @@ enum sof_tokens {
 	SOF_GAIN_TOKENS,
 	SOF_ACPDMIC_TOKENS,
 	SOF_ACPI2S_TOKENS,
+	SOF_MICFIL_TOKENS,
 
 	/* this should be the last */
 	SOF_TOKEN_COUNT,
diff --git a/sound/soc/sof/topology.c b/sound/soc/sof/topology.c
index a3a3af252259..f1add75afbe1 100644
--- a/sound/soc/sof/topology.c
+++ b/sound/soc/sof/topology.c
@@ -296,6 +296,7 @@ static const struct sof_dai_types sof_dais[] = {
 	{"AFE", SOF_DAI_MEDIATEK_AFE},
 	{"ACPSP_VIRTUAL", SOF_DAI_AMD_SP_VIRTUAL},
 	{"ACPHS_VIRTUAL", SOF_DAI_AMD_HS_VIRTUAL},
+	{"MICFIL", SOF_DAI_IMX_MICFIL},
 
 };
 
@@ -1736,8 +1737,10 @@ static int sof_dai_load(struct snd_soc_component *scomp, int index,
 	/* perform pcm set op */
 	if (ipc_pcm_ops && ipc_pcm_ops->pcm_setup) {
 		ret = ipc_pcm_ops->pcm_setup(sdev, spcm);
-		if (ret < 0)
+		if (ret < 0) {
+			kfree(spcm);
 			return ret;
+		}
 	}
 
 	dai_drv->dobj.private = spcm;
@@ -1960,6 +1963,10 @@ static int sof_link_load(struct snd_soc_component *scomp, int index, struct snd_
 		token_id = SOF_ACPI2S_TOKENS;
 		num_tuples += token_list[SOF_ACPI2S_TOKENS].count;
 		break;
+	case SOF_DAI_IMX_MICFIL:
+		token_id = SOF_MICFIL_TOKENS;
+		num_tuples += token_list[SOF_MICFIL_TOKENS].count;
+		break;
 	default:
 		break;
 	}
diff --git a/sound/soc/sti/sti_uniperif.c b/sound/soc/sti/sti_uniperif.c
index 2c21a86421e6..ba824f14a39c 100644
--- a/sound/soc/sti/sti_uniperif.c
+++ b/sound/soc/sti/sti_uniperif.c
@@ -461,10 +461,6 @@ static int sti_uniperiph_cpu_dai_of(struct device_node *node,
 	return 0;
 }
 
-static const struct snd_dmaengine_pcm_config dmaengine_pcm_config = {
-	.prepare_slave_config = snd_dmaengine_pcm_prepare_slave_config,
-};
-
 static int sti_uniperiph_probe(struct platform_device *pdev)
 {
 	struct sti_uniperiph_data *priv;
@@ -493,8 +489,7 @@ static int sti_uniperiph_probe(struct platform_device *pdev)
 	if (ret < 0)
 		return ret;
 
-	return devm_snd_dmaengine_pcm_register(&pdev->dev,
-					       &dmaengine_pcm_config, 0);
+	return devm_snd_dmaengine_pcm_register(&pdev->dev, NULL, 0);
 }
 
 static struct platform_driver sti_uniperiph_driver = {
diff --git a/sound/soc/tegra/tegra_pcm.c b/sound/soc/tegra/tegra_pcm.c
index 142e8d4eefd5..42acb56543db 100644
--- a/sound/soc/tegra/tegra_pcm.c
+++ b/sound/soc/tegra/tegra_pcm.c
@@ -98,8 +98,8 @@ int tegra_pcm_open(struct snd_soc_component *component,
 		return ret;
 	}
 
-	chan = dma_request_slave_channel(cpu_dai->dev, dmap->chan_name);
-	if (!chan) {
+	chan = dma_request_chan(cpu_dai->dev, dmap->chan_name);
+	if (IS_ERR(chan)) {
 		dev_err(cpu_dai->dev,
 			"dmaengine request slave channel failed! (%s)\n",
 			dmap->chan_name);
diff --git a/sound/x86/intel_hdmi_audio.c b/sound/x86/intel_hdmi_audio.c
index ab95fb34a635..02f5a7f9b728 100644
--- a/sound/x86/intel_hdmi_audio.c
+++ b/sound/x86/intel_hdmi_audio.c
@@ -30,6 +30,7 @@
 #include <sound/control.h>
 #include <sound/jack.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_eld.h>
 #include <drm/intel_lpe_audio.h>
 #include "intel_hdmi_audio.h"
 
diff --git a/tools/bpf/bpftool/Documentation/bpftool.rst b/tools/bpf/bpftool/Documentation/bpftool.rst
index 6965c94dfdaf..09e4f2ff5658 100644
--- a/tools/bpf/bpftool/Documentation/bpftool.rst
+++ b/tools/bpf/bpftool/Documentation/bpftool.rst
@@ -20,7 +20,7 @@ SYNOPSIS
 
 	**bpftool** **version**
 
-	*OBJECT* := { **map** | **program** | **link** | **cgroup** | **perf** | **net** | **feature** |
+	*OBJECT* := { **map** | **prog** | **link** | **cgroup** | **perf** | **net** | **feature** |
 	**btf** | **gen** | **struct_ops** | **iter** }
 
 	*OPTIONS* := { { **-V** | **--version** } | |COMMON_OPTIONS| }
diff --git a/tools/build/Makefile.feature b/tools/build/Makefile.feature
index 934e2777a2db..64df118376df 100644
--- a/tools/build/Makefile.feature
+++ b/tools/build/Makefile.feature
@@ -32,6 +32,7 @@ FEATURE_TESTS_BASIC :=                  \
         backtrace                       \
         dwarf                           \
         dwarf_getlocations              \
+        dwarf_getcfi                    \
         eventfd                         \
         fortify-source                  \
         get_current_dir_name            \
diff --git a/tools/build/feature/Makefile b/tools/build/feature/Makefile
index dad79ede4e0a..37722e509eb9 100644
--- a/tools/build/feature/Makefile
+++ b/tools/build/feature/Makefile
@@ -7,6 +7,7 @@ FILES=                                          \
          test-bionic.bin                        \
          test-dwarf.bin                         \
          test-dwarf_getlocations.bin            \
+         test-dwarf_getcfi.bin                  \
          test-eventfd.bin                       \
          test-fortify-source.bin                \
          test-get_current_dir_name.bin          \
@@ -154,6 +155,9 @@ $(OUTPUT)test-dwarf.bin:
 $(OUTPUT)test-dwarf_getlocations.bin:
 	$(BUILD) $(DWARFLIBS)
 
+$(OUTPUT)test-dwarf_getcfi.bin:
+	$(BUILD) $(DWARFLIBS)
+
 $(OUTPUT)test-libelf-getphdrnum.bin:
 	$(BUILD) -lelf
 
diff --git a/tools/build/feature/test-dwarf_getcfi.c b/tools/build/feature/test-dwarf_getcfi.c
new file mode 100644
index 000000000000..50e7d7cb7bdf
--- /dev/null
+++ b/tools/build/feature/test-dwarf_getcfi.c
@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <stdio.h>
+#include <elfutils/libdw.h>
+
+int main(void)
+{
+	Dwarf *dwarf = NULL;
+	return dwarf_getcfi(dwarf) == NULL;
+}
diff --git a/tools/hv/hv_kvp_daemon.c b/tools/hv/hv_kvp_daemon.c
index 264eeb9c46a9..318e2dad27e0 100644
--- a/tools/hv/hv_kvp_daemon.c
+++ b/tools/hv/hv_kvp_daemon.c
@@ -1421,7 +1421,7 @@ static int kvp_set_ip_info(char *if_name, struct hv_kvp_ipaddr_value *new_val)
 	if (error)
 		goto setval_error;
 
-	if (new_val->addr_family == ADDR_FAMILY_IPV6) {
+	if (new_val->addr_family & ADDR_FAMILY_IPV6) {
 		error = fprintf(nmfile, "\n[ipv6]\n");
 		if (error < 0)
 			goto setval_error;
@@ -1455,14 +1455,18 @@ static int kvp_set_ip_info(char *if_name, struct hv_kvp_ipaddr_value *new_val)
 	if (error < 0)
 		goto setval_error;
 
-	error = fprintf(nmfile, "gateway=%s\n", (char *)new_val->gate_way);
-	if (error < 0)
-		goto setval_error;
-
-	error = fprintf(nmfile, "dns=%s\n", (char *)new_val->dns_addr);
-	if (error < 0)
-		goto setval_error;
+	/* we do not want ipv4 addresses in ipv6 section and vice versa */
+	if (is_ipv6 != is_ipv4((char *)new_val->gate_way)) {
+		error = fprintf(nmfile, "gateway=%s\n", (char *)new_val->gate_way);
+		if (error < 0)
+			goto setval_error;
+	}
 
+	if (is_ipv6 != is_ipv4((char *)new_val->dns_addr)) {
+		error = fprintf(nmfile, "dns=%s\n", (char *)new_val->dns_addr);
+		if (error < 0)
+			goto setval_error;
+	}
 	fclose(nmfile);
 	fclose(ifcfg_file);
 
diff --git a/tools/hv/hv_set_ifconfig.sh b/tools/hv/hv_set_ifconfig.sh
index ae5a7a8249a2..440a91b35823 100755
--- a/tools/hv/hv_set_ifconfig.sh
+++ b/tools/hv/hv_set_ifconfig.sh
@@ -53,7 +53,7 @@
 #                       or "manual" if no boot-time protocol should be used)
 #
 # address1=ipaddr1/plen
-# address=ipaddr2/plen
+# address2=ipaddr2/plen
 #
 # gateway=gateway1;gateway2
 #
@@ -61,7 +61,7 @@
 #
 # [ipv6]
 # address1=ipaddr1/plen
-# address2=ipaddr1/plen
+# address2=ipaddr2/plen
 #
 # gateway=gateway1;gateway2
 #
diff --git a/tools/include/asm-generic/unaligned.h b/tools/include/asm-generic/unaligned.h
index 156743d399ae..2fd551915c20 100644
--- a/tools/include/asm-generic/unaligned.h
+++ b/tools/include/asm-generic/unaligned.h
@@ -8,6 +8,7 @@
  */
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wpacked"
+#pragma GCC diagnostic ignored "-Wattributes"
 
 #define __get_unaligned_t(type, ptr) ({						\
 	const struct { type x; } __packed *__pptr = (typeof(__pptr))(ptr);	\
diff --git a/tools/include/linux/rwsem.h b/tools/include/linux/rwsem.h
index 83971b3cbfce..f8bffd4a987c 100644
--- a/tools/include/linux/rwsem.h
+++ b/tools/include/linux/rwsem.h
@@ -37,4 +37,8 @@ static inline int up_write(struct rw_semaphore *sem)
 {
 	return pthread_rwlock_unlock(&sem->lock);
 }
+
+#define down_read_nested(sem, subclass)		down_read(sem)
+#define down_write_nested(sem, subclass)	down_write(sem)
+
 #endif /* _TOOLS_RWSEM_H */
diff --git a/tools/include/linux/spinlock.h b/tools/include/linux/spinlock.h
index 622266b197d0..a6cdf25b6b9d 100644
--- a/tools/include/linux/spinlock.h
+++ b/tools/include/linux/spinlock.h
@@ -11,6 +11,7 @@
 #define spin_lock_init(x)	pthread_mutex_init(x, NULL)
 
 #define spin_lock(x)			pthread_mutex_lock(x)
+#define spin_lock_nested(x, subclass)	pthread_mutex_lock(x)
 #define spin_unlock(x)			pthread_mutex_unlock(x)
 #define spin_lock_bh(x)			pthread_mutex_lock(x)
 #define spin_unlock_bh(x)		pthread_mutex_unlock(x)
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 0f6cdf52b1da..7a5498242eaa 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -1200,6 +1200,9 @@ enum bpf_perf_event_type {
  */
 #define BPF_F_XDP_DEV_BOUND_ONLY	(1U << 6)
 
+/* The verifier internal test flag. Behavior is undefined */
+#define BPF_F_TEST_REG_INVARIANTS	(1U << 7)
+
 /* link_create.kprobe_multi.flags used in LINK_CREATE command for
  * BPF_TRACE_KPROBE_MULTI attach type to create return probe.
  */
@@ -4517,6 +4520,8 @@ union bpf_attr {
  * long bpf_get_task_stack(struct task_struct *task, void *buf, u32 size, u64 flags)
  *	Description
  *		Return a user or a kernel stack in bpf program provided buffer.
+ *		Note: the user stack will only be populated if the *task* is
+ *		the current task; all other tasks will return -EOPNOTSUPP.
  *		To achieve this, the helper needs *task*, which is a valid
  *		pointer to **struct task_struct**. To store the stacktrace, the
  *		bpf program provides *buf* with a nonnegative *size*.
@@ -4528,6 +4533,7 @@ union bpf_attr {
  *
  *		**BPF_F_USER_STACK**
  *			Collect a user space stack instead of a kernel stack.
+ *			The *task* must be the current task.
  *		**BPF_F_USER_BUILD_ID**
  *			Collect buildid+offset instead of ips for user stack,
  *			only valid if **BPF_F_USER_STACK** is also specified.
@@ -7151,40 +7157,31 @@ struct bpf_spin_lock {
 };
 
 struct bpf_timer {
-	__u64 :64;
-	__u64 :64;
+	__u64 __opaque[2];
 } __attribute__((aligned(8)));
 
 struct bpf_dynptr {
-	__u64 :64;
-	__u64 :64;
+	__u64 __opaque[2];
 } __attribute__((aligned(8)));
 
 struct bpf_list_head {
-	__u64 :64;
-	__u64 :64;
+	__u64 __opaque[2];
 } __attribute__((aligned(8)));
 
 struct bpf_list_node {
-	__u64 :64;
-	__u64 :64;
-	__u64 :64;
+	__u64 __opaque[3];
 } __attribute__((aligned(8)));
 
 struct bpf_rb_root {
-	__u64 :64;
-	__u64 :64;
+	__u64 __opaque[2];
 } __attribute__((aligned(8)));
 
 struct bpf_rb_node {
-	__u64 :64;
-	__u64 :64;
-	__u64 :64;
-	__u64 :64;
+	__u64 __opaque[4];
 } __attribute__((aligned(8)));
 
 struct bpf_refcount {
-	__u32 :32;
+	__u32 __opaque[1];
 } __attribute__((aligned(4)));
 
 struct bpf_sysctl {
diff --git a/tools/include/uapi/linux/fs.h b/tools/include/uapi/linux/fs.h
index da43810b7485..48ad69f7722e 100644
--- a/tools/include/uapi/linux/fs.h
+++ b/tools/include/uapi/linux/fs.h
@@ -316,6 +316,7 @@ typedef int __bitwise __kernel_rwf_t;
 #define PAGE_IS_SWAPPED		(1 << 4)
 #define PAGE_IS_PFNZERO		(1 << 5)
 #define PAGE_IS_HUGE		(1 << 6)
+#define PAGE_IS_SOFT_DIRTY	(1 << 7)
 
 /*
  * struct page_region - Page region with flags
diff --git a/tools/include/uapi/linux/perf_event.h b/tools/include/uapi/linux/perf_event.h
index 39c6a250dd1b..3a64499b0f5d 100644
--- a/tools/include/uapi/linux/perf_event.h
+++ b/tools/include/uapi/linux/perf_event.h
@@ -204,6 +204,8 @@ enum perf_branch_sample_type_shift {
 
 	PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT	= 18, /* save privilege mode */
 
+	PERF_SAMPLE_BRANCH_COUNTERS_SHIFT	= 19, /* save occurrences of events on a branch */
+
 	PERF_SAMPLE_BRANCH_MAX_SHIFT		/* non-ABI */
 };
 
@@ -235,6 +237,8 @@ enum perf_branch_sample_type {
 
 	PERF_SAMPLE_BRANCH_PRIV_SAVE	= 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT,
 
+	PERF_SAMPLE_BRANCH_COUNTERS	= 1U << PERF_SAMPLE_BRANCH_COUNTERS_SHIFT,
+
 	PERF_SAMPLE_BRANCH_MAX		= 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT,
 };
 
@@ -982,6 +986,12 @@ enum perf_event_type {
 	 *	{ u64                   nr;
 	 *	  { u64	hw_idx; } && PERF_SAMPLE_BRANCH_HW_INDEX
 	 *        { u64 from, to, flags } lbr[nr];
+	 *        #
+	 *        # The format of the counters is decided by the
+	 *        # "branch_counter_nr" and "branch_counter_width",
+	 *        # which are defined in the ABI.
+	 *        #
+	 *        { u64 counters; } cntr[nr] && PERF_SAMPLE_BRANCH_COUNTERS
 	 *      } && PERF_SAMPLE_BRANCH_STACK
 	 *
 	 * 	{ u64			abi; # enum perf_sample_regs_abi
@@ -1427,6 +1437,9 @@ struct perf_branch_entry {
 		reserved:31;
 };
 
+/* Size of used info bits in struct perf_branch_entry */
+#define PERF_BRANCH_ENTRY_INFO_BITS_MAX		33
+
 union perf_sample_weight {
 	__u64		full;
 #if defined(__LITTLE_ENDIAN_BITFIELD)
diff --git a/tools/lib/bpf/libbpf_common.h b/tools/lib/bpf/libbpf_common.h
index b7060f254486..8fe248e14eb6 100644
--- a/tools/lib/bpf/libbpf_common.h
+++ b/tools/lib/bpf/libbpf_common.h
@@ -79,11 +79,14 @@
  */
 #define LIBBPF_OPTS_RESET(NAME, ...)					    \
 	do {								    \
-		memset(&NAME, 0, sizeof(NAME));				    \
-		NAME = (typeof(NAME)) {					    \
-			.sz = sizeof(NAME),				    \
-			__VA_ARGS__					    \
-		};							    \
+		typeof(NAME) ___##NAME = ({ 				    \
+			memset(&___##NAME, 0, sizeof(NAME));		    \
+			(typeof(NAME)) {				    \
+				.sz = sizeof(NAME),			    \
+				__VA_ARGS__				    \
+			};						    \
+		});							    \
+		memcpy(&NAME, &___##NAME, sizeof(NAME));		    \
 	} while (0)
 
 #endif /* __LIBBPF_LIBBPF_COMMON_H */
diff --git a/tools/objtool/check.c b/tools/objtool/check.c
index e94756e09ca9..548ec3cd7c00 100644
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -291,7 +291,7 @@ static void init_insn_state(struct objtool_file *file, struct insn_state *state,
 
 static struct cfi_state *cfi_alloc(void)
 {
-	struct cfi_state *cfi = calloc(sizeof(struct cfi_state), 1);
+	struct cfi_state *cfi = calloc(1, sizeof(struct cfi_state));
 	if (!cfi) {
 		WARN("calloc failed");
 		exit(1);
diff --git a/tools/perf/.gitignore b/tools/perf/.gitignore
index f533e76fb480..ee5c14f3b8b1 100644
--- a/tools/perf/.gitignore
+++ b/tools/perf/.gitignore
@@ -49,3 +49,4 @@ libtraceevent/
 libtraceevent_plugins/
 fixdep
 Documentation/doc.dep
+python_ext_build/
diff --git a/tools/perf/Documentation/perf-config.txt b/tools/perf/Documentation/perf-config.txt
index 0b4e79dbd3f6..16398babd1ef 100644
--- a/tools/perf/Documentation/perf-config.txt
+++ b/tools/perf/Documentation/perf-config.txt
@@ -722,6 +722,10 @@ session-<NAME>.*::
 		Defines new record session for daemon. The value is record's
 		command line without the 'record' keyword.
 
+test.*::
+
+	test.objdump::
+		objdump binary to use for disassembly and annotations.
 
 SEE ALSO
 --------
diff --git a/tools/perf/Documentation/perf-list.txt b/tools/perf/Documentation/perf-list.txt
index d5f78e125efe..1b90575ee3c8 100644
--- a/tools/perf/Documentation/perf-list.txt
+++ b/tools/perf/Documentation/perf-list.txt
@@ -81,11 +81,13 @@ For Intel systems precise event sampling is implemented with PEBS
 which supports up to precise-level 2, and precise level 3 for
 some special cases
 
-On AMD systems it is implemented using IBS (up to precise-level 2).
-The precise modifier works with event types 0x76 (cpu-cycles, CPU
-clocks not halted) and 0xC1 (micro-ops retired). Both events map to
-IBS execution sampling (IBS op) with the IBS Op Counter Control bit
-(IbsOpCntCtl) set respectively (see the
+On AMD systems it is implemented using IBS OP (up to precise-level 2).
+Unlike Intel PEBS which provides levels of precision, AMD core pmu is
+inherently non-precise and IBS is inherently precise. (i.e. ibs_op//,
+ibs_op//p, ibs_op//pp and ibs_op//ppp are all same). The precise modifier
+works with event types 0x76 (cpu-cycles, CPU clocks not halted) and 0xC1
+(micro-ops retired). Both events map to IBS execution sampling (IBS op)
+with the IBS Op Counter Control bit (IbsOpCntCtl) set respectively (see the
 Core Complex (CCX) -> Processor x86 Core -> Instruction Based Sampling (IBS)
 section of the [AMD Processor Programming Reference (PPR)] relevant to the
 family, model and stepping of the processor being used).
diff --git a/tools/perf/Documentation/perf-record.txt b/tools/perf/Documentation/perf-record.txt
index 1889f66addf2..6015fdd08fb6 100644
--- a/tools/perf/Documentation/perf-record.txt
+++ b/tools/perf/Documentation/perf-record.txt
@@ -445,6 +445,10 @@ following filters are defined:
 		     4th-Gen Xeon+ server), the save branch type is unconditionally enabled
 		     when the taken branch stack sampling is enabled.
 	- priv: save privilege state during sampling in case binary is not available later
+	- counter: save occurrences of the event since the last branch entry. Currently, the
+		   feature is only supported by a newer CPU, e.g., Intel Sierra Forest and
+		   later platforms. An error out is expected if it's used on the unsupported
+		   kernel or CPUs.
 
 +
 The option requires at least one branch type among any, any_call, any_ret, ind_call, cond.
diff --git a/tools/perf/Makefile.config b/tools/perf/Makefile.config
index b3e6ed10f40c..aa55850fbc21 100644
--- a/tools/perf/Makefile.config
+++ b/tools/perf/Makefile.config
@@ -476,6 +476,11 @@ else
       else
         CFLAGS += -DHAVE_DWARF_GETLOCATIONS_SUPPORT
       endif # dwarf_getlocations
+      ifneq ($(feature-dwarf_getcfi), 1)
+        msg := $(warning Old libdw.h, finding variables at given 'perf probe' point will not work, install elfutils-devel/libdw-dev >= 0.142);
+      else
+        CFLAGS += -DHAVE_DWARF_CFI_SUPPORT
+      endif # dwarf_getcfi
     endif # Dwarf support
   endif # libelf support
 endif # NO_LIBELF
@@ -680,15 +685,15 @@ ifndef BUILD_BPF_SKEL
 endif
 
 ifeq ($(BUILD_BPF_SKEL),1)
-  ifeq ($(filter -DHAVE_LIBBPF_SUPPORT, $(CFLAGS)),)
-    dummy := $(warning Warning: Disabled BPF skeletons as libbpf is required)
-    BUILD_BPF_SKEL := 0
-  else ifeq ($(filter -DHAVE_LIBELF_SUPPORT, $(CFLAGS)),)
+  ifeq ($(filter -DHAVE_LIBELF_SUPPORT, $(CFLAGS)),)
     dummy := $(warning Warning: Disabled BPF skeletons as libelf is required by bpftool)
     BUILD_BPF_SKEL := 0
   else ifeq ($(filter -DHAVE_ZLIB_SUPPORT, $(CFLAGS)),)
     dummy := $(warning Warning: Disabled BPF skeletons as zlib is required by bpftool)
     BUILD_BPF_SKEL := 0
+  else ifeq ($(filter -DHAVE_LIBBPF_SUPPORT, $(CFLAGS)),)
+    dummy := $(warning Warning: Disabled BPF skeletons as libbpf is required)
+    BUILD_BPF_SKEL := 0
   else ifeq ($(call get-executable,$(CLANG)),)
     dummy := $(warning Warning: Disabled BPF skeletons as clang ($(CLANG)) is missing)
     BUILD_BPF_SKEL := 0
diff --git a/tools/perf/arch/loongarch/annotate/instructions.c b/tools/perf/arch/loongarch/annotate/instructions.c
index 98e19c5366ac..21cc7e4149f7 100644
--- a/tools/perf/arch/loongarch/annotate/instructions.c
+++ b/tools/perf/arch/loongarch/annotate/instructions.c
@@ -61,10 +61,10 @@ static int loongarch_jump__parse(struct arch *arch, struct ins_operands *ops, st
 	const char *c = strchr(ops->raw, '#');
 	u64 start, end;
 
-	ops->raw_comment = strchr(ops->raw, arch->objdump.comment_char);
-	ops->raw_func_start = strchr(ops->raw, '<');
+	ops->jump.raw_comment = strchr(ops->raw, arch->objdump.comment_char);
+	ops->jump.raw_func_start = strchr(ops->raw, '<');
 
-	if (ops->raw_func_start && c > ops->raw_func_start)
+	if (ops->jump.raw_func_start && c > ops->jump.raw_func_start)
 		c = NULL;
 
 	if (c++ != NULL)
diff --git a/tools/perf/arch/mips/entry/syscalls/syscall_n64.tbl b/tools/perf/arch/mips/entry/syscalls/syscall_n64.tbl
index 80be0e98ea0c..81c772c0f5c8 100644
--- a/tools/perf/arch/mips/entry/syscalls/syscall_n64.tbl
+++ b/tools/perf/arch/mips/entry/syscalls/syscall_n64.tbl
@@ -367,3 +367,6 @@
 450	common	set_mempolicy_home_node		sys_set_mempolicy_home_node
 451	n64	cachestat			sys_cachestat
 452	n64	fchmodat2			sys_fchmodat2
+453	n64	lsm_get_self_attr		sys_lsm_get_self_attr
+454	n64	lsm_set_self_attr		sys_lsm_set_self_attr
+455	n64	lsm_list_modules		sys_lsm_list_modules
diff --git a/tools/perf/arch/powerpc/entry/syscalls/syscall.tbl b/tools/perf/arch/powerpc/entry/syscalls/syscall.tbl
index e1412519b4ad..861c6ca0a8c3 100644
--- a/tools/perf/arch/powerpc/entry/syscalls/syscall.tbl
+++ b/tools/perf/arch/powerpc/entry/syscalls/syscall.tbl
@@ -539,3 +539,6 @@
 450 	nospu	set_mempolicy_home_node		sys_set_mempolicy_home_node
 451	common	cachestat			sys_cachestat
 452	common	fchmodat2			sys_fchmodat2
+453	common	lsm_get_self_attr		sys_lsm_get_self_attr
+454	common	lsm_set_self_attr		sys_lsm_set_self_attr
+455	common	lsm_list_modules		sys_lsm_list_modules
diff --git a/tools/perf/arch/s390/entry/syscalls/syscall.tbl b/tools/perf/arch/s390/entry/syscalls/syscall.tbl
index cc0bc144b661..5a422443cb16 100644
--- a/tools/perf/arch/s390/entry/syscalls/syscall.tbl
+++ b/tools/perf/arch/s390/entry/syscalls/syscall.tbl
@@ -455,3 +455,6 @@
 450  common	set_mempolicy_home_node	sys_set_mempolicy_home_node	sys_set_mempolicy_home_node
 451  common	cachestat		sys_cachestat			sys_cachestat
 452  common	fchmodat2		sys_fchmodat2			sys_fchmodat2
+453  common	lsm_get_self_attr	sys_lsm_get_self_attr	sys_lsm_get_self_attr
+454  common	lsm_set_self_attr	sys_lsm_set_self_attr	sys_lsm_set_self_attr
+455  common	lsm_list_modules	sys_lsm_list_modules	sys_lsm_list_modules
diff --git a/tools/perf/arch/x86/entry/syscalls/syscall_64.tbl b/tools/perf/arch/x86/entry/syscalls/syscall_64.tbl
index 2a62eaf30d69..e692c88105a6 100644
--- a/tools/perf/arch/x86/entry/syscalls/syscall_64.tbl
+++ b/tools/perf/arch/x86/entry/syscalls/syscall_64.tbl
@@ -375,6 +375,9 @@
 451	common	cachestat		sys_cachestat
 452	common	fchmodat2		sys_fchmodat2
 453	64	map_shadow_stack	sys_map_shadow_stack
+454	common	lsm_get_self_attr	sys_lsm_get_self_attr
+455	common	lsm_set_self_attr	sys_lsm_set_self_attr
+456	common	lsm_list_modules	sys_lsm_list_modules
 
 #
 # Due to a historical design error, certain syscalls are numbered differently
diff --git a/tools/perf/builtin-annotate.c b/tools/perf/builtin-annotate.c
index aeeb801f1ed7..a9129b51d511 100644
--- a/tools/perf/builtin-annotate.c
+++ b/tools/perf/builtin-annotate.c
@@ -94,6 +94,7 @@ static void process_basic_block(struct addr_map_symbol *start,
 	struct annotation *notes = sym ? symbol__annotation(sym) : NULL;
 	struct block_range_iter iter;
 	struct block_range *entry;
+	struct annotated_branch *branch;
 
 	/*
 	 * Sanity; NULL isn't executable and the CPU cannot execute backwards
@@ -105,6 +106,8 @@ static void process_basic_block(struct addr_map_symbol *start,
 	if (!block_range_iter__valid(&iter))
 		return;
 
+	branch = annotation__get_branch(notes);
+
 	/*
 	 * First block in range is a branch target.
 	 */
@@ -118,8 +121,8 @@ static void process_basic_block(struct addr_map_symbol *start,
 		entry->coverage++;
 		entry->sym = sym;
 
-		if (notes)
-			notes->max_coverage = max(notes->max_coverage, entry->coverage);
+		if (branch)
+			branch->max_coverage = max(branch->max_coverage, entry->coverage);
 
 	} while (block_range_iter__next(&iter));
 
diff --git a/tools/perf/builtin-inject.c b/tools/perf/builtin-inject.c
index c8cf2fdd9cff..eb3ef5c24b66 100644
--- a/tools/perf/builtin-inject.c
+++ b/tools/perf/builtin-inject.c
@@ -2265,6 +2265,12 @@ int cmd_inject(int argc, const char **argv)
 		"perf inject [<options>]",
 		NULL
 	};
+
+	if (!inject.itrace_synth_opts.set) {
+		/* Disable eager loading of kernel symbols that adds overhead to perf inject. */
+		symbol_conf.lazy_load_kernel_maps = true;
+	}
+
 #ifndef HAVE_JITDUMP
 	set_option_nobuild(options, 'j', "jit", "NO_LIBELF=1", true);
 #endif
diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c
index dcf288a4fb9a..8ec818568662 100644
--- a/tools/perf/builtin-record.c
+++ b/tools/perf/builtin-record.c
@@ -3989,6 +3989,8 @@ int cmd_record(int argc, const char **argv)
 # undef set_nobuild
 #endif
 
+	/* Disable eager loading of kernel symbols that adds overhead to perf record. */
+	symbol_conf.lazy_load_kernel_maps = true;
 	rec->opts.affinity = PERF_AFFINITY_SYS;
 
 	rec->evlist = evlist__new();
diff --git a/tools/perf/builtin-report.c b/tools/perf/builtin-report.c
index 9cb1da2dc0c0..121a2781323c 100644
--- a/tools/perf/builtin-report.c
+++ b/tools/perf/builtin-report.c
@@ -1426,6 +1426,13 @@ int cmd_report(int argc, const char **argv)
 	if (ret < 0)
 		goto exit;
 
+	/*
+	 * tasks_mode require access to exited threads to list those that are in
+	 * the data file. Off-cpu events are synthesized after other events and
+	 * reference exited threads.
+	 */
+	symbol_conf.keep_exited_threads = true;
+
 	annotation_options__init(&report.annotation_opts);
 
 	ret = perf_config(report__config, &report);
diff --git a/tools/perf/pmu-events/arch/riscv/mapfile.csv b/tools/perf/pmu-events/arch/riscv/mapfile.csv
index c61b3d6ef616..ee61e26f90cd 100644
--- a/tools/perf/pmu-events/arch/riscv/mapfile.csv
+++ b/tools/perf/pmu-events/arch/riscv/mapfile.csv
@@ -15,3 +15,4 @@
 #
 #MVENDORID-MARCHID-MIMPID,Version,Filename,EventType
 0x489-0x8000000000000007-0x[[:xdigit:]]+,v1,sifive/u74,core
+0x67e-0x80000000db000080-0x[[:xdigit:]]+,v1,starfive/dubhe-80,core
diff --git a/tools/perf/pmu-events/arch/riscv/starfive/dubhe-80/common.json b/tools/perf/pmu-events/arch/riscv/starfive/dubhe-80/common.json
new file mode 100644
index 000000000000..fbffcacb2ace
--- /dev/null
+++ b/tools/perf/pmu-events/arch/riscv/starfive/dubhe-80/common.json
@@ -0,0 +1,172 @@
+[
+  {
+    "EventName": "ACCESS_MMU_STLB",
+    "EventCode": "0x1",
+    "BriefDescription": "access MMU STLB"
+  },
+  {
+    "EventName": "MISS_MMU_STLB",
+    "EventCode": "0x2",
+    "BriefDescription": "miss MMU STLB"
+  },
+  {
+    "EventName": "ACCESS_MMU_PTE_C",
+    "EventCode": "0x3",
+    "BriefDescription": "access MMU PTE-Cache"
+  },
+  {
+    "EventName": "MISS_MMU_PTE_C",
+    "EventCode": "0x4",
+    "BriefDescription": "miss MMU PTE-Cache"
+  },
+  {
+    "EventName": "ROB_FLUSH",
+    "EventCode": "0x5",
+    "BriefDescription": "ROB flush (all kinds of exceptions)"
+  },
+  {
+    "EventName": "BTB_PREDICTION_MISS",
+    "EventCode": "0x6",
+    "BriefDescription": "BTB prediction miss"
+  },
+  {
+    "EventName": "ITLB_MISS",
+    "EventCode": "0x7",
+    "BriefDescription": "ITLB miss"
+  },
+  {
+    "EventName": "SYNC_DEL_FETCH_G",
+    "EventCode": "0x8",
+    "BriefDescription": "SYNC delivery a fetch-group"
+  },
+  {
+    "EventName": "ICACHE_MISS",
+    "EventCode": "0x9",
+    "BriefDescription": "ICache miss"
+  },
+  {
+    "EventName": "BPU_BR_RETIRE",
+    "EventCode": "0xA",
+    "BriefDescription": "condition branch instruction retire"
+  },
+  {
+    "EventName": "BPU_BR_MISS",
+    "EventCode": "0xB",
+    "BriefDescription": "condition branch instruction miss"
+  },
+  {
+    "EventName": "RET_INS_RETIRE",
+    "EventCode": "0xC",
+    "BriefDescription": "return instruction retire"
+  },
+  {
+    "EventName": "RET_INS_MISS",
+    "EventCode": "0xD",
+    "BriefDescription": "return instruction miss"
+  },
+  {
+    "EventName": "INDIRECT_JR_MISS",
+    "EventCode": "0xE",
+    "BriefDescription": "indirect JR instruction miss (inlcude without target)"
+  },
+  {
+    "EventName": "IBUF_VAL_ID_NORDY",
+    "EventCode": "0xF",
+    "BriefDescription": "IBUF valid while ID not ready"
+  },
+  {
+    "EventName": "IBUF_NOVAL_ID_RDY",
+    "EventCode": "0x10",
+    "BriefDescription": "IBUF not valid while ID ready"
+  },
+  {
+    "EventName": "REN_INT_PHY_REG_NORDY",
+    "EventCode": "0x11",
+    "BriefDescription": "REN integer physical register file is not ready"
+  },
+  {
+    "EventName": "REN_FP_PHY_REG_NORDY",
+    "EventCode": "0x12",
+    "BriefDescription": "REN floating point physical register file is not ready"
+  },
+  {
+    "EventName": "REN_CP_NORDY",
+    "EventCode": "0x13",
+    "BriefDescription": "REN checkpoint is not ready"
+  },
+  {
+    "EventName": "DEC_VAL_ROB_NORDY",
+    "EventCode": "0x14",
+    "BriefDescription": "DEC is valid and ROB is not ready"
+  },
+  {
+    "EventName": "OOD_FLUSH_LS_DEP",
+    "EventCode": "0x15",
+    "BriefDescription": "out of order flush due to load/store dependency"
+  },
+  {
+    "EventName": "BRU_RET_IJR_INS",
+    "EventCode": "0x16",
+    "BriefDescription": "BRU retire an IJR instruction"
+  },
+  {
+    "EventName": "ACCESS_DTLB",
+    "EventCode": "0x17",
+    "BriefDescription": "access DTLB"
+  },
+  {
+    "EventName": "MISS_DTLB",
+    "EventCode": "0x18",
+    "BriefDescription": "miss DTLB"
+  },
+  {
+    "EventName": "LOAD_INS_DCACHE",
+    "EventCode": "0x19",
+    "BriefDescription": "load instruction access DCache"
+  },
+  {
+    "EventName": "LOAD_INS_MISS_DCACHE",
+    "EventCode": "0x1A",
+    "BriefDescription": "load instruction miss DCache"
+  },
+  {
+    "EventName": "STORE_INS_DCACHE",
+    "EventCode": "0x1B",
+    "BriefDescription": "store/amo instruction access DCache"
+  },
+  {
+    "EventName": "STORE_INS_MISS_DCACHE",
+    "EventCode": "0x1C",
+    "BriefDescription": "store/amo instruction miss DCache"
+  },
+  {
+    "EventName": "LOAD_SCACHE",
+    "EventCode": "0x1D",
+    "BriefDescription": "load access SCache"
+  },
+  {
+    "EventName": "STORE_SCACHE",
+    "EventCode": "0x1E",
+    "BriefDescription": "store access SCache"
+  },
+  {
+    "EventName": "LOAD_MISS_SCACHE",
+    "EventCode": "0x1F",
+    "BriefDescription": "load miss SCache"
+  },
+  {
+    "EventName": "STORE_MISS_SCACHE",
+    "EventCode": "0x20",
+    "BriefDescription": "store miss SCache"
+  },
+  {
+    "EventName": "L2C_PF_REQ",
+    "EventCode": "0x21",
+    "BriefDescription": "L2C data-prefetcher request"
+  },
+  {
+    "EventName": "L2C_PF_HIT",
+    "EventCode": "0x22",
+    "BriefDescription": "L2C data-prefetcher hit"
+  }
+]
diff --git a/tools/perf/pmu-events/arch/riscv/starfive/dubhe-80/firmware.json b/tools/perf/pmu-events/arch/riscv/starfive/dubhe-80/firmware.json
new file mode 100644
index 000000000000..9b4a032186a7
--- /dev/null
+++ b/tools/perf/pmu-events/arch/riscv/starfive/dubhe-80/firmware.json
@@ -0,0 +1,68 @@
+[
+  {
+    "ArchStdEvent": "FW_MISALIGNED_LOAD"
+  },
+  {
+    "ArchStdEvent": "FW_MISALIGNED_STORE"
+  },
+  {
+    "ArchStdEvent": "FW_ACCESS_LOAD"
+  },
+  {
+    "ArchStdEvent": "FW_ACCESS_STORE"
+  },
+  {
+    "ArchStdEvent": "FW_ILLEGAL_INSN"
+  },
+  {
+    "ArchStdEvent": "FW_SET_TIMER"
+  },
+  {
+    "ArchStdEvent": "FW_IPI_SENT"
+  },
+  {
+    "ArchStdEvent": "FW_IPI_RECEIVED"
+  },
+  {
+    "ArchStdEvent": "FW_FENCE_I_SENT"
+  },
+  {
+    "ArchStdEvent": "FW_FENCE_I_RECEIVED"
+  },
+  {
+    "ArchStdEvent": "FW_SFENCE_VMA_SENT"
+  },
+  {
+    "ArchStdEvent": "FW_SFENCE_VMA_RECEIVED"
+  },
+  {
+    "ArchStdEvent": "FW_SFENCE_VMA_RECEIVED"
+  },
+  {
+    "ArchStdEvent": "FW_SFENCE_VMA_ASID_RECEIVED"
+  },
+  {
+    "ArchStdEvent": "FW_HFENCE_GVMA_SENT"
+  },
+  {
+    "ArchStdEvent": "FW_HFENCE_GVMA_RECEIVED"
+  },
+  {
+    "ArchStdEvent": "FW_HFENCE_GVMA_VMID_SENT"
+  },
+  {
+    "ArchStdEvent": "FW_HFENCE_GVMA_VMID_RECEIVED"
+  },
+  {
+    "ArchStdEvent": "FW_HFENCE_VVMA_SENT"
+  },
+  {
+    "ArchStdEvent": "FW_HFENCE_VVMA_RECEIVED"
+  },
+  {
+    "ArchStdEvent": "FW_HFENCE_VVMA_ASID_SENT"
+  },
+  {
+    "ArchStdEvent": "FW_HFENCE_VVMA_ASID_RECEIVED"
+  }
+]
diff --git a/tools/perf/tests/attr.c b/tools/perf/tests/attr.c
index 61186d0d1cfa..97e1bdd6ec0e 100644
--- a/tools/perf/tests/attr.c
+++ b/tools/perf/tests/attr.c
@@ -188,7 +188,7 @@ static int test__attr(struct test_suite *test __maybe_unused, int subtest __mayb
 	if (perf_pmus__num_core_pmus() > 1) {
 		/*
 		 * TODO: Attribute tests hard code the PMU type. If there are >1
-		 * core PMU then each PMU will have a different type whic
+		 * core PMU then each PMU will have a different type which
 		 * requires additional support.
 		 */
 		pr_debug("Skip test on hybrid systems");
diff --git a/tools/perf/tests/builtin-test.c b/tools/perf/tests/builtin-test.c
index cb6f1dd00dc4..113e92119e1d 100644
--- a/tools/perf/tests/builtin-test.c
+++ b/tools/perf/tests/builtin-test.c
@@ -14,6 +14,7 @@
 #include <sys/wait.h>
 #include <sys/stat.h>
 #include "builtin.h"
+#include "config.h"
 #include "hist.h"
 #include "intlist.h"
 #include "tests.h"
@@ -32,6 +33,7 @@
 
 static bool dont_fork;
 const char *dso_to_test;
+const char *test_objdump_path = "objdump";
 
 /*
  * List of architecture specific tests. Not a weak symbol as the array length is
@@ -513,6 +515,15 @@ static int run_workload(const char *work, int argc, const char **argv)
 	return -1;
 }
 
+static int perf_test__config(const char *var, const char *value,
+			     void *data __maybe_unused)
+{
+	if (!strcmp(var, "test.objdump"))
+		test_objdump_path = value;
+
+	return 0;
+}
+
 int cmd_test(int argc, const char **argv)
 {
 	const char *test_usage[] = {
@@ -529,6 +540,8 @@ int cmd_test(int argc, const char **argv)
 		    "Do not fork for testcase"),
 	OPT_STRING('w', "workload", &workload, "work", "workload to run for testing"),
 	OPT_STRING(0, "dso", &dso_to_test, "dso", "dso to test"),
+	OPT_STRING(0, "objdump", &test_objdump_path, "path",
+		   "objdump binary to use for disassembly and annotations"),
 	OPT_END()
 	};
 	const char * const test_subcommands[] = { "list", NULL };
@@ -538,6 +551,8 @@ int cmd_test(int argc, const char **argv)
         if (ret < 0)
                 return ret;
 
+	perf_config(perf_test__config, NULL);
+
 	/* Unbuffered output */
 	setvbuf(stdout, NULL, _IONBF, 0);
 
diff --git a/tools/perf/tests/code-reading.c b/tools/perf/tests/code-reading.c
index 3af81012014e..8620146d0378 100644
--- a/tools/perf/tests/code-reading.c
+++ b/tools/perf/tests/code-reading.c
@@ -185,7 +185,7 @@ static int read_via_objdump(const char *filename, u64 addr, void *buf,
 	int ret;
 
 	fmt = "%s -z -d --start-address=0x%"PRIx64" --stop-address=0x%"PRIx64" %s";
-	ret = snprintf(cmd, sizeof(cmd), fmt, "objdump", addr, addr + len,
+	ret = snprintf(cmd, sizeof(cmd), fmt, test_objdump_path, addr, addr + len,
 		       filename);
 	if (ret <= 0 || (size_t)ret >= sizeof(cmd))
 		return -1;
@@ -511,38 +511,6 @@ static void fs_something(void)
 	}
 }
 
-#ifdef __s390x__
-#include "header.h" // for get_cpuid()
-#endif
-
-static const char *do_determine_event(bool excl_kernel)
-{
-	const char *event = excl_kernel ? "cycles:u" : "cycles";
-
-#ifdef __s390x__
-	char cpuid[128], model[16], model_c[16], cpum_cf_v[16];
-	unsigned int family;
-	int ret, cpum_cf_a;
-
-	if (get_cpuid(cpuid, sizeof(cpuid)))
-		goto out_clocks;
-	ret = sscanf(cpuid, "%*[^,],%u,%[^,],%[^,],%[^,],%x", &family, model_c,
-		     model, cpum_cf_v, &cpum_cf_a);
-	if (ret != 5)		 /* Not available */
-		goto out_clocks;
-	if (excl_kernel && (cpum_cf_a & 4))
-		return event;
-	if (!excl_kernel && (cpum_cf_a & 2))
-		return event;
-
-	/* Fall through: missing authorization */
-out_clocks:
-	event = excl_kernel ? "cpu-clock:u" : "cpu-clock";
-
-#endif
-	return event;
-}
-
 static void do_something(void)
 {
 	fs_something();
@@ -583,8 +551,10 @@ static int do_test_code_reading(bool try_kcore)
 	int err = -1, ret;
 	pid_t pid;
 	struct map *map;
-	bool have_vmlinux, have_kcore, excl_kernel = false;
+	bool have_vmlinux, have_kcore;
 	struct dso *dso;
+	const char *events[] = { "cycles", "cycles:u", "cpu-clock", "cpu-clock:u", NULL };
+	int evidx = 0;
 
 	pid = getpid();
 
@@ -618,7 +588,7 @@ static int do_test_code_reading(bool try_kcore)
 
 	/* No point getting kernel events if there is no kernel object */
 	if (!have_vmlinux && !have_kcore)
-		excl_kernel = true;
+		evidx++;
 
 	threads = thread_map__new_by_tid(pid);
 	if (!threads) {
@@ -646,7 +616,7 @@ static int do_test_code_reading(bool try_kcore)
 		goto out_put;
 	}
 
-	while (1) {
+	while (events[evidx]) {
 		const char *str;
 
 		evlist = evlist__new();
@@ -657,7 +627,7 @@ static int do_test_code_reading(bool try_kcore)
 
 		perf_evlist__set_maps(&evlist->core, cpus, threads);
 
-		str = do_determine_event(excl_kernel);
+		str = events[evidx];
 		pr_debug("Parsing event '%s'\n", str);
 		ret = parse_event(evlist, str);
 		if (ret < 0) {
@@ -675,32 +645,32 @@ static int do_test_code_reading(bool try_kcore)
 
 		ret = evlist__open(evlist);
 		if (ret < 0) {
-			if (!excl_kernel) {
-				excl_kernel = true;
-				/*
-				 * Both cpus and threads are now owned by evlist
-				 * and will be freed by following perf_evlist__set_maps
-				 * call. Getting reference to keep them alive.
-				 */
-				perf_cpu_map__get(cpus);
-				perf_thread_map__get(threads);
-				perf_evlist__set_maps(&evlist->core, NULL, NULL);
-				evlist__delete(evlist);
-				evlist = NULL;
-				continue;
-			}
+			evidx++;
 
-			if (verbose > 0) {
+			if (events[evidx] == NULL && verbose > 0) {
 				char errbuf[512];
 				evlist__strerror_open(evlist, errno, errbuf, sizeof(errbuf));
 				pr_debug("perf_evlist__open() failed!\n%s\n", errbuf);
 			}
 
-			goto out_put;
+			/*
+			 * Both cpus and threads are now owned by evlist
+			 * and will be freed by following perf_evlist__set_maps
+			 * call. Getting reference to keep them alive.
+			 */
+			perf_cpu_map__get(cpus);
+			perf_thread_map__get(threads);
+			perf_evlist__set_maps(&evlist->core, NULL, NULL);
+			evlist__delete(evlist);
+			evlist = NULL;
+			continue;
 		}
 		break;
 	}
 
+	if (events[evidx] == NULL)
+		goto out_put;
+
 	ret = evlist__mmap(evlist, UINT_MAX);
 	if (ret < 0) {
 		pr_debug("evlist__mmap failed\n");
@@ -721,7 +691,7 @@ static int do_test_code_reading(bool try_kcore)
 		err = TEST_CODE_READING_NO_KERNEL_OBJ;
 	else if (!have_vmlinux && !try_kcore)
 		err = TEST_CODE_READING_NO_VMLINUX;
-	else if (excl_kernel)
+	else if (strstr(events[evidx], ":u"))
 		err = TEST_CODE_READING_NO_ACCESS;
 	else
 		err = TEST_CODE_READING_OK;
diff --git a/tools/perf/tests/make b/tools/perf/tests/make
index d9945ed25bc5..8a4da7eb637a 100644
--- a/tools/perf/tests/make
+++ b/tools/perf/tests/make
@@ -183,7 +183,7 @@ run += make_install_prefix_slash
 # run += make_install_pdf
 run += make_minimal
 
-old_libbpf := $(shell echo '\#include <bpf/libbpf.h>' | $(CC) -E -dM -x c -| egrep -q "define[[:space:]]+LIBBPF_MAJOR_VERSION[[:space:]]+0{1}")
+old_libbpf := $(shell echo '\#include <bpf/libbpf.h>' | $(CC) -E -dM -x c -| grep -q -E "define[[:space:]]+LIBBPF_MAJOR_VERSION[[:space:]]+0{1}")
 
 ifneq ($(old_libbpf),)
 run += make_libbpf_dynamic
diff --git a/tools/perf/tests/parse-events.c b/tools/perf/tests/parse-events.c
index f78be21a5999..e52f45c7c3d1 100644
--- a/tools/perf/tests/parse-events.c
+++ b/tools/perf/tests/parse-events.c
@@ -2549,7 +2549,7 @@ static int test__pmu_events(struct test_suite *test __maybe_unused, int subtest
 			if (strchr(ent->d_name, '.'))
 				continue;
 
-			/* exclude parametrized ones (name contains '?') */
+			/* exclude parameterized ones (name contains '?') */
 			n = snprintf(pmu_event, sizeof(pmu_event), "%s%s", path, ent->d_name);
 			if (n >= PATH_MAX) {
 				pr_err("pmu event name crossed PATH_MAX(%d) size\n", PATH_MAX);
@@ -2578,7 +2578,7 @@ static int test__pmu_events(struct test_suite *test __maybe_unused, int subtest
 			fclose(file);
 
 			if (is_event_parameterized == 1) {
-				pr_debug("skipping parametrized PMU event: %s which contains ?\n", pmu_event);
+				pr_debug("skipping parameterized PMU event: %s which contains ?\n", pmu_event);
 				continue;
 			}
 
diff --git a/tools/perf/tests/shell/record_offcpu.sh b/tools/perf/tests/shell/record_offcpu.sh
index a1ef8f0d2b5c..67c925f3a15a 100755
--- a/tools/perf/tests/shell/record_offcpu.sh
+++ b/tools/perf/tests/shell/record_offcpu.sh
@@ -77,9 +77,9 @@ test_offcpu_child() {
     err=1
     return
   fi
-  # each process waits for read and write, so it should be more than 800 events
+  # each process waits at least for poll, so it should be more than 400 events
   if ! perf report -i ${perfdata} -s comm -q -n -t ';' --percent-limit=90 | \
-    awk -F ";" '{ if (NF > 3 && int($3) < 800) exit 1; }'
+    awk -F ";" '{ if (NF > 3 && int($3) < 400) exit 1; }'
   then
     echo "Child task off-cpu test [Failed invalid output]"
     err=1
diff --git a/tools/perf/tests/shell/stat_all_pmu.sh b/tools/perf/tests/shell/stat_all_pmu.sh
index c77955419173..d2a3506e0d19 100755
--- a/tools/perf/tests/shell/stat_all_pmu.sh
+++ b/tools/perf/tests/shell/stat_all_pmu.sh
@@ -4,7 +4,7 @@
 
 set -e
 
-# Test all PMU events; however exclude parametrized ones (name contains '?')
+# Test all PMU events; however exclude parameterized ones (name contains '?')
 for p in $(perf list --raw-dump pmu | sed 's/[[:graph:]]\+?[[:graph:]]\+[[:space:]]//g'); do
   echo "Testing $p"
   result=$(perf stat -e "$p" true 2>&1)
diff --git a/tools/perf/tests/tests.h b/tools/perf/tests/tests.h
index b394f3ac2d66..dad3d7414142 100644
--- a/tools/perf/tests/tests.h
+++ b/tools/perf/tests/tests.h
@@ -207,5 +207,6 @@ DECLARE_WORKLOAD(brstack);
 DECLARE_WORKLOAD(datasym);
 
 extern const char *dso_to_test;
+extern const char *test_objdump_path;
 
 #endif /* TESTS_H */
diff --git a/tools/perf/trace/beauty/prctl_option.sh b/tools/perf/trace/beauty/prctl_option.sh
index 8059342ca412..9455d9672f14 100755
--- a/tools/perf/trace/beauty/prctl_option.sh
+++ b/tools/perf/trace/beauty/prctl_option.sh
@@ -4,9 +4,9 @@
 [ $# -eq 1 ] && header_dir=$1 || header_dir=tools/include/uapi/linux/
 
 printf "static const char *prctl_options[] = {\n"
-regex='^#define[[:space:]]{1}PR_(\w+)[[:space:]]*([[:xdigit:]]+)([[:space:]]*\/.*)?$'
+regex='^#define[[:space:]]{1}PR_(\w+)[[:space:]]*([[:xdigit:]]+)([[:space:]]*/.*)?$'
 grep -E $regex ${header_dir}/prctl.h | grep -v PR_SET_PTRACER | \
-	sed -r "s/$regex/\2 \1/g"	| \
+	sed -E "s%$regex%\2 \1%g"	| \
 	sort -n | xargs printf "\t[%s] = \"%s\",\n"
 printf "};\n"
 
diff --git a/tools/perf/trace/beauty/socket.sh b/tools/perf/trace/beauty/socket.sh
index 8bc7ba62203e..670c6db298ae 100755
--- a/tools/perf/trace/beauty/socket.sh
+++ b/tools/perf/trace/beauty/socket.sh
@@ -18,10 +18,10 @@ grep -E $ipproto_regex ${uapi_header_dir}/in.h | \
 printf "};\n\n"
 
 printf "static const char *socket_level[] = {\n"
-socket_level_regex='^#define[[:space:]]+SOL_(\w+)[[:space:]]+([[:digit:]]+)([[:space:]]+\/.*)?'
+socket_level_regex='^#define[[:space:]]+SOL_(\w+)[[:space:]]+([[:digit:]]+)([[:space:]]+/.*)?'
 
 grep -E $socket_level_regex ${beauty_header_dir}/socket.h | \
-	sed -r "s/$socket_level_regex/\2 \1/g"	| \
+	sed -E "s%$socket_level_regex%\2 \1%g"	| \
 	sort -n | xargs printf "\t[%s] = \"%s\",\n"
 printf "};\n\n"
 
diff --git a/tools/perf/ui/browsers/annotate.c b/tools/perf/ui/browsers/annotate.c
index ccdb2cd11fbf..163f916fff68 100644
--- a/tools/perf/ui/browsers/annotate.c
+++ b/tools/perf/ui/browsers/annotate.c
@@ -188,7 +188,7 @@ static void annotate_browser__draw_current_jump(struct ui_browser *browser)
 	 *  name right after the '<' token and probably treating this like a
 	 *  'call' instruction.
 	 */
-	target = notes->offsets[cursor->ops.target.offset];
+	target = notes->src->offsets[cursor->ops.target.offset];
 	if (target == NULL) {
 		ui_helpline__printf("WARN: jump target inconsistency, press 'o', notes->offsets[%#x] = NULL\n",
 				    cursor->ops.target.offset);
@@ -337,7 +337,7 @@ static void annotate_browser__calc_percent(struct annotate_browser *browser,
 				max_percent = percent;
 		}
 
-		if (max_percent < 0.01 && pos->al.ipc == 0) {
+		if (max_percent < 0.01 && (!pos->al.cycles || pos->al.cycles->ipc == 0)) {
 			RB_CLEAR_NODE(&pos->al.rb_node);
 			continue;
 		}
@@ -384,7 +384,7 @@ static bool annotate_browser__toggle_source(struct annotate_browser *browser)
 		if (al->idx_asm < offset)
 			offset = al->idx;
 
-		browser->b.nr_entries = notes->nr_entries;
+		browser->b.nr_entries = notes->src->nr_entries;
 		notes->options->hide_src_code = false;
 		browser->b.seek(&browser->b, -offset, SEEK_CUR);
 		browser->b.top_idx = al->idx - offset;
@@ -402,7 +402,7 @@ static bool annotate_browser__toggle_source(struct annotate_browser *browser)
 		if (al->idx_asm < offset)
 			offset = al->idx_asm;
 
-		browser->b.nr_entries = notes->nr_asm_entries;
+		browser->b.nr_entries = notes->src->nr_asm_entries;
 		notes->options->hide_src_code = true;
 		browser->b.seek(&browser->b, -offset, SEEK_CUR);
 		browser->b.top_idx = al->idx_asm - offset;
@@ -435,7 +435,7 @@ static void ui_browser__init_asm_mode(struct ui_browser *browser)
 {
 	struct annotation *notes = browser__annotation(browser);
 	ui_browser__reset_index(browser);
-	browser->nr_entries = notes->nr_asm_entries;
+	browser->nr_entries = notes->src->nr_asm_entries;
 }
 
 static int sym_title(struct symbol *sym, struct map *map, char *title,
@@ -860,7 +860,7 @@ static int annotate_browser__run(struct annotate_browser *browser,
 					   browser->b.height,
 					   browser->b.index,
 					   browser->b.top_idx,
-					   notes->nr_asm_entries);
+					   notes->src->nr_asm_entries);
 		}
 			continue;
 		case K_ENTER:
@@ -991,8 +991,8 @@ int symbol__tui_annotate(struct map_symbol *ms, struct evsel *evsel,
 
 	ui_helpline__push("Press ESC to exit");
 
-	browser.b.width = notes->max_line_len;
-	browser.b.nr_entries = notes->nr_entries;
+	browser.b.width = notes->src->max_line_len;
+	browser.b.nr_entries = notes->src->nr_entries;
 	browser.b.entries = &notes->src->source,
 	browser.b.width += 18; /* Percentage */
 
@@ -1006,6 +1006,6 @@ int symbol__tui_annotate(struct map_symbol *ms, struct evsel *evsel,
 
 out_free_offsets:
 	if(not_annotated)
-		zfree(&notes->offsets);
+		zfree(&notes->src->offsets);
 	return ret;
 }
diff --git a/tools/perf/util/Build b/tools/perf/util/Build
index fb661c48992f..5e5e6c0d96ca 100644
--- a/tools/perf/util/Build
+++ b/tools/perf/util/Build
@@ -195,6 +195,7 @@ endif
 perf-$(CONFIG_DWARF) += probe-finder.o
 perf-$(CONFIG_DWARF) += dwarf-aux.o
 perf-$(CONFIG_DWARF) += dwarf-regs.o
+perf-$(CONFIG_DWARF) += debuginfo.o
 
 perf-$(CONFIG_LIBDW_DWARF_UNWIND) += unwind-libdw.o
 perf-$(CONFIG_LOCAL_LIBUNWIND)    += unwind-libunwind-local.o
diff --git a/tools/perf/util/annotate.c b/tools/perf/util/annotate.c
index 82956adf9963..3364edf30f50 100644
--- a/tools/perf/util/annotate.c
+++ b/tools/perf/util/annotate.c
@@ -340,10 +340,10 @@ bool ins__is_call(const struct ins *ins)
  */
 static inline const char *validate_comma(const char *c, struct ins_operands *ops)
 {
-	if (ops->raw_comment && c > ops->raw_comment)
+	if (ops->jump.raw_comment && c > ops->jump.raw_comment)
 		return NULL;
 
-	if (ops->raw_func_start && c > ops->raw_func_start)
+	if (ops->jump.raw_func_start && c > ops->jump.raw_func_start)
 		return NULL;
 
 	return c;
@@ -359,8 +359,8 @@ static int jump__parse(struct arch *arch, struct ins_operands *ops, struct map_s
 	const char *c = strchr(ops->raw, ',');
 	u64 start, end;
 
-	ops->raw_comment = strchr(ops->raw, arch->objdump.comment_char);
-	ops->raw_func_start = strchr(ops->raw, '<');
+	ops->jump.raw_comment = strchr(ops->raw, arch->objdump.comment_char);
+	ops->jump.raw_func_start = strchr(ops->raw, '<');
 
 	c = validate_comma(c, ops);
 
@@ -462,7 +462,16 @@ static int jump__scnprintf(struct ins *ins, char *bf, size_t size,
 			 ops->target.offset);
 }
 
+static void jump__delete(struct ins_operands *ops __maybe_unused)
+{
+	/*
+	 * The ops->jump.raw_comment and ops->jump.raw_func_start belong to the
+	 * raw string, don't free them.
+	 */
+}
+
 static struct ins_ops jump_ops = {
+	.free	   = jump__delete,
 	.parse	   = jump__parse,
 	.scnprintf = jump__scnprintf,
 };
@@ -810,7 +819,6 @@ static __maybe_unused void annotated_source__delete(struct annotated_source *src
 	if (src == NULL)
 		return;
 	zfree(&src->histograms);
-	zfree(&src->cycles_hist);
 	free(src);
 }
 
@@ -845,18 +853,6 @@ static int annotated_source__alloc_histograms(struct annotated_source *src,
 	return src->histograms ? 0 : -1;
 }
 
-/* The cycles histogram is lazily allocated. */
-static int symbol__alloc_hist_cycles(struct symbol *sym)
-{
-	struct annotation *notes = symbol__annotation(sym);
-	const size_t size = symbol__size(sym);
-
-	notes->src->cycles_hist = calloc(size, sizeof(struct cyc_hist));
-	if (notes->src->cycles_hist == NULL)
-		return -1;
-	return 0;
-}
-
 void symbol__annotate_zero_histograms(struct symbol *sym)
 {
 	struct annotation *notes = symbol__annotation(sym);
@@ -865,9 +861,10 @@ void symbol__annotate_zero_histograms(struct symbol *sym)
 	if (notes->src != NULL) {
 		memset(notes->src->histograms, 0,
 		       notes->src->nr_histograms * notes->src->sizeof_sym_hist);
-		if (notes->src->cycles_hist)
-			memset(notes->src->cycles_hist, 0,
-				symbol__size(sym) * sizeof(struct cyc_hist));
+	}
+	if (notes->branch && notes->branch->cycles_hist) {
+		memset(notes->branch->cycles_hist, 0,
+		       symbol__size(sym) * sizeof(struct cyc_hist));
 	}
 	annotation__unlock(notes);
 }
@@ -958,23 +955,33 @@ static int __symbol__inc_addr_samples(struct map_symbol *ms,
 	return 0;
 }
 
+struct annotated_branch *annotation__get_branch(struct annotation *notes)
+{
+	if (notes == NULL)
+		return NULL;
+
+	if (notes->branch == NULL)
+		notes->branch = zalloc(sizeof(*notes->branch));
+
+	return notes->branch;
+}
+
 static struct cyc_hist *symbol__cycles_hist(struct symbol *sym)
 {
 	struct annotation *notes = symbol__annotation(sym);
+	struct annotated_branch *branch;
 
-	if (notes->src == NULL) {
-		notes->src = annotated_source__new();
-		if (notes->src == NULL)
-			return NULL;
-		goto alloc_cycles_hist;
-	}
+	branch = annotation__get_branch(notes);
+	if (branch == NULL)
+		return NULL;
+
+	if (branch->cycles_hist == NULL) {
+		const size_t size = symbol__size(sym);
 
-	if (!notes->src->cycles_hist) {
-alloc_cycles_hist:
-		symbol__alloc_hist_cycles(sym);
+		branch->cycles_hist = calloc(size, sizeof(struct cyc_hist));
 	}
 
-	return notes->src->cycles_hist;
+	return branch->cycles_hist;
 }
 
 struct annotated_source *symbol__hists(struct symbol *sym, int nr_hists)
@@ -1077,12 +1084,20 @@ static unsigned annotation__count_insn(struct annotation *notes, u64 start, u64
 	u64 offset;
 
 	for (offset = start; offset <= end; offset++) {
-		if (notes->offsets[offset])
+		if (notes->src->offsets[offset])
 			n_insn++;
 	}
 	return n_insn;
 }
 
+static void annotated_branch__delete(struct annotated_branch *branch)
+{
+	if (branch) {
+		zfree(&branch->cycles_hist);
+		free(branch);
+	}
+}
+
 static void annotation__count_and_fill(struct annotation *notes, u64 start, u64 end, struct cyc_hist *ch)
 {
 	unsigned n_insn;
@@ -1091,6 +1106,7 @@ static void annotation__count_and_fill(struct annotation *notes, u64 start, u64
 
 	n_insn = annotation__count_insn(notes, start, end);
 	if (n_insn && ch->num && ch->cycles) {
+		struct annotated_branch *branch;
 		float ipc = n_insn / ((double)ch->cycles / (double)ch->num);
 
 		/* Hide data when there are too many overlaps. */
@@ -1098,54 +1114,76 @@ static void annotation__count_and_fill(struct annotation *notes, u64 start, u64
 			return;
 
 		for (offset = start; offset <= end; offset++) {
-			struct annotation_line *al = notes->offsets[offset];
+			struct annotation_line *al = notes->src->offsets[offset];
 
-			if (al && al->ipc == 0.0) {
-				al->ipc = ipc;
+			if (al && al->cycles && al->cycles->ipc == 0.0) {
+				al->cycles->ipc = ipc;
 				cover_insn++;
 			}
 		}
 
-		if (cover_insn) {
-			notes->hit_cycles += ch->cycles;
-			notes->hit_insn += n_insn * ch->num;
-			notes->cover_insn += cover_insn;
+		branch = annotation__get_branch(notes);
+		if (cover_insn && branch) {
+			branch->hit_cycles += ch->cycles;
+			branch->hit_insn += n_insn * ch->num;
+			branch->cover_insn += cover_insn;
 		}
 	}
 }
 
-void annotation__compute_ipc(struct annotation *notes, size_t size)
+static int annotation__compute_ipc(struct annotation *notes, size_t size)
 {
+	int err = 0;
 	s64 offset;
 
-	if (!notes->src || !notes->src->cycles_hist)
-		return;
+	if (!notes->branch || !notes->branch->cycles_hist)
+		return 0;
 
-	notes->total_insn = annotation__count_insn(notes, 0, size - 1);
-	notes->hit_cycles = 0;
-	notes->hit_insn = 0;
-	notes->cover_insn = 0;
+	notes->branch->total_insn = annotation__count_insn(notes, 0, size - 1);
+	notes->branch->hit_cycles = 0;
+	notes->branch->hit_insn = 0;
+	notes->branch->cover_insn = 0;
 
 	annotation__lock(notes);
 	for (offset = size - 1; offset >= 0; --offset) {
 		struct cyc_hist *ch;
 
-		ch = &notes->src->cycles_hist[offset];
+		ch = &notes->branch->cycles_hist[offset];
 		if (ch && ch->cycles) {
 			struct annotation_line *al;
 
+			al = notes->src->offsets[offset];
+			if (al && al->cycles == NULL) {
+				al->cycles = zalloc(sizeof(*al->cycles));
+				if (al->cycles == NULL) {
+					err = ENOMEM;
+					break;
+				}
+			}
 			if (ch->have_start)
 				annotation__count_and_fill(notes, ch->start, offset, ch);
-			al = notes->offsets[offset];
 			if (al && ch->num_aggr) {
-				al->cycles = ch->cycles_aggr / ch->num_aggr;
-				al->cycles_max = ch->cycles_max;
-				al->cycles_min = ch->cycles_min;
+				al->cycles->avg = ch->cycles_aggr / ch->num_aggr;
+				al->cycles->max = ch->cycles_max;
+				al->cycles->min = ch->cycles_min;
+			}
+		}
+	}
+
+	if (err) {
+		while (++offset < (s64)size) {
+			struct cyc_hist *ch = &notes->branch->cycles_hist[offset];
+
+			if (ch && ch->cycles) {
+				struct annotation_line *al = notes->src->offsets[offset];
+				if (al)
+					zfree(&al->cycles);
 			}
-			notes->have_cycles = true;
 		}
 	}
+
 	annotation__unlock(notes);
+	return 0;
 }
 
 int addr_map_symbol__inc_samples(struct addr_map_symbol *ams, struct perf_sample *sample,
@@ -1225,6 +1263,7 @@ static void annotation_line__exit(struct annotation_line *al)
 {
 	zfree_srcline(&al->path);
 	zfree(&al->line);
+	zfree(&al->cycles);
 }
 
 static size_t disasm_line_size(int nr)
@@ -1299,6 +1338,7 @@ int disasm_line__scnprintf(struct disasm_line *dl, char *bf, size_t size, bool r
 void annotation__exit(struct annotation *notes)
 {
 	annotated_source__delete(notes->src);
+	annotated_branch__delete(notes->branch);
 }
 
 static struct sharded_mutex *sharded_mutex;
@@ -2113,12 +2153,13 @@ static int symbol__disassemble(struct symbol *sym, struct annotate_args *args)
 	err = asprintf(&command,
 		 "%s %s%s --start-address=0x%016" PRIx64
 		 " --stop-address=0x%016" PRIx64
-		 " -l -d %s %s %s %c%s%c %s%s -C \"$1\"",
+		 " %s -d %s %s %s %c%s%c %s%s -C \"$1\"",
 		 opts->objdump_path ?: "objdump",
 		 opts->disassembler_style ? "-M " : "",
 		 opts->disassembler_style ?: "",
 		 map__rip_2objdump(map, sym->start),
 		 map__rip_2objdump(map, sym->end),
+		 opts->show_linenr ? "-l" : "",
 		 opts->show_asm_raw ? "" : "--no-show-raw-insn",
 		 opts->annotate_src ? "-S" : "",
 		 opts->prefix ? "--prefix " : "",
@@ -2769,7 +2810,7 @@ void annotation__mark_jump_targets(struct annotation *notes, struct symbol *sym)
 		return;
 
 	for (offset = 0; offset < size; ++offset) {
-		struct annotation_line *al = notes->offsets[offset];
+		struct annotation_line *al = notes->src->offsets[offset];
 		struct disasm_line *dl;
 
 		dl = disasm_line(al);
@@ -2777,7 +2818,7 @@ void annotation__mark_jump_targets(struct annotation *notes, struct symbol *sym)
 		if (!disasm_line__is_valid_local_jump(dl, sym))
 			continue;
 
-		al = notes->offsets[dl->ops.target.offset];
+		al = notes->src->offsets[dl->ops.target.offset];
 
 		/*
 		 * FIXME: Oops, no jump target? Buggy disassembler? Or do we
@@ -2794,19 +2835,20 @@ void annotation__mark_jump_targets(struct annotation *notes, struct symbol *sym)
 void annotation__set_offsets(struct annotation *notes, s64 size)
 {
 	struct annotation_line *al;
+	struct annotated_source *src = notes->src;
 
-	notes->max_line_len = 0;
-	notes->nr_entries = 0;
-	notes->nr_asm_entries = 0;
+	src->max_line_len = 0;
+	src->nr_entries = 0;
+	src->nr_asm_entries = 0;
 
-	list_for_each_entry(al, &notes->src->source, node) {
+	list_for_each_entry(al, &src->source, node) {
 		size_t line_len = strlen(al->line);
 
-		if (notes->max_line_len < line_len)
-			notes->max_line_len = line_len;
-		al->idx = notes->nr_entries++;
+		if (src->max_line_len < line_len)
+			src->max_line_len = line_len;
+		al->idx = src->nr_entries++;
 		if (al->offset != -1) {
-			al->idx_asm = notes->nr_asm_entries++;
+			al->idx_asm = src->nr_asm_entries++;
 			/*
 			 * FIXME: short term bandaid to cope with assembly
 			 * routines that comes with labels in the same column
@@ -2815,7 +2857,7 @@ void annotation__set_offsets(struct annotation *notes, s64 size)
 			 * E.g. copy_user_generic_unrolled
  			 */
 			if (al->offset < size)
-				notes->offsets[al->offset] = al;
+				notes->src->offsets[al->offset] = al;
 		} else
 			al->idx_asm = -1;
 	}
@@ -3052,13 +3094,14 @@ static void disasm_line__write(struct disasm_line *dl, struct annotation *notes,
 static void ipc_coverage_string(char *bf, int size, struct annotation *notes)
 {
 	double ipc = 0.0, coverage = 0.0;
+	struct annotated_branch *branch = annotation__get_branch(notes);
 
-	if (notes->hit_cycles)
-		ipc = notes->hit_insn / ((double)notes->hit_cycles);
+	if (branch && branch->hit_cycles)
+		ipc = branch->hit_insn / ((double)branch->hit_cycles);
 
-	if (notes->total_insn) {
-		coverage = notes->cover_insn * 100.0 /
-			((double)notes->total_insn);
+	if (branch && branch->total_insn) {
+		coverage = branch->cover_insn * 100.0 /
+			((double)branch->total_insn);
 	}
 
 	scnprintf(bf, size, "(Average IPC: %.2f, IPC Coverage: %.1f%%)",
@@ -3083,8 +3126,8 @@ static void __annotation_line__write(struct annotation_line *al, struct annotati
 	int printed;
 
 	if (first_line && (al->offset == -1 || percent_max == 0.0)) {
-		if (notes->have_cycles) {
-			if (al->ipc == 0.0 && al->cycles == 0)
+		if (notes->branch && al->cycles) {
+			if (al->cycles->ipc == 0.0 && al->cycles->avg == 0)
 				show_title = true;
 		} else
 			show_title = true;
@@ -3120,18 +3163,18 @@ static void __annotation_line__write(struct annotation_line *al, struct annotati
 		}
 	}
 
-	if (notes->have_cycles) {
-		if (al->ipc)
-			obj__printf(obj, "%*.2f ", ANNOTATION__IPC_WIDTH - 1, al->ipc);
+	if (notes->branch) {
+		if (al->cycles && al->cycles->ipc)
+			obj__printf(obj, "%*.2f ", ANNOTATION__IPC_WIDTH - 1, al->cycles->ipc);
 		else if (!show_title)
 			obj__printf(obj, "%*s", ANNOTATION__IPC_WIDTH, " ");
 		else
 			obj__printf(obj, "%*s ", ANNOTATION__IPC_WIDTH - 1, "IPC");
 
 		if (!notes->options->show_minmax_cycle) {
-			if (al->cycles)
+			if (al->cycles && al->cycles->avg)
 				obj__printf(obj, "%*" PRIu64 " ",
-					   ANNOTATION__CYCLES_WIDTH - 1, al->cycles);
+					   ANNOTATION__CYCLES_WIDTH - 1, al->cycles->avg);
 			else if (!show_title)
 				obj__printf(obj, "%*s",
 					    ANNOTATION__CYCLES_WIDTH, " ");
@@ -3145,8 +3188,8 @@ static void __annotation_line__write(struct annotation_line *al, struct annotati
 
 				scnprintf(str, sizeof(str),
 					"%" PRIu64 "(%" PRIu64 "/%" PRIu64 ")",
-					al->cycles, al->cycles_min,
-					al->cycles_max);
+					al->cycles->avg, al->cycles->min,
+					al->cycles->max);
 
 				obj__printf(obj, "%*s ",
 					    ANNOTATION__MINMAX_CYCLES_WIDTH - 1,
@@ -3247,8 +3290,8 @@ int symbol__annotate2(struct map_symbol *ms, struct evsel *evsel,
 	size_t size = symbol__size(sym);
 	int nr_pcnt = 1, err;
 
-	notes->offsets = zalloc(size * sizeof(struct annotation_line *));
-	if (notes->offsets == NULL)
+	notes->src->offsets = zalloc(size * sizeof(struct annotation_line *));
+	if (notes->src->offsets == NULL)
 		return ENOMEM;
 
 	if (evsel__is_group_event(evsel))
@@ -3264,7 +3307,11 @@ int symbol__annotate2(struct map_symbol *ms, struct evsel *evsel,
 
 	annotation__set_offsets(notes, size);
 	annotation__mark_jump_targets(notes, sym);
-	annotation__compute_ipc(notes, size);
+
+	err = annotation__compute_ipc(notes, size);
+	if (err)
+		goto out_free_offsets;
+
 	annotation__init_column_widths(notes, sym);
 	notes->nr_events = nr_pcnt;
 
@@ -3274,7 +3321,7 @@ int symbol__annotate2(struct map_symbol *ms, struct evsel *evsel,
 	return 0;
 
 out_free_offsets:
-	zfree(&notes->offsets);
+	zfree(&notes->src->offsets);
 	return err;
 }
 
diff --git a/tools/perf/util/annotate.h b/tools/perf/util/annotate.h
index 962780559176..bc8b95e8b1be 100644
--- a/tools/perf/util/annotate.h
+++ b/tools/perf/util/annotate.h
@@ -31,8 +31,6 @@ struct ins {
 
 struct ins_operands {
 	char	*raw;
-	char	*raw_comment;
-	char	*raw_func_start;
 	struct {
 		char	*raw;
 		char	*name;
@@ -52,6 +50,10 @@ struct ins_operands {
 			struct ins	    ins;
 			struct ins_operands *ops;
 		} locked;
+		struct {
+			char	*raw_comment;
+			char	*raw_func_start;
+		} jump;
 	};
 };
 
@@ -130,6 +132,13 @@ struct annotation_data {
 	struct sym_hist_entry	 he;
 };
 
+struct cycles_info {
+	float			 ipc;
+	u64			 avg;
+	u64			 max;
+	u64			 min;
+};
+
 struct annotation_line {
 	struct list_head	 node;
 	struct rb_node		 rb_node;
@@ -137,12 +146,9 @@ struct annotation_line {
 	char			*line;
 	int			 line_nr;
 	char			*fileloc;
-	int			 jump_sources;
-	float			 ipc;
-	u64			 cycles;
-	u64			 cycles_max;
-	u64			 cycles_min;
 	char			*path;
+	struct cycles_info	*cycles;
+	int			 jump_sources;
 	u32			 idx;
 	int			 idx_asm;
 	int			 data_nr;
@@ -264,27 +270,30 @@ struct cyc_hist {
  * returns.
  */
 struct annotated_source {
-	struct list_head   source;
-	int    		   nr_histograms;
-	size_t		   sizeof_sym_hist;
-	struct cyc_hist	   *cycles_hist;
-	struct sym_hist	   *histograms;
+	struct list_head	source;
+	size_t			sizeof_sym_hist;
+	struct sym_hist		*histograms;
+	struct annotation_line	**offsets;
+	int    			nr_histograms;
+	int			nr_entries;
+	int			nr_asm_entries;
+	u16			max_line_len;
 };
 
-struct LOCKABLE annotation {
-	u64			max_coverage;
-	u64			start;
+struct annotated_branch {
 	u64			hit_cycles;
 	u64			hit_insn;
 	unsigned int		total_insn;
 	unsigned int		cover_insn;
+	struct cyc_hist		*cycles_hist;
+	u64			max_coverage;
+};
+
+struct LOCKABLE annotation {
+	u64			start;
 	struct annotation_options *options;
-	struct annotation_line	**offsets;
 	int			nr_events;
 	int			max_jump_sources;
-	int			nr_entries;
-	int			nr_asm_entries;
-	u16			max_line_len;
 	struct {
 		u8		addr;
 		u8		jumps;
@@ -293,8 +302,8 @@ struct LOCKABLE annotation {
 		u8		max_addr;
 		u8		max_ins_name;
 	} widths;
-	bool			have_cycles;
 	struct annotated_source *src;
+	struct annotated_branch *branch;
 };
 
 static inline void annotation__init(struct annotation *notes __maybe_unused)
@@ -308,10 +317,10 @@ bool annotation__trylock(struct annotation *notes) EXCLUSIVE_TRYLOCK_FUNCTION(tr
 
 static inline int annotation__cycles_width(struct annotation *notes)
 {
-	if (notes->have_cycles && notes->options->show_minmax_cycle)
+	if (notes->branch && notes->options->show_minmax_cycle)
 		return ANNOTATION__IPC_WIDTH + ANNOTATION__MINMAX_CYCLES_WIDTH;
 
-	return notes->have_cycles ? ANNOTATION__IPC_WIDTH + ANNOTATION__CYCLES_WIDTH : 0;
+	return notes->branch ? ANNOTATION__IPC_WIDTH + ANNOTATION__CYCLES_WIDTH : 0;
 }
 
 static inline int annotation__pcnt_width(struct annotation *notes)
@@ -325,7 +334,6 @@ static inline bool annotation_line__filter(struct annotation_line *al, struct an
 }
 
 void annotation__set_offsets(struct annotation *notes, s64 size);
-void annotation__compute_ipc(struct annotation *notes, size_t size);
 void annotation__mark_jump_targets(struct annotation *notes, struct symbol *sym);
 void annotation__update_column_widths(struct annotation *notes);
 void annotation__init_column_widths(struct annotation *notes, struct symbol *sym);
@@ -349,6 +357,8 @@ static inline struct annotation *symbol__annotation(struct symbol *sym)
 int addr_map_symbol__inc_samples(struct addr_map_symbol *ams, struct perf_sample *sample,
 				 struct evsel *evsel);
 
+struct annotated_branch *annotation__get_branch(struct annotation *notes);
+
 int addr_map_symbol__account_cycles(struct addr_map_symbol *ams,
 				    struct addr_map_symbol *start,
 				    unsigned cycles);
diff --git a/tools/perf/util/block-info.c b/tools/perf/util/block-info.c
index 591fc1edd385..08f82c1f166c 100644
--- a/tools/perf/util/block-info.c
+++ b/tools/perf/util/block-info.c
@@ -129,9 +129,9 @@ int block_info__process_sym(struct hist_entry *he, struct block_hist *bh,
 	al.sym = he->ms.sym;
 
 	notes = symbol__annotation(he->ms.sym);
-	if (!notes || !notes->src || !notes->src->cycles_hist)
+	if (!notes || !notes->branch || !notes->branch->cycles_hist)
 		return 0;
-	ch = notes->src->cycles_hist;
+	ch = notes->branch->cycles_hist;
 	for (unsigned int i = 0; i < symbol__size(he->ms.sym); i++) {
 		if (ch[i].num_aggr) {
 			struct block_info *bi;
diff --git a/tools/perf/util/block-range.c b/tools/perf/util/block-range.c
index 680e92774d0c..15c42196c24c 100644
--- a/tools/perf/util/block-range.c
+++ b/tools/perf/util/block-range.c
@@ -311,6 +311,7 @@ struct block_range_iter block_range__create(u64 start, u64 end)
 double block_range__coverage(struct block_range *br)
 {
 	struct symbol *sym;
+	struct annotated_branch *branch;
 
 	if (!br) {
 		if (block_ranges.blocks)
@@ -323,5 +324,9 @@ double block_range__coverage(struct block_range *br)
 	if (!sym)
 		return -1;
 
-	return (double)br->coverage / symbol__annotation(sym)->max_coverage;
+	branch = symbol__annotation(sym)->branch;
+	if (!branch)
+		return -1;
+
+	return (double)br->coverage / branch->max_coverage;
 }
diff --git a/tools/perf/util/bpf-event.c b/tools/perf/util/bpf-event.c
index 38fcf3ba5749..830711cae30d 100644
--- a/tools/perf/util/bpf-event.c
+++ b/tools/perf/util/bpf-event.c
@@ -386,6 +386,9 @@ int perf_event__synthesize_bpf_events(struct perf_session *session,
 	int err;
 	int fd;
 
+	if (opts->no_bpf_event)
+		return 0;
+
 	event = malloc(sizeof(event->bpf) + KSYM_NAME_LEN + machine->id_hdr_size);
 	if (!event)
 		return -1;
diff --git a/tools/perf/util/debuginfo.c b/tools/perf/util/debuginfo.c
new file mode 100644
index 000000000000..19acf4775d35
--- /dev/null
+++ b/tools/perf/util/debuginfo.c
@@ -0,0 +1,205 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * DWARF debug information handling code.  Copied from probe-finder.c.
+ *
+ * Written by Masami Hiramatsu <mhiramat@redhat.com>
+ */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <linux/zalloc.h>
+
+#include "build-id.h"
+#include "dso.h"
+#include "debug.h"
+#include "debuginfo.h"
+#include "symbol.h"
+
+#ifdef HAVE_DEBUGINFOD_SUPPORT
+#include <elfutils/debuginfod.h>
+#endif
+
+/* Dwarf FL wrappers */
+static char *debuginfo_path;	/* Currently dummy */
+
+static const Dwfl_Callbacks offline_callbacks = {
+	.find_debuginfo = dwfl_standard_find_debuginfo,
+	.debuginfo_path = &debuginfo_path,
+
+	.section_address = dwfl_offline_section_address,
+
+	/* We use this table for core files too.  */
+	.find_elf = dwfl_build_id_find_elf,
+};
+
+/* Get a Dwarf from offline image */
+static int debuginfo__init_offline_dwarf(struct debuginfo *dbg,
+					 const char *path)
+{
+	GElf_Addr dummy;
+	int fd;
+
+	fd = open(path, O_RDONLY);
+	if (fd < 0)
+		return fd;
+
+	dbg->dwfl = dwfl_begin(&offline_callbacks);
+	if (!dbg->dwfl)
+		goto error;
+
+	dwfl_report_begin(dbg->dwfl);
+	dbg->mod = dwfl_report_offline(dbg->dwfl, "", "", fd);
+	if (!dbg->mod)
+		goto error;
+
+	dbg->dbg = dwfl_module_getdwarf(dbg->mod, &dbg->bias);
+	if (!dbg->dbg)
+		goto error;
+
+	dwfl_module_build_id(dbg->mod, &dbg->build_id, &dummy);
+
+	dwfl_report_end(dbg->dwfl, NULL, NULL);
+
+	return 0;
+error:
+	if (dbg->dwfl)
+		dwfl_end(dbg->dwfl);
+	else
+		close(fd);
+	memset(dbg, 0, sizeof(*dbg));
+
+	return -ENOENT;
+}
+
+static struct debuginfo *__debuginfo__new(const char *path)
+{
+	struct debuginfo *dbg = zalloc(sizeof(*dbg));
+	if (!dbg)
+		return NULL;
+
+	if (debuginfo__init_offline_dwarf(dbg, path) < 0)
+		zfree(&dbg);
+	if (dbg)
+		pr_debug("Open Debuginfo file: %s\n", path);
+	return dbg;
+}
+
+enum dso_binary_type distro_dwarf_types[] = {
+	DSO_BINARY_TYPE__FEDORA_DEBUGINFO,
+	DSO_BINARY_TYPE__UBUNTU_DEBUGINFO,
+	DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO,
+	DSO_BINARY_TYPE__BUILDID_DEBUGINFO,
+	DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO,
+	DSO_BINARY_TYPE__NOT_FOUND,
+};
+
+struct debuginfo *debuginfo__new(const char *path)
+{
+	enum dso_binary_type *type;
+	char buf[PATH_MAX], nil = '\0';
+	struct dso *dso;
+	struct debuginfo *dinfo = NULL;
+	struct build_id bid;
+
+	/* Try to open distro debuginfo files */
+	dso = dso__new(path);
+	if (!dso)
+		goto out;
+
+	/* Set the build id for DSO_BINARY_TYPE__BUILDID_DEBUGINFO */
+	if (is_regular_file(path) && filename__read_build_id(path, &bid) > 0)
+		dso__set_build_id(dso, &bid);
+
+	for (type = distro_dwarf_types;
+	     !dinfo && *type != DSO_BINARY_TYPE__NOT_FOUND;
+	     type++) {
+		if (dso__read_binary_type_filename(dso, *type, &nil,
+						   buf, PATH_MAX) < 0)
+			continue;
+		dinfo = __debuginfo__new(buf);
+	}
+	dso__put(dso);
+
+out:
+	/* if failed to open all distro debuginfo, open given binary */
+	return dinfo ? : __debuginfo__new(path);
+}
+
+void debuginfo__delete(struct debuginfo *dbg)
+{
+	if (dbg) {
+		if (dbg->dwfl)
+			dwfl_end(dbg->dwfl);
+		free(dbg);
+	}
+}
+
+/* For the kernel module, we need a special code to get a DIE */
+int debuginfo__get_text_offset(struct debuginfo *dbg, Dwarf_Addr *offs,
+				bool adjust_offset)
+{
+	int n, i;
+	Elf32_Word shndx;
+	Elf_Scn *scn;
+	Elf *elf;
+	GElf_Shdr mem, *shdr;
+	const char *p;
+
+	elf = dwfl_module_getelf(dbg->mod, &dbg->bias);
+	if (!elf)
+		return -EINVAL;
+
+	/* Get the number of relocations */
+	n = dwfl_module_relocations(dbg->mod);
+	if (n < 0)
+		return -ENOENT;
+	/* Search the relocation related .text section */
+	for (i = 0; i < n; i++) {
+		p = dwfl_module_relocation_info(dbg->mod, i, &shndx);
+		if (strcmp(p, ".text") == 0) {
+			/* OK, get the section header */
+			scn = elf_getscn(elf, shndx);
+			if (!scn)
+				return -ENOENT;
+			shdr = gelf_getshdr(scn, &mem);
+			if (!shdr)
+				return -ENOENT;
+			*offs = shdr->sh_addr;
+			if (adjust_offset)
+				*offs -= shdr->sh_offset;
+		}
+	}
+	return 0;
+}
+
+#ifdef HAVE_DEBUGINFOD_SUPPORT
+int get_source_from_debuginfod(const char *raw_path,
+			       const char *sbuild_id, char **new_path)
+{
+	debuginfod_client *c = debuginfod_begin();
+	const char *p = raw_path;
+	int fd;
+
+	if (!c)
+		return -ENOMEM;
+
+	fd = debuginfod_find_source(c, (const unsigned char *)sbuild_id,
+				0, p, new_path);
+	pr_debug("Search %s from debuginfod -> %d\n", p, fd);
+	if (fd >= 0)
+		close(fd);
+	debuginfod_end(c);
+	if (fd < 0) {
+		pr_debug("Failed to find %s in debuginfod (%s)\n",
+			raw_path, sbuild_id);
+		return -ENOENT;
+	}
+	pr_debug("Got a source %s\n", *new_path);
+
+	return 0;
+}
+#endif /* HAVE_DEBUGINFOD_SUPPORT */
diff --git a/tools/perf/util/debuginfo.h b/tools/perf/util/debuginfo.h
new file mode 100644
index 000000000000..4d65b8c605fc
--- /dev/null
+++ b/tools/perf/util/debuginfo.h
@@ -0,0 +1,64 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#ifndef _PERF_DEBUGINFO_H
+#define _PERF_DEBUGINFO_H
+
+#include <errno.h>
+#include <linux/compiler.h>
+
+#ifdef HAVE_DWARF_SUPPORT
+
+#include "dwarf-aux.h"
+
+/* debug information structure */
+struct debuginfo {
+	Dwarf		*dbg;
+	Dwfl_Module	*mod;
+	Dwfl		*dwfl;
+	Dwarf_Addr	bias;
+	const unsigned char	*build_id;
+};
+
+/* This also tries to open distro debuginfo */
+struct debuginfo *debuginfo__new(const char *path);
+void debuginfo__delete(struct debuginfo *dbg);
+
+int debuginfo__get_text_offset(struct debuginfo *dbg, Dwarf_Addr *offs,
+			       bool adjust_offset);
+
+#else /* HAVE_DWARF_SUPPORT */
+
+/* dummy debug information structure */
+struct debuginfo {
+};
+
+static inline struct debuginfo *debuginfo__new(const char *path __maybe_unused)
+{
+	return NULL;
+}
+
+static inline void debuginfo__delete(struct debuginfo *dbg __maybe_unused)
+{
+}
+
+static inline int debuginfo__get_text_offset(struct debuginfo *dbg __maybe_unused,
+					     Dwarf_Addr *offs __maybe_unused,
+					     bool adjust_offset __maybe_unused)
+{
+	return -EINVAL;
+}
+
+#endif /* HAVE_DWARF_SUPPORT */
+
+#ifdef HAVE_DEBUGINFOD_SUPPORT
+int get_source_from_debuginfod(const char *raw_path, const char *sbuild_id,
+			       char **new_path);
+#else /* HAVE_DEBUGINFOD_SUPPORT */
+static inline int get_source_from_debuginfod(const char *raw_path __maybe_unused,
+					     const char *sbuild_id __maybe_unused,
+					     char **new_path __maybe_unused)
+{
+	return -ENOTSUP;
+}
+#endif /* HAVE_DEBUGINFOD_SUPPORT */
+
+#endif /* _PERF_DEBUGINFO_H */
diff --git a/tools/perf/util/dwarf-aux.c b/tools/perf/util/dwarf-aux.c
index 2941d88f2199..652e6e7368a2 100644
--- a/tools/perf/util/dwarf-aux.c
+++ b/tools/perf/util/dwarf-aux.c
@@ -1090,7 +1090,14 @@ int die_get_typename(Dwarf_Die *vr_die, struct strbuf *buf)
 		return strbuf_addf(buf, "%s%s", tmp, name ?: "");
 	}
 	ret = die_get_typename(&type, buf);
-	return ret ? ret : strbuf_addstr(buf, tmp);
+	if (ret < 0) {
+		/* void pointer has no type attribute */
+		if (tag == DW_TAG_pointer_type && ret == -ENOENT)
+			return strbuf_addf(buf, "void*");
+
+		return ret;
+	}
+	return strbuf_addstr(buf, tmp);
 }
 
 /**
@@ -1238,12 +1245,72 @@ int die_get_var_range(Dwarf_Die *sp_die, Dwarf_Die *vr_die, struct strbuf *buf)
 out:
 	return ret;
 }
-#else
-int die_get_var_range(Dwarf_Die *sp_die __maybe_unused,
-		      Dwarf_Die *vr_die __maybe_unused,
-		      struct strbuf *buf __maybe_unused)
+
+/* Interval parameters for __die_find_var_reg_cb() */
+struct find_var_data {
+	/* Target instruction address */
+	Dwarf_Addr pc;
+	/* Target register */
+	unsigned reg;
+};
+
+/* Max number of registers DW_OP_regN supports */
+#define DWARF_OP_DIRECT_REGS  32
+
+/* Only checks direct child DIEs in the given scope. */
+static int __die_find_var_reg_cb(Dwarf_Die *die_mem, void *arg)
+{
+	struct find_var_data *data = arg;
+	int tag = dwarf_tag(die_mem);
+	ptrdiff_t off = 0;
+	Dwarf_Attribute attr;
+	Dwarf_Addr base, start, end;
+	Dwarf_Op *ops;
+	size_t nops;
+
+	if (tag != DW_TAG_variable && tag != DW_TAG_formal_parameter)
+		return DIE_FIND_CB_SIBLING;
+
+	if (dwarf_attr(die_mem, DW_AT_location, &attr) == NULL)
+		return DIE_FIND_CB_SIBLING;
+
+	while ((off = dwarf_getlocations(&attr, off, &base, &start, &end, &ops, &nops)) > 0) {
+		/* Assuming the location list is sorted by address */
+		if (end < data->pc)
+			continue;
+		if (start > data->pc)
+			break;
+
+		/* Only match with a simple case */
+		if (data->reg < DWARF_OP_DIRECT_REGS) {
+			if (ops->atom == (DW_OP_reg0 + data->reg) && nops == 1)
+				return DIE_FIND_CB_END;
+		} else {
+			if (ops->atom == DW_OP_regx && ops->number == data->reg &&
+			    nops == 1)
+				return DIE_FIND_CB_END;
+		}
+	}
+	return DIE_FIND_CB_SIBLING;
+}
+
+/**
+ * die_find_variable_by_reg - Find a variable saved in a register
+ * @sc_die: a scope DIE
+ * @pc: the program address to find
+ * @reg: the register number to find
+ * @die_mem: a buffer to save the resulting DIE
+ *
+ * Find the variable DIE accessed by the given register.
+ */
+Dwarf_Die *die_find_variable_by_reg(Dwarf_Die *sc_die, Dwarf_Addr pc, int reg,
+				    Dwarf_Die *die_mem)
 {
-	return -ENOTSUP;
+	struct find_var_data data = {
+		.pc = pc,
+		.reg = reg,
+	};
+	return die_find_child(sc_die, __die_find_var_reg_cb, &data, die_mem);
 }
 #endif
 
@@ -1425,3 +1492,56 @@ void die_skip_prologue(Dwarf_Die *sp_die, Dwarf_Die *cu_die,
 
 	*entrypc = postprologue_addr;
 }
+
+/* Internal parameters for __die_find_scope_cb() */
+struct find_scope_data {
+	/* Target instruction address */
+	Dwarf_Addr pc;
+	/* Number of scopes found [output] */
+	int nr;
+	/* Array of scopes found, 0 for the outermost one. [output] */
+	Dwarf_Die *scopes;
+};
+
+static int __die_find_scope_cb(Dwarf_Die *die_mem, void *arg)
+{
+	struct find_scope_data *data = arg;
+
+	if (dwarf_haspc(die_mem, data->pc)) {
+		Dwarf_Die *tmp;
+
+		tmp = realloc(data->scopes, (data->nr + 1) * sizeof(*tmp));
+		if (tmp == NULL)
+			return DIE_FIND_CB_END;
+
+		memcpy(tmp + data->nr, die_mem, sizeof(*die_mem));
+		data->scopes = tmp;
+		data->nr++;
+		return DIE_FIND_CB_CHILD;
+	}
+	return DIE_FIND_CB_SIBLING;
+}
+
+/**
+ * die_get_scopes - Return a list of scopes including the address
+ * @cu_die: a compile unit DIE
+ * @pc: the address to find
+ * @scopes: the array of DIEs for scopes (result)
+ *
+ * This function does the same as the dwarf_getscopes() but doesn't follow
+ * the origins of inlined functions.  It returns the number of scopes saved
+ * in the @scopes argument.  The outer scope will be saved first (index 0) and
+ * the last one is the innermost scope at the @pc.
+ */
+int die_get_scopes(Dwarf_Die *cu_die, Dwarf_Addr pc, Dwarf_Die **scopes)
+{
+	struct find_scope_data data = {
+		.pc = pc,
+	};
+	Dwarf_Die die_mem;
+
+	die_find_child(cu_die, __die_find_scope_cb, &data, &die_mem);
+
+	*scopes = data.scopes;
+	return data.nr;
+}
diff --git a/tools/perf/util/dwarf-aux.h b/tools/perf/util/dwarf-aux.h
index 7ec8bc1083bb..b6f430730bd1 100644
--- a/tools/perf/util/dwarf-aux.h
+++ b/tools/perf/util/dwarf-aux.h
@@ -121,7 +121,6 @@ int die_get_typename(Dwarf_Die *vr_die, struct strbuf *buf);
 
 /* Get the name and type of given variable DIE, stored as "type\tname" */
 int die_get_varname(Dwarf_Die *vr_die, struct strbuf *buf);
-int die_get_var_range(Dwarf_Die *sp_die, Dwarf_Die *vr_die, struct strbuf *buf);
 
 /* Check if target program is compiled with optimization */
 bool die_is_optimized_target(Dwarf_Die *cu_die);
@@ -130,4 +129,35 @@ bool die_is_optimized_target(Dwarf_Die *cu_die);
 void die_skip_prologue(Dwarf_Die *sp_die, Dwarf_Die *cu_die,
 		       Dwarf_Addr *entrypc);
 
-#endif
+/* Get the list of including scopes */
+int die_get_scopes(Dwarf_Die *cu_die, Dwarf_Addr pc, Dwarf_Die **scopes);
+
+#ifdef HAVE_DWARF_GETLOCATIONS_SUPPORT
+
+/* Get byte offset range of given variable DIE */
+int die_get_var_range(Dwarf_Die *sp_die, Dwarf_Die *vr_die, struct strbuf *buf);
+
+/* Find a variable saved in the 'reg' at given address */
+Dwarf_Die *die_find_variable_by_reg(Dwarf_Die *sc_die, Dwarf_Addr pc, int reg,
+				    Dwarf_Die *die_mem);
+
+#else /*  HAVE_DWARF_GETLOCATIONS_SUPPORT */
+
+static inline int die_get_var_range(Dwarf_Die *sp_die __maybe_unused,
+				    Dwarf_Die *vr_die __maybe_unused,
+				    struct strbuf *buf __maybe_unused)
+{
+	return -ENOTSUP;
+}
+
+static inline Dwarf_Die *die_find_variable_by_reg(Dwarf_Die *sc_die __maybe_unused,
+						  Dwarf_Addr pc __maybe_unused,
+						  int reg __maybe_unused,
+						  Dwarf_Die *die_mem __maybe_unused)
+{
+	return NULL;
+}
+
+#endif /* HAVE_DWARF_GETLOCATIONS_SUPPORT */
+
+#endif /* _DWARF_AUX_H */
diff --git a/tools/perf/util/env.c b/tools/perf/util/env.c
index 44140b7f596a..cbc18b22ace5 100644
--- a/tools/perf/util/env.c
+++ b/tools/perf/util/env.c
@@ -531,6 +531,24 @@ int perf_env__numa_node(struct perf_env *env, struct perf_cpu cpu)
 	return cpu.cpu >= 0 && cpu.cpu < env->nr_numa_map ? env->numa_map[cpu.cpu] : -1;
 }
 
+bool perf_env__has_pmu_mapping(struct perf_env *env, const char *pmu_name)
+{
+	char *pmu_mapping = env->pmu_mappings, *colon;
+
+	for (int i = 0; i < env->nr_pmu_mappings; ++i) {
+		if (strtoul(pmu_mapping, &colon, 0) == ULONG_MAX || *colon != ':')
+			goto out_error;
+
+		pmu_mapping = colon + 1;
+		if (strcmp(pmu_mapping, pmu_name) == 0)
+			return true;
+
+		pmu_mapping += strlen(pmu_mapping) + 1;
+	}
+out_error:
+	return false;
+}
+
 char *perf_env__find_pmu_cap(struct perf_env *env, const char *pmu_name,
 			     const char *cap)
 {
diff --git a/tools/perf/util/env.h b/tools/perf/util/env.h
index 4566c51f2fd9..94596ff124d5 100644
--- a/tools/perf/util/env.h
+++ b/tools/perf/util/env.h
@@ -46,6 +46,9 @@ struct hybrid_node {
 struct pmu_caps {
 	int		nr_caps;
 	unsigned int    max_branches;
+	unsigned int	br_cntr_nr;
+	unsigned int	br_cntr_width;
+
 	char            **caps;
 	char            *pmu_name;
 };
@@ -62,6 +65,8 @@ struct perf_env {
 	unsigned long long	total_mem;
 	unsigned int		msr_pmu_type;
 	unsigned int		max_branches;
+	unsigned int		br_cntr_nr;
+	unsigned int		br_cntr_width;
 	int			kernel_is_64_bit;
 
 	int			nr_cmdline;
@@ -174,4 +179,6 @@ struct btf_node *perf_env__find_btf(struct perf_env *env, __u32 btf_id);
 int perf_env__numa_node(struct perf_env *env, struct perf_cpu cpu);
 char *perf_env__find_pmu_cap(struct perf_env *env, const char *pmu_name,
 			     const char *cap);
+
+bool perf_env__has_pmu_mapping(struct perf_env *env, const char *pmu_name);
 #endif /* __PERF_ENV_H */
diff --git a/tools/perf/util/event.c b/tools/perf/util/event.c
index 923c0fb15122..68f45e9e63b6 100644
--- a/tools/perf/util/event.c
+++ b/tools/perf/util/event.c
@@ -617,13 +617,13 @@ struct map *thread__find_map(struct thread *thread, u8 cpumode, u64 addr,
 	if (cpumode == PERF_RECORD_MISC_KERNEL && perf_host) {
 		al->level = 'k';
 		maps = machine__kernel_maps(machine);
-		load_map = true;
+		load_map = !symbol_conf.lazy_load_kernel_maps;
 	} else if (cpumode == PERF_RECORD_MISC_USER && perf_host) {
 		al->level = '.';
 	} else if (cpumode == PERF_RECORD_MISC_GUEST_KERNEL && perf_guest) {
 		al->level = 'g';
 		maps = machine__kernel_maps(machine);
-		load_map = true;
+		load_map = !symbol_conf.lazy_load_kernel_maps;
 	} else if (cpumode == PERF_RECORD_MISC_GUEST_USER && perf_guest) {
 		al->level = 'u';
 	} else {
diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c
index 72a5dfc38d38..a5da74e3a517 100644
--- a/tools/perf/util/evsel.c
+++ b/tools/perf/util/evsel.c
@@ -1832,6 +1832,8 @@ static int __evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus,
 
 static void evsel__disable_missing_features(struct evsel *evsel)
 {
+	if (perf_missing_features.branch_counters)
+		evsel->core.attr.branch_sample_type &= ~PERF_SAMPLE_BRANCH_COUNTERS;
 	if (perf_missing_features.read_lost)
 		evsel->core.attr.read_format &= ~PERF_FORMAT_LOST;
 	if (perf_missing_features.weight_struct) {
@@ -1885,7 +1887,12 @@ bool evsel__detect_missing_features(struct evsel *evsel)
 	 * Must probe features in the order they were added to the
 	 * perf_event_attr interface.
 	 */
-	if (!perf_missing_features.read_lost &&
+	if (!perf_missing_features.branch_counters &&
+	    (evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS)) {
+		perf_missing_features.branch_counters = true;
+		pr_debug2("switching off branch counters support\n");
+		return true;
+	} else if (!perf_missing_features.read_lost &&
 	    (evsel->core.attr.read_format & PERF_FORMAT_LOST)) {
 		perf_missing_features.read_lost = true;
 		pr_debug2("switching off PERF_FORMAT_LOST support\n");
@@ -2318,6 +2325,22 @@ u64 evsel__bitfield_swap_branch_flags(u64 value)
 	return new_val;
 }
 
+static inline bool evsel__has_branch_counters(const struct evsel *evsel)
+{
+	struct evsel *cur, *leader = evsel__leader(evsel);
+
+	/* The branch counters feature only supports group */
+	if (!leader || !evsel->evlist)
+		return false;
+
+	evlist__for_each_entry(evsel->evlist, cur) {
+		if ((leader == evsel__leader(cur)) &&
+		    (cur->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS))
+			return true;
+	}
+	return false;
+}
+
 int evsel__parse_sample(struct evsel *evsel, union perf_event *event,
 			struct perf_sample *data)
 {
@@ -2551,6 +2574,16 @@ int evsel__parse_sample(struct evsel *evsel, union perf_event *event,
 
 		OVERFLOW_CHECK(array, sz, max_size);
 		array = (void *)array + sz;
+
+		if (evsel__has_branch_counters(evsel)) {
+			OVERFLOW_CHECK_u64(array);
+
+			data->branch_stack_cntr = (u64 *)array;
+			sz = data->branch_stack->nr * sizeof(u64);
+
+			OVERFLOW_CHECK(array, sz, max_size);
+			array = (void *)array + sz;
+		}
 	}
 
 	if (type & PERF_SAMPLE_REGS_USER) {
diff --git a/tools/perf/util/evsel.h b/tools/perf/util/evsel.h
index d791316a1792..f19ac9f027ef 100644
--- a/tools/perf/util/evsel.h
+++ b/tools/perf/util/evsel.h
@@ -191,6 +191,7 @@ struct perf_missing_features {
 	bool code_page_size;
 	bool weight_struct;
 	bool read_lost;
+	bool branch_counters;
 };
 
 extern struct perf_missing_features perf_missing_features;
diff --git a/tools/perf/util/header.c b/tools/perf/util/header.c
index e86b9439ffee..1c687b5789c0 100644
--- a/tools/perf/util/header.c
+++ b/tools/perf/util/header.c
@@ -2145,6 +2145,14 @@ static void print_pmu_caps(struct feat_fd *ff, FILE *fp)
 		__print_pmu_caps(fp, pmu_caps->nr_caps, pmu_caps->caps,
 				 pmu_caps->pmu_name);
 	}
+
+	if (strcmp(perf_env__arch(&ff->ph->env), "x86") == 0 &&
+	    perf_env__has_pmu_mapping(&ff->ph->env, "ibs_op")) {
+		char *max_precise = perf_env__find_pmu_cap(&ff->ph->env, "cpu", "max_precise");
+
+		if (max_precise != NULL && atoi(max_precise) == 0)
+			fprintf(fp, "# AMD systems uses ibs_op// PMU for some precise events, e.g.: cycles:p, see the 'perf list' man page for further details.\n");
+	}
 }
 
 static void print_pmu_mappings(struct feat_fd *ff, FILE *fp)
@@ -3259,7 +3267,9 @@ static int process_compressed(struct feat_fd *ff,
 }
 
 static int __process_pmu_caps(struct feat_fd *ff, int *nr_caps,
-			      char ***caps, unsigned int *max_branches)
+			      char ***caps, unsigned int *max_branches,
+			      unsigned int *br_cntr_nr,
+			      unsigned int *br_cntr_width)
 {
 	char *name, *value, *ptr;
 	u32 nr_pmu_caps, i;
@@ -3294,6 +3304,12 @@ static int __process_pmu_caps(struct feat_fd *ff, int *nr_caps,
 		if (!strcmp(name, "branches"))
 			*max_branches = atoi(value);
 
+		if (!strcmp(name, "branch_counter_nr"))
+			*br_cntr_nr = atoi(value);
+
+		if (!strcmp(name, "branch_counter_width"))
+			*br_cntr_width = atoi(value);
+
 		free(value);
 		free(name);
 	}
@@ -3318,7 +3334,9 @@ static int process_cpu_pmu_caps(struct feat_fd *ff,
 {
 	int ret = __process_pmu_caps(ff, &ff->ph->env.nr_cpu_pmu_caps,
 				     &ff->ph->env.cpu_pmu_caps,
-				     &ff->ph->env.max_branches);
+				     &ff->ph->env.max_branches,
+				     &ff->ph->env.br_cntr_nr,
+				     &ff->ph->env.br_cntr_width);
 
 	if (!ret && !ff->ph->env.cpu_pmu_caps)
 		pr_debug("cpu pmu capabilities not available\n");
@@ -3347,7 +3365,9 @@ static int process_pmu_caps(struct feat_fd *ff, void *data __maybe_unused)
 	for (i = 0; i < nr_pmu; i++) {
 		ret = __process_pmu_caps(ff, &pmu_caps[i].nr_caps,
 					 &pmu_caps[i].caps,
-					 &pmu_caps[i].max_branches);
+					 &pmu_caps[i].max_branches,
+					 &pmu_caps[i].br_cntr_nr,
+					 &pmu_caps[i].br_cntr_width);
 		if (ret)
 			goto err;
 
diff --git a/tools/perf/util/machine.c b/tools/perf/util/machine.c
index 90c750150b19..a985d004aa8d 100644
--- a/tools/perf/util/machine.c
+++ b/tools/perf/util/machine.c
@@ -2157,9 +2157,13 @@ int machine__process_exit_event(struct machine *machine, union perf_event *event
 	if (dump_trace)
 		perf_event__fprintf_task(event, stdout);
 
-	if (thread != NULL)
-		thread__put(thread);
-
+	if (thread != NULL) {
+		if (symbol_conf.keep_exited_threads)
+			thread__set_exited(thread, /*exited=*/true);
+		else
+			machine__remove_thread(machine, thread);
+	}
+	thread__put(thread);
 	return 0;
 }
 
diff --git a/tools/perf/util/parse-branch-options.c b/tools/perf/util/parse-branch-options.c
index fd67d204d720..f7f7aff3d85a 100644
--- a/tools/perf/util/parse-branch-options.c
+++ b/tools/perf/util/parse-branch-options.c
@@ -36,6 +36,7 @@ static const struct branch_mode branch_modes[] = {
 	BRANCH_OPT("stack", PERF_SAMPLE_BRANCH_CALL_STACK),
 	BRANCH_OPT("hw_index", PERF_SAMPLE_BRANCH_HW_INDEX),
 	BRANCH_OPT("priv", PERF_SAMPLE_BRANCH_PRIV_SAVE),
+	BRANCH_OPT("counter", PERF_SAMPLE_BRANCH_COUNTERS),
 	BRANCH_END
 };
 
diff --git a/tools/perf/util/perf_event_attr_fprintf.c b/tools/perf/util/perf_event_attr_fprintf.c
index 2247991451f3..8f04d3b7f3ec 100644
--- a/tools/perf/util/perf_event_attr_fprintf.c
+++ b/tools/perf/util/perf_event_attr_fprintf.c
@@ -55,6 +55,7 @@ static void __p_branch_sample_type(char *buf, size_t size, u64 value)
 		bit_name(COND), bit_name(CALL_STACK), bit_name(IND_JUMP),
 		bit_name(CALL), bit_name(NO_FLAGS), bit_name(NO_CYCLES),
 		bit_name(TYPE_SAVE), bit_name(HW_INDEX), bit_name(PRIV_SAVE),
+		bit_name(COUNTERS),
 		{ .name = NULL, }
 	};
 #undef bit_name
diff --git a/tools/perf/util/probe-finder.c b/tools/perf/util/probe-finder.c
index f171360b0ef4..c8923375e30d 100644
--- a/tools/perf/util/probe-finder.c
+++ b/tools/perf/util/probe-finder.c
@@ -23,6 +23,7 @@
 #include "event.h"
 #include "dso.h"
 #include "debug.h"
+#include "debuginfo.h"
 #include "intlist.h"
 #include "strbuf.h"
 #include "strlist.h"
@@ -31,128 +32,9 @@
 #include "probe-file.h"
 #include "string2.h"
 
-#ifdef HAVE_DEBUGINFOD_SUPPORT
-#include <elfutils/debuginfod.h>
-#endif
-
 /* Kprobe tracer basic type is up to u64 */
 #define MAX_BASIC_TYPE_BITS	64
 
-/* Dwarf FL wrappers */
-static char *debuginfo_path;	/* Currently dummy */
-
-static const Dwfl_Callbacks offline_callbacks = {
-	.find_debuginfo = dwfl_standard_find_debuginfo,
-	.debuginfo_path = &debuginfo_path,
-
-	.section_address = dwfl_offline_section_address,
-
-	/* We use this table for core files too.  */
-	.find_elf = dwfl_build_id_find_elf,
-};
-
-/* Get a Dwarf from offline image */
-static int debuginfo__init_offline_dwarf(struct debuginfo *dbg,
-					 const char *path)
-{
-	GElf_Addr dummy;
-	int fd;
-
-	fd = open(path, O_RDONLY);
-	if (fd < 0)
-		return fd;
-
-	dbg->dwfl = dwfl_begin(&offline_callbacks);
-	if (!dbg->dwfl)
-		goto error;
-
-	dwfl_report_begin(dbg->dwfl);
-	dbg->mod = dwfl_report_offline(dbg->dwfl, "", "", fd);
-	if (!dbg->mod)
-		goto error;
-
-	dbg->dbg = dwfl_module_getdwarf(dbg->mod, &dbg->bias);
-	if (!dbg->dbg)
-		goto error;
-
-	dwfl_module_build_id(dbg->mod, &dbg->build_id, &dummy);
-
-	dwfl_report_end(dbg->dwfl, NULL, NULL);
-
-	return 0;
-error:
-	if (dbg->dwfl)
-		dwfl_end(dbg->dwfl);
-	else
-		close(fd);
-	memset(dbg, 0, sizeof(*dbg));
-
-	return -ENOENT;
-}
-
-static struct debuginfo *__debuginfo__new(const char *path)
-{
-	struct debuginfo *dbg = zalloc(sizeof(*dbg));
-	if (!dbg)
-		return NULL;
-
-	if (debuginfo__init_offline_dwarf(dbg, path) < 0)
-		zfree(&dbg);
-	if (dbg)
-		pr_debug("Open Debuginfo file: %s\n", path);
-	return dbg;
-}
-
-enum dso_binary_type distro_dwarf_types[] = {
-	DSO_BINARY_TYPE__FEDORA_DEBUGINFO,
-	DSO_BINARY_TYPE__UBUNTU_DEBUGINFO,
-	DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO,
-	DSO_BINARY_TYPE__BUILDID_DEBUGINFO,
-	DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO,
-	DSO_BINARY_TYPE__NOT_FOUND,
-};
-
-struct debuginfo *debuginfo__new(const char *path)
-{
-	enum dso_binary_type *type;
-	char buf[PATH_MAX], nil = '\0';
-	struct dso *dso;
-	struct debuginfo *dinfo = NULL;
-	struct build_id bid;
-
-	/* Try to open distro debuginfo files */
-	dso = dso__new(path);
-	if (!dso)
-		goto out;
-
-	/* Set the build id for DSO_BINARY_TYPE__BUILDID_DEBUGINFO */
-	if (is_regular_file(path) && filename__read_build_id(path, &bid) > 0)
-		dso__set_build_id(dso, &bid);
-
-	for (type = distro_dwarf_types;
-	     !dinfo && *type != DSO_BINARY_TYPE__NOT_FOUND;
-	     type++) {
-		if (dso__read_binary_type_filename(dso, *type, &nil,
-						   buf, PATH_MAX) < 0)
-			continue;
-		dinfo = __debuginfo__new(buf);
-	}
-	dso__put(dso);
-
-out:
-	/* if failed to open all distro debuginfo, open given binary */
-	return dinfo ? : __debuginfo__new(path);
-}
-
-void debuginfo__delete(struct debuginfo *dbg)
-{
-	if (dbg) {
-		if (dbg->dwfl)
-			dwfl_end(dbg->dwfl);
-		free(dbg);
-	}
-}
-
 /*
  * Probe finder related functions
  */
@@ -722,7 +604,7 @@ static int call_probe_finder(Dwarf_Die *sc_die, struct probe_finder *pf)
 	ret = dwarf_getlocation_addr(&fb_attr, pf->addr, &pf->fb_ops, &nops, 1);
 	if (ret <= 0 || nops == 0) {
 		pf->fb_ops = NULL;
-#if _ELFUTILS_PREREQ(0, 142)
+#ifdef HAVE_DWARF_CFI_SUPPORT
 	} else if (nops == 1 && pf->fb_ops[0].atom == DW_OP_call_frame_cfa &&
 		   (pf->cfi_eh != NULL || pf->cfi_dbg != NULL)) {
 		if ((dwarf_cfi_addrframe(pf->cfi_eh, pf->addr, &frame) != 0 &&
@@ -733,7 +615,7 @@ static int call_probe_finder(Dwarf_Die *sc_die, struct probe_finder *pf)
 			free(frame);
 			return -ENOENT;
 		}
-#endif
+#endif /* HAVE_DWARF_CFI_SUPPORT */
 	}
 
 	/* Call finder's callback handler */
@@ -1258,7 +1140,7 @@ static int debuginfo__find_probes(struct debuginfo *dbg,
 
 	pf->machine = ehdr.e_machine;
 
-#if _ELFUTILS_PREREQ(0, 142)
+#ifdef HAVE_DWARF_CFI_SUPPORT
 	do {
 		GElf_Shdr shdr;
 
@@ -1268,7 +1150,7 @@ static int debuginfo__find_probes(struct debuginfo *dbg,
 
 		pf->cfi_dbg = dwarf_getcfi(dbg->dbg);
 	} while (0);
-#endif
+#endif /* HAVE_DWARF_CFI_SUPPORT */
 
 	ret = debuginfo__find_probe_location(dbg, pf);
 	return ret;
@@ -1677,44 +1559,6 @@ int debuginfo__find_available_vars_at(struct debuginfo *dbg,
 	return (ret < 0) ? ret : af.nvls;
 }
 
-/* For the kernel module, we need a special code to get a DIE */
-int debuginfo__get_text_offset(struct debuginfo *dbg, Dwarf_Addr *offs,
-				bool adjust_offset)
-{
-	int n, i;
-	Elf32_Word shndx;
-	Elf_Scn *scn;
-	Elf *elf;
-	GElf_Shdr mem, *shdr;
-	const char *p;
-
-	elf = dwfl_module_getelf(dbg->mod, &dbg->bias);
-	if (!elf)
-		return -EINVAL;
-
-	/* Get the number of relocations */
-	n = dwfl_module_relocations(dbg->mod);
-	if (n < 0)
-		return -ENOENT;
-	/* Search the relocation related .text section */
-	for (i = 0; i < n; i++) {
-		p = dwfl_module_relocation_info(dbg->mod, i, &shndx);
-		if (strcmp(p, ".text") == 0) {
-			/* OK, get the section header */
-			scn = elf_getscn(elf, shndx);
-			if (!scn)
-				return -ENOENT;
-			shdr = gelf_getshdr(scn, &mem);
-			if (!shdr)
-				return -ENOENT;
-			*offs = shdr->sh_addr;
-			if (adjust_offset)
-				*offs -= shdr->sh_offset;
-		}
-	}
-	return 0;
-}
-
 /* Reverse search */
 int debuginfo__find_probe_point(struct debuginfo *dbg, u64 addr,
 				struct perf_probe_point *ppt)
@@ -2009,41 +1853,6 @@ int debuginfo__find_line_range(struct debuginfo *dbg, struct line_range *lr)
 	return (ret < 0) ? ret : lf.found;
 }
 
-#ifdef HAVE_DEBUGINFOD_SUPPORT
-/* debuginfod doesn't require the comp_dir but buildid is required */
-static int get_source_from_debuginfod(const char *raw_path,
-				const char *sbuild_id, char **new_path)
-{
-	debuginfod_client *c = debuginfod_begin();
-	const char *p = raw_path;
-	int fd;
-
-	if (!c)
-		return -ENOMEM;
-
-	fd = debuginfod_find_source(c, (const unsigned char *)sbuild_id,
-				0, p, new_path);
-	pr_debug("Search %s from debuginfod -> %d\n", p, fd);
-	if (fd >= 0)
-		close(fd);
-	debuginfod_end(c);
-	if (fd < 0) {
-		pr_debug("Failed to find %s in debuginfod (%s)\n",
-			raw_path, sbuild_id);
-		return -ENOENT;
-	}
-	pr_debug("Got a source %s\n", *new_path);
-
-	return 0;
-}
-#else
-static inline int get_source_from_debuginfod(const char *raw_path __maybe_unused,
-				const char *sbuild_id __maybe_unused,
-				char **new_path __maybe_unused)
-{
-	return -ENOTSUP;
-}
-#endif
 /*
  * Find a src file from a DWARF tag path. Prepend optional source path prefix
  * and chop off leading directories that do not exist. Result is passed back as
diff --git a/tools/perf/util/probe-finder.h b/tools/perf/util/probe-finder.h
index 8bc1c80d3c1c..3add5ff516e1 100644
--- a/tools/perf/util/probe-finder.h
+++ b/tools/perf/util/probe-finder.h
@@ -24,21 +24,7 @@ static inline int is_c_varname(const char *name)
 #ifdef HAVE_DWARF_SUPPORT
 
 #include "dwarf-aux.h"
-
-/* TODO: export debuginfo data structure even if no dwarf support */
-
-/* debug information structure */
-struct debuginfo {
-	Dwarf		*dbg;
-	Dwfl_Module	*mod;
-	Dwfl		*dwfl;
-	Dwarf_Addr	bias;
-	const unsigned char	*build_id;
-};
-
-/* This also tries to open distro debuginfo */
-struct debuginfo *debuginfo__new(const char *path);
-void debuginfo__delete(struct debuginfo *dbg);
+#include "debuginfo.h"
 
 /* Find probe_trace_events specified by perf_probe_event from debuginfo */
 int debuginfo__find_trace_events(struct debuginfo *dbg,
@@ -49,9 +35,6 @@ int debuginfo__find_trace_events(struct debuginfo *dbg,
 int debuginfo__find_probe_point(struct debuginfo *dbg, u64 addr,
 				struct perf_probe_point *ppt);
 
-int debuginfo__get_text_offset(struct debuginfo *dbg, Dwarf_Addr *offs,
-			       bool adjust_offset);
-
 /* Find a line range */
 int debuginfo__find_line_range(struct debuginfo *dbg, struct line_range *lr);
 
diff --git a/tools/perf/util/s390-cpumcf-kernel.h b/tools/perf/util/s390-cpumcf-kernel.h
index f55ca07f3ca1..74b36644e384 100644
--- a/tools/perf/util/s390-cpumcf-kernel.h
+++ b/tools/perf/util/s390-cpumcf-kernel.h
@@ -12,6 +12,8 @@
 #define	S390_CPUMCF_DIAG_DEF	0xfeef	/* Counter diagnostic entry ID */
 #define	PERF_EVENT_CPUM_CF_DIAG	0xBC000	/* Event: Counter sets */
 #define PERF_EVENT_CPUM_SF_DIAG	0xBD000 /* Event: Combined-sampling */
+#define PERF_EVENT_PAI_CRYPTO_ALL	0x1000 /* Event: CRYPTO_ALL */
+#define PERF_EVENT_PAI_NNPA_ALL	0x1800 /* Event: NNPA_ALL */
 
 struct cf_ctrset_entry {	/* CPU-M CF counter set entry (8 byte) */
 	unsigned int def:16;	/* 0-15  Data Entry Format */
diff --git a/tools/perf/util/s390-sample-raw.c b/tools/perf/util/s390-sample-raw.c
index 115b16edb451..29a744eeb71e 100644
--- a/tools/perf/util/s390-sample-raw.c
+++ b/tools/perf/util/s390-sample-raw.c
@@ -125,6 +125,9 @@ static int get_counterset_start(int setnr)
 		return 128;
 	case CPUMF_CTR_SET_MT_DIAG:		/* Diagnostic counter set */
 		return 448;
+	case PERF_EVENT_PAI_NNPA_ALL:		/* PAI NNPA counter set */
+	case PERF_EVENT_PAI_CRYPTO_ALL:		/* PAI CRYPTO counter set */
+		return setnr;
 	default:
 		return -1;
 	}
@@ -212,27 +215,117 @@ static void s390_cpumcfdg_dump(struct perf_pmu *pmu, struct perf_sample *sample)
 	}
 }
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpacked"
+#pragma GCC diagnostic ignored "-Wattributes"
+/*
+ * Check for consistency of PAI_CRYPTO/PAI_NNPA raw data.
+ */
+struct pai_data {		/* Event number and value */
+	u16 event_nr;
+	u64 event_val;
+} __packed;
+
+#pragma GCC diagnostic pop
+
+/*
+ * Test for valid raw data. At least one PAI event should be in the raw
+ * data section.
+ */
+static bool s390_pai_all_test(struct perf_sample *sample)
+{
+	unsigned char *buf = sample->raw_data;
+	size_t len = sample->raw_size;
+
+	if (len < 0xa || !buf)
+		return false;
+	return true;
+}
+
+static void s390_pai_all_dump(struct evsel *evsel, struct perf_sample *sample)
+{
+	size_t len = sample->raw_size, offset = 0;
+	unsigned char *p = sample->raw_data;
+	const char *color = PERF_COLOR_BLUE;
+	struct pai_data pai_data;
+	char *ev_name;
+
+	while (offset < len) {
+		memcpy(&pai_data.event_nr, p, sizeof(pai_data.event_nr));
+		pai_data.event_nr = be16_to_cpu(pai_data.event_nr);
+		p += sizeof(pai_data.event_nr);
+		offset += sizeof(pai_data.event_nr);
+
+		memcpy(&pai_data.event_val, p, sizeof(pai_data.event_val));
+		pai_data.event_val = be64_to_cpu(pai_data.event_val);
+		p += sizeof(pai_data.event_val);
+		offset += sizeof(pai_data.event_val);
+
+		ev_name = get_counter_name(evsel->core.attr.config,
+					   pai_data.event_nr, evsel->pmu);
+		color_fprintf(stdout, color, "\tCounter:%03d %s Value:%#018lx\n",
+			      pai_data.event_nr, ev_name ?: "<unknown>",
+			      pai_data.event_val);
+		free(ev_name);
+
+		if (offset + 0xa > len)
+			break;
+	}
+	color_fprintf(stdout, color, "\n");
+}
+
 /* S390 specific trace event function. Check for PERF_RECORD_SAMPLE events
- * and if the event was triggered by a counter set diagnostic event display
- * its raw data.
+ * and if the event was triggered by a
+ * - counter set diagnostic event
+ * - processor activity assist (PAI) crypto counter event
+ * - processor activity assist (PAI) neural network processor assist (NNPA)
+ *   counter event
+ * display its raw data.
  * The function is only invoked when the dump flag -D is set.
+ *
+ * Function evlist__s390_sample_raw() is defined as call back after it has
+ * been verified that the perf.data file was created on s390 platform.
  */
-void evlist__s390_sample_raw(struct evlist *evlist, union perf_event *event, struct perf_sample *sample)
+void evlist__s390_sample_raw(struct evlist *evlist, union perf_event *event,
+			     struct perf_sample *sample)
 {
+	const char *pai_name;
 	struct evsel *evsel;
 
 	if (event->header.type != PERF_RECORD_SAMPLE)
 		return;
 
 	evsel = evlist__event2evsel(evlist, event);
-	if (evsel == NULL ||
-	    evsel->core.attr.config != PERF_EVENT_CPUM_CF_DIAG)
+	if (!evsel)
 		return;
 
 	/* Display raw data on screen */
-	if (!s390_cpumcfdg_testctr(sample)) {
-		pr_err("Invalid counter set data encountered\n");
+	if (evsel->core.attr.config == PERF_EVENT_CPUM_CF_DIAG) {
+		if (!evsel->pmu)
+			evsel->pmu = perf_pmus__find("cpum_cf");
+		if (!s390_cpumcfdg_testctr(sample))
+			pr_err("Invalid counter set data encountered\n");
+		else
+			s390_cpumcfdg_dump(evsel->pmu, sample);
 		return;
 	}
-	s390_cpumcfdg_dump(evsel->pmu, sample);
+
+	switch (evsel->core.attr.config) {
+	case PERF_EVENT_PAI_NNPA_ALL:
+		pai_name = "NNPA_ALL";
+		break;
+	case PERF_EVENT_PAI_CRYPTO_ALL:
+		pai_name = "CRYPTO_ALL";
+		break;
+	default:
+		return;
+	}
+
+	if (!s390_pai_all_test(sample)) {
+		pr_err("Invalid %s raw data encountered\n", pai_name);
+	} else {
+		if (!evsel->pmu)
+			evsel->pmu = perf_pmus__find_by_type(evsel->core.attr.type);
+		s390_pai_all_dump(evsel, sample);
+	}
 }
diff --git a/tools/perf/util/sample.h b/tools/perf/util/sample.h
index c92ad0f51ecd..70b2c3135555 100644
--- a/tools/perf/util/sample.h
+++ b/tools/perf/util/sample.h
@@ -113,6 +113,7 @@ struct perf_sample {
 	void *raw_data;
 	struct ip_callchain *callchain;
 	struct branch_stack *branch_stack;
+	u64 *branch_stack_cntr;
 	struct regs_dump  user_regs;
 	struct regs_dump  intr_regs;
 	struct stack_dump user_stack;
diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c
index 1e9aa8ed15b6..199d3e8df315 100644
--- a/tools/perf/util/session.c
+++ b/tools/perf/util/session.c
@@ -115,6 +115,11 @@ static int perf_session__open(struct perf_session *session, int repipe_fd)
 		return -1;
 	}
 
+	if (perf_header__has_feat(&session->header, HEADER_AUXTRACE)) {
+		/* Auxiliary events may reference exited threads, hold onto dead ones. */
+		symbol_conf.keep_exited_threads = true;
+	}
+
 	if (perf_data__is_pipe(data))
 		return 0;
 
@@ -1150,9 +1155,13 @@ static void callchain__printf(struct evsel *evsel,
 		       i, callchain->ips[i]);
 }
 
-static void branch_stack__printf(struct perf_sample *sample, bool callstack)
+static void branch_stack__printf(struct perf_sample *sample,
+				 struct evsel *evsel)
 {
 	struct branch_entry *entries = perf_sample__branch_entries(sample);
+	bool callstack = evsel__has_branch_callstack(evsel);
+	u64 *branch_stack_cntr = sample->branch_stack_cntr;
+	struct perf_env *env = evsel__env(evsel);
 	uint64_t i;
 
 	if (!callstack) {
@@ -1194,6 +1203,13 @@ static void branch_stack__printf(struct perf_sample *sample, bool callstack)
 			}
 		}
 	}
+
+	if (branch_stack_cntr) {
+		printf("... branch stack counters: nr:%" PRIu64 " (counter width: %u max counter nr:%u)\n",
+			sample->branch_stack->nr, env->br_cntr_width, env->br_cntr_nr);
+		for (i = 0; i < sample->branch_stack->nr; i++)
+			printf("..... %2"PRIu64": %016" PRIx64 "\n", i, branch_stack_cntr[i]);
+	}
 }
 
 static void regs_dump__printf(u64 mask, u64 *regs, const char *arch)
@@ -1355,7 +1371,7 @@ static void dump_sample(struct evsel *evsel, union perf_event *event,
 		callchain__printf(evsel, sample);
 
 	if (evsel__has_br_stack(evsel))
-		branch_stack__printf(sample, evsel__has_branch_callstack(evsel));
+		branch_stack__printf(sample, evsel);
 
 	if (sample_type & PERF_SAMPLE_REGS_USER)
 		regs_user__printf(sample, arch);
diff --git a/tools/perf/util/sort.c b/tools/perf/util/sort.c
index 80e4f6132740..27b123ccd2d1 100644
--- a/tools/perf/util/sort.c
+++ b/tools/perf/util/sort.c
@@ -583,21 +583,21 @@ static int hist_entry__sym_ipc_snprintf(struct hist_entry *he, char *bf,
 {
 
 	struct symbol *sym = he->ms.sym;
-	struct annotation *notes;
+	struct annotated_branch *branch;
 	double ipc = 0.0, coverage = 0.0;
 	char tmp[64];
 
 	if (!sym)
 		return repsep_snprintf(bf, size, "%-*s", width, "-");
 
-	notes = symbol__annotation(sym);
+	branch = symbol__annotation(sym)->branch;
 
-	if (notes->hit_cycles)
-		ipc = notes->hit_insn / ((double)notes->hit_cycles);
+	if (branch && branch->hit_cycles)
+		ipc = branch->hit_insn / ((double)branch->hit_cycles);
 
-	if (notes->total_insn) {
-		coverage = notes->cover_insn * 100.0 /
-			((double)notes->total_insn);
+	if (branch && branch->total_insn) {
+		coverage = branch->cover_insn * 100.0 /
+			((double)branch->total_insn);
 	}
 
 	snprintf(tmp, sizeof(tmp), "%-5.2f [%5.1f%%]", ipc, coverage);
diff --git a/tools/perf/util/symbol_conf.h b/tools/perf/util/symbol_conf.h
index 0b589570d1d0..6040286e07a6 100644
--- a/tools/perf/util/symbol_conf.h
+++ b/tools/perf/util/symbol_conf.h
@@ -42,7 +42,9 @@ struct symbol_conf {
 			inline_name,
 			disable_add2line_warn,
 			buildid_mmap2,
-			guest_code;
+			guest_code,
+			lazy_load_kernel_maps,
+			keep_exited_threads;
 	const char	*vmlinux_name,
 			*kallsyms_name,
 			*source_prefix,
diff --git a/tools/perf/util/thread.h b/tools/perf/util/thread.h
index e79225a0ea46..0df775b5c110 100644
--- a/tools/perf/util/thread.h
+++ b/tools/perf/util/thread.h
@@ -36,13 +36,22 @@ struct thread_rb_node {
 };
 
 DECLARE_RC_STRUCT(thread) {
+	/** @maps: mmaps associated with this thread. */
 	struct maps		*maps;
 	pid_t			pid_; /* Not all tools update this */
+	/** @tid: thread ID number unique to a machine. */
 	pid_t			tid;
+	/** @ppid: parent process of the process this thread belongs to. */
 	pid_t			ppid;
 	int			cpu;
 	int			guest_cpu; /* For QEMU thread */
 	refcount_t		refcnt;
+	/**
+	 * @exited: Has the thread had an exit event. Such threads are usually
+	 * removed from the machine's threads but some events/tools require
+	 * access to dead threads.
+	 */
+	bool			exited;
 	bool			comm_set;
 	int			comm_len;
 	struct list_head	namespaces_list;
@@ -189,6 +198,11 @@ static inline refcount_t *thread__refcnt(struct thread *thread)
 	return &RC_CHK_ACCESS(thread)->refcnt;
 }
 
+static inline void thread__set_exited(struct thread *thread, bool exited)
+{
+	RC_CHK_ACCESS(thread)->exited = exited;
+}
+
 static inline bool thread__comm_set(const struct thread *thread)
 {
 	return RC_CHK_ACCESS(thread)->comm_set;
diff --git a/tools/power/pm-graph/sleepgraph.py b/tools/power/pm-graph/sleepgraph.py
index 4a356a706785..40ad221e8881 100755
--- a/tools/power/pm-graph/sleepgraph.py
+++ b/tools/power/pm-graph/sleepgraph.py
@@ -4151,7 +4151,7 @@ def parseKernelLog(data):
 			elif(re.match('Enabling non-boot CPUs .*', msg)):
 				# start of first cpu resume
 				cpu_start = ktime
-			elif(re.match('smpboot: CPU (?P<cpu>[0-9]*) is now offline', msg)) \
+			elif(re.match('smpboot: CPU (?P<cpu>[0-9]*) is now offline', msg) \
 				or re.match('psci: CPU(?P<cpu>[0-9]*) killed.*', msg)):
 				# end of a cpu suspend, start of the next
 				m = re.match('smpboot: CPU (?P<cpu>[0-9]*) is now offline', msg)
diff --git a/tools/testing/radix-tree/linux.c b/tools/testing/radix-tree/linux.c
index 61fe2601cb3a..4eb442206d01 100644
--- a/tools/testing/radix-tree/linux.c
+++ b/tools/testing/radix-tree/linux.c
@@ -93,13 +93,9 @@ void *kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru,
 	return p;
 }
 
-void kmem_cache_free_locked(struct kmem_cache *cachep, void *objp)
+void __kmem_cache_free_locked(struct kmem_cache *cachep, void *objp)
 {
 	assert(objp);
-	uatomic_dec(&nr_allocated);
-	uatomic_dec(&cachep->nr_allocated);
-	if (kmalloc_verbose)
-		printf("Freeing %p to slab\n", objp);
 	if (cachep->nr_objs > 10 || cachep->align) {
 		memset(objp, POISON_FREE, cachep->size);
 		free(objp);
@@ -111,6 +107,15 @@ void kmem_cache_free_locked(struct kmem_cache *cachep, void *objp)
 	}
 }
 
+void kmem_cache_free_locked(struct kmem_cache *cachep, void *objp)
+{
+	uatomic_dec(&nr_allocated);
+	uatomic_dec(&cachep->nr_allocated);
+	if (kmalloc_verbose)
+		printf("Freeing %p to slab\n", objp);
+	__kmem_cache_free_locked(cachep, objp);
+}
+
 void kmem_cache_free(struct kmem_cache *cachep, void *objp)
 {
 	pthread_mutex_lock(&cachep->lock);
@@ -141,18 +146,17 @@ int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
 	if (kmalloc_verbose)
 		pr_debug("Bulk alloc %lu\n", size);
 
-	if (!(gfp & __GFP_DIRECT_RECLAIM)) {
-		if (cachep->non_kernel < size)
-			return 0;
-
-		cachep->non_kernel -= size;
-	}
-
 	pthread_mutex_lock(&cachep->lock);
 	if (cachep->nr_objs >= size) {
 		struct radix_tree_node *node;
 
 		for (i = 0; i < size; i++) {
+			if (!(gfp & __GFP_DIRECT_RECLAIM)) {
+				if (!cachep->non_kernel)
+					break;
+				cachep->non_kernel--;
+			}
+
 			node = cachep->objs;
 			cachep->nr_objs--;
 			cachep->objs = node->parent;
@@ -163,11 +167,19 @@ int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
 	} else {
 		pthread_mutex_unlock(&cachep->lock);
 		for (i = 0; i < size; i++) {
+			if (!(gfp & __GFP_DIRECT_RECLAIM)) {
+				if (!cachep->non_kernel)
+					break;
+				cachep->non_kernel--;
+			}
+
 			if (cachep->align) {
 				posix_memalign(&p[i], cachep->align,
 					       cachep->size);
 			} else {
 				p[i] = malloc(cachep->size);
+				if (!p[i])
+					break;
 			}
 			if (cachep->ctor)
 				cachep->ctor(p[i]);
@@ -176,6 +188,15 @@ int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
 		}
 	}
 
+	if (i < size) {
+		size = i;
+		pthread_mutex_lock(&cachep->lock);
+		for (i = 0; i < size; i++)
+			__kmem_cache_free_locked(cachep, p[i]);
+		pthread_mutex_unlock(&cachep->lock);
+		return 0;
+	}
+
 	for (i = 0; i < size; i++) {
 		uatomic_inc(&nr_allocated);
 		uatomic_inc(&cachep->nr_allocated);
diff --git a/tools/testing/radix-tree/linux/maple_tree.h b/tools/testing/radix-tree/linux/maple_tree.h
index 7d8d1f445b89..06c89bdcc515 100644
--- a/tools/testing/radix-tree/linux/maple_tree.h
+++ b/tools/testing/radix-tree/linux/maple_tree.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0+ */
 #define atomic_t int32_t
-#include "../../../../include/linux/maple_tree.h"
 #define atomic_inc(x) uatomic_inc(x)
 #define atomic_read(x) uatomic_read(x)
 #define atomic_set(x, y) do {} while (0)
 #define U8_MAX UCHAR_MAX
+#include "../../../../include/linux/maple_tree.h"
diff --git a/tools/testing/radix-tree/maple.c b/tools/testing/radix-tree/maple.c
index e5da1cad70ba..56ae47291ee0 100644
--- a/tools/testing/radix-tree/maple.c
+++ b/tools/testing/radix-tree/maple.c
@@ -118,6 +118,7 @@ static noinline void __init check_new_node(struct maple_tree *mt)
 	MT_BUG_ON(mt, mas.alloc == NULL);
 	MT_BUG_ON(mt, mas.alloc->slot[0] == NULL);
 	mas_push_node(&mas, mn);
+	mas_reset(&mas);
 	mas_nomem(&mas, GFP_KERNEL); /* free */
 	mtree_unlock(mt);
 
@@ -141,7 +142,7 @@ static noinline void __init check_new_node(struct maple_tree *mt)
 
 	mn->parent = ma_parent_ptr(mn);
 	ma_free_rcu(mn);
-	mas.node = MAS_START;
+	mas.status = ma_start;
 	mas_nomem(&mas, GFP_KERNEL);
 	/* Allocate 3 nodes, will fail. */
 	mas_node_count(&mas, 3);
@@ -158,6 +159,7 @@ static noinline void __init check_new_node(struct maple_tree *mt)
 	/* Ensure we counted 3. */
 	MT_BUG_ON(mt, mas_allocated(&mas) != 3);
 	/* Free. */
+	mas_reset(&mas);
 	mas_nomem(&mas, GFP_KERNEL);
 
 	/* Set allocation request to 1. */
@@ -272,6 +274,7 @@ static noinline void __init check_new_node(struct maple_tree *mt)
 			ma_free_rcu(mn);
 			MT_BUG_ON(mt, mas_allocated(&mas) != i - j - 1);
 		}
+		mas_reset(&mas);
 		MT_BUG_ON(mt, mas_nomem(&mas, GFP_KERNEL));
 
 	}
@@ -294,6 +297,7 @@ static noinline void __init check_new_node(struct maple_tree *mt)
 		smn = smn->slot[0]; /* next. */
 	}
 	MT_BUG_ON(mt, mas_allocated(&mas) != total);
+	mas_reset(&mas);
 	mas_nomem(&mas, GFP_KERNEL); /* Free. */
 
 	MT_BUG_ON(mt, mas_allocated(&mas) != 0);
@@ -441,7 +445,7 @@ static noinline void __init check_new_node(struct maple_tree *mt)
 	mas.node = MA_ERROR(-ENOMEM);
 	mas_node_count(&mas, 10); /* Request */
 	mas_nomem(&mas, GFP_KERNEL); /* Fill request */
-	mas.node = MAS_START;
+	mas.status = ma_start;
 	MT_BUG_ON(mt, mas_allocated(&mas) != 10);
 	mas_destroy(&mas);
 
@@ -452,7 +456,7 @@ static noinline void __init check_new_node(struct maple_tree *mt)
 	mas.node = MA_ERROR(-ENOMEM);
 	mas_node_count(&mas, 10 + MAPLE_ALLOC_SLOTS - 1); /* Request */
 	mas_nomem(&mas, GFP_KERNEL); /* Fill request */
-	mas.node = MAS_START;
+	mas.status = ma_start;
 	MT_BUG_ON(mt, mas_allocated(&mas) != 10 + MAPLE_ALLOC_SLOTS - 1);
 	mas_destroy(&mas);
 
@@ -941,10 +945,11 @@ static inline bool mas_tree_walk(struct ma_state *mas, unsigned long *range_min,
 
 	ret = mas_descend_walk(mas, range_min, range_max);
 	if (unlikely(mte_dead_node(mas->node))) {
-		mas->node = MAS_START;
+		mas->status = ma_start;
 		goto retry;
 	}
 
+	mas->end = mas_data_end(mas);
 	return ret;
 
 not_found:
@@ -960,17 +965,19 @@ static inline void *mas_range_load(struct ma_state *mas,
 	unsigned long index = mas->index;
 
 	if (mas_is_none(mas) || mas_is_paused(mas))
-		mas->node = MAS_START;
+		mas->status = ma_start;
 retry:
 	if (mas_tree_walk(mas, range_min, range_max))
-		if (unlikely(mas->node == MAS_ROOT))
+		if (unlikely(mas->status == ma_root))
 			return mas_root(mas);
 
 	if (likely(mas->offset != MAPLE_NODE_SLOTS))
 		entry = mas_get_slot(mas, mas->offset);
 
-	if (mas_dead_node(mas, index))
+	if (mas_is_active(mas) && mte_dead_node(mas->node)) {
+		mas_set(mas, index);
 		goto retry;
+	}
 
 	return entry;
 }
@@ -35336,7 +35343,7 @@ static void mas_dfs_preorder(struct ma_state *mas)
 	unsigned char end, slot = 0;
 	unsigned long *pivots;
 
-	if (mas->node == MAS_START) {
+	if (mas->status == ma_start) {
 		mas_start(mas);
 		return;
 	}
@@ -35373,7 +35380,7 @@ static void mas_dfs_preorder(struct ma_state *mas)
 
 	return;
 done:
-	mas->node = MAS_NONE;
+	mas->status = ma_none;
 }
 
 
@@ -35832,7 +35839,7 @@ static noinline void __init check_nomem(struct maple_tree *mt)
 	mas_store(&ms, &ms); /* insert 1 -> &ms, fails. */
 	MT_BUG_ON(mt, ms.node != MA_ERROR(-ENOMEM));
 	mas_nomem(&ms, GFP_KERNEL); /* Node allocated in here. */
-	MT_BUG_ON(mt, ms.node != MAS_START);
+	MT_BUG_ON(mt, ms.status != ma_start);
 	mtree_unlock(mt);
 	MT_BUG_ON(mt, mtree_insert(mt, 2, mt, GFP_KERNEL) != 0);
 	mtree_lock(mt);
@@ -35857,6 +35864,363 @@ static noinline void __init check_locky(struct maple_tree *mt)
 	mt_clear_in_rcu(mt);
 }
 
+/*
+ * Compares two nodes except for the addresses stored in the nodes.
+ * Returns zero if they are the same, otherwise returns non-zero.
+ */
+static int __init compare_node(struct maple_enode *enode_a,
+			       struct maple_enode *enode_b)
+{
+	struct maple_node *node_a, *node_b;
+	struct maple_node a, b;
+	void **slots_a, **slots_b; /* Do not use the rcu tag. */
+	enum maple_type type;
+	int i;
+
+	if (((unsigned long)enode_a & MAPLE_NODE_MASK) !=
+	    ((unsigned long)enode_b & MAPLE_NODE_MASK)) {
+		pr_err("The lower 8 bits of enode are different.\n");
+		return -1;
+	}
+
+	type = mte_node_type(enode_a);
+	node_a = mte_to_node(enode_a);
+	node_b = mte_to_node(enode_b);
+	a = *node_a;
+	b = *node_b;
+
+	/* Do not compare addresses. */
+	if (ma_is_root(node_a) || ma_is_root(node_b)) {
+		a.parent = (struct maple_pnode *)((unsigned long)a.parent &
+						  MA_ROOT_PARENT);
+		b.parent = (struct maple_pnode *)((unsigned long)b.parent &
+						  MA_ROOT_PARENT);
+	} else {
+		a.parent = (struct maple_pnode *)((unsigned long)a.parent &
+						  MAPLE_NODE_MASK);
+		b.parent = (struct maple_pnode *)((unsigned long)b.parent &
+						  MAPLE_NODE_MASK);
+	}
+
+	if (a.parent != b.parent) {
+		pr_err("The lower 8 bits of parents are different. %p %p\n",
+			a.parent, b.parent);
+		return -1;
+	}
+
+	/*
+	 * If it is a leaf node, the slots do not contain the node address, and
+	 * no special processing of slots is required.
+	 */
+	if (ma_is_leaf(type))
+		goto cmp;
+
+	slots_a = ma_slots(&a, type);
+	slots_b = ma_slots(&b, type);
+
+	for (i = 0; i < mt_slots[type]; i++) {
+		if (!slots_a[i] && !slots_b[i])
+			break;
+
+		if (!slots_a[i] || !slots_b[i]) {
+			pr_err("The number of slots is different.\n");
+			return -1;
+		}
+
+		/* Do not compare addresses in slots. */
+		((unsigned long *)slots_a)[i] &= MAPLE_NODE_MASK;
+		((unsigned long *)slots_b)[i] &= MAPLE_NODE_MASK;
+	}
+
+cmp:
+	/*
+	 * Compare all contents of two nodes, including parent (except address),
+	 * slots (except address), pivots, gaps and metadata.
+	 */
+	return memcmp(&a, &b, sizeof(struct maple_node));
+}
+
+/*
+ * Compare two trees and return 0 if they are the same, non-zero otherwise.
+ */
+static int __init compare_tree(struct maple_tree *mt_a, struct maple_tree *mt_b)
+{
+	MA_STATE(mas_a, mt_a, 0, 0);
+	MA_STATE(mas_b, mt_b, 0, 0);
+
+	if (mt_a->ma_flags != mt_b->ma_flags) {
+		pr_err("The flags of the two trees are different.\n");
+		return -1;
+	}
+
+	mas_dfs_preorder(&mas_a);
+	mas_dfs_preorder(&mas_b);
+
+	if (mas_is_ptr(&mas_a) || mas_is_ptr(&mas_b)) {
+		if (!(mas_is_ptr(&mas_a) && mas_is_ptr(&mas_b))) {
+			pr_err("One is ma_root and the other is not.\n");
+			return -1;
+		}
+		return 0;
+	}
+
+	while (!mas_is_none(&mas_a) || !mas_is_none(&mas_b)) {
+
+		if (mas_is_none(&mas_a) || mas_is_none(&mas_b)) {
+			pr_err("One is ma_none and the other is not.\n");
+			return -1;
+		}
+
+		if (mas_a.min != mas_b.min ||
+		    mas_a.max != mas_b.max) {
+			pr_err("mas->min, mas->max do not match.\n");
+			return -1;
+		}
+
+		if (compare_node(mas_a.node, mas_b.node)) {
+			pr_err("The contents of nodes %p and %p are different.\n",
+			       mas_a.node, mas_b.node);
+			mt_dump(mt_a, mt_dump_dec);
+			mt_dump(mt_b, mt_dump_dec);
+			return -1;
+		}
+
+		mas_dfs_preorder(&mas_a);
+		mas_dfs_preorder(&mas_b);
+	}
+
+	return 0;
+}
+
+static __init void mas_subtree_max_range(struct ma_state *mas)
+{
+	unsigned long limit = mas->max;
+	MA_STATE(newmas, mas->tree, 0, 0);
+	void *entry;
+
+	mas_for_each(mas, entry, limit) {
+		if (mas->last - mas->index >=
+		    newmas.last - newmas.index) {
+			newmas = *mas;
+		}
+	}
+
+	*mas = newmas;
+}
+
+/*
+ * build_full_tree() - Build a full tree.
+ * @mt: The tree to build.
+ * @flags: Use @flags to build the tree.
+ * @height: The height of the tree to build.
+ *
+ * Build a tree with full leaf nodes and internal nodes. Note that the height
+ * should not exceed 3, otherwise it will take a long time to build.
+ * Return: zero if the build is successful, non-zero if it fails.
+ */
+static __init int build_full_tree(struct maple_tree *mt, unsigned int flags,
+		int height)
+{
+	MA_STATE(mas, mt, 0, 0);
+	unsigned long step;
+	int ret = 0, cnt = 1;
+	enum maple_type type;
+
+	mt_init_flags(mt, flags);
+	mtree_insert_range(mt, 0, ULONG_MAX, xa_mk_value(5), GFP_KERNEL);
+
+	mtree_lock(mt);
+
+	while (1) {
+		mas_set(&mas, 0);
+		if (mt_height(mt) < height) {
+			mas.max = ULONG_MAX;
+			goto store;
+		}
+
+		while (1) {
+			mas_dfs_preorder(&mas);
+			if (mas_is_none(&mas))
+				goto unlock;
+
+			type = mte_node_type(mas.node);
+			if (mas_data_end(&mas) + 1 < mt_slots[type]) {
+				mas_set(&mas, mas.min);
+				goto store;
+			}
+		}
+store:
+		mas_subtree_max_range(&mas);
+		step = mas.last - mas.index;
+		if (step < 1) {
+			ret = -1;
+			goto unlock;
+		}
+
+		step /= 2;
+		mas.last = mas.index + step;
+		mas_store_gfp(&mas, xa_mk_value(5),
+				GFP_KERNEL);
+		++cnt;
+	}
+unlock:
+	mtree_unlock(mt);
+
+	MT_BUG_ON(mt, mt_height(mt) != height);
+	/* pr_info("height:%u number of elements:%d\n", mt_height(mt), cnt); */
+	return ret;
+}
+
+static noinline void __init check_mtree_dup(struct maple_tree *mt)
+{
+	DEFINE_MTREE(new);
+	int i, j, ret, count = 0;
+	unsigned int rand_seed = 17, rand;
+
+	/* store a value at [0, 0] */
+	mt_init_flags(mt, 0);
+	mtree_store_range(mt, 0, 0, xa_mk_value(0), GFP_KERNEL);
+	ret = mtree_dup(mt, &new, GFP_KERNEL);
+	MT_BUG_ON(&new, ret);
+	mt_validate(&new);
+	if (compare_tree(mt, &new))
+		MT_BUG_ON(&new, 1);
+
+	mtree_destroy(mt);
+	mtree_destroy(&new);
+
+	/* The two trees have different attributes. */
+	mt_init_flags(mt, 0);
+	mt_init_flags(&new, MT_FLAGS_ALLOC_RANGE);
+	ret = mtree_dup(mt, &new, GFP_KERNEL);
+	MT_BUG_ON(&new, ret != -EINVAL);
+	mtree_destroy(mt);
+	mtree_destroy(&new);
+
+	/* The new tree is not empty */
+	mt_init_flags(mt, 0);
+	mt_init_flags(&new, 0);
+	mtree_store(&new, 5, xa_mk_value(5), GFP_KERNEL);
+	ret = mtree_dup(mt, &new, GFP_KERNEL);
+	MT_BUG_ON(&new, ret != -EINVAL);
+	mtree_destroy(mt);
+	mtree_destroy(&new);
+
+	/* Test for duplicating full trees. */
+	for (i = 1; i <= 3; i++) {
+		ret = build_full_tree(mt, 0, i);
+		MT_BUG_ON(mt, ret);
+		mt_init_flags(&new, 0);
+
+		ret = mtree_dup(mt, &new, GFP_KERNEL);
+		MT_BUG_ON(&new, ret);
+		mt_validate(&new);
+		if (compare_tree(mt, &new))
+			MT_BUG_ON(&new, 1);
+
+		mtree_destroy(mt);
+		mtree_destroy(&new);
+	}
+
+	for (i = 1; i <= 3; i++) {
+		ret = build_full_tree(mt, MT_FLAGS_ALLOC_RANGE, i);
+		MT_BUG_ON(mt, ret);
+		mt_init_flags(&new, MT_FLAGS_ALLOC_RANGE);
+
+		ret = mtree_dup(mt, &new, GFP_KERNEL);
+		MT_BUG_ON(&new, ret);
+		mt_validate(&new);
+		if (compare_tree(mt, &new))
+			MT_BUG_ON(&new, 1);
+
+		mtree_destroy(mt);
+		mtree_destroy(&new);
+	}
+
+	/* Test for normal duplicating. */
+	for (i = 0; i < 1000; i += 3) {
+		if (i & 1) {
+			mt_init_flags(mt, 0);
+			mt_init_flags(&new, 0);
+		} else {
+			mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+			mt_init_flags(&new, MT_FLAGS_ALLOC_RANGE);
+		}
+
+		for (j = 0; j < i; j++) {
+			mtree_store_range(mt, j * 10, j * 10 + 5,
+					  xa_mk_value(j), GFP_KERNEL);
+		}
+
+		ret = mtree_dup(mt, &new, GFP_KERNEL);
+		MT_BUG_ON(&new, ret);
+		mt_validate(&new);
+		if (compare_tree(mt, &new))
+			MT_BUG_ON(&new, 1);
+
+		mtree_destroy(mt);
+		mtree_destroy(&new);
+	}
+
+	/* Test memory allocation failed. */
+	mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+	for (i = 0; i < 30; i += 3) {
+		mtree_store_range(mt, j * 10, j * 10 + 5,
+					  xa_mk_value(j), GFP_KERNEL);
+	}
+
+	/* Failed at the first node. */
+	mt_init_flags(&new, MT_FLAGS_ALLOC_RANGE);
+	mt_set_non_kernel(0);
+	ret = mtree_dup(mt, &new, GFP_NOWAIT);
+	mt_set_non_kernel(0);
+	MT_BUG_ON(&new, ret != -ENOMEM);
+	mtree_destroy(mt);
+	mtree_destroy(&new);
+
+	/* Random maple tree fails at a random node. */
+	for (i = 0; i < 1000; i += 3) {
+		if (i & 1) {
+			mt_init_flags(mt, 0);
+			mt_init_flags(&new, 0);
+		} else {
+			mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE);
+			mt_init_flags(&new, MT_FLAGS_ALLOC_RANGE);
+		}
+
+		for (j = 0; j < i; j++) {
+			mtree_store_range(mt, j * 10, j * 10 + 5,
+					  xa_mk_value(j), GFP_KERNEL);
+		}
+		/*
+		 * The rand() library function is not used, so we can generate
+		 * the same random numbers on any platform.
+		 */
+		rand_seed = rand_seed * 1103515245 + 12345;
+		rand = rand_seed / 65536 % 128;
+		mt_set_non_kernel(rand);
+
+		ret = mtree_dup(mt, &new, GFP_NOWAIT);
+		mt_set_non_kernel(0);
+		if (ret != 0) {
+			MT_BUG_ON(&new, ret != -ENOMEM);
+			count++;
+			mtree_destroy(mt);
+			continue;
+		}
+
+		mt_validate(&new);
+		if (compare_tree(mt, &new))
+			MT_BUG_ON(&new, 1);
+
+		mtree_destroy(mt);
+		mtree_destroy(&new);
+	}
+
+	/* pr_info("mtree_dup() fail %d times\n", count); */
+	BUG_ON(!count);
+}
+
 extern void test_kmem_cache_bulk(void);
 
 void farmer_tests(void)
@@ -35904,6 +36268,10 @@ void farmer_tests(void)
 	check_null_expand(&tree);
 	mtree_destroy(&tree);
 
+	mt_init_flags(&tree, 0);
+	check_mtree_dup(&tree);
+	mtree_destroy(&tree);
+
 	/* RCU testing */
 	mt_init_flags(&tree, 0);
 	check_erase_testset(&tree);
@@ -35938,7 +36306,9 @@ void farmer_tests(void)
 
 void maple_tree_tests(void)
 {
+#if !defined(BENCH)
 	farmer_tests();
+#endif
 	maple_tree_seed();
 	maple_tree_harvest();
 }
diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile
index 3b2061d1c1a5..1107be84ea95 100644
--- a/tools/testing/selftests/Makefile
+++ b/tools/testing/selftests/Makefile
@@ -43,6 +43,7 @@ TARGETS += landlock
 TARGETS += lib
 TARGETS += livepatch
 TARGETS += lkdtm
+TARGETS += lsm
 TARGETS += membarrier
 TARGETS += memfd
 TARGETS += memory-hotplug
diff --git a/tools/testing/selftests/bpf/cgroup_helpers.c b/tools/testing/selftests/bpf/cgroup_helpers.c
index 5b1da2a32ea7..5aa133bf3688 100644
--- a/tools/testing/selftests/bpf/cgroup_helpers.c
+++ b/tools/testing/selftests/bpf/cgroup_helpers.c
@@ -45,9 +45,12 @@
 #define format_parent_cgroup_path(buf, path) \
 	format_cgroup_path_pid(buf, path, getppid())
 
-#define format_classid_path(buf)				\
-	snprintf(buf, sizeof(buf), "%s%s", NETCLS_MOUNT_PATH,	\
-		 CGROUP_WORK_DIR)
+#define format_classid_path_pid(buf, pid)				\
+	snprintf(buf, sizeof(buf), "%s%s%d", NETCLS_MOUNT_PATH,	\
+		 CGROUP_WORK_DIR, pid)
+
+#define format_classid_path(buf)	\
+	format_classid_path_pid(buf, getpid())
 
 static __thread bool cgroup_workdir_mounted;
 
@@ -419,26 +422,23 @@ int create_and_get_cgroup(const char *relative_path)
 }
 
 /**
- * get_cgroup_id() - Get cgroup id for a particular cgroup path
- * @relative_path: The cgroup path, relative to the workdir, to join
+ * get_cgroup_id_from_path - Get cgroup id for a particular cgroup path
+ * @cgroup_workdir: The absolute cgroup path
  *
  * On success, it returns the cgroup id. On failure it returns 0,
  * which is an invalid cgroup id.
  * If there is a failure, it prints the error to stderr.
  */
-unsigned long long get_cgroup_id(const char *relative_path)
+unsigned long long get_cgroup_id_from_path(const char *cgroup_workdir)
 {
 	int dirfd, err, flags, mount_id, fhsize;
 	union {
 		unsigned long long cgid;
 		unsigned char raw_bytes[8];
 	} id;
-	char cgroup_workdir[PATH_MAX + 1];
 	struct file_handle *fhp, *fhp2;
 	unsigned long long ret = 0;
 
-	format_cgroup_path(cgroup_workdir, relative_path);
-
 	dirfd = AT_FDCWD;
 	flags = 0;
 	fhsize = sizeof(*fhp);
@@ -474,6 +474,14 @@ unsigned long long get_cgroup_id(const char *relative_path)
 	return ret;
 }
 
+unsigned long long get_cgroup_id(const char *relative_path)
+{
+	char cgroup_workdir[PATH_MAX + 1];
+
+	format_cgroup_path(cgroup_workdir, relative_path);
+	return get_cgroup_id_from_path(cgroup_workdir);
+}
+
 int cgroup_setup_and_join(const char *path) {
 	int cg_fd;
 
@@ -523,10 +531,20 @@ int setup_classid_environment(void)
 		return 1;
 	}
 
-	if (mount("net_cls", NETCLS_MOUNT_PATH, "cgroup", 0, "net_cls") &&
-	    errno != EBUSY) {
-		log_err("mount cgroup net_cls");
-		return 1;
+	if (mount("net_cls", NETCLS_MOUNT_PATH, "cgroup", 0, "net_cls")) {
+		if (errno != EBUSY) {
+			log_err("mount cgroup net_cls");
+			return 1;
+		}
+
+		if (rmdir(NETCLS_MOUNT_PATH)) {
+			log_err("rmdir cgroup net_cls");
+			return 1;
+		}
+		if (umount(CGROUP_MOUNT_DFLT)) {
+			log_err("umount cgroup base");
+			return 1;
+		}
 	}
 
 	cleanup_classid_environment();
@@ -541,15 +559,16 @@ int setup_classid_environment(void)
 
 /**
  * set_classid() - Set a cgroupv1 net_cls classid
- * @id: the numeric classid
  *
- * Writes the passed classid into the cgroup work dir's net_cls.classid
+ * Writes the classid into the cgroup work dir's net_cls.classid
  * file in order to later on trigger socket tagging.
  *
+ * We leverage the current pid as the classid, ensuring unique identification.
+ *
  * On success, it returns 0, otherwise on failure it returns 1. If there
  * is a failure, it prints the error to stderr.
  */
-int set_classid(unsigned int id)
+int set_classid(void)
 {
 	char cgroup_workdir[PATH_MAX - 42];
 	char cgroup_classid_path[PATH_MAX + 1];
@@ -565,7 +584,7 @@ int set_classid(unsigned int id)
 		return 1;
 	}
 
-	if (dprintf(fd, "%u\n", id) < 0) {
+	if (dprintf(fd, "%u\n", getpid()) < 0) {
 		log_err("Setting cgroup classid");
 		rc = 1;
 	}
@@ -607,3 +626,66 @@ void cleanup_classid_environment(void)
 	join_cgroup_from_top(NETCLS_MOUNT_PATH);
 	nftw(cgroup_workdir, nftwfunc, WALK_FD_LIMIT, FTW_DEPTH | FTW_MOUNT);
 }
+
+/**
+ * get_classid_cgroup_id - Get the cgroup id of a net_cls cgroup
+ */
+unsigned long long get_classid_cgroup_id(void)
+{
+	char cgroup_workdir[PATH_MAX + 1];
+
+	format_classid_path(cgroup_workdir);
+	return get_cgroup_id_from_path(cgroup_workdir);
+}
+
+/**
+ * get_cgroup1_hierarchy_id - Retrieves the ID of a cgroup1 hierarchy from the cgroup1 subsys name.
+ * @subsys_name: The cgroup1 subsys name, which can be retrieved from /proc/self/cgroup. It can be
+ * a named cgroup like "name=systemd", a controller name like "net_cls", or multi-contollers like
+ * "net_cls,net_prio".
+ */
+int get_cgroup1_hierarchy_id(const char *subsys_name)
+{
+	char *c, *c2, *c3, *c4;
+	bool found = false;
+	char line[1024];
+	FILE *file;
+	int i, id;
+
+	if (!subsys_name)
+		return -1;
+
+	file = fopen("/proc/self/cgroup", "r");
+	if (!file) {
+		log_err("fopen /proc/self/cgroup");
+		return -1;
+	}
+
+	while (fgets(line, 1024, file)) {
+		i = 0;
+		for (c = strtok_r(line, ":", &c2); c && i < 2; c = strtok_r(NULL, ":", &c2)) {
+			if (i == 0) {
+				id = strtol(c, NULL, 10);
+			} else if (i == 1) {
+				if (!strcmp(c, subsys_name)) {
+					found = true;
+					break;
+				}
+
+				/* Multiple subsystems may share one single mount point */
+				for (c3 = strtok_r(c, ",", &c4); c3;
+				     c3 = strtok_r(NULL, ",", &c4)) {
+					if (!strcmp(c, subsys_name)) {
+						found = true;
+						break;
+					}
+				}
+			}
+			i++;
+		}
+		if (found)
+			break;
+	}
+	fclose(file);
+	return found ? id : -1;
+}
diff --git a/tools/testing/selftests/bpf/cgroup_helpers.h b/tools/testing/selftests/bpf/cgroup_helpers.h
index 5c2cb9c8b546..ee053641c026 100644
--- a/tools/testing/selftests/bpf/cgroup_helpers.h
+++ b/tools/testing/selftests/bpf/cgroup_helpers.h
@@ -20,6 +20,7 @@ int get_root_cgroup(void);
 int create_and_get_cgroup(const char *relative_path);
 void remove_cgroup(const char *relative_path);
 unsigned long long get_cgroup_id(const char *relative_path);
+int get_cgroup1_hierarchy_id(const char *subsys_name);
 
 int join_cgroup(const char *relative_path);
 int join_root_cgroup(void);
@@ -29,8 +30,9 @@ int setup_cgroup_environment(void);
 void cleanup_cgroup_environment(void);
 
 /* cgroupv1 related */
-int set_classid(unsigned int id);
+int set_classid(void);
 int join_classid(void);
+unsigned long long get_classid_cgroup_id(void);
 
 int setup_classid_environment(void);
 void cleanup_classid_environment(void);
diff --git a/tools/testing/selftests/bpf/config.aarch64 b/tools/testing/selftests/bpf/config.aarch64
index 253821494884..29c8635c5722 100644
--- a/tools/testing/selftests/bpf/config.aarch64
+++ b/tools/testing/selftests/bpf/config.aarch64
@@ -1,4 +1,3 @@
-CONFIG_9P_FS=y
 CONFIG_ARCH_VEXPRESS=y
 CONFIG_ARCH_WANT_DEFAULT_BPF_JIT=y
 CONFIG_ARM_SMMU_V3=y
@@ -37,6 +36,7 @@ CONFIG_CRYPTO_USER_API_SKCIPHER=y
 CONFIG_DEBUG_ATOMIC_SLEEP=y
 CONFIG_DEBUG_INFO_BTF=y
 CONFIG_DEBUG_INFO_DWARF4=y
+CONFIG_DEBUG_INFO_REDUCED=n
 CONFIG_DEBUG_LIST=y
 CONFIG_DEBUG_LOCKDEP=y
 CONFIG_DEBUG_NOTIFIERS=y
@@ -46,7 +46,6 @@ CONFIG_DEBUG_SG=y
 CONFIG_DETECT_HUNG_TASK=y
 CONFIG_DEVTMPFS_MOUNT=y
 CONFIG_DEVTMPFS=y
-CONFIG_DRM_VIRTIO_GPU=y
 CONFIG_DRM=y
 CONFIG_DUMMY=y
 CONFIG_EXPERT=y
@@ -67,7 +66,6 @@ CONFIG_HAVE_KRETPROBES=y
 CONFIG_HEADERS_INSTALL=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_HUGETLBFS=y
-CONFIG_HW_RANDOM_VIRTIO=y
 CONFIG_HW_RANDOM=y
 CONFIG_HZ_100=y
 CONFIG_IDLE_PAGE_TRACKING=y
@@ -99,8 +97,6 @@ CONFIG_MEMCG=y
 CONFIG_MEMORY_HOTPLUG=y
 CONFIG_MEMORY_HOTREMOVE=y
 CONFIG_NAMESPACES=y
-CONFIG_NET_9P_VIRTIO=y
-CONFIG_NET_9P=y
 CONFIG_NET_ACT_BPF=y
 CONFIG_NET_ACT_GACT=y
 CONFIG_NETDEVICES=y
@@ -140,7 +136,6 @@ CONFIG_SCHED_TRACER=y
 CONFIG_SCSI_CONSTANTS=y
 CONFIG_SCSI_LOGGING=y
 CONFIG_SCSI_SCAN_ASYNC=y
-CONFIG_SCSI_VIRTIO=y
 CONFIG_SCSI=y
 CONFIG_SECURITY_NETWORK=y
 CONFIG_SERIAL_AMBA_PL011_CONSOLE=y
@@ -167,16 +162,6 @@ CONFIG_UPROBES=y
 CONFIG_USELIB=y
 CONFIG_USER_NS=y
 CONFIG_VETH=y
-CONFIG_VIRTIO_BALLOON=y
-CONFIG_VIRTIO_BLK=y
-CONFIG_VIRTIO_CONSOLE=y
-CONFIG_VIRTIO_FS=y
-CONFIG_VIRTIO_INPUT=y
-CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y
-CONFIG_VIRTIO_MMIO=y
-CONFIG_VIRTIO_NET=y
-CONFIG_VIRTIO_PCI=y
-CONFIG_VIRTIO_VSOCKETS_COMMON=y
 CONFIG_VLAN_8021Q=y
 CONFIG_VSOCKETS=y
 CONFIG_VSOCKETS_LOOPBACK=y
diff --git a/tools/testing/selftests/bpf/config.s390x b/tools/testing/selftests/bpf/config.s390x
index 2ba92167be35..e93330382849 100644
--- a/tools/testing/selftests/bpf/config.s390x
+++ b/tools/testing/selftests/bpf/config.s390x
@@ -1,4 +1,3 @@
-CONFIG_9P_FS=y
 CONFIG_ARCH_WANT_DEFAULT_BPF_JIT=y
 CONFIG_AUDIT=y
 CONFIG_BLK_CGROUP=y
@@ -84,8 +83,6 @@ CONFIG_MEMORY_HOTPLUG=y
 CONFIG_MEMORY_HOTREMOVE=y
 CONFIG_NAMESPACES=y
 CONFIG_NET=y
-CONFIG_NET_9P=y
-CONFIG_NET_9P_VIRTIO=y
 CONFIG_NET_ACT_BPF=y
 CONFIG_NET_ACT_GACT=y
 CONFIG_NET_KEY=y
@@ -114,7 +111,6 @@ CONFIG_SAMPLE_SECCOMP=y
 CONFIG_SAMPLES=y
 CONFIG_SCHED_TRACER=y
 CONFIG_SCSI=y
-CONFIG_SCSI_VIRTIO=y
 CONFIG_SECURITY_NETWORK=y
 CONFIG_STACK_TRACER=y
 CONFIG_STATIC_KEYS_SELFTEST=y
@@ -136,11 +132,6 @@ CONFIG_UPROBES=y
 CONFIG_USELIB=y
 CONFIG_USER_NS=y
 CONFIG_VETH=y
-CONFIG_VIRTIO_BALLOON=y
-CONFIG_VIRTIO_BLK=y
-CONFIG_VIRTIO_NET=y
-CONFIG_VIRTIO_PCI=y
-CONFIG_VIRTIO_VSOCKETS_COMMON=y
 CONFIG_VLAN_8021Q=y
 CONFIG_VSOCKETS=y
 CONFIG_VSOCKETS_LOOPBACK=y
diff --git a/tools/testing/selftests/bpf/config.vm b/tools/testing/selftests/bpf/config.vm
new file mode 100644
index 000000000000..a9746ca78777
--- /dev/null
+++ b/tools/testing/selftests/bpf/config.vm
@@ -0,0 +1,12 @@
+CONFIG_9P_FS=y
+CONFIG_9P_FS_POSIX_ACL=y
+CONFIG_9P_FS_SECURITY=y
+CONFIG_CRYPTO_DEV_VIRTIO=y
+CONFIG_NET_9P=y
+CONFIG_NET_9P_VIRTIO=y
+CONFIG_VIRTIO_BALLOON=y
+CONFIG_VIRTIO_BLK=y
+CONFIG_VIRTIO_CONSOLE=y
+CONFIG_VIRTIO_NET=y
+CONFIG_VIRTIO_PCI=y
+CONFIG_VIRTIO_VSOCKETS_COMMON=y
diff --git a/tools/testing/selftests/bpf/config.x86_64 b/tools/testing/selftests/bpf/config.x86_64
index 2e70a6048278..f7bfb2b09c82 100644
--- a/tools/testing/selftests/bpf/config.x86_64
+++ b/tools/testing/selftests/bpf/config.x86_64
@@ -1,6 +1,3 @@
-CONFIG_9P_FS=y
-CONFIG_9P_FS_POSIX_ACL=y
-CONFIG_9P_FS_SECURITY=y
 CONFIG_AGP=y
 CONFIG_AGP_AMD64=y
 CONFIG_AGP_INTEL=y
@@ -45,7 +42,6 @@ CONFIG_CPU_IDLE_GOV_LADDER=y
 CONFIG_CPUSETS=y
 CONFIG_CRC_T10DIF=y
 CONFIG_CRYPTO_BLAKE2B=y
-CONFIG_CRYPTO_DEV_VIRTIO=y
 CONFIG_CRYPTO_SEQIV=y
 CONFIG_CRYPTO_XXHASH=y
 CONFIG_DCB=y
@@ -145,8 +141,6 @@ CONFIG_MEMORY_FAILURE=y
 CONFIG_MINIX_SUBPARTITION=y
 CONFIG_NAMESPACES=y
 CONFIG_NET=y
-CONFIG_NET_9P=y
-CONFIG_NET_9P_VIRTIO=y
 CONFIG_NET_ACT_BPF=y
 CONFIG_NET_CLS_CGROUP=y
 CONFIG_NET_EMATCH=y
@@ -228,12 +222,6 @@ CONFIG_USER_NS=y
 CONFIG_VALIDATE_FS_PARSER=y
 CONFIG_VETH=y
 CONFIG_VIRT_DRIVERS=y
-CONFIG_VIRTIO_BALLOON=y
-CONFIG_VIRTIO_BLK=y
-CONFIG_VIRTIO_CONSOLE=y
-CONFIG_VIRTIO_NET=y
-CONFIG_VIRTIO_PCI=y
-CONFIG_VIRTIO_VSOCKETS_COMMON=y
 CONFIG_VLAN_8021Q=y
 CONFIG_VSOCKETS=y
 CONFIG_VSOCKETS_LOOPBACK=y
diff --git a/tools/testing/selftests/bpf/map_tests/map_percpu_stats.c b/tools/testing/selftests/bpf/map_tests/map_percpu_stats.c
index 8bf497a9843e..2ea36408816b 100644
--- a/tools/testing/selftests/bpf/map_tests/map_percpu_stats.c
+++ b/tools/testing/selftests/bpf/map_tests/map_percpu_stats.c
@@ -131,10 +131,17 @@ static bool is_lru(__u32 map_type)
 	       map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH;
 }
 
+static bool is_percpu(__u32 map_type)
+{
+	return map_type == BPF_MAP_TYPE_PERCPU_HASH ||
+	       map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH;
+}
+
 struct upsert_opts {
 	__u32 map_type;
 	int map_fd;
 	__u32 n;
+	bool retry_for_nomem;
 };
 
 static int create_small_hash(void)
@@ -148,19 +155,38 @@ static int create_small_hash(void)
 	return map_fd;
 }
 
+static bool retry_for_nomem_fn(int err)
+{
+	return err == ENOMEM;
+}
+
 static void *patch_map_thread(void *arg)
 {
+	/* 8KB is enough for 1024 CPUs. And it is shared between N_THREADS. */
+	static __u8 blob[8 << 10];
 	struct upsert_opts *opts = arg;
+	void *val_ptr;
 	int val;
 	int ret;
 	int i;
 
 	for (i = 0; i < opts->n; i++) {
-		if (opts->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
+		if (opts->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
 			val = create_small_hash();
-		else
+			val_ptr = &val;
+		} else if (is_percpu(opts->map_type)) {
+			val_ptr = blob;
+		} else {
 			val = rand();
-		ret = bpf_map_update_elem(opts->map_fd, &i, &val, 0);
+			val_ptr = &val;
+		}
+
+		/* 2 seconds may be enough ? */
+		if (opts->retry_for_nomem)
+			ret = map_update_retriable(opts->map_fd, &i, val_ptr, 0,
+						   40, retry_for_nomem_fn);
+		else
+			ret = bpf_map_update_elem(opts->map_fd, &i, val_ptr, 0);
 		CHECK(ret < 0, "bpf_map_update_elem", "key=%d error: %s\n", i, strerror(errno));
 
 		if (opts->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
@@ -281,6 +307,13 @@ static void __test(int map_fd)
 	else
 		opts.n /= 2;
 
+	/* per-cpu bpf memory allocator may not be able to allocate per-cpu
+	 * pointer successfully and it can not refill free llist timely, and
+	 * bpf_map_update_elem() will return -ENOMEM. so just retry to mitigate
+	 * the problem temporarily.
+	 */
+	opts.retry_for_nomem = is_percpu(opts.map_type) && (info.map_flags & BPF_F_NO_PREALLOC);
+
 	/*
 	 * Upsert keys [0, n) under some competition: with random values from
 	 * N_THREADS threads. Check values, then delete all elements and check
diff --git a/tools/testing/selftests/bpf/prog_tests/align.c b/tools/testing/selftests/bpf/prog_tests/align.c
index 465c1c3a3d3c..4ebd0da898f5 100644
--- a/tools/testing/selftests/bpf/prog_tests/align.c
+++ b/tools/testing/selftests/bpf/prog_tests/align.c
@@ -40,7 +40,7 @@ static struct bpf_align_test tests[] = {
 		},
 		.prog_type = BPF_PROG_TYPE_SCHED_CLS,
 		.matches = {
-			{0, "R1", "ctx(off=0,imm=0)"},
+			{0, "R1", "ctx()"},
 			{0, "R10", "fp0"},
 			{0, "R3_w", "2"},
 			{1, "R3_w", "4"},
@@ -68,7 +68,7 @@ static struct bpf_align_test tests[] = {
 		},
 		.prog_type = BPF_PROG_TYPE_SCHED_CLS,
 		.matches = {
-			{0, "R1", "ctx(off=0,imm=0)"},
+			{0, "R1", "ctx()"},
 			{0, "R10", "fp0"},
 			{0, "R3_w", "1"},
 			{1, "R3_w", "2"},
@@ -97,7 +97,7 @@ static struct bpf_align_test tests[] = {
 		},
 		.prog_type = BPF_PROG_TYPE_SCHED_CLS,
 		.matches = {
-			{0, "R1", "ctx(off=0,imm=0)"},
+			{0, "R1", "ctx()"},
 			{0, "R10", "fp0"},
 			{0, "R3_w", "4"},
 			{1, "R3_w", "8"},
@@ -119,7 +119,7 @@ static struct bpf_align_test tests[] = {
 		},
 		.prog_type = BPF_PROG_TYPE_SCHED_CLS,
 		.matches = {
-			{0, "R1", "ctx(off=0,imm=0)"},
+			{0, "R1", "ctx()"},
 			{0, "R10", "fp0"},
 			{0, "R3_w", "7"},
 			{1, "R3_w", "7"},
@@ -162,13 +162,13 @@ static struct bpf_align_test tests[] = {
 		},
 		.prog_type = BPF_PROG_TYPE_SCHED_CLS,
 		.matches = {
-			{6, "R0_w", "pkt(off=8,r=8,imm=0)"},
+			{6, "R0_w", "pkt(off=8,r=8)"},
 			{6, "R3_w", "var_off=(0x0; 0xff)"},
 			{7, "R3_w", "var_off=(0x0; 0x1fe)"},
 			{8, "R3_w", "var_off=(0x0; 0x3fc)"},
 			{9, "R3_w", "var_off=(0x0; 0x7f8)"},
 			{10, "R3_w", "var_off=(0x0; 0xff0)"},
-			{12, "R3_w", "pkt_end(off=0,imm=0)"},
+			{12, "R3_w", "pkt_end()"},
 			{17, "R4_w", "var_off=(0x0; 0xff)"},
 			{18, "R4_w", "var_off=(0x0; 0x1fe0)"},
 			{19, "R4_w", "var_off=(0x0; 0xff0)"},
@@ -235,11 +235,11 @@ static struct bpf_align_test tests[] = {
 		},
 		.prog_type = BPF_PROG_TYPE_SCHED_CLS,
 		.matches = {
-			{2, "R5_w", "pkt(off=0,r=0,imm=0)"},
-			{4, "R5_w", "pkt(off=14,r=0,imm=0)"},
-			{5, "R4_w", "pkt(off=14,r=0,imm=0)"},
-			{9, "R2", "pkt(off=0,r=18,imm=0)"},
-			{10, "R5", "pkt(off=14,r=18,imm=0)"},
+			{2, "R5_w", "pkt(r=0)"},
+			{4, "R5_w", "pkt(off=14,r=0)"},
+			{5, "R4_w", "pkt(off=14,r=0)"},
+			{9, "R2", "pkt(r=18)"},
+			{10, "R5", "pkt(off=14,r=18)"},
 			{10, "R4_w", "var_off=(0x0; 0xff)"},
 			{13, "R4_w", "var_off=(0x0; 0xffff)"},
 			{14, "R4_w", "var_off=(0x0; 0xffff)"},
@@ -299,7 +299,7 @@ static struct bpf_align_test tests[] = {
 			/* Calculated offset in R6 has unknown value, but known
 			 * alignment of 4.
 			 */
-			{6, "R2_w", "pkt(off=0,r=8,imm=0)"},
+			{6, "R2_w", "pkt(r=8)"},
 			{7, "R6_w", "var_off=(0x0; 0x3fc)"},
 			/* Offset is added to packet pointer R5, resulting in
 			 * known fixed offset, and variable offset from R6.
@@ -337,7 +337,7 @@ static struct bpf_align_test tests[] = {
 			/* Constant offset is added to R5 packet pointer,
 			 * resulting in reg->off value of 14.
 			 */
-			{26, "R5_w", "pkt(off=14,r=8,"},
+			{26, "R5_w", "pkt(off=14,r=8)"},
 			/* Variable offset is added to R5, resulting in a
 			 * variable offset of (4n). See comment for insn #18
 			 * for R4 = R5 trick.
@@ -397,7 +397,7 @@ static struct bpf_align_test tests[] = {
 			/* Calculated offset in R6 has unknown value, but known
 			 * alignment of 4.
 			 */
-			{6, "R2_w", "pkt(off=0,r=8,imm=0)"},
+			{6, "R2_w", "pkt(r=8)"},
 			{7, "R6_w", "var_off=(0x0; 0x3fc)"},
 			/* Adding 14 makes R6 be (4n+2) */
 			{8, "R6_w", "var_off=(0x2; 0x7fc)"},
@@ -459,7 +459,7 @@ static struct bpf_align_test tests[] = {
 		.prog_type = BPF_PROG_TYPE_SCHED_CLS,
 		.result = REJECT,
 		.matches = {
-			{3, "R5_w", "pkt_end(off=0,imm=0)"},
+			{3, "R5_w", "pkt_end()"},
 			/* (ptr - ptr) << 2 == unknown, (4n) */
 			{5, "R5_w", "var_off=(0x0; 0xfffffffffffffffc)"},
 			/* (4n) + 14 == (4n+2).  We blow our bounds, because
@@ -513,7 +513,7 @@ static struct bpf_align_test tests[] = {
 			/* Calculated offset in R6 has unknown value, but known
 			 * alignment of 4.
 			 */
-			{6, "R2_w", "pkt(off=0,r=8,imm=0)"},
+			{6, "R2_w", "pkt(r=8)"},
 			{8, "R6_w", "var_off=(0x0; 0x3fc)"},
 			/* Adding 14 makes R6 be (4n+2) */
 			{9, "R6_w", "var_off=(0x2; 0x7fc)"},
@@ -566,7 +566,7 @@ static struct bpf_align_test tests[] = {
 			/* Calculated offset in R6 has unknown value, but known
 			 * alignment of 4.
 			 */
-			{6, "R2_w", "pkt(off=0,r=8,imm=0)"},
+			{6, "R2_w", "pkt(r=8)"},
 			{9, "R6_w", "var_off=(0x0; 0x3c)"},
 			/* Adding 14 makes R6 be (4n+2) */
 			{10, "R6_w", "var_off=(0x2; 0x7c)"},
@@ -659,14 +659,14 @@ static int do_test_single(struct bpf_align_test *test)
 			/* Check the next line as well in case the previous line
 			 * did not have a corresponding bpf insn. Example:
 			 * func#0 @0
-			 * 0: R1=ctx(off=0,imm=0) R10=fp0
+			 * 0: R1=ctx() R10=fp0
 			 * 0: (b7) r3 = 2                 ; R3_w=2
 			 *
 			 * Sometimes it's actually two lines below, e.g. when
 			 * searching for "6: R3_w=scalar(umax=255,var_off=(0x0; 0xff))":
-			 *   from 4 to 6: R0_w=pkt(off=8,r=8,imm=0) R1=ctx(off=0,imm=0) R2_w=pkt(off=0,r=8,imm=0) R3_w=pkt_end(off=0,imm=0) R10=fp0
-			 *   6: R0_w=pkt(off=8,r=8,imm=0) R1=ctx(off=0,imm=0) R2_w=pkt(off=0,r=8,imm=0) R3_w=pkt_end(off=0,imm=0) R10=fp0
-			 *   6: (71) r3 = *(u8 *)(r2 +0)           ; R2_w=pkt(off=0,r=8,imm=0) R3_w=scalar(umax=255,var_off=(0x0; 0xff))
+			 *   from 4 to 6: R0_w=pkt(off=8,r=8) R1=ctx() R2_w=pkt(r=8) R3_w=pkt_end() R10=fp0
+			 *   6: R0_w=pkt(off=8,r=8) R1=ctx() R2_w=pkt(r=8) R3_w=pkt_end() R10=fp0
+			 *   6: (71) r3 = *(u8 *)(r2 +0)           ; R2_w=pkt(r=8) R3_w=scalar(umax=255,var_off=(0x0; 0xff))
 			 */
 			while (!(p = strstr(line_ptr, m.reg)) || !strstr(p, m.match)) {
 				cur_line = -1;
diff --git a/tools/testing/selftests/bpf/prog_tests/bind_perm.c b/tools/testing/selftests/bpf/prog_tests/bind_perm.c
index a1766a298bb7..f7cd129cb82b 100644
--- a/tools/testing/selftests/bpf/prog_tests/bind_perm.c
+++ b/tools/testing/selftests/bpf/prog_tests/bind_perm.c
@@ -9,8 +9,6 @@
 #include "cap_helpers.h"
 #include "bind_perm.skel.h"
 
-static int duration;
-
 static int create_netns(void)
 {
 	if (!ASSERT_OK(unshare(CLONE_NEWNET), "create netns"))
@@ -27,7 +25,7 @@ void try_bind(int family, int port, int expected_errno)
 	int fd = -1;
 
 	fd = socket(family, SOCK_STREAM, 0);
-	if (CHECK(fd < 0, "fd", "errno %d", errno))
+	if (!ASSERT_GE(fd, 0, "socket"))
 		goto close_socket;
 
 	if (family == AF_INET) {
@@ -60,7 +58,7 @@ void test_bind_perm(void)
 		return;
 
 	cgroup_fd = test__join_cgroup("/bind_perm");
-	if (CHECK(cgroup_fd < 0, "cg-join", "errno %d", errno))
+	if (!ASSERT_GE(cgroup_fd, 0, "test__join_cgroup"))
 		return;
 
 	skel = bind_perm__open_and_load();
diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_iter.c b/tools/testing/selftests/bpf/prog_tests/bpf_iter.c
index e3498f607b49..618af9dfae9b 100644
--- a/tools/testing/selftests/bpf/prog_tests/bpf_iter.c
+++ b/tools/testing/selftests/bpf/prog_tests/bpf_iter.c
@@ -34,8 +34,6 @@
 #include "bpf_iter_ksym.skel.h"
 #include "bpf_iter_sockmap.skel.h"
 
-static int duration;
-
 static void test_btf_id_or_null(void)
 {
 	struct bpf_iter_test_kern3 *skel;
@@ -64,7 +62,7 @@ static void do_dummy_read_opts(struct bpf_program *prog, struct bpf_iter_attach_
 	/* not check contents, but ensure read() ends without error */
 	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
 		;
-	CHECK(len < 0, "read", "read failed: %s\n", strerror(errno));
+	ASSERT_GE(len, 0, "read");
 
 	close(iter_fd);
 
@@ -334,6 +332,8 @@ static void test_task_stack(void)
 	do_dummy_read(skel->progs.dump_task_stack);
 	do_dummy_read(skel->progs.get_task_user_stacks);
 
+	ASSERT_EQ(skel->bss->num_user_stacks, 1, "num_user_stacks");
+
 	bpf_iter_task_stack__destroy(skel);
 }
 
@@ -413,7 +413,7 @@ static int do_btf_read(struct bpf_iter_task_btf *skel)
 		goto free_link;
 	}
 
-	if (CHECK(err < 0, "read", "read failed: %s\n", strerror(errno)))
+	if (!ASSERT_GE(err, 0, "read"))
 		goto free_link;
 
 	ASSERT_HAS_SUBSTR(taskbuf, "(struct task_struct)",
@@ -526,11 +526,11 @@ static int do_read_with_fd(int iter_fd, const char *expected,
 	start = 0;
 	while ((len = read(iter_fd, buf + start, read_buf_len)) > 0) {
 		start += len;
-		if (CHECK(start >= 16, "read", "read len %d\n", len))
+		if (!ASSERT_LT(start, 16, "read"))
 			return -1;
 		read_buf_len = read_one_char ? 1 : 16 - start;
 	}
-	if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno)))
+	if (!ASSERT_GE(len, 0, "read"))
 		return -1;
 
 	if (!ASSERT_STREQ(buf, expected, "read"))
@@ -571,8 +571,7 @@ static int do_read(const char *path, const char *expected)
 	int err, iter_fd;
 
 	iter_fd = open(path, O_RDONLY);
-	if (CHECK(iter_fd < 0, "open", "open %s failed: %s\n",
-		  path, strerror(errno)))
+	if (!ASSERT_GE(iter_fd, 0, "open"))
 		return -1;
 
 	err = do_read_with_fd(iter_fd, expected, false);
@@ -600,7 +599,7 @@ static void test_file_iter(void)
 	unlink(path);
 
 	err = bpf_link__pin(link, path);
-	if (CHECK(err, "pin_iter", "pin_iter to %s failed: %d\n", path, err))
+	if (!ASSERT_OK(err, "pin_iter"))
 		goto free_link;
 
 	err = do_read(path, "abcd");
@@ -651,12 +650,10 @@ static void test_overflow(bool test_e2big_overflow, bool ret1)
 	 * overflow and needs restart.
 	 */
 	map1_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL);
-	if (CHECK(map1_fd < 0, "bpf_map_create",
-		  "map_creation failed: %s\n", strerror(errno)))
+	if (!ASSERT_GE(map1_fd, 0, "bpf_map_create"))
 		goto out;
 	map2_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL);
-	if (CHECK(map2_fd < 0, "bpf_map_create",
-		  "map_creation failed: %s\n", strerror(errno)))
+	if (!ASSERT_GE(map2_fd, 0, "bpf_map_create"))
 		goto free_map1;
 
 	/* bpf_seq_printf kernel buffer is 8 pages, so one map
@@ -685,14 +682,12 @@ static void test_overflow(bool test_e2big_overflow, bool ret1)
 	/* setup filtering map_id in bpf program */
 	map_info_len = sizeof(map_info);
 	err = bpf_map_get_info_by_fd(map1_fd, &map_info, &map_info_len);
-	if (CHECK(err, "get_map_info", "get map info failed: %s\n",
-		  strerror(errno)))
+	if (!ASSERT_OK(err, "get_map_info"))
 		goto free_map2;
 	skel->bss->map1_id = map_info.id;
 
 	err = bpf_map_get_info_by_fd(map2_fd, &map_info, &map_info_len);
-	if (CHECK(err, "get_map_info", "get map info failed: %s\n",
-		  strerror(errno)))
+	if (!ASSERT_OK(err, "get_map_info"))
 		goto free_map2;
 	skel->bss->map2_id = map_info.id;
 
@@ -705,7 +700,7 @@ static void test_overflow(bool test_e2big_overflow, bool ret1)
 		goto free_link;
 
 	buf = malloc(expected_read_len);
-	if (!buf)
+	if (!ASSERT_OK_PTR(buf, "malloc"))
 		goto close_iter;
 
 	/* do read */
@@ -714,16 +709,14 @@ static void test_overflow(bool test_e2big_overflow, bool ret1)
 		while ((len = read(iter_fd, buf, expected_read_len)) > 0)
 			total_read_len += len;
 
-		CHECK(len != -1 || errno != E2BIG, "read",
-		      "expected ret -1, errno E2BIG, but get ret %d, error %s\n",
-			  len, strerror(errno));
+		ASSERT_EQ(len, -1, "read");
+		ASSERT_EQ(errno, E2BIG, "read");
 		goto free_buf;
 	} else if (!ret1) {
 		while ((len = read(iter_fd, buf, expected_read_len)) > 0)
 			total_read_len += len;
 
-		if (CHECK(len < 0, "read", "read failed: %s\n",
-			  strerror(errno)))
+		if (!ASSERT_GE(len, 0, "read"))
 			goto free_buf;
 	} else {
 		do {
@@ -732,8 +725,7 @@ static void test_overflow(bool test_e2big_overflow, bool ret1)
 				total_read_len += len;
 		} while (len > 0 || len == -EAGAIN);
 
-		if (CHECK(len < 0, "read", "read failed: %s\n",
-			  strerror(errno)))
+		if (!ASSERT_GE(len, 0, "read"))
 			goto free_buf;
 	}
 
@@ -836,7 +828,7 @@ static void test_bpf_hash_map(void)
 	/* do some tests */
 	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
 		;
-	if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno)))
+	if (!ASSERT_GE(len, 0, "read"))
 		goto close_iter;
 
 	/* test results */
@@ -878,6 +870,8 @@ static void test_bpf_percpu_hash_map(void)
 
 	skel->rodata->num_cpus = bpf_num_possible_cpus();
 	val = malloc(8 * bpf_num_possible_cpus());
+	if (!ASSERT_OK_PTR(val, "malloc"))
+		goto out;
 
 	err = bpf_iter_bpf_percpu_hash_map__load(skel);
 	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__load"))
@@ -917,7 +911,7 @@ static void test_bpf_percpu_hash_map(void)
 	/* do some tests */
 	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
 		;
-	if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno)))
+	if (!ASSERT_GE(len, 0, "read"))
 		goto close_iter;
 
 	/* test results */
@@ -983,17 +977,14 @@ static void test_bpf_array_map(void)
 	start = 0;
 	while ((len = read(iter_fd, buf + start, sizeof(buf) - start)) > 0)
 		start += len;
-	if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno)))
+	if (!ASSERT_GE(len, 0, "read"))
 		goto close_iter;
 
 	/* test results */
 	res_first_key = *(__u32 *)buf;
 	res_first_val = *(__u64 *)(buf + sizeof(__u32));
-	if (CHECK(res_first_key != 0 || res_first_val != first_val,
-		  "bpf_seq_write",
-		  "seq_write failure: first key %u vs expected 0, "
-		  " first value %llu vs expected %llu\n",
-		  res_first_key, res_first_val, first_val))
+	if (!ASSERT_EQ(res_first_key, 0, "bpf_seq_write") ||
+			!ASSERT_EQ(res_first_val, first_val, "bpf_seq_write"))
 		goto close_iter;
 
 	if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum"))
@@ -1057,6 +1048,8 @@ static void test_bpf_percpu_array_map(void)
 
 	skel->rodata->num_cpus = bpf_num_possible_cpus();
 	val = malloc(8 * bpf_num_possible_cpus());
+	if (!ASSERT_OK_PTR(val, "malloc"))
+		goto out;
 
 	err = bpf_iter_bpf_percpu_array_map__load(skel);
 	if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__load"))
@@ -1092,7 +1085,7 @@ static void test_bpf_percpu_array_map(void)
 	/* do some tests */
 	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
 		;
-	if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno)))
+	if (!ASSERT_GE(len, 0, "read"))
 		goto close_iter;
 
 	/* test results */
@@ -1131,6 +1124,7 @@ static void test_bpf_sk_storage_delete(void)
 	sock_fd = socket(AF_INET6, SOCK_STREAM, 0);
 	if (!ASSERT_GE(sock_fd, 0, "socket"))
 		goto out;
+
 	err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST);
 	if (!ASSERT_OK(err, "map_update"))
 		goto out;
@@ -1151,14 +1145,19 @@ static void test_bpf_sk_storage_delete(void)
 	/* do some tests */
 	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
 		;
-	if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno)))
+	if (!ASSERT_GE(len, 0, "read"))
 		goto close_iter;
 
 	/* test results */
 	err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
-	if (CHECK(!err || errno != ENOENT, "bpf_map_lookup_elem",
-		  "map value wasn't deleted (err=%d, errno=%d)\n", err, errno))
-		goto close_iter;
+
+	 /* Note: The following assertions serve to ensure
+	  * the value was deleted. It does so by asserting
+	  * that bpf_map_lookup_elem has failed. This might
+	  * seem counterintuitive at first.
+	  */
+	ASSERT_ERR(err, "bpf_map_lookup_elem");
+	ASSERT_EQ(errno, ENOENT, "bpf_map_lookup_elem");
 
 close_iter:
 	close(iter_fd);
@@ -1203,17 +1202,15 @@ static void test_bpf_sk_storage_get(void)
 	do_dummy_read(skel->progs.fill_socket_owner);
 
 	err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
-	if (CHECK(err || val != getpid(), "bpf_map_lookup_elem",
-	    "map value wasn't set correctly (expected %d, got %d, err=%d)\n",
-	    getpid(), val, err))
+	if (!ASSERT_OK(err, "bpf_map_lookup_elem") ||
+			!ASSERT_EQ(val, getpid(), "bpf_map_lookup_elem"))
 		goto close_socket;
 
 	do_dummy_read(skel->progs.negate_socket_local_storage);
 
 	err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);
-	CHECK(err || val != -getpid(), "bpf_map_lookup_elem",
-	      "map value wasn't set correctly (expected %d, got %d, err=%d)\n",
-	      -getpid(), val, err);
+	ASSERT_OK(err, "bpf_map_lookup_elem");
+	ASSERT_EQ(val, -getpid(), "bpf_map_lookup_elem");
 
 close_socket:
 	close(sock_fd);
@@ -1290,7 +1287,7 @@ static void test_bpf_sk_storage_map(void)
 	/* do some tests */
 	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
 		;
-	if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno)))
+	if (!ASSERT_GE(len, 0, "read"))
 		goto close_iter;
 
 	/* test results */
diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c b/tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c
index 675b90b15280..f09d6ac2ef09 100644
--- a/tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c
+++ b/tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c
@@ -25,7 +25,7 @@ void serial_test_bpf_obj_id(void)
 	 */
 	__u32 map_ids[nr_iters + 1];
 	char jited_insns[128], xlated_insns[128], zeros[128], tp_name[128];
-	__u32 i, next_id, info_len, nr_id_found, duration = 0;
+	__u32 i, next_id, info_len, nr_id_found;
 	struct timespec real_time_ts, boot_time_ts;
 	int err = 0;
 	__u64 array_value;
@@ -33,16 +33,16 @@ void serial_test_bpf_obj_id(void)
 	time_t now, load_time;
 
 	err = bpf_prog_get_fd_by_id(0);
-	CHECK(err >= 0 || errno != ENOENT,
-	      "get-fd-by-notexist-prog-id", "err %d errno %d\n", err, errno);
+	ASSERT_LT(err, 0, "bpf_prog_get_fd_by_id");
+	ASSERT_EQ(errno, ENOENT, "bpf_prog_get_fd_by_id");
 
 	err = bpf_map_get_fd_by_id(0);
-	CHECK(err >= 0 || errno != ENOENT,
-	      "get-fd-by-notexist-map-id", "err %d errno %d\n", err, errno);
+	ASSERT_LT(err, 0, "bpf_map_get_fd_by_id");
+	ASSERT_EQ(errno, ENOENT, "bpf_map_get_fd_by_id");
 
 	err = bpf_link_get_fd_by_id(0);
-	CHECK(err >= 0 || errno != ENOENT,
-	      "get-fd-by-notexist-link-id", "err %d errno %d\n", err, errno);
+	ASSERT_LT(err, 0, "bpf_map_get_fd_by_id");
+	ASSERT_EQ(errno, ENOENT, "bpf_map_get_fd_by_id");
 
 	/* Check bpf_map_get_info_by_fd() */
 	bzero(zeros, sizeof(zeros));
@@ -53,25 +53,26 @@ void serial_test_bpf_obj_id(void)
 		/* test_obj_id.o is a dumb prog. It should never fail
 		 * to load.
 		 */
-		if (CHECK_FAIL(err))
+		if (!ASSERT_OK(err, "bpf_prog_test_load"))
 			continue;
 
 		/* Insert a magic value to the map */
 		map_fds[i] = bpf_find_map(__func__, objs[i], "test_map_id");
-		if (CHECK_FAIL(map_fds[i] < 0))
+		if (!ASSERT_GE(map_fds[i], 0, "bpf_find_map"))
 			goto done;
+
 		err = bpf_map_update_elem(map_fds[i], &array_key,
 					  &array_magic_value, 0);
-		if (CHECK_FAIL(err))
+		if (!ASSERT_OK(err, "bpf_map_update_elem"))
 			goto done;
 
-		prog = bpf_object__find_program_by_name(objs[i],
-							"test_obj_id");
-		if (CHECK_FAIL(!prog))
+		prog = bpf_object__find_program_by_name(objs[i], "test_obj_id");
+		if (!ASSERT_OK_PTR(prog, "bpf_object__find_program_by_name"))
 			goto done;
+
 		links[i] = bpf_program__attach(prog);
 		err = libbpf_get_error(links[i]);
-		if (CHECK(err, "prog_attach", "prog #%d, err %d\n", i, err)) {
+		if (!ASSERT_OK(err, "bpf_program__attach")) {
 			links[i] = NULL;
 			goto done;
 		}
@@ -81,24 +82,14 @@ void serial_test_bpf_obj_id(void)
 		bzero(&map_infos[i], info_len);
 		err = bpf_map_get_info_by_fd(map_fds[i], &map_infos[i],
 					     &info_len);
-		if (CHECK(err ||
-			  map_infos[i].type != BPF_MAP_TYPE_ARRAY ||
-			  map_infos[i].key_size != sizeof(__u32) ||
-			  map_infos[i].value_size != sizeof(__u64) ||
-			  map_infos[i].max_entries != 1 ||
-			  map_infos[i].map_flags != 0 ||
-			  info_len != sizeof(struct bpf_map_info) ||
-			  strcmp((char *)map_infos[i].name, expected_map_name),
-			  "get-map-info(fd)",
-			  "err %d errno %d type %d(%d) info_len %u(%zu) key_size %u value_size %u max_entries %u map_flags %X name %s(%s)\n",
-			  err, errno,
-			  map_infos[i].type, BPF_MAP_TYPE_ARRAY,
-			  info_len, sizeof(struct bpf_map_info),
-			  map_infos[i].key_size,
-			  map_infos[i].value_size,
-			  map_infos[i].max_entries,
-			  map_infos[i].map_flags,
-			  map_infos[i].name, expected_map_name))
+		if (!ASSERT_OK(err, "bpf_map_get_info_by_fd") ||
+				!ASSERT_EQ(map_infos[i].type, BPF_MAP_TYPE_ARRAY, "map_type") ||
+				!ASSERT_EQ(map_infos[i].key_size, sizeof(__u32), "key_size") ||
+				!ASSERT_EQ(map_infos[i].value_size, sizeof(__u64), "value_size") ||
+				!ASSERT_EQ(map_infos[i].max_entries, 1, "max_entries") ||
+				!ASSERT_EQ(map_infos[i].map_flags, 0, "map_flags") ||
+				!ASSERT_EQ(info_len, sizeof(struct bpf_map_info), "map_info_len") ||
+				!ASSERT_STREQ((char *)map_infos[i].name, expected_map_name, "map_name"))
 			goto done;
 
 		/* Check getting prog info */
@@ -112,48 +103,34 @@ void serial_test_bpf_obj_id(void)
 		prog_infos[i].xlated_prog_len = sizeof(xlated_insns);
 		prog_infos[i].map_ids = ptr_to_u64(map_ids + i);
 		prog_infos[i].nr_map_ids = 2;
+
 		err = clock_gettime(CLOCK_REALTIME, &real_time_ts);
-		if (CHECK_FAIL(err))
+		if (!ASSERT_OK(err, "clock_gettime"))
 			goto done;
+
 		err = clock_gettime(CLOCK_BOOTTIME, &boot_time_ts);
-		if (CHECK_FAIL(err))
+		if (!ASSERT_OK(err, "clock_gettime"))
 			goto done;
+
 		err = bpf_prog_get_info_by_fd(prog_fds[i], &prog_infos[i],
 					      &info_len);
 		load_time = (real_time_ts.tv_sec - boot_time_ts.tv_sec)
 			+ (prog_infos[i].load_time / nsec_per_sec);
-		if (CHECK(err ||
-			  prog_infos[i].type != BPF_PROG_TYPE_RAW_TRACEPOINT ||
-			  info_len != sizeof(struct bpf_prog_info) ||
-			  (env.jit_enabled && !prog_infos[i].jited_prog_len) ||
-			  (env.jit_enabled &&
-			   !memcmp(jited_insns, zeros, sizeof(zeros))) ||
-			  !prog_infos[i].xlated_prog_len ||
-			  !memcmp(xlated_insns, zeros, sizeof(zeros)) ||
-			  load_time < now - 60 || load_time > now + 60 ||
-			  prog_infos[i].created_by_uid != my_uid ||
-			  prog_infos[i].nr_map_ids != 1 ||
-			  *(int *)(long)prog_infos[i].map_ids != map_infos[i].id ||
-			  strcmp((char *)prog_infos[i].name, expected_prog_name),
-			  "get-prog-info(fd)",
-			  "err %d errno %d i %d type %d(%d) info_len %u(%zu) "
-			  "jit_enabled %d jited_prog_len %u xlated_prog_len %u "
-			  "jited_prog %d xlated_prog %d load_time %lu(%lu) "
-			  "uid %u(%u) nr_map_ids %u(%u) map_id %u(%u) "
-			  "name %s(%s)\n",
-			  err, errno, i,
-			  prog_infos[i].type, BPF_PROG_TYPE_SOCKET_FILTER,
-			  info_len, sizeof(struct bpf_prog_info),
-			  env.jit_enabled,
-			  prog_infos[i].jited_prog_len,
-			  prog_infos[i].xlated_prog_len,
-			  !!memcmp(jited_insns, zeros, sizeof(zeros)),
-			  !!memcmp(xlated_insns, zeros, sizeof(zeros)),
-			  load_time, now,
-			  prog_infos[i].created_by_uid, my_uid,
-			  prog_infos[i].nr_map_ids, 1,
-			  *(int *)(long)prog_infos[i].map_ids, map_infos[i].id,
-			  prog_infos[i].name, expected_prog_name))
+
+		if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd") ||
+				!ASSERT_EQ(prog_infos[i].type, BPF_PROG_TYPE_RAW_TRACEPOINT, "prog_type") ||
+				!ASSERT_EQ(info_len, sizeof(struct bpf_prog_info), "prog_info_len") ||
+				!ASSERT_FALSE((env.jit_enabled && !prog_infos[i].jited_prog_len), "jited_prog_len") ||
+				!ASSERT_FALSE((env.jit_enabled && !memcmp(jited_insns, zeros, sizeof(zeros))),
+					"jited_insns") ||
+				!ASSERT_NEQ(prog_infos[i].xlated_prog_len, 0, "xlated_prog_len") ||
+				!ASSERT_NEQ(memcmp(xlated_insns, zeros, sizeof(zeros)), 0, "xlated_insns") ||
+				!ASSERT_GE(load_time, (now - 60), "load_time") ||
+				!ASSERT_LE(load_time, (now + 60), "load_time") ||
+				!ASSERT_EQ(prog_infos[i].created_by_uid, my_uid, "created_by_uid") ||
+				!ASSERT_EQ(prog_infos[i].nr_map_ids, 1, "nr_map_ids") ||
+				!ASSERT_EQ(*(int *)(long)prog_infos[i].map_ids, map_infos[i].id, "map_ids") ||
+				!ASSERT_STREQ((char *)prog_infos[i].name, expected_prog_name, "prog_name"))
 			goto done;
 
 		/* Check getting link info */
@@ -163,25 +140,12 @@ void serial_test_bpf_obj_id(void)
 		link_infos[i].raw_tracepoint.tp_name_len = sizeof(tp_name);
 		err = bpf_link_get_info_by_fd(bpf_link__fd(links[i]),
 					      &link_infos[i], &info_len);
-		if (CHECK(err ||
-			  link_infos[i].type != BPF_LINK_TYPE_RAW_TRACEPOINT ||
-			  link_infos[i].prog_id != prog_infos[i].id ||
-			  link_infos[i].raw_tracepoint.tp_name != ptr_to_u64(&tp_name) ||
-			  strcmp(u64_to_ptr(link_infos[i].raw_tracepoint.tp_name),
-				 "sys_enter") ||
-			  info_len != sizeof(struct bpf_link_info),
-			  "get-link-info(fd)",
-			  "err %d errno %d info_len %u(%zu) type %d(%d) id %d "
-			  "prog_id %d (%d) tp_name %s(%s)\n",
-			  err, errno,
-			  info_len, sizeof(struct bpf_link_info),
-			  link_infos[i].type, BPF_LINK_TYPE_RAW_TRACEPOINT,
-			  link_infos[i].id,
-			  link_infos[i].prog_id, prog_infos[i].id,
-			  (const char *)u64_to_ptr(link_infos[i].raw_tracepoint.tp_name),
-			  "sys_enter"))
+		if (!ASSERT_OK(err, "bpf_link_get_info_by_fd") ||
+				!ASSERT_EQ(link_infos[i].type, BPF_LINK_TYPE_RAW_TRACEPOINT, "link_type") ||
+				!ASSERT_EQ(link_infos[i].prog_id, prog_infos[i].id, "prog_id") ||
+				!ASSERT_EQ(link_infos[i].raw_tracepoint.tp_name, ptr_to_u64(&tp_name), "&tp_name") ||
+				!ASSERT_STREQ(u64_to_ptr(link_infos[i].raw_tracepoint.tp_name), "sys_enter", "tp_name"))
 			goto done;
-
 	}
 
 	/* Check bpf_prog_get_next_id() */
@@ -190,7 +154,7 @@ void serial_test_bpf_obj_id(void)
 	while (!bpf_prog_get_next_id(next_id, &next_id)) {
 		struct bpf_prog_info prog_info = {};
 		__u32 saved_map_id;
-		int prog_fd;
+		int prog_fd, cmp_res;
 
 		info_len = sizeof(prog_info);
 
@@ -198,9 +162,7 @@ void serial_test_bpf_obj_id(void)
 		if (prog_fd < 0 && errno == ENOENT)
 			/* The bpf_prog is in the dead row */
 			continue;
-		if (CHECK(prog_fd < 0, "get-prog-fd(next_id)",
-			  "prog_fd %d next_id %d errno %d\n",
-			  prog_fd, next_id, errno))
+		if (!ASSERT_GE(prog_fd, 0, "bpf_prog_get_fd_by_id"))
 			break;
 
 		for (i = 0; i < nr_iters; i++)
@@ -218,9 +180,8 @@ void serial_test_bpf_obj_id(void)
 		 */
 		prog_info.nr_map_ids = 1;
 		err = bpf_prog_get_info_by_fd(prog_fd, &prog_info, &info_len);
-		if (CHECK(!err || errno != EFAULT,
-			  "get-prog-fd-bad-nr-map-ids", "err %d errno %d(%d)",
-			  err, errno, EFAULT))
+		if (!ASSERT_ERR(err, "bpf_prog_get_info_by_fd") ||
+				!ASSERT_EQ(errno, EFAULT, "bpf_prog_get_info_by_fd"))
 			break;
 		bzero(&prog_info, sizeof(prog_info));
 		info_len = sizeof(prog_info);
@@ -231,27 +192,22 @@ void serial_test_bpf_obj_id(void)
 		err = bpf_prog_get_info_by_fd(prog_fd, &prog_info, &info_len);
 		prog_infos[i].jited_prog_insns = 0;
 		prog_infos[i].xlated_prog_insns = 0;
-		CHECK(err || info_len != sizeof(struct bpf_prog_info) ||
-		      memcmp(&prog_info, &prog_infos[i], info_len) ||
-		      *(int *)(long)prog_info.map_ids != saved_map_id,
-		      "get-prog-info(next_id->fd)",
-		      "err %d errno %d info_len %u(%zu) memcmp %d map_id %u(%u)\n",
-		      err, errno, info_len, sizeof(struct bpf_prog_info),
-		      memcmp(&prog_info, &prog_infos[i], info_len),
-		      *(int *)(long)prog_info.map_ids, saved_map_id);
+		cmp_res = memcmp(&prog_info, &prog_infos[i], info_len);
+
+		ASSERT_OK(err, "bpf_prog_get_info_by_fd");
+		ASSERT_EQ(info_len, sizeof(struct bpf_prog_info), "prog_info_len");
+		ASSERT_OK(cmp_res, "memcmp");
+		ASSERT_EQ(*(int *)(long)prog_info.map_ids, saved_map_id, "map_id");
 		close(prog_fd);
 	}
-	CHECK(nr_id_found != nr_iters,
-	      "check total prog id found by get_next_id",
-	      "nr_id_found %u(%u)\n",
-	      nr_id_found, nr_iters);
+	ASSERT_EQ(nr_id_found, nr_iters, "prog_nr_id_found");
 
 	/* Check bpf_map_get_next_id() */
 	nr_id_found = 0;
 	next_id = 0;
 	while (!bpf_map_get_next_id(next_id, &next_id)) {
 		struct bpf_map_info map_info = {};
-		int map_fd;
+		int map_fd, cmp_res;
 
 		info_len = sizeof(map_info);
 
@@ -259,9 +215,7 @@ void serial_test_bpf_obj_id(void)
 		if (map_fd < 0 && errno == ENOENT)
 			/* The bpf_map is in the dead row */
 			continue;
-		if (CHECK(map_fd < 0, "get-map-fd(next_id)",
-			  "map_fd %d next_id %u errno %d\n",
-			  map_fd, next_id, errno))
+		if (!ASSERT_GE(map_fd, 0, "bpf_map_get_fd_by_id"))
 			break;
 
 		for (i = 0; i < nr_iters; i++)
@@ -274,25 +228,19 @@ void serial_test_bpf_obj_id(void)
 		nr_id_found++;
 
 		err = bpf_map_lookup_elem(map_fd, &array_key, &array_value);
-		if (CHECK_FAIL(err))
+		if (!ASSERT_OK(err, "bpf_map_lookup_elem"))
 			goto done;
 
 		err = bpf_map_get_info_by_fd(map_fd, &map_info, &info_len);
-		CHECK(err || info_len != sizeof(struct bpf_map_info) ||
-		      memcmp(&map_info, &map_infos[i], info_len) ||
-		      array_value != array_magic_value,
-		      "check get-map-info(next_id->fd)",
-		      "err %d errno %d info_len %u(%zu) memcmp %d array_value %llu(%llu)\n",
-		      err, errno, info_len, sizeof(struct bpf_map_info),
-		      memcmp(&map_info, &map_infos[i], info_len),
-		      array_value, array_magic_value);
+		cmp_res = memcmp(&map_info, &map_infos[i], info_len);
+		ASSERT_OK(err, "bpf_map_get_info_by_fd");
+		ASSERT_EQ(info_len, sizeof(struct bpf_map_info), "info_len");
+		ASSERT_OK(cmp_res, "memcmp");
+		ASSERT_EQ(array_value, array_magic_value, "array_value");
 
 		close(map_fd);
 	}
-	CHECK(nr_id_found != nr_iters,
-	      "check total map id found by get_next_id",
-	      "nr_id_found %u(%u)\n",
-	      nr_id_found, nr_iters);
+	ASSERT_EQ(nr_id_found, nr_iters, "map_nr_id_found");
 
 	/* Check bpf_link_get_next_id() */
 	nr_id_found = 0;
@@ -308,9 +256,7 @@ void serial_test_bpf_obj_id(void)
 		if (link_fd < 0 && errno == ENOENT)
 			/* The bpf_link is in the dead row */
 			continue;
-		if (CHECK(link_fd < 0, "get-link-fd(next_id)",
-			  "link_fd %d next_id %u errno %d\n",
-			  link_fd, next_id, errno))
+		if (!ASSERT_GE(link_fd, 0, "bpf_link_get_fd_by_id"))
 			break;
 
 		for (i = 0; i < nr_iters; i++)
@@ -325,17 +271,13 @@ void serial_test_bpf_obj_id(void)
 		err = bpf_link_get_info_by_fd(link_fd, &link_info, &info_len);
 		cmp_res = memcmp(&link_info, &link_infos[i],
 				offsetof(struct bpf_link_info, raw_tracepoint));
-		CHECK(err || info_len != sizeof(link_info) || cmp_res,
-		      "check get-link-info(next_id->fd)",
-		      "err %d errno %d info_len %u(%zu) memcmp %d\n",
-		      err, errno, info_len, sizeof(struct bpf_link_info),
-		      cmp_res);
+		ASSERT_OK(err, "bpf_link_get_info_by_fd");
+		ASSERT_EQ(info_len, sizeof(link_info), "info_len");
+		ASSERT_OK(cmp_res, "memcmp");
 
 		close(link_fd);
 	}
-	CHECK(nr_id_found != nr_iters,
-	      "check total link id found by get_next_id",
-	      "nr_id_found %u(%u)\n", nr_id_found, nr_iters);
+	ASSERT_EQ(nr_id_found, nr_iters, "link_nr_id_found");
 
 done:
 	for (i = 0; i < nr_iters; i++) {
diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c
index 4aabeaa525d4..a88e6e07e4f5 100644
--- a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c
+++ b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c
@@ -20,15 +20,14 @@
 
 static const unsigned int total_bytes = 10 * 1024 * 1024;
 static int expected_stg = 0xeB9F;
-static int stop, duration;
+static int stop;
 
 static int settcpca(int fd, const char *tcp_ca)
 {
 	int err;
 
 	err = setsockopt(fd, IPPROTO_TCP, TCP_CONGESTION, tcp_ca, strlen(tcp_ca));
-	if (CHECK(err == -1, "setsockopt(fd, TCP_CONGESTION)", "errno:%d\n",
-		  errno))
+	if (!ASSERT_NEQ(err, -1, "setsockopt"))
 		return -1;
 
 	return 0;
@@ -65,8 +64,7 @@ static void *server(void *arg)
 		bytes += nr_sent;
 	}
 
-	CHECK(bytes != total_bytes, "send", "%zd != %u nr_sent:%zd errno:%d\n",
-	      bytes, total_bytes, nr_sent, errno);
+	ASSERT_EQ(bytes, total_bytes, "send");
 
 done:
 	if (fd >= 0)
@@ -92,10 +90,11 @@ static void do_test(const char *tcp_ca, const struct bpf_map *sk_stg_map)
 	WRITE_ONCE(stop, 0);
 
 	lfd = socket(AF_INET6, SOCK_STREAM, 0);
-	if (CHECK(lfd == -1, "socket", "errno:%d\n", errno))
+	if (!ASSERT_NEQ(lfd, -1, "socket"))
 		return;
+
 	fd = socket(AF_INET6, SOCK_STREAM, 0);
-	if (CHECK(fd == -1, "socket", "errno:%d\n", errno)) {
+	if (!ASSERT_NEQ(fd, -1, "socket")) {
 		close(lfd);
 		return;
 	}
@@ -108,26 +107,27 @@ static void do_test(const char *tcp_ca, const struct bpf_map *sk_stg_map)
 	sa6.sin6_family = AF_INET6;
 	sa6.sin6_addr = in6addr_loopback;
 	err = bind(lfd, (struct sockaddr *)&sa6, addrlen);
-	if (CHECK(err == -1, "bind", "errno:%d\n", errno))
+	if (!ASSERT_NEQ(err, -1, "bind"))
 		goto done;
+
 	err = getsockname(lfd, (struct sockaddr *)&sa6, &addrlen);
-	if (CHECK(err == -1, "getsockname", "errno:%d\n", errno))
+	if (!ASSERT_NEQ(err, -1, "getsockname"))
 		goto done;
+
 	err = listen(lfd, 1);
-	if (CHECK(err == -1, "listen", "errno:%d\n", errno))
+	if (!ASSERT_NEQ(err, -1, "listen"))
 		goto done;
 
 	if (sk_stg_map) {
 		err = bpf_map_update_elem(bpf_map__fd(sk_stg_map), &fd,
 					  &expected_stg, BPF_NOEXIST);
-		if (CHECK(err, "bpf_map_update_elem(sk_stg_map)",
-			  "err:%d errno:%d\n", err, errno))
+		if (!ASSERT_OK(err, "bpf_map_update_elem(sk_stg_map)"))
 			goto done;
 	}
 
 	/* connect to server */
 	err = connect(fd, (struct sockaddr *)&sa6, addrlen);
-	if (CHECK(err == -1, "connect", "errno:%d\n", errno))
+	if (!ASSERT_NEQ(err, -1, "connect"))
 		goto done;
 
 	if (sk_stg_map) {
@@ -135,14 +135,13 @@ static void do_test(const char *tcp_ca, const struct bpf_map *sk_stg_map)
 
 		err = bpf_map_lookup_elem(bpf_map__fd(sk_stg_map), &fd,
 					  &tmp_stg);
-		if (CHECK(!err || errno != ENOENT,
-			  "bpf_map_lookup_elem(sk_stg_map)",
-			  "err:%d errno:%d\n", err, errno))
+		if (!ASSERT_ERR(err, "bpf_map_lookup_elem(sk_stg_map)") ||
+				!ASSERT_EQ(errno, ENOENT, "bpf_map_lookup_elem(sk_stg_map)"))
 			goto done;
 	}
 
 	err = pthread_create(&srv_thread, NULL, server, (void *)(long)lfd);
-	if (CHECK(err != 0, "pthread_create", "err:%d errno:%d\n", err, errno))
+	if (!ASSERT_OK(err, "pthread_create"))
 		goto done;
 
 	/* recv total_bytes */
@@ -156,13 +155,12 @@ static void do_test(const char *tcp_ca, const struct bpf_map *sk_stg_map)
 		bytes += nr_recv;
 	}
 
-	CHECK(bytes != total_bytes, "recv", "%zd != %u nr_recv:%zd errno:%d\n",
-	      bytes, total_bytes, nr_recv, errno);
+	ASSERT_EQ(bytes, total_bytes, "recv");
 
 	WRITE_ONCE(stop, 1);
 	pthread_join(srv_thread, &thread_ret);
-	CHECK(IS_ERR(thread_ret), "pthread_join", "thread_ret:%ld",
-	      PTR_ERR(thread_ret));
+	ASSERT_OK(IS_ERR(thread_ret), "thread_ret");
+
 done:
 	close(lfd);
 	close(fd);
@@ -174,7 +172,7 @@ static void test_cubic(void)
 	struct bpf_link *link;
 
 	cubic_skel = bpf_cubic__open_and_load();
-	if (CHECK(!cubic_skel, "bpf_cubic__open_and_load", "failed\n"))
+	if (!ASSERT_OK_PTR(cubic_skel, "bpf_cubic__open_and_load"))
 		return;
 
 	link = bpf_map__attach_struct_ops(cubic_skel->maps.cubic);
@@ -197,7 +195,7 @@ static void test_dctcp(void)
 	struct bpf_link *link;
 
 	dctcp_skel = bpf_dctcp__open_and_load();
-	if (CHECK(!dctcp_skel, "bpf_dctcp__open_and_load", "failed\n"))
+	if (!ASSERT_OK_PTR(dctcp_skel, "bpf_dctcp__open_and_load"))
 		return;
 
 	link = bpf_map__attach_struct_ops(dctcp_skel->maps.dctcp);
@@ -207,9 +205,7 @@ static void test_dctcp(void)
 	}
 
 	do_test("bpf_dctcp", dctcp_skel->maps.sk_stg_map);
-	CHECK(dctcp_skel->bss->stg_result != expected_stg,
-	      "Unexpected stg_result", "stg_result (%x) != expected_stg (%x)\n",
-	      dctcp_skel->bss->stg_result, expected_stg);
+	ASSERT_EQ(dctcp_skel->bss->stg_result, expected_stg, "stg_result");
 
 	bpf_link__destroy(link);
 	bpf_dctcp__destroy(dctcp_skel);
diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c
index 731c343897d8..e770912fc1d2 100644
--- a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c
+++ b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c
@@ -35,7 +35,7 @@ static int check_load(const char *file, enum bpf_prog_type type)
 	}
 
 	bpf_program__set_type(prog, type);
-	bpf_program__set_flags(prog, BPF_F_TEST_RND_HI32);
+	bpf_program__set_flags(prog, BPF_F_TEST_RND_HI32 | BPF_F_TEST_REG_INVARIANTS);
 	bpf_program__set_log_level(prog, 4 | extra_prog_load_log_flags);
 
 	err = bpf_object__load(obj);
diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c
index 92d51f377fe5..8fb4a04fbbc0 100644
--- a/tools/testing/selftests/bpf/prog_tests/btf.c
+++ b/tools/testing/selftests/bpf/prog_tests/btf.c
@@ -5265,6 +5265,7 @@ static size_t get_pprint_mapv_size(enum pprint_mapv_kind_t mapv_kind)
 #endif
 
 	assert(0);
+	return 0;
 }
 
 static void set_pprint_mapv(enum pprint_mapv_kind_t mapv_kind,
diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup1_hierarchy.c b/tools/testing/selftests/bpf/prog_tests/cgroup1_hierarchy.c
new file mode 100644
index 000000000000..74d6d7546f40
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/cgroup1_hierarchy.c
@@ -0,0 +1,158 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (C) 2023 Yafang Shao <laoar.shao@gmail.com> */
+
+#include <sys/types.h>
+#include <unistd.h>
+#include <test_progs.h>
+#include "cgroup_helpers.h"
+#include "test_cgroup1_hierarchy.skel.h"
+
+static void bpf_cgroup1(struct test_cgroup1_hierarchy *skel)
+{
+	struct bpf_link *lsm_link, *fentry_link;
+	int err;
+
+	/* Attach LSM prog first */
+	lsm_link = bpf_program__attach_lsm(skel->progs.lsm_run);
+	if (!ASSERT_OK_PTR(lsm_link, "lsm_attach"))
+		return;
+
+	/* LSM prog will be triggered when attaching fentry */
+	fentry_link = bpf_program__attach_trace(skel->progs.fentry_run);
+	ASSERT_NULL(fentry_link, "fentry_attach_fail");
+
+	err = bpf_link__destroy(lsm_link);
+	ASSERT_OK(err, "destroy_lsm");
+}
+
+static void bpf_cgroup1_sleepable(struct test_cgroup1_hierarchy *skel)
+{
+	struct bpf_link *lsm_link, *fentry_link;
+	int err;
+
+	/* Attach LSM prog first */
+	lsm_link = bpf_program__attach_lsm(skel->progs.lsm_s_run);
+	if (!ASSERT_OK_PTR(lsm_link, "lsm_attach"))
+		return;
+
+	/* LSM prog will be triggered when attaching fentry */
+	fentry_link = bpf_program__attach_trace(skel->progs.fentry_run);
+	ASSERT_NULL(fentry_link, "fentry_attach_fail");
+
+	err = bpf_link__destroy(lsm_link);
+	ASSERT_OK(err, "destroy_lsm");
+}
+
+static void bpf_cgroup1_invalid_id(struct test_cgroup1_hierarchy *skel)
+{
+	struct bpf_link *lsm_link, *fentry_link;
+	int err;
+
+	/* Attach LSM prog first */
+	lsm_link = bpf_program__attach_lsm(skel->progs.lsm_run);
+	if (!ASSERT_OK_PTR(lsm_link, "lsm_attach"))
+		return;
+
+	/* LSM prog will be triggered when attaching fentry */
+	fentry_link = bpf_program__attach_trace(skel->progs.fentry_run);
+	if (!ASSERT_OK_PTR(fentry_link, "fentry_attach_success"))
+		goto cleanup;
+
+	err = bpf_link__destroy(fentry_link);
+	ASSERT_OK(err, "destroy_lsm");
+
+cleanup:
+	err = bpf_link__destroy(lsm_link);
+	ASSERT_OK(err, "destroy_fentry");
+}
+
+void test_cgroup1_hierarchy(void)
+{
+	struct test_cgroup1_hierarchy *skel;
+	__u64 current_cgid;
+	int hid, err;
+
+	skel = test_cgroup1_hierarchy__open();
+	if (!ASSERT_OK_PTR(skel, "open"))
+		return;
+
+	skel->bss->target_pid = getpid();
+
+	err = bpf_program__set_attach_target(skel->progs.fentry_run, 0, "bpf_fentry_test1");
+	if (!ASSERT_OK(err, "fentry_set_target"))
+		goto destroy;
+
+	err = test_cgroup1_hierarchy__load(skel);
+	if (!ASSERT_OK(err, "load"))
+		goto destroy;
+
+	/* Setup cgroup1 hierarchy */
+	err = setup_classid_environment();
+	if (!ASSERT_OK(err, "setup_classid_environment"))
+		goto destroy;
+
+	err = join_classid();
+	if (!ASSERT_OK(err, "join_cgroup1"))
+		goto cleanup;
+
+	current_cgid = get_classid_cgroup_id();
+	if (!ASSERT_GE(current_cgid, 0, "cgroup1 id"))
+		goto cleanup;
+
+	hid = get_cgroup1_hierarchy_id("net_cls");
+	if (!ASSERT_GE(hid, 0, "cgroup1 id"))
+		goto cleanup;
+	skel->bss->target_hid = hid;
+
+	if (test__start_subtest("test_cgroup1_hierarchy")) {
+		skel->bss->target_ancestor_cgid = current_cgid;
+		bpf_cgroup1(skel);
+	}
+
+	if (test__start_subtest("test_root_cgid")) {
+		skel->bss->target_ancestor_cgid = 1;
+		skel->bss->target_ancestor_level = 0;
+		bpf_cgroup1(skel);
+	}
+
+	if (test__start_subtest("test_invalid_level")) {
+		skel->bss->target_ancestor_cgid = 1;
+		skel->bss->target_ancestor_level = 1;
+		bpf_cgroup1_invalid_id(skel);
+	}
+
+	if (test__start_subtest("test_invalid_cgid")) {
+		skel->bss->target_ancestor_cgid = 0;
+		bpf_cgroup1_invalid_id(skel);
+	}
+
+	if (test__start_subtest("test_invalid_hid")) {
+		skel->bss->target_ancestor_cgid = 1;
+		skel->bss->target_ancestor_level = 0;
+		skel->bss->target_hid = -1;
+		bpf_cgroup1_invalid_id(skel);
+	}
+
+	if (test__start_subtest("test_invalid_cgrp_name")) {
+		skel->bss->target_hid = get_cgroup1_hierarchy_id("net_cl");
+		skel->bss->target_ancestor_cgid = current_cgid;
+		bpf_cgroup1_invalid_id(skel);
+	}
+
+	if (test__start_subtest("test_invalid_cgrp_name2")) {
+		skel->bss->target_hid = get_cgroup1_hierarchy_id("net_cls,");
+		skel->bss->target_ancestor_cgid = current_cgid;
+		bpf_cgroup1_invalid_id(skel);
+	}
+
+	if (test__start_subtest("test_sleepable_prog")) {
+		skel->bss->target_hid = hid;
+		skel->bss->target_ancestor_cgid = current_cgid;
+		bpf_cgroup1_sleepable(skel);
+	}
+
+cleanup:
+	cleanup_classid_environment();
+destroy:
+	test_cgroup1_hierarchy__destroy(skel);
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_v1v2.c b/tools/testing/selftests/bpf/prog_tests/cgroup_v1v2.c
index 9026b42914d3..addf720428f7 100644
--- a/tools/testing/selftests/bpf/prog_tests/cgroup_v1v2.c
+++ b/tools/testing/selftests/bpf/prog_tests/cgroup_v1v2.c
@@ -71,7 +71,7 @@ void test_cgroup_v1v2(void)
 	}
 	ASSERT_OK(run_test(cgroup_fd, server_fd, false), "cgroup-v2-only");
 	setup_classid_environment();
-	set_classid(42);
+	set_classid();
 	ASSERT_OK(run_test(cgroup_fd, server_fd, true), "cgroup-v1v2");
 	cleanup_classid_environment();
 	close(server_fd);
diff --git a/tools/testing/selftests/bpf/prog_tests/local_kptr_stash.c b/tools/testing/selftests/bpf/prog_tests/local_kptr_stash.c
index b25b870f87ba..e6e50a394472 100644
--- a/tools/testing/selftests/bpf/prog_tests/local_kptr_stash.c
+++ b/tools/testing/selftests/bpf/prog_tests/local_kptr_stash.c
@@ -73,6 +73,37 @@ static void test_local_kptr_stash_unstash(void)
 	local_kptr_stash__destroy(skel);
 }
 
+static void test_refcount_acquire_without_unstash(void)
+{
+	LIBBPF_OPTS(bpf_test_run_opts, opts,
+		    .data_in = &pkt_v4,
+		    .data_size_in = sizeof(pkt_v4),
+		    .repeat = 1,
+	);
+	struct local_kptr_stash *skel;
+	int ret;
+
+	skel = local_kptr_stash__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "local_kptr_stash__open_and_load"))
+		return;
+
+	ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.refcount_acquire_without_unstash),
+				     &opts);
+	ASSERT_OK(ret, "refcount_acquire_without_unstash run");
+	ASSERT_EQ(opts.retval, 2, "refcount_acquire_without_unstash retval");
+
+	ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.stash_refcounted_node), &opts);
+	ASSERT_OK(ret, "stash_refcounted_node run");
+	ASSERT_OK(opts.retval, "stash_refcounted_node retval");
+
+	ret = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.refcount_acquire_without_unstash),
+				     &opts);
+	ASSERT_OK(ret, "refcount_acquire_without_unstash (2) run");
+	ASSERT_EQ(opts.retval, 42, "refcount_acquire_without_unstash (2) retval");
+
+	local_kptr_stash__destroy(skel);
+}
+
 static void test_local_kptr_stash_fail(void)
 {
 	RUN_TESTS(local_kptr_stash_fail);
@@ -86,6 +117,8 @@ void test_local_kptr_stash(void)
 		test_local_kptr_stash_plain();
 	if (test__start_subtest("local_kptr_stash_unstash"))
 		test_local_kptr_stash_unstash();
+	if (test__start_subtest("refcount_acquire_without_unstash"))
+		test_refcount_acquire_without_unstash();
 	if (test__start_subtest("local_kptr_stash_fail"))
 		test_local_kptr_stash_fail();
 }
diff --git a/tools/testing/selftests/bpf/prog_tests/log_buf.c b/tools/testing/selftests/bpf/prog_tests/log_buf.c
index fe9a23e65ef4..0f7ea4d7d9f6 100644
--- a/tools/testing/selftests/bpf/prog_tests/log_buf.c
+++ b/tools/testing/selftests/bpf/prog_tests/log_buf.c
@@ -78,7 +78,7 @@ static void obj_load_log_buf(void)
 	ASSERT_OK_PTR(strstr(libbpf_log_buf, "prog 'bad_prog': BPF program load failed"),
 		      "libbpf_log_not_empty");
 	ASSERT_OK_PTR(strstr(obj_log_buf, "DATASEC license"), "obj_log_not_empty");
-	ASSERT_OK_PTR(strstr(good_log_buf, "0: R1=ctx(off=0,imm=0) R10=fp0"),
+	ASSERT_OK_PTR(strstr(good_log_buf, "0: R1=ctx() R10=fp0"),
 		      "good_log_verbose");
 	ASSERT_OK_PTR(strstr(bad_log_buf, "invalid access to map value, value_size=16 off=16000 size=4"),
 		      "bad_log_not_empty");
@@ -175,7 +175,7 @@ static void bpf_prog_load_log_buf(void)
 	opts.log_level = 2;
 	fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "good_prog", "GPL",
 			   good_prog_insns, good_prog_insn_cnt, &opts);
-	ASSERT_OK_PTR(strstr(log_buf, "0: R1=ctx(off=0,imm=0) R10=fp0"), "good_log_2");
+	ASSERT_OK_PTR(strstr(log_buf, "0: R1=ctx() R10=fp0"), "good_log_2");
 	ASSERT_GE(fd, 0, "good_fd2");
 	if (fd >= 0)
 		close(fd);
diff --git a/tools/testing/selftests/bpf/prog_tests/reg_bounds.c b/tools/testing/selftests/bpf/prog_tests/reg_bounds.c
new file mode 100644
index 000000000000..0c9abd279e18
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/reg_bounds.c
@@ -0,0 +1,2124 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */
+
+#define _GNU_SOURCE
+#include <limits.h>
+#include <test_progs.h>
+#include <linux/filter.h>
+#include <linux/bpf.h>
+
+/* =================================
+ * SHORT AND CONSISTENT NUMBER TYPES
+ * =================================
+ */
+#define U64_MAX ((u64)UINT64_MAX)
+#define U32_MAX ((u32)UINT_MAX)
+#define U16_MAX ((u32)UINT_MAX)
+#define S64_MIN ((s64)INT64_MIN)
+#define S64_MAX ((s64)INT64_MAX)
+#define S32_MIN ((s32)INT_MIN)
+#define S32_MAX ((s32)INT_MAX)
+#define S16_MIN ((s16)0x80000000)
+#define S16_MAX ((s16)0x7fffffff)
+
+typedef unsigned long long ___u64;
+typedef unsigned int ___u32;
+typedef long long ___s64;
+typedef int ___s32;
+
+/* avoid conflicts with already defined types in kernel headers */
+#define u64 ___u64
+#define u32 ___u32
+#define s64 ___s64
+#define s32 ___s32
+
+/* ==================================
+ * STRING BUF ABSTRACTION AND HELPERS
+ * ==================================
+ */
+struct strbuf {
+	size_t buf_sz;
+	int pos;
+	char buf[0];
+};
+
+#define DEFINE_STRBUF(name, N)						\
+	struct { struct strbuf buf; char data[(N)]; } ___##name;	\
+	struct strbuf *name = (___##name.buf.buf_sz = (N), ___##name.buf.pos = 0, &___##name.buf)
+
+__printf(2, 3)
+static inline void snappendf(struct strbuf *s, const char *fmt, ...)
+{
+	va_list args;
+
+	va_start(args, fmt);
+	s->pos += vsnprintf(s->buf + s->pos,
+			    s->pos < s->buf_sz ? s->buf_sz - s->pos : 0,
+			    fmt, args);
+	va_end(args);
+}
+
+/* ==================================
+ * GENERIC NUMBER TYPE AND OPERATIONS
+ * ==================================
+ */
+enum num_t { U64, first_t = U64, U32, S64, S32, last_t = S32 };
+
+static __always_inline u64 min_t(enum num_t t, u64 x, u64 y)
+{
+	switch (t) {
+	case U64: return (u64)x < (u64)y ? (u64)x : (u64)y;
+	case U32: return (u32)x < (u32)y ? (u32)x : (u32)y;
+	case S64: return (s64)x < (s64)y ? (s64)x : (s64)y;
+	case S32: return (s32)x < (s32)y ? (s32)x : (s32)y;
+	default: printf("min_t!\n"); exit(1);
+	}
+}
+
+static __always_inline u64 max_t(enum num_t t, u64 x, u64 y)
+{
+	switch (t) {
+	case U64: return (u64)x > (u64)y ? (u64)x : (u64)y;
+	case U32: return (u32)x > (u32)y ? (u32)x : (u32)y;
+	case S64: return (s64)x > (s64)y ? (s64)x : (s64)y;
+	case S32: return (s32)x > (s32)y ? (u32)(s32)x : (u32)(s32)y;
+	default: printf("max_t!\n"); exit(1);
+	}
+}
+
+static __always_inline u64 cast_t(enum num_t t, u64 x)
+{
+	switch (t) {
+	case U64: return (u64)x;
+	case U32: return (u32)x;
+	case S64: return (s64)x;
+	case S32: return (u32)(s32)x;
+	default: printf("cast_t!\n"); exit(1);
+	}
+}
+
+static const char *t_str(enum num_t t)
+{
+	switch (t) {
+	case U64: return "u64";
+	case U32: return "u32";
+	case S64: return "s64";
+	case S32: return "s32";
+	default: printf("t_str!\n"); exit(1);
+	}
+}
+
+static enum num_t t_is_32(enum num_t t)
+{
+	switch (t) {
+	case U64: return false;
+	case U32: return true;
+	case S64: return false;
+	case S32: return true;
+	default: printf("t_is_32!\n"); exit(1);
+	}
+}
+
+static enum num_t t_signed(enum num_t t)
+{
+	switch (t) {
+	case U64: return S64;
+	case U32: return S32;
+	case S64: return S64;
+	case S32: return S32;
+	default: printf("t_signed!\n"); exit(1);
+	}
+}
+
+static enum num_t t_unsigned(enum num_t t)
+{
+	switch (t) {
+	case U64: return U64;
+	case U32: return U32;
+	case S64: return U64;
+	case S32: return U32;
+	default: printf("t_unsigned!\n"); exit(1);
+	}
+}
+
+#define UNUM_MAX_DECIMAL U16_MAX
+#define SNUM_MAX_DECIMAL S16_MAX
+#define SNUM_MIN_DECIMAL S16_MIN
+
+static bool num_is_small(enum num_t t, u64 x)
+{
+	switch (t) {
+	case U64: return (u64)x <= UNUM_MAX_DECIMAL;
+	case U32: return (u32)x <= UNUM_MAX_DECIMAL;
+	case S64: return (s64)x >= SNUM_MIN_DECIMAL && (s64)x <= SNUM_MAX_DECIMAL;
+	case S32: return (s32)x >= SNUM_MIN_DECIMAL && (s32)x <= SNUM_MAX_DECIMAL;
+	default: printf("num_is_small!\n"); exit(1);
+	}
+}
+
+static void snprintf_num(enum num_t t, struct strbuf *sb, u64 x)
+{
+	bool is_small = num_is_small(t, x);
+
+	if (is_small) {
+		switch (t) {
+		case U64: return snappendf(sb, "%llu", (u64)x);
+		case U32: return snappendf(sb, "%u", (u32)x);
+		case S64: return snappendf(sb, "%lld", (s64)x);
+		case S32: return snappendf(sb, "%d", (s32)x);
+		default: printf("snprintf_num!\n"); exit(1);
+		}
+	} else {
+		switch (t) {
+		case U64:
+			if (x == U64_MAX)
+				return snappendf(sb, "U64_MAX");
+			else if (x >= U64_MAX - 256)
+				return snappendf(sb, "U64_MAX-%llu", U64_MAX - x);
+			else
+				return snappendf(sb, "%#llx", (u64)x);
+		case U32:
+			if ((u32)x == U32_MAX)
+				return snappendf(sb, "U32_MAX");
+			else if ((u32)x >= U32_MAX - 256)
+				return snappendf(sb, "U32_MAX-%u", U32_MAX - (u32)x);
+			else
+				return snappendf(sb, "%#x", (u32)x);
+		case S64:
+			if ((s64)x == S64_MAX)
+				return snappendf(sb, "S64_MAX");
+			else if ((s64)x >= S64_MAX - 256)
+				return snappendf(sb, "S64_MAX-%lld", S64_MAX - (s64)x);
+			else if ((s64)x == S64_MIN)
+				return snappendf(sb, "S64_MIN");
+			else if ((s64)x <= S64_MIN + 256)
+				return snappendf(sb, "S64_MIN+%lld", (s64)x - S64_MIN);
+			else
+				return snappendf(sb, "%#llx", (s64)x);
+		case S32:
+			if ((s32)x == S32_MAX)
+				return snappendf(sb, "S32_MAX");
+			else if ((s32)x >= S32_MAX - 256)
+				return snappendf(sb, "S32_MAX-%d", S32_MAX - (s32)x);
+			else if ((s32)x == S32_MIN)
+				return snappendf(sb, "S32_MIN");
+			else if ((s32)x <= S32_MIN + 256)
+				return snappendf(sb, "S32_MIN+%d", (s32)x - S32_MIN);
+			else
+				return snappendf(sb, "%#x", (s32)x);
+		default: printf("snprintf_num!\n"); exit(1);
+		}
+	}
+}
+
+/* ===================================
+ * GENERIC RANGE STRUCT AND OPERATIONS
+ * ===================================
+ */
+struct range {
+	u64 a, b;
+};
+
+static void snprintf_range(enum num_t t, struct strbuf *sb, struct range x)
+{
+	if (x.a == x.b)
+		return snprintf_num(t, sb, x.a);
+
+	snappendf(sb, "[");
+	snprintf_num(t, sb, x.a);
+	snappendf(sb, "; ");
+	snprintf_num(t, sb, x.b);
+	snappendf(sb, "]");
+}
+
+static void print_range(enum num_t t, struct range x, const char *sfx)
+{
+	DEFINE_STRBUF(sb, 128);
+
+	snprintf_range(t, sb, x);
+	printf("%s%s", sb->buf, sfx);
+}
+
+static const struct range unkn[] = {
+	[U64] = { 0, U64_MAX },
+	[U32] = { 0, U32_MAX },
+	[S64] = { (u64)S64_MIN, (u64)S64_MAX },
+	[S32] = { (u64)(u32)S32_MIN, (u64)(u32)S32_MAX },
+};
+
+static struct range unkn_subreg(enum num_t t)
+{
+	switch (t) {
+	case U64: return unkn[U32];
+	case U32: return unkn[U32];
+	case S64: return unkn[U32];
+	case S32: return unkn[S32];
+	default: printf("unkn_subreg!\n"); exit(1);
+	}
+}
+
+static struct range range(enum num_t t, u64 a, u64 b)
+{
+	switch (t) {
+	case U64: return (struct range){ (u64)a, (u64)b };
+	case U32: return (struct range){ (u32)a, (u32)b };
+	case S64: return (struct range){ (s64)a, (s64)b };
+	case S32: return (struct range){ (u32)(s32)a, (u32)(s32)b };
+	default: printf("range!\n"); exit(1);
+	}
+}
+
+static __always_inline u32 sign64(u64 x) { return (x >> 63) & 1; }
+static __always_inline u32 sign32(u64 x) { return ((u32)x >> 31) & 1; }
+static __always_inline u32 upper32(u64 x) { return (u32)(x >> 32); }
+static __always_inline u64 swap_low32(u64 x, u32 y) { return (x & 0xffffffff00000000ULL) | y; }
+
+static bool range_eq(struct range x, struct range y)
+{
+	return x.a == y.a && x.b == y.b;
+}
+
+static struct range range_cast_to_s32(struct range x)
+{
+	u64 a = x.a, b = x.b;
+
+	/* if upper 32 bits are constant, lower 32 bits should form a proper
+	 * s32 range to be correct
+	 */
+	if (upper32(a) == upper32(b) && (s32)a <= (s32)b)
+		return range(S32, a, b);
+
+	/* Special case where upper bits form a small sequence of two
+	 * sequential numbers (in 32-bit unsigned space, so 0xffffffff to
+	 * 0x00000000 is also valid), while lower bits form a proper s32 range
+	 * going from negative numbers to positive numbers.
+	 *
+	 * E.g.: [0xfffffff0ffffff00; 0xfffffff100000010]. Iterating
+	 * over full 64-bit numbers range will form a proper [-16, 16]
+	 * ([0xffffff00; 0x00000010]) range in its lower 32 bits.
+	 */
+	if (upper32(a) + 1 == upper32(b) && (s32)a < 0 && (s32)b >= 0)
+		return range(S32, a, b);
+
+	/* otherwise we can't derive much meaningful information */
+	return unkn[S32];
+}
+
+static struct range range_cast_u64(enum num_t to_t, struct range x)
+{
+	u64 a = (u64)x.a, b = (u64)x.b;
+
+	switch (to_t) {
+	case U64:
+		return x;
+	case U32:
+		if (upper32(a) != upper32(b))
+			return unkn[U32];
+		return range(U32, a, b);
+	case S64:
+		if (sign64(a) != sign64(b))
+			return unkn[S64];
+		return range(S64, a, b);
+	case S32:
+		return range_cast_to_s32(x);
+	default: printf("range_cast_u64!\n"); exit(1);
+	}
+}
+
+static struct range range_cast_s64(enum num_t to_t, struct range x)
+{
+	s64 a = (s64)x.a, b = (s64)x.b;
+
+	switch (to_t) {
+	case U64:
+		/* equivalent to (s64)a <= (s64)b check */
+		if (sign64(a) != sign64(b))
+			return unkn[U64];
+		return range(U64, a, b);
+	case U32:
+		if (upper32(a) != upper32(b) || sign32(a) != sign32(b))
+			return unkn[U32];
+		return range(U32, a, b);
+	case S64:
+		return x;
+	case S32:
+		return range_cast_to_s32(x);
+	default: printf("range_cast_s64!\n"); exit(1);
+	}
+}
+
+static struct range range_cast_u32(enum num_t to_t, struct range x)
+{
+	u32 a = (u32)x.a, b = (u32)x.b;
+
+	switch (to_t) {
+	case U64:
+	case S64:
+		/* u32 is always a valid zero-extended u64/s64 */
+		return range(to_t, a, b);
+	case U32:
+		return x;
+	case S32:
+		return range_cast_to_s32(range(U32, a, b));
+	default: printf("range_cast_u32!\n"); exit(1);
+	}
+}
+
+static struct range range_cast_s32(enum num_t to_t, struct range x)
+{
+	s32 a = (s32)x.a, b = (s32)x.b;
+
+	switch (to_t) {
+	case U64:
+	case U32:
+	case S64:
+		if (sign32(a) != sign32(b))
+			return unkn[to_t];
+		return range(to_t, a, b);
+	case S32:
+		return x;
+	default: printf("range_cast_s32!\n"); exit(1);
+	}
+}
+
+/* Reinterpret range in *from_t* domain as a range in *to_t* domain preserving
+ * all possible information. Worst case, it will be unknown range within
+ * *to_t* domain, if nothing more specific can be guaranteed during the
+ * conversion
+ */
+static struct range range_cast(enum num_t from_t, enum num_t to_t, struct range from)
+{
+	switch (from_t) {
+	case U64: return range_cast_u64(to_t, from);
+	case U32: return range_cast_u32(to_t, from);
+	case S64: return range_cast_s64(to_t, from);
+	case S32: return range_cast_s32(to_t, from);
+	default: printf("range_cast!\n"); exit(1);
+	}
+}
+
+static bool is_valid_num(enum num_t t, u64 x)
+{
+	switch (t) {
+	case U64: return true;
+	case U32: return upper32(x) == 0;
+	case S64: return true;
+	case S32: return upper32(x) == 0;
+	default: printf("is_valid_num!\n"); exit(1);
+	}
+}
+
+static bool is_valid_range(enum num_t t, struct range x)
+{
+	if (!is_valid_num(t, x.a) || !is_valid_num(t, x.b))
+		return false;
+
+	switch (t) {
+	case U64: return (u64)x.a <= (u64)x.b;
+	case U32: return (u32)x.a <= (u32)x.b;
+	case S64: return (s64)x.a <= (s64)x.b;
+	case S32: return (s32)x.a <= (s32)x.b;
+	default: printf("is_valid_range!\n"); exit(1);
+	}
+}
+
+static struct range range_improve(enum num_t t, struct range old, struct range new)
+{
+	return range(t, max_t(t, old.a, new.a), min_t(t, old.b, new.b));
+}
+
+static struct range range_refine(enum num_t x_t, struct range x, enum num_t y_t, struct range y)
+{
+	struct range y_cast;
+
+	y_cast = range_cast(y_t, x_t, y);
+
+	/* the case when new range knowledge, *y*, is a 32-bit subregister
+	 * range, while previous range knowledge, *x*, is a full register
+	 * 64-bit range, needs special treatment to take into account upper 32
+	 * bits of full register range
+	 */
+	if (t_is_32(y_t) && !t_is_32(x_t)) {
+		struct range x_swap;
+
+		/* some combinations of upper 32 bits and sign bit can lead to
+		 * invalid ranges, in such cases it's easier to detect them
+		 * after cast/swap than try to enumerate all the conditions
+		 * under which transformation and knowledge transfer is valid
+		 */
+		x_swap = range(x_t, swap_low32(x.a, y_cast.a), swap_low32(x.b, y_cast.b));
+		if (!is_valid_range(x_t, x_swap))
+			return x;
+		return range_improve(x_t, x, x_swap);
+	}
+
+	/* otherwise, plain range cast and intersection works */
+	return range_improve(x_t, x, y_cast);
+}
+
+/* =======================
+ * GENERIC CONDITIONAL OPS
+ * =======================
+ */
+enum op { OP_LT, OP_LE, OP_GT, OP_GE, OP_EQ, OP_NE, first_op = OP_LT, last_op = OP_NE };
+
+static enum op complement_op(enum op op)
+{
+	switch (op) {
+	case OP_LT: return OP_GE;
+	case OP_LE: return OP_GT;
+	case OP_GT: return OP_LE;
+	case OP_GE: return OP_LT;
+	case OP_EQ: return OP_NE;
+	case OP_NE: return OP_EQ;
+	default: printf("complement_op!\n"); exit(1);
+	}
+}
+
+static const char *op_str(enum op op)
+{
+	switch (op) {
+	case OP_LT: return "<";
+	case OP_LE: return "<=";
+	case OP_GT: return ">";
+	case OP_GE: return ">=";
+	case OP_EQ: return "==";
+	case OP_NE: return "!=";
+	default: printf("op_str!\n"); exit(1);
+	}
+}
+
+/* Can register with range [x.a, x.b] *EVER* satisfy
+ * OP (<, <=, >, >=, ==, !=) relation to
+ * a regsiter with range [y.a, y.b]
+ * _in *num_t* domain_
+ */
+static bool range_canbe_op(enum num_t t, struct range x, struct range y, enum op op)
+{
+#define range_canbe(T) do {									\
+	switch (op) {										\
+	case OP_LT: return (T)x.a < (T)y.b;							\
+	case OP_LE: return (T)x.a <= (T)y.b;							\
+	case OP_GT: return (T)x.b > (T)y.a;							\
+	case OP_GE: return (T)x.b >= (T)y.a;							\
+	case OP_EQ: return (T)max_t(t, x.a, y.a) <= (T)min_t(t, x.b, y.b);			\
+	case OP_NE: return !((T)x.a == (T)x.b && (T)y.a == (T)y.b && (T)x.a == (T)y.a);		\
+	default: printf("range_canbe op %d\n", op); exit(1);					\
+	}											\
+} while (0)
+
+	switch (t) {
+	case U64: { range_canbe(u64); }
+	case U32: { range_canbe(u32); }
+	case S64: { range_canbe(s64); }
+	case S32: { range_canbe(s32); }
+	default: printf("range_canbe!\n"); exit(1);
+	}
+#undef range_canbe
+}
+
+/* Does register with range [x.a, x.b] *ALWAYS* satisfy
+ * OP (<, <=, >, >=, ==, !=) relation to
+ * a regsiter with range [y.a, y.b]
+ * _in *num_t* domain_
+ */
+static bool range_always_op(enum num_t t, struct range x, struct range y, enum op op)
+{
+	/* always op <=> ! canbe complement(op) */
+	return !range_canbe_op(t, x, y, complement_op(op));
+}
+
+/* Does register with range [x.a, x.b] *NEVER* satisfy
+ * OP (<, <=, >, >=, ==, !=) relation to
+ * a regsiter with range [y.a, y.b]
+ * _in *num_t* domain_
+ */
+static bool range_never_op(enum num_t t, struct range x, struct range y, enum op op)
+{
+	return !range_canbe_op(t, x, y, op);
+}
+
+/* similar to verifier's is_branch_taken():
+ *    1 - always taken;
+ *    0 - never taken,
+ *   -1 - unsure.
+ */
+static int range_branch_taken_op(enum num_t t, struct range x, struct range y, enum op op)
+{
+	if (range_always_op(t, x, y, op))
+		return 1;
+	if (range_never_op(t, x, y, op))
+		return 0;
+	return -1;
+}
+
+/* What would be the new estimates for register x and y ranges assuming truthful
+ * OP comparison between them. I.e., (x OP y == true) => x <- newx, y <- newy.
+ *
+ * We assume "interesting" cases where ranges overlap. Cases where it's
+ * obvious that (x OP y) is either always true or false should be filtered with
+ * range_never and range_always checks.
+ */
+static void range_cond(enum num_t t, struct range x, struct range y,
+		       enum op op, struct range *newx, struct range *newy)
+{
+	if (!range_canbe_op(t, x, y, op)) {
+		/* nothing to adjust, can't happen, return original values */
+		*newx = x;
+		*newy = y;
+		return;
+	}
+	switch (op) {
+	case OP_LT:
+		*newx = range(t, x.a, min_t(t, x.b, y.b - 1));
+		*newy = range(t, max_t(t, x.a + 1, y.a), y.b);
+		break;
+	case OP_LE:
+		*newx = range(t, x.a, min_t(t, x.b, y.b));
+		*newy = range(t, max_t(t, x.a, y.a), y.b);
+		break;
+	case OP_GT:
+		*newx = range(t, max_t(t, x.a, y.a + 1), x.b);
+		*newy = range(t, y.a, min_t(t, x.b - 1, y.b));
+		break;
+	case OP_GE:
+		*newx = range(t, max_t(t, x.a, y.a), x.b);
+		*newy = range(t, y.a, min_t(t, x.b, y.b));
+		break;
+	case OP_EQ:
+		*newx = range(t, max_t(t, x.a, y.a), min_t(t, x.b, y.b));
+		*newy = range(t, max_t(t, x.a, y.a), min_t(t, x.b, y.b));
+		break;
+	case OP_NE:
+		/* generic case, can't derive more information */
+		*newx = range(t, x.a, x.b);
+		*newy = range(t, y.a, y.b);
+		break;
+
+		/* below extended logic is not supported by verifier just yet */
+		if (x.a == x.b && x.a == y.a) {
+			/* X is a constant matching left side of Y */
+			*newx = range(t, x.a, x.b);
+			*newy = range(t, y.a + 1, y.b);
+		} else if (x.a == x.b && x.b == y.b) {
+			/* X is a constant matching rigth side of Y */
+			*newx = range(t, x.a, x.b);
+			*newy = range(t, y.a, y.b - 1);
+		} else if (y.a == y.b && x.a == y.a) {
+			/* Y is a constant matching left side of X */
+			*newx = range(t, x.a + 1, x.b);
+			*newy = range(t, y.a, y.b);
+		} else if (y.a == y.b && x.b == y.b) {
+			/* Y is a constant matching rigth side of X */
+			*newx = range(t, x.a, x.b - 1);
+			*newy = range(t, y.a, y.b);
+		} else {
+			/* generic case, can't derive more information */
+			*newx = range(t, x.a, x.b);
+			*newy = range(t, y.a, y.b);
+		}
+
+		break;
+	default:
+		break;
+	}
+}
+
+/* =======================
+ * REGISTER STATE HANDLING
+ * =======================
+ */
+struct reg_state {
+	struct range r[4]; /* indexed by enum num_t: U64, U32, S64, S32 */
+	bool valid;
+};
+
+static void print_reg_state(struct reg_state *r, const char *sfx)
+{
+	DEFINE_STRBUF(sb, 512);
+	enum num_t t;
+	int cnt = 0;
+
+	if (!r->valid) {
+		printf("<not found>%s", sfx);
+		return;
+	}
+
+	snappendf(sb, "scalar(");
+	for (t = first_t; t <= last_t; t++) {
+		snappendf(sb, "%s%s=", cnt++ ? "," : "", t_str(t));
+		snprintf_range(t, sb, r->r[t]);
+	}
+	snappendf(sb, ")");
+
+	printf("%s%s", sb->buf, sfx);
+}
+
+static void print_refinement(enum num_t s_t, struct range src,
+			     enum num_t d_t, struct range old, struct range new,
+			     const char *ctx)
+{
+	printf("REFINING (%s) (%s)SRC=", ctx, t_str(s_t));
+	print_range(s_t, src, "");
+	printf(" (%s)DST_OLD=", t_str(d_t));
+	print_range(d_t, old, "");
+	printf(" (%s)DST_NEW=", t_str(d_t));
+	print_range(d_t, new, "\n");
+}
+
+static void reg_state_refine(struct reg_state *r, enum num_t t, struct range x, const char *ctx)
+{
+	enum num_t d_t, s_t;
+	struct range old;
+	bool keep_going = false;
+
+again:
+	/* try to derive new knowledge from just learned range x of type t */
+	for (d_t = first_t; d_t <= last_t; d_t++) {
+		old = r->r[d_t];
+		r->r[d_t] = range_refine(d_t, r->r[d_t], t, x);
+		if (!range_eq(r->r[d_t], old)) {
+			keep_going = true;
+			if (env.verbosity >= VERBOSE_VERY)
+				print_refinement(t, x, d_t, old, r->r[d_t], ctx);
+		}
+	}
+
+	/* now see if we can derive anything new from updated reg_state's ranges */
+	for (s_t = first_t; s_t <= last_t; s_t++) {
+		for (d_t = first_t; d_t <= last_t; d_t++) {
+			old = r->r[d_t];
+			r->r[d_t] = range_refine(d_t, r->r[d_t], s_t, r->r[s_t]);
+			if (!range_eq(r->r[d_t], old)) {
+				keep_going = true;
+				if (env.verbosity >= VERBOSE_VERY)
+					print_refinement(s_t, r->r[s_t], d_t, old, r->r[d_t], ctx);
+			}
+		}
+	}
+
+	/* keep refining until we converge */
+	if (keep_going) {
+		keep_going = false;
+		goto again;
+	}
+}
+
+static void reg_state_set_const(struct reg_state *rs, enum num_t t, u64 val)
+{
+	enum num_t tt;
+
+	rs->valid = true;
+	for (tt = first_t; tt <= last_t; tt++)
+		rs->r[tt] = tt == t ? range(t, val, val) : unkn[tt];
+
+	reg_state_refine(rs, t, rs->r[t], "CONST");
+}
+
+static void reg_state_cond(enum num_t t, struct reg_state *x, struct reg_state *y, enum op op,
+			   struct reg_state *newx, struct reg_state *newy, const char *ctx)
+{
+	char buf[32];
+	enum num_t ts[2];
+	struct reg_state xx = *x, yy = *y;
+	int i, t_cnt;
+	struct range z1, z2;
+
+	if (op == OP_EQ || op == OP_NE) {
+		/* OP_EQ and OP_NE are sign-agnostic, so we need to process
+		 * both signed and unsigned domains at the same time
+		 */
+		ts[0] = t_unsigned(t);
+		ts[1] = t_signed(t);
+		t_cnt = 2;
+	} else {
+		ts[0] = t;
+		t_cnt = 1;
+	}
+
+	for (i = 0; i < t_cnt; i++) {
+		t = ts[i];
+		z1 = x->r[t];
+		z2 = y->r[t];
+
+		range_cond(t, z1, z2, op, &z1, &z2);
+
+		if (newx) {
+			snprintf(buf, sizeof(buf), "%s R1", ctx);
+			reg_state_refine(&xx, t, z1, buf);
+		}
+		if (newy) {
+			snprintf(buf, sizeof(buf), "%s R2", ctx);
+			reg_state_refine(&yy, t, z2, buf);
+		}
+	}
+
+	if (newx)
+		*newx = xx;
+	if (newy)
+		*newy = yy;
+}
+
+static int reg_state_branch_taken_op(enum num_t t, struct reg_state *x, struct reg_state *y,
+				     enum op op)
+{
+	if (op == OP_EQ || op == OP_NE) {
+		/* OP_EQ and OP_NE are sign-agnostic */
+		enum num_t tu = t_unsigned(t);
+		enum num_t ts = t_signed(t);
+		int br_u, br_s, br;
+
+		br_u = range_branch_taken_op(tu, x->r[tu], y->r[tu], op);
+		br_s = range_branch_taken_op(ts, x->r[ts], y->r[ts], op);
+
+		if (br_u >= 0 && br_s >= 0 && br_u != br_s)
+			ASSERT_FALSE(true, "branch taken inconsistency!\n");
+
+		/* if 64-bit ranges are indecisive, use 32-bit subranges to
+		 * eliminate always/never taken branches, if possible
+		 */
+		if (br_u == -1 && (t == U64 || t == S64)) {
+			br = range_branch_taken_op(U32, x->r[U32], y->r[U32], op);
+			/* we can only reject for OP_EQ, never take branch
+			 * based on lower 32 bits
+			 */
+			if (op == OP_EQ && br == 0)
+				return 0;
+			/* for OP_NEQ we can be conclusive only if lower 32 bits
+			 * differ and thus inequality branch is always taken
+			 */
+			if (op == OP_NE && br == 1)
+				return 1;
+
+			br = range_branch_taken_op(S32, x->r[S32], y->r[S32], op);
+			if (op == OP_EQ && br == 0)
+				return 0;
+			if (op == OP_NE && br == 1)
+				return 1;
+		}
+
+		return br_u >= 0 ? br_u : br_s;
+	}
+	return range_branch_taken_op(t, x->r[t], y->r[t], op);
+}
+
+/* =====================================
+ * BPF PROGS GENERATION AND VERIFICATION
+ * =====================================
+ */
+struct case_spec {
+	/* whether to init full register (r1) or sub-register (w1) */
+	bool init_subregs;
+	/* whether to establish initial value range on full register (r1) or
+	 * sub-register (w1)
+	 */
+	bool setup_subregs;
+	/* whether to establish initial value range using signed or unsigned
+	 * comparisons (i.e., initialize umin/umax or smin/smax directly)
+	 */
+	bool setup_signed;
+	/* whether to perform comparison on full registers or sub-registers */
+	bool compare_subregs;
+	/* whether to perform comparison using signed or unsigned operations */
+	bool compare_signed;
+};
+
+/* Generate test BPF program based on provided test ranges, operation, and
+ * specifications about register bitness and signedness.
+ */
+static int load_range_cmp_prog(struct range x, struct range y, enum op op,
+			       int branch_taken, struct case_spec spec,
+			       char *log_buf, size_t log_sz,
+			       int *false_pos, int *true_pos)
+{
+#define emit(insn) ({							\
+	struct bpf_insn __insns[] = { insn };				\
+	int __i;							\
+	for (__i = 0; __i < ARRAY_SIZE(__insns); __i++)			\
+		insns[cur_pos + __i] = __insns[__i];			\
+	cur_pos += __i;							\
+})
+#define JMP_TO(target) (target - cur_pos - 1)
+	int cur_pos = 0, exit_pos, fd, op_code;
+	struct bpf_insn insns[64];
+	LIBBPF_OPTS(bpf_prog_load_opts, opts,
+		.log_level = 2,
+		.log_buf = log_buf,
+		.log_size = log_sz,
+		.prog_flags = BPF_F_TEST_REG_INVARIANTS,
+	);
+
+	/* ; skip exit block below
+	 * goto +2;
+	 */
+	emit(BPF_JMP_A(2));
+	exit_pos = cur_pos;
+	/* ; exit block for all the preparatory conditionals
+	 * out:
+	 * r0 = 0;
+	 * exit;
+	 */
+	emit(BPF_MOV64_IMM(BPF_REG_0, 0));
+	emit(BPF_EXIT_INSN());
+	/*
+	 * ; assign r6/w6 and r7/w7 unpredictable u64/u32 value
+	 * call bpf_get_current_pid_tgid;
+	 * r6 = r0;               | w6 = w0;
+	 * call bpf_get_current_pid_tgid;
+	 * r7 = r0;               | w7 = w0;
+	 */
+	emit(BPF_EMIT_CALL(BPF_FUNC_get_current_pid_tgid));
+	if (spec.init_subregs)
+		emit(BPF_MOV32_REG(BPF_REG_6, BPF_REG_0));
+	else
+		emit(BPF_MOV64_REG(BPF_REG_6, BPF_REG_0));
+	emit(BPF_EMIT_CALL(BPF_FUNC_get_current_pid_tgid));
+	if (spec.init_subregs)
+		emit(BPF_MOV32_REG(BPF_REG_7, BPF_REG_0));
+	else
+		emit(BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
+	/* ; setup initial r6/w6 possible value range ([x.a, x.b])
+	 * r1 = %[x.a] ll;        | w1 = %[x.a];
+	 * r2 = %[x.b] ll;        | w2 = %[x.b];
+	 * if r6 < r1 goto out;   | if w6 < w1 goto out;
+	 * if r6 > r2 goto out;   | if w6 > w2 goto out;
+	 */
+	if (spec.setup_subregs) {
+		emit(BPF_MOV32_IMM(BPF_REG_1, (s32)x.a));
+		emit(BPF_MOV32_IMM(BPF_REG_2, (s32)x.b));
+		emit(BPF_JMP32_REG(spec.setup_signed ? BPF_JSLT : BPF_JLT,
+				   BPF_REG_6, BPF_REG_1, JMP_TO(exit_pos)));
+		emit(BPF_JMP32_REG(spec.setup_signed ? BPF_JSGT : BPF_JGT,
+				   BPF_REG_6, BPF_REG_2, JMP_TO(exit_pos)));
+	} else {
+		emit(BPF_LD_IMM64(BPF_REG_1, x.a));
+		emit(BPF_LD_IMM64(BPF_REG_2, x.b));
+		emit(BPF_JMP_REG(spec.setup_signed ? BPF_JSLT : BPF_JLT,
+				 BPF_REG_6, BPF_REG_1, JMP_TO(exit_pos)));
+		emit(BPF_JMP_REG(spec.setup_signed ? BPF_JSGT : BPF_JGT,
+				 BPF_REG_6, BPF_REG_2, JMP_TO(exit_pos)));
+	}
+	/* ; setup initial r7/w7 possible value range ([y.a, y.b])
+	 * r1 = %[y.a] ll;        | w1 = %[y.a];
+	 * r2 = %[y.b] ll;        | w2 = %[y.b];
+	 * if r7 < r1 goto out;   | if w7 < w1 goto out;
+	 * if r7 > r2 goto out;   | if w7 > w2 goto out;
+	 */
+	if (spec.setup_subregs) {
+		emit(BPF_MOV32_IMM(BPF_REG_1, (s32)y.a));
+		emit(BPF_MOV32_IMM(BPF_REG_2, (s32)y.b));
+		emit(BPF_JMP32_REG(spec.setup_signed ? BPF_JSLT : BPF_JLT,
+				   BPF_REG_7, BPF_REG_1, JMP_TO(exit_pos)));
+		emit(BPF_JMP32_REG(spec.setup_signed ? BPF_JSGT : BPF_JGT,
+				   BPF_REG_7, BPF_REG_2, JMP_TO(exit_pos)));
+	} else {
+		emit(BPF_LD_IMM64(BPF_REG_1, y.a));
+		emit(BPF_LD_IMM64(BPF_REG_2, y.b));
+		emit(BPF_JMP_REG(spec.setup_signed ? BPF_JSLT : BPF_JLT,
+				 BPF_REG_7, BPF_REG_1, JMP_TO(exit_pos)));
+		emit(BPF_JMP_REG(spec.setup_signed ? BPF_JSGT : BPF_JGT,
+				 BPF_REG_7, BPF_REG_2, JMP_TO(exit_pos)));
+	}
+	/* ; range test instruction
+	 * if r6 <op> r7 goto +3; | if w6 <op> w7 goto +3;
+	 */
+	switch (op) {
+	case OP_LT: op_code = spec.compare_signed ? BPF_JSLT : BPF_JLT; break;
+	case OP_LE: op_code = spec.compare_signed ? BPF_JSLE : BPF_JLE; break;
+	case OP_GT: op_code = spec.compare_signed ? BPF_JSGT : BPF_JGT; break;
+	case OP_GE: op_code = spec.compare_signed ? BPF_JSGE : BPF_JGE; break;
+	case OP_EQ: op_code = BPF_JEQ; break;
+	case OP_NE: op_code = BPF_JNE; break;
+	default:
+		printf("unrecognized op %d\n", op);
+		return -ENOTSUP;
+	}
+	/* ; BEFORE conditional, r0/w0 = {r6/w6,r7/w7} is to extract verifier state reliably
+	 * ; this is used for debugging, as verifier doesn't always print
+	 * ; registers states as of condition jump instruction (e.g., when
+	 * ; precision marking happens)
+	 * r0 = r6;               | w0 = w6;
+	 * r0 = r7;               | w0 = w7;
+	 */
+	if (spec.compare_subregs) {
+		emit(BPF_MOV32_REG(BPF_REG_0, BPF_REG_6));
+		emit(BPF_MOV32_REG(BPF_REG_0, BPF_REG_7));
+	} else {
+		emit(BPF_MOV64_REG(BPF_REG_0, BPF_REG_6));
+		emit(BPF_MOV64_REG(BPF_REG_0, BPF_REG_7));
+	}
+	if (spec.compare_subregs)
+		emit(BPF_JMP32_REG(op_code, BPF_REG_6, BPF_REG_7, 3));
+	else
+		emit(BPF_JMP_REG(op_code, BPF_REG_6, BPF_REG_7, 3));
+	/* ; FALSE branch, r0/w0 = {r6/w6,r7/w7} is to extract verifier state reliably
+	 * r0 = r6;               | w0 = w6;
+	 * r0 = r7;               | w0 = w7;
+	 * exit;
+	 */
+	*false_pos = cur_pos;
+	if (spec.compare_subregs) {
+		emit(BPF_MOV32_REG(BPF_REG_0, BPF_REG_6));
+		emit(BPF_MOV32_REG(BPF_REG_0, BPF_REG_7));
+	} else {
+		emit(BPF_MOV64_REG(BPF_REG_0, BPF_REG_6));
+		emit(BPF_MOV64_REG(BPF_REG_0, BPF_REG_7));
+	}
+	if (branch_taken == 1) /* false branch is never taken */
+		emit(BPF_EMIT_CALL(0xDEAD)); /* poison this branch */
+	else
+		emit(BPF_EXIT_INSN());
+	/* ; TRUE branch, r0/w0 = {r6/w6,r7/w7} is to extract verifier state reliably
+	 * r0 = r6;               | w0 = w6;
+	 * r0 = r7;               | w0 = w7;
+	 * exit;
+	 */
+	*true_pos = cur_pos;
+	if (spec.compare_subregs) {
+		emit(BPF_MOV32_REG(BPF_REG_0, BPF_REG_6));
+		emit(BPF_MOV32_REG(BPF_REG_0, BPF_REG_7));
+	} else {
+		emit(BPF_MOV64_REG(BPF_REG_0, BPF_REG_6));
+		emit(BPF_MOV64_REG(BPF_REG_0, BPF_REG_7));
+	}
+	if (branch_taken == 0) /* true branch is never taken */
+		emit(BPF_EMIT_CALL(0xDEAD)); /* poison this branch */
+	emit(BPF_EXIT_INSN()); /* last instruction has to be exit */
+
+	fd = bpf_prog_load(BPF_PROG_TYPE_RAW_TRACEPOINT, "reg_bounds_test",
+			   "GPL", insns, cur_pos, &opts);
+	if (fd < 0)
+		return fd;
+
+	close(fd);
+	return 0;
+#undef emit
+#undef JMP_TO
+}
+
+#define str_has_pfx(str, pfx) (strncmp(str, pfx, strlen(pfx)) == 0)
+
+/* Parse register state from verifier log.
+ * `s` should point to the start of "Rx = ..." substring in the verifier log.
+ */
+static int parse_reg_state(const char *s, struct reg_state *reg)
+{
+	/* There are two generic forms for SCALAR register:
+	 * - known constant: R6_rwD=P%lld
+	 * - range: R6_rwD=scalar(id=1,...), where "..." is a comma-separated
+	 *   list of optional range specifiers:
+	 *     - umin=%llu, if missing, assumed 0;
+	 *     - umax=%llu, if missing, assumed U64_MAX;
+	 *     - smin=%lld, if missing, assumed S64_MIN;
+	 *     - smax=%lld, if missing, assummed S64_MAX;
+	 *     - umin32=%d, if missing, assumed 0;
+	 *     - umax32=%d, if missing, assumed U32_MAX;
+	 *     - smin32=%d, if missing, assumed S32_MIN;
+	 *     - smax32=%d, if missing, assummed S32_MAX;
+	 *     - var_off=(%#llx; %#llx), tnum part, we don't care about it.
+	 *
+	 * If some of the values are equal, they will be grouped (but min/max
+	 * are not mixed together, and similarly negative values are not
+	 * grouped with non-negative ones). E.g.:
+	 *
+	 *   R6_w=Pscalar(smin=smin32=0, smax=umax=umax32=1000)
+	 *
+	 * _rwD part is optional (and any of the letters can be missing).
+	 * P (precision mark) is optional as well.
+	 *
+	 * Anything inside scalar() is optional, including id, of course.
+	 */
+	struct {
+		const char *pfx;
+		u64 *dst, def;
+		bool is_32, is_set;
+	} *f, fields[8] = {
+		{"smin=", &reg->r[S64].a, S64_MIN},
+		{"smax=", &reg->r[S64].b, S64_MAX},
+		{"umin=", &reg->r[U64].a, 0},
+		{"umax=", &reg->r[U64].b, U64_MAX},
+		{"smin32=", &reg->r[S32].a, (u32)S32_MIN, true},
+		{"smax32=", &reg->r[S32].b, (u32)S32_MAX, true},
+		{"umin32=", &reg->r[U32].a, 0,            true},
+		{"umax32=", &reg->r[U32].b, U32_MAX,      true},
+	};
+	const char *p;
+	int i;
+
+	p = strchr(s, '=');
+	if (!p)
+		return -EINVAL;
+	p++;
+	if (*p == 'P')
+		p++;
+
+	if (!str_has_pfx(p, "scalar(")) {
+		long long sval;
+		enum num_t t;
+
+		if (p[0] == '0' && p[1] == 'x') {
+			if (sscanf(p, "%llx", &sval) != 1)
+				return -EINVAL;
+		} else {
+			if (sscanf(p, "%lld", &sval) != 1)
+				return -EINVAL;
+		}
+
+		reg->valid = true;
+		for (t = first_t; t <= last_t; t++) {
+			reg->r[t] = range(t, sval, sval);
+		}
+		return 0;
+	}
+
+	p += sizeof("scalar");
+	while (p) {
+		int midxs[ARRAY_SIZE(fields)], mcnt = 0;
+		u64 val;
+
+		for (i = 0; i < ARRAY_SIZE(fields); i++) {
+			f = &fields[i];
+			if (!str_has_pfx(p, f->pfx))
+				continue;
+			midxs[mcnt++] = i;
+			p += strlen(f->pfx);
+		}
+
+		if (mcnt) {
+			/* populate all matched fields */
+			if (p[0] == '0' && p[1] == 'x') {
+				if (sscanf(p, "%llx", &val) != 1)
+					return -EINVAL;
+			} else {
+				if (sscanf(p, "%lld", &val) != 1)
+					return -EINVAL;
+			}
+
+			for (i = 0; i < mcnt; i++) {
+				f = &fields[midxs[i]];
+				f->is_set = true;
+				*f->dst = f->is_32 ? (u64)(u32)val : val;
+			}
+		} else if (str_has_pfx(p, "var_off")) {
+			/* skip "var_off=(0x0; 0x3f)" part completely */
+			p = strchr(p, ')');
+			if (!p)
+				return -EINVAL;
+			p++;
+		}
+
+		p = strpbrk(p, ",)");
+		if (*p == ')')
+			break;
+		if (p)
+			p++;
+	}
+
+	reg->valid = true;
+
+	for (i = 0; i < ARRAY_SIZE(fields); i++) {
+		f = &fields[i];
+		if (!f->is_set)
+			*f->dst = f->def;
+	}
+
+	return 0;
+}
+
+
+/* Parse all register states (TRUE/FALSE branches and DST/SRC registers)
+ * out of the verifier log for a corresponding test case BPF program.
+ */
+static int parse_range_cmp_log(const char *log_buf, struct case_spec spec,
+			       int false_pos, int true_pos,
+			       struct reg_state *false1_reg, struct reg_state *false2_reg,
+			       struct reg_state *true1_reg, struct reg_state *true2_reg)
+{
+	struct {
+		int insn_idx;
+		int reg_idx;
+		const char *reg_upper;
+		struct reg_state *state;
+	} specs[] = {
+		{false_pos,     6, "R6=", false1_reg},
+		{false_pos + 1, 7, "R7=", false2_reg},
+		{true_pos,      6, "R6=", true1_reg},
+		{true_pos + 1,  7, "R7=", true2_reg},
+	};
+	char buf[32];
+	const char *p = log_buf, *q;
+	int i, err;
+
+	for (i = 0; i < 4; i++) {
+		sprintf(buf, "%d: (%s) %s = %s%d", specs[i].insn_idx,
+			spec.compare_subregs ? "bc" : "bf",
+			spec.compare_subregs ? "w0" : "r0",
+			spec.compare_subregs ? "w" : "r", specs[i].reg_idx);
+
+		q = strstr(p, buf);
+		if (!q) {
+			*specs[i].state = (struct reg_state){.valid = false};
+			continue;
+		}
+		p = strstr(q, specs[i].reg_upper);
+		if (!p)
+			return -EINVAL;
+		err = parse_reg_state(p, specs[i].state);
+		if (err)
+			return -EINVAL;
+	}
+	return 0;
+}
+
+/* Validate ranges match, and print details if they don't */
+static bool assert_range_eq(enum num_t t, struct range x, struct range y,
+			    const char *ctx1, const char *ctx2)
+{
+	DEFINE_STRBUF(sb, 512);
+
+	if (range_eq(x, y))
+		return true;
+
+	snappendf(sb, "MISMATCH %s.%s: ", ctx1, ctx2);
+	snprintf_range(t, sb, x);
+	snappendf(sb, " != ");
+	snprintf_range(t, sb, y);
+
+	printf("%s\n", sb->buf);
+
+	return false;
+}
+
+/* Validate that register states match, and print details if they don't */
+static bool assert_reg_state_eq(struct reg_state *r, struct reg_state *e, const char *ctx)
+{
+	bool ok = true;
+	enum num_t t;
+
+	if (r->valid != e->valid) {
+		printf("MISMATCH %s: actual %s != expected %s\n", ctx,
+		       r->valid ? "<valid>" : "<invalid>",
+		       e->valid ? "<valid>" : "<invalid>");
+		return false;
+	}
+
+	if (!r->valid)
+		return true;
+
+	for (t = first_t; t <= last_t; t++) {
+		if (!assert_range_eq(t, r->r[t], e->r[t], ctx, t_str(t)))
+			ok = false;
+	}
+
+	return ok;
+}
+
+/* Printf verifier log, filtering out irrelevant noise */
+static void print_verifier_log(const char *buf)
+{
+	const char *p;
+
+	while (buf[0]) {
+		p = strchrnul(buf, '\n');
+
+		/* filter out irrelevant precision backtracking logs */
+		if (str_has_pfx(buf, "mark_precise: "))
+			goto skip_line;
+
+		printf("%.*s\n", (int)(p - buf), buf);
+
+skip_line:
+		buf = *p == '\0' ? p : p + 1;
+	}
+}
+
+/* Simulate provided test case purely with our own range-based logic.
+ * This is done to set up expectations for verifier's branch_taken logic and
+ * verifier's register states in the verifier log.
+ */
+static void sim_case(enum num_t init_t, enum num_t cond_t,
+		     struct range x, struct range y, enum op op,
+		     struct reg_state *fr1, struct reg_state *fr2,
+		     struct reg_state *tr1, struct reg_state *tr2,
+		     int *branch_taken)
+{
+	const u64 A = x.a;
+	const u64 B = x.b;
+	const u64 C = y.a;
+	const u64 D = y.b;
+	struct reg_state rc;
+	enum op rev_op = complement_op(op);
+	enum num_t t;
+
+	fr1->valid = fr2->valid = true;
+	tr1->valid = tr2->valid = true;
+	for (t = first_t; t <= last_t; t++) {
+		/* if we are initializing using 32-bit subregisters,
+		 * full registers get upper 32 bits zeroed automatically
+		 */
+		struct range z = t_is_32(init_t) ? unkn_subreg(t) : unkn[t];
+
+		fr1->r[t] = fr2->r[t] = tr1->r[t] = tr2->r[t] = z;
+	}
+
+	/* step 1: r1 >= A, r2 >= C */
+	reg_state_set_const(&rc, init_t, A);
+	reg_state_cond(init_t, fr1, &rc, OP_GE, fr1, NULL, "r1>=A");
+	reg_state_set_const(&rc, init_t, C);
+	reg_state_cond(init_t, fr2, &rc, OP_GE, fr2, NULL, "r2>=C");
+	*tr1 = *fr1;
+	*tr2 = *fr2;
+	if (env.verbosity >= VERBOSE_VERY) {
+		printf("STEP1 (%s) R1: ", t_str(init_t)); print_reg_state(fr1, "\n");
+		printf("STEP1 (%s) R2: ", t_str(init_t)); print_reg_state(fr2, "\n");
+	}
+
+	/* step 2: r1 <= B, r2 <= D */
+	reg_state_set_const(&rc, init_t, B);
+	reg_state_cond(init_t, fr1, &rc, OP_LE, fr1, NULL, "r1<=B");
+	reg_state_set_const(&rc, init_t, D);
+	reg_state_cond(init_t, fr2, &rc, OP_LE, fr2, NULL, "r2<=D");
+	*tr1 = *fr1;
+	*tr2 = *fr2;
+	if (env.verbosity >= VERBOSE_VERY) {
+		printf("STEP2 (%s) R1: ", t_str(init_t)); print_reg_state(fr1, "\n");
+		printf("STEP2 (%s) R2: ", t_str(init_t)); print_reg_state(fr2, "\n");
+	}
+
+	/* step 3: r1 <op> r2 */
+	*branch_taken = reg_state_branch_taken_op(cond_t, fr1, fr2, op);
+	fr1->valid = fr2->valid = false;
+	tr1->valid = tr2->valid = false;
+	if (*branch_taken != 1) { /* FALSE is possible */
+		fr1->valid = fr2->valid = true;
+		reg_state_cond(cond_t, fr1, fr2, rev_op, fr1, fr2, "FALSE");
+	}
+	if (*branch_taken != 0) { /* TRUE is possible */
+		tr1->valid = tr2->valid = true;
+		reg_state_cond(cond_t, tr1, tr2, op, tr1, tr2, "TRUE");
+	}
+	if (env.verbosity >= VERBOSE_VERY) {
+		printf("STEP3 (%s) FALSE R1:", t_str(cond_t)); print_reg_state(fr1, "\n");
+		printf("STEP3 (%s) FALSE R2:", t_str(cond_t)); print_reg_state(fr2, "\n");
+		printf("STEP3 (%s) TRUE  R1:", t_str(cond_t)); print_reg_state(tr1, "\n");
+		printf("STEP3 (%s) TRUE  R2:", t_str(cond_t)); print_reg_state(tr2, "\n");
+	}
+}
+
+/* ===============================
+ * HIGH-LEVEL TEST CASE VALIDATION
+ * ===============================
+ */
+static u32 upper_seeds[] = {
+	0,
+	1,
+	U32_MAX,
+	U32_MAX - 1,
+	S32_MAX,
+	(u32)S32_MIN,
+};
+
+static u32 lower_seeds[] = {
+	0,
+	1,
+	2, (u32)-2,
+	255, (u32)-255,
+	UINT_MAX,
+	UINT_MAX - 1,
+	INT_MAX,
+	(u32)INT_MIN,
+};
+
+struct ctx {
+	int val_cnt, subval_cnt, range_cnt, subrange_cnt;
+	u64 uvals[ARRAY_SIZE(upper_seeds) * ARRAY_SIZE(lower_seeds)];
+	s64 svals[ARRAY_SIZE(upper_seeds) * ARRAY_SIZE(lower_seeds)];
+	u32 usubvals[ARRAY_SIZE(lower_seeds)];
+	s32 ssubvals[ARRAY_SIZE(lower_seeds)];
+	struct range *uranges, *sranges;
+	struct range *usubranges, *ssubranges;
+	int max_failure_cnt, cur_failure_cnt;
+	int total_case_cnt, case_cnt;
+	int rand_case_cnt;
+	unsigned rand_seed;
+	__u64 start_ns;
+	char progress_ctx[64];
+};
+
+static void cleanup_ctx(struct ctx *ctx)
+{
+	free(ctx->uranges);
+	free(ctx->sranges);
+	free(ctx->usubranges);
+	free(ctx->ssubranges);
+}
+
+struct subtest_case {
+	enum num_t init_t;
+	enum num_t cond_t;
+	struct range x;
+	struct range y;
+	enum op op;
+};
+
+static void subtest_case_str(struct strbuf *sb, struct subtest_case *t, bool use_op)
+{
+	snappendf(sb, "(%s)", t_str(t->init_t));
+	snprintf_range(t->init_t, sb, t->x);
+	snappendf(sb, " (%s)%s ", t_str(t->cond_t), use_op ? op_str(t->op) : "<op>");
+	snprintf_range(t->init_t, sb, t->y);
+}
+
+/* Generate and validate test case based on specific combination of setup
+ * register ranges (including their expected num_t domain), and conditional
+ * operation to perform (including num_t domain in which it has to be
+ * performed)
+ */
+static int verify_case_op(enum num_t init_t, enum num_t cond_t,
+			  struct range x, struct range y, enum op op)
+{
+	char log_buf[256 * 1024];
+	size_t log_sz = sizeof(log_buf);
+	int err, false_pos = 0, true_pos = 0, branch_taken;
+	struct reg_state fr1, fr2, tr1, tr2;
+	struct reg_state fe1, fe2, te1, te2;
+	bool failed = false;
+	struct case_spec spec = {
+		.init_subregs = (init_t == U32 || init_t == S32),
+		.setup_subregs = (init_t == U32 || init_t == S32),
+		.setup_signed = (init_t == S64 || init_t == S32),
+		.compare_subregs = (cond_t == U32 || cond_t == S32),
+		.compare_signed = (cond_t == S64 || cond_t == S32),
+	};
+
+	log_buf[0] = '\0';
+
+	sim_case(init_t, cond_t, x, y, op, &fe1, &fe2, &te1, &te2, &branch_taken);
+
+	err = load_range_cmp_prog(x, y, op, branch_taken, spec,
+				  log_buf, log_sz, &false_pos, &true_pos);
+	if (err) {
+		ASSERT_OK(err, "load_range_cmp_prog");
+		failed = true;
+	}
+
+	err = parse_range_cmp_log(log_buf, spec, false_pos, true_pos,
+				  &fr1, &fr2, &tr1, &tr2);
+	if (err) {
+		ASSERT_OK(err, "parse_range_cmp_log");
+		failed = true;
+	}
+
+	if (!assert_reg_state_eq(&fr1, &fe1, "false_reg1") ||
+	    !assert_reg_state_eq(&fr2, &fe2, "false_reg2") ||
+	    !assert_reg_state_eq(&tr1, &te1, "true_reg1") ||
+	    !assert_reg_state_eq(&tr2, &te2, "true_reg2")) {
+		failed = true;
+	}
+
+	if (failed || env.verbosity >= VERBOSE_NORMAL) {
+		if (failed || env.verbosity >= VERBOSE_VERY) {
+			printf("VERIFIER LOG:\n========================\n");
+			print_verifier_log(log_buf);
+			printf("=====================\n");
+		}
+		printf("ACTUAL   FALSE1: "); print_reg_state(&fr1, "\n");
+		printf("EXPECTED FALSE1: "); print_reg_state(&fe1, "\n");
+		printf("ACTUAL   FALSE2: "); print_reg_state(&fr2, "\n");
+		printf("EXPECTED FALSE2: "); print_reg_state(&fe2, "\n");
+		printf("ACTUAL   TRUE1:  "); print_reg_state(&tr1, "\n");
+		printf("EXPECTED TRUE1:  "); print_reg_state(&te1, "\n");
+		printf("ACTUAL   TRUE2:  "); print_reg_state(&tr2, "\n");
+		printf("EXPECTED TRUE2:  "); print_reg_state(&te2, "\n");
+
+		return failed ? -EINVAL : 0;
+	}
+
+	return 0;
+}
+
+/* Given setup ranges and number types, go over all supported operations,
+ * generating individual subtest for each allowed combination
+ */
+static int verify_case_opt(struct ctx *ctx, enum num_t init_t, enum num_t cond_t,
+			   struct range x, struct range y, bool is_subtest)
+{
+	DEFINE_STRBUF(sb, 256);
+	int err;
+	struct subtest_case sub = {
+		.init_t = init_t,
+		.cond_t = cond_t,
+		.x = x,
+		.y = y,
+	};
+
+	sb->pos = 0; /* reset position in strbuf */
+	subtest_case_str(sb, &sub, false /* ignore op */);
+	if (is_subtest && !test__start_subtest(sb->buf))
+		return 0;
+
+	for (sub.op = first_op; sub.op <= last_op; sub.op++) {
+		sb->pos = 0; /* reset position in strbuf */
+		subtest_case_str(sb, &sub, true /* print op */);
+
+		if (env.verbosity >= VERBOSE_NORMAL) /* this speeds up debugging */
+			printf("TEST CASE: %s\n", sb->buf);
+
+		err = verify_case_op(init_t, cond_t, x, y, sub.op);
+		if (err || env.verbosity >= VERBOSE_NORMAL)
+			ASSERT_OK(err, sb->buf);
+		if (err) {
+			ctx->cur_failure_cnt++;
+			if (ctx->cur_failure_cnt > ctx->max_failure_cnt)
+				return err;
+			return 0; /* keep testing other cases */
+		}
+		ctx->case_cnt++;
+		if ((ctx->case_cnt % 10000) == 0) {
+			double progress = (ctx->case_cnt + 0.0) / ctx->total_case_cnt;
+			u64 elapsed_ns = get_time_ns() - ctx->start_ns;
+			double remain_ns = elapsed_ns / progress * (1 - progress);
+
+			fprintf(env.stderr, "PROGRESS (%s): %d/%d (%.2lf%%), "
+					    "elapsed %llu mins (%.2lf hrs), "
+					    "ETA %.0lf mins (%.2lf hrs)\n",
+				ctx->progress_ctx,
+				ctx->case_cnt, ctx->total_case_cnt, 100.0 * progress,
+				elapsed_ns / 1000000000 / 60,
+				elapsed_ns / 1000000000.0 / 3600,
+				remain_ns / 1000000000.0 / 60,
+				remain_ns / 1000000000.0 / 3600);
+		}
+	}
+
+	return 0;
+}
+
+static int verify_case(struct ctx *ctx, enum num_t init_t, enum num_t cond_t,
+		       struct range x, struct range y)
+{
+	return verify_case_opt(ctx, init_t, cond_t, x, y, true /* is_subtest */);
+}
+
+/* ================================
+ * GENERATED CASES FROM SEED VALUES
+ * ================================
+ */
+static int u64_cmp(const void *p1, const void *p2)
+{
+	u64 x1 = *(const u64 *)p1, x2 = *(const u64 *)p2;
+
+	return x1 != x2 ? (x1 < x2 ? -1 : 1) : 0;
+}
+
+static int u32_cmp(const void *p1, const void *p2)
+{
+	u32 x1 = *(const u32 *)p1, x2 = *(const u32 *)p2;
+
+	return x1 != x2 ? (x1 < x2 ? -1 : 1) : 0;
+}
+
+static int s64_cmp(const void *p1, const void *p2)
+{
+	s64 x1 = *(const s64 *)p1, x2 = *(const s64 *)p2;
+
+	return x1 != x2 ? (x1 < x2 ? -1 : 1) : 0;
+}
+
+static int s32_cmp(const void *p1, const void *p2)
+{
+	s32 x1 = *(const s32 *)p1, x2 = *(const s32 *)p2;
+
+	return x1 != x2 ? (x1 < x2 ? -1 : 1) : 0;
+}
+
+/* Generate valid unique constants from seeds, both signed and unsigned */
+static void gen_vals(struct ctx *ctx)
+{
+	int i, j, cnt = 0;
+
+	for (i = 0; i < ARRAY_SIZE(upper_seeds); i++) {
+		for (j = 0; j < ARRAY_SIZE(lower_seeds); j++) {
+			ctx->uvals[cnt++] = (((u64)upper_seeds[i]) << 32) | lower_seeds[j];
+		}
+	}
+
+	/* sort and compact uvals (i.e., it's `sort | uniq`) */
+	qsort(ctx->uvals, cnt, sizeof(*ctx->uvals), u64_cmp);
+	for (i = 1, j = 0; i < cnt; i++) {
+		if (ctx->uvals[j] == ctx->uvals[i])
+			continue;
+		j++;
+		ctx->uvals[j] = ctx->uvals[i];
+	}
+	ctx->val_cnt = j + 1;
+
+	/* we have exactly the same number of s64 values, they are just in
+	 * a different order than u64s, so just sort them differently
+	 */
+	for (i = 0; i < ctx->val_cnt; i++)
+		ctx->svals[i] = ctx->uvals[i];
+	qsort(ctx->svals, ctx->val_cnt, sizeof(*ctx->svals), s64_cmp);
+
+	if (env.verbosity >= VERBOSE_SUPER) {
+		DEFINE_STRBUF(sb1, 256);
+		DEFINE_STRBUF(sb2, 256);
+
+		for (i = 0; i < ctx->val_cnt; i++) {
+			sb1->pos = sb2->pos = 0;
+			snprintf_num(U64, sb1, ctx->uvals[i]);
+			snprintf_num(S64, sb2, ctx->svals[i]);
+			printf("SEED #%d: u64=%-20s s64=%-20s\n", i, sb1->buf, sb2->buf);
+		}
+	}
+
+	/* 32-bit values are generated separately */
+	cnt = 0;
+	for (i = 0; i < ARRAY_SIZE(lower_seeds); i++) {
+		ctx->usubvals[cnt++] = lower_seeds[i];
+	}
+
+	/* sort and compact usubvals (i.e., it's `sort | uniq`) */
+	qsort(ctx->usubvals, cnt, sizeof(*ctx->usubvals), u32_cmp);
+	for (i = 1, j = 0; i < cnt; i++) {
+		if (ctx->usubvals[j] == ctx->usubvals[i])
+			continue;
+		j++;
+		ctx->usubvals[j] = ctx->usubvals[i];
+	}
+	ctx->subval_cnt = j + 1;
+
+	for (i = 0; i < ctx->subval_cnt; i++)
+		ctx->ssubvals[i] = ctx->usubvals[i];
+	qsort(ctx->ssubvals, ctx->subval_cnt, sizeof(*ctx->ssubvals), s32_cmp);
+
+	if (env.verbosity >= VERBOSE_SUPER) {
+		DEFINE_STRBUF(sb1, 256);
+		DEFINE_STRBUF(sb2, 256);
+
+		for (i = 0; i < ctx->subval_cnt; i++) {
+			sb1->pos = sb2->pos = 0;
+			snprintf_num(U32, sb1, ctx->usubvals[i]);
+			snprintf_num(S32, sb2, ctx->ssubvals[i]);
+			printf("SUBSEED #%d: u32=%-10s s32=%-10s\n", i, sb1->buf, sb2->buf);
+		}
+	}
+}
+
+/* Generate valid ranges from upper/lower seeds */
+static int gen_ranges(struct ctx *ctx)
+{
+	int i, j, cnt = 0;
+
+	for (i = 0; i < ctx->val_cnt; i++) {
+		for (j = i; j < ctx->val_cnt; j++) {
+			if (env.verbosity >= VERBOSE_SUPER) {
+				DEFINE_STRBUF(sb1, 256);
+				DEFINE_STRBUF(sb2, 256);
+
+				sb1->pos = sb2->pos = 0;
+				snprintf_range(U64, sb1, range(U64, ctx->uvals[i], ctx->uvals[j]));
+				snprintf_range(S64, sb2, range(S64, ctx->svals[i], ctx->svals[j]));
+				printf("RANGE #%d: u64=%-40s s64=%-40s\n", cnt, sb1->buf, sb2->buf);
+			}
+			cnt++;
+		}
+	}
+	ctx->range_cnt = cnt;
+
+	ctx->uranges = calloc(ctx->range_cnt, sizeof(*ctx->uranges));
+	if (!ASSERT_OK_PTR(ctx->uranges, "uranges_calloc"))
+		return -EINVAL;
+	ctx->sranges = calloc(ctx->range_cnt, sizeof(*ctx->sranges));
+	if (!ASSERT_OK_PTR(ctx->sranges, "sranges_calloc"))
+		return -EINVAL;
+
+	cnt = 0;
+	for (i = 0; i < ctx->val_cnt; i++) {
+		for (j = i; j < ctx->val_cnt; j++) {
+			ctx->uranges[cnt] = range(U64, ctx->uvals[i], ctx->uvals[j]);
+			ctx->sranges[cnt] = range(S64, ctx->svals[i], ctx->svals[j]);
+			cnt++;
+		}
+	}
+
+	cnt = 0;
+	for (i = 0; i < ctx->subval_cnt; i++) {
+		for (j = i; j < ctx->subval_cnt; j++) {
+			if (env.verbosity >= VERBOSE_SUPER) {
+				DEFINE_STRBUF(sb1, 256);
+				DEFINE_STRBUF(sb2, 256);
+
+				sb1->pos = sb2->pos = 0;
+				snprintf_range(U32, sb1, range(U32, ctx->usubvals[i], ctx->usubvals[j]));
+				snprintf_range(S32, sb2, range(S32, ctx->ssubvals[i], ctx->ssubvals[j]));
+				printf("SUBRANGE #%d: u32=%-20s s32=%-20s\n", cnt, sb1->buf, sb2->buf);
+			}
+			cnt++;
+		}
+	}
+	ctx->subrange_cnt = cnt;
+
+	ctx->usubranges = calloc(ctx->subrange_cnt, sizeof(*ctx->usubranges));
+	if (!ASSERT_OK_PTR(ctx->usubranges, "usubranges_calloc"))
+		return -EINVAL;
+	ctx->ssubranges = calloc(ctx->subrange_cnt, sizeof(*ctx->ssubranges));
+	if (!ASSERT_OK_PTR(ctx->ssubranges, "ssubranges_calloc"))
+		return -EINVAL;
+
+	cnt = 0;
+	for (i = 0; i < ctx->subval_cnt; i++) {
+		for (j = i; j < ctx->subval_cnt; j++) {
+			ctx->usubranges[cnt] = range(U32, ctx->usubvals[i], ctx->usubvals[j]);
+			ctx->ssubranges[cnt] = range(S32, ctx->ssubvals[i], ctx->ssubvals[j]);
+			cnt++;
+		}
+	}
+
+	return 0;
+}
+
+static int parse_env_vars(struct ctx *ctx)
+{
+	const char *s;
+
+	if ((s = getenv("REG_BOUNDS_MAX_FAILURE_CNT"))) {
+		errno = 0;
+		ctx->max_failure_cnt = strtol(s, NULL, 10);
+		if (errno || ctx->max_failure_cnt < 0) {
+			ASSERT_OK(-errno, "REG_BOUNDS_MAX_FAILURE_CNT");
+			return -EINVAL;
+		}
+	}
+
+	if ((s = getenv("REG_BOUNDS_RAND_CASE_CNT"))) {
+		errno = 0;
+		ctx->rand_case_cnt = strtol(s, NULL, 10);
+		if (errno || ctx->rand_case_cnt < 0) {
+			ASSERT_OK(-errno, "REG_BOUNDS_RAND_CASE_CNT");
+			return -EINVAL;
+		}
+	}
+
+	if ((s = getenv("REG_BOUNDS_RAND_SEED"))) {
+		errno = 0;
+		ctx->rand_seed = strtoul(s, NULL, 10);
+		if (errno) {
+			ASSERT_OK(-errno, "REG_BOUNDS_RAND_SEED");
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int prepare_gen_tests(struct ctx *ctx)
+{
+	const char *s;
+	int err;
+
+	if (!(s = getenv("SLOW_TESTS")) || strcmp(s, "1") != 0) {
+		test__skip();
+		return -ENOTSUP;
+	}
+
+	err = parse_env_vars(ctx);
+	if (err)
+		return err;
+
+	gen_vals(ctx);
+	err = gen_ranges(ctx);
+	if (err) {
+		ASSERT_OK(err, "gen_ranges");
+		return err;
+	}
+
+	return 0;
+}
+
+/* Go over generated constants and ranges and validate various supported
+ * combinations of them
+ */
+static void validate_gen_range_vs_const_64(enum num_t init_t, enum num_t cond_t)
+{
+	struct ctx ctx;
+	struct range rconst;
+	const struct range *ranges;
+	const u64 *vals;
+	int i, j;
+
+	memset(&ctx, 0, sizeof(ctx));
+
+	if (prepare_gen_tests(&ctx))
+		goto cleanup;
+
+	ranges = init_t == U64 ? ctx.uranges : ctx.sranges;
+	vals = init_t == U64 ? ctx.uvals : (const u64 *)ctx.svals;
+
+	ctx.total_case_cnt = (last_op - first_op + 1) * (2 * ctx.range_cnt * ctx.val_cnt);
+	ctx.start_ns = get_time_ns();
+	snprintf(ctx.progress_ctx, sizeof(ctx.progress_ctx),
+		 "RANGE x CONST, %s -> %s",
+		 t_str(init_t), t_str(cond_t));
+
+	for (i = 0; i < ctx.val_cnt; i++) {
+		for (j = 0; j < ctx.range_cnt; j++) {
+			rconst = range(init_t, vals[i], vals[i]);
+
+			/* (u64|s64)(<range> x <const>) */
+			if (verify_case(&ctx, init_t, cond_t, ranges[j], rconst))
+				goto cleanup;
+			/* (u64|s64)(<const> x <range>) */
+			if (verify_case(&ctx, init_t, cond_t, rconst, ranges[j]))
+				goto cleanup;
+		}
+	}
+
+cleanup:
+	cleanup_ctx(&ctx);
+}
+
+static void validate_gen_range_vs_const_32(enum num_t init_t, enum num_t cond_t)
+{
+	struct ctx ctx;
+	struct range rconst;
+	const struct range *ranges;
+	const u32 *vals;
+	int i, j;
+
+	memset(&ctx, 0, sizeof(ctx));
+
+	if (prepare_gen_tests(&ctx))
+		goto cleanup;
+
+	ranges = init_t == U32 ? ctx.usubranges : ctx.ssubranges;
+	vals = init_t == U32 ? ctx.usubvals : (const u32 *)ctx.ssubvals;
+
+	ctx.total_case_cnt = (last_op - first_op + 1) * (2 * ctx.subrange_cnt * ctx.subval_cnt);
+	ctx.start_ns = get_time_ns();
+	snprintf(ctx.progress_ctx, sizeof(ctx.progress_ctx),
+		 "RANGE x CONST, %s -> %s",
+		 t_str(init_t), t_str(cond_t));
+
+	for (i = 0; i < ctx.subval_cnt; i++) {
+		for (j = 0; j < ctx.subrange_cnt; j++) {
+			rconst = range(init_t, vals[i], vals[i]);
+
+			/* (u32|s32)(<range> x <const>) */
+			if (verify_case(&ctx, init_t, cond_t, ranges[j], rconst))
+				goto cleanup;
+			/* (u32|s32)(<const> x <range>) */
+			if (verify_case(&ctx, init_t, cond_t, rconst, ranges[j]))
+				goto cleanup;
+		}
+	}
+
+cleanup:
+	cleanup_ctx(&ctx);
+}
+
+static void validate_gen_range_vs_range(enum num_t init_t, enum num_t cond_t)
+{
+	struct ctx ctx;
+	const struct range *ranges;
+	int i, j, rcnt;
+
+	memset(&ctx, 0, sizeof(ctx));
+
+	if (prepare_gen_tests(&ctx))
+		goto cleanup;
+
+	switch (init_t)
+	{
+	case U64:
+		ranges = ctx.uranges;
+		rcnt = ctx.range_cnt;
+		break;
+	case U32:
+		ranges = ctx.usubranges;
+		rcnt = ctx.subrange_cnt;
+		break;
+	case S64:
+		ranges = ctx.sranges;
+		rcnt = ctx.range_cnt;
+		break;
+	case S32:
+		ranges = ctx.ssubranges;
+		rcnt = ctx.subrange_cnt;
+		break;
+	default:
+		printf("validate_gen_range_vs_range!\n");
+		exit(1);
+	}
+
+	ctx.total_case_cnt = (last_op - first_op + 1) * (2 * rcnt * (rcnt + 1) / 2);
+	ctx.start_ns = get_time_ns();
+	snprintf(ctx.progress_ctx, sizeof(ctx.progress_ctx),
+		 "RANGE x RANGE, %s -> %s",
+		 t_str(init_t), t_str(cond_t));
+
+	for (i = 0; i < rcnt; i++) {
+		for (j = i; j < rcnt; j++) {
+			/* (<range> x <range>) */
+			if (verify_case(&ctx, init_t, cond_t, ranges[i], ranges[j]))
+				goto cleanup;
+			if (verify_case(&ctx, init_t, cond_t, ranges[j], ranges[i]))
+				goto cleanup;
+		}
+	}
+
+cleanup:
+	cleanup_ctx(&ctx);
+}
+
+/* Go over thousands of test cases generated from initial seed values.
+ * Given this take a long time, guard this begind SLOW_TESTS=1 envvar. If
+ * envvar is not set, this test is skipped during test_progs testing.
+ *
+ * We split this up into smaller subsets based on initialization and
+ * conditiona numeric domains to get an easy parallelization with test_progs'
+ * -j argument.
+ */
+
+/* RANGE x CONST, U64 initial range */
+void test_reg_bounds_gen_consts_u64_u64(void) { validate_gen_range_vs_const_64(U64, U64); }
+void test_reg_bounds_gen_consts_u64_s64(void) { validate_gen_range_vs_const_64(U64, S64); }
+void test_reg_bounds_gen_consts_u64_u32(void) { validate_gen_range_vs_const_64(U64, U32); }
+void test_reg_bounds_gen_consts_u64_s32(void) { validate_gen_range_vs_const_64(U64, S32); }
+/* RANGE x CONST, S64 initial range */
+void test_reg_bounds_gen_consts_s64_u64(void) { validate_gen_range_vs_const_64(S64, U64); }
+void test_reg_bounds_gen_consts_s64_s64(void) { validate_gen_range_vs_const_64(S64, S64); }
+void test_reg_bounds_gen_consts_s64_u32(void) { validate_gen_range_vs_const_64(S64, U32); }
+void test_reg_bounds_gen_consts_s64_s32(void) { validate_gen_range_vs_const_64(S64, S32); }
+/* RANGE x CONST, U32 initial range */
+void test_reg_bounds_gen_consts_u32_u64(void) { validate_gen_range_vs_const_32(U32, U64); }
+void test_reg_bounds_gen_consts_u32_s64(void) { validate_gen_range_vs_const_32(U32, S64); }
+void test_reg_bounds_gen_consts_u32_u32(void) { validate_gen_range_vs_const_32(U32, U32); }
+void test_reg_bounds_gen_consts_u32_s32(void) { validate_gen_range_vs_const_32(U32, S32); }
+/* RANGE x CONST, S32 initial range */
+void test_reg_bounds_gen_consts_s32_u64(void) { validate_gen_range_vs_const_32(S32, U64); }
+void test_reg_bounds_gen_consts_s32_s64(void) { validate_gen_range_vs_const_32(S32, S64); }
+void test_reg_bounds_gen_consts_s32_u32(void) { validate_gen_range_vs_const_32(S32, U32); }
+void test_reg_bounds_gen_consts_s32_s32(void) { validate_gen_range_vs_const_32(S32, S32); }
+
+/* RANGE x RANGE, U64 initial range */
+void test_reg_bounds_gen_ranges_u64_u64(void) { validate_gen_range_vs_range(U64, U64); }
+void test_reg_bounds_gen_ranges_u64_s64(void) { validate_gen_range_vs_range(U64, S64); }
+void test_reg_bounds_gen_ranges_u64_u32(void) { validate_gen_range_vs_range(U64, U32); }
+void test_reg_bounds_gen_ranges_u64_s32(void) { validate_gen_range_vs_range(U64, S32); }
+/* RANGE x RANGE, S64 initial range */
+void test_reg_bounds_gen_ranges_s64_u64(void) { validate_gen_range_vs_range(S64, U64); }
+void test_reg_bounds_gen_ranges_s64_s64(void) { validate_gen_range_vs_range(S64, S64); }
+void test_reg_bounds_gen_ranges_s64_u32(void) { validate_gen_range_vs_range(S64, U32); }
+void test_reg_bounds_gen_ranges_s64_s32(void) { validate_gen_range_vs_range(S64, S32); }
+/* RANGE x RANGE, U32 initial range */
+void test_reg_bounds_gen_ranges_u32_u64(void) { validate_gen_range_vs_range(U32, U64); }
+void test_reg_bounds_gen_ranges_u32_s64(void) { validate_gen_range_vs_range(U32, S64); }
+void test_reg_bounds_gen_ranges_u32_u32(void) { validate_gen_range_vs_range(U32, U32); }
+void test_reg_bounds_gen_ranges_u32_s32(void) { validate_gen_range_vs_range(U32, S32); }
+/* RANGE x RANGE, S32 initial range */
+void test_reg_bounds_gen_ranges_s32_u64(void) { validate_gen_range_vs_range(S32, U64); }
+void test_reg_bounds_gen_ranges_s32_s64(void) { validate_gen_range_vs_range(S32, S64); }
+void test_reg_bounds_gen_ranges_s32_u32(void) { validate_gen_range_vs_range(S32, U32); }
+void test_reg_bounds_gen_ranges_s32_s32(void) { validate_gen_range_vs_range(S32, S32); }
+
+#define DEFAULT_RAND_CASE_CNT 100
+
+#define RAND_21BIT_MASK ((1 << 22) - 1)
+
+static u64 rand_u64()
+{
+	/* RAND_MAX is guaranteed to be at least 1<<15, but in practice it
+	 * seems to be 1<<31, so we need to call it thrice to get full u64;
+	 * we'll use rougly equal split: 22 + 21 + 21 bits
+	 */
+	return ((u64)random() << 42) |
+	       (((u64)random() & RAND_21BIT_MASK) << 21) |
+	       (random() & RAND_21BIT_MASK);
+}
+
+static u64 rand_const(enum num_t t)
+{
+	return cast_t(t, rand_u64());
+}
+
+static struct range rand_range(enum num_t t)
+{
+	u64 x = rand_const(t), y = rand_const(t);
+
+	return range(t, min_t(t, x, y), max_t(t, x, y));
+}
+
+static void validate_rand_ranges(enum num_t init_t, enum num_t cond_t, bool const_range)
+{
+	struct ctx ctx;
+	struct range range1, range2;
+	int err, i;
+	u64 t;
+
+	memset(&ctx, 0, sizeof(ctx));
+
+	err = parse_env_vars(&ctx);
+	if (err) {
+		ASSERT_OK(err, "parse_env_vars");
+		return;
+	}
+
+	if (ctx.rand_case_cnt == 0)
+		ctx.rand_case_cnt = DEFAULT_RAND_CASE_CNT;
+	if (ctx.rand_seed == 0)
+		ctx.rand_seed = (unsigned)get_time_ns();
+
+	srandom(ctx.rand_seed);
+
+	ctx.total_case_cnt = (last_op - first_op + 1) * (2 * ctx.rand_case_cnt);
+	ctx.start_ns = get_time_ns();
+	snprintf(ctx.progress_ctx, sizeof(ctx.progress_ctx),
+		 "[RANDOM SEED %u] RANGE x %s, %s -> %s",
+		 ctx.rand_seed, const_range ? "CONST" : "RANGE",
+		 t_str(init_t), t_str(cond_t));
+
+	for (i = 0; i < ctx.rand_case_cnt; i++) {
+		range1 = rand_range(init_t);
+		if (const_range) {
+			t = rand_const(init_t);
+			range2 = range(init_t, t, t);
+		} else {
+			range2 = rand_range(init_t);
+		}
+
+		/* <range1> x <range2> */
+		if (verify_case_opt(&ctx, init_t, cond_t, range1, range2, false /* !is_subtest */))
+			goto cleanup;
+		/* <range2> x <range1> */
+		if (verify_case_opt(&ctx, init_t, cond_t, range2, range1, false /* !is_subtest */))
+			goto cleanup;
+	}
+
+cleanup:
+	/* make sure we report random seed for reproducing */
+	ASSERT_TRUE(true, ctx.progress_ctx);
+	cleanup_ctx(&ctx);
+}
+
+/* [RANDOM] RANGE x CONST, U64 initial range */
+void test_reg_bounds_rand_consts_u64_u64(void) { validate_rand_ranges(U64, U64, true /* const */); }
+void test_reg_bounds_rand_consts_u64_s64(void) { validate_rand_ranges(U64, S64, true /* const */); }
+void test_reg_bounds_rand_consts_u64_u32(void) { validate_rand_ranges(U64, U32, true /* const */); }
+void test_reg_bounds_rand_consts_u64_s32(void) { validate_rand_ranges(U64, S32, true /* const */); }
+/* [RANDOM] RANGE x CONST, S64 initial range */
+void test_reg_bounds_rand_consts_s64_u64(void) { validate_rand_ranges(S64, U64, true /* const */); }
+void test_reg_bounds_rand_consts_s64_s64(void) { validate_rand_ranges(S64, S64, true /* const */); }
+void test_reg_bounds_rand_consts_s64_u32(void) { validate_rand_ranges(S64, U32, true /* const */); }
+void test_reg_bounds_rand_consts_s64_s32(void) { validate_rand_ranges(S64, S32, true /* const */); }
+/* [RANDOM] RANGE x CONST, U32 initial range */
+void test_reg_bounds_rand_consts_u32_u64(void) { validate_rand_ranges(U32, U64, true /* const */); }
+void test_reg_bounds_rand_consts_u32_s64(void) { validate_rand_ranges(U32, S64, true /* const */); }
+void test_reg_bounds_rand_consts_u32_u32(void) { validate_rand_ranges(U32, U32, true /* const */); }
+void test_reg_bounds_rand_consts_u32_s32(void) { validate_rand_ranges(U32, S32, true /* const */); }
+/* [RANDOM] RANGE x CONST, S32 initial range */
+void test_reg_bounds_rand_consts_s32_u64(void) { validate_rand_ranges(S32, U64, true /* const */); }
+void test_reg_bounds_rand_consts_s32_s64(void) { validate_rand_ranges(S32, S64, true /* const */); }
+void test_reg_bounds_rand_consts_s32_u32(void) { validate_rand_ranges(S32, U32, true /* const */); }
+void test_reg_bounds_rand_consts_s32_s32(void) { validate_rand_ranges(S32, S32, true /* const */); }
+
+/* [RANDOM] RANGE x RANGE, U64 initial range */
+void test_reg_bounds_rand_ranges_u64_u64(void) { validate_rand_ranges(U64, U64, false /* range */); }
+void test_reg_bounds_rand_ranges_u64_s64(void) { validate_rand_ranges(U64, S64, false /* range */); }
+void test_reg_bounds_rand_ranges_u64_u32(void) { validate_rand_ranges(U64, U32, false /* range */); }
+void test_reg_bounds_rand_ranges_u64_s32(void) { validate_rand_ranges(U64, S32, false /* range */); }
+/* [RANDOM] RANGE x RANGE, S64 initial range */
+void test_reg_bounds_rand_ranges_s64_u64(void) { validate_rand_ranges(S64, U64, false /* range */); }
+void test_reg_bounds_rand_ranges_s64_s64(void) { validate_rand_ranges(S64, S64, false /* range */); }
+void test_reg_bounds_rand_ranges_s64_u32(void) { validate_rand_ranges(S64, U32, false /* range */); }
+void test_reg_bounds_rand_ranges_s64_s32(void) { validate_rand_ranges(S64, S32, false /* range */); }
+/* [RANDOM] RANGE x RANGE, U32 initial range */
+void test_reg_bounds_rand_ranges_u32_u64(void) { validate_rand_ranges(U32, U64, false /* range */); }
+void test_reg_bounds_rand_ranges_u32_s64(void) { validate_rand_ranges(U32, S64, false /* range */); }
+void test_reg_bounds_rand_ranges_u32_u32(void) { validate_rand_ranges(U32, U32, false /* range */); }
+void test_reg_bounds_rand_ranges_u32_s32(void) { validate_rand_ranges(U32, S32, false /* range */); }
+/* [RANDOM] RANGE x RANGE, S32 initial range */
+void test_reg_bounds_rand_ranges_s32_u64(void) { validate_rand_ranges(S32, U64, false /* range */); }
+void test_reg_bounds_rand_ranges_s32_s64(void) { validate_rand_ranges(S32, S64, false /* range */); }
+void test_reg_bounds_rand_ranges_s32_u32(void) { validate_rand_ranges(S32, U32, false /* range */); }
+void test_reg_bounds_rand_ranges_s32_s32(void) { validate_rand_ranges(S32, S32, false /* range */); }
+
+/* A set of hard-coded "interesting" cases to validate as part of normal
+ * test_progs test runs
+ */
+static struct subtest_case crafted_cases[] = {
+	{U64, U64, {0, 0xffffffff}, {0, 0}},
+	{U64, U64, {0, 0x80000000}, {0, 0}},
+	{U64, U64, {0x100000000ULL, 0x100000100ULL}, {0, 0}},
+	{U64, U64, {0x100000000ULL, 0x180000000ULL}, {0, 0}},
+	{U64, U64, {0x100000000ULL, 0x1ffffff00ULL}, {0, 0}},
+	{U64, U64, {0x100000000ULL, 0x1ffffff01ULL}, {0, 0}},
+	{U64, U64, {0x100000000ULL, 0x1fffffffeULL}, {0, 0}},
+	{U64, U64, {0x100000001ULL, 0x1000000ffULL}, {0, 0}},
+
+	/* single point overlap, interesting BPF_EQ and BPF_NE interactions */
+	{U64, U64, {0, 1}, {1, 0x80000000}},
+	{U64, S64, {0, 1}, {1, 0x80000000}},
+	{U64, U32, {0, 1}, {1, 0x80000000}},
+	{U64, S32, {0, 1}, {1, 0x80000000}},
+
+	{U64, S64, {0, 0xffffffff00000000ULL}, {0, 0}},
+	{U64, S64, {0x7fffffffffffffffULL, 0xffffffff00000000ULL}, {0, 0}},
+	{U64, S64, {0x7fffffff00000001ULL, 0xffffffff00000000ULL}, {0, 0}},
+	{U64, S64, {0, 0xffffffffULL}, {1, 1}},
+	{U64, S64, {0, 0xffffffffULL}, {0x7fffffff, 0x7fffffff}},
+
+	{U64, U32, {0, 0x100000000}, {0, 0}},
+	{U64, U32, {0xfffffffe, 0x100000000}, {0x80000000, 0x80000000}},
+
+	{U64, S32, {0, 0xffffffff00000000ULL}, {0, 0}},
+	/* these are tricky cases where lower 32 bits allow to tighten 64
+	 * bit boundaries based on tightened lower 32 bit boundaries
+	 */
+	{U64, S32, {0, 0x0ffffffffULL}, {0, 0}},
+	{U64, S32, {0, 0x100000000ULL}, {0, 0}},
+	{U64, S32, {0, 0x100000001ULL}, {0, 0}},
+	{U64, S32, {0, 0x180000000ULL}, {0, 0}},
+	{U64, S32, {0, 0x17fffffffULL}, {0, 0}},
+	{U64, S32, {0, 0x180000001ULL}, {0, 0}},
+
+	/* verifier knows about [-1, 0] range for s32 for this case already */
+	{S64, S64, {0xffffffffffffffffULL, 0}, {0xffffffff00000000ULL, 0xffffffff00000000ULL}},
+	/* but didn't know about these cases initially */
+	{U64, U64, {0xffffffff, 0x100000000ULL}, {0, 0}}, /* s32: [-1, 0] */
+	{U64, U64, {0xffffffff, 0x100000001ULL}, {0, 0}}, /* s32: [-1, 1] */
+
+	/* longer convergence case: learning from u64 -> s64 -> u64 -> u32,
+	 * arriving at u32: [1, U32_MAX] (instead of more pessimistic [0, U32_MAX])
+	 */
+	{S64, U64, {0xffffffff00000001ULL, 0}, {0xffffffff00000000ULL, 0xffffffff00000000ULL}},
+
+	{U32, U32, {1, U32_MAX}, {0, 0}},
+
+	{U32, S32, {0, U32_MAX}, {U32_MAX, U32_MAX}},
+
+	{S32, U64, {(u32)(s32)S32_MIN, (u32)(s32)S32_MIN}, {(u32)(s32)-255, 0}},
+	{S32, S64, {(u32)(s32)S32_MIN, (u32)(s32)-255}, {(u32)(s32)-2, 0}},
+	{S32, S64, {0, 1}, {(u32)(s32)S32_MIN, (u32)(s32)S32_MIN}},
+	{S32, U32, {(u32)(s32)S32_MIN, (u32)(s32)S32_MIN}, {(u32)(s32)S32_MIN, (u32)(s32)S32_MIN}},
+};
+
+/* Go over crafted hard-coded cases. This is fast, so we do it as part of
+ * normal test_progs run.
+ */
+void test_reg_bounds_crafted(void)
+{
+	struct ctx ctx;
+	int i;
+
+	memset(&ctx, 0, sizeof(ctx));
+
+	for (i = 0; i < ARRAY_SIZE(crafted_cases); i++) {
+		struct subtest_case *c = &crafted_cases[i];
+
+		verify_case(&ctx, c->init_t, c->cond_t, c->x, c->y);
+		verify_case(&ctx, c->init_t, c->cond_t, c->y, c->x);
+	}
+
+	cleanup_ctx(&ctx);
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/spin_lock.c b/tools/testing/selftests/bpf/prog_tests/spin_lock.c
index f29c08d93beb..18d451be57c8 100644
--- a/tools/testing/selftests/bpf/prog_tests/spin_lock.c
+++ b/tools/testing/selftests/bpf/prog_tests/spin_lock.c
@@ -13,22 +13,22 @@ static struct {
 	const char *err_msg;
 } spin_lock_fail_tests[] = {
 	{ "lock_id_kptr_preserve",
-	  "5: (bf) r1 = r0                       ; R0_w=ptr_foo(id=2,ref_obj_id=2,off=0,imm=0) "
-	  "R1_w=ptr_foo(id=2,ref_obj_id=2,off=0,imm=0) refs=2\n6: (85) call bpf_this_cpu_ptr#154\n"
+	  "5: (bf) r1 = r0                       ; R0_w=ptr_foo(id=2,ref_obj_id=2) "
+	  "R1_w=ptr_foo(id=2,ref_obj_id=2) refs=2\n6: (85) call bpf_this_cpu_ptr#154\n"
 	  "R1 type=ptr_ expected=percpu_ptr_" },
 	{ "lock_id_global_zero",
-	  "; R1_w=map_value(off=0,ks=4,vs=4,imm=0)\n2: (85) call bpf_this_cpu_ptr#154\n"
+	  "; R1_w=map_value(map=.data.A,ks=4,vs=4)\n2: (85) call bpf_this_cpu_ptr#154\n"
 	  "R1 type=map_value expected=percpu_ptr_" },
 	{ "lock_id_mapval_preserve",
 	  "[0-9]\\+: (bf) r1 = r0                       ;"
-	  " R0_w=map_value(id=1,off=0,ks=4,vs=8,imm=0)"
-	  " R1_w=map_value(id=1,off=0,ks=4,vs=8,imm=0)\n"
+	  " R0_w=map_value(id=1,map=array_map,ks=4,vs=8)"
+	  " R1_w=map_value(id=1,map=array_map,ks=4,vs=8)\n"
 	  "[0-9]\\+: (85) call bpf_this_cpu_ptr#154\n"
 	  "R1 type=map_value expected=percpu_ptr_" },
 	{ "lock_id_innermapval_preserve",
 	  "[0-9]\\+: (bf) r1 = r0                      ;"
-	  " R0=map_value(id=2,off=0,ks=4,vs=8,imm=0)"
-	  " R1_w=map_value(id=2,off=0,ks=4,vs=8,imm=0)\n"
+	  " R0=map_value(id=2,ks=4,vs=8)"
+	  " R1_w=map_value(id=2,ks=4,vs=8)\n"
 	  "[0-9]\\+: (85) call bpf_this_cpu_ptr#154\n"
 	  "R1 type=map_value expected=percpu_ptr_" },
 	{ "lock_id_mismatch_kptr_kptr", "bpf_spin_unlock of different lock" },
diff --git a/tools/testing/selftests/bpf/prog_tests/tc_opts.c b/tools/testing/selftests/bpf/prog_tests/tc_opts.c
index 51883ccb8020..196abf223465 100644
--- a/tools/testing/selftests/bpf/prog_tests/tc_opts.c
+++ b/tools/testing/selftests/bpf/prog_tests/tc_opts.c
@@ -2387,12 +2387,9 @@ static int generate_dummy_prog(void)
 	const size_t prog_insn_cnt = sizeof(prog_insns) / sizeof(struct bpf_insn);
 	LIBBPF_OPTS(bpf_prog_load_opts, opts);
 	const size_t log_buf_sz = 256;
-	char *log_buf;
+	char log_buf[log_buf_sz];
 	int fd = -1;
 
-	log_buf = malloc(log_buf_sz);
-	if (!ASSERT_OK_PTR(log_buf, "log_buf_alloc"))
-		return fd;
 	opts.log_buf = log_buf;
 	opts.log_size = log_buf_sz;
 
@@ -2402,7 +2399,6 @@ static int generate_dummy_prog(void)
 			   prog_insns, prog_insn_cnt, &opts);
 	ASSERT_STREQ(log_buf, "", "log_0");
 	ASSERT_GE(fd, 0, "prog_fd");
-	free(log_buf);
 	return fd;
 }
 
diff --git a/tools/testing/selftests/bpf/prog_tests/tc_redirect.c b/tools/testing/selftests/bpf/prog_tests/tc_redirect.c
index 6ee22c3b251a..518f143c5b0f 100644
--- a/tools/testing/selftests/bpf/prog_tests/tc_redirect.c
+++ b/tools/testing/selftests/bpf/prog_tests/tc_redirect.c
@@ -24,6 +24,7 @@
 
 #include "test_progs.h"
 #include "network_helpers.h"
+#include "netlink_helpers.h"
 #include "test_tc_neigh_fib.skel.h"
 #include "test_tc_neigh.skel.h"
 #include "test_tc_peer.skel.h"
@@ -110,11 +111,17 @@ static void netns_setup_namespaces_nofail(const char *verb)
 	}
 }
 
+enum dev_mode {
+	MODE_VETH,
+	MODE_NETKIT,
+};
+
 struct netns_setup_result {
-	int ifindex_veth_src;
-	int ifindex_veth_src_fwd;
-	int ifindex_veth_dst;
-	int ifindex_veth_dst_fwd;
+	enum dev_mode dev_mode;
+	int ifindex_src;
+	int ifindex_src_fwd;
+	int ifindex_dst;
+	int ifindex_dst_fwd;
 };
 
 static int get_ifaddr(const char *name, char *ifaddr)
@@ -137,58 +144,110 @@ static int get_ifaddr(const char *name, char *ifaddr)
 	return 0;
 }
 
+static int create_netkit(int mode, char *prim, char *peer)
+{
+	struct rtattr *linkinfo, *data, *peer_info;
+	struct rtnl_handle rth = { .fd = -1 };
+	const char *type = "netkit";
+	struct {
+		struct nlmsghdr n;
+		struct ifinfomsg i;
+		char buf[1024];
+	} req = {};
+	int err;
+
+	err = rtnl_open(&rth, 0);
+	if (!ASSERT_OK(err, "open_rtnetlink"))
+		return err;
+
+	memset(&req, 0, sizeof(req));
+	req.n.nlmsg_len = NLMSG_LENGTH(sizeof(struct ifinfomsg));
+	req.n.nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
+	req.n.nlmsg_type = RTM_NEWLINK;
+	req.i.ifi_family = AF_UNSPEC;
+
+	addattr_l(&req.n, sizeof(req), IFLA_IFNAME, prim, strlen(prim));
+	linkinfo = addattr_nest(&req.n, sizeof(req), IFLA_LINKINFO);
+	addattr_l(&req.n, sizeof(req), IFLA_INFO_KIND, type, strlen(type));
+	data = addattr_nest(&req.n, sizeof(req), IFLA_INFO_DATA);
+	addattr32(&req.n, sizeof(req), IFLA_NETKIT_MODE, mode);
+	peer_info = addattr_nest(&req.n, sizeof(req), IFLA_NETKIT_PEER_INFO);
+	req.n.nlmsg_len += sizeof(struct ifinfomsg);
+	addattr_l(&req.n, sizeof(req), IFLA_IFNAME, peer, strlen(peer));
+	addattr_nest_end(&req.n, peer_info);
+	addattr_nest_end(&req.n, data);
+	addattr_nest_end(&req.n, linkinfo);
+
+	err = rtnl_talk(&rth, &req.n, NULL);
+	ASSERT_OK(err, "talk_rtnetlink");
+	rtnl_close(&rth);
+	return err;
+}
+
 static int netns_setup_links_and_routes(struct netns_setup_result *result)
 {
 	struct nstoken *nstoken = NULL;
-	char veth_src_fwd_addr[IFADDR_STR_LEN+1] = {};
-
-	SYS(fail, "ip link add veth_src type veth peer name veth_src_fwd");
-	SYS(fail, "ip link add veth_dst type veth peer name veth_dst_fwd");
+	char src_fwd_addr[IFADDR_STR_LEN+1] = {};
+	int err;
 
-	SYS(fail, "ip link set veth_dst_fwd address " MAC_DST_FWD);
-	SYS(fail, "ip link set veth_dst address " MAC_DST);
+	if (result->dev_mode == MODE_VETH) {
+		SYS(fail, "ip link add src type veth peer name src_fwd");
+		SYS(fail, "ip link add dst type veth peer name dst_fwd");
+
+		SYS(fail, "ip link set dst_fwd address " MAC_DST_FWD);
+		SYS(fail, "ip link set dst address " MAC_DST);
+	} else if (result->dev_mode == MODE_NETKIT) {
+		err = create_netkit(NETKIT_L3, "src", "src_fwd");
+		if (!ASSERT_OK(err, "create_ifindex_src"))
+			goto fail;
+		err = create_netkit(NETKIT_L3, "dst", "dst_fwd");
+		if (!ASSERT_OK(err, "create_ifindex_dst"))
+			goto fail;
+	}
 
-	if (get_ifaddr("veth_src_fwd", veth_src_fwd_addr))
+	if (get_ifaddr("src_fwd", src_fwd_addr))
 		goto fail;
 
-	result->ifindex_veth_src = if_nametoindex("veth_src");
-	if (!ASSERT_GT(result->ifindex_veth_src, 0, "ifindex_veth_src"))
+	result->ifindex_src = if_nametoindex("src");
+	if (!ASSERT_GT(result->ifindex_src, 0, "ifindex_src"))
 		goto fail;
 
-	result->ifindex_veth_src_fwd = if_nametoindex("veth_src_fwd");
-	if (!ASSERT_GT(result->ifindex_veth_src_fwd, 0, "ifindex_veth_src_fwd"))
+	result->ifindex_src_fwd = if_nametoindex("src_fwd");
+	if (!ASSERT_GT(result->ifindex_src_fwd, 0, "ifindex_src_fwd"))
 		goto fail;
 
-	result->ifindex_veth_dst = if_nametoindex("veth_dst");
-	if (!ASSERT_GT(result->ifindex_veth_dst, 0, "ifindex_veth_dst"))
+	result->ifindex_dst = if_nametoindex("dst");
+	if (!ASSERT_GT(result->ifindex_dst, 0, "ifindex_dst"))
 		goto fail;
 
-	result->ifindex_veth_dst_fwd = if_nametoindex("veth_dst_fwd");
-	if (!ASSERT_GT(result->ifindex_veth_dst_fwd, 0, "ifindex_veth_dst_fwd"))
+	result->ifindex_dst_fwd = if_nametoindex("dst_fwd");
+	if (!ASSERT_GT(result->ifindex_dst_fwd, 0, "ifindex_dst_fwd"))
 		goto fail;
 
-	SYS(fail, "ip link set veth_src netns " NS_SRC);
-	SYS(fail, "ip link set veth_src_fwd netns " NS_FWD);
-	SYS(fail, "ip link set veth_dst_fwd netns " NS_FWD);
-	SYS(fail, "ip link set veth_dst netns " NS_DST);
+	SYS(fail, "ip link set src netns " NS_SRC);
+	SYS(fail, "ip link set src_fwd netns " NS_FWD);
+	SYS(fail, "ip link set dst_fwd netns " NS_FWD);
+	SYS(fail, "ip link set dst netns " NS_DST);
 
 	/** setup in 'src' namespace */
 	nstoken = open_netns(NS_SRC);
 	if (!ASSERT_OK_PTR(nstoken, "setns src"))
 		goto fail;
 
-	SYS(fail, "ip addr add " IP4_SRC "/32 dev veth_src");
-	SYS(fail, "ip addr add " IP6_SRC "/128 dev veth_src nodad");
-	SYS(fail, "ip link set dev veth_src up");
+	SYS(fail, "ip addr add " IP4_SRC "/32 dev src");
+	SYS(fail, "ip addr add " IP6_SRC "/128 dev src nodad");
+	SYS(fail, "ip link set dev src up");
 
-	SYS(fail, "ip route add " IP4_DST "/32 dev veth_src scope global");
-	SYS(fail, "ip route add " IP4_NET "/16 dev veth_src scope global");
-	SYS(fail, "ip route add " IP6_DST "/128 dev veth_src scope global");
+	SYS(fail, "ip route add " IP4_DST "/32 dev src scope global");
+	SYS(fail, "ip route add " IP4_NET "/16 dev src scope global");
+	SYS(fail, "ip route add " IP6_DST "/128 dev src scope global");
 
-	SYS(fail, "ip neigh add " IP4_DST " dev veth_src lladdr %s",
-	    veth_src_fwd_addr);
-	SYS(fail, "ip neigh add " IP6_DST " dev veth_src lladdr %s",
-	    veth_src_fwd_addr);
+	if (result->dev_mode == MODE_VETH) {
+		SYS(fail, "ip neigh add " IP4_DST " dev src lladdr %s",
+		    src_fwd_addr);
+		SYS(fail, "ip neigh add " IP6_DST " dev src lladdr %s",
+		    src_fwd_addr);
+	}
 
 	close_netns(nstoken);
 
@@ -201,15 +260,15 @@ static int netns_setup_links_and_routes(struct netns_setup_result *result)
 	 * needs v4 one in order to start ARP probing. IP4_NET route is added
 	 * to the endpoints so that the ARP processing will reply.
 	 */
-	SYS(fail, "ip addr add " IP4_SLL "/32 dev veth_src_fwd");
-	SYS(fail, "ip addr add " IP4_DLL "/32 dev veth_dst_fwd");
-	SYS(fail, "ip link set dev veth_src_fwd up");
-	SYS(fail, "ip link set dev veth_dst_fwd up");
+	SYS(fail, "ip addr add " IP4_SLL "/32 dev src_fwd");
+	SYS(fail, "ip addr add " IP4_DLL "/32 dev dst_fwd");
+	SYS(fail, "ip link set dev src_fwd up");
+	SYS(fail, "ip link set dev dst_fwd up");
 
-	SYS(fail, "ip route add " IP4_SRC "/32 dev veth_src_fwd scope global");
-	SYS(fail, "ip route add " IP6_SRC "/128 dev veth_src_fwd scope global");
-	SYS(fail, "ip route add " IP4_DST "/32 dev veth_dst_fwd scope global");
-	SYS(fail, "ip route add " IP6_DST "/128 dev veth_dst_fwd scope global");
+	SYS(fail, "ip route add " IP4_SRC "/32 dev src_fwd scope global");
+	SYS(fail, "ip route add " IP6_SRC "/128 dev src_fwd scope global");
+	SYS(fail, "ip route add " IP4_DST "/32 dev dst_fwd scope global");
+	SYS(fail, "ip route add " IP6_DST "/128 dev dst_fwd scope global");
 
 	close_netns(nstoken);
 
@@ -218,16 +277,18 @@ static int netns_setup_links_and_routes(struct netns_setup_result *result)
 	if (!ASSERT_OK_PTR(nstoken, "setns dst"))
 		goto fail;
 
-	SYS(fail, "ip addr add " IP4_DST "/32 dev veth_dst");
-	SYS(fail, "ip addr add " IP6_DST "/128 dev veth_dst nodad");
-	SYS(fail, "ip link set dev veth_dst up");
+	SYS(fail, "ip addr add " IP4_DST "/32 dev dst");
+	SYS(fail, "ip addr add " IP6_DST "/128 dev dst nodad");
+	SYS(fail, "ip link set dev dst up");
 
-	SYS(fail, "ip route add " IP4_SRC "/32 dev veth_dst scope global");
-	SYS(fail, "ip route add " IP4_NET "/16 dev veth_dst scope global");
-	SYS(fail, "ip route add " IP6_SRC "/128 dev veth_dst scope global");
+	SYS(fail, "ip route add " IP4_SRC "/32 dev dst scope global");
+	SYS(fail, "ip route add " IP4_NET "/16 dev dst scope global");
+	SYS(fail, "ip route add " IP6_SRC "/128 dev dst scope global");
 
-	SYS(fail, "ip neigh add " IP4_SRC " dev veth_dst lladdr " MAC_DST_FWD);
-	SYS(fail, "ip neigh add " IP6_SRC " dev veth_dst lladdr " MAC_DST_FWD);
+	if (result->dev_mode == MODE_VETH) {
+		SYS(fail, "ip neigh add " IP4_SRC " dev dst lladdr " MAC_DST_FWD);
+		SYS(fail, "ip neigh add " IP6_SRC " dev dst lladdr " MAC_DST_FWD);
+	}
 
 	close_netns(nstoken);
 
@@ -293,23 +354,23 @@ static int netns_load_bpf(const struct bpf_program *src_prog,
 			  const struct bpf_program *chk_prog,
 			  const struct netns_setup_result *setup_result)
 {
-	LIBBPF_OPTS(bpf_tc_hook, qdisc_veth_src_fwd);
-	LIBBPF_OPTS(bpf_tc_hook, qdisc_veth_dst_fwd);
+	LIBBPF_OPTS(bpf_tc_hook, qdisc_src_fwd);
+	LIBBPF_OPTS(bpf_tc_hook, qdisc_dst_fwd);
 	int err;
 
-	/* tc qdisc add dev veth_src_fwd clsact */
-	QDISC_CLSACT_CREATE(&qdisc_veth_src_fwd, setup_result->ifindex_veth_src_fwd);
-	/* tc filter add dev veth_src_fwd ingress bpf da src_prog */
-	XGRESS_FILTER_ADD(&qdisc_veth_src_fwd, BPF_TC_INGRESS, src_prog, 0);
-	/* tc filter add dev veth_src_fwd egress bpf da chk_prog */
-	XGRESS_FILTER_ADD(&qdisc_veth_src_fwd, BPF_TC_EGRESS, chk_prog, 0);
+	/* tc qdisc add dev src_fwd clsact */
+	QDISC_CLSACT_CREATE(&qdisc_src_fwd, setup_result->ifindex_src_fwd);
+	/* tc filter add dev src_fwd ingress bpf da src_prog */
+	XGRESS_FILTER_ADD(&qdisc_src_fwd, BPF_TC_INGRESS, src_prog, 0);
+	/* tc filter add dev src_fwd egress bpf da chk_prog */
+	XGRESS_FILTER_ADD(&qdisc_src_fwd, BPF_TC_EGRESS, chk_prog, 0);
 
-	/* tc qdisc add dev veth_dst_fwd clsact */
-	QDISC_CLSACT_CREATE(&qdisc_veth_dst_fwd, setup_result->ifindex_veth_dst_fwd);
-	/* tc filter add dev veth_dst_fwd ingress bpf da dst_prog */
-	XGRESS_FILTER_ADD(&qdisc_veth_dst_fwd, BPF_TC_INGRESS, dst_prog, 0);
-	/* tc filter add dev veth_dst_fwd egress bpf da chk_prog */
-	XGRESS_FILTER_ADD(&qdisc_veth_dst_fwd, BPF_TC_EGRESS, chk_prog, 0);
+	/* tc qdisc add dev dst_fwd clsact */
+	QDISC_CLSACT_CREATE(&qdisc_dst_fwd, setup_result->ifindex_dst_fwd);
+	/* tc filter add dev dst_fwd ingress bpf da dst_prog */
+	XGRESS_FILTER_ADD(&qdisc_dst_fwd, BPF_TC_INGRESS, dst_prog, 0);
+	/* tc filter add dev dst_fwd egress bpf da chk_prog */
+	XGRESS_FILTER_ADD(&qdisc_dst_fwd, BPF_TC_EGRESS, chk_prog, 0);
 
 	return 0;
 fail:
@@ -539,10 +600,10 @@ static void test_inet_dtime(int family, int type, const char *addr, __u16 port)
 static int netns_load_dtime_bpf(struct test_tc_dtime *skel,
 				const struct netns_setup_result *setup_result)
 {
-	LIBBPF_OPTS(bpf_tc_hook, qdisc_veth_src_fwd);
-	LIBBPF_OPTS(bpf_tc_hook, qdisc_veth_dst_fwd);
-	LIBBPF_OPTS(bpf_tc_hook, qdisc_veth_src);
-	LIBBPF_OPTS(bpf_tc_hook, qdisc_veth_dst);
+	LIBBPF_OPTS(bpf_tc_hook, qdisc_src_fwd);
+	LIBBPF_OPTS(bpf_tc_hook, qdisc_dst_fwd);
+	LIBBPF_OPTS(bpf_tc_hook, qdisc_src);
+	LIBBPF_OPTS(bpf_tc_hook, qdisc_dst);
 	struct nstoken *nstoken;
 	int err;
 
@@ -550,58 +611,58 @@ static int netns_load_dtime_bpf(struct test_tc_dtime *skel,
 	nstoken = open_netns(NS_SRC);
 	if (!ASSERT_OK_PTR(nstoken, "setns " NS_SRC))
 		return -1;
-	/* tc qdisc add dev veth_src clsact */
-	QDISC_CLSACT_CREATE(&qdisc_veth_src, setup_result->ifindex_veth_src);
-	/* tc filter add dev veth_src ingress bpf da ingress_host */
-	XGRESS_FILTER_ADD(&qdisc_veth_src, BPF_TC_INGRESS, skel->progs.ingress_host, 0);
-	/* tc filter add dev veth_src egress bpf da egress_host */
-	XGRESS_FILTER_ADD(&qdisc_veth_src, BPF_TC_EGRESS, skel->progs.egress_host, 0);
+	/* tc qdisc add dev src clsact */
+	QDISC_CLSACT_CREATE(&qdisc_src, setup_result->ifindex_src);
+	/* tc filter add dev src ingress bpf da ingress_host */
+	XGRESS_FILTER_ADD(&qdisc_src, BPF_TC_INGRESS, skel->progs.ingress_host, 0);
+	/* tc filter add dev src egress bpf da egress_host */
+	XGRESS_FILTER_ADD(&qdisc_src, BPF_TC_EGRESS, skel->progs.egress_host, 0);
 	close_netns(nstoken);
 
 	/* setup ns_dst tc progs */
 	nstoken = open_netns(NS_DST);
 	if (!ASSERT_OK_PTR(nstoken, "setns " NS_DST))
 		return -1;
-	/* tc qdisc add dev veth_dst clsact */
-	QDISC_CLSACT_CREATE(&qdisc_veth_dst, setup_result->ifindex_veth_dst);
-	/* tc filter add dev veth_dst ingress bpf da ingress_host */
-	XGRESS_FILTER_ADD(&qdisc_veth_dst, BPF_TC_INGRESS, skel->progs.ingress_host, 0);
-	/* tc filter add dev veth_dst egress bpf da egress_host */
-	XGRESS_FILTER_ADD(&qdisc_veth_dst, BPF_TC_EGRESS, skel->progs.egress_host, 0);
+	/* tc qdisc add dev dst clsact */
+	QDISC_CLSACT_CREATE(&qdisc_dst, setup_result->ifindex_dst);
+	/* tc filter add dev dst ingress bpf da ingress_host */
+	XGRESS_FILTER_ADD(&qdisc_dst, BPF_TC_INGRESS, skel->progs.ingress_host, 0);
+	/* tc filter add dev dst egress bpf da egress_host */
+	XGRESS_FILTER_ADD(&qdisc_dst, BPF_TC_EGRESS, skel->progs.egress_host, 0);
 	close_netns(nstoken);
 
 	/* setup ns_fwd tc progs */
 	nstoken = open_netns(NS_FWD);
 	if (!ASSERT_OK_PTR(nstoken, "setns " NS_FWD))
 		return -1;
-	/* tc qdisc add dev veth_dst_fwd clsact */
-	QDISC_CLSACT_CREATE(&qdisc_veth_dst_fwd, setup_result->ifindex_veth_dst_fwd);
-	/* tc filter add dev veth_dst_fwd ingress prio 100 bpf da ingress_fwdns_prio100 */
-	XGRESS_FILTER_ADD(&qdisc_veth_dst_fwd, BPF_TC_INGRESS,
+	/* tc qdisc add dev dst_fwd clsact */
+	QDISC_CLSACT_CREATE(&qdisc_dst_fwd, setup_result->ifindex_dst_fwd);
+	/* tc filter add dev dst_fwd ingress prio 100 bpf da ingress_fwdns_prio100 */
+	XGRESS_FILTER_ADD(&qdisc_dst_fwd, BPF_TC_INGRESS,
 			  skel->progs.ingress_fwdns_prio100, 100);
-	/* tc filter add dev veth_dst_fwd ingress prio 101 bpf da ingress_fwdns_prio101 */
-	XGRESS_FILTER_ADD(&qdisc_veth_dst_fwd, BPF_TC_INGRESS,
+	/* tc filter add dev dst_fwd ingress prio 101 bpf da ingress_fwdns_prio101 */
+	XGRESS_FILTER_ADD(&qdisc_dst_fwd, BPF_TC_INGRESS,
 			  skel->progs.ingress_fwdns_prio101, 101);
-	/* tc filter add dev veth_dst_fwd egress prio 100 bpf da egress_fwdns_prio100 */
-	XGRESS_FILTER_ADD(&qdisc_veth_dst_fwd, BPF_TC_EGRESS,
+	/* tc filter add dev dst_fwd egress prio 100 bpf da egress_fwdns_prio100 */
+	XGRESS_FILTER_ADD(&qdisc_dst_fwd, BPF_TC_EGRESS,
 			  skel->progs.egress_fwdns_prio100, 100);
-	/* tc filter add dev veth_dst_fwd egress prio 101 bpf da egress_fwdns_prio101 */
-	XGRESS_FILTER_ADD(&qdisc_veth_dst_fwd, BPF_TC_EGRESS,
+	/* tc filter add dev dst_fwd egress prio 101 bpf da egress_fwdns_prio101 */
+	XGRESS_FILTER_ADD(&qdisc_dst_fwd, BPF_TC_EGRESS,
 			  skel->progs.egress_fwdns_prio101, 101);
 
-	/* tc qdisc add dev veth_src_fwd clsact */
-	QDISC_CLSACT_CREATE(&qdisc_veth_src_fwd, setup_result->ifindex_veth_src_fwd);
-	/* tc filter add dev veth_src_fwd ingress prio 100 bpf da ingress_fwdns_prio100 */
-	XGRESS_FILTER_ADD(&qdisc_veth_src_fwd, BPF_TC_INGRESS,
+	/* tc qdisc add dev src_fwd clsact */
+	QDISC_CLSACT_CREATE(&qdisc_src_fwd, setup_result->ifindex_src_fwd);
+	/* tc filter add dev src_fwd ingress prio 100 bpf da ingress_fwdns_prio100 */
+	XGRESS_FILTER_ADD(&qdisc_src_fwd, BPF_TC_INGRESS,
 			  skel->progs.ingress_fwdns_prio100, 100);
-	/* tc filter add dev veth_src_fwd ingress prio 101 bpf da ingress_fwdns_prio101 */
-	XGRESS_FILTER_ADD(&qdisc_veth_src_fwd, BPF_TC_INGRESS,
+	/* tc filter add dev src_fwd ingress prio 101 bpf da ingress_fwdns_prio101 */
+	XGRESS_FILTER_ADD(&qdisc_src_fwd, BPF_TC_INGRESS,
 			  skel->progs.ingress_fwdns_prio101, 101);
-	/* tc filter add dev veth_src_fwd egress prio 100 bpf da egress_fwdns_prio100 */
-	XGRESS_FILTER_ADD(&qdisc_veth_src_fwd, BPF_TC_EGRESS,
+	/* tc filter add dev src_fwd egress prio 100 bpf da egress_fwdns_prio100 */
+	XGRESS_FILTER_ADD(&qdisc_src_fwd, BPF_TC_EGRESS,
 			  skel->progs.egress_fwdns_prio100, 100);
-	/* tc filter add dev veth_src_fwd egress prio 101 bpf da egress_fwdns_prio101 */
-	XGRESS_FILTER_ADD(&qdisc_veth_src_fwd, BPF_TC_EGRESS,
+	/* tc filter add dev src_fwd egress prio 101 bpf da egress_fwdns_prio101 */
+	XGRESS_FILTER_ADD(&qdisc_src_fwd, BPF_TC_EGRESS,
 			  skel->progs.egress_fwdns_prio101, 101);
 	close_netns(nstoken);
 	return 0;
@@ -777,8 +838,8 @@ static void test_tc_redirect_dtime(struct netns_setup_result *setup_result)
 	if (!ASSERT_OK_PTR(skel, "test_tc_dtime__open"))
 		return;
 
-	skel->rodata->IFINDEX_SRC = setup_result->ifindex_veth_src_fwd;
-	skel->rodata->IFINDEX_DST = setup_result->ifindex_veth_dst_fwd;
+	skel->rodata->IFINDEX_SRC = setup_result->ifindex_src_fwd;
+	skel->rodata->IFINDEX_DST = setup_result->ifindex_dst_fwd;
 
 	err = test_tc_dtime__load(skel);
 	if (!ASSERT_OK(err, "test_tc_dtime__load"))
@@ -868,8 +929,8 @@ static void test_tc_redirect_neigh(struct netns_setup_result *setup_result)
 	if (!ASSERT_OK_PTR(skel, "test_tc_neigh__open"))
 		goto done;
 
-	skel->rodata->IFINDEX_SRC = setup_result->ifindex_veth_src_fwd;
-	skel->rodata->IFINDEX_DST = setup_result->ifindex_veth_dst_fwd;
+	skel->rodata->IFINDEX_SRC = setup_result->ifindex_src_fwd;
+	skel->rodata->IFINDEX_DST = setup_result->ifindex_dst_fwd;
 
 	err = test_tc_neigh__load(skel);
 	if (!ASSERT_OK(err, "test_tc_neigh__load"))
@@ -904,8 +965,8 @@ static void test_tc_redirect_peer(struct netns_setup_result *setup_result)
 	if (!ASSERT_OK_PTR(skel, "test_tc_peer__open"))
 		goto done;
 
-	skel->rodata->IFINDEX_SRC = setup_result->ifindex_veth_src_fwd;
-	skel->rodata->IFINDEX_DST = setup_result->ifindex_veth_dst_fwd;
+	skel->rodata->IFINDEX_SRC = setup_result->ifindex_src_fwd;
+	skel->rodata->IFINDEX_DST = setup_result->ifindex_dst_fwd;
 
 	err = test_tc_peer__load(skel);
 	if (!ASSERT_OK(err, "test_tc_peer__load"))
@@ -996,7 +1057,7 @@ static int tun_relay_loop(int src_fd, int target_fd)
 static void test_tc_redirect_peer_l3(struct netns_setup_result *setup_result)
 {
 	LIBBPF_OPTS(bpf_tc_hook, qdisc_tun_fwd);
-	LIBBPF_OPTS(bpf_tc_hook, qdisc_veth_dst_fwd);
+	LIBBPF_OPTS(bpf_tc_hook, qdisc_dst_fwd);
 	struct test_tc_peer *skel = NULL;
 	struct nstoken *nstoken = NULL;
 	int err;
@@ -1045,7 +1106,7 @@ static void test_tc_redirect_peer_l3(struct netns_setup_result *setup_result)
 		goto fail;
 
 	skel->rodata->IFINDEX_SRC = ifindex;
-	skel->rodata->IFINDEX_DST = setup_result->ifindex_veth_dst_fwd;
+	skel->rodata->IFINDEX_DST = setup_result->ifindex_dst_fwd;
 
 	err = test_tc_peer__load(skel);
 	if (!ASSERT_OK(err, "test_tc_peer__load"))
@@ -1053,19 +1114,19 @@ static void test_tc_redirect_peer_l3(struct netns_setup_result *setup_result)
 
 	/* Load "tc_src_l3" to the tun_fwd interface to redirect packets
 	 * towards dst, and "tc_dst" to redirect packets
-	 * and "tc_chk" on veth_dst_fwd to drop non-redirected packets.
+	 * and "tc_chk" on dst_fwd to drop non-redirected packets.
 	 */
 	/* tc qdisc add dev tun_fwd clsact */
 	QDISC_CLSACT_CREATE(&qdisc_tun_fwd, ifindex);
 	/* tc filter add dev tun_fwd ingress bpf da tc_src_l3 */
 	XGRESS_FILTER_ADD(&qdisc_tun_fwd, BPF_TC_INGRESS, skel->progs.tc_src_l3, 0);
 
-	/* tc qdisc add dev veth_dst_fwd clsact */
-	QDISC_CLSACT_CREATE(&qdisc_veth_dst_fwd, setup_result->ifindex_veth_dst_fwd);
-	/* tc filter add dev veth_dst_fwd ingress bpf da tc_dst_l3 */
-	XGRESS_FILTER_ADD(&qdisc_veth_dst_fwd, BPF_TC_INGRESS, skel->progs.tc_dst_l3, 0);
-	/* tc filter add dev veth_dst_fwd egress bpf da tc_chk */
-	XGRESS_FILTER_ADD(&qdisc_veth_dst_fwd, BPF_TC_EGRESS, skel->progs.tc_chk, 0);
+	/* tc qdisc add dev dst_fwd clsact */
+	QDISC_CLSACT_CREATE(&qdisc_dst_fwd, setup_result->ifindex_dst_fwd);
+	/* tc filter add dev dst_fwd ingress bpf da tc_dst_l3 */
+	XGRESS_FILTER_ADD(&qdisc_dst_fwd, BPF_TC_INGRESS, skel->progs.tc_dst_l3, 0);
+	/* tc filter add dev dst_fwd egress bpf da tc_chk */
+	XGRESS_FILTER_ADD(&qdisc_dst_fwd, BPF_TC_EGRESS, skel->progs.tc_chk, 0);
 
 	/* Setup route and neigh tables */
 	SYS(fail, "ip -netns " NS_SRC " addr add dev tun_src " IP4_TUN_SRC "/24");
@@ -1074,17 +1135,17 @@ static void test_tc_redirect_peer_l3(struct netns_setup_result *setup_result)
 	SYS(fail, "ip -netns " NS_SRC " addr add dev tun_src " IP6_TUN_SRC "/64 nodad");
 	SYS(fail, "ip -netns " NS_FWD " addr add dev tun_fwd " IP6_TUN_FWD "/64 nodad");
 
-	SYS(fail, "ip -netns " NS_SRC " route del " IP4_DST "/32 dev veth_src scope global");
+	SYS(fail, "ip -netns " NS_SRC " route del " IP4_DST "/32 dev src scope global");
 	SYS(fail, "ip -netns " NS_SRC " route add " IP4_DST "/32 via " IP4_TUN_FWD
 	    " dev tun_src scope global");
-	SYS(fail, "ip -netns " NS_DST " route add " IP4_TUN_SRC "/32 dev veth_dst scope global");
-	SYS(fail, "ip -netns " NS_SRC " route del " IP6_DST "/128 dev veth_src scope global");
+	SYS(fail, "ip -netns " NS_DST " route add " IP4_TUN_SRC "/32 dev dst scope global");
+	SYS(fail, "ip -netns " NS_SRC " route del " IP6_DST "/128 dev src scope global");
 	SYS(fail, "ip -netns " NS_SRC " route add " IP6_DST "/128 via " IP6_TUN_FWD
 	    " dev tun_src scope global");
-	SYS(fail, "ip -netns " NS_DST " route add " IP6_TUN_SRC "/128 dev veth_dst scope global");
+	SYS(fail, "ip -netns " NS_DST " route add " IP6_TUN_SRC "/128 dev dst scope global");
 
-	SYS(fail, "ip -netns " NS_DST " neigh add " IP4_TUN_SRC " dev veth_dst lladdr " MAC_DST_FWD);
-	SYS(fail, "ip -netns " NS_DST " neigh add " IP6_TUN_SRC " dev veth_dst lladdr " MAC_DST_FWD);
+	SYS(fail, "ip -netns " NS_DST " neigh add " IP4_TUN_SRC " dev dst lladdr " MAC_DST_FWD);
+	SYS(fail, "ip -netns " NS_DST " neigh add " IP6_TUN_SRC " dev dst lladdr " MAC_DST_FWD);
 
 	if (!ASSERT_OK(set_forwarding(false), "disable forwarding"))
 		goto fail;
@@ -1106,9 +1167,9 @@ static void test_tc_redirect_peer_l3(struct netns_setup_result *setup_result)
 		close_netns(nstoken);
 }
 
-#define RUN_TEST(name)                                                                      \
+#define RUN_TEST(name, mode)                                                                \
 	({                                                                                  \
-		struct netns_setup_result setup_result;                                     \
+		struct netns_setup_result setup_result = { .dev_mode = mode, };             \
 		if (test__start_subtest(#name))                                             \
 			if (ASSERT_OK(netns_setup_namespaces("add"), "setup namespaces")) { \
 				if (ASSERT_OK(netns_setup_links_and_routes(&setup_result),  \
@@ -1122,11 +1183,13 @@ static void *test_tc_redirect_run_tests(void *arg)
 {
 	netns_setup_namespaces_nofail("delete");
 
-	RUN_TEST(tc_redirect_peer);
-	RUN_TEST(tc_redirect_peer_l3);
-	RUN_TEST(tc_redirect_neigh);
-	RUN_TEST(tc_redirect_neigh_fib);
-	RUN_TEST(tc_redirect_dtime);
+	RUN_TEST(tc_redirect_peer, MODE_VETH);
+	RUN_TEST(tc_redirect_peer, MODE_NETKIT);
+	RUN_TEST(tc_redirect_peer_l3, MODE_VETH);
+	RUN_TEST(tc_redirect_peer_l3, MODE_NETKIT);
+	RUN_TEST(tc_redirect_neigh, MODE_VETH);
+	RUN_TEST(tc_redirect_neigh_fib, MODE_VETH);
+	RUN_TEST(tc_redirect_dtime, MODE_VETH);
 	return NULL;
 }
 
diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c
index e5c61aa6604a..5cfa7a6316b6 100644
--- a/tools/testing/selftests/bpf/prog_tests/verifier.c
+++ b/tools/testing/selftests/bpf/prog_tests/verifier.c
@@ -31,6 +31,7 @@
 #include "verifier_helper_restricted.skel.h"
 #include "verifier_helper_value_access.skel.h"
 #include "verifier_int_ptr.skel.h"
+#include "verifier_iterating_callbacks.skel.h"
 #include "verifier_jeq_infer_not_null.skel.h"
 #include "verifier_ld_ind.skel.h"
 #include "verifier_ldsx.skel.h"
@@ -139,6 +140,7 @@ void test_verifier_helper_packet_access(void) { RUN(verifier_helper_packet_acces
 void test_verifier_helper_restricted(void)    { RUN(verifier_helper_restricted); }
 void test_verifier_helper_value_access(void)  { RUN(verifier_helper_value_access); }
 void test_verifier_int_ptr(void)              { RUN(verifier_int_ptr); }
+void test_verifier_iterating_callbacks(void)  { RUN(verifier_iterating_callbacks); }
 void test_verifier_jeq_infer_not_null(void)   { RUN(verifier_jeq_infer_not_null); }
 void test_verifier_ld_ind(void)               { RUN(verifier_ld_ind); }
 void test_verifier_ldsx(void)                  { RUN(verifier_ldsx); }
diff --git a/tools/testing/selftests/bpf/prog_tests/vmlinux.c b/tools/testing/selftests/bpf/prog_tests/vmlinux.c
index 72310cfc6474..6fb2217d940b 100644
--- a/tools/testing/selftests/bpf/prog_tests/vmlinux.c
+++ b/tools/testing/selftests/bpf/prog_tests/vmlinux.c
@@ -16,27 +16,27 @@ static void nsleep()
 
 void test_vmlinux(void)
 {
-	int duration = 0, err;
+	int err;
 	struct test_vmlinux* skel;
 	struct test_vmlinux__bss *bss;
 
 	skel = test_vmlinux__open_and_load();
-	if (CHECK(!skel, "skel_open", "failed to open skeleton\n"))
+	if (!ASSERT_OK_PTR(skel, "test_vmlinux__open_and_load"))
 		return;
 	bss = skel->bss;
 
 	err = test_vmlinux__attach(skel);
-	if (CHECK(err, "skel_attach", "skeleton attach failed: %d\n", err))
+	if (!ASSERT_OK(err, "test_vmlinux__attach"))
 		goto cleanup;
 
 	/* trigger everything */
 	nsleep();
 
-	CHECK(!bss->tp_called, "tp", "not called\n");
-	CHECK(!bss->raw_tp_called, "raw_tp", "not called\n");
-	CHECK(!bss->tp_btf_called, "tp_btf", "not called\n");
-	CHECK(!bss->kprobe_called, "kprobe", "not called\n");
-	CHECK(!bss->fentry_called, "fentry", "not called\n");
+	ASSERT_TRUE(bss->tp_called, "tp");
+	ASSERT_TRUE(bss->raw_tp_called, "raw_tp");
+	ASSERT_TRUE(bss->tp_btf_called, "tp_btf");
+	ASSERT_TRUE(bss->kprobe_called, "kprobe");
+	ASSERT_TRUE(bss->fentry_called, "fentry");
 
 cleanup:
 	test_vmlinux__destroy(skel);
diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_task_stack.c b/tools/testing/selftests/bpf/progs/bpf_iter_task_stack.c
index f2b8167b72a8..442f4ca39fd7 100644
--- a/tools/testing/selftests/bpf/progs/bpf_iter_task_stack.c
+++ b/tools/testing/selftests/bpf/progs/bpf_iter_task_stack.c
@@ -35,6 +35,8 @@ int dump_task_stack(struct bpf_iter__task *ctx)
 	return 0;
 }
 
+int num_user_stacks = 0;
+
 SEC("iter/task")
 int get_task_user_stacks(struct bpf_iter__task *ctx)
 {
@@ -51,6 +53,9 @@ int get_task_user_stacks(struct bpf_iter__task *ctx)
 	if (res <= 0)
 		return 0;
 
+	/* Only one task, the current one, should succeed */
+	++num_user_stacks;
+
 	buf_sz += res;
 
 	/* If the verifier doesn't refine bpf_get_task_stack res, and instead
diff --git a/tools/testing/selftests/bpf/progs/bpf_loop_bench.c b/tools/testing/selftests/bpf/progs/bpf_loop_bench.c
index 4ce76eb064c4..d461746fd3c1 100644
--- a/tools/testing/selftests/bpf/progs/bpf_loop_bench.c
+++ b/tools/testing/selftests/bpf/progs/bpf_loop_bench.c
@@ -15,13 +15,16 @@ static int empty_callback(__u32 index, void *data)
 	return 0;
 }
 
+static int outer_loop(__u32 index, void *data)
+{
+	bpf_loop(nr_loops, empty_callback, NULL, 0);
+	__sync_add_and_fetch(&hits, nr_loops);
+	return 0;
+}
+
 SEC("fentry/" SYS_PREFIX "sys_getpgid")
 int benchmark(void *ctx)
 {
-	for (int i = 0; i < 1000; i++) {
-		bpf_loop(nr_loops, empty_callback, NULL, 0);
-
-		__sync_add_and_fetch(&hits, nr_loops);
-	}
+	bpf_loop(1000, outer_loop, NULL, 0);
 	return 0;
 }
diff --git a/tools/testing/selftests/bpf/progs/cb_refs.c b/tools/testing/selftests/bpf/progs/cb_refs.c
index 76d661b20e87..56c764df8196 100644
--- a/tools/testing/selftests/bpf/progs/cb_refs.c
+++ b/tools/testing/selftests/bpf/progs/cb_refs.c
@@ -33,6 +33,7 @@ int underflow_prog(void *ctx)
 	if (!p)
 		return 0;
 	bpf_for_each_map_elem(&array_map, cb1, &p, 0);
+	bpf_kfunc_call_test_release(p);
 	return 0;
 }
 
diff --git a/tools/testing/selftests/bpf/progs/exceptions_assert.c b/tools/testing/selftests/bpf/progs/exceptions_assert.c
index e1e5c54a6a11..49efaed143fc 100644
--- a/tools/testing/selftests/bpf/progs/exceptions_assert.c
+++ b/tools/testing/selftests/bpf/progs/exceptions_assert.c
@@ -18,48 +18,48 @@
 		return *(u64 *)num;					\
 	}
 
-__msg(": R0_w=-2147483648 R10=fp0")
+__msg(": R0_w=0xffffffff80000000 R10=fp0")
 check_assert(s64, eq, int_min, INT_MIN);
-__msg(": R0_w=2147483647 R10=fp0")
+__msg(": R0_w=0x7fffffff R10=fp0")
 check_assert(s64, eq, int_max, INT_MAX);
 __msg(": R0_w=0 R10=fp0")
 check_assert(s64, eq, zero, 0);
-__msg(": R0_w=-9223372036854775808 R1_w=-9223372036854775808 R10=fp0")
+__msg(": R0_w=0x8000000000000000 R1_w=0x8000000000000000 R10=fp0")
 check_assert(s64, eq, llong_min, LLONG_MIN);
-__msg(": R0_w=9223372036854775807 R1_w=9223372036854775807 R10=fp0")
+__msg(": R0_w=0x7fffffffffffffff R1_w=0x7fffffffffffffff R10=fp0")
 check_assert(s64, eq, llong_max, LLONG_MAX);
 
-__msg(": R0_w=scalar(smax=2147483646) R10=fp0")
+__msg(": R0_w=scalar(smax=0x7ffffffe) R10=fp0")
 check_assert(s64, lt, pos, INT_MAX);
-__msg(": R0_w=scalar(smax=-1,umin=9223372036854775808,var_off=(0x8000000000000000; 0x7fffffffffffffff))")
+__msg(": R0_w=scalar(smax=-1,umin=0x8000000000000000,var_off=(0x8000000000000000; 0x7fffffffffffffff))")
 check_assert(s64, lt, zero, 0);
-__msg(": R0_w=scalar(smax=-2147483649,umin=9223372036854775808,umax=18446744071562067967,var_off=(0x8000000000000000; 0x7fffffffffffffff))")
+__msg(": R0_w=scalar(smax=0xffffffff7fffffff,umin=0x8000000000000000,umax=0xffffffff7fffffff,var_off=(0x8000000000000000; 0x7fffffffffffffff))")
 check_assert(s64, lt, neg, INT_MIN);
 
-__msg(": R0_w=scalar(smax=2147483647) R10=fp0")
+__msg(": R0_w=scalar(smax=0x7fffffff) R10=fp0")
 check_assert(s64, le, pos, INT_MAX);
 __msg(": R0_w=scalar(smax=0) R10=fp0")
 check_assert(s64, le, zero, 0);
-__msg(": R0_w=scalar(smax=-2147483648,umin=9223372036854775808,umax=18446744071562067968,var_off=(0x8000000000000000; 0x7fffffffffffffff))")
+__msg(": R0_w=scalar(smax=0xffffffff80000000,umin=0x8000000000000000,umax=0xffffffff80000000,var_off=(0x8000000000000000; 0x7fffffffffffffff))")
 check_assert(s64, le, neg, INT_MIN);
 
-__msg(": R0_w=scalar(smin=umin=2147483648,umax=9223372036854775807,var_off=(0x0; 0x7fffffffffffffff))")
+__msg(": R0_w=scalar(smin=umin=0x80000000,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))")
 check_assert(s64, gt, pos, INT_MAX);
-__msg(": R0_w=scalar(smin=umin=1,umax=9223372036854775807,var_off=(0x0; 0x7fffffffffffffff))")
+__msg(": R0_w=scalar(smin=umin=1,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))")
 check_assert(s64, gt, zero, 0);
-__msg(": R0_w=scalar(smin=-2147483647) R10=fp0")
+__msg(": R0_w=scalar(smin=0xffffffff80000001) R10=fp0")
 check_assert(s64, gt, neg, INT_MIN);
 
-__msg(": R0_w=scalar(smin=umin=2147483647,umax=9223372036854775807,var_off=(0x0; 0x7fffffffffffffff))")
+__msg(": R0_w=scalar(smin=umin=0x7fffffff,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff))")
 check_assert(s64, ge, pos, INT_MAX);
-__msg(": R0_w=scalar(smin=0,umax=9223372036854775807,var_off=(0x0; 0x7fffffffffffffff)) R10=fp0")
+__msg(": R0_w=scalar(smin=0,umax=0x7fffffffffffffff,var_off=(0x0; 0x7fffffffffffffff)) R10=fp0")
 check_assert(s64, ge, zero, 0);
-__msg(": R0_w=scalar(smin=-2147483648) R10=fp0")
+__msg(": R0_w=scalar(smin=0xffffffff80000000) R10=fp0")
 check_assert(s64, ge, neg, INT_MIN);
 
 SEC("?tc")
 __log_level(2) __failure
-__msg(": R0=0 R1=ctx(off=0,imm=0) R2=scalar(smin=smin32=-2147483646,smax=smax32=2147483645) R10=fp0")
+__msg(": R0=0 R1=ctx() R2=scalar(smin=0xffffffff80000002,smax=smax32=0x7ffffffd,smin32=0x80000002) R10=fp0")
 int check_assert_range_s64(struct __sk_buff *ctx)
 {
 	struct bpf_sock *sk = ctx->sk;
@@ -75,7 +75,7 @@ int check_assert_range_s64(struct __sk_buff *ctx)
 
 SEC("?tc")
 __log_level(2) __failure
-__msg(": R1=ctx(off=0,imm=0) R2=scalar(smin=umin=smin32=umin32=4096,smax=umax=smax32=umax32=8192,var_off=(0x0; 0x3fff))")
+__msg(": R1=ctx() R2=scalar(smin=umin=smin32=umin32=4096,smax=umax=smax32=umax32=8192,var_off=(0x0; 0x3fff))")
 int check_assert_range_u64(struct __sk_buff *ctx)
 {
 	u64 num = ctx->len;
@@ -86,7 +86,7 @@ int check_assert_range_u64(struct __sk_buff *ctx)
 
 SEC("?tc")
 __log_level(2) __failure
-__msg(": R0=0 R1=ctx(off=0,imm=0) R2=4096 R10=fp0")
+__msg(": R0=0 R1=ctx() R2=4096 R10=fp0")
 int check_assert_single_range_s64(struct __sk_buff *ctx)
 {
 	struct bpf_sock *sk = ctx->sk;
@@ -103,7 +103,7 @@ int check_assert_single_range_s64(struct __sk_buff *ctx)
 
 SEC("?tc")
 __log_level(2) __failure
-__msg(": R1=ctx(off=0,imm=0) R2=4096 R10=fp0")
+__msg(": R1=ctx() R2=4096 R10=fp0")
 int check_assert_single_range_u64(struct __sk_buff *ctx)
 {
 	u64 num = ctx->len;
@@ -114,7 +114,7 @@ int check_assert_single_range_u64(struct __sk_buff *ctx)
 
 SEC("?tc")
 __log_level(2) __failure
-__msg(": R1=pkt(off=64,r=64,imm=0) R2=pkt_end(off=0,imm=0) R6=pkt(off=0,r=64,imm=0) R10=fp0")
+__msg(": R1=pkt(off=64,r=64) R2=pkt_end() R6=pkt(r=64) R10=fp0")
 int check_assert_generic(struct __sk_buff *ctx)
 {
 	u8 *data_end = (void *)(long)ctx->data_end;
diff --git a/tools/testing/selftests/bpf/progs/exceptions_fail.c b/tools/testing/selftests/bpf/progs/exceptions_fail.c
index 4c39e920dac2..8c0ef2742208 100644
--- a/tools/testing/selftests/bpf/progs/exceptions_fail.c
+++ b/tools/testing/selftests/bpf/progs/exceptions_fail.c
@@ -171,6 +171,7 @@ int reject_with_rbtree_add_throw(void *ctx)
 		return 0;
 	bpf_spin_lock(&lock);
 	bpf_rbtree_add(&rbtree, &f->node, rbless);
+	bpf_spin_unlock(&lock);
 	return 0;
 }
 
@@ -214,6 +215,7 @@ int reject_with_cb_reference(void *ctx)
 	if (!f)
 		return 0;
 	bpf_loop(5, subprog_cb_ref, NULL, 0);
+	bpf_obj_drop(f);
 	return 0;
 }
 
diff --git a/tools/testing/selftests/bpf/progs/iters.c b/tools/testing/selftests/bpf/progs/iters.c
index c20c4e38b71c..b2181f850d3e 100644
--- a/tools/testing/selftests/bpf/progs/iters.c
+++ b/tools/testing/selftests/bpf/progs/iters.c
@@ -1411,4 +1411,26 @@ __naked int checkpoint_states_deletion(void)
 	);
 }
 
+struct {
+	int data[32];
+	int n;
+} loop_data;
+
+SEC("raw_tp")
+__success
+int iter_arr_with_actual_elem_count(const void *ctx)
+{
+	int i, n = loop_data.n, sum = 0;
+
+	if (n > ARRAY_SIZE(loop_data.data))
+		return 0;
+
+	bpf_for(i, 0, n) {
+		/* no rechecking of i against ARRAY_SIZE(loop_data.n) */
+		sum += loop_data.data[i];
+	}
+
+	return sum;
+}
+
 char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/local_kptr_stash.c b/tools/testing/selftests/bpf/progs/local_kptr_stash.c
index b567a666d2b8..1769fdff6aea 100644
--- a/tools/testing/selftests/bpf/progs/local_kptr_stash.c
+++ b/tools/testing/selftests/bpf/progs/local_kptr_stash.c
@@ -14,6 +14,24 @@ struct node_data {
 	struct bpf_rb_node node;
 };
 
+struct refcounted_node {
+	long data;
+	struct bpf_rb_node rb_node;
+	struct bpf_refcount refcount;
+};
+
+struct stash {
+	struct bpf_spin_lock l;
+	struct refcounted_node __kptr *stashed;
+};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_ARRAY);
+	__type(key, int);
+	__type(value, struct stash);
+	__uint(max_entries, 10);
+} refcounted_node_stash SEC(".maps");
+
 struct plain_local {
 	long key;
 	long data;
@@ -38,6 +56,7 @@ struct map_value {
  * Had to do the same w/ bpf_kfunc_call_test_release below
  */
 struct node_data *just_here_because_btf_bug;
+struct refcounted_node *just_here_because_btf_bug2;
 
 struct {
 	__uint(type, BPF_MAP_TYPE_ARRAY);
@@ -132,4 +151,56 @@ long stash_test_ref_kfunc(void *ctx)
 	return 0;
 }
 
+SEC("tc")
+long refcount_acquire_without_unstash(void *ctx)
+{
+	struct refcounted_node *p;
+	struct stash *s;
+	int ret = 0;
+
+	s = bpf_map_lookup_elem(&refcounted_node_stash, &ret);
+	if (!s)
+		return 1;
+
+	if (!s->stashed)
+		/* refcount_acquire failure is expected when no refcounted_node
+		 * has been stashed before this program executes
+		 */
+		return 2;
+
+	p = bpf_refcount_acquire(s->stashed);
+	if (!p)
+		return 3;
+
+	ret = s->stashed ? s->stashed->data : -1;
+	bpf_obj_drop(p);
+	return ret;
+}
+
+/* Helper for refcount_acquire_without_unstash test */
+SEC("tc")
+long stash_refcounted_node(void *ctx)
+{
+	struct refcounted_node *p;
+	struct stash *s;
+	int key = 0;
+
+	s = bpf_map_lookup_elem(&refcounted_node_stash, &key);
+	if (!s)
+		return 1;
+
+	p = bpf_obj_new(typeof(*p));
+	if (!p)
+		return 2;
+	p->data = 42;
+
+	p = bpf_kptr_xchg(&s->stashed, p);
+	if (p) {
+		bpf_obj_drop(p);
+		return 3;
+	}
+
+	return 0;
+}
+
 char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/pyperf180.c b/tools/testing/selftests/bpf/progs/pyperf180.c
index c39f559d3100..42c4a8b62e36 100644
--- a/tools/testing/selftests/bpf/progs/pyperf180.c
+++ b/tools/testing/selftests/bpf/progs/pyperf180.c
@@ -1,4 +1,26 @@
 // SPDX-License-Identifier: GPL-2.0
 // Copyright (c) 2019 Facebook
 #define STACK_MAX_LEN 180
+
+/* llvm upstream commit at clang18
+ *   https://github.com/llvm/llvm-project/commit/1a2e77cf9e11dbf56b5720c607313a566eebb16e
+ * changed inlining behavior and caused compilation failure as some branch
+ * target distance exceeded 16bit representation which is the maximum for
+ * cpu v1/v2/v3. Macro __BPF_CPU_VERSION__ is later implemented in clang18
+ * to specify which cpu version is used for compilation. So a smaller
+ * unroll_count can be set if __BPF_CPU_VERSION__ is less than 4, which
+ * reduced some branch target distances and resolved the compilation failure.
+ *
+ * To capture the case where a developer/ci uses clang18 but the corresponding
+ * repo checkpoint does not have __BPF_CPU_VERSION__, a smaller unroll_count
+ * will be set as well to prevent potential compilation failures.
+ */
+#ifdef __BPF_CPU_VERSION__
+#if __BPF_CPU_VERSION__ < 4
+#define UNROLL_COUNT 90
+#endif
+#elif __clang_major__ == 18
+#define UNROLL_COUNT 90
+#endif
+
 #include "pyperf.h"
diff --git a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c
index 1ef07f6ee580..1553b9c16aa7 100644
--- a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c
+++ b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c
@@ -53,6 +53,25 @@ long rbtree_refcounted_node_ref_escapes(void *ctx)
 	return 0;
 }
 
+SEC("?tc")
+__failure __msg("Possibly NULL pointer passed to trusted arg0")
+long refcount_acquire_maybe_null(void *ctx)
+{
+	struct node_acquire *n, *m;
+
+	n = bpf_obj_new(typeof(*n));
+	/* Intentionally not testing !n
+	 * it's MAYBE_NULL for refcount_acquire
+	 */
+	m = bpf_refcount_acquire(n);
+	if (m)
+		bpf_obj_drop(m);
+	if (n)
+		bpf_obj_drop(n);
+
+	return 0;
+}
+
 SEC("?tc")
 __failure __msg("Unreleased reference id=3 alloc_insn=9")
 long rbtree_refcounted_node_ref_escapes_owning_input(void *ctx)
diff --git a/tools/testing/selftests/bpf/progs/strobemeta.h b/tools/testing/selftests/bpf/progs/strobemeta.h
index e02cfd380746..40df2cc26eaf 100644
--- a/tools/testing/selftests/bpf/progs/strobemeta.h
+++ b/tools/testing/selftests/bpf/progs/strobemeta.h
@@ -24,9 +24,11 @@ struct task_struct {};
 #define STACK_TABLE_EPOCH_SHIFT 20
 #define STROBE_MAX_STR_LEN 1
 #define STROBE_MAX_CFGS 32
+#define READ_MAP_VAR_PAYLOAD_CAP					\
+	((1 + STROBE_MAX_MAP_ENTRIES * 2) * STROBE_MAX_STR_LEN)
 #define STROBE_MAX_PAYLOAD						\
 	(STROBE_MAX_STRS * STROBE_MAX_STR_LEN +				\
-	STROBE_MAX_MAPS * (1 + STROBE_MAX_MAP_ENTRIES * 2) * STROBE_MAX_STR_LEN)
+	 STROBE_MAX_MAPS * READ_MAP_VAR_PAYLOAD_CAP)
 
 struct strobe_value_header {
 	/*
@@ -355,7 +357,7 @@ static __always_inline uint64_t read_str_var(struct strobemeta_cfg *cfg,
 					     size_t idx, void *tls_base,
 					     struct strobe_value_generic *value,
 					     struct strobemeta_payload *data,
-					     void *payload)
+					     size_t off)
 {
 	void *location;
 	uint64_t len;
@@ -366,7 +368,7 @@ static __always_inline uint64_t read_str_var(struct strobemeta_cfg *cfg,
 		return 0;
 
 	bpf_probe_read_user(value, sizeof(struct strobe_value_generic), location);
-	len = bpf_probe_read_user_str(payload, STROBE_MAX_STR_LEN, value->ptr);
+	len = bpf_probe_read_user_str(&data->payload[off], STROBE_MAX_STR_LEN, value->ptr);
 	/*
 	 * if bpf_probe_read_user_str returns error (<0), due to casting to
 	 * unsinged int, it will become big number, so next check is
@@ -378,14 +380,14 @@ static __always_inline uint64_t read_str_var(struct strobemeta_cfg *cfg,
 		return 0;
 
 	data->str_lens[idx] = len;
-	return len;
+	return off + len;
 }
 
-static __always_inline void *read_map_var(struct strobemeta_cfg *cfg,
-					  size_t idx, void *tls_base,
-					  struct strobe_value_generic *value,
-					  struct strobemeta_payload *data,
-					  void *payload)
+static __always_inline uint64_t read_map_var(struct strobemeta_cfg *cfg,
+					     size_t idx, void *tls_base,
+					     struct strobe_value_generic *value,
+					     struct strobemeta_payload *data,
+					     size_t off)
 {
 	struct strobe_map_descr* descr = &data->map_descrs[idx];
 	struct strobe_map_raw map;
@@ -397,11 +399,11 @@ static __always_inline void *read_map_var(struct strobemeta_cfg *cfg,
 
 	location = calc_location(&cfg->map_locs[idx], tls_base);
 	if (!location)
-		return payload;
+		return off;
 
 	bpf_probe_read_user(value, sizeof(struct strobe_value_generic), location);
 	if (bpf_probe_read_user(&map, sizeof(struct strobe_map_raw), value->ptr))
-		return payload;
+		return off;
 
 	descr->id = map.id;
 	descr->cnt = map.cnt;
@@ -410,10 +412,10 @@ static __always_inline void *read_map_var(struct strobemeta_cfg *cfg,
 		data->req_meta_valid = 1;
 	}
 
-	len = bpf_probe_read_user_str(payload, STROBE_MAX_STR_LEN, map.tag);
+	len = bpf_probe_read_user_str(&data->payload[off], STROBE_MAX_STR_LEN, map.tag);
 	if (len <= STROBE_MAX_STR_LEN) {
 		descr->tag_len = len;
-		payload += len;
+		off += len;
 	}
 
 #ifdef NO_UNROLL
@@ -426,22 +428,22 @@ static __always_inline void *read_map_var(struct strobemeta_cfg *cfg,
 			break;
 
 		descr->key_lens[i] = 0;
-		len = bpf_probe_read_user_str(payload, STROBE_MAX_STR_LEN,
+		len = bpf_probe_read_user_str(&data->payload[off], STROBE_MAX_STR_LEN,
 					      map.entries[i].key);
 		if (len <= STROBE_MAX_STR_LEN) {
 			descr->key_lens[i] = len;
-			payload += len;
+			off += len;
 		}
 		descr->val_lens[i] = 0;
-		len = bpf_probe_read_user_str(payload, STROBE_MAX_STR_LEN,
+		len = bpf_probe_read_user_str(&data->payload[off], STROBE_MAX_STR_LEN,
 					      map.entries[i].val);
 		if (len <= STROBE_MAX_STR_LEN) {
 			descr->val_lens[i] = len;
-			payload += len;
+			off += len;
 		}
 	}
 
-	return payload;
+	return off;
 }
 
 #ifdef USE_BPF_LOOP
@@ -455,14 +457,20 @@ struct read_var_ctx {
 	struct strobemeta_payload *data;
 	void *tls_base;
 	struct strobemeta_cfg *cfg;
-	void *payload;
+	size_t payload_off;
 	/* value gets mutated */
 	struct strobe_value_generic *value;
 	enum read_type type;
 };
 
-static int read_var_callback(__u32 index, struct read_var_ctx *ctx)
+static int read_var_callback(__u64 index, struct read_var_ctx *ctx)
 {
+	/* lose precision info for ctx->payload_off, verifier won't track
+	 * double xor, barrier_var() is needed to force clang keep both xors.
+	 */
+	ctx->payload_off ^= index;
+	barrier_var(ctx->payload_off);
+	ctx->payload_off ^= index;
 	switch (ctx->type) {
 	case READ_INT_VAR:
 		if (index >= STROBE_MAX_INTS)
@@ -472,14 +480,18 @@ static int read_var_callback(__u32 index, struct read_var_ctx *ctx)
 	case READ_MAP_VAR:
 		if (index >= STROBE_MAX_MAPS)
 			return 1;
-		ctx->payload = read_map_var(ctx->cfg, index, ctx->tls_base,
-					    ctx->value, ctx->data, ctx->payload);
+		if (ctx->payload_off > sizeof(ctx->data->payload) - READ_MAP_VAR_PAYLOAD_CAP)
+			return 1;
+		ctx->payload_off = read_map_var(ctx->cfg, index, ctx->tls_base,
+						ctx->value, ctx->data, ctx->payload_off);
 		break;
 	case READ_STR_VAR:
 		if (index >= STROBE_MAX_STRS)
 			return 1;
-		ctx->payload += read_str_var(ctx->cfg, index, ctx->tls_base,
-					     ctx->value, ctx->data, ctx->payload);
+		if (ctx->payload_off > sizeof(ctx->data->payload) - STROBE_MAX_STR_LEN)
+			return 1;
+		ctx->payload_off = read_str_var(ctx->cfg, index, ctx->tls_base,
+						ctx->value, ctx->data, ctx->payload_off);
 		break;
 	}
 	return 0;
@@ -501,7 +513,8 @@ static void *read_strobe_meta(struct task_struct *task,
 	pid_t pid = bpf_get_current_pid_tgid() >> 32;
 	struct strobe_value_generic value = {0};
 	struct strobemeta_cfg *cfg;
-	void *tls_base, *payload;
+	size_t payload_off;
+	void *tls_base;
 
 	cfg = bpf_map_lookup_elem(&strobemeta_cfgs, &pid);
 	if (!cfg)
@@ -509,7 +522,7 @@ static void *read_strobe_meta(struct task_struct *task,
 
 	data->int_vals_set_mask = 0;
 	data->req_meta_valid = 0;
-	payload = data->payload;
+	payload_off = 0;
 	/*
 	 * we don't have struct task_struct definition, it should be:
 	 * tls_base = (void *)task->thread.fsbase;
@@ -522,7 +535,7 @@ static void *read_strobe_meta(struct task_struct *task,
 		.tls_base = tls_base,
 		.value = &value,
 		.data = data,
-		.payload = payload,
+		.payload_off = 0,
 	};
 	int err;
 
@@ -540,6 +553,11 @@ static void *read_strobe_meta(struct task_struct *task,
 	err = bpf_loop(STROBE_MAX_MAPS, read_var_callback, &ctx, 0);
 	if (err != STROBE_MAX_MAPS)
 		return NULL;
+
+	payload_off = ctx.payload_off;
+	/* this should not really happen, here only to satisfy verifer */
+	if (payload_off > sizeof(data->payload))
+		payload_off = sizeof(data->payload);
 #else
 #ifdef NO_UNROLL
 #pragma clang loop unroll(disable)
@@ -555,7 +573,7 @@ static void *read_strobe_meta(struct task_struct *task,
 #pragma unroll
 #endif /* NO_UNROLL */
 	for (int i = 0; i < STROBE_MAX_STRS; ++i) {
-		payload += read_str_var(cfg, i, tls_base, &value, data, payload);
+		payload_off = read_str_var(cfg, i, tls_base, &value, data, payload_off);
 	}
 #ifdef NO_UNROLL
 #pragma clang loop unroll(disable)
@@ -563,7 +581,7 @@ static void *read_strobe_meta(struct task_struct *task,
 #pragma unroll
 #endif /* NO_UNROLL */
 	for (int i = 0; i < STROBE_MAX_MAPS; ++i) {
-		payload = read_map_var(cfg, i, tls_base, &value, data, payload);
+		payload_off = read_map_var(cfg, i, tls_base, &value, data, payload_off);
 	}
 #endif /* USE_BPF_LOOP */
 
@@ -571,7 +589,7 @@ static void *read_strobe_meta(struct task_struct *task,
 	 * return pointer right after end of payload, so it's possible to
 	 * calculate exact amount of useful data that needs to be sent
 	 */
-	return payload;
+	return &data->payload[payload_off];
 }
 
 SEC("raw_tracepoint/kfree_skb")
diff --git a/tools/testing/selftests/bpf/progs/test_cgroup1_hierarchy.c b/tools/testing/selftests/bpf/progs/test_cgroup1_hierarchy.c
new file mode 100644
index 000000000000..44628865fe1d
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/test_cgroup1_hierarchy.c
@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (C) 2023 Yafang Shao <laoar.shao@gmail.com> */
+
+#include "vmlinux.h"
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include <bpf/bpf_core_read.h>
+
+__u32 target_ancestor_level;
+__u64 target_ancestor_cgid;
+int target_pid, target_hid;
+
+struct cgroup *bpf_task_get_cgroup1(struct task_struct *task, int hierarchy_id) __ksym;
+struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym;
+void bpf_cgroup_release(struct cgroup *cgrp) __ksym;
+
+static int bpf_link_create_verify(int cmd)
+{
+	struct cgroup *cgrp, *ancestor;
+	struct task_struct *task;
+	int ret = 0;
+
+	if (cmd != BPF_LINK_CREATE)
+		return 0;
+
+	task = bpf_get_current_task_btf();
+
+	/* Then it can run in parallel with others */
+	if (task->pid != target_pid)
+		return 0;
+
+	cgrp = bpf_task_get_cgroup1(task, target_hid);
+	if (!cgrp)
+		return 0;
+
+	/* Refuse it if its cgid or its ancestor's cgid is the target cgid */
+	if (cgrp->kn->id == target_ancestor_cgid)
+		ret = -1;
+
+	ancestor = bpf_cgroup_ancestor(cgrp, target_ancestor_level);
+	if (!ancestor)
+		goto out;
+
+	if (ancestor->kn->id == target_ancestor_cgid)
+		ret = -1;
+	bpf_cgroup_release(ancestor);
+
+out:
+	bpf_cgroup_release(cgrp);
+	return ret;
+}
+
+SEC("lsm/bpf")
+int BPF_PROG(lsm_run, int cmd, union bpf_attr *attr, unsigned int size)
+{
+	return bpf_link_create_verify(cmd);
+}
+
+SEC("lsm.s/bpf")
+int BPF_PROG(lsm_s_run, int cmd, union bpf_attr *attr, unsigned int size)
+{
+	return bpf_link_create_verify(cmd);
+}
+
+SEC("fentry")
+int BPF_PROG(fentry_run)
+{
+	return 0;
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds.c b/tools/testing/selftests/bpf/progs/verifier_bounds.c
index c5588a14fe2e..ec430b71730b 100644
--- a/tools/testing/selftests/bpf/progs/verifier_bounds.c
+++ b/tools/testing/selftests/bpf/progs/verifier_bounds.c
@@ -965,6 +965,7 @@ l0_%=:	r0 = 0;						\
 SEC("xdp")
 __description("bound check with JMP_JSLT for crossing 64-bit signed boundary")
 __success __retval(0)
+__flag(!BPF_F_TEST_REG_INVARIANTS) /* known invariants violation */
 __naked void crossing_64_bit_signed_boundary_2(void)
 {
 	asm volatile ("					\
@@ -1046,6 +1047,7 @@ l0_%=:	r0 = 0;						\
 SEC("xdp")
 __description("bound check with JMP32_JSLT for crossing 32-bit signed boundary")
 __success __retval(0)
+__flag(!BPF_F_TEST_REG_INVARIANTS) /* known invariants violation */
 __naked void crossing_32_bit_signed_boundary_2(void)
 {
 	asm volatile ("					\
diff --git a/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c b/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c
new file mode 100644
index 000000000000..5905e036e0ea
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c
@@ -0,0 +1,242 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include "bpf_misc.h"
+
+struct {
+	__uint(type, BPF_MAP_TYPE_ARRAY);
+	__uint(max_entries, 8);
+	__type(key, __u32);
+	__type(value, __u64);
+} map SEC(".maps");
+
+struct {
+	__uint(type, BPF_MAP_TYPE_USER_RINGBUF);
+	__uint(max_entries, 8);
+} ringbuf SEC(".maps");
+
+struct vm_area_struct;
+struct bpf_map;
+
+struct buf_context {
+	char *buf;
+};
+
+struct num_context {
+	__u64 i;
+	__u64 j;
+};
+
+__u8 choice_arr[2] = { 0, 1 };
+
+static int unsafe_on_2nd_iter_cb(__u32 idx, struct buf_context *ctx)
+{
+	if (idx == 0) {
+		ctx->buf = (char *)(0xDEAD);
+		return 0;
+	}
+
+	if (bpf_probe_read_user(ctx->buf, 8, (void *)(0xBADC0FFEE)))
+		return 1;
+
+	return 0;
+}
+
+SEC("?raw_tp")
+__failure __msg("R1 type=scalar expected=fp")
+int unsafe_on_2nd_iter(void *unused)
+{
+	char buf[4];
+	struct buf_context loop_ctx = { .buf = buf };
+
+	bpf_loop(100, unsafe_on_2nd_iter_cb, &loop_ctx, 0);
+	return 0;
+}
+
+static int unsafe_on_zero_iter_cb(__u32 idx, struct num_context *ctx)
+{
+	ctx->i = 0;
+	return 0;
+}
+
+SEC("?raw_tp")
+__failure __msg("invalid access to map value, value_size=2 off=32 size=1")
+int unsafe_on_zero_iter(void *unused)
+{
+	struct num_context loop_ctx = { .i = 32 };
+
+	bpf_loop(100, unsafe_on_zero_iter_cb, &loop_ctx, 0);
+	return choice_arr[loop_ctx.i];
+}
+
+static int widening_cb(__u32 idx, struct num_context *ctx)
+{
+	++ctx->i;
+	return 0;
+}
+
+SEC("?raw_tp")
+__success
+int widening(void *unused)
+{
+	struct num_context loop_ctx = { .i = 0, .j = 1 };
+
+	bpf_loop(100, widening_cb, &loop_ctx, 0);
+	/* loop_ctx.j is not changed during callback iteration,
+	 * verifier should not apply widening to it.
+	 */
+	return choice_arr[loop_ctx.j];
+}
+
+static int loop_detection_cb(__u32 idx, struct num_context *ctx)
+{
+	for (;;) {}
+	return 0;
+}
+
+SEC("?raw_tp")
+__failure __msg("infinite loop detected")
+int loop_detection(void *unused)
+{
+	struct num_context loop_ctx = { .i = 0 };
+
+	bpf_loop(100, loop_detection_cb, &loop_ctx, 0);
+	return 0;
+}
+
+static __always_inline __u64 oob_state_machine(struct num_context *ctx)
+{
+	switch (ctx->i) {
+	case 0:
+		ctx->i = 1;
+		break;
+	case 1:
+		ctx->i = 32;
+		break;
+	}
+	return 0;
+}
+
+static __u64 for_each_map_elem_cb(struct bpf_map *map, __u32 *key, __u64 *val, void *data)
+{
+	return oob_state_machine(data);
+}
+
+SEC("?raw_tp")
+__failure __msg("invalid access to map value, value_size=2 off=32 size=1")
+int unsafe_for_each_map_elem(void *unused)
+{
+	struct num_context loop_ctx = { .i = 0 };
+
+	bpf_for_each_map_elem(&map, for_each_map_elem_cb, &loop_ctx, 0);
+	return choice_arr[loop_ctx.i];
+}
+
+static __u64 ringbuf_drain_cb(struct bpf_dynptr *dynptr, void *data)
+{
+	return oob_state_machine(data);
+}
+
+SEC("?raw_tp")
+__failure __msg("invalid access to map value, value_size=2 off=32 size=1")
+int unsafe_ringbuf_drain(void *unused)
+{
+	struct num_context loop_ctx = { .i = 0 };
+
+	bpf_user_ringbuf_drain(&ringbuf, ringbuf_drain_cb, &loop_ctx, 0);
+	return choice_arr[loop_ctx.i];
+}
+
+static __u64 find_vma_cb(struct task_struct *task, struct vm_area_struct *vma, void *data)
+{
+	return oob_state_machine(data);
+}
+
+SEC("?raw_tp")
+__failure __msg("invalid access to map value, value_size=2 off=32 size=1")
+int unsafe_find_vma(void *unused)
+{
+	struct task_struct *task = bpf_get_current_task_btf();
+	struct num_context loop_ctx = { .i = 0 };
+
+	bpf_find_vma(task, 0, find_vma_cb, &loop_ctx, 0);
+	return choice_arr[loop_ctx.i];
+}
+
+static int iter_limit_cb(__u32 idx, struct num_context *ctx)
+{
+	ctx->i++;
+	return 0;
+}
+
+SEC("?raw_tp")
+__success
+int bpf_loop_iter_limit_ok(void *unused)
+{
+	struct num_context ctx = { .i = 0 };
+
+	bpf_loop(1, iter_limit_cb, &ctx, 0);
+	return choice_arr[ctx.i];
+}
+
+SEC("?raw_tp")
+__failure __msg("invalid access to map value, value_size=2 off=2 size=1")
+int bpf_loop_iter_limit_overflow(void *unused)
+{
+	struct num_context ctx = { .i = 0 };
+
+	bpf_loop(2, iter_limit_cb, &ctx, 0);
+	return choice_arr[ctx.i];
+}
+
+static int iter_limit_level2a_cb(__u32 idx, struct num_context *ctx)
+{
+	ctx->i += 100;
+	return 0;
+}
+
+static int iter_limit_level2b_cb(__u32 idx, struct num_context *ctx)
+{
+	ctx->i += 10;
+	return 0;
+}
+
+static int iter_limit_level1_cb(__u32 idx, struct num_context *ctx)
+{
+	ctx->i += 1;
+	bpf_loop(1, iter_limit_level2a_cb, ctx, 0);
+	bpf_loop(1, iter_limit_level2b_cb, ctx, 0);
+	return 0;
+}
+
+/* Check that path visiting every callback function once had been
+ * reached by verifier. Variables 'ctx{1,2}i' below serve as flags,
+ * with each decimal digit corresponding to a callback visit marker.
+ */
+SEC("socket")
+__success __retval(111111)
+int bpf_loop_iter_limit_nested(void *unused)
+{
+	struct num_context ctx1 = { .i = 0 };
+	struct num_context ctx2 = { .i = 0 };
+	__u64 a, b, c;
+
+	bpf_loop(1, iter_limit_level1_cb, &ctx1, 0);
+	bpf_loop(1, iter_limit_level1_cb, &ctx2, 0);
+	a = ctx1.i;
+	b = ctx2.i;
+	/* Force 'ctx1.i' and 'ctx2.i' precise. */
+	c = choice_arr[(a + b) % 2];
+	/* This makes 'c' zero, but neither clang nor verifier know it. */
+	c /= 10;
+	/* Make sure that verifier does not visit 'impossible' states:
+	 * enumerate all possible callback visit masks.
+	 */
+	if (a != 0 && a != 1 && a != 11 && a != 101 && a != 111 &&
+	    b != 0 && b != 1 && b != 11 && b != 101 && b != 111)
+		asm volatile ("r0 /= 0;" ::: "r0");
+	return 1000 * a + b + c;
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c b/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c
index db6b3143338b..f61d623b1ce8 100644
--- a/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c
+++ b/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c
@@ -119,15 +119,41 @@ __naked int global_subprog_result_precise(void)
 
 SEC("?raw_tp")
 __success __log_level(2)
+/* First simulated path does not include callback body,
+ * r1 and r4 are always precise for bpf_loop() calls.
+ */
+__msg("9: (85) call bpf_loop#181")
+__msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1")
+__msg("mark_precise: frame0: parent state regs=r4 stack=:")
+__msg("mark_precise: frame0: last_idx 8 first_idx 0 subseq_idx 9")
+__msg("mark_precise: frame0: regs=r4 stack= before 8: (b7) r4 = 0")
+__msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1")
+__msg("mark_precise: frame0: parent state regs=r1 stack=:")
+__msg("mark_precise: frame0: last_idx 8 first_idx 0 subseq_idx 9")
+__msg("mark_precise: frame0: regs=r1 stack= before 8: (b7) r4 = 0")
+__msg("mark_precise: frame0: regs=r1 stack= before 7: (b7) r3 = 0")
+__msg("mark_precise: frame0: regs=r1 stack= before 6: (bf) r2 = r8")
+__msg("mark_precise: frame0: regs=r1 stack= before 5: (bf) r1 = r6")
+__msg("mark_precise: frame0: regs=r6 stack= before 4: (b7) r6 = 3")
+/* r6 precision propagation */
 __msg("14: (0f) r1 += r6")
-__msg("mark_precise: frame0: last_idx 14 first_idx 10")
+__msg("mark_precise: frame0: last_idx 14 first_idx 9")
 __msg("mark_precise: frame0: regs=r6 stack= before 13: (bf) r1 = r7")
 __msg("mark_precise: frame0: regs=r6 stack= before 12: (27) r6 *= 4")
 __msg("mark_precise: frame0: regs=r6 stack= before 11: (25) if r6 > 0x3 goto pc+4")
 __msg("mark_precise: frame0: regs=r6 stack= before 10: (bf) r6 = r0")
-__msg("mark_precise: frame0: parent state regs=r0 stack=:")
-__msg("mark_precise: frame0: last_idx 18 first_idx 0")
-__msg("mark_precise: frame0: regs=r0 stack= before 18: (95) exit")
+__msg("mark_precise: frame0: regs=r0 stack= before 9: (85) call bpf_loop")
+/* State entering callback body popped from states stack */
+__msg("from 9 to 17: frame1:")
+__msg("17: frame1: R1=scalar() R2=0 R10=fp0 cb")
+__msg("17: (b7) r0 = 0")
+__msg("18: (95) exit")
+__msg("returning from callee:")
+__msg("to caller at 9:")
+__msg("frame 0: propagating r1,r4")
+__msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1")
+__msg("mark_precise: frame0: regs=r1,r4 stack= before 18: (95) exit")
+__msg("from 18 to 9: safe")
 __naked int callback_result_precise(void)
 {
 	asm volatile (
@@ -233,20 +259,36 @@ __naked int parent_callee_saved_reg_precise_global(void)
 
 SEC("?raw_tp")
 __success __log_level(2)
+/* First simulated path does not include callback body */
 __msg("12: (0f) r1 += r6")
-__msg("mark_precise: frame0: last_idx 12 first_idx 10")
+__msg("mark_precise: frame0: last_idx 12 first_idx 9")
 __msg("mark_precise: frame0: regs=r6 stack= before 11: (bf) r1 = r7")
 __msg("mark_precise: frame0: regs=r6 stack= before 10: (27) r6 *= 4")
+__msg("mark_precise: frame0: regs=r6 stack= before 9: (85) call bpf_loop")
 __msg("mark_precise: frame0: parent state regs=r6 stack=:")
-__msg("mark_precise: frame0: last_idx 16 first_idx 0")
-__msg("mark_precise: frame0: regs=r6 stack= before 16: (95) exit")
-__msg("mark_precise: frame1: regs= stack= before 15: (b7) r0 = 0")
-__msg("mark_precise: frame1: regs= stack= before 9: (85) call bpf_loop#181")
+__msg("mark_precise: frame0: last_idx 8 first_idx 0 subseq_idx 9")
 __msg("mark_precise: frame0: regs=r6 stack= before 8: (b7) r4 = 0")
 __msg("mark_precise: frame0: regs=r6 stack= before 7: (b7) r3 = 0")
 __msg("mark_precise: frame0: regs=r6 stack= before 6: (bf) r2 = r8")
 __msg("mark_precise: frame0: regs=r6 stack= before 5: (b7) r1 = 1")
 __msg("mark_precise: frame0: regs=r6 stack= before 4: (b7) r6 = 3")
+/* State entering callback body popped from states stack */
+__msg("from 9 to 15: frame1:")
+__msg("15: frame1: R1=scalar() R2=0 R10=fp0 cb")
+__msg("15: (b7) r0 = 0")
+__msg("16: (95) exit")
+__msg("returning from callee:")
+__msg("to caller at 9:")
+/* r1, r4 are always precise for bpf_loop(),
+ * r6 was marked before backtracking to callback body.
+ */
+__msg("frame 0: propagating r1,r4,r6")
+__msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1")
+__msg("mark_precise: frame0: regs=r1,r4,r6 stack= before 16: (95) exit")
+__msg("mark_precise: frame1: regs= stack= before 15: (b7) r0 = 0")
+__msg("mark_precise: frame1: regs= stack= before 9: (85) call bpf_loop")
+__msg("mark_precise: frame0: parent state regs= stack=:")
+__msg("from 16 to 9: safe")
 __naked int parent_callee_saved_reg_precise_with_callback(void)
 {
 	asm volatile (
@@ -373,22 +415,38 @@ __naked int parent_stack_slot_precise_global(void)
 
 SEC("?raw_tp")
 __success __log_level(2)
+/* First simulated path does not include callback body */
 __msg("14: (0f) r1 += r6")
-__msg("mark_precise: frame0: last_idx 14 first_idx 11")
+__msg("mark_precise: frame0: last_idx 14 first_idx 10")
 __msg("mark_precise: frame0: regs=r6 stack= before 13: (bf) r1 = r7")
 __msg("mark_precise: frame0: regs=r6 stack= before 12: (27) r6 *= 4")
 __msg("mark_precise: frame0: regs=r6 stack= before 11: (79) r6 = *(u64 *)(r10 -8)")
+__msg("mark_precise: frame0: regs= stack=-8 before 10: (85) call bpf_loop")
 __msg("mark_precise: frame0: parent state regs= stack=-8:")
-__msg("mark_precise: frame0: last_idx 18 first_idx 0")
-__msg("mark_precise: frame0: regs= stack=-8 before 18: (95) exit")
-__msg("mark_precise: frame1: regs= stack= before 17: (b7) r0 = 0")
-__msg("mark_precise: frame1: regs= stack= before 10: (85) call bpf_loop#181")
+__msg("mark_precise: frame0: last_idx 9 first_idx 0 subseq_idx 10")
 __msg("mark_precise: frame0: regs= stack=-8 before 9: (b7) r4 = 0")
 __msg("mark_precise: frame0: regs= stack=-8 before 8: (b7) r3 = 0")
 __msg("mark_precise: frame0: regs= stack=-8 before 7: (bf) r2 = r8")
 __msg("mark_precise: frame0: regs= stack=-8 before 6: (bf) r1 = r6")
 __msg("mark_precise: frame0: regs= stack=-8 before 5: (7b) *(u64 *)(r10 -8) = r6")
 __msg("mark_precise: frame0: regs=r6 stack= before 4: (b7) r6 = 3")
+/* State entering callback body popped from states stack */
+__msg("from 10 to 17: frame1:")
+__msg("17: frame1: R1=scalar() R2=0 R10=fp0 cb")
+__msg("17: (b7) r0 = 0")
+__msg("18: (95) exit")
+__msg("returning from callee:")
+__msg("to caller at 10:")
+/* r1, r4 are always precise for bpf_loop(),
+ * fp-8 was marked before backtracking to callback body.
+ */
+__msg("frame 0: propagating r1,r4,fp-8")
+__msg("mark_precise: frame0: last_idx 10 first_idx 10 subseq_idx -1")
+__msg("mark_precise: frame0: regs=r1,r4 stack=-8 before 18: (95) exit")
+__msg("mark_precise: frame1: regs= stack= before 17: (b7) r0 = 0")
+__msg("mark_precise: frame1: regs= stack= before 10: (85) call bpf_loop#181")
+__msg("mark_precise: frame0: parent state regs= stack=:")
+__msg("from 18 to 10: safe")
 __naked int parent_stack_slot_precise_with_callback(void)
 {
 	asm volatile (
diff --git a/tools/testing/selftests/bpf/progs/xdp_synproxy_kern.c b/tools/testing/selftests/bpf/progs/xdp_synproxy_kern.c
index e959336c7a73..80f620602d50 100644
--- a/tools/testing/selftests/bpf/progs/xdp_synproxy_kern.c
+++ b/tools/testing/selftests/bpf/progs/xdp_synproxy_kern.c
@@ -53,6 +53,8 @@
 #define DEFAULT_TTL 64
 #define MAX_ALLOWED_PORTS 8
 
+#define MAX_PACKET_OFF 0xffff
+
 #define swap(a, b) \
 	do { typeof(a) __tmp = (a); (a) = (b); (b) = __tmp; } while (0)
 
@@ -183,63 +185,76 @@ static __always_inline __u32 tcp_clock_ms(void)
 }
 
 struct tcpopt_context {
-	__u8 *ptr;
-	__u8 *end;
+	void *data;
 	void *data_end;
 	__be32 *tsecr;
 	__u8 wscale;
 	bool option_timestamp;
 	bool option_sack;
+	__u32 off;
 };
 
-static int tscookie_tcpopt_parse(struct tcpopt_context *ctx)
+static __always_inline u8 *next(struct tcpopt_context *ctx, __u32 sz)
 {
-	__u8 opcode, opsize;
+	__u64 off = ctx->off;
+	__u8 *data;
 
-	if (ctx->ptr >= ctx->end)
-		return 1;
-	if (ctx->ptr >= ctx->data_end)
-		return 1;
+	/* Verifier forbids access to packet when offset exceeds MAX_PACKET_OFF */
+	if (off > MAX_PACKET_OFF - sz)
+		return NULL;
 
-	opcode = ctx->ptr[0];
+	data = ctx->data + off;
+	barrier_var(data);
+	if (data + sz >= ctx->data_end)
+		return NULL;
 
-	if (opcode == TCPOPT_EOL)
-		return 1;
-	if (opcode == TCPOPT_NOP) {
-		++ctx->ptr;
-		return 0;
-	}
+	ctx->off += sz;
+	return data;
+}
 
-	if (ctx->ptr + 1 >= ctx->end)
-		return 1;
-	if (ctx->ptr + 1 >= ctx->data_end)
+static int tscookie_tcpopt_parse(struct tcpopt_context *ctx)
+{
+	__u8 *opcode, *opsize, *wscale, *tsecr;
+	__u32 off = ctx->off;
+
+	opcode = next(ctx, 1);
+	if (!opcode)
 		return 1;
-	opsize = ctx->ptr[1];
-	if (opsize < 2)
+
+	if (*opcode == TCPOPT_EOL)
 		return 1;
+	if (*opcode == TCPOPT_NOP)
+		return 0;
 
-	if (ctx->ptr + opsize > ctx->end)
+	opsize = next(ctx, 1);
+	if (!opsize || *opsize < 2)
 		return 1;
 
-	switch (opcode) {
+	switch (*opcode) {
 	case TCPOPT_WINDOW:
-		if (opsize == TCPOLEN_WINDOW && ctx->ptr + TCPOLEN_WINDOW <= ctx->data_end)
-			ctx->wscale = ctx->ptr[2] < TCP_MAX_WSCALE ? ctx->ptr[2] : TCP_MAX_WSCALE;
+		wscale = next(ctx, 1);
+		if (!wscale)
+			return 1;
+		if (*opsize == TCPOLEN_WINDOW)
+			ctx->wscale = *wscale < TCP_MAX_WSCALE ? *wscale : TCP_MAX_WSCALE;
 		break;
 	case TCPOPT_TIMESTAMP:
-		if (opsize == TCPOLEN_TIMESTAMP && ctx->ptr + TCPOLEN_TIMESTAMP <= ctx->data_end) {
+		tsecr = next(ctx, 4);
+		if (!tsecr)
+			return 1;
+		if (*opsize == TCPOLEN_TIMESTAMP) {
 			ctx->option_timestamp = true;
 			/* Client's tsval becomes our tsecr. */
-			*ctx->tsecr = get_unaligned((__be32 *)(ctx->ptr + 2));
+			*ctx->tsecr = get_unaligned((__be32 *)tsecr);
 		}
 		break;
 	case TCPOPT_SACK_PERM:
-		if (opsize == TCPOLEN_SACK_PERM)
+		if (*opsize == TCPOLEN_SACK_PERM)
 			ctx->option_sack = true;
 		break;
 	}
 
-	ctx->ptr += opsize;
+	ctx->off = off + *opsize;
 
 	return 0;
 }
@@ -256,16 +271,21 @@ static int tscookie_tcpopt_parse_batch(__u32 index, void *context)
 
 static __always_inline bool tscookie_init(struct tcphdr *tcp_header,
 					  __u16 tcp_len, __be32 *tsval,
-					  __be32 *tsecr, void *data_end)
+					  __be32 *tsecr, void *data, void *data_end)
 {
 	struct tcpopt_context loop_ctx = {
-		.ptr = (__u8 *)(tcp_header + 1),
-		.end = (__u8 *)tcp_header + tcp_len,
+		.data = data,
 		.data_end = data_end,
 		.tsecr = tsecr,
 		.wscale = TS_OPT_WSCALE_MASK,
 		.option_timestamp = false,
 		.option_sack = false,
+		/* Note: currently verifier would track .off as unbound scalar.
+		 *       In case if verifier would at some point get smarter and
+		 *       compute bounded value for this var, beware that it might
+		 *       hinder bpf_loop() convergence validation.
+		 */
+		.off = (__u8 *)(tcp_header + 1) - (__u8 *)data,
 	};
 	u32 cookie;
 
@@ -635,7 +655,7 @@ static __always_inline int syncookie_handle_syn(struct header_pointers *hdr,
 	cookie = (__u32)value;
 
 	if (tscookie_init((void *)hdr->tcp, hdr->tcp_len,
-			  &tsopt_buf[0], &tsopt_buf[1], data_end))
+			  &tsopt_buf[0], &tsopt_buf[1], data, data_end))
 		tsopt = tsopt_buf;
 
 	/* Check that there is enough space for a SYNACK. It also covers
diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c
index 37ffa57f28a1..a350ecdfba4a 100644
--- a/tools/testing/selftests/bpf/test_loader.c
+++ b/tools/testing/selftests/bpf/test_loader.c
@@ -153,6 +153,14 @@ static int parse_retval(const char *str, int *val, const char *name)
 	return parse_int(str, val, name);
 }
 
+static void update_flags(int *flags, int flag, bool clear)
+{
+	if (clear)
+		*flags &= ~flag;
+	else
+		*flags |= flag;
+}
+
 /* Uses btf_decl_tag attributes to describe the expected test
  * behavior, see bpf_misc.h for detailed description of each attribute
  * and attribute combinations.
@@ -171,6 +179,7 @@ static int parse_test_spec(struct test_loader *tester,
 	memset(spec, 0, sizeof(*spec));
 
 	spec->prog_name = bpf_program__name(prog);
+	spec->prog_flags = BPF_F_TEST_REG_INVARIANTS; /* by default be strict */
 
 	btf = bpf_object__btf(obj);
 	if (!btf) {
@@ -187,7 +196,8 @@ static int parse_test_spec(struct test_loader *tester,
 	for (i = 1; i < btf__type_cnt(btf); i++) {
 		const char *s, *val, *msg;
 		const struct btf_type *t;
-		int tmp;
+		bool clear;
+		int flags;
 
 		t = btf__type_by_id(btf, i);
 		if (!btf_is_decl_tag(t))
@@ -253,23 +263,30 @@ static int parse_test_spec(struct test_loader *tester,
 				goto cleanup;
 		} else if (str_has_pfx(s, TEST_TAG_PROG_FLAGS_PFX)) {
 			val = s + sizeof(TEST_TAG_PROG_FLAGS_PFX) - 1;
+
+			clear = val[0] == '!';
+			if (clear)
+				val++;
+
 			if (strcmp(val, "BPF_F_STRICT_ALIGNMENT") == 0) {
-				spec->prog_flags |= BPF_F_STRICT_ALIGNMENT;
+				update_flags(&spec->prog_flags, BPF_F_STRICT_ALIGNMENT, clear);
 			} else if (strcmp(val, "BPF_F_ANY_ALIGNMENT") == 0) {
-				spec->prog_flags |= BPF_F_ANY_ALIGNMENT;
+				update_flags(&spec->prog_flags, BPF_F_ANY_ALIGNMENT, clear);
 			} else if (strcmp(val, "BPF_F_TEST_RND_HI32") == 0) {
-				spec->prog_flags |= BPF_F_TEST_RND_HI32;
+				update_flags(&spec->prog_flags, BPF_F_TEST_RND_HI32, clear);
 			} else if (strcmp(val, "BPF_F_TEST_STATE_FREQ") == 0) {
-				spec->prog_flags |= BPF_F_TEST_STATE_FREQ;
+				update_flags(&spec->prog_flags, BPF_F_TEST_STATE_FREQ, clear);
 			} else if (strcmp(val, "BPF_F_SLEEPABLE") == 0) {
-				spec->prog_flags |= BPF_F_SLEEPABLE;
+				update_flags(&spec->prog_flags, BPF_F_SLEEPABLE, clear);
 			} else if (strcmp(val, "BPF_F_XDP_HAS_FRAGS") == 0) {
-				spec->prog_flags |= BPF_F_XDP_HAS_FRAGS;
+				update_flags(&spec->prog_flags, BPF_F_XDP_HAS_FRAGS, clear);
+			} else if (strcmp(val, "BPF_F_TEST_REG_INVARIANTS") == 0) {
+				update_flags(&spec->prog_flags, BPF_F_TEST_REG_INVARIANTS, clear);
 			} else /* assume numeric value */ {
-				err = parse_int(val, &tmp, "test prog flags");
+				err = parse_int(val, &flags, "test prog flags");
 				if (err)
 					goto cleanup;
-				spec->prog_flags |= tmp;
+				update_flags(&spec->prog_flags, flags, clear);
 			}
 		}
 	}
diff --git a/tools/testing/selftests/bpf/test_maps.c b/tools/testing/selftests/bpf/test_maps.c
index 7fc00e423e4d..767e0693df10 100644
--- a/tools/testing/selftests/bpf/test_maps.c
+++ b/tools/testing/selftests/bpf/test_maps.c
@@ -1396,13 +1396,18 @@ static void test_map_stress(void)
 #define MAX_DELAY_US 50000
 #define MIN_DELAY_RANGE_US 5000
 
-static int map_update_retriable(int map_fd, const void *key, const void *value,
-				int flags, int attempts)
+static bool retry_for_again_or_busy(int err)
+{
+	return (err == EAGAIN || err == EBUSY);
+}
+
+int map_update_retriable(int map_fd, const void *key, const void *value, int flags, int attempts,
+			 retry_for_error_fn need_retry)
 {
 	int delay = rand() % MIN_DELAY_RANGE_US;
 
 	while (bpf_map_update_elem(map_fd, key, value, flags)) {
-		if (!attempts || (errno != EAGAIN && errno != EBUSY))
+		if (!attempts || !need_retry(errno))
 			return -errno;
 
 		if (delay <= MAX_DELAY_US / 2)
@@ -1445,11 +1450,13 @@ static void test_update_delete(unsigned int fn, void *data)
 		key = value = i;
 
 		if (do_update) {
-			err = map_update_retriable(fd, &key, &value, BPF_NOEXIST, MAP_RETRIES);
+			err = map_update_retriable(fd, &key, &value, BPF_NOEXIST, MAP_RETRIES,
+						   retry_for_again_or_busy);
 			if (err)
 				printf("error %d %d\n", err, errno);
 			assert(err == 0);
-			err = map_update_retriable(fd, &key, &value, BPF_EXIST, MAP_RETRIES);
+			err = map_update_retriable(fd, &key, &value, BPF_EXIST, MAP_RETRIES,
+						   retry_for_again_or_busy);
 			if (err)
 				printf("error %d %d\n", err, errno);
 			assert(err == 0);
diff --git a/tools/testing/selftests/bpf/test_maps.h b/tools/testing/selftests/bpf/test_maps.h
index f6fbca761732..e4ac704a536c 100644
--- a/tools/testing/selftests/bpf/test_maps.h
+++ b/tools/testing/selftests/bpf/test_maps.h
@@ -4,6 +4,7 @@
 
 #include <stdio.h>
 #include <stdlib.h>
+#include <stdbool.h>
 
 #define CHECK(condition, tag, format...) ({				\
 	int __ret = !!(condition);					\
@@ -16,4 +17,8 @@
 
 extern int skips;
 
+typedef bool (*retry_for_error_fn)(int err);
+int map_update_retriable(int map_fd, const void *key, const void *value, int flags, int attempts,
+			 retry_for_error_fn need_retry);
+
 #endif
diff --git a/tools/testing/selftests/bpf/test_sock_addr.c b/tools/testing/selftests/bpf/test_sock_addr.c
index 2c89674fc62c..b0068a9d2cfe 100644
--- a/tools/testing/selftests/bpf/test_sock_addr.c
+++ b/tools/testing/selftests/bpf/test_sock_addr.c
@@ -679,7 +679,7 @@ static int load_path(const struct sock_addr_test *test, const char *path)
 
 	bpf_program__set_type(prog, BPF_PROG_TYPE_CGROUP_SOCK_ADDR);
 	bpf_program__set_expected_attach_type(prog, test->expected_attach_type);
-	bpf_program__set_flags(prog, BPF_F_TEST_RND_HI32);
+	bpf_program__set_flags(prog, BPF_F_TEST_RND_HI32 | BPF_F_TEST_REG_INVARIANTS);
 
 	err = bpf_object__load(obj);
 	if (err) {
diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c
index 98107e0452d3..f36e41435be7 100644
--- a/tools/testing/selftests/bpf/test_verifier.c
+++ b/tools/testing/selftests/bpf/test_verifier.c
@@ -1588,7 +1588,7 @@ static void do_test_single(struct bpf_test *test, bool unpriv,
 	if (fixup_skips != skips)
 		return;
 
-	pflags = BPF_F_TEST_RND_HI32;
+	pflags = BPF_F_TEST_RND_HI32 | BPF_F_TEST_REG_INVARIANTS;
 	if (test->flags & F_LOAD_WITH_STRICT_ALIGNMENT)
 		pflags |= BPF_F_STRICT_ALIGNMENT;
 	if (test->flags & F_NEEDS_EFFICIENT_UNALIGNED_ACCESS)
diff --git a/tools/testing/selftests/bpf/testing_helpers.c b/tools/testing/selftests/bpf/testing_helpers.c
index 8d994884c7b4..d2458c1b1671 100644
--- a/tools/testing/selftests/bpf/testing_helpers.c
+++ b/tools/testing/selftests/bpf/testing_helpers.c
@@ -276,7 +276,7 @@ int bpf_prog_test_load(const char *file, enum bpf_prog_type type,
 	if (type != BPF_PROG_TYPE_UNSPEC && bpf_program__type(prog) != type)
 		bpf_program__set_type(prog, type);
 
-	flags = bpf_program__flags(prog) | BPF_F_TEST_RND_HI32;
+	flags = bpf_program__flags(prog) | BPF_F_TEST_RND_HI32 | BPF_F_TEST_REG_INVARIANTS;
 	bpf_program__set_flags(prog, flags);
 
 	err = bpf_object__load(obj);
@@ -299,7 +299,7 @@ int bpf_test_load_program(enum bpf_prog_type type, const struct bpf_insn *insns,
 {
 	LIBBPF_OPTS(bpf_prog_load_opts, opts,
 		.kern_version = kern_version,
-		.prog_flags = BPF_F_TEST_RND_HI32,
+		.prog_flags = BPF_F_TEST_RND_HI32 | BPF_F_TEST_REG_INVARIANTS,
 		.log_level = extra_prog_load_log_flags,
 		.log_buf = log_buf,
 		.log_size = log_buf_sz,
diff --git a/tools/testing/selftests/bpf/veristat.c b/tools/testing/selftests/bpf/veristat.c
index 655095810d4a..1d418d66e375 100644
--- a/tools/testing/selftests/bpf/veristat.c
+++ b/tools/testing/selftests/bpf/veristat.c
@@ -18,6 +18,7 @@
 #include <libelf.h>
 #include <gelf.h>
 #include <float.h>
+#include <math.h>
 
 #ifndef ARRAY_SIZE
 #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
@@ -99,6 +100,7 @@ struct stat_specs {
 	enum stat_id ids[ALL_STATS_CNT];
 	enum stat_variant variants[ALL_STATS_CNT];
 	bool asc[ALL_STATS_CNT];
+	bool abs[ALL_STATS_CNT];
 	int lens[ALL_STATS_CNT * 3]; /* 3x for comparison mode */
 };
 
@@ -133,6 +135,7 @@ struct filter {
 	int stat_id;
 	enum stat_variant stat_var;
 	long value;
+	bool abs;
 };
 
 static struct env {
@@ -142,10 +145,12 @@ static struct env {
 	bool debug;
 	bool quiet;
 	bool force_checkpoints;
+	bool force_reg_invariants;
 	enum resfmt out_fmt;
 	bool show_version;
 	bool comparison_mode;
 	bool replay_mode;
+	int top_n;
 
 	int log_level;
 	int log_size;
@@ -210,8 +215,7 @@ static const struct argp_option opts[] = {
 	{ "log-level", 'l', "LEVEL", 0, "Verifier log level (default 0 for normal mode, 1 for verbose mode)" },
 	{ "log-fixed", OPT_LOG_FIXED, NULL, 0, "Disable verifier log rotation" },
 	{ "log-size", OPT_LOG_SIZE, "BYTES", 0, "Customize verifier log size (default to 16MB)" },
-	{ "test-states", 't', NULL, 0,
-	  "Force frequent BPF verifier state checkpointing (set BPF_F_TEST_STATE_FREQ program flag)" },
+	{ "top-n", 'n', "N", 0, "Emit only up to first N results." },
 	{ "quiet", 'q', NULL, 0, "Quiet mode" },
 	{ "emit", 'e', "SPEC", 0, "Specify stats to be emitted" },
 	{ "sort", 's', "SPEC", 0, "Specify sort order" },
@@ -219,6 +223,10 @@ static const struct argp_option opts[] = {
 	{ "compare", 'C', NULL, 0, "Comparison mode" },
 	{ "replay", 'R', NULL, 0, "Replay mode" },
 	{ "filter", 'f', "FILTER", 0, "Filter expressions (or @filename for file with expressions)." },
+	{ "test-states", 't', NULL, 0,
+	  "Force frequent BPF verifier state checkpointing (set BPF_F_TEST_STATE_FREQ program flag)" },
+	{ "test-reg-invariants", 'r', NULL, 0,
+	  "Force BPF verifier failure on register invariant violation (BPF_F_TEST_REG_INVARIANTS program flag)" },
 	{},
 };
 
@@ -290,6 +298,16 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state)
 	case 't':
 		env.force_checkpoints = true;
 		break;
+	case 'r':
+		env.force_reg_invariants = true;
+		break;
+	case 'n':
+		errno = 0;
+		env.top_n = strtol(arg, NULL, 10);
+		if (errno) {
+			fprintf(stderr, "invalid top N specifier: %s\n", arg);
+			argp_usage(state);
+		}
 	case 'C':
 		env.comparison_mode = true;
 		break;
@@ -455,7 +473,8 @@ static struct {
 	{ OP_EQ, "=" },
 };
 
-static bool parse_stat_id_var(const char *name, size_t len, int *id, enum stat_variant *var);
+static bool parse_stat_id_var(const char *name, size_t len, int *id,
+			      enum stat_variant *var, bool *is_abs);
 
 static int append_filter(struct filter **filters, int *cnt, const char *str)
 {
@@ -488,13 +507,14 @@ static int append_filter(struct filter **filters, int *cnt, const char *str)
 		long val;
 		const char *end = str;
 		const char *op_str;
+		bool is_abs;
 
 		op_str = operators[i].op_str;
 		p = strstr(str, op_str);
 		if (!p)
 			continue;
 
-		if (!parse_stat_id_var(str, p - str, &id, &var)) {
+		if (!parse_stat_id_var(str, p - str, &id, &var, &is_abs)) {
 			fprintf(stderr, "Unrecognized stat name in '%s'!\n", str);
 			return -EINVAL;
 		}
@@ -533,6 +553,7 @@ static int append_filter(struct filter **filters, int *cnt, const char *str)
 		f->stat_id = id;
 		f->stat_var = var;
 		f->op = operators[i].op_kind;
+		f->abs = true;
 		f->value = val;
 
 		*cnt += 1;
@@ -657,7 +678,8 @@ static struct stat_def {
 	[MARK_READ_MAX_LEN] = { "Max mark read length", {"max_mark_read_len", "mark_read"}, },
 };
 
-static bool parse_stat_id_var(const char *name, size_t len, int *id, enum stat_variant *var)
+static bool parse_stat_id_var(const char *name, size_t len, int *id,
+			      enum stat_variant *var, bool *is_abs)
 {
 	static const char *var_sfxs[] = {
 		[VARIANT_A] = "_a",
@@ -667,6 +689,14 @@ static bool parse_stat_id_var(const char *name, size_t len, int *id, enum stat_v
 	};
 	int i, j, k;
 
+	/* |<stat>| means we take absolute value of given stat */
+	*is_abs = false;
+	if (len > 2 && name[0] == '|' && name[len - 1] == '|') {
+		*is_abs = true;
+		name += 1;
+		len -= 2;
+	}
+
 	for (i = 0; i < ARRAY_SIZE(stat_defs); i++) {
 		struct stat_def *def = &stat_defs[i];
 		size_t alias_len, sfx_len;
@@ -722,7 +752,7 @@ static bool is_desc_sym(char c)
 static int parse_stat(const char *stat_name, struct stat_specs *specs)
 {
 	int id;
-	bool has_order = false, is_asc = false;
+	bool has_order = false, is_asc = false, is_abs = false;
 	size_t len = strlen(stat_name);
 	enum stat_variant var;
 
@@ -737,7 +767,7 @@ static int parse_stat(const char *stat_name, struct stat_specs *specs)
 		len -= 1;
 	}
 
-	if (!parse_stat_id_var(stat_name, len, &id, &var)) {
+	if (!parse_stat_id_var(stat_name, len, &id, &var, &is_abs)) {
 		fprintf(stderr, "Unrecognized stat name '%s'\n", stat_name);
 		return -ESRCH;
 	}
@@ -745,6 +775,7 @@ static int parse_stat(const char *stat_name, struct stat_specs *specs)
 	specs->ids[specs->spec_cnt] = id;
 	specs->variants[specs->spec_cnt] = var;
 	specs->asc[specs->spec_cnt] = has_order ? is_asc : stat_defs[id].asc_by_default;
+	specs->abs[specs->spec_cnt] = is_abs;
 	specs->spec_cnt++;
 
 	return 0;
@@ -997,6 +1028,8 @@ static int process_prog(const char *filename, struct bpf_object *obj, struct bpf
 
 	if (env.force_checkpoints)
 		bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_TEST_STATE_FREQ);
+	if (env.force_reg_invariants)
+		bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_TEST_REG_INVARIANTS);
 
 	err = bpf_object__load(obj);
 	env.progs_processed++;
@@ -1103,7 +1136,7 @@ static int process_obj(const char *filename)
 }
 
 static int cmp_stat(const struct verif_stats *s1, const struct verif_stats *s2,
-		    enum stat_id id, bool asc)
+		    enum stat_id id, bool asc, bool abs)
 {
 	int cmp = 0;
 
@@ -1124,6 +1157,11 @@ static int cmp_stat(const struct verif_stats *s1, const struct verif_stats *s2,
 		long v1 = s1->stats[id];
 		long v2 = s2->stats[id];
 
+		if (abs) {
+			v1 = v1 < 0 ? -v1 : v1;
+			v2 = v2 < 0 ? -v2 : v2;
+		}
+
 		if (v1 != v2)
 			cmp = v1 < v2 ? -1 : 1;
 		break;
@@ -1142,7 +1180,8 @@ static int cmp_prog_stats(const void *v1, const void *v2)
 	int i, cmp;
 
 	for (i = 0; i < env.sort_spec.spec_cnt; i++) {
-		cmp = cmp_stat(s1, s2, env.sort_spec.ids[i], env.sort_spec.asc[i]);
+		cmp = cmp_stat(s1, s2, env.sort_spec.ids[i],
+			       env.sort_spec.asc[i], env.sort_spec.abs[i]);
 		if (cmp != 0)
 			return cmp;
 	}
@@ -1211,7 +1250,8 @@ static void fetch_join_stat_value(const struct verif_stats_join *s,
 
 static int cmp_join_stat(const struct verif_stats_join *s1,
 			 const struct verif_stats_join *s2,
-			 enum stat_id id, enum stat_variant var, bool asc)
+			 enum stat_id id, enum stat_variant var,
+			 bool asc, bool abs)
 {
 	const char *str1 = NULL, *str2 = NULL;
 	double v1, v2;
@@ -1220,6 +1260,11 @@ static int cmp_join_stat(const struct verif_stats_join *s1,
 	fetch_join_stat_value(s1, id, var, &str1, &v1);
 	fetch_join_stat_value(s2, id, var, &str2, &v2);
 
+	if (abs) {
+		v1 = fabs(v1);
+		v2 = fabs(v2);
+	}
+
 	if (str1)
 		cmp = strcmp(str1, str2);
 	else if (v1 != v2)
@@ -1237,7 +1282,8 @@ static int cmp_join_stats(const void *v1, const void *v2)
 		cmp = cmp_join_stat(s1, s2,
 				    env.sort_spec.ids[i],
 				    env.sort_spec.variants[i],
-				    env.sort_spec.asc[i]);
+				    env.sort_spec.asc[i],
+				    env.sort_spec.abs[i]);
 		if (cmp != 0)
 			return cmp;
 	}
@@ -1720,6 +1766,9 @@ static bool is_join_stat_filter_matched(struct filter *f, const struct verif_sta
 
 	fetch_join_stat_value(stats, f->stat_id, f->stat_var, &str, &value);
 
+	if (f->abs)
+		value = fabs(value);
+
 	switch (f->op) {
 	case OP_EQ: return value > f->value - eps && value < f->value + eps;
 	case OP_NEQ: return value < f->value - eps || value > f->value + eps;
@@ -1766,7 +1815,7 @@ static int handle_comparison_mode(void)
 	struct stat_specs base_specs = {}, comp_specs = {};
 	struct stat_specs tmp_sort_spec;
 	enum resfmt cur_fmt;
-	int err, i, j, last_idx;
+	int err, i, j, last_idx, cnt;
 
 	if (env.filename_cnt != 2) {
 		fprintf(stderr, "Comparison mode expects exactly two input CSV files!\n\n");
@@ -1879,7 +1928,7 @@ static int handle_comparison_mode(void)
 		env.join_stat_cnt += 1;
 	}
 
-	/* now sort joined results accorsing to sort spec */
+	/* now sort joined results according to sort spec */
 	qsort(env.join_stats, env.join_stat_cnt, sizeof(*env.join_stats), cmp_join_stats);
 
 	/* for human-readable table output we need to do extra pass to
@@ -1896,16 +1945,22 @@ static int handle_comparison_mode(void)
 	output_comp_headers(cur_fmt);
 
 	last_idx = -1;
+	cnt = 0;
 	for (i = 0; i < env.join_stat_cnt; i++) {
 		const struct verif_stats_join *join = &env.join_stats[i];
 
 		if (!should_output_join_stats(join))
 			continue;
 
+		if (env.top_n && cnt >= env.top_n)
+			break;
+
 		if (cur_fmt == RESFMT_TABLE_CALCLEN)
 			last_idx = i;
 
 		output_comp_stats(join, cur_fmt, i == last_idx);
+
+		cnt++;
 	}
 
 	if (cur_fmt == RESFMT_TABLE_CALCLEN) {
@@ -1920,6 +1975,9 @@ static bool is_stat_filter_matched(struct filter *f, const struct verif_stats *s
 {
 	long value = stats->stats[f->stat_id];
 
+	if (f->abs)
+		value = value < 0 ? -value : value;
+
 	switch (f->op) {
 	case OP_EQ: return value == f->value;
 	case OP_NEQ: return value != f->value;
@@ -1964,7 +2022,7 @@ static bool should_output_stats(const struct verif_stats *stats)
 static void output_prog_stats(void)
 {
 	const struct verif_stats *stats;
-	int i, last_stat_idx = 0;
+	int i, last_stat_idx = 0, cnt = 0;
 
 	if (env.out_fmt == RESFMT_TABLE) {
 		/* calculate column widths */
@@ -1984,7 +2042,10 @@ static void output_prog_stats(void)
 		stats = &env.prog_stats[i];
 		if (!should_output_stats(stats))
 			continue;
+		if (env.top_n && cnt >= env.top_n)
+			break;
 		output_stats(stats, env.out_fmt, i == last_stat_idx);
+		cnt++;
 	}
 }
 
diff --git a/tools/testing/selftests/bpf/vmtest.sh b/tools/testing/selftests/bpf/vmtest.sh
index 685034528018..65d14f3bbe30 100755
--- a/tools/testing/selftests/bpf/vmtest.sh
+++ b/tools/testing/selftests/bpf/vmtest.sh
@@ -36,7 +36,9 @@ DEFAULT_COMMAND="./test_progs"
 MOUNT_DIR="mnt"
 ROOTFS_IMAGE="root.img"
 OUTPUT_DIR="$HOME/.bpf_selftests"
-KCONFIG_REL_PATHS=("tools/testing/selftests/bpf/config" "tools/testing/selftests/bpf/config.${ARCH}")
+KCONFIG_REL_PATHS=("tools/testing/selftests/bpf/config"
+	"tools/testing/selftests/bpf/config.vm"
+	"tools/testing/selftests/bpf/config.${ARCH}")
 INDEX_URL="https://raw.githubusercontent.com/libbpf/ci/master/INDEX"
 NUM_COMPILE_JOBS="$(nproc)"
 LOG_FILE_BASE="$(date +"bpf_selftests.%Y-%m-%d_%H-%M-%S")"
diff --git a/tools/testing/selftests/cgroup/test_cpuset_prs.sh b/tools/testing/selftests/cgroup/test_cpuset_prs.sh
index a6e9848189d6..7b7c4c2b6d85 100755
--- a/tools/testing/selftests/cgroup/test_cpuset_prs.sh
+++ b/tools/testing/selftests/cgroup/test_cpuset_prs.sh
@@ -146,71 +146,6 @@ test_add_proc()
 	echo $$ > $CGROUP2/cgroup.procs	# Move out the task
 }
 
-#
-# Testing the new "isolated" partition root type
-#
-test_isolated()
-{
-	cd $CGROUP2/test
-	echo 2-3 > cpuset.cpus
-	TYPE=$(cat cpuset.cpus.partition)
-	[[ $TYPE = member ]] || echo member > cpuset.cpus.partition
-
-	console_msg "Change from member to root"
-	test_partition root
-
-	console_msg "Change from root to isolated"
-	test_partition isolated
-
-	console_msg "Change from isolated to member"
-	test_partition member
-
-	console_msg "Change from member to isolated"
-	test_partition isolated
-
-	console_msg "Change from isolated to root"
-	test_partition root
-
-	console_msg "Change from root to member"
-	test_partition member
-
-	#
-	# Testing partition root with no cpu
-	#
-	console_msg "Distribute all cpus to child partition"
-	echo +cpuset > cgroup.subtree_control
-	test_partition root
-
-	mkdir A1
-	cd A1
-	echo 2-3 > cpuset.cpus
-	test_partition root
-	test_effective_cpus 2-3
-	cd ..
-	test_effective_cpus ""
-
-	console_msg "Moving task to partition test"
-	test_add_proc "No space left"
-	cd A1
-	test_add_proc ""
-	cd ..
-
-	console_msg "Shrink and expand child partition"
-	cd A1
-	echo 2 > cpuset.cpus
-	cd ..
-	test_effective_cpus 3
-	cd A1
-	echo 2-3 > cpuset.cpus
-	cd ..
-	test_effective_cpus ""
-
-	# Cleaning up
-	console_msg "Cleaning up"
-	echo $$ > $CGROUP2/cgroup.procs
-	[[ -d A1 ]] && rmdir A1
-}
-
 #
 # Cpuset controller state transition test matrix.
 #
@@ -297,14 +232,14 @@ TEST_MATRIX=(
 	" C0-3:S+ C1-3:S+ C2-3   C4-5   X2-3  X2-3:P1   P2     P1    0 A1:0-1,A2:,A3:2-3,B1:4-5 \
 								       A1:P0,A2:P1,A3:P2,B1:P1 2-3"
 	" C0-3:S+ C1-3:S+ C2-3    C4    X2-3  X2-3:P1   P2     P1    0 A1:0-1,A2:,A3:2-3,B1:4 \
-								       A1:P0,A2:P1,A3:P2,B1:P1 2-4"
+								       A1:P0,A2:P1,A3:P2,B1:P1 2-4,2-3"
 	" C0-3:S+ C1-3:S+  C3     C4    X2-3  X2-3:P1   P2     P1    0 A1:0-1,A2:2,A3:3,B1:4 \
-								       A1:P0,A2:P1,A3:P2,B1:P1 2-4"
+								       A1:P0,A2:P1,A3:P2,B1:P1 2-4,3"
 	" C0-4:S+ C1-4:S+ C2-4     .    X2-4  X2-4:P2  X4:P1    .    0 A1:0-1,A2:2-3,A3:4 \
-								       A1:P0,A2:P2,A3:P1 2-4"
+								       A1:P0,A2:P2,A3:P1 2-4,2-3"
 	" C0-4:X2-4:S+ C1-4:X2-4:S+:P2 C2-4:X4:P1 \
 				   .      .      X5      .      .    0 A1:0-4,A2:1-4,A3:2-4 \
-								       A1:P0,A2:P-2,A3:P-1 ."
+								       A1:P0,A2:P-2,A3:P-1"
 	" C0-4:X2-4:S+ C1-4:X2-4:S+:P2 C2-4:X4:P1 \
 				   .      .      .      X1      .    0 A1:0-1,A2:2-4,A3:2-4 \
 								       A1:P0,A2:P2,A3:P-1 2-4"
@@ -313,7 +248,7 @@ TEST_MATRIX=(
 	" C0-3:S+ C1-3:S+ C2-3     .    X2-3   X2-3 X2-3:P2:O2=0 .   0 A1:0-1,A2:1,A3:3 A1:P0,A3:P2 2-3"
 	" C0-3:S+ C1-3:S+ C2-3     .    X2-3   X2-3 X2-3:P2:O2=0 O2=1 0 A1:0-1,A2:1,A3:2-3 A1:P0,A3:P2 2-3"
 	" C0-3:S+ C1-3:S+  C3      .    X2-3   X2-3    P2:O3=0   .   0 A1:0-2,A2:1-2,A3: A1:P0,A3:P2 3"
-	" C0-3:S+ C1-3:S+  C3      .    X2-3   X2-3   T:P2:O3=0  .   0 A1:0-2,A2:1-2,A3:1-2 A1:P0,A3:P-2 3"
+	" C0-3:S+ C1-3:S+  C3      .    X2-3   X2-3   T:P2:O3=0  .   0 A1:0-2,A2:1-2,A3:1-2 A1:P0,A3:P-2 3,"
 
 	# An invalidated remote partition cannot self-recover from hotplug
 	" C0-3:S+ C1-3:S+  C2      .    X2-3   X2-3   T:P2:O2=0 O2=1 0 A1:0-3,A2:1-3,A3:2 A1:P0,A3:P-2"
@@ -347,10 +282,10 @@ TEST_MATRIX=(
 	# cpus_allowed/exclusive_cpus update tests
 	" C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \
 				   .     C4      .      P2     .     0 A1:4,A2:4,XA2:,XA3:,A3:4 \
-								       A1:P0,A3:P-2 ."
+								       A1:P0,A3:P-2"
 	" C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \
 				   .     X1      .      P2     .     0 A1:0-3,A2:1-3,XA1:1,XA2:,XA3:,A3:2-3 \
-								       A1:P0,A3:P-2 ."
+								       A1:P0,A3:P-2"
 	" C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \
 				   .      .     C3      P2     .     0 A1:0-2,A2:0-2,XA2:3,XA3:3,A3:3 \
 								       A1:P0,A3:P2 3"
@@ -359,13 +294,13 @@ TEST_MATRIX=(
 								       A1:P0,A3:P2 3"
 	" C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3:P2 \
 				   .      .     X3      .      .     0 A1:0-3,A2:1-3,XA2:3,XA3:3,A3:2-3 \
-								       A1:P0,A3:P-2 ."
+								       A1:P0,A3:P-2"
 	" C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3:P2 \
 				   .      .     C3      .      .     0 A1:0-3,A2:3,XA2:3,XA3:3,A3:3 \
-								       A1:P0,A3:P-2 ."
+								       A1:P0,A3:P-2"
 	" C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3:P2 \
 				   .     C4      .      .      .     0 A1:4,A2:4,A3:4,XA1:,XA2:,XA3 \
-								       A1:P0,A3:P-2 ."
+								       A1:P0,A3:P-2"
 
 	#  old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS
 	#  ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ --------
@@ -441,7 +376,7 @@ write_cpu_online()
 		}
 	fi
 	echo $VAL > $CPUFILE
-	pause 0.01
+	pause 0.05
 }
 
 #
@@ -573,12 +508,14 @@ dump_states()
 		XECPUS=$DIR/cpuset.cpus.exclusive.effective
 		PRS=$DIR/cpuset.cpus.partition
 		PCPUS=$DIR/.__DEBUG__.cpuset.cpus.subpartitions
+		ISCPUS=$DIR/.__DEBUG__.cpuset.cpus.isolated
 		[[ -e $CPUS   ]] && echo "$CPUS: $(cat $CPUS)"
 		[[ -e $XCPUS  ]] && echo "$XCPUS: $(cat $XCPUS)"
 		[[ -e $ECPUS  ]] && echo "$ECPUS: $(cat $ECPUS)"
 		[[ -e $XECPUS ]] && echo "$XECPUS: $(cat $XECPUS)"
 		[[ -e $PRS    ]] && echo "$PRS: $(cat $PRS)"
 		[[ -e $PCPUS  ]] && echo "$PCPUS: $(cat $PCPUS)"
+		[[ -e $ISCPUS ]] && echo "$ISCPUS: $(cat $ISCPUS)"
 	done
 }
 
@@ -656,11 +593,17 @@ check_cgroup_states()
 
 #
 # Get isolated (including offline) CPUs by looking at
-# /sys/kernel/debug/sched/domains and compare that with the expected value.
+# /sys/kernel/debug/sched/domains and *cpuset.cpus.isolated control file,
+# if available, and compare that with the expected value.
 #
-# Note that a sched domain of just 1 CPU will be considered isolated.
+# Note that isolated CPUs from the sched/domains context include offline
+# CPUs as well as CPUs in non-isolated 1-CPU partition. Those CPUs may
+# not be included in the *cpuset.cpus.isolated control file which contains
+# only CPUs in isolated partitions.
 #
-# $1 - expected isolated cpu list
+# $1 - expected isolated cpu list(s) <isolcpus1>{,<isolcpus2>}
+# <isolcpus1> - expected sched/domains value
+# <isolcpus2> - *cpuset.cpus.isolated value = <isolcpus1> if not defined
 #
 check_isolcpus()
 {
@@ -668,8 +611,38 @@ check_isolcpus()
 	ISOLCPUS=
 	LASTISOLCPU=
 	SCHED_DOMAINS=/sys/kernel/debug/sched/domains
+	ISCPUS=${CGROUP2}/.__DEBUG__.cpuset.cpus.isolated
+	if [[ $EXPECT_VAL = . ]]
+	then
+		EXPECT_VAL=
+		EXPECT_VAL2=
+	elif [[ $(expr $EXPECT_VAL : ".*,.*") > 0 ]]
+	then
+		set -- $(echo $EXPECT_VAL | sed -e "s/,/ /g")
+		EXPECT_VAL=$1
+		EXPECT_VAL2=$2
+	else
+		EXPECT_VAL2=$EXPECT_VAL
+	fi
+
+	#
+	# Check the debug isolated cpumask, if present
+	#
+	[[ -f $ISCPUS ]] && {
+		ISOLCPUS=$(cat $ISCPUS)
+		[[ "$EXPECT_VAL2" != "$ISOLCPUS" ]] && {
+			# Take a 50ms pause and try again
+			pause 0.05
+			ISOLCPUS=$(cat $ISCPUS)
+		}
+		[[ "$EXPECT_VAL2" != "$ISOLCPUS" ]] && return 1
+		ISOLCPUS=
+	}
+
+	#
+	# Use the sched domain in debugfs to check isolated CPUs, if available
+	#
 	[[ -d $SCHED_DOMAINS ]] || return 0
-	[[ $EXPECT_VAL = . ]] && EXPECT_VAL=
 
 	for ((CPU=0; CPU < $NR_CPUS; CPU++))
 	do
@@ -713,6 +686,22 @@ test_fail()
 	exit 1
 }
 
+#
+# Check to see if there are unexpected isolated CPUs left
+#
+null_isolcpus_check()
+{
+	[[ $VERBOSE -gt 0 ]] || return 0
+	pause 0.02
+	check_isolcpus "."
+	if [[ $? -ne 0 ]]
+	then
+		echo "Unexpected isolated CPUs: $ISOLCPUS"
+		dump_states
+		exit 1
+	fi
+}
+
 #
 # Run cpuset state transition test
 #  $1 - test matrix name
@@ -798,12 +787,79 @@ run_state_test()
 			echo "Effective cpus changed to $NEWLIST after test $I!"
 			exit 1
 		}
+		null_isolcpus_check
 		[[ $VERBOSE -gt 0 ]] && echo "Test $I done."
 		((I++))
 	done
 	echo "All $I tests of $TEST PASSED."
 }
 
+#
+# Testing the new "isolated" partition root type
+#
+test_isolated()
+{
+	cd $CGROUP2/test
+	echo 2-3 > cpuset.cpus
+	TYPE=$(cat cpuset.cpus.partition)
+	[[ $TYPE = member ]] || echo member > cpuset.cpus.partition
+
+	console_msg "Change from member to root"
+	test_partition root
+
+	console_msg "Change from root to isolated"
+	test_partition isolated
+
+	console_msg "Change from isolated to member"
+	test_partition member
+
+	console_msg "Change from member to isolated"
+	test_partition isolated
+
+	console_msg "Change from isolated to root"
+	test_partition root
+
+	console_msg "Change from root to member"
+	test_partition member
+
+	#
+	# Testing partition root with no cpu
+	#
+	console_msg "Distribute all cpus to child partition"
+	echo +cpuset > cgroup.subtree_control
+	test_partition root
+
+	mkdir A1
+	cd A1
+	echo 2-3 > cpuset.cpus
+	test_partition root
+	test_effective_cpus 2-3
+	cd ..
+	test_effective_cpus ""
+
+	console_msg "Moving task to partition test"
+	test_add_proc "No space left"
+	cd A1
+	test_add_proc ""
+	cd ..
+
+	console_msg "Shrink and expand child partition"
+	cd A1
+	echo 2 > cpuset.cpus
+	cd ..
+	test_effective_cpus 3
+	cd A1
+	echo 2-3 > cpuset.cpus
+	cd ..
+	test_effective_cpus ""
+
+	# Cleaning up
+	console_msg "Cleaning up"
+	echo $$ > $CGROUP2/cgroup.procs
+	[[ -d A1 ]] && rmdir A1
+	null_isolcpus_check
+}
+
 #
 # Wait for inotify event for the given file and read it
 # $1: cgroup file to wait for
diff --git a/tools/testing/selftests/cgroup/test_freezer.c b/tools/testing/selftests/cgroup/test_freezer.c
index ff519029f6f4..8845353aca53 100644
--- a/tools/testing/selftests/cgroup/test_freezer.c
+++ b/tools/testing/selftests/cgroup/test_freezer.c
@@ -740,7 +740,7 @@ static int test_cgfreezer_ptraced(const char *root)
 
 	/*
 	 * cg_check_frozen(cgroup, true) will fail here,
-	 * because the task in in the TRACEd state.
+	 * because the task is in the TRACEd state.
 	 */
 	if (cg_freeze_wait(cgroup, false))
 		goto cleanup;
diff --git a/tools/testing/selftests/drivers/net/mlxsw/pci_reset.sh b/tools/testing/selftests/drivers/net/mlxsw/pci_reset.sh
new file mode 100755
index 000000000000..fe0343b95e6c
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/mlxsw/pci_reset.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# Test that PCI reset works correctly by verifying that only the expected reset
+# methods are supported and that after issuing the reset the ifindex of the
+# port changes.
+
+lib_dir=$(dirname $0)/../../../net/forwarding
+
+ALL_TESTS="
+	pci_reset_test
+"
+NUM_NETIFS=1
+source $lib_dir/lib.sh
+source $lib_dir/devlink_lib.sh
+
+pci_reset_test()
+{
+	RET=0
+
+	local bus=$(echo $DEVLINK_DEV | cut -d '/' -f 1)
+	local bdf=$(echo $DEVLINK_DEV | cut -d '/' -f 2)
+
+	if [ $bus != "pci" ]; then
+		check_err 1 "devlink device is not a PCI device"
+		log_test "pci reset"
+		return
+	fi
+
+	if [ ! -f /sys/bus/pci/devices/$bdf/reset_method ]; then
+		check_err 1 "reset is not supported"
+		log_test "pci reset"
+		return
+	fi
+
+	[[ $(cat /sys/bus/pci/devices/$bdf/reset_method) == "bus" ]]
+	check_err $? "only \"bus\" reset method should be supported"
+
+	local ifindex_pre=$(ip -j link show dev $swp1 | jq '.[]["ifindex"]')
+
+	echo 1 > /sys/bus/pci/devices/$bdf/reset
+	check_err $? "reset failed"
+
+	# Wait for udev to rename newly created netdev.
+	udevadm settle
+
+	local ifindex_post=$(ip -j link show dev $swp1 | jq '.[]["ifindex"]')
+
+	[[ $ifindex_pre != $ifindex_post ]]
+	check_err $? "reset not performed"
+
+	log_test "pci reset"
+}
+
+swp1=${NETIFS[p1]}
+tests_run
+
+exit $EXIT_STATUS
diff --git a/tools/testing/selftests/iommu/iommufd_utils.h b/tools/testing/selftests/iommu/iommufd_utils.h
index 050e9751321c..ad9202335656 100644
--- a/tools/testing/selftests/iommu/iommufd_utils.h
+++ b/tools/testing/selftests/iommu/iommufd_utils.h
@@ -293,15 +293,13 @@ static int _test_mock_dirty_bitmaps(int fd, __u32 hwpt_id, size_t length,
 				    __u64 bitmap_size, __u32 flags,
 				    struct __test_metadata *_metadata)
 {
-	unsigned long i, count, nbits = bitmap_size * BITS_PER_BYTE;
+	unsigned long i, nbits = bitmap_size * BITS_PER_BYTE;
 	unsigned long nr = nbits / 2;
 	__u64 out_dirty = 0;
 
 	/* Mark all even bits as dirty in the mock domain */
-	for (count = 0, i = 0; i < nbits; count += !(i % 2), i++)
-		if (!(i % 2))
-			set_bit(i, (unsigned long *)bitmap);
-	ASSERT_EQ(nr, count);
+	for (i = 0; i < nbits; i += 2)
+		set_bit(i, (unsigned long *)bitmap);
 
 	test_cmd_mock_domain_set_dirty(fd, hwpt_id, length, iova, page_size,
 				       bitmap, &out_dirty);
@@ -311,9 +309,10 @@ static int _test_mock_dirty_bitmaps(int fd, __u32 hwpt_id, size_t length,
 	memset(bitmap, 0, bitmap_size);
 	test_cmd_get_dirty_bitmap(fd, hwpt_id, length, iova, page_size, bitmap,
 				  flags);
-	for (count = 0, i = 0; i < nbits; count += !(i % 2), i++)
+	/* Beware ASSERT_EQ() is two statements -- braces are not redundant! */
+	for (i = 0; i < nbits; i++) {
 		ASSERT_EQ(!(i % 2), test_bit(i, (unsigned long *)bitmap));
-	ASSERT_EQ(count, out_dirty);
+	}
 
 	memset(bitmap, 0, bitmap_size);
 	test_cmd_get_dirty_bitmap(fd, hwpt_id, length, iova, page_size, bitmap,
diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile
index a5963ab9215b..69ce8e06b3a3 100644
--- a/tools/testing/selftests/kvm/Makefile
+++ b/tools/testing/selftests/kvm/Makefile
@@ -91,6 +91,8 @@ TEST_GEN_PROGS_x86_64 += x86_64/monitor_mwait_test
 TEST_GEN_PROGS_x86_64 += x86_64/nested_exceptions_test
 TEST_GEN_PROGS_x86_64 += x86_64/platform_info_test
 TEST_GEN_PROGS_x86_64 += x86_64/pmu_event_filter_test
+TEST_GEN_PROGS_x86_64 += x86_64/private_mem_conversions_test
+TEST_GEN_PROGS_x86_64 += x86_64/private_mem_kvm_exits_test
 TEST_GEN_PROGS_x86_64 += x86_64/set_boot_cpu_id
 TEST_GEN_PROGS_x86_64 += x86_64/set_sregs_test
 TEST_GEN_PROGS_x86_64 += x86_64/smaller_maxphyaddr_emulation_test
@@ -133,6 +135,7 @@ TEST_GEN_PROGS_x86_64 += access_tracking_perf_test
 TEST_GEN_PROGS_x86_64 += demand_paging_test
 TEST_GEN_PROGS_x86_64 += dirty_log_test
 TEST_GEN_PROGS_x86_64 += dirty_log_perf_test
+TEST_GEN_PROGS_x86_64 += guest_memfd_test
 TEST_GEN_PROGS_x86_64 += guest_print_test
 TEST_GEN_PROGS_x86_64 += hardware_disable_test
 TEST_GEN_PROGS_x86_64 += kvm_create_max_vcpus
diff --git a/tools/testing/selftests/kvm/aarch64/page_fault_test.c b/tools/testing/selftests/kvm/aarch64/page_fault_test.c
index eb4217b7c768..08a5ca5bed56 100644
--- a/tools/testing/selftests/kvm/aarch64/page_fault_test.c
+++ b/tools/testing/selftests/kvm/aarch64/page_fault_test.c
@@ -705,7 +705,7 @@ static void run_test(enum vm_guest_mode mode, void *arg)
 
 	print_test_banner(mode, p);
 
-	vm = ____vm_create(mode);
+	vm = ____vm_create(VM_SHAPE(mode));
 	setup_memslots(vm, p);
 	kvm_vm_elf_load(vm, program_invocation_name);
 	setup_ucall(vm);
diff --git a/tools/testing/selftests/kvm/dirty_log_test.c b/tools/testing/selftests/kvm/dirty_log_test.c
index 936f3a8d1b83..6cbecf499767 100644
--- a/tools/testing/selftests/kvm/dirty_log_test.c
+++ b/tools/testing/selftests/kvm/dirty_log_test.c
@@ -699,7 +699,7 @@ static struct kvm_vm *create_vm(enum vm_guest_mode mode, struct kvm_vcpu **vcpu,
 
 	pr_info("Testing guest mode: %s\n", vm_guest_mode_string(mode));
 
-	vm = __vm_create(mode, 1, extra_mem_pages);
+	vm = __vm_create(VM_SHAPE(mode), 1, extra_mem_pages);
 
 	log_mode_create_vm_done(vm);
 	*vcpu = vm_vcpu_add(vm, 0, guest_code);
diff --git a/tools/testing/selftests/kvm/guest_memfd_test.c b/tools/testing/selftests/kvm/guest_memfd_test.c
new file mode 100644
index 000000000000..318ba6ba8bd3
--- /dev/null
+++ b/tools/testing/selftests/kvm/guest_memfd_test.c
@@ -0,0 +1,200 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright Intel Corporation, 2023
+ *
+ * Author: Chao Peng <chao.p.peng@linux.intel.com>
+ */
+
+#define _GNU_SOURCE
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <stdio.h>
+#include <fcntl.h>
+
+#include <linux/bitmap.h>
+#include <linux/falloc.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include "test_util.h"
+#include "kvm_util_base.h"
+
+static void test_file_read_write(int fd)
+{
+	char buf[64];
+
+	TEST_ASSERT(read(fd, buf, sizeof(buf)) < 0,
+		    "read on a guest_mem fd should fail");
+	TEST_ASSERT(write(fd, buf, sizeof(buf)) < 0,
+		    "write on a guest_mem fd should fail");
+	TEST_ASSERT(pread(fd, buf, sizeof(buf), 0) < 0,
+		    "pread on a guest_mem fd should fail");
+	TEST_ASSERT(pwrite(fd, buf, sizeof(buf), 0) < 0,
+		    "pwrite on a guest_mem fd should fail");
+}
+
+static void test_mmap(int fd, size_t page_size)
+{
+	char *mem;
+
+	mem = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+	TEST_ASSERT_EQ(mem, MAP_FAILED);
+}
+
+static void test_file_size(int fd, size_t page_size, size_t total_size)
+{
+	struct stat sb;
+	int ret;
+
+	ret = fstat(fd, &sb);
+	TEST_ASSERT(!ret, "fstat should succeed");
+	TEST_ASSERT_EQ(sb.st_size, total_size);
+	TEST_ASSERT_EQ(sb.st_blksize, page_size);
+}
+
+static void test_fallocate(int fd, size_t page_size, size_t total_size)
+{
+	int ret;
+
+	ret = fallocate(fd, FALLOC_FL_KEEP_SIZE, 0, total_size);
+	TEST_ASSERT(!ret, "fallocate with aligned offset and size should succeed");
+
+	ret = fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE,
+			page_size - 1, page_size);
+	TEST_ASSERT(ret, "fallocate with unaligned offset should fail");
+
+	ret = fallocate(fd, FALLOC_FL_KEEP_SIZE, total_size, page_size);
+	TEST_ASSERT(ret, "fallocate beginning at total_size should fail");
+
+	ret = fallocate(fd, FALLOC_FL_KEEP_SIZE, total_size + page_size, page_size);
+	TEST_ASSERT(ret, "fallocate beginning after total_size should fail");
+
+	ret = fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE,
+			total_size, page_size);
+	TEST_ASSERT(!ret, "fallocate(PUNCH_HOLE) at total_size should succeed");
+
+	ret = fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE,
+			total_size + page_size, page_size);
+	TEST_ASSERT(!ret, "fallocate(PUNCH_HOLE) after total_size should succeed");
+
+	ret = fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE,
+			page_size, page_size - 1);
+	TEST_ASSERT(ret, "fallocate with unaligned size should fail");
+
+	ret = fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE,
+			page_size, page_size);
+	TEST_ASSERT(!ret, "fallocate(PUNCH_HOLE) with aligned offset and size should succeed");
+
+	ret = fallocate(fd, FALLOC_FL_KEEP_SIZE, page_size, page_size);
+	TEST_ASSERT(!ret, "fallocate to restore punched hole should succeed");
+}
+
+static void test_invalid_punch_hole(int fd, size_t page_size, size_t total_size)
+{
+	struct {
+		off_t offset;
+		off_t len;
+	} testcases[] = {
+		{0, 1},
+		{0, page_size - 1},
+		{0, page_size + 1},
+
+		{1, 1},
+		{1, page_size - 1},
+		{1, page_size},
+		{1, page_size + 1},
+
+		{page_size, 1},
+		{page_size, page_size - 1},
+		{page_size, page_size + 1},
+	};
+	int ret, i;
+
+	for (i = 0; i < ARRAY_SIZE(testcases); i++) {
+		ret = fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE,
+				testcases[i].offset, testcases[i].len);
+		TEST_ASSERT(ret == -1 && errno == EINVAL,
+			    "PUNCH_HOLE with !PAGE_SIZE offset (%lx) and/or length (%lx) should fail",
+			    testcases[i].offset, testcases[i].len);
+	}
+}
+
+static void test_create_guest_memfd_invalid(struct kvm_vm *vm)
+{
+	size_t page_size = getpagesize();
+	uint64_t flag;
+	size_t size;
+	int fd;
+
+	for (size = 1; size < page_size; size++) {
+		fd = __vm_create_guest_memfd(vm, size, 0);
+		TEST_ASSERT(fd == -1 && errno == EINVAL,
+			    "guest_memfd() with non-page-aligned page size '0x%lx' should fail with EINVAL",
+			    size);
+	}
+
+	for (flag = 0; flag; flag <<= 1) {
+		uint64_t bit;
+
+		fd = __vm_create_guest_memfd(vm, page_size, flag);
+		TEST_ASSERT(fd == -1 && errno == EINVAL,
+			    "guest_memfd() with flag '0x%lx' should fail with EINVAL",
+			    flag);
+	}
+}
+
+static void test_create_guest_memfd_multiple(struct kvm_vm *vm)
+{
+	int fd1, fd2, ret;
+	struct stat st1, st2;
+
+	fd1 = __vm_create_guest_memfd(vm, 4096, 0);
+	TEST_ASSERT(fd1 != -1, "memfd creation should succeed");
+
+	ret = fstat(fd1, &st1);
+	TEST_ASSERT(ret != -1, "memfd fstat should succeed");
+	TEST_ASSERT(st1.st_size == 4096, "memfd st_size should match requested size");
+
+	fd2 = __vm_create_guest_memfd(vm, 8192, 0);
+	TEST_ASSERT(fd2 != -1, "memfd creation should succeed");
+
+	ret = fstat(fd2, &st2);
+	TEST_ASSERT(ret != -1, "memfd fstat should succeed");
+	TEST_ASSERT(st2.st_size == 8192, "second memfd st_size should match requested size");
+
+	ret = fstat(fd1, &st1);
+	TEST_ASSERT(ret != -1, "memfd fstat should succeed");
+	TEST_ASSERT(st1.st_size == 4096, "first memfd st_size should still match requested size");
+	TEST_ASSERT(st1.st_ino != st2.st_ino, "different memfd should have different inode numbers");
+}
+
+int main(int argc, char *argv[])
+{
+	size_t page_size;
+	size_t total_size;
+	int fd;
+	struct kvm_vm *vm;
+
+	TEST_REQUIRE(kvm_has_cap(KVM_CAP_GUEST_MEMFD));
+
+	page_size = getpagesize();
+	total_size = page_size * 4;
+
+	vm = vm_create_barebones();
+
+	test_create_guest_memfd_invalid(vm);
+	test_create_guest_memfd_multiple(vm);
+
+	fd = vm_create_guest_memfd(vm, total_size, 0);
+
+	test_file_read_write(fd);
+	test_mmap(fd, page_size);
+	test_file_size(fd, page_size, total_size);
+	test_fallocate(fd, page_size, total_size);
+	test_invalid_punch_hole(fd, page_size, total_size);
+
+	close(fd);
+}
diff --git a/tools/testing/selftests/kvm/include/kvm_util_base.h b/tools/testing/selftests/kvm/include/kvm_util_base.h
index a18db6a7b3cf..1b58f943562f 100644
--- a/tools/testing/selftests/kvm/include/kvm_util_base.h
+++ b/tools/testing/selftests/kvm/include/kvm_util_base.h
@@ -44,7 +44,7 @@ typedef uint64_t vm_paddr_t; /* Virtual Machine (Guest) physical address */
 typedef uint64_t vm_vaddr_t; /* Virtual Machine (Guest) virtual address */
 
 struct userspace_mem_region {
-	struct kvm_userspace_memory_region region;
+	struct kvm_userspace_memory_region2 region;
 	struct sparsebit *unused_phy_pages;
 	int fd;
 	off_t offset;
@@ -188,6 +188,23 @@ enum vm_guest_mode {
 	NUM_VM_MODES,
 };
 
+struct vm_shape {
+	enum vm_guest_mode mode;
+	unsigned int type;
+};
+
+#define VM_TYPE_DEFAULT			0
+
+#define VM_SHAPE(__mode)			\
+({						\
+	struct vm_shape shape = {		\
+		.mode = (__mode),		\
+		.type = VM_TYPE_DEFAULT		\
+	};					\
+						\
+	shape;					\
+})
+
 #if defined(__aarch64__)
 
 extern enum vm_guest_mode vm_mode_default;
@@ -220,6 +237,8 @@ extern enum vm_guest_mode vm_mode_default;
 
 #endif
 
+#define VM_SHAPE_DEFAULT	VM_SHAPE(VM_MODE_DEFAULT)
+
 #define MIN_PAGE_SIZE		(1U << MIN_PAGE_SHIFT)
 #define PTES_PER_MIN_PAGE	ptes_per_page(MIN_PAGE_SIZE)
 
@@ -333,6 +352,54 @@ static inline void vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0)
 	vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap);
 }
 
+static inline void vm_set_memory_attributes(struct kvm_vm *vm, uint64_t gpa,
+					    uint64_t size, uint64_t attributes)
+{
+	struct kvm_memory_attributes attr = {
+		.attributes = attributes,
+		.address = gpa,
+		.size = size,
+		.flags = 0,
+	};
+
+	/*
+	 * KVM_SET_MEMORY_ATTRIBUTES overwrites _all_ attributes.  These flows
+	 * need significant enhancements to support multiple attributes.
+	 */
+	TEST_ASSERT(!attributes || attributes == KVM_MEMORY_ATTRIBUTE_PRIVATE,
+		    "Update me to support multiple attributes!");
+
+	vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr);
+}
+
+
+static inline void vm_mem_set_private(struct kvm_vm *vm, uint64_t gpa,
+				      uint64_t size)
+{
+	vm_set_memory_attributes(vm, gpa, size, KVM_MEMORY_ATTRIBUTE_PRIVATE);
+}
+
+static inline void vm_mem_set_shared(struct kvm_vm *vm, uint64_t gpa,
+				     uint64_t size)
+{
+	vm_set_memory_attributes(vm, gpa, size, 0);
+}
+
+void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t gpa, uint64_t size,
+			    bool punch_hole);
+
+static inline void vm_guest_mem_punch_hole(struct kvm_vm *vm, uint64_t gpa,
+					   uint64_t size)
+{
+	vm_guest_mem_fallocate(vm, gpa, size, true);
+}
+
+static inline void vm_guest_mem_allocate(struct kvm_vm *vm, uint64_t gpa,
+					 uint64_t size)
+{
+	vm_guest_mem_fallocate(vm, gpa, size, false);
+}
+
 void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size);
 const char *vm_guest_mode_string(uint32_t i);
 
@@ -431,14 +498,44 @@ static inline uint64_t vm_get_stat(struct kvm_vm *vm, const char *stat_name)
 
 void vm_create_irqchip(struct kvm_vm *vm);
 
+static inline int __vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size,
+					uint64_t flags)
+{
+	struct kvm_create_guest_memfd guest_memfd = {
+		.size = size,
+		.flags = flags,
+	};
+
+	return __vm_ioctl(vm, KVM_CREATE_GUEST_MEMFD, &guest_memfd);
+}
+
+static inline int vm_create_guest_memfd(struct kvm_vm *vm, uint64_t size,
+					uint64_t flags)
+{
+	int fd = __vm_create_guest_memfd(vm, size, flags);
+
+	TEST_ASSERT(fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_GUEST_MEMFD, fd));
+	return fd;
+}
+
 void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
 			       uint64_t gpa, uint64_t size, void *hva);
 int __vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
 				uint64_t gpa, uint64_t size, void *hva);
+void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
+				uint64_t gpa, uint64_t size, void *hva,
+				uint32_t guest_memfd, uint64_t guest_memfd_offset);
+int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
+				 uint64_t gpa, uint64_t size, void *hva,
+				 uint32_t guest_memfd, uint64_t guest_memfd_offset);
+
 void vm_userspace_mem_region_add(struct kvm_vm *vm,
 	enum vm_mem_backing_src_type src_type,
 	uint64_t guest_paddr, uint32_t slot, uint64_t npages,
 	uint32_t flags);
+void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
+		uint64_t guest_paddr, uint32_t slot, uint64_t npages,
+		uint32_t flags, int guest_memfd_fd, uint64_t guest_memfd_offset);
 
 void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags);
 void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa);
@@ -713,21 +810,33 @@ vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm);
  * __vm_create() does NOT create vCPUs, @nr_runnable_vcpus is used purely to
  * calculate the amount of memory needed for per-vCPU data, e.g. stacks.
  */
-struct kvm_vm *____vm_create(enum vm_guest_mode mode);
-struct kvm_vm *__vm_create(enum vm_guest_mode mode, uint32_t nr_runnable_vcpus,
+struct kvm_vm *____vm_create(struct vm_shape shape);
+struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus,
 			   uint64_t nr_extra_pages);
 
 static inline struct kvm_vm *vm_create_barebones(void)
 {
-	return ____vm_create(VM_MODE_DEFAULT);
+	return ____vm_create(VM_SHAPE_DEFAULT);
 }
 
+#ifdef __x86_64__
+static inline struct kvm_vm *vm_create_barebones_protected_vm(void)
+{
+	const struct vm_shape shape = {
+		.mode = VM_MODE_DEFAULT,
+		.type = KVM_X86_SW_PROTECTED_VM,
+	};
+
+	return ____vm_create(shape);
+}
+#endif
+
 static inline struct kvm_vm *vm_create(uint32_t nr_runnable_vcpus)
 {
-	return __vm_create(VM_MODE_DEFAULT, nr_runnable_vcpus, 0);
+	return __vm_create(VM_SHAPE_DEFAULT, nr_runnable_vcpus, 0);
 }
 
-struct kvm_vm *__vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus,
+struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus,
 				      uint64_t extra_mem_pages,
 				      void *guest_code, struct kvm_vcpu *vcpus[]);
 
@@ -735,17 +844,27 @@ static inline struct kvm_vm *vm_create_with_vcpus(uint32_t nr_vcpus,
 						  void *guest_code,
 						  struct kvm_vcpu *vcpus[])
 {
-	return __vm_create_with_vcpus(VM_MODE_DEFAULT, nr_vcpus, 0,
+	return __vm_create_with_vcpus(VM_SHAPE_DEFAULT, nr_vcpus, 0,
 				      guest_code, vcpus);
 }
 
+
+struct kvm_vm *__vm_create_shape_with_one_vcpu(struct vm_shape shape,
+					       struct kvm_vcpu **vcpu,
+					       uint64_t extra_mem_pages,
+					       void *guest_code);
+
 /*
  * Create a VM with a single vCPU with reasonable defaults and @extra_mem_pages
  * additional pages of guest memory.  Returns the VM and vCPU (via out param).
  */
-struct kvm_vm *__vm_create_with_one_vcpu(struct kvm_vcpu **vcpu,
-					 uint64_t extra_mem_pages,
-					 void *guest_code);
+static inline struct kvm_vm *__vm_create_with_one_vcpu(struct kvm_vcpu **vcpu,
+						       uint64_t extra_mem_pages,
+						       void *guest_code)
+{
+	return __vm_create_shape_with_one_vcpu(VM_SHAPE_DEFAULT, vcpu,
+					       extra_mem_pages, guest_code);
+}
 
 static inline struct kvm_vm *vm_create_with_one_vcpu(struct kvm_vcpu **vcpu,
 						     void *guest_code)
@@ -753,6 +872,13 @@ static inline struct kvm_vm *vm_create_with_one_vcpu(struct kvm_vcpu **vcpu,
 	return __vm_create_with_one_vcpu(vcpu, 0, guest_code);
 }
 
+static inline struct kvm_vm *vm_create_shape_with_one_vcpu(struct vm_shape shape,
+							   struct kvm_vcpu **vcpu,
+							   void *guest_code)
+{
+	return __vm_create_shape_with_one_vcpu(shape, vcpu, 0, guest_code);
+}
+
 struct kvm_vcpu *vm_recreate_with_one_vcpu(struct kvm_vm *vm);
 
 void kvm_pin_this_task_to_pcpu(uint32_t pcpu);
@@ -776,10 +902,6 @@ vm_adjust_num_guest_pages(enum vm_guest_mode mode, unsigned int num_guest_pages)
 	return n;
 }
 
-struct kvm_userspace_memory_region *
-kvm_userspace_memory_region_find(struct kvm_vm *vm, uint64_t start,
-				 uint64_t end);
-
 #define sync_global_to_guest(vm, g) ({				\
 	typeof(g) *_p = addr_gva2hva(vm, (vm_vaddr_t)&(g));	\
 	memcpy(_p, &(g), sizeof(g));				\
diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h
index 8e5f413a593d..a0c7dd3a5b30 100644
--- a/tools/testing/selftests/kvm/include/test_util.h
+++ b/tools/testing/selftests/kvm/include/test_util.h
@@ -142,6 +142,11 @@ static inline bool backing_src_is_shared(enum vm_mem_backing_src_type t)
 	return vm_mem_backing_src_alias(t)->flag & MAP_SHARED;
 }
 
+static inline bool backing_src_can_be_huge(enum vm_mem_backing_src_type t)
+{
+	return t != VM_MEM_SRC_ANONYMOUS && t != VM_MEM_SRC_SHMEM;
+}
+
 /* Aligns x up to the next multiple of size. Size must be a power of 2. */
 static inline uint64_t align_up(uint64_t x, uint64_t size)
 {
diff --git a/tools/testing/selftests/kvm/include/ucall_common.h b/tools/testing/selftests/kvm/include/ucall_common.h
index ce33d306c2cb..0fb472a5a058 100644
--- a/tools/testing/selftests/kvm/include/ucall_common.h
+++ b/tools/testing/selftests/kvm/include/ucall_common.h
@@ -52,6 +52,17 @@ int ucall_nr_pages_required(uint64_t page_size);
 #define GUEST_SYNC_ARGS(stage, arg1, arg2, arg3, arg4)	\
 				ucall(UCALL_SYNC, 6, "hello", stage, arg1, arg2, arg3, arg4)
 #define GUEST_SYNC(stage)	ucall(UCALL_SYNC, 2, "hello", stage)
+#define GUEST_SYNC1(arg0)	ucall(UCALL_SYNC, 1, arg0)
+#define GUEST_SYNC2(arg0, arg1)	ucall(UCALL_SYNC, 2, arg0, arg1)
+#define GUEST_SYNC3(arg0, arg1, arg2) \
+				ucall(UCALL_SYNC, 3, arg0, arg1, arg2)
+#define GUEST_SYNC4(arg0, arg1, arg2, arg3) \
+				ucall(UCALL_SYNC, 4, arg0, arg1, arg2, arg3)
+#define GUEST_SYNC5(arg0, arg1, arg2, arg3, arg4) \
+				ucall(UCALL_SYNC, 5, arg0, arg1, arg2, arg3, arg4)
+#define GUEST_SYNC6(arg0, arg1, arg2, arg3, arg4, arg5) \
+				ucall(UCALL_SYNC, 6, arg0, arg1, arg2, arg3, arg4, arg5)
+
 #define GUEST_PRINTF(_fmt, _args...) ucall_fmt(UCALL_PRINTF, _fmt, ##_args)
 #define GUEST_DONE()		ucall(UCALL_DONE, 0)
 
diff --git a/tools/testing/selftests/kvm/include/x86_64/processor.h b/tools/testing/selftests/kvm/include/x86_64/processor.h
index 25bc61dac5fb..a84863503fcb 100644
--- a/tools/testing/selftests/kvm/include/x86_64/processor.h
+++ b/tools/testing/selftests/kvm/include/x86_64/processor.h
@@ -15,6 +15,7 @@
 #include <asm/msr-index.h>
 #include <asm/prctl.h>
 
+#include <linux/kvm_para.h>
 #include <linux/stringify.h>
 
 #include "../kvm_util.h"
@@ -1194,6 +1195,20 @@ uint64_t kvm_hypercall(uint64_t nr, uint64_t a0, uint64_t a1, uint64_t a2,
 uint64_t __xen_hypercall(uint64_t nr, uint64_t a0, void *a1);
 void xen_hypercall(uint64_t nr, uint64_t a0, void *a1);
 
+static inline uint64_t __kvm_hypercall_map_gpa_range(uint64_t gpa,
+						     uint64_t size, uint64_t flags)
+{
+	return kvm_hypercall(KVM_HC_MAP_GPA_RANGE, gpa, size >> PAGE_SHIFT, flags, 0);
+}
+
+static inline void kvm_hypercall_map_gpa_range(uint64_t gpa, uint64_t size,
+					       uint64_t flags)
+{
+	uint64_t ret = __kvm_hypercall_map_gpa_range(gpa, size, flags);
+
+	GUEST_ASSERT(!ret);
+}
+
 void __vm_xsave_require_permission(uint64_t xfeature, const char *name);
 
 #define vm_xsave_require_permission(xfeature)	\
diff --git a/tools/testing/selftests/kvm/kvm_page_table_test.c b/tools/testing/selftests/kvm/kvm_page_table_test.c
index 69f26d80c821..e37dc9c21888 100644
--- a/tools/testing/selftests/kvm/kvm_page_table_test.c
+++ b/tools/testing/selftests/kvm/kvm_page_table_test.c
@@ -254,7 +254,7 @@ static struct kvm_vm *pre_init_before_test(enum vm_guest_mode mode, void *arg)
 
 	/* Create a VM with enough guest pages */
 	guest_num_pages = test_mem_size / guest_page_size;
-	vm = __vm_create_with_vcpus(mode, nr_vcpus, guest_num_pages,
+	vm = __vm_create_with_vcpus(VM_SHAPE(mode), nr_vcpus, guest_num_pages,
 				    guest_code, test_args.vcpus);
 
 	/* Align down GPA of the testing memslot */
diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c
index 7a8af1821f5d..9b29cbf49476 100644
--- a/tools/testing/selftests/kvm/lib/kvm_util.c
+++ b/tools/testing/selftests/kvm/lib/kvm_util.c
@@ -209,7 +209,7 @@ __weak void vm_vaddr_populate_bitmap(struct kvm_vm *vm)
 		(1ULL << (vm->va_bits - 1)) >> vm->page_shift);
 }
 
-struct kvm_vm *____vm_create(enum vm_guest_mode mode)
+struct kvm_vm *____vm_create(struct vm_shape shape)
 {
 	struct kvm_vm *vm;
 
@@ -221,13 +221,13 @@ struct kvm_vm *____vm_create(enum vm_guest_mode mode)
 	vm->regions.hva_tree = RB_ROOT;
 	hash_init(vm->regions.slot_hash);
 
-	vm->mode = mode;
-	vm->type = 0;
+	vm->mode = shape.mode;
+	vm->type = shape.type;
 
-	vm->pa_bits = vm_guest_mode_params[mode].pa_bits;
-	vm->va_bits = vm_guest_mode_params[mode].va_bits;
-	vm->page_size = vm_guest_mode_params[mode].page_size;
-	vm->page_shift = vm_guest_mode_params[mode].page_shift;
+	vm->pa_bits = vm_guest_mode_params[vm->mode].pa_bits;
+	vm->va_bits = vm_guest_mode_params[vm->mode].va_bits;
+	vm->page_size = vm_guest_mode_params[vm->mode].page_size;
+	vm->page_shift = vm_guest_mode_params[vm->mode].page_shift;
 
 	/* Setup mode specific traits. */
 	switch (vm->mode) {
@@ -265,7 +265,7 @@ struct kvm_vm *____vm_create(enum vm_guest_mode mode)
 		/*
 		 * Ignore KVM support for 5-level paging (vm->va_bits == 57),
 		 * it doesn't take effect unless a CR4.LA57 is set, which it
-		 * isn't for this VM_MODE.
+		 * isn't for this mode (48-bit virtual address space).
 		 */
 		TEST_ASSERT(vm->va_bits == 48 || vm->va_bits == 57,
 			    "Linear address width (%d bits) not supported",
@@ -285,10 +285,11 @@ struct kvm_vm *____vm_create(enum vm_guest_mode mode)
 		vm->pgtable_levels = 5;
 		break;
 	default:
-		TEST_FAIL("Unknown guest mode, mode: 0x%x", mode);
+		TEST_FAIL("Unknown guest mode: 0x%x", vm->mode);
 	}
 
 #ifdef __aarch64__
+	TEST_ASSERT(!vm->type, "ARM doesn't support test-provided types");
 	if (vm->pa_bits != 40)
 		vm->type = KVM_VM_TYPE_ARM_IPA_SIZE(vm->pa_bits);
 #endif
@@ -347,19 +348,19 @@ static uint64_t vm_nr_pages_required(enum vm_guest_mode mode,
 	return vm_adjust_num_guest_pages(mode, nr_pages);
 }
 
-struct kvm_vm *__vm_create(enum vm_guest_mode mode, uint32_t nr_runnable_vcpus,
+struct kvm_vm *__vm_create(struct vm_shape shape, uint32_t nr_runnable_vcpus,
 			   uint64_t nr_extra_pages)
 {
-	uint64_t nr_pages = vm_nr_pages_required(mode, nr_runnable_vcpus,
+	uint64_t nr_pages = vm_nr_pages_required(shape.mode, nr_runnable_vcpus,
 						 nr_extra_pages);
 	struct userspace_mem_region *slot0;
 	struct kvm_vm *vm;
 	int i;
 
-	pr_debug("%s: mode='%s' pages='%ld'\n", __func__,
-		 vm_guest_mode_string(mode), nr_pages);
+	pr_debug("%s: mode='%s' type='%d', pages='%ld'\n", __func__,
+		 vm_guest_mode_string(shape.mode), shape.type, nr_pages);
 
-	vm = ____vm_create(mode);
+	vm = ____vm_create(shape);
 
 	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, nr_pages, 0);
 	for (i = 0; i < NR_MEM_REGIONS; i++)
@@ -400,7 +401,7 @@ struct kvm_vm *__vm_create(enum vm_guest_mode mode, uint32_t nr_runnable_vcpus,
  * extra_mem_pages is only used to calculate the maximum page table size,
  * no real memory allocation for non-slot0 memory in this function.
  */
-struct kvm_vm *__vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus,
+struct kvm_vm *__vm_create_with_vcpus(struct vm_shape shape, uint32_t nr_vcpus,
 				      uint64_t extra_mem_pages,
 				      void *guest_code, struct kvm_vcpu *vcpus[])
 {
@@ -409,7 +410,7 @@ struct kvm_vm *__vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus
 
 	TEST_ASSERT(!nr_vcpus || vcpus, "Must provide vCPU array");
 
-	vm = __vm_create(mode, nr_vcpus, extra_mem_pages);
+	vm = __vm_create(shape, nr_vcpus, extra_mem_pages);
 
 	for (i = 0; i < nr_vcpus; ++i)
 		vcpus[i] = vm_vcpu_add(vm, i, guest_code);
@@ -417,15 +418,15 @@ struct kvm_vm *__vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus
 	return vm;
 }
 
-struct kvm_vm *__vm_create_with_one_vcpu(struct kvm_vcpu **vcpu,
-					 uint64_t extra_mem_pages,
-					 void *guest_code)
+struct kvm_vm *__vm_create_shape_with_one_vcpu(struct vm_shape shape,
+					       struct kvm_vcpu **vcpu,
+					       uint64_t extra_mem_pages,
+					       void *guest_code)
 {
 	struct kvm_vcpu *vcpus[1];
 	struct kvm_vm *vm;
 
-	vm = __vm_create_with_vcpus(VM_MODE_DEFAULT, 1, extra_mem_pages,
-				    guest_code, vcpus);
+	vm = __vm_create_with_vcpus(shape, 1, extra_mem_pages, guest_code, vcpus);
 
 	*vcpu = vcpus[0];
 	return vm;
@@ -453,8 +454,9 @@ void kvm_vm_restart(struct kvm_vm *vmp)
 		vm_create_irqchip(vmp);
 
 	hash_for_each(vmp->regions.slot_hash, ctr, region, slot_node) {
-		int ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION, &region->region);
-		TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
+		int ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION2, &region->region);
+
+		TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n"
 			    "  rc: %i errno: %i\n"
 			    "  slot: %u flags: 0x%x\n"
 			    "  guest_phys_addr: 0x%llx size: 0x%llx",
@@ -590,35 +592,6 @@ userspace_mem_region_find(struct kvm_vm *vm, uint64_t start, uint64_t end)
 	return NULL;
 }
 
-/*
- * KVM Userspace Memory Region Find
- *
- * Input Args:
- *   vm - Virtual Machine
- *   start - Starting VM physical address
- *   end - Ending VM physical address, inclusive.
- *
- * Output Args: None
- *
- * Return:
- *   Pointer to overlapping region, NULL if no such region.
- *
- * Public interface to userspace_mem_region_find. Allows tests to look up
- * the memslot datastructure for a given range of guest physical memory.
- */
-struct kvm_userspace_memory_region *
-kvm_userspace_memory_region_find(struct kvm_vm *vm, uint64_t start,
-				 uint64_t end)
-{
-	struct userspace_mem_region *region;
-
-	region = userspace_mem_region_find(vm, start, end);
-	if (!region)
-		return NULL;
-
-	return &region->region;
-}
-
 __weak void vcpu_arch_free(struct kvm_vcpu *vcpu)
 {
 
@@ -686,7 +659,7 @@ static void __vm_mem_region_delete(struct kvm_vm *vm,
 	}
 
 	region->region.memory_size = 0;
-	vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION, &region->region);
+	vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, &region->region);
 
 	sparsebit_free(&region->unused_phy_pages);
 	ret = munmap(region->mmap_start, region->mmap_size);
@@ -697,6 +670,8 @@ static void __vm_mem_region_delete(struct kvm_vm *vm,
 		TEST_ASSERT(!ret, __KVM_SYSCALL_ERROR("munmap()", ret));
 		close(region->fd);
 	}
+	if (region->region.guest_memfd >= 0)
+		close(region->region.guest_memfd);
 
 	free(region);
 }
@@ -898,36 +873,44 @@ void vm_set_user_memory_region(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
 		    errno, strerror(errno));
 }
 
-/*
- * VM Userspace Memory Region Add
- *
- * Input Args:
- *   vm - Virtual Machine
- *   src_type - Storage source for this region.
- *              NULL to use anonymous memory.
- *   guest_paddr - Starting guest physical address
- *   slot - KVM region slot
- *   npages - Number of physical pages
- *   flags - KVM memory region flags (e.g. KVM_MEM_LOG_DIRTY_PAGES)
- *
- * Output Args: None
- *
- * Return: None
- *
- * Allocates a memory area of the number of pages specified by npages
- * and maps it to the VM specified by vm, at a starting physical address
- * given by guest_paddr.  The region is created with a KVM region slot
- * given by slot, which must be unique and < KVM_MEM_SLOTS_NUM.  The
- * region is created with the flags given by flags.
- */
-void vm_userspace_mem_region_add(struct kvm_vm *vm,
-	enum vm_mem_backing_src_type src_type,
-	uint64_t guest_paddr, uint32_t slot, uint64_t npages,
-	uint32_t flags)
+int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
+				 uint64_t gpa, uint64_t size, void *hva,
+				 uint32_t guest_memfd, uint64_t guest_memfd_offset)
+{
+	struct kvm_userspace_memory_region2 region = {
+		.slot = slot,
+		.flags = flags,
+		.guest_phys_addr = gpa,
+		.memory_size = size,
+		.userspace_addr = (uintptr_t)hva,
+		.guest_memfd = guest_memfd,
+		.guest_memfd_offset = guest_memfd_offset,
+	};
+
+	return ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION2, &region);
+}
+
+void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags,
+				uint64_t gpa, uint64_t size, void *hva,
+				uint32_t guest_memfd, uint64_t guest_memfd_offset)
+{
+	int ret = __vm_set_user_memory_region2(vm, slot, flags, gpa, size, hva,
+					       guest_memfd, guest_memfd_offset);
+
+	TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION2 failed, errno = %d (%s)",
+		    errno, strerror(errno));
+}
+
+
+/* FIXME: This thing needs to be ripped apart and rewritten. */
+void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
+		uint64_t guest_paddr, uint32_t slot, uint64_t npages,
+		uint32_t flags, int guest_memfd, uint64_t guest_memfd_offset)
 {
 	int ret;
 	struct userspace_mem_region *region;
 	size_t backing_src_pagesz = get_backing_src_pagesz(src_type);
+	size_t mem_size = npages * vm->page_size;
 	size_t alignment;
 
 	TEST_ASSERT(vm_adjust_num_guest_pages(vm->mode, npages) == npages,
@@ -980,7 +963,7 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm,
 	/* Allocate and initialize new mem region structure. */
 	region = calloc(1, sizeof(*region));
 	TEST_ASSERT(region != NULL, "Insufficient Memory");
-	region->mmap_size = npages * vm->page_size;
+	region->mmap_size = mem_size;
 
 #ifdef __s390x__
 	/* On s390x, the host address must be aligned to 1M (due to PGSTEs) */
@@ -1027,14 +1010,38 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm,
 	/* As needed perform madvise */
 	if ((src_type == VM_MEM_SRC_ANONYMOUS ||
 	     src_type == VM_MEM_SRC_ANONYMOUS_THP) && thp_configured()) {
-		ret = madvise(region->host_mem, npages * vm->page_size,
+		ret = madvise(region->host_mem, mem_size,
 			      src_type == VM_MEM_SRC_ANONYMOUS ? MADV_NOHUGEPAGE : MADV_HUGEPAGE);
 		TEST_ASSERT(ret == 0, "madvise failed, addr: %p length: 0x%lx src_type: %s",
-			    region->host_mem, npages * vm->page_size,
+			    region->host_mem, mem_size,
 			    vm_mem_backing_src_alias(src_type)->name);
 	}
 
 	region->backing_src_type = src_type;
+
+	if (flags & KVM_MEM_GUEST_MEMFD) {
+		if (guest_memfd < 0) {
+			uint32_t guest_memfd_flags = 0;
+			TEST_ASSERT(!guest_memfd_offset,
+				    "Offset must be zero when creating new guest_memfd");
+			guest_memfd = vm_create_guest_memfd(vm, mem_size, guest_memfd_flags);
+		} else {
+			/*
+			 * Install a unique fd for each memslot so that the fd
+			 * can be closed when the region is deleted without
+			 * needing to track if the fd is owned by the framework
+			 * or by the caller.
+			 */
+			guest_memfd = dup(guest_memfd);
+			TEST_ASSERT(guest_memfd >= 0, __KVM_SYSCALL_ERROR("dup()", guest_memfd));
+		}
+
+		region->region.guest_memfd = guest_memfd;
+		region->region.guest_memfd_offset = guest_memfd_offset;
+	} else {
+		region->region.guest_memfd = -1;
+	}
+
 	region->unused_phy_pages = sparsebit_alloc();
 	sparsebit_set_num(region->unused_phy_pages,
 		guest_paddr >> vm->page_shift, npages);
@@ -1043,13 +1050,14 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm,
 	region->region.guest_phys_addr = guest_paddr;
 	region->region.memory_size = npages * vm->page_size;
 	region->region.userspace_addr = (uintptr_t) region->host_mem;
-	ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION, &region->region);
-	TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
+	ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, &region->region);
+	TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n"
 		"  rc: %i errno: %i\n"
 		"  slot: %u flags: 0x%x\n"
-		"  guest_phys_addr: 0x%lx size: 0x%lx",
+		"  guest_phys_addr: 0x%lx size: 0x%lx guest_memfd: %d\n",
 		ret, errno, slot, flags,
-		guest_paddr, (uint64_t) region->region.memory_size);
+		guest_paddr, (uint64_t) region->region.memory_size,
+		region->region.guest_memfd);
 
 	/* Add to quick lookup data structures */
 	vm_userspace_mem_region_gpa_insert(&vm->regions.gpa_tree, region);
@@ -1070,6 +1078,14 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm,
 	}
 }
 
+void vm_userspace_mem_region_add(struct kvm_vm *vm,
+				 enum vm_mem_backing_src_type src_type,
+				 uint64_t guest_paddr, uint32_t slot,
+				 uint64_t npages, uint32_t flags)
+{
+	vm_mem_add(vm, src_type, guest_paddr, slot, npages, flags, -1, 0);
+}
+
 /*
  * Memslot to region
  *
@@ -1126,9 +1142,9 @@ void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags)
 
 	region->region.flags = flags;
 
-	ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION, &region->region);
+	ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, &region->region);
 
-	TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
+	TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n"
 		"  rc: %i errno: %i slot: %u flags: 0x%x",
 		ret, errno, slot, flags);
 }
@@ -1156,9 +1172,9 @@ void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa)
 
 	region->region.guest_phys_addr = new_gpa;
 
-	ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION, &region->region);
+	ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, &region->region);
 
-	TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION failed\n"
+	TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION2 failed\n"
 		    "ret: %i errno: %i slot: %u new_gpa: 0x%lx",
 		    ret, errno, slot, new_gpa);
 }
@@ -1181,6 +1197,34 @@ void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot)
 	__vm_mem_region_delete(vm, memslot2region(vm, slot), true);
 }
 
+void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t base, uint64_t size,
+			    bool punch_hole)
+{
+	const int mode = FALLOC_FL_KEEP_SIZE | (punch_hole ? FALLOC_FL_PUNCH_HOLE : 0);
+	struct userspace_mem_region *region;
+	uint64_t end = base + size;
+	uint64_t gpa, len;
+	off_t fd_offset;
+	int ret;
+
+	for (gpa = base; gpa < end; gpa += len) {
+		uint64_t offset;
+
+		region = userspace_mem_region_find(vm, gpa, gpa);
+		TEST_ASSERT(region && region->region.flags & KVM_MEM_GUEST_MEMFD,
+			    "Private memory region not found for GPA 0x%lx", gpa);
+
+		offset = gpa - region->region.guest_phys_addr;
+		fd_offset = region->region.guest_memfd_offset + offset;
+		len = min_t(uint64_t, end - gpa, region->region.memory_size - offset);
+
+		ret = fallocate(region->region.guest_memfd, mode, fd_offset, len);
+		TEST_ASSERT(!ret, "fallocate() failed to %s at %lx (len = %lu), fd = %d, mode = %x, offset = %lx\n",
+			    punch_hole ? "punch hole" : "allocate", gpa, len,
+			    region->region.guest_memfd, mode, fd_offset);
+	}
+}
+
 /* Returns the size of a vCPU's kvm_run structure. */
 static int vcpu_mmap_sz(void)
 {
diff --git a/tools/testing/selftests/kvm/lib/memstress.c b/tools/testing/selftests/kvm/lib/memstress.c
index df457452d146..d05487e5a371 100644
--- a/tools/testing/selftests/kvm/lib/memstress.c
+++ b/tools/testing/selftests/kvm/lib/memstress.c
@@ -168,7 +168,8 @@ struct kvm_vm *memstress_create_vm(enum vm_guest_mode mode, int nr_vcpus,
 	 * The memory is also added to memslot 0, but that's a benign side
 	 * effect as KVM allows aliasing HVAs in meslots.
 	 */
-	vm = __vm_create_with_vcpus(mode, nr_vcpus, slot0_pages + guest_num_pages,
+	vm = __vm_create_with_vcpus(VM_SHAPE(mode), nr_vcpus,
+				    slot0_pages + guest_num_pages,
 				    memstress_guest_code, vcpus);
 
 	args->vm = vm;
diff --git a/tools/testing/selftests/kvm/s390x/cmma_test.c b/tools/testing/selftests/kvm/s390x/cmma_test.c
index c8e0a6495a63..626a2b8a2037 100644
--- a/tools/testing/selftests/kvm/s390x/cmma_test.c
+++ b/tools/testing/selftests/kvm/s390x/cmma_test.c
@@ -94,11 +94,6 @@ static void guest_dirty_test_data(void)
 	);
 }
 
-static struct kvm_vm *create_vm(void)
-{
-	return ____vm_create(VM_MODE_DEFAULT);
-}
-
 static void create_main_memslot(struct kvm_vm *vm)
 {
 	int i;
@@ -157,7 +152,7 @@ static struct kvm_vm *create_vm_two_memslots(void)
 {
 	struct kvm_vm *vm;
 
-	vm = create_vm();
+	vm = vm_create_barebones();
 
 	create_memslots(vm);
 
@@ -276,7 +271,7 @@ static void assert_exit_was_hypercall(struct kvm_vcpu *vcpu)
 
 static void test_migration_mode(void)
 {
-	struct kvm_vm *vm = create_vm();
+	struct kvm_vm *vm = vm_create_barebones();
 	struct kvm_vcpu *vcpu;
 	u64 orig_psw;
 	int rc;
@@ -670,7 +665,7 @@ struct testdef {
  */
 static int machine_has_cmma(void)
 {
-	struct kvm_vm *vm = create_vm();
+	struct kvm_vm *vm = vm_create_barebones();
 	int r;
 
 	r = !__kvm_has_device_attr(vm->fd, KVM_S390_VM_MEM_CTRL, KVM_S390_VM_MEM_ENABLE_CMMA);
diff --git a/tools/testing/selftests/kvm/set_memory_region_test.c b/tools/testing/selftests/kvm/set_memory_region_test.c
index b32960189f5f..1efee1cfcff0 100644
--- a/tools/testing/selftests/kvm/set_memory_region_test.c
+++ b/tools/testing/selftests/kvm/set_memory_region_test.c
@@ -326,6 +326,53 @@ static void test_zero_memory_regions(void)
 }
 #endif /* __x86_64__ */
 
+static void test_invalid_memory_region_flags(void)
+{
+	uint32_t supported_flags = KVM_MEM_LOG_DIRTY_PAGES;
+	const uint32_t v2_only_flags = KVM_MEM_GUEST_MEMFD;
+	struct kvm_vm *vm;
+	int r, i;
+
+#ifdef __x86_64__
+	supported_flags |= KVM_MEM_READONLY;
+
+	if (kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM))
+		vm = vm_create_barebones_protected_vm();
+	else
+#endif
+		vm = vm_create_barebones();
+
+	if (kvm_check_cap(KVM_CAP_MEMORY_ATTRIBUTES) & KVM_MEMORY_ATTRIBUTE_PRIVATE)
+		supported_flags |= KVM_MEM_GUEST_MEMFD;
+
+	for (i = 0; i < 32; i++) {
+		if ((supported_flags & BIT(i)) && !(v2_only_flags & BIT(i)))
+			continue;
+
+		r = __vm_set_user_memory_region(vm, MEM_REGION_SLOT, BIT(i),
+						MEM_REGION_GPA, MEM_REGION_SIZE, NULL);
+
+		TEST_ASSERT(r && errno == EINVAL,
+			    "KVM_SET_USER_MEMORY_REGION should have failed on v2 only flag 0x%lx", BIT(i));
+
+		if (supported_flags & BIT(i))
+			continue;
+
+		r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, BIT(i),
+						 MEM_REGION_GPA, MEM_REGION_SIZE, NULL, 0, 0);
+		TEST_ASSERT(r && errno == EINVAL,
+			    "KVM_SET_USER_MEMORY_REGION2 should have failed on unsupported flag 0x%lx", BIT(i));
+	}
+
+	if (supported_flags & KVM_MEM_GUEST_MEMFD) {
+		r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT,
+						 KVM_MEM_LOG_DIRTY_PAGES | KVM_MEM_GUEST_MEMFD,
+						 MEM_REGION_GPA, MEM_REGION_SIZE, NULL, 0, 0);
+		TEST_ASSERT(r && errno == EINVAL,
+			    "KVM_SET_USER_MEMORY_REGION2 should have failed, dirty logging private memory is unsupported");
+	}
+}
+
 /*
  * Test it can be added memory slots up to KVM_CAP_NR_MEMSLOTS, then any
  * tentative to add further slots should fail.
@@ -385,13 +432,105 @@ static void test_add_max_memory_regions(void)
 	kvm_vm_free(vm);
 }
 
+
+#ifdef __x86_64__
+static void test_invalid_guest_memfd(struct kvm_vm *vm, int memfd,
+				     size_t offset, const char *msg)
+{
+	int r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD,
+					     MEM_REGION_GPA, MEM_REGION_SIZE,
+					     0, memfd, offset);
+	TEST_ASSERT(r == -1 && errno == EINVAL, "%s", msg);
+}
+
+static void test_add_private_memory_region(void)
+{
+	struct kvm_vm *vm, *vm2;
+	int memfd, i;
+
+	pr_info("Testing ADD of KVM_MEM_GUEST_MEMFD memory regions\n");
+
+	vm = vm_create_barebones_protected_vm();
+
+	test_invalid_guest_memfd(vm, vm->kvm_fd, 0, "KVM fd should fail");
+	test_invalid_guest_memfd(vm, vm->fd, 0, "VM's fd should fail");
+
+	memfd = kvm_memfd_alloc(MEM_REGION_SIZE, false);
+	test_invalid_guest_memfd(vm, memfd, 0, "Regular memfd() should fail");
+	close(memfd);
+
+	vm2 = vm_create_barebones_protected_vm();
+	memfd = vm_create_guest_memfd(vm2, MEM_REGION_SIZE, 0);
+	test_invalid_guest_memfd(vm, memfd, 0, "Other VM's guest_memfd() should fail");
+
+	vm_set_user_memory_region2(vm2, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD,
+				   MEM_REGION_GPA, MEM_REGION_SIZE, 0, memfd, 0);
+	close(memfd);
+	kvm_vm_free(vm2);
+
+	memfd = vm_create_guest_memfd(vm, MEM_REGION_SIZE, 0);
+	for (i = 1; i < PAGE_SIZE; i++)
+		test_invalid_guest_memfd(vm, memfd, i, "Unaligned offset should fail");
+
+	vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD,
+				   MEM_REGION_GPA, MEM_REGION_SIZE, 0, memfd, 0);
+	close(memfd);
+
+	kvm_vm_free(vm);
+}
+
+static void test_add_overlapping_private_memory_regions(void)
+{
+	struct kvm_vm *vm;
+	int memfd;
+	int r;
+
+	pr_info("Testing ADD of overlapping KVM_MEM_GUEST_MEMFD memory regions\n");
+
+	vm = vm_create_barebones_protected_vm();
+
+	memfd = vm_create_guest_memfd(vm, MEM_REGION_SIZE * 4, 0);
+
+	vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD,
+				   MEM_REGION_GPA, MEM_REGION_SIZE * 2, 0, memfd, 0);
+
+	vm_set_user_memory_region2(vm, MEM_REGION_SLOT + 1, KVM_MEM_GUEST_MEMFD,
+				   MEM_REGION_GPA * 2, MEM_REGION_SIZE * 2,
+				   0, memfd, MEM_REGION_SIZE * 2);
+
+	/*
+	 * Delete the first memslot, and then attempt to recreate it except
+	 * with a "bad" offset that results in overlap in the guest_memfd().
+	 */
+	vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD,
+				   MEM_REGION_GPA, 0, NULL, -1, 0);
+
+	/* Overlap the front half of the other slot. */
+	r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD,
+					 MEM_REGION_GPA * 2 - MEM_REGION_SIZE,
+					 MEM_REGION_SIZE * 2,
+					 0, memfd, 0);
+	TEST_ASSERT(r == -1 && errno == EEXIST, "%s",
+		    "Overlapping guest_memfd() bindings should fail with EEXIST");
+
+	/* And now the back half of the other slot. */
+	r = __vm_set_user_memory_region2(vm, MEM_REGION_SLOT, KVM_MEM_GUEST_MEMFD,
+					 MEM_REGION_GPA * 2 + MEM_REGION_SIZE,
+					 MEM_REGION_SIZE * 2,
+					 0, memfd, 0);
+	TEST_ASSERT(r == -1 && errno == EEXIST, "%s",
+		    "Overlapping guest_memfd() bindings should fail with EEXIST");
+
+	close(memfd);
+	kvm_vm_free(vm);
+}
+#endif
+
 int main(int argc, char *argv[])
 {
 #ifdef __x86_64__
 	int i, loops;
-#endif
 
-#ifdef __x86_64__
 	/*
 	 * FIXME: the zero-memslot test fails on aarch64 and s390x because
 	 * KVM_RUN fails with ENOEXEC or EFAULT.
@@ -399,9 +538,19 @@ int main(int argc, char *argv[])
 	test_zero_memory_regions();
 #endif
 
+	test_invalid_memory_region_flags();
+
 	test_add_max_memory_regions();
 
 #ifdef __x86_64__
+	if (kvm_has_cap(KVM_CAP_GUEST_MEMFD) &&
+	    (kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM))) {
+		test_add_private_memory_region();
+		test_add_overlapping_private_memory_regions();
+	} else {
+		pr_info("Skipping tests for KVM_MEM_GUEST_MEMFD memory regions\n");
+	}
+
 	if (argc > 1)
 		loops = atoi_positive("Number of iterations", argv[1]);
 	else
diff --git a/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.c b/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.c
new file mode 100644
index 000000000000..4d6a37a5d896
--- /dev/null
+++ b/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.c
@@ -0,0 +1,482 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2022, Google LLC.
+ */
+#define _GNU_SOURCE /* for program_invocation_short_name */
+#include <fcntl.h>
+#include <limits.h>
+#include <pthread.h>
+#include <sched.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ioctl.h>
+
+#include <linux/compiler.h>
+#include <linux/kernel.h>
+#include <linux/kvm_para.h>
+#include <linux/memfd.h>
+#include <linux/sizes.h>
+
+#include <test_util.h>
+#include <kvm_util.h>
+#include <processor.h>
+
+#define BASE_DATA_SLOT		10
+#define BASE_DATA_GPA		((uint64_t)(1ull << 32))
+#define PER_CPU_DATA_SIZE	((uint64_t)(SZ_2M + PAGE_SIZE))
+
+/* Horrific macro so that the line info is captured accurately :-( */
+#define memcmp_g(gpa, pattern,  size)								\
+do {												\
+	uint8_t *mem = (uint8_t *)gpa;								\
+	size_t i;										\
+												\
+	for (i = 0; i < size; i++)								\
+		__GUEST_ASSERT(mem[i] == pattern,						\
+			       "Guest expected 0x%x at offset %lu (gpa 0x%llx), got 0x%x",	\
+			       pattern, i, gpa + i, mem[i]);					\
+} while (0)
+
+static void memcmp_h(uint8_t *mem, uint64_t gpa, uint8_t pattern, size_t size)
+{
+	size_t i;
+
+	for (i = 0; i < size; i++)
+		TEST_ASSERT(mem[i] == pattern,
+			    "Host expected 0x%x at gpa 0x%lx, got 0x%x",
+			    pattern, gpa + i, mem[i]);
+}
+
+/*
+ * Run memory conversion tests with explicit conversion:
+ * Execute KVM hypercall to map/unmap gpa range which will cause userspace exit
+ * to back/unback private memory. Subsequent accesses by guest to the gpa range
+ * will not cause exit to userspace.
+ *
+ * Test memory conversion scenarios with following steps:
+ * 1) Access private memory using private access and verify that memory contents
+ *   are not visible to userspace.
+ * 2) Convert memory to shared using explicit conversions and ensure that
+ *   userspace is able to access the shared regions.
+ * 3) Convert memory back to private using explicit conversions and ensure that
+ *   userspace is again not able to access converted private regions.
+ */
+
+#define GUEST_STAGE(o, s) { .offset = o, .size = s }
+
+enum ucall_syncs {
+	SYNC_SHARED,
+	SYNC_PRIVATE,
+};
+
+static void guest_sync_shared(uint64_t gpa, uint64_t size,
+			      uint8_t current_pattern, uint8_t new_pattern)
+{
+	GUEST_SYNC5(SYNC_SHARED, gpa, size, current_pattern, new_pattern);
+}
+
+static void guest_sync_private(uint64_t gpa, uint64_t size, uint8_t pattern)
+{
+	GUEST_SYNC4(SYNC_PRIVATE, gpa, size, pattern);
+}
+
+/* Arbitrary values, KVM doesn't care about the attribute flags. */
+#define MAP_GPA_SET_ATTRIBUTES	BIT(0)
+#define MAP_GPA_SHARED		BIT(1)
+#define MAP_GPA_DO_FALLOCATE	BIT(2)
+
+static void guest_map_mem(uint64_t gpa, uint64_t size, bool map_shared,
+			  bool do_fallocate)
+{
+	uint64_t flags = MAP_GPA_SET_ATTRIBUTES;
+
+	if (map_shared)
+		flags |= MAP_GPA_SHARED;
+	if (do_fallocate)
+		flags |= MAP_GPA_DO_FALLOCATE;
+	kvm_hypercall_map_gpa_range(gpa, size, flags);
+}
+
+static void guest_map_shared(uint64_t gpa, uint64_t size, bool do_fallocate)
+{
+	guest_map_mem(gpa, size, true, do_fallocate);
+}
+
+static void guest_map_private(uint64_t gpa, uint64_t size, bool do_fallocate)
+{
+	guest_map_mem(gpa, size, false, do_fallocate);
+}
+
+struct {
+	uint64_t offset;
+	uint64_t size;
+} static const test_ranges[] = {
+	GUEST_STAGE(0, PAGE_SIZE),
+	GUEST_STAGE(0, SZ_2M),
+	GUEST_STAGE(PAGE_SIZE, PAGE_SIZE),
+	GUEST_STAGE(PAGE_SIZE, SZ_2M),
+	GUEST_STAGE(SZ_2M, PAGE_SIZE),
+};
+
+static void guest_test_explicit_conversion(uint64_t base_gpa, bool do_fallocate)
+{
+	const uint8_t def_p = 0xaa;
+	const uint8_t init_p = 0xcc;
+	uint64_t j;
+	int i;
+
+	/* Memory should be shared by default. */
+	memset((void *)base_gpa, def_p, PER_CPU_DATA_SIZE);
+	memcmp_g(base_gpa, def_p, PER_CPU_DATA_SIZE);
+	guest_sync_shared(base_gpa, PER_CPU_DATA_SIZE, def_p, init_p);
+
+	memcmp_g(base_gpa, init_p, PER_CPU_DATA_SIZE);
+
+	for (i = 0; i < ARRAY_SIZE(test_ranges); i++) {
+		uint64_t gpa = base_gpa + test_ranges[i].offset;
+		uint64_t size = test_ranges[i].size;
+		uint8_t p1 = 0x11;
+		uint8_t p2 = 0x22;
+		uint8_t p3 = 0x33;
+		uint8_t p4 = 0x44;
+
+		/*
+		 * Set the test region to pattern one to differentiate it from
+		 * the data range as a whole (contains the initial pattern).
+		 */
+		memset((void *)gpa, p1, size);
+
+		/*
+		 * Convert to private, set and verify the private data, and
+		 * then verify that the rest of the data (map shared) still
+		 * holds the initial pattern, and that the host always sees the
+		 * shared memory (initial pattern).  Unlike shared memory,
+		 * punching a hole in private memory is destructive, i.e.
+		 * previous values aren't guaranteed to be preserved.
+		 */
+		guest_map_private(gpa, size, do_fallocate);
+
+		if (size > PAGE_SIZE) {
+			memset((void *)gpa, p2, PAGE_SIZE);
+			goto skip;
+		}
+
+		memset((void *)gpa, p2, size);
+		guest_sync_private(gpa, size, p1);
+
+		/*
+		 * Verify that the private memory was set to pattern two, and
+		 * that shared memory still holds the initial pattern.
+		 */
+		memcmp_g(gpa, p2, size);
+		if (gpa > base_gpa)
+			memcmp_g(base_gpa, init_p, gpa - base_gpa);
+		if (gpa + size < base_gpa + PER_CPU_DATA_SIZE)
+			memcmp_g(gpa + size, init_p,
+				 (base_gpa + PER_CPU_DATA_SIZE) - (gpa + size));
+
+		/*
+		 * Convert odd-number page frames back to shared to verify KVM
+		 * also correctly handles holes in private ranges.
+		 */
+		for (j = 0; j < size; j += PAGE_SIZE) {
+			if ((j >> PAGE_SHIFT) & 1) {
+				guest_map_shared(gpa + j, PAGE_SIZE, do_fallocate);
+				guest_sync_shared(gpa + j, PAGE_SIZE, p1, p3);
+
+				memcmp_g(gpa + j, p3, PAGE_SIZE);
+			} else {
+				guest_sync_private(gpa + j, PAGE_SIZE, p1);
+			}
+		}
+
+skip:
+		/*
+		 * Convert the entire region back to shared, explicitly write
+		 * pattern three to fill in the even-number frames before
+		 * asking the host to verify (and write pattern four).
+		 */
+		guest_map_shared(gpa, size, do_fallocate);
+		memset((void *)gpa, p3, size);
+		guest_sync_shared(gpa, size, p3, p4);
+		memcmp_g(gpa, p4, size);
+
+		/* Reset the shared memory back to the initial pattern. */
+		memset((void *)gpa, init_p, size);
+
+		/*
+		 * Free (via PUNCH_HOLE) *all* private memory so that the next
+		 * iteration starts from a clean slate, e.g. with respect to
+		 * whether or not there are pages/folios in guest_mem.
+		 */
+		guest_map_shared(base_gpa, PER_CPU_DATA_SIZE, true);
+	}
+}
+
+static void guest_punch_hole(uint64_t gpa, uint64_t size)
+{
+	/* "Mapping" memory shared via fallocate() is done via PUNCH_HOLE. */
+	uint64_t flags = MAP_GPA_SHARED | MAP_GPA_DO_FALLOCATE;
+
+	kvm_hypercall_map_gpa_range(gpa, size, flags);
+}
+
+/*
+ * Test that PUNCH_HOLE actually frees memory by punching holes without doing a
+ * proper conversion.  Freeing (PUNCH_HOLE) should zap SPTEs, and reallocating
+ * (subsequent fault) should zero memory.
+ */
+static void guest_test_punch_hole(uint64_t base_gpa, bool precise)
+{
+	const uint8_t init_p = 0xcc;
+	int i;
+
+	/*
+	 * Convert the entire range to private, this testcase is all about
+	 * punching holes in guest_memfd, i.e. shared mappings aren't needed.
+	 */
+	guest_map_private(base_gpa, PER_CPU_DATA_SIZE, false);
+
+	for (i = 0; i < ARRAY_SIZE(test_ranges); i++) {
+		uint64_t gpa = base_gpa + test_ranges[i].offset;
+		uint64_t size = test_ranges[i].size;
+
+		/*
+		 * Free all memory before each iteration, even for the !precise
+		 * case where the memory will be faulted back in.  Freeing and
+		 * reallocating should obviously work, and freeing all memory
+		 * minimizes the probability of cross-testcase influence.
+		 */
+		guest_punch_hole(base_gpa, PER_CPU_DATA_SIZE);
+
+		/* Fault-in and initialize memory, and verify the pattern. */
+		if (precise) {
+			memset((void *)gpa, init_p, size);
+			memcmp_g(gpa, init_p, size);
+		} else {
+			memset((void *)base_gpa, init_p, PER_CPU_DATA_SIZE);
+			memcmp_g(base_gpa, init_p, PER_CPU_DATA_SIZE);
+		}
+
+		/*
+		 * Punch a hole at the target range and verify that reads from
+		 * the guest succeed and return zeroes.
+		 */
+		guest_punch_hole(gpa, size);
+		memcmp_g(gpa, 0, size);
+	}
+}
+
+static void guest_code(uint64_t base_gpa)
+{
+	/*
+	 * Run the conversion test twice, with and without doing fallocate() on
+	 * the guest_memfd backing when converting between shared and private.
+	 */
+	guest_test_explicit_conversion(base_gpa, false);
+	guest_test_explicit_conversion(base_gpa, true);
+
+	/*
+	 * Run the PUNCH_HOLE test twice too, once with the entire guest_memfd
+	 * faulted in, once with only the target range faulted in.
+	 */
+	guest_test_punch_hole(base_gpa, false);
+	guest_test_punch_hole(base_gpa, true);
+	GUEST_DONE();
+}
+
+static void handle_exit_hypercall(struct kvm_vcpu *vcpu)
+{
+	struct kvm_run *run = vcpu->run;
+	uint64_t gpa = run->hypercall.args[0];
+	uint64_t size = run->hypercall.args[1] * PAGE_SIZE;
+	bool set_attributes = run->hypercall.args[2] & MAP_GPA_SET_ATTRIBUTES;
+	bool map_shared = run->hypercall.args[2] & MAP_GPA_SHARED;
+	bool do_fallocate = run->hypercall.args[2] & MAP_GPA_DO_FALLOCATE;
+	struct kvm_vm *vm = vcpu->vm;
+
+	TEST_ASSERT(run->hypercall.nr == KVM_HC_MAP_GPA_RANGE,
+		    "Wanted MAP_GPA_RANGE (%u), got '%llu'",
+		    KVM_HC_MAP_GPA_RANGE, run->hypercall.nr);
+
+	if (do_fallocate)
+		vm_guest_mem_fallocate(vm, gpa, size, map_shared);
+
+	if (set_attributes)
+		vm_set_memory_attributes(vm, gpa, size,
+					 map_shared ? 0 : KVM_MEMORY_ATTRIBUTE_PRIVATE);
+	run->hypercall.ret = 0;
+}
+
+static bool run_vcpus;
+
+static void *__test_mem_conversions(void *__vcpu)
+{
+	struct kvm_vcpu *vcpu = __vcpu;
+	struct kvm_run *run = vcpu->run;
+	struct kvm_vm *vm = vcpu->vm;
+	struct ucall uc;
+
+	while (!READ_ONCE(run_vcpus))
+		;
+
+	for ( ;; ) {
+		vcpu_run(vcpu);
+
+		if (run->exit_reason == KVM_EXIT_HYPERCALL) {
+			handle_exit_hypercall(vcpu);
+			continue;
+		}
+
+		TEST_ASSERT(run->exit_reason == KVM_EXIT_IO,
+			    "Wanted KVM_EXIT_IO, got exit reason: %u (%s)",
+			    run->exit_reason, exit_reason_str(run->exit_reason));
+
+		switch (get_ucall(vcpu, &uc)) {
+		case UCALL_ABORT:
+			REPORT_GUEST_ASSERT(uc);
+		case UCALL_SYNC: {
+			uint64_t gpa  = uc.args[1];
+			size_t size = uc.args[2];
+			size_t i;
+
+			TEST_ASSERT(uc.args[0] == SYNC_SHARED ||
+				    uc.args[0] == SYNC_PRIVATE,
+				    "Unknown sync command '%ld'", uc.args[0]);
+
+			for (i = 0; i < size; i += vm->page_size) {
+				size_t nr_bytes = min_t(size_t, vm->page_size, size - i);
+				uint8_t *hva = addr_gpa2hva(vm, gpa + i);
+
+				/* In all cases, the host should observe the shared data. */
+				memcmp_h(hva, gpa + i, uc.args[3], nr_bytes);
+
+				/* For shared, write the new pattern to guest memory. */
+				if (uc.args[0] == SYNC_SHARED)
+					memset(hva, uc.args[4], nr_bytes);
+			}
+			break;
+		}
+		case UCALL_DONE:
+			return NULL;
+		default:
+			TEST_FAIL("Unknown ucall 0x%lx.", uc.cmd);
+		}
+	}
+}
+
+static void test_mem_conversions(enum vm_mem_backing_src_type src_type, uint32_t nr_vcpus,
+				 uint32_t nr_memslots)
+{
+	/*
+	 * Allocate enough memory so that each vCPU's chunk of memory can be
+	 * naturally aligned with respect to the size of the backing store.
+	 */
+	const size_t alignment = max_t(size_t, SZ_2M, get_backing_src_pagesz(src_type));
+	const size_t per_cpu_size = align_up(PER_CPU_DATA_SIZE, alignment);
+	const size_t memfd_size = per_cpu_size * nr_vcpus;
+	const size_t slot_size = memfd_size / nr_memslots;
+	struct kvm_vcpu *vcpus[KVM_MAX_VCPUS];
+	pthread_t threads[KVM_MAX_VCPUS];
+	struct kvm_vm *vm;
+	int memfd, i, r;
+
+	const struct vm_shape shape = {
+		.mode = VM_MODE_DEFAULT,
+		.type = KVM_X86_SW_PROTECTED_VM,
+	};
+
+	TEST_ASSERT(slot_size * nr_memslots == memfd_size,
+		    "The memfd size (0x%lx) needs to be cleanly divisible by the number of memslots (%u)",
+		    memfd_size, nr_memslots);
+	vm = __vm_create_with_vcpus(shape, nr_vcpus, 0, guest_code, vcpus);
+
+	vm_enable_cap(vm, KVM_CAP_EXIT_HYPERCALL, (1 << KVM_HC_MAP_GPA_RANGE));
+
+	memfd = vm_create_guest_memfd(vm, memfd_size, 0);
+
+	for (i = 0; i < nr_memslots; i++)
+		vm_mem_add(vm, src_type, BASE_DATA_GPA + slot_size * i,
+			   BASE_DATA_SLOT + i, slot_size / vm->page_size,
+			   KVM_MEM_GUEST_MEMFD, memfd, slot_size * i);
+
+	for (i = 0; i < nr_vcpus; i++) {
+		uint64_t gpa =  BASE_DATA_GPA + i * per_cpu_size;
+
+		vcpu_args_set(vcpus[i], 1, gpa);
+
+		/*
+		 * Map only what is needed so that an out-of-bounds access
+		 * results #PF => SHUTDOWN instead of data corruption.
+		 */
+		virt_map(vm, gpa, gpa, PER_CPU_DATA_SIZE / vm->page_size);
+
+		pthread_create(&threads[i], NULL, __test_mem_conversions, vcpus[i]);
+	}
+
+	WRITE_ONCE(run_vcpus, true);
+
+	for (i = 0; i < nr_vcpus; i++)
+		pthread_join(threads[i], NULL);
+
+	kvm_vm_free(vm);
+
+	/*
+	 * Allocate and free memory from the guest_memfd after closing the VM
+	 * fd.  The guest_memfd is gifted a reference to its owning VM, i.e.
+	 * should prevent the VM from being fully destroyed until the last
+	 * reference to the guest_memfd is also put.
+	 */
+	r = fallocate(memfd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, memfd_size);
+	TEST_ASSERT(!r, __KVM_SYSCALL_ERROR("fallocate()", r));
+
+	r = fallocate(memfd, FALLOC_FL_KEEP_SIZE, 0, memfd_size);
+	TEST_ASSERT(!r, __KVM_SYSCALL_ERROR("fallocate()", r));
+}
+
+static void usage(const char *cmd)
+{
+	puts("");
+	printf("usage: %s [-h] [-m nr_memslots] [-s mem_type] [-n nr_vcpus]\n", cmd);
+	puts("");
+	backing_src_help("-s");
+	puts("");
+	puts(" -n: specify the number of vcpus (default: 1)");
+	puts("");
+	puts(" -m: specify the number of memslots (default: 1)");
+	puts("");
+}
+
+int main(int argc, char *argv[])
+{
+	enum vm_mem_backing_src_type src_type = DEFAULT_VM_MEM_SRC;
+	uint32_t nr_memslots = 1;
+	uint32_t nr_vcpus = 1;
+	int opt;
+
+	TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM));
+
+	while ((opt = getopt(argc, argv, "hm:s:n:")) != -1) {
+		switch (opt) {
+		case 's':
+			src_type = parse_backing_src_type(optarg);
+			break;
+		case 'n':
+			nr_vcpus = atoi_positive("nr_vcpus", optarg);
+			break;
+		case 'm':
+			nr_memslots = atoi_positive("nr_memslots", optarg);
+			break;
+		case 'h':
+		default:
+			usage(argv[0]);
+			exit(0);
+		}
+	}
+
+	test_mem_conversions(src_type, nr_vcpus, nr_memslots);
+
+	return 0;
+}
diff --git a/tools/testing/selftests/kvm/x86_64/private_mem_kvm_exits_test.c b/tools/testing/selftests/kvm/x86_64/private_mem_kvm_exits_test.c
new file mode 100644
index 000000000000..13e72fcec8dd
--- /dev/null
+++ b/tools/testing/selftests/kvm/x86_64/private_mem_kvm_exits_test.c
@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2023, Google LLC.
+ */
+#include <linux/kvm.h>
+#include <pthread.h>
+#include <stdint.h>
+
+#include "kvm_util.h"
+#include "processor.h"
+#include "test_util.h"
+
+/* Arbitrarily selected to avoid overlaps with anything else */
+#define EXITS_TEST_GVA 0xc0000000
+#define EXITS_TEST_GPA EXITS_TEST_GVA
+#define EXITS_TEST_NPAGES 1
+#define EXITS_TEST_SIZE (EXITS_TEST_NPAGES * PAGE_SIZE)
+#define EXITS_TEST_SLOT 10
+
+static uint64_t guest_repeatedly_read(void)
+{
+	volatile uint64_t value;
+
+	while (true)
+		value = *((uint64_t *) EXITS_TEST_GVA);
+
+	return value;
+}
+
+static uint32_t run_vcpu_get_exit_reason(struct kvm_vcpu *vcpu)
+{
+	int r;
+
+	r = _vcpu_run(vcpu);
+	if (r) {
+		TEST_ASSERT(errno == EFAULT, KVM_IOCTL_ERROR(KVM_RUN, r));
+		TEST_ASSERT_EQ(vcpu->run->exit_reason, KVM_EXIT_MEMORY_FAULT);
+	}
+	return vcpu->run->exit_reason;
+}
+
+const struct vm_shape protected_vm_shape = {
+	.mode = VM_MODE_DEFAULT,
+	.type = KVM_X86_SW_PROTECTED_VM,
+};
+
+static void test_private_access_memslot_deleted(void)
+{
+	struct kvm_vm *vm;
+	struct kvm_vcpu *vcpu;
+	pthread_t vm_thread;
+	void *thread_return;
+	uint32_t exit_reason;
+
+	vm = vm_create_shape_with_one_vcpu(protected_vm_shape, &vcpu,
+					   guest_repeatedly_read);
+
+	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
+				    EXITS_TEST_GPA, EXITS_TEST_SLOT,
+				    EXITS_TEST_NPAGES,
+				    KVM_MEM_GUEST_MEMFD);
+
+	virt_map(vm, EXITS_TEST_GVA, EXITS_TEST_GPA, EXITS_TEST_NPAGES);
+
+	/* Request to access page privately */
+	vm_mem_set_private(vm, EXITS_TEST_GPA, EXITS_TEST_SIZE);
+
+	pthread_create(&vm_thread, NULL,
+		       (void *(*)(void *))run_vcpu_get_exit_reason,
+		       (void *)vcpu);
+
+	vm_mem_region_delete(vm, EXITS_TEST_SLOT);
+
+	pthread_join(vm_thread, &thread_return);
+	exit_reason = (uint32_t)(uint64_t)thread_return;
+
+	TEST_ASSERT_EQ(exit_reason, KVM_EXIT_MEMORY_FAULT);
+	TEST_ASSERT_EQ(vcpu->run->memory_fault.flags, KVM_MEMORY_EXIT_FLAG_PRIVATE);
+	TEST_ASSERT_EQ(vcpu->run->memory_fault.gpa, EXITS_TEST_GPA);
+	TEST_ASSERT_EQ(vcpu->run->memory_fault.size, EXITS_TEST_SIZE);
+
+	kvm_vm_free(vm);
+}
+
+static void test_private_access_memslot_not_private(void)
+{
+	struct kvm_vm *vm;
+	struct kvm_vcpu *vcpu;
+	uint32_t exit_reason;
+
+	vm = vm_create_shape_with_one_vcpu(protected_vm_shape, &vcpu,
+					   guest_repeatedly_read);
+
+	/* Add a non-private memslot (flags = 0) */
+	vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
+				    EXITS_TEST_GPA, EXITS_TEST_SLOT,
+				    EXITS_TEST_NPAGES, 0);
+
+	virt_map(vm, EXITS_TEST_GVA, EXITS_TEST_GPA, EXITS_TEST_NPAGES);
+
+	/* Request to access page privately */
+	vm_mem_set_private(vm, EXITS_TEST_GPA, EXITS_TEST_SIZE);
+
+	exit_reason = run_vcpu_get_exit_reason(vcpu);
+
+	TEST_ASSERT_EQ(exit_reason, KVM_EXIT_MEMORY_FAULT);
+	TEST_ASSERT_EQ(vcpu->run->memory_fault.flags, KVM_MEMORY_EXIT_FLAG_PRIVATE);
+	TEST_ASSERT_EQ(vcpu->run->memory_fault.gpa, EXITS_TEST_GPA);
+	TEST_ASSERT_EQ(vcpu->run->memory_fault.size, EXITS_TEST_SIZE);
+
+	kvm_vm_free(vm);
+}
+
+int main(int argc, char *argv[])
+{
+	TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM));
+
+	test_private_access_memslot_deleted();
+	test_private_access_memslot_not_private();
+}
diff --git a/tools/testing/selftests/kvm/x86_64/ucna_injection_test.c b/tools/testing/selftests/kvm/x86_64/ucna_injection_test.c
index 85f34ca7e49e..0ed32ec903d0 100644
--- a/tools/testing/selftests/kvm/x86_64/ucna_injection_test.c
+++ b/tools/testing/selftests/kvm/x86_64/ucna_injection_test.c
@@ -271,7 +271,7 @@ int main(int argc, char *argv[])
 
 	kvm_check_cap(KVM_CAP_MCE);
 
-	vm = __vm_create(VM_MODE_DEFAULT, 3, 0);
+	vm = __vm_create(VM_SHAPE_DEFAULT, 3, 0);
 
 	kvm_ioctl(vm->kvm_fd, KVM_X86_GET_MCE_CAP_SUPPORTED,
 		  &supported_mcg_caps);
diff --git a/tools/testing/selftests/landlock/base_test.c b/tools/testing/selftests/landlock/base_test.c
index 646f778dfb1e..d292b419ccba 100644
--- a/tools/testing/selftests/landlock/base_test.c
+++ b/tools/testing/selftests/landlock/base_test.c
@@ -75,7 +75,7 @@ TEST(abi_version)
 	const struct landlock_ruleset_attr ruleset_attr = {
 		.handled_access_fs = LANDLOCK_ACCESS_FS_READ_FILE,
 	};
-	ASSERT_EQ(4, landlock_create_ruleset(NULL, 0,
+	ASSERT_EQ(5, landlock_create_ruleset(NULL, 0,
 					     LANDLOCK_CREATE_RULESET_VERSION));
 
 	ASSERT_EQ(-1, landlock_create_ruleset(&ruleset_attr, 0,
diff --git a/tools/testing/selftests/landlock/fs_test.c b/tools/testing/selftests/landlock/fs_test.c
index 18e1f86a6234..06c47c816c51 100644
--- a/tools/testing/selftests/landlock/fs_test.c
+++ b/tools/testing/selftests/landlock/fs_test.c
@@ -9,6 +9,7 @@
 
 #define _GNU_SOURCE
 #include <fcntl.h>
+#include <linux/fs.h>
 #include <linux/landlock.h>
 #include <linux/magic.h>
 #include <sched.h>
@@ -525,9 +526,10 @@ TEST_F_FORK(layout1, inval)
 	LANDLOCK_ACCESS_FS_EXECUTE | \
 	LANDLOCK_ACCESS_FS_WRITE_FILE | \
 	LANDLOCK_ACCESS_FS_READ_FILE | \
-	LANDLOCK_ACCESS_FS_TRUNCATE)
+	LANDLOCK_ACCESS_FS_TRUNCATE | \
+	LANDLOCK_ACCESS_FS_IOCTL)
 
-#define ACCESS_LAST LANDLOCK_ACCESS_FS_TRUNCATE
+#define ACCESS_LAST LANDLOCK_ACCESS_FS_IOCTL
 
 #define ACCESS_ALL ( \
 	ACCESS_FILE | \
@@ -3379,7 +3381,7 @@ TEST_F_FORK(layout1, truncate_unhandled)
 			      LANDLOCK_ACCESS_FS_WRITE_FILE;
 	int ruleset_fd;
 
-	/* Enable Landlock. */
+	/* Enables Landlock. */
 	ruleset_fd = create_ruleset(_metadata, handled, rules);
 
 	ASSERT_LE(0, ruleset_fd);
@@ -3462,7 +3464,7 @@ TEST_F_FORK(layout1, truncate)
 			      LANDLOCK_ACCESS_FS_TRUNCATE;
 	int ruleset_fd;
 
-	/* Enable Landlock. */
+	/* Enables Landlock. */
 	ruleset_fd = create_ruleset(_metadata, handled, rules);
 
 	ASSERT_LE(0, ruleset_fd);
@@ -3689,7 +3691,7 @@ TEST_F_FORK(ftruncate, open_and_ftruncate)
 	};
 	int fd, ruleset_fd;
 
-	/* Enable Landlock. */
+	/* Enables Landlock. */
 	ruleset_fd = create_ruleset(_metadata, variant->handled, rules);
 	ASSERT_LE(0, ruleset_fd);
 	enforce_ruleset(_metadata, ruleset_fd);
@@ -3766,18 +3768,492 @@ TEST_F_FORK(ftruncate, open_and_ftruncate_in_different_processes)
 	ASSERT_EQ(0, close(socket_fds[1]));
 }
 
-TEST(memfd_ftruncate)
+/* Invokes the FS_IOC_GETFLAGS IOCTL and returns its errno or 0. */
+static int test_fs_ioc_getflags_ioctl(int fd)
 {
-	int fd;
+	uint32_t flags;
+
+	if (ioctl(fd, FS_IOC_GETFLAGS, &flags) < 0)
+		return errno;
+	return 0;
+}
+
+TEST(memfd_ftruncate_and_ioctl)
+{
+	const struct landlock_ruleset_attr attr = {
+		.handled_access_fs = ACCESS_ALL,
+	};
+	int ruleset_fd, fd, i;
+
+	/*
+	 * We exercise the same test both with and without Landlock enabled, to
+	 * ensure that it behaves the same in both cases.
+	 */
+	for (i = 0; i < 2; i++) {
+		/* Creates a new memfd. */
+		fd = memfd_create("name", MFD_CLOEXEC);
+		ASSERT_LE(0, fd);
+
+		/*
+		 * Checks that operations associated with the opened file
+		 * (ftruncate, ioctl) are permitted on file descriptors that are
+		 * created in ways other than open(2).
+		 */
+		EXPECT_EQ(0, test_ftruncate(fd));
+		EXPECT_EQ(0, test_fs_ioc_getflags_ioctl(fd));
+
+		ASSERT_EQ(0, close(fd));
+
+		/* Enables Landlock. */
+		ruleset_fd = landlock_create_ruleset(&attr, sizeof(attr), 0);
+		ASSERT_LE(0, ruleset_fd);
+		enforce_ruleset(_metadata, ruleset_fd);
+		ASSERT_EQ(0, close(ruleset_fd));
+	}
+}
+
+TEST_F_FORK(layout1, o_path_ftruncate_and_ioctl)
+{
+	const struct landlock_ruleset_attr attr = {
+		.handled_access_fs = ACCESS_ALL,
+	};
+	int ruleset_fd, fd;
+
+	/*
+	 * Checks that for files opened with O_PATH, both ioctl(2) and
+	 * ftruncate(2) yield EBADF, as it is documented in open(2) for the
+	 * O_PATH flag.
+	 */
+	fd = open(dir_s1d1, O_PATH | O_CLOEXEC);
+	ASSERT_LE(0, fd);
+
+	EXPECT_EQ(EBADF, test_ftruncate(fd));
+	EXPECT_EQ(EBADF, test_fs_ioc_getflags_ioctl(fd));
+
+	ASSERT_EQ(0, close(fd));
+
+	/* Enables Landlock. */
+	ruleset_fd = landlock_create_ruleset(&attr, sizeof(attr), 0);
+	ASSERT_LE(0, ruleset_fd);
+	enforce_ruleset(_metadata, ruleset_fd);
+	ASSERT_EQ(0, close(ruleset_fd));
+
+	/*
+	 * Checks that after enabling Landlock,
+	 * - the file can still be opened with O_PATH
+	 * - both ioctl and truncate still yield EBADF (not EACCES).
+	 */
+	fd = open(dir_s1d1, O_PATH | O_CLOEXEC);
+	ASSERT_LE(0, fd);
+
+	EXPECT_EQ(EBADF, test_ftruncate(fd));
+	EXPECT_EQ(EBADF, test_fs_ioc_getflags_ioctl(fd));
+
+	ASSERT_EQ(0, close(fd));
+}
+
+/* clang-format off */
+FIXTURE(ioctl) {};
+/* clang-format on */
+
+FIXTURE_SETUP(ioctl)
+{
+	prepare_layout(_metadata);
+	create_file(_metadata, file1_s1d1);
+}
+
+FIXTURE_TEARDOWN(ioctl)
+{
+	EXPECT_EQ(0, remove_path(file1_s1d1));
+	cleanup_layout(_metadata);
+}
+
+FIXTURE_VARIANT(ioctl)
+{
+	const __u64 handled;
+	const __u64 permitted;
+	const mode_t open_mode;
+	/*
+	 * These are the expected IOCTL results for a representative IOCTL from
+	 * each of the IOCTL groups.  We only distinguish the 0 and EACCES
+	 * results here, and treat other errors as 0.
+	 */
+	const int expected_fioqsize_result; /* G1 */
+	const int expected_fibmap_result; /* G2 */
+	const int expected_fionread_result; /* G3 */
+	const int expected_fs_ioc_zero_range_result; /* G4 */
+	const int expected_fs_ioc_getflags_result; /* other */
+};
+
+/* clang-format off */
+FIXTURE_VARIANT_ADD(ioctl, ioctl_handled_i_permitted_none) {
+	/* clang-format on */
+	.handled = LANDLOCK_ACCESS_FS_EXECUTE | LANDLOCK_ACCESS_FS_IOCTL,
+	.permitted = LANDLOCK_ACCESS_FS_EXECUTE,
+	.open_mode = O_RDWR,
+	.expected_fioqsize_result = EACCES,
+	.expected_fibmap_result = EACCES,
+	.expected_fionread_result = EACCES,
+	.expected_fs_ioc_zero_range_result = EACCES,
+	.expected_fs_ioc_getflags_result = EACCES,
+};
+
+/* clang-format off */
+FIXTURE_VARIANT_ADD(ioctl, ioctl_handled_i_permitted_i) {
+	/* clang-format on */
+	.handled = LANDLOCK_ACCESS_FS_IOCTL,
+	.permitted = LANDLOCK_ACCESS_FS_IOCTL,
+	.open_mode = O_RDWR,
+	.expected_fioqsize_result = 0,
+	.expected_fibmap_result = 0,
+	.expected_fionread_result = 0,
+	.expected_fs_ioc_zero_range_result = 0,
+	.expected_fs_ioc_getflags_result = 0,
+};
+
+/* clang-format off */
+FIXTURE_VARIANT_ADD(ioctl, ioctl_unhandled) {
+	/* clang-format on */
+	.handled = LANDLOCK_ACCESS_FS_EXECUTE,
+	.permitted = LANDLOCK_ACCESS_FS_EXECUTE,
+	.open_mode = O_RDWR,
+	.expected_fioqsize_result = 0,
+	.expected_fibmap_result = 0,
+	.expected_fionread_result = 0,
+	.expected_fs_ioc_zero_range_result = 0,
+	.expected_fs_ioc_getflags_result = 0,
+};
+
+/* clang-format off */
+FIXTURE_VARIANT_ADD(ioctl, ioctl_handled_rwd_permitted_r) {
+	/* clang-format on */
+	.handled = LANDLOCK_ACCESS_FS_READ_FILE |
+		   LANDLOCK_ACCESS_FS_WRITE_FILE | LANDLOCK_ACCESS_FS_READ_DIR,
+	.permitted = LANDLOCK_ACCESS_FS_READ_FILE,
+	.open_mode = O_RDONLY,
+	/* If LANDLOCK_ACCESS_FS_IOCTL is not handled, all IOCTLs work. */
+	.expected_fioqsize_result = 0,
+	.expected_fibmap_result = 0,
+	.expected_fionread_result = 0,
+	.expected_fs_ioc_zero_range_result = 0,
+	.expected_fs_ioc_getflags_result = 0,
+};
+
+/* clang-format off */
+FIXTURE_VARIANT_ADD(ioctl, ioctl_handled_rwd_permitted_w) {
+	/* clang-format on */
+	.handled = LANDLOCK_ACCESS_FS_READ_FILE |
+		   LANDLOCK_ACCESS_FS_WRITE_FILE | LANDLOCK_ACCESS_FS_READ_DIR,
+	.permitted = LANDLOCK_ACCESS_FS_WRITE_FILE,
+	.open_mode = O_WRONLY,
+	/* If LANDLOCK_ACCESS_FS_IOCTL is not handled, all IOCTLs work. */
+	.expected_fioqsize_result = 0,
+	.expected_fibmap_result = 0,
+	.expected_fionread_result = 0,
+	.expected_fs_ioc_zero_range_result = 0,
+	.expected_fs_ioc_getflags_result = 0,
+};
+
+/* clang-format off */
+FIXTURE_VARIANT_ADD(ioctl, ioctl_handled_ri_permitted_r) {
+	/* clang-format on */
+	.handled = LANDLOCK_ACCESS_FS_READ_FILE | LANDLOCK_ACCESS_FS_IOCTL,
+	.permitted = LANDLOCK_ACCESS_FS_READ_FILE,
+	.open_mode = O_RDONLY,
+	.expected_fioqsize_result = 0,
+	.expected_fibmap_result = 0,
+	.expected_fionread_result = 0,
+	.expected_fs_ioc_zero_range_result = EACCES,
+	.expected_fs_ioc_getflags_result = EACCES,
+};
+
+/* clang-format off */
+FIXTURE_VARIANT_ADD(ioctl, ioctl_handled_wi_permitted_w) {
+	/* clang-format on */
+	.handled = LANDLOCK_ACCESS_FS_WRITE_FILE | LANDLOCK_ACCESS_FS_IOCTL,
+	.permitted = LANDLOCK_ACCESS_FS_WRITE_FILE,
+	.open_mode = O_WRONLY,
+	.expected_fioqsize_result = 0,
+	.expected_fibmap_result = 0,
+	.expected_fionread_result = EACCES,
+	.expected_fs_ioc_zero_range_result = 0,
+	.expected_fs_ioc_getflags_result = EACCES,
+};
+
+/* clang-format off */
+FIXTURE_VARIANT_ADD(ioctl, ioctl_handled_di_permitted_d) {
+	/* clang-format on */
+	.handled = LANDLOCK_ACCESS_FS_READ_DIR | LANDLOCK_ACCESS_FS_IOCTL,
+	.permitted = LANDLOCK_ACCESS_FS_READ_DIR,
+	.open_mode = O_RDWR,
+	.expected_fioqsize_result = 0,
+	.expected_fibmap_result = EACCES,
+	.expected_fionread_result = EACCES,
+	.expected_fs_ioc_zero_range_result = EACCES,
+	.expected_fs_ioc_getflags_result = EACCES,
+};
+
+/* clang-format off */
+FIXTURE_VARIANT_ADD(ioctl, ioctl_handled_rwi_permitted_rw) {
+	/* clang-format on */
+	.handled = LANDLOCK_ACCESS_FS_READ_FILE |
+		   LANDLOCK_ACCESS_FS_WRITE_FILE | LANDLOCK_ACCESS_FS_IOCTL,
+	.permitted = LANDLOCK_ACCESS_FS_READ_FILE |
+		     LANDLOCK_ACCESS_FS_WRITE_FILE,
+	.open_mode = O_RDWR,
+	.expected_fioqsize_result = 0,
+	.expected_fibmap_result = 0,
+	.expected_fionread_result = 0,
+	.expected_fs_ioc_zero_range_result = 0,
+	.expected_fs_ioc_getflags_result = EACCES,
+};
+
+/* clang-format off */
+FIXTURE_VARIANT_ADD(ioctl, ioctl_handled_rwi_permitted_r) {
+	/* clang-format on */
+	.handled = LANDLOCK_ACCESS_FS_READ_FILE |
+		   LANDLOCK_ACCESS_FS_WRITE_FILE | LANDLOCK_ACCESS_FS_IOCTL,
+	.permitted = LANDLOCK_ACCESS_FS_READ_FILE,
+	.open_mode = O_RDONLY,
+	.expected_fioqsize_result = 0,
+	.expected_fibmap_result = 0,
+	.expected_fionread_result = 0,
+	.expected_fs_ioc_zero_range_result = EACCES,
+	.expected_fs_ioc_getflags_result = EACCES,
+};
+
+/* clang-format off */
+FIXTURE_VARIANT_ADD(ioctl, ioctl_handled_rwi_permitted_ri) {
+	/* clang-format on */
+	.handled = LANDLOCK_ACCESS_FS_READ_FILE |
+		   LANDLOCK_ACCESS_FS_WRITE_FILE | LANDLOCK_ACCESS_FS_IOCTL,
+	.permitted = LANDLOCK_ACCESS_FS_READ_FILE | LANDLOCK_ACCESS_FS_IOCTL,
+	.open_mode = O_RDONLY,
+	.expected_fioqsize_result = 0,
+	.expected_fibmap_result = 0,
+	.expected_fionread_result = 0,
+	.expected_fs_ioc_zero_range_result = EACCES,
+	.expected_fs_ioc_getflags_result = 0,
+};
+
+/* clang-format off */
+FIXTURE_VARIANT_ADD(ioctl, ioctl_handled_rwi_permitted_w) {
+	/* clang-format on */
+	.handled = LANDLOCK_ACCESS_FS_READ_FILE |
+		   LANDLOCK_ACCESS_FS_WRITE_FILE | LANDLOCK_ACCESS_FS_IOCTL,
+	.permitted = LANDLOCK_ACCESS_FS_WRITE_FILE,
+	.open_mode = O_WRONLY,
+	.expected_fioqsize_result = 0,
+	.expected_fibmap_result = 0,
+	.expected_fionread_result = EACCES,
+	.expected_fs_ioc_zero_range_result = 0,
+	.expected_fs_ioc_getflags_result = EACCES,
+};
+
+/* clang-format off */
+FIXTURE_VARIANT_ADD(ioctl, ioctl_handled_rwi_permitted_wi) {
+	/* clang-format on */
+	.handled = LANDLOCK_ACCESS_FS_READ_FILE |
+		   LANDLOCK_ACCESS_FS_WRITE_FILE | LANDLOCK_ACCESS_FS_IOCTL,
+	.permitted = LANDLOCK_ACCESS_FS_WRITE_FILE | LANDLOCK_ACCESS_FS_IOCTL,
+	.open_mode = O_WRONLY,
+	.expected_fioqsize_result = 0,
+	.expected_fibmap_result = 0,
+	.expected_fionread_result = EACCES,
+	.expected_fs_ioc_zero_range_result = 0,
+	.expected_fs_ioc_getflags_result = 0,
+};
+
+static int test_fioqsize_ioctl(int fd)
+{
+	size_t sz;
+
+	if (ioctl(fd, FIOQSIZE, &sz) < 0)
+		return errno;
+	return 0;
+}
+
+static int test_fibmap_ioctl(int fd)
+{
+	int blk = 0;
+
+	/*
+	 * We only want to distinguish here whether Landlock already caught it,
+	 * so we treat anything but EACCESS as success.  (It commonly returns
+	 * EPERM when missing CAP_SYS_RAWIO.)
+	 */
+	if (ioctl(fd, FIBMAP, &blk) < 0 && errno == EACCES)
+		return errno;
+	return 0;
+}
+
+static int test_fionread_ioctl(int fd)
+{
+	size_t sz = 0;
+
+	if (ioctl(fd, FIONREAD, &sz) < 0 && errno == EACCES)
+		return errno;
+	return 0;
+}
+
+#define FS_IOC_ZERO_RANGE _IOW('X', 57, struct space_resv)
+
+static int test_fs_ioc_zero_range_ioctl(int fd)
+{
+	struct space_resv {
+		__s16 l_type;
+		__s16 l_whence;
+		__s64 l_start;
+		__s64 l_len; /* len == 0 means until end of file */
+		__s32 l_sysid;
+		__u32 l_pid;
+		__s32 l_pad[4]; /* reserved area */
+	} reservation = {};
+	/*
+	 * This can fail for various reasons, but we only want to distinguish
+	 * here whether Landlock already caught it, so we treat anything but
+	 * EACCES as success.
+	 */
+	if (ioctl(fd, FS_IOC_ZERO_RANGE, &reservation) < 0 && errno == EACCES)
+		return errno;
+	return 0;
+}
+
+TEST_F_FORK(ioctl, handle_dir_access_file)
+{
+	const int flag = 0;
+	const struct rule rules[] = {
+		{
+			.path = dir_s1d1,
+			.access = variant->permitted,
+		},
+		{},
+	};
+	int fd, ruleset_fd;
+
+	/* Enables Landlock. */
+	ruleset_fd = create_ruleset(_metadata, variant->handled, rules);
+	ASSERT_LE(0, ruleset_fd);
+	enforce_ruleset(_metadata, ruleset_fd);
+	ASSERT_EQ(0, close(ruleset_fd));
+
+	fd = open(file1_s1d1, variant->open_mode);
+	ASSERT_LE(0, fd);
+
+	/*
+	 * Checks that IOCTL commands in each IOCTL group return the expected
+	 * errors.
+	 */
+	EXPECT_EQ(variant->expected_fioqsize_result, test_fioqsize_ioctl(fd));
+	EXPECT_EQ(variant->expected_fibmap_result, test_fibmap_ioctl(fd));
+	EXPECT_EQ(variant->expected_fionread_result, test_fionread_ioctl(fd));
+	EXPECT_EQ(variant->expected_fs_ioc_zero_range_result,
+		  test_fs_ioc_zero_range_ioctl(fd));
+	EXPECT_EQ(variant->expected_fs_ioc_getflags_result,
+		  test_fs_ioc_getflags_ioctl(fd));
+
+	/* Checks that unrestrictable commands are unrestricted. */
+	EXPECT_EQ(0, ioctl(fd, FIOCLEX));
+	EXPECT_EQ(0, ioctl(fd, FIONCLEX));
+	EXPECT_EQ(0, ioctl(fd, FIONBIO, &flag));
+	EXPECT_EQ(0, ioctl(fd, FIOASYNC, &flag));
+
+	ASSERT_EQ(0, close(fd));
+}
+
+TEST_F_FORK(ioctl, handle_dir_access_dir)
+{
+	const char *const path = dir_s1d1;
+	const int flag = 0;
+	const struct rule rules[] = {
+		{
+			.path = path,
+			.access = variant->permitted,
+		},
+		{},
+	};
+	int fd, ruleset_fd;
+
+	/* Enables Landlock. */
+	ruleset_fd = create_ruleset(_metadata, variant->handled, rules);
+	ASSERT_LE(0, ruleset_fd);
+	enforce_ruleset(_metadata, ruleset_fd);
+	ASSERT_EQ(0, close(ruleset_fd));
+
+	/*
+	 * Ignore variant->open_mode for this test, as we intend to open a
+	 * directory.  If the directory can not be opened, the variant is
+	 * infeasible to test with an opened directory.
+	 */
+	fd = open(path, O_RDONLY);
+	if (fd < 0)
+		return;
+
+	/*
+	 * Checks that IOCTL commands in each IOCTL group return the expected
+	 * errors.
+	 */
+	EXPECT_EQ(variant->expected_fioqsize_result, test_fioqsize_ioctl(fd));
+	EXPECT_EQ(variant->expected_fibmap_result, test_fibmap_ioctl(fd));
+	EXPECT_EQ(variant->expected_fionread_result, test_fionread_ioctl(fd));
+	EXPECT_EQ(variant->expected_fs_ioc_zero_range_result,
+		  test_fs_ioc_zero_range_ioctl(fd));
+	EXPECT_EQ(variant->expected_fs_ioc_getflags_result,
+		  test_fs_ioc_getflags_ioctl(fd));
+
+	/* Checks that unrestrictable commands are unrestricted. */
+	EXPECT_EQ(0, ioctl(fd, FIOCLEX));
+	EXPECT_EQ(0, ioctl(fd, FIONCLEX));
+	EXPECT_EQ(0, ioctl(fd, FIONBIO, &flag));
+	EXPECT_EQ(0, ioctl(fd, FIOASYNC, &flag));
+
+	ASSERT_EQ(0, close(fd));
+}
+
+TEST_F_FORK(ioctl, handle_file_access_file)
+{
+	const char *const path = file1_s1d1;
+	const int flag = 0;
+	const struct rule rules[] = {
+		{
+			.path = path,
+			.access = variant->permitted,
+		},
+		{},
+	};
+	int fd, ruleset_fd;
+
+	if (variant->permitted & LANDLOCK_ACCESS_FS_READ_DIR) {
+		/* This access right can not be granted on files. */
+		return;
+	}
+
+	/* Enables Landlock. */
+	ruleset_fd = create_ruleset(_metadata, variant->handled, rules);
+	ASSERT_LE(0, ruleset_fd);
+	enforce_ruleset(_metadata, ruleset_fd);
+	ASSERT_EQ(0, close(ruleset_fd));
 
-	fd = memfd_create("name", MFD_CLOEXEC);
+	fd = open(path, variant->open_mode);
 	ASSERT_LE(0, fd);
 
 	/*
-	 * Checks that ftruncate is permitted on file descriptors that are
-	 * created in ways other than open(2).
+	 * Checks that IOCTL commands in each IOCTL group return the expected
+	 * errors.
 	 */
-	EXPECT_EQ(0, test_ftruncate(fd));
+	EXPECT_EQ(variant->expected_fioqsize_result, test_fioqsize_ioctl(fd));
+	EXPECT_EQ(variant->expected_fibmap_result, test_fibmap_ioctl(fd));
+	EXPECT_EQ(variant->expected_fionread_result, test_fionread_ioctl(fd));
+	EXPECT_EQ(variant->expected_fs_ioc_zero_range_result,
+		  test_fs_ioc_zero_range_ioctl(fd));
+	EXPECT_EQ(variant->expected_fs_ioc_getflags_result,
+		  test_fs_ioc_getflags_ioctl(fd));
+
+	/* Checks that unrestrictable commands are unrestricted. */
+	EXPECT_EQ(0, ioctl(fd, FIOCLEX));
+	EXPECT_EQ(0, ioctl(fd, FIONCLEX));
+	EXPECT_EQ(0, ioctl(fd, FIONBIO, &flag));
+	EXPECT_EQ(0, ioctl(fd, FIOASYNC, &flag));
 
 	ASSERT_EQ(0, close(fd));
 }
diff --git a/tools/testing/selftests/lsm/.gitignore b/tools/testing/selftests/lsm/.gitignore
new file mode 100644
index 000000000000..bd68f6c3fd07
--- /dev/null
+++ b/tools/testing/selftests/lsm/.gitignore
@@ -0,0 +1 @@
+/*_test
diff --git a/tools/testing/selftests/lsm/Makefile b/tools/testing/selftests/lsm/Makefile
new file mode 100644
index 000000000000..3f80c0bc093d
--- /dev/null
+++ b/tools/testing/selftests/lsm/Makefile
@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# First run: make -C ../../../.. headers_install
+
+CFLAGS += -Wall -O2 $(KHDR_INCLUDES)
+LOCAL_HDRS += common.h
+
+TEST_GEN_PROGS := lsm_get_self_attr_test lsm_list_modules_test \
+		  lsm_set_self_attr_test
+
+include ../lib.mk
+
+$(OUTPUT)/lsm_get_self_attr_test: lsm_get_self_attr_test.c common.c
+$(OUTPUT)/lsm_set_self_attr_test: lsm_set_self_attr_test.c common.c
+$(OUTPUT)/lsm_list_modules_test: lsm_list_modules_test.c common.c
+
+EXTRA_CLEAN = $(OUTPUT)/common.o
diff --git a/tools/testing/selftests/lsm/common.c b/tools/testing/selftests/lsm/common.c
new file mode 100644
index 000000000000..9ad258912646
--- /dev/null
+++ b/tools/testing/selftests/lsm/common.c
@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Linux Security Module infrastructure tests
+ *
+ * Copyright © 2023 Casey Schaufler <casey@schaufler-ca.com>
+ */
+
+#define _GNU_SOURCE
+#include <linux/lsm.h>
+#include <fcntl.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include "common.h"
+
+#define PROCATTR "/proc/self/attr/"
+
+int read_proc_attr(const char *attr, char *value, size_t size)
+{
+	int fd;
+	int len;
+	char *path;
+
+	len = strlen(PROCATTR) + strlen(attr) + 1;
+	path = calloc(len, 1);
+	if (path == NULL)
+		return -1;
+	sprintf(path, "%s%s", PROCATTR, attr);
+
+	fd = open(path, O_RDONLY);
+	free(path);
+
+	if (fd < 0)
+		return -1;
+	len = read(fd, value, size);
+
+	close(fd);
+
+	/* Ensure value is terminated */
+	if (len <= 0 || len == size)
+		return -1;
+	value[len] = '\0';
+
+	path = strchr(value, '\n');
+	if (path)
+		*path = '\0';
+
+	return 0;
+}
+
+int read_sysfs_lsms(char *lsms, size_t size)
+{
+	FILE *fp;
+	size_t red;
+
+	fp = fopen("/sys/kernel/security/lsm", "r");
+	if (fp == NULL)
+		return -1;
+	red = fread(lsms, 1, size, fp);
+	fclose(fp);
+
+	if (red <= 0 || red == size)
+		return -1;
+	lsms[red] = '\0';
+	return 0;
+}
+
+int attr_lsm_count(void)
+{
+	char *names = calloc(sysconf(_SC_PAGESIZE), 1);
+	int count = 0;
+
+	if (!names)
+		return 0;
+
+	if (read_sysfs_lsms(names, sysconf(_SC_PAGESIZE)))
+		return 0;
+
+	if (strstr(names, "selinux"))
+		count++;
+	if (strstr(names, "smack"))
+		count++;
+	if (strstr(names, "apparmor"))
+		count++;
+
+	return count;
+}
diff --git a/tools/testing/selftests/lsm/common.h b/tools/testing/selftests/lsm/common.h
new file mode 100644
index 000000000000..d404329e5eeb
--- /dev/null
+++ b/tools/testing/selftests/lsm/common.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Linux Security Module infrastructure tests
+ *
+ * Copyright © 2023 Casey Schaufler <casey@schaufler-ca.com>
+ */
+
+#ifndef lsm_get_self_attr
+static inline int lsm_get_self_attr(unsigned int attr, struct lsm_ctx *ctx,
+				    size_t *size, __u32 flags)
+{
+	return syscall(__NR_lsm_get_self_attr, attr, ctx, size, flags);
+}
+#endif
+
+#ifndef lsm_set_self_attr
+static inline int lsm_set_self_attr(unsigned int attr, struct lsm_ctx *ctx,
+				    size_t size, __u32 flags)
+{
+	return syscall(__NR_lsm_set_self_attr, attr, ctx, size, flags);
+}
+#endif
+
+#ifndef lsm_list_modules
+static inline int lsm_list_modules(__u64 *ids, size_t *size, __u32 flags)
+{
+	return syscall(__NR_lsm_list_modules, ids, size, flags);
+}
+#endif
+
+extern int read_proc_attr(const char *attr, char *value, size_t size);
+extern int read_sysfs_lsms(char *lsms, size_t size);
+int attr_lsm_count(void);
diff --git a/tools/testing/selftests/lsm/config b/tools/testing/selftests/lsm/config
new file mode 100644
index 000000000000..1c0c4c020f9c
--- /dev/null
+++ b/tools/testing/selftests/lsm/config
@@ -0,0 +1,3 @@
+CONFIG_SYSFS=y
+CONFIG_SECURITY=y
+CONFIG_SECURITYFS=y
diff --git a/tools/testing/selftests/lsm/lsm_get_self_attr_test.c b/tools/testing/selftests/lsm/lsm_get_self_attr_test.c
new file mode 100644
index 000000000000..e0e313d9047a
--- /dev/null
+++ b/tools/testing/selftests/lsm/lsm_get_self_attr_test.c
@@ -0,0 +1,275 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Linux Security Module infrastructure tests
+ * Tests for the lsm_get_self_attr system call
+ *
+ * Copyright © 2022 Casey Schaufler <casey@schaufler-ca.com>
+ */
+
+#define _GNU_SOURCE
+#include <linux/lsm.h>
+#include <fcntl.h>
+#include <string.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include "../kselftest_harness.h"
+#include "common.h"
+
+static struct lsm_ctx *next_ctx(struct lsm_ctx *ctxp)
+{
+	void *vp;
+
+	vp = (void *)ctxp + sizeof(*ctxp) + ctxp->ctx_len;
+	return (struct lsm_ctx *)vp;
+}
+
+TEST(size_null_lsm_get_self_attr)
+{
+	const long page_size = sysconf(_SC_PAGESIZE);
+	struct lsm_ctx *ctx = calloc(page_size, 1);
+
+	ASSERT_NE(NULL, ctx);
+	errno = 0;
+	ASSERT_EQ(-1, lsm_get_self_attr(LSM_ATTR_CURRENT, ctx, NULL, 0));
+	ASSERT_EQ(EINVAL, errno);
+
+	free(ctx);
+}
+
+TEST(ctx_null_lsm_get_self_attr)
+{
+	const long page_size = sysconf(_SC_PAGESIZE);
+	size_t size = page_size;
+	int rc;
+
+	rc = lsm_get_self_attr(LSM_ATTR_CURRENT, NULL, &size, 0);
+
+	if (attr_lsm_count()) {
+		ASSERT_NE(-1, rc);
+		ASSERT_NE(1, size);
+	} else {
+		ASSERT_EQ(-1, rc);
+	}
+}
+
+TEST(size_too_small_lsm_get_self_attr)
+{
+	const long page_size = sysconf(_SC_PAGESIZE);
+	struct lsm_ctx *ctx = calloc(page_size, 1);
+	size_t size = 1;
+
+	ASSERT_NE(NULL, ctx);
+	errno = 0;
+	ASSERT_EQ(-1, lsm_get_self_attr(LSM_ATTR_CURRENT, ctx, &size, 0));
+	if (attr_lsm_count()) {
+		ASSERT_EQ(E2BIG, errno);
+	} else {
+		ASSERT_EQ(EOPNOTSUPP, errno);
+	}
+	ASSERT_NE(1, size);
+
+	free(ctx);
+}
+
+TEST(flags_zero_lsm_get_self_attr)
+{
+	const long page_size = sysconf(_SC_PAGESIZE);
+	struct lsm_ctx *ctx = calloc(page_size, 1);
+	__u64 *syscall_lsms = calloc(page_size, 1);
+	size_t size;
+	int lsmcount;
+	int i;
+
+	ASSERT_NE(NULL, ctx);
+	errno = 0;
+	size = page_size;
+	ASSERT_EQ(-1, lsm_get_self_attr(LSM_ATTR_CURRENT, ctx, &size,
+					LSM_FLAG_SINGLE));
+	ASSERT_EQ(EINVAL, errno);
+	ASSERT_EQ(page_size, size);
+
+	lsmcount = syscall(__NR_lsm_list_modules, syscall_lsms, &size, 0);
+	ASSERT_LE(1, lsmcount);
+	ASSERT_NE(NULL, syscall_lsms);
+
+	for (i = 0; i < lsmcount; i++) {
+		errno = 0;
+		size = page_size;
+		ctx->id = syscall_lsms[i];
+
+		if (syscall_lsms[i] == LSM_ID_SELINUX ||
+		    syscall_lsms[i] == LSM_ID_SMACK ||
+		    syscall_lsms[i] == LSM_ID_APPARMOR) {
+			ASSERT_EQ(1, lsm_get_self_attr(LSM_ATTR_CURRENT, ctx,
+						       &size, LSM_FLAG_SINGLE));
+		} else {
+			ASSERT_EQ(-1, lsm_get_self_attr(LSM_ATTR_CURRENT, ctx,
+							&size,
+							LSM_FLAG_SINGLE));
+		}
+	}
+
+	free(ctx);
+}
+
+TEST(flags_overset_lsm_get_self_attr)
+{
+	const long page_size = sysconf(_SC_PAGESIZE);
+	struct lsm_ctx *ctx = calloc(page_size, 1);
+	size_t size;
+
+	ASSERT_NE(NULL, ctx);
+
+	errno = 0;
+	size = page_size;
+	ASSERT_EQ(-1, lsm_get_self_attr(LSM_ATTR_CURRENT | LSM_ATTR_PREV, ctx,
+					&size, 0));
+	ASSERT_EQ(EOPNOTSUPP, errno);
+
+	errno = 0;
+	size = page_size;
+	ASSERT_EQ(-1, lsm_get_self_attr(LSM_ATTR_CURRENT, ctx, &size,
+					LSM_FLAG_SINGLE |
+					(LSM_FLAG_SINGLE << 1)));
+	ASSERT_EQ(EINVAL, errno);
+
+	free(ctx);
+}
+
+TEST(basic_lsm_get_self_attr)
+{
+	const long page_size = sysconf(_SC_PAGESIZE);
+	size_t size = page_size;
+	struct lsm_ctx *ctx = calloc(page_size, 1);
+	struct lsm_ctx *tctx = NULL;
+	__u64 *syscall_lsms = calloc(page_size, 1);
+	char *attr = calloc(page_size, 1);
+	int cnt_current = 0;
+	int cnt_exec = 0;
+	int cnt_fscreate = 0;
+	int cnt_keycreate = 0;
+	int cnt_prev = 0;
+	int cnt_sockcreate = 0;
+	int lsmcount;
+	int count;
+	int i;
+
+	ASSERT_NE(NULL, ctx);
+	ASSERT_NE(NULL, syscall_lsms);
+
+	lsmcount = syscall(__NR_lsm_list_modules, syscall_lsms, &size, 0);
+	ASSERT_LE(1, lsmcount);
+
+	for (i = 0; i < lsmcount; i++) {
+		switch (syscall_lsms[i]) {
+		case LSM_ID_SELINUX:
+			cnt_current++;
+			cnt_exec++;
+			cnt_fscreate++;
+			cnt_keycreate++;
+			cnt_prev++;
+			cnt_sockcreate++;
+			break;
+		case LSM_ID_SMACK:
+			cnt_current++;
+			break;
+		case LSM_ID_APPARMOR:
+			cnt_current++;
+			cnt_exec++;
+			cnt_prev++;
+			break;
+		default:
+			break;
+		}
+	}
+
+	if (cnt_current) {
+		size = page_size;
+		count = lsm_get_self_attr(LSM_ATTR_CURRENT, ctx, &size, 0);
+		ASSERT_EQ(cnt_current, count);
+		tctx = ctx;
+		ASSERT_EQ(0, read_proc_attr("current", attr, page_size));
+		ASSERT_EQ(0, strcmp((char *)tctx->ctx, attr));
+		for (i = 1; i < count; i++) {
+			tctx = next_ctx(tctx);
+			ASSERT_NE(0, strcmp((char *)tctx->ctx, attr));
+		}
+	}
+	if (cnt_exec) {
+		size = page_size;
+		count = lsm_get_self_attr(LSM_ATTR_EXEC, ctx, &size, 0);
+		ASSERT_GE(cnt_exec, count);
+		if (count > 0) {
+			tctx = ctx;
+			if (read_proc_attr("exec", attr, page_size) == 0)
+				ASSERT_EQ(0, strcmp((char *)tctx->ctx, attr));
+		}
+		for (i = 1; i < count; i++) {
+			tctx = next_ctx(tctx);
+			ASSERT_NE(0, strcmp((char *)tctx->ctx, attr));
+		}
+	}
+	if (cnt_fscreate) {
+		size = page_size;
+		count = lsm_get_self_attr(LSM_ATTR_FSCREATE, ctx, &size, 0);
+		ASSERT_GE(cnt_fscreate, count);
+		if (count > 0) {
+			tctx = ctx;
+			if (read_proc_attr("fscreate", attr, page_size) == 0)
+				ASSERT_EQ(0, strcmp((char *)tctx->ctx, attr));
+		}
+		for (i = 1; i < count; i++) {
+			tctx = next_ctx(tctx);
+			ASSERT_NE(0, strcmp((char *)tctx->ctx, attr));
+		}
+	}
+	if (cnt_keycreate) {
+		size = page_size;
+		count = lsm_get_self_attr(LSM_ATTR_KEYCREATE, ctx, &size, 0);
+		ASSERT_GE(cnt_keycreate, count);
+		if (count > 0) {
+			tctx = ctx;
+			if (read_proc_attr("keycreate", attr, page_size) == 0)
+				ASSERT_EQ(0, strcmp((char *)tctx->ctx, attr));
+		}
+		for (i = 1; i < count; i++) {
+			tctx = next_ctx(tctx);
+			ASSERT_NE(0, strcmp((char *)tctx->ctx, attr));
+		}
+	}
+	if (cnt_prev) {
+		size = page_size;
+		count = lsm_get_self_attr(LSM_ATTR_PREV, ctx, &size, 0);
+		ASSERT_GE(cnt_prev, count);
+		if (count > 0) {
+			tctx = ctx;
+			ASSERT_EQ(0, read_proc_attr("prev", attr, page_size));
+			ASSERT_EQ(0, strcmp((char *)tctx->ctx, attr));
+			for (i = 1; i < count; i++) {
+				tctx = next_ctx(tctx);
+				ASSERT_NE(0, strcmp((char *)tctx->ctx, attr));
+			}
+		}
+	}
+	if (cnt_sockcreate) {
+		size = page_size;
+		count = lsm_get_self_attr(LSM_ATTR_SOCKCREATE, ctx, &size, 0);
+		ASSERT_GE(cnt_sockcreate, count);
+		if (count > 0) {
+			tctx = ctx;
+			if (read_proc_attr("sockcreate", attr, page_size) == 0)
+				ASSERT_EQ(0, strcmp((char *)tctx->ctx, attr));
+		}
+		for (i = 1; i < count; i++) {
+			tctx = next_ctx(tctx);
+			ASSERT_NE(0, strcmp((char *)tctx->ctx, attr));
+		}
+	}
+
+	free(ctx);
+	free(attr);
+	free(syscall_lsms);
+}
+
+TEST_HARNESS_MAIN
diff --git a/tools/testing/selftests/lsm/lsm_list_modules_test.c b/tools/testing/selftests/lsm/lsm_list_modules_test.c
new file mode 100644
index 000000000000..445c02f09c74
--- /dev/null
+++ b/tools/testing/selftests/lsm/lsm_list_modules_test.c
@@ -0,0 +1,140 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Linux Security Module infrastructure tests
+ * Tests for the lsm_list_modules system call
+ *
+ * Copyright © 2022 Casey Schaufler <casey@schaufler-ca.com>
+ */
+
+#define _GNU_SOURCE
+#include <linux/lsm.h>
+#include <string.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include "../kselftest_harness.h"
+#include "common.h"
+
+TEST(size_null_lsm_list_modules)
+{
+	const long page_size = sysconf(_SC_PAGESIZE);
+	__u64 *syscall_lsms = calloc(page_size, 1);
+
+	ASSERT_NE(NULL, syscall_lsms);
+	errno = 0;
+	ASSERT_EQ(-1, lsm_list_modules(syscall_lsms, NULL, 0));
+	ASSERT_EQ(EFAULT, errno);
+
+	free(syscall_lsms);
+}
+
+TEST(ids_null_lsm_list_modules)
+{
+	const long page_size = sysconf(_SC_PAGESIZE);
+	size_t size = page_size;
+
+	errno = 0;
+	ASSERT_EQ(-1, lsm_list_modules(NULL, &size, 0));
+	ASSERT_EQ(EFAULT, errno);
+	ASSERT_NE(1, size);
+}
+
+TEST(size_too_small_lsm_list_modules)
+{
+	const long page_size = sysconf(_SC_PAGESIZE);
+	__u64 *syscall_lsms = calloc(page_size, 1);
+	size_t size = 1;
+
+	ASSERT_NE(NULL, syscall_lsms);
+	errno = 0;
+	ASSERT_EQ(-1, lsm_list_modules(syscall_lsms, &size, 0));
+	ASSERT_EQ(E2BIG, errno);
+	ASSERT_NE(1, size);
+
+	free(syscall_lsms);
+}
+
+TEST(flags_set_lsm_list_modules)
+{
+	const long page_size = sysconf(_SC_PAGESIZE);
+	__u64 *syscall_lsms = calloc(page_size, 1);
+	size_t size = page_size;
+
+	ASSERT_NE(NULL, syscall_lsms);
+	errno = 0;
+	ASSERT_EQ(-1, lsm_list_modules(syscall_lsms, &size, 7));
+	ASSERT_EQ(EINVAL, errno);
+	ASSERT_EQ(page_size, size);
+
+	free(syscall_lsms);
+}
+
+TEST(correct_lsm_list_modules)
+{
+	const long page_size = sysconf(_SC_PAGESIZE);
+	size_t size = page_size;
+	__u64 *syscall_lsms = calloc(page_size, 1);
+	char *sysfs_lsms = calloc(page_size, 1);
+	char *name;
+	char *cp;
+	int count;
+	int i;
+
+	ASSERT_NE(NULL, sysfs_lsms);
+	ASSERT_NE(NULL, syscall_lsms);
+	ASSERT_EQ(0, read_sysfs_lsms(sysfs_lsms, page_size));
+
+	count = lsm_list_modules(syscall_lsms, &size, 0);
+	ASSERT_LE(1, count);
+	cp = sysfs_lsms;
+	for (i = 0; i < count; i++) {
+		switch (syscall_lsms[i]) {
+		case LSM_ID_CAPABILITY:
+			name = "capability";
+			break;
+		case LSM_ID_SELINUX:
+			name = "selinux";
+			break;
+		case LSM_ID_SMACK:
+			name = "smack";
+			break;
+		case LSM_ID_TOMOYO:
+			name = "tomoyo";
+			break;
+		case LSM_ID_IMA:
+			name = "ima";
+			break;
+		case LSM_ID_APPARMOR:
+			name = "apparmor";
+			break;
+		case LSM_ID_YAMA:
+			name = "yama";
+			break;
+		case LSM_ID_LOADPIN:
+			name = "loadpin";
+			break;
+		case LSM_ID_SAFESETID:
+			name = "safesetid";
+			break;
+		case LSM_ID_LOCKDOWN:
+			name = "lockdown";
+			break;
+		case LSM_ID_BPF:
+			name = "bpf";
+			break;
+		case LSM_ID_LANDLOCK:
+			name = "landlock";
+			break;
+		default:
+			name = "INVALID";
+			break;
+		}
+		ASSERT_EQ(0, strncmp(cp, name, strlen(name)));
+		cp += strlen(name) + 1;
+	}
+
+	free(sysfs_lsms);
+	free(syscall_lsms);
+}
+
+TEST_HARNESS_MAIN
diff --git a/tools/testing/selftests/lsm/lsm_set_self_attr_test.c b/tools/testing/selftests/lsm/lsm_set_self_attr_test.c
new file mode 100644
index 000000000000..e9712c6cf596
--- /dev/null
+++ b/tools/testing/selftests/lsm/lsm_set_self_attr_test.c
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Linux Security Module infrastructure tests
+ * Tests for the lsm_set_self_attr system call
+ *
+ * Copyright © 2022 Casey Schaufler <casey@schaufler-ca.com>
+ */
+
+#define _GNU_SOURCE
+#include <linux/lsm.h>
+#include <string.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include "../kselftest_harness.h"
+#include "common.h"
+
+TEST(ctx_null_lsm_set_self_attr)
+{
+	ASSERT_EQ(-1, lsm_set_self_attr(LSM_ATTR_CURRENT, NULL,
+					sizeof(struct lsm_ctx), 0));
+}
+
+TEST(size_too_small_lsm_set_self_attr)
+{
+	const long page_size = sysconf(_SC_PAGESIZE);
+	struct lsm_ctx *ctx = calloc(page_size, 1);
+	size_t size = page_size;
+
+	ASSERT_NE(NULL, ctx);
+	if (attr_lsm_count()) {
+		ASSERT_LE(1, lsm_get_self_attr(LSM_ATTR_CURRENT, ctx, &size,
+					       0));
+	}
+	ASSERT_EQ(-1, lsm_set_self_attr(LSM_ATTR_CURRENT, ctx, 1, 0));
+
+	free(ctx);
+}
+
+TEST(flags_zero_lsm_set_self_attr)
+{
+	const long page_size = sysconf(_SC_PAGESIZE);
+	struct lsm_ctx *ctx = calloc(page_size, 1);
+	size_t size = page_size;
+
+	ASSERT_NE(NULL, ctx);
+	if (attr_lsm_count()) {
+		ASSERT_LE(1, lsm_get_self_attr(LSM_ATTR_CURRENT, ctx, &size,
+					       0));
+	}
+	ASSERT_EQ(-1, lsm_set_self_attr(LSM_ATTR_CURRENT, ctx, size, 1));
+
+	free(ctx);
+}
+
+TEST(flags_overset_lsm_set_self_attr)
+{
+	const long page_size = sysconf(_SC_PAGESIZE);
+	char *ctx = calloc(page_size, 1);
+	size_t size = page_size;
+	struct lsm_ctx *tctx = (struct lsm_ctx *)ctx;
+
+	ASSERT_NE(NULL, ctx);
+	if (attr_lsm_count()) {
+		ASSERT_LE(1, lsm_get_self_attr(LSM_ATTR_CURRENT, tctx, &size,
+					       0));
+	}
+	ASSERT_EQ(-1, lsm_set_self_attr(LSM_ATTR_CURRENT | LSM_ATTR_PREV, tctx,
+					size, 0));
+
+	free(ctx);
+}
+
+TEST_HARNESS_MAIN
diff --git a/tools/testing/selftests/mm/pagemap_ioctl.c b/tools/testing/selftests/mm/pagemap_ioctl.c
index befab43719ba..d59517ed3d48 100644
--- a/tools/testing/selftests/mm/pagemap_ioctl.c
+++ b/tools/testing/selftests/mm/pagemap_ioctl.c
@@ -36,6 +36,7 @@ int pagemap_fd;
 int uffd;
 int page_size;
 int hpage_size;
+const char *progname;
 
 #define LEN(region)	((region.end - region.start)/page_size)
 
@@ -1149,11 +1150,11 @@ int sanity_tests(void)
 	munmap(mem, mem_size);
 
 	/* 9. Memory mapped file */
-	fd = open(__FILE__, O_RDONLY);
+	fd = open(progname, O_RDONLY);
 	if (fd < 0)
 		ksft_exit_fail_msg("%s Memory mapped file\n", __func__);
 
-	ret = stat(__FILE__, &sbuf);
+	ret = stat(progname, &sbuf);
 	if (ret < 0)
 		ksft_exit_fail_msg("error %d %d %s\n", ret, errno, strerror(errno));
 
@@ -1472,12 +1473,14 @@ static void transact_test(int page_size)
 			      extra_thread_faults);
 }
 
-int main(void)
+int main(int argc, char *argv[])
 {
 	int mem_size, shmid, buf_size, fd, i, ret;
 	char *mem, *map, *fmem;
 	struct stat sbuf;
 
+	progname = argv[0];
+
 	ksft_print_header();
 
 	if (init_uffd())
diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c
index 3082b40492dd..4aeb8d5299ff 100644
--- a/tools/testing/selftests/mm/vm_util.c
+++ b/tools/testing/selftests/mm/vm_util.c
@@ -4,6 +4,7 @@
 #include <dirent.h>
 #include <sys/ioctl.h>
 #include <linux/userfaultfd.h>
+#include <linux/fs.h>
 #include <sys/syscall.h>
 #include <unistd.h>
 #include "../kselftest.h"
@@ -28,19 +29,92 @@ uint64_t pagemap_get_entry(int fd, char *start)
 	return entry;
 }
 
+static uint64_t __pagemap_scan_get_categories(int fd, char *start, struct page_region *r)
+{
+	struct pm_scan_arg arg;
+
+	arg.start = (uintptr_t)start;
+	arg.end = (uintptr_t)(start + psize());
+	arg.vec = (uintptr_t)r;
+	arg.vec_len = 1;
+	arg.flags = 0;
+	arg.size = sizeof(struct pm_scan_arg);
+	arg.max_pages = 0;
+	arg.category_inverted = 0;
+	arg.category_mask = 0;
+	arg.category_anyof_mask = PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | PAGE_IS_FILE |
+				  PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_PFNZERO |
+				  PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY;
+	arg.return_mask = arg.category_anyof_mask;
+
+	return ioctl(fd, PAGEMAP_SCAN, &arg);
+}
+
+static uint64_t pagemap_scan_get_categories(int fd, char *start)
+{
+	struct page_region r;
+	long ret;
+
+	ret = __pagemap_scan_get_categories(fd, start, &r);
+	if (ret < 0)
+		ksft_exit_fail_msg("PAGEMAP_SCAN failed: %s\n", strerror(errno));
+	if (ret == 0)
+		return 0;
+	return r.categories;
+}
+
+/* `start` is any valid address. */
+static bool pagemap_scan_supported(int fd, char *start)
+{
+	static int supported = -1;
+	int ret;
+
+	if (supported != -1)
+		return supported;
+
+	/* Provide an invalid address in order to trigger EFAULT. */
+	ret = __pagemap_scan_get_categories(fd, start, (struct page_region *) ~0UL);
+	if (ret == 0)
+		ksft_exit_fail_msg("PAGEMAP_SCAN succedded unexpectedly\n");
+
+	supported = errno == EFAULT;
+
+	return supported;
+}
+
+static bool page_entry_is(int fd, char *start, char *desc,
+			  uint64_t pagemap_flags, uint64_t pagescan_flags)
+{
+	bool m = pagemap_get_entry(fd, start) & pagemap_flags;
+
+	if (pagemap_scan_supported(fd, start)) {
+		bool s = pagemap_scan_get_categories(fd, start) & pagescan_flags;
+
+		if (m == s)
+			return m;
+
+		ksft_exit_fail_msg(
+			"read and ioctl return unmatched results for %s: %d %d", desc, m, s);
+	}
+	return m;
+}
+
 bool pagemap_is_softdirty(int fd, char *start)
 {
-	return pagemap_get_entry(fd, start) & PM_SOFT_DIRTY;
+	return page_entry_is(fd, start, "soft-dirty",
+				PM_SOFT_DIRTY, PAGE_IS_SOFT_DIRTY);
 }
 
 bool pagemap_is_swapped(int fd, char *start)
 {
-	return pagemap_get_entry(fd, start) & PM_SWAP;
+	return page_entry_is(fd, start, "swap", PM_SWAP, PAGE_IS_SWAPPED);
 }
 
 bool pagemap_is_populated(int fd, char *start)
 {
-	return pagemap_get_entry(fd, start) & (PM_PRESENT | PM_SWAP);
+	return page_entry_is(fd, start, "populated",
+				PM_PRESENT | PM_SWAP,
+				PAGE_IS_PRESENT | PAGE_IS_SWAPPED);
 }
 
 unsigned long pagemap_get_pfn(int fd, char *start)
diff --git a/tools/testing/selftests/net/Makefile b/tools/testing/selftests/net/Makefile
index 5b2aca4c5f10..9274edfb76ff 100644
--- a/tools/testing/selftests/net/Makefile
+++ b/tools/testing/selftests/net/Makefile
@@ -91,6 +91,7 @@ TEST_PROGS += test_bridge_neigh_suppress.sh
 TEST_PROGS += test_vxlan_nolocalbypass.sh
 TEST_PROGS += test_bridge_backup_port.sh
 TEST_PROGS += fdb_flush.sh
+TEST_PROGS += fq_band_pktlimit.sh
 
 TEST_FILES := settings
 
diff --git a/tools/testing/selftests/net/cmsg_sender.c b/tools/testing/selftests/net/cmsg_sender.c
index 24b21b15ed3f..8d7575389f58 100644
--- a/tools/testing/selftests/net/cmsg_sender.c
+++ b/tools/testing/selftests/net/cmsg_sender.c
@@ -45,11 +45,13 @@ struct options {
 	const char *host;
 	const char *service;
 	unsigned int size;
+	unsigned int num_pkt;
 	struct {
 		unsigned int mark;
 		unsigned int dontfrag;
 		unsigned int tclass;
 		unsigned int hlimit;
+		unsigned int priority;
 	} sockopt;
 	struct {
 		unsigned int family;
@@ -72,6 +74,7 @@ struct options {
 	} v6;
 } opt = {
 	.size = 13,
+	.num_pkt = 1,
 	.sock = {
 		.family	= AF_UNSPEC,
 		.type	= SOCK_DGRAM,
@@ -112,7 +115,7 @@ static void cs_parse_args(int argc, char *argv[])
 {
 	int o;
 
-	while ((o = getopt(argc, argv, "46sS:p:m:M:d:tf:F:c:C:l:L:H:")) != -1) {
+	while ((o = getopt(argc, argv, "46sS:p:P:m:M:n:d:tf:F:c:C:l:L:H:")) != -1) {
 		switch (o) {
 		case 's':
 			opt.silent_send = true;
@@ -138,7 +141,9 @@ static void cs_parse_args(int argc, char *argv[])
 				cs_usage(argv[0]);
 			}
 			break;
-
+		case 'P':
+			opt.sockopt.priority = atoi(optarg);
+			break;
 		case 'm':
 			opt.mark.ena = true;
 			opt.mark.val = atoi(optarg);
@@ -146,6 +151,9 @@ static void cs_parse_args(int argc, char *argv[])
 		case 'M':
 			opt.sockopt.mark = atoi(optarg);
 			break;
+		case 'n':
+			opt.num_pkt = atoi(optarg);
+			break;
 		case 'd':
 			opt.txtime.ena = true;
 			opt.txtime.delay = atoi(optarg);
@@ -410,6 +418,10 @@ static void ca_set_sockopts(int fd)
 	    setsockopt(fd, SOL_IPV6, IPV6_UNICAST_HOPS,
 		       &opt.sockopt.hlimit, sizeof(opt.sockopt.hlimit)))
 		error(ERN_SOCKOPT, errno, "setsockopt IPV6_HOPLIMIT");
+	if (opt.sockopt.priority &&
+	    setsockopt(fd, SOL_SOCKET, SO_PRIORITY,
+		       &opt.sockopt.priority, sizeof(opt.sockopt.priority)))
+		error(ERN_SOCKOPT, errno, "setsockopt SO_PRIORITY");
 }
 
 int main(int argc, char *argv[])
@@ -421,6 +433,7 @@ int main(int argc, char *argv[])
 	char *buf;
 	int err;
 	int fd;
+	int i;
 
 	cs_parse_args(argc, argv);
 
@@ -480,24 +493,27 @@ int main(int argc, char *argv[])
 
 	cs_write_cmsg(fd, &msg, cbuf, sizeof(cbuf));
 
-	err = sendmsg(fd, &msg, 0);
-	if (err < 0) {
-		if (!opt.silent_send)
-			fprintf(stderr, "send failed: %s\n", strerror(errno));
-		err = ERN_SEND;
-		goto err_out;
-	} else if (err != (int)opt.size) {
-		fprintf(stderr, "short send\n");
-		err = ERN_SEND_SHORT;
-		goto err_out;
-	} else {
-		err = ERN_SUCCESS;
+	for (i = 0; i < opt.num_pkt; i++) {
+		err = sendmsg(fd, &msg, 0);
+		if (err < 0) {
+			if (!opt.silent_send)
+				fprintf(stderr, "send failed: %s\n", strerror(errno));
+			err = ERN_SEND;
+			goto err_out;
+		} else if (err != (int)opt.size) {
+			fprintf(stderr, "short send\n");
+			err = ERN_SEND_SHORT;
+			goto err_out;
+		}
 	}
+	err = ERN_SUCCESS;
 
-	/* Make sure all timestamps have time to loop back */
-	usleep(opt.txtime.delay);
+	if (opt.ts.ena) {
+		/* Make sure all timestamps have time to loop back */
+		usleep(opt.txtime.delay);
 
-	cs_read_cmsg(fd, &msg, cbuf, sizeof(cbuf));
+		cs_read_cmsg(fd, &msg, cbuf, sizeof(cbuf));
+	}
 
 err_out:
 	close(fd);
diff --git a/tools/testing/selftests/net/fq_band_pktlimit.sh b/tools/testing/selftests/net/fq_band_pktlimit.sh
new file mode 100755
index 000000000000..24b77bdf41ff
--- /dev/null
+++ b/tools/testing/selftests/net/fq_band_pktlimit.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# Verify that FQ has a packet limit per band:
+#
+# 1. set the limit to 10 per band
+# 2. send 20 pkts on band A: verify that 10 are queued, 10 dropped
+# 3. send 20 pkts on band A: verify that  0 are queued, 20 dropped
+# 4. send 20 pkts on band B: verify that 10 are queued, 10 dropped
+#
+# Send packets with a 100ms delay to ensure that previously sent
+# packets are still queued when later ones are sent.
+# Use SO_TXTIME for this.
+
+die() {
+	echo "$1"
+	exit 1
+}
+
+# run inside private netns
+if [[ $# -eq 0 ]]; then
+	./in_netns.sh "$0" __subprocess
+	exit
+fi
+
+ip link add type dummy
+ip link set dev dummy0 up
+ip -6 addr add fdaa::1/128 dev dummy0
+ip -6 route add fdaa::/64 dev dummy0
+tc qdisc replace dev dummy0 root handle 1: fq quantum 1514 initial_quantum 1514 limit 10
+
+./cmsg_sender -6 -p u -d 100000 -n 20 fdaa::2 8000
+OUT1="$(tc -s qdisc show dev dummy0 | grep '^\ Sent')"
+
+./cmsg_sender -6 -p u -d 100000 -n 20 fdaa::2 8000
+OUT2="$(tc -s qdisc show dev dummy0 | grep '^\ Sent')"
+
+./cmsg_sender -6 -p u -d 100000 -n 20 -P 7 fdaa::2 8000
+OUT3="$(tc -s qdisc show dev dummy0 | grep '^\ Sent')"
+
+# Initial stats will report zero sent, as all packets are still
+# queued in FQ. Sleep for the delay period (100ms) and see that
+# twenty are now sent.
+sleep 0.1
+OUT4="$(tc -s qdisc show dev dummy0 | grep '^\ Sent')"
+
+# Log the output after the test
+echo "${OUT1}"
+echo "${OUT2}"
+echo "${OUT3}"
+echo "${OUT4}"
+
+# Test the output for expected values
+echo "${OUT1}" | grep -q '0\ pkt\ (dropped\ 10'  || die "unexpected drop count at 1"
+echo "${OUT2}" | grep -q '0\ pkt\ (dropped\ 30'  || die "unexpected drop count at 2"
+echo "${OUT3}" | grep -q '0\ pkt\ (dropped\ 40'  || die "unexpected drop count at 3"
+echo "${OUT4}" | grep -q '20\ pkt\ (dropped\ 40' || die "unexpected accept count at 4"
diff --git a/tools/testing/selftests/net/net_helper.sh b/tools/testing/selftests/net/net_helper.sh
new file mode 100755
index 000000000000..4fe0befa13fb
--- /dev/null
+++ b/tools/testing/selftests/net/net_helper.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# Helper functions
+
+wait_local_port_listen()
+{
+	local listener_ns="${1}"
+	local port="${2}"
+	local protocol="${3}"
+	local port_hex
+	local i
+
+	port_hex="$(printf "%04X" "${port}")"
+	for i in $(seq 10); do
+		if ip netns exec "${listener_ns}" cat /proc/net/"${protocol}"* | \
+		   grep -q "${port_hex}"; then
+			break
+		fi
+		sleep 0.1
+	done
+}
diff --git a/tools/testing/selftests/net/rtnetlink.sh b/tools/testing/selftests/net/rtnetlink.sh
index 5f2b3f6c0d74..38be9706c45f 100755
--- a/tools/testing/selftests/net/rtnetlink.sh
+++ b/tools/testing/selftests/net/rtnetlink.sh
@@ -859,7 +859,7 @@ kci_test_gretap()
 
 
 	run_cmd ip -netns "$testns" addr add dev "$DEV_NS" 10.1.1.100/24
-	run_cmd ip -netns "$testns" link set dev $DEV_NS ups
+	run_cmd ip -netns "$testns" link set dev $DEV_NS up
 	run_cmd ip -netns "$testns" link del "$DEV_NS"
 
 	# test external mode
diff --git a/tools/testing/selftests/net/udpgro.sh b/tools/testing/selftests/net/udpgro.sh
index 0c743752669a..af5dc57c8ce9 100755
--- a/tools/testing/selftests/net/udpgro.sh
+++ b/tools/testing/selftests/net/udpgro.sh
@@ -3,6 +3,8 @@
 #
 # Run a series of udpgro functional tests.
 
+source net_helper.sh
+
 readonly PEER_NS="ns-peer-$(mktemp -u XXXXXX)"
 
 BPF_FILE="../bpf/xdp_dummy.bpf.o"
@@ -51,8 +53,7 @@ run_one() {
 		echo "ok" || \
 		echo "failed" &
 
-	# Hack: let bg programs complete the startup
-	sleep 0.2
+	wait_local_port_listen ${PEER_NS} 8000 udp
 	./udpgso_bench_tx ${tx_args}
 	ret=$?
 	wait $(jobs -p)
@@ -97,7 +98,7 @@ run_one_nat() {
 		echo "ok" || \
 		echo "failed"&
 
-	sleep 0.1
+	wait_local_port_listen "${PEER_NS}" 8000 udp
 	./udpgso_bench_tx ${tx_args}
 	ret=$?
 	kill -INT $pid
@@ -118,11 +119,9 @@ run_one_2sock() {
 		echo "ok" || \
 		echo "failed" &
 
-	# Hack: let bg programs complete the startup
-	sleep 0.2
+	wait_local_port_listen "${PEER_NS}" 12345 udp
 	./udpgso_bench_tx ${tx_args} -p 12345
-	sleep 0.1
-	# first UDP GSO socket should be closed at this point
+	wait_local_port_listen "${PEER_NS}" 8000 udp
 	./udpgso_bench_tx ${tx_args}
 	ret=$?
 	wait $(jobs -p)
diff --git a/tools/testing/selftests/net/udpgro_bench.sh b/tools/testing/selftests/net/udpgro_bench.sh
index 894972877e8b..cb664679b434 100755
--- a/tools/testing/selftests/net/udpgro_bench.sh
+++ b/tools/testing/selftests/net/udpgro_bench.sh
@@ -3,6 +3,8 @@
 #
 # Run a series of udpgro benchmarks
 
+source net_helper.sh
+
 readonly PEER_NS="ns-peer-$(mktemp -u XXXXXX)"
 
 BPF_FILE="../bpf/xdp_dummy.bpf.o"
@@ -40,8 +42,7 @@ run_one() {
 	ip netns exec "${PEER_NS}" ./udpgso_bench_rx ${rx_args} -r &
 	ip netns exec "${PEER_NS}" ./udpgso_bench_rx -t ${rx_args} -r &
 
-	# Hack: let bg programs complete the startup
-	sleep 0.2
+	wait_local_port_listen "${PEER_NS}" 8000 udp
 	./udpgso_bench_tx ${tx_args}
 }
 
diff --git a/tools/testing/selftests/net/udpgro_frglist.sh b/tools/testing/selftests/net/udpgro_frglist.sh
index 0a6359bed0b9..dd47fa96f6b3 100755
--- a/tools/testing/selftests/net/udpgro_frglist.sh
+++ b/tools/testing/selftests/net/udpgro_frglist.sh
@@ -3,6 +3,8 @@
 #
 # Run a series of udpgro benchmarks
 
+source net_helper.sh
+
 readonly PEER_NS="ns-peer-$(mktemp -u XXXXXX)"
 
 BPF_FILE="../bpf/xdp_dummy.bpf.o"
@@ -45,8 +47,7 @@ run_one() {
         echo ${rx_args}
 	ip netns exec "${PEER_NS}" ./udpgso_bench_rx ${rx_args} -r &
 
-	# Hack: let bg programs complete the startup
-	sleep 0.2
+	wait_local_port_listen "${PEER_NS}" 8000 udp
 	./udpgso_bench_tx ${tx_args}
 }
 
diff --git a/tools/testing/selftests/tc-testing/plugin-lib/nsPlugin.py b/tools/testing/selftests/tc-testing/plugin-lib/nsPlugin.py
index b62429b0fcdb..65c8f3f983b9 100644
--- a/tools/testing/selftests/tc-testing/plugin-lib/nsPlugin.py
+++ b/tools/testing/selftests/tc-testing/plugin-lib/nsPlugin.py
@@ -9,43 +9,13 @@ from TdcPlugin import TdcPlugin
 
 from tdc_config import *
 
-def prepare_suite(obj, test):
-    original = obj.args.NAMES
-
-    if 'skip' in test and test['skip'] == 'yes':
-        return
-
-    if 'nsPlugin' not in test['plugins']:
-        return
-
-    shadow = {}
-    shadow['IP'] = original['IP']
-    shadow['TC'] = original['TC']
-    shadow['NS'] = '{}-{}'.format(original['NS'], test['random'])
-    shadow['DEV0'] = '{}id{}'.format(original['DEV0'], test['id'])
-    shadow['DEV1'] = '{}id{}'.format(original['DEV1'], test['id'])
-    shadow['DUMMY'] = '{}id{}'.format(original['DUMMY'], test['id'])
-    shadow['DEV2'] = original['DEV2']
-    obj.args.NAMES = shadow
-
-    if obj.args.namespace:
-        obj._ns_create()
-    else:
-        obj._ports_create()
-
-    # Make sure the netns is visible in the fs
-    while True:
-        obj._proc_check()
-        try:
-            ns = obj.args.NAMES['NS']
-            f = open('/run/netns/{}'.format(ns))
-            f.close()
-            break
-        except:
-            time.sleep(0.1)
-            continue
-
-    obj.args.NAMES = original
+try:
+    from pyroute2 import netns
+    from pyroute2 import IPRoute
+    netlink = True
+except ImportError:
+    netlink = False
+    print("!!! Consider installing pyroute2 !!!")
 
 class SubPlugin(TdcPlugin):
     def __init__(self):
@@ -57,60 +27,69 @@ class SubPlugin(TdcPlugin):
 
         super().pre_suite(testcount, testlist)
 
-        print("Setting up namespaces and devices...")
+    def prepare_test(self, test):
+        if 'skip' in test and test['skip'] == 'yes':
+            return
 
-        with Pool(self.args.mp) as p:
-            it = zip(cycle([self]), testlist)
-            p.starmap(prepare_suite, it)
+        if 'nsPlugin' not in test['plugins']:
+            return
 
-    def pre_case(self, caseinfo, test_skip):
+        if netlink == True:
+            self._nl_ns_create()
+        else:
+            self._ns_create()
+
+        # Make sure the netns is visible in the fs
+        ticks = 20
+        while True:
+            if ticks == 0:
+                raise TimeoutError
+            self._proc_check()
+            try:
+                ns = self.args.NAMES['NS']
+                f = open('/run/netns/{}'.format(ns))
+                f.close()
+                break
+            except:
+                time.sleep(0.1)
+                ticks -= 1
+                continue
+
+    def pre_case(self, test, test_skip):
         if self.args.verbose:
             print('{}.pre_case'.format(self.sub_class))
 
         if test_skip:
             return
 
+        self.prepare_test(test)
 
     def post_case(self):
         if self.args.verbose:
             print('{}.post_case'.format(self.sub_class))
 
-        if self.args.namespace:
-            self._ns_destroy()
+        if netlink == True:
+            self._nl_ns_destroy()
         else:
-            self._ports_destroy()
+            self._ns_destroy()
 
     def post_suite(self, index):
         if self.args.verbose:
             print('{}.post_suite'.format(self.sub_class))
 
         # Make sure we don't leak resources
-        for f in os.listdir('/run/netns/'):
-            cmd = self._replace_keywords("$IP netns del {}".format(f))
-
-            if self.args.verbose > 3:
-                print('_exec_cmd:  command "{}"'.format(cmd))
+        cmd = "$IP -a netns del"
 
-            subprocess.run(cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        if self.args.verbose > 3:
+            print('_exec_cmd:  command "{}"'.format(cmd))
 
-    def add_args(self, parser):
-        super().add_args(parser)
-        self.argparser_group = self.argparser.add_argument_group(
-            'netns',
-            'options for nsPlugin(run commands in net namespace)')
-        self.argparser_group.add_argument(
-            '-N', '--no-namespace', action='store_false', default=True,
-            dest='namespace', help='Don\'t run commands in namespace')
-        return self.argparser
+        subprocess.run(cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
 
     def adjust_command(self, stage, command):
         super().adjust_command(stage, command)
         cmdform = 'list'
         cmdlist = list()
 
-        if not self.args.namespace:
-            return command
-
         if self.args.verbose:
             print('{}.adjust_command'.format(self.sub_class))
 
@@ -138,41 +117,67 @@ class SubPlugin(TdcPlugin):
             print('adjust_command:  return command [{}]'.format(command))
         return command
 
-    def _ports_create_cmds(self):
-        cmds = []
-
-        cmds.append(self._replace_keywords('link add $DEV0 type veth peer name $DEV1'))
-        cmds.append(self._replace_keywords('link set $DEV0 up'))
-        cmds.append(self._replace_keywords('link add $DUMMY type dummy'))
-        if not self.args.namespace:
-            cmds.append(self._replace_keywords('link set $DEV1 up'))
-
-        return cmds
+    def _nl_ns_create(self):
+        ns = self.args.NAMES["NS"];
+        dev0 = self.args.NAMES["DEV0"];
+        dev1 = self.args.NAMES["DEV1"];
+        dummy = self.args.NAMES["DUMMY"];
 
-    def _ports_create(self):
-        self._exec_cmd_batched('pre', self._ports_create_cmds())
-
-    def _ports_destroy_cmd(self):
-        return self._replace_keywords('link del $DEV0')
-
-    def _ports_destroy(self):
-        self._exec_cmd('post', self._ports_destroy_cmd())
+        if self.args.verbose:
+            print('{}._nl_ns_create'.format(self.sub_class))
+
+        netns.create(ns)
+        netns.pushns(newns=ns)
+        with IPRoute() as ip:
+            ip.link('add', ifname=dev1, kind='veth', peer={'ifname': dev0, 'net_ns_fd':'/proc/1/ns/net'})
+            ip.link('add', ifname=dummy, kind='dummy')
+            ticks = 20
+            while True:
+                if ticks == 0:
+                    raise TimeoutError
+                try:
+                    dev1_idx = ip.link_lookup(ifname=dev1)[0]
+                    dummy_idx = ip.link_lookup(ifname=dummy)[0]
+                    ip.link('set', index=dev1_idx, state='up')
+                    ip.link('set', index=dummy_idx, state='up')
+                    break
+                except:
+                    time.sleep(0.1)
+                    ticks -= 1
+                    continue
+        netns.popns()
+
+        with IPRoute() as ip:
+            ticks = 20
+            while True:
+                if ticks == 0:
+                    raise TimeoutError
+                try:
+                    dev0_idx = ip.link_lookup(ifname=dev0)[0]
+                    ip.link('set', index=dev0_idx, state='up')
+                    break
+                except:
+                    time.sleep(0.1)
+                    ticks -= 1
+                    continue
 
     def _ns_create_cmds(self):
         cmds = []
 
-        if self.args.namespace:
-            ns = self.args.NAMES['NS']
+        ns = self.args.NAMES['NS']
 
-            cmds.append(self._replace_keywords('netns add {}'.format(ns)))
-            cmds.append(self._replace_keywords('link set $DEV1 netns {}'.format(ns)))
-            cmds.append(self._replace_keywords('link set $DUMMY netns {}'.format(ns)))
-            cmds.append(self._replace_keywords('netns exec {} $IP link set $DEV1 up'.format(ns)))
-            cmds.append(self._replace_keywords('netns exec {} $IP link set $DUMMY up'.format(ns)))
+        cmds.append(self._replace_keywords('netns add {}'.format(ns)))
+        cmds.append(self._replace_keywords('link add $DEV1 type veth peer name $DEV0'))
+        cmds.append(self._replace_keywords('link set $DEV1 netns {}'.format(ns)))
+        cmds.append(self._replace_keywords('link add $DUMMY type dummy'.format(ns)))
+        cmds.append(self._replace_keywords('link set $DUMMY netns {}'.format(ns)))
+        cmds.append(self._replace_keywords('netns exec {} $IP link set $DEV1 up'.format(ns)))
+        cmds.append(self._replace_keywords('netns exec {} $IP link set $DUMMY up'.format(ns)))
+        cmds.append(self._replace_keywords('link set $DEV0 up'.format(ns)))
 
-            if self.args.device:
-                cmds.append(self._replace_keywords('link set $DEV2 netns {}'.format(ns)))
-                cmds.append(self._replace_keywords('netns exec {} $IP link set $DEV2 up'.format(ns)))
+        if self.args.device:
+            cmds.append(self._replace_keywords('link set $DEV2 netns {}'.format(ns)))
+            cmds.append(self._replace_keywords('netns exec {} $IP link set $DEV2 up'.format(ns)))
 
         return cmds
 
@@ -181,9 +186,12 @@ class SubPlugin(TdcPlugin):
         Create the network namespace in which the tests will be run and set up
         the required network devices for it.
         '''
-        self._ports_create()
         self._exec_cmd_batched('pre', self._ns_create_cmds())
 
+    def _nl_ns_destroy(self):
+        ns = self.args.NAMES['NS']
+        netns.remove(ns)
+
     def _ns_destroy_cmd(self):
         return self._replace_keywords('netns delete {}'.format(self.args.NAMES['NS']))
 
@@ -192,9 +200,7 @@ class SubPlugin(TdcPlugin):
         Destroy the network namespace for testing (and any associated network
         devices as well)
         '''
-        if self.args.namespace:
-            self._exec_cmd('post', self._ns_destroy_cmd())
-            self._ports_destroy()
+        self._exec_cmd('post', self._ns_destroy_cmd())
 
     @cached_property
     def _proc(self):
diff --git a/tools/testing/selftests/tc-testing/tc-tests/filters/u32.json b/tools/testing/selftests/tc-testing/tc-tests/filters/u32.json
index ddc7c355be0a..24bd0c2a3014 100644
--- a/tools/testing/selftests/tc-testing/tc-tests/filters/u32.json
+++ b/tools/testing/selftests/tc-testing/tc-tests/filters/u32.json
@@ -272,5 +272,62 @@
         "teardown": [
             "$TC qdisc del dev $DEV1 parent root drr"
         ]
+    },
+    {
+        "id": "bd32",
+        "name": "Try to delete hashtable referenced by another u32 filter",
+        "category": [
+            "filter",
+            "u32"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 parent root handle 10: drr",
+            "$TC filter add dev $DEV1 parent 10:0 protocol ip prio 2 handle 1: u32 divisor 1",
+            "$TC filter add dev $DEV1 parent 10:0 protocol ip prio 2 u32 ht 800: match ip src any link 1:"
+        ],
+        "cmdUnderTest": "$TC filter delete dev $DEV1 parent 10: prio 2 handle 1: u32",
+        "expExitCode": "2",
+        "verifyCmd": "$TC filter show dev $DEV1",
+        "matchPattern": "protocol ip pref 2 u32 chain 0 fh 1:",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 parent root drr"
+        ]
+    },
+    {
+        "id": "4585",
+        "name": "Delete small tree of u32 hashtables and filters",
+        "category": [
+            "filter",
+            "u32"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 parent root handle 10: drr",
+            "$TC filter add dev $DEV1 parent 10:0 protocol ip prio 2 handle 1: u32 divisor 1",
+            "$TC filter add dev $DEV1 parent 10:0 protocol ip prio 2 handle 2: u32 divisor 1",
+            "$TC filter add dev $DEV1 parent 10:0 protocol ip prio 2 handle 3: u32 divisor 2",
+            "$TC filter add dev $DEV1 parent 10:0 protocol ip prio 2 handle 4: u32 divisor 1",
+            "$TC filter add dev $DEV1 parent 10:0 protocol ip prio 2 u32 ht 1: match ip src any action drop",
+            "$TC filter add dev $DEV1 parent 10:0 protocol ip prio 2 u32 ht 2: match ip src any action drop",
+            "$TC filter add dev $DEV1 parent 10:0 protocol ip prio 2 u32 ht 3: match ip src any link 2:",
+            "$TC filter add dev $DEV1 parent 10:0 protocol ip prio 2 u32 ht 3: match ip src any link 1:",
+            "$TC filter add dev $DEV1 parent 10:0 protocol ip prio 2 u32 ht 4: match ip src any action drop",
+            "$TC filter add dev $DEV1 parent 10:0 protocol ip prio 2 u32 ht 800: match ip src any link 3:",
+            "$TC filter add dev $DEV1 parent 10:0 protocol ip prio 2 u32 ht 800: match ip src any link 4:"
+        ],
+        "cmdUnderTest": "$TC filter delete dev $DEV1 parent 10:",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1",
+        "matchPattern": "protocol ip pref 2 u32",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 parent root drr"
+        ]
     }
 ]
diff --git a/tools/testing/selftests/tc-testing/tdc.py b/tools/testing/selftests/tc-testing/tdc.py
index a6718192aff3..669ec89ebfe1 100755
--- a/tools/testing/selftests/tc-testing/tdc.py
+++ b/tools/testing/selftests/tc-testing/tdc.py
@@ -616,7 +616,7 @@ def test_runner_mp(pm, args, alltests):
     batches.insert(0, serial)
 
     print("Executing {} tests in parallel and {} in serial".format(len(parallel), len(serial)))
-    print("Using {} batches".format(len(batches)))
+    print("Using {} batches and {} workers".format(len(batches), args.mp))
 
     # We can't pickle these objects so workaround them
     global mp_pm
@@ -1017,6 +1017,7 @@ def main():
     parser = pm.call_add_args(parser)
     (args, remaining) = parser.parse_known_args()
     args.NAMES = NAMES
+    args.mp = min(args.mp, 4)
     pm.set_args(args)
     check_default_settings(args, remaining, pm)
     if args.verbose > 2:
diff --git a/tools/testing/selftests/tc-testing/tdc.sh b/tools/testing/selftests/tc-testing/tdc.sh
index eb357bd7923c..4dbe50bde5a0 100755
--- a/tools/testing/selftests/tc-testing/tdc.sh
+++ b/tools/testing/selftests/tc-testing/tdc.sh
@@ -1,7 +1,68 @@
 #!/bin/sh
 # SPDX-License-Identifier: GPL-2.0
 
-modprobe netdevsim
-modprobe sch_teql
-./tdc.py -c actions --nobuildebpf
-./tdc.py -c qdisc
+# If a module is required and was not compiled
+# the test that requires it will fail anyways
+try_modprobe() {
+   modprobe -q -R "$1"
+   if [ $? -ne 0 ]; then
+      echo "Module $1 not found... skipping."
+   else
+      modprobe "$1"
+   fi
+}
+
+try_modprobe netdevsim
+try_modprobe act_bpf
+try_modprobe act_connmark
+try_modprobe act_csum
+try_modprobe act_ct
+try_modprobe act_ctinfo
+try_modprobe act_gact
+try_modprobe act_gate
+try_modprobe act_ipt
+try_modprobe act_mirred
+try_modprobe act_mpls
+try_modprobe act_nat
+try_modprobe act_pedit
+try_modprobe act_police
+try_modprobe act_sample
+try_modprobe act_simple
+try_modprobe act_skbedit
+try_modprobe act_skbmod
+try_modprobe act_tunnel_key
+try_modprobe act_vlan
+try_modprobe cls_basic
+try_modprobe cls_bpf
+try_modprobe cls_cgroup
+try_modprobe cls_flow
+try_modprobe cls_flower
+try_modprobe cls_fw
+try_modprobe cls_matchall
+try_modprobe cls_route
+try_modprobe cls_u32
+try_modprobe em_canid
+try_modprobe em_cmp
+try_modprobe em_ipset
+try_modprobe em_ipt
+try_modprobe em_meta
+try_modprobe em_nbyte
+try_modprobe em_text
+try_modprobe em_u32
+try_modprobe sch_cake
+try_modprobe sch_cbs
+try_modprobe sch_choke
+try_modprobe sch_codel
+try_modprobe sch_drr
+try_modprobe sch_etf
+try_modprobe sch_ets
+try_modprobe sch_fq
+try_modprobe sch_fq_codel
+try_modprobe sch_fq_pie
+try_modprobe sch_gred
+try_modprobe sch_hfsc
+try_modprobe sch_hhf
+try_modprobe sch_htb
+try_modprobe sch_teql
+./tdc.py -J`nproc` -c actions --nobuildebpf
+./tdc.py -J`nproc` -c qdisc
diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig
index 484d0873061c..2c964586aa14 100644
--- a/virt/kvm/Kconfig
+++ b/virt/kvm/Kconfig
@@ -92,3 +92,20 @@ config HAVE_KVM_PM_NOTIFIER
 
 config KVM_GENERIC_HARDWARE_ENABLING
        bool
+
+config KVM_GENERIC_MMU_NOTIFIER
+       select MMU_NOTIFIER
+       bool
+
+config KVM_GENERIC_MEMORY_ATTRIBUTES
+       select KVM_GENERIC_MMU_NOTIFIER
+       bool
+
+config KVM_PRIVATE_MEM
+       select XARRAY_MULTI
+       bool
+
+config KVM_GENERIC_PRIVATE_MEM
+       select KVM_GENERIC_MEMORY_ATTRIBUTES
+       select KVM_PRIVATE_MEM
+       bool
diff --git a/virt/kvm/Makefile.kvm b/virt/kvm/Makefile.kvm
index 2c27d5d0c367..724c89af78af 100644
--- a/virt/kvm/Makefile.kvm
+++ b/virt/kvm/Makefile.kvm
@@ -12,3 +12,4 @@ kvm-$(CONFIG_KVM_ASYNC_PF) += $(KVM)/async_pf.o
 kvm-$(CONFIG_HAVE_KVM_IRQ_ROUTING) += $(KVM)/irqchip.o
 kvm-$(CONFIG_HAVE_KVM_DIRTY_RING) += $(KVM)/dirty_ring.o
 kvm-$(CONFIG_HAVE_KVM_PFNCACHE) += $(KVM)/pfncache.o
+kvm-$(CONFIG_KVM_PRIVATE_MEM) += $(KVM)/guest_memfd.o
diff --git a/virt/kvm/dirty_ring.c b/virt/kvm/dirty_ring.c
index c1cd7dfe4a90..86d267db87bb 100644
--- a/virt/kvm/dirty_ring.c
+++ b/virt/kvm/dirty_ring.c
@@ -58,7 +58,7 @@ static void kvm_reset_dirty_gfn(struct kvm *kvm, u32 slot, u64 offset, u64 mask)
 	as_id = slot >> 16;
 	id = (u16)slot;
 
-	if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
+	if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS)
 		return;
 
 	memslot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c
new file mode 100644
index 000000000000..16d58806e913
--- /dev/null
+++ b/virt/kvm/guest_memfd.c
@@ -0,0 +1,535 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/backing-dev.h>
+#include <linux/falloc.h>
+#include <linux/kvm_host.h>
+#include <linux/pagemap.h>
+#include <linux/anon_inodes.h>
+
+#include "kvm_mm.h"
+
+struct kvm_gmem {
+	struct kvm *kvm;
+	struct xarray bindings;
+	struct list_head entry;
+};
+
+static struct folio *kvm_gmem_get_folio(struct inode *inode, pgoff_t index)
+{
+	struct folio *folio;
+
+	/* TODO: Support huge pages. */
+	folio = filemap_grab_folio(inode->i_mapping, index);
+	if (IS_ERR_OR_NULL(folio))
+		return NULL;
+
+	/*
+	 * Use the up-to-date flag to track whether or not the memory has been
+	 * zeroed before being handed off to the guest.  There is no backing
+	 * storage for the memory, so the folio will remain up-to-date until
+	 * it's removed.
+	 *
+	 * TODO: Skip clearing pages when trusted firmware will do it when
+	 * assigning memory to the guest.
+	 */
+	if (!folio_test_uptodate(folio)) {
+		unsigned long nr_pages = folio_nr_pages(folio);
+		unsigned long i;
+
+		for (i = 0; i < nr_pages; i++)
+			clear_highpage(folio_page(folio, i));
+
+		folio_mark_uptodate(folio);
+	}
+
+	/*
+	 * Ignore accessed, referenced, and dirty flags.  The memory is
+	 * unevictable and there is no storage to write back to.
+	 */
+	return folio;
+}
+
+static void kvm_gmem_invalidate_begin(struct kvm_gmem *gmem, pgoff_t start,
+				      pgoff_t end)
+{
+	bool flush = false, found_memslot = false;
+	struct kvm_memory_slot *slot;
+	struct kvm *kvm = gmem->kvm;
+	unsigned long index;
+
+	xa_for_each_range(&gmem->bindings, index, slot, start, end - 1) {
+		pgoff_t pgoff = slot->gmem.pgoff;
+
+		struct kvm_gfn_range gfn_range = {
+			.start = slot->base_gfn + max(pgoff, start) - pgoff,
+			.end = slot->base_gfn + min(pgoff + slot->npages, end) - pgoff,
+			.slot = slot,
+			.may_block = true,
+		};
+
+		if (!found_memslot) {
+			found_memslot = true;
+
+			KVM_MMU_LOCK(kvm);
+			kvm_mmu_invalidate_begin(kvm);
+		}
+
+		flush |= kvm_mmu_unmap_gfn_range(kvm, &gfn_range);
+	}
+
+	if (flush)
+		kvm_flush_remote_tlbs(kvm);
+
+	if (found_memslot)
+		KVM_MMU_UNLOCK(kvm);
+}
+
+static void kvm_gmem_invalidate_end(struct kvm_gmem *gmem, pgoff_t start,
+				    pgoff_t end)
+{
+	struct kvm *kvm = gmem->kvm;
+
+	if (xa_find(&gmem->bindings, &start, end - 1, XA_PRESENT)) {
+		KVM_MMU_LOCK(kvm);
+		kvm_mmu_invalidate_end(kvm);
+		KVM_MMU_UNLOCK(kvm);
+	}
+}
+
+static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len)
+{
+	struct list_head *gmem_list = &inode->i_mapping->i_private_list;
+	pgoff_t start = offset >> PAGE_SHIFT;
+	pgoff_t end = (offset + len) >> PAGE_SHIFT;
+	struct kvm_gmem *gmem;
+
+	/*
+	 * Bindings must be stable across invalidation to ensure the start+end
+	 * are balanced.
+	 */
+	filemap_invalidate_lock(inode->i_mapping);
+
+	list_for_each_entry(gmem, gmem_list, entry)
+		kvm_gmem_invalidate_begin(gmem, start, end);
+
+	truncate_inode_pages_range(inode->i_mapping, offset, offset + len - 1);
+
+	list_for_each_entry(gmem, gmem_list, entry)
+		kvm_gmem_invalidate_end(gmem, start, end);
+
+	filemap_invalidate_unlock(inode->i_mapping);
+
+	return 0;
+}
+
+static long kvm_gmem_allocate(struct inode *inode, loff_t offset, loff_t len)
+{
+	struct address_space *mapping = inode->i_mapping;
+	pgoff_t start, index, end;
+	int r;
+
+	/* Dedicated guest is immutable by default. */
+	if (offset + len > i_size_read(inode))
+		return -EINVAL;
+
+	filemap_invalidate_lock_shared(mapping);
+
+	start = offset >> PAGE_SHIFT;
+	end = (offset + len) >> PAGE_SHIFT;
+
+	r = 0;
+	for (index = start; index < end; ) {
+		struct folio *folio;
+
+		if (signal_pending(current)) {
+			r = -EINTR;
+			break;
+		}
+
+		folio = kvm_gmem_get_folio(inode, index);
+		if (!folio) {
+			r = -ENOMEM;
+			break;
+		}
+
+		index = folio_next_index(folio);
+
+		folio_unlock(folio);
+		folio_put(folio);
+
+		/* 64-bit only, wrapping the index should be impossible. */
+		if (WARN_ON_ONCE(!index))
+			break;
+
+		cond_resched();
+	}
+
+	filemap_invalidate_unlock_shared(mapping);
+
+	return r;
+}
+
+static long kvm_gmem_fallocate(struct file *file, int mode, loff_t offset,
+			       loff_t len)
+{
+	int ret;
+
+	if (!(mode & FALLOC_FL_KEEP_SIZE))
+		return -EOPNOTSUPP;
+
+	if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
+		return -EOPNOTSUPP;
+
+	if (!PAGE_ALIGNED(offset) || !PAGE_ALIGNED(len))
+		return -EINVAL;
+
+	if (mode & FALLOC_FL_PUNCH_HOLE)
+		ret = kvm_gmem_punch_hole(file_inode(file), offset, len);
+	else
+		ret = kvm_gmem_allocate(file_inode(file), offset, len);
+
+	if (!ret)
+		file_modified(file);
+	return ret;
+}
+
+static int kvm_gmem_release(struct inode *inode, struct file *file)
+{
+	struct kvm_gmem *gmem = file->private_data;
+	struct kvm_memory_slot *slot;
+	struct kvm *kvm = gmem->kvm;
+	unsigned long index;
+
+	/*
+	 * Prevent concurrent attempts to *unbind* a memslot.  This is the last
+	 * reference to the file and thus no new bindings can be created, but
+	 * dereferencing the slot for existing bindings needs to be protected
+	 * against memslot updates, specifically so that unbind doesn't race
+	 * and free the memslot (kvm_gmem_get_file() will return NULL).
+	 */
+	mutex_lock(&kvm->slots_lock);
+
+	filemap_invalidate_lock(inode->i_mapping);
+
+	xa_for_each(&gmem->bindings, index, slot)
+		rcu_assign_pointer(slot->gmem.file, NULL);
+
+	synchronize_rcu();
+
+	/*
+	 * All in-flight operations are gone and new bindings can be created.
+	 * Zap all SPTEs pointed at by this file.  Do not free the backing
+	 * memory, as its lifetime is associated with the inode, not the file.
+	 */
+	kvm_gmem_invalidate_begin(gmem, 0, -1ul);
+	kvm_gmem_invalidate_end(gmem, 0, -1ul);
+
+	list_del(&gmem->entry);
+
+	filemap_invalidate_unlock(inode->i_mapping);
+
+	mutex_unlock(&kvm->slots_lock);
+
+	xa_destroy(&gmem->bindings);
+	kfree(gmem);
+
+	kvm_put_kvm(kvm);
+
+	return 0;
+}
+
+static inline struct file *kvm_gmem_get_file(struct kvm_memory_slot *slot)
+{
+	/*
+	 * Do not return slot->gmem.file if it has already been closed;
+	 * there might be some time between the last fput() and when
+	 * kvm_gmem_release() clears slot->gmem.file, and you do not
+	 * want to spin in the meanwhile.
+	 */
+	return get_file_active(&slot->gmem.file);
+}
+
+static struct file_operations kvm_gmem_fops = {
+	.open		= generic_file_open,
+	.release	= kvm_gmem_release,
+	.fallocate	= kvm_gmem_fallocate,
+};
+
+void kvm_gmem_init(struct module *module)
+{
+	kvm_gmem_fops.owner = module;
+}
+
+static int kvm_gmem_migrate_folio(struct address_space *mapping,
+				  struct folio *dst, struct folio *src,
+				  enum migrate_mode mode)
+{
+	WARN_ON_ONCE(1);
+	return -EINVAL;
+}
+
+static int kvm_gmem_error_folio(struct address_space *mapping,
+		struct folio *folio)
+{
+	struct list_head *gmem_list = &mapping->i_private_list;
+	struct kvm_gmem *gmem;
+	pgoff_t start, end;
+
+	filemap_invalidate_lock_shared(mapping);
+
+	start = folio->index;
+	end = start + folio_nr_pages(folio);
+
+	list_for_each_entry(gmem, gmem_list, entry)
+		kvm_gmem_invalidate_begin(gmem, start, end);
+
+	/*
+	 * Do not truncate the range, what action is taken in response to the
+	 * error is userspace's decision (assuming the architecture supports
+	 * gracefully handling memory errors).  If/when the guest attempts to
+	 * access a poisoned page, kvm_gmem_get_pfn() will return -EHWPOISON,
+	 * at which point KVM can either terminate the VM or propagate the
+	 * error to userspace.
+	 */
+
+	list_for_each_entry(gmem, gmem_list, entry)
+		kvm_gmem_invalidate_end(gmem, start, end);
+
+	filemap_invalidate_unlock_shared(mapping);
+
+	return MF_DELAYED;
+}
+
+static const struct address_space_operations kvm_gmem_aops = {
+	.dirty_folio = noop_dirty_folio,
+#ifdef CONFIG_MIGRATION
+	.migrate_folio	= kvm_gmem_migrate_folio,
+#endif
+	.error_remove_folio = kvm_gmem_error_folio,
+};
+
+static int kvm_gmem_getattr(struct mnt_idmap *idmap, const struct path *path,
+			    struct kstat *stat, u32 request_mask,
+			    unsigned int query_flags)
+{
+	struct inode *inode = path->dentry->d_inode;
+
+	generic_fillattr(idmap, request_mask, inode, stat);
+	return 0;
+}
+
+static int kvm_gmem_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
+			    struct iattr *attr)
+{
+	return -EINVAL;
+}
+static const struct inode_operations kvm_gmem_iops = {
+	.getattr	= kvm_gmem_getattr,
+	.setattr	= kvm_gmem_setattr,
+};
+
+static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags)
+{
+	const char *anon_name = "[kvm-gmem]";
+	struct kvm_gmem *gmem;
+	struct inode *inode;
+	struct file *file;
+	int fd, err;
+
+	fd = get_unused_fd_flags(0);
+	if (fd < 0)
+		return fd;
+
+	gmem = kzalloc(sizeof(*gmem), GFP_KERNEL);
+	if (!gmem) {
+		err = -ENOMEM;
+		goto err_fd;
+	}
+
+	file = anon_inode_create_getfile(anon_name, &kvm_gmem_fops, gmem,
+					 O_RDWR, NULL);
+	if (IS_ERR(file)) {
+		err = PTR_ERR(file);
+		goto err_gmem;
+	}
+
+	file->f_flags |= O_LARGEFILE;
+
+	inode = file->f_inode;
+	WARN_ON(file->f_mapping != inode->i_mapping);
+
+	inode->i_private = (void *)(unsigned long)flags;
+	inode->i_op = &kvm_gmem_iops;
+	inode->i_mapping->a_ops = &kvm_gmem_aops;
+	inode->i_mode |= S_IFREG;
+	inode->i_size = size;
+	mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER);
+	mapping_set_unmovable(inode->i_mapping);
+	/* Unmovable mappings are supposed to be marked unevictable as well. */
+	WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping));
+
+	kvm_get_kvm(kvm);
+	gmem->kvm = kvm;
+	xa_init(&gmem->bindings);
+	list_add(&gmem->entry, &inode->i_mapping->i_private_list);
+
+	fd_install(fd, file);
+	return fd;
+
+err_gmem:
+	kfree(gmem);
+err_fd:
+	put_unused_fd(fd);
+	return err;
+}
+
+int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args)
+{
+	loff_t size = args->size;
+	u64 flags = args->flags;
+	u64 valid_flags = 0;
+
+	if (flags & ~valid_flags)
+		return -EINVAL;
+
+	if (size <= 0 || !PAGE_ALIGNED(size))
+		return -EINVAL;
+
+	return __kvm_gmem_create(kvm, size, flags);
+}
+
+int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot,
+		  unsigned int fd, loff_t offset)
+{
+	loff_t size = slot->npages << PAGE_SHIFT;
+	unsigned long start, end;
+	struct kvm_gmem *gmem;
+	struct inode *inode;
+	struct file *file;
+	int r = -EINVAL;
+
+	BUILD_BUG_ON(sizeof(gfn_t) != sizeof(slot->gmem.pgoff));
+
+	file = fget(fd);
+	if (!file)
+		return -EBADF;
+
+	if (file->f_op != &kvm_gmem_fops)
+		goto err;
+
+	gmem = file->private_data;
+	if (gmem->kvm != kvm)
+		goto err;
+
+	inode = file_inode(file);
+
+	if (offset < 0 || !PAGE_ALIGNED(offset) ||
+	    offset + size > i_size_read(inode))
+		goto err;
+
+	filemap_invalidate_lock(inode->i_mapping);
+
+	start = offset >> PAGE_SHIFT;
+	end = start + slot->npages;
+
+	if (!xa_empty(&gmem->bindings) &&
+	    xa_find(&gmem->bindings, &start, end - 1, XA_PRESENT)) {
+		filemap_invalidate_unlock(inode->i_mapping);
+		goto err;
+	}
+
+	/*
+	 * No synchronize_rcu() needed, any in-flight readers are guaranteed to
+	 * be see either a NULL file or this new file, no need for them to go
+	 * away.
+	 */
+	rcu_assign_pointer(slot->gmem.file, file);
+	slot->gmem.pgoff = start;
+
+	xa_store_range(&gmem->bindings, start, end - 1, slot, GFP_KERNEL);
+	filemap_invalidate_unlock(inode->i_mapping);
+
+	/*
+	 * Drop the reference to the file, even on success.  The file pins KVM,
+	 * not the other way 'round.  Active bindings are invalidated if the
+	 * file is closed before memslots are destroyed.
+	 */
+	r = 0;
+err:
+	fput(file);
+	return r;
+}
+
+void kvm_gmem_unbind(struct kvm_memory_slot *slot)
+{
+	unsigned long start = slot->gmem.pgoff;
+	unsigned long end = start + slot->npages;
+	struct kvm_gmem *gmem;
+	struct file *file;
+
+	/*
+	 * Nothing to do if the underlying file was already closed (or is being
+	 * closed right now), kvm_gmem_release() invalidates all bindings.
+	 */
+	file = kvm_gmem_get_file(slot);
+	if (!file)
+		return;
+
+	gmem = file->private_data;
+
+	filemap_invalidate_lock(file->f_mapping);
+	xa_store_range(&gmem->bindings, start, end - 1, NULL, GFP_KERNEL);
+	rcu_assign_pointer(slot->gmem.file, NULL);
+	synchronize_rcu();
+	filemap_invalidate_unlock(file->f_mapping);
+
+	fput(file);
+}
+
+int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot,
+		     gfn_t gfn, kvm_pfn_t *pfn, int *max_order)
+{
+	pgoff_t index = gfn - slot->base_gfn + slot->gmem.pgoff;
+	struct kvm_gmem *gmem;
+	struct folio *folio;
+	struct page *page;
+	struct file *file;
+	int r;
+
+	file = kvm_gmem_get_file(slot);
+	if (!file)
+		return -EFAULT;
+
+	gmem = file->private_data;
+
+	if (WARN_ON_ONCE(xa_load(&gmem->bindings, index) != slot)) {
+		r = -EIO;
+		goto out_fput;
+	}
+
+	folio = kvm_gmem_get_folio(file_inode(file), index);
+	if (!folio) {
+		r = -ENOMEM;
+		goto out_fput;
+	}
+
+	if (folio_test_hwpoison(folio)) {
+		r = -EHWPOISON;
+		goto out_unlock;
+	}
+
+	page = folio_file_page(folio, index);
+
+	*pfn = page_to_pfn(page);
+	if (max_order)
+		*max_order = 0;
+
+	r = 0;
+
+out_unlock:
+	folio_unlock(folio);
+out_fput:
+	fput(file);
+
+	return r;
+}
+EXPORT_SYMBOL_GPL(kvm_gmem_get_pfn);
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 486800a7024b..8758cb799e18 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -535,30 +535,43 @@ void kvm_destroy_vcpus(struct kvm *kvm)
 }
 EXPORT_SYMBOL_GPL(kvm_destroy_vcpus);
 
-#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
+#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER
 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
 {
 	return container_of(mn, struct kvm, mmu_notifier);
 }
 
-typedef bool (*hva_handler_t)(struct kvm *kvm, struct kvm_gfn_range *range);
+typedef bool (*gfn_handler_t)(struct kvm *kvm, struct kvm_gfn_range *range);
 
-typedef void (*on_lock_fn_t)(struct kvm *kvm, unsigned long start,
-			     unsigned long end);
+typedef void (*on_lock_fn_t)(struct kvm *kvm);
 
-typedef void (*on_unlock_fn_t)(struct kvm *kvm);
-
-struct kvm_hva_range {
-	unsigned long start;
-	unsigned long end;
+struct kvm_mmu_notifier_range {
+	/*
+	 * 64-bit addresses, as KVM notifiers can operate on host virtual
+	 * addresses (unsigned long) and guest physical addresses (64-bit).
+	 */
+	u64 start;
+	u64 end;
 	union kvm_mmu_notifier_arg arg;
-	hva_handler_t handler;
+	gfn_handler_t handler;
 	on_lock_fn_t on_lock;
-	on_unlock_fn_t on_unlock;
 	bool flush_on_ret;
 	bool may_block;
 };
 
+/*
+ * The inner-most helper returns a tuple containing the return value from the
+ * arch- and action-specific handler, plus a flag indicating whether or not at
+ * least one memslot was found, i.e. if the handler found guest memory.
+ *
+ * Note, most notifiers are averse to booleans, so even though KVM tracks the
+ * return from arch code as a bool, outer helpers will cast it to an int. :-(
+ */
+typedef struct kvm_mmu_notifier_return {
+	bool ret;
+	bool found_memslot;
+} kvm_mn_ret_t;
+
 /*
  * Use a dedicated stub instead of NULL to indicate that there is no callback
  * function/handler.  The compiler technically can't guarantee that a real
@@ -580,26 +593,29 @@ static const union kvm_mmu_notifier_arg KVM_MMU_NOTIFIER_NO_ARG;
 	     node;							     \
 	     node = interval_tree_iter_next(node, start, last))	     \
 
-static __always_inline int __kvm_handle_hva_range(struct kvm *kvm,
-						  const struct kvm_hva_range *range)
+static __always_inline kvm_mn_ret_t __kvm_handle_hva_range(struct kvm *kvm,
+							   const struct kvm_mmu_notifier_range *range)
 {
-	bool ret = false, locked = false;
+	struct kvm_mmu_notifier_return r = {
+		.ret = false,
+		.found_memslot = false,
+	};
 	struct kvm_gfn_range gfn_range;
 	struct kvm_memory_slot *slot;
 	struct kvm_memslots *slots;
 	int i, idx;
 
 	if (WARN_ON_ONCE(range->end <= range->start))
-		return 0;
+		return r;
 
 	/* A null handler is allowed if and only if on_lock() is provided. */
 	if (WARN_ON_ONCE(IS_KVM_NULL_FN(range->on_lock) &&
 			 IS_KVM_NULL_FN(range->handler)))
-		return 0;
+		return r;
 
 	idx = srcu_read_lock(&kvm->srcu);
 
-	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
+	for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
 		struct interval_tree_node *node;
 
 		slots = __kvm_memslots(kvm, i);
@@ -608,9 +624,9 @@ static __always_inline int __kvm_handle_hva_range(struct kvm *kvm,
 			unsigned long hva_start, hva_end;
 
 			slot = container_of(node, struct kvm_memory_slot, hva_node[slots->node_idx]);
-			hva_start = max(range->start, slot->userspace_addr);
-			hva_end = min(range->end, slot->userspace_addr +
-						  (slot->npages << PAGE_SHIFT));
+			hva_start = max_t(unsigned long, range->start, slot->userspace_addr);
+			hva_end = min_t(unsigned long, range->end,
+					slot->userspace_addr + (slot->npages << PAGE_SHIFT));
 
 			/*
 			 * To optimize for the likely case where the address
@@ -629,71 +645,66 @@ static __always_inline int __kvm_handle_hva_range(struct kvm *kvm,
 			gfn_range.end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, slot);
 			gfn_range.slot = slot;
 
-			if (!locked) {
-				locked = true;
+			if (!r.found_memslot) {
+				r.found_memslot = true;
 				KVM_MMU_LOCK(kvm);
 				if (!IS_KVM_NULL_FN(range->on_lock))
-					range->on_lock(kvm, range->start, range->end);
+					range->on_lock(kvm);
+
 				if (IS_KVM_NULL_FN(range->handler))
 					break;
 			}
-			ret |= range->handler(kvm, &gfn_range);
+			r.ret |= range->handler(kvm, &gfn_range);
 		}
 	}
 
-	if (range->flush_on_ret && ret)
+	if (range->flush_on_ret && r.ret)
 		kvm_flush_remote_tlbs(kvm);
 
-	if (locked) {
+	if (r.found_memslot)
 		KVM_MMU_UNLOCK(kvm);
-		if (!IS_KVM_NULL_FN(range->on_unlock))
-			range->on_unlock(kvm);
-	}
 
 	srcu_read_unlock(&kvm->srcu, idx);
 
-	/* The notifiers are averse to booleans. :-( */
-	return (int)ret;
+	return r;
 }
 
 static __always_inline int kvm_handle_hva_range(struct mmu_notifier *mn,
 						unsigned long start,
 						unsigned long end,
 						union kvm_mmu_notifier_arg arg,
-						hva_handler_t handler)
+						gfn_handler_t handler)
 {
 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
-	const struct kvm_hva_range range = {
+	const struct kvm_mmu_notifier_range range = {
 		.start		= start,
 		.end		= end,
 		.arg		= arg,
 		.handler	= handler,
 		.on_lock	= (void *)kvm_null_fn,
-		.on_unlock	= (void *)kvm_null_fn,
 		.flush_on_ret	= true,
 		.may_block	= false,
 	};
 
-	return __kvm_handle_hva_range(kvm, &range);
+	return __kvm_handle_hva_range(kvm, &range).ret;
 }
 
 static __always_inline int kvm_handle_hva_range_no_flush(struct mmu_notifier *mn,
 							 unsigned long start,
 							 unsigned long end,
-							 hva_handler_t handler)
+							 gfn_handler_t handler)
 {
 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
-	const struct kvm_hva_range range = {
+	const struct kvm_mmu_notifier_range range = {
 		.start		= start,
 		.end		= end,
 		.handler	= handler,
 		.on_lock	= (void *)kvm_null_fn,
-		.on_unlock	= (void *)kvm_null_fn,
 		.flush_on_ret	= false,
 		.may_block	= false,
 	};
 
-	return __kvm_handle_hva_range(kvm, &range);
+	return __kvm_handle_hva_range(kvm, &range).ret;
 }
 
 static bool kvm_change_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
@@ -738,16 +749,29 @@ static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
 	kvm_handle_hva_range(mn, address, address + 1, arg, kvm_change_spte_gfn);
 }
 
-void kvm_mmu_invalidate_begin(struct kvm *kvm, unsigned long start,
-			      unsigned long end)
+void kvm_mmu_invalidate_begin(struct kvm *kvm)
 {
+	lockdep_assert_held_write(&kvm->mmu_lock);
 	/*
 	 * The count increase must become visible at unlock time as no
 	 * spte can be established without taking the mmu_lock and
 	 * count is also read inside the mmu_lock critical section.
 	 */
 	kvm->mmu_invalidate_in_progress++;
+
 	if (likely(kvm->mmu_invalidate_in_progress == 1)) {
+		kvm->mmu_invalidate_range_start = INVALID_GPA;
+		kvm->mmu_invalidate_range_end = INVALID_GPA;
+	}
+}
+
+void kvm_mmu_invalidate_range_add(struct kvm *kvm, gfn_t start, gfn_t end)
+{
+	lockdep_assert_held_write(&kvm->mmu_lock);
+
+	WARN_ON_ONCE(!kvm->mmu_invalidate_in_progress);
+
+	if (likely(kvm->mmu_invalidate_range_start == INVALID_GPA)) {
 		kvm->mmu_invalidate_range_start = start;
 		kvm->mmu_invalidate_range_end = end;
 	} else {
@@ -767,16 +791,21 @@ void kvm_mmu_invalidate_begin(struct kvm *kvm, unsigned long start,
 	}
 }
 
+bool kvm_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
+{
+	kvm_mmu_invalidate_range_add(kvm, range->start, range->end);
+	return kvm_unmap_gfn_range(kvm, range);
+}
+
 static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
 					const struct mmu_notifier_range *range)
 {
 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
-	const struct kvm_hva_range hva_range = {
+	const struct kvm_mmu_notifier_range hva_range = {
 		.start		= range->start,
 		.end		= range->end,
-		.handler	= kvm_unmap_gfn_range,
+		.handler	= kvm_mmu_unmap_gfn_range,
 		.on_lock	= kvm_mmu_invalidate_begin,
-		.on_unlock	= kvm_arch_guest_memory_reclaimed,
 		.flush_on_ret	= true,
 		.may_block	= mmu_notifier_range_blockable(range),
 	};
@@ -808,14 +837,21 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
 	gfn_to_pfn_cache_invalidate_start(kvm, range->start, range->end,
 					  hva_range.may_block);
 
-	__kvm_handle_hva_range(kvm, &hva_range);
+	/*
+	 * If one or more memslots were found and thus zapped, notify arch code
+	 * that guest memory has been reclaimed.  This needs to be done *after*
+	 * dropping mmu_lock, as x86's reclaim path is slooooow.
+	 */
+	if (__kvm_handle_hva_range(kvm, &hva_range).found_memslot)
+		kvm_arch_guest_memory_reclaimed(kvm);
 
 	return 0;
 }
 
-void kvm_mmu_invalidate_end(struct kvm *kvm, unsigned long start,
-			    unsigned long end)
+void kvm_mmu_invalidate_end(struct kvm *kvm)
 {
+	lockdep_assert_held_write(&kvm->mmu_lock);
+
 	/*
 	 * This sequence increase will notify the kvm page fault that
 	 * the page that is going to be mapped in the spte could have
@@ -829,18 +865,24 @@ void kvm_mmu_invalidate_end(struct kvm *kvm, unsigned long start,
 	 * in conjunction with the smp_rmb in mmu_invalidate_retry().
 	 */
 	kvm->mmu_invalidate_in_progress--;
+	KVM_BUG_ON(kvm->mmu_invalidate_in_progress < 0, kvm);
+
+	/*
+	 * Assert that at least one range was added between start() and end().
+	 * Not adding a range isn't fatal, but it is a KVM bug.
+	 */
+	WARN_ON_ONCE(kvm->mmu_invalidate_range_start == INVALID_GPA);
 }
 
 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
 					const struct mmu_notifier_range *range)
 {
 	struct kvm *kvm = mmu_notifier_to_kvm(mn);
-	const struct kvm_hva_range hva_range = {
+	const struct kvm_mmu_notifier_range hva_range = {
 		.start		= range->start,
 		.end		= range->end,
 		.handler	= (void *)kvm_null_fn,
 		.on_lock	= kvm_mmu_invalidate_end,
-		.on_unlock	= (void *)kvm_null_fn,
 		.flush_on_ret	= false,
 		.may_block	= mmu_notifier_range_blockable(range),
 	};
@@ -859,8 +901,6 @@ static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
 	 */
 	if (wake)
 		rcuwait_wake_up(&kvm->mn_memslots_update_rcuwait);
-
-	BUG_ON(kvm->mmu_invalidate_in_progress < 0);
 }
 
 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
@@ -934,14 +974,14 @@ static int kvm_init_mmu_notifier(struct kvm *kvm)
 	return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
 }
 
-#else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
+#else  /* !CONFIG_KVM_GENERIC_MMU_NOTIFIER */
 
 static int kvm_init_mmu_notifier(struct kvm *kvm)
 {
 	return 0;
 }
 
-#endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
+#endif /* CONFIG_KVM_GENERIC_MMU_NOTIFIER */
 
 #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER
 static int kvm_pm_notifier_call(struct notifier_block *bl,
@@ -987,6 +1027,9 @@ static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
 /* This does not remove the slot from struct kvm_memslots data structures */
 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
 {
+	if (slot->flags & KVM_MEM_GUEST_MEMFD)
+		kvm_gmem_unbind(slot);
+
 	kvm_destroy_dirty_bitmap(slot);
 
 	kvm_arch_free_memslot(kvm, slot);
@@ -1171,6 +1214,9 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname)
 	spin_lock_init(&kvm->mn_invalidate_lock);
 	rcuwait_init(&kvm->mn_memslots_update_rcuwait);
 	xa_init(&kvm->vcpu_array);
+#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
+	xa_init(&kvm->mem_attr_array);
+#endif
 
 	INIT_LIST_HEAD(&kvm->gpc_list);
 	spin_lock_init(&kvm->gpc_lock);
@@ -1195,7 +1241,7 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname)
 		goto out_err_no_irq_srcu;
 
 	refcount_set(&kvm->users_count, 1);
-	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
+	for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
 		for (j = 0; j < 2; j++) {
 			slots = &kvm->__memslots[i][j];
 
@@ -1261,7 +1307,7 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname)
 out_err_no_debugfs:
 	kvm_coalesced_mmio_free(kvm);
 out_no_coalesced_mmio:
-#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
+#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER
 	if (kvm->mmu_notifier.ops)
 		mmu_notifier_unregister(&kvm->mmu_notifier, current->mm);
 #endif
@@ -1321,7 +1367,7 @@ static void kvm_destroy_vm(struct kvm *kvm)
 		kvm->buses[i] = NULL;
 	}
 	kvm_coalesced_mmio_free(kvm);
-#if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
+#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER
 	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
 	/*
 	 * At this point, pending calls to invalidate_range_start()
@@ -1330,20 +1376,30 @@ static void kvm_destroy_vm(struct kvm *kvm)
 	 * No threads can be waiting in kvm_swap_active_memslots() as the
 	 * last reference on KVM has been dropped, but freeing
 	 * memslots would deadlock without this manual intervention.
+	 *
+	 * If the count isn't unbalanced, i.e. KVM did NOT unregister its MMU
+	 * notifier between a start() and end(), then there shouldn't be any
+	 * in-progress invalidations.
 	 */
 	WARN_ON(rcuwait_active(&kvm->mn_memslots_update_rcuwait));
-	kvm->mn_active_invalidate_count = 0;
+	if (kvm->mn_active_invalidate_count)
+		kvm->mn_active_invalidate_count = 0;
+	else
+		WARN_ON(kvm->mmu_invalidate_in_progress);
 #else
 	kvm_flush_shadow_all(kvm);
 #endif
 	kvm_arch_destroy_vm(kvm);
 	kvm_destroy_devices(kvm);
-	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
+	for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
 		kvm_free_memslots(kvm, &kvm->__memslots[i][0]);
 		kvm_free_memslots(kvm, &kvm->__memslots[i][1]);
 	}
 	cleanup_srcu_struct(&kvm->irq_srcu);
 	cleanup_srcu_struct(&kvm->srcu);
+#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
+	xa_destroy(&kvm->mem_attr_array);
+#endif
 	kvm_arch_free_vm(kvm);
 	preempt_notifier_dec();
 	hardware_disable_all();
@@ -1545,10 +1601,26 @@ static void kvm_replace_memslot(struct kvm *kvm,
 	}
 }
 
-static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem)
+/*
+ * Flags that do not access any of the extra space of struct
+ * kvm_userspace_memory_region2.  KVM_SET_USER_MEMORY_REGION_V1_FLAGS
+ * only allows these.
+ */
+#define KVM_SET_USER_MEMORY_REGION_V1_FLAGS \
+	(KVM_MEM_LOG_DIRTY_PAGES | KVM_MEM_READONLY)
+
+static int check_memory_region_flags(struct kvm *kvm,
+				     const struct kvm_userspace_memory_region2 *mem)
 {
 	u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
 
+	if (kvm_arch_has_private_mem(kvm))
+		valid_flags |= KVM_MEM_GUEST_MEMFD;
+
+	/* Dirty logging private memory is not currently supported. */
+	if (mem->flags & KVM_MEM_GUEST_MEMFD)
+		valid_flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
+
 #ifdef __KVM_HAVE_READONLY_MEM
 	valid_flags |= KVM_MEM_READONLY;
 #endif
@@ -1610,7 +1682,7 @@ static void kvm_swap_active_memslots(struct kvm *kvm, int as_id)
 	 * space 0 will use generations 0, 2, 4, ... while address space 1 will
 	 * use generations 1, 3, 5, ...
 	 */
-	gen += KVM_ADDRESS_SPACE_NUM;
+	gen += kvm_arch_nr_memslot_as_ids(kvm);
 
 	kvm_arch_memslots_updated(kvm, gen);
 
@@ -1947,7 +2019,7 @@ static bool kvm_check_memslot_overlap(struct kvm_memslots *slots, int id,
  * Must be called holding kvm->slots_lock for write.
  */
 int __kvm_set_memory_region(struct kvm *kvm,
-			    const struct kvm_userspace_memory_region *mem)
+			    const struct kvm_userspace_memory_region2 *mem)
 {
 	struct kvm_memory_slot *old, *new;
 	struct kvm_memslots *slots;
@@ -1957,7 +2029,7 @@ int __kvm_set_memory_region(struct kvm *kvm,
 	int as_id, id;
 	int r;
 
-	r = check_memory_region_flags(mem);
+	r = check_memory_region_flags(kvm, mem);
 	if (r)
 		return r;
 
@@ -1976,7 +2048,11 @@ int __kvm_set_memory_region(struct kvm *kvm,
 	     !access_ok((void __user *)(unsigned long)mem->userspace_addr,
 			mem->memory_size))
 		return -EINVAL;
-	if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM)
+	if (mem->flags & KVM_MEM_GUEST_MEMFD &&
+	    (mem->guest_memfd_offset & (PAGE_SIZE - 1) ||
+	     mem->guest_memfd_offset + mem->memory_size < mem->guest_memfd_offset))
+		return -EINVAL;
+	if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_MEM_SLOTS_NUM)
 		return -EINVAL;
 	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
 		return -EINVAL;
@@ -2014,6 +2090,9 @@ int __kvm_set_memory_region(struct kvm *kvm,
 		if ((kvm->nr_memslot_pages + npages) < kvm->nr_memslot_pages)
 			return -EINVAL;
 	} else { /* Modify an existing slot. */
+		/* Private memslots are immutable, they can only be deleted. */
+		if (mem->flags & KVM_MEM_GUEST_MEMFD)
+			return -EINVAL;
 		if ((mem->userspace_addr != old->userspace_addr) ||
 		    (npages != old->npages) ||
 		    ((mem->flags ^ old->flags) & KVM_MEM_READONLY))
@@ -2042,16 +2121,29 @@ int __kvm_set_memory_region(struct kvm *kvm,
 	new->npages = npages;
 	new->flags = mem->flags;
 	new->userspace_addr = mem->userspace_addr;
+	if (mem->flags & KVM_MEM_GUEST_MEMFD) {
+		r = kvm_gmem_bind(kvm, new, mem->guest_memfd, mem->guest_memfd_offset);
+		if (r)
+			goto out;
+	}
 
 	r = kvm_set_memslot(kvm, old, new, change);
 	if (r)
-		kfree(new);
+		goto out_unbind;
+
+	return 0;
+
+out_unbind:
+	if (mem->flags & KVM_MEM_GUEST_MEMFD)
+		kvm_gmem_unbind(new);
+out:
+	kfree(new);
 	return r;
 }
 EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
 
 int kvm_set_memory_region(struct kvm *kvm,
-			  const struct kvm_userspace_memory_region *mem)
+			  const struct kvm_userspace_memory_region2 *mem)
 {
 	int r;
 
@@ -2063,7 +2155,7 @@ int kvm_set_memory_region(struct kvm *kvm,
 EXPORT_SYMBOL_GPL(kvm_set_memory_region);
 
 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
-					  struct kvm_userspace_memory_region *mem)
+					  struct kvm_userspace_memory_region2 *mem)
 {
 	if ((u16)mem->slot >= KVM_USER_MEM_SLOTS)
 		return -EINVAL;
@@ -2096,7 +2188,7 @@ int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log,
 
 	as_id = log->slot >> 16;
 	id = (u16)log->slot;
-	if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
+	if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS)
 		return -EINVAL;
 
 	slots = __kvm_memslots(kvm, as_id);
@@ -2158,7 +2250,7 @@ static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log)
 
 	as_id = log->slot >> 16;
 	id = (u16)log->slot;
-	if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
+	if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS)
 		return -EINVAL;
 
 	slots = __kvm_memslots(kvm, as_id);
@@ -2270,7 +2362,7 @@ static int kvm_clear_dirty_log_protect(struct kvm *kvm,
 
 	as_id = log->slot >> 16;
 	id = (u16)log->slot;
-	if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
+	if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS)
 		return -EINVAL;
 
 	if (log->first_page & 63)
@@ -2342,6 +2434,200 @@ static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm,
 }
 #endif /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */
 
+#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
+/*
+ * Returns true if _all_ gfns in the range [@start, @end) have attributes
+ * matching @attrs.
+ */
+bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
+				     unsigned long attrs)
+{
+	XA_STATE(xas, &kvm->mem_attr_array, start);
+	unsigned long index;
+	bool has_attrs;
+	void *entry;
+
+	rcu_read_lock();
+
+	if (!attrs) {
+		has_attrs = !xas_find(&xas, end - 1);
+		goto out;
+	}
+
+	has_attrs = true;
+	for (index = start; index < end; index++) {
+		do {
+			entry = xas_next(&xas);
+		} while (xas_retry(&xas, entry));
+
+		if (xas.xa_index != index || xa_to_value(entry) != attrs) {
+			has_attrs = false;
+			break;
+		}
+	}
+
+out:
+	rcu_read_unlock();
+	return has_attrs;
+}
+
+static u64 kvm_supported_mem_attributes(struct kvm *kvm)
+{
+	if (!kvm || kvm_arch_has_private_mem(kvm))
+		return KVM_MEMORY_ATTRIBUTE_PRIVATE;
+
+	return 0;
+}
+
+static __always_inline void kvm_handle_gfn_range(struct kvm *kvm,
+						 struct kvm_mmu_notifier_range *range)
+{
+	struct kvm_gfn_range gfn_range;
+	struct kvm_memory_slot *slot;
+	struct kvm_memslots *slots;
+	struct kvm_memslot_iter iter;
+	bool found_memslot = false;
+	bool ret = false;
+	int i;
+
+	gfn_range.arg = range->arg;
+	gfn_range.may_block = range->may_block;
+
+	for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
+		slots = __kvm_memslots(kvm, i);
+
+		kvm_for_each_memslot_in_gfn_range(&iter, slots, range->start, range->end) {
+			slot = iter.slot;
+			gfn_range.slot = slot;
+
+			gfn_range.start = max(range->start, slot->base_gfn);
+			gfn_range.end = min(range->end, slot->base_gfn + slot->npages);
+			if (gfn_range.start >= gfn_range.end)
+				continue;
+
+			if (!found_memslot) {
+				found_memslot = true;
+				KVM_MMU_LOCK(kvm);
+				if (!IS_KVM_NULL_FN(range->on_lock))
+					range->on_lock(kvm);
+			}
+
+			ret |= range->handler(kvm, &gfn_range);
+		}
+	}
+
+	if (range->flush_on_ret && ret)
+		kvm_flush_remote_tlbs(kvm);
+
+	if (found_memslot)
+		KVM_MMU_UNLOCK(kvm);
+}
+
+static bool kvm_pre_set_memory_attributes(struct kvm *kvm,
+					  struct kvm_gfn_range *range)
+{
+	/*
+	 * Unconditionally add the range to the invalidation set, regardless of
+	 * whether or not the arch callback actually needs to zap SPTEs.  E.g.
+	 * if KVM supports RWX attributes in the future and the attributes are
+	 * going from R=>RW, zapping isn't strictly necessary.  Unconditionally
+	 * adding the range allows KVM to require that MMU invalidations add at
+	 * least one range between begin() and end(), e.g. allows KVM to detect
+	 * bugs where the add() is missed.  Relaxing the rule *might* be safe,
+	 * but it's not obvious that allowing new mappings while the attributes
+	 * are in flux is desirable or worth the complexity.
+	 */
+	kvm_mmu_invalidate_range_add(kvm, range->start, range->end);
+
+	return kvm_arch_pre_set_memory_attributes(kvm, range);
+}
+
+/* Set @attributes for the gfn range [@start, @end). */
+static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
+				     unsigned long attributes)
+{
+	struct kvm_mmu_notifier_range pre_set_range = {
+		.start = start,
+		.end = end,
+		.handler = kvm_pre_set_memory_attributes,
+		.on_lock = kvm_mmu_invalidate_begin,
+		.flush_on_ret = true,
+		.may_block = true,
+	};
+	struct kvm_mmu_notifier_range post_set_range = {
+		.start = start,
+		.end = end,
+		.arg.attributes = attributes,
+		.handler = kvm_arch_post_set_memory_attributes,
+		.on_lock = kvm_mmu_invalidate_end,
+		.may_block = true,
+	};
+	unsigned long i;
+	void *entry;
+	int r = 0;
+
+	entry = attributes ? xa_mk_value(attributes) : NULL;
+
+	mutex_lock(&kvm->slots_lock);
+
+	/* Nothing to do if the entire range as the desired attributes. */
+	if (kvm_range_has_memory_attributes(kvm, start, end, attributes))
+		goto out_unlock;
+
+	/*
+	 * Reserve memory ahead of time to avoid having to deal with failures
+	 * partway through setting the new attributes.
+	 */
+	for (i = start; i < end; i++) {
+		r = xa_reserve(&kvm->mem_attr_array, i, GFP_KERNEL_ACCOUNT);
+		if (r)
+			goto out_unlock;
+	}
+
+	kvm_handle_gfn_range(kvm, &pre_set_range);
+
+	for (i = start; i < end; i++) {
+		r = xa_err(xa_store(&kvm->mem_attr_array, i, entry,
+				    GFP_KERNEL_ACCOUNT));
+		KVM_BUG_ON(r, kvm);
+	}
+
+	kvm_handle_gfn_range(kvm, &post_set_range);
+
+out_unlock:
+	mutex_unlock(&kvm->slots_lock);
+
+	return r;
+}
+static int kvm_vm_ioctl_set_mem_attributes(struct kvm *kvm,
+					   struct kvm_memory_attributes *attrs)
+{
+	gfn_t start, end;
+
+	/* flags is currently not used. */
+	if (attrs->flags)
+		return -EINVAL;
+	if (attrs->attributes & ~kvm_supported_mem_attributes(kvm))
+		return -EINVAL;
+	if (attrs->size == 0 || attrs->address + attrs->size < attrs->address)
+		return -EINVAL;
+	if (!PAGE_ALIGNED(attrs->address) || !PAGE_ALIGNED(attrs->size))
+		return -EINVAL;
+
+	start = attrs->address >> PAGE_SHIFT;
+	end = (attrs->address + attrs->size) >> PAGE_SHIFT;
+
+	/*
+	 * xarray tracks data using "unsigned long", and as a result so does
+	 * KVM.  For simplicity, supports generic attributes only on 64-bit
+	 * architectures.
+	 */
+	BUILD_BUG_ON(sizeof(attrs->attributes) != sizeof(unsigned long));
+
+	return kvm_vm_set_mem_attributes(kvm, start, end, attrs->attributes);
+}
+#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */
+
 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
 {
 	return __gfn_to_memslot(kvm_memslots(kvm), gfn);
@@ -4533,6 +4819,7 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
 {
 	switch (arg) {
 	case KVM_CAP_USER_MEMORY:
+	case KVM_CAP_USER_MEMORY2:
 	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
 	case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
 	case KVM_CAP_INTERNAL_ERROR_DATA:
@@ -4561,9 +4848,11 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
 	case KVM_CAP_IRQ_ROUTING:
 		return KVM_MAX_IRQ_ROUTES;
 #endif
-#if KVM_ADDRESS_SPACE_NUM > 1
+#if KVM_MAX_NR_ADDRESS_SPACES > 1
 	case KVM_CAP_MULTI_ADDRESS_SPACE:
-		return KVM_ADDRESS_SPACE_NUM;
+		if (kvm)
+			return kvm_arch_nr_memslot_as_ids(kvm);
+		return KVM_MAX_NR_ADDRESS_SPACES;
 #endif
 	case KVM_CAP_NR_MEMSLOTS:
 		return KVM_USER_MEM_SLOTS;
@@ -4585,6 +4874,14 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
 	case KVM_CAP_BINARY_STATS_FD:
 	case KVM_CAP_SYSTEM_EVENT_DATA:
 		return 1;
+#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
+	case KVM_CAP_MEMORY_ATTRIBUTES:
+		return kvm_supported_mem_attributes(kvm);
+#endif
+#ifdef CONFIG_KVM_PRIVATE_MEM
+	case KVM_CAP_GUEST_MEMFD:
+		return !kvm || kvm_arch_has_private_mem(kvm);
+#endif
 	default:
 		break;
 	}
@@ -4663,7 +4960,7 @@ bool kvm_are_all_memslots_empty(struct kvm *kvm)
 
 	lockdep_assert_held(&kvm->slots_lock);
 
-	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
+	for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
 		if (!kvm_memslots_empty(__kvm_memslots(kvm, i)))
 			return false;
 	}
@@ -4788,6 +5085,14 @@ static int kvm_vm_ioctl_get_stats_fd(struct kvm *kvm)
 	return fd;
 }
 
+#define SANITY_CHECK_MEM_REGION_FIELD(field)					\
+do {										\
+	BUILD_BUG_ON(offsetof(struct kvm_userspace_memory_region, field) !=		\
+		     offsetof(struct kvm_userspace_memory_region2, field));	\
+	BUILD_BUG_ON(sizeof_field(struct kvm_userspace_memory_region, field) !=		\
+		     sizeof_field(struct kvm_userspace_memory_region2, field));	\
+} while (0)
+
 static long kvm_vm_ioctl(struct file *filp,
 			   unsigned int ioctl, unsigned long arg)
 {
@@ -4810,15 +5115,39 @@ static long kvm_vm_ioctl(struct file *filp,
 		r = kvm_vm_ioctl_enable_cap_generic(kvm, &cap);
 		break;
 	}
+	case KVM_SET_USER_MEMORY_REGION2:
 	case KVM_SET_USER_MEMORY_REGION: {
-		struct kvm_userspace_memory_region kvm_userspace_mem;
+		struct kvm_userspace_memory_region2 mem;
+		unsigned long size;
+
+		if (ioctl == KVM_SET_USER_MEMORY_REGION) {
+			/*
+			 * Fields beyond struct kvm_userspace_memory_region shouldn't be
+			 * accessed, but avoid leaking kernel memory in case of a bug.
+			 */
+			memset(&mem, 0, sizeof(mem));
+			size = sizeof(struct kvm_userspace_memory_region);
+		} else {
+			size = sizeof(struct kvm_userspace_memory_region2);
+		}
+
+		/* Ensure the common parts of the two structs are identical. */
+		SANITY_CHECK_MEM_REGION_FIELD(slot);
+		SANITY_CHECK_MEM_REGION_FIELD(flags);
+		SANITY_CHECK_MEM_REGION_FIELD(guest_phys_addr);
+		SANITY_CHECK_MEM_REGION_FIELD(memory_size);
+		SANITY_CHECK_MEM_REGION_FIELD(userspace_addr);
 
 		r = -EFAULT;
-		if (copy_from_user(&kvm_userspace_mem, argp,
-						sizeof(kvm_userspace_mem)))
+		if (copy_from_user(&mem, argp, size))
 			goto out;
 
-		r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
+		r = -EINVAL;
+		if (ioctl == KVM_SET_USER_MEMORY_REGION &&
+		    (mem.flags & ~KVM_SET_USER_MEMORY_REGION_V1_FLAGS))
+			goto out;
+
+		r = kvm_vm_ioctl_set_memory_region(kvm, &mem);
 		break;
 	}
 	case KVM_GET_DIRTY_LOG: {
@@ -4946,6 +5275,18 @@ static long kvm_vm_ioctl(struct file *filp,
 		break;
 	}
 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
+#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
+	case KVM_SET_MEMORY_ATTRIBUTES: {
+		struct kvm_memory_attributes attrs;
+
+		r = -EFAULT;
+		if (copy_from_user(&attrs, argp, sizeof(attrs)))
+			goto out;
+
+		r = kvm_vm_ioctl_set_mem_attributes(kvm, &attrs);
+		break;
+	}
+#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */
 	case KVM_CREATE_DEVICE: {
 		struct kvm_create_device cd;
 
@@ -4973,6 +5314,18 @@ static long kvm_vm_ioctl(struct file *filp,
 	case KVM_GET_STATS_FD:
 		r = kvm_vm_ioctl_get_stats_fd(kvm);
 		break;
+#ifdef CONFIG_KVM_PRIVATE_MEM
+	case KVM_CREATE_GUEST_MEMFD: {
+		struct kvm_create_guest_memfd guest_memfd;
+
+		r = -EFAULT;
+		if (copy_from_user(&guest_memfd, argp, sizeof(guest_memfd)))
+			goto out;
+
+		r = kvm_gmem_create(kvm, &guest_memfd);
+		break;
+	}
+#endif
 	default:
 		r = kvm_arch_vm_ioctl(filp, ioctl, arg);
 	}
@@ -6105,6 +6458,8 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module)
 	if (WARN_ON_ONCE(r))
 		goto err_vfio;
 
+	kvm_gmem_init(module);
+
 	/*
 	 * Registration _must_ be the very last thing done, as this exposes
 	 * /dev/kvm to userspace, i.e. all infrastructure must be setup!
diff --git a/virt/kvm/kvm_mm.h b/virt/kvm/kvm_mm.h
index 180f1a09e6ba..ecefc7ec51af 100644
--- a/virt/kvm/kvm_mm.h
+++ b/virt/kvm/kvm_mm.h
@@ -37,4 +37,30 @@ static inline void gfn_to_pfn_cache_invalidate_start(struct kvm *kvm,
 }
 #endif /* HAVE_KVM_PFNCACHE */
 
+#ifdef CONFIG_KVM_PRIVATE_MEM
+void kvm_gmem_init(struct module *module);
+int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args);
+int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot,
+		  unsigned int fd, loff_t offset);
+void kvm_gmem_unbind(struct kvm_memory_slot *slot);
+#else
+static inline void kvm_gmem_init(struct module *module)
+{
+
+}
+
+static inline int kvm_gmem_bind(struct kvm *kvm,
+					 struct kvm_memory_slot *slot,
+					 unsigned int fd, loff_t offset)
+{
+	WARN_ON_ONCE(1);
+	return -EIO;
+}
+
+static inline void kvm_gmem_unbind(struct kvm_memory_slot *slot)
+{
+	WARN_ON_ONCE(1);
+}
+#endif /* CONFIG_KVM_PRIVATE_MEM */
+
 #endif /* __KVM_MM_H__ */
